History

RDMA/core: Introduce generic buffer descriptor infrastructure for umem

Introduce a per-attribute UVERBS_ATTR_UMEM model so each uverbs command's umem set is explicit in its UAPI definition. Add driver-facing wrapper helpers that pin a umem on demand from an attribute or a VA addr; the driver owns the returned umem and releases it from its destroy/error paths.

Jiri Pirko committed 5 months ago

Commit 3cfdff41 parent 28fb7015 files · +346 −0Browse files

Unified · Split

5 files
  1. drivers/infiniband/core/ib_core_uverbs.c +25 −0
  2. drivers/infiniband/core/umem.c +228 −0
  3. include/rdma/ib_umem.h +36 −0
  4. include/rdma/uverbs_ioctl.h +30 −0
  5. include/uapi/rdma/ib_user_ioctl_verbs.h +27 −0
drivers/infiniband/core/ib_core_uverbs.c+25−0
@@ -708,6 +708,31 @@ int uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle, · 707 unmodified lines
708}708}
709EXPORT_SYMBOL(uverbs_get_flags32);709EXPORT_SYMBOL(uverbs_get_flags32);
710 710
711/**
712 * uverbs_get_buffer_desc - Read a buffer descriptor from a uverbs attr.
713 * @attrs_bundle: uverbs attribute bundle.
714 * @attr_id: id of an UVERBS_ATTR_UMEM-typed attribute.
715 * @desc: descriptor to fill.
716 *
717 * Return: 0 on success, -ENOENT if @attr_id is not set, -EINVAL on a
718 * malformed descriptor, or any other negative errno propagated from
719 * uverbs_copy_from() (notably -EFAULT on copy_from_user() failure).
720 */
721int uverbs_get_buffer_desc(const struct uverbs_attr_bundle *attrs_bundle,
722 u16 attr_id, struct ib_uverbs_buffer_desc *desc)
723{
724 int ret;
725
726 ret = uverbs_copy_from(desc, attrs_bundle, attr_id);
727 if (ret)
728 return ret;
729 if (desc->flags & ~IB_UVERBS_BUFFER_DESC_FLAGS_KNOWN_MASK)
730 return -EINVAL;
731 desc->optional_flags &= IB_UVERBS_BUFFER_DESC_OPTIONAL_FLAGS_KNOWN_MASK;
732 return 0;
733}
734EXPORT_SYMBOL(uverbs_get_buffer_desc);
735
711/* Once called an abort will call through to the type's destroy_hw() */736/* Once called an abort will call through to the type's destroy_hw() */
712void uverbs_finalize_uobj_create(const struct uverbs_attr_bundle *bundle,737void uverbs_finalize_uobj_create(const struct uverbs_attr_bundle *bundle,
713 u16 idx)738 u16 idx)
drivers/infiniband/core/umem.c+228−0
@@ -269,6 +269,175 @@ umem_kfree: · 268 unmodified lines
269 return ret ? ERR_PTR(ret) : umem;269 return ret ? ERR_PTR(ret) : umem;
270}270}
271 271
272/**
273 * ib_umem_get_desc - Pin a umem from a buffer descriptor.
274 * @device: IB device.
275 * @desc: buffer descriptor (VA or DMABUF).
276 * @access: IB access flags.
277 *
278 * Return: caller-owned umem on success, ERR_PTR(...) on error.
279 */
280struct ib_umem *ib_umem_get_desc(struct ib_device *device,
281 const struct ib_uverbs_buffer_desc *desc,
282 int access)
283{
284 struct ib_umem_dmabuf *umem_dmabuf;
285
286 if (desc->flags & ~IB_UVERBS_BUFFER_DESC_FLAGS_KNOWN_MASK)
287 return ERR_PTR(-EINVAL);
288
289 if (overflows_type(desc->addr, unsigned long) ||
290 overflows_type(desc->length, size_t))
291 return ERR_PTR(-EOVERFLOW);
292
293 switch (desc->type) {
294 case IB_UVERBS_BUFFER_TYPE_DMABUF:
295 umem_dmabuf = ib_umem_dmabuf_get_pinned(device, desc->addr,
296 desc->length, desc->fd,
297 access);
298 if (IS_ERR(umem_dmabuf))
299 return ERR_CAST(umem_dmabuf);
300 return &umem_dmabuf->umem;
301 case IB_UVERBS_BUFFER_TYPE_VA:
302 return __ib_umem_get_va(device, desc->addr, desc->length,
303 access);
304 default:
305 return ERR_PTR(-EINVAL);
306 }
307}
308EXPORT_SYMBOL(ib_umem_get_desc);
309
310/*
311 * Per-command legacy buffer-desc filler.
312 * Returns 0 on success (desc filled), -ENODATA if no legacy attrs apply,
313 * negative errno on validation failure.
314 */
315typedef int (*ib_umem_buf_desc_filler_t)(const struct uverbs_attr_bundle *attrs,
316 struct ib_uverbs_buffer_desc *desc);
317
318/*
319 * ib_umem_resolve_desc - Resolve a buffer descriptor from a per-command UMEM
320 * attribute and/or a legacy attr filler.
321 *
322 * Return:
323 * 0 @desc filled.
324 * -ENOENT no source produced a buffer.
325 * -EINVAL both the UMEM attribute and the legacy filler produced a buffer.
326 * -errno propagated from attr read / filler validation.
327 */
328static int ib_umem_resolve_desc(const struct uverbs_attr_bundle *attrs,
329 u16 attr_id,
330 ib_umem_buf_desc_filler_t legacy_filler,
331 struct ib_uverbs_buffer_desc *desc)
332{
333 bool have_desc = false;
334 int ret;
335
336 if (!attrs)
337 return -ENOENT;
338
339 ret = uverbs_get_buffer_desc(attrs, attr_id, desc);
340 if (!ret)
341 have_desc = true;
342 else if (ret != -ENOENT)
343 return ret;
344
345 if (legacy_filler) {
346 struct ib_uverbs_buffer_desc legacy_desc = {};
347
348 ret = legacy_filler(attrs, &legacy_desc);
349 if (!ret) {
350 if (have_desc)
351 return -EINVAL;
352 *desc = legacy_desc;
353 have_desc = true;
354 } else if (ret != -ENODATA) {
355 return ret;
356 }
357 }
358
359 return have_desc ? 0 : -ENOENT;
360}
361
362/*
363 * ib_umem_get_desc_check - Pin a umem from @desc and verify it meets
364 * @min_size.
365 */
366static struct ib_umem *
367ib_umem_get_desc_check(struct ib_device *device,
368 const struct ib_uverbs_buffer_desc *desc,
369 size_t min_size, int access)
370{
371 struct ib_umem *umem;
372
373 umem = ib_umem_get_desc(device, desc, access);
374 if (IS_ERR(umem))
375 return umem;
376 if (umem->length < min_size) {
377 ib_umem_release(umem);
378 return ERR_PTR(-EINVAL);
379 }
380 return umem;
381}
382
383/*
384 * ib_umem_get_from_attrs - Pin a umem from a per-command UMEM attribute
385 * and/or a legacy attr filler.
386 *
387 * Return: caller-owned umem on success; NULL when no source supplied a
388 * buffer; ERR_PTR(...) on error.
389 */
390static struct ib_umem *
391ib_umem_get_from_attrs(struct ib_device *device,
392 const struct uverbs_attr_bundle *attrs,
393 u16 attr_id, ib_umem_buf_desc_filler_t legacy_filler,
394 size_t size, int access)
395{
396 struct ib_uverbs_buffer_desc desc = {};
397 int ret;
398
399 ret = ib_umem_resolve_desc(attrs, attr_id, legacy_filler, &desc);
400 if (ret == -ENOENT)
401 return NULL;
402 if (ret)
403 return ERR_PTR(ret);
404 return ib_umem_get_desc_check(device, &desc, size, access);
405}
406
407/*
408 * ib_umem_get_from_attrs_or_va - Pin a umem from a per-command UMEM
409 * attribute and/or a legacy attr filler,
410 * falling back to a UHW VA when no source
411 * matched.
412 *
413 * @size is always consumed: it is the length to pin on the VA fallback
414 * path AND the post-pin minimum-length check on the attr / legacy paths.
415 * Callers must always pass a meaningful, validated value.
416 *
417 * Return: caller-owned umem on success, ERR_PTR(...) on error.
418 */
419static struct ib_umem *
420ib_umem_get_from_attrs_or_va(struct ib_device *device,
421 const struct uverbs_attr_bundle *attrs,
422 u16 attr_id,
423 ib_umem_buf_desc_filler_t legacy_filler,
424 u64 addr, size_t size, int access)
425{
426 struct ib_uverbs_buffer_desc desc = {};
427 int ret;
428
429 ret = ib_umem_resolve_desc(attrs, attr_id, legacy_filler, &desc);
430 if (ret == -ENOENT)
431 desc = (struct ib_uverbs_buffer_desc){
432 .type = IB_UVERBS_BUFFER_TYPE_VA,
433 .addr = addr,
434 .length = size,
435 };
436 else if (ret)
437 return ERR_PTR(ret);
438 return ib_umem_get_desc_check(device, &desc, size, access);
439}
440
272/**441/**
273 * ib_umem_get_va - Pin and DMA map userspace memory.442 * ib_umem_get_va - Pin and DMA map userspace memory.
274 *443 *
@@ -284,6 +453,65 @@ struct ib_umem *ib_umem_get_va(struct ib_device *device, unsigned long addr, · 9 unmodified lines
284}453}
285EXPORT_SYMBOL(ib_umem_get_va);454EXPORT_SYMBOL(ib_umem_get_va);
286 455
456/**
457 * ib_umem_get_attr - Pin a umem from a per-command UMEM attribute.
458 * @device: IB device.
459 * @attrs: uverbs attribute bundle (may be NULL).
460 * @attr_id: per-command UMEM attribute id.
461 * @size: minimum required umem length.
462 * @access: IB access flags.
463 *
464 * Return: caller-owned umem on success; NULL when no source supplied
465 * a buffer; ERR_PTR(...) on error.
466 */
467struct ib_umem *ib_umem_get_attr(struct ib_device *device,
468 const struct uverbs_attr_bundle *attrs,
469 u16 attr_id, size_t size, int access)
470{
471 return ib_umem_get_from_attrs(device, attrs, attr_id, NULL, size,
472 access);
473}
474EXPORT_SYMBOL(ib_umem_get_attr);
475
476/**
477 * ib_umem_get_attr_or_va - Pin a umem from a per-command UMEM attribute,
478 * falling back to a UHW VA.
479 * @device: IB device.
480 * @attrs: uverbs attribute bundle (may be NULL).
481 * @attr_id: per-command UMEM attribute id.
482 * @addr: UHW user VA used when no per-command attribute matched.
483 * @size: on the attr / legacy paths, the minimum required umem length
484 * validated post-pin; on the VA fallback path, the length to pin.
485 * @access: IB access flags.
486 *
487 * Like ib_umem_get_attr(), but pins @addr/@size when no per-command
488 * UMEM attribute is supplied.
489 *
490 * IMPORTANT: @size is always consumed. On the attr / legacy paths it is
491 * used as the post-pin minimum-length check; on the VA fallback path it
492 * is the length to pin. Callers MUST pass a meaningful, validated value
493 * even when they expect an attribute-supplied buffer to be used.
494 *
495 * Every in-tree caller passes the same value for the two roles of @size
496 * because no driver today distinguishes a user-passed buffer length from
497 * a driver-computed minimum. Drivers that currently accept a user-supplied
498 * length without cross-checking it against a driver minimum (vmw_pvrdma
499 * CQ/QP/SRQ, qedr CQ/QP/SRQ, mana WQ/QP, ionic CQ/QP), once tightened to
500 * compute and check a real minimum, will want to introduce a separate
501 * helper that passes these as distinct values.
502 *
503 * Return: caller-owned umem on success, ERR_PTR(...) on error.
504 */
505struct ib_umem *ib_umem_get_attr_or_va(struct ib_device *device,
506 const struct uverbs_attr_bundle *attrs,
507 u16 attr_id, u64 addr, size_t size,
508 int access)
509{
510 return ib_umem_get_from_attrs_or_va(device, attrs, attr_id, NULL, addr,
511 size, access);
512}
513EXPORT_SYMBOL(ib_umem_get_attr_or_va);
514
287/**515/**
288 * ib_umem_release - release pinned memory516 * ib_umem_release - release pinned memory
289 * @umem: umem struct to release517 * @umem: umem struct to release
include/rdma/ib_umem.h+36−0
@@ -73,10 +73,26 @@ static inline size_t ib_umem_num_pages(struct ib_umem *umem) · 72 unmodified lines
73{73{
74 return ib_umem_num_dma_blocks(umem, PAGE_SIZE);74 return ib_umem_num_dma_blocks(umem, PAGE_SIZE);
75}75}
76
77struct ib_udata;
78struct ib_uverbs_buffer_desc;
79struct uverbs_attr_bundle;
80
76#ifdef CONFIG_INFINIBAND_USER_MEM81#ifdef CONFIG_INFINIBAND_USER_MEM
77 82
83struct ib_umem *ib_umem_get_desc(struct ib_device *device,
84 const struct ib_uverbs_buffer_desc *desc,
85 int access);
78struct ib_umem *ib_umem_get_va(struct ib_device *device, unsigned long addr,86struct ib_umem *ib_umem_get_va(struct ib_device *device, unsigned long addr,
79 size_t size, int access);87 size_t size, int access);
88struct ib_umem *ib_umem_get_attr(struct ib_device *device,
89 const struct uverbs_attr_bundle *attrs,
90 u16 attr_id, size_t size, int access);
91struct ib_umem *ib_umem_get_attr_or_va(struct ib_device *device,
92 const struct uverbs_attr_bundle *attrs,
93 u16 attr_id, u64 addr, size_t size,
94 int access);
95
80void ib_umem_release(struct ib_umem *umem);96void ib_umem_release(struct ib_umem *umem);
81int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset,97int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset,
82 size_t length);98 size_t length);
@@ -160,12 +176,32 @@ void ib_umem_dmabuf_revoke(struct ib_umem_dmabuf *umem_dmabuf); · 77 unmodified lines
160 176
161#include <linux/err.h>177#include <linux/err.h>
162 178
179static inline struct ib_umem *
180ib_umem_get_desc(struct ib_device *device,
181 const struct ib_uverbs_buffer_desc *desc, int access)
182{
183 return ERR_PTR(-EOPNOTSUPP);
184}
163static inline struct ib_umem *ib_umem_get_va(struct ib_device *device,185static inline struct ib_umem *ib_umem_get_va(struct ib_device *device,
164 unsigned long addr, size_t size,186 unsigned long addr, size_t size,
165 int access)187 int access)
166{188{
167 return ERR_PTR(-EOPNOTSUPP);189 return ERR_PTR(-EOPNOTSUPP);
168}190}
191static inline struct ib_umem *
192ib_umem_get_attr(struct ib_device *device,
193 const struct uverbs_attr_bundle *attrs, u16 attr_id,
194 size_t size, int access)
195{
196 return ERR_PTR(-EOPNOTSUPP);
197}
198static inline struct ib_umem *
199ib_umem_get_attr_or_va(struct ib_device *device,
200 const struct uverbs_attr_bundle *attrs, u16 attr_id,
201 u64 addr, size_t size, int access)
202{
203 return ERR_PTR(-EOPNOTSUPP);
204}
169static inline void ib_umem_release(struct ib_umem *umem) { }205static inline void ib_umem_release(struct ib_umem *umem) { }
170static inline int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset,206static inline int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset,
171 size_t length) {207 size_t length) {
include/rdma/uverbs_ioctl.h+30−0
@@ -590,6 +590,28 @@ struct uapi_definition { · 589 unmodified lines
590 UA_OPTIONAL, \590 UA_OPTIONAL, \
591 .is_udata = 1)591 .is_udata = 1)
592 592
593/*
594 * Per-attribute UMEM descriptor. The payload is a single
595 * struct ib_uverbs_buffer_desc identifying a memory region backed by
596 * dma-buf or user virtual address. _access selects UA_OPTIONAL or
597 * UA_MANDATORY. Drivers obtain a umem from the attribute via the
598 * ib_umem_get_*() wrapper helpers.
599 */
600#define UVERBS_ATTR_UMEM(_attr_id, _access) \
601 UVERBS_ATTR_PTR_IN(_attr_id, \
602 UVERBS_ATTR_TYPE(struct ib_uverbs_buffer_desc), \
603 _access)
604
605/*
606 * Bit masks of the @flags / @optional_flags fields of struct
607 * ib_uverbs_buffer_desc that the kernel understands. @flags is strict:
608 * any bit outside the known mask makes the call fail with -EINVAL.
609 * @optional_flags is advisory: bits outside the known mask are silently
610 * dropped. Both masks are extended as new bits are introduced.
611 */
612#define IB_UVERBS_BUFFER_DESC_FLAGS_KNOWN_MASK 0U
613#define IB_UVERBS_BUFFER_DESC_OPTIONAL_FLAGS_KNOWN_MASK 0U
614
593/* =================================================615/* =================================================
594 * Parsing infrastructure616 * Parsing infrastructure
595 * =================================================617 * =================================================
@@ -862,6 +884,8 @@ int uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle, · 266 unmodified lines
862 size_t idx, u64 allowed_bits);884 size_t idx, u64 allowed_bits);
863int uverbs_copy_to(const struct uverbs_attr_bundle *attrs_bundle, size_t idx,885int uverbs_copy_to(const struct uverbs_attr_bundle *attrs_bundle, size_t idx,
864 const void *from, size_t size);886 const void *from, size_t size);
887int uverbs_get_buffer_desc(const struct uverbs_attr_bundle *attrs_bundle,
888 u16 attr_id, struct ib_uverbs_buffer_desc *desc);
865__malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size,889__malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size,
866 gfp_t flags);890 gfp_t flags);
867 891
@@ -920,6 +944,12 @@ static inline int uverbs_copy_to(const struct uverbs_attr_bundle *attrs_bundle, · 52 unmodified lines
920{944{
921 return -EINVAL;945 return -EINVAL;
922}946}
947static inline int
948uverbs_get_buffer_desc(const struct uverbs_attr_bundle *attrs_bundle,
949 u16 attr_id, struct ib_uverbs_buffer_desc *desc)
950{
951 return -EINVAL;
952}
923static inline __malloc void *uverbs_alloc(struct uverbs_attr_bundle *bundle,953static inline __malloc void *uverbs_alloc(struct uverbs_attr_bundle *bundle,
924 size_t size)954 size_t size)
925{955{
include/uapi/rdma/ib_user_ioctl_verbs.h+27−0
@@ -273,4 +273,31 @@ struct ib_uverbs_gid_entry { · 272 unmodified lines
273 __u32 netdev_ifindex; /* It is 0 if there is no netdev associated with it */273 __u32 netdev_ifindex; /* It is 0 if there is no netdev associated with it */
274};274};
275 275
276enum ib_uverbs_buffer_type {
277 IB_UVERBS_BUFFER_TYPE_DMABUF,
278 IB_UVERBS_BUFFER_TYPE_VA,
279};
280
281/*
282 * Describes a single buffer backed by dma-buf or user virtual address.
283 * Used as the payload of a per-attribute UVERBS_ATTR_UMEM-typed attribute.
284 *
285 * @type: buffer type from enum ib_uverbs_buffer_type
286 * @fd: dma-buf file descriptor (valid for IB_UVERBS_BUFFER_TYPE_DMABUF)
287 * @flags: required flags; the kernel rejects the call with -EINVAL if any
288 * bit is not understood. No bits are defined yet.
289 * @optional_flags: advisory flags; bits the kernel does not understand are
290 * silently ignored. No bits are defined yet.
291 * @addr: offset within dma-buf, or user virtual address for VA
292 * @length: buffer length in bytes
293 */
294struct ib_uverbs_buffer_desc {
295 __u32 type;
296 __s32 fd;
297 __u32 flags;
298 __u32 optional_flags;
299 __aligned_u64 addr;
300 __aligned_u64 length;
301};
302
276#endif303#endif