// SPDX-License-Identifier: GPL-2.0#include <linux/kernel.h>#include <linux/errno.h>#include <linux/fs.h>#include <linux/file.h>#include <linux/mm.h>#include <linux/slab.h>#include <linux/nospec.h>#include <linux/hugetlb.h>#include <linux/compat.h>#include <linux/io_uring.h>#include <linux/io_uring/cmd.h> #include <uapi/linux/io_uring.h> #include "filetable.h"#include "io_uring.h"#include "openclose.h"#include "rsrc.h"#include "memmap.h"#include "register.h" struct io_rsrc_update { struct file *file; u64 arg; u32 nr_args; u32 offset;}; static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov); static int hpage_acct_ref(struct io_ring_ctx *ctx, struct page *hpage, bool *acct_new){ unsigned long key = (unsigned long) hpage; unsigned long count; void *entry; int ret; lockdep_assert_held(&ctx->uring_lock); entry = xa_load(&ctx->hpage_acct, key); if (entry) { *acct_new = false; count = xa_to_value(entry) + 1; } else { ret = xa_reserve(&ctx->hpage_acct, key, GFP_KERNEL_ACCOUNT); if (ret) return ret; *acct_new = true; count = 1; } xa_store(&ctx->hpage_acct, key, xa_mk_value(count), GFP_KERNEL_ACCOUNT); return 0;} static bool hpage_acct_unref(struct io_ring_ctx *ctx, struct page *hpage){ unsigned long key = (unsigned long) hpage; unsigned long count; void *entry; lockdep_assert_held(&ctx->uring_lock); entry = xa_load(&ctx->hpage_acct, key); if (WARN_ON_ONCE(!entry)) return false; count = xa_to_value(entry); if (count == 1) { xa_erase(&ctx->hpage_acct, key); return true; } xa_store(&ctx->hpage_acct, key, xa_mk_value(count - 1), GFP_KERNEL_ACCOUNT); return false;} /* only define max */#define IORING_MAX_FIXED_FILES (1U << 20)#define IORING_MAX_REG_BUFFERS (1U << 14) #define IO_CACHED_BVECS_SEGS 32 int __io_account_mem(struct user_struct *user, unsigned long nr_pages){ unsigned long page_limit, cur_pages, new_pages; if (!nr_pages) return 0; /* Don't allow more pages than we can safely lock */ page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; cur_pages = atomic_long_read(&user->locked_vm); do { new_pages = cur_pages + nr_pages; if (new_pages > page_limit) return -ENOMEM; } while (!atomic_long_try_cmpxchg(&user->locked_vm, &cur_pages, new_pages)); return 0;} void io_unaccount_mem(struct user_struct *user, struct mm_struct *mm_account, unsigned long nr_pages){ if (user) __io_unaccount_mem(user, nr_pages); if (mm_account) atomic64_sub(nr_pages, &mm_account->pinned_vm);} int io_account_mem(struct user_struct *user, struct mm_struct *mm_account, unsigned long nr_pages){ int ret; if (user) { ret = __io_account_mem(user, nr_pages); if (ret) return ret; } if (mm_account) atomic64_add(nr_pages, &mm_account->pinned_vm); return 0;} int io_validate_user_buf_range(u64 uaddr, u64 ulen){ unsigned long tmp, base = (unsigned long)uaddr; unsigned long acct_len = (unsigned long)PAGE_ALIGN(ulen); if (!ulen) return -EFAULT; /* 32-bit sanity checking */ if (ulen > ULONG_MAX || uaddr > ULONG_MAX) return -EFAULT; /* cap to 1TB for 64-bit */ if (ulen > SZ_1T) return -EINVAL; if (check_add_overflow(base, acct_len, &tmp)) return -EOVERFLOW; return 0;} static void io_release_ubuf(void *priv){ struct io_mapped_ubuf *imu = priv; unsigned int i; for (i = 0; i < imu->nr_bvecs; i++) { struct folio *folio = bvec_folio(&imu->bvec[i]); unpin_user_folio(folio, 1); }} static struct io_mapped_ubuf *io_alloc_imu(struct io_ring_ctx *ctx, int nr_bvecs){ if (nr_bvecs <= IO_CACHED_BVECS_SEGS) return io_cache_alloc(&ctx->imu_cache, GFP_KERNEL); return kvmalloc_flex(struct io_mapped_ubuf, bvec, nr_bvecs);} static void io_free_imu(struct io_ring_ctx *ctx, struct io_mapped_ubuf *imu){ if (imu->nr_bvecs <= IO_CACHED_BVECS_SEGS) io_cache_free(&ctx->imu_cache, imu); else kvfree(imu);} static unsigned long io_buffer_unaccount_pages(struct io_ring_ctx *ctx, struct io_mapped_ubuf *imu){ struct page *seen = NULL; unsigned long acct = 0; int i; if (imu->flags & IO_REGBUF_F_KBUF || !ctx->user) return 0; for (i = 0; i < imu->nr_bvecs; i++) { struct page *page = imu->bvec[i].bv_page; struct page *hpage; if (!PageCompound(page)) { acct++; continue; } hpage = compound_head(page); if (hpage == seen) continue; seen = hpage; /* Unaccount on last reference */ if (hpage_acct_unref(ctx, hpage)) acct += page_size(hpage) >> PAGE_SHIFT; cond_resched(); } return acct;} static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf *imu){ unsigned long acct_pages = 0; /* Always decrement, so it works for cloned buffers too */ acct_pages = io_buffer_unaccount_pages(ctx, imu); if (unlikely(refcount_read(&imu->refs) > 1)) { if (!refcount_dec_and_test(&imu->refs)) return; } if (acct_pages) io_unaccount_mem(ctx->user, ctx->mm_account, acct_pages); imu->release(imu->priv); io_free_imu(ctx, imu);} struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx, int type){ struct io_rsrc_node *node; node = io_cache_alloc(&ctx->node_cache, GFP_KERNEL); if (node) { node->type = type; node->refs = 1; node->tag = 0; node->file_ptr = 0; } return node;} bool io_rsrc_cache_init(struct io_ring_ctx *ctx){ const int imu_cache_size = struct_size_t(struct io_mapped_ubuf, bvec, IO_CACHED_BVECS_SEGS); const int node_size = sizeof(struct io_rsrc_node); bool ret; ret = io_alloc_cache_init(&ctx->node_cache, IO_ALLOC_CACHE_MAX, node_size, 0); ret |= io_alloc_cache_init(&ctx->imu_cache, IO_ALLOC_CACHE_MAX, imu_cache_size, 0); return ret;} void io_rsrc_cache_free(struct io_ring_ctx *ctx){ io_alloc_cache_free(&ctx->node_cache, kfree); io_alloc_cache_free(&ctx->imu_cache, kvfree);} static void io_clear_table_tags(struct io_rsrc_data *data){ int i; for (i = 0; i < data->nr; i++) { struct io_rsrc_node *node = data->nodes[i]; if (node) node->tag = 0; }} __cold void io_rsrc_data_free(struct io_ring_ctx *ctx, struct io_rsrc_data *data){ if (!data->nr) return; while (data->nr--) { if (data->nodes[data->nr]) io_put_rsrc_node(ctx, data->nodes[data->nr]); } kvfree(data->nodes); data->nodes = NULL; data->nr = 0;} __cold int io_rsrc_data_alloc(struct io_rsrc_data *data, unsigned nr){ data->nodes = kvmalloc_objs(struct io_rsrc_node *, nr, GFP_KERNEL_ACCOUNT | __GFP_ZERO); if (data->nodes) { data->nr = nr; return 0; } return -ENOMEM;} static int __io_sqe_files_update(struct io_ring_ctx *ctx, struct io_uring_rsrc_update2 *up, unsigned nr_args){ u64 __user *tags = u64_to_user_ptr(up->tags); __s32 __user *fds = u64_to_user_ptr(up->data); int fd, i, err = 0; unsigned int done; if (!ctx->file_table.data.nr) return -ENXIO; if (up->offset + nr_args > ctx->file_table.data.nr) return -EINVAL; for (done = 0; done < nr_args; done++) { u64 tag = 0; if ((tags && copy_from_user(&tag, &tags[done], sizeof(tag))) || copy_from_user(&fd, &fds[done], sizeof(fd))) { err = -EFAULT; break; } if ((fd == IORING_REGISTER_FILES_SKIP || fd == -1) && tag) { err = -EINVAL; break; } if (fd == IORING_REGISTER_FILES_SKIP) continue; i = up->offset + done; if (i >= ctx->file_table.data.nr) break; i = array_index_nospec(i, ctx->file_table.data.nr); if (io_reset_rsrc_node(ctx, &ctx->file_table.data, i)) io_file_bitmap_clear(&ctx->file_table, i); if (fd != -1) { struct file *file = fget(fd); struct io_rsrc_node *node; if (!file) { err = -EBADF; break; } /* * Don't allow io_uring instances to be registered. */ if (io_is_uring_fops(file)) { fput(file); err = -EBADF; break; } node = io_rsrc_node_alloc(ctx, IORING_RSRC_FILE); if (!node) { err = -ENOMEM; fput(file); break; } ctx->file_table.data.nodes[i] = node; if (tag) node->tag = tag; io_fixed_file_set(node, file); io_file_bitmap_set(&ctx->file_table, i); } } return done ? done : err;} static int __io_sqe_buffers_update(struct io_ring_ctx *ctx, struct io_uring_rsrc_update2 *up, unsigned int nr_args){ u64 __user *tags = u64_to_user_ptr(up->tags); struct iovec fast_iov, *iov; struct iovec __user *uvec; u64 user_data = up->data; __u32 done; int i, err; if (!ctx->buf_table.nr) return -ENXIO; if (up->offset + nr_args > ctx->buf_table.nr) return -EINVAL; for (done = 0; done < nr_args; done++) { struct io_rsrc_node *node; u64 tag = 0; uvec = u64_to_user_ptr(user_data); iov = iovec_from_user(uvec, 1, 1, &fast_iov, io_is_compat(ctx)); if (IS_ERR(iov)) { err = PTR_ERR(iov); break; } if (tags && copy_from_user(&tag, &tags[done], sizeof(tag))) { err = -EFAULT; break; } node = io_sqe_buffer_register(ctx, iov); if (IS_ERR(node)) { err = PTR_ERR(node); break; } if (tag) { if (!node) { err = -EINVAL; break; } node->tag = tag; } i = array_index_nospec(up->offset + done, ctx->buf_table.nr); io_reset_rsrc_node(ctx, &ctx->buf_table, i); ctx->buf_table.nodes[i] = node; if (io_is_compat(ctx)) user_data += sizeof(struct compat_iovec); else user_data += sizeof(struct iovec); } return done ? done : err;} static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type, struct io_uring_rsrc_update2 *up, unsigned nr_args){ __u32 tmp; lockdep_assert_held(&ctx->uring_lock); if (check_add_overflow(up->offset, nr_args, &tmp)) return -EOVERFLOW; switch (type) { case IORING_RSRC_FILE: return __io_sqe_files_update(ctx, up, nr_args); case IORING_RSRC_BUFFER: return __io_sqe_buffers_update(ctx, up, nr_args); } return -EINVAL;} int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args){ struct io_uring_rsrc_update2 up; if (!nr_args) return -EINVAL; memset(&up, 0, sizeof(up)); if (copy_from_user(&up, arg, sizeof(struct io_uring_rsrc_update))) return -EFAULT; if (up.resv || up.resv2) return -EINVAL; return __io_register_rsrc_update(ctx, IORING_RSRC_FILE, &up, nr_args);} int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg, unsigned size, unsigned type){ struct io_uring_rsrc_update2 up; if (size != sizeof(up)) return -EINVAL; if (copy_from_user(&up, arg, sizeof(up))) return -EFAULT; if (!up.nr || up.resv || up.resv2) return -EINVAL; return __io_register_rsrc_update(ctx, type, &up, up.nr);} __cold int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg, unsigned int size, unsigned int type){ struct io_uring_rsrc_register rr; /* keep it extendible */ if (size != sizeof(rr)) return -EINVAL; memset(&rr, 0, sizeof(rr)); if (copy_from_user(&rr, arg, size)) return -EFAULT; if (!rr.nr || rr.resv2) return -EINVAL; if (rr.flags & ~IORING_RSRC_REGISTER_SPARSE) return -EINVAL; switch (type) { case IORING_RSRC_FILE: if (rr.flags & IORING_RSRC_REGISTER_SPARSE && rr.data) break; return io_sqe_files_register(ctx, u64_to_user_ptr(rr.data), rr.nr, u64_to_user_ptr(rr.tags)); case IORING_RSRC_BUFFER: if (rr.flags & IORING_RSRC_REGISTER_SPARSE && rr.data) break; return io_sqe_buffers_register(ctx, u64_to_user_ptr(rr.data), rr.nr, u64_to_user_ptr(rr.tags)); } return -EINVAL;} Lines 1–500 of 1765. Load more
// SPDX-License-Identifier: GPL-2.0 #include <linux/kernel.h> #include <linux/errno.h> #include <linux/fs.h> #include <linux/file.h> #include <linux/mm.h> #include <linux/slab.h> #include <linux/nospec.h> #include <linux/hugetlb.h> #include <linux/compat.h> #include <linux/io_uring.h> #include <linux/io_uring/cmd.h> #include <uapi/linux/io_uring.h> #include "filetable.h" #include "io_uring.h" #include "openclose.h" #include "rsrc.h" #include "memmap.h" #include "register.h" struct io_rsrc_update { struct file *file; u64 arg; u32 nr_args; u32 offset; }; static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov); static int hpage_acct_ref(struct io_ring_ctx *ctx, struct page *hpage, bool *acct_new) { unsigned long key = (unsigned long) hpage; unsigned long count; void *entry; int ret; lockdep_assert_held(&ctx->uring_lock); entry = xa_load(&ctx->hpage_acct, key); if (entry) { *acct_new = false; count = xa_to_value(entry) + 1; } else { ret = xa_reserve(&ctx->hpage_acct, key, GFP_KERNEL_ACCOUNT); if (ret) return ret; *acct_new = true; count = 1; } xa_store(&ctx->hpage_acct, key, xa_mk_value(count), GFP_KERNEL_ACCOUNT); return 0; } static bool hpage_acct_unref(struct io_ring_ctx *ctx, struct page *hpage) { unsigned long key = (unsigned long) hpage; unsigned long count; void *entry; lockdep_assert_held(&ctx->uring_lock); entry = xa_load(&ctx->hpage_acct, key); if (WARN_ON_ONCE(!entry)) return false; count = xa_to_value(entry); if (count == 1) { xa_erase(&ctx->hpage_acct, key); return true; } xa_store(&ctx->hpage_acct, key, xa_mk_value(count - 1), GFP_KERNEL_ACCOUNT); return false; } /* only define max */ #define IORING_MAX_FIXED_FILES (1U << 20) #define IORING_MAX_REG_BUFFERS (1U << 14) #define IO_CACHED_BVECS_SEGS 32 int __io_account_mem(struct user_struct *user, unsigned long nr_pages) { unsigned long page_limit, cur_pages, new_pages; if (!nr_pages) return 0; /* Don't allow more pages than we can safely lock */ page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; cur_pages = atomic_long_read(&user->locked_vm); do { new_pages = cur_pages + nr_pages; if (new_pages > page_limit) return -ENOMEM; } while (!atomic_long_try_cmpxchg(&user->locked_vm, &cur_pages, new_pages)); return 0; } void io_unaccount_mem(struct user_struct *user, struct mm_struct *mm_account, unsigned long nr_pages) { if (user) __io_unaccount_mem(user, nr_pages); if (mm_account) atomic64_sub(nr_pages, &mm_account->pinned_vm); } int io_account_mem(struct user_struct *user, struct mm_struct *mm_account, unsigned long nr_pages) { int ret; if (user) { ret = __io_account_mem(user, nr_pages); if (ret) return ret; } if (mm_account) atomic64_add(nr_pages, &mm_account->pinned_vm); return 0; } int io_validate_user_buf_range(u64 uaddr, u64 ulen) { unsigned long tmp, base = (unsigned long)uaddr; unsigned long acct_len = (unsigned long)PAGE_ALIGN(ulen); if (!ulen) return -EFAULT; /* 32-bit sanity checking */ if (ulen > ULONG_MAX || uaddr > ULONG_MAX) return -EFAULT; /* cap to 1TB for 64-bit */ if (ulen > SZ_1T) return -EINVAL; if (check_add_overflow(base, acct_len, &tmp)) return -EOVERFLOW; return 0; } static void io_release_ubuf(void *priv) { struct io_mapped_ubuf *imu = priv; unsigned int i; for (i = 0; i < imu->nr_bvecs; i++) { struct folio *folio = bvec_folio(&imu->bvec[i]); unpin_user_folio(folio, 1); } } static struct io_mapped_ubuf *io_alloc_imu(struct io_ring_ctx *ctx, int nr_bvecs) { if (nr_bvecs <= IO_CACHED_BVECS_SEGS) return io_cache_alloc(&ctx->imu_cache, GFP_KERNEL); return kvmalloc_flex(struct io_mapped_ubuf, bvec, nr_bvecs); } static void io_free_imu(struct io_ring_ctx *ctx, struct io_mapped_ubuf *imu) { if (imu->nr_bvecs <= IO_CACHED_BVECS_SEGS) io_cache_free(&ctx->imu_cache, imu); else kvfree(imu); } static unsigned long io_buffer_unaccount_pages(struct io_ring_ctx *ctx, struct io_mapped_ubuf *imu) { struct page *seen = NULL; unsigned long acct = 0; int i; if (imu->flags & IO_REGBUF_F_KBUF || !ctx->user) return 0; for (i = 0; i < imu->nr_bvecs; i++) { struct page *page = imu->bvec[i].bv_page; struct page *hpage; if (!PageCompound(page)) { acct++; continue; } hpage = compound_head(page); if (hpage == seen) continue; seen = hpage; /* Unaccount on last reference */ if (hpage_acct_unref(ctx, hpage)) acct += page_size(hpage) >> PAGE_SHIFT; cond_resched(); } return acct; } static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf *imu) { unsigned long acct_pages = 0; /* Always decrement, so it works for cloned buffers too */ acct_pages = io_buffer_unaccount_pages(ctx, imu); if (unlikely(refcount_read(&imu->refs) > 1)) { if (!refcount_dec_and_test(&imu->refs)) return; } if (acct_pages) io_unaccount_mem(ctx->user, ctx->mm_account, acct_pages); imu->release(imu->priv); io_free_imu(ctx, imu); } struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx, int type) { struct io_rsrc_node *node; node = io_cache_alloc(&ctx->node_cache, GFP_KERNEL); if (node) { node->type = type; node->refs = 1; node->tag = 0; node->file_ptr = 0; } return node; } bool io_rsrc_cache_init(struct io_ring_ctx *ctx) { const int imu_cache_size = struct_size_t(struct io_mapped_ubuf, bvec, IO_CACHED_BVECS_SEGS); const int node_size = sizeof(struct io_rsrc_node); bool ret; ret = io_alloc_cache_init(&ctx->node_cache, IO_ALLOC_CACHE_MAX, node_size, 0); ret |= io_alloc_cache_init(&ctx->imu_cache, IO_ALLOC_CACHE_MAX, imu_cache_size, 0); return ret; } void io_rsrc_cache_free(struct io_ring_ctx *ctx) { io_alloc_cache_free(&ctx->node_cache, kfree); io_alloc_cache_free(&ctx->imu_cache, kvfree); } static void io_clear_table_tags(struct io_rsrc_data *data) { int i; for (i = 0; i < data->nr; i++) { struct io_rsrc_node *node = data->nodes[i]; if (node) node->tag = 0; } } __cold void io_rsrc_data_free(struct io_ring_ctx *ctx, struct io_rsrc_data *data) { if (!data->nr) return; while (data->nr--) { if (data->nodes[data->nr]) io_put_rsrc_node(ctx, data->nodes[data->nr]); } kvfree(data->nodes); data->nodes = NULL; data->nr = 0; } __cold int io_rsrc_data_alloc(struct io_rsrc_data *data, unsigned nr) { data->nodes = kvmalloc_objs(struct io_rsrc_node *, nr, GFP_KERNEL_ACCOUNT | __GFP_ZERO); if (data->nodes) { data->nr = nr; return 0; } return -ENOMEM; } static int __io_sqe_files_update(struct io_ring_ctx *ctx, struct io_uring_rsrc_update2 *up, unsigned nr_args) { u64 __user *tags = u64_to_user_ptr(up->tags); __s32 __user *fds = u64_to_user_ptr(up->data); int fd, i, err = 0; unsigned int done; if (!ctx->file_table.data.nr) return -ENXIO; if (up->offset + nr_args > ctx->file_table.data.nr) return -EINVAL; for (done = 0; done < nr_args; done++) { u64 tag = 0; if ((tags && copy_from_user(&tag, &tags[done], sizeof(tag))) || copy_from_user(&fd, &fds[done], sizeof(fd))) { err = -EFAULT; break; } if ((fd == IORING_REGISTER_FILES_SKIP || fd == -1) && tag) { err = -EINVAL; break; } if (fd == IORING_REGISTER_FILES_SKIP) continue; i = up->offset + done; if (i >= ctx->file_table.data.nr) break; i = array_index_nospec(i, ctx->file_table.data.nr); if (io_reset_rsrc_node(ctx, &ctx->file_table.data, i)) io_file_bitmap_clear(&ctx->file_table, i); if (fd != -1) { struct file *file = fget(fd); struct io_rsrc_node *node; if (!file) { err = -EBADF; break; } /* * Don't allow io_uring instances to be registered. */ if (io_is_uring_fops(file)) { fput(file); err = -EBADF; break; } node = io_rsrc_node_alloc(ctx, IORING_RSRC_FILE); if (!node) { err = -ENOMEM; fput(file); break; } ctx->file_table.data.nodes[i] = node; if (tag) node->tag = tag; io_fixed_file_set(node, file); io_file_bitmap_set(&ctx->file_table, i); } } return done ? done : err; } static int __io_sqe_buffers_update(struct io_ring_ctx *ctx, struct io_uring_rsrc_update2 *up, unsigned int nr_args) { u64 __user *tags = u64_to_user_ptr(up->tags); struct iovec fast_iov, *iov; struct iovec __user *uvec; u64 user_data = up->data; __u32 done; int i, err; if (!ctx->buf_table.nr) return -ENXIO; if (up->offset + nr_args > ctx->buf_table.nr) return -EINVAL; for (done = 0; done < nr_args; done++) { struct io_rsrc_node *node; u64 tag = 0; uvec = u64_to_user_ptr(user_data); iov = iovec_from_user(uvec, 1, 1, &fast_iov, io_is_compat(ctx)); if (IS_ERR(iov)) { err = PTR_ERR(iov); break; } if (tags && copy_from_user(&tag, &tags[done], sizeof(tag))) { err = -EFAULT; break; } node = io_sqe_buffer_register(ctx, iov); if (IS_ERR(node)) { err = PTR_ERR(node); break; } if (tag) { if (!node) { err = -EINVAL; break; } node->tag = tag; } i = array_index_nospec(up->offset + done, ctx->buf_table.nr); io_reset_rsrc_node(ctx, &ctx->buf_table, i); ctx->buf_table.nodes[i] = node; if (io_is_compat(ctx)) user_data += sizeof(struct compat_iovec); else user_data += sizeof(struct iovec); } return done ? done : err; } static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type, struct io_uring_rsrc_update2 *up, unsigned nr_args) { __u32 tmp; lockdep_assert_held(&ctx->uring_lock); if (check_add_overflow(up->offset, nr_args, &tmp)) return -EOVERFLOW; switch (type) { case IORING_RSRC_FILE: return __io_sqe_files_update(ctx, up, nr_args); case IORING_RSRC_BUFFER: return __io_sqe_buffers_update(ctx, up, nr_args); } return -EINVAL; } int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args) { struct io_uring_rsrc_update2 up; if (!nr_args) return -EINVAL; memset(&up, 0, sizeof(up)); if (copy_from_user(&up, arg, sizeof(struct io_uring_rsrc_update))) return -EFAULT; if (up.resv || up.resv2) return -EINVAL; return __io_register_rsrc_update(ctx, IORING_RSRC_FILE, &up, nr_args); } int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg, unsigned size, unsigned type) { struct io_uring_rsrc_update2 up; if (size != sizeof(up)) return -EINVAL; if (copy_from_user(&up, arg, sizeof(up))) return -EFAULT; if (!up.nr || up.resv || up.resv2) return -EINVAL; return __io_register_rsrc_update(ctx, type, &up, up.nr); } __cold int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg, unsigned int size, unsigned int type) { struct io_uring_rsrc_register rr; /* keep it extendible */ if (size != sizeof(rr)) return -EINVAL; memset(&rr, 0, sizeof(rr)); if (copy_from_user(&rr, arg, size)) return -EFAULT; if (!rr.nr || rr.resv2) return -EINVAL; if (rr.flags & ~IORING_RSRC_REGISTER_SPARSE) return -EINVAL; switch (type) { case IORING_RSRC_FILE: if (rr.flags & IORING_RSRC_REGISTER_SPARSE && rr.data) break; return io_sqe_files_register(ctx, u64_to_user_ptr(rr.data), rr.nr, u64_to_user_ptr(rr.tags)); case IORING_RSRC_BUFFER: if (rr.flags & IORING_RSRC_REGISTER_SPARSE && rr.data) break; return io_sqe_buffers_register(ctx, u64_to_user_ptr(rr.data), rr.nr, u64_to_user_ptr(rr.tags)); } return -EINVAL; }