// SPDX-License-Identifier: GPL-2.0/* * Code related to the io_uring_register() syscall * * Copyright (C) 2023 Jens Axboe */#include <linux/kernel.h>#include <linux/errno.h>#include <linux/syscalls.h>#include <linux/refcount.h>#include <linux/bits.h>#include <linux/fs.h>#include <linux/file.h>#include <linux/slab.h>#include <linux/uaccess.h>#include <linux/nospec.h>#include <linux/compat.h>#include <linux/io_uring.h>#include <linux/io_uring_types.h> #include "filetable.h"#include "io_uring.h"#include "opdef.h"#include "tctx.h"#include "rsrc.h"#include "sqpoll.h"#include "register.h"#include "cancel.h"#include "kbuf.h"#include "napi.h"#include "eventfd.h"#include "msg_ring.h"#include "memmap.h"#include "zcrx.h"#include "query.h"#include "bpf_filter.h" #define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \ IORING_REGISTER_LAST + IORING_OP_LAST) static __cold int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args){ struct io_uring_probe *p; size_t size; int i, ret; if (nr_args > IORING_OP_LAST) nr_args = IORING_OP_LAST; size = struct_size(p, ops, nr_args); p = memdup_user(arg, size); if (IS_ERR(p)) return PTR_ERR(p); ret = -EINVAL; if (memchr_inv(p, 0, size)) goto out; p->last_op = IORING_OP_LAST - 1; for (i = 0; i < nr_args; i++) { p->ops[i].op = i; if (io_uring_op_supported(i)) p->ops[i].flags = IO_URING_OP_SUPPORTED; } p->ops_len = i; ret = 0; if (copy_to_user(arg, p, size)) ret = -EFAULT;out: kfree(p); return ret;} int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id){ const struct cred *creds; creds = xa_erase(&ctx->personalities, id); if (creds) { put_cred(creds); return 0; } return -EINVAL;} static int io_register_personality(struct io_ring_ctx *ctx){ const struct cred *creds; u32 id; int ret; creds = get_current_cred(); ret = xa_alloc_cyclic(&ctx->personalities, &id, (void *)creds, XA_LIMIT(0, USHRT_MAX), &ctx->pers_next, GFP_KERNEL); if (ret < 0) { put_cred(creds); return ret; } return id;} /* * Returns number of restrictions parsed and added on success, or < 0 for * an error. */static __cold int io_parse_restrictions(void __user *arg, unsigned int nr_args, struct io_restriction *restrictions){ struct io_uring_restriction *res; size_t size; int i, ret; if (!arg || nr_args > IORING_MAX_RESTRICTIONS) return -EINVAL; size = array_size(nr_args, sizeof(*res)); if (size == SIZE_MAX) return -EOVERFLOW; res = memdup_user(arg, size); if (IS_ERR(res)) return PTR_ERR(res); ret = -EINVAL; for (i = 0; i < nr_args; i++) { switch (res[i].opcode) { case IORING_RESTRICTION_REGISTER_OP: if (res[i].register_op >= IORING_REGISTER_LAST) goto err; __set_bit(res[i].register_op, restrictions->register_op); restrictions->reg_registered = true; break; case IORING_RESTRICTION_SQE_OP: if (res[i].sqe_op >= IORING_OP_LAST) goto err; __set_bit(res[i].sqe_op, restrictions->sqe_op); restrictions->op_registered = true; break; case IORING_RESTRICTION_SQE_FLAGS_ALLOWED: restrictions->sqe_flags_allowed = res[i].sqe_flags; restrictions->op_registered = true; break; case IORING_RESTRICTION_SQE_FLAGS_REQUIRED: restrictions->sqe_flags_required = res[i].sqe_flags; restrictions->op_registered = true; break; default: goto err; } } ret = nr_args; if (!nr_args) { restrictions->op_registered = true; restrictions->reg_registered = true; }err: kfree(res); return ret;} static __cold int io_register_restrictions(struct io_ring_ctx *ctx, void __user *arg, unsigned int nr_args){ int ret; /* Restrictions allowed only if rings started disabled */ if (!(ctx->flags & IORING_SETUP_R_DISABLED)) return -EBADFD; /* We allow only a single restrictions registration */ if (ctx->restrictions.op_registered || ctx->restrictions.reg_registered) return -EBUSY; ret = io_parse_restrictions(arg, nr_args, &ctx->restrictions); /* * Reset all restrictions if an error happened, but retain any COW'ed * settings. */ if (ret < 0) { struct io_bpf_filters *bpf = ctx->restrictions.bpf_filters; bool cowed = ctx->restrictions.bpf_filters_cow; memset(&ctx->restrictions, 0, sizeof(ctx->restrictions)); ctx->restrictions.bpf_filters = bpf; ctx->restrictions.bpf_filters_cow = cowed; return ret; } if (ctx->restrictions.op_registered) ctx->int_flags |= IO_RING_F_OP_RESTRICTED; if (ctx->restrictions.reg_registered) ctx->int_flags |= IO_RING_F_REG_RESTRICTED; return 0;} static int io_register_restrictions_task(void __user *arg, unsigned int nr_args){ struct io_uring_task_restriction __user *ures = arg; struct io_uring_task_restriction tres; struct io_restriction *res; int ret; /* Disallow if task already has registered restrictions */ if (current->io_uring_restrict) return -EPERM; /* * Similar to seccomp, disallow setting a filter if task_no_new_privs * is false and we're not CAP_SYS_ADMIN. */ if (!task_no_new_privs(current) && !ns_capable_noaudit(current_user_ns(), CAP_SYS_ADMIN)) return -EACCES; if (nr_args != 1) return -EINVAL; if (copy_from_user(&tres, arg, sizeof(tres))) return -EFAULT; if (tres.flags) return -EINVAL; if (!mem_is_zero(tres.resv, sizeof(tres.resv))) return -EINVAL; res = kzalloc_obj(*res, GFP_KERNEL_ACCOUNT); if (!res) return -ENOMEM; ret = io_parse_restrictions(ures->restrictions, tres.nr_res, res); if (ret < 0) { kfree(res); return ret; } current->io_uring_restrict = res; return 0;} static int io_register_bpf_filter_task(void __user *arg, unsigned int nr_args){ struct io_restriction *res; int ret; /* * Similar to seccomp, disallow setting a filter if task_no_new_privs * is false and we're not CAP_SYS_ADMIN. */ if (!task_no_new_privs(current) && !ns_capable_noaudit(current_user_ns(), CAP_SYS_ADMIN)) return -EACCES; if (nr_args != 1) return -EINVAL; /* If no task restrictions exist, setup a new set */ res = current->io_uring_restrict; if (!res) { res = kzalloc_obj(*res, GFP_KERNEL_ACCOUNT); if (!res) return -ENOMEM; } ret = io_register_bpf_filter(res, arg); if (ret) { if (res != current->io_uring_restrict) kfree(res); return ret; } if (!current->io_uring_restrict) current->io_uring_restrict = res; return 0;} static int io_register_enable_rings(struct io_ring_ctx *ctx){ if (!(ctx->flags & IORING_SETUP_R_DISABLED)) return -EBADFD; if (ctx->flags & IORING_SETUP_SINGLE_ISSUER) { ctx->submitter_task = get_task_struct(current); /* * Lazy activation attempts would fail if it was polled before * submitter_task is set. */ if (wq_has_sleeper(&ctx->poll_wq)) io_activate_pollwq(ctx); } /* Keep submitter_task store before clearing IORING_SETUP_R_DISABLED */ smp_store_release(&ctx->flags, ctx->flags & ~IORING_SETUP_R_DISABLED); if (ctx->sq_data && wq_has_sleeper(&ctx->sq_data->wait)) wake_up(&ctx->sq_data->wait); return 0;} static __cold int __io_register_iowq_aff(struct io_ring_ctx *ctx, cpumask_var_t new_mask){ int ret; if (!(ctx->flags & IORING_SETUP_SQPOLL)) { ret = io_wq_cpu_affinity(current->io_uring, new_mask); } else { mutex_unlock(&ctx->uring_lock); ret = io_sqpoll_wq_cpu_affinity(ctx, new_mask); mutex_lock(&ctx->uring_lock); } return ret;} static __cold int io_register_iowq_aff(struct io_ring_ctx *ctx, void __user *arg, unsigned len){ cpumask_var_t new_mask; int ret; if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) return -ENOMEM; cpumask_clear(new_mask); if (len > cpumask_size()) len = cpumask_size(); #ifdef CONFIG_COMPAT if (in_compat_syscall()) ret = compat_get_bitmap(cpumask_bits(new_mask), (const compat_ulong_t __user *)arg, len * 8 /* CHAR_BIT */); else#endif ret = copy_from_user(new_mask, arg, len); if (ret) { free_cpumask_var(new_mask); return -EFAULT; } ret = __io_register_iowq_aff(ctx, new_mask); free_cpumask_var(new_mask); return ret;} static __cold int io_unregister_iowq_aff(struct io_ring_ctx *ctx){ return __io_register_iowq_aff(ctx, NULL);} static __cold int io_register_iowq_max_workers(struct io_ring_ctx *ctx, void __user *arg) __must_hold(&ctx->uring_lock){ struct io_tctx_node *node; struct io_uring_task *tctx = NULL; struct io_sq_data *sqd = NULL; __u32 new_count[2]; int i, ret; if (copy_from_user(new_count, arg, sizeof(new_count))) return -EFAULT; for (i = 0; i < ARRAY_SIZE(new_count); i++) if (new_count[i] > INT_MAX) return -EINVAL; if (ctx->flags & IORING_SETUP_SQPOLL) { sqd = ctx->sq_data; if (sqd) { struct task_struct *tsk; /* * Observe the correct sqd->lock -> ctx->uring_lock * ordering. Fine to drop uring_lock here, we hold * a ref to the ctx. */ refcount_inc(&sqd->refs); mutex_unlock(&ctx->uring_lock); mutex_lock(&sqd->lock); mutex_lock(&ctx->uring_lock); tsk = sqpoll_task_locked(sqd); if (tsk) tctx = tsk->io_uring; } } else { tctx = current->io_uring; } BUILD_BUG_ON(sizeof(new_count) != sizeof(ctx->iowq_limits)); for (i = 0; i < ARRAY_SIZE(new_count); i++) if (new_count[i]) ctx->iowq_limits[i] = new_count[i]; ctx->int_flags |= IO_RING_F_IOWQ_LIMITS_SET; if (tctx && tctx->io_wq) { ret = io_wq_max_workers(tctx->io_wq, new_count); if (ret) goto err; } else { memset(new_count, 0, sizeof(new_count)); } if (sqd) { mutex_unlock(&ctx->uring_lock); mutex_unlock(&sqd->lock); io_put_sq_data(sqd); mutex_lock(&ctx->uring_lock); } if (copy_to_user(arg, new_count, sizeof(new_count))) return -EFAULT; /* that's it for SQPOLL, only the SQPOLL task creates requests */ if (sqd) return 0; /* now propagate the restriction to all registered users */ mutex_lock(&ctx->tctx_lock); list_for_each_entry(node, &ctx->tctx_list, ctx_node) { tctx = node->task->io_uring; if (WARN_ON_ONCE(!tctx->io_wq)) continue; for (i = 0; i < ARRAY_SIZE(new_count); i++) new_count[i] = ctx->iowq_limits[i]; /* ignore errors, it always returns zero anyway */ (void)io_wq_max_workers(tctx->io_wq, new_count); } mutex_unlock(&ctx->tctx_lock); return 0;err: if (sqd) { mutex_unlock(&ctx->uring_lock); mutex_unlock(&sqd->lock); io_put_sq_data(sqd); mutex_lock(&ctx->uring_lock); } return ret;} static int io_register_clock(struct io_ring_ctx *ctx, struct io_uring_clock_register __user *arg){ struct io_uring_clock_register reg; if (copy_from_user(®, arg, sizeof(reg))) return -EFAULT; if (memchr_inv(®.__resv, 0, sizeof(reg.__resv))) return -EINVAL; switch (reg.clockid) { case CLOCK_MONOTONIC: ctx->clock_offset = 0; break; case CLOCK_BOOTTIME: ctx->clock_offset = TK_OFFS_BOOT; break; default: return -EINVAL; } ctx->clockid = reg.clockid; return 0;} /* * State to maintain until we can swap. Both new and old state, used for * either mapping or freeing. */struct io_ring_ctx_rings { struct io_rings *rings; struct io_uring_sqe *sq_sqes; struct io_mapped_region sq_region; struct io_mapped_region ring_region;}; static void io_register_free_rings(struct io_ring_ctx *ctx, struct io_ring_ctx_rings *r){ io_free_region(ctx->user, &r->sq_region); io_free_region(ctx->user, &r->ring_region);} #define swap_old(ctx, o, n, field) \ do { \ (o).field = (ctx)->field; \ (ctx)->field = (n).field; \ } while (0) #define RESIZE_FLAGS (IORING_SETUP_CQSIZE | IORING_SETUP_CLAMP)#define COPY_FLAGS (IORING_SETUP_NO_SQARRAY | IORING_SETUP_SQE128 | \ IORING_SETUP_CQE32 | IORING_SETUP_NO_MMAP | \ IORING_SETUP_CQE_MIXED | IORING_SETUP_SQE_MIXED) static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg){ struct io_ctx_config config;Lines 1–500 of 1049. Load more
// SPDX-License-Identifier: GPL-2.0 /* * Code related to the io_uring_register() syscall * * Copyright (C) 2023 Jens Axboe */ #include <linux/kernel.h> #include <linux/errno.h> #include <linux/syscalls.h> #include <linux/refcount.h> #include <linux/bits.h> #include <linux/fs.h> #include <linux/file.h> #include <linux/slab.h> #include <linux/uaccess.h> #include <linux/nospec.h> #include <linux/compat.h> #include <linux/io_uring.h> #include <linux/io_uring_types.h> #include "filetable.h" #include "io_uring.h" #include "opdef.h" #include "tctx.h" #include "rsrc.h" #include "sqpoll.h" #include "register.h" #include "cancel.h" #include "kbuf.h" #include "napi.h" #include "eventfd.h" #include "msg_ring.h" #include "memmap.h" #include "zcrx.h" #include "query.h" #include "bpf_filter.h" #define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \ IORING_REGISTER_LAST + IORING_OP_LAST) static __cold int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args) { struct io_uring_probe *p; size_t size; int i, ret; if (nr_args > IORING_OP_LAST) nr_args = IORING_OP_LAST; size = struct_size(p, ops, nr_args); p = memdup_user(arg, size); if (IS_ERR(p)) return PTR_ERR(p); ret = -EINVAL; if (memchr_inv(p, 0, size)) goto out; p->last_op = IORING_OP_LAST - 1; for (i = 0; i < nr_args; i++) { p->ops[i].op = i; if (io_uring_op_supported(i)) p->ops[i].flags = IO_URING_OP_SUPPORTED; } p->ops_len = i; ret = 0; if (copy_to_user(arg, p, size)) ret = -EFAULT; out: kfree(p); return ret; } int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id) { const struct cred *creds; creds = xa_erase(&ctx->personalities, id); if (creds) { put_cred(creds); return 0; } return -EINVAL; } static int io_register_personality(struct io_ring_ctx *ctx) { const struct cred *creds; u32 id; int ret; creds = get_current_cred(); ret = xa_alloc_cyclic(&ctx->personalities, &id, (void *)creds, XA_LIMIT(0, USHRT_MAX), &ctx->pers_next, GFP_KERNEL); if (ret < 0) { put_cred(creds); return ret; } return id; } /* * Returns number of restrictions parsed and added on success, or < 0 for * an error. */ static __cold int io_parse_restrictions(void __user *arg, unsigned int nr_args, struct io_restriction *restrictions) { struct io_uring_restriction *res; size_t size; int i, ret; if (!arg || nr_args > IORING_MAX_RESTRICTIONS) return -EINVAL; size = array_size(nr_args, sizeof(*res)); if (size == SIZE_MAX) return -EOVERFLOW; res = memdup_user(arg, size); if (IS_ERR(res)) return PTR_ERR(res); ret = -EINVAL; for (i = 0; i < nr_args; i++) { switch (res[i].opcode) { case IORING_RESTRICTION_REGISTER_OP: if (res[i].register_op >= IORING_REGISTER_LAST) goto err; __set_bit(res[i].register_op, restrictions->register_op); restrictions->reg_registered = true; break; case IORING_RESTRICTION_SQE_OP: if (res[i].sqe_op >= IORING_OP_LAST) goto err; __set_bit(res[i].sqe_op, restrictions->sqe_op); restrictions->op_registered = true; break; case IORING_RESTRICTION_SQE_FLAGS_ALLOWED: restrictions->sqe_flags_allowed = res[i].sqe_flags; restrictions->op_registered = true; break; case IORING_RESTRICTION_SQE_FLAGS_REQUIRED: restrictions->sqe_flags_required = res[i].sqe_flags; restrictions->op_registered = true; break; default: goto err; } } ret = nr_args; if (!nr_args) { restrictions->op_registered = true; restrictions->reg_registered = true; } err: kfree(res); return ret; } static __cold int io_register_restrictions(struct io_ring_ctx *ctx, void __user *arg, unsigned int nr_args) { int ret; /* Restrictions allowed only if rings started disabled */ if (!(ctx->flags & IORING_SETUP_R_DISABLED)) return -EBADFD; /* We allow only a single restrictions registration */ if (ctx->restrictions.op_registered || ctx->restrictions.reg_registered) return -EBUSY; ret = io_parse_restrictions(arg, nr_args, &ctx->restrictions); /* * Reset all restrictions if an error happened, but retain any COW'ed * settings. */ if (ret < 0) { struct io_bpf_filters *bpf = ctx->restrictions.bpf_filters; bool cowed = ctx->restrictions.bpf_filters_cow; memset(&ctx->restrictions, 0, sizeof(ctx->restrictions)); ctx->restrictions.bpf_filters = bpf; ctx->restrictions.bpf_filters_cow = cowed; return ret; } if (ctx->restrictions.op_registered) ctx->int_flags |= IO_RING_F_OP_RESTRICTED; if (ctx->restrictions.reg_registered) ctx->int_flags |= IO_RING_F_REG_RESTRICTED; return 0; } static int io_register_restrictions_task(void __user *arg, unsigned int nr_args) { struct io_uring_task_restriction __user *ures = arg; struct io_uring_task_restriction tres; struct io_restriction *res; int ret; /* Disallow if task already has registered restrictions */ if (current->io_uring_restrict) return -EPERM; /* * Similar to seccomp, disallow setting a filter if task_no_new_privs * is false and we're not CAP_SYS_ADMIN. */ if (!task_no_new_privs(current) && !ns_capable_noaudit(current_user_ns(), CAP_SYS_ADMIN)) return -EACCES; if (nr_args != 1) return -EINVAL; if (copy_from_user(&tres, arg, sizeof(tres))) return -EFAULT; if (tres.flags) return -EINVAL; if (!mem_is_zero(tres.resv, sizeof(tres.resv))) return -EINVAL; res = kzalloc_obj(*res, GFP_KERNEL_ACCOUNT); if (!res) return -ENOMEM; ret = io_parse_restrictions(ures->restrictions, tres.nr_res, res); if (ret < 0) { kfree(res); return ret; } current->io_uring_restrict = res; return 0; } static int io_register_bpf_filter_task(void __user *arg, unsigned int nr_args) { struct io_restriction *res; int ret; /* * Similar to seccomp, disallow setting a filter if task_no_new_privs * is false and we're not CAP_SYS_ADMIN. */ if (!task_no_new_privs(current) && !ns_capable_noaudit(current_user_ns(), CAP_SYS_ADMIN)) return -EACCES; if (nr_args != 1) return -EINVAL; /* If no task restrictions exist, setup a new set */ res = current->io_uring_restrict; if (!res) { res = kzalloc_obj(*res, GFP_KERNEL_ACCOUNT); if (!res) return -ENOMEM; } ret = io_register_bpf_filter(res, arg); if (ret) { if (res != current->io_uring_restrict) kfree(res); return ret; } if (!current->io_uring_restrict) current->io_uring_restrict = res; return 0; } static int io_register_enable_rings(struct io_ring_ctx *ctx) { if (!(ctx->flags & IORING_SETUP_R_DISABLED)) return -EBADFD; if (ctx->flags & IORING_SETUP_SINGLE_ISSUER) { ctx->submitter_task = get_task_struct(current); /* * Lazy activation attempts would fail if it was polled before * submitter_task is set. */ if (wq_has_sleeper(&ctx->poll_wq)) io_activate_pollwq(ctx); } /* Keep submitter_task store before clearing IORING_SETUP_R_DISABLED */ smp_store_release(&ctx->flags, ctx->flags & ~IORING_SETUP_R_DISABLED); if (ctx->sq_data && wq_has_sleeper(&ctx->sq_data->wait)) wake_up(&ctx->sq_data->wait); return 0; } static __cold int __io_register_iowq_aff(struct io_ring_ctx *ctx, cpumask_var_t new_mask) { int ret; if (!(ctx->flags & IORING_SETUP_SQPOLL)) { ret = io_wq_cpu_affinity(current->io_uring, new_mask); } else { mutex_unlock(&ctx->uring_lock); ret = io_sqpoll_wq_cpu_affinity(ctx, new_mask); mutex_lock(&ctx->uring_lock); } return ret; } static __cold int io_register_iowq_aff(struct io_ring_ctx *ctx, void __user *arg, unsigned len) { cpumask_var_t new_mask; int ret; if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) return -ENOMEM; cpumask_clear(new_mask); if (len > cpumask_size()) len = cpumask_size(); #ifdef CONFIG_COMPAT if (in_compat_syscall()) ret = compat_get_bitmap(cpumask_bits(new_mask), (const compat_ulong_t __user *)arg, len * 8 /* CHAR_BIT */); else #endif ret = copy_from_user(new_mask, arg, len); if (ret) { free_cpumask_var(new_mask); return -EFAULT; } ret = __io_register_iowq_aff(ctx, new_mask); free_cpumask_var(new_mask); return ret; } static __cold int io_unregister_iowq_aff(struct io_ring_ctx *ctx) { return __io_register_iowq_aff(ctx, NULL); } static __cold int io_register_iowq_max_workers(struct io_ring_ctx *ctx, void __user *arg) __must_hold(&ctx->uring_lock) { struct io_tctx_node *node; struct io_uring_task *tctx = NULL; struct io_sq_data *sqd = NULL; __u32 new_count[2]; int i, ret; if (copy_from_user(new_count, arg, sizeof(new_count))) return -EFAULT; for (i = 0; i < ARRAY_SIZE(new_count); i++) if (new_count[i] > INT_MAX) return -EINVAL; if (ctx->flags & IORING_SETUP_SQPOLL) { sqd = ctx->sq_data; if (sqd) { struct task_struct *tsk; /* * Observe the correct sqd->lock -> ctx->uring_lock * ordering. Fine to drop uring_lock here, we hold * a ref to the ctx. */ refcount_inc(&sqd->refs); mutex_unlock(&ctx->uring_lock); mutex_lock(&sqd->lock); mutex_lock(&ctx->uring_lock); tsk = sqpoll_task_locked(sqd); if (tsk) tctx = tsk->io_uring; } } else { tctx = current->io_uring; } BUILD_BUG_ON(sizeof(new_count) != sizeof(ctx->iowq_limits)); for (i = 0; i < ARRAY_SIZE(new_count); i++) if (new_count[i]) ctx->iowq_limits[i] = new_count[i]; ctx->int_flags |= IO_RING_F_IOWQ_LIMITS_SET; if (tctx && tctx->io_wq) { ret = io_wq_max_workers(tctx->io_wq, new_count); if (ret) goto err; } else { memset(new_count, 0, sizeof(new_count)); } if (sqd) { mutex_unlock(&ctx->uring_lock); mutex_unlock(&sqd->lock); io_put_sq_data(sqd); mutex_lock(&ctx->uring_lock); } if (copy_to_user(arg, new_count, sizeof(new_count))) return -EFAULT; /* that's it for SQPOLL, only the SQPOLL task creates requests */ if (sqd) return 0; /* now propagate the restriction to all registered users */ mutex_lock(&ctx->tctx_lock); list_for_each_entry(node, &ctx->tctx_list, ctx_node) { tctx = node->task->io_uring; if (WARN_ON_ONCE(!tctx->io_wq)) continue; for (i = 0; i < ARRAY_SIZE(new_count); i++) new_count[i] = ctx->iowq_limits[i]; /* ignore errors, it always returns zero anyway */ (void)io_wq_max_workers(tctx->io_wq, new_count); } mutex_unlock(&ctx->tctx_lock); return 0; err: if (sqd) { mutex_unlock(&ctx->uring_lock); mutex_unlock(&sqd->lock); io_put_sq_data(sqd); mutex_lock(&ctx->uring_lock); } return ret; } static int io_register_clock(struct io_ring_ctx *ctx, struct io_uring_clock_register __user *arg) { struct io_uring_clock_register reg; if (copy_from_user(®, arg, sizeof(reg))) return -EFAULT; if (memchr_inv(®.__resv, 0, sizeof(reg.__resv))) return -EINVAL; switch (reg.clockid) { case CLOCK_MONOTONIC: ctx->clock_offset = 0; break; case CLOCK_BOOTTIME: ctx->clock_offset = TK_OFFS_BOOT; break; default: return -EINVAL; } ctx->clockid = reg.clockid; return 0; } /* * State to maintain until we can swap. Both new and old state, used for * either mapping or freeing. */ struct io_ring_ctx_rings { struct io_rings *rings; struct io_uring_sqe *sq_sqes; struct io_mapped_region sq_region; struct io_mapped_region ring_region; }; static void io_register_free_rings(struct io_ring_ctx *ctx, struct io_ring_ctx_rings *r) { io_free_region(ctx->user, &r->sq_region); io_free_region(ctx->user, &r->ring_region); } #define swap_old(ctx, o, n, field) \ do { \ (o).field = (ctx)->field; \ (ctx)->field = (n).field; \ } while (0) #define RESIZE_FLAGS (IORING_SETUP_CQSIZE | IORING_SETUP_CLAMP) #define COPY_FLAGS (IORING_SETUP_NO_SQARRAY | IORING_SETUP_SQE128 | \ IORING_SETUP_CQE32 | IORING_SETUP_NO_MMAP | \ IORING_SETUP_CQE_MIXED | IORING_SETUP_SQE_MIXED) static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) { struct io_ctx_config config;