// SPDX-License-Identifier: GPL-2.0/* * Code related to the io_uring_register() syscall * * Copyright (C) 2023 Jens Axboe */#include <linux/kernel.h>#include <linux/errno.h>#include <linux/syscalls.h>#include <linux/refcount.h>#include <linux/bits.h>#include <linux/fs.h>#include <linux/file.h>#include <linux/slab.h>#include <linux/uaccess.h>#include <linux/nospec.h>#include <linux/compat.h>#include <linux/io_uring.h>#include <linux/io_uring_types.h> #include "io_uring.h"#include "opdef.h"#include "tctx.h"#include "rsrc.h"#include "sqpoll.h"#include "register.h"#include "cancel.h"#include "kbuf.h"#include "napi.h"#include "eventfd.h"#include "msg_ring.h"#include "memmap.h" #define IORING_MAX_RESTRICTIONS	(IORING_RESTRICTION_LAST + \				 IORING_REGISTER_LAST + IORING_OP_LAST) static __cold int io_probe(struct io_ring_ctx *ctx, void __user *arg,			   unsigned nr_args){	struct io_uring_probe *p;	size_t size;	int i, ret; 	if (nr_args > IORING_OP_LAST)		nr_args = IORING_OP_LAST; 	size = struct_size(p, ops, nr_args);	p = kzalloc(size, GFP_KERNEL);	if (!p)		return -ENOMEM; 	ret = -EFAULT;	if (copy_from_user(p, arg, size))		goto out;	ret = -EINVAL;	if (memchr_inv(p, 0, size))		goto out; 	p->last_op = IORING_OP_LAST - 1; 	for (i = 0; i < nr_args; i++) {		p->ops[i].op = i;		if (io_uring_op_supported(i))			p->ops[i].flags = IO_URING_OP_SUPPORTED;	}	p->ops_len = i; 	ret = 0;	if (copy_to_user(arg, p, size))		ret = -EFAULT;out:	kfree(p);	return ret;} int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id){	const struct cred *creds; 	creds = xa_erase(&ctx->personalities, id);	if (creds) {		put_cred(creds);		return 0;	} 	return -EINVAL;}  static int io_register_personality(struct io_ring_ctx *ctx){	const struct cred *creds;	u32 id;	int ret; 	creds = get_current_cred(); 	ret = xa_alloc_cyclic(&ctx->personalities, &id, (void *)creds,			XA_LIMIT(0, USHRT_MAX), &ctx->pers_next, GFP_KERNEL);	if (ret < 0) {		put_cred(creds);		return ret;	}	return id;} static __cold int io_register_restrictions(struct io_ring_ctx *ctx,					   void __user *arg, unsigned int nr_args){	struct io_uring_restriction *res;	size_t size;	int i, ret; 	/* Restrictions allowed only if rings started disabled */	if (!(ctx->flags & IORING_SETUP_R_DISABLED))		return -EBADFD; 	/* We allow only a single restrictions registration */	if (ctx->restrictions.registered)		return -EBUSY; 	if (!arg || nr_args > IORING_MAX_RESTRICTIONS)		return -EINVAL; 	size = array_size(nr_args, sizeof(*res));	if (size == SIZE_MAX)		return -EOVERFLOW; 	res = memdup_user(arg, size);	if (IS_ERR(res))		return PTR_ERR(res); 	ret = 0; 	for (i = 0; i < nr_args; i++) {		switch (res[i].opcode) {		case IORING_RESTRICTION_REGISTER_OP:			if (res[i].register_op >= IORING_REGISTER_LAST) {				ret = -EINVAL;				goto out;			} 			__set_bit(res[i].register_op,				  ctx->restrictions.register_op);			break;		case IORING_RESTRICTION_SQE_OP:			if (res[i].sqe_op >= IORING_OP_LAST) {				ret = -EINVAL;				goto out;			} 			__set_bit(res[i].sqe_op, ctx->restrictions.sqe_op);			break;		case IORING_RESTRICTION_SQE_FLAGS_ALLOWED:			ctx->restrictions.sqe_flags_allowed = res[i].sqe_flags;			break;		case IORING_RESTRICTION_SQE_FLAGS_REQUIRED:			ctx->restrictions.sqe_flags_required = res[i].sqe_flags;			break;		default:			ret = -EINVAL;			goto out;		}	} out:	/* Reset all restrictions if an error happened */	if (ret != 0)		memset(&ctx->restrictions, 0, sizeof(ctx->restrictions));	else		ctx->restrictions.registered = true; 	kfree(res);	return ret;} static int io_register_enable_rings(struct io_ring_ctx *ctx){	if (!(ctx->flags & IORING_SETUP_R_DISABLED))		return -EBADFD; 	if (ctx->flags & IORING_SETUP_SINGLE_ISSUER && !ctx->submitter_task) {		WRITE_ONCE(ctx->submitter_task, get_task_struct(current));		/*		 * Lazy activation attempts would fail if it was polled before		 * submitter_task is set.		 */		if (wq_has_sleeper(&ctx->poll_wq))			io_activate_pollwq(ctx);	} 	if (ctx->restrictions.registered)		ctx->restricted = 1; 	ctx->flags &= ~IORING_SETUP_R_DISABLED;	if (ctx->sq_data && wq_has_sleeper(&ctx->sq_data->wait))		wake_up(&ctx->sq_data->wait);	return 0;} static __cold int __io_register_iowq_aff(struct io_ring_ctx *ctx,					 cpumask_var_t new_mask){	int ret; 	if (!(ctx->flags & IORING_SETUP_SQPOLL)) {		ret = io_wq_cpu_affinity(current->io_uring, new_mask);	} else {		mutex_unlock(&ctx->uring_lock);		ret = io_sqpoll_wq_cpu_affinity(ctx, new_mask);		mutex_lock(&ctx->uring_lock);	} 	return ret;} static __cold int io_register_iowq_aff(struct io_ring_ctx *ctx,				       void __user *arg, unsigned len){	cpumask_var_t new_mask;	int ret; 	if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))		return -ENOMEM; 	cpumask_clear(new_mask);	if (len > cpumask_size())		len = cpumask_size(); #ifdef CONFIG_COMPAT	if (in_compat_syscall())		ret = compat_get_bitmap(cpumask_bits(new_mask),					(const compat_ulong_t __user *)arg,					len * 8 /* CHAR_BIT */);	else#endif		ret = copy_from_user(new_mask, arg, len); 	if (ret) {		free_cpumask_var(new_mask);		return -EFAULT;	} 	ret = __io_register_iowq_aff(ctx, new_mask);	free_cpumask_var(new_mask);	return ret;} static __cold int io_unregister_iowq_aff(struct io_ring_ctx *ctx){	return __io_register_iowq_aff(ctx, NULL);} static __cold int io_register_iowq_max_workers(struct io_ring_ctx *ctx,					       void __user *arg)	__must_hold(&ctx->uring_lock){	struct io_tctx_node *node;	struct io_uring_task *tctx = NULL;	struct io_sq_data *sqd = NULL;	__u32 new_count[2];	int i, ret; 	if (copy_from_user(new_count, arg, sizeof(new_count)))		return -EFAULT;	for (i = 0; i < ARRAY_SIZE(new_count); i++)		if (new_count[i] > INT_MAX)			return -EINVAL; 	if (ctx->flags & IORING_SETUP_SQPOLL) {		sqd = ctx->sq_data;		if (sqd) {			/*			 * Observe the correct sqd->lock -> ctx->uring_lock			 * ordering. Fine to drop uring_lock here, we hold			 * a ref to the ctx.			 */			refcount_inc(&sqd->refs);			mutex_unlock(&ctx->uring_lock);			mutex_lock(&sqd->lock);			mutex_lock(&ctx->uring_lock);			if (sqd->thread)				tctx = sqd->thread->io_uring;		}	} else {		tctx = current->io_uring;	} 	BUILD_BUG_ON(sizeof(new_count) != sizeof(ctx->iowq_limits)); 	for (i = 0; i < ARRAY_SIZE(new_count); i++)		if (new_count[i])			ctx->iowq_limits[i] = new_count[i];	ctx->iowq_limits_set = true; 	if (tctx && tctx->io_wq) {		ret = io_wq_max_workers(tctx->io_wq, new_count);		if (ret)			goto err;	} else {		memset(new_count, 0, sizeof(new_count));	} 	if (sqd) {		mutex_unlock(&ctx->uring_lock);		mutex_unlock(&sqd->lock);		io_put_sq_data(sqd);		mutex_lock(&ctx->uring_lock);	} 	if (copy_to_user(arg, new_count, sizeof(new_count)))		return -EFAULT; 	/* that's it for SQPOLL, only the SQPOLL task creates requests */	if (sqd)		return 0; 	/* now propagate the restriction to all registered users */	list_for_each_entry(node, &ctx->tctx_list, ctx_node) {		tctx = node->task->io_uring;		if (WARN_ON_ONCE(!tctx->io_wq))			continue; 		for (i = 0; i < ARRAY_SIZE(new_count); i++)			new_count[i] = ctx->iowq_limits[i];		/* ignore errors, it always returns zero anyway */		(void)io_wq_max_workers(tctx->io_wq, new_count);	}	return 0;err:	if (sqd) {		mutex_unlock(&ctx->uring_lock);		mutex_unlock(&sqd->lock);		io_put_sq_data(sqd);		mutex_lock(&ctx->uring_lock);	}	return ret;} static int io_register_clock(struct io_ring_ctx *ctx,			     struct io_uring_clock_register __user *arg){	struct io_uring_clock_register reg; 	if (copy_from_user(&reg, arg, sizeof(reg)))		return -EFAULT;	if (memchr_inv(&reg.__resv, 0, sizeof(reg.__resv)))		return -EINVAL; 	switch (reg.clockid) {	case CLOCK_MONOTONIC:		ctx->clock_offset = 0;		break;	case CLOCK_BOOTTIME:		ctx->clock_offset = TK_OFFS_BOOT;		break;	default:		return -EINVAL;	} 	ctx->clockid = reg.clockid;	return 0;} /* * State to maintain until we can swap. Both new and old state, used for * either mapping or freeing. */struct io_ring_ctx_rings {	unsigned short n_ring_pages;	unsigned short n_sqe_pages;	struct page **ring_pages;	struct page **sqe_pages;	struct io_uring_sqe *sq_sqes;	struct io_rings *rings;}; static void io_register_free_rings(struct io_uring_params *p,				   struct io_ring_ctx_rings *r){	if (!(p->flags & IORING_SETUP_NO_MMAP)) {		io_pages_unmap(r->rings, &r->ring_pages, &r->n_ring_pages,				true);		io_pages_unmap(r->sq_sqes, &r->sqe_pages, &r->n_sqe_pages,				true);	} else {		io_pages_free(&r->ring_pages, r->n_ring_pages);		io_pages_free(&r->sqe_pages, r->n_sqe_pages);		vunmap(r->rings);		vunmap(r->sq_sqes);	}} #define swap_old(ctx, o, n, field)		\	do {					\		(o).field = (ctx)->field;	\		(ctx)->field = (n).field;	\	} while (0) #define RESIZE_FLAGS	(IORING_SETUP_CQSIZE | IORING_SETUP_CLAMP)#define COPY_FLAGS	(IORING_SETUP_NO_SQARRAY | IORING_SETUP_SQE128 | \			 IORING_SETUP_CQE32 | IORING_SETUP_NO_MMAP) static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg){	struct io_ring_ctx_rings o = { }, n = { }, *to_free = NULL;	size_t size, sq_array_offset;	unsigned i, tail, old_head;	struct io_uring_params p;	void *ptr;	int ret; 	/* for single issuer, must be owner resizing */	if (ctx->flags & IORING_SETUP_SINGLE_ISSUER &&	    current != ctx->submitter_task)		return -EEXIST;	/* limited to DEFER_TASKRUN for now */	if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN))		return -EINVAL;	if (copy_from_user(&p, arg, sizeof(p)))		return -EFAULT;	if (p.flags & ~RESIZE_FLAGS)		return -EINVAL; 	/* properties that are always inherited */	p.flags |= (ctx->flags & COPY_FLAGS); 	ret = io_uring_fill_params(p.sq_entries, &p);	if (unlikely(ret))		return ret; 	/* nothing to do, but copy params back */	if (p.sq_entries == ctx->sq_entries && p.cq_entries == ctx->cq_entries) {		if (copy_to_user(arg, &p, sizeof(p)))			return -EFAULT;		return 0;	} 	size = rings_size(p.flags, p.sq_entries, p.cq_entries,				&sq_array_offset);	if (size == SIZE_MAX)		return -EOVERFLOW; 	if (!(p.flags & IORING_SETUP_NO_MMAP))		n.rings = io_pages_map(&n.ring_pages, &n.n_ring_pages, size);	else		n.rings = __io_uaddr_map(&n.ring_pages, &n.n_ring_pages,						p.cq_off.user_addr, size);	if (IS_ERR(n.rings))		return PTR_ERR(n.rings); 	/*	 * At this point n.rings is shared with userspace, just like o.rings	 * is as well. While we don't expect userspace to modify it while	 * a resize is in progress, and it's most likely that userspace will	 * shoot itself in the foot if it does, we can't always assume good	 * intent... Use read/write once helpers from here on to indicate the	 * shared nature of it.	 */	WRITE_ONCE(n.rings->sq_ring_mask, p.sq_entries - 1);	WRITE_ONCE(n.rings->cq_ring_mask, p.cq_entries - 1);	WRITE_ONCE(n.rings->sq_ring_entries, p.sq_entries);	WRITE_ONCE(n.rings->cq_ring_entries, p.cq_entries); 	if (copy_to_user(arg, &p, sizeof(p))) {		io_register_free_rings(&p, &n);		return -EFAULT;	} 	if (p.flags & IORING_SETUP_SQE128)		size = array_size(2 * sizeof(struct io_uring_sqe), p.sq_entries);	else		size = array_size(sizeof(struct io_uring_sqe), p.sq_entries);	if (size == SIZE_MAX) {		io_register_free_rings(&p, &n);		return -EOVERFLOW;	} 	if (!(p.flags & IORING_SETUP_NO_MMAP))		ptr = io_pages_map(&n.sqe_pages, &n.n_sqe_pages, size);	else		ptr = __io_uaddr_map(&n.sqe_pages, &n.n_sqe_pages,					p.sq_off.user_addr,					size);	if (IS_ERR(ptr)) {		io_register_free_rings(&p, &n);		return PTR_ERR(ptr);	} 	/*	 * If using SQPOLL, park the thread	 */	if (ctx->sq_data) {		mutex_unlock(&ctx->uring_lock);		io_sq_thread_park(ctx->sq_data);		mutex_lock(&ctx->uring_lock);	} 	/*	 * We'll do the swap. Grab the ctx->resize_lock, which will exclude

Lines 1–500 of 928. Load more