// SPDX-License-Identifier: GPL-2.0/* * Shared application/kernel submission and completion ring pairs, for * supporting fast/efficient IO. * * A note on the read/write ordering memory barriers that are matched between * the application and kernel side. * * After the application reads the CQ ring tail, it must use an * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses * before writing the tail (using smp_load_acquire to read the tail will * do). It also needs a smp_mb() before updating CQ head (ordering the * entry load(s) with the head store), pairing with an implicit barrier * through a control-dependency in io_get_cqe (smp_store_release to * store head will do). Failure to do so could lead to reading invalid * CQ entries. * * Likewise, the application must use an appropriate smp_wmb() before * writing the SQ tail (ordering SQ entry stores with the tail store), * which pairs with smp_load_acquire in io_get_sqring (smp_store_release * to store the tail will do). And it needs a barrier ordering the SQ * head load before writing new SQ entries (smp_load_acquire to read * head will do). * * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after* * updating the SQ tail; a full memory barrier smp_mb() is needed * between. * * Also see the examples in the liburing library: * *	git://git.kernel.dk/liburing * * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens * from data shared between the kernel and application. This is done both * for ordering purposes, but also to ensure that once a value is loaded from * data that the application could potentially modify, it remains stable. * * Copyright (C) 2018-2019 Jens Axboe * Copyright (c) 2018-2019 Christoph Hellwig */#include <linux/kernel.h>#include <linux/init.h>#include <linux/errno.h>#include <linux/syscalls.h>#include <net/compat.h>#include <linux/refcount.h>#include <linux/uio.h>#include <linux/bits.h> #include <linux/sched/signal.h>#include <linux/fs.h>#include <linux/file.h>#include <linux/fdtable.h>#include <linux/mm.h>#include <linux/mman.h>#include <linux/percpu.h>#include <linux/slab.h>#include <linux/bvec.h>#include <linux/net.h>#include <net/sock.h>#include <net/af_unix.h>#include <net/scm.h>#include <linux/anon_inodes.h>#include <linux/sched/mm.h>#include <linux/uaccess.h>#include <linux/nospec.h>#include <linux/highmem.h>#include <linux/fsnotify.h>#include <linux/fadvise.h>#include <linux/task_work.h>#include <linux/io_uring.h>#include <linux/audit.h>#include <linux/security.h> #define CREATE_TRACE_POINTS#include <trace/events/io_uring.h> #include <uapi/linux/io_uring.h> #include "io-wq.h" #include "io_uring.h"#include "opdef.h"#include "refs.h"#include "tctx.h"#include "sqpoll.h"#include "fdinfo.h"#include "kbuf.h"#include "rsrc.h"#include "cancel.h"#include "net.h"#include "notif.h" #include "timeout.h"#include "poll.h"#include "alloc_cache.h" #define IORING_MAX_ENTRIES	32768#define IORING_MAX_CQ_ENTRIES	(2 * IORING_MAX_ENTRIES) #define IORING_MAX_RESTRICTIONS	(IORING_RESTRICTION_LAST + \				 IORING_REGISTER_LAST + IORING_OP_LAST) #define SQE_COMMON_FLAGS (IOSQE_FIXED_FILE | IOSQE_IO_LINK | \			  IOSQE_IO_HARDLINK | IOSQE_ASYNC) #define SQE_VALID_FLAGS	(SQE_COMMON_FLAGS | IOSQE_BUFFER_SELECT | \			IOSQE_IO_DRAIN | IOSQE_CQE_SKIP_SUCCESS) #define IO_REQ_CLEAN_FLAGS (REQ_F_BUFFER_SELECTED | REQ_F_NEED_CLEANUP | \				REQ_F_POLLED | REQ_F_INFLIGHT | REQ_F_CREDS | \				REQ_F_ASYNC_DATA) #define IO_REQ_CLEAN_SLOW_FLAGS (REQ_F_REFCOUNT | REQ_F_LINK | REQ_F_HARDLINK |\				 IO_REQ_CLEAN_FLAGS) #define IO_TCTX_REFS_CACHE_NR	(1U << 10) #define IO_COMPL_BATCH			32#define IO_REQ_ALLOC_BATCH		8 enum {	IO_CHECK_CQ_OVERFLOW_BIT,	IO_CHECK_CQ_DROPPED_BIT,}; enum {	IO_EVENTFD_OP_SIGNAL_BIT,	IO_EVENTFD_OP_FREE_BIT,}; struct io_defer_entry {	struct list_head	list;	struct io_kiocb		*req;	u32			seq;}; /* requests with any of those set should undergo io_disarm_next() */#define IO_DISARM_MASK (REQ_F_ARM_LTIMEOUT | REQ_F_LINK_TIMEOUT | REQ_F_FAIL)#define IO_REQ_LINK_FLAGS (REQ_F_LINK | REQ_F_HARDLINK) static bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,					 struct task_struct *task,					 bool cancel_all); static void io_dismantle_req(struct io_kiocb *req);static void io_clean_op(struct io_kiocb *req);static void io_queue_sqe(struct io_kiocb *req);static void io_move_task_work_from_local(struct io_ring_ctx *ctx);static void __io_submit_flush_completions(struct io_ring_ctx *ctx);static __cold void io_fallback_tw(struct io_uring_task *tctx); static struct kmem_cache *req_cachep; struct sock *io_uring_get_socket(struct file *file){#if defined(CONFIG_UNIX)	if (io_is_uring_fops(file)) {		struct io_ring_ctx *ctx = file->private_data; 		return ctx->ring_sock->sk;	}#endif	return NULL;}EXPORT_SYMBOL(io_uring_get_socket); static inline void io_submit_flush_completions(struct io_ring_ctx *ctx){	if (!wq_list_empty(&ctx->submit_state.compl_reqs) ||	    ctx->submit_state.cqes_count)		__io_submit_flush_completions(ctx);} static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx){	return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head);} static inline unsigned int __io_cqring_events_user(struct io_ring_ctx *ctx){	return READ_ONCE(ctx->rings->cq.tail) - READ_ONCE(ctx->rings->cq.head);} static bool io_match_linked(struct io_kiocb *head){	struct io_kiocb *req; 	io_for_each_link(req, head) {		if (req->flags & REQ_F_INFLIGHT)			return true;	}	return false;} /* * As io_match_task() but protected against racing with linked timeouts. * User must not hold timeout_lock. */bool io_match_task_safe(struct io_kiocb *head, struct task_struct *task,			bool cancel_all){	bool matched; 	if (task && head->task != task)		return false;	if (cancel_all)		return true; 	if (head->flags & REQ_F_LINK_TIMEOUT) {		struct io_ring_ctx *ctx = head->ctx; 		/* protect against races with linked timeouts */		spin_lock_irq(&ctx->timeout_lock);		matched = io_match_linked(head);		spin_unlock_irq(&ctx->timeout_lock);	} else {		matched = io_match_linked(head);	}	return matched;} static inline void req_fail_link_node(struct io_kiocb *req, int res){	req_set_fail(req);	io_req_set_res(req, res, 0);} static inline void io_req_add_to_cache(struct io_kiocb *req, struct io_ring_ctx *ctx){	wq_stack_add_head(&req->comp_list, &ctx->submit_state.free_list);} static __cold void io_ring_ctx_ref_free(struct percpu_ref *ref){	struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs); 	complete(&ctx->ref_comp);} static __cold void io_fallback_req_func(struct work_struct *work){	struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx,						fallback_work.work);	struct llist_node *node = llist_del_all(&ctx->fallback_llist);	struct io_kiocb *req, *tmp;	bool locked = false; 	percpu_ref_get(&ctx->refs);	llist_for_each_entry_safe(req, tmp, node, io_task_work.node)		req->io_task_work.func(req, &locked); 	if (locked) {		io_submit_flush_completions(ctx);		mutex_unlock(&ctx->uring_lock);	}	percpu_ref_put(&ctx->refs);} static int io_alloc_hash_table(struct io_hash_table *table, unsigned bits){	unsigned hash_buckets = 1U << bits;	size_t hash_size = hash_buckets * sizeof(table->hbs[0]); 	table->hbs = kmalloc(hash_size, GFP_KERNEL);	if (!table->hbs)		return -ENOMEM; 	table->hash_bits = bits;	init_hash_table(table, hash_buckets);	return 0;} static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p){	struct io_ring_ctx *ctx;	int hash_bits; 	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);	if (!ctx)		return NULL; 	xa_init(&ctx->io_bl_xa); 	/*	 * Use 5 bits less than the max cq entries, that should give us around	 * 32 entries per hash list if totally full and uniformly spread, but	 * don't keep too many buckets to not overconsume memory.	 */	hash_bits = ilog2(p->cq_entries) - 5;	hash_bits = clamp(hash_bits, 1, 8);	if (io_alloc_hash_table(&ctx->cancel_table, hash_bits))		goto err;	if (io_alloc_hash_table(&ctx->cancel_table_locked, hash_bits))		goto err; 	ctx->dummy_ubuf = kzalloc(sizeof(*ctx->dummy_ubuf), GFP_KERNEL);	if (!ctx->dummy_ubuf)		goto err;	/* set invalid range, so io_import_fixed() fails meeting it */	ctx->dummy_ubuf->ubuf = -1UL; 	if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,			    0, GFP_KERNEL))		goto err; 	ctx->flags = p->flags;	init_waitqueue_head(&ctx->sqo_sq_wait);	INIT_LIST_HEAD(&ctx->sqd_list);	INIT_LIST_HEAD(&ctx->cq_overflow_list);	INIT_LIST_HEAD(&ctx->io_buffers_cache);	io_alloc_cache_init(&ctx->apoll_cache);	io_alloc_cache_init(&ctx->netmsg_cache);	init_completion(&ctx->ref_comp);	xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1);	mutex_init(&ctx->uring_lock);	init_waitqueue_head(&ctx->cq_wait);	spin_lock_init(&ctx->completion_lock);	spin_lock_init(&ctx->timeout_lock);	INIT_WQ_LIST(&ctx->iopoll_list);	INIT_LIST_HEAD(&ctx->io_buffers_pages);	INIT_LIST_HEAD(&ctx->io_buffers_comp);	INIT_LIST_HEAD(&ctx->defer_list);	INIT_LIST_HEAD(&ctx->timeout_list);	INIT_LIST_HEAD(&ctx->ltimeout_list);	spin_lock_init(&ctx->rsrc_ref_lock);	INIT_LIST_HEAD(&ctx->rsrc_ref_list);	INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work);	init_task_work(&ctx->rsrc_put_tw, io_rsrc_put_tw);	init_llist_head(&ctx->rsrc_put_llist);	init_llist_head(&ctx->work_llist);	INIT_LIST_HEAD(&ctx->tctx_list);	ctx->submit_state.free_list.next = NULL;	INIT_WQ_LIST(&ctx->locked_free_list);	INIT_DELAYED_WORK(&ctx->fallback_work, io_fallback_req_func);	INIT_WQ_LIST(&ctx->submit_state.compl_reqs);	return ctx;err:	kfree(ctx->dummy_ubuf);	kfree(ctx->cancel_table.hbs);	kfree(ctx->cancel_table_locked.hbs);	kfree(ctx->io_bl);	xa_destroy(&ctx->io_bl_xa);	kfree(ctx);	return NULL;} static void io_account_cq_overflow(struct io_ring_ctx *ctx){	struct io_rings *r = ctx->rings; 	WRITE_ONCE(r->cq_overflow, READ_ONCE(r->cq_overflow) + 1);	ctx->cq_extra--;} static bool req_need_defer(struct io_kiocb *req, u32 seq){	if (unlikely(req->flags & REQ_F_IO_DRAIN)) {		struct io_ring_ctx *ctx = req->ctx; 		return seq + READ_ONCE(ctx->cq_extra) != ctx->cached_cq_tail;	} 	return false;} static inline void io_req_track_inflight(struct io_kiocb *req){	if (!(req->flags & REQ_F_INFLIGHT)) {		req->flags |= REQ_F_INFLIGHT;		atomic_inc(&req->task->io_uring->inflight_tracked);	}} static struct io_kiocb *__io_prep_linked_timeout(struct io_kiocb *req){	if (WARN_ON_ONCE(!req->link))		return NULL; 	req->flags &= ~REQ_F_ARM_LTIMEOUT;	req->flags |= REQ_F_LINK_TIMEOUT; 	/* linked timeouts should have two refs once prep'ed */	io_req_set_refcount(req);	__io_req_set_refcount(req->link, 2);	return req->link;} static inline struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req){	if (likely(!(req->flags & REQ_F_ARM_LTIMEOUT)))		return NULL;	return __io_prep_linked_timeout(req);} static noinline void __io_arm_ltimeout(struct io_kiocb *req){	io_queue_linked_timeout(__io_prep_linked_timeout(req));} static inline void io_arm_ltimeout(struct io_kiocb *req){	if (unlikely(req->flags & REQ_F_ARM_LTIMEOUT))		__io_arm_ltimeout(req);} static void io_prep_async_work(struct io_kiocb *req){	const struct io_op_def *def = &io_op_defs[req->opcode];	struct io_ring_ctx *ctx = req->ctx; 	if (!(req->flags & REQ_F_CREDS)) {		req->flags |= REQ_F_CREDS;		req->creds = get_current_cred();	} 	req->work.list.next = NULL;	req->work.flags = 0;	req->work.cancel_seq = atomic_read(&ctx->cancel_seq);	if (req->flags & REQ_F_FORCE_ASYNC)		req->work.flags |= IO_WQ_WORK_CONCURRENT; 	if (req->file && !io_req_ffs_set(req))		req->flags |= io_file_get_flags(req->file) << REQ_F_SUPPORT_NOWAIT_BIT; 	if (req->flags & REQ_F_ISREG) {		if (def->hash_reg_file || (ctx->flags & IORING_SETUP_IOPOLL))			io_wq_hash_work(&req->work, file_inode(req->file));	} else if (!req->file || !S_ISBLK(file_inode(req->file)->i_mode)) {		if (def->unbound_nonreg_file)			req->work.flags |= IO_WQ_WORK_UNBOUND;	}} static void io_prep_async_link(struct io_kiocb *req){	struct io_kiocb *cur; 	if (req->flags & REQ_F_LINK_TIMEOUT) {		struct io_ring_ctx *ctx = req->ctx; 		spin_lock_irq(&ctx->timeout_lock);		io_for_each_link(cur, req)			io_prep_async_work(cur);		spin_unlock_irq(&ctx->timeout_lock);	} else {		io_for_each_link(cur, req)			io_prep_async_work(cur);	}} void io_queue_iowq(struct io_kiocb *req, bool *dont_use){	struct io_kiocb *link = io_prep_linked_timeout(req);	struct io_uring_task *tctx = req->task->io_uring; 	BUG_ON(!tctx);	BUG_ON(!tctx->io_wq); 	/* init ->work of the whole link before punting */	io_prep_async_link(req); 	/*	 * Not expected to happen, but if we do have a bug where this _can_	 * happen, catch it here and ensure the request is marked as	 * canceled. That will make io-wq go through the usual work cancel	 * procedure rather than attempt to run this request (or create a new	 * worker for it).	 */	if (WARN_ON_ONCE(!same_thread_group(req->task, current)))		req->work.flags |= IO_WQ_WORK_CANCEL; 	trace_io_uring_queue_async_work(req, io_wq_is_hashed(&req->work));	io_wq_enqueue(tctx->io_wq, &req->work);	if (link)		io_queue_linked_timeout(link);} static __cold void io_queue_deferred(struct io_ring_ctx *ctx){	while (!list_empty(&ctx->defer_list)) {		struct io_defer_entry *de = list_first_entry(&ctx->defer_list,						struct io_defer_entry, list); 		if (req_need_defer(de->req, de->seq))			break;		list_del_init(&de->list);		io_req_task_queue(de->req);		kfree(de);	}}  static void io_eventfd_ops(struct rcu_head *rcu){	struct io_ev_fd *ev_fd = container_of(rcu, struct io_ev_fd, rcu);	int ops = atomic_xchg(&ev_fd->ops, 0); 	if (ops & BIT(IO_EVENTFD_OP_SIGNAL_BIT))

Lines 1–500 of 4283. Load more