// SPDX-License-Identifier: GPL-2.0/* * Shared application/kernel submission and completion ring pairs, for * supporting fast/efficient IO. * * A note on the read/write ordering memory barriers that are matched between * the application and kernel side. * * After the application reads the CQ ring tail, it must use an * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses * before writing the tail (using smp_load_acquire to read the tail will * do). It also needs a smp_mb() before updating CQ head (ordering the * entry load(s) with the head store), pairing with an implicit barrier * through a control-dependency in io_get_cqring (smp_store_release to * store head will do). Failure to do so could lead to reading invalid * CQ entries. * * Likewise, the application must use an appropriate smp_wmb() before * writing the SQ tail (ordering SQ entry stores with the tail store), * which pairs with smp_load_acquire in io_get_sqring (smp_store_release * to store the tail will do). And it needs a barrier ordering the SQ * head load before writing new SQ entries (smp_load_acquire to read * head will do). * * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after* * updating the SQ tail; a full memory barrier smp_mb() is needed * between. * * Also see the examples in the liburing library: * *	git://git.kernel.dk/liburing * * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens * from data shared between the kernel and application. This is done both * for ordering purposes, but also to ensure that once a value is loaded from * data that the application could potentially modify, it remains stable. * * Copyright (C) 2018-2019 Jens Axboe * Copyright (c) 2018-2019 Christoph Hellwig */#include <linux/kernel.h>#include <linux/init.h>#include <linux/errno.h>#include <linux/syscalls.h>#include <linux/compat.h>#include <net/compat.h>#include <linux/refcount.h>#include <linux/uio.h>#include <linux/bits.h> #include <linux/sched/signal.h>#include <linux/fs.h>#include <linux/file.h>#include <linux/fdtable.h>#include <linux/mm.h>#include <linux/mman.h>#include <linux/percpu.h>#include <linux/slab.h>#include <linux/kthread.h>#include <linux/blkdev.h>#include <linux/bvec.h>#include <linux/net.h>#include <net/sock.h>#include <net/af_unix.h>#include <net/scm.h>#include <linux/anon_inodes.h>#include <linux/sched/mm.h>#include <linux/uaccess.h>#include <linux/nospec.h>#include <linux/sizes.h>#include <linux/hugetlb.h>#include <linux/highmem.h>#include <linux/namei.h>#include <linux/fsnotify.h>#include <linux/fadvise.h>#include <linux/eventpoll.h>#include <linux/fs_struct.h>#include <linux/splice.h>#include <linux/task_work.h>#include <linux/pagemap.h> #define CREATE_TRACE_POINTS#include <trace/events/io_uring.h> #include <uapi/linux/io_uring.h> #include "internal.h"#include "io-wq.h" #define IORING_MAX_ENTRIES	32768#define IORING_MAX_CQ_ENTRIES	(2 * IORING_MAX_ENTRIES) /* * Shift of 9 is 512 entries, or exactly one page on 64-bit archs */#define IORING_FILE_TABLE_SHIFT	9#define IORING_MAX_FILES_TABLE	(1U << IORING_FILE_TABLE_SHIFT)#define IORING_FILE_TABLE_MASK	(IORING_MAX_FILES_TABLE - 1)#define IORING_MAX_FIXED_FILES	(64 * IORING_MAX_FILES_TABLE) struct io_uring {	u32 head ____cacheline_aligned_in_smp;	u32 tail ____cacheline_aligned_in_smp;}; /* * This data is shared with the application through the mmap at offsets * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING. * * The offsets to the member fields are published through struct * io_sqring_offsets when calling io_uring_setup. */struct io_rings {	/*	 * Head and tail offsets into the ring; the offsets need to be	 * masked to get valid indices.	 *	 * The kernel controls head of the sq ring and the tail of the cq ring,	 * and the application controls tail of the sq ring and the head of the	 * cq ring.	 */	struct io_uring		sq, cq;	/*	 * Bitmasks to apply to head and tail offsets (constant, equals	 * ring_entries - 1)	 */	u32			sq_ring_mask, cq_ring_mask;	/* Ring sizes (constant, power of 2) */	u32			sq_ring_entries, cq_ring_entries;	/*	 * Number of invalid entries dropped by the kernel due to	 * invalid index stored in array	 *	 * Written by the kernel, shouldn't be modified by the	 * application (i.e. get number of "new events" by comparing to	 * cached value).	 *	 * After a new SQ head value was read by the application this	 * counter includes all submissions that were dropped reaching	 * the new SQ head (and possibly more).	 */	u32			sq_dropped;	/*	 * Runtime SQ flags	 *	 * Written by the kernel, shouldn't be modified by the	 * application.	 *	 * The application needs a full memory barrier before checking	 * for IORING_SQ_NEED_WAKEUP after updating the sq tail.	 */	u32			sq_flags;	/*	 * Runtime CQ flags	 *	 * Written by the application, shouldn't be modified by the	 * kernel.	 */	u32                     cq_flags;	/*	 * Number of completion events lost because the queue was full;	 * this should be avoided by the application by making sure	 * there are not more requests pending than there is space in	 * the completion queue.	 *	 * Written by the kernel, shouldn't be modified by the	 * application (i.e. get number of "new events" by comparing to	 * cached value).	 *	 * As completion events come in out of order this counter is not	 * ordered with any other data.	 */	u32			cq_overflow;	/*	 * Ring buffer of completion events.	 *	 * The kernel writes completion events fresh every time they are	 * produced, so the application is allowed to modify pending	 * entries.	 */	struct io_uring_cqe	cqes[] ____cacheline_aligned_in_smp;}; struct io_mapped_ubuf {	u64		ubuf;	size_t		len;	struct		bio_vec *bvec;	unsigned int	nr_bvecs;}; struct fixed_file_table {	struct file		**files;}; struct fixed_file_ref_node {	struct percpu_ref		refs;	struct list_head		node;	struct list_head		file_list;	struct fixed_file_data		*file_data;	struct llist_node		llist;}; struct fixed_file_data {	struct fixed_file_table		*table;	struct io_ring_ctx		*ctx; 	struct percpu_ref		*cur_refs;	struct percpu_ref		refs;	struct completion		done;	struct list_head		ref_list;	spinlock_t			lock;}; struct io_buffer {	struct list_head list;	__u64 addr;	__s32 len;	__u16 bid;}; struct io_ring_ctx {	struct {		struct percpu_ref	refs;	} ____cacheline_aligned_in_smp; 	struct {		unsigned int		flags;		unsigned int		compat: 1;		unsigned int		limit_mem: 1;		unsigned int		cq_overflow_flushed: 1;		unsigned int		drain_next: 1;		unsigned int		eventfd_async: 1; 		/*		 * Ring buffer of indices into array of io_uring_sqe, which is		 * mmapped by the application using the IORING_OFF_SQES offset.		 *		 * This indirection could e.g. be used to assign fixed		 * io_uring_sqe entries to operations and only submit them to		 * the queue when needed.		 *		 * The kernel modifies neither the indices array nor the entries		 * array.		 */		u32			*sq_array;		unsigned		cached_sq_head;		unsigned		sq_entries;		unsigned		sq_mask;		unsigned		sq_thread_idle;		unsigned		cached_sq_dropped;		atomic_t		cached_cq_overflow;		unsigned long		sq_check_overflow; 		struct list_head	defer_list;		struct list_head	timeout_list;		struct list_head	cq_overflow_list; 		wait_queue_head_t	inflight_wait;		struct io_uring_sqe	*sq_sqes;	} ____cacheline_aligned_in_smp; 	struct io_rings	*rings; 	/* IO offload */	struct io_wq		*io_wq;	struct task_struct	*sqo_thread;	/* if using sq thread polling */	struct mm_struct	*sqo_mm;	wait_queue_head_t	sqo_wait; 	/*	 * If used, fixed file set. Writers must ensure that ->refs is dead,	 * readers must ensure that ->refs is alive as long as the file* is	 * used. Only updated through io_uring_register(2).	 */	struct fixed_file_data	*file_data;	unsigned		nr_user_files;	int 			ring_fd;	struct file 		*ring_file; 	/* if used, fixed mapped user buffers */	unsigned		nr_user_bufs;	struct io_mapped_ubuf	*user_bufs; 	struct user_struct	*user; 	const struct cred	*creds; 	struct completion	ref_comp;	struct completion	sq_thread_comp; 	/* if all else fails... */	struct io_kiocb		*fallback_req; #if defined(CONFIG_UNIX)	struct socket		*ring_sock;#endif 	struct idr		io_buffer_idr; 	struct idr		personality_idr; 	struct {		unsigned		cached_cq_tail;		unsigned		cq_entries;		unsigned		cq_mask;		atomic_t		cq_timeouts;		unsigned long		cq_check_overflow;		struct wait_queue_head	cq_wait;		struct fasync_struct	*cq_fasync;		struct eventfd_ctx	*cq_ev_fd;	} ____cacheline_aligned_in_smp; 	struct {		struct mutex		uring_lock;		wait_queue_head_t	wait;	} ____cacheline_aligned_in_smp; 	struct {		spinlock_t		completion_lock; 		/*		 * ->iopoll_list is protected by the ctx->uring_lock for		 * io_uring instances that don't use IORING_SETUP_SQPOLL.		 * For SQPOLL, only the single threaded io_sq_thread() will		 * manipulate the list, hence no extra locking is needed there.		 */		struct list_head	iopoll_list;		struct hlist_head	*cancel_hash;		unsigned		cancel_hash_bits;		bool			poll_multi_file; 		spinlock_t		inflight_lock;		struct list_head	inflight_list;	} ____cacheline_aligned_in_smp; 	struct delayed_work		file_put_work;	struct llist_head		file_put_llist; 	struct work_struct		exit_work;}; /* * First field must be the file pointer in all the * iocb unions! See also 'struct kiocb' in <linux/fs.h> */struct io_poll_iocb {	struct file			*file;	union {		struct wait_queue_head	*head;		u64			addr;	};	__poll_t			events;	bool				done;	bool				canceled;	struct wait_queue_entry		wait;}; struct io_close {	struct file			*file;	struct file			*put_file;	int				fd;}; struct io_timeout_data {	struct io_kiocb			*req;	struct hrtimer			timer;	struct timespec64		ts;	enum hrtimer_mode		mode;}; struct io_accept {	struct file			*file;	struct sockaddr __user		*addr;	int __user			*addr_len;	int				flags;	unsigned long			nofile;}; struct io_sync {	struct file			*file;	loff_t				len;	loff_t				off;	int				flags;	int				mode;}; struct io_cancel {	struct file			*file;	u64				addr;}; struct io_timeout {	struct file			*file;	u64				addr;	int				flags;	u32				off;	u32				target_seq;	struct list_head		list;}; struct io_rw {	/* NOTE: kiocb has the file as the first member, so don't do it here */	struct kiocb			kiocb;	u64				addr;	u64				len;}; struct io_connect {	struct file			*file;	struct sockaddr __user		*addr;	int				addr_len;}; struct io_sr_msg {	struct file			*file;	union {		struct user_msghdr __user *umsg;		void __user		*buf;	};	int				msg_flags;	int				bgid;	size_t				len;	struct io_buffer		*kbuf;}; struct io_open {	struct file			*file;	int				dfd;	struct filename			*filename;	struct open_how			how;	unsigned long			nofile;}; struct io_files_update {	struct file			*file;	u64				arg;	u32				nr_args;	u32				offset;}; struct io_fadvise {	struct file			*file;	u64				offset;	u32				len;	u32				advice;}; struct io_madvise {	struct file			*file;	u64				addr;	u32				len;	u32				advice;}; struct io_epoll {	struct file			*file;	int				epfd;	int				op;	int				fd;	struct epoll_event		event;}; struct io_splice {	struct file			*file_out;	struct file			*file_in;	loff_t				off_out;	loff_t				off_in;	u64				len;	unsigned int			flags;}; struct io_provide_buf {	struct file			*file;	__u64				addr;	__s32				len;	__u32				bgid;	__u16				nbufs;	__u16				bid;}; struct io_statx {	struct file			*file;	int				dfd;	unsigned int			mask;	unsigned int			flags;	const char __user		*filename;	struct statx __user		*buffer;}; struct io_completion {	struct file			*file;	struct list_head		list;	int				cflags;}; struct io_async_connect {	struct sockaddr_storage		address;}; 

Lines 1–500 of 8901. Load more