// SPDX-License-Identifier: GPL-2.0/* * Shared application/kernel submission and completion ring pairs, for * supporting fast/efficient IO. * * A note on the read/write ordering memory barriers that are matched between * the application and kernel side. * * After the application reads the CQ ring tail, it must use an * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses * before writing the tail (using smp_load_acquire to read the tail will * do). It also needs a smp_mb() before updating CQ head (ordering the * entry load(s) with the head store), pairing with an implicit barrier * through a control-dependency in io_get_cqring (smp_store_release to * store head will do). Failure to do so could lead to reading invalid * CQ entries. * * Likewise, the application must use an appropriate smp_wmb() before * writing the SQ tail (ordering SQ entry stores with the tail store), * which pairs with smp_load_acquire in io_get_sqring (smp_store_release * to store the tail will do). And it needs a barrier ordering the SQ * head load before writing new SQ entries (smp_load_acquire to read * head will do). * * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after* * updating the SQ tail; a full memory barrier smp_mb() is needed * between. * * Also see the examples in the liburing library: * * git://git.kernel.dk/liburing * * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens * from data shared between the kernel and application. This is done both * for ordering purposes, but also to ensure that once a value is loaded from * data that the application could potentially modify, it remains stable. * * Copyright (C) 2018-2019 Jens Axboe * Copyright (c) 2018-2019 Christoph Hellwig */#include <linux/kernel.h>#include <linux/init.h>#include <linux/errno.h>#include <linux/syscalls.h>#include <linux/compat.h>#include <net/compat.h>#include <linux/refcount.h>#include <linux/uio.h>#include <linux/bits.h> #include <linux/sched/signal.h>#include <linux/fs.h>#include <linux/file.h>#include <linux/fdtable.h>#include <linux/mm.h>#include <linux/mman.h>#include <linux/percpu.h>#include <linux/slab.h>#include <linux/blkdev.h>#include <linux/bvec.h>#include <linux/net.h>#include <net/sock.h>#include <net/af_unix.h>#include <net/scm.h>#include <linux/anon_inodes.h>#include <linux/sched/mm.h>#include <linux/uaccess.h>#include <linux/nospec.h>#include <linux/sizes.h>#include <linux/hugetlb.h>#include <linux/highmem.h>#include <linux/namei.h>#include <linux/fsnotify.h>#include <linux/fadvise.h>#include <linux/eventpoll.h>#include <linux/splice.h>#include <linux/task_work.h>#include <linux/pagemap.h>#include <linux/io_uring.h> #define CREATE_TRACE_POINTS#include <trace/events/io_uring.h> #include <uapi/linux/io_uring.h> #include "internal.h"#include "io-wq.h" #define IORING_MAX_ENTRIES 32768#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES) /* * Shift of 9 is 512 entries, or exactly one page on 64-bit archs */#define IORING_FILE_TABLE_SHIFT 9#define IORING_MAX_FILES_TABLE (1U << IORING_FILE_TABLE_SHIFT)#define IORING_FILE_TABLE_MASK (IORING_MAX_FILES_TABLE - 1)#define IORING_MAX_FIXED_FILES (64 * IORING_MAX_FILES_TABLE)#define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \ IORING_REGISTER_LAST + IORING_OP_LAST) #define IORING_MAX_REG_BUFFERS (1U << 14) #define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK| \ IOSQE_IO_HARDLINK | IOSQE_ASYNC | \ IOSQE_BUFFER_SELECT) struct io_uring { u32 head ____cacheline_aligned_in_smp; u32 tail ____cacheline_aligned_in_smp;}; /* * This data is shared with the application through the mmap at offsets * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING. * * The offsets to the member fields are published through struct * io_sqring_offsets when calling io_uring_setup. */struct io_rings { /* * Head and tail offsets into the ring; the offsets need to be * masked to get valid indices. * * The kernel controls head of the sq ring and the tail of the cq ring, * and the application controls tail of the sq ring and the head of the * cq ring. */ struct io_uring sq, cq; /* * Bitmasks to apply to head and tail offsets (constant, equals * ring_entries - 1) */ u32 sq_ring_mask, cq_ring_mask; /* Ring sizes (constant, power of 2) */ u32 sq_ring_entries, cq_ring_entries; /* * Number of invalid entries dropped by the kernel due to * invalid index stored in array * * Written by the kernel, shouldn't be modified by the * application (i.e. get number of "new events" by comparing to * cached value). * * After a new SQ head value was read by the application this * counter includes all submissions that were dropped reaching * the new SQ head (and possibly more). */ u32 sq_dropped; /* * Runtime SQ flags * * Written by the kernel, shouldn't be modified by the * application. * * The application needs a full memory barrier before checking * for IORING_SQ_NEED_WAKEUP after updating the sq tail. */ u32 sq_flags; /* * Runtime CQ flags * * Written by the application, shouldn't be modified by the * kernel. */ u32 cq_flags; /* * Number of completion events lost because the queue was full; * this should be avoided by the application by making sure * there are not more requests pending than there is space in * the completion queue. * * Written by the kernel, shouldn't be modified by the * application (i.e. get number of "new events" by comparing to * cached value). * * As completion events come in out of order this counter is not * ordered with any other data. */ u32 cq_overflow; /* * Ring buffer of completion events. * * The kernel writes completion events fresh every time they are * produced, so the application is allowed to modify pending * entries. */ struct io_uring_cqe cqes[] ____cacheline_aligned_in_smp;}; enum io_uring_cmd_flags { IO_URING_F_NONBLOCK = 1, IO_URING_F_COMPLETE_DEFER = 2,}; struct io_mapped_ubuf { u64 ubuf; u64 ubuf_end; unsigned int nr_bvecs; unsigned long acct_pages; struct bio_vec bvec[];}; struct io_ring_ctx; struct io_overflow_cqe { struct io_uring_cqe cqe; struct list_head list;}; struct io_fixed_file { /* file * with additional FFS_* flags */ unsigned long file_ptr;}; struct io_rsrc_put { struct list_head list; u64 tag; union { void *rsrc; struct file *file; struct io_mapped_ubuf *buf; };}; struct io_file_table { /* two level table */ struct io_fixed_file **files;}; struct io_rsrc_node { struct percpu_ref refs; struct list_head node; struct list_head rsrc_list; struct io_rsrc_data *rsrc_data; struct llist_node llist; bool done;}; typedef void (rsrc_put_fn)(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc); struct io_rsrc_data { struct io_ring_ctx *ctx; u64 *tags; rsrc_put_fn *do_put; atomic_t refs; struct completion done; bool quiesce;}; struct io_buffer { struct list_head list; __u64 addr; __u32 len; __u16 bid;}; struct io_restriction { DECLARE_BITMAP(register_op, IORING_REGISTER_LAST); DECLARE_BITMAP(sqe_op, IORING_OP_LAST); u8 sqe_flags_allowed; u8 sqe_flags_required; bool registered;}; enum { IO_SQ_THREAD_SHOULD_STOP = 0, IO_SQ_THREAD_SHOULD_PARK,}; struct io_sq_data { refcount_t refs; atomic_t park_pending; struct mutex lock; /* ctx's that are using this sqd */ struct list_head ctx_list; struct task_struct *thread; struct wait_queue_head wait; unsigned sq_thread_idle; int sq_cpu; pid_t task_pid; pid_t task_tgid; unsigned long state; struct completion exited; struct callback_head *park_task_work;}; #define IO_IOPOLL_BATCH 8#define IO_COMPL_BATCH 32#define IO_REQ_CACHE_SIZE 32#define IO_REQ_ALLOC_BATCH 8 struct io_comp_state { struct io_kiocb *reqs[IO_COMPL_BATCH]; unsigned int nr; unsigned int locked_free_nr; /* inline/task_work completion list, under ->uring_lock */ struct list_head free_list; /* IRQ completion list, under ->completion_lock */ struct list_head locked_free_list;}; struct io_submit_link { struct io_kiocb *head; struct io_kiocb *last;}; struct io_submit_state { struct blk_plug plug; struct io_submit_link link; /* * io_kiocb alloc cache */ void *reqs[IO_REQ_CACHE_SIZE]; unsigned int free_reqs; bool plug_started; /* * Batch completion logic */ struct io_comp_state comp; /* * File reference cache */ struct file *file; unsigned int fd; unsigned int file_refs; unsigned int ios_left;}; struct io_ring_ctx { struct { struct percpu_ref refs; } ____cacheline_aligned_in_smp; struct { unsigned int flags; unsigned int compat: 1; unsigned int drain_next: 1; unsigned int eventfd_async: 1; unsigned int restricted: 1; /* * Ring buffer of indices into array of io_uring_sqe, which is * mmapped by the application using the IORING_OFF_SQES offset. * * This indirection could e.g. be used to assign fixed * io_uring_sqe entries to operations and only submit them to * the queue when needed. * * The kernel modifies neither the indices array nor the entries * array. */ u32 *sq_array; unsigned cached_sq_head; unsigned sq_entries; unsigned sq_mask; unsigned sq_thread_idle; unsigned cached_sq_dropped; unsigned cached_cq_overflow; unsigned long sq_check_overflow; /* hashed buffered write serialization */ struct io_wq_hash *hash_map; struct list_head defer_list; struct list_head timeout_list; struct list_head cq_overflow_list; struct io_uring_sqe *sq_sqes; } ____cacheline_aligned_in_smp; struct { struct mutex uring_lock; wait_queue_head_t wait; } ____cacheline_aligned_in_smp; struct io_submit_state submit_state; struct io_rings *rings; /* Only used for accounting purposes */ struct mm_struct *mm_account; const struct cred *sq_creds; /* cred used for __io_sq_thread() */ struct io_sq_data *sq_data; /* if using sq thread polling */ struct wait_queue_head sqo_sq_wait; struct list_head sqd_list; /* * If used, fixed file set. Writers must ensure that ->refs is dead, * readers must ensure that ->refs is alive as long as the file* is * used. Only updated through io_uring_register(2). */ struct io_rsrc_data *file_data; struct io_file_table file_table; unsigned nr_user_files; /* if used, fixed mapped user buffers */ struct io_rsrc_data *buf_data; unsigned nr_user_bufs; struct io_mapped_ubuf **user_bufs; struct user_struct *user; struct completion ref_comp; #if defined(CONFIG_UNIX) struct socket *ring_sock;#endif struct xarray io_buffers; struct xarray personalities; u32 pers_next; struct { unsigned cached_cq_tail; unsigned cq_entries; unsigned cq_mask; atomic_t cq_timeouts; unsigned cq_last_tm_flush; unsigned cq_extra; unsigned long cq_check_overflow; struct wait_queue_head cq_wait; struct fasync_struct *cq_fasync; struct eventfd_ctx *cq_ev_fd; } ____cacheline_aligned_in_smp; struct { spinlock_t completion_lock; /* * ->iopoll_list is protected by the ctx->uring_lock for * io_uring instances that don't use IORING_SETUP_SQPOLL. * For SQPOLL, only the single threaded io_sq_thread() will * manipulate the list, hence no extra locking is needed there. */ struct list_head iopoll_list; struct hlist_head *cancel_hash; unsigned cancel_hash_bits; bool poll_multi_file; } ____cacheline_aligned_in_smp; struct delayed_work rsrc_put_work; struct llist_head rsrc_put_llist; struct list_head rsrc_ref_list; spinlock_t rsrc_ref_lock; struct io_rsrc_node *rsrc_node; struct io_rsrc_node *rsrc_backup_node; struct io_mapped_ubuf *dummy_ubuf; struct io_restriction restrictions; /* exit task_work */ struct callback_head *exit_task_work; /* Keep this last, we don't need it for the fast path */ struct work_struct exit_work; struct list_head tctx_list;}; struct io_uring_task { /* submission side */ struct xarray xa; struct wait_queue_head wait; const struct io_ring_ctx *last; struct io_wq *io_wq; struct percpu_counter inflight; atomic_t inflight_tracked; atomic_t in_idle; spinlock_t task_lock; struct io_wq_work_list task_list; unsigned long task_state; struct callback_head task_work;}; /* * First field must be the file pointer in all the * iocb unions! See also 'struct kiocb' in <linux/fs.h> */struct io_poll_iocb { struct file *file; struct wait_queue_head *head; __poll_t events; bool done; bool canceled; struct wait_queue_entry wait;};Lines 1–500 of 10207. Load more
// SPDX-License-Identifier: GPL-2.0 /* * Shared application/kernel submission and completion ring pairs, for * supporting fast/efficient IO. * * A note on the read/write ordering memory barriers that are matched between * the application and kernel side. * * After the application reads the CQ ring tail, it must use an * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses * before writing the tail (using smp_load_acquire to read the tail will * do). It also needs a smp_mb() before updating CQ head (ordering the * entry load(s) with the head store), pairing with an implicit barrier * through a control-dependency in io_get_cqring (smp_store_release to * store head will do). Failure to do so could lead to reading invalid * CQ entries. * * Likewise, the application must use an appropriate smp_wmb() before * writing the SQ tail (ordering SQ entry stores with the tail store), * which pairs with smp_load_acquire in io_get_sqring (smp_store_release * to store the tail will do). And it needs a barrier ordering the SQ * head load before writing new SQ entries (smp_load_acquire to read * head will do). * * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after* * updating the SQ tail; a full memory barrier smp_mb() is needed * between. * * Also see the examples in the liburing library: * * git://git.kernel.dk/liburing * * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens * from data shared between the kernel and application. This is done both * for ordering purposes, but also to ensure that once a value is loaded from * data that the application could potentially modify, it remains stable. * * Copyright (C) 2018-2019 Jens Axboe * Copyright (c) 2018-2019 Christoph Hellwig */ #include <linux/kernel.h> #include <linux/init.h> #include <linux/errno.h> #include <linux/syscalls.h> #include <linux/compat.h> #include <net/compat.h> #include <linux/refcount.h> #include <linux/uio.h> #include <linux/bits.h> #include <linux/sched/signal.h> #include <linux/fs.h> #include <linux/file.h> #include <linux/fdtable.h> #include <linux/mm.h> #include <linux/mman.h> #include <linux/percpu.h> #include <linux/slab.h> #include <linux/blkdev.h> #include <linux/bvec.h> #include <linux/net.h> #include <net/sock.h> #include <net/af_unix.h> #include <net/scm.h> #include <linux/anon_inodes.h> #include <linux/sched/mm.h> #include <linux/uaccess.h> #include <linux/nospec.h> #include <linux/sizes.h> #include <linux/hugetlb.h> #include <linux/highmem.h> #include <linux/namei.h> #include <linux/fsnotify.h> #include <linux/fadvise.h> #include <linux/eventpoll.h> #include <linux/splice.h> #include <linux/task_work.h> #include <linux/pagemap.h> #include <linux/io_uring.h> #define CREATE_TRACE_POINTS #include <trace/events/io_uring.h> #include <uapi/linux/io_uring.h> #include "internal.h" #include "io-wq.h" #define IORING_MAX_ENTRIES 32768 #define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES) /* * Shift of 9 is 512 entries, or exactly one page on 64-bit archs */ #define IORING_FILE_TABLE_SHIFT 9 #define IORING_MAX_FILES_TABLE (1U << IORING_FILE_TABLE_SHIFT) #define IORING_FILE_TABLE_MASK (IORING_MAX_FILES_TABLE - 1) #define IORING_MAX_FIXED_FILES (64 * IORING_MAX_FILES_TABLE) #define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \ IORING_REGISTER_LAST + IORING_OP_LAST) #define IORING_MAX_REG_BUFFERS (1U << 14) #define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK| \ IOSQE_IO_HARDLINK | IOSQE_ASYNC | \ IOSQE_BUFFER_SELECT) struct io_uring { u32 head ____cacheline_aligned_in_smp; u32 tail ____cacheline_aligned_in_smp; }; /* * This data is shared with the application through the mmap at offsets * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING. * * The offsets to the member fields are published through struct * io_sqring_offsets when calling io_uring_setup. */ struct io_rings { /* * Head and tail offsets into the ring; the offsets need to be * masked to get valid indices. * * The kernel controls head of the sq ring and the tail of the cq ring, * and the application controls tail of the sq ring and the head of the * cq ring. */ struct io_uring sq, cq; /* * Bitmasks to apply to head and tail offsets (constant, equals * ring_entries - 1) */ u32 sq_ring_mask, cq_ring_mask; /* Ring sizes (constant, power of 2) */ u32 sq_ring_entries, cq_ring_entries; /* * Number of invalid entries dropped by the kernel due to * invalid index stored in array * * Written by the kernel, shouldn't be modified by the * application (i.e. get number of "new events" by comparing to * cached value). * * After a new SQ head value was read by the application this * counter includes all submissions that were dropped reaching * the new SQ head (and possibly more). */ u32 sq_dropped; /* * Runtime SQ flags * * Written by the kernel, shouldn't be modified by the * application. * * The application needs a full memory barrier before checking * for IORING_SQ_NEED_WAKEUP after updating the sq tail. */ u32 sq_flags; /* * Runtime CQ flags * * Written by the application, shouldn't be modified by the * kernel. */ u32 cq_flags; /* * Number of completion events lost because the queue was full; * this should be avoided by the application by making sure * there are not more requests pending than there is space in * the completion queue. * * Written by the kernel, shouldn't be modified by the * application (i.e. get number of "new events" by comparing to * cached value). * * As completion events come in out of order this counter is not * ordered with any other data. */ u32 cq_overflow; /* * Ring buffer of completion events. * * The kernel writes completion events fresh every time they are * produced, so the application is allowed to modify pending * entries. */ struct io_uring_cqe cqes[] ____cacheline_aligned_in_smp; }; enum io_uring_cmd_flags { IO_URING_F_NONBLOCK = 1, IO_URING_F_COMPLETE_DEFER = 2, }; struct io_mapped_ubuf { u64 ubuf; u64 ubuf_end; unsigned int nr_bvecs; unsigned long acct_pages; struct bio_vec bvec[]; }; struct io_ring_ctx; struct io_overflow_cqe { struct io_uring_cqe cqe; struct list_head list; }; struct io_fixed_file { /* file * with additional FFS_* flags */ unsigned long file_ptr; }; struct io_rsrc_put { struct list_head list; u64 tag; union { void *rsrc; struct file *file; struct io_mapped_ubuf *buf; }; }; struct io_file_table { /* two level table */ struct io_fixed_file **files; }; struct io_rsrc_node { struct percpu_ref refs; struct list_head node; struct list_head rsrc_list; struct io_rsrc_data *rsrc_data; struct llist_node llist; bool done; }; typedef void (rsrc_put_fn)(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc); struct io_rsrc_data { struct io_ring_ctx *ctx; u64 *tags; rsrc_put_fn *do_put; atomic_t refs; struct completion done; bool quiesce; }; struct io_buffer { struct list_head list; __u64 addr; __u32 len; __u16 bid; }; struct io_restriction { DECLARE_BITMAP(register_op, IORING_REGISTER_LAST); DECLARE_BITMAP(sqe_op, IORING_OP_LAST); u8 sqe_flags_allowed; u8 sqe_flags_required; bool registered; }; enum { IO_SQ_THREAD_SHOULD_STOP = 0, IO_SQ_THREAD_SHOULD_PARK, }; struct io_sq_data { refcount_t refs; atomic_t park_pending; struct mutex lock; /* ctx's that are using this sqd */ struct list_head ctx_list; struct task_struct *thread; struct wait_queue_head wait; unsigned sq_thread_idle; int sq_cpu; pid_t task_pid; pid_t task_tgid; unsigned long state; struct completion exited; struct callback_head *park_task_work; }; #define IO_IOPOLL_BATCH 8 #define IO_COMPL_BATCH 32 #define IO_REQ_CACHE_SIZE 32 #define IO_REQ_ALLOC_BATCH 8 struct io_comp_state { struct io_kiocb *reqs[IO_COMPL_BATCH]; unsigned int nr; unsigned int locked_free_nr; /* inline/task_work completion list, under ->uring_lock */ struct list_head free_list; /* IRQ completion list, under ->completion_lock */ struct list_head locked_free_list; }; struct io_submit_link { struct io_kiocb *head; struct io_kiocb *last; }; struct io_submit_state { struct blk_plug plug; struct io_submit_link link; /* * io_kiocb alloc cache */ void *reqs[IO_REQ_CACHE_SIZE]; unsigned int free_reqs; bool plug_started; /* * Batch completion logic */ struct io_comp_state comp; /* * File reference cache */ struct file *file; unsigned int fd; unsigned int file_refs; unsigned int ios_left; }; struct io_ring_ctx { struct { struct percpu_ref refs; } ____cacheline_aligned_in_smp; struct { unsigned int flags; unsigned int compat: 1; unsigned int drain_next: 1; unsigned int eventfd_async: 1; unsigned int restricted: 1; /* * Ring buffer of indices into array of io_uring_sqe, which is * mmapped by the application using the IORING_OFF_SQES offset. * * This indirection could e.g. be used to assign fixed * io_uring_sqe entries to operations and only submit them to * the queue when needed. * * The kernel modifies neither the indices array nor the entries * array. */ u32 *sq_array; unsigned cached_sq_head; unsigned sq_entries; unsigned sq_mask; unsigned sq_thread_idle; unsigned cached_sq_dropped; unsigned cached_cq_overflow; unsigned long sq_check_overflow; /* hashed buffered write serialization */ struct io_wq_hash *hash_map; struct list_head defer_list; struct list_head timeout_list; struct list_head cq_overflow_list; struct io_uring_sqe *sq_sqes; } ____cacheline_aligned_in_smp; struct { struct mutex uring_lock; wait_queue_head_t wait; } ____cacheline_aligned_in_smp; struct io_submit_state submit_state; struct io_rings *rings; /* Only used for accounting purposes */ struct mm_struct *mm_account; const struct cred *sq_creds; /* cred used for __io_sq_thread() */ struct io_sq_data *sq_data; /* if using sq thread polling */ struct wait_queue_head sqo_sq_wait; struct list_head sqd_list; /* * If used, fixed file set. Writers must ensure that ->refs is dead, * readers must ensure that ->refs is alive as long as the file* is * used. Only updated through io_uring_register(2). */ struct io_rsrc_data *file_data; struct io_file_table file_table; unsigned nr_user_files; /* if used, fixed mapped user buffers */ struct io_rsrc_data *buf_data; unsigned nr_user_bufs; struct io_mapped_ubuf **user_bufs; struct user_struct *user; struct completion ref_comp; #if defined(CONFIG_UNIX) struct socket *ring_sock; #endif struct xarray io_buffers; struct xarray personalities; u32 pers_next; struct { unsigned cached_cq_tail; unsigned cq_entries; unsigned cq_mask; atomic_t cq_timeouts; unsigned cq_last_tm_flush; unsigned cq_extra; unsigned long cq_check_overflow; struct wait_queue_head cq_wait; struct fasync_struct *cq_fasync; struct eventfd_ctx *cq_ev_fd; } ____cacheline_aligned_in_smp; struct { spinlock_t completion_lock; /* * ->iopoll_list is protected by the ctx->uring_lock for * io_uring instances that don't use IORING_SETUP_SQPOLL. * For SQPOLL, only the single threaded io_sq_thread() will * manipulate the list, hence no extra locking is needed there. */ struct list_head iopoll_list; struct hlist_head *cancel_hash; unsigned cancel_hash_bits; bool poll_multi_file; } ____cacheline_aligned_in_smp; struct delayed_work rsrc_put_work; struct llist_head rsrc_put_llist; struct list_head rsrc_ref_list; spinlock_t rsrc_ref_lock; struct io_rsrc_node *rsrc_node; struct io_rsrc_node *rsrc_backup_node; struct io_mapped_ubuf *dummy_ubuf; struct io_restriction restrictions; /* exit task_work */ struct callback_head *exit_task_work; /* Keep this last, we don't need it for the fast path */ struct work_struct exit_work; struct list_head tctx_list; }; struct io_uring_task { /* submission side */ struct xarray xa; struct wait_queue_head wait; const struct io_ring_ctx *last; struct io_wq *io_wq; struct percpu_counter inflight; atomic_t inflight_tracked; atomic_t in_idle; spinlock_t task_lock; struct io_wq_work_list task_list; unsigned long task_state; struct callback_head task_work; }; /* * First field must be the file pointer in all the * iocb unions! See also 'struct kiocb' in <linux/fs.h> */ struct io_poll_iocb { struct file *file; struct wait_queue_head *head; __poll_t events; bool done; bool canceled; struct wait_queue_entry wait; };