// SPDX-License-Identifier: GPL-2.0-or-later/* * fs/eventpoll.c (Efficient event retrieval implementation) * Copyright (C) 2001,...,2009 Davide Libenzi * * Davide Libenzi <davidel@xmailserver.org> */ #include <linux/init.h>#include <linux/kernel.h>#include <linux/sched/signal.h>#include <linux/fs.h>#include <linux/file.h>#include <linux/signal.h>#include <linux/errno.h>#include <linux/mm.h>#include <linux/slab.h>#include <linux/poll.h>#include <linux/string.h>#include <linux/list.h>#include <linux/hash.h>#include <linux/spinlock.h>#include <linux/syscalls.h>#include <linux/rbtree.h>#include <linux/wait.h>#include <linux/eventpoll.h>#include <linux/mount.h>#include <linux/bitops.h>#include <linux/mutex.h>#include <linux/anon_inodes.h>#include <linux/device.h>#include <linux/uaccess.h>#include <asm/io.h>#include <asm/mman.h>#include <linux/atomic.h>#include <linux/proc_fs.h>#include <linux/seq_file.h>#include <linux/compat.h>#include <linux/rculist.h>#include <linux/capability.h>#include <linux/seqlock.h>#include <net/busy_poll.h> /* * fs/eventpoll.c - Efficient event polling ("epoll") kernel implementation. * * * Overview * -------- * * Each epoll_create(2) returns an anonymous [eventpoll] file whose * ->private_data is a struct eventpoll. Each EPOLL_CTL_ADD installs * a struct epitem linking one (watched file, fd) pair back to that * eventpoll via the watched file's f_op->poll() wait queue(s). When * the watched file signals readiness, ep_poll_callback() fires and * marks the epitem ready. epoll_wait(2) drains the ready list under * ep->mtx, re-queueing items in level-triggered mode. * * epoll instances can watch other epoll instances up to EP_MAX_NESTS * deep; cycles are forbidden and detected at EPOLL_CTL_ADD time. * * * Locking * ------- * * Three levels, acquired from outer to inner: * * epnested_mutex (global; rare; taken only for EPOLL_CTL_ADD * loop / path checks) * > ep->mtx (per-eventpoll; sleepable; serializes most ops) * > ep->lock (per-eventpoll; IRQ-safe spinlock) * * file->f_lock (per-file; NOT IRQ-safe; guards f_ep hlist ops; * nested inside ep->mtx, outside ep->lock) * * Rationale: * - ep->lock is a spinlock because ep_poll_callback() is called from * wake_up() which may run in hard-IRQ context. All ep->lock * critical sections use spin_lock_irqsave(). * - ep->mtx is a sleepable mutex because the event delivery loop * calls copy_to_user(), and ep_insert() may sleep in * kmem_cache_alloc() and f_op->poll(). * - epnested_mutex is global because cycle detection needs a global * view of the epoll topology; a per-object scheme would let two * concurrent inserts (A into B, B into A) construct a cycle * without either observer seeing it. * - Per-ep ep->mtx is preferred for scalability elsewhere. Events * that require epnested_mutex are rare. * * When EPOLL_CTL_ADD nests one eventpoll inside another we acquire * ep->mtx on both: outer first, target second. Since cycles are * forbidden the set of live ep->mtx holds is always a strict chain, * communicated to lockdep via mutex_lock_nested() subclasses derived * from the current recursion depth. * * * Field protection * ---------------- * * struct eventpoll: * mtx - self * rbr - ep->mtx * ovflist, rdllist - ep->lock (IRQ-safe) * wq - ep->lock for queue mutation * poll_wait - internal waitqueue spinlock * refs - file->f_lock for adds; ep->mtx for removes; * RCU for readers (hlist_del_rcu + kfree_rcu(ep)) * ws - ep->mtx * gen, loop_check_depth - epnested_mutex * file, user - immutable after setup * refcount - atomic (refcount_t) * napi_* - READ_ONCE / WRITE_ONCE * * struct epitem: * rbn / rcu union - rbn: ep->mtx (while epi is linked in ep->rbr). * rcu: written only by kfree_rcu(epi) on the free * path; otherwise untouched by epoll code. * rdllink, next - ep->lock * ffd, ep - immutable after ep_insert() * pwqlist - ep->mtx for writes; POLLFREE clears pwq->whead * via smp_store_release(), see below * fllink - file->f_lock for mutation; hlist_del_rcu + * kfree_rcu(epi) for safe RCU readers * ws - RCU (rcu_assign_pointer / * rcu_dereference_check(mtx)) * event - ep->mtx for writes; lockless read in * ep_poll_callback pairs with smp_mb() in * ep_modify() * * * Ready-list state machine * ------------------------ * * Readiness is tracked in two lists under ep->lock: * * rdllist - doubly-linked FIFO; the "current" ready list. * ovflist - singly-linked LIFO; used during a scan to catch * events that arrive while rdllist is being iterated * without ep->lock. * * Encoded in ep->ovflist: * EP_UNACTIVE_PTR - no scan active; callback appends to rdllist. * NULL - scan active, no spill yet. * pointer to epi - scan active with spilled items (LIFO). * * Encoded in epi->ovflist_next: * EP_UNACTIVE_PTR - epi is not on ovflist. * otherwise - next epi on ovflist (NULL at tail). * * ep_start_scan() flips "not scanning" to "scanning" and splices * rdllist into a caller-local scan_batch. ep_done_scan() drains ovflist * back to rdllist (list_add head-insert reverses LIFO to FIFO), * flips back to "not scanning", and re-splices any items the caller * left in scan_batch (e.g., level-triggered re-queues). * * * Removal paths * ------------- * * Three paths dispose of epitems and/or eventpolls: * * A. ep_remove() - EPOLL_CTL_DEL and ep_insert() * rollback. Caller holds ep->mtx. * B. ep_clear_and_put() - close of the epoll fd itself * (ep_eventpoll_release). * C. eventpoll_release_file() - close of a watched file, invoked * from __fput(). * * Coordination: * A and C exclude each other via the watched file's refcount. * A pins the file with epi_fget() before touching file->f_ep or * file->f_lock; if the pin fails, __fput() is in flight and C * will clean this epi up. See the epi_fget() block comment. * A and B both hold ep->mtx serially. B walks the rbtree with * rb_next() captured before ep_remove() erases the current node. * B and C both take ep->mtx; the loser sees fewer entries or an * empty file->f_ep. * * Within every path the internal order is strict: * ep_unregister_pollwait() - drain pwqlist; synchronizes with any * in-flight ep_poll_callback via the * watched wait-queue head's lock. * ep_remove_file() - hlist_del_rcu of epi->fllink and, * if last watcher, clear file->f_ep, * under file->f_lock. * ep_remove_epi() - rb_erase, rdllist unlink (ep->lock), * wakeup_source_unregister, * kfree_rcu(epi). * * kfree_rcu(epi) defers the free past RCU readers in * reverse_path_check_proc(); kfree_rcu(ep) defers past readers in * ep_get_upwards_depth_proc(). * * * POLLFREE handshake * ------------------ * * When a subsystem tears down a wait-queue head that an epitem is * registered on (binder, signalfd, ...), it wakes the callback with * POLLFREE and must RCU-defer the head's free. The store/load pair: * * ep_poll_callback() POLLFREE branch: * smp_store_release(&pwq->whead, NULL) * * ep_remove_wait_queue(): * smp_load_acquire(&pwq->whead) * * See those sites for the full argument. */ /* Epoll private bits inside the event mask */#define EP_PRIVATE_BITS (EPOLLWAKEUP | EPOLLONESHOT | EPOLLET | EPOLLEXCLUSIVE) #define EPOLLINOUT_BITS (EPOLLIN | EPOLLOUT) #define EPOLLEXCLUSIVE_OK_BITS (EPOLLINOUT_BITS | EPOLLERR | EPOLLHUP | \ EPOLLWAKEUP | EPOLLET | EPOLLEXCLUSIVE) /* Maximum number of nesting allowed inside epoll sets */#define EP_MAX_NESTS 4 #define EP_MAX_EVENTS (INT_MAX / sizeof(struct epoll_event)) #define EP_UNACTIVE_PTR ((void *) -1L) #define EP_ITEM_COST (sizeof(struct epitem) + sizeof(struct eppoll_entry)) /* Wait structure used by the poll hooks */struct eppoll_entry { /* List header used to link this structure to the "struct epitem" */ struct eppoll_entry *next; /* The "base" pointer is set to the container "struct epitem" */ struct epitem *base; /* * Wait queue item that will be linked to the target file wait * queue head. */ wait_queue_entry_t wait; /* The wait queue head that linked the "wait" wait queue item */ wait_queue_head_t *whead;}; /* * Each file descriptor added to the eventpoll interface will * have an entry of this type linked to the "rbr" RB tree. * Avoid increasing the size of this struct, there can be many thousands * of these on a server and we do not want this to take another cache line. */struct epitem { union { /* RB tree node links this structure to the eventpoll RB tree */ struct rb_node rbn; /* Used to free the struct epitem */ struct rcu_head rcu; }; /* Link on the owning eventpoll's ready list (ep->rdllist). */ struct list_head rdllink; /* * Link on the owning eventpoll's scan-overflow list (ep->ovflist), * EP_UNACTIVE_PTR when not linked. See epi_on_ovflist() / * epi_clear_ovflist() and the "Ready-list state machine" section * in the top-of-file banner. */ struct epitem *ovflist_next; /* The file descriptor information this item refers to */ struct epoll_key ffd; /* List containing poll wait queues */ struct eppoll_entry *pwqlist; /* The "container" of this item */ struct eventpoll *ep; /* List header used to link this item to the "struct file" items list */ struct hlist_node fllink; /* wakeup_source used when EPOLLWAKEUP is set */ struct wakeup_source __rcu *ws; /* The structure that describe the interested events and the source fd */ struct epoll_event event;}; /* * This structure is stored inside the "private_data" member of the file * structure and represents the main data structure for the eventpoll * interface. */struct eventpoll { /* * This mutex is used to ensure that files are not removed * while epoll is using them. This is held during the event * collection loop, the file cleanup path, the epoll file exit * code and the ctl operations. */ struct mutex mtx; /* Wait queue used by sys_epoll_wait() */ wait_queue_head_t wq; /* Wait queue used by file->poll() */ wait_queue_head_t poll_wait; /* List of ready file descriptors */ struct list_head rdllist; /* Lock which protects rdllist and ovflist */ spinlock_t lock; /* Protect switching between rdllist and ovflist */ seqcount_spinlock_t seq; /* RB tree root used to store monitored fd structs */ struct rb_root_cached rbr; /* * This is a single linked list that chains all the "struct epitem" that * happened while transferring ready events to userspace w/out * holding ->lock. */ struct epitem *ovflist; /* wakeup_source used when ep_send_events or __ep_eventpoll_poll is running */ struct wakeup_source *ws; /* The user that created the eventpoll descriptor */ struct user_struct *user; struct file *file; /* used to optimize loop detection check */ u64 gen; struct hlist_head refs; u8 loop_check_depth; /* usage count, orchestrates "struct eventpoll" disposal */ refcount_t refcount; /* used to defer freeing past ep_get_upwards_depth_proc() RCU walk */ struct rcu_head rcu; #ifdef CONFIG_NET_RX_BUSY_POLL /* used to track busy poll napi_id */ unsigned int napi_id; /* busy poll timeout */ u32 busy_poll_usecs; /* busy poll packet budget */ u16 busy_poll_budget; bool prefer_busy_poll;#endif #ifdef CONFIG_DEBUG_LOCK_ALLOC /* tracks wakeup nests for lockdep validation */ u8 nests;#endif}; /* Wrapper struct used by poll queueing */struct ep_pqueue { poll_table pt; struct epitem *epi;}; /* * Configuration options available inside /proc/sys/fs/epoll/ *//* Maximum number of epoll watched descriptors, per user */static long max_user_watches __read_mostly; /* * Cycle and path-length checks at EPOLL_CTL_ADD * --------------------------------------------- * * When EPOLL_CTL_ADD creates a link that either targets an eventpoll * file or extends an existing chain of eventpolls, two checks run: * * 1. no cycle is being formed -- ep_loop_check() walks downward * from the candidate target, and ep_get_upwards_depth_proc() * walks upward from the outer ep, both bounded by EP_MAX_NESTS. * 2. no file accumulates more than path_limits[depth] wakeup paths * of a given length -- reverse_path_check(). * * Both need a global view of the epoll topology and must be atomic * with the insertion, so the check is serialized by epnested_mutex * and carries its scratch state on a stack-allocated struct * ep_ctl_ctx scoped to one do_epoll_ctl() call. Non-nested inserts * skip this machinery entirely and take only ep->mtx. * * epnested_mutex Serializes the whole check. * loop_check_gen Global monotonic stamp, bumped at the start of * a check and again at the end. ep->gen caches * the value under which ep was last visited by * ep_loop_check_proc() or * ep_get_upwards_depth_proc(); the post-check * bump ensures those cached stamps can no longer * equal loop_check_gen, so the * "ep->gen == loop_check_gen" trigger in * ep_ctl_lock() only fires while another check * is in flight. * * struct ep_ctl_ctx carries the rest (inserting_into, tfile_check_list, * path_count[]) through the walk; see its declaration below. * * Commits fdcfce93073d ("eventpoll: Fix integer overflow in * ep_loop_check_proc()") and f2e467a48287 ("eventpoll: Fix * semi-unbounded recursion") hardened the walk; any refactor must * preserve both bail-outs. */static DEFINE_MUTEX(epnested_mutex);static u64 loop_check_gen = 0; #define PATH_ARR_SIZE 5 /* * Per-do_epoll_ctl() scratch for the loop / path checks. Allocated on * the caller's stack; populated by ep_ctl_lock() and the downward * walk; consumed by reverse_path_check(); released by ep_ctl_unlock(). * Only valid while the caller holds epnested_mutex. */struct ep_ctl_ctx { /* * Outer eventpoll for one ep_loop_check(); if the downward walk * reaches it the insert would form a cycle. */ struct eventpoll *inserting_into; /* * Singly-linked list of epitems_head objects collected during * ep_loop_check_proc(), then walked by reverse_path_check(). * Terminated by EP_UNACTIVE_PTR, not NULL: epitems_head->next * doubles as a membership flag (a NULL ->next means "not on this * list", see ep_remove_file()), so the list uses a non-NULL * sentinel to keep the tail head distinguishable from an unlisted * one. */ struct epitems_head *tfile_check_list; /* * Per-depth wakeup-path tally used by reverse_path_check_proc(); * reinitialized to zero at the start of each reverse_path_check() * iteration. */ int path_count[PATH_ARR_SIZE];}; /* Slab cache used to allocate "struct epitem" */static struct kmem_cache *epi_cache __ro_after_init; /* Slab cache used to allocate "struct eppoll_entry" */static struct kmem_cache *pwq_cache __ro_after_init; /* * Wrapper anchor for file->f_ep when the watched file is not itself an * eventpoll; for the epoll-watches-epoll case, file->f_ep points at * &watched_ep->refs directly. The ->next field threads * ctx->tfile_check_list during one EPOLL_CTL_ADD path check. The ->file * field holds a reference to the associated file while the head is on * the list. */struct epitems_head { struct hlist_head epitems; struct epitems_head *next; struct file *file;}; static struct kmem_cache *ephead_cache __ro_after_init; static inline void free_ephead(struct epitems_head *head){ if (head) kmem_cache_free(ephead_cache, head);} static void list_file(struct file *file, struct ep_ctl_ctx *ctx){ struct epitems_head *head; head = container_of(file->f_ep, struct epitems_head, epitems); if (!head->next) { /* * The caller owns a reference to @file or holds the ep->mtx for the * epitem that led here. The latter blocks eventpoll_release_file() * before the file allocation can be freed and reused. A dying leaf * can be skipped since removing links cannot increase the reverse * path count. */ if (!file_ref_get(&file->f_ref)) return; head->file = file; head->next = ctx->tfile_check_list; ctx->tfile_check_list = head; }} Lines 1–500 of 3030. Load more
// SPDX-License-Identifier: GPL-2.0-or-later /* * fs/eventpoll.c (Efficient event retrieval implementation) * Copyright (C) 2001,...,2009 Davide Libenzi * * Davide Libenzi <davidel@xmailserver.org> */ #include <linux/init.h> #include <linux/kernel.h> #include <linux/sched/signal.h> #include <linux/fs.h> #include <linux/file.h> #include <linux/signal.h> #include <linux/errno.h> #include <linux/mm.h> #include <linux/slab.h> #include <linux/poll.h> #include <linux/string.h> #include <linux/list.h> #include <linux/hash.h> #include <linux/spinlock.h> #include <linux/syscalls.h> #include <linux/rbtree.h> #include <linux/wait.h> #include <linux/eventpoll.h> #include <linux/mount.h> #include <linux/bitops.h> #include <linux/mutex.h> #include <linux/anon_inodes.h> #include <linux/device.h> #include <linux/uaccess.h> #include <asm/io.h> #include <asm/mman.h> #include <linux/atomic.h> #include <linux/proc_fs.h> #include <linux/seq_file.h> #include <linux/compat.h> #include <linux/rculist.h> #include <linux/capability.h> #include <linux/seqlock.h> #include <net/busy_poll.h> /* * fs/eventpoll.c - Efficient event polling ("epoll") kernel implementation. * * * Overview * -------- * * Each epoll_create(2) returns an anonymous [eventpoll] file whose * ->private_data is a struct eventpoll. Each EPOLL_CTL_ADD installs * a struct epitem linking one (watched file, fd) pair back to that * eventpoll via the watched file's f_op->poll() wait queue(s). When * the watched file signals readiness, ep_poll_callback() fires and * marks the epitem ready. epoll_wait(2) drains the ready list under * ep->mtx, re-queueing items in level-triggered mode. * * epoll instances can watch other epoll instances up to EP_MAX_NESTS * deep; cycles are forbidden and detected at EPOLL_CTL_ADD time. * * * Locking * ------- * * Three levels, acquired from outer to inner: * * epnested_mutex (global; rare; taken only for EPOLL_CTL_ADD * loop / path checks) * > ep->mtx (per-eventpoll; sleepable; serializes most ops) * > ep->lock (per-eventpoll; IRQ-safe spinlock) * * file->f_lock (per-file; NOT IRQ-safe; guards f_ep hlist ops; * nested inside ep->mtx, outside ep->lock) * * Rationale: * - ep->lock is a spinlock because ep_poll_callback() is called from * wake_up() which may run in hard-IRQ context. All ep->lock * critical sections use spin_lock_irqsave(). * - ep->mtx is a sleepable mutex because the event delivery loop * calls copy_to_user(), and ep_insert() may sleep in * kmem_cache_alloc() and f_op->poll(). * - epnested_mutex is global because cycle detection needs a global * view of the epoll topology; a per-object scheme would let two * concurrent inserts (A into B, B into A) construct a cycle * without either observer seeing it. * - Per-ep ep->mtx is preferred for scalability elsewhere. Events * that require epnested_mutex are rare. * * When EPOLL_CTL_ADD nests one eventpoll inside another we acquire * ep->mtx on both: outer first, target second. Since cycles are * forbidden the set of live ep->mtx holds is always a strict chain, * communicated to lockdep via mutex_lock_nested() subclasses derived * from the current recursion depth. * * * Field protection * ---------------- * * struct eventpoll: * mtx - self * rbr - ep->mtx * ovflist, rdllist - ep->lock (IRQ-safe) * wq - ep->lock for queue mutation * poll_wait - internal waitqueue spinlock * refs - file->f_lock for adds; ep->mtx for removes; * RCU for readers (hlist_del_rcu + kfree_rcu(ep)) * ws - ep->mtx * gen, loop_check_depth - epnested_mutex * file, user - immutable after setup * refcount - atomic (refcount_t) * napi_* - READ_ONCE / WRITE_ONCE * * struct epitem: * rbn / rcu union - rbn: ep->mtx (while epi is linked in ep->rbr). * rcu: written only by kfree_rcu(epi) on the free * path; otherwise untouched by epoll code. * rdllink, next - ep->lock * ffd, ep - immutable after ep_insert() * pwqlist - ep->mtx for writes; POLLFREE clears pwq->whead * via smp_store_release(), see below * fllink - file->f_lock for mutation; hlist_del_rcu + * kfree_rcu(epi) for safe RCU readers * ws - RCU (rcu_assign_pointer / * rcu_dereference_check(mtx)) * event - ep->mtx for writes; lockless read in * ep_poll_callback pairs with smp_mb() in * ep_modify() * * * Ready-list state machine * ------------------------ * * Readiness is tracked in two lists under ep->lock: * * rdllist - doubly-linked FIFO; the "current" ready list. * ovflist - singly-linked LIFO; used during a scan to catch * events that arrive while rdllist is being iterated * without ep->lock. * * Encoded in ep->ovflist: * EP_UNACTIVE_PTR - no scan active; callback appends to rdllist. * NULL - scan active, no spill yet. * pointer to epi - scan active with spilled items (LIFO). * * Encoded in epi->ovflist_next: * EP_UNACTIVE_PTR - epi is not on ovflist. * otherwise - next epi on ovflist (NULL at tail). * * ep_start_scan() flips "not scanning" to "scanning" and splices * rdllist into a caller-local scan_batch. ep_done_scan() drains ovflist * back to rdllist (list_add head-insert reverses LIFO to FIFO), * flips back to "not scanning", and re-splices any items the caller * left in scan_batch (e.g., level-triggered re-queues). * * * Removal paths * ------------- * * Three paths dispose of epitems and/or eventpolls: * * A. ep_remove() - EPOLL_CTL_DEL and ep_insert() * rollback. Caller holds ep->mtx. * B. ep_clear_and_put() - close of the epoll fd itself * (ep_eventpoll_release). * C. eventpoll_release_file() - close of a watched file, invoked * from __fput(). * * Coordination: * A and C exclude each other via the watched file's refcount. * A pins the file with epi_fget() before touching file->f_ep or * file->f_lock; if the pin fails, __fput() is in flight and C * will clean this epi up. See the epi_fget() block comment. * A and B both hold ep->mtx serially. B walks the rbtree with * rb_next() captured before ep_remove() erases the current node. * B and C both take ep->mtx; the loser sees fewer entries or an * empty file->f_ep. * * Within every path the internal order is strict: * ep_unregister_pollwait() - drain pwqlist; synchronizes with any * in-flight ep_poll_callback via the * watched wait-queue head's lock. * ep_remove_file() - hlist_del_rcu of epi->fllink and, * if last watcher, clear file->f_ep, * under file->f_lock. * ep_remove_epi() - rb_erase, rdllist unlink (ep->lock), * wakeup_source_unregister, * kfree_rcu(epi). * * kfree_rcu(epi) defers the free past RCU readers in * reverse_path_check_proc(); kfree_rcu(ep) defers past readers in * ep_get_upwards_depth_proc(). * * * POLLFREE handshake * ------------------ * * When a subsystem tears down a wait-queue head that an epitem is * registered on (binder, signalfd, ...), it wakes the callback with * POLLFREE and must RCU-defer the head's free. The store/load pair: * * ep_poll_callback() POLLFREE branch: * smp_store_release(&pwq->whead, NULL) * * ep_remove_wait_queue(): * smp_load_acquire(&pwq->whead) * * See those sites for the full argument. */ /* Epoll private bits inside the event mask */ #define EP_PRIVATE_BITS (EPOLLWAKEUP | EPOLLONESHOT | EPOLLET | EPOLLEXCLUSIVE) #define EPOLLINOUT_BITS (EPOLLIN | EPOLLOUT) #define EPOLLEXCLUSIVE_OK_BITS (EPOLLINOUT_BITS | EPOLLERR | EPOLLHUP | \ EPOLLWAKEUP | EPOLLET | EPOLLEXCLUSIVE) /* Maximum number of nesting allowed inside epoll sets */ #define EP_MAX_NESTS 4 #define EP_MAX_EVENTS (INT_MAX / sizeof(struct epoll_event)) #define EP_UNACTIVE_PTR ((void *) -1L) #define EP_ITEM_COST (sizeof(struct epitem) + sizeof(struct eppoll_entry)) /* Wait structure used by the poll hooks */ struct eppoll_entry { /* List header used to link this structure to the "struct epitem" */ struct eppoll_entry *next; /* The "base" pointer is set to the container "struct epitem" */ struct epitem *base; /* * Wait queue item that will be linked to the target file wait * queue head. */ wait_queue_entry_t wait; /* The wait queue head that linked the "wait" wait queue item */ wait_queue_head_t *whead; }; /* * Each file descriptor added to the eventpoll interface will * have an entry of this type linked to the "rbr" RB tree. * Avoid increasing the size of this struct, there can be many thousands * of these on a server and we do not want this to take another cache line. */ struct epitem { union { /* RB tree node links this structure to the eventpoll RB tree */ struct rb_node rbn; /* Used to free the struct epitem */ struct rcu_head rcu; }; /* Link on the owning eventpoll's ready list (ep->rdllist). */ struct list_head rdllink; /* * Link on the owning eventpoll's scan-overflow list (ep->ovflist), * EP_UNACTIVE_PTR when not linked. See epi_on_ovflist() / * epi_clear_ovflist() and the "Ready-list state machine" section * in the top-of-file banner. */ struct epitem *ovflist_next; /* The file descriptor information this item refers to */ struct epoll_key ffd; /* List containing poll wait queues */ struct eppoll_entry *pwqlist; /* The "container" of this item */ struct eventpoll *ep; /* List header used to link this item to the "struct file" items list */ struct hlist_node fllink; /* wakeup_source used when EPOLLWAKEUP is set */ struct wakeup_source __rcu *ws; /* The structure that describe the interested events and the source fd */ struct epoll_event event; }; /* * This structure is stored inside the "private_data" member of the file * structure and represents the main data structure for the eventpoll * interface. */ struct eventpoll { /* * This mutex is used to ensure that files are not removed * while epoll is using them. This is held during the event * collection loop, the file cleanup path, the epoll file exit * code and the ctl operations. */ struct mutex mtx; /* Wait queue used by sys_epoll_wait() */ wait_queue_head_t wq; /* Wait queue used by file->poll() */ wait_queue_head_t poll_wait; /* List of ready file descriptors */ struct list_head rdllist; /* Lock which protects rdllist and ovflist */ spinlock_t lock; /* Protect switching between rdllist and ovflist */ seqcount_spinlock_t seq; /* RB tree root used to store monitored fd structs */ struct rb_root_cached rbr; /* * This is a single linked list that chains all the "struct epitem" that * happened while transferring ready events to userspace w/out * holding ->lock. */ struct epitem *ovflist; /* wakeup_source used when ep_send_events or __ep_eventpoll_poll is running */ struct wakeup_source *ws; /* The user that created the eventpoll descriptor */ struct user_struct *user; struct file *file; /* used to optimize loop detection check */ u64 gen; struct hlist_head refs; u8 loop_check_depth; /* usage count, orchestrates "struct eventpoll" disposal */ refcount_t refcount; /* used to defer freeing past ep_get_upwards_depth_proc() RCU walk */ struct rcu_head rcu; #ifdef CONFIG_NET_RX_BUSY_POLL /* used to track busy poll napi_id */ unsigned int napi_id; /* busy poll timeout */ u32 busy_poll_usecs; /* busy poll packet budget */ u16 busy_poll_budget; bool prefer_busy_poll; #endif #ifdef CONFIG_DEBUG_LOCK_ALLOC /* tracks wakeup nests for lockdep validation */ u8 nests; #endif }; /* Wrapper struct used by poll queueing */ struct ep_pqueue { poll_table pt; struct epitem *epi; }; /* * Configuration options available inside /proc/sys/fs/epoll/ */ /* Maximum number of epoll watched descriptors, per user */ static long max_user_watches __read_mostly; /* * Cycle and path-length checks at EPOLL_CTL_ADD * --------------------------------------------- * * When EPOLL_CTL_ADD creates a link that either targets an eventpoll * file or extends an existing chain of eventpolls, two checks run: * * 1. no cycle is being formed -- ep_loop_check() walks downward * from the candidate target, and ep_get_upwards_depth_proc() * walks upward from the outer ep, both bounded by EP_MAX_NESTS. * 2. no file accumulates more than path_limits[depth] wakeup paths * of a given length -- reverse_path_check(). * * Both need a global view of the epoll topology and must be atomic * with the insertion, so the check is serialized by epnested_mutex * and carries its scratch state on a stack-allocated struct * ep_ctl_ctx scoped to one do_epoll_ctl() call. Non-nested inserts * skip this machinery entirely and take only ep->mtx. * * epnested_mutex Serializes the whole check. * loop_check_gen Global monotonic stamp, bumped at the start of * a check and again at the end. ep->gen caches * the value under which ep was last visited by * ep_loop_check_proc() or * ep_get_upwards_depth_proc(); the post-check * bump ensures those cached stamps can no longer * equal loop_check_gen, so the * "ep->gen == loop_check_gen" trigger in * ep_ctl_lock() only fires while another check * is in flight. * * struct ep_ctl_ctx carries the rest (inserting_into, tfile_check_list, * path_count[]) through the walk; see its declaration below. * * Commits fdcfce93073d ("eventpoll: Fix integer overflow in * ep_loop_check_proc()") and f2e467a48287 ("eventpoll: Fix * semi-unbounded recursion") hardened the walk; any refactor must * preserve both bail-outs. */ static DEFINE_MUTEX(epnested_mutex); static u64 loop_check_gen = 0; #define PATH_ARR_SIZE 5 /* * Per-do_epoll_ctl() scratch for the loop / path checks. Allocated on * the caller's stack; populated by ep_ctl_lock() and the downward * walk; consumed by reverse_path_check(); released by ep_ctl_unlock(). * Only valid while the caller holds epnested_mutex. */ struct ep_ctl_ctx { /* * Outer eventpoll for one ep_loop_check(); if the downward walk * reaches it the insert would form a cycle. */ struct eventpoll *inserting_into; /* * Singly-linked list of epitems_head objects collected during * ep_loop_check_proc(), then walked by reverse_path_check(). * Terminated by EP_UNACTIVE_PTR, not NULL: epitems_head->next * doubles as a membership flag (a NULL ->next means "not on this * list", see ep_remove_file()), so the list uses a non-NULL * sentinel to keep the tail head distinguishable from an unlisted * one. */ struct epitems_head *tfile_check_list; /* * Per-depth wakeup-path tally used by reverse_path_check_proc(); * reinitialized to zero at the start of each reverse_path_check() * iteration. */ int path_count[PATH_ARR_SIZE]; }; /* Slab cache used to allocate "struct epitem" */ static struct kmem_cache *epi_cache __ro_after_init; /* Slab cache used to allocate "struct eppoll_entry" */ static struct kmem_cache *pwq_cache __ro_after_init; /* * Wrapper anchor for file->f_ep when the watched file is not itself an * eventpoll; for the epoll-watches-epoll case, file->f_ep points at * &watched_ep->refs directly. The ->next field threads * ctx->tfile_check_list during one EPOLL_CTL_ADD path check. The ->file * field holds a reference to the associated file while the head is on * the list. */ struct epitems_head { struct hlist_head epitems; struct epitems_head *next; struct file *file; }; static struct kmem_cache *ephead_cache __ro_after_init; static inline void free_ephead(struct epitems_head *head) { if (head) kmem_cache_free(ephead_cache, head); } static void list_file(struct file *file, struct ep_ctl_ctx *ctx) { struct epitems_head *head; head = container_of(file->f_ep, struct epitems_head, epitems); if (!head->next) { /* * The caller owns a reference to @file or holds the ep->mtx for the * epitem that led here. The latter blocks eventpoll_release_file() * before the file allocation can be freed and reused. A dying leaf * can be skipped since removing links cannot increase the reverse * path count. */ if (!file_ref_get(&file->f_ref)) return; head->file = file; head->next = ctx->tfile_check_list; ctx->tfile_check_list = head; } }