/* *  fs/eventpoll.c (Efficient event retrieval implementation) *  Copyright (C) 2001,...,2009	 Davide Libenzi * *  This program is free software; you can redistribute it and/or modify *  it under the terms of the GNU General Public License as published by *  the Free Software Foundation; either version 2 of the License, or *  (at your option) any later version. * *  Davide Libenzi <davidel@xmailserver.org> * */ #include <linux/init.h>#include <linux/kernel.h>#include <linux/sched.h>#include <linux/fs.h>#include <linux/file.h>#include <linux/signal.h>#include <linux/errno.h>#include <linux/mm.h>#include <linux/slab.h>#include <linux/poll.h>#include <linux/string.h>#include <linux/list.h>#include <linux/hash.h>#include <linux/spinlock.h>#include <linux/syscalls.h>#include <linux/rbtree.h>#include <linux/wait.h>#include <linux/eventpoll.h>#include <linux/mount.h>#include <linux/bitops.h>#include <linux/mutex.h>#include <linux/anon_inodes.h>#include <linux/device.h>#include <asm/uaccess.h>#include <asm/io.h>#include <asm/mman.h>#include <linux/atomic.h> /* * LOCKING: * There are three level of locking required by epoll : * * 1) epmutex (mutex) * 2) ep->mtx (mutex) * 3) ep->lock (spinlock) * * The acquire order is the one listed above, from 1 to 3. * We need a spinlock (ep->lock) because we manipulate objects * from inside the poll callback, that might be triggered from * a wake_up() that in turn might be called from IRQ context. * So we can't sleep inside the poll callback and hence we need * a spinlock. During the event transfer loop (from kernel to * user space) we could end up sleeping due a copy_to_user(), so * we need a lock that will allow us to sleep. This lock is a * mutex (ep->mtx). It is acquired during the event transfer loop, * during epoll_ctl(EPOLL_CTL_DEL) and during eventpoll_release_file(). * Then we also need a global mutex to serialize eventpoll_release_file() * and ep_free(). * This mutex is acquired by ep_free() during the epoll file * cleanup path and it is also acquired by eventpoll_release_file() * if a file has been pushed inside an epoll set and it is then * close()d without a previous call to epoll_ctl(EPOLL_CTL_DEL). * It is also acquired when inserting an epoll fd onto another epoll * fd. We do this so that we walk the epoll tree and ensure that this * insertion does not create a cycle of epoll file descriptors, which * could lead to deadlock. We need a global mutex to prevent two * simultaneous inserts (A into B and B into A) from racing and * constructing a cycle without either insert observing that it is * going to. * It is necessary to acquire multiple "ep->mtx"es at once in the * case when one epoll fd is added to another. In this case, we * always acquire the locks in the order of nesting (i.e. after * epoll_ctl(e1, EPOLL_CTL_ADD, e2), e1->mtx will always be acquired * before e2->mtx). Since we disallow cycles of epoll file * descriptors, this ensures that the mutexes are well-ordered. In * order to communicate this nesting to lockdep, when walking a tree * of epoll file descriptors, we use the current recursion depth as * the lockdep subkey. * It is possible to drop the "ep->mtx" and to use the global * mutex "epmutex" (together with "ep->lock") to have it working, * but having "ep->mtx" will make the interface more scalable. * Events that require holding "epmutex" are very rare, while for * normal operations the epoll private "ep->mtx" will guarantee * a better scalability. */ /* Epoll private bits inside the event mask */#define EP_PRIVATE_BITS (EPOLLWAKEUP | EPOLLONESHOT | EPOLLET) /* Maximum number of nesting allowed inside epoll sets */#define EP_MAX_NESTS 4 #define EP_MAX_EVENTS (INT_MAX / sizeof(struct epoll_event)) #define EP_UNACTIVE_PTR ((void *) -1L) #define EP_ITEM_COST (sizeof(struct epitem) + sizeof(struct eppoll_entry)) struct epoll_filefd {	struct file *file;	int fd;}; /* * Structure used to track possible nested calls, for too deep recursions * and loop cycles. */struct nested_call_node {	struct list_head llink;	void *cookie;	void *ctx;}; /* * This structure is used as collector for nested calls, to check for * maximum recursion dept and loop cycles. */struct nested_calls {	struct list_head tasks_call_list;	spinlock_t lock;}; /* * Each file descriptor added to the eventpoll interface will * have an entry of this type linked to the "rbr" RB tree. */struct epitem {	/* RB tree node used to link this structure to the eventpoll RB tree */	struct rb_node rbn; 	/* List header used to link this structure to the eventpoll ready list */	struct list_head rdllink; 	/*	 * Works together "struct eventpoll"->ovflist in keeping the	 * single linked chain of items.	 */	struct epitem *next; 	/* The file descriptor information this item refers to */	struct epoll_filefd ffd; 	/* Number of active wait queue attached to poll operations */	int nwait; 	/* List containing poll wait queues */	struct list_head pwqlist; 	/* The "container" of this item */	struct eventpoll *ep; 	/* List header used to link this item to the "struct file" items list */	struct list_head fllink; 	/* wakeup_source used when EPOLLWAKEUP is set */	struct wakeup_source *ws; 	/* The structure that describe the interested events and the source fd */	struct epoll_event event;}; /* * This structure is stored inside the "private_data" member of the file * structure and represents the main data structure for the eventpoll * interface. */struct eventpoll {	/* Protect the access to this structure */	spinlock_t lock; 	/*	 * This mutex is used to ensure that files are not removed	 * while epoll is using them. This is held during the event	 * collection loop, the file cleanup path, the epoll file exit	 * code and the ctl operations.	 */	struct mutex mtx; 	/* Wait queue used by sys_epoll_wait() */	wait_queue_head_t wq; 	/* Wait queue used by file->poll() */	wait_queue_head_t poll_wait; 	/* List of ready file descriptors */	struct list_head rdllist; 	/* RB tree root used to store monitored fd structs */	struct rb_root rbr; 	/*	 * This is a single linked list that chains all the "struct epitem" that	 * happened while transferring ready events to userspace w/out	 * holding ->lock.	 */	struct epitem *ovflist; 	/* wakeup_source used when ep_scan_ready_list is running */	struct wakeup_source *ws; 	/* The user that created the eventpoll descriptor */	struct user_struct *user; 	struct file *file; 	/* used to optimize loop detection check */	int visited;	struct list_head visited_list_link;}; /* Wait structure used by the poll hooks */struct eppoll_entry {	/* List header used to link this structure to the "struct epitem" */	struct list_head llink; 	/* The "base" pointer is set to the container "struct epitem" */	struct epitem *base; 	/*	 * Wait queue item that will be linked to the target file wait	 * queue head.	 */	wait_queue_t wait; 	/* The wait queue head that linked the "wait" wait queue item */	wait_queue_head_t *whead;}; /* Wrapper struct used by poll queueing */struct ep_pqueue {	poll_table pt;	struct epitem *epi;}; /* Used by the ep_send_events() function as callback private data */struct ep_send_events_data {	int maxevents;	struct epoll_event __user *events;}; /* * Configuration options available inside /proc/sys/fs/epoll/ *//* Maximum number of epoll watched descriptors, per user */static long max_user_watches __read_mostly; /* * This mutex is used to serialize ep_free() and eventpoll_release_file(). */static DEFINE_MUTEX(epmutex); /* Used to check for epoll file descriptor inclusion loops */static struct nested_calls poll_loop_ncalls; /* Used for safe wake up implementation */static struct nested_calls poll_safewake_ncalls; /* Used to call file's f_op->poll() under the nested calls boundaries */static struct nested_calls poll_readywalk_ncalls; /* Slab cache used to allocate "struct epitem" */static struct kmem_cache *epi_cache __read_mostly; /* Slab cache used to allocate "struct eppoll_entry" */static struct kmem_cache *pwq_cache __read_mostly; /* Visited nodes during ep_loop_check(), so we can unset them when we finish */static LIST_HEAD(visited_list); /* * List of files with newly added links, where we may need to limit the number * of emanating paths. Protected by the epmutex. */static LIST_HEAD(tfile_check_list); #ifdef CONFIG_SYSCTL #include <linux/sysctl.h> static long zero;static long long_max = LONG_MAX; ctl_table epoll_table[] = {	{		.procname	= "max_user_watches",		.data		= &max_user_watches,		.maxlen		= sizeof(max_user_watches),		.mode		= 0644,		.proc_handler	= proc_doulongvec_minmax,		.extra1		= &zero,		.extra2		= &long_max,	},	{ }};#endif /* CONFIG_SYSCTL */ static const struct file_operations eventpoll_fops; static inline int is_file_epoll(struct file *f){	return f->f_op == &eventpoll_fops;} /* Setup the structure that is used as key for the RB tree */static inline void ep_set_ffd(struct epoll_filefd *ffd,			      struct file *file, int fd){	ffd->file = file;	ffd->fd = fd;} /* Compare RB tree keys */static inline int ep_cmp_ffd(struct epoll_filefd *p1,			     struct epoll_filefd *p2){	return (p1->file > p2->file ? +1:	        (p1->file < p2->file ? -1 : p1->fd - p2->fd));} /* Tells us if the item is currently linked */static inline int ep_is_linked(struct list_head *p){	return !list_empty(p);} static inline struct eppoll_entry *ep_pwq_from_wait(wait_queue_t *p){	return container_of(p, struct eppoll_entry, wait);} /* Get the "struct epitem" from a wait queue pointer */static inline struct epitem *ep_item_from_wait(wait_queue_t *p){	return container_of(p, struct eppoll_entry, wait)->base;} /* Get the "struct epitem" from an epoll queue wrapper */static inline struct epitem *ep_item_from_epqueue(poll_table *p){	return container_of(p, struct ep_pqueue, pt)->epi;} /* Tells if the epoll_ctl(2) operation needs an event copy from userspace */static inline int ep_op_has_event(int op){	return op != EPOLL_CTL_DEL;} /* Initialize the poll safe wake up structure */static void ep_nested_calls_init(struct nested_calls *ncalls){	INIT_LIST_HEAD(&ncalls->tasks_call_list);	spin_lock_init(&ncalls->lock);} /** * ep_events_available - Checks if ready events might be available. * * @ep: Pointer to the eventpoll context. * * Returns: Returns a value different than zero if ready events are available, *          or zero otherwise. */static inline int ep_events_available(struct eventpoll *ep){	return !list_empty(&ep->rdllist) || ep->ovflist != EP_UNACTIVE_PTR;} /** * ep_call_nested - Perform a bound (possibly) nested call, by checking *                  that the recursion limit is not exceeded, and that *                  the same nested call (by the meaning of same cookie) is *                  no re-entered. * * @ncalls: Pointer to the nested_calls structure to be used for this call. * @max_nests: Maximum number of allowed nesting calls. * @nproc: Nested call core function pointer. * @priv: Opaque data to be passed to the @nproc callback. * @cookie: Cookie to be used to identify this nested call. * @ctx: This instance context. * * Returns: Returns the code returned by the @nproc callback, or -1 if *          the maximum recursion limit has been exceeded. */static int ep_call_nested(struct nested_calls *ncalls, int max_nests,			  int (*nproc)(void *, void *, int), void *priv,			  void *cookie, void *ctx){	int error, call_nests = 0;	unsigned long flags;	struct list_head *lsthead = &ncalls->tasks_call_list;	struct nested_call_node *tncur;	struct nested_call_node tnode; 	spin_lock_irqsave(&ncalls->lock, flags); 	/*	 * Try to see if the current task is already inside this wakeup call.	 * We use a list here, since the population inside this set is always	 * very much limited.	 */	list_for_each_entry(tncur, lsthead, llink) {		if (tncur->ctx == ctx &&		    (tncur->cookie == cookie || ++call_nests > max_nests)) {			/*			 * Ops ... loop detected or maximum nest level reached.			 * We abort this wake by breaking the cycle itself.			 */			error = -1;			goto out_unlock;		}	} 	/* Add the current task and cookie to the list */	tnode.ctx = ctx;	tnode.cookie = cookie;	list_add(&tnode.llink, lsthead); 	spin_unlock_irqrestore(&ncalls->lock, flags); 	/* Call the nested function */	error = (*nproc)(priv, cookie, call_nests); 	/* Remove the current task from the list */	spin_lock_irqsave(&ncalls->lock, flags);	list_del(&tnode.llink);out_unlock:	spin_unlock_irqrestore(&ncalls->lock, flags); 	return error;} /* * As described in commit 0ccf831cb lockdep: annotate epoll * the use of wait queues used by epoll is done in a very controlled * manner. Wake ups can nest inside each other, but are never done * with the same locking. For example: * *   dfd = socket(...); *   efd1 = epoll_create(); *   efd2 = epoll_create(); *   epoll_ctl(efd1, EPOLL_CTL_ADD, dfd, ...); *   epoll_ctl(efd2, EPOLL_CTL_ADD, efd1, ...); * * When a packet arrives to the device underneath "dfd", the net code will * issue a wake_up() on its poll wake list. Epoll (efd1) has installed a * callback wakeup entry on that queue, and the wake_up() performed by the * "dfd" net code will end up in ep_poll_callback(). At this point epoll * (efd1) notices that it may have some event ready, so it needs to wake up * the waiters on its poll wait list (efd2). So it calls ep_poll_safewake() * that ends up in another wake_up(), after having checked about the * recursion constraints. That are, no more than EP_MAX_POLLWAKE_NESTS, to * avoid stack blasting. * * When CONFIG_DEBUG_LOCK_ALLOC is enabled, make sure lockdep can handle * this special case of epoll. */#ifdef CONFIG_DEBUG_LOCK_ALLOCstatic inline void ep_wake_up_nested(wait_queue_head_t *wqueue,				     unsigned long events, int subclass){	unsigned long flags; 	spin_lock_irqsave_nested(&wqueue->lock, flags, subclass);	wake_up_locked_poll(wqueue, events);	spin_unlock_irqrestore(&wqueue->lock, flags);}#elsestatic inline void ep_wake_up_nested(wait_queue_head_t *wqueue,				     unsigned long events, int subclass){	wake_up_poll(wqueue, events);}#endif static int ep_poll_wakeup_proc(void *priv, void *cookie, int call_nests){	ep_wake_up_nested((wait_queue_head_t *) cookie, POLLIN,			  1 + call_nests);	return 0;} /* * Perform a safe wake up of the poll wait list. The problem is that * with the new callback'd wake up system, it is possible that the * poll callback is reentered from inside the call to wake_up() done * on the poll wait queue head. The rule is that we cannot reenter the * wake up code from the same task more than EP_MAX_NESTS times, * and we cannot reenter the same wait queue head at all. This will * enable to have a hierarchy of epoll file descriptor of no more than * EP_MAX_NESTS deep. */static void ep_poll_safewake(wait_queue_head_t *wq){	int this_cpu = get_cpu(); 	ep_call_nested(&poll_safewake_ncalls, EP_MAX_NESTS,

Lines 1–500 of 1934. Load more