Also make it available outside of epoll, along with the helper that decides if we need to copy the passed in epoll_event.
Unified · Split
| @@ -354,12 +354,6 @@ static inline struct epitem *ep_item_from_epqueue(poll_table *p) · 353 unmodified lines | ||
| 354 | 354 | return container_of(p, struct ep_pqueue, pt)->epi; |
| 355 | 355 | } |
| 356 | 356 | |
| 357 | −/* Tells if the epoll_ctl(2) operation needs an event copy from userspace */ | |
| 358 | −static inline int ep_op_has_event(int op) | |
| 359 | −{ | |
| 360 | − return op != EPOLL_CTL_DEL; | |
| 361 | −} | |
| 362 | − | |
| 363 | 357 | /* Initialize the poll safe wake up structure */ |
| 364 | 358 | static void ep_nested_calls_init(struct nested_calls *ncalls) |
| 365 | 359 | { |
| @@ -2074,7 +2068,20 @@ SYSCALL_DEFINE1(epoll_create, int, size) · 1708 unmodified lines | ||
| 2074 | 2068 | return do_epoll_create(0); |
| 2075 | 2069 | } |
| 2076 | 2070 | |
| 2077 | −static int do_epoll_ctl(int epfd, int op, int fd, struct epoll_event *epds) | |
| 2071 | +static inline int epoll_mutex_lock(struct mutex *mutex, int depth, | |
| 2072 | + bool nonblock) | |
| 2073 | +{ | |
| 2074 | + if (!nonblock) { | |
| 2075 | + mutex_lock_nested(mutex, depth); | |
| 2076 | + return 0; | |
| 2077 | + } | |
| 2078 | + if (mutex_trylock(mutex)) | |
| 2079 | + return 0; | |
| 2080 | + return -EAGAIN; | |
| 2081 | +} | |
| 2082 | + | |
| 2083 | +int do_epoll_ctl(int epfd, int op, int fd, struct epoll_event *epds, | |
| 2084 | + bool nonblock) | |
| 2078 | 2085 | { |
| 2079 | 2086 | int error; |
| 2080 | 2087 | int full_check = 0; |
| @@ -2145,13 +2152,17 @@ static int do_epoll_ctl(int epfd, int op, int fd, struct epoll_event *epds) · 64 unmodified lines | ||
| 2145 | 2152 | * deep wakeup paths from forming in parallel through multiple |
| 2146 | 2153 | * EPOLL_CTL_ADD operations. |
| 2147 | 2154 | */ |
| 2148 | − mutex_lock_nested(&ep->mtx, 0); | |
| 2155 | + error = epoll_mutex_lock(&ep->mtx, 0, nonblock); | |
| 2156 | + if (error) | |
| 2157 | + goto error_tgt_fput; | |
| 2149 | 2158 | if (op == EPOLL_CTL_ADD) { |
| 2150 | 2159 | if (!list_empty(&f.file->f_ep_links) || |
| 2151 | 2160 | is_file_epoll(tf.file)) { |
| 2152 | − full_check = 1; | |
| 2153 | 2161 | mutex_unlock(&ep->mtx); |
| 2154 | − mutex_lock(&epmutex); | |
| 2162 | + error = epoll_mutex_lock(&epmutex, 0, nonblock); | |
| 2163 | + if (error) | |
| 2164 | + goto error_tgt_fput; | |
| 2165 | + full_check = 1; | |
| 2155 | 2166 | if (is_file_epoll(tf.file)) { |
| 2156 | 2167 | error = -ELOOP; |
| 2157 | 2168 | if (ep_loop_check(ep, tf.file) != 0) { |
| @@ -2161,10 +2172,19 @@ static int do_epoll_ctl(int epfd, int op, int fd, struct epoll_event *epds) · 3 unmodified lines | ||
| 2161 | 2172 | } else |
| 2162 | 2173 | list_add(&tf.file->f_tfile_llink, |
| 2163 | 2174 | &tfile_check_list); |
| 2164 | − mutex_lock_nested(&ep->mtx, 0); | |
| 2175 | + error = epoll_mutex_lock(&ep->mtx, 0, nonblock); | |
| 2176 | + if (error) { | |
| 2177 | +out_del: | |
| 2178 | + list_del(&tf.file->f_tfile_llink); | |
| 2179 | + goto error_tgt_fput; | |
| 2180 | + } | |
| 2165 | 2181 | if (is_file_epoll(tf.file)) { |
| 2166 | 2182 | tep = tf.file->private_data; |
| 2167 | − mutex_lock_nested(&tep->mtx, 1); | |
| 2183 | + error = epoll_mutex_lock(&tep->mtx, 1, nonblock); | |
| 2184 | + if (error) { | |
| 2185 | + mutex_unlock(&ep->mtx); | |
| 2186 | + goto out_del; | |
| 2187 | + } | |
| 2168 | 2188 | } |
| 2169 | 2189 | } |
| 2170 | 2190 | } |
| @@ -2233,7 +2253,7 @@ SYSCALL_DEFINE4(epoll_ctl, int, epfd, int, op, int, fd, · 62 unmodified lines | ||
| 2233 | 2253 | copy_from_user(&epds, event, sizeof(struct epoll_event))) |
| 2234 | 2254 | return -EFAULT; |
| 2235 | 2255 | |
| 2236 | − return do_epoll_ctl(epfd, op, fd, &epds); | |
| 2256 | + return do_epoll_ctl(epfd, op, fd, &epds, false); | |
| 2237 | 2257 | } |
| 2238 | 2258 | |
| 2239 | 2259 | /* |
| @@ -61,6 +61,15 @@ static inline void eventpoll_release(struct file *file) · 60 unmodified lines | ||
| 61 | 61 | eventpoll_release_file(file); |
| 62 | 62 | } |
| 63 | 63 | |
| 64 | +int do_epoll_ctl(int epfd, int op, int fd, struct epoll_event *epds, | |
| 65 | + bool nonblock); | |
| 66 | + | |
| 67 | +/* Tells if the epoll_ctl(2) operation needs an event copy from userspace */ | |
| 68 | +static inline int ep_op_has_event(int op) | |
| 69 | +{ | |
| 70 | + return op != EPOLL_CTL_DEL; | |
| 71 | +} | |
| 72 | + | |
| 64 | 73 | #else |
| 65 | 74 | |
| 66 | 75 | static inline void eventpoll_init_file(struct file *file) {} |