io_req_local_work_add() signals the CQ ring eventfd inline when it is the one to push the first entry onto ->work_list. For DEFER_TASKRUN rings that add is frequently done from a waitqueue wakeup handler, where an arbitrary waitqueue lock is held. eventfd_signal_mask() only refuses to recurse when current->in_eventfd is set, but that bit is set by eventfd_signal_mask() itself. If the wake chain starts somewhere else, signal goes out inline and can feed back into epoll. Add IOU_F_TWQ_IN_WAKE, set it on the task_work add done from the three waitqueue callbacks, and use it to force io_eventfd_signal() down the existing call_rcu_hurry() deferral instead of signaling inline.
Unified · Split
| @@ -20,6 +20,14 @@ enum { · 19 unmodified lines | ||
| 20 | 20 | * It's also ignored unless IORING_SETUP_DEFER_TASKRUN is set. |
| 21 | 21 | */ |
| 22 | 22 | IOU_F_TWQ_LAZY_WAKE = 1, |
| 23 | + | |
| 24 | + /* | |
| 25 | + * Set when task_work is queued from a waitqueue wakeup handler, where | |
| 26 | + * an arbitrary provider waitqueue lock is held. Signaling the CQ ring | |
| 27 | + * eventfd inline from there can recurse back into that lock through | |
| 28 | + * epoll, so the eventfd signal must be deferred. | |
| 29 | + */ | |
| 30 | + IOU_F_TWQ_IN_WAKE = 2, | |
| 23 | 31 | }; |
| 24 | 32 | |
| 25 | 33 | enum io_uring_cmd_flags { |
| @@ -51,9 +51,9 @@ static void io_eventfd_do_signal(struct rcu_head *rcu) · 50 unmodified lines | ||
| 51 | 51 | /* |
| 52 | 52 | * Returns true if the caller should put the ev_fd reference, false if not. |
| 53 | 53 | */ |
| 54 | −static bool __io_eventfd_signal(struct io_ev_fd *ev_fd) | |
| 54 | +static bool __io_eventfd_signal(struct io_ev_fd *ev_fd, bool defer) | |
| 55 | 55 | { |
| 56 | − if (eventfd_signal_allowed()) { | |
| 56 | + if (!defer && eventfd_signal_allowed()) { | |
| 57 | 57 | eventfd_signal_mask(ev_fd->cq_ev_fd, EPOLL_URING_WAKE); |
| 58 | 58 | return true; |
| 59 | 59 | } |
| @@ -73,7 +73,7 @@ static bool io_eventfd_trigger(struct io_ev_fd *ev_fd) · 13 unmodified lines | ||
| 73 | 73 | return !ev_fd->eventfd_async || io_wq_current_is_worker(); |
| 74 | 74 | } |
| 75 | 75 | |
| 76 | −void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event) | |
| 76 | +void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event, bool defer) | |
| 77 | 77 | { |
| 78 | 78 | bool skip = false; |
| 79 | 79 | struct io_ev_fd *ev_fd; |
| @@ -113,7 +113,7 @@ void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event) · 33 unmodified lines | ||
| 113 | 113 | spin_unlock(&ctx->completion_lock); |
| 114 | 114 | } |
| 115 | 115 | |
| 116 | − if (skip || __io_eventfd_signal(ev_fd)) | |
| 116 | + if (skip || __io_eventfd_signal(ev_fd, defer)) | |
| 117 | 117 | io_eventfd_put(ev_fd); |
| 118 | 118 | } |
| 119 | 119 | |
| @@ -5,4 +5,4 @@ int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg, · 4 unmodified lines | ||
| 5 | 5 | unsigned int eventfd_async); |
| 6 | 6 | int io_eventfd_unregister(struct io_ring_ctx *ctx); |
| 7 | 7 | |
| 8 | −void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event); | |
| 8 | +void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event, bool defer); | |
| @@ -181,7 +181,7 @@ static void io_futex_wakev_fn(struct wake_q_head *wake_q, struct futex_q *q) · 180 unmodified lines | ||
| 181 | 181 | |
| 182 | 182 | io_req_set_res(req, 0, 0); |
| 183 | 183 | req->io_task_work.func = io_futexv_complete; |
| 184 | − io_req_task_work_add(req); | |
| 184 | + __io_req_task_work_add(req, IOU_F_TWQ_IN_WAKE); | |
| 185 | 185 | } |
| 186 | 186 | |
| 187 | 187 | int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) |
| @@ -237,7 +237,7 @@ static void io_futex_wake_fn(struct wake_q_head *wake_q, struct futex_q *q) · 49 unmodified lines | ||
| 237 | 237 | |
| 238 | 238 | io_req_set_res(req, 0, 0); |
| 239 | 239 | req->io_task_work.func = io_futex_complete; |
| 240 | − io_req_task_work_add(req); | |
| 240 | + __io_req_task_work_add(req, IOU_F_TWQ_IN_WAKE); | |
| 241 | 241 | } |
| 242 | 242 | |
| 243 | 243 | int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags) |
| @@ -475,7 +475,7 @@ void __io_commit_cqring_flush(struct io_ring_ctx *ctx) · 474 unmodified lines | ||
| 475 | 475 | if (ctx->int_flags & IO_RING_F_OFF_TIMEOUT_USED) |
| 476 | 476 | io_flush_timeouts(ctx); |
| 477 | 477 | if (ctx->int_flags & IO_RING_F_HAS_EVFD) |
| 478 | − io_eventfd_signal(ctx, true); | |
| 478 | + io_eventfd_signal(ctx, true, false); | |
| 479 | 479 | } |
| 480 | 480 | |
| 481 | 481 | static inline void __io_cq_lock(struct io_ring_ctx *ctx) |
| @@ -208,9 +208,9 @@ enum { · 207 unmodified lines | ||
| 208 | 208 | IOU_POLL_REQUEUE = 4, |
| 209 | 209 | }; |
| 210 | 210 | |
| 211 | −static void __io_poll_execute(struct io_kiocb *req, int mask) | |
| 211 | +static void __io_poll_execute(struct io_kiocb *req, int mask, unsigned tw_flags) | |
| 212 | 212 | { |
| 213 | − unsigned flags = 0; | |
| 213 | + unsigned flags = tw_flags; | |
| 214 | 214 | |
| 215 | 215 | io_req_set_res(req, mask, 0); |
| 216 | 216 | req->io_task_work.func = io_poll_task_func; |
| @@ -218,14 +218,15 @@ static void __io_poll_execute(struct io_kiocb *req, int mask) · 1 unmodified line | ||
| 218 | 218 | trace_io_uring_task_add(req, mask); |
| 219 | 219 | |
| 220 | 220 | if (!(req->flags & REQ_F_POLL_NO_LAZY)) |
| 221 | − flags = IOU_F_TWQ_LAZY_WAKE; | |
| 221 | + flags |= IOU_F_TWQ_LAZY_WAKE; | |
| 222 | 222 | __io_req_task_work_add(req, flags); |
| 223 | 223 | } |
| 224 | 224 | |
| 225 | −static inline void io_poll_execute(struct io_kiocb *req, int res) | |
| 225 | +static inline void io_poll_execute(struct io_kiocb *req, int res, | |
| 226 | + unsigned tw_flags) | |
| 226 | 227 | { |
| 227 | 228 | if (io_poll_get_ownership(req)) |
| 228 | − __io_poll_execute(req, res); | |
| 229 | + __io_poll_execute(req, res, tw_flags); | |
| 229 | 230 | } |
| 230 | 231 | |
| 231 | 232 | /* |
| @@ -344,7 +345,7 @@ void io_poll_task_func(struct io_tw_req tw_req, io_tw_token_t tw) · 112 unmodified lines | ||
| 344 | 345 | if (ret == IOU_POLL_NO_ACTION) { |
| 345 | 346 | return; |
| 346 | 347 | } else if (ret == IOU_POLL_REQUEUE) { |
| 347 | − __io_poll_execute(req, 0); | |
| 348 | + __io_poll_execute(req, 0, 0); | |
| 348 | 349 | return; |
| 349 | 350 | } |
| 350 | 351 | io_poll_remove_entries(req); |
| @@ -383,7 +384,7 @@ static void io_poll_cancel_req(struct io_kiocb *req) · 32 unmodified lines | ||
| 383 | 384 | { |
| 384 | 385 | io_poll_mark_cancelled(req); |
| 385 | 386 | /* kick tw, which should complete the request */ |
| 386 | − io_poll_execute(req, 0); | |
| 387 | + io_poll_execute(req, 0, 0); | |
| 387 | 388 | } |
| 388 | 389 | |
| 389 | 390 | #define IO_ASYNC_POLL_COMMON (EPOLLONESHOT | EPOLLPRI) |
| @@ -392,7 +393,7 @@ static __cold int io_pollfree_wake(struct io_kiocb *req, struct io_poll *poll) · 2 unmodified lines | ||
| 392 | 393 | { |
| 393 | 394 | io_poll_mark_cancelled(req); |
| 394 | 395 | /* we have to kick tw in case it's not already */ |
| 395 | − io_poll_execute(req, 0); | |
| 396 | + io_poll_execute(req, 0, IOU_F_TWQ_IN_WAKE); | |
| 396 | 397 | io_poll_remove_waitq(poll); |
| 397 | 398 | return 1; |
| 398 | 399 | } |
| @@ -430,7 +431,7 @@ static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, · 31 unmodified lines | ||
| 430 | 431 | else |
| 431 | 432 | req->flags &= ~REQ_F_SINGLE_POLL; |
| 432 | 433 | } |
| 433 | − __io_poll_execute(req, mask); | |
| 434 | + __io_poll_execute(req, mask, IOU_F_TWQ_IN_WAKE); | |
| 434 | 435 | } |
| 435 | 436 | return 1; |
| 436 | 437 | } |
| @@ -618,7 +619,7 @@ static int __io_arm_poll_handler(struct io_kiocb *req, · 181 unmodified lines | ||
| 618 | 619 | |
| 619 | 620 | if (mask && (poll->events & EPOLLET) && |
| 620 | 621 | io_poll_can_finish_inline(req, ipt)) { |
| 621 | − __io_poll_execute(req, mask); | |
| 622 | + __io_poll_execute(req, mask, 0); | |
| 622 | 623 | return 0; |
| 623 | 624 | } |
| 624 | 625 | io_napi_add(req); |
| @@ -629,7 +630,7 @@ static int __io_arm_poll_handler(struct io_kiocb *req, · 4 unmodified lines | ||
| 629 | 630 | * poll was waken up, queue up a tw, it'll deal with it. |
| 630 | 631 | */ |
| 631 | 632 | if (atomic_cmpxchg(&req->poll_refs, 1, 0) != 1) |
| 632 | − __io_poll_execute(req, 0); | |
| 633 | + __io_poll_execute(req, 0, 0); | |
| 633 | 634 | } |
| 634 | 635 | return 0; |
| 635 | 636 | } |
| @@ -170,7 +170,7 @@ void io_req_local_work_add(struct io_kiocb *req, unsigned flags) · 169 unmodified lines | ||
| 170 | 170 | if (mpscq_push(&ctx->work_list, &req->io_task_work.node)) { |
| 171 | 171 | io_ctx_mark_taskrun(ctx); |
| 172 | 172 | if (data_race(ctx->int_flags) & IO_RING_F_HAS_EVFD) |
| 173 | − io_eventfd_signal(ctx, false); | |
| 173 | + io_eventfd_signal(ctx, false, flags & IOU_F_TWQ_IN_WAKE); | |
| 174 | 174 | } |
| 175 | 175 | |
| 176 | 176 | /* |
| @@ -253,7 +253,7 @@ static int io_waitid_wait(struct wait_queue_entry *wait, unsigned mode, · 252 unmodified lines | ||
| 253 | 253 | return 1; |
| 254 | 254 | |
| 255 | 255 | req->io_task_work.func = io_waitid_cb; |
| 256 | − io_req_task_work_add(req); | |
| 256 | + __io_req_task_work_add(req, IOU_F_TWQ_IN_WAKE); | |
| 257 | 257 | return 1; |
| 258 | 258 | } |
| 259 | 259 | |