io_req_local_work_add() signals the CQ ring eventfd inline when it is the one to push the first entry onto ->work_list. For DEFER_TASKRUN rings that add is frequently done from a waitqueue wakeup handler, where an arbitrary waitqueue lock is held. eventfd_signal_mask() only refuses to recurse when current->in_eventfd is set, but that bit is set by eventfd_signal_mask() itself. If the wake chain starts somewhere else, signal goes out inline and can feed back into epoll. Add IOU_F_TWQ_IN_WAKE, set it on the task_work add done from the three waitqueue callbacks, and use it to force io_eventfd_signal() down the existing call_rcu_hurry() deferral instead of signaling inline.
Unified · Split
| @@ -20,6 +20,14 @@ enum { · 19 unmodified lines | |||
| 20 | * It's also ignored unless IORING_SETUP_DEFER_TASKRUN is set. | 20 | * It's also ignored unless IORING_SETUP_DEFER_TASKRUN is set. |
| 21 | */ | 21 | */ |
| 22 | IOU_F_TWQ_LAZY_WAKE = 1, | 22 | IOU_F_TWQ_LAZY_WAKE = 1, |
| 23 | |||
| 24 | /* | ||
| 25 | * Set when task_work is queued from a waitqueue wakeup handler, where | ||
| 26 | * an arbitrary provider waitqueue lock is held. Signaling the CQ ring | ||
| 27 | * eventfd inline from there can recurse back into that lock through | ||
| 28 | * epoll, so the eventfd signal must be deferred. | ||
| 29 | */ | ||
| 30 | IOU_F_TWQ_IN_WAKE = 2, | ||
| 23 | }; | 31 | }; |
| 24 | 32 | ||
| 25 | enum io_uring_cmd_flags { | 33 | enum io_uring_cmd_flags { |
| @@ -51,9 +51,9 @@ static void io_eventfd_do_signal(struct rcu_head *rcu) · 50 unmodified lines | |||
| 51 | /* | 51 | /* |
| 52 | * Returns true if the caller should put the ev_fd reference, false if not. | 52 | * Returns true if the caller should put the ev_fd reference, false if not. |
| 53 | */ | 53 | */ |
| 54 | static bool __io_eventfd_signal(struct io_ev_fd *ev_fd) | 54 | static bool __io_eventfd_signal(struct io_ev_fd *ev_fd, bool defer) |
| 55 | { | 55 | { |
| 56 | if (eventfd_signal_allowed()) { | 56 | if (!defer && eventfd_signal_allowed()) { |
| 57 | eventfd_signal_mask(ev_fd->cq_ev_fd, EPOLL_URING_WAKE); | 57 | eventfd_signal_mask(ev_fd->cq_ev_fd, EPOLL_URING_WAKE); |
| 58 | return true; | 58 | return true; |
| 59 | } | 59 | } |
| @@ -73,7 +73,7 @@ static bool io_eventfd_trigger(struct io_ev_fd *ev_fd) · 13 unmodified lines | |||
| 73 | return !ev_fd->eventfd_async || io_wq_current_is_worker(); | 73 | return !ev_fd->eventfd_async || io_wq_current_is_worker(); |
| 74 | } | 74 | } |
| 75 | 75 | ||
| 76 | void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event) | 76 | void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event, bool defer) |
| 77 | { | 77 | { |
| 78 | bool skip = false; | 78 | bool skip = false; |
| 79 | struct io_ev_fd *ev_fd; | 79 | struct io_ev_fd *ev_fd; |
| @@ -113,7 +113,7 @@ void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event) · 33 unmodified lines | |||
| 113 | spin_unlock(&ctx->completion_lock); | 113 | spin_unlock(&ctx->completion_lock); |
| 114 | } | 114 | } |
| 115 | 115 | ||
| 116 | if (skip || __io_eventfd_signal(ev_fd)) | 116 | if (skip || __io_eventfd_signal(ev_fd, defer)) |
| 117 | io_eventfd_put(ev_fd); | 117 | io_eventfd_put(ev_fd); |
| 118 | } | 118 | } |
| 119 | 119 | ||
| @@ -5,4 +5,4 @@ int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg, · 4 unmodified lines | |||
| 5 | unsigned int eventfd_async); | 5 | unsigned int eventfd_async); |
| 6 | int io_eventfd_unregister(struct io_ring_ctx *ctx); | 6 | int io_eventfd_unregister(struct io_ring_ctx *ctx); |
| 7 | 7 | ||
| 8 | void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event); | 8 | void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event, bool defer); |
| @@ -181,7 +181,7 @@ static void io_futex_wakev_fn(struct wake_q_head *wake_q, struct futex_q *q) · 180 unmodified lines | |||
| 181 | 181 | ||
| 182 | io_req_set_res(req, 0, 0); | 182 | io_req_set_res(req, 0, 0); |
| 183 | req->io_task_work.func = io_futexv_complete; | 183 | req->io_task_work.func = io_futexv_complete; |
| 184 | io_req_task_work_add(req); | 184 | __io_req_task_work_add(req, IOU_F_TWQ_IN_WAKE); |
| 185 | } | 185 | } |
| 186 | 186 | ||
| 187 | int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) | 187 | int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) |
| @@ -237,7 +237,7 @@ static void io_futex_wake_fn(struct wake_q_head *wake_q, struct futex_q *q) · 49 unmodified lines | |||
| 237 | 237 | ||
| 238 | io_req_set_res(req, 0, 0); | 238 | io_req_set_res(req, 0, 0); |
| 239 | req->io_task_work.func = io_futex_complete; | 239 | req->io_task_work.func = io_futex_complete; |
| 240 | io_req_task_work_add(req); | 240 | __io_req_task_work_add(req, IOU_F_TWQ_IN_WAKE); |
| 241 | } | 241 | } |
| 242 | 242 | ||
| 243 | int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags) | 243 | int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags) |
| @@ -475,7 +475,7 @@ void __io_commit_cqring_flush(struct io_ring_ctx *ctx) · 474 unmodified lines | |||
| 475 | if (ctx->int_flags & IO_RING_F_OFF_TIMEOUT_USED) | 475 | if (ctx->int_flags & IO_RING_F_OFF_TIMEOUT_USED) |
| 476 | io_flush_timeouts(ctx); | 476 | io_flush_timeouts(ctx); |
| 477 | if (ctx->int_flags & IO_RING_F_HAS_EVFD) | 477 | if (ctx->int_flags & IO_RING_F_HAS_EVFD) |
| 478 | io_eventfd_signal(ctx, true); | 478 | io_eventfd_signal(ctx, true, false); |
| 479 | } | 479 | } |
| 480 | 480 | ||
| 481 | static inline void __io_cq_lock(struct io_ring_ctx *ctx) | 481 | static inline void __io_cq_lock(struct io_ring_ctx *ctx) |
| @@ -208,9 +208,9 @@ enum { · 207 unmodified lines | |||
| 208 | IOU_POLL_REQUEUE = 4, | 208 | IOU_POLL_REQUEUE = 4, |
| 209 | }; | 209 | }; |
| 210 | 210 | ||
| 211 | static void __io_poll_execute(struct io_kiocb *req, int mask) | 211 | static void __io_poll_execute(struct io_kiocb *req, int mask, unsigned tw_flags) |
| 212 | { | 212 | { |
| 213 | unsigned flags = 0; | 213 | unsigned flags = tw_flags; |
| 214 | 214 | ||
| 215 | io_req_set_res(req, mask, 0); | 215 | io_req_set_res(req, mask, 0); |
| 216 | req->io_task_work.func = io_poll_task_func; | 216 | req->io_task_work.func = io_poll_task_func; |
| @@ -218,14 +218,15 @@ static void __io_poll_execute(struct io_kiocb *req, int mask) · 1 unmodified line | |||
| 218 | trace_io_uring_task_add(req, mask); | 218 | trace_io_uring_task_add(req, mask); |
| 219 | 219 | ||
| 220 | if (!(req->flags & REQ_F_POLL_NO_LAZY)) | 220 | if (!(req->flags & REQ_F_POLL_NO_LAZY)) |
| 221 | flags = IOU_F_TWQ_LAZY_WAKE; | 221 | flags |= IOU_F_TWQ_LAZY_WAKE; |
| 222 | __io_req_task_work_add(req, flags); | 222 | __io_req_task_work_add(req, flags); |
| 223 | } | 223 | } |
| 224 | 224 | ||
| 225 | static inline void io_poll_execute(struct io_kiocb *req, int res) | 225 | static inline void io_poll_execute(struct io_kiocb *req, int res, |
| 226 | unsigned tw_flags) | ||
| 226 | { | 227 | { |
| 227 | if (io_poll_get_ownership(req)) | 228 | if (io_poll_get_ownership(req)) |
| 228 | __io_poll_execute(req, res); | 229 | __io_poll_execute(req, res, tw_flags); |
| 229 | } | 230 | } |
| 230 | 231 | ||
| 231 | /* | 232 | /* |
| @@ -344,7 +345,7 @@ void io_poll_task_func(struct io_tw_req tw_req, io_tw_token_t tw) · 112 unmodified lines | |||
| 344 | if (ret == IOU_POLL_NO_ACTION) { | 345 | if (ret == IOU_POLL_NO_ACTION) { |
| 345 | return; | 346 | return; |
| 346 | } else if (ret == IOU_POLL_REQUEUE) { | 347 | } else if (ret == IOU_POLL_REQUEUE) { |
| 347 | __io_poll_execute(req, 0); | 348 | __io_poll_execute(req, 0, 0); |
| 348 | return; | 349 | return; |
| 349 | } | 350 | } |
| 350 | io_poll_remove_entries(req); | 351 | io_poll_remove_entries(req); |
| @@ -383,7 +384,7 @@ static void io_poll_cancel_req(struct io_kiocb *req) · 32 unmodified lines | |||
| 383 | { | 384 | { |
| 384 | io_poll_mark_cancelled(req); | 385 | io_poll_mark_cancelled(req); |
| 385 | /* kick tw, which should complete the request */ | 386 | /* kick tw, which should complete the request */ |
| 386 | io_poll_execute(req, 0); | 387 | io_poll_execute(req, 0, 0); |
| 387 | } | 388 | } |
| 388 | 389 | ||
| 389 | #define IO_ASYNC_POLL_COMMON (EPOLLONESHOT | EPOLLPRI) | 390 | #define IO_ASYNC_POLL_COMMON (EPOLLONESHOT | EPOLLPRI) |
| @@ -392,7 +393,7 @@ static __cold int io_pollfree_wake(struct io_kiocb *req, struct io_poll *poll) · 2 unmodified lines | |||
| 392 | { | 393 | { |
| 393 | io_poll_mark_cancelled(req); | 394 | io_poll_mark_cancelled(req); |
| 394 | /* we have to kick tw in case it's not already */ | 395 | /* we have to kick tw in case it's not already */ |
| 395 | io_poll_execute(req, 0); | 396 | io_poll_execute(req, 0, IOU_F_TWQ_IN_WAKE); |
| 396 | io_poll_remove_waitq(poll); | 397 | io_poll_remove_waitq(poll); |
| 397 | return 1; | 398 | return 1; |
| 398 | } | 399 | } |
| @@ -430,7 +431,7 @@ static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, · 31 unmodified lines | |||
| 430 | else | 431 | else |
| 431 | req->flags &= ~REQ_F_SINGLE_POLL; | 432 | req->flags &= ~REQ_F_SINGLE_POLL; |
| 432 | } | 433 | } |
| 433 | __io_poll_execute(req, mask); | 434 | __io_poll_execute(req, mask, IOU_F_TWQ_IN_WAKE); |
| 434 | } | 435 | } |
| 435 | return 1; | 436 | return 1; |
| 436 | } | 437 | } |
| @@ -618,7 +619,7 @@ static int __io_arm_poll_handler(struct io_kiocb *req, · 181 unmodified lines | |||
| 618 | 619 | ||
| 619 | if (mask && (poll->events & EPOLLET) && | 620 | if (mask && (poll->events & EPOLLET) && |
| 620 | io_poll_can_finish_inline(req, ipt)) { | 621 | io_poll_can_finish_inline(req, ipt)) { |
| 621 | __io_poll_execute(req, mask); | 622 | __io_poll_execute(req, mask, 0); |
| 622 | return 0; | 623 | return 0; |
| 623 | } | 624 | } |
| 624 | io_napi_add(req); | 625 | io_napi_add(req); |
| @@ -629,7 +630,7 @@ static int __io_arm_poll_handler(struct io_kiocb *req, · 4 unmodified lines | |||
| 629 | * poll was waken up, queue up a tw, it'll deal with it. | 630 | * poll was waken up, queue up a tw, it'll deal with it. |
| 630 | */ | 631 | */ |
| 631 | if (atomic_cmpxchg(&req->poll_refs, 1, 0) != 1) | 632 | if (atomic_cmpxchg(&req->poll_refs, 1, 0) != 1) |
| 632 | __io_poll_execute(req, 0); | 633 | __io_poll_execute(req, 0, 0); |
| 633 | } | 634 | } |
| 634 | return 0; | 635 | return 0; |
| 635 | } | 636 | } |
| @@ -170,7 +170,7 @@ void io_req_local_work_add(struct io_kiocb *req, unsigned flags) · 169 unmodified lines | |||
| 170 | if (mpscq_push(&ctx->work_list, &req->io_task_work.node)) { | 170 | if (mpscq_push(&ctx->work_list, &req->io_task_work.node)) { |
| 171 | io_ctx_mark_taskrun(ctx); | 171 | io_ctx_mark_taskrun(ctx); |
| 172 | if (data_race(ctx->int_flags) & IO_RING_F_HAS_EVFD) | 172 | if (data_race(ctx->int_flags) & IO_RING_F_HAS_EVFD) |
| 173 | io_eventfd_signal(ctx, false); | 173 | io_eventfd_signal(ctx, false, flags & IOU_F_TWQ_IN_WAKE); |
| 174 | } | 174 | } |
| 175 | 175 | ||
| 176 | /* | 176 | /* |
| @@ -253,7 +253,7 @@ static int io_waitid_wait(struct wait_queue_entry *wait, unsigned mode, · 252 unmodified lines | |||
| 253 | return 1; | 253 | return 1; |
| 254 | 254 | ||
| 255 | req->io_task_work.func = io_waitid_cb; | 255 | req->io_task_work.func = io_waitid_cb; |
| 256 | io_req_task_work_add(req); | 256 | __io_req_task_work_add(req, IOU_F_TWQ_IN_WAKE); |
| 257 | return 1; | 257 | return 1; |
| 258 | } | 258 | } |
| 259 | 259 | ||