History

io_uring: defer eventfd signaling when queued from a wakeup handler

io_req_local_work_add() signals the CQ ring eventfd inline when it is the one to push the first entry onto ->work_list. For DEFER_TASKRUN rings that add is frequently done from a waitqueue wakeup handler, where an arbitrary waitqueue lock is held. eventfd_signal_mask() only refuses to recurse when current->in_eventfd is set, but that bit is set by eventfd_signal_mask() itself. If the wake chain starts somewhere else, signal goes out inline and can feed back into epoll. Add IOU_F_TWQ_IN_WAKE, set it on the task_work add done from the three waitqueue callbacks, and use it to force io_eventfd_signal() down the existing call_rcu_hurry() deferral instead of signaling inline.

Jens Axboe committed 3 months ago

Commit cd305ee1 parent 3f3a6a18 files · +30 −21Browse files

Unified · Split

8 files
  1. include/linux/io_uring_types.h +8 −0
  2. io_uring/eventfd.c +4 −4
  3. io_uring/eventfd.h +1 −1
  4. io_uring/futex.c +2 −2
  5. io_uring/io_uring.c +1 −1
  6. io_uring/poll.c +12 −11
  7. io_uring/tw.c +1 −1
  8. io_uring/waitid.c +1 −1
include/linux/io_uring_types.h+8−0
@@ -20,6 +20,14 @@ enum { · 19 unmodified lines
20 * It's also ignored unless IORING_SETUP_DEFER_TASKRUN is set.20 * It's also ignored unless IORING_SETUP_DEFER_TASKRUN is set.
21 */21 */
22 IOU_F_TWQ_LAZY_WAKE = 1,22 IOU_F_TWQ_LAZY_WAKE = 1,
23
24 /*
25 * Set when task_work is queued from a waitqueue wakeup handler, where
26 * an arbitrary provider waitqueue lock is held. Signaling the CQ ring
27 * eventfd inline from there can recurse back into that lock through
28 * epoll, so the eventfd signal must be deferred.
29 */
30 IOU_F_TWQ_IN_WAKE = 2,
23};31};
24 32
25enum io_uring_cmd_flags {33enum io_uring_cmd_flags {
io_uring/eventfd.c+4−4
@@ -51,9 +51,9 @@ static void io_eventfd_do_signal(struct rcu_head *rcu) · 50 unmodified lines
51/*51/*
52 * Returns true if the caller should put the ev_fd reference, false if not.52 * Returns true if the caller should put the ev_fd reference, false if not.
53 */53 */
54static bool __io_eventfd_signal(struct io_ev_fd *ev_fd)54static bool __io_eventfd_signal(struct io_ev_fd *ev_fd, bool defer)
55{55{
56 if (eventfd_signal_allowed()) {56 if (!defer && eventfd_signal_allowed()) {
57 eventfd_signal_mask(ev_fd->cq_ev_fd, EPOLL_URING_WAKE);57 eventfd_signal_mask(ev_fd->cq_ev_fd, EPOLL_URING_WAKE);
58 return true;58 return true;
59 }59 }
@@ -73,7 +73,7 @@ static bool io_eventfd_trigger(struct io_ev_fd *ev_fd) · 13 unmodified lines
73 return !ev_fd->eventfd_async || io_wq_current_is_worker();73 return !ev_fd->eventfd_async || io_wq_current_is_worker();
74}74}
75 75
76void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event)76void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event, bool defer)
77{77{
78 bool skip = false;78 bool skip = false;
79 struct io_ev_fd *ev_fd;79 struct io_ev_fd *ev_fd;
@@ -113,7 +113,7 @@ void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event) · 33 unmodified lines
113 spin_unlock(&ctx->completion_lock);113 spin_unlock(&ctx->completion_lock);
114 }114 }
115 115
116 if (skip || __io_eventfd_signal(ev_fd))116 if (skip || __io_eventfd_signal(ev_fd, defer))
117 io_eventfd_put(ev_fd);117 io_eventfd_put(ev_fd);
118}118}
119 119
io_uring/eventfd.h+1−1
@@ -5,4 +5,4 @@ int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg, · 4 unmodified lines
5 unsigned int eventfd_async);5 unsigned int eventfd_async);
6int io_eventfd_unregister(struct io_ring_ctx *ctx);6int io_eventfd_unregister(struct io_ring_ctx *ctx);
7 7
8void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event);8void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event, bool defer);
io_uring/futex.c+2−2
@@ -181,7 +181,7 @@ static void io_futex_wakev_fn(struct wake_q_head *wake_q, struct futex_q *q) · 180 unmodified lines
181 181
182 io_req_set_res(req, 0, 0);182 io_req_set_res(req, 0, 0);
183 req->io_task_work.func = io_futexv_complete;183 req->io_task_work.func = io_futexv_complete;
184 io_req_task_work_add(req);184 __io_req_task_work_add(req, IOU_F_TWQ_IN_WAKE);
185}185}
186 186
187int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)187int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
@@ -237,7 +237,7 @@ static void io_futex_wake_fn(struct wake_q_head *wake_q, struct futex_q *q) · 49 unmodified lines
237 237
238 io_req_set_res(req, 0, 0);238 io_req_set_res(req, 0, 0);
239 req->io_task_work.func = io_futex_complete;239 req->io_task_work.func = io_futex_complete;
240 io_req_task_work_add(req);240 __io_req_task_work_add(req, IOU_F_TWQ_IN_WAKE);
241}241}
242 242
243int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags)243int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags)
io_uring/io_uring.c+1−1
@@ -475,7 +475,7 @@ void __io_commit_cqring_flush(struct io_ring_ctx *ctx) · 474 unmodified lines
475 if (ctx->int_flags & IO_RING_F_OFF_TIMEOUT_USED)475 if (ctx->int_flags & IO_RING_F_OFF_TIMEOUT_USED)
476 io_flush_timeouts(ctx);476 io_flush_timeouts(ctx);
477 if (ctx->int_flags & IO_RING_F_HAS_EVFD)477 if (ctx->int_flags & IO_RING_F_HAS_EVFD)
478 io_eventfd_signal(ctx, true);478 io_eventfd_signal(ctx, true, false);
479}479}
480 480
481static inline void __io_cq_lock(struct io_ring_ctx *ctx)481static inline void __io_cq_lock(struct io_ring_ctx *ctx)
io_uring/poll.c+12−11
@@ -208,9 +208,9 @@ enum { · 207 unmodified lines
208 IOU_POLL_REQUEUE = 4,208 IOU_POLL_REQUEUE = 4,
209};209};
210 210
211static void __io_poll_execute(struct io_kiocb *req, int mask)211static void __io_poll_execute(struct io_kiocb *req, int mask, unsigned tw_flags)
212{212{
213 unsigned flags = 0;213 unsigned flags = tw_flags;
214 214
215 io_req_set_res(req, mask, 0);215 io_req_set_res(req, mask, 0);
216 req->io_task_work.func = io_poll_task_func;216 req->io_task_work.func = io_poll_task_func;
@@ -218,14 +218,15 @@ static void __io_poll_execute(struct io_kiocb *req, int mask) · 1 unmodified line
218 trace_io_uring_task_add(req, mask);218 trace_io_uring_task_add(req, mask);
219 219
220 if (!(req->flags & REQ_F_POLL_NO_LAZY))220 if (!(req->flags & REQ_F_POLL_NO_LAZY))
221 flags = IOU_F_TWQ_LAZY_WAKE;221 flags |= IOU_F_TWQ_LAZY_WAKE;
222 __io_req_task_work_add(req, flags);222 __io_req_task_work_add(req, flags);
223}223}
224 224
225static inline void io_poll_execute(struct io_kiocb *req, int res)225static inline void io_poll_execute(struct io_kiocb *req, int res,
226 unsigned tw_flags)
226{227{
227 if (io_poll_get_ownership(req))228 if (io_poll_get_ownership(req))
228 __io_poll_execute(req, res);229 __io_poll_execute(req, res, tw_flags);
229}230}
230 231
231/*232/*
@@ -344,7 +345,7 @@ void io_poll_task_func(struct io_tw_req tw_req, io_tw_token_t tw) · 112 unmodified lines
344 if (ret == IOU_POLL_NO_ACTION) {345 if (ret == IOU_POLL_NO_ACTION) {
345 return;346 return;
346 } else if (ret == IOU_POLL_REQUEUE) {347 } else if (ret == IOU_POLL_REQUEUE) {
347 __io_poll_execute(req, 0);348 __io_poll_execute(req, 0, 0);
348 return;349 return;
349 }350 }
350 io_poll_remove_entries(req);351 io_poll_remove_entries(req);
@@ -383,7 +384,7 @@ static void io_poll_cancel_req(struct io_kiocb *req) · 32 unmodified lines
383{384{
384 io_poll_mark_cancelled(req);385 io_poll_mark_cancelled(req);
385 /* kick tw, which should complete the request */386 /* kick tw, which should complete the request */
386 io_poll_execute(req, 0);387 io_poll_execute(req, 0, 0);
387}388}
388 389
389#define IO_ASYNC_POLL_COMMON (EPOLLONESHOT | EPOLLPRI)390#define IO_ASYNC_POLL_COMMON (EPOLLONESHOT | EPOLLPRI)
@@ -392,7 +393,7 @@ static __cold int io_pollfree_wake(struct io_kiocb *req, struct io_poll *poll) · 2 unmodified lines
392{393{
393 io_poll_mark_cancelled(req);394 io_poll_mark_cancelled(req);
394 /* we have to kick tw in case it's not already */395 /* we have to kick tw in case it's not already */
395 io_poll_execute(req, 0);396 io_poll_execute(req, 0, IOU_F_TWQ_IN_WAKE);
396 io_poll_remove_waitq(poll);397 io_poll_remove_waitq(poll);
397 return 1;398 return 1;
398}399}
@@ -430,7 +431,7 @@ static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, · 31 unmodified lines
430 else431 else
431 req->flags &= ~REQ_F_SINGLE_POLL;432 req->flags &= ~REQ_F_SINGLE_POLL;
432 }433 }
433 __io_poll_execute(req, mask);434 __io_poll_execute(req, mask, IOU_F_TWQ_IN_WAKE);
434 }435 }
435 return 1;436 return 1;
436}437}
@@ -618,7 +619,7 @@ static int __io_arm_poll_handler(struct io_kiocb *req, · 181 unmodified lines
618 619
619 if (mask && (poll->events & EPOLLET) &&620 if (mask && (poll->events & EPOLLET) &&
620 io_poll_can_finish_inline(req, ipt)) {621 io_poll_can_finish_inline(req, ipt)) {
621 __io_poll_execute(req, mask);622 __io_poll_execute(req, mask, 0);
622 return 0;623 return 0;
623 }624 }
624 io_napi_add(req);625 io_napi_add(req);
@@ -629,7 +630,7 @@ static int __io_arm_poll_handler(struct io_kiocb *req, · 4 unmodified lines
629 * poll was waken up, queue up a tw, it'll deal with it.630 * poll was waken up, queue up a tw, it'll deal with it.
630 */631 */
631 if (atomic_cmpxchg(&req->poll_refs, 1, 0) != 1)632 if (atomic_cmpxchg(&req->poll_refs, 1, 0) != 1)
632 __io_poll_execute(req, 0);633 __io_poll_execute(req, 0, 0);
633 }634 }
634 return 0;635 return 0;
635}636}
io_uring/tw.c+1−1
@@ -170,7 +170,7 @@ void io_req_local_work_add(struct io_kiocb *req, unsigned flags) · 169 unmodified lines
170 if (mpscq_push(&ctx->work_list, &req->io_task_work.node)) {170 if (mpscq_push(&ctx->work_list, &req->io_task_work.node)) {
171 io_ctx_mark_taskrun(ctx);171 io_ctx_mark_taskrun(ctx);
172 if (data_race(ctx->int_flags) & IO_RING_F_HAS_EVFD)172 if (data_race(ctx->int_flags) & IO_RING_F_HAS_EVFD)
173 io_eventfd_signal(ctx, false);173 io_eventfd_signal(ctx, false, flags & IOU_F_TWQ_IN_WAKE);
174 }174 }
175 175
176 /*176 /*
io_uring/waitid.c+1−1
@@ -253,7 +253,7 @@ static int io_waitid_wait(struct wait_queue_entry *wait, unsigned mode, · 252 unmodified lines
253 return 1;253 return 1;
254 254
255 req->io_task_work.func = io_waitid_cb;255 req->io_task_work.func = io_waitid_cb;
256 io_req_task_work_add(req);256 __io_req_task_work_add(req, IOU_F_TWQ_IN_WAKE);
257 return 1;257 return 1;
258}258}
259 259

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t