History

io_uring: defer eventfd signaling when queued from a wakeup handler

io_req_local_work_add() signals the CQ ring eventfd inline when it is the one to push the first entry onto ->work_list. For DEFER_TASKRUN rings that add is frequently done from a waitqueue wakeup handler, where an arbitrary waitqueue lock is held. eventfd_signal_mask() only refuses to recurse when current->in_eventfd is set, but that bit is set by eventfd_signal_mask() itself. If the wake chain starts somewhere else, signal goes out inline and can feed back into epoll. Add IOU_F_TWQ_IN_WAKE, set it on the task_work add done from the three waitqueue callbacks, and use it to force io_eventfd_signal() down the existing call_rcu_hurry() deferral instead of signaling inline.

Jens Axboe committed 3 months ago

Commit cd305ee1 parent 3f3a6a18 files · +30 −21Browse files

Unified · Split

8 files
  1. include/linux/io_uring_types.h +8 −0
  2. io_uring/eventfd.c +4 −4
  3. io_uring/eventfd.h +1 −1
  4. io_uring/futex.c +2 −2
  5. io_uring/io_uring.c +1 −1
  6. io_uring/poll.c +12 −11
  7. io_uring/tw.c +1 −1
  8. io_uring/waitid.c +1 −1
include/linux/io_uring_types.h+8−0
@@ -20,6 +20,14 @@ enum { · 19 unmodified lines
2020 * It's also ignored unless IORING_SETUP_DEFER_TASKRUN is set.
2121 */
2222 IOU_F_TWQ_LAZY_WAKE = 1,
23+
24+ /*
25+ * Set when task_work is queued from a waitqueue wakeup handler, where
26+ * an arbitrary provider waitqueue lock is held. Signaling the CQ ring
27+ * eventfd inline from there can recurse back into that lock through
28+ * epoll, so the eventfd signal must be deferred.
29+ */
30+ IOU_F_TWQ_IN_WAKE = 2,
2331 };
2432
2533 enum io_uring_cmd_flags {
io_uring/eventfd.c+4−4
@@ -51,9 +51,9 @@ static void io_eventfd_do_signal(struct rcu_head *rcu) · 50 unmodified lines
5151 /*
5252 * Returns true if the caller should put the ev_fd reference, false if not.
5353 */
54−static bool __io_eventfd_signal(struct io_ev_fd *ev_fd)
54+static bool __io_eventfd_signal(struct io_ev_fd *ev_fd, bool defer)
5555 {
56− if (eventfd_signal_allowed()) {
56+ if (!defer && eventfd_signal_allowed()) {
5757 eventfd_signal_mask(ev_fd->cq_ev_fd, EPOLL_URING_WAKE);
5858 return true;
5959 }
@@ -73,7 +73,7 @@ static bool io_eventfd_trigger(struct io_ev_fd *ev_fd) · 13 unmodified lines
7373 return !ev_fd->eventfd_async || io_wq_current_is_worker();
7474 }
7575
76−void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event)
76+void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event, bool defer)
7777 {
7878 bool skip = false;
7979 struct io_ev_fd *ev_fd;
@@ -113,7 +113,7 @@ void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event) · 33 unmodified lines
113113 spin_unlock(&ctx->completion_lock);
114114 }
115115
116− if (skip || __io_eventfd_signal(ev_fd))
116+ if (skip || __io_eventfd_signal(ev_fd, defer))
117117 io_eventfd_put(ev_fd);
118118 }
119119
io_uring/eventfd.h+1−1
@@ -5,4 +5,4 @@ int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg, · 4 unmodified lines
55 unsigned int eventfd_async);
66 int io_eventfd_unregister(struct io_ring_ctx *ctx);
77
8−void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event);
8+void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event, bool defer);
io_uring/futex.c+2−2
@@ -181,7 +181,7 @@ static void io_futex_wakev_fn(struct wake_q_head *wake_q, struct futex_q *q) · 180 unmodified lines
181181
182182 io_req_set_res(req, 0, 0);
183183 req->io_task_work.func = io_futexv_complete;
184− io_req_task_work_add(req);
184+ __io_req_task_work_add(req, IOU_F_TWQ_IN_WAKE);
185185 }
186186
187187 int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
@@ -237,7 +237,7 @@ static void io_futex_wake_fn(struct wake_q_head *wake_q, struct futex_q *q) · 49 unmodified lines
237237
238238 io_req_set_res(req, 0, 0);
239239 req->io_task_work.func = io_futex_complete;
240− io_req_task_work_add(req);
240+ __io_req_task_work_add(req, IOU_F_TWQ_IN_WAKE);
241241 }
242242
243243 int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags)
io_uring/io_uring.c+1−1
@@ -475,7 +475,7 @@ void __io_commit_cqring_flush(struct io_ring_ctx *ctx) · 474 unmodified lines
475475 if (ctx->int_flags & IO_RING_F_OFF_TIMEOUT_USED)
476476 io_flush_timeouts(ctx);
477477 if (ctx->int_flags & IO_RING_F_HAS_EVFD)
478− io_eventfd_signal(ctx, true);
478+ io_eventfd_signal(ctx, true, false);
479479 }
480480
481481 static inline void __io_cq_lock(struct io_ring_ctx *ctx)
io_uring/poll.c+12−11
@@ -208,9 +208,9 @@ enum { · 207 unmodified lines
208208 IOU_POLL_REQUEUE = 4,
209209 };
210210
211−static void __io_poll_execute(struct io_kiocb *req, int mask)
211+static void __io_poll_execute(struct io_kiocb *req, int mask, unsigned tw_flags)
212212 {
213− unsigned flags = 0;
213+ unsigned flags = tw_flags;
214214
215215 io_req_set_res(req, mask, 0);
216216 req->io_task_work.func = io_poll_task_func;
@@ -218,14 +218,15 @@ static void __io_poll_execute(struct io_kiocb *req, int mask) · 1 unmodified line
218218 trace_io_uring_task_add(req, mask);
219219
220220 if (!(req->flags & REQ_F_POLL_NO_LAZY))
221− flags = IOU_F_TWQ_LAZY_WAKE;
221+ flags |= IOU_F_TWQ_LAZY_WAKE;
222222 __io_req_task_work_add(req, flags);
223223 }
224224
225−static inline void io_poll_execute(struct io_kiocb *req, int res)
225+static inline void io_poll_execute(struct io_kiocb *req, int res,
226+ unsigned tw_flags)
226227 {
227228 if (io_poll_get_ownership(req))
228− __io_poll_execute(req, res);
229+ __io_poll_execute(req, res, tw_flags);
229230 }
230231
231232 /*
@@ -344,7 +345,7 @@ void io_poll_task_func(struct io_tw_req tw_req, io_tw_token_t tw) · 112 unmodified lines
344345 if (ret == IOU_POLL_NO_ACTION) {
345346 return;
346347 } else if (ret == IOU_POLL_REQUEUE) {
347− __io_poll_execute(req, 0);
348+ __io_poll_execute(req, 0, 0);
348349 return;
349350 }
350351 io_poll_remove_entries(req);
@@ -383,7 +384,7 @@ static void io_poll_cancel_req(struct io_kiocb *req) · 32 unmodified lines
383384 {
384385 io_poll_mark_cancelled(req);
385386 /* kick tw, which should complete the request */
386− io_poll_execute(req, 0);
387+ io_poll_execute(req, 0, 0);
387388 }
388389
389390 #define IO_ASYNC_POLL_COMMON (EPOLLONESHOT | EPOLLPRI)
@@ -392,7 +393,7 @@ static __cold int io_pollfree_wake(struct io_kiocb *req, struct io_poll *poll) · 2 unmodified lines
392393 {
393394 io_poll_mark_cancelled(req);
394395 /* we have to kick tw in case it's not already */
395− io_poll_execute(req, 0);
396+ io_poll_execute(req, 0, IOU_F_TWQ_IN_WAKE);
396397 io_poll_remove_waitq(poll);
397398 return 1;
398399 }
@@ -430,7 +431,7 @@ static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, · 31 unmodified lines
430431 else
431432 req->flags &= ~REQ_F_SINGLE_POLL;
432433 }
433− __io_poll_execute(req, mask);
434+ __io_poll_execute(req, mask, IOU_F_TWQ_IN_WAKE);
434435 }
435436 return 1;
436437 }
@@ -618,7 +619,7 @@ static int __io_arm_poll_handler(struct io_kiocb *req, · 181 unmodified lines
618619
619620 if (mask && (poll->events & EPOLLET) &&
620621 io_poll_can_finish_inline(req, ipt)) {
621− __io_poll_execute(req, mask);
622+ __io_poll_execute(req, mask, 0);
622623 return 0;
623624 }
624625 io_napi_add(req);
@@ -629,7 +630,7 @@ static int __io_arm_poll_handler(struct io_kiocb *req, · 4 unmodified lines
629630 * poll was waken up, queue up a tw, it'll deal with it.
630631 */
631632 if (atomic_cmpxchg(&req->poll_refs, 1, 0) != 1)
632− __io_poll_execute(req, 0);
633+ __io_poll_execute(req, 0, 0);
633634 }
634635 return 0;
635636 }
io_uring/tw.c+1−1
@@ -170,7 +170,7 @@ void io_req_local_work_add(struct io_kiocb *req, unsigned flags) · 169 unmodified lines
170170 if (mpscq_push(&ctx->work_list, &req->io_task_work.node)) {
171171 io_ctx_mark_taskrun(ctx);
172172 if (data_race(ctx->int_flags) & IO_RING_F_HAS_EVFD)
173− io_eventfd_signal(ctx, false);
173+ io_eventfd_signal(ctx, false, flags & IOU_F_TWQ_IN_WAKE);
174174 }
175175
176176 /*
io_uring/waitid.c+1−1
@@ -253,7 +253,7 @@ static int io_waitid_wait(struct wait_queue_entry *wait, unsigned mode, · 252 unmodified lines
253253 return 1;
254254
255255 req->io_task_work.func = io_waitid_cb;
256− io_req_task_work_add(req);
256+ __io_req_task_work_add(req, IOU_F_TWQ_IN_WAKE);
257257 return 1;
258258 }
259259

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t