History

Merge tag 'io_uring-6.1-2022-11-11' of git://git.kernel.dk/linux

Pull io_uring fixes from Jens Axboe: "Nothing major, just a few minor tweaks: - Tweak for the TCP zero-copy io_uring self test (Pavel) - Rather than use our internal cached value of number of CQ events available, use what the user can see (Dylan) - Fix a typo in a comment, added in this release (me) - Don't allow wrapping while adding provided buffers (me) - Fix a double poll race, and add a lockdep assertion for it too (Pavel)" * tag 'io_uring-6.1-2022-11-11' of git://git.kernel.dk/linux: io_uring/poll: lockdep annote io_poll_req_insert_locked io_uring/poll: fix double poll req->flags races io_uring: check for rollover of buffer ID when providing buffers io_uring: calculate CQEs from the user visible value io_uring: fix typo in io_uring.h comment selftests/net: don't tests batched TCP io_uring zc

Linus Torvalds committed 4 years ago

Commit 4e6b2b22 parents f5020a0 55760355 files · +31 −16Browse files

Unified · Split

5 files
  1. include/uapi/linux/io_uring.h +1 −1
  2. io_uring/io_uring.c +8 −2
  3. io_uring/kbuf.c +2 −0
  4. io_uring/poll.c +19 −12
  5. tools/testing/selftests/net/io_uring_zerocopy_tx.sh +1 −1
include/uapi/linux/io_uring.h+1−1
@@ -222,7 +222,7 @@ enum io_uring_op { · 221 unmodified lines
222 222
223/*223/*
224 * sqe->uring_cmd_flags224 * sqe->uring_cmd_flags
225 * IORING_URING_CMD_FIXED use registered buffer; pass thig flag225 * IORING_URING_CMD_FIXED use registered buffer; pass this flag
226 * along with setting sqe->buf_index.226 * along with setting sqe->buf_index.
227 */227 */
228#define IORING_URING_CMD_FIXED (1U << 0)228#define IORING_URING_CMD_FIXED (1U << 0)
io_uring/io_uring.c+8−2
@@ -176,6 +176,11 @@ static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx) · 175 unmodified lines
176 return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head);176 return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head);
177}177}
178 178
179static inline unsigned int __io_cqring_events_user(struct io_ring_ctx *ctx)
180{
181 return READ_ONCE(ctx->rings->cq.tail) - READ_ONCE(ctx->rings->cq.head);
182}
183
179static bool io_match_linked(struct io_kiocb *head)184static bool io_match_linked(struct io_kiocb *head)
180{185{
181 struct io_kiocb *req;186 struct io_kiocb *req;
@@ -2315,7 +2320,7 @@ static inline bool io_has_work(struct io_ring_ctx *ctx) · 2133 unmodified lines
2315static inline bool io_should_wake(struct io_wait_queue *iowq)2320static inline bool io_should_wake(struct io_wait_queue *iowq)
2316{2321{
2317 struct io_ring_ctx *ctx = iowq->ctx;2322 struct io_ring_ctx *ctx = iowq->ctx;
2318 int dist = ctx->cached_cq_tail - (int) iowq->cq_tail;2323 int dist = READ_ONCE(ctx->rings->cq.tail) - (int) iowq->cq_tail;
2319 2324
2320 /*2325 /*
2321 * Wake up if we have enough events, or if a timeout occurred since we2326 * Wake up if we have enough events, or if a timeout occurred since we
@@ -2399,7 +2404,8 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, · 77 unmodified lines
2399 return ret;2404 return ret;
2400 io_cqring_overflow_flush(ctx);2405 io_cqring_overflow_flush(ctx);
2401 2406
2402 if (io_cqring_events(ctx) >= min_events)2407 /* if user messes with these they will just get an early return */
2408 if (__io_cqring_events_user(ctx) >= min_events)
2403 return 0;2409 return 0;
2404 } while (ret > 0);2410 } while (ret > 0);
2405 2411
io_uring/kbuf.c+2−0
@@ -346,6 +346,8 @@ int io_provide_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe · 345 unmodified lines
346 tmp = READ_ONCE(sqe->off);346 tmp = READ_ONCE(sqe->off);
347 if (tmp > USHRT_MAX)347 if (tmp > USHRT_MAX)
348 return -E2BIG;348 return -E2BIG;
349 if (tmp + p->nbufs >= USHRT_MAX)
350 return -EINVAL;
349 p->bid = tmp;351 p->bid = tmp;
350 return 0;352 return 0;
351}353}
io_uring/poll.c+19−12
@@ -116,6 +116,8 @@ static void io_poll_req_insert_locked(struct io_kiocb *req) · 115 unmodified lines
116 struct io_hash_table *table = &req->ctx->cancel_table_locked;116 struct io_hash_table *table = &req->ctx->cancel_table_locked;
117 u32 index = hash_long(req->cqe.user_data, table->hash_bits);117 u32 index = hash_long(req->cqe.user_data, table->hash_bits);
118 118
119 lockdep_assert_held(&req->ctx->uring_lock);
120
119 hlist_add_head(&req->hash_node, &table->hbs[index].list);121 hlist_add_head(&req->hash_node, &table->hbs[index].list);
120}122}
121 123
@@ -394,7 +396,8 @@ static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, · 272 unmodified lines
394 return 1;396 return 1;
395}397}
396 398
397static void io_poll_double_prepare(struct io_kiocb *req)399/* fails only when polling is already completing by the first entry */
400static bool io_poll_double_prepare(struct io_kiocb *req)
398{401{
399 struct wait_queue_head *head;402 struct wait_queue_head *head;
400 struct io_poll *poll = io_poll_get_single(req);403 struct io_poll *poll = io_poll_get_single(req);
@@ -403,20 +406,20 @@ static void io_poll_double_prepare(struct io_kiocb *req) · 2 unmodified lines
403 rcu_read_lock();406 rcu_read_lock();
404 head = smp_load_acquire(&poll->head);407 head = smp_load_acquire(&poll->head);
405 /*408 /*
406 * poll arm may not hold ownership and so race with409 * poll arm might not hold ownership and so race for req->flags with
407 * io_poll_wake() by modifying req->flags. There is only one410 * io_poll_wake(). There is only one poll entry queued, serialise with
408 * poll entry queued, serialise with it by taking its head lock.411 * it by taking its head lock. As we're still arming the tw hanlder
412 * is not going to be run, so there are no races with it.
409 */413 */
410 if (head)414 if (head) {
411 spin_lock_irq(&head->lock);415 spin_lock_irq(&head->lock);
412 416 req->flags |= REQ_F_DOUBLE_POLL;
413 req->flags |= REQ_F_DOUBLE_POLL;417 if (req->opcode == IORING_OP_POLL_ADD)
414 if (req->opcode == IORING_OP_POLL_ADD)418 req->flags |= REQ_F_ASYNC_DATA;
415 req->flags |= REQ_F_ASYNC_DATA;
416
417 if (head)
418 spin_unlock_irq(&head->lock);419 spin_unlock_irq(&head->lock);
420 }
419 rcu_read_unlock();421 rcu_read_unlock();
422 return !!head;
420}423}
421 424
422static void __io_queue_proc(struct io_poll *poll, struct io_poll_table *pt,425static void __io_queue_proc(struct io_poll *poll, struct io_poll_table *pt,
@@ -454,7 +457,11 @@ static void __io_queue_proc(struct io_poll *poll, struct io_poll_table *pt, · 31 unmodified lines
454 /* mark as double wq entry */457 /* mark as double wq entry */
455 wqe_private |= IO_WQE_F_DOUBLE;458 wqe_private |= IO_WQE_F_DOUBLE;
456 io_init_poll_iocb(poll, first->events, first->wait.func);459 io_init_poll_iocb(poll, first->events, first->wait.func);
457 io_poll_double_prepare(req);460 if (!io_poll_double_prepare(req)) {
461 /* the request is completing, just back off */
462 kfree(poll);
463 return;
464 }
458 *poll_ptr = poll;465 *poll_ptr = poll;
459 } else {466 } else {
460 /* fine to modify, there is no poll queued to race with us */467 /* fine to modify, there is no poll queued to race with us */
tools/testing/selftests/net/io_uring_zerocopy_tx.sh+1−1
@@ -29,7 +29,7 @@ if [[ "$#" -eq "0" ]]; then · 28 unmodified lines
29 for IP in "${IPs[@]}"; do29 for IP in "${IPs[@]}"; do
30 for mode in $(seq 1 3); do30 for mode in $(seq 1 3); do
31 $0 "$IP" udp -m "$mode" -t 1 -n 3231 $0 "$IP" udp -m "$mode" -t 1 -n 32
32 $0 "$IP" tcp -m "$mode" -t 1 -n 3232 $0 "$IP" tcp -m "$mode" -t 1 -n 1
33 done33 done
34 done34 done
35 35