History

Merge tag 'io_uring-7.0-20260403' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux

Pull io_uring fixes from Jens Axboe: - A previous fix in this release covered the case of the rings being RCU protected during resize, but it missed a few spots. This covers the rest - Fix the cBPF filters when COW'ed, introduced in this merge window - Fix for an attempt to import a zero sized buffer - Fix for a missing clamp in importing bundle buffers * tag 'io_uring-7.0-20260403' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux: io_uring/bpf_filters: retain COW'ed settings on parse failures io_uring: protect remaining lockless ctx->rings accesses with RCU io_uring/rsrc: reject zero-length fixed buffer import io_uring/net: fix slab-out-of-bounds read in io_bundle_nbufs()

Linus Torvalds committed 7 months ago

Commit e41255c2 parents c514f73 aa35dd67 files · +87 −29Browse files

Unified · Split

7 files
  1. io_uring/io_uring.c +5 −2
  2. io_uring/io_uring.h +29 −5
  3. io_uring/net.c +4 −0
  4. io_uring/register.c +9 −1
  5. io_uring/rsrc.c +4 −0
  6. io_uring/wait.c +31 −19
  7. io_uring/wait.h +5 −2
io_uring/io_uring.c+5−2
@@ -2015,7 +2015,7 @@ int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) · 2014 unmodified lines
2015 if (ctx->flags & IORING_SETUP_SQ_REWIND)2015 if (ctx->flags & IORING_SETUP_SQ_REWIND)
2016 entries = ctx->sq_entries;2016 entries = ctx->sq_entries;
2017 else2017 else
2018 entries = io_sqring_entries(ctx);2018 entries = __io_sqring_entries(ctx);
2019 2019
2020 entries = min(nr, entries);2020 entries = min(nr, entries);
2021 if (unlikely(!entries))2021 if (unlikely(!entries))
@@ -2250,7 +2250,9 @@ static __poll_t io_uring_poll(struct file *file, poll_table *wait) · 228 unmodified lines
2250 */2250 */
2251 poll_wait(file, &ctx->poll_wq, wait);2251 poll_wait(file, &ctx->poll_wq, wait);
2252 2252
2253 if (!io_sqring_full(ctx))2253 rcu_read_lock();
2254
2255 if (!__io_sqring_full(ctx))
2254 mask |= EPOLLOUT | EPOLLWRNORM;2256 mask |= EPOLLOUT | EPOLLWRNORM;
2255 2257
2256 /*2258 /*
@@ -2270,6 +2272,7 @@ static __poll_t io_uring_poll(struct file *file, poll_table *wait) · 13 unmodified lines
2270 if (__io_cqring_events_user(ctx) || io_has_work(ctx))2272 if (__io_cqring_events_user(ctx) || io_has_work(ctx))
2271 mask |= EPOLLIN | EPOLLRDNORM;2273 mask |= EPOLLIN | EPOLLRDNORM;
2272 2274
2275 rcu_read_unlock();
2273 return mask;2276 return mask;
2274}2277}
2275 2278
io_uring/io_uring.h+29−5
@@ -142,16 +142,28 @@ struct io_wait_queue { · 141 unmodified lines
142#endif142#endif
143};143};
144 144
145static inline struct io_rings *io_get_rings(struct io_ring_ctx *ctx)
146{
147 return rcu_dereference_check(ctx->rings_rcu,
148 lockdep_is_held(&ctx->uring_lock) ||
149 lockdep_is_held(&ctx->completion_lock));
150}
151
145static inline bool io_should_wake(struct io_wait_queue *iowq)152static inline bool io_should_wake(struct io_wait_queue *iowq)
146{153{
147 struct io_ring_ctx *ctx = iowq->ctx;154 struct io_ring_ctx *ctx = iowq->ctx;
148 int dist = READ_ONCE(ctx->rings->cq.tail) - (int) iowq->cq_tail;155 struct io_rings *rings;
156 int dist;
157
158 guard(rcu)();
159 rings = io_get_rings(ctx);
149 160
150 /*161 /*
151 * Wake up if we have enough events, or if a timeout occurred since we162 * Wake up if we have enough events, or if a timeout occurred since we
152 * started waiting. For timeouts, we always want to return to userspace,163 * started waiting. For timeouts, we always want to return to userspace,
153 * regardless of event count.164 * regardless of event count.
154 */165 */
166 dist = READ_ONCE(rings->cq.tail) - (int) iowq->cq_tail;
155 return dist >= 0 || atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts;167 return dist >= 0 || atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts;
156}168}
157 169
@@ -431,9 +443,9 @@ static inline void io_cqring_wake(struct io_ring_ctx *ctx) · 273 unmodified lines
431 __io_wq_wake(&ctx->cq_wait);443 __io_wq_wake(&ctx->cq_wait);
432}444}
433 445
434static inline bool io_sqring_full(struct io_ring_ctx *ctx)446static inline bool __io_sqring_full(struct io_ring_ctx *ctx)
435{447{
436 struct io_rings *r = ctx->rings;448 struct io_rings *r = io_get_rings(ctx);
437 449
438 /*450 /*
439 * SQPOLL must use the actual sqring head, as using the cached_sq_head451 * SQPOLL must use the actual sqring head, as using the cached_sq_head
@@ -445,9 +457,15 @@ static inline bool io_sqring_full(struct io_ring_ctx *ctx) · 5 unmodified lines
445 return READ_ONCE(r->sq.tail) - READ_ONCE(r->sq.head) == ctx->sq_entries;457 return READ_ONCE(r->sq.tail) - READ_ONCE(r->sq.head) == ctx->sq_entries;
446}458}
447 459
448static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)460static inline bool io_sqring_full(struct io_ring_ctx *ctx)
449{461{
450 struct io_rings *rings = ctx->rings;462 guard(rcu)();
463 return __io_sqring_full(ctx);
464}
465
466static inline unsigned int __io_sqring_entries(struct io_ring_ctx *ctx)
467{
468 struct io_rings *rings = io_get_rings(ctx);
451 unsigned int entries;469 unsigned int entries;
452 470
453 /* make sure SQ entry isn't read before tail */471 /* make sure SQ entry isn't read before tail */
@@ -455,6 +473,12 @@ static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx) · 1 unmodified line
455 return min(entries, ctx->sq_entries);473 return min(entries, ctx->sq_entries);
456}474}
457 475
476static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)
477{
478 guard(rcu)();
479 return __io_sqring_entries(ctx);
480}
481
458/*482/*
459 * Don't complete immediately but use deferred completion infrastructure.483 * Don't complete immediately but use deferred completion infrastructure.
460 * Protected by ->uring_lock and can only be used either with484 * Protected by ->uring_lock and can only be used either with
io_uring/net.c+4−0
@@ -421,6 +421,8 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 420 unmodified lines
421 421
422 sr->done_io = 0;422 sr->done_io = 0;
423 sr->len = READ_ONCE(sqe->len);423 sr->len = READ_ONCE(sqe->len);
424 if (unlikely(sr->len < 0))
425 return -EINVAL;
424 sr->flags = READ_ONCE(sqe->ioprio);426 sr->flags = READ_ONCE(sqe->ioprio);
425 if (sr->flags & ~SENDMSG_FLAGS)427 if (sr->flags & ~SENDMSG_FLAGS)
426 return -EINVAL;428 return -EINVAL;
@@ -791,6 +793,8 @@ int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 364 unmodified lines
791 793
792 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));794 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
793 sr->len = READ_ONCE(sqe->len);795 sr->len = READ_ONCE(sqe->len);
796 if (unlikely(sr->len < 0))
797 return -EINVAL;
794 sr->flags = READ_ONCE(sqe->ioprio);798 sr->flags = READ_ONCE(sqe->ioprio);
795 if (sr->flags & ~RECVMSG_FLAGS)799 if (sr->flags & ~RECVMSG_FLAGS)
796 return -EINVAL;800 return -EINVAL;
io_uring/register.c+9−1
@@ -178,9 +178,17 @@ static __cold int io_register_restrictions(struct io_ring_ctx *ctx, · 177 unmodified lines
178 return -EBUSY;178 return -EBUSY;
179 179
180 ret = io_parse_restrictions(arg, nr_args, &ctx->restrictions);180 ret = io_parse_restrictions(arg, nr_args, &ctx->restrictions);
181 /* Reset all restrictions if an error happened */181 /*
182 * Reset all restrictions if an error happened, but retain any COW'ed
183 * settings.
184 */
182 if (ret < 0) {185 if (ret < 0) {
186 struct io_bpf_filters *bpf = ctx->restrictions.bpf_filters;
187 bool cowed = ctx->restrictions.bpf_filters_cow;
188
183 memset(&ctx->restrictions, 0, sizeof(ctx->restrictions));189 memset(&ctx->restrictions, 0, sizeof(ctx->restrictions));
190 ctx->restrictions.bpf_filters = bpf;
191 ctx->restrictions.bpf_filters_cow = cowed;
184 return ret;192 return ret;
185 }193 }
186 if (ctx->restrictions.op_registered)194 if (ctx->restrictions.op_registered)
io_uring/rsrc.c+4−0
@@ -1061,6 +1061,10 @@ static int io_import_fixed(int ddir, struct iov_iter *iter, · 1060 unmodified lines
1061 return ret;1061 return ret;
1062 if (!(imu->dir & (1 << ddir)))1062 if (!(imu->dir & (1 << ddir)))
1063 return -EFAULT;1063 return -EFAULT;
1064 if (unlikely(!len)) {
1065 iov_iter_bvec(iter, ddir, NULL, 0, 0);
1066 return 0;
1067 }
1064 1068
1065 offset = buf_addr - imu->ubuf;1069 offset = buf_addr - imu->ubuf;
1066 1070
io_uring/wait.c+31−19
@@ -79,12 +79,15 @@ static enum hrtimer_restart io_cqring_min_timer_wakeup(struct hrtimer *timer) · 78 unmodified lines
79 if (io_has_work(ctx))79 if (io_has_work(ctx))
80 goto out_wake;80 goto out_wake;
81 /* got events since we started waiting, min timeout is done */81 /* got events since we started waiting, min timeout is done */
82 if (iowq->cq_min_tail != READ_ONCE(ctx->rings->cq.tail))82 scoped_guard(rcu) {
83 goto out_wake;83 struct io_rings *rings = io_get_rings(ctx);
84 /* if we have any events and min timeout expired, we're done */
85 if (io_cqring_events(ctx))
86 goto out_wake;
87 84
85 if (iowq->cq_min_tail != READ_ONCE(rings->cq.tail))
86 goto out_wake;
87 /* if we have any events and min timeout expired, we're done */
88 if (io_cqring_events(ctx))
89 goto out_wake;
90 }
88 /*91 /*
89 * If using deferred task_work running and application is waiting on92 * If using deferred task_work running and application is waiting on
90 * more than one request, ensure we reset it now where we are switching93 * more than one request, ensure we reset it now where we are switching
@@ -186,9 +189,9 @@ int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags, · 95 unmodified lines
186 struct ext_arg *ext_arg)189 struct ext_arg *ext_arg)
187{190{
188 struct io_wait_queue iowq;191 struct io_wait_queue iowq;
189 struct io_rings *rings = ctx->rings;192 struct io_rings *rings;
190 ktime_t start_time;193 ktime_t start_time;
191 int ret;194 int ret, nr_wait;
192 195
193 min_events = min_t(int, min_events, ctx->cq_entries);196 min_events = min_t(int, min_events, ctx->cq_entries);
194 197
@@ -201,15 +204,23 @@ int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags, · 6 unmodified lines
201 204
202 if (unlikely(test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq)))205 if (unlikely(test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq)))
203 io_cqring_do_overflow_flush(ctx);206 io_cqring_do_overflow_flush(ctx);
204 if (__io_cqring_events_user(ctx) >= min_events)207
208 rcu_read_lock();
209 rings = io_get_rings(ctx);
210 if (__io_cqring_events_user(ctx) >= min_events) {
211 rcu_read_unlock();
205 return 0;212 return 0;
213 }
206 214
207 init_waitqueue_func_entry(&iowq.wq, io_wake_function);215 init_waitqueue_func_entry(&iowq.wq, io_wake_function);
208 iowq.wq.private = current;216 iowq.wq.private = current;
209 INIT_LIST_HEAD(&iowq.wq.entry);217 INIT_LIST_HEAD(&iowq.wq.entry);
210 iowq.ctx = ctx;218 iowq.ctx = ctx;
211 iowq.cq_tail = READ_ONCE(ctx->rings->cq.head) + min_events;219 iowq.cq_tail = READ_ONCE(rings->cq.head) + min_events;
212 iowq.cq_min_tail = READ_ONCE(ctx->rings->cq.tail);220 iowq.cq_min_tail = READ_ONCE(rings->cq.tail);
221 nr_wait = (int) iowq.cq_tail - READ_ONCE(rings->cq.tail);
222 rcu_read_unlock();
223 rings = NULL;
213 iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts);224 iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts);
214 iowq.hit_timeout = 0;225 iowq.hit_timeout = 0;
215 iowq.min_timeout = ext_arg->min_time;226 iowq.min_timeout = ext_arg->min_time;
@@ -240,14 +251,6 @@ int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags, · 24 unmodified lines
240 trace_io_uring_cqring_wait(ctx, min_events);251 trace_io_uring_cqring_wait(ctx, min_events);
241 do {252 do {
242 unsigned long check_cq;253 unsigned long check_cq;
243 int nr_wait;
244
245 /* if min timeout has been hit, don't reset wait count */
246 if (!iowq.hit_timeout)
247 nr_wait = (int) iowq.cq_tail -
248 READ_ONCE(ctx->rings->cq.tail);
249 else
250 nr_wait = 1;
251 254
252 if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) {255 if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) {
253 atomic_set(&ctx->cq_wait_nr, nr_wait);256 atomic_set(&ctx->cq_wait_nr, nr_wait);
@@ -298,11 +301,20 @@ int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags, · 44 unmodified lines
298 break;301 break;
299 }302 }
300 cond_resched();303 cond_resched();
304
305 /* if min timeout has been hit, don't reset wait count */
306 if (!iowq.hit_timeout)
307 scoped_guard(rcu)
308 nr_wait = (int) iowq.cq_tail -
309 READ_ONCE(io_get_rings(ctx)->cq.tail);
310 else
311 nr_wait = 1;
301 } while (1);312 } while (1);
302 313
303 if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN))314 if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN))
304 finish_wait(&ctx->cq_wait, &iowq.wq);315 finish_wait(&ctx->cq_wait, &iowq.wq);
305 restore_saved_sigmask_unless(ret == -EINTR);316 restore_saved_sigmask_unless(ret == -EINTR);
306 317
307 return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;318 guard(rcu)();
319 return READ_ONCE(io_get_rings(ctx)->cq.head) == READ_ONCE(io_get_rings(ctx)->cq.tail) ? ret : 0;
308}320}
io_uring/wait.h+5−2
@@ -28,12 +28,15 @@ void io_cqring_do_overflow_flush(struct io_ring_ctx *ctx); · 27 unmodified lines
28 28
29static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx)29static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx)
30{30{
31 return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head);31 struct io_rings *rings = io_get_rings(ctx);
32 return ctx->cached_cq_tail - READ_ONCE(rings->cq.head);
32}33}
33 34
34static inline unsigned int __io_cqring_events_user(struct io_ring_ctx *ctx)35static inline unsigned int __io_cqring_events_user(struct io_ring_ctx *ctx)
35{36{
36 return READ_ONCE(ctx->rings->cq.tail) - READ_ONCE(ctx->rings->cq.head);37 struct io_rings *rings = io_get_rings(ctx);
38
39 return READ_ONCE(rings->cq.tail) - READ_ONCE(rings->cq.head);
37}40}
38 41
39/*42/*

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t