Pull io_uring fixes from Jens Axboe: - A previous fix in this release covered the case of the rings being RCU protected during resize, but it missed a few spots. This covers the rest - Fix the cBPF filters when COW'ed, introduced in this merge window - Fix for an attempt to import a zero sized buffer - Fix for a missing clamp in importing bundle buffers * tag 'io_uring-7.0-20260403' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux: io_uring/bpf_filters: retain COW'ed settings on parse failures io_uring: protect remaining lockless ctx->rings accesses with RCU io_uring/rsrc: reject zero-length fixed buffer import io_uring/net: fix slab-out-of-bounds read in io_bundle_nbufs()
Unified · Split
| @@ -2015,7 +2015,7 @@ int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) · 2014 unmodified lines | ||
| 2015 | 2015 | if (ctx->flags & IORING_SETUP_SQ_REWIND) |
| 2016 | 2016 | entries = ctx->sq_entries; |
| 2017 | 2017 | else |
| 2018 | − entries = io_sqring_entries(ctx); | |
| 2018 | + entries = __io_sqring_entries(ctx); | |
| 2019 | 2019 | |
| 2020 | 2020 | entries = min(nr, entries); |
| 2021 | 2021 | if (unlikely(!entries)) |
| @@ -2250,7 +2250,9 @@ static __poll_t io_uring_poll(struct file *file, poll_table *wait) · 228 unmodified lines | ||
| 2250 | 2250 | */ |
| 2251 | 2251 | poll_wait(file, &ctx->poll_wq, wait); |
| 2252 | 2252 | |
| 2253 | − if (!io_sqring_full(ctx)) | |
| 2253 | + rcu_read_lock(); | |
| 2254 | + | |
| 2255 | + if (!__io_sqring_full(ctx)) | |
| 2254 | 2256 | mask |= EPOLLOUT | EPOLLWRNORM; |
| 2255 | 2257 | |
| 2256 | 2258 | /* |
| @@ -2270,6 +2272,7 @@ static __poll_t io_uring_poll(struct file *file, poll_table *wait) · 13 unmodified lines | ||
| 2270 | 2272 | if (__io_cqring_events_user(ctx) || io_has_work(ctx)) |
| 2271 | 2273 | mask |= EPOLLIN | EPOLLRDNORM; |
| 2272 | 2274 | |
| 2275 | + rcu_read_unlock(); | |
| 2273 | 2276 | return mask; |
| 2274 | 2277 | } |
| 2275 | 2278 | |
| @@ -142,16 +142,28 @@ struct io_wait_queue { · 141 unmodified lines | ||
| 142 | 142 | #endif |
| 143 | 143 | }; |
| 144 | 144 | |
| 145 | +static inline struct io_rings *io_get_rings(struct io_ring_ctx *ctx) | |
| 146 | +{ | |
| 147 | + return rcu_dereference_check(ctx->rings_rcu, | |
| 148 | + lockdep_is_held(&ctx->uring_lock) || | |
| 149 | + lockdep_is_held(&ctx->completion_lock)); | |
| 150 | +} | |
| 151 | + | |
| 145 | 152 | static inline bool io_should_wake(struct io_wait_queue *iowq) |
| 146 | 153 | { |
| 147 | 154 | struct io_ring_ctx *ctx = iowq->ctx; |
| 148 | − int dist = READ_ONCE(ctx->rings->cq.tail) - (int) iowq->cq_tail; | |
| 155 | + struct io_rings *rings; | |
| 156 | + int dist; | |
| 157 | + | |
| 158 | + guard(rcu)(); | |
| 159 | + rings = io_get_rings(ctx); | |
| 149 | 160 | |
| 150 | 161 | /* |
| 151 | 162 | * Wake up if we have enough events, or if a timeout occurred since we |
| 152 | 163 | * started waiting. For timeouts, we always want to return to userspace, |
| 153 | 164 | * regardless of event count. |
| 154 | 165 | */ |
| 166 | + dist = READ_ONCE(rings->cq.tail) - (int) iowq->cq_tail; | |
| 155 | 167 | return dist >= 0 || atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts; |
| 156 | 168 | } |
| 157 | 169 | |
| @@ -431,9 +443,9 @@ static inline void io_cqring_wake(struct io_ring_ctx *ctx) · 273 unmodified lines | ||
| 431 | 443 | __io_wq_wake(&ctx->cq_wait); |
| 432 | 444 | } |
| 433 | 445 | |
| 434 | −static inline bool io_sqring_full(struct io_ring_ctx *ctx) | |
| 446 | +static inline bool __io_sqring_full(struct io_ring_ctx *ctx) | |
| 435 | 447 | { |
| 436 | − struct io_rings *r = ctx->rings; | |
| 448 | + struct io_rings *r = io_get_rings(ctx); | |
| 437 | 449 | |
| 438 | 450 | /* |
| 439 | 451 | * SQPOLL must use the actual sqring head, as using the cached_sq_head |
| @@ -445,9 +457,15 @@ static inline bool io_sqring_full(struct io_ring_ctx *ctx) · 5 unmodified lines | ||
| 445 | 457 | return READ_ONCE(r->sq.tail) - READ_ONCE(r->sq.head) == ctx->sq_entries; |
| 446 | 458 | } |
| 447 | 459 | |
| 448 | −static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx) | |
| 460 | +static inline bool io_sqring_full(struct io_ring_ctx *ctx) | |
| 449 | 461 | { |
| 450 | − struct io_rings *rings = ctx->rings; | |
| 462 | + guard(rcu)(); | |
| 463 | + return __io_sqring_full(ctx); | |
| 464 | +} | |
| 465 | + | |
| 466 | +static inline unsigned int __io_sqring_entries(struct io_ring_ctx *ctx) | |
| 467 | +{ | |
| 468 | + struct io_rings *rings = io_get_rings(ctx); | |
| 451 | 469 | unsigned int entries; |
| 452 | 470 | |
| 453 | 471 | /* make sure SQ entry isn't read before tail */ |
| @@ -455,6 +473,12 @@ static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx) · 1 unmodified line | ||
| 455 | 473 | return min(entries, ctx->sq_entries); |
| 456 | 474 | } |
| 457 | 475 | |
| 476 | +static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx) | |
| 477 | +{ | |
| 478 | + guard(rcu)(); | |
| 479 | + return __io_sqring_entries(ctx); | |
| 480 | +} | |
| 481 | + | |
| 458 | 482 | /* |
| 459 | 483 | * Don't complete immediately but use deferred completion infrastructure. |
| 460 | 484 | * Protected by ->uring_lock and can only be used either with |
| @@ -421,6 +421,8 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 420 unmodified lines | ||
| 421 | 421 | |
| 422 | 422 | sr->done_io = 0; |
| 423 | 423 | sr->len = READ_ONCE(sqe->len); |
| 424 | + if (unlikely(sr->len < 0)) | |
| 425 | + return -EINVAL; | |
| 424 | 426 | sr->flags = READ_ONCE(sqe->ioprio); |
| 425 | 427 | if (sr->flags & ~SENDMSG_FLAGS) |
| 426 | 428 | return -EINVAL; |
| @@ -791,6 +793,8 @@ int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 364 unmodified lines | ||
| 791 | 793 | |
| 792 | 794 | sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); |
| 793 | 795 | sr->len = READ_ONCE(sqe->len); |
| 796 | + if (unlikely(sr->len < 0)) | |
| 797 | + return -EINVAL; | |
| 794 | 798 | sr->flags = READ_ONCE(sqe->ioprio); |
| 795 | 799 | if (sr->flags & ~RECVMSG_FLAGS) |
| 796 | 800 | return -EINVAL; |
| @@ -178,9 +178,17 @@ static __cold int io_register_restrictions(struct io_ring_ctx *ctx, · 177 unmodified lines | ||
| 178 | 178 | return -EBUSY; |
| 179 | 179 | |
| 180 | 180 | ret = io_parse_restrictions(arg, nr_args, &ctx->restrictions); |
| 181 | − /* Reset all restrictions if an error happened */ | |
| 181 | + /* | |
| 182 | + * Reset all restrictions if an error happened, but retain any COW'ed | |
| 183 | + * settings. | |
| 184 | + */ | |
| 182 | 185 | if (ret < 0) { |
| 186 | + struct io_bpf_filters *bpf = ctx->restrictions.bpf_filters; | |
| 187 | + bool cowed = ctx->restrictions.bpf_filters_cow; | |
| 188 | + | |
| 183 | 189 | memset(&ctx->restrictions, 0, sizeof(ctx->restrictions)); |
| 190 | + ctx->restrictions.bpf_filters = bpf; | |
| 191 | + ctx->restrictions.bpf_filters_cow = cowed; | |
| 184 | 192 | return ret; |
| 185 | 193 | } |
| 186 | 194 | if (ctx->restrictions.op_registered) |
| @@ -1061,6 +1061,10 @@ static int io_import_fixed(int ddir, struct iov_iter *iter, · 1060 unmodified lines | ||
| 1061 | 1061 | return ret; |
| 1062 | 1062 | if (!(imu->dir & (1 << ddir))) |
| 1063 | 1063 | return -EFAULT; |
| 1064 | + if (unlikely(!len)) { | |
| 1065 | + iov_iter_bvec(iter, ddir, NULL, 0, 0); | |
| 1066 | + return 0; | |
| 1067 | + } | |
| 1064 | 1068 | |
| 1065 | 1069 | offset = buf_addr - imu->ubuf; |
| 1066 | 1070 | |
| @@ -79,12 +79,15 @@ static enum hrtimer_restart io_cqring_min_timer_wakeup(struct hrtimer *timer) · 78 unmodified lines | ||
| 79 | 79 | if (io_has_work(ctx)) |
| 80 | 80 | goto out_wake; |
| 81 | 81 | /* got events since we started waiting, min timeout is done */ |
| 82 | − if (iowq->cq_min_tail != READ_ONCE(ctx->rings->cq.tail)) | |
| 83 | − goto out_wake; | |
| 84 | − /* if we have any events and min timeout expired, we're done */ | |
| 85 | − if (io_cqring_events(ctx)) | |
| 86 | − goto out_wake; | |
| 82 | + scoped_guard(rcu) { | |
| 83 | + struct io_rings *rings = io_get_rings(ctx); | |
| 87 | 84 | |
| 85 | + if (iowq->cq_min_tail != READ_ONCE(rings->cq.tail)) | |
| 86 | + goto out_wake; | |
| 87 | + /* if we have any events and min timeout expired, we're done */ | |
| 88 | + if (io_cqring_events(ctx)) | |
| 89 | + goto out_wake; | |
| 90 | + } | |
| 88 | 91 | /* |
| 89 | 92 | * If using deferred task_work running and application is waiting on |
| 90 | 93 | * more than one request, ensure we reset it now where we are switching |
| @@ -186,9 +189,9 @@ int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags, · 95 unmodified lines | ||
| 186 | 189 | struct ext_arg *ext_arg) |
| 187 | 190 | { |
| 188 | 191 | struct io_wait_queue iowq; |
| 189 | − struct io_rings *rings = ctx->rings; | |
| 192 | + struct io_rings *rings; | |
| 190 | 193 | ktime_t start_time; |
| 191 | − int ret; | |
| 194 | + int ret, nr_wait; | |
| 192 | 195 | |
| 193 | 196 | min_events = min_t(int, min_events, ctx->cq_entries); |
| 194 | 197 | |
| @@ -201,15 +204,23 @@ int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags, · 6 unmodified lines | ||
| 201 | 204 | |
| 202 | 205 | if (unlikely(test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq))) |
| 203 | 206 | io_cqring_do_overflow_flush(ctx); |
| 204 | − if (__io_cqring_events_user(ctx) >= min_events) | |
| 207 | + | |
| 208 | + rcu_read_lock(); | |
| 209 | + rings = io_get_rings(ctx); | |
| 210 | + if (__io_cqring_events_user(ctx) >= min_events) { | |
| 211 | + rcu_read_unlock(); | |
| 205 | 212 | return 0; |
| 213 | + } | |
| 206 | 214 | |
| 207 | 215 | init_waitqueue_func_entry(&iowq.wq, io_wake_function); |
| 208 | 216 | iowq.wq.private = current; |
| 209 | 217 | INIT_LIST_HEAD(&iowq.wq.entry); |
| 210 | 218 | iowq.ctx = ctx; |
| 211 | − iowq.cq_tail = READ_ONCE(ctx->rings->cq.head) + min_events; | |
| 212 | − iowq.cq_min_tail = READ_ONCE(ctx->rings->cq.tail); | |
| 219 | + iowq.cq_tail = READ_ONCE(rings->cq.head) + min_events; | |
| 220 | + iowq.cq_min_tail = READ_ONCE(rings->cq.tail); | |
| 221 | + nr_wait = (int) iowq.cq_tail - READ_ONCE(rings->cq.tail); | |
| 222 | + rcu_read_unlock(); | |
| 223 | + rings = NULL; | |
| 213 | 224 | iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts); |
| 214 | 225 | iowq.hit_timeout = 0; |
| 215 | 226 | iowq.min_timeout = ext_arg->min_time; |
| @@ -240,14 +251,6 @@ int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags, · 24 unmodified lines | ||
| 240 | 251 | trace_io_uring_cqring_wait(ctx, min_events); |
| 241 | 252 | do { |
| 242 | 253 | unsigned long check_cq; |
| 243 | − int nr_wait; | |
| 244 | − | |
| 245 | − /* if min timeout has been hit, don't reset wait count */ | |
| 246 | − if (!iowq.hit_timeout) | |
| 247 | − nr_wait = (int) iowq.cq_tail - | |
| 248 | − READ_ONCE(ctx->rings->cq.tail); | |
| 249 | − else | |
| 250 | − nr_wait = 1; | |
| 251 | 254 | |
| 252 | 255 | if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) { |
| 253 | 256 | atomic_set(&ctx->cq_wait_nr, nr_wait); |
| @@ -298,11 +301,20 @@ int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags, · 44 unmodified lines | ||
| 298 | 301 | break; |
| 299 | 302 | } |
| 300 | 303 | cond_resched(); |
| 304 | + | |
| 305 | + /* if min timeout has been hit, don't reset wait count */ | |
| 306 | + if (!iowq.hit_timeout) | |
| 307 | + scoped_guard(rcu) | |
| 308 | + nr_wait = (int) iowq.cq_tail - | |
| 309 | + READ_ONCE(io_get_rings(ctx)->cq.tail); | |
| 310 | + else | |
| 311 | + nr_wait = 1; | |
| 301 | 312 | } while (1); |
| 302 | 313 | |
| 303 | 314 | if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN)) |
| 304 | 315 | finish_wait(&ctx->cq_wait, &iowq.wq); |
| 305 | 316 | restore_saved_sigmask_unless(ret == -EINTR); |
| 306 | 317 | |
| 307 | − return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0; | |
| 318 | + guard(rcu)(); | |
| 319 | + return READ_ONCE(io_get_rings(ctx)->cq.head) == READ_ONCE(io_get_rings(ctx)->cq.tail) ? ret : 0; | |
| 308 | 320 | } |
| @@ -28,12 +28,15 @@ void io_cqring_do_overflow_flush(struct io_ring_ctx *ctx); · 27 unmodified lines | ||
| 28 | 28 | |
| 29 | 29 | static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx) |
| 30 | 30 | { |
| 31 | − return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head); | |
| 31 | + struct io_rings *rings = io_get_rings(ctx); | |
| 32 | + return ctx->cached_cq_tail - READ_ONCE(rings->cq.head); | |
| 32 | 33 | } |
| 33 | 34 | |
| 34 | 35 | static inline unsigned int __io_cqring_events_user(struct io_ring_ctx *ctx) |
| 35 | 36 | { |
| 36 | − return READ_ONCE(ctx->rings->cq.tail) - READ_ONCE(ctx->rings->cq.head); | |
| 37 | + struct io_rings *rings = io_get_rings(ctx); | |
| 38 | + | |
| 39 | + return READ_ONCE(rings->cq.tail) - READ_ONCE(rings->cq.head); | |
| 37 | 40 | } |
| 38 | 41 | |
| 39 | 42 | /* |