Pull io_uring fixes from Jens Axboe: - A previous fix in this release covered the case of the rings being RCU protected during resize, but it missed a few spots. This covers the rest - Fix the cBPF filters when COW'ed, introduced in this merge window - Fix for an attempt to import a zero sized buffer - Fix for a missing clamp in importing bundle buffers * tag 'io_uring-7.0-20260403' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux: io_uring/bpf_filters: retain COW'ed settings on parse failures io_uring: protect remaining lockless ctx->rings accesses with RCU io_uring/rsrc: reject zero-length fixed buffer import io_uring/net: fix slab-out-of-bounds read in io_bundle_nbufs()
Unified · Split
| @@ -2015,7 +2015,7 @@ int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) · 2014 unmodified lines | |||
| 2015 | if (ctx->flags & IORING_SETUP_SQ_REWIND) | 2015 | if (ctx->flags & IORING_SETUP_SQ_REWIND) |
| 2016 | entries = ctx->sq_entries; | 2016 | entries = ctx->sq_entries; |
| 2017 | else | 2017 | else |
| 2018 | entries = io_sqring_entries(ctx); | 2018 | entries = __io_sqring_entries(ctx); |
| 2019 | 2019 | ||
| 2020 | entries = min(nr, entries); | 2020 | entries = min(nr, entries); |
| 2021 | if (unlikely(!entries)) | 2021 | if (unlikely(!entries)) |
| @@ -2250,7 +2250,9 @@ static __poll_t io_uring_poll(struct file *file, poll_table *wait) · 228 unmodified lines | |||
| 2250 | */ | 2250 | */ |
| 2251 | poll_wait(file, &ctx->poll_wq, wait); | 2251 | poll_wait(file, &ctx->poll_wq, wait); |
| 2252 | 2252 | ||
| 2253 | if (!io_sqring_full(ctx)) | 2253 | rcu_read_lock(); |
| 2254 | |||
| 2255 | if (!__io_sqring_full(ctx)) | ||
| 2254 | mask |= EPOLLOUT | EPOLLWRNORM; | 2256 | mask |= EPOLLOUT | EPOLLWRNORM; |
| 2255 | 2257 | ||
| 2256 | /* | 2258 | /* |
| @@ -2270,6 +2272,7 @@ static __poll_t io_uring_poll(struct file *file, poll_table *wait) · 13 unmodified lines | |||
| 2270 | if (__io_cqring_events_user(ctx) || io_has_work(ctx)) | 2272 | if (__io_cqring_events_user(ctx) || io_has_work(ctx)) |
| 2271 | mask |= EPOLLIN | EPOLLRDNORM; | 2273 | mask |= EPOLLIN | EPOLLRDNORM; |
| 2272 | 2274 | ||
| 2275 | rcu_read_unlock(); | ||
| 2273 | return mask; | 2276 | return mask; |
| 2274 | } | 2277 | } |
| 2275 | 2278 | ||
| @@ -142,16 +142,28 @@ struct io_wait_queue { · 141 unmodified lines | |||
| 142 | #endif | 142 | #endif |
| 143 | }; | 143 | }; |
| 144 | 144 | ||
| 145 | static inline struct io_rings *io_get_rings(struct io_ring_ctx *ctx) | ||
| 146 | { | ||
| 147 | return rcu_dereference_check(ctx->rings_rcu, | ||
| 148 | lockdep_is_held(&ctx->uring_lock) || | ||
| 149 | lockdep_is_held(&ctx->completion_lock)); | ||
| 150 | } | ||
| 151 | |||
| 145 | static inline bool io_should_wake(struct io_wait_queue *iowq) | 152 | static inline bool io_should_wake(struct io_wait_queue *iowq) |
| 146 | { | 153 | { |
| 147 | struct io_ring_ctx *ctx = iowq->ctx; | 154 | struct io_ring_ctx *ctx = iowq->ctx; |
| 148 | int dist = READ_ONCE(ctx->rings->cq.tail) - (int) iowq->cq_tail; | 155 | struct io_rings *rings; |
| 156 | int dist; | ||
| 157 | |||
| 158 | guard(rcu)(); | ||
| 159 | rings = io_get_rings(ctx); | ||
| 149 | 160 | ||
| 150 | /* | 161 | /* |
| 151 | * Wake up if we have enough events, or if a timeout occurred since we | 162 | * Wake up if we have enough events, or if a timeout occurred since we |
| 152 | * started waiting. For timeouts, we always want to return to userspace, | 163 | * started waiting. For timeouts, we always want to return to userspace, |
| 153 | * regardless of event count. | 164 | * regardless of event count. |
| 154 | */ | 165 | */ |
| 166 | dist = READ_ONCE(rings->cq.tail) - (int) iowq->cq_tail; | ||
| 155 | return dist >= 0 || atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts; | 167 | return dist >= 0 || atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts; |
| 156 | } | 168 | } |
| 157 | 169 | ||
| @@ -431,9 +443,9 @@ static inline void io_cqring_wake(struct io_ring_ctx *ctx) · 273 unmodified lines | |||
| 431 | __io_wq_wake(&ctx->cq_wait); | 443 | __io_wq_wake(&ctx->cq_wait); |
| 432 | } | 444 | } |
| 433 | 445 | ||
| 434 | static inline bool io_sqring_full(struct io_ring_ctx *ctx) | 446 | static inline bool __io_sqring_full(struct io_ring_ctx *ctx) |
| 435 | { | 447 | { |
| 436 | struct io_rings *r = ctx->rings; | 448 | struct io_rings *r = io_get_rings(ctx); |
| 437 | 449 | ||
| 438 | /* | 450 | /* |
| 439 | * SQPOLL must use the actual sqring head, as using the cached_sq_head | 451 | * SQPOLL must use the actual sqring head, as using the cached_sq_head |
| @@ -445,9 +457,15 @@ static inline bool io_sqring_full(struct io_ring_ctx *ctx) · 5 unmodified lines | |||
| 445 | return READ_ONCE(r->sq.tail) - READ_ONCE(r->sq.head) == ctx->sq_entries; | 457 | return READ_ONCE(r->sq.tail) - READ_ONCE(r->sq.head) == ctx->sq_entries; |
| 446 | } | 458 | } |
| 447 | 459 | ||
| 448 | static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx) | 460 | static inline bool io_sqring_full(struct io_ring_ctx *ctx) |
| 449 | { | 461 | { |
| 450 | struct io_rings *rings = ctx->rings; | 462 | guard(rcu)(); |
| 463 | return __io_sqring_full(ctx); | ||
| 464 | } | ||
| 465 | |||
| 466 | static inline unsigned int __io_sqring_entries(struct io_ring_ctx *ctx) | ||
| 467 | { | ||
| 468 | struct io_rings *rings = io_get_rings(ctx); | ||
| 451 | unsigned int entries; | 469 | unsigned int entries; |
| 452 | 470 | ||
| 453 | /* make sure SQ entry isn't read before tail */ | 471 | /* make sure SQ entry isn't read before tail */ |
| @@ -455,6 +473,12 @@ static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx) · 1 unmodified line | |||
| 455 | return min(entries, ctx->sq_entries); | 473 | return min(entries, ctx->sq_entries); |
| 456 | } | 474 | } |
| 457 | 475 | ||
| 476 | static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx) | ||
| 477 | { | ||
| 478 | guard(rcu)(); | ||
| 479 | return __io_sqring_entries(ctx); | ||
| 480 | } | ||
| 481 | |||
| 458 | /* | 482 | /* |
| 459 | * Don't complete immediately but use deferred completion infrastructure. | 483 | * Don't complete immediately but use deferred completion infrastructure. |
| 460 | * Protected by ->uring_lock and can only be used either with | 484 | * Protected by ->uring_lock and can only be used either with |
| @@ -421,6 +421,8 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 420 unmodified lines | |||
| 421 | 421 | ||
| 422 | sr->done_io = 0; | 422 | sr->done_io = 0; |
| 423 | sr->len = READ_ONCE(sqe->len); | 423 | sr->len = READ_ONCE(sqe->len); |
| 424 | if (unlikely(sr->len < 0)) | ||
| 425 | return -EINVAL; | ||
| 424 | sr->flags = READ_ONCE(sqe->ioprio); | 426 | sr->flags = READ_ONCE(sqe->ioprio); |
| 425 | if (sr->flags & ~SENDMSG_FLAGS) | 427 | if (sr->flags & ~SENDMSG_FLAGS) |
| 426 | return -EINVAL; | 428 | return -EINVAL; |
| @@ -791,6 +793,8 @@ int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 364 unmodified lines | |||
| 791 | 793 | ||
| 792 | sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); | 794 | sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); |
| 793 | sr->len = READ_ONCE(sqe->len); | 795 | sr->len = READ_ONCE(sqe->len); |
| 796 | if (unlikely(sr->len < 0)) | ||
| 797 | return -EINVAL; | ||
| 794 | sr->flags = READ_ONCE(sqe->ioprio); | 798 | sr->flags = READ_ONCE(sqe->ioprio); |
| 795 | if (sr->flags & ~RECVMSG_FLAGS) | 799 | if (sr->flags & ~RECVMSG_FLAGS) |
| 796 | return -EINVAL; | 800 | return -EINVAL; |
| @@ -178,9 +178,17 @@ static __cold int io_register_restrictions(struct io_ring_ctx *ctx, · 177 unmodified lines | |||
| 178 | return -EBUSY; | 178 | return -EBUSY; |
| 179 | 179 | ||
| 180 | ret = io_parse_restrictions(arg, nr_args, &ctx->restrictions); | 180 | ret = io_parse_restrictions(arg, nr_args, &ctx->restrictions); |
| 181 | /* Reset all restrictions if an error happened */ | 181 | /* |
| 182 | * Reset all restrictions if an error happened, but retain any COW'ed | ||
| 183 | * settings. | ||
| 184 | */ | ||
| 182 | if (ret < 0) { | 185 | if (ret < 0) { |
| 186 | struct io_bpf_filters *bpf = ctx->restrictions.bpf_filters; | ||
| 187 | bool cowed = ctx->restrictions.bpf_filters_cow; | ||
| 188 | |||
| 183 | memset(&ctx->restrictions, 0, sizeof(ctx->restrictions)); | 189 | memset(&ctx->restrictions, 0, sizeof(ctx->restrictions)); |
| 190 | ctx->restrictions.bpf_filters = bpf; | ||
| 191 | ctx->restrictions.bpf_filters_cow = cowed; | ||
| 184 | return ret; | 192 | return ret; |
| 185 | } | 193 | } |
| 186 | if (ctx->restrictions.op_registered) | 194 | if (ctx->restrictions.op_registered) |
| @@ -1061,6 +1061,10 @@ static int io_import_fixed(int ddir, struct iov_iter *iter, · 1060 unmodified lines | |||
| 1061 | return ret; | 1061 | return ret; |
| 1062 | if (!(imu->dir & (1 << ddir))) | 1062 | if (!(imu->dir & (1 << ddir))) |
| 1063 | return -EFAULT; | 1063 | return -EFAULT; |
| 1064 | if (unlikely(!len)) { | ||
| 1065 | iov_iter_bvec(iter, ddir, NULL, 0, 0); | ||
| 1066 | return 0; | ||
| 1067 | } | ||
| 1064 | 1068 | ||
| 1065 | offset = buf_addr - imu->ubuf; | 1069 | offset = buf_addr - imu->ubuf; |
| 1066 | 1070 | ||
| @@ -79,12 +79,15 @@ static enum hrtimer_restart io_cqring_min_timer_wakeup(struct hrtimer *timer) · 78 unmodified lines | |||
| 79 | if (io_has_work(ctx)) | 79 | if (io_has_work(ctx)) |
| 80 | goto out_wake; | 80 | goto out_wake; |
| 81 | /* got events since we started waiting, min timeout is done */ | 81 | /* got events since we started waiting, min timeout is done */ |
| 82 | if (iowq->cq_min_tail != READ_ONCE(ctx->rings->cq.tail)) | 82 | scoped_guard(rcu) { |
| 83 | goto out_wake; | 83 | struct io_rings *rings = io_get_rings(ctx); |
| 84 | /* if we have any events and min timeout expired, we're done */ | ||
| 85 | if (io_cqring_events(ctx)) | ||
| 86 | goto out_wake; | ||
| 87 | 84 | ||
| 85 | if (iowq->cq_min_tail != READ_ONCE(rings->cq.tail)) | ||
| 86 | goto out_wake; | ||
| 87 | /* if we have any events and min timeout expired, we're done */ | ||
| 88 | if (io_cqring_events(ctx)) | ||
| 89 | goto out_wake; | ||
| 90 | } | ||
| 88 | /* | 91 | /* |
| 89 | * If using deferred task_work running and application is waiting on | 92 | * If using deferred task_work running and application is waiting on |
| 90 | * more than one request, ensure we reset it now where we are switching | 93 | * more than one request, ensure we reset it now where we are switching |
| @@ -186,9 +189,9 @@ int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags, · 95 unmodified lines | |||
| 186 | struct ext_arg *ext_arg) | 189 | struct ext_arg *ext_arg) |
| 187 | { | 190 | { |
| 188 | struct io_wait_queue iowq; | 191 | struct io_wait_queue iowq; |
| 189 | struct io_rings *rings = ctx->rings; | 192 | struct io_rings *rings; |
| 190 | ktime_t start_time; | 193 | ktime_t start_time; |
| 191 | int ret; | 194 | int ret, nr_wait; |
| 192 | 195 | ||
| 193 | min_events = min_t(int, min_events, ctx->cq_entries); | 196 | min_events = min_t(int, min_events, ctx->cq_entries); |
| 194 | 197 | ||
| @@ -201,15 +204,23 @@ int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags, · 6 unmodified lines | |||
| 201 | 204 | ||
| 202 | if (unlikely(test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq))) | 205 | if (unlikely(test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq))) |
| 203 | io_cqring_do_overflow_flush(ctx); | 206 | io_cqring_do_overflow_flush(ctx); |
| 204 | if (__io_cqring_events_user(ctx) >= min_events) | 207 | |
| 208 | rcu_read_lock(); | ||
| 209 | rings = io_get_rings(ctx); | ||
| 210 | if (__io_cqring_events_user(ctx) >= min_events) { | ||
| 211 | rcu_read_unlock(); | ||
| 205 | return 0; | 212 | return 0; |
| 213 | } | ||
| 206 | 214 | ||
| 207 | init_waitqueue_func_entry(&iowq.wq, io_wake_function); | 215 | init_waitqueue_func_entry(&iowq.wq, io_wake_function); |
| 208 | iowq.wq.private = current; | 216 | iowq.wq.private = current; |
| 209 | INIT_LIST_HEAD(&iowq.wq.entry); | 217 | INIT_LIST_HEAD(&iowq.wq.entry); |
| 210 | iowq.ctx = ctx; | 218 | iowq.ctx = ctx; |
| 211 | iowq.cq_tail = READ_ONCE(ctx->rings->cq.head) + min_events; | 219 | iowq.cq_tail = READ_ONCE(rings->cq.head) + min_events; |
| 212 | iowq.cq_min_tail = READ_ONCE(ctx->rings->cq.tail); | 220 | iowq.cq_min_tail = READ_ONCE(rings->cq.tail); |
| 221 | nr_wait = (int) iowq.cq_tail - READ_ONCE(rings->cq.tail); | ||
| 222 | rcu_read_unlock(); | ||
| 223 | rings = NULL; | ||
| 213 | iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts); | 224 | iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts); |
| 214 | iowq.hit_timeout = 0; | 225 | iowq.hit_timeout = 0; |
| 215 | iowq.min_timeout = ext_arg->min_time; | 226 | iowq.min_timeout = ext_arg->min_time; |
| @@ -240,14 +251,6 @@ int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags, · 24 unmodified lines | |||
| 240 | trace_io_uring_cqring_wait(ctx, min_events); | 251 | trace_io_uring_cqring_wait(ctx, min_events); |
| 241 | do { | 252 | do { |
| 242 | unsigned long check_cq; | 253 | unsigned long check_cq; |
| 243 | int nr_wait; | ||
| 244 | |||
| 245 | /* if min timeout has been hit, don't reset wait count */ | ||
| 246 | if (!iowq.hit_timeout) | ||
| 247 | nr_wait = (int) iowq.cq_tail - | ||
| 248 | READ_ONCE(ctx->rings->cq.tail); | ||
| 249 | else | ||
| 250 | nr_wait = 1; | ||
| 251 | 254 | ||
| 252 | if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) { | 255 | if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) { |
| 253 | atomic_set(&ctx->cq_wait_nr, nr_wait); | 256 | atomic_set(&ctx->cq_wait_nr, nr_wait); |
| @@ -298,11 +301,20 @@ int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags, · 44 unmodified lines | |||
| 298 | break; | 301 | break; |
| 299 | } | 302 | } |
| 300 | cond_resched(); | 303 | cond_resched(); |
| 304 | |||
| 305 | /* if min timeout has been hit, don't reset wait count */ | ||
| 306 | if (!iowq.hit_timeout) | ||
| 307 | scoped_guard(rcu) | ||
| 308 | nr_wait = (int) iowq.cq_tail - | ||
| 309 | READ_ONCE(io_get_rings(ctx)->cq.tail); | ||
| 310 | else | ||
| 311 | nr_wait = 1; | ||
| 301 | } while (1); | 312 | } while (1); |
| 302 | 313 | ||
| 303 | if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN)) | 314 | if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN)) |
| 304 | finish_wait(&ctx->cq_wait, &iowq.wq); | 315 | finish_wait(&ctx->cq_wait, &iowq.wq); |
| 305 | restore_saved_sigmask_unless(ret == -EINTR); | 316 | restore_saved_sigmask_unless(ret == -EINTR); |
| 306 | 317 | ||
| 307 | return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0; | 318 | guard(rcu)(); |
| 319 | return READ_ONCE(io_get_rings(ctx)->cq.head) == READ_ONCE(io_get_rings(ctx)->cq.tail) ? ret : 0; | ||
| 308 | } | 320 | } |
| @@ -28,12 +28,15 @@ void io_cqring_do_overflow_flush(struct io_ring_ctx *ctx); · 27 unmodified lines | |||
| 28 | 28 | ||
| 29 | static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx) | 29 | static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx) |
| 30 | { | 30 | { |
| 31 | return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head); | 31 | struct io_rings *rings = io_get_rings(ctx); |
| 32 | return ctx->cached_cq_tail - READ_ONCE(rings->cq.head); | ||
| 32 | } | 33 | } |
| 33 | 34 | ||
| 34 | static inline unsigned int __io_cqring_events_user(struct io_ring_ctx *ctx) | 35 | static inline unsigned int __io_cqring_events_user(struct io_ring_ctx *ctx) |
| 35 | { | 36 | { |
| 36 | return READ_ONCE(ctx->rings->cq.tail) - READ_ONCE(ctx->rings->cq.head); | 37 | struct io_rings *rings = io_get_rings(ctx); |
| 38 | |||
| 39 | return READ_ONCE(rings->cq.tail) - READ_ONCE(rings->cq.head); | ||
| 37 | } | 40 | } |
| 38 | 41 | ||
| 39 | /* | 42 | /* |