History

Merge tag 'io_uring-7.0-20260216' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux

Pull more io_uring updates from Jens Axboe: "This is a mix of cleanups and fixes. No major fixes in here, just a bunch of little fixes. Some of them marked for stable as it fixes behavioral issues - Fix an issue with SOCKET_URING_OP_SETSOCKOPT for netlink sockets, due to a too restrictive check on it having an ioctl handler - Remove a redundant SQPOLL check in ring creation - Kill dead accounting for zero-copy send, which doesn't use ->buf or ->len post the initial setup - Fix missing clamp of the allocation hint, which could cause allocations to fall outside of the range the application asked for. Still within the allowed limits. - Fix for IORING_OP_PIPE's handling of direct descriptors - Tweak to the API for the newly added BPF filters, making them more future proof in terms of how applications deal with them - A few fixes for zcrx, fixing a few error handling conditions - Fix for zcrx request flag checking - Add support for querying the zcrx page size - Improve the NO_SQARRAY static branch inc/dec, avoiding busy conditions causing too much traffic - Various little cleanups" * tag 'io_uring-7.0-20260216' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux: io_uring/bpf_filter: pass in expected filter payload size io_uring/bpf_filter: move filter size and populate helper into struct io_uring/cancel: de-unionize file and user_data in struct io_cancel_data io_uring/rsrc: improve regbuf iov validation io_uring: remove unneeded io_send_zc accounting io_uring/cmd_net: fix too strict requirement on ioctl io_uring: delay sqarray static branch disablement io_uring/query: add query.h copyright notice io_uring/query: return support for custom rx page size io_uring/zcrx: check unsupported flags on import io_uring/zcrx: fix post open error handling io_uring/zcrx: fix sgtable leak on mapping failures io_uring: use the right type for creds iteration io_uring/openclose: fix io_pipe_fixed() slot tracking for specific slots io_uring/filetable: clamp alloc_hint to the configured alloc range io_uring/rsrc: replace reg buffer bit field with flags io_uring/zcrx: improve types for size calculation io_uring/tctx: avoid modifying loop variable in io_ring_add_registered_file io_uring: simplify IORING_SETUP_DEFER_TASKRUN && !SQPOLL check

Linus Torvalds committed 9 months ago

Commit 7b751b02 parents 9702969 be3573118 files · +149 −90Browse files

Unified · Split

18 files
  1. include/uapi/linux/io_uring.h +8 −0
  2. include/uapi/linux/io_uring/bpf_filter.h +7 −1
  3. include/uapi/linux/io_uring/query.h +5 −1
  4. io_uring/bpf_filter.c +55 −27
  5. io_uring/cancel.h +2 −4
  6. io_uring/cmd_net.c +6 −3
  7. io_uring/filetable.c +4 −0
  8. io_uring/io_uring.c +6 −7
  9. io_uring/net.c +0 −2
  10. io_uring/opdef.c +6 −0
  11. io_uring/opdef.h +6 −0
  12. io_uring/openclose.c +6 −3
  13. io_uring/query.c +1 −1
  14. io_uring/rsrc.c +16 −27
  15. io_uring/rsrc.h +5 −1
  16. io_uring/rw.c +2 −1
  17. io_uring/tctx.c +5 −5
  18. io_uring/zcrx.c +9 −7
include/uapi/linux/io_uring.h+8−0
@@ -1090,6 +1090,14 @@ enum zcrx_reg_flags { · 1089 unmodified lines
1090 ZCRX_REG_IMPORT = 1,1090 ZCRX_REG_IMPORT = 1,
1091};1091};
1092 1092
1093enum zcrx_features {
1094 /*
1095 * The user can ask for the desired rx page size by passing the
1096 * value in struct io_uring_zcrx_ifq_reg::rx_buf_len.
1097 */
1098 ZCRX_FEATURE_RX_PAGE_SIZE = 1 << 0,
1099};
1100
1093/*1101/*
1094 * Argument for IORING_REGISTER_ZCRX_IFQ1102 * Argument for IORING_REGISTER_ZCRX_IFQ
1095 */1103 */
include/uapi/linux/io_uring/bpf_filter.h+7−1
@@ -35,13 +35,19 @@ enum { · 34 unmodified lines
35 * If set, any currently unset opcode will have a deny filter attached35 * If set, any currently unset opcode will have a deny filter attached
36 */36 */
37 IO_URING_BPF_FILTER_DENY_REST = 1,37 IO_URING_BPF_FILTER_DENY_REST = 1,
38 /*
39 * If set, if kernel and application don't agree on pdu_size for
40 * the given opcode, fail the registration of the filter.
41 */
42 IO_URING_BPF_FILTER_SZ_STRICT = 2,
38};43};
39 44
40struct io_uring_bpf_filter {45struct io_uring_bpf_filter {
41 __u32 opcode; /* io_uring opcode to filter */46 __u32 opcode; /* io_uring opcode to filter */
42 __u32 flags;47 __u32 flags;
43 __u32 filter_len; /* number of BPF instructions */48 __u32 filter_len; /* number of BPF instructions */
44 __u32 resv;49 __u8 pdu_size; /* expected pdu size for opcode */
50 __u8 resv[3];
45 __u64 filter_ptr; /* pointer to BPF filter */51 __u64 filter_ptr; /* pointer to BPF filter */
46 __u64 resv2[5];52 __u64 resv2[5];
47};53};
include/uapi/linux/io_uring/query.h+5−1
@@ -1,6 +1,9 @@
1/* SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) OR MIT */1/* SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) OR MIT */
2/*2/*
3 * Header file for the io_uring query interface.3 * Header file for the io_uring query interface.
4 *
5 * Copyright (C) 2026 Pavel Begunkov <asml.silence@gmail.com>
6 * Copyright (C) Meta Platforms, Inc.
4 */7 */
5#ifndef LINUX_IO_URING_QUERY_H8#ifndef LINUX_IO_URING_QUERY_H
6#define LINUX_IO_URING_QUERY_H9#define LINUX_IO_URING_QUERY_H
@@ -50,7 +53,8 @@ struct io_uring_query_zcrx { · 43 unmodified lines
50 __u64 area_flags;53 __u64 area_flags;
51 /* The number of supported ZCRX_CTRL_* opcodes */54 /* The number of supported ZCRX_CTRL_* opcodes */
52 __u32 nr_ctrl_opcodes;55 __u32 nr_ctrl_opcodes;
53 __u32 __resv1;56 /* Bitmask of ZCRX_FEATURE_* indicating which features are available */
57 __u32 features;
54 /* The refill ring header size */58 /* The refill ring header size */
55 __u32 rq_hdr_size;59 __u32 rq_hdr_size;
56 /* The alignment for the header */60 /* The alignment for the header */
io_uring/bpf_filter.c+55−27
@@ -26,6 +26,8 @@ static const struct io_bpf_filter dummy_filter; · 25 unmodified lines
26static void io_uring_populate_bpf_ctx(struct io_uring_bpf_ctx *bctx,26static void io_uring_populate_bpf_ctx(struct io_uring_bpf_ctx *bctx,
27 struct io_kiocb *req)27 struct io_kiocb *req)
28{28{
29 const struct io_issue_def *def = &io_issue_defs[req->opcode];
30
29 bctx->opcode = req->opcode;31 bctx->opcode = req->opcode;
30 bctx->sqe_flags = (__force int) req->flags & SQE_VALID_FLAGS;32 bctx->sqe_flags = (__force int) req->flags & SQE_VALID_FLAGS;
31 bctx->user_data = req->cqe.user_data;33 bctx->user_data = req->cqe.user_data;
@@ -34,19 +36,12 @@ static void io_uring_populate_bpf_ctx(struct io_uring_bpf_ctx *bctx, · 2 unmodified lines
34 sizeof(*bctx) - offsetof(struct io_uring_bpf_ctx, pdu_size));36 sizeof(*bctx) - offsetof(struct io_uring_bpf_ctx, pdu_size));
35 37
36 /*38 /*
37 * Opcodes can provide a handler fo populating more data into bctx,39 * Opcodes can provide a handler for populating more data into bctx,
38 * for filters to use.40 * for filters to use.
39 */41 */
40 switch (req->opcode) {42 if (def->filter_pdu_size) {
41 case IORING_OP_SOCKET:43 bctx->pdu_size = def->filter_pdu_size;
42 bctx->pdu_size = sizeof(bctx->socket);44 def->filter_populate(bctx, req);
43 io_socket_bpf_populate(bctx, req);
44 break;
45 case IORING_OP_OPENAT:
46 case IORING_OP_OPENAT2:
47 bctx->pdu_size = sizeof(bctx->open);
48 io_openat_bpf_populate(bctx, req);
49 break;
50 }45 }
51}46}
52 47
@@ -313,36 +308,69 @@ err: · 260 unmodified lines
313 return ERR_PTR(-EBUSY);308 return ERR_PTR(-EBUSY);
314}309}
315 310
316#define IO_URING_BPF_FILTER_FLAGS IO_URING_BPF_FILTER_DENY_REST311#define IO_URING_BPF_FILTER_FLAGS (IO_URING_BPF_FILTER_DENY_REST | \
312 IO_URING_BPF_FILTER_SZ_STRICT)
317 313
318int io_register_bpf_filter(struct io_restriction *res,314static int io_bpf_filter_import(struct io_uring_bpf *reg,
319 struct io_uring_bpf __user *arg)315 struct io_uring_bpf __user *arg)
320{316{
321 struct io_bpf_filters *filters, *old_filters = NULL;317 const struct io_issue_def *def;
322 struct io_bpf_filter *filter, *old_filter;
323 struct io_uring_bpf reg;
324 struct bpf_prog *prog;
325 struct sock_fprog fprog;
326 int ret;318 int ret;
327 319
328 if (copy_from_user(&reg, arg, sizeof(reg)))320 if (copy_from_user(reg, arg, sizeof(*reg)))
329 return -EFAULT;321 return -EFAULT;
330 if (reg.cmd_type != IO_URING_BPF_CMD_FILTER)322 if (reg->cmd_type != IO_URING_BPF_CMD_FILTER)
331 return -EINVAL;323 return -EINVAL;
332 if (reg.cmd_flags || reg.resv)324 if (reg->cmd_flags || reg->resv)
333 return -EINVAL;325 return -EINVAL;
334 326
335 if (reg.filter.opcode >= IORING_OP_LAST)327 if (reg->filter.opcode >= IORING_OP_LAST)
336 return -EINVAL;328 return -EINVAL;
337 if (reg.filter.flags & ~IO_URING_BPF_FILTER_FLAGS)329 if (reg->filter.flags & ~IO_URING_BPF_FILTER_FLAGS)
338 return -EINVAL;330 return -EINVAL;
339 if (reg.filter.resv)331 if (!mem_is_zero(reg->filter.resv, sizeof(reg->filter.resv)))
340 return -EINVAL;332 return -EINVAL;
341 if (!mem_is_zero(reg.filter.resv2, sizeof(reg.filter.resv2)))333 if (!mem_is_zero(reg->filter.resv2, sizeof(reg->filter.resv2)))
342 return -EINVAL;334 return -EINVAL;
343 if (!reg.filter.filter_len || reg.filter.filter_len > BPF_MAXINSNS)335 if (!reg->filter.filter_len || reg->filter.filter_len > BPF_MAXINSNS)
344 return -EINVAL;336 return -EINVAL;
345 337
338 /* Verify filter size */
339 def = &io_issue_defs[array_index_nospec(reg->filter.opcode, IORING_OP_LAST)];
340
341 /* same size, always ok */
342 ret = 0;
343 if (reg->filter.pdu_size == def->filter_pdu_size)
344 ;
345 /* size differs, fail in strict mode */
346 else if (reg->filter.flags & IO_URING_BPF_FILTER_SZ_STRICT)
347 ret = -EMSGSIZE;
348 /* userspace filter is bigger, always disallow */
349 else if (reg->filter.pdu_size > def->filter_pdu_size)
350 ret = -EMSGSIZE;
351
352 /* copy back kernel filter size */
353 reg->filter.pdu_size = def->filter_pdu_size;
354 if (copy_to_user(&arg->filter, &reg->filter, sizeof(reg->filter)))
355 return -EFAULT;
356
357 return ret;
358}
359
360int io_register_bpf_filter(struct io_restriction *res,
361 struct io_uring_bpf __user *arg)
362{
363 struct io_bpf_filters *filters, *old_filters = NULL;
364 struct io_bpf_filter *filter, *old_filter;
365 struct io_uring_bpf reg;
366 struct bpf_prog *prog;
367 struct sock_fprog fprog;
368 int ret;
369
370 ret = io_bpf_filter_import(&reg, arg);
371 if (ret)
372 return ret;
373
346 fprog.len = reg.filter.filter_len;374 fprog.len = reg.filter.filter_len;
347 fprog.filter = u64_to_user_ptr(reg.filter.filter_ptr);375 fprog.filter = u64_to_user_ptr(reg.filter.filter_ptr);
348 376
io_uring/cancel.h+2−4
@@ -6,10 +6,8 @@ · 5 unmodified lines
6 6
7struct io_cancel_data {7struct io_cancel_data {
8 struct io_ring_ctx *ctx;8 struct io_ring_ctx *ctx;
9 union {9 u64 data;
10 u64 data;10 struct file *file;
11 struct file *file;
12 };
13 u8 opcode;11 u8 opcode;
14 u32 flags;12 u32 flags;
15 int seq;13 int seq;
io_uring/cmd_net.c+6−3
@@ -160,16 +160,19 @@ int io_uring_cmd_sock(struct io_uring_cmd *cmd, unsigned int issue_flags) · 159 unmodified lines
160 struct proto *prot = READ_ONCE(sk->sk_prot);160 struct proto *prot = READ_ONCE(sk->sk_prot);
161 int ret, arg = 0;161 int ret, arg = 0;
162 162
163 if (!prot || !prot->ioctl)
164 return -EOPNOTSUPP;
165
166 switch (cmd->cmd_op) {163 switch (cmd->cmd_op) {
167 case SOCKET_URING_OP_SIOCINQ:164 case SOCKET_URING_OP_SIOCINQ:
165 if (!prot || !prot->ioctl)
166 return -EOPNOTSUPP;
167
168 ret = prot->ioctl(sk, SIOCINQ, &arg);168 ret = prot->ioctl(sk, SIOCINQ, &arg);
169 if (ret)169 if (ret)
170 return ret;170 return ret;
171 return arg;171 return arg;
172 case SOCKET_URING_OP_SIOCOUTQ:172 case SOCKET_URING_OP_SIOCOUTQ:
173 if (!prot || !prot->ioctl)
174 return -EOPNOTSUPP;
175
173 ret = prot->ioctl(sk, SIOCOUTQ, &arg);176 ret = prot->ioctl(sk, SIOCOUTQ, &arg);
174 if (ret)177 if (ret)
175 return ret;178 return ret;
io_uring/filetable.c+4−0
@@ -22,6 +22,10 @@ static int io_file_bitmap_get(struct io_ring_ctx *ctx) · 21 unmodified lines
22 if (!table->bitmap)22 if (!table->bitmap)
23 return -ENFILE;23 return -ENFILE;
24 24
25 if (table->alloc_hint < ctx->file_alloc_start ||
26 table->alloc_hint >= ctx->file_alloc_end)
27 table->alloc_hint = ctx->file_alloc_start;
28
25 do {29 do {
26 ret = find_next_zero_bit(table->bitmap, nr, table->alloc_hint);30 ret = find_next_zero_bit(table->bitmap, nr, table->alloc_hint);
27 if (ret != nr)31 if (ret != nr)
io_uring/io_uring.c+6−7
@@ -119,7 +119,7 @@ · 118 unmodified lines
119static void io_queue_sqe(struct io_kiocb *req, unsigned int extra_flags);119static void io_queue_sqe(struct io_kiocb *req, unsigned int extra_flags);
120static void __io_req_caches_free(struct io_ring_ctx *ctx);120static void __io_req_caches_free(struct io_ring_ctx *ctx);
121 121
122static __read_mostly DEFINE_STATIC_KEY_FALSE(io_key_has_sqarray);122static __read_mostly DEFINE_STATIC_KEY_DEFERRED_FALSE(io_key_has_sqarray, HZ);
123 123
124struct kmem_cache *req_cachep;124struct kmem_cache *req_cachep;
125static struct workqueue_struct *iou_wq __ro_after_init;125static struct workqueue_struct *iou_wq __ro_after_init;
@@ -1978,7 +1978,7 @@ static bool io_get_sqe(struct io_ring_ctx *ctx, const struct io_uring_sqe **sqe) · 1852 unmodified lines
1978 unsigned mask = ctx->sq_entries - 1;1978 unsigned mask = ctx->sq_entries - 1;
1979 unsigned head = ctx->cached_sq_head++ & mask;1979 unsigned head = ctx->cached_sq_head++ & mask;
1980 1980
1981 if (static_branch_unlikely(&io_key_has_sqarray) &&1981 if (static_branch_unlikely(&io_key_has_sqarray.key) &&
1982 (!(ctx->flags & IORING_SETUP_NO_SQARRAY))) {1982 (!(ctx->flags & IORING_SETUP_NO_SQARRAY))) {
1983 head = READ_ONCE(ctx->sq_array[head]);1983 head = READ_ONCE(ctx->sq_array[head]);
1984 if (unlikely(head >= ctx->sq_entries)) {1984 if (unlikely(head >= ctx->sq_entries)) {
@@ -2173,7 +2173,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) · 188 unmodified lines
2173 io_rings_free(ctx);2173 io_rings_free(ctx);
2174 2174
2175 if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))2175 if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
2176 static_branch_dec(&io_key_has_sqarray);2176 static_branch_slow_dec_deferred(&io_key_has_sqarray);
2177 2177
2178 percpu_ref_exit(&ctx->refs);2178 percpu_ref_exit(&ctx->refs);
2179 free_uid(ctx->user);2179 free_uid(ctx->user);
@@ -2398,7 +2398,7 @@ static __cold void io_ring_exit_work(struct work_struct *work) · 218 unmodified lines
2398static __cold void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)2398static __cold void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
2399{2399{
2400 unsigned long index;2400 unsigned long index;
2401 struct creds *creds;2401 struct cred *creds;
2402 2402
2403 mutex_lock(&ctx->uring_lock);2403 mutex_lock(&ctx->uring_lock);
2404 percpu_ref_kill(&ctx->refs);2404 percpu_ref_kill(&ctx->refs);
@@ -2946,11 +2946,10 @@ static __cold int io_uring_create(struct io_ctx_config *config) · 541 unmodified lines
2946 ctx->clock_offset = 0;2946 ctx->clock_offset = 0;
2947 2947
2948 if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))2948 if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
2949 static_branch_inc(&io_key_has_sqarray);2949 static_branch_deferred_inc(&io_key_has_sqarray);
2950 2950
2951 if ((ctx->flags & IORING_SETUP_DEFER_TASKRUN) &&2951 if ((ctx->flags & IORING_SETUP_DEFER_TASKRUN) &&
2952 !(ctx->flags & IORING_SETUP_IOPOLL) &&2952 !(ctx->flags & IORING_SETUP_IOPOLL))
2953 !(ctx->flags & IORING_SETUP_SQPOLL))
2954 ctx->task_complete = true;2953 ctx->task_complete = true;
2955 2954
2956 if (ctx->task_complete || (ctx->flags & IORING_SETUP_IOPOLL))2955 if (ctx->task_complete || (ctx->flags & IORING_SETUP_IOPOLL))
io_uring/net.c+0−2
@@ -1493,8 +1493,6 @@ int io_send_zc(struct io_kiocb *req, unsigned int issue_flags) · 1492 unmodified lines
1493 return -EAGAIN;1493 return -EAGAIN;
1494 1494
1495 if (ret > 0 && io_net_retry(sock, kmsg->msg.msg_flags)) {1495 if (ret > 0 && io_net_retry(sock, kmsg->msg.msg_flags)) {
1496 zc->len -= ret;
1497 zc->buf += ret;
1498 zc->done_io += ret;1496 zc->done_io += ret;
1499 return -EAGAIN;1497 return -EAGAIN;
1500 }1498 }
io_uring/opdef.c+6−0
@@ -221,8 +221,10 @@ const struct io_issue_def io_issue_defs[] = { · 220 unmodified lines
221 .issue = io_fallocate,221 .issue = io_fallocate,
222 },222 },
223 [IORING_OP_OPENAT] = {223 [IORING_OP_OPENAT] = {
224 .filter_pdu_size = sizeof_field(struct io_uring_bpf_ctx, open),
224 .prep = io_openat_prep,225 .prep = io_openat_prep,
225 .issue = io_openat,226 .issue = io_openat,
227 .filter_populate = io_openat_bpf_populate,
226 },228 },
227 [IORING_OP_CLOSE] = {229 [IORING_OP_CLOSE] = {
228 .prep = io_close_prep,230 .prep = io_close_prep,
@@ -309,8 +311,10 @@ const struct io_issue_def io_issue_defs[] = { · 80 unmodified lines
309#endif311#endif
310 },312 },
311 [IORING_OP_OPENAT2] = {313 [IORING_OP_OPENAT2] = {
314 .filter_pdu_size = sizeof_field(struct io_uring_bpf_ctx, open),
312 .prep = io_openat2_prep,315 .prep = io_openat2_prep,
313 .issue = io_openat2,316 .issue = io_openat2,
317 .filter_populate = io_openat_bpf_populate,
314 },318 },
315 [IORING_OP_EPOLL_CTL] = {319 [IORING_OP_EPOLL_CTL] = {
316 .unbound_nonreg_file = 1,320 .unbound_nonreg_file = 1,
@@ -406,8 +410,10 @@ const struct io_issue_def io_issue_defs[] = { · 89 unmodified lines
406 [IORING_OP_SOCKET] = {410 [IORING_OP_SOCKET] = {
407 .audit_skip = 1,411 .audit_skip = 1,
408#if defined(CONFIG_NET)412#if defined(CONFIG_NET)
413 .filter_pdu_size = sizeof_field(struct io_uring_bpf_ctx, socket),
409 .prep = io_socket_prep,414 .prep = io_socket_prep,
410 .issue = io_socket,415 .issue = io_socket,
416 .filter_populate = io_socket_bpf_populate,
411#else417#else
412 .prep = io_eopnotsupp_prep,418 .prep = io_eopnotsupp_prep,
413#endif419#endif
io_uring/opdef.h+6−0
@@ -2,6 +2,8 @@ · 1 unmodified line
2#ifndef IOU_OP_DEF_H2#ifndef IOU_OP_DEF_H
3#define IOU_OP_DEF_H3#define IOU_OP_DEF_H
4 4
5struct io_uring_bpf_ctx;
6
5struct io_issue_def {7struct io_issue_def {
6 /* needs req->file assigned */8 /* needs req->file assigned */
7 unsigned needs_file : 1;9 unsigned needs_file : 1;
@@ -33,8 +35,12 @@ struct io_issue_def { · 25 unmodified lines
33 /* size of async data needed, if any */35 /* size of async data needed, if any */
34 unsigned short async_size;36 unsigned short async_size;
35 37
38 /* bpf filter pdu size, if any */
39 unsigned short filter_pdu_size;
40
36 int (*issue)(struct io_kiocb *, unsigned int);41 int (*issue)(struct io_kiocb *, unsigned int);
37 int (*prep)(struct io_kiocb *, const struct io_uring_sqe *);42 int (*prep)(struct io_kiocb *, const struct io_uring_sqe *);
43 void (*filter_populate)(struct io_uring_bpf_ctx *, struct io_kiocb *);
38};44};
39 45
40struct io_cold_def {46struct io_cold_def {
io_uring/openclose.c+6−3
@@ -345,31 +345,34 @@ static int io_pipe_fixed(struct io_kiocb *req, struct file **files, · 344 unmodified lines
345{345{
346 struct io_pipe *p = io_kiocb_to_cmd(req, struct io_pipe);346 struct io_pipe *p = io_kiocb_to_cmd(req, struct io_pipe);
347 struct io_ring_ctx *ctx = req->ctx;347 struct io_ring_ctx *ctx = req->ctx;
348 bool alloc_slot;
348 int ret, fds[2] = { -1, -1 };349 int ret, fds[2] = { -1, -1 };
349 int slot = p->file_slot;350 int slot = p->file_slot;
350 351
351 if (p->flags & O_CLOEXEC)352 if (p->flags & O_CLOEXEC)
352 return -EINVAL;353 return -EINVAL;
353 354
355 alloc_slot = slot == IORING_FILE_INDEX_ALLOC;
356
354 io_ring_submit_lock(ctx, issue_flags);357 io_ring_submit_lock(ctx, issue_flags);
355 358
356 ret = __io_fixed_fd_install(ctx, files[0], slot);359 ret = __io_fixed_fd_install(ctx, files[0], slot);
357 if (ret < 0)360 if (ret < 0)
358 goto err;361 goto err;
359 fds[0] = ret;362 fds[0] = alloc_slot ? ret : slot - 1;
360 files[0] = NULL;363 files[0] = NULL;
361 364
362 /*365 /*
363 * If a specific slot is given, next one will be used for366 * If a specific slot is given, next one will be used for
364 * the write side.367 * the write side.
365 */368 */
366 if (slot != IORING_FILE_INDEX_ALLOC)369 if (!alloc_slot)
367 slot++;370 slot++;
368 371
369 ret = __io_fixed_fd_install(ctx, files[1], slot);372 ret = __io_fixed_fd_install(ctx, files[1], slot);
370 if (ret < 0)373 if (ret < 0)
371 goto err;374 goto err;
372 fds[1] = ret;375 fds[1] = alloc_slot ? ret : slot - 1;
373 files[1] = NULL;376 files[1] = NULL;
374 377
375 io_ring_submit_unlock(ctx, issue_flags);378 io_ring_submit_unlock(ctx, issue_flags);
io_uring/query.c+1−1
@@ -39,7 +39,7 @@ static ssize_t io_query_zcrx(union io_query_data *data) · 38 unmodified lines
39 e->nr_ctrl_opcodes = __ZCRX_CTRL_LAST;39 e->nr_ctrl_opcodes = __ZCRX_CTRL_LAST;
40 e->rq_hdr_size = sizeof(struct io_uring);40 e->rq_hdr_size = sizeof(struct io_uring);
41 e->rq_hdr_alignment = L1_CACHE_BYTES;41 e->rq_hdr_alignment = L1_CACHE_BYTES;
42 e->__resv1 = 0;42 e->features = ZCRX_FEATURE_RX_PAGE_SIZE;
43 e->__resv2 = 0;43 e->__resv2 = 0;
44 return sizeof(*e);44 return sizeof(*e);
45}45}
io_uring/rsrc.c+16−27
@@ -96,20 +96,6 @@ int io_validate_user_buf_range(u64 uaddr, u64 ulen) · 95 unmodified lines
96 return 0;96 return 0;
97}97}
98 98
99static int io_buffer_validate(struct iovec *iov)
100{
101 /*
102 * Don't impose further limits on the size and buffer
103 * constraints here, we'll -EINVAL later when IO is
104 * submitted if they are wrong.
105 */
106 if (!iov->iov_base)
107 return iov->iov_len ? -EFAULT : 0;
108
109 return io_validate_user_buf_range((unsigned long)iov->iov_base,
110 iov->iov_len);
111}
112
113static void io_release_ubuf(void *priv)99static void io_release_ubuf(void *priv)
114{100{
115 struct io_mapped_ubuf *imu = priv;101 struct io_mapped_ubuf *imu = priv;
@@ -319,9 +305,6 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx, · 203 unmodified lines
319 err = -EFAULT;305 err = -EFAULT;
320 break;306 break;
321 }307 }
322 err = io_buffer_validate(iov);
323 if (err)
324 break;
325 node = io_sqe_buffer_register(ctx, iov, &last_hpage);308 node = io_sqe_buffer_register(ctx, iov, &last_hpage);
326 if (IS_ERR(node)) {309 if (IS_ERR(node)) {
327 err = PTR_ERR(node);310 err = PTR_ERR(node);
@@ -790,8 +773,17 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, · 462 unmodified lines
790 struct io_imu_folio_data data;773 struct io_imu_folio_data data;
791 bool coalesced = false;774 bool coalesced = false;
792 775
793 if (!iov->iov_base)776 if (!iov->iov_base) {
777 if (iov->iov_len)
778 return ERR_PTR(-EFAULT);
779 /* remove the buffer without installing a new one */
794 return NULL;780 return NULL;
781 }
782
783 ret = io_validate_user_buf_range((unsigned long)iov->iov_base,
784 iov->iov_len);
785 if (ret)
786 return ERR_PTR(ret);
795 787
796 node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER);788 node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER);
797 if (!node)789 if (!node)
@@ -828,7 +820,7 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, · 30 unmodified lines
828 imu->folio_shift = PAGE_SHIFT;820 imu->folio_shift = PAGE_SHIFT;
829 imu->release = io_release_ubuf;821 imu->release = io_release_ubuf;
830 imu->priv = imu;822 imu->priv = imu;
831 imu->is_kbuf = false;823 imu->flags = 0;
832 imu->dir = IO_IMU_DEST | IO_IMU_SOURCE;824 imu->dir = IO_IMU_DEST | IO_IMU_SOURCE;
833 if (coalesced)825 if (coalesced)
834 imu->folio_shift = data.folio_shift;826 imu->folio_shift = data.folio_shift;
@@ -897,9 +889,6 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, · 62 unmodified lines
897 ret = PTR_ERR(iov);889 ret = PTR_ERR(iov);
898 break;890 break;
899 }891 }
900 ret = io_buffer_validate(iov);
901 if (ret)
902 break;
903 if (ctx->compat)892 if (ctx->compat)
904 arg += sizeof(struct compat_iovec);893 arg += sizeof(struct compat_iovec);
905 else894 else
@@ -985,7 +974,7 @@ int io_buffer_register_bvec(struct io_uring_cmd *cmd, struct request *rq, · 79 unmodified lines
985 refcount_set(&imu->refs, 1);974 refcount_set(&imu->refs, 1);
986 imu->release = release;975 imu->release = release;
987 imu->priv = rq;976 imu->priv = rq;
988 imu->is_kbuf = true;977 imu->flags = IO_REGBUF_F_KBUF;
989 imu->dir = 1 << rq_data_dir(rq);978 imu->dir = 1 << rq_data_dir(rq);
990 979
991 rq_for_each_bvec(bv, rq, rq_iter)980 rq_for_each_bvec(bv, rq, rq_iter)
@@ -1020,7 +1009,7 @@ int io_buffer_unregister_bvec(struct io_uring_cmd *cmd, unsigned int index, · 28 unmodified lines
1020 ret = -EINVAL;1009 ret = -EINVAL;
1021 goto unlock;1010 goto unlock;
1022 }1011 }
1023 if (!node->buf->is_kbuf) {1012 if (!(node->buf->flags & IO_REGBUF_F_KBUF)) {
1024 ret = -EBUSY;1013 ret = -EBUSY;
1025 goto unlock;1014 goto unlock;
1026 }1015 }
@@ -1076,7 +1065,7 @@ static int io_import_fixed(int ddir, struct iov_iter *iter, · 49 unmodified lines
1076 1065
1077 offset = buf_addr - imu->ubuf;1066 offset = buf_addr - imu->ubuf;
1078 1067
1079 if (imu->is_kbuf)1068 if (imu->flags & IO_REGBUF_F_KBUF)
1080 return io_import_kbuf(ddir, iter, imu, len, offset);1069 return io_import_kbuf(ddir, iter, imu, len, offset);
1081 1070
1082 /*1071 /*
@@ -1496,7 +1485,7 @@ int io_import_reg_vec(int ddir, struct iov_iter *iter, · 413 unmodified lines
1496 iovec_off = vec->nr - nr_iovs;1485 iovec_off = vec->nr - nr_iovs;
1497 iov = vec->iovec + iovec_off;1486 iov = vec->iovec + iovec_off;
1498 1487
1499 if (imu->is_kbuf) {1488 if (imu->flags & IO_REGBUF_F_KBUF) {
1500 int ret = io_kern_bvec_size(iov, nr_iovs, imu, &nr_segs);1489 int ret = io_kern_bvec_size(iov, nr_iovs, imu, &nr_segs);
1501 1490
1502 if (unlikely(ret))1491 if (unlikely(ret))
@@ -1534,7 +1523,7 @@ int io_import_reg_vec(int ddir, struct iov_iter *iter, · 31 unmodified lines
1534 req->flags |= REQ_F_NEED_CLEANUP;1523 req->flags |= REQ_F_NEED_CLEANUP;
1535 }1524 }
1536 1525
1537 if (imu->is_kbuf)1526 if (imu->flags & IO_REGBUF_F_KBUF)
1538 return io_vec_fill_kern_bvec(ddir, iter, imu, iov, nr_iovs, vec);1527 return io_vec_fill_kern_bvec(ddir, iter, imu, iov, nr_iovs, vec);
1539 1528
1540 return io_vec_fill_bvec(ddir, iter, imu, iov, nr_iovs, vec);1529 return io_vec_fill_bvec(ddir, iter, imu, iov, nr_iovs, vec);
io_uring/rsrc.h+5−1
@@ -28,6 +28,10 @@ enum { · 27 unmodified lines
28 IO_IMU_SOURCE = 1 << ITER_SOURCE,28 IO_IMU_SOURCE = 1 << ITER_SOURCE,
29};29};
30 30
31enum {
32 IO_REGBUF_F_KBUF = 1,
33};
34
31struct io_mapped_ubuf {35struct io_mapped_ubuf {
32 u64 ubuf;36 u64 ubuf;
33 unsigned int len;37 unsigned int len;
@@ -37,7 +41,7 @@ struct io_mapped_ubuf { · 3 unmodified lines
37 unsigned long acct_pages;41 unsigned long acct_pages;
38 void (*release)(void *);42 void (*release)(void *);
39 void *priv;43 void *priv;
40 bool is_kbuf;44 u8 flags;
41 u8 dir;45 u8 dir;
42 struct bio_vec bvec[] __counted_by(nr_bvecs);46 struct bio_vec bvec[] __counted_by(nr_bvecs);
43};47};
io_uring/rw.c+2−1
@@ -702,7 +702,8 @@ static ssize_t loop_rw_iter(int ddir, struct io_rw *rw, struct iov_iter *iter) · 701 unmodified lines
702 if ((kiocb->ki_flags & IOCB_NOWAIT) &&702 if ((kiocb->ki_flags & IOCB_NOWAIT) &&
703 !(kiocb->ki_filp->f_flags & O_NONBLOCK))703 !(kiocb->ki_filp->f_flags & O_NONBLOCK))
704 return -EAGAIN;704 return -EAGAIN;
705 if ((req->flags & REQ_F_BUF_NODE) && req->buf_node->buf->is_kbuf)705 if ((req->flags & REQ_F_BUF_NODE) &&
706 (req->buf_node->buf->flags & IO_REGBUF_F_KBUF))
706 return -EFAULT;707 return -EFAULT;
707 708
708 ppos = io_kiocb_ppos(kiocb);709 ppos = io_kiocb_ppos(kiocb);
io_uring/tctx.c+5−5
@@ -240,14 +240,14 @@ void io_uring_unreg_ringfd(void) · 239 unmodified lines
240int io_ring_add_registered_file(struct io_uring_task *tctx, struct file *file,240int io_ring_add_registered_file(struct io_uring_task *tctx, struct file *file,
241 int start, int end)241 int start, int end)
242{242{
243 int offset;243 int offset, idx;
244 for (offset = start; offset < end; offset++) {244 for (offset = start; offset < end; offset++) {
245 offset = array_index_nospec(offset, IO_RINGFD_REG_MAX);245 idx = array_index_nospec(offset, IO_RINGFD_REG_MAX);
246 if (tctx->registered_rings[offset])246 if (tctx->registered_rings[idx])
247 continue;247 continue;
248 248
249 tctx->registered_rings[offset] = file;249 tctx->registered_rings[idx] = file;
250 return offset;250 return idx;
251 }251 }
252 return -EBUSY;252 return -EBUSY;
253}253}
io_uring/zcrx.c+9−7
@@ -205,7 +205,7 @@ static int io_import_umem(struct io_zcrx_ifq *ifq, · 204 unmodified lines
205 return PTR_ERR(pages);205 return PTR_ERR(pages);
206 206
207 ret = sg_alloc_table_from_pages(&mem->page_sg_table, pages, nr_pages,207 ret = sg_alloc_table_from_pages(&mem->page_sg_table, pages, nr_pages,
208 0, nr_pages << PAGE_SHIFT,208 0, (unsigned long)nr_pages << PAGE_SHIFT,
209 GFP_KERNEL_ACCOUNT);209 GFP_KERNEL_ACCOUNT);
210 if (ret) {210 if (ret) {
211 unpin_user_pages(pages, nr_pages);211 unpin_user_pages(pages, nr_pages);
@@ -300,6 +300,9 @@ static int io_zcrx_map_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area) · 88 unmodified lines
300 }300 }
301 301
302 ret = io_populate_area_dma(ifq, area);302 ret = io_populate_area_dma(ifq, area);
303 if (ret && !area->mem.is_dmabuf)
304 dma_unmap_sgtable(ifq->dev, &area->mem.page_sg_table,
305 DMA_FROM_DEVICE, IO_DMA_ATTR);
303 if (ret == 0)306 if (ret == 0)
304 area->is_mapped = true;307 area->is_mapped = true;
305 return ret;308 return ret;
@@ -538,9 +541,6 @@ static void io_close_queue(struct io_zcrx_ifq *ifq) · 232 unmodified lines
538 .mp_priv = ifq,541 .mp_priv = ifq,
539 };542 };
540 543
541 if (ifq->if_rxq == -1)
542 return;
543
544 scoped_guard(mutex, &ifq->pp_lock) {544 scoped_guard(mutex, &ifq->pp_lock) {
545 netdev = ifq->netdev;545 netdev = ifq->netdev;
546 netdev_tracker = ifq->netdev_tracker;546 netdev_tracker = ifq->netdev_tracker;
@@ -548,7 +548,8 @@ static void io_close_queue(struct io_zcrx_ifq *ifq) · 1 unmodified line
548 }548 }
549 549
550 if (netdev) {550 if (netdev) {
551 net_mp_close_rxq(netdev, ifq->if_rxq, &p);551 if (ifq->if_rxq != -1)
552 net_mp_close_rxq(netdev, ifq->if_rxq, &p);
552 netdev_put(netdev, &netdev_tracker);553 netdev_put(netdev, &netdev_tracker);
553 }554 }
554 ifq->if_rxq = -1;555 ifq->if_rxq = -1;
@@ -702,6 +703,8 @@ static int import_zcrx(struct io_ring_ctx *ctx, · 147 unmodified lines
702 return -EINVAL;703 return -EINVAL;
703 if (reg->if_rxq || reg->rq_entries || reg->area_ptr || reg->region_ptr)704 if (reg->if_rxq || reg->rq_entries || reg->area_ptr || reg->region_ptr)
704 return -EINVAL;705 return -EINVAL;
706 if (reg->flags & ~ZCRX_REG_IMPORT)
707 return -EINVAL;
705 708
706 fd = reg->if_idx;709 fd = reg->if_idx;
707 CLASS(fd, f)(fd);710 CLASS(fd, f)(fd);
@@ -858,13 +861,12 @@ int io_register_zcrx_ifq(struct io_ring_ctx *ctx, · 150 unmodified lines
858 }861 }
859 return 0;862 return 0;
860netdev_put_unlock:863netdev_put_unlock:
861 netdev_put(ifq->netdev, &ifq->netdev_tracker);
862 netdev_unlock(ifq->netdev);864 netdev_unlock(ifq->netdev);
863err:865err:
864 scoped_guard(mutex, &ctx->mmap_lock)866 scoped_guard(mutex, &ctx->mmap_lock)
865 xa_erase(&ctx->zcrx_ctxs, id);867 xa_erase(&ctx->zcrx_ctxs, id);
866ifq_free:868ifq_free:
867 io_zcrx_ifq_free(ifq);869 zcrx_unregister(ifq);
868 return ret;870 return ret;
869}871}
870 872