History

Merge tag 'io_uring-7.0-20260216' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux

Pull more io_uring updates from Jens Axboe: "This is a mix of cleanups and fixes. No major fixes in here, just a bunch of little fixes. Some of them marked for stable as it fixes behavioral issues - Fix an issue with SOCKET_URING_OP_SETSOCKOPT for netlink sockets, due to a too restrictive check on it having an ioctl handler - Remove a redundant SQPOLL check in ring creation - Kill dead accounting for zero-copy send, which doesn't use ->buf or ->len post the initial setup - Fix missing clamp of the allocation hint, which could cause allocations to fall outside of the range the application asked for. Still within the allowed limits. - Fix for IORING_OP_PIPE's handling of direct descriptors - Tweak to the API for the newly added BPF filters, making them more future proof in terms of how applications deal with them - A few fixes for zcrx, fixing a few error handling conditions - Fix for zcrx request flag checking - Add support for querying the zcrx page size - Improve the NO_SQARRAY static branch inc/dec, avoiding busy conditions causing too much traffic - Various little cleanups" * tag 'io_uring-7.0-20260216' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux: io_uring/bpf_filter: pass in expected filter payload size io_uring/bpf_filter: move filter size and populate helper into struct io_uring/cancel: de-unionize file and user_data in struct io_cancel_data io_uring/rsrc: improve regbuf iov validation io_uring: remove unneeded io_send_zc accounting io_uring/cmd_net: fix too strict requirement on ioctl io_uring: delay sqarray static branch disablement io_uring/query: add query.h copyright notice io_uring/query: return support for custom rx page size io_uring/zcrx: check unsupported flags on import io_uring/zcrx: fix post open error handling io_uring/zcrx: fix sgtable leak on mapping failures io_uring: use the right type for creds iteration io_uring/openclose: fix io_pipe_fixed() slot tracking for specific slots io_uring/filetable: clamp alloc_hint to the configured alloc range io_uring/rsrc: replace reg buffer bit field with flags io_uring/zcrx: improve types for size calculation io_uring/tctx: avoid modifying loop variable in io_ring_add_registered_file io_uring: simplify IORING_SETUP_DEFER_TASKRUN && !SQPOLL check

Linus Torvalds committed 9 months ago

Commit 7b751b02 parents 9702969 be3573118 files · +149 −90Browse files

Unified · Split

18 files
  1. include/uapi/linux/io_uring.h +8 −0
  2. include/uapi/linux/io_uring/bpf_filter.h +7 −1
  3. include/uapi/linux/io_uring/query.h +5 −1
  4. io_uring/bpf_filter.c +55 −27
  5. io_uring/cancel.h +2 −4
  6. io_uring/cmd_net.c +6 −3
  7. io_uring/filetable.c +4 −0
  8. io_uring/io_uring.c +6 −7
  9. io_uring/net.c +0 −2
  10. io_uring/opdef.c +6 −0
  11. io_uring/opdef.h +6 −0
  12. io_uring/openclose.c +6 −3
  13. io_uring/query.c +1 −1
  14. io_uring/rsrc.c +16 −27
  15. io_uring/rsrc.h +5 −1
  16. io_uring/rw.c +2 −1
  17. io_uring/tctx.c +5 −5
  18. io_uring/zcrx.c +9 −7
include/uapi/linux/io_uring.h+8−0
@@ -1090,6 +1090,14 @@ enum zcrx_reg_flags { · 1089 unmodified lines
10901090 ZCRX_REG_IMPORT = 1,
10911091 };
10921092
1093+enum zcrx_features {
1094+ /*
1095+ * The user can ask for the desired rx page size by passing the
1096+ * value in struct io_uring_zcrx_ifq_reg::rx_buf_len.
1097+ */
1098+ ZCRX_FEATURE_RX_PAGE_SIZE = 1 << 0,
1099+};
1100+
10931101 /*
10941102 * Argument for IORING_REGISTER_ZCRX_IFQ
10951103 */
include/uapi/linux/io_uring/bpf_filter.h+7−1
@@ -35,13 +35,19 @@ enum { · 34 unmodified lines
3535 * If set, any currently unset opcode will have a deny filter attached
3636 */
3737 IO_URING_BPF_FILTER_DENY_REST = 1,
38+ /*
39+ * If set, if kernel and application don't agree on pdu_size for
40+ * the given opcode, fail the registration of the filter.
41+ */
42+ IO_URING_BPF_FILTER_SZ_STRICT = 2,
3843 };
3944
4045 struct io_uring_bpf_filter {
4146 __u32 opcode; /* io_uring opcode to filter */
4247 __u32 flags;
4348 __u32 filter_len; /* number of BPF instructions */
44− __u32 resv;
49+ __u8 pdu_size; /* expected pdu size for opcode */
50+ __u8 resv[3];
4551 __u64 filter_ptr; /* pointer to BPF filter */
4652 __u64 resv2[5];
4753 };
include/uapi/linux/io_uring/query.h+5−1
@@ -1,6 +1,9 @@
11 /* SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) OR MIT */
22 /*
33 * Header file for the io_uring query interface.
4+ *
5+ * Copyright (C) 2026 Pavel Begunkov <asml.silence@gmail.com>
6+ * Copyright (C) Meta Platforms, Inc.
47 */
58 #ifndef LINUX_IO_URING_QUERY_H
69 #define LINUX_IO_URING_QUERY_H
@@ -50,7 +53,8 @@ struct io_uring_query_zcrx { · 43 unmodified lines
5053 __u64 area_flags;
5154 /* The number of supported ZCRX_CTRL_* opcodes */
5255 __u32 nr_ctrl_opcodes;
53− __u32 __resv1;
56+ /* Bitmask of ZCRX_FEATURE_* indicating which features are available */
57+ __u32 features;
5458 /* The refill ring header size */
5559 __u32 rq_hdr_size;
5660 /* The alignment for the header */
io_uring/bpf_filter.c+55−27
@@ -26,6 +26,8 @@ static const struct io_bpf_filter dummy_filter; · 25 unmodified lines
2626 static void io_uring_populate_bpf_ctx(struct io_uring_bpf_ctx *bctx,
2727 struct io_kiocb *req)
2828 {
29+ const struct io_issue_def *def = &io_issue_defs[req->opcode];
30+
2931 bctx->opcode = req->opcode;
3032 bctx->sqe_flags = (__force int) req->flags & SQE_VALID_FLAGS;
3133 bctx->user_data = req->cqe.user_data;
@@ -34,19 +36,12 @@ static void io_uring_populate_bpf_ctx(struct io_uring_bpf_ctx *bctx, · 2 unmodified lines
3436 sizeof(*bctx) - offsetof(struct io_uring_bpf_ctx, pdu_size));
3537
3638 /*
37− * Opcodes can provide a handler fo populating more data into bctx,
39+ * Opcodes can provide a handler for populating more data into bctx,
3840 * for filters to use.
3941 */
40− switch (req->opcode) {
41− case IORING_OP_SOCKET:
42− bctx->pdu_size = sizeof(bctx->socket);
43− io_socket_bpf_populate(bctx, req);
44− break;
45− case IORING_OP_OPENAT:
46− case IORING_OP_OPENAT2:
47− bctx->pdu_size = sizeof(bctx->open);
48− io_openat_bpf_populate(bctx, req);
49− break;
42+ if (def->filter_pdu_size) {
43+ bctx->pdu_size = def->filter_pdu_size;
44+ def->filter_populate(bctx, req);
5045 }
5146 }
5247
@@ -313,36 +308,69 @@ err: · 260 unmodified lines
313308 return ERR_PTR(-EBUSY);
314309 }
315310
316−#define IO_URING_BPF_FILTER_FLAGS IO_URING_BPF_FILTER_DENY_REST
311+#define IO_URING_BPF_FILTER_FLAGS (IO_URING_BPF_FILTER_DENY_REST | \
312+ IO_URING_BPF_FILTER_SZ_STRICT)
317313
318−int io_register_bpf_filter(struct io_restriction *res,
319− struct io_uring_bpf __user *arg)
314+static int io_bpf_filter_import(struct io_uring_bpf *reg,
315+ struct io_uring_bpf __user *arg)
320316 {
321− struct io_bpf_filters *filters, *old_filters = NULL;
322− struct io_bpf_filter *filter, *old_filter;
323− struct io_uring_bpf reg;
324− struct bpf_prog *prog;
325− struct sock_fprog fprog;
317+ const struct io_issue_def *def;
326318 int ret;
327319
328− if (copy_from_user(&reg, arg, sizeof(reg)))
320+ if (copy_from_user(reg, arg, sizeof(*reg)))
329321 return -EFAULT;
330− if (reg.cmd_type != IO_URING_BPF_CMD_FILTER)
322+ if (reg->cmd_type != IO_URING_BPF_CMD_FILTER)
331323 return -EINVAL;
332− if (reg.cmd_flags || reg.resv)
324+ if (reg->cmd_flags || reg->resv)
333325 return -EINVAL;
334326
335− if (reg.filter.opcode >= IORING_OP_LAST)
327+ if (reg->filter.opcode >= IORING_OP_LAST)
336328 return -EINVAL;
337− if (reg.filter.flags & ~IO_URING_BPF_FILTER_FLAGS)
329+ if (reg->filter.flags & ~IO_URING_BPF_FILTER_FLAGS)
338330 return -EINVAL;
339− if (reg.filter.resv)
331+ if (!mem_is_zero(reg->filter.resv, sizeof(reg->filter.resv)))
340332 return -EINVAL;
341− if (!mem_is_zero(reg.filter.resv2, sizeof(reg.filter.resv2)))
333+ if (!mem_is_zero(reg->filter.resv2, sizeof(reg->filter.resv2)))
342334 return -EINVAL;
343− if (!reg.filter.filter_len || reg.filter.filter_len > BPF_MAXINSNS)
335+ if (!reg->filter.filter_len || reg->filter.filter_len > BPF_MAXINSNS)
344336 return -EINVAL;
345337
338+ /* Verify filter size */
339+ def = &io_issue_defs[array_index_nospec(reg->filter.opcode, IORING_OP_LAST)];
340+
341+ /* same size, always ok */
342+ ret = 0;
343+ if (reg->filter.pdu_size == def->filter_pdu_size)
344+ ;
345+ /* size differs, fail in strict mode */
346+ else if (reg->filter.flags & IO_URING_BPF_FILTER_SZ_STRICT)
347+ ret = -EMSGSIZE;
348+ /* userspace filter is bigger, always disallow */
349+ else if (reg->filter.pdu_size > def->filter_pdu_size)
350+ ret = -EMSGSIZE;
351+
352+ /* copy back kernel filter size */
353+ reg->filter.pdu_size = def->filter_pdu_size;
354+ if (copy_to_user(&arg->filter, &reg->filter, sizeof(reg->filter)))
355+ return -EFAULT;
356+
357+ return ret;
358+}
359+
360+int io_register_bpf_filter(struct io_restriction *res,
361+ struct io_uring_bpf __user *arg)
362+{
363+ struct io_bpf_filters *filters, *old_filters = NULL;
364+ struct io_bpf_filter *filter, *old_filter;
365+ struct io_uring_bpf reg;
366+ struct bpf_prog *prog;
367+ struct sock_fprog fprog;
368+ int ret;
369+
370+ ret = io_bpf_filter_import(&reg, arg);
371+ if (ret)
372+ return ret;
373+
346374 fprog.len = reg.filter.filter_len;
347375 fprog.filter = u64_to_user_ptr(reg.filter.filter_ptr);
348376
io_uring/cancel.h+2−4
@@ -6,10 +6,8 @@ · 5 unmodified lines
66
77 struct io_cancel_data {
88 struct io_ring_ctx *ctx;
9− union {
10− u64 data;
11− struct file *file;
12− };
9+ u64 data;
10+ struct file *file;
1311 u8 opcode;
1412 u32 flags;
1513 int seq;
io_uring/cmd_net.c+6−3
@@ -160,16 +160,19 @@ int io_uring_cmd_sock(struct io_uring_cmd *cmd, unsigned int issue_flags) · 159 unmodified lines
160160 struct proto *prot = READ_ONCE(sk->sk_prot);
161161 int ret, arg = 0;
162162
163− if (!prot || !prot->ioctl)
164− return -EOPNOTSUPP;
165−
166163 switch (cmd->cmd_op) {
167164 case SOCKET_URING_OP_SIOCINQ:
165+ if (!prot || !prot->ioctl)
166+ return -EOPNOTSUPP;
167+
168168 ret = prot->ioctl(sk, SIOCINQ, &arg);
169169 if (ret)
170170 return ret;
171171 return arg;
172172 case SOCKET_URING_OP_SIOCOUTQ:
173+ if (!prot || !prot->ioctl)
174+ return -EOPNOTSUPP;
175+
173176 ret = prot->ioctl(sk, SIOCOUTQ, &arg);
174177 if (ret)
175178 return ret;
io_uring/filetable.c+4−0
@@ -22,6 +22,10 @@ static int io_file_bitmap_get(struct io_ring_ctx *ctx) · 21 unmodified lines
2222 if (!table->bitmap)
2323 return -ENFILE;
2424
25+ if (table->alloc_hint < ctx->file_alloc_start ||
26+ table->alloc_hint >= ctx->file_alloc_end)
27+ table->alloc_hint = ctx->file_alloc_start;
28+
2529 do {
2630 ret = find_next_zero_bit(table->bitmap, nr, table->alloc_hint);
2731 if (ret != nr)
io_uring/io_uring.c+6−7
@@ -119,7 +119,7 @@ · 118 unmodified lines
119119 static void io_queue_sqe(struct io_kiocb *req, unsigned int extra_flags);
120120 static void __io_req_caches_free(struct io_ring_ctx *ctx);
121121
122−static __read_mostly DEFINE_STATIC_KEY_FALSE(io_key_has_sqarray);
122+static __read_mostly DEFINE_STATIC_KEY_DEFERRED_FALSE(io_key_has_sqarray, HZ);
123123
124124 struct kmem_cache *req_cachep;
125125 static struct workqueue_struct *iou_wq __ro_after_init;
@@ -1978,7 +1978,7 @@ static bool io_get_sqe(struct io_ring_ctx *ctx, const struct io_uring_sqe **sqe) · 1852 unmodified lines
19781978 unsigned mask = ctx->sq_entries - 1;
19791979 unsigned head = ctx->cached_sq_head++ & mask;
19801980
1981− if (static_branch_unlikely(&io_key_has_sqarray) &&
1981+ if (static_branch_unlikely(&io_key_has_sqarray.key) &&
19821982 (!(ctx->flags & IORING_SETUP_NO_SQARRAY))) {
19831983 head = READ_ONCE(ctx->sq_array[head]);
19841984 if (unlikely(head >= ctx->sq_entries)) {
@@ -2173,7 +2173,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) · 188 unmodified lines
21732173 io_rings_free(ctx);
21742174
21752175 if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
2176− static_branch_dec(&io_key_has_sqarray);
2176+ static_branch_slow_dec_deferred(&io_key_has_sqarray);
21772177
21782178 percpu_ref_exit(&ctx->refs);
21792179 free_uid(ctx->user);
@@ -2398,7 +2398,7 @@ static __cold void io_ring_exit_work(struct work_struct *work) · 218 unmodified lines
23982398 static __cold void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
23992399 {
24002400 unsigned long index;
2401− struct creds *creds;
2401+ struct cred *creds;
24022402
24032403 mutex_lock(&ctx->uring_lock);
24042404 percpu_ref_kill(&ctx->refs);
@@ -2946,11 +2946,10 @@ static __cold int io_uring_create(struct io_ctx_config *config) · 541 unmodified lines
29462946 ctx->clock_offset = 0;
29472947
29482948 if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
2949− static_branch_inc(&io_key_has_sqarray);
2949+ static_branch_deferred_inc(&io_key_has_sqarray);
29502950
29512951 if ((ctx->flags & IORING_SETUP_DEFER_TASKRUN) &&
2952− !(ctx->flags & IORING_SETUP_IOPOLL) &&
2953− !(ctx->flags & IORING_SETUP_SQPOLL))
2952+ !(ctx->flags & IORING_SETUP_IOPOLL))
29542953 ctx->task_complete = true;
29552954
29562955 if (ctx->task_complete || (ctx->flags & IORING_SETUP_IOPOLL))
io_uring/net.c+0−2
@@ -1493,8 +1493,6 @@ int io_send_zc(struct io_kiocb *req, unsigned int issue_flags) · 1492 unmodified lines
14931493 return -EAGAIN;
14941494
14951495 if (ret > 0 && io_net_retry(sock, kmsg->msg.msg_flags)) {
1496− zc->len -= ret;
1497− zc->buf += ret;
14981496 zc->done_io += ret;
14991497 return -EAGAIN;
15001498 }
io_uring/opdef.c+6−0
@@ -221,8 +221,10 @@ const struct io_issue_def io_issue_defs[] = { · 220 unmodified lines
221221 .issue = io_fallocate,
222222 },
223223 [IORING_OP_OPENAT] = {
224+ .filter_pdu_size = sizeof_field(struct io_uring_bpf_ctx, open),
224225 .prep = io_openat_prep,
225226 .issue = io_openat,
227+ .filter_populate = io_openat_bpf_populate,
226228 },
227229 [IORING_OP_CLOSE] = {
228230 .prep = io_close_prep,
@@ -309,8 +311,10 @@ const struct io_issue_def io_issue_defs[] = { · 80 unmodified lines
309311 #endif
310312 },
311313 [IORING_OP_OPENAT2] = {
314+ .filter_pdu_size = sizeof_field(struct io_uring_bpf_ctx, open),
312315 .prep = io_openat2_prep,
313316 .issue = io_openat2,
317+ .filter_populate = io_openat_bpf_populate,
314318 },
315319 [IORING_OP_EPOLL_CTL] = {
316320 .unbound_nonreg_file = 1,
@@ -406,8 +410,10 @@ const struct io_issue_def io_issue_defs[] = { · 89 unmodified lines
406410 [IORING_OP_SOCKET] = {
407411 .audit_skip = 1,
408412 #if defined(CONFIG_NET)
413+ .filter_pdu_size = sizeof_field(struct io_uring_bpf_ctx, socket),
409414 .prep = io_socket_prep,
410415 .issue = io_socket,
416+ .filter_populate = io_socket_bpf_populate,
411417 #else
412418 .prep = io_eopnotsupp_prep,
413419 #endif
io_uring/opdef.h+6−0
@@ -2,6 +2,8 @@ · 1 unmodified line
22 #ifndef IOU_OP_DEF_H
33 #define IOU_OP_DEF_H
44
5+struct io_uring_bpf_ctx;
6+
57 struct io_issue_def {
68 /* needs req->file assigned */
79 unsigned needs_file : 1;
@@ -33,8 +35,12 @@ struct io_issue_def { · 25 unmodified lines
3335 /* size of async data needed, if any */
3436 unsigned short async_size;
3537
38+ /* bpf filter pdu size, if any */
39+ unsigned short filter_pdu_size;
40+
3641 int (*issue)(struct io_kiocb *, unsigned int);
3742 int (*prep)(struct io_kiocb *, const struct io_uring_sqe *);
43+ void (*filter_populate)(struct io_uring_bpf_ctx *, struct io_kiocb *);
3844 };
3945
4046 struct io_cold_def {
io_uring/openclose.c+6−3
@@ -345,31 +345,34 @@ static int io_pipe_fixed(struct io_kiocb *req, struct file **files, · 344 unmodified lines
345345 {
346346 struct io_pipe *p = io_kiocb_to_cmd(req, struct io_pipe);
347347 struct io_ring_ctx *ctx = req->ctx;
348+ bool alloc_slot;
348349 int ret, fds[2] = { -1, -1 };
349350 int slot = p->file_slot;
350351
351352 if (p->flags & O_CLOEXEC)
352353 return -EINVAL;
353354
355+ alloc_slot = slot == IORING_FILE_INDEX_ALLOC;
356+
354357 io_ring_submit_lock(ctx, issue_flags);
355358
356359 ret = __io_fixed_fd_install(ctx, files[0], slot);
357360 if (ret < 0)
358361 goto err;
359− fds[0] = ret;
362+ fds[0] = alloc_slot ? ret : slot - 1;
360363 files[0] = NULL;
361364
362365 /*
363366 * If a specific slot is given, next one will be used for
364367 * the write side.
365368 */
366− if (slot != IORING_FILE_INDEX_ALLOC)
369+ if (!alloc_slot)
367370 slot++;
368371
369372 ret = __io_fixed_fd_install(ctx, files[1], slot);
370373 if (ret < 0)
371374 goto err;
372− fds[1] = ret;
375+ fds[1] = alloc_slot ? ret : slot - 1;
373376 files[1] = NULL;
374377
375378 io_ring_submit_unlock(ctx, issue_flags);
io_uring/query.c+1−1
@@ -39,7 +39,7 @@ static ssize_t io_query_zcrx(union io_query_data *data) · 38 unmodified lines
3939 e->nr_ctrl_opcodes = __ZCRX_CTRL_LAST;
4040 e->rq_hdr_size = sizeof(struct io_uring);
4141 e->rq_hdr_alignment = L1_CACHE_BYTES;
42− e->__resv1 = 0;
42+ e->features = ZCRX_FEATURE_RX_PAGE_SIZE;
4343 e->__resv2 = 0;
4444 return sizeof(*e);
4545 }
io_uring/rsrc.c+16−27
@@ -96,20 +96,6 @@ int io_validate_user_buf_range(u64 uaddr, u64 ulen) · 95 unmodified lines
9696 return 0;
9797 }
9898
99−static int io_buffer_validate(struct iovec *iov)
100−{
101− /*
102− * Don't impose further limits on the size and buffer
103− * constraints here, we'll -EINVAL later when IO is
104− * submitted if they are wrong.
105− */
106− if (!iov->iov_base)
107− return iov->iov_len ? -EFAULT : 0;
108−
109− return io_validate_user_buf_range((unsigned long)iov->iov_base,
110− iov->iov_len);
111−}
112−
11399 static void io_release_ubuf(void *priv)
114100 {
115101 struct io_mapped_ubuf *imu = priv;
@@ -319,9 +305,6 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx, · 203 unmodified lines
319305 err = -EFAULT;
320306 break;
321307 }
322− err = io_buffer_validate(iov);
323− if (err)
324− break;
325308 node = io_sqe_buffer_register(ctx, iov, &last_hpage);
326309 if (IS_ERR(node)) {
327310 err = PTR_ERR(node);
@@ -790,8 +773,17 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, · 462 unmodified lines
790773 struct io_imu_folio_data data;
791774 bool coalesced = false;
792775
793− if (!iov->iov_base)
776+ if (!iov->iov_base) {
777+ if (iov->iov_len)
778+ return ERR_PTR(-EFAULT);
779+ /* remove the buffer without installing a new one */
794780 return NULL;
781+ }
782+
783+ ret = io_validate_user_buf_range((unsigned long)iov->iov_base,
784+ iov->iov_len);
785+ if (ret)
786+ return ERR_PTR(ret);
795787
796788 node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER);
797789 if (!node)
@@ -828,7 +820,7 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, · 30 unmodified lines
828820 imu->folio_shift = PAGE_SHIFT;
829821 imu->release = io_release_ubuf;
830822 imu->priv = imu;
831− imu->is_kbuf = false;
823+ imu->flags = 0;
832824 imu->dir = IO_IMU_DEST | IO_IMU_SOURCE;
833825 if (coalesced)
834826 imu->folio_shift = data.folio_shift;
@@ -897,9 +889,6 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, · 62 unmodified lines
897889 ret = PTR_ERR(iov);
898890 break;
899891 }
900− ret = io_buffer_validate(iov);
901− if (ret)
902− break;
903892 if (ctx->compat)
904893 arg += sizeof(struct compat_iovec);
905894 else
@@ -985,7 +974,7 @@ int io_buffer_register_bvec(struct io_uring_cmd *cmd, struct request *rq, · 79 unmodified lines
985974 refcount_set(&imu->refs, 1);
986975 imu->release = release;
987976 imu->priv = rq;
988− imu->is_kbuf = true;
977+ imu->flags = IO_REGBUF_F_KBUF;
989978 imu->dir = 1 << rq_data_dir(rq);
990979
991980 rq_for_each_bvec(bv, rq, rq_iter)
@@ -1020,7 +1009,7 @@ int io_buffer_unregister_bvec(struct io_uring_cmd *cmd, unsigned int index, · 28 unmodified lines
10201009 ret = -EINVAL;
10211010 goto unlock;
10221011 }
1023− if (!node->buf->is_kbuf) {
1012+ if (!(node->buf->flags & IO_REGBUF_F_KBUF)) {
10241013 ret = -EBUSY;
10251014 goto unlock;
10261015 }
@@ -1076,7 +1065,7 @@ static int io_import_fixed(int ddir, struct iov_iter *iter, · 49 unmodified lines
10761065
10771066 offset = buf_addr - imu->ubuf;
10781067
1079− if (imu->is_kbuf)
1068+ if (imu->flags & IO_REGBUF_F_KBUF)
10801069 return io_import_kbuf(ddir, iter, imu, len, offset);
10811070
10821071 /*
@@ -1496,7 +1485,7 @@ int io_import_reg_vec(int ddir, struct iov_iter *iter, · 413 unmodified lines
14961485 iovec_off = vec->nr - nr_iovs;
14971486 iov = vec->iovec + iovec_off;
14981487
1499− if (imu->is_kbuf) {
1488+ if (imu->flags & IO_REGBUF_F_KBUF) {
15001489 int ret = io_kern_bvec_size(iov, nr_iovs, imu, &nr_segs);
15011490
15021491 if (unlikely(ret))
@@ -1534,7 +1523,7 @@ int io_import_reg_vec(int ddir, struct iov_iter *iter, · 31 unmodified lines
15341523 req->flags |= REQ_F_NEED_CLEANUP;
15351524 }
15361525
1537− if (imu->is_kbuf)
1526+ if (imu->flags & IO_REGBUF_F_KBUF)
15381527 return io_vec_fill_kern_bvec(ddir, iter, imu, iov, nr_iovs, vec);
15391528
15401529 return io_vec_fill_bvec(ddir, iter, imu, iov, nr_iovs, vec);
io_uring/rsrc.h+5−1
@@ -28,6 +28,10 @@ enum { · 27 unmodified lines
2828 IO_IMU_SOURCE = 1 << ITER_SOURCE,
2929 };
3030
31+enum {
32+ IO_REGBUF_F_KBUF = 1,
33+};
34+
3135 struct io_mapped_ubuf {
3236 u64 ubuf;
3337 unsigned int len;
@@ -37,7 +41,7 @@ struct io_mapped_ubuf { · 3 unmodified lines
3741 unsigned long acct_pages;
3842 void (*release)(void *);
3943 void *priv;
40− bool is_kbuf;
44+ u8 flags;
4145 u8 dir;
4246 struct bio_vec bvec[] __counted_by(nr_bvecs);
4347 };
io_uring/rw.c+2−1
@@ -702,7 +702,8 @@ static ssize_t loop_rw_iter(int ddir, struct io_rw *rw, struct iov_iter *iter) · 701 unmodified lines
702702 if ((kiocb->ki_flags & IOCB_NOWAIT) &&
703703 !(kiocb->ki_filp->f_flags & O_NONBLOCK))
704704 return -EAGAIN;
705− if ((req->flags & REQ_F_BUF_NODE) && req->buf_node->buf->is_kbuf)
705+ if ((req->flags & REQ_F_BUF_NODE) &&
706+ (req->buf_node->buf->flags & IO_REGBUF_F_KBUF))
706707 return -EFAULT;
707708
708709 ppos = io_kiocb_ppos(kiocb);
io_uring/tctx.c+5−5
@@ -240,14 +240,14 @@ void io_uring_unreg_ringfd(void) · 239 unmodified lines
240240 int io_ring_add_registered_file(struct io_uring_task *tctx, struct file *file,
241241 int start, int end)
242242 {
243− int offset;
243+ int offset, idx;
244244 for (offset = start; offset < end; offset++) {
245− offset = array_index_nospec(offset, IO_RINGFD_REG_MAX);
246− if (tctx->registered_rings[offset])
245+ idx = array_index_nospec(offset, IO_RINGFD_REG_MAX);
246+ if (tctx->registered_rings[idx])
247247 continue;
248248
249− tctx->registered_rings[offset] = file;
250− return offset;
249+ tctx->registered_rings[idx] = file;
250+ return idx;
251251 }
252252 return -EBUSY;
253253 }
io_uring/zcrx.c+9−7
@@ -205,7 +205,7 @@ static int io_import_umem(struct io_zcrx_ifq *ifq, · 204 unmodified lines
205205 return PTR_ERR(pages);
206206
207207 ret = sg_alloc_table_from_pages(&mem->page_sg_table, pages, nr_pages,
208− 0, nr_pages << PAGE_SHIFT,
208+ 0, (unsigned long)nr_pages << PAGE_SHIFT,
209209 GFP_KERNEL_ACCOUNT);
210210 if (ret) {
211211 unpin_user_pages(pages, nr_pages);
@@ -300,6 +300,9 @@ static int io_zcrx_map_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area) · 88 unmodified lines
300300 }
301301
302302 ret = io_populate_area_dma(ifq, area);
303+ if (ret && !area->mem.is_dmabuf)
304+ dma_unmap_sgtable(ifq->dev, &area->mem.page_sg_table,
305+ DMA_FROM_DEVICE, IO_DMA_ATTR);
303306 if (ret == 0)
304307 area->is_mapped = true;
305308 return ret;
@@ -538,9 +541,6 @@ static void io_close_queue(struct io_zcrx_ifq *ifq) · 232 unmodified lines
538541 .mp_priv = ifq,
539542 };
540543
541− if (ifq->if_rxq == -1)
542− return;
543−
544544 scoped_guard(mutex, &ifq->pp_lock) {
545545 netdev = ifq->netdev;
546546 netdev_tracker = ifq->netdev_tracker;
@@ -548,7 +548,8 @@ static void io_close_queue(struct io_zcrx_ifq *ifq) · 1 unmodified line
548548 }
549549
550550 if (netdev) {
551− net_mp_close_rxq(netdev, ifq->if_rxq, &p);
551+ if (ifq->if_rxq != -1)
552+ net_mp_close_rxq(netdev, ifq->if_rxq, &p);
552553 netdev_put(netdev, &netdev_tracker);
553554 }
554555 ifq->if_rxq = -1;
@@ -702,6 +703,8 @@ static int import_zcrx(struct io_ring_ctx *ctx, · 147 unmodified lines
702703 return -EINVAL;
703704 if (reg->if_rxq || reg->rq_entries || reg->area_ptr || reg->region_ptr)
704705 return -EINVAL;
706+ if (reg->flags & ~ZCRX_REG_IMPORT)
707+ return -EINVAL;
705708
706709 fd = reg->if_idx;
707710 CLASS(fd, f)(fd);
@@ -858,13 +861,12 @@ int io_register_zcrx_ifq(struct io_ring_ctx *ctx, · 150 unmodified lines
858861 }
859862 return 0;
860863 netdev_put_unlock:
861− netdev_put(ifq->netdev, &ifq->netdev_tracker);
862864 netdev_unlock(ifq->netdev);
863865 err:
864866 scoped_guard(mutex, &ctx->mmap_lock)
865867 xa_erase(&ctx->zcrx_ctxs, id);
866868 ifq_free:
867− io_zcrx_ifq_free(ifq);
869+ zcrx_unregister(ifq);
868870 return ret;
869871 }
870872