Returning buffers via a ring is performant and convenient, but it becomes a problem when/if the user misconfigured the ring size and it becomes full. Add a synchronous way to return buffers back to the page pool via a new register opcode. It's supposed to be a reliable slow path for refilling.
Unified · Split
| @@ -689,6 +689,9 @@ enum io_uring_register_op { · 688 unmodified lines | ||
| 689 | 689 | /* query various aspects of io_uring, see linux/io_uring/query.h */ |
| 690 | 690 | IORING_REGISTER_QUERY = 35, |
| 691 | 691 | |
| 692 | + /* return zcrx buffers back into circulation */ | |
| 693 | + IORING_REGISTER_ZCRX_REFILL = 36, | |
| 694 | + | |
| 692 | 695 | /* this goes last */ |
| 693 | 696 | IORING_REGISTER_LAST, |
| 694 | 697 | |
| @@ -1070,6 +1073,15 @@ struct io_uring_zcrx_ifq_reg { · 375 unmodified lines | ||
| 1070 | 1073 | __u64 __resv[3]; |
| 1071 | 1074 | }; |
| 1072 | 1075 | |
| 1076 | +struct io_uring_zcrx_sync_refill { | |
| 1077 | + __u32 zcrx_id; | |
| 1078 | + /* the number of entries to return */ | |
| 1079 | + __u32 nr_entries; | |
| 1080 | + /* pointer to an array of struct io_uring_zcrx_rqe */ | |
| 1081 | + __u64 rqes; | |
| 1082 | + __u64 __resv[2]; | |
| 1083 | +}; | |
| 1084 | + | |
| 1073 | 1085 | #ifdef __cplusplus |
| 1074 | 1086 | } |
| 1075 | 1087 | #endif |
| @@ -833,6 +833,9 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, · 832 unmodified lines | ||
| 833 | 833 | case IORING_REGISTER_QUERY: |
| 834 | 834 | ret = io_query(ctx, arg, nr_args); |
| 835 | 835 | break; |
| 836 | + case IORING_REGISTER_ZCRX_REFILL: | |
| 837 | + ret = io_zcrx_return_bufs(ctx, arg, nr_args); | |
| 838 | + break; | |
| 836 | 839 | default: |
| 837 | 840 | ret = -EINVAL; |
| 838 | 841 | break; |
| @@ -927,6 +927,74 @@ static const struct memory_provider_ops io_uring_pp_zc_ops = { · 926 unmodified lines | ||
| 927 | 927 | .uninstall = io_pp_uninstall, |
| 928 | 928 | }; |
| 929 | 929 | |
| 930 | +#define IO_ZCRX_MAX_SYS_REFILL_BUFS (1 << 16) | |
| 931 | +#define IO_ZCRX_SYS_REFILL_BATCH 32 | |
| 932 | + | |
| 933 | +static void io_return_buffers(struct io_zcrx_ifq *ifq, | |
| 934 | + struct io_uring_zcrx_rqe *rqes, unsigned nr) | |
| 935 | +{ | |
| 936 | + int i; | |
| 937 | + | |
| 938 | + for (i = 0; i < nr; i++) { | |
| 939 | + struct net_iov *niov; | |
| 940 | + netmem_ref netmem; | |
| 941 | + | |
| 942 | + if (!io_parse_rqe(&rqes[i], ifq, &niov)) | |
| 943 | + continue; | |
| 944 | + | |
| 945 | + scoped_guard(spinlock_bh, &ifq->rq_lock) { | |
| 946 | + if (!io_zcrx_put_niov_uref(niov)) | |
| 947 | + continue; | |
| 948 | + } | |
| 949 | + | |
| 950 | + netmem = net_iov_to_netmem(niov); | |
| 951 | + if (!page_pool_unref_and_test(netmem)) | |
| 952 | + continue; | |
| 953 | + io_zcrx_return_niov(niov); | |
| 954 | + } | |
| 955 | +} | |
| 956 | + | |
| 957 | +int io_zcrx_return_bufs(struct io_ring_ctx *ctx, | |
| 958 | + void __user *arg, unsigned nr_arg) | |
| 959 | +{ | |
| 960 | + struct io_uring_zcrx_rqe rqes[IO_ZCRX_SYS_REFILL_BATCH]; | |
| 961 | + struct io_uring_zcrx_rqe __user *user_rqes; | |
| 962 | + struct io_uring_zcrx_sync_refill zr; | |
| 963 | + struct io_zcrx_ifq *ifq; | |
| 964 | + unsigned nr, i; | |
| 965 | + | |
| 966 | + if (nr_arg) | |
| 967 | + return -EINVAL; | |
| 968 | + if (copy_from_user(&zr, arg, sizeof(zr))) | |
| 969 | + return -EFAULT; | |
| 970 | + if (!zr.nr_entries || zr.nr_entries > IO_ZCRX_MAX_SYS_REFILL_BUFS) | |
| 971 | + return -EINVAL; | |
| 972 | + if (!mem_is_zero(&zr.__resv, sizeof(zr.__resv))) | |
| 973 | + return -EINVAL; | |
| 974 | + | |
| 975 | + ifq = xa_load(&ctx->zcrx_ctxs, zr.zcrx_id); | |
| 976 | + if (!ifq) | |
| 977 | + return -EINVAL; | |
| 978 | + nr = zr.nr_entries; | |
| 979 | + user_rqes = u64_to_user_ptr(zr.rqes); | |
| 980 | + | |
| 981 | + for (i = 0; i < nr;) { | |
| 982 | + unsigned batch = min(nr - i, IO_ZCRX_SYS_REFILL_BATCH); | |
| 983 | + size_t size = batch * sizeof(rqes[0]); | |
| 984 | + | |
| 985 | + if (copy_from_user(rqes, user_rqes + i, size)) | |
| 986 | + return i ? i : -EFAULT; | |
| 987 | + io_return_buffers(ifq, rqes, batch); | |
| 988 | + | |
| 989 | + i += batch; | |
| 990 | + | |
| 991 | + if (fatal_signal_pending(current)) | |
| 992 | + return i; | |
| 993 | + cond_resched(); | |
| 994 | + } | |
| 995 | + return nr; | |
| 996 | +} | |
| 997 | + | |
| 930 | 998 | static bool io_zcrx_queue_cqe(struct io_kiocb *req, struct net_iov *niov, |
| 931 | 999 | struct io_zcrx_ifq *ifq, int off, int len) |
| 932 | 1000 | { |
| @@ -63,6 +63,8 @@ struct io_zcrx_ifq { · 62 unmodified lines | ||
| 63 | 63 | }; |
| 64 | 64 | |
| 65 | 65 | #if defined(CONFIG_IO_URING_ZCRX) |
| 66 | +int io_zcrx_return_bufs(struct io_ring_ctx *ctx, | |
| 67 | + void __user *arg, unsigned nr_arg); | |
| 66 | 68 | int io_register_zcrx_ifq(struct io_ring_ctx *ctx, |
| 67 | 69 | struct io_uring_zcrx_ifq_reg __user *arg); |
| 68 | 70 | void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx); |
| @@ -95,6 +97,11 @@ static inline struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ct · 26 unmodified lines | ||
| 95 | 97 | { |
| 96 | 98 | return NULL; |
| 97 | 99 | } |
| 100 | +static inline int io_zcrx_return_bufs(struct io_ring_ctx *ctx, | |
| 101 | + void __user *arg, unsigned nr_arg) | |
| 102 | +{ | |
| 103 | + return -EOPNOTSUPP; | |
| 104 | +} | |
| 98 | 105 | #endif |
| 99 | 106 | |
| 100 | 107 | int io_recvzc(struct io_kiocb *req, unsigned int issue_flags); |