History

Merge tag 'for-5.20/io_uring-zerocopy-send-2022-07-29' of git://git.kernel.dk/linux-block

Pull io_uring zerocopy support from Jens Axboe: "This adds support for efficient support for zerocopy sends through io_uring. Both ipv4 and ipv6 is supported, as well as both TCP and UDP. The core network changes to support this is in a stable branch from Jakub that both io_uring and net-next has pulled in, and the io_uring changes are layered on top of that. All of the work has been done by Pavel" * tag 'for-5.20/io_uring-zerocopy-send-2022-07-29' of git://git.kernel.dk/linux-block: (34 commits) io_uring: notification completion optimisation io_uring: export req alloc from core io_uring/net: use unsigned for flags io_uring/net: make page accounting more consistent io_uring/net: checks errors of zc mem accounting io_uring/net: improve io_get_notif_slot types selftests/io_uring: test zerocopy send io_uring: enable managed frags with register buffers io_uring: add zc notification flush requests io_uring: rename IORING_OP_FILES_UPDATE io_uring: flush notifiers after sendzc io_uring: sendzc with fixed buffers io_uring: allow to pass addr into sendzc io_uring: account locked pages for non-fixed zc io_uring: wire send zc request type io_uring: add notification slot registration io_uring: add rsrc referencing for notifiers io_uring: complete notifiers in tw io_uring: cache struct io_notif io_uring: add zc notification infrastructure ...

Linus Torvalds committed 4 years ago

Commit 42df1cb2 parents 98e2474 14b146b25 files · +1604 −158Browse files

Showing the changes to include/uapi (1 of 25 files). Show all

Unified · Split

include/uapi/linux/io_uring.h+43−2
@@ -66,6 +66,10 @@ struct io_uring_sqe { · 65 unmodified lines
66 union {66 union {
67 __s32 splice_fd_in;67 __s32 splice_fd_in;
68 __u32 file_index;68 __u32 file_index;
69 struct {
70 __u16 notification_idx;
71 __u16 addr_len;
72 };
69 };73 };
70 union {74 union {
71 struct {75 struct {
@@ -170,7 +174,8 @@ enum io_uring_op { · 98 unmodified lines
170 IORING_OP_FALLOCATE,174 IORING_OP_FALLOCATE,
171 IORING_OP_OPENAT,175 IORING_OP_OPENAT,
172 IORING_OP_CLOSE,176 IORING_OP_CLOSE,
173 IORING_OP_FILES_UPDATE,177 IORING_OP_RSRC_UPDATE,
178 IORING_OP_FILES_UPDATE = IORING_OP_RSRC_UPDATE,
174 IORING_OP_STATX,179 IORING_OP_STATX,
175 IORING_OP_READ,180 IORING_OP_READ,
176 IORING_OP_WRITE,181 IORING_OP_WRITE,
@@ -197,6 +202,7 @@ enum io_uring_op { · 20 unmodified lines
197 IORING_OP_GETXATTR,202 IORING_OP_GETXATTR,
198 IORING_OP_SOCKET,203 IORING_OP_SOCKET,
199 IORING_OP_URING_CMD,204 IORING_OP_URING_CMD,
205 IORING_OP_SENDZC_NOTIF,
200 206
201 /* this goes last, obviously */207 /* this goes last, obviously */
202 IORING_OP_LAST,208 IORING_OP_LAST,
@@ -218,6 +224,7 @@ enum io_uring_op { · 15 unmodified lines
218#define IORING_TIMEOUT_ETIME_SUCCESS (1U << 5)224#define IORING_TIMEOUT_ETIME_SUCCESS (1U << 5)
219#define IORING_TIMEOUT_CLOCK_MASK (IORING_TIMEOUT_BOOTTIME | IORING_TIMEOUT_REALTIME)225#define IORING_TIMEOUT_CLOCK_MASK (IORING_TIMEOUT_BOOTTIME | IORING_TIMEOUT_REALTIME)
220#define IORING_TIMEOUT_UPDATE_MASK (IORING_TIMEOUT_UPDATE | IORING_LINK_TIMEOUT_UPDATE)226#define IORING_TIMEOUT_UPDATE_MASK (IORING_TIMEOUT_UPDATE | IORING_LINK_TIMEOUT_UPDATE)
227
221/*228/*
222 * sqe->splice_flags229 * sqe->splice_flags
223 * extends splice(2) flags230 * extends splice(2) flags
@@ -267,15 +274,32 @@ enum io_uring_op { · 43 unmodified lines
267 * IORING_RECV_MULTISHOT Multishot recv. Sets IORING_CQE_F_MORE if274 * IORING_RECV_MULTISHOT Multishot recv. Sets IORING_CQE_F_MORE if
268 * the handler will continue to report275 * the handler will continue to report
269 * CQEs on behalf of the same SQE.276 * CQEs on behalf of the same SQE.
277 *
278 * IORING_RECVSEND_FIXED_BUF Use registered buffers, the index is stored in
279 * the buf_index field.
280 *
281 * IORING_RECVSEND_NOTIF_FLUSH Flush a notification after a successful
282 * successful. Only for zerocopy sends.
270 */283 */
271#define IORING_RECVSEND_POLL_FIRST (1U << 0)284#define IORING_RECVSEND_POLL_FIRST (1U << 0)
272#define IORING_RECV_MULTISHOT (1U << 1)285#define IORING_RECV_MULTISHOT (1U << 1)
286#define IORING_RECVSEND_FIXED_BUF (1U << 2)
287#define IORING_RECVSEND_NOTIF_FLUSH (1U << 3)
273 288
274/*289/*
275 * accept flags stored in sqe->ioprio290 * accept flags stored in sqe->ioprio
276 */291 */
277#define IORING_ACCEPT_MULTISHOT (1U << 0)292#define IORING_ACCEPT_MULTISHOT (1U << 0)
278 293
294
295/*
296 * IORING_OP_RSRC_UPDATE flags
297 */
298enum {
299 IORING_RSRC_UPDATE_FILES,
300 IORING_RSRC_UPDATE_NOTIF,
301};
302
279/*303/*
280 * IORING_OP_MSG_RING command types, stored in sqe->addr304 * IORING_OP_MSG_RING command types, stored in sqe->addr
281 */305 */
@@ -457,6 +481,10 @@ enum { · 175 unmodified lines
457 /* register a range of fixed file slots for automatic slot allocation */481 /* register a range of fixed file slots for automatic slot allocation */
458 IORING_REGISTER_FILE_ALLOC_RANGE = 25,482 IORING_REGISTER_FILE_ALLOC_RANGE = 25,
459 483
484 /* zerocopy notification API */
485 IORING_REGISTER_NOTIFIERS = 26,
486 IORING_UNREGISTER_NOTIFIERS = 27,
487
460 /* this goes last */488 /* this goes last */
461 IORING_REGISTER_LAST489 IORING_REGISTER_LAST
462};490};
@@ -503,6 +531,19 @@ struct io_uring_rsrc_update2 { · 40 unmodified lines
503 __u32 resv2;531 __u32 resv2;
504};532};
505 533
534struct io_uring_notification_slot {
535 __u64 tag;
536 __u64 resv[3];
537};
538
539struct io_uring_notification_register {
540 __u32 nr_slots;
541 __u32 resv;
542 __u64 resv2;
543 __u64 data;
544 __u64 resv3;
545};
546
506/* Skip updating fd indexes set to this value in the fd table */547/* Skip updating fd indexes set to this value in the fd table */
507#define IORING_REGISTER_FILES_SKIP (-2)548#define IORING_REGISTER_FILES_SKIP (-2)
508 549