Add a new object called an interface queue (ifq) that represents a net rx queue that has been configured for zero copy. Each ifq is registered using a new registration opcode IORING_REGISTER_ZCRX_IFQ. The refill queue is allocated by the kernel and mapped by userspace using a new offset IORING_OFF_RQ_RING, in a similar fashion to the main SQ/CQ. It is used by userspace to return buffers that it is done with, which will then be re-used by the netdev again. The main CQ ring is used to notify userspace of received data by using the upper 16 bytes of a big CQE as a new struct io_uring_zcrx_cqe. Each entry contains the offset + len to the data. For now, each io_uring instance only has a single ifq.
Showing the changes to include/uapi/linux (1 of 10 files). Show all
Unified · Split
| @@ -639,7 +639,8 @@ enum io_uring_register_op { · 638 unmodified lines | ||
| 639 | 639 | /* send MSG_RING without having a ring */ |
| 640 | 640 | IORING_REGISTER_SEND_MSG_RING = 31, |
| 641 | 641 | |
| 642 | − /* 32 reserved for zc rx */ | |
| 642 | + /* register a netdev hw rx queue for zerocopy */ | |
| 643 | + IORING_REGISTER_ZCRX_IFQ = 32, | |
| 643 | 644 | |
| 644 | 645 | /* resize CQ ring */ |
| 645 | 646 | IORING_REGISTER_RESIZE_RINGS = 33, |
| @@ -956,6 +957,46 @@ enum io_uring_socket_op { · 310 unmodified lines | ||
| 956 | 957 | SOCKET_URING_OP_SETSOCKOPT, |
| 957 | 958 | }; |
| 958 | 959 | |
| 960 | +/* Zero copy receive refill queue entry */ | |
| 961 | +struct io_uring_zcrx_rqe { | |
| 962 | + __u64 off; | |
| 963 | + __u32 len; | |
| 964 | + __u32 __pad; | |
| 965 | +}; | |
| 966 | + | |
| 967 | +struct io_uring_zcrx_cqe { | |
| 968 | + __u64 off; | |
| 969 | + __u64 __pad; | |
| 970 | +}; | |
| 971 | + | |
| 972 | +/* The bit from which area id is encoded into offsets */ | |
| 973 | +#define IORING_ZCRX_AREA_SHIFT 48 | |
| 974 | +#define IORING_ZCRX_AREA_MASK (~(((__u64)1 << IORING_ZCRX_AREA_SHIFT) - 1)) | |
| 975 | + | |
| 976 | +struct io_uring_zcrx_offsets { | |
| 977 | + __u32 head; | |
| 978 | + __u32 tail; | |
| 979 | + __u32 rqes; | |
| 980 | + __u32 __resv2; | |
| 981 | + __u64 __resv[2]; | |
| 982 | +}; | |
| 983 | + | |
| 984 | +/* | |
| 985 | + * Argument for IORING_REGISTER_ZCRX_IFQ | |
| 986 | + */ | |
| 987 | +struct io_uring_zcrx_ifq_reg { | |
| 988 | + __u32 if_idx; | |
| 989 | + __u32 if_rxq; | |
| 990 | + __u32 rq_entries; | |
| 991 | + __u32 flags; | |
| 992 | + | |
| 993 | + __u64 area_ptr; /* pointer to struct io_uring_zcrx_area_reg */ | |
| 994 | + __u64 region_ptr; /* struct io_uring_region_desc * */ | |
| 995 | + | |
| 996 | + struct io_uring_zcrx_offsets offsets; | |
| 997 | + __u64 __resv[4]; | |
| 998 | +}; | |
| 999 | + | |
| 959 | 1000 | #ifdef __cplusplus |
| 960 | 1001 | } |
| 961 | 1002 | #endif |