Not many aware, but io_uring submission queue has two levels. The first level usually appears as sq_array and stores indexes into the actual SQ. To my knowledge, no one has ever seriously used it, nor liburing exposes it to users. Add IORING_SETUP_NO_SQARRAY, when set we don't bother creating and using the sq_array and SQ heads/tails will be pointing directly into the SQ. Improves memory footprint, in term of both allocations as well as cache usage, and also should make io_get_sqe() less branchy in the end.
Unified · Split
| @@ -185,6 +185,11 @@ enum { · 184 unmodified lines | ||
| 185 | 185 | */ |
| 186 | 186 | #define IORING_SETUP_REGISTERED_FD_ONLY (1U << 15) |
| 187 | 187 | |
| 188 | +/* | |
| 189 | + * Removes indirection through the SQ index array. | |
| 190 | + */ | |
| 191 | +#define IORING_SETUP_NO_SQARRAY (1U << 16) | |
| 192 | + | |
| 188 | 193 | enum io_uring_op { |
| 189 | 194 | IORING_OP_NOP, |
| 190 | 195 | IORING_OP_READV, |
| @@ -2339,8 +2339,21 @@ static void io_commit_sqring(struct io_ring_ctx *ctx) · 2338 unmodified lines | ||
| 2339 | 2339 | */ |
| 2340 | 2340 | static bool io_get_sqe(struct io_ring_ctx *ctx, const struct io_uring_sqe **sqe) |
| 2341 | 2341 | { |
| 2342 | − unsigned head, mask = ctx->sq_entries - 1; | |
| 2343 | − unsigned sq_idx = ctx->cached_sq_head++ & mask; | |
| 2342 | + unsigned mask = ctx->sq_entries - 1; | |
| 2343 | + unsigned head = ctx->cached_sq_head++ & mask; | |
| 2344 | + | |
| 2345 | + if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) { | |
| 2346 | + head = READ_ONCE(ctx->sq_array[head]); | |
| 2347 | + if (unlikely(head >= ctx->sq_entries)) { | |
| 2348 | + /* drop invalid entries */ | |
| 2349 | + spin_lock(&ctx->completion_lock); | |
| 2350 | + ctx->cq_extra--; | |
| 2351 | + spin_unlock(&ctx->completion_lock); | |
| 2352 | + WRITE_ONCE(ctx->rings->sq_dropped, | |
| 2353 | + READ_ONCE(ctx->rings->sq_dropped) + 1); | |
| 2354 | + return false; | |
| 2355 | + } | |
| 2356 | + } | |
| 2344 | 2357 | |
| 2345 | 2358 | /* |
| 2346 | 2359 | * The cached sq head (or cq tail) serves two purposes: |
| @@ -2350,22 +2363,12 @@ static bool io_get_sqe(struct io_ring_ctx *ctx, const struct io_uring_sqe **sqe) · 3 unmodified lines | ||
| 2350 | 2363 | * 2) allows the kernel side to track the head on its own, even |
| 2351 | 2364 | * though the application is the one updating it. |
| 2352 | 2365 | */ |
| 2353 | − head = READ_ONCE(ctx->sq_array[sq_idx]); | |
| 2354 | − if (likely(head < ctx->sq_entries)) { | |
| 2355 | − /* double index for 128-byte SQEs, twice as long */ | |
| 2356 | − if (ctx->flags & IORING_SETUP_SQE128) | |
| 2357 | − head <<= 1; | |
| 2358 | − *sqe = &ctx->sq_sqes[head]; | |
| 2359 | − return true; | |
| 2360 | − } | |
| 2361 | 2366 | |
| 2362 | − /* drop invalid entries */ | |
| 2363 | − spin_lock(&ctx->completion_lock); | |
| 2364 | − ctx->cq_extra--; | |
| 2365 | − spin_unlock(&ctx->completion_lock); | |
| 2366 | − WRITE_ONCE(ctx->rings->sq_dropped, | |
| 2367 | − READ_ONCE(ctx->rings->sq_dropped) + 1); | |
| 2368 | − return false; | |
| 2367 | + /* double index for 128-byte SQEs, twice as long */ | |
| 2368 | + if (ctx->flags & IORING_SETUP_SQE128) | |
| 2369 | + head <<= 1; | |
| 2370 | + *sqe = &ctx->sq_sqes[head]; | |
| 2371 | + return true; | |
| 2369 | 2372 | } |
| 2370 | 2373 | |
| 2371 | 2374 | int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) |
| @@ -2734,6 +2737,12 @@ static unsigned long rings_size(struct io_ring_ctx *ctx, unsigned int sq_entries · 362 unmodified lines | ||
| 2734 | 2737 | return SIZE_MAX; |
| 2735 | 2738 | #endif |
| 2736 | 2739 | |
| 2740 | + if (ctx->flags & IORING_SETUP_NO_SQARRAY) { | |
| 2741 | + if (sq_offset) | |
| 2742 | + *sq_offset = SIZE_MAX; | |
| 2743 | + return off; | |
| 2744 | + } | |
| 2745 | + | |
| 2737 | 2746 | if (sq_offset) |
| 2738 | 2747 | *sq_offset = off; |
| 2739 | 2748 | |
| @@ -3710,7 +3719,8 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, · 970 unmodified lines | ||
| 3710 | 3719 | return PTR_ERR(rings); |
| 3711 | 3720 | |
| 3712 | 3721 | ctx->rings = rings; |
| 3713 | − ctx->sq_array = (u32 *)((char *)rings + sq_array_offset); | |
| 3722 | + if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) | |
| 3723 | + ctx->sq_array = (u32 *)((char *)rings + sq_array_offset); | |
| 3714 | 3724 | rings->sq_ring_mask = p->sq_entries - 1; |
| 3715 | 3725 | rings->cq_ring_mask = p->cq_entries - 1; |
| 3716 | 3726 | rings->sq_ring_entries = p->sq_entries; |
| @@ -3921,7 +3931,8 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, · 204 unmodified lines | ||
| 3921 | 3931 | p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries); |
| 3922 | 3932 | p->sq_off.flags = offsetof(struct io_rings, sq_flags); |
| 3923 | 3933 | p->sq_off.dropped = offsetof(struct io_rings, sq_dropped); |
| 3924 | − p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings; | |
| 3934 | + if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) | |
| 3935 | + p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings; | |
| 3925 | 3936 | p->sq_off.resv1 = 0; |
| 3926 | 3937 | if (!(ctx->flags & IORING_SETUP_NO_MMAP)) |
| 3927 | 3938 | p->sq_off.user_addr = 0; |
| @@ -4010,7 +4021,8 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params) · 82 unmodified lines | ||
| 4010 | 4021 | IORING_SETUP_COOP_TASKRUN | IORING_SETUP_TASKRUN_FLAG | |
| 4011 | 4022 | IORING_SETUP_SQE128 | IORING_SETUP_CQE32 | |
| 4012 | 4023 | IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_DEFER_TASKRUN | |
| 4013 | − IORING_SETUP_NO_MMAP | IORING_SETUP_REGISTERED_FD_ONLY)) | |
| 4024 | + IORING_SETUP_NO_MMAP | IORING_SETUP_REGISTERED_FD_ONLY | | |
| 4025 | + IORING_SETUP_NO_SQARRAY)) | |
| 4014 | 4026 | return -EINVAL; |
| 4015 | 4027 | |
| 4016 | 4028 | return io_uring_create(entries, &p, params); |