Not many aware, but io_uring submission queue has two levels. The first level usually appears as sq_array and stores indexes into the actual SQ. To my knowledge, no one has ever seriously used it, nor liburing exposes it to users. Add IORING_SETUP_NO_SQARRAY, when set we don't bother creating and using the sq_array and SQ heads/tails will be pointing directly into the SQ. Improves memory footprint, in term of both allocations as well as cache usage, and also should make io_get_sqe() less branchy in the end.
Unified · Split
| @@ -185,6 +185,11 @@ enum { · 184 unmodified lines | |||
| 185 | */ | 185 | */ |
| 186 | #define IORING_SETUP_REGISTERED_FD_ONLY (1U << 15) | 186 | #define IORING_SETUP_REGISTERED_FD_ONLY (1U << 15) |
| 187 | 187 | ||
| 188 | /* | ||
| 189 | * Removes indirection through the SQ index array. | ||
| 190 | */ | ||
| 191 | #define IORING_SETUP_NO_SQARRAY (1U << 16) | ||
| 192 | |||
| 188 | enum io_uring_op { | 193 | enum io_uring_op { |
| 189 | IORING_OP_NOP, | 194 | IORING_OP_NOP, |
| 190 | IORING_OP_READV, | 195 | IORING_OP_READV, |
| @@ -2339,8 +2339,21 @@ static void io_commit_sqring(struct io_ring_ctx *ctx) · 2338 unmodified lines | |||
| 2339 | */ | 2339 | */ |
| 2340 | static bool io_get_sqe(struct io_ring_ctx *ctx, const struct io_uring_sqe **sqe) | 2340 | static bool io_get_sqe(struct io_ring_ctx *ctx, const struct io_uring_sqe **sqe) |
| 2341 | { | 2341 | { |
| 2342 | unsigned head, mask = ctx->sq_entries - 1; | 2342 | unsigned mask = ctx->sq_entries - 1; |
| 2343 | unsigned sq_idx = ctx->cached_sq_head++ & mask; | 2343 | unsigned head = ctx->cached_sq_head++ & mask; |
| 2344 | |||
| 2345 | if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) { | ||
| 2346 | head = READ_ONCE(ctx->sq_array[head]); | ||
| 2347 | if (unlikely(head >= ctx->sq_entries)) { | ||
| 2348 | /* drop invalid entries */ | ||
| 2349 | spin_lock(&ctx->completion_lock); | ||
| 2350 | ctx->cq_extra--; | ||
| 2351 | spin_unlock(&ctx->completion_lock); | ||
| 2352 | WRITE_ONCE(ctx->rings->sq_dropped, | ||
| 2353 | READ_ONCE(ctx->rings->sq_dropped) + 1); | ||
| 2354 | return false; | ||
| 2355 | } | ||
| 2356 | } | ||
| 2344 | 2357 | ||
| 2345 | /* | 2358 | /* |
| 2346 | * The cached sq head (or cq tail) serves two purposes: | 2359 | * The cached sq head (or cq tail) serves two purposes: |
| @@ -2350,22 +2363,12 @@ static bool io_get_sqe(struct io_ring_ctx *ctx, const struct io_uring_sqe **sqe) · 3 unmodified lines | |||
| 2350 | * 2) allows the kernel side to track the head on its own, even | 2363 | * 2) allows the kernel side to track the head on its own, even |
| 2351 | * though the application is the one updating it. | 2364 | * though the application is the one updating it. |
| 2352 | */ | 2365 | */ |
| 2353 | head = READ_ONCE(ctx->sq_array[sq_idx]); | ||
| 2354 | if (likely(head < ctx->sq_entries)) { | ||
| 2355 | /* double index for 128-byte SQEs, twice as long */ | ||
| 2356 | if (ctx->flags & IORING_SETUP_SQE128) | ||
| 2357 | head <<= 1; | ||
| 2358 | *sqe = &ctx->sq_sqes[head]; | ||
| 2359 | return true; | ||
| 2360 | } | ||
| 2361 | 2366 | ||
| 2362 | /* drop invalid entries */ | 2367 | /* double index for 128-byte SQEs, twice as long */ |
| 2363 | spin_lock(&ctx->completion_lock); | 2368 | if (ctx->flags & IORING_SETUP_SQE128) |
| 2364 | ctx->cq_extra--; | 2369 | head <<= 1; |
| 2365 | spin_unlock(&ctx->completion_lock); | 2370 | *sqe = &ctx->sq_sqes[head]; |
| 2366 | WRITE_ONCE(ctx->rings->sq_dropped, | 2371 | return true; |
| 2367 | READ_ONCE(ctx->rings->sq_dropped) + 1); | ||
| 2368 | return false; | ||
| 2369 | } | 2372 | } |
| 2370 | 2373 | ||
| 2371 | int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) | 2374 | int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) |
| @@ -2734,6 +2737,12 @@ static unsigned long rings_size(struct io_ring_ctx *ctx, unsigned int sq_entries · 362 unmodified lines | |||
| 2734 | return SIZE_MAX; | 2737 | return SIZE_MAX; |
| 2735 | #endif | 2738 | #endif |
| 2736 | 2739 | ||
| 2740 | if (ctx->flags & IORING_SETUP_NO_SQARRAY) { | ||
| 2741 | if (sq_offset) | ||
| 2742 | *sq_offset = SIZE_MAX; | ||
| 2743 | return off; | ||
| 2744 | } | ||
| 2745 | |||
| 2737 | if (sq_offset) | 2746 | if (sq_offset) |
| 2738 | *sq_offset = off; | 2747 | *sq_offset = off; |
| 2739 | 2748 | ||
| @@ -3710,7 +3719,8 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, · 970 unmodified lines | |||
| 3710 | return PTR_ERR(rings); | 3719 | return PTR_ERR(rings); |
| 3711 | 3720 | ||
| 3712 | ctx->rings = rings; | 3721 | ctx->rings = rings; |
| 3713 | ctx->sq_array = (u32 *)((char *)rings + sq_array_offset); | 3722 | if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) |
| 3723 | ctx->sq_array = (u32 *)((char *)rings + sq_array_offset); | ||
| 3714 | rings->sq_ring_mask = p->sq_entries - 1; | 3724 | rings->sq_ring_mask = p->sq_entries - 1; |
| 3715 | rings->cq_ring_mask = p->cq_entries - 1; | 3725 | rings->cq_ring_mask = p->cq_entries - 1; |
| 3716 | rings->sq_ring_entries = p->sq_entries; | 3726 | rings->sq_ring_entries = p->sq_entries; |
| @@ -3921,7 +3931,8 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, · 204 unmodified lines | |||
| 3921 | p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries); | 3931 | p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries); |
| 3922 | p->sq_off.flags = offsetof(struct io_rings, sq_flags); | 3932 | p->sq_off.flags = offsetof(struct io_rings, sq_flags); |
| 3923 | p->sq_off.dropped = offsetof(struct io_rings, sq_dropped); | 3933 | p->sq_off.dropped = offsetof(struct io_rings, sq_dropped); |
| 3924 | p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings; | 3934 | if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) |
| 3935 | p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings; | ||
| 3925 | p->sq_off.resv1 = 0; | 3936 | p->sq_off.resv1 = 0; |
| 3926 | if (!(ctx->flags & IORING_SETUP_NO_MMAP)) | 3937 | if (!(ctx->flags & IORING_SETUP_NO_MMAP)) |
| 3927 | p->sq_off.user_addr = 0; | 3938 | p->sq_off.user_addr = 0; |
| @@ -4010,7 +4021,8 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params) · 82 unmodified lines | |||
| 4010 | IORING_SETUP_COOP_TASKRUN | IORING_SETUP_TASKRUN_FLAG | | 4021 | IORING_SETUP_COOP_TASKRUN | IORING_SETUP_TASKRUN_FLAG | |
| 4011 | IORING_SETUP_SQE128 | IORING_SETUP_CQE32 | | 4022 | IORING_SETUP_SQE128 | IORING_SETUP_CQE32 | |
| 4012 | IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_DEFER_TASKRUN | | 4023 | IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_DEFER_TASKRUN | |
| 4013 | IORING_SETUP_NO_MMAP | IORING_SETUP_REGISTERED_FD_ONLY)) | 4024 | IORING_SETUP_NO_MMAP | IORING_SETUP_REGISTERED_FD_ONLY | |
| 4025 | IORING_SETUP_NO_SQARRAY)) | ||
| 4014 | return -EINVAL; | 4026 | return -EINVAL; |
| 4015 | 4027 | ||
| 4016 | return io_uring_create(entries, &p, params); | 4028 | return io_uring_create(entries, &p, params); |