This adds the sqpoll support to the io-uring napi.
Unified · Split
| @@ -252,4 +252,28 @@ void __io_napi_busy_loop(struct io_ring_ctx *ctx, struct io_wait_queue *iowq) · 251 unmodified lines | |||
| 252 | io_napi_blocking_busy_loop(ctx, iowq); | 252 | io_napi_blocking_busy_loop(ctx, iowq); |
| 253 | } | 253 | } |
| 254 | 254 | ||
| 255 | /* | ||
| 256 | * io_napi_sqpoll_busy_poll() - busy poll loop for sqpoll | ||
| 257 | * @ctx: pointer to io-uring context structure | ||
| 258 | * | ||
| 259 | * Splice of the napi list and execute the napi busy poll loop. | ||
| 260 | */ | ||
| 261 | int io_napi_sqpoll_busy_poll(struct io_ring_ctx *ctx) | ||
| 262 | { | ||
| 263 | LIST_HEAD(napi_list); | ||
| 264 | bool is_stale = false; | ||
| 265 | |||
| 266 | if (!READ_ONCE(ctx->napi_busy_poll_to)) | ||
| 267 | return 0; | ||
| 268 | if (list_empty_careful(&ctx->napi_list)) | ||
| 269 | return 0; | ||
| 270 | |||
| 271 | rcu_read_lock(); | ||
| 272 | is_stale = __io_napi_do_busy_loop(ctx, NULL); | ||
| 273 | rcu_read_unlock(); | ||
| 274 | |||
| 275 | io_napi_remove_stale(ctx, is_stale); | ||
| 276 | return 1; | ||
| 277 | } | ||
| 278 | |||
| 255 | #endif | 279 | #endif |
| @@ -17,6 +17,7 @@ void __io_napi_add(struct io_ring_ctx *ctx, struct socket *sock); · 16 unmodified lines | |||
| 17 | void __io_napi_adjust_timeout(struct io_ring_ctx *ctx, | 17 | void __io_napi_adjust_timeout(struct io_ring_ctx *ctx, |
| 18 | struct io_wait_queue *iowq, struct timespec64 *ts); | 18 | struct io_wait_queue *iowq, struct timespec64 *ts); |
| 19 | void __io_napi_busy_loop(struct io_ring_ctx *ctx, struct io_wait_queue *iowq); | 19 | void __io_napi_busy_loop(struct io_ring_ctx *ctx, struct io_wait_queue *iowq); |
| 20 | int io_napi_sqpoll_busy_poll(struct io_ring_ctx *ctx); | ||
| 20 | 21 | ||
| 21 | static inline bool io_napi(struct io_ring_ctx *ctx) | 22 | static inline bool io_napi(struct io_ring_ctx *ctx) |
| 22 | { | 23 | { |
| @@ -83,7 +84,10 @@ static inline void io_napi_busy_loop(struct io_ring_ctx *ctx, · 60 unmodified lines | |||
| 83 | struct io_wait_queue *iowq) | 84 | struct io_wait_queue *iowq) |
| 84 | { | 85 | { |
| 85 | } | 86 | } |
| 86 | 87 | static inline int io_napi_sqpoll_busy_poll(struct io_ring_ctx *ctx) | |
| 88 | { | ||
| 89 | return 0; | ||
| 90 | } | ||
| 87 | #endif /* CONFIG_NET_RX_BUSY_POLL */ | 91 | #endif /* CONFIG_NET_RX_BUSY_POLL */ |
| 88 | 92 | ||
| 89 | #endif | 93 | #endif |
| @@ -15,6 +15,7 @@ · 14 unmodified lines | |||
| 15 | #include <uapi/linux/io_uring.h> | 15 | #include <uapi/linux/io_uring.h> |
| 16 | 16 | ||
| 17 | #include "io_uring.h" | 17 | #include "io_uring.h" |
| 18 | #include "napi.h" | ||
| 18 | #include "sqpoll.h" | 19 | #include "sqpoll.h" |
| 19 | 20 | ||
| 20 | #define IORING_SQPOLL_CAP_ENTRIES_VALUE 8 | 21 | #define IORING_SQPOLL_CAP_ENTRIES_VALUE 8 |
| @@ -194,6 +195,9 @@ static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries) · 173 unmodified lines | |||
| 194 | ret = io_submit_sqes(ctx, to_submit); | 195 | ret = io_submit_sqes(ctx, to_submit); |
| 195 | mutex_unlock(&ctx->uring_lock); | 196 | mutex_unlock(&ctx->uring_lock); |
| 196 | 197 | ||
| 198 | if (io_napi(ctx)) | ||
| 199 | ret += io_napi_sqpoll_busy_poll(ctx); | ||
| 200 | |||
| 197 | if (to_submit && wq_has_sleeper(&ctx->sqo_sq_wait)) | 201 | if (to_submit && wq_has_sleeper(&ctx->sqo_sq_wait)) |
| 198 | wake_up(&ctx->sqo_sq_wait); | 202 | wake_up(&ctx->sqo_sq_wait); |
| 199 | if (creds) | 203 | if (creds) |