This adds the sqpoll support to the io-uring napi.
Unified · Split
| @@ -252,4 +252,28 @@ void __io_napi_busy_loop(struct io_ring_ctx *ctx, struct io_wait_queue *iowq) · 251 unmodified lines | ||
| 252 | 252 | io_napi_blocking_busy_loop(ctx, iowq); |
| 253 | 253 | } |
| 254 | 254 | |
| 255 | +/* | |
| 256 | + * io_napi_sqpoll_busy_poll() - busy poll loop for sqpoll | |
| 257 | + * @ctx: pointer to io-uring context structure | |
| 258 | + * | |
| 259 | + * Splice of the napi list and execute the napi busy poll loop. | |
| 260 | + */ | |
| 261 | +int io_napi_sqpoll_busy_poll(struct io_ring_ctx *ctx) | |
| 262 | +{ | |
| 263 | + LIST_HEAD(napi_list); | |
| 264 | + bool is_stale = false; | |
| 265 | + | |
| 266 | + if (!READ_ONCE(ctx->napi_busy_poll_to)) | |
| 267 | + return 0; | |
| 268 | + if (list_empty_careful(&ctx->napi_list)) | |
| 269 | + return 0; | |
| 270 | + | |
| 271 | + rcu_read_lock(); | |
| 272 | + is_stale = __io_napi_do_busy_loop(ctx, NULL); | |
| 273 | + rcu_read_unlock(); | |
| 274 | + | |
| 275 | + io_napi_remove_stale(ctx, is_stale); | |
| 276 | + return 1; | |
| 277 | +} | |
| 278 | + | |
| 255 | 279 | #endif |
| @@ -17,6 +17,7 @@ void __io_napi_add(struct io_ring_ctx *ctx, struct socket *sock); · 16 unmodified lines | ||
| 17 | 17 | void __io_napi_adjust_timeout(struct io_ring_ctx *ctx, |
| 18 | 18 | struct io_wait_queue *iowq, struct timespec64 *ts); |
| 19 | 19 | void __io_napi_busy_loop(struct io_ring_ctx *ctx, struct io_wait_queue *iowq); |
| 20 | +int io_napi_sqpoll_busy_poll(struct io_ring_ctx *ctx); | |
| 20 | 21 | |
| 21 | 22 | static inline bool io_napi(struct io_ring_ctx *ctx) |
| 22 | 23 | { |
| @@ -83,7 +84,10 @@ static inline void io_napi_busy_loop(struct io_ring_ctx *ctx, · 60 unmodified lines | ||
| 83 | 84 | struct io_wait_queue *iowq) |
| 84 | 85 | { |
| 85 | 86 | } |
| 86 | − | |
| 87 | +static inline int io_napi_sqpoll_busy_poll(struct io_ring_ctx *ctx) | |
| 88 | +{ | |
| 89 | + return 0; | |
| 90 | +} | |
| 87 | 91 | #endif /* CONFIG_NET_RX_BUSY_POLL */ |
| 88 | 92 | |
| 89 | 93 | #endif |
| @@ -15,6 +15,7 @@ · 14 unmodified lines | ||
| 15 | 15 | #include <uapi/linux/io_uring.h> |
| 16 | 16 | |
| 17 | 17 | #include "io_uring.h" |
| 18 | +#include "napi.h" | |
| 18 | 19 | #include "sqpoll.h" |
| 19 | 20 | |
| 20 | 21 | #define IORING_SQPOLL_CAP_ENTRIES_VALUE 8 |
| @@ -194,6 +195,9 @@ static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries) · 173 unmodified lines | ||
| 194 | 195 | ret = io_submit_sqes(ctx, to_submit); |
| 195 | 196 | mutex_unlock(&ctx->uring_lock); |
| 196 | 197 | |
| 198 | + if (io_napi(ctx)) | |
| 199 | + ret += io_napi_sqpoll_busy_poll(ctx); | |
| 200 | + | |
| 197 | 201 | if (to_submit && wq_has_sleeper(&ctx->sqo_sq_wait)) |
| 198 | 202 | wake_up(&ctx->sqo_sq_wait); |
| 199 | 203 | if (creds) |