This adds the overflow processing for large CQE's. This adds two parameters to the io_cqring_event_overflow function and uses these fields to initialize the large CQE fields. Allocate enough space for large CQE's in the overflow structue. If no large CQE's are used, the size of the allocation is unchanged. The cqe field can have a different size depending if its a large CQE or not. To be able to allocate different sizes, the two fields in the structure are re-ordered.
Unified · Split
| @@ -220,8 +220,8 @@ struct io_mapped_ubuf { · 219 unmodified lines | ||
| 220 | 220 | struct io_ring_ctx; |
| 221 | 221 | |
| 222 | 222 | struct io_overflow_cqe { |
| 223 | − struct io_uring_cqe cqe; | |
| 224 | 223 | struct list_head list; |
| 224 | + struct io_uring_cqe cqe; | |
| 225 | 225 | }; |
| 226 | 226 | |
| 227 | 227 | /* |
| @@ -2177,10 +2177,14 @@ static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx) · 1949 unmodified lines | ||
| 2177 | 2177 | static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force) |
| 2178 | 2178 | { |
| 2179 | 2179 | bool all_flushed, posted; |
| 2180 | + size_t cqe_size = sizeof(struct io_uring_cqe); | |
| 2180 | 2181 | |
| 2181 | 2182 | if (!force && __io_cqring_events(ctx) == ctx->cq_entries) |
| 2182 | 2183 | return false; |
| 2183 | 2184 | |
| 2185 | + if (ctx->flags & IORING_SETUP_CQE32) | |
| 2186 | + cqe_size <<= 1; | |
| 2187 | + | |
| 2184 | 2188 | posted = false; |
| 2185 | 2189 | spin_lock(&ctx->completion_lock); |
| 2186 | 2190 | while (!list_empty(&ctx->cq_overflow_list)) { |
| @@ -2192,7 +2196,7 @@ static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force) · 5 unmodified lines | ||
| 2192 | 2196 | ocqe = list_first_entry(&ctx->cq_overflow_list, |
| 2193 | 2197 | struct io_overflow_cqe, list); |
| 2194 | 2198 | if (cqe) |
| 2195 | − memcpy(cqe, &ocqe->cqe, sizeof(*cqe)); | |
| 2199 | + memcpy(cqe, &ocqe->cqe, cqe_size); | |
| 2196 | 2200 | else |
| 2197 | 2201 | io_account_cq_overflow(ctx); |
| 2198 | 2202 | |
| @@ -2280,11 +2284,17 @@ static __cold void io_uring_drop_tctx_refs(struct task_struct *task) · 81 unmodified lines | ||
| 2280 | 2284 | } |
| 2281 | 2285 | |
| 2282 | 2286 | static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data, |
| 2283 | − s32 res, u32 cflags) | |
| 2287 | + s32 res, u32 cflags, u64 extra1, | |
| 2288 | + u64 extra2) | |
| 2284 | 2289 | { |
| 2285 | 2290 | struct io_overflow_cqe *ocqe; |
| 2291 | + size_t ocq_size = sizeof(struct io_overflow_cqe); | |
| 2292 | + bool is_cqe32 = (ctx->flags & IORING_SETUP_CQE32); | |
| 2293 | + | |
| 2294 | + if (is_cqe32) | |
| 2295 | + ocq_size += sizeof(struct io_uring_cqe); | |
| 2286 | 2296 | |
| 2287 | − ocqe = kmalloc(sizeof(*ocqe), GFP_ATOMIC | __GFP_ACCOUNT); | |
| 2297 | + ocqe = kmalloc(ocq_size, GFP_ATOMIC | __GFP_ACCOUNT); | |
| 2288 | 2298 | trace_io_uring_cqe_overflow(ctx, user_data, res, cflags, ocqe); |
| 2289 | 2299 | if (!ocqe) { |
| 2290 | 2300 | /* |
| @@ -2304,6 +2314,10 @@ static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data, · 13 unmodified lines | ||
| 2304 | 2314 | ocqe->cqe.user_data = user_data; |
| 2305 | 2315 | ocqe->cqe.res = res; |
| 2306 | 2316 | ocqe->cqe.flags = cflags; |
| 2317 | + if (is_cqe32) { | |
| 2318 | + ocqe->cqe.big_cqe[0] = extra1; | |
| 2319 | + ocqe->cqe.big_cqe[1] = extra2; | |
| 2320 | + } | |
| 2307 | 2321 | list_add_tail(&ocqe->list, &ctx->cq_overflow_list); |
| 2308 | 2322 | return true; |
| 2309 | 2323 | } |
| @@ -2325,7 +2339,7 @@ static inline bool __io_fill_cqe(struct io_ring_ctx *ctx, u64 user_data, · 15 unmodified lines | ||
| 2325 | 2339 | WRITE_ONCE(cqe->flags, cflags); |
| 2326 | 2340 | return true; |
| 2327 | 2341 | } |
| 2328 | − return io_cqring_event_overflow(ctx, user_data, res, cflags); | |
| 2342 | + return io_cqring_event_overflow(ctx, user_data, res, cflags, 0, 0); | |
| 2329 | 2343 | } |
| 2330 | 2344 | |
| 2331 | 2345 | static inline bool __io_fill_cqe_req_filled(struct io_ring_ctx *ctx, |
| @@ -2347,7 +2361,7 @@ static inline bool __io_fill_cqe_req_filled(struct io_ring_ctx *ctx, · 15 unmodified lines | ||
| 2347 | 2361 | return true; |
| 2348 | 2362 | } |
| 2349 | 2363 | return io_cqring_event_overflow(ctx, req->cqe.user_data, |
| 2350 | − req->cqe.res, req->cqe.flags); | |
| 2364 | + req->cqe.res, req->cqe.flags, 0, 0); | |
| 2351 | 2365 | } |
| 2352 | 2366 | |
| 2353 | 2367 | static inline bool __io_fill_cqe32_req_filled(struct io_ring_ctx *ctx, |
| @@ -2373,8 +2387,8 @@ static inline bool __io_fill_cqe32_req_filled(struct io_ring_ctx *ctx, · 19 unmodified lines | ||
| 2373 | 2387 | return true; |
| 2374 | 2388 | } |
| 2375 | 2389 | |
| 2376 | − return io_cqring_event_overflow(ctx, req->cqe.user_data, | |
| 2377 | − req->cqe.res, req->cqe.flags); | |
| 2390 | + return io_cqring_event_overflow(ctx, req->cqe.user_data, req->cqe.res, | |
| 2391 | + req->cqe.flags, extra1, extra2); | |
| 2378 | 2392 | } |
| 2379 | 2393 | |
| 2380 | 2394 | static inline bool __io_fill_cqe_req(struct io_kiocb *req, s32 res, u32 cflags) |
| @@ -2411,7 +2425,7 @@ static inline void __io_fill_cqe32_req(struct io_kiocb *req, s32 res, u32 cflags · 30 unmodified lines | ||
| 2411 | 2425 | return; |
| 2412 | 2426 | } |
| 2413 | 2427 | |
| 2414 | − io_cqring_event_overflow(ctx, req->cqe.user_data, res, cflags); | |
| 2428 | + io_cqring_event_overflow(ctx, req->cqe.user_data, res, cflags, extra1, extra2); | |
| 2415 | 2429 | } |
| 2416 | 2430 | |
| 2417 | 2431 | static noinline bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, |