History

io_uring: overflow processing for CQE32

This adds the overflow processing for large CQE's. This adds two parameters to the io_cqring_event_overflow function and uses these fields to initialize the large CQE fields. Allocate enough space for large CQE's in the overflow structue. If no large CQE's are used, the size of the allocation is unchanged. The cqe field can have a different size depending if its a large CQE or not. To be able to allocate different sizes, the two fields in the structure are re-ordered.

Stefan Roesch committed 4 years ago

Commit e45a3e01 parent 0e2e5c41 file · +23 −9Browse files

Unified · Split

fs/io_uring.c+23−9
@@ -220,8 +220,8 @@ struct io_mapped_ubuf { · 219 unmodified lines
220struct io_ring_ctx;220struct io_ring_ctx;
221 221
222struct io_overflow_cqe {222struct io_overflow_cqe {
223 struct io_uring_cqe cqe;
224 struct list_head list;223 struct list_head list;
224 struct io_uring_cqe cqe;
225};225};
226 226
227/*227/*
@@ -2177,10 +2177,14 @@ static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx) · 1949 unmodified lines
2177static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)2177static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
2178{2178{
2179 bool all_flushed, posted;2179 bool all_flushed, posted;
2180 size_t cqe_size = sizeof(struct io_uring_cqe);
2180 2181
2181 if (!force && __io_cqring_events(ctx) == ctx->cq_entries)2182 if (!force && __io_cqring_events(ctx) == ctx->cq_entries)
2182 return false;2183 return false;
2183 2184
2185 if (ctx->flags & IORING_SETUP_CQE32)
2186 cqe_size <<= 1;
2187
2184 posted = false;2188 posted = false;
2185 spin_lock(&ctx->completion_lock);2189 spin_lock(&ctx->completion_lock);
2186 while (!list_empty(&ctx->cq_overflow_list)) {2190 while (!list_empty(&ctx->cq_overflow_list)) {
@@ -2192,7 +2196,7 @@ static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force) · 5 unmodified lines
2192 ocqe = list_first_entry(&ctx->cq_overflow_list,2196 ocqe = list_first_entry(&ctx->cq_overflow_list,
2193 struct io_overflow_cqe, list);2197 struct io_overflow_cqe, list);
2194 if (cqe)2198 if (cqe)
2195 memcpy(cqe, &ocqe->cqe, sizeof(*cqe));2199 memcpy(cqe, &ocqe->cqe, cqe_size);
2196 else2200 else
2197 io_account_cq_overflow(ctx);2201 io_account_cq_overflow(ctx);
2198 2202
@@ -2280,11 +2284,17 @@ static __cold void io_uring_drop_tctx_refs(struct task_struct *task) · 81 unmodified lines
2280}2284}
2281 2285
2282static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data,2286static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data,
2283 s32 res, u32 cflags)2287 s32 res, u32 cflags, u64 extra1,
2288 u64 extra2)
2284{2289{
2285 struct io_overflow_cqe *ocqe;2290 struct io_overflow_cqe *ocqe;
2291 size_t ocq_size = sizeof(struct io_overflow_cqe);
2292 bool is_cqe32 = (ctx->flags & IORING_SETUP_CQE32);
2293
2294 if (is_cqe32)
2295 ocq_size += sizeof(struct io_uring_cqe);
2286 2296
2287 ocqe = kmalloc(sizeof(*ocqe), GFP_ATOMIC | __GFP_ACCOUNT);2297 ocqe = kmalloc(ocq_size, GFP_ATOMIC | __GFP_ACCOUNT);
2288 trace_io_uring_cqe_overflow(ctx, user_data, res, cflags, ocqe);2298 trace_io_uring_cqe_overflow(ctx, user_data, res, cflags, ocqe);
2289 if (!ocqe) {2299 if (!ocqe) {
2290 /*2300 /*
@@ -2304,6 +2314,10 @@ static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data, · 13 unmodified lines
2304 ocqe->cqe.user_data = user_data;2314 ocqe->cqe.user_data = user_data;
2305 ocqe->cqe.res = res;2315 ocqe->cqe.res = res;
2306 ocqe->cqe.flags = cflags;2316 ocqe->cqe.flags = cflags;
2317 if (is_cqe32) {
2318 ocqe->cqe.big_cqe[0] = extra1;
2319 ocqe->cqe.big_cqe[1] = extra2;
2320 }
2307 list_add_tail(&ocqe->list, &ctx->cq_overflow_list);2321 list_add_tail(&ocqe->list, &ctx->cq_overflow_list);
2308 return true;2322 return true;
2309}2323}
@@ -2325,7 +2339,7 @@ static inline bool __io_fill_cqe(struct io_ring_ctx *ctx, u64 user_data, · 15 unmodified lines
2325 WRITE_ONCE(cqe->flags, cflags);2339 WRITE_ONCE(cqe->flags, cflags);
2326 return true;2340 return true;
2327 }2341 }
2328 return io_cqring_event_overflow(ctx, user_data, res, cflags);2342 return io_cqring_event_overflow(ctx, user_data, res, cflags, 0, 0);
2329}2343}
2330 2344
2331static inline bool __io_fill_cqe_req_filled(struct io_ring_ctx *ctx,2345static inline bool __io_fill_cqe_req_filled(struct io_ring_ctx *ctx,
@@ -2347,7 +2361,7 @@ static inline bool __io_fill_cqe_req_filled(struct io_ring_ctx *ctx, · 15 unmodified lines
2347 return true;2361 return true;
2348 }2362 }
2349 return io_cqring_event_overflow(ctx, req->cqe.user_data,2363 return io_cqring_event_overflow(ctx, req->cqe.user_data,
2350 req->cqe.res, req->cqe.flags);2364 req->cqe.res, req->cqe.flags, 0, 0);
2351}2365}
2352 2366
2353static inline bool __io_fill_cqe32_req_filled(struct io_ring_ctx *ctx,2367static inline bool __io_fill_cqe32_req_filled(struct io_ring_ctx *ctx,
@@ -2373,8 +2387,8 @@ static inline bool __io_fill_cqe32_req_filled(struct io_ring_ctx *ctx, · 19 unmodified lines
2373 return true;2387 return true;
2374 }2388 }
2375 2389
2376 return io_cqring_event_overflow(ctx, req->cqe.user_data,2390 return io_cqring_event_overflow(ctx, req->cqe.user_data, req->cqe.res,
2377 req->cqe.res, req->cqe.flags);2391 req->cqe.flags, extra1, extra2);
2378}2392}
2379 2393
2380static inline bool __io_fill_cqe_req(struct io_kiocb *req, s32 res, u32 cflags)2394static inline bool __io_fill_cqe_req(struct io_kiocb *req, s32 res, u32 cflags)
@@ -2411,7 +2425,7 @@ static inline void __io_fill_cqe32_req(struct io_kiocb *req, s32 res, u32 cflags · 30 unmodified lines
2411 return;2425 return;
2412 }2426 }
2413 2427
2414 io_cqring_event_overflow(ctx, req->cqe.user_data, res, cflags);2428 io_cqring_event_overflow(ctx, req->cqe.user_data, res, cflags, extra1, extra2);
2415}2429}
2416 2430
2417static noinline bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data,2431static noinline bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data,