We can't assume that the whole batch has fixed files in it. If it's a mix, or none at all, then we can end up doing a ref put that either messes up accounting, or causes an oops if we have no fixed files at all. Also ensure we free requests properly between inflight accounted and normal requests.
Unified · Split
| @@ -1207,21 +1207,24 @@ struct req_batch { · 1206 unmodified lines | |||
| 1207 | 1207 | ||
| 1208 | static void io_free_req_many(struct io_ring_ctx *ctx, struct req_batch *rb) | 1208 | static void io_free_req_many(struct io_ring_ctx *ctx, struct req_batch *rb) |
| 1209 | { | 1209 | { |
| 1210 | int fixed_refs = rb->to_free; | ||
| 1211 | |||
| 1210 | if (!rb->to_free) | 1212 | if (!rb->to_free) |
| 1211 | return; | 1213 | return; |
| 1212 | if (rb->need_iter) { | 1214 | if (rb->need_iter) { |
| 1213 | int i, inflight = 0; | 1215 | int i, inflight = 0; |
| 1214 | unsigned long flags; | 1216 | unsigned long flags; |
| 1215 | 1217 | ||
| 1218 | fixed_refs = 0; | ||
| 1216 | for (i = 0; i < rb->to_free; i++) { | 1219 | for (i = 0; i < rb->to_free; i++) { |
| 1217 | struct io_kiocb *req = rb->reqs[i]; | 1220 | struct io_kiocb *req = rb->reqs[i]; |
| 1218 | 1221 | ||
| 1219 | if (req->flags & REQ_F_FIXED_FILE) | 1222 | if (req->flags & REQ_F_FIXED_FILE) { |
| 1220 | req->file = NULL; | 1223 | req->file = NULL; |
| 1224 | fixed_refs++; | ||
| 1225 | } | ||
| 1221 | if (req->flags & REQ_F_INFLIGHT) | 1226 | if (req->flags & REQ_F_INFLIGHT) |
| 1222 | inflight++; | 1227 | inflight++; |
| 1223 | else | ||
| 1224 | rb->reqs[i] = NULL; | ||
| 1225 | __io_req_aux_free(req); | 1228 | __io_req_aux_free(req); |
| 1226 | } | 1229 | } |
| 1227 | if (!inflight) | 1230 | if (!inflight) |
| @@ -1231,7 +1234,7 @@ static void io_free_req_many(struct io_ring_ctx *ctx, struct req_batch *rb) · 3 unmodified lines | |||
| 1231 | for (i = 0; i < rb->to_free; i++) { | 1234 | for (i = 0; i < rb->to_free; i++) { |
| 1232 | struct io_kiocb *req = rb->reqs[i]; | 1235 | struct io_kiocb *req = rb->reqs[i]; |
| 1233 | 1236 | ||
| 1234 | if (req) { | 1237 | if (req->flags & REQ_F_INFLIGHT) { |
| 1235 | list_del(&req->inflight_entry); | 1238 | list_del(&req->inflight_entry); |
| 1236 | if (!--inflight) | 1239 | if (!--inflight) |
| 1237 | break; | 1240 | break; |
| @@ -1244,8 +1247,9 @@ static void io_free_req_many(struct io_ring_ctx *ctx, struct req_batch *rb) · 6 unmodified lines | |||
| 1244 | } | 1247 | } |
| 1245 | do_free: | 1248 | do_free: |
| 1246 | kmem_cache_free_bulk(req_cachep, rb->to_free, rb->reqs); | 1249 | kmem_cache_free_bulk(req_cachep, rb->to_free, rb->reqs); |
| 1250 | if (fixed_refs) | ||
| 1251 | percpu_ref_put_many(&ctx->file_data->refs, fixed_refs); | ||
| 1247 | percpu_ref_put_many(&ctx->refs, rb->to_free); | 1252 | percpu_ref_put_many(&ctx->refs, rb->to_free); |
| 1248 | percpu_ref_put_many(&ctx->file_data->refs, rb->to_free); | ||
| 1249 | rb->to_free = rb->need_iter = 0; | 1253 | rb->to_free = rb->need_iter = 0; |
| 1250 | } | 1254 | } |
| 1251 | 1255 | ||