We could queue a work for each req in defer and link list without increasing async_list->cnt, so we shouldn't decrease it while exiting from workqueue as well if we didn't process the req in async list. Thanks to Jens Axboe <axboe@kernel.dk> for his guidance.
Unified · Split
| @@ -333,7 +333,8 @@ struct io_kiocb { · 332 unmodified lines | |||
| 333 | #define REQ_F_IO_DRAIN 16 /* drain existing IO first */ | 333 | #define REQ_F_IO_DRAIN 16 /* drain existing IO first */ |
| 334 | #define REQ_F_IO_DRAINED 32 /* drain done */ | 334 | #define REQ_F_IO_DRAINED 32 /* drain done */ |
| 335 | #define REQ_F_LINK 64 /* linked sqes */ | 335 | #define REQ_F_LINK 64 /* linked sqes */ |
| 336 | #define REQ_F_FAIL_LINK 128 /* fail rest of links */ | 336 | #define REQ_F_LINK_DONE 128 /* linked sqes done */ |
| 337 | #define REQ_F_FAIL_LINK 256 /* fail rest of links */ | ||
| 337 | u64 user_data; | 338 | u64 user_data; |
| 338 | u32 result; | 339 | u32 result; |
| 339 | u32 sequence; | 340 | u32 sequence; |
| @@ -632,6 +633,7 @@ static void io_req_link_next(struct io_kiocb *req) · 292 unmodified lines | |||
| 632 | nxt->flags |= REQ_F_LINK; | 633 | nxt->flags |= REQ_F_LINK; |
| 633 | } | 634 | } |
| 634 | 635 | ||
| 636 | nxt->flags |= REQ_F_LINK_DONE; | ||
| 635 | INIT_WORK(&nxt->work, io_sq_wq_submit_work); | 637 | INIT_WORK(&nxt->work, io_sq_wq_submit_work); |
| 636 | queue_work(req->ctx->sqo_wq, &nxt->work); | 638 | queue_work(req->ctx->sqo_wq, &nxt->work); |
| 637 | } | 639 | } |
| @@ -1844,6 +1846,10 @@ restart: · 1206 unmodified lines | |||
| 1844 | /* async context always use a copy of the sqe */ | 1846 | /* async context always use a copy of the sqe */ |
| 1845 | kfree(sqe); | 1847 | kfree(sqe); |
| 1846 | 1848 | ||
| 1849 | /* req from defer and link list needn't decrease async cnt */ | ||
| 1850 | if (req->flags & (REQ_F_IO_DRAINED | REQ_F_LINK_DONE)) | ||
| 1851 | goto out; | ||
| 1852 | |||
| 1847 | if (!async_list) | 1853 | if (!async_list) |
| 1848 | break; | 1854 | break; |
| 1849 | if (!list_empty(&req_list)) { | 1855 | if (!list_empty(&req_list)) { |
| @@ -1891,6 +1897,7 @@ restart: · 41 unmodified lines | |||
| 1891 | } | 1897 | } |
| 1892 | } | 1898 | } |
| 1893 | 1899 | ||
| 1900 | out: | ||
| 1894 | if (cur_mm) { | 1901 | if (cur_mm) { |
| 1895 | set_fs(old_fs); | 1902 | set_fs(old_fs); |
| 1896 | unuse_mm(cur_mm); | 1903 | unuse_mm(cur_mm); |