We could queue a work for each req in defer and link list without increasing async_list->cnt, so we shouldn't decrease it while exiting from workqueue as well if we didn't process the req in async list. Thanks to Jens Axboe <axboe@kernel.dk> for his guidance.
Unified · Split
| @@ -333,7 +333,8 @@ struct io_kiocb { · 332 unmodified lines | ||
| 333 | 333 | #define REQ_F_IO_DRAIN 16 /* drain existing IO first */ |
| 334 | 334 | #define REQ_F_IO_DRAINED 32 /* drain done */ |
| 335 | 335 | #define REQ_F_LINK 64 /* linked sqes */ |
| 336 | −#define REQ_F_FAIL_LINK 128 /* fail rest of links */ | |
| 336 | +#define REQ_F_LINK_DONE 128 /* linked sqes done */ | |
| 337 | +#define REQ_F_FAIL_LINK 256 /* fail rest of links */ | |
| 337 | 338 | u64 user_data; |
| 338 | 339 | u32 result; |
| 339 | 340 | u32 sequence; |
| @@ -632,6 +633,7 @@ static void io_req_link_next(struct io_kiocb *req) · 292 unmodified lines | ||
| 632 | 633 | nxt->flags |= REQ_F_LINK; |
| 633 | 634 | } |
| 634 | 635 | |
| 636 | + nxt->flags |= REQ_F_LINK_DONE; | |
| 635 | 637 | INIT_WORK(&nxt->work, io_sq_wq_submit_work); |
| 636 | 638 | queue_work(req->ctx->sqo_wq, &nxt->work); |
| 637 | 639 | } |
| @@ -1844,6 +1846,10 @@ restart: · 1206 unmodified lines | ||
| 1844 | 1846 | /* async context always use a copy of the sqe */ |
| 1845 | 1847 | kfree(sqe); |
| 1846 | 1848 | |
| 1849 | + /* req from defer and link list needn't decrease async cnt */ | |
| 1850 | + if (req->flags & (REQ_F_IO_DRAINED | REQ_F_LINK_DONE)) | |
| 1851 | + goto out; | |
| 1852 | + | |
| 1847 | 1853 | if (!async_list) |
| 1848 | 1854 | break; |
| 1849 | 1855 | if (!list_empty(&req_list)) { |
| @@ -1891,6 +1897,7 @@ restart: · 41 unmodified lines | ||
| 1891 | 1897 | } |
| 1892 | 1898 | } |
| 1893 | 1899 | |
| 1900 | +out: | |
| 1894 | 1901 | if (cur_mm) { |
| 1895 | 1902 | set_fs(old_fs); |
| 1896 | 1903 | unuse_mm(cur_mm); |