INFO: task iou-wrk-6609:6612 blocked for more than 143 seconds. Not tainted 5.15.0-rc5-syzkaller #0 "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. task:iou-wrk-6609 state:D stack:27944 pid: 6612 ppid: 6526 flags:0x00004006 Call Trace: context_switch kernel/sched/core.c:4940 [inline] __schedule+0xb44/0x5960 kernel/sched/core.c:6287 schedule+0xd3/0x270 kernel/sched/core.c:6366 schedule_timeout+0x1db/0x2a0 kernel/time/timer.c:1857 do_wait_for_common kernel/sched/completion.c:85 [inline] __wait_for_common kernel/sched/completion.c:106 [inline] wait_for_common kernel/sched/completion.c:117 [inline] wait_for_completion+0x176/0x280 kernel/sched/completion.c:138 io_worker_exit fs/io-wq.c:183 [inline] io_wqe_worker+0x66d/0xc40 fs/io-wq.c:597 ret_from_fork+0x1f/0x30 arch/x86/entry/entry_64.S:295 io-wq worker may submit a task_work to the master task and upon io_worker_exit() wait for the tw to get executed. The problem appears when the master task is waiting in coredump.c: 468 freezer_do_not_count(); 469 wait_for_completion(&core_state->startup); 470 freezer_count(); Apparently having some dependency on children threads getting everything stuck. Workaround it by cancelling the taks_work callback that causes it before going into io_worker_exit() waiting. p.s. probably a better option is to not submit tw elevating the refcount in the first place, but let's leave this excercise for the future.
Unified · Split
| @@ -140,6 +140,7 @@ static void io_wqe_dec_running(struct io_worker *worker); · 139 unmodified lines | |||
| 140 | static bool io_acct_cancel_pending_work(struct io_wqe *wqe, | 140 | static bool io_acct_cancel_pending_work(struct io_wqe *wqe, |
| 141 | struct io_wqe_acct *acct, | 141 | struct io_wqe_acct *acct, |
| 142 | struct io_cb_cancel_data *match); | 142 | struct io_cb_cancel_data *match); |
| 143 | static void create_worker_cb(struct callback_head *cb); | ||
| 143 | 144 | ||
| 144 | static bool io_worker_get(struct io_worker *worker) | 145 | static bool io_worker_get(struct io_worker *worker) |
| 145 | { | 146 | { |
| @@ -174,9 +175,44 @@ static void io_worker_ref_put(struct io_wq *wq) · 28 unmodified lines | |||
| 174 | complete(&wq->worker_done); | 175 | complete(&wq->worker_done); |
| 175 | } | 176 | } |
| 176 | 177 | ||
| 178 | static void io_worker_cancel_cb(struct io_worker *worker) | ||
| 179 | { | ||
| 180 | struct io_wqe_acct *acct = io_wqe_get_acct(worker); | ||
| 181 | struct io_wqe *wqe = worker->wqe; | ||
| 182 | struct io_wq *wq = wqe->wq; | ||
| 183 | |||
| 184 | atomic_dec(&acct->nr_running); | ||
| 185 | raw_spin_lock(&worker->wqe->lock); | ||
| 186 | acct->nr_workers--; | ||
| 187 | raw_spin_unlock(&worker->wqe->lock); | ||
| 188 | io_worker_ref_put(wq); | ||
| 189 | clear_bit_unlock(0, &worker->create_state); | ||
| 190 | io_worker_release(worker); | ||
| 191 | } | ||
| 192 | |||
| 193 | static bool io_task_worker_match(struct callback_head *cb, void *data) | ||
| 194 | { | ||
| 195 | struct io_worker *worker; | ||
| 196 | |||
| 197 | if (cb->func != create_worker_cb) | ||
| 198 | return false; | ||
| 199 | worker = container_of(cb, struct io_worker, create_work); | ||
| 200 | return worker == data; | ||
| 201 | } | ||
| 202 | |||
| 177 | static void io_worker_exit(struct io_worker *worker) | 203 | static void io_worker_exit(struct io_worker *worker) |
| 178 | { | 204 | { |
| 179 | struct io_wqe *wqe = worker->wqe; | 205 | struct io_wqe *wqe = worker->wqe; |
| 206 | struct io_wq *wq = wqe->wq; | ||
| 207 | |||
| 208 | while (1) { | ||
| 209 | struct callback_head *cb = task_work_cancel_match(wq->task, | ||
| 210 | io_task_worker_match, worker); | ||
| 211 | |||
| 212 | if (!cb) | ||
| 213 | break; | ||
| 214 | io_worker_cancel_cb(worker); | ||
| 215 | } | ||
| 180 | 216 | ||
| 181 | io_worker_release(worker); | 217 | io_worker_release(worker); |
| 182 | wait_for_completion(&worker->ref_done); | 218 | wait_for_completion(&worker->ref_done); |
| @@ -1148,17 +1184,9 @@ static void io_wq_exit_workers(struct io_wq *wq) · 965 unmodified lines | |||
| 1148 | 1184 | ||
| 1149 | while ((cb = task_work_cancel_match(wq->task, io_task_work_match, wq)) != NULL) { | 1185 | while ((cb = task_work_cancel_match(wq->task, io_task_work_match, wq)) != NULL) { |
| 1150 | struct io_worker *worker; | 1186 | struct io_worker *worker; |
| 1151 | struct io_wqe_acct *acct; | ||
| 1152 | 1187 | ||
| 1153 | worker = container_of(cb, struct io_worker, create_work); | 1188 | worker = container_of(cb, struct io_worker, create_work); |
| 1154 | acct = io_wqe_get_acct(worker); | 1189 | io_worker_cancel_cb(worker); |
| 1155 | atomic_dec(&acct->nr_running); | ||
| 1156 | raw_spin_lock(&worker->wqe->lock); | ||
| 1157 | acct->nr_workers--; | ||
| 1158 | raw_spin_unlock(&worker->wqe->lock); | ||
| 1159 | io_worker_ref_put(wq); | ||
| 1160 | clear_bit_unlock(0, &worker->create_state); | ||
| 1161 | io_worker_release(worker); | ||
| 1162 | } | 1190 | } |
| 1163 | 1191 | ||
| 1164 | rcu_read_lock(); | 1192 | rcu_read_lock(); |