History

workqueue: dump the last woken worker for stalled pools

To identify the task most likely responsible for a stall, add last_woken_worker (L: pool->lock) to worker_pool and record it in kick_pool() just before wake_up_process(). This captures the idle worker that was kicked to take over when the last running worker went to sleep; if the pool is now stuck with no running worker, that task is the prime suspect and its backtrace is dumped by show_pool_no_running_worker(). Using struct worker * rather than struct task_struct * avoids any lifetime concern: workers are only destroyed via set_worker_dying() which requires pool->lock, and set_worker_dying() clears last_woken_worker when the dying worker matches. show_cpu_pool_busy_workers() holds pool->lock while calling sched_show_task(), so last_woken_worker is either NULL or points to a live worker with a valid task. More precisely, set_worker_dying() clears last_woken_worker before setting WORKER_DIE, so a non-NULL last_woken_worker means the kthread has not yet exited and worker->task is still alive.

Breno Leitao committed 4 months ago

Commit e73c2901 parent f7dc9331 file · +27 −2Browse files

Unified · Split

kernel/workqueue.c+27−2
@@ -226,6 +226,8 @@ struct worker_pool { · 225 unmodified lines
226226 /* L: hash of busy workers */
227227
228228 struct worker *manager; /* L: purely informational */
229+ /* L: last worker woken by kick_pool() */
230+ struct worker *last_woken_worker;
229231 struct list_head workers; /* A: attached workers */
230232
231233 struct ida worker_ida; /* worker IDs for task name */
@@ -1318,6 +1320,9 @@ static bool kick_pool_pick(struct worker_pool *pool, struct task_struct **wakep) · 1086 unmodified lines
13181320 }
13191321 }
13201322 #endif
1323+ /* Track the last idle worker woken, used for stall diagnostics. */
1324+ pool->last_woken_worker = worker;
1325+
13211326 *wakep = p;
13221327 return true;
13231328 }
@@ -2976,6 +2981,13 @@ static void set_worker_dying(struct worker *worker, struct list_head *list) · 1652 unmodified lines
29762981 pool->nr_workers--;
29772982 pool->nr_idle--;
29782983
2984+ /*
2985+ * Clear last_woken_worker if it points to this worker, so that
2986+ * show_cpu_pool_busy_workers() cannot dereference a freed worker.
2987+ */
2988+ if (pool->last_woken_worker == worker)
2989+ pool->last_woken_worker = NULL;
2990+
29792991 worker->flags |= WORKER_DIE;
29802992
29812993 list_move(&worker->entry, list);
@@ -7740,13 +7752,25 @@ static void show_pool_no_running_worker(struct worker_pool *pool) · 4758 unmodified lines
77407752 idle_cpu(pool->cpu) ? "idle" : "busy",
77417753 pool->nr_workers, pool->nr_idle);
77427754 pr_info("The pool might have trouble waking an idle worker.\n");
7755+ /*
7756+ * last_woken_worker and its task are valid here: set_worker_dying()
7757+ * clears it under pool->lock before setting WORKER_DIE, so if
7758+ * last_woken_worker is non-NULL the kthread has not yet exited and
7759+ * worker->task is still alive.
7760+ */
7761+ if (pool->last_woken_worker) {
7762+ pr_info("Backtrace of last woken worker:\n");
7763+ sched_show_task(pool->last_woken_worker->task);
7764+ } else {
7765+ pr_info("Last woken worker empty\n");
7766+ }
77437767 printk_deferred_exit();
77447768 }
77457769
77467770 /*
77477771 * Show running workers that might prevent the processing of pending work items.
77487772 * If no running worker is found, the pool may be stuck waiting for an idle
7749− * worker to be woken, so report the pool state.
7773+ * worker to be woken, so report the pool state and the last woken worker.
77507774 */
77517775 static void show_cpu_pool_busy_workers(struct worker_pool *pool)
77527776 {
@@ -7781,7 +7805,8 @@ static void show_cpu_pool_busy_workers(struct worker_pool *pool) · 28 unmodified lines
77817805
77827806 /*
77837807 * If no running worker was found, the pool is likely stuck. Print pool
7784− * state.
7808+ * state and the backtrace of the last woken worker, which is the prime
7809+ * suspect for the stall.
77857810 */
77867811 if (!found_running)
77877812 show_pool_no_running_worker(pool);