workqueue: dump the last woken worker for stalled pools

To identify the task most likely responsible for a stall, add last_woken_worker (L: pool->lock) to worker_pool and record it in kick_pool() just before wake_up_process(). This captures the idle worker that was kicked to take over when the last running worker went to sleep; if the pool is now stuck with no running worker, that task is the prime suspect and its backtrace is dumped by show_pool_no_running_worker(). Using struct worker * rather than struct task_struct * avoids any lifetime concern: workers are only destroyed via set_worker_dying() which requires pool->lock, and set_worker_dying() clears last_woken_worker when the dying worker matches. show_cpu_pool_busy_workers() holds pool->lock while calling sched_show_task(), so last_woken_worker is either NULL or points to a live worker with a valid task. More precisely, set_worker_dying() clears last_woken_worker before setting WORKER_DIE, so a non-NULL last_woken_worker means the kthread has not yet exited and worker->task is still alive.

Breno Leitao committed 4 months ago

Commit e73c2901 parent f7dc9331 file · +27 −2Browse files

Showing the changes to kernel/workqueue.c (1 of 1 file). Show all

Unified · Split

kernel/workqueue.c+27−2

 						/* L: hash of busy workers */
 	struct worker		*manager;	/* L: purely informational */
+	/* L: last worker woken by kick_pool() */
+	struct worker		*last_woken_worker;
 	struct list_head	workers;	/* A: attached workers */
 	struct ida		worker_ida;	/* worker IDs for task name */
 		}
 	}
 #endif
+	/* Track the last idle worker woken, used for stall diagnostics. */
+	pool->last_woken_worker = worker;
 	*wakep = p;
 	return true;
 }
 	pool->nr_workers--;
 	pool->nr_idle--;
+	/*
+	 * Clear last_woken_worker if it points to this worker, so that
+	 * show_cpu_pool_busy_workers() cannot dereference a freed worker.
+	 */
+	if (pool->last_woken_worker == worker)
+		pool->last_woken_worker = NULL;
 	worker->flags |= WORKER_DIE;
 	list_move(&worker->entry, list);
 		idle_cpu(pool->cpu) ? "idle" : "busy",
 		pool->nr_workers, pool->nr_idle);
 	pr_info("The pool might have trouble waking an idle worker.\n");
+	/*
+	 * last_woken_worker and its task are valid here: set_worker_dying()
+	 * clears it under pool->lock before setting WORKER_DIE, so if
+	 * last_woken_worker is non-NULL the kthread has not yet exited and
+	 * worker->task is still alive.
+	 */
+	if (pool->last_woken_worker) {
+		pr_info("Backtrace of last woken worker:\n");
+		sched_show_task(pool->last_woken_worker->task);
+	} else {
+		pr_info("Last woken worker empty\n");
+	}
 	printk_deferred_exit();
 }
 /*
  * Show running workers that might prevent the processing of pending work items.
  * If no running worker is found, the pool may be stuck waiting for an idle
- * worker to be woken, so report the pool state.
+ * worker to be woken, so report the pool state and the last woken worker.
  */
 static void show_cpu_pool_busy_workers(struct worker_pool *pool)
 {
 	/*
 	 * If no running worker was found, the pool is likely stuck. Print pool
-	 * state.
+	 * state and the backtrace of the last woken worker, which is the prime
+	 * suspect for the stall.
 	 */
 	if (!found_running)
 		show_pool_no_running_worker(pool);