show_cpu_pool_busy_workers() dumps every in-flight worker in the pool's busy_hash, including workers that are not currently running on the CPU. Restore the task_is_running() filter so only running workers are dumped. When no running worker is found the pool may be stuck, unable to wake an idle worker to process pending work, and the watchdog would otherwise give no feedback. Add show_pool_no_running_worker() to report the pool id, CPU, idle state, and worker counts in that case. The pool info message is printed inside pool->lock using printk_deferred_enter/exit, the same pattern used by the existing busy-worker loop, to avoid deadlocks with console drivers that queue work while holding locks also taken in their write paths. This has been running on the Meta fleet for a while and caught some real issues, for instance EFI stalls stalling the workqueue [1].
Showing the changes to kernel/workqueue.c (1 of 1 file). Show all
Unified · Split
| @@ -7726,13 +7726,31 @@ module_param_named(panic_on_stall_time, wq_panic_on_stall_time, uint, 0644); · 7725 unmodified lines | |||
| 7726 | MODULE_PARM_DESC(panic_on_stall_time, "Panic if stall exceeds this many seconds (0=disabled)"); | 7726 | MODULE_PARM_DESC(panic_on_stall_time, "Panic if stall exceeds this many seconds (0=disabled)"); |
| 7727 | 7727 | ||
| 7728 | /* | 7728 | /* |
| 7729 | * Show workers that might prevent the processing of pending work items. | 7729 | * Report that a pool has no worker in running state, which is a sign that the |
| 7730 | * A busy worker that is not running on the CPU (e.g. sleeping in | 7730 | * pool may be stuck. Print pool info. Must be called with pool->lock held and |
| 7731 | * wait_event_idle() with PF_WQ_WORKER cleared) can stall the pool just as | 7731 | * inside a printk_deferred_enter/exit region. |
| 7732 | * effectively as a CPU-bound one, so dump every in-flight worker. | 7732 | */ |
| 7733 | static void show_pool_no_running_worker(struct worker_pool *pool) | ||
| 7734 | { | ||
| 7735 | lockdep_assert_held(&pool->lock); | ||
| 7736 | |||
| 7737 | printk_deferred_enter(); | ||
| 7738 | pr_info("pool %d: no worker in running state, cpu=%d is %s (nr_workers=%d nr_idle=%d)\n", | ||
| 7739 | pool->id, pool->cpu, | ||
| 7740 | idle_cpu(pool->cpu) ? "idle" : "busy", | ||
| 7741 | pool->nr_workers, pool->nr_idle); | ||
| 7742 | pr_info("The pool might have trouble waking an idle worker.\n"); | ||
| 7743 | printk_deferred_exit(); | ||
| 7744 | } | ||
| 7745 | |||
| 7746 | /* | ||
| 7747 | * Show running workers that might prevent the processing of pending work items. | ||
| 7748 | * If no running worker is found, the pool may be stuck waiting for an idle | ||
| 7749 | * worker to be woken, so report the pool state. | ||
| 7733 | */ | 7750 | */ |
| 7734 | static void show_cpu_pool_busy_workers(struct worker_pool *pool) | 7751 | static void show_cpu_pool_busy_workers(struct worker_pool *pool) |
| 7735 | { | 7752 | { |
| 7753 | bool found_running = false; | ||
| 7736 | struct worker *worker; | 7754 | struct worker *worker; |
| 7737 | unsigned long irq_flags; | 7755 | unsigned long irq_flags; |
| 7738 | int bkt; | 7756 | int bkt; |
| @@ -7740,6 +7758,11 @@ static void show_cpu_pool_busy_workers(struct worker_pool *pool) · 1 unmodified line | |||
| 7740 | raw_spin_lock_irqsave(&pool->lock, irq_flags); | 7758 | raw_spin_lock_irqsave(&pool->lock, irq_flags); |
| 7741 | 7759 | ||
| 7742 | hash_for_each(pool->busy_hash, bkt, worker, hentry) { | 7760 | hash_for_each(pool->busy_hash, bkt, worker, hentry) { |
| 7761 | /* Skip workers that are not actively running on the CPU. */ | ||
| 7762 | if (!task_is_running(worker->task)) | ||
| 7763 | continue; | ||
| 7764 | |||
| 7765 | found_running = true; | ||
| 7743 | /* | 7766 | /* |
| 7744 | * Defer printing to avoid deadlocks in console | 7767 | * Defer printing to avoid deadlocks in console |
| 7745 | * drivers that queue work while holding locks | 7768 | * drivers that queue work while holding locks |
| @@ -7753,6 +7776,13 @@ static void show_cpu_pool_busy_workers(struct worker_pool *pool) · 7 unmodified lines | |||
| 7753 | printk_deferred_exit(); | 7776 | printk_deferred_exit(); |
| 7754 | } | 7777 | } |
| 7755 | 7778 | ||
| 7779 | /* | ||
| 7780 | * If no running worker was found, the pool is likely stuck. Print pool | ||
| 7781 | * state. | ||
| 7782 | */ | ||
| 7783 | if (!found_running) | ||
| 7784 | show_pool_no_running_worker(pool); | ||
| 7785 | |||
| 7756 | raw_spin_unlock_irqrestore(&pool->lock, irq_flags); | 7786 | raw_spin_unlock_irqrestore(&pool->lock, irq_flags); |
| 7757 | } | 7787 | } |
| 7758 | 7788 | ||