Currently, pool->nr_running can be modified from timer tick, that means the timer tick can run nested inside a not-irq-protected section that's in the process of modifying nr_running. Consider the following scenario: CPU0 kworker/0:2 (events) worker_clr_flags(worker, WORKER_PREP | WORKER_REBOUND); ->pool->nr_running++; (1) process_one_work() ->worker->current_func(work); ->schedule() ->wq_worker_sleeping() ->worker->sleeping = 1; ->pool->nr_running--; (0) .... ->wq_worker_running() .... CPU0 by interrupt: wq_worker_tick() ->worker_set_flags(worker, WORKER_CPU_INTENSIVE); ->pool->nr_running--; (-1) ->worker->flags |= WORKER_CPU_INTENSIVE; .... ->if (!(worker->flags & WORKER_NOT_RUNNING)) ->pool->nr_running++; (will not execute) ->worker->sleeping = 0; .... ->worker_clr_flags(worker, WORKER_CPU_INTENSIVE); ->pool->nr_running++; (0) .... worker_set_flags(worker, WORKER_PREP); ->pool->nr_running--; (-1) .... worker_enter_idle() ->WARN_ON_ONCE(pool->nr_workers == pool->nr_idle && pool->nr_running); if the nr_workers is equal to nr_idle, due to the nr_running is not zero, will trigger WARN_ON_ONCE(). [ 2.460602] WARNING: CPU: 0 PID: 63 at kernel/workqueue.c:1999 worker_enter_idle+0xb2/0xc0 [ 2.462163] Modules linked in: [ 2.463401] CPU: 0 PID: 63 Comm: kworker/0:2 Not tainted 6.4.0-rc2-next-20230519 #1 [ 2.463771] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.14.0-2 04/01/2014 [ 2.465127] Workqueue: 0x0 (events) [ 2.465678] RIP: 0010:worker_enter_idle+0xb2/0xc0 ... [ 2.472614] Call Trace: [ 2.473152] <TASK> [ 2.474182] worker_thread+0x71/0x430 [ 2.474992] ? _raw_spin_unlock_irqrestore+0x28/0x50 [ 2.475263] kthread+0x103/0x120 [ 2.475493] ? __pfx_worker_thread+0x10/0x10 [ 2.476355] ? __pfx_kthread+0x10/0x10 [ 2.476635] ret_from_fork+0x2c/0x50 [ 2.477051] </TASK> This commit therefore add the check of worker->sleeping in wq_worker_tick(), if the worker->sleeping is not zero, directly return. tj: Updated comment and description.
Showing the changes to kernel/workqueue.c (1 of 1 file). Show all
Unified · Split
| @@ -1051,7 +1051,7 @@ void wq_worker_running(struct task_struct *task) · 1050 unmodified lines | ||
| 1051 | 1051 | { |
| 1052 | 1052 | struct worker *worker = kthread_data(task); |
| 1053 | 1053 | |
| 1054 | − if (!worker->sleeping) | |
| 1054 | + if (!READ_ONCE(worker->sleeping)) | |
| 1055 | 1055 | return; |
| 1056 | 1056 | |
| 1057 | 1057 | /* |
| @@ -1071,7 +1071,7 @@ void wq_worker_running(struct task_struct *task) · 13 unmodified lines | ||
| 1071 | 1071 | */ |
| 1072 | 1072 | worker->current_at = worker->task->se.sum_exec_runtime; |
| 1073 | 1073 | |
| 1074 | − worker->sleeping = 0; | |
| 1074 | + WRITE_ONCE(worker->sleeping, 0); | |
| 1075 | 1075 | } |
| 1076 | 1076 | |
| 1077 | 1077 | /** |
| @@ -1097,10 +1097,10 @@ void wq_worker_sleeping(struct task_struct *task) · 19 unmodified lines | ||
| 1097 | 1097 | pool = worker->pool; |
| 1098 | 1098 | |
| 1099 | 1099 | /* Return if preempted before wq_worker_running() was reached */ |
| 1100 | − if (worker->sleeping) | |
| 1100 | + if (READ_ONCE(worker->sleeping)) | |
| 1101 | 1101 | return; |
| 1102 | 1102 | |
| 1103 | − worker->sleeping = 1; | |
| 1103 | + WRITE_ONCE(worker->sleeping, 1); | |
| 1104 | 1104 | raw_spin_lock_irq(&pool->lock); |
| 1105 | 1105 | |
| 1106 | 1106 | /* |
| @@ -1143,8 +1143,15 @@ void wq_worker_tick(struct task_struct *task) · 36 unmodified lines | ||
| 1143 | 1143 | * If the current worker is concurrency managed and hogged the CPU for |
| 1144 | 1144 | * longer than wq_cpu_intensive_thresh_us, it's automatically marked |
| 1145 | 1145 | * CPU_INTENSIVE to avoid stalling other concurrency-managed work items. |
| 1146 | + * | |
| 1147 | + * Set @worker->sleeping means that @worker is in the process of | |
| 1148 | + * switching out voluntarily and won't be contributing to | |
| 1149 | + * @pool->nr_running until it wakes up. As wq_worker_sleeping() also | |
| 1150 | + * decrements ->nr_running, setting CPU_INTENSIVE here can lead to | |
| 1151 | + * double decrements. The task is releasing the CPU anyway. Let's skip. | |
| 1152 | + * We probably want to make this prettier in the future. | |
| 1146 | 1153 | */ |
| 1147 | − if ((worker->flags & WORKER_NOT_RUNNING) || | |
| 1154 | + if ((worker->flags & WORKER_NOT_RUNNING) || READ_ONCE(worker->sleeping) || | |
| 1148 | 1155 | worker->task->se.sum_exec_runtime - worker->current_at < |
| 1149 | 1156 | wq_cpu_intensive_thresh_us * NSEC_PER_USEC) |
| 1150 | 1157 | return; |