If a per-cpu work item hogs the CPU, it can prevent other work items from starting through concurrency management. A per-cpu workqueue which intends to host such CPU-hogging work items can choose to not participate in concurrency management by setting %WQ_CPU_INTENSIVE; however, this can be error-prone and difficult to debug when missed. This patch adds an automatic CPU usage based detection. If a concurrency-managed work item consumes more CPU time than the threshold (10ms by default) continuously without intervening sleeps, wq_worker_tick() which is called from scheduler_tick() will detect the condition and automatically mark it CPU_INTENSIVE. The mechanism isn't foolproof: * Detection depends on tick hitting the work item. Getting preempted at the right timings may allow a violating work item to evade detection at least temporarily. * nohz_full CPUs may not be running ticks and thus can fail detection. * Even when detection is working, the 10ms detection delays can add up if many CPU-hogging work items are queued at the same time. However, in vast majority of cases, this should be able to detect violations reliably and provide reasonable protection with a small increase in code complexity. If some work items trigger this condition repeatedly, the bigger problem likely is the CPU being saturated with such per-cpu work items and the solution would be making them UNBOUND. The next patch will add a debug mechanism to help spot such cases. v4: Documentation for workqueue.cpu_intensive_thresh_us added to kernel-parameters.txt. v3: Switch to use wq_worker_tick() instead of hooking into preemptions as suggested by Peter. v2: Lai pointed out that wq_worker_stopping() also needs to be called from preemption and rtlock paths and an earlier patch was updated accordingly. This patch adds a comment describing the risk of infinte recursions and how they're avoided.
Showing the changes to kernel/workqueue.c (1 of 6 files). Show all
Unified · Split
| @@ -212,6 +212,7 @@ struct worker_pool { · 211 unmodified lines | |||
| 212 | enum pool_workqueue_stats { | 212 | enum pool_workqueue_stats { |
| 213 | PWQ_STAT_STARTED, /* work items started execution */ | 213 | PWQ_STAT_STARTED, /* work items started execution */ |
| 214 | PWQ_STAT_COMPLETED, /* work items completed execution */ | 214 | PWQ_STAT_COMPLETED, /* work items completed execution */ |
| 215 | PWQ_STAT_CPU_INTENSIVE, /* wq_cpu_intensive_thresh_us violations */ | ||
| 215 | PWQ_STAT_CM_WAKEUP, /* concurrency-management worker wakeups */ | 216 | PWQ_STAT_CM_WAKEUP, /* concurrency-management worker wakeups */ |
| 216 | PWQ_STAT_MAYDAY, /* maydays to rescuer */ | 217 | PWQ_STAT_MAYDAY, /* maydays to rescuer */ |
| 217 | PWQ_STAT_RESCUED, /* linked work items executed by rescuer */ | 218 | PWQ_STAT_RESCUED, /* linked work items executed by rescuer */ |
| @@ -332,6 +333,14 @@ static struct kmem_cache *pwq_cache; · 114 unmodified lines | |||
| 332 | static cpumask_var_t *wq_numa_possible_cpumask; | 333 | static cpumask_var_t *wq_numa_possible_cpumask; |
| 333 | /* possible CPUs of each node */ | 334 | /* possible CPUs of each node */ |
| 334 | 335 | ||
| 336 | /* | ||
| 337 | * Per-cpu work items which run for longer than the following threshold are | ||
| 338 | * automatically considered CPU intensive and excluded from concurrency | ||
| 339 | * management to prevent them from noticeably delaying other per-cpu work items. | ||
| 340 | */ | ||
| 341 | static unsigned long wq_cpu_intensive_thresh_us = 10000; | ||
| 342 | module_param_named(cpu_intensive_thresh_us, wq_cpu_intensive_thresh_us, ulong, 0644); | ||
| 343 | |||
| 335 | static bool wq_disable_numa; | 344 | static bool wq_disable_numa; |
| 336 | module_param_named(disable_numa, wq_disable_numa, bool, 0444); | 345 | module_param_named(disable_numa, wq_disable_numa, bool, 0444); |
| 337 | 346 | ||
| @@ -962,6 +971,13 @@ void wq_worker_running(struct task_struct *task) · 624 unmodified lines | |||
| 962 | if (!(worker->flags & WORKER_NOT_RUNNING)) | 971 | if (!(worker->flags & WORKER_NOT_RUNNING)) |
| 963 | worker->pool->nr_running++; | 972 | worker->pool->nr_running++; |
| 964 | preempt_enable(); | 973 | preempt_enable(); |
| 974 | |||
| 975 | /* | ||
| 976 | * CPU intensive auto-detection cares about how long a work item hogged | ||
| 977 | * CPU without sleeping. Reset the starting timestamp on wakeup. | ||
| 978 | */ | ||
| 979 | worker->current_at = worker->task->se.sum_exec_runtime; | ||
| 980 | |||
| 965 | worker->sleeping = 0; | 981 | worker->sleeping = 0; |
| 966 | } | 982 | } |
| 967 | 983 | ||
| @@ -1012,6 +1028,45 @@ void wq_worker_sleeping(struct task_struct *task) · 44 unmodified lines | |||
| 1012 | raw_spin_unlock_irq(&pool->lock); | 1028 | raw_spin_unlock_irq(&pool->lock); |
| 1013 | } | 1029 | } |
| 1014 | 1030 | ||
| 1031 | /** | ||
| 1032 | * wq_worker_tick - a scheduler tick occurred while a kworker is running | ||
| 1033 | * @task: task currently running | ||
| 1034 | * | ||
| 1035 | * Called from scheduler_tick(). We're in the IRQ context and the current | ||
| 1036 | * worker's fields which follow the 'K' locking rule can be accessed safely. | ||
| 1037 | */ | ||
| 1038 | void wq_worker_tick(struct task_struct *task) | ||
| 1039 | { | ||
| 1040 | struct worker *worker = kthread_data(task); | ||
| 1041 | struct pool_workqueue *pwq = worker->current_pwq; | ||
| 1042 | struct worker_pool *pool = worker->pool; | ||
| 1043 | |||
| 1044 | if (!pwq) | ||
| 1045 | return; | ||
| 1046 | |||
| 1047 | /* | ||
| 1048 | * If the current worker is concurrency managed and hogged the CPU for | ||
| 1049 | * longer than wq_cpu_intensive_thresh_us, it's automatically marked | ||
| 1050 | * CPU_INTENSIVE to avoid stalling other concurrency-managed work items. | ||
| 1051 | */ | ||
| 1052 | if ((worker->flags & WORKER_NOT_RUNNING) || | ||
| 1053 | worker->task->se.sum_exec_runtime - worker->current_at < | ||
| 1054 | wq_cpu_intensive_thresh_us * NSEC_PER_USEC) | ||
| 1055 | return; | ||
| 1056 | |||
| 1057 | raw_spin_lock(&pool->lock); | ||
| 1058 | |||
| 1059 | worker_set_flags(worker, WORKER_CPU_INTENSIVE); | ||
| 1060 | pwq->stats[PWQ_STAT_CPU_INTENSIVE]++; | ||
| 1061 | |||
| 1062 | if (need_more_worker(pool)) { | ||
| 1063 | pwq->stats[PWQ_STAT_CM_WAKEUP]++; | ||
| 1064 | wake_up_worker(pool); | ||
| 1065 | } | ||
| 1066 | |||
| 1067 | raw_spin_unlock(&pool->lock); | ||
| 1068 | } | ||
| 1069 | |||
| 1015 | /** | 1070 | /** |
| 1016 | * wq_worker_last_func - retrieve worker's last work function | 1071 | * wq_worker_last_func - retrieve worker's last work function |
| 1017 | * @task: Task to retrieve last work function of. | 1072 | * @task: Task to retrieve last work function of. |
| @@ -2327,7 +2382,6 @@ __acquires(&pool->lock) · 1309 unmodified lines | |||
| 2327 | { | 2382 | { |
| 2328 | struct pool_workqueue *pwq = get_work_pwq(work); | 2383 | struct pool_workqueue *pwq = get_work_pwq(work); |
| 2329 | struct worker_pool *pool = worker->pool; | 2384 | struct worker_pool *pool = worker->pool; |
| 2330 | bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE; | ||
| 2331 | unsigned long work_data; | 2385 | unsigned long work_data; |
| 2332 | struct worker *collision; | 2386 | struct worker *collision; |
| 2333 | #ifdef CONFIG_LOCKDEP | 2387 | #ifdef CONFIG_LOCKDEP |
| @@ -2364,6 +2418,7 @@ __acquires(&pool->lock) · 30 unmodified lines | |||
| 2364 | worker->current_work = work; | 2418 | worker->current_work = work; |
| 2365 | worker->current_func = work->func; | 2419 | worker->current_func = work->func; |
| 2366 | worker->current_pwq = pwq; | 2420 | worker->current_pwq = pwq; |
| 2421 | worker->current_at = worker->task->se.sum_exec_runtime; | ||
| 2367 | work_data = *work_data_bits(work); | 2422 | work_data = *work_data_bits(work); |
| 2368 | worker->current_color = get_work_color(work_data); | 2423 | worker->current_color = get_work_color(work_data); |
| 2369 | 2424 | ||
| @@ -2381,7 +2436,7 @@ __acquires(&pool->lock) · 11 unmodified lines | |||
| 2381 | * of concurrency management and the next code block will chain | 2436 | * of concurrency management and the next code block will chain |
| 2382 | * execution of the pending work items. | 2437 | * execution of the pending work items. |
| 2383 | */ | 2438 | */ |
| 2384 | if (unlikely(cpu_intensive)) | 2439 | if (unlikely(pwq->wq->flags & WQ_CPU_INTENSIVE)) |
| 2385 | worker_set_flags(worker, WORKER_CPU_INTENSIVE); | 2440 | worker_set_flags(worker, WORKER_CPU_INTENSIVE); |
| 2386 | 2441 | ||
| 2387 | /* | 2442 | /* |
| @@ -2461,9 +2516,12 @@ __acquires(&pool->lock) · 73 unmodified lines | |||
| 2461 | 2516 | ||
| 2462 | raw_spin_lock_irq(&pool->lock); | 2517 | raw_spin_lock_irq(&pool->lock); |
| 2463 | 2518 | ||
| 2464 | /* clear cpu intensive status */ | 2519 | /* |
| 2465 | if (unlikely(cpu_intensive)) | 2520 | * In addition to %WQ_CPU_INTENSIVE, @worker may also have been marked |
| 2466 | worker_clr_flags(worker, WORKER_CPU_INTENSIVE); | 2521 | * CPU intensive by wq_worker_tick() if @work hogged CPU longer than |
| 2522 | * wq_cpu_intensive_thresh_us. Clear it. | ||
| 2523 | */ | ||
| 2524 | worker_clr_flags(worker, WORKER_CPU_INTENSIVE); | ||
| 2467 | 2525 | ||
| 2468 | /* tag the worker for identification in schedule() */ | 2526 | /* tag the worker for identification in schedule() */ |
| 2469 | worker->last_func = worker->current_func; | 2527 | worker->last_func = worker->current_func; |