History

Merge tag 'wq-for-6.5' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq

Pull workqueue updates from Tejun Heo: - Concurrency-managed per-cpu work items that hog CPUs and delay the execution of other work items are now automatically detected and excluded from concurrency management. Reporting on such work items can also be enabled through a config option. - Added tools/workqueue/wq_monitor.py which improves visibility into workqueue usages and behaviors. - Arnd's minimal fix for gcc-13 enum warning on 32bit compiles, superseded by commit afa4bb778e48 in mainline. * tag 'wq-for-6.5' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq: workqueue: Disable per-cpu CPU hog detection when wq_cpu_intensive_thresh_us is 0 workqueue: Fix WARN_ON_ONCE() triggers in worker_enter_idle() workqueue: fix enum type for gcc-13 workqueue: Track and monitor per-workqueue CPU time usage workqueue: Report work funcs that trigger automatic CPU_INTENSIVE mechanism workqueue: Automatically mark CPU-hogging work items CPU_INTENSIVE workqueue: Improve locking rule description for worker fields workqueue: Move worker_set/clr_flags() upwards workqueue: Re-order struct worker fields workqueue: Add pwq->stats[] and a monitoring script Further upgrade queue_work_on() comment

Linus Torvalds committed 3 years ago

Commit 7ab044a2 parents 18eb3b6 18c8ae87 files · +499 −75Browse files

Showing the changes to kernel/workqueue.c (1 of 7 files). Show all

Unified · Split

kernel/workqueue.c+258−64
@@ -126,6 +126,12 @@ enum { · 125 unmodified lines
126 * cpu or grabbing pool->lock is enough for read access. If126 * cpu or grabbing pool->lock is enough for read access. If
127 * POOL_DISASSOCIATED is set, it's identical to L.127 * POOL_DISASSOCIATED is set, it's identical to L.
128 *128 *
129 * K: Only modified by worker while holding pool->lock. Can be safely read by
130 * self, while holding pool->lock or from IRQ context if %current is the
131 * kworker.
132 *
133 * S: Only modified by worker self.
134 *
129 * A: wq_pool_attach_mutex protected.135 * A: wq_pool_attach_mutex protected.
130 *136 *
131 * PL: wq_pool_mutex protected.137 * PL: wq_pool_mutex protected.
@@ -199,6 +205,22 @@ struct worker_pool { · 67 unmodified lines
199 struct rcu_head rcu;205 struct rcu_head rcu;
200};206};
201 207
208/*
209 * Per-pool_workqueue statistics. These can be monitored using
210 * tools/workqueue/wq_monitor.py.
211 */
212enum pool_workqueue_stats {
213 PWQ_STAT_STARTED, /* work items started execution */
214 PWQ_STAT_COMPLETED, /* work items completed execution */
215 PWQ_STAT_CPU_TIME, /* total CPU time consumed */
216 PWQ_STAT_CPU_INTENSIVE, /* wq_cpu_intensive_thresh_us violations */
217 PWQ_STAT_CM_WAKEUP, /* concurrency-management worker wakeups */
218 PWQ_STAT_MAYDAY, /* maydays to rescuer */
219 PWQ_STAT_RESCUED, /* linked work items executed by rescuer */
220
221 PWQ_NR_STATS,
222};
223
202/*224/*
203 * The per-pool workqueue. While queued, the lower WORK_STRUCT_FLAG_BITS225 * The per-pool workqueue. While queued, the lower WORK_STRUCT_FLAG_BITS
204 * of work_struct->data are used for flags and the remaining high bits226 * of work_struct->data are used for flags and the remaining high bits
@@ -236,6 +258,8 @@ struct pool_workqueue { · 31 unmodified lines
236 struct list_head pwqs_node; /* WR: node on wq->pwqs */258 struct list_head pwqs_node; /* WR: node on wq->pwqs */
237 struct list_head mayday_node; /* MD: node on wq->maydays */259 struct list_head mayday_node; /* MD: node on wq->maydays */
238 260
261 u64 stats[PWQ_NR_STATS];
262
239 /*263 /*
240 * Release of unbound pwq is punted to system_wq. See put_pwq()264 * Release of unbound pwq is punted to system_wq. See put_pwq()
241 * and pwq_unbound_release_workfn() for details. pool_workqueue265 * and pwq_unbound_release_workfn() for details. pool_workqueue
@@ -310,6 +334,14 @@ static struct kmem_cache *pwq_cache; · 68 unmodified lines
310static cpumask_var_t *wq_numa_possible_cpumask;334static cpumask_var_t *wq_numa_possible_cpumask;
311 /* possible CPUs of each node */335 /* possible CPUs of each node */
312 336
337/*
338 * Per-cpu work items which run for longer than the following threshold are
339 * automatically considered CPU intensive and excluded from concurrency
340 * management to prevent them from noticeably delaying other per-cpu work items.
341 */
342static unsigned long wq_cpu_intensive_thresh_us = 10000;
343module_param_named(cpu_intensive_thresh_us, wq_cpu_intensive_thresh_us, ulong, 0644);
344
313static bool wq_disable_numa;345static bool wq_disable_numa;
314module_param_named(disable_numa, wq_disable_numa, bool, 0444);346module_param_named(disable_numa, wq_disable_numa, bool, 0444);
315 347
@@ -866,6 +898,152 @@ static void wake_up_worker(struct worker_pool *pool) · 550 unmodified lines
866 wake_up_process(worker->task);898 wake_up_process(worker->task);
867}899}
868 900
901/**
902 * worker_set_flags - set worker flags and adjust nr_running accordingly
903 * @worker: self
904 * @flags: flags to set
905 *
906 * Set @flags in @worker->flags and adjust nr_running accordingly.
907 *
908 * CONTEXT:
909 * raw_spin_lock_irq(pool->lock)
910 */
911static inline void worker_set_flags(struct worker *worker, unsigned int flags)
912{
913 struct worker_pool *pool = worker->pool;
914
915 WARN_ON_ONCE(worker->task != current);
916
917 /* If transitioning into NOT_RUNNING, adjust nr_running. */
918 if ((flags & WORKER_NOT_RUNNING) &&
919 !(worker->flags & WORKER_NOT_RUNNING)) {
920 pool->nr_running--;
921 }
922
923 worker->flags |= flags;
924}
925
926/**
927 * worker_clr_flags - clear worker flags and adjust nr_running accordingly
928 * @worker: self
929 * @flags: flags to clear
930 *
931 * Clear @flags in @worker->flags and adjust nr_running accordingly.
932 *
933 * CONTEXT:
934 * raw_spin_lock_irq(pool->lock)
935 */
936static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
937{
938 struct worker_pool *pool = worker->pool;
939 unsigned int oflags = worker->flags;
940
941 WARN_ON_ONCE(worker->task != current);
942
943 worker->flags &= ~flags;
944
945 /*
946 * If transitioning out of NOT_RUNNING, increment nr_running. Note
947 * that the nested NOT_RUNNING is not a noop. NOT_RUNNING is mask
948 * of multiple flags, not a single flag.
949 */
950 if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
951 if (!(worker->flags & WORKER_NOT_RUNNING))
952 pool->nr_running++;
953}
954
955#ifdef CONFIG_WQ_CPU_INTENSIVE_REPORT
956
957/*
958 * Concurrency-managed per-cpu work items that hog CPU for longer than
959 * wq_cpu_intensive_thresh_us trigger the automatic CPU_INTENSIVE mechanism,
960 * which prevents them from stalling other concurrency-managed work items. If a
961 * work function keeps triggering this mechanism, it's likely that the work item
962 * should be using an unbound workqueue instead.
963 *
964 * wq_cpu_intensive_report() tracks work functions which trigger such conditions
965 * and report them so that they can be examined and converted to use unbound
966 * workqueues as appropriate. To avoid flooding the console, each violating work
967 * function is tracked and reported with exponential backoff.
968 */
969#define WCI_MAX_ENTS 128
970
971struct wci_ent {
972 work_func_t func;
973 atomic64_t cnt;
974 struct hlist_node hash_node;
975};
976
977static struct wci_ent wci_ents[WCI_MAX_ENTS];
978static int wci_nr_ents;
979static DEFINE_RAW_SPINLOCK(wci_lock);
980static DEFINE_HASHTABLE(wci_hash, ilog2(WCI_MAX_ENTS));
981
982static struct wci_ent *wci_find_ent(work_func_t func)
983{
984 struct wci_ent *ent;
985
986 hash_for_each_possible_rcu(wci_hash, ent, hash_node,
987 (unsigned long)func) {
988 if (ent->func == func)
989 return ent;
990 }
991 return NULL;
992}
993
994static void wq_cpu_intensive_report(work_func_t func)
995{
996 struct wci_ent *ent;
997
998restart:
999 ent = wci_find_ent(func);
1000 if (ent) {
1001 u64 cnt;
1002
1003 /*
1004 * Start reporting from the fourth time and back off
1005 * exponentially.
1006 */
1007 cnt = atomic64_inc_return_relaxed(&ent->cnt);
1008 if (cnt >= 4 && is_power_of_2(cnt))
1009 printk_deferred(KERN_WARNING "workqueue: %ps hogged CPU for >%luus %llu times, consider switching to WQ_UNBOUND\n",
1010 ent->func, wq_cpu_intensive_thresh_us,
1011 atomic64_read(&ent->cnt));
1012 return;
1013 }
1014
1015 /*
1016 * @func is a new violation. Allocate a new entry for it. If wcn_ents[]
1017 * is exhausted, something went really wrong and we probably made enough
1018 * noise already.
1019 */
1020 if (wci_nr_ents >= WCI_MAX_ENTS)
1021 return;
1022
1023 raw_spin_lock(&wci_lock);
1024
1025 if (wci_nr_ents >= WCI_MAX_ENTS) {
1026 raw_spin_unlock(&wci_lock);
1027 return;
1028 }
1029
1030 if (wci_find_ent(func)) {
1031 raw_spin_unlock(&wci_lock);
1032 goto restart;
1033 }
1034
1035 ent = &wci_ents[wci_nr_ents++];
1036 ent->func = func;
1037 atomic64_set(&ent->cnt, 1);
1038 hash_add_rcu(wci_hash, &ent->hash_node, (unsigned long)func);
1039
1040 raw_spin_unlock(&wci_lock);
1041}
1042
1043#else /* CONFIG_WQ_CPU_INTENSIVE_REPORT */
1044static void wq_cpu_intensive_report(work_func_t func) {}
1045#endif /* CONFIG_WQ_CPU_INTENSIVE_REPORT */
1046
869/**1047/**
870 * wq_worker_running - a worker is running again1048 * wq_worker_running - a worker is running again
871 * @task: task waking up1049 * @task: task waking up
@@ -876,7 +1054,7 @@ void wq_worker_running(struct task_struct *task) · 4 unmodified lines
876{1054{
877 struct worker *worker = kthread_data(task);1055 struct worker *worker = kthread_data(task);
878 1056
879 if (!worker->sleeping)1057 if (!READ_ONCE(worker->sleeping))
880 return;1058 return;
881 1059
882 /*1060 /*
@@ -889,7 +1067,14 @@ void wq_worker_running(struct task_struct *task) · 6 unmodified lines
889 if (!(worker->flags & WORKER_NOT_RUNNING))1067 if (!(worker->flags & WORKER_NOT_RUNNING))
890 worker->pool->nr_running++;1068 worker->pool->nr_running++;
891 preempt_enable();1069 preempt_enable();
892 worker->sleeping = 0;1070
1071 /*
1072 * CPU intensive auto-detection cares about how long a work item hogged
1073 * CPU without sleeping. Reset the starting timestamp on wakeup.
1074 */
1075 worker->current_at = worker->task->se.sum_exec_runtime;
1076
1077 WRITE_ONCE(worker->sleeping, 0);
893}1078}
894 1079
895/**1080/**
@@ -915,10 +1100,10 @@ void wq_worker_sleeping(struct task_struct *task) · 19 unmodified lines
915 pool = worker->pool;1100 pool = worker->pool;
916 1101
917 /* Return if preempted before wq_worker_running() was reached */1102 /* Return if preempted before wq_worker_running() was reached */
918 if (worker->sleeping)1103 if (READ_ONCE(worker->sleeping))
919 return;1104 return;
920 1105
921 worker->sleeping = 1;1106 WRITE_ONCE(worker->sleeping, 1);
922 raw_spin_lock_irq(&pool->lock);1107 raw_spin_lock_irq(&pool->lock);
923 1108
924 /*1109 /*
@@ -932,11 +1117,65 @@ void wq_worker_sleeping(struct task_struct *task) · 7 unmodified lines
932 }1117 }
933 1118
934 pool->nr_running--;1119 pool->nr_running--;
935 if (need_more_worker(pool))1120 if (need_more_worker(pool)) {
1121 worker->current_pwq->stats[PWQ_STAT_CM_WAKEUP]++;
936 wake_up_worker(pool);1122 wake_up_worker(pool);
1123 }
937 raw_spin_unlock_irq(&pool->lock);1124 raw_spin_unlock_irq(&pool->lock);
938}1125}
939 1126
1127/**
1128 * wq_worker_tick - a scheduler tick occurred while a kworker is running
1129 * @task: task currently running
1130 *
1131 * Called from scheduler_tick(). We're in the IRQ context and the current
1132 * worker's fields which follow the 'K' locking rule can be accessed safely.
1133 */
1134void wq_worker_tick(struct task_struct *task)
1135{
1136 struct worker *worker = kthread_data(task);
1137 struct pool_workqueue *pwq = worker->current_pwq;
1138 struct worker_pool *pool = worker->pool;
1139
1140 if (!pwq)
1141 return;
1142
1143 pwq->stats[PWQ_STAT_CPU_TIME] += TICK_USEC;
1144
1145 if (!wq_cpu_intensive_thresh_us)
1146 return;
1147
1148 /*
1149 * If the current worker is concurrency managed and hogged the CPU for
1150 * longer than wq_cpu_intensive_thresh_us, it's automatically marked
1151 * CPU_INTENSIVE to avoid stalling other concurrency-managed work items.
1152 *
1153 * Set @worker->sleeping means that @worker is in the process of
1154 * switching out voluntarily and won't be contributing to
1155 * @pool->nr_running until it wakes up. As wq_worker_sleeping() also
1156 * decrements ->nr_running, setting CPU_INTENSIVE here can lead to
1157 * double decrements. The task is releasing the CPU anyway. Let's skip.
1158 * We probably want to make this prettier in the future.
1159 */
1160 if ((worker->flags & WORKER_NOT_RUNNING) || READ_ONCE(worker->sleeping) ||
1161 worker->task->se.sum_exec_runtime - worker->current_at <
1162 wq_cpu_intensive_thresh_us * NSEC_PER_USEC)
1163 return;
1164
1165 raw_spin_lock(&pool->lock);
1166
1167 worker_set_flags(worker, WORKER_CPU_INTENSIVE);
1168 wq_cpu_intensive_report(worker->current_func);
1169 pwq->stats[PWQ_STAT_CPU_INTENSIVE]++;
1170
1171 if (need_more_worker(pool)) {
1172 pwq->stats[PWQ_STAT_CM_WAKEUP]++;
1173 wake_up_worker(pool);
1174 }
1175
1176 raw_spin_unlock(&pool->lock);
1177}
1178
940/**1179/**
941 * wq_worker_last_func - retrieve worker's last work function1180 * wq_worker_last_func - retrieve worker's last work function
942 * @task: Task to retrieve last work function of.1181 * @task: Task to retrieve last work function of.
@@ -968,60 +1207,6 @@ work_func_t wq_worker_last_func(struct task_struct *task) · 25 unmodified lines
968 return worker->last_func;1207 return worker->last_func;
969}1208}
970 1209
971/**
972 * worker_set_flags - set worker flags and adjust nr_running accordingly
973 * @worker: self
974 * @flags: flags to set
975 *
976 * Set @flags in @worker->flags and adjust nr_running accordingly.
977 *
978 * CONTEXT:
979 * raw_spin_lock_irq(pool->lock)
980 */
981static inline void worker_set_flags(struct worker *worker, unsigned int flags)
982{
983 struct worker_pool *pool = worker->pool;
984
985 WARN_ON_ONCE(worker->task != current);
986
987 /* If transitioning into NOT_RUNNING, adjust nr_running. */
988 if ((flags & WORKER_NOT_RUNNING) &&
989 !(worker->flags & WORKER_NOT_RUNNING)) {
990 pool->nr_running--;
991 }
992
993 worker->flags |= flags;
994}
995
996/**
997 * worker_clr_flags - clear worker flags and adjust nr_running accordingly
998 * @worker: self
999 * @flags: flags to clear
1000 *
1001 * Clear @flags in @worker->flags and adjust nr_running accordingly.
1002 *
1003 * CONTEXT:
1004 * raw_spin_lock_irq(pool->lock)
1005 */
1006static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
1007{
1008 struct worker_pool *pool = worker->pool;
1009 unsigned int oflags = worker->flags;
1010
1011 WARN_ON_ONCE(worker->task != current);
1012
1013 worker->flags &= ~flags;
1014
1015 /*
1016 * If transitioning out of NOT_RUNNING, increment nr_running. Note
1017 * that the nested NOT_RUNNING is not a noop. NOT_RUNNING is mask
1018 * of multiple flags, not a single flag.
1019 */
1020 if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
1021 if (!(worker->flags & WORKER_NOT_RUNNING))
1022 pool->nr_running++;
1023}
1024
1025/**1210/**
1026 * find_worker_executing_work - find worker which is executing a work1211 * find_worker_executing_work - find worker which is executing a work
1027 * @pool: pool of interest1212 * @pool: pool of interest
@@ -1542,6 +1727,8 @@ out: · 514 unmodified lines
1542 * We queue the work to a specific CPU, the caller must ensure it1727 * We queue the work to a specific CPU, the caller must ensure it
1543 * can't go away. Callers that fail to ensure that the specified1728 * can't go away. Callers that fail to ensure that the specified
1544 * CPU cannot go away will execute on a randomly chosen CPU.1729 * CPU cannot go away will execute on a randomly chosen CPU.
1730 * But note well that callers specifying a CPU that never has been
1731 * online will get a splat.
1545 *1732 *
1546 * Return: %false if @work was already on a queue, %true otherwise.1733 * Return: %false if @work was already on a queue, %true otherwise.
1547 */1734 */
@@ -2166,6 +2353,7 @@ static void send_mayday(struct work_struct *work) · 618 unmodified lines
2166 get_pwq(pwq);2353 get_pwq(pwq);
2167 list_add_tail(&pwq->mayday_node, &wq->maydays);2354 list_add_tail(&pwq->mayday_node, &wq->maydays);
2168 wake_up_process(wq->rescuer->task);2355 wake_up_process(wq->rescuer->task);
2356 pwq->stats[PWQ_STAT_MAYDAY]++;
2169 }2357 }
2170}2358}
2171 2359
@@ -2303,7 +2491,6 @@ __acquires(&pool->lock) · 131 unmodified lines
2303{2491{
2304 struct pool_workqueue *pwq = get_work_pwq(work);2492 struct pool_workqueue *pwq = get_work_pwq(work);
2305 struct worker_pool *pool = worker->pool;2493 struct worker_pool *pool = worker->pool;
2306 bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE;
2307 unsigned long work_data;2494 unsigned long work_data;
2308 struct worker *collision;2495 struct worker *collision;
2309#ifdef CONFIG_LOCKDEP2496#ifdef CONFIG_LOCKDEP
@@ -2340,6 +2527,7 @@ __acquires(&pool->lock) · 30 unmodified lines
2340 worker->current_work = work;2527 worker->current_work = work;
2341 worker->current_func = work->func;2528 worker->current_func = work->func;
2342 worker->current_pwq = pwq;2529 worker->current_pwq = pwq;
2530 worker->current_at = worker->task->se.sum_exec_runtime;
2343 work_data = *work_data_bits(work);2531 work_data = *work_data_bits(work);
2344 worker->current_color = get_work_color(work_data);2532 worker->current_color = get_work_color(work_data);
2345 2533
@@ -2357,7 +2545,7 @@ __acquires(&pool->lock) · 11 unmodified lines
2357 * of concurrency management and the next code block will chain2545 * of concurrency management and the next code block will chain
2358 * execution of the pending work items.2546 * execution of the pending work items.
2359 */2547 */
2360 if (unlikely(cpu_intensive))2548 if (unlikely(pwq->wq->flags & WQ_CPU_INTENSIVE))
2361 worker_set_flags(worker, WORKER_CPU_INTENSIVE);2549 worker_set_flags(worker, WORKER_CPU_INTENSIVE);
2362 2550
2363 /*2551 /*
@@ -2404,6 +2592,7 @@ __acquires(&pool->lock) · 40 unmodified lines
2404 * workqueues), so hiding them isn't a problem.2592 * workqueues), so hiding them isn't a problem.
2405 */2593 */
2406 lockdep_invariant_state(true);2594 lockdep_invariant_state(true);
2595 pwq->stats[PWQ_STAT_STARTED]++;
2407 trace_workqueue_execute_start(work);2596 trace_workqueue_execute_start(work);
2408 worker->current_func(work);2597 worker->current_func(work);
2409 /*2598 /*
@@ -2411,6 +2600,7 @@ __acquires(&pool->lock) · 1 unmodified line
2411 * point will only record its address.2600 * point will only record its address.
2412 */2601 */
2413 trace_workqueue_execute_end(work, worker->current_func);2602 trace_workqueue_execute_end(work, worker->current_func);
2603 pwq->stats[PWQ_STAT_COMPLETED]++;
2414 lock_map_release(&lockdep_map);2604 lock_map_release(&lockdep_map);
2415 lock_map_release(&pwq->wq->lockdep_map);2605 lock_map_release(&pwq->wq->lockdep_map);
2416 2606
@@ -2435,9 +2625,12 @@ __acquires(&pool->lock) · 18 unmodified lines
2435 2625
2436 raw_spin_lock_irq(&pool->lock);2626 raw_spin_lock_irq(&pool->lock);
2437 2627
2438 /* clear cpu intensive status */2628 /*
2439 if (unlikely(cpu_intensive))2629 * In addition to %WQ_CPU_INTENSIVE, @worker may also have been marked
2440 worker_clr_flags(worker, WORKER_CPU_INTENSIVE);2630 * CPU intensive by wq_worker_tick() if @work hogged CPU longer than
2631 * wq_cpu_intensive_thresh_us. Clear it.
2632 */
2633 worker_clr_flags(worker, WORKER_CPU_INTENSIVE);
2441 2634
2442 /* tag the worker for identification in schedule() */2635 /* tag the worker for identification in schedule() */
2443 worker->last_func = worker->current_func;2636 worker->last_func = worker->current_func;
@@ -2654,6 +2847,7 @@ repeat: · 210 unmodified lines
2654 if (first)2847 if (first)
2655 pool->watchdog_ts = jiffies;2848 pool->watchdog_ts = jiffies;
2656 move_linked_works(work, scheduled, &n);2849 move_linked_works(work, scheduled, &n);
2850 pwq->stats[PWQ_STAT_RESCUED]++;
2657 }2851 }
2658 first = false;2852 first = false;
2659 }2853 }