History

Merge tag 'wq-for-6.5' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq

Pull workqueue updates from Tejun Heo: - Concurrency-managed per-cpu work items that hog CPUs and delay the execution of other work items are now automatically detected and excluded from concurrency management. Reporting on such work items can also be enabled through a config option. - Added tools/workqueue/wq_monitor.py which improves visibility into workqueue usages and behaviors. - Arnd's minimal fix for gcc-13 enum warning on 32bit compiles, superseded by commit afa4bb778e48 in mainline. * tag 'wq-for-6.5' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq: workqueue: Disable per-cpu CPU hog detection when wq_cpu_intensive_thresh_us is 0 workqueue: Fix WARN_ON_ONCE() triggers in worker_enter_idle() workqueue: fix enum type for gcc-13 workqueue: Track and monitor per-workqueue CPU time usage workqueue: Report work funcs that trigger automatic CPU_INTENSIVE mechanism workqueue: Automatically mark CPU-hogging work items CPU_INTENSIVE workqueue: Improve locking rule description for worker fields workqueue: Move worker_set/clr_flags() upwards workqueue: Re-order struct worker fields workqueue: Add pwq->stats[] and a monitoring script Further upgrade queue_work_on() comment

Linus Torvalds committed 3 years ago

Commit 7ab044a2 parents 18eb3b6 18c8ae87 files · +499 −75Browse files

Showing the changes to kernel/workqueue.c (1 of 7 files). Show all

Unified · Split

kernel/workqueue.c+258−64
@@ -126,6 +126,12 @@ enum { · 125 unmodified lines
126126 * cpu or grabbing pool->lock is enough for read access. If
127127 * POOL_DISASSOCIATED is set, it's identical to L.
128128 *
129+ * K: Only modified by worker while holding pool->lock. Can be safely read by
130+ * self, while holding pool->lock or from IRQ context if %current is the
131+ * kworker.
132+ *
133+ * S: Only modified by worker self.
134+ *
129135 * A: wq_pool_attach_mutex protected.
130136 *
131137 * PL: wq_pool_mutex protected.
@@ -199,6 +205,22 @@ struct worker_pool { · 67 unmodified lines
199205 struct rcu_head rcu;
200206 };
201207
208+/*
209+ * Per-pool_workqueue statistics. These can be monitored using
210+ * tools/workqueue/wq_monitor.py.
211+ */
212+enum pool_workqueue_stats {
213+ PWQ_STAT_STARTED, /* work items started execution */
214+ PWQ_STAT_COMPLETED, /* work items completed execution */
215+ PWQ_STAT_CPU_TIME, /* total CPU time consumed */
216+ PWQ_STAT_CPU_INTENSIVE, /* wq_cpu_intensive_thresh_us violations */
217+ PWQ_STAT_CM_WAKEUP, /* concurrency-management worker wakeups */
218+ PWQ_STAT_MAYDAY, /* maydays to rescuer */
219+ PWQ_STAT_RESCUED, /* linked work items executed by rescuer */
220+
221+ PWQ_NR_STATS,
222+};
223+
202224 /*
203225 * The per-pool workqueue. While queued, the lower WORK_STRUCT_FLAG_BITS
204226 * of work_struct->data are used for flags and the remaining high bits
@@ -236,6 +258,8 @@ struct pool_workqueue { · 31 unmodified lines
236258 struct list_head pwqs_node; /* WR: node on wq->pwqs */
237259 struct list_head mayday_node; /* MD: node on wq->maydays */
238260
261+ u64 stats[PWQ_NR_STATS];
262+
239263 /*
240264 * Release of unbound pwq is punted to system_wq. See put_pwq()
241265 * and pwq_unbound_release_workfn() for details. pool_workqueue
@@ -310,6 +334,14 @@ static struct kmem_cache *pwq_cache; · 68 unmodified lines
310334 static cpumask_var_t *wq_numa_possible_cpumask;
311335 /* possible CPUs of each node */
312336
337+/*
338+ * Per-cpu work items which run for longer than the following threshold are
339+ * automatically considered CPU intensive and excluded from concurrency
340+ * management to prevent them from noticeably delaying other per-cpu work items.
341+ */
342+static unsigned long wq_cpu_intensive_thresh_us = 10000;
343+module_param_named(cpu_intensive_thresh_us, wq_cpu_intensive_thresh_us, ulong, 0644);
344+
313345 static bool wq_disable_numa;
314346 module_param_named(disable_numa, wq_disable_numa, bool, 0444);
315347
@@ -866,6 +898,152 @@ static void wake_up_worker(struct worker_pool *pool) · 550 unmodified lines
866898 wake_up_process(worker->task);
867899 }
868900
901+/**
902+ * worker_set_flags - set worker flags and adjust nr_running accordingly
903+ * @worker: self
904+ * @flags: flags to set
905+ *
906+ * Set @flags in @worker->flags and adjust nr_running accordingly.
907+ *
908+ * CONTEXT:
909+ * raw_spin_lock_irq(pool->lock)
910+ */
911+static inline void worker_set_flags(struct worker *worker, unsigned int flags)
912+{
913+ struct worker_pool *pool = worker->pool;
914+
915+ WARN_ON_ONCE(worker->task != current);
916+
917+ /* If transitioning into NOT_RUNNING, adjust nr_running. */
918+ if ((flags & WORKER_NOT_RUNNING) &&
919+ !(worker->flags & WORKER_NOT_RUNNING)) {
920+ pool->nr_running--;
921+ }
922+
923+ worker->flags |= flags;
924+}
925+
926+/**
927+ * worker_clr_flags - clear worker flags and adjust nr_running accordingly
928+ * @worker: self
929+ * @flags: flags to clear
930+ *
931+ * Clear @flags in @worker->flags and adjust nr_running accordingly.
932+ *
933+ * CONTEXT:
934+ * raw_spin_lock_irq(pool->lock)
935+ */
936+static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
937+{
938+ struct worker_pool *pool = worker->pool;
939+ unsigned int oflags = worker->flags;
940+
941+ WARN_ON_ONCE(worker->task != current);
942+
943+ worker->flags &= ~flags;
944+
945+ /*
946+ * If transitioning out of NOT_RUNNING, increment nr_running. Note
947+ * that the nested NOT_RUNNING is not a noop. NOT_RUNNING is mask
948+ * of multiple flags, not a single flag.
949+ */
950+ if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
951+ if (!(worker->flags & WORKER_NOT_RUNNING))
952+ pool->nr_running++;
953+}
954+
955+#ifdef CONFIG_WQ_CPU_INTENSIVE_REPORT
956+
957+/*
958+ * Concurrency-managed per-cpu work items that hog CPU for longer than
959+ * wq_cpu_intensive_thresh_us trigger the automatic CPU_INTENSIVE mechanism,
960+ * which prevents them from stalling other concurrency-managed work items. If a
961+ * work function keeps triggering this mechanism, it's likely that the work item
962+ * should be using an unbound workqueue instead.
963+ *
964+ * wq_cpu_intensive_report() tracks work functions which trigger such conditions
965+ * and report them so that they can be examined and converted to use unbound
966+ * workqueues as appropriate. To avoid flooding the console, each violating work
967+ * function is tracked and reported with exponential backoff.
968+ */
969+#define WCI_MAX_ENTS 128
970+
971+struct wci_ent {
972+ work_func_t func;
973+ atomic64_t cnt;
974+ struct hlist_node hash_node;
975+};
976+
977+static struct wci_ent wci_ents[WCI_MAX_ENTS];
978+static int wci_nr_ents;
979+static DEFINE_RAW_SPINLOCK(wci_lock);
980+static DEFINE_HASHTABLE(wci_hash, ilog2(WCI_MAX_ENTS));
981+
982+static struct wci_ent *wci_find_ent(work_func_t func)
983+{
984+ struct wci_ent *ent;
985+
986+ hash_for_each_possible_rcu(wci_hash, ent, hash_node,
987+ (unsigned long)func) {
988+ if (ent->func == func)
989+ return ent;
990+ }
991+ return NULL;
992+}
993+
994+static void wq_cpu_intensive_report(work_func_t func)
995+{
996+ struct wci_ent *ent;
997+
998+restart:
999+ ent = wci_find_ent(func);
1000+ if (ent) {
1001+ u64 cnt;
1002+
1003+ /*
1004+ * Start reporting from the fourth time and back off
1005+ * exponentially.
1006+ */
1007+ cnt = atomic64_inc_return_relaxed(&ent->cnt);
1008+ if (cnt >= 4 && is_power_of_2(cnt))
1009+ printk_deferred(KERN_WARNING "workqueue: %ps hogged CPU for >%luus %llu times, consider switching to WQ_UNBOUND\n",
1010+ ent->func, wq_cpu_intensive_thresh_us,
1011+ atomic64_read(&ent->cnt));
1012+ return;
1013+ }
1014+
1015+ /*
1016+ * @func is a new violation. Allocate a new entry for it. If wcn_ents[]
1017+ * is exhausted, something went really wrong and we probably made enough
1018+ * noise already.
1019+ */
1020+ if (wci_nr_ents >= WCI_MAX_ENTS)
1021+ return;
1022+
1023+ raw_spin_lock(&wci_lock);
1024+
1025+ if (wci_nr_ents >= WCI_MAX_ENTS) {
1026+ raw_spin_unlock(&wci_lock);
1027+ return;
1028+ }
1029+
1030+ if (wci_find_ent(func)) {
1031+ raw_spin_unlock(&wci_lock);
1032+ goto restart;
1033+ }
1034+
1035+ ent = &wci_ents[wci_nr_ents++];
1036+ ent->func = func;
1037+ atomic64_set(&ent->cnt, 1);
1038+ hash_add_rcu(wci_hash, &ent->hash_node, (unsigned long)func);
1039+
1040+ raw_spin_unlock(&wci_lock);
1041+}
1042+
1043+#else /* CONFIG_WQ_CPU_INTENSIVE_REPORT */
1044+static void wq_cpu_intensive_report(work_func_t func) {}
1045+#endif /* CONFIG_WQ_CPU_INTENSIVE_REPORT */
1046+
8691047 /**
8701048 * wq_worker_running - a worker is running again
8711049 * @task: task waking up
@@ -876,7 +1054,7 @@ void wq_worker_running(struct task_struct *task) · 4 unmodified lines
8761054 {
8771055 struct worker *worker = kthread_data(task);
8781056
879− if (!worker->sleeping)
1057+ if (!READ_ONCE(worker->sleeping))
8801058 return;
8811059
8821060 /*
@@ -889,7 +1067,14 @@ void wq_worker_running(struct task_struct *task) · 6 unmodified lines
8891067 if (!(worker->flags & WORKER_NOT_RUNNING))
8901068 worker->pool->nr_running++;
8911069 preempt_enable();
892− worker->sleeping = 0;
1070+
1071+ /*
1072+ * CPU intensive auto-detection cares about how long a work item hogged
1073+ * CPU without sleeping. Reset the starting timestamp on wakeup.
1074+ */
1075+ worker->current_at = worker->task->se.sum_exec_runtime;
1076+
1077+ WRITE_ONCE(worker->sleeping, 0);
8931078 }
8941079
8951080 /**
@@ -915,10 +1100,10 @@ void wq_worker_sleeping(struct task_struct *task) · 19 unmodified lines
9151100 pool = worker->pool;
9161101
9171102 /* Return if preempted before wq_worker_running() was reached */
918− if (worker->sleeping)
1103+ if (READ_ONCE(worker->sleeping))
9191104 return;
9201105
921− worker->sleeping = 1;
1106+ WRITE_ONCE(worker->sleeping, 1);
9221107 raw_spin_lock_irq(&pool->lock);
9231108
9241109 /*
@@ -932,11 +1117,65 @@ void wq_worker_sleeping(struct task_struct *task) · 7 unmodified lines
9321117 }
9331118
9341119 pool->nr_running--;
935− if (need_more_worker(pool))
1120+ if (need_more_worker(pool)) {
1121+ worker->current_pwq->stats[PWQ_STAT_CM_WAKEUP]++;
9361122 wake_up_worker(pool);
1123+ }
9371124 raw_spin_unlock_irq(&pool->lock);
9381125 }
9391126
1127+/**
1128+ * wq_worker_tick - a scheduler tick occurred while a kworker is running
1129+ * @task: task currently running
1130+ *
1131+ * Called from scheduler_tick(). We're in the IRQ context and the current
1132+ * worker's fields which follow the 'K' locking rule can be accessed safely.
1133+ */
1134+void wq_worker_tick(struct task_struct *task)
1135+{
1136+ struct worker *worker = kthread_data(task);
1137+ struct pool_workqueue *pwq = worker->current_pwq;
1138+ struct worker_pool *pool = worker->pool;
1139+
1140+ if (!pwq)
1141+ return;
1142+
1143+ pwq->stats[PWQ_STAT_CPU_TIME] += TICK_USEC;
1144+
1145+ if (!wq_cpu_intensive_thresh_us)
1146+ return;
1147+
1148+ /*
1149+ * If the current worker is concurrency managed and hogged the CPU for
1150+ * longer than wq_cpu_intensive_thresh_us, it's automatically marked
1151+ * CPU_INTENSIVE to avoid stalling other concurrency-managed work items.
1152+ *
1153+ * Set @worker->sleeping means that @worker is in the process of
1154+ * switching out voluntarily and won't be contributing to
1155+ * @pool->nr_running until it wakes up. As wq_worker_sleeping() also
1156+ * decrements ->nr_running, setting CPU_INTENSIVE here can lead to
1157+ * double decrements. The task is releasing the CPU anyway. Let's skip.
1158+ * We probably want to make this prettier in the future.
1159+ */
1160+ if ((worker->flags & WORKER_NOT_RUNNING) || READ_ONCE(worker->sleeping) ||
1161+ worker->task->se.sum_exec_runtime - worker->current_at <
1162+ wq_cpu_intensive_thresh_us * NSEC_PER_USEC)
1163+ return;
1164+
1165+ raw_spin_lock(&pool->lock);
1166+
1167+ worker_set_flags(worker, WORKER_CPU_INTENSIVE);
1168+ wq_cpu_intensive_report(worker->current_func);
1169+ pwq->stats[PWQ_STAT_CPU_INTENSIVE]++;
1170+
1171+ if (need_more_worker(pool)) {
1172+ pwq->stats[PWQ_STAT_CM_WAKEUP]++;
1173+ wake_up_worker(pool);
1174+ }
1175+
1176+ raw_spin_unlock(&pool->lock);
1177+}
1178+
9401179 /**
9411180 * wq_worker_last_func - retrieve worker's last work function
9421181 * @task: Task to retrieve last work function of.
@@ -968,60 +1207,6 @@ work_func_t wq_worker_last_func(struct task_struct *task) · 25 unmodified lines
9681207 return worker->last_func;
9691208 }
9701209
971−/**
972− * worker_set_flags - set worker flags and adjust nr_running accordingly
973− * @worker: self
974− * @flags: flags to set
975− *
976− * Set @flags in @worker->flags and adjust nr_running accordingly.
977− *
978− * CONTEXT:
979− * raw_spin_lock_irq(pool->lock)
980− */
981−static inline void worker_set_flags(struct worker *worker, unsigned int flags)
982−{
983− struct worker_pool *pool = worker->pool;
984−
985− WARN_ON_ONCE(worker->task != current);
986−
987− /* If transitioning into NOT_RUNNING, adjust nr_running. */
988− if ((flags & WORKER_NOT_RUNNING) &&
989− !(worker->flags & WORKER_NOT_RUNNING)) {
990− pool->nr_running--;
991− }
992−
993− worker->flags |= flags;
994−}
995−
996−/**
997− * worker_clr_flags - clear worker flags and adjust nr_running accordingly
998− * @worker: self
999− * @flags: flags to clear
1000− *
1001− * Clear @flags in @worker->flags and adjust nr_running accordingly.
1002− *
1003− * CONTEXT:
1004− * raw_spin_lock_irq(pool->lock)
1005− */
1006−static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
1007−{
1008− struct worker_pool *pool = worker->pool;
1009− unsigned int oflags = worker->flags;
1010−
1011− WARN_ON_ONCE(worker->task != current);
1012−
1013− worker->flags &= ~flags;
1014−
1015− /*
1016− * If transitioning out of NOT_RUNNING, increment nr_running. Note
1017− * that the nested NOT_RUNNING is not a noop. NOT_RUNNING is mask
1018− * of multiple flags, not a single flag.
1019− */
1020− if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
1021− if (!(worker->flags & WORKER_NOT_RUNNING))
1022− pool->nr_running++;
1023−}
1024−
10251210 /**
10261211 * find_worker_executing_work - find worker which is executing a work
10271212 * @pool: pool of interest
@@ -1542,6 +1727,8 @@ out: · 514 unmodified lines
15421727 * We queue the work to a specific CPU, the caller must ensure it
15431728 * can't go away. Callers that fail to ensure that the specified
15441729 * CPU cannot go away will execute on a randomly chosen CPU.
1730+ * But note well that callers specifying a CPU that never has been
1731+ * online will get a splat.
15451732 *
15461733 * Return: %false if @work was already on a queue, %true otherwise.
15471734 */
@@ -2166,6 +2353,7 @@ static void send_mayday(struct work_struct *work) · 618 unmodified lines
21662353 get_pwq(pwq);
21672354 list_add_tail(&pwq->mayday_node, &wq->maydays);
21682355 wake_up_process(wq->rescuer->task);
2356+ pwq->stats[PWQ_STAT_MAYDAY]++;
21692357 }
21702358 }
21712359
@@ -2303,7 +2491,6 @@ __acquires(&pool->lock) · 131 unmodified lines
23032491 {
23042492 struct pool_workqueue *pwq = get_work_pwq(work);
23052493 struct worker_pool *pool = worker->pool;
2306− bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE;
23072494 unsigned long work_data;
23082495 struct worker *collision;
23092496 #ifdef CONFIG_LOCKDEP
@@ -2340,6 +2527,7 @@ __acquires(&pool->lock) · 30 unmodified lines
23402527 worker->current_work = work;
23412528 worker->current_func = work->func;
23422529 worker->current_pwq = pwq;
2530+ worker->current_at = worker->task->se.sum_exec_runtime;
23432531 work_data = *work_data_bits(work);
23442532 worker->current_color = get_work_color(work_data);
23452533
@@ -2357,7 +2545,7 @@ __acquires(&pool->lock) · 11 unmodified lines
23572545 * of concurrency management and the next code block will chain
23582546 * execution of the pending work items.
23592547 */
2360− if (unlikely(cpu_intensive))
2548+ if (unlikely(pwq->wq->flags & WQ_CPU_INTENSIVE))
23612549 worker_set_flags(worker, WORKER_CPU_INTENSIVE);
23622550
23632551 /*
@@ -2404,6 +2592,7 @@ __acquires(&pool->lock) · 40 unmodified lines
24042592 * workqueues), so hiding them isn't a problem.
24052593 */
24062594 lockdep_invariant_state(true);
2595+ pwq->stats[PWQ_STAT_STARTED]++;
24072596 trace_workqueue_execute_start(work);
24082597 worker->current_func(work);
24092598 /*
@@ -2411,6 +2600,7 @@ __acquires(&pool->lock) · 1 unmodified line
24112600 * point will only record its address.
24122601 */
24132602 trace_workqueue_execute_end(work, worker->current_func);
2603+ pwq->stats[PWQ_STAT_COMPLETED]++;
24142604 lock_map_release(&lockdep_map);
24152605 lock_map_release(&pwq->wq->lockdep_map);
24162606
@@ -2435,9 +2625,12 @@ __acquires(&pool->lock) · 18 unmodified lines
24352625
24362626 raw_spin_lock_irq(&pool->lock);
24372627
2438− /* clear cpu intensive status */
2439− if (unlikely(cpu_intensive))
2440− worker_clr_flags(worker, WORKER_CPU_INTENSIVE);
2628+ /*
2629+ * In addition to %WQ_CPU_INTENSIVE, @worker may also have been marked
2630+ * CPU intensive by wq_worker_tick() if @work hogged CPU longer than
2631+ * wq_cpu_intensive_thresh_us. Clear it.
2632+ */
2633+ worker_clr_flags(worker, WORKER_CPU_INTENSIVE);
24412634
24422635 /* tag the worker for identification in schedule() */
24432636 worker->last_func = worker->current_func;
@@ -2654,6 +2847,7 @@ repeat: · 210 unmodified lines
26542847 if (first)
26552848 pool->watchdog_ts = jiffies;
26562849 move_linked_works(work, scheduled, &n);
2850+ pwq->stats[PWQ_STAT_RESCUED]++;
26572851 }
26582852 first = false;
26592853 }