Merge tag 'wq-for-6.5' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq

Pull workqueue updates from Tejun Heo: - Concurrency-managed per-cpu work items that hog CPUs and delay the execution of other work items are now automatically detected and excluded from concurrency management. Reporting on such work items can also be enabled through a config option. - Added tools/workqueue/wq_monitor.py which improves visibility into workqueue usages and behaviors. - Arnd's minimal fix for gcc-13 enum warning on 32bit compiles, superseded by commit afa4bb778e48 in mainline. * tag 'wq-for-6.5' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq: workqueue: Disable per-cpu CPU hog detection when wq_cpu_intensive_thresh_us is 0 workqueue: Fix WARN_ON_ONCE() triggers in worker_enter_idle() workqueue: fix enum type for gcc-13 workqueue: Track and monitor per-workqueue CPU time usage workqueue: Report work funcs that trigger automatic CPU_INTENSIVE mechanism workqueue: Automatically mark CPU-hogging work items CPU_INTENSIVE workqueue: Improve locking rule description for worker fields workqueue: Move worker_set/clr_flags() upwards workqueue: Re-order struct worker fields workqueue: Add pwq->stats[] and a monitoring script Further upgrade queue_work_on() comment

Linus Torvalds committed 3 years ago

Commit 7ab044a2 parents 18eb3b6 18c8ae87 files · +499 −75Browse files

Unified · Split

7 files

Documentation/admin-guide/kernel-parameters.txt +12 −0
Documentation/core-api/workqueue.rst +32 −0
kernel/sched/core.c +3 −0
kernel/workqueue.c +258 −64
kernel/workqueue_internal.h +13 −11
lib/Kconfig.debug +13 −0
tools/workqueue/wq_monitor.py +168 −0

Documentation/admin-guide/kernel-parameters.txt+12−0

@@ -6972,6 +6972,18 @@ · 6971 unmodified lines
6972	6972	it can be updated at runtime by writing to the
6973	6973	corresponding sysfs file.
6974	6974
	6975	+ workqueue.cpu_intensive_thresh_us=
	6976	+ Per-cpu work items which run for longer than this
	6977	+ threshold are automatically considered CPU intensive
	6978	+ and excluded from concurrency management to prevent
	6979	+ them from noticeably delaying other per-cpu work
	6980	+ items. Default is 10000 (10ms).
	6981	+
	6982	+ If CONFIG_WQ_CPU_INTENSIVE_REPORT is set, the kernel
	6983	+ will report the work functions which violate this
	6984	+ threshold repeatedly. They are likely good
	6985	+ candidates for using WQ_UNBOUND workqueues instead.
	6986	+
6975	6987	workqueue.disable_numa
6976	6988	By default, all work items queued to unbound
6977	6989	workqueues are affine to the NUMA nodes they're

Documentation/core-api/workqueue.rst+32−0

@@ -348,6 +348,37 @@ Guidelines · 347 unmodified lines
348	348	level of locality in wq operations and work item execution.
349	349
350	350
	351	+Monitoring
	352	+==========
	353	+
	354	+Use tools/workqueue/wq_monitor.py to monitor workqueue operations: ::
	355	+
	356	+ $ tools/workqueue/wq_monitor.py events
	357	+ total infl CPUtime CPUhog CMwake mayday rescued
	358	+ events 18545 0 6.1 0 5 - -
	359	+ events_highpri 8 0 0.0 0 0 - -
	360	+ events_long 3 0 0.0 0 0 - -
	361	+ events_unbound 38306 0 0.1 - - - -
	362	+ events_freezable 0 0 0.0 0 0 - -
	363	+ events_power_efficient 29598 0 0.2 0 0 - -
	364	+ events_freezable_power_ 10 0 0.0 0 0 - -
	365	+ sock_diag_events 0 0 0.0 0 0 - -
	366	+
	367	+ total infl CPUtime CPUhog CMwake mayday rescued
	368	+ events 18548 0 6.1 0 5 - -
	369	+ events_highpri 8 0 0.0 0 0 - -
	370	+ events_long 3 0 0.0 0 0 - -
	371	+ events_unbound 38322 0 0.1 - - - -
	372	+ events_freezable 0 0 0.0 0 0 - -
	373	+ events_power_efficient 29603 0 0.2 0 0 - -
	374	+ events_freezable_power_ 10 0 0.0 0 0 - -
	375	+ sock_diag_events 0 0 0.0 0 0 - -
	376	+
	377	+ ...
	378	+
	379	+See the command's help message for more info.
	380	+
	381	+
351	382	Debugging
352	383	=========
353	384
@@ -387,6 +418,7 @@ the stack trace of the offending worker thread. :: · 33 unmodified lines
387	418	The work item's function should be trivially visible in the stack
388	419	trace.
389	420
	421	+
390	422	Non-reentrance Conditions
391	423	=========================
392	424

kernel/sched/core.c+3−0

@@ -5670,6 +5670,9 @@ void scheduler_tick(void) · 5669 unmodified lines
5670	5670
5671	5671	perf_event_task_tick();
5672	5672
	5673	+ if (curr->flags & PF_WQ_WORKER)
	5674	+ wq_worker_tick(curr);
	5675	+
5673	5676	#ifdef CONFIG_SMP
5674	5677	rq->idle_balance = idle_cpu(cpu);
5675	5678	trigger_load_balance(rq);

kernel/workqueue.c+258−64

@@ -126,6 +126,12 @@ enum { · 125 unmodified lines
126	126	* cpu or grabbing pool->lock is enough for read access. If
127	127	* POOL_DISASSOCIATED is set, it's identical to L.
128	128	*
	129	+ * K: Only modified by worker while holding pool->lock. Can be safely read by
	130	+ * self, while holding pool->lock or from IRQ context if %current is the
	131	+ * kworker.
	132	+ *
	133	+ * S: Only modified by worker self.
	134	+ *
129	135	* A: wq_pool_attach_mutex protected.
130	136	*
131	137	* PL: wq_pool_mutex protected.
@@ -199,6 +205,22 @@ struct worker_pool { · 67 unmodified lines
199	205	struct rcu_head rcu;
200	206	};
201	207
	208	+/*
	209	+ * Per-pool_workqueue statistics. These can be monitored using
	210	+ * tools/workqueue/wq_monitor.py.
	211	+ */
	212	+enum pool_workqueue_stats {
	213	+ PWQ_STAT_STARTED, /* work items started execution */
	214	+ PWQ_STAT_COMPLETED, /* work items completed execution */
	215	+ PWQ_STAT_CPU_TIME, /* total CPU time consumed */
	216	+ PWQ_STAT_CPU_INTENSIVE, /* wq_cpu_intensive_thresh_us violations */
	217	+ PWQ_STAT_CM_WAKEUP, /* concurrency-management worker wakeups */
	218	+ PWQ_STAT_MAYDAY, /* maydays to rescuer */
	219	+ PWQ_STAT_RESCUED, /* linked work items executed by rescuer */
	220	+
	221	+ PWQ_NR_STATS,
	222	+};
	223	+
202	224	/*
203	225	* The per-pool workqueue. While queued, the lower WORK_STRUCT_FLAG_BITS
204	226	* of work_struct->data are used for flags and the remaining high bits
@@ -236,6 +258,8 @@ struct pool_workqueue { · 31 unmodified lines
236	258	struct list_head pwqs_node; /* WR: node on wq->pwqs */
237	259	struct list_head mayday_node; /* MD: node on wq->maydays */
238	260
	261	+ u64 stats[PWQ_NR_STATS];
	262	+
239	263	/*
240	264	* Release of unbound pwq is punted to system_wq. See put_pwq()
241	265	* and pwq_unbound_release_workfn() for details. pool_workqueue
@@ -310,6 +334,14 @@ static struct kmem_cache *pwq_cache; · 68 unmodified lines
310	334	static cpumask_var_t *wq_numa_possible_cpumask;
311	335	/* possible CPUs of each node */
312	336
	337	+/*
	338	+ * Per-cpu work items which run for longer than the following threshold are
	339	+ * automatically considered CPU intensive and excluded from concurrency
	340	+ * management to prevent them from noticeably delaying other per-cpu work items.
	341	+ */
	342	+static unsigned long wq_cpu_intensive_thresh_us = 10000;
	343	+module_param_named(cpu_intensive_thresh_us, wq_cpu_intensive_thresh_us, ulong, 0644);
	344	+
313	345	static bool wq_disable_numa;
314	346	module_param_named(disable_numa, wq_disable_numa, bool, 0444);
315	347
@@ -866,6 +898,152 @@ static void wake_up_worker(struct worker_pool *pool) · 550 unmodified lines
866	898	wake_up_process(worker->task);
867	899	}
868	900
	901	+/**
	902	+ * worker_set_flags - set worker flags and adjust nr_running accordingly
	903	+ * @worker: self
	904	+ * @flags: flags to set
	905	+ *
	906	+ * Set @flags in @worker->flags and adjust nr_running accordingly.
	907	+ *
	908	+ * CONTEXT:
	909	+ * raw_spin_lock_irq(pool->lock)
	910	+ */
	911	+static inline void worker_set_flags(struct worker *worker, unsigned int flags)
	912	+{
	913	+ struct worker_pool *pool = worker->pool;
	914	+
	915	+ WARN_ON_ONCE(worker->task != current);
	916	+
	917	+ /* If transitioning into NOT_RUNNING, adjust nr_running. */
	918	+ if ((flags & WORKER_NOT_RUNNING) &&
	919	+ !(worker->flags & WORKER_NOT_RUNNING)) {
	920	+ pool->nr_running--;
	921	+ }
	922	+
	923	+ worker->flags \|= flags;
	924	+}
	925	+
	926	+/**
	927	+ * worker_clr_flags - clear worker flags and adjust nr_running accordingly
	928	+ * @worker: self
	929	+ * @flags: flags to clear
	930	+ *
	931	+ * Clear @flags in @worker->flags and adjust nr_running accordingly.
	932	+ *
	933	+ * CONTEXT:
	934	+ * raw_spin_lock_irq(pool->lock)
	935	+ */
	936	+static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
	937	+{
	938	+ struct worker_pool *pool = worker->pool;
	939	+ unsigned int oflags = worker->flags;
	940	+
	941	+ WARN_ON_ONCE(worker->task != current);
	942	+
	943	+ worker->flags &= ~flags;
	944	+
	945	+ /*
	946	+ * If transitioning out of NOT_RUNNING, increment nr_running. Note
	947	+ * that the nested NOT_RUNNING is not a noop. NOT_RUNNING is mask
	948	+ * of multiple flags, not a single flag.
	949	+ */
	950	+ if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
	951	+ if (!(worker->flags & WORKER_NOT_RUNNING))
	952	+ pool->nr_running++;
	953	+}
	954	+
	955	+#ifdef CONFIG_WQ_CPU_INTENSIVE_REPORT
	956	+
	957	+/*
	958	+ * Concurrency-managed per-cpu work items that hog CPU for longer than
	959	+ * wq_cpu_intensive_thresh_us trigger the automatic CPU_INTENSIVE mechanism,
	960	+ * which prevents them from stalling other concurrency-managed work items. If a
	961	+ * work function keeps triggering this mechanism, it's likely that the work item
	962	+ * should be using an unbound workqueue instead.
	963	+ *
	964	+ * wq_cpu_intensive_report() tracks work functions which trigger such conditions
	965	+ * and report them so that they can be examined and converted to use unbound
	966	+ * workqueues as appropriate. To avoid flooding the console, each violating work
	967	+ * function is tracked and reported with exponential backoff.
	968	+ */
	969	+#define WCI_MAX_ENTS 128
	970	+
	971	+struct wci_ent {
	972	+ work_func_t func;
	973	+ atomic64_t cnt;
	974	+ struct hlist_node hash_node;
	975	+};
	976	+
	977	+static struct wci_ent wci_ents[WCI_MAX_ENTS];
	978	+static int wci_nr_ents;
	979	+static DEFINE_RAW_SPINLOCK(wci_lock);
	980	+static DEFINE_HASHTABLE(wci_hash, ilog2(WCI_MAX_ENTS));
	981	+
	982	+static struct wci_ent *wci_find_ent(work_func_t func)
	983	+{
	984	+ struct wci_ent *ent;
	985	+
	986	+ hash_for_each_possible_rcu(wci_hash, ent, hash_node,
	987	+ (unsigned long)func) {
	988	+ if (ent->func == func)
	989	+ return ent;
	990	+ }
	991	+ return NULL;
	992	+}
	993	+
	994	+static void wq_cpu_intensive_report(work_func_t func)
	995	+{
	996	+ struct wci_ent *ent;
	997	+
	998	+restart:
	999	+ ent = wci_find_ent(func);
	1000	+ if (ent) {
	1001	+ u64 cnt;
	1002	+
	1003	+ /*
	1004	+ * Start reporting from the fourth time and back off
	1005	+ * exponentially.
	1006	+ */
	1007	+ cnt = atomic64_inc_return_relaxed(&ent->cnt);
	1008	+ if (cnt >= 4 && is_power_of_2(cnt))
	1009	+ printk_deferred(KERN_WARNING "workqueue: %ps hogged CPU for >%luus %llu times, consider switching to WQ_UNBOUND\n",
	1010	+ ent->func, wq_cpu_intensive_thresh_us,
	1011	+ atomic64_read(&ent->cnt));
	1012	+ return;
	1013	+ }
	1014	+
	1015	+ /*
	1016	+ * @func is a new violation. Allocate a new entry for it. If wcn_ents[]
	1017	+ * is exhausted, something went really wrong and we probably made enough
	1018	+ * noise already.
	1019	+ */
	1020	+ if (wci_nr_ents >= WCI_MAX_ENTS)
	1021	+ return;
	1022	+
	1023	+ raw_spin_lock(&wci_lock);
	1024	+
	1025	+ if (wci_nr_ents >= WCI_MAX_ENTS) {
	1026	+ raw_spin_unlock(&wci_lock);
	1027	+ return;
	1028	+ }
	1029	+
	1030	+ if (wci_find_ent(func)) {
	1031	+ raw_spin_unlock(&wci_lock);
	1032	+ goto restart;
	1033	+ }
	1034	+
	1035	+ ent = &wci_ents[wci_nr_ents++];
	1036	+ ent->func = func;
	1037	+ atomic64_set(&ent->cnt, 1);
	1038	+ hash_add_rcu(wci_hash, &ent->hash_node, (unsigned long)func);
	1039	+
	1040	+ raw_spin_unlock(&wci_lock);
	1041	+}
	1042	+
	1043	+#else /* CONFIG_WQ_CPU_INTENSIVE_REPORT */
	1044	+static void wq_cpu_intensive_report(work_func_t func) {}
	1045	+#endif /* CONFIG_WQ_CPU_INTENSIVE_REPORT */
	1046	+
869	1047	/**
870	1048	* wq_worker_running - a worker is running again
871	1049	* @task: task waking up
@@ -876,7 +1054,7 @@ void wq_worker_running(struct task_struct *task) · 4 unmodified lines
876	1054	{
877	1055	struct worker *worker = kthread_data(task);
878	1056
879		− if (!worker->sleeping)
	1057	+ if (!READ_ONCE(worker->sleeping))
880	1058	return;
881	1059
882	1060	/*
@@ -889,7 +1067,14 @@ void wq_worker_running(struct task_struct *task) · 6 unmodified lines
889	1067	if (!(worker->flags & WORKER_NOT_RUNNING))
890	1068	worker->pool->nr_running++;
891	1069	preempt_enable();
892		− worker->sleeping = 0;
	1070	+
	1071	+ /*
	1072	+ * CPU intensive auto-detection cares about how long a work item hogged
	1073	+ * CPU without sleeping. Reset the starting timestamp on wakeup.
	1074	+ */
	1075	+ worker->current_at = worker->task->se.sum_exec_runtime;
	1076	+
	1077	+ WRITE_ONCE(worker->sleeping, 0);
893	1078	}
894	1079
895	1080	/**
@@ -915,10 +1100,10 @@ void wq_worker_sleeping(struct task_struct *task) · 19 unmodified lines
915	1100	pool = worker->pool;
916	1101
917	1102	/* Return if preempted before wq_worker_running() was reached */
918		− if (worker->sleeping)
	1103	+ if (READ_ONCE(worker->sleeping))
919	1104	return;
920	1105
921		− worker->sleeping = 1;
	1106	+ WRITE_ONCE(worker->sleeping, 1);
922	1107	raw_spin_lock_irq(&pool->lock);
923	1108
924	1109	/*
@@ -932,11 +1117,65 @@ void wq_worker_sleeping(struct task_struct *task) · 7 unmodified lines
932	1117	}
933	1118
934	1119	pool->nr_running--;
935		− if (need_more_worker(pool))
	1120	+ if (need_more_worker(pool)) {
	1121	+ worker->current_pwq->stats[PWQ_STAT_CM_WAKEUP]++;
936	1122	wake_up_worker(pool);
	1123	+ }
937	1124	raw_spin_unlock_irq(&pool->lock);
938	1125	}
939	1126
	1127	+/**
	1128	+ * wq_worker_tick - a scheduler tick occurred while a kworker is running
	1129	+ * @task: task currently running
	1130	+ *
	1131	+ * Called from scheduler_tick(). We're in the IRQ context and the current
	1132	+ * worker's fields which follow the 'K' locking rule can be accessed safely.
	1133	+ */
	1134	+void wq_worker_tick(struct task_struct *task)
	1135	+{
	1136	+ struct worker *worker = kthread_data(task);
	1137	+ struct pool_workqueue *pwq = worker->current_pwq;
	1138	+ struct worker_pool *pool = worker->pool;
	1139	+
	1140	+ if (!pwq)
	1141	+ return;
	1142	+
	1143	+ pwq->stats[PWQ_STAT_CPU_TIME] += TICK_USEC;
	1144	+
	1145	+ if (!wq_cpu_intensive_thresh_us)
	1146	+ return;
	1147	+
	1148	+ /*
	1149	+ * If the current worker is concurrency managed and hogged the CPU for
	1150	+ * longer than wq_cpu_intensive_thresh_us, it's automatically marked
	1151	+ * CPU_INTENSIVE to avoid stalling other concurrency-managed work items.
	1152	+ *
	1153	+ * Set @worker->sleeping means that @worker is in the process of
	1154	+ * switching out voluntarily and won't be contributing to
	1155	+ * @pool->nr_running until it wakes up. As wq_worker_sleeping() also
	1156	+ * decrements ->nr_running, setting CPU_INTENSIVE here can lead to
	1157	+ * double decrements. The task is releasing the CPU anyway. Let's skip.
	1158	+ * We probably want to make this prettier in the future.
	1159	+ */
	1160	+ if ((worker->flags & WORKER_NOT_RUNNING) \|\| READ_ONCE(worker->sleeping) \|\|
	1161	+ worker->task->se.sum_exec_runtime - worker->current_at <
	1162	+ wq_cpu_intensive_thresh_us * NSEC_PER_USEC)
	1163	+ return;
	1164	+
	1165	+ raw_spin_lock(&pool->lock);
	1166	+
	1167	+ worker_set_flags(worker, WORKER_CPU_INTENSIVE);
	1168	+ wq_cpu_intensive_report(worker->current_func);
	1169	+ pwq->stats[PWQ_STAT_CPU_INTENSIVE]++;
	1170	+
	1171	+ if (need_more_worker(pool)) {
	1172	+ pwq->stats[PWQ_STAT_CM_WAKEUP]++;
	1173	+ wake_up_worker(pool);
	1174	+ }
	1175	+
	1176	+ raw_spin_unlock(&pool->lock);
	1177	+}
	1178	+
940	1179	/**
941	1180	* wq_worker_last_func - retrieve worker's last work function
942	1181	* @task: Task to retrieve last work function of.
@@ -968,60 +1207,6 @@ work_func_t wq_worker_last_func(struct task_struct *task) · 25 unmodified lines
968	1207	return worker->last_func;
969	1208	}
970	1209
971		−/**
972		− * worker_set_flags - set worker flags and adjust nr_running accordingly
973		− * @worker: self
974		− * @flags: flags to set
975		− *
976		− * Set @flags in @worker->flags and adjust nr_running accordingly.
977		− *
978		− * CONTEXT:
979		− * raw_spin_lock_irq(pool->lock)
980		− */
981		−static inline void worker_set_flags(struct worker *worker, unsigned int flags)
982		−{
983		− struct worker_pool *pool = worker->pool;
984		−
985		− WARN_ON_ONCE(worker->task != current);
986		−
987		− /* If transitioning into NOT_RUNNING, adjust nr_running. */
988		− if ((flags & WORKER_NOT_RUNNING) &&
989		− !(worker->flags & WORKER_NOT_RUNNING)) {
990		− pool->nr_running--;
991		− }
992		−
993		− worker->flags \|= flags;
994		−}
995		−
996		−/**
997		− * worker_clr_flags - clear worker flags and adjust nr_running accordingly
998		− * @worker: self
999		− * @flags: flags to clear
1000		− *
1001		− * Clear @flags in @worker->flags and adjust nr_running accordingly.
1002		− *
1003		− * CONTEXT:
1004		− * raw_spin_lock_irq(pool->lock)
1005		− */
1006		−static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
1007		−{
1008		− struct worker_pool *pool = worker->pool;
1009		− unsigned int oflags = worker->flags;
1010		−
1011		− WARN_ON_ONCE(worker->task != current);
1012		−
1013		− worker->flags &= ~flags;
1014		−
1015		− /*
1016		− * If transitioning out of NOT_RUNNING, increment nr_running. Note
1017		− * that the nested NOT_RUNNING is not a noop. NOT_RUNNING is mask
1018		− * of multiple flags, not a single flag.
1019		− */
1020		− if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
1021		− if (!(worker->flags & WORKER_NOT_RUNNING))
1022		− pool->nr_running++;
1023		−}
1024		−
1025	1210	/**
1026	1211	* find_worker_executing_work - find worker which is executing a work
1027	1212	* @pool: pool of interest
@@ -1542,6 +1727,8 @@ out: · 514 unmodified lines
1542	1727	* We queue the work to a specific CPU, the caller must ensure it
1543	1728	* can't go away. Callers that fail to ensure that the specified
1544	1729	* CPU cannot go away will execute on a randomly chosen CPU.
	1730	+ * But note well that callers specifying a CPU that never has been
	1731	+ * online will get a splat.
1545	1732	*
1546	1733	* Return: %false if @work was already on a queue, %true otherwise.
1547	1734	*/
@@ -2166,6 +2353,7 @@ static void send_mayday(struct work_struct *work) · 618 unmodified lines
2166	2353	get_pwq(pwq);
2167	2354	list_add_tail(&pwq->mayday_node, &wq->maydays);
2168	2355	wake_up_process(wq->rescuer->task);
	2356	+ pwq->stats[PWQ_STAT_MAYDAY]++;
2169	2357	}
2170	2358	}
2171	2359
@@ -2303,7 +2491,6 @@ __acquires(&pool->lock) · 131 unmodified lines
2303	2491	{
2304	2492	struct pool_workqueue *pwq = get_work_pwq(work);
2305	2493	struct worker_pool *pool = worker->pool;
2306		− bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE;
2307	2494	unsigned long work_data;
2308	2495	struct worker *collision;
2309	2496	#ifdef CONFIG_LOCKDEP
@@ -2340,6 +2527,7 @@ __acquires(&pool->lock) · 30 unmodified lines
2340	2527	worker->current_work = work;
2341	2528	worker->current_func = work->func;
2342	2529	worker->current_pwq = pwq;
	2530	+ worker->current_at = worker->task->se.sum_exec_runtime;
2343	2531	work_data = *work_data_bits(work);
2344	2532	worker->current_color = get_work_color(work_data);
2345	2533
@@ -2357,7 +2545,7 @@ __acquires(&pool->lock) · 11 unmodified lines
2357	2545	* of concurrency management and the next code block will chain
2358	2546	* execution of the pending work items.
2359	2547	*/
2360		− if (unlikely(cpu_intensive))
	2548	+ if (unlikely(pwq->wq->flags & WQ_CPU_INTENSIVE))
2361	2549	worker_set_flags(worker, WORKER_CPU_INTENSIVE);
2362	2550
2363	2551	/*
@@ -2404,6 +2592,7 @@ __acquires(&pool->lock) · 40 unmodified lines
2404	2592	* workqueues), so hiding them isn't a problem.
2405	2593	*/
2406	2594	lockdep_invariant_state(true);
	2595	+ pwq->stats[PWQ_STAT_STARTED]++;
2407	2596	trace_workqueue_execute_start(work);
2408	2597	worker->current_func(work);
2409	2598	/*
@@ -2411,6 +2600,7 @@ __acquires(&pool->lock) · 1 unmodified line
2411	2600	* point will only record its address.
2412	2601	*/
2413	2602	trace_workqueue_execute_end(work, worker->current_func);
	2603	+ pwq->stats[PWQ_STAT_COMPLETED]++;
2414	2604	lock_map_release(&lockdep_map);
2415	2605	lock_map_release(&pwq->wq->lockdep_map);
2416	2606
@@ -2435,9 +2625,12 @@ __acquires(&pool->lock) · 18 unmodified lines
2435	2625
2436	2626	raw_spin_lock_irq(&pool->lock);
2437	2627
2438		− /* clear cpu intensive status */
2439		− if (unlikely(cpu_intensive))
2440		− worker_clr_flags(worker, WORKER_CPU_INTENSIVE);
	2628	+ /*
	2629	+ * In addition to %WQ_CPU_INTENSIVE, @worker may also have been marked
	2630	+ * CPU intensive by wq_worker_tick() if @work hogged CPU longer than
	2631	+ * wq_cpu_intensive_thresh_us. Clear it.
	2632	+ */
	2633	+ worker_clr_flags(worker, WORKER_CPU_INTENSIVE);
2441	2634
2442	2635	/* tag the worker for identification in schedule() */
2443	2636	worker->last_func = worker->current_func;
@@ -2654,6 +2847,7 @@ repeat: · 210 unmodified lines
2654	2847	if (first)
2655	2848	pool->watchdog_ts = jiffies;
2656	2849	move_linked_works(work, scheduled, &n);
	2850	+ pwq->stats[PWQ_STAT_RESCUED]++;
2657	2851	}
2658	2852	first = false;
2659	2853	}

kernel/workqueue_internal.h+13−11

@@ -28,13 +28,18 @@ struct worker { · 27 unmodified lines
28	28	struct hlist_node hentry; /* L: while busy */
29	29	};
30	30
31		− struct work_struct current_work; / L: work being processed */
32		− work_func_t current_func; /* L: current_work's fn */
33		− struct pool_workqueue current_pwq; / L: current_work's pwq */
34		− unsigned int current_color; /* L: current_work's color */
35		− struct list_head scheduled; /* L: scheduled works */
	31	+ struct work_struct current_work; / K: work being processed and its */
	32	+ work_func_t current_func; /* K: function */
	33	+ struct pool_workqueue current_pwq; / K: pwq */
	34	+ u64 current_at; /* K: runtime at start or last wakeup */
	35	+ unsigned int current_color; /* K: color */
	36	+
	37	+ int sleeping; /* S: is worker sleeping? */
36	38
37		− /* 64 bytes boundary on 64bit, 32 on 32bit */
	39	+ /* used by the scheduler to determine a worker's last known identity */
	40	+ work_func_t last_func; /* K: last work's fn */
	41	+
	42	+ struct list_head scheduled; /* L: scheduled works */
38	43
39	44	struct task_struct task; / I: worker task */
40	45	struct worker_pool pool; / A: the associated pool */
@@ -42,10 +47,9 @@ struct worker { · 1 unmodified line
42	47	struct list_head node; /* A: anchored at pool->workers */
43	48	/* A: runs through worker->node */
44	49
45		− unsigned long last_active; /* L: last active timestamp */
	50	+ unsigned long last_active; /* K: last active timestamp */
46	51	unsigned int flags; /* X: flags */
47	52	int id; /* I: worker id */
48		− int sleeping; /* None */
49	53
50	54	/*
51	55	* Opaque string set with work_set_desc(). Printed out with task
@@ -55,9 +59,6 @@ struct worker { · 3 unmodified lines
55	59
56	60	/* used only by rescuers to point to the target workqueue */
57	61	struct workqueue_struct rescue_wq; / I: the workqueue to rescue */
58		−
59		− /* used by the scheduler to determine a worker's last known identity */
60		− work_func_t last_func;
61	62	};
62	63
63	64	/**
@@ -76,6 +77,7 @@ static inline struct worker *current_wq_worker(void) · 12 unmodified lines
76	77	*/
77	78	void wq_worker_running(struct task_struct *task);
78	79	void wq_worker_sleeping(struct task_struct *task);
	80	+void wq_worker_tick(struct task_struct *task);
79	81	work_func_t wq_worker_last_func(struct task_struct *task);
80	82
81	83	#endif /* _KERNEL_WORKQUEUE_INTERNAL_H */

lib/Kconfig.debug+13−0

@@ -1134,6 +1134,19 @@ config WQ_WATCHDOG · 1133 unmodified lines
1134	1134	state. This can be configured through kernel parameter
1135	1135	"workqueue.watchdog_thresh" and its sysfs counterpart.
1136	1136
	1137	+config WQ_CPU_INTENSIVE_REPORT
	1138	+ bool "Report per-cpu work items which hog CPU for too long"
	1139	+ depends on DEBUG_KERNEL
	1140	+ help
	1141	+ Say Y here to enable reporting of concurrency-managed per-cpu work
	1142	+ items that hog CPUs for longer than
	1143	+ workqueue.cpu_intensive_threshold_us. Workqueue automatically
	1144	+ detects and excludes them from concurrency management to prevent
	1145	+ them from stalling other per-cpu work items. Occassional
	1146	+ triggering may not necessarily indicate a problem. Repeated
	1147	+ triggering likely indicates that the work item should be switched
	1148	+ to use an unbound workqueue.
	1149	+
1137	1150	config TEST_LOCKUP
1138	1151	tristate "Test module to generate lockups"
1139	1152	depends on m

tools/workqueue/wq_monitor.py+168−0

@@ -0,0 +1,168 @@
	1	+#!/usr/bin/env drgn
	2	+#
	3	+# Copyright (C) 2023 Tejun Heo <tj@kernel.org>
	4	+# Copyright (C) 2023 Meta Platforms, Inc. and affiliates.
	5	+
	6	+desc = """
	7	+This is a drgn script to monitor workqueues. For more info on drgn, visit
	8	+https://github.com/osandov/drgn.
	9	+
	10	+ total Total number of work items executed by the workqueue.
	11	+
	12	+ infl The number of currently in-flight work items.
	13	+
	14	+ CPUtime Total CPU time consumed by the workqueue in seconds. This is
	15	+ sampled from scheduler ticks and only provides ballpark
	16	+ measurement. "nohz_full=" CPUs are excluded from measurement.
	17	+
	18	+ CPUitsv The number of times a concurrency-managed work item hogged CPU
	19	+ longer than the threshold (workqueue.cpu_intensive_thresh_us)
	20	+ and got excluded from concurrency management to avoid stalling
	21	+ other work items.
	22	+
	23	+ CMwake The number of concurrency-management wake-ups while executing a
	24	+ work item of the workqueue.
	25	+
	26	+ mayday The number of times the rescuer was requested while waiting for
	27	+ new worker creation.
	28	+
	29	+ rescued The number of work items executed by the rescuer.
	30	+"""
	31	+
	32	+import sys
	33	+import signal
	34	+import os
	35	+import re
	36	+import time
	37	+import json
	38	+
	39	+import drgn
	40	+from drgn.helpers.linux.list import list_for_each_entry,list_empty
	41	+from drgn.helpers.linux.cpumask import for_each_possible_cpu
	42	+
	43	+import argparse
	44	+parser = argparse.ArgumentParser(description=desc,
	45	+ formatter_class=argparse.RawTextHelpFormatter)
	46	+parser.add_argument('workqueue', metavar='REGEX', nargs='*',
	47	+ help='Target workqueue name patterns (all if empty)')
	48	+parser.add_argument('-i', '--interval', metavar='SECS', type=float, default=1,
	49	+ help='Monitoring interval (0 to print once and exit)')
	50	+parser.add_argument('-j', '--json', action='store_true',
	51	+ help='Output in json')
	52	+args = parser.parse_args()
	53	+
	54	+def err(s):
	55	+ print(s, file=sys.stderr, flush=True)
	56	+ sys.exit(1)
	57	+
	58	+workqueues = prog['workqueues']
	59	+
	60	+WQ_UNBOUND = prog['WQ_UNBOUND']
	61	+WQ_MEM_RECLAIM = prog['WQ_MEM_RECLAIM']
	62	+
	63	+PWQ_STAT_STARTED = prog['PWQ_STAT_STARTED'] # work items started execution
	64	+PWQ_STAT_COMPLETED = prog['PWQ_STAT_COMPLETED'] # work items completed execution
	65	+PWQ_STAT_CPU_TIME = prog['PWQ_STAT_CPU_TIME'] # total CPU time consumed
	66	+PWQ_STAT_CPU_INTENSIVE = prog['PWQ_STAT_CPU_INTENSIVE'] # wq_cpu_intensive_thresh_us violations
	67	+PWQ_STAT_CM_WAKEUP = prog['PWQ_STAT_CM_WAKEUP'] # concurrency-management worker wakeups
	68	+PWQ_STAT_MAYDAY = prog['PWQ_STAT_MAYDAY'] # maydays to rescuer
	69	+PWQ_STAT_RESCUED = prog['PWQ_STAT_RESCUED'] # linked work items executed by rescuer
	70	+PWQ_NR_STATS = prog['PWQ_NR_STATS']
	71	+
	72	+class WqStats:
	73	+ def __init__(self, wq):
	74	+ self.name = wq.name.string_().decode()
	75	+ self.unbound = wq.flags & WQ_UNBOUND != 0
	76	+ self.mem_reclaim = wq.flags & WQ_MEM_RECLAIM != 0
	77	+ self.stats = [0] * PWQ_NR_STATS
	78	+ for pwq in list_for_each_entry('struct pool_workqueue', wq.pwqs.address_of_(), 'pwqs_node'):
	79	+ for i in range(PWQ_NR_STATS):
	80	+ self.stats[i] += int(pwq.stats[i])
	81	+
	82	+ def dict(self, now):
	83	+ return { 'timestamp' : now,
	84	+ 'name' : self.name,
	85	+ 'unbound' : self.unbound,
	86	+ 'mem_reclaim' : self.mem_reclaim,
	87	+ 'started' : self.stats[PWQ_STAT_STARTED],
	88	+ 'completed' : self.stats[PWQ_STAT_COMPLETED],
	89	+ 'cpu_time' : self.stats[PWQ_STAT_CPU_TIME],
	90	+ 'cpu_intensive' : self.stats[PWQ_STAT_CPU_INTENSIVE],
	91	+ 'cm_wakeup' : self.stats[PWQ_STAT_CM_WAKEUP],
	92	+ 'mayday' : self.stats[PWQ_STAT_MAYDAY],
	93	+ 'rescued' : self.stats[PWQ_STAT_RESCUED], }
	94	+
	95	+ def table_header_str():
	96	+ return f'{"":>24} {"total":>8} {"infl":>5} {"CPUtime":>8} '\
	97	+ f'{"CPUitsv":>7} {"CMwake":>7} {"mayday":>7} {"rescued":>7}'
	98	+
	99	+ def table_row_str(self):
	100	+ cpu_intensive = '-'
	101	+ cm_wakeup = '-'
	102	+ mayday = '-'
	103	+ rescued = '-'
	104	+
	105	+ if not self.unbound:
	106	+ cpu_intensive = str(self.stats[PWQ_STAT_CPU_INTENSIVE])
	107	+ cm_wakeup = str(self.stats[PWQ_STAT_CM_WAKEUP])
	108	+
	109	+ if self.mem_reclaim:
	110	+ mayday = str(self.stats[PWQ_STAT_MAYDAY])
	111	+ rescued = str(self.stats[PWQ_STAT_RESCUED])
	112	+
	113	+ out = f'{self.name[-24:]:24} ' \
	114	+ f'{self.stats[PWQ_STAT_STARTED]:8} ' \
	115	+ f'{max(self.stats[PWQ_STAT_STARTED] - self.stats[PWQ_STAT_COMPLETED], 0):5} ' \
	116	+ f'{self.stats[PWQ_STAT_CPU_TIME] / 1000000:8.1f} ' \
	117	+ f'{cpu_intensive:>7} ' \
	118	+ f'{cm_wakeup:>7} ' \
	119	+ f'{mayday:>7} ' \
	120	+ f'{rescued:>7} '
	121	+ return out.rstrip(':')
	122	+
	123	+exit_req = False
	124	+
	125	+def sigint_handler(signr, frame):
	126	+ global exit_req
	127	+ exit_req = True
	128	+
	129	+def main():
	130	+ # handle args
	131	+ table_fmt = not args.json
	132	+ interval = args.interval
	133	+
	134	+ re_str = None
	135	+ if args.workqueue:
	136	+ for r in args.workqueue:
	137	+ if re_str is None:
	138	+ re_str = r
	139	+ else:
	140	+ re_str += '\|' + r
	141	+
	142	+ filter_re = re.compile(re_str) if re_str else None
	143	+
	144	+ # monitoring loop
	145	+ signal.signal(signal.SIGINT, sigint_handler)
	146	+
	147	+ while not exit_req:
	148	+ now = time.time()
	149	+
	150	+ if table_fmt:
	151	+ print()
	152	+ print(WqStats.table_header_str())
	153	+
	154	+ for wq in list_for_each_entry('struct workqueue_struct', workqueues.address_of_(), 'list'):
	155	+ stats = WqStats(wq)
	156	+ if filter_re and not filter_re.search(stats.name):
	157	+ continue
	158	+ if table_fmt:
	159	+ print(stats.table_row_str())
	160	+ else:
	161	+ print(stats.dict(now))
	162	+
	163	+ if interval == 0:
	164	+ break
	165	+ time.sleep(interval)
	166	+
	167	+if __name__ == "__main__":
	168	+ main()