Merge tag 'wq-for-6.5' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq

Pull workqueue updates from Tejun Heo: - Concurrency-managed per-cpu work items that hog CPUs and delay the execution of other work items are now automatically detected and excluded from concurrency management. Reporting on such work items can also be enabled through a config option. - Added tools/workqueue/wq_monitor.py which improves visibility into workqueue usages and behaviors. - Arnd's minimal fix for gcc-13 enum warning on 32bit compiles, superseded by commit afa4bb778e48 in mainline. * tag 'wq-for-6.5' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq: workqueue: Disable per-cpu CPU hog detection when wq_cpu_intensive_thresh_us is 0 workqueue: Fix WARN_ON_ONCE() triggers in worker_enter_idle() workqueue: fix enum type for gcc-13 workqueue: Track and monitor per-workqueue CPU time usage workqueue: Report work funcs that trigger automatic CPU_INTENSIVE mechanism workqueue: Automatically mark CPU-hogging work items CPU_INTENSIVE workqueue: Improve locking rule description for worker fields workqueue: Move worker_set/clr_flags() upwards workqueue: Re-order struct worker fields workqueue: Add pwq->stats[] and a monitoring script Further upgrade queue_work_on() comment

Linus Torvalds committed 3 years ago

Commit 7ab044a2 parents 18eb3b6 18c8ae87 files · +499 −75Browse files

Unified · Split

7 files

Documentation/admin-guide/kernel-parameters.txt +12 −0
Documentation/core-api/workqueue.rst +32 −0
kernel/sched/core.c +3 −0
kernel/workqueue.c +258 −64
kernel/workqueue_internal.h +13 −11
lib/Kconfig.debug +13 −0
tools/workqueue/wq_monitor.py +168 −0

Documentation/admin-guide/kernel-parameters.txt+12−0

@@ -6972,6 +6972,18 @@ · 6971 unmodified lines
6972	it can be updated at runtime by writing to the	6972	it can be updated at runtime by writing to the
6973	corresponding sysfs file.	6973	corresponding sysfs file.
6974		6974
		6975	workqueue.cpu_intensive_thresh_us=
		6976	Per-cpu work items which run for longer than this
		6977	threshold are automatically considered CPU intensive
		6978	and excluded from concurrency management to prevent
		6979	them from noticeably delaying other per-cpu work
		6980	items. Default is 10000 (10ms).
		6981
		6982	If CONFIG_WQ_CPU_INTENSIVE_REPORT is set, the kernel
		6983	will report the work functions which violate this
		6984	threshold repeatedly. They are likely good
		6985	candidates for using WQ_UNBOUND workqueues instead.
		6986
6975	workqueue.disable_numa	6987	workqueue.disable_numa
6976	By default, all work items queued to unbound	6988	By default, all work items queued to unbound
6977	workqueues are affine to the NUMA nodes they're	6989	workqueues are affine to the NUMA nodes they're

Documentation/core-api/workqueue.rst+32−0

@@ -348,6 +348,37 @@ Guidelines · 347 unmodified lines
348	level of locality in wq operations and work item execution.	348	level of locality in wq operations and work item execution.
349		349
350		350
		351	Monitoring
		352	==========
		353
		354	Use tools/workqueue/wq_monitor.py to monitor workqueue operations: ::
		355
		356	$ tools/workqueue/wq_monitor.py events
		357	total infl CPUtime CPUhog CMwake mayday rescued
		358	events 18545 0 6.1 0 5 - -
		359	events_highpri 8 0 0.0 0 0 - -
		360	events_long 3 0 0.0 0 0 - -
		361	events_unbound 38306 0 0.1 - - - -
		362	events_freezable 0 0 0.0 0 0 - -
		363	events_power_efficient 29598 0 0.2 0 0 - -
		364	events_freezable_power_ 10 0 0.0 0 0 - -
		365	sock_diag_events 0 0 0.0 0 0 - -
		366
		367	total infl CPUtime CPUhog CMwake mayday rescued
		368	events 18548 0 6.1 0 5 - -
		369	events_highpri 8 0 0.0 0 0 - -
		370	events_long 3 0 0.0 0 0 - -
		371	events_unbound 38322 0 0.1 - - - -
		372	events_freezable 0 0 0.0 0 0 - -
		373	events_power_efficient 29603 0 0.2 0 0 - -
		374	events_freezable_power_ 10 0 0.0 0 0 - -
		375	sock_diag_events 0 0 0.0 0 0 - -
		376
		377	...
		378
		379	See the command's help message for more info.
		380
		381
351	Debugging	382	Debugging
352	=========	383	=========
353		384
@@ -387,6 +418,7 @@ the stack trace of the offending worker thread. :: · 33 unmodified lines
387	The work item's function should be trivially visible in the stack	418	The work item's function should be trivially visible in the stack
388	trace.	419	trace.
389		420
		421
390	Non-reentrance Conditions	422	Non-reentrance Conditions
391	=========================	423	=========================
392		424

kernel/sched/core.c+3−0

@@ -5670,6 +5670,9 @@ void scheduler_tick(void) · 5669 unmodified lines
5670		5670
5671	perf_event_task_tick();	5671	perf_event_task_tick();
5672		5672
		5673	if (curr->flags & PF_WQ_WORKER)
		5674	wq_worker_tick(curr);
		5675
5673	#ifdef CONFIG_SMP	5676	#ifdef CONFIG_SMP
5674	rq->idle_balance = idle_cpu(cpu);	5677	rq->idle_balance = idle_cpu(cpu);
5675	trigger_load_balance(rq);	5678	trigger_load_balance(rq);

kernel/workqueue.c+258−64

@@ -126,6 +126,12 @@ enum { · 125 unmodified lines
126	* cpu or grabbing pool->lock is enough for read access. If	126	* cpu or grabbing pool->lock is enough for read access. If
127	* POOL_DISASSOCIATED is set, it's identical to L.	127	* POOL_DISASSOCIATED is set, it's identical to L.
128	*	128	*
		129	* K: Only modified by worker while holding pool->lock. Can be safely read by
		130	* self, while holding pool->lock or from IRQ context if %current is the
		131	* kworker.
		132	*
		133	* S: Only modified by worker self.
		134	*
129	* A: wq_pool_attach_mutex protected.	135	* A: wq_pool_attach_mutex protected.
130	*	136	*
131	* PL: wq_pool_mutex protected.	137	* PL: wq_pool_mutex protected.
@@ -199,6 +205,22 @@ struct worker_pool { · 67 unmodified lines
199	struct rcu_head rcu;	205	struct rcu_head rcu;
200	};	206	};
201		207
		208	/*
		209	* Per-pool_workqueue statistics. These can be monitored using
		210	* tools/workqueue/wq_monitor.py.
		211	*/
		212	enum pool_workqueue_stats {
		213	PWQ_STAT_STARTED, /* work items started execution */
		214	PWQ_STAT_COMPLETED, /* work items completed execution */
		215	PWQ_STAT_CPU_TIME, /* total CPU time consumed */
		216	PWQ_STAT_CPU_INTENSIVE, /* wq_cpu_intensive_thresh_us violations */
		217	PWQ_STAT_CM_WAKEUP, /* concurrency-management worker wakeups */
		218	PWQ_STAT_MAYDAY, /* maydays to rescuer */
		219	PWQ_STAT_RESCUED, /* linked work items executed by rescuer */
		220
		221	PWQ_NR_STATS,
		222	};
		223
202	/*	224	/*
203	* The per-pool workqueue. While queued, the lower WORK_STRUCT_FLAG_BITS	225	* The per-pool workqueue. While queued, the lower WORK_STRUCT_FLAG_BITS
204	* of work_struct->data are used for flags and the remaining high bits	226	* of work_struct->data are used for flags and the remaining high bits
@@ -236,6 +258,8 @@ struct pool_workqueue { · 31 unmodified lines
236	struct list_head pwqs_node; /* WR: node on wq->pwqs */	258	struct list_head pwqs_node; /* WR: node on wq->pwqs */
237	struct list_head mayday_node; /* MD: node on wq->maydays */	259	struct list_head mayday_node; /* MD: node on wq->maydays */
238		260
		261	u64 stats[PWQ_NR_STATS];
		262
239	/*	263	/*
240	* Release of unbound pwq is punted to system_wq. See put_pwq()	264	* Release of unbound pwq is punted to system_wq. See put_pwq()
241	* and pwq_unbound_release_workfn() for details. pool_workqueue	265	* and pwq_unbound_release_workfn() for details. pool_workqueue
@@ -310,6 +334,14 @@ static struct kmem_cache *pwq_cache; · 68 unmodified lines
310	static cpumask_var_t *wq_numa_possible_cpumask;	334	static cpumask_var_t *wq_numa_possible_cpumask;
311	/* possible CPUs of each node */	335	/* possible CPUs of each node */
312		336
		337	/*
		338	* Per-cpu work items which run for longer than the following threshold are
		339	* automatically considered CPU intensive and excluded from concurrency
		340	* management to prevent them from noticeably delaying other per-cpu work items.
		341	*/
		342	static unsigned long wq_cpu_intensive_thresh_us = 10000;
		343	module_param_named(cpu_intensive_thresh_us, wq_cpu_intensive_thresh_us, ulong, 0644);
		344
313	static bool wq_disable_numa;	345	static bool wq_disable_numa;
314	module_param_named(disable_numa, wq_disable_numa, bool, 0444);	346	module_param_named(disable_numa, wq_disable_numa, bool, 0444);
315		347
@@ -866,6 +898,152 @@ static void wake_up_worker(struct worker_pool *pool) · 550 unmodified lines
866	wake_up_process(worker->task);	898	wake_up_process(worker->task);
867	}	899	}
868		900
		901	/**
		902	* worker_set_flags - set worker flags and adjust nr_running accordingly
		903	* @worker: self
		904	* @flags: flags to set
		905	*
		906	* Set @flags in @worker->flags and adjust nr_running accordingly.
		907	*
		908	* CONTEXT:
		909	* raw_spin_lock_irq(pool->lock)
		910	*/
		911	static inline void worker_set_flags(struct worker *worker, unsigned int flags)
		912	{
		913	struct worker_pool *pool = worker->pool;
		914
		915	WARN_ON_ONCE(worker->task != current);
		916
		917	/* If transitioning into NOT_RUNNING, adjust nr_running. */
		918	if ((flags & WORKER_NOT_RUNNING) &&
		919	!(worker->flags & WORKER_NOT_RUNNING)) {
		920	pool->nr_running--;
		921	}
		922
		923	worker->flags \|= flags;
		924	}
		925
		926	/**
		927	* worker_clr_flags - clear worker flags and adjust nr_running accordingly
		928	* @worker: self
		929	* @flags: flags to clear
		930	*
		931	* Clear @flags in @worker->flags and adjust nr_running accordingly.
		932	*
		933	* CONTEXT:
		934	* raw_spin_lock_irq(pool->lock)
		935	*/
		936	static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
		937	{
		938	struct worker_pool *pool = worker->pool;
		939	unsigned int oflags = worker->flags;
		940
		941	WARN_ON_ONCE(worker->task != current);
		942
		943	worker->flags &= ~flags;
		944
		945	/*
		946	* If transitioning out of NOT_RUNNING, increment nr_running. Note
		947	* that the nested NOT_RUNNING is not a noop. NOT_RUNNING is mask
		948	* of multiple flags, not a single flag.
		949	*/
		950	if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
		951	if (!(worker->flags & WORKER_NOT_RUNNING))
		952	pool->nr_running++;
		953	}
		954
		955	#ifdef CONFIG_WQ_CPU_INTENSIVE_REPORT
		956
		957	/*
		958	* Concurrency-managed per-cpu work items that hog CPU for longer than
		959	* wq_cpu_intensive_thresh_us trigger the automatic CPU_INTENSIVE mechanism,
		960	* which prevents them from stalling other concurrency-managed work items. If a
		961	* work function keeps triggering this mechanism, it's likely that the work item
		962	* should be using an unbound workqueue instead.
		963	*
		964	* wq_cpu_intensive_report() tracks work functions which trigger such conditions
		965	* and report them so that they can be examined and converted to use unbound
		966	* workqueues as appropriate. To avoid flooding the console, each violating work
		967	* function is tracked and reported with exponential backoff.
		968	*/
		969	#define WCI_MAX_ENTS 128
		970
		971	struct wci_ent {
		972	work_func_t func;
		973	atomic64_t cnt;
		974	struct hlist_node hash_node;
		975	};
		976
		977	static struct wci_ent wci_ents[WCI_MAX_ENTS];
		978	static int wci_nr_ents;
		979	static DEFINE_RAW_SPINLOCK(wci_lock);
		980	static DEFINE_HASHTABLE(wci_hash, ilog2(WCI_MAX_ENTS));
		981
		982	static struct wci_ent *wci_find_ent(work_func_t func)
		983	{
		984	struct wci_ent *ent;
		985
		986	hash_for_each_possible_rcu(wci_hash, ent, hash_node,
		987	(unsigned long)func) {
		988	if (ent->func == func)
		989	return ent;
		990	}
		991	return NULL;
		992	}
		993
		994	static void wq_cpu_intensive_report(work_func_t func)
		995	{
		996	struct wci_ent *ent;
		997
		998	restart:
		999	ent = wci_find_ent(func);
		1000	if (ent) {
		1001	u64 cnt;
		1002
		1003	/*
		1004	* Start reporting from the fourth time and back off
		1005	* exponentially.
		1006	*/
		1007	cnt = atomic64_inc_return_relaxed(&ent->cnt);
		1008	if (cnt >= 4 && is_power_of_2(cnt))
		1009	printk_deferred(KERN_WARNING "workqueue: %ps hogged CPU for >%luus %llu times, consider switching to WQ_UNBOUND\n",
		1010	ent->func, wq_cpu_intensive_thresh_us,
		1011	atomic64_read(&ent->cnt));
		1012	return;
		1013	}
		1014
		1015	/*
		1016	* @func is a new violation. Allocate a new entry for it. If wcn_ents[]
		1017	* is exhausted, something went really wrong and we probably made enough
		1018	* noise already.
		1019	*/
		1020	if (wci_nr_ents >= WCI_MAX_ENTS)
		1021	return;
		1022
		1023	raw_spin_lock(&wci_lock);
		1024
		1025	if (wci_nr_ents >= WCI_MAX_ENTS) {
		1026	raw_spin_unlock(&wci_lock);
		1027	return;
		1028	}
		1029
		1030	if (wci_find_ent(func)) {
		1031	raw_spin_unlock(&wci_lock);
		1032	goto restart;
		1033	}
		1034
		1035	ent = &wci_ents[wci_nr_ents++];
		1036	ent->func = func;
		1037	atomic64_set(&ent->cnt, 1);
		1038	hash_add_rcu(wci_hash, &ent->hash_node, (unsigned long)func);
		1039
		1040	raw_spin_unlock(&wci_lock);
		1041	}
		1042
		1043	#else /* CONFIG_WQ_CPU_INTENSIVE_REPORT */
		1044	static void wq_cpu_intensive_report(work_func_t func) {}
		1045	#endif /* CONFIG_WQ_CPU_INTENSIVE_REPORT */
		1046
869	/**	1047	/**
870	* wq_worker_running - a worker is running again	1048	* wq_worker_running - a worker is running again
871	* @task: task waking up	1049	* @task: task waking up
@@ -876,7 +1054,7 @@ void wq_worker_running(struct task_struct *task) · 4 unmodified lines
876	{	1054	{
877	struct worker *worker = kthread_data(task);	1055	struct worker *worker = kthread_data(task);
878		1056
879	if (!worker->sleeping)	1057	if (!READ_ONCE(worker->sleeping))
880	return;	1058	return;
881		1059
882	/*	1060	/*
@@ -889,7 +1067,14 @@ void wq_worker_running(struct task_struct *task) · 6 unmodified lines
889	if (!(worker->flags & WORKER_NOT_RUNNING))	1067	if (!(worker->flags & WORKER_NOT_RUNNING))
890	worker->pool->nr_running++;	1068	worker->pool->nr_running++;
891	preempt_enable();	1069	preempt_enable();
892	worker->sleeping = 0;	1070
		1071	/*
		1072	* CPU intensive auto-detection cares about how long a work item hogged
		1073	* CPU without sleeping. Reset the starting timestamp on wakeup.
		1074	*/
		1075	worker->current_at = worker->task->se.sum_exec_runtime;
		1076
		1077	WRITE_ONCE(worker->sleeping, 0);
893	}	1078	}
894		1079
895	/**	1080	/**
@@ -915,10 +1100,10 @@ void wq_worker_sleeping(struct task_struct *task) · 19 unmodified lines
915	pool = worker->pool;	1100	pool = worker->pool;
916		1101
917	/* Return if preempted before wq_worker_running() was reached */	1102	/* Return if preempted before wq_worker_running() was reached */
918	if (worker->sleeping)	1103	if (READ_ONCE(worker->sleeping))
919	return;	1104	return;
920		1105
921	worker->sleeping = 1;	1106	WRITE_ONCE(worker->sleeping, 1);
922	raw_spin_lock_irq(&pool->lock);	1107	raw_spin_lock_irq(&pool->lock);
923		1108
924	/*	1109	/*
@@ -932,11 +1117,65 @@ void wq_worker_sleeping(struct task_struct *task) · 7 unmodified lines
932	}	1117	}
933		1118
934	pool->nr_running--;	1119	pool->nr_running--;
935	if (need_more_worker(pool))	1120	if (need_more_worker(pool)) {
		1121	worker->current_pwq->stats[PWQ_STAT_CM_WAKEUP]++;
936	wake_up_worker(pool);	1122	wake_up_worker(pool);
		1123	}
937	raw_spin_unlock_irq(&pool->lock);	1124	raw_spin_unlock_irq(&pool->lock);
938	}	1125	}
939		1126
		1127	/**
		1128	* wq_worker_tick - a scheduler tick occurred while a kworker is running
		1129	* @task: task currently running
		1130	*
		1131	* Called from scheduler_tick(). We're in the IRQ context and the current
		1132	* worker's fields which follow the 'K' locking rule can be accessed safely.
		1133	*/
		1134	void wq_worker_tick(struct task_struct *task)
		1135	{
		1136	struct worker *worker = kthread_data(task);
		1137	struct pool_workqueue *pwq = worker->current_pwq;
		1138	struct worker_pool *pool = worker->pool;
		1139
		1140	if (!pwq)
		1141	return;
		1142
		1143	pwq->stats[PWQ_STAT_CPU_TIME] += TICK_USEC;
		1144
		1145	if (!wq_cpu_intensive_thresh_us)
		1146	return;
		1147
		1148	/*
		1149	* If the current worker is concurrency managed and hogged the CPU for
		1150	* longer than wq_cpu_intensive_thresh_us, it's automatically marked
		1151	* CPU_INTENSIVE to avoid stalling other concurrency-managed work items.
		1152	*
		1153	* Set @worker->sleeping means that @worker is in the process of
		1154	* switching out voluntarily and won't be contributing to
		1155	* @pool->nr_running until it wakes up. As wq_worker_sleeping() also
		1156	* decrements ->nr_running, setting CPU_INTENSIVE here can lead to
		1157	* double decrements. The task is releasing the CPU anyway. Let's skip.
		1158	* We probably want to make this prettier in the future.
		1159	*/
		1160	if ((worker->flags & WORKER_NOT_RUNNING) \|\| READ_ONCE(worker->sleeping) \|\|
		1161	worker->task->se.sum_exec_runtime - worker->current_at <
		1162	wq_cpu_intensive_thresh_us * NSEC_PER_USEC)
		1163	return;
		1164
		1165	raw_spin_lock(&pool->lock);
		1166
		1167	worker_set_flags(worker, WORKER_CPU_INTENSIVE);
		1168	wq_cpu_intensive_report(worker->current_func);
		1169	pwq->stats[PWQ_STAT_CPU_INTENSIVE]++;
		1170
		1171	if (need_more_worker(pool)) {
		1172	pwq->stats[PWQ_STAT_CM_WAKEUP]++;
		1173	wake_up_worker(pool);
		1174	}
		1175
		1176	raw_spin_unlock(&pool->lock);
		1177	}
		1178
940	/**	1179	/**
941	* wq_worker_last_func - retrieve worker's last work function	1180	* wq_worker_last_func - retrieve worker's last work function
942	* @task: Task to retrieve last work function of.	1181	* @task: Task to retrieve last work function of.
@@ -968,60 +1207,6 @@ work_func_t wq_worker_last_func(struct task_struct *task) · 25 unmodified lines
968	return worker->last_func;	1207	return worker->last_func;
969	}	1208	}
970		1209
971	/**
972	* worker_set_flags - set worker flags and adjust nr_running accordingly
973	* @worker: self
974	* @flags: flags to set
975	*
976	* Set @flags in @worker->flags and adjust nr_running accordingly.
977	*
978	* CONTEXT:
979	* raw_spin_lock_irq(pool->lock)
980	*/
981	static inline void worker_set_flags(struct worker *worker, unsigned int flags)
982	{
983	struct worker_pool *pool = worker->pool;
984
985	WARN_ON_ONCE(worker->task != current);
986
987	/* If transitioning into NOT_RUNNING, adjust nr_running. */
988	if ((flags & WORKER_NOT_RUNNING) &&
989	!(worker->flags & WORKER_NOT_RUNNING)) {
990	pool->nr_running--;
991	}
992
993	worker->flags \|= flags;
994	}
995
996	/**
997	* worker_clr_flags - clear worker flags and adjust nr_running accordingly
998	* @worker: self
999	* @flags: flags to clear
1000	*
1001	* Clear @flags in @worker->flags and adjust nr_running accordingly.
1002	*
1003	* CONTEXT:
1004	* raw_spin_lock_irq(pool->lock)
1005	*/
1006	static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
1007	{
1008	struct worker_pool *pool = worker->pool;
1009	unsigned int oflags = worker->flags;
1010
1011	WARN_ON_ONCE(worker->task != current);
1012
1013	worker->flags &= ~flags;
1014
1015	/*
1016	* If transitioning out of NOT_RUNNING, increment nr_running. Note
1017	* that the nested NOT_RUNNING is not a noop. NOT_RUNNING is mask
1018	* of multiple flags, not a single flag.
1019	*/
1020	if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
1021	if (!(worker->flags & WORKER_NOT_RUNNING))
1022	pool->nr_running++;
1023	}
1024
1025	/**	1210	/**
1026	* find_worker_executing_work - find worker which is executing a work	1211	* find_worker_executing_work - find worker which is executing a work
1027	* @pool: pool of interest	1212	* @pool: pool of interest
@@ -1542,6 +1727,8 @@ out: · 514 unmodified lines
1542	* We queue the work to a specific CPU, the caller must ensure it	1727	* We queue the work to a specific CPU, the caller must ensure it
1543	* can't go away. Callers that fail to ensure that the specified	1728	* can't go away. Callers that fail to ensure that the specified
1544	* CPU cannot go away will execute on a randomly chosen CPU.	1729	* CPU cannot go away will execute on a randomly chosen CPU.
		1730	* But note well that callers specifying a CPU that never has been
		1731	* online will get a splat.
1545	*	1732	*
1546	* Return: %false if @work was already on a queue, %true otherwise.	1733	* Return: %false if @work was already on a queue, %true otherwise.
1547	*/	1734	*/
@@ -2166,6 +2353,7 @@ static void send_mayday(struct work_struct *work) · 618 unmodified lines
2166	get_pwq(pwq);	2353	get_pwq(pwq);
2167	list_add_tail(&pwq->mayday_node, &wq->maydays);	2354	list_add_tail(&pwq->mayday_node, &wq->maydays);
2168	wake_up_process(wq->rescuer->task);	2355	wake_up_process(wq->rescuer->task);
		2356	pwq->stats[PWQ_STAT_MAYDAY]++;
2169	}	2357	}
2170	}	2358	}
2171		2359
@@ -2303,7 +2491,6 @@ __acquires(&pool->lock) · 131 unmodified lines
2303	{	2491	{
2304	struct pool_workqueue *pwq = get_work_pwq(work);	2492	struct pool_workqueue *pwq = get_work_pwq(work);
2305	struct worker_pool *pool = worker->pool;	2493	struct worker_pool *pool = worker->pool;
2306	bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE;
2307	unsigned long work_data;	2494	unsigned long work_data;
2308	struct worker *collision;	2495	struct worker *collision;
2309	#ifdef CONFIG_LOCKDEP	2496	#ifdef CONFIG_LOCKDEP
@@ -2340,6 +2527,7 @@ __acquires(&pool->lock) · 30 unmodified lines
2340	worker->current_work = work;	2527	worker->current_work = work;
2341	worker->current_func = work->func;	2528	worker->current_func = work->func;
2342	worker->current_pwq = pwq;	2529	worker->current_pwq = pwq;
		2530	worker->current_at = worker->task->se.sum_exec_runtime;
2343	work_data = *work_data_bits(work);	2531	work_data = *work_data_bits(work);
2344	worker->current_color = get_work_color(work_data);	2532	worker->current_color = get_work_color(work_data);
2345		2533
@@ -2357,7 +2545,7 @@ __acquires(&pool->lock) · 11 unmodified lines
2357	* of concurrency management and the next code block will chain	2545	* of concurrency management and the next code block will chain
2358	* execution of the pending work items.	2546	* execution of the pending work items.
2359	*/	2547	*/
2360	if (unlikely(cpu_intensive))	2548	if (unlikely(pwq->wq->flags & WQ_CPU_INTENSIVE))
2361	worker_set_flags(worker, WORKER_CPU_INTENSIVE);	2549	worker_set_flags(worker, WORKER_CPU_INTENSIVE);
2362		2550
2363	/*	2551	/*
@@ -2404,6 +2592,7 @@ __acquires(&pool->lock) · 40 unmodified lines
2404	* workqueues), so hiding them isn't a problem.	2592	* workqueues), so hiding them isn't a problem.
2405	*/	2593	*/
2406	lockdep_invariant_state(true);	2594	lockdep_invariant_state(true);
		2595	pwq->stats[PWQ_STAT_STARTED]++;
2407	trace_workqueue_execute_start(work);	2596	trace_workqueue_execute_start(work);
2408	worker->current_func(work);	2597	worker->current_func(work);
2409	/*	2598	/*
@@ -2411,6 +2600,7 @@ __acquires(&pool->lock) · 1 unmodified line
2411	* point will only record its address.	2600	* point will only record its address.
2412	*/	2601	*/
2413	trace_workqueue_execute_end(work, worker->current_func);	2602	trace_workqueue_execute_end(work, worker->current_func);
		2603	pwq->stats[PWQ_STAT_COMPLETED]++;
2414	lock_map_release(&lockdep_map);	2604	lock_map_release(&lockdep_map);
2415	lock_map_release(&pwq->wq->lockdep_map);	2605	lock_map_release(&pwq->wq->lockdep_map);
2416		2606
@@ -2435,9 +2625,12 @@ __acquires(&pool->lock) · 18 unmodified lines
2435		2625
2436	raw_spin_lock_irq(&pool->lock);	2626	raw_spin_lock_irq(&pool->lock);
2437		2627
2438	/* clear cpu intensive status */	2628	/*
2439	if (unlikely(cpu_intensive))	2629	* In addition to %WQ_CPU_INTENSIVE, @worker may also have been marked
2440	worker_clr_flags(worker, WORKER_CPU_INTENSIVE);	2630	* CPU intensive by wq_worker_tick() if @work hogged CPU longer than
		2631	* wq_cpu_intensive_thresh_us. Clear it.
		2632	*/
		2633	worker_clr_flags(worker, WORKER_CPU_INTENSIVE);
2441		2634
2442	/* tag the worker for identification in schedule() */	2635	/* tag the worker for identification in schedule() */
2443	worker->last_func = worker->current_func;	2636	worker->last_func = worker->current_func;
@@ -2654,6 +2847,7 @@ repeat: · 210 unmodified lines
2654	if (first)	2847	if (first)
2655	pool->watchdog_ts = jiffies;	2848	pool->watchdog_ts = jiffies;
2656	move_linked_works(work, scheduled, &n);	2849	move_linked_works(work, scheduled, &n);
		2850	pwq->stats[PWQ_STAT_RESCUED]++;
2657	}	2851	}
2658	first = false;	2852	first = false;
2659	}	2853	}

kernel/workqueue_internal.h+13−11

 		struct hlist_node	hentry;	/* L: while busy */
 	};
-	struct work_struct	*current_work;	/* L: work being processed */
+	struct work_struct	*current_work;	/* K: work being processed and its */
-	work_func_t		current_func;	/* L: current_work's fn */
+	work_func_t		current_func;	/* K: function */
-	struct pool_workqueue	*current_pwq;	/* L: current_work's pwq */
+	struct pool_workqueue	*current_pwq;	/* K: pwq */
-	unsigned int		current_color;	/* L: current_work's color */
+	u64			current_at;	/* K: runtime at start or last wakeup */
-	struct list_head	scheduled;	/* L: scheduled works */
+	unsigned int		current_color;	/* K: color */
+	int			sleeping;	/* S: is worker sleeping? */
-	/* 64 bytes boundary on 64bit, 32 on 32bit */
+	/* used by the scheduler to determine a worker's last known identity */
+	work_func_t		last_func;	/* K: last work's fn */
+	struct list_head	scheduled;	/* L: scheduled works */
 	struct task_struct	*task;		/* I: worker task */
 	struct worker_pool	*pool;		/* A: the associated pool */
 	struct list_head	node;		/* A: anchored at pool->workers */
 						/* A: runs through worker->node */
-	unsigned long		last_active;	/* L: last active timestamp */
+	unsigned long		last_active;	/* K: last active timestamp */
 	unsigned int		flags;		/* X: flags */
 	int			id;		/* I: worker id */
-	int			sleeping;	/* None */
 	/*
 	 * Opaque string set with work_set_desc().  Printed out with task
 	/* used only by rescuers to point to the target workqueue */
 	struct workqueue_struct	*rescue_wq;	/* I: the workqueue to rescue */
-	/* used by the scheduler to determine a worker's last known identity */
-	work_func_t		last_func;
 };
 /**
  */
 void wq_worker_running(struct task_struct *task);
 void wq_worker_sleeping(struct task_struct *task);
+void wq_worker_tick(struct task_struct *task);
 work_func_t wq_worker_last_func(struct task_struct *task);
 #endif /* _KERNEL_WORKQUEUE_INTERNAL_H */

lib/Kconfig.debug+13−0

@@ -1134,6 +1134,19 @@ config WQ_WATCHDOG · 1133 unmodified lines
1134	state. This can be configured through kernel parameter	1134	state. This can be configured through kernel parameter
1135	"workqueue.watchdog_thresh" and its sysfs counterpart.	1135	"workqueue.watchdog_thresh" and its sysfs counterpart.
1136		1136
		1137	config WQ_CPU_INTENSIVE_REPORT
		1138	bool "Report per-cpu work items which hog CPU for too long"
		1139	depends on DEBUG_KERNEL
		1140	help
		1141	Say Y here to enable reporting of concurrency-managed per-cpu work
		1142	items that hog CPUs for longer than
		1143	workqueue.cpu_intensive_threshold_us. Workqueue automatically
		1144	detects and excludes them from concurrency management to prevent
		1145	them from stalling other per-cpu work items. Occassional
		1146	triggering may not necessarily indicate a problem. Repeated
		1147	triggering likely indicates that the work item should be switched
		1148	to use an unbound workqueue.
		1149
1137	config TEST_LOCKUP	1150	config TEST_LOCKUP
1138	tristate "Test module to generate lockups"	1151	tristate "Test module to generate lockups"
1139	depends on m	1152	depends on m

tools/workqueue/wq_monitor.py+168−0

@@ -0,0 +1,168 @@
	1	#!/usr/bin/env drgn
	2	#
	3	# Copyright (C) 2023 Tejun Heo <tj@kernel.org>
	4	# Copyright (C) 2023 Meta Platforms, Inc. and affiliates.
	5
	6	desc = """
	7	This is a drgn script to monitor workqueues. For more info on drgn, visit
	8	https://github.com/osandov/drgn.
	9
	10	total Total number of work items executed by the workqueue.
	11
	12	infl The number of currently in-flight work items.
	13
	14	CPUtime Total CPU time consumed by the workqueue in seconds. This is
	15	sampled from scheduler ticks and only provides ballpark
	16	measurement. "nohz_full=" CPUs are excluded from measurement.
	17
	18	CPUitsv The number of times a concurrency-managed work item hogged CPU
	19	longer than the threshold (workqueue.cpu_intensive_thresh_us)
	20	and got excluded from concurrency management to avoid stalling
	21	other work items.
	22
	23	CMwake The number of concurrency-management wake-ups while executing a
	24	work item of the workqueue.
	25
	26	mayday The number of times the rescuer was requested while waiting for
	27	new worker creation.
	28
	29	rescued The number of work items executed by the rescuer.
	30	"""
	31
	32	import sys
	33	import signal
	34	import os
	35	import re
	36	import time
	37	import json
	38
	39	import drgn
	40	from drgn.helpers.linux.list import list_for_each_entry,list_empty
	41	from drgn.helpers.linux.cpumask import for_each_possible_cpu
	42
	43	import argparse
	44	parser = argparse.ArgumentParser(description=desc,
	45	formatter_class=argparse.RawTextHelpFormatter)
	46	parser.add_argument('workqueue', metavar='REGEX', nargs='*',
	47	help='Target workqueue name patterns (all if empty)')
	48	parser.add_argument('-i', '--interval', metavar='SECS', type=float, default=1,
	49	help='Monitoring interval (0 to print once and exit)')
	50	parser.add_argument('-j', '--json', action='store_true',
	51	help='Output in json')
	52	args = parser.parse_args()
	53
	54	def err(s):
	55	print(s, file=sys.stderr, flush=True)
	56	sys.exit(1)
	57
	58	workqueues = prog['workqueues']
	59
	60	WQ_UNBOUND = prog['WQ_UNBOUND']
	61	WQ_MEM_RECLAIM = prog['WQ_MEM_RECLAIM']
	62
	63	PWQ_STAT_STARTED = prog['PWQ_STAT_STARTED'] # work items started execution
	64	PWQ_STAT_COMPLETED = prog['PWQ_STAT_COMPLETED'] # work items completed execution
	65	PWQ_STAT_CPU_TIME = prog['PWQ_STAT_CPU_TIME'] # total CPU time consumed
	66	PWQ_STAT_CPU_INTENSIVE = prog['PWQ_STAT_CPU_INTENSIVE'] # wq_cpu_intensive_thresh_us violations
	67	PWQ_STAT_CM_WAKEUP = prog['PWQ_STAT_CM_WAKEUP'] # concurrency-management worker wakeups
	68	PWQ_STAT_MAYDAY = prog['PWQ_STAT_MAYDAY'] # maydays to rescuer
	69	PWQ_STAT_RESCUED = prog['PWQ_STAT_RESCUED'] # linked work items executed by rescuer
	70	PWQ_NR_STATS = prog['PWQ_NR_STATS']
	71
	72	class WqStats:
	73	def __init__(self, wq):
	74	self.name = wq.name.string_().decode()
	75	self.unbound = wq.flags & WQ_UNBOUND != 0
	76	self.mem_reclaim = wq.flags & WQ_MEM_RECLAIM != 0
	77	self.stats = [0] * PWQ_NR_STATS
	78	for pwq in list_for_each_entry('struct pool_workqueue', wq.pwqs.address_of_(), 'pwqs_node'):
	79	for i in range(PWQ_NR_STATS):
	80	self.stats[i] += int(pwq.stats[i])
	81
	82	def dict(self, now):
	83	return { 'timestamp' : now,
	84	'name' : self.name,
	85	'unbound' : self.unbound,
	86	'mem_reclaim' : self.mem_reclaim,
	87	'started' : self.stats[PWQ_STAT_STARTED],
	88	'completed' : self.stats[PWQ_STAT_COMPLETED],
	89	'cpu_time' : self.stats[PWQ_STAT_CPU_TIME],
	90	'cpu_intensive' : self.stats[PWQ_STAT_CPU_INTENSIVE],
	91	'cm_wakeup' : self.stats[PWQ_STAT_CM_WAKEUP],
	92	'mayday' : self.stats[PWQ_STAT_MAYDAY],
	93	'rescued' : self.stats[PWQ_STAT_RESCUED], }
	94
	95	def table_header_str():
	96	return f'{"":>24} {"total":>8} {"infl":>5} {"CPUtime":>8} '\
	97	f'{"CPUitsv":>7} {"CMwake":>7} {"mayday":>7} {"rescued":>7}'
	98
	99	def table_row_str(self):
	100	cpu_intensive = '-'
	101	cm_wakeup = '-'
	102	mayday = '-'
	103	rescued = '-'
	104
	105	if not self.unbound:
	106	cpu_intensive = str(self.stats[PWQ_STAT_CPU_INTENSIVE])
	107	cm_wakeup = str(self.stats[PWQ_STAT_CM_WAKEUP])
	108
	109	if self.mem_reclaim:
	110	mayday = str(self.stats[PWQ_STAT_MAYDAY])
	111	rescued = str(self.stats[PWQ_STAT_RESCUED])
	112
	113	out = f'{self.name[-24:]:24} ' \
	114	f'{self.stats[PWQ_STAT_STARTED]:8} ' \
	115	f'{max(self.stats[PWQ_STAT_STARTED] - self.stats[PWQ_STAT_COMPLETED], 0):5} ' \
	116	f'{self.stats[PWQ_STAT_CPU_TIME] / 1000000:8.1f} ' \
	117	f'{cpu_intensive:>7} ' \
	118	f'{cm_wakeup:>7} ' \
	119	f'{mayday:>7} ' \
	120	f'{rescued:>7} '
	121	return out.rstrip(':')
	122
	123	exit_req = False
	124
	125	def sigint_handler(signr, frame):
	126	global exit_req
	127	exit_req = True
	128
	129	def main():
	130	# handle args
	131	table_fmt = not args.json
	132	interval = args.interval
	133
	134	re_str = None
	135	if args.workqueue:
	136	for r in args.workqueue:
	137	if re_str is None:
	138	re_str = r
	139	else:
	140	re_str += '\|' + r
	141
	142	filter_re = re.compile(re_str) if re_str else None
	143
	144	# monitoring loop
	145	signal.signal(signal.SIGINT, sigint_handler)
	146
	147	while not exit_req:
	148	now = time.time()
	149
	150	if table_fmt:
	151	print()
	152	print(WqStats.table_header_str())
	153
	154	for wq in list_for_each_entry('struct workqueue_struct', workqueues.address_of_(), 'list'):
	155	stats = WqStats(wq)
	156	if filter_re and not filter_re.search(stats.name):
	157	continue
	158	if table_fmt:
	159	print(stats.table_row_str())
	160	else:
	161	print(stats.dict(now))
	162
	163	if interval == 0:
	164	break
	165	time.sleep(interval)
	166
	167	if __name__ == "__main__":
	168	main()