5816 lines · 159.4 KB · C
RawDownload
 * * This mechanism is controlled through the kernel parameter * "workqueue.watchdog_thresh" which can be updated at runtime through the * corresponding sysfs parameter file. */#ifdef CONFIG_WQ_WATCHDOG static unsigned long wq_watchdog_thresh = 30;static struct timer_list wq_watchdog_timer; static unsigned long wq_watchdog_touched = INITIAL_JIFFIES;static DEFINE_PER_CPU(unsigned long, wq_watchdog_touched_cpu) = INITIAL_JIFFIES; static void wq_watchdog_reset_touched(void){	int cpu; 	wq_watchdog_touched = jiffies;	for_each_possible_cpu(cpu)		per_cpu(wq_watchdog_touched_cpu, cpu) = jiffies;} static void wq_watchdog_timer_fn(struct timer_list *unused){	unsigned long thresh = READ_ONCE(wq_watchdog_thresh) * HZ;	bool lockup_detected = false;	struct worker_pool *pool;	int pi; 	if (!thresh)		return; 	rcu_read_lock(); 	for_each_pool(pool, pi) {		unsigned long pool_ts, touched, ts; 		if (list_empty(&pool->worklist))			continue; 		/* get the latest of pool and touched timestamps */		pool_ts = READ_ONCE(pool->watchdog_ts);		touched = READ_ONCE(wq_watchdog_touched); 		if (time_after(pool_ts, touched))			ts = pool_ts;		else			ts = touched; 		if (pool->cpu >= 0) {			unsigned long cpu_touched =				READ_ONCE(per_cpu(wq_watchdog_touched_cpu,						  pool->cpu));			if (time_after(cpu_touched, ts))				ts = cpu_touched;		} 		/* did we stall? */		if (time_after(jiffies, ts + thresh)) {			lockup_detected = true;			pr_emerg("BUG: workqueue lockup - pool");			pr_cont_pool_info(pool);			pr_cont(" stuck for %us!\n",				jiffies_to_msecs(jiffies - pool_ts) / 1000);		}	} 	rcu_read_unlock(); 	if (lockup_detected)		show_workqueue_state(); 	wq_watchdog_reset_touched();	mod_timer(&wq_watchdog_timer, jiffies + thresh);} notrace void wq_watchdog_touch(int cpu){	if (cpu >= 0)		per_cpu(wq_watchdog_touched_cpu, cpu) = jiffies;	else		wq_watchdog_touched = jiffies;} static void wq_watchdog_set_thresh(unsigned long thresh){	wq_watchdog_thresh = 0;	del_timer_sync(&wq_watchdog_timer); 	if (thresh) {		wq_watchdog_thresh = thresh;		wq_watchdog_reset_touched();		mod_timer(&wq_watchdog_timer, jiffies + thresh * HZ);	}} static int wq_watchdog_param_set_thresh(const char *val,					const struct kernel_param *kp){	unsigned long thresh;	int ret; 	ret = kstrtoul(val, 0, &thresh);	if (ret)		return ret; 	if (system_wq)		wq_watchdog_set_thresh(thresh);	else		wq_watchdog_thresh = thresh; 	return 0;} static const struct kernel_param_ops wq_watchdog_thresh_ops = {	.set	= wq_watchdog_param_set_thresh,	.get	= param_get_ulong,}; module_param_cb(watchdog_thresh, &wq_watchdog_thresh_ops, &wq_watchdog_thresh,		0644); static void wq_watchdog_init(void){	timer_setup(&wq_watchdog_timer, wq_watchdog_timer_fn, TIMER_DEFERRABLE);	wq_watchdog_set_thresh(wq_watchdog_thresh);} #else	/* CONFIG_WQ_WATCHDOG */ static inline void wq_watchdog_init(void) { } #endif	/* CONFIG_WQ_WATCHDOG */ static void __init wq_numa_init(void){	cpumask_var_t *tbl;	int node, cpu; 	if (num_possible_nodes() <= 1)		return; 	if (wq_disable_numa) {		pr_info("workqueue: NUMA affinity support disabled\n");		return;	} 	wq_update_unbound_numa_attrs_buf = alloc_workqueue_attrs(GFP_KERNEL);	BUG_ON(!wq_update_unbound_numa_attrs_buf); 	/*	 * We want masks of possible CPUs of each node which isn't readily	 * available.  Build one from cpu_to_node() which should have been	 * fully initialized by now.	 */	tbl = kcalloc(nr_node_ids, sizeof(tbl[0]), GFP_KERNEL);	BUG_ON(!tbl); 	for_each_node(node)		BUG_ON(!zalloc_cpumask_var_node(&tbl[node], GFP_KERNEL,				node_online(node) ? node : NUMA_NO_NODE)); 	for_each_possible_cpu(cpu) {		node = cpu_to_node(cpu);		if (WARN_ON(node == NUMA_NO_NODE)) {			pr_warn("workqueue: NUMA node mapping not available for cpu%d, disabling NUMA support\n", cpu);			/* happens iff arch is bonkers, let's just proceed */			return;		}		cpumask_set_cpu(cpu, tbl[node]);	} 	wq_numa_possible_cpumask = tbl;	wq_numa_enabled = true;} /** * workqueue_init_early - early init for workqueue subsystem * * This is the first half of two-staged workqueue subsystem initialization * and invoked as soon as the bare basics - memory allocation, cpumasks and * idr are up.  It sets up all the data structures and system workqueues * and allows early boot code to create workqueues and queue/cancel work * items.  Actual work item execution starts only after kthreads can be * created and scheduled right before early initcalls. */int __init workqueue_init_early(void){	int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL };	int hk_flags = HK_FLAG_DOMAIN | HK_FLAG_WQ;	int i, cpu; 	WARN_ON(__alignof__(struct pool_workqueue) < __alignof__(long long)); 	BUG_ON(!alloc_cpumask_var(&wq_unbound_cpumask, GFP_KERNEL));	cpumask_copy(wq_unbound_cpumask, housekeeping_cpumask(hk_flags)); 	pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC); 	/* initialize CPU pools */	for_each_possible_cpu(cpu) {		struct worker_pool *pool; 		i = 0;		for_each_cpu_worker_pool(pool, cpu) {			BUG_ON(init_worker_pool(pool));			pool->cpu = cpu;			cpumask_copy(pool->attrs->cpumask, cpumask_of(cpu));			pool->attrs->nice = std_nice[i++];			pool->node = cpu_to_node(cpu); 			/* alloc pool ID */			mutex_lock(&wq_pool_mutex);			BUG_ON(worker_pool_assign_id(pool));			mutex_unlock(&wq_pool_mutex);		}	} 	/* create default unbound and ordered wq attrs */	for (i = 0; i < NR_STD_WORKER_POOLS; i++) {		struct workqueue_attrs *attrs; 		BUG_ON(!(attrs = alloc_workqueue_attrs(GFP_KERNEL)));		attrs->nice = std_nice[i];		unbound_std_wq_attrs[i] = attrs; 		/*		 * An ordered wq should have only one pwq as ordering is		 * guaranteed by max_active which is enforced by pwqs.		 * Turn off NUMA so that dfl_pwq is used for all nodes.		 */		BUG_ON(!(attrs = alloc_workqueue_attrs(GFP_KERNEL)));		attrs->nice = std_nice[i];		attrs->no_numa = true;		ordered_wq_attrs[i] = attrs;	} 	system_wq = alloc_workqueue("events", 0, 0);	system_highpri_wq = alloc_workqueue("events_highpri", WQ_HIGHPRI, 0);	system_long_wq = alloc_workqueue("events_long", 0, 0);	system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND,					    WQ_UNBOUND_MAX_ACTIVE);	system_freezable_wq = alloc_workqueue("events_freezable",					      WQ_FREEZABLE, 0);	system_power_efficient_wq = alloc_workqueue("events_power_efficient",					      WQ_POWER_EFFICIENT, 0);	system_freezable_power_efficient_wq = alloc_workqueue("events_freezable_power_efficient",					      WQ_FREEZABLE | WQ_POWER_EFFICIENT,					      0);	BUG_ON(!system_wq || !system_highpri_wq || !system_long_wq ||	       !system_unbound_wq || !system_freezable_wq ||	       !system_power_efficient_wq ||	       !system_freezable_power_efficient_wq); 	return 0;} /** * workqueue_init - bring workqueue subsystem fully online * * This is the latter half of two-staged workqueue subsystem initialization * and invoked as soon as kthreads can be created and scheduled. * Workqueues have been created and work items queued on them, but there * are no kworkers executing the work items yet.  Populate the worker pools * with the initial workers and enable future kworker creations. */int __init workqueue_init(void){	struct workqueue_struct *wq;	struct worker_pool *pool;	int cpu, bkt; 	/*	 * It'd be simpler to initialize NUMA in workqueue_init_early() but	 * CPU to node mapping may not be available that early on some	 * archs such as power and arm64.  As per-cpu pools created	 * previously could be missing node hint and unbound pools NUMA	 * affinity, fix them up.	 *	 * Also, while iterating workqueues, create rescuers if requested.	 */	wq_numa_init(); 	mutex_lock(&wq_pool_mutex); 	for_each_possible_cpu(cpu) {		for_each_cpu_worker_pool(pool, cpu) {			pool->node = cpu_to_node(cpu);		}	} 	list_for_each_entry(wq, &workqueues, list) {		wq_update_unbound_numa(wq, smp_processor_id(), true);		WARN(init_rescuer(wq),		     "workqueue: failed to create early rescuer for %s",		     wq->name);	} 	mutex_unlock(&wq_pool_mutex); 	/* create the initial workers */	for_each_online_cpu(cpu) {		for_each_cpu_worker_pool(pool, cpu) {			pool->flags &= ~POOL_DISASSOCIATED;			BUG_ON(!create_worker(pool));		}	} 	hash_for_each(unbound_pool_hash, bkt, pool, hash_node)		BUG_ON(!create_worker(pool)); 	wq_online = true;	wq_watchdog_init(); 	return 0;}

Lines 5501–5816 of 5816. Previous lines

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t