/* * kernel/workqueue.c - generic async execution with shared worker pool * * Copyright (C) 2002		Ingo Molnar * *   Derived from the taskqueue/keventd code by: *     David Woodhouse <dwmw2@infradead.org> *     Andrew Morton *     Kai Petzke <wpp@marie.physik.tu-berlin.de> *     Theodore Ts'o <tytso@mit.edu> * * Made to use alloc_percpu by Christoph Lameter. * * Copyright (C) 2010		SUSE Linux Products GmbH * Copyright (C) 2010		Tejun Heo <tj@kernel.org> * * This is the generic async execution mechanism.  Work items as are * executed in process context.  The worker pool is shared and * automatically managed.  There are two worker pools for each CPU (one for * normal work items and the other for high priority ones) and some extra * pools for workqueues which are not bound to any specific CPU - the * number of these backing pools is dynamic. * * Please read Documentation/core-api/workqueue.rst for details. */ #include <linux/export.h>#include <linux/kernel.h>#include <linux/sched.h>#include <linux/init.h>#include <linux/signal.h>#include <linux/completion.h>#include <linux/workqueue.h>#include <linux/slab.h>#include <linux/cpu.h>#include <linux/notifier.h>#include <linux/kthread.h>#include <linux/hardirq.h>#include <linux/mempolicy.h>#include <linux/freezer.h>#include <linux/debug_locks.h>#include <linux/lockdep.h>#include <linux/idr.h>#include <linux/jhash.h>#include <linux/hashtable.h>#include <linux/rculist.h>#include <linux/nodemask.h>#include <linux/moduleparam.h>#include <linux/uaccess.h>#include <linux/sched/isolation.h>#include <linux/nmi.h> #include "workqueue_internal.h" enum {	/*	 * worker_pool flags	 *	 * A bound pool is either associated or disassociated with its CPU.	 * While associated (!DISASSOCIATED), all workers are bound to the	 * CPU and none has %WORKER_UNBOUND set and concurrency management	 * is in effect.	 *	 * While DISASSOCIATED, the cpu may be offline and all workers have	 * %WORKER_UNBOUND set and concurrency management disabled, and may	 * be executing on any CPU.  The pool behaves as an unbound one.	 *	 * Note that DISASSOCIATED should be flipped only while holding	 * wq_pool_attach_mutex to avoid changing binding state while	 * worker_attach_to_pool() is in progress.	 */	POOL_MANAGER_ACTIVE	= 1 << 0,	/* being managed */	POOL_DISASSOCIATED	= 1 << 2,	/* cpu can't serve workers */ 	/* worker flags */	WORKER_DIE		= 1 << 1,	/* die die die */	WORKER_IDLE		= 1 << 2,	/* is idle */	WORKER_PREP		= 1 << 3,	/* preparing to run works */	WORKER_CPU_INTENSIVE	= 1 << 6,	/* cpu intensive */	WORKER_UNBOUND		= 1 << 7,	/* worker is unbound */	WORKER_REBOUND		= 1 << 8,	/* worker was rebound */ 	WORKER_NOT_RUNNING	= WORKER_PREP | WORKER_CPU_INTENSIVE |				  WORKER_UNBOUND | WORKER_REBOUND, 	NR_STD_WORKER_POOLS	= 2,		/* # standard pools per cpu */ 	UNBOUND_POOL_HASH_ORDER	= 6,		/* hashed by pool->attrs */	BUSY_WORKER_HASH_ORDER	= 6,		/* 64 pointers */ 	MAX_IDLE_WORKERS_RATIO	= 4,		/* 1/4 of busy can be idle */	IDLE_WORKER_TIMEOUT	= 300 * HZ,	/* keep idle ones for 5 mins */ 	MAYDAY_INITIAL_TIMEOUT  = HZ / 100 >= 2 ? HZ / 100 : 2,						/* call for help after 10ms						   (min two ticks) */	MAYDAY_INTERVAL		= HZ / 10,	/* and then every 100ms */	CREATE_COOLDOWN		= HZ,		/* time to breath after fail */ 	/*	 * Rescue workers are used only on emergencies and shared by	 * all cpus.  Give MIN_NICE.	 */	RESCUER_NICE_LEVEL	= MIN_NICE,	HIGHPRI_NICE_LEVEL	= MIN_NICE, 	WQ_NAME_LEN		= 24,}; /* * Structure fields follow one of the following exclusion rules. * * I: Modifiable by initialization/destruction paths and read-only for *    everyone else. * * P: Preemption protected.  Disabling preemption is enough and should *    only be modified and accessed from the local cpu. * * L: pool->lock protected.  Access with pool->lock held. * * X: During normal operation, modification requires pool->lock and should *    be done only from local cpu.  Either disabling preemption on local *    cpu or grabbing pool->lock is enough for read access.  If *    POOL_DISASSOCIATED is set, it's identical to L. * * A: wq_pool_attach_mutex protected. * * PL: wq_pool_mutex protected. * * PR: wq_pool_mutex protected for writes.  Sched-RCU protected for reads. * * PW: wq_pool_mutex and wq->mutex protected for writes.  Either for reads. * * PWR: wq_pool_mutex and wq->mutex protected for writes.  Either or *      sched-RCU for reads. * * WQ: wq->mutex protected. * * WR: wq->mutex protected for writes.  Sched-RCU protected for reads. * * MD: wq_mayday_lock protected. */ /* struct worker is defined in workqueue_internal.h */ struct worker_pool {	spinlock_t		lock;		/* the pool lock */	int			cpu;		/* I: the associated cpu */	int			node;		/* I: the associated node ID */	int			id;		/* I: pool ID */	unsigned int		flags;		/* X: flags */ 	unsigned long		watchdog_ts;	/* L: watchdog timestamp */ 	struct list_head	worklist;	/* L: list of pending works */ 	int			nr_workers;	/* L: total number of workers */	int			nr_idle;	/* L: currently idle workers */ 	struct list_head	idle_list;	/* X: list of idle workers */	struct timer_list	idle_timer;	/* L: worker idle timeout */	struct timer_list	mayday_timer;	/* L: SOS timer for workers */ 	/* a workers is either on busy_hash or idle_list, or the manager */	DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER);						/* L: hash of busy workers */ 	struct worker		*manager;	/* L: purely informational */	struct list_head	workers;	/* A: attached workers */	struct completion	*detach_completion; /* all workers detached */ 	struct ida		worker_ida;	/* worker IDs for task name */ 	struct workqueue_attrs	*attrs;		/* I: worker attributes */	struct hlist_node	hash_node;	/* PL: unbound_pool_hash node */	int			refcnt;		/* PL: refcnt for unbound pools */ 	/*	 * The current concurrency level.  As it's likely to be accessed	 * from other CPUs during try_to_wake_up(), put it in a separate	 * cacheline.	 */	atomic_t		nr_running ____cacheline_aligned_in_smp; 	/*	 * Destruction of pool is sched-RCU protected to allow dereferences	 * from get_work_pool().	 */	struct rcu_head		rcu;} ____cacheline_aligned_in_smp; /* * The per-pool workqueue.  While queued, the lower WORK_STRUCT_FLAG_BITS * of work_struct->data are used for flags and the remaining high bits * point to the pwq; thus, pwqs need to be aligned at two's power of the * number of flag bits. */struct pool_workqueue {	struct worker_pool	*pool;		/* I: the associated pool */	struct workqueue_struct *wq;		/* I: the owning workqueue */	int			work_color;	/* L: current color */	int			flush_color;	/* L: flushing color */	int			refcnt;		/* L: reference count */	int			nr_in_flight[WORK_NR_COLORS];						/* L: nr of in_flight works */	int			nr_active;	/* L: nr of active works */	int			max_active;	/* L: max active works */	struct list_head	delayed_works;	/* L: delayed works */	struct list_head	pwqs_node;	/* WR: node on wq->pwqs */	struct list_head	mayday_node;	/* MD: node on wq->maydays */ 	/*	 * Release of unbound pwq is punted to system_wq.  See put_pwq()	 * and pwq_unbound_release_workfn() for details.  pool_workqueue	 * itself is also sched-RCU protected so that the first pwq can be	 * determined without grabbing wq->mutex.	 */	struct work_struct	unbound_release_work;	struct rcu_head		rcu;} __aligned(1 << WORK_STRUCT_FLAG_BITS); /* * Structure used to wait for workqueue flush. */struct wq_flusher {	struct list_head	list;		/* WQ: list of flushers */	int			flush_color;	/* WQ: flush color waiting for */	struct completion	done;		/* flush completion */}; struct wq_device; /* * The externally visible workqueue.  It relays the issued work items to * the appropriate worker_pool through its pool_workqueues. */struct workqueue_struct {	struct list_head	pwqs;		/* WR: all pwqs of this wq */	struct list_head	list;		/* PR: list of all workqueues */ 	struct mutex		mutex;		/* protects this wq */	int			work_color;	/* WQ: current work color */	int			flush_color;	/* WQ: current flush color */	atomic_t		nr_pwqs_to_flush; /* flush in progress */	struct wq_flusher	*first_flusher;	/* WQ: first flusher */	struct list_head	flusher_queue;	/* WQ: flush waiters */	struct list_head	flusher_overflow; /* WQ: flush overflow list */ 	struct list_head	maydays;	/* MD: pwqs requesting rescue */	struct worker		*rescuer;	/* I: rescue worker */ 	int			nr_drainers;	/* WQ: drain in progress */	int			saved_max_active; /* WQ: saved pwq max_active */ 	struct workqueue_attrs	*unbound_attrs;	/* PW: only for unbound wqs */	struct pool_workqueue	*dfl_pwq;	/* PW: only for unbound wqs */ #ifdef CONFIG_SYSFS	struct wq_device	*wq_dev;	/* I: for sysfs interface */#endif#ifdef CONFIG_LOCKDEP	char			*lock_name;	struct lock_class_key	key;	struct lockdep_map	lockdep_map;#endif	char			name[WQ_NAME_LEN]; /* I: workqueue name */ 	/*	 * Destruction of workqueue_struct is sched-RCU protected to allow	 * walking the workqueues list without grabbing wq_pool_mutex.	 * This is used to dump all workqueues from sysrq.	 */	struct rcu_head		rcu; 	/* hot fields used during command issue, aligned to cacheline */	unsigned int		flags ____cacheline_aligned; /* WQ: WQ_* flags */	struct pool_workqueue __percpu *cpu_pwqs; /* I: per-cpu pwqs */	struct pool_workqueue __rcu *numa_pwq_tbl[]; /* PWR: unbound pwqs indexed by node */}; static struct kmem_cache *pwq_cache; static cpumask_var_t *wq_numa_possible_cpumask;					/* possible CPUs of each node */ static bool wq_disable_numa;module_param_named(disable_numa, wq_disable_numa, bool, 0444); /* see the comment above the definition of WQ_POWER_EFFICIENT */static bool wq_power_efficient = IS_ENABLED(CONFIG_WQ_POWER_EFFICIENT_DEFAULT);module_param_named(power_efficient, wq_power_efficient, bool, 0444); static bool wq_online;			/* can kworkers be created yet? */ static bool wq_numa_enabled;		/* unbound NUMA affinity enabled */ /* buf for wq_update_unbound_numa_attrs(), protected by CPU hotplug exclusion */static struct workqueue_attrs *wq_update_unbound_numa_attrs_buf; static DEFINE_MUTEX(wq_pool_mutex);	/* protects pools and workqueues list */static DEFINE_MUTEX(wq_pool_attach_mutex); /* protects worker attach/detach */static DEFINE_SPINLOCK(wq_mayday_lock);	/* protects wq->maydays list */static DECLARE_WAIT_QUEUE_HEAD(wq_manager_wait); /* wait for manager to go away */ static LIST_HEAD(workqueues);		/* PR: list of all workqueues */static bool workqueue_freezing;		/* PL: have wqs started freezing? */ /* PL: allowable cpus for unbound wqs and work items */static cpumask_var_t wq_unbound_cpumask; /* CPU where unbound work was last round robin scheduled from this CPU */static DEFINE_PER_CPU(int, wq_rr_cpu_last); /* * Local execution of unbound work items is no longer guaranteed.  The * following always forces round-robin CPU selection on unbound work items * to uncover usages which depend on it. */#ifdef CONFIG_DEBUG_WQ_FORCE_RR_CPUstatic bool wq_debug_force_rr_cpu = true;#elsestatic bool wq_debug_force_rr_cpu = false;#endifmodule_param_named(debug_force_rr_cpu, wq_debug_force_rr_cpu, bool, 0644); /* the per-cpu worker pools */static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS], cpu_worker_pools); static DEFINE_IDR(worker_pool_idr);	/* PR: idr of all pools */ /* PL: hash of all unbound pools keyed by pool->attrs */static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER); /* I: attributes used when instantiating standard unbound pools on demand */static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS]; /* I: attributes used when instantiating ordered pools on demand */static struct workqueue_attrs *ordered_wq_attrs[NR_STD_WORKER_POOLS]; struct workqueue_struct *system_wq __read_mostly;EXPORT_SYMBOL(system_wq);struct workqueue_struct *system_highpri_wq __read_mostly;EXPORT_SYMBOL_GPL(system_highpri_wq);struct workqueue_struct *system_long_wq __read_mostly;EXPORT_SYMBOL_GPL(system_long_wq);struct workqueue_struct *system_unbound_wq __read_mostly;EXPORT_SYMBOL_GPL(system_unbound_wq);struct workqueue_struct *system_freezable_wq __read_mostly;EXPORT_SYMBOL_GPL(system_freezable_wq);struct workqueue_struct *system_power_efficient_wq __read_mostly;EXPORT_SYMBOL_GPL(system_power_efficient_wq);struct workqueue_struct *system_freezable_power_efficient_wq __read_mostly;EXPORT_SYMBOL_GPL(system_freezable_power_efficient_wq); static int worker_thread(void *__worker);static void workqueue_sysfs_unregister(struct workqueue_struct *wq); #define CREATE_TRACE_POINTS#include <trace/events/workqueue.h> #define assert_rcu_or_pool_mutex()					\	RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held() &&			\			 !lockdep_is_held(&wq_pool_mutex),		\			 "sched RCU or wq_pool_mutex should be held") #define assert_rcu_or_wq_mutex(wq)					\	RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held() &&			\			 !lockdep_is_held(&wq->mutex),			\			 "sched RCU or wq->mutex should be held") #define assert_rcu_or_wq_mutex_or_pool_mutex(wq)			\	RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held() &&			\			 !lockdep_is_held(&wq->mutex) &&		\			 !lockdep_is_held(&wq_pool_mutex),		\			 "sched RCU, wq->mutex or wq_pool_mutex should be held") #define for_each_cpu_worker_pool(pool, cpu)				\	for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0];		\	     (pool) < &per_cpu(cpu_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \	     (pool)++) /** * for_each_pool - iterate through all worker_pools in the system * @pool: iteration cursor * @pi: integer used for iteration * * This must be called either with wq_pool_mutex held or sched RCU read * locked.  If the pool needs to be used beyond the locking in effect, the * caller is responsible for guaranteeing that the pool stays online. * * The if/else clause exists only for the lockdep assertion and can be * ignored. */#define for_each_pool(pool, pi)						\	idr_for_each_entry(&worker_pool_idr, pool, pi)			\		if (({ assert_rcu_or_pool_mutex(); false; })) { }	\		else /** * for_each_pool_worker - iterate through all workers of a worker_pool * @worker: iteration cursor * @pool: worker_pool to iterate workers of * * This must be called with wq_pool_attach_mutex. * * The if/else clause exists only for the lockdep assertion and can be * ignored. */#define for_each_pool_worker(worker, pool)				\	list_for_each_entry((worker), &(pool)->workers, node)		\		if (({ lockdep_assert_held(&wq_pool_attach_mutex); false; })) { } \		else /** * for_each_pwq - iterate through all pool_workqueues of the specified workqueue * @pwq: iteration cursor * @wq: the target workqueue * * This must be called either with wq->mutex held or sched RCU read locked. * If the pwq needs to be used beyond the locking in effect, the caller is * responsible for guaranteeing that the pwq stays online. * * The if/else clause exists only for the lockdep assertion and can be * ignored. */#define for_each_pwq(pwq, wq)						\	list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node)		\		if (({ assert_rcu_or_wq_mutex(wq); false; })) { }	\		else #ifdef CONFIG_DEBUG_OBJECTS_WORK static struct debug_obj_descr work_debug_descr; static void *work_debug_hint(void *addr){	return ((struct work_struct *) addr)->func;} static bool work_is_static_object(void *addr){	struct work_struct *work = addr; 	return test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work));} /* * fixup_init is called when: * - an active object is initialized */static bool work_fixup_init(void *addr, enum debug_obj_state state){	struct work_struct *work = addr; 	switch (state) {	case ODEBUG_STATE_ACTIVE:		cancel_work_sync(work);		debug_object_init(work, &work_debug_descr);		return true;	default:		return false;	}} /* * fixup_free is called when: * - an active object is freed */static bool work_fixup_free(void *addr, enum debug_obj_state state){	struct work_struct *work = addr; 	switch (state) {	case ODEBUG_STATE_ACTIVE:		cancel_work_sync(work);		debug_object_free(work, &work_debug_descr);		return true;	default:		return false;	}} static struct debug_obj_descr work_debug_descr = {	.name		= "work_struct",	.debug_hint	= work_debug_hint,	.is_static_object = work_is_static_object,	.fixup_init	= work_fixup_init,	.fixup_free	= work_fixup_free,}; static inline void debug_work_activate(struct work_struct *work){	debug_object_activate(work, &work_debug_descr);} static inline void debug_work_deactivate(struct work_struct *work){	debug_object_deactivate(work, &work_debug_descr);} 

Lines 1–500 of 5982. Load more