/* * kernel/workqueue.c - generic async execution with shared worker pool * * Copyright (C) 2002		Ingo Molnar * *   Derived from the taskqueue/keventd code by: *     David Woodhouse <dwmw2@infradead.org> *     Andrew Morton *     Kai Petzke <wpp@marie.physik.tu-berlin.de> *     Theodore Ts'o <tytso@mit.edu> * * Made to use alloc_percpu by Christoph Lameter. * * Copyright (C) 2010		SUSE Linux Products GmbH * Copyright (C) 2010		Tejun Heo <tj@kernel.org> * * This is the generic async execution mechanism.  Work items as are * executed in process context.  The worker pool is shared and * automatically managed.  There is one worker pool for each CPU and * one extra for works which are better served by workers which are * not bound to any specific CPU. * * Please read Documentation/workqueue.txt for details. */ #include <linux/export.h>#include <linux/kernel.h>#include <linux/sched.h>#include <linux/init.h>#include <linux/signal.h>#include <linux/completion.h>#include <linux/workqueue.h>#include <linux/slab.h>#include <linux/cpu.h>#include <linux/notifier.h>#include <linux/kthread.h>#include <linux/hardirq.h>#include <linux/mempolicy.h>#include <linux/freezer.h>#include <linux/kallsyms.h>#include <linux/debug_locks.h>#include <linux/lockdep.h>#include <linux/idr.h> #include "workqueue_sched.h" enum {	/* global_cwq flags */	GCWQ_MANAGE_WORKERS	= 1 << 0,	/* need to manage workers */	GCWQ_MANAGING_WORKERS	= 1 << 1,	/* managing workers */	GCWQ_DISASSOCIATED	= 1 << 2,	/* cpu can't serve workers */	GCWQ_FREEZING		= 1 << 3,	/* freeze in progress */	GCWQ_HIGHPRI_PENDING	= 1 << 4,	/* highpri works on queue */ 	/* worker flags */	WORKER_STARTED		= 1 << 0,	/* started */	WORKER_DIE		= 1 << 1,	/* die die die */	WORKER_IDLE		= 1 << 2,	/* is idle */	WORKER_PREP		= 1 << 3,	/* preparing to run works */	WORKER_ROGUE		= 1 << 4,	/* not bound to any cpu */	WORKER_REBIND		= 1 << 5,	/* mom is home, come back */	WORKER_CPU_INTENSIVE	= 1 << 6,	/* cpu intensive */	WORKER_UNBOUND		= 1 << 7,	/* worker is unbound */ 	WORKER_NOT_RUNNING	= WORKER_PREP | WORKER_ROGUE | WORKER_REBIND |				  WORKER_CPU_INTENSIVE | WORKER_UNBOUND, 	/* gcwq->trustee_state */	TRUSTEE_START		= 0,		/* start */	TRUSTEE_IN_CHARGE	= 1,		/* trustee in charge of gcwq */	TRUSTEE_BUTCHER		= 2,		/* butcher workers */	TRUSTEE_RELEASE		= 3,		/* release workers */	TRUSTEE_DONE		= 4,		/* trustee is done */ 	BUSY_WORKER_HASH_ORDER	= 6,		/* 64 pointers */	BUSY_WORKER_HASH_SIZE	= 1 << BUSY_WORKER_HASH_ORDER,	BUSY_WORKER_HASH_MASK	= BUSY_WORKER_HASH_SIZE - 1, 	MAX_IDLE_WORKERS_RATIO	= 4,		/* 1/4 of busy can be idle */	IDLE_WORKER_TIMEOUT	= 300 * HZ,	/* keep idle ones for 5 mins */ 	MAYDAY_INITIAL_TIMEOUT  = HZ / 100 >= 2 ? HZ / 100 : 2,						/* call for help after 10ms						   (min two ticks) */	MAYDAY_INTERVAL		= HZ / 10,	/* and then every 100ms */	CREATE_COOLDOWN		= HZ,		/* time to breath after fail */	TRUSTEE_COOLDOWN	= HZ / 10,	/* for trustee draining */ 	/*	 * Rescue workers are used only on emergencies and shared by	 * all cpus.  Give -20.	 */	RESCUER_NICE_LEVEL	= -20,}; /* * Structure fields follow one of the following exclusion rules. * * I: Modifiable by initialization/destruction paths and read-only for *    everyone else. * * P: Preemption protected.  Disabling preemption is enough and should *    only be modified and accessed from the local cpu. * * L: gcwq->lock protected.  Access with gcwq->lock held. * * X: During normal operation, modification requires gcwq->lock and *    should be done only from local cpu.  Either disabling preemption *    on local cpu or grabbing gcwq->lock is enough for read access. *    If GCWQ_DISASSOCIATED is set, it's identical to L. * * F: wq->flush_mutex protected. * * W: workqueue_lock protected. */ struct global_cwq; /* * The poor guys doing the actual heavy lifting.  All on-duty workers * are either serving the manager role, on idle list or on busy hash. */struct worker {	/* on idle list while idle, on busy hash table while busy */	union {		struct list_head	entry;	/* L: while idle */		struct hlist_node	hentry;	/* L: while busy */	}; 	struct work_struct	*current_work;	/* L: work being processed */	struct cpu_workqueue_struct *current_cwq; /* L: current_work's cwq */	struct list_head	scheduled;	/* L: scheduled works */	struct task_struct	*task;		/* I: worker task */	struct global_cwq	*gcwq;		/* I: the associated gcwq */	/* 64 bytes boundary on 64bit, 32 on 32bit */	unsigned long		last_active;	/* L: last active timestamp */	unsigned int		flags;		/* X: flags */	int			id;		/* I: worker id */	struct work_struct	rebind_work;	/* L: rebind worker to cpu */}; /* * Global per-cpu workqueue.  There's one and only one for each cpu * and all works are queued and processed here regardless of their * target workqueues. */struct global_cwq {	spinlock_t		lock;		/* the gcwq lock */	struct list_head	worklist;	/* L: list of pending works */	unsigned int		cpu;		/* I: the associated cpu */	unsigned int		flags;		/* L: GCWQ_* flags */ 	int			nr_workers;	/* L: total number of workers */	int			nr_idle;	/* L: currently idle ones */ 	/* workers are chained either in the idle_list or busy_hash */	struct list_head	idle_list;	/* X: list of idle workers */	struct hlist_head	busy_hash[BUSY_WORKER_HASH_SIZE];						/* L: hash of busy workers */ 	struct timer_list	idle_timer;	/* L: worker idle timeout */	struct timer_list	mayday_timer;	/* L: SOS timer for dworkers */ 	struct ida		worker_ida;	/* L: for worker IDs */ 	struct task_struct	*trustee;	/* L: for gcwq shutdown */	unsigned int		trustee_state;	/* L: trustee state */	wait_queue_head_t	trustee_wait;	/* trustee wait */	struct worker		*first_idle;	/* L: first idle worker */} ____cacheline_aligned_in_smp; /* * The per-CPU workqueue.  The lower WORK_STRUCT_FLAG_BITS of * work_struct->data are used for flags and thus cwqs need to be * aligned at two's power of the number of flag bits. */struct cpu_workqueue_struct {	struct global_cwq	*gcwq;		/* I: the associated gcwq */	struct workqueue_struct *wq;		/* I: the owning workqueue */	int			work_color;	/* L: current color */	int			flush_color;	/* L: flushing color */	int			nr_in_flight[WORK_NR_COLORS];						/* L: nr of in_flight works */	int			nr_active;	/* L: nr of active works */	int			max_active;	/* L: max active works */	struct list_head	delayed_works;	/* L: delayed works */}; /* * Structure used to wait for workqueue flush. */struct wq_flusher {	struct list_head	list;		/* F: list of flushers */	int			flush_color;	/* F: flush color waiting for */	struct completion	done;		/* flush completion */}; /* * All cpumasks are assumed to be always set on UP and thus can't be * used to determine whether there's something to be done. */#ifdef CONFIG_SMPtypedef cpumask_var_t mayday_mask_t;#define mayday_test_and_set_cpu(cpu, mask)	\	cpumask_test_and_set_cpu((cpu), (mask))#define mayday_clear_cpu(cpu, mask)		cpumask_clear_cpu((cpu), (mask))#define for_each_mayday_cpu(cpu, mask)		for_each_cpu((cpu), (mask))#define alloc_mayday_mask(maskp, gfp)		zalloc_cpumask_var((maskp), (gfp))#define free_mayday_mask(mask)			free_cpumask_var((mask))#elsetypedef unsigned long mayday_mask_t;#define mayday_test_and_set_cpu(cpu, mask)	test_and_set_bit(0, &(mask))#define mayday_clear_cpu(cpu, mask)		clear_bit(0, &(mask))#define for_each_mayday_cpu(cpu, mask)		if ((cpu) = 0, (mask))#define alloc_mayday_mask(maskp, gfp)		true#define free_mayday_mask(mask)			do { } while (0)#endif /* * The externally visible workqueue abstraction is an array of * per-CPU workqueues: */struct workqueue_struct {	unsigned int		flags;		/* W: WQ_* flags */	union {		struct cpu_workqueue_struct __percpu	*pcpu;		struct cpu_workqueue_struct		*single;		unsigned long				v;	} cpu_wq;				/* I: cwq's */	struct list_head	list;		/* W: list of all workqueues */ 	struct mutex		flush_mutex;	/* protects wq flushing */	int			work_color;	/* F: current work color */	int			flush_color;	/* F: current flush color */	atomic_t		nr_cwqs_to_flush; /* flush in progress */	struct wq_flusher	*first_flusher;	/* F: first flusher */	struct list_head	flusher_queue;	/* F: flush waiters */	struct list_head	flusher_overflow; /* F: flush overflow list */ 	mayday_mask_t		mayday_mask;	/* cpus requesting rescue */	struct worker		*rescuer;	/* I: rescue worker */ 	int			nr_drainers;	/* W: drain in progress */	int			saved_max_active; /* W: saved cwq max_active */#ifdef CONFIG_LOCKDEP	struct lockdep_map	lockdep_map;#endif	char			name[];		/* I: workqueue name */}; struct workqueue_struct *system_wq __read_mostly;struct workqueue_struct *system_long_wq __read_mostly;struct workqueue_struct *system_nrt_wq __read_mostly;struct workqueue_struct *system_unbound_wq __read_mostly;struct workqueue_struct *system_freezable_wq __read_mostly;struct workqueue_struct *system_nrt_freezable_wq __read_mostly;EXPORT_SYMBOL_GPL(system_wq);EXPORT_SYMBOL_GPL(system_long_wq);EXPORT_SYMBOL_GPL(system_nrt_wq);EXPORT_SYMBOL_GPL(system_unbound_wq);EXPORT_SYMBOL_GPL(system_freezable_wq);EXPORT_SYMBOL_GPL(system_nrt_freezable_wq); #define CREATE_TRACE_POINTS#include <trace/events/workqueue.h> #define for_each_busy_worker(worker, i, pos, gcwq)			\	for (i = 0; i < BUSY_WORKER_HASH_SIZE; i++)			\		hlist_for_each_entry(worker, pos, &gcwq->busy_hash[i], hentry) static inline int __next_gcwq_cpu(int cpu, const struct cpumask *mask,				  unsigned int sw){	if (cpu < nr_cpu_ids) {		if (sw & 1) {			cpu = cpumask_next(cpu, mask);			if (cpu < nr_cpu_ids)				return cpu;		}		if (sw & 2)			return WORK_CPU_UNBOUND;	}	return WORK_CPU_NONE;} static inline int __next_wq_cpu(int cpu, const struct cpumask *mask,				struct workqueue_struct *wq){	return __next_gcwq_cpu(cpu, mask, !(wq->flags & WQ_UNBOUND) ? 1 : 2);} /* * CPU iterators * * An extra gcwq is defined for an invalid cpu number * (WORK_CPU_UNBOUND) to host workqueues which are not bound to any * specific CPU.  The following iterators are similar to * for_each_*_cpu() iterators but also considers the unbound gcwq. * * for_each_gcwq_cpu()		: possible CPUs + WORK_CPU_UNBOUND * for_each_online_gcwq_cpu()	: online CPUs + WORK_CPU_UNBOUND * for_each_cwq_cpu()		: possible CPUs for bound workqueues, *				  WORK_CPU_UNBOUND for unbound workqueues */#define for_each_gcwq_cpu(cpu)						\	for ((cpu) = __next_gcwq_cpu(-1, cpu_possible_mask, 3);		\	     (cpu) < WORK_CPU_NONE;					\	     (cpu) = __next_gcwq_cpu((cpu), cpu_possible_mask, 3)) #define for_each_online_gcwq_cpu(cpu)					\	for ((cpu) = __next_gcwq_cpu(-1, cpu_online_mask, 3);		\	     (cpu) < WORK_CPU_NONE;					\	     (cpu) = __next_gcwq_cpu((cpu), cpu_online_mask, 3)) #define for_each_cwq_cpu(cpu, wq)					\	for ((cpu) = __next_wq_cpu(-1, cpu_possible_mask, (wq));	\	     (cpu) < WORK_CPU_NONE;					\	     (cpu) = __next_wq_cpu((cpu), cpu_possible_mask, (wq))) #ifdef CONFIG_DEBUG_OBJECTS_WORK static struct debug_obj_descr work_debug_descr; static void *work_debug_hint(void *addr){	return ((struct work_struct *) addr)->func;} /* * fixup_init is called when: * - an active object is initialized */static int work_fixup_init(void *addr, enum debug_obj_state state){	struct work_struct *work = addr; 	switch (state) {	case ODEBUG_STATE_ACTIVE:		cancel_work_sync(work);		debug_object_init(work, &work_debug_descr);		return 1;	default:		return 0;	}} /* * fixup_activate is called when: * - an active object is activated * - an unknown object is activated (might be a statically initialized object) */static int work_fixup_activate(void *addr, enum debug_obj_state state){	struct work_struct *work = addr; 	switch (state) { 	case ODEBUG_STATE_NOTAVAILABLE:		/*		 * This is not really a fixup. The work struct was		 * statically initialized. We just make sure that it		 * is tracked in the object tracker.		 */		if (test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work))) {			debug_object_init(work, &work_debug_descr);			debug_object_activate(work, &work_debug_descr);			return 0;		}		WARN_ON_ONCE(1);		return 0; 	case ODEBUG_STATE_ACTIVE:		WARN_ON(1); 	default:		return 0;	}} /* * fixup_free is called when: * - an active object is freed */static int work_fixup_free(void *addr, enum debug_obj_state state){	struct work_struct *work = addr; 	switch (state) {	case ODEBUG_STATE_ACTIVE:		cancel_work_sync(work);		debug_object_free(work, &work_debug_descr);		return 1;	default:		return 0;	}} static struct debug_obj_descr work_debug_descr = {	.name		= "work_struct",	.debug_hint	= work_debug_hint,	.fixup_init	= work_fixup_init,	.fixup_activate	= work_fixup_activate,	.fixup_free	= work_fixup_free,}; static inline void debug_work_activate(struct work_struct *work){	debug_object_activate(work, &work_debug_descr);} static inline void debug_work_deactivate(struct work_struct *work){	debug_object_deactivate(work, &work_debug_descr);} void __init_work(struct work_struct *work, int onstack){	if (onstack)		debug_object_init_on_stack(work, &work_debug_descr);	else		debug_object_init(work, &work_debug_descr);}EXPORT_SYMBOL_GPL(__init_work); void destroy_work_on_stack(struct work_struct *work){	debug_object_free(work, &work_debug_descr);}EXPORT_SYMBOL_GPL(destroy_work_on_stack); #elsestatic inline void debug_work_activate(struct work_struct *work) { }static inline void debug_work_deactivate(struct work_struct *work) { }#endif /* Serializes the accesses to the list of workqueues. */static DEFINE_SPINLOCK(workqueue_lock);static LIST_HEAD(workqueues);static bool workqueue_freezing;		/* W: have wqs started freezing? */ /* * The almighty global cpu workqueues.  nr_running is the only field * which is expected to be used frequently by other cpus via * try_to_wake_up().  Put it in a separate cacheline. */static DEFINE_PER_CPU(struct global_cwq, global_cwq);static DEFINE_PER_CPU_SHARED_ALIGNED(atomic_t, gcwq_nr_running); /* * Global cpu workqueue and nr_running counter for unbound gcwq.  The * gcwq is always online, has GCWQ_DISASSOCIATED set, and all its * workers have WORKER_UNBOUND set. */static struct global_cwq unbound_global_cwq;static atomic_t unbound_gcwq_nr_running = ATOMIC_INIT(0);	/* always 0 */ static int worker_thread(void *__worker); static struct global_cwq *get_gcwq(unsigned int cpu){	if (cpu != WORK_CPU_UNBOUND)		return &per_cpu(global_cwq, cpu);	else		return &unbound_global_cwq;} static atomic_t *get_gcwq_nr_running(unsigned int cpu){	if (cpu != WORK_CPU_UNBOUND)		return &per_cpu(gcwq_nr_running, cpu);	else		return &unbound_gcwq_nr_running;} static struct cpu_workqueue_struct *get_cwq(unsigned int cpu,					    struct workqueue_struct *wq){	if (!(wq->flags & WQ_UNBOUND)) {		if (likely(cpu < nr_cpu_ids)) {#ifdef CONFIG_SMP			return per_cpu_ptr(wq->cpu_wq.pcpu, cpu);#else			return wq->cpu_wq.single;#endif		}	} else if (likely(cpu == WORK_CPU_UNBOUND))		return wq->cpu_wq.single;	return NULL;} static unsigned int work_color_to_flags(int color){	return color << WORK_STRUCT_COLOR_SHIFT;} static int get_work_color(struct work_struct *work){	return (*work_data_bits(work) >> WORK_STRUCT_COLOR_SHIFT) &		((1 << WORK_STRUCT_COLOR_BITS) - 1);}

Lines 1–500 of 3845. Load more