5816 lines · 159.4 KB · C
RawDownload
 * true return, it's guaranteed that need_to_create_worker() is false * and may_start_working() is true. * * CONTEXT: * spin_lock_irq(pool->lock) which may be released and regrabbed * multiple times.  Does GFP_KERNEL allocations. * * Return: * %false if the pool doesn't need management and the caller can safely * start processing works, %true if management function was performed and * the conditions that the caller verified before calling the function may * no longer be true. */static bool manage_workers(struct worker *worker){	struct worker_pool *pool = worker->pool; 	if (pool->flags & POOL_MANAGER_ACTIVE)		return false; 	pool->flags |= POOL_MANAGER_ACTIVE;	pool->manager = worker; 	maybe_create_worker(pool); 	pool->manager = NULL;	pool->flags &= ~POOL_MANAGER_ACTIVE;	wake_up(&wq_manager_wait);	return true;} /** * process_one_work - process single work * @worker: self * @work: work to process * * Process @work.  This function contains all the logics necessary to * process a single work including synchronization against and * interaction with other workers on the same cpu, queueing and * flushing.  As long as context requirement is met, any worker can * call this function to process a work. * * CONTEXT: * spin_lock_irq(pool->lock) which is released and regrabbed. */static void process_one_work(struct worker *worker, struct work_struct *work)__releases(&pool->lock)__acquires(&pool->lock){	struct pool_workqueue *pwq = get_work_pwq(work);	struct worker_pool *pool = worker->pool;	bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE;	int work_color;	struct worker *collision;#ifdef CONFIG_LOCKDEP	/*	 * It is permissible to free the struct work_struct from	 * inside the function that is called from it, this we need to	 * take into account for lockdep too.  To avoid bogus "held	 * lock freed" warnings as well as problems when looking into	 * work->lockdep_map, make a copy and use that here.	 */	struct lockdep_map lockdep_map; 	lockdep_copy_map(&lockdep_map, &work->lockdep_map);#endif	/* ensure we're on the correct CPU */	WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&		     raw_smp_processor_id() != pool->cpu); 	/*	 * A single work shouldn't be executed concurrently by	 * multiple workers on a single cpu.  Check whether anyone is	 * already processing the work.  If so, defer the work to the	 * currently executing one.	 */	collision = find_worker_executing_work(pool, work);	if (unlikely(collision)) {		move_linked_works(work, &collision->scheduled, NULL);		return;	} 	/* claim and dequeue */	debug_work_deactivate(work);	hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work);	worker->current_work = work;	worker->current_func = work->func;	worker->current_pwq = pwq;	work_color = get_work_color(work); 	/*	 * Record wq name for cmdline and debug reporting, may get	 * overridden through set_worker_desc().	 */	strscpy(worker->desc, pwq->wq->name, WORKER_DESC_LEN); 	list_del_init(&work->entry); 	/*	 * CPU intensive works don't participate in concurrency management.	 * They're the scheduler's responsibility.  This takes @worker out	 * of concurrency management and the next code block will chain	 * execution of the pending work items.	 */	if (unlikely(cpu_intensive))		worker_set_flags(worker, WORKER_CPU_INTENSIVE); 	/*	 * Wake up another worker if necessary.  The condition is always	 * false for normal per-cpu workers since nr_running would always	 * be >= 1 at this point.  This is used to chain execution of the	 * pending work items for WORKER_NOT_RUNNING workers such as the	 * UNBOUND and CPU_INTENSIVE ones.	 */	if (need_more_worker(pool))		wake_up_worker(pool); 	/*	 * Record the last pool and clear PENDING which should be the last	 * update to @work.  Also, do this inside @pool->lock so that	 * PENDING and queued state changes happen together while IRQ is	 * disabled.	 */	set_work_pool_and_clear_pending(work, pool->id); 	spin_unlock_irq(&pool->lock); 	lock_map_acquire(&pwq->wq->lockdep_map);	lock_map_acquire(&lockdep_map);	/*	 * Strictly speaking we should mark the invariant state without holding	 * any locks, that is, before these two lock_map_acquire()'s.	 *	 * However, that would result in:	 *	 *   A(W1)	 *   WFC(C)	 *		A(W1)	 *		C(C)	 *	 * Which would create W1->C->W1 dependencies, even though there is no	 * actual deadlock possible. There are two solutions, using a	 * read-recursive acquire on the work(queue) 'locks', but this will then	 * hit the lockdep limitation on recursive locks, or simply discard	 * these locks.	 *	 * AFAICT there is no possible deadlock scenario between the	 * flush_work() and complete() primitives (except for single-threaded	 * workqueues), so hiding them isn't a problem.	 */	lockdep_invariant_state(true);	trace_workqueue_execute_start(work);	worker->current_func(work);	/*	 * While we must be careful to not use "work" after this, the trace	 * point will only record its address.	 */	trace_workqueue_execute_end(work);	lock_map_release(&lockdep_map);	lock_map_release(&pwq->wq->lockdep_map); 	if (unlikely(in_atomic() || lockdep_depth(current) > 0)) {		pr_err("BUG: workqueue leaked lock or atomic: %s/0x%08x/%d\n"		       "     last function: %pf\n",		       current->comm, preempt_count(), task_pid_nr(current),		       worker->current_func);		debug_show_held_locks(current);		dump_stack();	} 	/*	 * The following prevents a kworker from hogging CPU on !PREEMPT	 * kernels, where a requeueing work item waiting for something to	 * happen could deadlock with stop_machine as such work item could	 * indefinitely requeue itself while all other CPUs are trapped in	 * stop_machine. At the same time, report a quiescent RCU state so	 * the same condition doesn't freeze RCU.	 */	cond_resched(); 	spin_lock_irq(&pool->lock); 	/* clear cpu intensive status */	if (unlikely(cpu_intensive))		worker_clr_flags(worker, WORKER_CPU_INTENSIVE); 	/* we're done with it, release */	hash_del(&worker->hentry);	worker->current_work = NULL;	worker->current_func = NULL;	worker->current_pwq = NULL;	pwq_dec_nr_in_flight(pwq, work_color);} /** * process_scheduled_works - process scheduled works * @worker: self * * Process all scheduled works.  Please note that the scheduled list * may change while processing a work, so this function repeatedly * fetches a work from the top and executes it. * * CONTEXT: * spin_lock_irq(pool->lock) which may be released and regrabbed * multiple times. */static void process_scheduled_works(struct worker *worker){	while (!list_empty(&worker->scheduled)) {		struct work_struct *work = list_first_entry(&worker->scheduled,						struct work_struct, entry);		process_one_work(worker, work);	}} static void set_pf_worker(bool val){	mutex_lock(&wq_pool_attach_mutex);	if (val)		current->flags |= PF_WQ_WORKER;	else		current->flags &= ~PF_WQ_WORKER;	mutex_unlock(&wq_pool_attach_mutex);} /** * worker_thread - the worker thread function * @__worker: self * * The worker thread function.  All workers belong to a worker_pool - * either a per-cpu one or dynamic unbound one.  These workers process all * work items regardless of their specific target workqueue.  The only * exception is work items which belong to workqueues with a rescuer which * will be explained in rescuer_thread(). * * Return: 0 */static int worker_thread(void *__worker){	struct worker *worker = __worker;	struct worker_pool *pool = worker->pool; 	/* tell the scheduler that this is a workqueue worker */	set_pf_worker(true);woke_up:	spin_lock_irq(&pool->lock); 	/* am I supposed to die? */	if (unlikely(worker->flags & WORKER_DIE)) {		spin_unlock_irq(&pool->lock);		WARN_ON_ONCE(!list_empty(&worker->entry));		set_pf_worker(false); 		set_task_comm(worker->task, "kworker/dying");		ida_simple_remove(&pool->worker_ida, worker->id);		worker_detach_from_pool(worker);		kfree(worker);		return 0;	} 	worker_leave_idle(worker);recheck:	/* no more worker necessary? */	if (!need_more_worker(pool))		goto sleep; 	/* do we need to manage? */	if (unlikely(!may_start_working(pool)) && manage_workers(worker))		goto recheck; 	/*	 * ->scheduled list can only be filled while a worker is	 * preparing to process a work or actually processing it.	 * Make sure nobody diddled with it while I was sleeping.	 */	WARN_ON_ONCE(!list_empty(&worker->scheduled)); 	/*	 * Finish PREP stage.  We're guaranteed to have at least one idle	 * worker or that someone else has already assumed the manager	 * role.  This is where @worker starts participating in concurrency	 * management if applicable and concurrency management is restored	 * after being rebound.  See rebind_workers() for details.	 */	worker_clr_flags(worker, WORKER_PREP | WORKER_REBOUND); 	do {		struct work_struct *work =			list_first_entry(&pool->worklist,					 struct work_struct, entry); 		pool->watchdog_ts = jiffies; 		if (likely(!(*work_data_bits(work) & WORK_STRUCT_LINKED))) {			/* optimization path, not strictly necessary */			process_one_work(worker, work);			if (unlikely(!list_empty(&worker->scheduled)))				process_scheduled_works(worker);		} else {			move_linked_works(work, &worker->scheduled, NULL);			process_scheduled_works(worker);		}	} while (keep_working(pool)); 	worker_set_flags(worker, WORKER_PREP);sleep:	/*	 * pool->lock is held and there's no work to process and no need to	 * manage, sleep.  Workers are woken up only while holding	 * pool->lock or from local cpu, so setting the current state	 * before releasing pool->lock is enough to prevent losing any	 * event.	 */	worker_enter_idle(worker);	__set_current_state(TASK_IDLE);	spin_unlock_irq(&pool->lock);	schedule();	goto woke_up;} /** * rescuer_thread - the rescuer thread function * @__rescuer: self * * Workqueue rescuer thread function.  There's one rescuer for each * workqueue which has WQ_MEM_RECLAIM set. * * Regular work processing on a pool may block trying to create a new * worker which uses GFP_KERNEL allocation which has slight chance of * developing into deadlock if some works currently on the same queue * need to be processed to satisfy the GFP_KERNEL allocation.  This is * the problem rescuer solves. * * When such condition is possible, the pool summons rescuers of all * workqueues which have works queued on the pool and let them process * those works so that forward progress can be guaranteed. * * This should happen rarely. * * Return: 0 */static int rescuer_thread(void *__rescuer){	struct worker *rescuer = __rescuer;	struct workqueue_struct *wq = rescuer->rescue_wq;	struct list_head *scheduled = &rescuer->scheduled;	bool should_stop; 	set_user_nice(current, RESCUER_NICE_LEVEL); 	/*	 * Mark rescuer as worker too.  As WORKER_PREP is never cleared, it	 * doesn't participate in concurrency management.	 */	set_pf_worker(true);repeat:	set_current_state(TASK_IDLE); 	/*	 * By the time the rescuer is requested to stop, the workqueue	 * shouldn't have any work pending, but @wq->maydays may still have	 * pwq(s) queued.  This can happen by non-rescuer workers consuming	 * all the work items before the rescuer got to them.  Go through	 * @wq->maydays processing before acting on should_stop so that the	 * list is always empty on exit.	 */	should_stop = kthread_should_stop(); 	/* see whether any pwq is asking for help */	spin_lock_irq(&wq_mayday_lock); 	while (!list_empty(&wq->maydays)) {		struct pool_workqueue *pwq = list_first_entry(&wq->maydays,					struct pool_workqueue, mayday_node);		struct worker_pool *pool = pwq->pool;		struct work_struct *work, *n;		bool first = true; 		__set_current_state(TASK_RUNNING);		list_del_init(&pwq->mayday_node); 		spin_unlock_irq(&wq_mayday_lock); 		worker_attach_to_pool(rescuer, pool); 		spin_lock_irq(&pool->lock); 		/*		 * Slurp in all works issued via this workqueue and		 * process'em.		 */		WARN_ON_ONCE(!list_empty(scheduled));		list_for_each_entry_safe(work, n, &pool->worklist, entry) {			if (get_work_pwq(work) == pwq) {				if (first)					pool->watchdog_ts = jiffies;				move_linked_works(work, scheduled, &n);			}			first = false;		} 		if (!list_empty(scheduled)) {			process_scheduled_works(rescuer); 			/*			 * The above execution of rescued work items could			 * have created more to rescue through			 * pwq_activate_first_delayed() or chained			 * queueing.  Let's put @pwq back on mayday list so			 * that such back-to-back work items, which may be			 * being used to relieve memory pressure, don't			 * incur MAYDAY_INTERVAL delay inbetween.			 */			if (need_to_create_worker(pool)) {				spin_lock(&wq_mayday_lock);				get_pwq(pwq);				list_move_tail(&pwq->mayday_node, &wq->maydays);				spin_unlock(&wq_mayday_lock);			}		} 		/*		 * Put the reference grabbed by send_mayday().  @pool won't		 * go away while we're still attached to it.		 */		put_pwq(pwq); 		/*		 * Leave this pool.  If need_more_worker() is %true, notify a		 * regular worker; otherwise, we end up with 0 concurrency		 * and stalling the execution.		 */		if (need_more_worker(pool))			wake_up_worker(pool); 		spin_unlock_irq(&pool->lock); 		worker_detach_from_pool(rescuer); 		spin_lock_irq(&wq_mayday_lock);	} 	spin_unlock_irq(&wq_mayday_lock); 	if (should_stop) {		__set_current_state(TASK_RUNNING);		set_pf_worker(false);		return 0;	} 	/* rescuers should never participate in concurrency management */	WARN_ON_ONCE(!(rescuer->flags & WORKER_NOT_RUNNING));	schedule();	goto repeat;} /** * check_flush_dependency - check for flush dependency sanity * @target_wq: workqueue being flushed * @target_work: work item being flushed (NULL for workqueue flushes) * * %current is trying to flush the whole @target_wq or @target_work on it. * If @target_wq doesn't have %WQ_MEM_RECLAIM, verify that %current is not * reclaiming memory or running on a workqueue which doesn't have * %WQ_MEM_RECLAIM as that can break forward-progress guarantee leading to * a deadlock. */static void check_flush_dependency(struct workqueue_struct *target_wq,				   struct work_struct *target_work){	work_func_t target_func = target_work ? target_work->func : NULL;	struct worker *worker; 	if (target_wq->flags & WQ_MEM_RECLAIM)		return; 	worker = current_wq_worker(); 	WARN_ONCE(current->flags & PF_MEMALLOC,		  "workqueue: PF_MEMALLOC task %d(%s) is flushing !WQ_MEM_RECLAIM %s:%pf",		  current->pid, current->comm, target_wq->name, target_func);	WARN_ONCE(worker && ((worker->current_pwq->wq->flags &			      (WQ_MEM_RECLAIM | __WQ_LEGACY)) == WQ_MEM_RECLAIM),		  "workqueue: WQ_MEM_RECLAIM %s:%pf is flushing !WQ_MEM_RECLAIM %s:%pf",		  worker->current_pwq->wq->name, worker->current_func,		  target_wq->name, target_func);} struct wq_barrier {	struct work_struct	work;	struct completion	done;	struct task_struct	*task;	/* purely informational */}; static void wq_barrier_func(struct work_struct *work){	struct wq_barrier *barr = container_of(work, struct wq_barrier, work);	complete(&barr->done);} 

Lines 2001–2500 of 5816. Previous lines

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t