raw_spin_unlock_irq(&wq_mayday_lock); 		worker_attach_to_pool(rescuer, pool); 		raw_spin_lock_irq(&pool->lock); 		/*		 * Slurp in all works issued via this workqueue and		 * process'em.		 */		WARN_ON_ONCE(!list_empty(scheduled));		list_for_each_entry_safe(work, n, &pool->worklist, entry) {			if (get_work_pwq(work) == pwq) {				if (first)					pool->watchdog_ts = jiffies;				move_linked_works(work, scheduled, &n);			}			first = false;		} 		if (!list_empty(scheduled)) {			process_scheduled_works(rescuer); 			/*			 * The above execution of rescued work items could			 * have created more to rescue through			 * pwq_activate_first_delayed() or chained			 * queueing.  Let's put @pwq back on mayday list so			 * that such back-to-back work items, which may be			 * being used to relieve memory pressure, don't			 * incur MAYDAY_INTERVAL delay inbetween.			 */			if (pwq->nr_active && need_to_create_worker(pool)) {				raw_spin_lock(&wq_mayday_lock);				/*				 * Queue iff we aren't racing destruction				 * and somebody else hasn't queued it already.				 */				if (wq->rescuer && list_empty(&pwq->mayday_node)) {					get_pwq(pwq);					list_add_tail(&pwq->mayday_node, &wq->maydays);				}				raw_spin_unlock(&wq_mayday_lock);			}		} 		/*		 * Put the reference grabbed by send_mayday().  @pool won't		 * go away while we're still attached to it.		 */		put_pwq(pwq); 		/*		 * Leave this pool.  If need_more_worker() is %true, notify a		 * regular worker; otherwise, we end up with 0 concurrency		 * and stalling the execution.		 */		if (need_more_worker(pool))			wake_up_worker(pool); 		raw_spin_unlock_irq(&pool->lock); 		worker_detach_from_pool(rescuer); 		raw_spin_lock_irq(&wq_mayday_lock);	} 	raw_spin_unlock_irq(&wq_mayday_lock); 	if (should_stop) {		__set_current_state(TASK_RUNNING);		set_pf_worker(false);		return 0;	} 	/* rescuers should never participate in concurrency management */	WARN_ON_ONCE(!(rescuer->flags & WORKER_NOT_RUNNING));	schedule();	goto repeat;} /** * check_flush_dependency - check for flush dependency sanity * @target_wq: workqueue being flushed * @target_work: work item being flushed (NULL for workqueue flushes) * * %current is trying to flush the whole @target_wq or @target_work on it. * If @target_wq doesn't have %WQ_MEM_RECLAIM, verify that %current is not * reclaiming memory or running on a workqueue which doesn't have * %WQ_MEM_RECLAIM as that can break forward-progress guarantee leading to * a deadlock. */static void check_flush_dependency(struct workqueue_struct *target_wq,				   struct work_struct *target_work){	work_func_t target_func = target_work ? target_work->func : NULL;	struct worker *worker; 	if (target_wq->flags & WQ_MEM_RECLAIM)		return; 	worker = current_wq_worker(); 	WARN_ONCE(current->flags & PF_MEMALLOC,		  "workqueue: PF_MEMALLOC task %d(%s) is flushing !WQ_MEM_RECLAIM %s:%ps",		  current->pid, current->comm, target_wq->name, target_func);	WARN_ONCE(worker && ((worker->current_pwq->wq->flags &			      (WQ_MEM_RECLAIM | __WQ_LEGACY)) == WQ_MEM_RECLAIM),		  "workqueue: WQ_MEM_RECLAIM %s:%ps is flushing !WQ_MEM_RECLAIM %s:%ps",		  worker->current_pwq->wq->name, worker->current_func,		  target_wq->name, target_func);} struct wq_barrier {	struct work_struct	work;	struct completion	done;	struct task_struct	*task;	/* purely informational */}; static void wq_barrier_func(struct work_struct *work){	struct wq_barrier *barr = container_of(work, struct wq_barrier, work);	complete(&barr->done);} /** * insert_wq_barrier - insert a barrier work * @pwq: pwq to insert barrier into * @barr: wq_barrier to insert * @target: target work to attach @barr to * @worker: worker currently executing @target, NULL if @target is not executing * * @barr is linked to @target such that @barr is completed only after * @target finishes execution.  Please note that the ordering * guarantee is observed only with respect to @target and on the local * cpu. * * Currently, a queued barrier can't be canceled.  This is because * try_to_grab_pending() can't determine whether the work to be * grabbed is at the head of the queue and thus can't clear LINKED * flag of the previous work while there must be a valid next work * after a work with LINKED flag set. * * Note that when @worker is non-NULL, @target may be modified * underneath us, so we can't reliably determine pwq from @target. * * CONTEXT: * raw_spin_lock_irq(pool->lock). */static void insert_wq_barrier(struct pool_workqueue *pwq,			      struct wq_barrier *barr,			      struct work_struct *target, struct worker *worker){	struct list_head *head;	unsigned int linked = 0; 	/*	 * debugobject calls are safe here even with pool->lock locked	 * as we know for sure that this will not trigger any of the	 * checks and call back into the fixup functions where we	 * might deadlock.	 */	INIT_WORK_ONSTACK(&barr->work, wq_barrier_func);	__set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work)); 	init_completion_map(&barr->done, &target->lockdep_map); 	barr->task = current; 	/*	 * If @target is currently being executed, schedule the	 * barrier to the worker; otherwise, put it after @target.	 */	if (worker)		head = worker->scheduled.next;	else {		unsigned long *bits = work_data_bits(target); 		head = target->entry.next;		/* there can already be other linked works, inherit and set */		linked = *bits & WORK_STRUCT_LINKED;		__set_bit(WORK_STRUCT_LINKED_BIT, bits);	} 	debug_work_activate(&barr->work);	insert_work(pwq, &barr->work, head,		    work_color_to_flags(WORK_NO_COLOR) | linked);} /** * flush_workqueue_prep_pwqs - prepare pwqs for workqueue flushing * @wq: workqueue being flushed * @flush_color: new flush color, < 0 for no-op * @work_color: new work color, < 0 for no-op * * Prepare pwqs for workqueue flushing. * * If @flush_color is non-negative, flush_color on all pwqs should be * -1.  If no pwq has in-flight commands at the specified color, all * pwq->flush_color's stay at -1 and %false is returned.  If any pwq * has in flight commands, its pwq->flush_color is set to * @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq * wakeup logic is armed and %true is returned. * * The caller should have initialized @wq->first_flusher prior to * calling this function with non-negative @flush_color.  If * @flush_color is negative, no flush color update is done and %false * is returned. * * If @work_color is non-negative, all pwqs should have the same * work_color which is previous to @work_color and all will be * advanced to @work_color. * * CONTEXT: * mutex_lock(wq->mutex). * * Return: * %true if @flush_color >= 0 and there's something to flush.  %false * otherwise. */static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq,				      int flush_color, int work_color){	bool wait = false;	struct pool_workqueue *pwq; 	if (flush_color >= 0) {		WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush));		atomic_set(&wq->nr_pwqs_to_flush, 1);	} 	for_each_pwq(pwq, wq) {		struct worker_pool *pool = pwq->pool; 		raw_spin_lock_irq(&pool->lock); 		if (flush_color >= 0) {			WARN_ON_ONCE(pwq->flush_color != -1); 			if (pwq->nr_in_flight[flush_color]) {				pwq->flush_color = flush_color;				atomic_inc(&wq->nr_pwqs_to_flush);				wait = true;			}		} 		if (work_color >= 0) {			WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));			pwq->work_color = work_color;		} 		raw_spin_unlock_irq(&pool->lock);	} 	if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))		complete(&wq->first_flusher->done); 	return wait;} /** * flush_workqueue - ensure that any scheduled work has run to completion. * @wq: workqueue to flush * * This function sleeps until all work items which were queued on entry * have finished execution, but it is not livelocked by new incoming ones. */void flush_workqueue(struct workqueue_struct *wq){	struct wq_flusher this_flusher = {		.list = LIST_HEAD_INIT(this_flusher.list),		.flush_color = -1,		.done = COMPLETION_INITIALIZER_ONSTACK_MAP(this_flusher.done, wq->lockdep_map),	};	int next_color; 	if (WARN_ON(!wq_online))		return; 	lock_map_acquire(&wq->lockdep_map);	lock_map_release(&wq->lockdep_map); 	mutex_lock(&wq->mutex); 	/*	 * Start-to-wait phase	 */	next_color = work_next_color(wq->work_color); 	if (next_color != wq->flush_color) {		/*		 * Color space is not full.  The current work_color		 * becomes our flush_color and work_color is advanced		 * by one.		 */		WARN_ON_ONCE(!list_empty(&wq->flusher_overflow));		this_flusher.flush_color = wq->work_color;		wq->work_color = next_color; 		if (!wq->first_flusher) {			/* no flush in progress, become the first flusher */			WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color); 			wq->first_flusher = &this_flusher; 			if (!flush_workqueue_prep_pwqs(wq, wq->flush_color,						       wq->work_color)) {				/* nothing to flush, done */				wq->flush_color = next_color;				wq->first_flusher = NULL;				goto out_unlock;			}		} else {			/* wait in queue */			WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color);			list_add_tail(&this_flusher.list, &wq->flusher_queue);			flush_workqueue_prep_pwqs(wq, -1, wq->work_color);		}	} else {		/*		 * Oops, color space is full, wait on overflow queue.		 * The next flush completion will assign us		 * flush_color and transfer to flusher_queue.		 */		list_add_tail(&this_flusher.list, &wq->flusher_overflow);	} 	check_flush_dependency(wq, NULL); 	mutex_unlock(&wq->mutex); 	wait_for_completion(&this_flusher.done); 	/*	 * Wake-up-and-cascade phase	 *	 * First flushers are responsible for cascading flushes and	 * handling overflow.  Non-first flushers can simply return.	 */	if (READ_ONCE(wq->first_flusher) != &this_flusher)		return; 	mutex_lock(&wq->mutex); 	/* we might have raced, check again with mutex held */	if (wq->first_flusher != &this_flusher)		goto out_unlock; 	WRITE_ONCE(wq->first_flusher, NULL); 	WARN_ON_ONCE(!list_empty(&this_flusher.list));	WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color); 	while (true) {		struct wq_flusher *next, *tmp; 		/* complete all the flushers sharing the current flush color */		list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) {			if (next->flush_color != wq->flush_color)				break;			list_del_init(&next->list);			complete(&next->done);		} 		WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) &&			     wq->flush_color != work_next_color(wq->work_color)); 		/* this flush_color is finished, advance by one */		wq->flush_color = work_next_color(wq->flush_color); 		/* one color has been freed, handle overflow queue */		if (!list_empty(&wq->flusher_overflow)) {			/*			 * Assign the same color to all overflowed			 * flushers, advance work_color and append to			 * flusher_queue.  This is the start-to-wait			 * phase for these overflowed flushers.			 */			list_for_each_entry(tmp, &wq->flusher_overflow, list)				tmp->flush_color = wq->work_color; 			wq->work_color = work_next_color(wq->work_color); 			list_splice_tail_init(&wq->flusher_overflow,					      &wq->flusher_queue);			flush_workqueue_prep_pwqs(wq, -1, wq->work_color);		} 		if (list_empty(&wq->flusher_queue)) {			WARN_ON_ONCE(wq->flush_color != wq->work_color);			break;		} 		/*		 * Need to flush more colors.  Make the next flusher		 * the new first flusher and arm pwqs.		 */		WARN_ON_ONCE(wq->flush_color == wq->work_color);		WARN_ON_ONCE(wq->flush_color != next->flush_color); 		list_del_init(&next->list);		wq->first_flusher = next; 		if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1))			break; 		/*		 * Meh... this color is already done, clear first		 * flusher and repeat cascading.		 */		wq->first_flusher = NULL;	} out_unlock:	mutex_unlock(&wq->mutex);}EXPORT_SYMBOL(flush_workqueue); /** * drain_workqueue - drain a workqueue * @wq: workqueue to drain * * Wait until the workqueue becomes empty.  While draining is in progress, * only chain queueing is allowed.  IOW, only currently pending or running * work items on @wq can queue further work items on it.  @wq is flushed * repeatedly until it becomes empty.  The number of flushing is determined * by the depth of chaining and should be relatively short.  Whine if it * takes too long. */void drain_workqueue(struct workqueue_struct *wq){	unsigned int flush_cnt = 0;	struct pool_workqueue *pwq; 	/*	 * __queue_work() needs to test whether there are drainers, is much	 * hotter than drain_workqueue() and already looks at @wq->flags.	 * Use __WQ_DRAINING so that queue doesn't have to check nr_drainers.	 */	mutex_lock(&wq->mutex);	if (!wq->nr_drainers++)		wq->flags |= __WQ_DRAINING;	mutex_unlock(&wq->mutex);reflush:	flush_workqueue(wq); 	mutex_lock(&wq->mutex); 	for_each_pwq(pwq, wq) {		bool drained; 		raw_spin_lock_irq(&pwq->pool->lock);		drained = !pwq->nr_active && list_empty(&pwq->delayed_works);		raw_spin_unlock_irq(&pwq->pool->lock); 		if (drained)			continue; 		if (++flush_cnt == 10 ||		    (flush_cnt % 100 == 0 && flush_cnt <= 1000))			pr_warn("workqueue %s: drain_workqueue() isn't complete after %u tries\n",				wq->name, flush_cnt); 		mutex_unlock(&wq->mutex);		goto reflush;	} 	if (!--wq->nr_drainers)		wq->flags &= ~__WQ_DRAINING;	mutex_unlock(&wq->mutex);}EXPORT_SYMBOL_GPL(drain_workqueue); static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr,			     bool from_cancel){	struct worker *worker = NULL;	struct worker_pool *pool;	struct pool_workqueue *pwq; 	might_sleep(); 	rcu_read_lock();	pool = get_work_pool(work);	if (!pool) {		rcu_read_unlock();		return false;	} 	raw_spin_lock_irq(&pool->lock);	/* see the comment in try_to_grab_pending() with the same code */	pwq = get_work_pwq(work);	if (pwq) {		if (unlikely(pwq->pool != pool))			goto already_gone;	} else {		worker = find_worker_executing_work(pool, work);		if (!worker)			goto already_gone;		pwq = worker->current_pwq;

Lines 2501–3000 of 6039. Previous lines Load more