if (flush_color >= 0) {			WARN_ON_ONCE(pwq->flush_color != -1); 			if (pwq->nr_in_flight[flush_color]) {				pwq->flush_color = flush_color;				atomic_inc(&wq->nr_pwqs_to_flush);				wait = true;			}		} 		if (work_color >= 0) {			WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));			pwq->work_color = work_color;		} 	} 	if (current_pool)		raw_spin_unlock_irq(&current_pool->lock); 	if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))		complete(&wq->first_flusher->done); 	return wait;} static void touch_wq_lockdep_map(struct workqueue_struct *wq){#ifdef CONFIG_LOCKDEP	if (unlikely(!wq->lockdep_map))		return; 	if (wq->flags & WQ_BH)		local_bh_disable(); 	lock_map_acquire(wq->lockdep_map);	lock_map_release(wq->lockdep_map); 	if (wq->flags & WQ_BH)		local_bh_enable();#endif} static void touch_work_lockdep_map(struct work_struct *work,				   struct workqueue_struct *wq){#ifdef CONFIG_LOCKDEP	if (wq->flags & WQ_BH)		local_bh_disable(); 	lock_map_acquire(&work->lockdep_map);	lock_map_release(&work->lockdep_map); 	if (wq->flags & WQ_BH)		local_bh_enable();#endif} /** * __flush_workqueue - ensure that any scheduled work has run to completion. * @wq: workqueue to flush * * This function sleeps until all work items which were queued on entry * have finished execution, but it is not livelocked by new incoming ones. */void __flush_workqueue(struct workqueue_struct *wq){	struct wq_flusher this_flusher = {		.list = LIST_HEAD_INIT(this_flusher.list),		.flush_color = -1,		.done = COMPLETION_INITIALIZER_ONSTACK_MAP(this_flusher.done, (*wq->lockdep_map)),	};	int next_color; 	if (WARN_ON(!wq_online))		return; 	touch_wq_lockdep_map(wq); 	mutex_lock(&wq->mutex); 	/*	 * Start-to-wait phase	 */	next_color = work_next_color(wq->work_color); 	if (next_color != wq->flush_color) {		/*		 * Color space is not full.  The current work_color		 * becomes our flush_color and work_color is advanced		 * by one.		 */		WARN_ON_ONCE(!list_empty(&wq->flusher_overflow));		this_flusher.flush_color = wq->work_color;		wq->work_color = next_color; 		if (!wq->first_flusher) {			/* no flush in progress, become the first flusher */			WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color); 			wq->first_flusher = &this_flusher; 			if (!flush_workqueue_prep_pwqs(wq, wq->flush_color,						       wq->work_color)) {				/* nothing to flush, done */				wq->flush_color = next_color;				wq->first_flusher = NULL;				goto out_unlock;			}		} else {			/* wait in queue */			WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color);			list_add_tail(&this_flusher.list, &wq->flusher_queue);			flush_workqueue_prep_pwqs(wq, -1, wq->work_color);		}	} else {		/*		 * Oops, color space is full, wait on overflow queue.		 * The next flush completion will assign us		 * flush_color and transfer to flusher_queue.		 */		list_add_tail(&this_flusher.list, &wq->flusher_overflow);	} 	check_flush_dependency(wq, NULL, false); 	mutex_unlock(&wq->mutex); 	wait_for_completion(&this_flusher.done); 	/*	 * Wake-up-and-cascade phase	 *	 * First flushers are responsible for cascading flushes and	 * handling overflow.  Non-first flushers can simply return.	 */	if (READ_ONCE(wq->first_flusher) != &this_flusher)		return; 	mutex_lock(&wq->mutex); 	/* we might have raced, check again with mutex held */	if (wq->first_flusher != &this_flusher)		goto out_unlock; 	WRITE_ONCE(wq->first_flusher, NULL); 	WARN_ON_ONCE(!list_empty(&this_flusher.list));	WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color); 	while (true) {		struct wq_flusher *next, *tmp; 		/* complete all the flushers sharing the current flush color */		list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) {			if (next->flush_color != wq->flush_color)				break;			list_del_init(&next->list);			complete(&next->done);		} 		WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) &&			     wq->flush_color != work_next_color(wq->work_color)); 		/* this flush_color is finished, advance by one */		wq->flush_color = work_next_color(wq->flush_color); 		/* one color has been freed, handle overflow queue */		if (!list_empty(&wq->flusher_overflow)) {			/*			 * Assign the same color to all overflowed			 * flushers, advance work_color and append to			 * flusher_queue.  This is the start-to-wait			 * phase for these overflowed flushers.			 */			list_for_each_entry(tmp, &wq->flusher_overflow, list)				tmp->flush_color = wq->work_color; 			wq->work_color = work_next_color(wq->work_color); 			list_splice_tail_init(&wq->flusher_overflow,					      &wq->flusher_queue);			flush_workqueue_prep_pwqs(wq, -1, wq->work_color);		} 		if (list_empty(&wq->flusher_queue)) {			WARN_ON_ONCE(wq->flush_color != wq->work_color);			break;		} 		/*		 * Need to flush more colors.  Make the next flusher		 * the new first flusher and arm pwqs.		 */		WARN_ON_ONCE(wq->flush_color == wq->work_color);		WARN_ON_ONCE(wq->flush_color != next->flush_color); 		list_del_init(&next->list);		wq->first_flusher = next; 		if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1))			break; 		/*		 * Meh... this color is already done, clear first		 * flusher and repeat cascading.		 */		wq->first_flusher = NULL;	} out_unlock:	mutex_unlock(&wq->mutex);}EXPORT_SYMBOL(__flush_workqueue); /** * drain_workqueue - drain a workqueue * @wq: workqueue to drain * * Wait until the workqueue becomes empty.  While draining is in progress, * only chain queueing is allowed.  IOW, only currently pending or running * work items on @wq can queue further work items on it.  @wq is flushed * repeatedly until it becomes empty.  The number of flushing is determined * by the depth of chaining and should be relatively short.  Whine if it * takes too long. */void drain_workqueue(struct workqueue_struct *wq){	unsigned int flush_cnt = 0;	struct pool_workqueue *pwq; 	/*	 * __queue_work() needs to test whether there are drainers, is much	 * hotter than drain_workqueue() and already looks at @wq->flags.	 * Use __WQ_DRAINING so that queue doesn't have to check nr_drainers.	 */	mutex_lock(&wq->mutex);	if (!wq->nr_drainers++)		wq->flags |= __WQ_DRAINING;	mutex_unlock(&wq->mutex);reflush:	__flush_workqueue(wq); 	mutex_lock(&wq->mutex); 	for_each_pwq(pwq, wq) {		bool drained; 		raw_spin_lock_irq(&pwq->pool->lock);		drained = pwq_is_empty(pwq);		raw_spin_unlock_irq(&pwq->pool->lock); 		if (drained)			continue; 		if (++flush_cnt == 10 ||		    (flush_cnt % 100 == 0 && flush_cnt <= 1000))			pr_warn("workqueue %s: %s() isn't complete after %u tries\n",				wq->name, __func__, flush_cnt); 		mutex_unlock(&wq->mutex);		goto reflush;	} 	if (!--wq->nr_drainers)		wq->flags &= ~__WQ_DRAINING;	mutex_unlock(&wq->mutex);}EXPORT_SYMBOL_GPL(drain_workqueue); static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr,			     bool from_cancel){	struct worker *worker = NULL;	struct worker_pool *pool;	struct pool_workqueue *pwq;	struct workqueue_struct *wq; 	rcu_read_lock();	pool = get_work_pool(work);	if (!pool) {		rcu_read_unlock();		return false;	} 	raw_spin_lock_irq(&pool->lock);	/* see the comment in try_to_grab_pending() with the same code */	pwq = get_work_pwq(work);	if (pwq) {		if (unlikely(pwq->pool != pool))			goto already_gone;	} else {		worker = find_worker_executing_work(pool, work);		if (!worker)			goto already_gone;		pwq = worker->current_pwq;	} 	wq = pwq->wq;	check_flush_dependency(wq, work, from_cancel); 	insert_wq_barrier(pwq, barr, work, worker);	raw_spin_unlock_irq(&pool->lock); 	touch_work_lockdep_map(work, wq); 	/*	 * Force a lock recursion deadlock when using flush_work() inside a	 * single-threaded or rescuer equipped workqueue.	 *	 * For single threaded workqueues the deadlock happens when the work	 * is after the work issuing the flush_work(). For rescuer equipped	 * workqueues the deadlock happens when the rescuer stalls, blocking	 * forward progress.	 */	if (!from_cancel && (wq->saved_max_active == 1 || wq->rescuer))		touch_wq_lockdep_map(wq); 	rcu_read_unlock();	return true;already_gone:	raw_spin_unlock_irq(&pool->lock);	rcu_read_unlock();	return false;} static bool __flush_work(struct work_struct *work, bool from_cancel){	struct wq_barrier barr; 	if (WARN_ON(!wq_online))		return false; 	if (WARN_ON(!work->func))		return false; 	if (!start_flush_work(work, &barr, from_cancel))		return false; 	/*	 * start_flush_work() returned %true. If @from_cancel is set, we know	 * that @work must have been executing during start_flush_work() and	 * can't currently be queued. Its data must contain OFFQ bits. If @work	 * was queued on a BH workqueue, we also know that it was running in the	 * BH context and thus can be busy-waited.	 */	if (from_cancel) {		unsigned long data = *work_data_bits(work); 		if (!WARN_ON_ONCE(data & WORK_STRUCT_PWQ) &&		    (data & WORK_OFFQ_BH)) {			/*			 * On RT, prevent a live lock when %current preempted			 * soft interrupt processing by blocking on lock which			 * is owned by the thread invoking the callback.			 */			while (!try_wait_for_completion(&barr.done)) {				if (IS_ENABLED(CONFIG_PREEMPT_RT)) {					struct worker_pool *pool; 					guard(rcu)();					pool = get_work_pool(work);					if (pool)						workqueue_callback_cancel_wait_running(pool);				} else {					cpu_relax();				}			}			goto out_destroy;		}	} 	wait_for_completion(&barr.done); out_destroy:	destroy_work_on_stack(&barr.work);	return true;} /** * flush_work - wait for a work to finish executing the last queueing instance * @work: the work to flush * * Wait until @work has finished execution.  @work is guaranteed to be idle * on return if it hasn't been requeued since flush started. * * Return: * %true if flush_work() waited for the work to finish execution, * %false if it was already idle. */bool flush_work(struct work_struct *work){	might_sleep();	return __flush_work(work, false);}EXPORT_SYMBOL_GPL(flush_work); /** * flush_delayed_work - wait for a dwork to finish executing the last queueing * @dwork: the delayed work to flush * * Delayed timer is cancelled and the pending work is queued for * immediate execution.  Like flush_work(), this function only * considers the last queueing instance of @dwork. * * Return: * %true if flush_work() waited for the work to finish execution, * %false if it was already idle. */bool flush_delayed_work(struct delayed_work *dwork){	local_irq_disable();	if (timer_delete_sync(&dwork->timer))		__queue_work(dwork->cpu, dwork->wq, &dwork->work);	local_irq_enable();	return flush_work(&dwork->work);}EXPORT_SYMBOL(flush_delayed_work); /** * flush_rcu_work - wait for a rwork to finish executing the last queueing * @rwork: the rcu work to flush * * Return: * %true if flush_rcu_work() waited for the work to finish execution, * %false if it was already idle. */bool flush_rcu_work(struct rcu_work *rwork){	if (test_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&rwork->work))) {		rcu_barrier();		flush_work(&rwork->work);		return true;	} else {		return flush_work(&rwork->work);	}}EXPORT_SYMBOL(flush_rcu_work); static void work_offqd_disable(struct work_offq_data *offqd){	const unsigned long max = (1lu << WORK_OFFQ_DISABLE_BITS) - 1; 	if (likely(offqd->disable < max))		offqd->disable++;	else		WARN_ONCE(true, "workqueue: work disable count overflowed\n");} static void work_offqd_enable(struct work_offq_data *offqd){	if (likely(offqd->disable > 0))		offqd->disable--;	else		WARN_ONCE(true, "workqueue: work disable count underflowed\n");} static bool __cancel_work(struct work_struct *work, u32 cflags){	struct work_offq_data offqd;	unsigned long irq_flags;	int ret; 	ret = work_grab_pending(work, cflags, &irq_flags); 	work_offqd_unpack(&offqd, *work_data_bits(work)); 	if (cflags & WORK_CANCEL_DISABLE)		work_offqd_disable(&offqd); 	set_work_pool_and_clear_pending(work, offqd.pool_id,					work_offqd_pack_flags(&offqd));	local_irq_restore(irq_flags);	return ret;} static bool __cancel_work_sync(struct work_struct *work, u32 cflags){	bool ret; 	ret = __cancel_work(work, cflags | WORK_CANCEL_DISABLE); 	if (*work_data_bits(work) & WORK_OFFQ_BH)		WARN_ON_ONCE(in_hardirq());	else		might_sleep(); 	/*	 * Skip __flush_work() during early boot when we know that @work isn't	 * executing. This allows canceling during early boot.	 */	if (wq_online)		__flush_work(work, true); 	if (!(cflags & WORK_CANCEL_DISABLE))		enable_work(work); 	return ret;}

Lines 4001–4500 of 8475. Previous lines Load more