5816 lines · 159.4 KB · C
RawDownload
/** * insert_wq_barrier - insert a barrier work * @pwq: pwq to insert barrier into * @barr: wq_barrier to insert * @target: target work to attach @barr to * @worker: worker currently executing @target, NULL if @target is not executing * * @barr is linked to @target such that @barr is completed only after * @target finishes execution.  Please note that the ordering * guarantee is observed only with respect to @target and on the local * cpu. * * Currently, a queued barrier can't be canceled.  This is because * try_to_grab_pending() can't determine whether the work to be * grabbed is at the head of the queue and thus can't clear LINKED * flag of the previous work while there must be a valid next work * after a work with LINKED flag set. * * Note that when @worker is non-NULL, @target may be modified * underneath us, so we can't reliably determine pwq from @target. * * CONTEXT: * spin_lock_irq(pool->lock). */static void insert_wq_barrier(struct pool_workqueue *pwq,			      struct wq_barrier *barr,			      struct work_struct *target, struct worker *worker){	struct list_head *head;	unsigned int linked = 0; 	/*	 * debugobject calls are safe here even with pool->lock locked	 * as we know for sure that this will not trigger any of the	 * checks and call back into the fixup functions where we	 * might deadlock.	 */	INIT_WORK_ONSTACK(&barr->work, wq_barrier_func);	__set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work)); 	init_completion_map(&barr->done, &target->lockdep_map); 	barr->task = current; 	/*	 * If @target is currently being executed, schedule the	 * barrier to the worker; otherwise, put it after @target.	 */	if (worker)		head = worker->scheduled.next;	else {		unsigned long *bits = work_data_bits(target); 		head = target->entry.next;		/* there can already be other linked works, inherit and set */		linked = *bits & WORK_STRUCT_LINKED;		__set_bit(WORK_STRUCT_LINKED_BIT, bits);	} 	debug_work_activate(&barr->work);	insert_work(pwq, &barr->work, head,		    work_color_to_flags(WORK_NO_COLOR) | linked);} /** * flush_workqueue_prep_pwqs - prepare pwqs for workqueue flushing * @wq: workqueue being flushed * @flush_color: new flush color, < 0 for no-op * @work_color: new work color, < 0 for no-op * * Prepare pwqs for workqueue flushing. * * If @flush_color is non-negative, flush_color on all pwqs should be * -1.  If no pwq has in-flight commands at the specified color, all * pwq->flush_color's stay at -1 and %false is returned.  If any pwq * has in flight commands, its pwq->flush_color is set to * @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq * wakeup logic is armed and %true is returned. * * The caller should have initialized @wq->first_flusher prior to * calling this function with non-negative @flush_color.  If * @flush_color is negative, no flush color update is done and %false * is returned. * * If @work_color is non-negative, all pwqs should have the same * work_color which is previous to @work_color and all will be * advanced to @work_color. * * CONTEXT: * mutex_lock(wq->mutex). * * Return: * %true if @flush_color >= 0 and there's something to flush.  %false * otherwise. */static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq,				      int flush_color, int work_color){	bool wait = false;	struct pool_workqueue *pwq; 	if (flush_color >= 0) {		WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush));		atomic_set(&wq->nr_pwqs_to_flush, 1);	} 	for_each_pwq(pwq, wq) {		struct worker_pool *pool = pwq->pool; 		spin_lock_irq(&pool->lock); 		if (flush_color >= 0) {			WARN_ON_ONCE(pwq->flush_color != -1); 			if (pwq->nr_in_flight[flush_color]) {				pwq->flush_color = flush_color;				atomic_inc(&wq->nr_pwqs_to_flush);				wait = true;			}		} 		if (work_color >= 0) {			WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));			pwq->work_color = work_color;		} 		spin_unlock_irq(&pool->lock);	} 	if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))		complete(&wq->first_flusher->done); 	return wait;} /** * flush_workqueue - ensure that any scheduled work has run to completion. * @wq: workqueue to flush * * This function sleeps until all work items which were queued on entry * have finished execution, but it is not livelocked by new incoming ones. */void flush_workqueue(struct workqueue_struct *wq){	struct wq_flusher this_flusher = {		.list = LIST_HEAD_INIT(this_flusher.list),		.flush_color = -1,		.done = COMPLETION_INITIALIZER_ONSTACK_MAP(this_flusher.done, wq->lockdep_map),	};	int next_color; 	if (WARN_ON(!wq_online))		return; 	lock_map_acquire(&wq->lockdep_map);	lock_map_release(&wq->lockdep_map); 	mutex_lock(&wq->mutex); 	/*	 * Start-to-wait phase	 */	next_color = work_next_color(wq->work_color); 	if (next_color != wq->flush_color) {		/*		 * Color space is not full.  The current work_color		 * becomes our flush_color and work_color is advanced		 * by one.		 */		WARN_ON_ONCE(!list_empty(&wq->flusher_overflow));		this_flusher.flush_color = wq->work_color;		wq->work_color = next_color; 		if (!wq->first_flusher) {			/* no flush in progress, become the first flusher */			WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color); 			wq->first_flusher = &this_flusher; 			if (!flush_workqueue_prep_pwqs(wq, wq->flush_color,						       wq->work_color)) {				/* nothing to flush, done */				wq->flush_color = next_color;				wq->first_flusher = NULL;				goto out_unlock;			}		} else {			/* wait in queue */			WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color);			list_add_tail(&this_flusher.list, &wq->flusher_queue);			flush_workqueue_prep_pwqs(wq, -1, wq->work_color);		}	} else {		/*		 * Oops, color space is full, wait on overflow queue.		 * The next flush completion will assign us		 * flush_color and transfer to flusher_queue.		 */		list_add_tail(&this_flusher.list, &wq->flusher_overflow);	} 	check_flush_dependency(wq, NULL); 	mutex_unlock(&wq->mutex); 	wait_for_completion(&this_flusher.done); 	/*	 * Wake-up-and-cascade phase	 *	 * First flushers are responsible for cascading flushes and	 * handling overflow.  Non-first flushers can simply return.	 */	if (wq->first_flusher != &this_flusher)		return; 	mutex_lock(&wq->mutex); 	/* we might have raced, check again with mutex held */	if (wq->first_flusher != &this_flusher)		goto out_unlock; 	wq->first_flusher = NULL; 	WARN_ON_ONCE(!list_empty(&this_flusher.list));	WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color); 	while (true) {		struct wq_flusher *next, *tmp; 		/* complete all the flushers sharing the current flush color */		list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) {			if (next->flush_color != wq->flush_color)				break;			list_del_init(&next->list);			complete(&next->done);		} 		WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) &&			     wq->flush_color != work_next_color(wq->work_color)); 		/* this flush_color is finished, advance by one */		wq->flush_color = work_next_color(wq->flush_color); 		/* one color has been freed, handle overflow queue */		if (!list_empty(&wq->flusher_overflow)) {			/*			 * Assign the same color to all overflowed			 * flushers, advance work_color and append to			 * flusher_queue.  This is the start-to-wait			 * phase for these overflowed flushers.			 */			list_for_each_entry(tmp, &wq->flusher_overflow, list)				tmp->flush_color = wq->work_color; 			wq->work_color = work_next_color(wq->work_color); 			list_splice_tail_init(&wq->flusher_overflow,					      &wq->flusher_queue);			flush_workqueue_prep_pwqs(wq, -1, wq->work_color);		} 		if (list_empty(&wq->flusher_queue)) {			WARN_ON_ONCE(wq->flush_color != wq->work_color);			break;		} 		/*		 * Need to flush more colors.  Make the next flusher		 * the new first flusher and arm pwqs.		 */		WARN_ON_ONCE(wq->flush_color == wq->work_color);		WARN_ON_ONCE(wq->flush_color != next->flush_color); 		list_del_init(&next->list);		wq->first_flusher = next; 		if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1))			break; 		/*		 * Meh... this color is already done, clear first		 * flusher and repeat cascading.		 */		wq->first_flusher = NULL;	} out_unlock:	mutex_unlock(&wq->mutex);}EXPORT_SYMBOL(flush_workqueue); /** * drain_workqueue - drain a workqueue * @wq: workqueue to drain * * Wait until the workqueue becomes empty.  While draining is in progress, * only chain queueing is allowed.  IOW, only currently pending or running * work items on @wq can queue further work items on it.  @wq is flushed * repeatedly until it becomes empty.  The number of flushing is determined * by the depth of chaining and should be relatively short.  Whine if it * takes too long. */void drain_workqueue(struct workqueue_struct *wq){	unsigned int flush_cnt = 0;	struct pool_workqueue *pwq; 	/*	 * __queue_work() needs to test whether there are drainers, is much	 * hotter than drain_workqueue() and already looks at @wq->flags.	 * Use __WQ_DRAINING so that queue doesn't have to check nr_drainers.	 */	mutex_lock(&wq->mutex);	if (!wq->nr_drainers++)		wq->flags |= __WQ_DRAINING;	mutex_unlock(&wq->mutex);reflush:	flush_workqueue(wq); 	mutex_lock(&wq->mutex); 	for_each_pwq(pwq, wq) {		bool drained; 		spin_lock_irq(&pwq->pool->lock);		drained = !pwq->nr_active && list_empty(&pwq->delayed_works);		spin_unlock_irq(&pwq->pool->lock); 		if (drained)			continue; 		if (++flush_cnt == 10 ||		    (flush_cnt % 100 == 0 && flush_cnt <= 1000))			pr_warn("workqueue %s: drain_workqueue() isn't complete after %u tries\n",				wq->name, flush_cnt); 		mutex_unlock(&wq->mutex);		goto reflush;	} 	if (!--wq->nr_drainers)		wq->flags &= ~__WQ_DRAINING;	mutex_unlock(&wq->mutex);}EXPORT_SYMBOL_GPL(drain_workqueue); static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr,			     bool from_cancel){	struct worker *worker = NULL;	struct worker_pool *pool;	struct pool_workqueue *pwq; 	might_sleep(); 	local_irq_disable();	pool = get_work_pool(work);	if (!pool) {		local_irq_enable();		return false;	} 	spin_lock(&pool->lock);	/* see the comment in try_to_grab_pending() with the same code */	pwq = get_work_pwq(work);	if (pwq) {		if (unlikely(pwq->pool != pool))			goto already_gone;	} else {		worker = find_worker_executing_work(pool, work);		if (!worker)			goto already_gone;		pwq = worker->current_pwq;	} 	check_flush_dependency(pwq->wq, work); 	insert_wq_barrier(pwq, barr, work, worker);	spin_unlock_irq(&pool->lock); 	/*	 * Force a lock recursion deadlock when using flush_work() inside a	 * single-threaded or rescuer equipped workqueue.	 *	 * For single threaded workqueues the deadlock happens when the work	 * is after the work issuing the flush_work(). For rescuer equipped	 * workqueues the deadlock happens when the rescuer stalls, blocking	 * forward progress.	 */	if (!from_cancel &&	    (pwq->wq->saved_max_active == 1 || pwq->wq->rescuer)) {		lock_map_acquire(&pwq->wq->lockdep_map);		lock_map_release(&pwq->wq->lockdep_map);	} 	return true;already_gone:	spin_unlock_irq(&pool->lock);	return false;} static bool __flush_work(struct work_struct *work, bool from_cancel){	struct wq_barrier barr; 	if (WARN_ON(!wq_online))		return false; 	if (!from_cancel) {		lock_map_acquire(&work->lockdep_map);		lock_map_release(&work->lockdep_map);	} 	if (start_flush_work(work, &barr, from_cancel)) {		wait_for_completion(&barr.done);		destroy_work_on_stack(&barr.work);		return true;	} else {		return false;	}} /** * flush_work - wait for a work to finish executing the last queueing instance * @work: the work to flush * * Wait until @work has finished execution.  @work is guaranteed to be idle * on return if it hasn't been requeued since flush started. * * Return: * %true if flush_work() waited for the work to finish execution, * %false if it was already idle. */bool flush_work(struct work_struct *work){	return __flush_work(work, false);}EXPORT_SYMBOL_GPL(flush_work); struct cwt_wait {	wait_queue_entry_t		wait;	struct work_struct	*work;}; static int cwt_wakefn(wait_queue_entry_t *wait, unsigned mode, int sync, void *key){	struct cwt_wait *cwait = container_of(wait, struct cwt_wait, wait); 	if (cwait->work != key)		return 0;	return autoremove_wake_function(wait, mode, sync, key);} static bool __cancel_work_timer(struct work_struct *work, bool is_dwork){	static DECLARE_WAIT_QUEUE_HEAD(cancel_waitq);	unsigned long flags;	int ret; 	do {		ret = try_to_grab_pending(work, is_dwork, &flags);		/*		 * If someone else is already canceling, wait for it to		 * finish.  flush_work() doesn't work for PREEMPT_NONE		 * because we may get scheduled between @work's completion		 * and the other canceling task resuming and clearing		 * CANCELING - flush_work() will return false immediately		 * as @work is no longer busy, try_to_grab_pending() will		 * return -ENOENT as @work is still being canceled and the		 * other canceling task won't be able to clear CANCELING as		 * we're hogging the CPU.		 *		 * Let's wait for completion using a waitqueue.  As this		 * may lead to the thundering herd problem, use a custom		 * wake function which matches @work along with exclusive		 * wait and wakeup.		 */		if (unlikely(ret == -ENOENT)) {			struct cwt_wait cwait; 			init_wait(&cwait.wait);			cwait.wait.func = cwt_wakefn;			cwait.work = work; 			prepare_to_wait_exclusive(&cancel_waitq, &cwait.wait,						  TASK_UNINTERRUPTIBLE);			if (work_is_canceling(work))				schedule();			finish_wait(&cancel_waitq, &cwait.wait);		}	} while (unlikely(ret < 0)); 	/* tell other tasks trying to grab @work to back off */	mark_work_canceling(work);	local_irq_restore(flags); 	/*	 * This allows canceling during early boot.  We know that @work

Lines 2501–3000 of 5816. Previous lines

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t