/* * gcwq->lock is held and there's no work to process and no * need to manage, sleep. Workers are woken up only while * holding gcwq->lock or from local cpu, so setting the * current state before releasing gcwq->lock is enough to * prevent losing any event. */ worker_enter_idle(worker); __set_current_state(TASK_INTERRUPTIBLE); spin_unlock_irq(&gcwq->lock); schedule(); goto woke_up;} /** * rescuer_thread - the rescuer thread function * @__wq: the associated workqueue * * Workqueue rescuer thread function. There's one rescuer for each * workqueue which has WQ_RESCUER set. * * Regular work processing on a gcwq may block trying to create a new * worker which uses GFP_KERNEL allocation which has slight chance of * developing into deadlock if some works currently on the same queue * need to be processed to satisfy the GFP_KERNEL allocation. This is * the problem rescuer solves. * * When such condition is possible, the gcwq summons rescuers of all * workqueues which have works queued on the gcwq and let them process * those works so that forward progress can be guaranteed. * * This should happen rarely. */static int rescuer_thread(void *__wq){ struct workqueue_struct *wq = __wq; struct worker *rescuer = wq->rescuer; struct list_head *scheduled = &rescuer->scheduled; bool is_unbound = wq->flags & WQ_UNBOUND; unsigned int cpu; set_user_nice(current, RESCUER_NICE_LEVEL);repeat: set_current_state(TASK_INTERRUPTIBLE); if (kthread_should_stop()) return 0; /* * See whether any cpu is asking for help. Unbounded * workqueues use cpu 0 in mayday_mask for CPU_UNBOUND. */ for_each_mayday_cpu(cpu, wq->mayday_mask) { unsigned int tcpu = is_unbound ? WORK_CPU_UNBOUND : cpu; struct cpu_workqueue_struct *cwq = get_cwq(tcpu, wq); struct global_cwq *gcwq = cwq->gcwq; struct work_struct *work, *n; __set_current_state(TASK_RUNNING); mayday_clear_cpu(cpu, wq->mayday_mask); /* migrate to the target cpu if possible */ rescuer->gcwq = gcwq; worker_maybe_bind_and_lock(rescuer); /* * Slurp in all works issued via this workqueue and * process'em. */ BUG_ON(!list_empty(&rescuer->scheduled)); list_for_each_entry_safe(work, n, &gcwq->worklist, entry) if (get_work_cwq(work) == cwq) move_linked_works(work, scheduled, &n); process_scheduled_works(rescuer); /* * Leave this gcwq. If keep_working() is %true, notify a * regular worker; otherwise, we end up with 0 concurrency * and stalling the execution. */ if (keep_working(gcwq)) wake_up_worker(gcwq); spin_unlock_irq(&gcwq->lock); } schedule(); goto repeat;} struct wq_barrier { struct work_struct work; struct completion done;}; static void wq_barrier_func(struct work_struct *work){ struct wq_barrier *barr = container_of(work, struct wq_barrier, work); complete(&barr->done);} /** * insert_wq_barrier - insert a barrier work * @cwq: cwq to insert barrier into * @barr: wq_barrier to insert * @target: target work to attach @barr to * @worker: worker currently executing @target, NULL if @target is not executing * * @barr is linked to @target such that @barr is completed only after * @target finishes execution. Please note that the ordering * guarantee is observed only with respect to @target and on the local * cpu. * * Currently, a queued barrier can't be canceled. This is because * try_to_grab_pending() can't determine whether the work to be * grabbed is at the head of the queue and thus can't clear LINKED * flag of the previous work while there must be a valid next work * after a work with LINKED flag set. * * Note that when @worker is non-NULL, @target may be modified * underneath us, so we can't reliably determine cwq from @target. * * CONTEXT: * spin_lock_irq(gcwq->lock). */static void insert_wq_barrier(struct cpu_workqueue_struct *cwq, struct wq_barrier *barr, struct work_struct *target, struct worker *worker){ struct list_head *head; unsigned int linked = 0; /* * debugobject calls are safe here even with gcwq->lock locked * as we know for sure that this will not trigger any of the * checks and call back into the fixup functions where we * might deadlock. */ INIT_WORK_ONSTACK(&barr->work, wq_barrier_func); __set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work)); init_completion(&barr->done); /* * If @target is currently being executed, schedule the * barrier to the worker; otherwise, put it after @target. */ if (worker) head = worker->scheduled.next; else { unsigned long *bits = work_data_bits(target); head = target->entry.next; /* there can already be other linked works, inherit and set */ linked = *bits & WORK_STRUCT_LINKED; __set_bit(WORK_STRUCT_LINKED_BIT, bits); } debug_work_activate(&barr->work); insert_work(cwq, &barr->work, head, work_color_to_flags(WORK_NO_COLOR) | linked);} /** * flush_workqueue_prep_cwqs - prepare cwqs for workqueue flushing * @wq: workqueue being flushed * @flush_color: new flush color, < 0 for no-op * @work_color: new work color, < 0 for no-op * * Prepare cwqs for workqueue flushing. * * If @flush_color is non-negative, flush_color on all cwqs should be * -1. If no cwq has in-flight commands at the specified color, all * cwq->flush_color's stay at -1 and %false is returned. If any cwq * has in flight commands, its cwq->flush_color is set to * @flush_color, @wq->nr_cwqs_to_flush is updated accordingly, cwq * wakeup logic is armed and %true is returned. * * The caller should have initialized @wq->first_flusher prior to * calling this function with non-negative @flush_color. If * @flush_color is negative, no flush color update is done and %false * is returned. * * If @work_color is non-negative, all cwqs should have the same * work_color which is previous to @work_color and all will be * advanced to @work_color. * * CONTEXT: * mutex_lock(wq->flush_mutex). * * RETURNS: * %true if @flush_color >= 0 and there's something to flush. %false * otherwise. */static bool flush_workqueue_prep_cwqs(struct workqueue_struct *wq, int flush_color, int work_color){ bool wait = false; unsigned int cpu; if (flush_color >= 0) { BUG_ON(atomic_read(&wq->nr_cwqs_to_flush)); atomic_set(&wq->nr_cwqs_to_flush, 1); } for_each_cwq_cpu(cpu, wq) { struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); struct global_cwq *gcwq = cwq->gcwq; spin_lock_irq(&gcwq->lock); if (flush_color >= 0) { BUG_ON(cwq->flush_color != -1); if (cwq->nr_in_flight[flush_color]) { cwq->flush_color = flush_color; atomic_inc(&wq->nr_cwqs_to_flush); wait = true; } } if (work_color >= 0) { BUG_ON(work_color != work_next_color(cwq->work_color)); cwq->work_color = work_color; } spin_unlock_irq(&gcwq->lock); } if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_cwqs_to_flush)) complete(&wq->first_flusher->done); return wait;} /** * flush_workqueue - ensure that any scheduled work has run to completion. * @wq: workqueue to flush * * Forces execution of the workqueue and blocks until its completion. * This is typically used in driver shutdown handlers. * * We sleep until all works which were queued on entry have been handled, * but we are not livelocked by new incoming ones. */void flush_workqueue(struct workqueue_struct *wq){ struct wq_flusher this_flusher = { .list = LIST_HEAD_INIT(this_flusher.list), .flush_color = -1, .done = COMPLETION_INITIALIZER_ONSTACK(this_flusher.done), }; int next_color; lock_map_acquire(&wq->lockdep_map); lock_map_release(&wq->lockdep_map); mutex_lock(&wq->flush_mutex); /* * Start-to-wait phase */ next_color = work_next_color(wq->work_color); if (next_color != wq->flush_color) { /* * Color space is not full. The current work_color * becomes our flush_color and work_color is advanced * by one. */ BUG_ON(!list_empty(&wq->flusher_overflow)); this_flusher.flush_color = wq->work_color; wq->work_color = next_color; if (!wq->first_flusher) { /* no flush in progress, become the first flusher */ BUG_ON(wq->flush_color != this_flusher.flush_color); wq->first_flusher = &this_flusher; if (!flush_workqueue_prep_cwqs(wq, wq->flush_color, wq->work_color)) { /* nothing to flush, done */ wq->flush_color = next_color; wq->first_flusher = NULL; goto out_unlock; } } else { /* wait in queue */ BUG_ON(wq->flush_color == this_flusher.flush_color); list_add_tail(&this_flusher.list, &wq->flusher_queue); flush_workqueue_prep_cwqs(wq, -1, wq->work_color); } } else { /* * Oops, color space is full, wait on overflow queue. * The next flush completion will assign us * flush_color and transfer to flusher_queue. */ list_add_tail(&this_flusher.list, &wq->flusher_overflow); } mutex_unlock(&wq->flush_mutex); wait_for_completion(&this_flusher.done); /* * Wake-up-and-cascade phase * * First flushers are responsible for cascading flushes and * handling overflow. Non-first flushers can simply return. */ if (wq->first_flusher != &this_flusher) return; mutex_lock(&wq->flush_mutex); /* we might have raced, check again with mutex held */ if (wq->first_flusher != &this_flusher) goto out_unlock; wq->first_flusher = NULL; BUG_ON(!list_empty(&this_flusher.list)); BUG_ON(wq->flush_color != this_flusher.flush_color); while (true) { struct wq_flusher *next, *tmp; /* complete all the flushers sharing the current flush color */ list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) { if (next->flush_color != wq->flush_color) break; list_del_init(&next->list); complete(&next->done); } BUG_ON(!list_empty(&wq->flusher_overflow) && wq->flush_color != work_next_color(wq->work_color)); /* this flush_color is finished, advance by one */ wq->flush_color = work_next_color(wq->flush_color); /* one color has been freed, handle overflow queue */ if (!list_empty(&wq->flusher_overflow)) { /* * Assign the same color to all overflowed * flushers, advance work_color and append to * flusher_queue. This is the start-to-wait * phase for these overflowed flushers. */ list_for_each_entry(tmp, &wq->flusher_overflow, list) tmp->flush_color = wq->work_color; wq->work_color = work_next_color(wq->work_color); list_splice_tail_init(&wq->flusher_overflow, &wq->flusher_queue); flush_workqueue_prep_cwqs(wq, -1, wq->work_color); } if (list_empty(&wq->flusher_queue)) { BUG_ON(wq->flush_color != wq->work_color); break; } /* * Need to flush more colors. Make the next flusher * the new first flusher and arm cwqs. */ BUG_ON(wq->flush_color == wq->work_color); BUG_ON(wq->flush_color != next->flush_color); list_del_init(&next->list); wq->first_flusher = next; if (flush_workqueue_prep_cwqs(wq, wq->flush_color, -1)) break; /* * Meh... this color is already done, clear first * flusher and repeat cascading. */ wq->first_flusher = NULL; } out_unlock: mutex_unlock(&wq->flush_mutex);}EXPORT_SYMBOL_GPL(flush_workqueue); /** * drain_workqueue - drain a workqueue * @wq: workqueue to drain * * Wait until the workqueue becomes empty. While draining is in progress, * only chain queueing is allowed. IOW, only currently pending or running * work items on @wq can queue further work items on it. @wq is flushed * repeatedly until it becomes empty. The number of flushing is detemined * by the depth of chaining and should be relatively short. Whine if it * takes too long. */void drain_workqueue(struct workqueue_struct *wq){ unsigned int flush_cnt = 0; unsigned int cpu; /* * __queue_work() needs to test whether there are drainers, is much * hotter than drain_workqueue() and already looks at @wq->flags. * Use WQ_DRAINING so that queue doesn't have to check nr_drainers. */ spin_lock(&workqueue_lock); if (!wq->nr_drainers++) wq->flags |= WQ_DRAINING; spin_unlock(&workqueue_lock);reflush: flush_workqueue(wq); for_each_cwq_cpu(cpu, wq) { struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); bool drained; spin_lock_irq(&cwq->gcwq->lock); drained = !cwq->nr_active && list_empty(&cwq->delayed_works); spin_unlock_irq(&cwq->gcwq->lock); if (drained) continue; if (++flush_cnt == 10 || (flush_cnt % 100 == 0 && flush_cnt <= 1000)) pr_warning("workqueue %s: flush on destruction isn't complete after %u tries\n", wq->name, flush_cnt); goto reflush; } spin_lock(&workqueue_lock); if (!--wq->nr_drainers) wq->flags &= ~WQ_DRAINING; spin_unlock(&workqueue_lock);}EXPORT_SYMBOL_GPL(drain_workqueue); static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr, bool wait_executing){ struct worker *worker = NULL; struct global_cwq *gcwq; struct cpu_workqueue_struct *cwq; might_sleep(); gcwq = get_work_gcwq(work); if (!gcwq) return false; spin_lock_irq(&gcwq->lock); if (!list_empty(&work->entry)) { /* * See the comment near try_to_grab_pending()->smp_rmb(). * If it was re-queued to a different gcwq under us, we * are not going to wait. */ smp_rmb(); cwq = get_work_cwq(work); if (unlikely(!cwq || gcwq != cwq->gcwq)) goto already_gone; } else if (wait_executing) { worker = find_worker_executing_work(gcwq, work); if (!worker) goto already_gone; cwq = worker->current_cwq; } else goto already_gone; insert_wq_barrier(cwq, barr, work, worker); spin_unlock_irq(&gcwq->lock); /* * If @max_active is 1 or rescuer is in use, flushing another work * item on the same workqueue may lead to deadlock. Make sure the * flusher is not running on the same workqueue by verifying write * access. */ if (cwq->wq->saved_max_active == 1 || cwq->wq->flags & WQ_RESCUER) lock_map_acquire(&cwq->wq->lockdep_map); else lock_map_acquire_read(&cwq->wq->lockdep_map); lock_map_release(&cwq->wq->lockdep_map); return true;already_gone: spin_unlock_irq(&gcwq->lock); return false;} /** * flush_work - wait for a work to finish executing the last queueing instance * @work: the work to flush *Lines 2001–2500 of 3842. Previous lines Load more
/* * gcwq->lock is held and there's no work to process and no * need to manage, sleep. Workers are woken up only while * holding gcwq->lock or from local cpu, so setting the * current state before releasing gcwq->lock is enough to * prevent losing any event. */ worker_enter_idle(worker); __set_current_state(TASK_INTERRUPTIBLE); spin_unlock_irq(&gcwq->lock); schedule(); goto woke_up; } /** * rescuer_thread - the rescuer thread function * @__wq: the associated workqueue * * Workqueue rescuer thread function. There's one rescuer for each * workqueue which has WQ_RESCUER set. * * Regular work processing on a gcwq may block trying to create a new * worker which uses GFP_KERNEL allocation which has slight chance of * developing into deadlock if some works currently on the same queue * need to be processed to satisfy the GFP_KERNEL allocation. This is * the problem rescuer solves. * * When such condition is possible, the gcwq summons rescuers of all * workqueues which have works queued on the gcwq and let them process * those works so that forward progress can be guaranteed. * * This should happen rarely. */ static int rescuer_thread(void *__wq) { struct workqueue_struct *wq = __wq; struct worker *rescuer = wq->rescuer; struct list_head *scheduled = &rescuer->scheduled; bool is_unbound = wq->flags & WQ_UNBOUND; unsigned int cpu; set_user_nice(current, RESCUER_NICE_LEVEL); repeat: set_current_state(TASK_INTERRUPTIBLE); if (kthread_should_stop()) return 0; /* * See whether any cpu is asking for help. Unbounded * workqueues use cpu 0 in mayday_mask for CPU_UNBOUND. */ for_each_mayday_cpu(cpu, wq->mayday_mask) { unsigned int tcpu = is_unbound ? WORK_CPU_UNBOUND : cpu; struct cpu_workqueue_struct *cwq = get_cwq(tcpu, wq); struct global_cwq *gcwq = cwq->gcwq; struct work_struct *work, *n; __set_current_state(TASK_RUNNING); mayday_clear_cpu(cpu, wq->mayday_mask); /* migrate to the target cpu if possible */ rescuer->gcwq = gcwq; worker_maybe_bind_and_lock(rescuer); /* * Slurp in all works issued via this workqueue and * process'em. */ BUG_ON(!list_empty(&rescuer->scheduled)); list_for_each_entry_safe(work, n, &gcwq->worklist, entry) if (get_work_cwq(work) == cwq) move_linked_works(work, scheduled, &n); process_scheduled_works(rescuer); /* * Leave this gcwq. If keep_working() is %true, notify a * regular worker; otherwise, we end up with 0 concurrency * and stalling the execution. */ if (keep_working(gcwq)) wake_up_worker(gcwq); spin_unlock_irq(&gcwq->lock); } schedule(); goto repeat; } struct wq_barrier { struct work_struct work; struct completion done; }; static void wq_barrier_func(struct work_struct *work) { struct wq_barrier *barr = container_of(work, struct wq_barrier, work); complete(&barr->done); } /** * insert_wq_barrier - insert a barrier work * @cwq: cwq to insert barrier into * @barr: wq_barrier to insert * @target: target work to attach @barr to * @worker: worker currently executing @target, NULL if @target is not executing * * @barr is linked to @target such that @barr is completed only after * @target finishes execution. Please note that the ordering * guarantee is observed only with respect to @target and on the local * cpu. * * Currently, a queued barrier can't be canceled. This is because * try_to_grab_pending() can't determine whether the work to be * grabbed is at the head of the queue and thus can't clear LINKED * flag of the previous work while there must be a valid next work * after a work with LINKED flag set. * * Note that when @worker is non-NULL, @target may be modified * underneath us, so we can't reliably determine cwq from @target. * * CONTEXT: * spin_lock_irq(gcwq->lock). */ static void insert_wq_barrier(struct cpu_workqueue_struct *cwq, struct wq_barrier *barr, struct work_struct *target, struct worker *worker) { struct list_head *head; unsigned int linked = 0; /* * debugobject calls are safe here even with gcwq->lock locked * as we know for sure that this will not trigger any of the * checks and call back into the fixup functions where we * might deadlock. */ INIT_WORK_ONSTACK(&barr->work, wq_barrier_func); __set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work)); init_completion(&barr->done); /* * If @target is currently being executed, schedule the * barrier to the worker; otherwise, put it after @target. */ if (worker) head = worker->scheduled.next; else { unsigned long *bits = work_data_bits(target); head = target->entry.next; /* there can already be other linked works, inherit and set */ linked = *bits & WORK_STRUCT_LINKED; __set_bit(WORK_STRUCT_LINKED_BIT, bits); } debug_work_activate(&barr->work); insert_work(cwq, &barr->work, head, work_color_to_flags(WORK_NO_COLOR) | linked); } /** * flush_workqueue_prep_cwqs - prepare cwqs for workqueue flushing * @wq: workqueue being flushed * @flush_color: new flush color, < 0 for no-op * @work_color: new work color, < 0 for no-op * * Prepare cwqs for workqueue flushing. * * If @flush_color is non-negative, flush_color on all cwqs should be * -1. If no cwq has in-flight commands at the specified color, all * cwq->flush_color's stay at -1 and %false is returned. If any cwq * has in flight commands, its cwq->flush_color is set to * @flush_color, @wq->nr_cwqs_to_flush is updated accordingly, cwq * wakeup logic is armed and %true is returned. * * The caller should have initialized @wq->first_flusher prior to * calling this function with non-negative @flush_color. If * @flush_color is negative, no flush color update is done and %false * is returned. * * If @work_color is non-negative, all cwqs should have the same * work_color which is previous to @work_color and all will be * advanced to @work_color. * * CONTEXT: * mutex_lock(wq->flush_mutex). * * RETURNS: * %true if @flush_color >= 0 and there's something to flush. %false * otherwise. */ static bool flush_workqueue_prep_cwqs(struct workqueue_struct *wq, int flush_color, int work_color) { bool wait = false; unsigned int cpu; if (flush_color >= 0) { BUG_ON(atomic_read(&wq->nr_cwqs_to_flush)); atomic_set(&wq->nr_cwqs_to_flush, 1); } for_each_cwq_cpu(cpu, wq) { struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); struct global_cwq *gcwq = cwq->gcwq; spin_lock_irq(&gcwq->lock); if (flush_color >= 0) { BUG_ON(cwq->flush_color != -1); if (cwq->nr_in_flight[flush_color]) { cwq->flush_color = flush_color; atomic_inc(&wq->nr_cwqs_to_flush); wait = true; } } if (work_color >= 0) { BUG_ON(work_color != work_next_color(cwq->work_color)); cwq->work_color = work_color; } spin_unlock_irq(&gcwq->lock); } if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_cwqs_to_flush)) complete(&wq->first_flusher->done); return wait; } /** * flush_workqueue - ensure that any scheduled work has run to completion. * @wq: workqueue to flush * * Forces execution of the workqueue and blocks until its completion. * This is typically used in driver shutdown handlers. * * We sleep until all works which were queued on entry have been handled, * but we are not livelocked by new incoming ones. */ void flush_workqueue(struct workqueue_struct *wq) { struct wq_flusher this_flusher = { .list = LIST_HEAD_INIT(this_flusher.list), .flush_color = -1, .done = COMPLETION_INITIALIZER_ONSTACK(this_flusher.done), }; int next_color; lock_map_acquire(&wq->lockdep_map); lock_map_release(&wq->lockdep_map); mutex_lock(&wq->flush_mutex); /* * Start-to-wait phase */ next_color = work_next_color(wq->work_color); if (next_color != wq->flush_color) { /* * Color space is not full. The current work_color * becomes our flush_color and work_color is advanced * by one. */ BUG_ON(!list_empty(&wq->flusher_overflow)); this_flusher.flush_color = wq->work_color; wq->work_color = next_color; if (!wq->first_flusher) { /* no flush in progress, become the first flusher */ BUG_ON(wq->flush_color != this_flusher.flush_color); wq->first_flusher = &this_flusher; if (!flush_workqueue_prep_cwqs(wq, wq->flush_color, wq->work_color)) { /* nothing to flush, done */ wq->flush_color = next_color; wq->first_flusher = NULL; goto out_unlock; } } else { /* wait in queue */ BUG_ON(wq->flush_color == this_flusher.flush_color); list_add_tail(&this_flusher.list, &wq->flusher_queue); flush_workqueue_prep_cwqs(wq, -1, wq->work_color); } } else { /* * Oops, color space is full, wait on overflow queue. * The next flush completion will assign us * flush_color and transfer to flusher_queue. */ list_add_tail(&this_flusher.list, &wq->flusher_overflow); } mutex_unlock(&wq->flush_mutex); wait_for_completion(&this_flusher.done); /* * Wake-up-and-cascade phase * * First flushers are responsible for cascading flushes and * handling overflow. Non-first flushers can simply return. */ if (wq->first_flusher != &this_flusher) return; mutex_lock(&wq->flush_mutex); /* we might have raced, check again with mutex held */ if (wq->first_flusher != &this_flusher) goto out_unlock; wq->first_flusher = NULL; BUG_ON(!list_empty(&this_flusher.list)); BUG_ON(wq->flush_color != this_flusher.flush_color); while (true) { struct wq_flusher *next, *tmp; /* complete all the flushers sharing the current flush color */ list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) { if (next->flush_color != wq->flush_color) break; list_del_init(&next->list); complete(&next->done); } BUG_ON(!list_empty(&wq->flusher_overflow) && wq->flush_color != work_next_color(wq->work_color)); /* this flush_color is finished, advance by one */ wq->flush_color = work_next_color(wq->flush_color); /* one color has been freed, handle overflow queue */ if (!list_empty(&wq->flusher_overflow)) { /* * Assign the same color to all overflowed * flushers, advance work_color and append to * flusher_queue. This is the start-to-wait * phase for these overflowed flushers. */ list_for_each_entry(tmp, &wq->flusher_overflow, list) tmp->flush_color = wq->work_color; wq->work_color = work_next_color(wq->work_color); list_splice_tail_init(&wq->flusher_overflow, &wq->flusher_queue); flush_workqueue_prep_cwqs(wq, -1, wq->work_color); } if (list_empty(&wq->flusher_queue)) { BUG_ON(wq->flush_color != wq->work_color); break; } /* * Need to flush more colors. Make the next flusher * the new first flusher and arm cwqs. */ BUG_ON(wq->flush_color == wq->work_color); BUG_ON(wq->flush_color != next->flush_color); list_del_init(&next->list); wq->first_flusher = next; if (flush_workqueue_prep_cwqs(wq, wq->flush_color, -1)) break; /* * Meh... this color is already done, clear first * flusher and repeat cascading. */ wq->first_flusher = NULL; } out_unlock: mutex_unlock(&wq->flush_mutex); } EXPORT_SYMBOL_GPL(flush_workqueue); /** * drain_workqueue - drain a workqueue * @wq: workqueue to drain * * Wait until the workqueue becomes empty. While draining is in progress, * only chain queueing is allowed. IOW, only currently pending or running * work items on @wq can queue further work items on it. @wq is flushed * repeatedly until it becomes empty. The number of flushing is detemined * by the depth of chaining and should be relatively short. Whine if it * takes too long. */ void drain_workqueue(struct workqueue_struct *wq) { unsigned int flush_cnt = 0; unsigned int cpu; /* * __queue_work() needs to test whether there are drainers, is much * hotter than drain_workqueue() and already looks at @wq->flags. * Use WQ_DRAINING so that queue doesn't have to check nr_drainers. */ spin_lock(&workqueue_lock); if (!wq->nr_drainers++) wq->flags |= WQ_DRAINING; spin_unlock(&workqueue_lock); reflush: flush_workqueue(wq); for_each_cwq_cpu(cpu, wq) { struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); bool drained; spin_lock_irq(&cwq->gcwq->lock); drained = !cwq->nr_active && list_empty(&cwq->delayed_works); spin_unlock_irq(&cwq->gcwq->lock); if (drained) continue; if (++flush_cnt == 10 || (flush_cnt % 100 == 0 && flush_cnt <= 1000)) pr_warning("workqueue %s: flush on destruction isn't complete after %u tries\n", wq->name, flush_cnt); goto reflush; } spin_lock(&workqueue_lock); if (!--wq->nr_drainers) wq->flags &= ~WQ_DRAINING; spin_unlock(&workqueue_lock); } EXPORT_SYMBOL_GPL(drain_workqueue); static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr, bool wait_executing) { struct worker *worker = NULL; struct global_cwq *gcwq; struct cpu_workqueue_struct *cwq; might_sleep(); gcwq = get_work_gcwq(work); if (!gcwq) return false; spin_lock_irq(&gcwq->lock); if (!list_empty(&work->entry)) { /* * See the comment near try_to_grab_pending()->smp_rmb(). * If it was re-queued to a different gcwq under us, we * are not going to wait. */ smp_rmb(); cwq = get_work_cwq(work); if (unlikely(!cwq || gcwq != cwq->gcwq)) goto already_gone; } else if (wait_executing) { worker = find_worker_executing_work(gcwq, work); if (!worker) goto already_gone; cwq = worker->current_cwq; } else goto already_gone; insert_wq_barrier(cwq, barr, work, worker); spin_unlock_irq(&gcwq->lock); /* * If @max_active is 1 or rescuer is in use, flushing another work * item on the same workqueue may lead to deadlock. Make sure the * flusher is not running on the same workqueue by verifying write * access. */ if (cwq->wq->saved_max_active == 1 || cwq->wq->flags & WQ_RESCUER) lock_map_acquire(&cwq->wq->lockdep_map); else lock_map_acquire_read(&cwq->wq->lockdep_map); lock_map_release(&cwq->wq->lockdep_map); return true; already_gone: spin_unlock_irq(&gcwq->lock); return false; } /** * flush_work - wait for a work to finish executing the last queueing instance * @work: the work to flush *