unsigned long action, void *hcpu){ unsigned int cpu = (unsigned long)hcpu; struct global_cwq *gcwq = get_gcwq(cpu); struct task_struct *new_trustee = NULL; struct worker *uninitialized_var(new_worker); unsigned long flags; action &= ~CPU_TASKS_FROZEN; switch (action) { case CPU_DOWN_PREPARE: new_trustee = kthread_create(trustee_thread, gcwq, "workqueue_trustee/%d\n", cpu); if (IS_ERR(new_trustee)) return notifier_from_errno(PTR_ERR(new_trustee)); kthread_bind(new_trustee, cpu); /* fall through */ case CPU_UP_PREPARE: BUG_ON(gcwq->first_idle); new_worker = create_worker(gcwq, false); if (!new_worker) { if (new_trustee) kthread_stop(new_trustee); return NOTIFY_BAD; } } /* some are called w/ irq disabled, don't disturb irq status */ spin_lock_irqsave(&gcwq->lock, flags); switch (action) { case CPU_DOWN_PREPARE: /* initialize trustee and tell it to acquire the gcwq */ BUG_ON(gcwq->trustee || gcwq->trustee_state != TRUSTEE_DONE); gcwq->trustee = new_trustee; gcwq->trustee_state = TRUSTEE_START; wake_up_process(gcwq->trustee); wait_trustee_state(gcwq, TRUSTEE_IN_CHARGE); /* fall through */ case CPU_UP_PREPARE: BUG_ON(gcwq->first_idle); gcwq->first_idle = new_worker; break; case CPU_DYING: /* * Before this, the trustee and all workers except for * the ones which are still executing works from * before the last CPU down must be on the cpu. After * this, they'll all be diasporas. */ gcwq->flags |= GCWQ_DISASSOCIATED; break; case CPU_POST_DEAD: gcwq->trustee_state = TRUSTEE_BUTCHER; /* fall through */ case CPU_UP_CANCELED: destroy_worker(gcwq->first_idle); gcwq->first_idle = NULL; break; case CPU_DOWN_FAILED: case CPU_ONLINE: gcwq->flags &= ~GCWQ_DISASSOCIATED; if (gcwq->trustee_state != TRUSTEE_DONE) { gcwq->trustee_state = TRUSTEE_RELEASE; wake_up_process(gcwq->trustee); wait_trustee_state(gcwq, TRUSTEE_DONE); } /* * Trustee is done and there might be no worker left. * Put the first_idle in and request a real manager to * take a look. */ spin_unlock_irq(&gcwq->lock); kthread_bind(gcwq->first_idle->task, cpu); spin_lock_irq(&gcwq->lock); gcwq->flags |= GCWQ_MANAGE_WORKERS; start_worker(gcwq->first_idle); gcwq->first_idle = NULL; break; } spin_unlock_irqrestore(&gcwq->lock, flags); return notifier_from_errno(0);} #ifdef CONFIG_SMP struct work_for_cpu { struct completion completion; long (*fn)(void *); void *arg; long ret;}; static int do_work_for_cpu(void *_wfc){ struct work_for_cpu *wfc = _wfc; wfc->ret = wfc->fn(wfc->arg); complete(&wfc->completion); return 0;} /** * work_on_cpu - run a function in user context on a particular cpu * @cpu: the cpu to run on * @fn: the function to run * @arg: the function arg * * This will return the value @fn returns. * It is up to the caller to ensure that the cpu doesn't go offline. * The caller must not hold any locks which would prevent @fn from completing. */long work_on_cpu(unsigned int cpu, long (*fn)(void *), void *arg){ struct task_struct *sub_thread; struct work_for_cpu wfc = { .completion = COMPLETION_INITIALIZER_ONSTACK(wfc.completion), .fn = fn, .arg = arg, }; sub_thread = kthread_create(do_work_for_cpu, &wfc, "work_for_cpu"); if (IS_ERR(sub_thread)) return PTR_ERR(sub_thread); kthread_bind(sub_thread, cpu); wake_up_process(sub_thread); wait_for_completion(&wfc.completion); return wfc.ret;}EXPORT_SYMBOL_GPL(work_on_cpu);#endif /* CONFIG_SMP */ #ifdef CONFIG_FREEZER /** * freeze_workqueues_begin - begin freezing workqueues * * Start freezing workqueues. After this function returns, all freezable * workqueues will queue new works to their frozen_works list instead of * gcwq->worklist. * * CONTEXT: * Grabs and releases workqueue_lock and gcwq->lock's. */void freeze_workqueues_begin(void){ unsigned int cpu; spin_lock(&workqueue_lock); BUG_ON(workqueue_freezing); workqueue_freezing = true; for_each_gcwq_cpu(cpu) { struct global_cwq *gcwq = get_gcwq(cpu); struct workqueue_struct *wq; spin_lock_irq(&gcwq->lock); BUG_ON(gcwq->flags & GCWQ_FREEZING); gcwq->flags |= GCWQ_FREEZING; list_for_each_entry(wq, &workqueues, list) { struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); if (cwq && wq->flags & WQ_FREEZABLE) cwq->max_active = 0; } spin_unlock_irq(&gcwq->lock); } spin_unlock(&workqueue_lock);} /** * freeze_workqueues_busy - are freezable workqueues still busy? * * Check whether freezing is complete. This function must be called * between freeze_workqueues_begin() and thaw_workqueues(). * * CONTEXT: * Grabs and releases workqueue_lock. * * RETURNS: * %true if some freezable workqueues are still busy. %false if freezing * is complete. */bool freeze_workqueues_busy(void){ unsigned int cpu; bool busy = false; spin_lock(&workqueue_lock); BUG_ON(!workqueue_freezing); for_each_gcwq_cpu(cpu) { struct workqueue_struct *wq; /* * nr_active is monotonically decreasing. It's safe * to peek without lock. */ list_for_each_entry(wq, &workqueues, list) { struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); if (!cwq || !(wq->flags & WQ_FREEZABLE)) continue; BUG_ON(cwq->nr_active < 0); if (cwq->nr_active) { busy = true; goto out_unlock; } } }out_unlock: spin_unlock(&workqueue_lock); return busy;} /** * thaw_workqueues - thaw workqueues * * Thaw workqueues. Normal queueing is restored and all collected * frozen works are transferred to their respective gcwq worklists. * * CONTEXT: * Grabs and releases workqueue_lock and gcwq->lock's. */void thaw_workqueues(void){ unsigned int cpu; spin_lock(&workqueue_lock); if (!workqueue_freezing) goto out_unlock; for_each_gcwq_cpu(cpu) { struct global_cwq *gcwq = get_gcwq(cpu); struct workqueue_struct *wq; spin_lock_irq(&gcwq->lock); BUG_ON(!(gcwq->flags & GCWQ_FREEZING)); gcwq->flags &= ~GCWQ_FREEZING; list_for_each_entry(wq, &workqueues, list) { struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); if (!cwq || !(wq->flags & WQ_FREEZABLE)) continue; /* restore max_active and repopulate worklist */ cwq->max_active = wq->saved_max_active; while (!list_empty(&cwq->delayed_works) && cwq->nr_active < cwq->max_active) cwq_activate_first_delayed(cwq); } wake_up_worker(gcwq); spin_unlock_irq(&gcwq->lock); } workqueue_freezing = false;out_unlock: spin_unlock(&workqueue_lock);}#endif /* CONFIG_FREEZER */ static int __init init_workqueues(void){ unsigned int cpu; int i; cpu_notifier(workqueue_cpu_callback, CPU_PRI_WORKQUEUE); /* initialize gcwqs */ for_each_gcwq_cpu(cpu) { struct global_cwq *gcwq = get_gcwq(cpu); spin_lock_init(&gcwq->lock); INIT_LIST_HEAD(&gcwq->worklist); gcwq->cpu = cpu; gcwq->flags |= GCWQ_DISASSOCIATED; INIT_LIST_HEAD(&gcwq->idle_list); for (i = 0; i < BUSY_WORKER_HASH_SIZE; i++) INIT_HLIST_HEAD(&gcwq->busy_hash[i]); init_timer_deferrable(&gcwq->idle_timer); gcwq->idle_timer.function = idle_worker_timeout; gcwq->idle_timer.data = (unsigned long)gcwq; setup_timer(&gcwq->mayday_timer, gcwq_mayday_timeout, (unsigned long)gcwq); ida_init(&gcwq->worker_ida); gcwq->trustee_state = TRUSTEE_DONE; init_waitqueue_head(&gcwq->trustee_wait); } /* create the initial worker */ for_each_online_gcwq_cpu(cpu) { struct global_cwq *gcwq = get_gcwq(cpu); struct worker *worker; if (cpu != WORK_CPU_UNBOUND) gcwq->flags &= ~GCWQ_DISASSOCIATED; worker = create_worker(gcwq, true); BUG_ON(!worker); spin_lock_irq(&gcwq->lock); start_worker(worker); spin_unlock_irq(&gcwq->lock); } system_wq = alloc_workqueue("events", 0, 0); system_long_wq = alloc_workqueue("events_long", 0, 0); system_nrt_wq = alloc_workqueue("events_nrt", WQ_NON_REENTRANT, 0); system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND, WQ_UNBOUND_MAX_ACTIVE); system_freezable_wq = alloc_workqueue("events_freezable", WQ_FREEZABLE, 0); system_nrt_freezable_wq = alloc_workqueue("events_nrt_freezable", WQ_NON_REENTRANT | WQ_FREEZABLE, 0); BUG_ON(!system_wq || !system_long_wq || !system_nrt_wq || !system_unbound_wq || !system_freezable_wq || !system_nrt_freezable_wq); return 0;}early_initcall(init_workqueues);Lines 3501–3842 of 3842. Previous lines
unsigned long action, void *hcpu) { unsigned int cpu = (unsigned long)hcpu; struct global_cwq *gcwq = get_gcwq(cpu); struct task_struct *new_trustee = NULL; struct worker *uninitialized_var(new_worker); unsigned long flags; action &= ~CPU_TASKS_FROZEN; switch (action) { case CPU_DOWN_PREPARE: new_trustee = kthread_create(trustee_thread, gcwq, "workqueue_trustee/%d\n", cpu); if (IS_ERR(new_trustee)) return notifier_from_errno(PTR_ERR(new_trustee)); kthread_bind(new_trustee, cpu); /* fall through */ case CPU_UP_PREPARE: BUG_ON(gcwq->first_idle); new_worker = create_worker(gcwq, false); if (!new_worker) { if (new_trustee) kthread_stop(new_trustee); return NOTIFY_BAD; } } /* some are called w/ irq disabled, don't disturb irq status */ spin_lock_irqsave(&gcwq->lock, flags); switch (action) { case CPU_DOWN_PREPARE: /* initialize trustee and tell it to acquire the gcwq */ BUG_ON(gcwq->trustee || gcwq->trustee_state != TRUSTEE_DONE); gcwq->trustee = new_trustee; gcwq->trustee_state = TRUSTEE_START; wake_up_process(gcwq->trustee); wait_trustee_state(gcwq, TRUSTEE_IN_CHARGE); /* fall through */ case CPU_UP_PREPARE: BUG_ON(gcwq->first_idle); gcwq->first_idle = new_worker; break; case CPU_DYING: /* * Before this, the trustee and all workers except for * the ones which are still executing works from * before the last CPU down must be on the cpu. After * this, they'll all be diasporas. */ gcwq->flags |= GCWQ_DISASSOCIATED; break; case CPU_POST_DEAD: gcwq->trustee_state = TRUSTEE_BUTCHER; /* fall through */ case CPU_UP_CANCELED: destroy_worker(gcwq->first_idle); gcwq->first_idle = NULL; break; case CPU_DOWN_FAILED: case CPU_ONLINE: gcwq->flags &= ~GCWQ_DISASSOCIATED; if (gcwq->trustee_state != TRUSTEE_DONE) { gcwq->trustee_state = TRUSTEE_RELEASE; wake_up_process(gcwq->trustee); wait_trustee_state(gcwq, TRUSTEE_DONE); } /* * Trustee is done and there might be no worker left. * Put the first_idle in and request a real manager to * take a look. */ spin_unlock_irq(&gcwq->lock); kthread_bind(gcwq->first_idle->task, cpu); spin_lock_irq(&gcwq->lock); gcwq->flags |= GCWQ_MANAGE_WORKERS; start_worker(gcwq->first_idle); gcwq->first_idle = NULL; break; } spin_unlock_irqrestore(&gcwq->lock, flags); return notifier_from_errno(0); } #ifdef CONFIG_SMP struct work_for_cpu { struct completion completion; long (*fn)(void *); void *arg; long ret; }; static int do_work_for_cpu(void *_wfc) { struct work_for_cpu *wfc = _wfc; wfc->ret = wfc->fn(wfc->arg); complete(&wfc->completion); return 0; } /** * work_on_cpu - run a function in user context on a particular cpu * @cpu: the cpu to run on * @fn: the function to run * @arg: the function arg * * This will return the value @fn returns. * It is up to the caller to ensure that the cpu doesn't go offline. * The caller must not hold any locks which would prevent @fn from completing. */ long work_on_cpu(unsigned int cpu, long (*fn)(void *), void *arg) { struct task_struct *sub_thread; struct work_for_cpu wfc = { .completion = COMPLETION_INITIALIZER_ONSTACK(wfc.completion), .fn = fn, .arg = arg, }; sub_thread = kthread_create(do_work_for_cpu, &wfc, "work_for_cpu"); if (IS_ERR(sub_thread)) return PTR_ERR(sub_thread); kthread_bind(sub_thread, cpu); wake_up_process(sub_thread); wait_for_completion(&wfc.completion); return wfc.ret; } EXPORT_SYMBOL_GPL(work_on_cpu); #endif /* CONFIG_SMP */ #ifdef CONFIG_FREEZER /** * freeze_workqueues_begin - begin freezing workqueues * * Start freezing workqueues. After this function returns, all freezable * workqueues will queue new works to their frozen_works list instead of * gcwq->worklist. * * CONTEXT: * Grabs and releases workqueue_lock and gcwq->lock's. */ void freeze_workqueues_begin(void) { unsigned int cpu; spin_lock(&workqueue_lock); BUG_ON(workqueue_freezing); workqueue_freezing = true; for_each_gcwq_cpu(cpu) { struct global_cwq *gcwq = get_gcwq(cpu); struct workqueue_struct *wq; spin_lock_irq(&gcwq->lock); BUG_ON(gcwq->flags & GCWQ_FREEZING); gcwq->flags |= GCWQ_FREEZING; list_for_each_entry(wq, &workqueues, list) { struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); if (cwq && wq->flags & WQ_FREEZABLE) cwq->max_active = 0; } spin_unlock_irq(&gcwq->lock); } spin_unlock(&workqueue_lock); } /** * freeze_workqueues_busy - are freezable workqueues still busy? * * Check whether freezing is complete. This function must be called * between freeze_workqueues_begin() and thaw_workqueues(). * * CONTEXT: * Grabs and releases workqueue_lock. * * RETURNS: * %true if some freezable workqueues are still busy. %false if freezing * is complete. */ bool freeze_workqueues_busy(void) { unsigned int cpu; bool busy = false; spin_lock(&workqueue_lock); BUG_ON(!workqueue_freezing); for_each_gcwq_cpu(cpu) { struct workqueue_struct *wq; /* * nr_active is monotonically decreasing. It's safe * to peek without lock. */ list_for_each_entry(wq, &workqueues, list) { struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); if (!cwq || !(wq->flags & WQ_FREEZABLE)) continue; BUG_ON(cwq->nr_active < 0); if (cwq->nr_active) { busy = true; goto out_unlock; } } } out_unlock: spin_unlock(&workqueue_lock); return busy; } /** * thaw_workqueues - thaw workqueues * * Thaw workqueues. Normal queueing is restored and all collected * frozen works are transferred to their respective gcwq worklists. * * CONTEXT: * Grabs and releases workqueue_lock and gcwq->lock's. */ void thaw_workqueues(void) { unsigned int cpu; spin_lock(&workqueue_lock); if (!workqueue_freezing) goto out_unlock; for_each_gcwq_cpu(cpu) { struct global_cwq *gcwq = get_gcwq(cpu); struct workqueue_struct *wq; spin_lock_irq(&gcwq->lock); BUG_ON(!(gcwq->flags & GCWQ_FREEZING)); gcwq->flags &= ~GCWQ_FREEZING; list_for_each_entry(wq, &workqueues, list) { struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); if (!cwq || !(wq->flags & WQ_FREEZABLE)) continue; /* restore max_active and repopulate worklist */ cwq->max_active = wq->saved_max_active; while (!list_empty(&cwq->delayed_works) && cwq->nr_active < cwq->max_active) cwq_activate_first_delayed(cwq); } wake_up_worker(gcwq); spin_unlock_irq(&gcwq->lock); } workqueue_freezing = false; out_unlock: spin_unlock(&workqueue_lock); } #endif /* CONFIG_FREEZER */ static int __init init_workqueues(void) { unsigned int cpu; int i; cpu_notifier(workqueue_cpu_callback, CPU_PRI_WORKQUEUE); /* initialize gcwqs */ for_each_gcwq_cpu(cpu) { struct global_cwq *gcwq = get_gcwq(cpu); spin_lock_init(&gcwq->lock); INIT_LIST_HEAD(&gcwq->worklist); gcwq->cpu = cpu; gcwq->flags |= GCWQ_DISASSOCIATED; INIT_LIST_HEAD(&gcwq->idle_list); for (i = 0; i < BUSY_WORKER_HASH_SIZE; i++) INIT_HLIST_HEAD(&gcwq->busy_hash[i]); init_timer_deferrable(&gcwq->idle_timer); gcwq->idle_timer.function = idle_worker_timeout; gcwq->idle_timer.data = (unsigned long)gcwq; setup_timer(&gcwq->mayday_timer, gcwq_mayday_timeout, (unsigned long)gcwq); ida_init(&gcwq->worker_ida); gcwq->trustee_state = TRUSTEE_DONE; init_waitqueue_head(&gcwq->trustee_wait); } /* create the initial worker */ for_each_online_gcwq_cpu(cpu) { struct global_cwq *gcwq = get_gcwq(cpu); struct worker *worker; if (cpu != WORK_CPU_UNBOUND) gcwq->flags &= ~GCWQ_DISASSOCIATED; worker = create_worker(gcwq, true); BUG_ON(!worker); spin_lock_irq(&gcwq->lock); start_worker(worker); spin_unlock_irq(&gcwq->lock); } system_wq = alloc_workqueue("events", 0, 0); system_long_wq = alloc_workqueue("events_long", 0, 0); system_nrt_wq = alloc_workqueue("events_nrt", WQ_NON_REENTRANT, 0); system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND, WQ_UNBOUND_MAX_ACTIVE); system_freezable_wq = alloc_workqueue("events_freezable", WQ_FREEZABLE, 0); system_nrt_freezable_wq = alloc_workqueue("events_nrt_freezable", WQ_NON_REENTRANT | WQ_FREEZABLE, 0); BUG_ON(!system_wq || !system_long_wq || !system_nrt_wq || !system_unbound_wq || !system_freezable_wq || !system_nrt_freezable_wq); return 0; } early_initcall(init_workqueues);