/* * It's multi cpu. If @wq is non-reentrant and @work * was previously on a different cpu, it might still * be running there, in which case the work needs to * be queued on that cpu to guarantee non-reentrance. */ gcwq = get_gcwq(cpu); if (wq->flags & WQ_NON_REENTRANT && (last_gcwq = get_work_gcwq(work)) && last_gcwq != gcwq) { struct worker *worker; spin_lock_irqsave(&last_gcwq->lock, flags); worker = find_worker_executing_work(last_gcwq, work); if (worker && worker->current_cwq->wq == wq) gcwq = last_gcwq; else { /* meh... not running there, queue here */ spin_unlock_irqrestore(&last_gcwq->lock, flags); spin_lock_irqsave(&gcwq->lock, flags); } } else spin_lock_irqsave(&gcwq->lock, flags); } else { gcwq = get_gcwq(WORK_CPU_UNBOUND); spin_lock_irqsave(&gcwq->lock, flags); } /* gcwq determined, get cwq and queue */ cwq = get_cwq(gcwq->cpu, wq); trace_workqueue_queue_work(cpu, cwq, work); if (WARN_ON(!list_empty(&work->entry))) { spin_unlock_irqrestore(&gcwq->lock, flags); return; } cwq->nr_in_flight[cwq->work_color]++; work_flags = work_color_to_flags(cwq->work_color); if (likely(cwq->nr_active < cwq->max_active)) { trace_workqueue_activate_work(work); cwq->nr_active++; worklist = gcwq_determine_ins_pos(gcwq, cwq); } else { work_flags |= WORK_STRUCT_DELAYED; worklist = &cwq->delayed_works; } insert_work(cwq, work, worklist, work_flags); spin_unlock_irqrestore(&gcwq->lock, flags);} /** * queue_work - queue work on a workqueue * @wq: workqueue to use * @work: work to queue * * Returns 0 if @work was already on a queue, non-zero otherwise. * * We queue the work to the CPU on which it was submitted, but if the CPU dies * it can be processed by another CPU. */int queue_work(struct workqueue_struct *wq, struct work_struct *work){ int ret; ret = queue_work_on(get_cpu(), wq, work); put_cpu(); return ret;}EXPORT_SYMBOL_GPL(queue_work); /** * queue_work_on - queue work on specific cpu * @cpu: CPU number to execute work on * @wq: workqueue to use * @work: work to queue * * Returns 0 if @work was already on a queue, non-zero otherwise. * * We queue the work to a specific CPU, the caller must ensure it * can't go away. */intqueue_work_on(int cpu, struct workqueue_struct *wq, struct work_struct *work){ int ret = 0; if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) { __queue_work(cpu, wq, work); ret = 1; } return ret;}EXPORT_SYMBOL_GPL(queue_work_on); static void delayed_work_timer_fn(unsigned long __data){ struct delayed_work *dwork = (struct delayed_work *)__data; struct cpu_workqueue_struct *cwq = get_work_cwq(&dwork->work); __queue_work(smp_processor_id(), cwq->wq, &dwork->work);} /** * queue_delayed_work - queue work on a workqueue after delay * @wq: workqueue to use * @dwork: delayable work to queue * @delay: number of jiffies to wait before queueing * * Returns 0 if @work was already on a queue, non-zero otherwise. */int queue_delayed_work(struct workqueue_struct *wq, struct delayed_work *dwork, unsigned long delay){ if (delay == 0) return queue_work(wq, &dwork->work); return queue_delayed_work_on(-1, wq, dwork, delay);}EXPORT_SYMBOL_GPL(queue_delayed_work); /** * queue_delayed_work_on - queue work on specific CPU after delay * @cpu: CPU number to execute work on * @wq: workqueue to use * @dwork: work to queue * @delay: number of jiffies to wait before queueing * * Returns 0 if @work was already on a queue, non-zero otherwise. */int queue_delayed_work_on(int cpu, struct workqueue_struct *wq, struct delayed_work *dwork, unsigned long delay){ int ret = 0; struct timer_list *timer = &dwork->timer; struct work_struct *work = &dwork->work; if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) { unsigned int lcpu; BUG_ON(timer_pending(timer)); BUG_ON(!list_empty(&work->entry)); timer_stats_timer_set_start_info(&dwork->timer); /* * This stores cwq for the moment, for the timer_fn. * Note that the work's gcwq is preserved to allow * reentrance detection for delayed works. */ if (!(wq->flags & WQ_UNBOUND)) { struct global_cwq *gcwq = get_work_gcwq(work); if (gcwq && gcwq->cpu != WORK_CPU_UNBOUND) lcpu = gcwq->cpu; else lcpu = raw_smp_processor_id(); } else lcpu = WORK_CPU_UNBOUND; set_work_cwq(work, get_cwq(lcpu, wq), 0); timer->expires = jiffies + delay; timer->data = (unsigned long)dwork; timer->function = delayed_work_timer_fn; if (unlikely(cpu >= 0)) add_timer_on(timer, cpu); else add_timer(timer); ret = 1; } return ret;}EXPORT_SYMBOL_GPL(queue_delayed_work_on); /** * worker_enter_idle - enter idle state * @worker: worker which is entering idle state * * @worker is entering idle state. Update stats and idle timer if * necessary. * * LOCKING: * spin_lock_irq(gcwq->lock). */static void worker_enter_idle(struct worker *worker){ struct global_cwq *gcwq = worker->gcwq; BUG_ON(worker->flags & WORKER_IDLE); BUG_ON(!list_empty(&worker->entry) && (worker->hentry.next || worker->hentry.pprev)); /* can't use worker_set_flags(), also called from start_worker() */ worker->flags |= WORKER_IDLE; gcwq->nr_idle++; worker->last_active = jiffies; /* idle_list is LIFO */ list_add(&worker->entry, &gcwq->idle_list); if (likely(!(worker->flags & WORKER_ROGUE))) { if (too_many_workers(gcwq) && !timer_pending(&gcwq->idle_timer)) mod_timer(&gcwq->idle_timer, jiffies + IDLE_WORKER_TIMEOUT); } else wake_up_all(&gcwq->trustee_wait); /* * Sanity check nr_running. Because trustee releases gcwq->lock * between setting %WORKER_ROGUE and zapping nr_running, the * warning may trigger spuriously. Check iff trustee is idle. */ WARN_ON_ONCE(gcwq->trustee_state == TRUSTEE_DONE && gcwq->nr_workers == gcwq->nr_idle && atomic_read(get_gcwq_nr_running(gcwq->cpu)));} /** * worker_leave_idle - leave idle state * @worker: worker which is leaving idle state * * @worker is leaving idle state. Update stats. * * LOCKING: * spin_lock_irq(gcwq->lock). */static void worker_leave_idle(struct worker *worker){ struct global_cwq *gcwq = worker->gcwq; BUG_ON(!(worker->flags & WORKER_IDLE)); worker_clr_flags(worker, WORKER_IDLE); gcwq->nr_idle--; list_del_init(&worker->entry);} /** * worker_maybe_bind_and_lock - bind worker to its cpu if possible and lock gcwq * @worker: self * * Works which are scheduled while the cpu is online must at least be * scheduled to a worker which is bound to the cpu so that if they are * flushed from cpu callbacks while cpu is going down, they are * guaranteed to execute on the cpu. * * This function is to be used by rogue workers and rescuers to bind * themselves to the target cpu and may race with cpu going down or * coming online. kthread_bind() can't be used because it may put the * worker to already dead cpu and set_cpus_allowed_ptr() can't be used * verbatim as it's best effort and blocking and gcwq may be * [dis]associated in the meantime. * * This function tries set_cpus_allowed() and locks gcwq and verifies * the binding against GCWQ_DISASSOCIATED which is set during * CPU_DYING and cleared during CPU_ONLINE, so if the worker enters * idle state or fetches works without dropping lock, it can guarantee * the scheduling requirement described in the first paragraph. * * CONTEXT: * Might sleep. Called without any lock but returns with gcwq->lock * held. * * RETURNS: * %true if the associated gcwq is online (@worker is successfully * bound), %false if offline. */static bool worker_maybe_bind_and_lock(struct worker *worker)__acquires(&gcwq->lock){ struct global_cwq *gcwq = worker->gcwq; struct task_struct *task = worker->task; while (true) { /* * The following call may fail, succeed or succeed * without actually migrating the task to the cpu if * it races with cpu hotunplug operation. Verify * against GCWQ_DISASSOCIATED. */ if (!(gcwq->flags & GCWQ_DISASSOCIATED)) set_cpus_allowed_ptr(task, get_cpu_mask(gcwq->cpu)); spin_lock_irq(&gcwq->lock); if (gcwq->flags & GCWQ_DISASSOCIATED) return false; if (task_cpu(task) == gcwq->cpu && cpumask_equal(¤t->cpus_allowed, get_cpu_mask(gcwq->cpu))) return true; spin_unlock_irq(&gcwq->lock); /* * We've raced with CPU hot[un]plug. Give it a breather * and retry migration. cond_resched() is required here; * otherwise, we might deadlock against cpu_stop trying to * bring down the CPU on non-preemptive kernel. */ cpu_relax(); cond_resched(); }} /* * Function for worker->rebind_work used to rebind rogue busy workers * to the associated cpu which is coming back online. This is * scheduled by cpu up but can race with other cpu hotplug operations * and may be executed twice without intervening cpu down. */static void worker_rebind_fn(struct work_struct *work){ struct worker *worker = container_of(work, struct worker, rebind_work); struct global_cwq *gcwq = worker->gcwq; if (worker_maybe_bind_and_lock(worker)) worker_clr_flags(worker, WORKER_REBIND); spin_unlock_irq(&gcwq->lock);} static struct worker *alloc_worker(void){ struct worker *worker; worker = kzalloc(sizeof(*worker), GFP_KERNEL); if (worker) { INIT_LIST_HEAD(&worker->entry); INIT_LIST_HEAD(&worker->scheduled); INIT_WORK(&worker->rebind_work, worker_rebind_fn); /* on creation a worker is in !idle && prep state */ worker->flags = WORKER_PREP; } return worker;} /** * create_worker - create a new workqueue worker * @gcwq: gcwq the new worker will belong to * @bind: whether to set affinity to @cpu or not * * Create a new worker which is bound to @gcwq. The returned worker * can be started by calling start_worker() or destroyed using * destroy_worker(). * * CONTEXT: * Might sleep. Does GFP_KERNEL allocations. * * RETURNS: * Pointer to the newly created worker. */static struct worker *create_worker(struct global_cwq *gcwq, bool bind){ bool on_unbound_cpu = gcwq->cpu == WORK_CPU_UNBOUND; struct worker *worker = NULL; int id = -1; spin_lock_irq(&gcwq->lock); while (ida_get_new(&gcwq->worker_ida, &id)) { spin_unlock_irq(&gcwq->lock); if (!ida_pre_get(&gcwq->worker_ida, GFP_KERNEL)) goto fail; spin_lock_irq(&gcwq->lock); } spin_unlock_irq(&gcwq->lock); worker = alloc_worker(); if (!worker) goto fail; worker->gcwq = gcwq; worker->id = id; if (!on_unbound_cpu) worker->task = kthread_create_on_node(worker_thread, worker, cpu_to_node(gcwq->cpu), "kworker/%u:%d", gcwq->cpu, id); else worker->task = kthread_create(worker_thread, worker, "kworker/u:%d", id); if (IS_ERR(worker->task)) goto fail; /* * A rogue worker will become a regular one if CPU comes * online later on. Make sure every worker has * PF_THREAD_BOUND set. */ if (bind && !on_unbound_cpu) kthread_bind(worker->task, gcwq->cpu); else { worker->task->flags |= PF_THREAD_BOUND; if (on_unbound_cpu) worker->flags |= WORKER_UNBOUND; } return worker;fail: if (id >= 0) { spin_lock_irq(&gcwq->lock); ida_remove(&gcwq->worker_ida, id); spin_unlock_irq(&gcwq->lock); } kfree(worker); return NULL;} /** * start_worker - start a newly created worker * @worker: worker to start * * Make the gcwq aware of @worker and start it. * * CONTEXT: * spin_lock_irq(gcwq->lock). */static void start_worker(struct worker *worker){ worker->flags |= WORKER_STARTED; worker->gcwq->nr_workers++; worker_enter_idle(worker); wake_up_process(worker->task);} /** * destroy_worker - destroy a workqueue worker * @worker: worker to be destroyed * * Destroy @worker and adjust @gcwq stats accordingly. * * CONTEXT: * spin_lock_irq(gcwq->lock) which is released and regrabbed. */static void destroy_worker(struct worker *worker){ struct global_cwq *gcwq = worker->gcwq; int id = worker->id; /* sanity check frenzy */ BUG_ON(worker->current_work); BUG_ON(!list_empty(&worker->scheduled)); if (worker->flags & WORKER_STARTED) gcwq->nr_workers--; if (worker->flags & WORKER_IDLE) gcwq->nr_idle--; list_del_init(&worker->entry); worker->flags |= WORKER_DIE; spin_unlock_irq(&gcwq->lock); kthread_stop(worker->task); kfree(worker); spin_lock_irq(&gcwq->lock); ida_remove(&gcwq->worker_ida, id);} static void idle_worker_timeout(unsigned long __gcwq){ struct global_cwq *gcwq = (void *)__gcwq; spin_lock_irq(&gcwq->lock); if (too_many_workers(gcwq)) { struct worker *worker; unsigned long expires; /* idle_list is kept in LIFO order, check the last one */ worker = list_entry(gcwq->idle_list.prev, struct worker, entry); expires = worker->last_active + IDLE_WORKER_TIMEOUT; if (time_before(jiffies, expires)) mod_timer(&gcwq->idle_timer, expires); else { /* it's been idle for too long, wake up manager */ gcwq->flags |= GCWQ_MANAGE_WORKERS; wake_up_worker(gcwq); } } spin_unlock_irq(&gcwq->lock);} static bool send_mayday(struct work_struct *work){ struct cpu_workqueue_struct *cwq = get_work_cwq(work); struct workqueue_struct *wq = cwq->wq; unsigned int cpu; if (!(wq->flags & WQ_RESCUER)) return false;Lines 1001–1500 of 3842. Previous lines Load more
/* * It's multi cpu. If @wq is non-reentrant and @work * was previously on a different cpu, it might still * be running there, in which case the work needs to * be queued on that cpu to guarantee non-reentrance. */ gcwq = get_gcwq(cpu); if (wq->flags & WQ_NON_REENTRANT && (last_gcwq = get_work_gcwq(work)) && last_gcwq != gcwq) { struct worker *worker; spin_lock_irqsave(&last_gcwq->lock, flags); worker = find_worker_executing_work(last_gcwq, work); if (worker && worker->current_cwq->wq == wq) gcwq = last_gcwq; else { /* meh... not running there, queue here */ spin_unlock_irqrestore(&last_gcwq->lock, flags); spin_lock_irqsave(&gcwq->lock, flags); } } else spin_lock_irqsave(&gcwq->lock, flags); } else { gcwq = get_gcwq(WORK_CPU_UNBOUND); spin_lock_irqsave(&gcwq->lock, flags); } /* gcwq determined, get cwq and queue */ cwq = get_cwq(gcwq->cpu, wq); trace_workqueue_queue_work(cpu, cwq, work); if (WARN_ON(!list_empty(&work->entry))) { spin_unlock_irqrestore(&gcwq->lock, flags); return; } cwq->nr_in_flight[cwq->work_color]++; work_flags = work_color_to_flags(cwq->work_color); if (likely(cwq->nr_active < cwq->max_active)) { trace_workqueue_activate_work(work); cwq->nr_active++; worklist = gcwq_determine_ins_pos(gcwq, cwq); } else { work_flags |= WORK_STRUCT_DELAYED; worklist = &cwq->delayed_works; } insert_work(cwq, work, worklist, work_flags); spin_unlock_irqrestore(&gcwq->lock, flags); } /** * queue_work - queue work on a workqueue * @wq: workqueue to use * @work: work to queue * * Returns 0 if @work was already on a queue, non-zero otherwise. * * We queue the work to the CPU on which it was submitted, but if the CPU dies * it can be processed by another CPU. */ int queue_work(struct workqueue_struct *wq, struct work_struct *work) { int ret; ret = queue_work_on(get_cpu(), wq, work); put_cpu(); return ret; } EXPORT_SYMBOL_GPL(queue_work); /** * queue_work_on - queue work on specific cpu * @cpu: CPU number to execute work on * @wq: workqueue to use * @work: work to queue * * Returns 0 if @work was already on a queue, non-zero otherwise. * * We queue the work to a specific CPU, the caller must ensure it * can't go away. */ int queue_work_on(int cpu, struct workqueue_struct *wq, struct work_struct *work) { int ret = 0; if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) { __queue_work(cpu, wq, work); ret = 1; } return ret; } EXPORT_SYMBOL_GPL(queue_work_on); static void delayed_work_timer_fn(unsigned long __data) { struct delayed_work *dwork = (struct delayed_work *)__data; struct cpu_workqueue_struct *cwq = get_work_cwq(&dwork->work); __queue_work(smp_processor_id(), cwq->wq, &dwork->work); } /** * queue_delayed_work - queue work on a workqueue after delay * @wq: workqueue to use * @dwork: delayable work to queue * @delay: number of jiffies to wait before queueing * * Returns 0 if @work was already on a queue, non-zero otherwise. */ int queue_delayed_work(struct workqueue_struct *wq, struct delayed_work *dwork, unsigned long delay) { if (delay == 0) return queue_work(wq, &dwork->work); return queue_delayed_work_on(-1, wq, dwork, delay); } EXPORT_SYMBOL_GPL(queue_delayed_work); /** * queue_delayed_work_on - queue work on specific CPU after delay * @cpu: CPU number to execute work on * @wq: workqueue to use * @dwork: work to queue * @delay: number of jiffies to wait before queueing * * Returns 0 if @work was already on a queue, non-zero otherwise. */ int queue_delayed_work_on(int cpu, struct workqueue_struct *wq, struct delayed_work *dwork, unsigned long delay) { int ret = 0; struct timer_list *timer = &dwork->timer; struct work_struct *work = &dwork->work; if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) { unsigned int lcpu; BUG_ON(timer_pending(timer)); BUG_ON(!list_empty(&work->entry)); timer_stats_timer_set_start_info(&dwork->timer); /* * This stores cwq for the moment, for the timer_fn. * Note that the work's gcwq is preserved to allow * reentrance detection for delayed works. */ if (!(wq->flags & WQ_UNBOUND)) { struct global_cwq *gcwq = get_work_gcwq(work); if (gcwq && gcwq->cpu != WORK_CPU_UNBOUND) lcpu = gcwq->cpu; else lcpu = raw_smp_processor_id(); } else lcpu = WORK_CPU_UNBOUND; set_work_cwq(work, get_cwq(lcpu, wq), 0); timer->expires = jiffies + delay; timer->data = (unsigned long)dwork; timer->function = delayed_work_timer_fn; if (unlikely(cpu >= 0)) add_timer_on(timer, cpu); else add_timer(timer); ret = 1; } return ret; } EXPORT_SYMBOL_GPL(queue_delayed_work_on); /** * worker_enter_idle - enter idle state * @worker: worker which is entering idle state * * @worker is entering idle state. Update stats and idle timer if * necessary. * * LOCKING: * spin_lock_irq(gcwq->lock). */ static void worker_enter_idle(struct worker *worker) { struct global_cwq *gcwq = worker->gcwq; BUG_ON(worker->flags & WORKER_IDLE); BUG_ON(!list_empty(&worker->entry) && (worker->hentry.next || worker->hentry.pprev)); /* can't use worker_set_flags(), also called from start_worker() */ worker->flags |= WORKER_IDLE; gcwq->nr_idle++; worker->last_active = jiffies; /* idle_list is LIFO */ list_add(&worker->entry, &gcwq->idle_list); if (likely(!(worker->flags & WORKER_ROGUE))) { if (too_many_workers(gcwq) && !timer_pending(&gcwq->idle_timer)) mod_timer(&gcwq->idle_timer, jiffies + IDLE_WORKER_TIMEOUT); } else wake_up_all(&gcwq->trustee_wait); /* * Sanity check nr_running. Because trustee releases gcwq->lock * between setting %WORKER_ROGUE and zapping nr_running, the * warning may trigger spuriously. Check iff trustee is idle. */ WARN_ON_ONCE(gcwq->trustee_state == TRUSTEE_DONE && gcwq->nr_workers == gcwq->nr_idle && atomic_read(get_gcwq_nr_running(gcwq->cpu))); } /** * worker_leave_idle - leave idle state * @worker: worker which is leaving idle state * * @worker is leaving idle state. Update stats. * * LOCKING: * spin_lock_irq(gcwq->lock). */ static void worker_leave_idle(struct worker *worker) { struct global_cwq *gcwq = worker->gcwq; BUG_ON(!(worker->flags & WORKER_IDLE)); worker_clr_flags(worker, WORKER_IDLE); gcwq->nr_idle--; list_del_init(&worker->entry); } /** * worker_maybe_bind_and_lock - bind worker to its cpu if possible and lock gcwq * @worker: self * * Works which are scheduled while the cpu is online must at least be * scheduled to a worker which is bound to the cpu so that if they are * flushed from cpu callbacks while cpu is going down, they are * guaranteed to execute on the cpu. * * This function is to be used by rogue workers and rescuers to bind * themselves to the target cpu and may race with cpu going down or * coming online. kthread_bind() can't be used because it may put the * worker to already dead cpu and set_cpus_allowed_ptr() can't be used * verbatim as it's best effort and blocking and gcwq may be * [dis]associated in the meantime. * * This function tries set_cpus_allowed() and locks gcwq and verifies * the binding against GCWQ_DISASSOCIATED which is set during * CPU_DYING and cleared during CPU_ONLINE, so if the worker enters * idle state or fetches works without dropping lock, it can guarantee * the scheduling requirement described in the first paragraph. * * CONTEXT: * Might sleep. Called without any lock but returns with gcwq->lock * held. * * RETURNS: * %true if the associated gcwq is online (@worker is successfully * bound), %false if offline. */ static bool worker_maybe_bind_and_lock(struct worker *worker) __acquires(&gcwq->lock) { struct global_cwq *gcwq = worker->gcwq; struct task_struct *task = worker->task; while (true) { /* * The following call may fail, succeed or succeed * without actually migrating the task to the cpu if * it races with cpu hotunplug operation. Verify * against GCWQ_DISASSOCIATED. */ if (!(gcwq->flags & GCWQ_DISASSOCIATED)) set_cpus_allowed_ptr(task, get_cpu_mask(gcwq->cpu)); spin_lock_irq(&gcwq->lock); if (gcwq->flags & GCWQ_DISASSOCIATED) return false; if (task_cpu(task) == gcwq->cpu && cpumask_equal(¤t->cpus_allowed, get_cpu_mask(gcwq->cpu))) return true; spin_unlock_irq(&gcwq->lock); /* * We've raced with CPU hot[un]plug. Give it a breather * and retry migration. cond_resched() is required here; * otherwise, we might deadlock against cpu_stop trying to * bring down the CPU on non-preemptive kernel. */ cpu_relax(); cond_resched(); } } /* * Function for worker->rebind_work used to rebind rogue busy workers * to the associated cpu which is coming back online. This is * scheduled by cpu up but can race with other cpu hotplug operations * and may be executed twice without intervening cpu down. */ static void worker_rebind_fn(struct work_struct *work) { struct worker *worker = container_of(work, struct worker, rebind_work); struct global_cwq *gcwq = worker->gcwq; if (worker_maybe_bind_and_lock(worker)) worker_clr_flags(worker, WORKER_REBIND); spin_unlock_irq(&gcwq->lock); } static struct worker *alloc_worker(void) { struct worker *worker; worker = kzalloc(sizeof(*worker), GFP_KERNEL); if (worker) { INIT_LIST_HEAD(&worker->entry); INIT_LIST_HEAD(&worker->scheduled); INIT_WORK(&worker->rebind_work, worker_rebind_fn); /* on creation a worker is in !idle && prep state */ worker->flags = WORKER_PREP; } return worker; } /** * create_worker - create a new workqueue worker * @gcwq: gcwq the new worker will belong to * @bind: whether to set affinity to @cpu or not * * Create a new worker which is bound to @gcwq. The returned worker * can be started by calling start_worker() or destroyed using * destroy_worker(). * * CONTEXT: * Might sleep. Does GFP_KERNEL allocations. * * RETURNS: * Pointer to the newly created worker. */ static struct worker *create_worker(struct global_cwq *gcwq, bool bind) { bool on_unbound_cpu = gcwq->cpu == WORK_CPU_UNBOUND; struct worker *worker = NULL; int id = -1; spin_lock_irq(&gcwq->lock); while (ida_get_new(&gcwq->worker_ida, &id)) { spin_unlock_irq(&gcwq->lock); if (!ida_pre_get(&gcwq->worker_ida, GFP_KERNEL)) goto fail; spin_lock_irq(&gcwq->lock); } spin_unlock_irq(&gcwq->lock); worker = alloc_worker(); if (!worker) goto fail; worker->gcwq = gcwq; worker->id = id; if (!on_unbound_cpu) worker->task = kthread_create_on_node(worker_thread, worker, cpu_to_node(gcwq->cpu), "kworker/%u:%d", gcwq->cpu, id); else worker->task = kthread_create(worker_thread, worker, "kworker/u:%d", id); if (IS_ERR(worker->task)) goto fail; /* * A rogue worker will become a regular one if CPU comes * online later on. Make sure every worker has * PF_THREAD_BOUND set. */ if (bind && !on_unbound_cpu) kthread_bind(worker->task, gcwq->cpu); else { worker->task->flags |= PF_THREAD_BOUND; if (on_unbound_cpu) worker->flags |= WORKER_UNBOUND; } return worker; fail: if (id >= 0) { spin_lock_irq(&gcwq->lock); ida_remove(&gcwq->worker_ida, id); spin_unlock_irq(&gcwq->lock); } kfree(worker); return NULL; } /** * start_worker - start a newly created worker * @worker: worker to start * * Make the gcwq aware of @worker and start it. * * CONTEXT: * spin_lock_irq(gcwq->lock). */ static void start_worker(struct worker *worker) { worker->flags |= WORKER_STARTED; worker->gcwq->nr_workers++; worker_enter_idle(worker); wake_up_process(worker->task); } /** * destroy_worker - destroy a workqueue worker * @worker: worker to be destroyed * * Destroy @worker and adjust @gcwq stats accordingly. * * CONTEXT: * spin_lock_irq(gcwq->lock) which is released and regrabbed. */ static void destroy_worker(struct worker *worker) { struct global_cwq *gcwq = worker->gcwq; int id = worker->id; /* sanity check frenzy */ BUG_ON(worker->current_work); BUG_ON(!list_empty(&worker->scheduled)); if (worker->flags & WORKER_STARTED) gcwq->nr_workers--; if (worker->flags & WORKER_IDLE) gcwq->nr_idle--; list_del_init(&worker->entry); worker->flags |= WORKER_DIE; spin_unlock_irq(&gcwq->lock); kthread_stop(worker->task); kfree(worker); spin_lock_irq(&gcwq->lock); ida_remove(&gcwq->worker_ida, id); } static void idle_worker_timeout(unsigned long __gcwq) { struct global_cwq *gcwq = (void *)__gcwq; spin_lock_irq(&gcwq->lock); if (too_many_workers(gcwq)) { struct worker *worker; unsigned long expires; /* idle_list is kept in LIFO order, check the last one */ worker = list_entry(gcwq->idle_list.prev, struct worker, entry); expires = worker->last_active + IDLE_WORKER_TIMEOUT; if (time_before(jiffies, expires)) mod_timer(&gcwq->idle_timer, expires); else { /* it's been idle for too long, wake up manager */ gcwq->flags |= GCWQ_MANAGE_WORKERS; wake_up_worker(gcwq); } } spin_unlock_irq(&gcwq->lock); } static bool send_mayday(struct work_struct *work) { struct cpu_workqueue_struct *cwq = get_work_cwq(work); struct workqueue_struct *wq = cwq->wq; unsigned int cpu; if (!(wq->flags & WQ_RESCUER)) return false;