/* mayday mayday mayday */ cpu = cwq->gcwq->cpu; /* WORK_CPU_UNBOUND can't be set in cpumask, use cpu 0 instead */ if (cpu == WORK_CPU_UNBOUND) cpu = 0; if (!mayday_test_and_set_cpu(cpu, wq->mayday_mask)) wake_up_process(wq->rescuer->task); return true;} static void gcwq_mayday_timeout(unsigned long __gcwq){ struct global_cwq *gcwq = (void *)__gcwq; struct work_struct *work; spin_lock_irq(&gcwq->lock); if (need_to_create_worker(gcwq)) { /* * We've been trying to create a new worker but * haven't been successful. We might be hitting an * allocation deadlock. Send distress signals to * rescuers. */ list_for_each_entry(work, &gcwq->worklist, entry) send_mayday(work); } spin_unlock_irq(&gcwq->lock); mod_timer(&gcwq->mayday_timer, jiffies + MAYDAY_INTERVAL);} /** * maybe_create_worker - create a new worker if necessary * @gcwq: gcwq to create a new worker for * * Create a new worker for @gcwq if necessary. @gcwq is guaranteed to * have at least one idle worker on return from this function. If * creating a new worker takes longer than MAYDAY_INTERVAL, mayday is * sent to all rescuers with works scheduled on @gcwq to resolve * possible allocation deadlock. * * On return, need_to_create_worker() is guaranteed to be false and * may_start_working() true. * * LOCKING: * spin_lock_irq(gcwq->lock) which may be released and regrabbed * multiple times. Does GFP_KERNEL allocations. Called only from * manager. * * RETURNS: * false if no action was taken and gcwq->lock stayed locked, true * otherwise. */static bool maybe_create_worker(struct global_cwq *gcwq)__releases(&gcwq->lock)__acquires(&gcwq->lock){ if (!need_to_create_worker(gcwq)) return false;restart: spin_unlock_irq(&gcwq->lock); /* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */ mod_timer(&gcwq->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT); while (true) { struct worker *worker; worker = create_worker(gcwq, true); if (worker) { del_timer_sync(&gcwq->mayday_timer); spin_lock_irq(&gcwq->lock); start_worker(worker); BUG_ON(need_to_create_worker(gcwq)); return true; } if (!need_to_create_worker(gcwq)) break; __set_current_state(TASK_INTERRUPTIBLE); schedule_timeout(CREATE_COOLDOWN); if (!need_to_create_worker(gcwq)) break; } del_timer_sync(&gcwq->mayday_timer); spin_lock_irq(&gcwq->lock); if (need_to_create_worker(gcwq)) goto restart; return true;} /** * maybe_destroy_worker - destroy workers which have been idle for a while * @gcwq: gcwq to destroy workers for * * Destroy @gcwq workers which have been idle for longer than * IDLE_WORKER_TIMEOUT. * * LOCKING: * spin_lock_irq(gcwq->lock) which may be released and regrabbed * multiple times. Called only from manager. * * RETURNS: * false if no action was taken and gcwq->lock stayed locked, true * otherwise. */static bool maybe_destroy_workers(struct global_cwq *gcwq){ bool ret = false; while (too_many_workers(gcwq)) { struct worker *worker; unsigned long expires; worker = list_entry(gcwq->idle_list.prev, struct worker, entry); expires = worker->last_active + IDLE_WORKER_TIMEOUT; if (time_before(jiffies, expires)) { mod_timer(&gcwq->idle_timer, expires); break; } destroy_worker(worker); ret = true; } return ret;} /** * manage_workers - manage worker pool * @worker: self * * Assume the manager role and manage gcwq worker pool @worker belongs * to. At any given time, there can be only zero or one manager per * gcwq. The exclusion is handled automatically by this function. * * The caller can safely start processing works on false return. On * true return, it's guaranteed that need_to_create_worker() is false * and may_start_working() is true. * * CONTEXT: * spin_lock_irq(gcwq->lock) which may be released and regrabbed * multiple times. Does GFP_KERNEL allocations. * * RETURNS: * false if no action was taken and gcwq->lock stayed locked, true if * some action was taken. */static bool manage_workers(struct worker *worker){ struct global_cwq *gcwq = worker->gcwq; bool ret = false; if (gcwq->flags & GCWQ_MANAGING_WORKERS) return ret; gcwq->flags &= ~GCWQ_MANAGE_WORKERS; gcwq->flags |= GCWQ_MANAGING_WORKERS; /* * Destroy and then create so that may_start_working() is true * on return. */ ret |= maybe_destroy_workers(gcwq); ret |= maybe_create_worker(gcwq); gcwq->flags &= ~GCWQ_MANAGING_WORKERS; /* * The trustee might be waiting to take over the manager * position, tell it we're done. */ if (unlikely(gcwq->trustee)) wake_up_all(&gcwq->trustee_wait); return ret;} /** * move_linked_works - move linked works to a list * @work: start of series of works to be scheduled * @head: target list to append @work to * @nextp: out paramter for nested worklist walking * * Schedule linked works starting from @work to @head. Work series to * be scheduled starts at @work and includes any consecutive work with * WORK_STRUCT_LINKED set in its predecessor. * * If @nextp is not NULL, it's updated to point to the next work of * the last scheduled work. This allows move_linked_works() to be * nested inside outer list_for_each_entry_safe(). * * CONTEXT: * spin_lock_irq(gcwq->lock). */static void move_linked_works(struct work_struct *work, struct list_head *head, struct work_struct **nextp){ struct work_struct *n; /* * Linked worklist will always end before the end of the list, * use NULL for list head. */ list_for_each_entry_safe_from(work, n, NULL, entry) { list_move_tail(&work->entry, head); if (!(*work_data_bits(work) & WORK_STRUCT_LINKED)) break; } /* * If we're already inside safe list traversal and have moved * multiple works to the scheduled queue, the next position * needs to be updated. */ if (nextp) *nextp = n;} static void cwq_activate_first_delayed(struct cpu_workqueue_struct *cwq){ struct work_struct *work = list_first_entry(&cwq->delayed_works, struct work_struct, entry); struct list_head *pos = gcwq_determine_ins_pos(cwq->gcwq, cwq); trace_workqueue_activate_work(work); move_linked_works(work, pos, NULL); __clear_bit(WORK_STRUCT_DELAYED_BIT, work_data_bits(work)); cwq->nr_active++;} /** * cwq_dec_nr_in_flight - decrement cwq's nr_in_flight * @cwq: cwq of interest * @color: color of work which left the queue * @delayed: for a delayed work * * A work either has completed or is removed from pending queue, * decrement nr_in_flight of its cwq and handle workqueue flushing. * * CONTEXT: * spin_lock_irq(gcwq->lock). */static void cwq_dec_nr_in_flight(struct cpu_workqueue_struct *cwq, int color, bool delayed){ /* ignore uncolored works */ if (color == WORK_NO_COLOR) return; cwq->nr_in_flight[color]--; if (!delayed) { cwq->nr_active--; if (!list_empty(&cwq->delayed_works)) { /* one down, submit a delayed one */ if (cwq->nr_active < cwq->max_active) cwq_activate_first_delayed(cwq); } } /* is flush in progress and are we at the flushing tip? */ if (likely(cwq->flush_color != color)) return; /* are there still in-flight works? */ if (cwq->nr_in_flight[color]) return; /* this cwq is done, clear flush_color */ cwq->flush_color = -1; /* * If this was the last cwq, wake up the first flusher. It * will handle the rest. */ if (atomic_dec_and_test(&cwq->wq->nr_cwqs_to_flush)) complete(&cwq->wq->first_flusher->done);} /** * process_one_work - process single work * @worker: self * @work: work to process * * Process @work. This function contains all the logics necessary to * process a single work including synchronization against and * interaction with other workers on the same cpu, queueing and * flushing. As long as context requirement is met, any worker can * call this function to process a work. * * CONTEXT: * spin_lock_irq(gcwq->lock) which is released and regrabbed. */static void process_one_work(struct worker *worker, struct work_struct *work)__releases(&gcwq->lock)__acquires(&gcwq->lock){ struct cpu_workqueue_struct *cwq = get_work_cwq(work); struct global_cwq *gcwq = cwq->gcwq; struct hlist_head *bwh = busy_worker_head(gcwq, work); bool cpu_intensive = cwq->wq->flags & WQ_CPU_INTENSIVE; work_func_t f = work->func; int work_color; struct worker *collision;#ifdef CONFIG_LOCKDEP /* * It is permissible to free the struct work_struct from * inside the function that is called from it, this we need to * take into account for lockdep too. To avoid bogus "held * lock freed" warnings as well as problems when looking into * work->lockdep_map, make a copy and use that here. */ struct lockdep_map lockdep_map; lockdep_copy_map(&lockdep_map, &work->lockdep_map);#endif /* * A single work shouldn't be executed concurrently by * multiple workers on a single cpu. Check whether anyone is * already processing the work. If so, defer the work to the * currently executing one. */ collision = __find_worker_executing_work(gcwq, bwh, work); if (unlikely(collision)) { move_linked_works(work, &collision->scheduled, NULL); return; } /* claim and process */ debug_work_deactivate(work); hlist_add_head(&worker->hentry, bwh); worker->current_work = work; worker->current_cwq = cwq; work_color = get_work_color(work); /* record the current cpu number in the work data and dequeue */ set_work_cpu(work, gcwq->cpu); list_del_init(&work->entry); /* * If HIGHPRI_PENDING, check the next work, and, if HIGHPRI, * wake up another worker; otherwise, clear HIGHPRI_PENDING. */ if (unlikely(gcwq->flags & GCWQ_HIGHPRI_PENDING)) { struct work_struct *nwork = list_first_entry(&gcwq->worklist, struct work_struct, entry); if (!list_empty(&gcwq->worklist) && get_work_cwq(nwork)->wq->flags & WQ_HIGHPRI) wake_up_worker(gcwq); else gcwq->flags &= ~GCWQ_HIGHPRI_PENDING; } /* * CPU intensive works don't participate in concurrency * management. They're the scheduler's responsibility. */ if (unlikely(cpu_intensive)) worker_set_flags(worker, WORKER_CPU_INTENSIVE, true); spin_unlock_irq(&gcwq->lock); work_clear_pending(work); lock_map_acquire_read(&cwq->wq->lockdep_map); lock_map_acquire(&lockdep_map); trace_workqueue_execute_start(work); f(work); /* * While we must be careful to not use "work" after this, the trace * point will only record its address. */ trace_workqueue_execute_end(work); lock_map_release(&lockdep_map); lock_map_release(&cwq->wq->lockdep_map); if (unlikely(in_atomic() || lockdep_depth(current) > 0)) { printk(KERN_ERR "BUG: workqueue leaked lock or atomic: " "%s/0x%08x/%d\n", current->comm, preempt_count(), task_pid_nr(current)); printk(KERN_ERR " last function: "); print_symbol("%s\n", (unsigned long)f); debug_show_held_locks(current); dump_stack(); } spin_lock_irq(&gcwq->lock); /* clear cpu intensive status */ if (unlikely(cpu_intensive)) worker_clr_flags(worker, WORKER_CPU_INTENSIVE); /* we're done with it, release */ hlist_del_init(&worker->hentry); worker->current_work = NULL; worker->current_cwq = NULL; cwq_dec_nr_in_flight(cwq, work_color, false);} /** * process_scheduled_works - process scheduled works * @worker: self * * Process all scheduled works. Please note that the scheduled list * may change while processing a work, so this function repeatedly * fetches a work from the top and executes it. * * CONTEXT: * spin_lock_irq(gcwq->lock) which may be released and regrabbed * multiple times. */static void process_scheduled_works(struct worker *worker){ while (!list_empty(&worker->scheduled)) { struct work_struct *work = list_first_entry(&worker->scheduled, struct work_struct, entry); process_one_work(worker, work); }} /** * worker_thread - the worker thread function * @__worker: self * * The gcwq worker thread function. There's a single dynamic pool of * these per each cpu. These workers process all works regardless of * their specific target workqueue. The only exception is works which * belong to workqueues with a rescuer which will be explained in * rescuer_thread(). */static int worker_thread(void *__worker){ struct worker *worker = __worker; struct global_cwq *gcwq = worker->gcwq; /* tell the scheduler that this is a workqueue worker */ worker->task->flags |= PF_WQ_WORKER;woke_up: spin_lock_irq(&gcwq->lock); /* DIE can be set only while we're idle, checking here is enough */ if (worker->flags & WORKER_DIE) { spin_unlock_irq(&gcwq->lock); worker->task->flags &= ~PF_WQ_WORKER; return 0; } worker_leave_idle(worker);recheck: /* no more worker necessary? */ if (!need_more_worker(gcwq)) goto sleep; /* do we need to manage? */ if (unlikely(!may_start_working(gcwq)) && manage_workers(worker)) goto recheck; /* * ->scheduled list can only be filled while a worker is * preparing to process a work or actually processing it. * Make sure nobody diddled with it while I was sleeping. */ BUG_ON(!list_empty(&worker->scheduled)); /* * When control reaches this point, we're guaranteed to have * at least one idle worker or that someone else has already * assumed the manager role. */ worker_clr_flags(worker, WORKER_PREP); do { struct work_struct *work = list_first_entry(&gcwq->worklist, struct work_struct, entry); if (likely(!(*work_data_bits(work) & WORK_STRUCT_LINKED))) { /* optimization path, not strictly necessary */ process_one_work(worker, work); if (unlikely(!list_empty(&worker->scheduled))) process_scheduled_works(worker); } else { move_linked_works(work, &worker->scheduled, NULL); process_scheduled_works(worker); } } while (keep_working(gcwq)); worker_set_flags(worker, WORKER_PREP, false);sleep: if (unlikely(need_to_manage_workers(gcwq)) && manage_workers(worker)) goto recheck; Lines 1501–2000 of 3842. Previous lines Load more
/* mayday mayday mayday */ cpu = cwq->gcwq->cpu; /* WORK_CPU_UNBOUND can't be set in cpumask, use cpu 0 instead */ if (cpu == WORK_CPU_UNBOUND) cpu = 0; if (!mayday_test_and_set_cpu(cpu, wq->mayday_mask)) wake_up_process(wq->rescuer->task); return true; } static void gcwq_mayday_timeout(unsigned long __gcwq) { struct global_cwq *gcwq = (void *)__gcwq; struct work_struct *work; spin_lock_irq(&gcwq->lock); if (need_to_create_worker(gcwq)) { /* * We've been trying to create a new worker but * haven't been successful. We might be hitting an * allocation deadlock. Send distress signals to * rescuers. */ list_for_each_entry(work, &gcwq->worklist, entry) send_mayday(work); } spin_unlock_irq(&gcwq->lock); mod_timer(&gcwq->mayday_timer, jiffies + MAYDAY_INTERVAL); } /** * maybe_create_worker - create a new worker if necessary * @gcwq: gcwq to create a new worker for * * Create a new worker for @gcwq if necessary. @gcwq is guaranteed to * have at least one idle worker on return from this function. If * creating a new worker takes longer than MAYDAY_INTERVAL, mayday is * sent to all rescuers with works scheduled on @gcwq to resolve * possible allocation deadlock. * * On return, need_to_create_worker() is guaranteed to be false and * may_start_working() true. * * LOCKING: * spin_lock_irq(gcwq->lock) which may be released and regrabbed * multiple times. Does GFP_KERNEL allocations. Called only from * manager. * * RETURNS: * false if no action was taken and gcwq->lock stayed locked, true * otherwise. */ static bool maybe_create_worker(struct global_cwq *gcwq) __releases(&gcwq->lock) __acquires(&gcwq->lock) { if (!need_to_create_worker(gcwq)) return false; restart: spin_unlock_irq(&gcwq->lock); /* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */ mod_timer(&gcwq->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT); while (true) { struct worker *worker; worker = create_worker(gcwq, true); if (worker) { del_timer_sync(&gcwq->mayday_timer); spin_lock_irq(&gcwq->lock); start_worker(worker); BUG_ON(need_to_create_worker(gcwq)); return true; } if (!need_to_create_worker(gcwq)) break; __set_current_state(TASK_INTERRUPTIBLE); schedule_timeout(CREATE_COOLDOWN); if (!need_to_create_worker(gcwq)) break; } del_timer_sync(&gcwq->mayday_timer); spin_lock_irq(&gcwq->lock); if (need_to_create_worker(gcwq)) goto restart; return true; } /** * maybe_destroy_worker - destroy workers which have been idle for a while * @gcwq: gcwq to destroy workers for * * Destroy @gcwq workers which have been idle for longer than * IDLE_WORKER_TIMEOUT. * * LOCKING: * spin_lock_irq(gcwq->lock) which may be released and regrabbed * multiple times. Called only from manager. * * RETURNS: * false if no action was taken and gcwq->lock stayed locked, true * otherwise. */ static bool maybe_destroy_workers(struct global_cwq *gcwq) { bool ret = false; while (too_many_workers(gcwq)) { struct worker *worker; unsigned long expires; worker = list_entry(gcwq->idle_list.prev, struct worker, entry); expires = worker->last_active + IDLE_WORKER_TIMEOUT; if (time_before(jiffies, expires)) { mod_timer(&gcwq->idle_timer, expires); break; } destroy_worker(worker); ret = true; } return ret; } /** * manage_workers - manage worker pool * @worker: self * * Assume the manager role and manage gcwq worker pool @worker belongs * to. At any given time, there can be only zero or one manager per * gcwq. The exclusion is handled automatically by this function. * * The caller can safely start processing works on false return. On * true return, it's guaranteed that need_to_create_worker() is false * and may_start_working() is true. * * CONTEXT: * spin_lock_irq(gcwq->lock) which may be released and regrabbed * multiple times. Does GFP_KERNEL allocations. * * RETURNS: * false if no action was taken and gcwq->lock stayed locked, true if * some action was taken. */ static bool manage_workers(struct worker *worker) { struct global_cwq *gcwq = worker->gcwq; bool ret = false; if (gcwq->flags & GCWQ_MANAGING_WORKERS) return ret; gcwq->flags &= ~GCWQ_MANAGE_WORKERS; gcwq->flags |= GCWQ_MANAGING_WORKERS; /* * Destroy and then create so that may_start_working() is true * on return. */ ret |= maybe_destroy_workers(gcwq); ret |= maybe_create_worker(gcwq); gcwq->flags &= ~GCWQ_MANAGING_WORKERS; /* * The trustee might be waiting to take over the manager * position, tell it we're done. */ if (unlikely(gcwq->trustee)) wake_up_all(&gcwq->trustee_wait); return ret; } /** * move_linked_works - move linked works to a list * @work: start of series of works to be scheduled * @head: target list to append @work to * @nextp: out paramter for nested worklist walking * * Schedule linked works starting from @work to @head. Work series to * be scheduled starts at @work and includes any consecutive work with * WORK_STRUCT_LINKED set in its predecessor. * * If @nextp is not NULL, it's updated to point to the next work of * the last scheduled work. This allows move_linked_works() to be * nested inside outer list_for_each_entry_safe(). * * CONTEXT: * spin_lock_irq(gcwq->lock). */ static void move_linked_works(struct work_struct *work, struct list_head *head, struct work_struct **nextp) { struct work_struct *n; /* * Linked worklist will always end before the end of the list, * use NULL for list head. */ list_for_each_entry_safe_from(work, n, NULL, entry) { list_move_tail(&work->entry, head); if (!(*work_data_bits(work) & WORK_STRUCT_LINKED)) break; } /* * If we're already inside safe list traversal and have moved * multiple works to the scheduled queue, the next position * needs to be updated. */ if (nextp) *nextp = n; } static void cwq_activate_first_delayed(struct cpu_workqueue_struct *cwq) { struct work_struct *work = list_first_entry(&cwq->delayed_works, struct work_struct, entry); struct list_head *pos = gcwq_determine_ins_pos(cwq->gcwq, cwq); trace_workqueue_activate_work(work); move_linked_works(work, pos, NULL); __clear_bit(WORK_STRUCT_DELAYED_BIT, work_data_bits(work)); cwq->nr_active++; } /** * cwq_dec_nr_in_flight - decrement cwq's nr_in_flight * @cwq: cwq of interest * @color: color of work which left the queue * @delayed: for a delayed work * * A work either has completed or is removed from pending queue, * decrement nr_in_flight of its cwq and handle workqueue flushing. * * CONTEXT: * spin_lock_irq(gcwq->lock). */ static void cwq_dec_nr_in_flight(struct cpu_workqueue_struct *cwq, int color, bool delayed) { /* ignore uncolored works */ if (color == WORK_NO_COLOR) return; cwq->nr_in_flight[color]--; if (!delayed) { cwq->nr_active--; if (!list_empty(&cwq->delayed_works)) { /* one down, submit a delayed one */ if (cwq->nr_active < cwq->max_active) cwq_activate_first_delayed(cwq); } } /* is flush in progress and are we at the flushing tip? */ if (likely(cwq->flush_color != color)) return; /* are there still in-flight works? */ if (cwq->nr_in_flight[color]) return; /* this cwq is done, clear flush_color */ cwq->flush_color = -1; /* * If this was the last cwq, wake up the first flusher. It * will handle the rest. */ if (atomic_dec_and_test(&cwq->wq->nr_cwqs_to_flush)) complete(&cwq->wq->first_flusher->done); } /** * process_one_work - process single work * @worker: self * @work: work to process * * Process @work. This function contains all the logics necessary to * process a single work including synchronization against and * interaction with other workers on the same cpu, queueing and * flushing. As long as context requirement is met, any worker can * call this function to process a work. * * CONTEXT: * spin_lock_irq(gcwq->lock) which is released and regrabbed. */ static void process_one_work(struct worker *worker, struct work_struct *work) __releases(&gcwq->lock) __acquires(&gcwq->lock) { struct cpu_workqueue_struct *cwq = get_work_cwq(work); struct global_cwq *gcwq = cwq->gcwq; struct hlist_head *bwh = busy_worker_head(gcwq, work); bool cpu_intensive = cwq->wq->flags & WQ_CPU_INTENSIVE; work_func_t f = work->func; int work_color; struct worker *collision; #ifdef CONFIG_LOCKDEP /* * It is permissible to free the struct work_struct from * inside the function that is called from it, this we need to * take into account for lockdep too. To avoid bogus "held * lock freed" warnings as well as problems when looking into * work->lockdep_map, make a copy and use that here. */ struct lockdep_map lockdep_map; lockdep_copy_map(&lockdep_map, &work->lockdep_map); #endif /* * A single work shouldn't be executed concurrently by * multiple workers on a single cpu. Check whether anyone is * already processing the work. If so, defer the work to the * currently executing one. */ collision = __find_worker_executing_work(gcwq, bwh, work); if (unlikely(collision)) { move_linked_works(work, &collision->scheduled, NULL); return; } /* claim and process */ debug_work_deactivate(work); hlist_add_head(&worker->hentry, bwh); worker->current_work = work; worker->current_cwq = cwq; work_color = get_work_color(work); /* record the current cpu number in the work data and dequeue */ set_work_cpu(work, gcwq->cpu); list_del_init(&work->entry); /* * If HIGHPRI_PENDING, check the next work, and, if HIGHPRI, * wake up another worker; otherwise, clear HIGHPRI_PENDING. */ if (unlikely(gcwq->flags & GCWQ_HIGHPRI_PENDING)) { struct work_struct *nwork = list_first_entry(&gcwq->worklist, struct work_struct, entry); if (!list_empty(&gcwq->worklist) && get_work_cwq(nwork)->wq->flags & WQ_HIGHPRI) wake_up_worker(gcwq); else gcwq->flags &= ~GCWQ_HIGHPRI_PENDING; } /* * CPU intensive works don't participate in concurrency * management. They're the scheduler's responsibility. */ if (unlikely(cpu_intensive)) worker_set_flags(worker, WORKER_CPU_INTENSIVE, true); spin_unlock_irq(&gcwq->lock); work_clear_pending(work); lock_map_acquire_read(&cwq->wq->lockdep_map); lock_map_acquire(&lockdep_map); trace_workqueue_execute_start(work); f(work); /* * While we must be careful to not use "work" after this, the trace * point will only record its address. */ trace_workqueue_execute_end(work); lock_map_release(&lockdep_map); lock_map_release(&cwq->wq->lockdep_map); if (unlikely(in_atomic() || lockdep_depth(current) > 0)) { printk(KERN_ERR "BUG: workqueue leaked lock or atomic: " "%s/0x%08x/%d\n", current->comm, preempt_count(), task_pid_nr(current)); printk(KERN_ERR " last function: "); print_symbol("%s\n", (unsigned long)f); debug_show_held_locks(current); dump_stack(); } spin_lock_irq(&gcwq->lock); /* clear cpu intensive status */ if (unlikely(cpu_intensive)) worker_clr_flags(worker, WORKER_CPU_INTENSIVE); /* we're done with it, release */ hlist_del_init(&worker->hentry); worker->current_work = NULL; worker->current_cwq = NULL; cwq_dec_nr_in_flight(cwq, work_color, false); } /** * process_scheduled_works - process scheduled works * @worker: self * * Process all scheduled works. Please note that the scheduled list * may change while processing a work, so this function repeatedly * fetches a work from the top and executes it. * * CONTEXT: * spin_lock_irq(gcwq->lock) which may be released and regrabbed * multiple times. */ static void process_scheduled_works(struct worker *worker) { while (!list_empty(&worker->scheduled)) { struct work_struct *work = list_first_entry(&worker->scheduled, struct work_struct, entry); process_one_work(worker, work); } } /** * worker_thread - the worker thread function * @__worker: self * * The gcwq worker thread function. There's a single dynamic pool of * these per each cpu. These workers process all works regardless of * their specific target workqueue. The only exception is works which * belong to workqueues with a rescuer which will be explained in * rescuer_thread(). */ static int worker_thread(void *__worker) { struct worker *worker = __worker; struct global_cwq *gcwq = worker->gcwq; /* tell the scheduler that this is a workqueue worker */ worker->task->flags |= PF_WQ_WORKER; woke_up: spin_lock_irq(&gcwq->lock); /* DIE can be set only while we're idle, checking here is enough */ if (worker->flags & WORKER_DIE) { spin_unlock_irq(&gcwq->lock); worker->task->flags &= ~PF_WQ_WORKER; return 0; } worker_leave_idle(worker); recheck: /* no more worker necessary? */ if (!need_more_worker(gcwq)) goto sleep; /* do we need to manage? */ if (unlikely(!may_start_working(gcwq)) && manage_workers(worker)) goto recheck; /* * ->scheduled list can only be filled while a worker is * preparing to process a work or actually processing it. * Make sure nobody diddled with it while I was sleeping. */ BUG_ON(!list_empty(&worker->scheduled)); /* * When control reaches this point, we're guaranteed to have * at least one idle worker or that someone else has already * assumed the manager role. */ worker_clr_flags(worker, WORKER_PREP); do { struct work_struct *work = list_first_entry(&gcwq->worklist, struct work_struct, entry); if (likely(!(*work_data_bits(work) & WORK_STRUCT_LINKED))) { /* optimization path, not strictly necessary */ process_one_work(worker, work); if (unlikely(!list_empty(&worker->scheduled))) process_scheduled_works(worker); } else { move_linked_works(work, &worker->scheduled, NULL); process_scheduled_works(worker); } } while (keep_working(gcwq)); worker_set_flags(worker, WORKER_PREP, false); sleep: if (unlikely(need_to_manage_workers(gcwq)) && manage_workers(worker)) goto recheck;