/* * A work's data points to the cwq with WORK_STRUCT_CWQ set while the * work is on queue. Once execution starts, WORK_STRUCT_CWQ is * cleared and the work data contains the cpu number it was last on. * * set_work_{cwq|cpu}() and clear_work_data() can be used to set the * cwq, cpu or clear work->data. These functions should only be * called while the work is owned - ie. while the PENDING bit is set. * * get_work_[g]cwq() can be used to obtain the gcwq or cwq * corresponding to a work. gcwq is available once the work has been * queued anywhere after initialization. cwq is available only from * queueing until execution starts. */static inline void set_work_data(struct work_struct *work, unsigned long data, unsigned long flags){ BUG_ON(!work_pending(work)); atomic_long_set(&work->data, data | flags | work_static(work));} static void set_work_cwq(struct work_struct *work, struct cpu_workqueue_struct *cwq, unsigned long extra_flags){ set_work_data(work, (unsigned long)cwq, WORK_STRUCT_PENDING | WORK_STRUCT_CWQ | extra_flags);} static void set_work_cpu(struct work_struct *work, unsigned int cpu){ set_work_data(work, cpu << WORK_STRUCT_FLAG_BITS, WORK_STRUCT_PENDING);} static void clear_work_data(struct work_struct *work){ set_work_data(work, WORK_STRUCT_NO_CPU, 0);} static struct cpu_workqueue_struct *get_work_cwq(struct work_struct *work){ unsigned long data = atomic_long_read(&work->data); if (data & WORK_STRUCT_CWQ) return (void *)(data & WORK_STRUCT_WQ_DATA_MASK); else return NULL;} static struct global_cwq *get_work_gcwq(struct work_struct *work){ unsigned long data = atomic_long_read(&work->data); unsigned int cpu; if (data & WORK_STRUCT_CWQ) return ((struct cpu_workqueue_struct *) (data & WORK_STRUCT_WQ_DATA_MASK))->gcwq; cpu = data >> WORK_STRUCT_FLAG_BITS; if (cpu == WORK_CPU_NONE) return NULL; BUG_ON(cpu >= nr_cpu_ids && cpu != WORK_CPU_UNBOUND); return get_gcwq(cpu);} /* * Policy functions. These define the policies on how the global * worker pool is managed. Unless noted otherwise, these functions * assume that they're being called with gcwq->lock held. */ static bool __need_more_worker(struct global_cwq *gcwq){ return !atomic_read(get_gcwq_nr_running(gcwq->cpu)) || gcwq->flags & GCWQ_HIGHPRI_PENDING;} /* * Need to wake up a worker? Called from anything but currently * running workers. */static bool need_more_worker(struct global_cwq *gcwq){ return !list_empty(&gcwq->worklist) && __need_more_worker(gcwq);} /* Can I start working? Called from busy but !running workers. */static bool may_start_working(struct global_cwq *gcwq){ return gcwq->nr_idle;} /* Do I need to keep working? Called from currently running workers. */static bool keep_working(struct global_cwq *gcwq){ atomic_t *nr_running = get_gcwq_nr_running(gcwq->cpu); return !list_empty(&gcwq->worklist) && (atomic_read(nr_running) <= 1 || gcwq->flags & GCWQ_HIGHPRI_PENDING);} /* Do we need a new worker? Called from manager. */static bool need_to_create_worker(struct global_cwq *gcwq){ return need_more_worker(gcwq) && !may_start_working(gcwq);} /* Do I need to be the manager? */static bool need_to_manage_workers(struct global_cwq *gcwq){ return need_to_create_worker(gcwq) || gcwq->flags & GCWQ_MANAGE_WORKERS;} /* Do we have too many workers and should some go away? */static bool too_many_workers(struct global_cwq *gcwq){ bool managing = gcwq->flags & GCWQ_MANAGING_WORKERS; int nr_idle = gcwq->nr_idle + managing; /* manager is considered idle */ int nr_busy = gcwq->nr_workers - nr_idle; return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;} /* * Wake up functions. */ /* Return the first worker. Safe with preemption disabled */static struct worker *first_worker(struct global_cwq *gcwq){ if (unlikely(list_empty(&gcwq->idle_list))) return NULL; return list_first_entry(&gcwq->idle_list, struct worker, entry);} /** * wake_up_worker - wake up an idle worker * @gcwq: gcwq to wake worker for * * Wake up the first idle worker of @gcwq. * * CONTEXT: * spin_lock_irq(gcwq->lock). */static void wake_up_worker(struct global_cwq *gcwq){ struct worker *worker = first_worker(gcwq); if (likely(worker)) wake_up_process(worker->task);} /** * wq_worker_waking_up - a worker is waking up * @task: task waking up * @cpu: CPU @task is waking up to * * This function is called during try_to_wake_up() when a worker is * being awoken. * * CONTEXT: * spin_lock_irq(rq->lock) */void wq_worker_waking_up(struct task_struct *task, unsigned int cpu){ struct worker *worker = kthread_data(task); if (!(worker->flags & WORKER_NOT_RUNNING)) atomic_inc(get_gcwq_nr_running(cpu));} /** * wq_worker_sleeping - a worker is going to sleep * @task: task going to sleep * @cpu: CPU in question, must be the current CPU number * * This function is called during schedule() when a busy worker is * going to sleep. Worker on the same cpu can be woken up by * returning pointer to its task. * * CONTEXT: * spin_lock_irq(rq->lock) * * RETURNS: * Worker task on @cpu to wake up, %NULL if none. */struct task_struct *wq_worker_sleeping(struct task_struct *task, unsigned int cpu){ struct worker *worker = kthread_data(task), *to_wakeup = NULL; struct global_cwq *gcwq = get_gcwq(cpu); atomic_t *nr_running = get_gcwq_nr_running(cpu); if (worker->flags & WORKER_NOT_RUNNING) return NULL; /* this can only happen on the local cpu */ BUG_ON(cpu != raw_smp_processor_id()); /* * The counterpart of the following dec_and_test, implied mb, * worklist not empty test sequence is in insert_work(). * Please read comment there. * * NOT_RUNNING is clear. This means that trustee is not in * charge and we're running on the local cpu w/ rq lock held * and preemption disabled, which in turn means that none else * could be manipulating idle_list, so dereferencing idle_list * without gcwq lock is safe. */ if (atomic_dec_and_test(nr_running) && !list_empty(&gcwq->worklist)) to_wakeup = first_worker(gcwq); return to_wakeup ? to_wakeup->task : NULL;} /** * worker_set_flags - set worker flags and adjust nr_running accordingly * @worker: self * @flags: flags to set * @wakeup: wakeup an idle worker if necessary * * Set @flags in @worker->flags and adjust nr_running accordingly. If * nr_running becomes zero and @wakeup is %true, an idle worker is * woken up. * * CONTEXT: * spin_lock_irq(gcwq->lock) */static inline void worker_set_flags(struct worker *worker, unsigned int flags, bool wakeup){ struct global_cwq *gcwq = worker->gcwq; WARN_ON_ONCE(worker->task != current); /* * If transitioning into NOT_RUNNING, adjust nr_running and * wake up an idle worker as necessary if requested by * @wakeup. */ if ((flags & WORKER_NOT_RUNNING) && !(worker->flags & WORKER_NOT_RUNNING)) { atomic_t *nr_running = get_gcwq_nr_running(gcwq->cpu); if (wakeup) { if (atomic_dec_and_test(nr_running) && !list_empty(&gcwq->worklist)) wake_up_worker(gcwq); } else atomic_dec(nr_running); } worker->flags |= flags;} /** * worker_clr_flags - clear worker flags and adjust nr_running accordingly * @worker: self * @flags: flags to clear * * Clear @flags in @worker->flags and adjust nr_running accordingly. * * CONTEXT: * spin_lock_irq(gcwq->lock) */static inline void worker_clr_flags(struct worker *worker, unsigned int flags){ struct global_cwq *gcwq = worker->gcwq; unsigned int oflags = worker->flags; WARN_ON_ONCE(worker->task != current); worker->flags &= ~flags; /* * If transitioning out of NOT_RUNNING, increment nr_running. Note * that the nested NOT_RUNNING is not a noop. NOT_RUNNING is mask * of multiple flags, not a single flag. */ if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING)) if (!(worker->flags & WORKER_NOT_RUNNING)) atomic_inc(get_gcwq_nr_running(gcwq->cpu));} /** * busy_worker_head - return the busy hash head for a work * @gcwq: gcwq of interest * @work: work to be hashed * * Return hash head of @gcwq for @work. * * CONTEXT: * spin_lock_irq(gcwq->lock). * * RETURNS: * Pointer to the hash head. */static struct hlist_head *busy_worker_head(struct global_cwq *gcwq, struct work_struct *work){ const int base_shift = ilog2(sizeof(struct work_struct)); unsigned long v = (unsigned long)work; /* simple shift and fold hash, do we need something better? */ v >>= base_shift; v += v >> BUSY_WORKER_HASH_ORDER; v &= BUSY_WORKER_HASH_MASK; return &gcwq->busy_hash[v];} /** * __find_worker_executing_work - find worker which is executing a work * @gcwq: gcwq of interest * @bwh: hash head as returned by busy_worker_head() * @work: work to find worker for * * Find a worker which is executing @work on @gcwq. @bwh should be * the hash head obtained by calling busy_worker_head() with the same * work. * * CONTEXT: * spin_lock_irq(gcwq->lock). * * RETURNS: * Pointer to worker which is executing @work if found, NULL * otherwise. */static struct worker *__find_worker_executing_work(struct global_cwq *gcwq, struct hlist_head *bwh, struct work_struct *work){ struct worker *worker; struct hlist_node *tmp; hlist_for_each_entry(worker, tmp, bwh, hentry) if (worker->current_work == work) return worker; return NULL;} /** * find_worker_executing_work - find worker which is executing a work * @gcwq: gcwq of interest * @work: work to find worker for * * Find a worker which is executing @work on @gcwq. This function is * identical to __find_worker_executing_work() except that this * function calculates @bwh itself. * * CONTEXT: * spin_lock_irq(gcwq->lock). * * RETURNS: * Pointer to worker which is executing @work if found, NULL * otherwise. */static struct worker *find_worker_executing_work(struct global_cwq *gcwq, struct work_struct *work){ return __find_worker_executing_work(gcwq, busy_worker_head(gcwq, work), work);} /** * gcwq_determine_ins_pos - find insertion position * @gcwq: gcwq of interest * @cwq: cwq a work is being queued for * * A work for @cwq is about to be queued on @gcwq, determine insertion * position for the work. If @cwq is for HIGHPRI wq, the work is * queued at the head of the queue but in FIFO order with respect to * other HIGHPRI works; otherwise, at the end of the queue. This * function also sets GCWQ_HIGHPRI_PENDING flag to hint @gcwq that * there are HIGHPRI works pending. * * CONTEXT: * spin_lock_irq(gcwq->lock). * * RETURNS: * Pointer to inserstion position. */static inline struct list_head *gcwq_determine_ins_pos(struct global_cwq *gcwq, struct cpu_workqueue_struct *cwq){ struct work_struct *twork; if (likely(!(cwq->wq->flags & WQ_HIGHPRI))) return &gcwq->worklist; list_for_each_entry(twork, &gcwq->worklist, entry) { struct cpu_workqueue_struct *tcwq = get_work_cwq(twork); if (!(tcwq->wq->flags & WQ_HIGHPRI)) break; } gcwq->flags |= GCWQ_HIGHPRI_PENDING; return &twork->entry;} /** * insert_work - insert a work into gcwq * @cwq: cwq @work belongs to * @work: work to insert * @head: insertion point * @extra_flags: extra WORK_STRUCT_* flags to set * * Insert @work which belongs to @cwq into @gcwq after @head. * @extra_flags is or'd to work_struct flags. * * CONTEXT: * spin_lock_irq(gcwq->lock). */static void insert_work(struct cpu_workqueue_struct *cwq, struct work_struct *work, struct list_head *head, unsigned int extra_flags){ struct global_cwq *gcwq = cwq->gcwq; /* we own @work, set data and link */ set_work_cwq(work, cwq, extra_flags); /* * Ensure that we get the right work->data if we see the * result of list_add() below, see try_to_grab_pending(). */ smp_wmb(); list_add_tail(&work->entry, head); /* * Ensure either worker_sched_deactivated() sees the above * list_add_tail() or we see zero nr_running to avoid workers * lying around lazily while there are works to be processed. */ smp_mb(); if (__need_more_worker(gcwq)) wake_up_worker(gcwq);} /* * Test whether @work is being queued from another work executing on the * same workqueue. This is rather expensive and should only be used from * cold paths. */static bool is_chained_work(struct workqueue_struct *wq){ unsigned long flags; unsigned int cpu; for_each_gcwq_cpu(cpu) { struct global_cwq *gcwq = get_gcwq(cpu); struct worker *worker; struct hlist_node *pos; int i; spin_lock_irqsave(&gcwq->lock, flags); for_each_busy_worker(worker, i, pos, gcwq) { if (worker->task != current) continue; spin_unlock_irqrestore(&gcwq->lock, flags); /* * I'm @worker, no locking necessary. See if @work * is headed to the same workqueue. */ return worker->current_cwq->wq == wq; } spin_unlock_irqrestore(&gcwq->lock, flags); } return false;} static void __queue_work(unsigned int cpu, struct workqueue_struct *wq, struct work_struct *work){ struct global_cwq *gcwq; struct cpu_workqueue_struct *cwq; struct list_head *worklist; unsigned int work_flags; unsigned long flags; debug_work_activate(work); /* if dying, only works from the same workqueue are allowed */ if (unlikely(wq->flags & WQ_DRAINING) && WARN_ON_ONCE(!is_chained_work(wq))) return; /* determine gcwq to use */ if (!(wq->flags & WQ_UNBOUND)) { struct global_cwq *last_gcwq; if (unlikely(cpu == WORK_CPU_UNBOUND)) cpu = raw_smp_processor_id();Lines 501–1000 of 3842. Previous lines Load more
/* * A work's data points to the cwq with WORK_STRUCT_CWQ set while the * work is on queue. Once execution starts, WORK_STRUCT_CWQ is * cleared and the work data contains the cpu number it was last on. * * set_work_{cwq|cpu}() and clear_work_data() can be used to set the * cwq, cpu or clear work->data. These functions should only be * called while the work is owned - ie. while the PENDING bit is set. * * get_work_[g]cwq() can be used to obtain the gcwq or cwq * corresponding to a work. gcwq is available once the work has been * queued anywhere after initialization. cwq is available only from * queueing until execution starts. */ static inline void set_work_data(struct work_struct *work, unsigned long data, unsigned long flags) { BUG_ON(!work_pending(work)); atomic_long_set(&work->data, data | flags | work_static(work)); } static void set_work_cwq(struct work_struct *work, struct cpu_workqueue_struct *cwq, unsigned long extra_flags) { set_work_data(work, (unsigned long)cwq, WORK_STRUCT_PENDING | WORK_STRUCT_CWQ | extra_flags); } static void set_work_cpu(struct work_struct *work, unsigned int cpu) { set_work_data(work, cpu << WORK_STRUCT_FLAG_BITS, WORK_STRUCT_PENDING); } static void clear_work_data(struct work_struct *work) { set_work_data(work, WORK_STRUCT_NO_CPU, 0); } static struct cpu_workqueue_struct *get_work_cwq(struct work_struct *work) { unsigned long data = atomic_long_read(&work->data); if (data & WORK_STRUCT_CWQ) return (void *)(data & WORK_STRUCT_WQ_DATA_MASK); else return NULL; } static struct global_cwq *get_work_gcwq(struct work_struct *work) { unsigned long data = atomic_long_read(&work->data); unsigned int cpu; if (data & WORK_STRUCT_CWQ) return ((struct cpu_workqueue_struct *) (data & WORK_STRUCT_WQ_DATA_MASK))->gcwq; cpu = data >> WORK_STRUCT_FLAG_BITS; if (cpu == WORK_CPU_NONE) return NULL; BUG_ON(cpu >= nr_cpu_ids && cpu != WORK_CPU_UNBOUND); return get_gcwq(cpu); } /* * Policy functions. These define the policies on how the global * worker pool is managed. Unless noted otherwise, these functions * assume that they're being called with gcwq->lock held. */ static bool __need_more_worker(struct global_cwq *gcwq) { return !atomic_read(get_gcwq_nr_running(gcwq->cpu)) || gcwq->flags & GCWQ_HIGHPRI_PENDING; } /* * Need to wake up a worker? Called from anything but currently * running workers. */ static bool need_more_worker(struct global_cwq *gcwq) { return !list_empty(&gcwq->worklist) && __need_more_worker(gcwq); } /* Can I start working? Called from busy but !running workers. */ static bool may_start_working(struct global_cwq *gcwq) { return gcwq->nr_idle; } /* Do I need to keep working? Called from currently running workers. */ static bool keep_working(struct global_cwq *gcwq) { atomic_t *nr_running = get_gcwq_nr_running(gcwq->cpu); return !list_empty(&gcwq->worklist) && (atomic_read(nr_running) <= 1 || gcwq->flags & GCWQ_HIGHPRI_PENDING); } /* Do we need a new worker? Called from manager. */ static bool need_to_create_worker(struct global_cwq *gcwq) { return need_more_worker(gcwq) && !may_start_working(gcwq); } /* Do I need to be the manager? */ static bool need_to_manage_workers(struct global_cwq *gcwq) { return need_to_create_worker(gcwq) || gcwq->flags & GCWQ_MANAGE_WORKERS; } /* Do we have too many workers and should some go away? */ static bool too_many_workers(struct global_cwq *gcwq) { bool managing = gcwq->flags & GCWQ_MANAGING_WORKERS; int nr_idle = gcwq->nr_idle + managing; /* manager is considered idle */ int nr_busy = gcwq->nr_workers - nr_idle; return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy; } /* * Wake up functions. */ /* Return the first worker. Safe with preemption disabled */ static struct worker *first_worker(struct global_cwq *gcwq) { if (unlikely(list_empty(&gcwq->idle_list))) return NULL; return list_first_entry(&gcwq->idle_list, struct worker, entry); } /** * wake_up_worker - wake up an idle worker * @gcwq: gcwq to wake worker for * * Wake up the first idle worker of @gcwq. * * CONTEXT: * spin_lock_irq(gcwq->lock). */ static void wake_up_worker(struct global_cwq *gcwq) { struct worker *worker = first_worker(gcwq); if (likely(worker)) wake_up_process(worker->task); } /** * wq_worker_waking_up - a worker is waking up * @task: task waking up * @cpu: CPU @task is waking up to * * This function is called during try_to_wake_up() when a worker is * being awoken. * * CONTEXT: * spin_lock_irq(rq->lock) */ void wq_worker_waking_up(struct task_struct *task, unsigned int cpu) { struct worker *worker = kthread_data(task); if (!(worker->flags & WORKER_NOT_RUNNING)) atomic_inc(get_gcwq_nr_running(cpu)); } /** * wq_worker_sleeping - a worker is going to sleep * @task: task going to sleep * @cpu: CPU in question, must be the current CPU number * * This function is called during schedule() when a busy worker is * going to sleep. Worker on the same cpu can be woken up by * returning pointer to its task. * * CONTEXT: * spin_lock_irq(rq->lock) * * RETURNS: * Worker task on @cpu to wake up, %NULL if none. */ struct task_struct *wq_worker_sleeping(struct task_struct *task, unsigned int cpu) { struct worker *worker = kthread_data(task), *to_wakeup = NULL; struct global_cwq *gcwq = get_gcwq(cpu); atomic_t *nr_running = get_gcwq_nr_running(cpu); if (worker->flags & WORKER_NOT_RUNNING) return NULL; /* this can only happen on the local cpu */ BUG_ON(cpu != raw_smp_processor_id()); /* * The counterpart of the following dec_and_test, implied mb, * worklist not empty test sequence is in insert_work(). * Please read comment there. * * NOT_RUNNING is clear. This means that trustee is not in * charge and we're running on the local cpu w/ rq lock held * and preemption disabled, which in turn means that none else * could be manipulating idle_list, so dereferencing idle_list * without gcwq lock is safe. */ if (atomic_dec_and_test(nr_running) && !list_empty(&gcwq->worklist)) to_wakeup = first_worker(gcwq); return to_wakeup ? to_wakeup->task : NULL; } /** * worker_set_flags - set worker flags and adjust nr_running accordingly * @worker: self * @flags: flags to set * @wakeup: wakeup an idle worker if necessary * * Set @flags in @worker->flags and adjust nr_running accordingly. If * nr_running becomes zero and @wakeup is %true, an idle worker is * woken up. * * CONTEXT: * spin_lock_irq(gcwq->lock) */ static inline void worker_set_flags(struct worker *worker, unsigned int flags, bool wakeup) { struct global_cwq *gcwq = worker->gcwq; WARN_ON_ONCE(worker->task != current); /* * If transitioning into NOT_RUNNING, adjust nr_running and * wake up an idle worker as necessary if requested by * @wakeup. */ if ((flags & WORKER_NOT_RUNNING) && !(worker->flags & WORKER_NOT_RUNNING)) { atomic_t *nr_running = get_gcwq_nr_running(gcwq->cpu); if (wakeup) { if (atomic_dec_and_test(nr_running) && !list_empty(&gcwq->worklist)) wake_up_worker(gcwq); } else atomic_dec(nr_running); } worker->flags |= flags; } /** * worker_clr_flags - clear worker flags and adjust nr_running accordingly * @worker: self * @flags: flags to clear * * Clear @flags in @worker->flags and adjust nr_running accordingly. * * CONTEXT: * spin_lock_irq(gcwq->lock) */ static inline void worker_clr_flags(struct worker *worker, unsigned int flags) { struct global_cwq *gcwq = worker->gcwq; unsigned int oflags = worker->flags; WARN_ON_ONCE(worker->task != current); worker->flags &= ~flags; /* * If transitioning out of NOT_RUNNING, increment nr_running. Note * that the nested NOT_RUNNING is not a noop. NOT_RUNNING is mask * of multiple flags, not a single flag. */ if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING)) if (!(worker->flags & WORKER_NOT_RUNNING)) atomic_inc(get_gcwq_nr_running(gcwq->cpu)); } /** * busy_worker_head - return the busy hash head for a work * @gcwq: gcwq of interest * @work: work to be hashed * * Return hash head of @gcwq for @work. * * CONTEXT: * spin_lock_irq(gcwq->lock). * * RETURNS: * Pointer to the hash head. */ static struct hlist_head *busy_worker_head(struct global_cwq *gcwq, struct work_struct *work) { const int base_shift = ilog2(sizeof(struct work_struct)); unsigned long v = (unsigned long)work; /* simple shift and fold hash, do we need something better? */ v >>= base_shift; v += v >> BUSY_WORKER_HASH_ORDER; v &= BUSY_WORKER_HASH_MASK; return &gcwq->busy_hash[v]; } /** * __find_worker_executing_work - find worker which is executing a work * @gcwq: gcwq of interest * @bwh: hash head as returned by busy_worker_head() * @work: work to find worker for * * Find a worker which is executing @work on @gcwq. @bwh should be * the hash head obtained by calling busy_worker_head() with the same * work. * * CONTEXT: * spin_lock_irq(gcwq->lock). * * RETURNS: * Pointer to worker which is executing @work if found, NULL * otherwise. */ static struct worker *__find_worker_executing_work(struct global_cwq *gcwq, struct hlist_head *bwh, struct work_struct *work) { struct worker *worker; struct hlist_node *tmp; hlist_for_each_entry(worker, tmp, bwh, hentry) if (worker->current_work == work) return worker; return NULL; } /** * find_worker_executing_work - find worker which is executing a work * @gcwq: gcwq of interest * @work: work to find worker for * * Find a worker which is executing @work on @gcwq. This function is * identical to __find_worker_executing_work() except that this * function calculates @bwh itself. * * CONTEXT: * spin_lock_irq(gcwq->lock). * * RETURNS: * Pointer to worker which is executing @work if found, NULL * otherwise. */ static struct worker *find_worker_executing_work(struct global_cwq *gcwq, struct work_struct *work) { return __find_worker_executing_work(gcwq, busy_worker_head(gcwq, work), work); } /** * gcwq_determine_ins_pos - find insertion position * @gcwq: gcwq of interest * @cwq: cwq a work is being queued for * * A work for @cwq is about to be queued on @gcwq, determine insertion * position for the work. If @cwq is for HIGHPRI wq, the work is * queued at the head of the queue but in FIFO order with respect to * other HIGHPRI works; otherwise, at the end of the queue. This * function also sets GCWQ_HIGHPRI_PENDING flag to hint @gcwq that * there are HIGHPRI works pending. * * CONTEXT: * spin_lock_irq(gcwq->lock). * * RETURNS: * Pointer to inserstion position. */ static inline struct list_head *gcwq_determine_ins_pos(struct global_cwq *gcwq, struct cpu_workqueue_struct *cwq) { struct work_struct *twork; if (likely(!(cwq->wq->flags & WQ_HIGHPRI))) return &gcwq->worklist; list_for_each_entry(twork, &gcwq->worklist, entry) { struct cpu_workqueue_struct *tcwq = get_work_cwq(twork); if (!(tcwq->wq->flags & WQ_HIGHPRI)) break; } gcwq->flags |= GCWQ_HIGHPRI_PENDING; return &twork->entry; } /** * insert_work - insert a work into gcwq * @cwq: cwq @work belongs to * @work: work to insert * @head: insertion point * @extra_flags: extra WORK_STRUCT_* flags to set * * Insert @work which belongs to @cwq into @gcwq after @head. * @extra_flags is or'd to work_struct flags. * * CONTEXT: * spin_lock_irq(gcwq->lock). */ static void insert_work(struct cpu_workqueue_struct *cwq, struct work_struct *work, struct list_head *head, unsigned int extra_flags) { struct global_cwq *gcwq = cwq->gcwq; /* we own @work, set data and link */ set_work_cwq(work, cwq, extra_flags); /* * Ensure that we get the right work->data if we see the * result of list_add() below, see try_to_grab_pending(). */ smp_wmb(); list_add_tail(&work->entry, head); /* * Ensure either worker_sched_deactivated() sees the above * list_add_tail() or we see zero nr_running to avoid workers * lying around lazily while there are works to be processed. */ smp_mb(); if (__need_more_worker(gcwq)) wake_up_worker(gcwq); } /* * Test whether @work is being queued from another work executing on the * same workqueue. This is rather expensive and should only be used from * cold paths. */ static bool is_chained_work(struct workqueue_struct *wq) { unsigned long flags; unsigned int cpu; for_each_gcwq_cpu(cpu) { struct global_cwq *gcwq = get_gcwq(cpu); struct worker *worker; struct hlist_node *pos; int i; spin_lock_irqsave(&gcwq->lock, flags); for_each_busy_worker(worker, i, pos, gcwq) { if (worker->task != current) continue; spin_unlock_irqrestore(&gcwq->lock, flags); /* * I'm @worker, no locking necessary. See if @work * is headed to the same workqueue. */ return worker->current_cwq->wq == wq; } spin_unlock_irqrestore(&gcwq->lock, flags); } return false; } static void __queue_work(unsigned int cpu, struct workqueue_struct *wq, struct work_struct *work) { struct global_cwq *gcwq; struct cpu_workqueue_struct *cwq; struct list_head *worklist; unsigned int work_flags; unsigned long flags; debug_work_activate(work); /* if dying, only works from the same workqueue are allowed */ if (unlikely(wq->flags & WQ_DRAINING) && WARN_ON_ONCE(!is_chained_work(wq))) return; /* determine gcwq to use */ if (!(wq->flags & WQ_UNBOUND)) { struct global_cwq *last_gcwq; if (unlikely(cpu == WORK_CPU_UNBOUND)) cpu = raw_smp_processor_id();