History

workqueue: use @pool instead of @gcwq or @cpu where applicable

Modify all functions which deal with per-pool properties to pass around @pool instead of @gcwq or @cpu. The changes in this patch are mechanical and don't caues any functional difference. This is to prepare for multiple pools per gcwq.

Tejun Heo committed 14 years ago

Commit 63d95a91 parent bd7bdd41 file · +111 −107Browse files

Unified · Split

kernel/workqueue.c+111−107
@@ -471,8 +471,10 @@ static struct global_cwq *get_gcwq(unsigned int cpu) · 470 unmodified lines
471471 return &unbound_global_cwq;
472472 }
473473
474−static atomic_t *get_gcwq_nr_running(unsigned int cpu)
474+static atomic_t *get_pool_nr_running(struct worker_pool *pool)
475475 {
476+ int cpu = pool->gcwq->cpu;
477+
476478 if (cpu != WORK_CPU_UNBOUND)
477479 return &per_cpu(gcwq_nr_running, cpu);
478480 else
@@ -578,10 +580,10 @@ static struct global_cwq *get_work_gcwq(struct work_struct *work) · 99 unmodified lines
578580 * assume that they're being called with gcwq->lock held.
579581 */
580582
581−static bool __need_more_worker(struct global_cwq *gcwq)
583+static bool __need_more_worker(struct worker_pool *pool)
582584 {
583− return !atomic_read(get_gcwq_nr_running(gcwq->cpu)) ||
584− gcwq->flags & GCWQ_HIGHPRI_PENDING;
585+ return !atomic_read(get_pool_nr_running(pool)) ||
586+ pool->gcwq->flags & GCWQ_HIGHPRI_PENDING;
585587 }
586588
587589 /*
@@ -592,45 +594,46 @@ static bool __need_more_worker(struct global_cwq *gcwq) · 4 unmodified lines
592594 * function will always return %true for unbound gcwq as long as the
593595 * worklist isn't empty.
594596 */
595−static bool need_more_worker(struct global_cwq *gcwq)
597+static bool need_more_worker(struct worker_pool *pool)
596598 {
597− return !list_empty(&gcwq->pool.worklist) && __need_more_worker(gcwq);
599+ return !list_empty(&pool->worklist) && __need_more_worker(pool);
598600 }
599601
600602 /* Can I start working? Called from busy but !running workers. */
601−static bool may_start_working(struct global_cwq *gcwq)
603+static bool may_start_working(struct worker_pool *pool)
602604 {
603− return gcwq->pool.nr_idle;
605+ return pool->nr_idle;
604606 }
605607
606608 /* Do I need to keep working? Called from currently running workers. */
607−static bool keep_working(struct global_cwq *gcwq)
609+static bool keep_working(struct worker_pool *pool)
608610 {
609− atomic_t *nr_running = get_gcwq_nr_running(gcwq->cpu);
611+ atomic_t *nr_running = get_pool_nr_running(pool);
610612
611− return !list_empty(&gcwq->pool.worklist) &&
613+ return !list_empty(&pool->worklist) &&
612614 (atomic_read(nr_running) <= 1 ||
613− gcwq->flags & GCWQ_HIGHPRI_PENDING);
615+ pool->gcwq->flags & GCWQ_HIGHPRI_PENDING);
614616 }
615617
616618 /* Do we need a new worker? Called from manager. */
617−static bool need_to_create_worker(struct global_cwq *gcwq)
619+static bool need_to_create_worker(struct worker_pool *pool)
618620 {
619− return need_more_worker(gcwq) && !may_start_working(gcwq);
621+ return need_more_worker(pool) && !may_start_working(pool);
620622 }
621623
622624 /* Do I need to be the manager? */
623−static bool need_to_manage_workers(struct global_cwq *gcwq)
625+static bool need_to_manage_workers(struct worker_pool *pool)
624626 {
625− return need_to_create_worker(gcwq) || gcwq->flags & GCWQ_MANAGE_WORKERS;
627+ return need_to_create_worker(pool) ||
628+ pool->gcwq->flags & GCWQ_MANAGE_WORKERS;
626629 }
627630
628631 /* Do we have too many workers and should some go away? */
629−static bool too_many_workers(struct global_cwq *gcwq)
632+static bool too_many_workers(struct worker_pool *pool)
630633 {
631− bool managing = gcwq->flags & GCWQ_MANAGING_WORKERS;
632− int nr_idle = gcwq->pool.nr_idle + managing; /* manager is considered idle */
633− int nr_busy = gcwq->pool.nr_workers - nr_idle;
634+ bool managing = pool->gcwq->flags & GCWQ_MANAGING_WORKERS;
635+ int nr_idle = pool->nr_idle + managing; /* manager is considered idle */
636+ int nr_busy = pool->nr_workers - nr_idle;
634637
635638 return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;
636639 }
@@ -640,26 +643,26 @@ static bool too_many_workers(struct global_cwq *gcwq) · 3 unmodified lines
640643 */
641644
642645 /* Return the first worker. Safe with preemption disabled */
643−static struct worker *first_worker(struct global_cwq *gcwq)
646+static struct worker *first_worker(struct worker_pool *pool)
644647 {
645− if (unlikely(list_empty(&gcwq->pool.idle_list)))
648+ if (unlikely(list_empty(&pool->idle_list)))
646649 return NULL;
647650
648− return list_first_entry(&gcwq->pool.idle_list, struct worker, entry);
651+ return list_first_entry(&pool->idle_list, struct worker, entry);
649652 }
650653
651654 /**
652655 * wake_up_worker - wake up an idle worker
653− * @gcwq: gcwq to wake worker for
656+ * @pool: worker pool to wake worker from
654657 *
655− * Wake up the first idle worker of @gcwq.
658+ * Wake up the first idle worker of @pool.
656659 *
657660 * CONTEXT:
658661 * spin_lock_irq(gcwq->lock).
659662 */
660−static void wake_up_worker(struct global_cwq *gcwq)
663+static void wake_up_worker(struct worker_pool *pool)
661664 {
662− struct worker *worker = first_worker(gcwq);
665+ struct worker *worker = first_worker(pool);
663666
664667 if (likely(worker))
665668 wake_up_process(worker->task);
@@ -681,7 +684,7 @@ void wq_worker_waking_up(struct task_struct *task, unsigned int cpu) · 15 unmodified lines
681684 struct worker *worker = kthread_data(task);
682685
683686 if (!(worker->flags & WORKER_NOT_RUNNING))
684− atomic_inc(get_gcwq_nr_running(cpu));
687+ atomic_inc(get_pool_nr_running(worker->pool));
685688 }
686689
687690 /**
@@ -704,8 +707,7 @@ struct task_struct *wq_worker_sleeping(struct task_struct *task, · 16 unmodified lines
704707 {
705708 struct worker *worker = kthread_data(task), *to_wakeup = NULL;
706709 struct worker_pool *pool = worker->pool;
707− struct global_cwq *gcwq = pool->gcwq;
708− atomic_t *nr_running = get_gcwq_nr_running(cpu);
710+ atomic_t *nr_running = get_pool_nr_running(pool);
709711
710712 if (worker->flags & WORKER_NOT_RUNNING)
711713 return NULL;
@@ -725,7 +727,7 @@ struct task_struct *wq_worker_sleeping(struct task_struct *task, · 13 unmodified lines
725727 * without gcwq lock is safe.
726728 */
727729 if (atomic_dec_and_test(nr_running) && !list_empty(&pool->worklist))
728− to_wakeup = first_worker(gcwq);
730+ to_wakeup = first_worker(pool);
729731 return to_wakeup ? to_wakeup->task : NULL;
730732 }
731733
@@ -746,7 +748,6 @@ static inline void worker_set_flags(struct worker *worker, unsigned int flags, · 14 unmodified lines
746748 bool wakeup)
747749 {
748750 struct worker_pool *pool = worker->pool;
749− struct global_cwq *gcwq = pool->gcwq;
750751
751752 WARN_ON_ONCE(worker->task != current);
752753
@@ -757,12 +758,12 @@ static inline void worker_set_flags(struct worker *worker, unsigned int flags, · 4 unmodified lines
757758 */
758759 if ((flags & WORKER_NOT_RUNNING) &&
759760 !(worker->flags & WORKER_NOT_RUNNING)) {
760− atomic_t *nr_running = get_gcwq_nr_running(gcwq->cpu);
761+ atomic_t *nr_running = get_pool_nr_running(pool);
761762
762763 if (wakeup) {
763764 if (atomic_dec_and_test(nr_running) &&
764765 !list_empty(&pool->worklist))
765− wake_up_worker(gcwq);
766+ wake_up_worker(pool);
766767 } else
767768 atomic_dec(nr_running);
768769 }
@@ -782,7 +783,7 @@ static inline void worker_set_flags(struct worker *worker, unsigned int flags, · 13 unmodified lines
782783 */
783784 static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
784785 {
785− struct global_cwq *gcwq = worker->pool->gcwq;
786+ struct worker_pool *pool = worker->pool;
786787 unsigned int oflags = worker->flags;
787788
788789 WARN_ON_ONCE(worker->task != current);
@@ -796,7 +797,7 @@ static inline void worker_clr_flags(struct worker *worker, unsigned int flags) · 7 unmodified lines
796797 */
797798 if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
798799 if (!(worker->flags & WORKER_NOT_RUNNING))
799− atomic_inc(get_gcwq_nr_running(gcwq->cpu));
800+ atomic_inc(get_pool_nr_running(pool));
800801 }
801802
802803 /**
@@ -880,15 +881,15 @@ static struct worker *find_worker_executing_work(struct global_cwq *gcwq, · 77 unmodified lines
880881 }
881882
882883 /**
883− * gcwq_determine_ins_pos - find insertion position
884− * @gcwq: gcwq of interest
884+ * pool_determine_ins_pos - find insertion position
885+ * @pool: pool of interest
885886 * @cwq: cwq a work is being queued for
886887 *
887− * A work for @cwq is about to be queued on @gcwq, determine insertion
888+ * A work for @cwq is about to be queued on @pool, determine insertion
888889 * position for the work. If @cwq is for HIGHPRI wq, the work is
889890 * queued at the head of the queue but in FIFO order with respect to
890891 * other HIGHPRI works; otherwise, at the end of the queue. This
891− * function also sets GCWQ_HIGHPRI_PENDING flag to hint @gcwq that
892+ * function also sets GCWQ_HIGHPRI_PENDING flag to hint @pool that
892893 * there are HIGHPRI works pending.
893894 *
894895 * CONTEXT:
@@ -897,22 +898,22 @@ static struct worker *find_worker_executing_work(struct global_cwq *gcwq, · 2 unmodified lines
897898 * RETURNS:
898899 * Pointer to inserstion position.
899900 */
900−static inline struct list_head *gcwq_determine_ins_pos(struct global_cwq *gcwq,
901+static inline struct list_head *pool_determine_ins_pos(struct worker_pool *pool,
901902 struct cpu_workqueue_struct *cwq)
902903 {
903904 struct work_struct *twork;
904905
905906 if (likely(!(cwq->wq->flags & WQ_HIGHPRI)))
906− return &gcwq->pool.worklist;
907+ return &pool->worklist;
907908
908− list_for_each_entry(twork, &gcwq->pool.worklist, entry) {
909+ list_for_each_entry(twork, &pool->worklist, entry) {
909910 struct cpu_workqueue_struct *tcwq = get_work_cwq(twork);
910911
911912 if (!(tcwq->wq->flags & WQ_HIGHPRI))
912913 break;
913914 }
914915
915− gcwq->flags |= GCWQ_HIGHPRI_PENDING;
916+ pool->gcwq->flags |= GCWQ_HIGHPRI_PENDING;
916917 return &twork->entry;
917918 }
918919
@@ -933,7 +934,7 @@ static void insert_work(struct cpu_workqueue_struct *cwq, · 14 unmodified lines
933934 struct work_struct *work, struct list_head *head,
934935 unsigned int extra_flags)
935936 {
936− struct global_cwq *gcwq = cwq->pool->gcwq;
937+ struct worker_pool *pool = cwq->pool;
937938
938939 /* we own @work, set data and link */
939940 set_work_cwq(work, cwq, extra_flags);
@@ -953,8 +954,8 @@ static void insert_work(struct cpu_workqueue_struct *cwq, · 13 unmodified lines
953954 */
954955 smp_mb();
955956
956− if (__need_more_worker(gcwq))
957− wake_up_worker(gcwq);
957+ if (__need_more_worker(pool))
958+ wake_up_worker(pool);
958959 }
959960
960961 /*
@@ -1056,7 +1057,7 @@ static void __queue_work(unsigned int cpu, struct workqueue_struct *wq, · 95 unmodified lines
10561057 if (likely(cwq->nr_active < cwq->max_active)) {
10571058 trace_workqueue_activate_work(work);
10581059 cwq->nr_active++;
1059− worklist = gcwq_determine_ins_pos(gcwq, cwq);
1060+ worklist = pool_determine_ins_pos(cwq->pool, cwq);
10601061 } else {
10611062 work_flags |= WORK_STRUCT_DELAYED;
10621063 worklist = &cwq->delayed_works;
@@ -1221,7 +1222,7 @@ static void worker_enter_idle(struct worker *worker) · 158 unmodified lines
12211222 list_add(&worker->entry, &pool->idle_list);
12221223
12231224 if (likely(!(worker->flags & WORKER_ROGUE))) {
1224− if (too_many_workers(gcwq) && !timer_pending(&pool->idle_timer))
1225+ if (too_many_workers(pool) && !timer_pending(&pool->idle_timer))
12251226 mod_timer(&pool->idle_timer,
12261227 jiffies + IDLE_WORKER_TIMEOUT);
12271228 } else
@@ -1234,7 +1235,7 @@ static void worker_enter_idle(struct worker *worker) · 6 unmodified lines
12341235 */
12351236 WARN_ON_ONCE(gcwq->trustee_state == TRUSTEE_DONE &&
12361237 pool->nr_workers == pool->nr_idle &&
1237− atomic_read(get_gcwq_nr_running(gcwq->cpu)));
1238+ atomic_read(get_pool_nr_running(pool)));
12381239 }
12391240
12401241 /**
@@ -1356,10 +1357,10 @@ static struct worker *alloc_worker(void) · 115 unmodified lines
13561357
13571358 /**
13581359 * create_worker - create a new workqueue worker
1359− * @gcwq: gcwq the new worker will belong to
1360+ * @pool: pool the new worker will belong to
13601361 * @bind: whether to set affinity to @cpu or not
13611362 *
1362− * Create a new worker which is bound to @gcwq. The returned worker
1363+ * Create a new worker which is bound to @pool. The returned worker
13631364 * can be started by calling start_worker() or destroyed using
13641365 * destroy_worker().
13651366 *
@@ -1369,10 +1370,10 @@ static struct worker *alloc_worker(void) · 3 unmodified lines
13691370 * RETURNS:
13701371 * Pointer to the newly created worker.
13711372 */
1372−static struct worker *create_worker(struct global_cwq *gcwq, bool bind)
1373+static struct worker *create_worker(struct worker_pool *pool, bool bind)
13731374 {
1375+ struct global_cwq *gcwq = pool->gcwq;
13741376 bool on_unbound_cpu = gcwq->cpu == WORK_CPU_UNBOUND;
1375− struct worker_pool *pool = &gcwq->pool;
13761377 struct worker *worker = NULL;
13771378 int id = -1;
13781379
@@ -1480,27 +1481,27 @@ static void destroy_worker(struct worker *worker) · 101 unmodified lines
14801481 ida_remove(&pool->worker_ida, id);
14811482 }
14821483
1483−static void idle_worker_timeout(unsigned long __gcwq)
1484+static void idle_worker_timeout(unsigned long __pool)
14841485 {
1485− struct global_cwq *gcwq = (void *)__gcwq;
1486+ struct worker_pool *pool = (void *)__pool;
1487+ struct global_cwq *gcwq = pool->gcwq;
14861488
14871489 spin_lock_irq(&gcwq->lock);
14881490
1489− if (too_many_workers(gcwq)) {
1491+ if (too_many_workers(pool)) {
14901492 struct worker *worker;
14911493 unsigned long expires;
14921494
14931495 /* idle_list is kept in LIFO order, check the last one */
1494− worker = list_entry(gcwq->pool.idle_list.prev, struct worker,
1495− entry);
1496+ worker = list_entry(pool->idle_list.prev, struct worker, entry);
14961497 expires = worker->last_active + IDLE_WORKER_TIMEOUT;
14971498
14981499 if (time_before(jiffies, expires))
1499− mod_timer(&gcwq->pool.idle_timer, expires);
1500+ mod_timer(&pool->idle_timer, expires);
15001501 else {
15011502 /* it's been idle for too long, wake up manager */
15021503 gcwq->flags |= GCWQ_MANAGE_WORKERS;
1503− wake_up_worker(gcwq);
1504+ wake_up_worker(pool);
15041505 }
15051506 }
15061507
@@ -1526,37 +1527,38 @@ static bool send_mayday(struct work_struct *work) · 19 unmodified lines
15261527 return true;
15271528 }
15281529
1529−static void gcwq_mayday_timeout(unsigned long __gcwq)
1530+static void gcwq_mayday_timeout(unsigned long __pool)
15301531 {
1531− struct global_cwq *gcwq = (void *)__gcwq;
1532+ struct worker_pool *pool = (void *)__pool;
1533+ struct global_cwq *gcwq = pool->gcwq;
15321534 struct work_struct *work;
15331535
15341536 spin_lock_irq(&gcwq->lock);
15351537
1536− if (need_to_create_worker(gcwq)) {
1538+ if (need_to_create_worker(pool)) {
15371539 /*
15381540 * We've been trying to create a new worker but
15391541 * haven't been successful. We might be hitting an
15401542 * allocation deadlock. Send distress signals to
15411543 * rescuers.
15421544 */
1543− list_for_each_entry(work, &gcwq->pool.worklist, entry)
1545+ list_for_each_entry(work, &pool->worklist, entry)
15441546 send_mayday(work);
15451547 }
15461548
15471549 spin_unlock_irq(&gcwq->lock);
15481550
1549− mod_timer(&gcwq->pool.mayday_timer, jiffies + MAYDAY_INTERVAL);
1551+ mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);
15501552 }
15511553
15521554 /**
15531555 * maybe_create_worker - create a new worker if necessary
1554− * @gcwq: gcwq to create a new worker for
1556+ * @pool: pool to create a new worker for
15551557 *
1556− * Create a new worker for @gcwq if necessary. @gcwq is guaranteed to
1558+ * Create a new worker for @pool if necessary. @pool is guaranteed to
15571559 * have at least one idle worker on return from this function. If
15581560 * creating a new worker takes longer than MAYDAY_INTERVAL, mayday is
1559− * sent to all rescuers with works scheduled on @gcwq to resolve
1561+ * sent to all rescuers with works scheduled on @pool to resolve
15601562 * possible allocation deadlock.
15611563 *
15621564 * On return, need_to_create_worker() is guaranteed to be false and
@@ -1571,52 +1573,54 @@ static void gcwq_mayday_timeout(unsigned long __gcwq) · 8 unmodified lines
15711573 * false if no action was taken and gcwq->lock stayed locked, true
15721574 * otherwise.
15731575 */
1574−static bool maybe_create_worker(struct global_cwq *gcwq)
1576+static bool maybe_create_worker(struct worker_pool *pool)
15751577 __releases(&gcwq->lock)
15761578 __acquires(&gcwq->lock)
15771579 {
1578− if (!need_to_create_worker(gcwq))
1580+ struct global_cwq *gcwq = pool->gcwq;
1581+
1582+ if (!need_to_create_worker(pool))
15791583 return false;
15801584 restart:
15811585 spin_unlock_irq(&gcwq->lock);
15821586
15831587 /* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */
1584− mod_timer(&gcwq->pool.mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
1588+ mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
15851589
15861590 while (true) {
15871591 struct worker *worker;
15881592
1589− worker = create_worker(gcwq, true);
1593+ worker = create_worker(pool, true);
15901594 if (worker) {
1591− del_timer_sync(&gcwq->pool.mayday_timer);
1595+ del_timer_sync(&pool->mayday_timer);
15921596 spin_lock_irq(&gcwq->lock);
15931597 start_worker(worker);
1594− BUG_ON(need_to_create_worker(gcwq));
1598+ BUG_ON(need_to_create_worker(pool));
15951599 return true;
15961600 }
15971601
1598− if (!need_to_create_worker(gcwq))
1602+ if (!need_to_create_worker(pool))
15991603 break;
16001604
16011605 __set_current_state(TASK_INTERRUPTIBLE);
16021606 schedule_timeout(CREATE_COOLDOWN);
16031607
1604− if (!need_to_create_worker(gcwq))
1608+ if (!need_to_create_worker(pool))
16051609 break;
16061610 }
16071611
1608− del_timer_sync(&gcwq->pool.mayday_timer);
1612+ del_timer_sync(&pool->mayday_timer);
16091613 spin_lock_irq(&gcwq->lock);
1610− if (need_to_create_worker(gcwq))
1614+ if (need_to_create_worker(pool))
16111615 goto restart;
16121616 return true;
16131617 }
16141618
16151619 /**
16161620 * maybe_destroy_worker - destroy workers which have been idle for a while
1617− * @gcwq: gcwq to destroy workers for
1621+ * @pool: pool to destroy workers for
16181622 *
1619− * Destroy @gcwq workers which have been idle for longer than
1623+ * Destroy @pool workers which have been idle for longer than
16201624 * IDLE_WORKER_TIMEOUT.
16211625 *
16221626 * LOCKING:
@@ -1627,20 +1631,19 @@ restart: · 4 unmodified lines
16271631 * false if no action was taken and gcwq->lock stayed locked, true
16281632 * otherwise.
16291633 */
1630−static bool maybe_destroy_workers(struct global_cwq *gcwq)
1634+static bool maybe_destroy_workers(struct worker_pool *pool)
16311635 {
16321636 bool ret = false;
16331637
1634− while (too_many_workers(gcwq)) {
1638+ while (too_many_workers(pool)) {
16351639 struct worker *worker;
16361640 unsigned long expires;
16371641
1638− worker = list_entry(gcwq->pool.idle_list.prev, struct worker,
1639− entry);
1642+ worker = list_entry(pool->idle_list.prev, struct worker, entry);
16401643 expires = worker->last_active + IDLE_WORKER_TIMEOUT;
16411644
16421645 if (time_before(jiffies, expires)) {
1643− mod_timer(&gcwq->pool.idle_timer, expires);
1646+ mod_timer(&pool->idle_timer, expires);
16441647 break;
16451648 }
16461649
@@ -1673,7 +1676,8 @@ static bool maybe_destroy_workers(struct global_cwq *gcwq) · 26 unmodified lines
16731676 */
16741677 static bool manage_workers(struct worker *worker)
16751678 {
1676− struct global_cwq *gcwq = worker->pool->gcwq;
1679+ struct worker_pool *pool = worker->pool;
1680+ struct global_cwq *gcwq = pool->gcwq;
16771681 bool ret = false;
16781682
16791683 if (gcwq->flags & GCWQ_MANAGING_WORKERS)
@@ -1686,8 +1690,8 @@ static bool manage_workers(struct worker *worker) · 6 unmodified lines
16861690 * Destroy and then create so that may_start_working() is true
16871691 * on return.
16881692 */
1689− ret |= maybe_destroy_workers(gcwq);
1690− ret |= maybe_create_worker(gcwq);
1693+ ret |= maybe_destroy_workers(pool);
1694+ ret |= maybe_create_worker(pool);
16911695
16921696 gcwq->flags &= ~GCWQ_MANAGING_WORKERS;
16931697
@@ -1746,7 +1750,7 @@ static void cwq_activate_first_delayed(struct cpu_workqueue_struct *cwq) · 52 unmodified lines
17461750 {
17471751 struct work_struct *work = list_first_entry(&cwq->delayed_works,
17481752 struct work_struct, entry);
1749− struct list_head *pos = gcwq_determine_ins_pos(cwq->pool->gcwq, cwq);
1753+ struct list_head *pos = pool_determine_ins_pos(cwq->pool, cwq);
17501754
17511755 trace_workqueue_activate_work(work);
17521756 move_linked_works(work, pos, NULL);
@@ -1874,7 +1878,7 @@ __acquires(&gcwq->lock) · 121 unmodified lines
18741878
18751879 if (!list_empty(&pool->worklist) &&
18761880 get_work_cwq(nwork)->wq->flags & WQ_HIGHPRI)
1877− wake_up_worker(gcwq);
1881+ wake_up_worker(pool);
18781882 else
18791883 gcwq->flags &= ~GCWQ_HIGHPRI_PENDING;
18801884 }
@@ -1890,8 +1894,8 @@ __acquires(&gcwq->lock) · 9 unmodified lines
18901894 * Unbound gcwq isn't concurrency managed and work items should be
18911895 * executed ASAP. Wake up another worker if necessary.
18921896 */
1893− if ((worker->flags & WORKER_UNBOUND) && need_more_worker(gcwq))
1894− wake_up_worker(gcwq);
1897+ if ((worker->flags & WORKER_UNBOUND) && need_more_worker(pool))
1898+ wake_up_worker(pool);
18951899
18961900 spin_unlock_irq(&gcwq->lock);
18971901
@@ -1983,11 +1987,11 @@ woke_up: · 85 unmodified lines
19831987 worker_leave_idle(worker);
19841988 recheck:
19851989 /* no more worker necessary? */
1986− if (!need_more_worker(gcwq))
1990+ if (!need_more_worker(pool))
19871991 goto sleep;
19881992
19891993 /* do we need to manage? */
1990− if (unlikely(!may_start_working(gcwq)) && manage_workers(worker))
1994+ if (unlikely(!may_start_working(pool)) && manage_workers(worker))
19911995 goto recheck;
19921996
19931997 /*
@@ -2018,11 +2022,11 @@ recheck: · 24 unmodified lines
20182022 move_linked_works(work, &worker->scheduled, NULL);
20192023 process_scheduled_works(worker);
20202024 }
2021− } while (keep_working(gcwq));
2025+ } while (keep_working(pool));
20222026
20232027 worker_set_flags(worker, WORKER_PREP, false);
20242028 sleep:
2025− if (unlikely(need_to_manage_workers(gcwq)) && manage_workers(worker))
2029+ if (unlikely(need_to_manage_workers(pool)) && manage_workers(worker))
20262030 goto recheck;
20272031
20282032 /*
@@ -2107,8 +2111,8 @@ repeat: · 78 unmodified lines
21072111 * regular worker; otherwise, we end up with 0 concurrency
21082112 * and stalling the execution.
21092113 */
2110− if (keep_working(gcwq))
2111− wake_up_worker(gcwq);
2114+ if (keep_working(pool))
2115+ wake_up_worker(pool);
21122116
21132117 spin_unlock_irq(&gcwq->lock);
21142118 }
@@ -3383,7 +3387,7 @@ static int __cpuinit trustee_thread(void *__gcwq) · 1268 unmodified lines
33833387 * keep_working() are always true as long as the worklist is
33843388 * not empty.
33853389 */
3386− atomic_set(get_gcwq_nr_running(gcwq->cpu), 0);
3390+ atomic_set(get_pool_nr_running(&gcwq->pool), 0);
33873391
33883392 spin_unlock_irq(&gcwq->lock);
33893393 del_timer_sync(&gcwq->pool.idle_timer);
@@ -3424,9 +3428,9 @@ static int __cpuinit trustee_thread(void *__gcwq) · 34 unmodified lines
34243428 wake_up_process(worker->task);
34253429 }
34263430
3427− if (need_to_create_worker(gcwq)) {
3431+ if (need_to_create_worker(&gcwq->pool)) {
34283432 spin_unlock_irq(&gcwq->lock);
3429− worker = create_worker(gcwq, false);
3433+ worker = create_worker(&gcwq->pool, false);
34303434 spin_lock_irq(&gcwq->lock);
34313435 if (worker) {
34323436 worker->flags |= WORKER_ROGUE;
@@ -3540,7 +3544,7 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, · 107 unmodified lines
35403544 /* fall through */
35413545 case CPU_UP_PREPARE:
35423546 BUG_ON(gcwq->pool.first_idle);
3543− new_worker = create_worker(gcwq, false);
3547+ new_worker = create_worker(&gcwq->pool, false);
35443548 if (!new_worker) {
35453549 if (new_trustee)
35463550 kthread_stop(new_trustee);
@@ -3788,7 +3792,7 @@ void thaw_workqueues(void) · 241 unmodified lines
37883792 cwq_activate_first_delayed(cwq);
37893793 }
37903794
3791− wake_up_worker(gcwq);
3795+ wake_up_worker(&gcwq->pool);
37923796
37933797 spin_unlock_irq(&gcwq->lock);
37943798 }
@@ -3822,10 +3826,10 @@ static int __init init_workqueues(void) · 27 unmodified lines
38223826
38233827 init_timer_deferrable(&gcwq->pool.idle_timer);
38243828 gcwq->pool.idle_timer.function = idle_worker_timeout;
3825− gcwq->pool.idle_timer.data = (unsigned long)gcwq;
3829+ gcwq->pool.idle_timer.data = (unsigned long)&gcwq->pool;
38263830
38273831 setup_timer(&gcwq->pool.mayday_timer, gcwq_mayday_timeout,
3828− (unsigned long)gcwq);
3832+ (unsigned long)&gcwq->pool);
38293833
38303834 ida_init(&gcwq->pool.worker_ida);
38313835
@@ -3840,7 +3844,7 @@ static int __init init_workqueues(void) · 8 unmodified lines
38403844
38413845 if (cpu != WORK_CPU_UNBOUND)
38423846 gcwq->flags &= ~GCWQ_DISASSOCIATED;
3843− worker = create_worker(gcwq, true);
3847+ worker = create_worker(&gcwq->pool, true);
38443848 BUG_ON(!worker);
38453849 spin_lock_irq(&gcwq->lock);
38463850 start_worker(worker);