History

workqueue: factor out worker_pool from global_cwq

Move worklist and all worker management fields from global_cwq into the new struct worker_pool. worker_pool points back to the containing gcwq. worker and cpu_workqueue_struct are updated to point to worker_pool instead of gcwq too. This change is mechanical and doesn't introduce any functional difference other than rearranging of fields and an added level of indirection in some places. This is to prepare for multiple pools per gcwq. v2: Comment typo fixes as suggested by Namhyung.

Tejun Heo committed 14 years ago

Commit bd7bdd41 parent 974271c2 files · +118 −100Browse files

Unified · Split

2 files
  1. include/trace/events/workqueue.h +1 −1
  2. kernel/workqueue.c +117 −99
include/trace/events/workqueue.h+1−1
@@ -54,7 +54,7 @@ TRACE_EVENT(workqueue_queue_work, · 53 unmodified lines
5454 __entry->function = work->func;
5555 __entry->workqueue = cwq->wq;
5656 __entry->req_cpu = req_cpu;
57− __entry->cpu = cwq->gcwq->cpu;
57+ __entry->cpu = cwq->pool->gcwq->cpu;
5858 ),
5959
6060 TP_printk("work struct=%p function=%pf workqueue=%p req_cpu=%u cpu=%u",
kernel/workqueue.c+117−99
@@ -115,6 +115,7 @@ enum { · 114 unmodified lines
115115 */
116116
117117 struct global_cwq;
118+struct worker_pool;
118119
119120 /*
120121 * The poor guys doing the actual heavy lifting. All on-duty workers
@@ -131,7 +132,7 @@ struct worker { · 10 unmodified lines
131132 struct cpu_workqueue_struct *current_cwq; /* L: current_work's cwq */
132133 struct list_head scheduled; /* L: scheduled works */
133134 struct task_struct *task; /* I: worker task */
134− struct global_cwq *gcwq; /* I: the associated gcwq */
135+ struct worker_pool *pool; /* I: the associated pool */
135136 /* 64 bytes boundary on 64bit, 32 on 32bit */
136137 unsigned long last_active; /* L: last active timestamp */
137138 unsigned int flags; /* X: flags */
@@ -139,6 +140,21 @@ struct worker { · 1 unmodified line
139140 struct work_struct rebind_work; /* L: rebind worker to cpu */
140141 };
141142
143+struct worker_pool {
144+ struct global_cwq *gcwq; /* I: the owning gcwq */
145+
146+ struct list_head worklist; /* L: list of pending works */
147+ int nr_workers; /* L: total number of workers */
148+ int nr_idle; /* L: currently idle ones */
149+
150+ struct list_head idle_list; /* X: list of idle workers */
151+ struct timer_list idle_timer; /* L: worker idle timeout */
152+ struct timer_list mayday_timer; /* L: SOS timer for workers */
153+
154+ struct ida worker_ida; /* L: for worker IDs */
155+ struct worker *first_idle; /* L: first idle worker */
156+};
157+
142158 /*
143159 * Global per-cpu workqueue. There's one and only one for each cpu
144160 * and all works are queued and processed here regardless of their
@@ -146,27 +162,18 @@ struct worker { · 1 unmodified line
146162 */
147163 struct global_cwq {
148164 spinlock_t lock; /* the gcwq lock */
149− struct list_head worklist; /* L: list of pending works */
150165 unsigned int cpu; /* I: the associated cpu */
151166 unsigned int flags; /* L: GCWQ_* flags */
152167
153− int nr_workers; /* L: total number of workers */
154− int nr_idle; /* L: currently idle ones */
155−
156− /* workers are chained either in the idle_list or busy_hash */
157− struct list_head idle_list; /* X: list of idle workers */
168+ /* workers are chained either in busy_hash or pool idle_list */
158169 struct hlist_head busy_hash[BUSY_WORKER_HASH_SIZE];
159170 /* L: hash of busy workers */
160171
161− struct timer_list idle_timer; /* L: worker idle timeout */
162− struct timer_list mayday_timer; /* L: SOS timer for dworkers */
163−
164− struct ida worker_ida; /* L: for worker IDs */
172+ struct worker_pool pool; /* the worker pools */
165173
166174 struct task_struct *trustee; /* L: for gcwq shutdown */
167175 unsigned int trustee_state; /* L: trustee state */
168176 wait_queue_head_t trustee_wait; /* trustee wait */
169− struct worker *first_idle; /* L: first idle worker */
170177 } ____cacheline_aligned_in_smp;
171178
172179 /*
@@ -175,7 +182,7 @@ struct global_cwq { · 2 unmodified lines
175182 * aligned at two's power of the number of flag bits.
176183 */
177184 struct cpu_workqueue_struct {
178− struct global_cwq *gcwq; /* I: the associated gcwq */
185+ struct worker_pool *pool; /* I: the associated pool */
179186 struct workqueue_struct *wq; /* I: the owning workqueue */
180187 int work_color; /* L: current color */
181188 int flush_color; /* L: flushing color */
@@ -555,7 +562,7 @@ static struct global_cwq *get_work_gcwq(struct work_struct *work) · 373 unmodified lines
555562
556563 if (data & WORK_STRUCT_CWQ)
557564 return ((struct cpu_workqueue_struct *)
558− (data & WORK_STRUCT_WQ_DATA_MASK))->gcwq;
565+ (data & WORK_STRUCT_WQ_DATA_MASK))->pool->gcwq;
559566
560567 cpu = data >> WORK_STRUCT_FLAG_BITS;
561568 if (cpu == WORK_CPU_NONE)
@@ -587,13 +594,13 @@ static bool __need_more_worker(struct global_cwq *gcwq) · 25 unmodified lines
587594 */
588595 static bool need_more_worker(struct global_cwq *gcwq)
589596 {
590− return !list_empty(&gcwq->worklist) && __need_more_worker(gcwq);
597+ return !list_empty(&gcwq->pool.worklist) && __need_more_worker(gcwq);
591598 }
592599
593600 /* Can I start working? Called from busy but !running workers. */
594601 static bool may_start_working(struct global_cwq *gcwq)
595602 {
596− return gcwq->nr_idle;
603+ return gcwq->pool.nr_idle;
597604 }
598605
599606 /* Do I need to keep working? Called from currently running workers. */
@@ -601,7 +608,7 @@ static bool keep_working(struct global_cwq *gcwq) · 1 unmodified line
601608 {
602609 atomic_t *nr_running = get_gcwq_nr_running(gcwq->cpu);
603610
604− return !list_empty(&gcwq->worklist) &&
611+ return !list_empty(&gcwq->pool.worklist) &&
605612 (atomic_read(nr_running) <= 1 ||
606613 gcwq->flags & GCWQ_HIGHPRI_PENDING);
607614 }
@@ -622,8 +629,8 @@ static bool need_to_manage_workers(struct global_cwq *gcwq) · 14 unmodified lines
622629 static bool too_many_workers(struct global_cwq *gcwq)
623630 {
624631 bool managing = gcwq->flags & GCWQ_MANAGING_WORKERS;
625− int nr_idle = gcwq->nr_idle + managing; /* manager is considered idle */
626− int nr_busy = gcwq->nr_workers - nr_idle;
632+ int nr_idle = gcwq->pool.nr_idle + managing; /* manager is considered idle */
633+ int nr_busy = gcwq->pool.nr_workers - nr_idle;
627634
628635 return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;
629636 }
@@ -635,10 +642,10 @@ static bool too_many_workers(struct global_cwq *gcwq) · 5 unmodified lines
635642 /* Return the first worker. Safe with preemption disabled */
636643 static struct worker *first_worker(struct global_cwq *gcwq)
637644 {
638− if (unlikely(list_empty(&gcwq->idle_list)))
645+ if (unlikely(list_empty(&gcwq->pool.idle_list)))
639646 return NULL;
640647
641− return list_first_entry(&gcwq->idle_list, struct worker, entry);
648+ return list_first_entry(&gcwq->pool.idle_list, struct worker, entry);
642649 }
643650
644651 /**
@@ -696,7 +703,8 @@ struct task_struct *wq_worker_sleeping(struct task_struct *task, · 51 unmodified lines
696703 unsigned int cpu)
697704 {
698705 struct worker *worker = kthread_data(task), *to_wakeup = NULL;
699− struct global_cwq *gcwq = get_gcwq(cpu);
706+ struct worker_pool *pool = worker->pool;
707+ struct global_cwq *gcwq = pool->gcwq;
700708 atomic_t *nr_running = get_gcwq_nr_running(cpu);
701709
702710 if (worker->flags & WORKER_NOT_RUNNING)
@@ -716,7 +724,7 @@ struct task_struct *wq_worker_sleeping(struct task_struct *task, · 13 unmodified lines
716724 * could be manipulating idle_list, so dereferencing idle_list
717725 * without gcwq lock is safe.
718726 */
719− if (atomic_dec_and_test(nr_running) && !list_empty(&gcwq->worklist))
727+ if (atomic_dec_and_test(nr_running) && !list_empty(&pool->worklist))
720728 to_wakeup = first_worker(gcwq);
721729 return to_wakeup ? to_wakeup->task : NULL;
722730 }
@@ -737,7 +745,8 @@ struct task_struct *wq_worker_sleeping(struct task_struct *task, · 14 unmodified lines
737745 static inline void worker_set_flags(struct worker *worker, unsigned int flags,
738746 bool wakeup)
739747 {
740− struct global_cwq *gcwq = worker->gcwq;
748+ struct worker_pool *pool = worker->pool;
749+ struct global_cwq *gcwq = pool->gcwq;
741750
742751 WARN_ON_ONCE(worker->task != current);
743752
@@ -752,7 +761,7 @@ static inline void worker_set_flags(struct worker *worker, unsigned int flags, · 8 unmodified lines
752761
753762 if (wakeup) {
754763 if (atomic_dec_and_test(nr_running) &&
755− !list_empty(&gcwq->worklist))
764+ !list_empty(&pool->worklist))
756765 wake_up_worker(gcwq);
757766 } else
758767 atomic_dec(nr_running);
@@ -773,7 +782,7 @@ static inline void worker_set_flags(struct worker *worker, unsigned int flags, · 14 unmodified lines
773782 */
774783 static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
775784 {
776− struct global_cwq *gcwq = worker->gcwq;
785+ struct global_cwq *gcwq = worker->pool->gcwq;
777786 unsigned int oflags = worker->flags;
778787
779788 WARN_ON_ONCE(worker->task != current);
@@ -894,9 +903,9 @@ static inline struct list_head *gcwq_determine_ins_pos(struct global_cwq *gcwq, · 114 unmodified lines
894903 struct work_struct *twork;
895904
896905 if (likely(!(cwq->wq->flags & WQ_HIGHPRI)))
897− return &gcwq->worklist;
906+ return &gcwq->pool.worklist;
898907
899− list_for_each_entry(twork, &gcwq->worklist, entry) {
908+ list_for_each_entry(twork, &gcwq->pool.worklist, entry) {
900909 struct cpu_workqueue_struct *tcwq = get_work_cwq(twork);
901910
902911 if (!(tcwq->wq->flags & WQ_HIGHPRI))
@@ -924,7 +933,7 @@ static void insert_work(struct cpu_workqueue_struct *cwq, · 21 unmodified lines
924933 struct work_struct *work, struct list_head *head,
925934 unsigned int extra_flags)
926935 {
927− struct global_cwq *gcwq = cwq->gcwq;
936+ struct global_cwq *gcwq = cwq->pool->gcwq;
928937
929938 /* we own @work, set data and link */
930939 set_work_cwq(work, cwq, extra_flags);
@@ -1196,7 +1205,8 @@ EXPORT_SYMBOL_GPL(queue_delayed_work_on); · 265 unmodified lines
11961205 */
11971206 static void worker_enter_idle(struct worker *worker)
11981207 {
1199− struct global_cwq *gcwq = worker->gcwq;
1208+ struct worker_pool *pool = worker->pool;
1209+ struct global_cwq *gcwq = pool->gcwq;
12001210
12011211 BUG_ON(worker->flags & WORKER_IDLE);
12021212 BUG_ON(!list_empty(&worker->entry) &&
@@ -1204,15 +1214,15 @@ static void worker_enter_idle(struct worker *worker) · 1 unmodified line
12041214
12051215 /* can't use worker_set_flags(), also called from start_worker() */
12061216 worker->flags |= WORKER_IDLE;
1207− gcwq->nr_idle++;
1217+ pool->nr_idle++;
12081218 worker->last_active = jiffies;
12091219
12101220 /* idle_list is LIFO */
1211− list_add(&worker->entry, &gcwq->idle_list);
1221+ list_add(&worker->entry, &pool->idle_list);
12121222
12131223 if (likely(!(worker->flags & WORKER_ROGUE))) {
1214− if (too_many_workers(gcwq) && !timer_pending(&gcwq->idle_timer))
1215− mod_timer(&gcwq->idle_timer,
1224+ if (too_many_workers(gcwq) && !timer_pending(&pool->idle_timer))
1225+ mod_timer(&pool->idle_timer,
12161226 jiffies + IDLE_WORKER_TIMEOUT);
12171227 } else
12181228 wake_up_all(&gcwq->trustee_wait);
@@ -1223,7 +1233,7 @@ static void worker_enter_idle(struct worker *worker) · 4 unmodified lines
12231233 * warning may trigger spuriously. Check iff trustee is idle.
12241234 */
12251235 WARN_ON_ONCE(gcwq->trustee_state == TRUSTEE_DONE &&
1226− gcwq->nr_workers == gcwq->nr_idle &&
1236+ pool->nr_workers == pool->nr_idle &&
12271237 atomic_read(get_gcwq_nr_running(gcwq->cpu)));
12281238 }
12291239
@@ -1238,11 +1248,11 @@ static void worker_enter_idle(struct worker *worker) · 8 unmodified lines
12381248 */
12391249 static void worker_leave_idle(struct worker *worker)
12401250 {
1241− struct global_cwq *gcwq = worker->gcwq;
1251+ struct worker_pool *pool = worker->pool;
12421252
12431253 BUG_ON(!(worker->flags & WORKER_IDLE));
12441254 worker_clr_flags(worker, WORKER_IDLE);
1245− gcwq->nr_idle--;
1255+ pool->nr_idle--;
12461256 list_del_init(&worker->entry);
12471257 }
12481258
@@ -1279,7 +1289,7 @@ static void worker_leave_idle(struct worker *worker) · 30 unmodified lines
12791289 static bool worker_maybe_bind_and_lock(struct worker *worker)
12801290 __acquires(&gcwq->lock)
12811291 {
1282− struct global_cwq *gcwq = worker->gcwq;
1292+ struct global_cwq *gcwq = worker->pool->gcwq;
12831293 struct task_struct *task = worker->task;
12841294
12851295 while (true) {
@@ -1321,7 +1331,7 @@ __acquires(&gcwq->lock) · 35 unmodified lines
13211331 static void worker_rebind_fn(struct work_struct *work)
13221332 {
13231333 struct worker *worker = container_of(work, struct worker, rebind_work);
1324− struct global_cwq *gcwq = worker->gcwq;
1334+ struct global_cwq *gcwq = worker->pool->gcwq;
13251335
13261336 if (worker_maybe_bind_and_lock(worker))
13271337 worker_clr_flags(worker, WORKER_REBIND);
@@ -1362,13 +1372,14 @@ static struct worker *alloc_worker(void) · 34 unmodified lines
13621372 static struct worker *create_worker(struct global_cwq *gcwq, bool bind)
13631373 {
13641374 bool on_unbound_cpu = gcwq->cpu == WORK_CPU_UNBOUND;
1375+ struct worker_pool *pool = &gcwq->pool;
13651376 struct worker *worker = NULL;
13661377 int id = -1;
13671378
13681379 spin_lock_irq(&gcwq->lock);
1369− while (ida_get_new(&gcwq->worker_ida, &id)) {
1380+ while (ida_get_new(&pool->worker_ida, &id)) {
13701381 spin_unlock_irq(&gcwq->lock);
1371− if (!ida_pre_get(&gcwq->worker_ida, GFP_KERNEL))
1382+ if (!ida_pre_get(&pool->worker_ida, GFP_KERNEL))
13721383 goto fail;
13731384 spin_lock_irq(&gcwq->lock);
13741385 }
@@ -1378,7 +1389,7 @@ static struct worker *create_worker(struct global_cwq *gcwq, bool bind) · 3 unmodified lines
13781389 if (!worker)
13791390 goto fail;
13801391
1381− worker->gcwq = gcwq;
1392+ worker->pool = pool;
13821393 worker->id = id;
13831394
13841395 if (!on_unbound_cpu)
@@ -1409,7 +1420,7 @@ static struct worker *create_worker(struct global_cwq *gcwq, bool bind) · 24 unmodified lines
14091420 fail:
14101421 if (id >= 0) {
14111422 spin_lock_irq(&gcwq->lock);
1412− ida_remove(&gcwq->worker_ida, id);
1423+ ida_remove(&pool->worker_ida, id);
14131424 spin_unlock_irq(&gcwq->lock);
14141425 }
14151426 kfree(worker);
@@ -1428,7 +1439,7 @@ fail: · 12 unmodified lines
14281439 static void start_worker(struct worker *worker)
14291440 {
14301441 worker->flags |= WORKER_STARTED;
1431− worker->gcwq->nr_workers++;
1442+ worker->pool->nr_workers++;
14321443 worker_enter_idle(worker);
14331444 wake_up_process(worker->task);
14341445 }
@@ -1444,7 +1455,8 @@ static void start_worker(struct worker *worker) · 9 unmodified lines
14441455 */
14451456 static void destroy_worker(struct worker *worker)
14461457 {
1447− struct global_cwq *gcwq = worker->gcwq;
1458+ struct worker_pool *pool = worker->pool;
1459+ struct global_cwq *gcwq = pool->gcwq;
14481460 int id = worker->id;
14491461
14501462 /* sanity check frenzy */
@@ -1452,9 +1464,9 @@ static void destroy_worker(struct worker *worker) · 1 unmodified line
14521464 BUG_ON(!list_empty(&worker->scheduled));
14531465
14541466 if (worker->flags & WORKER_STARTED)
1455− gcwq->nr_workers--;
1467+ pool->nr_workers--;
14561468 if (worker->flags & WORKER_IDLE)
1457− gcwq->nr_idle--;
1469+ pool->nr_idle--;
14581470
14591471 list_del_init(&worker->entry);
14601472 worker->flags |= WORKER_DIE;
@@ -1465,7 +1477,7 @@ static void destroy_worker(struct worker *worker) · 4 unmodified lines
14651477 kfree(worker);
14661478
14671479 spin_lock_irq(&gcwq->lock);
1468− ida_remove(&gcwq->worker_ida, id);
1480+ ida_remove(&pool->worker_ida, id);
14691481 }
14701482
14711483 static void idle_worker_timeout(unsigned long __gcwq)
@@ -1479,11 +1491,12 @@ static void idle_worker_timeout(unsigned long __gcwq) · 7 unmodified lines
14791491 unsigned long expires;
14801492
14811493 /* idle_list is kept in LIFO order, check the last one */
1482− worker = list_entry(gcwq->idle_list.prev, struct worker, entry);
1494+ worker = list_entry(gcwq->pool.idle_list.prev, struct worker,
1495+ entry);
14831496 expires = worker->last_active + IDLE_WORKER_TIMEOUT;
14841497
14851498 if (time_before(jiffies, expires))
1486− mod_timer(&gcwq->idle_timer, expires);
1499+ mod_timer(&gcwq->pool.idle_timer, expires);
14871500 else {
14881501 /* it's been idle for too long, wake up manager */
14891502 gcwq->flags |= GCWQ_MANAGE_WORKERS;
@@ -1504,7 +1517,7 @@ static bool send_mayday(struct work_struct *work) · 14 unmodified lines
15041517 return false;
15051518
15061519 /* mayday mayday mayday */
1507− cpu = cwq->gcwq->cpu;
1520+ cpu = cwq->pool->gcwq->cpu;
15081521 /* WORK_CPU_UNBOUND can't be set in cpumask, use cpu 0 instead */
15091522 if (cpu == WORK_CPU_UNBOUND)
15101523 cpu = 0;
@@ -1527,13 +1540,13 @@ static void gcwq_mayday_timeout(unsigned long __gcwq) · 16 unmodified lines
15271540 * allocation deadlock. Send distress signals to
15281541 * rescuers.
15291542 */
1530− list_for_each_entry(work, &gcwq->worklist, entry)
1543+ list_for_each_entry(work, &gcwq->pool.worklist, entry)
15311544 send_mayday(work);
15321545 }
15331546
15341547 spin_unlock_irq(&gcwq->lock);
15351548
1536− mod_timer(&gcwq->mayday_timer, jiffies + MAYDAY_INTERVAL);
1549+ mod_timer(&gcwq->pool.mayday_timer, jiffies + MAYDAY_INTERVAL);
15371550 }
15381551
15391552 /**
@@ -1568,14 +1581,14 @@ restart: · 28 unmodified lines
15681581 spin_unlock_irq(&gcwq->lock);
15691582
15701583 /* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */
1571− mod_timer(&gcwq->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
1584+ mod_timer(&gcwq->pool.mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
15721585
15731586 while (true) {
15741587 struct worker *worker;
15751588
15761589 worker = create_worker(gcwq, true);
15771590 if (worker) {
1578− del_timer_sync(&gcwq->mayday_timer);
1591+ del_timer_sync(&gcwq->pool.mayday_timer);
15791592 spin_lock_irq(&gcwq->lock);
15801593 start_worker(worker);
15811594 BUG_ON(need_to_create_worker(gcwq));
@@ -1592,7 +1605,7 @@ restart: · 10 unmodified lines
15921605 break;
15931606 }
15941607
1595− del_timer_sync(&gcwq->mayday_timer);
1608+ del_timer_sync(&gcwq->pool.mayday_timer);
15961609 spin_lock_irq(&gcwq->lock);
15971610 if (need_to_create_worker(gcwq))
15981611 goto restart;
@@ -1622,11 +1635,12 @@ static bool maybe_destroy_workers(struct global_cwq *gcwq) · 23 unmodified lines
16221635 struct worker *worker;
16231636 unsigned long expires;
16241637
1625− worker = list_entry(gcwq->idle_list.prev, struct worker, entry);
1638+ worker = list_entry(gcwq->pool.idle_list.prev, struct worker,
1639+ entry);
16261640 expires = worker->last_active + IDLE_WORKER_TIMEOUT;
16271641
16281642 if (time_before(jiffies, expires)) {
1629− mod_timer(&gcwq->idle_timer, expires);
1643+ mod_timer(&gcwq->pool.idle_timer, expires);
16301644 break;
16311645 }
16321646
@@ -1659,7 +1673,7 @@ static bool maybe_destroy_workers(struct global_cwq *gcwq) · 26 unmodified lines
16591673 */
16601674 static bool manage_workers(struct worker *worker)
16611675 {
1662− struct global_cwq *gcwq = worker->gcwq;
1676+ struct global_cwq *gcwq = worker->pool->gcwq;
16631677 bool ret = false;
16641678
16651679 if (gcwq->flags & GCWQ_MANAGING_WORKERS)
@@ -1732,7 +1746,7 @@ static void cwq_activate_first_delayed(struct cpu_workqueue_struct *cwq) · 66 unmodified lines
17321746 {
17331747 struct work_struct *work = list_first_entry(&cwq->delayed_works,
17341748 struct work_struct, entry);
1735− struct list_head *pos = gcwq_determine_ins_pos(cwq->gcwq, cwq);
1749+ struct list_head *pos = gcwq_determine_ins_pos(cwq->pool->gcwq, cwq);
17361750
17371751 trace_workqueue_activate_work(work);
17381752 move_linked_works(work, pos, NULL);
@@ -1808,7 +1822,8 @@ __releases(&gcwq->lock) · 69 unmodified lines
18081822 __acquires(&gcwq->lock)
18091823 {
18101824 struct cpu_workqueue_struct *cwq = get_work_cwq(work);
1811− struct global_cwq *gcwq = cwq->gcwq;
1825+ struct worker_pool *pool = worker->pool;
1826+ struct global_cwq *gcwq = pool->gcwq;
18121827 struct hlist_head *bwh = busy_worker_head(gcwq, work);
18131828 bool cpu_intensive = cwq->wq->flags & WQ_CPU_INTENSIVE;
18141829 work_func_t f = work->func;
@@ -1854,10 +1869,10 @@ __acquires(&gcwq->lock) · 39 unmodified lines
18541869 * wake up another worker; otherwise, clear HIGHPRI_PENDING.
18551870 */
18561871 if (unlikely(gcwq->flags & GCWQ_HIGHPRI_PENDING)) {
1857− struct work_struct *nwork = list_first_entry(&gcwq->worklist,
1858− struct work_struct, entry);
1872+ struct work_struct *nwork = list_first_entry(&pool->worklist,
1873+ struct work_struct, entry);
18591874
1860− if (!list_empty(&gcwq->worklist) &&
1875+ if (!list_empty(&pool->worklist) &&
18611876 get_work_cwq(nwork)->wq->flags & WQ_HIGHPRI)
18621877 wake_up_worker(gcwq);
18631878 else
@@ -1950,7 +1965,8 @@ static void process_scheduled_works(struct worker *worker) · 86 unmodified lines
19501965 static int worker_thread(void *__worker)
19511966 {
19521967 struct worker *worker = __worker;
1953− struct global_cwq *gcwq = worker->gcwq;
1968+ struct worker_pool *pool = worker->pool;
1969+ struct global_cwq *gcwq = pool->gcwq;
19541970
19551971 /* tell the scheduler that this is a workqueue worker */
19561972 worker->task->flags |= PF_WQ_WORKER;
@@ -1990,7 +2006,7 @@ recheck: · 33 unmodified lines
19902006
19912007 do {
19922008 struct work_struct *work =
1993− list_first_entry(&gcwq->worklist,
2009+ list_first_entry(&pool->worklist,
19942010 struct work_struct, entry);
19952011
19962012 if (likely(!(*work_data_bits(work) & WORK_STRUCT_LINKED))) {
@@ -2064,14 +2080,15 @@ repeat: · 67 unmodified lines
20642080 for_each_mayday_cpu(cpu, wq->mayday_mask) {
20652081 unsigned int tcpu = is_unbound ? WORK_CPU_UNBOUND : cpu;
20662082 struct cpu_workqueue_struct *cwq = get_cwq(tcpu, wq);
2067− struct global_cwq *gcwq = cwq->gcwq;
2083+ struct worker_pool *pool = cwq->pool;
2084+ struct global_cwq *gcwq = pool->gcwq;
20682085 struct work_struct *work, *n;
20692086
20702087 __set_current_state(TASK_RUNNING);
20712088 mayday_clear_cpu(cpu, wq->mayday_mask);
20722089
20732090 /* migrate to the target cpu if possible */
2074− rescuer->gcwq = gcwq;
2091+ rescuer->pool = pool;
20752092 worker_maybe_bind_and_lock(rescuer);
20762093
20772094 /*
@@ -2079,7 +2096,7 @@ repeat: · 1 unmodified line
20792096 * process'em.
20802097 */
20812098 BUG_ON(!list_empty(&rescuer->scheduled));
2082− list_for_each_entry_safe(work, n, &gcwq->worklist, entry)
2099+ list_for_each_entry_safe(work, n, &pool->worklist, entry)
20832100 if (get_work_cwq(work) == cwq)
20842101 move_linked_works(work, scheduled, &n);
20852102
@@ -2216,7 +2233,7 @@ static bool flush_workqueue_prep_cwqs(struct workqueue_struct *wq, · 130 unmodified lines
22162233
22172234 for_each_cwq_cpu(cpu, wq) {
22182235 struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
2219− struct global_cwq *gcwq = cwq->gcwq;
2236+ struct global_cwq *gcwq = cwq->pool->gcwq;
22202237
22212238 spin_lock_irq(&gcwq->lock);
22222239
@@ -2432,9 +2449,9 @@ reflush: · 209 unmodified lines
24322449 struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
24332450 bool drained;
24342451
2435− spin_lock_irq(&cwq->gcwq->lock);
2452+ spin_lock_irq(&cwq->pool->gcwq->lock);
24362453 drained = !cwq->nr_active && list_empty(&cwq->delayed_works);
2437− spin_unlock_irq(&cwq->gcwq->lock);
2454+ spin_unlock_irq(&cwq->pool->gcwq->lock);
24382455
24392456 if (drained)
24402457 continue;
@@ -2474,7 +2491,7 @@ static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr, · 33 unmodified lines
24742491 */
24752492 smp_rmb();
24762493 cwq = get_work_cwq(work);
2477− if (unlikely(!cwq || gcwq != cwq->gcwq))
2494+ if (unlikely(!cwq || gcwq != cwq->pool->gcwq))
24782495 goto already_gone;
24792496 } else if (wait_executing) {
24802497 worker = find_worker_executing_work(gcwq, work);
@@ -3017,7 +3034,7 @@ struct workqueue_struct *__alloc_workqueue_key(const char *fmt, · 536 unmodified lines
30173034 struct global_cwq *gcwq = get_gcwq(cpu);
30183035
30193036 BUG_ON((unsigned long)cwq & WORK_STRUCT_FLAG_MASK);
3020− cwq->gcwq = gcwq;
3037+ cwq->pool = &gcwq->pool;
30213038 cwq->wq = wq;
30223039 cwq->flush_color = -1;
30233040 cwq->max_active = max_active;
@@ -3344,7 +3361,7 @@ static int __cpuinit trustee_thread(void *__gcwq) · 320 unmodified lines
33443361
33453362 gcwq->flags |= GCWQ_MANAGING_WORKERS;
33463363
3347− list_for_each_entry(worker, &gcwq->idle_list, entry)
3364+ list_for_each_entry(worker, &gcwq->pool.idle_list, entry)
33483365 worker->flags |= WORKER_ROGUE;
33493366
33503367 for_each_busy_worker(worker, i, pos, gcwq)
@@ -3369,7 +3386,7 @@ static int __cpuinit trustee_thread(void *__gcwq) · 18 unmodified lines
33693386 atomic_set(get_gcwq_nr_running(gcwq->cpu), 0);
33703387
33713388 spin_unlock_irq(&gcwq->lock);
3372− del_timer_sync(&gcwq->idle_timer);
3389+ del_timer_sync(&gcwq->pool.idle_timer);
33733390 spin_lock_irq(&gcwq->lock);
33743391
33753392 /*
@@ -3391,17 +3408,17 @@ static int __cpuinit trustee_thread(void *__gcwq) · 15 unmodified lines
33913408 * may be frozen works in freezable cwqs. Don't declare
33923409 * completion while frozen.
33933410 */
3394− while (gcwq->nr_workers != gcwq->nr_idle ||
3411+ while (gcwq->pool.nr_workers != gcwq->pool.nr_idle ||
33953412 gcwq->flags & GCWQ_FREEZING ||
33963413 gcwq->trustee_state == TRUSTEE_IN_CHARGE) {
33973414 int nr_works = 0;
33983415
3399− list_for_each_entry(work, &gcwq->worklist, entry) {
3416+ list_for_each_entry(work, &gcwq->pool.worklist, entry) {
34003417 send_mayday(work);
34013418 nr_works++;
34023419 }
34033420
3404− list_for_each_entry(worker, &gcwq->idle_list, entry) {
3421+ list_for_each_entry(worker, &gcwq->pool.idle_list, entry) {
34053422 if (!nr_works--)
34063423 break;
34073424 wake_up_process(worker->task);
@@ -3428,11 +3445,11 @@ static int __cpuinit trustee_thread(void *__gcwq) · 20 unmodified lines
34283445 * all workers till we're canceled.
34293446 */
34303447 do {
3431− rc = trustee_wait_event(!list_empty(&gcwq->idle_list));
3432− while (!list_empty(&gcwq->idle_list))
3433− destroy_worker(list_first_entry(&gcwq->idle_list,
3448+ rc = trustee_wait_event(!list_empty(&gcwq->pool.idle_list));
3449+ while (!list_empty(&gcwq->pool.idle_list))
3450+ destroy_worker(list_first_entry(&gcwq->pool.idle_list,
34343451 struct worker, entry));
3435− } while (gcwq->nr_workers && rc >= 0);
3452+ } while (gcwq->pool.nr_workers && rc >= 0);
34363453
34373454 /*
34383455 * At this point, either draining has completed and no worker
@@ -3441,7 +3458,7 @@ static int __cpuinit trustee_thread(void *__gcwq) · 2 unmodified lines
34413458 * Tell the remaining busy ones to rebind once it finishes the
34423459 * currently scheduled works by scheduling the rebind_work.
34433460 */
3444− WARN_ON(!list_empty(&gcwq->idle_list));
3461+ WARN_ON(!list_empty(&gcwq->pool.idle_list));
34453462
34463463 for_each_busy_worker(worker, i, pos, gcwq) {
34473464 struct work_struct *rebind_work = &worker->rebind_work;
@@ -3522,7 +3539,7 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, · 74 unmodified lines
35223539 kthread_bind(new_trustee, cpu);
35233540 /* fall through */
35243541 case CPU_UP_PREPARE:
3525− BUG_ON(gcwq->first_idle);
3542+ BUG_ON(gcwq->pool.first_idle);
35263543 new_worker = create_worker(gcwq, false);
35273544 if (!new_worker) {
35283545 if (new_trustee)
@@ -3544,8 +3561,8 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, · 15 unmodified lines
35443561 wait_trustee_state(gcwq, TRUSTEE_IN_CHARGE);
35453562 /* fall through */
35463563 case CPU_UP_PREPARE:
3547− BUG_ON(gcwq->first_idle);
3548− gcwq->first_idle = new_worker;
3564+ BUG_ON(gcwq->pool.first_idle);
3565+ gcwq->pool.first_idle = new_worker;
35493566 break;
35503567
35513568 case CPU_DYING:
@@ -3562,8 +3579,8 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, · 10 unmodified lines
35623579 gcwq->trustee_state = TRUSTEE_BUTCHER;
35633580 /* fall through */
35643581 case CPU_UP_CANCELED:
3565− destroy_worker(gcwq->first_idle);
3566− gcwq->first_idle = NULL;
3582+ destroy_worker(gcwq->pool.first_idle);
3583+ gcwq->pool.first_idle = NULL;
35673584 break;
35683585
35693586 case CPU_DOWN_FAILED:
@@ -3581,11 +3598,11 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, · 11 unmodified lines
35813598 * take a look.
35823599 */
35833600 spin_unlock_irq(&gcwq->lock);
3584− kthread_bind(gcwq->first_idle->task, cpu);
3601+ kthread_bind(gcwq->pool.first_idle->task, cpu);
35853602 spin_lock_irq(&gcwq->lock);
35863603 gcwq->flags |= GCWQ_MANAGE_WORKERS;
3587− start_worker(gcwq->first_idle);
3588− gcwq->first_idle = NULL;
3604+ start_worker(gcwq->pool.first_idle);
3605+ gcwq->pool.first_idle = NULL;
35893606 break;
35903607 }
35913608
@@ -3794,22 +3811,23 @@ static int __init init_workqueues(void) · 202 unmodified lines
37943811 struct global_cwq *gcwq = get_gcwq(cpu);
37953812
37963813 spin_lock_init(&gcwq->lock);
3797− INIT_LIST_HEAD(&gcwq->worklist);
3814+ gcwq->pool.gcwq = gcwq;
3815+ INIT_LIST_HEAD(&gcwq->pool.worklist);
37983816 gcwq->cpu = cpu;
37993817 gcwq->flags |= GCWQ_DISASSOCIATED;
38003818
3801− INIT_LIST_HEAD(&gcwq->idle_list);
3819+ INIT_LIST_HEAD(&gcwq->pool.idle_list);
38023820 for (i = 0; i < BUSY_WORKER_HASH_SIZE; i++)
38033821 INIT_HLIST_HEAD(&gcwq->busy_hash[i]);
38043822
3805− init_timer_deferrable(&gcwq->idle_timer);
3806− gcwq->idle_timer.function = idle_worker_timeout;
3807− gcwq->idle_timer.data = (unsigned long)gcwq;
3823+ init_timer_deferrable(&gcwq->pool.idle_timer);
3824+ gcwq->pool.idle_timer.function = idle_worker_timeout;
3825+ gcwq->pool.idle_timer.data = (unsigned long)gcwq;
38083826
3809− setup_timer(&gcwq->mayday_timer, gcwq_mayday_timeout,
3827+ setup_timer(&gcwq->pool.mayday_timer, gcwq_mayday_timeout,
38103828 (unsigned long)gcwq);
38113829
3812− ida_init(&gcwq->worker_ida);
3830+ ida_init(&gcwq->pool.worker_ida);
38133831
38143832 gcwq->trustee_state = TRUSTEE_DONE;
38153833 init_waitqueue_head(&gcwq->trustee_wait);