History

workqueue: move busy_hash from global_cwq to worker_pool

There's no functional necessity for the two pools on the same CPU to share the busy hash table. It's also likely to be a bottleneck when implementing pools with user-specified attributes. This patch makes busy_hash per-pool. The conversion is mostly straight-forward. Changes worth noting are, * Large block of changes in rebind_workers() is moving the block inside for_each_worker_pool() as now there are separate hash tables for each pool. This changes the order of operations but doesn't break anything. * Thre for_each_worker_pool() loops in gcwq_unbind_fn() are combined into one. This again changes the order of operaitons but doesn't break anything. This is part of an effort to remove global_cwq and make worker_pool the top level abstraction, which in turn will help implementing worker pools with user-specified attributes.

Tejun Heo committed 13 years ago

Commit c9e7cf21 parent 7c3eed51 file · +59 −52Browse files

Unified · Split

kernel/workqueue.c+59−52
@@ -137,6 +137,10 @@ struct worker_pool { · 136 unmodified lines
137137 struct timer_list idle_timer; /* L: worker idle timeout */
138138 struct timer_list mayday_timer; /* L: SOS timer for workers */
139139
140+ /* workers are chained either in busy_hash or idle_list */
141+ DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER);
142+ /* L: hash of busy workers */
143+
140144 struct mutex assoc_mutex; /* protect POOL_DISASSOCIATED */
141145 struct ida worker_ida; /* L: for worker IDs */
142146 };
@@ -150,10 +154,6 @@ struct global_cwq { · 7 unmodified lines
150154 spinlock_t lock; /* the gcwq lock */
151155 unsigned int cpu; /* I: the associated cpu */
152156
153− /* workers are chained either in busy_hash or pool idle_list */
154− DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER);
155− /* L: hash of busy workers */
156−
157157 struct worker_pool pools[NR_STD_WORKER_POOLS];
158158 /* normal and highpri pools */
159159 } ____cacheline_aligned_in_smp;
@@ -255,8 +255,8 @@ EXPORT_SYMBOL_GPL(system_freezable_wq); · 95 unmodified lines
255255 for ((pool) = &(gcwq)->pools[0]; \
256256 (pool) < &(gcwq)->pools[NR_STD_WORKER_POOLS]; (pool)++)
257257
258−#define for_each_busy_worker(worker, i, pos, gcwq) \
259− hash_for_each(gcwq->busy_hash, i, pos, worker, hentry)
258+#define for_each_busy_worker(worker, i, pos, pool) \
259+ hash_for_each(pool->busy_hash, i, pos, worker, hentry)
260260
261261 static inline int __next_gcwq_cpu(int cpu, const struct cpumask *mask,
262262 unsigned int sw)
@@ -892,11 +892,11 @@ static inline void worker_clr_flags(struct worker *worker, unsigned int flags) · 629 unmodified lines
892892
893893 /**
894894 * find_worker_executing_work - find worker which is executing a work
895− * @gcwq: gcwq of interest
895+ * @pool: pool of interest
896896 * @work: work to find worker for
897897 *
898− * Find a worker which is executing @work on @gcwq by searching
899− * @gcwq->busy_hash which is keyed by the address of @work. For a worker
898+ * Find a worker which is executing @work on @pool by searching
899+ * @pool->busy_hash which is keyed by the address of @work. For a worker
900900 * to match, its current execution should match the address of @work and
901901 * its work function. This is to avoid unwanted dependency between
902902 * unrelated work executions through a work item being recycled while still
@@ -924,13 +924,13 @@ static inline void worker_clr_flags(struct worker *worker, unsigned int flags) · 21 unmodified lines
924924 * Pointer to worker which is executing @work if found, NULL
925925 * otherwise.
926926 */
927−static struct worker *find_worker_executing_work(struct global_cwq *gcwq,
927+static struct worker *find_worker_executing_work(struct worker_pool *pool,
928928 struct work_struct *work)
929929 {
930930 struct worker *worker;
931931 struct hlist_node *tmp;
932932
933− hash_for_each_possible(gcwq->busy_hash, worker, tmp, hentry,
933+ hash_for_each_possible(pool->busy_hash, worker, tmp, hentry,
934934 (unsigned long)work)
935935 if (worker->current_work == work &&
936936 worker->current_func == work->func)
@@ -1191,13 +1191,15 @@ static bool is_chained_work(struct workqueue_struct *wq) · 254 unmodified lines
11911191 unsigned int cpu;
11921192
11931193 for_each_gcwq_cpu(cpu) {
1194− struct global_cwq *gcwq = get_gcwq(cpu);
1194+ struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
1195+ struct worker_pool *pool = cwq->pool;
1196+ struct global_cwq *gcwq = pool->gcwq;
11951197 struct worker *worker;
11961198 struct hlist_node *pos;
11971199 int i;
11981200
11991201 spin_lock_irqsave(&gcwq->lock, flags);
1200− for_each_busy_worker(worker, i, pos, gcwq) {
1202+ for_each_busy_worker(worker, i, pos, pool) {
12011203 if (worker->task != current)
12021204 continue;
12031205 spin_unlock_irqrestore(&gcwq->lock, flags);
@@ -1238,7 +1240,7 @@ static void __queue_work(unsigned int cpu, struct workqueue_struct *wq, · 34 unmodified lines
12381240
12391241 /* determine gcwq to use */
12401242 if (!(wq->flags & WQ_UNBOUND)) {
1241− struct global_cwq *last_gcwq;
1243+ struct worker_pool *last_pool;
12421244
12431245 if (cpu == WORK_CPU_UNBOUND)
12441246 cpu = raw_smp_processor_id();
@@ -1250,14 +1252,15 @@ static void __queue_work(unsigned int cpu, struct workqueue_struct *wq, · 5 unmodified lines
12501252 * non-reentrancy.
12511253 */
12521254 gcwq = get_gcwq(cpu);
1253− last_gcwq = get_work_gcwq(work);
1255+ last_pool = get_work_pool(work);
12541256
1255− if (last_gcwq && last_gcwq != gcwq) {
1257+ if (last_pool && last_pool->gcwq != gcwq) {
1258+ struct global_cwq *last_gcwq = last_pool->gcwq;
12561259 struct worker *worker;
12571260
12581261 spin_lock(&last_gcwq->lock);
12591262
1260− worker = find_worker_executing_work(last_gcwq, work);
1263+ worker = find_worker_executing_work(last_pool, work);
12611264
12621265 if (worker && worker->current_cwq->wq == wq)
12631266 gcwq = last_gcwq;
@@ -1722,31 +1725,32 @@ static void rebind_workers(struct global_cwq *gcwq) · 458 unmodified lines
17221725 */
17231726 wake_up_process(worker->task);
17241727 }
1725− }
17261728
1727− /* rebind busy workers */
1728− for_each_busy_worker(worker, i, pos, gcwq) {
1729− struct work_struct *rebind_work = &worker->rebind_work;
1730− struct workqueue_struct *wq;
1729+ /* rebind busy workers */
1730+ for_each_busy_worker(worker, i, pos, pool) {
1731+ struct work_struct *rebind_work = &worker->rebind_work;
1732+ struct workqueue_struct *wq;
17311733
1732− if (test_and_set_bit(WORK_STRUCT_PENDING_BIT,
1733− work_data_bits(rebind_work)))
1734− continue;
1734+ if (test_and_set_bit(WORK_STRUCT_PENDING_BIT,
1735+ work_data_bits(rebind_work)))
1736+ continue;
17351737
1736− debug_work_activate(rebind_work);
1738+ debug_work_activate(rebind_work);
17371739
1738− /*
1739− * wq doesn't really matter but let's keep @worker->pool
1740− * and @cwq->pool consistent for sanity.
1741− */
1742− if (std_worker_pool_pri(worker->pool))
1743− wq = system_highpri_wq;
1744− else
1745− wq = system_wq;
1746−
1747− insert_work(get_cwq(gcwq->cpu, wq), rebind_work,
1748− worker->scheduled.next,
1749− work_color_to_flags(WORK_NO_COLOR));
1740+ /*
1741+ * wq doesn't really matter but let's keep
1742+ * @worker->pool and @cwq->pool consistent for
1743+ * sanity.
1744+ */
1745+ if (std_worker_pool_pri(worker->pool))
1746+ wq = system_highpri_wq;
1747+ else
1748+ wq = system_wq;
1749+
1750+ insert_work(get_cwq(gcwq->cpu, wq), rebind_work,
1751+ worker->scheduled.next,
1752+ work_color_to_flags(WORK_NO_COLOR));
1753+ }
17501754 }
17511755 }
17521756
@@ -2197,7 +2201,7 @@ __acquires(&gcwq->lock) · 444 unmodified lines
21972201 * already processing the work. If so, defer the work to the
21982202 * currently executing one.
21992203 */
2200− collision = find_worker_executing_work(gcwq, work);
2204+ collision = find_worker_executing_work(pool, work);
22012205 if (unlikely(collision)) {
22022206 move_linked_works(work, &collision->scheduled, NULL);
22032207 return;
@@ -2205,7 +2209,7 @@ __acquires(&gcwq->lock) · 1 unmodified line
22052209
22062210 /* claim and dequeue */
22072211 debug_work_deactivate(work);
2208− hash_add(gcwq->busy_hash, &worker->hentry, (unsigned long)work);
2212+ hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work);
22092213 worker->current_work = work;
22102214 worker->current_func = work->func;
22112215 worker->current_cwq = cwq;
@@ -2833,13 +2837,15 @@ EXPORT_SYMBOL_GPL(drain_workqueue); · 621 unmodified lines
28332837 static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr)
28342838 {
28352839 struct worker *worker = NULL;
2840+ struct worker_pool *pool;
28362841 struct global_cwq *gcwq;
28372842 struct cpu_workqueue_struct *cwq;
28382843
28392844 might_sleep();
2840− gcwq = get_work_gcwq(work);
2841− if (!gcwq)
2845+ pool = get_work_pool(work);
2846+ if (!pool)
28422847 return false;
2848+ gcwq = pool->gcwq;
28432849
28442850 spin_lock_irq(&gcwq->lock);
28452851 if (!list_empty(&work->entry)) {
@@ -2853,7 +2859,7 @@ static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr) · 7 unmodified lines
28532859 if (unlikely(!cwq || gcwq != cwq->pool->gcwq))
28542860 goto already_gone;
28552861 } else {
2856− worker = find_worker_executing_work(gcwq, work);
2862+ worker = find_worker_executing_work(pool, work);
28572863 if (!worker)
28582864 goto already_gone;
28592865 cwq = worker->current_cwq;
@@ -3482,18 +3488,20 @@ EXPORT_SYMBOL_GPL(workqueue_congested); · 622 unmodified lines
34823488 */
34833489 unsigned int work_busy(struct work_struct *work)
34843490 {
3485− struct global_cwq *gcwq = get_work_gcwq(work);
3491+ struct worker_pool *pool = get_work_pool(work);
3492+ struct global_cwq *gcwq;
34863493 unsigned long flags;
34873494 unsigned int ret = 0;
34883495
3489− if (!gcwq)
3496+ if (!pool)
34903497 return 0;
3498+ gcwq = pool->gcwq;
34913499
34923500 spin_lock_irqsave(&gcwq->lock, flags);
34933501
34943502 if (work_pending(work))
34953503 ret |= WORK_BUSY_PENDING;
3496− if (find_worker_executing_work(gcwq, work))
3504+ if (find_worker_executing_work(pool, work))
34973505 ret |= WORK_BUSY_RUNNING;
34983506
34993507 spin_unlock_irqrestore(&gcwq->lock, flags);
@@ -3555,15 +3563,15 @@ static void gcwq_unbind_fn(struct work_struct *work) · 55 unmodified lines
35553563 * ones which are still executing works from before the last CPU
35563564 * down must be on the cpu. After this, they may become diasporas.
35573565 */
3558− for_each_worker_pool(pool, gcwq)
3566+ for_each_worker_pool(pool, gcwq) {
35593567 list_for_each_entry(worker, &pool->idle_list, entry)
35603568 worker->flags |= WORKER_UNBOUND;
35613569
3562− for_each_busy_worker(worker, i, pos, gcwq)
3563− worker->flags |= WORKER_UNBOUND;
3570+ for_each_busy_worker(worker, i, pos, pool)
3571+ worker->flags |= WORKER_UNBOUND;
35643572
3565− for_each_worker_pool(pool, gcwq)
35663573 pool->flags |= POOL_DISASSOCIATED;
3574+ }
35673575
35683576 gcwq_release_assoc_and_unlock(gcwq);
35693577
@@ -3854,13 +3862,12 @@ static int __init init_workqueues(void) · 284 unmodified lines
38543862 spin_lock_init(&gcwq->lock);
38553863 gcwq->cpu = cpu;
38563864
3857− hash_init(gcwq->busy_hash);
3858−
38593865 for_each_worker_pool(pool, gcwq) {
38603866 pool->gcwq = gcwq;
38613867 pool->flags |= POOL_DISASSOCIATED;
38623868 INIT_LIST_HEAD(&pool->worklist);
38633869 INIT_LIST_HEAD(&pool->idle_list);
3870+ hash_init(pool->busy_hash);
38643871
38653872 init_timer_deferrable(&pool->idle_timer);
38663873 pool->idle_timer.function = idle_worker_timeout;