History

workqueue: move busy_hash from global_cwq to worker_pool

There's no functional necessity for the two pools on the same CPU to share the busy hash table. It's also likely to be a bottleneck when implementing pools with user-specified attributes. This patch makes busy_hash per-pool. The conversion is mostly straight-forward. Changes worth noting are, * Large block of changes in rebind_workers() is moving the block inside for_each_worker_pool() as now there are separate hash tables for each pool. This changes the order of operations but doesn't break anything. * Thre for_each_worker_pool() loops in gcwq_unbind_fn() are combined into one. This again changes the order of operaitons but doesn't break anything. This is part of an effort to remove global_cwq and make worker_pool the top level abstraction, which in turn will help implementing worker pools with user-specified attributes.

Tejun Heo committed 13 years ago

Commit c9e7cf21 parent 7c3eed51 file · +59 −52Browse files

Unified · Split

kernel/workqueue.c+59−52
@@ -137,6 +137,10 @@ struct worker_pool { · 136 unmodified lines
137 struct timer_list idle_timer; /* L: worker idle timeout */137 struct timer_list idle_timer; /* L: worker idle timeout */
138 struct timer_list mayday_timer; /* L: SOS timer for workers */138 struct timer_list mayday_timer; /* L: SOS timer for workers */
139 139
140 /* workers are chained either in busy_hash or idle_list */
141 DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER);
142 /* L: hash of busy workers */
143
140 struct mutex assoc_mutex; /* protect POOL_DISASSOCIATED */144 struct mutex assoc_mutex; /* protect POOL_DISASSOCIATED */
141 struct ida worker_ida; /* L: for worker IDs */145 struct ida worker_ida; /* L: for worker IDs */
142};146};
@@ -150,10 +154,6 @@ struct global_cwq { · 7 unmodified lines
150 spinlock_t lock; /* the gcwq lock */154 spinlock_t lock; /* the gcwq lock */
151 unsigned int cpu; /* I: the associated cpu */155 unsigned int cpu; /* I: the associated cpu */
152 156
153 /* workers are chained either in busy_hash or pool idle_list */
154 DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER);
155 /* L: hash of busy workers */
156
157 struct worker_pool pools[NR_STD_WORKER_POOLS];157 struct worker_pool pools[NR_STD_WORKER_POOLS];
158 /* normal and highpri pools */158 /* normal and highpri pools */
159} ____cacheline_aligned_in_smp;159} ____cacheline_aligned_in_smp;
@@ -255,8 +255,8 @@ EXPORT_SYMBOL_GPL(system_freezable_wq); · 95 unmodified lines
255 for ((pool) = &(gcwq)->pools[0]; \255 for ((pool) = &(gcwq)->pools[0]; \
256 (pool) < &(gcwq)->pools[NR_STD_WORKER_POOLS]; (pool)++)256 (pool) < &(gcwq)->pools[NR_STD_WORKER_POOLS]; (pool)++)
257 257
258#define for_each_busy_worker(worker, i, pos, gcwq) \258#define for_each_busy_worker(worker, i, pos, pool) \
259 hash_for_each(gcwq->busy_hash, i, pos, worker, hentry)259 hash_for_each(pool->busy_hash, i, pos, worker, hentry)
260 260
261static inline int __next_gcwq_cpu(int cpu, const struct cpumask *mask,261static inline int __next_gcwq_cpu(int cpu, const struct cpumask *mask,
262 unsigned int sw)262 unsigned int sw)
@@ -892,11 +892,11 @@ static inline void worker_clr_flags(struct worker *worker, unsigned int flags) · 629 unmodified lines
892 892
893/**893/**
894 * find_worker_executing_work - find worker which is executing a work894 * find_worker_executing_work - find worker which is executing a work
895 * @gcwq: gcwq of interest895 * @pool: pool of interest
896 * @work: work to find worker for896 * @work: work to find worker for
897 *897 *
898 * Find a worker which is executing @work on @gcwq by searching898 * Find a worker which is executing @work on @pool by searching
899 * @gcwq->busy_hash which is keyed by the address of @work. For a worker899 * @pool->busy_hash which is keyed by the address of @work. For a worker
900 * to match, its current execution should match the address of @work and900 * to match, its current execution should match the address of @work and
901 * its work function. This is to avoid unwanted dependency between901 * its work function. This is to avoid unwanted dependency between
902 * unrelated work executions through a work item being recycled while still902 * unrelated work executions through a work item being recycled while still
@@ -924,13 +924,13 @@ static inline void worker_clr_flags(struct worker *worker, unsigned int flags) · 21 unmodified lines
924 * Pointer to worker which is executing @work if found, NULL924 * Pointer to worker which is executing @work if found, NULL
925 * otherwise.925 * otherwise.
926 */926 */
927static struct worker *find_worker_executing_work(struct global_cwq *gcwq,927static struct worker *find_worker_executing_work(struct worker_pool *pool,
928 struct work_struct *work)928 struct work_struct *work)
929{929{
930 struct worker *worker;930 struct worker *worker;
931 struct hlist_node *tmp;931 struct hlist_node *tmp;
932 932
933 hash_for_each_possible(gcwq->busy_hash, worker, tmp, hentry,933 hash_for_each_possible(pool->busy_hash, worker, tmp, hentry,
934 (unsigned long)work)934 (unsigned long)work)
935 if (worker->current_work == work &&935 if (worker->current_work == work &&
936 worker->current_func == work->func)936 worker->current_func == work->func)
@@ -1191,13 +1191,15 @@ static bool is_chained_work(struct workqueue_struct *wq) · 254 unmodified lines
1191 unsigned int cpu;1191 unsigned int cpu;
1192 1192
1193 for_each_gcwq_cpu(cpu) {1193 for_each_gcwq_cpu(cpu) {
1194 struct global_cwq *gcwq = get_gcwq(cpu);1194 struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
1195 struct worker_pool *pool = cwq->pool;
1196 struct global_cwq *gcwq = pool->gcwq;
1195 struct worker *worker;1197 struct worker *worker;
1196 struct hlist_node *pos;1198 struct hlist_node *pos;
1197 int i;1199 int i;
1198 1200
1199 spin_lock_irqsave(&gcwq->lock, flags);1201 spin_lock_irqsave(&gcwq->lock, flags);
1200 for_each_busy_worker(worker, i, pos, gcwq) {1202 for_each_busy_worker(worker, i, pos, pool) {
1201 if (worker->task != current)1203 if (worker->task != current)
1202 continue;1204 continue;
1203 spin_unlock_irqrestore(&gcwq->lock, flags);1205 spin_unlock_irqrestore(&gcwq->lock, flags);
@@ -1238,7 +1240,7 @@ static void __queue_work(unsigned int cpu, struct workqueue_struct *wq, · 34 unmodified lines
1238 1240
1239 /* determine gcwq to use */1241 /* determine gcwq to use */
1240 if (!(wq->flags & WQ_UNBOUND)) {1242 if (!(wq->flags & WQ_UNBOUND)) {
1241 struct global_cwq *last_gcwq;1243 struct worker_pool *last_pool;
1242 1244
1243 if (cpu == WORK_CPU_UNBOUND)1245 if (cpu == WORK_CPU_UNBOUND)
1244 cpu = raw_smp_processor_id();1246 cpu = raw_smp_processor_id();
@@ -1250,14 +1252,15 @@ static void __queue_work(unsigned int cpu, struct workqueue_struct *wq, · 5 unmodified lines
1250 * non-reentrancy.1252 * non-reentrancy.
1251 */1253 */
1252 gcwq = get_gcwq(cpu);1254 gcwq = get_gcwq(cpu);
1253 last_gcwq = get_work_gcwq(work);1255 last_pool = get_work_pool(work);
1254 1256
1255 if (last_gcwq && last_gcwq != gcwq) {1257 if (last_pool && last_pool->gcwq != gcwq) {
1258 struct global_cwq *last_gcwq = last_pool->gcwq;
1256 struct worker *worker;1259 struct worker *worker;
1257 1260
1258 spin_lock(&last_gcwq->lock);1261 spin_lock(&last_gcwq->lock);
1259 1262
1260 worker = find_worker_executing_work(last_gcwq, work);1263 worker = find_worker_executing_work(last_pool, work);
1261 1264
1262 if (worker && worker->current_cwq->wq == wq)1265 if (worker && worker->current_cwq->wq == wq)
1263 gcwq = last_gcwq;1266 gcwq = last_gcwq;
@@ -1722,31 +1725,32 @@ static void rebind_workers(struct global_cwq *gcwq) · 458 unmodified lines
1722 */1725 */
1723 wake_up_process(worker->task);1726 wake_up_process(worker->task);
1724 }1727 }
1725 }
1726 1728
1727 /* rebind busy workers */1729 /* rebind busy workers */
1728 for_each_busy_worker(worker, i, pos, gcwq) {1730 for_each_busy_worker(worker, i, pos, pool) {
1729 struct work_struct *rebind_work = &worker->rebind_work;1731 struct work_struct *rebind_work = &worker->rebind_work;
1730 struct workqueue_struct *wq;1732 struct workqueue_struct *wq;
1731 1733
1732 if (test_and_set_bit(WORK_STRUCT_PENDING_BIT,1734 if (test_and_set_bit(WORK_STRUCT_PENDING_BIT,
1733 work_data_bits(rebind_work)))1735 work_data_bits(rebind_work)))
1734 continue;1736 continue;
1735 1737
1736 debug_work_activate(rebind_work);1738 debug_work_activate(rebind_work);
1737 1739
1738 /*1740 /*
1739 * wq doesn't really matter but let's keep @worker->pool1741 * wq doesn't really matter but let's keep
1740 * and @cwq->pool consistent for sanity.1742 * @worker->pool and @cwq->pool consistent for
1741 */1743 * sanity.
1742 if (std_worker_pool_pri(worker->pool))1744 */
1743 wq = system_highpri_wq;1745 if (std_worker_pool_pri(worker->pool))
1744 else1746 wq = system_highpri_wq;
1745 wq = system_wq;1747 else
1746 1748 wq = system_wq;
1747 insert_work(get_cwq(gcwq->cpu, wq), rebind_work,1749
1748 worker->scheduled.next,1750 insert_work(get_cwq(gcwq->cpu, wq), rebind_work,
1749 work_color_to_flags(WORK_NO_COLOR));1751 worker->scheduled.next,
1752 work_color_to_flags(WORK_NO_COLOR));
1753 }
1750 }1754 }
1751}1755}
1752 1756
@@ -2197,7 +2201,7 @@ __acquires(&gcwq->lock) · 444 unmodified lines
2197 * already processing the work. If so, defer the work to the2201 * already processing the work. If so, defer the work to the
2198 * currently executing one.2202 * currently executing one.
2199 */2203 */
2200 collision = find_worker_executing_work(gcwq, work);2204 collision = find_worker_executing_work(pool, work);
2201 if (unlikely(collision)) {2205 if (unlikely(collision)) {
2202 move_linked_works(work, &collision->scheduled, NULL);2206 move_linked_works(work, &collision->scheduled, NULL);
2203 return;2207 return;
@@ -2205,7 +2209,7 @@ __acquires(&gcwq->lock) · 1 unmodified line
2205 2209
2206 /* claim and dequeue */2210 /* claim and dequeue */
2207 debug_work_deactivate(work);2211 debug_work_deactivate(work);
2208 hash_add(gcwq->busy_hash, &worker->hentry, (unsigned long)work);2212 hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work);
2209 worker->current_work = work;2213 worker->current_work = work;
2210 worker->current_func = work->func;2214 worker->current_func = work->func;
2211 worker->current_cwq = cwq;2215 worker->current_cwq = cwq;
@@ -2833,13 +2837,15 @@ EXPORT_SYMBOL_GPL(drain_workqueue); · 621 unmodified lines
2833static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr)2837static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr)
2834{2838{
2835 struct worker *worker = NULL;2839 struct worker *worker = NULL;
2840 struct worker_pool *pool;
2836 struct global_cwq *gcwq;2841 struct global_cwq *gcwq;
2837 struct cpu_workqueue_struct *cwq;2842 struct cpu_workqueue_struct *cwq;
2838 2843
2839 might_sleep();2844 might_sleep();
2840 gcwq = get_work_gcwq(work);2845 pool = get_work_pool(work);
2841 if (!gcwq)2846 if (!pool)
2842 return false;2847 return false;
2848 gcwq = pool->gcwq;
2843 2849
2844 spin_lock_irq(&gcwq->lock);2850 spin_lock_irq(&gcwq->lock);
2845 if (!list_empty(&work->entry)) {2851 if (!list_empty(&work->entry)) {
@@ -2853,7 +2859,7 @@ static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr) · 7 unmodified lines
2853 if (unlikely(!cwq || gcwq != cwq->pool->gcwq))2859 if (unlikely(!cwq || gcwq != cwq->pool->gcwq))
2854 goto already_gone;2860 goto already_gone;
2855 } else {2861 } else {
2856 worker = find_worker_executing_work(gcwq, work);2862 worker = find_worker_executing_work(pool, work);
2857 if (!worker)2863 if (!worker)
2858 goto already_gone;2864 goto already_gone;
2859 cwq = worker->current_cwq;2865 cwq = worker->current_cwq;
@@ -3482,18 +3488,20 @@ EXPORT_SYMBOL_GPL(workqueue_congested); · 622 unmodified lines
3482 */3488 */
3483unsigned int work_busy(struct work_struct *work)3489unsigned int work_busy(struct work_struct *work)
3484{3490{
3485 struct global_cwq *gcwq = get_work_gcwq(work);3491 struct worker_pool *pool = get_work_pool(work);
3492 struct global_cwq *gcwq;
3486 unsigned long flags;3493 unsigned long flags;
3487 unsigned int ret = 0;3494 unsigned int ret = 0;
3488 3495
3489 if (!gcwq)3496 if (!pool)
3490 return 0;3497 return 0;
3498 gcwq = pool->gcwq;
3491 3499
3492 spin_lock_irqsave(&gcwq->lock, flags);3500 spin_lock_irqsave(&gcwq->lock, flags);
3493 3501
3494 if (work_pending(work))3502 if (work_pending(work))
3495 ret |= WORK_BUSY_PENDING;3503 ret |= WORK_BUSY_PENDING;
3496 if (find_worker_executing_work(gcwq, work))3504 if (find_worker_executing_work(pool, work))
3497 ret |= WORK_BUSY_RUNNING;3505 ret |= WORK_BUSY_RUNNING;
3498 3506
3499 spin_unlock_irqrestore(&gcwq->lock, flags);3507 spin_unlock_irqrestore(&gcwq->lock, flags);
@@ -3555,15 +3563,15 @@ static void gcwq_unbind_fn(struct work_struct *work) · 55 unmodified lines
3555 * ones which are still executing works from before the last CPU3563 * ones which are still executing works from before the last CPU
3556 * down must be on the cpu. After this, they may become diasporas.3564 * down must be on the cpu. After this, they may become diasporas.
3557 */3565 */
3558 for_each_worker_pool(pool, gcwq)3566 for_each_worker_pool(pool, gcwq) {
3559 list_for_each_entry(worker, &pool->idle_list, entry)3567 list_for_each_entry(worker, &pool->idle_list, entry)
3560 worker->flags |= WORKER_UNBOUND;3568 worker->flags |= WORKER_UNBOUND;
3561 3569
3562 for_each_busy_worker(worker, i, pos, gcwq)3570 for_each_busy_worker(worker, i, pos, pool)
3563 worker->flags |= WORKER_UNBOUND;3571 worker->flags |= WORKER_UNBOUND;
3564 3572
3565 for_each_worker_pool(pool, gcwq)
3566 pool->flags |= POOL_DISASSOCIATED;3573 pool->flags |= POOL_DISASSOCIATED;
3574 }
3567 3575
3568 gcwq_release_assoc_and_unlock(gcwq);3576 gcwq_release_assoc_and_unlock(gcwq);
3569 3577
@@ -3854,13 +3862,12 @@ static int __init init_workqueues(void) · 284 unmodified lines
3854 spin_lock_init(&gcwq->lock);3862 spin_lock_init(&gcwq->lock);
3855 gcwq->cpu = cpu;3863 gcwq->cpu = cpu;
3856 3864
3857 hash_init(gcwq->busy_hash);
3858
3859 for_each_worker_pool(pool, gcwq) {3865 for_each_worker_pool(pool, gcwq) {
3860 pool->gcwq = gcwq;3866 pool->gcwq = gcwq;
3861 pool->flags |= POOL_DISASSOCIATED;3867 pool->flags |= POOL_DISASSOCIATED;
3862 INIT_LIST_HEAD(&pool->worklist);3868 INIT_LIST_HEAD(&pool->worklist);
3863 INIT_LIST_HEAD(&pool->idle_list);3869 INIT_LIST_HEAD(&pool->idle_list);
3870 hash_init(pool->busy_hash);
3864 3871
3865 init_timer_deferrable(&pool->idle_timer);3872 init_timer_deferrable(&pool->idle_timer);
3866 pool->idle_timer.function = idle_worker_timeout;3873 pool->idle_timer.function = idle_worker_timeout;