History

workqueue: move nr_running into worker_pool

As nr_running is likely to be accessed from other CPUs during try_to_wake_up(), it was kept outside worker_pool; however, while less frequent, other fields in worker_pool are accessed from other CPUs for, e.g., non-reentrancy check. Also, with recent pool related changes, accessing nr_running matching the worker_pool isn't as simple as it used to be. Move nr_running inside worker_pool. Keep it aligned to cacheline and define CPU pools using DEFINE_PER_CPU_SHARED_ALIGNED(). This should give at least the same cacheline behavior. get_pool_nr_running() is replaced with direct pool->nr_running accesses.

Tejun Heo committed 13 years ago

Commit e19e3971 parent 16062831 file · +22 −41Browse files

Unified · Split

kernel/workqueue.c+22−41
@@ -144,6 +144,13 @@ struct worker_pool { · 143 unmodified lines
144 144
145 struct mutex assoc_mutex; /* protect POOL_DISASSOCIATED */145 struct mutex assoc_mutex; /* protect POOL_DISASSOCIATED */
146 struct ida worker_ida; /* L: for worker IDs */146 struct ida worker_ida; /* L: for worker IDs */
147
148 /*
149 * The current concurrency level. As it's likely to be accessed
150 * from other CPUs during try_to_wake_up(), put it in a separate
151 * cacheline.
152 */
153 atomic_t nr_running ____cacheline_aligned_in_smp;
147} ____cacheline_aligned_in_smp;154} ____cacheline_aligned_in_smp;
148 155
149/*156/*
@@ -417,23 +424,12 @@ static LIST_HEAD(workqueues); · 267 unmodified lines
417static bool workqueue_freezing; /* W: have wqs started freezing? */424static bool workqueue_freezing; /* W: have wqs started freezing? */
418 425
419/*426/*
420 * The CPU standard worker pools. nr_running is the only field which is427 * The CPU and unbound standard worker pools. The unbound ones have
421 * expected to be used frequently by other cpus via try_to_wake_up(). Put428 * POOL_DISASSOCIATED set, and their workers have WORKER_UNBOUND set.
422 * it in a separate cacheline.
423 */
424static DEFINE_PER_CPU(struct worker_pool [NR_STD_WORKER_POOLS],
425 cpu_std_worker_pools);
426static DEFINE_PER_CPU_SHARED_ALIGNED(atomic_t [NR_STD_WORKER_POOLS],
427 cpu_std_pool_nr_running);
428
429/*
430 * Standard worker pools and nr_running counter for unbound CPU. The pools
431 * have POOL_DISASSOCIATED set, and all workers have WORKER_UNBOUND set.
432 */429 */
430static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS],
431 cpu_std_worker_pools);
433static struct worker_pool unbound_std_worker_pools[NR_STD_WORKER_POOLS];432static struct worker_pool unbound_std_worker_pools[NR_STD_WORKER_POOLS];
434static atomic_t unbound_std_pool_nr_running[NR_STD_WORKER_POOLS] = {
435 [0 ... NR_STD_WORKER_POOLS - 1] = ATOMIC_INIT(0), /* always 0 */
436};
437 433
438/* idr of all pools */434/* idr of all pools */
439static DEFINE_MUTEX(worker_pool_idr_mutex);435static DEFINE_MUTEX(worker_pool_idr_mutex);
@@ -483,17 +479,6 @@ static struct worker_pool *get_std_worker_pool(int cpu, bool highpri) · 43 unmodified lines
483 return &pools[highpri];479 return &pools[highpri];
484}480}
485 481
486static atomic_t *get_pool_nr_running(struct worker_pool *pool)
487{
488 int cpu = pool->cpu;
489 int idx = std_worker_pool_pri(pool);
490
491 if (cpu != WORK_CPU_UNBOUND)
492 return &per_cpu(cpu_std_pool_nr_running, cpu)[idx];
493 else
494 return &unbound_std_pool_nr_running[idx];
495}
496
497static struct cpu_workqueue_struct *get_cwq(unsigned int cpu,482static struct cpu_workqueue_struct *get_cwq(unsigned int cpu,
498 struct workqueue_struct *wq)483 struct workqueue_struct *wq)
499{484{
@@ -654,7 +639,7 @@ static bool work_is_canceling(struct work_struct *work) · 154 unmodified lines
654 639
655static bool __need_more_worker(struct worker_pool *pool)640static bool __need_more_worker(struct worker_pool *pool)
656{641{
657 return !atomic_read(get_pool_nr_running(pool));642 return !atomic_read(&pool->nr_running);
658}643}
659 644
660/*645/*
@@ -679,9 +664,8 @@ static bool may_start_working(struct worker_pool *pool) · 18 unmodified lines
679/* Do I need to keep working? Called from currently running workers. */664/* Do I need to keep working? Called from currently running workers. */
680static bool keep_working(struct worker_pool *pool)665static bool keep_working(struct worker_pool *pool)
681{666{
682 atomic_t *nr_running = get_pool_nr_running(pool);667 return !list_empty(&pool->worklist) &&
683 668 atomic_read(&pool->nr_running) <= 1;
684 return !list_empty(&pool->worklist) && atomic_read(nr_running) <= 1;
685}669}
686 670
687/* Do we need a new worker? Called from manager. */671/* Do we need a new worker? Called from manager. */
@@ -761,7 +745,7 @@ void wq_worker_waking_up(struct task_struct *task, unsigned int cpu) · 73 unmodified lines
761 745
762 if (!(worker->flags & WORKER_NOT_RUNNING)) {746 if (!(worker->flags & WORKER_NOT_RUNNING)) {
763 WARN_ON_ONCE(worker->pool->cpu != cpu);747 WARN_ON_ONCE(worker->pool->cpu != cpu);
764 atomic_inc(get_pool_nr_running(worker->pool));748 atomic_inc(&worker->pool->nr_running);
765 }749 }
766}750}
767 751
@@ -785,7 +769,6 @@ struct task_struct *wq_worker_sleeping(struct task_struct *task, · 17 unmodified lines
785{769{
786 struct worker *worker = kthread_data(task), *to_wakeup = NULL;770 struct worker *worker = kthread_data(task), *to_wakeup = NULL;
787 struct worker_pool *pool;771 struct worker_pool *pool;
788 atomic_t *nr_running;
789 772
790 /*773 /*
791 * Rescuers, which may not have all the fields set up like normal774 * Rescuers, which may not have all the fields set up like normal
@@ -796,7 +779,6 @@ struct task_struct *wq_worker_sleeping(struct task_struct *task, · 4 unmodified lines
796 return NULL;779 return NULL;
797 780
798 pool = worker->pool;781 pool = worker->pool;
799 nr_running = get_pool_nr_running(pool);
800 782
801 /* this can only happen on the local cpu */783 /* this can only happen on the local cpu */
802 BUG_ON(cpu != raw_smp_processor_id());784 BUG_ON(cpu != raw_smp_processor_id());
@@ -812,7 +794,8 @@ struct task_struct *wq_worker_sleeping(struct task_struct *task, · 9 unmodified lines
812 * manipulating idle_list, so dereferencing idle_list without pool794 * manipulating idle_list, so dereferencing idle_list without pool
813 * lock is safe.795 * lock is safe.
814 */796 */
815 if (atomic_dec_and_test(nr_running) && !list_empty(&pool->worklist))797 if (atomic_dec_and_test(&pool->nr_running) &&
798 !list_empty(&pool->worklist))
816 to_wakeup = first_worker(pool);799 to_wakeup = first_worker(pool);
817 return to_wakeup ? to_wakeup->task : NULL;800 return to_wakeup ? to_wakeup->task : NULL;
818}801}
@@ -844,14 +827,12 @@ static inline void worker_set_flags(struct worker *worker, unsigned int flags, · 25 unmodified lines
844 */827 */
845 if ((flags & WORKER_NOT_RUNNING) &&828 if ((flags & WORKER_NOT_RUNNING) &&
846 !(worker->flags & WORKER_NOT_RUNNING)) {829 !(worker->flags & WORKER_NOT_RUNNING)) {
847 atomic_t *nr_running = get_pool_nr_running(pool);
848
849 if (wakeup) {830 if (wakeup) {
850 if (atomic_dec_and_test(nr_running) &&831 if (atomic_dec_and_test(&pool->nr_running) &&
851 !list_empty(&pool->worklist))832 !list_empty(&pool->worklist))
852 wake_up_worker(pool);833 wake_up_worker(pool);
853 } else834 } else
854 atomic_dec(nr_running);835 atomic_dec(&pool->nr_running);
855 }836 }
856 837
857 worker->flags |= flags;838 worker->flags |= flags;
@@ -883,7 +864,7 @@ static inline void worker_clr_flags(struct worker *worker, unsigned int flags) · 25 unmodified lines
883 */864 */
884 if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))865 if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
885 if (!(worker->flags & WORKER_NOT_RUNNING))866 if (!(worker->flags & WORKER_NOT_RUNNING))
886 atomic_inc(get_pool_nr_running(pool));867 atomic_inc(&pool->nr_running);
887}868}
888 869
889/**870/**
@@ -1518,7 +1499,7 @@ static void worker_enter_idle(struct worker *worker) · 628 unmodified lines
1518 */1499 */
1519 WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&1500 WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
1520 pool->nr_workers == pool->nr_idle &&1501 pool->nr_workers == pool->nr_idle &&
1521 atomic_read(get_pool_nr_running(pool)));1502 atomic_read(&pool->nr_running));
1522}1503}
1523 1504
1524/**1505/**
@@ -3506,7 +3487,7 @@ static void wq_unbind_fn(struct work_struct *work) · 1981 unmodified lines
3506 * didn't already.3487 * didn't already.
3507 */3488 */
3508 for_each_std_worker_pool(pool, cpu)3489 for_each_std_worker_pool(pool, cpu)
3509 atomic_set(get_pool_nr_running(pool), 0);3490 atomic_set(&pool->nr_running, 0);
3510}3491}
3511 3492
3512/*3493/*