History

workqueue: move nr_running into worker_pool

As nr_running is likely to be accessed from other CPUs during try_to_wake_up(), it was kept outside worker_pool; however, while less frequent, other fields in worker_pool are accessed from other CPUs for, e.g., non-reentrancy check. Also, with recent pool related changes, accessing nr_running matching the worker_pool isn't as simple as it used to be. Move nr_running inside worker_pool. Keep it aligned to cacheline and define CPU pools using DEFINE_PER_CPU_SHARED_ALIGNED(). This should give at least the same cacheline behavior. get_pool_nr_running() is replaced with direct pool->nr_running accesses.

Tejun Heo committed 13 years ago

Commit e19e3971 parent 16062831 file · +22 −41Browse files

Unified · Split

kernel/workqueue.c+22−41
@@ -144,6 +144,13 @@ struct worker_pool { · 143 unmodified lines
144144
145145 struct mutex assoc_mutex; /* protect POOL_DISASSOCIATED */
146146 struct ida worker_ida; /* L: for worker IDs */
147+
148+ /*
149+ * The current concurrency level. As it's likely to be accessed
150+ * from other CPUs during try_to_wake_up(), put it in a separate
151+ * cacheline.
152+ */
153+ atomic_t nr_running ____cacheline_aligned_in_smp;
147154 } ____cacheline_aligned_in_smp;
148155
149156 /*
@@ -417,23 +424,12 @@ static LIST_HEAD(workqueues); · 267 unmodified lines
417424 static bool workqueue_freezing; /* W: have wqs started freezing? */
418425
419426 /*
420− * The CPU standard worker pools. nr_running is the only field which is
421− * expected to be used frequently by other cpus via try_to_wake_up(). Put
422− * it in a separate cacheline.
423− */
424−static DEFINE_PER_CPU(struct worker_pool [NR_STD_WORKER_POOLS],
425− cpu_std_worker_pools);
426−static DEFINE_PER_CPU_SHARED_ALIGNED(atomic_t [NR_STD_WORKER_POOLS],
427− cpu_std_pool_nr_running);
428−
429−/*
430− * Standard worker pools and nr_running counter for unbound CPU. The pools
431− * have POOL_DISASSOCIATED set, and all workers have WORKER_UNBOUND set.
427+ * The CPU and unbound standard worker pools. The unbound ones have
428+ * POOL_DISASSOCIATED set, and their workers have WORKER_UNBOUND set.
432429 */
430+static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS],
431+ cpu_std_worker_pools);
433432 static struct worker_pool unbound_std_worker_pools[NR_STD_WORKER_POOLS];
434−static atomic_t unbound_std_pool_nr_running[NR_STD_WORKER_POOLS] = {
435− [0 ... NR_STD_WORKER_POOLS - 1] = ATOMIC_INIT(0), /* always 0 */
436−};
437433
438434 /* idr of all pools */
439435 static DEFINE_MUTEX(worker_pool_idr_mutex);
@@ -483,17 +479,6 @@ static struct worker_pool *get_std_worker_pool(int cpu, bool highpri) · 43 unmodified lines
483479 return &pools[highpri];
484480 }
485481
486−static atomic_t *get_pool_nr_running(struct worker_pool *pool)
487−{
488− int cpu = pool->cpu;
489− int idx = std_worker_pool_pri(pool);
490−
491− if (cpu != WORK_CPU_UNBOUND)
492− return &per_cpu(cpu_std_pool_nr_running, cpu)[idx];
493− else
494− return &unbound_std_pool_nr_running[idx];
495−}
496−
497482 static struct cpu_workqueue_struct *get_cwq(unsigned int cpu,
498483 struct workqueue_struct *wq)
499484 {
@@ -654,7 +639,7 @@ static bool work_is_canceling(struct work_struct *work) · 154 unmodified lines
654639
655640 static bool __need_more_worker(struct worker_pool *pool)
656641 {
657− return !atomic_read(get_pool_nr_running(pool));
642+ return !atomic_read(&pool->nr_running);
658643 }
659644
660645 /*
@@ -679,9 +664,8 @@ static bool may_start_working(struct worker_pool *pool) · 18 unmodified lines
679664 /* Do I need to keep working? Called from currently running workers. */
680665 static bool keep_working(struct worker_pool *pool)
681666 {
682− atomic_t *nr_running = get_pool_nr_running(pool);
683−
684− return !list_empty(&pool->worklist) && atomic_read(nr_running) <= 1;
667+ return !list_empty(&pool->worklist) &&
668+ atomic_read(&pool->nr_running) <= 1;
685669 }
686670
687671 /* Do we need a new worker? Called from manager. */
@@ -761,7 +745,7 @@ void wq_worker_waking_up(struct task_struct *task, unsigned int cpu) · 73 unmodified lines
761745
762746 if (!(worker->flags & WORKER_NOT_RUNNING)) {
763747 WARN_ON_ONCE(worker->pool->cpu != cpu);
764− atomic_inc(get_pool_nr_running(worker->pool));
748+ atomic_inc(&worker->pool->nr_running);
765749 }
766750 }
767751
@@ -785,7 +769,6 @@ struct task_struct *wq_worker_sleeping(struct task_struct *task, · 17 unmodified lines
785769 {
786770 struct worker *worker = kthread_data(task), *to_wakeup = NULL;
787771 struct worker_pool *pool;
788− atomic_t *nr_running;
789772
790773 /*
791774 * Rescuers, which may not have all the fields set up like normal
@@ -796,7 +779,6 @@ struct task_struct *wq_worker_sleeping(struct task_struct *task, · 4 unmodified lines
796779 return NULL;
797780
798781 pool = worker->pool;
799− nr_running = get_pool_nr_running(pool);
800782
801783 /* this can only happen on the local cpu */
802784 BUG_ON(cpu != raw_smp_processor_id());
@@ -812,7 +794,8 @@ struct task_struct *wq_worker_sleeping(struct task_struct *task, · 9 unmodified lines
812794 * manipulating idle_list, so dereferencing idle_list without pool
813795 * lock is safe.
814796 */
815− if (atomic_dec_and_test(nr_running) && !list_empty(&pool->worklist))
797+ if (atomic_dec_and_test(&pool->nr_running) &&
798+ !list_empty(&pool->worklist))
816799 to_wakeup = first_worker(pool);
817800 return to_wakeup ? to_wakeup->task : NULL;
818801 }
@@ -844,14 +827,12 @@ static inline void worker_set_flags(struct worker *worker, unsigned int flags, · 25 unmodified lines
844827 */
845828 if ((flags & WORKER_NOT_RUNNING) &&
846829 !(worker->flags & WORKER_NOT_RUNNING)) {
847− atomic_t *nr_running = get_pool_nr_running(pool);
848−
849830 if (wakeup) {
850− if (atomic_dec_and_test(nr_running) &&
831+ if (atomic_dec_and_test(&pool->nr_running) &&
851832 !list_empty(&pool->worklist))
852833 wake_up_worker(pool);
853834 } else
854− atomic_dec(nr_running);
835+ atomic_dec(&pool->nr_running);
855836 }
856837
857838 worker->flags |= flags;
@@ -883,7 +864,7 @@ static inline void worker_clr_flags(struct worker *worker, unsigned int flags) · 25 unmodified lines
883864 */
884865 if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
885866 if (!(worker->flags & WORKER_NOT_RUNNING))
886− atomic_inc(get_pool_nr_running(pool));
867+ atomic_inc(&pool->nr_running);
887868 }
888869
889870 /**
@@ -1518,7 +1499,7 @@ static void worker_enter_idle(struct worker *worker) · 628 unmodified lines
15181499 */
15191500 WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
15201501 pool->nr_workers == pool->nr_idle &&
1521− atomic_read(get_pool_nr_running(pool)));
1502+ atomic_read(&pool->nr_running));
15221503 }
15231504
15241505 /**
@@ -3506,7 +3487,7 @@ static void wq_unbind_fn(struct work_struct *work) · 1981 unmodified lines
35063487 * didn't already.
35073488 */
35083489 for_each_std_worker_pool(pool, cpu)
3509− atomic_set(get_pool_nr_running(pool), 0);
3490+ atomic_set(&pool->nr_running, 0);
35103491 }
35113492
35123493 /*