History

Merge branch 'for-3.17' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq

Pull workqueue updates from Tejun Heo: "Lai has been doing a lot of cleanups of workqueue and kthread_work. No significant behavior change. Just a lot of cleanups all over the place. Some are a bit invasive but overall nothing too dangerous" * 'for-3.17' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq: kthread_work: remove the unused wait_queue_head kthread_work: wake up worker only when the worker is idle workqueue: use nr_node_ids instead of wq_numa_tbl_len workqueue: remove the misnamed out_unlock label in get_unbound_pool() workqueue: remove the stale comment in pwq_unbound_release_workfn() workqueue: move rescuer pool detachment to the end workqueue: unfold start_worker() into create_worker() workqueue: remove @wakeup from worker_set_flags() workqueue: remove an unneeded UNBOUND test before waking up the next worker workqueue: wake regular worker if need_more_worker() when rescuer leave the pool workqueue: alloc struct worker on its local node workqueue: reuse the already calculated pwq in try_to_grab_pending() workqueue: stronger test in process_one_work() workqueue: clear POOL_DISASSOCIATED in rebind_workers() workqueue: sanity check pool->cpu in wq_worker_sleeping() workqueue: clear leftover flags when detached workqueue: remove useless WARN_ON_ONCE() workqueue: use schedule_timeout_interruptible() instead of open code workqueue: remove the empty check in too_many_workers() workqueue: use "pool->cpu < 0" to stand for an unbound pool

Linus Torvalds committed 12 years ago

Commit c4c3f5f2 parents 1bff598 95847e13 files · +64 −151Browse files

Showing the changes to kernel/workqueue.c (1 of 3 files). Show all

Unified · Split

kernel/workqueue.c+61−139
@@ -265,7 +265,6 @@ struct workqueue_struct { · 264 unmodified lines
265265
266266 static struct kmem_cache *pwq_cache;
267267
268−static int wq_numa_tbl_len; /* highest possible NUMA node id + 1 */
269268 static cpumask_var_t *wq_numa_possible_cpumask;
270269 /* possible CPUs of each node */
271270
@@ -758,13 +757,6 @@ static bool too_many_workers(struct worker_pool *pool) · 486 unmodified lines
758757 int nr_idle = pool->nr_idle + managing; /* manager is considered idle */
759758 int nr_busy = pool->nr_workers - nr_idle;
760759
761− /*
762− * nr_idle and idle_list may disagree if idle rebinding is in
763− * progress. Never return %true if idle_list is empty.
764− */
765− if (list_empty(&pool->idle_list))
766− return false;
767−
768760 return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;
769761 }
770762
@@ -850,7 +842,7 @@ struct task_struct *wq_worker_sleeping(struct task_struct *task, int cpu) · 79 unmodified lines
850842 pool = worker->pool;
851843
852844 /* this can only happen on the local cpu */
853− if (WARN_ON_ONCE(cpu != raw_smp_processor_id()))
845+ if (WARN_ON_ONCE(cpu != raw_smp_processor_id() || pool->cpu != cpu))
854846 return NULL;
855847
856848 /*
@@ -874,35 +866,22 @@ struct task_struct *wq_worker_sleeping(struct task_struct *task, int cpu) · 17 unmodified lines
874866 * worker_set_flags - set worker flags and adjust nr_running accordingly
875867 * @worker: self
876868 * @flags: flags to set
877− * @wakeup: wakeup an idle worker if necessary
878869 *
879− * Set @flags in @worker->flags and adjust nr_running accordingly. If
880− * nr_running becomes zero and @wakeup is %true, an idle worker is
881− * woken up.
870+ * Set @flags in @worker->flags and adjust nr_running accordingly.
882871 *
883872 * CONTEXT:
884873 * spin_lock_irq(pool->lock)
885874 */
886−static inline void worker_set_flags(struct worker *worker, unsigned int flags,
887− bool wakeup)
875+static inline void worker_set_flags(struct worker *worker, unsigned int flags)
888876 {
889877 struct worker_pool *pool = worker->pool;
890878
891879 WARN_ON_ONCE(worker->task != current);
892880
893− /*
894− * If transitioning into NOT_RUNNING, adjust nr_running and
895− * wake up an idle worker as necessary if requested by
896− * @wakeup.
897− */
881+ /* If transitioning into NOT_RUNNING, adjust nr_running. */
898882 if ((flags & WORKER_NOT_RUNNING) &&
899883 !(worker->flags & WORKER_NOT_RUNNING)) {
900− if (wakeup) {
901− if (atomic_dec_and_test(&pool->nr_running) &&
902− !list_empty(&pool->worklist))
903− wake_up_worker(pool);
904− } else
905− atomic_dec(&pool->nr_running);
884+ atomic_dec(&pool->nr_running);
906885 }
907886
908887 worker->flags |= flags;
@@ -1232,7 +1211,7 @@ static int try_to_grab_pending(struct work_struct *work, bool is_dwork, · 323 unmodified lines
12321211 pwq_activate_delayed_work(work);
12331212
12341213 list_del_init(&work->entry);
1235− pwq_dec_nr_in_flight(get_work_pwq(work), get_work_color(work));
1214+ pwq_dec_nr_in_flight(pwq, get_work_color(work));
12361215
12371216 /* work->data points to pwq iff queued, point to pool */
12381217 set_work_pool_and_keep_pending(work, pool->id);
@@ -1560,7 +1539,7 @@ static void worker_enter_idle(struct worker *worker) · 321 unmodified lines
15601539 (worker->hentry.next || worker->hentry.pprev)))
15611540 return;
15621541
1563− /* can't use worker_set_flags(), also called from start_worker() */
1542+ /* can't use worker_set_flags(), also called from create_worker() */
15641543 worker->flags |= WORKER_IDLE;
15651544 pool->nr_idle++;
15661545 worker->last_active = jiffies;
@@ -1602,11 +1581,11 @@ static void worker_leave_idle(struct worker *worker) · 35 unmodified lines
16021581 list_del_init(&worker->entry);
16031582 }
16041583
1605−static struct worker *alloc_worker(void)
1584+static struct worker *alloc_worker(int node)
16061585 {
16071586 struct worker *worker;
16081587
1609− worker = kzalloc(sizeof(*worker), GFP_KERNEL);
1588+ worker = kzalloc_node(sizeof(*worker), GFP_KERNEL, node);
16101589 if (worker) {
16111590 INIT_LIST_HEAD(&worker->entry);
16121591 INIT_LIST_HEAD(&worker->scheduled);
@@ -1670,6 +1649,9 @@ static void worker_detach_from_pool(struct worker *worker, · 57 unmodified lines
16701649 detach_completion = pool->detach_completion;
16711650 mutex_unlock(&pool->attach_mutex);
16721651
1652+ /* clear leftover flags without pool->lock after it is detached */
1653+ worker->flags &= ~(WORKER_UNBOUND | WORKER_REBOUND);
1654+
16731655 if (detach_completion)
16741656 complete(detach_completion);
16751657 }
@@ -1678,8 +1660,7 @@ static void worker_detach_from_pool(struct worker *worker, · 2 unmodified lines
16781660 * create_worker - create a new workqueue worker
16791661 * @pool: pool the new worker will belong to
16801662 *
1681− * Create a new worker which is attached to @pool. The new worker must be
1682− * started by start_worker().
1663+ * Create and start a new worker which is attached to @pool.
16831664 *
16841665 * CONTEXT:
16851666 * Might sleep. Does GFP_KERNEL allocations.
@@ -1698,7 +1679,7 @@ static struct worker *create_worker(struct worker_pool *pool) · 12 unmodified lines
16981679 if (id < 0)
16991680 goto fail;
17001681
1701− worker = alloc_worker();
1682+ worker = alloc_worker(pool->node);
17021683 if (!worker)
17031684 goto fail;
17041685
@@ -1724,6 +1705,13 @@ static struct worker *create_worker(struct worker_pool *pool) · 19 unmodified lines
17241705 /* successful, attach the worker to the pool */
17251706 worker_attach_to_pool(worker, pool);
17261707
1708+ /* start the newly created worker */
1709+ spin_lock_irq(&pool->lock);
1710+ worker->pool->nr_workers++;
1711+ worker_enter_idle(worker);
1712+ wake_up_process(worker->task);
1713+ spin_unlock_irq(&pool->lock);
1714+
17271715 return worker;
17281716
17291717 fail:
@@ -1733,44 +1721,6 @@ fail: · 3 unmodified lines
17331721 return NULL;
17341722 }
17351723
1736−/**
1737− * start_worker - start a newly created worker
1738− * @worker: worker to start
1739− *
1740− * Make the pool aware of @worker and start it.
1741− *
1742− * CONTEXT:
1743− * spin_lock_irq(pool->lock).
1744− */
1745−static void start_worker(struct worker *worker)
1746−{
1747− worker->pool->nr_workers++;
1748− worker_enter_idle(worker);
1749− wake_up_process(worker->task);
1750−}
1751−
1752−/**
1753− * create_and_start_worker - create and start a worker for a pool
1754− * @pool: the target pool
1755− *
1756− * Grab the managership of @pool and create and start a new worker for it.
1757− *
1758− * Return: 0 on success. A negative error code otherwise.
1759− */
1760−static int create_and_start_worker(struct worker_pool *pool)
1761−{
1762− struct worker *worker;
1763−
1764− worker = create_worker(pool);
1765− if (worker) {
1766− spin_lock_irq(&pool->lock);
1767− start_worker(worker);
1768− spin_unlock_irq(&pool->lock);
1769− }
1770−
1771− return worker ? 0 : -ENOMEM;
1772−}
1773−
17741724 /**
17751725 * destroy_worker - destroy a workqueue worker
17761726 * @worker: worker to be destroyed
@@ -1909,23 +1859,10 @@ restart: · 132 unmodified lines
19091859 mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
19101860
19111861 while (true) {
1912− struct worker *worker;
1913−
1914− worker = create_worker(pool);
1915− if (worker) {
1916− del_timer_sync(&pool->mayday_timer);
1917− spin_lock_irq(&pool->lock);
1918− start_worker(worker);
1919− if (WARN_ON_ONCE(need_to_create_worker(pool)))
1920− goto restart;
1921− return true;
1922− }
1923−
1924− if (!need_to_create_worker(pool))
1862+ if (create_worker(pool) || !need_to_create_worker(pool))
19251863 break;
19261864
1927− __set_current_state(TASK_INTERRUPTIBLE);
1928− schedule_timeout(CREATE_COOLDOWN);
1865+ schedule_timeout_interruptible(CREATE_COOLDOWN);
19291866
19301867 if (!need_to_create_worker(pool))
19311868 break;
@@ -1933,6 +1870,11 @@ restart: · 1 unmodified line
19331870
19341871 del_timer_sync(&pool->mayday_timer);
19351872 spin_lock_irq(&pool->lock);
1873+ /*
1874+ * This is necessary even after a new worker was just successfully
1875+ * created as @pool->lock was dropped and the new worker might have
1876+ * already become busy.
1877+ */
19361878 if (need_to_create_worker(pool))
19371879 goto restart;
19381880 return true;
@@ -2020,13 +1962,7 @@ __acquires(&pool->lock) · 81 unmodified lines
20201962
20211963 lockdep_copy_map(&lockdep_map, &work->lockdep_map);
20221964 #endif
2023− /*
2024− * Ensure we're on the correct CPU. DISASSOCIATED test is
2025− * necessary to avoid spurious warnings from rescuers servicing the
2026− * unbound or a disassociated pool.
2027− */
2028− WARN_ON_ONCE(!(worker->flags & WORKER_UNBOUND) &&
2029− !(pool->flags & POOL_DISASSOCIATED) &&
1965+ WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
20301966 raw_smp_processor_id() != pool->cpu);
20311967
20321968 /*
@@ -2052,17 +1988,22 @@ __acquires(&pool->lock) · 19 unmodified lines
20521988 list_del_init(&work->entry);
20531989
20541990 /*
2055− * CPU intensive works don't participate in concurrency
2056− * management. They're the scheduler's responsibility.
1991+ * CPU intensive works don't participate in concurrency management.
1992+ * They're the scheduler's responsibility. This takes @worker out
1993+ * of concurrency management and the next code block will chain
1994+ * execution of the pending work items.
20571995 */
20581996 if (unlikely(cpu_intensive))
2059− worker_set_flags(worker, WORKER_CPU_INTENSIVE, true);
1997+ worker_set_flags(worker, WORKER_CPU_INTENSIVE);
20601998
20611999 /*
2062− * Unbound pool isn't concurrency managed and work items should be
2063− * executed ASAP. Wake up another worker if necessary.
2000+ * Wake up another worker if necessary. The condition is always
2001+ * false for normal per-cpu workers since nr_running would always
2002+ * be >= 1 at this point. This is used to chain execution of the
2003+ * pending work items for WORKER_NOT_RUNNING workers such as the
2004+ * UNBOUND and CPU_INTENSIVE ones.
20642005 */
2065− if ((worker->flags & WORKER_UNBOUND) && need_more_worker(pool))
2006+ if (need_more_worker(pool))
20662007 wake_up_worker(pool);
20672008
20682009 /*
@@ -2218,7 +2159,7 @@ recheck: · 149 unmodified lines
22182159 }
22192160 } while (keep_working(pool));
22202161
2221− worker_set_flags(worker, WORKER_PREP, false);
2162+ worker_set_flags(worker, WORKER_PREP);
22222163 sleep:
22232164 /*
22242165 * pool->lock is held and there's no work to process and no need to
@@ -2311,29 +2252,27 @@ repeat: · 86 unmodified lines
23112252 move_linked_works(work, scheduled, &n);
23122253
23132254 process_scheduled_works(rescuer);
2314− spin_unlock_irq(&pool->lock);
2315−
2316− worker_detach_from_pool(rescuer, pool);
2317−
2318− spin_lock_irq(&pool->lock);
23192255
23202256 /*
23212257 * Put the reference grabbed by send_mayday(). @pool won't
2322− * go away while we're holding its lock.
2258+ * go away while we're still attached to it.
23232259 */
23242260 put_pwq(pwq);
23252261
23262262 /*
2327− * Leave this pool. If keep_working() is %true, notify a
2263+ * Leave this pool. If need_more_worker() is %true, notify a
23282264 * regular worker; otherwise, we end up with 0 concurrency
23292265 * and stalling the execution.
23302266 */
2331− if (keep_working(pool))
2267+ if (need_more_worker(pool))
23322268 wake_up_worker(pool);
23332269
23342270 rescuer->pool = NULL;
2335− spin_unlock(&pool->lock);
2336− spin_lock(&wq_mayday_lock);
2271+ spin_unlock_irq(&pool->lock);
2272+
2273+ worker_detach_from_pool(rescuer, pool);
2274+
2275+ spin_lock_irq(&wq_mayday_lock);
23372276 }
23382277
23392278 spin_unlock_irq(&wq_mayday_lock);
@@ -3458,7 +3397,7 @@ static void put_unbound_pool(struct worker_pool *pool) · 1118 unmodified lines
34583397 return;
34593398
34603399 /* sanity checks */
3461− if (WARN_ON(!(pool->flags & POOL_DISASSOCIATED)) ||
3400+ if (WARN_ON(!(pool->cpu < 0)) ||
34623401 WARN_ON(!list_empty(&pool->worklist)))
34633402 return;
34643403
@@ -3524,7 +3463,7 @@ static struct worker_pool *get_unbound_pool(const struct workqueue_attrs *attrs) · 59 unmodified lines
35243463 hash_for_each_possible(unbound_pool_hash, pool, hash_node, hash) {
35253464 if (wqattrs_equal(pool->attrs, attrs)) {
35263465 pool->refcnt++;
3527− goto out_unlock;
3466+ return pool;
35283467 }
35293468 }
35303469
@@ -3557,12 +3496,12 @@ static struct worker_pool *get_unbound_pool(const struct workqueue_attrs *attrs) · 26 unmodified lines
35573496 goto fail;
35583497
35593498 /* create and start the initial worker */
3560− if (create_and_start_worker(pool) < 0)
3499+ if (!create_worker(pool))
35613500 goto fail;
35623501
35633502 /* install */
35643503 hash_add(unbound_pool_hash, &pool->hash_node, hash);
3565−out_unlock:
3504+
35663505 return pool;
35673506 fail:
35683507 if (pool)
@@ -3591,11 +3530,6 @@ static void pwq_unbound_release_workfn(struct work_struct *work) · 22 unmodified lines
35913530 if (WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND)))
35923531 return;
35933532
3594− /*
3595− * Unlink @pwq. Synchronization against wq->mutex isn't strictly
3596− * necessary on release but do it anyway. It's easier to verify
3597− * and consistent with the linking path.
3598− */
35993533 mutex_lock(&wq->mutex);
36003534 list_del_rcu(&pwq->pwqs_node);
36013535 is_last = list_empty(&wq->pwqs);
@@ -3692,10 +3626,7 @@ static void link_pwq(struct pool_workqueue *pwq) · 90 unmodified lines
36923626 if (!list_empty(&pwq->pwqs_node))
36933627 return;
36943628
3695− /*
3696− * Set the matching work_color. This is synchronized with
3697− * wq->mutex to avoid confusing flush_workqueue().
3698− */
3629+ /* set the matching work_color */
36993630 pwq->work_color = wq->work_color;
37003631
37013632 /* sync max_active to the current setting */
@@ -3832,7 +3763,7 @@ int apply_workqueue_attrs(struct workqueue_struct *wq, · 130 unmodified lines
38323763 if (WARN_ON((wq->flags & __WQ_ORDERED) && !list_empty(&wq->pwqs)))
38333764 return -EINVAL;
38343765
3835− pwq_tbl = kzalloc(wq_numa_tbl_len * sizeof(pwq_tbl[0]), GFP_KERNEL);
3766+ pwq_tbl = kzalloc(nr_node_ids * sizeof(pwq_tbl[0]), GFP_KERNEL);
38363767 new_attrs = alloc_workqueue_attrs(GFP_KERNEL);
38373768 tmp_attrs = alloc_workqueue_attrs(GFP_KERNEL);
38383769 if (!pwq_tbl || !new_attrs || !tmp_attrs)
@@ -4080,7 +4011,7 @@ struct workqueue_struct *__alloc_workqueue_key(const char *fmt, · 241 unmodified lines
40804011
40814012 /* allocate wq and format name */
40824013 if (flags & WQ_UNBOUND)
4083− tbl_size = wq_numa_tbl_len * sizeof(wq->numa_pwq_tbl[0]);
4014+ tbl_size = nr_node_ids * sizeof(wq->numa_pwq_tbl[0]);
40844015
40854016 wq = kzalloc(sizeof(*wq) + tbl_size, GFP_KERNEL);
40864017 if (!wq)
@@ -4122,7 +4053,7 @@ struct workqueue_struct *__alloc_workqueue_key(const char *fmt, · 35 unmodified lines
41224053 if (flags & WQ_MEM_RECLAIM) {
41234054 struct worker *rescuer;
41244055
4125− rescuer = alloc_worker();
4056+ rescuer = alloc_worker(NUMA_NO_NODE);
41264057 if (!rescuer)
41274058 goto err_destroy;
41284059
@@ -4470,8 +4401,6 @@ static void wq_unbind_fn(struct work_struct *work) · 341 unmodified lines
44704401 struct worker *worker;
44714402
44724403 for_each_cpu_worker_pool(pool, cpu) {
4473− WARN_ON_ONCE(cpu != smp_processor_id());
4474−
44754404 mutex_lock(&pool->attach_mutex);
44764405 spin_lock_irq(&pool->lock);
44774406
@@ -4543,6 +4472,7 @@ static void rebind_workers(struct worker_pool *pool) · 65 unmodified lines
45434472 pool->attrs->cpumask) < 0);
45444473
45454474 spin_lock_irq(&pool->lock);
4475+ pool->flags &= ~POOL_DISASSOCIATED;
45464476
45474477 for_each_pool_worker(worker, pool) {
45484478 unsigned int worker_flags = worker->flags;
@@ -4632,7 +4562,7 @@ static int workqueue_cpu_up_callback(struct notifier_block *nfb, · 83 unmodified lines
46324562 for_each_cpu_worker_pool(pool, cpu) {
46334563 if (pool->nr_workers)
46344564 continue;
4635− if (create_and_start_worker(pool) < 0)
4565+ if (!create_worker(pool))
46364566 return NOTIFY_BAD;
46374567 }
46384568 break;
@@ -4645,10 +4575,6 @@ static int workqueue_cpu_up_callback(struct notifier_block *nfb, · 6 unmodified lines
46454575 mutex_lock(&pool->attach_mutex);
46464576
46474577 if (pool->cpu == cpu) {
4648− spin_lock_irq(&pool->lock);
4649− pool->flags &= ~POOL_DISASSOCIATED;
4650− spin_unlock_irq(&pool->lock);
4651−
46524578 rebind_workers(pool);
46534579 } else if (pool->cpu < 0) {
46544580 restore_unbound_workers_cpumask(pool, cpu);
@@ -4856,10 +4782,6 @@ static void __init wq_numa_init(void) · 201 unmodified lines
48564782 cpumask_var_t *tbl;
48574783 int node, cpu;
48584784
4859− /* determine NUMA pwq table len - highest node id + 1 */
4860− for_each_node(node)
4861− wq_numa_tbl_len = max(wq_numa_tbl_len, node + 1);
4862−
48634785 if (num_possible_nodes() <= 1)
48644786 return;
48654787
@@ -4876,7 +4798,7 @@ static void __init wq_numa_init(void) · 10 unmodified lines
48764798 * available. Build one from cpu_to_node() which should have been
48774799 * fully initialized by now.
48784800 */
4879− tbl = kzalloc(wq_numa_tbl_len * sizeof(tbl[0]), GFP_KERNEL);
4801+ tbl = kzalloc(nr_node_ids * sizeof(tbl[0]), GFP_KERNEL);
48804802 BUG_ON(!tbl);
48814803
48824804 for_each_node(node)
@@ -4936,7 +4858,7 @@ static int __init init_workqueues(void) · 53 unmodified lines
49364858
49374859 for_each_cpu_worker_pool(pool, cpu) {
49384860 pool->flags &= ~POOL_DISASSOCIATED;
4939− BUG_ON(create_and_start_worker(pool) < 0);
4861+ BUG_ON(!create_worker(pool));
49404862 }
49414863 }
49424864