History

Merge branch 'for-3.16' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq

Pull workqueue updates from Tejun Heo: "Lai simplified worker destruction path and internal workqueue locking and there are some other minor changes. Except for the removal of some long-deprecated interfaces which haven't had any in-kernel user for quite a while, there shouldn't be any difference to workqueue users" * 'for-3.16' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq: kernel/workqueue.c: pr_warning/pr_warn & printk/pr_info workqueue: remove the confusing POOL_FREEZING workqueue: rename first_worker() to first_idle_worker() workqueue: remove unused work_clear_pending() workqueue: remove unused WORK_CPU_END workqueue: declare system_highpri_wq workqueue: use generic attach/detach routine for rescuers workqueue: separate pool-attaching code out from create_worker() workqueue: rename manager_mutex to attach_mutex workqueue: narrow the protection range of manager_mutex workqueue: convert worker_idr to worker_ida workqueue: separate iteration role from worker_idr workqueue: destroy worker directly in the idle timeout handler workqueue: async worker destruction workqueue: destroy_worker() should destroy idle workers only workqueue: use manager lock only to protect worker_idr workqueue: Remove deprecated system_nrt[_freezable]_wq workqueue: Remove deprecated flush[_delayed]_work_sync() kernel/workqueue.c: pr_warning/pr_warn & printk/pr_info workqueue: simplify wq_update_unbound_numa() by jumping to use_dfl_pwq if the target cpumask equals wq's

Linus Torvalds committed 12 years ago

Commit da85d192 parents 68a29ef 015af063 files · +154 −336Browse files

Showing the changes to kernel/workqueue.c (1 of 3 files). Show all

Unified · Split

kernel/workqueue.c+147−301
@@ -65,15 +65,12 @@ enum { · 64 unmodified lines
6565 * be executing on any CPU. The pool behaves as an unbound one.
6666 *
6767 * Note that DISASSOCIATED should be flipped only while holding
68− * manager_mutex to avoid changing binding state while
69− * create_worker() is in progress.
68+ * attach_mutex to avoid changing binding state while
69+ * worker_attach_to_pool() is in progress.
7070 */
71− POOL_MANAGE_WORKERS = 1 << 0, /* need to manage workers */
7271 POOL_DISASSOCIATED = 1 << 2, /* cpu can't serve workers */
73− POOL_FREEZING = 1 << 3, /* freeze in progress */
7472
7573 /* worker flags */
76− WORKER_STARTED = 1 << 0, /* started */
7774 WORKER_DIE = 1 << 1, /* die die die */
7875 WORKER_IDLE = 1 << 2, /* is idle */
7976 WORKER_PREP = 1 << 3, /* preparing to run works */
@@ -124,8 +121,7 @@ enum { · 44 unmodified lines
124121 * cpu or grabbing pool->lock is enough for read access. If
125122 * POOL_DISASSOCIATED is set, it's identical to L.
126123 *
127− * MG: pool->manager_mutex and pool->lock protected. Writes require both
128− * locks. Reads can happen under either lock.
124+ * A: pool->attach_mutex protected.
129125 *
130126 * PL: wq_pool_mutex protected.
131127 *
@@ -163,8 +159,11 @@ struct worker_pool { · 31 unmodified lines
163159
164160 /* see manage_workers() for details on the two manager mutexes */
165161 struct mutex manager_arb; /* manager arbitration */
166− struct mutex manager_mutex; /* manager exclusion */
167− struct idr worker_idr; /* MG: worker IDs and iteration */
162+ struct mutex attach_mutex; /* attach/detach exclusion */
163+ struct list_head workers; /* A: attached workers */
164+ struct completion *detach_completion; /* all workers detached */
165+
166+ struct ida worker_ida; /* worker IDs for task name */
168167
169168 struct workqueue_attrs *attrs; /* I: worker attributes */
170169 struct hlist_node hash_node; /* PL: unbound_pool_hash node */
@@ -340,16 +339,6 @@ static void copy_workqueue_attrs(struct workqueue_attrs *to, · 169 unmodified lines
340339 lockdep_is_held(&wq->mutex), \
341340 "sched RCU or wq->mutex should be held")
342341
343−#ifdef CONFIG_LOCKDEP
344−#define assert_manager_or_pool_lock(pool) \
345− WARN_ONCE(debug_locks && \
346− !lockdep_is_held(&(pool)->manager_mutex) && \
347− !lockdep_is_held(&(pool)->lock), \
348− "pool->manager_mutex or ->lock should be held")
349−#else
350−#define assert_manager_or_pool_lock(pool) do { } while (0)
351−#endif
352−
353342 #define for_each_cpu_worker_pool(pool, cpu) \
354343 for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0]; \
355344 (pool) < &per_cpu(cpu_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
@@ -375,17 +364,16 @@ static void copy_workqueue_attrs(struct workqueue_attrs *to, · 19 unmodified lines
375364 /**
376365 * for_each_pool_worker - iterate through all workers of a worker_pool
377366 * @worker: iteration cursor
378− * @wi: integer used for iteration
379367 * @pool: worker_pool to iterate workers of
380368 *
381− * This must be called with either @pool->manager_mutex or ->lock held.
369+ * This must be called with @pool->attach_mutex.
382370 *
383371 * The if/else clause exists only for the lockdep assertion and can be
384372 * ignored.
385373 */
386−#define for_each_pool_worker(worker, wi, pool) \
387− idr_for_each_entry(&(pool)->worker_idr, (worker), (wi)) \
388− if (({ assert_manager_or_pool_lock((pool)); false; })) { } \
374+#define for_each_pool_worker(worker, pool) \
375+ list_for_each_entry((worker), &(pool)->workers, node) \
376+ if (({ lockdep_assert_held(&pool->attach_mutex); false; })) { } \
389377 else
390378
391379 /**
@@ -763,13 +751,6 @@ static bool need_to_create_worker(struct worker_pool *pool) · 371 unmodified lines
763751 return need_more_worker(pool) && !may_start_working(pool);
764752 }
765753
766−/* Do I need to be the manager? */
767−static bool need_to_manage_workers(struct worker_pool *pool)
768−{
769− return need_to_create_worker(pool) ||
770− (pool->flags & POOL_MANAGE_WORKERS);
771−}
772−
773754 /* Do we have too many workers and should some go away? */
774755 static bool too_many_workers(struct worker_pool *pool)
775756 {
@@ -791,8 +772,8 @@ static bool too_many_workers(struct worker_pool *pool) · 15 unmodified lines
791772 * Wake up functions.
792773 */
793774
794−/* Return the first worker. Safe with preemption disabled */
795−static struct worker *first_worker(struct worker_pool *pool)
775+/* Return the first idle worker. Safe with preemption disabled */
776+static struct worker *first_idle_worker(struct worker_pool *pool)
796777 {
797778 if (unlikely(list_empty(&pool->idle_list)))
798779 return NULL;
@@ -811,7 +792,7 @@ static struct worker *first_worker(struct worker_pool *pool) · 12 unmodified lines
811792 */
812793 static void wake_up_worker(struct worker_pool *pool)
813794 {
814− struct worker *worker = first_worker(pool);
795+ struct worker *worker = first_idle_worker(pool);
815796
816797 if (likely(worker))
817798 wake_up_process(worker->task);
@@ -885,7 +866,7 @@ struct task_struct *wq_worker_sleeping(struct task_struct *task, int cpu) · 67 unmodified lines
885866 */
886867 if (atomic_dec_and_test(&pool->nr_running) &&
887868 !list_empty(&pool->worklist))
888− to_wakeup = first_worker(pool);
869+ to_wakeup = first_idle_worker(pool);
889870 return to_wakeup ? to_wakeup->task : NULL;
890871 }
891872
@@ -1621,70 +1602,6 @@ static void worker_leave_idle(struct worker *worker) · 729 unmodified lines
16211602 list_del_init(&worker->entry);
16221603 }
16231604
1624−/**
1625− * worker_maybe_bind_and_lock - try to bind %current to worker_pool and lock it
1626− * @pool: target worker_pool
1627− *
1628− * Bind %current to the cpu of @pool if it is associated and lock @pool.
1629− *
1630− * Works which are scheduled while the cpu is online must at least be
1631− * scheduled to a worker which is bound to the cpu so that if they are
1632− * flushed from cpu callbacks while cpu is going down, they are
1633− * guaranteed to execute on the cpu.
1634− *
1635− * This function is to be used by unbound workers and rescuers to bind
1636− * themselves to the target cpu and may race with cpu going down or
1637− * coming online. kthread_bind() can't be used because it may put the
1638− * worker to already dead cpu and set_cpus_allowed_ptr() can't be used
1639− * verbatim as it's best effort and blocking and pool may be
1640− * [dis]associated in the meantime.
1641− *
1642− * This function tries set_cpus_allowed() and locks pool and verifies the
1643− * binding against %POOL_DISASSOCIATED which is set during
1644− * %CPU_DOWN_PREPARE and cleared during %CPU_ONLINE, so if the worker
1645− * enters idle state or fetches works without dropping lock, it can
1646− * guarantee the scheduling requirement described in the first paragraph.
1647− *
1648− * CONTEXT:
1649− * Might sleep. Called without any lock but returns with pool->lock
1650− * held.
1651− *
1652− * Return:
1653− * %true if the associated pool is online (@worker is successfully
1654− * bound), %false if offline.
1655− */
1656−static bool worker_maybe_bind_and_lock(struct worker_pool *pool)
1657−__acquires(&pool->lock)
1658−{
1659− while (true) {
1660− /*
1661− * The following call may fail, succeed or succeed
1662− * without actually migrating the task to the cpu if
1663− * it races with cpu hotunplug operation. Verify
1664− * against POOL_DISASSOCIATED.
1665− */
1666− if (!(pool->flags & POOL_DISASSOCIATED))
1667− set_cpus_allowed_ptr(current, pool->attrs->cpumask);
1668−
1669− spin_lock_irq(&pool->lock);
1670− if (pool->flags & POOL_DISASSOCIATED)
1671− return false;
1672− if (task_cpu(current) == pool->cpu &&
1673− cpumask_equal(&current->cpus_allowed, pool->attrs->cpumask))
1674− return true;
1675− spin_unlock_irq(&pool->lock);
1676−
1677− /*
1678− * We've raced with CPU hot[un]plug. Give it a breather
1679− * and retry migration. cond_resched() is required here;
1680− * otherwise, we might deadlock against cpu_stop trying to
1681− * bring down the CPU on non-preemptive kernel.
1682− */
1683− cpu_relax();
1684− cond_resched();
1685− }
1686−}
1687−
16881605 static struct worker *alloc_worker(void)
16891606 {
16901607 struct worker *worker;
@@ -1693,19 +1610,76 @@ static struct worker *alloc_worker(void) · 2 unmodified lines
16931610 if (worker) {
16941611 INIT_LIST_HEAD(&worker->entry);
16951612 INIT_LIST_HEAD(&worker->scheduled);
1613+ INIT_LIST_HEAD(&worker->node);
16961614 /* on creation a worker is in !idle && prep state */
16971615 worker->flags = WORKER_PREP;
16981616 }
16991617 return worker;
17001618 }
17011619
1620+/**
1621+ * worker_attach_to_pool() - attach a worker to a pool
1622+ * @worker: worker to be attached
1623+ * @pool: the target pool
1624+ *
1625+ * Attach @worker to @pool. Once attached, the %WORKER_UNBOUND flag and
1626+ * cpu-binding of @worker are kept coordinated with the pool across
1627+ * cpu-[un]hotplugs.
1628+ */
1629+static void worker_attach_to_pool(struct worker *worker,
1630+ struct worker_pool *pool)
1631+{
1632+ mutex_lock(&pool->attach_mutex);
1633+
1634+ /*
1635+ * set_cpus_allowed_ptr() will fail if the cpumask doesn't have any
1636+ * online CPUs. It'll be re-applied when any of the CPUs come up.
1637+ */
1638+ set_cpus_allowed_ptr(worker->task, pool->attrs->cpumask);
1639+
1640+ /*
1641+ * The pool->attach_mutex ensures %POOL_DISASSOCIATED remains
1642+ * stable across this function. See the comments above the
1643+ * flag definition for details.
1644+ */
1645+ if (pool->flags & POOL_DISASSOCIATED)
1646+ worker->flags |= WORKER_UNBOUND;
1647+
1648+ list_add_tail(&worker->node, &pool->workers);
1649+
1650+ mutex_unlock(&pool->attach_mutex);
1651+}
1652+
1653+/**
1654+ * worker_detach_from_pool() - detach a worker from its pool
1655+ * @worker: worker which is attached to its pool
1656+ * @pool: the pool @worker is attached to
1657+ *
1658+ * Undo the attaching which had been done in worker_attach_to_pool(). The
1659+ * caller worker shouldn't access to the pool after detached except it has
1660+ * other reference to the pool.
1661+ */
1662+static void worker_detach_from_pool(struct worker *worker,
1663+ struct worker_pool *pool)
1664+{
1665+ struct completion *detach_completion = NULL;
1666+
1667+ mutex_lock(&pool->attach_mutex);
1668+ list_del(&worker->node);
1669+ if (list_empty(&pool->workers))
1670+ detach_completion = pool->detach_completion;
1671+ mutex_unlock(&pool->attach_mutex);
1672+
1673+ if (detach_completion)
1674+ complete(detach_completion);
1675+}
1676+
17021677 /**
17031678 * create_worker - create a new workqueue worker
17041679 * @pool: pool the new worker will belong to
17051680 *
1706− * Create a new worker which is bound to @pool. The returned worker
1707− * can be started by calling start_worker() or destroyed using
1708− * destroy_worker().
1681+ * Create a new worker which is attached to @pool. The new worker must be
1682+ * started by start_worker().
17091683 *
17101684 * CONTEXT:
17111685 * Might sleep. Does GFP_KERNEL allocations.
@@ -1719,19 +1693,8 @@ static struct worker *create_worker(struct worker_pool *pool) · 7 unmodified lines
17191693 int id = -1;
17201694 char id_buf[16];
17211695
1722− lockdep_assert_held(&pool->manager_mutex);
1723−
1724− /*
1725− * ID is needed to determine kthread name. Allocate ID first
1726− * without installing the pointer.
1727− */
1728− idr_preload(GFP_KERNEL);
1729− spin_lock_irq(&pool->lock);
1730−
1731− id = idr_alloc(&pool->worker_idr, NULL, 0, 0, GFP_NOWAIT);
1732−
1733− spin_unlock_irq(&pool->lock);
1734− idr_preload_end();
1696+ /* ID is needed to determine kthread name */
1697+ id = ida_simple_get(&pool->worker_ida, 0, 0, GFP_KERNEL);
17351698 if (id < 0)
17361699 goto fail;
17371700
@@ -1758,33 +1721,14 @@ static struct worker *create_worker(struct worker_pool *pool) · 20 unmodified lines
17581721 /* prevent userland from meddling with cpumask of workqueue workers */
17591722 worker->task->flags |= PF_NO_SETAFFINITY;
17601723
1761− /*
1762− * set_cpus_allowed_ptr() will fail if the cpumask doesn't have any
1763− * online CPUs. It'll be re-applied when any of the CPUs come up.
1764− */
1765− set_cpus_allowed_ptr(worker->task, pool->attrs->cpumask);
1766−
1767− /*
1768− * The caller is responsible for ensuring %POOL_DISASSOCIATED
1769− * remains stable across this function. See the comments above the
1770− * flag definition for details.
1771− */
1772− if (pool->flags & POOL_DISASSOCIATED)
1773− worker->flags |= WORKER_UNBOUND;
1774−
1775− /* successful, commit the pointer to idr */
1776− spin_lock_irq(&pool->lock);
1777− idr_replace(&pool->worker_idr, worker, worker->id);
1778− spin_unlock_irq(&pool->lock);
1724+ /* successful, attach the worker to the pool */
1725+ worker_attach_to_pool(worker, pool);
17791726
17801727 return worker;
17811728
17821729 fail:
1783− if (id >= 0) {
1784− spin_lock_irq(&pool->lock);
1785− idr_remove(&pool->worker_idr, id);
1786− spin_unlock_irq(&pool->lock);
1787− }
1730+ if (id >= 0)
1731+ ida_simple_remove(&pool->worker_ida, id);
17881732 kfree(worker);
17891733 return NULL;
17901734 }
@@ -1800,7 +1744,6 @@ fail: · 9 unmodified lines
18001744 */
18011745 static void start_worker(struct worker *worker)
18021746 {
1803− worker->flags |= WORKER_STARTED;
18041747 worker->pool->nr_workers++;
18051748 worker_enter_idle(worker);
18061749 wake_up_process(worker->task);
@@ -1818,8 +1761,6 @@ static int create_and_start_worker(struct worker_pool *pool) · 11 unmodified lines
18181761 {
18191762 struct worker *worker;
18201763
1821− mutex_lock(&pool->manager_mutex);
1822−
18231764 worker = create_worker(pool);
18241765 if (worker) {
18251766 spin_lock_irq(&pool->lock);
@@ -1827,8 +1768,6 @@ static int create_and_start_worker(struct worker_pool *pool) · 1 unmodified line
18271768 spin_unlock_irq(&pool->lock);
18281769 }
18291770
1830− mutex_unlock(&pool->manager_mutex);
1831−
18321771 return worker ? 0 : -ENOMEM;
18331772 }
18341773
@@ -1836,46 +1775,30 @@ static int create_and_start_worker(struct worker_pool *pool) · 1 unmodified line
18361775 * destroy_worker - destroy a workqueue worker
18371776 * @worker: worker to be destroyed
18381777 *
1839− * Destroy @worker and adjust @pool stats accordingly.
1778+ * Destroy @worker and adjust @pool stats accordingly. The worker should
1779+ * be idle.
18401780 *
18411781 * CONTEXT:
1842− * spin_lock_irq(pool->lock) which is released and regrabbed.
1782+ * spin_lock_irq(pool->lock).
18431783 */
18441784 static void destroy_worker(struct worker *worker)
18451785 {
18461786 struct worker_pool *pool = worker->pool;
18471787
1848− lockdep_assert_held(&pool->manager_mutex);
18491788 lockdep_assert_held(&pool->lock);
18501789
18511790 /* sanity check frenzy */
18521791 if (WARN_ON(worker->current_work) ||
1853− WARN_ON(!list_empty(&worker->scheduled)))
1792+ WARN_ON(!list_empty(&worker->scheduled)) ||
1793+ WARN_ON(!(worker->flags & WORKER_IDLE)))
18541794 return;
18551795
1856− if (worker->flags & WORKER_STARTED)
1857− pool->nr_workers--;
1858− if (worker->flags & WORKER_IDLE)
1859− pool->nr_idle--;
1860−
1861− /*
1862− * Once WORKER_DIE is set, the kworker may destroy itself at any
1863− * point. Pin to ensure the task stays until we're done with it.
1864− */
1865− get_task_struct(worker->task);
1796+ pool->nr_workers--;
1797+ pool->nr_idle--;
18661798
18671799 list_del_init(&worker->entry);
18681800 worker->flags |= WORKER_DIE;
1869−
1870− idr_remove(&pool->worker_idr, worker->id);
1871−
1872− spin_unlock_irq(&pool->lock);
1873−
1874− kthread_stop(worker->task);
1875− put_task_struct(worker->task);
1876− kfree(worker);
1877−
1878− spin_lock_irq(&pool->lock);
1801+ wake_up_process(worker->task);
18791802 }
18801803
18811804 static void idle_worker_timeout(unsigned long __pool)
@@ -1884,7 +1807,7 @@ static void idle_worker_timeout(unsigned long __pool) · 2 unmodified lines
18841807
18851808 spin_lock_irq(&pool->lock);
18861809
1887− if (too_many_workers(pool)) {
1810+ while (too_many_workers(pool)) {
18881811 struct worker *worker;
18891812 unsigned long expires;
18901813
@@ -1892,13 +1815,12 @@ static void idle_worker_timeout(unsigned long __pool) · 1 unmodified line
18921815 worker = list_entry(pool->idle_list.prev, struct worker, entry);
18931816 expires = worker->last_active + IDLE_WORKER_TIMEOUT;
18941817
1895− if (time_before(jiffies, expires))
1818+ if (time_before(jiffies, expires)) {
18961819 mod_timer(&pool->idle_timer, expires);
1897− else {
1898− /* it's been idle for too long, wake up manager */
1899− pool->flags |= POOL_MANAGE_WORKERS;
1900− wake_up_worker(pool);
1820+ break;
19011821 }
1822+
1823+ destroy_worker(worker);
19021824 }
19031825
19041826 spin_unlock_irq(&pool->lock);
@@ -2016,44 +1938,6 @@ restart: · 111 unmodified lines
20161938 return true;
20171939 }
20181940
2019−/**
2020− * maybe_destroy_worker - destroy workers which have been idle for a while
2021− * @pool: pool to destroy workers for
2022− *
2023− * Destroy @pool workers which have been idle for longer than
2024− * IDLE_WORKER_TIMEOUT.
2025− *
2026− * LOCKING:
2027− * spin_lock_irq(pool->lock) which may be released and regrabbed
2028− * multiple times. Called only from manager.
2029− *
2030− * Return:
2031− * %false if no action was taken and pool->lock stayed locked, %true
2032− * otherwise.
2033− */
2034−static bool maybe_destroy_workers(struct worker_pool *pool)
2035−{
2036− bool ret = false;
2037−
2038− while (too_many_workers(pool)) {
2039− struct worker *worker;
2040− unsigned long expires;
2041−
2042− worker = list_entry(pool->idle_list.prev, struct worker, entry);
2043− expires = worker->last_active + IDLE_WORKER_TIMEOUT;
2044−
2045− if (time_before(jiffies, expires)) {
2046− mod_timer(&pool->idle_timer, expires);
2047− break;
2048− }
2049−
2050− destroy_worker(worker);
2051− ret = true;
2052− }
2053−
2054− return ret;
2055−}
2056−
20571941 /**
20581942 * manage_workers - manage worker pool
20591943 * @worker: self
@@ -2083,8 +1967,6 @@ static bool manage_workers(struct worker *worker) · 23 unmodified lines
20831967 bool ret = false;
20841968
20851969 /*
2086− * Managership is governed by two mutexes - manager_arb and
2087− * manager_mutex. manager_arb handles arbitration of manager role.
20881970 * Anyone who successfully grabs manager_arb wins the arbitration
20891971 * and becomes the manager. mutex_trylock() on pool->manager_arb
20901972 * failure while holding pool->lock reliably indicates that someone
@@ -2093,40 +1975,12 @@ static bool manage_workers(struct worker *worker) · 2 unmodified lines
20931975 * grabbing manager_arb is responsible for actually performing
20941976 * manager duties. If manager_arb is grabbed and released without
20951977 * actual management, the pool may stall indefinitely.
2096− *
2097− * manager_mutex is used for exclusion of actual management
2098− * operations. The holder of manager_mutex can be sure that none
2099− * of management operations, including creation and destruction of
2100− * workers, won't take place until the mutex is released. Because
2101− * manager_mutex doesn't interfere with manager role arbitration,
2102− * it is guaranteed that the pool's management, while may be
2103− * delayed, won't be disturbed by someone else grabbing
2104− * manager_mutex.
21051978 */
21061979 if (!mutex_trylock(&pool->manager_arb))
21071980 return ret;
21081981
2109− /*
2110− * With manager arbitration won, manager_mutex would be free in
2111− * most cases. trylock first without dropping @pool->lock.
2112− */
2113− if (unlikely(!mutex_trylock(&pool->manager_mutex))) {
2114− spin_unlock_irq(&pool->lock);
2115− mutex_lock(&pool->manager_mutex);
2116− spin_lock_irq(&pool->lock);
2117− ret = true;
2118− }
2119−
2120− pool->flags &= ~POOL_MANAGE_WORKERS;
2121−
2122− /*
2123− * Destroy and then create so that may_start_working() is true
2124− * on return.
2125− */
2126− ret |= maybe_destroy_workers(pool);
21271982 ret |= maybe_create_worker(pool);
21281983
2129− mutex_unlock(&pool->manager_mutex);
21301984 mutex_unlock(&pool->manager_arb);
21311985 return ret;
21321986 }
@@ -2314,6 +2168,11 @@ woke_up: · 181 unmodified lines
23142168 spin_unlock_irq(&pool->lock);
23152169 WARN_ON_ONCE(!list_empty(&worker->entry));
23162170 worker->task->flags &= ~PF_WQ_WORKER;
2171+
2172+ set_task_comm(worker->task, "kworker/dying");
2173+ ida_simple_remove(&pool->worker_ida, worker->id);
2174+ worker_detach_from_pool(worker, pool);
2175+ kfree(worker);
23172176 return 0;
23182177 }
23192178
@@ -2361,9 +2220,6 @@ recheck: · 41 unmodified lines
23612220
23622221 worker_set_flags(worker, WORKER_PREP, false);
23632222 sleep:
2364− if (unlikely(need_to_manage_workers(pool)) && manage_workers(worker))
2365− goto recheck;
2366−
23672223 /*
23682224 * pool->lock is held and there's no work to process and no need to
23692225 * manage, sleep. Workers are woken up only while holding
@@ -2440,8 +2296,9 @@ repeat: · 70 unmodified lines
24402296
24412297 spin_unlock_irq(&wq_mayday_lock);
24422298
2443− /* migrate to the target cpu if possible */
2444− worker_maybe_bind_and_lock(pool);
2299+ worker_attach_to_pool(rescuer, pool);
2300+
2301+ spin_lock_irq(&pool->lock);
24452302 rescuer->pool = pool;
24462303
24472304 /*
@@ -2454,6 +2311,11 @@ repeat: · 6 unmodified lines
24542311 move_linked_works(work, scheduled, &n);
24552312
24562313 process_scheduled_works(rescuer);
2314+ spin_unlock_irq(&pool->lock);
2315+
2316+ worker_detach_from_pool(rescuer, pool);
2317+
2318+ spin_lock_irq(&pool->lock);
24572319
24582320 /*
24592321 * Put the reference grabbed by send_mayday(). @pool won't
@@ -3550,9 +3412,10 @@ static int init_worker_pool(struct worker_pool *pool) · 1090 unmodified lines
35503412 (unsigned long)pool);
35513413
35523414 mutex_init(&pool->manager_arb);
3553− mutex_init(&pool->manager_mutex);
3554− idr_init(&pool->worker_idr);
3415+ mutex_init(&pool->attach_mutex);
3416+ INIT_LIST_HEAD(&pool->workers);
35553417
3418+ ida_init(&pool->worker_ida);
35563419 INIT_HLIST_NODE(&pool->hash_node);
35573420 pool->refcnt = 1;
35583421
@@ -3567,7 +3430,7 @@ static void rcu_free_pool(struct rcu_head *rcu) · 8 unmodified lines
35673430 {
35683431 struct worker_pool *pool = container_of(rcu, struct worker_pool, rcu);
35693432
3570− idr_destroy(&pool->worker_idr);
3433+ ida_destroy(&pool->worker_ida);
35713434 free_workqueue_attrs(pool->attrs);
35723435 kfree(pool);
35733436 }
@@ -3585,6 +3448,7 @@ static void rcu_free_pool(struct rcu_head *rcu) · 11 unmodified lines
35853448 */
35863449 static void put_unbound_pool(struct worker_pool *pool)
35873450 {
3451+ DECLARE_COMPLETION_ONSTACK(detach_completion);
35883452 struct worker *worker;
35893453
35903454 lockdep_assert_held(&wq_pool_mutex);
@@ -3605,18 +3469,24 @@ static void put_unbound_pool(struct worker_pool *pool) · 14 unmodified lines
36053469 /*
36063470 * Become the manager and destroy all workers. Grabbing
36073471 * manager_arb prevents @pool's workers from blocking on
3608− * manager_mutex.
3472+ * attach_mutex.
36093473 */
36103474 mutex_lock(&pool->manager_arb);
3611− mutex_lock(&pool->manager_mutex);
3612− spin_lock_irq(&pool->lock);
36133475
3614− while ((worker = first_worker(pool)))
3476+ spin_lock_irq(&pool->lock);
3477+ while ((worker = first_idle_worker(pool)))
36153478 destroy_worker(worker);
36163479 WARN_ON(pool->nr_workers || pool->nr_idle);
3617−
36183480 spin_unlock_irq(&pool->lock);
3619− mutex_unlock(&pool->manager_mutex);
3481+
3482+ mutex_lock(&pool->attach_mutex);
3483+ if (!list_empty(&pool->workers))
3484+ pool->detach_completion = &detach_completion;
3485+ mutex_unlock(&pool->attach_mutex);
3486+
3487+ if (pool->detach_completion)
3488+ wait_for_completion(pool->detach_completion);
3489+
36203490 mutex_unlock(&pool->manager_arb);
36213491
36223492 /* shut down the timers */
@@ -3662,9 +3532,6 @@ static struct worker_pool *get_unbound_pool(const struct workqueue_attrs *attrs) · 39 unmodified lines
36623532 if (!pool || init_worker_pool(pool) < 0)
36633533 goto fail;
36643534
3665− if (workqueue_freezing)
3666− pool->flags |= POOL_FREEZING;
3667−
36683535 lockdep_set_subclass(&pool->lock, 1); /* see put_pwq() */
36693536 copy_workqueue_attrs(pool->attrs, attrs);
36703537
@@ -3771,7 +3638,12 @@ static void pwq_adjust_max_active(struct pool_workqueue *pwq) · 100 unmodified lines
37713638
37723639 spin_lock_irq(&pwq->pool->lock);
37733640
3774− if (!freezable || !(pwq->pool->flags & POOL_FREEZING)) {
3641+ /*
3642+ * During [un]freezing, the caller is responsible for ensuring that
3643+ * this function is called at least once after @workqueue_freezing
3644+ * is updated and visible.
3645+ */
3646+ if (!freezable || !workqueue_freezing) {
37753647 pwq->max_active = wq->saved_max_active;
37763648
37773649 while (!list_empty(&pwq->delayed_works) &&
@@ -4103,17 +3975,13 @@ static void wq_update_unbound_numa(struct workqueue_struct *wq, int cpu, · 325 unmodified lines
41033975 * Let's determine what needs to be done. If the target cpumask is
41043976 * different from wq's, we need to compare it to @pwq's and create
41053977 * a new one if they don't match. If the target cpumask equals
4106− * wq's, the default pwq should be used. If @pwq is already the
4107− * default one, nothing to do; otherwise, install the default one.
3978+ * wq's, the default pwq should be used.
41083979 */
41093980 if (wq_calc_node_cpumask(wq->unbound_attrs, node, cpu_off, cpumask)) {
41103981 if (cpumask_equal(cpumask, pwq->pool->attrs->cpumask))
41113982 goto out_unlock;
41123983 } else {
4113− if (pwq == wq->dfl_pwq)
4114− goto out_unlock;
4115− else
4116− goto use_dfl_pwq;
3984+ goto use_dfl_pwq;
41173985 }
41183986
41193987 mutex_unlock(&wq->mutex);
@@ -4121,8 +3989,8 @@ static void wq_update_unbound_numa(struct workqueue_struct *wq, int cpu, · 1 unmodified line
41213989 /* create a new pwq */
41223990 pwq = alloc_unbound_pwq(wq, target_attrs);
41233991 if (!pwq) {
4124− pr_warning("workqueue: allocation failed while updating NUMA affinity of \"%s\"\n",
4125− wq->name);
3992+ pr_warn("workqueue: allocation failed while updating NUMA affinity of \"%s\"\n",
3993+ wq->name);
41263994 mutex_lock(&wq->mutex);
41273995 goto use_dfl_pwq;
41283996 }
@@ -4599,28 +4467,27 @@ static void wq_unbind_fn(struct work_struct *work) · 470 unmodified lines
45994467 int cpu = smp_processor_id();
46004468 struct worker_pool *pool;
46014469 struct worker *worker;
4602− int wi;
46034470
46044471 for_each_cpu_worker_pool(pool, cpu) {
46054472 WARN_ON_ONCE(cpu != smp_processor_id());
46064473
4607− mutex_lock(&pool->manager_mutex);
4474+ mutex_lock(&pool->attach_mutex);
46084475 spin_lock_irq(&pool->lock);
46094476
46104477 /*
4611− * We've blocked all manager operations. Make all workers
4478+ * We've blocked all attach/detach operations. Make all workers
46124479 * unbound and set DISASSOCIATED. Before this, all workers
46134480 * except for the ones which are still executing works from
46144481 * before the last CPU down must be on the cpu. After
46154482 * this, they may become diasporas.
46164483 */
4617− for_each_pool_worker(worker, wi, pool)
4484+ for_each_pool_worker(worker, pool)
46184485 worker->flags |= WORKER_UNBOUND;
46194486
46204487 pool->flags |= POOL_DISASSOCIATED;
46214488
46224489 spin_unlock_irq(&pool->lock);
4623− mutex_unlock(&pool->manager_mutex);
4490+ mutex_unlock(&pool->attach_mutex);
46244491
46254492 /*
46264493 * Call schedule() so that we cross rq->lock and thus can
@@ -4660,9 +4527,8 @@ static void wq_unbind_fn(struct work_struct *work) · 33 unmodified lines
46604527 static void rebind_workers(struct worker_pool *pool)
46614528 {
46624529 struct worker *worker;
4663− int wi;
46644530
4665− lockdep_assert_held(&pool->manager_mutex);
4531+ lockdep_assert_held(&pool->attach_mutex);
46664532
46674533 /*
46684534 * Restore CPU affinity of all workers. As all idle workers should
@@ -4671,13 +4537,13 @@ static void rebind_workers(struct worker_pool *pool) · 2 unmodified lines
46714537 * of all workers first and then clear UNBOUND. As we're called
46724538 * from CPU_ONLINE, the following shouldn't fail.
46734539 */
4674− for_each_pool_worker(worker, wi, pool)
4540+ for_each_pool_worker(worker, pool)
46754541 WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
46764542 pool->attrs->cpumask) < 0);
46774543
46784544 spin_lock_irq(&pool->lock);
46794545
4680− for_each_pool_worker(worker, wi, pool) {
4546+ for_each_pool_worker(worker, pool) {
46814547 unsigned int worker_flags = worker->flags;
46824548
46834549 /*
@@ -4729,9 +4595,8 @@ static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu) · 45 unmodified lines
47294595 {
47304596 static cpumask_t cpumask;
47314597 struct worker *worker;
4732− int wi;
47334598
4734− lockdep_assert_held(&pool->manager_mutex);
4599+ lockdep_assert_held(&pool->attach_mutex);
47354600
47364601 /* is @cpu allowed for @pool? */
47374602 if (!cpumask_test_cpu(cpu, pool->attrs->cpumask))
@@ -4743,7 +4608,7 @@ static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu) · 5 unmodified lines
47434608 return;
47444609
47454610 /* as we're called from CPU_ONLINE, the following shouldn't fail */
4746− for_each_pool_worker(worker, wi, pool)
4611+ for_each_pool_worker(worker, pool)
47474612 WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
47484613 pool->attrs->cpumask) < 0);
47494614 }
@@ -4776,7 +4641,7 @@ static int workqueue_cpu_up_callback(struct notifier_block *nfb, · 26 unmodified lines
47764641 mutex_lock(&wq_pool_mutex);
47774642
47784643 for_each_pool(pool, pi) {
4779− mutex_lock(&pool->manager_mutex);
4644+ mutex_lock(&pool->attach_mutex);
47804645
47814646 if (pool->cpu == cpu) {
47824647 spin_lock_irq(&pool->lock);
@@ -4788,7 +4653,7 @@ static int workqueue_cpu_up_callback(struct notifier_block *nfb, · 5 unmodified lines
47884653 restore_unbound_workers_cpumask(pool, cpu);
47894654 }
47904655
4791− mutex_unlock(&pool->manager_mutex);
4656+ mutex_unlock(&pool->attach_mutex);
47924657 }
47934658
47944659 /* update NUMA affinity of unbound workqueues */
@@ -4887,24 +4752,14 @@ EXPORT_SYMBOL_GPL(work_on_cpu); · 92 unmodified lines
48874752 */
48884753 void freeze_workqueues_begin(void)
48894754 {
4890− struct worker_pool *pool;
48914755 struct workqueue_struct *wq;
48924756 struct pool_workqueue *pwq;
4893− int pi;
48944757
48954758 mutex_lock(&wq_pool_mutex);
48964759
48974760 WARN_ON_ONCE(workqueue_freezing);
48984761 workqueue_freezing = true;
48994762
4900− /* set FREEZING */
4901− for_each_pool(pool, pi) {
4902− spin_lock_irq(&pool->lock);
4903− WARN_ON_ONCE(pool->flags & POOL_FREEZING);
4904− pool->flags |= POOL_FREEZING;
4905− spin_unlock_irq(&pool->lock);
4906− }
4907−
49084763 list_for_each_entry(wq, &workqueues, list) {
49094764 mutex_lock(&wq->mutex);
49104765 for_each_pwq(pwq, wq)
@@ -4974,21 +4829,13 @@ void thaw_workqueues(void) · 63 unmodified lines
49744829 {
49754830 struct workqueue_struct *wq;
49764831 struct pool_workqueue *pwq;
4977− struct worker_pool *pool;
4978− int pi;
49794832
49804833 mutex_lock(&wq_pool_mutex);
49814834
49824835 if (!workqueue_freezing)
49834836 goto out_unlock;
49844837
4985− /* clear FREEZING */
4986− for_each_pool(pool, pi) {
4987− spin_lock_irq(&pool->lock);
4988− WARN_ON_ONCE(!(pool->flags & POOL_FREEZING));
4989− pool->flags &= ~POOL_FREEZING;
4990− spin_unlock_irq(&pool->lock);
4991− }
4838+ workqueue_freezing = false;
49924839
49934840 /* restore max_active and repopulate worklist */
49944841 list_for_each_entry(wq, &workqueues, list) {
@@ -4998,7 +4845,6 @@ void thaw_workqueues(void) · 3 unmodified lines
49984845 mutex_unlock(&wq->mutex);
49994846 }
50004847
5001− workqueue_freezing = false;
50024848 out_unlock:
50034849 mutex_unlock(&wq_pool_mutex);
50044850 }