History

Merge branch 'for-5.8' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq

Pull workqueue updates from Tejun Heo: "Mostly cleanups and other trivial changes. The only interesting change is Sebastian's rcuwait conversion for RT" * 'for-5.8' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq: workqueue: use BUILD_BUG_ON() for compile time test instead of WARN_ON() workqueue: fix a piece of comment about reserved bits for work flags workqueue: remove useless unlock() and lock() in series workqueue: void unneeded requeuing the pwq in rescuer thread workqueue: Convert the pool::lock and wq_mayday_lock to raw_spinlock_t workqueue: Use rcuwait for wq_manager_wait workqueue: Remove unnecessary kfree() call in rcu_free_wq() workqueue: Fix an use after free in init_rescuer() workqueue: Use IS_ERR and PTR_ERR instead of PTR_ERR_OR_ZERO.

Linus Torvalds committed 6 years ago

Commit fe3bc8a2 parents 4a7e89c 10cdb152 files · +110 −99Browse files

Unified · Split

2 files
  1. include/linux/workqueue.h +1 −1
  2. kernel/workqueue.c +109 −98
include/linux/workqueue.h+1−1
@@ -62,7 +62,7 @@ enum { · 61 unmodified lines
62 WORK_CPU_UNBOUND = NR_CPUS,62 WORK_CPU_UNBOUND = NR_CPUS,
63 63
64 /*64 /*
65 * Reserve 7 bits off of pwq pointer w/ debugobjects turned off.65 * Reserve 8 bits off of pwq pointer w/ debugobjects turned off.
66 * This makes pwqs aligned to 256 bytes and allows 15 workqueue66 * This makes pwqs aligned to 256 bytes and allows 15 workqueue
67 * flush colors.67 * flush colors.
68 */68 */
kernel/workqueue.c+109−98
@@ -145,7 +145,7 @@ enum { · 144 unmodified lines
145/* struct worker is defined in workqueue_internal.h */145/* struct worker is defined in workqueue_internal.h */
146 146
147struct worker_pool {147struct worker_pool {
148 spinlock_t lock; /* the pool lock */148 raw_spinlock_t lock; /* the pool lock */
149 int cpu; /* I: the associated cpu */149 int cpu; /* I: the associated cpu */
150 int node; /* I: the associated node ID */150 int node; /* I: the associated node ID */
151 int id; /* I: pool ID */151 int id; /* I: pool ID */
@@ -300,8 +300,9 @@ static struct workqueue_attrs *wq_update_unbound_numa_attrs_buf; · 148 unmodified lines
300 300
301static DEFINE_MUTEX(wq_pool_mutex); /* protects pools and workqueues list */301static DEFINE_MUTEX(wq_pool_mutex); /* protects pools and workqueues list */
302static DEFINE_MUTEX(wq_pool_attach_mutex); /* protects worker attach/detach */302static DEFINE_MUTEX(wq_pool_attach_mutex); /* protects worker attach/detach */
303static DEFINE_SPINLOCK(wq_mayday_lock); /* protects wq->maydays list */303static DEFINE_RAW_SPINLOCK(wq_mayday_lock); /* protects wq->maydays list */
304static DECLARE_WAIT_QUEUE_HEAD(wq_manager_wait); /* wait for manager to go away */304/* wait for manager to go away */
305static struct rcuwait manager_wait = __RCUWAIT_INITIALIZER(manager_wait);
305 306
306static LIST_HEAD(workqueues); /* PR: list of all workqueues */307static LIST_HEAD(workqueues); /* PR: list of all workqueues */
307static bool workqueue_freezing; /* PL: have wqs started freezing? */308static bool workqueue_freezing; /* PL: have wqs started freezing? */
@@ -826,7 +827,7 @@ static struct worker *first_idle_worker(struct worker_pool *pool) · 518 unmodified lines
826 * Wake up the first idle worker of @pool.827 * Wake up the first idle worker of @pool.
827 *828 *
828 * CONTEXT:829 * CONTEXT:
829 * spin_lock_irq(pool->lock).830 * raw_spin_lock_irq(pool->lock).
830 */831 */
831static void wake_up_worker(struct worker_pool *pool)832static void wake_up_worker(struct worker_pool *pool)
832{833{
@@ -881,7 +882,7 @@ void wq_worker_sleeping(struct task_struct *task) · 48 unmodified lines
881 return;882 return;
882 883
883 worker->sleeping = 1;884 worker->sleeping = 1;
884 spin_lock_irq(&pool->lock);885 raw_spin_lock_irq(&pool->lock);
885 886
886 /*887 /*
887 * The counterpart of the following dec_and_test, implied mb,888 * The counterpart of the following dec_and_test, implied mb,
@@ -900,7 +901,7 @@ void wq_worker_sleeping(struct task_struct *task) · 12 unmodified lines
900 if (next)901 if (next)
901 wake_up_process(next->task);902 wake_up_process(next->task);
902 }903 }
903 spin_unlock_irq(&pool->lock);904 raw_spin_unlock_irq(&pool->lock);
904}905}
905 906
906/**907/**
@@ -911,7 +912,7 @@ void wq_worker_sleeping(struct task_struct *task) · 4 unmodified lines
911 * the scheduler to get a worker's last known identity.912 * the scheduler to get a worker's last known identity.
912 *913 *
913 * CONTEXT:914 * CONTEXT:
914 * spin_lock_irq(rq->lock)915 * raw_spin_lock_irq(rq->lock)
915 *916 *
916 * This function is called during schedule() when a kworker is going917 * This function is called during schedule() when a kworker is going
917 * to sleep. It's used by psi to identify aggregation workers during918 * to sleep. It's used by psi to identify aggregation workers during
@@ -942,7 +943,7 @@ work_func_t wq_worker_last_func(struct task_struct *task) · 24 unmodified lines
942 * Set @flags in @worker->flags and adjust nr_running accordingly.943 * Set @flags in @worker->flags and adjust nr_running accordingly.
943 *944 *
944 * CONTEXT:945 * CONTEXT:
945 * spin_lock_irq(pool->lock)946 * raw_spin_lock_irq(pool->lock)
946 */947 */
947static inline void worker_set_flags(struct worker *worker, unsigned int flags)948static inline void worker_set_flags(struct worker *worker, unsigned int flags)
948{949{
@@ -967,7 +968,7 @@ static inline void worker_set_flags(struct worker *worker, unsigned int flags) · 18 unmodified lines
967 * Clear @flags in @worker->flags and adjust nr_running accordingly.968 * Clear @flags in @worker->flags and adjust nr_running accordingly.
968 *969 *
969 * CONTEXT:970 * CONTEXT:
970 * spin_lock_irq(pool->lock)971 * raw_spin_lock_irq(pool->lock)
971 */972 */
972static inline void worker_clr_flags(struct worker *worker, unsigned int flags)973static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
973{974{
@@ -1015,7 +1016,7 @@ static inline void worker_clr_flags(struct worker *worker, unsigned int flags) · 41 unmodified lines
1015 * actually occurs, it should be easy to locate the culprit work function.1016 * actually occurs, it should be easy to locate the culprit work function.
1016 *1017 *
1017 * CONTEXT:1018 * CONTEXT:
1018 * spin_lock_irq(pool->lock).1019 * raw_spin_lock_irq(pool->lock).
1019 *1020 *
1020 * Return:1021 * Return:
1021 * Pointer to worker which is executing @work if found, %NULL1022 * Pointer to worker which is executing @work if found, %NULL
@@ -1050,7 +1051,7 @@ static struct worker *find_worker_executing_work(struct worker_pool *pool, · 28 unmodified lines
1050 * nested inside outer list_for_each_entry_safe().1051 * nested inside outer list_for_each_entry_safe().
1051 *1052 *
1052 * CONTEXT:1053 * CONTEXT:
1053 * spin_lock_irq(pool->lock).1054 * raw_spin_lock_irq(pool->lock).
1054 */1055 */
1055static void move_linked_works(struct work_struct *work, struct list_head *head,1056static void move_linked_works(struct work_struct *work, struct list_head *head,
1056 struct work_struct **nextp)1057 struct work_struct **nextp)
@@ -1128,9 +1129,9 @@ static void put_pwq_unlocked(struct pool_workqueue *pwq) · 71 unmodified lines
1128 * As both pwqs and pools are RCU protected, the1129 * As both pwqs and pools are RCU protected, the
1129 * following lock operations are safe.1130 * following lock operations are safe.
1130 */1131 */
1131 spin_lock_irq(&pwq->pool->lock);1132 raw_spin_lock_irq(&pwq->pool->lock);
1132 put_pwq(pwq);1133 put_pwq(pwq);
1133 spin_unlock_irq(&pwq->pool->lock);1134 raw_spin_unlock_irq(&pwq->pool->lock);
1134 }1135 }
1135}1136}
1136 1137
@@ -1163,7 +1164,7 @@ static void pwq_activate_first_delayed(struct pool_workqueue *pwq) · 26 unmodified lines
1163 * decrement nr_in_flight of its pwq and handle workqueue flushing.1164 * decrement nr_in_flight of its pwq and handle workqueue flushing.
1164 *1165 *
1165 * CONTEXT:1166 * CONTEXT:
1166 * spin_lock_irq(pool->lock).1167 * raw_spin_lock_irq(pool->lock).
1167 */1168 */
1168static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, int color)1169static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, int color)
1169{1170{
@@ -1262,7 +1263,7 @@ static int try_to_grab_pending(struct work_struct *work, bool is_dwork, · 92 unmodified lines
1262 if (!pool)1263 if (!pool)
1263 goto fail;1264 goto fail;
1264 1265
1265 spin_lock(&pool->lock);1266 raw_spin_lock(&pool->lock);
1266 /*1267 /*
1267 * work->data is guaranteed to point to pwq only while the work1268 * work->data is guaranteed to point to pwq only while the work
1268 * item is queued on pwq->wq, and both updating work->data to point1269 * item is queued on pwq->wq, and both updating work->data to point
@@ -1291,11 +1292,11 @@ static int try_to_grab_pending(struct work_struct *work, bool is_dwork, · 22 unmodified lines
1291 /* work->data points to pwq iff queued, point to pool */1292 /* work->data points to pwq iff queued, point to pool */
1292 set_work_pool_and_keep_pending(work, pool->id);1293 set_work_pool_and_keep_pending(work, pool->id);
1293 1294
1294 spin_unlock(&pool->lock);1295 raw_spin_unlock(&pool->lock);
1295 rcu_read_unlock();1296 rcu_read_unlock();
1296 return 1;1297 return 1;
1297 }1298 }
1298 spin_unlock(&pool->lock);1299 raw_spin_unlock(&pool->lock);
1299fail:1300fail:
1300 rcu_read_unlock();1301 rcu_read_unlock();
1301 local_irq_restore(*flags);1302 local_irq_restore(*flags);
@@ -1316,7 +1317,7 @@ fail: · 14 unmodified lines
1316 * work_struct flags.1317 * work_struct flags.
1317 *1318 *
1318 * CONTEXT:1319 * CONTEXT:
1319 * spin_lock_irq(pool->lock).1320 * raw_spin_lock_irq(pool->lock).
1320 */1321 */
1321static void insert_work(struct pool_workqueue *pwq, struct work_struct *work,1322static void insert_work(struct pool_workqueue *pwq, struct work_struct *work,
1322 struct list_head *head, unsigned int extra_flags)1323 struct list_head *head, unsigned int extra_flags)
@@ -1433,7 +1434,7 @@ retry: · 110 unmodified lines
1433 if (last_pool && last_pool != pwq->pool) {1434 if (last_pool && last_pool != pwq->pool) {
1434 struct worker *worker;1435 struct worker *worker;
1435 1436
1436 spin_lock(&last_pool->lock);1437 raw_spin_lock(&last_pool->lock);
1437 1438
1438 worker = find_worker_executing_work(last_pool, work);1439 worker = find_worker_executing_work(last_pool, work);
1439 1440
@@ -1441,11 +1442,11 @@ retry: · 1 unmodified line
1441 pwq = worker->current_pwq;1442 pwq = worker->current_pwq;
1442 } else {1443 } else {
1443 /* meh... not running there, queue here */1444 /* meh... not running there, queue here */
1444 spin_unlock(&last_pool->lock);1445 raw_spin_unlock(&last_pool->lock);
1445 spin_lock(&pwq->pool->lock);1446 raw_spin_lock(&pwq->pool->lock);
1446 }1447 }
1447 } else {1448 } else {
1448 spin_lock(&pwq->pool->lock);1449 raw_spin_lock(&pwq->pool->lock);
1449 }1450 }
1450 1451
1451 /*1452 /*
@@ -1458,7 +1459,7 @@ retry: · 6 unmodified lines
1458 */1459 */
1459 if (unlikely(!pwq->refcnt)) {1460 if (unlikely(!pwq->refcnt)) {
1460 if (wq->flags & WQ_UNBOUND) {1461 if (wq->flags & WQ_UNBOUND) {
1461 spin_unlock(&pwq->pool->lock);1462 raw_spin_unlock(&pwq->pool->lock);
1462 cpu_relax();1463 cpu_relax();
1463 goto retry;1464 goto retry;
1464 }1465 }
@@ -1490,7 +1491,7 @@ retry: · 25 unmodified lines
1490 insert_work(pwq, work, worklist, work_flags);1491 insert_work(pwq, work, worklist, work_flags);
1491 1492
1492out:1493out:
1493 spin_unlock(&pwq->pool->lock);1494 raw_spin_unlock(&pwq->pool->lock);
1494 rcu_read_unlock();1495 rcu_read_unlock();
1495}1496}
1496 1497
@@ -1759,7 +1760,7 @@ EXPORT_SYMBOL(queue_rcu_work); · 262 unmodified lines
1759 * necessary.1760 * necessary.
1760 *1761 *
1761 * LOCKING:1762 * LOCKING:
1762 * spin_lock_irq(pool->lock).1763 * raw_spin_lock_irq(pool->lock).
1763 */1764 */
1764static void worker_enter_idle(struct worker *worker)1765static void worker_enter_idle(struct worker *worker)
1765{1766{
@@ -1799,7 +1800,7 @@ static void worker_enter_idle(struct worker *worker) · 33 unmodified lines
1799 * @worker is leaving idle state. Update stats.1800 * @worker is leaving idle state. Update stats.
1800 *1801 *
1801 * LOCKING:1802 * LOCKING:
1802 * spin_lock_irq(pool->lock).1803 * raw_spin_lock_irq(pool->lock).
1803 */1804 */
1804static void worker_leave_idle(struct worker *worker)1805static void worker_leave_idle(struct worker *worker)
1805{1806{
@@ -1937,11 +1938,11 @@ static struct worker *create_worker(struct worker_pool *pool) · 131 unmodified lines
1937 worker_attach_to_pool(worker, pool);1938 worker_attach_to_pool(worker, pool);
1938 1939
1939 /* start the newly created worker */1940 /* start the newly created worker */
1940 spin_lock_irq(&pool->lock);1941 raw_spin_lock_irq(&pool->lock);
1941 worker->pool->nr_workers++;1942 worker->pool->nr_workers++;
1942 worker_enter_idle(worker);1943 worker_enter_idle(worker);
1943 wake_up_process(worker->task);1944 wake_up_process(worker->task);
1944 spin_unlock_irq(&pool->lock);1945 raw_spin_unlock_irq(&pool->lock);
1945 1946
1946 return worker;1947 return worker;
1947 1948
@@ -1960,7 +1961,7 @@ fail: · 12 unmodified lines
1960 * be idle.1961 * be idle.
1961 *1962 *
1962 * CONTEXT:1963 * CONTEXT:
1963 * spin_lock_irq(pool->lock).1964 * raw_spin_lock_irq(pool->lock).
1964 */1965 */
1965static void destroy_worker(struct worker *worker)1966static void destroy_worker(struct worker *worker)
1966{1967{
@@ -1986,7 +1987,7 @@ static void idle_worker_timeout(struct timer_list *t) · 19 unmodified lines
1986{1987{
1987 struct worker_pool *pool = from_timer(pool, t, idle_timer);1988 struct worker_pool *pool = from_timer(pool, t, idle_timer);
1988 1989
1989 spin_lock_irq(&pool->lock);1990 raw_spin_lock_irq(&pool->lock);
1990 1991
1991 while (too_many_workers(pool)) {1992 while (too_many_workers(pool)) {
1992 struct worker *worker;1993 struct worker *worker;
@@ -2004,7 +2005,7 @@ static void idle_worker_timeout(struct timer_list *t) · 11 unmodified lines
2004 destroy_worker(worker);2005 destroy_worker(worker);
2005 }2006 }
2006 2007
2007 spin_unlock_irq(&pool->lock);2008 raw_spin_unlock_irq(&pool->lock);
2008}2009}
2009 2010
2010static void send_mayday(struct work_struct *work)2011static void send_mayday(struct work_struct *work)
@@ -2035,8 +2036,8 @@ static void pool_mayday_timeout(struct timer_list *t) · 24 unmodified lines
2035 struct worker_pool *pool = from_timer(pool, t, mayday_timer);2036 struct worker_pool *pool = from_timer(pool, t, mayday_timer);
2036 struct work_struct *work;2037 struct work_struct *work;
2037 2038
2038 spin_lock_irq(&pool->lock);2039 raw_spin_lock_irq(&pool->lock);
2039 spin_lock(&wq_mayday_lock); /* for wq->maydays */2040 raw_spin_lock(&wq_mayday_lock); /* for wq->maydays */
2040 2041
2041 if (need_to_create_worker(pool)) {2042 if (need_to_create_worker(pool)) {
2042 /*2043 /*
@@ -2049,8 +2050,8 @@ static void pool_mayday_timeout(struct timer_list *t) · 6 unmodified lines
2049 send_mayday(work);2050 send_mayday(work);
2050 }2051 }
2051 2052
2052 spin_unlock(&wq_mayday_lock);2053 raw_spin_unlock(&wq_mayday_lock);
2053 spin_unlock_irq(&pool->lock);2054 raw_spin_unlock_irq(&pool->lock);
2054 2055
2055 mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);2056 mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);
2056}2057}
@@ -2069,7 +2070,7 @@ static void pool_mayday_timeout(struct timer_list *t) · 12 unmodified lines
2069 * may_start_working() %true.2070 * may_start_working() %true.
2070 *2071 *
2071 * LOCKING:2072 * LOCKING:
2072 * spin_lock_irq(pool->lock) which may be released and regrabbed2073 * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
2073 * multiple times. Does GFP_KERNEL allocations. Called only from2074 * multiple times. Does GFP_KERNEL allocations. Called only from
2074 * manager.2075 * manager.
2075 */2076 */
@@ -2078,7 +2079,7 @@ __releases(&pool->lock) · 2 unmodified lines
2078__acquires(&pool->lock)2079__acquires(&pool->lock)
2079{2080{
2080restart:2081restart:
2081 spin_unlock_irq(&pool->lock);2082 raw_spin_unlock_irq(&pool->lock);
2082 2083
2083 /* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */2084 /* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */
2084 mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);2085 mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
@@ -2094,7 +2095,7 @@ restart: · 9 unmodified lines
2094 }2095 }
2095 2096
2096 del_timer_sync(&pool->mayday_timer);2097 del_timer_sync(&pool->mayday_timer);
2097 spin_lock_irq(&pool->lock);2098 raw_spin_lock_irq(&pool->lock);
2098 /*2099 /*
2099 * This is necessary even after a new worker was just successfully2100 * This is necessary even after a new worker was just successfully
2100 * created as @pool->lock was dropped and the new worker might have2101 * created as @pool->lock was dropped and the new worker might have
@@ -2117,7 +2118,7 @@ restart: · 16 unmodified lines
2117 * and may_start_working() is true.2118 * and may_start_working() is true.
2118 *2119 *
2119 * CONTEXT:2120 * CONTEXT:
2120 * spin_lock_irq(pool->lock) which may be released and regrabbed2121 * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
2121 * multiple times. Does GFP_KERNEL allocations.2122 * multiple times. Does GFP_KERNEL allocations.
2122 *2123 *
2123 * Return:2124 * Return:
@@ -2140,7 +2141,7 @@ static bool manage_workers(struct worker *worker) · 16 unmodified lines
2140 2141
2141 pool->manager = NULL;2142 pool->manager = NULL;
2142 pool->flags &= ~POOL_MANAGER_ACTIVE;2143 pool->flags &= ~POOL_MANAGER_ACTIVE;
2143 wake_up(&wq_manager_wait);2144 rcuwait_wake_up(&manager_wait);
2144 return true;2145 return true;
2145}2146}
2146 2147
@@ -2156,7 +2157,7 @@ static bool manage_workers(struct worker *worker) · 9 unmodified lines
2156 * call this function to process a work.2157 * call this function to process a work.
2157 *2158 *
2158 * CONTEXT:2159 * CONTEXT:
2159 * spin_lock_irq(pool->lock) which is released and regrabbed.2160 * raw_spin_lock_irq(pool->lock) which is released and regrabbed.
2160 */2161 */
2161static void process_one_work(struct worker *worker, struct work_struct *work)2162static void process_one_work(struct worker *worker, struct work_struct *work)
2162__releases(&pool->lock)2163__releases(&pool->lock)
@@ -2238,7 +2239,7 @@ __acquires(&pool->lock) · 75 unmodified lines
2238 */2239 */
2239 set_work_pool_and_clear_pending(work, pool->id);2240 set_work_pool_and_clear_pending(work, pool->id);
2240 2241
2241 spin_unlock_irq(&pool->lock);2242 raw_spin_unlock_irq(&pool->lock);
2242 2243
2243 lock_map_acquire(&pwq->wq->lockdep_map);2244 lock_map_acquire(&pwq->wq->lockdep_map);
2244 lock_map_acquire(&lockdep_map);2245 lock_map_acquire(&lockdep_map);
@@ -2293,7 +2294,7 @@ __acquires(&pool->lock) · 48 unmodified lines
2293 */2294 */
2294 cond_resched();2295 cond_resched();
2295 2296
2296 spin_lock_irq(&pool->lock);2297 raw_spin_lock_irq(&pool->lock);
2297 2298
2298 /* clear cpu intensive status */2299 /* clear cpu intensive status */
2299 if (unlikely(cpu_intensive))2300 if (unlikely(cpu_intensive))
@@ -2319,7 +2320,7 @@ __acquires(&pool->lock) · 19 unmodified lines
2319 * fetches a work from the top and executes it.2320 * fetches a work from the top and executes it.
2320 *2321 *
2321 * CONTEXT:2322 * CONTEXT:
2322 * spin_lock_irq(pool->lock) which may be released and regrabbed2323 * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
2323 * multiple times.2324 * multiple times.
2324 */2325 */
2325static void process_scheduled_works(struct worker *worker)2326static void process_scheduled_works(struct worker *worker)
@@ -2361,11 +2362,11 @@ static int worker_thread(void *__worker) · 35 unmodified lines
2361 /* tell the scheduler that this is a workqueue worker */2362 /* tell the scheduler that this is a workqueue worker */
2362 set_pf_worker(true);2363 set_pf_worker(true);
2363woke_up:2364woke_up:
2364 spin_lock_irq(&pool->lock);2365 raw_spin_lock_irq(&pool->lock);
2365 2366
2366 /* am I supposed to die? */2367 /* am I supposed to die? */
2367 if (unlikely(worker->flags & WORKER_DIE)) {2368 if (unlikely(worker->flags & WORKER_DIE)) {
2368 spin_unlock_irq(&pool->lock);2369 raw_spin_unlock_irq(&pool->lock);
2369 WARN_ON_ONCE(!list_empty(&worker->entry));2370 WARN_ON_ONCE(!list_empty(&worker->entry));
2370 set_pf_worker(false);2371 set_pf_worker(false);
2371 2372
@@ -2431,7 +2432,7 @@ sleep: · 59 unmodified lines
2431 */2432 */
2432 worker_enter_idle(worker);2433 worker_enter_idle(worker);
2433 __set_current_state(TASK_IDLE);2434 __set_current_state(TASK_IDLE);
2434 spin_unlock_irq(&pool->lock);2435 raw_spin_unlock_irq(&pool->lock);
2435 schedule();2436 schedule();
2436 goto woke_up;2437 goto woke_up;
2437}2438}
@@ -2485,7 +2486,7 @@ repeat: · 47 unmodified lines
2485 should_stop = kthread_should_stop();2486 should_stop = kthread_should_stop();
2486 2487
2487 /* see whether any pwq is asking for help */2488 /* see whether any pwq is asking for help */
2488 spin_lock_irq(&wq_mayday_lock);2489 raw_spin_lock_irq(&wq_mayday_lock);
2489 2490
2490 while (!list_empty(&wq->maydays)) {2491 while (!list_empty(&wq->maydays)) {
2491 struct pool_workqueue *pwq = list_first_entry(&wq->maydays,2492 struct pool_workqueue *pwq = list_first_entry(&wq->maydays,
@@ -2497,11 +2498,11 @@ repeat: · 5 unmodified lines
2497 __set_current_state(TASK_RUNNING);2498 __set_current_state(TASK_RUNNING);
2498 list_del_init(&pwq->mayday_node);2499 list_del_init(&pwq->mayday_node);
2499 2500
2500 spin_unlock_irq(&wq_mayday_lock);2501 raw_spin_unlock_irq(&wq_mayday_lock);
2501 2502
2502 worker_attach_to_pool(rescuer, pool);2503 worker_attach_to_pool(rescuer, pool);
2503 2504
2504 spin_lock_irq(&pool->lock);2505 raw_spin_lock_irq(&pool->lock);
2505 2506
2506 /*2507 /*
2507 * Slurp in all works issued via this workqueue and2508 * Slurp in all works issued via this workqueue and
@@ -2529,8 +2530,8 @@ repeat: · 21 unmodified lines
2529 * being used to relieve memory pressure, don't2530 * being used to relieve memory pressure, don't
2530 * incur MAYDAY_INTERVAL delay inbetween.2531 * incur MAYDAY_INTERVAL delay inbetween.
2531 */2532 */
2532 if (need_to_create_worker(pool)) {2533 if (pwq->nr_active && need_to_create_worker(pool)) {
2533 spin_lock(&wq_mayday_lock);2534 raw_spin_lock(&wq_mayday_lock);
2534 /*2535 /*
2535 * Queue iff we aren't racing destruction2536 * Queue iff we aren't racing destruction
2536 * and somebody else hasn't queued it already.2537 * and somebody else hasn't queued it already.
@@ -2539,7 +2540,7 @@ repeat: · 2 unmodified lines
2539 get_pwq(pwq);2540 get_pwq(pwq);
2540 list_add_tail(&pwq->mayday_node, &wq->maydays);2541 list_add_tail(&pwq->mayday_node, &wq->maydays);
2541 }2542 }
2542 spin_unlock(&wq_mayday_lock);2543 raw_spin_unlock(&wq_mayday_lock);
2543 }2544 }
2544 }2545 }
2545 2546
@@ -2557,14 +2558,14 @@ repeat: · 11 unmodified lines
2557 if (need_more_worker(pool))2558 if (need_more_worker(pool))
2558 wake_up_worker(pool);2559 wake_up_worker(pool);
2559 2560
2560 spin_unlock_irq(&pool->lock);2561 raw_spin_unlock_irq(&pool->lock);
2561 2562
2562 worker_detach_from_pool(rescuer);2563 worker_detach_from_pool(rescuer);
2563 2564
2564 spin_lock_irq(&wq_mayday_lock);2565 raw_spin_lock_irq(&wq_mayday_lock);
2565 }2566 }
2566 2567
2567 spin_unlock_irq(&wq_mayday_lock);2568 raw_spin_unlock_irq(&wq_mayday_lock);
2568 2569
2569 if (should_stop) {2570 if (should_stop) {
2570 __set_current_state(TASK_RUNNING);2571 __set_current_state(TASK_RUNNING);
@@ -2644,7 +2645,7 @@ static void wq_barrier_func(struct work_struct *work) · 73 unmodified lines
2644 * underneath us, so we can't reliably determine pwq from @target.2645 * underneath us, so we can't reliably determine pwq from @target.
2645 *2646 *
2646 * CONTEXT:2647 * CONTEXT:
2647 * spin_lock_irq(pool->lock).2648 * raw_spin_lock_irq(pool->lock).
2648 */2649 */
2649static void insert_wq_barrier(struct pool_workqueue *pwq,2650static void insert_wq_barrier(struct pool_workqueue *pwq,
2650 struct wq_barrier *barr,2651 struct wq_barrier *barr,
@@ -2731,7 +2732,7 @@ static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq, · 80 unmodified lines
2731 for_each_pwq(pwq, wq) {2732 for_each_pwq(pwq, wq) {
2732 struct worker_pool *pool = pwq->pool;2733 struct worker_pool *pool = pwq->pool;
2733 2734
2734 spin_lock_irq(&pool->lock);2735 raw_spin_lock_irq(&pool->lock);
2735 2736
2736 if (flush_color >= 0) {2737 if (flush_color >= 0) {
2737 WARN_ON_ONCE(pwq->flush_color != -1);2738 WARN_ON_ONCE(pwq->flush_color != -1);
@@ -2748,7 +2749,7 @@ static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq, · 10 unmodified lines
2748 pwq->work_color = work_color;2749 pwq->work_color = work_color;
2749 }2750 }
2750 2751
2751 spin_unlock_irq(&pool->lock);2752 raw_spin_unlock_irq(&pool->lock);
2752 }2753 }
2753 2754
2754 if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))2755 if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))
@@ -2948,9 +2949,9 @@ reflush: · 193 unmodified lines
2948 for_each_pwq(pwq, wq) {2949 for_each_pwq(pwq, wq) {
2949 bool drained;2950 bool drained;
2950 2951
2951 spin_lock_irq(&pwq->pool->lock);2952 raw_spin_lock_irq(&pwq->pool->lock);
2952 drained = !pwq->nr_active && list_empty(&pwq->delayed_works);2953 drained = !pwq->nr_active && list_empty(&pwq->delayed_works);
2953 spin_unlock_irq(&pwq->pool->lock);2954 raw_spin_unlock_irq(&pwq->pool->lock);
2954 2955
2955 if (drained)2956 if (drained)
2956 continue;2957 continue;
@@ -2986,7 +2987,7 @@ static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr, · 29 unmodified lines
2986 return false;2987 return false;
2987 }2988 }
2988 2989
2989 spin_lock_irq(&pool->lock);2990 raw_spin_lock_irq(&pool->lock);
2990 /* see the comment in try_to_grab_pending() with the same code */2991 /* see the comment in try_to_grab_pending() with the same code */
2991 pwq = get_work_pwq(work);2992 pwq = get_work_pwq(work);
2992 if (pwq) {2993 if (pwq) {
@@ -3002,7 +3003,7 @@ static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr, · 9 unmodified lines
3002 check_flush_dependency(pwq->wq, work);3003 check_flush_dependency(pwq->wq, work);
3003 3004
3004 insert_wq_barrier(pwq, barr, work, worker);3005 insert_wq_barrier(pwq, barr, work, worker);
3005 spin_unlock_irq(&pool->lock);3006 raw_spin_unlock_irq(&pool->lock);
3006 3007
3007 /*3008 /*
3008 * Force a lock recursion deadlock when using flush_work() inside a3009 * Force a lock recursion deadlock when using flush_work() inside a
@@ -3021,7 +3022,7 @@ static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr, · 12 unmodified lines
3021 rcu_read_unlock();3022 rcu_read_unlock();
3022 return true;3023 return true;
3023already_gone:3024already_gone:
3024 spin_unlock_irq(&pool->lock);3025 raw_spin_unlock_irq(&pool->lock);
3025 rcu_read_unlock();3026 rcu_read_unlock();
3026 return false;3027 return false;
3027}3028}
@@ -3414,7 +3415,7 @@ static bool wqattrs_equal(const struct workqueue_attrs *a, · 386 unmodified lines
3414 */3415 */
3415static int init_worker_pool(struct worker_pool *pool)3416static int init_worker_pool(struct worker_pool *pool)
3416{3417{
3417 spin_lock_init(&pool->lock);3418 raw_spin_lock_init(&pool->lock);
3418 pool->id = -1;3419 pool->id = -1;
3419 pool->cpu = -1;3420 pool->cpu = -1;
3420 pool->node = NUMA_NO_NODE;3421 pool->node = NUMA_NO_NODE;
@@ -3491,7 +3492,6 @@ static void rcu_free_wq(struct rcu_head *rcu) · 70 unmodified lines
3491 else3492 else
3492 free_workqueue_attrs(wq->unbound_attrs);3493 free_workqueue_attrs(wq->unbound_attrs);
3493 3494
3494 kfree(wq->rescuer);
3495 kfree(wq);3495 kfree(wq);
3496}3496}
3497 3497
@@ -3504,6 +3504,18 @@ static void rcu_free_pool(struct rcu_head *rcu) · 6 unmodified lines
3504 kfree(pool);3504 kfree(pool);
3505}3505}
3506 3506
3507/* This returns with the lock held on success (pool manager is inactive). */
3508static bool wq_manager_inactive(struct worker_pool *pool)
3509{
3510 raw_spin_lock_irq(&pool->lock);
3511
3512 if (pool->flags & POOL_MANAGER_ACTIVE) {
3513 raw_spin_unlock_irq(&pool->lock);
3514 return false;
3515 }
3516 return true;
3517}
3518
3507/**3519/**
3508 * put_unbound_pool - put a worker_pool3520 * put_unbound_pool - put a worker_pool
3509 * @pool: worker_pool to put3521 * @pool: worker_pool to put
@@ -3539,16 +3551,17 @@ static void put_unbound_pool(struct worker_pool *pool) · 29 unmodified lines
3539 * Become the manager and destroy all workers. This prevents3551 * Become the manager and destroy all workers. This prevents
3540 * @pool's workers from blocking on attach_mutex. We're the last3552 * @pool's workers from blocking on attach_mutex. We're the last
3541 * manager and @pool gets freed with the flag set.3553 * manager and @pool gets freed with the flag set.
3554 * Because of how wq_manager_inactive() works, we will hold the
3555 * spinlock after a successful wait.
3542 */3556 */
3543 spin_lock_irq(&pool->lock);3557 rcuwait_wait_event(&manager_wait, wq_manager_inactive(pool),
3544 wait_event_lock_irq(wq_manager_wait,3558 TASK_UNINTERRUPTIBLE);
3545 !(pool->flags & POOL_MANAGER_ACTIVE), pool->lock);
3546 pool->flags |= POOL_MANAGER_ACTIVE;3559 pool->flags |= POOL_MANAGER_ACTIVE;
3547 3560
3548 while ((worker = first_idle_worker(pool)))3561 while ((worker = first_idle_worker(pool)))
3549 destroy_worker(worker);3562 destroy_worker(worker);
3550 WARN_ON(pool->nr_workers || pool->nr_idle);3563 WARN_ON(pool->nr_workers || pool->nr_idle);
3551 spin_unlock_irq(&pool->lock);3564 raw_spin_unlock_irq(&pool->lock);
3552 3565
3553 mutex_lock(&wq_pool_attach_mutex);3566 mutex_lock(&wq_pool_attach_mutex);
3554 if (!list_empty(&pool->workers))3567 if (!list_empty(&pool->workers))
@@ -3704,7 +3717,7 @@ static void pwq_adjust_max_active(struct pool_workqueue *pwq) · 149 unmodified lines
3704 return;3717 return;
3705 3718
3706 /* this function can be called during early boot w/ irq disabled */3719 /* this function can be called during early boot w/ irq disabled */
3707 spin_lock_irqsave(&pwq->pool->lock, flags);3720 raw_spin_lock_irqsave(&pwq->pool->lock, flags);
3708 3721
3709 /*3722 /*
3710 * During [un]freezing, the caller is responsible for ensuring that3723 * During [un]freezing, the caller is responsible for ensuring that
@@ -3727,7 +3740,7 @@ static void pwq_adjust_max_active(struct pool_workqueue *pwq) · 16 unmodified lines
3727 pwq->max_active = 0;3740 pwq->max_active = 0;
3728 }3741 }
3729 3742
3730 spin_unlock_irqrestore(&pwq->pool->lock, flags);3743 raw_spin_unlock_irqrestore(&pwq->pool->lock, flags);
3731}3744}
3732 3745
3733/* initialize newly alloced @pwq which is associated with @wq and @pool */3746/* initialize newly alloced @pwq which is associated with @wq and @pool */
@@ -4129,9 +4142,9 @@ static void wq_update_unbound_numa(struct workqueue_struct *wq, int cpu, · 395 unmodified lines
4129 4142
4130use_dfl_pwq:4143use_dfl_pwq:
4131 mutex_lock(&wq->mutex);4144 mutex_lock(&wq->mutex);
4132 spin_lock_irq(&wq->dfl_pwq->pool->lock);4145 raw_spin_lock_irq(&wq->dfl_pwq->pool->lock);
4133 get_pwq(wq->dfl_pwq);4146 get_pwq(wq->dfl_pwq);
4134 spin_unlock_irq(&wq->dfl_pwq->pool->lock);4147 raw_spin_unlock_irq(&wq->dfl_pwq->pool->lock);
4135 old_pwq = numa_pwq_tbl_install(wq, node, wq->dfl_pwq);4148 old_pwq = numa_pwq_tbl_install(wq, node, wq->dfl_pwq);
4136out_unlock:4149out_unlock:
4137 mutex_unlock(&wq->mutex);4150 mutex_unlock(&wq->mutex);
@@ -4208,8 +4221,8 @@ static int init_rescuer(struct workqueue_struct *wq) · 70 unmodified lines
4208 4221
4209 rescuer->rescue_wq = wq;4222 rescuer->rescue_wq = wq;
4210 rescuer->task = kthread_create(rescuer_thread, rescuer, "%s", wq->name);4223 rescuer->task = kthread_create(rescuer_thread, rescuer, "%s", wq->name);
4211 ret = PTR_ERR_OR_ZERO(rescuer->task);4224 if (IS_ERR(rescuer->task)) {
4212 if (ret) {4225 ret = PTR_ERR(rescuer->task);
4213 kfree(rescuer);4226 kfree(rescuer);
4214 return ret;4227 return ret;
4215 }4228 }
@@ -4360,9 +4373,9 @@ void destroy_workqueue(struct workqueue_struct *wq) · 144 unmodified lines
4360 struct worker *rescuer = wq->rescuer;4373 struct worker *rescuer = wq->rescuer;
4361 4374
4362 /* this prevents new queueing */4375 /* this prevents new queueing */
4363 spin_lock_irq(&wq_mayday_lock);4376 raw_spin_lock_irq(&wq_mayday_lock);
4364 wq->rescuer = NULL;4377 wq->rescuer = NULL;
4365 spin_unlock_irq(&wq_mayday_lock);4378 raw_spin_unlock_irq(&wq_mayday_lock);
4366 4379
4367 /* rescuer will empty maydays list before exiting */4380 /* rescuer will empty maydays list before exiting */
4368 kthread_stop(rescuer->task);4381 kthread_stop(rescuer->task);
@@ -4376,27 +4389,25 @@ void destroy_workqueue(struct workqueue_struct *wq) · 7 unmodified lines
4376 mutex_lock(&wq_pool_mutex);4389 mutex_lock(&wq_pool_mutex);
4377 mutex_lock(&wq->mutex);4390 mutex_lock(&wq->mutex);
4378 for_each_pwq(pwq, wq) {4391 for_each_pwq(pwq, wq) {
4379 spin_lock_irq(&pwq->pool->lock);4392 raw_spin_lock_irq(&pwq->pool->lock);
4380 if (WARN_ON(pwq_busy(pwq))) {4393 if (WARN_ON(pwq_busy(pwq))) {
4381 pr_warn("%s: %s has the following busy pwq\n",4394 pr_warn("%s: %s has the following busy pwq\n",
4382 __func__, wq->name);4395 __func__, wq->name);
4383 show_pwq(pwq);4396 show_pwq(pwq);
4384 spin_unlock_irq(&pwq->pool->lock);4397 raw_spin_unlock_irq(&pwq->pool->lock);
4385 mutex_unlock(&wq->mutex);4398 mutex_unlock(&wq->mutex);
4386 mutex_unlock(&wq_pool_mutex);4399 mutex_unlock(&wq_pool_mutex);
4387 show_workqueue_state();4400 show_workqueue_state();
4388 return;4401 return;
4389 }4402 }
4390 spin_unlock_irq(&pwq->pool->lock);4403 raw_spin_unlock_irq(&pwq->pool->lock);
4391 }4404 }
4392 mutex_unlock(&wq->mutex);4405 mutex_unlock(&wq->mutex);
4393 mutex_unlock(&wq_pool_mutex);
4394 4406
4395 /*4407 /*
4396 * wq list is used to freeze wq, remove from list after4408 * wq list is used to freeze wq, remove from list after
4397 * flushing is complete in case freeze races us.4409 * flushing is complete in case freeze races us.
4398 */4410 */
4399 mutex_lock(&wq_pool_mutex);
4400 list_del_rcu(&wq->list);4411 list_del_rcu(&wq->list);
4401 mutex_unlock(&wq_pool_mutex);4412 mutex_unlock(&wq_pool_mutex);
4402 4413
@@ -4558,10 +4569,10 @@ unsigned int work_busy(struct work_struct *work) · 155 unmodified lines
4558 rcu_read_lock();4569 rcu_read_lock();
4559 pool = get_work_pool(work);4570 pool = get_work_pool(work);
4560 if (pool) {4571 if (pool) {
4561 spin_lock_irqsave(&pool->lock, flags);4572 raw_spin_lock_irqsave(&pool->lock, flags);
4562 if (find_worker_executing_work(pool, work))4573 if (find_worker_executing_work(pool, work))
4563 ret |= WORK_BUSY_RUNNING;4574 ret |= WORK_BUSY_RUNNING;
4564 spin_unlock_irqrestore(&pool->lock, flags);4575 raw_spin_unlock_irqrestore(&pool->lock, flags);
4565 }4576 }
4566 rcu_read_unlock();4577 rcu_read_unlock();
4567 4578
@@ -4768,10 +4779,10 @@ void show_workqueue_state(void) · 200 unmodified lines
4768 pr_info("workqueue %s: flags=0x%x\n", wq->name, wq->flags);4779 pr_info("workqueue %s: flags=0x%x\n", wq->name, wq->flags);
4769 4780
4770 for_each_pwq(pwq, wq) {4781 for_each_pwq(pwq, wq) {
4771 spin_lock_irqsave(&pwq->pool->lock, flags);4782 raw_spin_lock_irqsave(&pwq->pool->lock, flags);
4772 if (pwq->nr_active || !list_empty(&pwq->delayed_works))4783 if (pwq->nr_active || !list_empty(&pwq->delayed_works))
4773 show_pwq(pwq);4784 show_pwq(pwq);
4774 spin_unlock_irqrestore(&pwq->pool->lock, flags);4785 raw_spin_unlock_irqrestore(&pwq->pool->lock, flags);
4775 /*4786 /*
4776 * We could be printing a lot from atomic context, e.g.4787 * We could be printing a lot from atomic context, e.g.
4777 * sysrq-t -> show_workqueue_state(). Avoid triggering4788 * sysrq-t -> show_workqueue_state(). Avoid triggering
@@ -4785,7 +4796,7 @@ void show_workqueue_state(void) · 7 unmodified lines
4785 struct worker *worker;4796 struct worker *worker;
4786 bool first = true;4797 bool first = true;
4787 4798
4788 spin_lock_irqsave(&pool->lock, flags);4799 raw_spin_lock_irqsave(&pool->lock, flags);
4789 if (pool->nr_workers == pool->nr_idle)4800 if (pool->nr_workers == pool->nr_idle)
4790 goto next_pool;4801 goto next_pool;
4791 4802
@@ -4804,7 +4815,7 @@ void show_workqueue_state(void) · 12 unmodified lines
4804 }4815 }
4805 pr_cont("\n");4816 pr_cont("\n");
4806 next_pool:4817 next_pool:
4807 spin_unlock_irqrestore(&pool->lock, flags);4818 raw_spin_unlock_irqrestore(&pool->lock, flags);
4808 /*4819 /*
4809 * We could be printing a lot from atomic context, e.g.4820 * We could be printing a lot from atomic context, e.g.
4810 * sysrq-t -> show_workqueue_state(). Avoid triggering4821 * sysrq-t -> show_workqueue_state(). Avoid triggering
@@ -4834,7 +4845,7 @@ void wq_worker_comm(char *buf, size_t size, struct task_struct *task) · 23 unmodified lines
4834 struct worker_pool *pool = worker->pool;4845 struct worker_pool *pool = worker->pool;
4835 4846
4836 if (pool) {4847 if (pool) {
4837 spin_lock_irq(&pool->lock);4848 raw_spin_lock_irq(&pool->lock);
4838 /*4849 /*
4839 * ->desc tracks information (wq name or4850 * ->desc tracks information (wq name or
4840 * set_worker_desc()) for the latest execution. If4851 * set_worker_desc()) for the latest execution. If
@@ -4848,7 +4859,7 @@ void wq_worker_comm(char *buf, size_t size, struct task_struct *task) · 7 unmodified lines
4848 scnprintf(buf + off, size - off, "-%s",4859 scnprintf(buf + off, size - off, "-%s",
4849 worker->desc);4860 worker->desc);
4850 }4861 }
4851 spin_unlock_irq(&pool->lock);4862 raw_spin_unlock_irq(&pool->lock);
4852 }4863 }
4853 }4864 }
4854 4865
@@ -4879,7 +4890,7 @@ static void unbind_workers(int cpu) · 24 unmodified lines
4879 4890
4880 for_each_cpu_worker_pool(pool, cpu) {4891 for_each_cpu_worker_pool(pool, cpu) {
4881 mutex_lock(&wq_pool_attach_mutex);4892 mutex_lock(&wq_pool_attach_mutex);
4882 spin_lock_irq(&pool->lock);4893 raw_spin_lock_irq(&pool->lock);
4883 4894
4884 /*4895 /*
4885 * We've blocked all attach/detach operations. Make all workers4896 * We've blocked all attach/detach operations. Make all workers
@@ -4893,7 +4904,7 @@ static void unbind_workers(int cpu) · 7 unmodified lines
4893 4904
4894 pool->flags |= POOL_DISASSOCIATED;4905 pool->flags |= POOL_DISASSOCIATED;
4895 4906
4896 spin_unlock_irq(&pool->lock);4907 raw_spin_unlock_irq(&pool->lock);
4897 mutex_unlock(&wq_pool_attach_mutex);4908 mutex_unlock(&wq_pool_attach_mutex);
4898 4909
4899 /*4910 /*
@@ -4919,9 +4930,9 @@ static void unbind_workers(int cpu) · 19 unmodified lines
4919 * worker blocking could lead to lengthy stalls. Kick off4930 * worker blocking could lead to lengthy stalls. Kick off
4920 * unbound chain execution of currently pending work items.4931 * unbound chain execution of currently pending work items.
4921 */4932 */
4922 spin_lock_irq(&pool->lock);4933 raw_spin_lock_irq(&pool->lock);
4923 wake_up_worker(pool);4934 wake_up_worker(pool);
4924 spin_unlock_irq(&pool->lock);4935 raw_spin_unlock_irq(&pool->lock);
4925 }4936 }
4926}4937}
4927 4938
@@ -4948,7 +4959,7 @@ static void rebind_workers(struct worker_pool *pool) · 20 unmodified lines
4948 WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,4959 WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
4949 pool->attrs->cpumask) < 0);4960 pool->attrs->cpumask) < 0);
4950 4961
4951 spin_lock_irq(&pool->lock);4962 raw_spin_lock_irq(&pool->lock);
4952 4963
4953 pool->flags &= ~POOL_DISASSOCIATED;4964 pool->flags &= ~POOL_DISASSOCIATED;
4954 4965
@@ -4987,7 +4998,7 @@ static void rebind_workers(struct worker_pool *pool) · 32 unmodified lines
4987 WRITE_ONCE(worker->flags, worker_flags);4998 WRITE_ONCE(worker->flags, worker_flags);
4988 }4999 }
4989 5000
4990 spin_unlock_irq(&pool->lock);5001 raw_spin_unlock_irq(&pool->lock);
4991}5002}
4992 5003
4993/**5004/**
@@ -5906,7 +5917,7 @@ void __init workqueue_init_early(void) · 912 unmodified lines
5906 int hk_flags = HK_FLAG_DOMAIN | HK_FLAG_WQ;5917 int hk_flags = HK_FLAG_DOMAIN | HK_FLAG_WQ;
5907 int i, cpu;5918 int i, cpu;
5908 5919
5909 WARN_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));5920 BUILD_BUG_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));
5910 5921
5911 BUG_ON(!alloc_cpumask_var(&wq_unbound_cpumask, GFP_KERNEL));5922 BUG_ON(!alloc_cpumask_var(&wq_unbound_cpumask, GFP_KERNEL));
5912 cpumask_copy(wq_unbound_cpumask, housekeeping_cpumask(hk_flags));5923 cpumask_copy(wq_unbound_cpumask, housekeeping_cpumask(hk_flags));