Merge branch 'for-5.8' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq

Pull workqueue updates from Tejun Heo: "Mostly cleanups and other trivial changes. The only interesting change is Sebastian's rcuwait conversion for RT" * 'for-5.8' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq: workqueue: use BUILD_BUG_ON() for compile time test instead of WARN_ON() workqueue: fix a piece of comment about reserved bits for work flags workqueue: remove useless unlock() and lock() in series workqueue: void unneeded requeuing the pwq in rescuer thread workqueue: Convert the pool::lock and wq_mayday_lock to raw_spinlock_t workqueue: Use rcuwait for wq_manager_wait workqueue: Remove unnecessary kfree() call in rcu_free_wq() workqueue: Fix an use after free in init_rescuer() workqueue: Use IS_ERR and PTR_ERR instead of PTR_ERR_OR_ZERO.

Linus Torvalds committed 6 years ago

Commit fe3bc8a2 parents 4a7e89c 10cdb152 files · +110 −99Browse files

Unified · Split

2 files

include/linux/workqueue.h +1 −1
kernel/workqueue.c +109 −98

include/linux/workqueue.h+1−1

@@ -62,7 +62,7 @@ enum { · 61 unmodified lines
62	62	WORK_CPU_UNBOUND = NR_CPUS,
63	63
64	64	/*
65		− * Reserve 7 bits off of pwq pointer w/ debugobjects turned off.
	65	+ * Reserve 8 bits off of pwq pointer w/ debugobjects turned off.
66	66	* This makes pwqs aligned to 256 bytes and allows 15 workqueue
67	67	* flush colors.
68	68	*/

kernel/workqueue.c+109−98

@@ -145,7 +145,7 @@ enum { · 144 unmodified lines
145	145	/* struct worker is defined in workqueue_internal.h */
146	146
147	147	struct worker_pool {
148		− spinlock_t lock; /* the pool lock */
	148	+ raw_spinlock_t lock; /* the pool lock */
149	149	int cpu; /* I: the associated cpu */
150	150	int node; /* I: the associated node ID */
151	151	int id; /* I: pool ID */
@@ -300,8 +300,9 @@ static struct workqueue_attrs *wq_update_unbound_numa_attrs_buf; · 148 unmodified lines
300	300
301	301	static DEFINE_MUTEX(wq_pool_mutex); /* protects pools and workqueues list */
302	302	static DEFINE_MUTEX(wq_pool_attach_mutex); /* protects worker attach/detach */
303		−static DEFINE_SPINLOCK(wq_mayday_lock); /* protects wq->maydays list */
304		−static DECLARE_WAIT_QUEUE_HEAD(wq_manager_wait); /* wait for manager to go away */
	303	+static DEFINE_RAW_SPINLOCK(wq_mayday_lock); /* protects wq->maydays list */
	304	+/* wait for manager to go away */
	305	+static struct rcuwait manager_wait = __RCUWAIT_INITIALIZER(manager_wait);
305	306
306	307	static LIST_HEAD(workqueues); /* PR: list of all workqueues */
307	308	static bool workqueue_freezing; /* PL: have wqs started freezing? */
@@ -826,7 +827,7 @@ static struct worker first_idle_worker(struct worker_pool pool) · 518 unmodified lines
826	827	* Wake up the first idle worker of @pool.
827	828	*
828	829	* CONTEXT:
829		− * spin_lock_irq(pool->lock).
	830	+ * raw_spin_lock_irq(pool->lock).
830	831	*/
831	832	static void wake_up_worker(struct worker_pool *pool)
832	833	{
@@ -881,7 +882,7 @@ void wq_worker_sleeping(struct task_struct *task) · 48 unmodified lines
881	882	return;
882	883
883	884	worker->sleeping = 1;
884		− spin_lock_irq(&pool->lock);
	885	+ raw_spin_lock_irq(&pool->lock);
885	886
886	887	/*
887	888	* The counterpart of the following dec_and_test, implied mb,
@@ -900,7 +901,7 @@ void wq_worker_sleeping(struct task_struct *task) · 12 unmodified lines
900	901	if (next)
901	902	wake_up_process(next->task);
902	903	}
903		− spin_unlock_irq(&pool->lock);
	904	+ raw_spin_unlock_irq(&pool->lock);
904	905	}
905	906
906	907	/**
@@ -911,7 +912,7 @@ void wq_worker_sleeping(struct task_struct *task) · 4 unmodified lines
911	912	* the scheduler to get a worker's last known identity.
912	913	*
913	914	* CONTEXT:
914		− * spin_lock_irq(rq->lock)
	915	+ * raw_spin_lock_irq(rq->lock)
915	916	*
916	917	* This function is called during schedule() when a kworker is going
917	918	* to sleep. It's used by psi to identify aggregation workers during
@@ -942,7 +943,7 @@ work_func_t wq_worker_last_func(struct task_struct *task) · 24 unmodified lines
942	943	* Set @flags in @worker->flags and adjust nr_running accordingly.
943	944	*
944	945	* CONTEXT:
945		− * spin_lock_irq(pool->lock)
	946	+ * raw_spin_lock_irq(pool->lock)
946	947	*/
947	948	static inline void worker_set_flags(struct worker *worker, unsigned int flags)
948	949	{
@@ -967,7 +968,7 @@ static inline void worker_set_flags(struct worker *worker, unsigned int flags) · 18 unmodified lines
967	968	* Clear @flags in @worker->flags and adjust nr_running accordingly.
968	969	*
969	970	* CONTEXT:
970		− * spin_lock_irq(pool->lock)
	971	+ * raw_spin_lock_irq(pool->lock)
971	972	*/
972	973	static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
973	974	{
@@ -1015,7 +1016,7 @@ static inline void worker_clr_flags(struct worker *worker, unsigned int flags) · 41 unmodified lines
1015	1016	* actually occurs, it should be easy to locate the culprit work function.
1016	1017	*
1017	1018	* CONTEXT:
1018		− * spin_lock_irq(pool->lock).
	1019	+ * raw_spin_lock_irq(pool->lock).
1019	1020	*
1020	1021	* Return:
1021	1022	* Pointer to worker which is executing @work if found, %NULL
@@ -1050,7 +1051,7 @@ static struct worker find_worker_executing_work(struct worker_pool pool, · 28 unmodified lines
1050	1051	* nested inside outer list_for_each_entry_safe().
1051	1052	*
1052	1053	* CONTEXT:
1053		− * spin_lock_irq(pool->lock).
	1054	+ * raw_spin_lock_irq(pool->lock).
1054	1055	*/
1055	1056	static void move_linked_works(struct work_struct work, struct list_head head,
1056	1057	struct work_struct **nextp)
@@ -1128,9 +1129,9 @@ static void put_pwq_unlocked(struct pool_workqueue *pwq) · 71 unmodified lines
1128	1129	* As both pwqs and pools are RCU protected, the
1129	1130	* following lock operations are safe.
1130	1131	*/
1131		− spin_lock_irq(&pwq->pool->lock);
	1132	+ raw_spin_lock_irq(&pwq->pool->lock);
1132	1133	put_pwq(pwq);
1133		− spin_unlock_irq(&pwq->pool->lock);
	1134	+ raw_spin_unlock_irq(&pwq->pool->lock);
1134	1135	}
1135	1136	}
1136	1137
@@ -1163,7 +1164,7 @@ static void pwq_activate_first_delayed(struct pool_workqueue *pwq) · 26 unmodified lines
1163	1164	* decrement nr_in_flight of its pwq and handle workqueue flushing.
1164	1165	*
1165	1166	* CONTEXT:
1166		− * spin_lock_irq(pool->lock).
	1167	+ * raw_spin_lock_irq(pool->lock).
1167	1168	*/
1168	1169	static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, int color)
1169	1170	{
@@ -1262,7 +1263,7 @@ static int try_to_grab_pending(struct work_struct *work, bool is_dwork, · 92 unmodified lines
1262	1263	if (!pool)
1263	1264	goto fail;
1264	1265
1265		− spin_lock(&pool->lock);
	1266	+ raw_spin_lock(&pool->lock);
1266	1267	/*
1267	1268	* work->data is guaranteed to point to pwq only while the work
1268	1269	* item is queued on pwq->wq, and both updating work->data to point
@@ -1291,11 +1292,11 @@ static int try_to_grab_pending(struct work_struct *work, bool is_dwork, · 22 unmodified lines
1291	1292	/* work->data points to pwq iff queued, point to pool */
1292	1293	set_work_pool_and_keep_pending(work, pool->id);
1293	1294
1294		− spin_unlock(&pool->lock);
	1295	+ raw_spin_unlock(&pool->lock);
1295	1296	rcu_read_unlock();
1296	1297	return 1;
1297	1298	}
1298		− spin_unlock(&pool->lock);
	1299	+ raw_spin_unlock(&pool->lock);
1299	1300	fail:
1300	1301	rcu_read_unlock();
1301	1302	local_irq_restore(*flags);
@@ -1316,7 +1317,7 @@ fail: · 14 unmodified lines
1316	1317	* work_struct flags.
1317	1318	*
1318	1319	* CONTEXT:
1319		− * spin_lock_irq(pool->lock).
	1320	+ * raw_spin_lock_irq(pool->lock).
1320	1321	*/
1321	1322	static void insert_work(struct pool_workqueue pwq, struct work_struct work,
1322	1323	struct list_head *head, unsigned int extra_flags)
@@ -1433,7 +1434,7 @@ retry: · 110 unmodified lines
1433	1434	if (last_pool && last_pool != pwq->pool) {
1434	1435	struct worker *worker;
1435	1436
1436		− spin_lock(&last_pool->lock);
	1437	+ raw_spin_lock(&last_pool->lock);
1437	1438
1438	1439	worker = find_worker_executing_work(last_pool, work);
1439	1440
@@ -1441,11 +1442,11 @@ retry: · 1 unmodified line
1441	1442	pwq = worker->current_pwq;
1442	1443	} else {
1443	1444	/* meh... not running there, queue here */
1444		− spin_unlock(&last_pool->lock);
1445		− spin_lock(&pwq->pool->lock);
	1445	+ raw_spin_unlock(&last_pool->lock);
	1446	+ raw_spin_lock(&pwq->pool->lock);
1446	1447	}
1447	1448	} else {
1448		− spin_lock(&pwq->pool->lock);
	1449	+ raw_spin_lock(&pwq->pool->lock);
1449	1450	}
1450	1451
1451	1452	/*
@@ -1458,7 +1459,7 @@ retry: · 6 unmodified lines
1458	1459	*/
1459	1460	if (unlikely(!pwq->refcnt)) {
1460	1461	if (wq->flags & WQ_UNBOUND) {
1461		− spin_unlock(&pwq->pool->lock);
	1462	+ raw_spin_unlock(&pwq->pool->lock);
1462	1463	cpu_relax();
1463	1464	goto retry;
1464	1465	}
@@ -1490,7 +1491,7 @@ retry: · 25 unmodified lines
1490	1491	insert_work(pwq, work, worklist, work_flags);
1491	1492
1492	1493	out:
1493		− spin_unlock(&pwq->pool->lock);
	1494	+ raw_spin_unlock(&pwq->pool->lock);
1494	1495	rcu_read_unlock();
1495	1496	}
1496	1497
@@ -1759,7 +1760,7 @@ EXPORT_SYMBOL(queue_rcu_work); · 262 unmodified lines
1759	1760	* necessary.
1760	1761	*
1761	1762	* LOCKING:
1762		− * spin_lock_irq(pool->lock).
	1763	+ * raw_spin_lock_irq(pool->lock).
1763	1764	*/
1764	1765	static void worker_enter_idle(struct worker *worker)
1765	1766	{
@@ -1799,7 +1800,7 @@ static void worker_enter_idle(struct worker *worker) · 33 unmodified lines
1799	1800	* @worker is leaving idle state. Update stats.
1800	1801	*
1801	1802	* LOCKING:
1802		− * spin_lock_irq(pool->lock).
	1803	+ * raw_spin_lock_irq(pool->lock).
1803	1804	*/
1804	1805	static void worker_leave_idle(struct worker *worker)
1805	1806	{
@@ -1937,11 +1938,11 @@ static struct worker create_worker(struct worker_pool pool) · 131 unmodified lines
1937	1938	worker_attach_to_pool(worker, pool);
1938	1939
1939	1940	/* start the newly created worker */
1940		− spin_lock_irq(&pool->lock);
	1941	+ raw_spin_lock_irq(&pool->lock);
1941	1942	worker->pool->nr_workers++;
1942	1943	worker_enter_idle(worker);
1943	1944	wake_up_process(worker->task);
1944		− spin_unlock_irq(&pool->lock);
	1945	+ raw_spin_unlock_irq(&pool->lock);
1945	1946
1946	1947	return worker;
1947	1948
@@ -1960,7 +1961,7 @@ fail: · 12 unmodified lines
1960	1961	* be idle.
1961	1962	*
1962	1963	* CONTEXT:
1963		− * spin_lock_irq(pool->lock).
	1964	+ * raw_spin_lock_irq(pool->lock).
1964	1965	*/
1965	1966	static void destroy_worker(struct worker *worker)
1966	1967	{
@@ -1986,7 +1987,7 @@ static void idle_worker_timeout(struct timer_list *t) · 19 unmodified lines
1986	1987	{
1987	1988	struct worker_pool *pool = from_timer(pool, t, idle_timer);
1988	1989
1989		− spin_lock_irq(&pool->lock);
	1990	+ raw_spin_lock_irq(&pool->lock);
1990	1991
1991	1992	while (too_many_workers(pool)) {
1992	1993	struct worker *worker;
@@ -2004,7 +2005,7 @@ static void idle_worker_timeout(struct timer_list *t) · 11 unmodified lines
2004	2005	destroy_worker(worker);
2005	2006	}
2006	2007
2007		− spin_unlock_irq(&pool->lock);
	2008	+ raw_spin_unlock_irq(&pool->lock);
2008	2009	}
2009	2010
2010	2011	static void send_mayday(struct work_struct *work)
@@ -2035,8 +2036,8 @@ static void pool_mayday_timeout(struct timer_list *t) · 24 unmodified lines
2035	2036	struct worker_pool *pool = from_timer(pool, t, mayday_timer);
2036	2037	struct work_struct *work;
2037	2038
2038		− spin_lock_irq(&pool->lock);
2039		− spin_lock(&wq_mayday_lock); /* for wq->maydays */
	2039	+ raw_spin_lock_irq(&pool->lock);
	2040	+ raw_spin_lock(&wq_mayday_lock); /* for wq->maydays */
2040	2041
2041	2042	if (need_to_create_worker(pool)) {
2042	2043	/*
@@ -2049,8 +2050,8 @@ static void pool_mayday_timeout(struct timer_list *t) · 6 unmodified lines
2049	2050	send_mayday(work);
2050	2051	}
2051	2052
2052		− spin_unlock(&wq_mayday_lock);
2053		− spin_unlock_irq(&pool->lock);
	2053	+ raw_spin_unlock(&wq_mayday_lock);
	2054	+ raw_spin_unlock_irq(&pool->lock);
2054	2055
2055	2056	mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);
2056	2057	}
@@ -2069,7 +2070,7 @@ static void pool_mayday_timeout(struct timer_list *t) · 12 unmodified lines
2069	2070	* may_start_working() %true.
2070	2071	*
2071	2072	* LOCKING:
2072		− * spin_lock_irq(pool->lock) which may be released and regrabbed
	2073	+ * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
2073	2074	* multiple times. Does GFP_KERNEL allocations. Called only from
2074	2075	* manager.
2075	2076	*/
@@ -2078,7 +2079,7 @@ __releases(&pool->lock) · 2 unmodified lines
2078	2079	__acquires(&pool->lock)
2079	2080	{
2080	2081	restart:
2081		− spin_unlock_irq(&pool->lock);
	2082	+ raw_spin_unlock_irq(&pool->lock);
2082	2083
2083	2084	/* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */
2084	2085	mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
@@ -2094,7 +2095,7 @@ restart: · 9 unmodified lines
2094	2095	}
2095	2096
2096	2097	del_timer_sync(&pool->mayday_timer);
2097		− spin_lock_irq(&pool->lock);
	2098	+ raw_spin_lock_irq(&pool->lock);
2098	2099	/*
2099	2100	* This is necessary even after a new worker was just successfully
2100	2101	* created as @pool->lock was dropped and the new worker might have
@@ -2117,7 +2118,7 @@ restart: · 16 unmodified lines
2117	2118	* and may_start_working() is true.
2118	2119	*
2119	2120	* CONTEXT:
2120		− * spin_lock_irq(pool->lock) which may be released and regrabbed
	2121	+ * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
2121	2122	* multiple times. Does GFP_KERNEL allocations.
2122	2123	*
2123	2124	* Return:
@@ -2140,7 +2141,7 @@ static bool manage_workers(struct worker *worker) · 16 unmodified lines
2140	2141
2141	2142	pool->manager = NULL;
2142	2143	pool->flags &= ~POOL_MANAGER_ACTIVE;
2143		− wake_up(&wq_manager_wait);
	2144	+ rcuwait_wake_up(&manager_wait);
2144	2145	return true;
2145	2146	}
2146	2147
@@ -2156,7 +2157,7 @@ static bool manage_workers(struct worker *worker) · 9 unmodified lines
2156	2157	* call this function to process a work.
2157	2158	*
2158	2159	* CONTEXT:
2159		− * spin_lock_irq(pool->lock) which is released and regrabbed.
	2160	+ * raw_spin_lock_irq(pool->lock) which is released and regrabbed.
2160	2161	*/
2161	2162	static void process_one_work(struct worker worker, struct work_struct work)
2162	2163	__releases(&pool->lock)
@@ -2238,7 +2239,7 @@ __acquires(&pool->lock) · 75 unmodified lines
2238	2239	*/
2239	2240	set_work_pool_and_clear_pending(work, pool->id);
2240	2241
2241		− spin_unlock_irq(&pool->lock);
	2242	+ raw_spin_unlock_irq(&pool->lock);
2242	2243
2243	2244	lock_map_acquire(&pwq->wq->lockdep_map);
2244	2245	lock_map_acquire(&lockdep_map);
@@ -2293,7 +2294,7 @@ __acquires(&pool->lock) · 48 unmodified lines
2293	2294	*/
2294	2295	cond_resched();
2295	2296
2296		− spin_lock_irq(&pool->lock);
	2297	+ raw_spin_lock_irq(&pool->lock);
2297	2298
2298	2299	/* clear cpu intensive status */
2299	2300	if (unlikely(cpu_intensive))
@@ -2319,7 +2320,7 @@ __acquires(&pool->lock) · 19 unmodified lines
2319	2320	* fetches a work from the top and executes it.
2320	2321	*
2321	2322	* CONTEXT:
2322		− * spin_lock_irq(pool->lock) which may be released and regrabbed
	2323	+ * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
2323	2324	* multiple times.
2324	2325	*/
2325	2326	static void process_scheduled_works(struct worker *worker)
@@ -2361,11 +2362,11 @@ static int worker_thread(void *__worker) · 35 unmodified lines
2361	2362	/* tell the scheduler that this is a workqueue worker */
2362	2363	set_pf_worker(true);
2363	2364	woke_up:
2364		− spin_lock_irq(&pool->lock);
	2365	+ raw_spin_lock_irq(&pool->lock);
2365	2366
2366	2367	/* am I supposed to die? */
2367	2368	if (unlikely(worker->flags & WORKER_DIE)) {
2368		− spin_unlock_irq(&pool->lock);
	2369	+ raw_spin_unlock_irq(&pool->lock);
2369	2370	WARN_ON_ONCE(!list_empty(&worker->entry));
2370	2371	set_pf_worker(false);
2371	2372
@@ -2431,7 +2432,7 @@ sleep: · 59 unmodified lines
2431	2432	*/
2432	2433	worker_enter_idle(worker);
2433	2434	__set_current_state(TASK_IDLE);
2434		− spin_unlock_irq(&pool->lock);
	2435	+ raw_spin_unlock_irq(&pool->lock);
2435	2436	schedule();
2436	2437	goto woke_up;
2437	2438	}
@@ -2485,7 +2486,7 @@ repeat: · 47 unmodified lines
2485	2486	should_stop = kthread_should_stop();
2486	2487
2487	2488	/* see whether any pwq is asking for help */
2488		− spin_lock_irq(&wq_mayday_lock);
	2489	+ raw_spin_lock_irq(&wq_mayday_lock);
2489	2490
2490	2491	while (!list_empty(&wq->maydays)) {
2491	2492	struct pool_workqueue *pwq = list_first_entry(&wq->maydays,
@@ -2497,11 +2498,11 @@ repeat: · 5 unmodified lines
2497	2498	__set_current_state(TASK_RUNNING);
2498	2499	list_del_init(&pwq->mayday_node);
2499	2500
2500		− spin_unlock_irq(&wq_mayday_lock);
	2501	+ raw_spin_unlock_irq(&wq_mayday_lock);
2501	2502
2502	2503	worker_attach_to_pool(rescuer, pool);
2503	2504
2504		− spin_lock_irq(&pool->lock);
	2505	+ raw_spin_lock_irq(&pool->lock);
2505	2506
2506	2507	/*
2507	2508	* Slurp in all works issued via this workqueue and
@@ -2529,8 +2530,8 @@ repeat: · 21 unmodified lines
2529	2530	* being used to relieve memory pressure, don't
2530	2531	* incur MAYDAY_INTERVAL delay inbetween.
2531	2532	*/
2532		− if (need_to_create_worker(pool)) {
2533		− spin_lock(&wq_mayday_lock);
	2533	+ if (pwq->nr_active && need_to_create_worker(pool)) {
	2534	+ raw_spin_lock(&wq_mayday_lock);
2534	2535	/*
2535	2536	* Queue iff we aren't racing destruction
2536	2537	* and somebody else hasn't queued it already.
@@ -2539,7 +2540,7 @@ repeat: · 2 unmodified lines
2539	2540	get_pwq(pwq);
2540	2541	list_add_tail(&pwq->mayday_node, &wq->maydays);
2541	2542	}
2542		− spin_unlock(&wq_mayday_lock);
	2543	+ raw_spin_unlock(&wq_mayday_lock);
2543	2544	}
2544	2545	}
2545	2546
@@ -2557,14 +2558,14 @@ repeat: · 11 unmodified lines
2557	2558	if (need_more_worker(pool))
2558	2559	wake_up_worker(pool);
2559	2560
2560		− spin_unlock_irq(&pool->lock);
	2561	+ raw_spin_unlock_irq(&pool->lock);
2561	2562
2562	2563	worker_detach_from_pool(rescuer);
2563	2564
2564		− spin_lock_irq(&wq_mayday_lock);
	2565	+ raw_spin_lock_irq(&wq_mayday_lock);
2565	2566	}
2566	2567
2567		− spin_unlock_irq(&wq_mayday_lock);
	2568	+ raw_spin_unlock_irq(&wq_mayday_lock);
2568	2569
2569	2570	if (should_stop) {
2570	2571	__set_current_state(TASK_RUNNING);
@@ -2644,7 +2645,7 @@ static void wq_barrier_func(struct work_struct *work) · 73 unmodified lines
2644	2645	* underneath us, so we can't reliably determine pwq from @target.
2645	2646	*
2646	2647	* CONTEXT:
2647		− * spin_lock_irq(pool->lock).
	2648	+ * raw_spin_lock_irq(pool->lock).
2648	2649	*/
2649	2650	static void insert_wq_barrier(struct pool_workqueue *pwq,
2650	2651	struct wq_barrier *barr,
@@ -2731,7 +2732,7 @@ static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq, · 80 unmodified lines
2731	2732	for_each_pwq(pwq, wq) {
2732	2733	struct worker_pool *pool = pwq->pool;
2733	2734
2734		− spin_lock_irq(&pool->lock);
	2735	+ raw_spin_lock_irq(&pool->lock);
2735	2736
2736	2737	if (flush_color >= 0) {
2737	2738	WARN_ON_ONCE(pwq->flush_color != -1);
@@ -2748,7 +2749,7 @@ static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq, · 10 unmodified lines
2748	2749	pwq->work_color = work_color;
2749	2750	}
2750	2751
2751		− spin_unlock_irq(&pool->lock);
	2752	+ raw_spin_unlock_irq(&pool->lock);
2752	2753	}
2753	2754
2754	2755	if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))
@@ -2948,9 +2949,9 @@ reflush: · 193 unmodified lines
2948	2949	for_each_pwq(pwq, wq) {
2949	2950	bool drained;
2950	2951
2951		− spin_lock_irq(&pwq->pool->lock);
	2952	+ raw_spin_lock_irq(&pwq->pool->lock);
2952	2953	drained = !pwq->nr_active && list_empty(&pwq->delayed_works);
2953		− spin_unlock_irq(&pwq->pool->lock);
	2954	+ raw_spin_unlock_irq(&pwq->pool->lock);
2954	2955
2955	2956	if (drained)
2956	2957	continue;
@@ -2986,7 +2987,7 @@ static bool start_flush_work(struct work_struct work, struct wq_barrier barr, · 29 unmodified lines
2986	2987	return false;
2987	2988	}
2988	2989
2989		− spin_lock_irq(&pool->lock);
	2990	+ raw_spin_lock_irq(&pool->lock);
2990	2991	/* see the comment in try_to_grab_pending() with the same code */
2991	2992	pwq = get_work_pwq(work);
2992	2993	if (pwq) {
@@ -3002,7 +3003,7 @@ static bool start_flush_work(struct work_struct work, struct wq_barrier barr, · 9 unmodified lines
3002	3003	check_flush_dependency(pwq->wq, work);
3003	3004
3004	3005	insert_wq_barrier(pwq, barr, work, worker);
3005		− spin_unlock_irq(&pool->lock);
	3006	+ raw_spin_unlock_irq(&pool->lock);
3006	3007
3007	3008	/*
3008	3009	* Force a lock recursion deadlock when using flush_work() inside a
@@ -3021,7 +3022,7 @@ static bool start_flush_work(struct work_struct work, struct wq_barrier barr, · 12 unmodified lines
3021	3022	rcu_read_unlock();
3022	3023	return true;
3023	3024	already_gone:
3024		− spin_unlock_irq(&pool->lock);
	3025	+ raw_spin_unlock_irq(&pool->lock);
3025	3026	rcu_read_unlock();
3026	3027	return false;
3027	3028	}
@@ -3414,7 +3415,7 @@ static bool wqattrs_equal(const struct workqueue_attrs *a, · 386 unmodified lines
3414	3415	*/
3415	3416	static int init_worker_pool(struct worker_pool *pool)
3416	3417	{
3417		− spin_lock_init(&pool->lock);
	3418	+ raw_spin_lock_init(&pool->lock);
3418	3419	pool->id = -1;
3419	3420	pool->cpu = -1;
3420	3421	pool->node = NUMA_NO_NODE;
@@ -3491,7 +3492,6 @@ static void rcu_free_wq(struct rcu_head *rcu) · 70 unmodified lines
3491	3492	else
3492	3493	free_workqueue_attrs(wq->unbound_attrs);
3493	3494
3494		− kfree(wq->rescuer);
3495	3495	kfree(wq);
3496	3496	}
3497	3497
@@ -3504,6 +3504,18 @@ static void rcu_free_pool(struct rcu_head *rcu) · 6 unmodified lines
3504	3504	kfree(pool);
3505	3505	}
3506	3506
	3507	+/* This returns with the lock held on success (pool manager is inactive). */
	3508	+static bool wq_manager_inactive(struct worker_pool *pool)
	3509	+{
	3510	+ raw_spin_lock_irq(&pool->lock);
	3511	+
	3512	+ if (pool->flags & POOL_MANAGER_ACTIVE) {
	3513	+ raw_spin_unlock_irq(&pool->lock);
	3514	+ return false;
	3515	+ }
	3516	+ return true;
	3517	+}
	3518	+
3507	3519	/**
3508	3520	* put_unbound_pool - put a worker_pool
3509	3521	* @pool: worker_pool to put
@@ -3539,16 +3551,17 @@ static void put_unbound_pool(struct worker_pool *pool) · 29 unmodified lines
3539	3551	* Become the manager and destroy all workers. This prevents
3540	3552	* @pool's workers from blocking on attach_mutex. We're the last
3541	3553	* manager and @pool gets freed with the flag set.
	3554	+ * Because of how wq_manager_inactive() works, we will hold the
	3555	+ * spinlock after a successful wait.
3542	3556	*/
3543		− spin_lock_irq(&pool->lock);
3544		− wait_event_lock_irq(wq_manager_wait,
3545		− !(pool->flags & POOL_MANAGER_ACTIVE), pool->lock);
	3557	+ rcuwait_wait_event(&manager_wait, wq_manager_inactive(pool),
	3558	+ TASK_UNINTERRUPTIBLE);
3546	3559	pool->flags \|= POOL_MANAGER_ACTIVE;
3547	3560
3548	3561	while ((worker = first_idle_worker(pool)))
3549	3562	destroy_worker(worker);
3550	3563	WARN_ON(pool->nr_workers \|\| pool->nr_idle);
3551		− spin_unlock_irq(&pool->lock);
	3564	+ raw_spin_unlock_irq(&pool->lock);
3552	3565
3553	3566	mutex_lock(&wq_pool_attach_mutex);
3554	3567	if (!list_empty(&pool->workers))
@@ -3704,7 +3717,7 @@ static void pwq_adjust_max_active(struct pool_workqueue *pwq) · 149 unmodified lines
3704	3717	return;
3705	3718
3706	3719	/* this function can be called during early boot w/ irq disabled */
3707		− spin_lock_irqsave(&pwq->pool->lock, flags);
	3720	+ raw_spin_lock_irqsave(&pwq->pool->lock, flags);
3708	3721
3709	3722	/*
3710	3723	* During [un]freezing, the caller is responsible for ensuring that
@@ -3727,7 +3740,7 @@ static void pwq_adjust_max_active(struct pool_workqueue *pwq) · 16 unmodified lines
3727	3740	pwq->max_active = 0;
3728	3741	}
3729	3742
3730		− spin_unlock_irqrestore(&pwq->pool->lock, flags);
	3743	+ raw_spin_unlock_irqrestore(&pwq->pool->lock, flags);
3731	3744	}
3732	3745
3733	3746	/* initialize newly alloced @pwq which is associated with @wq and @pool */
@@ -4129,9 +4142,9 @@ static void wq_update_unbound_numa(struct workqueue_struct *wq, int cpu, · 395 unmodified lines
4129	4142
4130	4143	use_dfl_pwq:
4131	4144	mutex_lock(&wq->mutex);
4132		− spin_lock_irq(&wq->dfl_pwq->pool->lock);
	4145	+ raw_spin_lock_irq(&wq->dfl_pwq->pool->lock);
4133	4146	get_pwq(wq->dfl_pwq);
4134		− spin_unlock_irq(&wq->dfl_pwq->pool->lock);
	4147	+ raw_spin_unlock_irq(&wq->dfl_pwq->pool->lock);
4135	4148	old_pwq = numa_pwq_tbl_install(wq, node, wq->dfl_pwq);
4136	4149	out_unlock:
4137	4150	mutex_unlock(&wq->mutex);
@@ -4208,8 +4221,8 @@ static int init_rescuer(struct workqueue_struct *wq) · 70 unmodified lines
4208	4221
4209	4222	rescuer->rescue_wq = wq;
4210	4223	rescuer->task = kthread_create(rescuer_thread, rescuer, "%s", wq->name);
4211		− ret = PTR_ERR_OR_ZERO(rescuer->task);
4212		− if (ret) {
	4224	+ if (IS_ERR(rescuer->task)) {
	4225	+ ret = PTR_ERR(rescuer->task);
4213	4226	kfree(rescuer);
4214	4227	return ret;
4215	4228	}
@@ -4360,9 +4373,9 @@ void destroy_workqueue(struct workqueue_struct *wq) · 144 unmodified lines
4360	4373	struct worker *rescuer = wq->rescuer;
4361	4374
4362	4375	/* this prevents new queueing */
4363		− spin_lock_irq(&wq_mayday_lock);
	4376	+ raw_spin_lock_irq(&wq_mayday_lock);
4364	4377	wq->rescuer = NULL;
4365		− spin_unlock_irq(&wq_mayday_lock);
	4378	+ raw_spin_unlock_irq(&wq_mayday_lock);
4366	4379
4367	4380	/* rescuer will empty maydays list before exiting */
4368	4381	kthread_stop(rescuer->task);
@@ -4376,27 +4389,25 @@ void destroy_workqueue(struct workqueue_struct *wq) · 7 unmodified lines
4376	4389	mutex_lock(&wq_pool_mutex);
4377	4390	mutex_lock(&wq->mutex);
4378	4391	for_each_pwq(pwq, wq) {
4379		− spin_lock_irq(&pwq->pool->lock);
	4392	+ raw_spin_lock_irq(&pwq->pool->lock);
4380	4393	if (WARN_ON(pwq_busy(pwq))) {
4381	4394	pr_warn("%s: %s has the following busy pwq\n",
4382	4395	__func__, wq->name);
4383	4396	show_pwq(pwq);
4384		− spin_unlock_irq(&pwq->pool->lock);
	4397	+ raw_spin_unlock_irq(&pwq->pool->lock);
4385	4398	mutex_unlock(&wq->mutex);
4386	4399	mutex_unlock(&wq_pool_mutex);
4387	4400	show_workqueue_state();
4388	4401	return;
4389	4402	}
4390		− spin_unlock_irq(&pwq->pool->lock);
	4403	+ raw_spin_unlock_irq(&pwq->pool->lock);
4391	4404	}
4392	4405	mutex_unlock(&wq->mutex);
4393		− mutex_unlock(&wq_pool_mutex);
4394	4406
4395	4407	/*
4396	4408	* wq list is used to freeze wq, remove from list after
4397	4409	* flushing is complete in case freeze races us.
4398	4410	*/
4399		− mutex_lock(&wq_pool_mutex);
4400	4411	list_del_rcu(&wq->list);
4401	4412	mutex_unlock(&wq_pool_mutex);
4402	4413
@@ -4558,10 +4569,10 @@ unsigned int work_busy(struct work_struct *work) · 155 unmodified lines
4558	4569	rcu_read_lock();
4559	4570	pool = get_work_pool(work);
4560	4571	if (pool) {
4561		− spin_lock_irqsave(&pool->lock, flags);
	4572	+ raw_spin_lock_irqsave(&pool->lock, flags);
4562	4573	if (find_worker_executing_work(pool, work))
4563	4574	ret \|= WORK_BUSY_RUNNING;
4564		− spin_unlock_irqrestore(&pool->lock, flags);
	4575	+ raw_spin_unlock_irqrestore(&pool->lock, flags);
4565	4576	}
4566	4577	rcu_read_unlock();
4567	4578
@@ -4768,10 +4779,10 @@ void show_workqueue_state(void) · 200 unmodified lines
4768	4779	pr_info("workqueue %s: flags=0x%x\n", wq->name, wq->flags);
4769	4780
4770	4781	for_each_pwq(pwq, wq) {
4771		− spin_lock_irqsave(&pwq->pool->lock, flags);
	4782	+ raw_spin_lock_irqsave(&pwq->pool->lock, flags);
4772	4783	if (pwq->nr_active \|\| !list_empty(&pwq->delayed_works))
4773	4784	show_pwq(pwq);
4774		− spin_unlock_irqrestore(&pwq->pool->lock, flags);
	4785	+ raw_spin_unlock_irqrestore(&pwq->pool->lock, flags);
4775	4786	/*
4776	4787	* We could be printing a lot from atomic context, e.g.
4777	4788	* sysrq-t -> show_workqueue_state(). Avoid triggering
@@ -4785,7 +4796,7 @@ void show_workqueue_state(void) · 7 unmodified lines
4785	4796	struct worker *worker;
4786	4797	bool first = true;
4787	4798
4788		− spin_lock_irqsave(&pool->lock, flags);
	4799	+ raw_spin_lock_irqsave(&pool->lock, flags);
4789	4800	if (pool->nr_workers == pool->nr_idle)
4790	4801	goto next_pool;
4791	4802
@@ -4804,7 +4815,7 @@ void show_workqueue_state(void) · 12 unmodified lines
4804	4815	}
4805	4816	pr_cont("\n");
4806	4817	next_pool:
4807		− spin_unlock_irqrestore(&pool->lock, flags);
	4818	+ raw_spin_unlock_irqrestore(&pool->lock, flags);
4808	4819	/*
4809	4820	* We could be printing a lot from atomic context, e.g.
4810	4821	* sysrq-t -> show_workqueue_state(). Avoid triggering
@@ -4834,7 +4845,7 @@ void wq_worker_comm(char buf, size_t size, struct task_struct task) · 23 unmodified lines
4834	4845	struct worker_pool *pool = worker->pool;
4835	4846
4836	4847	if (pool) {
4837		− spin_lock_irq(&pool->lock);
	4848	+ raw_spin_lock_irq(&pool->lock);
4838	4849	/*
4839	4850	* ->desc tracks information (wq name or
4840	4851	* set_worker_desc()) for the latest execution. If
@@ -4848,7 +4859,7 @@ void wq_worker_comm(char buf, size_t size, struct task_struct task) · 7 unmodified lines
4848	4859	scnprintf(buf + off, size - off, "-%s",
4849	4860	worker->desc);
4850	4861	}
4851		− spin_unlock_irq(&pool->lock);
	4862	+ raw_spin_unlock_irq(&pool->lock);
4852	4863	}
4853	4864	}
4854	4865
@@ -4879,7 +4890,7 @@ static void unbind_workers(int cpu) · 24 unmodified lines
4879	4890
4880	4891	for_each_cpu_worker_pool(pool, cpu) {
4881	4892	mutex_lock(&wq_pool_attach_mutex);
4882		− spin_lock_irq(&pool->lock);
	4893	+ raw_spin_lock_irq(&pool->lock);
4883	4894
4884	4895	/*
4885	4896	* We've blocked all attach/detach operations. Make all workers
@@ -4893,7 +4904,7 @@ static void unbind_workers(int cpu) · 7 unmodified lines
4893	4904
4894	4905	pool->flags \|= POOL_DISASSOCIATED;
4895	4906
4896		− spin_unlock_irq(&pool->lock);
	4907	+ raw_spin_unlock_irq(&pool->lock);
4897	4908	mutex_unlock(&wq_pool_attach_mutex);
4898	4909
4899	4910	/*
@@ -4919,9 +4930,9 @@ static void unbind_workers(int cpu) · 19 unmodified lines
4919	4930	* worker blocking could lead to lengthy stalls. Kick off
4920	4931	* unbound chain execution of currently pending work items.
4921	4932	*/
4922		− spin_lock_irq(&pool->lock);
	4933	+ raw_spin_lock_irq(&pool->lock);
4923	4934	wake_up_worker(pool);
4924		− spin_unlock_irq(&pool->lock);
	4935	+ raw_spin_unlock_irq(&pool->lock);
4925	4936	}
4926	4937	}
4927	4938
@@ -4948,7 +4959,7 @@ static void rebind_workers(struct worker_pool *pool) · 20 unmodified lines
4948	4959	WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
4949	4960	pool->attrs->cpumask) < 0);
4950	4961
4951		− spin_lock_irq(&pool->lock);
	4962	+ raw_spin_lock_irq(&pool->lock);
4952	4963
4953	4964	pool->flags &= ~POOL_DISASSOCIATED;
4954	4965
@@ -4987,7 +4998,7 @@ static void rebind_workers(struct worker_pool *pool) · 32 unmodified lines
4987	4998	WRITE_ONCE(worker->flags, worker_flags);
4988	4999	}
4989	5000
4990		− spin_unlock_irq(&pool->lock);
	5001	+ raw_spin_unlock_irq(&pool->lock);
4991	5002	}
4992	5003
4993	5004	/**
@@ -5906,7 +5917,7 @@ void __init workqueue_init_early(void) · 912 unmodified lines
5906	5917	int hk_flags = HK_FLAG_DOMAIN \| HK_FLAG_WQ;
5907	5918	int i, cpu;
5908	5919
5909		− WARN_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));
	5920	+ BUILD_BUG_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));
5910	5921
5911	5922	BUG_ON(!alloc_cpumask_var(&wq_unbound_cpumask, GFP_KERNEL));
5912	5923	cpumask_copy(wq_unbound_cpumask, housekeeping_cpumask(hk_flags));