History

workqueue: separate out pool and workqueue locking into wq_mutex

Currently, workqueue_lock protects most shared workqueue resources - the pools, workqueues, pool_workqueues, draining, ID assignments, mayday handling and so on. The coverage has grown organically and there is no identified bottleneck coming from workqueue_lock, but it has grown a bit too much and scheduled rebinding changes need the pools and workqueues to be protected by a mutex instead of a spinlock. This patch breaks out pool and workqueue synchronization from workqueue_lock into a new mutex - wq_mutex. The followings are protected by wq_mutex. * worker_pool_idr and unbound_pool_hash * pool->refcnt * workqueues list * workqueue->flags, ->nr_drainers Most changes are mostly straight-forward. workqueue_lock is replaced with wq_mutex where applicable and workqueue_lock lock/unlocks are added where wq_mutex conversion leaves data structures not protected by wq_mutex without locking. irq / preemption flippings were added where the conversion affects them. Things worth noting are * New WQ and WR locking lables added along with assert_rcu_or_wq_mutex(). * worker_pool_assign_id() now expects to be called under wq_mutex. * create_mutex is removed from get_unbound_pool(). It now just holds wq_mutex. This patch shouldn't introduce any visible behavior changes.

Tejun Heo committed 13 years ago

Commit 5bcab331 parent 7d19c5c1 file · +77 −69Browse files

Showing the changes to kernel/workqueue.c (1 of 1 file). Show all

Unified · Split

kernel/workqueue.c+77−69
@@ -119,9 +119,11 @@ enum { · 118 unmodified lines
119119 *
120120 * F: wq->flush_mutex protected.
121121 *
122− * W: workqueue_lock protected.
122+ * WQ: wq_mutex protected.
123+ *
124+ * WR: wq_mutex protected for writes. Sched-RCU protected for reads.
123125 *
124− * R: workqueue_lock protected for writes. Sched-RCU protected for reads.
126+ * W: workqueue_lock protected.
125127 *
126128 * FR: wq->flush_mutex and workqueue_lock protected for writes. Sched-RCU
127129 * protected for reads.
@@ -155,8 +157,8 @@ struct worker_pool { · 27 unmodified lines
155157 struct ida worker_ida; /* L: for worker IDs */
156158
157159 struct workqueue_attrs *attrs; /* I: worker attributes */
158− struct hlist_node hash_node; /* W: unbound_pool_hash node */
159− int refcnt; /* W: refcnt for unbound pools */
160+ struct hlist_node hash_node; /* WQ: unbound_pool_hash node */
161+ int refcnt; /* WQ: refcnt for unbound pools */
160162
161163 /*
162164 * The current concurrency level. As it's likely to be accessed
@@ -218,10 +220,10 @@ struct wq_device; · 55 unmodified lines
218220 * the appropriate worker_pool through its pool_workqueues.
219221 */
220222 struct workqueue_struct {
221− unsigned int flags; /* W: WQ_* flags */
223+ unsigned int flags; /* WQ: WQ_* flags */
222224 struct pool_workqueue __percpu *cpu_pwqs; /* I: per-cpu pwq's */
223225 struct list_head pwqs; /* FR: all pwqs of this wq */
224− struct list_head list; /* W: list of all workqueues */
226+ struct list_head list; /* WQ: list of all workqueues */
225227
226228 struct mutex flush_mutex; /* protects wq flushing */
227229 int work_color; /* F: current work color */
@@ -234,7 +236,7 @@ struct workqueue_struct { · 6 unmodified lines
234236 struct list_head maydays; /* W: pwqs requesting rescue */
235237 struct worker *rescuer; /* I: rescue worker */
236238
237− int nr_drainers; /* W: drain in progress */
239+ int nr_drainers; /* WQ: drain in progress */
238240 int saved_max_active; /* W: saved pwq max_active */
239241
240242 #ifdef CONFIG_SYSFS
@@ -248,22 +250,19 @@ struct workqueue_struct { · 7 unmodified lines
248250
249251 static struct kmem_cache *pwq_cache;
250252
251−/* Serializes the accesses to the list of workqueues. */
253+static DEFINE_MUTEX(wq_mutex); /* protects workqueues and pools */
252254 static DEFINE_SPINLOCK(workqueue_lock);
253−static LIST_HEAD(workqueues);
254−static bool workqueue_freezing; /* W: have wqs started freezing? */
255+
256+static LIST_HEAD(workqueues); /* WQ: list of all workqueues */
257+static bool workqueue_freezing; /* WQ: have wqs started freezing? */
255258
256259 /* the per-cpu worker pools */
257260 static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS],
258261 cpu_worker_pools);
259262
260−/*
261− * R: idr of all pools. Modifications are protected by workqueue_lock.
262− * Read accesses are protected by sched-RCU protected.
263− */
264−static DEFINE_IDR(worker_pool_idr);
263+static DEFINE_IDR(worker_pool_idr); /* WR: idr of all pools */
265264
266−/* W: hash of all unbound pools keyed by pool->attrs */
265+/* WQ: hash of all unbound pools keyed by pool->attrs */
267266 static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER);
268267
269268 /* I: attributes used when instantiating standard unbound pools on demand */
@@ -287,6 +286,11 @@ static void copy_workqueue_attrs(struct workqueue_attrs *to, · 17 unmodified lines
287286 #define CREATE_TRACE_POINTS
288287 #include <trace/events/workqueue.h>
289288
289+#define assert_rcu_or_wq_mutex() \
290+ rcu_lockdep_assert(rcu_read_lock_sched_held() || \
291+ lockdep_is_held(&wq_mutex), \
292+ "sched RCU or wq_mutex should be held")
293+
290294 #define assert_rcu_or_wq_lock() \
291295 rcu_lockdep_assert(rcu_read_lock_sched_held() || \
292296 lockdep_is_held(&workqueue_lock), \
@@ -305,16 +309,16 @@ static void copy_workqueue_attrs(struct workqueue_attrs *to, · 12 unmodified lines
305309 * @pool: iteration cursor
306310 * @pi: integer used for iteration
307311 *
308− * This must be called either with workqueue_lock held or sched RCU read
309− * locked. If the pool needs to be used beyond the locking in effect, the
310− * caller is responsible for guaranteeing that the pool stays online.
312+ * This must be called either with wq_mutex held or sched RCU read locked.
313+ * If the pool needs to be used beyond the locking in effect, the caller is
314+ * responsible for guaranteeing that the pool stays online.
311315 *
312316 * The if/else clause exists only for the lockdep assertion and can be
313317 * ignored.
314318 */
315319 #define for_each_pool(pool, pi) \
316320 idr_for_each_entry(&worker_pool_idr, pool, pi) \
317− if (({ assert_rcu_or_wq_lock(); false; })) { } \
321+ if (({ assert_rcu_or_wq_mutex(); false; })) { } \
318322 else
319323
320324 /**
@@ -455,13 +459,12 @@ static int worker_pool_assign_id(struct worker_pool *pool) · 134 unmodified lines
455459 {
456460 int ret;
457461
462+ lockdep_assert_held(&wq_mutex);
463+
458464 do {
459465 if (!idr_pre_get(&worker_pool_idr, GFP_KERNEL))
460466 return -ENOMEM;
461−
462− spin_lock_irq(&workqueue_lock);
463467 ret = idr_get_new(&worker_pool_idr, pool, &pool->id);
464− spin_unlock_irq(&workqueue_lock);
465468 } while (ret == -EAGAIN);
466469
467470 return ret;
@@ -574,9 +577,9 @@ static struct pool_workqueue *get_work_pwq(struct work_struct *work) · 106 unmodified lines
574577 *
575578 * Return the worker_pool @work was last associated with. %NULL if none.
576579 *
577− * Pools are created and destroyed under workqueue_lock, and allows read
578− * access under sched-RCU read lock. As such, this function should be
579− * called under workqueue_lock or with preemption disabled.
580+ * Pools are created and destroyed under wq_mutex, and allows read access
581+ * under sched-RCU read lock. As such, this function should be called
582+ * under wq_mutex or with preemption disabled.
580583 *
581584 * All fields of the returned pool are accessible as long as the above
582585 * mentioned locking is in effect. If the returned pool needs to be used
@@ -588,7 +591,7 @@ static struct worker_pool *get_work_pool(struct work_struct *work) · 5 unmodified lines
588591 unsigned long data = atomic_long_read(&work->data);
589592 int pool_id;
590593
591− assert_rcu_or_wq_lock();
594+ assert_rcu_or_wq_mutex();
592595
593596 if (data & WORK_STRUCT_PWQ)
594597 return ((struct pool_workqueue *)
@@ -2768,10 +2771,10 @@ void drain_workqueue(struct workqueue_struct *wq) · 2173 unmodified lines
27682771 * hotter than drain_workqueue() and already looks at @wq->flags.
27692772 * Use __WQ_DRAINING so that queue doesn't have to check nr_drainers.
27702773 */
2771− spin_lock_irq(&workqueue_lock);
2774+ mutex_lock(&wq_mutex);
27722775 if (!wq->nr_drainers++)
27732776 wq->flags |= __WQ_DRAINING;
2774− spin_unlock_irq(&workqueue_lock);
2777+ mutex_unlock(&wq_mutex);
27752778 reflush:
27762779 flush_workqueue(wq);
27772780
@@ -2796,12 +2799,12 @@ reflush: · 18 unmodified lines
27962799 goto reflush;
27972800 }
27982801
2799− spin_lock(&workqueue_lock);
2802+ local_irq_enable();
2803+
2804+ mutex_lock(&wq_mutex);
28002805 if (!--wq->nr_drainers)
28012806 wq->flags &= ~__WQ_DRAINING;
2802− spin_unlock(&workqueue_lock);
2803−
2804− local_irq_enable();
2807+ mutex_unlock(&wq_mutex);
28052808 }
28062809 EXPORT_SYMBOL_GPL(drain_workqueue);
28072810
@@ -3514,16 +3517,16 @@ static void put_unbound_pool(struct worker_pool *pool) · 706 unmodified lines
35143517 {
35153518 struct worker *worker;
35163519
3517− spin_lock_irq(&workqueue_lock);
3520+ mutex_lock(&wq_mutex);
35183521 if (--pool->refcnt) {
3519− spin_unlock_irq(&workqueue_lock);
3522+ mutex_unlock(&wq_mutex);
35203523 return;
35213524 }
35223525
35233526 /* sanity checks */
35243527 if (WARN_ON(!(pool->flags & POOL_DISASSOCIATED)) ||
35253528 WARN_ON(!list_empty(&pool->worklist))) {
3526− spin_unlock_irq(&workqueue_lock);
3529+ mutex_unlock(&wq_mutex);
35273530 return;
35283531 }
35293532
@@ -3532,7 +3535,7 @@ static void put_unbound_pool(struct worker_pool *pool) · 2 unmodified lines
35323535 idr_remove(&worker_pool_idr, pool->id);
35333536 hash_del(&pool->hash_node);
35343537
3535− spin_unlock_irq(&workqueue_lock);
3538+ mutex_unlock(&wq_mutex);
35363539
35373540 /*
35383541 * Become the manager and destroy all workers. Grabbing
@@ -3570,21 +3573,18 @@ static void put_unbound_pool(struct worker_pool *pool) · 31 unmodified lines
35703573 */
35713574 static struct worker_pool *get_unbound_pool(const struct workqueue_attrs *attrs)
35723575 {
3573− static DEFINE_MUTEX(create_mutex);
35743576 u32 hash = wqattrs_hash(attrs);
35753577 struct worker_pool *pool;
35763578
3577− mutex_lock(&create_mutex);
3579+ mutex_lock(&wq_mutex);
35783580
35793581 /* do we already have a matching pool? */
3580− spin_lock_irq(&workqueue_lock);
35813582 hash_for_each_possible(unbound_pool_hash, pool, hash_node, hash) {
35823583 if (wqattrs_equal(pool->attrs, attrs)) {
35833584 pool->refcnt++;
35843585 goto out_unlock;
35853586 }
35863587 }
3587− spin_unlock_irq(&workqueue_lock);
35883588
35893589 /* nope, create a new one */
35903590 pool = kzalloc(sizeof(*pool), GFP_KERNEL);
@@ -3602,14 +3602,12 @@ static struct worker_pool *get_unbound_pool(const struct workqueue_attrs *attrs) · 11 unmodified lines
36023602 goto fail;
36033603
36043604 /* install */
3605− spin_lock_irq(&workqueue_lock);
36063605 hash_add(unbound_pool_hash, &pool->hash_node, hash);
36073606 out_unlock:
3608− spin_unlock_irq(&workqueue_lock);
3609− mutex_unlock(&create_mutex);
3607+ mutex_unlock(&wq_mutex);
36103608 return pool;
36113609 fail:
3612− mutex_unlock(&create_mutex);
3610+ mutex_unlock(&wq_mutex);
36133611 if (pool)
36143612 put_unbound_pool(pool);
36153613 return NULL;
@@ -3883,18 +3881,19 @@ struct workqueue_struct *__alloc_workqueue_key(const char *fmt, · 267 unmodified lines
38833881 goto err_destroy;
38843882
38853883 /*
3886− * workqueue_lock protects global freeze state and workqueues list.
3887− * Grab it, adjust max_active and add the new workqueue to
3888− * workqueues list.
3884+ * wq_mutex protects global freeze state and workqueues list. Grab
3885+ * it, adjust max_active and add the new @wq to workqueues list.
38893886 */
3890− spin_lock_irq(&workqueue_lock);
3887+ mutex_lock(&wq_mutex);
38913888
3889+ spin_lock_irq(&workqueue_lock);
38923890 for_each_pwq(pwq, wq)
38933891 pwq_adjust_max_active(pwq);
3892+ spin_unlock_irq(&workqueue_lock);
38943893
38953894 list_add(&wq->list, &workqueues);
38963895
3897− spin_unlock_irq(&workqueue_lock);
3896+ mutex_unlock(&wq_mutex);
38983897
38993898 return wq;
39003899
@@ -3920,9 +3919,8 @@ void destroy_workqueue(struct workqueue_struct *wq) · 19 unmodified lines
39203919 /* drain it before proceeding with destruction */
39213920 drain_workqueue(wq);
39223921
3923− spin_lock_irq(&workqueue_lock);
3924−
39253922 /* sanity checks */
3923+ spin_lock_irq(&workqueue_lock);
39263924 for_each_pwq(pwq, wq) {
39273925 int i;
39283926
@@ -3940,14 +3938,15 @@ void destroy_workqueue(struct workqueue_struct *wq) · 11 unmodified lines
39403938 return;
39413939 }
39423940 }
3941+ spin_unlock_irq(&workqueue_lock);
39433942
39443943 /*
39453944 * wq list is used to freeze wq, remove from list after
39463945 * flushing is complete in case freeze races us.
39473946 */
3947+ mutex_lock(&wq_mutex);
39483948 list_del_init(&wq->list);
3949−
3950− spin_unlock_irq(&workqueue_lock);
3949+ mutex_unlock(&wq_mutex);
39513950
39523951 workqueue_sysfs_unregister(wq);
39533952
@@ -4267,7 +4266,7 @@ EXPORT_SYMBOL_GPL(work_on_cpu); · 313 unmodified lines
42674266 * pool->worklist.
42684267 *
42694268 * CONTEXT:
4270− * Grabs and releases workqueue_lock and pool->lock's.
4269+ * Grabs and releases wq_mutex, workqueue_lock and pool->lock's.
42714270 */
42724271 void freeze_workqueues_begin(void)
42734272 {
@@ -4276,26 +4275,28 @@ void freeze_workqueues_begin(void) · 2 unmodified lines
42764275 struct pool_workqueue *pwq;
42774276 int pi;
42784277
4279− spin_lock_irq(&workqueue_lock);
4278+ mutex_lock(&wq_mutex);
42804279
42814280 WARN_ON_ONCE(workqueue_freezing);
42824281 workqueue_freezing = true;
42834282
42844283 /* set FREEZING */
42854284 for_each_pool(pool, pi) {
4286− spin_lock(&pool->lock);
4285+ spin_lock_irq(&pool->lock);
42874286 WARN_ON_ONCE(pool->flags & POOL_FREEZING);
42884287 pool->flags |= POOL_FREEZING;
4289− spin_unlock(&pool->lock);
4288+ spin_unlock_irq(&pool->lock);
42904289 }
42914290
42924291 /* suppress further executions by setting max_active to zero */
4292+ spin_lock_irq(&workqueue_lock);
42934293 list_for_each_entry(wq, &workqueues, list) {
42944294 for_each_pwq(pwq, wq)
42954295 pwq_adjust_max_active(pwq);
42964296 }
4297−
42984297 spin_unlock_irq(&workqueue_lock);
4298+
4299+ mutex_unlock(&wq_mutex);
42994300 }
43004301
43014302 /**
@@ -4305,7 +4306,7 @@ void freeze_workqueues_begin(void) · 3 unmodified lines
43054306 * between freeze_workqueues_begin() and thaw_workqueues().
43064307 *
43074308 * CONTEXT:
4308− * Grabs and releases workqueue_lock.
4309+ * Grabs and releases wq_mutex.
43094310 *
43104311 * RETURNS:
43114312 * %true if some freezable workqueues are still busy. %false if freezing
@@ -4317,7 +4318,7 @@ bool freeze_workqueues_busy(void) · 5 unmodified lines
43174318 struct workqueue_struct *wq;
43184319 struct pool_workqueue *pwq;
43194320
4320− spin_lock_irq(&workqueue_lock);
4321+ mutex_lock(&wq_mutex);
43214322
43224323 WARN_ON_ONCE(!workqueue_freezing);
43234324
@@ -4328,16 +4329,19 @@ bool freeze_workqueues_busy(void) · 4 unmodified lines
43284329 * nr_active is monotonically decreasing. It's safe
43294330 * to peek without lock.
43304331 */
4332+ preempt_disable();
43314333 for_each_pwq(pwq, wq) {
43324334 WARN_ON_ONCE(pwq->nr_active < 0);
43334335 if (pwq->nr_active) {
43344336 busy = true;
4337+ preempt_enable();
43354338 goto out_unlock;
43364339 }
43374340 }
4341+ preempt_enable();
43384342 }
43394343 out_unlock:
4340− spin_unlock_irq(&workqueue_lock);
4344+ mutex_unlock(&wq_mutex);
43414345 return busy;
43424346 }
43434347
@@ -4348,7 +4352,7 @@ out_unlock: · 4 unmodified lines
43484352 * frozen works are transferred to their respective pool worklists.
43494353 *
43504354 * CONTEXT:
4351− * Grabs and releases workqueue_lock and pool->lock's.
4355+ * Grabs and releases wq_mutex, workqueue_lock and pool->lock's.
43524356 */
43534357 void thaw_workqueues(void)
43544358 {
@@ -4357,35 +4361,37 @@ void thaw_workqueues(void) · 2 unmodified lines
43574361 struct worker_pool *pool;
43584362 int pi;
43594363
4360− spin_lock_irq(&workqueue_lock);
4364+ mutex_lock(&wq_mutex);
43614365
43624366 if (!workqueue_freezing)
43634367 goto out_unlock;
43644368
43654369 /* clear FREEZING */
43664370 for_each_pool(pool, pi) {
4367− spin_lock(&pool->lock);
4371+ spin_lock_irq(&pool->lock);
43684372 WARN_ON_ONCE(!(pool->flags & POOL_FREEZING));
43694373 pool->flags &= ~POOL_FREEZING;
4370− spin_unlock(&pool->lock);
4374+ spin_unlock_irq(&pool->lock);
43714375 }
43724376
43734377 /* restore max_active and repopulate worklist */
4378+ spin_lock_irq(&workqueue_lock);
43744379 list_for_each_entry(wq, &workqueues, list) {
43754380 for_each_pwq(pwq, wq)
43764381 pwq_adjust_max_active(pwq);
43774382 }
4383+ spin_unlock_irq(&workqueue_lock);
43784384
43794385 /* kick workers */
43804386 for_each_pool(pool, pi) {
4381− spin_lock(&pool->lock);
4387+ spin_lock_irq(&pool->lock);
43824388 wake_up_worker(pool);
4383− spin_unlock(&pool->lock);
4389+ spin_unlock_irq(&pool->lock);
43844390 }
43854391
43864392 workqueue_freezing = false;
43874393 out_unlock:
4388− spin_unlock_irq(&workqueue_lock);
4394+ mutex_unlock(&wq_mutex);
43894395 }
43904396 #endif /* CONFIG_FREEZER */
43914397
@@ -4417,7 +4423,9 @@ static int __init init_workqueues(void) · 25 unmodified lines
44174423 pool->attrs->nice = std_nice[i++];
44184424
44194425 /* alloc pool ID */
4426+ mutex_lock(&wq_mutex);
44204427 BUG_ON(worker_pool_assign_id(pool));
4428+ mutex_unlock(&wq_mutex);
44214429 }
44224430 }
44234431