History

workqueue: protect wq->pwqs and iteration with wq->mutex

We're expanding wq->mutex to cover all fields specific to each workqueue with the end goal of replacing pwq_lock which will make locking simpler and easier to understand. init_and_link_pwq() and pwq_unbound_release_workfn() already grab wq->mutex when adding or removing a pwq from wq->pwqs list. This patch makes it official that the list is wq->mutex protected for writes and updates readers accoridingly. Explicit IRQ toggles for sched-RCU read-locking in flush_workqueue_prep_pwqs() and drain_workqueues() are removed as the surrounding wq->mutex can provide sufficient synchronization. Also, assert_rcu_or_pwq_lock() is renamed to assert_rcu_or_wq_mutex() and checks for wq->mutex too. pwq_lock locking and assertion are not removed by this patch and a couple of for_each_pwq() iterations are still protected by it. They'll be removed by future patches. tj: Rebased on top of the current dev branch. Updated description. Folded in assert_rcu_or_wq_mutex() renaming from a later patch along with associated comment updates.

Lai Jiangshan committed 13 years ago

Commit b09f4fd1 parent 87fc7411 file · +18 −24Browse files

Showing the changes to kernel/workqueue.c (1 of 1 file). Show all

Unified · Split

kernel/workqueue.c+18−24
@@ -204,7 +204,7 @@ struct pool_workqueue { · 203 unmodified lines
204204 * Release of unbound pwq is punted to system_wq. See put_pwq()
205205 * and pwq_unbound_release_workfn() for details. pool_workqueue
206206 * itself is also sched-RCU protected so that the first pwq can be
207− * determined without grabbing pwq_lock.
207+ * determined without grabbing wq->mutex.
208208 */
209209 struct work_struct unbound_release_work;
210210 struct rcu_head rcu;
@@ -298,10 +298,11 @@ static void copy_workqueue_attrs(struct workqueue_attrs *to, · 87 unmodified lines
298298 lockdep_is_held(&wq_pool_mutex), \
299299 "sched RCU or wq_pool_mutex should be held")
300300
301−#define assert_rcu_or_pwq_lock() \
301+#define assert_rcu_or_wq_mutex(wq) \
302302 rcu_lockdep_assert(rcu_read_lock_sched_held() || \
303+ lockdep_is_held(&wq->mutex) || \
303304 lockdep_is_held(&pwq_lock), \
304− "sched RCU or pwq_lock should be held")
305+ "sched RCU or wq->mutex should be held")
305306
306307 #ifdef CONFIG_LOCKDEP
307308 #define assert_manager_or_pool_lock(pool) \
@@ -356,7 +357,7 @@ static void copy_workqueue_attrs(struct workqueue_attrs *to, · 48 unmodified lines
356357 * @pwq: iteration cursor
357358 * @wq: the target workqueue
358359 *
359− * This must be called either with pwq_lock held or sched RCU read locked.
360+ * This must be called either with wq->mutex held or sched RCU read locked.
360361 * If the pwq needs to be used beyond the locking in effect, the caller is
361362 * responsible for guaranteeing that the pwq stays online.
362363 *
@@ -365,7 +366,7 @@ static void copy_workqueue_attrs(struct workqueue_attrs *to, · 2 unmodified lines
365366 */
366367 #define for_each_pwq(pwq, wq) \
367368 list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node) \
368− if (({ assert_rcu_or_pwq_lock(); false; })) { } \
369+ if (({ assert_rcu_or_wq_mutex(wq); false; })) { } \
369370 else
370371
371372 #ifdef CONFIG_DEBUG_OBJECTS_WORK
@@ -504,13 +505,13 @@ static int worker_pool_assign_id(struct worker_pool *pool) · 132 unmodified lines
504505 * first_pwq - return the first pool_workqueue of the specified workqueue
505506 * @wq: the target workqueue
506507 *
507− * This must be called either with pwq_lock held or sched RCU read locked.
508+ * This must be called either with wq->mutex held or sched RCU read locked.
508509 * If the pwq needs to be used beyond the locking in effect, the caller is
509510 * responsible for guaranteeing that the pwq stays online.
510511 */
511512 static struct pool_workqueue *first_pwq(struct workqueue_struct *wq)
512513 {
513− assert_rcu_or_pwq_lock();
514+ assert_rcu_or_wq_mutex(wq);
514515 return list_first_or_null_rcu(&wq->pwqs, struct pool_workqueue,
515516 pwqs_node);
516517 }
@@ -2477,12 +2478,10 @@ static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq, · 1960 unmodified lines
24772478 atomic_set(&wq->nr_pwqs_to_flush, 1);
24782479 }
24792480
2480− local_irq_disable();
2481−
24822481 for_each_pwq(pwq, wq) {
24832482 struct worker_pool *pool = pwq->pool;
24842483
2485− spin_lock(&pool->lock);
2484+ spin_lock_irq(&pool->lock);
24862485
24872486 if (flush_color >= 0) {
24882487 WARN_ON_ONCE(pwq->flush_color != -1);
@@ -2499,11 +2498,9 @@ static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq, · 10 unmodified lines
24992498 pwq->work_color = work_color;
25002499 }
25012500
2502− spin_unlock(&pool->lock);
2501+ spin_unlock_irq(&pool->lock);
25032502 }
25042503
2505− local_irq_enable();
2506−
25072504 if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))
25082505 complete(&wq->first_flusher->done);
25092506
@@ -2691,14 +2688,14 @@ void drain_workqueue(struct workqueue_struct *wq) · 181 unmodified lines
26912688 reflush:
26922689 flush_workqueue(wq);
26932690
2694− local_irq_disable();
2691+ mutex_lock(&wq->mutex);
26952692
26962693 for_each_pwq(pwq, wq) {
26972694 bool drained;
26982695
2699− spin_lock(&pwq->pool->lock);
2696+ spin_lock_irq(&pwq->pool->lock);
27002697 drained = !pwq->nr_active && list_empty(&pwq->delayed_works);
2701− spin_unlock(&pwq->pool->lock);
2698+ spin_unlock_irq(&pwq->pool->lock);
27022699
27032700 if (drained)
27042701 continue;
@@ -2708,13 +2705,10 @@ reflush: · 3 unmodified lines
27082705 pr_warn("workqueue %s: drain_workqueue() isn't complete after %u tries\n",
27092706 wq->name, flush_cnt);
27102707
2711− local_irq_enable();
2708+ mutex_unlock(&wq->mutex);
27122709 goto reflush;
27132710 }
27142711
2715− local_irq_enable();
2716−
2717− mutex_lock(&wq->mutex);
27182712 if (!--wq->nr_drainers)
27192713 wq->flags &= ~__WQ_DRAINING;
27202714 mutex_unlock(&wq->mutex);
@@ -3843,13 +3837,13 @@ void destroy_workqueue(struct workqueue_struct *wq) · 1122 unmodified lines
38433837 drain_workqueue(wq);
38443838
38453839 /* sanity checks */
3846− spin_lock_irq(&pwq_lock);
3840+ mutex_lock(&wq->mutex);
38473841 for_each_pwq(pwq, wq) {
38483842 int i;
38493843
38503844 for (i = 0; i < WORK_NR_COLORS; i++) {
38513845 if (WARN_ON(pwq->nr_in_flight[i])) {
3852− spin_unlock_irq(&pwq_lock);
3846+ mutex_unlock(&wq->mutex);
38533847 return;
38543848 }
38553849 }
@@ -3857,11 +3851,11 @@ void destroy_workqueue(struct workqueue_struct *wq) · 1 unmodified line
38573851 if (WARN_ON(pwq->refcnt > 1) ||
38583852 WARN_ON(pwq->nr_active) ||
38593853 WARN_ON(!list_empty(&pwq->delayed_works))) {
3860− spin_unlock_irq(&pwq_lock);
3854+ mutex_unlock(&wq->mutex);
38613855 return;
38623856 }
38633857 }
3864− spin_unlock_irq(&pwq_lock);
3858+ mutex_unlock(&wq->mutex);
38653859
38663860 /*
38673861 * wq list is used to freeze wq, remove from list after