History

workqueue: RCU protect wq->dfl_pwq and implement accessors for it

wq->cpu_pwq is RCU protected but wq->dfl_pwq isn't. This is okay because currently wq->dfl_pwq is used only accessed to install it into wq->cpu_pwq which doesn't require RCU access. However, we want to be able to access wq->dfl_pwq under RCU in the future to access its __pod_cpumask and the code can be made easier to read by making the two pwq fields behave in the same way. - Make wq->dfl_pwq RCU protected. - Add unbound_pwq_slot() and unbound_pwq() which can access both ->dfl_pwq and ->cpu_pwq. The former returns the double pointer that can be used access and update the pwqs. The latter performs locking check and dereferences the double pointer. - pwq accesses and updates are converted to use unbound_pwq[_slot]().

Tejun Heo committed 2 years ago

Commit 9f66cff1 parent c5404d41 file · +40 −24Browse files

Showing the changes to kernel/workqueue.c (1 of 1 file). Show all

Unified · Split

kernel/workqueue.c+40−24
@@ -308,7 +308,7 @@ struct workqueue_struct { · 307 unmodified lines
308308 int saved_max_active; /* WQ: saved max_active */
309309
310310 struct workqueue_attrs *unbound_attrs; /* PW: only for unbound wqs */
311− struct pool_workqueue *dfl_pwq; /* PW: only for unbound wqs */
311+ struct pool_workqueue __rcu *dfl_pwq; /* PW: only for unbound wqs */
312312
313313 #ifdef CONFIG_SYSFS
314314 struct wq_device *wq_dev; /* I: for sysfs interface */
@@ -639,6 +639,23 @@ static int worker_pool_assign_id(struct worker_pool *pool) · 324 unmodified lines
639639 return ret;
640640 }
641641
642+static struct pool_workqueue __rcu **
643+unbound_pwq_slot(struct workqueue_struct *wq, int cpu)
644+{
645+ if (cpu >= 0)
646+ return per_cpu_ptr(wq->cpu_pwq, cpu);
647+ else
648+ return &wq->dfl_pwq;
649+}
650+
651+/* @cpu < 0 for dfl_pwq */
652+static struct pool_workqueue *unbound_pwq(struct workqueue_struct *wq, int cpu)
653+{
654+ return rcu_dereference_check(*unbound_pwq_slot(wq, cpu),
655+ lockdep_is_held(&wq_pool_mutex) ||
656+ lockdep_is_held(&wq->mutex));
657+}
658+
642659 static unsigned int work_color_to_flags(int color)
643660 {
644661 return color << WORK_STRUCT_COLOR_SHIFT;
@@ -4328,10 +4345,11 @@ static void wq_calc_pod_cpumask(struct workqueue_attrs *attrs, int cpu, · 3683 unmodified lines
43284345 "possible intersect\n");
43294346 }
43304347
4331−/* install @pwq into @wq's cpu_pwq and return the old pwq */
4348+/* install @pwq into @wq and return the old pwq, @cpu < 0 for dfl_pwq */
43324349 static struct pool_workqueue *install_unbound_pwq(struct workqueue_struct *wq,
43334350 int cpu, struct pool_workqueue *pwq)
43344351 {
4352+ struct pool_workqueue __rcu **slot = unbound_pwq_slot(wq, cpu);
43354353 struct pool_workqueue *old_pwq;
43364354
43374355 lockdep_assert_held(&wq_pool_mutex);
@@ -4340,8 +4358,8 @@ static struct pool_workqueue *install_unbound_pwq(struct workqueue_struct *wq, · 2 unmodified lines
43404358 /* link_pwq() can handle duplicate calls */
43414359 link_pwq(pwq);
43424360
4343− old_pwq = rcu_access_pointer(*per_cpu_ptr(wq->cpu_pwq, cpu));
4344− rcu_assign_pointer(*per_cpu_ptr(wq->cpu_pwq, cpu), pwq);
4361+ old_pwq = rcu_access_pointer(*slot);
4362+ rcu_assign_pointer(*slot, pwq);
43454363 return old_pwq;
43464364 }
43474365
@@ -4441,14 +4459,11 @@ static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx) · 93 unmodified lines
44414459
44424460 copy_workqueue_attrs(ctx->wq->unbound_attrs, ctx->attrs);
44434461
4444− /* save the previous pwq and install the new one */
4462+ /* save the previous pwqs and install the new ones */
44454463 for_each_possible_cpu(cpu)
44464464 ctx->pwq_tbl[cpu] = install_unbound_pwq(ctx->wq, cpu,
44474465 ctx->pwq_tbl[cpu]);
4448−
4449− /* @dfl_pwq might not have been used, ensure it's linked */
4450− link_pwq(ctx->dfl_pwq);
4451− swap(ctx->wq->dfl_pwq, ctx->dfl_pwq);
4466+ ctx->dfl_pwq = install_unbound_pwq(ctx->wq, -1, ctx->dfl_pwq);
44524467
44534468 mutex_unlock(&ctx->wq->mutex);
44544469 }
@@ -4558,9 +4573,7 @@ static void wq_update_pod(struct workqueue_struct *wq, int cpu, · 103 unmodified lines
45584573
45594574 /* nothing to do if the target cpumask matches the current pwq */
45604575 wq_calc_pod_cpumask(target_attrs, cpu, off_cpu);
4561− pwq = rcu_dereference_protected(*per_cpu_ptr(wq->cpu_pwq, cpu),
4562− lockdep_is_held(&wq_pool_mutex));
4563− if (wqattrs_equal(target_attrs, pwq->pool->attrs))
4576+ if (wqattrs_equal(target_attrs, unbound_pwq(wq, cpu)->pool->attrs))
45644577 return;
45654578
45664579 /* create a new pwq */
@@ -4578,10 +4591,11 @@ static void wq_update_pod(struct workqueue_struct *wq, int cpu, · 11 unmodified lines
45784591
45794592 use_dfl_pwq:
45804593 mutex_lock(&wq->mutex);
4581− raw_spin_lock_irq(&wq->dfl_pwq->pool->lock);
4582− get_pwq(wq->dfl_pwq);
4583− raw_spin_unlock_irq(&wq->dfl_pwq->pool->lock);
4584− old_pwq = install_unbound_pwq(wq, cpu, wq->dfl_pwq);
4594+ pwq = unbound_pwq(wq, -1);
4595+ raw_spin_lock_irq(&pwq->pool->lock);
4596+ get_pwq(pwq);
4597+ raw_spin_unlock_irq(&pwq->pool->lock);
4598+ old_pwq = install_unbound_pwq(wq, cpu, pwq);
45854599 out_unlock:
45864600 mutex_unlock(&wq->mutex);
45874601 put_pwq_unlocked(old_pwq);
@@ -4619,10 +4633,13 @@ static int alloc_and_link_pwqs(struct workqueue_struct *wq) · 31 unmodified lines
46194633
46204634 cpus_read_lock();
46214635 if (wq->flags & __WQ_ORDERED) {
4636+ struct pool_workqueue *dfl_pwq;
4637+
46224638 ret = apply_workqueue_attrs(wq, ordered_wq_attrs[highpri]);
46234639 /* there should only be single pwq for ordering guarantee */
4624− WARN(!ret && (wq->pwqs.next != &wq->dfl_pwq->pwqs_node ||
4625− wq->pwqs.prev != &wq->dfl_pwq->pwqs_node),
4640+ dfl_pwq = rcu_access_pointer(wq->dfl_pwq);
4641+ WARN(!ret && (wq->pwqs.next != &dfl_pwq->pwqs_node ||
4642+ wq->pwqs.prev != &dfl_pwq->pwqs_node),
46264643 "ordering guarantee broken for workqueue %s\n", wq->name);
46274644 } else {
46284645 ret = apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]);
@@ -4856,7 +4873,7 @@ static bool pwq_busy(struct pool_workqueue *pwq) · 227 unmodified lines
48564873 if (pwq->nr_in_flight[i])
48574874 return true;
48584875
4859− if ((pwq != pwq->wq->dfl_pwq) && (pwq->refcnt > 1))
4876+ if ((pwq != rcu_access_pointer(pwq->wq->dfl_pwq)) && (pwq->refcnt > 1))
48604877 return true;
48614878 if (!pwq_is_empty(pwq))
48624879 return true;
@@ -4940,13 +4957,12 @@ void destroy_workqueue(struct workqueue_struct *wq) · 77 unmodified lines
49404957 rcu_read_lock();
49414958
49424959 for_each_possible_cpu(cpu) {
4943− pwq = rcu_access_pointer(*per_cpu_ptr(wq->cpu_pwq, cpu));
4944− RCU_INIT_POINTER(*per_cpu_ptr(wq->cpu_pwq, cpu), NULL);
4945− put_pwq_unlocked(pwq);
4960+ put_pwq_unlocked(unbound_pwq(wq, cpu));
4961+ RCU_INIT_POINTER(*unbound_pwq_slot(wq, cpu), NULL);
49464962 }
49474963
4948− put_pwq_unlocked(wq->dfl_pwq);
4949− wq->dfl_pwq = NULL;
4964+ put_pwq_unlocked(unbound_pwq(wq, -1));
4965+ RCU_INIT_POINTER(*unbound_pwq_slot(wq, -1), NULL);
49504966
49514967 rcu_read_unlock();
49524968 }