For an unbound pool, multiple cpumasks are involved. U: The user-specified cpumask (may be filtered with cpu_possible_mask). A: The actual cpumask filtered by wq_unbound_cpumask. If the filtering leaves no CPU, wq_unbound_cpumask is used. P: Per-pod subsets of #A. wq->attrs stores #U, wq->dfl_pwq->pool->attrs->cpumask #A, and wq->cpu_pwq[CPU]->pool->attrs->cpumask #P. wq_update_pod() is called to update per-pod pwq's during CPU hotplug. To calculate the new #P for each workqueue, it needs to call wq_calc_pod_cpumask() with @attrs that contains #A. Currently, wq_update_pod() achieves this by calling wq_calc_pod_cpumask() with wq->dfl_pwq->pool->attrs. This is rather fragile because we're calling wq_calc_pod_cpumask() with @attrs of a worker_pool rather than the workqueue's actual attrs when what we want to calculate is the workqueue's cpumask on the pod. While this works fine currently, future changes will add fields which are used differently between workqueues and worker_pools and this subtlety will bite us. This patch factors out #U -> #A calculation from apply_wqattrs_prepare() into wqattrs_actualize_cpumask and updates wq_update_pod() to copy wq->unbound_attrs and use the new helper to obtain #A freshly instead of abusing wq->dfl_pwq->pool_attrs. This shouldn't cause any behavior changes in the current code.
Showing the changes to kernel/workqueue.c (1 of 1 file). Show all
Unified · Split
| @@ -348,6 +348,7 @@ static bool wq_pod_enabled; /* unbound CPU pod affinity enabled */ · 347 unmodified lines | |||
| 348 | 348 | ||
| 349 | /* buf for wq_update_unbound_pod_attrs(), protected by CPU hotplug exclusion */ | 349 | /* buf for wq_update_unbound_pod_attrs(), protected by CPU hotplug exclusion */ |
| 350 | static struct workqueue_attrs *wq_update_pod_attrs_buf; | 350 | static struct workqueue_attrs *wq_update_pod_attrs_buf; |
| 351 | static cpumask_var_t wq_update_pod_cpumask_buf; | ||
| 351 | 352 | ||
| 352 | static DEFINE_MUTEX(wq_pool_mutex); /* protects pools and workqueues list */ | 353 | static DEFINE_MUTEX(wq_pool_mutex); /* protects pools and workqueues list */ |
| 353 | static DEFINE_MUTEX(wq_pool_attach_mutex); /* protects worker attach/detach */ | 354 | static DEFINE_MUTEX(wq_pool_attach_mutex); /* protects worker attach/detach */ |
| @@ -3699,6 +3700,20 @@ static bool wqattrs_equal(const struct workqueue_attrs *a, · 3345 unmodified lines | |||
| 3699 | return true; | 3700 | return true; |
| 3700 | } | 3701 | } |
| 3701 | 3702 | ||
| 3703 | /* Update @attrs with actually available CPUs */ | ||
| 3704 | static void wqattrs_actualize_cpumask(struct workqueue_attrs *attrs, | ||
| 3705 | const cpumask_t *unbound_cpumask) | ||
| 3706 | { | ||
| 3707 | /* | ||
| 3708 | * Calculate the effective CPU mask of @attrs given @unbound_cpumask. If | ||
| 3709 | * @attrs->cpumask doesn't overlap with @unbound_cpumask, we fallback to | ||
| 3710 | * @unbound_cpumask. | ||
| 3711 | */ | ||
| 3712 | cpumask_and(attrs->cpumask, attrs->cpumask, unbound_cpumask); | ||
| 3713 | if (unlikely(cpumask_empty(attrs->cpumask))) | ||
| 3714 | cpumask_copy(attrs->cpumask, unbound_cpumask); | ||
| 3715 | } | ||
| 3716 | |||
| 3702 | /** | 3717 | /** |
| 3703 | * init_worker_pool - initialize a newly zalloc'd worker_pool | 3718 | * init_worker_pool - initialize a newly zalloc'd worker_pool |
| 3704 | * @pool: worker_pool to initialize | 3719 | * @pool: worker_pool to initialize |
| @@ -4221,33 +4236,23 @@ apply_wqattrs_prepare(struct workqueue_struct *wq, · 516 unmodified lines | |||
| 4221 | if (!ctx || !new_attrs || !tmp_attrs) | 4236 | if (!ctx || !new_attrs || !tmp_attrs) |
| 4222 | goto out_free; | 4237 | goto out_free; |
| 4223 | 4238 | ||
| 4224 | /* | ||
| 4225 | * Calculate the attrs of the default pwq with unbound_cpumask | ||
| 4226 | * which is wq_unbound_cpumask or to set to wq_unbound_cpumask. | ||
| 4227 | * If the user configured cpumask doesn't overlap with the | ||
| 4228 | * wq_unbound_cpumask, we fallback to the wq_unbound_cpumask. | ||
| 4229 | */ | ||
| 4230 | copy_workqueue_attrs(new_attrs, attrs); | ||
| 4231 | cpumask_and(new_attrs->cpumask, new_attrs->cpumask, unbound_cpumask); | ||
| 4232 | if (unlikely(cpumask_empty(new_attrs->cpumask))) | ||
| 4233 | cpumask_copy(new_attrs->cpumask, unbound_cpumask); | ||
| 4234 | |||
| 4235 | /* | ||
| 4236 | * We may create multiple pwqs with differing cpumasks. Make a | ||
| 4237 | * copy of @new_attrs which will be modified and used to obtain | ||
| 4238 | * pools. | ||
| 4239 | */ | ||
| 4240 | copy_workqueue_attrs(tmp_attrs, new_attrs); | ||
| 4241 | |||
| 4242 | /* | 4239 | /* |
| 4243 | * If something goes wrong during CPU up/down, we'll fall back to | 4240 | * If something goes wrong during CPU up/down, we'll fall back to |
| 4244 | * the default pwq covering whole @attrs->cpumask. Always create | 4241 | * the default pwq covering whole @attrs->cpumask. Always create |
| 4245 | * it even if we don't use it immediately. | 4242 | * it even if we don't use it immediately. |
| 4246 | */ | 4243 | */ |
| 4244 | copy_workqueue_attrs(new_attrs, attrs); | ||
| 4245 | wqattrs_actualize_cpumask(new_attrs, unbound_cpumask); | ||
| 4247 | ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs); | 4246 | ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs); |
| 4248 | if (!ctx->dfl_pwq) | 4247 | if (!ctx->dfl_pwq) |
| 4249 | goto out_free; | 4248 | goto out_free; |
| 4250 | 4249 | ||
| 4250 | /* | ||
| 4251 | * We may create multiple pwqs with differing cpumasks. Make a copy of | ||
| 4252 | * @new_attrs which will be modified and used to obtain pools. | ||
| 4253 | */ | ||
| 4254 | copy_workqueue_attrs(tmp_attrs, new_attrs); | ||
| 4255 | |||
| 4251 | for_each_possible_cpu(cpu) { | 4256 | for_each_possible_cpu(cpu) { |
| 4252 | if (new_attrs->ordered) { | 4257 | if (new_attrs->ordered) { |
| 4253 | ctx->dfl_pwq->refcnt++; | 4258 | ctx->dfl_pwq->refcnt++; |
| @@ -4414,18 +4419,20 @@ static void wq_update_pod(struct workqueue_struct *wq, int cpu, · 160 unmodified lines | |||
| 4414 | * CPU hotplug exclusion. | 4419 | * CPU hotplug exclusion. |
| 4415 | */ | 4420 | */ |
| 4416 | target_attrs = wq_update_pod_attrs_buf; | 4421 | target_attrs = wq_update_pod_attrs_buf; |
| 4417 | cpumask = target_attrs->cpumask; | 4422 | cpumask = wq_update_pod_cpumask_buf; |
| 4418 | 4423 | ||
| 4419 | copy_workqueue_attrs(target_attrs, wq->unbound_attrs); | 4424 | copy_workqueue_attrs(target_attrs, wq->unbound_attrs); |
| 4425 | wqattrs_actualize_cpumask(target_attrs, wq_unbound_cpumask); | ||
| 4420 | 4426 | ||
| 4421 | /* nothing to do if the target cpumask matches the current pwq */ | 4427 | /* nothing to do if the target cpumask matches the current pwq */ |
| 4422 | wq_calc_pod_cpumask(wq->dfl_pwq->pool->attrs, pod, off_cpu, cpumask); | 4428 | wq_calc_pod_cpumask(target_attrs, pod, off_cpu, cpumask); |
| 4423 | pwq = rcu_dereference_protected(*per_cpu_ptr(wq->cpu_pwq, cpu), | 4429 | pwq = rcu_dereference_protected(*per_cpu_ptr(wq->cpu_pwq, cpu), |
| 4424 | lockdep_is_held(&wq_pool_mutex)); | 4430 | lockdep_is_held(&wq_pool_mutex)); |
| 4425 | if (cpumask_equal(cpumask, pwq->pool->attrs->cpumask)) | 4431 | if (cpumask_equal(cpumask, pwq->pool->attrs->cpumask)) |
| 4426 | return; | 4432 | return; |
| 4427 | 4433 | ||
| 4428 | /* create a new pwq */ | 4434 | /* create a new pwq */ |
| 4435 | cpumask_copy(target_attrs->cpumask, cpumask); | ||
| 4429 | pwq = alloc_unbound_pwq(wq, target_attrs); | 4436 | pwq = alloc_unbound_pwq(wq, target_attrs); |
| 4430 | if (!pwq) { | 4437 | if (!pwq) { |
| 4431 | pr_warn("workqueue: allocation failed while updating CPU pod affinity of \"%s\"\n", | 4438 | pr_warn("workqueue: allocation failed while updating CPU pod affinity of \"%s\"\n", |
| @@ -6285,6 +6292,8 @@ void __init workqueue_init_early(void) · 1853 unmodified lines | |||
| 6285 | wq_update_pod_attrs_buf = alloc_workqueue_attrs(); | 6292 | wq_update_pod_attrs_buf = alloc_workqueue_attrs(); |
| 6286 | BUG_ON(!wq_update_pod_attrs_buf); | 6293 | BUG_ON(!wq_update_pod_attrs_buf); |
| 6287 | 6294 | ||
| 6295 | BUG_ON(!alloc_cpumask_var(&wq_update_pod_cpumask_buf, GFP_KERNEL)); | ||
| 6296 | |||
| 6288 | /* initialize CPU pools */ | 6297 | /* initialize CPU pools */ |
| 6289 | for_each_possible_cpu(cpu) { | 6298 | for_each_possible_cpu(cpu) { |
| 6290 | struct worker_pool *pool; | 6299 | struct worker_pool *pool; |