When a CPU went online or offline, wq_update_unbound_numa() was called only on the CPU which was going up or down. This works fine because all CPUs on the same NUMA node share the same pool_workqueue slot - one CPU updating it updates it for everyone in the node. However, future changes will make each CPU use a separate pool_workqueue even when they're sharing the same worker_pool, which requires updating pool_workqueue's for all CPUs which may be sharing the same pool_workqueue on hotplug. To accommodate the planned changes, this patch updates workqueue_on/offline_cpu() so that they call wq_update_unbound_numa() for all CPUs sharing the same NUMA node as the CPU going up or down. In the current code, the second+ calls would be noops and there shouldn't be any behavior changes. * As wq_update_unbound_numa() is now called on multiple CPUs per each hotplug event, @cpu is renamed to @hotplug_cpu and another @cpu argument is added. The former indicates the CPU being hot[un]plugged and the latter the CPU whose pool_workqueue is being updated. * In wq_update_unbound_numa(), cpu_off is renamed to off_cpu for consistency with the new @hotplug_cpu.
Showing the changes to kernel/workqueue.c (1 of 1 file). Show all
Unified · Split
| @@ -4422,7 +4422,8 @@ int apply_workqueue_attrs(struct workqueue_struct *wq, · 4421 unmodified lines | |||
| 4422 | /** | 4422 | /** |
| 4423 | * wq_update_unbound_numa - update NUMA affinity of a wq for CPU hot[un]plug | 4423 | * wq_update_unbound_numa - update NUMA affinity of a wq for CPU hot[un]plug |
| 4424 | * @wq: the target workqueue | 4424 | * @wq: the target workqueue |
| 4425 | * @cpu: the CPU coming up or going down | 4425 | * @cpu: the CPU to update pool association for |
| 4426 | * @hotplug_cpu: the CPU coming up or going down | ||
| 4426 | * @online: whether @cpu is coming up or going down | 4427 | * @online: whether @cpu is coming up or going down |
| 4427 | * | 4428 | * |
| 4428 | * This function is to be called from %CPU_DOWN_PREPARE, %CPU_ONLINE and | 4429 | * This function is to be called from %CPU_DOWN_PREPARE, %CPU_ONLINE and |
| @@ -4442,10 +4443,10 @@ int apply_workqueue_attrs(struct workqueue_struct *wq, · 13 unmodified lines | |||
| 4442 | * CPU_DOWN_PREPARE. | 4443 | * CPU_DOWN_PREPARE. |
| 4443 | */ | 4444 | */ |
| 4444 | static void wq_update_unbound_numa(struct workqueue_struct *wq, int cpu, | 4445 | static void wq_update_unbound_numa(struct workqueue_struct *wq, int cpu, |
| 4445 | bool online) | 4446 | int hotplug_cpu, bool online) |
| 4446 | { | 4447 | { |
| 4447 | int node = cpu_to_node(cpu); | 4448 | int node = cpu_to_node(cpu); |
| 4448 | int cpu_off = online ? -1 : cpu; | 4449 | int off_cpu = online ? -1 : hotplug_cpu; |
| 4449 | struct pool_workqueue *old_pwq = NULL, *pwq; | 4450 | struct pool_workqueue *old_pwq = NULL, *pwq; |
| 4450 | struct workqueue_attrs *target_attrs; | 4451 | struct workqueue_attrs *target_attrs; |
| 4451 | cpumask_t *cpumask; | 4452 | cpumask_t *cpumask; |
| @@ -4473,7 +4474,7 @@ static void wq_update_unbound_numa(struct workqueue_struct *wq, int cpu, · 21 unmodified lines | |||
| 4473 | * and create a new one if they don't match. If the target cpumask | 4474 | * and create a new one if they don't match. If the target cpumask |
| 4474 | * equals the default pwq's, the default pwq should be used. | 4475 | * equals the default pwq's, the default pwq should be used. |
| 4475 | */ | 4476 | */ |
| 4476 | if (wq_calc_node_cpumask(wq->dfl_pwq->pool->attrs, node, cpu_off, cpumask)) { | 4477 | if (wq_calc_node_cpumask(wq->dfl_pwq->pool->attrs, node, off_cpu, cpumask)) { |
| 4477 | if (cpumask_equal(cpumask, pwq->pool->attrs->cpumask)) | 4478 | if (cpumask_equal(cpumask, pwq->pool->attrs->cpumask)) |
| 4478 | return; | 4479 | return; |
| 4479 | } else { | 4480 | } else { |
| @@ -5514,8 +5515,15 @@ int workqueue_online_cpu(unsigned int cpu) · 1034 unmodified lines | |||
| 5514 | } | 5515 | } |
| 5515 | 5516 | ||
| 5516 | /* update NUMA affinity of unbound workqueues */ | 5517 | /* update NUMA affinity of unbound workqueues */ |
| 5517 | list_for_each_entry(wq, &workqueues, list) | 5518 | list_for_each_entry(wq, &workqueues, list) { |
| 5518 | wq_update_unbound_numa(wq, cpu, true); | 5519 | int tcpu; |
| 5520 | |||
| 5521 | for_each_possible_cpu(tcpu) { | ||
| 5522 | if (cpu_to_node(tcpu) == cpu_to_node(cpu)) { | ||
| 5523 | wq_update_unbound_numa(wq, tcpu, cpu, true); | ||
| 5524 | } | ||
| 5525 | } | ||
| 5526 | } | ||
| 5519 | 5527 | ||
| 5520 | mutex_unlock(&wq_pool_mutex); | 5528 | mutex_unlock(&wq_pool_mutex); |
| 5521 | return 0; | 5529 | return 0; |
| @@ -5533,8 +5541,15 @@ int workqueue_offline_cpu(unsigned int cpu) · 11 unmodified lines | |||
| 5533 | 5541 | ||
| 5534 | /* update NUMA affinity of unbound workqueues */ | 5542 | /* update NUMA affinity of unbound workqueues */ |
| 5535 | mutex_lock(&wq_pool_mutex); | 5543 | mutex_lock(&wq_pool_mutex); |
| 5536 | list_for_each_entry(wq, &workqueues, list) | 5544 | list_for_each_entry(wq, &workqueues, list) { |
| 5537 | wq_update_unbound_numa(wq, cpu, false); | 5545 | int tcpu; |
| 5546 | |||
| 5547 | for_each_possible_cpu(tcpu) { | ||
| 5548 | if (cpu_to_node(tcpu) == cpu_to_node(cpu)) { | ||
| 5549 | wq_update_unbound_numa(wq, tcpu, cpu, false); | ||
| 5550 | } | ||
| 5551 | } | ||
| 5552 | } | ||
| 5538 | mutex_unlock(&wq_pool_mutex); | 5553 | mutex_unlock(&wq_pool_mutex); |
| 5539 | 5554 | ||
| 5540 | return 0; | 5555 | return 0; |
| @@ -6509,7 +6524,8 @@ void __init workqueue_init(void) · 968 unmodified lines | |||
| 6509 | } | 6524 | } |
| 6510 | 6525 | ||
| 6511 | list_for_each_entry(wq, &workqueues, list) { | 6526 | list_for_each_entry(wq, &workqueues, list) { |
| 6512 | wq_update_unbound_numa(wq, smp_processor_id(), true); | 6527 | wq_update_unbound_numa(wq, smp_processor_id(), smp_processor_id(), |
| 6528 | true); | ||
| 6513 | WARN(init_rescuer(wq), | 6529 | WARN(init_rescuer(wq), |
| 6514 | "workqueue: failed to create early rescuer for %s", | 6530 | "workqueue: failed to create early rescuer for %s", |
| 6515 | wq->name); | 6531 | wq->name); |