History

workqueue: Initialize unbound CPU pods later in the boot

During boot, to initialize unbound CPU pods, wq_pod_init() was called from workqueue_init(). This is early enough for NUMA nodes to be set up but before SMP is brought up and CPU topology information is populated. Workqueue is in the process of improving CPU locality for unbound workqueues and will need access to topology information during pod init. This adds a new init function workqueue_init_topology() which is called after CPU topology information is available and replaces wq_pod_init(). As unbound CPU pods are now initialized after workqueues are activated, we need to revisit the workqueues to apply the pod configuration. Workqueues which are created before workqueue_init_topology() are set up so that they always use the default worker pool. After pods are set up in workqueue_init_topology(), wq_update_pod() is called on all existing workqueues to update the pool associations accordingly. Note that wq_update_pod_attrs_buf allocation is moved to workqueue_init_early(). This isn't necessary right now but enables further generalization of pod handling in the future. This patch changes the initialization sequence but the end result should be the same.

Tejun Heo committed 3 years ago

Commit 29301551 parent a86feae3 files · +43 −27Browse files

Showing the changes to kernel/workqueue.c (1 of 3 files). Show all

Unified · Split

kernel/workqueue.c+41−27
@@ -6256,17 +6256,15 @@ static inline void wq_watchdog_init(void) { } · 6255 unmodified lines
6256 6256
6257#endif /* CONFIG_WQ_WATCHDOG */6257#endif /* CONFIG_WQ_WATCHDOG */
6258 6258
6259static void wq_pod_init(void);
6260
6261/**6259/**
6262 * workqueue_init_early - early init for workqueue subsystem6260 * workqueue_init_early - early init for workqueue subsystem
6263 *6261 *
6264 * This is the first half of two-staged workqueue subsystem initialization6262 * This is the first step of three-staged workqueue subsystem initialization and
6265 * and invoked as soon as the bare basics - memory allocation, cpumasks and6263 * invoked as soon as the bare basics - memory allocation, cpumasks and idr are
6266 * idr are up. It sets up all the data structures and system workqueues6264 * up. It sets up all the data structures and system workqueues and allows early
6267 * and allows early boot code to create workqueues and queue/cancel work6265 * boot code to create workqueues and queue/cancel work items. Actual work item
6268 * items. Actual work item execution starts only after kthreads can be6266 * execution starts only after kthreads can be created and scheduled right
6269 * created and scheduled right before early initcalls.6267 * before early initcalls.
6270 */6268 */
6271void __init workqueue_init_early(void)6269void __init workqueue_init_early(void)
6272{6270{
@@ -6284,6 +6282,9 @@ void __init workqueue_init_early(void) · 11 unmodified lines
6284 6282
6285 pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);6283 pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);
6286 6284
6285 wq_update_pod_attrs_buf = alloc_workqueue_attrs();
6286 BUG_ON(!wq_update_pod_attrs_buf);
6287
6287 /* initialize CPU pools */6288 /* initialize CPU pools */
6288 for_each_possible_cpu(cpu) {6289 for_each_possible_cpu(cpu) {
6289 struct worker_pool *pool;6290 struct worker_pool *pool;
@@ -6381,11 +6382,11 @@ static void __init wq_cpu_intensive_thresh_init(void) · 91 unmodified lines
6381/**6382/**
6382 * workqueue_init - bring workqueue subsystem fully online6383 * workqueue_init - bring workqueue subsystem fully online
6383 *6384 *
6384 * This is the latter half of two-staged workqueue subsystem initialization6385 * This is the second step of three-staged workqueue subsystem initialization
6385 * and invoked as soon as kthreads can be created and scheduled.6386 * and invoked as soon as kthreads can be created and scheduled. Workqueues have
6386 * Workqueues have been created and work items queued on them, but there6387 * been created and work items queued on them, but there are no kworkers
6387 * are no kworkers executing the work items yet. Populate the worker pools6388 * executing the work items yet. Populate the worker pools with the initial
6388 * with the initial workers and enable future kworker creations.6389 * workers and enable future kworker creations.
6389 */6390 */
6390void __init workqueue_init(void)6391void __init workqueue_init(void)
6391{6392{
@@ -6395,18 +6396,12 @@ void __init workqueue_init(void) · 3 unmodified lines
6395 6396
6396 wq_cpu_intensive_thresh_init();6397 wq_cpu_intensive_thresh_init();
6397 6398
6398 /*
6399 * It'd be simpler to initialize pods in workqueue_init_early() but CPU
6400 * to node mapping may not be available that early on some archs such as
6401 * power and arm64. As per-cpu pools created previously could be missing
6402 * node hint and unbound pool pod affinity, fix them up.
6403 *
6404 * Also, while iterating workqueues, create rescuers if requested.
6405 */
6406 wq_pod_init();
6407
6408 mutex_lock(&wq_pool_mutex);6399 mutex_lock(&wq_pool_mutex);
6409 6400
6401 /*
6402 * Per-cpu pools created earlier could be missing node hint. Fix them
6403 * up. Also, create a rescuer for workqueues that requested it.
6404 */
6410 for_each_possible_cpu(cpu) {6405 for_each_possible_cpu(cpu) {
6411 for_each_cpu_worker_pool(pool, cpu) {6406 for_each_cpu_worker_pool(pool, cpu) {
6412 pool->node = cpu_to_node(cpu);6407 pool->node = cpu_to_node(cpu);
@@ -6414,7 +6409,6 @@ void __init workqueue_init(void) · 1 unmodified line
6414 }6409 }
6415 6410
6416 list_for_each_entry(wq, &workqueues, list) {6411 list_for_each_entry(wq, &workqueues, list) {
6417 wq_update_pod(wq, smp_processor_id(), smp_processor_id(), true);
6418 WARN(init_rescuer(wq),6412 WARN(init_rescuer(wq),
6419 "workqueue: failed to create early rescuer for %s",6413 "workqueue: failed to create early rescuer for %s",
6420 wq->name);6414 wq->name);
@@ -6437,8 +6431,16 @@ void __init workqueue_init(void) · 16 unmodified lines
6437 wq_watchdog_init();6431 wq_watchdog_init();
6438}6432}
6439 6433
6440static void __init wq_pod_init(void)6434/**
6435 * workqueue_init_topology - initialize CPU pods for unbound workqueues
6436 *
6437 * This is the third step of there-staged workqueue subsystem initialization and
6438 * invoked after SMP and topology information are fully initialized. It
6439 * initializes the unbound CPU pods accordingly.
6440 */
6441void __init workqueue_init_topology(void)
6441{6442{
6443 struct workqueue_struct *wq;
6442 cpumask_var_t *tbl;6444 cpumask_var_t *tbl;
6443 int node, cpu;6445 int node, cpu;
6444 6446
@@ -6452,8 +6454,7 @@ static void __init wq_pod_init(void) · 7 unmodified lines
6452 }6454 }
6453 }6455 }
6454 6456
6455 wq_update_pod_attrs_buf = alloc_workqueue_attrs();6457 mutex_lock(&wq_pool_mutex);
6456 BUG_ON(!wq_update_pod_attrs_buf);
6457 6458
6458 /*6459 /*
6459 * We want masks of possible CPUs of each node which isn't readily6460 * We want masks of possible CPUs of each node which isn't readily
@@ -6474,6 +6475,19 @@ static void __init wq_pod_init(void) · 14 unmodified lines
6474 6475
6475 wq_pod_cpus = tbl;6476 wq_pod_cpus = tbl;
6476 wq_pod_enabled = true;6477 wq_pod_enabled = true;
6478
6479 /*
6480 * Workqueues allocated earlier would have all CPUs sharing the default
6481 * worker pool. Explicitly call wq_update_pod() on all workqueue and CPU
6482 * combinations to apply per-pod sharing.
6483 */
6484 list_for_each_entry(wq, &workqueues, list) {
6485 for_each_online_cpu(cpu) {
6486 wq_update_pod(wq, cpu, cpu, true);
6487 }
6488 }
6489
6490 mutex_unlock(&wq_pool_mutex);
6477}6491}
6478 6492
6479void __warn_flushing_systemwide_wq(void)6493void __warn_flushing_systemwide_wq(void)

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t