History

workqueue: Initialize unbound CPU pods later in the boot

During boot, to initialize unbound CPU pods, wq_pod_init() was called from workqueue_init(). This is early enough for NUMA nodes to be set up but before SMP is brought up and CPU topology information is populated. Workqueue is in the process of improving CPU locality for unbound workqueues and will need access to topology information during pod init. This adds a new init function workqueue_init_topology() which is called after CPU topology information is available and replaces wq_pod_init(). As unbound CPU pods are now initialized after workqueues are activated, we need to revisit the workqueues to apply the pod configuration. Workqueues which are created before workqueue_init_topology() are set up so that they always use the default worker pool. After pods are set up in workqueue_init_topology(), wq_update_pod() is called on all existing workqueues to update the pool associations accordingly. Note that wq_update_pod_attrs_buf allocation is moved to workqueue_init_early(). This isn't necessary right now but enables further generalization of pod handling in the future. This patch changes the initialization sequence but the end result should be the same.

Tejun Heo committed 3 years ago

Commit 29301551 parent a86feae3 files · +43 −27Browse files

Showing the changes to kernel/workqueue.c (1 of 3 files). Show all

Unified · Split

kernel/workqueue.c+41−27
@@ -6256,17 +6256,15 @@ static inline void wq_watchdog_init(void) { } · 6255 unmodified lines
62566256
62576257 #endif /* CONFIG_WQ_WATCHDOG */
62586258
6259−static void wq_pod_init(void);
6260−
62616259 /**
62626260 * workqueue_init_early - early init for workqueue subsystem
62636261 *
6264− * This is the first half of two-staged workqueue subsystem initialization
6265− * and invoked as soon as the bare basics - memory allocation, cpumasks and
6266− * idr are up. It sets up all the data structures and system workqueues
6267− * and allows early boot code to create workqueues and queue/cancel work
6268− * items. Actual work item execution starts only after kthreads can be
6269− * created and scheduled right before early initcalls.
6262+ * This is the first step of three-staged workqueue subsystem initialization and
6263+ * invoked as soon as the bare basics - memory allocation, cpumasks and idr are
6264+ * up. It sets up all the data structures and system workqueues and allows early
6265+ * boot code to create workqueues and queue/cancel work items. Actual work item
6266+ * execution starts only after kthreads can be created and scheduled right
6267+ * before early initcalls.
62706268 */
62716269 void __init workqueue_init_early(void)
62726270 {
@@ -6284,6 +6282,9 @@ void __init workqueue_init_early(void) · 11 unmodified lines
62846282
62856283 pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);
62866284
6285+ wq_update_pod_attrs_buf = alloc_workqueue_attrs();
6286+ BUG_ON(!wq_update_pod_attrs_buf);
6287+
62876288 /* initialize CPU pools */
62886289 for_each_possible_cpu(cpu) {
62896290 struct worker_pool *pool;
@@ -6381,11 +6382,11 @@ static void __init wq_cpu_intensive_thresh_init(void) · 91 unmodified lines
63816382 /**
63826383 * workqueue_init - bring workqueue subsystem fully online
63836384 *
6384− * This is the latter half of two-staged workqueue subsystem initialization
6385− * and invoked as soon as kthreads can be created and scheduled.
6386− * Workqueues have been created and work items queued on them, but there
6387− * are no kworkers executing the work items yet. Populate the worker pools
6388− * with the initial workers and enable future kworker creations.
6385+ * This is the second step of three-staged workqueue subsystem initialization
6386+ * and invoked as soon as kthreads can be created and scheduled. Workqueues have
6387+ * been created and work items queued on them, but there are no kworkers
6388+ * executing the work items yet. Populate the worker pools with the initial
6389+ * workers and enable future kworker creations.
63896390 */
63906391 void __init workqueue_init(void)
63916392 {
@@ -6395,18 +6396,12 @@ void __init workqueue_init(void) · 3 unmodified lines
63956396
63966397 wq_cpu_intensive_thresh_init();
63976398
6398− /*
6399− * It'd be simpler to initialize pods in workqueue_init_early() but CPU
6400− * to node mapping may not be available that early on some archs such as
6401− * power and arm64. As per-cpu pools created previously could be missing
6402− * node hint and unbound pool pod affinity, fix them up.
6403− *
6404− * Also, while iterating workqueues, create rescuers if requested.
6405− */
6406− wq_pod_init();
6407−
64086399 mutex_lock(&wq_pool_mutex);
64096400
6401+ /*
6402+ * Per-cpu pools created earlier could be missing node hint. Fix them
6403+ * up. Also, create a rescuer for workqueues that requested it.
6404+ */
64106405 for_each_possible_cpu(cpu) {
64116406 for_each_cpu_worker_pool(pool, cpu) {
64126407 pool->node = cpu_to_node(cpu);
@@ -6414,7 +6409,6 @@ void __init workqueue_init(void) · 1 unmodified line
64146409 }
64156410
64166411 list_for_each_entry(wq, &workqueues, list) {
6417− wq_update_pod(wq, smp_processor_id(), smp_processor_id(), true);
64186412 WARN(init_rescuer(wq),
64196413 "workqueue: failed to create early rescuer for %s",
64206414 wq->name);
@@ -6437,8 +6431,16 @@ void __init workqueue_init(void) · 16 unmodified lines
64376431 wq_watchdog_init();
64386432 }
64396433
6440−static void __init wq_pod_init(void)
6434+/**
6435+ * workqueue_init_topology - initialize CPU pods for unbound workqueues
6436+ *
6437+ * This is the third step of there-staged workqueue subsystem initialization and
6438+ * invoked after SMP and topology information are fully initialized. It
6439+ * initializes the unbound CPU pods accordingly.
6440+ */
6441+void __init workqueue_init_topology(void)
64416442 {
6443+ struct workqueue_struct *wq;
64426444 cpumask_var_t *tbl;
64436445 int node, cpu;
64446446
@@ -6452,8 +6454,7 @@ static void __init wq_pod_init(void) · 7 unmodified lines
64526454 }
64536455 }
64546456
6455− wq_update_pod_attrs_buf = alloc_workqueue_attrs();
6456− BUG_ON(!wq_update_pod_attrs_buf);
6457+ mutex_lock(&wq_pool_mutex);
64576458
64586459 /*
64596460 * We want masks of possible CPUs of each node which isn't readily
@@ -6474,6 +6475,19 @@ static void __init wq_pod_init(void) · 14 unmodified lines
64746475
64756476 wq_pod_cpus = tbl;
64766477 wq_pod_enabled = true;
6478+
6479+ /*
6480+ * Workqueues allocated earlier would have all CPUs sharing the default
6481+ * worker pool. Explicitly call wq_update_pod() on all workqueue and CPU
6482+ * combinations to apply per-pod sharing.
6483+ */
6484+ list_for_each_entry(wq, &workqueues, list) {
6485+ for_each_online_cpu(cpu) {
6486+ wq_update_pod(wq, cpu, cpu, true);
6487+ }
6488+ }
6489+
6490+ mutex_unlock(&wq_pool_mutex);
64776491 }
64786492
64796493 void __warn_flushing_systemwide_wq(void)

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t