History

workqueue: Fix queue_work_on() with BH workqueues

When queue_work_on() is used to queue a BH work item on a remote CPU, the work item is queued on that CPU but kick_pool() raises softirq on the local CPU. This leads to stalls as the work item won't be executed until something else on the remote CPU schedules a BH work item or tasklet locally. Fix it by bouncing raising softirq to the target CPU using per-cpu irq_work.

Tejun Heo committed 2 years ago

Commit 2f34d731 parent bf52b1a1 file · +36 −5Browse files

Showing the changes to kernel/workqueue.c (1 of 1 file). Show all

Unified · Split

kernel/workqueue.c+36−5
@@ -54,6 +54,7 @@ · 53 unmodified lines
54#include <linux/nmi.h>54#include <linux/nmi.h>
55#include <linux/kvm_para.h>55#include <linux/kvm_para.h>
56#include <linux/delay.h>56#include <linux/delay.h>
57#include <linux/irq_work.h>
57 58
58#include "workqueue_internal.h"59#include "workqueue_internal.h"
59 60
@@ -457,6 +458,10 @@ static bool wq_debug_force_rr_cpu = false; · 397 unmodified lines
457#endif458#endif
458module_param_named(debug_force_rr_cpu, wq_debug_force_rr_cpu, bool, 0644);459module_param_named(debug_force_rr_cpu, wq_debug_force_rr_cpu, bool, 0644);
459 460
461/* to raise softirq for the BH worker pools on other CPUs */
462static DEFINE_PER_CPU_SHARED_ALIGNED(struct irq_work [NR_STD_WORKER_POOLS],
463 bh_pool_irq_works);
464
460/* the BH worker pools */465/* the BH worker pools */
461static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS],466static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS],
462 bh_worker_pools);467 bh_worker_pools);
@@ -1197,6 +1202,13 @@ static bool assign_work(struct work_struct *work, struct worker *worker, · 734 unmodified lines
1197 return true;1202 return true;
1198}1203}
1199 1204
1205static struct irq_work *bh_pool_irq_work(struct worker_pool *pool)
1206{
1207 int high = pool->attrs->nice == HIGHPRI_NICE_LEVEL ? 1 : 0;
1208
1209 return &per_cpu(bh_pool_irq_works, pool->cpu)[high];
1210}
1211
1200/**1212/**
1201 * kick_pool - wake up an idle worker if necessary1213 * kick_pool - wake up an idle worker if necessary
1202 * @pool: pool to kick1214 * @pool: pool to kick
@@ -1215,10 +1227,15 @@ static bool kick_pool(struct worker_pool *pool) · 12 unmodified lines
1215 return false;1227 return false;
1216 1228
1217 if (pool->flags & POOL_BH) {1229 if (pool->flags & POOL_BH) {
1218 if (pool->attrs->nice == HIGHPRI_NICE_LEVEL)1230 if (likely(pool->cpu == smp_processor_id())) {
1219 raise_softirq_irqoff(HI_SOFTIRQ);1231 if (pool->attrs->nice == HIGHPRI_NICE_LEVEL)
1220 else1232 raise_softirq_irqoff(HI_SOFTIRQ);
1221 raise_softirq_irqoff(TASKLET_SOFTIRQ);1233 else
1234 raise_softirq_irqoff(TASKLET_SOFTIRQ);
1235 } else {
1236 irq_work_queue_on(bh_pool_irq_work(pool), pool->cpu);
1237 }
1238
1222 return true;1239 return true;
1223 }1240 }
1224 1241
@@ -7367,6 +7384,16 @@ static inline void wq_watchdog_init(void) { } · 6142 unmodified lines
7367 7384
7368#endif /* CONFIG_WQ_WATCHDOG */7385#endif /* CONFIG_WQ_WATCHDOG */
7369 7386
7387static void bh_pool_kick_normal(struct irq_work *irq_work)
7388{
7389 raise_softirq_irqoff(TASKLET_SOFTIRQ);
7390}
7391
7392static void bh_pool_kick_highpri(struct irq_work *irq_work)
7393{
7394 raise_softirq_irqoff(HI_SOFTIRQ);
7395}
7396
7370static void __init restrict_unbound_cpumask(const char *name, const struct cpumask *mask)7397static void __init restrict_unbound_cpumask(const char *name, const struct cpumask *mask)
7371{7398{
7372 if (!cpumask_intersects(wq_unbound_cpumask, mask)) {7399 if (!cpumask_intersects(wq_unbound_cpumask, mask)) {
@@ -7408,6 +7435,8 @@ void __init workqueue_init_early(void) · 35 unmodified lines
7408{7435{
7409 struct wq_pod_type *pt = &wq_pod_types[WQ_AFFN_SYSTEM];7436 struct wq_pod_type *pt = &wq_pod_types[WQ_AFFN_SYSTEM];
7410 int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL };7437 int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL };
7438 void (*irq_work_fns[2])(struct irq_work *) = { bh_pool_kick_normal,
7439 bh_pool_kick_highpri };
7411 int i, cpu;7440 int i, cpu;
7412 7441
7413 BUILD_BUG_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));7442 BUILD_BUG_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));
@@ -7455,8 +7484,10 @@ void __init workqueue_init_early(void) · 41 unmodified lines
7455 7484
7456 i = 0;7485 i = 0;
7457 for_each_bh_worker_pool(pool, cpu) {7486 for_each_bh_worker_pool(pool, cpu) {
7458 init_cpu_worker_pool(pool, cpu, std_nice[i++]);7487 init_cpu_worker_pool(pool, cpu, std_nice[i]);
7459 pool->flags |= POOL_BH;7488 pool->flags |= POOL_BH;
7489 init_irq_work(bh_pool_irq_work(pool), irq_work_fns[i]);
7490 i++;
7460 }7491 }
7461 7492
7462 i = 0;7493 i = 0;