With CONFIG_IRQ_FORCED_THREADING=y, testing the boolean force_irqthreads could incur a cache line miss in invoke_softirq() and other places. Replace the test with a static key to avoid the potential cache miss. [ tglx: Dropped the IDE part, removed the export and updated blk-mq ]
Showing the changes to kernel (2 of 4 files). Show all
Unified · Split
| @@ -25,12 +25,11 @@ · 24 unmodified lines | ||
| 25 | 25 | #include "internals.h" |
| 26 | 26 | |
| 27 | 27 | #if defined(CONFIG_IRQ_FORCED_THREADING) && !defined(CONFIG_PREEMPT_RT) |
| 28 | −__read_mostly bool force_irqthreads; | |
| 29 | −EXPORT_SYMBOL_GPL(force_irqthreads); | |
| 28 | +DEFINE_STATIC_KEY_FALSE(force_irqthreads_key); | |
| 30 | 29 | |
| 31 | 30 | static int __init setup_forced_irqthreads(char *arg) |
| 32 | 31 | { |
| 33 | − force_irqthreads = true; | |
| 32 | + static_branch_enable(&force_irqthreads_key); | |
| 34 | 33 | return 0; |
| 35 | 34 | } |
| 36 | 35 | early_param("threadirqs", setup_forced_irqthreads); |
| @@ -1260,8 +1259,8 @@ static int irq_thread(void *data) · 1223 unmodified lines | ||
| 1260 | 1259 | irqreturn_t (*handler_fn)(struct irq_desc *desc, |
| 1261 | 1260 | struct irqaction *action); |
| 1262 | 1261 | |
| 1263 | − if (force_irqthreads && test_bit(IRQTF_FORCED_THREAD, | |
| 1264 | − &action->thread_flags)) | |
| 1262 | + if (force_irqthreads() && test_bit(IRQTF_FORCED_THREAD, | |
| 1263 | + &action->thread_flags)) | |
| 1265 | 1264 | handler_fn = irq_forced_thread_fn; |
| 1266 | 1265 | else |
| 1267 | 1266 | handler_fn = irq_thread_fn; |
| @@ -1322,7 +1321,7 @@ EXPORT_SYMBOL_GPL(irq_wake_thread); · 54 unmodified lines | ||
| 1322 | 1321 | |
| 1323 | 1322 | static int irq_setup_forced_threading(struct irqaction *new) |
| 1324 | 1323 | { |
| 1325 | − if (!force_irqthreads) | |
| 1324 | + if (!force_irqthreads()) | |
| 1326 | 1325 | return 0; |
| 1327 | 1326 | if (new->flags & (IRQF_NO_THREAD | IRQF_PERCPU | IRQF_ONESHOT)) |
| 1328 | 1327 | return 0; |
| @@ -422,7 +422,7 @@ static inline void invoke_softirq(void) · 421 unmodified lines | ||
| 422 | 422 | if (ksoftirqd_running(local_softirq_pending())) |
| 423 | 423 | return; |
| 424 | 424 | |
| 425 | − if (!force_irqthreads || !__this_cpu_read(ksoftirqd)) { | |
| 425 | + if (!force_irqthreads() || !__this_cpu_read(ksoftirqd)) { | |
| 426 | 426 | #ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK |
| 427 | 427 | /* |
| 428 | 428 | * We can safely execute softirq on the current stack if |