History

genirq: Change force_irqthreads to a static key

With CONFIG_IRQ_FORCED_THREADING=y, testing the boolean force_irqthreads could incur a cache line miss in invoke_softirq() and other places. Replace the test with a static key to avoid the potential cache miss. [ tglx: Dropped the IDE part, removed the export and updated blk-mq ]

Tanner Love committed 5 years ago

Commit 91cc4701 parent 5a6c76b4 files · +12 −11Browse files

Showing the changes to kernel (2 of 4 files). Show all

Unified · Split

2 files
  1. kernel/irq/manage.c +5 −6
  2. kernel/softirq.c +1 −1
kernel/irq/manage.c+5−6
@@ -25,12 +25,11 @@ · 24 unmodified lines
25#include "internals.h"25#include "internals.h"
26 26
27#if defined(CONFIG_IRQ_FORCED_THREADING) && !defined(CONFIG_PREEMPT_RT)27#if defined(CONFIG_IRQ_FORCED_THREADING) && !defined(CONFIG_PREEMPT_RT)
28__read_mostly bool force_irqthreads;28DEFINE_STATIC_KEY_FALSE(force_irqthreads_key);
29EXPORT_SYMBOL_GPL(force_irqthreads);
30 29
31static int __init setup_forced_irqthreads(char *arg)30static int __init setup_forced_irqthreads(char *arg)
32{31{
33 force_irqthreads = true;32 static_branch_enable(&force_irqthreads_key);
34 return 0;33 return 0;
35}34}
36early_param("threadirqs", setup_forced_irqthreads);35early_param("threadirqs", setup_forced_irqthreads);
@@ -1260,8 +1259,8 @@ static int irq_thread(void *data) · 1223 unmodified lines
1260 irqreturn_t (*handler_fn)(struct irq_desc *desc,1259 irqreturn_t (*handler_fn)(struct irq_desc *desc,
1261 struct irqaction *action);1260 struct irqaction *action);
1262 1261
1263 if (force_irqthreads && test_bit(IRQTF_FORCED_THREAD,1262 if (force_irqthreads() && test_bit(IRQTF_FORCED_THREAD,
1264 &action->thread_flags))1263 &action->thread_flags))
1265 handler_fn = irq_forced_thread_fn;1264 handler_fn = irq_forced_thread_fn;
1266 else1265 else
1267 handler_fn = irq_thread_fn;1266 handler_fn = irq_thread_fn;
@@ -1322,7 +1321,7 @@ EXPORT_SYMBOL_GPL(irq_wake_thread); · 54 unmodified lines
1322 1321
1323static int irq_setup_forced_threading(struct irqaction *new)1322static int irq_setup_forced_threading(struct irqaction *new)
1324{1323{
1325 if (!force_irqthreads)1324 if (!force_irqthreads())
1326 return 0;1325 return 0;
1327 if (new->flags & (IRQF_NO_THREAD | IRQF_PERCPU | IRQF_ONESHOT))1326 if (new->flags & (IRQF_NO_THREAD | IRQF_PERCPU | IRQF_ONESHOT))
1328 return 0;1327 return 0;
kernel/softirq.c+1−1
@@ -422,7 +422,7 @@ static inline void invoke_softirq(void) · 421 unmodified lines
422 if (ksoftirqd_running(local_softirq_pending()))422 if (ksoftirqd_running(local_softirq_pending()))
423 return;423 return;
424 424
425 if (!force_irqthreads || !__this_cpu_read(ksoftirqd)) {425 if (!force_irqthreads() || !__this_cpu_read(ksoftirqd)) {
426#ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK426#ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
427 /*427 /*
428 * We can safely execute softirq on the current stack if428 * We can safely execute softirq on the current stack if