History

genirq: Change force_irqthreads to a static key

With CONFIG_IRQ_FORCED_THREADING=y, testing the boolean force_irqthreads could incur a cache line miss in invoke_softirq() and other places. Replace the test with a static key to avoid the potential cache miss. [ tglx: Dropped the IDE part, removed the export and updated blk-mq ]

Tanner Love committed 5 years ago

Commit 91cc4701 parent 5a6c76b4 files · +12 −11Browse files

Unified · Split

4 files
  1. block/blk-mq.c +1 −1
  2. include/linux/interrupt.h +5 −3
  3. kernel/irq/manage.c +5 −6
  4. kernel/softirq.c +1 −1
block/blk-mq.c+1−1
@@ -606,7 +606,7 @@ static inline bool blk_mq_complete_need_ipi(struct request *rq) · 605 unmodified lines
606 * This is probably worse than completing the request on a different606 * This is probably worse than completing the request on a different
607 * cache domain.607 * cache domain.
608 */608 */
609 if (force_irqthreads)609 if (force_irqthreads())
610 return false;610 return false;
611 611
612 /* same CPU or cache domain? Complete locally */612 /* same CPU or cache domain? Complete locally */
include/linux/interrupt.h+5−3
@@ -13,6 +13,7 @@ · 12 unmodified lines
13#include <linux/hrtimer.h>13#include <linux/hrtimer.h>
14#include <linux/kref.h>14#include <linux/kref.h>
15#include <linux/workqueue.h>15#include <linux/workqueue.h>
16#include <linux/jump_label.h>
16 17
17#include <linux/atomic.h>18#include <linux/atomic.h>
18#include <asm/ptrace.h>19#include <asm/ptrace.h>
@@ -474,12 +475,13 @@ extern int irq_set_irqchip_state(unsigned int irq, enum irqchip_irq_state which, · 455 unmodified lines
474 475
475#ifdef CONFIG_IRQ_FORCED_THREADING476#ifdef CONFIG_IRQ_FORCED_THREADING
476# ifdef CONFIG_PREEMPT_RT477# ifdef CONFIG_PREEMPT_RT
477# define force_irqthreads (true)478# define force_irqthreads() (true)
478# else479# else
479extern bool force_irqthreads;480DECLARE_STATIC_KEY_FALSE(force_irqthreads_key);
481# define force_irqthreads() (static_branch_unlikely(&force_irqthreads_key))
480# endif482# endif
481#else483#else
482#define force_irqthreads (0)484#define force_irqthreads() (false)
483#endif485#endif
484 486
485#ifndef local_softirq_pending487#ifndef local_softirq_pending
kernel/irq/manage.c+5−6
@@ -25,12 +25,11 @@ · 24 unmodified lines
25#include "internals.h"25#include "internals.h"
26 26
27#if defined(CONFIG_IRQ_FORCED_THREADING) && !defined(CONFIG_PREEMPT_RT)27#if defined(CONFIG_IRQ_FORCED_THREADING) && !defined(CONFIG_PREEMPT_RT)
28__read_mostly bool force_irqthreads;28DEFINE_STATIC_KEY_FALSE(force_irqthreads_key);
29EXPORT_SYMBOL_GPL(force_irqthreads);
30 29
31static int __init setup_forced_irqthreads(char *arg)30static int __init setup_forced_irqthreads(char *arg)
32{31{
33 force_irqthreads = true;32 static_branch_enable(&force_irqthreads_key);
34 return 0;33 return 0;
35}34}
36early_param("threadirqs", setup_forced_irqthreads);35early_param("threadirqs", setup_forced_irqthreads);
@@ -1260,8 +1259,8 @@ static int irq_thread(void *data) · 1223 unmodified lines
1260 irqreturn_t (*handler_fn)(struct irq_desc *desc,1259 irqreturn_t (*handler_fn)(struct irq_desc *desc,
1261 struct irqaction *action);1260 struct irqaction *action);
1262 1261
1263 if (force_irqthreads && test_bit(IRQTF_FORCED_THREAD,1262 if (force_irqthreads() && test_bit(IRQTF_FORCED_THREAD,
1264 &action->thread_flags))1263 &action->thread_flags))
1265 handler_fn = irq_forced_thread_fn;1264 handler_fn = irq_forced_thread_fn;
1266 else1265 else
1267 handler_fn = irq_thread_fn;1266 handler_fn = irq_thread_fn;
@@ -1322,7 +1321,7 @@ EXPORT_SYMBOL_GPL(irq_wake_thread); · 54 unmodified lines
1322 1321
1323static int irq_setup_forced_threading(struct irqaction *new)1322static int irq_setup_forced_threading(struct irqaction *new)
1324{1323{
1325 if (!force_irqthreads)1324 if (!force_irqthreads())
1326 return 0;1325 return 0;
1327 if (new->flags & (IRQF_NO_THREAD | IRQF_PERCPU | IRQF_ONESHOT))1326 if (new->flags & (IRQF_NO_THREAD | IRQF_PERCPU | IRQF_ONESHOT))
1328 return 0;1327 return 0;
kernel/softirq.c+1−1
@@ -422,7 +422,7 @@ static inline void invoke_softirq(void) · 421 unmodified lines
422 if (ksoftirqd_running(local_softirq_pending()))422 if (ksoftirqd_running(local_softirq_pending()))
423 return;423 return;
424 424
425 if (!force_irqthreads || !__this_cpu_read(ksoftirqd)) {425 if (!force_irqthreads() || !__this_cpu_read(ksoftirqd)) {
426#ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK426#ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
427 /*427 /*
428 * We can safely execute softirq on the current stack if428 * We can safely execute softirq on the current stack if