genirq: Change force_irqthreads to a static key

With CONFIG_IRQ_FORCED_THREADING=y, testing the boolean force_irqthreads could incur a cache line miss in invoke_softirq() and other places. Replace the test with a static key to avoid the potential cache miss. [ tglx: Dropped the IDE part, removed the export and updated blk-mq ]

Tanner Love committed 5 years ago

Commit 91cc4701 parent 5a6c76b4 files · +12 −11Browse files

Unified · Split

4 files

block/blk-mq.c +1 −1
include/linux/interrupt.h +5 −3
kernel/irq/manage.c +5 −6
kernel/softirq.c +1 −1

block/blk-mq.c+1−1

@@ -606,7 +606,7 @@ static inline bool blk_mq_complete_need_ipi(struct request *rq) · 605 unmodified lines
606	606	* This is probably worse than completing the request on a different
607	607	* cache domain.
608	608	*/
609		− if (force_irqthreads)
	609	+ if (force_irqthreads())
610	610	return false;
611	611
612	612	/* same CPU or cache domain? Complete locally */

include/linux/interrupt.h+5−3

@@ -13,6 +13,7 @@ · 12 unmodified lines
13	13	#include <linux/hrtimer.h>
14	14	#include <linux/kref.h>
15	15	#include <linux/workqueue.h>
	16	+#include <linux/jump_label.h>
16	17
17	18	#include <linux/atomic.h>
18	19	#include <asm/ptrace.h>
@@ -474,12 +475,13 @@ extern int irq_set_irqchip_state(unsigned int irq, enum irqchip_irq_state which, · 455 unmodified lines
474	475
475	476	#ifdef CONFIG_IRQ_FORCED_THREADING
476	477	# ifdef CONFIG_PREEMPT_RT
477		−# define force_irqthreads (true)
	478	+# define force_irqthreads() (true)
478	479	# else
479		−extern bool force_irqthreads;
	480	+DECLARE_STATIC_KEY_FALSE(force_irqthreads_key);
	481	+# define force_irqthreads() (static_branch_unlikely(&force_irqthreads_key))
480	482	# endif
481	483	#else
482		−#define force_irqthreads (0)
	484	+#define force_irqthreads() (false)
483	485	#endif
484	486
485	487	#ifndef local_softirq_pending

kernel/irq/manage.c+5−6

@@ -25,12 +25,11 @@ · 24 unmodified lines
25	25	#include "internals.h"
26	26
27	27	#if defined(CONFIG_IRQ_FORCED_THREADING) && !defined(CONFIG_PREEMPT_RT)
28		−__read_mostly bool force_irqthreads;
29		−EXPORT_SYMBOL_GPL(force_irqthreads);
	28	+DEFINE_STATIC_KEY_FALSE(force_irqthreads_key);
30	29
31	30	static int __init setup_forced_irqthreads(char *arg)
32	31	{
33		− force_irqthreads = true;
	32	+ static_branch_enable(&force_irqthreads_key);
34	33	return 0;
35	34	}
36	35	early_param("threadirqs", setup_forced_irqthreads);
@@ -1260,8 +1259,8 @@ static int irq_thread(void *data) · 1223 unmodified lines
1260	1259	irqreturn_t (handler_fn)(struct irq_desc desc,
1261	1260	struct irqaction *action);
1262	1261
1263		− if (force_irqthreads && test_bit(IRQTF_FORCED_THREAD,
1264		− &action->thread_flags))
	1262	+ if (force_irqthreads() && test_bit(IRQTF_FORCED_THREAD,
	1263	+ &action->thread_flags))
1265	1264	handler_fn = irq_forced_thread_fn;
1266	1265	else
1267	1266	handler_fn = irq_thread_fn;
@@ -1322,7 +1321,7 @@ EXPORT_SYMBOL_GPL(irq_wake_thread); · 54 unmodified lines
1322	1321
1323	1322	static int irq_setup_forced_threading(struct irqaction *new)
1324	1323	{
1325		− if (!force_irqthreads)
	1324	+ if (!force_irqthreads())
1326	1325	return 0;
1327	1326	if (new->flags & (IRQF_NO_THREAD \| IRQF_PERCPU \| IRQF_ONESHOT))
1328	1327	return 0;

kernel/softirq.c+1−1

@@ -422,7 +422,7 @@ static inline void invoke_softirq(void) · 421 unmodified lines
422	422	if (ksoftirqd_running(local_softirq_pending()))
423	423	return;
424	424
425		− if (!force_irqthreads \|\| !__this_cpu_read(ksoftirqd)) {
	425	+ if (!force_irqthreads() \|\| !__this_cpu_read(ksoftirqd)) {
426	426	#ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
427	427	/*
428	428	* We can safely execute softirq on the current stack if