cpuidle: Fix ct_idle_*() usage

The whole disable-RCU, enable-IRQS dance is very intricate since changing IRQ state is traced, which depends on RCU. Add two helpers for the cpuidle case that mirror the entry code: ct_cpuidle_enter() ct_cpuidle_exit() And fix all the cases where the enter/exit dance was buggy.

Peter Zijlstra committed 3 years ago

Commit a01353c1 parent 0c5ffc315 files · +86 −66Browse files

Unified · Split

15 files

arch/arm/mach-imx/cpuidle-imx6q.c +2 −2
arch/arm/mach-imx/cpuidle-imx6sx.c +2 −2
arch/arm/mach-omap2/cpuidle34xx.c +2 −2
arch/arm/mach-omap2/cpuidle44xx.c +4 −4
drivers/acpi/processor_idle.c +6 −2
drivers/cpuidle/cpuidle-big_little.c +2 −2
drivers/cpuidle/cpuidle-mvebu-v7.c +2 −2
drivers/cpuidle/cpuidle-psci.c +2 −2
drivers/cpuidle/cpuidle-riscv-sbi.c +2 −2
drivers/cpuidle/cpuidle-tegra.c +4 −4
drivers/cpuidle/cpuidle.c +6 −5
include/linux/clockchips.h +2 −2
include/linux/cpuidle.h +32 −2
kernel/sched/idle.c +13 −32
kernel/time/tick-broadcast.c +5 −1

arch/arm/mach-imx/cpuidle-imx6q.c+2−2

@@ -25,9 +25,9 @@ static int imx6q_enter_wait(struct cpuidle_device *dev, · 24 unmodified lines
25	25	imx6_set_lpm(WAIT_UNCLOCKED);
26	26	raw_spin_unlock(&cpuidle_lock);
27	27
28		− ct_idle_enter();
	28	+ ct_cpuidle_enter();
29	29	cpu_do_idle();
30		− ct_idle_exit();
	30	+ ct_cpuidle_exit();
31	31
32	32	raw_spin_lock(&cpuidle_lock);
33	33	if (num_idle_cpus-- == num_online_cpus())

arch/arm/mach-imx/cpuidle-imx6sx.c+2−2

@@ -47,9 +47,9 @@ static int imx6sx_enter_wait(struct cpuidle_device *dev, · 46 unmodified lines
47	47	cpu_pm_enter();
48	48	cpu_cluster_pm_enter();
49	49
50		− ct_idle_enter();
	50	+ ct_cpuidle_enter();
51	51	cpu_suspend(0, imx6sx_idle_finish);
52		− ct_idle_exit();
	52	+ ct_cpuidle_exit();
53	53
54	54	cpu_cluster_pm_exit();
55	55	cpu_pm_exit();

arch/arm/mach-omap2/cpuidle34xx.c+2−2

@@ -133,9 +133,9 @@ static int omap3_enter_idle(struct cpuidle_device *dev, · 132 unmodified lines
133	133	}
134	134
135	135	/* Execute ARM wfi */
136		− ct_idle_enter();
	136	+ ct_cpuidle_enter();
137	137	omap_sram_idle();
138		− ct_idle_exit();
	138	+ ct_cpuidle_exit();
139	139
140	140	/*
141	141	* Call idle CPU PM enter notifier chain to restore

arch/arm/mach-omap2/cpuidle44xx.c+4−4

@@ -105,9 +105,9 @@ static int omap_enter_idle_smp(struct cpuidle_device *dev, · 104 unmodified lines
105	105	}
106	106	raw_spin_unlock_irqrestore(&mpu_lock, flag);
107	107
108		− ct_idle_enter();
	108	+ ct_cpuidle_enter();
109	109	omap4_enter_lowpower(dev->cpu, cx->cpu_state);
110		− ct_idle_exit();
	110	+ ct_cpuidle_exit();
111	111
112	112	raw_spin_lock_irqsave(&mpu_lock, flag);
113	113	if (cx->mpu_state_vote == num_online_cpus())
@@ -186,10 +186,10 @@ static int omap_enter_idle_coupled(struct cpuidle_device *dev, · 72 unmodified lines
186	186	}
187	187	}
188	188
189		− ct_idle_enter();
	189	+ ct_cpuidle_enter();
190	190	omap4_enter_lowpower(dev->cpu, cx->cpu_state);
191	191	cpu_done[dev->cpu] = true;
192		− ct_idle_exit();
	192	+ ct_cpuidle_exit();
193	193
194	194	/* Wakeup CPU1 only if it is not offlined */
195	195	if (dev->cpu == 0 && cpumask_test_cpu(1, cpu_online_mask)) {

drivers/acpi/processor_idle.c+6−2

@@ -642,6 +642,8 @@ static int __cpuidle acpi_idle_enter_bm(struct cpuidle_driver *drv, · 641 unmodified lines
642	642	*/
643	643	bool dis_bm = pr->flags.bm_control;
644	644
	645	+ instrumentation_begin();
	646	+
645	647	/* If we can skip BM, demote to a safe state. */
646	648	if (!cx->bm_sts_skip && acpi_idle_bm_check()) {
647	649	dis_bm = false;
@@ -663,11 +665,11 @@ static int __cpuidle acpi_idle_enter_bm(struct cpuidle_driver *drv, · 15 unmodified lines
663	665	raw_spin_unlock(&c3_lock);
664	666	}
665	667
666		− ct_idle_enter();
	668	+ ct_cpuidle_enter();
667	669
668	670	acpi_idle_do_entry(cx);
669	671
670		− ct_idle_exit();
	672	+ ct_cpuidle_exit();
671	673
672	674	/* Re-enable bus master arbitration */
673	675	if (dis_bm) {
@@ -677,6 +679,8 @@ static int __cpuidle acpi_idle_enter_bm(struct cpuidle_driver *drv, · 3 unmodified lines
677	679	raw_spin_unlock(&c3_lock);
678	680	}
679	681
	682	+ instrumentation_end();
	683	+
680	684	return index;
681	685	}
682	686

drivers/cpuidle/cpuidle-big_little.c+2−2

@@ -126,13 +126,13 @@ static int bl_enter_powerdown(struct cpuidle_device *dev, · 125 unmodified lines
126	126	struct cpuidle_driver *drv, int idx)
127	127	{
128	128	cpu_pm_enter();
129		− ct_idle_enter();
	129	+ ct_cpuidle_enter();
130	130
131	131	cpu_suspend(0, bl_powerdown_finisher);
132	132
133	133	/* signals the MCPM core that CPU is out of low power state */
134	134	mcpm_cpu_powered_up();
135		− ct_idle_exit();
	135	+ ct_cpuidle_exit();
136	136
137	137	cpu_pm_exit();
138	138

drivers/cpuidle/cpuidle-mvebu-v7.c+2−2

@@ -36,9 +36,9 @@ static int mvebu_v7_enter_idle(struct cpuidle_device *dev, · 35 unmodified lines
36	36	if (drv->states[index].flags & MVEBU_V7_FLAG_DEEP_IDLE)
37	37	deepidle = true;
38	38
39		− ct_idle_enter();
	39	+ ct_cpuidle_enter();
40	40	ret = mvebu_v7_cpu_suspend(deepidle);
41		− ct_idle_exit();
	41	+ ct_cpuidle_exit();
42	42
43	43	cpu_pm_exit();
44	44

drivers/cpuidle/cpuidle-psci.c+2−2

@@ -74,7 +74,7 @@ static int __psci_enter_domain_idle_state(struct cpuidle_device *dev, · 73 unmodified lines
74	74	else
75	75	pm_runtime_put_sync_suspend(pd_dev);
76	76
77		− ct_idle_enter();
	77	+ ct_cpuidle_enter();
78	78
79	79	state = psci_get_domain_state();
80	80	if (!state)
@@ -82,7 +82,7 @@ static int __psci_enter_domain_idle_state(struct cpuidle_device *dev, · 1 unmodified line
82	82
83	83	ret = psci_cpu_suspend_enter(state) ? -1 : idx;
84	84
85		− ct_idle_exit();
	85	+ ct_cpuidle_exit();
86	86
87	87	if (s2idle)
88	88	dev_pm_genpd_resume(pd_dev);

drivers/cpuidle/cpuidle-riscv-sbi.c+2−2

@@ -126,7 +126,7 @@ static int __sbi_enter_domain_idle_state(struct cpuidle_device *dev, · 125 unmodified lines
126	126	else
127	127	pm_runtime_put_sync_suspend(pd_dev);
128	128
129		− ct_idle_enter();
	129	+ ct_cpuidle_enter();
130	130
131	131	if (sbi_is_domain_state_available())
132	132	state = sbi_get_domain_state();
@@ -135,7 +135,7 @@ static int __sbi_enter_domain_idle_state(struct cpuidle_device *dev, · 2 unmodified lines
135	135
136	136	ret = sbi_suspend(state) ? -1 : idx;
137	137
138		− ct_idle_exit();
	138	+ ct_cpuidle_exit();
139	139
140	140	if (s2idle)
141	141	dev_pm_genpd_resume(pd_dev);

drivers/cpuidle/cpuidle-tegra.c+4−4

@@ -183,7 +183,7 @@ static int tegra_cpuidle_state_enter(struct cpuidle_device *dev, · 182 unmodified lines
183	183	tegra_pm_set_cpu_in_lp2();
184	184	cpu_pm_enter();
185	185
186		− ct_idle_enter();
	186	+ ct_cpuidle_enter();
187	187
188	188	switch (index) {
189	189	case TEGRA_C7:
@@ -199,7 +199,7 @@ static int tegra_cpuidle_state_enter(struct cpuidle_device *dev, · 9 unmodified lines
199	199	break;
200	200	}
201	201
202		− ct_idle_exit();
	202	+ ct_cpuidle_exit();
203	203
204	204	cpu_pm_exit();
205	205	tegra_pm_clear_cpu_in_lp2();
@@ -240,10 +240,10 @@ static int tegra_cpuidle_enter(struct cpuidle_device *dev, · 34 unmodified lines
240	240
241	241	if (index == TEGRA_C1) {
242	242	if (do_rcu)
243		− ct_idle_enter();
	243	+ ct_cpuidle_enter();
244	244	ret = arm_cpuidle_simple_enter(dev, drv, index);
245	245	if (do_rcu)
246		− ct_idle_exit();
	246	+ ct_cpuidle_exit();
247	247	} else
248	248	ret = tegra_cpuidle_state_enter(dev, index, cpu);
249	249

drivers/cpuidle/cpuidle.c+6−5

@@ -14,6 +14,7 @@ · 13 unmodified lines
14	14	#include <linux/mutex.h>
15	15	#include <linux/sched.h>
16	16	#include <linux/sched/clock.h>
	17	+#include <linux/sched/idle.h>
17	18	#include <linux/notifier.h>
18	19	#include <linux/pm_qos.h>
19	20	#include <linux/cpu.h>
@@ -152,12 +153,12 @@ static void enter_s2idle_proper(struct cpuidle_driver *drv, · 132 unmodified lines
152	153	*/
153	154	stop_critical_timings();
154	155	if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))
155		− ct_idle_enter();
	156	+ ct_cpuidle_enter();
156	157	target_state->enter_s2idle(dev, drv, index);
157	158	if (WARN_ON_ONCE(!irqs_disabled()))
158		− local_irq_disable();
	159	+ raw_local_irq_disable();
159	160	if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))
160		− ct_idle_exit();
	161	+ ct_cpuidle_exit();
161	162	tick_unfreeze();
162	163	start_critical_timings();
163	164
@@ -235,14 +236,14 @@ int cpuidle_enter_state(struct cpuidle_device dev, struct cpuidle_driver drv, · 71 unmodified lines
235	236
236	237	stop_critical_timings();
237	238	if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))
238		− ct_idle_enter();
	239	+ ct_cpuidle_enter();
239	240
240	241	entered_state = target_state->enter(dev, drv, index);
241	242	if (WARN_ONCE(!irqs_disabled(), "%ps leaked IRQ state", target_state->enter))
242	243	raw_local_irq_disable();
243	244
244	245	if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))
245		− ct_idle_exit();
	246	+ ct_cpuidle_exit();
246	247	start_critical_timings();
247	248
248	249	sched_clock_idle_wakeup_event();

include/linux/clockchips.h+2−2

@@ -211,7 +211,7 @@ extern int tick_receive_broadcast(void); · 210 unmodified lines
211	211	extern void tick_setup_hrtimer_broadcast(void);
212	212	extern int tick_check_broadcast_expired(void);
213	213	# else
214		−static inline int tick_check_broadcast_expired(void) { return 0; }
	214	+static __always_inline int tick_check_broadcast_expired(void) { return 0; }
215	215	static inline void tick_setup_hrtimer_broadcast(void) { }
216	216	# endif
217	217
@@ -219,7 +219,7 @@ static inline void tick_setup_hrtimer_broadcast(void) { } · 1 unmodified line
219	219
220	220	static inline void clockevents_suspend(void) { }
221	221	static inline void clockevents_resume(void) { }
222		−static inline int tick_check_broadcast_expired(void) { return 0; }
	222	+static __always_inline int tick_check_broadcast_expired(void) { return 0; }
223	223	static inline void tick_setup_hrtimer_broadcast(void) { }
224	224
225	225	#endif /* !CONFIG_GENERIC_CLOCKEVENTS */

include/linux/cpuidle.h+32−2

@@ -14,6 +14,7 @@ · 13 unmodified lines
14	14	#include <linux/percpu.h>
15	15	#include <linux/list.h>
16	16	#include <linux/hrtimer.h>
	17	+#include <linux/context_tracking.h>
17	18
18	19	#define CPUIDLE_STATE_MAX 10
19	20	#define CPUIDLE_NAME_LEN 16
@@ -115,6 +116,35 @@ struct cpuidle_device { · 95 unmodified lines
115	116	DECLARE_PER_CPU(struct cpuidle_device *, cpuidle_devices);
116	117	DECLARE_PER_CPU(struct cpuidle_device, cpuidle_dev);
117	118
	119	+static __always_inline void ct_cpuidle_enter(void)
	120	+{
	121	+ lockdep_assert_irqs_disabled();
	122	+ /*
	123	+ * Idle is allowed to (temporary) enable IRQs. It
	124	+ * will return with IRQs disabled.
	125	+ *
	126	+ * Trace IRQs enable here, then switch off RCU, and have
	127	+ * arch_cpu_idle() use raw_local_irq_enable(). Note that
	128	+ * ct_idle_enter() relies on lockdep IRQ state, so switch that
	129	+ * last -- this is very similar to the entry code.
	130	+ */
	131	+ trace_hardirqs_on_prepare();
	132	+ lockdep_hardirqs_on_prepare();
	133	+ instrumentation_end();
	134	+ ct_idle_enter();
	135	+ lockdep_hardirqs_on(_RET_IP_);
	136	+}
	137	+
	138	+static __always_inline void ct_cpuidle_exit(void)
	139	+{
	140	+ /*
	141	+ * Carefully undo the above.
	142	+ */
	143	+ lockdep_hardirqs_off(_RET_IP_);
	144	+ ct_idle_exit();
	145	+ instrumentation_begin();
	146	+}
	147	+
118	148	/****************************
119	149	* CPUIDLE DRIVER INTERFACE *
120	150	****************************/
@@ -289,9 +319,9 @@ extern s64 cpuidle_governor_latency_req(unsigned int cpu); · 168 unmodified lines
289	319	if (!is_retention) \
290	320	__ret = cpu_pm_enter(); \
291	321	if (!__ret) { \
292		− ct_idle_enter(); \
	322	+ ct_cpuidle_enter(); \
293	323	__ret = low_level_idle_enter(state); \
294		− ct_idle_exit(); \
	324	+ ct_cpuidle_exit(); \
295	325	if (!is_retention) \
296	326	cpu_pm_exit(); \
297	327	} \

kernel/sched/idle.c+13−32

@@ -51,18 +51,22 @@ __setup("hlt", cpu_idle_nopoll_setup); · 50 unmodified lines
51	51
52	52	static noinline int __cpuidle cpu_idle_poll(void)
53	53	{
	54	+ instrumentation_begin();
54	55	trace_cpu_idle(0, smp_processor_id());
55	56	stop_critical_timings();
56		− ct_idle_enter();
57		− local_irq_enable();
	57	+ ct_cpuidle_enter();
58	58
	59	+ raw_local_irq_enable();
59	60	while (!tif_need_resched() &&
60	61	(cpu_idle_force_poll \|\| tick_check_broadcast_expired()))
61	62	cpu_relax();
	63	+ raw_local_irq_disable();
62	64
63		− ct_idle_exit();
	65	+ ct_cpuidle_exit();
64	66	start_critical_timings();
65	67	trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id());
	68	+ local_irq_enable();
	69	+ instrumentation_end();
66	70
67	71	return 1;
68	72	}
@@ -85,44 +89,21 @@ void __weak arch_cpu_idle(void) · 16 unmodified lines
85	89	*/
86	90	void __cpuidle default_idle_call(void)
87	91	{
88		− if (current_clr_polling_and_test()) {
89		− local_irq_enable();
90		− } else {
91		−
	92	+ instrumentation_begin();
	93	+ if (!current_clr_polling_and_test()) {
92	94	trace_cpu_idle(1, smp_processor_id());
93	95	stop_critical_timings();
94	96
95		− /*
96		− * arch_cpu_idle() is supposed to enable IRQs, however
97		− * we can't do that because of RCU and tracing.
98		− *
99		− * Trace IRQs enable here, then switch off RCU, and have
100		− * arch_cpu_idle() use raw_local_irq_enable(). Note that
101		− * ct_idle_enter() relies on lockdep IRQ state, so switch that
102		− * last -- this is very similar to the entry code.
103		− */
104		− trace_hardirqs_on_prepare();
105		− lockdep_hardirqs_on_prepare();
106		− ct_idle_enter();
107		− lockdep_hardirqs_on(_THIS_IP_);
108		−
	97	+ ct_cpuidle_enter();
109	98	arch_cpu_idle();
110		−
111		− /*
112		− * OK, so IRQs are enabled here, but RCU needs them disabled to
113		− * turn itself back on.. funny thing is that disabling IRQs
114		− * will cause tracing, which needs RCU. Jump through hoops to
115		− * make it 'work'.
116		− */
117	99	raw_local_irq_disable();
118		− lockdep_hardirqs_off(_THIS_IP_);
119		− ct_idle_exit();
120		− lockdep_hardirqs_on(_THIS_IP_);
121		− raw_local_irq_enable();
	100	+ ct_cpuidle_exit();
122	101
123	102	start_critical_timings();
124	103	trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id());
125	104	}
	105	+ local_irq_enable();
	106	+ instrumentation_end();
126	107	}
127	108
128	109	static int call_cpuidle_s2idle(struct cpuidle_driver *drv,

kernel/time/tick-broadcast.c+5−1

@@ -622,9 +622,13 @@ struct cpumask *tick_get_broadcast_oneshot_mask(void) · 621 unmodified lines
622	622	* to avoid a deep idle transition as we are about to get the
623	623	* broadcast IPI right away.
624	624	*/
625		−int tick_check_broadcast_expired(void)
	625	+noinstr int tick_check_broadcast_expired(void)
626	626	{
	627	+#ifdef _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H
	628	+ return arch_test_bit(smp_processor_id(), cpumask_bits(tick_broadcast_force_mask));
	629	+#else
627	630	return cpumask_test_cpu(smp_processor_id(), tick_broadcast_force_mask);
	631	+#endif
628	632	}
629	633
630	634	/*