History

cpuidle: Fix ct_idle_*() usage

The whole disable-RCU, enable-IRQS dance is very intricate since changing IRQ state is traced, which depends on RCU. Add two helpers for the cpuidle case that mirror the entry code: ct_cpuidle_enter() ct_cpuidle_exit() And fix all the cases where the enter/exit dance was buggy.

Peter Zijlstra committed 3 years ago

Commit a01353c1 parent 0c5ffc315 files · +86 −66Browse files

Unified · Split

15 files
  1. arch/arm/mach-imx/cpuidle-imx6q.c +2 −2
  2. arch/arm/mach-imx/cpuidle-imx6sx.c +2 −2
  3. arch/arm/mach-omap2/cpuidle34xx.c +2 −2
  4. arch/arm/mach-omap2/cpuidle44xx.c +4 −4
  5. drivers/acpi/processor_idle.c +6 −2
  6. drivers/cpuidle/cpuidle-big_little.c +2 −2
  7. drivers/cpuidle/cpuidle-mvebu-v7.c +2 −2
  8. drivers/cpuidle/cpuidle-psci.c +2 −2
  9. drivers/cpuidle/cpuidle-riscv-sbi.c +2 −2
  10. drivers/cpuidle/cpuidle-tegra.c +4 −4
  11. drivers/cpuidle/cpuidle.c +6 −5
  12. include/linux/clockchips.h +2 −2
  13. include/linux/cpuidle.h +32 −2
  14. kernel/sched/idle.c +13 −32
  15. kernel/time/tick-broadcast.c +5 −1
arch/arm/mach-imx/cpuidle-imx6q.c+2−2
@@ -25,9 +25,9 @@ static int imx6q_enter_wait(struct cpuidle_device *dev, · 24 unmodified lines
25 imx6_set_lpm(WAIT_UNCLOCKED);25 imx6_set_lpm(WAIT_UNCLOCKED);
26 raw_spin_unlock(&cpuidle_lock);26 raw_spin_unlock(&cpuidle_lock);
27 27
28 ct_idle_enter();28 ct_cpuidle_enter();
29 cpu_do_idle();29 cpu_do_idle();
30 ct_idle_exit();30 ct_cpuidle_exit();
31 31
32 raw_spin_lock(&cpuidle_lock);32 raw_spin_lock(&cpuidle_lock);
33 if (num_idle_cpus-- == num_online_cpus())33 if (num_idle_cpus-- == num_online_cpus())
arch/arm/mach-imx/cpuidle-imx6sx.c+2−2
@@ -47,9 +47,9 @@ static int imx6sx_enter_wait(struct cpuidle_device *dev, · 46 unmodified lines
47 cpu_pm_enter();47 cpu_pm_enter();
48 cpu_cluster_pm_enter();48 cpu_cluster_pm_enter();
49 49
50 ct_idle_enter();50 ct_cpuidle_enter();
51 cpu_suspend(0, imx6sx_idle_finish);51 cpu_suspend(0, imx6sx_idle_finish);
52 ct_idle_exit();52 ct_cpuidle_exit();
53 53
54 cpu_cluster_pm_exit();54 cpu_cluster_pm_exit();
55 cpu_pm_exit();55 cpu_pm_exit();
arch/arm/mach-omap2/cpuidle34xx.c+2−2
@@ -133,9 +133,9 @@ static int omap3_enter_idle(struct cpuidle_device *dev, · 132 unmodified lines
133 }133 }
134 134
135 /* Execute ARM wfi */135 /* Execute ARM wfi */
136 ct_idle_enter();136 ct_cpuidle_enter();
137 omap_sram_idle();137 omap_sram_idle();
138 ct_idle_exit();138 ct_cpuidle_exit();
139 139
140 /*140 /*
141 * Call idle CPU PM enter notifier chain to restore141 * Call idle CPU PM enter notifier chain to restore
arch/arm/mach-omap2/cpuidle44xx.c+4−4
@@ -105,9 +105,9 @@ static int omap_enter_idle_smp(struct cpuidle_device *dev, · 104 unmodified lines
105 }105 }
106 raw_spin_unlock_irqrestore(&mpu_lock, flag);106 raw_spin_unlock_irqrestore(&mpu_lock, flag);
107 107
108 ct_idle_enter();108 ct_cpuidle_enter();
109 omap4_enter_lowpower(dev->cpu, cx->cpu_state);109 omap4_enter_lowpower(dev->cpu, cx->cpu_state);
110 ct_idle_exit();110 ct_cpuidle_exit();
111 111
112 raw_spin_lock_irqsave(&mpu_lock, flag);112 raw_spin_lock_irqsave(&mpu_lock, flag);
113 if (cx->mpu_state_vote == num_online_cpus())113 if (cx->mpu_state_vote == num_online_cpus())
@@ -186,10 +186,10 @@ static int omap_enter_idle_coupled(struct cpuidle_device *dev, · 72 unmodified lines
186 }186 }
187 }187 }
188 188
189 ct_idle_enter();189 ct_cpuidle_enter();
190 omap4_enter_lowpower(dev->cpu, cx->cpu_state);190 omap4_enter_lowpower(dev->cpu, cx->cpu_state);
191 cpu_done[dev->cpu] = true;191 cpu_done[dev->cpu] = true;
192 ct_idle_exit();192 ct_cpuidle_exit();
193 193
194 /* Wakeup CPU1 only if it is not offlined */194 /* Wakeup CPU1 only if it is not offlined */
195 if (dev->cpu == 0 && cpumask_test_cpu(1, cpu_online_mask)) {195 if (dev->cpu == 0 && cpumask_test_cpu(1, cpu_online_mask)) {
drivers/acpi/processor_idle.c+6−2
@@ -642,6 +642,8 @@ static int __cpuidle acpi_idle_enter_bm(struct cpuidle_driver *drv, · 641 unmodified lines
642 */642 */
643 bool dis_bm = pr->flags.bm_control;643 bool dis_bm = pr->flags.bm_control;
644 644
645 instrumentation_begin();
646
645 /* If we can skip BM, demote to a safe state. */647 /* If we can skip BM, demote to a safe state. */
646 if (!cx->bm_sts_skip && acpi_idle_bm_check()) {648 if (!cx->bm_sts_skip && acpi_idle_bm_check()) {
647 dis_bm = false;649 dis_bm = false;
@@ -663,11 +665,11 @@ static int __cpuidle acpi_idle_enter_bm(struct cpuidle_driver *drv, · 15 unmodified lines
663 raw_spin_unlock(&c3_lock);665 raw_spin_unlock(&c3_lock);
664 }666 }
665 667
666 ct_idle_enter();668 ct_cpuidle_enter();
667 669
668 acpi_idle_do_entry(cx);670 acpi_idle_do_entry(cx);
669 671
670 ct_idle_exit();672 ct_cpuidle_exit();
671 673
672 /* Re-enable bus master arbitration */674 /* Re-enable bus master arbitration */
673 if (dis_bm) {675 if (dis_bm) {
@@ -677,6 +679,8 @@ static int __cpuidle acpi_idle_enter_bm(struct cpuidle_driver *drv, · 3 unmodified lines
677 raw_spin_unlock(&c3_lock);679 raw_spin_unlock(&c3_lock);
678 }680 }
679 681
682 instrumentation_end();
683
680 return index;684 return index;
681}685}
682 686
drivers/cpuidle/cpuidle-big_little.c+2−2
@@ -126,13 +126,13 @@ static int bl_enter_powerdown(struct cpuidle_device *dev, · 125 unmodified lines
126 struct cpuidle_driver *drv, int idx)126 struct cpuidle_driver *drv, int idx)
127{127{
128 cpu_pm_enter();128 cpu_pm_enter();
129 ct_idle_enter();129 ct_cpuidle_enter();
130 130
131 cpu_suspend(0, bl_powerdown_finisher);131 cpu_suspend(0, bl_powerdown_finisher);
132 132
133 /* signals the MCPM core that CPU is out of low power state */133 /* signals the MCPM core that CPU is out of low power state */
134 mcpm_cpu_powered_up();134 mcpm_cpu_powered_up();
135 ct_idle_exit();135 ct_cpuidle_exit();
136 136
137 cpu_pm_exit();137 cpu_pm_exit();
138 138
drivers/cpuidle/cpuidle-mvebu-v7.c+2−2
@@ -36,9 +36,9 @@ static int mvebu_v7_enter_idle(struct cpuidle_device *dev, · 35 unmodified lines
36 if (drv->states[index].flags & MVEBU_V7_FLAG_DEEP_IDLE)36 if (drv->states[index].flags & MVEBU_V7_FLAG_DEEP_IDLE)
37 deepidle = true;37 deepidle = true;
38 38
39 ct_idle_enter();39 ct_cpuidle_enter();
40 ret = mvebu_v7_cpu_suspend(deepidle);40 ret = mvebu_v7_cpu_suspend(deepidle);
41 ct_idle_exit();41 ct_cpuidle_exit();
42 42
43 cpu_pm_exit();43 cpu_pm_exit();
44 44
drivers/cpuidle/cpuidle-psci.c+2−2
@@ -74,7 +74,7 @@ static int __psci_enter_domain_idle_state(struct cpuidle_device *dev, · 73 unmodified lines
74 else74 else
75 pm_runtime_put_sync_suspend(pd_dev);75 pm_runtime_put_sync_suspend(pd_dev);
76 76
77 ct_idle_enter();77 ct_cpuidle_enter();
78 78
79 state = psci_get_domain_state();79 state = psci_get_domain_state();
80 if (!state)80 if (!state)
@@ -82,7 +82,7 @@ static int __psci_enter_domain_idle_state(struct cpuidle_device *dev, · 1 unmodified line
82 82
83 ret = psci_cpu_suspend_enter(state) ? -1 : idx;83 ret = psci_cpu_suspend_enter(state) ? -1 : idx;
84 84
85 ct_idle_exit();85 ct_cpuidle_exit();
86 86
87 if (s2idle)87 if (s2idle)
88 dev_pm_genpd_resume(pd_dev);88 dev_pm_genpd_resume(pd_dev);
drivers/cpuidle/cpuidle-riscv-sbi.c+2−2
@@ -126,7 +126,7 @@ static int __sbi_enter_domain_idle_state(struct cpuidle_device *dev, · 125 unmodified lines
126 else126 else
127 pm_runtime_put_sync_suspend(pd_dev);127 pm_runtime_put_sync_suspend(pd_dev);
128 128
129 ct_idle_enter();129 ct_cpuidle_enter();
130 130
131 if (sbi_is_domain_state_available())131 if (sbi_is_domain_state_available())
132 state = sbi_get_domain_state();132 state = sbi_get_domain_state();
@@ -135,7 +135,7 @@ static int __sbi_enter_domain_idle_state(struct cpuidle_device *dev, · 2 unmodified lines
135 135
136 ret = sbi_suspend(state) ? -1 : idx;136 ret = sbi_suspend(state) ? -1 : idx;
137 137
138 ct_idle_exit();138 ct_cpuidle_exit();
139 139
140 if (s2idle)140 if (s2idle)
141 dev_pm_genpd_resume(pd_dev);141 dev_pm_genpd_resume(pd_dev);
drivers/cpuidle/cpuidle-tegra.c+4−4
@@ -183,7 +183,7 @@ static int tegra_cpuidle_state_enter(struct cpuidle_device *dev, · 182 unmodified lines
183 tegra_pm_set_cpu_in_lp2();183 tegra_pm_set_cpu_in_lp2();
184 cpu_pm_enter();184 cpu_pm_enter();
185 185
186 ct_idle_enter();186 ct_cpuidle_enter();
187 187
188 switch (index) {188 switch (index) {
189 case TEGRA_C7:189 case TEGRA_C7:
@@ -199,7 +199,7 @@ static int tegra_cpuidle_state_enter(struct cpuidle_device *dev, · 9 unmodified lines
199 break;199 break;
200 }200 }
201 201
202 ct_idle_exit();202 ct_cpuidle_exit();
203 203
204 cpu_pm_exit();204 cpu_pm_exit();
205 tegra_pm_clear_cpu_in_lp2();205 tegra_pm_clear_cpu_in_lp2();
@@ -240,10 +240,10 @@ static int tegra_cpuidle_enter(struct cpuidle_device *dev, · 34 unmodified lines
240 240
241 if (index == TEGRA_C1) {241 if (index == TEGRA_C1) {
242 if (do_rcu)242 if (do_rcu)
243 ct_idle_enter();243 ct_cpuidle_enter();
244 ret = arm_cpuidle_simple_enter(dev, drv, index);244 ret = arm_cpuidle_simple_enter(dev, drv, index);
245 if (do_rcu)245 if (do_rcu)
246 ct_idle_exit();246 ct_cpuidle_exit();
247 } else247 } else
248 ret = tegra_cpuidle_state_enter(dev, index, cpu);248 ret = tegra_cpuidle_state_enter(dev, index, cpu);
249 249
drivers/cpuidle/cpuidle.c+6−5
@@ -14,6 +14,7 @@ · 13 unmodified lines
14#include <linux/mutex.h>14#include <linux/mutex.h>
15#include <linux/sched.h>15#include <linux/sched.h>
16#include <linux/sched/clock.h>16#include <linux/sched/clock.h>
17#include <linux/sched/idle.h>
17#include <linux/notifier.h>18#include <linux/notifier.h>
18#include <linux/pm_qos.h>19#include <linux/pm_qos.h>
19#include <linux/cpu.h>20#include <linux/cpu.h>
@@ -152,12 +153,12 @@ static void enter_s2idle_proper(struct cpuidle_driver *drv, · 132 unmodified lines
152 */153 */
153 stop_critical_timings();154 stop_critical_timings();
154 if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))155 if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))
155 ct_idle_enter();156 ct_cpuidle_enter();
156 target_state->enter_s2idle(dev, drv, index);157 target_state->enter_s2idle(dev, drv, index);
157 if (WARN_ON_ONCE(!irqs_disabled()))158 if (WARN_ON_ONCE(!irqs_disabled()))
158 local_irq_disable();159 raw_local_irq_disable();
159 if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))160 if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))
160 ct_idle_exit();161 ct_cpuidle_exit();
161 tick_unfreeze();162 tick_unfreeze();
162 start_critical_timings();163 start_critical_timings();
163 164
@@ -235,14 +236,14 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv, · 71 unmodified lines
235 236
236 stop_critical_timings();237 stop_critical_timings();
237 if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))238 if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))
238 ct_idle_enter();239 ct_cpuidle_enter();
239 240
240 entered_state = target_state->enter(dev, drv, index);241 entered_state = target_state->enter(dev, drv, index);
241 if (WARN_ONCE(!irqs_disabled(), "%ps leaked IRQ state", target_state->enter))242 if (WARN_ONCE(!irqs_disabled(), "%ps leaked IRQ state", target_state->enter))
242 raw_local_irq_disable();243 raw_local_irq_disable();
243 244
244 if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))245 if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))
245 ct_idle_exit();246 ct_cpuidle_exit();
246 start_critical_timings();247 start_critical_timings();
247 248
248 sched_clock_idle_wakeup_event();249 sched_clock_idle_wakeup_event();
include/linux/clockchips.h+2−2
@@ -211,7 +211,7 @@ extern int tick_receive_broadcast(void); · 210 unmodified lines
211extern void tick_setup_hrtimer_broadcast(void);211extern void tick_setup_hrtimer_broadcast(void);
212extern int tick_check_broadcast_expired(void);212extern int tick_check_broadcast_expired(void);
213# else213# else
214static inline int tick_check_broadcast_expired(void) { return 0; }214static __always_inline int tick_check_broadcast_expired(void) { return 0; }
215static inline void tick_setup_hrtimer_broadcast(void) { }215static inline void tick_setup_hrtimer_broadcast(void) { }
216# endif216# endif
217 217
@@ -219,7 +219,7 @@ static inline void tick_setup_hrtimer_broadcast(void) { } · 1 unmodified line
219 219
220static inline void clockevents_suspend(void) { }220static inline void clockevents_suspend(void) { }
221static inline void clockevents_resume(void) { }221static inline void clockevents_resume(void) { }
222static inline int tick_check_broadcast_expired(void) { return 0; }222static __always_inline int tick_check_broadcast_expired(void) { return 0; }
223static inline void tick_setup_hrtimer_broadcast(void) { }223static inline void tick_setup_hrtimer_broadcast(void) { }
224 224
225#endif /* !CONFIG_GENERIC_CLOCKEVENTS */225#endif /* !CONFIG_GENERIC_CLOCKEVENTS */
include/linux/cpuidle.h+32−2
@@ -14,6 +14,7 @@ · 13 unmodified lines
14#include <linux/percpu.h>14#include <linux/percpu.h>
15#include <linux/list.h>15#include <linux/list.h>
16#include <linux/hrtimer.h>16#include <linux/hrtimer.h>
17#include <linux/context_tracking.h>
17 18
18#define CPUIDLE_STATE_MAX 1019#define CPUIDLE_STATE_MAX 10
19#define CPUIDLE_NAME_LEN 1620#define CPUIDLE_NAME_LEN 16
@@ -115,6 +116,35 @@ struct cpuidle_device { · 95 unmodified lines
115DECLARE_PER_CPU(struct cpuidle_device *, cpuidle_devices);116DECLARE_PER_CPU(struct cpuidle_device *, cpuidle_devices);
116DECLARE_PER_CPU(struct cpuidle_device, cpuidle_dev);117DECLARE_PER_CPU(struct cpuidle_device, cpuidle_dev);
117 118
119static __always_inline void ct_cpuidle_enter(void)
120{
121 lockdep_assert_irqs_disabled();
122 /*
123 * Idle is allowed to (temporary) enable IRQs. It
124 * will return with IRQs disabled.
125 *
126 * Trace IRQs enable here, then switch off RCU, and have
127 * arch_cpu_idle() use raw_local_irq_enable(). Note that
128 * ct_idle_enter() relies on lockdep IRQ state, so switch that
129 * last -- this is very similar to the entry code.
130 */
131 trace_hardirqs_on_prepare();
132 lockdep_hardirqs_on_prepare();
133 instrumentation_end();
134 ct_idle_enter();
135 lockdep_hardirqs_on(_RET_IP_);
136}
137
138static __always_inline void ct_cpuidle_exit(void)
139{
140 /*
141 * Carefully undo the above.
142 */
143 lockdep_hardirqs_off(_RET_IP_);
144 ct_idle_exit();
145 instrumentation_begin();
146}
147
118/****************************148/****************************
119 * CPUIDLE DRIVER INTERFACE *149 * CPUIDLE DRIVER INTERFACE *
120 ****************************/150 ****************************/
@@ -289,9 +319,9 @@ extern s64 cpuidle_governor_latency_req(unsigned int cpu); · 168 unmodified lines
289 if (!is_retention) \319 if (!is_retention) \
290 __ret = cpu_pm_enter(); \320 __ret = cpu_pm_enter(); \
291 if (!__ret) { \321 if (!__ret) { \
292 ct_idle_enter(); \322 ct_cpuidle_enter(); \
293 __ret = low_level_idle_enter(state); \323 __ret = low_level_idle_enter(state); \
294 ct_idle_exit(); \324 ct_cpuidle_exit(); \
295 if (!is_retention) \325 if (!is_retention) \
296 cpu_pm_exit(); \326 cpu_pm_exit(); \
297 } \327 } \
kernel/sched/idle.c+13−32
@@ -51,18 +51,22 @@ __setup("hlt", cpu_idle_nopoll_setup); · 50 unmodified lines
51 51
52static noinline int __cpuidle cpu_idle_poll(void)52static noinline int __cpuidle cpu_idle_poll(void)
53{53{
54 instrumentation_begin();
54 trace_cpu_idle(0, smp_processor_id());55 trace_cpu_idle(0, smp_processor_id());
55 stop_critical_timings();56 stop_critical_timings();
56 ct_idle_enter();57 ct_cpuidle_enter();
57 local_irq_enable();
58 58
59 raw_local_irq_enable();
59 while (!tif_need_resched() &&60 while (!tif_need_resched() &&
60 (cpu_idle_force_poll || tick_check_broadcast_expired()))61 (cpu_idle_force_poll || tick_check_broadcast_expired()))
61 cpu_relax();62 cpu_relax();
63 raw_local_irq_disable();
62 64
63 ct_idle_exit();65 ct_cpuidle_exit();
64 start_critical_timings();66 start_critical_timings();
65 trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id());67 trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id());
68 local_irq_enable();
69 instrumentation_end();
66 70
67 return 1;71 return 1;
68}72}
@@ -85,44 +89,21 @@ void __weak arch_cpu_idle(void) · 16 unmodified lines
85 */89 */
86void __cpuidle default_idle_call(void)90void __cpuidle default_idle_call(void)
87{91{
88 if (current_clr_polling_and_test()) {92 instrumentation_begin();
89 local_irq_enable();93 if (!current_clr_polling_and_test()) {
90 } else {
91
92 trace_cpu_idle(1, smp_processor_id());94 trace_cpu_idle(1, smp_processor_id());
93 stop_critical_timings();95 stop_critical_timings();
94 96
95 /*97 ct_cpuidle_enter();
96 * arch_cpu_idle() is supposed to enable IRQs, however
97 * we can't do that because of RCU and tracing.
98 *
99 * Trace IRQs enable here, then switch off RCU, and have
100 * arch_cpu_idle() use raw_local_irq_enable(). Note that
101 * ct_idle_enter() relies on lockdep IRQ state, so switch that
102 * last -- this is very similar to the entry code.
103 */
104 trace_hardirqs_on_prepare();
105 lockdep_hardirqs_on_prepare();
106 ct_idle_enter();
107 lockdep_hardirqs_on(_THIS_IP_);
108
109 arch_cpu_idle();98 arch_cpu_idle();
110
111 /*
112 * OK, so IRQs are enabled here, but RCU needs them disabled to
113 * turn itself back on.. funny thing is that disabling IRQs
114 * will cause tracing, which needs RCU. Jump through hoops to
115 * make it 'work'.
116 */
117 raw_local_irq_disable();99 raw_local_irq_disable();
118 lockdep_hardirqs_off(_THIS_IP_);100 ct_cpuidle_exit();
119 ct_idle_exit();
120 lockdep_hardirqs_on(_THIS_IP_);
121 raw_local_irq_enable();
122 101
123 start_critical_timings();102 start_critical_timings();
124 trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id());103 trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id());
125 }104 }
105 local_irq_enable();
106 instrumentation_end();
126}107}
127 108
128static int call_cpuidle_s2idle(struct cpuidle_driver *drv,109static int call_cpuidle_s2idle(struct cpuidle_driver *drv,
kernel/time/tick-broadcast.c+5−1
@@ -622,9 +622,13 @@ struct cpumask *tick_get_broadcast_oneshot_mask(void) · 621 unmodified lines
622 * to avoid a deep idle transition as we are about to get the622 * to avoid a deep idle transition as we are about to get the
623 * broadcast IPI right away.623 * broadcast IPI right away.
624 */624 */
625int tick_check_broadcast_expired(void)625noinstr int tick_check_broadcast_expired(void)
626{626{
627#ifdef _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H
628 return arch_test_bit(smp_processor_id(), cpumask_bits(tick_broadcast_force_mask));
629#else
627 return cpumask_test_cpu(smp_processor_id(), tick_broadcast_force_mask);630 return cpumask_test_cpu(smp_processor_id(), tick_broadcast_force_mask);
631#endif
628}632}
629 633
630/*634/*