| @@ -21,6 +21,7 @@ · 20 unmodified lines |
| 21 | 21 | #include "common.h" |
| 22 | 22 | #include "pm.h" |
| 23 | 23 | #include "prm.h" |
| 24 | +#include "clockdomain.h" |
| 24 | 25 | |
| 25 | 26 | /* Machine specific information */ |
| 26 | 27 | struct omap4_idle_statedata { |
| @@ -47,10 +48,14 @@ static struct omap4_idle_statedata omap4_idle_data[] = { · 20 unmodified lines |
| 47 | 48 | }, |
| 48 | 49 | }; |
| 49 | 50 | |
| 50 | | −static struct powerdomain *mpu_pd, *cpu0_pd, *cpu1_pd; |
| 51 | +static struct powerdomain *mpu_pd, *cpu_pd[NR_CPUS]; |
| 52 | +static struct clockdomain *cpu_clkdm[NR_CPUS]; |
| 53 | + |
| 54 | +static atomic_t abort_barrier; |
| 55 | +static bool cpu_done[NR_CPUS]; |
| 51 | 56 | |
| 52 | 57 | /** |
| 53 | | − * omap4_enter_idle - Programs OMAP4 to enter the specified state |
| 58 | + * omap4_enter_idle_coupled_[simple/coupled] - OMAP4 cpuidle entry functions |
| 54 | 59 | * @dev: cpuidle device |
| 55 | 60 | * @drv: cpuidle driver |
| 56 | 61 | * @index: the index of state to be entered |
| @@ -59,60 +64,84 @@ static struct powerdomain *mpu_pd, *cpu0_pd, *cpu1_pd; · 2 unmodified lines |
| 59 | 64 | * specified low power state selected by the governor. |
| 60 | 65 | * Returns the amount of time spent in the low power state. |
| 61 | 66 | */ |
| 62 | | −static int omap4_enter_idle(struct cpuidle_device *dev, |
| 67 | +static int omap4_enter_idle_simple(struct cpuidle_device *dev, |
| 68 | + struct cpuidle_driver *drv, |
| 69 | + int index) |
| 70 | +{ |
| 71 | + local_fiq_disable(); |
| 72 | + omap_do_wfi(); |
| 73 | + local_fiq_enable(); |
| 74 | + |
| 75 | + return index; |
| 76 | +} |
| 77 | + |
| 78 | +static int omap4_enter_idle_coupled(struct cpuidle_device *dev, |
| 63 | 79 | struct cpuidle_driver *drv, |
| 64 | 80 | int index) |
| 65 | 81 | { |
| 66 | 82 | struct omap4_idle_statedata *cx = &omap4_idle_data[index]; |
| 67 | | − u32 cpu1_state; |
| 68 | 83 | int cpu_id = smp_processor_id(); |
| 69 | 84 | |
| 70 | 85 | local_fiq_disable(); |
| 71 | 86 | |
| 72 | 87 | /* |
| 73 | | − * CPU0 has to stay ON (i.e in C1) until CPU1 is OFF state. |
| 88 | + * CPU0 has to wait and stay ON until CPU1 is OFF state. |
| 74 | 89 | * This is necessary to honour hardware recommondation |
| 75 | 90 | * of triggeing all the possible low power modes once CPU1 is |
| 76 | 91 | * out of coherency and in OFF mode. |
| 77 | | − * Update dev->last_state so that governor stats reflects right |
| 78 | | − * data. |
| 79 | 92 | */ |
| 80 | | − cpu1_state = pwrdm_read_pwrst(cpu1_pd); |
| 81 | | − if (cpu1_state != PWRDM_POWER_OFF) { |
| 82 | | − index = drv->safe_state_index; |
| 83 | | − cx = &omap4_idle_data[index]; |
| 93 | + if (dev->cpu == 0 && cpumask_test_cpu(1, cpu_online_mask)) { |
| 94 | + while (pwrdm_read_pwrst(cpu_pd[1]) != PWRDM_POWER_OFF) { |
| 95 | + cpu_relax(); |
| 96 | + |
| 97 | + /* |
| 98 | + * CPU1 could have already entered & exited idle |
| 99 | + * without hitting off because of a wakeup |
| 100 | + * or a failed attempt to hit off mode. Check for |
| 101 | + * that here, otherwise we could spin forever |
| 102 | + * waiting for CPU1 off. |
| 103 | + */ |
| 104 | + if (cpu_done[1]) |
| 105 | + goto fail; |
| 106 | + |
| 107 | + } |
| 84 | 108 | } |
| 85 | 109 | |
| 86 | | − if (index > 0) |
| 87 | | − clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &cpu_id); |
| 110 | + clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &cpu_id); |
| 88 | 111 | |
| 89 | 112 | /* |
| 90 | 113 | * Call idle CPU PM enter notifier chain so that |
| 91 | 114 | * VFP and per CPU interrupt context is saved. |
| 92 | 115 | */ |
| 93 | | − if (cx->cpu_state == PWRDM_POWER_OFF) |
| 94 | | − cpu_pm_enter(); |
| 95 | | − |
| 96 | | − pwrdm_set_logic_retst(mpu_pd, cx->mpu_logic_state); |
| 97 | | − omap_set_pwrdm_state(mpu_pd, cx->mpu_state); |
| 98 | | − |
| 99 | | − /* |
| 100 | | − * Call idle CPU cluster PM enter notifier chain |
| 101 | | − * to save GIC and wakeupgen context. |
| 102 | | − */ |
| 103 | | − if ((cx->mpu_state == PWRDM_POWER_RET) && |
| 104 | | − (cx->mpu_logic_state == PWRDM_POWER_OFF)) |
| 105 | | − cpu_cluster_pm_enter(); |
| 116 | + cpu_pm_enter(); |
| 117 | + |
| 118 | + if (dev->cpu == 0) { |
| 119 | + pwrdm_set_logic_retst(mpu_pd, cx->mpu_logic_state); |
| 120 | + omap_set_pwrdm_state(mpu_pd, cx->mpu_state); |
| 121 | + |
| 122 | + /* |
| 123 | + * Call idle CPU cluster PM enter notifier chain |
| 124 | + * to save GIC and wakeupgen context. |
| 125 | + */ |
| 126 | + if ((cx->mpu_state == PWRDM_POWER_RET) && |
| 127 | + (cx->mpu_logic_state == PWRDM_POWER_OFF)) |
| 128 | + cpu_cluster_pm_enter(); |
| 129 | + } |
| 106 | 130 | |
| 107 | 131 | omap4_enter_lowpower(dev->cpu, cx->cpu_state); |
| 132 | + cpu_done[dev->cpu] = true; |
| 133 | + |
| 134 | + /* Wakeup CPU1 only if it is not offlined */ |
| 135 | + if (dev->cpu == 0 && cpumask_test_cpu(1, cpu_online_mask)) { |
| 136 | + clkdm_wakeup(cpu_clkdm[1]); |
| 137 | + clkdm_allow_idle(cpu_clkdm[1]); |
| 138 | + } |
| 108 | 139 | |
| 109 | 140 | /* |
| 110 | 141 | * Call idle CPU PM exit notifier chain to restore |
| 111 | | − * VFP and per CPU IRQ context. Only CPU0 state is |
| 112 | | − * considered since CPU1 is managed by CPU hotplug. |
| 142 | + * VFP and per CPU IRQ context. |
| 113 | 143 | */ |
| 114 | | − if (pwrdm_read_prev_pwrst(cpu0_pd) == PWRDM_POWER_OFF) |
| 115 | | − cpu_pm_exit(); |
| 144 | + cpu_pm_exit(); |
| 116 | 145 | |
| 117 | 146 | /* |
| 118 | 147 | * Call idle CPU cluster PM exit notifier chain |
| @@ -121,8 +150,11 @@ static int omap4_enter_idle(struct cpuidle_device *dev, · 2 unmodified lines |
| 121 | 150 | if (omap4_mpuss_read_prev_context_state()) |
| 122 | 151 | cpu_cluster_pm_exit(); |
| 123 | 152 | |
| 124 | | − if (index > 0) |
| 125 | | − clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &cpu_id); |
| 153 | + clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &cpu_id); |
| 154 | + |
| 155 | +fail: |
| 156 | + cpuidle_coupled_parallel_barrier(dev, &abort_barrier); |
| 157 | + cpu_done[dev->cpu] = false; |
| 126 | 158 | |
| 127 | 159 | local_fiq_enable(); |
| 128 | 160 | |
| @@ -141,7 +173,7 @@ struct cpuidle_driver omap4_idle_driver = { · 12 unmodified lines |
| 141 | 173 | .exit_latency = 2 + 2, |
| 142 | 174 | .target_residency = 5, |
| 143 | 175 | .flags = CPUIDLE_FLAG_TIME_VALID, |
| 144 | | − .enter = omap4_enter_idle, |
| 176 | + .enter = omap4_enter_idle_simple, |
| 145 | 177 | .name = "C1", |
| 146 | 178 | .desc = "MPUSS ON" |
| 147 | 179 | }, |
| @@ -149,8 +181,8 @@ struct cpuidle_driver omap4_idle_driver = { · 1 unmodified line |
| 149 | 181 | /* C2 - CPU0 OFF + CPU1 OFF + MPU CSWR */ |
| 150 | 182 | .exit_latency = 328 + 440, |
| 151 | 183 | .target_residency = 960, |
| 152 | | − .flags = CPUIDLE_FLAG_TIME_VALID, |
| 153 | | − .enter = omap4_enter_idle, |
| 184 | + .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_COUPLED, |
| 185 | + .enter = omap4_enter_idle_coupled, |
| 154 | 186 | .name = "C2", |
| 155 | 187 | .desc = "MPUSS CSWR", |
| 156 | 188 | }, |
| @@ -158,8 +190,8 @@ struct cpuidle_driver omap4_idle_driver = { · 1 unmodified line |
| 158 | 190 | /* C3 - CPU0 OFF + CPU1 OFF + MPU OSWR */ |
| 159 | 191 | .exit_latency = 460 + 518, |
| 160 | 192 | .target_residency = 1100, |
| 161 | | − .flags = CPUIDLE_FLAG_TIME_VALID, |
| 162 | | − .enter = omap4_enter_idle, |
| 193 | + .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_COUPLED, |
| 194 | + .enter = omap4_enter_idle_coupled, |
| 163 | 195 | .name = "C3", |
| 164 | 196 | .desc = "MPUSS OSWR", |
| 165 | 197 | }, |
| @@ -168,6 +200,16 @@ struct cpuidle_driver omap4_idle_driver = { · 2 unmodified lines |
| 168 | 200 | .safe_state_index = 0, |
| 169 | 201 | }; |
| 170 | 202 | |
| 203 | +/* |
| 204 | + * For each cpu, setup the broadcast timer because local timers |
| 205 | + * stops for the states above C1. |
| 206 | + */ |
| 207 | +static void omap_setup_broadcast_timer(void *arg) |
| 208 | +{ |
| 209 | + int cpu = smp_processor_id(); |
| 210 | + clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ON, &cpu); |
| 211 | +} |
| 212 | + |
| 171 | 213 | /** |
| 172 | 214 | * omap4_idle_init - Init routine for OMAP4 idle |
| 173 | 215 | * |
| @@ -180,19 +222,30 @@ int __init omap4_idle_init(void) · 6 unmodified lines |
| 180 | 222 | unsigned int cpu_id = 0; |
| 181 | 223 | |
| 182 | 224 | mpu_pd = pwrdm_lookup("mpu_pwrdm"); |
| 183 | | − cpu0_pd = pwrdm_lookup("cpu0_pwrdm"); |
| 184 | | − cpu1_pd = pwrdm_lookup("cpu1_pwrdm"); |
| 185 | | − if ((!mpu_pd) || (!cpu0_pd) || (!cpu1_pd)) |
| 225 | + cpu_pd[0] = pwrdm_lookup("cpu0_pwrdm"); |
| 226 | + cpu_pd[1] = pwrdm_lookup("cpu1_pwrdm"); |
| 227 | + if ((!mpu_pd) || (!cpu_pd[0]) || (!cpu_pd[1])) |
| 186 | 228 | return -ENODEV; |
| 187 | 229 | |
| 188 | | − dev = &per_cpu(omap4_idle_dev, cpu_id); |
| 189 | | − dev->cpu = cpu_id; |
| 230 | + cpu_clkdm[0] = clkdm_lookup("mpu0_clkdm"); |
| 231 | + cpu_clkdm[1] = clkdm_lookup("mpu1_clkdm"); |
| 232 | + if (!cpu_clkdm[0] || !cpu_clkdm[1]) |
| 233 | + return -ENODEV; |
| 234 | + |
| 235 | + /* Configure the broadcast timer on each cpu */ |
| 236 | + on_each_cpu(omap_setup_broadcast_timer, NULL, 1); |
| 237 | + |
| 238 | + for_each_cpu(cpu_id, cpu_online_mask) { |
| 239 | + dev = &per_cpu(omap4_idle_dev, cpu_id); |
| 240 | + dev->cpu = cpu_id; |
| 241 | + dev->coupled_cpus = *cpu_online_mask; |
| 190 | 242 | |
| 191 | | − cpuidle_register_driver(&omap4_idle_driver); |
| 243 | + cpuidle_register_driver(&omap4_idle_driver); |
| 192 | 244 | |
| 193 | | − if (cpuidle_register_device(dev)) { |
| 194 | | − pr_err("%s: CPUidle register device failed\n", __func__); |
| 195 | | − return -EIO; |
| 245 | + if (cpuidle_register_device(dev)) { |
| 246 | + pr_err("%s: CPUidle register failed\n", __func__); |
| 247 | + return -EIO; |
| 248 | + } |
| 196 | 249 | } |
| 197 | 250 | |
| 198 | 251 | return 0; |