Unified · Split
| @@ -22,6 +22,7 @@ · 21 unmodified lines | ||
| 22 | 22 | #include <linux/sched.h> |
| 23 | 23 | #include <linux/suspend.h> |
| 24 | 24 | #include <linux/export.h> |
| 25 | +#include <linux/cpu.h> | |
| 25 | 26 | |
| 26 | 27 | #include "power.h" |
| 27 | 28 | |
| @@ -128,6 +129,7 @@ static const struct genpd_lock_ops genpd_spin_ops = { · 100 unmodified lines | ||
| 128 | 129 | #define genpd_is_irq_safe(genpd) (genpd->flags & GENPD_FLAG_IRQ_SAFE) |
| 129 | 130 | #define genpd_is_always_on(genpd) (genpd->flags & GENPD_FLAG_ALWAYS_ON) |
| 130 | 131 | #define genpd_is_active_wakeup(genpd) (genpd->flags & GENPD_FLAG_ACTIVE_WAKEUP) |
| 132 | +#define genpd_is_cpu_domain(genpd) (genpd->flags & GENPD_FLAG_CPU_DOMAIN) | |
| 131 | 133 | |
| 132 | 134 | static inline bool irq_safe_dev_in_no_sleep_domain(struct device *dev, |
| 133 | 135 | const struct generic_pm_domain *genpd) |
| @@ -1452,6 +1454,56 @@ static void genpd_free_dev_data(struct device *dev, · 1318 unmodified lines | ||
| 1452 | 1454 | dev_pm_put_subsys_data(dev); |
| 1453 | 1455 | } |
| 1454 | 1456 | |
| 1457 | +static void __genpd_update_cpumask(struct generic_pm_domain *genpd, | |
| 1458 | + int cpu, bool set, unsigned int depth) | |
| 1459 | +{ | |
| 1460 | + struct gpd_link *link; | |
| 1461 | + | |
| 1462 | + if (!genpd_is_cpu_domain(genpd)) | |
| 1463 | + return; | |
| 1464 | + | |
| 1465 | + list_for_each_entry(link, &genpd->slave_links, slave_node) { | |
| 1466 | + struct generic_pm_domain *master = link->master; | |
| 1467 | + | |
| 1468 | + genpd_lock_nested(master, depth + 1); | |
| 1469 | + __genpd_update_cpumask(master, cpu, set, depth + 1); | |
| 1470 | + genpd_unlock(master); | |
| 1471 | + } | |
| 1472 | + | |
| 1473 | + if (set) | |
| 1474 | + cpumask_set_cpu(cpu, genpd->cpus); | |
| 1475 | + else | |
| 1476 | + cpumask_clear_cpu(cpu, genpd->cpus); | |
| 1477 | +} | |
| 1478 | + | |
| 1479 | +static void genpd_update_cpumask(struct generic_pm_domain *genpd, | |
| 1480 | + struct device *dev, bool set) | |
| 1481 | +{ | |
| 1482 | + int cpu; | |
| 1483 | + | |
| 1484 | + if (!genpd_is_cpu_domain(genpd)) | |
| 1485 | + return; | |
| 1486 | + | |
| 1487 | + for_each_possible_cpu(cpu) { | |
| 1488 | + if (get_cpu_device(cpu) == dev) { | |
| 1489 | + __genpd_update_cpumask(genpd, cpu, set, 0); | |
| 1490 | + return; | |
| 1491 | + } | |
| 1492 | + } | |
| 1493 | +} | |
| 1494 | + | |
| 1495 | +static void genpd_set_cpumask(struct generic_pm_domain *genpd, | |
| 1496 | + struct device *dev) | |
| 1497 | +{ | |
| 1498 | + genpd_update_cpumask(genpd, dev, true); | |
| 1499 | +} | |
| 1500 | + | |
| 1501 | +static void genpd_clear_cpumask(struct generic_pm_domain *genpd, | |
| 1502 | + struct device *dev) | |
| 1503 | +{ | |
| 1504 | + genpd_update_cpumask(genpd, dev, false); | |
| 1505 | +} | |
| 1506 | + | |
| 1455 | 1507 | static int genpd_add_device(struct generic_pm_domain *genpd, struct device *dev, |
| 1456 | 1508 | struct gpd_timing_data *td) |
| 1457 | 1509 | { |
| @@ -1473,6 +1525,7 @@ static int genpd_add_device(struct generic_pm_domain *genpd, struct device *dev, · 15 unmodified lines | ||
| 1473 | 1525 | |
| 1474 | 1526 | genpd_lock(genpd); |
| 1475 | 1527 | |
| 1528 | + genpd_set_cpumask(genpd, dev); | |
| 1476 | 1529 | dev_pm_domain_set(dev, &genpd->domain); |
| 1477 | 1530 | |
| 1478 | 1531 | genpd->device_count++; |
| @@ -1530,6 +1583,7 @@ static int genpd_remove_device(struct generic_pm_domain *genpd, · 51 unmodified lines | ||
| 1530 | 1583 | genpd->device_count--; |
| 1531 | 1584 | genpd->max_off_time_changed = true; |
| 1532 | 1585 | |
| 1586 | + genpd_clear_cpumask(genpd, dev); | |
| 1533 | 1587 | dev_pm_domain_set(dev, NULL); |
| 1534 | 1588 | |
| 1535 | 1589 | list_del_init(&pdd->list_node); |
| @@ -1684,6 +1738,12 @@ out: · 148 unmodified lines | ||
| 1684 | 1738 | } |
| 1685 | 1739 | EXPORT_SYMBOL_GPL(pm_genpd_remove_subdomain); |
| 1686 | 1740 | |
| 1741 | +static void genpd_free_default_power_state(struct genpd_power_state *states, | |
| 1742 | + unsigned int state_count) | |
| 1743 | +{ | |
| 1744 | + kfree(states); | |
| 1745 | +} | |
| 1746 | + | |
| 1687 | 1747 | static int genpd_set_default_power_state(struct generic_pm_domain *genpd) |
| 1688 | 1748 | { |
| 1689 | 1749 | struct genpd_power_state *state; |
| @@ -1694,7 +1754,7 @@ static int genpd_set_default_power_state(struct generic_pm_domain *genpd) · 4 unmodified lines | ||
| 1694 | 1754 | |
| 1695 | 1755 | genpd->states = state; |
| 1696 | 1756 | genpd->state_count = 1; |
| 1697 | − genpd->free = state; | |
| 1757 | + genpd->free_states = genpd_free_default_power_state; | |
| 1698 | 1758 | |
| 1699 | 1759 | return 0; |
| 1700 | 1760 | } |
| @@ -1760,11 +1820,18 @@ int pm_genpd_init(struct generic_pm_domain *genpd, · 59 unmodified lines | ||
| 1760 | 1820 | if (genpd_is_always_on(genpd) && !genpd_status_on(genpd)) |
| 1761 | 1821 | return -EINVAL; |
| 1762 | 1822 | |
| 1823 | + if (genpd_is_cpu_domain(genpd) && | |
| 1824 | + !zalloc_cpumask_var(&genpd->cpus, GFP_KERNEL)) | |
| 1825 | + return -ENOMEM; | |
| 1826 | + | |
| 1763 | 1827 | /* Use only one "off" state if there were no states declared */ |
| 1764 | 1828 | if (genpd->state_count == 0) { |
| 1765 | 1829 | ret = genpd_set_default_power_state(genpd); |
| 1766 | − if (ret) | |
| 1830 | + if (ret) { | |
| 1831 | + if (genpd_is_cpu_domain(genpd)) | |
| 1832 | + free_cpumask_var(genpd->cpus); | |
| 1767 | 1833 | return ret; |
| 1834 | + } | |
| 1768 | 1835 | } else if (!gov && genpd->state_count > 1) { |
| 1769 | 1836 | pr_warn("%s: no governor for states\n", genpd->name); |
| 1770 | 1837 | } |
| @@ -1810,7 +1877,11 @@ static int genpd_remove(struct generic_pm_domain *genpd) · 39 unmodified lines | ||
| 1810 | 1877 | list_del(&genpd->gpd_list_node); |
| 1811 | 1878 | genpd_unlock(genpd); |
| 1812 | 1879 | cancel_work_sync(&genpd->power_off_work); |
| 1813 | − kfree(genpd->free); | |
| 1880 | + if (genpd_is_cpu_domain(genpd)) | |
| 1881 | + free_cpumask_var(genpd->cpus); | |
| 1882 | + if (genpd->free_states) | |
| 1883 | + genpd->free_states(genpd->states, genpd->state_count); | |
| 1884 | + | |
| 1814 | 1885 | pr_debug("%s: removed %s\n", __func__, genpd->name); |
| 1815 | 1886 | |
| 1816 | 1887 | return 0; |
| @@ -10,6 +10,9 @@ · 9 unmodified lines | ||
| 10 | 10 | #include <linux/pm_domain.h> |
| 11 | 11 | #include <linux/pm_qos.h> |
| 12 | 12 | #include <linux/hrtimer.h> |
| 13 | +#include <linux/cpuidle.h> | |
| 14 | +#include <linux/cpumask.h> | |
| 15 | +#include <linux/ktime.h> | |
| 13 | 16 | |
| 14 | 17 | static int dev_update_qos_constraint(struct device *dev, void *data) |
| 15 | 18 | { |
| @@ -210,8 +213,10 @@ static bool default_power_down_ok(struct dev_pm_domain *pd) · 194 unmodified lines | ||
| 210 | 213 | struct generic_pm_domain *genpd = pd_to_genpd(pd); |
| 211 | 214 | struct gpd_link *link; |
| 212 | 215 | |
| 213 | − if (!genpd->max_off_time_changed) | |
| 216 | + if (!genpd->max_off_time_changed) { | |
| 217 | + genpd->state_idx = genpd->cached_power_down_state_idx; | |
| 214 | 218 | return genpd->cached_power_down_ok; |
| 219 | + } | |
| 215 | 220 | |
| 216 | 221 | /* |
| 217 | 222 | * We have to invalidate the cached results for the masters, so |
| @@ -236,6 +241,7 @@ static bool default_power_down_ok(struct dev_pm_domain *pd) · 18 unmodified lines | ||
| 236 | 241 | genpd->state_idx--; |
| 237 | 242 | } |
| 238 | 243 | |
| 244 | + genpd->cached_power_down_state_idx = genpd->state_idx; | |
| 239 | 245 | return genpd->cached_power_down_ok; |
| 240 | 246 | } |
| 241 | 247 | |
| @@ -244,6 +250,65 @@ static bool always_on_power_down_ok(struct dev_pm_domain *domain) · 2 unmodified lines | ||
| 244 | 250 | return false; |
| 245 | 251 | } |
| 246 | 252 | |
| 253 | +#ifdef CONFIG_CPU_IDLE | |
| 254 | +static bool cpu_power_down_ok(struct dev_pm_domain *pd) | |
| 255 | +{ | |
| 256 | + struct generic_pm_domain *genpd = pd_to_genpd(pd); | |
| 257 | + struct cpuidle_device *dev; | |
| 258 | + ktime_t domain_wakeup, next_hrtimer; | |
| 259 | + s64 idle_duration_ns; | |
| 260 | + int cpu, i; | |
| 261 | + | |
| 262 | + /* Validate dev PM QoS constraints. */ | |
| 263 | + if (!default_power_down_ok(pd)) | |
| 264 | + return false; | |
| 265 | + | |
| 266 | + if (!(genpd->flags & GENPD_FLAG_CPU_DOMAIN)) | |
| 267 | + return true; | |
| 268 | + | |
| 269 | + /* | |
| 270 | + * Find the next wakeup for any of the online CPUs within the PM domain | |
| 271 | + * and its subdomains. Note, we only need the genpd->cpus, as it already | |
| 272 | + * contains a mask of all CPUs from subdomains. | |
| 273 | + */ | |
| 274 | + domain_wakeup = ktime_set(KTIME_SEC_MAX, 0); | |
| 275 | + for_each_cpu_and(cpu, genpd->cpus, cpu_online_mask) { | |
| 276 | + dev = per_cpu(cpuidle_devices, cpu); | |
| 277 | + if (dev) { | |
| 278 | + next_hrtimer = READ_ONCE(dev->next_hrtimer); | |
| 279 | + if (ktime_before(next_hrtimer, domain_wakeup)) | |
| 280 | + domain_wakeup = next_hrtimer; | |
| 281 | + } | |
| 282 | + } | |
| 283 | + | |
| 284 | + /* The minimum idle duration is from now - until the next wakeup. */ | |
| 285 | + idle_duration_ns = ktime_to_ns(ktime_sub(domain_wakeup, ktime_get())); | |
| 286 | + if (idle_duration_ns <= 0) | |
| 287 | + return false; | |
| 288 | + | |
| 289 | + /* | |
| 290 | + * Find the deepest idle state that has its residency value satisfied | |
| 291 | + * and by also taking into account the power off latency for the state. | |
| 292 | + * Start at the state picked by the dev PM QoS constraint validation. | |
| 293 | + */ | |
| 294 | + i = genpd->state_idx; | |
| 295 | + do { | |
| 296 | + if (idle_duration_ns >= (genpd->states[i].residency_ns + | |
| 297 | + genpd->states[i].power_off_latency_ns)) { | |
| 298 | + genpd->state_idx = i; | |
| 299 | + return true; | |
| 300 | + } | |
| 301 | + } while (--i >= 0); | |
| 302 | + | |
| 303 | + return false; | |
| 304 | +} | |
| 305 | + | |
| 306 | +struct dev_power_governor pm_domain_cpu_gov = { | |
| 307 | + .suspend_ok = default_suspend_ok, | |
| 308 | + .power_down_ok = cpu_power_down_ok, | |
| 309 | +}; | |
| 310 | +#endif | |
| 311 | + | |
| 247 | 312 | struct dev_power_governor simple_qos_governor = { |
| 248 | 313 | .suspend_ok = default_suspend_ok, |
| 249 | 314 | .power_down_ok = default_power_down_ok, |
| @@ -84,7 +84,7 @@ static struct cpuidle_driver exynos_idle_driver = { · 83 unmodified lines | ||
| 84 | 84 | [1] = { |
| 85 | 85 | .enter = exynos_enter_lowpower, |
| 86 | 86 | .exit_latency = 300, |
| 87 | − .target_residency = 100000, | |
| 87 | + .target_residency = 10000, | |
| 88 | 88 | .name = "C1", |
| 89 | 89 | .desc = "ARM power down", |
| 90 | 90 | }, |
| @@ -328,9 +328,23 @@ int cpuidle_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, · 327 unmodified lines | ||
| 328 | 328 | int cpuidle_enter(struct cpuidle_driver *drv, struct cpuidle_device *dev, |
| 329 | 329 | int index) |
| 330 | 330 | { |
| 331 | + int ret = 0; | |
| 332 | + | |
| 333 | + /* | |
| 334 | + * Store the next hrtimer, which becomes either next tick or the next | |
| 335 | + * timer event, whatever expires first. Additionally, to make this data | |
| 336 | + * useful for consumers outside cpuidle, we rely on that the governor's | |
| 337 | + * ->select() callback have decided, whether to stop the tick or not. | |
| 338 | + */ | |
| 339 | + WRITE_ONCE(dev->next_hrtimer, tick_nohz_get_next_hrtimer()); | |
| 340 | + | |
| 331 | 341 | if (cpuidle_state_is_coupled(drv, index)) |
| 332 | − return cpuidle_enter_state_coupled(dev, drv, index); | |
| 333 | − return cpuidle_enter_state(dev, drv, index); | |
| 342 | + ret = cpuidle_enter_state_coupled(dev, drv, index); | |
| 343 | + else | |
| 344 | + ret = cpuidle_enter_state(dev, drv, index); | |
| 345 | + | |
| 346 | + WRITE_ONCE(dev->next_hrtimer, 0); | |
| 347 | + return ret; | |
| 334 | 348 | } |
| 335 | 349 | |
| 336 | 350 | /** |
| @@ -511,6 +525,7 @@ static void __cpuidle_device_init(struct cpuidle_device *dev) · 174 unmodified lines | ||
| 511 | 525 | { |
| 512 | 526 | memset(dev->states_usage, 0, sizeof(dev->states_usage)); |
| 513 | 527 | dev->last_residency = 0; |
| 528 | + dev->next_hrtimer = 0; | |
| 514 | 529 | } |
| 515 | 530 | |
| 516 | 531 | /** |
| @@ -83,6 +83,7 @@ struct cpuidle_device { · 82 unmodified lines | ||
| 83 | 83 | unsigned int use_deepest_state:1; |
| 84 | 84 | unsigned int poll_time_limit:1; |
| 85 | 85 | unsigned int cpu; |
| 86 | + ktime_t next_hrtimer; | |
| 86 | 87 | |
| 87 | 88 | int last_residency; |
| 88 | 89 | struct cpuidle_state_usage states_usage[CPUIDLE_STATE_MAX]; |
| @@ -16,6 +16,7 @@ · 15 unmodified lines | ||
| 16 | 16 | #include <linux/of.h> |
| 17 | 17 | #include <linux/notifier.h> |
| 18 | 18 | #include <linux/spinlock.h> |
| 19 | +#include <linux/cpumask.h> | |
| 19 | 20 | |
| 20 | 21 | /* |
| 21 | 22 | * Flags to control the behaviour of a genpd. |
| @@ -42,11 +43,22 @@ · 20 unmodified lines | ||
| 42 | 43 | * GENPD_FLAG_ACTIVE_WAKEUP: Instructs genpd to keep the PM domain powered |
| 43 | 44 | * on, in case any of its attached devices is used |
| 44 | 45 | * in the wakeup path to serve system wakeups. |
| 46 | + * | |
| 47 | + * GENPD_FLAG_CPU_DOMAIN: Instructs genpd that it should expect to get | |
| 48 | + * devices attached, which may belong to CPUs or | |
| 49 | + * possibly have subdomains with CPUs attached. | |
| 50 | + * This flag enables the genpd backend driver to | |
| 51 | + * deploy idle power management support for CPUs | |
| 52 | + * and groups of CPUs. Note that, the backend | |
| 53 | + * driver must then comply with the so called, | |
| 54 | + * last-man-standing algorithm, for the CPUs in the | |
| 55 | + * PM domain. | |
| 45 | 56 | */ |
| 46 | 57 | #define GENPD_FLAG_PM_CLK (1U << 0) |
| 47 | 58 | #define GENPD_FLAG_IRQ_SAFE (1U << 1) |
| 48 | 59 | #define GENPD_FLAG_ALWAYS_ON (1U << 2) |
| 49 | 60 | #define GENPD_FLAG_ACTIVE_WAKEUP (1U << 3) |
| 61 | +#define GENPD_FLAG_CPU_DOMAIN (1U << 4) | |
| 50 | 62 | |
| 51 | 63 | enum gpd_status { |
| 52 | 64 | GPD_STATE_ACTIVE = 0, /* PM domain is active */ |
| @@ -69,6 +81,7 @@ struct genpd_power_state { · 16 unmodified lines | ||
| 69 | 81 | s64 residency_ns; |
| 70 | 82 | struct fwnode_handle *fwnode; |
| 71 | 83 | ktime_t idle_time; |
| 84 | + void *data; | |
| 72 | 85 | }; |
| 73 | 86 | |
| 74 | 87 | struct genpd_lock_ops; |
| @@ -93,6 +106,7 @@ struct generic_pm_domain { · 18 unmodified lines | ||
| 93 | 106 | unsigned int suspended_count; /* System suspend device counter */ |
| 94 | 107 | unsigned int prepared_count; /* Suspend counter of prepared devices */ |
| 95 | 108 | unsigned int performance_state; /* Aggregated max performance state */ |
| 109 | + cpumask_var_t cpus; /* A cpumask of the attached CPUs */ | |
| 96 | 110 | int (*power_off)(struct generic_pm_domain *domain); |
| 97 | 111 | int (*power_on)(struct generic_pm_domain *domain); |
| 98 | 112 | struct opp_table *opp_table; /* OPP table of the genpd */ |
| @@ -104,15 +118,17 @@ struct generic_pm_domain { · 5 unmodified lines | ||
| 104 | 118 | s64 max_off_time_ns; /* Maximum allowed "suspended" time. */ |
| 105 | 119 | bool max_off_time_changed; |
| 106 | 120 | bool cached_power_down_ok; |
| 121 | + bool cached_power_down_state_idx; | |
| 107 | 122 | int (*attach_dev)(struct generic_pm_domain *domain, |
| 108 | 123 | struct device *dev); |
| 109 | 124 | void (*detach_dev)(struct generic_pm_domain *domain, |
| 110 | 125 | struct device *dev); |
| 111 | 126 | unsigned int flags; /* Bit field of configs for genpd */ |
| 112 | 127 | struct genpd_power_state *states; |
| 128 | + void (*free_states)(struct genpd_power_state *states, | |
| 129 | + unsigned int state_count); | |
| 113 | 130 | unsigned int state_count; /* number of states */ |
| 114 | 131 | unsigned int state_idx; /* state that genpd will go to when off */ |
| 115 | − void *free; /* Free the state that was allocated for default */ | |
| 116 | 132 | ktime_t on_time; |
| 117 | 133 | ktime_t accounting_time; |
| 118 | 134 | const struct genpd_lock_ops *lock_ops; |
| @@ -187,6 +203,9 @@ int dev_pm_genpd_set_performance_state(struct device *dev, unsigned int state); · 68 unmodified lines | ||
| 187 | 203 | |
| 188 | 204 | extern struct dev_power_governor simple_qos_governor; |
| 189 | 205 | extern struct dev_power_governor pm_domain_always_on_gov; |
| 206 | +#ifdef CONFIG_CPU_IDLE | |
| 207 | +extern struct dev_power_governor pm_domain_cpu_gov; | |
| 208 | +#endif | |
| 190 | 209 | #else |
| 191 | 210 | |
| 192 | 211 | static inline struct generic_pm_domain_data *dev_gpd_data(struct device *dev) |
| @@ -122,6 +122,7 @@ extern void tick_nohz_idle_enter(void); · 121 unmodified lines | ||
| 122 | 122 | extern void tick_nohz_idle_exit(void); |
| 123 | 123 | extern void tick_nohz_irq_exit(void); |
| 124 | 124 | extern bool tick_nohz_idle_got_tick(void); |
| 125 | +extern ktime_t tick_nohz_get_next_hrtimer(void); | |
| 125 | 126 | extern ktime_t tick_nohz_get_sleep_length(ktime_t *delta_next); |
| 126 | 127 | extern unsigned long tick_nohz_get_idle_calls(void); |
| 127 | 128 | extern unsigned long tick_nohz_get_idle_calls_cpu(int cpu); |
| @@ -145,7 +146,11 @@ static inline void tick_nohz_idle_restart_tick(void) { } · 17 unmodified lines | ||
| 145 | 146 | static inline void tick_nohz_idle_enter(void) { } |
| 146 | 147 | static inline void tick_nohz_idle_exit(void) { } |
| 147 | 148 | static inline bool tick_nohz_idle_got_tick(void) { return false; } |
| 148 | − | |
| 149 | +static inline ktime_t tick_nohz_get_next_hrtimer(void) | |
| 150 | +{ | |
| 151 | + /* Next wake up is the tick period, assume it starts now */ | |
| 152 | + return ktime_add(ktime_get(), TICK_NSEC); | |
| 153 | +} | |
| 149 | 154 | static inline ktime_t tick_nohz_get_sleep_length(ktime_t *delta_next) |
| 150 | 155 | { |
| 151 | 156 | *delta_next = TICK_NSEC; |
| @@ -1022,6 +1022,18 @@ bool tick_nohz_idle_got_tick(void) · 1021 unmodified lines | ||
| 1022 | 1022 | return false; |
| 1023 | 1023 | } |
| 1024 | 1024 | |
| 1025 | +/** | |
| 1026 | + * tick_nohz_get_next_hrtimer - return the next expiration time for the hrtimer | |
| 1027 | + * or the tick, whatever that expires first. Note that, if the tick has been | |
| 1028 | + * stopped, it returns the next hrtimer. | |
| 1029 | + * | |
| 1030 | + * Called from power state control code with interrupts disabled | |
| 1031 | + */ | |
| 1032 | +ktime_t tick_nohz_get_next_hrtimer(void) | |
| 1033 | +{ | |
| 1034 | + return __this_cpu_read(tick_cpu_device.evtdev)->next_event; | |
| 1035 | +} | |
| 1036 | + | |
| 1025 | 1037 | /** |
| 1026 | 1038 | * tick_nohz_get_sleep_length - return the expected length of the current sleep |
| 1027 | 1039 | * @delta_next: duration until the next event if the tick cannot be stopped |