In order to minimize the interference of migrate_disable() on lower priority tasks, which can be deprived of runtime due to being stuck below a higher priority task. Teach the RT/DL balancers to push away these higher priority tasks when a lower priority task gets selected to run on a freshly demoted CPU (pull). This adds migration interference to the higher priority task, but restores bandwidth to system that would otherwise be irrevocably lost. Without this it would be possible to have all tasks on the system stuck on a single CPU, each task preempted in a migrate_disable() section with a single high priority task running. This way we can still approximate running the M highest priority tasks on the system. Migrating the top task away is (ofcourse) still subject to migrate_disable() too, which means the lower task is subject to an interference equivalent to the worst case migrate_disable() section.
Showing the changes to kernel (4 of 6 files). Show all
Unified · Split
| @@ -1763,11 +1763,6 @@ void migrate_enable(void) · 1762 unmodified lines | |||
| 1763 | } | 1763 | } |
| 1764 | EXPORT_SYMBOL_GPL(migrate_enable); | 1764 | EXPORT_SYMBOL_GPL(migrate_enable); |
| 1765 | 1765 | ||
| 1766 | static inline bool is_migration_disabled(struct task_struct *p) | ||
| 1767 | { | ||
| 1768 | return p->migration_disabled; | ||
| 1769 | } | ||
| 1770 | |||
| 1771 | static inline bool rq_has_pinned_tasks(struct rq *rq) | 1766 | static inline bool rq_has_pinned_tasks(struct rq *rq) |
| 1772 | { | 1767 | { |
| 1773 | return rq->nr_pinned; | 1768 | return rq->nr_pinned; |
| @@ -1972,6 +1967,49 @@ out: · 198 unmodified lines | |||
| 1972 | return 0; | 1967 | return 0; |
| 1973 | } | 1968 | } |
| 1974 | 1969 | ||
| 1970 | int push_cpu_stop(void *arg) | ||
| 1971 | { | ||
| 1972 | struct rq *lowest_rq = NULL, *rq = this_rq(); | ||
| 1973 | struct task_struct *p = arg; | ||
| 1974 | |||
| 1975 | raw_spin_lock_irq(&p->pi_lock); | ||
| 1976 | raw_spin_lock(&rq->lock); | ||
| 1977 | |||
| 1978 | if (task_rq(p) != rq) | ||
| 1979 | goto out_unlock; | ||
| 1980 | |||
| 1981 | if (is_migration_disabled(p)) { | ||
| 1982 | p->migration_flags |= MDF_PUSH; | ||
| 1983 | goto out_unlock; | ||
| 1984 | } | ||
| 1985 | |||
| 1986 | p->migration_flags &= ~MDF_PUSH; | ||
| 1987 | |||
| 1988 | if (p->sched_class->find_lock_rq) | ||
| 1989 | lowest_rq = p->sched_class->find_lock_rq(p, rq); | ||
| 1990 | |||
| 1991 | if (!lowest_rq) | ||
| 1992 | goto out_unlock; | ||
| 1993 | |||
| 1994 | // XXX validate p is still the highest prio task | ||
| 1995 | if (task_rq(p) == rq) { | ||
| 1996 | deactivate_task(rq, p, 0); | ||
| 1997 | set_task_cpu(p, lowest_rq->cpu); | ||
| 1998 | activate_task(lowest_rq, p, 0); | ||
| 1999 | resched_curr(lowest_rq); | ||
| 2000 | } | ||
| 2001 | |||
| 2002 | double_unlock_balance(rq, lowest_rq); | ||
| 2003 | |||
| 2004 | out_unlock: | ||
| 2005 | rq->push_busy = false; | ||
| 2006 | raw_spin_unlock(&rq->lock); | ||
| 2007 | raw_spin_unlock_irq(&p->pi_lock); | ||
| 2008 | |||
| 2009 | put_task_struct(p); | ||
| 2010 | return 0; | ||
| 2011 | } | ||
| 2012 | |||
| 1975 | /* | 2013 | /* |
| 1976 | * sched_class::set_cpus_allowed must do the below, but is not required to | 2014 | * sched_class::set_cpus_allowed must do the below, but is not required to |
| 1977 | * actually call this function. | 2015 | * actually call this function. |
| @@ -2052,6 +2090,14 @@ static int affine_move_task(struct rq *rq, struct task_struct *p, struct rq_flag · 74 unmodified lines | |||
| 2052 | 2090 | ||
| 2053 | /* Can the task run on the task's current CPU? If so, we're done */ | 2091 | /* Can the task run on the task's current CPU? If so, we're done */ |
| 2054 | if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask)) { | 2092 | if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask)) { |
| 2093 | struct task_struct *push_task = NULL; | ||
| 2094 | |||
| 2095 | if ((flags & SCA_MIGRATE_ENABLE) && | ||
| 2096 | (p->migration_flags & MDF_PUSH) && !rq->push_busy) { | ||
| 2097 | rq->push_busy = true; | ||
| 2098 | push_task = get_task_struct(p); | ||
| 2099 | } | ||
| 2100 | |||
| 2055 | pending = p->migration_pending; | 2101 | pending = p->migration_pending; |
| 2056 | if (pending) { | 2102 | if (pending) { |
| 2057 | refcount_inc(&pending->refs); | 2103 | refcount_inc(&pending->refs); |
| @@ -2060,6 +2106,11 @@ static int affine_move_task(struct rq *rq, struct task_struct *p, struct rq_flag · 2 unmodified lines | |||
| 2060 | } | 2106 | } |
| 2061 | task_rq_unlock(rq, p, rf); | 2107 | task_rq_unlock(rq, p, rf); |
| 2062 | 2108 | ||
| 2109 | if (push_task) { | ||
| 2110 | stop_one_cpu_nowait(rq->cpu, push_cpu_stop, | ||
| 2111 | p, &rq->push_work); | ||
| 2112 | } | ||
| 2113 | |||
| 2063 | if (complete) | 2114 | if (complete) |
| 2064 | goto do_complete; | 2115 | goto do_complete; |
| 2065 | 2116 | ||
| @@ -2098,6 +2149,7 @@ static int affine_move_task(struct rq *rq, struct task_struct *p, struct rq_flag · 32 unmodified lines | |||
| 2098 | if (flags & SCA_MIGRATE_ENABLE) { | 2149 | if (flags & SCA_MIGRATE_ENABLE) { |
| 2099 | 2150 | ||
| 2100 | refcount_inc(&pending->refs); /* pending->{arg,stop_work} */ | 2151 | refcount_inc(&pending->refs); /* pending->{arg,stop_work} */ |
| 2152 | p->migration_flags &= ~MDF_PUSH; | ||
| 2101 | task_rq_unlock(rq, p, rf); | 2153 | task_rq_unlock(rq, p, rf); |
| 2102 | 2154 | ||
| 2103 | pending->arg = (struct migration_arg) { | 2155 | pending->arg = (struct migration_arg) { |
| @@ -2716,11 +2768,6 @@ static inline int __set_cpus_allowed_ptr(struct task_struct *p, · 612 unmodified lines | |||
| 2716 | 2768 | ||
| 2717 | static inline void migrate_disable_switch(struct rq *rq, struct task_struct *p) { } | 2769 | static inline void migrate_disable_switch(struct rq *rq, struct task_struct *p) { } |
| 2718 | 2770 | ||
| 2719 | static inline bool is_migration_disabled(struct task_struct *p) | ||
| 2720 | { | ||
| 2721 | return false; | ||
| 2722 | } | ||
| 2723 | |||
| 2724 | static inline bool rq_has_pinned_tasks(struct rq *rq) | 2771 | static inline bool rq_has_pinned_tasks(struct rq *rq) |
| 2725 | { | 2772 | { |
| 2726 | return false; | 2773 | return false; |
| @@ -2129,6 +2129,9 @@ static int push_dl_task(struct rq *rq) · 2128 unmodified lines | |||
| 2129 | return 0; | 2129 | return 0; |
| 2130 | 2130 | ||
| 2131 | retry: | 2131 | retry: |
| 2132 | if (is_migration_disabled(next_task)) | ||
| 2133 | return 0; | ||
| 2134 | |||
| 2132 | if (WARN_ON(next_task == rq->curr)) | 2135 | if (WARN_ON(next_task == rq->curr)) |
| 2133 | return 0; | 2136 | return 0; |
| 2134 | 2137 | ||
| @@ -2206,7 +2209,7 @@ static void push_dl_tasks(struct rq *rq) · 71 unmodified lines | |||
| 2206 | static void pull_dl_task(struct rq *this_rq) | 2209 | static void pull_dl_task(struct rq *this_rq) |
| 2207 | { | 2210 | { |
| 2208 | int this_cpu = this_rq->cpu, cpu; | 2211 | int this_cpu = this_rq->cpu, cpu; |
| 2209 | struct task_struct *p; | 2212 | struct task_struct *p, *push_task; |
| 2210 | bool resched = false; | 2213 | bool resched = false; |
| 2211 | struct rq *src_rq; | 2214 | struct rq *src_rq; |
| 2212 | u64 dmin = LONG_MAX; | 2215 | u64 dmin = LONG_MAX; |
| @@ -2236,6 +2239,7 @@ static void pull_dl_task(struct rq *this_rq) · 23 unmodified lines | |||
| 2236 | continue; | 2239 | continue; |
| 2237 | 2240 | ||
| 2238 | /* Might drop this_rq->lock */ | 2241 | /* Might drop this_rq->lock */ |
| 2242 | push_task = NULL; | ||
| 2239 | double_lock_balance(this_rq, src_rq); | 2243 | double_lock_balance(this_rq, src_rq); |
| 2240 | 2244 | ||
| 2241 | /* | 2245 | /* |
| @@ -2267,17 +2271,27 @@ static void pull_dl_task(struct rq *this_rq) · 25 unmodified lines | |||
| 2267 | src_rq->curr->dl.deadline)) | 2271 | src_rq->curr->dl.deadline)) |
| 2268 | goto skip; | 2272 | goto skip; |
| 2269 | 2273 | ||
| 2270 | resched = true; | 2274 | if (is_migration_disabled(p)) { |
| 2271 | 2275 | push_task = get_push_task(src_rq); | |
| 2272 | deactivate_task(src_rq, p, 0); | 2276 | } else { |
| 2273 | set_task_cpu(p, this_cpu); | 2277 | deactivate_task(src_rq, p, 0); |
| 2274 | activate_task(this_rq, p, 0); | 2278 | set_task_cpu(p, this_cpu); |
| 2275 | dmin = p->dl.deadline; | 2279 | activate_task(this_rq, p, 0); |
| 2280 | dmin = p->dl.deadline; | ||
| 2281 | resched = true; | ||
| 2282 | } | ||
| 2276 | 2283 | ||
| 2277 | /* Is there any other task even earlier? */ | 2284 | /* Is there any other task even earlier? */ |
| 2278 | } | 2285 | } |
| 2279 | skip: | 2286 | skip: |
| 2280 | double_unlock_balance(this_rq, src_rq); | 2287 | double_unlock_balance(this_rq, src_rq); |
| 2288 | |||
| 2289 | if (push_task) { | ||
| 2290 | raw_spin_unlock(&this_rq->lock); | ||
| 2291 | stop_one_cpu_nowait(src_rq->cpu, push_cpu_stop, | ||
| 2292 | push_task, &src_rq->push_work); | ||
| 2293 | raw_spin_lock(&this_rq->lock); | ||
| 2294 | } | ||
| 2281 | } | 2295 | } |
| 2282 | 2296 | ||
| 2283 | if (resched) | 2297 | if (resched) |
| @@ -2524,6 +2538,7 @@ const struct sched_class dl_sched_class · 240 unmodified lines | |||
| 2524 | .rq_online = rq_online_dl, | 2538 | .rq_online = rq_online_dl, |
| 2525 | .rq_offline = rq_offline_dl, | 2539 | .rq_offline = rq_offline_dl, |
| 2526 | .task_woken = task_woken_dl, | 2540 | .task_woken = task_woken_dl, |
| 2541 | .find_lock_rq = find_lock_later_rq, | ||
| 2527 | #endif | 2542 | #endif |
| 2528 | 2543 | ||
| 2529 | .task_tick = task_tick_dl, | 2544 | .task_tick = task_tick_dl, |
| @@ -1859,7 +1859,7 @@ static struct task_struct *pick_next_pushable_task(struct rq *rq) · 1858 unmodified lines | |||
| 1859 | * running task can migrate over to a CPU that is running a task | 1859 | * running task can migrate over to a CPU that is running a task |
| 1860 | * of lesser priority. | 1860 | * of lesser priority. |
| 1861 | */ | 1861 | */ |
| 1862 | static int push_rt_task(struct rq *rq) | 1862 | static int push_rt_task(struct rq *rq, bool pull) |
| 1863 | { | 1863 | { |
| 1864 | struct task_struct *next_task; | 1864 | struct task_struct *next_task; |
| 1865 | struct rq *lowest_rq; | 1865 | struct rq *lowest_rq; |
| @@ -1873,6 +1873,34 @@ static int push_rt_task(struct rq *rq) · 7 unmodified lines | |||
| 1873 | return 0; | 1873 | return 0; |
| 1874 | 1874 | ||
| 1875 | retry: | 1875 | retry: |
| 1876 | if (is_migration_disabled(next_task)) { | ||
| 1877 | struct task_struct *push_task = NULL; | ||
| 1878 | int cpu; | ||
| 1879 | |||
| 1880 | if (!pull || rq->push_busy) | ||
| 1881 | return 0; | ||
| 1882 | |||
| 1883 | cpu = find_lowest_rq(rq->curr); | ||
| 1884 | if (cpu == -1 || cpu == rq->cpu) | ||
| 1885 | return 0; | ||
| 1886 | |||
| 1887 | /* | ||
| 1888 | * Given we found a CPU with lower priority than @next_task, | ||
| 1889 | * therefore it should be running. However we cannot migrate it | ||
| 1890 | * to this other CPU, instead attempt to push the current | ||
| 1891 | * running task on this CPU away. | ||
| 1892 | */ | ||
| 1893 | push_task = get_push_task(rq); | ||
| 1894 | if (push_task) { | ||
| 1895 | raw_spin_unlock(&rq->lock); | ||
| 1896 | stop_one_cpu_nowait(rq->cpu, push_cpu_stop, | ||
| 1897 | push_task, &rq->push_work); | ||
| 1898 | raw_spin_lock(&rq->lock); | ||
| 1899 | } | ||
| 1900 | |||
| 1901 | return 0; | ||
| 1902 | } | ||
| 1903 | |||
| 1876 | if (WARN_ON(next_task == rq->curr)) | 1904 | if (WARN_ON(next_task == rq->curr)) |
| 1877 | return 0; | 1905 | return 0; |
| 1878 | 1906 | ||
| @@ -1927,12 +1955,10 @@ retry: · 48 unmodified lines | |||
| 1927 | deactivate_task(rq, next_task, 0); | 1955 | deactivate_task(rq, next_task, 0); |
| 1928 | set_task_cpu(next_task, lowest_rq->cpu); | 1956 | set_task_cpu(next_task, lowest_rq->cpu); |
| 1929 | activate_task(lowest_rq, next_task, 0); | 1957 | activate_task(lowest_rq, next_task, 0); |
| 1930 | ret = 1; | ||
| 1931 | |||
| 1932 | resched_curr(lowest_rq); | 1958 | resched_curr(lowest_rq); |
| 1959 | ret = 1; | ||
| 1933 | 1960 | ||
| 1934 | double_unlock_balance(rq, lowest_rq); | 1961 | double_unlock_balance(rq, lowest_rq); |
| 1935 | |||
| 1936 | out: | 1962 | out: |
| 1937 | put_task_struct(next_task); | 1963 | put_task_struct(next_task); |
| 1938 | 1964 | ||
| @@ -1942,7 +1968,7 @@ out: · 3 unmodified lines | |||
| 1942 | static void push_rt_tasks(struct rq *rq) | 1968 | static void push_rt_tasks(struct rq *rq) |
| 1943 | { | 1969 | { |
| 1944 | /* push_rt_task will return true if it moved an RT */ | 1970 | /* push_rt_task will return true if it moved an RT */ |
| 1945 | while (push_rt_task(rq)) | 1971 | while (push_rt_task(rq, false)) |
| 1946 | ; | 1972 | ; |
| 1947 | } | 1973 | } |
| 1948 | 1974 | ||
| @@ -2095,7 +2121,8 @@ void rto_push_irq_work_func(struct irq_work *work) · 146 unmodified lines | |||
| 2095 | */ | 2121 | */ |
| 2096 | if (has_pushable_tasks(rq)) { | 2122 | if (has_pushable_tasks(rq)) { |
| 2097 | raw_spin_lock(&rq->lock); | 2123 | raw_spin_lock(&rq->lock); |
| 2098 | push_rt_tasks(rq); | 2124 | while (push_rt_task(rq, true)) |
| 2125 | ; | ||
| 2099 | raw_spin_unlock(&rq->lock); | 2126 | raw_spin_unlock(&rq->lock); |
| 2100 | } | 2127 | } |
| 2101 | 2128 | ||
| @@ -2120,7 +2147,7 @@ static void pull_rt_task(struct rq *this_rq) · 18 unmodified lines | |||
| 2120 | { | 2147 | { |
| 2121 | int this_cpu = this_rq->cpu, cpu; | 2148 | int this_cpu = this_rq->cpu, cpu; |
| 2122 | bool resched = false; | 2149 | bool resched = false; |
| 2123 | struct task_struct *p; | 2150 | struct task_struct *p, *push_task; |
| 2124 | struct rq *src_rq; | 2151 | struct rq *src_rq; |
| 2125 | int rt_overload_count = rt_overloaded(this_rq); | 2152 | int rt_overload_count = rt_overloaded(this_rq); |
| 2126 | 2153 | ||
| @@ -2167,6 +2194,7 @@ static void pull_rt_task(struct rq *this_rq) · 40 unmodified lines | |||
| 2167 | * double_lock_balance, and another CPU could | 2194 | * double_lock_balance, and another CPU could |
| 2168 | * alter this_rq | 2195 | * alter this_rq |
| 2169 | */ | 2196 | */ |
| 2197 | push_task = NULL; | ||
| 2170 | double_lock_balance(this_rq, src_rq); | 2198 | double_lock_balance(this_rq, src_rq); |
| 2171 | 2199 | ||
| 2172 | /* | 2200 | /* |
| @@ -2194,11 +2222,14 @@ static void pull_rt_task(struct rq *this_rq) · 21 unmodified lines | |||
| 2194 | if (p->prio < src_rq->curr->prio) | 2222 | if (p->prio < src_rq->curr->prio) |
| 2195 | goto skip; | 2223 | goto skip; |
| 2196 | 2224 | ||
| 2197 | resched = true; | 2225 | if (is_migration_disabled(p)) { |
| 2198 | 2226 | push_task = get_push_task(src_rq); | |
| 2199 | deactivate_task(src_rq, p, 0); | 2227 | } else { |
| 2200 | set_task_cpu(p, this_cpu); | 2228 | deactivate_task(src_rq, p, 0); |
| 2201 | activate_task(this_rq, p, 0); | 2229 | set_task_cpu(p, this_cpu); |
| 2230 | activate_task(this_rq, p, 0); | ||
| 2231 | resched = true; | ||
| 2232 | } | ||
| 2202 | /* | 2233 | /* |
| 2203 | * We continue with the search, just in | 2234 | * We continue with the search, just in |
| 2204 | * case there's an even higher prio task | 2235 | * case there's an even higher prio task |
| @@ -2208,6 +2239,13 @@ static void pull_rt_task(struct rq *this_rq) · 3 unmodified lines | |||
| 2208 | } | 2239 | } |
| 2209 | skip: | 2240 | skip: |
| 2210 | double_unlock_balance(this_rq, src_rq); | 2241 | double_unlock_balance(this_rq, src_rq); |
| 2242 | |||
| 2243 | if (push_task) { | ||
| 2244 | raw_spin_unlock(&this_rq->lock); | ||
| 2245 | stop_one_cpu_nowait(src_rq->cpu, push_cpu_stop, | ||
| 2246 | push_task, &src_rq->push_work); | ||
| 2247 | raw_spin_lock(&this_rq->lock); | ||
| 2248 | } | ||
| 2211 | } | 2249 | } |
| 2212 | 2250 | ||
| 2213 | if (resched) | 2251 | if (resched) |
| @@ -2449,6 +2487,7 @@ const struct sched_class rt_sched_class · 235 unmodified lines | |||
| 2449 | .rq_offline = rq_offline_rt, | 2487 | .rq_offline = rq_offline_rt, |
| 2450 | .task_woken = task_woken_rt, | 2488 | .task_woken = task_woken_rt, |
| 2451 | .switched_from = switched_from_rt, | 2489 | .switched_from = switched_from_rt, |
| 2490 | .find_lock_rq = find_lock_lowest_rq, | ||
| 2452 | #endif | 2491 | #endif |
| 2453 | 2492 | ||
| 2454 | .task_tick = task_tick_rt, | 2493 | .task_tick = task_tick_rt, |
| @@ -1057,6 +1057,8 @@ struct rq { · 1056 unmodified lines | |||
| 1057 | #if defined(CONFIG_PREEMPT_RT) && defined(CONFIG_SMP) | 1057 | #if defined(CONFIG_PREEMPT_RT) && defined(CONFIG_SMP) |
| 1058 | unsigned int nr_pinned; | 1058 | unsigned int nr_pinned; |
| 1059 | #endif | 1059 | #endif |
| 1060 | unsigned int push_busy; | ||
| 1061 | struct cpu_stop_work push_work; | ||
| 1060 | }; | 1062 | }; |
| 1061 | 1063 | ||
| 1062 | #ifdef CONFIG_FAIR_GROUP_SCHED | 1064 | #ifdef CONFIG_FAIR_GROUP_SCHED |
| @@ -1084,6 +1086,16 @@ static inline int cpu_of(struct rq *rq) · 21 unmodified lines | |||
| 1084 | #endif | 1086 | #endif |
| 1085 | } | 1087 | } |
| 1086 | 1088 | ||
| 1089 | #define MDF_PUSH 0x01 | ||
| 1090 | |||
| 1091 | static inline bool is_migration_disabled(struct task_struct *p) | ||
| 1092 | { | ||
| 1093 | #if defined(CONFIG_SMP) && defined(CONFIG_PREEMPT_RT) | ||
| 1094 | return p->migration_disabled; | ||
| 1095 | #else | ||
| 1096 | return false; | ||
| 1097 | #endif | ||
| 1098 | } | ||
| 1087 | 1099 | ||
| 1088 | #ifdef CONFIG_SCHED_SMT | 1100 | #ifdef CONFIG_SCHED_SMT |
| 1089 | extern void __update_idle_core(struct rq *rq); | 1101 | extern void __update_idle_core(struct rq *rq); |
| @@ -1823,6 +1835,8 @@ struct sched_class { · 733 unmodified lines | |||
| 1823 | 1835 | ||
| 1824 | void (*rq_online)(struct rq *rq); | 1836 | void (*rq_online)(struct rq *rq); |
| 1825 | void (*rq_offline)(struct rq *rq); | 1837 | void (*rq_offline)(struct rq *rq); |
| 1838 | |||
| 1839 | struct rq *(*find_lock_rq)(struct task_struct *p, struct rq *rq); | ||
| 1826 | #endif | 1840 | #endif |
| 1827 | 1841 | ||
| 1828 | void (*task_tick)(struct rq *rq, struct task_struct *p, int queued); | 1842 | void (*task_tick)(struct rq *rq, struct task_struct *p, int queued); |
| @@ -1918,6 +1932,24 @@ extern void trigger_load_balance(struct rq *rq); · 89 unmodified lines | |||
| 1918 | 1932 | ||
| 1919 | extern void set_cpus_allowed_common(struct task_struct *p, const struct cpumask *new_mask, u32 flags); | 1933 | extern void set_cpus_allowed_common(struct task_struct *p, const struct cpumask *new_mask, u32 flags); |
| 1920 | 1934 | ||
| 1935 | static inline struct task_struct *get_push_task(struct rq *rq) | ||
| 1936 | { | ||
| 1937 | struct task_struct *p = rq->curr; | ||
| 1938 | |||
| 1939 | lockdep_assert_held(&rq->lock); | ||
| 1940 | |||
| 1941 | if (rq->push_busy) | ||
| 1942 | return NULL; | ||
| 1943 | |||
| 1944 | if (p->nr_cpus_allowed == 1) | ||
| 1945 | return NULL; | ||
| 1946 | |||
| 1947 | rq->push_busy = true; | ||
| 1948 | return get_task_struct(p); | ||
| 1949 | } | ||
| 1950 | |||
| 1951 | extern int push_cpu_stop(void *arg); | ||
| 1952 | |||
| 1921 | #endif | 1953 | #endif |
| 1922 | 1954 | ||
| 1923 | #ifdef CONFIG_CPU_IDLE | 1955 | #ifdef CONFIG_CPU_IDLE |