History

Merge branch 'sched/migrate-disable'

Peter Zijlstra committed 6 years ago

Commit 12fa97c2 parents b6d37a7 c777d8419 files · +1038 −227Browse files

Showing the changes to kernel (9 of 19 files). Show all

Unified · Split

9 files
  1. kernel/cpu.c +8 −1
  2. kernel/sched/core.c +739 −181
  3. kernel/sched/cpudeadline.c +2 −2
  4. kernel/sched/cpupri.c +2 −2
  5. kernel/sched/deadline.c +31 −15
  6. kernel/sched/rt.c +57 −18
  7. kernel/sched/sched.h +56 −3
  8. kernel/stop_machine.c +24 −3
  9. kernel/workqueue.c +4 −0
kernel/cpu.c+8−1
@@ -1602,7 +1602,7 @@ static struct cpuhp_step cpuhp_hp_states[] = { · 1601 unmodified lines
1602 .name = "ap:online",1602 .name = "ap:online",
1603 },1603 },
1604 /*1604 /*
1605 * Handled on controll processor until the plugged processor manages1605 * Handled on control processor until the plugged processor manages
1606 * this itself.1606 * this itself.
1607 */1607 */
1608 [CPUHP_TEARDOWN_CPU] = {1608 [CPUHP_TEARDOWN_CPU] = {
@@ -1611,6 +1611,13 @@ static struct cpuhp_step cpuhp_hp_states[] = { · 2 unmodified lines
1611 .teardown.single = takedown_cpu,1611 .teardown.single = takedown_cpu,
1612 .cant_stop = true,1612 .cant_stop = true,
1613 },1613 },
1614
1615 [CPUHP_AP_SCHED_WAIT_EMPTY] = {
1616 .name = "sched:waitempty",
1617 .startup.single = NULL,
1618 .teardown.single = sched_cpu_wait_empty,
1619 },
1620
1614 /* Handle smpboot threads park/unpark */1621 /* Handle smpboot threads park/unpark */
1615 [CPUHP_AP_SMPBOOT_THREADS] = {1622 [CPUHP_AP_SMPBOOT_THREADS] = {
1616 .name = "smpboot/threads:online",1623 .name = "smpboot/threads:online",
kernel/sched/core.c+739−181
@@ -1696,6 +1696,80 @@ void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags) · 1695 unmodified lines
1696 1696
1697#ifdef CONFIG_SMP1697#ifdef CONFIG_SMP
1698 1698
1699#ifdef CONFIG_PREEMPT_RT
1700
1701static void
1702__do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask, u32 flags);
1703
1704static int __set_cpus_allowed_ptr(struct task_struct *p,
1705 const struct cpumask *new_mask,
1706 u32 flags);
1707
1708static void migrate_disable_switch(struct rq *rq, struct task_struct *p)
1709{
1710 if (likely(!p->migration_disabled))
1711 return;
1712
1713 if (p->cpus_ptr != &p->cpus_mask)
1714 return;
1715
1716 /*
1717 * Violates locking rules! see comment in __do_set_cpus_allowed().
1718 */
1719 __do_set_cpus_allowed(p, cpumask_of(rq->cpu), SCA_MIGRATE_DISABLE);
1720}
1721
1722void migrate_disable(void)
1723{
1724 struct task_struct *p = current;
1725
1726 if (p->migration_disabled) {
1727 p->migration_disabled++;
1728 return;
1729 }
1730
1731 preempt_disable();
1732 this_rq()->nr_pinned++;
1733 p->migration_disabled = 1;
1734 preempt_enable();
1735}
1736EXPORT_SYMBOL_GPL(migrate_disable);
1737
1738void migrate_enable(void)
1739{
1740 struct task_struct *p = current;
1741
1742 if (p->migration_disabled > 1) {
1743 p->migration_disabled--;
1744 return;
1745 }
1746
1747 /*
1748 * Ensure stop_task runs either before or after this, and that
1749 * __set_cpus_allowed_ptr(SCA_MIGRATE_ENABLE) doesn't schedule().
1750 */
1751 preempt_disable();
1752 if (p->cpus_ptr != &p->cpus_mask)
1753 __set_cpus_allowed_ptr(p, &p->cpus_mask, SCA_MIGRATE_ENABLE);
1754 /*
1755 * Mustn't clear migration_disabled() until cpus_ptr points back at the
1756 * regular cpus_mask, otherwise things that race (eg.
1757 * select_fallback_rq) get confused.
1758 */
1759 barrier();
1760 p->migration_disabled = 0;
1761 this_rq()->nr_pinned--;
1762 preempt_enable();
1763}
1764EXPORT_SYMBOL_GPL(migrate_enable);
1765
1766static inline bool rq_has_pinned_tasks(struct rq *rq)
1767{
1768 return rq->nr_pinned;
1769}
1770
1771#endif
1772
1699/*1773/*
1700 * Per-CPU kthreads are allowed to run on !active && online CPUs, see1774 * Per-CPU kthreads are allowed to run on !active && online CPUs, see
1701 * __set_cpus_allowed_ptr() and select_fallback_rq().1775 * __set_cpus_allowed_ptr() and select_fallback_rq().
@@ -1705,7 +1779,7 @@ static inline bool is_cpu_allowed(struct task_struct *p, int cpu) · 3 unmodified lines
1705 if (!cpumask_test_cpu(cpu, p->cpus_ptr))1779 if (!cpumask_test_cpu(cpu, p->cpus_ptr))
1706 return false;1780 return false;
1707 1781
1708 if (is_per_cpu_kthread(p))1782 if (is_per_cpu_kthread(p) || is_migration_disabled(p))
1709 return cpu_online(cpu);1783 return cpu_online(cpu);
1710 1784
1711 return cpu_active(cpu);1785 return cpu_active(cpu);
@@ -1750,8 +1824,16 @@ static struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf, · 38 unmodified lines
1750}1824}
1751 1825
1752struct migration_arg {1826struct migration_arg {
1753 struct task_struct *task;1827 struct task_struct *task;
1754 int dest_cpu;1828 int dest_cpu;
1829 struct set_affinity_pending *pending;
1830};
1831
1832struct set_affinity_pending {
1833 refcount_t refs;
1834 struct completion done;
1835 struct cpu_stop_work stop_work;
1836 struct migration_arg arg;
1755};1837};
1756 1838
1757/*1839/*
@@ -1783,16 +1865,19 @@ static struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf, · 25 unmodified lines
1783 */1865 */
1784static int migration_cpu_stop(void *data)1866static int migration_cpu_stop(void *data)
1785{1867{
1868 struct set_affinity_pending *pending;
1786 struct migration_arg *arg = data;1869 struct migration_arg *arg = data;
1787 struct task_struct *p = arg->task;1870 struct task_struct *p = arg->task;
1871 int dest_cpu = arg->dest_cpu;
1788 struct rq *rq = this_rq();1872 struct rq *rq = this_rq();
1873 bool complete = false;
1789 struct rq_flags rf;1874 struct rq_flags rf;
1790 1875
1791 /*1876 /*
1792 * The original target CPU might have gone down and we might1877 * The original target CPU might have gone down and we might
1793 * be on another CPU but it doesn't matter.1878 * be on another CPU but it doesn't matter.
1794 */1879 */
1795 local_irq_disable();1880 local_irq_save(rf.flags);
1796 /*1881 /*
1797 * We need to explicitly wake pending tasks before running1882 * We need to explicitly wake pending tasks before running
1798 * __migrate_task() such that we will not miss enforcing cpus_ptr1883 * __migrate_task() such that we will not miss enforcing cpus_ptr
@@ -1802,21 +1887,126 @@ static int migration_cpu_stop(void *data) · 3 unmodified lines
1802 1887
1803 raw_spin_lock(&p->pi_lock);1888 raw_spin_lock(&p->pi_lock);
1804 rq_lock(rq, &rf);1889 rq_lock(rq, &rf);
1890
1891 pending = p->migration_pending;
1805 /*1892 /*
1806 * If task_rq(p) != rq, it cannot be migrated here, because we're1893 * If task_rq(p) != rq, it cannot be migrated here, because we're
1807 * holding rq->lock, if p->on_rq == 0 it cannot get enqueued because1894 * holding rq->lock, if p->on_rq == 0 it cannot get enqueued because
1808 * we're holding p->pi_lock.1895 * we're holding p->pi_lock.
1809 */1896 */
1810 if (task_rq(p) == rq) {1897 if (task_rq(p) == rq) {
1898 if (is_migration_disabled(p))
1899 goto out;
1900
1901 if (pending) {
1902 p->migration_pending = NULL;
1903 complete = true;
1904 }
1905
1906 /* migrate_enable() -- we must not race against SCA */
1907 if (dest_cpu < 0) {
1908 /*
1909 * When this was migrate_enable() but we no longer
1910 * have a @pending, a concurrent SCA 'fixed' things
1911 * and we should be valid again. Nothing to do.
1912 */
1913 if (!pending) {
1914 WARN_ON_ONCE(!is_cpu_allowed(p, cpu_of(rq)));
1915 goto out;
1916 }
1917
1918 dest_cpu = cpumask_any_distribute(&p->cpus_mask);
1919 }
1920
1811 if (task_on_rq_queued(p))1921 if (task_on_rq_queued(p))
1812 rq = __migrate_task(rq, &rf, p, arg->dest_cpu);1922 rq = __migrate_task(rq, &rf, p, dest_cpu);
1813 else1923 else
1814 p->wake_cpu = arg->dest_cpu;1924 p->wake_cpu = dest_cpu;
1925
1926 } else if (dest_cpu < 0) {
1927 /*
1928 * This happens when we get migrated between migrate_enable()'s
1929 * preempt_enable() and scheduling the stopper task. At that
1930 * point we're a regular task again and not current anymore.
1931 *
1932 * A !PREEMPT kernel has a giant hole here, which makes it far
1933 * more likely.
1934 */
1935
1936 /*
1937 * When this was migrate_enable() but we no longer have an
1938 * @pending, a concurrent SCA 'fixed' things and we should be
1939 * valid again. Nothing to do.
1940 */
1941 if (!pending) {
1942 WARN_ON_ONCE(!is_cpu_allowed(p, cpu_of(rq)));
1943 goto out;
1944 }
1945
1946 /*
1947 * When migrate_enable() hits a rq mis-match we can't reliably
1948 * determine is_migration_disabled() and so have to chase after
1949 * it.
1950 */
1951 task_rq_unlock(rq, p, &rf);
1952 stop_one_cpu_nowait(task_cpu(p), migration_cpu_stop,
1953 &pending->arg, &pending->stop_work);
1954 return 0;
1815 }1955 }
1816 rq_unlock(rq, &rf);1956out:
1817 raw_spin_unlock(&p->pi_lock);1957 task_rq_unlock(rq, p, &rf);
1958
1959 if (complete)
1960 complete_all(&pending->done);
1961
1962 /* For pending->{arg,stop_work} */
1963 pending = arg->pending;
1964 if (pending && refcount_dec_and_test(&pending->refs))
1965 wake_up_var(&pending->refs);
1818 1966
1819 local_irq_enable();1967 return 0;
1968}
1969
1970int push_cpu_stop(void *arg)
1971{
1972 struct rq *lowest_rq = NULL, *rq = this_rq();
1973 struct task_struct *p = arg;
1974
1975 raw_spin_lock_irq(&p->pi_lock);
1976 raw_spin_lock(&rq->lock);
1977
1978 if (task_rq(p) != rq)
1979 goto out_unlock;
1980
1981 if (is_migration_disabled(p)) {
1982 p->migration_flags |= MDF_PUSH;
1983 goto out_unlock;
1984 }
1985
1986 p->migration_flags &= ~MDF_PUSH;
1987
1988 if (p->sched_class->find_lock_rq)
1989 lowest_rq = p->sched_class->find_lock_rq(p, rq);
1990
1991 if (!lowest_rq)
1992 goto out_unlock;
1993
1994 // XXX validate p is still the highest prio task
1995 if (task_rq(p) == rq) {
1996 deactivate_task(rq, p, 0);
1997 set_task_cpu(p, lowest_rq->cpu);
1998 activate_task(lowest_rq, p, 0);
1999 resched_curr(lowest_rq);
2000 }
2001
2002 double_unlock_balance(rq, lowest_rq);
2003
2004out_unlock:
2005 rq->push_busy = false;
2006 raw_spin_unlock(&rq->lock);
2007 raw_spin_unlock_irq(&p->pi_lock);
2008
2009 put_task_struct(p);
1820 return 0;2010 return 0;
1821}2011}
1822 2012
@@ -1824,18 +2014,39 @@ static int migration_cpu_stop(void *data) · 1 unmodified line
1824 * sched_class::set_cpus_allowed must do the below, but is not required to2014 * sched_class::set_cpus_allowed must do the below, but is not required to
1825 * actually call this function.2015 * actually call this function.
1826 */2016 */
1827void set_cpus_allowed_common(struct task_struct *p, const struct cpumask *new_mask)2017void set_cpus_allowed_common(struct task_struct *p, const struct cpumask *new_mask, u32 flags)
1828{2018{
2019 if (flags & (SCA_MIGRATE_ENABLE | SCA_MIGRATE_DISABLE)) {
2020 p->cpus_ptr = new_mask;
2021 return;
2022 }
2023
1829 cpumask_copy(&p->cpus_mask, new_mask);2024 cpumask_copy(&p->cpus_mask, new_mask);
1830 p->nr_cpus_allowed = cpumask_weight(new_mask);2025 p->nr_cpus_allowed = cpumask_weight(new_mask);
1831}2026}
1832 2027
1833void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)2028static void
2029__do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask, u32 flags)
1834{2030{
1835 struct rq *rq = task_rq(p);2031 struct rq *rq = task_rq(p);
1836 bool queued, running;2032 bool queued, running;
1837 2033
1838 lockdep_assert_held(&p->pi_lock);2034 /*
2035 * This here violates the locking rules for affinity, since we're only
2036 * supposed to change these variables while holding both rq->lock and
2037 * p->pi_lock.
2038 *
2039 * HOWEVER, it magically works, because ttwu() is the only code that
2040 * accesses these variables under p->pi_lock and only does so after
2041 * smp_cond_load_acquire(&p->on_cpu, !VAL), and we're in __schedule()
2042 * before finish_task().
2043 *
2044 * XXX do further audits, this smells like something putrid.
2045 */
2046 if (flags & SCA_MIGRATE_DISABLE)
2047 SCHED_WARN_ON(!p->on_cpu);
2048 else
2049 lockdep_assert_held(&p->pi_lock);
1839 2050
1840 queued = task_on_rq_queued(p);2051 queued = task_on_rq_queued(p);
1841 running = task_current(rq, p);2052 running = task_current(rq, p);
@@ -1851,7 +2062,7 @@ void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) · 9 unmodified lines
1851 if (running)2062 if (running)
1852 put_prev_task(rq, p);2063 put_prev_task(rq, p);
1853 2064
1854 p->sched_class->set_cpus_allowed(p, new_mask);2065 p->sched_class->set_cpus_allowed(p, new_mask, flags);
1855 2066
1856 if (queued)2067 if (queued)
1857 enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK);2068 enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK);
@@ -1859,6 +2070,208 @@ void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) · 1 unmodified line
1859 set_next_task(rq, p);2070 set_next_task(rq, p);
1860}2071}
1861 2072
2073void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
2074{
2075 __do_set_cpus_allowed(p, new_mask, 0);
2076}
2077
2078/*
2079 * This function is wildly self concurrent; here be dragons.
2080 *
2081 *
2082 * When given a valid mask, __set_cpus_allowed_ptr() must block until the
2083 * designated task is enqueued on an allowed CPU. If that task is currently
2084 * running, we have to kick it out using the CPU stopper.
2085 *
2086 * Migrate-Disable comes along and tramples all over our nice sandcastle.
2087 * Consider:
2088 *
2089 * Initial conditions: P0->cpus_mask = [0, 1]
2090 *
2091 * P0@CPU0 P1
2092 *
2093 * migrate_disable();
2094 * <preempted>
2095 * set_cpus_allowed_ptr(P0, [1]);
2096 *
2097 * P1 *cannot* return from this set_cpus_allowed_ptr() call until P0 executes
2098 * its outermost migrate_enable() (i.e. it exits its Migrate-Disable region).
2099 * This means we need the following scheme:
2100 *
2101 * P0@CPU0 P1
2102 *
2103 * migrate_disable();
2104 * <preempted>
2105 * set_cpus_allowed_ptr(P0, [1]);
2106 * <blocks>
2107 * <resumes>
2108 * migrate_enable();
2109 * __set_cpus_allowed_ptr();
2110 * <wakes local stopper>
2111 * `--> <woken on migration completion>
2112 *
2113 * Now the fun stuff: there may be several P1-like tasks, i.e. multiple
2114 * concurrent set_cpus_allowed_ptr(P0, [*]) calls. CPU affinity changes of any
2115 * task p are serialized by p->pi_lock, which we can leverage: the one that
2116 * should come into effect at the end of the Migrate-Disable region is the last
2117 * one. This means we only need to track a single cpumask (i.e. p->cpus_mask),
2118 * but we still need to properly signal those waiting tasks at the appropriate
2119 * moment.
2120 *
2121 * This is implemented using struct set_affinity_pending. The first
2122 * __set_cpus_allowed_ptr() caller within a given Migrate-Disable region will
2123 * setup an instance of that struct and install it on the targeted task_struct.
2124 * Any and all further callers will reuse that instance. Those then wait for
2125 * a completion signaled at the tail of the CPU stopper callback (1), triggered
2126 * on the end of the Migrate-Disable region (i.e. outermost migrate_enable()).
2127 *
2128 *
2129 * (1) In the cases covered above. There is one more where the completion is
2130 * signaled within affine_move_task() itself: when a subsequent affinity request
2131 * cancels the need for an active migration. Consider:
2132 *
2133 * Initial conditions: P0->cpus_mask = [0, 1]
2134 *
2135 * P0@CPU0 P1 P2
2136 *
2137 * migrate_disable();
2138 * <preempted>
2139 * set_cpus_allowed_ptr(P0, [1]);
2140 * <blocks>
2141 * set_cpus_allowed_ptr(P0, [0, 1]);
2142 * <signal completion>
2143 * <awakes>
2144 *
2145 * Note that the above is safe vs a concurrent migrate_enable(), as any
2146 * pending affinity completion is preceded by an uninstallation of
2147 * p->migration_pending done with p->pi_lock held.
2148 */
2149static int affine_move_task(struct rq *rq, struct task_struct *p, struct rq_flags *rf,
2150 int dest_cpu, unsigned int flags)
2151{
2152 struct set_affinity_pending my_pending = { }, *pending = NULL;
2153 struct migration_arg arg = {
2154 .task = p,
2155 .dest_cpu = dest_cpu,
2156 };
2157 bool complete = false;
2158
2159 /* Can the task run on the task's current CPU? If so, we're done */
2160 if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask)) {
2161 struct task_struct *push_task = NULL;
2162
2163 if ((flags & SCA_MIGRATE_ENABLE) &&
2164 (p->migration_flags & MDF_PUSH) && !rq->push_busy) {
2165 rq->push_busy = true;
2166 push_task = get_task_struct(p);
2167 }
2168
2169 pending = p->migration_pending;
2170 if (pending) {
2171 refcount_inc(&pending->refs);
2172 p->migration_pending = NULL;
2173 complete = true;
2174 }
2175 task_rq_unlock(rq, p, rf);
2176
2177 if (push_task) {
2178 stop_one_cpu_nowait(rq->cpu, push_cpu_stop,
2179 p, &rq->push_work);
2180 }
2181
2182 if (complete)
2183 goto do_complete;
2184
2185 return 0;
2186 }
2187
2188 if (!(flags & SCA_MIGRATE_ENABLE)) {
2189 /* serialized by p->pi_lock */
2190 if (!p->migration_pending) {
2191 /* Install the request */
2192 refcount_set(&my_pending.refs, 1);
2193 init_completion(&my_pending.done);
2194 p->migration_pending = &my_pending;
2195 } else {
2196 pending = p->migration_pending;
2197 refcount_inc(&pending->refs);
2198 }
2199 }
2200 pending = p->migration_pending;
2201 /*
2202 * - !MIGRATE_ENABLE:
2203 * we'll have installed a pending if there wasn't one already.
2204 *
2205 * - MIGRATE_ENABLE:
2206 * we're here because the current CPU isn't matching anymore,
2207 * the only way that can happen is because of a concurrent
2208 * set_cpus_allowed_ptr() call, which should then still be
2209 * pending completion.
2210 *
2211 * Either way, we really should have a @pending here.
2212 */
2213 if (WARN_ON_ONCE(!pending)) {
2214 task_rq_unlock(rq, p, rf);
2215 return -EINVAL;
2216 }
2217
2218 if (flags & SCA_MIGRATE_ENABLE) {
2219
2220 refcount_inc(&pending->refs); /* pending->{arg,stop_work} */
2221 p->migration_flags &= ~MDF_PUSH;
2222 task_rq_unlock(rq, p, rf);
2223
2224 pending->arg = (struct migration_arg) {
2225 .task = p,
2226 .dest_cpu = -1,
2227 .pending = pending,
2228 };
2229
2230 stop_one_cpu_nowait(cpu_of(rq), migration_cpu_stop,
2231 &pending->arg, &pending->stop_work);
2232
2233 return 0;
2234 }
2235
2236 if (task_running(rq, p) || p->state == TASK_WAKING) {
2237 /*
2238 * Lessen races (and headaches) by delegating
2239 * is_migration_disabled(p) checks to the stopper, which will
2240 * run on the same CPU as said p.
2241 */
2242 task_rq_unlock(rq, p, rf);
2243 stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
2244
2245 } else {
2246
2247 if (!is_migration_disabled(p)) {
2248 if (task_on_rq_queued(p))
2249 rq = move_queued_task(rq, rf, p, dest_cpu);
2250
2251 p->migration_pending = NULL;
2252 complete = true;
2253 }
2254 task_rq_unlock(rq, p, rf);
2255
2256do_complete:
2257 if (complete)
2258 complete_all(&pending->done);
2259 }
2260
2261 wait_for_completion(&pending->done);
2262
2263 if (refcount_dec_and_test(&pending->refs))
2264 wake_up_var(&pending->refs);
2265
2266 /*
2267 * Block the original owner of &pending until all subsequent callers
2268 * have seen the completion and decremented the refcount
2269 */
2270 wait_var_event(&my_pending.refs, !refcount_read(&my_pending.refs));
2271
2272 return 0;
2273}
2274
1862/*2275/*
1863 * Change a given task's CPU affinity. Migrate the thread to a2276 * Change a given task's CPU affinity. Migrate the thread to a
1864 * proper CPU and schedule it away if the CPU it's executing on2277 * proper CPU and schedule it away if the CPU it's executing on
@@ -1869,7 +2282,8 @@ void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) · 4 unmodified lines
1869 * call is not atomic; no spinlocks may be held.2282 * call is not atomic; no spinlocks may be held.
1870 */2283 */
1871static int __set_cpus_allowed_ptr(struct task_struct *p,2284static int __set_cpus_allowed_ptr(struct task_struct *p,
1872 const struct cpumask *new_mask, bool check)2285 const struct cpumask *new_mask,
2286 u32 flags)
1873{2287{
1874 const struct cpumask *cpu_valid_mask = cpu_active_mask;2288 const struct cpumask *cpu_valid_mask = cpu_active_mask;
1875 unsigned int dest_cpu;2289 unsigned int dest_cpu;
@@ -1880,9 +2294,14 @@ static int __set_cpus_allowed_ptr(struct task_struct *p, · 4 unmodified lines
1880 rq = task_rq_lock(p, &rf);2294 rq = task_rq_lock(p, &rf);
1881 update_rq_clock(rq);2295 update_rq_clock(rq);
1882 2296
1883 if (p->flags & PF_KTHREAD) {2297 if (p->flags & PF_KTHREAD || is_migration_disabled(p)) {
1884 /*2298 /*
1885 * Kernel threads are allowed on online && !active CPUs2299 * Kernel threads are allowed on online && !active CPUs.
2300 *
2301 * Specifically, migration_disabled() tasks must not fail the
2302 * cpumask_any_and_distribute() pick below, esp. so on
2303 * SCA_MIGRATE_ENABLE, otherwise we'll not call
2304 * set_cpus_allowed_common() and actually reset p->cpus_ptr.
1886 */2305 */
1887 cpu_valid_mask = cpu_online_mask;2306 cpu_valid_mask = cpu_online_mask;
1888 }2307 }
@@ -1891,13 +2310,22 @@ static int __set_cpus_allowed_ptr(struct task_struct *p, · 2 unmodified lines
1891 * Must re-check here, to close a race against __kthread_bind(),2310 * Must re-check here, to close a race against __kthread_bind(),
1892 * sched_setaffinity() is not guaranteed to observe the flag.2311 * sched_setaffinity() is not guaranteed to observe the flag.
1893 */2312 */
1894 if (check && (p->flags & PF_NO_SETAFFINITY)) {2313 if ((flags & SCA_CHECK) && (p->flags & PF_NO_SETAFFINITY)) {
1895 ret = -EINVAL;2314 ret = -EINVAL;
1896 goto out;2315 goto out;
1897 }2316 }
1898 2317
1899 if (cpumask_equal(&p->cpus_mask, new_mask))2318 if (!(flags & SCA_MIGRATE_ENABLE)) {
1900 goto out;2319 if (cpumask_equal(&p->cpus_mask, new_mask))
2320 goto out;
2321
2322 if (WARN_ON_ONCE(p == current &&
2323 is_migration_disabled(p) &&
2324 !cpumask_test_cpu(task_cpu(p), new_mask))) {
2325 ret = -EBUSY;
2326 goto out;
2327 }
2328 }
1901 2329
1902 /*2330 /*
1903 * Picking a ~random cpu helps in cases where we are changing affinity2331 * Picking a ~random cpu helps in cases where we are changing affinity
@@ -1910,7 +2338,7 @@ static int __set_cpus_allowed_ptr(struct task_struct *p, · 6 unmodified lines
1910 goto out;2338 goto out;
1911 }2339 }
1912 2340
1913 do_set_cpus_allowed(p, new_mask);2341 __do_set_cpus_allowed(p, new_mask, flags);
1914 2342
1915 if (p->flags & PF_KTHREAD) {2343 if (p->flags & PF_KTHREAD) {
1916 /*2344 /*
@@ -1922,23 +2350,8 @@ static int __set_cpus_allowed_ptr(struct task_struct *p, · 5 unmodified lines
1922 p->nr_cpus_allowed != 1);2350 p->nr_cpus_allowed != 1);
1923 }2351 }
1924 2352
1925 /* Can the task run on the task's current CPU? If so, we're done */2353 return affine_move_task(rq, p, &rf, dest_cpu, flags);
1926 if (cpumask_test_cpu(task_cpu(p), new_mask))
1927 goto out;
1928 2354
1929 if (task_running(rq, p) || p->state == TASK_WAKING) {
1930 struct migration_arg arg = { p, dest_cpu };
1931 /* Need help from migration thread: drop lock and wait. */
1932 task_rq_unlock(rq, p, &rf);
1933 stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
1934 return 0;
1935 } else if (task_on_rq_queued(p)) {
1936 /*
1937 * OK, since we're going to drop the lock immediately
1938 * afterwards anyway.
1939 */
1940 rq = move_queued_task(rq, &rf, p, dest_cpu);
1941 }
1942out:2355out:
1943 task_rq_unlock(rq, p, &rf);2356 task_rq_unlock(rq, p, &rf);
1944 2357
@@ -1947,7 +2360,7 @@ out: · 2 unmodified lines
1947 2360
1948int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)2361int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
1949{2362{
1950 return __set_cpus_allowed_ptr(p, new_mask, false);2363 return __set_cpus_allowed_ptr(p, new_mask, 0);
1951}2364}
1952EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);2365EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
1953 2366
@@ -1988,6 +2401,8 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu) · 34 unmodified lines
1988 * Clearly, migrating tasks to offline CPUs is a fairly daft thing.2401 * Clearly, migrating tasks to offline CPUs is a fairly daft thing.
1989 */2402 */
1990 WARN_ON_ONCE(!cpu_online(new_cpu));2403 WARN_ON_ONCE(!cpu_online(new_cpu));
2404
2405 WARN_ON_ONCE(is_migration_disabled(p));
1991#endif2406#endif
1992 2407
1993 trace_sched_migrate_task(p, new_cpu);2408 trace_sched_migrate_task(p, new_cpu);
@@ -2318,6 +2733,12 @@ static int select_fallback_rq(int cpu, struct task_struct *p) · 324 unmodified lines
2318 }2733 }
2319 fallthrough;2734 fallthrough;
2320 case possible:2735 case possible:
2736 /*
2737 * XXX When called from select_task_rq() we only
2738 * hold p->pi_lock and again violate locking order.
2739 *
2740 * More yuck to audit.
2741 */
2321 do_set_cpus_allowed(p, cpu_possible_mask);2742 do_set_cpus_allowed(p, cpu_possible_mask);
2322 state = fail;2743 state = fail;
2323 break;2744 break;
@@ -2352,7 +2773,7 @@ int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags) · 28 unmodified lines
2352{2773{
2353 lockdep_assert_held(&p->pi_lock);2774 lockdep_assert_held(&p->pi_lock);
2354 2775
2355 if (p->nr_cpus_allowed > 1)2776 if (p->nr_cpus_allowed > 1 && !is_migration_disabled(p))
2356 cpu = p->sched_class->select_task_rq(p, cpu, sd_flags, wake_flags);2777 cpu = p->sched_class->select_task_rq(p, cpu, sd_flags, wake_flags);
2357 else2778 else
2358 cpu = cpumask_any(p->cpus_ptr);2779 cpu = cpumask_any(p->cpus_ptr);
@@ -2375,6 +2796,7 @@ int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags) · 16 unmodified lines
2375 2796
2376void sched_set_stop_task(int cpu, struct task_struct *stop)2797void sched_set_stop_task(int cpu, struct task_struct *stop)
2377{2798{
2799 static struct lock_class_key stop_pi_lock;
2378 struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };2800 struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };
2379 struct task_struct *old_stop = cpu_rq(cpu)->stop;2801 struct task_struct *old_stop = cpu_rq(cpu)->stop;
2380 2802
@@ -2390,6 +2812,20 @@ void sched_set_stop_task(int cpu, struct task_struct *stop) · 9 unmodified lines
2390 sched_setscheduler_nocheck(stop, SCHED_FIFO, &param);2812 sched_setscheduler_nocheck(stop, SCHED_FIFO, &param);
2391 2813
2392 stop->sched_class = &stop_sched_class;2814 stop->sched_class = &stop_sched_class;
2815
2816 /*
2817 * The PI code calls rt_mutex_setprio() with ->pi_lock held to
2818 * adjust the effective priority of a task. As a result,
2819 * rt_mutex_setprio() can trigger (RT) balancing operations,
2820 * which can then trigger wakeups of the stop thread to push
2821 * around the current task.
2822 *
2823 * The stop task itself will never be part of the PI-chain, it
2824 * never blocks, therefore that ->pi_lock recursion is safe.
2825 * Tell lockdep about this by placing the stop->pi_lock in its
2826 * own class.
2827 */
2828 lockdep_set_class(&stop->pi_lock, &stop_pi_lock);
2393 }2829 }
2394 2830
2395 cpu_rq(cpu)->stop = stop;2831 cpu_rq(cpu)->stop = stop;
@@ -2406,13 +2842,25 @@ void sched_set_stop_task(int cpu, struct task_struct *stop) · 10 unmodified lines
2406#else2842#else
2407 2843
2408static inline int __set_cpus_allowed_ptr(struct task_struct *p,2844static inline int __set_cpus_allowed_ptr(struct task_struct *p,
2409 const struct cpumask *new_mask, bool check)2845 const struct cpumask *new_mask,
2846 u32 flags)
2410{2847{
2411 return set_cpus_allowed_ptr(p, new_mask);2848 return set_cpus_allowed_ptr(p, new_mask);
2412}2849}
2413 2850
2414#endif /* CONFIG_SMP */2851#endif /* CONFIG_SMP */
2415 2852
2853#if !defined(CONFIG_SMP) || !defined(CONFIG_PREEMPT_RT)
2854
2855static inline void migrate_disable_switch(struct rq *rq, struct task_struct *p) { }
2856
2857static inline bool rq_has_pinned_tasks(struct rq *rq)
2858{
2859 return false;
2860}
2861
2862#endif
2863
2416static void2864static void
2417ttwu_stat(struct task_struct *p, int cpu, int wake_flags)2865ttwu_stat(struct task_struct *p, int cpu, int wake_flags)
2418{2866{
@@ -3098,6 +3546,7 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p) · 679 unmodified lines
3098 init_numa_balancing(clone_flags, p);3546 init_numa_balancing(clone_flags, p);
3099#ifdef CONFIG_SMP3547#ifdef CONFIG_SMP
3100 p->wake_entry.u_flags = CSD_TYPE_TTWU;3548 p->wake_entry.u_flags = CSD_TYPE_TTWU;
3549 p->migration_pending = NULL;
3101#endif3550#endif
3102}3551}
3103 3552
@@ -3485,6 +3934,90 @@ static inline void finish_task(struct task_struct *prev) · 381 unmodified lines
3485#endif3934#endif
3486}3935}
3487 3936
3937#ifdef CONFIG_SMP
3938
3939static void do_balance_callbacks(struct rq *rq, struct callback_head *head)
3940{
3941 void (*func)(struct rq *rq);
3942 struct callback_head *next;
3943
3944 lockdep_assert_held(&rq->lock);
3945
3946 while (head) {
3947 func = (void (*)(struct rq *))head->func;
3948 next = head->next;
3949 head->next = NULL;
3950 head = next;
3951
3952 func(rq);
3953 }
3954}
3955
3956static inline struct callback_head *splice_balance_callbacks(struct rq *rq)
3957{
3958 struct callback_head *head = rq->balance_callback;
3959
3960 lockdep_assert_held(&rq->lock);
3961 if (head) {
3962 rq->balance_callback = NULL;
3963 rq->balance_flags &= ~BALANCE_WORK;
3964 }
3965
3966 return head;
3967}
3968
3969static void __balance_callbacks(struct rq *rq)
3970{
3971 do_balance_callbacks(rq, splice_balance_callbacks(rq));
3972}
3973
3974static inline void balance_callbacks(struct rq *rq, struct callback_head *head)
3975{
3976 unsigned long flags;
3977
3978 if (unlikely(head)) {
3979 raw_spin_lock_irqsave(&rq->lock, flags);
3980 do_balance_callbacks(rq, head);
3981 raw_spin_unlock_irqrestore(&rq->lock, flags);
3982 }
3983}
3984
3985static void balance_push(struct rq *rq);
3986
3987static inline void balance_switch(struct rq *rq)
3988{
3989 if (likely(!rq->balance_flags))
3990 return;
3991
3992 if (rq->balance_flags & BALANCE_PUSH) {
3993 balance_push(rq);
3994 return;
3995 }
3996
3997 __balance_callbacks(rq);
3998}
3999
4000#else
4001
4002static inline void __balance_callbacks(struct rq *rq)
4003{
4004}
4005
4006static inline struct callback_head *splice_balance_callbacks(struct rq *rq)
4007{
4008 return NULL;
4009}
4010
4011static inline void balance_callbacks(struct rq *rq, struct callback_head *head)
4012{
4013}
4014
4015static inline void balance_switch(struct rq *rq)
4016{
4017}
4018
4019#endif
4020
3488static inline void4021static inline void
3489prepare_lock_switch(struct rq *rq, struct task_struct *next, struct rq_flags *rf)4022prepare_lock_switch(struct rq *rq, struct task_struct *next, struct rq_flags *rf)
3490{4023{
@@ -3510,6 +4043,7 @@ static inline void finish_lock_switch(struct rq *rq) · 19 unmodified lines
3510 * prev into current:4043 * prev into current:
3511 */4044 */
3512 spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_);4045 spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_);
4046 balance_switch(rq);
3513 raw_spin_unlock_irq(&rq->lock);4047 raw_spin_unlock_irq(&rq->lock);
3514}4048}
3515 4049
@@ -3651,43 +4185,6 @@ static struct rq *finish_task_switch(struct task_struct *prev) · 135 unmodified lines
3651 return rq;4185 return rq;
3652}4186}
3653 4187
3654#ifdef CONFIG_SMP
3655
3656/* rq->lock is NOT held, but preemption is disabled */
3657static void __balance_callback(struct rq *rq)
3658{
3659 struct callback_head *head, *next;
3660 void (*func)(struct rq *rq);
3661 unsigned long flags;
3662
3663 raw_spin_lock_irqsave(&rq->lock, flags);
3664 head = rq->balance_callback;
3665 rq->balance_callback = NULL;
3666 while (head) {
3667 func = (void (*)(struct rq *))head->func;
3668 next = head->next;
3669 head->next = NULL;
3670 head = next;
3671
3672 func(rq);
3673 }
3674 raw_spin_unlock_irqrestore(&rq->lock, flags);
3675}
3676
3677static inline void balance_callback(struct rq *rq)
3678{
3679 if (unlikely(rq->balance_callback))
3680 __balance_callback(rq);
3681}
3682
3683#else
3684
3685static inline void balance_callback(struct rq *rq)
3686{
3687}
3688
3689#endif
3690
3691/**4188/**
3692 * schedule_tail - first thing a freshly forked thread must call.4189 * schedule_tail - first thing a freshly forked thread must call.
3693 * @prev: the thread we just switched away from.4190 * @prev: the thread we just switched away from.
@@ -3707,7 +4204,6 @@ asmlinkage __visible void schedule_tail(struct task_struct *prev) · 13 unmodified lines
3707 */4204 */
3708 4205
3709 rq = finish_task_switch(prev);4206 rq = finish_task_switch(prev);
3710 balance_callback(rq);
3711 preempt_enable();4207 preempt_enable();
3712 4208
3713 if (current->set_child_tid)4209 if (current->set_child_tid)
@@ -4515,6 +5011,7 @@ static void __sched notrace __schedule(bool preempt) · 801 unmodified lines
4515 */5011 */
4516 ++*switch_count;5012 ++*switch_count;
4517 5013
5014 migrate_disable_switch(rq, prev);
4518 psi_sched_switch(prev, next, !task_on_rq_queued(prev));5015 psi_sched_switch(prev, next, !task_on_rq_queued(prev));
4519 5016
4520 trace_sched_switch(preempt, prev, next);5017 trace_sched_switch(preempt, prev, next);
@@ -4523,10 +5020,11 @@ static void __sched notrace __schedule(bool preempt) · 2 unmodified lines
4523 rq = context_switch(rq, prev, next, &rf);5020 rq = context_switch(rq, prev, next, &rf);
4524 } else {5021 } else {
4525 rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP);5022 rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP);
4526 rq_unlock_irq(rq, &rf);
4527 }
4528 5023
4529 balance_callback(rq);5024 rq_unpin_lock(rq, &rf);
5025 __balance_callbacks(rq);
5026 raw_spin_unlock_irq(&rq->lock);
5027 }
4530}5028}
4531 5029
4532void __noreturn do_task_dead(void)5030void __noreturn do_task_dead(void)
@@ -4937,9 +5435,11 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) · 404 unmodified lines
4937out_unlock:5435out_unlock:
4938 /* Avoid rq from going away on us: */5436 /* Avoid rq from going away on us: */
4939 preempt_disable();5437 preempt_disable();
4940 __task_rq_unlock(rq, &rf);
4941 5438
4942 balance_callback(rq);5439 rq_unpin_lock(rq, &rf);
5440 __balance_callbacks(rq);
5441 raw_spin_unlock(&rq->lock);
5442
4943 preempt_enable();5443 preempt_enable();
4944}5444}
4945#else5445#else
@@ -5213,6 +5713,7 @@ static int __sched_setscheduler(struct task_struct *p, · 267 unmodified lines
5213 int retval, oldprio, oldpolicy = -1, queued, running;5713 int retval, oldprio, oldpolicy = -1, queued, running;
5214 int new_effective_prio, policy = attr->sched_policy;5714 int new_effective_prio, policy = attr->sched_policy;
5215 const struct sched_class *prev_class;5715 const struct sched_class *prev_class;
5716 struct callback_head *head;
5216 struct rq_flags rf;5717 struct rq_flags rf;
5217 int reset_on_fork;5718 int reset_on_fork;
5218 int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK;5719 int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK;
@@ -5451,6 +5952,7 @@ change: · 232 unmodified lines
5451 5952
5452 /* Avoid rq from going away on us: */5953 /* Avoid rq from going away on us: */
5453 preempt_disable();5954 preempt_disable();
5955 head = splice_balance_callbacks(rq);
5454 task_rq_unlock(rq, p, &rf);5956 task_rq_unlock(rq, p, &rf);
5455 5957
5456 if (pi) {5958 if (pi) {
@@ -5459,7 +5961,7 @@ change: · 2 unmodified lines
5459 }5961 }
5460 5962
5461 /* Run balance callbacks after we've adjusted the PI chain: */5963 /* Run balance callbacks after we've adjusted the PI chain: */
5462 balance_callback(rq);5964 balance_callbacks(rq, head);
5463 preempt_enable();5965 preempt_enable();
5464 5966
5465 return 0;5967 return 0;
@@ -5954,7 +6456,7 @@ long sched_setaffinity(pid_t pid, const struct cpumask *in_mask) · 488 unmodified lines
5954 }6456 }
5955#endif6457#endif
5956again:6458again:
5957 retval = __set_cpus_allowed_ptr(p, new_mask, true);6459 retval = __set_cpus_allowed_ptr(p, new_mask, SCA_CHECK);
5958 6460
5959 if (!retval) {6461 if (!retval) {
5960 cpuset_cpus_allowed(p, cpus_allowed);6462 cpuset_cpus_allowed(p, cpus_allowed);
@@ -6443,6 +6945,7 @@ void sched_show_task(struct task_struct *p) · 482 unmodified lines
6443 (unsigned long)task_thread_info(p)->flags);6945 (unsigned long)task_thread_info(p)->flags);
6444 6946
6445 print_worker_info(KERN_INFO, p);6947 print_worker_info(KERN_INFO, p);
6948 print_stop_info(KERN_INFO, p);
6446 show_stack(p, NULL, KERN_INFO);6949 show_stack(p, NULL, KERN_INFO);
6447 put_task_stack(p);6950 put_task_stack(p);
6448}6951}
@@ -6533,7 +7036,7 @@ void init_idle(struct task_struct *idle, int cpu) · 84 unmodified lines
6533 *7036 *
6534 * And since this is boot we can forgo the serialization.7037 * And since this is boot we can forgo the serialization.
6535 */7038 */
6536 set_cpus_allowed_common(idle, cpumask_of(cpu));7039 set_cpus_allowed_common(idle, cpumask_of(cpu), 0);
6537#endif7040#endif
6538 /*7041 /*
6539 * We're having a chicken and egg problem, even though we are7042 * We're having a chicken and egg problem, even though we are
@@ -6684,119 +7187,126 @@ void idle_task_exit(void) · 144 unmodified lines
6684 /* finish_cpu(), as ran on the BP, will clean up the active_mm state */7187 /* finish_cpu(), as ran on the BP, will clean up the active_mm state */
6685}7188}
6686 7189
6687/*7190static int __balance_push_cpu_stop(void *arg)
6688 * Since this CPU is going 'away' for a while, fold any nr_active delta
6689 * we might have. Assumes we're called after migrate_tasks() so that the
6690 * nr_active count is stable. We need to take the teardown thread which
6691 * is calling this into account, so we hand in adjust = 1 to the load
6692 * calculation.
6693 *
6694 * Also see the comment "Global load-average calculations".
6695 */
6696static void calc_load_migrate(struct rq *rq)
6697{7191{
6698 long delta = calc_load_fold_active(rq, 1);7192 struct task_struct *p = arg;
6699 if (delta)7193 struct rq *rq = this_rq();
6700 atomic_long_add(delta, &calc_load_tasks);7194 struct rq_flags rf;
6701}7195 int cpu;
6702 7196
6703static struct task_struct *__pick_migrate_task(struct rq *rq)7197 raw_spin_lock_irq(&p->pi_lock);
6704{7198 rq_lock(rq, &rf);
6705 const struct sched_class *class;
6706 struct task_struct *next;
6707 7199
6708 for_each_class(class) {7200 update_rq_clock(rq);
6709 next = class->pick_next_task(rq);7201
6710 if (next) {7202 if (task_rq(p) == rq && task_on_rq_queued(p)) {
6711 next->sched_class->put_prev_task(rq, next);7203 cpu = select_fallback_rq(rq->cpu, p);
6712 return next;7204 rq = __migrate_task(rq, &rf, p, cpu);
6713 }
6714 }7205 }
6715 7206
6716 /* The idle class should always have a runnable task */7207 rq_unlock(rq, &rf);
6717 BUG();7208 raw_spin_unlock_irq(&p->pi_lock);
7209
7210 put_task_struct(p);
7211
7212 return 0;
6718}7213}
6719 7214
7215static DEFINE_PER_CPU(struct cpu_stop_work, push_work);
7216
6720/*7217/*
6721 * Migrate all tasks from the rq, sleeping tasks will be migrated by7218 * Ensure we only run per-cpu kthreads once the CPU goes !active.
6722 * try_to_wake_up()->select_task_rq().
6723 *
6724 * Called with rq->lock held even though we'er in stop_machine() and
6725 * there's no concurrency possible, we hold the required locks anyway
6726 * because of lock validation efforts.
6727 */7219 */
6728static void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf)7220static void balance_push(struct rq *rq)
6729{7221{
6730 struct rq *rq = dead_rq;7222 struct task_struct *push_task = rq->curr;
6731 struct task_struct *next, *stop = rq->stop;7223
6732 struct rq_flags orf = *rf;7224 lockdep_assert_held(&rq->lock);
6733 int dest_cpu;7225 SCHED_WARN_ON(rq->cpu != smp_processor_id());
6734 7226
6735 /*7227 /*
6736 * Fudge the rq selection such that the below task selection loop7228 * Both the cpu-hotplug and stop task are in this case and are
6737 * doesn't get stuck on the currently eligible stop task.7229 * required to complete the hotplug process.
6738 *
6739 * We're currently inside stop_machine() and the rq is either stuck
6740 * in the stop_machine_cpu_stop() loop, or we're executing this code,
6741 * either way we should never end up calling schedule() until we're
6742 * done here.
6743 */7230 */
6744 rq->stop = NULL;7231 if (is_per_cpu_kthread(push_task) || is_migration_disabled(push_task)) {
7232 /*
7233 * If this is the idle task on the outgoing CPU try to wake
7234 * up the hotplug control thread which might wait for the
7235 * last task to vanish. The rcuwait_active() check is
7236 * accurate here because the waiter is pinned on this CPU
7237 * and can't obviously be running in parallel.
7238 *
7239 * On RT kernels this also has to check whether there are
7240 * pinned and scheduled out tasks on the runqueue. They
7241 * need to leave the migrate disabled section first.
7242 */
7243 if (!rq->nr_running && !rq_has_pinned_tasks(rq) &&
7244 rcuwait_active(&rq->hotplug_wait)) {
7245 raw_spin_unlock(&rq->lock);
7246 rcuwait_wake_up(&rq->hotplug_wait);
7247 raw_spin_lock(&rq->lock);
7248 }
7249 return;
7250 }
6745 7251
7252 get_task_struct(push_task);
6746 /*7253 /*
6747 * put_prev_task() and pick_next_task() sched7254 * Temporarily drop rq->lock such that we can wake-up the stop task.
6748 * class method both need to have an up-to-date7255 * Both preemption and IRQs are still disabled.
6749 * value of rq->clock[_task]
6750 */7256 */
6751 update_rq_clock(rq);7257 raw_spin_unlock(&rq->lock);
7258 stop_one_cpu_nowait(rq->cpu, __balance_push_cpu_stop, push_task,
7259 this_cpu_ptr(&push_work));
7260 /*
7261 * At this point need_resched() is true and we'll take the loop in
7262 * schedule(). The next pick is obviously going to be the stop task
7263 * which is_per_cpu_kthread() and will push this task away.
7264 */
7265 raw_spin_lock(&rq->lock);
7266}
6752 7267
6753 for (;;) {7268static void balance_push_set(int cpu, bool on)
6754 /*7269{
6755 * There's this thread running, bail when that's the only7270 struct rq *rq = cpu_rq(cpu);
6756 * remaining thread:7271 struct rq_flags rf;
6757 */
6758 if (rq->nr_running == 1)
6759 break;
6760 7272
6761 next = __pick_migrate_task(rq);7273 rq_lock_irqsave(rq, &rf);
7274 if (on)
7275 rq->balance_flags |= BALANCE_PUSH;
7276 else
7277 rq->balance_flags &= ~BALANCE_PUSH;
7278 rq_unlock_irqrestore(rq, &rf);
7279}
6762 7280
6763 /*7281/*
6764 * Rules for changing task_struct::cpus_mask are holding7282 * Invoked from a CPUs hotplug control thread after the CPU has been marked
6765 * both pi_lock and rq->lock, such that holding either7283 * inactive. All tasks which are not per CPU kernel threads are either
6766 * stabilizes the mask.7284 * pushed off this CPU now via balance_push() or placed on a different CPU
6767 *7285 * during wakeup. Wait until the CPU is quiescent.
6768 * Drop rq->lock is not quite as disastrous as it usually is7286 */
6769 * because !cpu_active at this point, which means load-balance7287static void balance_hotplug_wait(void)
6770 * will not interfere. Also, stop-machine.7288{
6771 */7289 struct rq *rq = this_rq();
6772 rq_unlock(rq, rf);
6773 raw_spin_lock(&next->pi_lock);
6774 rq_relock(rq, rf);
6775 7290
6776 /*7291 rcuwait_wait_event(&rq->hotplug_wait,
6777 * Since we're inside stop-machine, _nothing_ should have7292 rq->nr_running == 1 && !rq_has_pinned_tasks(rq),
6778 * changed the task, WARN if weird stuff happened, because in7293 TASK_UNINTERRUPTIBLE);
6779 * that case the above rq->lock drop is a fail too.7294}
6780 */
6781 if (WARN_ON(task_rq(next) != rq || !task_on_rq_queued(next))) {
6782 raw_spin_unlock(&next->pi_lock);
6783 continue;
6784 }
6785 7295
6786 /* Find suitable destination for @next, with force if needed. */7296#else
6787 dest_cpu = select_fallback_rq(dead_rq->cpu, next);
6788 rq = __migrate_task(rq, rf, next, dest_cpu);
6789 if (rq != dead_rq) {
6790 rq_unlock(rq, rf);
6791 rq = dead_rq;
6792 *rf = orf;
6793 rq_relock(rq, rf);
6794 }
6795 raw_spin_unlock(&next->pi_lock);
6796 }
6797 7297
6798 rq->stop = stop;7298static inline void balance_push(struct rq *rq)
7299{
6799}7300}
7301
7302static inline void balance_push_set(int cpu, bool on)
7303{
7304}
7305
7306static inline void balance_hotplug_wait(void)
7307{
7308}
7309
6800#endif /* CONFIG_HOTPLUG_CPU */7310#endif /* CONFIG_HOTPLUG_CPU */
6801 7311
6802void set_rq_online(struct rq *rq)7312void set_rq_online(struct rq *rq)
@@ -6882,6 +7392,8 @@ int sched_cpu_activate(unsigned int cpu) · 79 unmodified lines
6882 struct rq *rq = cpu_rq(cpu);7392 struct rq *rq = cpu_rq(cpu);
6883 struct rq_flags rf;7393 struct rq_flags rf;
6884 7394
7395 balance_push_set(cpu, false);
7396
6885#ifdef CONFIG_SCHED_SMT7397#ifdef CONFIG_SCHED_SMT
6886 /*7398 /*
6887 * When going up, increment the number of cores with SMT present.7399 * When going up, increment the number of cores with SMT present.
@@ -6917,6 +7429,8 @@ int sched_cpu_activate(unsigned int cpu) · 29 unmodified lines
6917 7429
6918int sched_cpu_deactivate(unsigned int cpu)7430int sched_cpu_deactivate(unsigned int cpu)
6919{7431{
7432 struct rq *rq = cpu_rq(cpu);
7433 struct rq_flags rf;
6920 int ret;7434 int ret;
6921 7435
6922 set_cpu_active(cpu, false);7436 set_cpu_active(cpu, false);
@@ -6929,6 +7443,16 @@ int sched_cpu_deactivate(unsigned int cpu) · 6 unmodified lines
6929 */7443 */
6930 synchronize_rcu();7444 synchronize_rcu();
6931 7445
7446 balance_push_set(cpu, true);
7447
7448 rq_lock_irqsave(rq, &rf);
7449 if (rq->rd) {
7450 update_rq_clock(rq);
7451 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
7452 set_rq_offline(rq);
7453 }
7454 rq_unlock_irqrestore(rq, &rf);
7455
6932#ifdef CONFIG_SCHED_SMT7456#ifdef CONFIG_SCHED_SMT
6933 /*7457 /*
6934 * When going down, decrement the number of cores with SMT present.7458 * When going down, decrement the number of cores with SMT present.
@@ -6942,6 +7466,7 @@ int sched_cpu_deactivate(unsigned int cpu) · 7 unmodified lines
6942 7466
6943 ret = cpuset_cpu_inactive(cpu);7467 ret = cpuset_cpu_inactive(cpu);
6944 if (ret) {7468 if (ret) {
7469 balance_push_set(cpu, false);
6945 set_cpu_active(cpu, true);7470 set_cpu_active(cpu, true);
6946 return ret;7471 return ret;
6947 }7472 }
@@ -6965,6 +7490,41 @@ int sched_cpu_starting(unsigned int cpu) · 17 unmodified lines
6965}7490}
6966 7491
6967#ifdef CONFIG_HOTPLUG_CPU7492#ifdef CONFIG_HOTPLUG_CPU
7493
7494/*
7495 * Invoked immediately before the stopper thread is invoked to bring the
7496 * CPU down completely. At this point all per CPU kthreads except the
7497 * hotplug thread (current) and the stopper thread (inactive) have been
7498 * either parked or have been unbound from the outgoing CPU. Ensure that
7499 * any of those which might be on the way out are gone.
7500 *
7501 * If after this point a bound task is being woken on this CPU then the
7502 * responsible hotplug callback has failed to do it's job.
7503 * sched_cpu_dying() will catch it with the appropriate fireworks.
7504 */
7505int sched_cpu_wait_empty(unsigned int cpu)
7506{
7507 balance_hotplug_wait();
7508 return 0;
7509}
7510
7511/*
7512 * Since this CPU is going 'away' for a while, fold any nr_active delta we
7513 * might have. Called from the CPU stopper task after ensuring that the
7514 * stopper is the last running task on the CPU, so nr_active count is
7515 * stable. We need to take the teardown thread which is calling this into
7516 * account, so we hand in adjust = 1 to the load calculation.
7517 *
7518 * Also see the comment "Global load-average calculations".
7519 */
7520static void calc_load_migrate(struct rq *rq)
7521{
7522 long delta = calc_load_fold_active(rq, 1);
7523
7524 if (delta)
7525 atomic_long_add(delta, &calc_load_tasks);
7526}
7527
6968int sched_cpu_dying(unsigned int cpu)7528int sched_cpu_dying(unsigned int cpu)
6969{7529{
6970 struct rq *rq = cpu_rq(cpu);7530 struct rq *rq = cpu_rq(cpu);
@@ -6974,12 +7534,7 @@ int sched_cpu_dying(unsigned int cpu) · 3 unmodified lines
6974 sched_tick_stop(cpu);7534 sched_tick_stop(cpu);
6975 7535
6976 rq_lock_irqsave(rq, &rf);7536 rq_lock_irqsave(rq, &rf);
6977 if (rq->rd) {7537 BUG_ON(rq->nr_running != 1 || rq_has_pinned_tasks(rq));
6978 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
6979 set_rq_offline(rq);
6980 }
6981 migrate_tasks(rq, &rf);
6982 BUG_ON(rq->nr_running != 1);
6983 rq_unlock_irqrestore(rq, &rf);7538 rq_unlock_irqrestore(rq, &rf);
6984 7539
6985 calc_load_migrate(rq);7540 calc_load_migrate(rq);
@@ -7186,6 +7741,9 @@ void __init sched_init(void) · 200 unmodified lines
7186 7741
7187 rq_csd_init(rq, &rq->nohz_csd, nohz_csd_func);7742 rq_csd_init(rq, &rq->nohz_csd, nohz_csd_func);
7188#endif7743#endif
7744#ifdef CONFIG_HOTPLUG_CPU
7745 rcuwait_init(&rq->hotplug_wait);
7746#endif
7189#endif /* CONFIG_SMP */7747#endif /* CONFIG_SMP */
7190 hrtick_rq_init(rq);7748 hrtick_rq_init(rq);
7191 atomic_set(&rq->nr_iowait, 0);7749 atomic_set(&rq->nr_iowait, 0);
kernel/sched/cpudeadline.c+2−2
@@ -120,7 +120,7 @@ int cpudl_find(struct cpudl *cp, struct task_struct *p, · 119 unmodified lines
120 const struct sched_dl_entity *dl_se = &p->dl;120 const struct sched_dl_entity *dl_se = &p->dl;
121 121
122 if (later_mask &&122 if (later_mask &&
123 cpumask_and(later_mask, cp->free_cpus, p->cpus_ptr)) {123 cpumask_and(later_mask, cp->free_cpus, &p->cpus_mask)) {
124 unsigned long cap, max_cap = 0;124 unsigned long cap, max_cap = 0;
125 int cpu, max_cpu = -1;125 int cpu, max_cpu = -1;
126 126
@@ -151,7 +151,7 @@ int cpudl_find(struct cpudl *cp, struct task_struct *p, · 24 unmodified lines
151 151
152 WARN_ON(best_cpu != -1 && !cpu_present(best_cpu));152 WARN_ON(best_cpu != -1 && !cpu_present(best_cpu));
153 153
154 if (cpumask_test_cpu(best_cpu, p->cpus_ptr) &&154 if (cpumask_test_cpu(best_cpu, &p->cpus_mask) &&
155 dl_time_before(dl_se->deadline, cp->elements[0].dl)) {155 dl_time_before(dl_se->deadline, cp->elements[0].dl)) {
156 if (later_mask)156 if (later_mask)
157 cpumask_set_cpu(best_cpu, later_mask);157 cpumask_set_cpu(best_cpu, later_mask);
kernel/sched/cpupri.c+2−2
@@ -97,11 +97,11 @@ static inline int __cpupri_find(struct cpupri *cp, struct task_struct *p, · 96 unmodified lines
97 if (skip)97 if (skip)
98 return 0;98 return 0;
99 99
100 if (cpumask_any_and(p->cpus_ptr, vec->mask) >= nr_cpu_ids)100 if (cpumask_any_and(&p->cpus_mask, vec->mask) >= nr_cpu_ids)
101 return 0;101 return 0;
102 102
103 if (lowest_mask) {103 if (lowest_mask) {
104 cpumask_and(lowest_mask, p->cpus_ptr, vec->mask);104 cpumask_and(lowest_mask, &p->cpus_mask, vec->mask);
105 105
106 /*106 /*
107 * We have to ensure that we have at least one bit107 * We have to ensure that we have at least one bit
kernel/sched/deadline.c+31−15
@@ -559,7 +559,7 @@ static int push_dl_task(struct rq *rq); · 558 unmodified lines
559 559
560static inline bool need_pull_dl_task(struct rq *rq, struct task_struct *prev)560static inline bool need_pull_dl_task(struct rq *rq, struct task_struct *prev)
561{561{
562 return dl_task(prev);562 return rq->online && dl_task(prev);
563}563}
564 564
565static DEFINE_PER_CPU(struct callback_head, dl_push_head);565static DEFINE_PER_CPU(struct callback_head, dl_push_head);
@@ -1931,7 +1931,7 @@ static void task_fork_dl(struct task_struct *p) · 1365 unmodified lines
1931static int pick_dl_task(struct rq *rq, struct task_struct *p, int cpu)1931static int pick_dl_task(struct rq *rq, struct task_struct *p, int cpu)
1932{1932{
1933 if (!task_running(rq, p) &&1933 if (!task_running(rq, p) &&
1934 cpumask_test_cpu(cpu, p->cpus_ptr))1934 cpumask_test_cpu(cpu, &p->cpus_mask))
1935 return 1;1935 return 1;
1936 return 0;1936 return 0;
1937}1937}
@@ -2021,8 +2021,8 @@ static int find_later_rq(struct task_struct *task) · 83 unmodified lines
2021 return this_cpu;2021 return this_cpu;
2022 }2022 }
2023 2023
2024 best_cpu = cpumask_first_and(later_mask,2024 best_cpu = cpumask_any_and_distribute(later_mask,
2025 sched_domain_span(sd));2025 sched_domain_span(sd));
2026 /*2026 /*
2027 * Last chance: if a CPU being in both later_mask2027 * Last chance: if a CPU being in both later_mask
2028 * and current sd span is valid, that becomes our2028 * and current sd span is valid, that becomes our
@@ -2044,7 +2044,7 @@ static int find_later_rq(struct task_struct *task) · 15 unmodified lines
2044 if (this_cpu != -1)2044 if (this_cpu != -1)
2045 return this_cpu;2045 return this_cpu;
2046 2046
2047 cpu = cpumask_any(later_mask);2047 cpu = cpumask_any_distribute(later_mask);
2048 if (cpu < nr_cpu_ids)2048 if (cpu < nr_cpu_ids)
2049 return cpu;2049 return cpu;
2050 2050
@@ -2081,7 +2081,7 @@ static struct rq *find_lock_later_rq(struct task_struct *task, struct rq *rq) · 30 unmodified lines
2081 /* Retry if something changed. */2081 /* Retry if something changed. */
2082 if (double_lock_balance(rq, later_rq)) {2082 if (double_lock_balance(rq, later_rq)) {
2083 if (unlikely(task_rq(task) != rq ||2083 if (unlikely(task_rq(task) != rq ||
2084 !cpumask_test_cpu(later_rq->cpu, task->cpus_ptr) ||2084 !cpumask_test_cpu(later_rq->cpu, &task->cpus_mask) ||
2085 task_running(rq, task) ||2085 task_running(rq, task) ||
2086 !dl_task(task) ||2086 !dl_task(task) ||
2087 !task_on_rq_queued(task))) {2087 !task_on_rq_queued(task))) {
@@ -2148,6 +2148,9 @@ static int push_dl_task(struct rq *rq) · 60 unmodified lines
2148 return 0;2148 return 0;
2149 2149
2150retry:2150retry:
2151 if (is_migration_disabled(next_task))
2152 return 0;
2153
2151 if (WARN_ON(next_task == rq->curr))2154 if (WARN_ON(next_task == rq->curr))
2152 return 0;2155 return 0;
2153 2156
@@ -2225,7 +2228,7 @@ static void push_dl_tasks(struct rq *rq) · 71 unmodified lines
2225static void pull_dl_task(struct rq *this_rq)2228static void pull_dl_task(struct rq *this_rq)
2226{2229{
2227 int this_cpu = this_rq->cpu, cpu;2230 int this_cpu = this_rq->cpu, cpu;
2228 struct task_struct *p;2231 struct task_struct *p, *push_task;
2229 bool resched = false;2232 bool resched = false;
2230 struct rq *src_rq;2233 struct rq *src_rq;
2231 u64 dmin = LONG_MAX;2234 u64 dmin = LONG_MAX;
@@ -2255,6 +2258,7 @@ static void pull_dl_task(struct rq *this_rq) · 23 unmodified lines
2255 continue;2258 continue;
2256 2259
2257 /* Might drop this_rq->lock */2260 /* Might drop this_rq->lock */
2261 push_task = NULL;
2258 double_lock_balance(this_rq, src_rq);2262 double_lock_balance(this_rq, src_rq);
2259 2263
2260 /*2264 /*
@@ -2286,17 +2290,27 @@ static void pull_dl_task(struct rq *this_rq) · 25 unmodified lines
2286 src_rq->curr->dl.deadline))2290 src_rq->curr->dl.deadline))
2287 goto skip;2291 goto skip;
2288 2292
2289 resched = true;2293 if (is_migration_disabled(p)) {
2290 2294 push_task = get_push_task(src_rq);
2291 deactivate_task(src_rq, p, 0);2295 } else {
2292 set_task_cpu(p, this_cpu);2296 deactivate_task(src_rq, p, 0);
2293 activate_task(this_rq, p, 0);2297 set_task_cpu(p, this_cpu);
2294 dmin = p->dl.deadline;2298 activate_task(this_rq, p, 0);
2299 dmin = p->dl.deadline;
2300 resched = true;
2301 }
2295 2302
2296 /* Is there any other task even earlier? */2303 /* Is there any other task even earlier? */
2297 }2304 }
2298skip:2305skip:
2299 double_unlock_balance(this_rq, src_rq);2306 double_unlock_balance(this_rq, src_rq);
2307
2308 if (push_task) {
2309 raw_spin_unlock(&this_rq->lock);
2310 stop_one_cpu_nowait(src_rq->cpu, push_cpu_stop,
2311 push_task, &src_rq->push_work);
2312 raw_spin_lock(&this_rq->lock);
2313 }
2300 }2314 }
2301 2315
2302 if (resched)2316 if (resched)
@@ -2320,7 +2334,8 @@ static void task_woken_dl(struct rq *rq, struct task_struct *p) · 17 unmodified lines
2320}2334}
2321 2335
2322static void set_cpus_allowed_dl(struct task_struct *p,2336static void set_cpus_allowed_dl(struct task_struct *p,
2323 const struct cpumask *new_mask)2337 const struct cpumask *new_mask,
2338 u32 flags)
2324{2339{
2325 struct root_domain *src_rd;2340 struct root_domain *src_rd;
2326 struct rq *rq;2341 struct rq *rq;
@@ -2349,7 +2364,7 @@ static void set_cpus_allowed_dl(struct task_struct *p, · 22 unmodified lines
2349 raw_spin_unlock(&src_dl_b->lock);2364 raw_spin_unlock(&src_dl_b->lock);
2350 }2365 }
2351 2366
2352 set_cpus_allowed_common(p, new_mask);2367 set_cpus_allowed_common(p, new_mask, flags);
2353}2368}
2354 2369
2355/* Assumes rq->lock is held */2370/* Assumes rq->lock is held */
@@ -2542,6 +2557,7 @@ DEFINE_SCHED_CLASS(dl) = { · 186 unmodified lines
2542 .rq_online = rq_online_dl,2557 .rq_online = rq_online_dl,
2543 .rq_offline = rq_offline_dl,2558 .rq_offline = rq_offline_dl,
2544 .task_woken = task_woken_dl,2559 .task_woken = task_woken_dl,
2560 .find_lock_rq = find_lock_later_rq,
2545#endif2561#endif
2546 2562
2547 .task_tick = task_tick_dl,2563 .task_tick = task_tick_dl,
kernel/sched/rt.c+57−18
@@ -265,7 +265,7 @@ static void pull_rt_task(struct rq *this_rq); · 264 unmodified lines
265static inline bool need_pull_rt_task(struct rq *rq, struct task_struct *prev)265static inline bool need_pull_rt_task(struct rq *rq, struct task_struct *prev)
266{266{
267 /* Try to pull RT tasks here if we lower this rq's prio */267 /* Try to pull RT tasks here if we lower this rq's prio */
268 return rq->rt.highest_prio.curr > prev->prio;268 return rq->online && rq->rt.highest_prio.curr > prev->prio;
269}269}
270 270
271static inline int rt_overloaded(struct rq *rq)271static inline int rt_overloaded(struct rq *rq)
@@ -1660,7 +1660,7 @@ static void put_prev_task_rt(struct rq *rq, struct task_struct *p) · 1388 unmodified lines
1660static int pick_rt_task(struct rq *rq, struct task_struct *p, int cpu)1660static int pick_rt_task(struct rq *rq, struct task_struct *p, int cpu)
1661{1661{
1662 if (!task_running(rq, p) &&1662 if (!task_running(rq, p) &&
1663 cpumask_test_cpu(cpu, p->cpus_ptr))1663 cpumask_test_cpu(cpu, &p->cpus_mask))
1664 return 1;1664 return 1;
1665 1665
1666 return 0;1666 return 0;
@@ -1754,8 +1754,8 @@ static int find_lowest_rq(struct task_struct *task) · 87 unmodified lines
1754 return this_cpu;1754 return this_cpu;
1755 }1755 }
1756 1756
1757 best_cpu = cpumask_first_and(lowest_mask,1757 best_cpu = cpumask_any_and_distribute(lowest_mask,
1758 sched_domain_span(sd));1758 sched_domain_span(sd));
1759 if (best_cpu < nr_cpu_ids) {1759 if (best_cpu < nr_cpu_ids) {
1760 rcu_read_unlock();1760 rcu_read_unlock();
1761 return best_cpu;1761 return best_cpu;
@@ -1772,7 +1772,7 @@ static int find_lowest_rq(struct task_struct *task) · 10 unmodified lines
1772 if (this_cpu != -1)1772 if (this_cpu != -1)
1773 return this_cpu;1773 return this_cpu;
1774 1774
1775 cpu = cpumask_any(lowest_mask);1775 cpu = cpumask_any_distribute(lowest_mask);
1776 if (cpu < nr_cpu_ids)1776 if (cpu < nr_cpu_ids)
1777 return cpu;1777 return cpu;
1778 1778
@@ -1813,7 +1813,7 @@ static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq) · 34 unmodified lines
1813 * Also make sure that it wasn't scheduled on its rq.1813 * Also make sure that it wasn't scheduled on its rq.
1814 */1814 */
1815 if (unlikely(task_rq(task) != rq ||1815 if (unlikely(task_rq(task) != rq ||
1816 !cpumask_test_cpu(lowest_rq->cpu, task->cpus_ptr) ||1816 !cpumask_test_cpu(lowest_rq->cpu, &task->cpus_mask) ||
1817 task_running(rq, task) ||1817 task_running(rq, task) ||
1818 !rt_task(task) ||1818 !rt_task(task) ||
1819 !task_on_rq_queued(task))) {1819 !task_on_rq_queued(task))) {
@@ -1861,7 +1861,7 @@ static struct task_struct *pick_next_pushable_task(struct rq *rq) · 41 unmodified lines
1861 * running task can migrate over to a CPU that is running a task1861 * running task can migrate over to a CPU that is running a task
1862 * of lesser priority.1862 * of lesser priority.
1863 */1863 */
1864static int push_rt_task(struct rq *rq)1864static int push_rt_task(struct rq *rq, bool pull)
1865{1865{
1866 struct task_struct *next_task;1866 struct task_struct *next_task;
1867 struct rq *lowest_rq;1867 struct rq *lowest_rq;
@@ -1875,6 +1875,34 @@ static int push_rt_task(struct rq *rq) · 7 unmodified lines
1875 return 0;1875 return 0;
1876 1876
1877retry:1877retry:
1878 if (is_migration_disabled(next_task)) {
1879 struct task_struct *push_task = NULL;
1880 int cpu;
1881
1882 if (!pull || rq->push_busy)
1883 return 0;
1884
1885 cpu = find_lowest_rq(rq->curr);
1886 if (cpu == -1 || cpu == rq->cpu)
1887 return 0;
1888
1889 /*
1890 * Given we found a CPU with lower priority than @next_task,
1891 * therefore it should be running. However we cannot migrate it
1892 * to this other CPU, instead attempt to push the current
1893 * running task on this CPU away.
1894 */
1895 push_task = get_push_task(rq);
1896 if (push_task) {
1897 raw_spin_unlock(&rq->lock);
1898 stop_one_cpu_nowait(rq->cpu, push_cpu_stop,
1899 push_task, &rq->push_work);
1900 raw_spin_lock(&rq->lock);
1901 }
1902
1903 return 0;
1904 }
1905
1878 if (WARN_ON(next_task == rq->curr))1906 if (WARN_ON(next_task == rq->curr))
1879 return 0;1907 return 0;
1880 1908
@@ -1929,12 +1957,10 @@ retry: · 48 unmodified lines
1929 deactivate_task(rq, next_task, 0);1957 deactivate_task(rq, next_task, 0);
1930 set_task_cpu(next_task, lowest_rq->cpu);1958 set_task_cpu(next_task, lowest_rq->cpu);
1931 activate_task(lowest_rq, next_task, 0);1959 activate_task(lowest_rq, next_task, 0);
1932 ret = 1;
1933
1934 resched_curr(lowest_rq);1960 resched_curr(lowest_rq);
1961 ret = 1;
1935 1962
1936 double_unlock_balance(rq, lowest_rq);1963 double_unlock_balance(rq, lowest_rq);
1937
1938out:1964out:
1939 put_task_struct(next_task);1965 put_task_struct(next_task);
1940 1966
@@ -1944,7 +1970,7 @@ out: · 3 unmodified lines
1944static void push_rt_tasks(struct rq *rq)1970static void push_rt_tasks(struct rq *rq)
1945{1971{
1946 /* push_rt_task will return true if it moved an RT */1972 /* push_rt_task will return true if it moved an RT */
1947 while (push_rt_task(rq))1973 while (push_rt_task(rq, false))
1948 ;1974 ;
1949}1975}
1950 1976
@@ -2097,7 +2123,8 @@ void rto_push_irq_work_func(struct irq_work *work) · 146 unmodified lines
2097 */2123 */
2098 if (has_pushable_tasks(rq)) {2124 if (has_pushable_tasks(rq)) {
2099 raw_spin_lock(&rq->lock);2125 raw_spin_lock(&rq->lock);
2100 push_rt_tasks(rq);2126 while (push_rt_task(rq, true))
2127 ;
2101 raw_spin_unlock(&rq->lock);2128 raw_spin_unlock(&rq->lock);
2102 }2129 }
2103 2130
@@ -2122,7 +2149,7 @@ static void pull_rt_task(struct rq *this_rq) · 18 unmodified lines
2122{2149{
2123 int this_cpu = this_rq->cpu, cpu;2150 int this_cpu = this_rq->cpu, cpu;
2124 bool resched = false;2151 bool resched = false;
2125 struct task_struct *p;2152 struct task_struct *p, *push_task;
2126 struct rq *src_rq;2153 struct rq *src_rq;
2127 int rt_overload_count = rt_overloaded(this_rq);2154 int rt_overload_count = rt_overloaded(this_rq);
2128 2155
@@ -2169,6 +2196,7 @@ static void pull_rt_task(struct rq *this_rq) · 40 unmodified lines
2169 * double_lock_balance, and another CPU could2196 * double_lock_balance, and another CPU could
2170 * alter this_rq2197 * alter this_rq
2171 */2198 */
2199 push_task = NULL;
2172 double_lock_balance(this_rq, src_rq);2200 double_lock_balance(this_rq, src_rq);
2173 2201
2174 /*2202 /*
@@ -2196,11 +2224,14 @@ static void pull_rt_task(struct rq *this_rq) · 21 unmodified lines
2196 if (p->prio < src_rq->curr->prio)2224 if (p->prio < src_rq->curr->prio)
2197 goto skip;2225 goto skip;
2198 2226
2199 resched = true;2227 if (is_migration_disabled(p)) {
2200 2228 push_task = get_push_task(src_rq);
2201 deactivate_task(src_rq, p, 0);2229 } else {
2202 set_task_cpu(p, this_cpu);2230 deactivate_task(src_rq, p, 0);
2203 activate_task(this_rq, p, 0);2231 set_task_cpu(p, this_cpu);
2232 activate_task(this_rq, p, 0);
2233 resched = true;
2234 }
2204 /*2235 /*
2205 * We continue with the search, just in2236 * We continue with the search, just in
2206 * case there's an even higher prio task2237 * case there's an even higher prio task
@@ -2210,6 +2241,13 @@ static void pull_rt_task(struct rq *this_rq) · 3 unmodified lines
2210 }2241 }
2211skip:2242skip:
2212 double_unlock_balance(this_rq, src_rq);2243 double_unlock_balance(this_rq, src_rq);
2244
2245 if (push_task) {
2246 raw_spin_unlock(&this_rq->lock);
2247 stop_one_cpu_nowait(src_rq->cpu, push_cpu_stop,
2248 push_task, &src_rq->push_work);
2249 raw_spin_lock(&this_rq->lock);
2250 }
2213 }2251 }
2214 2252
2215 if (resched)2253 if (resched)
@@ -2451,6 +2489,7 @@ DEFINE_SCHED_CLASS(rt) = { · 235 unmodified lines
2451 .rq_offline = rq_offline_rt,2489 .rq_offline = rq_offline_rt,
2452 .task_woken = task_woken_rt,2490 .task_woken = task_woken_rt,
2453 .switched_from = switched_from_rt,2491 .switched_from = switched_from_rt,
2492 .find_lock_rq = find_lock_lowest_rq,
2454#endif2493#endif
2455 2494
2456 .task_tick = task_tick_rt,2495 .task_tick = task_tick_rt,
kernel/sched/sched.h+56−3
@@ -975,6 +975,7 @@ struct rq { · 974 unmodified lines
975 unsigned long cpu_capacity_orig;975 unsigned long cpu_capacity_orig;
976 976
977 struct callback_head *balance_callback;977 struct callback_head *balance_callback;
978 unsigned char balance_flags;
978 979
979 unsigned char nohz_idle_balance;980 unsigned char nohz_idle_balance;
980 unsigned char idle_balance;981 unsigned char idle_balance;
@@ -1005,6 +1006,10 @@ struct rq { · 24 unmodified lines
1005 1006
1006 /* This is used to determine avg_idle's max value */1007 /* This is used to determine avg_idle's max value */
1007 u64 max_idle_balance_cost;1008 u64 max_idle_balance_cost;
1009
1010#ifdef CONFIG_HOTPLUG_CPU
1011 struct rcuwait hotplug_wait;
1012#endif
1008#endif /* CONFIG_SMP */1013#endif /* CONFIG_SMP */
1009 1014
1010#ifdef CONFIG_IRQ_TIME_ACCOUNTING1015#ifdef CONFIG_IRQ_TIME_ACCOUNTING
@@ -1050,6 +1055,12 @@ struct rq { · 39 unmodified lines
1050 /* Must be inspected within a rcu lock section */1055 /* Must be inspected within a rcu lock section */
1051 struct cpuidle_state *idle_state;1056 struct cpuidle_state *idle_state;
1052#endif1057#endif
1058
1059#if defined(CONFIG_PREEMPT_RT) && defined(CONFIG_SMP)
1060 unsigned int nr_pinned;
1061#endif
1062 unsigned int push_busy;
1063 struct cpu_stop_work push_work;
1053};1064};
1054 1065
1055#ifdef CONFIG_FAIR_GROUP_SCHED1066#ifdef CONFIG_FAIR_GROUP_SCHED
@@ -1077,6 +1088,16 @@ static inline int cpu_of(struct rq *rq) · 21 unmodified lines
1077#endif1088#endif
1078}1089}
1079 1090
1091#define MDF_PUSH 0x01
1092
1093static inline bool is_migration_disabled(struct task_struct *p)
1094{
1095#if defined(CONFIG_SMP) && defined(CONFIG_PREEMPT_RT)
1096 return p->migration_disabled;
1097#else
1098 return false;
1099#endif
1100}
1080 1101
1081#ifdef CONFIG_SCHED_SMT1102#ifdef CONFIG_SCHED_SMT
1082extern void __update_idle_core(struct rq *rq);1103extern void __update_idle_core(struct rq *rq);
@@ -1223,6 +1244,9 @@ static inline void rq_pin_lock(struct rq *rq, struct rq_flags *rf) · 140 unmodified lines
1223 rq->clock_update_flags &= (RQCF_REQ_SKIP|RQCF_ACT_SKIP);1244 rq->clock_update_flags &= (RQCF_REQ_SKIP|RQCF_ACT_SKIP);
1224 rf->clock_update_flags = 0;1245 rf->clock_update_flags = 0;
1225#endif1246#endif
1247#ifdef CONFIG_SMP
1248 SCHED_WARN_ON(rq->balance_callback);
1249#endif
1226}1250}
1227 1251
1228static inline void rq_unpin_lock(struct rq *rq, struct rq_flags *rf)1252static inline void rq_unpin_lock(struct rq *rq, struct rq_flags *rf)
@@ -1384,6 +1408,9 @@ init_numa_balancing(unsigned long clone_flags, struct task_struct *p) · 155 unmodified lines
1384 1408
1385#ifdef CONFIG_SMP1409#ifdef CONFIG_SMP
1386 1410
1411#define BALANCE_WORK 0x01
1412#define BALANCE_PUSH 0x02
1413
1387static inline void1414static inline void
1388queue_balance_callback(struct rq *rq,1415queue_balance_callback(struct rq *rq,
1389 struct callback_head *head,1416 struct callback_head *head,
@@ -1391,12 +1418,13 @@ queue_balance_callback(struct rq *rq, · 1 unmodified line
1391{1418{
1392 lockdep_assert_held(&rq->lock);1419 lockdep_assert_held(&rq->lock);
1393 1420
1394 if (unlikely(head->next))1421 if (unlikely(head->next || (rq->balance_flags & BALANCE_PUSH)))
1395 return;1422 return;
1396 1423
1397 head->func = (void (*)(struct callback_head *))func;1424 head->func = (void (*)(struct callback_head *))func;
1398 head->next = rq->balance_callback;1425 head->next = rq->balance_callback;
1399 rq->balance_callback = head;1426 rq->balance_callback = head;
1427 rq->balance_flags |= BALANCE_WORK;
1400}1428}
1401 1429
1402#define rcu_dereference_check_sched_domain(p) \1430#define rcu_dereference_check_sched_domain(p) \
@@ -1804,10 +1832,13 @@ struct sched_class { · 401 unmodified lines
1804 void (*task_woken)(struct rq *this_rq, struct task_struct *task);1832 void (*task_woken)(struct rq *this_rq, struct task_struct *task);
1805 1833
1806 void (*set_cpus_allowed)(struct task_struct *p,1834 void (*set_cpus_allowed)(struct task_struct *p,
1807 const struct cpumask *newmask);1835 const struct cpumask *newmask,
1836 u32 flags);
1808 1837
1809 void (*rq_online)(struct rq *rq);1838 void (*rq_online)(struct rq *rq);
1810 void (*rq_offline)(struct rq *rq);1839 void (*rq_offline)(struct rq *rq);
1840
1841 struct rq *(*find_lock_rq)(struct task_struct *p, struct rq *rq);
1811#endif1842#endif
1812 1843
1813 void (*task_tick)(struct rq *rq, struct task_struct *p, int queued);1844 void (*task_tick)(struct rq *rq, struct task_struct *p, int queued);
@@ -1905,13 +1936,35 @@ static inline bool sched_fair_runnable(struct rq *rq) · 91 unmodified lines
1905extern struct task_struct *pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf);1936extern struct task_struct *pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf);
1906extern struct task_struct *pick_next_task_idle(struct rq *rq);1937extern struct task_struct *pick_next_task_idle(struct rq *rq);
1907 1938
1939#define SCA_CHECK 0x01
1940#define SCA_MIGRATE_DISABLE 0x02
1941#define SCA_MIGRATE_ENABLE 0x04
1942
1908#ifdef CONFIG_SMP1943#ifdef CONFIG_SMP
1909 1944
1910extern void update_group_capacity(struct sched_domain *sd, int cpu);1945extern void update_group_capacity(struct sched_domain *sd, int cpu);
1911 1946
1912extern void trigger_load_balance(struct rq *rq);1947extern void trigger_load_balance(struct rq *rq);
1913 1948
1914extern void set_cpus_allowed_common(struct task_struct *p, const struct cpumask *new_mask);1949extern void set_cpus_allowed_common(struct task_struct *p, const struct cpumask *new_mask, u32 flags);
1950
1951static inline struct task_struct *get_push_task(struct rq *rq)
1952{
1953 struct task_struct *p = rq->curr;
1954
1955 lockdep_assert_held(&rq->lock);
1956
1957 if (rq->push_busy)
1958 return NULL;
1959
1960 if (p->nr_cpus_allowed == 1)
1961 return NULL;
1962
1963 rq->push_busy = true;
1964 return get_task_struct(p);
1965}
1966
1967extern int push_cpu_stop(void *arg);
1915 1968
1916#endif1969#endif
1917 1970
kernel/stop_machine.c+24−3
@@ -42,11 +42,27 @@ struct cpu_stopper { · 41 unmodified lines
42 struct list_head works; /* list of pending works */42 struct list_head works; /* list of pending works */
43 43
44 struct cpu_stop_work stop_work; /* for stop_cpus */44 struct cpu_stop_work stop_work; /* for stop_cpus */
45 unsigned long caller;
46 cpu_stop_fn_t fn;
45};47};
46 48
47static DEFINE_PER_CPU(struct cpu_stopper, cpu_stopper);49static DEFINE_PER_CPU(struct cpu_stopper, cpu_stopper);
48static bool stop_machine_initialized = false;50static bool stop_machine_initialized = false;
49 51
52void print_stop_info(const char *log_lvl, struct task_struct *task)
53{
54 /*
55 * If @task is a stopper task, it cannot migrate and task_cpu() is
56 * stable.
57 */
58 struct cpu_stopper *stopper = per_cpu_ptr(&cpu_stopper, task_cpu(task));
59
60 if (task != stopper->thread)
61 return;
62
63 printk("%sStopper: %pS <- %pS\n", log_lvl, stopper->fn, (void *)stopper->caller);
64}
65
50/* static data for stop_cpus */66/* static data for stop_cpus */
51static DEFINE_MUTEX(stop_cpus_mutex);67static DEFINE_MUTEX(stop_cpus_mutex);
52static bool stop_cpus_in_progress;68static bool stop_cpus_in_progress;
@@ -123,7 +139,7 @@ static bool cpu_stop_queue_work(unsigned int cpu, struct cpu_stop_work *work) · 70 unmodified lines
123int stop_one_cpu(unsigned int cpu, cpu_stop_fn_t fn, void *arg)139int stop_one_cpu(unsigned int cpu, cpu_stop_fn_t fn, void *arg)
124{140{
125 struct cpu_stop_done done;141 struct cpu_stop_done done;
126 struct cpu_stop_work work = { .fn = fn, .arg = arg, .done = &done };142 struct cpu_stop_work work = { .fn = fn, .arg = arg, .done = &done, .caller = _RET_IP_ };
127 143
128 cpu_stop_init_done(&done, 1);144 cpu_stop_init_done(&done, 1);
129 if (!cpu_stop_queue_work(cpu, &work))145 if (!cpu_stop_queue_work(cpu, &work))
@@ -331,7 +347,8 @@ int stop_two_cpus(unsigned int cpu1, unsigned int cpu2, cpu_stop_fn_t fn, void * · 201 unmodified lines
331 work1 = work2 = (struct cpu_stop_work){347 work1 = work2 = (struct cpu_stop_work){
332 .fn = multi_cpu_stop,348 .fn = multi_cpu_stop,
333 .arg = &msdata,349 .arg = &msdata,
334 .done = &done350 .done = &done,
351 .caller = _RET_IP_,
335 };352 };
336 353
337 cpu_stop_init_done(&done, 2);354 cpu_stop_init_done(&done, 2);
@@ -367,7 +384,7 @@ int stop_two_cpus(unsigned int cpu1, unsigned int cpu2, cpu_stop_fn_t fn, void * · 29 unmodified lines
367bool stop_one_cpu_nowait(unsigned int cpu, cpu_stop_fn_t fn, void *arg,384bool stop_one_cpu_nowait(unsigned int cpu, cpu_stop_fn_t fn, void *arg,
368 struct cpu_stop_work *work_buf)385 struct cpu_stop_work *work_buf)
369{386{
370 *work_buf = (struct cpu_stop_work){ .fn = fn, .arg = arg, };387 *work_buf = (struct cpu_stop_work){ .fn = fn, .arg = arg, .caller = _RET_IP_, };
371 return cpu_stop_queue_work(cpu, work_buf);388 return cpu_stop_queue_work(cpu, work_buf);
372}389}
373 390
@@ -487,6 +504,8 @@ repeat: · 113 unmodified lines
487 int ret;504 int ret;
488 505
489 /* cpu stop callbacks must not sleep, make in_atomic() == T */506 /* cpu stop callbacks must not sleep, make in_atomic() == T */
507 stopper->caller = work->caller;
508 stopper->fn = fn;
490 preempt_count_inc();509 preempt_count_inc();
491 ret = fn(arg);510 ret = fn(arg);
492 if (done) {511 if (done) {
@@ -495,6 +514,8 @@ repeat: · 2 unmodified lines
495 cpu_stop_signal_done(done);514 cpu_stop_signal_done(done);
496 }515 }
497 preempt_count_dec();516 preempt_count_dec();
517 stopper->fn = NULL;
518 stopper->caller = 0;
498 WARN_ONCE(preempt_count(),519 WARN_ONCE(preempt_count(),
499 "cpu_stop: %ps(%p) leaked preempt count\n", fn, arg);520 "cpu_stop: %ps(%p) leaked preempt count\n", fn, arg);
500 goto repeat;521 goto repeat;
kernel/workqueue.c+4−0
@@ -4908,6 +4908,10 @@ static void unbind_workers(int cpu) · 4907 unmodified lines
4908 pool->flags |= POOL_DISASSOCIATED;4908 pool->flags |= POOL_DISASSOCIATED;
4909 4909
4910 raw_spin_unlock_irq(&pool->lock);4910 raw_spin_unlock_irq(&pool->lock);
4911
4912 for_each_pool_worker(worker, pool)
4913 WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, cpu_active_mask) < 0);
4914
4911 mutex_unlock(&wq_pool_attach_mutex);4915 mutex_unlock(&wq_pool_attach_mutex);
4912 4916
4913 /*4917 /*

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t