Merge branch 'sched/migrate-disable'

Peter Zijlstra committed 6 years ago

Commit 12fa97c2 parents b6d37a7 c777d8419 files · +1038 −227Browse files

Showing the changes to kernel (9 of 19 files). Show all

Unified · Split

9 files

kernel/cpu.c +8 −1
kernel/sched/core.c +739 −181
kernel/sched/cpudeadline.c +2 −2
kernel/sched/cpupri.c +2 −2
kernel/sched/deadline.c +31 −15
kernel/sched/rt.c +57 −18
kernel/sched/sched.h +56 −3
kernel/stop_machine.c +24 −3
kernel/workqueue.c +4 −0

@@ -1602,7 +1602,7 @@ static struct cpuhp_step cpuhp_hp_states[] = { · 1601 unmodified lines
1602	1602	.name = "ap:online",
1603	1603	},
1604	1604	/*
1605		− * Handled on controll processor until the plugged processor manages
	1605	+ * Handled on control processor until the plugged processor manages
1606	1606	* this itself.
1607	1607	*/
1608	1608	[CPUHP_TEARDOWN_CPU] = {
@@ -1611,6 +1611,13 @@ static struct cpuhp_step cpuhp_hp_states[] = { · 2 unmodified lines
1611	1611	.teardown.single = takedown_cpu,
1612	1612	.cant_stop = true,
1613	1613	},
	1614	+
	1615	+ [CPUHP_AP_SCHED_WAIT_EMPTY] = {
	1616	+ .name = "sched:waitempty",
	1617	+ .startup.single = NULL,
	1618	+ .teardown.single = sched_cpu_wait_empty,
	1619	+ },
	1620	+
1614	1621	/* Handle smpboot threads park/unpark */
1615	1622	[CPUHP_AP_SMPBOOT_THREADS] = {
1616	1623	.name = "smpboot/threads:online",

kernel/sched/core.c+739−181

@@ -1696,6 +1696,80 @@ void check_preempt_curr(struct rq rq, struct task_struct p, int flags) · 1695 unmodified lines
1696	1696
1697	1697	#ifdef CONFIG_SMP
1698	1698
	1699	+#ifdef CONFIG_PREEMPT_RT
	1700	+
	1701	+static void
	1702	+__do_set_cpus_allowed(struct task_struct p, const struct cpumask new_mask, u32 flags);
	1703	+
	1704	+static int __set_cpus_allowed_ptr(struct task_struct *p,
	1705	+ const struct cpumask *new_mask,
	1706	+ u32 flags);
	1707	+
	1708	+static void migrate_disable_switch(struct rq rq, struct task_struct p)
	1709	+{
	1710	+ if (likely(!p->migration_disabled))
	1711	+ return;
	1712	+
	1713	+ if (p->cpus_ptr != &p->cpus_mask)
	1714	+ return;
	1715	+
	1716	+ /*
	1717	+ * Violates locking rules! see comment in __do_set_cpus_allowed().
	1718	+ */
	1719	+ __do_set_cpus_allowed(p, cpumask_of(rq->cpu), SCA_MIGRATE_DISABLE);
	1720	+}
	1721	+
	1722	+void migrate_disable(void)
	1723	+{
	1724	+ struct task_struct *p = current;
	1725	+
	1726	+ if (p->migration_disabled) {
	1727	+ p->migration_disabled++;
	1728	+ return;
	1729	+ }
	1730	+
	1731	+ preempt_disable();
	1732	+ this_rq()->nr_pinned++;
	1733	+ p->migration_disabled = 1;
	1734	+ preempt_enable();
	1735	+}
	1736	+EXPORT_SYMBOL_GPL(migrate_disable);
	1737	+
	1738	+void migrate_enable(void)
	1739	+{
	1740	+ struct task_struct *p = current;
	1741	+
	1742	+ if (p->migration_disabled > 1) {
	1743	+ p->migration_disabled--;
	1744	+ return;
	1745	+ }
	1746	+
	1747	+ /*
	1748	+ * Ensure stop_task runs either before or after this, and that
	1749	+ * __set_cpus_allowed_ptr(SCA_MIGRATE_ENABLE) doesn't schedule().
	1750	+ */
	1751	+ preempt_disable();
	1752	+ if (p->cpus_ptr != &p->cpus_mask)
	1753	+ __set_cpus_allowed_ptr(p, &p->cpus_mask, SCA_MIGRATE_ENABLE);
	1754	+ /*
	1755	+ * Mustn't clear migration_disabled() until cpus_ptr points back at the
	1756	+ * regular cpus_mask, otherwise things that race (eg.
	1757	+ * select_fallback_rq) get confused.
	1758	+ */
	1759	+ barrier();
	1760	+ p->migration_disabled = 0;
	1761	+ this_rq()->nr_pinned--;
	1762	+ preempt_enable();
	1763	+}
	1764	+EXPORT_SYMBOL_GPL(migrate_enable);
	1765	+
	1766	+static inline bool rq_has_pinned_tasks(struct rq *rq)
	1767	+{
	1768	+ return rq->nr_pinned;
	1769	+}
	1770	+
	1771	+#endif
	1772	+
1699	1773	/*
1700	1774	* Per-CPU kthreads are allowed to run on !active && online CPUs, see
1701	1775	* __set_cpus_allowed_ptr() and select_fallback_rq().
@@ -1705,7 +1779,7 @@ static inline bool is_cpu_allowed(struct task_struct *p, int cpu) · 3 unmodified lines
1705	1779	if (!cpumask_test_cpu(cpu, p->cpus_ptr))
1706	1780	return false;
1707	1781
1708		− if (is_per_cpu_kthread(p))
	1782	+ if (is_per_cpu_kthread(p) \|\| is_migration_disabled(p))
1709	1783	return cpu_online(cpu);
1710	1784
1711	1785	return cpu_active(cpu);
@@ -1750,8 +1824,16 @@ static struct rq move_queued_task(struct rq rq, struct rq_flags *rf, · 38 unmodified lines
1750	1824	}
1751	1825
1752	1826	struct migration_arg {
1753		− struct task_struct *task;
1754		− int dest_cpu;
	1827	+ struct task_struct *task;
	1828	+ int dest_cpu;
	1829	+ struct set_affinity_pending *pending;
	1830	+};
	1831	+
	1832	+struct set_affinity_pending {
	1833	+ refcount_t refs;
	1834	+ struct completion done;
	1835	+ struct cpu_stop_work stop_work;
	1836	+ struct migration_arg arg;
1755	1837	};
1756	1838
1757	1839	/*
@@ -1783,16 +1865,19 @@ static struct rq __migrate_task(struct rq rq, struct rq_flags *rf, · 25 unmodified lines
1783	1865	*/
1784	1866	static int migration_cpu_stop(void *data)
1785	1867	{
	1868	+ struct set_affinity_pending *pending;
1786	1869	struct migration_arg *arg = data;
1787	1870	struct task_struct *p = arg->task;
	1871	+ int dest_cpu = arg->dest_cpu;
1788	1872	struct rq *rq = this_rq();
	1873	+ bool complete = false;
1789	1874	struct rq_flags rf;
1790	1875
1791	1876	/*
1792	1877	* The original target CPU might have gone down and we might
1793	1878	* be on another CPU but it doesn't matter.
1794	1879	*/
1795		− local_irq_disable();
	1880	+ local_irq_save(rf.flags);
1796	1881	/*
1797	1882	* We need to explicitly wake pending tasks before running
1798	1883	* __migrate_task() such that we will not miss enforcing cpus_ptr
@@ -1802,21 +1887,126 @@ static int migration_cpu_stop(void *data) · 3 unmodified lines
1802	1887
1803	1888	raw_spin_lock(&p->pi_lock);
1804	1889	rq_lock(rq, &rf);
	1890	+
	1891	+ pending = p->migration_pending;
1805	1892	/*
1806	1893	* If task_rq(p) != rq, it cannot be migrated here, because we're
1807	1894	* holding rq->lock, if p->on_rq == 0 it cannot get enqueued because
1808	1895	* we're holding p->pi_lock.
1809	1896	*/
1810	1897	if (task_rq(p) == rq) {
	1898	+ if (is_migration_disabled(p))
	1899	+ goto out;
	1900	+
	1901	+ if (pending) {
	1902	+ p->migration_pending = NULL;
	1903	+ complete = true;
	1904	+ }
	1905	+
	1906	+ /* migrate_enable() -- we must not race against SCA */
	1907	+ if (dest_cpu < 0) {
	1908	+ /*
	1909	+ * When this was migrate_enable() but we no longer
	1910	+ * have a @pending, a concurrent SCA 'fixed' things
	1911	+ * and we should be valid again. Nothing to do.
	1912	+ */
	1913	+ if (!pending) {
	1914	+ WARN_ON_ONCE(!is_cpu_allowed(p, cpu_of(rq)));
	1915	+ goto out;
	1916	+ }
	1917	+
	1918	+ dest_cpu = cpumask_any_distribute(&p->cpus_mask);
	1919	+ }
	1920	+
1811	1921	if (task_on_rq_queued(p))
1812		− rq = __migrate_task(rq, &rf, p, arg->dest_cpu);
	1922	+ rq = __migrate_task(rq, &rf, p, dest_cpu);
1813	1923	else
1814		− p->wake_cpu = arg->dest_cpu;
	1924	+ p->wake_cpu = dest_cpu;
	1925	+
	1926	+ } else if (dest_cpu < 0) {
	1927	+ /*
	1928	+ * This happens when we get migrated between migrate_enable()'s
	1929	+ * preempt_enable() and scheduling the stopper task. At that
	1930	+ * point we're a regular task again and not current anymore.
	1931	+ *
	1932	+ * A !PREEMPT kernel has a giant hole here, which makes it far
	1933	+ * more likely.
	1934	+ */
	1935	+
	1936	+ /*
	1937	+ * When this was migrate_enable() but we no longer have an
	1938	+ * @pending, a concurrent SCA 'fixed' things and we should be
	1939	+ * valid again. Nothing to do.
	1940	+ */
	1941	+ if (!pending) {
	1942	+ WARN_ON_ONCE(!is_cpu_allowed(p, cpu_of(rq)));
	1943	+ goto out;
	1944	+ }
	1945	+
	1946	+ /*
	1947	+ * When migrate_enable() hits a rq mis-match we can't reliably
	1948	+ * determine is_migration_disabled() and so have to chase after
	1949	+ * it.
	1950	+ */
	1951	+ task_rq_unlock(rq, p, &rf);
	1952	+ stop_one_cpu_nowait(task_cpu(p), migration_cpu_stop,
	1953	+ &pending->arg, &pending->stop_work);
	1954	+ return 0;
1815	1955	}
1816		− rq_unlock(rq, &rf);
1817		− raw_spin_unlock(&p->pi_lock);
	1956	+out:
	1957	+ task_rq_unlock(rq, p, &rf);
	1958	+
	1959	+ if (complete)
	1960	+ complete_all(&pending->done);
	1961	+
	1962	+ /* For pending->{arg,stop_work} */
	1963	+ pending = arg->pending;
	1964	+ if (pending && refcount_dec_and_test(&pending->refs))
	1965	+ wake_up_var(&pending->refs);
1818	1966
1819		− local_irq_enable();
	1967	+ return 0;
	1968	+}
	1969	+
	1970	+int push_cpu_stop(void *arg)
	1971	+{
	1972	+ struct rq lowest_rq = NULL, rq = this_rq();
	1973	+ struct task_struct *p = arg;
	1974	+
	1975	+ raw_spin_lock_irq(&p->pi_lock);
	1976	+ raw_spin_lock(&rq->lock);
	1977	+
	1978	+ if (task_rq(p) != rq)
	1979	+ goto out_unlock;
	1980	+
	1981	+ if (is_migration_disabled(p)) {
	1982	+ p->migration_flags \|= MDF_PUSH;
	1983	+ goto out_unlock;
	1984	+ }
	1985	+
	1986	+ p->migration_flags &= ~MDF_PUSH;
	1987	+
	1988	+ if (p->sched_class->find_lock_rq)
	1989	+ lowest_rq = p->sched_class->find_lock_rq(p, rq);
	1990	+
	1991	+ if (!lowest_rq)
	1992	+ goto out_unlock;
	1993	+
	1994	+ // XXX validate p is still the highest prio task
	1995	+ if (task_rq(p) == rq) {
	1996	+ deactivate_task(rq, p, 0);
	1997	+ set_task_cpu(p, lowest_rq->cpu);
	1998	+ activate_task(lowest_rq, p, 0);
	1999	+ resched_curr(lowest_rq);
	2000	+ }
	2001	+
	2002	+ double_unlock_balance(rq, lowest_rq);
	2003	+
	2004	+out_unlock:
	2005	+ rq->push_busy = false;
	2006	+ raw_spin_unlock(&rq->lock);
	2007	+ raw_spin_unlock_irq(&p->pi_lock);
	2008	+
	2009	+ put_task_struct(p);
1820	2010	return 0;
1821	2011	}
1822	2012
@@ -1824,18 +2014,39 @@ static int migration_cpu_stop(void *data) · 1 unmodified line
1824	2014	* sched_class::set_cpus_allowed must do the below, but is not required to
1825	2015	* actually call this function.
1826	2016	*/
1827		−void set_cpus_allowed_common(struct task_struct p, const struct cpumask new_mask)
	2017	+void set_cpus_allowed_common(struct task_struct p, const struct cpumask new_mask, u32 flags)
1828	2018	{
	2019	+ if (flags & (SCA_MIGRATE_ENABLE \| SCA_MIGRATE_DISABLE)) {
	2020	+ p->cpus_ptr = new_mask;
	2021	+ return;
	2022	+ }
	2023	+
1829	2024	cpumask_copy(&p->cpus_mask, new_mask);
1830	2025	p->nr_cpus_allowed = cpumask_weight(new_mask);
1831	2026	}
1832	2027
1833		−void do_set_cpus_allowed(struct task_struct p, const struct cpumask new_mask)
	2028	+static void
	2029	+__do_set_cpus_allowed(struct task_struct p, const struct cpumask new_mask, u32 flags)
1834	2030	{
1835	2031	struct rq *rq = task_rq(p);
1836	2032	bool queued, running;
1837	2033
1838		− lockdep_assert_held(&p->pi_lock);
	2034	+ /*
	2035	+ * This here violates the locking rules for affinity, since we're only
	2036	+ * supposed to change these variables while holding both rq->lock and
	2037	+ * p->pi_lock.
	2038	+ *
	2039	+ * HOWEVER, it magically works, because ttwu() is the only code that
	2040	+ * accesses these variables under p->pi_lock and only does so after
	2041	+ * smp_cond_load_acquire(&p->on_cpu, !VAL), and we're in __schedule()
	2042	+ * before finish_task().
	2043	+ *
	2044	+ * XXX do further audits, this smells like something putrid.
	2045	+ */
	2046	+ if (flags & SCA_MIGRATE_DISABLE)
	2047	+ SCHED_WARN_ON(!p->on_cpu);
	2048	+ else
	2049	+ lockdep_assert_held(&p->pi_lock);
1839	2050
1840	2051	queued = task_on_rq_queued(p);
1841	2052	running = task_current(rq, p);
@@ -1851,7 +2062,7 @@ void do_set_cpus_allowed(struct task_struct p, const struct cpumask new_mask) · 9 unmodified lines
1851	2062	if (running)
1852	2063	put_prev_task(rq, p);
1853	2064
1854		− p->sched_class->set_cpus_allowed(p, new_mask);
	2065	+ p->sched_class->set_cpus_allowed(p, new_mask, flags);
1855	2066
1856	2067	if (queued)
1857	2068	enqueue_task(rq, p, ENQUEUE_RESTORE \| ENQUEUE_NOCLOCK);
@@ -1859,6 +2070,208 @@ void do_set_cpus_allowed(struct task_struct p, const struct cpumask new_mask) · 1 unmodified line
1859	2070	set_next_task(rq, p);
1860	2071	}
1861	2072
	2073	+void do_set_cpus_allowed(struct task_struct p, const struct cpumask new_mask)
	2074	+{
	2075	+ __do_set_cpus_allowed(p, new_mask, 0);
	2076	+}
	2077	+
	2078	+/*
	2079	+ * This function is wildly self concurrent; here be dragons.
	2080	+ *
	2081	+ *
	2082	+ * When given a valid mask, __set_cpus_allowed_ptr() must block until the
	2083	+ * designated task is enqueued on an allowed CPU. If that task is currently
	2084	+ * running, we have to kick it out using the CPU stopper.
	2085	+ *
	2086	+ * Migrate-Disable comes along and tramples all over our nice sandcastle.
	2087	+ * Consider:
	2088	+ *
	2089	+ * Initial conditions: P0->cpus_mask = [0, 1]
	2090	+ *
	2091	+ * P0@CPU0 P1
	2092	+ *
	2093	+ * migrate_disable();
	2094	+ * <preempted>
	2095	+ * set_cpus_allowed_ptr(P0, [1]);
	2096	+ *
	2097	+ * P1 cannot return from this set_cpus_allowed_ptr() call until P0 executes
	2098	+ * its outermost migrate_enable() (i.e. it exits its Migrate-Disable region).
	2099	+ * This means we need the following scheme:
	2100	+ *
	2101	+ * P0@CPU0 P1
	2102	+ *
	2103	+ * migrate_disable();
	2104	+ * <preempted>
	2105	+ * set_cpus_allowed_ptr(P0, [1]);
	2106	+ * <blocks>
	2107	+ * <resumes>
	2108	+ * migrate_enable();
	2109	+ * __set_cpus_allowed_ptr();
	2110	+ * <wakes local stopper>
	2111	+ * `--> <woken on migration completion>
	2112	+ *
	2113	+ * Now the fun stuff: there may be several P1-like tasks, i.e. multiple
	2114	+ * concurrent set_cpus_allowed_ptr(P0, [*]) calls. CPU affinity changes of any
	2115	+ * task p are serialized by p->pi_lock, which we can leverage: the one that
	2116	+ * should come into effect at the end of the Migrate-Disable region is the last
	2117	+ * one. This means we only need to track a single cpumask (i.e. p->cpus_mask),
	2118	+ * but we still need to properly signal those waiting tasks at the appropriate
	2119	+ * moment.
	2120	+ *
	2121	+ * This is implemented using struct set_affinity_pending. The first
	2122	+ * __set_cpus_allowed_ptr() caller within a given Migrate-Disable region will
	2123	+ * setup an instance of that struct and install it on the targeted task_struct.
	2124	+ * Any and all further callers will reuse that instance. Those then wait for
	2125	+ * a completion signaled at the tail of the CPU stopper callback (1), triggered
	2126	+ * on the end of the Migrate-Disable region (i.e. outermost migrate_enable()).
	2127	+ *
	2128	+ *
	2129	+ * (1) In the cases covered above. There is one more where the completion is
	2130	+ * signaled within affine_move_task() itself: when a subsequent affinity request
	2131	+ * cancels the need for an active migration. Consider:
	2132	+ *
	2133	+ * Initial conditions: P0->cpus_mask = [0, 1]
	2134	+ *
	2135	+ * P0@CPU0 P1 P2
	2136	+ *
	2137	+ * migrate_disable();
	2138	+ * <preempted>
	2139	+ * set_cpus_allowed_ptr(P0, [1]);
	2140	+ * <blocks>
	2141	+ * set_cpus_allowed_ptr(P0, [0, 1]);
	2142	+ * <signal completion>
	2143	+ * <awakes>
	2144	+ *
	2145	+ * Note that the above is safe vs a concurrent migrate_enable(), as any
	2146	+ * pending affinity completion is preceded by an uninstallation of
	2147	+ * p->migration_pending done with p->pi_lock held.
	2148	+ */
	2149	+static int affine_move_task(struct rq rq, struct task_struct p, struct rq_flags *rf,
	2150	+ int dest_cpu, unsigned int flags)
	2151	+{
	2152	+ struct set_affinity_pending my_pending = { }, *pending = NULL;
	2153	+ struct migration_arg arg = {
	2154	+ .task = p,
	2155	+ .dest_cpu = dest_cpu,
	2156	+ };
	2157	+ bool complete = false;
	2158	+
	2159	+ /* Can the task run on the task's current CPU? If so, we're done */
	2160	+ if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask)) {
	2161	+ struct task_struct *push_task = NULL;
	2162	+
	2163	+ if ((flags & SCA_MIGRATE_ENABLE) &&
	2164	+ (p->migration_flags & MDF_PUSH) && !rq->push_busy) {
	2165	+ rq->push_busy = true;
	2166	+ push_task = get_task_struct(p);
	2167	+ }
	2168	+
	2169	+ pending = p->migration_pending;
	2170	+ if (pending) {
	2171	+ refcount_inc(&pending->refs);
	2172	+ p->migration_pending = NULL;
	2173	+ complete = true;
	2174	+ }
	2175	+ task_rq_unlock(rq, p, rf);
	2176	+
	2177	+ if (push_task) {
	2178	+ stop_one_cpu_nowait(rq->cpu, push_cpu_stop,
	2179	+ p, &rq->push_work);
	2180	+ }
	2181	+
	2182	+ if (complete)
	2183	+ goto do_complete;
	2184	+
	2185	+ return 0;
	2186	+ }
	2187	+
	2188	+ if (!(flags & SCA_MIGRATE_ENABLE)) {
	2189	+ /* serialized by p->pi_lock */
	2190	+ if (!p->migration_pending) {
	2191	+ /* Install the request */
	2192	+ refcount_set(&my_pending.refs, 1);
	2193	+ init_completion(&my_pending.done);
	2194	+ p->migration_pending = &my_pending;
	2195	+ } else {
	2196	+ pending = p->migration_pending;
	2197	+ refcount_inc(&pending->refs);
	2198	+ }
	2199	+ }
	2200	+ pending = p->migration_pending;
	2201	+ /*
	2202	+ * - !MIGRATE_ENABLE:
	2203	+ * we'll have installed a pending if there wasn't one already.
	2204	+ *
	2205	+ * - MIGRATE_ENABLE:
	2206	+ * we're here because the current CPU isn't matching anymore,
	2207	+ * the only way that can happen is because of a concurrent
	2208	+ * set_cpus_allowed_ptr() call, which should then still be
	2209	+ * pending completion.
	2210	+ *
	2211	+ * Either way, we really should have a @pending here.
	2212	+ */
	2213	+ if (WARN_ON_ONCE(!pending)) {
	2214	+ task_rq_unlock(rq, p, rf);
	2215	+ return -EINVAL;
	2216	+ }
	2217	+
	2218	+ if (flags & SCA_MIGRATE_ENABLE) {
	2219	+
	2220	+ refcount_inc(&pending->refs); /* pending->{arg,stop_work} */
	2221	+ p->migration_flags &= ~MDF_PUSH;
	2222	+ task_rq_unlock(rq, p, rf);
	2223	+
	2224	+ pending->arg = (struct migration_arg) {
	2225	+ .task = p,
	2226	+ .dest_cpu = -1,
	2227	+ .pending = pending,
	2228	+ };
	2229	+
	2230	+ stop_one_cpu_nowait(cpu_of(rq), migration_cpu_stop,
	2231	+ &pending->arg, &pending->stop_work);
	2232	+
	2233	+ return 0;
	2234	+ }
	2235	+
	2236	+ if (task_running(rq, p) \|\| p->state == TASK_WAKING) {
	2237	+ /*
	2238	+ * Lessen races (and headaches) by delegating
	2239	+ * is_migration_disabled(p) checks to the stopper, which will
	2240	+ * run on the same CPU as said p.
	2241	+ */
	2242	+ task_rq_unlock(rq, p, rf);
	2243	+ stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
	2244	+
	2245	+ } else {
	2246	+
	2247	+ if (!is_migration_disabled(p)) {
	2248	+ if (task_on_rq_queued(p))
	2249	+ rq = move_queued_task(rq, rf, p, dest_cpu);
	2250	+
	2251	+ p->migration_pending = NULL;
	2252	+ complete = true;
	2253	+ }
	2254	+ task_rq_unlock(rq, p, rf);
	2255	+
	2256	+do_complete:
	2257	+ if (complete)
	2258	+ complete_all(&pending->done);
	2259	+ }
	2260	+
	2261	+ wait_for_completion(&pending->done);
	2262	+
	2263	+ if (refcount_dec_and_test(&pending->refs))
	2264	+ wake_up_var(&pending->refs);
	2265	+
	2266	+ /*
	2267	+ * Block the original owner of &pending until all subsequent callers
	2268	+ * have seen the completion and decremented the refcount
	2269	+ */
	2270	+ wait_var_event(&my_pending.refs, !refcount_read(&my_pending.refs));
	2271	+
	2272	+ return 0;
	2273	+}
	2274	+
1862	2275	/*
1863	2276	* Change a given task's CPU affinity. Migrate the thread to a
1864	2277	* proper CPU and schedule it away if the CPU it's executing on
@@ -1869,7 +2282,8 @@ void do_set_cpus_allowed(struct task_struct p, const struct cpumask new_mask) · 4 unmodified lines
1869	2282	* call is not atomic; no spinlocks may be held.
1870	2283	*/
1871	2284	static int __set_cpus_allowed_ptr(struct task_struct *p,
1872		− const struct cpumask *new_mask, bool check)
	2285	+ const struct cpumask *new_mask,
	2286	+ u32 flags)
1873	2287	{
1874	2288	const struct cpumask *cpu_valid_mask = cpu_active_mask;
1875	2289	unsigned int dest_cpu;
@@ -1880,9 +2294,14 @@ static int __set_cpus_allowed_ptr(struct task_struct *p, · 4 unmodified lines
1880	2294	rq = task_rq_lock(p, &rf);
1881	2295	update_rq_clock(rq);
1882	2296
1883		− if (p->flags & PF_KTHREAD) {
	2297	+ if (p->flags & PF_KTHREAD \|\| is_migration_disabled(p)) {
1884	2298	/*
1885		− * Kernel threads are allowed on online && !active CPUs
	2299	+ * Kernel threads are allowed on online && !active CPUs.
	2300	+ *
	2301	+ * Specifically, migration_disabled() tasks must not fail the
	2302	+ * cpumask_any_and_distribute() pick below, esp. so on
	2303	+ * SCA_MIGRATE_ENABLE, otherwise we'll not call
	2304	+ * set_cpus_allowed_common() and actually reset p->cpus_ptr.
1886	2305	*/
1887	2306	cpu_valid_mask = cpu_online_mask;
1888	2307	}
@@ -1891,13 +2310,22 @@ static int __set_cpus_allowed_ptr(struct task_struct *p, · 2 unmodified lines
1891	2310	* Must re-check here, to close a race against __kthread_bind(),
1892	2311	* sched_setaffinity() is not guaranteed to observe the flag.
1893	2312	*/
1894		− if (check && (p->flags & PF_NO_SETAFFINITY)) {
	2313	+ if ((flags & SCA_CHECK) && (p->flags & PF_NO_SETAFFINITY)) {
1895	2314	ret = -EINVAL;
1896	2315	goto out;
1897	2316	}
1898	2317
1899		− if (cpumask_equal(&p->cpus_mask, new_mask))
1900		− goto out;
	2318	+ if (!(flags & SCA_MIGRATE_ENABLE)) {
	2319	+ if (cpumask_equal(&p->cpus_mask, new_mask))
	2320	+ goto out;
	2321	+
	2322	+ if (WARN_ON_ONCE(p == current &&
	2323	+ is_migration_disabled(p) &&
	2324	+ !cpumask_test_cpu(task_cpu(p), new_mask))) {
	2325	+ ret = -EBUSY;
	2326	+ goto out;
	2327	+ }
	2328	+ }
1901	2329
1902	2330	/*
1903	2331	* Picking a ~random cpu helps in cases where we are changing affinity
@@ -1910,7 +2338,7 @@ static int __set_cpus_allowed_ptr(struct task_struct *p, · 6 unmodified lines
1910	2338	goto out;
1911	2339	}
1912	2340
1913		− do_set_cpus_allowed(p, new_mask);
	2341	+ __do_set_cpus_allowed(p, new_mask, flags);
1914	2342
1915	2343	if (p->flags & PF_KTHREAD) {
1916	2344	/*
@@ -1922,23 +2350,8 @@ static int __set_cpus_allowed_ptr(struct task_struct *p, · 5 unmodified lines
1922	2350	p->nr_cpus_allowed != 1);
1923	2351	}
1924	2352
1925		− /* Can the task run on the task's current CPU? If so, we're done */
1926		− if (cpumask_test_cpu(task_cpu(p), new_mask))
1927		− goto out;
	2353	+ return affine_move_task(rq, p, &rf, dest_cpu, flags);
1928	2354
1929		− if (task_running(rq, p) \|\| p->state == TASK_WAKING) {
1930		− struct migration_arg arg = { p, dest_cpu };
1931		− /* Need help from migration thread: drop lock and wait. */
1932		− task_rq_unlock(rq, p, &rf);
1933		− stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
1934		− return 0;
1935		− } else if (task_on_rq_queued(p)) {
1936		− /*
1937		− * OK, since we're going to drop the lock immediately
1938		− * afterwards anyway.
1939		− */
1940		− rq = move_queued_task(rq, &rf, p, dest_cpu);
1941		− }
1942	2355	out:
1943	2356	task_rq_unlock(rq, p, &rf);
1944	2357
@@ -1947,7 +2360,7 @@ out: · 2 unmodified lines
1947	2360
1948	2361	int set_cpus_allowed_ptr(struct task_struct p, const struct cpumask new_mask)
1949	2362	{
1950		− return __set_cpus_allowed_ptr(p, new_mask, false);
	2363	+ return __set_cpus_allowed_ptr(p, new_mask, 0);
1951	2364	}
1952	2365	EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
1953	2366
@@ -1988,6 +2401,8 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu) · 34 unmodified lines
1988	2401	* Clearly, migrating tasks to offline CPUs is a fairly daft thing.
1989	2402	*/
1990	2403	WARN_ON_ONCE(!cpu_online(new_cpu));
	2404	+
	2405	+ WARN_ON_ONCE(is_migration_disabled(p));
1991	2406	#endif
1992	2407
1993	2408	trace_sched_migrate_task(p, new_cpu);
@@ -2318,6 +2733,12 @@ static int select_fallback_rq(int cpu, struct task_struct *p) · 324 unmodified lines
2318	2733	}
2319	2734	fallthrough;
2320	2735	case possible:
	2736	+ /*
	2737	+ * XXX When called from select_task_rq() we only
	2738	+ * hold p->pi_lock and again violate locking order.
	2739	+ *
	2740	+ * More yuck to audit.
	2741	+ */
2321	2742	do_set_cpus_allowed(p, cpu_possible_mask);
2322	2743	state = fail;
2323	2744	break;
@@ -2352,7 +2773,7 @@ int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags) · 28 unmodified lines
2352	2773	{
2353	2774	lockdep_assert_held(&p->pi_lock);
2354	2775
2355		− if (p->nr_cpus_allowed > 1)
	2776	+ if (p->nr_cpus_allowed > 1 && !is_migration_disabled(p))
2356	2777	cpu = p->sched_class->select_task_rq(p, cpu, sd_flags, wake_flags);
2357	2778	else
2358	2779	cpu = cpumask_any(p->cpus_ptr);
@@ -2375,6 +2796,7 @@ int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags) · 16 unmodified lines
2375	2796
2376	2797	void sched_set_stop_task(int cpu, struct task_struct *stop)
2377	2798	{
	2799	+ static struct lock_class_key stop_pi_lock;
2378	2800	struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };
2379	2801	struct task_struct *old_stop = cpu_rq(cpu)->stop;
2380	2802
@@ -2390,6 +2812,20 @@ void sched_set_stop_task(int cpu, struct task_struct *stop) · 9 unmodified lines
2390	2812	sched_setscheduler_nocheck(stop, SCHED_FIFO, &param);
2391	2813
2392	2814	stop->sched_class = &stop_sched_class;
	2815	+
	2816	+ /*
	2817	+ * The PI code calls rt_mutex_setprio() with ->pi_lock held to
	2818	+ * adjust the effective priority of a task. As a result,
	2819	+ * rt_mutex_setprio() can trigger (RT) balancing operations,
	2820	+ * which can then trigger wakeups of the stop thread to push
	2821	+ * around the current task.
	2822	+ *
	2823	+ * The stop task itself will never be part of the PI-chain, it
	2824	+ * never blocks, therefore that ->pi_lock recursion is safe.
	2825	+ * Tell lockdep about this by placing the stop->pi_lock in its
	2826	+ * own class.
	2827	+ */
	2828	+ lockdep_set_class(&stop->pi_lock, &stop_pi_lock);
2393	2829	}
2394	2830
2395	2831	cpu_rq(cpu)->stop = stop;
@@ -2406,13 +2842,25 @@ void sched_set_stop_task(int cpu, struct task_struct *stop) · 10 unmodified lines
2406	2842	#else
2407	2843
2408	2844	static inline int __set_cpus_allowed_ptr(struct task_struct *p,
2409		− const struct cpumask *new_mask, bool check)
	2845	+ const struct cpumask *new_mask,
	2846	+ u32 flags)
2410	2847	{
2411	2848	return set_cpus_allowed_ptr(p, new_mask);
2412	2849	}
2413	2850
2414	2851	#endif /* CONFIG_SMP */
2415	2852
	2853	+#if !defined(CONFIG_SMP) \|\| !defined(CONFIG_PREEMPT_RT)
	2854	+
	2855	+static inline void migrate_disable_switch(struct rq rq, struct task_struct p) { }
	2856	+
	2857	+static inline bool rq_has_pinned_tasks(struct rq *rq)
	2858	+{
	2859	+ return false;
	2860	+}
	2861	+
	2862	+#endif
	2863	+
2416	2864	static void
2417	2865	ttwu_stat(struct task_struct *p, int cpu, int wake_flags)
2418	2866	{
@@ -3098,6 +3546,7 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p) · 679 unmodified lines
3098	3546	init_numa_balancing(clone_flags, p);
3099	3547	#ifdef CONFIG_SMP
3100	3548	p->wake_entry.u_flags = CSD_TYPE_TTWU;
	3549	+ p->migration_pending = NULL;
3101	3550	#endif
3102	3551	}
3103	3552
@@ -3485,6 +3934,90 @@ static inline void finish_task(struct task_struct *prev) · 381 unmodified lines
3485	3934	#endif
3486	3935	}
3487	3936
	3937	+#ifdef CONFIG_SMP
	3938	+
	3939	+static void do_balance_callbacks(struct rq rq, struct callback_head head)
	3940	+{
	3941	+ void (func)(struct rq rq);
	3942	+ struct callback_head *next;
	3943	+
	3944	+ lockdep_assert_held(&rq->lock);
	3945	+
	3946	+ while (head) {
	3947	+ func = (void ()(struct rq ))head->func;
	3948	+ next = head->next;
	3949	+ head->next = NULL;
	3950	+ head = next;
	3951	+
	3952	+ func(rq);
	3953	+ }
	3954	+}
	3955	+
	3956	+static inline struct callback_head splice_balance_callbacks(struct rq rq)
	3957	+{
	3958	+ struct callback_head *head = rq->balance_callback;
	3959	+
	3960	+ lockdep_assert_held(&rq->lock);
	3961	+ if (head) {
	3962	+ rq->balance_callback = NULL;
	3963	+ rq->balance_flags &= ~BALANCE_WORK;
	3964	+ }
	3965	+
	3966	+ return head;
	3967	+}
	3968	+
	3969	+static void __balance_callbacks(struct rq *rq)
	3970	+{
	3971	+ do_balance_callbacks(rq, splice_balance_callbacks(rq));
	3972	+}
	3973	+
	3974	+static inline void balance_callbacks(struct rq rq, struct callback_head head)
	3975	+{
	3976	+ unsigned long flags;
	3977	+
	3978	+ if (unlikely(head)) {
	3979	+ raw_spin_lock_irqsave(&rq->lock, flags);
	3980	+ do_balance_callbacks(rq, head);
	3981	+ raw_spin_unlock_irqrestore(&rq->lock, flags);
	3982	+ }
	3983	+}
	3984	+
	3985	+static void balance_push(struct rq *rq);
	3986	+
	3987	+static inline void balance_switch(struct rq *rq)
	3988	+{
	3989	+ if (likely(!rq->balance_flags))
	3990	+ return;
	3991	+
	3992	+ if (rq->balance_flags & BALANCE_PUSH) {
	3993	+ balance_push(rq);
	3994	+ return;
	3995	+ }
	3996	+
	3997	+ __balance_callbacks(rq);
	3998	+}
	3999	+
	4000	+#else
	4001	+
	4002	+static inline void __balance_callbacks(struct rq *rq)
	4003	+{
	4004	+}
	4005	+
	4006	+static inline struct callback_head splice_balance_callbacks(struct rq rq)
	4007	+{
	4008	+ return NULL;
	4009	+}
	4010	+
	4011	+static inline void balance_callbacks(struct rq rq, struct callback_head head)
	4012	+{
	4013	+}
	4014	+
	4015	+static inline void balance_switch(struct rq *rq)
	4016	+{
	4017	+}
	4018	+
	4019	+#endif
	4020	+
3488	4021	static inline void
3489	4022	prepare_lock_switch(struct rq rq, struct task_struct next, struct rq_flags *rf)
3490	4023	{
@@ -3510,6 +4043,7 @@ static inline void finish_lock_switch(struct rq *rq) · 19 unmodified lines
3510	4043	* prev into current:
3511	4044	*/
3512	4045	spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_);
	4046	+ balance_switch(rq);
3513	4047	raw_spin_unlock_irq(&rq->lock);
3514	4048	}
3515	4049
@@ -3651,43 +4185,6 @@ static struct rq finish_task_switch(struct task_struct prev) · 135 unmodified lines
3651	4185	return rq;
3652	4186	}
3653	4187
3654		−#ifdef CONFIG_SMP
3655		−
3656		−/* rq->lock is NOT held, but preemption is disabled */
3657		−static void __balance_callback(struct rq *rq)
3658		−{
3659		− struct callback_head head, next;
3660		− void (func)(struct rq rq);
3661		− unsigned long flags;
3662		−
3663		− raw_spin_lock_irqsave(&rq->lock, flags);
3664		− head = rq->balance_callback;
3665		− rq->balance_callback = NULL;
3666		− while (head) {
3667		− func = (void ()(struct rq ))head->func;
3668		− next = head->next;
3669		− head->next = NULL;
3670		− head = next;
3671		−
3672		− func(rq);
3673		− }
3674		− raw_spin_unlock_irqrestore(&rq->lock, flags);
3675		−}
3676		−
3677		−static inline void balance_callback(struct rq *rq)
3678		−{
3679		− if (unlikely(rq->balance_callback))
3680		− __balance_callback(rq);
3681		−}
3682		−
3683		−#else
3684		−
3685		−static inline void balance_callback(struct rq *rq)
3686		−{
3687		−}
3688		−
3689		−#endif
3690		−
3691	4188	/**
3692	4189	* schedule_tail - first thing a freshly forked thread must call.
3693	4190	* @prev: the thread we just switched away from.
@@ -3707,7 +4204,6 @@ asmlinkage __visible void schedule_tail(struct task_struct *prev) · 13 unmodified lines
3707	4204	*/
3708	4205
3709	4206	rq = finish_task_switch(prev);
3710		− balance_callback(rq);
3711	4207	preempt_enable();
3712	4208
3713	4209	if (current->set_child_tid)
@@ -4515,6 +5011,7 @@ static void __sched notrace __schedule(bool preempt) · 801 unmodified lines
4515	5011	*/
4516	5012	++*switch_count;
4517	5013
	5014	+ migrate_disable_switch(rq, prev);
4518	5015	psi_sched_switch(prev, next, !task_on_rq_queued(prev));
4519	5016
4520	5017	trace_sched_switch(preempt, prev, next);
@@ -4523,10 +5020,11 @@ static void __sched notrace __schedule(bool preempt) · 2 unmodified lines
4523	5020	rq = context_switch(rq, prev, next, &rf);
4524	5021	} else {
4525	5022	rq->clock_update_flags &= ~(RQCF_ACT_SKIP\|RQCF_REQ_SKIP);
4526		− rq_unlock_irq(rq, &rf);
4527		− }
4528	5023
4529		− balance_callback(rq);
	5024	+ rq_unpin_lock(rq, &rf);
	5025	+ __balance_callbacks(rq);
	5026	+ raw_spin_unlock_irq(&rq->lock);
	5027	+ }
4530	5028	}
4531	5029
4532	5030	void __noreturn do_task_dead(void)
@@ -4937,9 +5435,11 @@ void rt_mutex_setprio(struct task_struct p, struct task_struct pi_task) · 404 unmodified lines
4937	5435	out_unlock:
4938	5436	/* Avoid rq from going away on us: */
4939	5437	preempt_disable();
4940		− __task_rq_unlock(rq, &rf);
4941	5438
4942		− balance_callback(rq);
	5439	+ rq_unpin_lock(rq, &rf);
	5440	+ __balance_callbacks(rq);
	5441	+ raw_spin_unlock(&rq->lock);
	5442	+
4943	5443	preempt_enable();
4944	5444	}
4945	5445	#else
@@ -5213,6 +5713,7 @@ static int __sched_setscheduler(struct task_struct *p, · 267 unmodified lines
5213	5713	int retval, oldprio, oldpolicy = -1, queued, running;
5214	5714	int new_effective_prio, policy = attr->sched_policy;
5215	5715	const struct sched_class *prev_class;
	5716	+ struct callback_head *head;
5216	5717	struct rq_flags rf;
5217	5718	int reset_on_fork;
5218	5719	int queue_flags = DEQUEUE_SAVE \| DEQUEUE_MOVE \| DEQUEUE_NOCLOCK;
@@ -5451,6 +5952,7 @@ change: · 232 unmodified lines
5451	5952
5452	5953	/* Avoid rq from going away on us: */
5453	5954	preempt_disable();
	5955	+ head = splice_balance_callbacks(rq);
5454	5956	task_rq_unlock(rq, p, &rf);
5455	5957
5456	5958	if (pi) {
@@ -5459,7 +5961,7 @@ change: · 2 unmodified lines
5459	5961	}
5460	5962
5461	5963	/* Run balance callbacks after we've adjusted the PI chain: */
5462		− balance_callback(rq);
	5964	+ balance_callbacks(rq, head);
5463	5965	preempt_enable();
5464	5966
5465	5967	return 0;
@@ -5954,7 +6456,7 @@ long sched_setaffinity(pid_t pid, const struct cpumask *in_mask) · 488 unmodified lines
5954	6456	}
5955	6457	#endif
5956	6458	again:
5957		− retval = __set_cpus_allowed_ptr(p, new_mask, true);
	6459	+ retval = __set_cpus_allowed_ptr(p, new_mask, SCA_CHECK);
5958	6460
5959	6461	if (!retval) {
5960	6462	cpuset_cpus_allowed(p, cpus_allowed);
@@ -6443,6 +6945,7 @@ void sched_show_task(struct task_struct *p) · 482 unmodified lines
6443	6945	(unsigned long)task_thread_info(p)->flags);
6444	6946
6445	6947	print_worker_info(KERN_INFO, p);
	6948	+ print_stop_info(KERN_INFO, p);
6446	6949	show_stack(p, NULL, KERN_INFO);
6447	6950	put_task_stack(p);
6448	6951	}
@@ -6533,7 +7036,7 @@ void init_idle(struct task_struct *idle, int cpu) · 84 unmodified lines
6533	7036	*
6534	7037	* And since this is boot we can forgo the serialization.
6535	7038	*/
6536		− set_cpus_allowed_common(idle, cpumask_of(cpu));
	7039	+ set_cpus_allowed_common(idle, cpumask_of(cpu), 0);
6537	7040	#endif
6538	7041	/*
6539	7042	* We're having a chicken and egg problem, even though we are
@@ -6684,119 +7187,126 @@ void idle_task_exit(void) · 144 unmodified lines
6684	7187	/* finish_cpu(), as ran on the BP, will clean up the active_mm state */
6685	7188	}
6686	7189
6687		−/*
6688		− * Since this CPU is going 'away' for a while, fold any nr_active delta
6689		− * we might have. Assumes we're called after migrate_tasks() so that the
6690		− * nr_active count is stable. We need to take the teardown thread which
6691		− * is calling this into account, so we hand in adjust = 1 to the load
6692		− * calculation.
6693		− *
6694		− * Also see the comment "Global load-average calculations".
6695		− */
6696		−static void calc_load_migrate(struct rq *rq)
	7190	+static int __balance_push_cpu_stop(void *arg)
6697	7191	{
6698		− long delta = calc_load_fold_active(rq, 1);
6699		− if (delta)
6700		− atomic_long_add(delta, &calc_load_tasks);
6701		−}
	7192	+ struct task_struct *p = arg;
	7193	+ struct rq *rq = this_rq();
	7194	+ struct rq_flags rf;
	7195	+ int cpu;
6702	7196
6703		−static struct task_struct __pick_migrate_task(struct rq rq)
6704		−{
6705		− const struct sched_class *class;
6706		− struct task_struct *next;
	7197	+ raw_spin_lock_irq(&p->pi_lock);
	7198	+ rq_lock(rq, &rf);
6707	7199
6708		− for_each_class(class) {
6709		− next = class->pick_next_task(rq);
6710		− if (next) {
6711		− next->sched_class->put_prev_task(rq, next);
6712		− return next;
6713		− }
	7200	+ update_rq_clock(rq);
	7201	+
	7202	+ if (task_rq(p) == rq && task_on_rq_queued(p)) {
	7203	+ cpu = select_fallback_rq(rq->cpu, p);
	7204	+ rq = __migrate_task(rq, &rf, p, cpu);
6714	7205	}
6715	7206
6716		− /* The idle class should always have a runnable task */
6717		− BUG();
	7207	+ rq_unlock(rq, &rf);
	7208	+ raw_spin_unlock_irq(&p->pi_lock);
	7209	+
	7210	+ put_task_struct(p);
	7211	+
	7212	+ return 0;
6718	7213	}
6719	7214
	7215	+static DEFINE_PER_CPU(struct cpu_stop_work, push_work);
	7216	+
6720	7217	/*
6721		− * Migrate all tasks from the rq, sleeping tasks will be migrated by
6722		− * try_to_wake_up()->select_task_rq().
6723		− *
6724		− * Called with rq->lock held even though we'er in stop_machine() and
6725		− * there's no concurrency possible, we hold the required locks anyway
6726		− * because of lock validation efforts.
	7218	+ * Ensure we only run per-cpu kthreads once the CPU goes !active.
6727	7219	*/
6728		−static void migrate_tasks(struct rq dead_rq, struct rq_flags rf)
	7220	+static void balance_push(struct rq *rq)
6729	7221	{
6730		− struct rq *rq = dead_rq;
6731		− struct task_struct next, stop = rq->stop;
6732		− struct rq_flags orf = *rf;
6733		− int dest_cpu;
	7222	+ struct task_struct *push_task = rq->curr;
	7223	+
	7224	+ lockdep_assert_held(&rq->lock);
	7225	+ SCHED_WARN_ON(rq->cpu != smp_processor_id());
6734	7226
6735	7227	/*
6736		− * Fudge the rq selection such that the below task selection loop
6737		− * doesn't get stuck on the currently eligible stop task.
6738		− *
6739		− * We're currently inside stop_machine() and the rq is either stuck
6740		− * in the stop_machine_cpu_stop() loop, or we're executing this code,
6741		− * either way we should never end up calling schedule() until we're
6742		− * done here.
	7228	+ * Both the cpu-hotplug and stop task are in this case and are
	7229	+ * required to complete the hotplug process.
6743	7230	*/
6744		− rq->stop = NULL;
	7231	+ if (is_per_cpu_kthread(push_task) \|\| is_migration_disabled(push_task)) {
	7232	+ /*
	7233	+ * If this is the idle task on the outgoing CPU try to wake
	7234	+ * up the hotplug control thread which might wait for the
	7235	+ * last task to vanish. The rcuwait_active() check is
	7236	+ * accurate here because the waiter is pinned on this CPU
	7237	+ * and can't obviously be running in parallel.
	7238	+ *
	7239	+ * On RT kernels this also has to check whether there are
	7240	+ * pinned and scheduled out tasks on the runqueue. They
	7241	+ * need to leave the migrate disabled section first.
	7242	+ */
	7243	+ if (!rq->nr_running && !rq_has_pinned_tasks(rq) &&
	7244	+ rcuwait_active(&rq->hotplug_wait)) {
	7245	+ raw_spin_unlock(&rq->lock);
	7246	+ rcuwait_wake_up(&rq->hotplug_wait);
	7247	+ raw_spin_lock(&rq->lock);
	7248	+ }
	7249	+ return;
	7250	+ }
6745	7251
	7252	+ get_task_struct(push_task);
6746	7253	/*
6747		− * put_prev_task() and pick_next_task() sched
6748		− * class method both need to have an up-to-date
6749		− * value of rq->clock[_task]
	7254	+ * Temporarily drop rq->lock such that we can wake-up the stop task.
	7255	+ * Both preemption and IRQs are still disabled.
6750	7256	*/
6751		− update_rq_clock(rq);
	7257	+ raw_spin_unlock(&rq->lock);
	7258	+ stop_one_cpu_nowait(rq->cpu, __balance_push_cpu_stop, push_task,
	7259	+ this_cpu_ptr(&push_work));
	7260	+ /*
	7261	+ * At this point need_resched() is true and we'll take the loop in
	7262	+ * schedule(). The next pick is obviously going to be the stop task
	7263	+ * which is_per_cpu_kthread() and will push this task away.
	7264	+ */
	7265	+ raw_spin_lock(&rq->lock);
	7266	+}
6752	7267
6753		− for (;;) {
6754		− /*
6755		− * There's this thread running, bail when that's the only
6756		− * remaining thread:
6757		− */
6758		− if (rq->nr_running == 1)
6759		− break;
	7268	+static void balance_push_set(int cpu, bool on)
	7269	+{
	7270	+ struct rq *rq = cpu_rq(cpu);
	7271	+ struct rq_flags rf;
6760	7272
6761		− next = __pick_migrate_task(rq);
	7273	+ rq_lock_irqsave(rq, &rf);
	7274	+ if (on)
	7275	+ rq->balance_flags \|= BALANCE_PUSH;
	7276	+ else
	7277	+ rq->balance_flags &= ~BALANCE_PUSH;
	7278	+ rq_unlock_irqrestore(rq, &rf);
	7279	+}
6762	7280
6763		− /*
6764		− * Rules for changing task_struct::cpus_mask are holding
6765		− * both pi_lock and rq->lock, such that holding either
6766		− * stabilizes the mask.
6767		− *
6768		− * Drop rq->lock is not quite as disastrous as it usually is
6769		− * because !cpu_active at this point, which means load-balance
6770		− * will not interfere. Also, stop-machine.
6771		− */
6772		− rq_unlock(rq, rf);
6773		− raw_spin_lock(&next->pi_lock);
6774		− rq_relock(rq, rf);
	7281	+/*
	7282	+ * Invoked from a CPUs hotplug control thread after the CPU has been marked
	7283	+ * inactive. All tasks which are not per CPU kernel threads are either
	7284	+ * pushed off this CPU now via balance_push() or placed on a different CPU
	7285	+ * during wakeup. Wait until the CPU is quiescent.
	7286	+ */
	7287	+static void balance_hotplug_wait(void)
	7288	+{
	7289	+ struct rq *rq = this_rq();
6775	7290
6776		− /*
6777		− * Since we're inside stop-machine, _nothing_ should have
6778		− * changed the task, WARN if weird stuff happened, because in
6779		− * that case the above rq->lock drop is a fail too.
6780		− */
6781		− if (WARN_ON(task_rq(next) != rq \|\| !task_on_rq_queued(next))) {
6782		− raw_spin_unlock(&next->pi_lock);
6783		− continue;
6784		− }
	7291	+ rcuwait_wait_event(&rq->hotplug_wait,
	7292	+ rq->nr_running == 1 && !rq_has_pinned_tasks(rq),
	7293	+ TASK_UNINTERRUPTIBLE);
	7294	+}
6785	7295
6786		− /* Find suitable destination for @next, with force if needed. */
6787		− dest_cpu = select_fallback_rq(dead_rq->cpu, next);
6788		− rq = __migrate_task(rq, rf, next, dest_cpu);
6789		− if (rq != dead_rq) {
6790		− rq_unlock(rq, rf);
6791		− rq = dead_rq;
6792		− *rf = orf;
6793		− rq_relock(rq, rf);
6794		− }
6795		− raw_spin_unlock(&next->pi_lock);
6796		− }
	7296	+#else
6797	7297
6798		− rq->stop = stop;
	7298	+static inline void balance_push(struct rq *rq)
	7299	+{
6799	7300	}
	7301	+
	7302	+static inline void balance_push_set(int cpu, bool on)
	7303	+{
	7304	+}
	7305	+
	7306	+static inline void balance_hotplug_wait(void)
	7307	+{
	7308	+}
	7309	+
6800	7310	#endif /* CONFIG_HOTPLUG_CPU */
6801	7311
6802	7312	void set_rq_online(struct rq *rq)
@@ -6882,6 +7392,8 @@ int sched_cpu_activate(unsigned int cpu) · 79 unmodified lines
6882	7392	struct rq *rq = cpu_rq(cpu);
6883	7393	struct rq_flags rf;
6884	7394
	7395	+ balance_push_set(cpu, false);
	7396	+
6885	7397	#ifdef CONFIG_SCHED_SMT
6886	7398	/*
6887	7399	* When going up, increment the number of cores with SMT present.
@@ -6917,6 +7429,8 @@ int sched_cpu_activate(unsigned int cpu) · 29 unmodified lines
6917	7429
6918	7430	int sched_cpu_deactivate(unsigned int cpu)
6919	7431	{
	7432	+ struct rq *rq = cpu_rq(cpu);
	7433	+ struct rq_flags rf;
6920	7434	int ret;
6921	7435
6922	7436	set_cpu_active(cpu, false);
@@ -6929,6 +7443,16 @@ int sched_cpu_deactivate(unsigned int cpu) · 6 unmodified lines
6929	7443	*/
6930	7444	synchronize_rcu();
6931	7445
	7446	+ balance_push_set(cpu, true);
	7447	+
	7448	+ rq_lock_irqsave(rq, &rf);
	7449	+ if (rq->rd) {
	7450	+ update_rq_clock(rq);
	7451	+ BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
	7452	+ set_rq_offline(rq);
	7453	+ }
	7454	+ rq_unlock_irqrestore(rq, &rf);
	7455	+
6932	7456	#ifdef CONFIG_SCHED_SMT
6933	7457	/*
6934	7458	* When going down, decrement the number of cores with SMT present.
@@ -6942,6 +7466,7 @@ int sched_cpu_deactivate(unsigned int cpu) · 7 unmodified lines
6942	7466
6943	7467	ret = cpuset_cpu_inactive(cpu);
6944	7468	if (ret) {
	7469	+ balance_push_set(cpu, false);
6945	7470	set_cpu_active(cpu, true);
6946	7471	return ret;
6947	7472	}
@@ -6965,6 +7490,41 @@ int sched_cpu_starting(unsigned int cpu) · 17 unmodified lines
6965	7490	}
6966	7491
6967	7492	#ifdef CONFIG_HOTPLUG_CPU
	7493	+
	7494	+/*
	7495	+ * Invoked immediately before the stopper thread is invoked to bring the
	7496	+ * CPU down completely. At this point all per CPU kthreads except the
	7497	+ * hotplug thread (current) and the stopper thread (inactive) have been
	7498	+ * either parked or have been unbound from the outgoing CPU. Ensure that
	7499	+ * any of those which might be on the way out are gone.
	7500	+ *
	7501	+ * If after this point a bound task is being woken on this CPU then the
	7502	+ * responsible hotplug callback has failed to do it's job.
	7503	+ * sched_cpu_dying() will catch it with the appropriate fireworks.
	7504	+ */
	7505	+int sched_cpu_wait_empty(unsigned int cpu)
	7506	+{
	7507	+ balance_hotplug_wait();
	7508	+ return 0;
	7509	+}
	7510	+
	7511	+/*
	7512	+ * Since this CPU is going 'away' for a while, fold any nr_active delta we
	7513	+ * might have. Called from the CPU stopper task after ensuring that the
	7514	+ * stopper is the last running task on the CPU, so nr_active count is
	7515	+ * stable. We need to take the teardown thread which is calling this into
	7516	+ * account, so we hand in adjust = 1 to the load calculation.
	7517	+ *
	7518	+ * Also see the comment "Global load-average calculations".
	7519	+ */
	7520	+static void calc_load_migrate(struct rq *rq)
	7521	+{
	7522	+ long delta = calc_load_fold_active(rq, 1);
	7523	+
	7524	+ if (delta)
	7525	+ atomic_long_add(delta, &calc_load_tasks);
	7526	+}
	7527	+
6968	7528	int sched_cpu_dying(unsigned int cpu)
6969	7529	{
6970	7530	struct rq *rq = cpu_rq(cpu);
@@ -6974,12 +7534,7 @@ int sched_cpu_dying(unsigned int cpu) · 3 unmodified lines
6974	7534	sched_tick_stop(cpu);
6975	7535
6976	7536	rq_lock_irqsave(rq, &rf);
6977		− if (rq->rd) {
6978		− BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
6979		− set_rq_offline(rq);
6980		− }
6981		− migrate_tasks(rq, &rf);
6982		− BUG_ON(rq->nr_running != 1);
	7537	+ BUG_ON(rq->nr_running != 1 \|\| rq_has_pinned_tasks(rq));
6983	7538	rq_unlock_irqrestore(rq, &rf);
6984	7539
6985	7540	calc_load_migrate(rq);
@@ -7186,6 +7741,9 @@ void __init sched_init(void) · 200 unmodified lines
7186	7741
7187	7742	rq_csd_init(rq, &rq->nohz_csd, nohz_csd_func);
7188	7743	#endif
	7744	+#ifdef CONFIG_HOTPLUG_CPU
	7745	+ rcuwait_init(&rq->hotplug_wait);
	7746	+#endif
7189	7747	#endif /* CONFIG_SMP */
7190	7748	hrtick_rq_init(rq);
7191	7749	atomic_set(&rq->nr_iowait, 0);

kernel/sched/cpudeadline.c+2−2

@@ -120,7 +120,7 @@ int cpudl_find(struct cpudl cp, struct task_struct p, · 119 unmodified lines
120	120	const struct sched_dl_entity *dl_se = &p->dl;
121	121
122	122	if (later_mask &&
123		− cpumask_and(later_mask, cp->free_cpus, p->cpus_ptr)) {
	123	+ cpumask_and(later_mask, cp->free_cpus, &p->cpus_mask)) {
124	124	unsigned long cap, max_cap = 0;
125	125	int cpu, max_cpu = -1;
126	126
@@ -151,7 +151,7 @@ int cpudl_find(struct cpudl cp, struct task_struct p, · 24 unmodified lines
151	151
152	152	WARN_ON(best_cpu != -1 && !cpu_present(best_cpu));
153	153
154		− if (cpumask_test_cpu(best_cpu, p->cpus_ptr) &&
	154	+ if (cpumask_test_cpu(best_cpu, &p->cpus_mask) &&
155	155	dl_time_before(dl_se->deadline, cp->elements[0].dl)) {
156	156	if (later_mask)
157	157	cpumask_set_cpu(best_cpu, later_mask);

kernel/sched/cpupri.c+2−2

@@ -97,11 +97,11 @@ static inline int __cpupri_find(struct cpupri cp, struct task_struct p, · 96 unmodified lines
97	97	if (skip)
98	98	return 0;
99	99
100		− if (cpumask_any_and(p->cpus_ptr, vec->mask) >= nr_cpu_ids)
	100	+ if (cpumask_any_and(&p->cpus_mask, vec->mask) >= nr_cpu_ids)
101	101	return 0;
102	102
103	103	if (lowest_mask) {
104		− cpumask_and(lowest_mask, p->cpus_ptr, vec->mask);
	104	+ cpumask_and(lowest_mask, &p->cpus_mask, vec->mask);
105	105
106	106	/*
107	107	* We have to ensure that we have at least one bit

kernel/sched/deadline.c+31−15

@@ -559,7 +559,7 @@ static int push_dl_task(struct rq *rq); · 558 unmodified lines
559	559
560	560	static inline bool need_pull_dl_task(struct rq rq, struct task_struct prev)
561	561	{
562		− return dl_task(prev);
	562	+ return rq->online && dl_task(prev);
563	563	}
564	564
565	565	static DEFINE_PER_CPU(struct callback_head, dl_push_head);
@@ -1931,7 +1931,7 @@ static void task_fork_dl(struct task_struct *p) · 1365 unmodified lines
1931	1931	static int pick_dl_task(struct rq rq, struct task_struct p, int cpu)
1932	1932	{
1933	1933	if (!task_running(rq, p) &&
1934		− cpumask_test_cpu(cpu, p->cpus_ptr))
	1934	+ cpumask_test_cpu(cpu, &p->cpus_mask))
1935	1935	return 1;
1936	1936	return 0;
1937	1937	}
@@ -2021,8 +2021,8 @@ static int find_later_rq(struct task_struct *task) · 83 unmodified lines
2021	2021	return this_cpu;
2022	2022	}
2023	2023
2024		− best_cpu = cpumask_first_and(later_mask,
2025		− sched_domain_span(sd));
	2024	+ best_cpu = cpumask_any_and_distribute(later_mask,
	2025	+ sched_domain_span(sd));
2026	2026	/*
2027	2027	* Last chance: if a CPU being in both later_mask
2028	2028	* and current sd span is valid, that becomes our
@@ -2044,7 +2044,7 @@ static int find_later_rq(struct task_struct *task) · 15 unmodified lines
2044	2044	if (this_cpu != -1)
2045	2045	return this_cpu;
2046	2046
2047		− cpu = cpumask_any(later_mask);
	2047	+ cpu = cpumask_any_distribute(later_mask);
2048	2048	if (cpu < nr_cpu_ids)
2049	2049	return cpu;
2050	2050
@@ -2081,7 +2081,7 @@ static struct rq find_lock_later_rq(struct task_struct task, struct rq *rq) · 30 unmodified lines
2081	2081	/* Retry if something changed. */
2082	2082	if (double_lock_balance(rq, later_rq)) {
2083	2083	if (unlikely(task_rq(task) != rq \|\|
2084		− !cpumask_test_cpu(later_rq->cpu, task->cpus_ptr) \|\|
	2084	+ !cpumask_test_cpu(later_rq->cpu, &task->cpus_mask) \|\|
2085	2085	task_running(rq, task) \|\|
2086	2086	!dl_task(task) \|\|
2087	2087	!task_on_rq_queued(task))) {
@@ -2148,6 +2148,9 @@ static int push_dl_task(struct rq *rq) · 60 unmodified lines
2148	2148	return 0;
2149	2149
2150	2150	retry:
	2151	+ if (is_migration_disabled(next_task))
	2152	+ return 0;
	2153	+
2151	2154	if (WARN_ON(next_task == rq->curr))
2152	2155	return 0;
2153	2156
@@ -2225,7 +2228,7 @@ static void push_dl_tasks(struct rq *rq) · 71 unmodified lines
2225	2228	static void pull_dl_task(struct rq *this_rq)
2226	2229	{
2227	2230	int this_cpu = this_rq->cpu, cpu;
2228		− struct task_struct *p;
	2231	+ struct task_struct p, push_task;
2229	2232	bool resched = false;
2230	2233	struct rq *src_rq;
2231	2234	u64 dmin = LONG_MAX;
@@ -2255,6 +2258,7 @@ static void pull_dl_task(struct rq *this_rq) · 23 unmodified lines
2255	2258	continue;
2256	2259
2257	2260	/* Might drop this_rq->lock */
	2261	+ push_task = NULL;
2258	2262	double_lock_balance(this_rq, src_rq);
2259	2263
2260	2264	/*
@@ -2286,17 +2290,27 @@ static void pull_dl_task(struct rq *this_rq) · 25 unmodified lines
2286	2290	src_rq->curr->dl.deadline))
2287	2291	goto skip;
2288	2292
2289		− resched = true;
2290		−
2291		− deactivate_task(src_rq, p, 0);
2292		− set_task_cpu(p, this_cpu);
2293		− activate_task(this_rq, p, 0);
2294		− dmin = p->dl.deadline;
	2293	+ if (is_migration_disabled(p)) {
	2294	+ push_task = get_push_task(src_rq);
	2295	+ } else {
	2296	+ deactivate_task(src_rq, p, 0);
	2297	+ set_task_cpu(p, this_cpu);
	2298	+ activate_task(this_rq, p, 0);
	2299	+ dmin = p->dl.deadline;
	2300	+ resched = true;
	2301	+ }
2295	2302
2296	2303	/* Is there any other task even earlier? */
2297	2304	}
2298	2305	skip:
2299	2306	double_unlock_balance(this_rq, src_rq);
	2307	+
	2308	+ if (push_task) {
	2309	+ raw_spin_unlock(&this_rq->lock);
	2310	+ stop_one_cpu_nowait(src_rq->cpu, push_cpu_stop,
	2311	+ push_task, &src_rq->push_work);
	2312	+ raw_spin_lock(&this_rq->lock);
	2313	+ }
2300	2314	}
2301	2315
2302	2316	if (resched)
@@ -2320,7 +2334,8 @@ static void task_woken_dl(struct rq rq, struct task_struct p) · 17 unmodified lines
2320	2334	}
2321	2335
2322	2336	static void set_cpus_allowed_dl(struct task_struct *p,
2323		− const struct cpumask *new_mask)
	2337	+ const struct cpumask *new_mask,
	2338	+ u32 flags)
2324	2339	{
2325	2340	struct root_domain *src_rd;
2326	2341	struct rq *rq;
@@ -2349,7 +2364,7 @@ static void set_cpus_allowed_dl(struct task_struct *p, · 22 unmodified lines
2349	2364	raw_spin_unlock(&src_dl_b->lock);
2350	2365	}
2351	2366
2352		− set_cpus_allowed_common(p, new_mask);
	2367	+ set_cpus_allowed_common(p, new_mask, flags);
2353	2368	}
2354	2369
2355	2370	/* Assumes rq->lock is held */
@@ -2542,6 +2557,7 @@ DEFINE_SCHED_CLASS(dl) = { · 186 unmodified lines
2542	2557	.rq_online = rq_online_dl,
2543	2558	.rq_offline = rq_offline_dl,
2544	2559	.task_woken = task_woken_dl,
	2560	+ .find_lock_rq = find_lock_later_rq,
2545	2561	#endif
2546	2562
2547	2563	.task_tick = task_tick_dl,

kernel/sched/rt.c+57−18

@@ -265,7 +265,7 @@ static void pull_rt_task(struct rq *this_rq); · 264 unmodified lines
265	265	static inline bool need_pull_rt_task(struct rq rq, struct task_struct prev)
266	266	{
267	267	/* Try to pull RT tasks here if we lower this rq's prio */
268		− return rq->rt.highest_prio.curr > prev->prio;
	268	+ return rq->online && rq->rt.highest_prio.curr > prev->prio;
269	269	}
270	270
271	271	static inline int rt_overloaded(struct rq *rq)
@@ -1660,7 +1660,7 @@ static void put_prev_task_rt(struct rq rq, struct task_struct p) · 1388 unmodified lines
1660	1660	static int pick_rt_task(struct rq rq, struct task_struct p, int cpu)
1661	1661	{
1662	1662	if (!task_running(rq, p) &&
1663		− cpumask_test_cpu(cpu, p->cpus_ptr))
	1663	+ cpumask_test_cpu(cpu, &p->cpus_mask))
1664	1664	return 1;
1665	1665
1666	1666	return 0;
@@ -1754,8 +1754,8 @@ static int find_lowest_rq(struct task_struct *task) · 87 unmodified lines
1754	1754	return this_cpu;
1755	1755	}
1756	1756
1757		− best_cpu = cpumask_first_and(lowest_mask,
1758		− sched_domain_span(sd));
	1757	+ best_cpu = cpumask_any_and_distribute(lowest_mask,
	1758	+ sched_domain_span(sd));
1759	1759	if (best_cpu < nr_cpu_ids) {
1760	1760	rcu_read_unlock();
1761	1761	return best_cpu;
@@ -1772,7 +1772,7 @@ static int find_lowest_rq(struct task_struct *task) · 10 unmodified lines
1772	1772	if (this_cpu != -1)
1773	1773	return this_cpu;
1774	1774
1775		− cpu = cpumask_any(lowest_mask);
	1775	+ cpu = cpumask_any_distribute(lowest_mask);
1776	1776	if (cpu < nr_cpu_ids)
1777	1777	return cpu;
1778	1778
@@ -1813,7 +1813,7 @@ static struct rq find_lock_lowest_rq(struct task_struct task, struct rq *rq) · 34 unmodified lines
1813	1813	* Also make sure that it wasn't scheduled on its rq.
1814	1814	*/
1815	1815	if (unlikely(task_rq(task) != rq \|\|
1816		− !cpumask_test_cpu(lowest_rq->cpu, task->cpus_ptr) \|\|
	1816	+ !cpumask_test_cpu(lowest_rq->cpu, &task->cpus_mask) \|\|
1817	1817	task_running(rq, task) \|\|
1818	1818	!rt_task(task) \|\|
1819	1819	!task_on_rq_queued(task))) {
@@ -1861,7 +1861,7 @@ static struct task_struct pick_next_pushable_task(struct rq rq) · 41 unmodified lines
1861	1861	* running task can migrate over to a CPU that is running a task
1862	1862	* of lesser priority.
1863	1863	*/
1864		−static int push_rt_task(struct rq *rq)
	1864	+static int push_rt_task(struct rq *rq, bool pull)
1865	1865	{
1866	1866	struct task_struct *next_task;
1867	1867	struct rq *lowest_rq;
@@ -1875,6 +1875,34 @@ static int push_rt_task(struct rq *rq) · 7 unmodified lines
1875	1875	return 0;
1876	1876
1877	1877	retry:
	1878	+ if (is_migration_disabled(next_task)) {
	1879	+ struct task_struct *push_task = NULL;
	1880	+ int cpu;
	1881	+
	1882	+ if (!pull \|\| rq->push_busy)
	1883	+ return 0;
	1884	+
	1885	+ cpu = find_lowest_rq(rq->curr);
	1886	+ if (cpu == -1 \|\| cpu == rq->cpu)
	1887	+ return 0;
	1888	+
	1889	+ /*
	1890	+ * Given we found a CPU with lower priority than @next_task,
	1891	+ * therefore it should be running. However we cannot migrate it
	1892	+ * to this other CPU, instead attempt to push the current
	1893	+ * running task on this CPU away.
	1894	+ */
	1895	+ push_task = get_push_task(rq);
	1896	+ if (push_task) {
	1897	+ raw_spin_unlock(&rq->lock);
	1898	+ stop_one_cpu_nowait(rq->cpu, push_cpu_stop,
	1899	+ push_task, &rq->push_work);
	1900	+ raw_spin_lock(&rq->lock);
	1901	+ }
	1902	+
	1903	+ return 0;
	1904	+ }
	1905	+
1878	1906	if (WARN_ON(next_task == rq->curr))
1879	1907	return 0;
1880	1908
@@ -1929,12 +1957,10 @@ retry: · 48 unmodified lines
1929	1957	deactivate_task(rq, next_task, 0);
1930	1958	set_task_cpu(next_task, lowest_rq->cpu);
1931	1959	activate_task(lowest_rq, next_task, 0);
1932		− ret = 1;
1933		−
1934	1960	resched_curr(lowest_rq);
	1961	+ ret = 1;
1935	1962
1936	1963	double_unlock_balance(rq, lowest_rq);
1937		−
1938	1964	out:
1939	1965	put_task_struct(next_task);
1940	1966
@@ -1944,7 +1970,7 @@ out: · 3 unmodified lines
1944	1970	static void push_rt_tasks(struct rq *rq)
1945	1971	{
1946	1972	/* push_rt_task will return true if it moved an RT */
1947		− while (push_rt_task(rq))
	1973	+ while (push_rt_task(rq, false))
1948	1974	;
1949	1975	}
1950	1976
@@ -2097,7 +2123,8 @@ void rto_push_irq_work_func(struct irq_work *work) · 146 unmodified lines
2097	2123	*/
2098	2124	if (has_pushable_tasks(rq)) {
2099	2125	raw_spin_lock(&rq->lock);
2100		− push_rt_tasks(rq);
	2126	+ while (push_rt_task(rq, true))
	2127	+ ;
2101	2128	raw_spin_unlock(&rq->lock);
2102	2129	}
2103	2130
@@ -2122,7 +2149,7 @@ static void pull_rt_task(struct rq *this_rq) · 18 unmodified lines
2122	2149	{
2123	2150	int this_cpu = this_rq->cpu, cpu;
2124	2151	bool resched = false;
2125		− struct task_struct *p;
	2152	+ struct task_struct p, push_task;
2126	2153	struct rq *src_rq;
2127	2154	int rt_overload_count = rt_overloaded(this_rq);
2128	2155
@@ -2169,6 +2196,7 @@ static void pull_rt_task(struct rq *this_rq) · 40 unmodified lines
2169	2196	* double_lock_balance, and another CPU could
2170	2197	* alter this_rq
2171	2198	*/
	2199	+ push_task = NULL;
2172	2200	double_lock_balance(this_rq, src_rq);
2173	2201
2174	2202	/*
@@ -2196,11 +2224,14 @@ static void pull_rt_task(struct rq *this_rq) · 21 unmodified lines
2196	2224	if (p->prio < src_rq->curr->prio)
2197	2225	goto skip;
2198	2226
2199		− resched = true;
2200		−
2201		− deactivate_task(src_rq, p, 0);
2202		− set_task_cpu(p, this_cpu);
2203		− activate_task(this_rq, p, 0);
	2227	+ if (is_migration_disabled(p)) {
	2228	+ push_task = get_push_task(src_rq);
	2229	+ } else {
	2230	+ deactivate_task(src_rq, p, 0);
	2231	+ set_task_cpu(p, this_cpu);
	2232	+ activate_task(this_rq, p, 0);
	2233	+ resched = true;
	2234	+ }
2204	2235	/*
2205	2236	* We continue with the search, just in
2206	2237	* case there's an even higher prio task
@@ -2210,6 +2241,13 @@ static void pull_rt_task(struct rq *this_rq) · 3 unmodified lines
2210	2241	}
2211	2242	skip:
2212	2243	double_unlock_balance(this_rq, src_rq);
	2244	+
	2245	+ if (push_task) {
	2246	+ raw_spin_unlock(&this_rq->lock);
	2247	+ stop_one_cpu_nowait(src_rq->cpu, push_cpu_stop,
	2248	+ push_task, &src_rq->push_work);
	2249	+ raw_spin_lock(&this_rq->lock);
	2250	+ }
2213	2251	}
2214	2252
2215	2253	if (resched)
@@ -2451,6 +2489,7 @@ DEFINE_SCHED_CLASS(rt) = { · 235 unmodified lines
2451	2489	.rq_offline = rq_offline_rt,
2452	2490	.task_woken = task_woken_rt,
2453	2491	.switched_from = switched_from_rt,
	2492	+ .find_lock_rq = find_lock_lowest_rq,
2454	2493	#endif
2455	2494
2456	2495	.task_tick = task_tick_rt,

kernel/sched/sched.h+56−3

@@ -975,6 +975,7 @@ struct rq { · 974 unmodified lines
975	975	unsigned long cpu_capacity_orig;
976	976
977	977	struct callback_head *balance_callback;
	978	+ unsigned char balance_flags;
978	979
979	980	unsigned char nohz_idle_balance;
980	981	unsigned char idle_balance;
@@ -1005,6 +1006,10 @@ struct rq { · 24 unmodified lines
1005	1006
1006	1007	/* This is used to determine avg_idle's max value */
1007	1008	u64 max_idle_balance_cost;
	1009	+
	1010	+#ifdef CONFIG_HOTPLUG_CPU
	1011	+ struct rcuwait hotplug_wait;
	1012	+#endif
1008	1013	#endif /* CONFIG_SMP */
1009	1014
1010	1015	#ifdef CONFIG_IRQ_TIME_ACCOUNTING
@@ -1050,6 +1055,12 @@ struct rq { · 39 unmodified lines
1050	1055	/* Must be inspected within a rcu lock section */
1051	1056	struct cpuidle_state *idle_state;
1052	1057	#endif
	1058	+
	1059	+#if defined(CONFIG_PREEMPT_RT) && defined(CONFIG_SMP)
	1060	+ unsigned int nr_pinned;
	1061	+#endif
	1062	+ unsigned int push_busy;
	1063	+ struct cpu_stop_work push_work;
1053	1064	};
1054	1065
1055	1066	#ifdef CONFIG_FAIR_GROUP_SCHED
@@ -1077,6 +1088,16 @@ static inline int cpu_of(struct rq *rq) · 21 unmodified lines
1077	1088	#endif
1078	1089	}
1079	1090
	1091	+#define MDF_PUSH 0x01
	1092	+
	1093	+static inline bool is_migration_disabled(struct task_struct *p)
	1094	+{
	1095	+#if defined(CONFIG_SMP) && defined(CONFIG_PREEMPT_RT)
	1096	+ return p->migration_disabled;
	1097	+#else
	1098	+ return false;
	1099	+#endif
	1100	+}
1080	1101
1081	1102	#ifdef CONFIG_SCHED_SMT
1082	1103	extern void __update_idle_core(struct rq *rq);
@@ -1223,6 +1244,9 @@ static inline void rq_pin_lock(struct rq rq, struct rq_flags rf) · 140 unmodified lines
1223	1244	rq->clock_update_flags &= (RQCF_REQ_SKIP\|RQCF_ACT_SKIP);
1224	1245	rf->clock_update_flags = 0;
1225	1246	#endif
	1247	+#ifdef CONFIG_SMP
	1248	+ SCHED_WARN_ON(rq->balance_callback);
	1249	+#endif
1226	1250	}
1227	1251
1228	1252	static inline void rq_unpin_lock(struct rq rq, struct rq_flags rf)
@@ -1384,6 +1408,9 @@ init_numa_balancing(unsigned long clone_flags, struct task_struct *p) · 155 unmodified lines
1384	1408
1385	1409	#ifdef CONFIG_SMP
1386	1410
	1411	+#define BALANCE_WORK 0x01
	1412	+#define BALANCE_PUSH 0x02
	1413	+
1387	1414	static inline void
1388	1415	queue_balance_callback(struct rq *rq,
1389	1416	struct callback_head *head,
@@ -1391,12 +1418,13 @@ queue_balance_callback(struct rq *rq, · 1 unmodified line
1391	1418	{
1392	1419	lockdep_assert_held(&rq->lock);
1393	1420
1394		− if (unlikely(head->next))
	1421	+ if (unlikely(head->next \|\| (rq->balance_flags & BALANCE_PUSH)))
1395	1422	return;
1396	1423
1397	1424	head->func = (void ()(struct callback_head ))func;
1398	1425	head->next = rq->balance_callback;
1399	1426	rq->balance_callback = head;
	1427	+ rq->balance_flags \|= BALANCE_WORK;
1400	1428	}
1401	1429
1402	1430	#define rcu_dereference_check_sched_domain(p) \
@@ -1804,10 +1832,13 @@ struct sched_class { · 401 unmodified lines
1804	1832	void (task_woken)(struct rq this_rq, struct task_struct *task);
1805	1833
1806	1834	void (set_cpus_allowed)(struct task_struct p,
1807		− const struct cpumask *newmask);
	1835	+ const struct cpumask *newmask,
	1836	+ u32 flags);
1808	1837
1809	1838	void (rq_online)(struct rq rq);
1810	1839	void (rq_offline)(struct rq rq);
	1840	+
	1841	+ struct rq (find_lock_rq)(struct task_struct p, struct rq rq);
1811	1842	#endif
1812	1843
1813	1844	void (task_tick)(struct rq rq, struct task_struct *p, int queued);
@@ -1905,13 +1936,35 @@ static inline bool sched_fair_runnable(struct rq *rq) · 91 unmodified lines
1905	1936	extern struct task_struct pick_next_task_fair(struct rq rq, struct task_struct prev, struct rq_flags rf);
1906	1937	extern struct task_struct pick_next_task_idle(struct rq rq);
1907	1938
	1939	+#define SCA_CHECK 0x01
	1940	+#define SCA_MIGRATE_DISABLE 0x02
	1941	+#define SCA_MIGRATE_ENABLE 0x04
	1942	+
1908	1943	#ifdef CONFIG_SMP
1909	1944
1910	1945	extern void update_group_capacity(struct sched_domain *sd, int cpu);
1911	1946
1912	1947	extern void trigger_load_balance(struct rq *rq);
1913	1948
1914		−extern void set_cpus_allowed_common(struct task_struct p, const struct cpumask new_mask);
	1949	+extern void set_cpus_allowed_common(struct task_struct p, const struct cpumask new_mask, u32 flags);
	1950	+
	1951	+static inline struct task_struct get_push_task(struct rq rq)
	1952	+{
	1953	+ struct task_struct *p = rq->curr;
	1954	+
	1955	+ lockdep_assert_held(&rq->lock);
	1956	+
	1957	+ if (rq->push_busy)
	1958	+ return NULL;
	1959	+
	1960	+ if (p->nr_cpus_allowed == 1)
	1961	+ return NULL;
	1962	+
	1963	+ rq->push_busy = true;
	1964	+ return get_task_struct(p);
	1965	+}
	1966	+
	1967	+extern int push_cpu_stop(void *arg);
1915	1968
1916	1969	#endif
1917	1970

kernel/stop_machine.c+24−3

@@ -42,11 +42,27 @@ struct cpu_stopper { · 41 unmodified lines
42	42	struct list_head works; /* list of pending works */
43	43
44	44	struct cpu_stop_work stop_work; /* for stop_cpus */
	45	+ unsigned long caller;
	46	+ cpu_stop_fn_t fn;
45	47	};
46	48
47	49	static DEFINE_PER_CPU(struct cpu_stopper, cpu_stopper);
48	50	static bool stop_machine_initialized = false;
49	51
	52	+void print_stop_info(const char log_lvl, struct task_struct task)
	53	+{
	54	+ /*
	55	+ * If @task is a stopper task, it cannot migrate and task_cpu() is
	56	+ * stable.
	57	+ */
	58	+ struct cpu_stopper *stopper = per_cpu_ptr(&cpu_stopper, task_cpu(task));
	59	+
	60	+ if (task != stopper->thread)
	61	+ return;
	62	+
	63	+ printk("%sStopper: %pS <- %pS\n", log_lvl, stopper->fn, (void *)stopper->caller);
	64	+}
	65	+
50	66	/* static data for stop_cpus */
51	67	static DEFINE_MUTEX(stop_cpus_mutex);
52	68	static bool stop_cpus_in_progress;
@@ -123,7 +139,7 @@ static bool cpu_stop_queue_work(unsigned int cpu, struct cpu_stop_work *work) · 70 unmodified lines
123	139	int stop_one_cpu(unsigned int cpu, cpu_stop_fn_t fn, void *arg)
124	140	{
125	141	struct cpu_stop_done done;
126		− struct cpu_stop_work work = { .fn = fn, .arg = arg, .done = &done };
	142	+ struct cpu_stop_work work = { .fn = fn, .arg = arg, .done = &done, .caller = _RET_IP_ };
127	143
128	144	cpu_stop_init_done(&done, 1);
129	145	if (!cpu_stop_queue_work(cpu, &work))
@@ -331,7 +347,8 @@ int stop_two_cpus(unsigned int cpu1, unsigned int cpu2, cpu_stop_fn_t fn, void * · 201 unmodified lines
331	347	work1 = work2 = (struct cpu_stop_work){
332	348	.fn = multi_cpu_stop,
333	349	.arg = &msdata,
334		− .done = &done
	350	+ .done = &done,
	351	+ .caller = _RET_IP_,
335	352	};
336	353
337	354	cpu_stop_init_done(&done, 2);
@@ -367,7 +384,7 @@ int stop_two_cpus(unsigned int cpu1, unsigned int cpu2, cpu_stop_fn_t fn, void * · 29 unmodified lines
367	384	bool stop_one_cpu_nowait(unsigned int cpu, cpu_stop_fn_t fn, void *arg,
368	385	struct cpu_stop_work *work_buf)
369	386	{
370		− *work_buf = (struct cpu_stop_work){ .fn = fn, .arg = arg, };
	387	+ *work_buf = (struct cpu_stop_work){ .fn = fn, .arg = arg, .caller = _RET_IP_, };
371	388	return cpu_stop_queue_work(cpu, work_buf);
372	389	}
373	390
@@ -487,6 +504,8 @@ repeat: · 113 unmodified lines
487	504	int ret;
488	505
489	506	/* cpu stop callbacks must not sleep, make in_atomic() == T */
	507	+ stopper->caller = work->caller;
	508	+ stopper->fn = fn;
490	509	preempt_count_inc();
491	510	ret = fn(arg);
492	511	if (done) {
@@ -495,6 +514,8 @@ repeat: · 2 unmodified lines
495	514	cpu_stop_signal_done(done);
496	515	}
497	516	preempt_count_dec();
	517	+ stopper->fn = NULL;
	518	+ stopper->caller = 0;
498	519	WARN_ONCE(preempt_count(),
499	520	"cpu_stop: %ps(%p) leaked preempt count\n", fn, arg);
500	521	goto repeat;

kernel/workqueue.c+4−0

@@ -4908,6 +4908,10 @@ static void unbind_workers(int cpu) · 4907 unmodified lines
4908	4908	pool->flags \|= POOL_DISASSOCIATED;
4909	4909
4910	4910	raw_spin_unlock_irq(&pool->lock);
	4911	+
	4912	+ for_each_pool_worker(worker, pool)
	4913	+ WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, cpu_active_mask) < 0);
	4914	+
4911	4915	mutex_unlock(&wq_pool_attach_mutex);
4912	4916
4913	4917	/*

Merge branch 'sched/migrate-disable'

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code