Merge tag 'sched-core-2022-05-23' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull scheduler updates from Ingo Molnar: - Updates to scheduler metrics: - PELT fixes & enhancements - PSI fixes & enhancements - Refactor cpu_util_without() - Updates to instrumentation/debugging: - Remove sched_trace_*() helper functions - can be done via debug info - Fix double update_rq_clock() warnings - Introduce & use "preemption model accessors" to simplify some of the Kconfig complexity. - Make softirq handling RT-safe. - Misc smaller fixes & cleanups. * tag 'sched-core-2022-05-23' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: topology: Remove unused cpu_cluster_mask() sched: Reverse sched_class layout sched/deadline: Remove superfluous rq clock update in push_dl_task() sched/core: Avoid obvious double update_rq_clock warning smp: Make softirq handling RT safe in flush_smp_call_function_queue() smp: Rename flush_smp_call_function_from_idle() sched: Fix missing prototype warnings sched/fair: Remove cfs_rq_tg_path() sched/fair: Remove sched_trace_*() helper functions sched/fair: Refactor cpu_util_without() sched/fair: Revise comment about lb decision matrix sched/psi: report zeroes for CPU full at the system level sched/fair: Delete useless condition in tg_unthrottle_up() sched/fair: Fix cfs_rq_clock_pelt() for throttled cfs_rq sched/fair: Move calculate of avg_load to a better location mailmap: Update my email address to @redhat.com MAINTAINERS: Add myself as scheduler topology reviewer psi: Fix trigger being fired unexpectedly at initial ftrace: Use preemption model accessors for trace header printout kcsan: Use preemption model accessors

Linus Torvalds committed 4 years ago

Commit 6f3f04c2 parents cfeb252 991d8d823 files · +213 −331Browse files

Unified · Split

23 files

.mailmap +1 −0
Documentation/accounting/psi.rst +4 −5
MAINTAINERS +1 −0
include/asm-generic/vmlinux.lds.h +6 −6
include/linux/interrupt.h +9 −0
include/linux/sched.h +2 −14
include/linux/topology.h +0 −7
kernel/kcsan/kcsan_test.c +3 −2
kernel/sched/build_policy.c +2 −0
kernel/sched/build_utility.c +1 −0
kernel/sched/core.c +13 −10
kernel/sched/deadline.c +4 −11
kernel/sched/fair.c +67 −225
kernel/sched/idle.c +1 −1
kernel/sched/pelt.h +2 −2
kernel/sched/psi.c +11 −7
kernel/sched/rt.c +3 −2
kernel/sched/sched.h +36 −19
kernel/sched/smp.h +6 −0
kernel/smp.c +24 −8
kernel/softirq.c +13 −0
kernel/stop_machine.c +0 −2
kernel/trace/trace.c +4 −10

.mailmap+1−0

@@ -398,6 +398,7 @@ Vasily Averin <vasily.averin@linux.dev> <vvs@virtuozzo.com> · 397 unmodified lines
398	398	Vasily Averin <vasily.averin@linux.dev> <vvs@openvz.org>
399	399	Vasily Averin <vasily.averin@linux.dev> <vvs@parallels.com>
400	400	Vasily Averin <vasily.averin@linux.dev> <vvs@sw.ru>
	401	+Valentin Schneider <vschneid@redhat.com> <valentin.schneider@arm.com>
401	402	Vinod Koul <vkoul@kernel.org> <vinod.koul@intel.com>
402	403	Vinod Koul <vkoul@kernel.org> <vinod.koul@linux.intel.com>
403	404	Vinod Koul <vkoul@kernel.org> <vkoul@infradead.org>

Documentation/accounting/psi.rst+4−5

@@ -37,11 +37,7 @@ Pressure interface · 36 unmodified lines
37	37	Pressure information for each resource is exported through the
38	38	respective file in /proc/pressure/ -- cpu, memory, and io.
39	39
40		−The format for CPU is as such::
41		−
42		− some avg10=0.00 avg60=0.00 avg300=0.00 total=0
43		−
44		−and for memory and IO::
	40	+The format is as such::
45	41
46	42	some avg10=0.00 avg60=0.00 avg300=0.00 total=0
47	43	full avg10=0.00 avg60=0.00 avg300=0.00 total=0
@@ -58,6 +54,9 @@ situation from a state where some tasks are stalled but the CPU is · 10 unmodified lines
58	54	still doing productive work. As such, time spent in this subset of the
59	55	stall state is tracked separately and exported in the "full" averages.
60	56
	57	+CPU full is undefined at the system level, but has been reported
	58	+since 5.13, so it is set to zero for backward compatibility.
	59	+
61	60	The ratios (in %) are tracked as recent trends over ten, sixty, and
62	61	three hundred second windows, which gives insight into short term events
63	62	as well as medium and long term trends. The total absolute stall time

MAINTAINERS+1−0

@@ -17524,6 +17524,7 @@ R: Steven Rostedt <rostedt@goodmis.org> (SCHED_FIFO/SCHED_RR) · 17523 unmodified lines
17524	17524	R: Ben Segall <bsegall@google.com> (CONFIG_CFS_BANDWIDTH)
17525	17525	R: Mel Gorman <mgorman@suse.de> (CONFIG_NUMA_BALANCING)
17526	17526	R: Daniel Bristot de Oliveira <bristot@redhat.com> (SCHED_DEADLINE)
	17527	+R: Valentin Schneider <vschneid@redhat.com> (TOPOLOGY)
17527	17528	L: linux-kernel@vger.kernel.org
17528	17529	S: Maintained
17529	17530	T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git sched/core

include/asm-generic/vmlinux.lds.h+6−6

@@ -126,13 +126,13 @@ · 125 unmodified lines
126	126	*/
127	127	#define SCHED_DATA \
128	128	STRUCT_ALIGN(); \
129		− __begin_sched_classes = .; \
130		− *(__idle_sched_class) \
131		− *(__fair_sched_class) \
132		− *(__rt_sched_class) \
133		− *(__dl_sched_class) \
	129	+ __sched_class_highest = .; \
134	130	*(__stop_sched_class) \
135		− __end_sched_classes = .;
	131	+ *(__dl_sched_class) \
	132	+ *(__rt_sched_class) \
	133	+ *(__fair_sched_class) \
	134	+ *(__idle_sched_class) \
	135	+ __sched_class_lowest = .;
136	136
137	137	/* The actual configuration determine if the init/exit sections
138	138	* are handled as text/data or they can be discarded (which

include/linux/interrupt.h+9−0

@@ -589,6 +589,15 @@ struct softirq_action · 588 unmodified lines
589	589	asmlinkage void do_softirq(void);
590	590	asmlinkage void __do_softirq(void);
591	591
	592	+#ifdef CONFIG_PREEMPT_RT
	593	+extern void do_softirq_post_smp_call_flush(unsigned int was_pending);
	594	+#else
	595	+static inline void do_softirq_post_smp_call_flush(unsigned int unused)
	596	+{
	597	+ do_softirq();
	598	+}
	599	+#endif
	600	+
592	601	extern void open_softirq(int nr, void (action)(struct softirq_action ));
593	602	extern void softirq_init(void);
594	603	extern void __raise_softirq_irqoff(unsigned int nr);

include/linux/sched.h+2−14

@@ -2382,20 +2382,6 @@ static inline void rseq_syscall(struct pt_regs *regs) · 2381 unmodified lines
2382	2382
2383	2383	#endif
2384	2384
2385		−const struct sched_avg sched_trace_cfs_rq_avg(struct cfs_rq cfs_rq);
2386		−char sched_trace_cfs_rq_path(struct cfs_rq cfs_rq, char *str, int len);
2387		−int sched_trace_cfs_rq_cpu(struct cfs_rq *cfs_rq);
2388		−
2389		−const struct sched_avg sched_trace_rq_avg_rt(struct rq rq);
2390		−const struct sched_avg sched_trace_rq_avg_dl(struct rq rq);
2391		−const struct sched_avg sched_trace_rq_avg_irq(struct rq rq);
2392		−
2393		−int sched_trace_rq_cpu(struct rq *rq);
2394		−int sched_trace_rq_cpu_capacity(struct rq *rq);
2395		−int sched_trace_rq_nr_running(struct rq *rq);
2396		−
2397		−const struct cpumask sched_trace_rd_span(struct root_domain rd);
2398		−
2399	2385	#ifdef CONFIG_SCHED_CORE
2400	2386	extern void sched_core_free(struct task_struct *tsk);
2401	2387	extern void sched_core_fork(struct task_struct *p);
@@ -2406,4 +2392,6 @@ static inline void sched_core_free(struct task_struct *tsk) { } · 4 unmodified lines
2406	2392	static inline void sched_core_fork(struct task_struct *p) { }
2407	2393	#endif
2408	2394
	2395	+extern void sched_set_stop_task(int cpu, struct task_struct *stop);
	2396	+
2409	2397	#endif

include/linux/topology.h+0−7

@@ -240,13 +240,6 @@ static inline const struct cpumask *cpu_smt_mask(int cpu) · 239 unmodified lines
240	240	}
241	241	#endif
242	242
243		−#if defined(CONFIG_SCHED_CLUSTER) && !defined(cpu_cluster_mask)
244		−static inline const struct cpumask *cpu_cluster_mask(int cpu)
245		−{
246		− return topology_cluster_cpumask(cpu);
247		−}
248		−#endif
249		−
250	243	static inline const struct cpumask *cpu_cpu_mask(int cpu)
251	244	{
252	245	return cpumask_of_node(cpu_to_node(cpu));

kernel/kcsan/kcsan_test.c+3−2

@@ -1380,13 +1380,14 @@ static const void nthreads_gen_params(const void prev, char *desc) · 1379 unmodified lines
1380	1380	else
1381	1381	nthreads *= 2;
1382	1382
1383		− if (!IS_ENABLED(CONFIG_PREEMPT) \|\| !IS_ENABLED(CONFIG_KCSAN_INTERRUPT_WATCHER)) {
	1383	+ if (!preempt_model_preemptible() \|\|
	1384	+ !IS_ENABLED(CONFIG_KCSAN_INTERRUPT_WATCHER)) {
1384	1385	/*
1385	1386	* Without any preemption, keep 2 CPUs free for other tasks, one
1386	1387	* of which is the main test case function checking for
1387	1388	* completion or failure.
1388	1389	*/
1389		− const long min_unused_cpus = IS_ENABLED(CONFIG_PREEMPT_NONE) ? 2 : 0;
	1390	+ const long min_unused_cpus = preempt_model_none() ? 2 : 0;
1390	1391	const long min_required_cpus = 2 + min_unused_cpus;
1391	1392
1392	1393	if (num_online_cpus() < min_required_cpus) {

kernel/sched/build_policy.c+2−0

@@ -15,6 +15,7 @@ · 14 unmodified lines
15	15	/* Headers: */
16	16	#include <linux/sched/clock.h>
17	17	#include <linux/sched/cputime.h>
	18	+#include <linux/sched/hotplug.h>
18	19	#include <linux/sched/posix-timers.h>
19	20	#include <linux/sched/rt.h>
20	21
@@ -31,6 +32,7 @@ · 10 unmodified lines
31	32	#include <uapi/linux/sched/types.h>
32	33
33	34	#include "sched.h"
	35	+#include "smp.h"
34	36
35	37	#include "autogroup.h"
36	38	#include "stats.h"

kernel/sched/build_utility.c+1−0

@@ -14,6 +14,7 @@ · 13 unmodified lines
14	14	#include <linux/sched/debug.h>
15	15	#include <linux/sched/isolation.h>
16	16	#include <linux/sched/loadavg.h>
	17	+#include <linux/sched/nohz.h>
17	18	#include <linux/sched/mm.h>
18	19	#include <linux/sched/rseq_api.h>
19	20	#include <linux/sched/task_stack.h>

kernel/sched/core.c+13−10

@@ -26,7 +26,10 @@ · 25 unmodified lines
26	26	#include <linux/topology.h>
27	27	#include <linux/sched/clock.h>
28	28	#include <linux/sched/cond_resched.h>
	29	+#include <linux/sched/cputime.h>
29	30	#include <linux/sched/debug.h>
	31	+#include <linux/sched/hotplug.h>
	32	+#include <linux/sched/init.h>
30	33	#include <linux/sched/isolation.h>
31	34	#include <linux/sched/loadavg.h>
32	35	#include <linux/sched/mm.h>
@@ -610,10 +613,10 @@ void double_rq_lock(struct rq rq1, struct rq rq2) · 577 unmodified lines
610	613	swap(rq1, rq2);
611	614
612	615	raw_spin_rq_lock(rq1);
613		− if (__rq_lockp(rq1) == __rq_lockp(rq2))
614		− return;
	616	+ if (__rq_lockp(rq1) != __rq_lockp(rq2))
	617	+ raw_spin_rq_lock_nested(rq2, SINGLE_DEPTH_NESTING);
615	618
616		− raw_spin_rq_lock_nested(rq2, SINGLE_DEPTH_NESTING);
	619	+ double_rq_clock_clear_update(rq1, rq2);
617	620	}
618	621	#endif
619	622
@@ -2190,7 +2193,7 @@ void check_preempt_curr(struct rq rq, struct task_struct p, int flags) · 1570 unmodified lines
2190	2193	{
2191	2194	if (p->sched_class == rq->curr->sched_class)
2192	2195	rq->curr->sched_class->check_preempt_curr(rq, p, flags);
2193		− else if (p->sched_class > rq->curr->sched_class)
	2196	+ else if (sched_class_above(p->sched_class, rq->curr->sched_class))
2194	2197	resched_curr(rq);
2195	2198
2196	2199	/*
@@ -2408,7 +2411,7 @@ static int migration_cpu_stop(void *data) · 211 unmodified lines
2408	2411	* __migrate_task() such that we will not miss enforcing cpus_ptr
2409	2412	* during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test.
2410	2413	*/
2411		− flush_smp_call_function_from_idle();
	2414	+ flush_smp_call_function_queue();
2412	2415
2413	2416	raw_spin_lock(&p->pi_lock);
2414	2417	rq_lock(rq, &rf);
@@ -5689,7 +5692,7 @@ __pick_next_task(struct rq rq, struct task_struct prev, struct rq_flags *rf) · 3274 unmodified lines
5689	5692	* higher scheduling class, because otherwise those lose the
5690	5693	* opportunity to pull in more work from other CPUs.
5691	5694	*/
5692		− if (likely(prev->sched_class <= &fair_sched_class &&
	5695	+ if (likely(!sched_class_above(prev->sched_class, &fair_sched_class) &&
5693	5696	rq->nr_running == rq->cfs.h_nr_running)) {
5694	5697
5695	5698	p = pick_next_task_fair(rq, prev, rf);
@@ -9469,11 +9472,11 @@ void __init sched_init(void) · 3773 unmodified lines
9469	9472	int i;
9470	9473
9471	9474	/* Make sure the linker didn't screw up */
9472		− BUG_ON(&idle_sched_class + 1 != &fair_sched_class \|\|
9473		− &fair_sched_class + 1 != &rt_sched_class \|\|
9474		− &rt_sched_class + 1 != &dl_sched_class);
	9475	+ BUG_ON(&idle_sched_class != &fair_sched_class + 1 \|\|
	9476	+ &fair_sched_class != &rt_sched_class + 1 \|\|
	9477	+ &rt_sched_class != &dl_sched_class + 1);
9475	9478	#ifdef CONFIG_SMP
9476		− BUG_ON(&dl_sched_class + 1 != &stop_sched_class);
	9479	+ BUG_ON(&dl_sched_class != &stop_sched_class + 1);
9477	9480	#endif
9478	9481
9479	9482	wait_bit_init();

kernel/sched/deadline.c+4−11

@@ -1220,8 +1220,6 @@ int dl_runtime_exceeded(struct sched_dl_entity *dl_se) · 1219 unmodified lines
1220	1220	return (dl_se->runtime <= 0);
1221	1221	}
1222	1222
1223		−extern bool sched_rt_bandwidth_account(struct rt_rq *rt_rq);
1224		−
1225	1223	/*
1226	1224	* This function implements the GRUB accounting rule:
1227	1225	* according to the GRUB reclaiming algorithm, the runtime is
@@ -1832,6 +1830,7 @@ out: · 604 unmodified lines
1832	1830
1833	1831	static void migrate_task_rq_dl(struct task_struct *p, int new_cpu __maybe_unused)
1834	1832	{
	1833	+ struct rq_flags rf;
1835	1834	struct rq *rq;
1836	1835
1837	1836	if (READ_ONCE(p->__state) != TASK_WAKING)
@@ -1843,7 +1842,7 @@ static void migrate_task_rq_dl(struct task_struct *p, int new_cpu __maybe_unused · 5 unmodified lines
1843	1842	* from try_to_wake_up(). Hence, p->pi_lock is locked, but
1844	1843	* rq->lock is not... So, lock it
1845	1844	*/
1846		− raw_spin_rq_lock(rq);
	1845	+ rq_lock(rq, &rf);
1847	1846	if (p->dl.dl_non_contending) {
1848	1847	update_rq_clock(rq);
1849	1848	sub_running_bw(&p->dl, &rq->dl);
@@ -1859,7 +1858,7 @@ static void migrate_task_rq_dl(struct task_struct *p, int new_cpu __maybe_unused · 9 unmodified lines
1859	1858	put_task_struct(p);
1860	1859	}
1861	1860	sub_rq_bw(&p->dl, &rq->dl);
1862		− raw_spin_rq_unlock(rq);
	1861	+ rq_unlock(rq, &rf);
1863	1862	}
1864	1863
1865	1864	static void check_preempt_equal_dl(struct rq rq, struct task_struct p)
@@ -2319,13 +2318,7 @@ retry: · 453 unmodified lines
2319	2318
2320	2319	deactivate_task(rq, next_task, 0);
2321	2320	set_task_cpu(next_task, later_rq->cpu);
2322		−
2323		− /*
2324		− * Update the later_rq clock here, because the clock is used
2325		− * by the cpufreq_update_util() inside __add_running_bw().
2326		− */
2327		− update_rq_clock(later_rq);
2328		− activate_task(later_rq, next_task, ENQUEUE_NOCLOCK);
	2321	+ activate_task(later_rq, next_task, 0);
2329	2322	ret = 1;
2330	2323
2331	2324	resched_curr(later_rq);

kernel/sched/fair.c+67−225

@@ -36,6 +36,7 @@ · 35 unmodified lines
36	36	#include <linux/sched/cond_resched.h>
37	37	#include <linux/sched/cputime.h>
38	38	#include <linux/sched/isolation.h>
	39	+#include <linux/sched/nohz.h>
39	40
40	41	#include <linux/cpuidle.h>
41	42	#include <linux/interrupt.h>
@@ -313,19 +314,6 @@ const struct sched_class fair_sched_class; · 271 unmodified lines
313	314	#define for_each_sched_entity(se) \
314	315	for (; se; se = se->parent)
315	316
316		−static inline void cfs_rq_tg_path(struct cfs_rq cfs_rq, char path, int len)
317		−{
318		− if (!path)
319		− return;
320		−
321		− if (cfs_rq && task_group_is_autogroup(cfs_rq->tg))
322		− autogroup_path(cfs_rq->tg, path, len);
323		− else if (cfs_rq && cfs_rq->tg->css.cgroup)
324		− cgroup_path(cfs_rq->tg->css.cgroup, path, len);
325		− else
326		− strlcpy(path, "(null)", len);
327		−}
328		−
329	317	static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
330	318	{
331	319	struct rq *rq = rq_of(cfs_rq);
@@ -493,12 +481,6 @@ static int se_is_idle(struct sched_entity *se) · 161 unmodified lines
493	481	#define for_each_sched_entity(se) \
494	482	for (; se; se = NULL)
495	483
496		−static inline void cfs_rq_tg_path(struct cfs_rq cfs_rq, char path, int len)
497		−{
498		− if (path)
499		− strlcpy(path, "(null)", len);
500		−}
501		−
502	484	static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
503	485	{
504	486	return true;
@@ -4846,11 +4828,11 @@ static int tg_unthrottle_up(struct task_group tg, void data) · 4341 unmodified lines
4846	4828
4847	4829	cfs_rq->throttle_count--;
4848	4830	if (!cfs_rq->throttle_count) {
4849		− cfs_rq->throttled_clock_task_time += rq_clock_task(rq) -
4850		− cfs_rq->throttled_clock_task;
	4831	+ cfs_rq->throttled_clock_pelt_time += rq_clock_pelt(rq) -
	4832	+ cfs_rq->throttled_clock_pelt;
4851	4833
4852	4834	/* Add cfs_rq with load or one or more already running entities to the list */
4853		− if (!cfs_rq_is_decayed(cfs_rq) \|\| cfs_rq->nr_running)
	4835	+ if (!cfs_rq_is_decayed(cfs_rq))
4854	4836	list_add_leaf_cfs_rq(cfs_rq);
4855	4837	}
4856	4838
@@ -4864,7 +4846,7 @@ static int tg_throttle_down(struct task_group tg, void data) · 7 unmodified lines
4864	4846
4865	4847	/* group is entering throttled state, stop time */
4866	4848	if (!cfs_rq->throttle_count) {
4867		− cfs_rq->throttled_clock_task = rq_clock_task(rq);
	4849	+ cfs_rq->throttled_clock_pelt = rq_clock_pelt(rq);
4868	4850	list_del_leaf_cfs_rq(cfs_rq);
4869	4851	}
4870	4852	cfs_rq->throttle_count++;
@@ -5308,7 +5290,7 @@ static void sync_throttle(struct task_group *tg, int cpu) · 437 unmodified lines
5308	5290	pcfs_rq = tg->parent->cfs_rq[cpu];
5309	5291
5310	5292	cfs_rq->throttle_count = pcfs_rq->throttle_count;
5311		− cfs_rq->throttled_clock_task = rq_clock_task(cpu_rq(cpu));
	5293	+ cfs_rq->throttled_clock_pelt = rq_clock_pelt(cpu_rq(cpu));
5312	5294	}
5313	5295
5314	5296	/* conditionally throttle active cfs_rq's from put_prev_entity() */
@@ -6544,108 +6526,19 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) · 1229 unmodified lines
6544	6526	}
6545	6527
6546	6528	/*
6547		− * cpu_util_without: compute cpu utilization without any contributions from *p
6548		− * @cpu: the CPU which utilization is requested
6549		− * @p: the task which utilization should be discounted
6550		− *
6551		− * The utilization of a CPU is defined by the utilization of tasks currently
6552		− * enqueued on that CPU as well as tasks which are currently sleeping after an
6553		− * execution on that CPU.
6554		− *
6555		− * This method returns the utilization of the specified CPU by discounting the
6556		− * utilization of the specified task, whenever the task is currently
6557		− * contributing to the CPU utilization.
6558		− */
6559		−static unsigned long cpu_util_without(int cpu, struct task_struct *p)
6560		−{
6561		− struct cfs_rq *cfs_rq;
6562		− unsigned int util;
6563		−
6564		− /* Task has no contribution or is new */
6565		− if (cpu != task_cpu(p) \|\| !READ_ONCE(p->se.avg.last_update_time))
6566		− return cpu_util_cfs(cpu);
6567		−
6568		− cfs_rq = &cpu_rq(cpu)->cfs;
6569		− util = READ_ONCE(cfs_rq->avg.util_avg);
6570		−
6571		− /* Discount task's util from CPU's util */
6572		− lsub_positive(&util, task_util(p));
6573		−
6574		− /*
6575		− * Covered cases:
6576		− *
6577		− * a) if *p is the only task sleeping on this CPU, then:
6578		− * cpu_util (== task_util) > util_est (== 0)
6579		− * and thus we return:
6580		− * cpu_util_without = (cpu_util - task_util) = 0
6581		− *
6582		− * b) if other tasks are SLEEPING on this CPU, which is now exiting
6583		− * IDLE, then:
6584		− * cpu_util >= task_util
6585		− * cpu_util > util_est (== 0)
6586		− * and thus we discount *p's blocked utilization to return:
6587		− * cpu_util_without = (cpu_util - task_util) >= 0
6588		− *
6589		− * c) if other tasks are RUNNABLE on that CPU and
6590		− * util_est > cpu_util
6591		− * then we use util_est since it returns a more restrictive
6592		− * estimation of the spare capacity on that CPU, by just
6593		− * considering the expected utilization of tasks already
6594		− * runnable on that CPU.
6595		− *
6596		− * Cases a) and b) are covered by the above code, while case c) is
6597		− * covered by the following code when estimated utilization is
6598		− * enabled.
6599		− */
6600		− if (sched_feat(UTIL_EST)) {
6601		− unsigned int estimated =
6602		− READ_ONCE(cfs_rq->avg.util_est.enqueued);
6603		−
6604		− /*
6605		− * Despite the following checks we still have a small window
6606		− * for a possible race, when an execl's select_task_rq_fair()
6607		− * races with LB's detach_task():
6608		− *
6609		− * detach_task()
6610		− * p->on_rq = TASK_ON_RQ_MIGRATING;
6611		− * ---------------------------------- A
6612		− * deactivate_task() \
6613		− * dequeue_task() + RaceTime
6614		− * util_est_dequeue() /
6615		− * ---------------------------------- B
6616		− *
6617		− * The additional check on "current == p" it's required to
6618		− * properly fix the execl regression and it helps in further
6619		− * reducing the chances for the above race.
6620		− */
6621		− if (unlikely(task_on_rq_queued(p) \|\| current == p))
6622		− lsub_positive(&estimated, _task_util_est(p));
6623		−
6624		− util = max(util, estimated);
6625		− }
6626		−
6627		− /*
6628		− * Utilization (estimated) can exceed the CPU capacity, thus let's
6629		− * clamp to the maximum CPU capacity to ensure consistency with
6630		− * cpu_util.
6631		− */
6632		− return min_t(unsigned long, util, capacity_orig_of(cpu));
6633		−}
6634		−
6635		−/*
6636		− * Predicts what cpu_util(@cpu) would return if @p was migrated (and enqueued)
6637		− * to @dst_cpu.
	6529	+ * Predicts what cpu_util(@cpu) would return if @p was removed from @cpu
	6530	+ * (@dst_cpu = -1) or migrated to @dst_cpu.
6638	6531	*/
6639	6532	static unsigned long cpu_util_next(int cpu, struct task_struct *p, int dst_cpu)
6640	6533	{
6641	6534	struct cfs_rq *cfs_rq = &cpu_rq(cpu)->cfs;
6642		− unsigned long util_est, util = READ_ONCE(cfs_rq->avg.util_avg);
	6535	+ unsigned long util = READ_ONCE(cfs_rq->avg.util_avg);
6643	6536
6644	6537	/*
6645		− * If @p migrates from @cpu to another, remove its contribution. Or,
6646		− * if @p migrates from another CPU to @cpu, add its contribution. In
6647		− * the other cases, @cpu is not impacted by the migration, so the
6648		− * util_avg should already be correct.
	6538	+ * If @dst_cpu is -1 or @p migrates from @cpu to @dst_cpu remove its
	6539	+ * contribution. If @p migrates from another CPU to @cpu add its
	6540	+ * contribution. In all the other cases @cpu is not impacted by the
	6541	+ * migration so its util_avg is already correct.
6649	6542	*/
6650	6543	if (task_cpu(p) == cpu && dst_cpu != cpu)
6651	6544	lsub_positive(&util, task_util(p));
@@ -6653,16 +6546,40 @@ static unsigned long cpu_util_next(int cpu, struct task_struct *p, int dst_cpu) · 1 unmodified line
6653	6546	util += task_util(p);
6654	6547
6655	6548	if (sched_feat(UTIL_EST)) {
	6549	+ unsigned long util_est;
	6550	+
6656	6551	util_est = READ_ONCE(cfs_rq->avg.util_est.enqueued);
6657	6552
6658	6553	/*
6659		− * During wake-up, the task isn't enqueued yet and doesn't
6660		− * appear in the cfs_rq->avg.util_est.enqueued of any rq,
6661		− * so just add it (if needed) to "simulate" what will be
6662		− * cpu_util after the task has been enqueued.
	6554	+ * During wake-up @p isn't enqueued yet and doesn't contribute
	6555	+ * to any cpu_rq(cpu)->cfs.avg.util_est.enqueued.
	6556	+ * If @dst_cpu == @cpu add it to "simulate" cpu_util after @p
	6557	+ * has been enqueued.
	6558	+ *
	6559	+ * During exec (@dst_cpu = -1) @p is enqueued and does
	6560	+ * contribute to cpu_rq(cpu)->cfs.util_est.enqueued.
	6561	+ * Remove it to "simulate" cpu_util without @p's contribution.
	6562	+ *
	6563	+ * Despite the task_on_rq_queued(@p) check there is still a
	6564	+ * small window for a possible race when an exec
	6565	+ * select_task_rq_fair() races with LB's detach_task().
	6566	+ *
	6567	+ * detach_task()
	6568	+ * deactivate_task()
	6569	+ * p->on_rq = TASK_ON_RQ_MIGRATING;
	6570	+ * -------------------------------- A
	6571	+ * dequeue_task() \
	6572	+ * dequeue_task_fair() + Race Time
	6573	+ * util_est_dequeue() /
	6574	+ * -------------------------------- B
	6575	+ *
	6576	+ * The additional check "current == p" is required to further
	6577	+ * reduce the race window.
6663	6578	*/
6664	6579	if (dst_cpu == cpu)
6665	6580	util_est += _task_util_est(p);
	6581	+ else if (unlikely(task_on_rq_queued(p) \|\| current == p))
	6582	+ lsub_positive(&util_est, _task_util_est(p));
6666	6583
6667	6584	util = max(util, util_est);
6668	6585	}
@@ -6670,6 +6587,28 @@ static unsigned long cpu_util_next(int cpu, struct task_struct *p, int dst_cpu) · 1 unmodified line
6670	6587	return min(util, capacity_orig_of(cpu));
6671	6588	}
6672	6589
	6590	+/*
	6591	+ * cpu_util_without: compute cpu utilization without any contributions from *p
	6592	+ * @cpu: the CPU which utilization is requested
	6593	+ * @p: the task which utilization should be discounted
	6594	+ *
	6595	+ * The utilization of a CPU is defined by the utilization of tasks currently
	6596	+ * enqueued on that CPU as well as tasks which are currently sleeping after an
	6597	+ * execution on that CPU.
	6598	+ *
	6599	+ * This method returns the utilization of the specified CPU by discounting the
	6600	+ * utilization of the specified task, whenever the task is currently
	6601	+ * contributing to the CPU utilization.
	6602	+ */
	6603	+static unsigned long cpu_util_without(int cpu, struct task_struct *p)
	6604	+{
	6605	+ /* Task has no contribution or is new */
	6606	+ if (cpu != task_cpu(p) \|\| !READ_ONCE(p->se.avg.last_update_time))
	6607	+ return cpu_util_cfs(cpu);
	6608	+
	6609	+ return cpu_util_next(cpu, p, -1);
	6610	+}
	6611	+
6673	6612	/*
6674	6613	* compute_energy(): Estimates the energy that @pd would consume if @p was
6675	6614	* migrated to @dst_cpu. compute_energy() predicts what will be the utilization
@@ -9460,8 +9399,6 @@ static inline void calculate_imbalance(struct lb_env env, struct sd_lb_stats s · 2784 unmodified lines
9460	9399	local->avg_load = (local->group_load * SCHED_CAPACITY_SCALE) /
9461	9400	local->group_capacity;
9462	9401
9463		− sds->avg_load = (sds->total_load * SCHED_CAPACITY_SCALE) /
9464		− sds->total_capacity;
9465	9402	/*
9466	9403	* If the local group is more loaded than the selected
9467	9404	* busiest group don't try to pull any tasks.
@@ -9470,6 +9407,9 @@ static inline void calculate_imbalance(struct lb_env env, struct sd_lb_stats s · 2 unmodified lines
9470	9407	env->imbalance = 0;
9471	9408	return;
9472	9409	}
	9410	+
	9411	+ sds->avg_load = (sds->total_load * SCHED_CAPACITY_SCALE) /
	9412	+ sds->total_capacity;
9473	9413	}
9474	9414
9475	9415	/*
@@ -9495,7 +9435,7 @@ static inline void calculate_imbalance(struct lb_env env, struct sd_lb_stats s · 19 unmodified lines
9495	9435	* busiest \ local has_spare fully_busy misfit asym imbalanced overloaded
9496	9436	* has_spare nr_idle balanced N/A N/A balanced balanced
9497	9437	* fully_busy nr_idle nr_idle N/A N/A balanced balanced
9498		− * misfit_task force N/A N/A N/A force force
	9438	+ * misfit_task force N/A N/A N/A N/A N/A
9499	9439	* asym_packing force force N/A N/A force force
9500	9440	* imbalanced force force N/A N/A force force
9501	9441	* overloaded force force N/A N/A force avg_load
@@ -11881,101 +11821,3 @@ __init void init_sched_fair_class(void) · 2379 unmodified lines
11881	11821	#endif /* SMP */
11882	11822
11883	11823	}
11884		−
11885		−/*
11886		− * Helper functions to facilitate extracting info from tracepoints.
11887		− */
11888		−
11889		−const struct sched_avg sched_trace_cfs_rq_avg(struct cfs_rq cfs_rq)
11890		−{
11891		−#ifdef CONFIG_SMP
11892		− return cfs_rq ? &cfs_rq->avg : NULL;
11893		−#else
11894		− return NULL;
11895		−#endif
11896		−}
11897		−EXPORT_SYMBOL_GPL(sched_trace_cfs_rq_avg);
11898		−
11899		−char sched_trace_cfs_rq_path(struct cfs_rq cfs_rq, char *str, int len)
11900		−{
11901		− if (!cfs_rq) {
11902		− if (str)
11903		− strlcpy(str, "(null)", len);
11904		− else
11905		− return NULL;
11906		− }
11907		−
11908		− cfs_rq_tg_path(cfs_rq, str, len);
11909		− return str;
11910		−}
11911		−EXPORT_SYMBOL_GPL(sched_trace_cfs_rq_path);
11912		−
11913		−int sched_trace_cfs_rq_cpu(struct cfs_rq *cfs_rq)
11914		−{
11915		− return cfs_rq ? cpu_of(rq_of(cfs_rq)) : -1;
11916		−}
11917		−EXPORT_SYMBOL_GPL(sched_trace_cfs_rq_cpu);
11918		−
11919		−const struct sched_avg sched_trace_rq_avg_rt(struct rq rq)
11920		−{
11921		−#ifdef CONFIG_SMP
11922		− return rq ? &rq->avg_rt : NULL;
11923		−#else
11924		− return NULL;
11925		−#endif
11926		−}
11927		−EXPORT_SYMBOL_GPL(sched_trace_rq_avg_rt);
11928		−
11929		−const struct sched_avg sched_trace_rq_avg_dl(struct rq rq)
11930		−{
11931		−#ifdef CONFIG_SMP
11932		− return rq ? &rq->avg_dl : NULL;
11933		−#else
11934		− return NULL;
11935		−#endif
11936		−}
11937		−EXPORT_SYMBOL_GPL(sched_trace_rq_avg_dl);
11938		−
11939		−const struct sched_avg sched_trace_rq_avg_irq(struct rq rq)
11940		−{
11941		−#if defined(CONFIG_SMP) && defined(CONFIG_HAVE_SCHED_AVG_IRQ)
11942		− return rq ? &rq->avg_irq : NULL;
11943		−#else
11944		− return NULL;
11945		−#endif
11946		−}
11947		−EXPORT_SYMBOL_GPL(sched_trace_rq_avg_irq);
11948		−
11949		−int sched_trace_rq_cpu(struct rq *rq)
11950		−{
11951		− return rq ? cpu_of(rq) : -1;
11952		−}
11953		−EXPORT_SYMBOL_GPL(sched_trace_rq_cpu);
11954		−
11955		−int sched_trace_rq_cpu_capacity(struct rq *rq)
11956		−{
11957		− return rq ?
11958		−#ifdef CONFIG_SMP
11959		− rq->cpu_capacity
11960		−#else
11961		− SCHED_CAPACITY_SCALE
11962		−#endif
11963		− : -1;
11964		−}
11965		−EXPORT_SYMBOL_GPL(sched_trace_rq_cpu_capacity);
11966		−
11967		−const struct cpumask sched_trace_rd_span(struct root_domain rd)
11968		−{
11969		−#ifdef CONFIG_SMP
11970		− return rd ? rd->span : NULL;
11971		−#else
11972		− return NULL;
11973		−#endif
11974		−}
11975		−EXPORT_SYMBOL_GPL(sched_trace_rd_span);
11976		−
11977		−int sched_trace_rq_nr_running(struct rq *rq)
11978		−{
11979		− return rq ? rq->nr_running : -1;
11980		−}
11981		−EXPORT_SYMBOL_GPL(sched_trace_rq_nr_running);

kernel/sched/idle.c+1−1

@@ -327,7 +327,7 @@ static void do_idle(void) · 326 unmodified lines
327	327	* RCU relies on this call to be done outside of an RCU read-side
328	328	* critical section.
329	329	*/
330		− flush_smp_call_function_from_idle();
	330	+ flush_smp_call_function_queue();
331	331	schedule_idle();
332	332
333	333	if (unlikely(klp_patch_pending(current)))

kernel/sched/pelt.h+2−2

@@ -145,9 +145,9 @@ static inline u64 rq_clock_pelt(struct rq *rq) · 144 unmodified lines
145	145	static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
146	146	{
147	147	if (unlikely(cfs_rq->throttle_count))
148		− return cfs_rq->throttled_clock_task - cfs_rq->throttled_clock_task_time;
	148	+ return cfs_rq->throttled_clock_pelt - cfs_rq->throttled_clock_pelt_time;
149	149
150		− return rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_task_time;
	150	+ return rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_pelt_time;
151	151	}
152	152	#else
153	153	static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)

kernel/sched/psi.c+11−7

@@ -1060,14 +1060,17 @@ int psi_show(struct seq_file m, struct psi_group group, enum psi_res res) · 1059 unmodified lines
1060	1060	mutex_unlock(&group->avgs_lock);
1061	1061
1062	1062	for (full = 0; full < 2; full++) {
1063		− unsigned long avg[3];
1064		− u64 total;
	1063	+ unsigned long avg[3] = { 0, };
	1064	+ u64 total = 0;
1065	1065	int w;
1066	1066
1067		− for (w = 0; w < 3; w++)
1068		− avg[w] = group->avg[res * 2 + full][w];
1069		− total = div_u64(group->total[PSI_AVGS][res * 2 + full],
1070		− NSEC_PER_USEC);
	1067	+ /* CPU FULL is undefined at the system level */
	1068	+ if (!(group == &psi_system && res == PSI_CPU && full)) {
	1069	+ for (w = 0; w < 3; w++)
	1070	+ avg[w] = group->avg[res * 2 + full][w];
	1071	+ total = div_u64(group->total[PSI_AVGS][res * 2 + full],
	1072	+ NSEC_PER_USEC);
	1073	+ }
1071	1074
1072	1075	seq_printf(m, "%s avg10=%lu.%02lu avg60=%lu.%02lu avg300=%lu.%02lu total=%llu\n",
1073	1076	full ? "full" : "some",
@@ -1117,7 +1120,8 @@ struct psi_trigger psi_trigger_create(struct psi_group group, · 43 unmodified lines
1117	1120	t->state = state;
1118	1121	t->threshold = threshold_us * NSEC_PER_USEC;
1119	1122	t->win.size = window_us * NSEC_PER_USEC;
1120		− window_reset(&t->win, 0, 0, 0);
	1123	+ window_reset(&t->win, sched_clock(),
	1124	+ group->total[PSI_POLL][t->state], 0);
1121	1125
1122	1126	t->event = 0;
1123	1127	t->last_event_time = 0;

kernel/sched/rt.c+3−2

@@ -871,6 +871,7 @@ static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun) · 870 unmodified lines
871	871	int enqueue = 0;
872	872	struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i);
873	873	struct rq *rq = rq_of_rt_rq(rt_rq);
	874	+ struct rq_flags rf;
874	875	int skip;
875	876
876	877	/*
@@ -885,7 +886,7 @@ static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun) · 8 unmodified lines
885	886	if (skip)
886	887	continue;
887	888
888		− raw_spin_rq_lock(rq);
	889	+ rq_lock(rq, &rf);
889	890	update_rq_clock(rq);
890	891
891	892	if (rt_rq->rt_time) {
@@ -923,7 +924,7 @@ static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun) · 31 unmodified lines
923	924
924	925	if (enqueue)
925	926	sched_rt_rq_enqueue(rt_rq);
926		− raw_spin_rq_unlock(rq);
	927	+ rq_unlock(rq, &rf);
927	928	}
928	929
929	930	if (!throttled && (!rt_bandwidth_enabled() \|\| rt_b->rt_runtime == RUNTIME_INF))

kernel/sched/sched.h+36−19

@@ -603,8 +603,8 @@ struct cfs_rq { · 602 unmodified lines
603	603	s64 runtime_remaining;
604	604
605	605	u64 throttled_clock;
606		− u64 throttled_clock_task;
607		− u64 throttled_clock_task_time;
	606	+ u64 throttled_clock_pelt;
	607	+ u64 throttled_clock_pelt_time;
608	608	int throttled;
609	609	int throttle_count;
610	610	struct list_head throttled_list;
@@ -1827,12 +1827,7 @@ static inline void dirty_sched_domain_sysctl(int cpu) · 1216 unmodified lines
1827	1827	#endif
1828	1828
1829	1829	extern int sched_update_scaling(void);
1830		−
1831		−extern void flush_smp_call_function_from_idle(void);
1832		−
1833		−#else /* !CONFIG_SMP: */
1834		−static inline void flush_smp_call_function_from_idle(void) { }
1835		−#endif
	1830	+#endif /* CONFIG_SMP */
1836	1831
1837	1832	#include "stats.h"
1838	1833
@@ -2182,6 +2177,8 @@ static inline void set_next_task(struct rq rq, struct task_struct next) · 343 unmodified lines
2182	2177	*
2183	2178	* include/asm-generic/vmlinux.lds.h
2184	2179	*
	2180	+ * CAREFUL they are laid out in REVERSE order!!!
	2181	+ *
2185	2182	* Also enforce alignment on the instance, not the type, to guarantee layout.
2186	2183	*/
2187	2184	#define DEFINE_SCHED_CLASS(name) \
@@ -2190,17 +2187,16 @@ const struct sched_class name##_sched_class \ · 2 unmodified lines
2190	2187	__section("__" #name "_sched_class")
2191	2188
2192	2189	/* Defined in include/asm-generic/vmlinux.lds.h */
2193		−extern struct sched_class __begin_sched_classes[];
2194		−extern struct sched_class __end_sched_classes[];
2195		−
2196		−#define sched_class_highest (__end_sched_classes - 1)
2197		−#define sched_class_lowest (__begin_sched_classes - 1)
	2190	+extern struct sched_class __sched_class_highest[];
	2191	+extern struct sched_class __sched_class_lowest[];
2198	2192
2199	2193	#define for_class_range(class, _from, _to) \
2200		− for (class = (_from); class != (_to); class--)
	2194	+ for (class = (_from); class < (_to); class++)
2201	2195
2202	2196	#define for_each_class(class) \
2203		− for_class_range(class, sched_class_highest, sched_class_lowest)
	2197	+ for_class_range(class, __sched_class_highest, __sched_class_lowest)
	2198	+
	2199	+#define sched_class_above(_a, _b) ((_a) < (_b))
2204	2200
2205	2201	extern const struct sched_class stop_sched_class;
2206	2202	extern const struct sched_class dl_sched_class;
@@ -2309,6 +2305,7 @@ extern void resched_cpu(int cpu); · 102 unmodified lines
2309	2305
2310	2306	extern struct rt_bandwidth def_rt_bandwidth;
2311	2307	extern void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime);
	2308	+extern bool sched_rt_bandwidth_account(struct rt_rq *rt_rq);
2312	2309
2313	2310	extern void init_dl_bandwidth(struct dl_bandwidth *dl_b, u64 period, u64 runtime);
2314	2311	extern void init_dl_task_timer(struct sched_dl_entity *dl_se);
@@ -2478,6 +2475,24 @@ unsigned long arch_scale_freq_capacity(int cpu) · 163 unmodified lines
2478	2475	}
2479	2476	#endif
2480	2477
	2478	+#ifdef CONFIG_SCHED_DEBUG
	2479	+/*
	2480	+ * In double_lock_balance()/double_rq_lock(), we use raw_spin_rq_lock() to
	2481	+ * acquire rq lock instead of rq_lock(). So at the end of these two functions
	2482	+ * we need to call double_rq_clock_clear_update() to clear RQCF_UPDATED of
	2483	+ * rq->clock_update_flags to avoid the WARN_DOUBLE_CLOCK warning.
	2484	+ */
	2485	+static inline void double_rq_clock_clear_update(struct rq rq1, struct rq rq2)
	2486	+{
	2487	+ rq1->clock_update_flags &= (RQCF_REQ_SKIP\|RQCF_ACT_SKIP);
	2488	+ /* rq1 == rq2 for !CONFIG_SMP, so just clear RQCF_UPDATED once. */
	2489	+#ifdef CONFIG_SMP
	2490	+ rq2->clock_update_flags &= (RQCF_REQ_SKIP\|RQCF_ACT_SKIP);
	2491	+#endif
	2492	+}
	2493	+#else
	2494	+static inline void double_rq_clock_clear_update(struct rq rq1, struct rq rq2) {}
	2495	+#endif
2481	2496
2482	2497	#ifdef CONFIG_SMP
2483	2498
@@ -2543,14 +2558,15 @@ static inline int _double_lock_balance(struct rq this_rq, struct rq busiest) · 59 unmodified lines
2543	2558	__acquires(busiest->lock)
2544	2559	__acquires(this_rq->lock)
2545	2560	{
2546		− if (__rq_lockp(this_rq) == __rq_lockp(busiest))
2547		− return 0;
2548		−
2549		− if (likely(raw_spin_rq_trylock(busiest)))
	2561	+ if (__rq_lockp(this_rq) == __rq_lockp(busiest) \|\|
	2562	+ likely(raw_spin_rq_trylock(busiest))) {
	2563	+ double_rq_clock_clear_update(this_rq, busiest);
2550	2564	return 0;
	2565	+ }
2551	2566
2552	2567	if (rq_order_less(this_rq, busiest)) {
2553	2568	raw_spin_rq_lock_nested(busiest, SINGLE_DEPTH_NESTING);
	2569	+ double_rq_clock_clear_update(this_rq, busiest);
2554	2570	return 0;
2555	2571	}
2556	2572
@@ -2644,6 +2660,7 @@ static inline void double_rq_lock(struct rq rq1, struct rq rq2) · 87 unmodified lines
2644	2660	BUG_ON(rq1 != rq2);
2645	2661	raw_spin_rq_lock(rq1);
2646	2662	__acquire(rq2->lock); /* Fake it out ;) */
	2663	+ double_rq_clock_clear_update(rq1, rq2);
2647	2664	}
2648	2665
2649	2666	/*

kernel/sched/smp.h+6−0

@@ -7,3 +7,9 @@ · 6 unmodified lines
7	7	extern void sched_ttwu_pending(void *arg);
8	8
9	9	extern void send_call_function_single_ipi(int cpu);
	10	+
	11	+#ifdef CONFIG_SMP
	12	+extern void flush_smp_call_function_queue(void);
	13	+#else
	14	+static inline void flush_smp_call_function_queue(void) { }
	15	+#endif

kernel/smp.c+24−8

@@ -96,7 +96,7 @@ static DEFINE_PER_CPU_ALIGNED(struct call_function_data, cfd_data); · 95 unmodified lines
96	96
97	97	static DEFINE_PER_CPU_SHARED_ALIGNED(struct llist_head, call_single_queue);
98	98
99		−static void flush_smp_call_function_queue(bool warn_cpu_offline);
	99	+static void __flush_smp_call_function_queue(bool warn_cpu_offline);
100	100
101	101	int smpcfd_prepare_cpu(unsigned int cpu)
102	102	{
@@ -141,7 +141,7 @@ int smpcfd_dying_cpu(unsigned int cpu) · 38 unmodified lines
141	141	* ensure that the outgoing CPU doesn't go offline with work
142	142	* still pending.
143	143	*/
144		− flush_smp_call_function_queue(false);
	144	+ __flush_smp_call_function_queue(false);
145	145	irq_work_run();
146	146	return 0;
147	147	}
@@ -544,11 +544,11 @@ void generic_smp_call_function_single_interrupt(void) · 396 unmodified lines
544	544	{
545	545	cfd_seq_store(this_cpu_ptr(&cfd_seq_local)->gotipi, CFD_SEQ_NOCPU,
546	546	smp_processor_id(), CFD_SEQ_GOTIPI);
547		− flush_smp_call_function_queue(true);
	547	+ __flush_smp_call_function_queue(true);
548	548	}
549	549
550	550	/**
551		− * flush_smp_call_function_queue - Flush pending smp-call-function callbacks
	551	+ * __flush_smp_call_function_queue - Flush pending smp-call-function callbacks
552	552	*
553	553	* @warn_cpu_offline: If set to 'true', warn if callbacks were queued on an
554	554	* offline CPU. Skip this check if set to 'false'.
@@ -561,7 +561,7 @@ void generic_smp_call_function_single_interrupt(void) · 6 unmodified lines
561	561	* Loop through the call_single_queue and run all the queued callbacks.
562	562	* Must be called with interrupts disabled.
563	563	*/
564		−static void flush_smp_call_function_queue(bool warn_cpu_offline)
	564	+static void __flush_smp_call_function_queue(bool warn_cpu_offline)
565	565	{
566	566	call_single_data_t csd, csd_next;
567	567	struct llist_node entry, prev;
@@ -684,8 +684,22 @@ static void flush_smp_call_function_queue(bool warn_cpu_offline) · 116 unmodified lines
684	684	smp_processor_id(), CFD_SEQ_HDLEND);
685	685	}
686	686
687		−void flush_smp_call_function_from_idle(void)
	687	+
	688	+/**
	689	+ * flush_smp_call_function_queue - Flush pending smp-call-function callbacks
	690	+ * from task context (idle, migration thread)
	691	+ *
	692	+ * When TIF_POLLING_NRFLAG is supported and a CPU is in idle and has it
	693	+ * set, then remote CPUs can avoid sending IPIs and wake the idle CPU by
	694	+ * setting TIF_NEED_RESCHED. The idle task on the woken up CPU has to
	695	+ * handle queued SMP function calls before scheduling.
	696	+ *
	697	+ * The migration thread has to ensure that an eventually pending wakeup has
	698	+ * been handled before it migrates a task.
	699	+ */
	700	+void flush_smp_call_function_queue(void)
688	701	{
	702	+ unsigned int was_pending;
689	703	unsigned long flags;
690	704
691	705	if (llist_empty(this_cpu_ptr(&call_single_queue)))
@@ -694,9 +708,11 @@ void flush_smp_call_function_from_idle(void) · 2 unmodified lines
694	708	cfd_seq_store(this_cpu_ptr(&cfd_seq_local)->idle, CFD_SEQ_NOCPU,
695	709	smp_processor_id(), CFD_SEQ_IDLE);
696	710	local_irq_save(flags);
697		− flush_smp_call_function_queue(true);
	711	+ /* Get the already pending soft interrupts for RT enabled kernels */
	712	+ was_pending = local_softirq_pending();
	713	+ __flush_smp_call_function_queue(true);
698	714	if (local_softirq_pending())
699		− do_softirq();
	715	+ do_softirq_post_smp_call_flush(was_pending);
700	716
701	717	local_irq_restore(flags);
702	718	}

kernel/softirq.c+13−0

@@ -294,6 +294,19 @@ static inline void invoke_softirq(void) · 293 unmodified lines
294	294	wakeup_softirqd();
295	295	}
296	296
	297	+/*
	298	+ * flush_smp_call_function_queue() can raise a soft interrupt in a function
	299	+ * call. On RT kernels this is undesired and the only known functionality
	300	+ * in the block layer which does this is disabled on RT. If soft interrupts
	301	+ * get raised which haven't been raised before the flush, warn so it can be
	302	+ * investigated.
	303	+ */
	304	+void do_softirq_post_smp_call_flush(unsigned int was_pending)
	305	+{
	306	+ if (WARN_ON_ONCE(was_pending != local_softirq_pending()))
	307	+ invoke_softirq();
	308	+}
	309	+
297	310	#else /* CONFIG_PREEMPT_RT */
298	311
299	312	/*

kernel/stop_machine.c+0−2

@@ -535,8 +535,6 @@ void stop_machine_park(int cpu) · 534 unmodified lines
535	535	kthread_park(stopper->thread);
536	536	}
537	537
538		−extern void sched_set_stop_task(int cpu, struct task_struct *stop);
539		−
540	538	static void cpu_stop_create(unsigned int cpu)
541	539	{
542	540	sched_set_stop_task(cpu, per_cpu(cpu_stopper.thread, cpu));

kernel/trace/trace.c+4−10

@@ -4289,17 +4289,11 @@ print_trace_header(struct seq_file m, struct trace_iterator iter) · 4288 unmodified lines
4289	4289	entries,
4290	4290	total,
4291	4291	buf->cpu,
4292		−#if defined(CONFIG_PREEMPT_NONE)
4293		− "server",
4294		−#elif defined(CONFIG_PREEMPT_VOLUNTARY)
4295		− "desktop",
4296		−#elif defined(CONFIG_PREEMPT)
4297		− "preempt",
4298		−#elif defined(CONFIG_PREEMPT_RT)
4299		− "preempt_rt",
4300		−#else
	4292	+ preempt_model_none() ? "server" :
	4293	+ preempt_model_voluntary() ? "desktop" :
	4294	+ preempt_model_full() ? "preempt" :
	4295	+ preempt_model_rt() ? "preempt_rt" :
4301	4296	"unknown",
4302		−#endif
4303	4297	/* These are reserved for later use */
4304	4298	0, 0, 0, 0);
4305	4299	#ifdef CONFIG_SMP