Merge tag 'sched-core-2022-05-23' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull scheduler updates from Ingo Molnar: - Updates to scheduler metrics: - PELT fixes & enhancements - PSI fixes & enhancements - Refactor cpu_util_without() - Updates to instrumentation/debugging: - Remove sched_trace_*() helper functions - can be done via debug info - Fix double update_rq_clock() warnings - Introduce & use "preemption model accessors" to simplify some of the Kconfig complexity. - Make softirq handling RT-safe. - Misc smaller fixes & cleanups. * tag 'sched-core-2022-05-23' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: topology: Remove unused cpu_cluster_mask() sched: Reverse sched_class layout sched/deadline: Remove superfluous rq clock update in push_dl_task() sched/core: Avoid obvious double update_rq_clock warning smp: Make softirq handling RT safe in flush_smp_call_function_queue() smp: Rename flush_smp_call_function_from_idle() sched: Fix missing prototype warnings sched/fair: Remove cfs_rq_tg_path() sched/fair: Remove sched_trace_*() helper functions sched/fair: Refactor cpu_util_without() sched/fair: Revise comment about lb decision matrix sched/psi: report zeroes for CPU full at the system level sched/fair: Delete useless condition in tg_unthrottle_up() sched/fair: Fix cfs_rq_clock_pelt() for throttled cfs_rq sched/fair: Move calculate of avg_load to a better location mailmap: Update my email address to @redhat.com MAINTAINERS: Add myself as scheduler topology reviewer psi: Fix trigger being fired unexpectedly at initial ftrace: Use preemption model accessors for trace header printout kcsan: Use preemption model accessors

Linus Torvalds committed 4 years ago

Commit 6f3f04c2 parents cfeb252 991d8d823 files · +213 −331Browse files

Showing the changes to kernel (16 of 23 files). Show all

Unified · Split

16 files

kernel/kcsan/kcsan_test.c +3 −2
kernel/sched/build_policy.c +2 −0
kernel/sched/build_utility.c +1 −0
kernel/sched/core.c +13 −10
kernel/sched/deadline.c +4 −11
kernel/sched/fair.c +67 −225
kernel/sched/idle.c +1 −1
kernel/sched/pelt.h +2 −2
kernel/sched/psi.c +11 −7
kernel/sched/rt.c +3 −2
kernel/sched/sched.h +36 −19
kernel/sched/smp.h +6 −0
kernel/smp.c +24 −8
kernel/softirq.c +13 −0
kernel/stop_machine.c +0 −2
kernel/trace/trace.c +4 −10

kernel/kcsan/kcsan_test.c+3−2

@@ -1380,13 +1380,14 @@ static const void nthreads_gen_params(const void prev, char *desc) · 1379 unmodified lines
1380	1380	else
1381	1381	nthreads *= 2;
1382	1382
1383		− if (!IS_ENABLED(CONFIG_PREEMPT) \|\| !IS_ENABLED(CONFIG_KCSAN_INTERRUPT_WATCHER)) {
	1383	+ if (!preempt_model_preemptible() \|\|
	1384	+ !IS_ENABLED(CONFIG_KCSAN_INTERRUPT_WATCHER)) {
1384	1385	/*
1385	1386	* Without any preemption, keep 2 CPUs free for other tasks, one
1386	1387	* of which is the main test case function checking for
1387	1388	* completion or failure.
1388	1389	*/
1389		− const long min_unused_cpus = IS_ENABLED(CONFIG_PREEMPT_NONE) ? 2 : 0;
	1390	+ const long min_unused_cpus = preempt_model_none() ? 2 : 0;
1390	1391	const long min_required_cpus = 2 + min_unused_cpus;
1391	1392
1392	1393	if (num_online_cpus() < min_required_cpus) {

kernel/sched/build_policy.c+2−0

@@ -15,6 +15,7 @@ · 14 unmodified lines
15	15	/* Headers: */
16	16	#include <linux/sched/clock.h>
17	17	#include <linux/sched/cputime.h>
	18	+#include <linux/sched/hotplug.h>
18	19	#include <linux/sched/posix-timers.h>
19	20	#include <linux/sched/rt.h>
20	21
@@ -31,6 +32,7 @@ · 10 unmodified lines
31	32	#include <uapi/linux/sched/types.h>
32	33
33	34	#include "sched.h"
	35	+#include "smp.h"
34	36
35	37	#include "autogroup.h"
36	38	#include "stats.h"

kernel/sched/build_utility.c+1−0

@@ -14,6 +14,7 @@ · 13 unmodified lines
14	14	#include <linux/sched/debug.h>
15	15	#include <linux/sched/isolation.h>
16	16	#include <linux/sched/loadavg.h>
	17	+#include <linux/sched/nohz.h>
17	18	#include <linux/sched/mm.h>
18	19	#include <linux/sched/rseq_api.h>
19	20	#include <linux/sched/task_stack.h>

kernel/sched/core.c+13−10

@@ -26,7 +26,10 @@ · 25 unmodified lines
26	26	#include <linux/topology.h>
27	27	#include <linux/sched/clock.h>
28	28	#include <linux/sched/cond_resched.h>
	29	+#include <linux/sched/cputime.h>
29	30	#include <linux/sched/debug.h>
	31	+#include <linux/sched/hotplug.h>
	32	+#include <linux/sched/init.h>
30	33	#include <linux/sched/isolation.h>
31	34	#include <linux/sched/loadavg.h>
32	35	#include <linux/sched/mm.h>
@@ -610,10 +613,10 @@ void double_rq_lock(struct rq rq1, struct rq rq2) · 577 unmodified lines
610	613	swap(rq1, rq2);
611	614
612	615	raw_spin_rq_lock(rq1);
613		− if (__rq_lockp(rq1) == __rq_lockp(rq2))
614		− return;
	616	+ if (__rq_lockp(rq1) != __rq_lockp(rq2))
	617	+ raw_spin_rq_lock_nested(rq2, SINGLE_DEPTH_NESTING);
615	618
616		− raw_spin_rq_lock_nested(rq2, SINGLE_DEPTH_NESTING);
	619	+ double_rq_clock_clear_update(rq1, rq2);
617	620	}
618	621	#endif
619	622
@@ -2190,7 +2193,7 @@ void check_preempt_curr(struct rq rq, struct task_struct p, int flags) · 1570 unmodified lines
2190	2193	{
2191	2194	if (p->sched_class == rq->curr->sched_class)
2192	2195	rq->curr->sched_class->check_preempt_curr(rq, p, flags);
2193		− else if (p->sched_class > rq->curr->sched_class)
	2196	+ else if (sched_class_above(p->sched_class, rq->curr->sched_class))
2194	2197	resched_curr(rq);
2195	2198
2196	2199	/*
@@ -2408,7 +2411,7 @@ static int migration_cpu_stop(void *data) · 211 unmodified lines
2408	2411	* __migrate_task() such that we will not miss enforcing cpus_ptr
2409	2412	* during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test.
2410	2413	*/
2411		− flush_smp_call_function_from_idle();
	2414	+ flush_smp_call_function_queue();
2412	2415
2413	2416	raw_spin_lock(&p->pi_lock);
2414	2417	rq_lock(rq, &rf);
@@ -5689,7 +5692,7 @@ __pick_next_task(struct rq rq, struct task_struct prev, struct rq_flags *rf) · 3274 unmodified lines
5689	5692	* higher scheduling class, because otherwise those lose the
5690	5693	* opportunity to pull in more work from other CPUs.
5691	5694	*/
5692		− if (likely(prev->sched_class <= &fair_sched_class &&
	5695	+ if (likely(!sched_class_above(prev->sched_class, &fair_sched_class) &&
5693	5696	rq->nr_running == rq->cfs.h_nr_running)) {
5694	5697
5695	5698	p = pick_next_task_fair(rq, prev, rf);
@@ -9469,11 +9472,11 @@ void __init sched_init(void) · 3773 unmodified lines
9469	9472	int i;
9470	9473
9471	9474	/* Make sure the linker didn't screw up */
9472		− BUG_ON(&idle_sched_class + 1 != &fair_sched_class \|\|
9473		− &fair_sched_class + 1 != &rt_sched_class \|\|
9474		− &rt_sched_class + 1 != &dl_sched_class);
	9475	+ BUG_ON(&idle_sched_class != &fair_sched_class + 1 \|\|
	9476	+ &fair_sched_class != &rt_sched_class + 1 \|\|
	9477	+ &rt_sched_class != &dl_sched_class + 1);
9475	9478	#ifdef CONFIG_SMP
9476		− BUG_ON(&dl_sched_class + 1 != &stop_sched_class);
	9479	+ BUG_ON(&dl_sched_class != &stop_sched_class + 1);
9477	9480	#endif
9478	9481
9479	9482	wait_bit_init();

kernel/sched/deadline.c+4−11

@@ -1220,8 +1220,6 @@ int dl_runtime_exceeded(struct sched_dl_entity *dl_se) · 1219 unmodified lines
1220	1220	return (dl_se->runtime <= 0);
1221	1221	}
1222	1222
1223		−extern bool sched_rt_bandwidth_account(struct rt_rq *rt_rq);
1224		−
1225	1223	/*
1226	1224	* This function implements the GRUB accounting rule:
1227	1225	* according to the GRUB reclaiming algorithm, the runtime is
@@ -1832,6 +1830,7 @@ out: · 604 unmodified lines
1832	1830
1833	1831	static void migrate_task_rq_dl(struct task_struct *p, int new_cpu __maybe_unused)
1834	1832	{
	1833	+ struct rq_flags rf;
1835	1834	struct rq *rq;
1836	1835
1837	1836	if (READ_ONCE(p->__state) != TASK_WAKING)
@@ -1843,7 +1842,7 @@ static void migrate_task_rq_dl(struct task_struct *p, int new_cpu __maybe_unused · 5 unmodified lines
1843	1842	* from try_to_wake_up(). Hence, p->pi_lock is locked, but
1844	1843	* rq->lock is not... So, lock it
1845	1844	*/
1846		− raw_spin_rq_lock(rq);
	1845	+ rq_lock(rq, &rf);
1847	1846	if (p->dl.dl_non_contending) {
1848	1847	update_rq_clock(rq);
1849	1848	sub_running_bw(&p->dl, &rq->dl);
@@ -1859,7 +1858,7 @@ static void migrate_task_rq_dl(struct task_struct *p, int new_cpu __maybe_unused · 9 unmodified lines
1859	1858	put_task_struct(p);
1860	1859	}
1861	1860	sub_rq_bw(&p->dl, &rq->dl);
1862		− raw_spin_rq_unlock(rq);
	1861	+ rq_unlock(rq, &rf);
1863	1862	}
1864	1863
1865	1864	static void check_preempt_equal_dl(struct rq rq, struct task_struct p)
@@ -2319,13 +2318,7 @@ retry: · 453 unmodified lines
2319	2318
2320	2319	deactivate_task(rq, next_task, 0);
2321	2320	set_task_cpu(next_task, later_rq->cpu);
2322		−
2323		− /*
2324		− * Update the later_rq clock here, because the clock is used
2325		− * by the cpufreq_update_util() inside __add_running_bw().
2326		− */
2327		− update_rq_clock(later_rq);
2328		− activate_task(later_rq, next_task, ENQUEUE_NOCLOCK);
	2321	+ activate_task(later_rq, next_task, 0);
2329	2322	ret = 1;
2330	2323
2331	2324	resched_curr(later_rq);

kernel/sched/fair.c+67−225

@@ -36,6 +36,7 @@ · 35 unmodified lines
36	36	#include <linux/sched/cond_resched.h>
37	37	#include <linux/sched/cputime.h>
38	38	#include <linux/sched/isolation.h>
	39	+#include <linux/sched/nohz.h>
39	40
40	41	#include <linux/cpuidle.h>
41	42	#include <linux/interrupt.h>
@@ -313,19 +314,6 @@ const struct sched_class fair_sched_class; · 271 unmodified lines
313	314	#define for_each_sched_entity(se) \
314	315	for (; se; se = se->parent)
315	316
316		−static inline void cfs_rq_tg_path(struct cfs_rq cfs_rq, char path, int len)
317		−{
318		− if (!path)
319		− return;
320		−
321		− if (cfs_rq && task_group_is_autogroup(cfs_rq->tg))
322		− autogroup_path(cfs_rq->tg, path, len);
323		− else if (cfs_rq && cfs_rq->tg->css.cgroup)
324		− cgroup_path(cfs_rq->tg->css.cgroup, path, len);
325		− else
326		− strlcpy(path, "(null)", len);
327		−}
328		−
329	317	static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
330	318	{
331	319	struct rq *rq = rq_of(cfs_rq);
@@ -493,12 +481,6 @@ static int se_is_idle(struct sched_entity *se) · 161 unmodified lines
493	481	#define for_each_sched_entity(se) \
494	482	for (; se; se = NULL)
495	483
496		−static inline void cfs_rq_tg_path(struct cfs_rq cfs_rq, char path, int len)
497		−{
498		− if (path)
499		− strlcpy(path, "(null)", len);
500		−}
501		−
502	484	static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
503	485	{
504	486	return true;
@@ -4846,11 +4828,11 @@ static int tg_unthrottle_up(struct task_group tg, void data) · 4341 unmodified lines
4846	4828
4847	4829	cfs_rq->throttle_count--;
4848	4830	if (!cfs_rq->throttle_count) {
4849		− cfs_rq->throttled_clock_task_time += rq_clock_task(rq) -
4850		− cfs_rq->throttled_clock_task;
	4831	+ cfs_rq->throttled_clock_pelt_time += rq_clock_pelt(rq) -
	4832	+ cfs_rq->throttled_clock_pelt;
4851	4833
4852	4834	/* Add cfs_rq with load or one or more already running entities to the list */
4853		− if (!cfs_rq_is_decayed(cfs_rq) \|\| cfs_rq->nr_running)
	4835	+ if (!cfs_rq_is_decayed(cfs_rq))
4854	4836	list_add_leaf_cfs_rq(cfs_rq);
4855	4837	}
4856	4838
@@ -4864,7 +4846,7 @@ static int tg_throttle_down(struct task_group tg, void data) · 7 unmodified lines
4864	4846
4865	4847	/* group is entering throttled state, stop time */
4866	4848	if (!cfs_rq->throttle_count) {
4867		− cfs_rq->throttled_clock_task = rq_clock_task(rq);
	4849	+ cfs_rq->throttled_clock_pelt = rq_clock_pelt(rq);
4868	4850	list_del_leaf_cfs_rq(cfs_rq);
4869	4851	}
4870	4852	cfs_rq->throttle_count++;
@@ -5308,7 +5290,7 @@ static void sync_throttle(struct task_group *tg, int cpu) · 437 unmodified lines
5308	5290	pcfs_rq = tg->parent->cfs_rq[cpu];
5309	5291
5310	5292	cfs_rq->throttle_count = pcfs_rq->throttle_count;
5311		− cfs_rq->throttled_clock_task = rq_clock_task(cpu_rq(cpu));
	5293	+ cfs_rq->throttled_clock_pelt = rq_clock_pelt(cpu_rq(cpu));
5312	5294	}
5313	5295
5314	5296	/* conditionally throttle active cfs_rq's from put_prev_entity() */
@@ -6544,108 +6526,19 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) · 1229 unmodified lines
6544	6526	}
6545	6527
6546	6528	/*
6547		− * cpu_util_without: compute cpu utilization without any contributions from *p
6548		− * @cpu: the CPU which utilization is requested
6549		− * @p: the task which utilization should be discounted
6550		− *
6551		− * The utilization of a CPU is defined by the utilization of tasks currently
6552		− * enqueued on that CPU as well as tasks which are currently sleeping after an
6553		− * execution on that CPU.
6554		− *
6555		− * This method returns the utilization of the specified CPU by discounting the
6556		− * utilization of the specified task, whenever the task is currently
6557		− * contributing to the CPU utilization.
6558		− */
6559		−static unsigned long cpu_util_without(int cpu, struct task_struct *p)
6560		−{
6561		− struct cfs_rq *cfs_rq;
6562		− unsigned int util;
6563		−
6564		− /* Task has no contribution or is new */
6565		− if (cpu != task_cpu(p) \|\| !READ_ONCE(p->se.avg.last_update_time))
6566		− return cpu_util_cfs(cpu);
6567		−
6568		− cfs_rq = &cpu_rq(cpu)->cfs;
6569		− util = READ_ONCE(cfs_rq->avg.util_avg);
6570		−
6571		− /* Discount task's util from CPU's util */
6572		− lsub_positive(&util, task_util(p));
6573		−
6574		− /*
6575		− * Covered cases:
6576		− *
6577		− * a) if *p is the only task sleeping on this CPU, then:
6578		− * cpu_util (== task_util) > util_est (== 0)
6579		− * and thus we return:
6580		− * cpu_util_without = (cpu_util - task_util) = 0
6581		− *
6582		− * b) if other tasks are SLEEPING on this CPU, which is now exiting
6583		− * IDLE, then:
6584		− * cpu_util >= task_util
6585		− * cpu_util > util_est (== 0)
6586		− * and thus we discount *p's blocked utilization to return:
6587		− * cpu_util_without = (cpu_util - task_util) >= 0
6588		− *
6589		− * c) if other tasks are RUNNABLE on that CPU and
6590		− * util_est > cpu_util
6591		− * then we use util_est since it returns a more restrictive
6592		− * estimation of the spare capacity on that CPU, by just
6593		− * considering the expected utilization of tasks already
6594		− * runnable on that CPU.
6595		− *
6596		− * Cases a) and b) are covered by the above code, while case c) is
6597		− * covered by the following code when estimated utilization is
6598		− * enabled.
6599		− */
6600		− if (sched_feat(UTIL_EST)) {
6601		− unsigned int estimated =
6602		− READ_ONCE(cfs_rq->avg.util_est.enqueued);
6603		−
6604		− /*
6605		− * Despite the following checks we still have a small window
6606		− * for a possible race, when an execl's select_task_rq_fair()
6607		− * races with LB's detach_task():
6608		− *
6609		− * detach_task()
6610		− * p->on_rq = TASK_ON_RQ_MIGRATING;
6611		− * ---------------------------------- A
6612		− * deactivate_task() \
6613		− * dequeue_task() + RaceTime
6614		− * util_est_dequeue() /
6615		− * ---------------------------------- B
6616		− *
6617		− * The additional check on "current == p" it's required to
6618		− * properly fix the execl regression and it helps in further
6619		− * reducing the chances for the above race.
6620		− */
6621		− if (unlikely(task_on_rq_queued(p) \|\| current == p))
6622		− lsub_positive(&estimated, _task_util_est(p));
6623		−
6624		− util = max(util, estimated);
6625		− }
6626		−
6627		− /*
6628		− * Utilization (estimated) can exceed the CPU capacity, thus let's
6629		− * clamp to the maximum CPU capacity to ensure consistency with
6630		− * cpu_util.
6631		− */
6632		− return min_t(unsigned long, util, capacity_orig_of(cpu));
6633		−}
6634		−
6635		−/*
6636		− * Predicts what cpu_util(@cpu) would return if @p was migrated (and enqueued)
6637		− * to @dst_cpu.
	6529	+ * Predicts what cpu_util(@cpu) would return if @p was removed from @cpu
	6530	+ * (@dst_cpu = -1) or migrated to @dst_cpu.
6638	6531	*/
6639	6532	static unsigned long cpu_util_next(int cpu, struct task_struct *p, int dst_cpu)
6640	6533	{
6641	6534	struct cfs_rq *cfs_rq = &cpu_rq(cpu)->cfs;
6642		− unsigned long util_est, util = READ_ONCE(cfs_rq->avg.util_avg);
	6535	+ unsigned long util = READ_ONCE(cfs_rq->avg.util_avg);
6643	6536
6644	6537	/*
6645		− * If @p migrates from @cpu to another, remove its contribution. Or,
6646		− * if @p migrates from another CPU to @cpu, add its contribution. In
6647		− * the other cases, @cpu is not impacted by the migration, so the
6648		− * util_avg should already be correct.
	6538	+ * If @dst_cpu is -1 or @p migrates from @cpu to @dst_cpu remove its
	6539	+ * contribution. If @p migrates from another CPU to @cpu add its
	6540	+ * contribution. In all the other cases @cpu is not impacted by the
	6541	+ * migration so its util_avg is already correct.
6649	6542	*/
6650	6543	if (task_cpu(p) == cpu && dst_cpu != cpu)
6651	6544	lsub_positive(&util, task_util(p));
@@ -6653,16 +6546,40 @@ static unsigned long cpu_util_next(int cpu, struct task_struct *p, int dst_cpu) · 1 unmodified line
6653	6546	util += task_util(p);
6654	6547
6655	6548	if (sched_feat(UTIL_EST)) {
	6549	+ unsigned long util_est;
	6550	+
6656	6551	util_est = READ_ONCE(cfs_rq->avg.util_est.enqueued);
6657	6552
6658	6553	/*
6659		− * During wake-up, the task isn't enqueued yet and doesn't
6660		− * appear in the cfs_rq->avg.util_est.enqueued of any rq,
6661		− * so just add it (if needed) to "simulate" what will be
6662		− * cpu_util after the task has been enqueued.
	6554	+ * During wake-up @p isn't enqueued yet and doesn't contribute
	6555	+ * to any cpu_rq(cpu)->cfs.avg.util_est.enqueued.
	6556	+ * If @dst_cpu == @cpu add it to "simulate" cpu_util after @p
	6557	+ * has been enqueued.
	6558	+ *
	6559	+ * During exec (@dst_cpu = -1) @p is enqueued and does
	6560	+ * contribute to cpu_rq(cpu)->cfs.util_est.enqueued.
	6561	+ * Remove it to "simulate" cpu_util without @p's contribution.
	6562	+ *
	6563	+ * Despite the task_on_rq_queued(@p) check there is still a
	6564	+ * small window for a possible race when an exec
	6565	+ * select_task_rq_fair() races with LB's detach_task().
	6566	+ *
	6567	+ * detach_task()
	6568	+ * deactivate_task()
	6569	+ * p->on_rq = TASK_ON_RQ_MIGRATING;
	6570	+ * -------------------------------- A
	6571	+ * dequeue_task() \
	6572	+ * dequeue_task_fair() + Race Time
	6573	+ * util_est_dequeue() /
	6574	+ * -------------------------------- B
	6575	+ *
	6576	+ * The additional check "current == p" is required to further
	6577	+ * reduce the race window.
6663	6578	*/
6664	6579	if (dst_cpu == cpu)
6665	6580	util_est += _task_util_est(p);
	6581	+ else if (unlikely(task_on_rq_queued(p) \|\| current == p))
	6582	+ lsub_positive(&util_est, _task_util_est(p));
6666	6583
6667	6584	util = max(util, util_est);
6668	6585	}
@@ -6670,6 +6587,28 @@ static unsigned long cpu_util_next(int cpu, struct task_struct *p, int dst_cpu) · 1 unmodified line
6670	6587	return min(util, capacity_orig_of(cpu));
6671	6588	}
6672	6589
	6590	+/*
	6591	+ * cpu_util_without: compute cpu utilization without any contributions from *p
	6592	+ * @cpu: the CPU which utilization is requested
	6593	+ * @p: the task which utilization should be discounted
	6594	+ *
	6595	+ * The utilization of a CPU is defined by the utilization of tasks currently
	6596	+ * enqueued on that CPU as well as tasks which are currently sleeping after an
	6597	+ * execution on that CPU.
	6598	+ *
	6599	+ * This method returns the utilization of the specified CPU by discounting the
	6600	+ * utilization of the specified task, whenever the task is currently
	6601	+ * contributing to the CPU utilization.
	6602	+ */
	6603	+static unsigned long cpu_util_without(int cpu, struct task_struct *p)
	6604	+{
	6605	+ /* Task has no contribution or is new */
	6606	+ if (cpu != task_cpu(p) \|\| !READ_ONCE(p->se.avg.last_update_time))
	6607	+ return cpu_util_cfs(cpu);
	6608	+
	6609	+ return cpu_util_next(cpu, p, -1);
	6610	+}
	6611	+
6673	6612	/*
6674	6613	* compute_energy(): Estimates the energy that @pd would consume if @p was
6675	6614	* migrated to @dst_cpu. compute_energy() predicts what will be the utilization
@@ -9460,8 +9399,6 @@ static inline void calculate_imbalance(struct lb_env env, struct sd_lb_stats s · 2784 unmodified lines
9460	9399	local->avg_load = (local->group_load * SCHED_CAPACITY_SCALE) /
9461	9400	local->group_capacity;
9462	9401
9463		− sds->avg_load = (sds->total_load * SCHED_CAPACITY_SCALE) /
9464		− sds->total_capacity;
9465	9402	/*
9466	9403	* If the local group is more loaded than the selected
9467	9404	* busiest group don't try to pull any tasks.
@@ -9470,6 +9407,9 @@ static inline void calculate_imbalance(struct lb_env env, struct sd_lb_stats s · 2 unmodified lines
9470	9407	env->imbalance = 0;
9471	9408	return;
9472	9409	}
	9410	+
	9411	+ sds->avg_load = (sds->total_load * SCHED_CAPACITY_SCALE) /
	9412	+ sds->total_capacity;
9473	9413	}
9474	9414
9475	9415	/*
@@ -9495,7 +9435,7 @@ static inline void calculate_imbalance(struct lb_env env, struct sd_lb_stats s · 19 unmodified lines
9495	9435	* busiest \ local has_spare fully_busy misfit asym imbalanced overloaded
9496	9436	* has_spare nr_idle balanced N/A N/A balanced balanced
9497	9437	* fully_busy nr_idle nr_idle N/A N/A balanced balanced
9498		− * misfit_task force N/A N/A N/A force force
	9438	+ * misfit_task force N/A N/A N/A N/A N/A
9499	9439	* asym_packing force force N/A N/A force force
9500	9440	* imbalanced force force N/A N/A force force
9501	9441	* overloaded force force N/A N/A force avg_load
@@ -11881,101 +11821,3 @@ __init void init_sched_fair_class(void) · 2379 unmodified lines
11881	11821	#endif /* SMP */
11882	11822
11883	11823	}
11884		−
11885		−/*
11886		− * Helper functions to facilitate extracting info from tracepoints.
11887		− */
11888		−
11889		−const struct sched_avg sched_trace_cfs_rq_avg(struct cfs_rq cfs_rq)
11890		−{
11891		−#ifdef CONFIG_SMP
11892		− return cfs_rq ? &cfs_rq->avg : NULL;
11893		−#else
11894		− return NULL;
11895		−#endif
11896		−}
11897		−EXPORT_SYMBOL_GPL(sched_trace_cfs_rq_avg);
11898		−
11899		−char sched_trace_cfs_rq_path(struct cfs_rq cfs_rq, char *str, int len)
11900		−{
11901		− if (!cfs_rq) {
11902		− if (str)
11903		− strlcpy(str, "(null)", len);
11904		− else
11905		− return NULL;
11906		− }
11907		−
11908		− cfs_rq_tg_path(cfs_rq, str, len);
11909		− return str;
11910		−}
11911		−EXPORT_SYMBOL_GPL(sched_trace_cfs_rq_path);
11912		−
11913		−int sched_trace_cfs_rq_cpu(struct cfs_rq *cfs_rq)
11914		−{
11915		− return cfs_rq ? cpu_of(rq_of(cfs_rq)) : -1;
11916		−}
11917		−EXPORT_SYMBOL_GPL(sched_trace_cfs_rq_cpu);
11918		−
11919		−const struct sched_avg sched_trace_rq_avg_rt(struct rq rq)
11920		−{
11921		−#ifdef CONFIG_SMP
11922		− return rq ? &rq->avg_rt : NULL;
11923		−#else
11924		− return NULL;
11925		−#endif
11926		−}
11927		−EXPORT_SYMBOL_GPL(sched_trace_rq_avg_rt);
11928		−
11929		−const struct sched_avg sched_trace_rq_avg_dl(struct rq rq)
11930		−{
11931		−#ifdef CONFIG_SMP
11932		− return rq ? &rq->avg_dl : NULL;
11933		−#else
11934		− return NULL;
11935		−#endif
11936		−}
11937		−EXPORT_SYMBOL_GPL(sched_trace_rq_avg_dl);
11938		−
11939		−const struct sched_avg sched_trace_rq_avg_irq(struct rq rq)
11940		−{
11941		−#if defined(CONFIG_SMP) && defined(CONFIG_HAVE_SCHED_AVG_IRQ)
11942		− return rq ? &rq->avg_irq : NULL;
11943		−#else
11944		− return NULL;
11945		−#endif
11946		−}
11947		−EXPORT_SYMBOL_GPL(sched_trace_rq_avg_irq);
11948		−
11949		−int sched_trace_rq_cpu(struct rq *rq)
11950		−{
11951		− return rq ? cpu_of(rq) : -1;
11952		−}
11953		−EXPORT_SYMBOL_GPL(sched_trace_rq_cpu);
11954		−
11955		−int sched_trace_rq_cpu_capacity(struct rq *rq)
11956		−{
11957		− return rq ?
11958		−#ifdef CONFIG_SMP
11959		− rq->cpu_capacity
11960		−#else
11961		− SCHED_CAPACITY_SCALE
11962		−#endif
11963		− : -1;
11964		−}
11965		−EXPORT_SYMBOL_GPL(sched_trace_rq_cpu_capacity);
11966		−
11967		−const struct cpumask sched_trace_rd_span(struct root_domain rd)
11968		−{
11969		−#ifdef CONFIG_SMP
11970		− return rd ? rd->span : NULL;
11971		−#else
11972		− return NULL;
11973		−#endif
11974		−}
11975		−EXPORT_SYMBOL_GPL(sched_trace_rd_span);
11976		−
11977		−int sched_trace_rq_nr_running(struct rq *rq)
11978		−{
11979		− return rq ? rq->nr_running : -1;
11980		−}
11981		−EXPORT_SYMBOL_GPL(sched_trace_rq_nr_running);

kernel/sched/idle.c+1−1

@@ -327,7 +327,7 @@ static void do_idle(void) · 326 unmodified lines
327	327	* RCU relies on this call to be done outside of an RCU read-side
328	328	* critical section.
329	329	*/
330		− flush_smp_call_function_from_idle();
	330	+ flush_smp_call_function_queue();
331	331	schedule_idle();
332	332
333	333	if (unlikely(klp_patch_pending(current)))

kernel/sched/pelt.h+2−2

@@ -145,9 +145,9 @@ static inline u64 rq_clock_pelt(struct rq *rq) · 144 unmodified lines
145	145	static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
146	146	{
147	147	if (unlikely(cfs_rq->throttle_count))
148		− return cfs_rq->throttled_clock_task - cfs_rq->throttled_clock_task_time;
	148	+ return cfs_rq->throttled_clock_pelt - cfs_rq->throttled_clock_pelt_time;
149	149
150		− return rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_task_time;
	150	+ return rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_pelt_time;
151	151	}
152	152	#else
153	153	static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)

kernel/sched/psi.c+11−7

@@ -1060,14 +1060,17 @@ int psi_show(struct seq_file m, struct psi_group group, enum psi_res res) · 1059 unmodified lines
1060	1060	mutex_unlock(&group->avgs_lock);
1061	1061
1062	1062	for (full = 0; full < 2; full++) {
1063		− unsigned long avg[3];
1064		− u64 total;
	1063	+ unsigned long avg[3] = { 0, };
	1064	+ u64 total = 0;
1065	1065	int w;
1066	1066
1067		− for (w = 0; w < 3; w++)
1068		− avg[w] = group->avg[res * 2 + full][w];
1069		− total = div_u64(group->total[PSI_AVGS][res * 2 + full],
1070		− NSEC_PER_USEC);
	1067	+ /* CPU FULL is undefined at the system level */
	1068	+ if (!(group == &psi_system && res == PSI_CPU && full)) {
	1069	+ for (w = 0; w < 3; w++)
	1070	+ avg[w] = group->avg[res * 2 + full][w];
	1071	+ total = div_u64(group->total[PSI_AVGS][res * 2 + full],
	1072	+ NSEC_PER_USEC);
	1073	+ }
1071	1074
1072	1075	seq_printf(m, "%s avg10=%lu.%02lu avg60=%lu.%02lu avg300=%lu.%02lu total=%llu\n",
1073	1076	full ? "full" : "some",
@@ -1117,7 +1120,8 @@ struct psi_trigger psi_trigger_create(struct psi_group group, · 43 unmodified lines
1117	1120	t->state = state;
1118	1121	t->threshold = threshold_us * NSEC_PER_USEC;
1119	1122	t->win.size = window_us * NSEC_PER_USEC;
1120		− window_reset(&t->win, 0, 0, 0);
	1123	+ window_reset(&t->win, sched_clock(),
	1124	+ group->total[PSI_POLL][t->state], 0);
1121	1125
1122	1126	t->event = 0;
1123	1127	t->last_event_time = 0;

kernel/sched/rt.c+3−2

@@ -871,6 +871,7 @@ static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun) · 870 unmodified lines
871	871	int enqueue = 0;
872	872	struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i);
873	873	struct rq *rq = rq_of_rt_rq(rt_rq);
	874	+ struct rq_flags rf;
874	875	int skip;
875	876
876	877	/*
@@ -885,7 +886,7 @@ static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun) · 8 unmodified lines
885	886	if (skip)
886	887	continue;
887	888
888		− raw_spin_rq_lock(rq);
	889	+ rq_lock(rq, &rf);
889	890	update_rq_clock(rq);
890	891
891	892	if (rt_rq->rt_time) {
@@ -923,7 +924,7 @@ static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun) · 31 unmodified lines
923	924
924	925	if (enqueue)
925	926	sched_rt_rq_enqueue(rt_rq);
926		− raw_spin_rq_unlock(rq);
	927	+ rq_unlock(rq, &rf);
927	928	}
928	929
929	930	if (!throttled && (!rt_bandwidth_enabled() \|\| rt_b->rt_runtime == RUNTIME_INF))

kernel/sched/sched.h+36−19

@@ -603,8 +603,8 @@ struct cfs_rq { · 602 unmodified lines
603	603	s64 runtime_remaining;
604	604
605	605	u64 throttled_clock;
606		− u64 throttled_clock_task;
607		− u64 throttled_clock_task_time;
	606	+ u64 throttled_clock_pelt;
	607	+ u64 throttled_clock_pelt_time;
608	608	int throttled;
609	609	int throttle_count;
610	610	struct list_head throttled_list;
@@ -1827,12 +1827,7 @@ static inline void dirty_sched_domain_sysctl(int cpu) · 1216 unmodified lines
1827	1827	#endif
1828	1828
1829	1829	extern int sched_update_scaling(void);
1830		−
1831		−extern void flush_smp_call_function_from_idle(void);
1832		−
1833		−#else /* !CONFIG_SMP: */
1834		−static inline void flush_smp_call_function_from_idle(void) { }
1835		−#endif
	1830	+#endif /* CONFIG_SMP */
1836	1831
1837	1832	#include "stats.h"
1838	1833
@@ -2182,6 +2177,8 @@ static inline void set_next_task(struct rq rq, struct task_struct next) · 343 unmodified lines
2182	2177	*
2183	2178	* include/asm-generic/vmlinux.lds.h
2184	2179	*
	2180	+ * CAREFUL they are laid out in REVERSE order!!!
	2181	+ *
2185	2182	* Also enforce alignment on the instance, not the type, to guarantee layout.
2186	2183	*/
2187	2184	#define DEFINE_SCHED_CLASS(name) \
@@ -2190,17 +2187,16 @@ const struct sched_class name##_sched_class \ · 2 unmodified lines
2190	2187	__section("__" #name "_sched_class")
2191	2188
2192	2189	/* Defined in include/asm-generic/vmlinux.lds.h */
2193		−extern struct sched_class __begin_sched_classes[];
2194		−extern struct sched_class __end_sched_classes[];
2195		−
2196		−#define sched_class_highest (__end_sched_classes - 1)
2197		−#define sched_class_lowest (__begin_sched_classes - 1)
	2190	+extern struct sched_class __sched_class_highest[];
	2191	+extern struct sched_class __sched_class_lowest[];
2198	2192
2199	2193	#define for_class_range(class, _from, _to) \
2200		− for (class = (_from); class != (_to); class--)
	2194	+ for (class = (_from); class < (_to); class++)
2201	2195
2202	2196	#define for_each_class(class) \
2203		− for_class_range(class, sched_class_highest, sched_class_lowest)
	2197	+ for_class_range(class, __sched_class_highest, __sched_class_lowest)
	2198	+
	2199	+#define sched_class_above(_a, _b) ((_a) < (_b))
2204	2200
2205	2201	extern const struct sched_class stop_sched_class;
2206	2202	extern const struct sched_class dl_sched_class;
@@ -2309,6 +2305,7 @@ extern void resched_cpu(int cpu); · 102 unmodified lines
2309	2305
2310	2306	extern struct rt_bandwidth def_rt_bandwidth;
2311	2307	extern void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime);
	2308	+extern bool sched_rt_bandwidth_account(struct rt_rq *rt_rq);
2312	2309
2313	2310	extern void init_dl_bandwidth(struct dl_bandwidth *dl_b, u64 period, u64 runtime);
2314	2311	extern void init_dl_task_timer(struct sched_dl_entity *dl_se);
@@ -2478,6 +2475,24 @@ unsigned long arch_scale_freq_capacity(int cpu) · 163 unmodified lines
2478	2475	}
2479	2476	#endif
2480	2477
	2478	+#ifdef CONFIG_SCHED_DEBUG
	2479	+/*
	2480	+ * In double_lock_balance()/double_rq_lock(), we use raw_spin_rq_lock() to
	2481	+ * acquire rq lock instead of rq_lock(). So at the end of these two functions
	2482	+ * we need to call double_rq_clock_clear_update() to clear RQCF_UPDATED of
	2483	+ * rq->clock_update_flags to avoid the WARN_DOUBLE_CLOCK warning.
	2484	+ */
	2485	+static inline void double_rq_clock_clear_update(struct rq rq1, struct rq rq2)
	2486	+{
	2487	+ rq1->clock_update_flags &= (RQCF_REQ_SKIP\|RQCF_ACT_SKIP);
	2488	+ /* rq1 == rq2 for !CONFIG_SMP, so just clear RQCF_UPDATED once. */
	2489	+#ifdef CONFIG_SMP
	2490	+ rq2->clock_update_flags &= (RQCF_REQ_SKIP\|RQCF_ACT_SKIP);
	2491	+#endif
	2492	+}
	2493	+#else
	2494	+static inline void double_rq_clock_clear_update(struct rq rq1, struct rq rq2) {}
	2495	+#endif
2481	2496
2482	2497	#ifdef CONFIG_SMP
2483	2498
@@ -2543,14 +2558,15 @@ static inline int _double_lock_balance(struct rq this_rq, struct rq busiest) · 59 unmodified lines
2543	2558	__acquires(busiest->lock)
2544	2559	__acquires(this_rq->lock)
2545	2560	{
2546		− if (__rq_lockp(this_rq) == __rq_lockp(busiest))
2547		− return 0;
2548		−
2549		− if (likely(raw_spin_rq_trylock(busiest)))
	2561	+ if (__rq_lockp(this_rq) == __rq_lockp(busiest) \|\|
	2562	+ likely(raw_spin_rq_trylock(busiest))) {
	2563	+ double_rq_clock_clear_update(this_rq, busiest);
2550	2564	return 0;
	2565	+ }
2551	2566
2552	2567	if (rq_order_less(this_rq, busiest)) {
2553	2568	raw_spin_rq_lock_nested(busiest, SINGLE_DEPTH_NESTING);
	2569	+ double_rq_clock_clear_update(this_rq, busiest);
2554	2570	return 0;
2555	2571	}
2556	2572
@@ -2644,6 +2660,7 @@ static inline void double_rq_lock(struct rq rq1, struct rq rq2) · 87 unmodified lines
2644	2660	BUG_ON(rq1 != rq2);
2645	2661	raw_spin_rq_lock(rq1);
2646	2662	__acquire(rq2->lock); /* Fake it out ;) */
	2663	+ double_rq_clock_clear_update(rq1, rq2);
2647	2664	}
2648	2665
2649	2666	/*

kernel/sched/smp.h+6−0

@@ -7,3 +7,9 @@ · 6 unmodified lines
7	7	extern void sched_ttwu_pending(void *arg);
8	8
9	9	extern void send_call_function_single_ipi(int cpu);
	10	+
	11	+#ifdef CONFIG_SMP
	12	+extern void flush_smp_call_function_queue(void);
	13	+#else
	14	+static inline void flush_smp_call_function_queue(void) { }
	15	+#endif

kernel/smp.c+24−8

@@ -96,7 +96,7 @@ static DEFINE_PER_CPU_ALIGNED(struct call_function_data, cfd_data); · 95 unmodified lines
96	96
97	97	static DEFINE_PER_CPU_SHARED_ALIGNED(struct llist_head, call_single_queue);
98	98
99		−static void flush_smp_call_function_queue(bool warn_cpu_offline);
	99	+static void __flush_smp_call_function_queue(bool warn_cpu_offline);
100	100
101	101	int smpcfd_prepare_cpu(unsigned int cpu)
102	102	{
@@ -141,7 +141,7 @@ int smpcfd_dying_cpu(unsigned int cpu) · 38 unmodified lines
141	141	* ensure that the outgoing CPU doesn't go offline with work
142	142	* still pending.
143	143	*/
144		− flush_smp_call_function_queue(false);
	144	+ __flush_smp_call_function_queue(false);
145	145	irq_work_run();
146	146	return 0;
147	147	}
@@ -544,11 +544,11 @@ void generic_smp_call_function_single_interrupt(void) · 396 unmodified lines
544	544	{
545	545	cfd_seq_store(this_cpu_ptr(&cfd_seq_local)->gotipi, CFD_SEQ_NOCPU,
546	546	smp_processor_id(), CFD_SEQ_GOTIPI);
547		− flush_smp_call_function_queue(true);
	547	+ __flush_smp_call_function_queue(true);
548	548	}
549	549
550	550	/**
551		− * flush_smp_call_function_queue - Flush pending smp-call-function callbacks
	551	+ * __flush_smp_call_function_queue - Flush pending smp-call-function callbacks
552	552	*
553	553	* @warn_cpu_offline: If set to 'true', warn if callbacks were queued on an
554	554	* offline CPU. Skip this check if set to 'false'.
@@ -561,7 +561,7 @@ void generic_smp_call_function_single_interrupt(void) · 6 unmodified lines
561	561	* Loop through the call_single_queue and run all the queued callbacks.
562	562	* Must be called with interrupts disabled.
563	563	*/
564		−static void flush_smp_call_function_queue(bool warn_cpu_offline)
	564	+static void __flush_smp_call_function_queue(bool warn_cpu_offline)
565	565	{
566	566	call_single_data_t csd, csd_next;
567	567	struct llist_node entry, prev;
@@ -684,8 +684,22 @@ static void flush_smp_call_function_queue(bool warn_cpu_offline) · 116 unmodified lines
684	684	smp_processor_id(), CFD_SEQ_HDLEND);
685	685	}
686	686
687		−void flush_smp_call_function_from_idle(void)
	687	+
	688	+/**
	689	+ * flush_smp_call_function_queue - Flush pending smp-call-function callbacks
	690	+ * from task context (idle, migration thread)
	691	+ *
	692	+ * When TIF_POLLING_NRFLAG is supported and a CPU is in idle and has it
	693	+ * set, then remote CPUs can avoid sending IPIs and wake the idle CPU by
	694	+ * setting TIF_NEED_RESCHED. The idle task on the woken up CPU has to
	695	+ * handle queued SMP function calls before scheduling.
	696	+ *
	697	+ * The migration thread has to ensure that an eventually pending wakeup has
	698	+ * been handled before it migrates a task.
	699	+ */
	700	+void flush_smp_call_function_queue(void)
688	701	{
	702	+ unsigned int was_pending;
689	703	unsigned long flags;
690	704
691	705	if (llist_empty(this_cpu_ptr(&call_single_queue)))
@@ -694,9 +708,11 @@ void flush_smp_call_function_from_idle(void) · 2 unmodified lines
694	708	cfd_seq_store(this_cpu_ptr(&cfd_seq_local)->idle, CFD_SEQ_NOCPU,
695	709	smp_processor_id(), CFD_SEQ_IDLE);
696	710	local_irq_save(flags);
697		− flush_smp_call_function_queue(true);
	711	+ /* Get the already pending soft interrupts for RT enabled kernels */
	712	+ was_pending = local_softirq_pending();
	713	+ __flush_smp_call_function_queue(true);
698	714	if (local_softirq_pending())
699		− do_softirq();
	715	+ do_softirq_post_smp_call_flush(was_pending);
700	716
701	717	local_irq_restore(flags);
702	718	}

kernel/softirq.c+13−0

@@ -294,6 +294,19 @@ static inline void invoke_softirq(void) · 293 unmodified lines
294	294	wakeup_softirqd();
295	295	}
296	296
	297	+/*
	298	+ * flush_smp_call_function_queue() can raise a soft interrupt in a function
	299	+ * call. On RT kernels this is undesired and the only known functionality
	300	+ * in the block layer which does this is disabled on RT. If soft interrupts
	301	+ * get raised which haven't been raised before the flush, warn so it can be
	302	+ * investigated.
	303	+ */
	304	+void do_softirq_post_smp_call_flush(unsigned int was_pending)
	305	+{
	306	+ if (WARN_ON_ONCE(was_pending != local_softirq_pending()))
	307	+ invoke_softirq();
	308	+}
	309	+
297	310	#else /* CONFIG_PREEMPT_RT */
298	311
299	312	/*

kernel/stop_machine.c+0−2

@@ -535,8 +535,6 @@ void stop_machine_park(int cpu) · 534 unmodified lines
535	535	kthread_park(stopper->thread);
536	536	}
537	537
538		−extern void sched_set_stop_task(int cpu, struct task_struct *stop);
539		−
540	538	static void cpu_stop_create(unsigned int cpu)
541	539	{
542	540	sched_set_stop_task(cpu, per_cpu(cpu_stopper.thread, cpu));

kernel/trace/trace.c+4−10

@@ -4289,17 +4289,11 @@ print_trace_header(struct seq_file m, struct trace_iterator iter) · 4288 unmodified lines
4289	4289	entries,
4290	4290	total,
4291	4291	buf->cpu,
4292		−#if defined(CONFIG_PREEMPT_NONE)
4293		− "server",
4294		−#elif defined(CONFIG_PREEMPT_VOLUNTARY)
4295		− "desktop",
4296		−#elif defined(CONFIG_PREEMPT)
4297		− "preempt",
4298		−#elif defined(CONFIG_PREEMPT_RT)
4299		− "preempt_rt",
4300		−#else
	4292	+ preempt_model_none() ? "server" :
	4293	+ preempt_model_voluntary() ? "desktop" :
	4294	+ preempt_model_full() ? "preempt" :
	4295	+ preempt_model_rt() ? "preempt_rt" :
4301	4296	"unknown",
4302		−#endif
4303	4297	/* These are reserved for later use */
4304	4298	0, 0, 0, 0);
4305	4299	#ifdef CONFIG_SMP