Merge tag 'sched-core-2024-09-19' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull scheduler updates from Ingo Molnar: - Implement the SCHED_DEADLINE server infrastructure - Daniel Bristot de Oliveira's last major contribution to the kernel: "SCHED_DEADLINE servers can help fixing starvation issues of low priority tasks (e.g., SCHED_OTHER) when higher priority tasks monopolize CPU cycles. Today we have RT Throttling; DEADLINE servers should be able to replace and improve that." (Daniel Bristot de Oliveira, Peter Zijlstra, Joel Fernandes, Youssef Esmat, Huang Shijie) - Preparatory changes for sched_ext integration: - Use set_next_task(.first) where required - Fix up set_next_task() implementations - Clean up DL server vs. core sched - Split up put_prev_task_balance() - Rework pick_next_task() - Combine the last put_prev_task() and the first set_next_task() - Rework dl_server - Add put_prev_task(.next) (Peter Zijlstra, with a fix by Tejun Heo) - Complete the EEVDF transition and refine EEVDF scheduling: - Implement delayed dequeue - Allow shorter slices to wakeup-preempt - Use sched_attr::sched_runtime to set request/slice suggestion - Document the new feature flags - Remove unused and duplicate-functionality fields - Simplify & unify pick_next_task_fair() - Misc debuggability enhancements (Peter Zijlstra, with fixes/cleanups by Dietmar Eggemann, Valentin Schneider and Chuyi Zhou) - Initialize the vruntime of a new task when it is first enqueued, resulting in significant decrease in latency of newly woken tasks (Zhang Qiao) - Introduce SM_IDLE and an idle re-entry fast-path in __schedule() (K Prateek Nayak, Peter Zijlstra) - Clean up and clarify the usage of Clean up usage of rt_task() (Qais Yousef) - Preempt SCHED_IDLE entities in strict cgroup hierarchies (Tianchen Ding) - Clarify the documentation of time units for deadline scheduler parameters (Christian Loehle) - Remove the HZ_BW chicken-bit feature flag introduced a year ago, the original change seems to be working fine (Phil Auld) - Misc fixes and cleanups (Chen Yu, Dan Carpenter, Huang Shijie, Peilin He, Qais Yousefm and Vincent Guittot) * tag 'sched-core-2024-09-19' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (64 commits) sched/cpufreq: Use NSEC_PER_MSEC for deadline task cpufreq/cppc: Use NSEC_PER_MSEC for deadline task sched/deadline: Clarify nanoseconds in uapi sched/deadline: Convert schedtool example to chrt sched/debug: Fix the runnable tasks output sched: Fix sched_delayed vs sched_core kernel/sched: Fix util_est accounting for DELAY_DEQUEUE kthread: Fix task state in kthread worker if being frozen sched/pelt: Use rq_clock_task() for hw_pressure sched/fair: Move effective_cpu_util() and effective_cpu_util() in fair.c sched/core: Introduce SM_IDLE and an idle re-entry fast-path in __schedule() sched: Add put_prev_task(.next) sched: Rework dl_server sched: Combine the last put_prev_task() and the first set_next_task() sched: Rework pick_next_task() sched: Split up put_prev_task_balance() sched: Clean up DL server vs core sched sched: Fixup set_next_task() implementations sched: Use set_next_task(.first) where required sched/fair: Properly deactivate sched_delayed task upon class change ...

Linus Torvalds committed 2 years ago

Commit 2004cef2 parents 509d2cd bc9057d32 files · +1695 −747Browse files

Unified · Split

32 files

Documentation/scheduler/sched-deadline.rst +6 −8
drivers/cpufreq/cppc_cpufreq.c +3 −3
fs/bcachefs/six.c +1 −1
fs/proc/base.c +1 −1
include/linux/ioprio.h +1 −1
include/linux/sched.h +24 −4
include/linux/sched/deadline.h +7 −7
include/linux/sched/prio.h +1 −0
include/linux/sched/rt.h +27 −6
include/uapi/linux/sched/types.h +3 −3
kernel/freezer.c +1 −1
kernel/kthread.c +9 −1
kernel/locking/rtmutex.c +2 −2
kernel/locking/rwsem.c +2 −2
kernel/locking/ww_mutex.h +1 −1
kernel/sched/core.c +166 −82
kernel/sched/cpufreq_schedutil.c +3 −3
kernel/sched/deadline.c +403 −100
kernel/sched/debug.c +186 −12
kernel/sched/fair.c +580 −190
kernel/sched/features.h +28 −2
kernel/sched/idle.c +7 −16
kernel/sched/rt.c +112 −149
kernel/sched/sched.h +78 −23
kernel/sched/stop_task.c +4 −14
kernel/sched/syscalls.c +25 −109
kernel/sched/topology.c +8 −0
kernel/sys.c +1 −1
kernel/time/hrtimer.c +1 −1
kernel/trace/trace_sched_wakeup.c +1 −1
mm/page-writeback.c +2 −2
mm/page_alloc.c +1 −1

Documentation/scheduler/sched-deadline.rst+6−8

@@ -749,21 +749,19 @@ Appendix A. Test suite · 748 unmodified lines
749	749	of the command line options. Please refer to rt-app documentation for more
750	750	details (`<rt-app-sources>/doc/*.json`).
751	751
752		− The second testing application is a modification of schedtool, called
753		− schedtool-dl, which can be used to setup SCHED_DEADLINE parameters for a
754		− certain pid/application. schedtool-dl is available at:
755		− https://github.com/scheduler-tools/schedtool-dl.git.
	752	+ The second testing application is done using chrt which has support
	753	+ for SCHED_DEADLINE.
756	754
757	755	The usage is straightforward::
758	756
759		− # schedtool -E -t 10000000:100000000 -e ./my_cpuhog_app
	757	+ # chrt -d -T 10000000 -D 100000000 0 ./my_cpuhog_app
760	758
761	759	With this, my_cpuhog_app is put to run inside a SCHED_DEADLINE reservation
762		− of 10ms every 100ms (note that parameters are expressed in microseconds).
763		− You can also use schedtool to create a reservation for an already running
	760	+ of 10ms every 100ms (note that parameters are expressed in nanoseconds).
	761	+ You can also use chrt to create a reservation for an already running
764	762	application, given that you know its pid::
765	763
766		− # schedtool -E -t 10000000:100000000 my_app_pid
	764	+ # chrt -d -T 10000000 -D 100000000 -p 0 my_app_pid
767	765
768	766	Appendix B. Minimal main()
769	767	==========================

drivers/cpufreq/cppc_cpufreq.c+3−3

@@ -224,9 +224,9 @@ static void __init cppc_freq_invariance_init(void) · 223 unmodified lines
224	224	* Fake (unused) bandwidth; workaround to "fix"
225	225	* priority inheritance.
226	226	*/
227		− .sched_runtime = 1000000,
228		− .sched_deadline = 10000000,
229		− .sched_period = 10000000,
	227	+ .sched_runtime = NSEC_PER_MSEC,
	228	+ .sched_deadline = 10 * NSEC_PER_MSEC,
	229	+ .sched_period = 10 * NSEC_PER_MSEC,
230	230	};
231	231	int ret;
232	232

fs/bcachefs/six.c+1−1

@@ -335,7 +335,7 @@ static inline bool six_owner_running(struct six_lock *lock) · 334 unmodified lines
335	335	*/
336	336	rcu_read_lock();
337	337	struct task_struct *owner = READ_ONCE(lock->owner);
338		− bool ret = owner ? owner_on_cpu(owner) : !rt_task(current);
	338	+ bool ret = owner ? owner_on_cpu(owner) : !rt_or_dl_task(current);
339	339	rcu_read_unlock();
340	340
341	341	return ret;

fs/proc/base.c+1−1

@@ -2626,7 +2626,7 @@ static ssize_t timerslack_ns_write(struct file file, const char __user buf, · 2625 unmodified lines
2626	2626	}
2627	2627
2628	2628	task_lock(p);
2629		− if (task_is_realtime(p))
	2629	+ if (rt_or_dl_task_policy(p))
2630	2630	slack_ns = 0;
2631	2631	else if (slack_ns == 0)
2632	2632	slack_ns = p->default_timer_slack_ns;

include/linux/ioprio.h+1−1

@@ -40,7 +40,7 @@ static inline int task_nice_ioclass(struct task_struct *task) · 39 unmodified lines
40	40	{
41	41	if (task->policy == SCHED_IDLE)
42	42	return IOPRIO_CLASS_IDLE;
43		− else if (task_is_realtime(task))
	43	+ else if (rt_or_dl_task_policy(task))
44	44	return IOPRIO_CLASS_RT;
45	45	else
46	46	return IOPRIO_CLASS_BE;

include/linux/sched.h+24−4

@@ -149,8 +149,9 @@ struct user_event_mm; · 148 unmodified lines
149	149	* Special states are those that do not use the normal wait-loop pattern. See
150	150	* the comment with set_special_state().
151	151	*/
152		−#define is_special_task_state(state) \
153		− ((state) & (__TASK_STOPPED \| __TASK_TRACED \| TASK_PARKED \| TASK_DEAD))
	152	+#define is_special_task_state(state) \
	153	+ ((state) & (__TASK_STOPPED \| __TASK_TRACED \| TASK_PARKED \| \
	154	+ TASK_DEAD \| TASK_FROZEN))
154	155
155	156	#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
156	157	# define debug_normal_state_change(state_value) \
@@ -541,9 +542,14 @@ struct sched_entity { · 384 unmodified lines
541	542	struct rb_node run_node;
542	543	u64 deadline;
543	544	u64 min_vruntime;
	545	+ u64 min_slice;
544	546
545	547	struct list_head group_node;
546		− unsigned int on_rq;
	548	+ unsigned char on_rq;
	549	+ unsigned char sched_delayed;
	550	+ unsigned char rel_deadline;
	551	+ unsigned char custom_slice;
	552	+ /* hole */
547	553
548	554	u64 exec_start;
549	555	u64 sum_exec_runtime;
@@ -639,12 +645,26 @@ struct sched_dl_entity { · 89 unmodified lines
639	645	*
640	646	* @dl_overrun tells if the task asked to be informed about runtime
641	647	* overruns.
	648	+ *
	649	+ * @dl_server tells if this is a server entity.
	650	+ *
	651	+ * @dl_defer tells if this is a deferred or regular server. For
	652	+ * now only defer server exists.
	653	+ *
	654	+ * @dl_defer_armed tells if the deferrable server is waiting
	655	+ * for the replenishment timer to activate it.
	656	+ *
	657	+ * @dl_defer_running tells if the deferrable server is actually
	658	+ * running, skipping the defer phase.
642	659	*/
643	660	unsigned int dl_throttled : 1;
644	661	unsigned int dl_yielded : 1;
645	662	unsigned int dl_non_contending : 1;
646	663	unsigned int dl_overrun : 1;
647	664	unsigned int dl_server : 1;
	665	+ unsigned int dl_defer : 1;
	666	+ unsigned int dl_defer_armed : 1;
	667	+ unsigned int dl_defer_running : 1;
648	668
649	669	/*
650	670	* Bandwidth enforcement timer. Each -deadline task has its
@@ -672,7 +692,7 @@ struct sched_dl_entity { · 21 unmodified lines
672	692	*/
673	693	struct rq *rq;
674	694	dl_server_has_tasks_f server_has_tasks;
675		− dl_server_pick_f server_pick;
	695	+ dl_server_pick_f server_pick_task;
676	696
677	697	#ifdef CONFIG_RT_MUTEXES
678	698	/*

include/linux/sched/deadline.h+7−7

@@ -10,16 +10,16 @@ · 9 unmodified lines
10	10
11	11	#include <linux/sched.h>
12	12
13		−#define MAX_DL_PRIO 0
14		−
15		−static inline int dl_prio(int prio)
	13	+static inline bool dl_prio(int prio)
16	14	{
17		− if (unlikely(prio < MAX_DL_PRIO))
18		− return 1;
19		− return 0;
	15	+ return unlikely(prio < MAX_DL_PRIO);
20	16	}
21	17
22		−static inline int dl_task(struct task_struct *p)
	18	+/*
	19	+ * Returns true if a task has a priority that belongs to DL class. PI-boosted
	20	+ * tasks will return true. Use dl_policy() to ignore PI-boosted tasks.
	21	+ */
	22	+static inline bool dl_task(struct task_struct *p)
23	23	{
24	24	return dl_prio(p->prio);
25	25	}

include/linux/sched/prio.h+1−0

@@ -14,6 +14,7 @@ · 13 unmodified lines
14	14	*/
15	15
16	16	#define MAX_RT_PRIO 100
	17	+#define MAX_DL_PRIO 0
17	18
18	19	#define MAX_PRIO (MAX_RT_PRIO + NICE_WIDTH)
19	20	#define DEFAULT_PRIO (MAX_RT_PRIO + NICE_WIDTH / 2)

include/linux/sched/rt.h+27−6

@@ -6,19 +6,40 @@ · 5 unmodified lines
6	6
7	7	struct task_struct;
8	8
9		−static inline int rt_prio(int prio)
	9	+static inline bool rt_prio(int prio)
10	10	{
11		− if (unlikely(prio < MAX_RT_PRIO))
12		− return 1;
13		− return 0;
	11	+ return unlikely(prio < MAX_RT_PRIO && prio >= MAX_DL_PRIO);
14	12	}
15	13
16		−static inline int rt_task(struct task_struct *p)
	14	+static inline bool rt_or_dl_prio(int prio)
	15	+{
	16	+ return unlikely(prio < MAX_RT_PRIO);
	17	+}
	18	+
	19	+/*
	20	+ * Returns true if a task has a priority that belongs to RT class. PI-boosted
	21	+ * tasks will return true. Use rt_policy() to ignore PI-boosted tasks.
	22	+ */
	23	+static inline bool rt_task(struct task_struct *p)
17	24	{
18	25	return rt_prio(p->prio);
19	26	}
20	27
21		−static inline bool task_is_realtime(struct task_struct *tsk)
	28	+/*
	29	+ * Returns true if a task has a priority that belongs to RT or DL classes.
	30	+ * PI-boosted tasks will return true. Use rt_or_dl_task_policy() to ignore
	31	+ * PI-boosted tasks.
	32	+ */
	33	+static inline bool rt_or_dl_task(struct task_struct *p)
	34	+{
	35	+ return rt_or_dl_prio(p->prio);
	36	+}
	37	+
	38	+/*
	39	+ * Returns true if a task has a policy that belongs to RT or DL classes.
	40	+ * PI-boosted tasks will return false.
	41	+ */
	42	+static inline bool rt_or_dl_task_policy(struct task_struct *tsk)
22	43	{
23	44	int policy = tsk->policy;
24	45

include/uapi/linux/sched/types.h+3−3

@@ -58,9 +58,9 @@ · 57 unmodified lines
58	58	*
59	59	* This is reflected by the following fields of the sched_attr structure:
60	60	*
61		− * @sched_deadline representative of the task's deadline
62		− * @sched_runtime representative of the task's runtime
63		− * @sched_period representative of the task's period
	61	+ * @sched_deadline representative of the task's deadline in nanoseconds
	62	+ * @sched_runtime representative of the task's runtime in nanoseconds
	63	+ * @sched_period representative of the task's period in nanoseconds
64	64	*
65	65	* Given this task model, there are a multiplicity of scheduling algorithms
66	66	* and policies, that can be used to ensure all the tasks will make their

kernel/freezer.c+1−1

@@ -72,7 +72,7 @@ bool __refrigerator(bool check_kthr_stop) · 71 unmodified lines
72	72	bool freeze;
73	73
74	74	raw_spin_lock_irq(&current->pi_lock);
75		− set_current_state(TASK_FROZEN);
	75	+ WRITE_ONCE(current->__state, TASK_FROZEN);
76	76	/* unstale saved_state so that __thaw_task() will wake us up */
77	77	current->saved_state = TASK_RUNNING;
78	78	raw_spin_unlock_irq(&current->pi_lock);

kernel/kthread.c+9−1

@@ -845,8 +845,16 @@ repeat: · 844 unmodified lines
845	845	* event only cares about the address.
846	846	*/
847	847	trace_sched_kthread_work_execute_end(work, func);
848		− } else if (!freezing(current))
	848	+ } else if (!freezing(current)) {
849	849	schedule();
	850	+ } else {
	851	+ /*
	852	+ * Handle the case where the current remains
	853	+ * TASK_INTERRUPTIBLE. try_to_freeze() expects
	854	+ * the current to be TASK_RUNNING.
	855	+ */
	856	+ __set_current_state(TASK_RUNNING);
	857	+ }
850	858
851	859	try_to_freeze();
852	860	cond_resched();

kernel/locking/rtmutex.c+2−2

@@ -347,7 +347,7 @@ static __always_inline int __waiter_prio(struct task_struct *task) · 346 unmodified lines
347	347	{
348	348	int prio = task->prio;
349	349
350		− if (!rt_prio(prio))
	350	+ if (!rt_or_dl_prio(prio))
351	351	return DEFAULT_PRIO;
352	352
353	353	return prio;
@@ -435,7 +435,7 @@ static inline bool rt_mutex_steal(struct rt_mutex_waiter *waiter, · 81 unmodified lines
435	435	* Note that RT tasks are excluded from same priority (lateral)
436	436	* steals to prevent the introduction of an unbounded latency.
437	437	*/
438		− if (rt_prio(waiter->tree.prio) \|\| dl_prio(waiter->tree.prio))
	438	+ if (rt_or_dl_prio(waiter->tree.prio))
439	439	return false;
440	440
441	441	return rt_waiter_node_equal(&waiter->tree, &top_waiter->tree);

kernel/locking/rwsem.c+2−2

@@ -631,7 +631,7 @@ static inline bool rwsem_try_write_lock(struct rw_semaphore *sem, · 630 unmodified lines
631	631	* if it is an RT task or wait in the wait queue
632	632	* for too long.
633	633	*/
634		− if (has_handoff \|\| (!rt_task(waiter->task) &&
	634	+ if (has_handoff \|\| (!rt_or_dl_task(waiter->task) &&
635	635	!time_after(jiffies, waiter->timeout)))
636	636	return false;
637	637
@@ -914,7 +914,7 @@ static bool rwsem_optimistic_spin(struct rw_semaphore *sem) · 276 unmodified lines
914	914	if (owner_state != OWNER_WRITER) {
915	915	if (need_resched())
916	916	break;
917		− if (rt_task(current) &&
	917	+ if (rt_or_dl_task(current) &&
918	918	(prev_owner_state != OWNER_WRITER))
919	919	break;
920	920	}

kernel/locking/ww_mutex.h+1−1

@@ -237,7 +237,7 @@ __ww_ctx_less(struct ww_acquire_ctx a, struct ww_acquire_ctx b) · 236 unmodified lines
237	237	int a_prio = a->task->prio;
238	238	int b_prio = b->task->prio;
239	239
240		− if (rt_prio(a_prio) \|\| rt_prio(b_prio)) {
	240	+ if (rt_or_dl_prio(a_prio) \|\| rt_or_dl_prio(b_prio)) {
241	241
242	242	if (a_prio > b_prio)
243	243	return true;

kernel/sched/core.c+166−82

@@ -163,7 +163,10 @@ static inline int __task_prio(const struct task_struct *p) · 162 unmodified lines
163	163	if (p->sched_class == &stop_sched_class) /* trumps deadline */
164	164	return -2;
165	165
166		− if (rt_prio(p->prio)) /* includes deadline */
	166	+ if (p->dl_server)
	167	+ return -1; /* deadline */
	168	+
	169	+ if (rt_or_dl_prio(p->prio))
167	170	return p->prio; /* [-1, 99] */
168	171
169	172	if (p->sched_class == &idle_sched_class)
@@ -192,8 +195,24 @@ static inline bool prio_less(const struct task_struct *a, · 22 unmodified lines
192	195	if (-pb < -pa)
193	196	return false;
194	197
195		− if (pa == -1) /* dl_prio() doesn't work because of stop_class above */
196		− return !dl_time_before(a->dl.deadline, b->dl.deadline);
	198	+ if (pa == -1) { /* dl_prio() doesn't work because of stop_class above */
	199	+ const struct sched_dl_entity a_dl, b_dl;
	200	+
	201	+ a_dl = &a->dl;
	202	+ /*
	203	+ * Since,'a' and 'b' can be CFS tasks served by DL server,
	204	+ * __task_prio() can return -1 (for DL) even for those. In that
	205	+ * case, get to the dl_server's DL entity.
	206	+ */
	207	+ if (a->dl_server)
	208	+ a_dl = a->dl_server;
	209	+
	210	+ b_dl = &b->dl;
	211	+ if (b->dl_server)
	212	+ b_dl = b->dl_server;
	213	+
	214	+ return !dl_time_before(a_dl->deadline, b_dl->deadline);
	215	+ }
197	216
198	217	if (pa == MAX_RT_PRIO + MAX_NICE) /* fair */
199	218	return cfs_prio_less(a, b, in_fi);
@@ -240,6 +259,9 @@ static inline int rb_sched_core_cmp(const void key, const struct rb_node node) · 40 unmodified lines
240	259
241	260	void sched_core_enqueue(struct rq rq, struct task_struct p)
242	261	{
	262	+ if (p->se.sched_delayed)
	263	+ return;
	264	+
243	265	rq->core->core_task_seq++;
244	266
245	267	if (!p->core_cookie)
@@ -250,6 +272,9 @@ void sched_core_enqueue(struct rq rq, struct task_struct p) · 4 unmodified lines
250	272
251	273	void sched_core_dequeue(struct rq rq, struct task_struct p, int flags)
252	274	{
	275	+ if (p->se.sched_delayed)
	276	+ return;
	277	+
253	278	rq->core->core_task_seq++;
254	279
255	280	if (sched_core_enqueued(p)) {
@@ -1269,7 +1294,7 @@ bool sched_can_stop_tick(struct rq *rq) · 1013 unmodified lines
1269	1294	* dequeued by migrating while the constrained task continues to run.
1270	1295	* E.g. going from 2->1 without going through pick_next_task().
1271	1296	*/
1272		− if (sched_feat(HZ_BW) && __need_bw_check(rq, rq->curr)) {
	1297	+ if (__need_bw_check(rq, rq->curr)) {
1273	1298	if (cfs_task_bw_constrained(rq->curr))
1274	1299	return false;
1275	1300	}
@@ -1672,6 +1697,9 @@ static inline void uclamp_rq_inc(struct rq rq, struct task_struct p) · 396 unmodified lines
1672	1697	if (unlikely(!p->sched_class->uclamp_enabled))
1673	1698	return;
1674	1699
	1700	+ if (p->se.sched_delayed)
	1701	+ return;
	1702	+
1675	1703	for_each_clamp_id(clamp_id)
1676	1704	uclamp_rq_inc_id(rq, p, clamp_id);
1677	1705
@@ -1696,6 +1724,9 @@ static inline void uclamp_rq_dec(struct rq rq, struct task_struct p) · 18 unmodified lines
1696	1724	if (unlikely(!p->sched_class->uclamp_enabled))
1697	1725	return;
1698	1726
	1727	+ if (p->se.sched_delayed)
	1728	+ return;
	1729	+
1699	1730	for_each_clamp_id(clamp_id)
1700	1731	uclamp_rq_dec_id(rq, p, clamp_id);
1701	1732	}
@@ -1975,14 +2006,21 @@ void enqueue_task(struct rq rq, struct task_struct p, int flags) · 273 unmodified lines
1975	2006	psi_enqueue(p, (flags & ENQUEUE_WAKEUP) && !(flags & ENQUEUE_MIGRATED));
1976	2007	}
1977	2008
1978		− uclamp_rq_inc(rq, p);
1979	2009	p->sched_class->enqueue_task(rq, p, flags);
	2010	+ /*
	2011	+ * Must be after ->enqueue_task() because ENQUEUE_DELAYED can clear
	2012	+ * ->sched_delayed.
	2013	+ */
	2014	+ uclamp_rq_inc(rq, p);
1980	2015
1981	2016	if (sched_core_enabled(rq))
1982	2017	sched_core_enqueue(rq, p);
1983	2018	}
1984	2019
1985		−void dequeue_task(struct rq rq, struct task_struct p, int flags)
	2020	+/*
	2021	+ * Must only return false when DEQUEUE_SLEEP.
	2022	+ */
	2023	+inline bool dequeue_task(struct rq rq, struct task_struct p, int flags)
1986	2024	{
1987	2025	if (sched_core_enabled(rq))
1988	2026	sched_core_dequeue(rq, p, flags);
@@ -1995,8 +2033,12 @@ void dequeue_task(struct rq rq, struct task_struct p, int flags) · 6 unmodified lines
1995	2033	psi_dequeue(p, flags & DEQUEUE_SLEEP);
1996	2034	}
1997	2035
	2036	+ /*
	2037	+ * Must be before ->dequeue_task() because ->dequeue_task() can 'fail'
	2038	+ * and mark the task ->sched_delayed.
	2039	+ */
1998	2040	uclamp_rq_dec(rq, p);
1999		− p->sched_class->dequeue_task(rq, p, flags);
	2041	+ return p->sched_class->dequeue_task(rq, p, flags);
2000	2042	}
2001	2043
2002	2044	void activate_task(struct rq rq, struct task_struct p, int flags)
@@ -2014,12 +2056,25 @@ void activate_task(struct rq rq, struct task_struct p, int flags) · 11 unmodified lines
2014	2056
2015	2057	void deactivate_task(struct rq rq, struct task_struct p, int flags)
2016	2058	{
2017		− WRITE_ONCE(p->on_rq, (flags & DEQUEUE_SLEEP) ? 0 : TASK_ON_RQ_MIGRATING);
	2059	+ SCHED_WARN_ON(flags & DEQUEUE_SLEEP);
	2060	+
	2061	+ WRITE_ONCE(p->on_rq, TASK_ON_RQ_MIGRATING);
2018	2062	ASSERT_EXCLUSIVE_WRITER(p->on_rq);
2019	2063
	2064	+ /*
	2065	+ * Code explicitly relies on TASK_ON_RQ_MIGRATING begin set before
	2066	+ * dequeue_task() and cleared after enqueue_task().
	2067	+ */
	2068	+
2020	2069	dequeue_task(rq, p, flags);
2021	2070	}
2022	2071
	2072	+static void block_task(struct rq rq, struct task_struct p, int flags)
	2073	+{
	2074	+ if (dequeue_task(rq, p, DEQUEUE_SLEEP \| flags))
	2075	+ __block_task(rq, p);
	2076	+}
	2077	+
2023	2078	/**
2024	2079	* task_curr - is this task currently executing on a CPU?
2025	2080	* @p: the task in question.
@@ -2233,6 +2288,12 @@ void migrate_disable(void) · 207 unmodified lines
2233	2288	struct task_struct *p = current;
2234	2289
2235	2290	if (p->migration_disabled) {
	2291	+#ifdef CONFIG_DEBUG_PREEMPT
	2292	+ /*
	2293	+ *Warn about overflow half-way through the range.
	2294	+ */
	2295	+ WARN_ON_ONCE((s16)p->migration_disabled < 0);
	2296	+#endif
2236	2297	p->migration_disabled++;
2237	2298	return;
2238	2299	}
@@ -2251,14 +2312,20 @@ void migrate_enable(void) · 12 unmodified lines
2251	2312	.flags = SCA_MIGRATE_ENABLE,
2252	2313	};
2253	2314
	2315	+#ifdef CONFIG_DEBUG_PREEMPT
	2316	+ /*
	2317	+ * Check both overflow from migrate_disable() and superfluous
	2318	+ * migrate_enable().
	2319	+ */
	2320	+ if (WARN_ON_ONCE((s16)p->migration_disabled <= 0))
	2321	+ return;
	2322	+#endif
	2323	+
2254	2324	if (p->migration_disabled > 1) {
2255	2325	p->migration_disabled--;
2256	2326	return;
2257	2327	}
2258	2328
2259		− if (WARN_ON_ONCE(!p->migration_disabled))
2260		− return;
2261		−
2262	2329	/*
2263	2330	* Ensure stop_task runs either before or after this, and that
2264	2331	* __set_cpus_allowed_ptr(SCA_MIGRATE_ENABLE) doesn't schedule().
@@ -3607,8 +3674,6 @@ ttwu_do_activate(struct rq rq, struct task_struct p, int wake_flags, · 1342 unmodified lines
3607	3674	rq->idle_stamp = 0;
3608	3675	}
3609	3676	#endif
3610		−
3611		− p->dl_server = NULL;
3612	3677	}
3613	3678
3614	3679	/*
@@ -3644,12 +3709,14 @@ static int ttwu_runnable(struct task_struct *p, int wake_flags) · 29 unmodified lines
3644	3709
3645	3710	rq = __task_rq_lock(p, &rf);
3646	3711	if (task_on_rq_queued(p)) {
	3712	+ update_rq_clock(rq);
	3713	+ if (p->se.sched_delayed)
	3714	+ enqueue_task(rq, p, ENQUEUE_NOCLOCK \| ENQUEUE_DELAYED);
3647	3715	if (!task_on_cpu(rq, p)) {
3648	3716	/*
3649	3717	* When on_rq && !on_cpu the task is preempted, see if
3650	3718	* it should preempt the task that is current now.
3651	3719	*/
3652		− update_rq_clock(rq);
3653	3720	wakeup_preempt(rq, p, wake_flags);
3654	3721	}
3655	3722	ttwu_do_wakeup(p);
@@ -4029,11 +4096,16 @@ int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) · 373 unmodified lines
4029	4096	* case the whole 'p->on_rq && ttwu_runnable()' case below
4030	4097	* without taking any locks.
4031	4098	*
	4099	+ * Specifically, given current runs ttwu() we must be before
	4100	+ * schedule()'s block_task(), as such this must not observe
	4101	+ * sched_delayed.
	4102	+ *
4032	4103	* In particular:
4033	4104	* - we rely on Program-Order guarantees for all the ordering,
4034	4105	* - we're serialized against set_special_state() by virtue of
4035	4106	* it disabling IRQs (this allows not taking ->pi_lock).
4036	4107	*/
	4108	+ SCHED_WARN_ON(p->se.sched_delayed);
4037	4109	if (!ttwu_state_match(p, state, &success))
4038	4110	goto out;
4039	4111
@@ -4322,9 +4394,11 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p) · 282 unmodified lines
4322	4394	p->se.nr_migrations = 0;
4323	4395	p->se.vruntime = 0;
4324	4396	p->se.vlag = 0;
4325		− p->se.slice = sysctl_sched_base_slice;
4326	4397	INIT_LIST_HEAD(&p->se.group_node);
4327	4398
	4399	+ /* A delayed task cannot be in clone(). */
	4400	+ SCHED_WARN_ON(p->se.sched_delayed);
	4401	+
4328	4402	#ifdef CONFIG_FAIR_GROUP_SCHED
4329	4403	p->se.cfs_rq = NULL;
4330	4404	#endif
@@ -4572,6 +4646,8 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) · 241 unmodified lines
4572	4646
4573	4647	p->prio = p->normal_prio = p->static_prio;
4574	4648	set_load_weight(p, false);
	4649	+ p->se.custom_slice = 0;
	4650	+ p->se.slice = sysctl_sched_base_slice;
4575	4651
4576	4652	/*
4577	4653	* We don't need the reset flag anymore after the fork. It has
@@ -4686,7 +4762,7 @@ void wake_up_new_task(struct task_struct *p) · 108 unmodified lines
4686	4762	update_rq_clock(rq);
4687	4763	post_init_entity_util_avg(p);
4688	4764
4689		− activate_task(rq, p, ENQUEUE_NOCLOCK);
	4765	+ activate_task(rq, p, ENQUEUE_NOCLOCK \| ENQUEUE_INITIAL);
4690	4766	trace_sched_wakeup_new(p);
4691	4767	wakeup_preempt(rq, p, WF_FORK);
4692	4768	#ifdef CONFIG_SMP
@@ -5769,8 +5845,8 @@ static inline void schedule_debug(struct task_struct *prev, bool preempt) · 1076 unmodified lines
5769	5845	schedstat_inc(this_rq()->sched_count);
5770	5846	}
5771	5847
5772		−static void put_prev_task_balance(struct rq rq, struct task_struct prev,
5773		− struct rq_flags *rf)
	5848	+static void prev_balance(struct rq rq, struct task_struct prev,
	5849	+ struct rq_flags *rf)
5774	5850	{
5775	5851	#ifdef CONFIG_SMP
5776	5852	const struct sched_class *class;
@@ -5787,8 +5863,6 @@ static void put_prev_task_balance(struct rq rq, struct task_struct prev, · 10 unmodified lines
5787	5863	break;
5788	5864	}
5789	5865	#endif
5790		−
5791		− put_prev_task(rq, prev);
5792	5866	}
5793	5867
5794	5868	/*
@@ -5800,6 +5874,8 @@ __pick_next_task(struct rq rq, struct task_struct prev, struct rq_flags *rf) · 5 unmodified lines
5800	5874	const struct sched_class *class;
5801	5875	struct task_struct *p;
5802	5876
	5877	+ rq->dl_server = NULL;
	5878	+
5803	5879	/*
5804	5880	* Optimization: we know that if all tasks are in the fair class we can
5805	5881	* call that function directly, but only if the @prev task wasn't of a
@@ -5815,35 +5891,28 @@ __pick_next_task(struct rq rq, struct task_struct prev, struct rq_flags *rf) · 9 unmodified lines
5815	5891
5816	5892	/* Assume the next prioritized class is idle_sched_class */
5817	5893	if (!p) {
5818		− put_prev_task(rq, prev);
5819		− p = pick_next_task_idle(rq);
	5894	+ p = pick_task_idle(rq);
	5895	+ put_prev_set_next_task(rq, prev, p);
5820	5896	}
5821	5897
5822		− /*
5823		− * This is the fast path; it cannot be a DL server pick;
5824		− * therefore even if @p == @prev, ->dl_server must be NULL.
5825		− */
5826		− if (p->dl_server)
5827		− p->dl_server = NULL;
5828		−
5829	5898	return p;
5830	5899	}
5831	5900
5832	5901	restart:
5833		− put_prev_task_balance(rq, prev, rf);
5834		−
5835		− /*
5836		− * We've updated @prev and no longer need the server link, clear it.
5837		− * Must be done before ->pick_next_task() because that can (re)set
5838		− * ->dl_server.
5839		− */
5840		− if (prev->dl_server)
5841		− prev->dl_server = NULL;
	5902	+ prev_balance(rq, prev, rf);
5842	5903
5843	5904	for_each_class(class) {
5844		− p = class->pick_next_task(rq);
5845		− if (p)
5846		− return p;
	5905	+ if (class->pick_next_task) {
	5906	+ p = class->pick_next_task(rq, prev);
	5907	+ if (p)
	5908	+ return p;
	5909	+ } else {
	5910	+ p = class->pick_task(rq);
	5911	+ if (p) {
	5912	+ put_prev_set_next_task(rq, prev, p);
	5913	+ return p;
	5914	+ }
	5915	+ }
5847	5916	}
5848	5917
5849	5918	BUG(); /* The idle class should always have a runnable task. */
@@ -5873,6 +5942,8 @@ static inline struct task_struct pick_task(struct rq rq) · 23 unmodified lines
5873	5942	const struct sched_class *class;
5874	5943	struct task_struct *p;
5875	5944
	5945	+ rq->dl_server = NULL;
	5946	+
5876	5947	for_each_class(class) {
5877	5948	p = class->pick_task(rq);
5878	5949	if (p)
@@ -5911,6 +5982,7 @@ pick_next_task(struct rq rq, struct task_struct prev, struct rq_flags *rf) · 32 unmodified lines
5911	5982	* another cpu during offline.
5912	5983	*/
5913	5984	rq->core_pick = NULL;
	5985	+ rq->core_dl_server = NULL;
5914	5986	return __pick_next_task(rq, prev, rf);
5915	5987	}
5916	5988
@@ -5929,16 +6001,13 @@ pick_next_task(struct rq rq, struct task_struct prev, struct rq_flags *rf) · 12 unmodified lines
5929	6001	WRITE_ONCE(rq->core_sched_seq, rq->core->core_pick_seq);
5930	6002
5931	6003	next = rq->core_pick;
5932		− if (next != prev) {
5933		− put_prev_task(rq, prev);
5934		− set_next_task(rq, next);
5935		− }
5936		−
	6004	+ rq->dl_server = rq->core_dl_server;
5937	6005	rq->core_pick = NULL;
5938		− goto out;
	6006	+ rq->core_dl_server = NULL;
	6007	+ goto out_set_next;
5939	6008	}
5940	6009
5941		− put_prev_task_balance(rq, prev, rf);
	6010	+ prev_balance(rq, prev, rf);
5942	6011
5943	6012	smt_mask = cpu_smt_mask(cpu);
5944	6013	need_sync = !!rq->core->core_cookie;
@@ -5979,6 +6048,7 @@ pick_next_task(struct rq rq, struct task_struct prev, struct rq_flags *rf) · 34 unmodified lines
5979	6048	next = pick_task(rq);
5980	6049	if (!next->core_cookie) {
5981	6050	rq->core_pick = NULL;
	6051	+ rq->core_dl_server = NULL;
5982	6052	/*
5983	6053	* For robustness, update the min_vruntime_fi for
5984	6054	* unconstrained picks as well.
@@ -6006,7 +6076,9 @@ pick_next_task(struct rq rq, struct task_struct prev, struct rq_flags *rf) · 21 unmodified lines
6006	6076	if (i != cpu && (rq_i != rq->core \|\| !core_clock_updated))
6007	6077	update_rq_clock(rq_i);
6008	6078
6009		− p = rq_i->core_pick = pick_task(rq_i);
	6079	+ rq_i->core_pick = p = pick_task(rq_i);
	6080	+ rq_i->core_dl_server = rq_i->dl_server;
	6081	+
6010	6082	if (!max \|\| prio_less(max, p, fi_before))
6011	6083	max = p;
6012	6084	}
@@ -6030,6 +6102,7 @@ pick_next_task(struct rq rq, struct task_struct prev, struct rq_flags *rf) · 17 unmodified lines
6030	6102	}
6031	6103
6032	6104	rq_i->core_pick = p;
	6105	+ rq_i->core_dl_server = NULL;
6033	6106
6034	6107	if (p == rq_i->idle) {
6035	6108	if (rq_i->nr_running) {
@@ -6090,6 +6163,7 @@ pick_next_task(struct rq rq, struct task_struct prev, struct rq_flags *rf) · 54 unmodified lines
6090	6163
6091	6164	if (i == cpu) {
6092	6165	rq_i->core_pick = NULL;
	6166	+ rq_i->core_dl_server = NULL;
6093	6167	continue;
6094	6168	}
6095	6169
@@ -6098,6 +6172,7 @@ pick_next_task(struct rq rq, struct task_struct prev, struct rq_flags *rf) · 2 unmodified lines
6098	6172
6099	6173	if (rq_i->curr == rq_i->core_pick) {
6100	6174	rq_i->core_pick = NULL;
	6175	+ rq_i->core_dl_server = NULL;
6101	6176	continue;
6102	6177	}
6103	6178
@@ -6105,8 +6180,7 @@ pick_next_task(struct rq rq, struct task_struct prev, struct rq_flags *rf) · 1 unmodified line
6105	6180	}
6106	6181
6107	6182	out_set_next:
6108		− set_next_task(rq, next);
6109		−out:
	6183	+ put_prev_set_next_task(rq, prev, next);
6110	6184	if (rq->core->core_forceidle_count && next == rq->idle)
6111	6185	queue_core_balance(rq);
6112	6186
@@ -6342,19 +6416,12 @@ pick_next_task(struct rq rq, struct task_struct prev, struct rq_flags *rf) · 229 unmodified lines
6342	6416	* Constants for the sched_mode argument of __schedule().
6343	6417	*
6344	6418	* The mode argument allows RT enabled kernels to differentiate a
6345		− * preemption from blocking on an 'sleeping' spin/rwlock. Note that
6346		− * SM_MASK_PREEMPT for !RT has all bits set, which allows the compiler to
6347		− * optimize the AND operation out and just check for zero.
	6419	+ * preemption from blocking on an 'sleeping' spin/rwlock.
6348	6420	*/
6349		−#define SM_NONE 0x0
6350		−#define SM_PREEMPT 0x1
6351		−#define SM_RTLOCK_WAIT 0x2
6352		−
6353		−#ifndef CONFIG_PREEMPT_RT
6354		−# define SM_MASK_PREEMPT (~0U)
6355		−#else
6356		−# define SM_MASK_PREEMPT SM_PREEMPT
6357		−#endif
	6421	+#define SM_IDLE (-1)
	6422	+#define SM_NONE 0
	6423	+#define SM_PREEMPT 1
	6424	+#define SM_RTLOCK_WAIT 2
6358	6425
6359	6426	/*
6360	6427	* __schedule() is the main scheduler function.
@@ -6395,9 +6462,14 @@ pick_next_task(struct rq rq, struct task_struct prev, struct rq_flags *rf) · 34 unmodified lines
6395	6462	*
6396	6463	* WARNING: must be called with preemption disabled!
6397	6464	*/
6398		−static void __sched notrace __schedule(unsigned int sched_mode)
	6465	+static void __sched notrace __schedule(int sched_mode)
6399	6466	{
6400	6467	struct task_struct prev, next;
	6468	+ /*
	6469	+ * On PREEMPT_RT kernel, SM_RTLOCK_WAIT is noted
	6470	+ * as a preemption by schedule_debug() and RCU.
	6471	+ */
	6472	+ bool preempt = sched_mode > SM_NONE;
6401	6473	unsigned long *switch_count;
6402	6474	unsigned long prev_state;
6403	6475	struct rq_flags rf;
@@ -6408,13 +6480,13 @@ static void __sched notrace __schedule(unsigned int sched_mode) · 4 unmodified lines
6408	6480	rq = cpu_rq(cpu);
6409	6481	prev = rq->curr;
6410	6482
6411		− schedule_debug(prev, !!sched_mode);
	6483	+ schedule_debug(prev, preempt);
6412	6484
6413	6485	if (sched_feat(HRTICK) \|\| sched_feat(HRTICK_DL))
6414	6486	hrtick_clear(rq);
6415	6487
6416	6488	local_irq_disable();
6417		− rcu_note_context_switch(!!sched_mode);
	6489	+ rcu_note_context_switch(preempt);
6418	6490
6419	6491	/*
6420	6492	* Make sure that signal_pending_state()->signal_pending() below
@@ -6443,22 +6515,32 @@ static void __sched notrace __schedule(unsigned int sched_mode) · 22 unmodified lines
6443	6515
6444	6516	switch_count = &prev->nivcsw;
6445	6517
	6518	+ /* Task state changes only considers SM_PREEMPT as preemption */
	6519	+ preempt = sched_mode == SM_PREEMPT;
	6520	+
6446	6521	/*
6447	6522	* We must load prev->state once (task_struct::state is volatile), such
6448	6523	* that we form a control dependency vs deactivate_task() below.
6449	6524	*/
6450	6525	prev_state = READ_ONCE(prev->__state);
6451		− if (!(sched_mode & SM_MASK_PREEMPT) && prev_state) {
	6526	+ if (sched_mode == SM_IDLE) {
	6527	+ if (!rq->nr_running) {
	6528	+ next = prev;
	6529	+ goto picked;
	6530	+ }
	6531	+ } else if (!preempt && prev_state) {
6452	6532	if (signal_pending_state(prev_state, prev)) {
6453	6533	WRITE_ONCE(prev->__state, TASK_RUNNING);
6454	6534	} else {
	6535	+ int flags = DEQUEUE_NOCLOCK;
	6536	+
6455	6537	prev->sched_contributes_to_load =
6456	6538	(prev_state & TASK_UNINTERRUPTIBLE) &&
6457	6539	!(prev_state & TASK_NOLOAD) &&
6458	6540	!(prev_state & TASK_FROZEN);
6459	6541
6460		− if (prev->sched_contributes_to_load)
6461		− rq->nr_uninterruptible++;
	6542	+ if (unlikely(is_special_task_state(prev_state)))
	6543	+ flags \|= DEQUEUE_SPECIAL;
6462	6544
6463	6545	/*
6464	6546	* __schedule() ttwu()
@@ -6471,17 +6553,13 @@ static void __sched notrace __schedule(unsigned int sched_mode) · 6 unmodified lines
6471	6553	*
6472	6554	* After this, schedule() must not care about p->state any more.
6473	6555	*/
6474		− deactivate_task(rq, prev, DEQUEUE_SLEEP \| DEQUEUE_NOCLOCK);
6475		−
6476		− if (prev->in_iowait) {
6477		− atomic_inc(&rq->nr_iowait);
6478		− delayacct_blkio_start();
6479		− }
	6556	+ block_task(rq, prev, flags);
6480	6557	}
6481	6558	switch_count = &prev->nvcsw;
6482	6559	}
6483	6560
6484	6561	next = pick_next_task(rq, prev, &rf);
	6562	+picked:
6485	6563	clear_tsk_need_resched(prev);
6486	6564	clear_preempt_need_resched();
6487	6565	#ifdef CONFIG_SCHED_DEBUG
@@ -6523,7 +6601,7 @@ static void __sched notrace __schedule(unsigned int sched_mode) · 35 unmodified lines
6523	6601	psi_account_irqtime(rq, prev, next);
6524	6602	psi_sched_switch(prev, next, !task_on_rq_queued(prev));
6525	6603
6526		− trace_sched_switch(sched_mode & SM_MASK_PREEMPT, prev, next, prev_state);
	6604	+ trace_sched_switch(preempt, prev, next, prev_state);
6527	6605
6528	6606	/* Also unlocks the rq: */
6529	6607	rq = context_switch(rq, prev, next, &rf);
@@ -6599,7 +6677,7 @@ static void sched_update_worker(struct task_struct *tsk) · 69 unmodified lines
6599	6677	}
6600	6678	}
6601	6679
6602		−static __always_inline void __schedule_loop(unsigned int sched_mode)
	6680	+static __always_inline void __schedule_loop(int sched_mode)
6603	6681	{
6604	6682	do {
6605	6683	preempt_disable();
@@ -6644,7 +6722,7 @@ void __sched schedule_idle(void) · 38 unmodified lines
6644	6722	*/
6645	6723	WARN_ON_ONCE(current->__state);
6646	6724	do {
6647		− __schedule(SM_NONE);
	6725	+ __schedule(SM_IDLE);
6648	6726	} while (need_resched());
6649	6727	}
6650	6728
@@ -8228,8 +8306,6 @@ void __init sched_init(void) · 1577 unmodified lines
8228	8306	#endif /* CONFIG_RT_GROUP_SCHED */
8229	8307	}
8230	8308
8231		− init_rt_bandwidth(&def_rt_bandwidth, global_rt_period(), global_rt_runtime());
8232		−
8233	8309	#ifdef CONFIG_SMP
8234	8310	init_defrootdomain();
8235	8311	#endif
@@ -8284,8 +8360,13 @@ void __init sched_init(void) · 48 unmodified lines
8284	8360	init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL);
8285	8361	#endif /* CONFIG_FAIR_GROUP_SCHED */
8286	8362
8287		− rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime;
8288	8363	#ifdef CONFIG_RT_GROUP_SCHED
	8364	+ /*
	8365	+ * This is required for init cpu because rt.c:__enable_runtime()
	8366	+ * starts working after scheduler_running, which is not the case
	8367	+ * yet.
	8368	+ */
	8369	+ rq->rt.rt_runtime = global_rt_runtime();
8289	8370	init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL);
8290	8371	#endif
8291	8372	#ifdef CONFIG_SMP
@@ -8317,10 +8398,12 @@ void __init sched_init(void) · 25 unmodified lines
8317	8398	#endif /* CONFIG_SMP */
8318	8399	hrtick_rq_init(rq);
8319	8400	atomic_set(&rq->nr_iowait, 0);
	8401	+ fair_server_init(rq);
8320	8402
8321	8403	#ifdef CONFIG_SCHED_CORE
8322	8404	rq->core = rq;
8323	8405	rq->core_pick = NULL;
	8406	+ rq->core_dl_server = NULL;
8324	8407	rq->core_enabled = 0;
8325	8408	rq->core_tree = RB_ROOT;
8326	8409	rq->core_forceidle_count = 0;
@@ -8333,6 +8416,7 @@ void __init sched_init(void) · 6 unmodified lines
8333	8416	}
8334	8417
8335	8418	set_load_weight(&init_task, false);
	8419	+ init_task.se.slice = sysctl_sched_base_slice,
8336	8420
8337	8421	/*
8338	8422	* The boot idle thread does lazy MMU switching as well:
@@ -8548,7 +8632,7 @@ void normalize_rt_tasks(void) · 209 unmodified lines
8548	8632	schedstat_set(p->stats.sleep_start, 0);
8549	8633	schedstat_set(p->stats.block_start, 0);
8550	8634
8551		− if (!dl_task(p) && !rt_task(p)) {
	8635	+ if (!rt_or_dl_task(p)) {
8552	8636	/*
8553	8637	* Renice negative nice level userspace
8554	8638	* tasks back to 0:

kernel/sched/cpufreq_schedutil.c+3−3

@@ -654,9 +654,9 @@ static int sugov_kthread_create(struct sugov_policy *sg_policy) · 653 unmodified lines
654	654	* Fake (unused) bandwidth; workaround to "fix"
655	655	* priority inheritance.
656	656	*/
657		− .sched_runtime = 1000000,
658		− .sched_deadline = 10000000,
659		− .sched_period = 10000000,
	657	+ .sched_runtime = NSEC_PER_MSEC,
	658	+ .sched_deadline = 10 * NSEC_PER_MSEC,
	659	+ .sched_period = 10 * NSEC_PER_MSEC,
660	660	};
661	661	struct cpufreq_policy *policy = sg_policy->policy;
662	662	int ret;

kernel/sched/deadline.c+403−100

@@ -320,19 +320,12 @@ void sub_running_bw(struct sched_dl_entity dl_se, struct dl_rq dl_rq) · 319 unmodified lines
320	320	__sub_running_bw(dl_se->dl_bw, dl_rq);
321	321	}
322	322
323		−static void dl_change_utilization(struct task_struct *p, u64 new_bw)
	323	+static void dl_rq_change_utilization(struct rq rq, struct sched_dl_entity dl_se, u64 new_bw)
324	324	{
325		− struct rq *rq;
326		−
327		− WARN_ON_ONCE(p->dl.flags & SCHED_FLAG_SUGOV);
328		−
329		− if (task_on_rq_queued(p))
330		− return;
	325	+ if (dl_se->dl_non_contending) {
	326	+ sub_running_bw(dl_se, &rq->dl);
	327	+ dl_se->dl_non_contending = 0;
331	328
332		− rq = task_rq(p);
333		− if (p->dl.dl_non_contending) {
334		− sub_running_bw(&p->dl, &rq->dl);
335		− p->dl.dl_non_contending = 0;
336	329	/*
337	330	* If the timer handler is currently running and the
338	331	* timer cannot be canceled, inactive_task_timer()
@@ -340,13 +333,25 @@ static void dl_change_utilization(struct task_struct *p, u64 new_bw) · 1 unmodified line
340	333	* will not touch the rq's active utilization,
341	334	* so we are still safe.
342	335	*/
343		− if (hrtimer_try_to_cancel(&p->dl.inactive_timer) == 1)
344		− put_task_struct(p);
	336	+ if (hrtimer_try_to_cancel(&dl_se->inactive_timer) == 1) {
	337	+ if (!dl_server(dl_se))
	338	+ put_task_struct(dl_task_of(dl_se));
	339	+ }
345	340	}
346		− __sub_rq_bw(p->dl.dl_bw, &rq->dl);
	341	+ __sub_rq_bw(dl_se->dl_bw, &rq->dl);
347	342	__add_rq_bw(new_bw, &rq->dl);
348	343	}
349	344
	345	+static void dl_change_utilization(struct task_struct *p, u64 new_bw)
	346	+{
	347	+ WARN_ON_ONCE(p->dl.flags & SCHED_FLAG_SUGOV);
	348	+
	349	+ if (task_on_rq_queued(p))
	350	+ return;
	351	+
	352	+ dl_rq_change_utilization(task_rq(p), &p->dl, new_bw);
	353	+}
	354	+
350	355	static void __dl_clear_params(struct sched_dl_entity *dl_se);
351	356
352	357	/*
@@ -771,6 +776,15 @@ static inline void replenish_dl_new_period(struct sched_dl_entity *dl_se, · 418 unmodified lines
771	776	/* for non-boosted task, pi_of(dl_se) == dl_se */
772	777	dl_se->deadline = rq_clock(rq) + pi_of(dl_se)->dl_deadline;
773	778	dl_se->runtime = pi_of(dl_se)->dl_runtime;
	779	+
	780	+ /*
	781	+ * If it is a deferred reservation, and the server
	782	+ * is not handling an starvation case, defer it.
	783	+ */
	784	+ if (dl_se->dl_defer & !dl_se->dl_defer_running) {
	785	+ dl_se->dl_throttled = 1;
	786	+ dl_se->dl_defer_armed = 1;
	787	+ }
774	788	}
775	789
776	790	/*
@@ -809,6 +823,9 @@ static inline void setup_new_dl_entity(struct sched_dl_entity *dl_se) · 32 unmodified lines
809	823	replenish_dl_new_period(dl_se, rq);
810	824	}
811	825
	826	+static int start_dl_timer(struct sched_dl_entity *dl_se);
	827	+static bool dl_entity_overflow(struct sched_dl_entity *dl_se, u64 t);
	828	+
812	829	/*
813	830	* Pure Earliest Deadline First (EDF) scheduling does not deal with the
814	831	* possibility of a entity lasting more than what it declared, and thus
@@ -837,9 +854,18 @@ static void replenish_dl_entity(struct sched_dl_entity *dl_se) · 22 unmodified lines
837	854	/*
838	855	* This could be the case for a !-dl task that is boosted.
839	856	* Just go with full inherited parameters.
	857	+ *
	858	+ * Or, it could be the case of a deferred reservation that
	859	+ * was not able to consume its runtime in background and
	860	+ * reached this point with current u > U.
	861	+ *
	862	+ * In both cases, set a new period.
840	863	*/
841		− if (dl_se->dl_deadline == 0)
842		− replenish_dl_new_period(dl_se, rq);
	864	+ if (dl_se->dl_deadline == 0 \|\|
	865	+ (dl_se->dl_defer_armed && dl_entity_overflow(dl_se, rq_clock(rq)))) {
	866	+ dl_se->deadline = rq_clock(rq) + pi_of(dl_se)->dl_deadline;
	867	+ dl_se->runtime = pi_of(dl_se)->dl_runtime;
	868	+ }
843	869
844	870	if (dl_se->dl_yielded && dl_se->runtime > 0)
845	871	dl_se->runtime = 0;
@@ -873,6 +899,44 @@ static void replenish_dl_entity(struct sched_dl_entity *dl_se) · 27 unmodified lines
873	899	dl_se->dl_yielded = 0;
874	900	if (dl_se->dl_throttled)
875	901	dl_se->dl_throttled = 0;
	902	+
	903	+ /*
	904	+ * If this is the replenishment of a deferred reservation,
	905	+ * clear the flag and return.
	906	+ */
	907	+ if (dl_se->dl_defer_armed) {
	908	+ dl_se->dl_defer_armed = 0;
	909	+ return;
	910	+ }
	911	+
	912	+ /*
	913	+ * A this point, if the deferred server is not armed, and the deadline
	914	+ * is in the future, if it is not running already, throttle the server
	915	+ * and arm the defer timer.
	916	+ */
	917	+ if (dl_se->dl_defer && !dl_se->dl_defer_running &&
	918	+ dl_time_before(rq_clock(dl_se->rq), dl_se->deadline - dl_se->runtime)) {
	919	+ if (!is_dl_boosted(dl_se) && dl_se->server_has_tasks(dl_se)) {
	920	+
	921	+ /*
	922	+ * Set dl_se->dl_defer_armed and dl_throttled variables to
	923	+ * inform the start_dl_timer() that this is a deferred
	924	+ * activation.
	925	+ */
	926	+ dl_se->dl_defer_armed = 1;
	927	+ dl_se->dl_throttled = 1;
	928	+ if (!start_dl_timer(dl_se)) {
	929	+ /*
	930	+ * If for whatever reason (delays), a previous timer was
	931	+ * queued but not serviced, cancel it and clean the
	932	+ * deferrable server variables intended for start_dl_timer().
	933	+ */
	934	+ hrtimer_try_to_cancel(&dl_se->dl_timer);
	935	+ dl_se->dl_defer_armed = 0;
	936	+ dl_se->dl_throttled = 0;
	937	+ }
	938	+ }
	939	+ }
876	940	}
877	941
878	942	/*
@@ -1023,6 +1087,15 @@ static void update_dl_entity(struct sched_dl_entity *dl_se) · 144 unmodified lines
1023	1087	}
1024	1088
1025	1089	replenish_dl_new_period(dl_se, rq);
	1090	+ } else if (dl_server(dl_se) && dl_se->dl_defer) {
	1091	+ /*
	1092	+ * The server can still use its previous deadline, so check if
	1093	+ * it left the dl_defer_running state.
	1094	+ */
	1095	+ if (!dl_se->dl_defer_running) {
	1096	+ dl_se->dl_defer_armed = 1;
	1097	+ dl_se->dl_throttled = 1;
	1098	+ }
1026	1099	}
1027	1100	}
1028	1101
@@ -1055,8 +1128,21 @@ static int start_dl_timer(struct sched_dl_entity *dl_se) · 26 unmodified lines
1055	1128	* We want the timer to fire at the deadline, but considering
1056	1129	* that it is actually coming from rq->clock and not from
1057	1130	* hrtimer's time base reading.
	1131	+ *
	1132	+ * The deferred reservation will have its timer set to
	1133	+ * (deadline - runtime). At that point, the CBS rule will decide
	1134	+ * if the current deadline can be used, or if a replenishment is
	1135	+ * required to avoid add too much pressure on the system
	1136	+ * (current u > U).
1058	1137	*/
1059		− act = ns_to_ktime(dl_next_period(dl_se));
	1138	+ if (dl_se->dl_defer_armed) {
	1139	+ WARN_ON_ONCE(!dl_se->dl_throttled);
	1140	+ act = ns_to_ktime(dl_se->deadline - dl_se->runtime);
	1141	+ } else {
	1142	+ /* act = deadline - rel-deadline + period */
	1143	+ act = ns_to_ktime(dl_next_period(dl_se));
	1144	+ }
	1145	+
1060	1146	now = hrtimer_cb_get_time(timer);
1061	1147	delta = ktime_to_ns(now) - rq_clock(rq);
1062	1148	act = ktime_add_ns(act, delta);
@@ -1106,6 +1192,62 @@ static void __push_dl_task(struct rq rq, struct rq_flags rf) · 43 unmodified lines
1106	1192	#endif
1107	1193	}
1108	1194
	1195	+/* a defer timer will not be reset if the runtime consumed was < dl_server_min_res */
	1196	+static const u64 dl_server_min_res = 1 * NSEC_PER_MSEC;
	1197	+
	1198	+static enum hrtimer_restart dl_server_timer(struct hrtimer timer, struct sched_dl_entity dl_se)
	1199	+{
	1200	+ struct rq *rq = rq_of_dl_se(dl_se);
	1201	+ u64 fw;
	1202	+
	1203	+ scoped_guard (rq_lock, rq) {
	1204	+ struct rq_flags *rf = &scope.rf;
	1205	+
	1206	+ if (!dl_se->dl_throttled \|\| !dl_se->dl_runtime)
	1207	+ return HRTIMER_NORESTART;
	1208	+
	1209	+ sched_clock_tick();
	1210	+ update_rq_clock(rq);
	1211	+
	1212	+ if (!dl_se->dl_runtime)
	1213	+ return HRTIMER_NORESTART;
	1214	+
	1215	+ if (!dl_se->server_has_tasks(dl_se)) {
	1216	+ replenish_dl_entity(dl_se);
	1217	+ return HRTIMER_NORESTART;
	1218	+ }
	1219	+
	1220	+ if (dl_se->dl_defer_armed) {
	1221	+ /*
	1222	+ * First check if the server could consume runtime in background.
	1223	+ * If so, it is possible to push the defer timer for this amount
	1224	+ * of time. The dl_server_min_res serves as a limit to avoid
	1225	+ * forwarding the timer for a too small amount of time.
	1226	+ */
	1227	+ if (dl_time_before(rq_clock(dl_se->rq),
	1228	+ (dl_se->deadline - dl_se->runtime - dl_server_min_res))) {
	1229	+
	1230	+ /* reset the defer timer */
	1231	+ fw = dl_se->deadline - rq_clock(dl_se->rq) - dl_se->runtime;
	1232	+
	1233	+ hrtimer_forward_now(timer, ns_to_ktime(fw));
	1234	+ return HRTIMER_RESTART;
	1235	+ }
	1236	+
	1237	+ dl_se->dl_defer_running = 1;
	1238	+ }
	1239	+
	1240	+ enqueue_dl_entity(dl_se, ENQUEUE_REPLENISH);
	1241	+
	1242	+ if (!dl_task(dl_se->rq->curr) \|\| dl_entity_preempt(dl_se, &dl_se->rq->curr->dl))
	1243	+ resched_curr(rq);
	1244	+
	1245	+ __push_dl_task(rq, rf);
	1246	+ }
	1247	+
	1248	+ return HRTIMER_NORESTART;
	1249	+}
	1250	+
1109	1251	/*
1110	1252	* This is the bandwidth enforcement timer callback. If here, we know
1111	1253	* a task is not on its dl_rq, since the fact that the timer was running
@@ -1128,28 +1270,8 @@ static enum hrtimer_restart dl_task_timer(struct hrtimer *timer) · 16 unmodified lines
1128	1270	struct rq_flags rf;
1129	1271	struct rq *rq;
1130	1272
1131		− if (dl_server(dl_se)) {
1132		− struct rq *rq = rq_of_dl_se(dl_se);
1133		− struct rq_flags rf;
1134		−
1135		− rq_lock(rq, &rf);
1136		− if (dl_se->dl_throttled) {
1137		− sched_clock_tick();
1138		− update_rq_clock(rq);
1139		−
1140		− if (dl_se->server_has_tasks(dl_se)) {
1141		− enqueue_dl_entity(dl_se, ENQUEUE_REPLENISH);
1142		− resched_curr(rq);
1143		− __push_dl_task(rq, &rf);
1144		− } else {
1145		− replenish_dl_entity(dl_se);
1146		− }
1147		−
1148		− }
1149		− rq_unlock(rq, &rf);
1150		−
1151		− return HRTIMER_NORESTART;
1152		− }
	1273	+ if (dl_server(dl_se))
	1274	+ return dl_server_timer(timer, dl_se);
1153	1275
1154	1276	p = dl_task_of(dl_se);
1155	1277	rq = task_rq_lock(p, &rf);
@@ -1319,22 +1441,10 @@ static u64 grub_reclaim(u64 delta, struct rq rq, struct sched_dl_entity dl_se) · 163 unmodified lines
1319	1441	return (delta * u_act) >> BW_SHIFT;
1320	1442	}
1321	1443
1322		−static inline void
1323		−update_stats_dequeue_dl(struct dl_rq dl_rq, struct sched_dl_entity dl_se,
1324		− int flags);
1325		−static void update_curr_dl_se(struct rq rq, struct sched_dl_entity dl_se, s64 delta_exec)
	1444	+s64 dl_scaled_delta_exec(struct rq rq, struct sched_dl_entity dl_se, s64 delta_exec)
1326	1445	{
1327	1446	s64 scaled_delta_exec;
1328	1447
1329		− if (unlikely(delta_exec <= 0)) {
1330		− if (unlikely(dl_se->dl_yielded))
1331		− goto throttle;
1332		− return;
1333		− }
1334		−
1335		− if (dl_entity_is_special(dl_se))
1336		− return;
1337		−
1338	1448	/*
1339	1449	* For tasks that participate in GRUB, we implement GRUB-PA: the
1340	1450	* spare reclaimed bandwidth is used to clock down frequency.
@@ -1353,8 +1463,64 @@ static void update_curr_dl_se(struct rq rq, struct sched_dl_entity dl_se, s64 · 12 unmodified lines
1353	1463	scaled_delta_exec = cap_scale(scaled_delta_exec, scale_cpu);
1354	1464	}
1355	1465
	1466	+ return scaled_delta_exec;
	1467	+}
	1468	+
	1469	+static inline void
	1470	+update_stats_dequeue_dl(struct dl_rq dl_rq, struct sched_dl_entity dl_se,
	1471	+ int flags);
	1472	+static void update_curr_dl_se(struct rq rq, struct sched_dl_entity dl_se, s64 delta_exec)
	1473	+{
	1474	+ s64 scaled_delta_exec;
	1475	+
	1476	+ if (unlikely(delta_exec <= 0)) {
	1477	+ if (unlikely(dl_se->dl_yielded))
	1478	+ goto throttle;
	1479	+ return;
	1480	+ }
	1481	+
	1482	+ if (dl_server(dl_se) && dl_se->dl_throttled && !dl_se->dl_defer)
	1483	+ return;
	1484	+
	1485	+ if (dl_entity_is_special(dl_se))
	1486	+ return;
	1487	+
	1488	+ scaled_delta_exec = dl_scaled_delta_exec(rq, dl_se, delta_exec);
	1489	+
1356	1490	dl_se->runtime -= scaled_delta_exec;
1357	1491
	1492	+ /*
	1493	+ * The fair server can consume its runtime while throttled (not queued/
	1494	+ * running as regular CFS).
	1495	+ *
	1496	+ * If the server consumes its entire runtime in this state. The server
	1497	+ * is not required for the current period. Thus, reset the server by
	1498	+ * starting a new period, pushing the activation.
	1499	+ */
	1500	+ if (dl_se->dl_defer && dl_se->dl_throttled && dl_runtime_exceeded(dl_se)) {
	1501	+ /*
	1502	+ * If the server was previously activated - the starving condition
	1503	+ * took place, it this point it went away because the fair scheduler
	1504	+ * was able to get runtime in background. So return to the initial
	1505	+ * state.
	1506	+ */
	1507	+ dl_se->dl_defer_running = 0;
	1508	+
	1509	+ hrtimer_try_to_cancel(&dl_se->dl_timer);
	1510	+
	1511	+ replenish_dl_new_period(dl_se, dl_se->rq);
	1512	+
	1513	+ /*
	1514	+ * Not being able to start the timer seems problematic. If it could not
	1515	+ * be started for whatever reason, we need to "unthrottle" the DL server
	1516	+ * and queue right away. Otherwise nothing might queue it. That's similar
	1517	+ * to what enqueue_dl_entity() does on start_dl_timer==0. For now, just warn.
	1518	+ */
	1519	+ WARN_ON_ONCE(!start_dl_timer(dl_se));
	1520	+
	1521	+ return;
	1522	+ }
	1523	+
1358	1524	throttle:
1359	1525	if (dl_runtime_exceeded(dl_se) \|\| dl_se->dl_yielded) {
1360	1526	dl_se->dl_throttled = 1;
@@ -1381,6 +1547,14 @@ throttle: · 20 unmodified lines
1381	1547	resched_curr(rq);
1382	1548	}
1383	1549
	1550	+ /*
	1551	+ * The fair server (sole dl_server) does not account for real-time
	1552	+ * workload because it is running fair work.
	1553	+ */
	1554	+ if (dl_se == &rq->fair_server)
	1555	+ return;
	1556	+
	1557	+#ifdef CONFIG_RT_GROUP_SCHED
1384	1558	/*
1385	1559	* Because -- for now -- we share the rt bandwidth, we need to
1386	1560	* account our runtime there too, otherwise actual rt tasks
@@ -1405,34 +1579,155 @@ throttle: · 18 unmodified lines
1405	1579	rt_rq->rt_time += delta_exec;
1406	1580	raw_spin_unlock(&rt_rq->rt_runtime_lock);
1407	1581	}
	1582	+#endif
	1583	+}
	1584	+
	1585	+/*
	1586	+ * In the non-defer mode, the idle time is not accounted, as the
	1587	+ * server provides a guarantee.
	1588	+ *
	1589	+ * If the dl_server is in defer mode, the idle time is also considered
	1590	+ * as time available for the fair server, avoiding a penalty for the
	1591	+ * rt scheduler that did not consumed that time.
	1592	+ */
	1593	+void dl_server_update_idle_time(struct rq rq, struct task_struct p)
	1594	+{
	1595	+ s64 delta_exec, scaled_delta_exec;
	1596	+
	1597	+ if (!rq->fair_server.dl_defer)
	1598	+ return;
	1599	+
	1600	+ /* no need to discount more */
	1601	+ if (rq->fair_server.runtime < 0)
	1602	+ return;
	1603	+
	1604	+ delta_exec = rq_clock_task(rq) - p->se.exec_start;
	1605	+ if (delta_exec < 0)
	1606	+ return;
	1607	+
	1608	+ scaled_delta_exec = dl_scaled_delta_exec(rq, &rq->fair_server, delta_exec);
	1609	+
	1610	+ rq->fair_server.runtime -= scaled_delta_exec;
	1611	+
	1612	+ if (rq->fair_server.runtime < 0) {
	1613	+ rq->fair_server.dl_defer_running = 0;
	1614	+ rq->fair_server.runtime = 0;
	1615	+ }
	1616	+
	1617	+ p->se.exec_start = rq_clock_task(rq);
1408	1618	}
1409	1619
1410	1620	void dl_server_update(struct sched_dl_entity *dl_se, s64 delta_exec)
1411	1621	{
1412		− update_curr_dl_se(dl_se->rq, dl_se, delta_exec);
	1622	+ /* 0 runtime = fair server disabled */
	1623	+ if (dl_se->dl_runtime)
	1624	+ update_curr_dl_se(dl_se->rq, dl_se, delta_exec);
1413	1625	}
1414	1626
1415	1627	void dl_server_start(struct sched_dl_entity *dl_se)
1416	1628	{
	1629	+ struct rq *rq = dl_se->rq;
	1630	+
	1631	+ /*
	1632	+ * XXX: the apply do not work fine at the init phase for the
	1633	+ * fair server because things are not yet set. We need to improve
	1634	+ * this before getting generic.
	1635	+ */
1417	1636	if (!dl_server(dl_se)) {
	1637	+ u64 runtime = 50 * NSEC_PER_MSEC;
	1638	+ u64 period = 1000 * NSEC_PER_MSEC;
	1639	+
	1640	+ dl_server_apply_params(dl_se, runtime, period, 1);
	1641	+
1418	1642	dl_se->dl_server = 1;
	1643	+ dl_se->dl_defer = 1;
1419	1644	setup_new_dl_entity(dl_se);
1420	1645	}
	1646	+
	1647	+ if (!dl_se->dl_runtime)
	1648	+ return;
	1649	+
1421	1650	enqueue_dl_entity(dl_se, ENQUEUE_WAKEUP);
	1651	+ if (!dl_task(dl_se->rq->curr) \|\| dl_entity_preempt(dl_se, &rq->curr->dl))
	1652	+ resched_curr(dl_se->rq);
1422	1653	}
1423	1654
1424	1655	void dl_server_stop(struct sched_dl_entity *dl_se)
1425	1656	{
	1657	+ if (!dl_se->dl_runtime)
	1658	+ return;
	1659	+
1426	1660	dequeue_dl_entity(dl_se, DEQUEUE_SLEEP);
	1661	+ hrtimer_try_to_cancel(&dl_se->dl_timer);
	1662	+ dl_se->dl_defer_armed = 0;
	1663	+ dl_se->dl_throttled = 0;
1427	1664	}
1428	1665
1429	1666	void dl_server_init(struct sched_dl_entity dl_se, struct rq rq,
1430	1667	dl_server_has_tasks_f has_tasks,
1431		− dl_server_pick_f pick)
	1668	+ dl_server_pick_f pick_task)
1432	1669	{
1433	1670	dl_se->rq = rq;
1434	1671	dl_se->server_has_tasks = has_tasks;
1435		− dl_se->server_pick = pick;
	1672	+ dl_se->server_pick_task = pick_task;
	1673	+}
	1674	+
	1675	+void __dl_server_attach_root(struct sched_dl_entity dl_se, struct rq rq)
	1676	+{
	1677	+ u64 new_bw = dl_se->dl_bw;
	1678	+ int cpu = cpu_of(rq);
	1679	+ struct dl_bw *dl_b;
	1680	+
	1681	+ dl_b = dl_bw_of(cpu_of(rq));
	1682	+ guard(raw_spinlock)(&dl_b->lock);
	1683	+
	1684	+ if (!dl_bw_cpus(cpu))
	1685	+ return;
	1686	+
	1687	+ __dl_add(dl_b, new_bw, dl_bw_cpus(cpu));
	1688	+}
	1689	+
	1690	+int dl_server_apply_params(struct sched_dl_entity *dl_se, u64 runtime, u64 period, bool init)
	1691	+{
	1692	+ u64 old_bw = init ? 0 : to_ratio(dl_se->dl_period, dl_se->dl_runtime);
	1693	+ u64 new_bw = to_ratio(period, runtime);
	1694	+ struct rq *rq = dl_se->rq;
	1695	+ int cpu = cpu_of(rq);
	1696	+ struct dl_bw *dl_b;
	1697	+ unsigned long cap;
	1698	+ int retval = 0;
	1699	+ int cpus;
	1700	+
	1701	+ dl_b = dl_bw_of(cpu);
	1702	+ guard(raw_spinlock)(&dl_b->lock);
	1703	+
	1704	+ cpus = dl_bw_cpus(cpu);
	1705	+ cap = dl_bw_capacity(cpu);
	1706	+
	1707	+ if (__dl_overflow(dl_b, cap, old_bw, new_bw))
	1708	+ return -EBUSY;
	1709	+
	1710	+ if (init) {
	1711	+ __add_rq_bw(new_bw, &rq->dl);
	1712	+ __dl_add(dl_b, new_bw, cpus);
	1713	+ } else {
	1714	+ __dl_sub(dl_b, dl_se->dl_bw, cpus);
	1715	+ __dl_add(dl_b, new_bw, cpus);
	1716	+
	1717	+ dl_rq_change_utilization(rq, dl_se, new_bw);
	1718	+ }
	1719	+
	1720	+ dl_se->dl_runtime = runtime;
	1721	+ dl_se->dl_deadline = period;
	1722	+ dl_se->dl_period = period;
	1723	+
	1724	+ dl_se->runtime = 0;
	1725	+ dl_se->deadline = 0;
	1726	+
	1727	+ dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime);
	1728	+ dl_se->dl_density = to_ratio(dl_se->dl_deadline, dl_se->dl_runtime);
	1729	+
	1730	+ return retval;
1436	1731	}
1437	1732
1438	1733	/*
@@ -1599,46 +1894,40 @@ static inline bool __dl_less(struct rb_node a, const struct rb_node b) · 160 unmodified lines
1599	1894	return dl_time_before(__node_2_dle(a)->deadline, __node_2_dle(b)->deadline);
1600	1895	}
1601	1896
1602		−static inline struct sched_statistics *
	1897	+static __always_inline struct sched_statistics *
1603	1898	__schedstats_from_dl_se(struct sched_dl_entity *dl_se)
1604	1899	{
	1900	+ if (!schedstat_enabled())
	1901	+ return NULL;
	1902	+
	1903	+ if (dl_server(dl_se))
	1904	+ return NULL;
	1905	+
1605	1906	return &dl_task_of(dl_se)->stats;
1606	1907	}
1607	1908
1608	1909	static inline void
1609	1910	update_stats_wait_start_dl(struct dl_rq dl_rq, struct sched_dl_entity dl_se)
1610	1911	{
1611		− struct sched_statistics *stats;
1612		−
1613		− if (!schedstat_enabled())
1614		− return;
1615		−
1616		− stats = __schedstats_from_dl_se(dl_se);
1617		− __update_stats_wait_start(rq_of_dl_rq(dl_rq), dl_task_of(dl_se), stats);
	1912	+ struct sched_statistics *stats = __schedstats_from_dl_se(dl_se);
	1913	+ if (stats)
	1914	+ __update_stats_wait_start(rq_of_dl_rq(dl_rq), dl_task_of(dl_se), stats);
1618	1915	}
1619	1916
1620	1917	static inline void
1621	1918	update_stats_wait_end_dl(struct dl_rq dl_rq, struct sched_dl_entity dl_se)
1622	1919	{
1623		− struct sched_statistics *stats;
1624		−
1625		− if (!schedstat_enabled())
1626		− return;
1627		−
1628		− stats = __schedstats_from_dl_se(dl_se);
1629		− __update_stats_wait_end(rq_of_dl_rq(dl_rq), dl_task_of(dl_se), stats);
	1920	+ struct sched_statistics *stats = __schedstats_from_dl_se(dl_se);
	1921	+ if (stats)
	1922	+ __update_stats_wait_end(rq_of_dl_rq(dl_rq), dl_task_of(dl_se), stats);
1630	1923	}
1631	1924
1632	1925	static inline void
1633	1926	update_stats_enqueue_sleeper_dl(struct dl_rq dl_rq, struct sched_dl_entity dl_se)
1634	1927	{
1635		− struct sched_statistics *stats;
1636		−
1637		− if (!schedstat_enabled())
1638		− return;
1639		−
1640		− stats = __schedstats_from_dl_se(dl_se);
1641		− __update_stats_enqueue_sleeper(rq_of_dl_rq(dl_rq), dl_task_of(dl_se), stats);
	1928	+ struct sched_statistics *stats = __schedstats_from_dl_se(dl_se);
	1929	+ if (stats)
	1930	+ __update_stats_enqueue_sleeper(rq_of_dl_rq(dl_rq), dl_task_of(dl_se), stats);
1642	1931	}
1643	1932
1644	1933	static inline void
@@ -1735,7 +2024,7 @@ enqueue_dl_entity(struct sched_dl_entity *dl_se, int flags) · 90 unmodified lines
1735	2024	* be counted in the active utilization; hence, we need to call
1736	2025	* add_running_bw().
1737	2026	*/
1738		− if (dl_se->dl_throttled && !(flags & ENQUEUE_REPLENISH)) {
	2027	+ if (!dl_se->dl_defer && dl_se->dl_throttled && !(flags & ENQUEUE_REPLENISH)) {
1739	2028	if (flags & ENQUEUE_WAKEUP)
1740	2029	task_contending(dl_se, flags);
1741	2030
@@ -1757,6 +2046,25 @@ enqueue_dl_entity(struct sched_dl_entity *dl_se, int flags) · 15 unmodified lines
1757	2046	setup_new_dl_entity(dl_se);
1758	2047	}
1759	2048
	2049	+ /*
	2050	+ * If the reservation is still throttled, e.g., it got replenished but is a
	2051	+ * deferred task and still got to wait, don't enqueue.
	2052	+ */
	2053	+ if (dl_se->dl_throttled && start_dl_timer(dl_se))
	2054	+ return;
	2055	+
	2056	+ /*
	2057	+ * We're about to enqueue, make sure we're not ->dl_throttled!
	2058	+ * In case the timer was not started, say because the defer time
	2059	+ * has passed, mark as not throttled and mark unarmed.
	2060	+ * Also cancel earlier timers, since letting those run is pointless.
	2061	+ */
	2062	+ if (dl_se->dl_throttled) {
	2063	+ hrtimer_try_to_cancel(&dl_se->dl_timer);
	2064	+ dl_se->dl_defer_armed = 0;
	2065	+ dl_se->dl_throttled = 0;
	2066	+ }
	2067	+
1760	2068	__enqueue_dl_entity(dl_se);
1761	2069	}
1762	2070
@@ -1846,7 +2154,7 @@ static void enqueue_task_dl(struct rq rq, struct task_struct p, int flags) · 83 unmodified lines
1846	2154	enqueue_pushable_dl_task(rq, p);
1847	2155	}
1848	2156
1849		−static void dequeue_task_dl(struct rq rq, struct task_struct p, int flags)
	2157	+static bool dequeue_task_dl(struct rq rq, struct task_struct p, int flags)
1850	2158	{
1851	2159	update_curr_dl(rq);
1852	2160
@@ -1856,6 +2164,8 @@ static void dequeue_task_dl(struct rq rq, struct task_struct p, int flags) · 3 unmodified lines
1856	2164	dequeue_dl_entity(&p->dl, flags);
1857	2165	if (!p->dl.dl_throttled && !dl_server(&p->dl))
1858	2166	dequeue_pushable_dl_task(rq, p);
	2167	+
	2168	+ return true;
1859	2169	}
1860	2170
1861	2171	/*
@@ -2074,6 +2384,9 @@ static void set_next_task_dl(struct rq rq, struct task_struct p, bool first) · 212 unmodified lines
2074	2384	update_dl_rq_load_avg(rq_clock_pelt(rq), rq, 0);
2075	2385
2076	2386	deadline_queue_push_tasks(rq);
	2387	+
	2388	+ if (hrtick_enabled(rq))
	2389	+ start_hrtick_dl(rq, &p->dl);
2077	2390	}
2078	2391
2079	2392	static struct sched_dl_entity pick_next_dl_entity(struct dl_rq dl_rq)
@@ -2086,7 +2399,11 @@ static struct sched_dl_entity pick_next_dl_entity(struct dl_rq dl_rq) · 6 unmodified lines
2086	2399	return __node_2_dle(left);
2087	2400	}
2088	2401
2089		−static struct task_struct pick_task_dl(struct rq rq)
	2402	+/*
	2403	+ * __pick_next_task_dl - Helper to pick the next -deadline task to run.
	2404	+ * @rq: The runqueue to pick the next task from.
	2405	+ */
	2406	+static struct task_struct __pick_task_dl(struct rq rq)
2090	2407	{
2091	2408	struct sched_dl_entity *dl_se;
2092	2409	struct dl_rq *dl_rq = &rq->dl;
@@ -2100,14 +2417,13 @@ again: · 7 unmodified lines
2100	2417	WARN_ON_ONCE(!dl_se);
2101	2418
2102	2419	if (dl_server(dl_se)) {
2103		− p = dl_se->server_pick(dl_se);
	2420	+ p = dl_se->server_pick_task(dl_se);
2104	2421	if (!p) {
2105		− WARN_ON_ONCE(1);
2106	2422	dl_se->dl_yielded = 1;
2107	2423	update_curr_dl_se(rq, dl_se, 0);
2108	2424	goto again;
2109	2425	}
2110		− p->dl_server = dl_se;
	2426	+ rq->dl_server = dl_se;
2111	2427	} else {
2112	2428	p = dl_task_of(dl_se);
2113	2429	}
@@ -2115,24 +2431,12 @@ again: · 1 unmodified line
2115	2431	return p;
2116	2432	}
2117	2433
2118		−static struct task_struct pick_next_task_dl(struct rq rq)
	2434	+static struct task_struct pick_task_dl(struct rq rq)
2119	2435	{
2120		− struct task_struct *p;
2121		−
2122		− p = pick_task_dl(rq);
2123		− if (!p)
2124		− return p;
2125		−
2126		− if (!p->dl_server)
2127		− set_next_task_dl(rq, p, true);
2128		−
2129		− if (hrtick_enabled(rq))
2130		− start_hrtick_dl(rq, &p->dl);
2131		−
2132		− return p;
	2436	+ return __pick_task_dl(rq);
2133	2437	}
2134	2438
2135		−static void put_prev_task_dl(struct rq rq, struct task_struct p)
	2439	+static void put_prev_task_dl(struct rq rq, struct task_struct p, struct task_struct *next)
2136	2440	{
2137	2441	struct sched_dl_entity *dl_se = &p->dl;
2138	2442	struct dl_rq *dl_rq = &rq->dl;
@@ -2824,13 +3128,12 @@ DEFINE_SCHED_CLASS(dl) = { · 685 unmodified lines
2824	3128
2825	3129	.wakeup_preempt = wakeup_preempt_dl,
2826	3130
2827		− .pick_next_task = pick_next_task_dl,
	3131	+ .pick_task = pick_task_dl,
2828	3132	.put_prev_task = put_prev_task_dl,
2829	3133	.set_next_task = set_next_task_dl,
2830	3134
2831	3135	#ifdef CONFIG_SMP
2832	3136	.balance = balance_dl,
2833		− .pick_task = pick_task_dl,
2834	3137	.select_task_rq = select_task_rq_dl,
2835	3138	.migrate_task_rq = migrate_task_rq_dl,
2836	3139	.set_cpus_allowed = set_cpus_allowed_dl,

kernel/sched/debug.c+186−12

@@ -333,8 +333,165 @@ static const struct file_operations sched_debug_fops = { · 332 unmodified lines
333	333	.release = seq_release,
334	334	};
335	335
	336	+enum dl_param {
	337	+ DL_RUNTIME = 0,
	338	+ DL_PERIOD,
	339	+};
	340	+
	341	+static unsigned long fair_server_period_max = (1UL << 22) * NSEC_PER_USEC; /* ~4 seconds */
	342	+static unsigned long fair_server_period_min = (100) * NSEC_PER_USEC; /* 100 us */
	343	+
	344	+static ssize_t sched_fair_server_write(struct file filp, const char __user ubuf,
	345	+ size_t cnt, loff_t *ppos, enum dl_param param)
	346	+{
	347	+ long cpu = (long) ((struct seq_file *) filp->private_data)->private;
	348	+ struct rq *rq = cpu_rq(cpu);
	349	+ u64 runtime, period;
	350	+ size_t err;
	351	+ int retval;
	352	+ u64 value;
	353	+
	354	+ err = kstrtoull_from_user(ubuf, cnt, 10, &value);
	355	+ if (err)
	356	+ return err;
	357	+
	358	+ scoped_guard (rq_lock_irqsave, rq) {
	359	+ runtime = rq->fair_server.dl_runtime;
	360	+ period = rq->fair_server.dl_period;
	361	+
	362	+ switch (param) {
	363	+ case DL_RUNTIME:
	364	+ if (runtime == value)
	365	+ break;
	366	+ runtime = value;
	367	+ break;
	368	+ case DL_PERIOD:
	369	+ if (value == period)
	370	+ break;
	371	+ period = value;
	372	+ break;
	373	+ }
	374	+
	375	+ if (runtime > period \|\|
	376	+ period > fair_server_period_max \|\|
	377	+ period < fair_server_period_min) {
	378	+ return -EINVAL;
	379	+ }
	380	+
	381	+ if (rq->cfs.h_nr_running) {
	382	+ update_rq_clock(rq);
	383	+ dl_server_stop(&rq->fair_server);
	384	+ }
	385	+
	386	+ retval = dl_server_apply_params(&rq->fair_server, runtime, period, 0);
	387	+ if (retval)
	388	+ cnt = retval;
	389	+
	390	+ if (!runtime)
	391	+ printk_deferred("Fair server disabled in CPU %d, system may crash due to starvation.\n",
	392	+ cpu_of(rq));
	393	+
	394	+ if (rq->cfs.h_nr_running)
	395	+ dl_server_start(&rq->fair_server);
	396	+ }
	397	+
	398	+ *ppos += cnt;
	399	+ return cnt;
	400	+}
	401	+
	402	+static size_t sched_fair_server_show(struct seq_file m, void v, enum dl_param param)
	403	+{
	404	+ unsigned long cpu = (unsigned long) m->private;
	405	+ struct rq *rq = cpu_rq(cpu);
	406	+ u64 value;
	407	+
	408	+ switch (param) {
	409	+ case DL_RUNTIME:
	410	+ value = rq->fair_server.dl_runtime;
	411	+ break;
	412	+ case DL_PERIOD:
	413	+ value = rq->fair_server.dl_period;
	414	+ break;
	415	+ }
	416	+
	417	+ seq_printf(m, "%llu\n", value);
	418	+ return 0;
	419	+
	420	+}
	421	+
	422	+static ssize_t
	423	+sched_fair_server_runtime_write(struct file filp, const char __user ubuf,
	424	+ size_t cnt, loff_t *ppos)
	425	+{
	426	+ return sched_fair_server_write(filp, ubuf, cnt, ppos, DL_RUNTIME);
	427	+}
	428	+
	429	+static int sched_fair_server_runtime_show(struct seq_file m, void v)
	430	+{
	431	+ return sched_fair_server_show(m, v, DL_RUNTIME);
	432	+}
	433	+
	434	+static int sched_fair_server_runtime_open(struct inode inode, struct file filp)
	435	+{
	436	+ return single_open(filp, sched_fair_server_runtime_show, inode->i_private);
	437	+}
	438	+
	439	+static const struct file_operations fair_server_runtime_fops = {
	440	+ .open = sched_fair_server_runtime_open,
	441	+ .write = sched_fair_server_runtime_write,
	442	+ .read = seq_read,
	443	+ .llseek = seq_lseek,
	444	+ .release = single_release,
	445	+};
	446	+
	447	+static ssize_t
	448	+sched_fair_server_period_write(struct file filp, const char __user ubuf,
	449	+ size_t cnt, loff_t *ppos)
	450	+{
	451	+ return sched_fair_server_write(filp, ubuf, cnt, ppos, DL_PERIOD);
	452	+}
	453	+
	454	+static int sched_fair_server_period_show(struct seq_file m, void v)
	455	+{
	456	+ return sched_fair_server_show(m, v, DL_PERIOD);
	457	+}
	458	+
	459	+static int sched_fair_server_period_open(struct inode inode, struct file filp)
	460	+{
	461	+ return single_open(filp, sched_fair_server_period_show, inode->i_private);
	462	+}
	463	+
	464	+static const struct file_operations fair_server_period_fops = {
	465	+ .open = sched_fair_server_period_open,
	466	+ .write = sched_fair_server_period_write,
	467	+ .read = seq_read,
	468	+ .llseek = seq_lseek,
	469	+ .release = single_release,
	470	+};
	471	+
336	472	static struct dentry *debugfs_sched;
337	473
	474	+static void debugfs_fair_server_init(void)
	475	+{
	476	+ struct dentry *d_fair;
	477	+ unsigned long cpu;
	478	+
	479	+ d_fair = debugfs_create_dir("fair_server", debugfs_sched);
	480	+ if (!d_fair)
	481	+ return;
	482	+
	483	+ for_each_possible_cpu(cpu) {
	484	+ struct dentry *d_cpu;
	485	+ char buf[32];
	486	+
	487	+ snprintf(buf, sizeof(buf), "cpu%lu", cpu);
	488	+ d_cpu = debugfs_create_dir(buf, d_fair);
	489	+
	490	+ debugfs_create_file("runtime", 0644, d_cpu, (void *) cpu, &fair_server_runtime_fops);
	491	+ debugfs_create_file("period", 0644, d_cpu, (void *) cpu, &fair_server_period_fops);
	492	+ }
	493	+}
	494	+
338	495	static __init int sched_init_debug(void)
339	496	{
340	497	struct dentry __maybe_unused *numa;
@@ -374,6 +531,8 @@ static __init int sched_init_debug(void) · 33 unmodified lines
374	531
375	532	debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops);
376	533
	534	+ debugfs_fair_server_init();
	535	+
377	536	return 0;
378	537	}
379	538	late_initcall(sched_init_debug);
@@ -580,27 +739,27 @@ print_task(struct seq_file m, struct rq rq, struct task_struct *p) · 200 unmodified lines
580	739	else
581	740	SEQ_printf(m, " %c", task_state_to_char(p));
582	741
583		− SEQ_printf(m, "%15s %5d %9Ld.%06ld %c %9Ld.%06ld %9Ld.%06ld %9Ld.%06ld %9Ld %5d ",
	742	+ SEQ_printf(m, " %15s %5d %9Ld.%06ld %c %9Ld.%06ld %c %9Ld.%06ld %9Ld.%06ld %9Ld %5d ",
584	743	p->comm, task_pid_nr(p),
585	744	SPLIT_NS(p->se.vruntime),
586	745	entity_eligible(cfs_rq_of(&p->se), &p->se) ? 'E' : 'N',
587	746	SPLIT_NS(p->se.deadline),
	747	+ p->se.custom_slice ? 'S' : ' ',
588	748	SPLIT_NS(p->se.slice),
589	749	SPLIT_NS(p->se.sum_exec_runtime),
590	750	(long long)(p->nvcsw + p->nivcsw),
591	751	p->prio);
592	752
593		− SEQ_printf(m, "%9lld.%06ld %9lld.%06ld %9lld.%06ld %9lld.%06ld",
	753	+ SEQ_printf(m, "%9lld.%06ld %9lld.%06ld %9lld.%06ld",
594	754	SPLIT_NS(schedstat_val_or_zero(p->stats.wait_sum)),
595		− SPLIT_NS(p->se.sum_exec_runtime),
596	755	SPLIT_NS(schedstat_val_or_zero(p->stats.sum_sleep_runtime)),
597	756	SPLIT_NS(schedstat_val_or_zero(p->stats.sum_block_runtime)));
598	757
599	758	#ifdef CONFIG_NUMA_BALANCING
600		− SEQ_printf(m, " %d %d", task_node(p), task_numa_group_id(p));
	759	+ SEQ_printf(m, " %d %d", task_node(p), task_numa_group_id(p));
601	760	#endif
602	761	#ifdef CONFIG_CGROUP_SCHED
603		− SEQ_printf_task_group_path(m, task_group(p), " %s")
	762	+ SEQ_printf_task_group_path(m, task_group(p), " %s")
604	763	#endif
605	764
606	765	SEQ_printf(m, "\n");
@@ -612,10 +771,26 @@ static void print_rq(struct seq_file m, struct rq rq, int rq_cpu) · 5 unmodified lines
612	771
613	772	SEQ_printf(m, "\n");
614	773	SEQ_printf(m, "runnable tasks:\n");
615		− SEQ_printf(m, " S task PID tree-key switches prio"
616		− " wait-time sum-exec sum-sleep\n");
	774	+ SEQ_printf(m, " S task PID vruntime eligible "
	775	+ "deadline slice sum-exec switches "
	776	+ "prio wait-time sum-sleep sum-block"
	777	+#ifdef CONFIG_NUMA_BALANCING
	778	+ " node group-id"
	779	+#endif
	780	+#ifdef CONFIG_CGROUP_SCHED
	781	+ " group-path"
	782	+#endif
	783	+ "\n");
617	784	SEQ_printf(m, "-------------------------------------------------------"
618		− "------------------------------------------------------\n");
	785	+ "------------------------------------------------------"
	786	+ "------------------------------------------------------"
	787	+#ifdef CONFIG_NUMA_BALANCING
	788	+ "--------------"
	789	+#endif
	790	+#ifdef CONFIG_CGROUP_SCHED
	791	+ "--------------"
	792	+#endif
	793	+ "\n");
619	794
620	795	rcu_read_lock();
621	796	for_each_process_thread(g, p) {
@@ -641,8 +816,6 @@ void print_cfs_rq(struct seq_file m, int cpu, struct cfs_rq cfs_rq) · 19 unmodified lines
641	816	SEQ_printf(m, "\n");
642	817	SEQ_printf(m, "cfs_rq[%d]:\n", cpu);
643	818	#endif
644		− SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "exec_clock",
645		− SPLIT_NS(cfs_rq->exec_clock));
646	819
647	820	raw_spin_rq_lock_irqsave(rq, flags);
648	821	root = __pick_root_entity(cfs_rq);
@@ -669,8 +842,6 @@ void print_cfs_rq(struct seq_file m, int cpu, struct cfs_rq cfs_rq) · 20 unmodified lines
669	842	SPLIT_NS(right_vruntime));
670	843	spread = right_vruntime - left_vruntime;
671	844	SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "spread", SPLIT_NS(spread));
672		− SEQ_printf(m, " .%-30s: %d\n", "nr_spread_over",
673		− cfs_rq->nr_spread_over);
674	845	SEQ_printf(m, " .%-30s: %d\n", "nr_running", cfs_rq->nr_running);
675	846	SEQ_printf(m, " .%-30s: %d\n", "h_nr_running", cfs_rq->h_nr_running);
676	847	SEQ_printf(m, " .%-30s: %d\n", "idle_nr_running",
@@ -730,9 +901,12 @@ void print_rt_rq(struct seq_file m, int cpu, struct rt_rq rt_rq) · 53 unmodified lines
730	901	SEQ_printf(m, " .%-30s: %Ld.%06ld\n", #x, SPLIT_NS(rt_rq->x))
731	902
732	903	PU(rt_nr_running);
	904	+
	905	+#ifdef CONFIG_RT_GROUP_SCHED
733	906	P(rt_throttled);
734	907	PN(rt_time);
735	908	PN(rt_runtime);
	909	+#endif
736	910
737	911	#undef PN
738	912	#undef PU

kernel/sched/fair.c+580−190

@@ -511,7 +511,7 @@ static int cfs_rq_is_idle(struct cfs_rq *cfs_rq) · 510 unmodified lines
511	511
512	512	static int se_is_idle(struct sched_entity *se)
513	513	{
514		− return 0;
	514	+ return task_has_idle_policy(task_of(se));
515	515	}
516	516
517	517	#endif /* CONFIG_FAIR_GROUP_SCHED */
@@ -779,8 +779,22 @@ static void update_min_vruntime(struct cfs_rq *cfs_rq) · 261 unmodified lines
779	779	}
780	780
781	781	/* ensure we never gain time by being placed backwards. */
782		− u64_u32_store(cfs_rq->min_vruntime,
783		− __update_min_vruntime(cfs_rq, vruntime));
	782	+ cfs_rq->min_vruntime = __update_min_vruntime(cfs_rq, vruntime);
	783	+}
	784	+
	785	+static inline u64 cfs_rq_min_slice(struct cfs_rq *cfs_rq)
	786	+{
	787	+ struct sched_entity *root = __pick_root_entity(cfs_rq);
	788	+ struct sched_entity *curr = cfs_rq->curr;
	789	+ u64 min_slice = ~0ULL;
	790	+
	791	+ if (curr && curr->on_rq)
	792	+ min_slice = curr->slice;
	793	+
	794	+ if (root)
	795	+ min_slice = min(min_slice, root->min_slice);
	796	+
	797	+ return min_slice;
784	798	}
785	799
786	800	static inline bool __entity_less(struct rb_node a, const struct rb_node b)
@@ -799,19 +813,34 @@ static inline void __min_vruntime_update(struct sched_entity *se, struct rb_node · 12 unmodified lines
799	813	}
800	814	}
801	815
	816	+static inline void __min_slice_update(struct sched_entity se, struct rb_node node)
	817	+{
	818	+ if (node) {
	819	+ struct sched_entity *rse = __node_2_se(node);
	820	+ if (rse->min_slice < se->min_slice)
	821	+ se->min_slice = rse->min_slice;
	822	+ }
	823	+}
	824	+
802	825	/*
803	826	* se->min_vruntime = min(se->vruntime, {left,right}->min_vruntime)
804	827	*/
805	828	static inline bool min_vruntime_update(struct sched_entity *se, bool exit)
806	829	{
807	830	u64 old_min_vruntime = se->min_vruntime;
	831	+ u64 old_min_slice = se->min_slice;
808	832	struct rb_node *node = &se->run_node;
809	833
810	834	se->min_vruntime = se->vruntime;
811	835	__min_vruntime_update(se, node->rb_right);
812	836	__min_vruntime_update(se, node->rb_left);
813	837
814		− return se->min_vruntime == old_min_vruntime;
	838	+ se->min_slice = se->slice;
	839	+ __min_slice_update(se, node->rb_right);
	840	+ __min_slice_update(se, node->rb_left);
	841	+
	842	+ return se->min_vruntime == old_min_vruntime &&
	843	+ se->min_slice == old_min_slice;
815	844	}
816	845
817	846	RB_DECLARE_CALLBACKS(static, min_vruntime_cb, struct sched_entity,
@@ -824,6 +853,7 @@ static void __enqueue_entity(struct cfs_rq cfs_rq, struct sched_entity se) · 6 unmodified lines
824	853	{
825	854	avg_vruntime_add(cfs_rq, se);
826	855	se->min_vruntime = se->vruntime;
	856	+ se->min_slice = se->slice;
827	857	rb_add_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline,
828	858	__entity_less, &min_vruntime_cb);
829	859	}
@@ -974,17 +1004,18 @@ static void clear_buddies(struct cfs_rq cfs_rq, struct sched_entity se); · 144 unmodified lines
974	1004	* XXX: strictly: vd_i += N*r_i/w_i such that: vd_i > ve_i
975	1005	* this is probably good enough.
976	1006	*/
977		−static void update_deadline(struct cfs_rq cfs_rq, struct sched_entity se)
	1007	+static bool update_deadline(struct cfs_rq cfs_rq, struct sched_entity se)
978	1008	{
979	1009	if ((s64)(se->vruntime - se->deadline) < 0)
980		− return;
	1010	+ return false;
981	1011
982	1012	/*
983	1013	* For EEVDF the virtual time slope is determined by w_i (iow.
984	1014	* nice) while the request time r_i is determined by
985	1015	* sysctl_sched_base_slice.
986	1016	*/
987		− se->slice = sysctl_sched_base_slice;
	1017	+ if (!se->custom_slice)
	1018	+ se->slice = sysctl_sched_base_slice;
988	1019
989	1020	/*
990	1021	* EEVDF: vd_i = ve_i + r_i / w_i
@@ -994,10 +1025,7 @@ static void update_deadline(struct cfs_rq cfs_rq, struct sched_entity se) · 3 unmodified lines
994	1025	/*
995	1026	* The task has consumed its request, reschedule.
996	1027	*/
997		− if (cfs_rq->nr_running > 1) {
998		− resched_curr(rq_of(cfs_rq));
999		− clear_buddies(cfs_rq, se);
1000		− }
	1028	+ return true;
1001	1029	}
1002	1030
1003	1031	#include "pelt.h"
@@ -1135,6 +1163,38 @@ static inline void update_curr_task(struct task_struct *p, s64 delta_exec) · 131 unmodified lines
1135	1163	dl_server_update(p->dl_server, delta_exec);
1136	1164	}
1137	1165
	1166	+static inline bool did_preempt_short(struct cfs_rq cfs_rq, struct sched_entity curr)
	1167	+{
	1168	+ if (!sched_feat(PREEMPT_SHORT))
	1169	+ return false;
	1170	+
	1171	+ if (curr->vlag == curr->deadline)
	1172	+ return false;
	1173	+
	1174	+ return !entity_eligible(cfs_rq, curr);
	1175	+}
	1176	+
	1177	+static inline bool do_preempt_short(struct cfs_rq *cfs_rq,
	1178	+ struct sched_entity pse, struct sched_entity se)
	1179	+{
	1180	+ if (!sched_feat(PREEMPT_SHORT))
	1181	+ return false;
	1182	+
	1183	+ if (pse->slice >= se->slice)
	1184	+ return false;
	1185	+
	1186	+ if (!entity_eligible(cfs_rq, pse))
	1187	+ return false;
	1188	+
	1189	+ if (entity_before(pse, se))
	1190	+ return true;
	1191	+
	1192	+ if (!entity_eligible(cfs_rq, se))
	1193	+ return true;
	1194	+
	1195	+ return false;
	1196	+}
	1197	+
1138	1198	/*
1139	1199	* Used by other classes to account runtime.
1140	1200	*/
@@ -1156,23 +1216,44 @@ s64 update_curr_common(struct rq *rq) · 15 unmodified lines
1156	1216	static void update_curr(struct cfs_rq *cfs_rq)
1157	1217	{
1158	1218	struct sched_entity *curr = cfs_rq->curr;
	1219	+ struct rq *rq = rq_of(cfs_rq);
1159	1220	s64 delta_exec;
	1221	+ bool resched;
1160	1222
1161	1223	if (unlikely(!curr))
1162	1224	return;
1163	1225
1164		− delta_exec = update_curr_se(rq_of(cfs_rq), curr);
	1226	+ delta_exec = update_curr_se(rq, curr);
1165	1227	if (unlikely(delta_exec <= 0))
1166	1228	return;
1167	1229
1168	1230	curr->vruntime += calc_delta_fair(delta_exec, curr);
1169		− update_deadline(cfs_rq, curr);
	1231	+ resched = update_deadline(cfs_rq, curr);
1170	1232	update_min_vruntime(cfs_rq);
1171	1233
1172		− if (entity_is_task(curr))
1173		− update_curr_task(task_of(curr), delta_exec);
	1234	+ if (entity_is_task(curr)) {
	1235	+ struct task_struct *p = task_of(curr);
	1236	+
	1237	+ update_curr_task(p, delta_exec);
	1238	+
	1239	+ /*
	1240	+ * Any fair task that runs outside of fair_server should
	1241	+ * account against fair_server such that it can account for
	1242	+ * this time and possibly avoid running this period.
	1243	+ */
	1244	+ if (p->dl_server != &rq->fair_server)
	1245	+ dl_server_update(&rq->fair_server, delta_exec);
	1246	+ }
1174	1247
1175	1248	account_cfs_rq_runtime(cfs_rq, delta_exec);
	1249	+
	1250	+ if (rq->nr_running == 1)
	1251	+ return;
	1252	+
	1253	+ if (resched \|\| did_preempt_short(cfs_rq, curr)) {
	1254	+ resched_curr(rq);
	1255	+ clear_buddies(cfs_rq, curr);
	1256	+ }
1176	1257	}
1177	1258
1178	1259	static void update_curr_fair(struct rq *rq)
@@ -5178,7 +5259,8 @@ place_entity(struct cfs_rq cfs_rq, struct sched_entity se, int flags) · 3999 unmodified lines
5178	5259	u64 vslice, vruntime = avg_vruntime(cfs_rq);
5179	5260	s64 lag = 0;
5180	5261
5181		− se->slice = sysctl_sched_base_slice;
	5262	+ if (!se->custom_slice)
	5263	+ se->slice = sysctl_sched_base_slice;
5182	5264	vslice = calc_delta_fair(se->slice, se);
5183	5265
5184	5266	/*
@@ -5259,6 +5341,12 @@ place_entity(struct cfs_rq cfs_rq, struct sched_entity se, int flags) · 74 unmodified lines
5259	5341
5260	5342	se->vruntime = vruntime - lag;
5261	5343
	5344	+ if (sched_feat(PLACE_REL_DEADLINE) && se->rel_deadline) {
	5345	+ se->deadline += se->vruntime;
	5346	+ se->rel_deadline = 0;
	5347	+ return;
	5348	+ }
	5349	+
5262	5350	/*
5263	5351	* When joining the competition; the existing tasks will be,
5264	5352	* on average, halfway through their slice, as such start tasks
@@ -5278,6 +5366,9 @@ static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq); · 13 unmodified lines
5278	5366
5279	5367	static inline bool cfs_bandwidth_used(void);
5280	5368
	5369	+static void
	5370	+requeue_delayed_entity(struct sched_entity *se);
	5371	+
5281	5372	static void
5282	5373	enqueue_entity(struct cfs_rq cfs_rq, struct sched_entity se, int flags)
5283	5374	{
@@ -5365,19 +5456,47 @@ static void clear_buddies(struct cfs_rq cfs_rq, struct sched_entity se) · 81 unmodified lines
5365	5456
5366	5457	static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq);
5367	5458
5368		−static void
	5459	+static inline void finish_delayed_dequeue_entity(struct sched_entity *se)
	5460	+{
	5461	+ se->sched_delayed = 0;
	5462	+ if (sched_feat(DELAY_ZERO) && se->vlag > 0)
	5463	+ se->vlag = 0;
	5464	+}
	5465	+
	5466	+static bool
5369	5467	dequeue_entity(struct cfs_rq cfs_rq, struct sched_entity se, int flags)
5370	5468	{
5371		− int action = UPDATE_TG;
	5469	+ bool sleep = flags & DEQUEUE_SLEEP;
5372	5470
	5471	+ update_curr(cfs_rq);
	5472	+
	5473	+ if (flags & DEQUEUE_DELAYED) {
	5474	+ SCHED_WARN_ON(!se->sched_delayed);
	5475	+ } else {
	5476	+ bool delay = sleep;
	5477	+ /*
	5478	+ * DELAY_DEQUEUE relies on spurious wakeups, special task
	5479	+ * states must not suffer spurious wakeups, excempt them.
	5480	+ */
	5481	+ if (flags & DEQUEUE_SPECIAL)
	5482	+ delay = false;
	5483	+
	5484	+ SCHED_WARN_ON(delay && se->sched_delayed);
	5485	+
	5486	+ if (sched_feat(DELAY_DEQUEUE) && delay &&
	5487	+ !entity_eligible(cfs_rq, se)) {
	5488	+ if (cfs_rq->next == se)
	5489	+ cfs_rq->next = NULL;
	5490	+ update_load_avg(cfs_rq, se, 0);
	5491	+ se->sched_delayed = 1;
	5492	+ return false;
	5493	+ }
	5494	+ }
	5495	+
	5496	+ int action = UPDATE_TG;
5373	5497	if (entity_is_task(se) && task_on_rq_migrating(task_of(se)))
5374	5498	action \|= DO_DETACH;
5375	5499
5376		− /*
5377		− * Update run-time statistics of the 'current'.
5378		− */
5379		− update_curr(cfs_rq);
5380		−
5381	5500	/*
5382	5501	* When dequeuing a sched_entity, we must:
5383	5502	* - Update loads to have both entity and cfs_rq synced with now.
@@ -5395,6 +5514,11 @@ dequeue_entity(struct cfs_rq cfs_rq, struct sched_entity se, int flags) · 11 unmodified lines
5395	5514	clear_buddies(cfs_rq, se);
5396	5515
5397	5516	update_entity_lag(cfs_rq, se);
	5517	+ if (sched_feat(PLACE_REL_DEADLINE) && !sleep) {
	5518	+ se->deadline -= se->vruntime;
	5519	+ se->rel_deadline = 1;
	5520	+ }
	5521	+
5398	5522	if (se != cfs_rq->curr)
5399	5523	__dequeue_entity(cfs_rq, se);
5400	5524	se->on_rq = 0;
@@ -5414,8 +5538,13 @@ dequeue_entity(struct cfs_rq cfs_rq, struct sched_entity se, int flags) · 13 unmodified lines
5414	5538	if ((flags & (DEQUEUE_SAVE \| DEQUEUE_MOVE)) != DEQUEUE_SAVE)
5415	5539	update_min_vruntime(cfs_rq);
5416	5540
	5541	+ if (flags & DEQUEUE_DELAYED)
	5542	+ finish_delayed_dequeue_entity(se);
	5543	+
5417	5544	if (cfs_rq->nr_running == 0)
5418	5545	update_idle_cfs_rq_clock_pelt(cfs_rq);
	5546	+
	5547	+ return true;
5419	5548	}
5420	5549
5421	5550	static void
@@ -5441,6 +5570,7 @@ set_next_entity(struct cfs_rq cfs_rq, struct sched_entity se) · 19 unmodified lines
5441	5570	}
5442	5571
5443	5572	update_stats_curr_start(cfs_rq, se);
	5573	+ SCHED_WARN_ON(cfs_rq->curr);
5444	5574	cfs_rq->curr = se;
5445	5575
5446	5576	/*
@@ -5461,6 +5591,8 @@ set_next_entity(struct cfs_rq cfs_rq, struct sched_entity se) · 14 unmodified lines
5461	5591	se->prev_sum_exec_runtime = se->sum_exec_runtime;
5462	5592	}
5463	5593
	5594	+static int dequeue_entities(struct rq rq, struct sched_entity se, int flags);
	5595	+
5464	5596	/*
5465	5597	* Pick the next process, keeping these things in mind, in this order:
5466	5598	* 1) keep things fair between processes/task groups
@@ -5469,16 +5601,26 @@ set_next_entity(struct cfs_rq cfs_rq, struct sched_entity se) · 2 unmodified lines
5469	5601	* 4) do not run the "skip" process, if something else is available
5470	5602	*/
5471	5603	static struct sched_entity *
5472		−pick_next_entity(struct cfs_rq *cfs_rq)
	5604	+pick_next_entity(struct rq rq, struct cfs_rq cfs_rq)
5473	5605	{
5474	5606	/*
5475	5607	* Enabling NEXT_BUDDY will affect latency but not fairness.
5476	5608	*/
5477	5609	if (sched_feat(NEXT_BUDDY) &&
5478		− cfs_rq->next && entity_eligible(cfs_rq, cfs_rq->next))
	5610	+ cfs_rq->next && entity_eligible(cfs_rq, cfs_rq->next)) {
	5611	+ /* ->next will never be delayed */
	5612	+ SCHED_WARN_ON(cfs_rq->next->sched_delayed);
5479	5613	return cfs_rq->next;
	5614	+ }
5480	5615
5481		− return pick_eevdf(cfs_rq);
	5616	+ struct sched_entity *se = pick_eevdf(cfs_rq);
	5617	+ if (se->sched_delayed) {
	5618	+ dequeue_entities(rq, se, DEQUEUE_SLEEP \| DEQUEUE_DELAYED);
	5619	+ SCHED_WARN_ON(se->sched_delayed);
	5620	+ SCHED_WARN_ON(se->on_rq);
	5621	+ return NULL;
	5622	+ }
	5623	+ return se;
5482	5624	}
5483	5625
5484	5626	static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq);
@@ -5502,6 +5644,7 @@ static void put_prev_entity(struct cfs_rq cfs_rq, struct sched_entity prev) · 17 unmodified lines
5502	5644	/* in !on_rq case, update occurred at dequeue */
5503	5645	update_load_avg(cfs_rq, prev, 0);
5504	5646	}
	5647	+ SCHED_WARN_ON(cfs_rq->curr != prev);
5505	5648	cfs_rq->curr = NULL;
5506	5649	}
5507	5650
@@ -5765,6 +5908,7 @@ static bool throttle_cfs_rq(struct cfs_rq *cfs_rq) · 257 unmodified lines
5765	5908	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
5766	5909	struct sched_entity *se;
5767	5910	long task_delta, idle_task_delta, dequeue = 1;
	5911	+ long rq_h_nr_running = rq->cfs.h_nr_running;
5768	5912
5769	5913	raw_spin_lock(&cfs_b->lock);
5770	5914	/* This will start the period timer if necessary */
@@ -5798,11 +5942,21 @@ static bool throttle_cfs_rq(struct cfs_rq *cfs_rq) · 27 unmodified lines
5798	5942	idle_task_delta = cfs_rq->idle_h_nr_running;
5799	5943	for_each_sched_entity(se) {
5800	5944	struct cfs_rq *qcfs_rq = cfs_rq_of(se);
	5945	+ int flags;
	5946	+
5801	5947	/* throttled entity or throttle-on-deactivate */
5802	5948	if (!se->on_rq)
5803	5949	goto done;
5804	5950
5805		− dequeue_entity(qcfs_rq, se, DEQUEUE_SLEEP);
	5951	+ /*
	5952	+ * Abuse SPECIAL to avoid delayed dequeue in this instance.
	5953	+ * This avoids teaching dequeue_entities() about throttled
	5954	+ * entities and keeps things relatively simple.
	5955	+ */
	5956	+ flags = DEQUEUE_SLEEP \| DEQUEUE_SPECIAL;
	5957	+ if (se->sched_delayed)
	5958	+ flags \|= DEQUEUE_DELAYED;
	5959	+ dequeue_entity(qcfs_rq, se, flags);
5806	5960
5807	5961	if (cfs_rq_is_idle(group_cfs_rq(se)))
5808	5962	idle_task_delta = cfs_rq->h_nr_running;
@@ -5836,6 +5990,9 @@ static bool throttle_cfs_rq(struct cfs_rq *cfs_rq) · 27 unmodified lines
5836	5990	/* At this point se is NULL and we are at root level*/
5837	5991	sub_nr_running(rq, task_delta);
5838	5992
	5993	+ /* Stop the fair server if throttling resulted in no runnable tasks */
	5994	+ if (rq_h_nr_running && !rq->cfs.h_nr_running)
	5995	+ dl_server_stop(&rq->fair_server);
5839	5996	done:
5840	5997	/*
5841	5998	* Note: distribution will already see us throttled via the
@@ -5854,6 +6011,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) · 12 unmodified lines
5854	6011	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
5855	6012	struct sched_entity *se;
5856	6013	long task_delta, idle_task_delta;
	6014	+ long rq_h_nr_running = rq->cfs.h_nr_running;
5857	6015
5858	6016	se = cfs_rq->tg->se[cpu_of(rq)];
5859	6017
@@ -5891,8 +6049,10 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) · 31 unmodified lines
5891	6049	for_each_sched_entity(se) {
5892	6050	struct cfs_rq *qcfs_rq = cfs_rq_of(se);
5893	6051
5894		− if (se->on_rq)
	6052	+ if (se->on_rq) {
	6053	+ SCHED_WARN_ON(se->sched_delayed);
5895	6054	break;
	6055	+ }
5896	6056	enqueue_entity(qcfs_rq, se, ENQUEUE_WAKEUP);
5897	6057
5898	6058	if (cfs_rq_is_idle(group_cfs_rq(se)))
@@ -5923,6 +6083,10 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) · 24 unmodified lines
5923	6083	goto unthrottle_throttle;
5924	6084	}
5925	6085
	6086	+ /* Start the fair server if un-throttling resulted in new runnable tasks */
	6087	+ if (!rq_h_nr_running && rq->cfs.h_nr_running)
	6088	+ dl_server_start(&rq->fair_server);
	6089	+
5926	6090	/* At this point se is NULL and we are at root level*/
5927	6091	add_nr_running(rq, task_delta);
5928	6092
@@ -6555,7 +6719,7 @@ static void sched_fair_update_stop_tick(struct rq rq, struct task_struct p) · 626 unmodified lines
6555	6719	{
6556	6720	int cpu = cpu_of(rq);
6557	6721
6558		− if (!sched_feat(HZ_BW) \|\| !cfs_bandwidth_used())
	6722	+ if (!cfs_bandwidth_used())
6559	6723	return;
6560	6724
6561	6725	if (!tick_nohz_full_cpu(cpu))
@@ -6738,6 +6902,37 @@ static int sched_idle_cpu(int cpu) · 176 unmodified lines
6738	6902	}
6739	6903	#endif
6740	6904
	6905	+static void
	6906	+requeue_delayed_entity(struct sched_entity *se)
	6907	+{
	6908	+ struct cfs_rq *cfs_rq = cfs_rq_of(se);
	6909	+
	6910	+ /*
	6911	+ * se->sched_delayed should imply: se->on_rq == 1.
	6912	+ * Because a delayed entity is one that is still on
	6913	+ * the runqueue competing until elegibility.
	6914	+ */
	6915	+ SCHED_WARN_ON(!se->sched_delayed);
	6916	+ SCHED_WARN_ON(!se->on_rq);
	6917	+
	6918	+ if (sched_feat(DELAY_ZERO)) {
	6919	+ update_entity_lag(cfs_rq, se);
	6920	+ if (se->vlag > 0) {
	6921	+ cfs_rq->nr_running--;
	6922	+ if (se != cfs_rq->curr)
	6923	+ __dequeue_entity(cfs_rq, se);
	6924	+ se->vlag = 0;
	6925	+ place_entity(cfs_rq, se, 0);
	6926	+ if (se != cfs_rq->curr)
	6927	+ __enqueue_entity(cfs_rq, se);
	6928	+ cfs_rq->nr_running++;
	6929	+ }
	6930	+ }
	6931	+
	6932	+ update_load_avg(cfs_rq, se, 0);
	6933	+ se->sched_delayed = 0;
	6934	+}
	6935	+
6741	6936	/*
6742	6937	* The enqueue_task method is called before nr_running is
6743	6938	* increased. Here we update the fair scheduling stats and
@@ -6750,6 +6945,8 @@ enqueue_task_fair(struct rq rq, struct task_struct p, int flags) · 6 unmodified lines
6750	6945	struct sched_entity *se = &p->se;
6751	6946	int idle_h_nr_running = task_has_idle_policy(p);
6752	6947	int task_new = !(flags & ENQUEUE_WAKEUP);
	6948	+ int rq_h_nr_running = rq->cfs.h_nr_running;
	6949	+ u64 slice = 0;
6753	6950
6754	6951	/*
6755	6952	* The code below (indirectly) updates schedutil which looks at
@@ -6757,7 +6954,13 @@ enqueue_task_fair(struct rq rq, struct task_struct p, int flags) · 1 unmodified line
6757	6954	* Let's add the task's estimated utilization to the cfs_rq's
6758	6955	* estimated utilization, before we update schedutil.
6759	6956	*/
6760		− util_est_enqueue(&rq->cfs, p);
	6957	+ if (!(p->se.sched_delayed && (task_on_rq_migrating(p) \|\| (flags & ENQUEUE_RESTORE))))
	6958	+ util_est_enqueue(&rq->cfs, p);
	6959	+
	6960	+ if (flags & ENQUEUE_DELAYED) {
	6961	+ requeue_delayed_entity(se);
	6962	+ return;
	6963	+ }
6761	6964
6762	6965	/*
6763	6966	* If in_iowait is set, the code below may not trigger any cpufreq
@@ -6768,10 +6971,24 @@ enqueue_task_fair(struct rq rq, struct task_struct p, int flags) · 4 unmodified lines
6768	6971	cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT);
6769	6972
6770	6973	for_each_sched_entity(se) {
6771		− if (se->on_rq)
	6974	+ if (se->on_rq) {
	6975	+ if (se->sched_delayed)
	6976	+ requeue_delayed_entity(se);
6772	6977	break;
	6978	+ }
6773	6979	cfs_rq = cfs_rq_of(se);
	6980	+
	6981	+ /*
	6982	+ * Basically set the slice of group entries to the min_slice of
	6983	+ * their respective cfs_rq. This ensures the group can service
	6984	+ * its entities in the desired time-frame.
	6985	+ */
	6986	+ if (slice) {
	6987	+ se->slice = slice;
	6988	+ se->custom_slice = 1;
	6989	+ }
6774	6990	enqueue_entity(cfs_rq, se, flags);
	6991	+ slice = cfs_rq_min_slice(cfs_rq);
6775	6992
6776	6993	cfs_rq->h_nr_running++;
6777	6994	cfs_rq->idle_h_nr_running += idle_h_nr_running;
@@ -6793,6 +7010,9 @@ enqueue_task_fair(struct rq rq, struct task_struct p, int flags) · 15 unmodified lines
6793	7010	se_update_runnable(se);
6794	7011	update_cfs_group(se);
6795	7012
	7013	+ se->slice = slice;
	7014	+ slice = cfs_rq_min_slice(cfs_rq);
	7015	+
6796	7016	cfs_rq->h_nr_running++;
6797	7017	cfs_rq->idle_h_nr_running += idle_h_nr_running;
6798	7018
@@ -6804,6 +7024,13 @@ enqueue_task_fair(struct rq rq, struct task_struct p, int flags) · 5 unmodified lines
6804	7024	goto enqueue_throttle;
6805	7025	}
6806	7026
	7027	+ if (!rq_h_nr_running && rq->cfs.h_nr_running) {
	7028	+ /* Account for idle runtime */
	7029	+ if (!rq->nr_running)
	7030	+ dl_server_update_idle_time(rq, rq->curr);
	7031	+ dl_server_start(&rq->fair_server);
	7032	+ }
	7033	+
6807	7034	/* At this point se is NULL and we are at root level*/
6808	7035	add_nr_running(rq, 1);
6809	7036
@@ -6833,36 +7060,59 @@ enqueue_throttle: · 23 unmodified lines
6833	7060	static void set_next_buddy(struct sched_entity *se);
6834	7061
6835	7062	/*
6836		− * The dequeue_task method is called before nr_running is
6837		− * decreased. We remove the task from the rbtree and
6838		− * update the fair scheduling stats:
	7063	+ * Basically dequeue_task_fair(), except it can deal with dequeue_entity()
	7064	+ * failing half-way through and resume the dequeue later.
	7065	+ *
	7066	+ * Returns:
	7067	+ * -1 - dequeue delayed
	7068	+ * 0 - dequeue throttled
	7069	+ * 1 - dequeue complete
6839	7070	*/
6840		−static void dequeue_task_fair(struct rq rq, struct task_struct p, int flags)
	7071	+static int dequeue_entities(struct rq rq, struct sched_entity se, int flags)
6841	7072	{
6842		− struct cfs_rq *cfs_rq;
6843		− struct sched_entity *se = &p->se;
6844		− int task_sleep = flags & DEQUEUE_SLEEP;
6845		− int idle_h_nr_running = task_has_idle_policy(p);
6846	7073	bool was_sched_idle = sched_idle_rq(rq);
	7074	+ int rq_h_nr_running = rq->cfs.h_nr_running;
	7075	+ bool task_sleep = flags & DEQUEUE_SLEEP;
	7076	+ bool task_delayed = flags & DEQUEUE_DELAYED;
	7077	+ struct task_struct *p = NULL;
	7078	+ int idle_h_nr_running = 0;
	7079	+ int h_nr_running = 0;
	7080	+ struct cfs_rq *cfs_rq;
	7081	+ u64 slice = 0;
6847	7082
6848		− util_est_dequeue(&rq->cfs, p);
	7083	+ if (entity_is_task(se)) {
	7084	+ p = task_of(se);
	7085	+ h_nr_running = 1;
	7086	+ idle_h_nr_running = task_has_idle_policy(p);
	7087	+ } else {
	7088	+ cfs_rq = group_cfs_rq(se);
	7089	+ slice = cfs_rq_min_slice(cfs_rq);
	7090	+ }
6849	7091
6850	7092	for_each_sched_entity(se) {
6851	7093	cfs_rq = cfs_rq_of(se);
6852		− dequeue_entity(cfs_rq, se, flags);
6853	7094
6854		− cfs_rq->h_nr_running--;
	7095	+ if (!dequeue_entity(cfs_rq, se, flags)) {
	7096	+ if (p && &p->se == se)
	7097	+ return -1;
	7098	+
	7099	+ break;
	7100	+ }
	7101	+
	7102	+ cfs_rq->h_nr_running -= h_nr_running;
6855	7103	cfs_rq->idle_h_nr_running -= idle_h_nr_running;
6856	7104
6857	7105	if (cfs_rq_is_idle(cfs_rq))
6858		− idle_h_nr_running = 1;
	7106	+ idle_h_nr_running = h_nr_running;
6859	7107
6860	7108	/* end evaluation on encountering a throttled cfs_rq */
6861	7109	if (cfs_rq_throttled(cfs_rq))
6862		− goto dequeue_throttle;
	7110	+ return 0;
6863	7111
6864	7112	/* Don't dequeue parent if it has other entities besides us */
6865	7113	if (cfs_rq->load.weight) {
	7114	+ slice = cfs_rq_min_slice(cfs_rq);
	7115	+
6866	7116	/* Avoid re-evaluating load for this entity: */
6867	7117	se = parent_entity(se);
6868	7118	/*
@@ -6874,6 +7124,7 @@ static void dequeue_task_fair(struct rq rq, struct task_struct p, int flags) · 5 unmodified lines
6874	7124	break;
6875	7125	}
6876	7126	flags \|= DEQUEUE_SLEEP;
	7127	+ flags &= ~(DEQUEUE_DELAYED \| DEQUEUE_SPECIAL);
6877	7128	}
6878	7129
6879	7130	for_each_sched_entity(se) {
@@ -6883,28 +7134,61 @@ static void dequeue_task_fair(struct rq rq, struct task_struct p, int flags) · 3 unmodified lines
6883	7134	se_update_runnable(se);
6884	7135	update_cfs_group(se);
6885	7136
6886		− cfs_rq->h_nr_running--;
	7137	+ se->slice = slice;
	7138	+ slice = cfs_rq_min_slice(cfs_rq);
	7139	+
	7140	+ cfs_rq->h_nr_running -= h_nr_running;
6887	7141	cfs_rq->idle_h_nr_running -= idle_h_nr_running;
6888	7142
6889	7143	if (cfs_rq_is_idle(cfs_rq))
6890		− idle_h_nr_running = 1;
	7144	+ idle_h_nr_running = h_nr_running;
6891	7145
6892	7146	/* end evaluation on encountering a throttled cfs_rq */
6893	7147	if (cfs_rq_throttled(cfs_rq))
6894		− goto dequeue_throttle;
6895		−
	7148	+ return 0;
6896	7149	}
6897	7150
6898		− /* At this point se is NULL and we are at root level*/
6899		− sub_nr_running(rq, 1);
	7151	+ sub_nr_running(rq, h_nr_running);
	7152	+
	7153	+ if (rq_h_nr_running && !rq->cfs.h_nr_running)
	7154	+ dl_server_stop(&rq->fair_server);
6900	7155
6901	7156	/* balance early to pull high priority tasks */
6902	7157	if (unlikely(!was_sched_idle && sched_idle_rq(rq)))
6903	7158	rq->next_balance = jiffies;
6904	7159
6905		−dequeue_throttle:
6906		− util_est_update(&rq->cfs, p, task_sleep);
	7160	+ if (p && task_delayed) {
	7161	+ SCHED_WARN_ON(!task_sleep);
	7162	+ SCHED_WARN_ON(p->on_rq != 1);
	7163	+
	7164	+ /* Fix-up what dequeue_task_fair() skipped */
	7165	+ hrtick_update(rq);
	7166	+
	7167	+ /* Fix-up what block_task() skipped. */
	7168	+ __block_task(rq, p);
	7169	+ }
	7170	+
	7171	+ return 1;
	7172	+}
	7173	+
	7174	+/*
	7175	+ * The dequeue_task method is called before nr_running is
	7176	+ * decreased. We remove the task from the rbtree and
	7177	+ * update the fair scheduling stats:
	7178	+ */
	7179	+static bool dequeue_task_fair(struct rq rq, struct task_struct p, int flags)
	7180	+{
	7181	+ if (!(p->se.sched_delayed && (task_on_rq_migrating(p) \|\| (flags & DEQUEUE_SAVE))))
	7182	+ util_est_dequeue(&rq->cfs, p);
	7183	+
	7184	+ if (dequeue_entities(rq, &p->se, flags) < 0) {
	7185	+ util_est_update(&rq->cfs, p, DEQUEUE_SLEEP);
	7186	+ return false;
	7187	+ }
	7188	+
	7189	+ util_est_update(&rq->cfs, p, flags & DEQUEUE_SLEEP);
6907	7190	hrtick_update(rq);
	7191	+ return true;
6908	7192	}
6909	7193
6910	7194	#ifdef CONFIG_SMP
@@ -7802,6 +8086,105 @@ static unsigned long cpu_util_without(int cpu, struct task_struct *p) · 891 unmodified lines
7802	8086	return cpu_util(cpu, p, -1, 0);
7803	8087	}
7804	8088
	8089	+/*
	8090	+ * This function computes an effective utilization for the given CPU, to be
	8091	+ * used for frequency selection given the linear relation: f = u * f_max.
	8092	+ *
	8093	+ * The scheduler tracks the following metrics:
	8094	+ *
	8095	+ * cpu_util_{cfs,rt,dl,irq}()
	8096	+ * cpu_bw_dl()
	8097	+ *
	8098	+ * Where the cfs,rt and dl util numbers are tracked with the same metric and
	8099	+ * synchronized windows and are thus directly comparable.
	8100	+ *
	8101	+ * The cfs,rt,dl utilization are the running times measured with rq->clock_task
	8102	+ * which excludes things like IRQ and steal-time. These latter are then accrued
	8103	+ * in the IRQ utilization.
	8104	+ *
	8105	+ * The DL bandwidth number OTOH is not a measured metric but a value computed
	8106	+ * based on the task model parameters and gives the minimal utilization
	8107	+ * required to meet deadlines.
	8108	+ */
	8109	+unsigned long effective_cpu_util(int cpu, unsigned long util_cfs,
	8110	+ unsigned long *min,
	8111	+ unsigned long *max)
	8112	+{
	8113	+ unsigned long util, irq, scale;
	8114	+ struct rq *rq = cpu_rq(cpu);
	8115	+
	8116	+ scale = arch_scale_cpu_capacity(cpu);
	8117	+
	8118	+ /*
	8119	+ * Early check to see if IRQ/steal time saturates the CPU, can be
	8120	+ * because of inaccuracies in how we track these -- see
	8121	+ * update_irq_load_avg().
	8122	+ */
	8123	+ irq = cpu_util_irq(rq);
	8124	+ if (unlikely(irq >= scale)) {
	8125	+ if (min)
	8126	+ *min = scale;
	8127	+ if (max)
	8128	+ *max = scale;
	8129	+ return scale;
	8130	+ }
	8131	+
	8132	+ if (min) {
	8133	+ /*
	8134	+ * The minimum utilization returns the highest level between:
	8135	+ * - the computed DL bandwidth needed with the IRQ pressure which
	8136	+ * steals time to the deadline task.
	8137	+ * - The minimum performance requirement for CFS and/or RT.
	8138	+ */
	8139	+ *min = max(irq + cpu_bw_dl(rq), uclamp_rq_get(rq, UCLAMP_MIN));
	8140	+
	8141	+ /*
	8142	+ * When an RT task is runnable and uclamp is not used, we must
	8143	+ * ensure that the task will run at maximum compute capacity.
	8144	+ */
	8145	+ if (!uclamp_is_used() && rt_rq_is_runnable(&rq->rt))
	8146	+ min = max(min, scale);
	8147	+ }
	8148	+
	8149	+ /*
	8150	+ * Because the time spend on RT/DL tasks is visible as 'lost' time to
	8151	+ * CFS tasks and we use the same metric to track the effective
	8152	+ * utilization (PELT windows are synchronized) we can directly add them
	8153	+ * to obtain the CPU's actual utilization.
	8154	+ */
	8155	+ util = util_cfs + cpu_util_rt(rq);
	8156	+ util += cpu_util_dl(rq);
	8157	+
	8158	+ /*
	8159	+ * The maximum hint is a soft bandwidth requirement, which can be lower
	8160	+ * than the actual utilization because of uclamp_max requirements.
	8161	+ */
	8162	+ if (max)
	8163	+ *max = min(scale, uclamp_rq_get(rq, UCLAMP_MAX));
	8164	+
	8165	+ if (util >= scale)
	8166	+ return scale;
	8167	+
	8168	+ /*
	8169	+ * There is still idle time; further improve the number by using the
	8170	+ * IRQ metric. Because IRQ/steal time is hidden from the task clock we
	8171	+ * need to scale the task numbers:
	8172	+ *
	8173	+ * max - irq
	8174	+ * U' = irq + --------- * U
	8175	+ * max
	8176	+ */
	8177	+ util = scale_irq_capacity(util, irq, scale);
	8178	+ util += irq;
	8179	+
	8180	+ return min(scale, util);
	8181	+}
	8182	+
	8183	+unsigned long sched_cpu_util(int cpu)
	8184	+{
	8185	+ return effective_cpu_util(cpu, cpu_util_cfs(cpu), NULL, NULL);
	8186	+}
	8187	+
7805	8188	/*
7806	8189	* energy_env - Utilization landscape for energy estimation.
7807	8190	* @task_busy_time: Utilization contribution by the task for which we test the
@@ -8286,7 +8669,21 @@ static void migrate_task_rq_fair(struct task_struct *p, int new_cpu) · 478 unmodified lines
8286	8669
8287	8670	static void task_dead_fair(struct task_struct *p)
8288	8671	{
8289		− remove_entity_load_avg(&p->se);
	8672	+ struct sched_entity *se = &p->se;
	8673	+
	8674	+ if (se->sched_delayed) {
	8675	+ struct rq_flags rf;
	8676	+ struct rq *rq;
	8677	+
	8678	+ rq = task_rq_lock(p, &rf);
	8679	+ if (se->sched_delayed) {
	8680	+ update_rq_clock(rq);
	8681	+ dequeue_entities(rq, se, DEQUEUE_SLEEP \| DEQUEUE_DELAYED);
	8682	+ }
	8683	+ task_rq_unlock(rq, p, &rf);
	8684	+ }
	8685	+
	8686	+ remove_entity_load_avg(se);
8290	8687	}
8291	8688
8292	8689	/*
@@ -8322,7 +8719,7 @@ static void set_cpus_allowed_fair(struct task_struct *p, struct affinity_context · 29 unmodified lines
8322	8719	static int
8323	8720	balance_fair(struct rq rq, struct task_struct prev, struct rq_flags *rf)
8324	8721	{
8325		− if (rq->nr_running)
	8722	+ if (sched_fair_runnable(rq))
8326	8723	return 1;
8327	8724
8328	8725	return sched_balance_newidle(rq, rf) != 0;
@@ -8381,16 +8778,7 @@ static void check_preempt_wakeup_fair(struct rq rq, struct task_struct p, int · 52 unmodified lines
8381	8778	if (test_tsk_need_resched(curr))
8382	8779	return;
8383	8780
8384		− /* Idle tasks are by definition preempted by non-idle tasks. */
8385		− if (unlikely(task_has_idle_policy(curr)) &&
8386		− likely(!task_has_idle_policy(p)))
8387		− goto preempt;
8388		−
8389		− /*
8390		− * Batch and idle tasks do not preempt non-idle tasks (their preemption
8391		− * is driven by the tick):
8392		− */
8393		− if (unlikely(p->policy != SCHED_NORMAL) \|\| !sched_feat(WAKEUP_PREEMPTION))
	8781	+ if (!sched_feat(WAKEUP_PREEMPTION))
8394	8782	return;
8395	8783
8396	8784	find_matching_se(&se, &pse);
@@ -8400,7 +8788,7 @@ static void check_preempt_wakeup_fair(struct rq rq, struct task_struct p, int · 3 unmodified lines
8400	8788	pse_is_idle = se_is_idle(pse);
8401	8789
8402	8790	/*
8403		− * Preempt an idle group in favor of a non-idle group (and don't preempt
	8791	+ * Preempt an idle entity in favor of a non-idle entity (and don't preempt
8404	8792	* in the inverse case).
8405	8793	*/
8406	8794	if (cse_is_idle && !pse_is_idle)
@@ -8408,11 +8796,26 @@ static void check_preempt_wakeup_fair(struct rq rq, struct task_struct p, int · 1 unmodified line
8408	8796	if (cse_is_idle != pse_is_idle)
8409	8797	return;
8410	8798
	8799	+ /*
	8800	+ * BATCH and IDLE tasks do not preempt others.
	8801	+ */
	8802	+ if (unlikely(p->policy != SCHED_NORMAL))
	8803	+ return;
	8804	+
8411	8805	cfs_rq = cfs_rq_of(se);
8412	8806	update_curr(cfs_rq);
	8807	+ /*
	8808	+ * If @p has a shorter slice than current and @p is eligible, override
	8809	+ * current's slice protection in order to allow preemption.
	8810	+ *
	8811	+ * Note that even if @p does not turn out to be the most eligible
	8812	+ * task at this moment, current's slice protection will be lost.
	8813	+ */
	8814	+ if (do_preempt_short(cfs_rq, pse, se) && se->vlag == se->deadline)
	8815	+ se->vlag = se->deadline + 1;
8413	8816
8414	8817	/*
8415		− * XXX pick_eevdf(cfs_rq) != se ?
	8818	+ * If @p has become the most eligible task, force preemption.
8416	8819	*/
8417	8820	if (pick_eevdf(cfs_rq) == pse)
8418	8821	goto preempt;
@@ -8423,7 +8826,6 @@ preempt: · 4 unmodified lines
8423	8826	resched_curr(rq);
8424	8827	}
8425	8828
8426		−#ifdef CONFIG_SMP
8427	8829	static struct task_struct pick_task_fair(struct rq rq)
8428	8830	{
8429	8831	struct sched_entity *se;
@@ -8435,95 +8837,58 @@ again: · 5 unmodified lines
8435	8837	return NULL;
8436	8838
8437	8839	do {
8438		− struct sched_entity *curr = cfs_rq->curr;
8439		−
8440		− /* When we pick for a remote RQ, we'll not have done put_prev_entity() */
8441		− if (curr) {
8442		− if (curr->on_rq)
8443		− update_curr(cfs_rq);
8444		− else
8445		− curr = NULL;
	8840	+ /* Might not have done put_prev_entity() */
	8841	+ if (cfs_rq->curr && cfs_rq->curr->on_rq)
	8842	+ update_curr(cfs_rq);
8446	8843
8447		− if (unlikely(check_cfs_rq_runtime(cfs_rq)))
8448		− goto again;
8449		− }
	8844	+ if (unlikely(check_cfs_rq_runtime(cfs_rq)))
	8845	+ goto again;
8450	8846
8451		− se = pick_next_entity(cfs_rq);
	8847	+ se = pick_next_entity(rq, cfs_rq);
	8848	+ if (!se)
	8849	+ goto again;
8452	8850	cfs_rq = group_cfs_rq(se);
8453	8851	} while (cfs_rq);
8454	8852
8455	8853	return task_of(se);
8456	8854	}
8457		−#endif
	8855	+
	8856	+static void __set_next_task_fair(struct rq rq, struct task_struct p, bool first);
	8857	+static void set_next_task_fair(struct rq rq, struct task_struct p, bool first);
8458	8858
8459	8859	struct task_struct *
8460	8860	pick_next_task_fair(struct rq rq, struct task_struct prev, struct rq_flags *rf)
8461	8861	{
8462		− struct cfs_rq *cfs_rq = &rq->cfs;
8463	8862	struct sched_entity *se;
8464	8863	struct task_struct *p;
8465	8864	int new_tasks;
8466	8865
8467	8866	again:
8468		− if (!sched_fair_runnable(rq))
	8867	+ p = pick_task_fair(rq);
	8868	+ if (!p)
8469	8869	goto idle;
	8870	+ se = &p->se;
8470	8871
8471	8872	#ifdef CONFIG_FAIR_GROUP_SCHED
8472		− if (!prev \|\| prev->sched_class != &fair_sched_class)
	8873	+ if (prev->sched_class != &fair_sched_class)
8473	8874	goto simple;
8474	8875
	8876	+ __put_prev_set_next_dl_server(rq, prev, p);
	8877	+
8475	8878	/*
8476	8879	* Because of the set_next_buddy() in dequeue_task_fair() it is rather
8477	8880	* likely that a next task is from the same cgroup as the current.
8478	8881	*
8479	8882	* Therefore attempt to avoid putting and setting the entire cgroup
8480	8883	* hierarchy, only change the part that actually changes.
8481		− */
8482		−
8483		− do {
8484		− struct sched_entity *curr = cfs_rq->curr;
8485		−
8486		− /*
8487		− * Since we got here without doing put_prev_entity() we also
8488		− * have to consider cfs_rq->curr. If it is still a runnable
8489		− * entity, update_curr() will update its vruntime, otherwise
8490		− * forget we've ever seen it.
8491		− */
8492		− if (curr) {
8493		− if (curr->on_rq)
8494		− update_curr(cfs_rq);
8495		− else
8496		− curr = NULL;
8497		−
8498		− /*
8499		− * This call to check_cfs_rq_runtime() will do the
8500		− * throttle and dequeue its entity in the parent(s).
8501		− * Therefore the nr_running test will indeed
8502		− * be correct.
8503		− */
8504		− if (unlikely(check_cfs_rq_runtime(cfs_rq))) {
8505		− cfs_rq = &rq->cfs;
8506		−
8507		− if (!cfs_rq->nr_running)
8508		− goto idle;
8509		−
8510		− goto simple;
8511		− }
8512		− }
8513		−
8514		− se = pick_next_entity(cfs_rq);
8515		− cfs_rq = group_cfs_rq(se);
8516		− } while (cfs_rq);
8517		−
8518		− p = task_of(se);
8519		−
8520		− /*
	8884	+ *
8521	8885	* Since we haven't yet done put_prev_entity and if the selected task
8522	8886	* is a different task than we started out with, try and touch the
8523	8887	* least amount of cfs_rqs.
8524	8888	*/
8525	8889	if (prev != p) {
8526	8890	struct sched_entity *pse = &prev->se;
	8891	+ struct cfs_rq *cfs_rq;
8527	8892
8528	8893	while (!(cfs_rq = is_same_group(se, pse))) {
8529	8894	int se_depth = se->depth;
@@ -8541,38 +8906,15 @@ again: · 11 unmodified lines
8541	8906
8542	8907	put_prev_entity(cfs_rq, pse);
8543	8908	set_next_entity(cfs_rq, se);
8544		− }
8545		−
8546		− goto done;
8547		−simple:
8548		−#endif
8549		− if (prev)
8550		− put_prev_task(rq, prev);
8551	8909
8552		− do {
8553		− se = pick_next_entity(cfs_rq);
8554		− set_next_entity(cfs_rq, se);
8555		− cfs_rq = group_cfs_rq(se);
8556		− } while (cfs_rq);
	8910	+ __set_next_task_fair(rq, p, true);
	8911	+ }
8557	8912
8558		− p = task_of(se);
	8913	+ return p;
8559	8914
8560		−done: __maybe_unused;
8561		−#ifdef CONFIG_SMP
8562		− /*
8563		− * Move the next running task to the front of
8564		− * the list, so our cfs_tasks list becomes MRU
8565		− * one.
8566		− */
8567		− list_move(&p->se.group_node, &rq->cfs_tasks);
	8915	+simple:
8568	8916	#endif
8569		−
8570		− if (hrtick_enabled_fair(rq))
8571		− hrtick_start_fair(rq, p);
8572		−
8573		− update_misfit_status(p, rq);
8574		− sched_fair_update_stop_tick(rq, p);
8575		−
	8917	+ put_prev_set_next_task(rq, prev, p);
8576	8918	return p;
8577	8919
8578	8920	idle:
@@ -8601,15 +8943,34 @@ idle: · 22 unmodified lines
8601	8943	return NULL;
8602	8944	}
8603	8945
8604		−static struct task_struct __pick_next_task_fair(struct rq rq)
	8946	+static struct task_struct __pick_next_task_fair(struct rq rq, struct task_struct *prev)
8605	8947	{
8606		− return pick_next_task_fair(rq, NULL, NULL);
	8948	+ return pick_next_task_fair(rq, prev, NULL);
	8949	+}
	8950	+
	8951	+static bool fair_server_has_tasks(struct sched_dl_entity *dl_se)
	8952	+{
	8953	+ return !!dl_se->rq->cfs.nr_running;
	8954	+}
	8955	+
	8956	+static struct task_struct fair_server_pick_task(struct sched_dl_entity dl_se)
	8957	+{
	8958	+ return pick_task_fair(dl_se->rq);
	8959	+}
	8960	+
	8961	+void fair_server_init(struct rq *rq)
	8962	+{
	8963	+ struct sched_dl_entity *dl_se = &rq->fair_server;
	8964	+
	8965	+ init_dl_entity(dl_se);
	8966	+
	8967	+ dl_server_init(dl_se, rq, fair_server_has_tasks, fair_server_pick_task);
8607	8968	}
8608	8969
8609	8970	/*
8610	8971	* Account for a descheduled task:
8611	8972	*/
8612		−static void put_prev_task_fair(struct rq rq, struct task_struct prev)
	8973	+static void put_prev_task_fair(struct rq rq, struct task_struct prev, struct task_struct *next)
8613	8974	{
8614	8975	struct sched_entity *se = &prev->se;
8615	8976	struct cfs_rq *cfs_rq;
@@ -9360,9 +9721,10 @@ static bool __update_blocked_others(struct rq rq, bool done) · 744 unmodified lines
9360	9721
9361	9722	hw_pressure = arch_scale_hw_pressure(cpu_of(rq));
9362	9723
	9724	+ /* hw_pressure doesn't care about invariance */
9363	9725	decayed = update_rt_rq_load_avg(now, rq, curr_class == &rt_sched_class) \|
9364	9726	update_dl_rq_load_avg(now, rq, curr_class == &dl_sched_class) \|
9365		− update_hw_load_avg(now, rq, hw_pressure) \|
	9727	+ update_hw_load_avg(rq_clock_task(rq), rq, hw_pressure) \|
9366	9728	update_irq_load_avg(rq, 0);
9367	9729
9368	9730	if (others_have_blocked(rq))
@@ -12702,22 +13064,7 @@ static void task_tick_fair(struct rq rq, struct task_struct curr, int queued) · 3333 unmodified lines
12702	13064	*/
12703	13065	static void task_fork_fair(struct task_struct *p)
12704	13066	{
12705		− struct sched_entity se = &p->se, curr;
12706		− struct cfs_rq *cfs_rq;
12707		− struct rq *rq = this_rq();
12708		− struct rq_flags rf;
12709		−
12710		− rq_lock(rq, &rf);
12711		− update_rq_clock(rq);
12712		−
12713	13067	set_task_max_allowed_capacity(p);
12714		−
12715		− cfs_rq = task_cfs_rq(current);
12716		− curr = cfs_rq->curr;
12717		− if (curr)
12718		− update_curr(cfs_rq);
12719		− place_entity(cfs_rq, se, ENQUEUE_INITIAL);
12720		− rq_unlock(rq, &rf);
12721	13068	}
12722	13069
12723	13070	/*
@@ -12829,10 +13176,28 @@ static void attach_task_cfs_rq(struct task_struct *p) · 105 unmodified lines
12829	13176	static void switched_from_fair(struct rq rq, struct task_struct p)
12830	13177	{
12831	13178	detach_task_cfs_rq(p);
	13179	+ /*
	13180	+ * Since this is called after changing class, this is a little weird
	13181	+ * and we cannot use DEQUEUE_DELAYED.
	13182	+ */
	13183	+ if (p->se.sched_delayed) {
	13184	+ /* First, dequeue it from its new class' structures */
	13185	+ dequeue_task(rq, p, DEQUEUE_NOCLOCK \| DEQUEUE_SLEEP);
	13186	+ /*
	13187	+ * Now, clean up the fair_sched_class side of things
	13188	+ * related to sched_delayed being true and that wasn't done
	13189	+ * due to the generic dequeue not using DEQUEUE_DELAYED.
	13190	+ */
	13191	+ finish_delayed_dequeue_entity(&p->se);
	13192	+ p->se.rel_deadline = 0;
	13193	+ __block_task(rq, p);
	13194	+ }
12832	13195	}
12833	13196
12834	13197	static void switched_to_fair(struct rq rq, struct task_struct p)
12835	13198	{
	13199	+ SCHED_WARN_ON(p->se.sched_delayed);
	13200	+
12836	13201	attach_task_cfs_rq(p);
12837	13202
12838	13203	set_task_max_allowed_capacity(p);
@@ -12850,12 +13215,7 @@ static void switched_to_fair(struct rq rq, struct task_struct p) · 11 unmodified lines
12850	13215	}
12851	13216	}
12852	13217
12853		−/* Account for a task changing its policy or group.
12854		− *
12855		− * This routine is mostly called to set cfs_rq->curr field when a task
12856		− * migrates between groups/classes.
12857		− */
12858		−static void set_next_task_fair(struct rq rq, struct task_struct p, bool first)
	13218	+static void __set_next_task_fair(struct rq rq, struct task_struct p, bool first)
12859	13219	{
12860	13220	struct sched_entity *se = &p->se;
12861	13221
@@ -12868,6 +13228,27 @@ static void set_next_task_fair(struct rq rq, struct task_struct p, bool first) · 6 unmodified lines
12868	13228	list_move(&se->group_node, &rq->cfs_tasks);
12869	13229	}
12870	13230	#endif
	13231	+ if (!first)
	13232	+ return;
	13233	+
	13234	+ SCHED_WARN_ON(se->sched_delayed);
	13235	+
	13236	+ if (hrtick_enabled_fair(rq))
	13237	+ hrtick_start_fair(rq, p);
	13238	+
	13239	+ update_misfit_status(p, rq);
	13240	+ sched_fair_update_stop_tick(rq, p);
	13241	+}
	13242	+
	13243	+/*
	13244	+ * Account for a task changing its policy or group.
	13245	+ *
	13246	+ * This routine is mostly called to set cfs_rq->curr field when a task
	13247	+ * migrates between groups/classes.
	13248	+ */
	13249	+static void set_next_task_fair(struct rq rq, struct task_struct p, bool first)
	13250	+{
	13251	+ struct sched_entity *se = &p->se;
12871	13252
12872	13253	for_each_sched_entity(se) {
12873	13254	struct cfs_rq *cfs_rq = cfs_rq_of(se);
@@ -12876,12 +13257,14 @@ static void set_next_task_fair(struct rq rq, struct task_struct p, bool first) · 2 unmodified lines
12876	13257	/* ensure bandwidth has been allocated on our new cfs_rq */
12877	13258	account_cfs_rq_runtime(cfs_rq, 0);
12878	13259	}
	13260	+
	13261	+ __set_next_task_fair(rq, p, first);
12879	13262	}
12880	13263
12881	13264	void init_cfs_rq(struct cfs_rq *cfs_rq)
12882	13265	{
12883	13266	cfs_rq->tasks_timeline = RB_ROOT_CACHED;
12884		− u64_u32_store(cfs_rq->min_vruntime, (u64)(-(1LL << 20)));
	13267	+ cfs_rq->min_vruntime = (u64)(-(1LL << 20));
12885	13268	#ifdef CONFIG_SMP
12886	13269	raw_spin_lock_init(&cfs_rq->removed.lock);
12887	13270	#endif
@@ -12983,28 +13366,35 @@ void online_fair_sched_group(struct task_group *tg) · 95 unmodified lines
12983	13366
12984	13367	void unregister_fair_sched_group(struct task_group *tg)
12985	13368	{
12986		− unsigned long flags;
12987		− struct rq *rq;
12988	13369	int cpu;
12989	13370
12990	13371	destroy_cfs_bandwidth(tg_cfs_bandwidth(tg));
12991	13372
12992	13373	for_each_possible_cpu(cpu) {
12993		− if (tg->se[cpu])
12994		− remove_entity_load_avg(tg->se[cpu]);
	13374	+ struct cfs_rq *cfs_rq = tg->cfs_rq[cpu];
	13375	+ struct sched_entity *se = tg->se[cpu];
	13376	+ struct rq *rq = cpu_rq(cpu);
	13377	+
	13378	+ if (se) {
	13379	+ if (se->sched_delayed) {
	13380	+ guard(rq_lock_irqsave)(rq);
	13381	+ if (se->sched_delayed) {
	13382	+ update_rq_clock(rq);
	13383	+ dequeue_entities(rq, se, DEQUEUE_SLEEP \| DEQUEUE_DELAYED);
	13384	+ }
	13385	+ list_del_leaf_cfs_rq(cfs_rq);
	13386	+ }
	13387	+ remove_entity_load_avg(se);
	13388	+ }
12995	13389
12996	13390	/*
12997	13391	* Only empty task groups can be destroyed; so we can speculatively
12998	13392	* check on_list without danger of it being re-added.
12999	13393	*/
13000		− if (!tg->cfs_rq[cpu]->on_list)
13001		− continue;
13002		−
13003		− rq = cpu_rq(cpu);
13004		−
13005		− raw_spin_rq_lock_irqsave(rq, flags);
13006		− list_del_leaf_cfs_rq(tg->cfs_rq[cpu]);
13007		− raw_spin_rq_unlock_irqrestore(rq, flags);
	13394	+ if (cfs_rq->on_list) {
	13395	+ guard(rq_lock_irqsave)(rq);
	13396	+ list_del_leaf_cfs_rq(cfs_rq);
	13397	+ }
13008	13398	}
13009	13399	}
13010	13400
@@ -13194,13 +13584,13 @@ DEFINE_SCHED_CLASS(fair) = { · 183 unmodified lines
13194	13584
13195	13585	.wakeup_preempt = check_preempt_wakeup_fair,
13196	13586
	13587	+ .pick_task = pick_task_fair,
13197	13588	.pick_next_task = __pick_next_task_fair,
13198	13589	.put_prev_task = put_prev_task_fair,
13199	13590	.set_next_task = set_next_task_fair,
13200	13591
13201	13592	#ifdef CONFIG_SMP
13202	13593	.balance = balance_fair,
13203		− .pick_task = pick_task_fair,
13204	13594	.select_task_rq = select_task_rq_fair,
13205	13595	.migrate_task_rq = migrate_task_rq_fair,
13206	13596

kernel/sched/features.h+28−2

@@ -5,8 +5,24 @@ · 4 unmodified lines
5	5	* sleep+wake cycles. EEVDF placement strategy #1, #2 if disabled.
6	6	*/
7	7	SCHED_FEAT(PLACE_LAG, true)
	8	+/*
	9	+ * Give new tasks half a slice to ease into the competition.
	10	+ */
8	11	SCHED_FEAT(PLACE_DEADLINE_INITIAL, true)
	12	+/*
	13	+ * Preserve relative virtual deadline on 'migration'.
	14	+ */
	15	+SCHED_FEAT(PLACE_REL_DEADLINE, true)
	16	+/*
	17	+ * Inhibit (wakeup) preemption until the current task has either matched the
	18	+ * 0-lag point or until is has exhausted it's slice.
	19	+ */
9	20	SCHED_FEAT(RUN_TO_PARITY, true)
	21	+/*
	22	+ * Allow wakeup of tasks with a shorter slice to cancel RESPECT_SLICE for
	23	+ * current.
	24	+ */
	25	+SCHED_FEAT(PREEMPT_SHORT, true)
10	26
11	27	/*
12	28	* Prefer to schedule the task we woke last (assuming it failed
@@ -21,6 +37,18 @@ SCHED_FEAT(NEXT_BUDDY, false) · 8 unmodified lines
21	37	*/
22	38	SCHED_FEAT(CACHE_HOT_BUDDY, true)
23	39
	40	+/*
	41	+ * Delay dequeueing tasks until they get selected or woken.
	42	+ *
	43	+ * By delaying the dequeue for non-eligible tasks, they remain in the
	44	+ * competition and can burn off their negative lag. When they get selected
	45	+ * they'll have positive lag by definition.
	46	+ *
	47	+ * DELAY_ZERO clips the lag on dequeue (or wakeup) to 0.
	48	+ */
	49	+SCHED_FEAT(DELAY_DEQUEUE, true)
	50	+SCHED_FEAT(DELAY_ZERO, true)
	51	+
24	52	/*
25	53	* Allow wakeup-time preemption of the current task:
26	54	*/
@@ -85,5 +113,3 @@ SCHED_FEAT(WA_BIAS, true) · 58 unmodified lines
85	113	SCHED_FEAT(UTIL_EST, true)
86	114
87	115	SCHED_FEAT(LATENCY_WARN, false)
88		−
89		−SCHED_FEAT(HZ_BW, true)

kernel/sched/idle.c+7−16

@@ -450,43 +450,35 @@ static void wakeup_preempt_idle(struct rq rq, struct task_struct p, int flags) · 449 unmodified lines
450	450	resched_curr(rq);
451	451	}
452	452
453		−static void put_prev_task_idle(struct rq rq, struct task_struct prev)
	453	+static void put_prev_task_idle(struct rq rq, struct task_struct prev, struct task_struct *next)
454	454	{
	455	+ dl_server_update_idle_time(rq, prev);
455	456	}
456	457
457	458	static void set_next_task_idle(struct rq rq, struct task_struct next, bool first)
458	459	{
459	460	update_idle_core(rq);
460	461	schedstat_inc(rq->sched_goidle);
	462	+ next->se.exec_start = rq_clock_task(rq);
461	463	}
462	464
463		−#ifdef CONFIG_SMP
464		−static struct task_struct pick_task_idle(struct rq rq)
	465	+struct task_struct pick_task_idle(struct rq rq)
465	466	{
466	467	return rq->idle;
467	468	}
468		−#endif
469		−
470		−struct task_struct pick_next_task_idle(struct rq rq)
471		−{
472		− struct task_struct *next = rq->idle;
473		−
474		− set_next_task_idle(rq, next, true);
475		−
476		− return next;
477		−}
478	469
479	470	/*
480	471	* It is not legal to sleep in the idle task - print a warning
481	472	* message if some code attempts to do it:
482	473	*/
483		−static void
	474	+static bool
484	475	dequeue_task_idle(struct rq rq, struct task_struct p, int flags)
485	476	{
486	477	raw_spin_rq_unlock_irq(rq);
487	478	printk(KERN_ERR "bad: scheduling from the idle thread!\n");
488	479	dump_stack();
489	480	raw_spin_rq_lock_irq(rq);
	481	+ return true;
490	482	}
491	483
492	484	/*
@@ -528,13 +520,12 @@ DEFINE_SCHED_CLASS(idle) = { · 35 unmodified lines
528	520
529	521	.wakeup_preempt = wakeup_preempt_idle,
530	522
531		− .pick_next_task = pick_next_task_idle,
	523	+ .pick_task = pick_task_idle,
532	524	.put_prev_task = put_prev_task_idle,
533	525	.set_next_task = set_next_task_idle,
534	526
535	527	#ifdef CONFIG_SMP
536	528	.balance = balance_idle,
537		− .pick_task = pick_task_idle,
538	529	.select_task_rq = select_task_rq_idle,
539	530	.set_cpus_allowed = set_cpus_allowed_common,
540	531	#endif

kernel/sched/rt.c+112−149

@@ -8,10 +8,6 @@ int sched_rr_timeslice = RR_TIMESLICE; · 7 unmodified lines
8	8	/* More than 4 hours if BW_SHIFT equals 20. */
9	9	static const u64 max_rt_runtime = MAX_BW;
10	10
11		−static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun);
12		−
13		−struct rt_bandwidth def_rt_bandwidth;
14		−
15	11	/*
16	12	* period over which we measure -rt task CPU usage in us.
17	13	* default: 1s
@@ -66,6 +62,40 @@ static int __init sched_rt_sysctl_init(void) · 48 unmodified lines
66	62	late_initcall(sched_rt_sysctl_init);
67	63	#endif
68	64
	65	+void init_rt_rq(struct rt_rq *rt_rq)
	66	+{
	67	+ struct rt_prio_array *array;
	68	+ int i;
	69	+
	70	+ array = &rt_rq->active;
	71	+ for (i = 0; i < MAX_RT_PRIO; i++) {
	72	+ INIT_LIST_HEAD(array->queue + i);
	73	+ __clear_bit(i, array->bitmap);
	74	+ }
	75	+ /* delimiter for bitsearch: */
	76	+ __set_bit(MAX_RT_PRIO, array->bitmap);
	77	+
	78	+#if defined CONFIG_SMP
	79	+ rt_rq->highest_prio.curr = MAX_RT_PRIO-1;
	80	+ rt_rq->highest_prio.next = MAX_RT_PRIO-1;
	81	+ rt_rq->overloaded = 0;
	82	+ plist_head_init(&rt_rq->pushable_tasks);
	83	+#endif /* CONFIG_SMP */
	84	+ /* We start is dequeued state, because no RT tasks are queued */
	85	+ rt_rq->rt_queued = 0;
	86	+
	87	+#ifdef CONFIG_RT_GROUP_SCHED
	88	+ rt_rq->rt_time = 0;
	89	+ rt_rq->rt_throttled = 0;
	90	+ rt_rq->rt_runtime = 0;
	91	+ raw_spin_lock_init(&rt_rq->rt_runtime_lock);
	92	+#endif
	93	+}
	94	+
	95	+#ifdef CONFIG_RT_GROUP_SCHED
	96	+
	97	+static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun);
	98	+
69	99	static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer)
70	100	{
71	101	struct rt_bandwidth *rt_b =
@@ -130,35 +160,6 @@ static void start_rt_bandwidth(struct rt_bandwidth *rt_b) · 58 unmodified lines
130	160	do_start_rt_bandwidth(rt_b);
131	161	}
132	162
133		−void init_rt_rq(struct rt_rq *rt_rq)
134		−{
135		− struct rt_prio_array *array;
136		− int i;
137		−
138		− array = &rt_rq->active;
139		− for (i = 0; i < MAX_RT_PRIO; i++) {
140		− INIT_LIST_HEAD(array->queue + i);
141		− __clear_bit(i, array->bitmap);
142		− }
143		− /* delimiter for bit-search: */
144		− __set_bit(MAX_RT_PRIO, array->bitmap);
145		−
146		−#if defined CONFIG_SMP
147		− rt_rq->highest_prio.curr = MAX_RT_PRIO-1;
148		− rt_rq->highest_prio.next = MAX_RT_PRIO-1;
149		− rt_rq->overloaded = 0;
150		− plist_head_init(&rt_rq->pushable_tasks);
151		−#endif /* CONFIG_SMP */
152		− /* We start is dequeued state, because no RT tasks are queued */
153		− rt_rq->rt_queued = 0;
154		−
155		− rt_rq->rt_time = 0;
156		− rt_rq->rt_throttled = 0;
157		− rt_rq->rt_runtime = 0;
158		− raw_spin_lock_init(&rt_rq->rt_runtime_lock);
159		−}
160		−
161		−#ifdef CONFIG_RT_GROUP_SCHED
162	163	static void destroy_rt_bandwidth(struct rt_bandwidth *rt_b)
163	164	{
164	165	hrtimer_cancel(&rt_b->rt_period_timer);
@@ -195,7 +196,6 @@ void unregister_rt_sched_group(struct task_group *tg) · 30 unmodified lines
195	196	{
196	197	if (tg->rt_se)
197	198	destroy_rt_bandwidth(&tg->rt_bandwidth);
198		−
199	199	}
200	200
201	201	void free_rt_sched_group(struct task_group *tg)
@@ -253,8 +253,7 @@ int alloc_rt_sched_group(struct task_group tg, struct task_group parent) · 51 unmodified lines
253	253	if (!tg->rt_se)
254	254	goto err;
255	255
256		− init_rt_bandwidth(&tg->rt_bandwidth,
257		− ktime_to_ns(def_rt_bandwidth.rt_period), 0);
	256	+ init_rt_bandwidth(&tg->rt_bandwidth, ktime_to_ns(global_rt_period()), 0);
258	257
259	258	for_each_possible_cpu(i) {
260	259	rt_rq = kzalloc_node(sizeof(struct rt_rq),
@@ -604,70 +603,6 @@ static inline struct rt_bandwidth sched_rt_bandwidth(struct rt_rq rt_rq) · 343 unmodified lines
604	603	return &rt_rq->tg->rt_bandwidth;
605	604	}
606	605
607		−#else /* !CONFIG_RT_GROUP_SCHED */
608		−
609		−static inline u64 sched_rt_runtime(struct rt_rq *rt_rq)
610		−{
611		− return rt_rq->rt_runtime;
612		−}
613		−
614		−static inline u64 sched_rt_period(struct rt_rq *rt_rq)
615		−{
616		− return ktime_to_ns(def_rt_bandwidth.rt_period);
617		−}
618		−
619		−typedef struct rt_rq *rt_rq_iter_t;
620		−
621		−#define for_each_rt_rq(rt_rq, iter, rq) \
622		− for ((void) iter, rt_rq = &rq->rt; rt_rq; rt_rq = NULL)
623		−
624		−#define for_each_sched_rt_entity(rt_se) \
625		− for (; rt_se; rt_se = NULL)
626		−
627		−static inline struct rt_rq group_rt_rq(struct sched_rt_entity rt_se)
628		−{
629		− return NULL;
630		−}
631		−
632		−static inline void sched_rt_rq_enqueue(struct rt_rq *rt_rq)
633		−{
634		− struct rq *rq = rq_of_rt_rq(rt_rq);
635		−
636		− if (!rt_rq->rt_nr_running)
637		− return;
638		−
639		− enqueue_top_rt_rq(rt_rq);
640		− resched_curr(rq);
641		−}
642		−
643		−static inline void sched_rt_rq_dequeue(struct rt_rq *rt_rq)
644		−{
645		− dequeue_top_rt_rq(rt_rq, rt_rq->rt_nr_running);
646		−}
647		−
648		−static inline int rt_rq_throttled(struct rt_rq *rt_rq)
649		−{
650		− return rt_rq->rt_throttled;
651		−}
652		−
653		−static inline const struct cpumask *sched_rt_period_mask(void)
654		−{
655		− return cpu_online_mask;
656		−}
657		−
658		−static inline
659		−struct rt_rq sched_rt_period_rt_rq(struct rt_bandwidth rt_b, int cpu)
660		−{
661		− return &cpu_rq(cpu)->rt;
662		−}
663		−
664		−static inline struct rt_bandwidth sched_rt_bandwidth(struct rt_rq rt_rq)
665		−{
666		− return &def_rt_bandwidth;
667		−}
668		−
669		−#endif /* CONFIG_RT_GROUP_SCHED */
670		−
671	606	bool sched_rt_bandwidth_account(struct rt_rq *rt_rq)
672	607	{
673	608	struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
@@ -859,7 +794,7 @@ static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun) · 185 unmodified lines
859	794	const struct cpumask *span;
860	795
861	796	span = sched_rt_period_mask();
862		−#ifdef CONFIG_RT_GROUP_SCHED
	797	+
863	798	/*
864	799	* FIXME: isolated CPUs should really leave the root task group,
865	800	* whether they are isolcpus or were isolated via cpusets, lest
@@ -871,7 +806,7 @@ static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun) · 5 unmodified lines
871	806	*/
872	807	if (rt_b == &root_task_group.rt_bandwidth)
873	808	span = cpu_online_mask;
874		−#endif
	809	+
875	810	for_each_cpu(i, span) {
876	811	int enqueue = 0;
877	812	struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i);
@@ -938,18 +873,6 @@ static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun) · 60 unmodified lines
938	873	return idle;
939	874	}
940	875
941		−static inline int rt_se_prio(struct sched_rt_entity *rt_se)
942		−{
943		−#ifdef CONFIG_RT_GROUP_SCHED
944		− struct rt_rq *rt_rq = group_rt_rq(rt_se);
945		−
946		− if (rt_rq)
947		− return rt_rq->highest_prio.curr;
948		−#endif
949		−
950		− return rt_task_of(rt_se)->prio;
951		−}
952		−
953	876	static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq)
954	877	{
955	878	u64 runtime = sched_rt_runtime(rt_rq);
@@ -993,6 +916,72 @@ static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq) · 37 unmodified lines
993	916	return 0;
994	917	}
995	918
	919	+#else /* !CONFIG_RT_GROUP_SCHED */
	920	+
	921	+typedef struct rt_rq *rt_rq_iter_t;
	922	+
	923	+#define for_each_rt_rq(rt_rq, iter, rq) \
	924	+ for ((void) iter, rt_rq = &rq->rt; rt_rq; rt_rq = NULL)
	925	+
	926	+#define for_each_sched_rt_entity(rt_se) \
	927	+ for (; rt_se; rt_se = NULL)
	928	+
	929	+static inline struct rt_rq group_rt_rq(struct sched_rt_entity rt_se)
	930	+{
	931	+ return NULL;
	932	+}
	933	+
	934	+static inline void sched_rt_rq_enqueue(struct rt_rq *rt_rq)
	935	+{
	936	+ struct rq *rq = rq_of_rt_rq(rt_rq);
	937	+
	938	+ if (!rt_rq->rt_nr_running)
	939	+ return;
	940	+
	941	+ enqueue_top_rt_rq(rt_rq);
	942	+ resched_curr(rq);
	943	+}
	944	+
	945	+static inline void sched_rt_rq_dequeue(struct rt_rq *rt_rq)
	946	+{
	947	+ dequeue_top_rt_rq(rt_rq, rt_rq->rt_nr_running);
	948	+}
	949	+
	950	+static inline int rt_rq_throttled(struct rt_rq *rt_rq)
	951	+{
	952	+ return false;
	953	+}
	954	+
	955	+static inline const struct cpumask *sched_rt_period_mask(void)
	956	+{
	957	+ return cpu_online_mask;
	958	+}
	959	+
	960	+static inline
	961	+struct rt_rq sched_rt_period_rt_rq(struct rt_bandwidth rt_b, int cpu)
	962	+{
	963	+ return &cpu_rq(cpu)->rt;
	964	+}
	965	+
	966	+#ifdef CONFIG_SMP
	967	+static void __enable_runtime(struct rq *rq) { }
	968	+static void __disable_runtime(struct rq *rq) { }
	969	+#endif
	970	+
	971	+#endif /* CONFIG_RT_GROUP_SCHED */
	972	+
	973	+static inline int rt_se_prio(struct sched_rt_entity *rt_se)
	974	+{
	975	+#ifdef CONFIG_RT_GROUP_SCHED
	976	+ struct rt_rq *rt_rq = group_rt_rq(rt_se);
	977	+
	978	+ if (rt_rq)
	979	+ return rt_rq->highest_prio.curr;
	980	+#endif
	981	+
	982	+ return rt_task_of(rt_se)->prio;
	983	+}
	984	+
996	985	/*
997	986	* Update the current task's runtime statistics. Skip current tasks that
998	987	* are not in our scheduling class.
@@ -1000,7 +989,6 @@ static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq) · 1 unmodified line
1000	989	static void update_curr_rt(struct rq *rq)
1001	990	{
1002	991	struct task_struct *curr = rq->curr;
1003		− struct sched_rt_entity *rt_se = &curr->rt;
1004	992	s64 delta_exec;
1005	993
1006	994	if (curr->sched_class != &rt_sched_class)
@@ -1010,6 +998,9 @@ static void update_curr_rt(struct rq *rq) · 3 unmodified lines
1010	998	if (unlikely(delta_exec <= 0))
1011	999	return;
1012	1000
	1001	+#ifdef CONFIG_RT_GROUP_SCHED
	1002	+ struct sched_rt_entity *rt_se = &curr->rt;
	1003	+
1013	1004	if (!rt_bandwidth_enabled())
1014	1005	return;
1015	1006
@@ -1028,6 +1019,7 @@ static void update_curr_rt(struct rq *rq) · 12 unmodified lines
1028	1019	do_start_rt_bandwidth(sched_rt_bandwidth(rt_rq));
1029	1020	}
1030	1021	}
	1022	+#endif
1031	1023	}
1032	1024
1033	1025	static void
@@ -1184,7 +1176,6 @@ dec_rt_group(struct sched_rt_entity rt_se, struct rt_rq rt_rq) · 150 unmodified lines
1184	1176	static void
1185	1177	inc_rt_group(struct sched_rt_entity rt_se, struct rt_rq rt_rq)
1186	1178	{
1187		− start_rt_bandwidth(&def_rt_bandwidth);
1188	1179	}
1189	1180
1190	1181	static inline
@@ -1492,7 +1483,7 @@ enqueue_task_rt(struct rq rq, struct task_struct p, int flags) · 301 unmodified lines
1492	1483	enqueue_pushable_task(rq, p);
1493	1484	}
1494	1485
1495		−static void dequeue_task_rt(struct rq rq, struct task_struct p, int flags)
	1486	+static bool dequeue_task_rt(struct rq rq, struct task_struct p, int flags)
1496	1487	{
1497	1488	struct sched_rt_entity *rt_se = &p->rt;
1498	1489
@@ -1500,6 +1491,8 @@ static void dequeue_task_rt(struct rq rq, struct task_struct p, int flags) · 1 unmodified line
1500	1491	dequeue_rt_entity(rt_se, flags);
1501	1492
1502	1493	dequeue_pushable_task(rq, p);
	1494	+
	1495	+ return true;
1503	1496	}
1504	1497
1505	1498	/*
@@ -1755,17 +1748,7 @@ static struct task_struct pick_task_rt(struct rq rq) · 249 unmodified lines
1755	1748	return p;
1756	1749	}
1757	1750
1758		−static struct task_struct pick_next_task_rt(struct rq rq)
1759		−{
1760		− struct task_struct *p = pick_task_rt(rq);
1761		−
1762		− if (p)
1763		− set_next_task_rt(rq, p, true);
1764		−
1765		− return p;
1766		−}
1767		−
1768		−static void put_prev_task_rt(struct rq rq, struct task_struct p)
	1751	+static void put_prev_task_rt(struct rq rq, struct task_struct p, struct task_struct *next)
1769	1752	{
1770	1753	struct sched_rt_entity *rt_se = &p->rt;
1771	1754	struct rt_rq *rt_rq = &rq->rt;
@@ -2652,13 +2635,12 @@ DEFINE_SCHED_CLASS(rt) = { · 880 unmodified lines
2652	2635
2653	2636	.wakeup_preempt = wakeup_preempt_rt,
2654	2637
2655		− .pick_next_task = pick_next_task_rt,
	2638	+ .pick_task = pick_task_rt,
2656	2639	.put_prev_task = put_prev_task_rt,
2657	2640	.set_next_task = set_next_task_rt,
2658	2641
2659	2642	#ifdef CONFIG_SMP
2660	2643	.balance = balance_rt,
2661		− .pick_task = pick_task_rt,
2662	2644	.select_task_rq = select_task_rq_rt,
2663	2645	.set_cpus_allowed = set_cpus_allowed_common,
2664	2646	.rq_online = rq_online_rt,
@@ -2912,19 +2894,6 @@ int sched_rt_can_attach(struct task_group tg, struct task_struct tsk) · 247 unmodified lines
2912	2894	#ifdef CONFIG_SYSCTL
2913	2895	static int sched_rt_global_constraints(void)
2914	2896	{
2915		− unsigned long flags;
2916		− int i;
2917		−
2918		− raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
2919		− for_each_possible_cpu(i) {
2920		− struct rt_rq *rt_rq = &cpu_rq(i)->rt;
2921		−
2922		− raw_spin_lock(&rt_rq->rt_runtime_lock);
2923		− rt_rq->rt_runtime = global_rt_runtime();
2924		− raw_spin_unlock(&rt_rq->rt_runtime_lock);
2925		− }
2926		− raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);
2927		−
2928	2897	return 0;
2929	2898	}
2930	2899	#endif /* CONFIG_SYSCTL */
@@ -2944,12 +2913,6 @@ static int sched_rt_global_validate(void) · 13 unmodified lines
2944	2913
2945	2914	static void sched_rt_do_global(void)
2946	2915	{
2947		− unsigned long flags;
2948		−
2949		− raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
2950		− def_rt_bandwidth.rt_runtime = global_rt_runtime();
2951		− def_rt_bandwidth.rt_period = ns_to_ktime(global_rt_period());
2952		− raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);
2953	2916	}
2954	2917
2955	2918	static int sched_rt_handler(const struct ctl_table table, int write, void buffer,

kernel/sched/sched.h+78−23

@@ -68,6 +68,7 @@ · 67 unmodified lines
68	68	#include <linux/wait_api.h>
69	69	#include <linux/wait_bit.h>
70	70	#include <linux/workqueue_api.h>
	71	+#include <linux/delayacct.h>
71	72
72	73	#include <trace/events/power.h>
73	74	#include <trace/events/sched.h>
@@ -335,7 +336,7 @@ extern bool __checkparam_dl(const struct sched_attr *attr); · 261 unmodified lines
335	336	extern bool dl_param_changed(struct task_struct p, const struct sched_attr attr);
336	337	extern int dl_cpuset_cpumask_can_shrink(const struct cpumask cur, const struct cpumask trial);
337	338	extern int dl_bw_check_overflow(int cpu);
338		−
	339	+extern s64 dl_scaled_delta_exec(struct rq rq, struct sched_dl_entity dl_se, s64 delta_exec);
339	340	/*
340	341	* SCHED_DEADLINE supports servers (nested scheduling) with the following
341	342	* interface:
@@ -361,7 +362,14 @@ extern void dl_server_start(struct sched_dl_entity *dl_se); · 19 unmodified lines
361	362	extern void dl_server_stop(struct sched_dl_entity *dl_se);
362	363	extern void dl_server_init(struct sched_dl_entity dl_se, struct rq rq,
363	364	dl_server_has_tasks_f has_tasks,
364		− dl_server_pick_f pick);
	365	+ dl_server_pick_f pick_task);
	366	+
	367	+extern void dl_server_update_idle_time(struct rq *rq,
	368	+ struct task_struct *p);
	369	+extern void fair_server_init(struct rq *rq);
	370	+extern void __dl_server_attach_root(struct sched_dl_entity dl_se, struct rq rq);
	371	+extern int dl_server_apply_params(struct sched_dl_entity *dl_se,
	372	+ u64 runtime, u64 period, bool init);
365	373
366	374	#ifdef CONFIG_CGROUP_SCHED
367	375
@@ -599,17 +607,12 @@ struct cfs_rq { · 231 unmodified lines
599	607	s64 avg_vruntime;
600	608	u64 avg_load;
601	609
602		− u64 exec_clock;
603	610	u64 min_vruntime;
604	611	#ifdef CONFIG_SCHED_CORE
605	612	unsigned int forceidle_seq;
606	613	u64 min_vruntime_fi;
607	614	#endif
608	615
609		−#ifndef CONFIG_64BIT
610		− u64 min_vruntime_copy;
611		−#endif
612		−
613	616	struct rb_root_cached tasks_timeline;
614	617
615	618	/*
@@ -619,10 +622,6 @@ struct cfs_rq { · 3 unmodified lines
619	622	struct sched_entity *curr;
620	623	struct sched_entity *next;
621	624
622		−#ifdef CONFIG_SCHED_DEBUG
623		− unsigned int nr_spread_over;
624		−#endif
625		−
626	625	#ifdef CONFIG_SMP
627	626	/*
628	627	* CFS load tracking
@@ -726,13 +725,13 @@ struct rt_rq { · 97 unmodified lines
726	725	#endif /* CONFIG_SMP */
727	726	int rt_queued;
728	727
	728	+#ifdef CONFIG_RT_GROUP_SCHED
729	729	int rt_throttled;
730	730	u64 rt_time;
731	731	u64 rt_runtime;
732	732	/* Nests inside the rq lock: */
733	733	raw_spinlock_t rt_runtime_lock;
734	734
735		−#ifdef CONFIG_RT_GROUP_SCHED
736	735	unsigned int rt_nr_boosted;
737	736
738	737	struct rq *rq;
@@ -820,6 +819,9 @@ static inline void se_update_runnable(struct sched_entity *se) · 81 unmodified lines
820	819
821	820	static inline long se_runnable(struct sched_entity *se)
822	821	{
	822	+ if (se->sched_delayed)
	823	+ return false;
	824	+
823	825	if (entity_is_task(se))
824	826	return !!se->on_rq;
825	827	else
@@ -834,6 +836,9 @@ static inline void se_update_runnable(struct sched_entity *se) { } · 8 unmodified lines
834	836
835	837	static inline long se_runnable(struct sched_entity *se)
836	838	{
	839	+ if (se->sched_delayed)
	840	+ return false;
	841	+
837	842	return !!se->on_rq;
838	843	}
839	844
@@ -1044,6 +1049,8 @@ struct rq { · 204 unmodified lines
1044	1049	struct rt_rq rt;
1045	1050	struct dl_rq dl;
1046	1051
	1052	+ struct sched_dl_entity fair_server;
	1053	+
1047	1054	#ifdef CONFIG_FAIR_GROUP_SCHED
1048	1055	/* list of leaf cfs_rq on this CPU: */
1049	1056	struct list_head leaf_cfs_rq_list;
@@ -1059,6 +1066,7 @@ struct rq { · 9 unmodified lines
1059	1066	unsigned int nr_uninterruptible;
1060	1067
1061	1068	struct task_struct __rcu *curr;
	1069	+ struct sched_dl_entity *dl_server;
1062	1070	struct task_struct *idle;
1063	1071	struct task_struct *stop;
1064	1072	unsigned long next_balance;
@@ -1158,7 +1166,6 @@ struct rq { · 93 unmodified lines
1158	1166	/* latency stats */
1159	1167	struct sched_info rq_sched_info;
1160	1168	unsigned long long rq_cpu_time;
1161		− /* could above be rq->cfs_rq.exec_clock + rq->rt_rq.rt_runtime ? */
1162	1169
1163	1170	/* sys_sched_yield() stats */
1164	1171	unsigned int yld_count;
@@ -1187,6 +1194,7 @@ struct rq { · 22 unmodified lines
1187	1194	/* per rq */
1188	1195	struct rq *core;
1189	1196	struct task_struct *core_pick;
	1197	+ struct sched_dl_entity *core_dl_server;
1190	1198	unsigned int core_enabled;
1191	1199	unsigned int core_sched_seq;
1192	1200	struct rb_root core_tree;
@@ -2247,11 +2255,13 @@ extern const u32 sched_prio_to_wmult[40]; · 1054 unmodified lines
2247	2255	*
2248	2256	*/
2249	2257
2250		−#define DEQUEUE_SLEEP 0x01
	2258	+#define DEQUEUE_SLEEP 0x01 /* Matches ENQUEUE_WAKEUP */
2251	2259	#define DEQUEUE_SAVE 0x02 /* Matches ENQUEUE_RESTORE */
2252	2260	#define DEQUEUE_MOVE 0x04 /* Matches ENQUEUE_MOVE */
2253	2261	#define DEQUEUE_NOCLOCK 0x08 /* Matches ENQUEUE_NOCLOCK */
	2262	+#define DEQUEUE_SPECIAL 0x10
2254	2263	#define DEQUEUE_MIGRATING 0x100 /* Matches ENQUEUE_MIGRATING */
	2264	+#define DEQUEUE_DELAYED 0x200 /* Matches ENQUEUE_DELAYED */
2255	2265
2256	2266	#define ENQUEUE_WAKEUP 0x01
2257	2267	#define ENQUEUE_RESTORE 0x02
@@ -2267,6 +2277,7 @@ extern const u32 sched_prio_to_wmult[40]; · 9 unmodified lines
2267	2277	#endif
2268	2278	#define ENQUEUE_INITIAL 0x80
2269	2279	#define ENQUEUE_MIGRATING 0x100
	2280	+#define ENQUEUE_DELAYED 0x200
2270	2281
2271	2282	#define RETRY_TASK ((void *)-1UL)
2272	2283
@@ -2285,23 +2296,31 @@ struct sched_class { · 12 unmodified lines
2285	2296	#endif
2286	2297
2287	2298	void (enqueue_task) (struct rq rq, struct task_struct *p, int flags);
2288		− void (dequeue_task) (struct rq rq, struct task_struct *p, int flags);
	2299	+ bool (dequeue_task) (struct rq rq, struct task_struct *p, int flags);
2289	2300	void (yield_task) (struct rq rq);
2290	2301	bool (yield_to_task)(struct rq rq, struct task_struct *p);
2291	2302
2292	2303	void (wakeup_preempt)(struct rq rq, struct task_struct *p, int flags);
2293	2304
2294		− struct task_struct (pick_next_task)(struct rq *rq);
	2305	+ struct task_struct (pick_task)(struct rq *rq);
	2306	+ /*
	2307	+ * Optional! When implemented pick_next_task() should be equivalent to:
	2308	+ *
	2309	+ * next = pick_task();
	2310	+ * if (next) {
	2311	+ * put_prev_task(prev);
	2312	+ * set_next_task_first(next);
	2313	+ * }
	2314	+ */
	2315	+ struct task_struct (pick_next_task)(struct rq rq, struct task_struct prev);
2295	2316
2296		− void (put_prev_task)(struct rq rq, struct task_struct *p);
	2317	+ void (put_prev_task)(struct rq rq, struct task_struct p, struct task_struct next);
2297	2318	void (set_next_task)(struct rq rq, struct task_struct *p, bool first);
2298	2319
2299	2320	#ifdef CONFIG_SMP
2300	2321	int (balance)(struct rq rq, struct task_struct prev, struct rq_flags rf);
2301	2322	int (select_task_rq)(struct task_struct p, int task_cpu, int flags);
2302	2323
2303		− struct task_struct * (pick_task)(struct rq rq);
2304		−
2305	2324	void (migrate_task_rq)(struct task_struct p, int new_cpu);
2306	2325
2307	2326	void (task_woken)(struct rq this_rq, struct task_struct *task);
@@ -2345,7 +2364,7 @@ struct sched_class { · 37 unmodified lines
2345	2364	static inline void put_prev_task(struct rq rq, struct task_struct prev)
2346	2365	{
2347	2366	WARN_ON_ONCE(rq->curr != prev);
2348		− prev->sched_class->put_prev_task(rq, prev);
	2367	+ prev->sched_class->put_prev_task(rq, prev, NULL);
2349	2368	}
2350	2369
2351	2370	static inline void set_next_task(struct rq rq, struct task_struct next)
@@ -2353,6 +2372,30 @@ static inline void set_next_task(struct rq rq, struct task_struct next) · 1 unmodified line
2353	2372	next->sched_class->set_next_task(rq, next, false);
2354	2373	}
2355	2374
	2375	+static inline void
	2376	+__put_prev_set_next_dl_server(struct rq *rq,
	2377	+ struct task_struct *prev,
	2378	+ struct task_struct *next)
	2379	+{
	2380	+ prev->dl_server = NULL;
	2381	+ next->dl_server = rq->dl_server;
	2382	+ rq->dl_server = NULL;
	2383	+}
	2384	+
	2385	+static inline void put_prev_set_next_task(struct rq *rq,
	2386	+ struct task_struct *prev,
	2387	+ struct task_struct *next)
	2388	+{
	2389	+ WARN_ON_ONCE(rq->curr != prev);
	2390	+
	2391	+ __put_prev_set_next_dl_server(rq, prev, next);
	2392	+
	2393	+ if (next == prev)
	2394	+ return;
	2395	+
	2396	+ prev->sched_class->put_prev_task(rq, prev, next);
	2397	+ next->sched_class->set_next_task(rq, next, true);
	2398	+}
2356	2399
2357	2400	/*
2358	2401	* Helper to define a sched_class instance; each one is placed in a separate
@@ -2408,7 +2451,7 @@ static inline bool sched_fair_runnable(struct rq *rq) · 49 unmodified lines
2408	2451	}
2409	2452
2410	2453	extern struct task_struct pick_next_task_fair(struct rq rq, struct task_struct prev, struct rq_flags rf);
2411		−extern struct task_struct pick_next_task_idle(struct rq rq);
	2454	+extern struct task_struct pick_task_idle(struct rq rq);
2412	2455
2413	2456	#define SCA_CHECK 0x01
2414	2457	#define SCA_MIGRATE_DISABLE 0x02
@@ -2515,7 +2558,6 @@ extern void reweight_task(struct task_struct p, const struct load_weight lw); · 100 unmodified lines
2515	2558	extern void resched_curr(struct rq *rq);
2516	2559	extern void resched_cpu(int cpu);
2517	2560
2518		−extern struct rt_bandwidth def_rt_bandwidth;
2519	2561	extern void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime);
2520	2562	extern bool sched_rt_bandwidth_account(struct rt_rq *rt_rq);
2521	2563
@@ -2586,6 +2628,19 @@ static inline void sub_nr_running(struct rq *rq, unsigned count) · 64 unmodified lines
2586	2628	sched_update_tick_dependency(rq);
2587	2629	}
2588	2630
	2631	+static inline void __block_task(struct rq rq, struct task_struct p)
	2632	+{
	2633	+ WRITE_ONCE(p->on_rq, 0);
	2634	+ ASSERT_EXCLUSIVE_WRITER(p->on_rq);
	2635	+ if (p->sched_contributes_to_load)
	2636	+ rq->nr_uninterruptible++;
	2637	+
	2638	+ if (p->in_iowait) {
	2639	+ atomic_inc(&rq->nr_iowait);
	2640	+ delayacct_blkio_start();
	2641	+ }
	2642	+}
	2643	+
2589	2644	extern void activate_task(struct rq rq, struct task_struct p, int flags);
2590	2645	extern void deactivate_task(struct rq rq, struct task_struct p, int flags);
2591	2646
@@ -3607,7 +3662,7 @@ extern int __sched_setaffinity(struct task_struct p, struct affinity_context c · 1015 unmodified lines
3607	3662	extern void __setscheduler_prio(struct task_struct *p, int prio);
3608	3663	extern void set_load_weight(struct task_struct *p, bool update_load);
3609	3664	extern void enqueue_task(struct rq rq, struct task_struct p, int flags);
3610		−extern void dequeue_task(struct rq rq, struct task_struct p, int flags);
	3665	+extern bool dequeue_task(struct rq rq, struct task_struct p, int flags);
3611	3666
3612	3667	extern void check_class_changed(struct rq rq, struct task_struct p,
3613	3668	const struct sched_class *prev_class,

kernel/sched/stop_task.c+4−14

@@ -41,26 +41,17 @@ static struct task_struct pick_task_stop(struct rq rq) · 40 unmodified lines
41	41	return rq->stop;
42	42	}
43	43
44		−static struct task_struct pick_next_task_stop(struct rq rq)
45		−{
46		− struct task_struct *p = pick_task_stop(rq);
47		−
48		− if (p)
49		− set_next_task_stop(rq, p, true);
50		−
51		− return p;
52		−}
53		−
54	44	static void
55	45	enqueue_task_stop(struct rq rq, struct task_struct p, int flags)
56	46	{
57	47	add_nr_running(rq, 1);
58	48	}
59	49
60		−static void
	50	+static bool
61	51	dequeue_task_stop(struct rq rq, struct task_struct p, int flags)
62	52	{
63	53	sub_nr_running(rq, 1);
	54	+ return true;
64	55	}
65	56
66	57	static void yield_task_stop(struct rq *rq)
@@ -68,7 +59,7 @@ static void yield_task_stop(struct rq *rq) · 1 unmodified line
68	59	BUG(); /* the stop task should never yield, its pointless. */
69	60	}
70	61
71		−static void put_prev_task_stop(struct rq rq, struct task_struct prev)
	62	+static void put_prev_task_stop(struct rq rq, struct task_struct prev, struct task_struct *next)
72	63	{
73	64	update_curr_common(rq);
74	65	}
@@ -111,13 +102,12 @@ DEFINE_SCHED_CLASS(stop) = { · 36 unmodified lines
111	102
112	103	.wakeup_preempt = wakeup_preempt_stop,
113	104
114		− .pick_next_task = pick_next_task_stop,
	105	+ .pick_task = pick_task_stop,
115	106	.put_prev_task = put_prev_task_stop,
116	107	.set_next_task = set_next_task_stop,
117	108
118	109	#ifdef CONFIG_SMP
119	110	.balance = balance_stop,
120		− .pick_task = pick_task_stop,
121	111	.select_task_rq = select_task_rq_stop,
122	112	.set_cpus_allowed = set_cpus_allowed_common,
123	113	#endif

kernel/sched/syscalls.c+25−109

@@ -57,7 +57,7 @@ static int effective_prio(struct task_struct *p) · 56 unmodified lines
57	57	* keep the priority unchanged. Otherwise, update priority
58	58	* to the normal priority:
59	59	*/
60		− if (!rt_prio(p->prio))
	60	+ if (!rt_or_dl_prio(p->prio))
61	61	return p->normal_prio;
62	62	return p->prio;
63	63	}
@@ -258,107 +258,6 @@ int sched_core_idle_cpu(int cpu) · 194 unmodified lines
258	258
259	259	#endif
260	260
261		−#ifdef CONFIG_SMP
262		−/*
263		− * This function computes an effective utilization for the given CPU, to be
264		− * used for frequency selection given the linear relation: f = u * f_max.
265		− *
266		− * The scheduler tracks the following metrics:
267		− *
268		− * cpu_util_{cfs,rt,dl,irq}()
269		− * cpu_bw_dl()
270		− *
271		− * Where the cfs,rt and dl util numbers are tracked with the same metric and
272		− * synchronized windows and are thus directly comparable.
273		− *
274		− * The cfs,rt,dl utilization are the running times measured with rq->clock_task
275		− * which excludes things like IRQ and steal-time. These latter are then accrued
276		− * in the IRQ utilization.
277		− *
278		− * The DL bandwidth number OTOH is not a measured metric but a value computed
279		− * based on the task model parameters and gives the minimal utilization
280		− * required to meet deadlines.
281		− */
282		−unsigned long effective_cpu_util(int cpu, unsigned long util_cfs,
283		− unsigned long *min,
284		− unsigned long *max)
285		−{
286		− unsigned long util, irq, scale;
287		− struct rq *rq = cpu_rq(cpu);
288		−
289		− scale = arch_scale_cpu_capacity(cpu);
290		−
291		− /*
292		− * Early check to see if IRQ/steal time saturates the CPU, can be
293		− * because of inaccuracies in how we track these -- see
294		− * update_irq_load_avg().
295		− */
296		− irq = cpu_util_irq(rq);
297		− if (unlikely(irq >= scale)) {
298		− if (min)
299		− *min = scale;
300		− if (max)
301		− *max = scale;
302		− return scale;
303		− }
304		−
305		− if (min) {
306		− /*
307		− * The minimum utilization returns the highest level between:
308		− * - the computed DL bandwidth needed with the IRQ pressure which
309		− * steals time to the deadline task.
310		− * - The minimum performance requirement for CFS and/or RT.
311		− */
312		− *min = max(irq + cpu_bw_dl(rq), uclamp_rq_get(rq, UCLAMP_MIN));
313		−
314		− /*
315		− * When an RT task is runnable and uclamp is not used, we must
316		− * ensure that the task will run at maximum compute capacity.
317		− */
318		− if (!uclamp_is_used() && rt_rq_is_runnable(&rq->rt))
319		− min = max(min, scale);
320		− }
321		−
322		− /*
323		− * Because the time spend on RT/DL tasks is visible as 'lost' time to
324		− * CFS tasks and we use the same metric to track the effective
325		− * utilization (PELT windows are synchronized) we can directly add them
326		− * to obtain the CPU's actual utilization.
327		− */
328		− util = util_cfs + cpu_util_rt(rq);
329		− util += cpu_util_dl(rq);
330		−
331		− /*
332		− * The maximum hint is a soft bandwidth requirement, which can be lower
333		− * than the actual utilization because of uclamp_max requirements.
334		− */
335		− if (max)
336		− *max = min(scale, uclamp_rq_get(rq, UCLAMP_MAX));
337		−
338		− if (util >= scale)
339		− return scale;
340		−
341		− /*
342		− * There is still idle time; further improve the number by using the
343		− * IRQ metric. Because IRQ/steal time is hidden from the task clock we
344		− * need to scale the task numbers:
345		− *
346		− * max - irq
347		− * U' = irq + --------- * U
348		− * max
349		− */
350		− util = scale_irq_capacity(util, irq, scale);
351		− util += irq;
352		−
353		− return min(scale, util);
354		−}
355		−
356		−unsigned long sched_cpu_util(int cpu)
357		−{
358		− return effective_cpu_util(cpu, cpu_util_cfs(cpu), NULL, NULL);
359		−}
360		−#endif /* CONFIG_SMP */
361		−
362	261	/**
363	262	* find_process_by_pid - find a process with a matching PID value.
364	263	* @pid: the pid in question.
@@ -401,13 +300,23 @@ static void __setscheduler_params(struct task_struct *p, · 36 unmodified lines
401	300
402	301	p->policy = policy;
403	302
404		− if (dl_policy(policy))
	303	+ if (dl_policy(policy)) {
405	304	__setparam_dl(p, attr);
406		− else if (fair_policy(policy))
	305	+ } else if (fair_policy(policy)) {
407	306	p->static_prio = NICE_TO_PRIO(attr->sched_nice);
	307	+ if (attr->sched_runtime) {
	308	+ p->se.custom_slice = 1;
	309	+ p->se.slice = clamp_t(u64, attr->sched_runtime,
	310	+ NSEC_PER_MSEC/10, /* HZ=1000 * 10 */
	311	+ NSEC_PER_MSEC100); / HZ=100 / 10 */
	312	+ } else {
	313	+ p->se.custom_slice = 0;
	314	+ p->se.slice = sysctl_sched_base_slice;
	315	+ }
	316	+ }
408	317
409	318	/* rt-policy tasks do not have a timerslack */
410		− if (task_is_realtime(p)) {
	319	+ if (rt_or_dl_task_policy(p)) {
411	320	p->timer_slack_ns = 0;
412	321	} else if (p->timer_slack_ns == 0) {
413	322	/* when switching back to non-rt policy, restore timerslack */
@@ -708,7 +617,9 @@ recheck: · 294 unmodified lines
708	617	* but store a possible modification of reset_on_fork.
709	618	*/
710	619	if (unlikely(policy == p->policy)) {
711		− if (fair_policy(policy) && attr->sched_nice != task_nice(p))
	620	+ if (fair_policy(policy) &&
	621	+ (attr->sched_nice != task_nice(p) \|\|
	622	+ (attr->sched_runtime != p->se.slice)))
712	623	goto change;
713	624	if (rt_policy(policy) && attr->sched_priority != p->rt_priority)
714	625	goto change;
@@ -854,6 +765,9 @@ static int _sched_setscheduler(struct task_struct *p, int policy, · 139 unmodified lines
854	765	.sched_nice = PRIO_TO_NICE(p->static_prio),
855	766	};
856	767
	768	+ if (p->se.custom_slice)
	769	+ attr.sched_runtime = p->se.slice;
	770	+
857	771	/* Fixup the legacy SCHED_RESET_ON_FORK hack. */
858	772	if ((policy != SETPARAM_POLICY) && (policy & SCHED_RESET_ON_FORK)) {
859	773	attr.sched_flags \|= SCHED_FLAG_RESET_ON_FORK;
@@ -1020,12 +934,14 @@ err_size: · 160 unmodified lines
1020	934
1021	935	static void get_params(struct task_struct p, struct sched_attr attr)
1022	936	{
1023		− if (task_has_dl_policy(p))
	937	+ if (task_has_dl_policy(p)) {
1024	938	__getparam_dl(p, attr);
1025		− else if (task_has_rt_policy(p))
	939	+ } else if (task_has_rt_policy(p)) {
1026	940	attr->sched_priority = p->rt_priority;
1027		− else
	941	+ } else {
1028	942	attr->sched_nice = task_nice(p);
	943	+ attr->sched_runtime = p->se.slice;
	944	+ }
1029	945	}
1030	946
1031	947	/**

kernel/sched/topology.c+8−0

@@ -516,6 +516,14 @@ void rq_attach_root(struct rq rq, struct root_domain rd) · 515 unmodified lines
516	516	if (cpumask_test_cpu(rq->cpu, cpu_active_mask))
517	517	set_rq_online(rq);
518	518
	519	+ /*
	520	+ * Because the rq is not a task, dl_add_task_root_domain() did not
	521	+ * move the fair server bw to the rd if it already started.
	522	+ * Add it now.
	523	+ */
	524	+ if (rq->fair_server.dl_server)
	525	+ __dl_server_attach_root(&rq->fair_server, rq);
	526	+
519	527	rq_unlock_irqrestore(rq, &rf);
520	528
521	529	if (old_rd)

kernel/sys.c+1−1

@@ -2557,7 +2557,7 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, · 2556 unmodified lines
2557	2557	error = current->timer_slack_ns;
2558	2558	break;
2559	2559	case PR_SET_TIMERSLACK:
2560		− if (task_is_realtime(current))
	2560	+ if (rt_or_dl_task_policy(current))
2561	2561	break;
2562	2562	if (arg2 <= 0)
2563	2563	current->timer_slack_ns =

kernel/time/hrtimer.c+1−1

@@ -1977,7 +1977,7 @@ static void __hrtimer_init_sleeper(struct hrtimer_sleeper *sl, · 1976 unmodified lines
1977	1977	* expiry.
1978	1978	*/
1979	1979	if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
1980		− if (task_is_realtime(current) && !(mode & HRTIMER_MODE_SOFT))
	1980	+ if (rt_or_dl_task_policy(current) && !(mode & HRTIMER_MODE_SOFT))
1981	1981	mode \|= HRTIMER_MODE_HARD;
1982	1982	}
1983	1983

kernel/trace/trace_sched_wakeup.c+1−1

@@ -547,7 +547,7 @@ probe_wakeup(void ignore, struct task_struct p) · 546 unmodified lines
547	547	* - wakeup_dl handles tasks belonging to sched_dl class only.
548	548	*/
549	549	if (tracing_dl \|\| (wakeup_dl && !dl_task(p)) \|\|
550		− (wakeup_rt && !dl_task(p) && !rt_task(p)) \|\|
	550	+ (wakeup_rt && !rt_or_dl_task(p)) \|\|
551	551	(!dl_task(p) && (p->prio >= wakeup_prio \|\| p->prio >= current->prio)))
552	552	return;
553	553

mm/page-writeback.c+2−2

@@ -418,7 +418,7 @@ static void domain_dirty_limits(struct dirty_throttle_control *dtc) · 417 unmodified lines
418	418	bg_thresh = (bg_ratio * available_memory) / PAGE_SIZE;
419	419
420	420	tsk = current;
421		− if (rt_task(tsk)) {
	421	+ if (rt_or_dl_task(tsk)) {
422	422	bg_thresh += bg_thresh / 4 + global_wb_domain.dirty_limit / 32;
423	423	thresh += thresh / 4 + global_wb_domain.dirty_limit / 32;
424	424	}
@@ -477,7 +477,7 @@ static unsigned long node_dirty_limit(struct pglist_data *pgdat) · 52 unmodified lines
477	477	else
478	478	dirty = vm_dirty_ratio * node_memory / 100;
479	479
480		− if (rt_task(tsk))
	480	+ if (rt_or_dl_task(tsk))
481	481	dirty += dirty / 4;
482	482
483	483	/*

mm/page_alloc.c+1−1

@@ -4004,7 +4004,7 @@ gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order) · 4003 unmodified lines
4004	4004	*/
4005	4005	if (alloc_flags & ALLOC_MIN_RESERVE)
4006	4006	alloc_flags &= ~ALLOC_CPUSET;
4007		− } else if (unlikely(rt_task(current)) && in_task())
	4007	+ } else if (unlikely(rt_or_dl_task(current)) && in_task())
4008	4008	alloc_flags \|= ALLOC_MIN_RESERVE;
4009	4009
4010	4010	alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, alloc_flags);