Merge tag 'sched-core-2024-09-19' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull scheduler updates from Ingo Molnar: - Implement the SCHED_DEADLINE server infrastructure - Daniel Bristot de Oliveira's last major contribution to the kernel: "SCHED_DEADLINE servers can help fixing starvation issues of low priority tasks (e.g., SCHED_OTHER) when higher priority tasks monopolize CPU cycles. Today we have RT Throttling; DEADLINE servers should be able to replace and improve that." (Daniel Bristot de Oliveira, Peter Zijlstra, Joel Fernandes, Youssef Esmat, Huang Shijie) - Preparatory changes for sched_ext integration: - Use set_next_task(.first) where required - Fix up set_next_task() implementations - Clean up DL server vs. core sched - Split up put_prev_task_balance() - Rework pick_next_task() - Combine the last put_prev_task() and the first set_next_task() - Rework dl_server - Add put_prev_task(.next) (Peter Zijlstra, with a fix by Tejun Heo) - Complete the EEVDF transition and refine EEVDF scheduling: - Implement delayed dequeue - Allow shorter slices to wakeup-preempt - Use sched_attr::sched_runtime to set request/slice suggestion - Document the new feature flags - Remove unused and duplicate-functionality fields - Simplify & unify pick_next_task_fair() - Misc debuggability enhancements (Peter Zijlstra, with fixes/cleanups by Dietmar Eggemann, Valentin Schneider and Chuyi Zhou) - Initialize the vruntime of a new task when it is first enqueued, resulting in significant decrease in latency of newly woken tasks (Zhang Qiao) - Introduce SM_IDLE and an idle re-entry fast-path in __schedule() (K Prateek Nayak, Peter Zijlstra) - Clean up and clarify the usage of Clean up usage of rt_task() (Qais Yousef) - Preempt SCHED_IDLE entities in strict cgroup hierarchies (Tianchen Ding) - Clarify the documentation of time units for deadline scheduler parameters (Christian Loehle) - Remove the HZ_BW chicken-bit feature flag introduced a year ago, the original change seems to be working fine (Phil Auld) - Misc fixes and cleanups (Chen Yu, Dan Carpenter, Huang Shijie, Peilin He, Qais Yousefm and Vincent Guittot) * tag 'sched-core-2024-09-19' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (64 commits) sched/cpufreq: Use NSEC_PER_MSEC for deadline task cpufreq/cppc: Use NSEC_PER_MSEC for deadline task sched/deadline: Clarify nanoseconds in uapi sched/deadline: Convert schedtool example to chrt sched/debug: Fix the runnable tasks output sched: Fix sched_delayed vs sched_core kernel/sched: Fix util_est accounting for DELAY_DEQUEUE kthread: Fix task state in kthread worker if being frozen sched/pelt: Use rq_clock_task() for hw_pressure sched/fair: Move effective_cpu_util() and effective_cpu_util() in fair.c sched/core: Introduce SM_IDLE and an idle re-entry fast-path in __schedule() sched: Add put_prev_task(.next) sched: Rework dl_server sched: Combine the last put_prev_task() and the first set_next_task() sched: Rework pick_next_task() sched: Split up put_prev_task_balance() sched: Clean up DL server vs core sched sched: Fixup set_next_task() implementations sched: Use set_next_task(.first) where required sched/fair: Properly deactivate sched_delayed task upon class change ...

Linus Torvalds committed 2 years ago

Commit 2004cef2 parents 509d2cd bc9057d32 files · +1695 −747Browse files

Unified · Split

32 files

Documentation/scheduler/sched-deadline.rst +6 −8
drivers/cpufreq/cppc_cpufreq.c +3 −3
fs/bcachefs/six.c +1 −1
fs/proc/base.c +1 −1
include/linux/ioprio.h +1 −1
include/linux/sched.h +24 −4
include/linux/sched/deadline.h +7 −7
include/linux/sched/prio.h +1 −0
include/linux/sched/rt.h +27 −6
include/uapi/linux/sched/types.h +3 −3
kernel/freezer.c +1 −1
kernel/kthread.c +9 −1
kernel/locking/rtmutex.c +2 −2
kernel/locking/rwsem.c +2 −2
kernel/locking/ww_mutex.h +1 −1
kernel/sched/core.c +166 −82
kernel/sched/cpufreq_schedutil.c +3 −3
kernel/sched/deadline.c +403 −100
kernel/sched/debug.c +186 −12
kernel/sched/fair.c +580 −190
kernel/sched/features.h +28 −2
kernel/sched/idle.c +7 −16
kernel/sched/rt.c +112 −149
kernel/sched/sched.h +78 −23
kernel/sched/stop_task.c +4 −14
kernel/sched/syscalls.c +25 −109
kernel/sched/topology.c +8 −0
kernel/sys.c +1 −1
kernel/time/hrtimer.c +1 −1
kernel/trace/trace_sched_wakeup.c +1 −1
mm/page-writeback.c +2 −2
mm/page_alloc.c +1 −1

Documentation/scheduler/sched-deadline.rst+6−8

  of the command line options. Please refer to rt-app documentation for more
  details (`<rt-app-sources>/doc/*.json`).
- The second testing application is a modification of schedtool, called
+ The second testing application is done using chrt which has support
- schedtool-dl, which can be used to setup SCHED_DEADLINE parameters for a
+ for SCHED_DEADLINE.
- certain pid/application. schedtool-dl is available at:
- https://github.com/scheduler-tools/schedtool-dl.git.
  The usage is straightforward::
-  # schedtool -E -t 10000000:100000000 -e ./my_cpuhog_app
+  # chrt -d -T 10000000 -D 100000000 0 ./my_cpuhog_app
  With this, my_cpuhog_app is put to run inside a SCHED_DEADLINE reservation
- of 10ms every 100ms (note that parameters are expressed in microseconds).
+ of 10ms every 100ms (note that parameters are expressed in nanoseconds).
- You can also use schedtool to create a reservation for an already running
+ You can also use chrt to create a reservation for an already running
  application, given that you know its pid::
-  # schedtool -E -t 10000000:100000000 my_app_pid
+  # chrt -d -T 10000000 -D 100000000 -p 0 my_app_pid
 Appendix B. Minimal main()
 ==========================

drivers/cpufreq/cppc_cpufreq.c+3−3

@@ -224,9 +224,9 @@ static void __init cppc_freq_invariance_init(void) · 223 unmodified lines
224	* Fake (unused) bandwidth; workaround to "fix"	224	* Fake (unused) bandwidth; workaround to "fix"
225	* priority inheritance.	225	* priority inheritance.
226	*/	226	*/
227	.sched_runtime = 1000000,	227	.sched_runtime = NSEC_PER_MSEC,
228	.sched_deadline = 10000000,	228	.sched_deadline = 10 * NSEC_PER_MSEC,
229	.sched_period = 10000000,	229	.sched_period = 10 * NSEC_PER_MSEC,
230	};	230	};
231	int ret;	231	int ret;
232		232

fs/bcachefs/six.c+1−1

@@ -335,7 +335,7 @@ static inline bool six_owner_running(struct six_lock *lock) · 334 unmodified lines
335	*/	335	*/
336	rcu_read_lock();	336	rcu_read_lock();
337	struct task_struct *owner = READ_ONCE(lock->owner);	337	struct task_struct *owner = READ_ONCE(lock->owner);
338	bool ret = owner ? owner_on_cpu(owner) : !rt_task(current);	338	bool ret = owner ? owner_on_cpu(owner) : !rt_or_dl_task(current);
339	rcu_read_unlock();	339	rcu_read_unlock();
340		340
341	return ret;	341	return ret;

fs/proc/base.c+1−1

@@ -2626,7 +2626,7 @@ static ssize_t timerslack_ns_write(struct file file, const char __user buf, · 2625 unmodified lines
2626	}	2626	}
2627		2627
2628	task_lock(p);	2628	task_lock(p);
2629	if (task_is_realtime(p))	2629	if (rt_or_dl_task_policy(p))
2630	slack_ns = 0;	2630	slack_ns = 0;
2631	else if (slack_ns == 0)	2631	else if (slack_ns == 0)
2632	slack_ns = p->default_timer_slack_ns;	2632	slack_ns = p->default_timer_slack_ns;

include/linux/ioprio.h+1−1

@@ -40,7 +40,7 @@ static inline int task_nice_ioclass(struct task_struct *task) · 39 unmodified lines
40	{	40	{
41	if (task->policy == SCHED_IDLE)	41	if (task->policy == SCHED_IDLE)
42	return IOPRIO_CLASS_IDLE;	42	return IOPRIO_CLASS_IDLE;
43	else if (task_is_realtime(task))	43	else if (rt_or_dl_task_policy(task))
44	return IOPRIO_CLASS_RT;	44	return IOPRIO_CLASS_RT;
45	else	45	else
46	return IOPRIO_CLASS_BE;	46	return IOPRIO_CLASS_BE;

include/linux/sched.h+24−4

  * Special states are those that do not use the normal wait-loop pattern. See
  * the comment with set_special_state().
  */
-#define is_special_task_state(state)				\
+#define is_special_task_state(state)					\
-	((state) & (__TASK_STOPPED | __TASK_TRACED | TASK_PARKED | TASK_DEAD))
+	((state) & (__TASK_STOPPED | __TASK_TRACED | TASK_PARKED |	\
+		    TASK_DEAD | TASK_FROZEN))
 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
 # define debug_normal_state_change(state_value)				\
 	struct rb_node			run_node;
 	u64				deadline;
 	u64				min_vruntime;
+	u64				min_slice;
 	struct list_head		group_node;
-	unsigned int			on_rq;
+	unsigned char			on_rq;
+	unsigned char			sched_delayed;
+	unsigned char			rel_deadline;
+	unsigned char			custom_slice;
+					/* hole */
 	u64				exec_start;
 	u64				sum_exec_runtime;
 	 *
 	 * @dl_overrun tells if the task asked to be informed about runtime
 	 * overruns.
+	 *
+	 * @dl_server tells if this is a server entity.
+	 *
+	 * @dl_defer tells if this is a deferred or regular server. For
+	 * now only defer server exists.
+	 *
+	 * @dl_defer_armed tells if the deferrable server is waiting
+	 * for the replenishment timer to activate it.
+	 *
+	 * @dl_defer_running tells if the deferrable server is actually
+	 * running, skipping the defer phase.
 	 */
 	unsigned int			dl_throttled      : 1;
 	unsigned int			dl_yielded        : 1;
 	unsigned int			dl_non_contending : 1;
 	unsigned int			dl_overrun	  : 1;
 	unsigned int			dl_server         : 1;
+	unsigned int			dl_defer	  : 1;
+	unsigned int			dl_defer_armed	  : 1;
+	unsigned int			dl_defer_running  : 1;
 	/*
 	 * Bandwidth enforcement timer. Each -deadline task has its
 	 */
 	struct rq			*rq;
 	dl_server_has_tasks_f		server_has_tasks;
-	dl_server_pick_f		server_pick;
+	dl_server_pick_f		server_pick_task;
 #ifdef CONFIG_RT_MUTEXES
 	/*

include/linux/sched/deadline.h+7−7

 #include <linux/sched.h>
-#define MAX_DL_PRIO		0
+static inline bool dl_prio(int prio)
-static inline int dl_prio(int prio)
 {
-	if (unlikely(prio < MAX_DL_PRIO))
+	return unlikely(prio < MAX_DL_PRIO);
-		return 1;
-	return 0;
 }
-static inline int dl_task(struct task_struct *p)
+/*
+ * Returns true if a task has a priority that belongs to DL class. PI-boosted
+ * tasks will return true. Use dl_policy() to ignore PI-boosted tasks.
+ */
+static inline bool dl_task(struct task_struct *p)
 {
 	return dl_prio(p->prio);
 }

include/linux/sched/prio.h+1−0

@@ -14,6 +14,7 @@ · 13 unmodified lines
14	*/	14	*/
15		15
16	#define MAX_RT_PRIO 100	16	#define MAX_RT_PRIO 100
		17	#define MAX_DL_PRIO 0
17		18
18	#define MAX_PRIO (MAX_RT_PRIO + NICE_WIDTH)	19	#define MAX_PRIO (MAX_RT_PRIO + NICE_WIDTH)
19	#define DEFAULT_PRIO (MAX_RT_PRIO + NICE_WIDTH / 2)	20	#define DEFAULT_PRIO (MAX_RT_PRIO + NICE_WIDTH / 2)

include/linux/sched/rt.h+27−6

@@ -6,19 +6,40 @@ · 5 unmodified lines
6		6
7	struct task_struct;	7	struct task_struct;
8		8
9	static inline int rt_prio(int prio)	9	static inline bool rt_prio(int prio)
10	{	10	{
11	if (unlikely(prio < MAX_RT_PRIO))	11	return unlikely(prio < MAX_RT_PRIO && prio >= MAX_DL_PRIO);
12	return 1;
13	return 0;
14	}	12	}
15		13
16	static inline int rt_task(struct task_struct *p)	14	static inline bool rt_or_dl_prio(int prio)
		15	{
		16	return unlikely(prio < MAX_RT_PRIO);
		17	}
		18
		19	/*
		20	* Returns true if a task has a priority that belongs to RT class. PI-boosted
		21	* tasks will return true. Use rt_policy() to ignore PI-boosted tasks.
		22	*/
		23	static inline bool rt_task(struct task_struct *p)
17	{	24	{
18	return rt_prio(p->prio);	25	return rt_prio(p->prio);
19	}	26	}
20		27
21	static inline bool task_is_realtime(struct task_struct *tsk)	28	/*
		29	* Returns true if a task has a priority that belongs to RT or DL classes.
		30	* PI-boosted tasks will return true. Use rt_or_dl_task_policy() to ignore
		31	* PI-boosted tasks.
		32	*/
		33	static inline bool rt_or_dl_task(struct task_struct *p)
		34	{
		35	return rt_or_dl_prio(p->prio);
		36	}
		37
		38	/*
		39	* Returns true if a task has a policy that belongs to RT or DL classes.
		40	* PI-boosted tasks will return false.
		41	*/
		42	static inline bool rt_or_dl_task_policy(struct task_struct *tsk)
22	{	43	{
23	int policy = tsk->policy;	44	int policy = tsk->policy;
24		45

include/uapi/linux/sched/types.h+3−3

@@ -58,9 +58,9 @@ · 57 unmodified lines
58	*	58	*
59	* This is reflected by the following fields of the sched_attr structure:	59	* This is reflected by the following fields of the sched_attr structure:
60	*	60	*
61	* @sched_deadline representative of the task's deadline	61	* @sched_deadline representative of the task's deadline in nanoseconds
62	* @sched_runtime representative of the task's runtime	62	* @sched_runtime representative of the task's runtime in nanoseconds
63	* @sched_period representative of the task's period	63	* @sched_period representative of the task's period in nanoseconds
64	*	64	*
65	* Given this task model, there are a multiplicity of scheduling algorithms	65	* Given this task model, there are a multiplicity of scheduling algorithms
66	* and policies, that can be used to ensure all the tasks will make their	66	* and policies, that can be used to ensure all the tasks will make their

kernel/freezer.c+1−1

@@ -72,7 +72,7 @@ bool __refrigerator(bool check_kthr_stop) · 71 unmodified lines
72	bool freeze;	72	bool freeze;
73		73
74	raw_spin_lock_irq(&current->pi_lock);	74	raw_spin_lock_irq(&current->pi_lock);
75	set_current_state(TASK_FROZEN);	75	WRITE_ONCE(current->__state, TASK_FROZEN);
76	/* unstale saved_state so that __thaw_task() will wake us up */	76	/* unstale saved_state so that __thaw_task() will wake us up */
77	current->saved_state = TASK_RUNNING;	77	current->saved_state = TASK_RUNNING;
78	raw_spin_unlock_irq(&current->pi_lock);	78	raw_spin_unlock_irq(&current->pi_lock);

kernel/kthread.c+9−1

 		 * event only cares about the address.
 		 */
 		trace_sched_kthread_work_execute_end(work, func);
-	} else if (!freezing(current))
+	} else if (!freezing(current)) {
 		schedule();
+	} else {
+		/*
+		 * Handle the case where the current remains
+		 * TASK_INTERRUPTIBLE. try_to_freeze() expects
+		 * the current to be TASK_RUNNING.
+		 */
+		__set_current_state(TASK_RUNNING);
+	}
 	try_to_freeze();
 	cond_resched();

kernel/locking/rtmutex.c+2−2

@@ -347,7 +347,7 @@ static __always_inline int __waiter_prio(struct task_struct *task) · 346 unmodified lines
347	{	347	{
348	int prio = task->prio;	348	int prio = task->prio;
349		349
350	if (!rt_prio(prio))	350	if (!rt_or_dl_prio(prio))
351	return DEFAULT_PRIO;	351	return DEFAULT_PRIO;
352		352
353	return prio;	353	return prio;
@@ -435,7 +435,7 @@ static inline bool rt_mutex_steal(struct rt_mutex_waiter *waiter, · 81 unmodified lines
435	* Note that RT tasks are excluded from same priority (lateral)	435	* Note that RT tasks are excluded from same priority (lateral)
436	* steals to prevent the introduction of an unbounded latency.	436	* steals to prevent the introduction of an unbounded latency.
437	*/	437	*/
438	if (rt_prio(waiter->tree.prio) \|\| dl_prio(waiter->tree.prio))	438	if (rt_or_dl_prio(waiter->tree.prio))
439	return false;	439	return false;
440		440
441	return rt_waiter_node_equal(&waiter->tree, &top_waiter->tree);	441	return rt_waiter_node_equal(&waiter->tree, &top_waiter->tree);

kernel/locking/rwsem.c+2−2

@@ -631,7 +631,7 @@ static inline bool rwsem_try_write_lock(struct rw_semaphore *sem, · 630 unmodified lines
631	* if it is an RT task or wait in the wait queue	631	* if it is an RT task or wait in the wait queue
632	* for too long.	632	* for too long.
633	*/	633	*/
634	if (has_handoff \|\| (!rt_task(waiter->task) &&	634	if (has_handoff \|\| (!rt_or_dl_task(waiter->task) &&
635	!time_after(jiffies, waiter->timeout)))	635	!time_after(jiffies, waiter->timeout)))
636	return false;	636	return false;
637		637
@@ -914,7 +914,7 @@ static bool rwsem_optimistic_spin(struct rw_semaphore *sem) · 276 unmodified lines
914	if (owner_state != OWNER_WRITER) {	914	if (owner_state != OWNER_WRITER) {
915	if (need_resched())	915	if (need_resched())
916	break;	916	break;
917	if (rt_task(current) &&	917	if (rt_or_dl_task(current) &&
918	(prev_owner_state != OWNER_WRITER))	918	(prev_owner_state != OWNER_WRITER))
919	break;	919	break;
920	}	920	}

kernel/locking/ww_mutex.h+1−1

@@ -237,7 +237,7 @@ __ww_ctx_less(struct ww_acquire_ctx a, struct ww_acquire_ctx b) · 236 unmodified lines
237	int a_prio = a->task->prio;	237	int a_prio = a->task->prio;
238	int b_prio = b->task->prio;	238	int b_prio = b->task->prio;
239		239
240	if (rt_prio(a_prio) \|\| rt_prio(b_prio)) {	240	if (rt_or_dl_prio(a_prio) \|\| rt_or_dl_prio(b_prio)) {
241		241
242	if (a_prio > b_prio)	242	if (a_prio > b_prio)
243	return true;	243	return true;

kernel/sched/core.c+166−82

@@ -163,7 +163,10 @@ static inline int __task_prio(const struct task_struct *p) · 162 unmodified lines
163	if (p->sched_class == &stop_sched_class) /* trumps deadline */	163	if (p->sched_class == &stop_sched_class) /* trumps deadline */
164	return -2;	164	return -2;
165		165
166	if (rt_prio(p->prio)) /* includes deadline */	166	if (p->dl_server)
		167	return -1; /* deadline */
		168
		169	if (rt_or_dl_prio(p->prio))
167	return p->prio; /* [-1, 99] */	170	return p->prio; /* [-1, 99] */
168		171
169	if (p->sched_class == &idle_sched_class)	172	if (p->sched_class == &idle_sched_class)
@@ -192,8 +195,24 @@ static inline bool prio_less(const struct task_struct *a, · 22 unmodified lines
192	if (-pb < -pa)	195	if (-pb < -pa)
193	return false;	196	return false;
194		197
195	if (pa == -1) /* dl_prio() doesn't work because of stop_class above */	198	if (pa == -1) { /* dl_prio() doesn't work because of stop_class above */
196	return !dl_time_before(a->dl.deadline, b->dl.deadline);	199	const struct sched_dl_entity a_dl, b_dl;
		200
		201	a_dl = &a->dl;
		202	/*
		203	* Since,'a' and 'b' can be CFS tasks served by DL server,
		204	* __task_prio() can return -1 (for DL) even for those. In that
		205	* case, get to the dl_server's DL entity.
		206	*/
		207	if (a->dl_server)
		208	a_dl = a->dl_server;
		209
		210	b_dl = &b->dl;
		211	if (b->dl_server)
		212	b_dl = b->dl_server;
		213
		214	return !dl_time_before(a_dl->deadline, b_dl->deadline);
		215	}
197		216
198	if (pa == MAX_RT_PRIO + MAX_NICE) /* fair */	217	if (pa == MAX_RT_PRIO + MAX_NICE) /* fair */
199	return cfs_prio_less(a, b, in_fi);	218	return cfs_prio_less(a, b, in_fi);
@@ -240,6 +259,9 @@ static inline int rb_sched_core_cmp(const void key, const struct rb_node node) · 40 unmodified lines
240		259
241	void sched_core_enqueue(struct rq rq, struct task_struct p)	260	void sched_core_enqueue(struct rq rq, struct task_struct p)
242	{	261	{
		262	if (p->se.sched_delayed)
		263	return;
		264
243	rq->core->core_task_seq++;	265	rq->core->core_task_seq++;
244		266
245	if (!p->core_cookie)	267	if (!p->core_cookie)
@@ -250,6 +272,9 @@ void sched_core_enqueue(struct rq rq, struct task_struct p) · 4 unmodified lines
250		272
251	void sched_core_dequeue(struct rq rq, struct task_struct p, int flags)	273	void sched_core_dequeue(struct rq rq, struct task_struct p, int flags)
252	{	274	{
		275	if (p->se.sched_delayed)
		276	return;
		277
253	rq->core->core_task_seq++;	278	rq->core->core_task_seq++;
254		279
255	if (sched_core_enqueued(p)) {	280	if (sched_core_enqueued(p)) {
@@ -1269,7 +1294,7 @@ bool sched_can_stop_tick(struct rq *rq) · 1013 unmodified lines
1269	* dequeued by migrating while the constrained task continues to run.	1294	* dequeued by migrating while the constrained task continues to run.
1270	* E.g. going from 2->1 without going through pick_next_task().	1295	* E.g. going from 2->1 without going through pick_next_task().
1271	*/	1296	*/
1272	if (sched_feat(HZ_BW) && __need_bw_check(rq, rq->curr)) {	1297	if (__need_bw_check(rq, rq->curr)) {
1273	if (cfs_task_bw_constrained(rq->curr))	1298	if (cfs_task_bw_constrained(rq->curr))
1274	return false;	1299	return false;
1275	}	1300	}
@@ -1672,6 +1697,9 @@ static inline void uclamp_rq_inc(struct rq rq, struct task_struct p) · 396 unmodified lines
1672	if (unlikely(!p->sched_class->uclamp_enabled))	1697	if (unlikely(!p->sched_class->uclamp_enabled))
1673	return;	1698	return;
1674		1699
		1700	if (p->se.sched_delayed)
		1701	return;
		1702
1675	for_each_clamp_id(clamp_id)	1703	for_each_clamp_id(clamp_id)
1676	uclamp_rq_inc_id(rq, p, clamp_id);	1704	uclamp_rq_inc_id(rq, p, clamp_id);
1677		1705
@@ -1696,6 +1724,9 @@ static inline void uclamp_rq_dec(struct rq rq, struct task_struct p) · 18 unmodified lines
1696	if (unlikely(!p->sched_class->uclamp_enabled))	1724	if (unlikely(!p->sched_class->uclamp_enabled))
1697	return;	1725	return;
1698		1726
		1727	if (p->se.sched_delayed)
		1728	return;
		1729
1699	for_each_clamp_id(clamp_id)	1730	for_each_clamp_id(clamp_id)
1700	uclamp_rq_dec_id(rq, p, clamp_id);	1731	uclamp_rq_dec_id(rq, p, clamp_id);
1701	}	1732	}
@@ -1975,14 +2006,21 @@ void enqueue_task(struct rq rq, struct task_struct p, int flags) · 273 unmodified lines
1975	psi_enqueue(p, (flags & ENQUEUE_WAKEUP) && !(flags & ENQUEUE_MIGRATED));	2006	psi_enqueue(p, (flags & ENQUEUE_WAKEUP) && !(flags & ENQUEUE_MIGRATED));
1976	}	2007	}
1977		2008
1978	uclamp_rq_inc(rq, p);
1979	p->sched_class->enqueue_task(rq, p, flags);	2009	p->sched_class->enqueue_task(rq, p, flags);
		2010	/*
		2011	* Must be after ->enqueue_task() because ENQUEUE_DELAYED can clear
		2012	* ->sched_delayed.
		2013	*/
		2014	uclamp_rq_inc(rq, p);
1980		2015
1981	if (sched_core_enabled(rq))	2016	if (sched_core_enabled(rq))
1982	sched_core_enqueue(rq, p);	2017	sched_core_enqueue(rq, p);
1983	}	2018	}
1984		2019
1985	void dequeue_task(struct rq rq, struct task_struct p, int flags)	2020	/*
		2021	* Must only return false when DEQUEUE_SLEEP.
		2022	*/
		2023	inline bool dequeue_task(struct rq rq, struct task_struct p, int flags)
1986	{	2024	{
1987	if (sched_core_enabled(rq))	2025	if (sched_core_enabled(rq))
1988	sched_core_dequeue(rq, p, flags);	2026	sched_core_dequeue(rq, p, flags);
@@ -1995,8 +2033,12 @@ void dequeue_task(struct rq rq, struct task_struct p, int flags) · 6 unmodified lines
1995	psi_dequeue(p, flags & DEQUEUE_SLEEP);	2033	psi_dequeue(p, flags & DEQUEUE_SLEEP);
1996	}	2034	}
1997		2035
		2036	/*
		2037	* Must be before ->dequeue_task() because ->dequeue_task() can 'fail'
		2038	* and mark the task ->sched_delayed.
		2039	*/
1998	uclamp_rq_dec(rq, p);	2040	uclamp_rq_dec(rq, p);
1999	p->sched_class->dequeue_task(rq, p, flags);	2041	return p->sched_class->dequeue_task(rq, p, flags);
2000	}	2042	}
2001		2043
2002	void activate_task(struct rq rq, struct task_struct p, int flags)	2044	void activate_task(struct rq rq, struct task_struct p, int flags)
@@ -2014,12 +2056,25 @@ void activate_task(struct rq rq, struct task_struct p, int flags) · 11 unmodified lines
2014		2056
2015	void deactivate_task(struct rq rq, struct task_struct p, int flags)	2057	void deactivate_task(struct rq rq, struct task_struct p, int flags)
2016	{	2058	{
2017	WRITE_ONCE(p->on_rq, (flags & DEQUEUE_SLEEP) ? 0 : TASK_ON_RQ_MIGRATING);	2059	SCHED_WARN_ON(flags & DEQUEUE_SLEEP);
		2060
		2061	WRITE_ONCE(p->on_rq, TASK_ON_RQ_MIGRATING);
2018	ASSERT_EXCLUSIVE_WRITER(p->on_rq);	2062	ASSERT_EXCLUSIVE_WRITER(p->on_rq);
2019		2063
		2064	/*
		2065	* Code explicitly relies on TASK_ON_RQ_MIGRATING begin set before
		2066	* dequeue_task() and cleared after enqueue_task().
		2067	*/
		2068
2020	dequeue_task(rq, p, flags);	2069	dequeue_task(rq, p, flags);
2021	}	2070	}
2022		2071
		2072	static void block_task(struct rq rq, struct task_struct p, int flags)
		2073	{
		2074	if (dequeue_task(rq, p, DEQUEUE_SLEEP \| flags))
		2075	__block_task(rq, p);
		2076	}
		2077
2023	/**	2078	/**
2024	* task_curr - is this task currently executing on a CPU?	2079	* task_curr - is this task currently executing on a CPU?
2025	* @p: the task in question.	2080	* @p: the task in question.
@@ -2233,6 +2288,12 @@ void migrate_disable(void) · 207 unmodified lines
2233	struct task_struct *p = current;	2288	struct task_struct *p = current;
2234		2289
2235	if (p->migration_disabled) {	2290	if (p->migration_disabled) {
		2291	#ifdef CONFIG_DEBUG_PREEMPT
		2292	/*
		2293	*Warn about overflow half-way through the range.
		2294	*/
		2295	WARN_ON_ONCE((s16)p->migration_disabled < 0);
		2296	#endif
2236	p->migration_disabled++;	2297	p->migration_disabled++;
2237	return;	2298	return;
2238	}	2299	}
@@ -2251,14 +2312,20 @@ void migrate_enable(void) · 12 unmodified lines
2251	.flags = SCA_MIGRATE_ENABLE,	2312	.flags = SCA_MIGRATE_ENABLE,
2252	};	2313	};
2253		2314
		2315	#ifdef CONFIG_DEBUG_PREEMPT
		2316	/*
		2317	* Check both overflow from migrate_disable() and superfluous
		2318	* migrate_enable().
		2319	*/
		2320	if (WARN_ON_ONCE((s16)p->migration_disabled <= 0))
		2321	return;
		2322	#endif
		2323
2254	if (p->migration_disabled > 1) {	2324	if (p->migration_disabled > 1) {
2255	p->migration_disabled--;	2325	p->migration_disabled--;
2256	return;	2326	return;
2257	}	2327	}
2258		2328
2259	if (WARN_ON_ONCE(!p->migration_disabled))
2260	return;
2261
2262	/*	2329	/*
2263	* Ensure stop_task runs either before or after this, and that	2330	* Ensure stop_task runs either before or after this, and that
2264	* __set_cpus_allowed_ptr(SCA_MIGRATE_ENABLE) doesn't schedule().	2331	* __set_cpus_allowed_ptr(SCA_MIGRATE_ENABLE) doesn't schedule().
@@ -3607,8 +3674,6 @@ ttwu_do_activate(struct rq rq, struct task_struct p, int wake_flags, · 1342 unmodified lines
3607	rq->idle_stamp = 0;	3674	rq->idle_stamp = 0;
3608	}	3675	}
3609	#endif	3676	#endif
3610
3611	p->dl_server = NULL;
3612	}	3677	}
3613		3678
3614	/*	3679	/*
@@ -3644,12 +3709,14 @@ static int ttwu_runnable(struct task_struct *p, int wake_flags) · 29 unmodified lines
3644		3709
3645	rq = __task_rq_lock(p, &rf);	3710	rq = __task_rq_lock(p, &rf);
3646	if (task_on_rq_queued(p)) {	3711	if (task_on_rq_queued(p)) {
		3712	update_rq_clock(rq);
		3713	if (p->se.sched_delayed)
		3714	enqueue_task(rq, p, ENQUEUE_NOCLOCK \| ENQUEUE_DELAYED);
3647	if (!task_on_cpu(rq, p)) {	3715	if (!task_on_cpu(rq, p)) {
3648	/*	3716	/*
3649	* When on_rq && !on_cpu the task is preempted, see if	3717	* When on_rq && !on_cpu the task is preempted, see if
3650	* it should preempt the task that is current now.	3718	* it should preempt the task that is current now.
3651	*/	3719	*/
3652	update_rq_clock(rq);
3653	wakeup_preempt(rq, p, wake_flags);	3720	wakeup_preempt(rq, p, wake_flags);
3654	}	3721	}
3655	ttwu_do_wakeup(p);	3722	ttwu_do_wakeup(p);
@@ -4029,11 +4096,16 @@ int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) · 373 unmodified lines
4029	* case the whole 'p->on_rq && ttwu_runnable()' case below	4096	* case the whole 'p->on_rq && ttwu_runnable()' case below
4030	* without taking any locks.	4097	* without taking any locks.
4031	*	4098	*
		4099	* Specifically, given current runs ttwu() we must be before
		4100	* schedule()'s block_task(), as such this must not observe
		4101	* sched_delayed.
		4102	*
4032	* In particular:	4103	* In particular:
4033	* - we rely on Program-Order guarantees for all the ordering,	4104	* - we rely on Program-Order guarantees for all the ordering,
4034	* - we're serialized against set_special_state() by virtue of	4105	* - we're serialized against set_special_state() by virtue of
4035	* it disabling IRQs (this allows not taking ->pi_lock).	4106	* it disabling IRQs (this allows not taking ->pi_lock).
4036	*/	4107	*/
		4108	SCHED_WARN_ON(p->se.sched_delayed);
4037	if (!ttwu_state_match(p, state, &success))	4109	if (!ttwu_state_match(p, state, &success))
4038	goto out;	4110	goto out;
4039		4111
@@ -4322,9 +4394,11 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p) · 282 unmodified lines
4322	p->se.nr_migrations = 0;	4394	p->se.nr_migrations = 0;
4323	p->se.vruntime = 0;	4395	p->se.vruntime = 0;
4324	p->se.vlag = 0;	4396	p->se.vlag = 0;
4325	p->se.slice = sysctl_sched_base_slice;
4326	INIT_LIST_HEAD(&p->se.group_node);	4397	INIT_LIST_HEAD(&p->se.group_node);
4327		4398
		4399	/* A delayed task cannot be in clone(). */
		4400	SCHED_WARN_ON(p->se.sched_delayed);
		4401
4328	#ifdef CONFIG_FAIR_GROUP_SCHED	4402	#ifdef CONFIG_FAIR_GROUP_SCHED
4329	p->se.cfs_rq = NULL;	4403	p->se.cfs_rq = NULL;
4330	#endif	4404	#endif
@@ -4572,6 +4646,8 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) · 241 unmodified lines
4572		4646
4573	p->prio = p->normal_prio = p->static_prio;	4647	p->prio = p->normal_prio = p->static_prio;
4574	set_load_weight(p, false);	4648	set_load_weight(p, false);
		4649	p->se.custom_slice = 0;
		4650	p->se.slice = sysctl_sched_base_slice;
4575		4651
4576	/*	4652	/*
4577	* We don't need the reset flag anymore after the fork. It has	4653	* We don't need the reset flag anymore after the fork. It has
@@ -4686,7 +4762,7 @@ void wake_up_new_task(struct task_struct *p) · 108 unmodified lines
4686	update_rq_clock(rq);	4762	update_rq_clock(rq);
4687	post_init_entity_util_avg(p);	4763	post_init_entity_util_avg(p);
4688		4764
4689	activate_task(rq, p, ENQUEUE_NOCLOCK);	4765	activate_task(rq, p, ENQUEUE_NOCLOCK \| ENQUEUE_INITIAL);
4690	trace_sched_wakeup_new(p);	4766	trace_sched_wakeup_new(p);
4691	wakeup_preempt(rq, p, WF_FORK);	4767	wakeup_preempt(rq, p, WF_FORK);
4692	#ifdef CONFIG_SMP	4768	#ifdef CONFIG_SMP
@@ -5769,8 +5845,8 @@ static inline void schedule_debug(struct task_struct *prev, bool preempt) · 1076 unmodified lines
5769	schedstat_inc(this_rq()->sched_count);	5845	schedstat_inc(this_rq()->sched_count);
5770	}	5846	}
5771		5847
5772	static void put_prev_task_balance(struct rq rq, struct task_struct prev,	5848	static void prev_balance(struct rq rq, struct task_struct prev,
5773	struct rq_flags *rf)	5849	struct rq_flags *rf)
5774	{	5850	{
5775	#ifdef CONFIG_SMP	5851	#ifdef CONFIG_SMP
5776	const struct sched_class *class;	5852	const struct sched_class *class;
@@ -5787,8 +5863,6 @@ static void put_prev_task_balance(struct rq rq, struct task_struct prev, · 10 unmodified lines
5787	break;	5863	break;
5788	}	5864	}
5789	#endif	5865	#endif
5790
5791	put_prev_task(rq, prev);
5792	}	5866	}
5793		5867
5794	/*	5868	/*
@@ -5800,6 +5874,8 @@ __pick_next_task(struct rq rq, struct task_struct prev, struct rq_flags *rf) · 5 unmodified lines
5800	const struct sched_class *class;	5874	const struct sched_class *class;
5801	struct task_struct *p;	5875	struct task_struct *p;
5802		5876
		5877	rq->dl_server = NULL;
		5878
5803	/*	5879	/*
5804	* Optimization: we know that if all tasks are in the fair class we can	5880	* Optimization: we know that if all tasks are in the fair class we can
5805	* call that function directly, but only if the @prev task wasn't of a	5881	* call that function directly, but only if the @prev task wasn't of a
@@ -5815,35 +5891,28 @@ __pick_next_task(struct rq rq, struct task_struct prev, struct rq_flags *rf) · 9 unmodified lines
5815		5891
5816	/* Assume the next prioritized class is idle_sched_class */	5892	/* Assume the next prioritized class is idle_sched_class */
5817	if (!p) {	5893	if (!p) {
5818	put_prev_task(rq, prev);	5894	p = pick_task_idle(rq);
5819	p = pick_next_task_idle(rq);	5895	put_prev_set_next_task(rq, prev, p);
5820	}	5896	}
5821		5897
5822	/*
5823	* This is the fast path; it cannot be a DL server pick;
5824	* therefore even if @p == @prev, ->dl_server must be NULL.
5825	*/
5826	if (p->dl_server)
5827	p->dl_server = NULL;
5828
5829	return p;	5898	return p;
5830	}	5899	}
5831		5900
5832	restart:	5901	restart:
5833	put_prev_task_balance(rq, prev, rf);	5902	prev_balance(rq, prev, rf);
5834
5835	/*
5836	* We've updated @prev and no longer need the server link, clear it.
5837	* Must be done before ->pick_next_task() because that can (re)set
5838	* ->dl_server.
5839	*/
5840	if (prev->dl_server)
5841	prev->dl_server = NULL;
5842		5903
5843	for_each_class(class) {	5904	for_each_class(class) {
5844	p = class->pick_next_task(rq);	5905	if (class->pick_next_task) {
5845	if (p)	5906	p = class->pick_next_task(rq, prev);
5846	return p;	5907	if (p)
		5908	return p;
		5909	} else {
		5910	p = class->pick_task(rq);
		5911	if (p) {
		5912	put_prev_set_next_task(rq, prev, p);
		5913	return p;
		5914	}
		5915	}
5847	}	5916	}
5848		5917
5849	BUG(); /* The idle class should always have a runnable task. */	5918	BUG(); /* The idle class should always have a runnable task. */
@@ -5873,6 +5942,8 @@ static inline struct task_struct pick_task(struct rq rq) · 23 unmodified lines
5873	const struct sched_class *class;	5942	const struct sched_class *class;
5874	struct task_struct *p;	5943	struct task_struct *p;
5875		5944
		5945	rq->dl_server = NULL;
		5946
5876	for_each_class(class) {	5947	for_each_class(class) {
5877	p = class->pick_task(rq);	5948	p = class->pick_task(rq);
5878	if (p)	5949	if (p)
@@ -5911,6 +5982,7 @@ pick_next_task(struct rq rq, struct task_struct prev, struct rq_flags *rf) · 32 unmodified lines
5911	* another cpu during offline.	5982	* another cpu during offline.
5912	*/	5983	*/
5913	rq->core_pick = NULL;	5984	rq->core_pick = NULL;
		5985	rq->core_dl_server = NULL;
5914	return __pick_next_task(rq, prev, rf);	5986	return __pick_next_task(rq, prev, rf);
5915	}	5987	}
5916		5988
@@ -5929,16 +6001,13 @@ pick_next_task(struct rq rq, struct task_struct prev, struct rq_flags *rf) · 12 unmodified lines
5929	WRITE_ONCE(rq->core_sched_seq, rq->core->core_pick_seq);	6001	WRITE_ONCE(rq->core_sched_seq, rq->core->core_pick_seq);
5930		6002
5931	next = rq->core_pick;	6003	next = rq->core_pick;
5932	if (next != prev) {	6004	rq->dl_server = rq->core_dl_server;
5933	put_prev_task(rq, prev);
5934	set_next_task(rq, next);
5935	}
5936
5937	rq->core_pick = NULL;	6005	rq->core_pick = NULL;
5938	goto out;	6006	rq->core_dl_server = NULL;
		6007	goto out_set_next;
5939	}	6008	}
5940		6009
5941	put_prev_task_balance(rq, prev, rf);	6010	prev_balance(rq, prev, rf);
5942		6011
5943	smt_mask = cpu_smt_mask(cpu);	6012	smt_mask = cpu_smt_mask(cpu);
5944	need_sync = !!rq->core->core_cookie;	6013	need_sync = !!rq->core->core_cookie;
@@ -5979,6 +6048,7 @@ pick_next_task(struct rq rq, struct task_struct prev, struct rq_flags *rf) · 34 unmodified lines
5979	next = pick_task(rq);	6048	next = pick_task(rq);
5980	if (!next->core_cookie) {	6049	if (!next->core_cookie) {
5981	rq->core_pick = NULL;	6050	rq->core_pick = NULL;
		6051	rq->core_dl_server = NULL;
5982	/*	6052	/*
5983	* For robustness, update the min_vruntime_fi for	6053	* For robustness, update the min_vruntime_fi for
5984	* unconstrained picks as well.	6054	* unconstrained picks as well.
@@ -6006,7 +6076,9 @@ pick_next_task(struct rq rq, struct task_struct prev, struct rq_flags *rf) · 21 unmodified lines
6006	if (i != cpu && (rq_i != rq->core \|\| !core_clock_updated))	6076	if (i != cpu && (rq_i != rq->core \|\| !core_clock_updated))
6007	update_rq_clock(rq_i);	6077	update_rq_clock(rq_i);
6008		6078
6009	p = rq_i->core_pick = pick_task(rq_i);	6079	rq_i->core_pick = p = pick_task(rq_i);
		6080	rq_i->core_dl_server = rq_i->dl_server;
		6081
6010	if (!max \|\| prio_less(max, p, fi_before))	6082	if (!max \|\| prio_less(max, p, fi_before))
6011	max = p;	6083	max = p;
6012	}	6084	}
@@ -6030,6 +6102,7 @@ pick_next_task(struct rq rq, struct task_struct prev, struct rq_flags *rf) · 17 unmodified lines
6030	}	6102	}
6031		6103
6032	rq_i->core_pick = p;	6104	rq_i->core_pick = p;
		6105	rq_i->core_dl_server = NULL;
6033		6106
6034	if (p == rq_i->idle) {	6107	if (p == rq_i->idle) {
6035	if (rq_i->nr_running) {	6108	if (rq_i->nr_running) {
@@ -6090,6 +6163,7 @@ pick_next_task(struct rq rq, struct task_struct prev, struct rq_flags *rf) · 54 unmodified lines
6090		6163
6091	if (i == cpu) {	6164	if (i == cpu) {
6092	rq_i->core_pick = NULL;	6165	rq_i->core_pick = NULL;
		6166	rq_i->core_dl_server = NULL;
6093	continue;	6167	continue;
6094	}	6168	}
6095		6169
@@ -6098,6 +6172,7 @@ pick_next_task(struct rq rq, struct task_struct prev, struct rq_flags *rf) · 2 unmodified lines
6098		6172
6099	if (rq_i->curr == rq_i->core_pick) {	6173	if (rq_i->curr == rq_i->core_pick) {
6100	rq_i->core_pick = NULL;	6174	rq_i->core_pick = NULL;
		6175	rq_i->core_dl_server = NULL;
6101	continue;	6176	continue;
6102	}	6177	}
6103		6178
@@ -6105,8 +6180,7 @@ pick_next_task(struct rq rq, struct task_struct prev, struct rq_flags *rf) · 1 unmodified line
6105	}	6180	}
6106		6181
6107	out_set_next:	6182	out_set_next:
6108	set_next_task(rq, next);	6183	put_prev_set_next_task(rq, prev, next);
6109	out:
6110	if (rq->core->core_forceidle_count && next == rq->idle)	6184	if (rq->core->core_forceidle_count && next == rq->idle)
6111	queue_core_balance(rq);	6185	queue_core_balance(rq);
6112		6186
@@ -6342,19 +6416,12 @@ pick_next_task(struct rq rq, struct task_struct prev, struct rq_flags *rf) · 229 unmodified lines
6342	* Constants for the sched_mode argument of __schedule().	6416	* Constants for the sched_mode argument of __schedule().
6343	*	6417	*
6344	* The mode argument allows RT enabled kernels to differentiate a	6418	* The mode argument allows RT enabled kernels to differentiate a
6345	* preemption from blocking on an 'sleeping' spin/rwlock. Note that	6419	* preemption from blocking on an 'sleeping' spin/rwlock.
6346	* SM_MASK_PREEMPT for !RT has all bits set, which allows the compiler to
6347	* optimize the AND operation out and just check for zero.
6348	*/	6420	*/
6349	#define SM_NONE 0x0	6421	#define SM_IDLE (-1)
6350	#define SM_PREEMPT 0x1	6422	#define SM_NONE 0
6351	#define SM_RTLOCK_WAIT 0x2	6423	#define SM_PREEMPT 1
6352		6424	#define SM_RTLOCK_WAIT 2
6353	#ifndef CONFIG_PREEMPT_RT
6354	# define SM_MASK_PREEMPT (~0U)
6355	#else
6356	# define SM_MASK_PREEMPT SM_PREEMPT
6357	#endif
6358		6425
6359	/*	6426	/*
6360	* __schedule() is the main scheduler function.	6427	* __schedule() is the main scheduler function.
@@ -6395,9 +6462,14 @@ pick_next_task(struct rq rq, struct task_struct prev, struct rq_flags *rf) · 34 unmodified lines
6395	*	6462	*
6396	* WARNING: must be called with preemption disabled!	6463	* WARNING: must be called with preemption disabled!
6397	*/	6464	*/
6398	static void __sched notrace __schedule(unsigned int sched_mode)	6465	static void __sched notrace __schedule(int sched_mode)
6399	{	6466	{
6400	struct task_struct prev, next;	6467	struct task_struct prev, next;
		6468	/*
		6469	* On PREEMPT_RT kernel, SM_RTLOCK_WAIT is noted
		6470	* as a preemption by schedule_debug() and RCU.
		6471	*/
		6472	bool preempt = sched_mode > SM_NONE;
6401	unsigned long *switch_count;	6473	unsigned long *switch_count;
6402	unsigned long prev_state;	6474	unsigned long prev_state;
6403	struct rq_flags rf;	6475	struct rq_flags rf;
@@ -6408,13 +6480,13 @@ static void __sched notrace __schedule(unsigned int sched_mode) · 4 unmodified lines
6408	rq = cpu_rq(cpu);	6480	rq = cpu_rq(cpu);
6409	prev = rq->curr;	6481	prev = rq->curr;
6410		6482
6411	schedule_debug(prev, !!sched_mode);	6483	schedule_debug(prev, preempt);
6412		6484
6413	if (sched_feat(HRTICK) \|\| sched_feat(HRTICK_DL))	6485	if (sched_feat(HRTICK) \|\| sched_feat(HRTICK_DL))
6414	hrtick_clear(rq);	6486	hrtick_clear(rq);
6415		6487
6416	local_irq_disable();	6488	local_irq_disable();
6417	rcu_note_context_switch(!!sched_mode);	6489	rcu_note_context_switch(preempt);
6418		6490
6419	/*	6491	/*
6420	* Make sure that signal_pending_state()->signal_pending() below	6492	* Make sure that signal_pending_state()->signal_pending() below
@@ -6443,22 +6515,32 @@ static void __sched notrace __schedule(unsigned int sched_mode) · 22 unmodified lines
6443		6515
6444	switch_count = &prev->nivcsw;	6516	switch_count = &prev->nivcsw;
6445		6517
		6518	/* Task state changes only considers SM_PREEMPT as preemption */
		6519	preempt = sched_mode == SM_PREEMPT;
		6520
6446	/*	6521	/*
6447	* We must load prev->state once (task_struct::state is volatile), such	6522	* We must load prev->state once (task_struct::state is volatile), such
6448	* that we form a control dependency vs deactivate_task() below.	6523	* that we form a control dependency vs deactivate_task() below.
6449	*/	6524	*/
6450	prev_state = READ_ONCE(prev->__state);	6525	prev_state = READ_ONCE(prev->__state);
6451	if (!(sched_mode & SM_MASK_PREEMPT) && prev_state) {	6526	if (sched_mode == SM_IDLE) {
		6527	if (!rq->nr_running) {
		6528	next = prev;
		6529	goto picked;
		6530	}
		6531	} else if (!preempt && prev_state) {
6452	if (signal_pending_state(prev_state, prev)) {	6532	if (signal_pending_state(prev_state, prev)) {
6453	WRITE_ONCE(prev->__state, TASK_RUNNING);	6533	WRITE_ONCE(prev->__state, TASK_RUNNING);
6454	} else {	6534	} else {
		6535	int flags = DEQUEUE_NOCLOCK;
		6536
6455	prev->sched_contributes_to_load =	6537	prev->sched_contributes_to_load =
6456	(prev_state & TASK_UNINTERRUPTIBLE) &&	6538	(prev_state & TASK_UNINTERRUPTIBLE) &&
6457	!(prev_state & TASK_NOLOAD) &&	6539	!(prev_state & TASK_NOLOAD) &&
6458	!(prev_state & TASK_FROZEN);	6540	!(prev_state & TASK_FROZEN);
6459		6541
6460	if (prev->sched_contributes_to_load)	6542	if (unlikely(is_special_task_state(prev_state)))
6461	rq->nr_uninterruptible++;	6543	flags \|= DEQUEUE_SPECIAL;
6462		6544
6463	/*	6545	/*
6464	* __schedule() ttwu()	6546	* __schedule() ttwu()
@@ -6471,17 +6553,13 @@ static void __sched notrace __schedule(unsigned int sched_mode) · 6 unmodified lines
6471	*	6553	*
6472	* After this, schedule() must not care about p->state any more.	6554	* After this, schedule() must not care about p->state any more.
6473	*/	6555	*/
6474	deactivate_task(rq, prev, DEQUEUE_SLEEP \| DEQUEUE_NOCLOCK);	6556	block_task(rq, prev, flags);
6475
6476	if (prev->in_iowait) {
6477	atomic_inc(&rq->nr_iowait);
6478	delayacct_blkio_start();
6479	}
6480	}	6557	}
6481	switch_count = &prev->nvcsw;	6558	switch_count = &prev->nvcsw;
6482	}	6559	}
6483		6560
6484	next = pick_next_task(rq, prev, &rf);	6561	next = pick_next_task(rq, prev, &rf);
		6562	picked:
6485	clear_tsk_need_resched(prev);	6563	clear_tsk_need_resched(prev);
6486	clear_preempt_need_resched();	6564	clear_preempt_need_resched();
6487	#ifdef CONFIG_SCHED_DEBUG	6565	#ifdef CONFIG_SCHED_DEBUG
@@ -6523,7 +6601,7 @@ static void __sched notrace __schedule(unsigned int sched_mode) · 35 unmodified lines
6523	psi_account_irqtime(rq, prev, next);	6601	psi_account_irqtime(rq, prev, next);
6524	psi_sched_switch(prev, next, !task_on_rq_queued(prev));	6602	psi_sched_switch(prev, next, !task_on_rq_queued(prev));
6525		6603
6526	trace_sched_switch(sched_mode & SM_MASK_PREEMPT, prev, next, prev_state);	6604	trace_sched_switch(preempt, prev, next, prev_state);
6527		6605
6528	/* Also unlocks the rq: */	6606	/* Also unlocks the rq: */
6529	rq = context_switch(rq, prev, next, &rf);	6607	rq = context_switch(rq, prev, next, &rf);
@@ -6599,7 +6677,7 @@ static void sched_update_worker(struct task_struct *tsk) · 69 unmodified lines
6599	}	6677	}
6600	}	6678	}
6601		6679
6602	static __always_inline void __schedule_loop(unsigned int sched_mode)	6680	static __always_inline void __schedule_loop(int sched_mode)
6603	{	6681	{
6604	do {	6682	do {
6605	preempt_disable();	6683	preempt_disable();
@@ -6644,7 +6722,7 @@ void __sched schedule_idle(void) · 38 unmodified lines
6644	*/	6722	*/
6645	WARN_ON_ONCE(current->__state);	6723	WARN_ON_ONCE(current->__state);
6646	do {	6724	do {
6647	__schedule(SM_NONE);	6725	__schedule(SM_IDLE);
6648	} while (need_resched());	6726	} while (need_resched());
6649	}	6727	}
6650		6728
@@ -8228,8 +8306,6 @@ void __init sched_init(void) · 1577 unmodified lines
8228	#endif /* CONFIG_RT_GROUP_SCHED */	8306	#endif /* CONFIG_RT_GROUP_SCHED */
8229	}	8307	}
8230		8308
8231	init_rt_bandwidth(&def_rt_bandwidth, global_rt_period(), global_rt_runtime());
8232
8233	#ifdef CONFIG_SMP	8309	#ifdef CONFIG_SMP
8234	init_defrootdomain();	8310	init_defrootdomain();
8235	#endif	8311	#endif
@@ -8284,8 +8360,13 @@ void __init sched_init(void) · 48 unmodified lines
8284	init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL);	8360	init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL);
8285	#endif /* CONFIG_FAIR_GROUP_SCHED */	8361	#endif /* CONFIG_FAIR_GROUP_SCHED */
8286		8362
8287	rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime;
8288	#ifdef CONFIG_RT_GROUP_SCHED	8363	#ifdef CONFIG_RT_GROUP_SCHED
		8364	/*
		8365	* This is required for init cpu because rt.c:__enable_runtime()
		8366	* starts working after scheduler_running, which is not the case
		8367	* yet.
		8368	*/
		8369	rq->rt.rt_runtime = global_rt_runtime();
8289	init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL);	8370	init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL);
8290	#endif	8371	#endif
8291	#ifdef CONFIG_SMP	8372	#ifdef CONFIG_SMP
@@ -8317,10 +8398,12 @@ void __init sched_init(void) · 25 unmodified lines
8317	#endif /* CONFIG_SMP */	8398	#endif /* CONFIG_SMP */
8318	hrtick_rq_init(rq);	8399	hrtick_rq_init(rq);
8319	atomic_set(&rq->nr_iowait, 0);	8400	atomic_set(&rq->nr_iowait, 0);
		8401	fair_server_init(rq);
8320		8402
8321	#ifdef CONFIG_SCHED_CORE	8403	#ifdef CONFIG_SCHED_CORE
8322	rq->core = rq;	8404	rq->core = rq;
8323	rq->core_pick = NULL;	8405	rq->core_pick = NULL;
		8406	rq->core_dl_server = NULL;
8324	rq->core_enabled = 0;	8407	rq->core_enabled = 0;
8325	rq->core_tree = RB_ROOT;	8408	rq->core_tree = RB_ROOT;
8326	rq->core_forceidle_count = 0;	8409	rq->core_forceidle_count = 0;
@@ -8333,6 +8416,7 @@ void __init sched_init(void) · 6 unmodified lines
8333	}	8416	}
8334		8417
8335	set_load_weight(&init_task, false);	8418	set_load_weight(&init_task, false);
		8419	init_task.se.slice = sysctl_sched_base_slice,
8336		8420
8337	/*	8421	/*
8338	* The boot idle thread does lazy MMU switching as well:	8422	* The boot idle thread does lazy MMU switching as well:
@@ -8548,7 +8632,7 @@ void normalize_rt_tasks(void) · 209 unmodified lines
8548	schedstat_set(p->stats.sleep_start, 0);	8632	schedstat_set(p->stats.sleep_start, 0);
8549	schedstat_set(p->stats.block_start, 0);	8633	schedstat_set(p->stats.block_start, 0);
8550		8634
8551	if (!dl_task(p) && !rt_task(p)) {	8635	if (!rt_or_dl_task(p)) {
8552	/*	8636	/*
8553	* Renice negative nice level userspace	8637	* Renice negative nice level userspace
8554	* tasks back to 0:	8638	* tasks back to 0:

kernel/sched/cpufreq_schedutil.c+3−3

@@ -654,9 +654,9 @@ static int sugov_kthread_create(struct sugov_policy *sg_policy) · 653 unmodified lines
654	* Fake (unused) bandwidth; workaround to "fix"	654	* Fake (unused) bandwidth; workaround to "fix"
655	* priority inheritance.	655	* priority inheritance.
656	*/	656	*/
657	.sched_runtime = 1000000,	657	.sched_runtime = NSEC_PER_MSEC,
658	.sched_deadline = 10000000,	658	.sched_deadline = 10 * NSEC_PER_MSEC,
659	.sched_period = 10000000,	659	.sched_period = 10 * NSEC_PER_MSEC,
660	};	660	};
661	struct cpufreq_policy *policy = sg_policy->policy;	661	struct cpufreq_policy *policy = sg_policy->policy;
662	int ret;	662	int ret;

kernel/sched/deadline.c+403−100

@@ -320,19 +320,12 @@ void sub_running_bw(struct sched_dl_entity dl_se, struct dl_rq dl_rq) · 319 unmodified lines
320	__sub_running_bw(dl_se->dl_bw, dl_rq);	320	__sub_running_bw(dl_se->dl_bw, dl_rq);
321	}	321	}
322		322
323	static void dl_change_utilization(struct task_struct *p, u64 new_bw)	323	static void dl_rq_change_utilization(struct rq rq, struct sched_dl_entity dl_se, u64 new_bw)
324	{	324	{
325	struct rq *rq;	325	if (dl_se->dl_non_contending) {
326		326	sub_running_bw(dl_se, &rq->dl);
327	WARN_ON_ONCE(p->dl.flags & SCHED_FLAG_SUGOV);	327	dl_se->dl_non_contending = 0;
328
329	if (task_on_rq_queued(p))
330	return;
331		328
332	rq = task_rq(p);
333	if (p->dl.dl_non_contending) {
334	sub_running_bw(&p->dl, &rq->dl);
335	p->dl.dl_non_contending = 0;
336	/*	329	/*
337	* If the timer handler is currently running and the	330	* If the timer handler is currently running and the
338	* timer cannot be canceled, inactive_task_timer()	331	* timer cannot be canceled, inactive_task_timer()
@@ -340,13 +333,25 @@ static void dl_change_utilization(struct task_struct *p, u64 new_bw) · 1 unmodified line
340	* will not touch the rq's active utilization,	333	* will not touch the rq's active utilization,
341	* so we are still safe.	334	* so we are still safe.
342	*/	335	*/
343	if (hrtimer_try_to_cancel(&p->dl.inactive_timer) == 1)	336	if (hrtimer_try_to_cancel(&dl_se->inactive_timer) == 1) {
344	put_task_struct(p);	337	if (!dl_server(dl_se))
		338	put_task_struct(dl_task_of(dl_se));
		339	}
345	}	340	}
346	__sub_rq_bw(p->dl.dl_bw, &rq->dl);	341	__sub_rq_bw(dl_se->dl_bw, &rq->dl);
347	__add_rq_bw(new_bw, &rq->dl);	342	__add_rq_bw(new_bw, &rq->dl);
348	}	343	}
349		344
		345	static void dl_change_utilization(struct task_struct *p, u64 new_bw)
		346	{
		347	WARN_ON_ONCE(p->dl.flags & SCHED_FLAG_SUGOV);
		348
		349	if (task_on_rq_queued(p))
		350	return;
		351
		352	dl_rq_change_utilization(task_rq(p), &p->dl, new_bw);
		353	}
		354
350	static void __dl_clear_params(struct sched_dl_entity *dl_se);	355	static void __dl_clear_params(struct sched_dl_entity *dl_se);
351		356
352	/*	357	/*
@@ -771,6 +776,15 @@ static inline void replenish_dl_new_period(struct sched_dl_entity *dl_se, · 418 unmodified lines
771	/* for non-boosted task, pi_of(dl_se) == dl_se */	776	/* for non-boosted task, pi_of(dl_se) == dl_se */
772	dl_se->deadline = rq_clock(rq) + pi_of(dl_se)->dl_deadline;	777	dl_se->deadline = rq_clock(rq) + pi_of(dl_se)->dl_deadline;
773	dl_se->runtime = pi_of(dl_se)->dl_runtime;	778	dl_se->runtime = pi_of(dl_se)->dl_runtime;
		779
		780	/*
		781	* If it is a deferred reservation, and the server
		782	* is not handling an starvation case, defer it.
		783	*/
		784	if (dl_se->dl_defer & !dl_se->dl_defer_running) {
		785	dl_se->dl_throttled = 1;
		786	dl_se->dl_defer_armed = 1;
		787	}
774	}	788	}
775		789
776	/*	790	/*
@@ -809,6 +823,9 @@ static inline void setup_new_dl_entity(struct sched_dl_entity *dl_se) · 32 unmodified lines
809	replenish_dl_new_period(dl_se, rq);	823	replenish_dl_new_period(dl_se, rq);
810	}	824	}
811		825
		826	static int start_dl_timer(struct sched_dl_entity *dl_se);
		827	static bool dl_entity_overflow(struct sched_dl_entity *dl_se, u64 t);
		828
812	/*	829	/*
813	* Pure Earliest Deadline First (EDF) scheduling does not deal with the	830	* Pure Earliest Deadline First (EDF) scheduling does not deal with the
814	* possibility of a entity lasting more than what it declared, and thus	831	* possibility of a entity lasting more than what it declared, and thus
@@ -837,9 +854,18 @@ static void replenish_dl_entity(struct sched_dl_entity *dl_se) · 22 unmodified lines
837	/*	854	/*
838	* This could be the case for a !-dl task that is boosted.	855	* This could be the case for a !-dl task that is boosted.
839	* Just go with full inherited parameters.	856	* Just go with full inherited parameters.
		857	*
		858	* Or, it could be the case of a deferred reservation that
		859	* was not able to consume its runtime in background and
		860	* reached this point with current u > U.
		861	*
		862	* In both cases, set a new period.
840	*/	863	*/
841	if (dl_se->dl_deadline == 0)	864	if (dl_se->dl_deadline == 0 \|\|
842	replenish_dl_new_period(dl_se, rq);	865	(dl_se->dl_defer_armed && dl_entity_overflow(dl_se, rq_clock(rq)))) {
		866	dl_se->deadline = rq_clock(rq) + pi_of(dl_se)->dl_deadline;
		867	dl_se->runtime = pi_of(dl_se)->dl_runtime;
		868	}
843		869
844	if (dl_se->dl_yielded && dl_se->runtime > 0)	870	if (dl_se->dl_yielded && dl_se->runtime > 0)
845	dl_se->runtime = 0;	871	dl_se->runtime = 0;
@@ -873,6 +899,44 @@ static void replenish_dl_entity(struct sched_dl_entity *dl_se) · 27 unmodified lines
873	dl_se->dl_yielded = 0;	899	dl_se->dl_yielded = 0;
874	if (dl_se->dl_throttled)	900	if (dl_se->dl_throttled)
875	dl_se->dl_throttled = 0;	901	dl_se->dl_throttled = 0;
		902
		903	/*
		904	* If this is the replenishment of a deferred reservation,
		905	* clear the flag and return.
		906	*/
		907	if (dl_se->dl_defer_armed) {
		908	dl_se->dl_defer_armed = 0;
		909	return;
		910	}
		911
		912	/*
		913	* A this point, if the deferred server is not armed, and the deadline
		914	* is in the future, if it is not running already, throttle the server
		915	* and arm the defer timer.
		916	*/
		917	if (dl_se->dl_defer && !dl_se->dl_defer_running &&
		918	dl_time_before(rq_clock(dl_se->rq), dl_se->deadline - dl_se->runtime)) {
		919	if (!is_dl_boosted(dl_se) && dl_se->server_has_tasks(dl_se)) {
		920
		921	/*
		922	* Set dl_se->dl_defer_armed and dl_throttled variables to
		923	* inform the start_dl_timer() that this is a deferred
		924	* activation.
		925	*/
		926	dl_se->dl_defer_armed = 1;
		927	dl_se->dl_throttled = 1;
		928	if (!start_dl_timer(dl_se)) {
		929	/*
		930	* If for whatever reason (delays), a previous timer was
		931	* queued but not serviced, cancel it and clean the
		932	* deferrable server variables intended for start_dl_timer().
		933	*/
		934	hrtimer_try_to_cancel(&dl_se->dl_timer);
		935	dl_se->dl_defer_armed = 0;
		936	dl_se->dl_throttled = 0;
		937	}
		938	}
		939	}
876	}	940	}
877		941
878	/*	942	/*
@@ -1023,6 +1087,15 @@ static void update_dl_entity(struct sched_dl_entity *dl_se) · 144 unmodified lines
1023	}	1087	}
1024		1088
1025	replenish_dl_new_period(dl_se, rq);	1089	replenish_dl_new_period(dl_se, rq);
		1090	} else if (dl_server(dl_se) && dl_se->dl_defer) {
		1091	/*
		1092	* The server can still use its previous deadline, so check if
		1093	* it left the dl_defer_running state.
		1094	*/
		1095	if (!dl_se->dl_defer_running) {
		1096	dl_se->dl_defer_armed = 1;
		1097	dl_se->dl_throttled = 1;
		1098	}
1026	}	1099	}
1027	}	1100	}
1028		1101
@@ -1055,8 +1128,21 @@ static int start_dl_timer(struct sched_dl_entity *dl_se) · 26 unmodified lines
1055	* We want the timer to fire at the deadline, but considering	1128	* We want the timer to fire at the deadline, but considering
1056	* that it is actually coming from rq->clock and not from	1129	* that it is actually coming from rq->clock and not from
1057	* hrtimer's time base reading.	1130	* hrtimer's time base reading.
		1131	*
		1132	* The deferred reservation will have its timer set to
		1133	* (deadline - runtime). At that point, the CBS rule will decide
		1134	* if the current deadline can be used, or if a replenishment is
		1135	* required to avoid add too much pressure on the system
		1136	* (current u > U).
1058	*/	1137	*/
1059	act = ns_to_ktime(dl_next_period(dl_se));	1138	if (dl_se->dl_defer_armed) {
		1139	WARN_ON_ONCE(!dl_se->dl_throttled);
		1140	act = ns_to_ktime(dl_se->deadline - dl_se->runtime);
		1141	} else {
		1142	/* act = deadline - rel-deadline + period */
		1143	act = ns_to_ktime(dl_next_period(dl_se));
		1144	}
		1145
1060	now = hrtimer_cb_get_time(timer);	1146	now = hrtimer_cb_get_time(timer);
1061	delta = ktime_to_ns(now) - rq_clock(rq);	1147	delta = ktime_to_ns(now) - rq_clock(rq);
1062	act = ktime_add_ns(act, delta);	1148	act = ktime_add_ns(act, delta);
@@ -1106,6 +1192,62 @@ static void __push_dl_task(struct rq rq, struct rq_flags rf) · 43 unmodified lines
1106	#endif	1192	#endif
1107	}	1193	}
1108		1194
		1195	/* a defer timer will not be reset if the runtime consumed was < dl_server_min_res */
		1196	static const u64 dl_server_min_res = 1 * NSEC_PER_MSEC;
		1197
		1198	static enum hrtimer_restart dl_server_timer(struct hrtimer timer, struct sched_dl_entity dl_se)
		1199	{
		1200	struct rq *rq = rq_of_dl_se(dl_se);
		1201	u64 fw;
		1202
		1203	scoped_guard (rq_lock, rq) {
		1204	struct rq_flags *rf = &scope.rf;
		1205
		1206	if (!dl_se->dl_throttled \|\| !dl_se->dl_runtime)
		1207	return HRTIMER_NORESTART;
		1208
		1209	sched_clock_tick();
		1210	update_rq_clock(rq);
		1211
		1212	if (!dl_se->dl_runtime)
		1213	return HRTIMER_NORESTART;
		1214
		1215	if (!dl_se->server_has_tasks(dl_se)) {
		1216	replenish_dl_entity(dl_se);
		1217	return HRTIMER_NORESTART;
		1218	}
		1219
		1220	if (dl_se->dl_defer_armed) {
		1221	/*
		1222	* First check if the server could consume runtime in background.
		1223	* If so, it is possible to push the defer timer for this amount
		1224	* of time. The dl_server_min_res serves as a limit to avoid
		1225	* forwarding the timer for a too small amount of time.
		1226	*/
		1227	if (dl_time_before(rq_clock(dl_se->rq),
		1228	(dl_se->deadline - dl_se->runtime - dl_server_min_res))) {
		1229
		1230	/* reset the defer timer */
		1231	fw = dl_se->deadline - rq_clock(dl_se->rq) - dl_se->runtime;
		1232
		1233	hrtimer_forward_now(timer, ns_to_ktime(fw));
		1234	return HRTIMER_RESTART;
		1235	}
		1236
		1237	dl_se->dl_defer_running = 1;
		1238	}
		1239
		1240	enqueue_dl_entity(dl_se, ENQUEUE_REPLENISH);
		1241
		1242	if (!dl_task(dl_se->rq->curr) \|\| dl_entity_preempt(dl_se, &dl_se->rq->curr->dl))
		1243	resched_curr(rq);
		1244
		1245	__push_dl_task(rq, rf);
		1246	}
		1247
		1248	return HRTIMER_NORESTART;
		1249	}
		1250
1109	/*	1251	/*
1110	* This is the bandwidth enforcement timer callback. If here, we know	1252	* This is the bandwidth enforcement timer callback. If here, we know
1111	* a task is not on its dl_rq, since the fact that the timer was running	1253	* a task is not on its dl_rq, since the fact that the timer was running
@@ -1128,28 +1270,8 @@ static enum hrtimer_restart dl_task_timer(struct hrtimer *timer) · 16 unmodified lines
1128	struct rq_flags rf;	1270	struct rq_flags rf;
1129	struct rq *rq;	1271	struct rq *rq;
1130		1272
1131	if (dl_server(dl_se)) {	1273	if (dl_server(dl_se))
1132	struct rq *rq = rq_of_dl_se(dl_se);	1274	return dl_server_timer(timer, dl_se);
1133	struct rq_flags rf;
1134
1135	rq_lock(rq, &rf);
1136	if (dl_se->dl_throttled) {
1137	sched_clock_tick();
1138	update_rq_clock(rq);
1139
1140	if (dl_se->server_has_tasks(dl_se)) {
1141	enqueue_dl_entity(dl_se, ENQUEUE_REPLENISH);
1142	resched_curr(rq);
1143	__push_dl_task(rq, &rf);
1144	} else {
1145	replenish_dl_entity(dl_se);
1146	}
1147
1148	}
1149	rq_unlock(rq, &rf);
1150
1151	return HRTIMER_NORESTART;
1152	}
1153		1275
1154	p = dl_task_of(dl_se);	1276	p = dl_task_of(dl_se);
1155	rq = task_rq_lock(p, &rf);	1277	rq = task_rq_lock(p, &rf);
@@ -1319,22 +1441,10 @@ static u64 grub_reclaim(u64 delta, struct rq rq, struct sched_dl_entity dl_se) · 163 unmodified lines
1319	return (delta * u_act) >> BW_SHIFT;	1441	return (delta * u_act) >> BW_SHIFT;
1320	}	1442	}
1321		1443
1322	static inline void	1444	s64 dl_scaled_delta_exec(struct rq rq, struct sched_dl_entity dl_se, s64 delta_exec)
1323	update_stats_dequeue_dl(struct dl_rq dl_rq, struct sched_dl_entity dl_se,
1324	int flags);
1325	static void update_curr_dl_se(struct rq rq, struct sched_dl_entity dl_se, s64 delta_exec)
1326	{	1445	{
1327	s64 scaled_delta_exec;	1446	s64 scaled_delta_exec;
1328		1447
1329	if (unlikely(delta_exec <= 0)) {
1330	if (unlikely(dl_se->dl_yielded))
1331	goto throttle;
1332	return;
1333	}
1334
1335	if (dl_entity_is_special(dl_se))
1336	return;
1337
1338	/*	1448	/*
1339	* For tasks that participate in GRUB, we implement GRUB-PA: the	1449	* For tasks that participate in GRUB, we implement GRUB-PA: the
1340	* spare reclaimed bandwidth is used to clock down frequency.	1450	* spare reclaimed bandwidth is used to clock down frequency.
@@ -1353,8 +1463,64 @@ static void update_curr_dl_se(struct rq rq, struct sched_dl_entity dl_se, s64 · 12 unmodified lines
1353	scaled_delta_exec = cap_scale(scaled_delta_exec, scale_cpu);	1463	scaled_delta_exec = cap_scale(scaled_delta_exec, scale_cpu);
1354	}	1464	}
1355		1465
		1466	return scaled_delta_exec;
		1467	}
		1468
		1469	static inline void
		1470	update_stats_dequeue_dl(struct dl_rq dl_rq, struct sched_dl_entity dl_se,
		1471	int flags);
		1472	static void update_curr_dl_se(struct rq rq, struct sched_dl_entity dl_se, s64 delta_exec)
		1473	{
		1474	s64 scaled_delta_exec;
		1475
		1476	if (unlikely(delta_exec <= 0)) {
		1477	if (unlikely(dl_se->dl_yielded))
		1478	goto throttle;
		1479	return;
		1480	}
		1481
		1482	if (dl_server(dl_se) && dl_se->dl_throttled && !dl_se->dl_defer)
		1483	return;
		1484
		1485	if (dl_entity_is_special(dl_se))
		1486	return;
		1487
		1488	scaled_delta_exec = dl_scaled_delta_exec(rq, dl_se, delta_exec);
		1489
1356	dl_se->runtime -= scaled_delta_exec;	1490	dl_se->runtime -= scaled_delta_exec;
1357		1491
		1492	/*
		1493	* The fair server can consume its runtime while throttled (not queued/
		1494	* running as regular CFS).
		1495	*
		1496	* If the server consumes its entire runtime in this state. The server
		1497	* is not required for the current period. Thus, reset the server by
		1498	* starting a new period, pushing the activation.
		1499	*/
		1500	if (dl_se->dl_defer && dl_se->dl_throttled && dl_runtime_exceeded(dl_se)) {
		1501	/*
		1502	* If the server was previously activated - the starving condition
		1503	* took place, it this point it went away because the fair scheduler
		1504	* was able to get runtime in background. So return to the initial
		1505	* state.
		1506	*/
		1507	dl_se->dl_defer_running = 0;
		1508
		1509	hrtimer_try_to_cancel(&dl_se->dl_timer);
		1510
		1511	replenish_dl_new_period(dl_se, dl_se->rq);
		1512
		1513	/*
		1514	* Not being able to start the timer seems problematic. If it could not
		1515	* be started for whatever reason, we need to "unthrottle" the DL server
		1516	* and queue right away. Otherwise nothing might queue it. That's similar
		1517	* to what enqueue_dl_entity() does on start_dl_timer==0. For now, just warn.
		1518	*/
		1519	WARN_ON_ONCE(!start_dl_timer(dl_se));
		1520
		1521	return;
		1522	}
		1523
1358	throttle:	1524	throttle:
1359	if (dl_runtime_exceeded(dl_se) \|\| dl_se->dl_yielded) {	1525	if (dl_runtime_exceeded(dl_se) \|\| dl_se->dl_yielded) {
1360	dl_se->dl_throttled = 1;	1526	dl_se->dl_throttled = 1;
@@ -1381,6 +1547,14 @@ throttle: · 20 unmodified lines
1381	resched_curr(rq);	1547	resched_curr(rq);
1382	}	1548	}
1383		1549
		1550	/*
		1551	* The fair server (sole dl_server) does not account for real-time
		1552	* workload because it is running fair work.
		1553	*/
		1554	if (dl_se == &rq->fair_server)
		1555	return;
		1556
		1557	#ifdef CONFIG_RT_GROUP_SCHED
1384	/*	1558	/*
1385	* Because -- for now -- we share the rt bandwidth, we need to	1559	* Because -- for now -- we share the rt bandwidth, we need to
1386	* account our runtime there too, otherwise actual rt tasks	1560	* account our runtime there too, otherwise actual rt tasks
@@ -1405,34 +1579,155 @@ throttle: · 18 unmodified lines
1405	rt_rq->rt_time += delta_exec;	1579	rt_rq->rt_time += delta_exec;
1406	raw_spin_unlock(&rt_rq->rt_runtime_lock);	1580	raw_spin_unlock(&rt_rq->rt_runtime_lock);
1407	}	1581	}
		1582	#endif
		1583	}
		1584
		1585	/*
		1586	* In the non-defer mode, the idle time is not accounted, as the
		1587	* server provides a guarantee.
		1588	*
		1589	* If the dl_server is in defer mode, the idle time is also considered
		1590	* as time available for the fair server, avoiding a penalty for the
		1591	* rt scheduler that did not consumed that time.
		1592	*/
		1593	void dl_server_update_idle_time(struct rq rq, struct task_struct p)
		1594	{
		1595	s64 delta_exec, scaled_delta_exec;
		1596
		1597	if (!rq->fair_server.dl_defer)
		1598	return;
		1599
		1600	/* no need to discount more */
		1601	if (rq->fair_server.runtime < 0)
		1602	return;
		1603
		1604	delta_exec = rq_clock_task(rq) - p->se.exec_start;
		1605	if (delta_exec < 0)
		1606	return;
		1607
		1608	scaled_delta_exec = dl_scaled_delta_exec(rq, &rq->fair_server, delta_exec);
		1609
		1610	rq->fair_server.runtime -= scaled_delta_exec;
		1611
		1612	if (rq->fair_server.runtime < 0) {
		1613	rq->fair_server.dl_defer_running = 0;
		1614	rq->fair_server.runtime = 0;
		1615	}
		1616
		1617	p->se.exec_start = rq_clock_task(rq);
1408	}	1618	}
1409		1619
1410	void dl_server_update(struct sched_dl_entity *dl_se, s64 delta_exec)	1620	void dl_server_update(struct sched_dl_entity *dl_se, s64 delta_exec)
1411	{	1621	{
1412	update_curr_dl_se(dl_se->rq, dl_se, delta_exec);	1622	/* 0 runtime = fair server disabled */
		1623	if (dl_se->dl_runtime)
		1624	update_curr_dl_se(dl_se->rq, dl_se, delta_exec);
1413	}	1625	}
1414		1626
1415	void dl_server_start(struct sched_dl_entity *dl_se)	1627	void dl_server_start(struct sched_dl_entity *dl_se)
1416	{	1628	{
		1629	struct rq *rq = dl_se->rq;
		1630
		1631	/*
		1632	* XXX: the apply do not work fine at the init phase for the
		1633	* fair server because things are not yet set. We need to improve
		1634	* this before getting generic.
		1635	*/
1417	if (!dl_server(dl_se)) {	1636	if (!dl_server(dl_se)) {
		1637	u64 runtime = 50 * NSEC_PER_MSEC;
		1638	u64 period = 1000 * NSEC_PER_MSEC;
		1639
		1640	dl_server_apply_params(dl_se, runtime, period, 1);
		1641
1418	dl_se->dl_server = 1;	1642	dl_se->dl_server = 1;
		1643	dl_se->dl_defer = 1;
1419	setup_new_dl_entity(dl_se);	1644	setup_new_dl_entity(dl_se);
1420	}	1645	}
		1646
		1647	if (!dl_se->dl_runtime)
		1648	return;
		1649
1421	enqueue_dl_entity(dl_se, ENQUEUE_WAKEUP);	1650	enqueue_dl_entity(dl_se, ENQUEUE_WAKEUP);
		1651	if (!dl_task(dl_se->rq->curr) \|\| dl_entity_preempt(dl_se, &rq->curr->dl))
		1652	resched_curr(dl_se->rq);
1422	}	1653	}
1423		1654
1424	void dl_server_stop(struct sched_dl_entity *dl_se)	1655	void dl_server_stop(struct sched_dl_entity *dl_se)
1425	{	1656	{
		1657	if (!dl_se->dl_runtime)
		1658	return;
		1659
1426	dequeue_dl_entity(dl_se, DEQUEUE_SLEEP);	1660	dequeue_dl_entity(dl_se, DEQUEUE_SLEEP);
		1661	hrtimer_try_to_cancel(&dl_se->dl_timer);
		1662	dl_se->dl_defer_armed = 0;
		1663	dl_se->dl_throttled = 0;
1427	}	1664	}
1428		1665
1429	void dl_server_init(struct sched_dl_entity dl_se, struct rq rq,	1666	void dl_server_init(struct sched_dl_entity dl_se, struct rq rq,
1430	dl_server_has_tasks_f has_tasks,	1667	dl_server_has_tasks_f has_tasks,
1431	dl_server_pick_f pick)	1668	dl_server_pick_f pick_task)
1432	{	1669	{
1433	dl_se->rq = rq;	1670	dl_se->rq = rq;
1434	dl_se->server_has_tasks = has_tasks;	1671	dl_se->server_has_tasks = has_tasks;
1435	dl_se->server_pick = pick;	1672	dl_se->server_pick_task = pick_task;
		1673	}
		1674
		1675	void __dl_server_attach_root(struct sched_dl_entity dl_se, struct rq rq)
		1676	{
		1677	u64 new_bw = dl_se->dl_bw;
		1678	int cpu = cpu_of(rq);
		1679	struct dl_bw *dl_b;
		1680
		1681	dl_b = dl_bw_of(cpu_of(rq));
		1682	guard(raw_spinlock)(&dl_b->lock);
		1683
		1684	if (!dl_bw_cpus(cpu))
		1685	return;
		1686
		1687	__dl_add(dl_b, new_bw, dl_bw_cpus(cpu));
		1688	}
		1689
		1690	int dl_server_apply_params(struct sched_dl_entity *dl_se, u64 runtime, u64 period, bool init)
		1691	{
		1692	u64 old_bw = init ? 0 : to_ratio(dl_se->dl_period, dl_se->dl_runtime);
		1693	u64 new_bw = to_ratio(period, runtime);
		1694	struct rq *rq = dl_se->rq;
		1695	int cpu = cpu_of(rq);
		1696	struct dl_bw *dl_b;
		1697	unsigned long cap;
		1698	int retval = 0;
		1699	int cpus;
		1700
		1701	dl_b = dl_bw_of(cpu);
		1702	guard(raw_spinlock)(&dl_b->lock);
		1703
		1704	cpus = dl_bw_cpus(cpu);
		1705	cap = dl_bw_capacity(cpu);
		1706
		1707	if (__dl_overflow(dl_b, cap, old_bw, new_bw))
		1708	return -EBUSY;
		1709
		1710	if (init) {
		1711	__add_rq_bw(new_bw, &rq->dl);
		1712	__dl_add(dl_b, new_bw, cpus);
		1713	} else {
		1714	__dl_sub(dl_b, dl_se->dl_bw, cpus);
		1715	__dl_add(dl_b, new_bw, cpus);
		1716
		1717	dl_rq_change_utilization(rq, dl_se, new_bw);
		1718	}
		1719
		1720	dl_se->dl_runtime = runtime;
		1721	dl_se->dl_deadline = period;
		1722	dl_se->dl_period = period;
		1723
		1724	dl_se->runtime = 0;
		1725	dl_se->deadline = 0;
		1726
		1727	dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime);
		1728	dl_se->dl_density = to_ratio(dl_se->dl_deadline, dl_se->dl_runtime);
		1729
		1730	return retval;
1436	}	1731	}
1437		1732
1438	/*	1733	/*
@@ -1599,46 +1894,40 @@ static inline bool __dl_less(struct rb_node a, const struct rb_node b) · 160 unmodified lines
1599	return dl_time_before(__node_2_dle(a)->deadline, __node_2_dle(b)->deadline);	1894	return dl_time_before(__node_2_dle(a)->deadline, __node_2_dle(b)->deadline);
1600	}	1895	}
1601		1896
1602	static inline struct sched_statistics *	1897	static __always_inline struct sched_statistics *
1603	__schedstats_from_dl_se(struct sched_dl_entity *dl_se)	1898	__schedstats_from_dl_se(struct sched_dl_entity *dl_se)
1604	{	1899	{
		1900	if (!schedstat_enabled())
		1901	return NULL;
		1902
		1903	if (dl_server(dl_se))
		1904	return NULL;
		1905
1605	return &dl_task_of(dl_se)->stats;	1906	return &dl_task_of(dl_se)->stats;
1606	}	1907	}
1607		1908
1608	static inline void	1909	static inline void
1609	update_stats_wait_start_dl(struct dl_rq dl_rq, struct sched_dl_entity dl_se)	1910	update_stats_wait_start_dl(struct dl_rq dl_rq, struct sched_dl_entity dl_se)
1610	{	1911	{
1611	struct sched_statistics *stats;	1912	struct sched_statistics *stats = __schedstats_from_dl_se(dl_se);
1612		1913	if (stats)
1613	if (!schedstat_enabled())	1914	__update_stats_wait_start(rq_of_dl_rq(dl_rq), dl_task_of(dl_se), stats);
1614	return;
1615
1616	stats = __schedstats_from_dl_se(dl_se);
1617	__update_stats_wait_start(rq_of_dl_rq(dl_rq), dl_task_of(dl_se), stats);
1618	}	1915	}
1619		1916
1620	static inline void	1917	static inline void
1621	update_stats_wait_end_dl(struct dl_rq dl_rq, struct sched_dl_entity dl_se)	1918	update_stats_wait_end_dl(struct dl_rq dl_rq, struct sched_dl_entity dl_se)
1622	{	1919	{
1623	struct sched_statistics *stats;	1920	struct sched_statistics *stats = __schedstats_from_dl_se(dl_se);
1624		1921	if (stats)
1625	if (!schedstat_enabled())	1922	__update_stats_wait_end(rq_of_dl_rq(dl_rq), dl_task_of(dl_se), stats);
1626	return;
1627
1628	stats = __schedstats_from_dl_se(dl_se);
1629	__update_stats_wait_end(rq_of_dl_rq(dl_rq), dl_task_of(dl_se), stats);
1630	}	1923	}
1631		1924
1632	static inline void	1925	static inline void
1633	update_stats_enqueue_sleeper_dl(struct dl_rq dl_rq, struct sched_dl_entity dl_se)	1926	update_stats_enqueue_sleeper_dl(struct dl_rq dl_rq, struct sched_dl_entity dl_se)
1634	{	1927	{
1635	struct sched_statistics *stats;	1928	struct sched_statistics *stats = __schedstats_from_dl_se(dl_se);
1636		1929	if (stats)
1637	if (!schedstat_enabled())	1930	__update_stats_enqueue_sleeper(rq_of_dl_rq(dl_rq), dl_task_of(dl_se), stats);
1638	return;
1639
1640	stats = __schedstats_from_dl_se(dl_se);
1641	__update_stats_enqueue_sleeper(rq_of_dl_rq(dl_rq), dl_task_of(dl_se), stats);
1642	}	1931	}
1643		1932
1644	static inline void	1933	static inline void
@@ -1735,7 +2024,7 @@ enqueue_dl_entity(struct sched_dl_entity *dl_se, int flags) · 90 unmodified lines
1735	* be counted in the active utilization; hence, we need to call	2024	* be counted in the active utilization; hence, we need to call
1736	* add_running_bw().	2025	* add_running_bw().
1737	*/	2026	*/
1738	if (dl_se->dl_throttled && !(flags & ENQUEUE_REPLENISH)) {	2027	if (!dl_se->dl_defer && dl_se->dl_throttled && !(flags & ENQUEUE_REPLENISH)) {
1739	if (flags & ENQUEUE_WAKEUP)	2028	if (flags & ENQUEUE_WAKEUP)
1740	task_contending(dl_se, flags);	2029	task_contending(dl_se, flags);
1741		2030
@@ -1757,6 +2046,25 @@ enqueue_dl_entity(struct sched_dl_entity *dl_se, int flags) · 15 unmodified lines
1757	setup_new_dl_entity(dl_se);	2046	setup_new_dl_entity(dl_se);
1758	}	2047	}
1759		2048
		2049	/*
		2050	* If the reservation is still throttled, e.g., it got replenished but is a
		2051	* deferred task and still got to wait, don't enqueue.
		2052	*/
		2053	if (dl_se->dl_throttled && start_dl_timer(dl_se))
		2054	return;
		2055
		2056	/*
		2057	* We're about to enqueue, make sure we're not ->dl_throttled!
		2058	* In case the timer was not started, say because the defer time
		2059	* has passed, mark as not throttled and mark unarmed.
		2060	* Also cancel earlier timers, since letting those run is pointless.
		2061	*/
		2062	if (dl_se->dl_throttled) {
		2063	hrtimer_try_to_cancel(&dl_se->dl_timer);
		2064	dl_se->dl_defer_armed = 0;
		2065	dl_se->dl_throttled = 0;
		2066	}
		2067
1760	__enqueue_dl_entity(dl_se);	2068	__enqueue_dl_entity(dl_se);
1761	}	2069	}
1762		2070
@@ -1846,7 +2154,7 @@ static void enqueue_task_dl(struct rq rq, struct task_struct p, int flags) · 83 unmodified lines
1846	enqueue_pushable_dl_task(rq, p);	2154	enqueue_pushable_dl_task(rq, p);
1847	}	2155	}
1848		2156
1849	static void dequeue_task_dl(struct rq rq, struct task_struct p, int flags)	2157	static bool dequeue_task_dl(struct rq rq, struct task_struct p, int flags)
1850	{	2158	{
1851	update_curr_dl(rq);	2159	update_curr_dl(rq);
1852		2160
@@ -1856,6 +2164,8 @@ static void dequeue_task_dl(struct rq rq, struct task_struct p, int flags) · 3 unmodified lines
1856	dequeue_dl_entity(&p->dl, flags);	2164	dequeue_dl_entity(&p->dl, flags);
1857	if (!p->dl.dl_throttled && !dl_server(&p->dl))	2165	if (!p->dl.dl_throttled && !dl_server(&p->dl))
1858	dequeue_pushable_dl_task(rq, p);	2166	dequeue_pushable_dl_task(rq, p);
		2167
		2168	return true;
1859	}	2169	}
1860		2170
1861	/*	2171	/*
@@ -2074,6 +2384,9 @@ static void set_next_task_dl(struct rq rq, struct task_struct p, bool first) · 212 unmodified lines
2074	update_dl_rq_load_avg(rq_clock_pelt(rq), rq, 0);	2384	update_dl_rq_load_avg(rq_clock_pelt(rq), rq, 0);
2075		2385
2076	deadline_queue_push_tasks(rq);	2386	deadline_queue_push_tasks(rq);
		2387
		2388	if (hrtick_enabled(rq))
		2389	start_hrtick_dl(rq, &p->dl);
2077	}	2390	}
2078		2391
2079	static struct sched_dl_entity pick_next_dl_entity(struct dl_rq dl_rq)	2392	static struct sched_dl_entity pick_next_dl_entity(struct dl_rq dl_rq)
@@ -2086,7 +2399,11 @@ static struct sched_dl_entity pick_next_dl_entity(struct dl_rq dl_rq) · 6 unmodified lines
2086	return __node_2_dle(left);	2399	return __node_2_dle(left);
2087	}	2400	}
2088		2401
2089	static struct task_struct pick_task_dl(struct rq rq)	2402	/*
		2403	* __pick_next_task_dl - Helper to pick the next -deadline task to run.
		2404	* @rq: The runqueue to pick the next task from.
		2405	*/
		2406	static struct task_struct __pick_task_dl(struct rq rq)
2090	{	2407	{
2091	struct sched_dl_entity *dl_se;	2408	struct sched_dl_entity *dl_se;
2092	struct dl_rq *dl_rq = &rq->dl;	2409	struct dl_rq *dl_rq = &rq->dl;
@@ -2100,14 +2417,13 @@ again: · 7 unmodified lines
2100	WARN_ON_ONCE(!dl_se);	2417	WARN_ON_ONCE(!dl_se);
2101		2418
2102	if (dl_server(dl_se)) {	2419	if (dl_server(dl_se)) {
2103	p = dl_se->server_pick(dl_se);	2420	p = dl_se->server_pick_task(dl_se);
2104	if (!p) {	2421	if (!p) {
2105	WARN_ON_ONCE(1);
2106	dl_se->dl_yielded = 1;	2422	dl_se->dl_yielded = 1;
2107	update_curr_dl_se(rq, dl_se, 0);	2423	update_curr_dl_se(rq, dl_se, 0);
2108	goto again;	2424	goto again;
2109	}	2425	}
2110	p->dl_server = dl_se;	2426	rq->dl_server = dl_se;
2111	} else {	2427	} else {
2112	p = dl_task_of(dl_se);	2428	p = dl_task_of(dl_se);
2113	}	2429	}
@@ -2115,24 +2431,12 @@ again: · 1 unmodified line
2115	return p;	2431	return p;
2116	}	2432	}
2117		2433
2118	static struct task_struct pick_next_task_dl(struct rq rq)	2434	static struct task_struct pick_task_dl(struct rq rq)
2119	{	2435	{
2120	struct task_struct *p;	2436	return __pick_task_dl(rq);
2121
2122	p = pick_task_dl(rq);
2123	if (!p)
2124	return p;
2125
2126	if (!p->dl_server)
2127	set_next_task_dl(rq, p, true);
2128
2129	if (hrtick_enabled(rq))
2130	start_hrtick_dl(rq, &p->dl);
2131
2132	return p;
2133	}	2437	}
2134		2438
2135	static void put_prev_task_dl(struct rq rq, struct task_struct p)	2439	static void put_prev_task_dl(struct rq rq, struct task_struct p, struct task_struct *next)
2136	{	2440	{
2137	struct sched_dl_entity *dl_se = &p->dl;	2441	struct sched_dl_entity *dl_se = &p->dl;
2138	struct dl_rq *dl_rq = &rq->dl;	2442	struct dl_rq *dl_rq = &rq->dl;
@@ -2824,13 +3128,12 @@ DEFINE_SCHED_CLASS(dl) = { · 685 unmodified lines
2824		3128
2825	.wakeup_preempt = wakeup_preempt_dl,	3129	.wakeup_preempt = wakeup_preempt_dl,
2826		3130
2827	.pick_next_task = pick_next_task_dl,	3131	.pick_task = pick_task_dl,
2828	.put_prev_task = put_prev_task_dl,	3132	.put_prev_task = put_prev_task_dl,
2829	.set_next_task = set_next_task_dl,	3133	.set_next_task = set_next_task_dl,
2830		3134
2831	#ifdef CONFIG_SMP	3135	#ifdef CONFIG_SMP
2832	.balance = balance_dl,	3136	.balance = balance_dl,
2833	.pick_task = pick_task_dl,
2834	.select_task_rq = select_task_rq_dl,	3137	.select_task_rq = select_task_rq_dl,
2835	.migrate_task_rq = migrate_task_rq_dl,	3138	.migrate_task_rq = migrate_task_rq_dl,
2836	.set_cpus_allowed = set_cpus_allowed_dl,	3139	.set_cpus_allowed = set_cpus_allowed_dl,

kernel/sched/debug.c+186−12

@@ -333,8 +333,165 @@ static const struct file_operations sched_debug_fops = { · 332 unmodified lines
333	.release = seq_release,	333	.release = seq_release,
334	};	334	};
335		335
		336	enum dl_param {
		337	DL_RUNTIME = 0,
		338	DL_PERIOD,
		339	};
		340
		341	static unsigned long fair_server_period_max = (1UL << 22) * NSEC_PER_USEC; /* ~4 seconds */
		342	static unsigned long fair_server_period_min = (100) * NSEC_PER_USEC; /* 100 us */
		343
		344	static ssize_t sched_fair_server_write(struct file filp, const char __user ubuf,
		345	size_t cnt, loff_t *ppos, enum dl_param param)
		346	{
		347	long cpu = (long) ((struct seq_file *) filp->private_data)->private;
		348	struct rq *rq = cpu_rq(cpu);
		349	u64 runtime, period;
		350	size_t err;
		351	int retval;
		352	u64 value;
		353
		354	err = kstrtoull_from_user(ubuf, cnt, 10, &value);
		355	if (err)
		356	return err;
		357
		358	scoped_guard (rq_lock_irqsave, rq) {
		359	runtime = rq->fair_server.dl_runtime;
		360	period = rq->fair_server.dl_period;
		361
		362	switch (param) {
		363	case DL_RUNTIME:
		364	if (runtime == value)
		365	break;
		366	runtime = value;
		367	break;
		368	case DL_PERIOD:
		369	if (value == period)
		370	break;
		371	period = value;
		372	break;
		373	}
		374
		375	if (runtime > period \|\|
		376	period > fair_server_period_max \|\|
		377	period < fair_server_period_min) {
		378	return -EINVAL;
		379	}
		380
		381	if (rq->cfs.h_nr_running) {
		382	update_rq_clock(rq);
		383	dl_server_stop(&rq->fair_server);
		384	}
		385
		386	retval = dl_server_apply_params(&rq->fair_server, runtime, period, 0);
		387	if (retval)
		388	cnt = retval;
		389
		390	if (!runtime)
		391	printk_deferred("Fair server disabled in CPU %d, system may crash due to starvation.\n",
		392	cpu_of(rq));
		393
		394	if (rq->cfs.h_nr_running)
		395	dl_server_start(&rq->fair_server);
		396	}
		397
		398	*ppos += cnt;
		399	return cnt;
		400	}
		401
		402	static size_t sched_fair_server_show(struct seq_file m, void v, enum dl_param param)
		403	{
		404	unsigned long cpu = (unsigned long) m->private;
		405	struct rq *rq = cpu_rq(cpu);
		406	u64 value;
		407
		408	switch (param) {
		409	case DL_RUNTIME:
		410	value = rq->fair_server.dl_runtime;
		411	break;
		412	case DL_PERIOD:
		413	value = rq->fair_server.dl_period;
		414	break;
		415	}
		416
		417	seq_printf(m, "%llu\n", value);
		418	return 0;
		419
		420	}
		421
		422	static ssize_t
		423	sched_fair_server_runtime_write(struct file filp, const char __user ubuf,
		424	size_t cnt, loff_t *ppos)
		425	{
		426	return sched_fair_server_write(filp, ubuf, cnt, ppos, DL_RUNTIME);
		427	}
		428
		429	static int sched_fair_server_runtime_show(struct seq_file m, void v)
		430	{
		431	return sched_fair_server_show(m, v, DL_RUNTIME);
		432	}
		433
		434	static int sched_fair_server_runtime_open(struct inode inode, struct file filp)
		435	{
		436	return single_open(filp, sched_fair_server_runtime_show, inode->i_private);
		437	}
		438
		439	static const struct file_operations fair_server_runtime_fops = {
		440	.open = sched_fair_server_runtime_open,
		441	.write = sched_fair_server_runtime_write,
		442	.read = seq_read,
		443	.llseek = seq_lseek,
		444	.release = single_release,
		445	};
		446
		447	static ssize_t
		448	sched_fair_server_period_write(struct file filp, const char __user ubuf,
		449	size_t cnt, loff_t *ppos)
		450	{
		451	return sched_fair_server_write(filp, ubuf, cnt, ppos, DL_PERIOD);
		452	}
		453
		454	static int sched_fair_server_period_show(struct seq_file m, void v)
		455	{
		456	return sched_fair_server_show(m, v, DL_PERIOD);
		457	}
		458
		459	static int sched_fair_server_period_open(struct inode inode, struct file filp)
		460	{
		461	return single_open(filp, sched_fair_server_period_show, inode->i_private);
		462	}
		463
		464	static const struct file_operations fair_server_period_fops = {
		465	.open = sched_fair_server_period_open,
		466	.write = sched_fair_server_period_write,
		467	.read = seq_read,
		468	.llseek = seq_lseek,
		469	.release = single_release,
		470	};
		471
336	static struct dentry *debugfs_sched;	472	static struct dentry *debugfs_sched;
337		473
		474	static void debugfs_fair_server_init(void)
		475	{
		476	struct dentry *d_fair;
		477	unsigned long cpu;
		478
		479	d_fair = debugfs_create_dir("fair_server", debugfs_sched);
		480	if (!d_fair)
		481	return;
		482
		483	for_each_possible_cpu(cpu) {
		484	struct dentry *d_cpu;
		485	char buf[32];
		486
		487	snprintf(buf, sizeof(buf), "cpu%lu", cpu);
		488	d_cpu = debugfs_create_dir(buf, d_fair);
		489
		490	debugfs_create_file("runtime", 0644, d_cpu, (void *) cpu, &fair_server_runtime_fops);
		491	debugfs_create_file("period", 0644, d_cpu, (void *) cpu, &fair_server_period_fops);
		492	}
		493	}
		494
338	static __init int sched_init_debug(void)	495	static __init int sched_init_debug(void)
339	{	496	{
340	struct dentry __maybe_unused *numa;	497	struct dentry __maybe_unused *numa;
@@ -374,6 +531,8 @@ static __init int sched_init_debug(void) · 33 unmodified lines
374		531
375	debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops);	532	debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops);
376		533
		534	debugfs_fair_server_init();
		535
377	return 0;	536	return 0;
378	}	537	}
379	late_initcall(sched_init_debug);	538	late_initcall(sched_init_debug);
@@ -580,27 +739,27 @@ print_task(struct seq_file m, struct rq rq, struct task_struct *p) · 200 unmodified lines
580	else	739	else
581	SEQ_printf(m, " %c", task_state_to_char(p));	740	SEQ_printf(m, " %c", task_state_to_char(p));
582		741
583	SEQ_printf(m, "%15s %5d %9Ld.%06ld %c %9Ld.%06ld %9Ld.%06ld %9Ld.%06ld %9Ld %5d ",	742	SEQ_printf(m, " %15s %5d %9Ld.%06ld %c %9Ld.%06ld %c %9Ld.%06ld %9Ld.%06ld %9Ld %5d ",
584	p->comm, task_pid_nr(p),	743	p->comm, task_pid_nr(p),
585	SPLIT_NS(p->se.vruntime),	744	SPLIT_NS(p->se.vruntime),
586	entity_eligible(cfs_rq_of(&p->se), &p->se) ? 'E' : 'N',	745	entity_eligible(cfs_rq_of(&p->se), &p->se) ? 'E' : 'N',
587	SPLIT_NS(p->se.deadline),	746	SPLIT_NS(p->se.deadline),
		747	p->se.custom_slice ? 'S' : ' ',
588	SPLIT_NS(p->se.slice),	748	SPLIT_NS(p->se.slice),
589	SPLIT_NS(p->se.sum_exec_runtime),	749	SPLIT_NS(p->se.sum_exec_runtime),
590	(long long)(p->nvcsw + p->nivcsw),	750	(long long)(p->nvcsw + p->nivcsw),
591	p->prio);	751	p->prio);
592		752
593	SEQ_printf(m, "%9lld.%06ld %9lld.%06ld %9lld.%06ld %9lld.%06ld",	753	SEQ_printf(m, "%9lld.%06ld %9lld.%06ld %9lld.%06ld",
594	SPLIT_NS(schedstat_val_or_zero(p->stats.wait_sum)),	754	SPLIT_NS(schedstat_val_or_zero(p->stats.wait_sum)),
595	SPLIT_NS(p->se.sum_exec_runtime),
596	SPLIT_NS(schedstat_val_or_zero(p->stats.sum_sleep_runtime)),	755	SPLIT_NS(schedstat_val_or_zero(p->stats.sum_sleep_runtime)),
597	SPLIT_NS(schedstat_val_or_zero(p->stats.sum_block_runtime)));	756	SPLIT_NS(schedstat_val_or_zero(p->stats.sum_block_runtime)));
598		757
599	#ifdef CONFIG_NUMA_BALANCING	758	#ifdef CONFIG_NUMA_BALANCING
600	SEQ_printf(m, " %d %d", task_node(p), task_numa_group_id(p));	759	SEQ_printf(m, " %d %d", task_node(p), task_numa_group_id(p));
601	#endif	760	#endif
602	#ifdef CONFIG_CGROUP_SCHED	761	#ifdef CONFIG_CGROUP_SCHED
603	SEQ_printf_task_group_path(m, task_group(p), " %s")	762	SEQ_printf_task_group_path(m, task_group(p), " %s")
604	#endif	763	#endif
605		764
606	SEQ_printf(m, "\n");	765	SEQ_printf(m, "\n");
@@ -612,10 +771,26 @@ static void print_rq(struct seq_file m, struct rq rq, int rq_cpu) · 5 unmodified lines
612		771
613	SEQ_printf(m, "\n");	772	SEQ_printf(m, "\n");
614	SEQ_printf(m, "runnable tasks:\n");	773	SEQ_printf(m, "runnable tasks:\n");
615	SEQ_printf(m, " S task PID tree-key switches prio"	774	SEQ_printf(m, " S task PID vruntime eligible "
616	" wait-time sum-exec sum-sleep\n");	775	"deadline slice sum-exec switches "
		776	"prio wait-time sum-sleep sum-block"
		777	#ifdef CONFIG_NUMA_BALANCING
		778	" node group-id"
		779	#endif
		780	#ifdef CONFIG_CGROUP_SCHED
		781	" group-path"
		782	#endif
		783	"\n");
617	SEQ_printf(m, "-------------------------------------------------------"	784	SEQ_printf(m, "-------------------------------------------------------"
618	"------------------------------------------------------\n");	785	"------------------------------------------------------"
		786	"------------------------------------------------------"
		787	#ifdef CONFIG_NUMA_BALANCING
		788	"--------------"
		789	#endif
		790	#ifdef CONFIG_CGROUP_SCHED
		791	"--------------"
		792	#endif
		793	"\n");
619		794
620	rcu_read_lock();	795	rcu_read_lock();
621	for_each_process_thread(g, p) {	796	for_each_process_thread(g, p) {
@@ -641,8 +816,6 @@ void print_cfs_rq(struct seq_file m, int cpu, struct cfs_rq cfs_rq) · 19 unmodified lines
641	SEQ_printf(m, "\n");	816	SEQ_printf(m, "\n");
642	SEQ_printf(m, "cfs_rq[%d]:\n", cpu);	817	SEQ_printf(m, "cfs_rq[%d]:\n", cpu);
643	#endif	818	#endif
644	SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "exec_clock",
645	SPLIT_NS(cfs_rq->exec_clock));
646		819
647	raw_spin_rq_lock_irqsave(rq, flags);	820	raw_spin_rq_lock_irqsave(rq, flags);
648	root = __pick_root_entity(cfs_rq);	821	root = __pick_root_entity(cfs_rq);
@@ -669,8 +842,6 @@ void print_cfs_rq(struct seq_file m, int cpu, struct cfs_rq cfs_rq) · 20 unmodified lines
669	SPLIT_NS(right_vruntime));	842	SPLIT_NS(right_vruntime));
670	spread = right_vruntime - left_vruntime;	843	spread = right_vruntime - left_vruntime;
671	SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "spread", SPLIT_NS(spread));	844	SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "spread", SPLIT_NS(spread));
672	SEQ_printf(m, " .%-30s: %d\n", "nr_spread_over",
673	cfs_rq->nr_spread_over);
674	SEQ_printf(m, " .%-30s: %d\n", "nr_running", cfs_rq->nr_running);	845	SEQ_printf(m, " .%-30s: %d\n", "nr_running", cfs_rq->nr_running);
675	SEQ_printf(m, " .%-30s: %d\n", "h_nr_running", cfs_rq->h_nr_running);	846	SEQ_printf(m, " .%-30s: %d\n", "h_nr_running", cfs_rq->h_nr_running);
676	SEQ_printf(m, " .%-30s: %d\n", "idle_nr_running",	847	SEQ_printf(m, " .%-30s: %d\n", "idle_nr_running",
@@ -730,9 +901,12 @@ void print_rt_rq(struct seq_file m, int cpu, struct rt_rq rt_rq) · 53 unmodified lines
730	SEQ_printf(m, " .%-30s: %Ld.%06ld\n", #x, SPLIT_NS(rt_rq->x))	901	SEQ_printf(m, " .%-30s: %Ld.%06ld\n", #x, SPLIT_NS(rt_rq->x))
731		902
732	PU(rt_nr_running);	903	PU(rt_nr_running);
		904
		905	#ifdef CONFIG_RT_GROUP_SCHED
733	P(rt_throttled);	906	P(rt_throttled);
734	PN(rt_time);	907	PN(rt_time);
735	PN(rt_runtime);	908	PN(rt_runtime);
		909	#endif
736		910
737	#undef PN	911	#undef PN
738	#undef PU	912	#undef PU

kernel/sched/fair.c+580−190

@@ -511,7 +511,7 @@ static int cfs_rq_is_idle(struct cfs_rq *cfs_rq) · 510 unmodified lines
511		511
512	static int se_is_idle(struct sched_entity *se)	512	static int se_is_idle(struct sched_entity *se)
513	{	513	{
514	return 0;	514	return task_has_idle_policy(task_of(se));
515	}	515	}
516		516
517	#endif /* CONFIG_FAIR_GROUP_SCHED */	517	#endif /* CONFIG_FAIR_GROUP_SCHED */
@@ -779,8 +779,22 @@ static void update_min_vruntime(struct cfs_rq *cfs_rq) · 261 unmodified lines
779	}	779	}
780		780
781	/* ensure we never gain time by being placed backwards. */	781	/* ensure we never gain time by being placed backwards. */
782	u64_u32_store(cfs_rq->min_vruntime,	782	cfs_rq->min_vruntime = __update_min_vruntime(cfs_rq, vruntime);
783	__update_min_vruntime(cfs_rq, vruntime));	783	}
		784
		785	static inline u64 cfs_rq_min_slice(struct cfs_rq *cfs_rq)
		786	{
		787	struct sched_entity *root = __pick_root_entity(cfs_rq);
		788	struct sched_entity *curr = cfs_rq->curr;
		789	u64 min_slice = ~0ULL;
		790
		791	if (curr && curr->on_rq)
		792	min_slice = curr->slice;
		793
		794	if (root)
		795	min_slice = min(min_slice, root->min_slice);
		796
		797	return min_slice;
784	}	798	}
785		799
786	static inline bool __entity_less(struct rb_node a, const struct rb_node b)	800	static inline bool __entity_less(struct rb_node a, const struct rb_node b)
@@ -799,19 +813,34 @@ static inline void __min_vruntime_update(struct sched_entity *se, struct rb_node · 12 unmodified lines
799	}	813	}
800	}	814	}
801		815
		816	static inline void __min_slice_update(struct sched_entity se, struct rb_node node)
		817	{
		818	if (node) {
		819	struct sched_entity *rse = __node_2_se(node);
		820	if (rse->min_slice < se->min_slice)
		821	se->min_slice = rse->min_slice;
		822	}
		823	}
		824
802	/*	825	/*
803	* se->min_vruntime = min(se->vruntime, {left,right}->min_vruntime)	826	* se->min_vruntime = min(se->vruntime, {left,right}->min_vruntime)
804	*/	827	*/
805	static inline bool min_vruntime_update(struct sched_entity *se, bool exit)	828	static inline bool min_vruntime_update(struct sched_entity *se, bool exit)
806	{	829	{
807	u64 old_min_vruntime = se->min_vruntime;	830	u64 old_min_vruntime = se->min_vruntime;
		831	u64 old_min_slice = se->min_slice;
808	struct rb_node *node = &se->run_node;	832	struct rb_node *node = &se->run_node;
809		833
810	se->min_vruntime = se->vruntime;	834	se->min_vruntime = se->vruntime;
811	__min_vruntime_update(se, node->rb_right);	835	__min_vruntime_update(se, node->rb_right);
812	__min_vruntime_update(se, node->rb_left);	836	__min_vruntime_update(se, node->rb_left);
813		837
814	return se->min_vruntime == old_min_vruntime;	838	se->min_slice = se->slice;
		839	__min_slice_update(se, node->rb_right);
		840	__min_slice_update(se, node->rb_left);
		841
		842	return se->min_vruntime == old_min_vruntime &&
		843	se->min_slice == old_min_slice;
815	}	844	}
816		845
817	RB_DECLARE_CALLBACKS(static, min_vruntime_cb, struct sched_entity,	846	RB_DECLARE_CALLBACKS(static, min_vruntime_cb, struct sched_entity,
@@ -824,6 +853,7 @@ static void __enqueue_entity(struct cfs_rq cfs_rq, struct sched_entity se) · 6 unmodified lines
824	{	853	{
825	avg_vruntime_add(cfs_rq, se);	854	avg_vruntime_add(cfs_rq, se);
826	se->min_vruntime = se->vruntime;	855	se->min_vruntime = se->vruntime;
		856	se->min_slice = se->slice;
827	rb_add_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline,	857	rb_add_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline,
828	__entity_less, &min_vruntime_cb);	858	__entity_less, &min_vruntime_cb);
829	}	859	}
@@ -974,17 +1004,18 @@ static void clear_buddies(struct cfs_rq cfs_rq, struct sched_entity se); · 144 unmodified lines
974	* XXX: strictly: vd_i += N*r_i/w_i such that: vd_i > ve_i	1004	* XXX: strictly: vd_i += N*r_i/w_i such that: vd_i > ve_i
975	* this is probably good enough.	1005	* this is probably good enough.
976	*/	1006	*/
977	static void update_deadline(struct cfs_rq cfs_rq, struct sched_entity se)	1007	static bool update_deadline(struct cfs_rq cfs_rq, struct sched_entity se)
978	{	1008	{
979	if ((s64)(se->vruntime - se->deadline) < 0)	1009	if ((s64)(se->vruntime - se->deadline) < 0)
980	return;	1010	return false;
981		1011
982	/*	1012	/*
983	* For EEVDF the virtual time slope is determined by w_i (iow.	1013	* For EEVDF the virtual time slope is determined by w_i (iow.
984	* nice) while the request time r_i is determined by	1014	* nice) while the request time r_i is determined by
985	* sysctl_sched_base_slice.	1015	* sysctl_sched_base_slice.
986	*/	1016	*/
987	se->slice = sysctl_sched_base_slice;	1017	if (!se->custom_slice)
		1018	se->slice = sysctl_sched_base_slice;
988		1019
989	/*	1020	/*
990	* EEVDF: vd_i = ve_i + r_i / w_i	1021	* EEVDF: vd_i = ve_i + r_i / w_i
@@ -994,10 +1025,7 @@ static void update_deadline(struct cfs_rq cfs_rq, struct sched_entity se) · 3 unmodified lines
994	/*	1025	/*
995	* The task has consumed its request, reschedule.	1026	* The task has consumed its request, reschedule.
996	*/	1027	*/
997	if (cfs_rq->nr_running > 1) {	1028	return true;
998	resched_curr(rq_of(cfs_rq));
999	clear_buddies(cfs_rq, se);
1000	}
1001	}	1029	}
1002		1030
1003	#include "pelt.h"	1031	#include "pelt.h"
@@ -1135,6 +1163,38 @@ static inline void update_curr_task(struct task_struct *p, s64 delta_exec) · 131 unmodified lines
1135	dl_server_update(p->dl_server, delta_exec);	1163	dl_server_update(p->dl_server, delta_exec);
1136	}	1164	}
1137		1165
		1166	static inline bool did_preempt_short(struct cfs_rq cfs_rq, struct sched_entity curr)
		1167	{
		1168	if (!sched_feat(PREEMPT_SHORT))
		1169	return false;
		1170
		1171	if (curr->vlag == curr->deadline)
		1172	return false;
		1173
		1174	return !entity_eligible(cfs_rq, curr);
		1175	}
		1176
		1177	static inline bool do_preempt_short(struct cfs_rq *cfs_rq,
		1178	struct sched_entity pse, struct sched_entity se)
		1179	{
		1180	if (!sched_feat(PREEMPT_SHORT))
		1181	return false;
		1182
		1183	if (pse->slice >= se->slice)
		1184	return false;
		1185
		1186	if (!entity_eligible(cfs_rq, pse))
		1187	return false;
		1188
		1189	if (entity_before(pse, se))
		1190	return true;
		1191
		1192	if (!entity_eligible(cfs_rq, se))
		1193	return true;
		1194
		1195	return false;
		1196	}
		1197
1138	/*	1198	/*
1139	* Used by other classes to account runtime.	1199	* Used by other classes to account runtime.
1140	*/	1200	*/
@@ -1156,23 +1216,44 @@ s64 update_curr_common(struct rq *rq) · 15 unmodified lines
1156	static void update_curr(struct cfs_rq *cfs_rq)	1216	static void update_curr(struct cfs_rq *cfs_rq)
1157	{	1217	{
1158	struct sched_entity *curr = cfs_rq->curr;	1218	struct sched_entity *curr = cfs_rq->curr;
		1219	struct rq *rq = rq_of(cfs_rq);
1159	s64 delta_exec;	1220	s64 delta_exec;
		1221	bool resched;
1160		1222
1161	if (unlikely(!curr))	1223	if (unlikely(!curr))
1162	return;	1224	return;
1163		1225
1164	delta_exec = update_curr_se(rq_of(cfs_rq), curr);	1226	delta_exec = update_curr_se(rq, curr);
1165	if (unlikely(delta_exec <= 0))	1227	if (unlikely(delta_exec <= 0))
1166	return;	1228	return;
1167		1229
1168	curr->vruntime += calc_delta_fair(delta_exec, curr);	1230	curr->vruntime += calc_delta_fair(delta_exec, curr);
1169	update_deadline(cfs_rq, curr);	1231	resched = update_deadline(cfs_rq, curr);
1170	update_min_vruntime(cfs_rq);	1232	update_min_vruntime(cfs_rq);
1171		1233
1172	if (entity_is_task(curr))	1234	if (entity_is_task(curr)) {
1173	update_curr_task(task_of(curr), delta_exec);	1235	struct task_struct *p = task_of(curr);
		1236
		1237	update_curr_task(p, delta_exec);
		1238
		1239	/*
		1240	* Any fair task that runs outside of fair_server should
		1241	* account against fair_server such that it can account for
		1242	* this time and possibly avoid running this period.
		1243	*/
		1244	if (p->dl_server != &rq->fair_server)
		1245	dl_server_update(&rq->fair_server, delta_exec);
		1246	}
1174		1247
1175	account_cfs_rq_runtime(cfs_rq, delta_exec);	1248	account_cfs_rq_runtime(cfs_rq, delta_exec);
		1249
		1250	if (rq->nr_running == 1)
		1251	return;
		1252
		1253	if (resched \|\| did_preempt_short(cfs_rq, curr)) {
		1254	resched_curr(rq);
		1255	clear_buddies(cfs_rq, curr);
		1256	}
1176	}	1257	}
1177		1258
1178	static void update_curr_fair(struct rq *rq)	1259	static void update_curr_fair(struct rq *rq)
@@ -5178,7 +5259,8 @@ place_entity(struct cfs_rq cfs_rq, struct sched_entity se, int flags) · 3999 unmodified lines
5178	u64 vslice, vruntime = avg_vruntime(cfs_rq);	5259	u64 vslice, vruntime = avg_vruntime(cfs_rq);
5179	s64 lag = 0;	5260	s64 lag = 0;
5180		5261
5181	se->slice = sysctl_sched_base_slice;	5262	if (!se->custom_slice)
		5263	se->slice = sysctl_sched_base_slice;
5182	vslice = calc_delta_fair(se->slice, se);	5264	vslice = calc_delta_fair(se->slice, se);
5183		5265
5184	/*	5266	/*
@@ -5259,6 +5341,12 @@ place_entity(struct cfs_rq cfs_rq, struct sched_entity se, int flags) · 74 unmodified lines
5259		5341
5260	se->vruntime = vruntime - lag;	5342	se->vruntime = vruntime - lag;
5261		5343
		5344	if (sched_feat(PLACE_REL_DEADLINE) && se->rel_deadline) {
		5345	se->deadline += se->vruntime;
		5346	se->rel_deadline = 0;
		5347	return;
		5348	}
		5349
5262	/*	5350	/*
5263	* When joining the competition; the existing tasks will be,	5351	* When joining the competition; the existing tasks will be,
5264	* on average, halfway through their slice, as such start tasks	5352	* on average, halfway through their slice, as such start tasks
@@ -5278,6 +5366,9 @@ static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq); · 13 unmodified lines
5278		5366
5279	static inline bool cfs_bandwidth_used(void);	5367	static inline bool cfs_bandwidth_used(void);
5280		5368
		5369	static void
		5370	requeue_delayed_entity(struct sched_entity *se);
		5371
5281	static void	5372	static void
5282	enqueue_entity(struct cfs_rq cfs_rq, struct sched_entity se, int flags)	5373	enqueue_entity(struct cfs_rq cfs_rq, struct sched_entity se, int flags)
5283	{	5374	{
@@ -5365,19 +5456,47 @@ static void clear_buddies(struct cfs_rq cfs_rq, struct sched_entity se) · 81 unmodified lines
5365		5456
5366	static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq);	5457	static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq);
5367		5458
5368	static void	5459	static inline void finish_delayed_dequeue_entity(struct sched_entity *se)
		5460	{
		5461	se->sched_delayed = 0;
		5462	if (sched_feat(DELAY_ZERO) && se->vlag > 0)
		5463	se->vlag = 0;
		5464	}
		5465
		5466	static bool
5369	dequeue_entity(struct cfs_rq cfs_rq, struct sched_entity se, int flags)	5467	dequeue_entity(struct cfs_rq cfs_rq, struct sched_entity se, int flags)
5370	{	5468	{
5371	int action = UPDATE_TG;	5469	bool sleep = flags & DEQUEUE_SLEEP;
5372		5470
		5471	update_curr(cfs_rq);
		5472
		5473	if (flags & DEQUEUE_DELAYED) {
		5474	SCHED_WARN_ON(!se->sched_delayed);
		5475	} else {
		5476	bool delay = sleep;
		5477	/*
		5478	* DELAY_DEQUEUE relies on spurious wakeups, special task
		5479	* states must not suffer spurious wakeups, excempt them.
		5480	*/
		5481	if (flags & DEQUEUE_SPECIAL)
		5482	delay = false;
		5483
		5484	SCHED_WARN_ON(delay && se->sched_delayed);
		5485
		5486	if (sched_feat(DELAY_DEQUEUE) && delay &&
		5487	!entity_eligible(cfs_rq, se)) {
		5488	if (cfs_rq->next == se)
		5489	cfs_rq->next = NULL;
		5490	update_load_avg(cfs_rq, se, 0);
		5491	se->sched_delayed = 1;
		5492	return false;
		5493	}
		5494	}
		5495
		5496	int action = UPDATE_TG;
5373	if (entity_is_task(se) && task_on_rq_migrating(task_of(se)))	5497	if (entity_is_task(se) && task_on_rq_migrating(task_of(se)))
5374	action \|= DO_DETACH;	5498	action \|= DO_DETACH;
5375		5499
5376	/*
5377	* Update run-time statistics of the 'current'.
5378	*/
5379	update_curr(cfs_rq);
5380
5381	/*	5500	/*
5382	* When dequeuing a sched_entity, we must:	5501	* When dequeuing a sched_entity, we must:
5383	* - Update loads to have both entity and cfs_rq synced with now.	5502	* - Update loads to have both entity and cfs_rq synced with now.
@@ -5395,6 +5514,11 @@ dequeue_entity(struct cfs_rq cfs_rq, struct sched_entity se, int flags) · 11 unmodified lines
5395	clear_buddies(cfs_rq, se);	5514	clear_buddies(cfs_rq, se);
5396		5515
5397	update_entity_lag(cfs_rq, se);	5516	update_entity_lag(cfs_rq, se);
		5517	if (sched_feat(PLACE_REL_DEADLINE) && !sleep) {
		5518	se->deadline -= se->vruntime;
		5519	se->rel_deadline = 1;
		5520	}
		5521
5398	if (se != cfs_rq->curr)	5522	if (se != cfs_rq->curr)
5399	__dequeue_entity(cfs_rq, se);	5523	__dequeue_entity(cfs_rq, se);
5400	se->on_rq = 0;	5524	se->on_rq = 0;
@@ -5414,8 +5538,13 @@ dequeue_entity(struct cfs_rq cfs_rq, struct sched_entity se, int flags) · 13 unmodified lines
5414	if ((flags & (DEQUEUE_SAVE \| DEQUEUE_MOVE)) != DEQUEUE_SAVE)	5538	if ((flags & (DEQUEUE_SAVE \| DEQUEUE_MOVE)) != DEQUEUE_SAVE)
5415	update_min_vruntime(cfs_rq);	5539	update_min_vruntime(cfs_rq);
5416		5540
		5541	if (flags & DEQUEUE_DELAYED)
		5542	finish_delayed_dequeue_entity(se);
		5543
5417	if (cfs_rq->nr_running == 0)	5544	if (cfs_rq->nr_running == 0)
5418	update_idle_cfs_rq_clock_pelt(cfs_rq);	5545	update_idle_cfs_rq_clock_pelt(cfs_rq);
		5546
		5547	return true;
5419	}	5548	}
5420		5549
5421	static void	5550	static void
@@ -5441,6 +5570,7 @@ set_next_entity(struct cfs_rq cfs_rq, struct sched_entity se) · 19 unmodified lines
5441	}	5570	}
5442		5571
5443	update_stats_curr_start(cfs_rq, se);	5572	update_stats_curr_start(cfs_rq, se);
		5573	SCHED_WARN_ON(cfs_rq->curr);
5444	cfs_rq->curr = se;	5574	cfs_rq->curr = se;
5445		5575
5446	/*	5576	/*
@@ -5461,6 +5591,8 @@ set_next_entity(struct cfs_rq cfs_rq, struct sched_entity se) · 14 unmodified lines
5461	se->prev_sum_exec_runtime = se->sum_exec_runtime;	5591	se->prev_sum_exec_runtime = se->sum_exec_runtime;
5462	}	5592	}
5463		5593
		5594	static int dequeue_entities(struct rq rq, struct sched_entity se, int flags);
		5595
5464	/*	5596	/*
5465	* Pick the next process, keeping these things in mind, in this order:	5597	* Pick the next process, keeping these things in mind, in this order:
5466	* 1) keep things fair between processes/task groups	5598	* 1) keep things fair between processes/task groups
@@ -5469,16 +5601,26 @@ set_next_entity(struct cfs_rq cfs_rq, struct sched_entity se) · 2 unmodified lines
5469	* 4) do not run the "skip" process, if something else is available	5601	* 4) do not run the "skip" process, if something else is available
5470	*/	5602	*/
5471	static struct sched_entity *	5603	static struct sched_entity *
5472	pick_next_entity(struct cfs_rq *cfs_rq)	5604	pick_next_entity(struct rq rq, struct cfs_rq cfs_rq)
5473	{	5605	{
5474	/*	5606	/*
5475	* Enabling NEXT_BUDDY will affect latency but not fairness.	5607	* Enabling NEXT_BUDDY will affect latency but not fairness.
5476	*/	5608	*/
5477	if (sched_feat(NEXT_BUDDY) &&	5609	if (sched_feat(NEXT_BUDDY) &&
5478	cfs_rq->next && entity_eligible(cfs_rq, cfs_rq->next))	5610	cfs_rq->next && entity_eligible(cfs_rq, cfs_rq->next)) {
		5611	/* ->next will never be delayed */
		5612	SCHED_WARN_ON(cfs_rq->next->sched_delayed);
5479	return cfs_rq->next;	5613	return cfs_rq->next;
		5614	}
5480		5615
5481	return pick_eevdf(cfs_rq);	5616	struct sched_entity *se = pick_eevdf(cfs_rq);
		5617	if (se->sched_delayed) {
		5618	dequeue_entities(rq, se, DEQUEUE_SLEEP \| DEQUEUE_DELAYED);
		5619	SCHED_WARN_ON(se->sched_delayed);
		5620	SCHED_WARN_ON(se->on_rq);
		5621	return NULL;
		5622	}
		5623	return se;
5482	}	5624	}
5483		5625
5484	static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq);	5626	static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq);
@@ -5502,6 +5644,7 @@ static void put_prev_entity(struct cfs_rq cfs_rq, struct sched_entity prev) · 17 unmodified lines
5502	/* in !on_rq case, update occurred at dequeue */	5644	/* in !on_rq case, update occurred at dequeue */
5503	update_load_avg(cfs_rq, prev, 0);	5645	update_load_avg(cfs_rq, prev, 0);
5504	}	5646	}
		5647	SCHED_WARN_ON(cfs_rq->curr != prev);
5505	cfs_rq->curr = NULL;	5648	cfs_rq->curr = NULL;
5506	}	5649	}
5507		5650
@@ -5765,6 +5908,7 @@ static bool throttle_cfs_rq(struct cfs_rq *cfs_rq) · 257 unmodified lines
5765	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);	5908	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
5766	struct sched_entity *se;	5909	struct sched_entity *se;
5767	long task_delta, idle_task_delta, dequeue = 1;	5910	long task_delta, idle_task_delta, dequeue = 1;
		5911	long rq_h_nr_running = rq->cfs.h_nr_running;
5768		5912
5769	raw_spin_lock(&cfs_b->lock);	5913	raw_spin_lock(&cfs_b->lock);
5770	/* This will start the period timer if necessary */	5914	/* This will start the period timer if necessary */
@@ -5798,11 +5942,21 @@ static bool throttle_cfs_rq(struct cfs_rq *cfs_rq) · 27 unmodified lines
5798	idle_task_delta = cfs_rq->idle_h_nr_running;	5942	idle_task_delta = cfs_rq->idle_h_nr_running;
5799	for_each_sched_entity(se) {	5943	for_each_sched_entity(se) {
5800	struct cfs_rq *qcfs_rq = cfs_rq_of(se);	5944	struct cfs_rq *qcfs_rq = cfs_rq_of(se);
		5945	int flags;
		5946
5801	/* throttled entity or throttle-on-deactivate */	5947	/* throttled entity or throttle-on-deactivate */
5802	if (!se->on_rq)	5948	if (!se->on_rq)
5803	goto done;	5949	goto done;
5804		5950
5805	dequeue_entity(qcfs_rq, se, DEQUEUE_SLEEP);	5951	/*
		5952	* Abuse SPECIAL to avoid delayed dequeue in this instance.
		5953	* This avoids teaching dequeue_entities() about throttled
		5954	* entities and keeps things relatively simple.
		5955	*/
		5956	flags = DEQUEUE_SLEEP \| DEQUEUE_SPECIAL;
		5957	if (se->sched_delayed)
		5958	flags \|= DEQUEUE_DELAYED;
		5959	dequeue_entity(qcfs_rq, se, flags);
5806		5960
5807	if (cfs_rq_is_idle(group_cfs_rq(se)))	5961	if (cfs_rq_is_idle(group_cfs_rq(se)))
5808	idle_task_delta = cfs_rq->h_nr_running;	5962	idle_task_delta = cfs_rq->h_nr_running;
@@ -5836,6 +5990,9 @@ static bool throttle_cfs_rq(struct cfs_rq *cfs_rq) · 27 unmodified lines
5836	/* At this point se is NULL and we are at root level*/	5990	/* At this point se is NULL and we are at root level*/
5837	sub_nr_running(rq, task_delta);	5991	sub_nr_running(rq, task_delta);
5838		5992
		5993	/* Stop the fair server if throttling resulted in no runnable tasks */
		5994	if (rq_h_nr_running && !rq->cfs.h_nr_running)
		5995	dl_server_stop(&rq->fair_server);
5839	done:	5996	done:
5840	/*	5997	/*
5841	* Note: distribution will already see us throttled via the	5998	* Note: distribution will already see us throttled via the
@@ -5854,6 +6011,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) · 12 unmodified lines
5854	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);	6011	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
5855	struct sched_entity *se;	6012	struct sched_entity *se;
5856	long task_delta, idle_task_delta;	6013	long task_delta, idle_task_delta;
		6014	long rq_h_nr_running = rq->cfs.h_nr_running;
5857		6015
5858	se = cfs_rq->tg->se[cpu_of(rq)];	6016	se = cfs_rq->tg->se[cpu_of(rq)];
5859		6017
@@ -5891,8 +6049,10 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) · 31 unmodified lines
5891	for_each_sched_entity(se) {	6049	for_each_sched_entity(se) {
5892	struct cfs_rq *qcfs_rq = cfs_rq_of(se);	6050	struct cfs_rq *qcfs_rq = cfs_rq_of(se);
5893		6051
5894	if (se->on_rq)	6052	if (se->on_rq) {
		6053	SCHED_WARN_ON(se->sched_delayed);
5895	break;	6054	break;
		6055	}
5896	enqueue_entity(qcfs_rq, se, ENQUEUE_WAKEUP);	6056	enqueue_entity(qcfs_rq, se, ENQUEUE_WAKEUP);
5897		6057
5898	if (cfs_rq_is_idle(group_cfs_rq(se)))	6058	if (cfs_rq_is_idle(group_cfs_rq(se)))
@@ -5923,6 +6083,10 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) · 24 unmodified lines
5923	goto unthrottle_throttle;	6083	goto unthrottle_throttle;
5924	}	6084	}
5925		6085
		6086	/* Start the fair server if un-throttling resulted in new runnable tasks */
		6087	if (!rq_h_nr_running && rq->cfs.h_nr_running)
		6088	dl_server_start(&rq->fair_server);
		6089
5926	/* At this point se is NULL and we are at root level*/	6090	/* At this point se is NULL and we are at root level*/
5927	add_nr_running(rq, task_delta);	6091	add_nr_running(rq, task_delta);
5928		6092
@@ -6555,7 +6719,7 @@ static void sched_fair_update_stop_tick(struct rq rq, struct task_struct p) · 626 unmodified lines
6555	{	6719	{
6556	int cpu = cpu_of(rq);	6720	int cpu = cpu_of(rq);
6557		6721
6558	if (!sched_feat(HZ_BW) \|\| !cfs_bandwidth_used())	6722	if (!cfs_bandwidth_used())
6559	return;	6723	return;
6560		6724
6561	if (!tick_nohz_full_cpu(cpu))	6725	if (!tick_nohz_full_cpu(cpu))
@@ -6738,6 +6902,37 @@ static int sched_idle_cpu(int cpu) · 176 unmodified lines
6738	}	6902	}
6739	#endif	6903	#endif
6740		6904
		6905	static void
		6906	requeue_delayed_entity(struct sched_entity *se)
		6907	{
		6908	struct cfs_rq *cfs_rq = cfs_rq_of(se);
		6909
		6910	/*
		6911	* se->sched_delayed should imply: se->on_rq == 1.
		6912	* Because a delayed entity is one that is still on
		6913	* the runqueue competing until elegibility.
		6914	*/
		6915	SCHED_WARN_ON(!se->sched_delayed);
		6916	SCHED_WARN_ON(!se->on_rq);
		6917
		6918	if (sched_feat(DELAY_ZERO)) {
		6919	update_entity_lag(cfs_rq, se);
		6920	if (se->vlag > 0) {
		6921	cfs_rq->nr_running--;
		6922	if (se != cfs_rq->curr)
		6923	__dequeue_entity(cfs_rq, se);
		6924	se->vlag = 0;
		6925	place_entity(cfs_rq, se, 0);
		6926	if (se != cfs_rq->curr)
		6927	__enqueue_entity(cfs_rq, se);
		6928	cfs_rq->nr_running++;
		6929	}
		6930	}
		6931
		6932	update_load_avg(cfs_rq, se, 0);
		6933	se->sched_delayed = 0;
		6934	}
		6935
6741	/*	6936	/*
6742	* The enqueue_task method is called before nr_running is	6937	* The enqueue_task method is called before nr_running is
6743	* increased. Here we update the fair scheduling stats and	6938	* increased. Here we update the fair scheduling stats and
@@ -6750,6 +6945,8 @@ enqueue_task_fair(struct rq rq, struct task_struct p, int flags) · 6 unmodified lines
6750	struct sched_entity *se = &p->se;	6945	struct sched_entity *se = &p->se;
6751	int idle_h_nr_running = task_has_idle_policy(p);	6946	int idle_h_nr_running = task_has_idle_policy(p);
6752	int task_new = !(flags & ENQUEUE_WAKEUP);	6947	int task_new = !(flags & ENQUEUE_WAKEUP);
		6948	int rq_h_nr_running = rq->cfs.h_nr_running;
		6949	u64 slice = 0;
6753		6950
6754	/*	6951	/*
6755	* The code below (indirectly) updates schedutil which looks at	6952	* The code below (indirectly) updates schedutil which looks at
@@ -6757,7 +6954,13 @@ enqueue_task_fair(struct rq rq, struct task_struct p, int flags) · 1 unmodified line
6757	* Let's add the task's estimated utilization to the cfs_rq's	6954	* Let's add the task's estimated utilization to the cfs_rq's
6758	* estimated utilization, before we update schedutil.	6955	* estimated utilization, before we update schedutil.
6759	*/	6956	*/
6760	util_est_enqueue(&rq->cfs, p);	6957	if (!(p->se.sched_delayed && (task_on_rq_migrating(p) \|\| (flags & ENQUEUE_RESTORE))))
		6958	util_est_enqueue(&rq->cfs, p);
		6959
		6960	if (flags & ENQUEUE_DELAYED) {
		6961	requeue_delayed_entity(se);
		6962	return;
		6963	}
6761		6964
6762	/*	6965	/*
6763	* If in_iowait is set, the code below may not trigger any cpufreq	6966	* If in_iowait is set, the code below may not trigger any cpufreq
@@ -6768,10 +6971,24 @@ enqueue_task_fair(struct rq rq, struct task_struct p, int flags) · 4 unmodified lines
6768	cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT);	6971	cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT);
6769		6972
6770	for_each_sched_entity(se) {	6973	for_each_sched_entity(se) {
6771	if (se->on_rq)	6974	if (se->on_rq) {
		6975	if (se->sched_delayed)
		6976	requeue_delayed_entity(se);
6772	break;	6977	break;
		6978	}
6773	cfs_rq = cfs_rq_of(se);	6979	cfs_rq = cfs_rq_of(se);
		6980
		6981	/*
		6982	* Basically set the slice of group entries to the min_slice of
		6983	* their respective cfs_rq. This ensures the group can service
		6984	* its entities in the desired time-frame.
		6985	*/
		6986	if (slice) {
		6987	se->slice = slice;
		6988	se->custom_slice = 1;
		6989	}
6774	enqueue_entity(cfs_rq, se, flags);	6990	enqueue_entity(cfs_rq, se, flags);
		6991	slice = cfs_rq_min_slice(cfs_rq);
6775		6992
6776	cfs_rq->h_nr_running++;	6993	cfs_rq->h_nr_running++;
6777	cfs_rq->idle_h_nr_running += idle_h_nr_running;	6994	cfs_rq->idle_h_nr_running += idle_h_nr_running;
@@ -6793,6 +7010,9 @@ enqueue_task_fair(struct rq rq, struct task_struct p, int flags) · 15 unmodified lines
6793	se_update_runnable(se);	7010	se_update_runnable(se);
6794	update_cfs_group(se);	7011	update_cfs_group(se);
6795		7012
		7013	se->slice = slice;
		7014	slice = cfs_rq_min_slice(cfs_rq);
		7015
6796	cfs_rq->h_nr_running++;	7016	cfs_rq->h_nr_running++;
6797	cfs_rq->idle_h_nr_running += idle_h_nr_running;	7017	cfs_rq->idle_h_nr_running += idle_h_nr_running;
6798		7018
@@ -6804,6 +7024,13 @@ enqueue_task_fair(struct rq rq, struct task_struct p, int flags) · 5 unmodified lines
6804	goto enqueue_throttle;	7024	goto enqueue_throttle;
6805	}	7025	}
6806		7026
		7027	if (!rq_h_nr_running && rq->cfs.h_nr_running) {
		7028	/* Account for idle runtime */
		7029	if (!rq->nr_running)
		7030	dl_server_update_idle_time(rq, rq->curr);
		7031	dl_server_start(&rq->fair_server);
		7032	}
		7033
6807	/* At this point se is NULL and we are at root level*/	7034	/* At this point se is NULL and we are at root level*/
6808	add_nr_running(rq, 1);	7035	add_nr_running(rq, 1);
6809		7036
@@ -6833,36 +7060,59 @@ enqueue_throttle: · 23 unmodified lines
6833	static void set_next_buddy(struct sched_entity *se);	7060	static void set_next_buddy(struct sched_entity *se);
6834		7061
6835	/*	7062	/*
6836	* The dequeue_task method is called before nr_running is	7063	* Basically dequeue_task_fair(), except it can deal with dequeue_entity()
6837	* decreased. We remove the task from the rbtree and	7064	* failing half-way through and resume the dequeue later.
6838	* update the fair scheduling stats:	7065	*
		7066	* Returns:
		7067	* -1 - dequeue delayed
		7068	* 0 - dequeue throttled
		7069	* 1 - dequeue complete
6839	*/	7070	*/
6840	static void dequeue_task_fair(struct rq rq, struct task_struct p, int flags)	7071	static int dequeue_entities(struct rq rq, struct sched_entity se, int flags)
6841	{	7072	{
6842	struct cfs_rq *cfs_rq;
6843	struct sched_entity *se = &p->se;
6844	int task_sleep = flags & DEQUEUE_SLEEP;
6845	int idle_h_nr_running = task_has_idle_policy(p);
6846	bool was_sched_idle = sched_idle_rq(rq);	7073	bool was_sched_idle = sched_idle_rq(rq);
		7074	int rq_h_nr_running = rq->cfs.h_nr_running;
		7075	bool task_sleep = flags & DEQUEUE_SLEEP;
		7076	bool task_delayed = flags & DEQUEUE_DELAYED;
		7077	struct task_struct *p = NULL;
		7078	int idle_h_nr_running = 0;
		7079	int h_nr_running = 0;
		7080	struct cfs_rq *cfs_rq;
		7081	u64 slice = 0;
6847		7082
6848	util_est_dequeue(&rq->cfs, p);	7083	if (entity_is_task(se)) {
		7084	p = task_of(se);
		7085	h_nr_running = 1;
		7086	idle_h_nr_running = task_has_idle_policy(p);
		7087	} else {
		7088	cfs_rq = group_cfs_rq(se);
		7089	slice = cfs_rq_min_slice(cfs_rq);
		7090	}
6849		7091
6850	for_each_sched_entity(se) {	7092	for_each_sched_entity(se) {
6851	cfs_rq = cfs_rq_of(se);	7093	cfs_rq = cfs_rq_of(se);
6852	dequeue_entity(cfs_rq, se, flags);
6853		7094
6854	cfs_rq->h_nr_running--;	7095	if (!dequeue_entity(cfs_rq, se, flags)) {
		7096	if (p && &p->se == se)
		7097	return -1;
		7098
		7099	break;
		7100	}
		7101
		7102	cfs_rq->h_nr_running -= h_nr_running;
6855	cfs_rq->idle_h_nr_running -= idle_h_nr_running;	7103	cfs_rq->idle_h_nr_running -= idle_h_nr_running;
6856		7104
6857	if (cfs_rq_is_idle(cfs_rq))	7105	if (cfs_rq_is_idle(cfs_rq))
6858	idle_h_nr_running = 1;	7106	idle_h_nr_running = h_nr_running;
6859		7107
6860	/* end evaluation on encountering a throttled cfs_rq */	7108	/* end evaluation on encountering a throttled cfs_rq */
6861	if (cfs_rq_throttled(cfs_rq))	7109	if (cfs_rq_throttled(cfs_rq))
6862	goto dequeue_throttle;	7110	return 0;
6863		7111
6864	/* Don't dequeue parent if it has other entities besides us */	7112	/* Don't dequeue parent if it has other entities besides us */
6865	if (cfs_rq->load.weight) {	7113	if (cfs_rq->load.weight) {
		7114	slice = cfs_rq_min_slice(cfs_rq);
		7115
6866	/* Avoid re-evaluating load for this entity: */	7116	/* Avoid re-evaluating load for this entity: */
6867	se = parent_entity(se);	7117	se = parent_entity(se);
6868	/*	7118	/*
@@ -6874,6 +7124,7 @@ static void dequeue_task_fair(struct rq rq, struct task_struct p, int flags) · 5 unmodified lines
6874	break;	7124	break;
6875	}	7125	}
6876	flags \|= DEQUEUE_SLEEP;	7126	flags \|= DEQUEUE_SLEEP;
		7127	flags &= ~(DEQUEUE_DELAYED \| DEQUEUE_SPECIAL);
6877	}	7128	}
6878		7129
6879	for_each_sched_entity(se) {	7130	for_each_sched_entity(se) {
@@ -6883,28 +7134,61 @@ static void dequeue_task_fair(struct rq rq, struct task_struct p, int flags) · 3 unmodified lines
6883	se_update_runnable(se);	7134	se_update_runnable(se);
6884	update_cfs_group(se);	7135	update_cfs_group(se);
6885		7136
6886	cfs_rq->h_nr_running--;	7137	se->slice = slice;
		7138	slice = cfs_rq_min_slice(cfs_rq);
		7139
		7140	cfs_rq->h_nr_running -= h_nr_running;
6887	cfs_rq->idle_h_nr_running -= idle_h_nr_running;	7141	cfs_rq->idle_h_nr_running -= idle_h_nr_running;
6888		7142
6889	if (cfs_rq_is_idle(cfs_rq))	7143	if (cfs_rq_is_idle(cfs_rq))
6890	idle_h_nr_running = 1;	7144	idle_h_nr_running = h_nr_running;
6891		7145
6892	/* end evaluation on encountering a throttled cfs_rq */	7146	/* end evaluation on encountering a throttled cfs_rq */
6893	if (cfs_rq_throttled(cfs_rq))	7147	if (cfs_rq_throttled(cfs_rq))
6894	goto dequeue_throttle;	7148	return 0;
6895
6896	}	7149	}
6897		7150
6898	/* At this point se is NULL and we are at root level*/	7151	sub_nr_running(rq, h_nr_running);
6899	sub_nr_running(rq, 1);	7152
		7153	if (rq_h_nr_running && !rq->cfs.h_nr_running)
		7154	dl_server_stop(&rq->fair_server);
6900		7155
6901	/* balance early to pull high priority tasks */	7156	/* balance early to pull high priority tasks */
6902	if (unlikely(!was_sched_idle && sched_idle_rq(rq)))	7157	if (unlikely(!was_sched_idle && sched_idle_rq(rq)))
6903	rq->next_balance = jiffies;	7158	rq->next_balance = jiffies;
6904		7159
6905	dequeue_throttle:	7160	if (p && task_delayed) {
6906	util_est_update(&rq->cfs, p, task_sleep);	7161	SCHED_WARN_ON(!task_sleep);
		7162	SCHED_WARN_ON(p->on_rq != 1);
		7163
		7164	/* Fix-up what dequeue_task_fair() skipped */
		7165	hrtick_update(rq);
		7166
		7167	/* Fix-up what block_task() skipped. */
		7168	__block_task(rq, p);
		7169	}
		7170
		7171	return 1;
		7172	}
		7173
		7174	/*
		7175	* The dequeue_task method is called before nr_running is
		7176	* decreased. We remove the task from the rbtree and
		7177	* update the fair scheduling stats:
		7178	*/
		7179	static bool dequeue_task_fair(struct rq rq, struct task_struct p, int flags)
		7180	{
		7181	if (!(p->se.sched_delayed && (task_on_rq_migrating(p) \|\| (flags & DEQUEUE_SAVE))))
		7182	util_est_dequeue(&rq->cfs, p);
		7183
		7184	if (dequeue_entities(rq, &p->se, flags) < 0) {
		7185	util_est_update(&rq->cfs, p, DEQUEUE_SLEEP);
		7186	return false;
		7187	}
		7188
		7189	util_est_update(&rq->cfs, p, flags & DEQUEUE_SLEEP);
6907	hrtick_update(rq);	7190	hrtick_update(rq);
		7191	return true;
6908	}	7192	}
6909		7193
6910	#ifdef CONFIG_SMP	7194	#ifdef CONFIG_SMP
@@ -7802,6 +8086,105 @@ static unsigned long cpu_util_without(int cpu, struct task_struct *p) · 891 unmodified lines
7802	return cpu_util(cpu, p, -1, 0);	8086	return cpu_util(cpu, p, -1, 0);
7803	}	8087	}
7804		8088
		8089	/*
		8090	* This function computes an effective utilization for the given CPU, to be
		8091	* used for frequency selection given the linear relation: f = u * f_max.
		8092	*
		8093	* The scheduler tracks the following metrics:
		8094	*
		8095	* cpu_util_{cfs,rt,dl,irq}()
		8096	* cpu_bw_dl()
		8097	*
		8098	* Where the cfs,rt and dl util numbers are tracked with the same metric and
		8099	* synchronized windows and are thus directly comparable.
		8100	*
		8101	* The cfs,rt,dl utilization are the running times measured with rq->clock_task
		8102	* which excludes things like IRQ and steal-time. These latter are then accrued
		8103	* in the IRQ utilization.
		8104	*
		8105	* The DL bandwidth number OTOH is not a measured metric but a value computed
		8106	* based on the task model parameters and gives the minimal utilization
		8107	* required to meet deadlines.
		8108	*/
		8109	unsigned long effective_cpu_util(int cpu, unsigned long util_cfs,
		8110	unsigned long *min,
		8111	unsigned long *max)
		8112	{
		8113	unsigned long util, irq, scale;
		8114	struct rq *rq = cpu_rq(cpu);
		8115
		8116	scale = arch_scale_cpu_capacity(cpu);
		8117
		8118	/*
		8119	* Early check to see if IRQ/steal time saturates the CPU, can be
		8120	* because of inaccuracies in how we track these -- see
		8121	* update_irq_load_avg().
		8122	*/
		8123	irq = cpu_util_irq(rq);
		8124	if (unlikely(irq >= scale)) {
		8125	if (min)
		8126	*min = scale;
		8127	if (max)
		8128	*max = scale;
		8129	return scale;
		8130	}
		8131
		8132	if (min) {
		8133	/*
		8134	* The minimum utilization returns the highest level between:
		8135	* - the computed DL bandwidth needed with the IRQ pressure which
		8136	* steals time to the deadline task.
		8137	* - The minimum performance requirement for CFS and/or RT.
		8138	*/
		8139	*min = max(irq + cpu_bw_dl(rq), uclamp_rq_get(rq, UCLAMP_MIN));
		8140
		8141	/*
		8142	* When an RT task is runnable and uclamp is not used, we must
		8143	* ensure that the task will run at maximum compute capacity.
		8144	*/
		8145	if (!uclamp_is_used() && rt_rq_is_runnable(&rq->rt))
		8146	min = max(min, scale);
		8147	}
		8148
		8149	/*
		8150	* Because the time spend on RT/DL tasks is visible as 'lost' time to
		8151	* CFS tasks and we use the same metric to track the effective
		8152	* utilization (PELT windows are synchronized) we can directly add them
		8153	* to obtain the CPU's actual utilization.
		8154	*/
		8155	util = util_cfs + cpu_util_rt(rq);
		8156	util += cpu_util_dl(rq);
		8157
		8158	/*
		8159	* The maximum hint is a soft bandwidth requirement, which can be lower
		8160	* than the actual utilization because of uclamp_max requirements.
		8161	*/
		8162	if (max)
		8163	*max = min(scale, uclamp_rq_get(rq, UCLAMP_MAX));
		8164
		8165	if (util >= scale)
		8166	return scale;
		8167
		8168	/*
		8169	* There is still idle time; further improve the number by using the
		8170	* IRQ metric. Because IRQ/steal time is hidden from the task clock we
		8171	* need to scale the task numbers:
		8172	*
		8173	* max - irq
		8174	* U' = irq + --------- * U
		8175	* max
		8176	*/
		8177	util = scale_irq_capacity(util, irq, scale);
		8178	util += irq;
		8179
		8180	return min(scale, util);
		8181	}
		8182
		8183	unsigned long sched_cpu_util(int cpu)
		8184	{
		8185	return effective_cpu_util(cpu, cpu_util_cfs(cpu), NULL, NULL);
		8186	}
		8187
7805	/*	8188	/*
7806	* energy_env - Utilization landscape for energy estimation.	8189	* energy_env - Utilization landscape for energy estimation.
7807	* @task_busy_time: Utilization contribution by the task for which we test the	8190	* @task_busy_time: Utilization contribution by the task for which we test the
@@ -8286,7 +8669,21 @@ static void migrate_task_rq_fair(struct task_struct *p, int new_cpu) · 478 unmodified lines
8286		8669
8287	static void task_dead_fair(struct task_struct *p)	8670	static void task_dead_fair(struct task_struct *p)
8288	{	8671	{
8289	remove_entity_load_avg(&p->se);	8672	struct sched_entity *se = &p->se;
		8673
		8674	if (se->sched_delayed) {
		8675	struct rq_flags rf;
		8676	struct rq *rq;
		8677
		8678	rq = task_rq_lock(p, &rf);
		8679	if (se->sched_delayed) {
		8680	update_rq_clock(rq);
		8681	dequeue_entities(rq, se, DEQUEUE_SLEEP \| DEQUEUE_DELAYED);
		8682	}
		8683	task_rq_unlock(rq, p, &rf);
		8684	}
		8685
		8686	remove_entity_load_avg(se);
8290	}	8687	}
8291		8688
8292	/*	8689	/*
@@ -8322,7 +8719,7 @@ static void set_cpus_allowed_fair(struct task_struct *p, struct affinity_context · 29 unmodified lines
8322	static int	8719	static int
8323	balance_fair(struct rq rq, struct task_struct prev, struct rq_flags *rf)	8720	balance_fair(struct rq rq, struct task_struct prev, struct rq_flags *rf)
8324	{	8721	{
8325	if (rq->nr_running)	8722	if (sched_fair_runnable(rq))
8326	return 1;	8723	return 1;
8327		8724
8328	return sched_balance_newidle(rq, rf) != 0;	8725	return sched_balance_newidle(rq, rf) != 0;
@@ -8381,16 +8778,7 @@ static void check_preempt_wakeup_fair(struct rq rq, struct task_struct p, int · 52 unmodified lines
8381	if (test_tsk_need_resched(curr))	8778	if (test_tsk_need_resched(curr))
8382	return;	8779	return;
8383		8780
8384	/* Idle tasks are by definition preempted by non-idle tasks. */	8781	if (!sched_feat(WAKEUP_PREEMPTION))
8385	if (unlikely(task_has_idle_policy(curr)) &&
8386	likely(!task_has_idle_policy(p)))
8387	goto preempt;
8388
8389	/*
8390	* Batch and idle tasks do not preempt non-idle tasks (their preemption
8391	* is driven by the tick):
8392	*/
8393	if (unlikely(p->policy != SCHED_NORMAL) \|\| !sched_feat(WAKEUP_PREEMPTION))
8394	return;	8782	return;
8395		8783
8396	find_matching_se(&se, &pse);	8784	find_matching_se(&se, &pse);
@@ -8400,7 +8788,7 @@ static void check_preempt_wakeup_fair(struct rq rq, struct task_struct p, int · 3 unmodified lines
8400	pse_is_idle = se_is_idle(pse);	8788	pse_is_idle = se_is_idle(pse);
8401		8789
8402	/*	8790	/*
8403	* Preempt an idle group in favor of a non-idle group (and don't preempt	8791	* Preempt an idle entity in favor of a non-idle entity (and don't preempt
8404	* in the inverse case).	8792	* in the inverse case).
8405	*/	8793	*/
8406	if (cse_is_idle && !pse_is_idle)	8794	if (cse_is_idle && !pse_is_idle)
@@ -8408,11 +8796,26 @@ static void check_preempt_wakeup_fair(struct rq rq, struct task_struct p, int · 1 unmodified line
8408	if (cse_is_idle != pse_is_idle)	8796	if (cse_is_idle != pse_is_idle)
8409	return;	8797	return;
8410		8798
		8799	/*
		8800	* BATCH and IDLE tasks do not preempt others.
		8801	*/
		8802	if (unlikely(p->policy != SCHED_NORMAL))
		8803	return;
		8804
8411	cfs_rq = cfs_rq_of(se);	8805	cfs_rq = cfs_rq_of(se);
8412	update_curr(cfs_rq);	8806	update_curr(cfs_rq);
		8807	/*
		8808	* If @p has a shorter slice than current and @p is eligible, override
		8809	* current's slice protection in order to allow preemption.
		8810	*
		8811	* Note that even if @p does not turn out to be the most eligible
		8812	* task at this moment, current's slice protection will be lost.
		8813	*/
		8814	if (do_preempt_short(cfs_rq, pse, se) && se->vlag == se->deadline)
		8815	se->vlag = se->deadline + 1;
8413		8816
8414	/*	8817	/*
8415	* XXX pick_eevdf(cfs_rq) != se ?	8818	* If @p has become the most eligible task, force preemption.
8416	*/	8819	*/
8417	if (pick_eevdf(cfs_rq) == pse)	8820	if (pick_eevdf(cfs_rq) == pse)
8418	goto preempt;	8821	goto preempt;
@@ -8423,7 +8826,6 @@ preempt: · 4 unmodified lines
8423	resched_curr(rq);	8826	resched_curr(rq);
8424	}	8827	}
8425		8828
8426	#ifdef CONFIG_SMP
8427	static struct task_struct pick_task_fair(struct rq rq)	8829	static struct task_struct pick_task_fair(struct rq rq)
8428	{	8830	{
8429	struct sched_entity *se;	8831	struct sched_entity *se;
@@ -8435,95 +8837,58 @@ again: · 5 unmodified lines
8435	return NULL;	8837	return NULL;
8436		8838
8437	do {	8839	do {
8438	struct sched_entity *curr = cfs_rq->curr;	8840	/* Might not have done put_prev_entity() */
8439		8841	if (cfs_rq->curr && cfs_rq->curr->on_rq)
8440	/* When we pick for a remote RQ, we'll not have done put_prev_entity() */	8842	update_curr(cfs_rq);
8441	if (curr) {
8442	if (curr->on_rq)
8443	update_curr(cfs_rq);
8444	else
8445	curr = NULL;
8446		8843
8447	if (unlikely(check_cfs_rq_runtime(cfs_rq)))	8844	if (unlikely(check_cfs_rq_runtime(cfs_rq)))
8448	goto again;	8845	goto again;
8449	}
8450		8846
8451	se = pick_next_entity(cfs_rq);	8847	se = pick_next_entity(rq, cfs_rq);
		8848	if (!se)
		8849	goto again;
8452	cfs_rq = group_cfs_rq(se);	8850	cfs_rq = group_cfs_rq(se);
8453	} while (cfs_rq);	8851	} while (cfs_rq);
8454		8852
8455	return task_of(se);	8853	return task_of(se);
8456	}	8854	}
8457	#endif	8855
		8856	static void __set_next_task_fair(struct rq rq, struct task_struct p, bool first);
		8857	static void set_next_task_fair(struct rq rq, struct task_struct p, bool first);
8458		8858
8459	struct task_struct *	8859	struct task_struct *
8460	pick_next_task_fair(struct rq rq, struct task_struct prev, struct rq_flags *rf)	8860	pick_next_task_fair(struct rq rq, struct task_struct prev, struct rq_flags *rf)
8461	{	8861	{
8462	struct cfs_rq *cfs_rq = &rq->cfs;
8463	struct sched_entity *se;	8862	struct sched_entity *se;
8464	struct task_struct *p;	8863	struct task_struct *p;
8465	int new_tasks;	8864	int new_tasks;
8466		8865
8467	again:	8866	again:
8468	if (!sched_fair_runnable(rq))	8867	p = pick_task_fair(rq);
		8868	if (!p)
8469	goto idle;	8869	goto idle;
		8870	se = &p->se;
8470		8871
8471	#ifdef CONFIG_FAIR_GROUP_SCHED	8872	#ifdef CONFIG_FAIR_GROUP_SCHED
8472	if (!prev \|\| prev->sched_class != &fair_sched_class)	8873	if (prev->sched_class != &fair_sched_class)
8473	goto simple;	8874	goto simple;
8474		8875
		8876	__put_prev_set_next_dl_server(rq, prev, p);
		8877
8475	/*	8878	/*
8476	* Because of the set_next_buddy() in dequeue_task_fair() it is rather	8879	* Because of the set_next_buddy() in dequeue_task_fair() it is rather
8477	* likely that a next task is from the same cgroup as the current.	8880	* likely that a next task is from the same cgroup as the current.
8478	*	8881	*
8479	* Therefore attempt to avoid putting and setting the entire cgroup	8882	* Therefore attempt to avoid putting and setting the entire cgroup
8480	* hierarchy, only change the part that actually changes.	8883	* hierarchy, only change the part that actually changes.
8481	*/	8884	*
8482
8483	do {
8484	struct sched_entity *curr = cfs_rq->curr;
8485
8486	/*
8487	* Since we got here without doing put_prev_entity() we also
8488	* have to consider cfs_rq->curr. If it is still a runnable
8489	* entity, update_curr() will update its vruntime, otherwise
8490	* forget we've ever seen it.
8491	*/
8492	if (curr) {
8493	if (curr->on_rq)
8494	update_curr(cfs_rq);
8495	else
8496	curr = NULL;
8497
8498	/*
8499	* This call to check_cfs_rq_runtime() will do the
8500	* throttle and dequeue its entity in the parent(s).
8501	* Therefore the nr_running test will indeed
8502	* be correct.
8503	*/
8504	if (unlikely(check_cfs_rq_runtime(cfs_rq))) {
8505	cfs_rq = &rq->cfs;
8506
8507	if (!cfs_rq->nr_running)
8508	goto idle;
8509
8510	goto simple;
8511	}
8512	}
8513
8514	se = pick_next_entity(cfs_rq);
8515	cfs_rq = group_cfs_rq(se);
8516	} while (cfs_rq);
8517
8518	p = task_of(se);
8519
8520	/*
8521	* Since we haven't yet done put_prev_entity and if the selected task	8885	* Since we haven't yet done put_prev_entity and if the selected task
8522	* is a different task than we started out with, try and touch the	8886	* is a different task than we started out with, try and touch the
8523	* least amount of cfs_rqs.	8887	* least amount of cfs_rqs.
8524	*/	8888	*/
8525	if (prev != p) {	8889	if (prev != p) {
8526	struct sched_entity *pse = &prev->se;	8890	struct sched_entity *pse = &prev->se;
		8891	struct cfs_rq *cfs_rq;
8527		8892
8528	while (!(cfs_rq = is_same_group(se, pse))) {	8893	while (!(cfs_rq = is_same_group(se, pse))) {
8529	int se_depth = se->depth;	8894	int se_depth = se->depth;
@@ -8541,38 +8906,15 @@ again: · 11 unmodified lines
8541		8906
8542	put_prev_entity(cfs_rq, pse);	8907	put_prev_entity(cfs_rq, pse);
8543	set_next_entity(cfs_rq, se);	8908	set_next_entity(cfs_rq, se);
8544	}
8545
8546	goto done;
8547	simple:
8548	#endif
8549	if (prev)
8550	put_prev_task(rq, prev);
8551		8909
8552	do {	8910	__set_next_task_fair(rq, p, true);
8553	se = pick_next_entity(cfs_rq);	8911	}
8554	set_next_entity(cfs_rq, se);
8555	cfs_rq = group_cfs_rq(se);
8556	} while (cfs_rq);
8557		8912
8558	p = task_of(se);	8913	return p;
8559		8914
8560	done: __maybe_unused;	8915	simple:
8561	#ifdef CONFIG_SMP
8562	/*
8563	* Move the next running task to the front of
8564	* the list, so our cfs_tasks list becomes MRU
8565	* one.
8566	*/
8567	list_move(&p->se.group_node, &rq->cfs_tasks);
8568	#endif	8916	#endif
8569		8917	put_prev_set_next_task(rq, prev, p);
8570	if (hrtick_enabled_fair(rq))
8571	hrtick_start_fair(rq, p);
8572
8573	update_misfit_status(p, rq);
8574	sched_fair_update_stop_tick(rq, p);
8575
8576	return p;	8918	return p;
8577		8919
8578	idle:	8920	idle:
@@ -8601,15 +8943,34 @@ idle: · 22 unmodified lines
8601	return NULL;	8943	return NULL;
8602	}	8944	}
8603		8945
8604	static struct task_struct __pick_next_task_fair(struct rq rq)	8946	static struct task_struct __pick_next_task_fair(struct rq rq, struct task_struct *prev)
8605	{	8947	{
8606	return pick_next_task_fair(rq, NULL, NULL);	8948	return pick_next_task_fair(rq, prev, NULL);
		8949	}
		8950
		8951	static bool fair_server_has_tasks(struct sched_dl_entity *dl_se)
		8952	{
		8953	return !!dl_se->rq->cfs.nr_running;
		8954	}
		8955
		8956	static struct task_struct fair_server_pick_task(struct sched_dl_entity dl_se)
		8957	{
		8958	return pick_task_fair(dl_se->rq);
		8959	}
		8960
		8961	void fair_server_init(struct rq *rq)
		8962	{
		8963	struct sched_dl_entity *dl_se = &rq->fair_server;
		8964
		8965	init_dl_entity(dl_se);
		8966
		8967	dl_server_init(dl_se, rq, fair_server_has_tasks, fair_server_pick_task);
8607	}	8968	}
8608		8969
8609	/*	8970	/*
8610	* Account for a descheduled task:	8971	* Account for a descheduled task:
8611	*/	8972	*/
8612	static void put_prev_task_fair(struct rq rq, struct task_struct prev)	8973	static void put_prev_task_fair(struct rq rq, struct task_struct prev, struct task_struct *next)
8613	{	8974	{
8614	struct sched_entity *se = &prev->se;	8975	struct sched_entity *se = &prev->se;
8615	struct cfs_rq *cfs_rq;	8976	struct cfs_rq *cfs_rq;
@@ -9360,9 +9721,10 @@ static bool __update_blocked_others(struct rq rq, bool done) · 744 unmodified lines
9360		9721
9361	hw_pressure = arch_scale_hw_pressure(cpu_of(rq));	9722	hw_pressure = arch_scale_hw_pressure(cpu_of(rq));
9362		9723
		9724	/* hw_pressure doesn't care about invariance */
9363	decayed = update_rt_rq_load_avg(now, rq, curr_class == &rt_sched_class) \|	9725	decayed = update_rt_rq_load_avg(now, rq, curr_class == &rt_sched_class) \|
9364	update_dl_rq_load_avg(now, rq, curr_class == &dl_sched_class) \|	9726	update_dl_rq_load_avg(now, rq, curr_class == &dl_sched_class) \|
9365	update_hw_load_avg(now, rq, hw_pressure) \|	9727	update_hw_load_avg(rq_clock_task(rq), rq, hw_pressure) \|
9366	update_irq_load_avg(rq, 0);	9728	update_irq_load_avg(rq, 0);
9367		9729
9368	if (others_have_blocked(rq))	9730	if (others_have_blocked(rq))
@@ -12702,22 +13064,7 @@ static void task_tick_fair(struct rq rq, struct task_struct curr, int queued) · 3333 unmodified lines
12702	*/	13064	*/
12703	static void task_fork_fair(struct task_struct *p)	13065	static void task_fork_fair(struct task_struct *p)
12704	{	13066	{
12705	struct sched_entity se = &p->se, curr;
12706	struct cfs_rq *cfs_rq;
12707	struct rq *rq = this_rq();
12708	struct rq_flags rf;
12709
12710	rq_lock(rq, &rf);
12711	update_rq_clock(rq);
12712
12713	set_task_max_allowed_capacity(p);	13067	set_task_max_allowed_capacity(p);
12714
12715	cfs_rq = task_cfs_rq(current);
12716	curr = cfs_rq->curr;
12717	if (curr)
12718	update_curr(cfs_rq);
12719	place_entity(cfs_rq, se, ENQUEUE_INITIAL);
12720	rq_unlock(rq, &rf);
12721	}	13068	}
12722		13069
12723	/*	13070	/*
@@ -12829,10 +13176,28 @@ static void attach_task_cfs_rq(struct task_struct *p) · 105 unmodified lines
12829	static void switched_from_fair(struct rq rq, struct task_struct p)	13176	static void switched_from_fair(struct rq rq, struct task_struct p)
12830	{	13177	{
12831	detach_task_cfs_rq(p);	13178	detach_task_cfs_rq(p);
		13179	/*
		13180	* Since this is called after changing class, this is a little weird
		13181	* and we cannot use DEQUEUE_DELAYED.
		13182	*/
		13183	if (p->se.sched_delayed) {
		13184	/* First, dequeue it from its new class' structures */
		13185	dequeue_task(rq, p, DEQUEUE_NOCLOCK \| DEQUEUE_SLEEP);
		13186	/*
		13187	* Now, clean up the fair_sched_class side of things
		13188	* related to sched_delayed being true and that wasn't done
		13189	* due to the generic dequeue not using DEQUEUE_DELAYED.
		13190	*/
		13191	finish_delayed_dequeue_entity(&p->se);
		13192	p->se.rel_deadline = 0;
		13193	__block_task(rq, p);
		13194	}
12832	}	13195	}
12833		13196
12834	static void switched_to_fair(struct rq rq, struct task_struct p)	13197	static void switched_to_fair(struct rq rq, struct task_struct p)
12835	{	13198	{
		13199	SCHED_WARN_ON(p->se.sched_delayed);
		13200
12836	attach_task_cfs_rq(p);	13201	attach_task_cfs_rq(p);
12837		13202
12838	set_task_max_allowed_capacity(p);	13203	set_task_max_allowed_capacity(p);
@@ -12850,12 +13215,7 @@ static void switched_to_fair(struct rq rq, struct task_struct p) · 11 unmodified lines
12850	}	13215	}
12851	}	13216	}
12852		13217
12853	/* Account for a task changing its policy or group.	13218	static void __set_next_task_fair(struct rq rq, struct task_struct p, bool first)
12854	*
12855	* This routine is mostly called to set cfs_rq->curr field when a task
12856	* migrates between groups/classes.
12857	*/
12858	static void set_next_task_fair(struct rq rq, struct task_struct p, bool first)
12859	{	13219	{
12860	struct sched_entity *se = &p->se;	13220	struct sched_entity *se = &p->se;
12861		13221
@@ -12868,6 +13228,27 @@ static void set_next_task_fair(struct rq rq, struct task_struct p, bool first) · 6 unmodified lines
12868	list_move(&se->group_node, &rq->cfs_tasks);	13228	list_move(&se->group_node, &rq->cfs_tasks);
12869	}	13229	}
12870	#endif	13230	#endif
		13231	if (!first)
		13232	return;
		13233
		13234	SCHED_WARN_ON(se->sched_delayed);
		13235
		13236	if (hrtick_enabled_fair(rq))
		13237	hrtick_start_fair(rq, p);
		13238
		13239	update_misfit_status(p, rq);
		13240	sched_fair_update_stop_tick(rq, p);
		13241	}
		13242
		13243	/*
		13244	* Account for a task changing its policy or group.
		13245	*
		13246	* This routine is mostly called to set cfs_rq->curr field when a task
		13247	* migrates between groups/classes.
		13248	*/
		13249	static void set_next_task_fair(struct rq rq, struct task_struct p, bool first)
		13250	{
		13251	struct sched_entity *se = &p->se;
12871		13252
12872	for_each_sched_entity(se) {	13253	for_each_sched_entity(se) {
12873	struct cfs_rq *cfs_rq = cfs_rq_of(se);	13254	struct cfs_rq *cfs_rq = cfs_rq_of(se);
@@ -12876,12 +13257,14 @@ static void set_next_task_fair(struct rq rq, struct task_struct p, bool first) · 2 unmodified lines
12876	/* ensure bandwidth has been allocated on our new cfs_rq */	13257	/* ensure bandwidth has been allocated on our new cfs_rq */
12877	account_cfs_rq_runtime(cfs_rq, 0);	13258	account_cfs_rq_runtime(cfs_rq, 0);
12878	}	13259	}
		13260
		13261	__set_next_task_fair(rq, p, first);
12879	}	13262	}
12880		13263
12881	void init_cfs_rq(struct cfs_rq *cfs_rq)	13264	void init_cfs_rq(struct cfs_rq *cfs_rq)
12882	{	13265	{
12883	cfs_rq->tasks_timeline = RB_ROOT_CACHED;	13266	cfs_rq->tasks_timeline = RB_ROOT_CACHED;
12884	u64_u32_store(cfs_rq->min_vruntime, (u64)(-(1LL << 20)));	13267	cfs_rq->min_vruntime = (u64)(-(1LL << 20));
12885	#ifdef CONFIG_SMP	13268	#ifdef CONFIG_SMP
12886	raw_spin_lock_init(&cfs_rq->removed.lock);	13269	raw_spin_lock_init(&cfs_rq->removed.lock);
12887	#endif	13270	#endif
@@ -12983,28 +13366,35 @@ void online_fair_sched_group(struct task_group *tg) · 95 unmodified lines
12983		13366
12984	void unregister_fair_sched_group(struct task_group *tg)	13367	void unregister_fair_sched_group(struct task_group *tg)
12985	{	13368	{
12986	unsigned long flags;
12987	struct rq *rq;
12988	int cpu;	13369	int cpu;
12989		13370
12990	destroy_cfs_bandwidth(tg_cfs_bandwidth(tg));	13371	destroy_cfs_bandwidth(tg_cfs_bandwidth(tg));
12991		13372
12992	for_each_possible_cpu(cpu) {	13373	for_each_possible_cpu(cpu) {
12993	if (tg->se[cpu])	13374	struct cfs_rq *cfs_rq = tg->cfs_rq[cpu];
12994	remove_entity_load_avg(tg->se[cpu]);	13375	struct sched_entity *se = tg->se[cpu];
		13376	struct rq *rq = cpu_rq(cpu);
		13377
		13378	if (se) {
		13379	if (se->sched_delayed) {
		13380	guard(rq_lock_irqsave)(rq);
		13381	if (se->sched_delayed) {
		13382	update_rq_clock(rq);
		13383	dequeue_entities(rq, se, DEQUEUE_SLEEP \| DEQUEUE_DELAYED);
		13384	}
		13385	list_del_leaf_cfs_rq(cfs_rq);
		13386	}
		13387	remove_entity_load_avg(se);
		13388	}
12995		13389
12996	/*	13390	/*
12997	* Only empty task groups can be destroyed; so we can speculatively	13391	* Only empty task groups can be destroyed; so we can speculatively
12998	* check on_list without danger of it being re-added.	13392	* check on_list without danger of it being re-added.
12999	*/	13393	*/
13000	if (!tg->cfs_rq[cpu]->on_list)	13394	if (cfs_rq->on_list) {
13001	continue;	13395	guard(rq_lock_irqsave)(rq);
13002		13396	list_del_leaf_cfs_rq(cfs_rq);
13003	rq = cpu_rq(cpu);	13397	}
13004
13005	raw_spin_rq_lock_irqsave(rq, flags);
13006	list_del_leaf_cfs_rq(tg->cfs_rq[cpu]);
13007	raw_spin_rq_unlock_irqrestore(rq, flags);
13008	}	13398	}
13009	}	13399	}
13010		13400
@@ -13194,13 +13584,13 @@ DEFINE_SCHED_CLASS(fair) = { · 183 unmodified lines
13194		13584
13195	.wakeup_preempt = check_preempt_wakeup_fair,	13585	.wakeup_preempt = check_preempt_wakeup_fair,
13196		13586
		13587	.pick_task = pick_task_fair,
13197	.pick_next_task = __pick_next_task_fair,	13588	.pick_next_task = __pick_next_task_fair,
13198	.put_prev_task = put_prev_task_fair,	13589	.put_prev_task = put_prev_task_fair,
13199	.set_next_task = set_next_task_fair,	13590	.set_next_task = set_next_task_fair,
13200		13591
13201	#ifdef CONFIG_SMP	13592	#ifdef CONFIG_SMP
13202	.balance = balance_fair,	13593	.balance = balance_fair,
13203	.pick_task = pick_task_fair,
13204	.select_task_rq = select_task_rq_fair,	13594	.select_task_rq = select_task_rq_fair,
13205	.migrate_task_rq = migrate_task_rq_fair,	13595	.migrate_task_rq = migrate_task_rq_fair,
13206		13596

kernel/sched/features.h+28−2

@@ -5,8 +5,24 @@ · 4 unmodified lines
5	* sleep+wake cycles. EEVDF placement strategy #1, #2 if disabled.	5	* sleep+wake cycles. EEVDF placement strategy #1, #2 if disabled.
6	*/	6	*/
7	SCHED_FEAT(PLACE_LAG, true)	7	SCHED_FEAT(PLACE_LAG, true)
		8	/*
		9	* Give new tasks half a slice to ease into the competition.
		10	*/
8	SCHED_FEAT(PLACE_DEADLINE_INITIAL, true)	11	SCHED_FEAT(PLACE_DEADLINE_INITIAL, true)
		12	/*
		13	* Preserve relative virtual deadline on 'migration'.
		14	*/
		15	SCHED_FEAT(PLACE_REL_DEADLINE, true)
		16	/*
		17	* Inhibit (wakeup) preemption until the current task has either matched the
		18	* 0-lag point or until is has exhausted it's slice.
		19	*/
9	SCHED_FEAT(RUN_TO_PARITY, true)	20	SCHED_FEAT(RUN_TO_PARITY, true)
		21	/*
		22	* Allow wakeup of tasks with a shorter slice to cancel RESPECT_SLICE for
		23	* current.
		24	*/
		25	SCHED_FEAT(PREEMPT_SHORT, true)
10		26
11	/*	27	/*
12	* Prefer to schedule the task we woke last (assuming it failed	28	* Prefer to schedule the task we woke last (assuming it failed
@@ -21,6 +37,18 @@ SCHED_FEAT(NEXT_BUDDY, false) · 8 unmodified lines
21	*/	37	*/
22	SCHED_FEAT(CACHE_HOT_BUDDY, true)	38	SCHED_FEAT(CACHE_HOT_BUDDY, true)
23		39
		40	/*
		41	* Delay dequeueing tasks until they get selected or woken.
		42	*
		43	* By delaying the dequeue for non-eligible tasks, they remain in the
		44	* competition and can burn off their negative lag. When they get selected
		45	* they'll have positive lag by definition.
		46	*
		47	* DELAY_ZERO clips the lag on dequeue (or wakeup) to 0.
		48	*/
		49	SCHED_FEAT(DELAY_DEQUEUE, true)
		50	SCHED_FEAT(DELAY_ZERO, true)
		51
24	/*	52	/*
25	* Allow wakeup-time preemption of the current task:	53	* Allow wakeup-time preemption of the current task:
26	*/	54	*/
@@ -85,5 +113,3 @@ SCHED_FEAT(WA_BIAS, true) · 58 unmodified lines
85	SCHED_FEAT(UTIL_EST, true)	113	SCHED_FEAT(UTIL_EST, true)
86		114
87	SCHED_FEAT(LATENCY_WARN, false)	115	SCHED_FEAT(LATENCY_WARN, false)
88
89	SCHED_FEAT(HZ_BW, true)

kernel/sched/idle.c+7−16

 	resched_curr(rq);
 }
-static void put_prev_task_idle(struct rq *rq, struct task_struct *prev)
+static void put_prev_task_idle(struct rq *rq, struct task_struct *prev, struct task_struct *next)
 {
+	dl_server_update_idle_time(rq, prev);
 }
 static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool first)
 {
 	update_idle_core(rq);
 	schedstat_inc(rq->sched_goidle);
+	next->se.exec_start = rq_clock_task(rq);
 }
-#ifdef CONFIG_SMP
+struct task_struct *pick_task_idle(struct rq *rq)
-static struct task_struct *pick_task_idle(struct rq *rq)
 {
 	return rq->idle;
 }
-#endif
-struct task_struct *pick_next_task_idle(struct rq *rq)
-{
-	struct task_struct *next = rq->idle;
-	set_next_task_idle(rq, next, true);
-	return next;
-}
 /*
  * It is not legal to sleep in the idle task - print a warning
  * message if some code attempts to do it:
  */
-static void
+static bool
 dequeue_task_idle(struct rq *rq, struct task_struct *p, int flags)
 {
 	raw_spin_rq_unlock_irq(rq);
 	printk(KERN_ERR "bad: scheduling from the idle thread!\n");
 	dump_stack();
 	raw_spin_rq_lock_irq(rq);
+	return true;
 }
 /*
 	.wakeup_preempt		= wakeup_preempt_idle,
-	.pick_next_task		= pick_next_task_idle,
+	.pick_task		= pick_task_idle,
 	.put_prev_task		= put_prev_task_idle,
 	.set_next_task          = set_next_task_idle,
 #ifdef CONFIG_SMP
 	.balance		= balance_idle,
-	.pick_task		= pick_task_idle,
 	.select_task_rq		= select_task_rq_idle,
 	.set_cpus_allowed	= set_cpus_allowed_common,
 #endif

kernel/sched/rt.c+112−149

@@ -8,10 +8,6 @@ int sched_rr_timeslice = RR_TIMESLICE; · 7 unmodified lines
8	/* More than 4 hours if BW_SHIFT equals 20. */	8	/* More than 4 hours if BW_SHIFT equals 20. */
9	static const u64 max_rt_runtime = MAX_BW;	9	static const u64 max_rt_runtime = MAX_BW;
10		10
11	static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun);
12
13	struct rt_bandwidth def_rt_bandwidth;
14
15	/*	11	/*
16	* period over which we measure -rt task CPU usage in us.	12	* period over which we measure -rt task CPU usage in us.
17	* default: 1s	13	* default: 1s
@@ -66,6 +62,40 @@ static int __init sched_rt_sysctl_init(void) · 48 unmodified lines
66	late_initcall(sched_rt_sysctl_init);	62	late_initcall(sched_rt_sysctl_init);
67	#endif	63	#endif
68		64
		65	void init_rt_rq(struct rt_rq *rt_rq)
		66	{
		67	struct rt_prio_array *array;
		68	int i;
		69
		70	array = &rt_rq->active;
		71	for (i = 0; i < MAX_RT_PRIO; i++) {
		72	INIT_LIST_HEAD(array->queue + i);
		73	__clear_bit(i, array->bitmap);
		74	}
		75	/* delimiter for bitsearch: */
		76	__set_bit(MAX_RT_PRIO, array->bitmap);
		77
		78	#if defined CONFIG_SMP
		79	rt_rq->highest_prio.curr = MAX_RT_PRIO-1;
		80	rt_rq->highest_prio.next = MAX_RT_PRIO-1;
		81	rt_rq->overloaded = 0;
		82	plist_head_init(&rt_rq->pushable_tasks);
		83	#endif /* CONFIG_SMP */
		84	/* We start is dequeued state, because no RT tasks are queued */
		85	rt_rq->rt_queued = 0;
		86
		87	#ifdef CONFIG_RT_GROUP_SCHED
		88	rt_rq->rt_time = 0;
		89	rt_rq->rt_throttled = 0;
		90	rt_rq->rt_runtime = 0;
		91	raw_spin_lock_init(&rt_rq->rt_runtime_lock);
		92	#endif
		93	}
		94
		95	#ifdef CONFIG_RT_GROUP_SCHED
		96
		97	static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun);
		98
69	static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer)	99	static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer)
70	{	100	{
71	struct rt_bandwidth *rt_b =	101	struct rt_bandwidth *rt_b =
@@ -130,35 +160,6 @@ static void start_rt_bandwidth(struct rt_bandwidth *rt_b) · 58 unmodified lines
130	do_start_rt_bandwidth(rt_b);	160	do_start_rt_bandwidth(rt_b);
131	}	161	}
132		162
133	void init_rt_rq(struct rt_rq *rt_rq)
134	{
135	struct rt_prio_array *array;
136	int i;
137
138	array = &rt_rq->active;
139	for (i = 0; i < MAX_RT_PRIO; i++) {
140	INIT_LIST_HEAD(array->queue + i);
141	__clear_bit(i, array->bitmap);
142	}
143	/* delimiter for bit-search: */
144	__set_bit(MAX_RT_PRIO, array->bitmap);
145
146	#if defined CONFIG_SMP
147	rt_rq->highest_prio.curr = MAX_RT_PRIO-1;
148	rt_rq->highest_prio.next = MAX_RT_PRIO-1;
149	rt_rq->overloaded = 0;
150	plist_head_init(&rt_rq->pushable_tasks);
151	#endif /* CONFIG_SMP */
152	/* We start is dequeued state, because no RT tasks are queued */
153	rt_rq->rt_queued = 0;
154
155	rt_rq->rt_time = 0;
156	rt_rq->rt_throttled = 0;
157	rt_rq->rt_runtime = 0;
158	raw_spin_lock_init(&rt_rq->rt_runtime_lock);
159	}
160
161	#ifdef CONFIG_RT_GROUP_SCHED
162	static void destroy_rt_bandwidth(struct rt_bandwidth *rt_b)	163	static void destroy_rt_bandwidth(struct rt_bandwidth *rt_b)
163	{	164	{
164	hrtimer_cancel(&rt_b->rt_period_timer);	165	hrtimer_cancel(&rt_b->rt_period_timer);
@@ -195,7 +196,6 @@ void unregister_rt_sched_group(struct task_group *tg) · 30 unmodified lines
195	{	196	{
196	if (tg->rt_se)	197	if (tg->rt_se)
197	destroy_rt_bandwidth(&tg->rt_bandwidth);	198	destroy_rt_bandwidth(&tg->rt_bandwidth);
198
199	}	199	}
200		200
201	void free_rt_sched_group(struct task_group *tg)	201	void free_rt_sched_group(struct task_group *tg)
@@ -253,8 +253,7 @@ int alloc_rt_sched_group(struct task_group tg, struct task_group parent) · 51 unmodified lines
253	if (!tg->rt_se)	253	if (!tg->rt_se)
254	goto err;	254	goto err;
255		255
256	init_rt_bandwidth(&tg->rt_bandwidth,	256	init_rt_bandwidth(&tg->rt_bandwidth, ktime_to_ns(global_rt_period()), 0);
257	ktime_to_ns(def_rt_bandwidth.rt_period), 0);
258		257
259	for_each_possible_cpu(i) {	258	for_each_possible_cpu(i) {
260	rt_rq = kzalloc_node(sizeof(struct rt_rq),	259	rt_rq = kzalloc_node(sizeof(struct rt_rq),
@@ -604,70 +603,6 @@ static inline struct rt_bandwidth sched_rt_bandwidth(struct rt_rq rt_rq) · 343 unmodified lines
604	return &rt_rq->tg->rt_bandwidth;	603	return &rt_rq->tg->rt_bandwidth;
605	}	604	}
606		605
607	#else /* !CONFIG_RT_GROUP_SCHED */
608
609	static inline u64 sched_rt_runtime(struct rt_rq *rt_rq)
610	{
611	return rt_rq->rt_runtime;
612	}
613
614	static inline u64 sched_rt_period(struct rt_rq *rt_rq)
615	{
616	return ktime_to_ns(def_rt_bandwidth.rt_period);
617	}
618
619	typedef struct rt_rq *rt_rq_iter_t;
620
621	#define for_each_rt_rq(rt_rq, iter, rq) \
622	for ((void) iter, rt_rq = &rq->rt; rt_rq; rt_rq = NULL)
623
624	#define for_each_sched_rt_entity(rt_se) \
625	for (; rt_se; rt_se = NULL)
626
627	static inline struct rt_rq group_rt_rq(struct sched_rt_entity rt_se)
628	{
629	return NULL;
630	}
631
632	static inline void sched_rt_rq_enqueue(struct rt_rq *rt_rq)
633	{
634	struct rq *rq = rq_of_rt_rq(rt_rq);
635
636	if (!rt_rq->rt_nr_running)
637	return;
638
639	enqueue_top_rt_rq(rt_rq);
640	resched_curr(rq);
641	}
642
643	static inline void sched_rt_rq_dequeue(struct rt_rq *rt_rq)
644	{
645	dequeue_top_rt_rq(rt_rq, rt_rq->rt_nr_running);
646	}
647
648	static inline int rt_rq_throttled(struct rt_rq *rt_rq)
649	{
650	return rt_rq->rt_throttled;
651	}
652
653	static inline const struct cpumask *sched_rt_period_mask(void)
654	{
655	return cpu_online_mask;
656	}
657
658	static inline
659	struct rt_rq sched_rt_period_rt_rq(struct rt_bandwidth rt_b, int cpu)
660	{
661	return &cpu_rq(cpu)->rt;
662	}
663
664	static inline struct rt_bandwidth sched_rt_bandwidth(struct rt_rq rt_rq)
665	{
666	return &def_rt_bandwidth;
667	}
668
669	#endif /* CONFIG_RT_GROUP_SCHED */
670
671	bool sched_rt_bandwidth_account(struct rt_rq *rt_rq)	606	bool sched_rt_bandwidth_account(struct rt_rq *rt_rq)
672	{	607	{
673	struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);	608	struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
@@ -859,7 +794,7 @@ static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun) · 185 unmodified lines
859	const struct cpumask *span;	794	const struct cpumask *span;
860		795
861	span = sched_rt_period_mask();	796	span = sched_rt_period_mask();
862	#ifdef CONFIG_RT_GROUP_SCHED	797
863	/*	798	/*
864	* FIXME: isolated CPUs should really leave the root task group,	799	* FIXME: isolated CPUs should really leave the root task group,
865	* whether they are isolcpus or were isolated via cpusets, lest	800	* whether they are isolcpus or were isolated via cpusets, lest
@@ -871,7 +806,7 @@ static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun) · 5 unmodified lines
871	*/	806	*/
872	if (rt_b == &root_task_group.rt_bandwidth)	807	if (rt_b == &root_task_group.rt_bandwidth)
873	span = cpu_online_mask;	808	span = cpu_online_mask;
874	#endif	809
875	for_each_cpu(i, span) {	810	for_each_cpu(i, span) {
876	int enqueue = 0;	811	int enqueue = 0;
877	struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i);	812	struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i);
@@ -938,18 +873,6 @@ static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun) · 60 unmodified lines
938	return idle;	873	return idle;
939	}	874	}
940		875
941	static inline int rt_se_prio(struct sched_rt_entity *rt_se)
942	{
943	#ifdef CONFIG_RT_GROUP_SCHED
944	struct rt_rq *rt_rq = group_rt_rq(rt_se);
945
946	if (rt_rq)
947	return rt_rq->highest_prio.curr;
948	#endif
949
950	return rt_task_of(rt_se)->prio;
951	}
952
953	static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq)	876	static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq)
954	{	877	{
955	u64 runtime = sched_rt_runtime(rt_rq);	878	u64 runtime = sched_rt_runtime(rt_rq);
@@ -993,6 +916,72 @@ static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq) · 37 unmodified lines
993	return 0;	916	return 0;
994	}	917	}
995		918
		919	#else /* !CONFIG_RT_GROUP_SCHED */
		920
		921	typedef struct rt_rq *rt_rq_iter_t;
		922
		923	#define for_each_rt_rq(rt_rq, iter, rq) \
		924	for ((void) iter, rt_rq = &rq->rt; rt_rq; rt_rq = NULL)
		925
		926	#define for_each_sched_rt_entity(rt_se) \
		927	for (; rt_se; rt_se = NULL)
		928
		929	static inline struct rt_rq group_rt_rq(struct sched_rt_entity rt_se)
		930	{
		931	return NULL;
		932	}
		933
		934	static inline void sched_rt_rq_enqueue(struct rt_rq *rt_rq)
		935	{
		936	struct rq *rq = rq_of_rt_rq(rt_rq);
		937
		938	if (!rt_rq->rt_nr_running)
		939	return;
		940
		941	enqueue_top_rt_rq(rt_rq);
		942	resched_curr(rq);
		943	}
		944
		945	static inline void sched_rt_rq_dequeue(struct rt_rq *rt_rq)
		946	{
		947	dequeue_top_rt_rq(rt_rq, rt_rq->rt_nr_running);
		948	}
		949
		950	static inline int rt_rq_throttled(struct rt_rq *rt_rq)
		951	{
		952	return false;
		953	}
		954
		955	static inline const struct cpumask *sched_rt_period_mask(void)
		956	{
		957	return cpu_online_mask;
		958	}
		959
		960	static inline
		961	struct rt_rq sched_rt_period_rt_rq(struct rt_bandwidth rt_b, int cpu)
		962	{
		963	return &cpu_rq(cpu)->rt;
		964	}
		965
		966	#ifdef CONFIG_SMP
		967	static void __enable_runtime(struct rq *rq) { }
		968	static void __disable_runtime(struct rq *rq) { }
		969	#endif
		970
		971	#endif /* CONFIG_RT_GROUP_SCHED */
		972
		973	static inline int rt_se_prio(struct sched_rt_entity *rt_se)
		974	{
		975	#ifdef CONFIG_RT_GROUP_SCHED
		976	struct rt_rq *rt_rq = group_rt_rq(rt_se);
		977
		978	if (rt_rq)
		979	return rt_rq->highest_prio.curr;
		980	#endif
		981
		982	return rt_task_of(rt_se)->prio;
		983	}
		984
996	/*	985	/*
997	* Update the current task's runtime statistics. Skip current tasks that	986	* Update the current task's runtime statistics. Skip current tasks that
998	* are not in our scheduling class.	987	* are not in our scheduling class.
@@ -1000,7 +989,6 @@ static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq) · 1 unmodified line
1000	static void update_curr_rt(struct rq *rq)	989	static void update_curr_rt(struct rq *rq)
1001	{	990	{
1002	struct task_struct *curr = rq->curr;	991	struct task_struct *curr = rq->curr;
1003	struct sched_rt_entity *rt_se = &curr->rt;
1004	s64 delta_exec;	992	s64 delta_exec;
1005		993
1006	if (curr->sched_class != &rt_sched_class)	994	if (curr->sched_class != &rt_sched_class)
@@ -1010,6 +998,9 @@ static void update_curr_rt(struct rq *rq) · 3 unmodified lines
1010	if (unlikely(delta_exec <= 0))	998	if (unlikely(delta_exec <= 0))
1011	return;	999	return;
1012		1000
		1001	#ifdef CONFIG_RT_GROUP_SCHED
		1002	struct sched_rt_entity *rt_se = &curr->rt;
		1003
1013	if (!rt_bandwidth_enabled())	1004	if (!rt_bandwidth_enabled())
1014	return;	1005	return;
1015		1006
@@ -1028,6 +1019,7 @@ static void update_curr_rt(struct rq *rq) · 12 unmodified lines
1028	do_start_rt_bandwidth(sched_rt_bandwidth(rt_rq));	1019	do_start_rt_bandwidth(sched_rt_bandwidth(rt_rq));
1029	}	1020	}
1030	}	1021	}
		1022	#endif
1031	}	1023	}
1032		1024
1033	static void	1025	static void
@@ -1184,7 +1176,6 @@ dec_rt_group(struct sched_rt_entity rt_se, struct rt_rq rt_rq) · 150 unmodified lines
1184	static void	1176	static void
1185	inc_rt_group(struct sched_rt_entity rt_se, struct rt_rq rt_rq)	1177	inc_rt_group(struct sched_rt_entity rt_se, struct rt_rq rt_rq)
1186	{	1178	{
1187	start_rt_bandwidth(&def_rt_bandwidth);
1188	}	1179	}
1189		1180
1190	static inline	1181	static inline
@@ -1492,7 +1483,7 @@ enqueue_task_rt(struct rq rq, struct task_struct p, int flags) · 301 unmodified lines
1492	enqueue_pushable_task(rq, p);	1483	enqueue_pushable_task(rq, p);
1493	}	1484	}
1494		1485
1495	static void dequeue_task_rt(struct rq rq, struct task_struct p, int flags)	1486	static bool dequeue_task_rt(struct rq rq, struct task_struct p, int flags)
1496	{	1487	{
1497	struct sched_rt_entity *rt_se = &p->rt;	1488	struct sched_rt_entity *rt_se = &p->rt;
1498		1489
@@ -1500,6 +1491,8 @@ static void dequeue_task_rt(struct rq rq, struct task_struct p, int flags) · 1 unmodified line
1500	dequeue_rt_entity(rt_se, flags);	1491	dequeue_rt_entity(rt_se, flags);
1501		1492
1502	dequeue_pushable_task(rq, p);	1493	dequeue_pushable_task(rq, p);
		1494
		1495	return true;
1503	}	1496	}
1504		1497
1505	/*	1498	/*
@@ -1755,17 +1748,7 @@ static struct task_struct pick_task_rt(struct rq rq) · 249 unmodified lines
1755	return p;	1748	return p;
1756	}	1749	}
1757		1750
1758	static struct task_struct pick_next_task_rt(struct rq rq)	1751	static void put_prev_task_rt(struct rq rq, struct task_struct p, struct task_struct *next)
1759	{
1760	struct task_struct *p = pick_task_rt(rq);
1761
1762	if (p)
1763	set_next_task_rt(rq, p, true);
1764
1765	return p;
1766	}
1767
1768	static void put_prev_task_rt(struct rq rq, struct task_struct p)
1769	{	1752	{
1770	struct sched_rt_entity *rt_se = &p->rt;	1753	struct sched_rt_entity *rt_se = &p->rt;
1771	struct rt_rq *rt_rq = &rq->rt;	1754	struct rt_rq *rt_rq = &rq->rt;
@@ -2652,13 +2635,12 @@ DEFINE_SCHED_CLASS(rt) = { · 880 unmodified lines
2652		2635
2653	.wakeup_preempt = wakeup_preempt_rt,	2636	.wakeup_preempt = wakeup_preempt_rt,
2654		2637
2655	.pick_next_task = pick_next_task_rt,	2638	.pick_task = pick_task_rt,
2656	.put_prev_task = put_prev_task_rt,	2639	.put_prev_task = put_prev_task_rt,
2657	.set_next_task = set_next_task_rt,	2640	.set_next_task = set_next_task_rt,
2658		2641
2659	#ifdef CONFIG_SMP	2642	#ifdef CONFIG_SMP
2660	.balance = balance_rt,	2643	.balance = balance_rt,
2661	.pick_task = pick_task_rt,
2662	.select_task_rq = select_task_rq_rt,	2644	.select_task_rq = select_task_rq_rt,
2663	.set_cpus_allowed = set_cpus_allowed_common,	2645	.set_cpus_allowed = set_cpus_allowed_common,
2664	.rq_online = rq_online_rt,	2646	.rq_online = rq_online_rt,
@@ -2912,19 +2894,6 @@ int sched_rt_can_attach(struct task_group tg, struct task_struct tsk) · 247 unmodified lines
2912	#ifdef CONFIG_SYSCTL	2894	#ifdef CONFIG_SYSCTL
2913	static int sched_rt_global_constraints(void)	2895	static int sched_rt_global_constraints(void)
2914	{	2896	{
2915	unsigned long flags;
2916	int i;
2917
2918	raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
2919	for_each_possible_cpu(i) {
2920	struct rt_rq *rt_rq = &cpu_rq(i)->rt;
2921
2922	raw_spin_lock(&rt_rq->rt_runtime_lock);
2923	rt_rq->rt_runtime = global_rt_runtime();
2924	raw_spin_unlock(&rt_rq->rt_runtime_lock);
2925	}
2926	raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);
2927
2928	return 0;	2897	return 0;
2929	}	2898	}
2930	#endif /* CONFIG_SYSCTL */	2899	#endif /* CONFIG_SYSCTL */
@@ -2944,12 +2913,6 @@ static int sched_rt_global_validate(void) · 13 unmodified lines
2944		2913
2945	static void sched_rt_do_global(void)	2914	static void sched_rt_do_global(void)
2946	{	2915	{
2947	unsigned long flags;
2948
2949	raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
2950	def_rt_bandwidth.rt_runtime = global_rt_runtime();
2951	def_rt_bandwidth.rt_period = ns_to_ktime(global_rt_period());
2952	raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);
2953	}	2916	}
2954		2917
2955	static int sched_rt_handler(const struct ctl_table table, int write, void buffer,	2918	static int sched_rt_handler(const struct ctl_table table, int write, void buffer,

kernel/sched/sched.h+78−23

@@ -68,6 +68,7 @@ · 67 unmodified lines
68	#include <linux/wait_api.h>	68	#include <linux/wait_api.h>
69	#include <linux/wait_bit.h>	69	#include <linux/wait_bit.h>
70	#include <linux/workqueue_api.h>	70	#include <linux/workqueue_api.h>
		71	#include <linux/delayacct.h>
71		72
72	#include <trace/events/power.h>	73	#include <trace/events/power.h>
73	#include <trace/events/sched.h>	74	#include <trace/events/sched.h>
@@ -335,7 +336,7 @@ extern bool __checkparam_dl(const struct sched_attr *attr); · 261 unmodified lines
335	extern bool dl_param_changed(struct task_struct p, const struct sched_attr attr);	336	extern bool dl_param_changed(struct task_struct p, const struct sched_attr attr);
336	extern int dl_cpuset_cpumask_can_shrink(const struct cpumask cur, const struct cpumask trial);	337	extern int dl_cpuset_cpumask_can_shrink(const struct cpumask cur, const struct cpumask trial);
337	extern int dl_bw_check_overflow(int cpu);	338	extern int dl_bw_check_overflow(int cpu);
338		339	extern s64 dl_scaled_delta_exec(struct rq rq, struct sched_dl_entity dl_se, s64 delta_exec);
339	/*	340	/*
340	* SCHED_DEADLINE supports servers (nested scheduling) with the following	341	* SCHED_DEADLINE supports servers (nested scheduling) with the following
341	* interface:	342	* interface:
@@ -361,7 +362,14 @@ extern void dl_server_start(struct sched_dl_entity *dl_se); · 19 unmodified lines
361	extern void dl_server_stop(struct sched_dl_entity *dl_se);	362	extern void dl_server_stop(struct sched_dl_entity *dl_se);
362	extern void dl_server_init(struct sched_dl_entity dl_se, struct rq rq,	363	extern void dl_server_init(struct sched_dl_entity dl_se, struct rq rq,
363	dl_server_has_tasks_f has_tasks,	364	dl_server_has_tasks_f has_tasks,
364	dl_server_pick_f pick);	365	dl_server_pick_f pick_task);
		366
		367	extern void dl_server_update_idle_time(struct rq *rq,
		368	struct task_struct *p);
		369	extern void fair_server_init(struct rq *rq);
		370	extern void __dl_server_attach_root(struct sched_dl_entity dl_se, struct rq rq);
		371	extern int dl_server_apply_params(struct sched_dl_entity *dl_se,
		372	u64 runtime, u64 period, bool init);
365		373
366	#ifdef CONFIG_CGROUP_SCHED	374	#ifdef CONFIG_CGROUP_SCHED
367		375
@@ -599,17 +607,12 @@ struct cfs_rq { · 231 unmodified lines
599	s64 avg_vruntime;	607	s64 avg_vruntime;
600	u64 avg_load;	608	u64 avg_load;
601		609
602	u64 exec_clock;
603	u64 min_vruntime;	610	u64 min_vruntime;
604	#ifdef CONFIG_SCHED_CORE	611	#ifdef CONFIG_SCHED_CORE
605	unsigned int forceidle_seq;	612	unsigned int forceidle_seq;
606	u64 min_vruntime_fi;	613	u64 min_vruntime_fi;
607	#endif	614	#endif
608		615
609	#ifndef CONFIG_64BIT
610	u64 min_vruntime_copy;
611	#endif
612
613	struct rb_root_cached tasks_timeline;	616	struct rb_root_cached tasks_timeline;
614		617
615	/*	618	/*
@@ -619,10 +622,6 @@ struct cfs_rq { · 3 unmodified lines
619	struct sched_entity *curr;	622	struct sched_entity *curr;
620	struct sched_entity *next;	623	struct sched_entity *next;
621		624
622	#ifdef CONFIG_SCHED_DEBUG
623	unsigned int nr_spread_over;
624	#endif
625
626	#ifdef CONFIG_SMP	625	#ifdef CONFIG_SMP
627	/*	626	/*
628	* CFS load tracking	627	* CFS load tracking
@@ -726,13 +725,13 @@ struct rt_rq { · 97 unmodified lines
726	#endif /* CONFIG_SMP */	725	#endif /* CONFIG_SMP */
727	int rt_queued;	726	int rt_queued;
728		727
		728	#ifdef CONFIG_RT_GROUP_SCHED
729	int rt_throttled;	729	int rt_throttled;
730	u64 rt_time;	730	u64 rt_time;
731	u64 rt_runtime;	731	u64 rt_runtime;
732	/* Nests inside the rq lock: */	732	/* Nests inside the rq lock: */
733	raw_spinlock_t rt_runtime_lock;	733	raw_spinlock_t rt_runtime_lock;
734		734
735	#ifdef CONFIG_RT_GROUP_SCHED
736	unsigned int rt_nr_boosted;	735	unsigned int rt_nr_boosted;
737		736
738	struct rq *rq;	737	struct rq *rq;
@@ -820,6 +819,9 @@ static inline void se_update_runnable(struct sched_entity *se) · 81 unmodified lines
820		819
821	static inline long se_runnable(struct sched_entity *se)	820	static inline long se_runnable(struct sched_entity *se)
822	{	821	{
		822	if (se->sched_delayed)
		823	return false;
		824
823	if (entity_is_task(se))	825	if (entity_is_task(se))
824	return !!se->on_rq;	826	return !!se->on_rq;
825	else	827	else
@@ -834,6 +836,9 @@ static inline void se_update_runnable(struct sched_entity *se) { } · 8 unmodified lines
834		836
835	static inline long se_runnable(struct sched_entity *se)	837	static inline long se_runnable(struct sched_entity *se)
836	{	838	{
		839	if (se->sched_delayed)
		840	return false;
		841
837	return !!se->on_rq;	842	return !!se->on_rq;
838	}	843	}
839		844
@@ -1044,6 +1049,8 @@ struct rq { · 204 unmodified lines
1044	struct rt_rq rt;	1049	struct rt_rq rt;
1045	struct dl_rq dl;	1050	struct dl_rq dl;
1046		1051
		1052	struct sched_dl_entity fair_server;
		1053
1047	#ifdef CONFIG_FAIR_GROUP_SCHED	1054	#ifdef CONFIG_FAIR_GROUP_SCHED
1048	/* list of leaf cfs_rq on this CPU: */	1055	/* list of leaf cfs_rq on this CPU: */
1049	struct list_head leaf_cfs_rq_list;	1056	struct list_head leaf_cfs_rq_list;
@@ -1059,6 +1066,7 @@ struct rq { · 9 unmodified lines
1059	unsigned int nr_uninterruptible;	1066	unsigned int nr_uninterruptible;
1060		1067
1061	struct task_struct __rcu *curr;	1068	struct task_struct __rcu *curr;
		1069	struct sched_dl_entity *dl_server;
1062	struct task_struct *idle;	1070	struct task_struct *idle;
1063	struct task_struct *stop;	1071	struct task_struct *stop;
1064	unsigned long next_balance;	1072	unsigned long next_balance;
@@ -1158,7 +1166,6 @@ struct rq { · 93 unmodified lines
1158	/* latency stats */	1166	/* latency stats */
1159	struct sched_info rq_sched_info;	1167	struct sched_info rq_sched_info;
1160	unsigned long long rq_cpu_time;	1168	unsigned long long rq_cpu_time;
1161	/* could above be rq->cfs_rq.exec_clock + rq->rt_rq.rt_runtime ? */
1162		1169
1163	/* sys_sched_yield() stats */	1170	/* sys_sched_yield() stats */
1164	unsigned int yld_count;	1171	unsigned int yld_count;
@@ -1187,6 +1194,7 @@ struct rq { · 22 unmodified lines
1187	/* per rq */	1194	/* per rq */
1188	struct rq *core;	1195	struct rq *core;
1189	struct task_struct *core_pick;	1196	struct task_struct *core_pick;
		1197	struct sched_dl_entity *core_dl_server;
1190	unsigned int core_enabled;	1198	unsigned int core_enabled;
1191	unsigned int core_sched_seq;	1199	unsigned int core_sched_seq;
1192	struct rb_root core_tree;	1200	struct rb_root core_tree;
@@ -2247,11 +2255,13 @@ extern const u32 sched_prio_to_wmult[40]; · 1054 unmodified lines
2247	*	2255	*
2248	*/	2256	*/
2249		2257
2250	#define DEQUEUE_SLEEP 0x01	2258	#define DEQUEUE_SLEEP 0x01 /* Matches ENQUEUE_WAKEUP */
2251	#define DEQUEUE_SAVE 0x02 /* Matches ENQUEUE_RESTORE */	2259	#define DEQUEUE_SAVE 0x02 /* Matches ENQUEUE_RESTORE */
2252	#define DEQUEUE_MOVE 0x04 /* Matches ENQUEUE_MOVE */	2260	#define DEQUEUE_MOVE 0x04 /* Matches ENQUEUE_MOVE */
2253	#define DEQUEUE_NOCLOCK 0x08 /* Matches ENQUEUE_NOCLOCK */	2261	#define DEQUEUE_NOCLOCK 0x08 /* Matches ENQUEUE_NOCLOCK */
		2262	#define DEQUEUE_SPECIAL 0x10
2254	#define DEQUEUE_MIGRATING 0x100 /* Matches ENQUEUE_MIGRATING */	2263	#define DEQUEUE_MIGRATING 0x100 /* Matches ENQUEUE_MIGRATING */
		2264	#define DEQUEUE_DELAYED 0x200 /* Matches ENQUEUE_DELAYED */
2255		2265
2256	#define ENQUEUE_WAKEUP 0x01	2266	#define ENQUEUE_WAKEUP 0x01
2257	#define ENQUEUE_RESTORE 0x02	2267	#define ENQUEUE_RESTORE 0x02
@@ -2267,6 +2277,7 @@ extern const u32 sched_prio_to_wmult[40]; · 9 unmodified lines
2267	#endif	2277	#endif
2268	#define ENQUEUE_INITIAL 0x80	2278	#define ENQUEUE_INITIAL 0x80
2269	#define ENQUEUE_MIGRATING 0x100	2279	#define ENQUEUE_MIGRATING 0x100
		2280	#define ENQUEUE_DELAYED 0x200
2270		2281
2271	#define RETRY_TASK ((void *)-1UL)	2282	#define RETRY_TASK ((void *)-1UL)
2272		2283
@@ -2285,23 +2296,31 @@ struct sched_class { · 12 unmodified lines
2285	#endif	2296	#endif
2286		2297
2287	void (enqueue_task) (struct rq rq, struct task_struct *p, int flags);	2298	void (enqueue_task) (struct rq rq, struct task_struct *p, int flags);
2288	void (dequeue_task) (struct rq rq, struct task_struct *p, int flags);	2299	bool (dequeue_task) (struct rq rq, struct task_struct *p, int flags);
2289	void (yield_task) (struct rq rq);	2300	void (yield_task) (struct rq rq);
2290	bool (yield_to_task)(struct rq rq, struct task_struct *p);	2301	bool (yield_to_task)(struct rq rq, struct task_struct *p);
2291		2302
2292	void (wakeup_preempt)(struct rq rq, struct task_struct *p, int flags);	2303	void (wakeup_preempt)(struct rq rq, struct task_struct *p, int flags);
2293		2304
2294	struct task_struct (pick_next_task)(struct rq *rq);	2305	struct task_struct (pick_task)(struct rq *rq);
		2306	/*
		2307	* Optional! When implemented pick_next_task() should be equivalent to:
		2308	*
		2309	* next = pick_task();
		2310	* if (next) {
		2311	* put_prev_task(prev);
		2312	* set_next_task_first(next);
		2313	* }
		2314	*/
		2315	struct task_struct (pick_next_task)(struct rq rq, struct task_struct prev);
2295		2316
2296	void (put_prev_task)(struct rq rq, struct task_struct *p);	2317	void (put_prev_task)(struct rq rq, struct task_struct p, struct task_struct next);
2297	void (set_next_task)(struct rq rq, struct task_struct *p, bool first);	2318	void (set_next_task)(struct rq rq, struct task_struct *p, bool first);
2298		2319
2299	#ifdef CONFIG_SMP	2320	#ifdef CONFIG_SMP
2300	int (balance)(struct rq rq, struct task_struct prev, struct rq_flags rf);	2321	int (balance)(struct rq rq, struct task_struct prev, struct rq_flags rf);
2301	int (select_task_rq)(struct task_struct p, int task_cpu, int flags);	2322	int (select_task_rq)(struct task_struct p, int task_cpu, int flags);
2302		2323
2303	struct task_struct * (pick_task)(struct rq rq);
2304
2305	void (migrate_task_rq)(struct task_struct p, int new_cpu);	2324	void (migrate_task_rq)(struct task_struct p, int new_cpu);
2306		2325
2307	void (task_woken)(struct rq this_rq, struct task_struct *task);	2326	void (task_woken)(struct rq this_rq, struct task_struct *task);
@@ -2345,7 +2364,7 @@ struct sched_class { · 37 unmodified lines
2345	static inline void put_prev_task(struct rq rq, struct task_struct prev)	2364	static inline void put_prev_task(struct rq rq, struct task_struct prev)
2346	{	2365	{
2347	WARN_ON_ONCE(rq->curr != prev);	2366	WARN_ON_ONCE(rq->curr != prev);
2348	prev->sched_class->put_prev_task(rq, prev);	2367	prev->sched_class->put_prev_task(rq, prev, NULL);
2349	}	2368	}
2350		2369
2351	static inline void set_next_task(struct rq rq, struct task_struct next)	2370	static inline void set_next_task(struct rq rq, struct task_struct next)
@@ -2353,6 +2372,30 @@ static inline void set_next_task(struct rq rq, struct task_struct next) · 1 unmodified line
2353	next->sched_class->set_next_task(rq, next, false);	2372	next->sched_class->set_next_task(rq, next, false);
2354	}	2373	}
2355		2374
		2375	static inline void
		2376	__put_prev_set_next_dl_server(struct rq *rq,
		2377	struct task_struct *prev,
		2378	struct task_struct *next)
		2379	{
		2380	prev->dl_server = NULL;
		2381	next->dl_server = rq->dl_server;
		2382	rq->dl_server = NULL;
		2383	}
		2384
		2385	static inline void put_prev_set_next_task(struct rq *rq,
		2386	struct task_struct *prev,
		2387	struct task_struct *next)
		2388	{
		2389	WARN_ON_ONCE(rq->curr != prev);
		2390
		2391	__put_prev_set_next_dl_server(rq, prev, next);
		2392
		2393	if (next == prev)
		2394	return;
		2395
		2396	prev->sched_class->put_prev_task(rq, prev, next);
		2397	next->sched_class->set_next_task(rq, next, true);
		2398	}
2356		2399
2357	/*	2400	/*
2358	* Helper to define a sched_class instance; each one is placed in a separate	2401	* Helper to define a sched_class instance; each one is placed in a separate
@@ -2408,7 +2451,7 @@ static inline bool sched_fair_runnable(struct rq *rq) · 49 unmodified lines
2408	}	2451	}
2409		2452
2410	extern struct task_struct pick_next_task_fair(struct rq rq, struct task_struct prev, struct rq_flags rf);	2453	extern struct task_struct pick_next_task_fair(struct rq rq, struct task_struct prev, struct rq_flags rf);
2411	extern struct task_struct pick_next_task_idle(struct rq rq);	2454	extern struct task_struct pick_task_idle(struct rq rq);
2412		2455
2413	#define SCA_CHECK 0x01	2456	#define SCA_CHECK 0x01
2414	#define SCA_MIGRATE_DISABLE 0x02	2457	#define SCA_MIGRATE_DISABLE 0x02
@@ -2515,7 +2558,6 @@ extern void reweight_task(struct task_struct p, const struct load_weight lw); · 100 unmodified lines
2515	extern void resched_curr(struct rq *rq);	2558	extern void resched_curr(struct rq *rq);
2516	extern void resched_cpu(int cpu);	2559	extern void resched_cpu(int cpu);
2517		2560
2518	extern struct rt_bandwidth def_rt_bandwidth;
2519	extern void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime);	2561	extern void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime);
2520	extern bool sched_rt_bandwidth_account(struct rt_rq *rt_rq);	2562	extern bool sched_rt_bandwidth_account(struct rt_rq *rt_rq);
2521		2563
@@ -2586,6 +2628,19 @@ static inline void sub_nr_running(struct rq *rq, unsigned count) · 64 unmodified lines
2586	sched_update_tick_dependency(rq);	2628	sched_update_tick_dependency(rq);
2587	}	2629	}
2588		2630
		2631	static inline void __block_task(struct rq rq, struct task_struct p)
		2632	{
		2633	WRITE_ONCE(p->on_rq, 0);
		2634	ASSERT_EXCLUSIVE_WRITER(p->on_rq);
		2635	if (p->sched_contributes_to_load)
		2636	rq->nr_uninterruptible++;
		2637
		2638	if (p->in_iowait) {
		2639	atomic_inc(&rq->nr_iowait);
		2640	delayacct_blkio_start();
		2641	}
		2642	}
		2643
2589	extern void activate_task(struct rq rq, struct task_struct p, int flags);	2644	extern void activate_task(struct rq rq, struct task_struct p, int flags);
2590	extern void deactivate_task(struct rq rq, struct task_struct p, int flags);	2645	extern void deactivate_task(struct rq rq, struct task_struct p, int flags);
2591		2646
@@ -3607,7 +3662,7 @@ extern int __sched_setaffinity(struct task_struct p, struct affinity_context c · 1015 unmodified lines
3607	extern void __setscheduler_prio(struct task_struct *p, int prio);	3662	extern void __setscheduler_prio(struct task_struct *p, int prio);
3608	extern void set_load_weight(struct task_struct *p, bool update_load);	3663	extern void set_load_weight(struct task_struct *p, bool update_load);
3609	extern void enqueue_task(struct rq rq, struct task_struct p, int flags);	3664	extern void enqueue_task(struct rq rq, struct task_struct p, int flags);
3610	extern void dequeue_task(struct rq rq, struct task_struct p, int flags);	3665	extern bool dequeue_task(struct rq rq, struct task_struct p, int flags);
3611		3666
3612	extern void check_class_changed(struct rq rq, struct task_struct p,	3667	extern void check_class_changed(struct rq rq, struct task_struct p,
3613	const struct sched_class *prev_class,	3668	const struct sched_class *prev_class,

kernel/sched/stop_task.c+4−14

 	return rq->stop;
 }
-static struct task_struct *pick_next_task_stop(struct rq *rq)
-{
-	struct task_struct *p = pick_task_stop(rq);
-	if (p)
-		set_next_task_stop(rq, p, true);
-	return p;
-}
 static void
 enqueue_task_stop(struct rq *rq, struct task_struct *p, int flags)
 {
 	add_nr_running(rq, 1);
 }
-static void
+static bool
 dequeue_task_stop(struct rq *rq, struct task_struct *p, int flags)
 {
 	sub_nr_running(rq, 1);
+	return true;
 }
 static void yield_task_stop(struct rq *rq)
 	BUG(); /* the stop task should never yield, its pointless. */
 }
-static void put_prev_task_stop(struct rq *rq, struct task_struct *prev)
+static void put_prev_task_stop(struct rq *rq, struct task_struct *prev, struct task_struct *next)
 {
 	update_curr_common(rq);
 }
 	.wakeup_preempt		= wakeup_preempt_stop,
-	.pick_next_task		= pick_next_task_stop,
+	.pick_task		= pick_task_stop,
 	.put_prev_task		= put_prev_task_stop,
 	.set_next_task          = set_next_task_stop,
 #ifdef CONFIG_SMP
 	.balance		= balance_stop,
-	.pick_task		= pick_task_stop,
 	.select_task_rq		= select_task_rq_stop,
 	.set_cpus_allowed	= set_cpus_allowed_common,
 #endif

kernel/sched/syscalls.c+25−109

@@ -57,7 +57,7 @@ static int effective_prio(struct task_struct *p) · 56 unmodified lines
57	* keep the priority unchanged. Otherwise, update priority	57	* keep the priority unchanged. Otherwise, update priority
58	* to the normal priority:	58	* to the normal priority:
59	*/	59	*/
60	if (!rt_prio(p->prio))	60	if (!rt_or_dl_prio(p->prio))
61	return p->normal_prio;	61	return p->normal_prio;
62	return p->prio;	62	return p->prio;
63	}	63	}
@@ -258,107 +258,6 @@ int sched_core_idle_cpu(int cpu) · 194 unmodified lines
258		258
259	#endif	259	#endif
260		260
261	#ifdef CONFIG_SMP
262	/*
263	* This function computes an effective utilization for the given CPU, to be
264	* used for frequency selection given the linear relation: f = u * f_max.
265	*
266	* The scheduler tracks the following metrics:
267	*
268	* cpu_util_{cfs,rt,dl,irq}()
269	* cpu_bw_dl()
270	*
271	* Where the cfs,rt and dl util numbers are tracked with the same metric and
272	* synchronized windows and are thus directly comparable.
273	*
274	* The cfs,rt,dl utilization are the running times measured with rq->clock_task
275	* which excludes things like IRQ and steal-time. These latter are then accrued
276	* in the IRQ utilization.
277	*
278	* The DL bandwidth number OTOH is not a measured metric but a value computed
279	* based on the task model parameters and gives the minimal utilization
280	* required to meet deadlines.
281	*/
282	unsigned long effective_cpu_util(int cpu, unsigned long util_cfs,
283	unsigned long *min,
284	unsigned long *max)
285	{
286	unsigned long util, irq, scale;
287	struct rq *rq = cpu_rq(cpu);
288
289	scale = arch_scale_cpu_capacity(cpu);
290
291	/*
292	* Early check to see if IRQ/steal time saturates the CPU, can be
293	* because of inaccuracies in how we track these -- see
294	* update_irq_load_avg().
295	*/
296	irq = cpu_util_irq(rq);
297	if (unlikely(irq >= scale)) {
298	if (min)
299	*min = scale;
300	if (max)
301	*max = scale;
302	return scale;
303	}
304
305	if (min) {
306	/*
307	* The minimum utilization returns the highest level between:
308	* - the computed DL bandwidth needed with the IRQ pressure which
309	* steals time to the deadline task.
310	* - The minimum performance requirement for CFS and/or RT.
311	*/
312	*min = max(irq + cpu_bw_dl(rq), uclamp_rq_get(rq, UCLAMP_MIN));
313
314	/*
315	* When an RT task is runnable and uclamp is not used, we must
316	* ensure that the task will run at maximum compute capacity.
317	*/
318	if (!uclamp_is_used() && rt_rq_is_runnable(&rq->rt))
319	min = max(min, scale);
320	}
321
322	/*
323	* Because the time spend on RT/DL tasks is visible as 'lost' time to
324	* CFS tasks and we use the same metric to track the effective
325	* utilization (PELT windows are synchronized) we can directly add them
326	* to obtain the CPU's actual utilization.
327	*/
328	util = util_cfs + cpu_util_rt(rq);
329	util += cpu_util_dl(rq);
330
331	/*
332	* The maximum hint is a soft bandwidth requirement, which can be lower
333	* than the actual utilization because of uclamp_max requirements.
334	*/
335	if (max)
336	*max = min(scale, uclamp_rq_get(rq, UCLAMP_MAX));
337
338	if (util >= scale)
339	return scale;
340
341	/*
342	* There is still idle time; further improve the number by using the
343	* IRQ metric. Because IRQ/steal time is hidden from the task clock we
344	* need to scale the task numbers:
345	*
346	* max - irq
347	* U' = irq + --------- * U
348	* max
349	*/
350	util = scale_irq_capacity(util, irq, scale);
351	util += irq;
352
353	return min(scale, util);
354	}
355
356	unsigned long sched_cpu_util(int cpu)
357	{
358	return effective_cpu_util(cpu, cpu_util_cfs(cpu), NULL, NULL);
359	}
360	#endif /* CONFIG_SMP */
361
362	/**	261	/**
363	* find_process_by_pid - find a process with a matching PID value.	262	* find_process_by_pid - find a process with a matching PID value.
364	* @pid: the pid in question.	263	* @pid: the pid in question.
@@ -401,13 +300,23 @@ static void __setscheduler_params(struct task_struct *p, · 36 unmodified lines
401		300
402	p->policy = policy;	301	p->policy = policy;
403		302
404	if (dl_policy(policy))	303	if (dl_policy(policy)) {
405	__setparam_dl(p, attr);	304	__setparam_dl(p, attr);
406	else if (fair_policy(policy))	305	} else if (fair_policy(policy)) {
407	p->static_prio = NICE_TO_PRIO(attr->sched_nice);	306	p->static_prio = NICE_TO_PRIO(attr->sched_nice);
		307	if (attr->sched_runtime) {
		308	p->se.custom_slice = 1;
		309	p->se.slice = clamp_t(u64, attr->sched_runtime,
		310	NSEC_PER_MSEC/10, /* HZ=1000 * 10 */
		311	NSEC_PER_MSEC100); / HZ=100 / 10 */
		312	} else {
		313	p->se.custom_slice = 0;
		314	p->se.slice = sysctl_sched_base_slice;
		315	}
		316	}
408		317
409	/* rt-policy tasks do not have a timerslack */	318	/* rt-policy tasks do not have a timerslack */
410	if (task_is_realtime(p)) {	319	if (rt_or_dl_task_policy(p)) {
411	p->timer_slack_ns = 0;	320	p->timer_slack_ns = 0;
412	} else if (p->timer_slack_ns == 0) {	321	} else if (p->timer_slack_ns == 0) {
413	/* when switching back to non-rt policy, restore timerslack */	322	/* when switching back to non-rt policy, restore timerslack */
@@ -708,7 +617,9 @@ recheck: · 294 unmodified lines
708	* but store a possible modification of reset_on_fork.	617	* but store a possible modification of reset_on_fork.
709	*/	618	*/
710	if (unlikely(policy == p->policy)) {	619	if (unlikely(policy == p->policy)) {
711	if (fair_policy(policy) && attr->sched_nice != task_nice(p))	620	if (fair_policy(policy) &&
		621	(attr->sched_nice != task_nice(p) \|\|
		622	(attr->sched_runtime != p->se.slice)))
712	goto change;	623	goto change;
713	if (rt_policy(policy) && attr->sched_priority != p->rt_priority)	624	if (rt_policy(policy) && attr->sched_priority != p->rt_priority)
714	goto change;	625	goto change;
@@ -854,6 +765,9 @@ static int _sched_setscheduler(struct task_struct *p, int policy, · 139 unmodified lines
854	.sched_nice = PRIO_TO_NICE(p->static_prio),	765	.sched_nice = PRIO_TO_NICE(p->static_prio),
855	};	766	};
856		767
		768	if (p->se.custom_slice)
		769	attr.sched_runtime = p->se.slice;
		770
857	/* Fixup the legacy SCHED_RESET_ON_FORK hack. */	771	/* Fixup the legacy SCHED_RESET_ON_FORK hack. */
858	if ((policy != SETPARAM_POLICY) && (policy & SCHED_RESET_ON_FORK)) {	772	if ((policy != SETPARAM_POLICY) && (policy & SCHED_RESET_ON_FORK)) {
859	attr.sched_flags \|= SCHED_FLAG_RESET_ON_FORK;	773	attr.sched_flags \|= SCHED_FLAG_RESET_ON_FORK;
@@ -1020,12 +934,14 @@ err_size: · 160 unmodified lines
1020		934
1021	static void get_params(struct task_struct p, struct sched_attr attr)	935	static void get_params(struct task_struct p, struct sched_attr attr)
1022	{	936	{
1023	if (task_has_dl_policy(p))	937	if (task_has_dl_policy(p)) {
1024	__getparam_dl(p, attr);	938	__getparam_dl(p, attr);
1025	else if (task_has_rt_policy(p))	939	} else if (task_has_rt_policy(p)) {
1026	attr->sched_priority = p->rt_priority;	940	attr->sched_priority = p->rt_priority;
1027	else	941	} else {
1028	attr->sched_nice = task_nice(p);	942	attr->sched_nice = task_nice(p);
		943	attr->sched_runtime = p->se.slice;
		944	}
1029	}	945	}
1030		946
1031	/**	947	/**

kernel/sched/topology.c+8−0

@@ -516,6 +516,14 @@ void rq_attach_root(struct rq rq, struct root_domain rd) · 515 unmodified lines
516	if (cpumask_test_cpu(rq->cpu, cpu_active_mask))	516	if (cpumask_test_cpu(rq->cpu, cpu_active_mask))
517	set_rq_online(rq);	517	set_rq_online(rq);
518		518
		519	/*
		520	* Because the rq is not a task, dl_add_task_root_domain() did not
		521	* move the fair server bw to the rd if it already started.
		522	* Add it now.
		523	*/
		524	if (rq->fair_server.dl_server)
		525	__dl_server_attach_root(&rq->fair_server, rq);
		526
519	rq_unlock_irqrestore(rq, &rf);	527	rq_unlock_irqrestore(rq, &rf);
520		528
521	if (old_rd)	529	if (old_rd)

kernel/sys.c+1−1

@@ -2557,7 +2557,7 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, · 2556 unmodified lines
2557	error = current->timer_slack_ns;	2557	error = current->timer_slack_ns;
2558	break;	2558	break;
2559	case PR_SET_TIMERSLACK:	2559	case PR_SET_TIMERSLACK:
2560	if (task_is_realtime(current))	2560	if (rt_or_dl_task_policy(current))
2561	break;	2561	break;
2562	if (arg2 <= 0)	2562	if (arg2 <= 0)
2563	current->timer_slack_ns =	2563	current->timer_slack_ns =

kernel/time/hrtimer.c+1−1

@@ -1977,7 +1977,7 @@ static void __hrtimer_init_sleeper(struct hrtimer_sleeper *sl, · 1976 unmodified lines
1977	* expiry.	1977	* expiry.
1978	*/	1978	*/
1979	if (IS_ENABLED(CONFIG_PREEMPT_RT)) {	1979	if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
1980	if (task_is_realtime(current) && !(mode & HRTIMER_MODE_SOFT))	1980	if (rt_or_dl_task_policy(current) && !(mode & HRTIMER_MODE_SOFT))
1981	mode \|= HRTIMER_MODE_HARD;	1981	mode \|= HRTIMER_MODE_HARD;
1982	}	1982	}
1983		1983

kernel/trace/trace_sched_wakeup.c+1−1

@@ -547,7 +547,7 @@ probe_wakeup(void ignore, struct task_struct p) · 546 unmodified lines
547	* - wakeup_dl handles tasks belonging to sched_dl class only.	547	* - wakeup_dl handles tasks belonging to sched_dl class only.
548	*/	548	*/
549	if (tracing_dl \|\| (wakeup_dl && !dl_task(p)) \|\|	549	if (tracing_dl \|\| (wakeup_dl && !dl_task(p)) \|\|
550	(wakeup_rt && !dl_task(p) && !rt_task(p)) \|\|	550	(wakeup_rt && !rt_or_dl_task(p)) \|\|
551	(!dl_task(p) && (p->prio >= wakeup_prio \|\| p->prio >= current->prio)))	551	(!dl_task(p) && (p->prio >= wakeup_prio \|\| p->prio >= current->prio)))
552	return;	552	return;
553		553

mm/page-writeback.c+2−2

@@ -418,7 +418,7 @@ static void domain_dirty_limits(struct dirty_throttle_control *dtc) · 417 unmodified lines
418	bg_thresh = (bg_ratio * available_memory) / PAGE_SIZE;	418	bg_thresh = (bg_ratio * available_memory) / PAGE_SIZE;
419		419
420	tsk = current;	420	tsk = current;
421	if (rt_task(tsk)) {	421	if (rt_or_dl_task(tsk)) {
422	bg_thresh += bg_thresh / 4 + global_wb_domain.dirty_limit / 32;	422	bg_thresh += bg_thresh / 4 + global_wb_domain.dirty_limit / 32;
423	thresh += thresh / 4 + global_wb_domain.dirty_limit / 32;	423	thresh += thresh / 4 + global_wb_domain.dirty_limit / 32;
424	}	424	}
@@ -477,7 +477,7 @@ static unsigned long node_dirty_limit(struct pglist_data *pgdat) · 52 unmodified lines
477	else	477	else
478	dirty = vm_dirty_ratio * node_memory / 100;	478	dirty = vm_dirty_ratio * node_memory / 100;
479		479
480	if (rt_task(tsk))	480	if (rt_or_dl_task(tsk))
481	dirty += dirty / 4;	481	dirty += dirty / 4;
482		482
483	/*	483	/*

mm/page_alloc.c+1−1

@@ -4004,7 +4004,7 @@ gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order) · 4003 unmodified lines
4004	*/	4004	*/
4005	if (alloc_flags & ALLOC_MIN_RESERVE)	4005	if (alloc_flags & ALLOC_MIN_RESERVE)
4006	alloc_flags &= ~ALLOC_CPUSET;	4006	alloc_flags &= ~ALLOC_CPUSET;
4007	} else if (unlikely(rt_task(current)) && in_task())	4007	} else if (unlikely(rt_or_dl_task(current)) && in_task())
4008	alloc_flags \|= ALLOC_MIN_RESERVE;	4008	alloc_flags \|= ALLOC_MIN_RESERVE;
4009		4009
4010	alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, alloc_flags);	4010	alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, alloc_flags);