Merge tag 'sched_urgent_for_v5.18_rc2' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull scheduler fixes from Borislav Petkov: - Use the correct static key checking primitive on the IRQ exit path - Two fixes for the new forceidle balancer * tag 'sched_urgent_for_v5.18_rc2' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: entry: Fix compile error in dynamic_irqentry_exit_cond_resched() sched: Teach the forced-newidle balancer about CPU affinity limitation. sched/core: Fix forceidle balancing

Linus Torvalds committed 4 years ago

Commit 71368492 parents 1862a69 0a700454 files · +12 −13Browse files

Unified · Split

4 files

kernel/entry/common.c +1 −1
kernel/sched/core.c +11 −5
kernel/sched/idle.c +0 −1
kernel/sched/sched.h +0 −6

kernel/entry/common.c+1−1

@@ -392,7 +392,7 @@ DEFINE_STATIC_CALL(irqentry_exit_cond_resched, raw_irqentry_exit_cond_resched); · 391 unmodified lines
392	392	DEFINE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched);
393	393	void dynamic_irqentry_exit_cond_resched(void)
394	394	{
395		− if (!static_key_unlikely(&sk_dynamic_irqentry_exit_cond_resched))
	395	+ if (!static_branch_unlikely(&sk_dynamic_irqentry_exit_cond_resched))
396	396	return;
397	397	raw_irqentry_exit_cond_resched();
398	398	}

kernel/sched/core.c+11−5

@@ -5752,6 +5752,8 @@ static inline struct task_struct pick_task(struct rq rq) · 5751 unmodified lines
5752	5752
5753	5753	extern void task_vruntime_update(struct rq rq, struct task_struct p, bool in_fi);
5754	5754
	5755	+static void queue_core_balance(struct rq *rq);
	5756	+
5755	5757	static struct task_struct *
5756	5758	pick_next_task(struct rq rq, struct task_struct prev, struct rq_flags *rf)
5757	5759	{
@@ -5801,7 +5803,7 @@ pick_next_task(struct rq rq, struct task_struct prev, struct rq_flags *rf) · 43 unmodified lines
5801	5803	}
5802	5804
5803	5805	rq->core_pick = NULL;
5804		− return next;
	5806	+ goto out;
5805	5807	}
5806	5808
5807	5809	put_prev_task_balance(rq, prev, rf);
@@ -5851,7 +5853,7 @@ pick_next_task(struct rq rq, struct task_struct prev, struct rq_flags *rf) · 43 unmodified lines
5851	5853	*/
5852	5854	WARN_ON_ONCE(fi_before);
5853	5855	task_vruntime_update(rq, next, false);
5854		− goto done;
	5856	+ goto out_set_next;
5855	5857	}
5856	5858	}
5857	5859
@@ -5970,8 +5972,12 @@ pick_next_task(struct rq rq, struct task_struct prev, struct rq_flags *rf) · 112 unmodified lines
5970	5972	resched_curr(rq_i);
5971	5973	}
5972	5974
5973		−done:
	5975	+out_set_next:
5974	5976	set_next_task(rq, next);
	5977	+out:
	5978	+ if (rq->core->core_forceidle_count && next == rq->idle)
	5979	+ queue_core_balance(rq);
	5980	+
5975	5981	return next;
5976	5982	}
5977	5983
@@ -6000,7 +6006,7 @@ static bool try_steal_cookie(int this, int that) · 22 unmodified lines
6000	6006	if (p == src->core_pick \|\| p == src->curr)
6001	6007	goto next;
6002	6008
6003		− if (!cpumask_test_cpu(this, &p->cpus_mask))
	6009	+ if (!is_cpu_allowed(p, this))
6004	6010	goto next;
6005	6011
6006	6012	if (p->core_occupation > dst->idle->core_occupation)
@@ -6066,7 +6072,7 @@ static void sched_core_balance(struct rq *rq) · 59 unmodified lines
6066	6072
6067	6073	static DEFINE_PER_CPU(struct callback_head, core_balance_head);
6068	6074
6069		−void queue_core_balance(struct rq *rq)
	6075	+static void queue_core_balance(struct rq *rq)
6070	6076	{
6071	6077	if (!sched_core_enabled(rq))
6072	6078	return;

kernel/sched/idle.c+0−1

@@ -434,7 +434,6 @@ static void set_next_task_idle(struct rq rq, struct task_struct next, bool fir · 433 unmodified lines
434	434	{
435	435	update_idle_core(rq);
436	436	schedstat_inc(rq->sched_goidle);
437		− queue_core_balance(rq);
438	437	}
439	438
440	439	#ifdef CONFIG_SMP

kernel/sched/sched.h+0−6

@@ -1232,8 +1232,6 @@ static inline bool sched_group_cookie_match(struct rq *rq, · 1231 unmodified lines
1232	1232	return false;
1233	1233	}
1234	1234
1235		−extern void queue_core_balance(struct rq *rq);
1236		−
1237	1235	static inline bool sched_core_enqueued(struct task_struct *p)
1238	1236	{
1239	1237	return !RB_EMPTY_NODE(&p->core_node);
@@ -1267,10 +1265,6 @@ static inline raw_spinlock_t __rq_lockp(struct rq rq) · 27 unmodified lines
1267	1265	return &rq->__lock;
1268	1266	}
1269	1267
1270		−static inline void queue_core_balance(struct rq *rq)
1271		−{
1272		−}
1273		−
1274	1268	static inline bool sched_cpu_cookie_match(struct rq rq, struct task_struct p)
1275	1269	{
1276	1270	return true;