Merge tag 'sched_urgent_for_v5.18_rc2' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull scheduler fixes from Borislav Petkov: - Use the correct static key checking primitive on the IRQ exit path - Two fixes for the new forceidle balancer * tag 'sched_urgent_for_v5.18_rc2' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: entry: Fix compile error in dynamic_irqentry_exit_cond_resched() sched: Teach the forced-newidle balancer about CPU affinity limitation. sched/core: Fix forceidle balancing

Linus Torvalds committed 4 years ago

Commit 71368492 parents 1862a69 0a700454 files · +12 −13Browse files

Unified · Split

4 files

kernel/entry/common.c +1 −1
kernel/sched/core.c +11 −5
kernel/sched/idle.c +0 −1
kernel/sched/sched.h +0 −6

kernel/entry/common.c+1−1

@@ -392,7 +392,7 @@ DEFINE_STATIC_CALL(irqentry_exit_cond_resched, raw_irqentry_exit_cond_resched); · 391 unmodified lines
392	DEFINE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched);	392	DEFINE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched);
393	void dynamic_irqentry_exit_cond_resched(void)	393	void dynamic_irqentry_exit_cond_resched(void)
394	{	394	{
395	if (!static_key_unlikely(&sk_dynamic_irqentry_exit_cond_resched))	395	if (!static_branch_unlikely(&sk_dynamic_irqentry_exit_cond_resched))
396	return;	396	return;
397	raw_irqentry_exit_cond_resched();	397	raw_irqentry_exit_cond_resched();
398	}	398	}

kernel/sched/core.c+11−5

 extern void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_fi);
+static void queue_core_balance(struct rq *rq);
 static struct task_struct *
 pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
 {
 		}
 		rq->core_pick = NULL;
-		return next;
+		goto out;
 	}
 	put_prev_task_balance(rq, prev, rf);
 			 */
 			WARN_ON_ONCE(fi_before);
 			task_vruntime_update(rq, next, false);
-			goto done;
+			goto out_set_next;
 		}
 	}
 		resched_curr(rq_i);
 	}
-done:
+out_set_next:
 	set_next_task(rq, next);
+out:
+	if (rq->core->core_forceidle_count && next == rq->idle)
+		queue_core_balance(rq);
 	return next;
 }
 		if (p == src->core_pick || p == src->curr)
 			goto next;
-		if (!cpumask_test_cpu(this, &p->cpus_mask))
+		if (!is_cpu_allowed(p, this))
 			goto next;
 		if (p->core_occupation > dst->idle->core_occupation)
 static DEFINE_PER_CPU(struct callback_head, core_balance_head);
-void queue_core_balance(struct rq *rq)
+static void queue_core_balance(struct rq *rq)
 {
 	if (!sched_core_enabled(rq))
 		return;

kernel/sched/idle.c+0−1

@@ -434,7 +434,6 @@ static void set_next_task_idle(struct rq rq, struct task_struct next, bool fir · 433 unmodified lines
434	{	434	{
435	update_idle_core(rq);	435	update_idle_core(rq);
436	schedstat_inc(rq->sched_goidle);	436	schedstat_inc(rq->sched_goidle);
437	queue_core_balance(rq);
438	}	437	}
439		438
440	#ifdef CONFIG_SMP	439	#ifdef CONFIG_SMP

kernel/sched/sched.h+0−6

 	return false;
 }
-extern void queue_core_balance(struct rq *rq);
 static inline bool sched_core_enqueued(struct task_struct *p)
 {
 	return !RB_EMPTY_NODE(&p->core_node);
 	return &rq->__lock;
 }
-static inline void queue_core_balance(struct rq *rq)
-{
-}
 static inline bool sched_cpu_cookie_match(struct rq *rq, struct task_struct *p)
 {
 	return true;