// SPDX-License-Identifier: GPL-2.0-only/* *  kernel/sched/core.c * *  Core kernel scheduler code and related syscalls * *  Copyright (C) 1991-2002  Linus Torvalds */#define CREATE_TRACE_POINTS#include <trace/events/sched.h>#undef CREATE_TRACE_POINTS #include "sched.h" #include <linux/nospec.h> #include <linux/kcov.h>#include <linux/scs.h> #include <asm/switch_to.h>#include <asm/tlb.h> #include "../workqueue_internal.h"#include "../../fs/io-wq.h"#include "../smpboot.h" #include "pelt.h"#include "smp.h" /* * Export tracepoints that act as a bare tracehook (ie: have no trace event * associated with them) to allow external modules to probe them. */EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_cfs_tp);EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_rt_tp);EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_dl_tp);EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_irq_tp);EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_se_tp);EXPORT_TRACEPOINT_SYMBOL_GPL(sched_cpu_capacity_tp);EXPORT_TRACEPOINT_SYMBOL_GPL(sched_overutilized_tp);EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_cfs_tp);EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_se_tp);EXPORT_TRACEPOINT_SYMBOL_GPL(sched_update_nr_running_tp); DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); #ifdef CONFIG_SCHED_DEBUG/* * Debugging: various feature bits * * If SCHED_DEBUG is disabled, each compilation unit has its own copy of * sysctl_sched_features, defined in sched.h, to allow constants propagation * at compile time and compiler optimization based on features default. */#define SCHED_FEAT(name, enabled)	\	(1UL << __SCHED_FEAT_##name) * enabled |const_debug unsigned int sysctl_sched_features =#include "features.h"	0;#undef SCHED_FEAT /* * Print a warning if need_resched is set for the given duration (if * LATENCY_WARN is enabled). * * If sysctl_resched_latency_warn_once is set, only one warning will be shown * per boot. */__read_mostly int sysctl_resched_latency_warn_ms = 100;__read_mostly int sysctl_resched_latency_warn_once = 1;#endif /* CONFIG_SCHED_DEBUG */ /* * Number of tasks to iterate in a single balance run. * Limited because this is done with IRQs disabled. */const_debug unsigned int sysctl_sched_nr_migrate = 32; /* * period over which we measure -rt task CPU usage in us. * default: 1s */unsigned int sysctl_sched_rt_period = 1000000; __read_mostly int scheduler_running; /* * part of the period that we allow rt tasks to run in us. * default: 0.95s */int sysctl_sched_rt_runtime = 950000;  /* * Serialization rules: * * Lock order: * *   p->pi_lock *     rq->lock *       hrtimer_cpu_base->lock (hrtimer_start() for bandwidth controls) * *  rq1->lock *    rq2->lock  where: rq1 < rq2 * * Regular state: * * Normal scheduling state is serialized by rq->lock. __schedule() takes the * local CPU's rq->lock, it optionally removes the task from the runqueue and * always looks at the local rq data structures to find the most eligible task * to run next. * * Task enqueue is also under rq->lock, possibly taken from another CPU. * Wakeups from another LLC domain might use an IPI to transfer the enqueue to * the local CPU to avoid bouncing the runqueue state around [ see * ttwu_queue_wakelist() ] * * Task wakeup, specifically wakeups that involve migration, are horribly * complicated to avoid having to take two rq->locks. * * Special state: * * System-calls and anything external will use task_rq_lock() which acquires * both p->pi_lock and rq->lock. As a consequence the state they change is * stable while holding either lock: * *  - sched_setaffinity()/ *    set_cpus_allowed_ptr():	p->cpus_ptr, p->nr_cpus_allowed *  - set_user_nice():		p->se.load, p->*prio *  - __sched_setscheduler():	p->sched_class, p->policy, p->*prio, *				p->se.load, p->rt_priority, *				p->dl.dl_{runtime, deadline, period, flags, bw, density} *  - sched_setnuma():		p->numa_preferred_nid *  - sched_move_task()/ *    cpu_cgroup_fork():	p->sched_task_group *  - uclamp_update_active()	p->uclamp* * * p->state <- TASK_*: * *   is changed locklessly using set_current_state(), __set_current_state() or *   set_special_state(), see their respective comments, or by *   try_to_wake_up(). This latter uses p->pi_lock to serialize against *   concurrent self. * * p->on_rq <- { 0, 1 = TASK_ON_RQ_QUEUED, 2 = TASK_ON_RQ_MIGRATING }: * *   is set by activate_task() and cleared by deactivate_task(), under *   rq->lock. Non-zero indicates the task is runnable, the special *   ON_RQ_MIGRATING state is used for migration without holding both *   rq->locks. It indicates task_cpu() is not stable, see task_rq_lock(). * * p->on_cpu <- { 0, 1 }: * *   is set by prepare_task() and cleared by finish_task() such that it will be *   set before p is scheduled-in and cleared after p is scheduled-out, both *   under rq->lock. Non-zero indicates the task is running on its CPU. * *   [ The astute reader will observe that it is possible for two tasks on one *     CPU to have ->on_cpu = 1 at the same time. ] * * task_cpu(p): is changed by set_task_cpu(), the rules are: * *  - Don't call set_task_cpu() on a blocked task: * *    We don't care what CPU we're not running on, this simplifies hotplug, *    the CPU assignment of blocked tasks isn't required to be valid. * *  - for try_to_wake_up(), called under p->pi_lock: * *    This allows try_to_wake_up() to only take one rq->lock, see its comment. * *  - for migration called under rq->lock: *    [ see task_on_rq_migrating() in task_rq_lock() ] * *    o move_queued_task() *    o detach_task() * *  - for migration called under double_rq_lock(): * *    o __migrate_swap_task() *    o push_rt_task() / pull_rt_task() *    o push_dl_task() / pull_dl_task() *    o dl_task_offline_migration() * */ /* * __task_rq_lock - lock the rq @p resides on. */struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf)	__acquires(rq->lock){	struct rq *rq; 	lockdep_assert_held(&p->pi_lock); 	for (;;) {		rq = task_rq(p);		raw_spin_lock(&rq->lock);		if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) {			rq_pin_lock(rq, rf);			return rq;		}		raw_spin_unlock(&rq->lock); 		while (unlikely(task_on_rq_migrating(p)))			cpu_relax();	}} /* * task_rq_lock - lock p->pi_lock and lock the rq @p resides on. */struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf)	__acquires(p->pi_lock)	__acquires(rq->lock){	struct rq *rq; 	for (;;) {		raw_spin_lock_irqsave(&p->pi_lock, rf->flags);		rq = task_rq(p);		raw_spin_lock(&rq->lock);		/*		 *	move_queued_task()		task_rq_lock()		 *		 *	ACQUIRE (rq->lock)		 *	[S] ->on_rq = MIGRATING		[L] rq = task_rq()		 *	WMB (__set_task_cpu())		ACQUIRE (rq->lock);		 *	[S] ->cpu = new_cpu		[L] task_rq()		 *					[L] ->on_rq		 *	RELEASE (rq->lock)		 *		 * If we observe the old CPU in task_rq_lock(), the acquire of		 * the old rq->lock will fully serialize against the stores.		 *		 * If we observe the new CPU in task_rq_lock(), the address		 * dependency headed by '[L] rq = task_rq()' and the acquire		 * will pair with the WMB to ensure we then also see migrating.		 */		if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) {			rq_pin_lock(rq, rf);			return rq;		}		raw_spin_unlock(&rq->lock);		raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags); 		while (unlikely(task_on_rq_migrating(p)))			cpu_relax();	}} /* * RQ-clock updating methods: */ static void update_rq_clock_task(struct rq *rq, s64 delta){/* * In theory, the compile should just see 0 here, and optimize out the call * to sched_rt_avg_update. But I don't trust it... */	s64 __maybe_unused steal = 0, irq_delta = 0; #ifdef CONFIG_IRQ_TIME_ACCOUNTING	irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time; 	/*	 * Since irq_time is only updated on {soft,}irq_exit, we might run into	 * this case when a previous update_rq_clock() happened inside a	 * {soft,}irq region.	 *	 * When this happens, we stop ->clock_task and only update the	 * prev_irq_time stamp to account for the part that fit, so that a next	 * update will consume the rest. This ensures ->clock_task is	 * monotonic.	 *	 * It does however cause some slight miss-attribution of {soft,}irq	 * time, a more accurate solution would be to update the irq_time using	 * the current rq->clock timestamp, except that would require using	 * atomic ops.	 */	if (irq_delta > delta)		irq_delta = delta; 	rq->prev_irq_time += irq_delta;	delta -= irq_delta;#endif#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING	if (static_key_false((&paravirt_steal_rq_enabled))) {		steal = paravirt_steal_clock(cpu_of(rq));		steal -= rq->prev_steal_time_rq; 		if (unlikely(steal > delta))			steal = delta; 		rq->prev_steal_time_rq += steal;		delta -= steal;	}#endif 	rq->clock_task += delta; #ifdef CONFIG_HAVE_SCHED_AVG_IRQ	if ((irq_delta + steal) && sched_feat(NONTASK_CAPACITY))		update_irq_load_avg(rq, irq_delta + steal);#endif	update_rq_clock_pelt(rq, delta);} void update_rq_clock(struct rq *rq){	s64 delta; 	lockdep_assert_held(&rq->lock); 	if (rq->clock_update_flags & RQCF_ACT_SKIP)		return; #ifdef CONFIG_SCHED_DEBUG	if (sched_feat(WARN_DOUBLE_CLOCK))		SCHED_WARN_ON(rq->clock_update_flags & RQCF_UPDATED);	rq->clock_update_flags |= RQCF_UPDATED;#endif 	delta = sched_clock_cpu(cpu_of(rq)) - rq->clock;	if (delta < 0)		return;	rq->clock += delta;	update_rq_clock_task(rq, delta);} #ifdef CONFIG_SCHED_HRTICK/* * Use HR-timers to deliver accurate preemption points. */ static void hrtick_clear(struct rq *rq){	if (hrtimer_active(&rq->hrtick_timer))		hrtimer_cancel(&rq->hrtick_timer);} /* * High-resolution timer tick. * Runs from hardirq context with interrupts disabled. */static enum hrtimer_restart hrtick(struct hrtimer *timer){	struct rq *rq = container_of(timer, struct rq, hrtick_timer);	struct rq_flags rf; 	WARN_ON_ONCE(cpu_of(rq) != smp_processor_id()); 	rq_lock(rq, &rf);	update_rq_clock(rq);	rq->curr->sched_class->task_tick(rq, rq->curr, 1);	rq_unlock(rq, &rf); 	return HRTIMER_NORESTART;} #ifdef CONFIG_SMP static void __hrtick_restart(struct rq *rq){	struct hrtimer *timer = &rq->hrtick_timer;	ktime_t time = rq->hrtick_time; 	hrtimer_start(timer, time, HRTIMER_MODE_ABS_PINNED_HARD);} /* * called from hardirq (IPI) context */static void __hrtick_start(void *arg){	struct rq *rq = arg;	struct rq_flags rf; 	rq_lock(rq, &rf);	__hrtick_restart(rq);	rq_unlock(rq, &rf);} /* * Called to set the hrtick timer state. * * called with rq->lock held and irqs disabled */void hrtick_start(struct rq *rq, u64 delay){	struct hrtimer *timer = &rq->hrtick_timer;	s64 delta; 	/*	 * Don't schedule slices shorter than 10000ns, that just	 * doesn't make sense and can cause timer DoS.	 */	delta = max_t(s64, delay, 10000LL);	rq->hrtick_time = ktime_add_ns(timer->base->get_time(), delta); 	if (rq == this_rq())		__hrtick_restart(rq);	else		smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd);} #else/* * Called to set the hrtick timer state. * * called with rq->lock held and irqs disabled */void hrtick_start(struct rq *rq, u64 delay){	/*	 * Don't schedule slices shorter than 10000ns, that just	 * doesn't make sense. Rely on vruntime for fairness.	 */	delay = max_t(u64, delay, 10000LL);	hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay),		      HRTIMER_MODE_REL_PINNED_HARD);} #endif /* CONFIG_SMP */ static void hrtick_rq_init(struct rq *rq){#ifdef CONFIG_SMP	INIT_CSD(&rq->hrtick_csd, __hrtick_start, rq);#endif	hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);	rq->hrtick_timer.function = hrtick;}#else	/* CONFIG_SCHED_HRTICK */static inline void hrtick_clear(struct rq *rq){} static inline void hrtick_rq_init(struct rq *rq){}#endif	/* CONFIG_SCHED_HRTICK */ /* * cmpxchg based fetch_or, macro so it works for different integer types */#define fetch_or(ptr, mask)						\	({								\		typeof(ptr) _ptr = (ptr);				\		typeof(mask) _mask = (mask);				\		typeof(*_ptr) _old, _val = *_ptr;			\									\		for (;;) {						\			_old = cmpxchg(_ptr, _val, _val | _mask);	\			if (_old == _val)				\				break;					\			_val = _old;					\		}							\	_old;								\}) #if defined(CONFIG_SMP) && defined(TIF_POLLING_NRFLAG)/* * Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG, * this avoids any races wrt polling state changes and thereby avoids * spurious IPIs. */static bool set_nr_and_not_polling(struct task_struct *p){	struct thread_info *ti = task_thread_info(p);	return !(fetch_or(&ti->flags, _TIF_NEED_RESCHED) & _TIF_POLLING_NRFLAG);} /* * Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set. * * If this returns true, then the idle task promises to call * sched_ttwu_pending() and reschedule soon. */static bool set_nr_if_polling(struct task_struct *p){	struct thread_info *ti = task_thread_info(p);	typeof(ti->flags) old, val = READ_ONCE(ti->flags); 	for (;;) {		if (!(val & _TIF_POLLING_NRFLAG))			return false;		if (val & _TIF_NEED_RESCHED)			return true;		old = cmpxchg(&ti->flags, val, val | _TIF_NEED_RESCHED);		if (old == val)			break;		val = old;	}	return true;} #else

Lines 1–500 of 9555. Load more