/* *  kernel/sched/core.c * *  Kernel scheduler and related syscalls * *  Copyright (C) 1991-2002  Linus Torvalds * *  1996-12-23  Modified by Dave Grothe to fix bugs in semaphores and *		make semaphores SMP safe *  1998-11-19	Implemented schedule_timeout() and related stuff *		by Andrea Arcangeli *  2002-01-04	New ultra-scalable O(1) scheduler by Ingo Molnar: *		hybrid priority-list and round-robin design with *		an array-switch method of distributing timeslices *		and per-CPU runqueues.  Cleanups and useful suggestions *		by Davide Libenzi, preemptible kernel bits by Robert Love. *  2003-09-03	Interactivity tuning by Con Kolivas. *  2004-04-02	Scheduler domains code by Nick Piggin *  2007-04-15  Work begun on replacing all interactivity tuning with a *              fair scheduling design by Con Kolivas. *  2007-05-05  Load balancing (smp-nice) and other improvements *              by Peter Williams *  2007-05-06  Interactivity improvements to CFS by Mike Galbraith *  2007-07-01  Group scheduling enhancements by Srivatsa Vaddagiri *  2007-11-29  RT balancing improvements by Steven Rostedt, Gregory Haskins, *              Thomas Gleixner, Mike Kravetz */ #include <linux/kasan.h>#include <linux/mm.h>#include <linux/module.h>#include <linux/nmi.h>#include <linux/init.h>#include <linux/uaccess.h>#include <linux/highmem.h>#include <linux/mmu_context.h>#include <linux/interrupt.h>#include <linux/capability.h>#include <linux/completion.h>#include <linux/kernel_stat.h>#include <linux/debug_locks.h>#include <linux/perf_event.h>#include <linux/security.h>#include <linux/notifier.h>#include <linux/profile.h>#include <linux/freezer.h>#include <linux/vmalloc.h>#include <linux/blkdev.h>#include <linux/delay.h>#include <linux/pid_namespace.h>#include <linux/smp.h>#include <linux/threads.h>#include <linux/timer.h>#include <linux/rcupdate.h>#include <linux/cpu.h>#include <linux/cpuset.h>#include <linux/percpu.h>#include <linux/proc_fs.h>#include <linux/seq_file.h>#include <linux/sysctl.h>#include <linux/syscalls.h>#include <linux/times.h>#include <linux/tsacct_kern.h>#include <linux/kprobes.h>#include <linux/delayacct.h>#include <linux/unistd.h>#include <linux/pagemap.h>#include <linux/hrtimer.h>#include <linux/tick.h>#include <linux/ctype.h>#include <linux/ftrace.h>#include <linux/slab.h>#include <linux/init_task.h>#include <linux/context_tracking.h>#include <linux/compiler.h>#include <linux/frame.h>#include <linux/prefetch.h> #include <asm/switch_to.h>#include <asm/tlb.h>#include <asm/irq_regs.h>#include <asm/mutex.h>#ifdef CONFIG_PARAVIRT#include <asm/paravirt.h>#endif #include "sched.h"#include "../workqueue_internal.h"#include "../smpboot.h" #define CREATE_TRACE_POINTS#include <trace/events/sched.h> DEFINE_MUTEX(sched_domains_mutex);DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); static void update_rq_clock_task(struct rq *rq, s64 delta); void update_rq_clock(struct rq *rq){	s64 delta; 	lockdep_assert_held(&rq->lock); 	if (rq->clock_skip_update & RQCF_ACT_SKIP)		return; 	delta = sched_clock_cpu(cpu_of(rq)) - rq->clock;	if (delta < 0)		return;	rq->clock += delta;	update_rq_clock_task(rq, delta);} /* * Debugging: various feature bits */ #define SCHED_FEAT(name, enabled)	\	(1UL << __SCHED_FEAT_##name) * enabled | const_debug unsigned int sysctl_sched_features =#include "features.h"	0; #undef SCHED_FEAT /* * Number of tasks to iterate in a single balance run. * Limited because this is done with IRQs disabled. */const_debug unsigned int sysctl_sched_nr_migrate = 32; /* * period over which we average the RT time consumption, measured * in ms. * * default: 1s */const_debug unsigned int sysctl_sched_time_avg = MSEC_PER_SEC; /* * period over which we measure -rt task cpu usage in us. * default: 1s */unsigned int sysctl_sched_rt_period = 1000000; __read_mostly int scheduler_running; /* * part of the period that we allow rt tasks to run in us. * default: 0.95s */int sysctl_sched_rt_runtime = 950000; /* cpus with isolated domains */cpumask_var_t cpu_isolated_map; /* * this_rq_lock - lock this runqueue and disable interrupts. */static struct rq *this_rq_lock(void)	__acquires(rq->lock){	struct rq *rq; 	local_irq_disable();	rq = this_rq();	raw_spin_lock(&rq->lock); 	return rq;} /* * __task_rq_lock - lock the rq @p resides on. */struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf)	__acquires(rq->lock){	struct rq *rq; 	lockdep_assert_held(&p->pi_lock); 	for (;;) {		rq = task_rq(p);		raw_spin_lock(&rq->lock);		if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) {			rf->cookie = lockdep_pin_lock(&rq->lock);			return rq;		}		raw_spin_unlock(&rq->lock); 		while (unlikely(task_on_rq_migrating(p)))			cpu_relax();	}} /* * task_rq_lock - lock p->pi_lock and lock the rq @p resides on. */struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf)	__acquires(p->pi_lock)	__acquires(rq->lock){	struct rq *rq; 	for (;;) {		raw_spin_lock_irqsave(&p->pi_lock, rf->flags);		rq = task_rq(p);		raw_spin_lock(&rq->lock);		/*		 *	move_queued_task()		task_rq_lock()		 *		 *	ACQUIRE (rq->lock)		 *	[S] ->on_rq = MIGRATING		[L] rq = task_rq()		 *	WMB (__set_task_cpu())		ACQUIRE (rq->lock);		 *	[S] ->cpu = new_cpu		[L] task_rq()		 *					[L] ->on_rq		 *	RELEASE (rq->lock)		 *		 * If we observe the old cpu in task_rq_lock, the acquire of		 * the old rq->lock will fully serialize against the stores.		 *		 * If we observe the new cpu in task_rq_lock, the acquire will		 * pair with the WMB to ensure we must then also see migrating.		 */		if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) {			rf->cookie = lockdep_pin_lock(&rq->lock);			return rq;		}		raw_spin_unlock(&rq->lock);		raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags); 		while (unlikely(task_on_rq_migrating(p)))			cpu_relax();	}} #ifdef CONFIG_SCHED_HRTICK/* * Use HR-timers to deliver accurate preemption points. */ static void hrtick_clear(struct rq *rq){	if (hrtimer_active(&rq->hrtick_timer))		hrtimer_cancel(&rq->hrtick_timer);} /* * High-resolution timer tick. * Runs from hardirq context with interrupts disabled. */static enum hrtimer_restart hrtick(struct hrtimer *timer){	struct rq *rq = container_of(timer, struct rq, hrtick_timer); 	WARN_ON_ONCE(cpu_of(rq) != smp_processor_id()); 	raw_spin_lock(&rq->lock);	update_rq_clock(rq);	rq->curr->sched_class->task_tick(rq, rq->curr, 1);	raw_spin_unlock(&rq->lock); 	return HRTIMER_NORESTART;} #ifdef CONFIG_SMP static void __hrtick_restart(struct rq *rq){	struct hrtimer *timer = &rq->hrtick_timer; 	hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED);} /* * called from hardirq (IPI) context */static void __hrtick_start(void *arg){	struct rq *rq = arg; 	raw_spin_lock(&rq->lock);	__hrtick_restart(rq);	rq->hrtick_csd_pending = 0;	raw_spin_unlock(&rq->lock);} /* * Called to set the hrtick timer state. * * called with rq->lock held and irqs disabled */void hrtick_start(struct rq *rq, u64 delay){	struct hrtimer *timer = &rq->hrtick_timer;	ktime_t time;	s64 delta; 	/*	 * Don't schedule slices shorter than 10000ns, that just	 * doesn't make sense and can cause timer DoS.	 */	delta = max_t(s64, delay, 10000LL);	time = ktime_add_ns(timer->base->get_time(), delta); 	hrtimer_set_expires(timer, time); 	if (rq == this_rq()) {		__hrtick_restart(rq);	} else if (!rq->hrtick_csd_pending) {		smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd);		rq->hrtick_csd_pending = 1;	}} #else/* * Called to set the hrtick timer state. * * called with rq->lock held and irqs disabled */void hrtick_start(struct rq *rq, u64 delay){	/*	 * Don't schedule slices shorter than 10000ns, that just	 * doesn't make sense. Rely on vruntime for fairness.	 */	delay = max_t(u64, delay, 10000LL);	hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay),		      HRTIMER_MODE_REL_PINNED);}#endif /* CONFIG_SMP */ static void init_rq_hrtick(struct rq *rq){#ifdef CONFIG_SMP	rq->hrtick_csd_pending = 0; 	rq->hrtick_csd.flags = 0;	rq->hrtick_csd.func = __hrtick_start;	rq->hrtick_csd.info = rq;#endif 	hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);	rq->hrtick_timer.function = hrtick;}#else	/* CONFIG_SCHED_HRTICK */static inline void hrtick_clear(struct rq *rq){} static inline void init_rq_hrtick(struct rq *rq){}#endif	/* CONFIG_SCHED_HRTICK */ /* * cmpxchg based fetch_or, macro so it works for different integer types */#define fetch_or(ptr, mask)						\	({								\		typeof(ptr) _ptr = (ptr);				\		typeof(mask) _mask = (mask);				\		typeof(*_ptr) _old, _val = *_ptr;			\									\		for (;;) {						\			_old = cmpxchg(_ptr, _val, _val | _mask);	\			if (_old == _val)				\				break;					\			_val = _old;					\		}							\	_old;								\}) #if defined(CONFIG_SMP) && defined(TIF_POLLING_NRFLAG)/* * Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG, * this avoids any races wrt polling state changes and thereby avoids * spurious IPIs. */static bool set_nr_and_not_polling(struct task_struct *p){	struct thread_info *ti = task_thread_info(p);	return !(fetch_or(&ti->flags, _TIF_NEED_RESCHED) & _TIF_POLLING_NRFLAG);} /* * Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set. * * If this returns true, then the idle task promises to call * sched_ttwu_pending() and reschedule soon. */static bool set_nr_if_polling(struct task_struct *p){	struct thread_info *ti = task_thread_info(p);	typeof(ti->flags) old, val = READ_ONCE(ti->flags); 	for (;;) {		if (!(val & _TIF_POLLING_NRFLAG))			return false;		if (val & _TIF_NEED_RESCHED)			return true;		old = cmpxchg(&ti->flags, val, val | _TIF_NEED_RESCHED);		if (old == val)			break;		val = old;	}	return true;} #elsestatic bool set_nr_and_not_polling(struct task_struct *p){	set_tsk_need_resched(p);	return true;} #ifdef CONFIG_SMPstatic bool set_nr_if_polling(struct task_struct *p){	return false;}#endif#endif void wake_q_add(struct wake_q_head *head, struct task_struct *task){	struct wake_q_node *node = &task->wake_q; 	/*	 * Atomically grab the task, if ->wake_q is !nil already it means	 * its already queued (either by us or someone else) and will get the	 * wakeup due to that.	 *	 * This cmpxchg() implies a full barrier, which pairs with the write	 * barrier implied by the wakeup in wake_up_q().	 */	if (cmpxchg(&node->next, NULL, WAKE_Q_TAIL))		return; 	get_task_struct(task); 	/*	 * The head is context local, there can be no concurrency.	 */	*head->lastp = node;	head->lastp = &node->next;} void wake_up_q(struct wake_q_head *head){	struct wake_q_node *node = head->first; 	while (node != WAKE_Q_TAIL) {		struct task_struct *task; 		task = container_of(node, struct task_struct, wake_q);		BUG_ON(!task);		/* task can safely be re-inserted now */		node = node->next;		task->wake_q.next = NULL; 		/*		 * wake_up_process() implies a wmb() to pair with the queueing		 * in wake_q_add() so as not to miss wakeups.		 */		wake_up_process(task);		put_task_struct(task);	}} /* * resched_curr - mark rq's current task 'to be rescheduled now'. * * On UP this means the setting of the need_resched flag, on SMP it * might also involve a cross-CPU call to trigger the scheduler on * the target CPU. */void resched_curr(struct rq *rq){	struct task_struct *curr = rq->curr;	int cpu; 	lockdep_assert_held(&rq->lock); 	if (test_tsk_need_resched(curr))		return; 	cpu = cpu_of(rq); 	if (cpu == smp_processor_id()) {		set_tsk_need_resched(curr);		set_preempt_need_resched();		return;	} 	if (set_nr_and_not_polling(curr))		smp_send_reschedule(cpu);

Lines 1–500 of 8845. Load more