/* *  kernel/sched/core.c * *  Kernel scheduler and related syscalls * *  Copyright (C) 1991-2002  Linus Torvalds * *  1996-12-23  Modified by Dave Grothe to fix bugs in semaphores and *		make semaphores SMP safe *  1998-11-19	Implemented schedule_timeout() and related stuff *		by Andrea Arcangeli *  2002-01-04	New ultra-scalable O(1) scheduler by Ingo Molnar: *		hybrid priority-list and round-robin design with *		an array-switch method of distributing timeslices *		and per-CPU runqueues.  Cleanups and useful suggestions *		by Davide Libenzi, preemptible kernel bits by Robert Love. *  2003-09-03	Interactivity tuning by Con Kolivas. *  2004-04-02	Scheduler domains code by Nick Piggin *  2007-04-15  Work begun on replacing all interactivity tuning with a *              fair scheduling design by Con Kolivas. *  2007-05-05  Load balancing (smp-nice) and other improvements *              by Peter Williams *  2007-05-06  Interactivity improvements to CFS by Mike Galbraith *  2007-07-01  Group scheduling enhancements by Srivatsa Vaddagiri *  2007-11-29  RT balancing improvements by Steven Rostedt, Gregory Haskins, *              Thomas Gleixner, Mike Kravetz */ #include <linux/mm.h>#include <linux/module.h>#include <linux/nmi.h>#include <linux/init.h>#include <linux/uaccess.h>#include <linux/highmem.h>#include <asm/mmu_context.h>#include <linux/interrupt.h>#include <linux/capability.h>#include <linux/completion.h>#include <linux/kernel_stat.h>#include <linux/debug_locks.h>#include <linux/perf_event.h>#include <linux/security.h>#include <linux/notifier.h>#include <linux/profile.h>#include <linux/freezer.h>#include <linux/vmalloc.h>#include <linux/blkdev.h>#include <linux/delay.h>#include <linux/pid_namespace.h>#include <linux/smp.h>#include <linux/threads.h>#include <linux/timer.h>#include <linux/rcupdate.h>#include <linux/cpu.h>#include <linux/cpuset.h>#include <linux/percpu.h>#include <linux/proc_fs.h>#include <linux/seq_file.h>#include <linux/sysctl.h>#include <linux/syscalls.h>#include <linux/times.h>#include <linux/tsacct_kern.h>#include <linux/kprobes.h>#include <linux/delayacct.h>#include <linux/unistd.h>#include <linux/pagemap.h>#include <linux/hrtimer.h>#include <linux/tick.h>#include <linux/debugfs.h>#include <linux/ctype.h>#include <linux/ftrace.h>#include <linux/slab.h>#include <linux/init_task.h>#include <linux/binfmts.h>#include <linux/context_tracking.h> #include <asm/switch_to.h>#include <asm/tlb.h>#include <asm/irq_regs.h>#include <asm/mutex.h>#ifdef CONFIG_PARAVIRT#include <asm/paravirt.h>#endif #include "sched.h"#include "../workqueue_internal.h"#include "../smpboot.h" #define CREATE_TRACE_POINTS#include <trace/events/sched.h> void start_bandwidth_timer(struct hrtimer *period_timer, ktime_t period){	unsigned long delta;	ktime_t soft, hard, now; 	for (;;) {		if (hrtimer_active(period_timer))			break; 		now = hrtimer_cb_get_time(period_timer);		hrtimer_forward(period_timer, now, period); 		soft = hrtimer_get_softexpires(period_timer);		hard = hrtimer_get_expires(period_timer);		delta = ktime_to_ns(ktime_sub(hard, soft));		__hrtimer_start_range_ns(period_timer, soft, delta,					 HRTIMER_MODE_ABS_PINNED, 0);	}} DEFINE_MUTEX(sched_domains_mutex);DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); static void update_rq_clock_task(struct rq *rq, s64 delta); void update_rq_clock(struct rq *rq){	s64 delta; 	if (rq->skip_clock_update > 0)		return; 	delta = sched_clock_cpu(cpu_of(rq)) - rq->clock;	rq->clock += delta;	update_rq_clock_task(rq, delta);} /* * Debugging: various feature bits */ #define SCHED_FEAT(name, enabled)	\	(1UL << __SCHED_FEAT_##name) * enabled | const_debug unsigned int sysctl_sched_features =#include "features.h"	0; #undef SCHED_FEAT #ifdef CONFIG_SCHED_DEBUG#define SCHED_FEAT(name, enabled)	\	#name , static const char * const sched_feat_names[] = {#include "features.h"}; #undef SCHED_FEAT static int sched_feat_show(struct seq_file *m, void *v){	int i; 	for (i = 0; i < __SCHED_FEAT_NR; i++) {		if (!(sysctl_sched_features & (1UL << i)))			seq_puts(m, "NO_");		seq_printf(m, "%s ", sched_feat_names[i]);	}	seq_puts(m, "\n"); 	return 0;} #ifdef HAVE_JUMP_LABEL #define jump_label_key__true  STATIC_KEY_INIT_TRUE#define jump_label_key__false STATIC_KEY_INIT_FALSE #define SCHED_FEAT(name, enabled)	\	jump_label_key__##enabled , struct static_key sched_feat_keys[__SCHED_FEAT_NR] = {#include "features.h"}; #undef SCHED_FEAT static void sched_feat_disable(int i){	if (static_key_enabled(&sched_feat_keys[i]))		static_key_slow_dec(&sched_feat_keys[i]);} static void sched_feat_enable(int i){	if (!static_key_enabled(&sched_feat_keys[i]))		static_key_slow_inc(&sched_feat_keys[i]);}#elsestatic void sched_feat_disable(int i) { };static void sched_feat_enable(int i) { };#endif /* HAVE_JUMP_LABEL */ static int sched_feat_set(char *cmp){	int i;	int neg = 0; 	if (strncmp(cmp, "NO_", 3) == 0) {		neg = 1;		cmp += 3;	} 	for (i = 0; i < __SCHED_FEAT_NR; i++) {		if (strcmp(cmp, sched_feat_names[i]) == 0) {			if (neg) {				sysctl_sched_features &= ~(1UL << i);				sched_feat_disable(i);			} else {				sysctl_sched_features |= (1UL << i);				sched_feat_enable(i);			}			break;		}	} 	return i;} static ssize_tsched_feat_write(struct file *filp, const char __user *ubuf,		size_t cnt, loff_t *ppos){	char buf[64];	char *cmp;	int i; 	if (cnt > 63)		cnt = 63; 	if (copy_from_user(&buf, ubuf, cnt))		return -EFAULT; 	buf[cnt] = 0;	cmp = strstrip(buf); 	i = sched_feat_set(cmp);	if (i == __SCHED_FEAT_NR)		return -EINVAL; 	*ppos += cnt; 	return cnt;} static int sched_feat_open(struct inode *inode, struct file *filp){	return single_open(filp, sched_feat_show, NULL);} static const struct file_operations sched_feat_fops = {	.open		= sched_feat_open,	.write		= sched_feat_write,	.read		= seq_read,	.llseek		= seq_lseek,	.release	= single_release,}; static __init int sched_init_debug(void){	debugfs_create_file("sched_features", 0644, NULL, NULL,			&sched_feat_fops); 	return 0;}late_initcall(sched_init_debug);#endif /* CONFIG_SCHED_DEBUG */ /* * Number of tasks to iterate in a single balance run. * Limited because this is done with IRQs disabled. */const_debug unsigned int sysctl_sched_nr_migrate = 32; /* * period over which we average the RT time consumption, measured * in ms. * * default: 1s */const_debug unsigned int sysctl_sched_time_avg = MSEC_PER_SEC; /* * period over which we measure -rt task cpu usage in us. * default: 1s */unsigned int sysctl_sched_rt_period = 1000000; __read_mostly int scheduler_running; /* * part of the period that we allow rt tasks to run in us. * default: 0.95s */int sysctl_sched_rt_runtime = 950000;   /* * __task_rq_lock - lock the rq @p resides on. */static inline struct rq *__task_rq_lock(struct task_struct *p)	__acquires(rq->lock){	struct rq *rq; 	lockdep_assert_held(&p->pi_lock); 	for (;;) {		rq = task_rq(p);		raw_spin_lock(&rq->lock);		if (likely(rq == task_rq(p)))			return rq;		raw_spin_unlock(&rq->lock);	}} /* * task_rq_lock - lock p->pi_lock and lock the rq @p resides on. */static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags)	__acquires(p->pi_lock)	__acquires(rq->lock){	struct rq *rq; 	for (;;) {		raw_spin_lock_irqsave(&p->pi_lock, *flags);		rq = task_rq(p);		raw_spin_lock(&rq->lock);		if (likely(rq == task_rq(p)))			return rq;		raw_spin_unlock(&rq->lock);		raw_spin_unlock_irqrestore(&p->pi_lock, *flags);	}} static void __task_rq_unlock(struct rq *rq)	__releases(rq->lock){	raw_spin_unlock(&rq->lock);} static inline voidtask_rq_unlock(struct rq *rq, struct task_struct *p, unsigned long *flags)	__releases(rq->lock)	__releases(p->pi_lock){	raw_spin_unlock(&rq->lock);	raw_spin_unlock_irqrestore(&p->pi_lock, *flags);} /* * this_rq_lock - lock this runqueue and disable interrupts. */static struct rq *this_rq_lock(void)	__acquires(rq->lock){	struct rq *rq; 	local_irq_disable();	rq = this_rq();	raw_spin_lock(&rq->lock); 	return rq;} #ifdef CONFIG_SCHED_HRTICK/* * Use HR-timers to deliver accurate preemption points. */ static void hrtick_clear(struct rq *rq){	if (hrtimer_active(&rq->hrtick_timer))		hrtimer_cancel(&rq->hrtick_timer);} /* * High-resolution timer tick. * Runs from hardirq context with interrupts disabled. */static enum hrtimer_restart hrtick(struct hrtimer *timer){	struct rq *rq = container_of(timer, struct rq, hrtick_timer); 	WARN_ON_ONCE(cpu_of(rq) != smp_processor_id()); 	raw_spin_lock(&rq->lock);	update_rq_clock(rq);	rq->curr->sched_class->task_tick(rq, rq->curr, 1);	raw_spin_unlock(&rq->lock); 	return HRTIMER_NORESTART;} #ifdef CONFIG_SMP static int __hrtick_restart(struct rq *rq){	struct hrtimer *timer = &rq->hrtick_timer;	ktime_t time = hrtimer_get_softexpires(timer); 	return __hrtimer_start_range_ns(timer, time, 0, HRTIMER_MODE_ABS_PINNED, 0);} /* * called from hardirq (IPI) context */static void __hrtick_start(void *arg){	struct rq *rq = arg; 	raw_spin_lock(&rq->lock);	__hrtick_restart(rq);	rq->hrtick_csd_pending = 0;	raw_spin_unlock(&rq->lock);} /* * Called to set the hrtick timer state. * * called with rq->lock held and irqs disabled */void hrtick_start(struct rq *rq, u64 delay){	struct hrtimer *timer = &rq->hrtick_timer;	ktime_t time = ktime_add_ns(timer->base->get_time(), delay); 	hrtimer_set_expires(timer, time); 	if (rq == this_rq()) {		__hrtick_restart(rq);	} else if (!rq->hrtick_csd_pending) {		__smp_call_function_single(cpu_of(rq), &rq->hrtick_csd, 0);		rq->hrtick_csd_pending = 1;	}} static inthotplug_hrtick(struct notifier_block *nfb, unsigned long action, void *hcpu){	int cpu = (int)(long)hcpu; 	switch (action) {	case CPU_UP_CANCELED:	case CPU_UP_CANCELED_FROZEN:	case CPU_DOWN_PREPARE:	case CPU_DOWN_PREPARE_FROZEN:	case CPU_DEAD:	case CPU_DEAD_FROZEN:		hrtick_clear(cpu_rq(cpu));		return NOTIFY_OK;	} 	return NOTIFY_DONE;} static __init void init_hrtick(void){	hotcpu_notifier(hotplug_hrtick, 0);}#else/* * Called to set the hrtick timer state. * * called with rq->lock held and irqs disabled */void hrtick_start(struct rq *rq, u64 delay){	__hrtimer_start_range_ns(&rq->hrtick_timer, ns_to_ktime(delay), 0,			HRTIMER_MODE_REL_PINNED, 0);} static inline void init_hrtick(void){}#endif /* CONFIG_SMP */ static void init_rq_hrtick(struct rq *rq){#ifdef CONFIG_SMP	rq->hrtick_csd_pending = 0; 	rq->hrtick_csd.flags = 0;	rq->hrtick_csd.func = __hrtick_start;	rq->hrtick_csd.info = rq;#endif 	hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);	rq->hrtick_timer.function = hrtick;}#else	/* CONFIG_SCHED_HRTICK */static inline void hrtick_clear(struct rq *rq){} static inline void init_rq_hrtick(struct rq *rq)

Lines 1–500 of 7361. Load more