/* *  linux/kernel/timer.c * *  Kernel internal timers * *  Copyright (C) 1991, 1992  Linus Torvalds * *  1997-01-28  Modified by Finn Arne Gangstad to make timers scale better. * *  1997-09-10  Updated NTP code according to technical memorandum Jan '96 *              "A Kernel Model for Precision Timekeeping" by Dave Mills *  1998-12-24  Fixed a xtime SMP race (we need the xtime_lock rw spinlock to *              serialize accesses to xtime/lost_ticks). *                              Copyright (C) 1998  Andrea Arcangeli *  1999-03-10  Improved NTP compatibility by Ulrich Windl *  2002-05-31	Move sys_sysinfo here and make its locking sane, Robert Love *  2000-10-05  Implemented scalable SMP per-CPU timer handling. *                              Copyright (C) 2000, 2001, 2002  Ingo Molnar *              Designed by David S. Miller, Alexey Kuznetsov and Ingo Molnar */ #include <linux/kernel_stat.h>#include <linux/export.h>#include <linux/interrupt.h>#include <linux/percpu.h>#include <linux/init.h>#include <linux/mm.h>#include <linux/swap.h>#include <linux/pid_namespace.h>#include <linux/notifier.h>#include <linux/thread_info.h>#include <linux/time.h>#include <linux/jiffies.h>#include <linux/posix-timers.h>#include <linux/cpu.h>#include <linux/syscalls.h>#include <linux/delay.h>#include <linux/tick.h>#include <linux/kallsyms.h>#include <linux/irq_work.h>#include <linux/sched.h>#include <linux/sched/sysctl.h>#include <linux/slab.h>#include <linux/compat.h> #include <asm/uaccess.h>#include <asm/unistd.h>#include <asm/div64.h>#include <asm/timex.h>#include <asm/io.h> #include "tick-internal.h" #define CREATE_TRACE_POINTS#include <trace/events/timer.h> __visible u64 jiffies_64 __cacheline_aligned_in_smp = INITIAL_JIFFIES; EXPORT_SYMBOL(jiffies_64); /* * per-CPU timer vector definitions: */#define TVN_BITS (CONFIG_BASE_SMALL ? 4 : 6)#define TVR_BITS (CONFIG_BASE_SMALL ? 6 : 8)#define TVN_SIZE (1 << TVN_BITS)#define TVR_SIZE (1 << TVR_BITS)#define TVN_MASK (TVN_SIZE - 1)#define TVR_MASK (TVR_SIZE - 1)#define MAX_TVAL ((unsigned long)((1ULL << (TVR_BITS + 4*TVN_BITS)) - 1)) struct tvec {	struct hlist_head vec[TVN_SIZE];}; struct tvec_root {	struct hlist_head vec[TVR_SIZE];}; struct tvec_base {	spinlock_t lock;	struct timer_list *running_timer;	unsigned long timer_jiffies;	unsigned long next_timer;	unsigned long active_timers;	unsigned long all_timers;	int cpu;	bool migration_enabled;	bool nohz_active;	struct tvec_root tv1;	struct tvec tv2;	struct tvec tv3;	struct tvec tv4;	struct tvec tv5;} ____cacheline_aligned;  static DEFINE_PER_CPU(struct tvec_base, tvec_bases); #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)unsigned int sysctl_timer_migration = 1; void timers_update_migration(bool update_nohz){	bool on = sysctl_timer_migration && tick_nohz_active;	unsigned int cpu; 	/* Avoid the loop, if nothing to update */	if (this_cpu_read(tvec_bases.migration_enabled) == on)		return; 	for_each_possible_cpu(cpu) {		per_cpu(tvec_bases.migration_enabled, cpu) = on;		per_cpu(hrtimer_bases.migration_enabled, cpu) = on;		if (!update_nohz)			continue;		per_cpu(tvec_bases.nohz_active, cpu) = true;		per_cpu(hrtimer_bases.nohz_active, cpu) = true;	}} int timer_migration_handler(struct ctl_table *table, int write,			    void __user *buffer, size_t *lenp,			    loff_t *ppos){	static DEFINE_MUTEX(mutex);	int ret; 	mutex_lock(&mutex);	ret = proc_dointvec(table, write, buffer, lenp, ppos);	if (!ret && write)		timers_update_migration(false);	mutex_unlock(&mutex);	return ret;} static inline struct tvec_base *get_target_base(struct tvec_base *base,						int pinned){	if (pinned || !base->migration_enabled)		return this_cpu_ptr(&tvec_bases);	return per_cpu_ptr(&tvec_bases, get_nohz_timer_target());}#elsestatic inline struct tvec_base *get_target_base(struct tvec_base *base,						int pinned){	return this_cpu_ptr(&tvec_bases);}#endif static unsigned long round_jiffies_common(unsigned long j, int cpu,		bool force_up){	int rem;	unsigned long original = j; 	/*	 * We don't want all cpus firing their timers at once hitting the	 * same lock or cachelines, so we skew each extra cpu with an extra	 * 3 jiffies. This 3 jiffies came originally from the mm/ code which	 * already did this.	 * The skew is done by adding 3*cpunr, then round, then subtract this	 * extra offset again.	 */	j += cpu * 3; 	rem = j % HZ; 	/*	 * If the target jiffie is just after a whole second (which can happen	 * due to delays of the timer irq, long irq off times etc etc) then	 * we should round down to the whole second, not up. Use 1/4th second	 * as cutoff for this rounding as an extreme upper bound for this.	 * But never round down if @force_up is set.	 */	if (rem < HZ/4 && !force_up) /* round down */		j = j - rem;	else /* round up */		j = j - rem + HZ; 	/* now that we have rounded, subtract the extra skew again */	j -= cpu * 3; 	/*	 * Make sure j is still in the future. Otherwise return the	 * unmodified value.	 */	return time_is_after_jiffies(j) ? j : original;} /** * __round_jiffies - function to round jiffies to a full second * @j: the time in (absolute) jiffies that should be rounded * @cpu: the processor number on which the timeout will happen * * __round_jiffies() rounds an absolute time in the future (in jiffies) * up or down to (approximately) full seconds. This is useful for timers * for which the exact time they fire does not matter too much, as long as * they fire approximately every X seconds. * * By rounding these timers to whole seconds, all such timers will fire * at the same time, rather than at various times spread out. The goal * of this is to have the CPU wake up less, which saves power. * * The exact rounding is skewed for each processor to avoid all * processors firing at the exact same time, which could lead * to lock contention or spurious cache line bouncing. * * The return value is the rounded version of the @j parameter. */unsigned long __round_jiffies(unsigned long j, int cpu){	return round_jiffies_common(j, cpu, false);}EXPORT_SYMBOL_GPL(__round_jiffies); /** * __round_jiffies_relative - function to round jiffies to a full second * @j: the time in (relative) jiffies that should be rounded * @cpu: the processor number on which the timeout will happen * * __round_jiffies_relative() rounds a time delta  in the future (in jiffies) * up or down to (approximately) full seconds. This is useful for timers * for which the exact time they fire does not matter too much, as long as * they fire approximately every X seconds. * * By rounding these timers to whole seconds, all such timers will fire * at the same time, rather than at various times spread out. The goal * of this is to have the CPU wake up less, which saves power. * * The exact rounding is skewed for each processor to avoid all * processors firing at the exact same time, which could lead * to lock contention or spurious cache line bouncing. * * The return value is the rounded version of the @j parameter. */unsigned long __round_jiffies_relative(unsigned long j, int cpu){	unsigned long j0 = jiffies; 	/* Use j0 because jiffies might change while we run */	return round_jiffies_common(j + j0, cpu, false) - j0;}EXPORT_SYMBOL_GPL(__round_jiffies_relative); /** * round_jiffies - function to round jiffies to a full second * @j: the time in (absolute) jiffies that should be rounded * * round_jiffies() rounds an absolute time in the future (in jiffies) * up or down to (approximately) full seconds. This is useful for timers * for which the exact time they fire does not matter too much, as long as * they fire approximately every X seconds. * * By rounding these timers to whole seconds, all such timers will fire * at the same time, rather than at various times spread out. The goal * of this is to have the CPU wake up less, which saves power. * * The return value is the rounded version of the @j parameter. */unsigned long round_jiffies(unsigned long j){	return round_jiffies_common(j, raw_smp_processor_id(), false);}EXPORT_SYMBOL_GPL(round_jiffies); /** * round_jiffies_relative - function to round jiffies to a full second * @j: the time in (relative) jiffies that should be rounded * * round_jiffies_relative() rounds a time delta  in the future (in jiffies) * up or down to (approximately) full seconds. This is useful for timers * for which the exact time they fire does not matter too much, as long as * they fire approximately every X seconds. * * By rounding these timers to whole seconds, all such timers will fire * at the same time, rather than at various times spread out. The goal * of this is to have the CPU wake up less, which saves power. * * The return value is the rounded version of the @j parameter. */unsigned long round_jiffies_relative(unsigned long j){	return __round_jiffies_relative(j, raw_smp_processor_id());}EXPORT_SYMBOL_GPL(round_jiffies_relative); /** * __round_jiffies_up - function to round jiffies up to a full second * @j: the time in (absolute) jiffies that should be rounded * @cpu: the processor number on which the timeout will happen * * This is the same as __round_jiffies() except that it will never * round down.  This is useful for timeouts for which the exact time * of firing does not matter too much, as long as they don't fire too * early. */unsigned long __round_jiffies_up(unsigned long j, int cpu){	return round_jiffies_common(j, cpu, true);}EXPORT_SYMBOL_GPL(__round_jiffies_up); /** * __round_jiffies_up_relative - function to round jiffies up to a full second * @j: the time in (relative) jiffies that should be rounded * @cpu: the processor number on which the timeout will happen * * This is the same as __round_jiffies_relative() except that it will never * round down.  This is useful for timeouts for which the exact time * of firing does not matter too much, as long as they don't fire too * early. */unsigned long __round_jiffies_up_relative(unsigned long j, int cpu){	unsigned long j0 = jiffies; 	/* Use j0 because jiffies might change while we run */	return round_jiffies_common(j + j0, cpu, true) - j0;}EXPORT_SYMBOL_GPL(__round_jiffies_up_relative); /** * round_jiffies_up - function to round jiffies up to a full second * @j: the time in (absolute) jiffies that should be rounded * * This is the same as round_jiffies() except that it will never * round down.  This is useful for timeouts for which the exact time * of firing does not matter too much, as long as they don't fire too * early. */unsigned long round_jiffies_up(unsigned long j){	return round_jiffies_common(j, raw_smp_processor_id(), true);}EXPORT_SYMBOL_GPL(round_jiffies_up); /** * round_jiffies_up_relative - function to round jiffies up to a full second * @j: the time in (relative) jiffies that should be rounded * * This is the same as round_jiffies_relative() except that it will never * round down.  This is useful for timeouts for which the exact time * of firing does not matter too much, as long as they don't fire too * early. */unsigned long round_jiffies_up_relative(unsigned long j){	return __round_jiffies_up_relative(j, raw_smp_processor_id());}EXPORT_SYMBOL_GPL(round_jiffies_up_relative); /** * set_timer_slack - set the allowed slack for a timer * @timer: the timer to be modified * @slack_hz: the amount of time (in jiffies) allowed for rounding * * Set the amount of time, in jiffies, that a certain timer has * in terms of slack. By setting this value, the timer subsystem * will schedule the actual timer somewhere between * the time mod_timer() asks for, and that time plus the slack. * * By setting the slack to -1, a percentage of the delay is used * instead. */void set_timer_slack(struct timer_list *timer, int slack_hz){	timer->slack = slack_hz;}EXPORT_SYMBOL_GPL(set_timer_slack); static void__internal_add_timer(struct tvec_base *base, struct timer_list *timer){	unsigned long expires = timer->expires;	unsigned long idx = expires - base->timer_jiffies;	struct hlist_head *vec; 	if (idx < TVR_SIZE) {		int i = expires & TVR_MASK;		vec = base->tv1.vec + i;	} else if (idx < 1 << (TVR_BITS + TVN_BITS)) {		int i = (expires >> TVR_BITS) & TVN_MASK;		vec = base->tv2.vec + i;	} else if (idx < 1 << (TVR_BITS + 2 * TVN_BITS)) {		int i = (expires >> (TVR_BITS + TVN_BITS)) & TVN_MASK;		vec = base->tv3.vec + i;	} else if (idx < 1 << (TVR_BITS + 3 * TVN_BITS)) {		int i = (expires >> (TVR_BITS + 2 * TVN_BITS)) & TVN_MASK;		vec = base->tv4.vec + i;	} else if ((signed long) idx < 0) {		/*		 * Can happen if you add a timer with expires == jiffies,		 * or you set a timer to go off in the past		 */		vec = base->tv1.vec + (base->timer_jiffies & TVR_MASK);	} else {		int i;		/* If the timeout is larger than MAX_TVAL (on 64-bit		 * architectures or with CONFIG_BASE_SMALL=1) then we		 * use the maximum timeout.		 */		if (idx > MAX_TVAL) {			idx = MAX_TVAL;			expires = idx + base->timer_jiffies;		}		i = (expires >> (TVR_BITS + 3 * TVN_BITS)) & TVN_MASK;		vec = base->tv5.vec + i;	} 	hlist_add_head(&timer->entry, vec);} static void internal_add_timer(struct tvec_base *base, struct timer_list *timer){	/* Advance base->jiffies, if the base is empty */	if (!base->all_timers++)		base->timer_jiffies = jiffies; 	__internal_add_timer(base, timer);	/*	 * Update base->active_timers and base->next_timer	 */	if (!(timer->flags & TIMER_DEFERRABLE)) {		if (!base->active_timers++ ||		    time_before(timer->expires, base->next_timer))			base->next_timer = timer->expires;	} 	/*	 * Check whether the other CPU is in dynticks mode and needs	 * to be triggered to reevaluate the timer wheel.	 * We are protected against the other CPU fiddling	 * with the timer by holding the timer base lock. This also	 * makes sure that a CPU on the way to stop its tick can not	 * evaluate the timer wheel.	 *	 * Spare the IPI for deferrable timers on idle targets though.	 * The next busy ticks will take care of it. Except full dynticks	 * require special care against races with idle_cpu(), lets deal	 * with that later.	 */	if (base->nohz_active) {		if (!(timer->flags & TIMER_DEFERRABLE) ||		    tick_nohz_full_cpu(base->cpu))			wake_up_nohz_cpu(base->cpu);	}} #ifdef CONFIG_TIMER_STATSvoid __timer_stats_timer_set_start_info(struct timer_list *timer, void *addr){	if (timer->start_site)		return; 	timer->start_site = addr;	memcpy(timer->start_comm, current->comm, TASK_COMM_LEN);	timer->start_pid = current->pid;} static void timer_stats_account_timer(struct timer_list *timer){	void *site; 	/*	 * start_site can be concurrently reset by	 * timer_stats_timer_clear_start_info()	 */	site = READ_ONCE(timer->start_site);	if (likely(!site))		return; 	timer_stats_update_stats(timer, timer->start_pid, site,				 timer->function, timer->start_comm,				 timer->flags);} #elsestatic void timer_stats_account_timer(struct timer_list *timer) {}#endif #ifdef CONFIG_DEBUG_OBJECTS_TIMERS static struct debug_obj_descr timer_debug_descr; static void *timer_debug_hint(void *addr){	return ((struct timer_list *) addr)->function;} /* * fixup_init is called when: * - an active object is initialized */static int timer_fixup_init(void *addr, enum debug_obj_state state){	struct timer_list *timer = addr; 	switch (state) {

Lines 1–500 of 1729. Load more