// SPDX-License-Identifier: GPL-2.0/* *  Kernel timekeeping code and accessor functions. Based on code from *  timer.c, moved in commit 8524070b7982. */#include <linux/audit.h>#include <linux/clocksource.h>#include <linux/compiler.h>#include <linux/jiffies.h>#include <linux/kobject.h>#include <linux/module.h>#include <linux/nmi.h>#include <linux/pvclock_gtod.h>#include <linux/random.h>#include <linux/sched/clock.h>#include <linux/sched/loadavg.h>#include <linux/static_key.h>#include <linux/stop_machine.h>#include <linux/syscore_ops.h>#include <linux/tick.h>#include <linux/time.h>#include <linux/timex.h>#include <linux/timekeeper_internal.h> #include <vdso/auxclock.h> #include "tick-internal.h"#include "timekeeping_internal.h"#include "ntp_internal.h" #define TK_CLEAR_NTP		(1 << 0)#define TK_CLOCK_WAS_SET	(1 << 1) #define TK_UPDATE_ALL		(TK_CLEAR_NTP | TK_CLOCK_WAS_SET) enum timekeeping_adv_mode {	/* Update timekeeper when a tick has passed */	TK_ADV_TICK, 	/* Update timekeeper on a direct frequency change */	TK_ADV_FREQ}; /* * The most important data for readout fits into a single 64 byte * cache line. */struct tk_data {	seqcount_raw_spinlock_t	seq;	struct timekeeper	timekeeper;	struct timekeeper	shadow_timekeeper;	raw_spinlock_t		lock;} ____cacheline_aligned; static struct tk_data timekeeper_data[TIMEKEEPERS_MAX]; /* The core timekeeper */#define tk_core		(timekeeper_data[TIMEKEEPER_CORE]) #ifdef CONFIG_POSIX_AUX_CLOCKSstatic inline bool tk_get_aux_ts64(unsigned int tkid, struct timespec64 *ts){	return ktime_get_aux_ts64(CLOCK_AUX + tkid - TIMEKEEPER_AUX_FIRST, ts);} static inline bool tk_is_aux(const struct timekeeper *tk){	return tk->id >= TIMEKEEPER_AUX_FIRST && tk->id <= TIMEKEEPER_AUX_LAST;}static inline struct tk_data *aux_get_tk_data(clockid_t id);#elsestatic inline bool tk_get_aux_ts64(unsigned int tkid, struct timespec64 *ts){	return false;} static inline bool tk_is_aux(const struct timekeeper *tk){	return false;}static inline struct tk_data *aux_get_tk_data(clockid_t id){	return NULL;}#endif static inline void tk_update_aux_offs(struct timekeeper *tk, ktime_t offs){	tk->offs_aux = offs;	tk->monotonic_to_aux = ktime_to_timespec64(offs);} /* flag for if timekeeping is suspended */int __read_mostly timekeeping_suspended; /** * struct tk_fast - NMI safe timekeeper * @seq:	Sequence counter for protecting updates. The lowest bit *		is the index for the tk_read_base array * @base:	tk_read_base array. Access is indexed by the lowest bit of *		@seq. * * See @update_fast_timekeeper() below. */struct tk_fast {	seqcount_latch_t	seq;	struct tk_read_base	base[2];}; /* Suspend-time cycles value for halted fast timekeeper. */static u64 cycles_at_suspend; static u64 dummy_clock_read(struct clocksource *cs){	if (timekeeping_suspended)		return cycles_at_suspend;	return local_clock();} static struct clocksource dummy_clock = {	.read = dummy_clock_read,}; /* * Boot time initialization which allows local_clock() to be utilized * during early boot when clocksources are not available. local_clock() * returns nanoseconds already so no conversion is required, hence mult=1 * and shift=0. When the first proper clocksource is installed then * the fast time keepers are updated with the correct values. */#define FAST_TK_INIT						\	{							\		.clock		= &dummy_clock,			\		.mask		= CLOCKSOURCE_MASK(64),		\		.mult		= 1,				\		.shift		= 0,				\	} static struct tk_fast tk_fast_mono ____cacheline_aligned = {	.seq     = SEQCNT_LATCH_ZERO(tk_fast_mono.seq),	.base[0] = FAST_TK_INIT,	.base[1] = FAST_TK_INIT,}; static struct tk_fast tk_fast_raw  ____cacheline_aligned = {	.seq     = SEQCNT_LATCH_ZERO(tk_fast_raw.seq),	.base[0] = FAST_TK_INIT,	.base[1] = FAST_TK_INIT,}; #ifdef CONFIG_POSIX_AUX_CLOCKSstatic __init void tk_aux_setup(void);static void tk_aux_update_clocksource(void);static void tk_aux_advance(void);#elsestatic inline void tk_aux_setup(void) { }static inline void tk_aux_update_clocksource(void) { }static inline void tk_aux_advance(void) { }#endif unsigned long timekeeper_lock_irqsave(void){	unsigned long flags; 	raw_spin_lock_irqsave(&tk_core.lock, flags);	return flags;} void timekeeper_unlock_irqrestore(unsigned long flags){	raw_spin_unlock_irqrestore(&tk_core.lock, flags);} /* * Multigrain timestamps require tracking the latest fine-grained timestamp * that has been issued, and never returning a coarse-grained timestamp that is * earlier than that value. * * mg_floor represents the latest fine-grained time that has been handed out as * a file timestamp on the system. This is tracked as a monotonic ktime_t, and * converted to a realtime clock value on an as-needed basis. * * Maintaining mg_floor ensures the multigrain interfaces never issue a * timestamp earlier than one that has been previously issued. * * The exception to this rule is when there is a backward realtime clock jump. If * such an event occurs, a timestamp can appear to be earlier than a previous one. */static __cacheline_aligned_in_smp atomic64_t mg_floor; static inline void tk_normalize_xtime(struct timekeeper *tk){	while (tk->tkr_mono.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_mono.shift)) {		tk->tkr_mono.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_mono.shift;		tk->xtime_sec++;	}	while (tk->tkr_raw.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_raw.shift)) {		tk->tkr_raw.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_raw.shift;		tk->raw_sec++;	}} static inline struct timespec64 tk_xtime(const struct timekeeper *tk){	struct timespec64 ts; 	ts.tv_sec = tk->xtime_sec;	ts.tv_nsec = (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);	return ts;} static inline struct timespec64 tk_xtime_coarse(const struct timekeeper *tk){	struct timespec64 ts; 	ts.tv_sec = tk->xtime_sec;	ts.tv_nsec = tk->coarse_nsec;	return ts;} /* * Update the nanoseconds part for the coarse time keepers. They can't rely * on xtime_nsec because xtime_nsec could be adjusted by a small negative * amount when the multiplication factor of the clock is adjusted, which * could cause the coarse clocks to go slightly backwards. See * timekeeping_apply_adjustment(). Thus we keep a separate copy for the coarse * clockids which only is updated when the clock has been set or  we have * accumulated time. */static inline void tk_update_coarse_nsecs(struct timekeeper *tk){	tk->coarse_nsec = tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift;} static void tk_set_xtime(struct timekeeper *tk, const struct timespec64 *ts){	tk->xtime_sec = ts->tv_sec;	tk->tkr_mono.xtime_nsec = (u64)ts->tv_nsec << tk->tkr_mono.shift;	tk_update_coarse_nsecs(tk);} static void tk_xtime_add(struct timekeeper *tk, const struct timespec64 *ts){	tk->xtime_sec += ts->tv_sec;	tk->tkr_mono.xtime_nsec += (u64)ts->tv_nsec << tk->tkr_mono.shift;	tk_normalize_xtime(tk);	tk_update_coarse_nsecs(tk);} static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec64 wtm){	struct timespec64 tmp; 	/*	 * Verify consistency of: offset_real = -wall_to_monotonic	 * before modifying anything	 */	set_normalized_timespec64(&tmp, -tk->wall_to_monotonic.tv_sec,					-tk->wall_to_monotonic.tv_nsec);	WARN_ON_ONCE(tk->offs_real != timespec64_to_ktime(tmp));	tk->wall_to_monotonic = wtm;	set_normalized_timespec64(&tmp, -wtm.tv_sec, -wtm.tv_nsec);	/* Paired with READ_ONCE() in ktime_mono_to_any() */	WRITE_ONCE(tk->offs_real, timespec64_to_ktime(tmp));	WRITE_ONCE(tk->offs_tai, ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0)));} static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta){	/* Paired with READ_ONCE() in ktime_mono_to_any() */	WRITE_ONCE(tk->offs_boot, ktime_add(tk->offs_boot, delta));	/*	 * Timespec representation for VDSO update to avoid 64bit division	 * on every update.	 */	tk->monotonic_to_boot = ktime_to_timespec64(tk->offs_boot);} #ifdef CONFIG_ARCH_WANTS_CLOCKSOURCE_READ_INLINE#include <asm/clock_inlined.h> static DEFINE_STATIC_KEY_FALSE(clocksource_read_inlined); /* * tk_clock_read - atomic clocksource read() helper * * This helper is necessary to use in the read paths because, while the * seqcount ensures we don't return a bad value while structures are updated, * it doesn't protect from potential crashes. There is the possibility that * the tkr's clocksource may change between the read reference, and the * clock reference passed to the read function.  This can cause crashes if * the wrong clocksource is passed to the wrong read function. * This isn't necessary to use when holding the tk_core.lock or doing * a read of the fast-timekeeper tkrs (which is protected by its own locking * and update logic). */static __always_inline u64 tk_clock_read(const struct tk_read_base *tkr){	struct clocksource *clock = READ_ONCE(tkr->clock); 	if (static_branch_likely(&clocksource_read_inlined))		return arch_inlined_clocksource_read(clock); 	return clock->read(clock);} static inline void clocksource_disable_inline_read(void){	static_branch_disable(&clocksource_read_inlined);} static inline void clocksource_enable_inline_read(void){	static_branch_enable(&clocksource_read_inlined);}#elsestatic __always_inline u64 tk_clock_read(const struct tk_read_base *tkr){	struct clocksource *clock = READ_ONCE(tkr->clock); 	return clock->read(clock);} static inline void clocksource_disable_inline_read(void) { }static inline void clocksource_enable_inline_read(void) { }#endif /** * tk_setup_internals - Set up internals to use clocksource clock. * * @tk:		The target timekeeper to setup. * @clock:		Pointer to clocksource. * * Calculates a fixed cycle/nsec interval for a given clocksource/adjustment * pair and interval request. * * Unless you're the timekeeping code, you should not be using this! */static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock){	u64 interval;	struct clocksource *old_clock; 	++tk->cs_was_changed_seq;	old_clock = tk->tkr_mono.clock;	tk->tkr_mono.clock = clock;	tk->tkr_mono.mask = clock->mask;	tk->tkr_mono.cycle_last = tk_clock_read(&tk->tkr_mono); 	tk->tkr_raw.clock = clock;	tk->tkr_raw.mask = clock->mask;	tk->tkr_raw.cycle_last = tk->tkr_mono.cycle_last; 	/* Do the ns -> cycle conversion first, using original mult */	interval = (u64)NTP_INTERVAL_LENGTH << clock->shift;	interval += clock->mult / 2;	do_div(interval, clock->mult);	if (interval == 0)		interval = 1; 	tk->cycle_interval = interval; 	/* Go back from cycles -> shifted ns */	tk->xtime_interval = interval * clock->mult;	tk->raw_interval = interval * clock->mult; 	 /* if changing clocks, convert xtime_nsec shift units */	if (old_clock) {		int shift_change = clock->shift - old_clock->shift;		if (shift_change < 0) {			tk->tkr_mono.xtime_nsec >>= -shift_change;			tk->tkr_raw.xtime_nsec >>= -shift_change;		} else {			tk->tkr_mono.xtime_nsec <<= shift_change;			tk->tkr_raw.xtime_nsec <<= shift_change;		}	} 	tk->tkr_mono.shift = clock->shift;	tk->tkr_raw.shift = clock->shift; 	tk->ntp_error = 0;	tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift; 	/*	 * ntp_tick is the tick length that NTP disciplines (its ±500 PPM	 * scales only this part), in NTP-shifted ns: the real interval of	 * a whole number of counter cycles. Because cycle_interval is	 * rounded to an integer number of cycles, this ntp_tick differs	 * from the true intended 1/HZ tick length by up to half a cycle	 * period.	 */	tk->ntp_tick = (u64)tk->xtime_interval << tk->ntp_error_shift; 	/*	 * cs_tick_adj is the constant difference between the disciplined	 * ntp_tick above and the true 1/HZ tick, expressed per-second to	 * match the ntp_update_frequency() addends and handed to NTP via	 * ntp_clear() to be explicitly included in its tick_length.	 *	 * Worked example: HZ=1000, ACPI PM timer at 3.579545 MHz, which	 * has 3579.545 cycles in 1ms, rounded to cycle_interval = 3580.	 *	 * So ntp_tick is actually 1.000127ms, as that is the amount of	 * time that 3580 cycles will take at the nominal frequency. This	 * is the part that NTP disciplines, causing each 3580 counts to	 * advance the clock by up to NTP's ±500PPM of that amount.	 *	 * The "extra" 127ns/tick is what's stored in cs_tick_adj and	 * applied as a constant correction by ntp_update_frequency() so	 * that NTP *believes* it's disciplining a 1ms tick.	 */	tk->cs_tick_adj = (s64)tk->ntp_tick -			  ((s64)NTP_INTERVAL_LENGTH << NTP_SCALE_SHIFT);	tk->cs_tick_adj *= NTP_INTERVAL_FREQ; 	/*	 * The timekeeper keeps its own mult values for the currently	 * active clocksource. These value will be adjusted via NTP	 * to counteract clock drifting.	 */	tk->tkr_mono.mult = clock->mult;	tk->tkr_raw.mult = clock->mult;	tk->ntp_err_mult = 0;	tk->skip_second_overflow = 0;	tk->skew_delta = 0; 	tk->cs_id = clock->id; 	/* Coupled clockevent data */	if (IS_ENABLED(CONFIG_GENERIC_CLOCKEVENTS_COUPLED) &&	    clock->flags & CLOCK_SOURCE_HAS_COUPLED_CLOCK_EVENT) {		/*		 * Aim for an one hour maximum delta and use KHz to handle		 * clocksources with a frequency above 4GHz correctly as		 * the frequency argument of clocks_calc_mult_shift() is u32.		 */		clocks_calc_mult_shift(&tk->cs_ns_to_cyc_mult, &tk->cs_ns_to_cyc_shift,				       NSEC_PER_MSEC, clock->freq_khz, 3600 * 1000);		/*		 * Initialize the conversion limit as the previous clocksource		 * might have the same shift/mult pair so the quick check in		 * tk_update_ns_to_cyc() fails to update it after a clocksource		 * change leaving it effectivly zero.		 */		tk->cs_ns_to_cyc_maxns = div_u64(clock->mask, tk->cs_ns_to_cyc_mult);	}} /* Timekeeper helper functions. */static noinline u64 delta_to_ns_safe(const struct tk_read_base *tkr, u64 delta){	return mul_u64_u32_add_u64_shr(delta, tkr->mult, tkr->xtime_nsec, tkr->shift);} static __always_inline u64 timekeeping_cycles_to_ns(const struct tk_read_base *tkr, u64 cycles){	/* Calculate the delta since the last update_wall_time() */	u64 mask = tkr->mask, delta = (cycles - tkr->cycle_last) & mask; 	/*	 * This detects both negative motion and the case where the delta	 * overflows the multiplication with tkr->mult.	 */	if (unlikely(delta > tkr->clock->max_cycles)) {		/*		 * Handle clocksource inconsistency between CPUs to prevent		 * time from going backwards by checking for the MSB of the		 * mask being set in the delta.		 */		if (delta & ~(mask >> 1))			return tkr->xtime_nsec >> tkr->shift; 		return delta_to_ns_safe(tkr, delta);	} 	return ((delta * tkr->mult) + tkr->xtime_nsec) >> tkr->shift;} static __always_inline u64 timekeeping_get_ns(const struct tk_read_base *tkr){	return timekeeping_cycles_to_ns(tkr, tk_clock_read(tkr));} /** * update_fast_timekeeper - Update the fast and NMI safe monotonic timekeeper. * @tkr: Timekeeping readout base from which we take the update * @tkf: Pointer to NMI safe timekeeper * * We want to use this from any context including NMI and tracing / * instrumenting the timekeeping code itself. * * Employ the latch technique; see @write_seqcount_latch. * * So if a NMI hits the update of base[0] then it will use base[1] * which is still consistent. In the worst case this can result is a * slightly wrong timestamp (a few nanoseconds). See * @ktime_get_mono_fast_ns. */static void update_fast_timekeeper(const struct tk_read_base *tkr,

Lines 1–500 of 3412. Load more