// SPDX-License-Identifier: GPL-2.0/* *  Kernel timekeeping code and accessor functions. Based on code from *  timer.c, moved in commit 8524070b7982. */#include <linux/timekeeper_internal.h>#include <linux/module.h>#include <linux/interrupt.h>#include <linux/kobject.h>#include <linux/percpu.h>#include <linux/init.h>#include <linux/mm.h>#include <linux/nmi.h>#include <linux/sched.h>#include <linux/sched/loadavg.h>#include <linux/sched/clock.h>#include <linux/syscore_ops.h>#include <linux/clocksource.h>#include <linux/jiffies.h>#include <linux/time.h>#include <linux/timex.h>#include <linux/tick.h>#include <linux/stop_machine.h>#include <linux/pvclock_gtod.h>#include <linux/compiler.h>#include <linux/audit.h>#include <linux/random.h> #include <vdso/auxclock.h> #include "tick-internal.h"#include "ntp_internal.h"#include "timekeeping_internal.h" #define TK_CLEAR_NTP		(1 << 0)#define TK_CLOCK_WAS_SET	(1 << 1) #define TK_UPDATE_ALL		(TK_CLEAR_NTP | TK_CLOCK_WAS_SET) enum timekeeping_adv_mode {	/* Update timekeeper when a tick has passed */	TK_ADV_TICK, 	/* Update timekeeper on a direct frequency change */	TK_ADV_FREQ}; /* * The most important data for readout fits into a single 64 byte * cache line. */struct tk_data {	seqcount_raw_spinlock_t	seq;	struct timekeeper	timekeeper;	struct timekeeper	shadow_timekeeper;	raw_spinlock_t		lock;} ____cacheline_aligned; static struct tk_data timekeeper_data[TIMEKEEPERS_MAX]; /* The core timekeeper */#define tk_core		(timekeeper_data[TIMEKEEPER_CORE]) #ifdef CONFIG_POSIX_AUX_CLOCKSstatic inline bool tk_get_aux_ts64(unsigned int tkid, struct timespec64 *ts){	return ktime_get_aux_ts64(CLOCK_AUX + tkid - TIMEKEEPER_AUX_FIRST, ts);} static inline bool tk_is_aux(const struct timekeeper *tk){	return tk->id >= TIMEKEEPER_AUX_FIRST && tk->id <= TIMEKEEPER_AUX_LAST;}#elsestatic inline bool tk_get_aux_ts64(unsigned int tkid, struct timespec64 *ts){	return false;} static inline bool tk_is_aux(const struct timekeeper *tk){	return false;}#endif static inline void tk_update_aux_offs(struct timekeeper *tk, ktime_t offs){	tk->offs_aux = offs;	tk->monotonic_to_aux = ktime_to_timespec64(offs);} /* flag for if timekeeping is suspended */int __read_mostly timekeeping_suspended; /** * struct tk_fast - NMI safe timekeeper * @seq:	Sequence counter for protecting updates. The lowest bit *		is the index for the tk_read_base array * @base:	tk_read_base array. Access is indexed by the lowest bit of *		@seq. * * See @update_fast_timekeeper() below. */struct tk_fast {	seqcount_latch_t	seq;	struct tk_read_base	base[2];}; /* Suspend-time cycles value for halted fast timekeeper. */static u64 cycles_at_suspend; static u64 dummy_clock_read(struct clocksource *cs){	if (timekeeping_suspended)		return cycles_at_suspend;	return local_clock();} static struct clocksource dummy_clock = {	.read = dummy_clock_read,}; /* * Boot time initialization which allows local_clock() to be utilized * during early boot when clocksources are not available. local_clock() * returns nanoseconds already so no conversion is required, hence mult=1 * and shift=0. When the first proper clocksource is installed then * the fast time keepers are updated with the correct values. */#define FAST_TK_INIT						\	{							\		.clock		= &dummy_clock,			\		.mask		= CLOCKSOURCE_MASK(64),		\		.mult		= 1,				\		.shift		= 0,				\	} static struct tk_fast tk_fast_mono ____cacheline_aligned = {	.seq     = SEQCNT_LATCH_ZERO(tk_fast_mono.seq),	.base[0] = FAST_TK_INIT,	.base[1] = FAST_TK_INIT,}; static struct tk_fast tk_fast_raw  ____cacheline_aligned = {	.seq     = SEQCNT_LATCH_ZERO(tk_fast_raw.seq),	.base[0] = FAST_TK_INIT,	.base[1] = FAST_TK_INIT,}; #ifdef CONFIG_POSIX_AUX_CLOCKSstatic __init void tk_aux_setup(void);static void tk_aux_update_clocksource(void);static void tk_aux_advance(void);#elsestatic inline void tk_aux_setup(void) { }static inline void tk_aux_update_clocksource(void) { }static inline void tk_aux_advance(void) { }#endif unsigned long timekeeper_lock_irqsave(void){	unsigned long flags; 	raw_spin_lock_irqsave(&tk_core.lock, flags);	return flags;} void timekeeper_unlock_irqrestore(unsigned long flags){	raw_spin_unlock_irqrestore(&tk_core.lock, flags);} /* * Multigrain timestamps require tracking the latest fine-grained timestamp * that has been issued, and never returning a coarse-grained timestamp that is * earlier than that value. * * mg_floor represents the latest fine-grained time that has been handed out as * a file timestamp on the system. This is tracked as a monotonic ktime_t, and * converted to a realtime clock value on an as-needed basis. * * Maintaining mg_floor ensures the multigrain interfaces never issue a * timestamp earlier than one that has been previously issued. * * The exception to this rule is when there is a backward realtime clock jump. If * such an event occurs, a timestamp can appear to be earlier than a previous one. */static __cacheline_aligned_in_smp atomic64_t mg_floor; static inline void tk_normalize_xtime(struct timekeeper *tk){	while (tk->tkr_mono.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_mono.shift)) {		tk->tkr_mono.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_mono.shift;		tk->xtime_sec++;	}	while (tk->tkr_raw.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_raw.shift)) {		tk->tkr_raw.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_raw.shift;		tk->raw_sec++;	}} static inline struct timespec64 tk_xtime(const struct timekeeper *tk){	struct timespec64 ts; 	ts.tv_sec = tk->xtime_sec;	ts.tv_nsec = (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);	return ts;} static inline struct timespec64 tk_xtime_coarse(const struct timekeeper *tk){	struct timespec64 ts; 	ts.tv_sec = tk->xtime_sec;	ts.tv_nsec = tk->coarse_nsec;	return ts;} /* * Update the nanoseconds part for the coarse time keepers. They can't rely * on xtime_nsec because xtime_nsec could be adjusted by a small negative * amount when the multiplication factor of the clock is adjusted, which * could cause the coarse clocks to go slightly backwards. See * timekeeping_apply_adjustment(). Thus we keep a separate copy for the coarse * clockids which only is updated when the clock has been set or  we have * accumulated time. */static inline void tk_update_coarse_nsecs(struct timekeeper *tk){	tk->coarse_nsec = tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift;} static void tk_set_xtime(struct timekeeper *tk, const struct timespec64 *ts){	tk->xtime_sec = ts->tv_sec;	tk->tkr_mono.xtime_nsec = (u64)ts->tv_nsec << tk->tkr_mono.shift;	tk_update_coarse_nsecs(tk);} static void tk_xtime_add(struct timekeeper *tk, const struct timespec64 *ts){	tk->xtime_sec += ts->tv_sec;	tk->tkr_mono.xtime_nsec += (u64)ts->tv_nsec << tk->tkr_mono.shift;	tk_normalize_xtime(tk);	tk_update_coarse_nsecs(tk);} static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec64 wtm){	struct timespec64 tmp; 	/*	 * Verify consistency of: offset_real = -wall_to_monotonic	 * before modifying anything	 */	set_normalized_timespec64(&tmp, -tk->wall_to_monotonic.tv_sec,					-tk->wall_to_monotonic.tv_nsec);	WARN_ON_ONCE(tk->offs_real != timespec64_to_ktime(tmp));	tk->wall_to_monotonic = wtm;	set_normalized_timespec64(&tmp, -wtm.tv_sec, -wtm.tv_nsec);	/* Paired with READ_ONCE() in ktime_mono_to_any() */	WRITE_ONCE(tk->offs_real, timespec64_to_ktime(tmp));	WRITE_ONCE(tk->offs_tai, ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0)));} static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta){	/* Paired with READ_ONCE() in ktime_mono_to_any() */	WRITE_ONCE(tk->offs_boot, ktime_add(tk->offs_boot, delta));	/*	 * Timespec representation for VDSO update to avoid 64bit division	 * on every update.	 */	tk->monotonic_to_boot = ktime_to_timespec64(tk->offs_boot);} /* * tk_clock_read - atomic clocksource read() helper * * This helper is necessary to use in the read paths because, while the * seqcount ensures we don't return a bad value while structures are updated, * it doesn't protect from potential crashes. There is the possibility that * the tkr's clocksource may change between the read reference, and the * clock reference passed to the read function.  This can cause crashes if * the wrong clocksource is passed to the wrong read function. * This isn't necessary to use when holding the tk_core.lock or doing * a read of the fast-timekeeper tkrs (which is protected by its own locking * and update logic). */static inline u64 tk_clock_read(const struct tk_read_base *tkr){	struct clocksource *clock = READ_ONCE(tkr->clock); 	return clock->read(clock);} /** * tk_setup_internals - Set up internals to use clocksource clock. * * @tk:		The target timekeeper to setup. * @clock:		Pointer to clocksource. * * Calculates a fixed cycle/nsec interval for a given clocksource/adjustment * pair and interval request. * * Unless you're the timekeeping code, you should not be using this! */static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock){	u64 interval;	u64 tmp, ntpinterval;	struct clocksource *old_clock; 	++tk->cs_was_changed_seq;	old_clock = tk->tkr_mono.clock;	tk->tkr_mono.clock = clock;	tk->tkr_mono.mask = clock->mask;	tk->tkr_mono.cycle_last = tk_clock_read(&tk->tkr_mono); 	tk->tkr_raw.clock = clock;	tk->tkr_raw.mask = clock->mask;	tk->tkr_raw.cycle_last = tk->tkr_mono.cycle_last; 	/* Do the ns -> cycle conversion first, using original mult */	tmp = NTP_INTERVAL_LENGTH;	tmp <<= clock->shift;	ntpinterval = tmp;	tmp += clock->mult/2;	do_div(tmp, clock->mult);	if (tmp == 0)		tmp = 1; 	interval = (u64) tmp;	tk->cycle_interval = interval; 	/* Go back from cycles -> shifted ns */	tk->xtime_interval = interval * clock->mult;	tk->xtime_remainder = ntpinterval - tk->xtime_interval;	tk->raw_interval = interval * clock->mult; 	 /* if changing clocks, convert xtime_nsec shift units */	if (old_clock) {		int shift_change = clock->shift - old_clock->shift;		if (shift_change < 0) {			tk->tkr_mono.xtime_nsec >>= -shift_change;			tk->tkr_raw.xtime_nsec >>= -shift_change;		} else {			tk->tkr_mono.xtime_nsec <<= shift_change;			tk->tkr_raw.xtime_nsec <<= shift_change;		}	} 	tk->tkr_mono.shift = clock->shift;	tk->tkr_raw.shift = clock->shift; 	tk->ntp_error = 0;	tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;	tk->ntp_tick = ntpinterval << tk->ntp_error_shift; 	/*	 * The timekeeper keeps its own mult values for the currently	 * active clocksource. These value will be adjusted via NTP	 * to counteract clock drifting.	 */	tk->tkr_mono.mult = clock->mult;	tk->tkr_raw.mult = clock->mult;	tk->ntp_err_mult = 0;	tk->skip_second_overflow = 0;} /* Timekeeper helper functions. */static noinline u64 delta_to_ns_safe(const struct tk_read_base *tkr, u64 delta){	return mul_u64_u32_add_u64_shr(delta, tkr->mult, tkr->xtime_nsec, tkr->shift);} static inline u64 timekeeping_cycles_to_ns(const struct tk_read_base *tkr, u64 cycles){	/* Calculate the delta since the last update_wall_time() */	u64 mask = tkr->mask, delta = (cycles - tkr->cycle_last) & mask; 	/*	 * This detects both negative motion and the case where the delta	 * overflows the multiplication with tkr->mult.	 */	if (unlikely(delta > tkr->clock->max_cycles)) {		/*		 * Handle clocksource inconsistency between CPUs to prevent		 * time from going backwards by checking for the MSB of the		 * mask being set in the delta.		 */		if (delta & ~(mask >> 1))			return tkr->xtime_nsec >> tkr->shift; 		return delta_to_ns_safe(tkr, delta);	} 	return ((delta * tkr->mult) + tkr->xtime_nsec) >> tkr->shift;} static __always_inline u64 timekeeping_get_ns(const struct tk_read_base *tkr){	return timekeeping_cycles_to_ns(tkr, tk_clock_read(tkr));} /** * update_fast_timekeeper - Update the fast and NMI safe monotonic timekeeper. * @tkr: Timekeeping readout base from which we take the update * @tkf: Pointer to NMI safe timekeeper * * We want to use this from any context including NMI and tracing / * instrumenting the timekeeping code itself. * * Employ the latch technique; see @write_seqcount_latch. * * So if a NMI hits the update of base[0] then it will use base[1] * which is still consistent. In the worst case this can result is a * slightly wrong timestamp (a few nanoseconds). See * @ktime_get_mono_fast_ns. */static void update_fast_timekeeper(const struct tk_read_base *tkr,				   struct tk_fast *tkf){	struct tk_read_base *base = tkf->base; 	/* Force readers off to base[1] */	write_seqcount_latch_begin(&tkf->seq); 	/* Update base[0] */	memcpy(base, tkr, sizeof(*base)); 	/* Force readers back to base[0] */	write_seqcount_latch(&tkf->seq); 	/* Update base[1] */	memcpy(base + 1, base, sizeof(*base)); 	write_seqcount_latch_end(&tkf->seq);} static __always_inline u64 __ktime_get_fast_ns(struct tk_fast *tkf){	struct tk_read_base *tkr;	unsigned int seq;	u64 now; 	do {		seq = read_seqcount_latch(&tkf->seq);		tkr = tkf->base + (seq & 0x01);		now = ktime_to_ns(tkr->base);		now += timekeeping_get_ns(tkr);	} while (read_seqcount_latch_retry(&tkf->seq, seq)); 	return now;} /** * ktime_get_mono_fast_ns - Fast NMI safe access to clock monotonic * * This timestamp is not guaranteed to be monotonic across an update. * The timestamp is calculated by: * *	now = base_mono + clock_delta * slope * * So if the update lowers the slope, readers who are forced to the * not yet updated second array are still using the old steeper slope. * * tmono * ^ * |    o  n * |   o n * |  u * | o * |o * |12345678---> reader order * * o = old slope * u = update * n = new slope * * So reader 6 will observe time going backwards versus reader 5. * * While other CPUs are likely to be able to observe that, the only way * for a CPU local observation is when an NMI hits in the middle of * the update. Timestamps taken from that NMI context might be ahead * of the following timestamps. Callers need to be aware of that and * deal with it. */u64 notrace ktime_get_mono_fast_ns(void){	return __ktime_get_fast_ns(&tk_fast_mono);}EXPORT_SYMBOL_GPL(ktime_get_mono_fast_ns); /** * ktime_get_raw_fast_ns - Fast NMI safe access to clock monotonic raw * * Contrary to ktime_get_mono_fast_ns() this is always correct because the * conversion factor is not affected by NTP/PTP correction.

Lines 1–500 of 3099. Load more