/* *  linux/kernel/time/timekeeping.c * *  Kernel timekeeping code and accessor functions * *  This code was moved from linux/kernel/timer.c. *  Please see that file for copyright and history logs. * */ #include <linux/timekeeper_internal.h>#include <linux/module.h>#include <linux/interrupt.h>#include <linux/percpu.h>#include <linux/init.h>#include <linux/mm.h>#include <linux/nmi.h>#include <linux/sched.h>#include <linux/sched/loadavg.h>#include <linux/syscore_ops.h>#include <linux/clocksource.h>#include <linux/jiffies.h>#include <linux/time.h>#include <linux/tick.h>#include <linux/stop_machine.h>#include <linux/pvclock_gtod.h>#include <linux/compiler.h> #include "tick-internal.h"#include "ntp_internal.h"#include "timekeeping_internal.h" #define TK_CLEAR_NTP		(1 << 0)#define TK_MIRROR		(1 << 1)#define TK_CLOCK_WAS_SET	(1 << 2) /* * The most important data for readout fits into a single 64 byte * cache line. */static struct {	seqcount_t		seq;	struct timekeeper	timekeeper;} tk_core ____cacheline_aligned; static DEFINE_RAW_SPINLOCK(timekeeper_lock);static struct timekeeper shadow_timekeeper; /** * struct tk_fast - NMI safe timekeeper * @seq:	Sequence counter for protecting updates. The lowest bit *		is the index for the tk_read_base array * @base:	tk_read_base array. Access is indexed by the lowest bit of *		@seq. * * See @update_fast_timekeeper() below. */struct tk_fast {	seqcount_t		seq;	struct tk_read_base	base[2];}; static struct tk_fast tk_fast_mono ____cacheline_aligned;static struct tk_fast tk_fast_raw  ____cacheline_aligned; /* flag for if timekeeping is suspended */int __read_mostly timekeeping_suspended; static inline void tk_normalize_xtime(struct timekeeper *tk){	while (tk->tkr_mono.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_mono.shift)) {		tk->tkr_mono.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_mono.shift;		tk->xtime_sec++;	}} static inline struct timespec64 tk_xtime(struct timekeeper *tk){	struct timespec64 ts; 	ts.tv_sec = tk->xtime_sec;	ts.tv_nsec = (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);	return ts;} static void tk_set_xtime(struct timekeeper *tk, const struct timespec64 *ts){	tk->xtime_sec = ts->tv_sec;	tk->tkr_mono.xtime_nsec = (u64)ts->tv_nsec << tk->tkr_mono.shift;} static void tk_xtime_add(struct timekeeper *tk, const struct timespec64 *ts){	tk->xtime_sec += ts->tv_sec;	tk->tkr_mono.xtime_nsec += (u64)ts->tv_nsec << tk->tkr_mono.shift;	tk_normalize_xtime(tk);} static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec64 wtm){	struct timespec64 tmp; 	/*	 * Verify consistency of: offset_real = -wall_to_monotonic	 * before modifying anything	 */	set_normalized_timespec64(&tmp, -tk->wall_to_monotonic.tv_sec,					-tk->wall_to_monotonic.tv_nsec);	WARN_ON_ONCE(tk->offs_real != timespec64_to_ktime(tmp));	tk->wall_to_monotonic = wtm;	set_normalized_timespec64(&tmp, -wtm.tv_sec, -wtm.tv_nsec);	tk->offs_real = timespec64_to_ktime(tmp);	tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0));} static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta){	tk->offs_boot = ktime_add(tk->offs_boot, delta);} #ifdef CONFIG_DEBUG_TIMEKEEPING#define WARNING_FREQ (HZ*300) /* 5 minute rate-limiting */ static void timekeeping_check_update(struct timekeeper *tk, u64 offset){ 	u64 max_cycles = tk->tkr_mono.clock->max_cycles;	const char *name = tk->tkr_mono.clock->name; 	if (offset > max_cycles) {		printk_deferred("WARNING: timekeeping: Cycle offset (%lld) is larger than allowed by the '%s' clock's max_cycles value (%lld): time overflow danger\n",				offset, name, max_cycles);		printk_deferred("         timekeeping: Your kernel is sick, but tries to cope by capping time updates\n");	} else {		if (offset > (max_cycles >> 1)) {			printk_deferred("INFO: timekeeping: Cycle offset (%lld) is larger than the '%s' clock's 50%% safety margin (%lld)\n",					offset, name, max_cycles >> 1);			printk_deferred("      timekeeping: Your kernel is still fine, but is feeling a bit nervous\n");		}	} 	if (tk->underflow_seen) {		if (jiffies - tk->last_warning > WARNING_FREQ) {			printk_deferred("WARNING: Underflow in clocksource '%s' observed, time update ignored.\n", name);			printk_deferred("         Please report this, consider using a different clocksource, if possible.\n");			printk_deferred("         Your kernel is probably still fine.\n");			tk->last_warning = jiffies;		}		tk->underflow_seen = 0;	} 	if (tk->overflow_seen) {		if (jiffies - tk->last_warning > WARNING_FREQ) {			printk_deferred("WARNING: Overflow in clocksource '%s' observed, time update capped.\n", name);			printk_deferred("         Please report this, consider using a different clocksource, if possible.\n");			printk_deferred("         Your kernel is probably still fine.\n");			tk->last_warning = jiffies;		}		tk->overflow_seen = 0;	}} static inline u64 timekeeping_get_delta(struct tk_read_base *tkr){	struct timekeeper *tk = &tk_core.timekeeper;	u64 now, last, mask, max, delta;	unsigned int seq; 	/*	 * Since we're called holding a seqlock, the data may shift	 * under us while we're doing the calculation. This can cause	 * false positives, since we'd note a problem but throw the	 * results away. So nest another seqlock here to atomically	 * grab the points we are checking with.	 */	do {		seq = read_seqcount_begin(&tk_core.seq);		now = tkr->read(tkr->clock);		last = tkr->cycle_last;		mask = tkr->mask;		max = tkr->clock->max_cycles;	} while (read_seqcount_retry(&tk_core.seq, seq)); 	delta = clocksource_delta(now, last, mask); 	/*	 * Try to catch underflows by checking if we are seeing small	 * mask-relative negative values.	 */	if (unlikely((~delta & mask) < (mask >> 3))) {		tk->underflow_seen = 1;		delta = 0;	} 	/* Cap delta value to the max_cycles values to avoid mult overflows */	if (unlikely(delta > max)) {		tk->overflow_seen = 1;		delta = tkr->clock->max_cycles;	} 	return delta;}#elsestatic inline void timekeeping_check_update(struct timekeeper *tk, u64 offset){}static inline u64 timekeeping_get_delta(struct tk_read_base *tkr){	u64 cycle_now, delta; 	/* read clocksource */	cycle_now = tkr->read(tkr->clock); 	/* calculate the delta since the last update_wall_time */	delta = clocksource_delta(cycle_now, tkr->cycle_last, tkr->mask); 	return delta;}#endif /** * tk_setup_internals - Set up internals to use clocksource clock. * * @tk:		The target timekeeper to setup. * @clock:		Pointer to clocksource. * * Calculates a fixed cycle/nsec interval for a given clocksource/adjustment * pair and interval request. * * Unless you're the timekeeping code, you should not be using this! */static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock){	u64 interval;	u64 tmp, ntpinterval;	struct clocksource *old_clock; 	++tk->cs_was_changed_seq;	old_clock = tk->tkr_mono.clock;	tk->tkr_mono.clock = clock;	tk->tkr_mono.read = clock->read;	tk->tkr_mono.mask = clock->mask;	tk->tkr_mono.cycle_last = tk->tkr_mono.read(clock); 	tk->tkr_raw.clock = clock;	tk->tkr_raw.read = clock->read;	tk->tkr_raw.mask = clock->mask;	tk->tkr_raw.cycle_last = tk->tkr_mono.cycle_last; 	/* Do the ns -> cycle conversion first, using original mult */	tmp = NTP_INTERVAL_LENGTH;	tmp <<= clock->shift;	ntpinterval = tmp;	tmp += clock->mult/2;	do_div(tmp, clock->mult);	if (tmp == 0)		tmp = 1; 	interval = (u64) tmp;	tk->cycle_interval = interval; 	/* Go back from cycles -> shifted ns */	tk->xtime_interval = interval * clock->mult;	tk->xtime_remainder = ntpinterval - tk->xtime_interval;	tk->raw_interval = (interval * clock->mult) >> clock->shift; 	 /* if changing clocks, convert xtime_nsec shift units */	if (old_clock) {		int shift_change = clock->shift - old_clock->shift;		if (shift_change < 0)			tk->tkr_mono.xtime_nsec >>= -shift_change;		else			tk->tkr_mono.xtime_nsec <<= shift_change;	}	tk->tkr_raw.xtime_nsec = 0; 	tk->tkr_mono.shift = clock->shift;	tk->tkr_raw.shift = clock->shift; 	tk->ntp_error = 0;	tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;	tk->ntp_tick = ntpinterval << tk->ntp_error_shift; 	/*	 * The timekeeper keeps its own mult values for the currently	 * active clocksource. These value will be adjusted via NTP	 * to counteract clock drifting.	 */	tk->tkr_mono.mult = clock->mult;	tk->tkr_raw.mult = clock->mult;	tk->ntp_err_mult = 0;} /* Timekeeper helper functions. */ #ifdef CONFIG_ARCH_USES_GETTIMEOFFSETstatic u32 default_arch_gettimeoffset(void) { return 0; }u32 (*arch_gettimeoffset)(void) = default_arch_gettimeoffset;#elsestatic inline u32 arch_gettimeoffset(void) { return 0; }#endif static inline u64 timekeeping_delta_to_ns(struct tk_read_base *tkr, u64 delta){	u64 nsec; 	nsec = delta * tkr->mult + tkr->xtime_nsec;	nsec >>= tkr->shift; 	/* If arch requires, add in get_arch_timeoffset() */	return nsec + arch_gettimeoffset();} static inline u64 timekeeping_get_ns(struct tk_read_base *tkr){	u64 delta; 	delta = timekeeping_get_delta(tkr);	return timekeeping_delta_to_ns(tkr, delta);} static inline u64 timekeeping_cycles_to_ns(struct tk_read_base *tkr, u64 cycles){	u64 delta; 	/* calculate the delta since the last update_wall_time */	delta = clocksource_delta(cycles, tkr->cycle_last, tkr->mask);	return timekeeping_delta_to_ns(tkr, delta);} /** * update_fast_timekeeper - Update the fast and NMI safe monotonic timekeeper. * @tkr: Timekeeping readout base from which we take the update * * We want to use this from any context including NMI and tracing / * instrumenting the timekeeping code itself. * * Employ the latch technique; see @raw_write_seqcount_latch. * * So if a NMI hits the update of base[0] then it will use base[1] * which is still consistent. In the worst case this can result is a * slightly wrong timestamp (a few nanoseconds). See * @ktime_get_mono_fast_ns. */static void update_fast_timekeeper(struct tk_read_base *tkr, struct tk_fast *tkf){	struct tk_read_base *base = tkf->base; 	/* Force readers off to base[1] */	raw_write_seqcount_latch(&tkf->seq); 	/* Update base[0] */	memcpy(base, tkr, sizeof(*base)); 	/* Force readers back to base[0] */	raw_write_seqcount_latch(&tkf->seq); 	/* Update base[1] */	memcpy(base + 1, base, sizeof(*base));} /** * ktime_get_mono_fast_ns - Fast NMI safe access to clock monotonic * * This timestamp is not guaranteed to be monotonic across an update. * The timestamp is calculated by: * *	now = base_mono + clock_delta * slope * * So if the update lowers the slope, readers who are forced to the * not yet updated second array are still using the old steeper slope. * * tmono * ^ * |    o  n * |   o n * |  u * | o * |o * |12345678---> reader order * * o = old slope * u = update * n = new slope * * So reader 6 will observe time going backwards versus reader 5. * * While other CPUs are likely to be able observe that, the only way * for a CPU local observation is when an NMI hits in the middle of * the update. Timestamps taken from that NMI context might be ahead * of the following timestamps. Callers need to be aware of that and * deal with it. */static __always_inline u64 __ktime_get_fast_ns(struct tk_fast *tkf){	struct tk_read_base *tkr;	unsigned int seq;	u64 now; 	do {		seq = raw_read_seqcount_latch(&tkf->seq);		tkr = tkf->base + (seq & 0x01);		now = ktime_to_ns(tkr->base); 		now += timekeeping_delta_to_ns(tkr,				clocksource_delta(					tkr->read(tkr->clock),					tkr->cycle_last,					tkr->mask));	} while (read_seqcount_retry(&tkf->seq, seq)); 	return now;} u64 ktime_get_mono_fast_ns(void){	return __ktime_get_fast_ns(&tk_fast_mono);}EXPORT_SYMBOL_GPL(ktime_get_mono_fast_ns); u64 ktime_get_raw_fast_ns(void){	return __ktime_get_fast_ns(&tk_fast_raw);}EXPORT_SYMBOL_GPL(ktime_get_raw_fast_ns); /** * ktime_get_boot_fast_ns - NMI safe and fast access to boot clock. * * To keep it NMI safe since we're accessing from tracing, we're not using a * separate timekeeper with updates to monotonic clock and boot offset * protected with seqlocks. This has the following minor side effects: * * (1) Its possible that a timestamp be taken after the boot offset is updated * but before the timekeeper is updated. If this happens, the new boot offset * is added to the old timekeeping making the clock appear to update slightly * earlier: *    CPU 0                                        CPU 1 *    timekeeping_inject_sleeptime64() *    __timekeeping_inject_sleeptime(tk, delta); *                                                 timestamp(); *    timekeeping_update(tk, TK_CLEAR_NTP...); * * (2) On 32-bit systems, the 64-bit boot offset (tk->offs_boot) may be * partially updated.  Since the tk->offs_boot update is a rare event, this * should be a rare occurrence which postprocessing should be able to handle. */u64 notrace ktime_get_boot_fast_ns(void){	struct timekeeper *tk = &tk_core.timekeeper; 	return (ktime_get_mono_fast_ns() + ktime_to_ns(tk->offs_boot));}EXPORT_SYMBOL_GPL(ktime_get_boot_fast_ns); /* Suspend-time cycles value for halted fast timekeeper. */static u64 cycles_at_suspend; static u64 dummy_clock_read(struct clocksource *cs){	return cycles_at_suspend;} /** * halt_fast_timekeeper - Prevent fast timekeeper from accessing clocksource. * @tk: Timekeeper to snapshot. * * It generally is unsafe to access the clocksource after timekeeping has been * suspended, so take a snapshot of the readout base of @tk and use it as the * fast timekeeper's readout base while suspended.  It will return the same * number of cycles every time until timekeeping is resumed at which time the * proper readout base for the fast timekeeper will be restored automatically. */static void halt_fast_timekeeper(struct timekeeper *tk){	static struct tk_read_base tkr_dummy;	struct tk_read_base *tkr = &tk->tkr_mono; 	memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy));	cycles_at_suspend = tkr->read(tkr->clock);	tkr_dummy.read = dummy_clock_read;	update_fast_timekeeper(&tkr_dummy, &tk_fast_mono); 	tkr = &tk->tkr_raw;	memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy));	tkr_dummy.read = dummy_clock_read;	update_fast_timekeeper(&tkr_dummy, &tk_fast_raw);} #ifdef CONFIG_GENERIC_TIME_VSYSCALL_OLD static inline void update_vsyscall(struct timekeeper *tk){	struct timespec xt, wm; 	xt = timespec64_to_timespec(tk_xtime(tk));	wm = timespec64_to_timespec(tk->wall_to_monotonic);	update_vsyscall_old(&xt, &wm, tk->tkr_mono.clock, tk->tkr_mono.mult,			    tk->tkr_mono.cycle_last);}

Lines 1–500 of 2309. Load more