/* * linux/kernel/time/timekeeping.c * * Kernel timekeeping code and accessor functions * * This code was moved from linux/kernel/timer.c. * Please see that file for copyright and history logs. * */ #include <linux/timekeeper_internal.h>#include <linux/module.h>#include <linux/interrupt.h>#include <linux/percpu.h>#include <linux/init.h>#include <linux/mm.h>#include <linux/sched.h>#include <linux/syscore_ops.h>#include <linux/clocksource.h>#include <linux/jiffies.h>#include <linux/time.h>#include <linux/tick.h>#include <linux/stop_machine.h>#include <linux/pvclock_gtod.h>#include <linux/compiler.h> #include "tick-internal.h"#include "ntp_internal.h"#include "timekeeping_internal.h" #define TK_CLEAR_NTP (1 << 0)#define TK_MIRROR (1 << 1)#define TK_CLOCK_WAS_SET (1 << 2) /* * The most important data for readout fits into a single 64 byte * cache line. */static struct { seqcount_t seq; struct timekeeper timekeeper;} tk_core ____cacheline_aligned; static DEFINE_RAW_SPINLOCK(timekeeper_lock);static struct timekeeper shadow_timekeeper; /** * struct tk_fast - NMI safe timekeeper * @seq: Sequence counter for protecting updates. The lowest bit * is the index for the tk_read_base array * @base: tk_read_base array. Access is indexed by the lowest bit of * @seq. * * See @update_fast_timekeeper() below. */struct tk_fast { seqcount_t seq; struct tk_read_base base[2];}; static struct tk_fast tk_fast_mono ____cacheline_aligned; /* flag for if timekeeping is suspended */int __read_mostly timekeeping_suspended; /* Flag for if there is a persistent clock on this platform */bool __read_mostly persistent_clock_exist = false; static inline void tk_normalize_xtime(struct timekeeper *tk){ while (tk->tkr.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr.shift)) { tk->tkr.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr.shift; tk->xtime_sec++; }} static inline struct timespec64 tk_xtime(struct timekeeper *tk){ struct timespec64 ts; ts.tv_sec = tk->xtime_sec; ts.tv_nsec = (long)(tk->tkr.xtime_nsec >> tk->tkr.shift); return ts;} static void tk_set_xtime(struct timekeeper *tk, const struct timespec64 *ts){ tk->xtime_sec = ts->tv_sec; tk->tkr.xtime_nsec = (u64)ts->tv_nsec << tk->tkr.shift;} static void tk_xtime_add(struct timekeeper *tk, const struct timespec64 *ts){ tk->xtime_sec += ts->tv_sec; tk->tkr.xtime_nsec += (u64)ts->tv_nsec << tk->tkr.shift; tk_normalize_xtime(tk);} static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec64 wtm){ struct timespec64 tmp; /* * Verify consistency of: offset_real = -wall_to_monotonic * before modifying anything */ set_normalized_timespec64(&tmp, -tk->wall_to_monotonic.tv_sec, -tk->wall_to_monotonic.tv_nsec); WARN_ON_ONCE(tk->offs_real.tv64 != timespec64_to_ktime(tmp).tv64); tk->wall_to_monotonic = wtm; set_normalized_timespec64(&tmp, -wtm.tv_sec, -wtm.tv_nsec); tk->offs_real = timespec64_to_ktime(tmp); tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0));} static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta){ tk->offs_boot = ktime_add(tk->offs_boot, delta);} /** * tk_setup_internals - Set up internals to use clocksource clock. * * @tk: The target timekeeper to setup. * @clock: Pointer to clocksource. * * Calculates a fixed cycle/nsec interval for a given clocksource/adjustment * pair and interval request. * * Unless you're the timekeeping code, you should not be using this! */static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock){ cycle_t interval; u64 tmp, ntpinterval; struct clocksource *old_clock; old_clock = tk->tkr.clock; tk->tkr.clock = clock; tk->tkr.read = clock->read; tk->tkr.mask = clock->mask; tk->tkr.cycle_last = tk->tkr.read(clock); /* Do the ns -> cycle conversion first, using original mult */ tmp = NTP_INTERVAL_LENGTH; tmp <<= clock->shift; ntpinterval = tmp; tmp += clock->mult/2; do_div(tmp, clock->mult); if (tmp == 0) tmp = 1; interval = (cycle_t) tmp; tk->cycle_interval = interval; /* Go back from cycles -> shifted ns */ tk->xtime_interval = (u64) interval * clock->mult; tk->xtime_remainder = ntpinterval - tk->xtime_interval; tk->raw_interval = ((u64) interval * clock->mult) >> clock->shift; /* if changing clocks, convert xtime_nsec shift units */ if (old_clock) { int shift_change = clock->shift - old_clock->shift; if (shift_change < 0) tk->tkr.xtime_nsec >>= -shift_change; else tk->tkr.xtime_nsec <<= shift_change; } tk->tkr.shift = clock->shift; tk->ntp_error = 0; tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift; tk->ntp_tick = ntpinterval << tk->ntp_error_shift; /* * The timekeeper keeps its own mult values for the currently * active clocksource. These value will be adjusted via NTP * to counteract clock drifting. */ tk->tkr.mult = clock->mult; tk->ntp_err_mult = 0;} /* Timekeeper helper functions. */ #ifdef CONFIG_ARCH_USES_GETTIMEOFFSETstatic u32 default_arch_gettimeoffset(void) { return 0; }u32 (*arch_gettimeoffset)(void) = default_arch_gettimeoffset;#elsestatic inline u32 arch_gettimeoffset(void) { return 0; }#endif static inline s64 timekeeping_get_ns(struct tk_read_base *tkr){ cycle_t cycle_now, delta; s64 nsec; /* read clocksource: */ cycle_now = tkr->read(tkr->clock); /* calculate the delta since the last update_wall_time: */ delta = clocksource_delta(cycle_now, tkr->cycle_last, tkr->mask); nsec = delta * tkr->mult + tkr->xtime_nsec; nsec >>= tkr->shift; /* If arch requires, add in get_arch_timeoffset() */ return nsec + arch_gettimeoffset();} static inline s64 timekeeping_get_ns_raw(struct timekeeper *tk){ struct clocksource *clock = tk->tkr.clock; cycle_t cycle_now, delta; s64 nsec; /* read clocksource: */ cycle_now = tk->tkr.read(clock); /* calculate the delta since the last update_wall_time: */ delta = clocksource_delta(cycle_now, tk->tkr.cycle_last, tk->tkr.mask); /* convert delta to nanoseconds. */ nsec = clocksource_cyc2ns(delta, clock->mult, clock->shift); /* If arch requires, add in get_arch_timeoffset() */ return nsec + arch_gettimeoffset();} /** * update_fast_timekeeper - Update the fast and NMI safe monotonic timekeeper. * @tk: The timekeeper from which we take the update * @tkf: The fast timekeeper to update * @tbase: The time base for the fast timekeeper (mono/raw) * * We want to use this from any context including NMI and tracing / * instrumenting the timekeeping code itself. * * So we handle this differently than the other timekeeping accessor * functions which retry when the sequence count has changed. The * update side does: * * smp_wmb(); <- Ensure that the last base[1] update is visible * tkf->seq++; * smp_wmb(); <- Ensure that the seqcount update is visible * update(tkf->base[0], tk); * smp_wmb(); <- Ensure that the base[0] update is visible * tkf->seq++; * smp_wmb(); <- Ensure that the seqcount update is visible * update(tkf->base[1], tk); * * The reader side does: * * do { * seq = tkf->seq; * smp_rmb(); * idx = seq & 0x01; * now = now(tkf->base[idx]); * smp_rmb(); * } while (seq != tkf->seq) * * As long as we update base[0] readers are forced off to * base[1]. Once base[0] is updated readers are redirected to base[0] * and the base[1] update takes place. * * So if a NMI hits the update of base[0] then it will use base[1] * which is still consistent. In the worst case this can result is a * slightly wrong timestamp (a few nanoseconds). See * @ktime_get_mono_fast_ns. */static void update_fast_timekeeper(struct timekeeper *tk){ struct tk_read_base *base = tk_fast_mono.base; /* Force readers off to base[1] */ raw_write_seqcount_latch(&tk_fast_mono.seq); /* Update base[0] */ memcpy(base, &tk->tkr, sizeof(*base)); /* Force readers back to base[0] */ raw_write_seqcount_latch(&tk_fast_mono.seq); /* Update base[1] */ memcpy(base + 1, base, sizeof(*base));} /** * ktime_get_mono_fast_ns - Fast NMI safe access to clock monotonic * * This timestamp is not guaranteed to be monotonic across an update. * The timestamp is calculated by: * * now = base_mono + clock_delta * slope * * So if the update lowers the slope, readers who are forced to the * not yet updated second array are still using the old steeper slope. * * tmono * ^ * | o n * | o n * | u * | o * |o * |12345678---> reader order * * o = old slope * u = update * n = new slope * * So reader 6 will observe time going backwards versus reader 5. * * While other CPUs are likely to be able observe that, the only way * for a CPU local observation is when an NMI hits in the middle of * the update. Timestamps taken from that NMI context might be ahead * of the following timestamps. Callers need to be aware of that and * deal with it. */u64 notrace ktime_get_mono_fast_ns(void){ struct tk_read_base *tkr; unsigned int seq; u64 now; do { seq = raw_read_seqcount(&tk_fast_mono.seq); tkr = tk_fast_mono.base + (seq & 0x01); now = ktime_to_ns(tkr->base_mono) + timekeeping_get_ns(tkr); } while (read_seqcount_retry(&tk_fast_mono.seq, seq)); return now;}EXPORT_SYMBOL_GPL(ktime_get_mono_fast_ns); #ifdef CONFIG_GENERIC_TIME_VSYSCALL_OLD static inline void update_vsyscall(struct timekeeper *tk){ struct timespec xt, wm; xt = timespec64_to_timespec(tk_xtime(tk)); wm = timespec64_to_timespec(tk->wall_to_monotonic); update_vsyscall_old(&xt, &wm, tk->tkr.clock, tk->tkr.mult, tk->tkr.cycle_last);} static inline void old_vsyscall_fixup(struct timekeeper *tk){ s64 remainder; /* * Store only full nanoseconds into xtime_nsec after rounding * it up and add the remainder to the error difference. * XXX - This is necessary to avoid small 1ns inconsistnecies caused * by truncating the remainder in vsyscalls. However, it causes * additional work to be done in timekeeping_adjust(). Once * the vsyscall implementations are converted to use xtime_nsec * (shifted nanoseconds), and CONFIG_GENERIC_TIME_VSYSCALL_OLD * users are removed, this can be killed. */ remainder = tk->tkr.xtime_nsec & ((1ULL << tk->tkr.shift) - 1); tk->tkr.xtime_nsec -= remainder; tk->tkr.xtime_nsec += 1ULL << tk->tkr.shift; tk->ntp_error += remainder << tk->ntp_error_shift; tk->ntp_error -= (1ULL << tk->tkr.shift) << tk->ntp_error_shift;}#else#define old_vsyscall_fixup(tk)#endif static RAW_NOTIFIER_HEAD(pvclock_gtod_chain); static void update_pvclock_gtod(struct timekeeper *tk, bool was_set){ raw_notifier_call_chain(&pvclock_gtod_chain, was_set, tk);} /** * pvclock_gtod_register_notifier - register a pvclock timedata update listener */int pvclock_gtod_register_notifier(struct notifier_block *nb){ struct timekeeper *tk = &tk_core.timekeeper; unsigned long flags; int ret; raw_spin_lock_irqsave(&timekeeper_lock, flags); ret = raw_notifier_chain_register(&pvclock_gtod_chain, nb); update_pvclock_gtod(tk, true); raw_spin_unlock_irqrestore(&timekeeper_lock, flags); return ret;}EXPORT_SYMBOL_GPL(pvclock_gtod_register_notifier); /** * pvclock_gtod_unregister_notifier - unregister a pvclock * timedata update listener */int pvclock_gtod_unregister_notifier(struct notifier_block *nb){ unsigned long flags; int ret; raw_spin_lock_irqsave(&timekeeper_lock, flags); ret = raw_notifier_chain_unregister(&pvclock_gtod_chain, nb); raw_spin_unlock_irqrestore(&timekeeper_lock, flags); return ret;}EXPORT_SYMBOL_GPL(pvclock_gtod_unregister_notifier); /* * Update the ktime_t based scalar nsec members of the timekeeper */static inline void tk_update_ktime_data(struct timekeeper *tk){ s64 nsec; /* * The xtime based monotonic readout is: * nsec = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec + now(); * The ktime based monotonic readout is: * nsec = base_mono + now(); * ==> base_mono = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec */ nsec = (s64)(tk->xtime_sec + tk->wall_to_monotonic.tv_sec); nsec *= NSEC_PER_SEC; nsec += tk->wall_to_monotonic.tv_nsec; tk->tkr.base_mono = ns_to_ktime(nsec); /* Update the monotonic raw base */ tk->base_raw = timespec64_to_ktime(tk->raw_time);} /* must hold timekeeper_lock */static void timekeeping_update(struct timekeeper *tk, unsigned int action){ if (action & TK_CLEAR_NTP) { tk->ntp_error = 0; ntp_clear(); } tk_update_ktime_data(tk); update_vsyscall(tk); update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET); if (action & TK_MIRROR) memcpy(&shadow_timekeeper, &tk_core.timekeeper, sizeof(tk_core.timekeeper)); update_fast_timekeeper(tk);} /** * timekeeping_forward_now - update clock to the current time * * Forward the current clock to update its state since the last call to * update_wall_time(). This is useful before significant clock changes, * as it avoids having to deal with this time offset explicitly. */static void timekeeping_forward_now(struct timekeeper *tk){ struct clocksource *clock = tk->tkr.clock; cycle_t cycle_now, delta; s64 nsec; cycle_now = tk->tkr.read(clock); delta = clocksource_delta(cycle_now, tk->tkr.cycle_last, tk->tkr.mask); tk->tkr.cycle_last = cycle_now; tk->tkr.xtime_nsec += delta * tk->tkr.mult; /* If arch requires, add in get_arch_timeoffset() */ tk->tkr.xtime_nsec += (u64)arch_gettimeoffset() << tk->tkr.shift; tk_normalize_xtime(tk); nsec = clocksource_cyc2ns(delta, clock->mult, clock->shift); timespec64_add_ns(&tk->raw_time, nsec);} /** * __getnstimeofday64 - Returns the time of day in a timespec64. * @ts: pointer to the timespec to be set * * Updates the time of day in the timespec. * Returns 0 on success, or -ve when suspended (timespec will be undefined). */int __getnstimeofday64(struct timespec64 *ts){ struct timekeeper *tk = &tk_core.timekeeper; unsigned long seq; s64 nsecs = 0; do { seq = read_seqcount_begin(&tk_core.seq);Lines 1–500 of 1821. Load more
/* * linux/kernel/time/timekeeping.c * * Kernel timekeeping code and accessor functions * * This code was moved from linux/kernel/timer.c. * Please see that file for copyright and history logs. * */ #include <linux/timekeeper_internal.h> #include <linux/module.h> #include <linux/interrupt.h> #include <linux/percpu.h> #include <linux/init.h> #include <linux/mm.h> #include <linux/sched.h> #include <linux/syscore_ops.h> #include <linux/clocksource.h> #include <linux/jiffies.h> #include <linux/time.h> #include <linux/tick.h> #include <linux/stop_machine.h> #include <linux/pvclock_gtod.h> #include <linux/compiler.h> #include "tick-internal.h" #include "ntp_internal.h" #include "timekeeping_internal.h" #define TK_CLEAR_NTP (1 << 0) #define TK_MIRROR (1 << 1) #define TK_CLOCK_WAS_SET (1 << 2) /* * The most important data for readout fits into a single 64 byte * cache line. */ static struct { seqcount_t seq; struct timekeeper timekeeper; } tk_core ____cacheline_aligned; static DEFINE_RAW_SPINLOCK(timekeeper_lock); static struct timekeeper shadow_timekeeper; /** * struct tk_fast - NMI safe timekeeper * @seq: Sequence counter for protecting updates. The lowest bit * is the index for the tk_read_base array * @base: tk_read_base array. Access is indexed by the lowest bit of * @seq. * * See @update_fast_timekeeper() below. */ struct tk_fast { seqcount_t seq; struct tk_read_base base[2]; }; static struct tk_fast tk_fast_mono ____cacheline_aligned; /* flag for if timekeeping is suspended */ int __read_mostly timekeeping_suspended; /* Flag for if there is a persistent clock on this platform */ bool __read_mostly persistent_clock_exist = false; static inline void tk_normalize_xtime(struct timekeeper *tk) { while (tk->tkr.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr.shift)) { tk->tkr.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr.shift; tk->xtime_sec++; } } static inline struct timespec64 tk_xtime(struct timekeeper *tk) { struct timespec64 ts; ts.tv_sec = tk->xtime_sec; ts.tv_nsec = (long)(tk->tkr.xtime_nsec >> tk->tkr.shift); return ts; } static void tk_set_xtime(struct timekeeper *tk, const struct timespec64 *ts) { tk->xtime_sec = ts->tv_sec; tk->tkr.xtime_nsec = (u64)ts->tv_nsec << tk->tkr.shift; } static void tk_xtime_add(struct timekeeper *tk, const struct timespec64 *ts) { tk->xtime_sec += ts->tv_sec; tk->tkr.xtime_nsec += (u64)ts->tv_nsec << tk->tkr.shift; tk_normalize_xtime(tk); } static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec64 wtm) { struct timespec64 tmp; /* * Verify consistency of: offset_real = -wall_to_monotonic * before modifying anything */ set_normalized_timespec64(&tmp, -tk->wall_to_monotonic.tv_sec, -tk->wall_to_monotonic.tv_nsec); WARN_ON_ONCE(tk->offs_real.tv64 != timespec64_to_ktime(tmp).tv64); tk->wall_to_monotonic = wtm; set_normalized_timespec64(&tmp, -wtm.tv_sec, -wtm.tv_nsec); tk->offs_real = timespec64_to_ktime(tmp); tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0)); } static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta) { tk->offs_boot = ktime_add(tk->offs_boot, delta); } /** * tk_setup_internals - Set up internals to use clocksource clock. * * @tk: The target timekeeper to setup. * @clock: Pointer to clocksource. * * Calculates a fixed cycle/nsec interval for a given clocksource/adjustment * pair and interval request. * * Unless you're the timekeeping code, you should not be using this! */ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock) { cycle_t interval; u64 tmp, ntpinterval; struct clocksource *old_clock; old_clock = tk->tkr.clock; tk->tkr.clock = clock; tk->tkr.read = clock->read; tk->tkr.mask = clock->mask; tk->tkr.cycle_last = tk->tkr.read(clock); /* Do the ns -> cycle conversion first, using original mult */ tmp = NTP_INTERVAL_LENGTH; tmp <<= clock->shift; ntpinterval = tmp; tmp += clock->mult/2; do_div(tmp, clock->mult); if (tmp == 0) tmp = 1; interval = (cycle_t) tmp; tk->cycle_interval = interval; /* Go back from cycles -> shifted ns */ tk->xtime_interval = (u64) interval * clock->mult; tk->xtime_remainder = ntpinterval - tk->xtime_interval; tk->raw_interval = ((u64) interval * clock->mult) >> clock->shift; /* if changing clocks, convert xtime_nsec shift units */ if (old_clock) { int shift_change = clock->shift - old_clock->shift; if (shift_change < 0) tk->tkr.xtime_nsec >>= -shift_change; else tk->tkr.xtime_nsec <<= shift_change; } tk->tkr.shift = clock->shift; tk->ntp_error = 0; tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift; tk->ntp_tick = ntpinterval << tk->ntp_error_shift; /* * The timekeeper keeps its own mult values for the currently * active clocksource. These value will be adjusted via NTP * to counteract clock drifting. */ tk->tkr.mult = clock->mult; tk->ntp_err_mult = 0; } /* Timekeeper helper functions. */ #ifdef CONFIG_ARCH_USES_GETTIMEOFFSET static u32 default_arch_gettimeoffset(void) { return 0; } u32 (*arch_gettimeoffset)(void) = default_arch_gettimeoffset; #else static inline u32 arch_gettimeoffset(void) { return 0; } #endif static inline s64 timekeeping_get_ns(struct tk_read_base *tkr) { cycle_t cycle_now, delta; s64 nsec; /* read clocksource: */ cycle_now = tkr->read(tkr->clock); /* calculate the delta since the last update_wall_time: */ delta = clocksource_delta(cycle_now, tkr->cycle_last, tkr->mask); nsec = delta * tkr->mult + tkr->xtime_nsec; nsec >>= tkr->shift; /* If arch requires, add in get_arch_timeoffset() */ return nsec + arch_gettimeoffset(); } static inline s64 timekeeping_get_ns_raw(struct timekeeper *tk) { struct clocksource *clock = tk->tkr.clock; cycle_t cycle_now, delta; s64 nsec; /* read clocksource: */ cycle_now = tk->tkr.read(clock); /* calculate the delta since the last update_wall_time: */ delta = clocksource_delta(cycle_now, tk->tkr.cycle_last, tk->tkr.mask); /* convert delta to nanoseconds. */ nsec = clocksource_cyc2ns(delta, clock->mult, clock->shift); /* If arch requires, add in get_arch_timeoffset() */ return nsec + arch_gettimeoffset(); } /** * update_fast_timekeeper - Update the fast and NMI safe monotonic timekeeper. * @tk: The timekeeper from which we take the update * @tkf: The fast timekeeper to update * @tbase: The time base for the fast timekeeper (mono/raw) * * We want to use this from any context including NMI and tracing / * instrumenting the timekeeping code itself. * * So we handle this differently than the other timekeeping accessor * functions which retry when the sequence count has changed. The * update side does: * * smp_wmb(); <- Ensure that the last base[1] update is visible * tkf->seq++; * smp_wmb(); <- Ensure that the seqcount update is visible * update(tkf->base[0], tk); * smp_wmb(); <- Ensure that the base[0] update is visible * tkf->seq++; * smp_wmb(); <- Ensure that the seqcount update is visible * update(tkf->base[1], tk); * * The reader side does: * * do { * seq = tkf->seq; * smp_rmb(); * idx = seq & 0x01; * now = now(tkf->base[idx]); * smp_rmb(); * } while (seq != tkf->seq) * * As long as we update base[0] readers are forced off to * base[1]. Once base[0] is updated readers are redirected to base[0] * and the base[1] update takes place. * * So if a NMI hits the update of base[0] then it will use base[1] * which is still consistent. In the worst case this can result is a * slightly wrong timestamp (a few nanoseconds). See * @ktime_get_mono_fast_ns. */ static void update_fast_timekeeper(struct timekeeper *tk) { struct tk_read_base *base = tk_fast_mono.base; /* Force readers off to base[1] */ raw_write_seqcount_latch(&tk_fast_mono.seq); /* Update base[0] */ memcpy(base, &tk->tkr, sizeof(*base)); /* Force readers back to base[0] */ raw_write_seqcount_latch(&tk_fast_mono.seq); /* Update base[1] */ memcpy(base + 1, base, sizeof(*base)); } /** * ktime_get_mono_fast_ns - Fast NMI safe access to clock monotonic * * This timestamp is not guaranteed to be monotonic across an update. * The timestamp is calculated by: * * now = base_mono + clock_delta * slope * * So if the update lowers the slope, readers who are forced to the * not yet updated second array are still using the old steeper slope. * * tmono * ^ * | o n * | o n * | u * | o * |o * |12345678---> reader order * * o = old slope * u = update * n = new slope * * So reader 6 will observe time going backwards versus reader 5. * * While other CPUs are likely to be able observe that, the only way * for a CPU local observation is when an NMI hits in the middle of * the update. Timestamps taken from that NMI context might be ahead * of the following timestamps. Callers need to be aware of that and * deal with it. */ u64 notrace ktime_get_mono_fast_ns(void) { struct tk_read_base *tkr; unsigned int seq; u64 now; do { seq = raw_read_seqcount(&tk_fast_mono.seq); tkr = tk_fast_mono.base + (seq & 0x01); now = ktime_to_ns(tkr->base_mono) + timekeeping_get_ns(tkr); } while (read_seqcount_retry(&tk_fast_mono.seq, seq)); return now; } EXPORT_SYMBOL_GPL(ktime_get_mono_fast_ns); #ifdef CONFIG_GENERIC_TIME_VSYSCALL_OLD static inline void update_vsyscall(struct timekeeper *tk) { struct timespec xt, wm; xt = timespec64_to_timespec(tk_xtime(tk)); wm = timespec64_to_timespec(tk->wall_to_monotonic); update_vsyscall_old(&xt, &wm, tk->tkr.clock, tk->tkr.mult, tk->tkr.cycle_last); } static inline void old_vsyscall_fixup(struct timekeeper *tk) { s64 remainder; /* * Store only full nanoseconds into xtime_nsec after rounding * it up and add the remainder to the error difference. * XXX - This is necessary to avoid small 1ns inconsistnecies caused * by truncating the remainder in vsyscalls. However, it causes * additional work to be done in timekeeping_adjust(). Once * the vsyscall implementations are converted to use xtime_nsec * (shifted nanoseconds), and CONFIG_GENERIC_TIME_VSYSCALL_OLD * users are removed, this can be killed. */ remainder = tk->tkr.xtime_nsec & ((1ULL << tk->tkr.shift) - 1); tk->tkr.xtime_nsec -= remainder; tk->tkr.xtime_nsec += 1ULL << tk->tkr.shift; tk->ntp_error += remainder << tk->ntp_error_shift; tk->ntp_error -= (1ULL << tk->tkr.shift) << tk->ntp_error_shift; } #else #define old_vsyscall_fixup(tk) #endif static RAW_NOTIFIER_HEAD(pvclock_gtod_chain); static void update_pvclock_gtod(struct timekeeper *tk, bool was_set) { raw_notifier_call_chain(&pvclock_gtod_chain, was_set, tk); } /** * pvclock_gtod_register_notifier - register a pvclock timedata update listener */ int pvclock_gtod_register_notifier(struct notifier_block *nb) { struct timekeeper *tk = &tk_core.timekeeper; unsigned long flags; int ret; raw_spin_lock_irqsave(&timekeeper_lock, flags); ret = raw_notifier_chain_register(&pvclock_gtod_chain, nb); update_pvclock_gtod(tk, true); raw_spin_unlock_irqrestore(&timekeeper_lock, flags); return ret; } EXPORT_SYMBOL_GPL(pvclock_gtod_register_notifier); /** * pvclock_gtod_unregister_notifier - unregister a pvclock * timedata update listener */ int pvclock_gtod_unregister_notifier(struct notifier_block *nb) { unsigned long flags; int ret; raw_spin_lock_irqsave(&timekeeper_lock, flags); ret = raw_notifier_chain_unregister(&pvclock_gtod_chain, nb); raw_spin_unlock_irqrestore(&timekeeper_lock, flags); return ret; } EXPORT_SYMBOL_GPL(pvclock_gtod_unregister_notifier); /* * Update the ktime_t based scalar nsec members of the timekeeper */ static inline void tk_update_ktime_data(struct timekeeper *tk) { s64 nsec; /* * The xtime based monotonic readout is: * nsec = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec + now(); * The ktime based monotonic readout is: * nsec = base_mono + now(); * ==> base_mono = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec */ nsec = (s64)(tk->xtime_sec + tk->wall_to_monotonic.tv_sec); nsec *= NSEC_PER_SEC; nsec += tk->wall_to_monotonic.tv_nsec; tk->tkr.base_mono = ns_to_ktime(nsec); /* Update the monotonic raw base */ tk->base_raw = timespec64_to_ktime(tk->raw_time); } /* must hold timekeeper_lock */ static void timekeeping_update(struct timekeeper *tk, unsigned int action) { if (action & TK_CLEAR_NTP) { tk->ntp_error = 0; ntp_clear(); } tk_update_ktime_data(tk); update_vsyscall(tk); update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET); if (action & TK_MIRROR) memcpy(&shadow_timekeeper, &tk_core.timekeeper, sizeof(tk_core.timekeeper)); update_fast_timekeeper(tk); } /** * timekeeping_forward_now - update clock to the current time * * Forward the current clock to update its state since the last call to * update_wall_time(). This is useful before significant clock changes, * as it avoids having to deal with this time offset explicitly. */ static void timekeeping_forward_now(struct timekeeper *tk) { struct clocksource *clock = tk->tkr.clock; cycle_t cycle_now, delta; s64 nsec; cycle_now = tk->tkr.read(clock); delta = clocksource_delta(cycle_now, tk->tkr.cycle_last, tk->tkr.mask); tk->tkr.cycle_last = cycle_now; tk->tkr.xtime_nsec += delta * tk->tkr.mult; /* If arch requires, add in get_arch_timeoffset() */ tk->tkr.xtime_nsec += (u64)arch_gettimeoffset() << tk->tkr.shift; tk_normalize_xtime(tk); nsec = clocksource_cyc2ns(delta, clock->mult, clock->shift); timespec64_add_ns(&tk->raw_time, nsec); } /** * __getnstimeofday64 - Returns the time of day in a timespec64. * @ts: pointer to the timespec to be set * * Updates the time of day in the timespec. * Returns 0 on success, or -ve when suspended (timespec will be undefined). */ int __getnstimeofday64(struct timespec64 *ts) { struct timekeeper *tk = &tk_core.timekeeper; unsigned long seq; s64 nsecs = 0; do { seq = read_seqcount_begin(&tk_core.seq);