// SPDX-License-Identifier: GPL-2.0+/* * This file contains the functions which manage clocksource drivers. * * Copyright (C) 2004, 2005 IBM, John Stultz (johnstul@us.ibm.com) */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/clocksource.h>#include <linux/cpu.h>#include <linux/delay.h>#include <linux/device.h>#include <linux/init.h>#include <linux/kthread.h>#include <linux/module.h>#include <linux/prandom.h>#include <linux/sched.h>#include <linux/tick.h>#include <linux/topology.h> #include "tick-internal.h"#include "timekeeping_internal.h" static void clocksource_enqueue(struct clocksource *cs); static noinline u64 cycles_to_nsec_safe(struct clocksource *cs, u64 start, u64 end){	u64 delta = clocksource_delta(end, start, cs->mask, cs->max_raw_delta); 	if (likely(delta < cs->max_cycles))		return clocksource_cyc2ns(delta, cs->mult, cs->shift); 	return mul_u64_u32_shr(delta, cs->mult, cs->shift);} /** * clocks_calc_mult_shift - calculate mult/shift factors for scaled math of clocks * @mult:	pointer to mult variable * @shift:	pointer to shift variable * @from:	frequency to convert from * @to:		frequency to convert to * @maxsec:	guaranteed runtime conversion range in seconds * * The function evaluates the shift/mult pair for the scaled math * operations of clocksources and clockevents. * * @to and @from are frequency values in HZ. For clock sources @to is * NSEC_PER_SEC == 1GHz and @from is the counter frequency. For clock * event @to is the counter frequency and @from is NSEC_PER_SEC. * * The @maxsec conversion range argument controls the time frame in * seconds which must be covered by the runtime conversion with the * calculated mult and shift factors. This guarantees that no 64bit * overflow happens when the input value of the conversion is * multiplied with the calculated mult factor. Larger ranges may * reduce the conversion accuracy by choosing smaller mult and shift * factors. */voidclocks_calc_mult_shift(u32 *mult, u32 *shift, u32 from, u32 to, u32 maxsec){	u64 tmp;	u32 sft, sftacc= 32; 	/*	 * Calculate the shift factor which is limiting the conversion	 * range:	 */	tmp = ((u64)maxsec * from) >> 32;	while (tmp) {		tmp >>=1;		sftacc--;	} 	/*	 * Find the conversion shift/mult pair which has the best	 * accuracy and fits the maxsec conversion range:	 */	for (sft = 32; sft > 0; sft--) {		tmp = (u64) to << sft;		tmp += from / 2;		do_div(tmp, from);		if ((tmp >> sftacc) == 0)			break;	}	*mult = tmp;	*shift = sft;}EXPORT_SYMBOL_GPL(clocks_calc_mult_shift); /*[Clocksource internal variables]--------- * curr_clocksource: *	currently selected clocksource. * suspend_clocksource: *	used to calculate the suspend time. * clocksource_list: *	linked list with the registered clocksources * clocksource_mutex: *	protects manipulations to curr_clocksource and the clocksource_list * override_name: *	Name of the user-specified clocksource. */static struct clocksource *curr_clocksource;static struct clocksource *suspend_clocksource;static LIST_HEAD(clocksource_list);static DEFINE_MUTEX(clocksource_mutex);static char override_name[CS_NAME_LEN];static int finished_booting;static u64 suspend_start; #ifdef CONFIG_CLOCKSOURCE_WATCHDOGstatic void clocksource_watchdog_work(struct work_struct *work);static void clocksource_select(void); static LIST_HEAD(watchdog_list);static struct clocksource *watchdog;static struct timer_list watchdog_timer;static DECLARE_WORK(watchdog_work, clocksource_watchdog_work);static DEFINE_SPINLOCK(watchdog_lock);static int watchdog_running;static atomic_t watchdog_reset_pending; /* Watchdog interval: 0.5sec. */#define WATCHDOG_INTERVAL		(HZ >> 1) /* Maximum time between two reference watchdog readouts */#define WATCHDOG_READOUT_MAX_NS		(50U * NSEC_PER_USEC) /* * Maximum time between two remote readouts for NUMA=n. On NUMA enabled systems * the timeout is calculated from the numa distance. */#define WATCHDOG_DEFAULT_TIMEOUT_NS	(50U * NSEC_PER_USEC) /* * Remote timeout NUMA distance multiplier. The local distance is 10. The * default remote distance is 20. ACPI tables provide more accurate numbers * which are guaranteed to be greater than the local distance. * * This results in a 5us base value, which is equivalent to the above !NUMA * default. */#define WATCHDOG_NUMA_MULTIPLIER_NS	((u64)(WATCHDOG_DEFAULT_TIMEOUT_NS / LOCAL_DISTANCE)) /* Limit the NUMA timeout in case the distance values are insanely big */#define WATCHDOG_NUMA_MAX_TIMEOUT_NS	((u64)(500U * NSEC_PER_USEC)) /* Shift values to calculate the approximate $N ppm of a given delta. */#define SHIFT_500PPM			11#define SHIFT_4000PPM			8 /* Number of attempts to read the watchdog */#define WATCHDOG_FREQ_RETRIES		3 /* Five reads local and remote for inter CPU skew detection */#define WATCHDOG_REMOTE_MAX_SEQ		10 static inline void clocksource_watchdog_lock(unsigned long *flags){	spin_lock_irqsave(&watchdog_lock, *flags);} static inline void clocksource_watchdog_unlock(unsigned long *flags){	spin_unlock_irqrestore(&watchdog_lock, *flags);} static int clocksource_watchdog_kthread(void *data); static void clocksource_watchdog_work(struct work_struct *work){	/*	 * We cannot directly run clocksource_watchdog_kthread() here, because	 * clocksource_select() calls timekeeping_notify() which uses	 * stop_machine(). One cannot use stop_machine() from a workqueue() due	 * lock inversions wrt CPU hotplug.	 *	 * Also, we only ever run this work once or twice during the lifetime	 * of the kernel, so there is no point in creating a more permanent	 * kthread for this.	 *	 * If kthread_run fails the next watchdog scan over the	 * watchdog_list will find the unstable clock again.	 */	kthread_run(clocksource_watchdog_kthread, NULL, "kwatchdog");} static void clocksource_change_rating(struct clocksource *cs, int rating){	list_del(&cs->list);	cs->rating = rating;	clocksource_enqueue(cs);} static void __clocksource_unstable(struct clocksource *cs){	cs->flags &= ~(CLOCK_SOURCE_VALID_FOR_HRES | CLOCK_SOURCE_WATCHDOG);	cs->flags |= CLOCK_SOURCE_UNSTABLE; 	/*	 * If the clocksource is registered clocksource_watchdog_kthread() will	 * re-rate and re-select.	 */	if (list_empty(&cs->list)) {		cs->rating = 0;		return;	} 	if (cs->mark_unstable)		cs->mark_unstable(cs); 	/* kick clocksource_watchdog_kthread() */	if (finished_booting)		schedule_work(&watchdog_work);} /** * clocksource_mark_unstable - mark clocksource unstable via watchdog * @cs:		clocksource to be marked unstable * * This function is called by the x86 TSC code to mark clocksources as unstable; * it defers demotion and re-selection to a kthread. */void clocksource_mark_unstable(struct clocksource *cs){	unsigned long flags; 	spin_lock_irqsave(&watchdog_lock, flags);	if (!(cs->flags & CLOCK_SOURCE_UNSTABLE)) {		if (!list_empty(&cs->list) && list_empty(&cs->wd_list))			list_add(&cs->wd_list, &watchdog_list);		__clocksource_unstable(cs);	}	spin_unlock_irqrestore(&watchdog_lock, flags);} static inline void clocksource_reset_watchdog(void){	struct clocksource *cs; 	list_for_each_entry(cs, &watchdog_list, wd_list)		cs->flags &= ~CLOCK_SOURCE_WATCHDOG;} enum wd_result {	WD_SUCCESS,	WD_FREQ_NO_WATCHDOG,	WD_FREQ_TIMEOUT,	WD_FREQ_RESET,	WD_FREQ_SKEWED,	WD_CPU_TIMEOUT,	WD_CPU_SKEWED,}; struct watchdog_cpu_data {	/* Keep first as it is 32 byte aligned */	call_single_data_t	csd;	atomic_t		remote_inprogress;	enum wd_result		result;	u64			cpu_ts[2];	struct clocksource	*cs;	/* Ensure that the sequence is in a separate cache line */	atomic_t		seq ____cacheline_aligned;	/* Set by the control CPU according to NUMA distance */	u64			timeout_ns;}; struct watchdog_data {	raw_spinlock_t	lock;	enum wd_result	result; 	u64		wd_seq;	u64		wd_delta;	u64		cs_delta;	u64		cpu_ts[2]; 	unsigned int	curr_cpu;} ____cacheline_aligned_in_smp; static void watchdog_check_skew_remote(void *unused); static DEFINE_PER_CPU_ALIGNED(struct watchdog_cpu_data, watchdog_cpu_data) = {	.csd	= CSD_INIT(watchdog_check_skew_remote, NULL),}; static struct watchdog_data watchdog_data = {	.lock	= __RAW_SPIN_LOCK_UNLOCKED(watchdog_data.lock),}; static inline void watchdog_set_result(struct watchdog_cpu_data *wd, enum wd_result result){	guard(raw_spinlock)(&watchdog_data.lock);	if (!wd->result) {		atomic_set(&wd->seq, WATCHDOG_REMOTE_MAX_SEQ);		WRITE_ONCE(wd->result, result);	}} /* Wait for the sequence number to hand over control. */static bool watchdog_wait_seq(struct watchdog_cpu_data *wd, u64 start, int seq){	for(int cnt = 0; atomic_read(&wd->seq) < seq; cnt++) {		/* Bail if the other side set an error result */		if (READ_ONCE(wd->result) != WD_SUCCESS)			return false; 		/* Prevent endless loops if the other CPU does not react. */		if (cnt == 5000) {			u64 nsecs = ktime_get_raw_fast_ns(); 			if (nsecs - start >=wd->timeout_ns) {				watchdog_set_result(wd, WD_CPU_TIMEOUT);				return false;			}			cnt = 0;		}		cpu_relax();	}	return seq < WATCHDOG_REMOTE_MAX_SEQ;} static void watchdog_check_skew(struct watchdog_cpu_data *wd, int index){	u64 prev, now, delta, start = ktime_get_raw_fast_ns();	int local = index, remote = (index + 1) & 0x1;	struct clocksource *cs = wd->cs; 	/* Set the local timestamp so that the first iteration works correctly */	wd->cpu_ts[local] = cs->read(cs); 	/* Signal arrival */	atomic_inc(&wd->seq); 	for (int seq = local + 2; seq < WATCHDOG_REMOTE_MAX_SEQ; seq += 2) {		if (!watchdog_wait_seq(wd, start, seq))			return; 		/* Capture local timestamp before possible non-local coherency overhead */		now = cs->read(cs); 		/* Store local timestamp before reading remote to limit coherency stalls */		wd->cpu_ts[local] = now; 		prev = wd->cpu_ts[remote];		delta = (now - prev) & cs->mask; 		if (delta > cs->max_raw_delta) {			watchdog_set_result(wd, WD_CPU_SKEWED);			return;		} 		/* Hand over to the remote CPU */		atomic_inc(&wd->seq);	}} static void watchdog_check_skew_remote(void *unused){	struct watchdog_cpu_data *wd = this_cpu_ptr(&watchdog_cpu_data); 	atomic_inc(&wd->remote_inprogress);	watchdog_check_skew(wd, 1);	atomic_dec(&wd->remote_inprogress);} static inline bool wd_csd_locked(struct watchdog_cpu_data *wd){	return READ_ONCE(wd->csd.node.u_flags) & CSD_FLAG_LOCK;} /* * This is only invoked for remote CPUs. See watchdog_check_cpu_skew(). */static inline u64 wd_get_remote_timeout(unsigned int remote_cpu){	unsigned int n1, n2;	u64 ns; 	if (nr_node_ids == 1)		return WATCHDOG_DEFAULT_TIMEOUT_NS; 	n1 = cpu_to_node(smp_processor_id());	n2 = cpu_to_node(remote_cpu);	ns = WATCHDOG_NUMA_MULTIPLIER_NS * node_distance(n1, n2);	return min(ns, WATCHDOG_NUMA_MAX_TIMEOUT_NS);} static void __watchdog_check_cpu_skew(struct clocksource *cs, unsigned int cpu){	struct watchdog_cpu_data *wd; 	wd = per_cpu_ptr(&watchdog_cpu_data, cpu);	if (atomic_read(&wd->remote_inprogress) || wd_csd_locked(wd)) {		watchdog_data.result = WD_CPU_TIMEOUT;		return;	} 	atomic_set(&wd->seq, 0);	wd->result = WD_SUCCESS;	wd->cs = cs;	/* Store the current CPU ID for the watchdog test unit */	cs->wd_cpu = smp_processor_id(); 	wd->timeout_ns = wd_get_remote_timeout(cpu); 	/* Kick the remote CPU into the watchdog function */	if (WARN_ON_ONCE(smp_call_function_single_async(cpu, &wd->csd))) {		watchdog_data.result = WD_CPU_TIMEOUT;		return;	} 	scoped_guard(irq)		watchdog_check_skew(wd, 0); 	scoped_guard(raw_spinlock_irq, &watchdog_data.lock) {		watchdog_data.result = wd->result;		memcpy(watchdog_data.cpu_ts, wd->cpu_ts, sizeof(wd->cpu_ts));	}} static void watchdog_check_cpu_skew(struct clocksource *cs){	unsigned int cpu = watchdog_data.curr_cpu; 	cpu = cpumask_next_wrap(cpu, cpu_online_mask);	watchdog_data.curr_cpu = cpu; 	/* Skip the current CPU. Handles num_online_cpus() == 1 as well */	if (cpu == smp_processor_id())		return; 	/* Don't interfere with the test mechanics */	if ((cs->flags & CLOCK_SOURCE_WDTEST) && !(cs->flags & CLOCK_SOURCE_WDTEST_PERCPU))		return; 	__watchdog_check_cpu_skew(cs, cpu);} static bool watchdog_check_freq(struct clocksource *cs, bool reset_pending){	unsigned int ppm_shift = SHIFT_4000PPM;	u64 wd_ts0, wd_ts1, cs_ts; 	watchdog_data.result = WD_SUCCESS;	if (!watchdog) {		watchdog_data.result = WD_FREQ_NO_WATCHDOG;		return false;	} 	if (cs->flags & CLOCK_SOURCE_WDTEST_PERCPU)		return true; 	/*	 * If both the clocksource and the watchdog claim they are	 * calibrated use 500ppm limit. Uncalibrated clocksources need a	 * larger allowance because thefirmware supplied frequencies can be	 * way off.	 */	if (watchdog->flags & CLOCK_SOURCE_CALIBRATED && cs->flags & CLOCK_SOURCE_CALIBRATED)		ppm_shift = SHIFT_500PPM; 	for (int retries = 0; retries < WATCHDOG_FREQ_RETRIES; retries++) {		s64 wd_last, cs_last, wd_seq, wd_delta, cs_delta, max_delta; 		scoped_guard(irq) {			wd_ts0 = watchdog->read(watchdog);			cs_ts = cs->read(cs);			wd_ts1 = watchdog->read(watchdog);		} 		wd_last = cs->wd_last;		cs_last = cs->cs_last; 		/* Validate the watchdog readout window */		wd_seq = cycles_to_nsec_safe(watchdog, wd_ts0, wd_ts1);		if (wd_seq > WATCHDOG_READOUT_MAX_NS) {			/* Store for printout in case all retries fail */			watchdog_data.wd_seq = wd_seq;			continue;		} 		/* Store for subsequent processing */		cs->wd_last = wd_ts0;		cs->cs_last = cs_ts; 		/* First round or reset pending? */		if (!(cs->flags & CLOCK_SOURCE_WATCHDOG) || reset_pending)			goto reset; 		/* Calculate the nanosecond deltas from the last invocation */		wd_delta = cycles_to_nsec_safe(watchdog, wd_last, wd_ts0);		cs_delta = cycles_to_nsec_safe(cs, cs_last, cs_ts); 		watchdog_data.wd_delta = wd_delta;		watchdog_data.cs_delta = cs_delta; 		/*		 * Ensure that the deltas are within the readout limits of		 * the clocksource and the watchdog. Long delays can cause

Lines 1–500 of 1616. Load more