// SPDX-License-Identifier: GPL-2.0-only/* *  linux/drivers/clocksource/arm_arch_timer.c * *  Copyright (C) 2011 ARM Ltd. *  All Rights Reserved */ #define pr_fmt(fmt) 	"arch_timer: " fmt #include <linux/init.h>#include <linux/kernel.h>#include <linux/device.h>#include <linux/smp.h>#include <linux/cpu.h>#include <linux/cpu_pm.h>#include <linux/clockchips.h>#include <linux/clocksource.h>#include <linux/clocksource_ids.h>#include <linux/interrupt.h>#include <linux/of_irq.h>#include <linux/of_address.h>#include <linux/io.h>#include <linux/slab.h>#include <linux/sched/clock.h>#include <linux/sched_clock.h>#include <linux/acpi.h>#include <linux/arm-smccc.h>#include <linux/ptp_kvm.h> #include <asm/arch_timer.h>#include <asm/virt.h> #include <clocksource/arm_arch_timer.h> #define CNTTIDR		0x08#define CNTTIDR_VIRT(n)	(BIT(1) << ((n) * 4)) #define CNTACR(n)	(0x40 + ((n) * 4))#define CNTACR_RPCT	BIT(0)#define CNTACR_RVCT	BIT(1)#define CNTACR_RFRQ	BIT(2)#define CNTACR_RVOFF	BIT(3)#define CNTACR_RWVT	BIT(4)#define CNTACR_RWPT	BIT(5) #define CNTVCT_LO	0x08#define CNTVCT_HI	0x0c#define CNTFRQ		0x10#define CNTP_TVAL	0x28#define CNTP_CTL	0x2c#define CNTV_TVAL	0x38#define CNTV_CTL	0x3c static unsigned arch_timers_present __initdata; static void __iomem *arch_counter_base __ro_after_init; struct arch_timer {	void __iomem *base;	struct clock_event_device evt;}; #define to_arch_timer(e) container_of(e, struct arch_timer, evt) static u32 arch_timer_rate __ro_after_init;u32 arch_timer_rate1 __ro_after_init;static int arch_timer_ppi[ARCH_TIMER_MAX_TIMER_PPI] __ro_after_init; static const char *arch_timer_ppi_names[ARCH_TIMER_MAX_TIMER_PPI] = {	[ARCH_TIMER_PHYS_SECURE_PPI]	= "sec-phys",	[ARCH_TIMER_PHYS_NONSECURE_PPI]	= "phys",	[ARCH_TIMER_VIRT_PPI]		= "virt",	[ARCH_TIMER_HYP_PPI]		= "hyp-phys",	[ARCH_TIMER_HYP_VIRT_PPI]	= "hyp-virt",}; static struct clock_event_device __percpu *arch_timer_evt; static enum arch_timer_ppi_nr arch_timer_uses_ppi __ro_after_init = ARCH_TIMER_VIRT_PPI;static bool arch_timer_c3stop __ro_after_init;static bool arch_timer_mem_use_virtual __ro_after_init;static bool arch_counter_suspend_stop __ro_after_init;#ifdef CONFIG_GENERIC_GETTIMEOFDAYstatic enum vdso_clock_mode vdso_default = VDSO_CLOCKMODE_ARCHTIMER;#elsestatic enum vdso_clock_mode vdso_default = VDSO_CLOCKMODE_NONE;#endif /* CONFIG_GENERIC_GETTIMEOFDAY */ static cpumask_t evtstrm_available = CPU_MASK_NONE;static bool evtstrm_enable __ro_after_init = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM); static int __init early_evtstrm_cfg(char *buf){	return strtobool(buf, &evtstrm_enable);}early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg); /* * Architected system timer support. */ static __always_inlinevoid arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val,			  struct clock_event_device *clk){	if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {		struct arch_timer *timer = to_arch_timer(clk);		switch (reg) {		case ARCH_TIMER_REG_CTRL:			writel_relaxed(val, timer->base + CNTP_CTL);			break;		case ARCH_TIMER_REG_TVAL:			writel_relaxed(val, timer->base + CNTP_TVAL);			break;		}	} else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {		struct arch_timer *timer = to_arch_timer(clk);		switch (reg) {		case ARCH_TIMER_REG_CTRL:			writel_relaxed(val, timer->base + CNTV_CTL);			break;		case ARCH_TIMER_REG_TVAL:			writel_relaxed(val, timer->base + CNTV_TVAL);			break;		}	} else {		arch_timer_reg_write_cp15(access, reg, val);	}} static __always_inlineu32 arch_timer_reg_read(int access, enum arch_timer_reg reg,			struct clock_event_device *clk){	u32 val; 	if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {		struct arch_timer *timer = to_arch_timer(clk);		switch (reg) {		case ARCH_TIMER_REG_CTRL:			val = readl_relaxed(timer->base + CNTP_CTL);			break;		case ARCH_TIMER_REG_TVAL:			val = readl_relaxed(timer->base + CNTP_TVAL);			break;		}	} else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {		struct arch_timer *timer = to_arch_timer(clk);		switch (reg) {		case ARCH_TIMER_REG_CTRL:			val = readl_relaxed(timer->base + CNTV_CTL);			break;		case ARCH_TIMER_REG_TVAL:			val = readl_relaxed(timer->base + CNTV_TVAL);			break;		}	} else {		val = arch_timer_reg_read_cp15(access, reg);	} 	return val;} static notrace u64 arch_counter_get_cntpct_stable(void){	return __arch_counter_get_cntpct_stable();} static notrace u64 arch_counter_get_cntpct(void){	return __arch_counter_get_cntpct();} static notrace u64 arch_counter_get_cntvct_stable(void){	return __arch_counter_get_cntvct_stable();} static notrace u64 arch_counter_get_cntvct(void){	return __arch_counter_get_cntvct();} /* * Default to cp15 based access because arm64 uses this function for * sched_clock() before DT is probed and the cp15 method is guaranteed * to exist on arm64. arm doesn't use this before DT is probed so even * if we don't have the cp15 accessors we won't have a problem. */u64 (*arch_timer_read_counter)(void) __ro_after_init = arch_counter_get_cntvct;EXPORT_SYMBOL_GPL(arch_timer_read_counter); static u64 arch_counter_read(struct clocksource *cs){	return arch_timer_read_counter();} static u64 arch_counter_read_cc(const struct cyclecounter *cc){	return arch_timer_read_counter();} static struct clocksource clocksource_counter = {	.name	= "arch_sys_counter",	.id	= CSID_ARM_ARCH_COUNTER,	.rating	= 400,	.read	= arch_counter_read,	.mask	= CLOCKSOURCE_MASK(56),	.flags	= CLOCK_SOURCE_IS_CONTINUOUS,}; static struct cyclecounter cyclecounter __ro_after_init = {	.read	= arch_counter_read_cc,	.mask	= CLOCKSOURCE_MASK(56),}; struct ate_acpi_oem_info {	char oem_id[ACPI_OEM_ID_SIZE + 1];	char oem_table_id[ACPI_OEM_TABLE_ID_SIZE + 1];	u32 oem_revision;}; #ifdef CONFIG_FSL_ERRATUM_A008585/* * The number of retries is an arbitrary value well beyond the highest number * of iterations the loop has been observed to take. */#define __fsl_a008585_read_reg(reg) ({			\	u64 _old, _new;					\	int _retries = 200;				\							\	do {						\		_old = read_sysreg(reg);		\		_new = read_sysreg(reg);		\		_retries--;				\	} while (unlikely(_old != _new) && _retries);	\							\	WARN_ON_ONCE(!_retries);			\	_new;						\}) static u32 notrace fsl_a008585_read_cntp_tval_el0(void){	return __fsl_a008585_read_reg(cntp_tval_el0);} static u32 notrace fsl_a008585_read_cntv_tval_el0(void){	return __fsl_a008585_read_reg(cntv_tval_el0);} static u64 notrace fsl_a008585_read_cntpct_el0(void){	return __fsl_a008585_read_reg(cntpct_el0);} static u64 notrace fsl_a008585_read_cntvct_el0(void){	return __fsl_a008585_read_reg(cntvct_el0);}#endif #ifdef CONFIG_HISILICON_ERRATUM_161010101/* * Verify whether the value of the second read is larger than the first by * less than 32 is the only way to confirm the value is correct, so clear the * lower 5 bits to check whether the difference is greater than 32 or not. * Theoretically the erratum should not occur more than twice in succession * when reading the system counter, but it is possible that some interrupts * may lead to more than twice read errors, triggering the warning, so setting * the number of retries far beyond the number of iterations the loop has been * observed to take. */#define __hisi_161010101_read_reg(reg) ({				\	u64 _old, _new;						\	int _retries = 50;					\								\	do {							\		_old = read_sysreg(reg);			\		_new = read_sysreg(reg);			\		_retries--;					\	} while (unlikely((_new - _old) >> 5) && _retries);	\								\	WARN_ON_ONCE(!_retries);				\	_new;							\}) static u32 notrace hisi_161010101_read_cntp_tval_el0(void){	return __hisi_161010101_read_reg(cntp_tval_el0);} static u32 notrace hisi_161010101_read_cntv_tval_el0(void){	return __hisi_161010101_read_reg(cntv_tval_el0);} static u64 notrace hisi_161010101_read_cntpct_el0(void){	return __hisi_161010101_read_reg(cntpct_el0);} static u64 notrace hisi_161010101_read_cntvct_el0(void){	return __hisi_161010101_read_reg(cntvct_el0);} static struct ate_acpi_oem_info hisi_161010101_oem_info[] = {	/*	 * Note that trailing spaces are required to properly match	 * the OEM table information.	 */	{		.oem_id		= "HISI  ",		.oem_table_id	= "HIP05   ",		.oem_revision	= 0,	},	{		.oem_id		= "HISI  ",		.oem_table_id	= "HIP06   ",		.oem_revision	= 0,	},	{		.oem_id		= "HISI  ",		.oem_table_id	= "HIP07   ",		.oem_revision	= 0,	},	{ /* Sentinel indicating the end of the OEM array */ },};#endif #ifdef CONFIG_ARM64_ERRATUM_858921static u64 notrace arm64_858921_read_cntpct_el0(void){	u64 old, new; 	old = read_sysreg(cntpct_el0);	new = read_sysreg(cntpct_el0);	return (((old ^ new) >> 32) & 1) ? old : new;} static u64 notrace arm64_858921_read_cntvct_el0(void){	u64 old, new; 	old = read_sysreg(cntvct_el0);	new = read_sysreg(cntvct_el0);	return (((old ^ new) >> 32) & 1) ? old : new;}#endif #ifdef CONFIG_SUN50I_ERRATUM_UNKNOWN1/* * The low bits of the counter registers are indeterminate while bit 10 or * greater is rolling over. Since the counter value can jump both backward * (7ff -> 000 -> 800) and forward (7ff -> fff -> 800), ignore register values * with all ones or all zeros in the low bits. Bound the loop by the maximum * number of CPU cycles in 3 consecutive 24 MHz counter periods. */#define __sun50i_a64_read_reg(reg) ({					\	u64 _val;							\	int _retries = 150;						\									\	do {								\		_val = read_sysreg(reg);				\		_retries--;						\	} while (((_val + 1) & GENMASK(9, 0)) <= 1 && _retries);	\									\	WARN_ON_ONCE(!_retries);					\	_val;								\}) static u64 notrace sun50i_a64_read_cntpct_el0(void){	return __sun50i_a64_read_reg(cntpct_el0);} static u64 notrace sun50i_a64_read_cntvct_el0(void){	return __sun50i_a64_read_reg(cntvct_el0);} static u32 notrace sun50i_a64_read_cntp_tval_el0(void){	return read_sysreg(cntp_cval_el0) - sun50i_a64_read_cntpct_el0();} static u32 notrace sun50i_a64_read_cntv_tval_el0(void){	return read_sysreg(cntv_cval_el0) - sun50i_a64_read_cntvct_el0();}#endif #ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUNDDEFINE_PER_CPU(const struct arch_timer_erratum_workaround *, timer_unstable_counter_workaround);EXPORT_SYMBOL_GPL(timer_unstable_counter_workaround); static atomic_t timer_unstable_counter_workaround_in_use = ATOMIC_INIT(0); static void erratum_set_next_event_tval_generic(const int access, unsigned long evt,						struct clock_event_device *clk){	unsigned long ctrl;	u64 cval; 	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);	ctrl |= ARCH_TIMER_CTRL_ENABLE;	ctrl &= ~ARCH_TIMER_CTRL_IT_MASK; 	if (access == ARCH_TIMER_PHYS_ACCESS) {		cval = evt + arch_counter_get_cntpct_stable();		write_sysreg(cval, cntp_cval_el0);	} else {		cval = evt + arch_counter_get_cntvct_stable();		write_sysreg(cval, cntv_cval_el0);	} 	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);} static __maybe_unused int erratum_set_next_event_tval_virt(unsigned long evt,					    struct clock_event_device *clk){	erratum_set_next_event_tval_generic(ARCH_TIMER_VIRT_ACCESS, evt, clk);	return 0;} static __maybe_unused int erratum_set_next_event_tval_phys(unsigned long evt,					    struct clock_event_device *clk){	erratum_set_next_event_tval_generic(ARCH_TIMER_PHYS_ACCESS, evt, clk);	return 0;} static const struct arch_timer_erratum_workaround ool_workarounds[] = {#ifdef CONFIG_FSL_ERRATUM_A008585	{		.match_type = ate_match_dt,		.id = "fsl,erratum-a008585",		.desc = "Freescale erratum a005858",		.read_cntp_tval_el0 = fsl_a008585_read_cntp_tval_el0,		.read_cntv_tval_el0 = fsl_a008585_read_cntv_tval_el0,		.read_cntpct_el0 = fsl_a008585_read_cntpct_el0,		.read_cntvct_el0 = fsl_a008585_read_cntvct_el0,		.set_next_event_phys = erratum_set_next_event_tval_phys,		.set_next_event_virt = erratum_set_next_event_tval_virt,	},#endif#ifdef CONFIG_HISILICON_ERRATUM_161010101	{		.match_type = ate_match_dt,		.id = "hisilicon,erratum-161010101",		.desc = "HiSilicon erratum 161010101",		.read_cntp_tval_el0 = hisi_161010101_read_cntp_tval_el0,		.read_cntv_tval_el0 = hisi_161010101_read_cntv_tval_el0,		.read_cntpct_el0 = hisi_161010101_read_cntpct_el0,		.read_cntvct_el0 = hisi_161010101_read_cntvct_el0,		.set_next_event_phys = erratum_set_next_event_tval_phys,		.set_next_event_virt = erratum_set_next_event_tval_virt,	},	{		.match_type = ate_match_acpi_oem_info,		.id = hisi_161010101_oem_info,		.desc = "HiSilicon erratum 161010101",		.read_cntp_tval_el0 = hisi_161010101_read_cntp_tval_el0,		.read_cntv_tval_el0 = hisi_161010101_read_cntv_tval_el0,		.read_cntpct_el0 = hisi_161010101_read_cntpct_el0,		.read_cntvct_el0 = hisi_161010101_read_cntvct_el0,		.set_next_event_phys = erratum_set_next_event_tval_phys,		.set_next_event_virt = erratum_set_next_event_tval_virt,	},#endif#ifdef CONFIG_ARM64_ERRATUM_858921	{		.match_type = ate_match_local_cap_id,		.id = (void *)ARM64_WORKAROUND_858921,		.desc = "ARM erratum 858921",		.read_cntpct_el0 = arm64_858921_read_cntpct_el0,		.read_cntvct_el0 = arm64_858921_read_cntvct_el0,	},#endif#ifdef CONFIG_SUN50I_ERRATUM_UNKNOWN1	{		.match_type = ate_match_dt,		.id = "allwinner,erratum-unknown1",		.desc = "Allwinner erratum UNKNOWN1",		.read_cntp_tval_el0 = sun50i_a64_read_cntp_tval_el0,		.read_cntv_tval_el0 = sun50i_a64_read_cntv_tval_el0,		.read_cntpct_el0 = sun50i_a64_read_cntpct_el0,		.read_cntvct_el0 = sun50i_a64_read_cntvct_el0,		.set_next_event_phys = erratum_set_next_event_tval_phys,		.set_next_event_virt = erratum_set_next_event_tval_virt,	},#endif#ifdef CONFIG_ARM64_ERRATUM_1418040	{		.match_type = ate_match_local_cap_id,		.id = (void *)ARM64_WORKAROUND_1418040,		.desc = "ARM erratum 1418040",

Lines 1–500 of 1714. Load more