/* *  linux/drivers/clocksource/arm_arch_timer.c * *  Copyright (C) 2011 ARM Ltd. *  All Rights Reserved * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. */ #define pr_fmt(fmt)	"arm_arch_timer: " fmt #include <linux/init.h>#include <linux/kernel.h>#include <linux/device.h>#include <linux/smp.h>#include <linux/cpu.h>#include <linux/cpu_pm.h>#include <linux/clockchips.h>#include <linux/clocksource.h>#include <linux/interrupt.h>#include <linux/of_irq.h>#include <linux/of_address.h>#include <linux/io.h>#include <linux/slab.h>#include <linux/sched/clock.h>#include <linux/sched_clock.h>#include <linux/acpi.h> #include <asm/arch_timer.h>#include <asm/virt.h> #include <clocksource/arm_arch_timer.h> #undef pr_fmt#define pr_fmt(fmt) "arch_timer: " fmt #define CNTTIDR		0x08#define CNTTIDR_VIRT(n)	(BIT(1) << ((n) * 4)) #define CNTACR(n)	(0x40 + ((n) * 4))#define CNTACR_RPCT	BIT(0)#define CNTACR_RVCT	BIT(1)#define CNTACR_RFRQ	BIT(2)#define CNTACR_RVOFF	BIT(3)#define CNTACR_RWVT	BIT(4)#define CNTACR_RWPT	BIT(5) #define CNTVCT_LO	0x08#define CNTVCT_HI	0x0c#define CNTFRQ		0x10#define CNTP_TVAL	0x28#define CNTP_CTL	0x2c#define CNTV_TVAL	0x38#define CNTV_CTL	0x3c static unsigned arch_timers_present __initdata; static void __iomem *arch_counter_base; struct arch_timer {	void __iomem *base;	struct clock_event_device evt;}; #define to_arch_timer(e) container_of(e, struct arch_timer, evt) static u32 arch_timer_rate;static int arch_timer_ppi[ARCH_TIMER_MAX_TIMER_PPI]; static struct clock_event_device __percpu *arch_timer_evt; static enum arch_timer_ppi_nr arch_timer_uses_ppi = ARCH_TIMER_VIRT_PPI;static bool arch_timer_c3stop;static bool arch_timer_mem_use_virtual;static bool arch_counter_suspend_stop;static bool vdso_default = true; static bool evtstrm_enable = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM); static int __init early_evtstrm_cfg(char *buf){	return strtobool(buf, &evtstrm_enable);}early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg); /* * Architected system timer support. */ static __always_inlinevoid arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val,			  struct clock_event_device *clk){	if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {		struct arch_timer *timer = to_arch_timer(clk);		switch (reg) {		case ARCH_TIMER_REG_CTRL:			writel_relaxed(val, timer->base + CNTP_CTL);			break;		case ARCH_TIMER_REG_TVAL:			writel_relaxed(val, timer->base + CNTP_TVAL);			break;		}	} else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {		struct arch_timer *timer = to_arch_timer(clk);		switch (reg) {		case ARCH_TIMER_REG_CTRL:			writel_relaxed(val, timer->base + CNTV_CTL);			break;		case ARCH_TIMER_REG_TVAL:			writel_relaxed(val, timer->base + CNTV_TVAL);			break;		}	} else {		arch_timer_reg_write_cp15(access, reg, val);	}} static __always_inlineu32 arch_timer_reg_read(int access, enum arch_timer_reg reg,			struct clock_event_device *clk){	u32 val; 	if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {		struct arch_timer *timer = to_arch_timer(clk);		switch (reg) {		case ARCH_TIMER_REG_CTRL:			val = readl_relaxed(timer->base + CNTP_CTL);			break;		case ARCH_TIMER_REG_TVAL:			val = readl_relaxed(timer->base + CNTP_TVAL);			break;		}	} else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {		struct arch_timer *timer = to_arch_timer(clk);		switch (reg) {		case ARCH_TIMER_REG_CTRL:			val = readl_relaxed(timer->base + CNTV_CTL);			break;		case ARCH_TIMER_REG_TVAL:			val = readl_relaxed(timer->base + CNTV_TVAL);			break;		}	} else {		val = arch_timer_reg_read_cp15(access, reg);	} 	return val;} /* * Default to cp15 based access because arm64 uses this function for * sched_clock() before DT is probed and the cp15 method is guaranteed * to exist on arm64. arm doesn't use this before DT is probed so even * if we don't have the cp15 accessors we won't have a problem. */u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct; static u64 arch_counter_read(struct clocksource *cs){	return arch_timer_read_counter();} static u64 arch_counter_read_cc(const struct cyclecounter *cc){	return arch_timer_read_counter();} static struct clocksource clocksource_counter = {	.name	= "arch_sys_counter",	.rating	= 400,	.read	= arch_counter_read,	.mask	= CLOCKSOURCE_MASK(56),	.flags	= CLOCK_SOURCE_IS_CONTINUOUS,}; static struct cyclecounter cyclecounter __ro_after_init = {	.read	= arch_counter_read_cc,	.mask	= CLOCKSOURCE_MASK(56),}; struct ate_acpi_oem_info {	char oem_id[ACPI_OEM_ID_SIZE + 1];	char oem_table_id[ACPI_OEM_TABLE_ID_SIZE + 1];	u32 oem_revision;}; #ifdef CONFIG_FSL_ERRATUM_A008585/* * The number of retries is an arbitrary value well beyond the highest number * of iterations the loop has been observed to take. */#define __fsl_a008585_read_reg(reg) ({			\	u64 _old, _new;					\	int _retries = 200;				\							\	do {						\		_old = read_sysreg(reg);		\		_new = read_sysreg(reg);		\		_retries--;				\	} while (unlikely(_old != _new) && _retries);	\							\	WARN_ON_ONCE(!_retries);			\	_new;						\}) static u32 notrace fsl_a008585_read_cntp_tval_el0(void){	return __fsl_a008585_read_reg(cntp_tval_el0);} static u32 notrace fsl_a008585_read_cntv_tval_el0(void){	return __fsl_a008585_read_reg(cntv_tval_el0);} static u64 notrace fsl_a008585_read_cntvct_el0(void){	return __fsl_a008585_read_reg(cntvct_el0);}#endif #ifdef CONFIG_HISILICON_ERRATUM_161010101/* * Verify whether the value of the second read is larger than the first by * less than 32 is the only way to confirm the value is correct, so clear the * lower 5 bits to check whether the difference is greater than 32 or not. * Theoretically the erratum should not occur more than twice in succession * when reading the system counter, but it is possible that some interrupts * may lead to more than twice read errors, triggering the warning, so setting * the number of retries far beyond the number of iterations the loop has been * observed to take. */#define __hisi_161010101_read_reg(reg) ({				\	u64 _old, _new;						\	int _retries = 50;					\								\	do {							\		_old = read_sysreg(reg);			\		_new = read_sysreg(reg);			\		_retries--;					\	} while (unlikely((_new - _old) >> 5) && _retries);	\								\	WARN_ON_ONCE(!_retries);				\	_new;							\}) static u32 notrace hisi_161010101_read_cntp_tval_el0(void){	return __hisi_161010101_read_reg(cntp_tval_el0);} static u32 notrace hisi_161010101_read_cntv_tval_el0(void){	return __hisi_161010101_read_reg(cntv_tval_el0);} static u64 notrace hisi_161010101_read_cntvct_el0(void){	return __hisi_161010101_read_reg(cntvct_el0);} static struct ate_acpi_oem_info hisi_161010101_oem_info[] = {	/*	 * Note that trailing spaces are required to properly match	 * the OEM table information.	 */	{		.oem_id		= "HISI  ",		.oem_table_id	= "HIP05   ",		.oem_revision	= 0,	},	{		.oem_id		= "HISI  ",		.oem_table_id	= "HIP06   ",		.oem_revision	= 0,	},	{		.oem_id		= "HISI  ",		.oem_table_id	= "HIP07   ",		.oem_revision	= 0,	},	{ /* Sentinel indicating the end of the OEM array */ },};#endif #ifdef CONFIG_ARM64_ERRATUM_858921static u64 notrace arm64_858921_read_cntvct_el0(void){	u64 old, new; 	old = read_sysreg(cntvct_el0);	new = read_sysreg(cntvct_el0);	return (((old ^ new) >> 32) & 1) ? old : new;}#endif #ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUNDDEFINE_PER_CPU(const struct arch_timer_erratum_workaround *,	       timer_unstable_counter_workaround);EXPORT_SYMBOL_GPL(timer_unstable_counter_workaround); DEFINE_STATIC_KEY_FALSE(arch_timer_read_ool_enabled);EXPORT_SYMBOL_GPL(arch_timer_read_ool_enabled); static void erratum_set_next_event_tval_generic(const int access, unsigned long evt,						struct clock_event_device *clk){	unsigned long ctrl;	u64 cval = evt + arch_counter_get_cntvct(); 	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);	ctrl |= ARCH_TIMER_CTRL_ENABLE;	ctrl &= ~ARCH_TIMER_CTRL_IT_MASK; 	if (access == ARCH_TIMER_PHYS_ACCESS)		write_sysreg(cval, cntp_cval_el0);	else		write_sysreg(cval, cntv_cval_el0); 	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);} static __maybe_unused int erratum_set_next_event_tval_virt(unsigned long evt,					    struct clock_event_device *clk){	erratum_set_next_event_tval_generic(ARCH_TIMER_VIRT_ACCESS, evt, clk);	return 0;} static __maybe_unused int erratum_set_next_event_tval_phys(unsigned long evt,					    struct clock_event_device *clk){	erratum_set_next_event_tval_generic(ARCH_TIMER_PHYS_ACCESS, evt, clk);	return 0;} static const struct arch_timer_erratum_workaround ool_workarounds[] = {#ifdef CONFIG_FSL_ERRATUM_A008585	{		.match_type = ate_match_dt,		.id = "fsl,erratum-a008585",		.desc = "Freescale erratum a005858",		.read_cntp_tval_el0 = fsl_a008585_read_cntp_tval_el0,		.read_cntv_tval_el0 = fsl_a008585_read_cntv_tval_el0,		.read_cntvct_el0 = fsl_a008585_read_cntvct_el0,		.set_next_event_phys = erratum_set_next_event_tval_phys,		.set_next_event_virt = erratum_set_next_event_tval_virt,	},#endif#ifdef CONFIG_HISILICON_ERRATUM_161010101	{		.match_type = ate_match_dt,		.id = "hisilicon,erratum-161010101",		.desc = "HiSilicon erratum 161010101",		.read_cntp_tval_el0 = hisi_161010101_read_cntp_tval_el0,		.read_cntv_tval_el0 = hisi_161010101_read_cntv_tval_el0,		.read_cntvct_el0 = hisi_161010101_read_cntvct_el0,		.set_next_event_phys = erratum_set_next_event_tval_phys,		.set_next_event_virt = erratum_set_next_event_tval_virt,	},	{		.match_type = ate_match_acpi_oem_info,		.id = hisi_161010101_oem_info,		.desc = "HiSilicon erratum 161010101",		.read_cntp_tval_el0 = hisi_161010101_read_cntp_tval_el0,		.read_cntv_tval_el0 = hisi_161010101_read_cntv_tval_el0,		.read_cntvct_el0 = hisi_161010101_read_cntvct_el0,		.set_next_event_phys = erratum_set_next_event_tval_phys,		.set_next_event_virt = erratum_set_next_event_tval_virt,	},#endif#ifdef CONFIG_ARM64_ERRATUM_858921	{		.match_type = ate_match_local_cap_id,		.id = (void *)ARM64_WORKAROUND_858921,		.desc = "ARM erratum 858921",		.read_cntvct_el0 = arm64_858921_read_cntvct_el0,	},#endif}; typedef bool (*ate_match_fn_t)(const struct arch_timer_erratum_workaround *,			       const void *); staticbool arch_timer_check_dt_erratum(const struct arch_timer_erratum_workaround *wa,				 const void *arg){	const struct device_node *np = arg; 	return of_property_read_bool(np, wa->id);} staticbool arch_timer_check_local_cap_erratum(const struct arch_timer_erratum_workaround *wa,					const void *arg){	return this_cpu_has_cap((uintptr_t)wa->id);}  staticbool arch_timer_check_acpi_oem_erratum(const struct arch_timer_erratum_workaround *wa,				       const void *arg){	static const struct ate_acpi_oem_info empty_oem_info = {};	const struct ate_acpi_oem_info *info = wa->id;	const struct acpi_table_header *table = arg; 	/* Iterate over the ACPI OEM info array, looking for a match */	while (memcmp(info, &empty_oem_info, sizeof(*info))) {		if (!memcmp(info->oem_id, table->oem_id, ACPI_OEM_ID_SIZE) &&		    !memcmp(info->oem_table_id, table->oem_table_id, ACPI_OEM_TABLE_ID_SIZE) &&		    info->oem_revision == table->oem_revision)			return true; 		info++;	} 	return false;} static const struct arch_timer_erratum_workaround *arch_timer_iterate_errata(enum arch_timer_erratum_match_type type,			  ate_match_fn_t match_fn,			  void *arg){	int i; 	for (i = 0; i < ARRAY_SIZE(ool_workarounds); i++) {		if (ool_workarounds[i].match_type != type)			continue; 		if (match_fn(&ool_workarounds[i], arg))			return &ool_workarounds[i];	} 	return NULL;} staticvoid arch_timer_enable_workaround(const struct arch_timer_erratum_workaround *wa,				  bool local){	int i; 	if (local) {		__this_cpu_write(timer_unstable_counter_workaround, wa);	} else {		for_each_possible_cpu(i)			per_cpu(timer_unstable_counter_workaround, i) = wa;	} 	static_branch_enable(&arch_timer_read_ool_enabled); 	/*	 * Don't use the vdso fastpath if errata require using the	 * out-of-line counter accessor. We may change our mind pretty	 * late in the game (with a per-CPU erratum, for example), so	 * change both the default value and the vdso itself.	 */	if (wa->read_cntvct_el0) {		clocksource_counter.archdata.vdso_direct = false;		vdso_default = false;	}} static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type type,					    void *arg){	const struct arch_timer_erratum_workaround *wa;	ate_match_fn_t match_fn = NULL;	bool local = false; 	switch (type) {	case ate_match_dt:		match_fn = arch_timer_check_dt_erratum;		break;	case ate_match_local_cap_id:		match_fn = arch_timer_check_local_cap_erratum;		local = true;		break;	case ate_match_acpi_oem_info:		match_fn = arch_timer_check_acpi_oem_erratum;		break;	default:		WARN_ON(1);		return;	} 	wa = arch_timer_iterate_errata(type, match_fn, arg);	if (!wa)		return; 	if (needs_unstable_timer_counter_workaround()) {		const struct arch_timer_erratum_workaround *__wa;

Lines 1–500 of 1520. Load more