1134 lines · 29.4 KB · C
RawDownload
/* *  linux/drivers/clocksource/arm_arch_timer.c * *  Copyright (C) 2011 ARM Ltd. *  All Rights Reserved * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. */ #define pr_fmt(fmt)	"arm_arch_timer: " fmt #include <linux/init.h>#include <linux/kernel.h>#include <linux/device.h>#include <linux/smp.h>#include <linux/cpu.h>#include <linux/cpu_pm.h>#include <linux/clockchips.h>#include <linux/clocksource.h>#include <linux/interrupt.h>#include <linux/of_irq.h>#include <linux/of_address.h>#include <linux/io.h>#include <linux/slab.h>#include <linux/sched/clock.h>#include <linux/sched_clock.h>#include <linux/acpi.h> #include <asm/arch_timer.h>#include <asm/virt.h> #include <clocksource/arm_arch_timer.h> #define CNTTIDR		0x08#define CNTTIDR_VIRT(n)	(BIT(1) << ((n) * 4)) #define CNTACR(n)	(0x40 + ((n) * 4))#define CNTACR_RPCT	BIT(0)#define CNTACR_RVCT	BIT(1)#define CNTACR_RFRQ	BIT(2)#define CNTACR_RVOFF	BIT(3)#define CNTACR_RWVT	BIT(4)#define CNTACR_RWPT	BIT(5) #define CNTVCT_LO	0x08#define CNTVCT_HI	0x0c#define CNTFRQ		0x10#define CNTP_TVAL	0x28#define CNTP_CTL	0x2c#define CNTV_TVAL	0x38#define CNTV_CTL	0x3c #define ARCH_CP15_TIMER	BIT(0)#define ARCH_MEM_TIMER	BIT(1)static unsigned arch_timers_present __initdata; static void __iomem *arch_counter_base; struct arch_timer {	void __iomem *base;	struct clock_event_device evt;}; #define to_arch_timer(e) container_of(e, struct arch_timer, evt) static u32 arch_timer_rate; enum ppi_nr {	PHYS_SECURE_PPI,	PHYS_NONSECURE_PPI,	VIRT_PPI,	HYP_PPI,	MAX_TIMER_PPI}; static int arch_timer_ppi[MAX_TIMER_PPI]; static struct clock_event_device __percpu *arch_timer_evt; static enum ppi_nr arch_timer_uses_ppi = VIRT_PPI;static bool arch_timer_c3stop;static bool arch_timer_mem_use_virtual;static bool arch_counter_suspend_stop; static bool evtstrm_enable = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM); static int __init early_evtstrm_cfg(char *buf){	return strtobool(buf, &evtstrm_enable);}early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg); /* * Architected system timer support. */ #ifdef CONFIG_FSL_ERRATUM_A008585/* * The number of retries is an arbitrary value well beyond the highest number * of iterations the loop has been observed to take. */#define __fsl_a008585_read_reg(reg) ({			\	u64 _old, _new;					\	int _retries = 200;				\							\	do {						\		_old = read_sysreg(reg);		\		_new = read_sysreg(reg);		\		_retries--;				\	} while (unlikely(_old != _new) && _retries);	\							\	WARN_ON_ONCE(!_retries);			\	_new;						\}) static u32 notrace fsl_a008585_read_cntp_tval_el0(void){	return __fsl_a008585_read_reg(cntp_tval_el0);} static u32 notrace fsl_a008585_read_cntv_tval_el0(void){	return __fsl_a008585_read_reg(cntv_tval_el0);} static u64 notrace fsl_a008585_read_cntvct_el0(void){	return __fsl_a008585_read_reg(cntvct_el0);}#endif #ifdef CONFIG_HISILICON_ERRATUM_161010101/* * Verify whether the value of the second read is larger than the first by * less than 32 is the only way to confirm the value is correct, so clear the * lower 5 bits to check whether the difference is greater than 32 or not. * Theoretically the erratum should not occur more than twice in succession * when reading the system counter, but it is possible that some interrupts * may lead to more than twice read errors, triggering the warning, so setting * the number of retries far beyond the number of iterations the loop has been * observed to take. */#define __hisi_161010101_read_reg(reg) ({				\	u64 _old, _new;						\	int _retries = 50;					\								\	do {							\		_old = read_sysreg(reg);			\		_new = read_sysreg(reg);			\		_retries--;					\	} while (unlikely((_new - _old) >> 5) && _retries);	\								\	WARN_ON_ONCE(!_retries);				\	_new;							\}) static u32 notrace hisi_161010101_read_cntp_tval_el0(void){	return __hisi_161010101_read_reg(cntp_tval_el0);} static u32 notrace hisi_161010101_read_cntv_tval_el0(void){	return __hisi_161010101_read_reg(cntv_tval_el0);} static u64 notrace hisi_161010101_read_cntvct_el0(void){	return __hisi_161010101_read_reg(cntvct_el0);}#endif #ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUNDconst struct arch_timer_erratum_workaround *timer_unstable_counter_workaround = NULL;EXPORT_SYMBOL_GPL(timer_unstable_counter_workaround); DEFINE_STATIC_KEY_FALSE(arch_timer_read_ool_enabled);EXPORT_SYMBOL_GPL(arch_timer_read_ool_enabled); static const struct arch_timer_erratum_workaround ool_workarounds[] = {#ifdef CONFIG_FSL_ERRATUM_A008585	{		.id = "fsl,erratum-a008585",		.read_cntp_tval_el0 = fsl_a008585_read_cntp_tval_el0,		.read_cntv_tval_el0 = fsl_a008585_read_cntv_tval_el0,		.read_cntvct_el0 = fsl_a008585_read_cntvct_el0,	},#endif#ifdef CONFIG_HISILICON_ERRATUM_161010101	{		.id = "hisilicon,erratum-161010101",		.read_cntp_tval_el0 = hisi_161010101_read_cntp_tval_el0,		.read_cntv_tval_el0 = hisi_161010101_read_cntv_tval_el0,		.read_cntvct_el0 = hisi_161010101_read_cntvct_el0,	},#endif};#endif /* CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND */ static __always_inlinevoid arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val,			  struct clock_event_device *clk){	if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {		struct arch_timer *timer = to_arch_timer(clk);		switch (reg) {		case ARCH_TIMER_REG_CTRL:			writel_relaxed(val, timer->base + CNTP_CTL);			break;		case ARCH_TIMER_REG_TVAL:			writel_relaxed(val, timer->base + CNTP_TVAL);			break;		}	} else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {		struct arch_timer *timer = to_arch_timer(clk);		switch (reg) {		case ARCH_TIMER_REG_CTRL:			writel_relaxed(val, timer->base + CNTV_CTL);			break;		case ARCH_TIMER_REG_TVAL:			writel_relaxed(val, timer->base + CNTV_TVAL);			break;		}	} else {		arch_timer_reg_write_cp15(access, reg, val);	}} static __always_inlineu32 arch_timer_reg_read(int access, enum arch_timer_reg reg,			struct clock_event_device *clk){	u32 val; 	if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {		struct arch_timer *timer = to_arch_timer(clk);		switch (reg) {		case ARCH_TIMER_REG_CTRL:			val = readl_relaxed(timer->base + CNTP_CTL);			break;		case ARCH_TIMER_REG_TVAL:			val = readl_relaxed(timer->base + CNTP_TVAL);			break;		}	} else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {		struct arch_timer *timer = to_arch_timer(clk);		switch (reg) {		case ARCH_TIMER_REG_CTRL:			val = readl_relaxed(timer->base + CNTV_CTL);			break;		case ARCH_TIMER_REG_TVAL:			val = readl_relaxed(timer->base + CNTV_TVAL);			break;		}	} else {		val = arch_timer_reg_read_cp15(access, reg);	} 	return val;} static __always_inline irqreturn_t timer_handler(const int access,					struct clock_event_device *evt){	unsigned long ctrl; 	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt);	if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {		ctrl |= ARCH_TIMER_CTRL_IT_MASK;		arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt);		evt->event_handler(evt);		return IRQ_HANDLED;	} 	return IRQ_NONE;} static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id){	struct clock_event_device *evt = dev_id; 	return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);} static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id){	struct clock_event_device *evt = dev_id; 	return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);} static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id){	struct clock_event_device *evt = dev_id; 	return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);} static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id){	struct clock_event_device *evt = dev_id; 	return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);} static __always_inline int timer_shutdown(const int access,					  struct clock_event_device *clk){	unsigned long ctrl; 	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);	ctrl &= ~ARCH_TIMER_CTRL_ENABLE;	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk); 	return 0;} static int arch_timer_shutdown_virt(struct clock_event_device *clk){	return timer_shutdown(ARCH_TIMER_VIRT_ACCESS, clk);} static int arch_timer_shutdown_phys(struct clock_event_device *clk){	return timer_shutdown(ARCH_TIMER_PHYS_ACCESS, clk);} static int arch_timer_shutdown_virt_mem(struct clock_event_device *clk){	return timer_shutdown(ARCH_TIMER_MEM_VIRT_ACCESS, clk);} static int arch_timer_shutdown_phys_mem(struct clock_event_device *clk){	return timer_shutdown(ARCH_TIMER_MEM_PHYS_ACCESS, clk);} static __always_inline void set_next_event(const int access, unsigned long evt,					   struct clock_event_device *clk){	unsigned long ctrl;	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);	ctrl |= ARCH_TIMER_CTRL_ENABLE;	ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;	arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt, clk);	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);} #ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUNDstatic __always_inline void erratum_set_next_event_generic(const int access,		unsigned long evt, struct clock_event_device *clk){	unsigned long ctrl;	u64 cval = evt + arch_counter_get_cntvct(); 	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);	ctrl |= ARCH_TIMER_CTRL_ENABLE;	ctrl &= ~ARCH_TIMER_CTRL_IT_MASK; 	if (access == ARCH_TIMER_PHYS_ACCESS)		write_sysreg(cval, cntp_cval_el0);	else if (access == ARCH_TIMER_VIRT_ACCESS)		write_sysreg(cval, cntv_cval_el0); 	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);} static int erratum_set_next_event_virt(unsigned long evt,					   struct clock_event_device *clk){	erratum_set_next_event_generic(ARCH_TIMER_VIRT_ACCESS, evt, clk);	return 0;} static int erratum_set_next_event_phys(unsigned long evt,					   struct clock_event_device *clk){	erratum_set_next_event_generic(ARCH_TIMER_PHYS_ACCESS, evt, clk);	return 0;}#endif /* CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND */ static int arch_timer_set_next_event_virt(unsigned long evt,					  struct clock_event_device *clk){	set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);	return 0;} static int arch_timer_set_next_event_phys(unsigned long evt,					  struct clock_event_device *clk){	set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);	return 0;} static int arch_timer_set_next_event_virt_mem(unsigned long evt,					      struct clock_event_device *clk){	set_next_event(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk);	return 0;} static int arch_timer_set_next_event_phys_mem(unsigned long evt,					      struct clock_event_device *clk){	set_next_event(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk);	return 0;} static void erratum_workaround_set_sne(struct clock_event_device *clk){#ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND	if (!static_branch_unlikely(&arch_timer_read_ool_enabled))		return; 	if (arch_timer_uses_ppi == VIRT_PPI)		clk->set_next_event = erratum_set_next_event_virt;	else		clk->set_next_event = erratum_set_next_event_phys;#endif} static void __arch_timer_setup(unsigned type,			       struct clock_event_device *clk){	clk->features = CLOCK_EVT_FEAT_ONESHOT; 	if (type == ARCH_CP15_TIMER) {		if (arch_timer_c3stop)			clk->features |= CLOCK_EVT_FEAT_C3STOP;		clk->name = "arch_sys_timer";		clk->rating = 450;		clk->cpumask = cpumask_of(smp_processor_id());		clk->irq = arch_timer_ppi[arch_timer_uses_ppi];		switch (arch_timer_uses_ppi) {		case VIRT_PPI:			clk->set_state_shutdown = arch_timer_shutdown_virt;			clk->set_state_oneshot_stopped = arch_timer_shutdown_virt;			clk->set_next_event = arch_timer_set_next_event_virt;			break;		case PHYS_SECURE_PPI:		case PHYS_NONSECURE_PPI:		case HYP_PPI:			clk->set_state_shutdown = arch_timer_shutdown_phys;			clk->set_state_oneshot_stopped = arch_timer_shutdown_phys;			clk->set_next_event = arch_timer_set_next_event_phys;			break;		default:			BUG();		} 		erratum_workaround_set_sne(clk);	} else {		clk->features |= CLOCK_EVT_FEAT_DYNIRQ;		clk->name = "arch_mem_timer";		clk->rating = 400;		clk->cpumask = cpu_all_mask;		if (arch_timer_mem_use_virtual) {			clk->set_state_shutdown = arch_timer_shutdown_virt_mem;			clk->set_state_oneshot_stopped = arch_timer_shutdown_virt_mem;			clk->set_next_event =				arch_timer_set_next_event_virt_mem;		} else {			clk->set_state_shutdown = arch_timer_shutdown_phys_mem;			clk->set_state_oneshot_stopped = arch_timer_shutdown_phys_mem;			clk->set_next_event =				arch_timer_set_next_event_phys_mem;		}	} 	clk->set_state_shutdown(clk); 	clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff);} static void arch_timer_evtstrm_enable(int divider){	u32 cntkctl = arch_timer_get_cntkctl(); 	cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK;	/* Set the divider and enable virtual event stream */	cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT)			| ARCH_TIMER_VIRT_EVT_EN;	arch_timer_set_cntkctl(cntkctl);	elf_hwcap |= HWCAP_EVTSTRM;#ifdef CONFIG_COMPAT	compat_elf_hwcap |= COMPAT_HWCAP_EVTSTRM;#endif} static void arch_timer_configure_evtstream(void){	int evt_stream_div, pos; 	/* Find the closest power of two to the divisor */	evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ;	pos = fls(evt_stream_div);

Lines 1–500 of 1134.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t