/* * linux/drivers/clocksource/arm_arch_timer.c * * Copyright (C) 2011 ARM Ltd. * All Rights Reserved * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. */ #define pr_fmt(fmt) "arm_arch_timer: " fmt #include <linux/init.h>#include <linux/kernel.h>#include <linux/device.h>#include <linux/smp.h>#include <linux/cpu.h>#include <linux/cpu_pm.h>#include <linux/clockchips.h>#include <linux/clocksource.h>#include <linux/interrupt.h>#include <linux/of_irq.h>#include <linux/of_address.h>#include <linux/io.h>#include <linux/slab.h>#include <linux/sched_clock.h>#include <linux/acpi.h> #include <asm/arch_timer.h>#include <asm/virt.h> #include <clocksource/arm_arch_timer.h> #define CNTTIDR 0x08#define CNTTIDR_VIRT(n) (BIT(1) << ((n) * 4)) #define CNTACR(n) (0x40 + ((n) * 4))#define CNTACR_RPCT BIT(0)#define CNTACR_RVCT BIT(1)#define CNTACR_RFRQ BIT(2)#define CNTACR_RVOFF BIT(3)#define CNTACR_RWVT BIT(4)#define CNTACR_RWPT BIT(5) #define CNTVCT_LO 0x08#define CNTVCT_HI 0x0c#define CNTFRQ 0x10#define CNTP_TVAL 0x28#define CNTP_CTL 0x2c#define CNTV_TVAL 0x38#define CNTV_CTL 0x3c #define ARCH_CP15_TIMER BIT(0)#define ARCH_MEM_TIMER BIT(1)static unsigned arch_timers_present __initdata; static void __iomem *arch_counter_base; struct arch_timer { void __iomem *base; struct clock_event_device evt;}; #define to_arch_timer(e) container_of(e, struct arch_timer, evt) static u32 arch_timer_rate; enum ppi_nr { PHYS_SECURE_PPI, PHYS_NONSECURE_PPI, VIRT_PPI, HYP_PPI, MAX_TIMER_PPI}; static int arch_timer_ppi[MAX_TIMER_PPI]; static struct clock_event_device __percpu *arch_timer_evt; static enum ppi_nr arch_timer_uses_ppi = VIRT_PPI;static bool arch_timer_c3stop;static bool arch_timer_mem_use_virtual; static bool evtstrm_enable = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM); static int __init early_evtstrm_cfg(char *buf){ return strtobool(buf, &evtstrm_enable);}early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg); /* * Architected system timer support. */ #ifdef CONFIG_FSL_ERRATUM_A008585DEFINE_STATIC_KEY_FALSE(arch_timer_read_ool_enabled);EXPORT_SYMBOL_GPL(arch_timer_read_ool_enabled); static int fsl_a008585_enable = -1; static int __init early_fsl_a008585_cfg(char *buf){ int ret; bool val; ret = strtobool(buf, &val); if (ret) return ret; fsl_a008585_enable = val; return 0;}early_param("clocksource.arm_arch_timer.fsl-a008585", early_fsl_a008585_cfg); u32 __fsl_a008585_read_cntp_tval_el0(void){ return __fsl_a008585_read_reg(cntp_tval_el0);} u32 __fsl_a008585_read_cntv_tval_el0(void){ return __fsl_a008585_read_reg(cntv_tval_el0);} u64 __fsl_a008585_read_cntvct_el0(void){ return __fsl_a008585_read_reg(cntvct_el0);}EXPORT_SYMBOL(__fsl_a008585_read_cntvct_el0);#endif /* CONFIG_FSL_ERRATUM_A008585 */ static __always_inlinevoid arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val, struct clock_event_device *clk){ if (access == ARCH_TIMER_MEM_PHYS_ACCESS) { struct arch_timer *timer = to_arch_timer(clk); switch (reg) { case ARCH_TIMER_REG_CTRL: writel_relaxed(val, timer->base + CNTP_CTL); break; case ARCH_TIMER_REG_TVAL: writel_relaxed(val, timer->base + CNTP_TVAL); break; } } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) { struct arch_timer *timer = to_arch_timer(clk); switch (reg) { case ARCH_TIMER_REG_CTRL: writel_relaxed(val, timer->base + CNTV_CTL); break; case ARCH_TIMER_REG_TVAL: writel_relaxed(val, timer->base + CNTV_TVAL); break; } } else { arch_timer_reg_write_cp15(access, reg, val); }} static __always_inlineu32 arch_timer_reg_read(int access, enum arch_timer_reg reg, struct clock_event_device *clk){ u32 val; if (access == ARCH_TIMER_MEM_PHYS_ACCESS) { struct arch_timer *timer = to_arch_timer(clk); switch (reg) { case ARCH_TIMER_REG_CTRL: val = readl_relaxed(timer->base + CNTP_CTL); break; case ARCH_TIMER_REG_TVAL: val = readl_relaxed(timer->base + CNTP_TVAL); break; } } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) { struct arch_timer *timer = to_arch_timer(clk); switch (reg) { case ARCH_TIMER_REG_CTRL: val = readl_relaxed(timer->base + CNTV_CTL); break; case ARCH_TIMER_REG_TVAL: val = readl_relaxed(timer->base + CNTV_TVAL); break; } } else { val = arch_timer_reg_read_cp15(access, reg); } return val;} static __always_inline irqreturn_t timer_handler(const int access, struct clock_event_device *evt){ unsigned long ctrl; ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt); if (ctrl & ARCH_TIMER_CTRL_IT_STAT) { ctrl |= ARCH_TIMER_CTRL_IT_MASK; arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt); evt->event_handler(evt); return IRQ_HANDLED; } return IRQ_NONE;} static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id){ struct clock_event_device *evt = dev_id; return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);} static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id){ struct clock_event_device *evt = dev_id; return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);} static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id){ struct clock_event_device *evt = dev_id; return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);} static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id){ struct clock_event_device *evt = dev_id; return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);} static __always_inline int timer_shutdown(const int access, struct clock_event_device *clk){ unsigned long ctrl; ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk); ctrl &= ~ARCH_TIMER_CTRL_ENABLE; arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk); return 0;} static int arch_timer_shutdown_virt(struct clock_event_device *clk){ return timer_shutdown(ARCH_TIMER_VIRT_ACCESS, clk);} static int arch_timer_shutdown_phys(struct clock_event_device *clk){ return timer_shutdown(ARCH_TIMER_PHYS_ACCESS, clk);} static int arch_timer_shutdown_virt_mem(struct clock_event_device *clk){ return timer_shutdown(ARCH_TIMER_MEM_VIRT_ACCESS, clk);} static int arch_timer_shutdown_phys_mem(struct clock_event_device *clk){ return timer_shutdown(ARCH_TIMER_MEM_PHYS_ACCESS, clk);} static __always_inline void set_next_event(const int access, unsigned long evt, struct clock_event_device *clk){ unsigned long ctrl; ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk); ctrl |= ARCH_TIMER_CTRL_ENABLE; ctrl &= ~ARCH_TIMER_CTRL_IT_MASK; arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt, clk); arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);} #ifdef CONFIG_FSL_ERRATUM_A008585static __always_inline void fsl_a008585_set_next_event(const int access, unsigned long evt, struct clock_event_device *clk){ unsigned long ctrl; u64 cval = evt + arch_counter_get_cntvct(); ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk); ctrl |= ARCH_TIMER_CTRL_ENABLE; ctrl &= ~ARCH_TIMER_CTRL_IT_MASK; if (access == ARCH_TIMER_PHYS_ACCESS) write_sysreg(cval, cntp_cval_el0); else if (access == ARCH_TIMER_VIRT_ACCESS) write_sysreg(cval, cntv_cval_el0); arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);} static int fsl_a008585_set_next_event_virt(unsigned long evt, struct clock_event_device *clk){ fsl_a008585_set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk); return 0;} static int fsl_a008585_set_next_event_phys(unsigned long evt, struct clock_event_device *clk){ fsl_a008585_set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk); return 0;}#endif /* CONFIG_FSL_ERRATUM_A008585 */ static int arch_timer_set_next_event_virt(unsigned long evt, struct clock_event_device *clk){ set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk); return 0;} static int arch_timer_set_next_event_phys(unsigned long evt, struct clock_event_device *clk){ set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk); return 0;} static int arch_timer_set_next_event_virt_mem(unsigned long evt, struct clock_event_device *clk){ set_next_event(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk); return 0;} static int arch_timer_set_next_event_phys_mem(unsigned long evt, struct clock_event_device *clk){ set_next_event(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk); return 0;} static void fsl_a008585_set_sne(struct clock_event_device *clk){#ifdef CONFIG_FSL_ERRATUM_A008585 if (!static_branch_unlikely(&arch_timer_read_ool_enabled)) return; if (arch_timer_uses_ppi == VIRT_PPI) clk->set_next_event = fsl_a008585_set_next_event_virt; else clk->set_next_event = fsl_a008585_set_next_event_phys;#endif} static void __arch_timer_setup(unsigned type, struct clock_event_device *clk){ clk->features = CLOCK_EVT_FEAT_ONESHOT; if (type == ARCH_CP15_TIMER) { if (arch_timer_c3stop) clk->features |= CLOCK_EVT_FEAT_C3STOP; clk->name = "arch_sys_timer"; clk->rating = 450; clk->cpumask = cpumask_of(smp_processor_id()); clk->irq = arch_timer_ppi[arch_timer_uses_ppi]; switch (arch_timer_uses_ppi) { case VIRT_PPI: clk->set_state_shutdown = arch_timer_shutdown_virt; clk->set_state_oneshot_stopped = arch_timer_shutdown_virt; clk->set_next_event = arch_timer_set_next_event_virt; break; case PHYS_SECURE_PPI: case PHYS_NONSECURE_PPI: case HYP_PPI: clk->set_state_shutdown = arch_timer_shutdown_phys; clk->set_state_oneshot_stopped = arch_timer_shutdown_phys; clk->set_next_event = arch_timer_set_next_event_phys; break; default: BUG(); } fsl_a008585_set_sne(clk); } else { clk->features |= CLOCK_EVT_FEAT_DYNIRQ; clk->name = "arch_mem_timer"; clk->rating = 400; clk->cpumask = cpu_all_mask; if (arch_timer_mem_use_virtual) { clk->set_state_shutdown = arch_timer_shutdown_virt_mem; clk->set_state_oneshot_stopped = arch_timer_shutdown_virt_mem; clk->set_next_event = arch_timer_set_next_event_virt_mem; } else { clk->set_state_shutdown = arch_timer_shutdown_phys_mem; clk->set_state_oneshot_stopped = arch_timer_shutdown_phys_mem; clk->set_next_event = arch_timer_set_next_event_phys_mem; } } clk->set_state_shutdown(clk); clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff);} static void arch_timer_evtstrm_enable(int divider){ u32 cntkctl = arch_timer_get_cntkctl(); cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK; /* Set the divider and enable virtual event stream */ cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT) | ARCH_TIMER_VIRT_EVT_EN; arch_timer_set_cntkctl(cntkctl); elf_hwcap |= HWCAP_EVTSTRM;#ifdef CONFIG_COMPAT compat_elf_hwcap |= COMPAT_HWCAP_EVTSTRM;#endif} static void arch_timer_configure_evtstream(void){ int evt_stream_div, pos; /* Find the closest power of two to the divisor */ evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ; pos = fls(evt_stream_div); if (pos > 1 && !(evt_stream_div & (1 << (pos - 2)))) pos--; /* enable event stream */ arch_timer_evtstrm_enable(min(pos, 15));} static void arch_counter_set_user_access(void){ u32 cntkctl = arch_timer_get_cntkctl(); /* Disable user access to the timers and the physical counter */ /* Also disable virtual event stream */ cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN | ARCH_TIMER_USR_VT_ACCESS_EN | ARCH_TIMER_VIRT_EVT_EN | ARCH_TIMER_USR_PCT_ACCESS_EN); /* Enable user access to the virtual counter */ cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN; arch_timer_set_cntkctl(cntkctl);} static bool arch_timer_has_nonsecure_ppi(void){ return (arch_timer_uses_ppi == PHYS_SECURE_PPI && arch_timer_ppi[PHYS_NONSECURE_PPI]);} static u32 check_ppi_trigger(int irq){ u32 flags = irq_get_trigger_type(irq); if (flags != IRQF_TRIGGER_HIGH && flags != IRQF_TRIGGER_LOW) { pr_warn("WARNING: Invalid trigger for IRQ%d, assuming level low\n", irq); pr_warn("WARNING: Please fix your firmware\n"); flags = IRQF_TRIGGER_LOW; } return flags;} static int arch_timer_starting_cpu(unsigned int cpu){ struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt); u32 flags; __arch_timer_setup(ARCH_CP15_TIMER, clk); flags = check_ppi_trigger(arch_timer_ppi[arch_timer_uses_ppi]); enable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], flags); if (arch_timer_has_nonsecure_ppi()) { flags = check_ppi_trigger(arch_timer_ppi[PHYS_NONSECURE_PPI]); enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], flags); } arch_counter_set_user_access(); if (evtstrm_enable) arch_timer_configure_evtstream(); return 0;} static voidarch_timer_detect_rate(void __iomem *cntbase, struct device_node *np){ /* Who has more than one independent system counter? */Lines 1–500 of 1056. Load more
/* * linux/drivers/clocksource/arm_arch_timer.c * * Copyright (C) 2011 ARM Ltd. * All Rights Reserved * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. */ #define pr_fmt(fmt) "arm_arch_timer: " fmt #include <linux/init.h> #include <linux/kernel.h> #include <linux/device.h> #include <linux/smp.h> #include <linux/cpu.h> #include <linux/cpu_pm.h> #include <linux/clockchips.h> #include <linux/clocksource.h> #include <linux/interrupt.h> #include <linux/of_irq.h> #include <linux/of_address.h> #include <linux/io.h> #include <linux/slab.h> #include <linux/sched_clock.h> #include <linux/acpi.h> #include <asm/arch_timer.h> #include <asm/virt.h> #include <clocksource/arm_arch_timer.h> #define CNTTIDR 0x08 #define CNTTIDR_VIRT(n) (BIT(1) << ((n) * 4)) #define CNTACR(n) (0x40 + ((n) * 4)) #define CNTACR_RPCT BIT(0) #define CNTACR_RVCT BIT(1) #define CNTACR_RFRQ BIT(2) #define CNTACR_RVOFF BIT(3) #define CNTACR_RWVT BIT(4) #define CNTACR_RWPT BIT(5) #define CNTVCT_LO 0x08 #define CNTVCT_HI 0x0c #define CNTFRQ 0x10 #define CNTP_TVAL 0x28 #define CNTP_CTL 0x2c #define CNTV_TVAL 0x38 #define CNTV_CTL 0x3c #define ARCH_CP15_TIMER BIT(0) #define ARCH_MEM_TIMER BIT(1) static unsigned arch_timers_present __initdata; static void __iomem *arch_counter_base; struct arch_timer { void __iomem *base; struct clock_event_device evt; }; #define to_arch_timer(e) container_of(e, struct arch_timer, evt) static u32 arch_timer_rate; enum ppi_nr { PHYS_SECURE_PPI, PHYS_NONSECURE_PPI, VIRT_PPI, HYP_PPI, MAX_TIMER_PPI }; static int arch_timer_ppi[MAX_TIMER_PPI]; static struct clock_event_device __percpu *arch_timer_evt; static enum ppi_nr arch_timer_uses_ppi = VIRT_PPI; static bool arch_timer_c3stop; static bool arch_timer_mem_use_virtual; static bool evtstrm_enable = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM); static int __init early_evtstrm_cfg(char *buf) { return strtobool(buf, &evtstrm_enable); } early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg); /* * Architected system timer support. */ #ifdef CONFIG_FSL_ERRATUM_A008585 DEFINE_STATIC_KEY_FALSE(arch_timer_read_ool_enabled); EXPORT_SYMBOL_GPL(arch_timer_read_ool_enabled); static int fsl_a008585_enable = -1; static int __init early_fsl_a008585_cfg(char *buf) { int ret; bool val; ret = strtobool(buf, &val); if (ret) return ret; fsl_a008585_enable = val; return 0; } early_param("clocksource.arm_arch_timer.fsl-a008585", early_fsl_a008585_cfg); u32 __fsl_a008585_read_cntp_tval_el0(void) { return __fsl_a008585_read_reg(cntp_tval_el0); } u32 __fsl_a008585_read_cntv_tval_el0(void) { return __fsl_a008585_read_reg(cntv_tval_el0); } u64 __fsl_a008585_read_cntvct_el0(void) { return __fsl_a008585_read_reg(cntvct_el0); } EXPORT_SYMBOL(__fsl_a008585_read_cntvct_el0); #endif /* CONFIG_FSL_ERRATUM_A008585 */ static __always_inline void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val, struct clock_event_device *clk) { if (access == ARCH_TIMER_MEM_PHYS_ACCESS) { struct arch_timer *timer = to_arch_timer(clk); switch (reg) { case ARCH_TIMER_REG_CTRL: writel_relaxed(val, timer->base + CNTP_CTL); break; case ARCH_TIMER_REG_TVAL: writel_relaxed(val, timer->base + CNTP_TVAL); break; } } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) { struct arch_timer *timer = to_arch_timer(clk); switch (reg) { case ARCH_TIMER_REG_CTRL: writel_relaxed(val, timer->base + CNTV_CTL); break; case ARCH_TIMER_REG_TVAL: writel_relaxed(val, timer->base + CNTV_TVAL); break; } } else { arch_timer_reg_write_cp15(access, reg, val); } } static __always_inline u32 arch_timer_reg_read(int access, enum arch_timer_reg reg, struct clock_event_device *clk) { u32 val; if (access == ARCH_TIMER_MEM_PHYS_ACCESS) { struct arch_timer *timer = to_arch_timer(clk); switch (reg) { case ARCH_TIMER_REG_CTRL: val = readl_relaxed(timer->base + CNTP_CTL); break; case ARCH_TIMER_REG_TVAL: val = readl_relaxed(timer->base + CNTP_TVAL); break; } } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) { struct arch_timer *timer = to_arch_timer(clk); switch (reg) { case ARCH_TIMER_REG_CTRL: val = readl_relaxed(timer->base + CNTV_CTL); break; case ARCH_TIMER_REG_TVAL: val = readl_relaxed(timer->base + CNTV_TVAL); break; } } else { val = arch_timer_reg_read_cp15(access, reg); } return val; } static __always_inline irqreturn_t timer_handler(const int access, struct clock_event_device *evt) { unsigned long ctrl; ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt); if (ctrl & ARCH_TIMER_CTRL_IT_STAT) { ctrl |= ARCH_TIMER_CTRL_IT_MASK; arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt); evt->event_handler(evt); return IRQ_HANDLED; } return IRQ_NONE; } static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id) { struct clock_event_device *evt = dev_id; return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt); } static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id) { struct clock_event_device *evt = dev_id; return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt); } static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id) { struct clock_event_device *evt = dev_id; return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt); } static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id) { struct clock_event_device *evt = dev_id; return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt); } static __always_inline int timer_shutdown(const int access, struct clock_event_device *clk) { unsigned long ctrl; ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk); ctrl &= ~ARCH_TIMER_CTRL_ENABLE; arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk); return 0; } static int arch_timer_shutdown_virt(struct clock_event_device *clk) { return timer_shutdown(ARCH_TIMER_VIRT_ACCESS, clk); } static int arch_timer_shutdown_phys(struct clock_event_device *clk) { return timer_shutdown(ARCH_TIMER_PHYS_ACCESS, clk); } static int arch_timer_shutdown_virt_mem(struct clock_event_device *clk) { return timer_shutdown(ARCH_TIMER_MEM_VIRT_ACCESS, clk); } static int arch_timer_shutdown_phys_mem(struct clock_event_device *clk) { return timer_shutdown(ARCH_TIMER_MEM_PHYS_ACCESS, clk); } static __always_inline void set_next_event(const int access, unsigned long evt, struct clock_event_device *clk) { unsigned long ctrl; ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk); ctrl |= ARCH_TIMER_CTRL_ENABLE; ctrl &= ~ARCH_TIMER_CTRL_IT_MASK; arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt, clk); arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk); } #ifdef CONFIG_FSL_ERRATUM_A008585 static __always_inline void fsl_a008585_set_next_event(const int access, unsigned long evt, struct clock_event_device *clk) { unsigned long ctrl; u64 cval = evt + arch_counter_get_cntvct(); ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk); ctrl |= ARCH_TIMER_CTRL_ENABLE; ctrl &= ~ARCH_TIMER_CTRL_IT_MASK; if (access == ARCH_TIMER_PHYS_ACCESS) write_sysreg(cval, cntp_cval_el0); else if (access == ARCH_TIMER_VIRT_ACCESS) write_sysreg(cval, cntv_cval_el0); arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk); } static int fsl_a008585_set_next_event_virt(unsigned long evt, struct clock_event_device *clk) { fsl_a008585_set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk); return 0; } static int fsl_a008585_set_next_event_phys(unsigned long evt, struct clock_event_device *clk) { fsl_a008585_set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk); return 0; } #endif /* CONFIG_FSL_ERRATUM_A008585 */ static int arch_timer_set_next_event_virt(unsigned long evt, struct clock_event_device *clk) { set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk); return 0; } static int arch_timer_set_next_event_phys(unsigned long evt, struct clock_event_device *clk) { set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk); return 0; } static int arch_timer_set_next_event_virt_mem(unsigned long evt, struct clock_event_device *clk) { set_next_event(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk); return 0; } static int arch_timer_set_next_event_phys_mem(unsigned long evt, struct clock_event_device *clk) { set_next_event(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk); return 0; } static void fsl_a008585_set_sne(struct clock_event_device *clk) { #ifdef CONFIG_FSL_ERRATUM_A008585 if (!static_branch_unlikely(&arch_timer_read_ool_enabled)) return; if (arch_timer_uses_ppi == VIRT_PPI) clk->set_next_event = fsl_a008585_set_next_event_virt; else clk->set_next_event = fsl_a008585_set_next_event_phys; #endif } static void __arch_timer_setup(unsigned type, struct clock_event_device *clk) { clk->features = CLOCK_EVT_FEAT_ONESHOT; if (type == ARCH_CP15_TIMER) { if (arch_timer_c3stop) clk->features |= CLOCK_EVT_FEAT_C3STOP; clk->name = "arch_sys_timer"; clk->rating = 450; clk->cpumask = cpumask_of(smp_processor_id()); clk->irq = arch_timer_ppi[arch_timer_uses_ppi]; switch (arch_timer_uses_ppi) { case VIRT_PPI: clk->set_state_shutdown = arch_timer_shutdown_virt; clk->set_state_oneshot_stopped = arch_timer_shutdown_virt; clk->set_next_event = arch_timer_set_next_event_virt; break; case PHYS_SECURE_PPI: case PHYS_NONSECURE_PPI: case HYP_PPI: clk->set_state_shutdown = arch_timer_shutdown_phys; clk->set_state_oneshot_stopped = arch_timer_shutdown_phys; clk->set_next_event = arch_timer_set_next_event_phys; break; default: BUG(); } fsl_a008585_set_sne(clk); } else { clk->features |= CLOCK_EVT_FEAT_DYNIRQ; clk->name = "arch_mem_timer"; clk->rating = 400; clk->cpumask = cpu_all_mask; if (arch_timer_mem_use_virtual) { clk->set_state_shutdown = arch_timer_shutdown_virt_mem; clk->set_state_oneshot_stopped = arch_timer_shutdown_virt_mem; clk->set_next_event = arch_timer_set_next_event_virt_mem; } else { clk->set_state_shutdown = arch_timer_shutdown_phys_mem; clk->set_state_oneshot_stopped = arch_timer_shutdown_phys_mem; clk->set_next_event = arch_timer_set_next_event_phys_mem; } } clk->set_state_shutdown(clk); clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff); } static void arch_timer_evtstrm_enable(int divider) { u32 cntkctl = arch_timer_get_cntkctl(); cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK; /* Set the divider and enable virtual event stream */ cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT) | ARCH_TIMER_VIRT_EVT_EN; arch_timer_set_cntkctl(cntkctl); elf_hwcap |= HWCAP_EVTSTRM; #ifdef CONFIG_COMPAT compat_elf_hwcap |= COMPAT_HWCAP_EVTSTRM; #endif } static void arch_timer_configure_evtstream(void) { int evt_stream_div, pos; /* Find the closest power of two to the divisor */ evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ; pos = fls(evt_stream_div); if (pos > 1 && !(evt_stream_div & (1 << (pos - 2)))) pos--; /* enable event stream */ arch_timer_evtstrm_enable(min(pos, 15)); } static void arch_counter_set_user_access(void) { u32 cntkctl = arch_timer_get_cntkctl(); /* Disable user access to the timers and the physical counter */ /* Also disable virtual event stream */ cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN | ARCH_TIMER_USR_VT_ACCESS_EN | ARCH_TIMER_VIRT_EVT_EN | ARCH_TIMER_USR_PCT_ACCESS_EN); /* Enable user access to the virtual counter */ cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN; arch_timer_set_cntkctl(cntkctl); } static bool arch_timer_has_nonsecure_ppi(void) { return (arch_timer_uses_ppi == PHYS_SECURE_PPI && arch_timer_ppi[PHYS_NONSECURE_PPI]); } static u32 check_ppi_trigger(int irq) { u32 flags = irq_get_trigger_type(irq); if (flags != IRQF_TRIGGER_HIGH && flags != IRQF_TRIGGER_LOW) { pr_warn("WARNING: Invalid trigger for IRQ%d, assuming level low\n", irq); pr_warn("WARNING: Please fix your firmware\n"); flags = IRQF_TRIGGER_LOW; } return flags; } static int arch_timer_starting_cpu(unsigned int cpu) { struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt); u32 flags; __arch_timer_setup(ARCH_CP15_TIMER, clk); flags = check_ppi_trigger(arch_timer_ppi[arch_timer_uses_ppi]); enable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], flags); if (arch_timer_has_nonsecure_ppi()) { flags = check_ppi_trigger(arch_timer_ppi[PHYS_NONSECURE_PPI]); enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], flags); } arch_counter_set_user_access(); if (evtstrm_enable) arch_timer_configure_evtstream(); return 0; } static void arch_timer_detect_rate(void __iomem *cntbase, struct device_node *np) { /* Who has more than one independent system counter? */