/* * linux/drivers/clocksource/arm_arch_timer.c * * Copyright (C) 2011 ARM Ltd. * All Rights Reserved * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. */#include <linux/init.h>#include <linux/kernel.h>#include <linux/device.h>#include <linux/smp.h>#include <linux/cpu.h>#include <linux/cpu_pm.h>#include <linux/clockchips.h>#include <linux/interrupt.h>#include <linux/of_irq.h>#include <linux/of_address.h>#include <linux/io.h>#include <linux/slab.h>#include <linux/sched_clock.h> #include <asm/arch_timer.h>#include <asm/virt.h> #include <clocksource/arm_arch_timer.h> #define CNTTIDR 0x08#define CNTTIDR_VIRT(n) (BIT(1) << ((n) * 4)) #define CNTVCT_LO 0x08#define CNTVCT_HI 0x0c#define CNTFRQ 0x10#define CNTP_TVAL 0x28#define CNTP_CTL 0x2c#define CNTV_TVAL 0x38#define CNTV_CTL 0x3c #define ARCH_CP15_TIMER BIT(0)#define ARCH_MEM_TIMER BIT(1)static unsigned arch_timers_present __initdata; static void __iomem *arch_counter_base; struct arch_timer { void __iomem *base; struct clock_event_device evt;}; #define to_arch_timer(e) container_of(e, struct arch_timer, evt) static u32 arch_timer_rate; enum ppi_nr { PHYS_SECURE_PPI, PHYS_NONSECURE_PPI, VIRT_PPI, HYP_PPI, MAX_TIMER_PPI}; static int arch_timer_ppi[MAX_TIMER_PPI]; static struct clock_event_device __percpu *arch_timer_evt; static bool arch_timer_use_virtual = true;static bool arch_timer_mem_use_virtual; /* * Architected system timer support. */ static __always_inlinevoid arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val, struct clock_event_device *clk){ if (access == ARCH_TIMER_MEM_PHYS_ACCESS) { struct arch_timer *timer = to_arch_timer(clk); switch (reg) { case ARCH_TIMER_REG_CTRL: writel_relaxed(val, timer->base + CNTP_CTL); break; case ARCH_TIMER_REG_TVAL: writel_relaxed(val, timer->base + CNTP_TVAL); break; } } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) { struct arch_timer *timer = to_arch_timer(clk); switch (reg) { case ARCH_TIMER_REG_CTRL: writel_relaxed(val, timer->base + CNTV_CTL); break; case ARCH_TIMER_REG_TVAL: writel_relaxed(val, timer->base + CNTV_TVAL); break; } } else { arch_timer_reg_write_cp15(access, reg, val); }} static __always_inlineu32 arch_timer_reg_read(int access, enum arch_timer_reg reg, struct clock_event_device *clk){ u32 val; if (access == ARCH_TIMER_MEM_PHYS_ACCESS) { struct arch_timer *timer = to_arch_timer(clk); switch (reg) { case ARCH_TIMER_REG_CTRL: val = readl_relaxed(timer->base + CNTP_CTL); break; case ARCH_TIMER_REG_TVAL: val = readl_relaxed(timer->base + CNTP_TVAL); break; } } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) { struct arch_timer *timer = to_arch_timer(clk); switch (reg) { case ARCH_TIMER_REG_CTRL: val = readl_relaxed(timer->base + CNTV_CTL); break; case ARCH_TIMER_REG_TVAL: val = readl_relaxed(timer->base + CNTV_TVAL); break; } } else { val = arch_timer_reg_read_cp15(access, reg); } return val;} static __always_inline irqreturn_t timer_handler(const int access, struct clock_event_device *evt){ unsigned long ctrl; ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt); if (ctrl & ARCH_TIMER_CTRL_IT_STAT) { ctrl |= ARCH_TIMER_CTRL_IT_MASK; arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt); evt->event_handler(evt); return IRQ_HANDLED; } return IRQ_NONE;} static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id){ struct clock_event_device *evt = dev_id; return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);} static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id){ struct clock_event_device *evt = dev_id; return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);} static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id){ struct clock_event_device *evt = dev_id; return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);} static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id){ struct clock_event_device *evt = dev_id; return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);} static __always_inline void timer_set_mode(const int access, int mode, struct clock_event_device *clk){ unsigned long ctrl; switch (mode) { case CLOCK_EVT_MODE_UNUSED: case CLOCK_EVT_MODE_SHUTDOWN: ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk); ctrl &= ~ARCH_TIMER_CTRL_ENABLE; arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk); break; default: break; }} static void arch_timer_set_mode_virt(enum clock_event_mode mode, struct clock_event_device *clk){ timer_set_mode(ARCH_TIMER_VIRT_ACCESS, mode, clk);} static void arch_timer_set_mode_phys(enum clock_event_mode mode, struct clock_event_device *clk){ timer_set_mode(ARCH_TIMER_PHYS_ACCESS, mode, clk);} static void arch_timer_set_mode_virt_mem(enum clock_event_mode mode, struct clock_event_device *clk){ timer_set_mode(ARCH_TIMER_MEM_VIRT_ACCESS, mode, clk);} static void arch_timer_set_mode_phys_mem(enum clock_event_mode mode, struct clock_event_device *clk){ timer_set_mode(ARCH_TIMER_MEM_PHYS_ACCESS, mode, clk);} static __always_inline void set_next_event(const int access, unsigned long evt, struct clock_event_device *clk){ unsigned long ctrl; ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk); ctrl |= ARCH_TIMER_CTRL_ENABLE; ctrl &= ~ARCH_TIMER_CTRL_IT_MASK; arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt, clk); arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);} static int arch_timer_set_next_event_virt(unsigned long evt, struct clock_event_device *clk){ set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk); return 0;} static int arch_timer_set_next_event_phys(unsigned long evt, struct clock_event_device *clk){ set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk); return 0;} static int arch_timer_set_next_event_virt_mem(unsigned long evt, struct clock_event_device *clk){ set_next_event(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk); return 0;} static int arch_timer_set_next_event_phys_mem(unsigned long evt, struct clock_event_device *clk){ set_next_event(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk); return 0;} static void __arch_timer_setup(unsigned type, struct clock_event_device *clk){ clk->features = CLOCK_EVT_FEAT_ONESHOT; if (type == ARCH_CP15_TIMER) { clk->features |= CLOCK_EVT_FEAT_C3STOP; clk->name = "arch_sys_timer"; clk->rating = 450; clk->cpumask = cpumask_of(smp_processor_id()); if (arch_timer_use_virtual) { clk->irq = arch_timer_ppi[VIRT_PPI]; clk->set_mode = arch_timer_set_mode_virt; clk->set_next_event = arch_timer_set_next_event_virt; } else { clk->irq = arch_timer_ppi[PHYS_SECURE_PPI]; clk->set_mode = arch_timer_set_mode_phys; clk->set_next_event = arch_timer_set_next_event_phys; } } else { clk->name = "arch_mem_timer"; clk->rating = 400; clk->cpumask = cpu_all_mask; if (arch_timer_mem_use_virtual) { clk->set_mode = arch_timer_set_mode_virt_mem; clk->set_next_event = arch_timer_set_next_event_virt_mem; } else { clk->set_mode = arch_timer_set_mode_phys_mem; clk->set_next_event = arch_timer_set_next_event_phys_mem; } } clk->set_mode(CLOCK_EVT_MODE_SHUTDOWN, clk); clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff);} static void arch_timer_configure_evtstream(void){ int evt_stream_div, pos; /* Find the closest power of two to the divisor */ evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ; pos = fls(evt_stream_div); if (pos > 1 && !(evt_stream_div & (1 << (pos - 2)))) pos--; /* enable event stream */ arch_timer_evtstrm_enable(min(pos, 15));} static int arch_timer_setup(struct clock_event_device *clk){ __arch_timer_setup(ARCH_CP15_TIMER, clk); if (arch_timer_use_virtual) enable_percpu_irq(arch_timer_ppi[VIRT_PPI], 0); else { enable_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI], 0); if (arch_timer_ppi[PHYS_NONSECURE_PPI]) enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], 0); } arch_counter_set_user_access(); if (IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM)) arch_timer_configure_evtstream(); return 0;} static voidarch_timer_detect_rate(void __iomem *cntbase, struct device_node *np){ /* Who has more than one independent system counter? */ if (arch_timer_rate) return; /* Try to determine the frequency from the device tree or CNTFRQ */ if (of_property_read_u32(np, "clock-frequency", &arch_timer_rate)) { if (cntbase) arch_timer_rate = readl_relaxed(cntbase + CNTFRQ); else arch_timer_rate = arch_timer_get_cntfrq(); } /* Check the timer frequency. */ if (arch_timer_rate == 0) pr_warn("Architected timer frequency not available\n");} static void arch_timer_banner(unsigned type){ pr_info("Architected %s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n", type & ARCH_CP15_TIMER ? "cp15" : "", type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ? " and " : "", type & ARCH_MEM_TIMER ? "mmio" : "", (unsigned long)arch_timer_rate / 1000000, (unsigned long)(arch_timer_rate / 10000) % 100, type & ARCH_CP15_TIMER ? arch_timer_use_virtual ? "virt" : "phys" : "", type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ? "/" : "", type & ARCH_MEM_TIMER ? arch_timer_mem_use_virtual ? "virt" : "phys" : "");} u32 arch_timer_get_rate(void){ return arch_timer_rate;} static u64 arch_counter_get_cntvct_mem(void){ u32 vct_lo, vct_hi, tmp_hi; do { vct_hi = readl_relaxed(arch_counter_base + CNTVCT_HI); vct_lo = readl_relaxed(arch_counter_base + CNTVCT_LO); tmp_hi = readl_relaxed(arch_counter_base + CNTVCT_HI); } while (vct_hi != tmp_hi); return ((u64) vct_hi << 32) | vct_lo;} /* * Default to cp15 based access because arm64 uses this function for * sched_clock() before DT is probed and the cp15 method is guaranteed * to exist on arm64. arm doesn't use this before DT is probed so even * if we don't have the cp15 accessors we won't have a problem. */u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct; static cycle_t arch_counter_read(struct clocksource *cs){ return arch_timer_read_counter();} static cycle_t arch_counter_read_cc(const struct cyclecounter *cc){ return arch_timer_read_counter();} static struct clocksource clocksource_counter = { .name = "arch_sys_counter", .rating = 400, .read = arch_counter_read, .mask = CLOCKSOURCE_MASK(56), .flags = CLOCK_SOURCE_IS_CONTINUOUS | CLOCK_SOURCE_SUSPEND_NONSTOP,}; static struct cyclecounter cyclecounter = { .read = arch_counter_read_cc, .mask = CLOCKSOURCE_MASK(56),}; static struct timecounter timecounter; struct timecounter *arch_timer_get_timecounter(void){ return &timecounter;} static void __init arch_counter_register(unsigned type){ u64 start_count; /* Register the CP15 based counter if we have one */ if (type & ARCH_CP15_TIMER) arch_timer_read_counter = arch_counter_get_cntvct; else arch_timer_read_counter = arch_counter_get_cntvct_mem; start_count = arch_timer_read_counter(); clocksource_register_hz(&clocksource_counter, arch_timer_rate); cyclecounter.mult = clocksource_counter.mult; cyclecounter.shift = clocksource_counter.shift; timecounter_init(&timecounter, &cyclecounter, start_count); /* 56 bits minimum, so we assume worst case rollover */ sched_clock_register(arch_timer_read_counter, 56, arch_timer_rate);} static void arch_timer_stop(struct clock_event_device *clk){ pr_debug("arch_timer_teardown disable IRQ%d cpu #%d\n", clk->irq, smp_processor_id()); if (arch_timer_use_virtual) disable_percpu_irq(arch_timer_ppi[VIRT_PPI]); else { disable_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI]); if (arch_timer_ppi[PHYS_NONSECURE_PPI]) disable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI]); } clk->set_mode(CLOCK_EVT_MODE_UNUSED, clk);} static int arch_timer_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu){ /* * Grab cpu pointer in each case to avoid spurious * preemptible warnings */ switch (action & ~CPU_TASKS_FROZEN) { case CPU_STARTING: arch_timer_setup(this_cpu_ptr(arch_timer_evt)); break; case CPU_DYING: arch_timer_stop(this_cpu_ptr(arch_timer_evt)); break; } return NOTIFY_OK;} static struct notifier_block arch_timer_cpu_nb = { .notifier_call = arch_timer_cpu_notify,}; #ifdef CONFIG_CPU_PMstatic unsigned int saved_cntkctl;static int arch_timer_cpu_pm_notify(struct notifier_block *self, unsigned long action, void *hcpu){ if (action == CPU_PM_ENTER) saved_cntkctl = arch_timer_get_cntkctl(); else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT) arch_timer_set_cntkctl(saved_cntkctl); return NOTIFY_OK;} static struct notifier_block arch_timer_cpu_pm_notifier = { .notifier_call = arch_timer_cpu_pm_notify,}; static int __init arch_timer_cpu_pm_init(void){Lines 1–500 of 737. Load more
/* * linux/drivers/clocksource/arm_arch_timer.c * * Copyright (C) 2011 ARM Ltd. * All Rights Reserved * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. */ #include <linux/init.h> #include <linux/kernel.h> #include <linux/device.h> #include <linux/smp.h> #include <linux/cpu.h> #include <linux/cpu_pm.h> #include <linux/clockchips.h> #include <linux/interrupt.h> #include <linux/of_irq.h> #include <linux/of_address.h> #include <linux/io.h> #include <linux/slab.h> #include <linux/sched_clock.h> #include <asm/arch_timer.h> #include <asm/virt.h> #include <clocksource/arm_arch_timer.h> #define CNTTIDR 0x08 #define CNTTIDR_VIRT(n) (BIT(1) << ((n) * 4)) #define CNTVCT_LO 0x08 #define CNTVCT_HI 0x0c #define CNTFRQ 0x10 #define CNTP_TVAL 0x28 #define CNTP_CTL 0x2c #define CNTV_TVAL 0x38 #define CNTV_CTL 0x3c #define ARCH_CP15_TIMER BIT(0) #define ARCH_MEM_TIMER BIT(1) static unsigned arch_timers_present __initdata; static void __iomem *arch_counter_base; struct arch_timer { void __iomem *base; struct clock_event_device evt; }; #define to_arch_timer(e) container_of(e, struct arch_timer, evt) static u32 arch_timer_rate; enum ppi_nr { PHYS_SECURE_PPI, PHYS_NONSECURE_PPI, VIRT_PPI, HYP_PPI, MAX_TIMER_PPI }; static int arch_timer_ppi[MAX_TIMER_PPI]; static struct clock_event_device __percpu *arch_timer_evt; static bool arch_timer_use_virtual = true; static bool arch_timer_mem_use_virtual; /* * Architected system timer support. */ static __always_inline void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val, struct clock_event_device *clk) { if (access == ARCH_TIMER_MEM_PHYS_ACCESS) { struct arch_timer *timer = to_arch_timer(clk); switch (reg) { case ARCH_TIMER_REG_CTRL: writel_relaxed(val, timer->base + CNTP_CTL); break; case ARCH_TIMER_REG_TVAL: writel_relaxed(val, timer->base + CNTP_TVAL); break; } } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) { struct arch_timer *timer = to_arch_timer(clk); switch (reg) { case ARCH_TIMER_REG_CTRL: writel_relaxed(val, timer->base + CNTV_CTL); break; case ARCH_TIMER_REG_TVAL: writel_relaxed(val, timer->base + CNTV_TVAL); break; } } else { arch_timer_reg_write_cp15(access, reg, val); } } static __always_inline u32 arch_timer_reg_read(int access, enum arch_timer_reg reg, struct clock_event_device *clk) { u32 val; if (access == ARCH_TIMER_MEM_PHYS_ACCESS) { struct arch_timer *timer = to_arch_timer(clk); switch (reg) { case ARCH_TIMER_REG_CTRL: val = readl_relaxed(timer->base + CNTP_CTL); break; case ARCH_TIMER_REG_TVAL: val = readl_relaxed(timer->base + CNTP_TVAL); break; } } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) { struct arch_timer *timer = to_arch_timer(clk); switch (reg) { case ARCH_TIMER_REG_CTRL: val = readl_relaxed(timer->base + CNTV_CTL); break; case ARCH_TIMER_REG_TVAL: val = readl_relaxed(timer->base + CNTV_TVAL); break; } } else { val = arch_timer_reg_read_cp15(access, reg); } return val; } static __always_inline irqreturn_t timer_handler(const int access, struct clock_event_device *evt) { unsigned long ctrl; ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt); if (ctrl & ARCH_TIMER_CTRL_IT_STAT) { ctrl |= ARCH_TIMER_CTRL_IT_MASK; arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt); evt->event_handler(evt); return IRQ_HANDLED; } return IRQ_NONE; } static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id) { struct clock_event_device *evt = dev_id; return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt); } static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id) { struct clock_event_device *evt = dev_id; return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt); } static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id) { struct clock_event_device *evt = dev_id; return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt); } static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id) { struct clock_event_device *evt = dev_id; return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt); } static __always_inline void timer_set_mode(const int access, int mode, struct clock_event_device *clk) { unsigned long ctrl; switch (mode) { case CLOCK_EVT_MODE_UNUSED: case CLOCK_EVT_MODE_SHUTDOWN: ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk); ctrl &= ~ARCH_TIMER_CTRL_ENABLE; arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk); break; default: break; } } static void arch_timer_set_mode_virt(enum clock_event_mode mode, struct clock_event_device *clk) { timer_set_mode(ARCH_TIMER_VIRT_ACCESS, mode, clk); } static void arch_timer_set_mode_phys(enum clock_event_mode mode, struct clock_event_device *clk) { timer_set_mode(ARCH_TIMER_PHYS_ACCESS, mode, clk); } static void arch_timer_set_mode_virt_mem(enum clock_event_mode mode, struct clock_event_device *clk) { timer_set_mode(ARCH_TIMER_MEM_VIRT_ACCESS, mode, clk); } static void arch_timer_set_mode_phys_mem(enum clock_event_mode mode, struct clock_event_device *clk) { timer_set_mode(ARCH_TIMER_MEM_PHYS_ACCESS, mode, clk); } static __always_inline void set_next_event(const int access, unsigned long evt, struct clock_event_device *clk) { unsigned long ctrl; ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk); ctrl |= ARCH_TIMER_CTRL_ENABLE; ctrl &= ~ARCH_TIMER_CTRL_IT_MASK; arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt, clk); arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk); } static int arch_timer_set_next_event_virt(unsigned long evt, struct clock_event_device *clk) { set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk); return 0; } static int arch_timer_set_next_event_phys(unsigned long evt, struct clock_event_device *clk) { set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk); return 0; } static int arch_timer_set_next_event_virt_mem(unsigned long evt, struct clock_event_device *clk) { set_next_event(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk); return 0; } static int arch_timer_set_next_event_phys_mem(unsigned long evt, struct clock_event_device *clk) { set_next_event(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk); return 0; } static void __arch_timer_setup(unsigned type, struct clock_event_device *clk) { clk->features = CLOCK_EVT_FEAT_ONESHOT; if (type == ARCH_CP15_TIMER) { clk->features |= CLOCK_EVT_FEAT_C3STOP; clk->name = "arch_sys_timer"; clk->rating = 450; clk->cpumask = cpumask_of(smp_processor_id()); if (arch_timer_use_virtual) { clk->irq = arch_timer_ppi[VIRT_PPI]; clk->set_mode = arch_timer_set_mode_virt; clk->set_next_event = arch_timer_set_next_event_virt; } else { clk->irq = arch_timer_ppi[PHYS_SECURE_PPI]; clk->set_mode = arch_timer_set_mode_phys; clk->set_next_event = arch_timer_set_next_event_phys; } } else { clk->name = "arch_mem_timer"; clk->rating = 400; clk->cpumask = cpu_all_mask; if (arch_timer_mem_use_virtual) { clk->set_mode = arch_timer_set_mode_virt_mem; clk->set_next_event = arch_timer_set_next_event_virt_mem; } else { clk->set_mode = arch_timer_set_mode_phys_mem; clk->set_next_event = arch_timer_set_next_event_phys_mem; } } clk->set_mode(CLOCK_EVT_MODE_SHUTDOWN, clk); clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff); } static void arch_timer_configure_evtstream(void) { int evt_stream_div, pos; /* Find the closest power of two to the divisor */ evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ; pos = fls(evt_stream_div); if (pos > 1 && !(evt_stream_div & (1 << (pos - 2)))) pos--; /* enable event stream */ arch_timer_evtstrm_enable(min(pos, 15)); } static int arch_timer_setup(struct clock_event_device *clk) { __arch_timer_setup(ARCH_CP15_TIMER, clk); if (arch_timer_use_virtual) enable_percpu_irq(arch_timer_ppi[VIRT_PPI], 0); else { enable_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI], 0); if (arch_timer_ppi[PHYS_NONSECURE_PPI]) enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], 0); } arch_counter_set_user_access(); if (IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM)) arch_timer_configure_evtstream(); return 0; } static void arch_timer_detect_rate(void __iomem *cntbase, struct device_node *np) { /* Who has more than one independent system counter? */ if (arch_timer_rate) return; /* Try to determine the frequency from the device tree or CNTFRQ */ if (of_property_read_u32(np, "clock-frequency", &arch_timer_rate)) { if (cntbase) arch_timer_rate = readl_relaxed(cntbase + CNTFRQ); else arch_timer_rate = arch_timer_get_cntfrq(); } /* Check the timer frequency. */ if (arch_timer_rate == 0) pr_warn("Architected timer frequency not available\n"); } static void arch_timer_banner(unsigned type) { pr_info("Architected %s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n", type & ARCH_CP15_TIMER ? "cp15" : "", type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ? " and " : "", type & ARCH_MEM_TIMER ? "mmio" : "", (unsigned long)arch_timer_rate / 1000000, (unsigned long)(arch_timer_rate / 10000) % 100, type & ARCH_CP15_TIMER ? arch_timer_use_virtual ? "virt" : "phys" : "", type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ? "/" : "", type & ARCH_MEM_TIMER ? arch_timer_mem_use_virtual ? "virt" : "phys" : ""); } u32 arch_timer_get_rate(void) { return arch_timer_rate; } static u64 arch_counter_get_cntvct_mem(void) { u32 vct_lo, vct_hi, tmp_hi; do { vct_hi = readl_relaxed(arch_counter_base + CNTVCT_HI); vct_lo = readl_relaxed(arch_counter_base + CNTVCT_LO); tmp_hi = readl_relaxed(arch_counter_base + CNTVCT_HI); } while (vct_hi != tmp_hi); return ((u64) vct_hi << 32) | vct_lo; } /* * Default to cp15 based access because arm64 uses this function for * sched_clock() before DT is probed and the cp15 method is guaranteed * to exist on arm64. arm doesn't use this before DT is probed so even * if we don't have the cp15 accessors we won't have a problem. */ u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct; static cycle_t arch_counter_read(struct clocksource *cs) { return arch_timer_read_counter(); } static cycle_t arch_counter_read_cc(const struct cyclecounter *cc) { return arch_timer_read_counter(); } static struct clocksource clocksource_counter = { .name = "arch_sys_counter", .rating = 400, .read = arch_counter_read, .mask = CLOCKSOURCE_MASK(56), .flags = CLOCK_SOURCE_IS_CONTINUOUS | CLOCK_SOURCE_SUSPEND_NONSTOP, }; static struct cyclecounter cyclecounter = { .read = arch_counter_read_cc, .mask = CLOCKSOURCE_MASK(56), }; static struct timecounter timecounter; struct timecounter *arch_timer_get_timecounter(void) { return &timecounter; } static void __init arch_counter_register(unsigned type) { u64 start_count; /* Register the CP15 based counter if we have one */ if (type & ARCH_CP15_TIMER) arch_timer_read_counter = arch_counter_get_cntvct; else arch_timer_read_counter = arch_counter_get_cntvct_mem; start_count = arch_timer_read_counter(); clocksource_register_hz(&clocksource_counter, arch_timer_rate); cyclecounter.mult = clocksource_counter.mult; cyclecounter.shift = clocksource_counter.shift; timecounter_init(&timecounter, &cyclecounter, start_count); /* 56 bits minimum, so we assume worst case rollover */ sched_clock_register(arch_timer_read_counter, 56, arch_timer_rate); } static void arch_timer_stop(struct clock_event_device *clk) { pr_debug("arch_timer_teardown disable IRQ%d cpu #%d\n", clk->irq, smp_processor_id()); if (arch_timer_use_virtual) disable_percpu_irq(arch_timer_ppi[VIRT_PPI]); else { disable_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI]); if (arch_timer_ppi[PHYS_NONSECURE_PPI]) disable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI]); } clk->set_mode(CLOCK_EVT_MODE_UNUSED, clk); } static int arch_timer_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { /* * Grab cpu pointer in each case to avoid spurious * preemptible warnings */ switch (action & ~CPU_TASKS_FROZEN) { case CPU_STARTING: arch_timer_setup(this_cpu_ptr(arch_timer_evt)); break; case CPU_DYING: arch_timer_stop(this_cpu_ptr(arch_timer_evt)); break; } return NOTIFY_OK; } static struct notifier_block arch_timer_cpu_nb = { .notifier_call = arch_timer_cpu_notify, }; #ifdef CONFIG_CPU_PM static unsigned int saved_cntkctl; static int arch_timer_cpu_pm_notify(struct notifier_block *self, unsigned long action, void *hcpu) { if (action == CPU_PM_ENTER) saved_cntkctl = arch_timer_get_cntkctl(); else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT) arch_timer_set_cntkctl(saved_cntkctl); return NOTIFY_OK; } static struct notifier_block arch_timer_cpu_pm_notifier = { .notifier_call = arch_timer_cpu_pm_notify, }; static int __init arch_timer_cpu_pm_init(void) {