/* * Common time routines among all ppc machines. * * Written by Cort Dougan (cort@cs.nmt.edu) to merge * Paul Mackerras' version and mine for PReP and Pmac. * MPC8xx/MBX changes by Dan Malek (dmalek@jlc.net). * Converted for 64-bit by Mike Corrigan (mikejc@us.ibm.com) * * First round of bugfixes by Gabriel Paubert (paubert@iram.es) * to make clock more stable (2.4.0-test5). The only thing * that this code assumes is that the timebases have been synchronized * by firmware on SMP and are never stopped (never do sleep * on SMP then, nap and doze are OK). * * Speeded up do_gettimeofday by getting rid of references to * xtime (which required locks for consistency). (mikejc@us.ibm.com) * * TODO (not necessarily in this file): * - improve precision and reproducibility of timebase frequency * measurement at boot time. * - for astronomical applications: add a new function to get * non ambiguous timestamps even around leap seconds. This needs * a new timestamp format and a good name. * * 1997-09-10 Updated NTP code according to technical memorandum Jan '96 * "A Kernel Model for Precision Timekeeping" by Dave Mills * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version * 2 of the License, or (at your option) any later version. */ #include <linux/errno.h>#include <linux/export.h>#include <linux/sched.h>#include <linux/kernel.h>#include <linux/param.h>#include <linux/string.h>#include <linux/mm.h>#include <linux/interrupt.h>#include <linux/timex.h>#include <linux/kernel_stat.h>#include <linux/time.h>#include <linux/init.h>#include <linux/profile.h>#include <linux/cpu.h>#include <linux/security.h>#include <linux/percpu.h>#include <linux/rtc.h>#include <linux/jiffies.h>#include <linux/posix-timers.h>#include <linux/irq.h>#include <linux/delay.h>#include <linux/irq_work.h>#include <asm/trace.h> #include <asm/io.h>#include <asm/processor.h>#include <asm/nvram.h>#include <asm/cache.h>#include <asm/machdep.h>#include <asm/uaccess.h>#include <asm/time.h>#include <asm/prom.h>#include <asm/irq.h>#include <asm/div64.h>#include <asm/smp.h>#include <asm/vdso_datapage.h>#include <asm/firmware.h>#include <asm/cputime.h> /* powerpc clocksource/clockevent code */ #include <linux/clockchips.h>#include <linux/timekeeper_internal.h> static cycle_t rtc_read(struct clocksource *);static struct clocksource clocksource_rtc = { .name = "rtc", .rating = 400, .flags = CLOCK_SOURCE_IS_CONTINUOUS, .mask = CLOCKSOURCE_MASK(64), .read = rtc_read,}; static cycle_t timebase_read(struct clocksource *);static struct clocksource clocksource_timebase = { .name = "timebase", .rating = 400, .flags = CLOCK_SOURCE_IS_CONTINUOUS, .mask = CLOCKSOURCE_MASK(64), .read = timebase_read,}; #define DECREMENTER_MAX 0x7fffffff static int decrementer_set_next_event(unsigned long evt, struct clock_event_device *dev);static void decrementer_set_mode(enum clock_event_mode mode, struct clock_event_device *dev); struct clock_event_device decrementer_clockevent = { .name = "decrementer", .rating = 200, .irq = 0, .set_next_event = decrementer_set_next_event, .set_mode = decrementer_set_mode, .features = CLOCK_EVT_FEAT_ONESHOT,};EXPORT_SYMBOL(decrementer_clockevent); DEFINE_PER_CPU(u64, decrementers_next_tb);static DEFINE_PER_CPU(struct clock_event_device, decrementers); #define XSEC_PER_SEC (1024*1024) #ifdef CONFIG_PPC64#define SCALE_XSEC(xsec, max) (((xsec) * max) / XSEC_PER_SEC)#else/* compute ((xsec << 12) * max) >> 32 */#define SCALE_XSEC(xsec, max) mulhwu((xsec) << 12, max)#endif unsigned long tb_ticks_per_jiffy;unsigned long tb_ticks_per_usec = 100; /* sane default */EXPORT_SYMBOL(tb_ticks_per_usec);unsigned long tb_ticks_per_sec;EXPORT_SYMBOL(tb_ticks_per_sec); /* for cputime_t conversions */ DEFINE_SPINLOCK(rtc_lock);EXPORT_SYMBOL_GPL(rtc_lock); static u64 tb_to_ns_scale __read_mostly;static unsigned tb_to_ns_shift __read_mostly;static u64 boot_tb __read_mostly; extern struct timezone sys_tz;static long timezone_offset; unsigned long ppc_proc_freq;EXPORT_SYMBOL_GPL(ppc_proc_freq);unsigned long ppc_tb_freq;EXPORT_SYMBOL_GPL(ppc_tb_freq); #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE/* * Factors for converting from cputime_t (timebase ticks) to * jiffies, microseconds, seconds, and clock_t (1/USER_HZ seconds). * These are all stored as 0.64 fixed-point binary fractions. */u64 __cputime_jiffies_factor;EXPORT_SYMBOL(__cputime_jiffies_factor);u64 __cputime_usec_factor;EXPORT_SYMBOL(__cputime_usec_factor);u64 __cputime_sec_factor;EXPORT_SYMBOL(__cputime_sec_factor);u64 __cputime_clockt_factor;EXPORT_SYMBOL(__cputime_clockt_factor);DEFINE_PER_CPU(unsigned long, cputime_last_delta);DEFINE_PER_CPU(unsigned long, cputime_scaled_last_delta); cputime_t cputime_one_jiffy; void (*dtl_consumer)(struct dtl_entry *, u64); static void calc_cputime_factors(void){ struct div_result res; div128_by_32(HZ, 0, tb_ticks_per_sec, &res); __cputime_jiffies_factor = res.result_low; div128_by_32(1000000, 0, tb_ticks_per_sec, &res); __cputime_usec_factor = res.result_low; div128_by_32(1, 0, tb_ticks_per_sec, &res); __cputime_sec_factor = res.result_low; div128_by_32(USER_HZ, 0, tb_ticks_per_sec, &res); __cputime_clockt_factor = res.result_low;} /* * Read the SPURR on systems that have it, otherwise the PURR, * or if that doesn't exist return the timebase value passed in. */static u64 read_spurr(u64 tb){ if (cpu_has_feature(CPU_FTR_SPURR)) return mfspr(SPRN_SPURR); if (cpu_has_feature(CPU_FTR_PURR)) return mfspr(SPRN_PURR); return tb;} #ifdef CONFIG_PPC_SPLPAR /* * Scan the dispatch trace log and count up the stolen time. * Should be called with interrupts disabled. */static u64 scan_dispatch_log(u64 stop_tb){ u64 i = local_paca->dtl_ridx; struct dtl_entry *dtl = local_paca->dtl_curr; struct dtl_entry *dtl_end = local_paca->dispatch_log_end; struct lppaca *vpa = local_paca->lppaca_ptr; u64 tb_delta; u64 stolen = 0; u64 dtb; if (!dtl) return 0; if (i == be64_to_cpu(vpa->dtl_idx)) return 0; while (i < be64_to_cpu(vpa->dtl_idx)) { dtb = be64_to_cpu(dtl->timebase); tb_delta = be32_to_cpu(dtl->enqueue_to_dispatch_time) + be32_to_cpu(dtl->ready_to_enqueue_time); barrier(); if (i + N_DISPATCH_LOG < be64_to_cpu(vpa->dtl_idx)) { /* buffer has overflowed */ i = be64_to_cpu(vpa->dtl_idx) - N_DISPATCH_LOG; dtl = local_paca->dispatch_log + (i % N_DISPATCH_LOG); continue; } if (dtb > stop_tb) break; if (dtl_consumer) dtl_consumer(dtl, i); stolen += tb_delta; ++i; ++dtl; if (dtl == dtl_end) dtl = local_paca->dispatch_log; } local_paca->dtl_ridx = i; local_paca->dtl_curr = dtl; return stolen;} /* * Accumulate stolen time by scanning the dispatch trace log. * Called on entry from user mode. */void accumulate_stolen_time(void){ u64 sst, ust; u8 save_soft_enabled = local_paca->soft_enabled; /* We are called early in the exception entry, before * soft/hard_enabled are sync'ed to the expected state * for the exception. We are hard disabled but the PACA * needs to reflect that so various debug stuff doesn't * complain */ local_paca->soft_enabled = 0; sst = scan_dispatch_log(local_paca->starttime_user); ust = scan_dispatch_log(local_paca->starttime); local_paca->system_time -= sst; local_paca->user_time -= ust; local_paca->stolen_time += ust + sst; local_paca->soft_enabled = save_soft_enabled;} static inline u64 calculate_stolen_time(u64 stop_tb){ u64 stolen = 0; if (get_paca()->dtl_ridx != be64_to_cpu(get_lppaca()->dtl_idx)) { stolen = scan_dispatch_log(stop_tb); get_paca()->system_time -= stolen; } stolen += get_paca()->stolen_time; get_paca()->stolen_time = 0; return stolen;} #else /* CONFIG_PPC_SPLPAR */static inline u64 calculate_stolen_time(u64 stop_tb){ return 0;} #endif /* CONFIG_PPC_SPLPAR */ /* * Account time for a transition between system, hard irq * or soft irq state. */static u64 vtime_delta(struct task_struct *tsk, u64 *sys_scaled, u64 *stolen){ u64 now, nowscaled, deltascaled; u64 udelta, delta, user_scaled; WARN_ON_ONCE(!irqs_disabled()); now = mftb(); nowscaled = read_spurr(now); get_paca()->system_time += now - get_paca()->starttime; get_paca()->starttime = now; deltascaled = nowscaled - get_paca()->startspurr; get_paca()->startspurr = nowscaled; *stolen = calculate_stolen_time(now); delta = get_paca()->system_time; get_paca()->system_time = 0; udelta = get_paca()->user_time - get_paca()->utime_sspurr; get_paca()->utime_sspurr = get_paca()->user_time; /* * Because we don't read the SPURR on every kernel entry/exit, * deltascaled includes both user and system SPURR ticks. * Apportion these ticks to system SPURR ticks and user * SPURR ticks in the same ratio as the system time (delta) * and user time (udelta) values obtained from the timebase * over the same interval. The system ticks get accounted here; * the user ticks get saved up in paca->user_time_scaled to be * used by account_process_tick. */ *sys_scaled = delta; user_scaled = udelta; if (deltascaled != delta + udelta) { if (udelta) { *sys_scaled = deltascaled * delta / (delta + udelta); user_scaled = deltascaled - *sys_scaled; } else { *sys_scaled = deltascaled; } } get_paca()->user_time_scaled += user_scaled; return delta;} void vtime_account_system(struct task_struct *tsk){ u64 delta, sys_scaled, stolen; delta = vtime_delta(tsk, &sys_scaled, &stolen); account_system_time(tsk, 0, delta, sys_scaled); if (stolen) account_steal_time(stolen);}EXPORT_SYMBOL_GPL(vtime_account_system); void vtime_account_idle(struct task_struct *tsk){ u64 delta, sys_scaled, stolen; delta = vtime_delta(tsk, &sys_scaled, &stolen); account_idle_time(delta + stolen);} /* * Transfer the user time accumulated in the paca * by the exception entry and exit code to the generic * process user time records. * Must be called with interrupts disabled. * Assumes that vtime_account_system/idle() has been called * recently (i.e. since the last entry from usermode) so that * get_paca()->user_time_scaled is up to date. */void vtime_account_user(struct task_struct *tsk){ cputime_t utime, utimescaled; utime = get_paca()->user_time; utimescaled = get_paca()->user_time_scaled; get_paca()->user_time = 0; get_paca()->user_time_scaled = 0; get_paca()->utime_sspurr = 0; account_user_time(tsk, utime, utimescaled);} #else /* ! CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */#define calc_cputime_factors()#endif void __delay(unsigned long loops){ unsigned long start; int diff; if (__USE_RTC()) { start = get_rtcl(); do { /* the RTCL register wraps at 1000000000 */ diff = get_rtcl() - start; if (diff < 0) diff += 1000000000; } while (diff < loops); } else { start = get_tbl(); while (get_tbl() - start < loops) HMT_low(); HMT_medium(); }}EXPORT_SYMBOL(__delay); void udelay(unsigned long usecs){ __delay(tb_ticks_per_usec * usecs);}EXPORT_SYMBOL(udelay); #ifdef CONFIG_SMPunsigned long profile_pc(struct pt_regs *regs){ unsigned long pc = instruction_pointer(regs); if (in_lock_functions(pc)) return regs->link; return pc;}EXPORT_SYMBOL(profile_pc);#endif #ifdef CONFIG_IRQ_WORK /* * 64-bit uses a byte in the PACA, 32-bit uses a per-cpu variable... */#ifdef CONFIG_PPC64static inline unsigned long test_irq_work_pending(void){ unsigned long x; asm volatile("lbz %0,%1(13)" : "=r" (x) : "i" (offsetof(struct paca_struct, irq_work_pending))); return x;} static inline void set_irq_work_pending_flag(void){ asm volatile("stb %0,%1(13)" : : "r" (1), "i" (offsetof(struct paca_struct, irq_work_pending)));} static inline void clear_irq_work_pending(void){ asm volatile("stb %0,%1(13)" : : "r" (0), "i" (offsetof(struct paca_struct, irq_work_pending)));} #else /* 32-bit */ DEFINE_PER_CPU(u8, irq_work_pending); #define set_irq_work_pending_flag() __get_cpu_var(irq_work_pending) = 1#define test_irq_work_pending() __get_cpu_var(irq_work_pending)#define clear_irq_work_pending() __get_cpu_var(irq_work_pending) = 0 #endif /* 32 vs 64 bit */ void arch_irq_work_raise(void){ preempt_disable(); set_irq_work_pending_flag(); set_dec(1); preempt_enable();} #else /* CONFIG_IRQ_WORK */ #define test_irq_work_pending() 0#define clear_irq_work_pending() #endif /* CONFIG_IRQ_WORK */ /* * timer_interrupt - gets called when the decrementer overflows, * with interrupts disabled. */void timer_interrupt(struct pt_regs * regs){ struct pt_regs *old_regs; u64 *next_tb = &__get_cpu_var(decrementers_next_tb); struct clock_event_device *evt = &__get_cpu_var(decrementers); u64 now; /* Ensure a positive value is written to the decrementer, or else * some CPUs will continue to take decrementer exceptions. */ set_dec(DECREMENTER_MAX); /* Some implementations of hotplug will get timer interrupts while * offline, just ignore these and we also need to set * decrementers_next_tb as MAX to make sure __check_irq_replay * don't replay timer interrupt when return, otherwise we'll trapLines 1–500 of 1055. Load more
/* * Common time routines among all ppc machines. * * Written by Cort Dougan (cort@cs.nmt.edu) to merge * Paul Mackerras' version and mine for PReP and Pmac. * MPC8xx/MBX changes by Dan Malek (dmalek@jlc.net). * Converted for 64-bit by Mike Corrigan (mikejc@us.ibm.com) * * First round of bugfixes by Gabriel Paubert (paubert@iram.es) * to make clock more stable (2.4.0-test5). The only thing * that this code assumes is that the timebases have been synchronized * by firmware on SMP and are never stopped (never do sleep * on SMP then, nap and doze are OK). * * Speeded up do_gettimeofday by getting rid of references to * xtime (which required locks for consistency). (mikejc@us.ibm.com) * * TODO (not necessarily in this file): * - improve precision and reproducibility of timebase frequency * measurement at boot time. * - for astronomical applications: add a new function to get * non ambiguous timestamps even around leap seconds. This needs * a new timestamp format and a good name. * * 1997-09-10 Updated NTP code according to technical memorandum Jan '96 * "A Kernel Model for Precision Timekeeping" by Dave Mills * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version * 2 of the License, or (at your option) any later version. */ #include <linux/errno.h> #include <linux/export.h> #include <linux/sched.h> #include <linux/kernel.h> #include <linux/param.h> #include <linux/string.h> #include <linux/mm.h> #include <linux/interrupt.h> #include <linux/timex.h> #include <linux/kernel_stat.h> #include <linux/time.h> #include <linux/init.h> #include <linux/profile.h> #include <linux/cpu.h> #include <linux/security.h> #include <linux/percpu.h> #include <linux/rtc.h> #include <linux/jiffies.h> #include <linux/posix-timers.h> #include <linux/irq.h> #include <linux/delay.h> #include <linux/irq_work.h> #include <asm/trace.h> #include <asm/io.h> #include <asm/processor.h> #include <asm/nvram.h> #include <asm/cache.h> #include <asm/machdep.h> #include <asm/uaccess.h> #include <asm/time.h> #include <asm/prom.h> #include <asm/irq.h> #include <asm/div64.h> #include <asm/smp.h> #include <asm/vdso_datapage.h> #include <asm/firmware.h> #include <asm/cputime.h> /* powerpc clocksource/clockevent code */ #include <linux/clockchips.h> #include <linux/timekeeper_internal.h> static cycle_t rtc_read(struct clocksource *); static struct clocksource clocksource_rtc = { .name = "rtc", .rating = 400, .flags = CLOCK_SOURCE_IS_CONTINUOUS, .mask = CLOCKSOURCE_MASK(64), .read = rtc_read, }; static cycle_t timebase_read(struct clocksource *); static struct clocksource clocksource_timebase = { .name = "timebase", .rating = 400, .flags = CLOCK_SOURCE_IS_CONTINUOUS, .mask = CLOCKSOURCE_MASK(64), .read = timebase_read, }; #define DECREMENTER_MAX 0x7fffffff static int decrementer_set_next_event(unsigned long evt, struct clock_event_device *dev); static void decrementer_set_mode(enum clock_event_mode mode, struct clock_event_device *dev); struct clock_event_device decrementer_clockevent = { .name = "decrementer", .rating = 200, .irq = 0, .set_next_event = decrementer_set_next_event, .set_mode = decrementer_set_mode, .features = CLOCK_EVT_FEAT_ONESHOT, }; EXPORT_SYMBOL(decrementer_clockevent); DEFINE_PER_CPU(u64, decrementers_next_tb); static DEFINE_PER_CPU(struct clock_event_device, decrementers); #define XSEC_PER_SEC (1024*1024) #ifdef CONFIG_PPC64 #define SCALE_XSEC(xsec, max) (((xsec) * max) / XSEC_PER_SEC) #else /* compute ((xsec << 12) * max) >> 32 */ #define SCALE_XSEC(xsec, max) mulhwu((xsec) << 12, max) #endif unsigned long tb_ticks_per_jiffy; unsigned long tb_ticks_per_usec = 100; /* sane default */ EXPORT_SYMBOL(tb_ticks_per_usec); unsigned long tb_ticks_per_sec; EXPORT_SYMBOL(tb_ticks_per_sec); /* for cputime_t conversions */ DEFINE_SPINLOCK(rtc_lock); EXPORT_SYMBOL_GPL(rtc_lock); static u64 tb_to_ns_scale __read_mostly; static unsigned tb_to_ns_shift __read_mostly; static u64 boot_tb __read_mostly; extern struct timezone sys_tz; static long timezone_offset; unsigned long ppc_proc_freq; EXPORT_SYMBOL_GPL(ppc_proc_freq); unsigned long ppc_tb_freq; EXPORT_SYMBOL_GPL(ppc_tb_freq); #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE /* * Factors for converting from cputime_t (timebase ticks) to * jiffies, microseconds, seconds, and clock_t (1/USER_HZ seconds). * These are all stored as 0.64 fixed-point binary fractions. */ u64 __cputime_jiffies_factor; EXPORT_SYMBOL(__cputime_jiffies_factor); u64 __cputime_usec_factor; EXPORT_SYMBOL(__cputime_usec_factor); u64 __cputime_sec_factor; EXPORT_SYMBOL(__cputime_sec_factor); u64 __cputime_clockt_factor; EXPORT_SYMBOL(__cputime_clockt_factor); DEFINE_PER_CPU(unsigned long, cputime_last_delta); DEFINE_PER_CPU(unsigned long, cputime_scaled_last_delta); cputime_t cputime_one_jiffy; void (*dtl_consumer)(struct dtl_entry *, u64); static void calc_cputime_factors(void) { struct div_result res; div128_by_32(HZ, 0, tb_ticks_per_sec, &res); __cputime_jiffies_factor = res.result_low; div128_by_32(1000000, 0, tb_ticks_per_sec, &res); __cputime_usec_factor = res.result_low; div128_by_32(1, 0, tb_ticks_per_sec, &res); __cputime_sec_factor = res.result_low; div128_by_32(USER_HZ, 0, tb_ticks_per_sec, &res); __cputime_clockt_factor = res.result_low; } /* * Read the SPURR on systems that have it, otherwise the PURR, * or if that doesn't exist return the timebase value passed in. */ static u64 read_spurr(u64 tb) { if (cpu_has_feature(CPU_FTR_SPURR)) return mfspr(SPRN_SPURR); if (cpu_has_feature(CPU_FTR_PURR)) return mfspr(SPRN_PURR); return tb; } #ifdef CONFIG_PPC_SPLPAR /* * Scan the dispatch trace log and count up the stolen time. * Should be called with interrupts disabled. */ static u64 scan_dispatch_log(u64 stop_tb) { u64 i = local_paca->dtl_ridx; struct dtl_entry *dtl = local_paca->dtl_curr; struct dtl_entry *dtl_end = local_paca->dispatch_log_end; struct lppaca *vpa = local_paca->lppaca_ptr; u64 tb_delta; u64 stolen = 0; u64 dtb; if (!dtl) return 0; if (i == be64_to_cpu(vpa->dtl_idx)) return 0; while (i < be64_to_cpu(vpa->dtl_idx)) { dtb = be64_to_cpu(dtl->timebase); tb_delta = be32_to_cpu(dtl->enqueue_to_dispatch_time) + be32_to_cpu(dtl->ready_to_enqueue_time); barrier(); if (i + N_DISPATCH_LOG < be64_to_cpu(vpa->dtl_idx)) { /* buffer has overflowed */ i = be64_to_cpu(vpa->dtl_idx) - N_DISPATCH_LOG; dtl = local_paca->dispatch_log + (i % N_DISPATCH_LOG); continue; } if (dtb > stop_tb) break; if (dtl_consumer) dtl_consumer(dtl, i); stolen += tb_delta; ++i; ++dtl; if (dtl == dtl_end) dtl = local_paca->dispatch_log; } local_paca->dtl_ridx = i; local_paca->dtl_curr = dtl; return stolen; } /* * Accumulate stolen time by scanning the dispatch trace log. * Called on entry from user mode. */ void accumulate_stolen_time(void) { u64 sst, ust; u8 save_soft_enabled = local_paca->soft_enabled; /* We are called early in the exception entry, before * soft/hard_enabled are sync'ed to the expected state * for the exception. We are hard disabled but the PACA * needs to reflect that so various debug stuff doesn't * complain */ local_paca->soft_enabled = 0; sst = scan_dispatch_log(local_paca->starttime_user); ust = scan_dispatch_log(local_paca->starttime); local_paca->system_time -= sst; local_paca->user_time -= ust; local_paca->stolen_time += ust + sst; local_paca->soft_enabled = save_soft_enabled; } static inline u64 calculate_stolen_time(u64 stop_tb) { u64 stolen = 0; if (get_paca()->dtl_ridx != be64_to_cpu(get_lppaca()->dtl_idx)) { stolen = scan_dispatch_log(stop_tb); get_paca()->system_time -= stolen; } stolen += get_paca()->stolen_time; get_paca()->stolen_time = 0; return stolen; } #else /* CONFIG_PPC_SPLPAR */ static inline u64 calculate_stolen_time(u64 stop_tb) { return 0; } #endif /* CONFIG_PPC_SPLPAR */ /* * Account time for a transition between system, hard irq * or soft irq state. */ static u64 vtime_delta(struct task_struct *tsk, u64 *sys_scaled, u64 *stolen) { u64 now, nowscaled, deltascaled; u64 udelta, delta, user_scaled; WARN_ON_ONCE(!irqs_disabled()); now = mftb(); nowscaled = read_spurr(now); get_paca()->system_time += now - get_paca()->starttime; get_paca()->starttime = now; deltascaled = nowscaled - get_paca()->startspurr; get_paca()->startspurr = nowscaled; *stolen = calculate_stolen_time(now); delta = get_paca()->system_time; get_paca()->system_time = 0; udelta = get_paca()->user_time - get_paca()->utime_sspurr; get_paca()->utime_sspurr = get_paca()->user_time; /* * Because we don't read the SPURR on every kernel entry/exit, * deltascaled includes both user and system SPURR ticks. * Apportion these ticks to system SPURR ticks and user * SPURR ticks in the same ratio as the system time (delta) * and user time (udelta) values obtained from the timebase * over the same interval. The system ticks get accounted here; * the user ticks get saved up in paca->user_time_scaled to be * used by account_process_tick. */ *sys_scaled = delta; user_scaled = udelta; if (deltascaled != delta + udelta) { if (udelta) { *sys_scaled = deltascaled * delta / (delta + udelta); user_scaled = deltascaled - *sys_scaled; } else { *sys_scaled = deltascaled; } } get_paca()->user_time_scaled += user_scaled; return delta; } void vtime_account_system(struct task_struct *tsk) { u64 delta, sys_scaled, stolen; delta = vtime_delta(tsk, &sys_scaled, &stolen); account_system_time(tsk, 0, delta, sys_scaled); if (stolen) account_steal_time(stolen); } EXPORT_SYMBOL_GPL(vtime_account_system); void vtime_account_idle(struct task_struct *tsk) { u64 delta, sys_scaled, stolen; delta = vtime_delta(tsk, &sys_scaled, &stolen); account_idle_time(delta + stolen); } /* * Transfer the user time accumulated in the paca * by the exception entry and exit code to the generic * process user time records. * Must be called with interrupts disabled. * Assumes that vtime_account_system/idle() has been called * recently (i.e. since the last entry from usermode) so that * get_paca()->user_time_scaled is up to date. */ void vtime_account_user(struct task_struct *tsk) { cputime_t utime, utimescaled; utime = get_paca()->user_time; utimescaled = get_paca()->user_time_scaled; get_paca()->user_time = 0; get_paca()->user_time_scaled = 0; get_paca()->utime_sspurr = 0; account_user_time(tsk, utime, utimescaled); } #else /* ! CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ #define calc_cputime_factors() #endif void __delay(unsigned long loops) { unsigned long start; int diff; if (__USE_RTC()) { start = get_rtcl(); do { /* the RTCL register wraps at 1000000000 */ diff = get_rtcl() - start; if (diff < 0) diff += 1000000000; } while (diff < loops); } else { start = get_tbl(); while (get_tbl() - start < loops) HMT_low(); HMT_medium(); } } EXPORT_SYMBOL(__delay); void udelay(unsigned long usecs) { __delay(tb_ticks_per_usec * usecs); } EXPORT_SYMBOL(udelay); #ifdef CONFIG_SMP unsigned long profile_pc(struct pt_regs *regs) { unsigned long pc = instruction_pointer(regs); if (in_lock_functions(pc)) return regs->link; return pc; } EXPORT_SYMBOL(profile_pc); #endif #ifdef CONFIG_IRQ_WORK /* * 64-bit uses a byte in the PACA, 32-bit uses a per-cpu variable... */ #ifdef CONFIG_PPC64 static inline unsigned long test_irq_work_pending(void) { unsigned long x; asm volatile("lbz %0,%1(13)" : "=r" (x) : "i" (offsetof(struct paca_struct, irq_work_pending))); return x; } static inline void set_irq_work_pending_flag(void) { asm volatile("stb %0,%1(13)" : : "r" (1), "i" (offsetof(struct paca_struct, irq_work_pending))); } static inline void clear_irq_work_pending(void) { asm volatile("stb %0,%1(13)" : : "r" (0), "i" (offsetof(struct paca_struct, irq_work_pending))); } #else /* 32-bit */ DEFINE_PER_CPU(u8, irq_work_pending); #define set_irq_work_pending_flag() __get_cpu_var(irq_work_pending) = 1 #define test_irq_work_pending() __get_cpu_var(irq_work_pending) #define clear_irq_work_pending() __get_cpu_var(irq_work_pending) = 0 #endif /* 32 vs 64 bit */ void arch_irq_work_raise(void) { preempt_disable(); set_irq_work_pending_flag(); set_dec(1); preempt_enable(); } #else /* CONFIG_IRQ_WORK */ #define test_irq_work_pending() 0 #define clear_irq_work_pending() #endif /* CONFIG_IRQ_WORK */ /* * timer_interrupt - gets called when the decrementer overflows, * with interrupts disabled. */ void timer_interrupt(struct pt_regs * regs) { struct pt_regs *old_regs; u64 *next_tb = &__get_cpu_var(decrementers_next_tb); struct clock_event_device *evt = &__get_cpu_var(decrementers); u64 now; /* Ensure a positive value is written to the decrementer, or else * some CPUs will continue to take decrementer exceptions. */ set_dec(DECREMENTER_MAX); /* Some implementations of hotplug will get timer interrupts while * offline, just ignore these and we also need to set * decrementers_next_tb as MAX to make sure __check_irq_replay * don't replay timer interrupt when return, otherwise we'll trap