// SPDX-License-Identifier: GPL-2.0-or-later/* * Common time routines among all ppc machines. * * Written by Cort Dougan (cort@cs.nmt.edu) to merge * Paul Mackerras' version and mine for PReP and Pmac. * MPC8xx/MBX changes by Dan Malek (dmalek@jlc.net). * Converted for 64-bit by Mike Corrigan (mikejc@us.ibm.com) * * First round of bugfixes by Gabriel Paubert (paubert@iram.es) * to make clock more stable (2.4.0-test5). The only thing * that this code assumes is that the timebases have been synchronized * by firmware on SMP and are never stopped (never do sleep * on SMP then, nap and doze are OK). * * Speeded up do_gettimeofday by getting rid of references to * xtime (which required locks for consistency). (mikejc@us.ibm.com) * * TODO (not necessarily in this file): * - improve precision and reproducibility of timebase frequency * measurement at boot time. * - for astronomical applications: add a new function to get * non ambiguous timestamps even around leap seconds. This needs * a new timestamp format and a good name. * * 1997-09-10 Updated NTP code according to technical memorandum Jan '96 * "A Kernel Model for Precision Timekeeping" by Dave Mills */ #include <linux/errno.h>#include <linux/export.h>#include <linux/sched.h>#include <linux/sched/clock.h>#include <linux/sched/cputime.h>#include <linux/kernel.h>#include <linux/param.h>#include <linux/string.h>#include <linux/mm.h>#include <linux/interrupt.h>#include <linux/timex.h>#include <linux/kernel_stat.h>#include <linux/time.h>#include <linux/init.h>#include <linux/profile.h>#include <linux/cpu.h>#include <linux/security.h>#include <linux/percpu.h>#include <linux/rtc.h>#include <linux/jiffies.h>#include <linux/posix-timers.h>#include <linux/irq.h>#include <linux/delay.h>#include <linux/irq_work.h>#include <linux/of_clk.h>#include <linux/suspend.h>#include <linux/processor.h>#include <linux/mc146818rtc.h>#include <linux/platform_device.h> #include <asm/trace.h>#include <asm/interrupt.h>#include <asm/io.h>#include <asm/nvram.h>#include <asm/cache.h>#include <asm/machdep.h>#include <linux/uaccess.h>#include <asm/time.h>#include <asm/irq.h>#include <asm/div64.h>#include <asm/smp.h>#include <asm/vdso_datapage.h>#include <asm/firmware.h>#include <asm/mce.h>#include <asm/systemcfg.h> /* powerpc clocksource/clockevent code */ #include <linux/clockchips.h> static u64 timebase_read(struct clocksource *);static struct clocksource clocksource_timebase = { .name = "timebase", .rating = 400, .flags = CLOCK_SOURCE_IS_CONTINUOUS, .mask = CLOCKSOURCE_MASK(64), .read = timebase_read, .vdso_clock_mode = VDSO_CLOCKMODE_ARCHTIMER,}; #define DECREMENTER_DEFAULT_MAX 0x7FFFFFFFu64 decrementer_max = DECREMENTER_DEFAULT_MAX;EXPORT_SYMBOL_GPL(decrementer_max); /* for KVM HDEC */ static int decrementer_set_next_event(unsigned long evt, struct clock_event_device *dev);static int decrementer_shutdown(struct clock_event_device *evt); struct clock_event_device decrementer_clockevent = { .name = "decrementer", .rating = 200, .irq = 0, .set_next_event = decrementer_set_next_event, .set_state_oneshot_stopped = decrementer_shutdown, .set_state_shutdown = decrementer_shutdown, .tick_resume = decrementer_shutdown, .features = CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_C3STOP,};EXPORT_SYMBOL(decrementer_clockevent); /* * This always puts next_tb beyond now, so the clock event will never fire * with the usual comparison, no need for a separate test for stopped. */#define DEC_CLOCKEVENT_STOPPED ~0ULLDEFINE_PER_CPU(u64, decrementers_next_tb) = DEC_CLOCKEVENT_STOPPED;EXPORT_SYMBOL_GPL(decrementers_next_tb);static DEFINE_PER_CPU(struct clock_event_device, decrementers); #define XSEC_PER_SEC (1024*1024) #ifdef CONFIG_PPC64#define SCALE_XSEC(xsec, max) (((xsec) * max) / XSEC_PER_SEC)#else/* compute ((xsec << 12) * max) >> 32 */#define SCALE_XSEC(xsec, max) mulhwu((xsec) << 12, max)#endif unsigned long tb_ticks_per_jiffy;unsigned long tb_ticks_per_usec = 100; /* sane default */EXPORT_SYMBOL(tb_ticks_per_usec);unsigned long tb_ticks_per_sec;EXPORT_SYMBOL(tb_ticks_per_sec); /* for cputime conversions */ DEFINE_SPINLOCK(rtc_lock);EXPORT_SYMBOL_GPL(rtc_lock); static u64 tb_to_ns_scale __read_mostly;static unsigned tb_to_ns_shift __read_mostly;static u64 boot_tb __ro_after_init; extern struct timezone sys_tz;static long timezone_offset; unsigned long ppc_proc_freq;EXPORT_SYMBOL_GPL(ppc_proc_freq);unsigned long ppc_tb_freq;EXPORT_SYMBOL_GPL(ppc_tb_freq); bool tb_invalid; #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE/* * Read the SPURR on systems that have it, otherwise the PURR, * or if that doesn't exist return the timebase value passed in. */static inline unsigned long read_spurr(unsigned long tb){ if (cpu_has_feature(CPU_FTR_SPURR)) return mfspr(SPRN_SPURR); if (cpu_has_feature(CPU_FTR_PURR)) return mfspr(SPRN_PURR); return tb;} /* * Account time for a transition between system, hard irq * or soft irq state. */static unsigned long vtime_delta_scaled(struct cpu_accounting_data *acct, unsigned long now, unsigned long stime){ unsigned long stime_scaled = 0;#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME unsigned long nowscaled, deltascaled; unsigned long utime, utime_scaled; nowscaled = read_spurr(now); deltascaled = nowscaled - acct->startspurr; acct->startspurr = nowscaled; utime = acct->utime - acct->utime_sspurr; acct->utime_sspurr = acct->utime; /* * Because we don't read the SPURR on every kernel entry/exit, * deltascaled includes both user and system SPURR ticks. * Apportion these ticks to system SPURR ticks and user * SPURR ticks in the same ratio as the system time (delta) * and user time (udelta) values obtained from the timebase * over the same interval. The system ticks get accounted here; * the user ticks get saved up in paca->user_time_scaled to be * used by account_process_tick. */ stime_scaled = stime; utime_scaled = utime; if (deltascaled != stime + utime) { if (utime) { stime_scaled = deltascaled * stime / (stime + utime); utime_scaled = deltascaled - stime_scaled; } else { stime_scaled = deltascaled; } } acct->utime_scaled += utime_scaled;#endif return stime_scaled;} static unsigned long vtime_delta(struct cpu_accounting_data *acct, unsigned long *stime_scaled, unsigned long *steal_time){ unsigned long now, stime; WARN_ON_ONCE(!irqs_disabled()); now = mftb(); stime = now - acct->starttime; acct->starttime = now; *stime_scaled = vtime_delta_scaled(acct, now, stime); if (IS_ENABLED(CONFIG_PPC_SPLPAR) && firmware_has_feature(FW_FEATURE_SPLPAR)) *steal_time = pseries_calculate_stolen_time(now); else *steal_time = 0; return stime;} static void vtime_delta_kernel(struct cpu_accounting_data *acct, unsigned long *stime, unsigned long *stime_scaled){ unsigned long steal_time; *stime = vtime_delta(acct, stime_scaled, &steal_time); *stime -= min(*stime, steal_time); acct->steal_time += steal_time;} void vtime_account_kernel(struct task_struct *tsk){ struct cpu_accounting_data *acct = get_accounting(tsk); unsigned long stime, stime_scaled; vtime_delta_kernel(acct, &stime, &stime_scaled); if (tsk->flags & PF_VCPU) { acct->gtime += stime;#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME acct->utime_scaled += stime_scaled;#endif } else { acct->stime += stime;#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME acct->stime_scaled += stime_scaled;#endif }}EXPORT_SYMBOL_GPL(vtime_account_kernel); void vtime_account_idle(struct task_struct *tsk){ unsigned long stime, stime_scaled, steal_time; struct cpu_accounting_data *acct = get_accounting(tsk); stime = vtime_delta(acct, &stime_scaled, &steal_time); acct->idle_time += stime + steal_time;} static void vtime_account_irq_field(struct cpu_accounting_data *acct, unsigned long *field){ unsigned long stime, stime_scaled; vtime_delta_kernel(acct, &stime, &stime_scaled); *field += stime;#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME acct->stime_scaled += stime_scaled;#endif} void vtime_account_softirq(struct task_struct *tsk){ struct cpu_accounting_data *acct = get_accounting(tsk); vtime_account_irq_field(acct, &acct->softirq_time);} void vtime_account_hardirq(struct task_struct *tsk){ struct cpu_accounting_data *acct = get_accounting(tsk); vtime_account_irq_field(acct, &acct->hardirq_time);} static void vtime_flush_scaled(struct task_struct *tsk, struct cpu_accounting_data *acct){#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME if (acct->utime_scaled) tsk->utimescaled += cputime_to_nsecs(acct->utime_scaled); if (acct->stime_scaled) tsk->stimescaled += cputime_to_nsecs(acct->stime_scaled); acct->utime_scaled = 0; acct->utime_sspurr = 0; acct->stime_scaled = 0;#endif} /* * Account the whole cputime accumulated in the paca * Must be called with interrupts disabled. * Assumes that vtime_account_kernel/idle() has been called * recently (i.e. since the last entry from usermode) so that * get_paca()->user_time_scaled is up to date. */void vtime_flush(struct task_struct *tsk){ struct cpu_accounting_data *acct = get_accounting(tsk); if (acct->utime) account_user_time(tsk, cputime_to_nsecs(acct->utime)); if (acct->gtime) account_guest_time(tsk, cputime_to_nsecs(acct->gtime)); if (IS_ENABLED(CONFIG_PPC_SPLPAR) && acct->steal_time) { account_steal_time(cputime_to_nsecs(acct->steal_time)); acct->steal_time = 0; } if (acct->idle_time) account_idle_time(cputime_to_nsecs(acct->idle_time)); if (acct->stime) account_system_index_time(tsk, cputime_to_nsecs(acct->stime), CPUTIME_SYSTEM); if (acct->hardirq_time) account_system_index_time(tsk, cputime_to_nsecs(acct->hardirq_time), CPUTIME_IRQ); if (acct->softirq_time) account_system_index_time(tsk, cputime_to_nsecs(acct->softirq_time), CPUTIME_SOFTIRQ); vtime_flush_scaled(tsk, acct); acct->utime = 0; acct->gtime = 0; acct->idle_time = 0; acct->stime = 0; acct->hardirq_time = 0; acct->softirq_time = 0;} /* * Called from the context switch with interrupts disabled, to charge all * accumulated times to the current process, and to prepare accounting on * the next process. */void vtime_task_switch(struct task_struct *prev){ if (is_idle_task(prev)) vtime_account_idle(prev); else vtime_account_kernel(prev); vtime_flush(prev); if (!IS_ENABLED(CONFIG_PPC64)) { struct cpu_accounting_data *acct = get_accounting(current); struct cpu_accounting_data *acct0 = get_accounting(prev); acct->starttime = acct0->starttime; }} /** * vtime_reset - Fast forward vtime entry clocks * * Called from dynticks idle IRQ entry to fast-forward the clocks to current time * so that the IRQ time is still accounted by vtime while nohz cputime is paused. */void vtime_reset(void){ struct cpu_accounting_data *acct = get_accounting(current); acct->starttime = mftb();#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME acct->startspurr = read_spurr(acct->starttime);#endif} #ifdef CONFIG_NO_HZ_COMMON/** * vtime_dyntick_start - Inform vtime about entry to idle-dynticks * * Called when idle enters in dyntick mode. The idle cputime that elapsed so far * is accumulated and the tick subsystem takes over the idle cputime accounting. */void vtime_dyntick_start(void){ vtime_account_idle(current);} /** * vtime_dyntick_stop - Inform vtime about exit from idle-dynticks * * Called when idle exits from dyntick mode. The vtime entry clocks are * fast-forward to current time so that idle accounting restarts elapsing from * now. */void vtime_dyntick_stop(void){ vtime_reset();}#endif /* CONFIG_NO_HZ_COMMON */#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ void __no_kcsan __delay(unsigned long loops){ unsigned long start; spin_begin(); if (tb_invalid) { /* * TB is in error state and isn't ticking anymore. * HMI handler was unable to recover from TB error. * Return immediately, so that kernel won't get stuck here. */ spin_cpu_relax(); } else { start = mftb(); while (mftb() - start < loops) spin_cpu_relax(); } spin_end();}EXPORT_SYMBOL(__delay); void __no_kcsan udelay(unsigned long usecs){ __delay(tb_ticks_per_usec * usecs);}EXPORT_SYMBOL(udelay); #ifdef CONFIG_SMPunsigned long profile_pc(struct pt_regs *regs){ unsigned long pc = instruction_pointer(regs); if (in_lock_functions(pc)) return regs->link; return pc;}EXPORT_SYMBOL(profile_pc);#endif #ifdef CONFIG_IRQ_WORK /* * 64-bit uses a byte in the PACA, 32-bit uses a per-cpu variable... */#ifdef CONFIG_PPC64static inline unsigned long test_irq_work_pending(void){ unsigned long x; asm volatile("lbz %0,%1(13)" : "=r" (x) : "i" (offsetof(struct paca_struct, irq_work_pending))); return x;} static inline void set_irq_work_pending_flag(void){ asm volatile("stb %0,%1(13)" : : "r" (1), "i" (offsetof(struct paca_struct, irq_work_pending)));} static inline void clear_irq_work_pending(void){ asm volatile("stb %0,%1(13)" : : "r" (0), "i" (offsetof(struct paca_struct, irq_work_pending)));} #else /* 32-bit */ DEFINE_PER_CPU(u8, irq_work_pending); #define set_irq_work_pending_flag() __this_cpu_write(irq_work_pending, 1)#define test_irq_work_pending() __this_cpu_read(irq_work_pending)#define clear_irq_work_pending() __this_cpu_write(irq_work_pending, 0) #endif /* 32 vs 64 bit */Lines 1–500 of 1107. Load more
// SPDX-License-Identifier: GPL-2.0-or-later /* * Common time routines among all ppc machines. * * Written by Cort Dougan (cort@cs.nmt.edu) to merge * Paul Mackerras' version and mine for PReP and Pmac. * MPC8xx/MBX changes by Dan Malek (dmalek@jlc.net). * Converted for 64-bit by Mike Corrigan (mikejc@us.ibm.com) * * First round of bugfixes by Gabriel Paubert (paubert@iram.es) * to make clock more stable (2.4.0-test5). The only thing * that this code assumes is that the timebases have been synchronized * by firmware on SMP and are never stopped (never do sleep * on SMP then, nap and doze are OK). * * Speeded up do_gettimeofday by getting rid of references to * xtime (which required locks for consistency). (mikejc@us.ibm.com) * * TODO (not necessarily in this file): * - improve precision and reproducibility of timebase frequency * measurement at boot time. * - for astronomical applications: add a new function to get * non ambiguous timestamps even around leap seconds. This needs * a new timestamp format and a good name. * * 1997-09-10 Updated NTP code according to technical memorandum Jan '96 * "A Kernel Model for Precision Timekeeping" by Dave Mills */ #include <linux/errno.h> #include <linux/export.h> #include <linux/sched.h> #include <linux/sched/clock.h> #include <linux/sched/cputime.h> #include <linux/kernel.h> #include <linux/param.h> #include <linux/string.h> #include <linux/mm.h> #include <linux/interrupt.h> #include <linux/timex.h> #include <linux/kernel_stat.h> #include <linux/time.h> #include <linux/init.h> #include <linux/profile.h> #include <linux/cpu.h> #include <linux/security.h> #include <linux/percpu.h> #include <linux/rtc.h> #include <linux/jiffies.h> #include <linux/posix-timers.h> #include <linux/irq.h> #include <linux/delay.h> #include <linux/irq_work.h> #include <linux/of_clk.h> #include <linux/suspend.h> #include <linux/processor.h> #include <linux/mc146818rtc.h> #include <linux/platform_device.h> #include <asm/trace.h> #include <asm/interrupt.h> #include <asm/io.h> #include <asm/nvram.h> #include <asm/cache.h> #include <asm/machdep.h> #include <linux/uaccess.h> #include <asm/time.h> #include <asm/irq.h> #include <asm/div64.h> #include <asm/smp.h> #include <asm/vdso_datapage.h> #include <asm/firmware.h> #include <asm/mce.h> #include <asm/systemcfg.h> /* powerpc clocksource/clockevent code */ #include <linux/clockchips.h> static u64 timebase_read(struct clocksource *); static struct clocksource clocksource_timebase = { .name = "timebase", .rating = 400, .flags = CLOCK_SOURCE_IS_CONTINUOUS, .mask = CLOCKSOURCE_MASK(64), .read = timebase_read, .vdso_clock_mode = VDSO_CLOCKMODE_ARCHTIMER, }; #define DECREMENTER_DEFAULT_MAX 0x7FFFFFFF u64 decrementer_max = DECREMENTER_DEFAULT_MAX; EXPORT_SYMBOL_GPL(decrementer_max); /* for KVM HDEC */ static int decrementer_set_next_event(unsigned long evt, struct clock_event_device *dev); static int decrementer_shutdown(struct clock_event_device *evt); struct clock_event_device decrementer_clockevent = { .name = "decrementer", .rating = 200, .irq = 0, .set_next_event = decrementer_set_next_event, .set_state_oneshot_stopped = decrementer_shutdown, .set_state_shutdown = decrementer_shutdown, .tick_resume = decrementer_shutdown, .features = CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_C3STOP, }; EXPORT_SYMBOL(decrementer_clockevent); /* * This always puts next_tb beyond now, so the clock event will never fire * with the usual comparison, no need for a separate test for stopped. */ #define DEC_CLOCKEVENT_STOPPED ~0ULL DEFINE_PER_CPU(u64, decrementers_next_tb) = DEC_CLOCKEVENT_STOPPED; EXPORT_SYMBOL_GPL(decrementers_next_tb); static DEFINE_PER_CPU(struct clock_event_device, decrementers); #define XSEC_PER_SEC (1024*1024) #ifdef CONFIG_PPC64 #define SCALE_XSEC(xsec, max) (((xsec) * max) / XSEC_PER_SEC) #else /* compute ((xsec << 12) * max) >> 32 */ #define SCALE_XSEC(xsec, max) mulhwu((xsec) << 12, max) #endif unsigned long tb_ticks_per_jiffy; unsigned long tb_ticks_per_usec = 100; /* sane default */ EXPORT_SYMBOL(tb_ticks_per_usec); unsigned long tb_ticks_per_sec; EXPORT_SYMBOL(tb_ticks_per_sec); /* for cputime conversions */ DEFINE_SPINLOCK(rtc_lock); EXPORT_SYMBOL_GPL(rtc_lock); static u64 tb_to_ns_scale __read_mostly; static unsigned tb_to_ns_shift __read_mostly; static u64 boot_tb __ro_after_init; extern struct timezone sys_tz; static long timezone_offset; unsigned long ppc_proc_freq; EXPORT_SYMBOL_GPL(ppc_proc_freq); unsigned long ppc_tb_freq; EXPORT_SYMBOL_GPL(ppc_tb_freq); bool tb_invalid; #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE /* * Read the SPURR on systems that have it, otherwise the PURR, * or if that doesn't exist return the timebase value passed in. */ static inline unsigned long read_spurr(unsigned long tb) { if (cpu_has_feature(CPU_FTR_SPURR)) return mfspr(SPRN_SPURR); if (cpu_has_feature(CPU_FTR_PURR)) return mfspr(SPRN_PURR); return tb; } /* * Account time for a transition between system, hard irq * or soft irq state. */ static unsigned long vtime_delta_scaled(struct cpu_accounting_data *acct, unsigned long now, unsigned long stime) { unsigned long stime_scaled = 0; #ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME unsigned long nowscaled, deltascaled; unsigned long utime, utime_scaled; nowscaled = read_spurr(now); deltascaled = nowscaled - acct->startspurr; acct->startspurr = nowscaled; utime = acct->utime - acct->utime_sspurr; acct->utime_sspurr = acct->utime; /* * Because we don't read the SPURR on every kernel entry/exit, * deltascaled includes both user and system SPURR ticks. * Apportion these ticks to system SPURR ticks and user * SPURR ticks in the same ratio as the system time (delta) * and user time (udelta) values obtained from the timebase * over the same interval. The system ticks get accounted here; * the user ticks get saved up in paca->user_time_scaled to be * used by account_process_tick. */ stime_scaled = stime; utime_scaled = utime; if (deltascaled != stime + utime) { if (utime) { stime_scaled = deltascaled * stime / (stime + utime); utime_scaled = deltascaled - stime_scaled; } else { stime_scaled = deltascaled; } } acct->utime_scaled += utime_scaled; #endif return stime_scaled; } static unsigned long vtime_delta(struct cpu_accounting_data *acct, unsigned long *stime_scaled, unsigned long *steal_time) { unsigned long now, stime; WARN_ON_ONCE(!irqs_disabled()); now = mftb(); stime = now - acct->starttime; acct->starttime = now; *stime_scaled = vtime_delta_scaled(acct, now, stime); if (IS_ENABLED(CONFIG_PPC_SPLPAR) && firmware_has_feature(FW_FEATURE_SPLPAR)) *steal_time = pseries_calculate_stolen_time(now); else *steal_time = 0; return stime; } static void vtime_delta_kernel(struct cpu_accounting_data *acct, unsigned long *stime, unsigned long *stime_scaled) { unsigned long steal_time; *stime = vtime_delta(acct, stime_scaled, &steal_time); *stime -= min(*stime, steal_time); acct->steal_time += steal_time; } void vtime_account_kernel(struct task_struct *tsk) { struct cpu_accounting_data *acct = get_accounting(tsk); unsigned long stime, stime_scaled; vtime_delta_kernel(acct, &stime, &stime_scaled); if (tsk->flags & PF_VCPU) { acct->gtime += stime; #ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME acct->utime_scaled += stime_scaled; #endif } else { acct->stime += stime; #ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME acct->stime_scaled += stime_scaled; #endif } } EXPORT_SYMBOL_GPL(vtime_account_kernel); void vtime_account_idle(struct task_struct *tsk) { unsigned long stime, stime_scaled, steal_time; struct cpu_accounting_data *acct = get_accounting(tsk); stime = vtime_delta(acct, &stime_scaled, &steal_time); acct->idle_time += stime + steal_time; } static void vtime_account_irq_field(struct cpu_accounting_data *acct, unsigned long *field) { unsigned long stime, stime_scaled; vtime_delta_kernel(acct, &stime, &stime_scaled); *field += stime; #ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME acct->stime_scaled += stime_scaled; #endif } void vtime_account_softirq(struct task_struct *tsk) { struct cpu_accounting_data *acct = get_accounting(tsk); vtime_account_irq_field(acct, &acct->softirq_time); } void vtime_account_hardirq(struct task_struct *tsk) { struct cpu_accounting_data *acct = get_accounting(tsk); vtime_account_irq_field(acct, &acct->hardirq_time); } static void vtime_flush_scaled(struct task_struct *tsk, struct cpu_accounting_data *acct) { #ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME if (acct->utime_scaled) tsk->utimescaled += cputime_to_nsecs(acct->utime_scaled); if (acct->stime_scaled) tsk->stimescaled += cputime_to_nsecs(acct->stime_scaled); acct->utime_scaled = 0; acct->utime_sspurr = 0; acct->stime_scaled = 0; #endif } /* * Account the whole cputime accumulated in the paca * Must be called with interrupts disabled. * Assumes that vtime_account_kernel/idle() has been called * recently (i.e. since the last entry from usermode) so that * get_paca()->user_time_scaled is up to date. */ void vtime_flush(struct task_struct *tsk) { struct cpu_accounting_data *acct = get_accounting(tsk); if (acct->utime) account_user_time(tsk, cputime_to_nsecs(acct->utime)); if (acct->gtime) account_guest_time(tsk, cputime_to_nsecs(acct->gtime)); if (IS_ENABLED(CONFIG_PPC_SPLPAR) && acct->steal_time) { account_steal_time(cputime_to_nsecs(acct->steal_time)); acct->steal_time = 0; } if (acct->idle_time) account_idle_time(cputime_to_nsecs(acct->idle_time)); if (acct->stime) account_system_index_time(tsk, cputime_to_nsecs(acct->stime), CPUTIME_SYSTEM); if (acct->hardirq_time) account_system_index_time(tsk, cputime_to_nsecs(acct->hardirq_time), CPUTIME_IRQ); if (acct->softirq_time) account_system_index_time(tsk, cputime_to_nsecs(acct->softirq_time), CPUTIME_SOFTIRQ); vtime_flush_scaled(tsk, acct); acct->utime = 0; acct->gtime = 0; acct->idle_time = 0; acct->stime = 0; acct->hardirq_time = 0; acct->softirq_time = 0; } /* * Called from the context switch with interrupts disabled, to charge all * accumulated times to the current process, and to prepare accounting on * the next process. */ void vtime_task_switch(struct task_struct *prev) { if (is_idle_task(prev)) vtime_account_idle(prev); else vtime_account_kernel(prev); vtime_flush(prev); if (!IS_ENABLED(CONFIG_PPC64)) { struct cpu_accounting_data *acct = get_accounting(current); struct cpu_accounting_data *acct0 = get_accounting(prev); acct->starttime = acct0->starttime; } } /** * vtime_reset - Fast forward vtime entry clocks * * Called from dynticks idle IRQ entry to fast-forward the clocks to current time * so that the IRQ time is still accounted by vtime while nohz cputime is paused. */ void vtime_reset(void) { struct cpu_accounting_data *acct = get_accounting(current); acct->starttime = mftb(); #ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME acct->startspurr = read_spurr(acct->starttime); #endif } #ifdef CONFIG_NO_HZ_COMMON /** * vtime_dyntick_start - Inform vtime about entry to idle-dynticks * * Called when idle enters in dyntick mode. The idle cputime that elapsed so far * is accumulated and the tick subsystem takes over the idle cputime accounting. */ void vtime_dyntick_start(void) { vtime_account_idle(current); } /** * vtime_dyntick_stop - Inform vtime about exit from idle-dynticks * * Called when idle exits from dyntick mode. The vtime entry clocks are * fast-forward to current time so that idle accounting restarts elapsing from * now. */ void vtime_dyntick_stop(void) { vtime_reset(); } #endif /* CONFIG_NO_HZ_COMMON */ #endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ void __no_kcsan __delay(unsigned long loops) { unsigned long start; spin_begin(); if (tb_invalid) { /* * TB is in error state and isn't ticking anymore. * HMI handler was unable to recover from TB error. * Return immediately, so that kernel won't get stuck here. */ spin_cpu_relax(); } else { start = mftb(); while (mftb() - start < loops) spin_cpu_relax(); } spin_end(); } EXPORT_SYMBOL(__delay); void __no_kcsan udelay(unsigned long usecs) { __delay(tb_ticks_per_usec * usecs); } EXPORT_SYMBOL(udelay); #ifdef CONFIG_SMP unsigned long profile_pc(struct pt_regs *regs) { unsigned long pc = instruction_pointer(regs); if (in_lock_functions(pc)) return regs->link; return pc; } EXPORT_SYMBOL(profile_pc); #endif #ifdef CONFIG_IRQ_WORK /* * 64-bit uses a byte in the PACA, 32-bit uses a per-cpu variable... */ #ifdef CONFIG_PPC64 static inline unsigned long test_irq_work_pending(void) { unsigned long x; asm volatile("lbz %0,%1(13)" : "=r" (x) : "i" (offsetof(struct paca_struct, irq_work_pending))); return x; } static inline void set_irq_work_pending_flag(void) { asm volatile("stb %0,%1(13)" : : "r" (1), "i" (offsetof(struct paca_struct, irq_work_pending))); } static inline void clear_irq_work_pending(void) { asm volatile("stb %0,%1(13)" : : "r" (0), "i" (offsetof(struct paca_struct, irq_work_pending))); } #else /* 32-bit */ DEFINE_PER_CPU(u8, irq_work_pending); #define set_irq_work_pending_flag() __this_cpu_write(irq_work_pending, 1) #define test_irq_work_pending() __this_cpu_read(irq_work_pending) #define clear_irq_work_pending() __this_cpu_write(irq_work_pending, 0) #endif /* 32 vs 64 bit */