Merge tag 'timers-core-2024-05-12' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull timers and timekeeping updates from Thomas Gleixner: "Core code: - Make timekeeping and VDSO time readouts resilent against math overflow: In guest context the kernel is prone to math overflow when the host defers the timer interrupt due to overload, malfunction or malice. This can be mitigated by checking the clocksource delta for the maximum deferrement which is readily available. If that value is exceeded then the code uses a slowpath function which can handle the multiplication overflow. This functionality is enabled unconditionally in the kernel, but made conditional in the VDSO code. The latter is conditional because it allows architectures to optimize the check so it is not causing performance regressions. On X86 this is achieved by reworking the existing check for negative TSC deltas as a negative delta obviously exceeds the maximum deferrement when it is evaluated as an unsigned value. That avoids two conditionals in the hotpath and allows to hide both the negative delta and the large delta handling in the same slow path. - Add an initial minimal ktime_t abstraction for Rust - The usual boring cleanups and enhancements Drivers: - Boring updates to device trees and trivial enhancements in various drivers" * tag 'timers-core-2024-05-12' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (33 commits) clocksource/drivers/arm_arch_timer: Mark hisi_161010101_oem_info const clocksource/drivers/timer-ti-dm: Remove an unused field in struct dmtimer clocksource/drivers/renesas-ostm: Avoid reprobe after successful early probe clocksource/drivers/renesas-ostm: Allow OSTM driver to reprobe for RZ/V2H(P) SoC dt-bindings: timer: renesas: ostm: Document Renesas RZ/V2H(P) SoC rust: time: doc: Add missing C header links clocksource: Make the int help prompt unit readable in ncurses hrtimer: Rename __hrtimer_hres_active() to hrtimer_hres_active() timerqueue: Remove never used function timerqueue_node_expires() rust: time: Add Ktime vdso: Fix powerpc build U64_MAX undeclared error clockevents: Convert s[n]printf() to sysfs_emit() clocksource: Convert s[n]printf() to sysfs_emit() clocksource: Make watchdog and suspend-timing multiplication overflow safe timekeeping: Let timekeeping_cycles_to_ns() handle both under and overflow timekeeping: Make delta calculation overflow safe timekeeping: Prepare timekeeping_cycles_to_ns() for overflow safety timekeeping: Fold in timekeeping_delta_to_ns() timekeeping: Consolidate timekeeping helpers timekeeping: Refactor timekeeping helpers ...

Linus Torvalds committed 2 years ago

Commit 2d9db772 parents 61deafa a3825a721 files · +284 −173Browse files

Unified · Split

21 files

Documentation/devicetree/bindings/timer/renesas,ostm.yaml +2 −0
arch/powerpc/include/asm/vdso/gettimeofday.h +11 −15
arch/s390/include/asm/vdso/gettimeofday.h +2 −5
arch/x86/Kconfig +1 −0
arch/x86/include/asm/vdso/gettimeofday.h +28 −16
drivers/clocksource/arm_arch_timer.c +1 −1
drivers/clocksource/renesas-ostm.c +2 −1
drivers/clocksource/timer-ti-dm.c +0 −1
include/linux/math64.h +2 −6
include/linux/timerqueue.h +0 −5
include/vdso/datapage.h +4 −0
include/vdso/math64.h +38 −0
kernel/time/Kconfig +1 −1
kernel/time/clockevents.c +1 −1
kernel/time/clocksource.c +21 −23
kernel/time/hrtimer.c +9 −32
kernel/time/timekeeping.c +49 −47
kernel/time/vsyscall.c +6 −0
lib/vdso/Kconfig +7 −0
lib/vdso/gettimeofday.c +36 −19
rust/kernel/time.rs +63 −0

Documentation/devicetree/bindings/timer/renesas,ostm.yaml+2−0

           - renesas,r9a07g043-ostm # RZ/G2UL and RZ/Five
           - renesas,r9a07g044-ostm # RZ/G2{L,LC}
           - renesas,r9a07g054-ostm # RZ/V2L
+          - renesas,r9a09g057-ostm # RZ/V2H(P)
       - const: renesas,ostm        # Generic
   reg:
           - renesas,r9a07g043-ostm
           - renesas,r9a07g044-ostm
           - renesas,r9a07g054-ostm
+          - renesas,r9a09g057-ostm
 then:
   required:
     - resets

arch/powerpc/include/asm/vdso/gettimeofday.h+11−15

 #define VDSO_HAS_TIME			1
+/*
+ * powerpc specific delta calculation.
+ *
+ * This variant removes the masking of the subtraction because the
+ * clocksource mask of all VDSO capable clocksources on powerpc is U64_MAX
+ * which would result in a pointless operation. The compiler cannot
+ * optimize it away as the mask comes from the vdso data and is not compile
+ * time constant.
+ */
+#define VDSO_DELTA_NOMASK		1
 static __always_inline int do_syscall_2(const unsigned long _r0, const unsigned long _r3,
 					const unsigned long _r4)
 {
 }
 #define vdso_clocksource_ok vdso_clocksource_ok
-/*
- * powerpc specific delta calculation.
- *
- * This variant removes the masking of the subtraction because the
- * clocksource mask of all VDSO capable clocksources on powerpc is U64_MAX
- * which would result in a pointless operation. The compiler cannot
- * optimize it away as the mask comes from the vdso data and is not compile
- * time constant.
- */
-static __always_inline u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult)
-{
-	return (cycles - last) * mult;
-}
-#define vdso_calc_delta vdso_calc_delta
 #ifndef __powerpc64__
 static __always_inline u64 vdso_shift_ns(u64 ns, unsigned long shift)
 {

arch/s390/include/asm/vdso/gettimeofday.h+2−5

 #define VDSO_HAS_CLOCK_GETRES 1
+#define VDSO_DELTA_NOMASK 1
 #include <asm/syscall.h>
 #include <asm/timex.h>
 #include <asm/unistd.h>
 #include <linux/compiler.h>
-#define vdso_calc_delta __arch_vdso_calc_delta
-static __always_inline u64 __arch_vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult)
-{
-	return (cycles - last) * mult;
-}
 static __always_inline const struct vdso_data *__arch_get_vdso_data(void)
 {

arch/x86/Kconfig+1−0

@@ -169,6 +169,7 @@ config X86 · 168 unmodified lines
169	select GENERIC_TIME_VSYSCALL	169	select GENERIC_TIME_VSYSCALL
170	select GENERIC_GETTIMEOFDAY	170	select GENERIC_GETTIMEOFDAY
171	select GENERIC_VDSO_TIME_NS	171	select GENERIC_VDSO_TIME_NS
		172	select GENERIC_VDSO_OVERFLOW_PROTECT
172	select GUP_GET_PXX_LOW_HIGH if X86_PAE	173	select GUP_GET_PXX_LOW_HIGH if X86_PAE
173	select HARDIRQS_SW_RESEND	174	select HARDIRQS_SW_RESEND
174	select HARDLOCKUP_CHECK_TIMESTAMP if X86_64	175	select HARDLOCKUP_CHECK_TIMESTAMP if X86_64

arch/x86/include/asm/vdso/gettimeofday.h+28−16

 #define vdso_cycles_ok arch_vdso_cycles_ok
 /*
- * x86 specific delta calculation.
+ * x86 specific calculation of nanoseconds for the current cycle count
  *
  * The regular implementation assumes that clocksource reads are globally
  * monotonic. The TSC can be slightly off across sockets which can cause
  * jump.
  *
  * Therefore it needs to be verified that @cycles are greater than
- * @last. If not then use @last, which is the base time of the current
+ * @vd->cycles_last. If not then use @vd->cycles_last, which is the base
- * conversion period.
+ * time of the current conversion period.
  *
  * This variant also uses a custom mask because while the clocksource mask of
  * all the VDSO capable clocksources on x86 is U64_MAX, the above code uses
  * declares everything with the MSB/Sign-bit set as invalid. Therefore the
  * effective mask is S64_MAX.
  */
-static __always_inline
+static __always_inline u64 vdso_calc_ns(const struct vdso_data *vd, u64 cycles, u64 base)
-u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult)
 {
-	/*
+	u64 delta = cycles - vd->cycle_last;
-	 * Due to the MSB/Sign-bit being used as invalid marker (see
-	 * arch_vdso_cycles_valid() above), the effective mask is S64_MAX.
-	 */
-	u64 delta = (cycles - last) & S64_MAX;
 	/*
-	 * Due to the above mentioned TSC wobbles, filter out negative motion.
+	 * Negative motion and deltas which can cause multiplication
-	 * Per the above masking, the effective sign bit is now bit 62.
+	 * overflow require special treatment. This check covers both as
+	 * negative motion is guaranteed to be greater than @vd::max_cycles
+	 * due to unsigned comparison.
+	 *
+	 * Due to the MSB/Sign-bit being used as invalid marker (see
+	 * arch_vdso_cycles_valid() above), the effective mask is S64_MAX,
+	 * but that case is also unlikely and will also take the unlikely path
+	 * here.
 	 */
-	if (unlikely(delta & (1ULL << 62)))
+	if (unlikely(delta > vd->max_cycles)) {
-		return 0;
+		/*
+		 * Due to the above mentioned TSC wobbles, filter out
+		 * negative motion.  Per the above masking, the effective
+		 * sign bit is now bit 62.
+		 */
+		if (delta & (1ULL << 62))
+			return base >> vd->shift;
+		/* Handle multiplication overflow gracefully */
+		return mul_u64_u32_add_u64_shr(delta & S64_MAX, vd->mult, base, vd->shift);
+	}
-	return delta * mult;
+	return ((delta * vd->mult) + base) >> vd->shift;
 }
-#define vdso_calc_delta vdso_calc_delta
+#define vdso_calc_ns vdso_calc_ns
 #endif /* !__ASSEMBLY__ */

drivers/clocksource/arm_arch_timer.c+1−1

@@ -331,7 +331,7 @@ static u64 notrace hisi_161010101_read_cntvct_el0(void) · 330 unmodified lines
331	return __hisi_161010101_read_reg(cntvct_el0);	331	return __hisi_161010101_read_reg(cntvct_el0);
332	}	332	}
333		333
334	static struct ate_acpi_oem_info hisi_161010101_oem_info[] = {	334	static const struct ate_acpi_oem_info hisi_161010101_oem_info[] = {
335	/*	335	/*
336	* Note that trailing spaces are required to properly match	336	* Note that trailing spaces are required to properly match
337	* the OEM table information.	337	* the OEM table information.

drivers/clocksource/renesas-ostm.c+2−1

 		pr_info("%pOF: used for clock events\n", np);
 	}
+	of_node_set_flag(np, OF_POPULATED);
 	return 0;
 err_cleanup:
 TIMER_OF_DECLARE(ostm, "renesas,ostm", ostm_init);
-#ifdef CONFIG_ARCH_RZG2L
+#if defined(CONFIG_ARCH_RZG2L) || defined(CONFIG_ARCH_R9A09G057)
 static int __init ostm_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;

drivers/clocksource/timer-ti-dm.c+0−1

@@ -129,7 +129,6 @@ struct dmtimer { · 128 unmodified lines
129	void __iomem func_base; / function register base */	129	void __iomem func_base; / function register base */
130		130
131	atomic_t enabled;	131	atomic_t enabled;
132	unsigned long rate;
133	unsigned reserved:1;	132	unsigned reserved:1;
134	unsigned posted:1;	133	unsigned posted:1;
135	unsigned omap1:1;	134	unsigned omap1:1;

include/linux/math64.h+2−6

 #include <linux/types.h>
 #include <linux/math.h>
-#include <vdso/math64.h>
 #include <asm/div64.h>
+#include <vdso/math64.h>
 #if BITS_PER_LONG == 64
 #ifndef mul_u64_u32_shr
 static __always_inline u64 mul_u64_u32_shr(u64 a, u32 mul, unsigned int shift)
 {
-	u32 ah, al;
+	u32 ah = a >> 32, al = a;
 	u64 ret;
-	al = a;
-	ah = a >> 32;
 	ret = mul_u32_u32(al, mul) >> shift;
 	if (ah)
 		ret += mul_u32_u32(ah, mul) << (32 - shift);
 	return ret;
 }
 #endif /* mul_u64_u32_shr */

include/linux/timerqueue.h+0−5

 	return !RB_EMPTY_NODE(&node->node);
 }
-static inline bool timerqueue_node_expires(struct timerqueue_node *node)
-{
-	return node->expires;
-}
 static inline void timerqueue_init_head(struct timerqueue_head *head)
 {
 	head->rb_root = RB_ROOT_CACHED;

include/vdso/datapage.h+4−0

  * @seq:		timebase sequence counter
  * @clock_mode:		clock mode
  * @cycle_last:		timebase at clocksource init
+ * @max_cycles:		maximum cycles which won't overflow 64bit multiplication
  * @mask:		clocksource mask
  * @mult:		clocksource multiplier
  * @shift:		clocksource shift
 	s32			clock_mode;
 	u64			cycle_last;
+#ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT
+	u64			max_cycles;
+#endif
 	u64			mask;
 	u32			mult;
 	u32			shift;

include/vdso/math64.h+38−0

@@ -21,4 +21,42 @@ __iter_div_u64_rem(u64 dividend, u32 divisor, u64 *remainder) · 20 unmodified lines
21	return ret;	21	return ret;
22	}	22	}
23		23
		24	#if defined(CONFIG_ARCH_SUPPORTS_INT128) && defined(__SIZEOF_INT128__)
		25
		26	#ifndef mul_u64_u32_add_u64_shr
		27	static __always_inline u64 mul_u64_u32_add_u64_shr(u64 a, u32 mul, u64 b, unsigned int shift)
		28	{
		29	return (u64)((((unsigned __int128)a * mul) + b) >> shift);
		30	}
		31	#endif /* mul_u64_u32_add_u64_shr */
		32
		33	#else
		34
		35	#ifndef mul_u64_u32_add_u64_shr
		36	#ifndef mul_u32_u32
		37	static inline u64 mul_u32_u32(u32 a, u32 b)
		38	{
		39	return (u64)a * b;
		40	}
		41	#define mul_u32_u32 mul_u32_u32
		42	#endif
		43	static __always_inline u64 mul_u64_u32_add_u64_shr(u64 a, u32 mul, u64 b, unsigned int shift)
		44	{
		45	u32 ah = a >> 32, al = a;
		46	bool ovf;
		47	u64 ret;
		48
		49	ovf = __builtin_add_overflow(mul_u32_u32(al, mul), b, &ret);
		50	ret >>= shift;
		51	if (ovf && shift)
		52	ret += 1ULL << (64 - shift);
		53	if (ah)
		54	ret += mul_u32_u32(ah, mul) << (32 - shift);
		55
		56	return ret;
		57	}
		58	#endif /* mul_u64_u32_add_u64_shr */
		59
		60	#endif
		61
24	#endif /* __VDSO_MATH64_H */	62	#endif /* __VDSO_MATH64_H */

kernel/time/Kconfig+1−1

@@ -202,7 +202,7 @@ config HIGH_RES_TIMERS · 201 unmodified lines
202	the size of the kernel image.	202	the size of the kernel image.
203		203
204	config CLOCKSOURCE_WATCHDOG_MAX_SKEW_US	204	config CLOCKSOURCE_WATCHDOG_MAX_SKEW_US
205	int "Clocksource watchdog maximum allowable skew (in μs)"	205	int "Clocksource watchdog maximum allowable skew (in microseconds)"
206	depends on CLOCKSOURCE_WATCHDOG	206	depends on CLOCKSOURCE_WATCHDOG
207	range 50 1000	207	range 50 1000
208	default 125	208	default 125

kernel/time/clockevents.c+1−1

@@ -677,7 +677,7 @@ static ssize_t current_device_show(struct device *dev, · 676 unmodified lines
677	raw_spin_lock_irq(&clockevents_lock);	677	raw_spin_lock_irq(&clockevents_lock);
678	td = tick_get_tick_dev(dev);	678	td = tick_get_tick_dev(dev);
679	if (td && td->evtdev)	679	if (td && td->evtdev)
680	count = snprintf(buf, PAGE_SIZE, "%s\n", td->evtdev->name);	680	count = sysfs_emit(buf, "%s\n", td->evtdev->name);
681	raw_spin_unlock_irq(&clockevents_lock);	681	raw_spin_unlock_irq(&clockevents_lock);
682	return count;	682	return count;
683	}	683	}

kernel/time/clocksource.c+21−23

 #include "tick-internal.h"
 #include "timekeeping_internal.h"
+static noinline u64 cycles_to_nsec_safe(struct clocksource *cs, u64 start, u64 end)
+{
+	u64 delta = clocksource_delta(end, start, cs->mask);
+	if (likely(delta < cs->max_cycles))
+		return clocksource_cyc2ns(delta, cs->mult, cs->shift);
+	return mul_u64_u32_shr(delta, cs->mult, cs->shift);
+}
 /**
  * clocks_calc_mult_shift - calculate mult/shift factors for scaled math of clocks
  * @mult:	pointer to mult variable
 static enum wd_read_status cs_watchdog_read(struct clocksource *cs, u64 *csnow, u64 *wdnow)
 {
 	unsigned int nretries, max_retries;
-	u64 wd_end, wd_end2, wd_delta;
 	int64_t wd_delay, wd_seq_delay;
+	u64 wd_end, wd_end2;
 	max_retries = clocksource_get_max_watchdog_retry();
 	for (nretries = 0; nretries <= max_retries; nretries++) {
 		wd_end2 = watchdog->read(watchdog);
 		local_irq_enable();
-		wd_delta = clocksource_delta(wd_end, *wdnow, watchdog->mask);
+		wd_delay = cycles_to_nsec_safe(watchdog, *wdnow, wd_end);
-		wd_delay = clocksource_cyc2ns(wd_delta, watchdog->mult,
-					      watchdog->shift);
 		if (wd_delay <= WATCHDOG_MAX_SKEW) {
 			if (nretries > 1 || nretries >= max_retries) {
 				pr_warn("timekeeping watchdog on CPU%d: %s retried %d times before success\n",
 		 * report system busy, reinit the watchdog and skip the current
 		 * watchdog test.
 		 */
-		wd_delta = clocksource_delta(wd_end2, wd_end, watchdog->mask);
+		wd_seq_delay = cycles_to_nsec_safe(watchdog, wd_end, wd_end2);
-		wd_seq_delay = clocksource_cyc2ns(wd_delta, watchdog->mult, watchdog->shift);
 		if (wd_seq_delay > WATCHDOG_MAX_SKEW/2)
 			goto skip_test;
 	}
 		delta = (csnow_end - csnow_mid) & cs->mask;
 		if (delta < 0)
 			cpumask_set_cpu(cpu, &cpus_ahead);
-		delta = clocksource_delta(csnow_end, csnow_begin, cs->mask);
+		cs_nsec = cycles_to_nsec_safe(cs, csnow_begin, csnow_end);
-		cs_nsec = clocksource_cyc2ns(delta, cs->mult, cs->shift);
 		if (cs_nsec > cs_nsec_max)
 			cs_nsec_max = cs_nsec;
 		if (cs_nsec < cs_nsec_min)
 static void clocksource_watchdog(struct timer_list *unused)
 {
-	u64 csnow, wdnow, cslast, wdlast, delta;
 	int64_t wd_nsec, cs_nsec, interval;
+	u64 csnow, wdnow, cslast, wdlast;
 	int next_cpu, reset_pending;
 	struct clocksource *cs;
 	enum wd_read_status read_ret;
 			continue;
 		}
-		delta = clocksource_delta(wdnow, cs->wd_last, watchdog->mask);
+		wd_nsec = cycles_to_nsec_safe(watchdog, cs->wd_last, wdnow);
-		wd_nsec = clocksource_cyc2ns(delta, watchdog->mult,
+		cs_nsec = cycles_to_nsec_safe(cs, cs->cs_last, csnow);
-					     watchdog->shift);
-		delta = clocksource_delta(csnow, cs->cs_last, cs->mask);
-		cs_nsec = clocksource_cyc2ns(delta, cs->mult, cs->shift);
 		wdlast = cs->wd_last; /* save these in case we print them */
 		cslast = cs->cs_last;
 		cs->cs_last = csnow;
  */
 u64 clocksource_stop_suspend_timing(struct clocksource *cs, u64 cycle_now)
 {
-	u64 now, delta, nsec = 0;
+	u64 now, nsec = 0;
 	if (!suspend_clocksource)
 		return 0;
 	else
 		now = suspend_clocksource->read(suspend_clocksource);
-	if (now > suspend_start) {
+	if (now > suspend_start)
-		delta = clocksource_delta(now, suspend_start,
+		nsec = cycles_to_nsec_safe(suspend_clocksource, suspend_start, now);
-					  suspend_clocksource->mask);
-		nsec = mul_u64_u32_shr(delta, suspend_clocksource->mult,
-				       suspend_clocksource->shift);
-	}
 	/*
 	 * Disable the suspend timer to save power if current clocksource is
 	ssize_t count = 0;
 	mutex_lock(&clocksource_mutex);
-	count = snprintf(buf, PAGE_SIZE, "%s\n", curr_clocksource->name);
+	count = sysfs_emit(buf, "%s\n", curr_clocksource->name);
 	mutex_unlock(&clocksource_mutex);
 	return count;

kernel/time/hrtimer.c+9−32

@@ -644,17 +644,12 @@ static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base) · 643 unmodified lines
644	/*	644	/*
645	* Is the high resolution mode active ?	645	* Is the high resolution mode active ?
646	*/	646	*/
647	static inline int __hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base)	647	static inline int hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base)
648	{	648	{
649	return IS_ENABLED(CONFIG_HIGH_RES_TIMERS) ?	649	return IS_ENABLED(CONFIG_HIGH_RES_TIMERS) ?
650	cpu_base->hres_active : 0;	650	cpu_base->hres_active : 0;
651	}	651	}
652		652
653	static inline int hrtimer_hres_active(void)
654	{
655	return __hrtimer_hres_active(this_cpu_ptr(&hrtimer_bases));
656	}
657
658	static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base,	653	static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base,
659	struct hrtimer *next_timer,	654	struct hrtimer *next_timer,
660	ktime_t expires_next)	655	ktime_t expires_next)
@@ -678,7 +673,7 @@ static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base, · 17 unmodified lines
678	* set. So we'd effectively block all timers until the T2 event	673	* set. So we'd effectively block all timers until the T2 event
679	* fires.	674	* fires.
680	*/	675	*/
681	if (!__hrtimer_hres_active(cpu_base) \|\| cpu_base->hang_detected)	676	if (!hrtimer_hres_active(cpu_base) \|\| cpu_base->hang_detected)
682	return;	677	return;
683		678
684	tick_program_event(expires_next, 1);	679	tick_program_event(expires_next, 1);
@@ -789,12 +784,12 @@ static void retrigger_next_event(void *arg) · 104 unmodified lines
789	* function call will take care of the reprogramming in case the	784	* function call will take care of the reprogramming in case the
790	* CPU was in a NOHZ idle sleep.	785	* CPU was in a NOHZ idle sleep.
791	*/	786	*/
792	if (!__hrtimer_hres_active(base) && !tick_nohz_active)	787	if (!hrtimer_hres_active(base) && !tick_nohz_active)
793	return;	788	return;
794		789
795	raw_spin_lock(&base->lock);	790	raw_spin_lock(&base->lock);
796	hrtimer_update_base(base);	791	hrtimer_update_base(base);
797	if (__hrtimer_hres_active(base))	792	if (hrtimer_hres_active(base))
798	hrtimer_force_reprogram(base, 0);	793	hrtimer_force_reprogram(base, 0);
799	else	794	else
800	hrtimer_update_next_event(base);	795	hrtimer_update_next_event(base);
@@ -951,7 +946,7 @@ void clock_was_set(unsigned int bases) · 150 unmodified lines
951	cpumask_var_t mask;	946	cpumask_var_t mask;
952	int cpu;	947	int cpu;
953		948
954	if (!__hrtimer_hres_active(cpu_base) && !tick_nohz_active)	949	if (!hrtimer_hres_active(cpu_base) && !tick_nohz_active)
955	goto out_timerfd;	950	goto out_timerfd;
956		951
957	if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {	952	if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
@@ -1491,7 +1486,7 @@ u64 hrtimer_get_next_event(void) · 533 unmodified lines
1491		1486
1492	raw_spin_lock_irqsave(&cpu_base->lock, flags);	1487	raw_spin_lock_irqsave(&cpu_base->lock, flags);
1493		1488
1494	if (!__hrtimer_hres_active(cpu_base))	1489	if (!hrtimer_hres_active(cpu_base))
1495	expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);	1490	expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
1496		1491
1497	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);	1492	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
@@ -1514,7 +1509,7 @@ u64 hrtimer_next_event_without(const struct hrtimer *exclude) · 16 unmodified lines
1514		1509
1515	raw_spin_lock_irqsave(&cpu_base->lock, flags);	1510	raw_spin_lock_irqsave(&cpu_base->lock, flags);
1516		1511
1517	if (__hrtimer_hres_active(cpu_base)) {	1512	if (hrtimer_hres_active(cpu_base)) {
1518	unsigned int active;	1513	unsigned int active;
1519		1514
1520	if (!cpu_base->softirq_activated) {	1515	if (!cpu_base->softirq_activated) {
@@ -1875,25 +1870,7 @@ retry: · 354 unmodified lines
1875	tick_program_event(expires_next, 1);	1870	tick_program_event(expires_next, 1);
1876	pr_warn_once("hrtimer: interrupt took %llu ns\n", ktime_to_ns(delta));	1871	pr_warn_once("hrtimer: interrupt took %llu ns\n", ktime_to_ns(delta));
1877	}	1872	}
1878		1873	#endif /* !CONFIG_HIGH_RES_TIMERS */
1879	/* called with interrupts disabled */
1880	static inline void __hrtimer_peek_ahead_timers(void)
1881	{
1882	struct tick_device *td;
1883
1884	if (!hrtimer_hres_active())
1885	return;
1886
1887	td = this_cpu_ptr(&tick_cpu_device);
1888	if (td && td->evtdev)
1889	hrtimer_interrupt(td->evtdev);
1890	}
1891
1892	#else /* CONFIG_HIGH_RES_TIMERS */
1893
1894	static inline void __hrtimer_peek_ahead_timers(void) { }
1895
1896	#endif /* !CONFIG_HIGH_RES_TIMERS */
1897		1874
1898	/*	1875	/*
1899	* Called from run_local_timers in hardirq context every jiffy	1876	* Called from run_local_timers in hardirq context every jiffy
@@ -1904,7 +1881,7 @@ void hrtimer_run_queues(void) · 4 unmodified lines
1904	unsigned long flags;	1881	unsigned long flags;
1905	ktime_t now;	1882	ktime_t now;
1906		1883
1907	if (__hrtimer_hres_active(cpu_base))	1884	if (hrtimer_hres_active(cpu_base))
1908	return;	1885	return;
1909		1886
1910	/*	1887	/*

kernel/time/timekeeping.c+49−47

 	}
 }
-static inline u64 timekeeping_get_delta(const struct tk_read_base *tkr)
+static inline u64 timekeeping_cycles_to_ns(const struct tk_read_base *tkr, u64 cycles);
+static inline u64 timekeeping_debug_get_ns(const struct tk_read_base *tkr)
 {
 	struct timekeeper *tk = &tk_core.timekeeper;
 	u64 now, last, mask, max, delta;
 	 * Try to catch underflows by checking if we are seeing small
 	 * mask-relative negative values.
 	 */
-	if (unlikely((~delta & mask) < (mask >> 3))) {
+	if (unlikely((~delta & mask) < (mask >> 3)))
 		tk->underflow_seen = 1;
-		delta = 0;
-	}
-	/* Cap delta value to the max_cycles values to avoid mult overflows */
+	/* Check for multiplication overflows */
-	if (unlikely(delta > max)) {
+	if (unlikely(delta > max))
 		tk->overflow_seen = 1;
-		delta = tkr->clock->max_cycles;
-	}
-	return delta;
+	/* timekeeping_cycles_to_ns() handles both under and overflow */
+	return timekeeping_cycles_to_ns(tkr, now);
 }
 #else
 static inline void timekeeping_check_update(struct timekeeper *tk, u64 offset)
 {
 }
-static inline u64 timekeeping_get_delta(const struct tk_read_base *tkr)
+static inline u64 timekeeping_debug_get_ns(const struct tk_read_base *tkr)
 {
-	u64 cycle_now, delta;
+	BUG();
-	/* read clocksource */
-	cycle_now = tk_clock_read(tkr);
-	/* calculate the delta since the last update_wall_time */
-	delta = clocksource_delta(cycle_now, tkr->cycle_last, tkr->mask);
-	return delta;
 }
 #endif
 }
 /* Timekeeper helper functions. */
+static noinline u64 delta_to_ns_safe(const struct tk_read_base *tkr, u64 delta)
+{
+	return mul_u64_u32_add_u64_shr(delta, tkr->mult, tkr->xtime_nsec, tkr->shift);
+}
-static inline u64 timekeeping_delta_to_ns(const struct tk_read_base *tkr, u64 delta)
+static inline u64 timekeeping_cycles_to_ns(const struct tk_read_base *tkr, u64 cycles)
 {
-	u64 nsec;
+	/* Calculate the delta since the last update_wall_time() */
+	u64 mask = tkr->mask, delta = (cycles - tkr->cycle_last) & mask;
-	nsec = delta * tkr->mult + tkr->xtime_nsec;
+	/*
-	nsec >>= tkr->shift;
+	 * This detects both negative motion and the case where the delta
+	 * overflows the multiplication with tkr->mult.
+	 */
+	if (unlikely(delta > tkr->clock->max_cycles)) {
+		/*
+		 * Handle clocksource inconsistency between CPUs to prevent
+		 * time from going backwards by checking for the MSB of the
+		 * mask being set in the delta.
+		 */
+		if (delta & ~(mask >> 1))
+			return tkr->xtime_nsec >> tkr->shift;
+		return delta_to_ns_safe(tkr, delta);
+	}
-	return nsec;
+	return ((delta * tkr->mult) + tkr->xtime_nsec) >> tkr->shift;
 }
-static inline u64 timekeeping_get_ns(const struct tk_read_base *tkr)
+static __always_inline u64 __timekeeping_get_ns(const struct tk_read_base *tkr)
 {
-	u64 delta;
+	return timekeeping_cycles_to_ns(tkr, tk_clock_read(tkr));
-	delta = timekeeping_get_delta(tkr);
-	return timekeeping_delta_to_ns(tkr, delta);
 }
-static inline u64 timekeeping_cycles_to_ns(const struct tk_read_base *tkr, u64 cycles)
+static inline u64 timekeeping_get_ns(const struct tk_read_base *tkr)
 {
-	u64 delta;
+	if (IS_ENABLED(CONFIG_DEBUG_TIMEKEEPING))
+		return timekeeping_debug_get_ns(tkr);
-	/* calculate the delta since the last update_wall_time */
+	return __timekeeping_get_ns(tkr);
-	delta = clocksource_delta(cycles, tkr->cycle_last, tkr->mask);
-	return timekeeping_delta_to_ns(tkr, delta);
 }
 /**
 	memcpy(base + 1, base, sizeof(*base));
 }
-static __always_inline u64 fast_tk_get_delta_ns(struct tk_read_base *tkr)
-{
-	u64 delta, cycles = tk_clock_read(tkr);
-	delta = clocksource_delta(cycles, tkr->cycle_last, tkr->mask);
-	return timekeeping_delta_to_ns(tkr, delta);
-}
 static __always_inline u64 __ktime_get_fast_ns(struct tk_fast *tkf)
 {
 	struct tk_read_base *tkr;
 		seq = raw_read_seqcount_latch(&tkf->seq);
 		tkr = tkf->base + (seq & 0x01);
 		now = ktime_to_ns(tkr->base);
-		now += fast_tk_get_delta_ns(tkr);
+		now += __timekeeping_get_ns(tkr);
 	} while (raw_read_seqcount_latch_retry(&tkf->seq, seq));
 	return now;
 		tkr = tkf->base + (seq & 0x01);
 		basem = ktime_to_ns(tkr->base);
 		baser = ktime_to_ns(tkr->base_real);
-		delta = fast_tk_get_delta_ns(tkr);
+		delta = __timekeeping_get_ns(tkr);
 	} while (raw_read_seqcount_latch_retry(&tkf->seq, seq));
 	if (mono)
 	tk->tkr_mono.cycle_last = cycle_now;
 	tk->tkr_raw.cycle_last  = cycle_now;
-	tk->tkr_mono.xtime_nsec += delta * tk->tkr_mono.mult;
+	while (delta > 0) {
-	tk->tkr_raw.xtime_nsec += delta * tk->tkr_raw.mult;
+		u64 max = tk->tkr_mono.clock->max_cycles;
+		u64 incr = delta < max ? delta : max;
-	tk_normalize_xtime(tk);
+		tk->tkr_mono.xtime_nsec += incr * tk->tkr_mono.mult;
+		tk->tkr_raw.xtime_nsec += incr * tk->tkr_raw.mult;
+		tk_normalize_xtime(tk);
+		delta -= incr;
+	}
 }
 /**

kernel/time/vsyscall.c+6−0

 	u64 nsec, sec;
 	vdata[CS_HRES_COARSE].cycle_last	= tk->tkr_mono.cycle_last;
+#ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT
+	vdata[CS_HRES_COARSE].max_cycles	= tk->tkr_mono.clock->max_cycles;
+#endif
 	vdata[CS_HRES_COARSE].mask		= tk->tkr_mono.mask;
 	vdata[CS_HRES_COARSE].mult		= tk->tkr_mono.mult;
 	vdata[CS_HRES_COARSE].shift		= tk->tkr_mono.shift;
 	vdata[CS_RAW].cycle_last		= tk->tkr_raw.cycle_last;
+#ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT
+	vdata[CS_RAW].max_cycles		= tk->tkr_raw.clock->max_cycles;
+#endif
 	vdata[CS_RAW].mask			= tk->tkr_raw.mask;
 	vdata[CS_RAW].mult			= tk->tkr_raw.mult;
 	vdata[CS_RAW].shift			= tk->tkr_raw.shift;

lib/vdso/Kconfig+7−0

@@ -30,4 +30,11 @@ config GENERIC_VDSO_TIME_NS · 29 unmodified lines
30	Selected by architectures which support time namespaces in the	30	Selected by architectures which support time namespaces in the
31	VDSO	31	VDSO
32		32
		33	config GENERIC_VDSO_OVERFLOW_PROTECT
		34	bool
		35	help
		36	Select to add multiplication overflow protection to the VDSO
		37	time getter functions for the price of an extra conditional
		38	in the hotpath.
		39
33	endif	40	endif

lib/vdso/gettimeofday.c+36−19

@@ -5,15 +5,23 @@ · 4 unmodified lines
5	#include <vdso/datapage.h>	5	#include <vdso/datapage.h>
6	#include <vdso/helpers.h>	6	#include <vdso/helpers.h>
7		7
8	#ifndef vdso_calc_delta	8	#ifndef vdso_calc_ns
9	/*	9
10	* Default implementation which works for all sane clocksources. That	10	#ifdef VDSO_DELTA_NOMASK
11	* obviously excludes x86/TSC.	11	# define VDSO_DELTA_MASK(vd) ULLONG_MAX
12	*/	12	#else
13	static __always_inline	13	# define VDSO_DELTA_MASK(vd) (vd->mask)
14	u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult)	14	#endif
		15
		16	#ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT
		17	static __always_inline bool vdso_delta_ok(const struct vdso_data *vd, u64 delta)
		18	{
		19	return delta < vd->max_cycles;
		20	}
		21	#else
		22	static __always_inline bool vdso_delta_ok(const struct vdso_data *vd, u64 delta)
15	{	23	{
16	return ((cycles - last) & mask) * mult;	24	return true;
17	}	25	}
18	#endif	26	#endif
19		27
@@ -24,6 +32,21 @@ static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift) · 4 unmodified lines
24	}	32	}
25	#endif	33	#endif
26		34
		35	/*
		36	* Default implementation which works for all sane clocksources. That
		37	* obviously excludes x86/TSC.
		38	*/
		39	static __always_inline u64 vdso_calc_ns(const struct vdso_data *vd, u64 cycles, u64 base)
		40	{
		41	u64 delta = (cycles - vd->cycle_last) & VDSO_DELTA_MASK(vd);
		42
		43	if (likely(vdso_delta_ok(vd, delta)))
		44	return vdso_shift_ns((delta * vd->mult) + base, vd->shift);
		45
		46	return mul_u64_u32_add_u64_shr(delta, vd->mult, base, vd->shift);
		47	}
		48	#endif /* vdso_calc_ns */
		49
27	#ifndef __arch_vdso_hres_capable	50	#ifndef __arch_vdso_hres_capable
28	static inline bool __arch_vdso_hres_capable(void)	51	static inline bool __arch_vdso_hres_capable(void)
29	{	52	{
@@ -49,10 +72,10 @@ static inline bool vdso_cycles_ok(u64 cycles) · 19 unmodified lines
49	static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,	72	static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
50	struct __kernel_timespec *ts)	73	struct __kernel_timespec *ts)
51	{	74	{
52	const struct vdso_data *vd;
53	const struct timens_offset *offs = &vdns->offset[clk];	75	const struct timens_offset *offs = &vdns->offset[clk];
54	const struct vdso_timestamp *vdso_ts;	76	const struct vdso_timestamp *vdso_ts;
55	u64 cycles, last, ns;	77	const struct vdso_data *vd;
		78	u64 cycles, ns;
56	u32 seq;	79	u32 seq;
57	s64 sec;	80	s64 sec;
58		81
@@ -73,10 +96,7 @@ static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_ · 14 unmodified lines
73	cycles = __arch_get_hw_counter(vd->clock_mode, vd);	96	cycles = __arch_get_hw_counter(vd->clock_mode, vd);
74	if (unlikely(!vdso_cycles_ok(cycles)))	97	if (unlikely(!vdso_cycles_ok(cycles)))
75	return -1;	98	return -1;
76	ns = vdso_ts->nsec;	99	ns = vdso_calc_ns(vd, cycles, vdso_ts->nsec);
77	last = vd->cycle_last;
78	ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
79	ns = vdso_shift_ns(ns, vd->shift);
80	sec = vdso_ts->sec;	100	sec = vdso_ts->sec;
81	} while (unlikely(vdso_read_retry(vd, seq)));	101	} while (unlikely(vdso_read_retry(vd, seq)));
82		102
@@ -111,7 +131,7 @@ static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, · 28 unmodified lines
111	struct __kernel_timespec *ts)	131	struct __kernel_timespec *ts)
112	{	132	{
113	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];	133	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
114	u64 cycles, last, sec, ns;	134	u64 cycles, sec, ns;
115	u32 seq;	135	u32 seq;
116		136
117	/* Allows to compile the high resolution parts out */	137	/* Allows to compile the high resolution parts out */
@@ -144,10 +164,7 @@ static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, · 26 unmodified lines
144	cycles = __arch_get_hw_counter(vd->clock_mode, vd);	164	cycles = __arch_get_hw_counter(vd->clock_mode, vd);
145	if (unlikely(!vdso_cycles_ok(cycles)))	165	if (unlikely(!vdso_cycles_ok(cycles)))
146	return -1;	166	return -1;
147	ns = vdso_ts->nsec;	167	ns = vdso_calc_ns(vd, cycles, vdso_ts->nsec);
148	last = vd->cycle_last;
149	ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
150	ns = vdso_shift_ns(ns, vd->shift);
151	sec = vdso_ts->sec;	168	sec = vdso_ts->sec;
152	} while (unlikely(vdso_read_retry(vd, seq)));	169	} while (unlikely(vdso_read_retry(vd, seq)));
153		170

rust/kernel/time.rs+63−0

@@ -4,6 +4,12 @@ · 3 unmodified lines
4	//!	4	//!
5	//! This module contains the kernel APIs related to time and timers that	5	//! This module contains the kernel APIs related to time and timers that
6	//! have been ported or wrapped for usage by Rust code in the kernel.	6	//! have been ported or wrapped for usage by Rust code in the kernel.
		7	//!
		8	//! C header: [`include/linux/jiffies.h`](srctree/include/linux/jiffies.h).
		9	//! C header: [`include/linux/ktime.h`](srctree/include/linux/ktime.h).
		10
		11	/// The number of nanoseconds per millisecond.
		12	pub const NSEC_PER_MSEC: i64 = bindings::NSEC_PER_MSEC as i64;
7		13
8	/// The time unit of Linux kernel. One jiffy equals (1/HZ) second.	14	/// The time unit of Linux kernel. One jiffy equals (1/HZ) second.
9	pub type Jiffies = core::ffi::c_ulong;	15	pub type Jiffies = core::ffi::c_ulong;
@@ -18,3 +24,60 @@ pub fn msecs_to_jiffies(msecs: Msecs) -> Jiffies { · 8 unmodified lines
18	// matter what the argument is.	24	// matter what the argument is.
19	unsafe { bindings::__msecs_to_jiffies(msecs) }	25	unsafe { bindings::__msecs_to_jiffies(msecs) }
20	}	26	}
		27
		28	/// A Rust wrapper around a `ktime_t`.
		29	#[repr(transparent)]
		30	#[derive(Copy, Clone)]
		31	pub struct Ktime {
		32	inner: bindings::ktime_t,
		33	}
		34
		35	impl Ktime {
		36	/// Create a `Ktime` from a raw `ktime_t`.
		37	#[inline]
		38	pub fn from_raw(inner: bindings::ktime_t) -> Self {
		39	Self { inner }
		40	}
		41
		42	/// Get the current time using `CLOCK_MONOTONIC`.
		43	#[inline]
		44	pub fn ktime_get() -> Self {
		45	// SAFETY: It is always safe to call `ktime_get` outside of NMI context.
		46	Self::from_raw(unsafe { bindings::ktime_get() })
		47	}
		48
		49	/// Divide the number of nanoseconds by a compile-time constant.
		50	#[inline]
		51	fn divns_constant<const DIV: i64>(self) -> i64 {
		52	self.to_ns() / DIV
		53	}
		54
		55	/// Returns the number of nanoseconds.
		56	#[inline]
		57	pub fn to_ns(self) -> i64 {
		58	self.inner
		59	}
		60
		61	/// Returns the number of milliseconds.
		62	#[inline]
		63	pub fn to_ms(self) -> i64 {
		64	self.divns_constant::<NSEC_PER_MSEC>()
		65	}
		66	}
		67
		68	/// Returns the number of milliseconds between two ktimes.
		69	#[inline]
		70	pub fn ktime_ms_delta(later: Ktime, earlier: Ktime) -> i64 {
		71	(later - earlier).to_ms()
		72	}
		73
		74	impl core::ops::Sub for Ktime {
		75	type Output = Ktime;
		76
		77	#[inline]
		78	fn sub(self, other: Ktime) -> Ktime {
		79	Self {
		80	inner: self.inner - other.inner,
		81	}
		82	}
		83	}