Merge tag 'timers-core-2024-05-12' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull timers and timekeeping updates from Thomas Gleixner: "Core code: - Make timekeeping and VDSO time readouts resilent against math overflow: In guest context the kernel is prone to math overflow when the host defers the timer interrupt due to overload, malfunction or malice. This can be mitigated by checking the clocksource delta for the maximum deferrement which is readily available. If that value is exceeded then the code uses a slowpath function which can handle the multiplication overflow. This functionality is enabled unconditionally in the kernel, but made conditional in the VDSO code. The latter is conditional because it allows architectures to optimize the check so it is not causing performance regressions. On X86 this is achieved by reworking the existing check for negative TSC deltas as a negative delta obviously exceeds the maximum deferrement when it is evaluated as an unsigned value. That avoids two conditionals in the hotpath and allows to hide both the negative delta and the large delta handling in the same slow path. - Add an initial minimal ktime_t abstraction for Rust - The usual boring cleanups and enhancements Drivers: - Boring updates to device trees and trivial enhancements in various drivers" * tag 'timers-core-2024-05-12' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (33 commits) clocksource/drivers/arm_arch_timer: Mark hisi_161010101_oem_info const clocksource/drivers/timer-ti-dm: Remove an unused field in struct dmtimer clocksource/drivers/renesas-ostm: Avoid reprobe after successful early probe clocksource/drivers/renesas-ostm: Allow OSTM driver to reprobe for RZ/V2H(P) SoC dt-bindings: timer: renesas: ostm: Document Renesas RZ/V2H(P) SoC rust: time: doc: Add missing C header links clocksource: Make the int help prompt unit readable in ncurses hrtimer: Rename __hrtimer_hres_active() to hrtimer_hres_active() timerqueue: Remove never used function timerqueue_node_expires() rust: time: Add Ktime vdso: Fix powerpc build U64_MAX undeclared error clockevents: Convert s[n]printf() to sysfs_emit() clocksource: Convert s[n]printf() to sysfs_emit() clocksource: Make watchdog and suspend-timing multiplication overflow safe timekeeping: Let timekeeping_cycles_to_ns() handle both under and overflow timekeeping: Make delta calculation overflow safe timekeeping: Prepare timekeeping_cycles_to_ns() for overflow safety timekeeping: Fold in timekeeping_delta_to_ns() timekeeping: Consolidate timekeeping helpers timekeeping: Refactor timekeeping helpers ...

Linus Torvalds committed 2 years ago

Commit 2d9db772 parents 61deafa a3825a721 files · +284 −173Browse files

Unified · Split

21 files

Documentation/devicetree/bindings/timer/renesas,ostm.yaml +2 −0
arch/powerpc/include/asm/vdso/gettimeofday.h +11 −15
arch/s390/include/asm/vdso/gettimeofday.h +2 −5
arch/x86/Kconfig +1 −0
arch/x86/include/asm/vdso/gettimeofday.h +28 −16
drivers/clocksource/arm_arch_timer.c +1 −1
drivers/clocksource/renesas-ostm.c +2 −1
drivers/clocksource/timer-ti-dm.c +0 −1
include/linux/math64.h +2 −6
include/linux/timerqueue.h +0 −5
include/vdso/datapage.h +4 −0
include/vdso/math64.h +38 −0
kernel/time/Kconfig +1 −1
kernel/time/clockevents.c +1 −1
kernel/time/clocksource.c +21 −23
kernel/time/hrtimer.c +9 −32
kernel/time/timekeeping.c +49 −47
kernel/time/vsyscall.c +6 −0
lib/vdso/Kconfig +7 −0
lib/vdso/gettimeofday.c +36 −19
rust/kernel/time.rs +63 −0

Documentation/devicetree/bindings/timer/renesas,ostm.yaml+2−0

@@ -26,6 +26,7 @@ properties: · 25 unmodified lines
26	26	- renesas,r9a07g043-ostm # RZ/G2UL and RZ/Five
27	27	- renesas,r9a07g044-ostm # RZ/G2{L,LC}
28	28	- renesas,r9a07g054-ostm # RZ/V2L
	29	+ - renesas,r9a09g057-ostm # RZ/V2H(P)
29	30	- const: renesas,ostm # Generic
30	31
31	32	reg:
@@ -58,6 +59,7 @@ if: · 26 unmodified lines
58	59	- renesas,r9a07g043-ostm
59	60	- renesas,r9a07g044-ostm
60	61	- renesas,r9a07g054-ostm
	62	+ - renesas,r9a09g057-ostm
61	63	then:
62	64	required:
63	65	- resets

arch/powerpc/include/asm/vdso/gettimeofday.h+11−15

@@ -13,6 +13,17 @@ · 12 unmodified lines
13	13
14	14	#define VDSO_HAS_TIME 1
15	15
	16	+/*
	17	+ * powerpc specific delta calculation.
	18	+ *
	19	+ * This variant removes the masking of the subtraction because the
	20	+ * clocksource mask of all VDSO capable clocksources on powerpc is U64_MAX
	21	+ * which would result in a pointless operation. The compiler cannot
	22	+ * optimize it away as the mask comes from the vdso data and is not compile
	23	+ * time constant.
	24	+ */
	25	+#define VDSO_DELTA_NOMASK 1
	26	+
16	27	static __always_inline int do_syscall_2(const unsigned long _r0, const unsigned long _r3,
17	28	const unsigned long _r4)
18	29	{
@@ -104,21 +115,6 @@ static inline bool vdso_clocksource_ok(const struct vdso_data *vd) · 85 unmodified lines
104	115	}
105	116	#define vdso_clocksource_ok vdso_clocksource_ok
106	117
107		−/*
108		− * powerpc specific delta calculation.
109		− *
110		− * This variant removes the masking of the subtraction because the
111		− * clocksource mask of all VDSO capable clocksources on powerpc is U64_MAX
112		− * which would result in a pointless operation. The compiler cannot
113		− * optimize it away as the mask comes from the vdso data and is not compile
114		− * time constant.
115		− */
116		−static __always_inline u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult)
117		−{
118		− return (cycles - last) * mult;
119		−}
120		−#define vdso_calc_delta vdso_calc_delta
121		−
122	118	#ifndef __powerpc64__
123	119	static __always_inline u64 vdso_shift_ns(u64 ns, unsigned long shift)
124	120	{

arch/s390/include/asm/vdso/gettimeofday.h+2−5

@@ -6,16 +6,13 @@ · 5 unmodified lines
6	6
7	7	#define VDSO_HAS_CLOCK_GETRES 1
8	8
	9	+#define VDSO_DELTA_NOMASK 1
	10	+
9	11	#include <asm/syscall.h>
10	12	#include <asm/timex.h>
11	13	#include <asm/unistd.h>
12	14	#include <linux/compiler.h>
13	15
14		−#define vdso_calc_delta __arch_vdso_calc_delta
15		−static __always_inline u64 __arch_vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult)
16		−{
17		− return (cycles - last) * mult;
18		−}
19	16
20	17	static __always_inline const struct vdso_data *__arch_get_vdso_data(void)
21	18	{

arch/x86/Kconfig+1−0

@@ -169,6 +169,7 @@ config X86 · 168 unmodified lines
169	169	select GENERIC_TIME_VSYSCALL
170	170	select GENERIC_GETTIMEOFDAY
171	171	select GENERIC_VDSO_TIME_NS
	172	+ select GENERIC_VDSO_OVERFLOW_PROTECT
172	173	select GUP_GET_PXX_LOW_HIGH if X86_PAE
173	174	select HARDIRQS_SW_RESEND
174	175	select HARDLOCKUP_CHECK_TIMESTAMP if X86_64

arch/x86/include/asm/vdso/gettimeofday.h+28−16

@@ -300,7 +300,7 @@ static inline bool arch_vdso_cycles_ok(u64 cycles) · 299 unmodified lines
300	300	#define vdso_cycles_ok arch_vdso_cycles_ok
301	301
302	302	/*
303		− * x86 specific delta calculation.
	303	+ * x86 specific calculation of nanoseconds for the current cycle count
304	304	*
305	305	* The regular implementation assumes that clocksource reads are globally
306	306	* monotonic. The TSC can be slightly off across sockets which can cause
@@ -308,8 +308,8 @@ static inline bool arch_vdso_cycles_ok(u64 cycles) · 1 unmodified line
308	308	* jump.
309	309	*
310	310	* Therefore it needs to be verified that @cycles are greater than
311		− * @last. If not then use @last, which is the base time of the current
312		− * conversion period.
	311	+ * @vd->cycles_last. If not then use @vd->cycles_last, which is the base
	312	+ * time of the current conversion period.
313	313	*
314	314	* This variant also uses a custom mask because while the clocksource mask of
315	315	* all the VDSO capable clocksources on x86 is U64_MAX, the above code uses
@@ -317,25 +317,37 @@ static inline bool arch_vdso_cycles_ok(u64 cycles) · 1 unmodified line
317	317	* declares everything with the MSB/Sign-bit set as invalid. Therefore the
318	318	* effective mask is S64_MAX.
319	319	*/
320		−static __always_inline
321		−u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult)
	320	+static __always_inline u64 vdso_calc_ns(const struct vdso_data *vd, u64 cycles, u64 base)
322	321	{
323		− /*
324		− * Due to the MSB/Sign-bit being used as invalid marker (see
325		− * arch_vdso_cycles_valid() above), the effective mask is S64_MAX.
326		− */
327		− u64 delta = (cycles - last) & S64_MAX;
	322	+ u64 delta = cycles - vd->cycle_last;
328	323
329	324	/*
330		− * Due to the above mentioned TSC wobbles, filter out negative motion.
331		− * Per the above masking, the effective sign bit is now bit 62.
	325	+ * Negative motion and deltas which can cause multiplication
	326	+ * overflow require special treatment. This check covers both as
	327	+ * negative motion is guaranteed to be greater than @vd::max_cycles
	328	+ * due to unsigned comparison.
	329	+ *
	330	+ * Due to the MSB/Sign-bit being used as invalid marker (see
	331	+ * arch_vdso_cycles_valid() above), the effective mask is S64_MAX,
	332	+ * but that case is also unlikely and will also take the unlikely path
	333	+ * here.
332	334	*/
333		− if (unlikely(delta & (1ULL << 62)))
334		− return 0;
	335	+ if (unlikely(delta > vd->max_cycles)) {
	336	+ /*
	337	+ * Due to the above mentioned TSC wobbles, filter out
	338	+ * negative motion. Per the above masking, the effective
	339	+ * sign bit is now bit 62.
	340	+ */
	341	+ if (delta & (1ULL << 62))
	342	+ return base >> vd->shift;
	343	+
	344	+ /* Handle multiplication overflow gracefully */
	345	+ return mul_u64_u32_add_u64_shr(delta & S64_MAX, vd->mult, base, vd->shift);
	346	+ }
335	347
336		− return delta * mult;
	348	+ return ((delta * vd->mult) + base) >> vd->shift;
337	349	}
338		−#define vdso_calc_delta vdso_calc_delta
	350	+#define vdso_calc_ns vdso_calc_ns
339	351
340	352	#endif /* !__ASSEMBLY__ */
341	353

drivers/clocksource/arm_arch_timer.c+1−1

@@ -331,7 +331,7 @@ static u64 notrace hisi_161010101_read_cntvct_el0(void) · 330 unmodified lines
331	331	return __hisi_161010101_read_reg(cntvct_el0);
332	332	}
333	333
334		−static struct ate_acpi_oem_info hisi_161010101_oem_info[] = {
	334	+static const struct ate_acpi_oem_info hisi_161010101_oem_info[] = {
335	335	/*
336	336	* Note that trailing spaces are required to properly match
337	337	* the OEM table information.

drivers/clocksource/renesas-ostm.c+2−1

@@ -210,6 +210,7 @@ static int __init ostm_init(struct device_node *np) · 209 unmodified lines
210	210	pr_info("%pOF: used for clock events\n", np);
211	211	}
212	212
	213	+ of_node_set_flag(np, OF_POPULATED);
213	214	return 0;
214	215
215	216	err_cleanup:
@@ -224,7 +225,7 @@ err_free: · 8 unmodified lines
224	225
225	226	TIMER_OF_DECLARE(ostm, "renesas,ostm", ostm_init);
226	227
227		−#ifdef CONFIG_ARCH_RZG2L
	228	+#if defined(CONFIG_ARCH_RZG2L) \|\| defined(CONFIG_ARCH_R9A09G057)
228	229	static int __init ostm_probe(struct platform_device *pdev)
229	230	{
230	231	struct device *dev = &pdev->dev;

drivers/clocksource/timer-ti-dm.c+0−1

@@ -129,7 +129,6 @@ struct dmtimer { · 128 unmodified lines
129	129	void __iomem func_base; / function register base */
130	130
131	131	atomic_t enabled;
132		− unsigned long rate;
133	132	unsigned reserved:1;
134	133	unsigned posted:1;
135	134	unsigned omap1:1;

include/linux/math64.h+2−6

@@ -4,8 +4,8 @@ · 3 unmodified lines
4	4
5	5	#include <linux/types.h>
6	6	#include <linux/math.h>
7		−#include <vdso/math64.h>
8	7	#include <asm/div64.h>
	8	+#include <vdso/math64.h>
9	9
10	10	#if BITS_PER_LONG == 64
11	11
@@ -179,16 +179,12 @@ static __always_inline u64 mul_u64_u64_shr(u64 a, u64 mul, unsigned int shift) · 167 unmodified lines
179	179	#ifndef mul_u64_u32_shr
180	180	static __always_inline u64 mul_u64_u32_shr(u64 a, u32 mul, unsigned int shift)
181	181	{
182		− u32 ah, al;
	182	+ u32 ah = a >> 32, al = a;
183	183	u64 ret;
184	184
185		− al = a;
186		− ah = a >> 32;
187		−
188	185	ret = mul_u32_u32(al, mul) >> shift;
189	186	if (ah)
190	187	ret += mul_u32_u32(ah, mul) << (32 - shift);
191		−
192	188	return ret;
193	189	}
194	190	#endif /* mul_u64_u32_shr */

include/linux/timerqueue.h+0−5

@@ -37,11 +37,6 @@ static inline bool timerqueue_node_queued(struct timerqueue_node *node) · 36 unmodified lines
37	37	return !RB_EMPTY_NODE(&node->node);
38	38	}
39	39
40		−static inline bool timerqueue_node_expires(struct timerqueue_node *node)
41		−{
42		− return node->expires;
43		−}
44		−
45	40	static inline void timerqueue_init_head(struct timerqueue_head *head)
46	41	{
47	42	head->rb_root = RB_ROOT_CACHED;

include/vdso/datapage.h+4−0

@@ -61,6 +61,7 @@ struct vdso_timestamp { · 60 unmodified lines
61	61	* @seq: timebase sequence counter
62	62	* @clock_mode: clock mode
63	63	* @cycle_last: timebase at clocksource init
	64	+ * @max_cycles: maximum cycles which won't overflow 64bit multiplication
64	65	* @mask: clocksource mask
65	66	* @mult: clocksource multiplier
66	67	* @shift: clocksource shift
@@ -92,6 +93,9 @@ struct vdso_data { · 25 unmodified lines
92	93
93	94	s32 clock_mode;
94	95	u64 cycle_last;
	96	+#ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT
	97	+ u64 max_cycles;
	98	+#endif
95	99	u64 mask;
96	100	u32 mult;
97	101	u32 shift;

include/vdso/math64.h+38−0

@@ -21,4 +21,42 @@ __iter_div_u64_rem(u64 dividend, u32 divisor, u64 *remainder) · 20 unmodified lines
21	21	return ret;
22	22	}
23	23
	24	+#if defined(CONFIG_ARCH_SUPPORTS_INT128) && defined(__SIZEOF_INT128__)
	25	+
	26	+#ifndef mul_u64_u32_add_u64_shr
	27	+static __always_inline u64 mul_u64_u32_add_u64_shr(u64 a, u32 mul, u64 b, unsigned int shift)
	28	+{
	29	+ return (u64)((((unsigned __int128)a * mul) + b) >> shift);
	30	+}
	31	+#endif /* mul_u64_u32_add_u64_shr */
	32	+
	33	+#else
	34	+
	35	+#ifndef mul_u64_u32_add_u64_shr
	36	+#ifndef mul_u32_u32
	37	+static inline u64 mul_u32_u32(u32 a, u32 b)
	38	+{
	39	+ return (u64)a * b;
	40	+}
	41	+#define mul_u32_u32 mul_u32_u32
	42	+#endif
	43	+static __always_inline u64 mul_u64_u32_add_u64_shr(u64 a, u32 mul, u64 b, unsigned int shift)
	44	+{
	45	+ u32 ah = a >> 32, al = a;
	46	+ bool ovf;
	47	+ u64 ret;
	48	+
	49	+ ovf = __builtin_add_overflow(mul_u32_u32(al, mul), b, &ret);
	50	+ ret >>= shift;
	51	+ if (ovf && shift)
	52	+ ret += 1ULL << (64 - shift);
	53	+ if (ah)
	54	+ ret += mul_u32_u32(ah, mul) << (32 - shift);
	55	+
	56	+ return ret;
	57	+}
	58	+#endif /* mul_u64_u32_add_u64_shr */
	59	+
	60	+#endif
	61	+
24	62	#endif /* __VDSO_MATH64_H */

kernel/time/Kconfig+1−1

@@ -202,7 +202,7 @@ config HIGH_RES_TIMERS · 201 unmodified lines
202	202	the size of the kernel image.
203	203
204	204	config CLOCKSOURCE_WATCHDOG_MAX_SKEW_US
205		− int "Clocksource watchdog maximum allowable skew (in μs)"
	205	+ int "Clocksource watchdog maximum allowable skew (in microseconds)"
206	206	depends on CLOCKSOURCE_WATCHDOG
207	207	range 50 1000
208	208	default 125

kernel/time/clockevents.c+1−1

@@ -677,7 +677,7 @@ static ssize_t current_device_show(struct device *dev, · 676 unmodified lines
677	677	raw_spin_lock_irq(&clockevents_lock);
678	678	td = tick_get_tick_dev(dev);
679	679	if (td && td->evtdev)
680		− count = snprintf(buf, PAGE_SIZE, "%s\n", td->evtdev->name);
	680	+ count = sysfs_emit(buf, "%s\n", td->evtdev->name);
681	681	raw_spin_unlock_irq(&clockevents_lock);
682	682	return count;
683	683	}

kernel/time/clocksource.c+21−23

@@ -20,6 +20,16 @@ · 19 unmodified lines
20	20	#include "tick-internal.h"
21	21	#include "timekeeping_internal.h"
22	22
	23	+static noinline u64 cycles_to_nsec_safe(struct clocksource *cs, u64 start, u64 end)
	24	+{
	25	+ u64 delta = clocksource_delta(end, start, cs->mask);
	26	+
	27	+ if (likely(delta < cs->max_cycles))
	28	+ return clocksource_cyc2ns(delta, cs->mult, cs->shift);
	29	+
	30	+ return mul_u64_u32_shr(delta, cs->mult, cs->shift);
	31	+}
	32	+
23	33	/**
24	34	* clocks_calc_mult_shift - calculate mult/shift factors for scaled math of clocks
25	35	* @mult: pointer to mult variable
@@ -222,8 +232,8 @@ enum wd_read_status { · 196 unmodified lines
222	232	static enum wd_read_status cs_watchdog_read(struct clocksource cs, u64 csnow, u64 *wdnow)
223	233	{
224	234	unsigned int nretries, max_retries;
225		− u64 wd_end, wd_end2, wd_delta;
226	235	int64_t wd_delay, wd_seq_delay;
	236	+ u64 wd_end, wd_end2;
227	237
228	238	max_retries = clocksource_get_max_watchdog_retry();
229	239	for (nretries = 0; nretries <= max_retries; nretries++) {
@@ -234,9 +244,7 @@ static enum wd_read_status cs_watchdog_read(struct clocksource cs, u64 csnow, · 4 unmodified lines
234	244	wd_end2 = watchdog->read(watchdog);
235	245	local_irq_enable();
236	246
237		− wd_delta = clocksource_delta(wd_end, *wdnow, watchdog->mask);
238		− wd_delay = clocksource_cyc2ns(wd_delta, watchdog->mult,
239		− watchdog->shift);
	247	+ wd_delay = cycles_to_nsec_safe(watchdog, *wdnow, wd_end);
240	248	if (wd_delay <= WATCHDOG_MAX_SKEW) {
241	249	if (nretries > 1 \|\| nretries >= max_retries) {
242	250	pr_warn("timekeeping watchdog on CPU%d: %s retried %d times before success\n",
@@ -254,8 +262,7 @@ static enum wd_read_status cs_watchdog_read(struct clocksource cs, u64 csnow, · 11 unmodified lines
254	262	* report system busy, reinit the watchdog and skip the current
255	263	* watchdog test.
256	264	*/
257		− wd_delta = clocksource_delta(wd_end2, wd_end, watchdog->mask);
258		− wd_seq_delay = clocksource_cyc2ns(wd_delta, watchdog->mult, watchdog->shift);
	265	+ wd_seq_delay = cycles_to_nsec_safe(watchdog, wd_end, wd_end2);
259	266	if (wd_seq_delay > WATCHDOG_MAX_SKEW/2)
260	267	goto skip_test;
261	268	}
@@ -366,8 +373,7 @@ void clocksource_verify_percpu(struct clocksource *cs) · 104 unmodified lines
366	373	delta = (csnow_end - csnow_mid) & cs->mask;
367	374	if (delta < 0)
368	375	cpumask_set_cpu(cpu, &cpus_ahead);
369		− delta = clocksource_delta(csnow_end, csnow_begin, cs->mask);
370		− cs_nsec = clocksource_cyc2ns(delta, cs->mult, cs->shift);
	376	+ cs_nsec = cycles_to_nsec_safe(cs, csnow_begin, csnow_end);
371	377	if (cs_nsec > cs_nsec_max)
372	378	cs_nsec_max = cs_nsec;
373	379	if (cs_nsec < cs_nsec_min)
@@ -398,8 +404,8 @@ static inline void clocksource_reset_watchdog(void) · 24 unmodified lines
398	404
399	405	static void clocksource_watchdog(struct timer_list *unused)
400	406	{
401		− u64 csnow, wdnow, cslast, wdlast, delta;
402	407	int64_t wd_nsec, cs_nsec, interval;
	408	+ u64 csnow, wdnow, cslast, wdlast;
403	409	int next_cpu, reset_pending;
404	410	struct clocksource *cs;
405	411	enum wd_read_status read_ret;
@@ -456,12 +462,8 @@ static void clocksource_watchdog(struct timer_list *unused) · 50 unmodified lines
456	462	continue;
457	463	}
458	464
459		− delta = clocksource_delta(wdnow, cs->wd_last, watchdog->mask);
460		− wd_nsec = clocksource_cyc2ns(delta, watchdog->mult,
461		− watchdog->shift);
462		−
463		− delta = clocksource_delta(csnow, cs->cs_last, cs->mask);
464		− cs_nsec = clocksource_cyc2ns(delta, cs->mult, cs->shift);
	465	+ wd_nsec = cycles_to_nsec_safe(watchdog, cs->wd_last, wdnow);
	466	+ cs_nsec = cycles_to_nsec_safe(cs, cs->cs_last, csnow);
465	467	wdlast = cs->wd_last; /* save these in case we print them */
466	468	cslast = cs->cs_last;
467	469	cs->cs_last = csnow;
@@ -832,7 +834,7 @@ void clocksource_start_suspend_timing(struct clocksource *cs, u64 start_cycles) · 364 unmodified lines
832	834	*/
833	835	u64 clocksource_stop_suspend_timing(struct clocksource *cs, u64 cycle_now)
834	836	{
835		− u64 now, delta, nsec = 0;
	837	+ u64 now, nsec = 0;
836	838
837	839	if (!suspend_clocksource)
838	840	return 0;
@@ -847,12 +849,8 @@ u64 clocksource_stop_suspend_timing(struct clocksource *cs, u64 cycle_now) · 8 unmodified lines
847	849	else
848	850	now = suspend_clocksource->read(suspend_clocksource);
849	851
850		− if (now > suspend_start) {
851		− delta = clocksource_delta(now, suspend_start,
852		− suspend_clocksource->mask);
853		− nsec = mul_u64_u32_shr(delta, suspend_clocksource->mult,
854		− suspend_clocksource->shift);
855		− }
	852	+ if (now > suspend_start)
	853	+ nsec = cycles_to_nsec_safe(suspend_clocksource, suspend_start, now);
856	854
857	855	/*
858	856	* Disable the suspend timer to save power if current clocksource is
@@ -1336,7 +1334,7 @@ static ssize_t current_clocksource_show(struct device *dev, · 477 unmodified lines
1336	1334	ssize_t count = 0;
1337	1335
1338	1336	mutex_lock(&clocksource_mutex);
1339		− count = snprintf(buf, PAGE_SIZE, "%s\n", curr_clocksource->name);
	1337	+ count = sysfs_emit(buf, "%s\n", curr_clocksource->name);
1340	1338	mutex_unlock(&clocksource_mutex);
1341	1339
1342	1340	return count;

kernel/time/hrtimer.c+9−32

@@ -644,17 +644,12 @@ static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base) · 643 unmodified lines
644	644	/*
645	645	* Is the high resolution mode active ?
646	646	*/
647		−static inline int __hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base)
	647	+static inline int hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base)
648	648	{
649	649	return IS_ENABLED(CONFIG_HIGH_RES_TIMERS) ?
650	650	cpu_base->hres_active : 0;
651	651	}
652	652
653		−static inline int hrtimer_hres_active(void)
654		−{
655		− return __hrtimer_hres_active(this_cpu_ptr(&hrtimer_bases));
656		−}
657		−
658	653	static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base,
659	654	struct hrtimer *next_timer,
660	655	ktime_t expires_next)
@@ -678,7 +673,7 @@ static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base, · 17 unmodified lines
678	673	* set. So we'd effectively block all timers until the T2 event
679	674	* fires.
680	675	*/
681		− if (!__hrtimer_hres_active(cpu_base) \|\| cpu_base->hang_detected)
	676	+ if (!hrtimer_hres_active(cpu_base) \|\| cpu_base->hang_detected)
682	677	return;
683	678
684	679	tick_program_event(expires_next, 1);
@@ -789,12 +784,12 @@ static void retrigger_next_event(void *arg) · 104 unmodified lines
789	784	* function call will take care of the reprogramming in case the
790	785	* CPU was in a NOHZ idle sleep.
791	786	*/
792		− if (!__hrtimer_hres_active(base) && !tick_nohz_active)
	787	+ if (!hrtimer_hres_active(base) && !tick_nohz_active)
793	788	return;
794	789
795	790	raw_spin_lock(&base->lock);
796	791	hrtimer_update_base(base);
797		− if (__hrtimer_hres_active(base))
	792	+ if (hrtimer_hres_active(base))
798	793	hrtimer_force_reprogram(base, 0);
799	794	else
800	795	hrtimer_update_next_event(base);
@@ -951,7 +946,7 @@ void clock_was_set(unsigned int bases) · 150 unmodified lines
951	946	cpumask_var_t mask;
952	947	int cpu;
953	948
954		− if (!__hrtimer_hres_active(cpu_base) && !tick_nohz_active)
	949	+ if (!hrtimer_hres_active(cpu_base) && !tick_nohz_active)
955	950	goto out_timerfd;
956	951
957	952	if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
@@ -1491,7 +1486,7 @@ u64 hrtimer_get_next_event(void) · 533 unmodified lines
1491	1486
1492	1487	raw_spin_lock_irqsave(&cpu_base->lock, flags);
1493	1488
1494		− if (!__hrtimer_hres_active(cpu_base))
	1489	+ if (!hrtimer_hres_active(cpu_base))
1495	1490	expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
1496	1491
1497	1492	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
@@ -1514,7 +1509,7 @@ u64 hrtimer_next_event_without(const struct hrtimer *exclude) · 16 unmodified lines
1514	1509
1515	1510	raw_spin_lock_irqsave(&cpu_base->lock, flags);
1516	1511
1517		− if (__hrtimer_hres_active(cpu_base)) {
	1512	+ if (hrtimer_hres_active(cpu_base)) {
1518	1513	unsigned int active;
1519	1514
1520	1515	if (!cpu_base->softirq_activated) {
@@ -1875,25 +1870,7 @@ retry: · 354 unmodified lines
1875	1870	tick_program_event(expires_next, 1);
1876	1871	pr_warn_once("hrtimer: interrupt took %llu ns\n", ktime_to_ns(delta));
1877	1872	}
1878		−
1879		−/* called with interrupts disabled */
1880		−static inline void __hrtimer_peek_ahead_timers(void)
1881		−{
1882		− struct tick_device *td;
1883		−
1884		− if (!hrtimer_hres_active())
1885		− return;
1886		−
1887		− td = this_cpu_ptr(&tick_cpu_device);
1888		− if (td && td->evtdev)
1889		− hrtimer_interrupt(td->evtdev);
1890		−}
1891		−
1892		−#else /* CONFIG_HIGH_RES_TIMERS */
1893		−
1894		−static inline void __hrtimer_peek_ahead_timers(void) { }
1895		−
1896		−#endif /* !CONFIG_HIGH_RES_TIMERS */
	1873	+#endif /* !CONFIG_HIGH_RES_TIMERS */
1897	1874
1898	1875	/*
1899	1876	* Called from run_local_timers in hardirq context every jiffy
@@ -1904,7 +1881,7 @@ void hrtimer_run_queues(void) · 4 unmodified lines
1904	1881	unsigned long flags;
1905	1882	ktime_t now;
1906	1883
1907		− if (__hrtimer_hres_active(cpu_base))
	1884	+ if (hrtimer_hres_active(cpu_base))
1908	1885	return;
1909	1886
1910	1887	/*

kernel/time/timekeeping.c+49−47

@@ -237,7 +237,9 @@ static void timekeeping_check_update(struct timekeeper *tk, u64 offset) · 236 unmodified lines
237	237	}
238	238	}
239	239
240		−static inline u64 timekeeping_get_delta(const struct tk_read_base *tkr)
	240	+static inline u64 timekeeping_cycles_to_ns(const struct tk_read_base *tkr, u64 cycles);
	241	+
	242	+static inline u64 timekeeping_debug_get_ns(const struct tk_read_base *tkr)
241	243	{
242	244	struct timekeeper *tk = &tk_core.timekeeper;
243	245	u64 now, last, mask, max, delta;
@@ -264,34 +266,23 @@ static inline u64 timekeeping_get_delta(const struct tk_read_base *tkr) · 20 unmodified lines
264	266	* Try to catch underflows by checking if we are seeing small
265	267	* mask-relative negative values.
266	268	*/
267		− if (unlikely((~delta & mask) < (mask >> 3))) {
	269	+ if (unlikely((~delta & mask) < (mask >> 3)))
268	270	tk->underflow_seen = 1;
269		− delta = 0;
270		− }
271	271
272		− /* Cap delta value to the max_cycles values to avoid mult overflows */
273		− if (unlikely(delta > max)) {
	272	+ /* Check for multiplication overflows */
	273	+ if (unlikely(delta > max))
274	274	tk->overflow_seen = 1;
275		− delta = tkr->clock->max_cycles;
276		− }
277	275
278		− return delta;
	276	+ /* timekeeping_cycles_to_ns() handles both under and overflow */
	277	+ return timekeeping_cycles_to_ns(tkr, now);
279	278	}
280	279	#else
281	280	static inline void timekeeping_check_update(struct timekeeper *tk, u64 offset)
282	281	{
283	282	}
284		−static inline u64 timekeeping_get_delta(const struct tk_read_base *tkr)
	283	+static inline u64 timekeeping_debug_get_ns(const struct tk_read_base *tkr)
285	284	{
286		− u64 cycle_now, delta;
287		−
288		− /* read clocksource */
289		− cycle_now = tk_clock_read(tkr);
290		−
291		− /* calculate the delta since the last update_wall_time */
292		− delta = clocksource_delta(cycle_now, tkr->cycle_last, tkr->mask);
293		−
294		− return delta;
	285	+ BUG();
295	286	}
296	287	#endif
297	288
@@ -370,32 +361,46 @@ static void tk_setup_internals(struct timekeeper tk, struct clocksource clock) · 72 unmodified lines
370	361	}
371	362
372	363	/* Timekeeper helper functions. */
	364	+static noinline u64 delta_to_ns_safe(const struct tk_read_base *tkr, u64 delta)
	365	+{
	366	+ return mul_u64_u32_add_u64_shr(delta, tkr->mult, tkr->xtime_nsec, tkr->shift);
	367	+}
373	368
374		−static inline u64 timekeeping_delta_to_ns(const struct tk_read_base *tkr, u64 delta)
	369	+static inline u64 timekeeping_cycles_to_ns(const struct tk_read_base *tkr, u64 cycles)
375	370	{
376		− u64 nsec;
	371	+ /* Calculate the delta since the last update_wall_time() */
	372	+ u64 mask = tkr->mask, delta = (cycles - tkr->cycle_last) & mask;
377	373
378		− nsec = delta * tkr->mult + tkr->xtime_nsec;
379		− nsec >>= tkr->shift;
	374	+ /*
	375	+ * This detects both negative motion and the case where the delta
	376	+ * overflows the multiplication with tkr->mult.
	377	+ */
	378	+ if (unlikely(delta > tkr->clock->max_cycles)) {
	379	+ /*
	380	+ * Handle clocksource inconsistency between CPUs to prevent
	381	+ * time from going backwards by checking for the MSB of the
	382	+ * mask being set in the delta.
	383	+ */
	384	+ if (delta & ~(mask >> 1))
	385	+ return tkr->xtime_nsec >> tkr->shift;
	386	+
	387	+ return delta_to_ns_safe(tkr, delta);
	388	+ }
380	389
381		− return nsec;
	390	+ return ((delta * tkr->mult) + tkr->xtime_nsec) >> tkr->shift;
382	391	}
383	392
384		−static inline u64 timekeeping_get_ns(const struct tk_read_base *tkr)
	393	+static __always_inline u64 __timekeeping_get_ns(const struct tk_read_base *tkr)
385	394	{
386		− u64 delta;
387		−
388		− delta = timekeeping_get_delta(tkr);
389		− return timekeeping_delta_to_ns(tkr, delta);
	395	+ return timekeeping_cycles_to_ns(tkr, tk_clock_read(tkr));
390	396	}
391	397
392		−static inline u64 timekeeping_cycles_to_ns(const struct tk_read_base *tkr, u64 cycles)
	398	+static inline u64 timekeeping_get_ns(const struct tk_read_base *tkr)
393	399	{
394		− u64 delta;
	400	+ if (IS_ENABLED(CONFIG_DEBUG_TIMEKEEPING))
	401	+ return timekeeping_debug_get_ns(tkr);
395	402
396		− /* calculate the delta since the last update_wall_time */
397		− delta = clocksource_delta(cycles, tkr->cycle_last, tkr->mask);
398		− return timekeeping_delta_to_ns(tkr, delta);
	403	+ return __timekeeping_get_ns(tkr);
399	404	}
400	405
401	406	/**
@@ -431,14 +436,6 @@ static void update_fast_timekeeper(const struct tk_read_base *tkr, · 29 unmodified lines
431	436	memcpy(base + 1, base, sizeof(*base));
432	437	}
433	438
434		−static __always_inline u64 fast_tk_get_delta_ns(struct tk_read_base *tkr)
435		−{
436		− u64 delta, cycles = tk_clock_read(tkr);
437		−
438		− delta = clocksource_delta(cycles, tkr->cycle_last, tkr->mask);
439		− return timekeeping_delta_to_ns(tkr, delta);
440		−}
441		−
442	439	static __always_inline u64 __ktime_get_fast_ns(struct tk_fast *tkf)
443	440	{
444	441	struct tk_read_base *tkr;
@@ -449,7 +446,7 @@ static __always_inline u64 __ktime_get_fast_ns(struct tk_fast *tkf) · 4 unmodified lines
449	446	seq = raw_read_seqcount_latch(&tkf->seq);
450	447	tkr = tkf->base + (seq & 0x01);
451	448	now = ktime_to_ns(tkr->base);
452		− now += fast_tk_get_delta_ns(tkr);
	449	+ now += __timekeeping_get_ns(tkr);
453	450	} while (raw_read_seqcount_latch_retry(&tkf->seq, seq));
454	451
455	452	return now;
@@ -565,7 +562,7 @@ static __always_inline u64 __ktime_get_real_fast(struct tk_fast tkf, u64 mono) · 109 unmodified lines
565	562	tkr = tkf->base + (seq & 0x01);
566	563	basem = ktime_to_ns(tkr->base);
567	564	baser = ktime_to_ns(tkr->base_real);
568		− delta = fast_tk_get_delta_ns(tkr);
	565	+ delta = __timekeeping_get_ns(tkr);
569	566	} while (raw_read_seqcount_latch_retry(&tkf->seq, seq));
570	567
571	568	if (mono)
@@ -800,10 +797,15 @@ static void timekeeping_forward_now(struct timekeeper *tk) · 228 unmodified lines
800	797	tk->tkr_mono.cycle_last = cycle_now;
801	798	tk->tkr_raw.cycle_last = cycle_now;
802	799
803		− tk->tkr_mono.xtime_nsec += delta * tk->tkr_mono.mult;
804		− tk->tkr_raw.xtime_nsec += delta * tk->tkr_raw.mult;
	800	+ while (delta > 0) {
	801	+ u64 max = tk->tkr_mono.clock->max_cycles;
	802	+ u64 incr = delta < max ? delta : max;
805	803
806		− tk_normalize_xtime(tk);
	804	+ tk->tkr_mono.xtime_nsec += incr * tk->tkr_mono.mult;
	805	+ tk->tkr_raw.xtime_nsec += incr * tk->tkr_raw.mult;
	806	+ tk_normalize_xtime(tk);
	807	+ delta -= incr;
	808	+ }
807	809	}
808	810
809	811	/**

kernel/time/vsyscall.c+6−0

@@ -22,10 +22,16 @@ static inline void update_vdso_data(struct vdso_data *vdata, · 21 unmodified lines
22	22	u64 nsec, sec;
23	23
24	24	vdata[CS_HRES_COARSE].cycle_last = tk->tkr_mono.cycle_last;
	25	+#ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT
	26	+ vdata[CS_HRES_COARSE].max_cycles = tk->tkr_mono.clock->max_cycles;
	27	+#endif
25	28	vdata[CS_HRES_COARSE].mask = tk->tkr_mono.mask;
26	29	vdata[CS_HRES_COARSE].mult = tk->tkr_mono.mult;
27	30	vdata[CS_HRES_COARSE].shift = tk->tkr_mono.shift;
28	31	vdata[CS_RAW].cycle_last = tk->tkr_raw.cycle_last;
	32	+#ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT
	33	+ vdata[CS_RAW].max_cycles = tk->tkr_raw.clock->max_cycles;
	34	+#endif
29	35	vdata[CS_RAW].mask = tk->tkr_raw.mask;
30	36	vdata[CS_RAW].mult = tk->tkr_raw.mult;
31	37	vdata[CS_RAW].shift = tk->tkr_raw.shift;

lib/vdso/Kconfig+7−0

@@ -30,4 +30,11 @@ config GENERIC_VDSO_TIME_NS · 29 unmodified lines
30	30	Selected by architectures which support time namespaces in the
31	31	VDSO
32	32
	33	+config GENERIC_VDSO_OVERFLOW_PROTECT
	34	+ bool
	35	+ help
	36	+ Select to add multiplication overflow protection to the VDSO
	37	+ time getter functions for the price of an extra conditional
	38	+ in the hotpath.
	39	+
33	40	endif

lib/vdso/gettimeofday.c+36−19

@@ -5,15 +5,23 @@ · 4 unmodified lines
5	5	#include <vdso/datapage.h>
6	6	#include <vdso/helpers.h>
7	7
8		−#ifndef vdso_calc_delta
9		−/*
10		− * Default implementation which works for all sane clocksources. That
11		− * obviously excludes x86/TSC.
12		− */
13		−static __always_inline
14		−u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult)
	8	+#ifndef vdso_calc_ns
	9	+
	10	+#ifdef VDSO_DELTA_NOMASK
	11	+# define VDSO_DELTA_MASK(vd) ULLONG_MAX
	12	+#else
	13	+# define VDSO_DELTA_MASK(vd) (vd->mask)
	14	+#endif
	15	+
	16	+#ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT
	17	+static __always_inline bool vdso_delta_ok(const struct vdso_data *vd, u64 delta)
	18	+{
	19	+ return delta < vd->max_cycles;
	20	+}
	21	+#else
	22	+static __always_inline bool vdso_delta_ok(const struct vdso_data *vd, u64 delta)
15	23	{
16		− return ((cycles - last) & mask) * mult;
	24	+ return true;
17	25	}
18	26	#endif
19	27
@@ -24,6 +32,21 @@ static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift) · 4 unmodified lines
24	32	}
25	33	#endif
26	34
	35	+/*
	36	+ * Default implementation which works for all sane clocksources. That
	37	+ * obviously excludes x86/TSC.
	38	+ */
	39	+static __always_inline u64 vdso_calc_ns(const struct vdso_data *vd, u64 cycles, u64 base)
	40	+{
	41	+ u64 delta = (cycles - vd->cycle_last) & VDSO_DELTA_MASK(vd);
	42	+
	43	+ if (likely(vdso_delta_ok(vd, delta)))
	44	+ return vdso_shift_ns((delta * vd->mult) + base, vd->shift);
	45	+
	46	+ return mul_u64_u32_add_u64_shr(delta, vd->mult, base, vd->shift);
	47	+}
	48	+#endif /* vdso_calc_ns */
	49	+
27	50	#ifndef __arch_vdso_hres_capable
28	51	static inline bool __arch_vdso_hres_capable(void)
29	52	{
@@ -49,10 +72,10 @@ static inline bool vdso_cycles_ok(u64 cycles) · 19 unmodified lines
49	72	static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
50	73	struct __kernel_timespec *ts)
51	74	{
52		− const struct vdso_data *vd;
53	75	const struct timens_offset *offs = &vdns->offset[clk];
54	76	const struct vdso_timestamp *vdso_ts;
55		− u64 cycles, last, ns;
	77	+ const struct vdso_data *vd;
	78	+ u64 cycles, ns;
56	79	u32 seq;
57	80	s64 sec;
58	81
@@ -73,10 +96,7 @@ static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_ · 14 unmodified lines
73	96	cycles = __arch_get_hw_counter(vd->clock_mode, vd);
74	97	if (unlikely(!vdso_cycles_ok(cycles)))
75	98	return -1;
76		− ns = vdso_ts->nsec;
77		− last = vd->cycle_last;
78		− ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
79		− ns = vdso_shift_ns(ns, vd->shift);
	99	+ ns = vdso_calc_ns(vd, cycles, vdso_ts->nsec);
80	100	sec = vdso_ts->sec;
81	101	} while (unlikely(vdso_read_retry(vd, seq)));
82	102
@@ -111,7 +131,7 @@ static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, · 28 unmodified lines
111	131	struct __kernel_timespec *ts)
112	132	{
113	133	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
114		− u64 cycles, last, sec, ns;
	134	+ u64 cycles, sec, ns;
115	135	u32 seq;
116	136
117	137	/* Allows to compile the high resolution parts out */
@@ -144,10 +164,7 @@ static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, · 26 unmodified lines
144	164	cycles = __arch_get_hw_counter(vd->clock_mode, vd);
145	165	if (unlikely(!vdso_cycles_ok(cycles)))
146	166	return -1;
147		− ns = vdso_ts->nsec;
148		− last = vd->cycle_last;
149		− ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
150		− ns = vdso_shift_ns(ns, vd->shift);
	167	+ ns = vdso_calc_ns(vd, cycles, vdso_ts->nsec);
151	168	sec = vdso_ts->sec;
152	169	} while (unlikely(vdso_read_retry(vd, seq)));
153	170

rust/kernel/time.rs+63−0

@@ -4,6 +4,12 @@ · 3 unmodified lines
4	4	//!
5	5	//! This module contains the kernel APIs related to time and timers that
6	6	//! have been ported or wrapped for usage by Rust code in the kernel.
	7	+//!
	8	+//! C header: [`include/linux/jiffies.h`](srctree/include/linux/jiffies.h).
	9	+//! C header: [`include/linux/ktime.h`](srctree/include/linux/ktime.h).
	10	+
	11	+/// The number of nanoseconds per millisecond.
	12	+pub const NSEC_PER_MSEC: i64 = bindings::NSEC_PER_MSEC as i64;
7	13
8	14	/// The time unit of Linux kernel. One jiffy equals (1/HZ) second.
9	15	pub type Jiffies = core::ffi::c_ulong;
@@ -18,3 +24,60 @@ pub fn msecs_to_jiffies(msecs: Msecs) -> Jiffies { · 8 unmodified lines
18	24	// matter what the argument is.
19	25	unsafe { bindings::__msecs_to_jiffies(msecs) }
20	26	}
	27	+
	28	+/// A Rust wrapper around a `ktime_t`.
	29	+#[repr(transparent)]
	30	+#[derive(Copy, Clone)]
	31	+pub struct Ktime {
	32	+ inner: bindings::ktime_t,
	33	+}
	34	+
	35	+impl Ktime {
	36	+ /// Create a `Ktime` from a raw `ktime_t`.
	37	+ #[inline]
	38	+ pub fn from_raw(inner: bindings::ktime_t) -> Self {
	39	+ Self { inner }
	40	+ }
	41	+
	42	+ /// Get the current time using `CLOCK_MONOTONIC`.
	43	+ #[inline]
	44	+ pub fn ktime_get() -> Self {
	45	+ // SAFETY: It is always safe to call `ktime_get` outside of NMI context.
	46	+ Self::from_raw(unsafe { bindings::ktime_get() })
	47	+ }
	48	+
	49	+ /// Divide the number of nanoseconds by a compile-time constant.
	50	+ #[inline]
	51	+ fn divns_constant<const DIV: i64>(self) -> i64 {
	52	+ self.to_ns() / DIV
	53	+ }
	54	+
	55	+ /// Returns the number of nanoseconds.
	56	+ #[inline]
	57	+ pub fn to_ns(self) -> i64 {
	58	+ self.inner
	59	+ }
	60	+
	61	+ /// Returns the number of milliseconds.
	62	+ #[inline]
	63	+ pub fn to_ms(self) -> i64 {
	64	+ self.divns_constant::<NSEC_PER_MSEC>()
	65	+ }
	66	+}
	67	+
	68	+/// Returns the number of milliseconds between two ktimes.
	69	+#[inline]
	70	+pub fn ktime_ms_delta(later: Ktime, earlier: Ktime) -> i64 {
	71	+ (later - earlier).to_ms()
	72	+}
	73	+
	74	+impl core::ops::Sub for Ktime {
	75	+ type Output = Ktime;
	76	+
	77	+ #[inline]
	78	+ fn sub(self, other: Ktime) -> Ktime {
	79	+ Self {
	80	+ inner: self.inner - other.inner,
	81	+ }
	82	+ }
	83	+}