Merge tag 'timers-urgent-2020-08-14' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull timekeeping updates from Thomas Gleixner: "A set of timekeeping/VDSO updates: - Preparatory work to allow S390 to switch over to the generic VDSO implementation. S390 requires that the VDSO data pointer is handed in to the counter read function when time namespace support is enabled. Adding the pointer is a NOOP for all other architectures because the compiler is supposed to optimize that out when it is unused in the architecture specific inline. The change also solved a similar problem for MIPS which fortunately has time namespaces not yet enabled. S390 needs to update clock related VDSO data independent of the timekeeping updates. This was solved so far with yet another sequence counter in the S390 implementation. A better solution is to utilize the already existing VDSO sequence count for this. The core code now exposes helper functions which allow to serialize against the timekeeper code and against concurrent readers. S390 needs extra data for their clock readout function. The initial common VDSO data structure did not provide a way to add that. It now has an embedded architecture specific struct embedded which defaults to an empty struct. Doing this now avoids tree dependencies and conflicts post rc1 and allows all other architectures which work on generic VDSO support to work from a common upstream base. - A trivial comment fix" * tag 'timers-urgent-2020-08-14' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: time: Delete repeated words in comments lib/vdso: Allow to add architecture-specific vdso data timekeeping/vsyscall: Provide vdso_update_begin/end() vdso/treewide: Add vdso_data pointer argument to __arch_get_hw_counter()

Linus Torvalds committed 6 years ago

Commit b923f122 parents b6b178e b0294f315 files · +84 −16Browse files

Unified · Split

15 files

arch/Kconfig +3 −0
arch/arm/include/asm/vdso/gettimeofday.h +2 −1
arch/arm64/include/asm/vdso/compat_gettimeofday.h +2 −1
arch/arm64/include/asm/vdso/gettimeofday.h +2 −1
arch/mips/include/asm/vdso/gettimeofday.h +3 −2
arch/riscv/include/asm/vdso/gettimeofday.h +2 −1
arch/x86/include/asm/vdso/gettimeofday.h +2 −1
include/vdso/datapage.h +10 −0
include/vdso/vsyscall.h +3 −0
kernel/time/alarmtimer.c +1 −1
kernel/time/sched_clock.c +1 −1
kernel/time/timekeeping.c +2 −2
kernel/time/timekeeping_internal.h +8 −3
kernel/time/vsyscall.c +41 −0
lib/vdso/gettimeofday.c +2 −2

arch/Kconfig+3−0

@@ -972,6 +972,9 @@ config HAVE_SPARSE_SYSCALL_NR · 971 unmodified lines
972	972	entries at 4000, 5000 and 6000 locations. This option turns on syscall
973	973	related optimizations for a given architecture.
974	974
	975	+config ARCH_HAS_VDSO_DATA
	976	+ bool
	977	+
975	978	source "kernel/gcov/Kconfig"
976	979
977	980	source "scripts/gcc-plugins/Kconfig"

arch/arm/include/asm/vdso/gettimeofday.h+2−1

@@ -113,7 +113,8 @@ static inline bool arm_vdso_hres_capable(void) · 112 unmodified lines
113	113	}
114	114	#define __arch_vdso_hres_capable arm_vdso_hres_capable
115	115
116		−static __always_inline u64 __arch_get_hw_counter(int clock_mode)
	116	+static __always_inline u64 __arch_get_hw_counter(int clock_mode,
	117	+ const struct vdso_data *vd)
117	118	{
118	119	#ifdef CONFIG_ARM_ARCH_TIMER
119	120	u64 cycle_now;

arch/arm64/include/asm/vdso/compat_gettimeofday.h+2−1

@@ -103,7 +103,8 @@ int clock_getres32_fallback(clockid_t _clkid, struct old_timespec32 *_ts) · 102 unmodified lines
103	103	return ret;
104	104	}
105	105
106		−static __always_inline u64 __arch_get_hw_counter(s32 clock_mode)
	106	+static __always_inline u64 __arch_get_hw_counter(s32 clock_mode,
	107	+ const struct vdso_data *vd)
107	108	{
108	109	u64 res;
109	110

arch/arm64/include/asm/vdso/gettimeofday.h+2−1

@@ -64,7 +64,8 @@ int clock_getres_fallback(clockid_t _clkid, struct __kernel_timespec *_ts) · 63 unmodified lines
64	64	return ret;
65	65	}
66	66
67		−static __always_inline u64 __arch_get_hw_counter(s32 clock_mode)
	67	+static __always_inline u64 __arch_get_hw_counter(s32 clock_mode,
	68	+ const struct vdso_data *vd)
68	69	{
69	70	u64 res;
70	71

arch/mips/include/asm/vdso/gettimeofday.h+3−2

@@ -167,7 +167,8 @@ static __always_inline u64 read_gic_count(const struct vdso_data *data) · 166 unmodified lines
167	167
168	168	#endif
169	169
170		−static __always_inline u64 __arch_get_hw_counter(s32 clock_mode)
	170	+static __always_inline u64 __arch_get_hw_counter(s32 clock_mode,
	171	+ const struct vdso_data *vd)
171	172	{
172	173	#ifdef CONFIG_CSRC_R4K
173	174	if (clock_mode == VDSO_CLOCKMODE_R4K)
@@ -175,7 +176,7 @@ static __always_inline u64 __arch_get_hw_counter(s32 clock_mode) · 1 unmodified line
175	176	#endif
176	177	#ifdef CONFIG_CLKSRC_MIPS_GIC
177	178	if (clock_mode == VDSO_CLOCKMODE_GIC)
178		− return read_gic_count(get_vdso_data());
	179	+ return read_gic_count(vd);
179	180	#endif
180	181	/*
181	182	* Core checks mode already. So this raced against a concurrent

arch/riscv/include/asm/vdso/gettimeofday.h+2−1

@@ -60,7 +60,8 @@ int clock_getres_fallback(clockid_t _clkid, struct __kernel_timespec *_ts) · 59 unmodified lines
60	60	return ret;
61	61	}
62	62
63		−static __always_inline u64 __arch_get_hw_counter(s32 clock_mode)
	63	+static __always_inline u64 __arch_get_hw_counter(s32 clock_mode,
	64	+ const struct vdso_data *vd)
64	65	{
65	66	/*
66	67	* The purpose of csr_read(CSR_TIME) is to trap the system into

arch/x86/include/asm/vdso/gettimeofday.h+2−1

@@ -241,7 +241,8 @@ static u64 vread_hvclock(void) · 240 unmodified lines
241	241	}
242	242	#endif
243	243
244		−static inline u64 __arch_get_hw_counter(s32 clock_mode)
	244	+static inline u64 __arch_get_hw_counter(s32 clock_mode,
	245	+ const struct vdso_data *vd)
245	246	{
246	247	if (likely(clock_mode == VDSO_CLOCKMODE_TSC))
247	248	return (u64)rdtsc_ordered();

include/vdso/datapage.h+10−0

@@ -19,6 +19,12 @@ · 18 unmodified lines
19	19	#include <vdso/time32.h>
20	20	#include <vdso/time64.h>
21	21
	22	+#ifdef CONFIG_ARCH_HAS_VDSO_DATA
	23	+#include <asm/vdso/data.h>
	24	+#else
	25	+struct arch_vdso_data {};
	26	+#endif
	27	+
22	28	#define VDSO_BASES (CLOCK_TAI + 1)
23	29	#define VDSO_HRES (BIT(CLOCK_REALTIME) \| \
24	30	BIT(CLOCK_MONOTONIC) \| \
@@ -64,6 +70,8 @@ struct vdso_timestamp { · 39 unmodified lines
64	70	* @tz_dsttime: type of DST correction
65	71	* @hrtimer_res: hrtimer resolution
66	72	* @__unused: unused
	73	+ * @arch_data: architecture specific data (optional, defaults
	74	+ * to an empty struct)
67	75	*
68	76	* vdso_data will be accessed by 64 bit and compat code at the same time
69	77	* so we should be careful before modifying this structure.
@@ -97,6 +105,8 @@ struct vdso_data { · 27 unmodified lines
97	105	s32 tz_dsttime;
98	106	u32 hrtimer_res;
99	107	u32 __unused;
	108	+
	109	+ struct arch_vdso_data arch_data;
100	110	};
101	111
102	112	/*

include/vdso/vsyscall.h+3−0

@@ -6,6 +6,9 @@ · 5 unmodified lines
6	6
7	7	#include <asm/vdso/vsyscall.h>
8	8
	9	+unsigned long vdso_update_begin(void);
	10	+void vdso_update_end(unsigned long flags);
	11	+
9	12	#endif /* !__ASSEMBLY__ */
10	13
11	14	#endif /* __VDSO_VSYSCALL_H */

kernel/time/alarmtimer.c+1−1

@@ -192,7 +192,7 @@ static void alarmtimer_dequeue(struct alarm_base base, struct alarm alarm) · 191 unmodified lines
192	192	* When a alarm timer fires, this runs through the timerqueue to
193	193	* see which alarms expired, and runs those. If there are more alarm
194	194	* timers queued for the future, we set the hrtimer to fire when
195		− * when the next future alarm timer expires.
	195	+ * the next future alarm timer expires.
196	196	*/
197	197	static enum hrtimer_restart alarmtimer_fired(struct hrtimer *timer)
198	198	{

kernel/time/sched_clock.c+1−1

@@ -229,7 +229,7 @@ void __init generic_sched_clock_init(void) · 228 unmodified lines
229	229	{
230	230	/*
231	231	* If no sched_clock() function has been provided at that point,
232		− * make it the final one one.
	232	+ * make it the final one.
233	233	*/
234	234	if (cd.actual_read_sched_clock == jiffy_sched_clock_read)
235	235	sched_clock_register(jiffy_sched_clock_read, BITS_PER_LONG, HZ);

kernel/time/timekeeping.c+2−2

@@ -39,7 +39,7 @@ enum timekeeping_adv_mode { · 38 unmodified lines
39	39	TK_ADV_FREQ
40	40	};
41	41
42		−static DEFINE_RAW_SPINLOCK(timekeeper_lock);
	42	+DEFINE_RAW_SPINLOCK(timekeeper_lock);
43	43
44	44	/*
45	45	* The most important data for readout fits into a single 64 byte
@@ -2004,7 +2004,7 @@ static inline unsigned int accumulate_nsecs_to_secs(struct timekeeper *tk) · 1958 unmodified lines
2004	2004	* logarithmic_accumulation - shifted accumulation of cycles
2005	2005	*
2006	2006	* This functions accumulates a shifted interval of cycles into
2007		− * into a shifted interval nanoseconds. Allows for O(log) accumulation
	2007	+ * a shifted interval nanoseconds. Allows for O(log) accumulation
2008	2008	* loop.
2009	2009	*
2010	2010	* Returns the unconsumed cycles.

kernel/time/timekeeping_internal.h+8−3

@@ -1,12 +1,14 @@
1	1	/* SPDX-License-Identifier: GPL-2.0 */
2	2	#ifndef _TIMEKEEPING_INTERNAL_H
3	3	#define _TIMEKEEPING_INTERNAL_H
4		−/*
5		− * timekeeping debug functions
6		− */
	4	+
7	5	#include <linux/clocksource.h>
	6	+#include <linux/spinlock.h>
8	7	#include <linux/time.h>
9	8
	9	+/*
	10	+ * timekeeping debug functions
	11	+ */
10	12	#ifdef CONFIG_DEBUG_FS
11	13	extern void tk_debug_account_sleep_time(const struct timespec64 *t);
12	14	#else
@@ -31,4 +33,7 @@ static inline u64 clocksource_delta(u64 now, u64 last, u64 mask) · 18 unmodified lines
31	33	}
32	34	#endif
33	35
	36	+/* Semi public for serialization of non timekeeper VDSO updates. */
	37	+extern raw_spinlock_t timekeeper_lock;
	38	+
34	39	#endif /* _TIMEKEEPING_INTERNAL_H */

kernel/time/vsyscall.c+41−0

@@ -13,6 +13,8 @@ · 12 unmodified lines
13	13	#include <vdso/helpers.h>
14	14	#include <vdso/vsyscall.h>
15	15
	16	+#include "timekeeping_internal.h"
	17	+
16	18	static inline void update_vdso_data(struct vdso_data *vdata,
17	19	struct timekeeper *tk)
18	20	{
@@ -127,3 +129,42 @@ void update_vsyscall_tz(void) · 108 unmodified lines
127	129
128	130	__arch_sync_vdso_data(vdata);
129	131	}
	132	+
	133	+/**
	134	+ * vdso_update_begin - Start of a VDSO update section
	135	+ *
	136	+ * Allows architecture code to safely update the architecture specific VDSO
	137	+ * data. Disables interrupts, acquires timekeeper lock to serialize against
	138	+ * concurrent updates from timekeeping and invalidates the VDSO data
	139	+ * sequence counter to prevent concurrent readers from accessing
	140	+ * inconsistent data.
	141	+ *
	142	+ * Returns: Saved interrupt flags which need to be handed in to
	143	+ * vdso_update_end().
	144	+ */
	145	+unsigned long vdso_update_begin(void)
	146	+{
	147	+ struct vdso_data *vdata = __arch_get_k_vdso_data();
	148	+ unsigned long flags;
	149	+
	150	+ raw_spin_lock_irqsave(&timekeeper_lock, flags);
	151	+ vdso_write_begin(vdata);
	152	+ return flags;
	153	+}
	154	+
	155	+/**
	156	+ * vdso_update_end - End of a VDSO update section
	157	+ * @flags: Interrupt flags as returned from vdso_update_begin()
	158	+ *
	159	+ * Pairs with vdso_update_begin(). Marks vdso data consistent, invokes data
	160	+ * synchronization if the architecture requires it, drops timekeeper lock
	161	+ * and restores interrupt flags.
	162	+ */
	163	+void vdso_update_end(unsigned long flags)
	164	+{
	165	+ struct vdso_data *vdata = __arch_get_k_vdso_data();
	166	+
	167	+ vdso_write_end(vdata);
	168	+ __arch_sync_vdso_data(vdata);
	169	+ raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
	170	+}

lib/vdso/gettimeofday.c+2−2

@@ -68,7 +68,7 @@ static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, · 67 unmodified lines
68	68	if (unlikely(!vdso_clocksource_ok(vd)))
69	69	return -1;
70	70
71		− cycles = __arch_get_hw_counter(vd->clock_mode);
	71	+ cycles = __arch_get_hw_counter(vd->clock_mode, vd);
72	72	if (unlikely(!vdso_cycles_ok(cycles)))
73	73	return -1;
74	74	ns = vdso_ts->nsec;
@@ -138,7 +138,7 @@ static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, · 63 unmodified lines
138	138	if (unlikely(!vdso_clocksource_ok(vd)))
139	139	return -1;
140	140
141		− cycles = __arch_get_hw_counter(vd->clock_mode);
	141	+ cycles = __arch_get_hw_counter(vd->clock_mode, vd);
142	142	if (unlikely(!vdso_cycles_ok(cycles)))
143	143	return -1;
144	144	ns = vdso_ts->nsec;

Merge tag 'timers-urgent-2020-08-14' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code