History

Merge tag 'timers-urgent-2020-08-14' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull timekeeping updates from Thomas Gleixner: "A set of timekeeping/VDSO updates: - Preparatory work to allow S390 to switch over to the generic VDSO implementation. S390 requires that the VDSO data pointer is handed in to the counter read function when time namespace support is enabled. Adding the pointer is a NOOP for all other architectures because the compiler is supposed to optimize that out when it is unused in the architecture specific inline. The change also solved a similar problem for MIPS which fortunately has time namespaces not yet enabled. S390 needs to update clock related VDSO data independent of the timekeeping updates. This was solved so far with yet another sequence counter in the S390 implementation. A better solution is to utilize the already existing VDSO sequence count for this. The core code now exposes helper functions which allow to serialize against the timekeeper code and against concurrent readers. S390 needs extra data for their clock readout function. The initial common VDSO data structure did not provide a way to add that. It now has an embedded architecture specific struct embedded which defaults to an empty struct. Doing this now avoids tree dependencies and conflicts post rc1 and allows all other architectures which work on generic VDSO support to work from a common upstream base. - A trivial comment fix" * tag 'timers-urgent-2020-08-14' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: time: Delete repeated words in comments lib/vdso: Allow to add architecture-specific vdso data timekeeping/vsyscall: Provide vdso_update_begin/end() vdso/treewide: Add vdso_data pointer argument to __arch_get_hw_counter()

Linus Torvalds committed 6 years ago

Commit b923f122 parents b6b178e b0294f315 files · +84 −16Browse files

Unified · Split

15 files
  1. arch/Kconfig +3 −0
  2. arch/arm/include/asm/vdso/gettimeofday.h +2 −1
  3. arch/arm64/include/asm/vdso/compat_gettimeofday.h +2 −1
  4. arch/arm64/include/asm/vdso/gettimeofday.h +2 −1
  5. arch/mips/include/asm/vdso/gettimeofday.h +3 −2
  6. arch/riscv/include/asm/vdso/gettimeofday.h +2 −1
  7. arch/x86/include/asm/vdso/gettimeofday.h +2 −1
  8. include/vdso/datapage.h +10 −0
  9. include/vdso/vsyscall.h +3 −0
  10. kernel/time/alarmtimer.c +1 −1
  11. kernel/time/sched_clock.c +1 −1
  12. kernel/time/timekeeping.c +2 −2
  13. kernel/time/timekeeping_internal.h +8 −3
  14. kernel/time/vsyscall.c +41 −0
  15. lib/vdso/gettimeofday.c +2 −2
arch/Kconfig+3−0
@@ -972,6 +972,9 @@ config HAVE_SPARSE_SYSCALL_NR · 971 unmodified lines
972 entries at 4000, 5000 and 6000 locations. This option turns on syscall972 entries at 4000, 5000 and 6000 locations. This option turns on syscall
973 related optimizations for a given architecture.973 related optimizations for a given architecture.
974 974
975config ARCH_HAS_VDSO_DATA
976 bool
977
975source "kernel/gcov/Kconfig"978source "kernel/gcov/Kconfig"
976 979
977source "scripts/gcc-plugins/Kconfig"980source "scripts/gcc-plugins/Kconfig"
arch/arm/include/asm/vdso/gettimeofday.h+2−1
@@ -113,7 +113,8 @@ static inline bool arm_vdso_hres_capable(void) · 112 unmodified lines
113}113}
114#define __arch_vdso_hres_capable arm_vdso_hres_capable114#define __arch_vdso_hres_capable arm_vdso_hres_capable
115 115
116static __always_inline u64 __arch_get_hw_counter(int clock_mode)116static __always_inline u64 __arch_get_hw_counter(int clock_mode,
117 const struct vdso_data *vd)
117{118{
118#ifdef CONFIG_ARM_ARCH_TIMER119#ifdef CONFIG_ARM_ARCH_TIMER
119 u64 cycle_now;120 u64 cycle_now;
arch/arm64/include/asm/vdso/compat_gettimeofday.h+2−1
@@ -103,7 +103,8 @@ int clock_getres32_fallback(clockid_t _clkid, struct old_timespec32 *_ts) · 102 unmodified lines
103 return ret;103 return ret;
104}104}
105 105
106static __always_inline u64 __arch_get_hw_counter(s32 clock_mode)106static __always_inline u64 __arch_get_hw_counter(s32 clock_mode,
107 const struct vdso_data *vd)
107{108{
108 u64 res;109 u64 res;
109 110
arch/arm64/include/asm/vdso/gettimeofday.h+2−1
@@ -64,7 +64,8 @@ int clock_getres_fallback(clockid_t _clkid, struct __kernel_timespec *_ts) · 63 unmodified lines
64 return ret;64 return ret;
65}65}
66 66
67static __always_inline u64 __arch_get_hw_counter(s32 clock_mode)67static __always_inline u64 __arch_get_hw_counter(s32 clock_mode,
68 const struct vdso_data *vd)
68{69{
69 u64 res;70 u64 res;
70 71
arch/mips/include/asm/vdso/gettimeofday.h+3−2
@@ -167,7 +167,8 @@ static __always_inline u64 read_gic_count(const struct vdso_data *data) · 166 unmodified lines
167 167
168#endif168#endif
169 169
170static __always_inline u64 __arch_get_hw_counter(s32 clock_mode)170static __always_inline u64 __arch_get_hw_counter(s32 clock_mode,
171 const struct vdso_data *vd)
171{172{
172#ifdef CONFIG_CSRC_R4K173#ifdef CONFIG_CSRC_R4K
173 if (clock_mode == VDSO_CLOCKMODE_R4K)174 if (clock_mode == VDSO_CLOCKMODE_R4K)
@@ -175,7 +176,7 @@ static __always_inline u64 __arch_get_hw_counter(s32 clock_mode) · 1 unmodified line
175#endif176#endif
176#ifdef CONFIG_CLKSRC_MIPS_GIC177#ifdef CONFIG_CLKSRC_MIPS_GIC
177 if (clock_mode == VDSO_CLOCKMODE_GIC)178 if (clock_mode == VDSO_CLOCKMODE_GIC)
178 return read_gic_count(get_vdso_data());179 return read_gic_count(vd);
179#endif180#endif
180 /*181 /*
181 * Core checks mode already. So this raced against a concurrent182 * Core checks mode already. So this raced against a concurrent
arch/riscv/include/asm/vdso/gettimeofday.h+2−1
@@ -60,7 +60,8 @@ int clock_getres_fallback(clockid_t _clkid, struct __kernel_timespec *_ts) · 59 unmodified lines
60 return ret;60 return ret;
61}61}
62 62
63static __always_inline u64 __arch_get_hw_counter(s32 clock_mode)63static __always_inline u64 __arch_get_hw_counter(s32 clock_mode,
64 const struct vdso_data *vd)
64{65{
65 /*66 /*
66 * The purpose of csr_read(CSR_TIME) is to trap the system into67 * The purpose of csr_read(CSR_TIME) is to trap the system into
arch/x86/include/asm/vdso/gettimeofday.h+2−1
@@ -241,7 +241,8 @@ static u64 vread_hvclock(void) · 240 unmodified lines
241}241}
242#endif242#endif
243 243
244static inline u64 __arch_get_hw_counter(s32 clock_mode)244static inline u64 __arch_get_hw_counter(s32 clock_mode,
245 const struct vdso_data *vd)
245{246{
246 if (likely(clock_mode == VDSO_CLOCKMODE_TSC))247 if (likely(clock_mode == VDSO_CLOCKMODE_TSC))
247 return (u64)rdtsc_ordered();248 return (u64)rdtsc_ordered();
include/vdso/datapage.h+10−0
@@ -19,6 +19,12 @@ · 18 unmodified lines
19#include <vdso/time32.h>19#include <vdso/time32.h>
20#include <vdso/time64.h>20#include <vdso/time64.h>
21 21
22#ifdef CONFIG_ARCH_HAS_VDSO_DATA
23#include <asm/vdso/data.h>
24#else
25struct arch_vdso_data {};
26#endif
27
22#define VDSO_BASES (CLOCK_TAI + 1)28#define VDSO_BASES (CLOCK_TAI + 1)
23#define VDSO_HRES (BIT(CLOCK_REALTIME) | \29#define VDSO_HRES (BIT(CLOCK_REALTIME) | \
24 BIT(CLOCK_MONOTONIC) | \30 BIT(CLOCK_MONOTONIC) | \
@@ -64,6 +70,8 @@ struct vdso_timestamp { · 39 unmodified lines
64 * @tz_dsttime: type of DST correction70 * @tz_dsttime: type of DST correction
65 * @hrtimer_res: hrtimer resolution71 * @hrtimer_res: hrtimer resolution
66 * @__unused: unused72 * @__unused: unused
73 * @arch_data: architecture specific data (optional, defaults
74 * to an empty struct)
67 *75 *
68 * vdso_data will be accessed by 64 bit and compat code at the same time76 * vdso_data will be accessed by 64 bit and compat code at the same time
69 * so we should be careful before modifying this structure.77 * so we should be careful before modifying this structure.
@@ -97,6 +105,8 @@ struct vdso_data { · 27 unmodified lines
97 s32 tz_dsttime;105 s32 tz_dsttime;
98 u32 hrtimer_res;106 u32 hrtimer_res;
99 u32 __unused;107 u32 __unused;
108
109 struct arch_vdso_data arch_data;
100};110};
101 111
102/*112/*
include/vdso/vsyscall.h+3−0
@@ -6,6 +6,9 @@ · 5 unmodified lines
6 6
7#include <asm/vdso/vsyscall.h>7#include <asm/vdso/vsyscall.h>
8 8
9unsigned long vdso_update_begin(void);
10void vdso_update_end(unsigned long flags);
11
9#endif /* !__ASSEMBLY__ */12#endif /* !__ASSEMBLY__ */
10 13
11#endif /* __VDSO_VSYSCALL_H */14#endif /* __VDSO_VSYSCALL_H */
kernel/time/alarmtimer.c+1−1
@@ -192,7 +192,7 @@ static void alarmtimer_dequeue(struct alarm_base *base, struct alarm *alarm) · 191 unmodified lines
192 * When a alarm timer fires, this runs through the timerqueue to192 * When a alarm timer fires, this runs through the timerqueue to
193 * see which alarms expired, and runs those. If there are more alarm193 * see which alarms expired, and runs those. If there are more alarm
194 * timers queued for the future, we set the hrtimer to fire when194 * timers queued for the future, we set the hrtimer to fire when
195 * when the next future alarm timer expires.195 * the next future alarm timer expires.
196 */196 */
197static enum hrtimer_restart alarmtimer_fired(struct hrtimer *timer)197static enum hrtimer_restart alarmtimer_fired(struct hrtimer *timer)
198{198{
kernel/time/sched_clock.c+1−1
@@ -229,7 +229,7 @@ void __init generic_sched_clock_init(void) · 228 unmodified lines
229{229{
230 /*230 /*
231 * If no sched_clock() function has been provided at that point,231 * If no sched_clock() function has been provided at that point,
232 * make it the final one one.232 * make it the final one.
233 */233 */
234 if (cd.actual_read_sched_clock == jiffy_sched_clock_read)234 if (cd.actual_read_sched_clock == jiffy_sched_clock_read)
235 sched_clock_register(jiffy_sched_clock_read, BITS_PER_LONG, HZ);235 sched_clock_register(jiffy_sched_clock_read, BITS_PER_LONG, HZ);
kernel/time/timekeeping.c+2−2
@@ -39,7 +39,7 @@ enum timekeeping_adv_mode { · 38 unmodified lines
39 TK_ADV_FREQ39 TK_ADV_FREQ
40};40};
41 41
42static DEFINE_RAW_SPINLOCK(timekeeper_lock);42DEFINE_RAW_SPINLOCK(timekeeper_lock);
43 43
44/*44/*
45 * The most important data for readout fits into a single 64 byte45 * The most important data for readout fits into a single 64 byte
@@ -2004,7 +2004,7 @@ static inline unsigned int accumulate_nsecs_to_secs(struct timekeeper *tk) · 1958 unmodified lines
2004 * logarithmic_accumulation - shifted accumulation of cycles2004 * logarithmic_accumulation - shifted accumulation of cycles
2005 *2005 *
2006 * This functions accumulates a shifted interval of cycles into2006 * This functions accumulates a shifted interval of cycles into
2007 * into a shifted interval nanoseconds. Allows for O(log) accumulation2007 * a shifted interval nanoseconds. Allows for O(log) accumulation
2008 * loop.2008 * loop.
2009 *2009 *
2010 * Returns the unconsumed cycles.2010 * Returns the unconsumed cycles.
kernel/time/timekeeping_internal.h+8−3
@@ -1,12 +1,14 @@
1/* SPDX-License-Identifier: GPL-2.0 */1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _TIMEKEEPING_INTERNAL_H2#ifndef _TIMEKEEPING_INTERNAL_H
3#define _TIMEKEEPING_INTERNAL_H3#define _TIMEKEEPING_INTERNAL_H
4/*4
5 * timekeeping debug functions
6 */
7#include <linux/clocksource.h>5#include <linux/clocksource.h>
6#include <linux/spinlock.h>
8#include <linux/time.h>7#include <linux/time.h>
9 8
9/*
10 * timekeeping debug functions
11 */
10#ifdef CONFIG_DEBUG_FS12#ifdef CONFIG_DEBUG_FS
11extern void tk_debug_account_sleep_time(const struct timespec64 *t);13extern void tk_debug_account_sleep_time(const struct timespec64 *t);
12#else14#else
@@ -31,4 +33,7 @@ static inline u64 clocksource_delta(u64 now, u64 last, u64 mask) · 18 unmodified lines
31}33}
32#endif34#endif
33 35
36/* Semi public for serialization of non timekeeper VDSO updates. */
37extern raw_spinlock_t timekeeper_lock;
38
34#endif /* _TIMEKEEPING_INTERNAL_H */39#endif /* _TIMEKEEPING_INTERNAL_H */
kernel/time/vsyscall.c+41−0
@@ -13,6 +13,8 @@ · 12 unmodified lines
13#include <vdso/helpers.h>13#include <vdso/helpers.h>
14#include <vdso/vsyscall.h>14#include <vdso/vsyscall.h>
15 15
16#include "timekeeping_internal.h"
17
16static inline void update_vdso_data(struct vdso_data *vdata,18static inline void update_vdso_data(struct vdso_data *vdata,
17 struct timekeeper *tk)19 struct timekeeper *tk)
18{20{
@@ -127,3 +129,42 @@ void update_vsyscall_tz(void) · 108 unmodified lines
127 129
128 __arch_sync_vdso_data(vdata);130 __arch_sync_vdso_data(vdata);
129}131}
132
133/**
134 * vdso_update_begin - Start of a VDSO update section
135 *
136 * Allows architecture code to safely update the architecture specific VDSO
137 * data. Disables interrupts, acquires timekeeper lock to serialize against
138 * concurrent updates from timekeeping and invalidates the VDSO data
139 * sequence counter to prevent concurrent readers from accessing
140 * inconsistent data.
141 *
142 * Returns: Saved interrupt flags which need to be handed in to
143 * vdso_update_end().
144 */
145unsigned long vdso_update_begin(void)
146{
147 struct vdso_data *vdata = __arch_get_k_vdso_data();
148 unsigned long flags;
149
150 raw_spin_lock_irqsave(&timekeeper_lock, flags);
151 vdso_write_begin(vdata);
152 return flags;
153}
154
155/**
156 * vdso_update_end - End of a VDSO update section
157 * @flags: Interrupt flags as returned from vdso_update_begin()
158 *
159 * Pairs with vdso_update_begin(). Marks vdso data consistent, invokes data
160 * synchronization if the architecture requires it, drops timekeeper lock
161 * and restores interrupt flags.
162 */
163void vdso_update_end(unsigned long flags)
164{
165 struct vdso_data *vdata = __arch_get_k_vdso_data();
166
167 vdso_write_end(vdata);
168 __arch_sync_vdso_data(vdata);
169 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
170}
lib/vdso/gettimeofday.c+2−2
@@ -68,7 +68,7 @@ static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, · 67 unmodified lines
68 if (unlikely(!vdso_clocksource_ok(vd)))68 if (unlikely(!vdso_clocksource_ok(vd)))
69 return -1;69 return -1;
70 70
71 cycles = __arch_get_hw_counter(vd->clock_mode);71 cycles = __arch_get_hw_counter(vd->clock_mode, vd);
72 if (unlikely(!vdso_cycles_ok(cycles)))72 if (unlikely(!vdso_cycles_ok(cycles)))
73 return -1;73 return -1;
74 ns = vdso_ts->nsec;74 ns = vdso_ts->nsec;
@@ -138,7 +138,7 @@ static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, · 63 unmodified lines
138 if (unlikely(!vdso_clocksource_ok(vd)))138 if (unlikely(!vdso_clocksource_ok(vd)))
139 return -1;139 return -1;
140 140
141 cycles = __arch_get_hw_counter(vd->clock_mode);141 cycles = __arch_get_hw_counter(vd->clock_mode, vd);
142 if (unlikely(!vdso_cycles_ok(cycles)))142 if (unlikely(!vdso_cycles_ok(cycles)))
143 return -1;143 return -1;
144 ns = vdso_ts->nsec;144 ns = vdso_ts->nsec;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t