Expose the auxiliary clocks through the vDSO. Architectures not using the generic vDSO time framework, namely SPARC64, are not supported.
Unified · Split
| @@ -5,6 +5,7 @@ · 4 unmodified lines | |||
| 5 | #ifndef __ASSEMBLY__ | 5 | #ifndef __ASSEMBLY__ |
| 6 | 6 | ||
| 7 | #include <linux/compiler.h> | 7 | #include <linux/compiler.h> |
| 8 | #include <uapi/linux/bits.h> | ||
| 8 | #include <uapi/linux/time.h> | 9 | #include <uapi/linux/time.h> |
| 9 | #include <uapi/linux/types.h> | 10 | #include <uapi/linux/types.h> |
| 10 | #include <uapi/asm-generic/errno-base.h> | 11 | #include <uapi/asm-generic/errno-base.h> |
| @@ -46,6 +47,7 @@ struct vdso_arch_data { · 35 unmodified lines | |||
| 46 | #define VDSO_COARSE (BIT(CLOCK_REALTIME_COARSE) | \ | 47 | #define VDSO_COARSE (BIT(CLOCK_REALTIME_COARSE) | \ |
| 47 | BIT(CLOCK_MONOTONIC_COARSE)) | 48 | BIT(CLOCK_MONOTONIC_COARSE)) |
| 48 | #define VDSO_RAW (BIT(CLOCK_MONOTONIC_RAW)) | 49 | #define VDSO_RAW (BIT(CLOCK_MONOTONIC_RAW)) |
| 50 | #define VDSO_AUX __GENMASK(CLOCK_AUX_LAST, CLOCK_AUX) | ||
| 49 | 51 | ||
| 50 | #define CS_HRES_COARSE 0 | 52 | #define CS_HRES_COARSE 0 |
| 51 | #define CS_RAW 1 | 53 | #define CS_RAW 1 |
| @@ -2,6 +2,7 @@ · 1 unmodified line | |||
| 2 | /* | 2 | /* |
| 3 | * Generic userspace implementations of gettimeofday() and similar. | 3 | * Generic userspace implementations of gettimeofday() and similar. |
| 4 | */ | 4 | */ |
| 5 | #include <vdso/auxclock.h> | ||
| 5 | #include <vdso/datapage.h> | 6 | #include <vdso/datapage.h> |
| 6 | #include <vdso/helpers.h> | 7 | #include <vdso/helpers.h> |
| 7 | 8 | ||
| @@ -74,7 +75,7 @@ static inline bool vdso_cycles_ok(u64 cycles) · 66 unmodified lines | |||
| 74 | static __always_inline bool vdso_clockid_valid(clockid_t clock) | 75 | static __always_inline bool vdso_clockid_valid(clockid_t clock) |
| 75 | { | 76 | { |
| 76 | /* Check for negative values or invalid clocks */ | 77 | /* Check for negative values or invalid clocks */ |
| 77 | return likely((u32) clock < MAX_CLOCKS); | 78 | return likely((u32) clock <= CLOCK_AUX_LAST); |
| 78 | } | 79 | } |
| 79 | 80 | ||
| 80 | /* | 81 | /* |
| @@ -268,6 +269,48 @@ bool do_coarse(const struct vdso_time_data *vd, const struct vdso_clock *vc, · 187 unmodified lines | |||
| 268 | return true; | 269 | return true; |
| 269 | } | 270 | } |
| 270 | 271 | ||
| 272 | static __always_inline | ||
| 273 | bool do_aux(const struct vdso_time_data *vd, clockid_t clock, struct __kernel_timespec *ts) | ||
| 274 | { | ||
| 275 | const struct vdso_clock *vc; | ||
| 276 | u32 seq, idx; | ||
| 277 | u64 sec, ns; | ||
| 278 | |||
| 279 | if (!IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS)) | ||
| 280 | return false; | ||
| 281 | |||
| 282 | idx = clock - CLOCK_AUX; | ||
| 283 | vc = &vd->aux_clock_data[idx]; | ||
| 284 | |||
| 285 | do { | ||
| 286 | /* | ||
| 287 | * Open coded function vdso_read_begin() to handle | ||
| 288 | * VDSO_CLOCK_TIMENS. See comment in do_hres(). | ||
| 289 | */ | ||
| 290 | while ((seq = READ_ONCE(vc->seq)) & 1) { | ||
| 291 | if (IS_ENABLED(CONFIG_TIME_NS) && vc->clock_mode == VDSO_CLOCKMODE_TIMENS) { | ||
| 292 | vd = __arch_get_vdso_u_timens_data(vd); | ||
| 293 | vc = &vd->aux_clock_data[idx]; | ||
| 294 | /* Re-read from the real time data page */ | ||
| 295 | continue; | ||
| 296 | } | ||
| 297 | cpu_relax(); | ||
| 298 | } | ||
| 299 | smp_rmb(); | ||
| 300 | |||
| 301 | /* Auxclock disabled? */ | ||
| 302 | if (vc->clock_mode == VDSO_CLOCKMODE_NONE) | ||
| 303 | return false; | ||
| 304 | |||
| 305 | if (!vdso_get_timestamp(vd, vc, VDSO_BASE_AUX, &sec, &ns)) | ||
| 306 | return false; | ||
| 307 | } while (unlikely(vdso_read_retry(vc, seq))); | ||
| 308 | |||
| 309 | vdso_set_timespec(ts, sec, ns); | ||
| 310 | |||
| 311 | return true; | ||
| 312 | } | ||
| 313 | |||
| 271 | static __always_inline bool | 314 | static __always_inline bool |
| 272 | __cvdso_clock_gettime_common(const struct vdso_time_data *vd, clockid_t clock, | 315 | __cvdso_clock_gettime_common(const struct vdso_time_data *vd, clockid_t clock, |
| 273 | struct __kernel_timespec *ts) | 316 | struct __kernel_timespec *ts) |
| @@ -289,6 +332,8 @@ __cvdso_clock_gettime_common(const struct vdso_time_data *vd, clockid_t clock, · 15 unmodified lines | |||
| 289 | return do_coarse(vd, &vc[CS_HRES_COARSE], clock, ts); | 332 | return do_coarse(vd, &vc[CS_HRES_COARSE], clock, ts); |
| 290 | else if (msk & VDSO_RAW) | 333 | else if (msk & VDSO_RAW) |
| 291 | vc = &vc[CS_RAW]; | 334 | vc = &vc[CS_RAW]; |
| 335 | else if (msk & VDSO_AUX) | ||
| 336 | return do_aux(vd, clock, ts); | ||
| 292 | else | 337 | else |
| 293 | return false; | 338 | return false; |
| 294 | 339 | ||
| @@ -433,6 +478,8 @@ bool __cvdso_clock_getres_common(const struct vdso_time_data *vd, clockid_t cloc · 138 unmodified lines | |||
| 433 | * Preserves the behaviour of posix_get_coarse_res(). | 478 | * Preserves the behaviour of posix_get_coarse_res(). |
| 434 | */ | 479 | */ |
| 435 | ns = LOW_RES_NSEC; | 480 | ns = LOW_RES_NSEC; |
| 481 | } else if (msk & VDSO_AUX) { | ||
| 482 | ns = aux_clock_resolution_ns(); | ||
| 436 | } else { | 483 | } else { |
| 437 | return false; | 484 | return false; |
| 438 | } | 485 | } |