History

timekeeping: Account for clocksource tick quantisation via NTP

cycle_interval is an integer number of counter cycles per NTP interval, so the real time it represents differs from the nominal NTP_INTERVAL_LENGTH by up to half a counter period. For coarse clocksources this is significant: the 3.579545 MHz ACPI PM timer at HZ=1000 rounds 3579.545 cycles up to 3580, making each tick 1.000127 ms (+127 PPM). Commit a386b5af8edd ("time: Compensate for rounding on odd-frequency clocksources") introduced xtime_remainder to compensate for exactly this, citing the same 127 PPM ACPI PM example. The compensation is correct and necessary, but it was applied inside the timekeeping accumulation in timekeeping.c: subtracted in the mult computation in timekeeping_adjust() and folded into the ntp_error update in logarithmic_accumulation(). That keeps the base rate correct and leaves NTP its full symmetric +/-MAXFREQ range rather than +373/-627 PPM, but the NTP code in ntp.c never sees it: tick_length is computed without the correction, so ntp.c's notion of how long a tick is disagrees with the rate timekeeping actually produces. Make the offset an explicit part of the NTP tick_length instead. Add ntp_data::cs_tick_adj, a fixed per-second addend that ntp_update_frequency() includes alongside ntp_tick_adj and time_freq. tk_setup_internals() computes it from the difference between the real cycle_interval duration and the nominal interval, stores it in the timekeeper, and hands it to NTP through a new argument to ntp_clear() -- which already recomputes the frequency and is invoked after every clocksource (re)configuration. timekeeping_init() now uses TK_UPDATE_ALL for this; clearing NTP there is otherwise redundant since ntp_init() has just initialised it. ntp.c now computes the true tick rate, giving a single source of truth. Like ntp_tick_adj, cs_tick_adj stays internal to the kernel: userspace still sees the nominal 1.000000 ms tick via adjtimex and is unaware of the addends. timekeeping_adjust() and logarithmic_accumulation() use ntp_tick / xtime_interval directly, and xtime_remainder is removed. The base-rate arithmetic is unchanged: ntp_tick becomes xtime_interval << ntp_error_shift, so the mult division yields the same base mult and the ntp_error accumulation still nets to zero per tick. Beyond the cleanup of treating all the tick_length contributions (nominal interval, ntp_tick_adj, cs_tick_adj, time_freq) consistently as addends in one place, it also prepares for feed-forward discipline: a future timekeeping_set_reference() will set tick_length to track an absolute external reference such as a vmclock, and that path needs ntp.c to own a tick_length that already reflects the clocksource quantisation, with no hidden correction applied elsewhere.

David Woodhouse committed 4 months ago

Commit 869a55e1 parent b7befd64 files · +78 −25Browse files

Unified · Split

4 files
  1. include/linux/timekeeper_internal.h +5 −3
  2. kernel/time/ntp.c +24 −3
  3. kernel/time/ntp_internal.h +1 −1
  4. kernel/time/timekeeping.c +48 −18
include/linux/timekeeper_internal.h+5−3
@@ -84,8 +84,6 @@ struct tk_read_base { · 83 unmodified lines
8484 * @cycle_interval: Number of clock cycles in one NTP interval
8585 * @xtime_interval: Number of clock shifted nano seconds in one NTP
8686 * interval.
87− * @xtime_remainder: Shifted nano seconds left over when rounding
88− * @cycle_interval
8987 * @raw_interval: Shifted raw nano seconds accumulated per NTP interval.
9088 * @next_leap_ktime: CLOCK_MONOTONIC time value of a pending leap-second
9189 * @ntp_tick: The ntp_tick_length() value currently being
@@ -99,6 +97,10 @@ struct tk_read_base { · 7 unmodified lines
9997 * @ntp_error_shift: Shift conversion between clock shifted nano seconds and
10098 * ntp shifted nano seconds.
10199 * @ntp_err_mult: Multiplication factor for scaled math conversion
100+ * @cs_tick_adj: Per-second adjustment handed to NTP via ntp_clear()
101+ * accounting for the difference between the nominal
102+ * NTP interval and the real time taken by the
103+ * clocksource's integer @cycle_interval (upscaled).
102104 * @skip_second_overflow: Flag used to avoid updating NTP twice with same second
103105 * @tai_offset: The current UTC to TAI offset in seconds
104106 *
@@ -178,7 +180,6 @@ struct timekeeper { · 73 unmodified lines
178180
179181 u64 cycle_interval;
180182 u64 xtime_interval;
181− s64 xtime_remainder;
182183 u64 raw_interval;
183184
184185 ktime_t next_leap_ktime;
@@ -186,6 +187,7 @@ struct timekeeper { · 1 unmodified line
186187 s64 ntp_error;
187188 u32 ntp_error_shift;
188189 u32 ntp_err_mult;
190+ s64 cs_tick_adj;
189191 u32 skip_second_overflow;
190192 s32 tai_offset;
191193 };
kernel/time/ntp.c+24−3
@@ -39,6 +39,10 @@ · 38 unmodified lines
3939 * @time_reftime: Time at last adjustment in seconds
4040 * @time_adjust: Adjustment value
4141 * @ntp_tick_adj: Constant boot-param configurable NTP tick adjustment (upscaled)
42+ * @cs_tick_adj: Fixed per-second adjustment compensating for the difference
43+ * between the nominal NTP interval and the real time taken
44+ * by the clocksource's integer @cycle_interval (upscaled).
45+ * Set by the timekeeping core via ntp_clear().
4246 * @ntp_next_leap_sec: Second value of the next pending leapsecond, or TIME64_MAX if no leap
4347 *
4448 * @pps_valid: PPS signal watchdog counter
@@ -70,6 +74,7 @@ struct ntp_data { · 25 unmodified lines
7074 time64_t time_reftime;
7175 long time_adjust;
7276 s64 ntp_tick_adj;
77+ s64 cs_tick_adj;
7378 time64_t ntp_next_leap_sec;
7479 #ifdef CONFIG_NTP_PPS
7580 int pps_valid;
@@ -255,6 +260,7 @@ static void ntp_update_frequency(struct ntp_data *ntpdata) · 179 unmodified lines
255260 second_length = (u64)(tick_usec * NSEC_PER_USEC * USER_HZ) << NTP_SCALE_SHIFT;
256261
257262 second_length += ntpdata->ntp_tick_adj;
263+ second_length += ntpdata->cs_tick_adj;
258264 second_length += ntpdata->time_freq;
259265
260266 new_base = div_u64(second_length, NTP_INTERVAL_FREQ);
@@ -350,11 +356,26 @@ static void __ntp_clear(struct ntp_data *ntpdata) · 89 unmodified lines
350356 }
351357
352358 /**
353− * ntp_clear - Clears the NTP state variables
354− * @tkid: Timekeeper ID to be able to select proper ntp data array member
359+ * ntp_clear - Clear NTP state and set the clocksource quantisation adjustment
360+ * @tkid: Timekeeper ID
361+ * @cs_tick_adj: Per-second adjustment in ns << NTP_SCALE_SHIFT
362+ *
363+ * The timekeeping core uses an integer number of cycles (@cycle_interval)
364+ * per NTP interval, so the real time that interval represents differs from
365+ * the nominal NTP_INTERVAL_LENGTH by up to half a counter period. Folding
366+ * this fixed offset into @cs_tick_adj makes it an explicit part of the NTP
367+ * tick_length computation in ntp.c, instead of being applied during
368+ * timekeeping accumulation where the NTP code never saw it. Like
369+ * @ntp_tick_adj it stays internal to the kernel; userspace still sees the
370+ * nominal tick via adjtimex. NTP retains its full symmetric ±MAXFREQ range
371+ * around the corrected base rate.
372+ *
373+ * Called whenever the clocksource is (re)configured, which is also when the
374+ * rest of the NTP state must be cleared, so the two are done together.
355375 */
356−void ntp_clear(unsigned int tkid)
376+void ntp_clear(unsigned int tkid, s64 cs_tick_adj)
357377 {
378+ tk_ntp_data[tkid].cs_tick_adj = cs_tick_adj;
358379 __ntp_clear(&tk_ntp_data[tkid]);
359380 }
360381
kernel/time/ntp_internal.h+1−1
@@ -3,7 +3,7 @@ · 2 unmodified lines
33 #define _LINUX_NTP_INTERNAL_H
44
55 extern void ntp_init(void);
6−extern void ntp_clear(unsigned int tkid);
6+extern void ntp_clear(unsigned int tkid, s64 cs_tick_adj);
77 /* Returns how long ticks are at present, in ns / 2^NTP_SCALE_SHIFT. */
88 extern u64 ntp_tick_length(unsigned int tkid);
99 extern ktime_t ntp_get_next_leap(unsigned int tkid);
kernel/time/timekeeping.c+48−18
@@ -339,7 +339,6 @@ static inline void clocksource_enable_inline_read(void) { } · 338 unmodified lines
339339 static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
340340 {
341341 u64 interval;
342− u64 tmp, ntpinterval;
343342 struct clocksource *old_clock;
344343
345344 ++tk->cs_was_changed_seq;
@@ -353,20 +352,16 @@ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock) · 7 unmodified lines
353352 tk->tkr_raw.cycle_last = tk->tkr_mono.cycle_last;
354353
355354 /* Do the ns -> cycle conversion first, using original mult */
356− tmp = NTP_INTERVAL_LENGTH;
357− tmp <<= clock->shift;
358− ntpinterval = tmp;
359− tmp += clock->mult/2;
360− do_div(tmp, clock->mult);
361− if (tmp == 0)
362− tmp = 1;
363−
364− interval = (u64) tmp;
355+ interval = (u64)NTP_INTERVAL_LENGTH << clock->shift;
356+ interval += clock->mult / 2;
357+ do_div(interval, clock->mult);
358+ if (interval == 0)
359+ interval = 1;
360+
365361 tk->cycle_interval = interval;
366362
367363 /* Go back from cycles -> shifted ns */
368364 tk->xtime_interval = interval * clock->mult;
369− tk->xtime_remainder = ntpinterval - tk->xtime_interval;
370365 tk->raw_interval = interval * clock->mult;
371366
372367 /* if changing clocks, convert xtime_nsec shift units */
@@ -386,7 +381,38 @@ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock) · 13 unmodified lines
386381
387382 tk->ntp_error = 0;
388383 tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;
389− tk->ntp_tick = ntpinterval << tk->ntp_error_shift;
384+
385+ /*
386+ * ntp_tick is the tick length that NTP disciplines (its ±500 PPM
387+ * scales only this part), in NTP-shifted ns: the real interval of
388+ * a whole number of counter cycles. Because cycle_interval is
389+ * rounded to an integer number of cycles, this ntp_tick differs
390+ * from the true intended 1/HZ tick length by up to half a cycle
391+ * period.
392+ */
393+ tk->ntp_tick = (u64)tk->xtime_interval << tk->ntp_error_shift;
394+
395+ /*
396+ * cs_tick_adj is the constant difference between the disciplined
397+ * ntp_tick above and the true 1/HZ tick, expressed per-second to
398+ * match the ntp_update_frequency() addends and handed to NTP via
399+ * ntp_clear() to be explicitly included in its tick_length.
400+ *
401+ * Worked example: HZ=1000, ACPI PM timer at 3.579545 MHz, which
402+ * has 3579.545 cycles in 1ms, rounded to cycle_interval = 3580.
403+ *
404+ * So ntp_tick is actually 1.000127ms, as that is the amount of
405+ * time that 3580 cycles will take at the nominal frequency. This
406+ * is the part that NTP disciplines, causing each 3580 counts to
407+ * advance the clock by up to NTP's ±500PPM of that amount.
408+ *
409+ * The "extra" 127ns/tick is what's stored in cs_tick_adj and
410+ * applied as a constant correction by ntp_update_frequency() so
411+ * that NTP *believes* it's disciplining a 1ms tick.
412+ */
413+ tk->cs_tick_adj = (s64)tk->ntp_tick -
414+ ((s64)NTP_INTERVAL_LENGTH << NTP_SCALE_SHIFT);
415+ tk->cs_tick_adj *= NTP_INTERVAL_FREQ;
390416
391417 /*
392418 * The timekeeper keeps its own mult values for the currently
@@ -803,7 +829,7 @@ static void timekeeping_update_from_shadow(struct tk_data *tkd, unsigned int act · 410 unmodified lines
803829
804830 if (action & TK_CLEAR_NTP) {
805831 tk->ntp_error = 0;
806− ntp_clear(tk->id);
832+ ntp_clear(tk->id, tk->cs_tick_adj);
807833 }
808834
809835 tk_update_leap_state(tk);
@@ -2090,7 +2116,12 @@ void __init timekeeping_init(void) · 1280 unmodified lines
20902116
20912117 tk_set_wall_to_mono(tks, wall_to_mono);
20922118
2093− timekeeping_update_from_shadow(&tk_core, TK_CLOCK_WAS_SET);
2119+ /*
2120+ * Use TK_UPDATE_ALL so the NTP layer picks up the clocksource's
2121+ * cs_tick_adj via ntp_clear(). Clearing NTP here is otherwise
2122+ * redundant as ntp_init() already initialised it above.
2123+ */
2124+ timekeeping_update_from_shadow(&tk_core, TK_UPDATE_ALL);
20942125 }
20952126
20962127 /* time in seconds when suspend began for persistent clock */
@@ -2439,8 +2470,8 @@ static void timekeeping_adjust(struct timekeeper *tk, s64 offset) · 342 unmodified lines
24392470 mult = tk->tkr_mono.mult - tk->ntp_err_mult;
24402471 } else {
24412472 tk->ntp_tick = ntp_tl;
2442− mult = div64_u64((tk->ntp_tick >> tk->ntp_error_shift) -
2443− tk->xtime_remainder, tk->cycle_interval);
2473+ mult = div64_u64(tk->ntp_tick >> tk->ntp_error_shift,
2474+ tk->cycle_interval);
24442475 }
24452476
24462477 /*
@@ -2565,8 +2596,7 @@ static u64 logarithmic_accumulation(struct timekeeper *tk, u64 offset, · 118 unmodified lines
25652596
25662597 /* Accumulate error between NTP and clock interval */
25672598 tk->ntp_error += tk->ntp_tick << shift;
2568− tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) <<
2569− (tk->ntp_error_shift + shift);
2599+ tk->ntp_error -= tk->xtime_interval << (tk->ntp_error_shift + shift);
25702600
25712601 return offset;
25722602 }