timekeeping: Account for clocksource tick quantisation via NTP

cycle_interval is an integer number of counter cycles per NTP interval, so the real time it represents differs from the nominal NTP_INTERVAL_LENGTH by up to half a counter period. For coarse clocksources this is significant: the 3.579545 MHz ACPI PM timer at HZ=1000 rounds 3579.545 cycles up to 3580, making each tick 1.000127 ms (+127 PPM). Commit a386b5af8edd ("time: Compensate for rounding on odd-frequency clocksources") introduced xtime_remainder to compensate for exactly this, citing the same 127 PPM ACPI PM example. The compensation is correct and necessary, but it was applied inside the timekeeping accumulation in timekeeping.c: subtracted in the mult computation in timekeeping_adjust() and folded into the ntp_error update in logarithmic_accumulation(). That keeps the base rate correct and leaves NTP its full symmetric +/-MAXFREQ range rather than +373/-627 PPM, but the NTP code in ntp.c never sees it: tick_length is computed without the correction, so ntp.c's notion of how long a tick is disagrees with the rate timekeeping actually produces. Make the offset an explicit part of the NTP tick_length instead. Add ntp_data::cs_tick_adj, a fixed per-second addend that ntp_update_frequency() includes alongside ntp_tick_adj and time_freq. tk_setup_internals() computes it from the difference between the real cycle_interval duration and the nominal interval, stores it in the timekeeper, and hands it to NTP through a new argument to ntp_clear() -- which already recomputes the frequency and is invoked after every clocksource (re)configuration. timekeeping_init() now uses TK_UPDATE_ALL for this; clearing NTP there is otherwise redundant since ntp_init() has just initialised it. ntp.c now computes the true tick rate, giving a single source of truth. Like ntp_tick_adj, cs_tick_adj stays internal to the kernel: userspace still sees the nominal 1.000000 ms tick via adjtimex and is unaware of the addends. timekeeping_adjust() and logarithmic_accumulation() use ntp_tick / xtime_interval directly, and xtime_remainder is removed. The base-rate arithmetic is unchanged: ntp_tick becomes xtime_interval << ntp_error_shift, so the mult division yields the same base mult and the ntp_error accumulation still nets to zero per tick. Beyond the cleanup of treating all the tick_length contributions (nominal interval, ntp_tick_adj, cs_tick_adj, time_freq) consistently as addends in one place, it also prepares for feed-forward discipline: a future timekeeping_set_reference() will set tick_length to track an absolute external reference such as a vmclock, and that path needs ntp.c to own a tick_length that already reflects the clocksource quantisation, with no hidden correction applied elsewhere.

David Woodhouse committed 4 months ago

Commit 869a55e1 parent b7befd64 files · +78 −25Browse files

Unified · Split

4 files

include/linux/timekeeper_internal.h +5 −3
kernel/time/ntp.c +24 −3
kernel/time/ntp_internal.h +1 −1
kernel/time/timekeeping.c +48 −18

include/linux/timekeeper_internal.h+5−3

  * @cycle_interval:		Number of clock cycles in one NTP interval
  * @xtime_interval:		Number of clock shifted nano seconds in one NTP
  *				interval.
- * @xtime_remainder:		Shifted nano seconds left over when rounding
- *				@cycle_interval
  * @raw_interval:		Shifted raw nano seconds accumulated per NTP interval.
  * @next_leap_ktime:		CLOCK_MONOTONIC time value of a pending leap-second
  * @ntp_tick:			The ntp_tick_length() value currently being
  * @ntp_error_shift:		Shift conversion between clock shifted nano seconds and
  *				ntp shifted nano seconds.
  * @ntp_err_mult:		Multiplication factor for scaled math conversion
+ * @cs_tick_adj:		Per-second adjustment handed to NTP via ntp_clear()
+ *				accounting for the difference between the nominal
+ *				NTP interval and the real time taken by the
+ *				clocksource's integer @cycle_interval (upscaled).
  * @skip_second_overflow:	Flag used to avoid updating NTP twice with same second
  * @tai_offset:			The current UTC to TAI offset in seconds
  *
 	u64			cycle_interval;
 	u64			xtime_interval;
-	s64			xtime_remainder;
 	u64			raw_interval;
 	ktime_t			next_leap_ktime;
 	s64			ntp_error;
 	u32			ntp_error_shift;
 	u32			ntp_err_mult;
+	s64			cs_tick_adj;
 	u32			skip_second_overflow;
 	s32			tai_offset;
 };

kernel/time/ntp.c+24−3

  * @time_reftime:	Time at last adjustment in seconds
  * @time_adjust:	Adjustment value
  * @ntp_tick_adj:	Constant boot-param configurable NTP tick adjustment (upscaled)
+ * @cs_tick_adj:	Fixed per-second adjustment compensating for the difference
+ *			between the nominal NTP interval and the real time taken
+ *			by the clocksource's integer @cycle_interval (upscaled).
+ *			Set by the timekeeping core via ntp_clear().
  * @ntp_next_leap_sec:	Second value of the next pending leapsecond, or TIME64_MAX if no leap
  *
  * @pps_valid:		PPS signal watchdog counter
 	time64_t		time_reftime;
 	long			time_adjust;
 	s64			ntp_tick_adj;
+	s64			cs_tick_adj;
 	time64_t		ntp_next_leap_sec;
 #ifdef CONFIG_NTP_PPS
 	int			pps_valid;
 	second_length		 = (u64)(tick_usec * NSEC_PER_USEC * USER_HZ) << NTP_SCALE_SHIFT;
 	second_length		+= ntpdata->ntp_tick_adj;
+	second_length		+= ntpdata->cs_tick_adj;
 	second_length		+= ntpdata->time_freq;
 	new_base		 = div_u64(second_length, NTP_INTERVAL_FREQ);
 }
 /**
- * ntp_clear - Clears the NTP state variables
+ * ntp_clear - Clear NTP state and set the clocksource quantisation adjustment
- * @tkid:	Timekeeper ID to be able to select proper ntp data array member
+ * @tkid:		Timekeeper ID
+ * @cs_tick_adj:	Per-second adjustment in ns << NTP_SCALE_SHIFT
+ *
+ * The timekeeping core uses an integer number of cycles (@cycle_interval)
+ * per NTP interval, so the real time that interval represents differs from
+ * the nominal NTP_INTERVAL_LENGTH by up to half a counter period. Folding
+ * this fixed offset into @cs_tick_adj makes it an explicit part of the NTP
+ * tick_length computation in ntp.c, instead of being applied during
+ * timekeeping accumulation where the NTP code never saw it. Like
+ * @ntp_tick_adj it stays internal to the kernel; userspace still sees the
+ * nominal tick via adjtimex. NTP retains its full symmetric ±MAXFREQ range
+ * around the corrected base rate.
+ *
+ * Called whenever the clocksource is (re)configured, which is also when the
+ * rest of the NTP state must be cleared, so the two are done together.
  */
-void ntp_clear(unsigned int tkid)
+void ntp_clear(unsigned int tkid, s64 cs_tick_adj)
 {
+	tk_ntp_data[tkid].cs_tick_adj = cs_tick_adj;
 	__ntp_clear(&tk_ntp_data[tkid]);
 }

kernel/time/ntp_internal.h+1−1

@@ -3,7 +3,7 @@ · 2 unmodified lines
3	#define _LINUX_NTP_INTERNAL_H	3	#define _LINUX_NTP_INTERNAL_H
4		4
5	extern void ntp_init(void);	5	extern void ntp_init(void);
6	extern void ntp_clear(unsigned int tkid);	6	extern void ntp_clear(unsigned int tkid, s64 cs_tick_adj);
7	/* Returns how long ticks are at present, in ns / 2^NTP_SCALE_SHIFT. */	7	/* Returns how long ticks are at present, in ns / 2^NTP_SCALE_SHIFT. */
8	extern u64 ntp_tick_length(unsigned int tkid);	8	extern u64 ntp_tick_length(unsigned int tkid);
9	extern ktime_t ntp_get_next_leap(unsigned int tkid);	9	extern ktime_t ntp_get_next_leap(unsigned int tkid);

kernel/time/timekeeping.c+48−18

@@ -339,7 +339,6 @@ static inline void clocksource_enable_inline_read(void) { } · 338 unmodified lines
339	static void tk_setup_internals(struct timekeeper tk, struct clocksource clock)	339	static void tk_setup_internals(struct timekeeper tk, struct clocksource clock)
340	{	340	{
341	u64 interval;	341	u64 interval;
342	u64 tmp, ntpinterval;
343	struct clocksource *old_clock;	342	struct clocksource *old_clock;
344		343
345	++tk->cs_was_changed_seq;	344	++tk->cs_was_changed_seq;
@@ -353,20 +352,16 @@ static void tk_setup_internals(struct timekeeper tk, struct clocksource clock) · 7 unmodified lines
353	tk->tkr_raw.cycle_last = tk->tkr_mono.cycle_last;	352	tk->tkr_raw.cycle_last = tk->tkr_mono.cycle_last;
354		353
355	/* Do the ns -> cycle conversion first, using original mult */	354	/* Do the ns -> cycle conversion first, using original mult */
356	tmp = NTP_INTERVAL_LENGTH;	355	interval = (u64)NTP_INTERVAL_LENGTH << clock->shift;
357	tmp <<= clock->shift;	356	interval += clock->mult / 2;
358	ntpinterval = tmp;	357	do_div(interval, clock->mult);
359	tmp += clock->mult/2;	358	if (interval == 0)
360	do_div(tmp, clock->mult);	359	interval = 1;
361	if (tmp == 0)	360
362	tmp = 1;
363
364	interval = (u64) tmp;
365	tk->cycle_interval = interval;	361	tk->cycle_interval = interval;
366		362
367	/* Go back from cycles -> shifted ns */	363	/* Go back from cycles -> shifted ns */
368	tk->xtime_interval = interval * clock->mult;	364	tk->xtime_interval = interval * clock->mult;
369	tk->xtime_remainder = ntpinterval - tk->xtime_interval;
370	tk->raw_interval = interval * clock->mult;	365	tk->raw_interval = interval * clock->mult;
371		366
372	/* if changing clocks, convert xtime_nsec shift units */	367	/* if changing clocks, convert xtime_nsec shift units */
@@ -386,7 +381,38 @@ static void tk_setup_internals(struct timekeeper tk, struct clocksource clock) · 13 unmodified lines
386		381
387	tk->ntp_error = 0;	382	tk->ntp_error = 0;
388	tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;	383	tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;
389	tk->ntp_tick = ntpinterval << tk->ntp_error_shift;	384
		385	/*
		386	* ntp_tick is the tick length that NTP disciplines (its ±500 PPM
		387	* scales only this part), in NTP-shifted ns: the real interval of
		388	* a whole number of counter cycles. Because cycle_interval is
		389	* rounded to an integer number of cycles, this ntp_tick differs
		390	* from the true intended 1/HZ tick length by up to half a cycle
		391	* period.
		392	*/
		393	tk->ntp_tick = (u64)tk->xtime_interval << tk->ntp_error_shift;
		394
		395	/*
		396	* cs_tick_adj is the constant difference between the disciplined
		397	* ntp_tick above and the true 1/HZ tick, expressed per-second to
		398	* match the ntp_update_frequency() addends and handed to NTP via
		399	* ntp_clear() to be explicitly included in its tick_length.
		400	*
		401	* Worked example: HZ=1000, ACPI PM timer at 3.579545 MHz, which
		402	* has 3579.545 cycles in 1ms, rounded to cycle_interval = 3580.
		403	*
		404	* So ntp_tick is actually 1.000127ms, as that is the amount of
		405	* time that 3580 cycles will take at the nominal frequency. This
		406	* is the part that NTP disciplines, causing each 3580 counts to
		407	* advance the clock by up to NTP's ±500PPM of that amount.
		408	*
		409	* The "extra" 127ns/tick is what's stored in cs_tick_adj and
		410	* applied as a constant correction by ntp_update_frequency() so
		411	* that NTP believes it's disciplining a 1ms tick.
		412	*/
		413	tk->cs_tick_adj = (s64)tk->ntp_tick -
		414	((s64)NTP_INTERVAL_LENGTH << NTP_SCALE_SHIFT);
		415	tk->cs_tick_adj *= NTP_INTERVAL_FREQ;
390		416
391	/*	417	/*
392	* The timekeeper keeps its own mult values for the currently	418	* The timekeeper keeps its own mult values for the currently
@@ -803,7 +829,7 @@ static void timekeeping_update_from_shadow(struct tk_data *tkd, unsigned int act · 410 unmodified lines
803		829
804	if (action & TK_CLEAR_NTP) {	830	if (action & TK_CLEAR_NTP) {
805	tk->ntp_error = 0;	831	tk->ntp_error = 0;
806	ntp_clear(tk->id);	832	ntp_clear(tk->id, tk->cs_tick_adj);
807	}	833	}
808		834
809	tk_update_leap_state(tk);	835	tk_update_leap_state(tk);
@@ -2090,7 +2116,12 @@ void __init timekeeping_init(void) · 1280 unmodified lines
2090		2116
2091	tk_set_wall_to_mono(tks, wall_to_mono);	2117	tk_set_wall_to_mono(tks, wall_to_mono);
2092		2118
2093	timekeeping_update_from_shadow(&tk_core, TK_CLOCK_WAS_SET);	2119	/*
		2120	* Use TK_UPDATE_ALL so the NTP layer picks up the clocksource's
		2121	* cs_tick_adj via ntp_clear(). Clearing NTP here is otherwise
		2122	* redundant as ntp_init() already initialised it above.
		2123	*/
		2124	timekeeping_update_from_shadow(&tk_core, TK_UPDATE_ALL);
2094	}	2125	}
2095		2126
2096	/* time in seconds when suspend began for persistent clock */	2127	/* time in seconds when suspend began for persistent clock */
@@ -2439,8 +2470,8 @@ static void timekeeping_adjust(struct timekeeper *tk, s64 offset) · 342 unmodified lines
2439	mult = tk->tkr_mono.mult - tk->ntp_err_mult;	2470	mult = tk->tkr_mono.mult - tk->ntp_err_mult;
2440	} else {	2471	} else {
2441	tk->ntp_tick = ntp_tl;	2472	tk->ntp_tick = ntp_tl;
2442	mult = div64_u64((tk->ntp_tick >> tk->ntp_error_shift) -	2473	mult = div64_u64(tk->ntp_tick >> tk->ntp_error_shift,
2443	tk->xtime_remainder, tk->cycle_interval);	2474	tk->cycle_interval);
2444	}	2475	}
2445		2476
2446	/*	2477	/*
@@ -2565,8 +2596,7 @@ static u64 logarithmic_accumulation(struct timekeeper *tk, u64 offset, · 118 unmodified lines
2565		2596
2566	/* Accumulate error between NTP and clock interval */	2597	/* Accumulate error between NTP and clock interval */
2567	tk->ntp_error += tk->ntp_tick << shift;	2598	tk->ntp_error += tk->ntp_tick << shift;
2568	tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) <<	2599	tk->ntp_error -= tk->xtime_interval << (tk->ntp_error_shift + shift);
2569	(tk->ntp_error_shift + shift);
2570		2600
2571	return offset;	2601	return offset;
2572	}	2602	}