History

clocksource: Scale the watchdog read retries automatically

On a 8-socket server the TSC is wrongly marked as 'unstable' and disabled during boot time on about one out of 120 boot attempts: clocksource: timekeeping watchdog on CPU227: wd-tsc-wd excessive read-back delay of 153560ns vs. limit of 125000ns, wd-wd read-back delay only 11440ns, attempt 3, marking tsc unstable tsc: Marking TSC unstable due to clocksource watchdog TSC found unstable after boot, most likely due to broken BIOS. Use 'tsc=unstable'. sched_clock: Marking unstable (119294969739, 159204297)<-(125446229205, -5992055152) clocksource: Checking clocksource tsc synchronization from CPU 319 to CPUs 0,99,136,180,210,542,601,896. clocksource: Switched to clocksource hpet The reason is that for platform with a large number of CPUs, there are sporadic big or huge read latencies while reading the watchog/clocksource during boot or when system is under stress work load, and the frequency and maximum value of the latency goes up with the number of online CPUs. The cCurrent code already has logic to detect and filter such high latency case by reading the watchdog twice and checking the two deltas. Due to the randomness of the latency, there is a low probabilty that the first delta (latency) is big, but the second delta is small and looks valid. The watchdog code retries the readouts by default twice, which is not necessarily sufficient for systems with a large number of CPUs. There is a command line parameter 'max_cswd_read_retries' which allows to increase the number of retries, but that's not user friendly as it needs to be tweaked per system. As the number of required retries is proportional to the number of online CPUs, this parameter can be calculated at runtime. Scale and enlarge the number of retries according to the number of online CPUs and remove the command line parameter completely. [ tglx: Massaged change log and comments ]

Feng Tang committed 2 years ago

Commit 2ed08e41 parent e0a12845 files · +25 −20Browse files

Unified · Split

5 files
  1. Documentation/admin-guide/kernel-parameters.txt +0 −6
  2. include/linux/clocksource.h +13 −1
  3. kernel/time/clocksource-wdtest.c +7 −6
  4. kernel/time/clocksource.c +4 −6
  5. tools/testing/selftests/rcutorture/bin/torture.sh +1 −1
Documentation/admin-guide/kernel-parameters.txt+0−6
@@ -679,12 +679,6 @@ · 678 unmodified lines
679 loops can be debugged more effectively on production679 loops can be debugged more effectively on production
680 systems.680 systems.
681 681
682 clocksource.max_cswd_read_retries= [KNL]
683 Number of clocksource_watchdog() retries due to
684 external delays before the clock will be marked
685 unstable. Defaults to two retries, that is,
686 three attempts to read the clock under test.
687
688 clocksource.verify_n_cpus= [KNL]682 clocksource.verify_n_cpus= [KNL]
689 Limit the number of CPUs checked for clocksources683 Limit the number of CPUs checked for clocksources
690 marked with CLOCK_SOURCE_VERIFY_PERCPU that684 marked with CLOCK_SOURCE_VERIFY_PERCPU that
include/linux/clocksource.h+13−1
@@ -291,7 +291,19 @@ static inline void timer_probe(void) {} · 290 unmodified lines
291#define TIMER_ACPI_DECLARE(name, table_id, fn) \291#define TIMER_ACPI_DECLARE(name, table_id, fn) \
292 ACPI_DECLARE_PROBE_ENTRY(timer, name, table_id, 0, NULL, 0, fn)292 ACPI_DECLARE_PROBE_ENTRY(timer, name, table_id, 0, NULL, 0, fn)
293 293
294extern ulong max_cswd_read_retries;294static inline unsigned int clocksource_get_max_watchdog_retry(void)
295{
296 /*
297 * When system is in the boot phase or under heavy workload, there
298 * can be random big latencies during the clocksource/watchdog
299 * read, so allow retries to filter the noise latency. As the
300 * latency's frequency and maximum value goes up with the number of
301 * CPUs, scale the number of retries with the number of online
302 * CPUs.
303 */
304 return (ilog2(num_online_cpus()) / 2) + 1;
305}
306
295void clocksource_verify_percpu(struct clocksource *cs);307void clocksource_verify_percpu(struct clocksource *cs);
296 308
297#endif /* _LINUX_CLOCKSOURCE_H */309#endif /* _LINUX_CLOCKSOURCE_H */
kernel/time/clocksource-wdtest.c+7−6
@@ -104,8 +104,8 @@ static void wdtest_ktime_clocksource_reset(void) · 103 unmodified lines
104static int wdtest_func(void *arg)104static int wdtest_func(void *arg)
105{105{
106 unsigned long j1, j2;106 unsigned long j1, j2;
107 int i, max_retries;
107 char *s;108 char *s;
108 int i;
109 109
110 schedule_timeout_uninterruptible(holdoff * HZ);110 schedule_timeout_uninterruptible(holdoff * HZ);
111 111
@@ -139,18 +139,19 @@ static int wdtest_func(void *arg) · 27 unmodified lines
139 WARN_ON_ONCE(time_before(j2, j1 + NSEC_PER_USEC));139 WARN_ON_ONCE(time_before(j2, j1 + NSEC_PER_USEC));
140 140
141 /* Verify tsc-like stability with various numbers of errors injected. */141 /* Verify tsc-like stability with various numbers of errors injected. */
142 for (i = 0; i <= max_cswd_read_retries + 1; i++) {142 max_retries = clocksource_get_max_watchdog_retry();
143 if (i <= 1 && i < max_cswd_read_retries)143 for (i = 0; i <= max_retries + 1; i++) {
144 if (i <= 1 && i < max_retries)
144 s = "";145 s = "";
145 else if (i <= max_cswd_read_retries)146 else if (i <= max_retries)
146 s = ", expect message";147 s = ", expect message";
147 else148 else
148 s = ", expect clock skew";149 s = ", expect clock skew";
149 pr_info("--- Watchdog with %dx error injection, %lu retries%s.\n", i, max_cswd_read_retries, s);150 pr_info("--- Watchdog with %dx error injection, %d retries%s.\n", i, max_retries, s);
150 WRITE_ONCE(wdtest_ktime_read_ndelays, i);151 WRITE_ONCE(wdtest_ktime_read_ndelays, i);
151 schedule_timeout_uninterruptible(2 * HZ);152 schedule_timeout_uninterruptible(2 * HZ);
152 WARN_ON_ONCE(READ_ONCE(wdtest_ktime_read_ndelays));153 WARN_ON_ONCE(READ_ONCE(wdtest_ktime_read_ndelays));
153 WARN_ON_ONCE((i <= max_cswd_read_retries) !=154 WARN_ON_ONCE((i <= max_retries) !=
154 !(clocksource_wdtest_ktime.flags & CLOCK_SOURCE_UNSTABLE));155 !(clocksource_wdtest_ktime.flags & CLOCK_SOURCE_UNSTABLE));
155 wdtest_ktime_clocksource_reset();156 wdtest_ktime_clocksource_reset();
156 }157 }
kernel/time/clocksource.c+4−6
@@ -210,9 +210,6 @@ void clocksource_mark_unstable(struct clocksource *cs) · 209 unmodified lines
210 spin_unlock_irqrestore(&watchdog_lock, flags);210 spin_unlock_irqrestore(&watchdog_lock, flags);
211}211}
212 212
213ulong max_cswd_read_retries = 2;
214module_param(max_cswd_read_retries, ulong, 0644);
215EXPORT_SYMBOL_GPL(max_cswd_read_retries);
216static int verify_n_cpus = 8;213static int verify_n_cpus = 8;
217module_param(verify_n_cpus, int, 0644);214module_param(verify_n_cpus, int, 0644);
218 215
@@ -224,11 +221,12 @@ enum wd_read_status { · 5 unmodified lines
224 221
225static enum wd_read_status cs_watchdog_read(struct clocksource *cs, u64 *csnow, u64 *wdnow)222static enum wd_read_status cs_watchdog_read(struct clocksource *cs, u64 *csnow, u64 *wdnow)
226{223{
227 unsigned int nretries;224 unsigned int nretries, max_retries;
228 u64 wd_end, wd_end2, wd_delta;225 u64 wd_end, wd_end2, wd_delta;
229 int64_t wd_delay, wd_seq_delay;226 int64_t wd_delay, wd_seq_delay;
230 227
231 for (nretries = 0; nretries <= max_cswd_read_retries; nretries++) {228 max_retries = clocksource_get_max_watchdog_retry();
229 for (nretries = 0; nretries <= max_retries; nretries++) {
232 local_irq_disable();230 local_irq_disable();
233 *wdnow = watchdog->read(watchdog);231 *wdnow = watchdog->read(watchdog);
234 *csnow = cs->read(cs);232 *csnow = cs->read(cs);
@@ -240,7 +238,7 @@ static enum wd_read_status cs_watchdog_read(struct clocksource *cs, u64 *csnow, · 5 unmodified lines
240 wd_delay = clocksource_cyc2ns(wd_delta, watchdog->mult,238 wd_delay = clocksource_cyc2ns(wd_delta, watchdog->mult,
241 watchdog->shift);239 watchdog->shift);
242 if (wd_delay <= WATCHDOG_MAX_SKEW) {240 if (wd_delay <= WATCHDOG_MAX_SKEW) {
243 if (nretries > 1 || nretries >= max_cswd_read_retries) {241 if (nretries > 1 || nretries >= max_retries) {
244 pr_warn("timekeeping watchdog on CPU%d: %s retried %d times before success\n",242 pr_warn("timekeeping watchdog on CPU%d: %s retried %d times before success\n",
245 smp_processor_id(), watchdog->name, nretries);243 smp_processor_id(), watchdog->name, nretries);
246 }244 }
tools/testing/selftests/rcutorture/bin/torture.sh+1−1
@@ -567,7 +567,7 @@ then · 566 unmodified lines
567 torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000 tsc=watchdog"567 torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000 tsc=watchdog"
568 torture_set "clocksourcewd-1" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration 45s --configs TREE03 --kconfig "CONFIG_TEST_CLOCKSOURCE_WATCHDOG=y" --trust-make568 torture_set "clocksourcewd-1" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration 45s --configs TREE03 --kconfig "CONFIG_TEST_CLOCKSOURCE_WATCHDOG=y" --trust-make
569 569
570 torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000 clocksource.max_cswd_read_retries=1 tsc=watchdog"570 torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000 tsc=watchdog"
571 torture_set "clocksourcewd-2" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration 45s --configs TREE03 --kconfig "CONFIG_TEST_CLOCKSOURCE_WATCHDOG=y" --trust-make571 torture_set "clocksourcewd-2" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration 45s --configs TREE03 --kconfig "CONFIG_TEST_CLOCKSOURCE_WATCHDOG=y" --trust-make
572 572
573 # In case our work is already done...573 # In case our work is already done...

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t