// SPDX-License-Identifier: GPL-2.0/* * Detect hard and soft lockups on a system * * started by Don Zickus, Copyright (C) 2010 Red Hat, Inc. * * Note: Most of this code is borrowed heavily from the original softlockup * detector, so thanks to Ingo for the initial implementation. * Some chunks also taken from the old x86-specific nmi watchdog code, thanks * to those contributors as well. */ #define pr_fmt(fmt) "watchdog: " fmt #include <linux/cpu.h>#include <linux/init.h>#include <linux/irq.h>#include <linux/irqdesc.h>#include <linux/kernel_stat.h>#include <linux/kvm_para.h>#include <linux/math64.h>#include <linux/mm.h>#include <linux/module.h>#include <linux/nmi.h>#include <linux/stop_machine.h>#include <linux/sysctl.h>#include <linux/tick.h> #include <linux/sched/clock.h>#include <linux/sched/debug.h>#include <linux/sched/isolation.h> #include <asm/irq_regs.h> static DEFINE_MUTEX(watchdog_mutex); #if defined(CONFIG_HARDLOCKUP_DETECTOR) || defined(CONFIG_HARDLOCKUP_DETECTOR_SPARC64)# define WATCHDOG_HARDLOCKUP_DEFAULT	1#else# define WATCHDOG_HARDLOCKUP_DEFAULT	0#endif #define NUM_SAMPLE_PERIODS	5 unsigned long __read_mostly watchdog_enabled;int __read_mostly watchdog_user_enabled = 1;static int __read_mostly watchdog_hardlockup_user_enabled = WATCHDOG_HARDLOCKUP_DEFAULT;static int __read_mostly watchdog_softlockup_user_enabled = 1;int __read_mostly watchdog_thresh = 10;static int __read_mostly watchdog_hardlockup_available; struct cpumask watchdog_cpumask __read_mostly;unsigned long *watchdog_cpumask_bits = cpumask_bits(&watchdog_cpumask); #ifdef CONFIG_HARDLOCKUP_DETECTOR # ifdef CONFIG_SMPint __read_mostly sysctl_hardlockup_all_cpu_backtrace;# endif /* CONFIG_SMP */ /* * Should we panic when a soft-lockup or hard-lockup occurs: */unsigned int __read_mostly hardlockup_panic =			IS_ENABLED(CONFIG_BOOTPARAM_HARDLOCKUP_PANIC);/* * We may not want to enable hard lockup detection by default in all cases, * for example when running the kernel as a guest on a hypervisor. In these * cases this function can be called to disable hard lockup detection. This * function should only be executed once by the boot processor before the * kernel command line parameters are parsed, because otherwise it is not * possible to override this in hardlockup_panic_setup(). */void __init hardlockup_detector_disable(void){	watchdog_hardlockup_user_enabled = 0;} static int __init hardlockup_panic_setup(char *str){next:	if (!strncmp(str, "panic", 5))		hardlockup_panic = 1;	else if (!strncmp(str, "nopanic", 7))		hardlockup_panic = 0;	else if (!strncmp(str, "0", 1))		watchdog_hardlockup_user_enabled = 0;	else if (!strncmp(str, "1", 1))		watchdog_hardlockup_user_enabled = 1;	else if (!strncmp(str, "r", 1))		hardlockup_config_perf_event(str + 1);	while (*(str++)) {		if (*str == ',') {			str++;			goto next;		}	}	return 1;}__setup("nmi_watchdog=", hardlockup_panic_setup); #endif /* CONFIG_HARDLOCKUP_DETECTOR */ #if defined(CONFIG_HARDLOCKUP_DETECTOR_COUNTS_HRTIMER) static DEFINE_PER_CPU(atomic_t, hrtimer_interrupts);static DEFINE_PER_CPU(int, hrtimer_interrupts_saved);static DEFINE_PER_CPU(bool, watchdog_hardlockup_warned);static DEFINE_PER_CPU(bool, watchdog_hardlockup_touched);static unsigned long hard_lockup_nmi_warn; notrace void arch_touch_nmi_watchdog(void){	/*	 * Using __raw here because some code paths have	 * preemption enabled.  If preemption is enabled	 * then interrupts should be enabled too, in which	 * case we shouldn't have to worry about the watchdog	 * going off.	 */	raw_cpu_write(watchdog_hardlockup_touched, true);}EXPORT_SYMBOL(arch_touch_nmi_watchdog); void watchdog_hardlockup_touch_cpu(unsigned int cpu){	per_cpu(watchdog_hardlockup_touched, cpu) = true;} static bool is_hardlockup(unsigned int cpu){	int hrint = atomic_read(&per_cpu(hrtimer_interrupts, cpu)); 	if (per_cpu(hrtimer_interrupts_saved, cpu) == hrint)		return true; 	/*	 * NOTE: we don't need any fancy atomic_t or READ_ONCE/WRITE_ONCE	 * for hrtimer_interrupts_saved. hrtimer_interrupts_saved is	 * written/read by a single CPU.	 */	per_cpu(hrtimer_interrupts_saved, cpu) = hrint; 	return false;} static void watchdog_hardlockup_kick(void){	int new_interrupts; 	new_interrupts = atomic_inc_return(this_cpu_ptr(&hrtimer_interrupts));	watchdog_buddy_check_hardlockup(new_interrupts);} void watchdog_hardlockup_check(unsigned int cpu, struct pt_regs *regs){	if (per_cpu(watchdog_hardlockup_touched, cpu)) {		per_cpu(watchdog_hardlockup_touched, cpu) = false;		return;	} 	/*	 * Check for a hardlockup by making sure the CPU's timer	 * interrupt is incrementing. The timer interrupt should have	 * fired multiple times before we overflow'd. If it hasn't	 * then this is a good indication the cpu is stuck	 */	if (is_hardlockup(cpu)) {		unsigned int this_cpu = smp_processor_id();		unsigned long flags; 		/* Only print hardlockups once. */		if (per_cpu(watchdog_hardlockup_warned, cpu))			return; 		/*		 * Prevent multiple hard-lockup reports if one cpu is already		 * engaged in dumping all cpu back traces.		 */		if (sysctl_hardlockup_all_cpu_backtrace) {			if (test_and_set_bit_lock(0, &hard_lockup_nmi_warn))				return;		} 		/*		 * NOTE: we call printk_cpu_sync_get_irqsave() after printing		 * the lockup message. While it would be nice to serialize		 * that printout, we really want to make sure that if some		 * other CPU somehow locked up while holding the lock associated		 * with printk_cpu_sync_get_irqsave() that we can still at least		 * get the message about the lockup out.		 */		pr_emerg("Watchdog detected hard LOCKUP on cpu %d\n", cpu);		printk_cpu_sync_get_irqsave(flags); 		print_modules();		print_irqtrace_events(current);		if (cpu == this_cpu) {			if (regs)				show_regs(regs);			else				dump_stack();			printk_cpu_sync_put_irqrestore(flags);		} else {			printk_cpu_sync_put_irqrestore(flags);			trigger_single_cpu_backtrace(cpu);		} 		if (sysctl_hardlockup_all_cpu_backtrace) {			trigger_allbutcpu_cpu_backtrace(cpu);			if (!hardlockup_panic)				clear_bit_unlock(0, &hard_lockup_nmi_warn);		} 		if (hardlockup_panic)			nmi_panic(regs, "Hard LOCKUP"); 		per_cpu(watchdog_hardlockup_warned, cpu) = true;	} else {		per_cpu(watchdog_hardlockup_warned, cpu) = false;	}} #else /* CONFIG_HARDLOCKUP_DETECTOR_COUNTS_HRTIMER */ static inline void watchdog_hardlockup_kick(void) { } #endif /* !CONFIG_HARDLOCKUP_DETECTOR_COUNTS_HRTIMER */ /* * These functions can be overridden based on the configured hardlockdup detector. * * watchdog_hardlockup_enable/disable can be implemented to start and stop when * softlockup watchdog start and stop. The detector must select the * SOFTLOCKUP_DETECTOR Kconfig. */void __weak watchdog_hardlockup_enable(unsigned int cpu) { } void __weak watchdog_hardlockup_disable(unsigned int cpu) { } /* * Watchdog-detector specific API. * * Return 0 when hardlockup watchdog is available, negative value otherwise. * Note that the negative value means that a delayed probe might * succeed later. */int __weak __init watchdog_hardlockup_probe(void){	return -ENODEV;} /** * watchdog_hardlockup_stop - Stop the watchdog for reconfiguration * * The reconfiguration steps are: * watchdog_hardlockup_stop(); * update_variables(); * watchdog_hardlockup_start(); */void __weak watchdog_hardlockup_stop(void) { } /** * watchdog_hardlockup_start - Start the watchdog after reconfiguration * * Counterpart to watchdog_hardlockup_stop(). * * The following variables have been updated in update_variables() and * contain the currently valid configuration: * - watchdog_enabled * - watchdog_thresh * - watchdog_cpumask */void __weak watchdog_hardlockup_start(void) { } /** * lockup_detector_update_enable - Update the sysctl enable bit * * Caller needs to make sure that the hard watchdogs are off, so this * can't race with watchdog_hardlockup_disable(). */static void lockup_detector_update_enable(void){	watchdog_enabled = 0;	if (!watchdog_user_enabled)		return;	if (watchdog_hardlockup_available && watchdog_hardlockup_user_enabled)		watchdog_enabled |= WATCHDOG_HARDLOCKUP_ENABLED;	if (watchdog_softlockup_user_enabled)		watchdog_enabled |= WATCHDOG_SOFTOCKUP_ENABLED;} #ifdef CONFIG_SOFTLOCKUP_DETECTOR /* * Delay the soflockup report when running a known slow code. * It does _not_ affect the timestamp of the last successdul reschedule. */#define SOFTLOCKUP_DELAY_REPORT	ULONG_MAX #ifdef CONFIG_SMPint __read_mostly sysctl_softlockup_all_cpu_backtrace;#endif static struct cpumask watchdog_allowed_mask __read_mostly; /* Global variables, exported for sysctl */unsigned int __read_mostly softlockup_panic =			IS_ENABLED(CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC); static bool softlockup_initialized __read_mostly;static u64 __read_mostly sample_period; /* Timestamp taken after the last successful reschedule. */static DEFINE_PER_CPU(unsigned long, watchdog_touch_ts);/* Timestamp of the last softlockup report. */static DEFINE_PER_CPU(unsigned long, watchdog_report_ts);static DEFINE_PER_CPU(struct hrtimer, watchdog_hrtimer);static DEFINE_PER_CPU(bool, softlockup_touch_sync);static unsigned long soft_lockup_nmi_warn; static int __init softlockup_panic_setup(char *str){	softlockup_panic = simple_strtoul(str, NULL, 0);	return 1;}__setup("softlockup_panic=", softlockup_panic_setup); static int __init nowatchdog_setup(char *str){	watchdog_user_enabled = 0;	return 1;}__setup("nowatchdog", nowatchdog_setup); static int __init nosoftlockup_setup(char *str){	watchdog_softlockup_user_enabled = 0;	return 1;}__setup("nosoftlockup", nosoftlockup_setup); static int __init watchdog_thresh_setup(char *str){	get_option(&str, &watchdog_thresh);	return 1;}__setup("watchdog_thresh=", watchdog_thresh_setup); static void __lockup_detector_cleanup(void); #ifdef CONFIG_SOFTLOCKUP_DETECTOR_INTR_STORMenum stats_per_group {	STATS_SYSTEM,	STATS_SOFTIRQ,	STATS_HARDIRQ,	STATS_IDLE,	NUM_STATS_PER_GROUP,}; static const enum cpu_usage_stat tracked_stats[NUM_STATS_PER_GROUP] = {	CPUTIME_SYSTEM,	CPUTIME_SOFTIRQ,	CPUTIME_IRQ,	CPUTIME_IDLE,}; static DEFINE_PER_CPU(u16, cpustat_old[NUM_STATS_PER_GROUP]);static DEFINE_PER_CPU(u8, cpustat_util[NUM_SAMPLE_PERIODS][NUM_STATS_PER_GROUP]);static DEFINE_PER_CPU(u8, cpustat_tail); /* * We don't need nanosecond resolution. A granularity of 16ms is * sufficient for our precision, allowing us to use u16 to store * cpustats, which will roll over roughly every ~1000 seconds. * 2^24 ~= 16 * 10^6 */static u16 get_16bit_precision(u64 data_ns){	return data_ns >> 24LL; /* 2^24ns ~= 16.8ms */} static void update_cpustat(void){	int i;	u8 util;	u16 old_stat, new_stat;	struct kernel_cpustat kcpustat;	u64 *cpustat = kcpustat.cpustat;	u8 tail = __this_cpu_read(cpustat_tail);	u16 sample_period_16 = get_16bit_precision(sample_period); 	kcpustat_cpu_fetch(&kcpustat, smp_processor_id()); 	for (i = 0; i < NUM_STATS_PER_GROUP; i++) {		old_stat = __this_cpu_read(cpustat_old[i]);		new_stat = get_16bit_precision(cpustat[tracked_stats[i]]);		util = DIV_ROUND_UP(100 * (new_stat - old_stat), sample_period_16);		__this_cpu_write(cpustat_util[tail][i], util);		__this_cpu_write(cpustat_old[i], new_stat);	} 	__this_cpu_write(cpustat_tail, (tail + 1) % NUM_SAMPLE_PERIODS);} static void print_cpustat(void){	int i, group;	u8 tail = __this_cpu_read(cpustat_tail);	u64 sample_period_second = sample_period; 	do_div(sample_period_second, NSEC_PER_SEC); 	/*	 * Outputting the "watchdog" prefix on every line is redundant and not	 * concise, and the original alarm information is sufficient for	 * positioning in logs, hence here printk() is used instead of pr_crit().	 */	printk(KERN_CRIT "CPU#%d Utilization every %llus during lockup:\n",	       smp_processor_id(), sample_period_second); 	for (i = 0; i < NUM_SAMPLE_PERIODS; i++) {		group = (tail + i) % NUM_SAMPLE_PERIODS;		printk(KERN_CRIT "\t#%d: %3u%% system,\t%3u%% softirq,\t"			"%3u%% hardirq,\t%3u%% idle\n", i + 1,			__this_cpu_read(cpustat_util[group][STATS_SYSTEM]),			__this_cpu_read(cpustat_util[group][STATS_SOFTIRQ]),			__this_cpu_read(cpustat_util[group][STATS_HARDIRQ]),			__this_cpu_read(cpustat_util[group][STATS_IDLE]));	}} #define HARDIRQ_PERCENT_THRESH          50#define NUM_HARDIRQ_REPORT              5struct irq_counts {	int irq;	u32 counts;}; static DEFINE_PER_CPU(bool, snapshot_taken); /* Tabulate the most frequent interrupts. */static void tabulate_irq_count(struct irq_counts *irq_counts, int irq, u32 counts, int rank){	int i;	struct irq_counts new_count = {irq, counts}; 	for (i = 0; i < rank; i++) {		if (counts > irq_counts[i].counts)			swap(new_count, irq_counts[i]);	}} /* * If the hardirq time exceeds HARDIRQ_PERCENT_THRESH% of the sample_period, * then the cause of softlockup might be interrupt storm. In this case, it * would be useful to start interrupt counting. */static bool need_counting_irqs(void){	u8 util;	int tail = __this_cpu_read(cpustat_tail); 	tail = (tail + NUM_HARDIRQ_REPORT - 1) % NUM_HARDIRQ_REPORT;	util = __this_cpu_read(cpustat_util[tail][STATS_HARDIRQ]);	return util > HARDIRQ_PERCENT_THRESH;} static void start_counting_irqs(void){	if (!__this_cpu_read(snapshot_taken)) {		kstat_snapshot_irqs();		__this_cpu_write(snapshot_taken, true);	}} static void stop_counting_irqs(void){	__this_cpu_write(snapshot_taken, false);} static void print_irq_counts(void){	unsigned int i, count;	struct irq_counts irq_counts_sorted[NUM_HARDIRQ_REPORT] = {		{-1, 0}, {-1, 0}, {-1, 0}, {-1, 0}, {-1, 0}	}; 	if (__this_cpu_read(snapshot_taken)) {		for_each_active_irq(i) {			count = kstat_get_irq_since_snapshot(i);			tabulate_irq_count(irq_counts_sorted, i, count, NUM_HARDIRQ_REPORT);		} 		/*		 * Outputting the "watchdog" prefix on every line is redundant and not		 * concise, and the original alarm information is sufficient for		 * positioning in logs, hence here printk() is used instead of pr_crit().		 */		printk(KERN_CRIT "CPU#%d Detect HardIRQ Time exceeds %d%%. Most frequent HardIRQs:\n",

Lines 1–500 of 1278. Load more