History

softlockup: automatically detect hung TASK_UNINTERRUPTIBLE tasks

this patch extends the soft-lockup detector to automatically detect hung TASK_UNINTERRUPTIBLE tasks. Such hung tasks are printed the following way: ------------------> INFO: task prctl:3042 blocked for more than 120 seconds. "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message prctl D fd5e3793 0 3042 2997 f6050f38 00000046 00000001 fd5e3793 00000009 c06d8264 c06dae80 00000286 f6050f40 f6050f00 f7d34d90 f7d34fc8 c1e1be80 00000001 f6050000 00000000 f7e92d00 00000286 f6050f18 c0489d1a f6050f40 00006605 00000000 c0133a5b Call Trace: [<c04883a5>] schedule_timeout+0x6d/0x8b [<c04883d8>] schedule_timeout_uninterruptible+0x15/0x17 [<c0133a76>] msleep+0x10/0x16 [<c0138974>] sys_prctl+0x30/0x1e2 [<c0104c52>] sysenter_past_esp+0x5f/0xa5 ======================= 2 locks held by prctl/3042: #0: (&sb->s_type->i_mutex_key#5){--..}, at: [<c0197d11>] do_fsync+0x38/0x7a #1: (jbd_handle){--..}, at: [<c01ca3d2>] journal_start+0xc7/0xe9 <------------------ the current default timeout is 120 seconds. Such messages are printed up to 10 times per bootup. If the system has crashed already then the messages are not printed. if lockdep is enabled then all held locks are printed as well. this feature is a natural extension to the softlockup-detector (kernel locked up without scheduling) and to the NMI watchdog (kernel locked up with IRQs disabled). [ Gautham R Shenoy <ego@in.ibm.com>: CPU hotplug fixes. ] [ Andrew Morton <akpm@linux-foundation.org>: build warning fix. ]

Ingo Molnar committed 18 years ago

Commit 82a1fcb1 parent d0d23b57 files · +164 −13Browse files

Unified · Split

7 files
  1. include/linux/debug_locks.h +5 −0
  2. include/linux/sched.h +10 −0
  3. kernel/fork.c +5 −0
  4. kernel/lockdep.c +11 −1
  5. kernel/sched.c +2 −2
  6. kernel/softlockup.c +104 −10
  7. kernel/sysctl.c +27 −0
include/linux/debug_locks.h+5−0
@@ -47,6 +47,7 @@ struct task_struct; · 46 unmodified lines
4747
4848 #ifdef CONFIG_LOCKDEP
4949 extern void debug_show_all_locks(void);
50+extern void __debug_show_held_locks(struct task_struct *task);
5051 extern void debug_show_held_locks(struct task_struct *task);
5152 extern void debug_check_no_locks_freed(const void *from, unsigned long len);
5253 extern void debug_check_no_locks_held(struct task_struct *task);
@@ -55,6 +56,10 @@ static inline void debug_show_all_locks(void) · 2 unmodified lines
5556 {
5657 }
5758
59+static inline void __debug_show_held_locks(struct task_struct *task)
60+{
61+}
62+
5863 static inline void debug_show_held_locks(struct task_struct *task)
5964 {
6065 }
include/linux/sched.h+10−0
@@ -258,12 +258,17 @@ extern void account_process_tick(struct task_struct *task, int user); · 257 unmodified lines
258258 extern void update_process_times(int user);
259259 extern void scheduler_tick(void);
260260
261+extern void sched_show_task(struct task_struct *p);
262+
261263 #ifdef CONFIG_DETECT_SOFTLOCKUP
262264 extern void softlockup_tick(void);
263265 extern void spawn_softlockup_task(void);
264266 extern void touch_softlockup_watchdog(void);
265267 extern void touch_all_softlockup_watchdogs(void);
266268 extern int softlockup_thresh;
269+extern unsigned long sysctl_hung_task_check_count;
270+extern unsigned long sysctl_hung_task_timeout_secs;
271+extern long sysctl_hung_task_warnings;
267272 #else
268273 static inline void softlockup_tick(void)
269274 {
@@ -1041,6 +1046,11 @@ struct task_struct { · 771 unmodified lines
10411046 /* ipc stuff */
10421047 struct sysv_sem sysvsem;
10431048 #endif
1049+#ifdef CONFIG_DETECT_SOFTLOCKUP
1050+/* hung task detection */
1051+ unsigned long last_switch_timestamp;
1052+ unsigned long last_switch_count;
1053+#endif
10441054 /* CPU-specific state of this task */
10451055 struct thread_struct thread;
10461056 /* filesystem information */
kernel/fork.c+5−0
@@ -1059,6 +1059,11 @@ static struct task_struct *copy_process(unsigned long clone_flags, · 1058 unmodified lines
10591059 p->prev_utime = cputime_zero;
10601060 p->prev_stime = cputime_zero;
10611061
1062+#ifdef CONFIG_DETECT_SOFTLOCKUP
1063+ p->last_switch_count = 0;
1064+ p->last_switch_timestamp = 0;
1065+#endif
1066+
10621067 #ifdef CONFIG_TASK_XACCT
10631068 p->rchar = 0; /* I/O counter: bytes read */
10641069 p->wchar = 0; /* I/O counter: bytes written */
kernel/lockdep.c+11−1
@@ -3206,7 +3206,11 @@ retry: · 3205 unmodified lines
32063206
32073207 EXPORT_SYMBOL_GPL(debug_show_all_locks);
32083208
3209−void debug_show_held_locks(struct task_struct *task)
3209+/*
3210+ * Careful: only use this function if you are sure that
3211+ * the task cannot run in parallel!
3212+ */
3213+void __debug_show_held_locks(struct task_struct *task)
32103214 {
32113215 if (unlikely(!debug_locks)) {
32123216 printk("INFO: lockdep is turned off.\n");
@@ -3214,6 +3218,12 @@ void debug_show_held_locks(struct task_struct *task) · 1 unmodified line
32143218 }
32153219 lockdep_print_held_locks(task);
32163220 }
3221+EXPORT_SYMBOL_GPL(__debug_show_held_locks);
3222+
3223+void debug_show_held_locks(struct task_struct *task)
3224+{
3225+ __debug_show_held_locks(task);
3226+}
32173227
32183228 EXPORT_SYMBOL_GPL(debug_show_held_locks);
32193229
kernel/sched.c+2−2
@@ -4945,7 +4945,7 @@ out_unlock: · 4944 unmodified lines
49454945
49464946 static const char stat_nam[] = "RSDTtZX";
49474947
4948−static void show_task(struct task_struct *p)
4948+void sched_show_task(struct task_struct *p)
49494949 {
49504950 unsigned long free = 0;
49514951 unsigned state;
@@ -4998,7 +4998,7 @@ void show_state_filter(unsigned long state_filter) · 46 unmodified lines
49984998 */
49994999 touch_nmi_watchdog();
50005000 if (!state_filter || (p->state & state_filter))
5001− show_task(p);
5001+ sched_show_task(p);
50025002 } while_each_thread(g, p);
50035003
50045004 touch_all_softlockup_watchdogs();
kernel/softlockup.c+104−10
@@ -8,6 +8,7 @@ · 7 unmodified lines
88 */
99 #include <linux/mm.h>
1010 #include <linux/cpu.h>
11+#include <linux/nmi.h>
1112 #include <linux/init.h>
1213 #include <linux/delay.h>
1314 #include <linux/freezer.h>
@@ -24,7 +25,7 @@ static DEFINE_PER_CPU(unsigned long, print_timestamp); · 10 unmodified lines
2425 static DEFINE_PER_CPU(struct task_struct *, watchdog_task);
2526
2627 static int did_panic;
27−int softlockup_thresh = 10;
28+int softlockup_thresh = 60;
2829
2930 static int
3031 softlock_panic(struct notifier_block *this, unsigned long event, void *ptr)
@@ -45,7 +46,7 @@ static struct notifier_block panic_block = { · 14 unmodified lines
4546 */
4647 static unsigned long get_timestamp(int this_cpu)
4748 {
48− return cpu_clock(this_cpu) >> 30; /* 2^30 ~= 10^9 */
49+ return cpu_clock(this_cpu) >> 30LL; /* 2^30 ~= 10^9 */
4950 }
5051
5152 void touch_softlockup_watchdog(void)
@@ -100,11 +101,7 @@ void softlockup_tick(void) · 48 unmodified lines
100101
101102 now = get_timestamp(this_cpu);
102103
103− /* Wake up the high-prio watchdog task every second: */
104− if (now > (touch_timestamp + 1))
105− wake_up_process(per_cpu(watchdog_task, this_cpu));
106−
107− /* Warn about unreasonable 10+ seconds delays: */
104+ /* Warn about unreasonable delays: */
108105 if (now <= (touch_timestamp + softlockup_thresh))
109106 return;
110107
@@ -121,12 +118,94 @@ void softlockup_tick(void) · 10 unmodified lines
121118 spin_unlock(&print_lock);
122119 }
123120
121+/*
122+ * Have a reasonable limit on the number of tasks checked:
123+ */
124+unsigned long sysctl_hung_task_check_count = 1024;
125+
126+/*
127+ * Zero means infinite timeout - no checking done:
128+ */
129+unsigned long sysctl_hung_task_timeout_secs = 120;
130+
131+long sysctl_hung_task_warnings = 10;
132+
133+/*
134+ * Only do the hung-tasks check on one CPU:
135+ */
136+static int check_cpu __read_mostly = -1;
137+
138+static void check_hung_task(struct task_struct *t, unsigned long now)
139+{
140+ unsigned long switch_count = t->nvcsw + t->nivcsw;
141+
142+ if (t->flags & PF_FROZEN)
143+ return;
144+
145+ if (switch_count != t->last_switch_count || !t->last_switch_timestamp) {
146+ t->last_switch_count = switch_count;
147+ t->last_switch_timestamp = now;
148+ return;
149+ }
150+ if ((long)(now - t->last_switch_timestamp) <
151+ sysctl_hung_task_timeout_secs)
152+ return;
153+ if (sysctl_hung_task_warnings < 0)
154+ return;
155+ sysctl_hung_task_warnings--;
156+
157+ /*
158+ * Ok, the task did not get scheduled for more than 2 minutes,
159+ * complain:
160+ */
161+ printk(KERN_ERR "INFO: task %s:%d blocked for more than "
162+ "%ld seconds.\n", t->comm, t->pid,
163+ sysctl_hung_task_timeout_secs);
164+ printk(KERN_ERR "\"echo 0 > /proc/sys/kernel/hung_task_timeout_secs\""
165+ " disables this message.\n");
166+ sched_show_task(t);
167+ __debug_show_held_locks(t);
168+
169+ t->last_switch_timestamp = now;
170+ touch_nmi_watchdog();
171+}
172+
173+/*
174+ * Check whether a TASK_UNINTERRUPTIBLE does not get woken up for
175+ * a really long time (120 seconds). If that happens, print out
176+ * a warning.
177+ */
178+static void check_hung_uninterruptible_tasks(int this_cpu)
179+{
180+ int max_count = sysctl_hung_task_check_count;
181+ unsigned long now = get_timestamp(this_cpu);
182+ struct task_struct *g, *t;
183+
184+ /*
185+ * If the system crashed already then all bets are off,
186+ * do not report extra hung tasks:
187+ */
188+ if ((tainted & TAINT_DIE) || did_panic)
189+ return;
190+
191+ read_lock(&tasklist_lock);
192+ do_each_thread(g, t) {
193+ if (!--max_count)
194+ break;
195+ if (t->state & TASK_UNINTERRUPTIBLE)
196+ check_hung_task(t, now);
197+ } while_each_thread(g, t);
198+
199+ read_unlock(&tasklist_lock);
200+}
201+
124202 /*
125203 * The watchdog thread - runs every second and touches the timestamp.
126204 */
127205 static int watchdog(void *__bind_cpu)
128206 {
129207 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
208+ int this_cpu = (long)__bind_cpu;
130209
131210 sched_setscheduler(current, SCHED_FIFO, &param);
132211
@@ -135,13 +214,18 @@ static int watchdog(void *__bind_cpu) · 2 unmodified lines
135214
136215 /*
137216 * Run briefly once per second to reset the softlockup timestamp.
138− * If this gets delayed for more than 10 seconds then the
217+ * If this gets delayed for more than 60 seconds then the
139218 * debug-printout triggers in softlockup_tick().
140219 */
141220 while (!kthread_should_stop()) {
142− set_current_state(TASK_INTERRUPTIBLE);
143221 touch_softlockup_watchdog();
144− schedule();
222+ msleep_interruptible(10000);
223+
224+ if (this_cpu != check_cpu)
225+ continue;
226+
227+ if (sysctl_hung_task_timeout_secs)
228+ check_hung_uninterruptible_tasks(this_cpu);
145229 }
146230
147231 return 0;
@@ -171,6 +255,7 @@ cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) · 23 unmodified lines
171255 break;
172256 case CPU_ONLINE:
173257 case CPU_ONLINE_FROZEN:
258+ check_cpu = any_online_cpu(cpu_online_map);
174259 wake_up_process(per_cpu(watchdog_task, hotcpu));
175260 break;
176261 #ifdef CONFIG_HOTPLUG_CPU
@@ -181,6 +266,15 @@ cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) · 4 unmodified lines
181266 /* Unbind so it can run. Fall thru. */
182267 kthread_bind(per_cpu(watchdog_task, hotcpu),
183268 any_online_cpu(cpu_online_map));
269+ case CPU_DOWN_PREPARE:
270+ case CPU_DOWN_PREPARE_FROZEN:
271+ if (hotcpu == check_cpu) {
272+ cpumask_t temp_cpu_online_map = cpu_online_map;
273+
274+ cpu_clear(hotcpu, temp_cpu_online_map);
275+ check_cpu = any_online_cpu(temp_cpu_online_map);
276+ }
277+ break;
184278 case CPU_DEAD:
185279 case CPU_DEAD_FROZEN:
186280 p = per_cpu(watchdog_task, hotcpu);
kernel/sysctl.c+27−0
@@ -753,6 +753,33 @@ static struct ctl_table kern_table[] = { · 752 unmodified lines
753753 .extra1 = &one,
754754 .extra2 = &sixty,
755755 },
756+ {
757+ .ctl_name = CTL_UNNUMBERED,
758+ .procname = "hung_task_check_count",
759+ .data = &sysctl_hung_task_check_count,
760+ .maxlen = sizeof(int),
761+ .mode = 0644,
762+ .proc_handler = &proc_dointvec_minmax,
763+ .strategy = &sysctl_intvec,
764+ },
765+ {
766+ .ctl_name = CTL_UNNUMBERED,
767+ .procname = "hung_task_timeout_secs",
768+ .data = &sysctl_hung_task_timeout_secs,
769+ .maxlen = sizeof(int),
770+ .mode = 0644,
771+ .proc_handler = &proc_dointvec_minmax,
772+ .strategy = &sysctl_intvec,
773+ },
774+ {
775+ .ctl_name = CTL_UNNUMBERED,
776+ .procname = "hung_task_warnings",
777+ .data = &sysctl_hung_task_warnings,
778+ .maxlen = sizeof(int),
779+ .mode = 0644,
780+ .proc_handler = &proc_dointvec_minmax,
781+ .strategy = &sysctl_intvec,
782+ },
756783 #endif
757784 #ifdef CONFIG_COMPAT
758785 {

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t