printk: Wait for the global console lock when the system is going down

There are reports that the console kthreads block the global console lock when the system is going down, for example, reboot, panic. First part of the solution was to block kthreads in these problematic system states so they stopped handling newly added messages. Second part of the solution is to wait when for the kthreads when they are actively printing. It solves the problem when a message was printed before the system entered the problematic state and the kthreads managed to step in. A busy waiting has to be used because panic() can be called in any context and in an unknown state of the scheduler. There must be a timeout because the kthread might get stuck or sleeping and never release the lock. The timeout 10s is an arbitrary value inspired by the softlockup timeout.

Petr Mladek committed 4 years ago

Commit b87f0231 parent c3230286 files · +47 −0Browse files

Unified · Split

6 files

include/linux/printk.h +5 −0
kernel/panic.c +2 −0
kernel/printk/internal.h +2 −0
kernel/printk/printk.c +4 −0
kernel/printk/printk_safe.c +32 −0
kernel/reboot.c +2 −0

include/linux/printk.h+5−0

@@ -174,6 +174,7 @@ extern void printk_prefer_direct_enter(void); · 173 unmodified lines
174	174	extern void printk_prefer_direct_exit(void);
175	175
176	176	extern bool pr_flush(int timeout_ms, bool reset_on_progress);
	177	+extern void try_block_console_kthreads(int timeout_ms);
177	178
178	179	/*
179	180	* Please don't use printk_ratelimit(), because it shares ratelimiting state
@@ -238,6 +239,10 @@ static inline bool pr_flush(int timeout_ms, bool reset_on_progress) · 58 unmodified lines
238	239	return true;
239	240	}
240	241
	242	+static inline void try_block_console_kthreads(int timeout_ms)
	243	+{
	244	+}
	245	+
241	246	static inline int printk_ratelimit(void)
242	247	{
243	248	return 0;

kernel/panic.c+2−0

@@ -273,6 +273,7 @@ void panic(const char *fmt, ...) · 272 unmodified lines
273	273	* unfortunately means it may not be hardened to work in a
274	274	* panic situation.
275	275	*/
	276	+ try_block_console_kthreads(10000);
276	277	smp_send_stop();
277	278	} else {
278	279	/*
@@ -280,6 +281,7 @@ void panic(const char *fmt, ...) · 1 unmodified line
280	281	* kmsg_dump, we will need architecture dependent extra
281	282	* works in addition to stopping other CPUs.
282	283	*/
	284	+ try_block_console_kthreads(10000);
283	285	crash_smp_send_stop();
284	286	}
285	287

kernel/printk/internal.h+2−0

@@ -20,6 +20,8 @@ enum printk_info_flags { · 19 unmodified lines
20	20	LOG_CONT = 8, /* text is a fragment of a continuation line */
21	21	};
22	22
	23	+extern bool block_console_kthreads;
	24	+
23	25	__printf(4, 0)
24	26	int vprintk_store(int facility, int level,
25	27	const struct dev_printk_info *dev_info,

kernel/printk/printk.c+4−0

@@ -250,6 +250,9 @@ static atomic_t console_kthreads_active = ATOMIC_INIT(0); · 249 unmodified lines
250	250	#define console_kthread_printing_exit() \
251	251	atomic_dec(&console_kthreads_active)
252	252
	253	+/* Block console kthreads to avoid processing new messages. */
	254	+bool block_console_kthreads;
	255	+
253	256	/*
254	257	* Helper macros to handle lockdep when locking/unlocking console_sem. We use
255	258	* macros instead of functions so that _RET_IP_ contains useful information.
@@ -3730,6 +3733,7 @@ static bool printer_should_wake(struct console *con, u64 seq) · 3474 unmodified lines
3730	3733
3731	3734	if (con->blocked \|\|
3732	3735	console_kthreads_atomically_blocked() \|\|
	3736	+ block_console_kthreads \|\|
3733	3737	system_state > SYSTEM_RUNNING \|\|
3734	3738	oops_in_progress) {
3735	3739	return false;

kernel/printk/printk_safe.c+32−0

@@ -8,7 +8,9 @@ · 7 unmodified lines
8	8	#include <linux/smp.h>
9	9	#include <linux/cpumask.h>
10	10	#include <linux/printk.h>
	11	+#include <linux/console.h>
11	12	#include <linux/kprobes.h>
	13	+#include <linux/delay.h>
12	14
13	15	#include "internal.h"
14	16
@@ -50,3 +52,33 @@ asmlinkage int vprintk(const char *fmt, va_list args) · 35 unmodified lines
50	52	return vprintk_default(fmt, args);
51	53	}
52	54	EXPORT_SYMBOL(vprintk);
	55	+
	56	+/**
	57	+ * try_block_console_kthreads() - Try to block console kthreads and
	58	+ * make the global console_lock() avaialble
	59	+ *
	60	+ * @timeout_ms: The maximum time (in ms) to wait.
	61	+ *
	62	+ * Prevent console kthreads from starting processing new messages. Wait
	63	+ * until the global console_lock() become available.
	64	+ *
	65	+ * Context: Can be called in any context.
	66	+ */
	67	+void try_block_console_kthreads(int timeout_ms)
	68	+{
	69	+ block_console_kthreads = true;
	70	+
	71	+ /* Do not wait when the console lock could not be safely taken. */
	72	+ if (this_cpu_read(printk_context) \|\| in_nmi())
	73	+ return;
	74	+
	75	+ while (timeout_ms > 0) {
	76	+ if (console_trylock()) {
	77	+ console_unlock();
	78	+ return;
	79	+ }
	80	+
	81	+ udelay(1000);
	82	+ timeout_ms -= 1;
	83	+ }
	84	+}

kernel/reboot.c+2−0

@@ -74,6 +74,7 @@ void kernel_restart_prepare(char *cmd) · 73 unmodified lines
74	74	{
75	75	blocking_notifier_call_chain(&reboot_notifier_list, SYS_RESTART, cmd);
76	76	system_state = SYSTEM_RESTART;
	77	+ try_block_console_kthreads(10000);
77	78	usermodehelper_disable();
78	79	device_shutdown();
79	80	}
@@ -262,6 +263,7 @@ static void kernel_shutdown_prepare(enum system_states state) · 182 unmodified lines
262	263	blocking_notifier_call_chain(&reboot_notifier_list,
263	264	(state == SYSTEM_HALT) ? SYS_HALT : SYS_POWER_OFF, NULL);
264	265	system_state = state;
	266	+ try_block_console_kthreads(10000);
265	267	usermodehelper_disable();
266	268	device_shutdown();
267	269	}