History

printk: Wait for the global console lock when the system is going down

There are reports that the console kthreads block the global console lock when the system is going down, for example, reboot, panic. First part of the solution was to block kthreads in these problematic system states so they stopped handling newly added messages. Second part of the solution is to wait when for the kthreads when they are actively printing. It solves the problem when a message was printed before the system entered the problematic state and the kthreads managed to step in. A busy waiting has to be used because panic() can be called in any context and in an unknown state of the scheduler. There must be a timeout because the kthread might get stuck or sleeping and never release the lock. The timeout 10s is an arbitrary value inspired by the softlockup timeout.

Petr Mladek committed 4 years ago

Commit b87f0231 parent c3230286 files · +47 −0Browse files

Unified · Split

6 files
  1. include/linux/printk.h +5 −0
  2. kernel/panic.c +2 −0
  3. kernel/printk/internal.h +2 −0
  4. kernel/printk/printk.c +4 −0
  5. kernel/printk/printk_safe.c +32 −0
  6. kernel/reboot.c +2 −0
include/linux/printk.h+5−0
@@ -174,6 +174,7 @@ extern void printk_prefer_direct_enter(void); · 173 unmodified lines
174extern void printk_prefer_direct_exit(void);174extern void printk_prefer_direct_exit(void);
175 175
176extern bool pr_flush(int timeout_ms, bool reset_on_progress);176extern bool pr_flush(int timeout_ms, bool reset_on_progress);
177extern void try_block_console_kthreads(int timeout_ms);
177 178
178/*179/*
179 * Please don't use printk_ratelimit(), because it shares ratelimiting state180 * Please don't use printk_ratelimit(), because it shares ratelimiting state
@@ -238,6 +239,10 @@ static inline bool pr_flush(int timeout_ms, bool reset_on_progress) · 58 unmodified lines
238 return true;239 return true;
239}240}
240 241
242static inline void try_block_console_kthreads(int timeout_ms)
243{
244}
245
241static inline int printk_ratelimit(void)246static inline int printk_ratelimit(void)
242{247{
243 return 0;248 return 0;
kernel/panic.c+2−0
@@ -273,6 +273,7 @@ void panic(const char *fmt, ...) · 272 unmodified lines
273 * unfortunately means it may not be hardened to work in a273 * unfortunately means it may not be hardened to work in a
274 * panic situation.274 * panic situation.
275 */275 */
276 try_block_console_kthreads(10000);
276 smp_send_stop();277 smp_send_stop();
277 } else {278 } else {
278 /*279 /*
@@ -280,6 +281,7 @@ void panic(const char *fmt, ...) · 1 unmodified line
280 * kmsg_dump, we will need architecture dependent extra281 * kmsg_dump, we will need architecture dependent extra
281 * works in addition to stopping other CPUs.282 * works in addition to stopping other CPUs.
282 */283 */
284 try_block_console_kthreads(10000);
283 crash_smp_send_stop();285 crash_smp_send_stop();
284 }286 }
285 287
kernel/printk/internal.h+2−0
@@ -20,6 +20,8 @@ enum printk_info_flags { · 19 unmodified lines
20 LOG_CONT = 8, /* text is a fragment of a continuation line */20 LOG_CONT = 8, /* text is a fragment of a continuation line */
21};21};
22 22
23extern bool block_console_kthreads;
24
23__printf(4, 0)25__printf(4, 0)
24int vprintk_store(int facility, int level,26int vprintk_store(int facility, int level,
25 const struct dev_printk_info *dev_info,27 const struct dev_printk_info *dev_info,
kernel/printk/printk.c+4−0
@@ -250,6 +250,9 @@ static atomic_t console_kthreads_active = ATOMIC_INIT(0); · 249 unmodified lines
250#define console_kthread_printing_exit() \250#define console_kthread_printing_exit() \
251 atomic_dec(&console_kthreads_active)251 atomic_dec(&console_kthreads_active)
252 252
253/* Block console kthreads to avoid processing new messages. */
254bool block_console_kthreads;
255
253/*256/*
254 * Helper macros to handle lockdep when locking/unlocking console_sem. We use257 * Helper macros to handle lockdep when locking/unlocking console_sem. We use
255 * macros instead of functions so that _RET_IP_ contains useful information.258 * macros instead of functions so that _RET_IP_ contains useful information.
@@ -3730,6 +3733,7 @@ static bool printer_should_wake(struct console *con, u64 seq) · 3474 unmodified lines
3730 3733
3731 if (con->blocked ||3734 if (con->blocked ||
3732 console_kthreads_atomically_blocked() ||3735 console_kthreads_atomically_blocked() ||
3736 block_console_kthreads ||
3733 system_state > SYSTEM_RUNNING ||3737 system_state > SYSTEM_RUNNING ||
3734 oops_in_progress) {3738 oops_in_progress) {
3735 return false;3739 return false;
kernel/printk/printk_safe.c+32−0
@@ -8,7 +8,9 @@ · 7 unmodified lines
8#include <linux/smp.h>8#include <linux/smp.h>
9#include <linux/cpumask.h>9#include <linux/cpumask.h>
10#include <linux/printk.h>10#include <linux/printk.h>
11#include <linux/console.h>
11#include <linux/kprobes.h>12#include <linux/kprobes.h>
13#include <linux/delay.h>
12 14
13#include "internal.h"15#include "internal.h"
14 16
@@ -50,3 +52,33 @@ asmlinkage int vprintk(const char *fmt, va_list args) · 35 unmodified lines
50 return vprintk_default(fmt, args);52 return vprintk_default(fmt, args);
51}53}
52EXPORT_SYMBOL(vprintk);54EXPORT_SYMBOL(vprintk);
55
56/**
57 * try_block_console_kthreads() - Try to block console kthreads and
58 * make the global console_lock() avaialble
59 *
60 * @timeout_ms: The maximum time (in ms) to wait.
61 *
62 * Prevent console kthreads from starting processing new messages. Wait
63 * until the global console_lock() become available.
64 *
65 * Context: Can be called in any context.
66 */
67void try_block_console_kthreads(int timeout_ms)
68{
69 block_console_kthreads = true;
70
71 /* Do not wait when the console lock could not be safely taken. */
72 if (this_cpu_read(printk_context) || in_nmi())
73 return;
74
75 while (timeout_ms > 0) {
76 if (console_trylock()) {
77 console_unlock();
78 return;
79 }
80
81 udelay(1000);
82 timeout_ms -= 1;
83 }
84}
kernel/reboot.c+2−0
@@ -74,6 +74,7 @@ void kernel_restart_prepare(char *cmd) · 73 unmodified lines
74{74{
75 blocking_notifier_call_chain(&reboot_notifier_list, SYS_RESTART, cmd);75 blocking_notifier_call_chain(&reboot_notifier_list, SYS_RESTART, cmd);
76 system_state = SYSTEM_RESTART;76 system_state = SYSTEM_RESTART;
77 try_block_console_kthreads(10000);
77 usermodehelper_disable();78 usermodehelper_disable();
78 device_shutdown();79 device_shutdown();
79}80}
@@ -262,6 +263,7 @@ static void kernel_shutdown_prepare(enum system_states state) · 182 unmodified lines
262 blocking_notifier_call_chain(&reboot_notifier_list,263 blocking_notifier_call_chain(&reboot_notifier_list,
263 (state == SYSTEM_HALT) ? SYS_HALT : SYS_POWER_OFF, NULL);264 (state == SYSTEM_HALT) ? SYS_HALT : SYS_POWER_OFF, NULL);
264 system_state = state;265 system_state = state;
266 try_block_console_kthreads(10000);
265 usermodehelper_disable();267 usermodehelper_disable();
266 device_shutdown();268 device_shutdown();
267}269}