History

printk: use the lockless ringbuffer

Replace the existing ringbuffer usage and implementation with lockless ringbuffer usage. Even though the new ringbuffer does not require locking, all existing locking is left in place. Therefore, this change is purely replacing the underlining ringbuffer. Changes that exist due to the ringbuffer replacement: - The VMCOREINFO has been updated for the new structures. - Dictionary data is now stored in a separate data buffer from the human-readable messages. The dictionary data buffer is set to the same size as the message buffer. Therefore, the total required memory for both dictionary and message data is 2 * (2 ^ CONFIG_LOG_BUF_SHIFT) for the initial static buffers and 2 * log_buf_len (the kernel parameter) for the dynamic buffers. - Record meta-data is now stored in a separate array of descriptors. This is an additional 72 * (2 ^ (CONFIG_LOG_BUF_SHIFT - 5)) bytes for the static array and 72 * (log_buf_len >> 5) bytes for the dynamic array.

John Ogness committed 6 years ago

Commit 896fbe21 parent 8749efc1 file · +493 −447Browse files

Unified · Split

kernel/printk/printk.c+493−447
@@ -55,6 +55,7 @@ · 54 unmodified lines
55#define CREATE_TRACE_POINTS55#define CREATE_TRACE_POINTS
56#include <trace/events/printk.h>56#include <trace/events/printk.h>
57 57
58#include "printk_ringbuffer.h"
58#include "console_cmdline.h"59#include "console_cmdline.h"
59#include "braille.h"60#include "braille.h"
60#include "internal.h"61#include "internal.h"
@@ -294,30 +295,24 @@ enum con_msg_format_flags { · 233 unmodified lines
294static int console_msg_format = MSG_FORMAT_DEFAULT;295static int console_msg_format = MSG_FORMAT_DEFAULT;
295 296
296/*297/*
297 * The printk log buffer consists of a chain of concatenated variable298 * The printk log buffer consists of a sequenced collection of records, each
298 * length records. Every record starts with a record header, containing299 * containing variable length message and dictionary text. Every record
299 * the overall length of the record.300 * also contains its own meta-data (@info).
300 *301 *
301 * The heads to the first and last entry in the buffer, as well as the302 * Every record meta-data carries the timestamp in microseconds, as well as
302 * sequence numbers of these entries are maintained when messages are303 * the standard userspace syslog level and syslog facility. The usual kernel
303 * stored.304 * messages use LOG_KERN; userspace-injected messages always carry a matching
305 * syslog facility, by default LOG_USER. The origin of every message can be
306 * reliably determined that way.
304 *307 *
305 * If the heads indicate available messages, the length in the header308 * The human readable log message of a record is available in @text, the
306 * tells the start next message. A length == 0 for the next message309 * length of the message text in @text_len. The stored message is not
307 * indicates a wrap-around to the beginning of the buffer.310 * terminated.
308 *311 *
309 * Every record carries the monotonic timestamp in microseconds, as well as312 * Optionally, a record can carry a dictionary of properties (key/value
310 * the standard userspace syslog level and syslog facility. The usual313 * pairs), to provide userspace with a machine-readable message context. The
311 * kernel messages use LOG_KERN; userspace-injected messages always carry314 * length of the dictionary is available in @dict_len. The dictionary is not
312 * a matching syslog facility, by default LOG_USER. The origin of every315 * terminated.
313 * message can be reliably determined that way.
314 *
315 * The human readable log message directly follows the message header. The
316 * length of the message text is stored in the header, the stored message
317 * is not terminated.
318 *
319 * Optionally, a message can carry a dictionary of properties (key/value pairs),
320 * to provide userspace with a machine-readable message context.
321 *316 *
322 * Examples for well-defined, commonly used property names are:317 * Examples for well-defined, commonly used property names are:
323 * DEVICE=b12:8 device identifier318 * DEVICE=b12:8 device identifier
@@ -331,21 +326,19 @@ static int console_msg_format = MSG_FORMAT_DEFAULT; · 7 unmodified lines
331 * follows directly after a '=' character. Every property is terminated by326 * follows directly after a '=' character. Every property is terminated by
332 * a '\0' character. The last property is not terminated.327 * a '\0' character. The last property is not terminated.
333 *328 *
334 * Example of a message structure:329 * Example of record values:
335 * 0000 ff 8f 00 00 00 00 00 00 monotonic time in nsec330 * record.text_buf = "it's a line" (unterminated)
336 * 0008 34 00 record is 52 bytes long331 * record.dict_buf = "DEVICE=b8:2\0DRIVER=bug" (unterminated)
337 * 000a 0b 00 text is 11 bytes long332 * record.info.seq = 56
338 * 000c 1f 00 dictionary is 23 bytes long333 * record.info.ts_nsec = 36863
339 * 000e 03 00 LOG_KERN (facility) LOG_ERR (level)334 * record.info.text_len = 11
340 * 0010 69 74 27 73 20 61 20 6c "it's a l"335 * record.info.dict_len = 22
341 * 69 6e 65 "ine"336 * record.info.facility = 0 (LOG_KERN)
342 * 001b 44 45 56 49 43 "DEVIC"337 * record.info.flags = 0
343 * 45 3d 62 38 3a 32 00 44 "E=b8:2\0D"338 * record.info.level = 3 (LOG_ERR)
344 * 52 49 56 45 52 3d 62 75 "RIVER=bu"339 * record.info.caller_id = 299 (task 299)
345 * 67 "g"340 *
346 * 0032 00 00 00 padding to next message header341 * The 'struct printk_info' buffer must never be directly exported to
347 *
348 * The 'struct printk_log' buffer header must never be directly exported to
349 * userspace, it is a kernel-private implementation detail that might342 * userspace, it is a kernel-private implementation detail that might
350 * need to be changed in the future, when the requirements change.343 * need to be changed in the future, when the requirements change.
351 *344 *
@@ -365,23 +358,6 @@ enum log_flags { · 13 unmodified lines
365 LOG_CONT = 8, /* text is a fragment of a continuation line */358 LOG_CONT = 8, /* text is a fragment of a continuation line */
366};359};
367 360
368struct printk_log {
369 u64 ts_nsec; /* timestamp in nanoseconds */
370 u16 len; /* length of entire record */
371 u16 text_len; /* length of text buffer */
372 u16 dict_len; /* length of dictionary buffer */
373 u8 facility; /* syslog facility */
374 u8 flags:5; /* internal record flags */
375 u8 level:3; /* syslog level */
376#ifdef CONFIG_PRINTK_CALLER
377 u32 caller_id; /* thread id or processor id */
378#endif
379}
380#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
381__packed __aligned(4)
382#endif
383;
384
385/*361/*
386 * The logbuf_lock protects kmsg buffer, indices, counters. This can be taken362 * The logbuf_lock protects kmsg buffer, indices, counters. This can be taken
387 * within the scheduler's rq lock. It must be released before calling363 * within the scheduler's rq lock. It must be released before calling
@@ -421,26 +397,16 @@ DEFINE_RAW_SPINLOCK(logbuf_lock); · 33 unmodified lines
421DECLARE_WAIT_QUEUE_HEAD(log_wait);397DECLARE_WAIT_QUEUE_HEAD(log_wait);
422/* the next printk record to read by syslog(READ) or /proc/kmsg */398/* the next printk record to read by syslog(READ) or /proc/kmsg */
423static u64 syslog_seq;399static u64 syslog_seq;
424static u32 syslog_idx;
425static size_t syslog_partial;400static size_t syslog_partial;
426static bool syslog_time;401static bool syslog_time;
427 402
428/* index and sequence number of the first record stored in the buffer */
429static u64 log_first_seq;
430static u32 log_first_idx;
431
432/* index and sequence number of the next record to store in the buffer */
433static u64 log_next_seq;
434static u32 log_next_idx;
435
436/* the next printk record to write to the console */403/* the next printk record to write to the console */
437static u64 console_seq;404static u64 console_seq;
438static u32 console_idx;
439static u64 exclusive_console_stop_seq;405static u64 exclusive_console_stop_seq;
406static unsigned long console_dropped;
440 407
441/* the next printk record to read after the last 'clear' command */408/* the next printk record to read after the last 'clear' command */
442static u64 clear_seq;409static u64 clear_seq;
443static u32 clear_idx;
444 410
445#ifdef CONFIG_PRINTK_CALLER411#ifdef CONFIG_PRINTK_CALLER
446#define PREFIX_MAX 48412#define PREFIX_MAX 48
@@ -453,13 +419,31 @@ static u32 clear_idx; · 6 unmodified lines
453#define LOG_FACILITY(v) ((v) >> 3 & 0xff)419#define LOG_FACILITY(v) ((v) >> 3 & 0xff)
454 420
455/* record buffer */421/* record buffer */
456#define LOG_ALIGN __alignof__(struct printk_log)422#define LOG_ALIGN __alignof__(unsigned long)
457#define __LOG_BUF_LEN (1 << CONFIG_LOG_BUF_SHIFT)423#define __LOG_BUF_LEN (1 << CONFIG_LOG_BUF_SHIFT)
458#define LOG_BUF_LEN_MAX (u32)(1 << 31)424#define LOG_BUF_LEN_MAX (u32)(1 << 31)
459static char __log_buf[__LOG_BUF_LEN] __aligned(LOG_ALIGN);425static char __log_buf[__LOG_BUF_LEN] __aligned(LOG_ALIGN);
460static char *log_buf = __log_buf;426static char *log_buf = __log_buf;
461static u32 log_buf_len = __LOG_BUF_LEN;427static u32 log_buf_len = __LOG_BUF_LEN;
462 428
429/*
430 * Define the average message size. This only affects the number of
431 * descriptors that will be available. Underestimating is better than
432 * overestimating (too many available descriptors is better than not enough).
433 * The dictionary buffer will be the same size as the text buffer.
434 */
435#define PRB_AVGBITS 5 /* 32 character average length */
436
437#if CONFIG_LOG_BUF_SHIFT <= PRB_AVGBITS
438#error CONFIG_LOG_BUF_SHIFT value too small.
439#endif
440_DEFINE_PRINTKRB(printk_rb_static, CONFIG_LOG_BUF_SHIFT - PRB_AVGBITS,
441 PRB_AVGBITS, PRB_AVGBITS, &__log_buf[0]);
442
443static struct printk_ringbuffer printk_rb_dynamic;
444
445static struct printk_ringbuffer *prb = &printk_rb_static;
446
463/*447/*
464 * We cannot access per-CPU data (e.g. per-CPU flush irq_work) before448 * We cannot access per-CPU data (e.g. per-CPU flush irq_work) before
465 * per_cpu_areas are initialised. This variable is set to true when449 * per_cpu_areas are initialised. This variable is set to true when
@@ -484,108 +468,6 @@ u32 log_buf_len_get(void) · 18 unmodified lines
484 return log_buf_len;468 return log_buf_len;
485}469}
486 470
487/* human readable text of the record */
488static char *log_text(const struct printk_log *msg)
489{
490 return (char *)msg + sizeof(struct printk_log);
491}
492
493/* optional key/value pair dictionary attached to the record */
494static char *log_dict(const struct printk_log *msg)
495{
496 return (char *)msg + sizeof(struct printk_log) + msg->text_len;
497}
498
499/* get record by index; idx must point to valid msg */
500static struct printk_log *log_from_idx(u32 idx)
501{
502 struct printk_log *msg = (struct printk_log *)(log_buf + idx);
503
504 /*
505 * A length == 0 record is the end of buffer marker. Wrap around and
506 * read the message at the start of the buffer.
507 */
508 if (!msg->len)
509 return (struct printk_log *)log_buf;
510 return msg;
511}
512
513/* get next record; idx must point to valid msg */
514static u32 log_next(u32 idx)
515{
516 struct printk_log *msg = (struct printk_log *)(log_buf + idx);
517
518 /* length == 0 indicates the end of the buffer; wrap */
519 /*
520 * A length == 0 record is the end of buffer marker. Wrap around and
521 * read the message at the start of the buffer as *this* one, and
522 * return the one after that.
523 */
524 if (!msg->len) {
525 msg = (struct printk_log *)log_buf;
526 return msg->len;
527 }
528 return idx + msg->len;
529}
530
531/*
532 * Check whether there is enough free space for the given message.
533 *
534 * The same values of first_idx and next_idx mean that the buffer
535 * is either empty or full.
536 *
537 * If the buffer is empty, we must respect the position of the indexes.
538 * They cannot be reset to the beginning of the buffer.
539 */
540static int logbuf_has_space(u32 msg_size, bool empty)
541{
542 u32 free;
543
544 if (log_next_idx > log_first_idx || empty)
545 free = max(log_buf_len - log_next_idx, log_first_idx);
546 else
547 free = log_first_idx - log_next_idx;
548
549 /*
550 * We need space also for an empty header that signalizes wrapping
551 * of the buffer.
552 */
553 return free >= msg_size + sizeof(struct printk_log);
554}
555
556static int log_make_free_space(u32 msg_size)
557{
558 while (log_first_seq < log_next_seq &&
559 !logbuf_has_space(msg_size, false)) {
560 /* drop old messages until we have enough contiguous space */
561 log_first_idx = log_next(log_first_idx);
562 log_first_seq++;
563 }
564
565 if (clear_seq < log_first_seq) {
566 clear_seq = log_first_seq;
567 clear_idx = log_first_idx;
568 }
569
570 /* sequence numbers are equal, so the log buffer is empty */
571 if (logbuf_has_space(msg_size, log_first_seq == log_next_seq))
572 return 0;
573
574 return -ENOMEM;
575}
576
577/* compute the message size including the padding bytes */
578static u32 msg_used_size(u16 text_len, u16 dict_len, u32 *pad_len)
579{
580 u32 size;
581
582 size = sizeof(struct printk_log) + text_len + dict_len;
583 *pad_len = (-size) & (LOG_ALIGN - 1);
584 size += *pad_len;
585
586 return size;
587}
588
589/*471/*
590 * Define how much of the log buffer we could take at maximum. The value472 * Define how much of the log buffer we could take at maximum. The value
591 * must be greater than two. Note that only half of the buffer is available473 * must be greater than two. Note that only half of the buffer is available
@@ -594,22 +476,23 @@ static u32 msg_used_size(u16 text_len, u16 dict_len, u32 *pad_len) · 2 unmodified lines
594#define MAX_LOG_TAKE_PART 4476#define MAX_LOG_TAKE_PART 4
595static const char trunc_msg[] = "<truncated>";477static const char trunc_msg[] = "<truncated>";
596 478
597static u32 truncate_msg(u16 *text_len, u16 *trunc_msg_len,479static void truncate_msg(u16 *text_len, u16 *trunc_msg_len)
598 u16 *dict_len, u32 *pad_len)
599{480{
600 /*481 /*
601 * The message should not take the whole buffer. Otherwise, it might482 * The message should not take the whole buffer. Otherwise, it might
602 * get removed too soon.483 * get removed too soon.
603 */484 */
604 u32 max_text_len = log_buf_len / MAX_LOG_TAKE_PART;485 u32 max_text_len = log_buf_len / MAX_LOG_TAKE_PART;
486
605 if (*text_len > max_text_len)487 if (*text_len > max_text_len)
606 *text_len = max_text_len;488 *text_len = max_text_len;
607 /* enable the warning message */489
490 /* enable the warning message (if there is room) */
608 *trunc_msg_len = strlen(trunc_msg);491 *trunc_msg_len = strlen(trunc_msg);
609 /* disable the "dict" completely */492 if (*text_len >= *trunc_msg_len)
610 *dict_len = 0;493 *text_len -= *trunc_msg_len;
611 /* compute the size again, count also the warning message */494 else
612 return msg_used_size(*text_len + *trunc_msg_len, 0, pad_len);495 *trunc_msg_len = 0;
613}496}
614 497
615/* insert record into the buffer, discard old ones, update heads */498/* insert record into the buffer, discard old ones, update heads */
@@ -618,60 +501,40 @@ static int log_store(u32 caller_id, int facility, int level, · 2 unmodified lines
618 const char *dict, u16 dict_len,501 const char *dict, u16 dict_len,
619 const char *text, u16 text_len)502 const char *text, u16 text_len)
620{503{
621 struct printk_log *msg;504 struct prb_reserved_entry e;
622 u32 size, pad_len;505 struct printk_record r;
623 u16 trunc_msg_len = 0;506 u16 trunc_msg_len = 0;
624 507
625 /* number of '\0' padding bytes to next message */508 prb_rec_init_wr(&r, text_len, dict_len);
626 size = msg_used_size(text_len, dict_len, &pad_len);
627 509
628 if (log_make_free_space(size)) {510 if (!prb_reserve(&e, prb, &r)) {
629 /* truncate the message if it is too long for empty buffer */511 /* truncate the message if it is too long for empty buffer */
630 size = truncate_msg(&text_len, &trunc_msg_len,512 truncate_msg(&text_len, &trunc_msg_len);
631 &dict_len, &pad_len);513 prb_rec_init_wr(&r, text_len + trunc_msg_len, dict_len);
632 /* survive when the log buffer is too small for trunc_msg */514 /* survive when the log buffer is too small for trunc_msg */
633 if (log_make_free_space(size))515 if (!prb_reserve(&e, prb, &r))
634 return 0;516 return 0;
635 }517 }
636 518
637 if (log_next_idx + size + sizeof(struct printk_log) > log_buf_len) {
638 /*
639 * This message + an additional empty header does not fit
640 * at the end of the buffer. Add an empty header with len == 0
641 * to signify a wrap around.
642 */
643 memset(log_buf + log_next_idx, 0, sizeof(struct printk_log));
644 log_next_idx = 0;
645 }
646
647 /* fill message */519 /* fill message */
648 msg = (struct printk_log *)(log_buf + log_next_idx);520 memcpy(&r.text_buf[0], text, text_len);
649 memcpy(log_text(msg), text, text_len);521 if (trunc_msg_len)
650 msg->text_len = text_len;522 memcpy(&r.text_buf[text_len], trunc_msg, trunc_msg_len);
651 if (trunc_msg_len) {523 if (r.dict_buf)
652 memcpy(log_text(msg) + text_len, trunc_msg, trunc_msg_len);524 memcpy(&r.dict_buf[0], dict, dict_len);
653 msg->text_len += trunc_msg_len;525 r.info->facility = facility;
654 }526 r.info->level = level & 7;
655 memcpy(log_dict(msg), dict, dict_len);527 r.info->flags = flags & 0x1f;
656 msg->dict_len = dict_len;
657 msg->facility = facility;
658 msg->level = level & 7;
659 msg->flags = flags & 0x1f;
660 if (ts_nsec > 0)528 if (ts_nsec > 0)
661 msg->ts_nsec = ts_nsec;529 r.info->ts_nsec = ts_nsec;
662 else530 else
663 msg->ts_nsec = local_clock();531 r.info->ts_nsec = local_clock();
664#ifdef CONFIG_PRINTK_CALLER532 r.info->caller_id = caller_id;
665 msg->caller_id = caller_id;
666#endif
667 memset(log_dict(msg) + dict_len, 0, pad_len);
668 msg->len = size;
669 533
670 /* insert message */534 /* insert message */
671 log_next_idx += msg->len;535 prb_commit(&e);
672 log_next_seq++;
673 536
674 return msg->text_len;537 return (text_len + trunc_msg_len);
675}538}
676 539
677int dmesg_restrict = IS_ENABLED(CONFIG_SECURITY_DMESG_RESTRICT);540int dmesg_restrict = IS_ENABLED(CONFIG_SECURITY_DMESG_RESTRICT);
@@ -723,13 +586,13 @@ static void append_char(char **pp, char *e, char c) · 45 unmodified lines
723 *(*pp)++ = c;586 *(*pp)++ = c;
724}587}
725 588
726static ssize_t msg_print_ext_header(char *buf, size_t size,589static ssize_t info_print_ext_header(char *buf, size_t size,
727 struct printk_log *msg, u64 seq)590 struct printk_info *info)
728{591{
729 u64 ts_usec = msg->ts_nsec;592 u64 ts_usec = info->ts_nsec;
730 char caller[20];593 char caller[20];
731#ifdef CONFIG_PRINTK_CALLER594#ifdef CONFIG_PRINTK_CALLER
732 u32 id = msg->caller_id;595 u32 id = info->caller_id;
733 596
734 snprintf(caller, sizeof(caller), ",caller=%c%u",597 snprintf(caller, sizeof(caller), ",caller=%c%u",
735 id & 0x80000000 ? 'C' : 'T', id & ~0x80000000);598 id & 0x80000000 ? 'C' : 'T', id & ~0x80000000);
@@ -740,8 +603,8 @@ static ssize_t msg_print_ext_header(char *buf, size_t size, · 4 unmodified lines
740 do_div(ts_usec, 1000);603 do_div(ts_usec, 1000);
741 604
742 return scnprintf(buf, size, "%u,%llu,%llu,%c%s;",605 return scnprintf(buf, size, "%u,%llu,%llu,%c%s;",
743 (msg->facility << 3) | msg->level, seq, ts_usec,606 (info->facility << 3) | info->level, info->seq,
744 msg->flags & LOG_CONT ? 'c' : '-', caller);607 ts_usec, info->flags & LOG_CONT ? 'c' : '-', caller);
745}608}
746 609
747static ssize_t msg_print_ext_body(char *buf, size_t size,610static ssize_t msg_print_ext_body(char *buf, size_t size,
@@ -795,10 +658,14 @@ static ssize_t msg_print_ext_body(char *buf, size_t size, · 47 unmodified lines
795/* /dev/kmsg - userspace message inject/listen interface */658/* /dev/kmsg - userspace message inject/listen interface */
796struct devkmsg_user {659struct devkmsg_user {
797 u64 seq;660 u64 seq;
798 u32 idx;
799 struct ratelimit_state rs;661 struct ratelimit_state rs;
800 struct mutex lock;662 struct mutex lock;
801 char buf[CONSOLE_EXT_LOG_MAX];663 char buf[CONSOLE_EXT_LOG_MAX];
664
665 struct printk_info info;
666 char text_buf[CONSOLE_EXT_LOG_MAX];
667 char dict_buf[CONSOLE_EXT_LOG_MAX];
668 struct printk_record record;
802};669};
803 670
804static __printf(3, 4) __cold671static __printf(3, 4) __cold
@@ -881,7 +748,7 @@ static ssize_t devkmsg_read(struct file *file, char __user *buf, · 76 unmodified lines
881 size_t count, loff_t *ppos)748 size_t count, loff_t *ppos)
882{749{
883 struct devkmsg_user *user = file->private_data;750 struct devkmsg_user *user = file->private_data;
884 struct printk_log *msg;751 struct printk_record *r = &user->record;
885 size_t len;752 size_t len;
886 ssize_t ret;753 ssize_t ret;
887 754
@@ -893,7 +760,7 @@ static ssize_t devkmsg_read(struct file *file, char __user *buf, · 5 unmodified lines
893 return ret;760 return ret;
894 761
895 logbuf_lock_irq();762 logbuf_lock_irq();
896 while (user->seq == log_next_seq) {763 if (!prb_read_valid(prb, user->seq, r)) {
897 if (file->f_flags & O_NONBLOCK) {764 if (file->f_flags & O_NONBLOCK) {
898 ret = -EAGAIN;765 ret = -EAGAIN;
899 logbuf_unlock_irq();766 logbuf_unlock_irq();
@@ -902,30 +769,26 @@ static ssize_t devkmsg_read(struct file *file, char __user *buf, · 2 unmodified lines
902 769
903 logbuf_unlock_irq();770 logbuf_unlock_irq();
904 ret = wait_event_interruptible(log_wait,771 ret = wait_event_interruptible(log_wait,
905 user->seq != log_next_seq);772 prb_read_valid(prb, user->seq, r));
906 if (ret)773 if (ret)
907 goto out;774 goto out;
908 logbuf_lock_irq();775 logbuf_lock_irq();
909 }776 }
910 777
911 if (user->seq < log_first_seq) {778 if (user->seq < prb_first_valid_seq(prb)) {
912 /* our last seen message is gone, return error and reset */779 /* our last seen message is gone, return error and reset */
913 user->idx = log_first_idx;780 user->seq = prb_first_valid_seq(prb);
914 user->seq = log_first_seq;
915 ret = -EPIPE;781 ret = -EPIPE;
916 logbuf_unlock_irq();782 logbuf_unlock_irq();
917 goto out;783 goto out;
918 }784 }
919 785
920 msg = log_from_idx(user->idx);786 len = info_print_ext_header(user->buf, sizeof(user->buf), r->info);
921 len = msg_print_ext_header(user->buf, sizeof(user->buf),
922 msg, user->seq);
923 len += msg_print_ext_body(user->buf + len, sizeof(user->buf) - len,787 len += msg_print_ext_body(user->buf + len, sizeof(user->buf) - len,
924 log_dict(msg), msg->dict_len,788 &r->dict_buf[0], r->info->dict_len,
925 log_text(msg), msg->text_len);789 &r->text_buf[0], r->info->text_len);
926 790
927 user->idx = log_next(user->idx);791 user->seq = r->info->seq + 1;
928 user->seq++;
929 logbuf_unlock_irq();792 logbuf_unlock_irq();
930 793
931 if (len > count) {794 if (len > count) {
@@ -957,8 +820,7 @@ static loff_t devkmsg_llseek(struct file *file, loff_t offset, int whence) · 25 unmodified lines
957 switch (whence) {820 switch (whence) {
958 case SEEK_SET:821 case SEEK_SET:
959 /* the first record */822 /* the first record */
960 user->idx = log_first_idx;823 user->seq = prb_first_valid_seq(prb);
961 user->seq = log_first_seq;
962 break;824 break;
963 case SEEK_DATA:825 case SEEK_DATA:
964 /*826 /*
@@ -966,13 +828,11 @@ static loff_t devkmsg_llseek(struct file *file, loff_t offset, int whence) · 1 unmodified line
966 * like issued by 'dmesg -c'. Reading /dev/kmsg itself828 * like issued by 'dmesg -c'. Reading /dev/kmsg itself
967 * changes no global state, and does not clear anything.829 * changes no global state, and does not clear anything.
968 */830 */
969 user->idx = clear_idx;
970 user->seq = clear_seq;831 user->seq = clear_seq;
971 break;832 break;
972 case SEEK_END:833 case SEEK_END:
973 /* after the last record */834 /* after the last record */
974 user->idx = log_next_idx;835 user->seq = prb_next_seq(prb);
975 user->seq = log_next_seq;
976 break;836 break;
977 default:837 default:
978 ret = -EINVAL;838 ret = -EINVAL;
@@ -992,9 +852,9 @@ static __poll_t devkmsg_poll(struct file *file, poll_table *wait) · 13 unmodified lines
992 poll_wait(file, &log_wait, wait);852 poll_wait(file, &log_wait, wait);
993 853
994 logbuf_lock_irq();854 logbuf_lock_irq();
995 if (user->seq < log_next_seq) {855 if (prb_read_valid(prb, user->seq, NULL)) {
996 /* return error when data has vanished underneath us */856 /* return error when data has vanished underneath us */
997 if (user->seq < log_first_seq)857 if (user->seq < prb_first_valid_seq(prb))
998 ret = EPOLLIN|EPOLLRDNORM|EPOLLERR|EPOLLPRI;858 ret = EPOLLIN|EPOLLRDNORM|EPOLLERR|EPOLLPRI;
999 else859 else
1000 ret = EPOLLIN|EPOLLRDNORM;860 ret = EPOLLIN|EPOLLRDNORM;
@@ -1029,9 +889,12 @@ static int devkmsg_open(struct inode *inode, struct file *file) · 28 unmodified lines
1029 889
1030 mutex_init(&user->lock);890 mutex_init(&user->lock);
1031 891
892 prb_rec_init_rd(&user->record, &user->info,
893 &user->text_buf[0], sizeof(user->text_buf),
894 &user->dict_buf[0], sizeof(user->dict_buf));
895
1032 logbuf_lock_irq();896 logbuf_lock_irq();
1033 user->idx = log_first_idx;897 user->seq = prb_first_valid_seq(prb);
1034 user->seq = log_first_seq;
1035 logbuf_unlock_irq();898 logbuf_unlock_irq();
1036 899
1037 file->private_data = user;900 file->private_data = user;
@@ -1072,23 +935,52 @@ const struct file_operations kmsg_fops = { · 34 unmodified lines
1072 */935 */
1073void log_buf_vmcoreinfo_setup(void)936void log_buf_vmcoreinfo_setup(void)
1074{937{
1075 VMCOREINFO_SYMBOL(log_buf);938 VMCOREINFO_SYMBOL(prb);
1076 VMCOREINFO_SYMBOL(log_buf_len);939 VMCOREINFO_SYMBOL(printk_rb_static);
1077 VMCOREINFO_SYMBOL(log_first_idx);940 VMCOREINFO_SYMBOL(clear_seq);
1078 VMCOREINFO_SYMBOL(clear_idx);941
1079 VMCOREINFO_SYMBOL(log_next_idx);
1080 /*942 /*
1081 * Export struct printk_log size and field offsets. User space tools can943 * Export struct size and field offsets. User space tools can
1082 * parse it and detect any changes to structure down the line.944 * parse it and detect any changes to structure down the line.
1083 */945 */
1084 VMCOREINFO_STRUCT_SIZE(printk_log);946
1085 VMCOREINFO_OFFSET(printk_log, ts_nsec);947 VMCOREINFO_STRUCT_SIZE(printk_ringbuffer);
1086 VMCOREINFO_OFFSET(printk_log, len);948 VMCOREINFO_OFFSET(printk_ringbuffer, desc_ring);
1087 VMCOREINFO_OFFSET(printk_log, text_len);949 VMCOREINFO_OFFSET(printk_ringbuffer, text_data_ring);
1088 VMCOREINFO_OFFSET(printk_log, dict_len);950 VMCOREINFO_OFFSET(printk_ringbuffer, dict_data_ring);
1089#ifdef CONFIG_PRINTK_CALLER951 VMCOREINFO_OFFSET(printk_ringbuffer, fail);
1090 VMCOREINFO_OFFSET(printk_log, caller_id);952
1091#endif953 VMCOREINFO_STRUCT_SIZE(prb_desc_ring);
954 VMCOREINFO_OFFSET(prb_desc_ring, count_bits);
955 VMCOREINFO_OFFSET(prb_desc_ring, descs);
956 VMCOREINFO_OFFSET(prb_desc_ring, head_id);
957 VMCOREINFO_OFFSET(prb_desc_ring, tail_id);
958
959 VMCOREINFO_STRUCT_SIZE(prb_desc);
960 VMCOREINFO_OFFSET(prb_desc, info);
961 VMCOREINFO_OFFSET(prb_desc, state_var);
962 VMCOREINFO_OFFSET(prb_desc, text_blk_lpos);
963 VMCOREINFO_OFFSET(prb_desc, dict_blk_lpos);
964
965 VMCOREINFO_STRUCT_SIZE(prb_data_blk_lpos);
966 VMCOREINFO_OFFSET(prb_data_blk_lpos, begin);
967 VMCOREINFO_OFFSET(prb_data_blk_lpos, next);
968
969 VMCOREINFO_STRUCT_SIZE(printk_info);
970 VMCOREINFO_OFFSET(printk_info, seq);
971 VMCOREINFO_OFFSET(printk_info, ts_nsec);
972 VMCOREINFO_OFFSET(printk_info, text_len);
973 VMCOREINFO_OFFSET(printk_info, dict_len);
974 VMCOREINFO_OFFSET(printk_info, caller_id);
975
976 VMCOREINFO_STRUCT_SIZE(prb_data_ring);
977 VMCOREINFO_OFFSET(prb_data_ring, size_bits);
978 VMCOREINFO_OFFSET(prb_data_ring, data);
979 VMCOREINFO_OFFSET(prb_data_ring, head_lpos);
980 VMCOREINFO_OFFSET(prb_data_ring, tail_lpos);
981
982 VMCOREINFO_SIZE(atomic_long_t);
983 VMCOREINFO_TYPE_OFFSET(atomic_long_t, counter);
1092}984}
1093#endif985#endif
1094 986
@@ -1166,11 +1058,48 @@ static void __init set_percpu_data_ready(void) · 71 unmodified lines
1166 __printk_percpu_data_ready = true;1058 __printk_percpu_data_ready = true;
1167}1059}
1168 1060
1061static unsigned int __init add_to_rb(struct printk_ringbuffer *rb,
1062 struct printk_record *r)
1063{
1064 struct prb_reserved_entry e;
1065 struct printk_record dest_r;
1066
1067 prb_rec_init_wr(&dest_r, r->info->text_len, r->info->dict_len);
1068
1069 if (!prb_reserve(&e, rb, &dest_r))
1070 return 0;
1071
1072 memcpy(&dest_r.text_buf[0], &r->text_buf[0], dest_r.text_buf_size);
1073 if (dest_r.dict_buf) {
1074 memcpy(&dest_r.dict_buf[0], &r->dict_buf[0],
1075 dest_r.dict_buf_size);
1076 }
1077 dest_r.info->facility = r->info->facility;
1078 dest_r.info->level = r->info->level;
1079 dest_r.info->flags = r->info->flags;
1080 dest_r.info->ts_nsec = r->info->ts_nsec;
1081 dest_r.info->caller_id = r->info->caller_id;
1082
1083 prb_commit(&e);
1084
1085 return prb_record_text_space(&e);
1086}
1087
1088static char setup_text_buf[CONSOLE_EXT_LOG_MAX] __initdata;
1089static char setup_dict_buf[CONSOLE_EXT_LOG_MAX] __initdata;
1090
1169void __init setup_log_buf(int early)1091void __init setup_log_buf(int early)
1170{1092{
1093 unsigned int new_descs_count;
1094 struct prb_desc *new_descs;
1095 struct printk_info info;
1096 struct printk_record r;
1097 size_t new_descs_size;
1171 unsigned long flags;1098 unsigned long flags;
1099 char *new_dict_buf;
1172 char *new_log_buf;1100 char *new_log_buf;
1173 unsigned int free;1101 unsigned int free;
1102 u64 seq;
1174 1103
1175 /*1104 /*
1176 * Some archs call setup_log_buf() multiple times - first is very1105 * Some archs call setup_log_buf() multiple times - first is very
@@ -1189,21 +1118,70 @@ void __init setup_log_buf(int early) · 12 unmodified lines
1189 if (!new_log_buf_len)1118 if (!new_log_buf_len)
1190 return;1119 return;
1191 1120
1121 new_descs_count = new_log_buf_len >> PRB_AVGBITS;
1122 if (new_descs_count == 0) {
1123 pr_err("new_log_buf_len: %lu too small\n", new_log_buf_len);
1124 return;
1125 }
1126
1192 new_log_buf = memblock_alloc(new_log_buf_len, LOG_ALIGN);1127 new_log_buf = memblock_alloc(new_log_buf_len, LOG_ALIGN);
1193 if (unlikely(!new_log_buf)) {1128 if (unlikely(!new_log_buf)) {
1194 pr_err("log_buf_len: %lu bytes not available\n",1129 pr_err("log_buf_len: %lu text bytes not available\n",
1195 new_log_buf_len);1130 new_log_buf_len);
1196 return;1131 return;
1197 }1132 }
1198 1133
1134 new_dict_buf = memblock_alloc(new_log_buf_len, LOG_ALIGN);
1135 if (unlikely(!new_dict_buf)) {
1136 pr_err("log_buf_len: %lu dict bytes not available\n",
1137 new_log_buf_len);
1138 memblock_free(__pa(new_log_buf), new_log_buf_len);
1139 return;
1140 }
1141
1142 new_descs_size = new_descs_count * sizeof(struct prb_desc);
1143 new_descs = memblock_alloc(new_descs_size, LOG_ALIGN);
1144 if (unlikely(!new_descs)) {
1145 pr_err("log_buf_len: %zu desc bytes not available\n",
1146 new_descs_size);
1147 memblock_free(__pa(new_dict_buf), new_log_buf_len);
1148 memblock_free(__pa(new_log_buf), new_log_buf_len);
1149 return;
1150 }
1151
1152 prb_rec_init_rd(&r, &info,
1153 &setup_text_buf[0], sizeof(setup_text_buf),
1154 &setup_dict_buf[0], sizeof(setup_dict_buf));
1155
1156 prb_init(&printk_rb_dynamic,
1157 new_log_buf, ilog2(new_log_buf_len),
1158 new_dict_buf, ilog2(new_log_buf_len),
1159 new_descs, ilog2(new_descs_count));
1160
1199 logbuf_lock_irqsave(flags);1161 logbuf_lock_irqsave(flags);
1162
1200 log_buf_len = new_log_buf_len;1163 log_buf_len = new_log_buf_len;
1201 log_buf = new_log_buf;1164 log_buf = new_log_buf;
1202 new_log_buf_len = 0;1165 new_log_buf_len = 0;
1203 free = __LOG_BUF_LEN - log_next_idx;1166
1204 memcpy(log_buf, __log_buf, __LOG_BUF_LEN);1167 free = __LOG_BUF_LEN;
1168 prb_for_each_record(0, &printk_rb_static, seq, &r)
1169 free -= add_to_rb(&printk_rb_dynamic, &r);
1170
1171 /*
1172 * This is early enough that everything is still running on the
1173 * boot CPU and interrupts are disabled. So no new messages will
1174 * appear during the transition to the dynamic buffer.
1175 */
1176 prb = &printk_rb_dynamic;
1177
1205 logbuf_unlock_irqrestore(flags);1178 logbuf_unlock_irqrestore(flags);
1206 1179
1180 if (seq != prb_next_seq(&printk_rb_static)) {
1181 pr_err("dropped %llu messages\n",
1182 prb_next_seq(&printk_rb_static) - seq);
1183 }
1184
1207 pr_info("log_buf_len: %u bytes\n", log_buf_len);1185 pr_info("log_buf_len: %u bytes\n", log_buf_len);
1208 pr_info("early log buf free: %u(%u%%)\n",1186 pr_info("early log buf free: %u(%u%%)\n",
1209 free, (free * 100) / __LOG_BUF_LEN);1187 free, (free * 100) / __LOG_BUF_LEN);
@@ -1313,18 +1291,18 @@ static size_t print_caller(u32 id, char *buf) · 103 unmodified lines
1313#define print_caller(id, buf) 01291#define print_caller(id, buf) 0
1314#endif1292#endif
1315 1293
1316static size_t print_prefix(const struct printk_log *msg, bool syslog,1294static size_t info_print_prefix(const struct printk_info *info, bool syslog,
1317 bool time, char *buf)1295 bool time, char *buf)
1318{1296{
1319 size_t len = 0;1297 size_t len = 0;
1320 1298
1321 if (syslog)1299 if (syslog)
1322 len = print_syslog((msg->facility << 3) | msg->level, buf);1300 len = print_syslog((info->facility << 3) | info->level, buf);
1323 1301
1324 if (time)1302 if (time)
1325 len += print_time(msg->ts_nsec, buf + len);1303 len += print_time(info->ts_nsec, buf + len);
1326 1304
1327 len += print_caller(msg->caller_id, buf + len);1305 len += print_caller(info->caller_id, buf + len);
1328 1306
1329 if (IS_ENABLED(CONFIG_PRINTK_CALLER) || time) {1307 if (IS_ENABLED(CONFIG_PRINTK_CALLER) || time) {
1330 buf[len++] = ' ';1308 buf[len++] = ' ';
@@ -1334,72 +1312,143 @@ static size_t print_prefix(const struct printk_log *msg, bool syslog, · 3 unmodified lines
1334 return len;1312 return len;
1335}1313}
1336 1314
1337static size_t msg_print_text(const struct printk_log *msg, bool syslog,1315/*
1338 bool time, char *buf, size_t size)1316 * Prepare the record for printing. The text is shifted within the given
1317 * buffer to avoid a need for another one. The following operations are
1318 * done:
1319 *
1320 * - Add prefix for each line.
1321 * - Add the trailing newline that has been removed in vprintk_store().
1322 * - Drop truncated lines that do not longer fit into the buffer.
1323 *
1324 * Return: The length of the updated/prepared text, including the added
1325 * prefixes and the newline. The dropped line(s) are not counted.
1326 */
1327static size_t record_print_text(struct printk_record *r, bool syslog,
1328 bool time)
1339{1329{
1340 const char *text = log_text(msg);1330 size_t text_len = r->info->text_len;
1341 size_t text_size = msg->text_len;1331 size_t buf_size = r->text_buf_size;
1342 size_t len = 0;1332 char *text = r->text_buf;
1343 char prefix[PREFIX_MAX];1333 char prefix[PREFIX_MAX];
1344 const size_t prefix_len = print_prefix(msg, syslog, time, prefix);1334 bool truncated = false;
1335 size_t prefix_len;
1336 size_t line_len;
1337 size_t len = 0;
1338 char *next;
1345 1339
1346 do {1340 prefix_len = info_print_prefix(r->info, syslog, time, prefix);
1347 const char *next = memchr(text, '\n', text_size);
1348 size_t text_len;
1349 1341
1342 /*
1343 * @text_len: bytes of unprocessed text
1344 * @line_len: bytes of current line _without_ newline
1345 * @text: pointer to beginning of current line
1346 * @len: number of bytes prepared in r->text_buf
1347 */
1348 for (;;) {
1349 next = memchr(text, '\n', text_len);
1350 if (next) {1350 if (next) {
1351 text_len = next - text;1351 line_len = next - text;
1352 next++;
1353 text_size -= next - text;
1354 } else {1352 } else {
1355 text_len = text_size;1353 /* Drop truncated line(s). */
1354 if (truncated)
1355 break;
1356 line_len = text_len;
1356 }1357 }
1357 1358
1358 if (buf) {1359 /*
1359 if (prefix_len + text_len + 1 >= size - len)1360 * Truncate the text if there is not enough space to add the
1361 * prefix and a trailing newline.
1362 */
1363 if (len + prefix_len + text_len + 1 > buf_size) {
1364 /* Drop even the current line if no space. */
1365 if (len + prefix_len + line_len + 1 > buf_size)
1360 break;1366 break;
1361 1367
1362 memcpy(buf + len, prefix, prefix_len);1368 text_len = buf_size - len - prefix_len - 1;
1363 len += prefix_len;1369 truncated = true;
1364 memcpy(buf + len, text, text_len);
1365 len += text_len;
1366 buf[len++] = '\n';
1367 } else {
1368 /* SYSLOG_ACTION_* buffer size only calculation */
1369 len += prefix_len + text_len + 1;
1370 }1370 }
1371 1371
1372 text = next;1372 memmove(text + prefix_len, text, text_len);
1373 } while (text);1373 memcpy(text, prefix, prefix_len);
1374
1375 len += prefix_len + line_len + 1;
1376
1377 if (text_len == line_len) {
1378 /*
1379 * Add the trailing newline removed in
1380 * vprintk_store().
1381 */
1382 text[prefix_len + line_len] = '\n';
1383 break;
1384 }
1385
1386 /*
1387 * Advance beyond the added prefix and the related line with
1388 * its newline.
1389 */
1390 text += prefix_len + line_len + 1;
1391
1392 /*
1393 * The remaining text has only decreased by the line with its
1394 * newline.
1395 *
1396 * Note that @text_len can become zero. It happens when @text
1397 * ended with a newline (either due to truncation or the
1398 * original string ending with "\n\n"). The loop is correctly
1399 * repeated and (if not truncated) an empty line with a prefix
1400 * will be prepared.
1401 */
1402 text_len -= line_len + 1;
1403 }
1374 1404
1375 return len;1405 return len;
1376}1406}
1377 1407
1408static size_t get_record_print_text_size(struct printk_info *info,
1409 unsigned int line_count,
1410 bool syslog, bool time)
1411{
1412 char prefix[PREFIX_MAX];
1413 size_t prefix_len;
1414
1415 prefix_len = info_print_prefix(info, syslog, time, prefix);
1416
1417 /*
1418 * Each line will be preceded with a prefix. The intermediate
1419 * newlines are already within the text, but a final trailing
1420 * newline will be added.
1421 */
1422 return ((prefix_len * line_count) + info->text_len + 1);
1423}
1424
1378static int syslog_print(char __user *buf, int size)1425static int syslog_print(char __user *buf, int size)
1379{1426{
1427 struct printk_info info;
1428 struct printk_record r;
1380 char *text;1429 char *text;
1381 struct printk_log *msg;
1382 int len = 0;1430 int len = 0;
1383 1431
1384 text = kmalloc(LOG_LINE_MAX + PREFIX_MAX, GFP_KERNEL);1432 text = kmalloc(LOG_LINE_MAX + PREFIX_MAX, GFP_KERNEL);
1385 if (!text)1433 if (!text)
1386 return -ENOMEM;1434 return -ENOMEM;
1387 1435
1436 prb_rec_init_rd(&r, &info, text, LOG_LINE_MAX + PREFIX_MAX, NULL, 0);
1437
1388 while (size > 0) {1438 while (size > 0) {
1389 size_t n;1439 size_t n;
1390 size_t skip;1440 size_t skip;
1391 1441
1392 logbuf_lock_irq();1442 logbuf_lock_irq();
1393 if (syslog_seq < log_first_seq) {1443 if (!prb_read_valid(prb, syslog_seq, &r)) {
1394 /* messages are gone, move to first one */
1395 syslog_seq = log_first_seq;
1396 syslog_idx = log_first_idx;
1397 syslog_partial = 0;
1398 }
1399 if (syslog_seq == log_next_seq) {
1400 logbuf_unlock_irq();1444 logbuf_unlock_irq();
1401 break;1445 break;
1402 }1446 }
1447 if (r.info->seq != syslog_seq) {
1448 /* message is gone, move to next valid one */
1449 syslog_seq = r.info->seq;
1450 syslog_partial = 0;
1451 }
1403 1452
1404 /*1453 /*
1405 * To keep reading/counting partial line consistent,1454 * To keep reading/counting partial line consistent,
@@ -1409,13 +1458,10 @@ static int syslog_print(char __user *buf, int size) · 3 unmodified lines
1409 syslog_time = printk_time;1458 syslog_time = printk_time;
1410 1459
1411 skip = syslog_partial;1460 skip = syslog_partial;
1412 msg = log_from_idx(syslog_idx);1461 n = record_print_text(&r, true, syslog_time);
1413 n = msg_print_text(msg, true, syslog_time, text,
1414 LOG_LINE_MAX + PREFIX_MAX);
1415 if (n - syslog_partial <= size) {1462 if (n - syslog_partial <= size) {
1416 /* message fits into buffer, move forward */1463 /* message fits into buffer, move forward */
1417 syslog_idx = log_next(syslog_idx);1464 syslog_seq = r.info->seq + 1;
1418 syslog_seq++;
1419 n -= syslog_partial;1465 n -= syslog_partial;
1420 syslog_partial = 0;1466 syslog_partial = 0;
1421 } else if (!len){1467 } else if (!len){
@@ -1446,11 +1492,12 @@ static int syslog_print(char __user *buf, int size) · 24 unmodified lines
1446 1492
1447static int syslog_print_all(char __user *buf, int size, bool clear)1493static int syslog_print_all(char __user *buf, int size, bool clear)
1448{1494{
1495 struct printk_info info;
1496 unsigned int line_count;
1497 struct printk_record r;
1449 char *text;1498 char *text;
1450 int len = 0;1499 int len = 0;
1451 u64 next_seq;
1452 u64 seq;1500 u64 seq;
1453 u32 idx;
1454 bool time;1501 bool time;
1455 1502
1456 text = kmalloc(LOG_LINE_MAX + PREFIX_MAX, GFP_KERNEL);1503 text = kmalloc(LOG_LINE_MAX + PREFIX_MAX, GFP_KERNEL);
@@ -1463,38 +1510,28 @@ static int syslog_print_all(char __user *buf, int size, bool clear) · 6 unmodified lines
1463 * Find first record that fits, including all following records,1510 * Find first record that fits, including all following records,
1464 * into the user-provided buffer for this dump.1511 * into the user-provided buffer for this dump.
1465 */1512 */
1466 seq = clear_seq;1513 prb_for_each_info(clear_seq, prb, seq, &info, &line_count)
1467 idx = clear_idx;1514 len += get_record_print_text_size(&info, line_count, true, time);
1468 while (seq < log_next_seq) {
1469 struct printk_log *msg = log_from_idx(idx);
1470
1471 len += msg_print_text(msg, true, time, NULL, 0);
1472 idx = log_next(idx);
1473 seq++;
1474 }
1475 1515
1476 /* move first record forward until length fits into the buffer */1516 /* move first record forward until length fits into the buffer */
1477 seq = clear_seq;1517 prb_for_each_info(clear_seq, prb, seq, &info, &line_count) {
1478 idx = clear_idx;1518 if (len <= size)
1479 while (len > size && seq < log_next_seq) {1519 break;
1480 struct printk_log *msg = log_from_idx(idx);1520 len -= get_record_print_text_size(&info, line_count, true, time);
1481
1482 len -= msg_print_text(msg, true, time, NULL, 0);
1483 idx = log_next(idx);
1484 seq++;
1485 }1521 }
1486 1522
1487 /* last message fitting into this dump */1523 prb_rec_init_rd(&r, &info, text, LOG_LINE_MAX + PREFIX_MAX, NULL, 0);
1488 next_seq = log_next_seq;
1489 1524
1490 len = 0;1525 len = 0;
1491 while (len >= 0 && seq < next_seq) {1526 prb_for_each_record(seq, prb, seq, &r) {
1492 struct printk_log *msg = log_from_idx(idx);1527 int textlen;
1493 int textlen = msg_print_text(msg, true, time, text,
1494 LOG_LINE_MAX + PREFIX_MAX);
1495 1528
1496 idx = log_next(idx);1529 textlen = record_print_text(&r, true, time);
1497 seq++;1530
1531 if (len + textlen > size) {
1532 seq--;
1533 break;
1534 }
1498 1535
1499 logbuf_unlock_irq();1536 logbuf_unlock_irq();
1500 if (copy_to_user(buf + len, text, textlen))1537 if (copy_to_user(buf + len, text, textlen))
@@ -1503,17 +1540,12 @@ static int syslog_print_all(char __user *buf, int size, bool clear) · 2 unmodified lines
1503 len += textlen;1540 len += textlen;
1504 logbuf_lock_irq();1541 logbuf_lock_irq();
1505 1542
1506 if (seq < log_first_seq) {1543 if (len < 0)
1507 /* messages are gone, move to next one */1544 break;
1508 seq = log_first_seq;
1509 idx = log_first_idx;
1510 }
1511 }1545 }
1512 1546
1513 if (clear) {1547 if (clear)
1514 clear_seq = log_next_seq;1548 clear_seq = seq;
1515 clear_idx = log_next_idx;
1516 }
1517 logbuf_unlock_irq();1549 logbuf_unlock_irq();
1518 1550
1519 kfree(text);1551 kfree(text);
@@ -1523,8 +1555,7 @@ static int syslog_print_all(char __user *buf, int size, bool clear) · 3 unmodified lines
1523static void syslog_clear(void)1555static void syslog_clear(void)
1524{1556{
1525 logbuf_lock_irq();1557 logbuf_lock_irq();
1526 clear_seq = log_next_seq;1558 clear_seq = prb_next_seq(prb);
1527 clear_idx = log_next_idx;
1528 logbuf_unlock_irq();1559 logbuf_unlock_irq();
1529}1560}
1530 1561
@@ -1551,7 +1582,7 @@ int do_syslog(int type, char __user *buf, int len, int source) · 20 unmodified lines
1551 if (!access_ok(buf, len))1582 if (!access_ok(buf, len))
1552 return -EFAULT;1583 return -EFAULT;
1553 error = wait_event_interruptible(log_wait,1584 error = wait_event_interruptible(log_wait,
1554 syslog_seq != log_next_seq);1585 prb_read_valid(prb, syslog_seq, NULL));
1555 if (error)1586 if (error)
1556 return error;1587 return error;
1557 error = syslog_print(buf, len);1588 error = syslog_print(buf, len);
@@ -1600,10 +1631,9 @@ int do_syslog(int type, char __user *buf, int len, int source) · 42 unmodified lines
1600 /* Number of chars in the log buffer */1631 /* Number of chars in the log buffer */
1601 case SYSLOG_ACTION_SIZE_UNREAD:1632 case SYSLOG_ACTION_SIZE_UNREAD:
1602 logbuf_lock_irq();1633 logbuf_lock_irq();
1603 if (syslog_seq < log_first_seq) {1634 if (syslog_seq < prb_first_valid_seq(prb)) {
1604 /* messages are gone, move to first one */1635 /* messages are gone, move to first one */
1605 syslog_seq = log_first_seq;1636 syslog_seq = prb_first_valid_seq(prb);
1606 syslog_idx = log_first_idx;
1607 syslog_partial = 0;1637 syslog_partial = 0;
1608 }1638 }
1609 if (source == SYSLOG_FROM_PROC) {1639 if (source == SYSLOG_FROM_PROC) {
@@ -1612,20 +1642,18 @@ int do_syslog(int type, char __user *buf, int len, int source) · 2 unmodified lines
1612 * for pending data, not the size; return the count of1642 * for pending data, not the size; return the count of
1613 * records, not the length.1643 * records, not the length.
1614 */1644 */
1615 error = log_next_seq - syslog_seq;1645 error = prb_next_seq(prb) - syslog_seq;
1616 } else {1646 } else {
1617 u64 seq = syslog_seq;
1618 u32 idx = syslog_idx;
1619 bool time = syslog_partial ? syslog_time : printk_time;1647 bool time = syslog_partial ? syslog_time : printk_time;
1620 1648 struct printk_info info;
1621 while (seq < log_next_seq) {1649 unsigned int line_count;
1622 struct printk_log *msg = log_from_idx(idx);1650 u64 seq;
1623 1651
1624 error += msg_print_text(msg, true, time, NULL,1652 prb_for_each_info(syslog_seq, prb, seq, &info,
1625 0);1653 &line_count) {
1654 error += get_record_print_text_size(&info, line_count,
1655 true, time);
1626 time = printk_time;1656 time = printk_time;
1627 idx = log_next(idx);
1628 seq++;
1629 }1657 }
1630 error -= syslog_partial;1658 error -= syslog_partial;
1631 }1659 }
@@ -1796,10 +1824,22 @@ static int console_trylock_spinning(void) · 164 unmodified lines
1796static void call_console_drivers(const char *ext_text, size_t ext_len,1824static void call_console_drivers(const char *ext_text, size_t ext_len,
1797 const char *text, size_t len)1825 const char *text, size_t len)
1798{1826{
1827 static char dropped_text[64];
1828 size_t dropped_len = 0;
1799 struct console *con;1829 struct console *con;
1800 1830
1801 trace_console_rcuidle(text, len);1831 trace_console_rcuidle(text, len);
1802 1832
1833 if (!console_drivers)
1834 return;
1835
1836 if (console_dropped) {
1837 dropped_len = snprintf(dropped_text, sizeof(dropped_text),
1838 "** %lu printk messages dropped **\n",
1839 console_dropped);
1840 console_dropped = 0;
1841 }
1842
1803 for_each_console(con) {1843 for_each_console(con) {
1804 if (exclusive_console && con != exclusive_console)1844 if (exclusive_console && con != exclusive_console)
1805 continue;1845 continue;
@@ -1812,8 +1852,11 @@ static void call_console_drivers(const char *ext_text, size_t ext_len, · 6 unmodified lines
1812 continue;1852 continue;
1813 if (con->flags & CON_EXTENDED)1853 if (con->flags & CON_EXTENDED)
1814 con->write(con, ext_text, ext_len);1854 con->write(con, ext_text, ext_len);
1815 else1855 else {
1856 if (dropped_len)
1857 con->write(con, dropped_text, dropped_len);
1816 con->write(con, text, len);1858 con->write(con, text, len);
1859 }
1817 }1860 }
1818}1861}
1819 1862
@@ -2076,21 +2119,24 @@ EXPORT_SYMBOL(printk); · 256 unmodified lines
2076#define PREFIX_MAX 02119#define PREFIX_MAX 0
2077#define printk_time false2120#define printk_time false
2078 2121
2122#define prb_read_valid(rb, seq, r) false
2123#define prb_first_valid_seq(rb) 0
2124
2079static u64 syslog_seq;2125static u64 syslog_seq;
2080static u32 syslog_idx;
2081static u64 console_seq;2126static u64 console_seq;
2082static u32 console_idx;
2083static u64 exclusive_console_stop_seq;2127static u64 exclusive_console_stop_seq;
2084static u64 log_first_seq;2128static unsigned long console_dropped;
2085static u32 log_first_idx;2129
2086static u64 log_next_seq;2130static size_t record_print_text(const struct printk_record *r,
2087static char *log_text(const struct printk_log *msg) { return NULL; }2131 bool syslog, bool time)
2088static char *log_dict(const struct printk_log *msg) { return NULL; }2132{
2089static struct printk_log *log_from_idx(u32 idx) { return NULL; }2133 return 0;
2090static u32 log_next(u32 idx) { return 0; }2134}
2091static ssize_t msg_print_ext_header(char *buf, size_t size,2135static ssize_t info_print_ext_header(char *buf, size_t size,
2092 struct printk_log *msg,2136 struct printk_info *info)
2093 u64 seq) { return 0; }2137{
2138 return 0;
2139}
2094static ssize_t msg_print_ext_body(char *buf, size_t size,2140static ssize_t msg_print_ext_body(char *buf, size_t size,
2095 char *dict, size_t dict_len,2141 char *dict, size_t dict_len,
2096 char *text, size_t text_len) { return 0; }2142 char *text, size_t text_len) { return 0; }
@@ -2098,8 +2144,6 @@ static void console_lock_spinning_enable(void) { } · 1 unmodified line
2098static int console_lock_spinning_disable_and_check(void) { return 0; }2144static int console_lock_spinning_disable_and_check(void) { return 0; }
2099static void call_console_drivers(const char *ext_text, size_t ext_len,2145static void call_console_drivers(const char *ext_text, size_t ext_len,
2100 const char *text, size_t len) {}2146 const char *text, size_t len) {}
2101static size_t msg_print_text(const struct printk_log *msg, bool syslog,
2102 bool time, char *buf, size_t size) { return 0; }
2103static bool suppress_message_printing(int level) { return false; }2147static bool suppress_message_printing(int level) { return false; }
2104 2148
2105#endif /* CONFIG_PRINTK */2149#endif /* CONFIG_PRINTK */
@@ -2384,14 +2428,19 @@ void console_unlock(void) · 278 unmodified lines
2384{2428{
2385 static char ext_text[CONSOLE_EXT_LOG_MAX];2429 static char ext_text[CONSOLE_EXT_LOG_MAX];
2386 static char text[LOG_LINE_MAX + PREFIX_MAX];2430 static char text[LOG_LINE_MAX + PREFIX_MAX];
2431 static char dict[LOG_LINE_MAX];
2387 unsigned long flags;2432 unsigned long flags;
2388 bool do_cond_resched, retry;2433 bool do_cond_resched, retry;
2434 struct printk_info info;
2435 struct printk_record r;
2389 2436
2390 if (console_suspended) {2437 if (console_suspended) {
2391 up_console_sem();2438 up_console_sem();
2392 return;2439 return;
2393 }2440 }
2394 2441
2442 prb_rec_init_rd(&r, &info, text, sizeof(text), dict, sizeof(dict));
2443
2395 /*2444 /*
2396 * Console drivers are called with interrupts disabled, so2445 * Console drivers are called with interrupts disabled, so
2397 * @console_may_schedule should be cleared before; however, we may2446 * @console_may_schedule should be cleared before; however, we may
@@ -2422,35 +2471,26 @@ again: · 24 unmodified lines
2422 }2471 }
2423 2472
2424 for (;;) {2473 for (;;) {
2425 struct printk_log *msg;
2426 size_t ext_len = 0;2474 size_t ext_len = 0;
2427 size_t len;2475 size_t len;
2428 2476
2429 printk_safe_enter_irqsave(flags);2477 printk_safe_enter_irqsave(flags);
2430 raw_spin_lock(&logbuf_lock);2478 raw_spin_lock(&logbuf_lock);
2431 if (console_seq < log_first_seq) {
2432 len = snprintf(text, sizeof(text),
2433 "** %llu printk messages dropped **\n",
2434 log_first_seq - console_seq);
2435
2436 /* messages are gone, move to first one */
2437 console_seq = log_first_seq;
2438 console_idx = log_first_idx;
2439 } else {
2440 len = 0;
2441 }
2442skip:2479skip:
2443 if (console_seq == log_next_seq)2480 if (!prb_read_valid(prb, console_seq, &r))
2444 break;2481 break;
2445 2482
2446 msg = log_from_idx(console_idx);2483 if (console_seq != r.info->seq) {
2447 if (suppress_message_printing(msg->level)) {2484 console_dropped += r.info->seq - console_seq;
2485 console_seq = r.info->seq;
2486 }
2487
2488 if (suppress_message_printing(r.info->level)) {
2448 /*2489 /*
2449 * Skip record we have buffered and already printed2490 * Skip record we have buffered and already printed
2450 * directly to the console when we received it, and2491 * directly to the console when we received it, and
2451 * record that has level above the console loglevel.2492 * record that has level above the console loglevel.
2452 */2493 */
2453 console_idx = log_next(console_idx);
2454 console_seq++;2494 console_seq++;
2455 goto skip;2495 goto skip;
2456 }2496 }
@@ -2461,19 +2501,24 @@ skip: · 4 unmodified lines
2461 exclusive_console = NULL;2501 exclusive_console = NULL;
2462 }2502 }
2463 2503
2464 len += msg_print_text(msg,2504 /*
2465 console_msg_format & MSG_FORMAT_SYSLOG,2505 * Handle extended console text first because later
2466 printk_time, text + len, sizeof(text) - len);2506 * record_print_text() will modify the record buffer in-place.
2507 */
2467 if (nr_ext_console_drivers) {2508 if (nr_ext_console_drivers) {
2468 ext_len = msg_print_ext_header(ext_text,2509 ext_len = info_print_ext_header(ext_text,
2469 sizeof(ext_text),2510 sizeof(ext_text),
2470 msg, console_seq);2511 r.info);
2471 ext_len += msg_print_ext_body(ext_text + ext_len,2512 ext_len += msg_print_ext_body(ext_text + ext_len,
2472 sizeof(ext_text) - ext_len,2513 sizeof(ext_text) - ext_len,
2473 log_dict(msg), msg->dict_len,2514 &r.dict_buf[0],
2474 log_text(msg), msg->text_len);2515 r.info->dict_len,
2516 &r.text_buf[0],
2517 r.info->text_len);
2475 }2518 }
2476 console_idx = log_next(console_idx);2519 len = record_print_text(&r,
2520 console_msg_format & MSG_FORMAT_SYSLOG,
2521 printk_time);
2477 console_seq++;2522 console_seq++;
2478 raw_spin_unlock(&logbuf_lock);2523 raw_spin_unlock(&logbuf_lock);
2479 2524
@@ -2513,7 +2558,7 @@ skip: · 33 unmodified lines
2513 * flush, no worries.2558 * flush, no worries.
2514 */2559 */
2515 raw_spin_lock(&logbuf_lock);2560 raw_spin_lock(&logbuf_lock);
2516 retry = console_seq != log_next_seq;2561 retry = prb_read_valid(prb, console_seq, NULL);
2517 raw_spin_unlock(&logbuf_lock);2562 raw_spin_unlock(&logbuf_lock);
2518 printk_safe_exit_irqrestore(flags);2563 printk_safe_exit_irqrestore(flags);
2519 2564
@@ -2582,8 +2627,7 @@ void console_flush_on_panic(enum con_flush_mode mode) · 62 unmodified lines
2582 unsigned long flags;2627 unsigned long flags;
2583 2628
2584 logbuf_lock_irqsave(flags);2629 logbuf_lock_irqsave(flags);
2585 console_seq = log_first_seq;2630 console_seq = prb_first_valid_seq(prb);
2586 console_idx = log_first_idx;
2587 logbuf_unlock_irqrestore(flags);2631 logbuf_unlock_irqrestore(flags);
2588 }2632 }
2589 console_unlock();2633 console_unlock();
@@ -2826,7 +2870,6 @@ void register_console(struct console *newcon) · 236 unmodified lines
2826 exclusive_console = newcon;2870 exclusive_console = newcon;
2827 exclusive_console_stop_seq = console_seq;2871 exclusive_console_stop_seq = console_seq;
2828 console_seq = syslog_seq;2872 console_seq = syslog_seq;
2829 console_idx = syslog_idx;
2830 logbuf_unlock_irqrestore(flags);2873 logbuf_unlock_irqrestore(flags);
2831 }2874 }
2832 console_unlock();2875 console_unlock();
@@ -3215,9 +3258,7 @@ void kmsg_dump(enum kmsg_dump_reason reason) · 382 unmodified lines
3215 3258
3216 logbuf_lock_irqsave(flags);3259 logbuf_lock_irqsave(flags);
3217 dumper->cur_seq = clear_seq;3260 dumper->cur_seq = clear_seq;
3218 dumper->cur_idx = clear_idx;3261 dumper->next_seq = prb_next_seq(prb);
3219 dumper->next_seq = log_next_seq;
3220 dumper->next_idx = log_next_idx;
3221 logbuf_unlock_irqrestore(flags);3262 logbuf_unlock_irqrestore(flags);
3222 3263
3223 /* invoke dumper which will iterate over records */3264 /* invoke dumper which will iterate over records */
@@ -3251,28 +3292,33 @@ void kmsg_dump(enum kmsg_dump_reason reason) · 27 unmodified lines
3251bool kmsg_dump_get_line_nolock(struct kmsg_dumper *dumper, bool syslog,3292bool kmsg_dump_get_line_nolock(struct kmsg_dumper *dumper, bool syslog,
3252 char *line, size_t size, size_t *len)3293 char *line, size_t size, size_t *len)
3253{3294{
3254 struct printk_log *msg;3295 struct printk_info info;
3296 unsigned int line_count;
3297 struct printk_record r;
3255 size_t l = 0;3298 size_t l = 0;
3256 bool ret = false;3299 bool ret = false;
3257 3300
3301 prb_rec_init_rd(&r, &info, line, size, NULL, 0);
3302
3258 if (!dumper->active)3303 if (!dumper->active)
3259 goto out;3304 goto out;
3260 3305
3261 if (dumper->cur_seq < log_first_seq) {3306 /* Read text or count text lines? */
3262 /* messages are gone, move to first available one */3307 if (line) {
3263 dumper->cur_seq = log_first_seq;3308 if (!prb_read_valid(prb, dumper->cur_seq, &r))
3264 dumper->cur_idx = log_first_idx;3309 goto out;
3265 }3310 l = record_print_text(&r, syslog, printk_time);
3266 3311 } else {
3267 /* last entry */3312 if (!prb_read_valid_info(prb, dumper->cur_seq,
3268 if (dumper->cur_seq >= log_next_seq)3313 &info, &line_count)) {
3269 goto out;3314 goto out;
3315 }
3316 l = get_record_print_text_size(&info, line_count, syslog,
3317 printk_time);
3270 3318
3271 msg = log_from_idx(dumper->cur_idx);3319 }
3272 l = msg_print_text(msg, syslog, printk_time, line, size);
3273 3320
3274 dumper->cur_idx = log_next(dumper->cur_idx);3321 dumper->cur_seq = r.info->seq + 1;
3275 dumper->cur_seq++;
3276 ret = true;3322 ret = true;
3277out:3323out:
3278 if (len)3324 if (len)
@@ -3333,23 +3379,25 @@ EXPORT_SYMBOL_GPL(kmsg_dump_get_line); · 54 unmodified lines
3333bool kmsg_dump_get_buffer(struct kmsg_dumper *dumper, bool syslog,3379bool kmsg_dump_get_buffer(struct kmsg_dumper *dumper, bool syslog,
3334 char *buf, size_t size, size_t *len)3380 char *buf, size_t size, size_t *len)
3335{3381{
3382 struct printk_info info;
3383 unsigned int line_count;
3384 struct printk_record r;
3336 unsigned long flags;3385 unsigned long flags;
3337 u64 seq;3386 u64 seq;
3338 u32 idx;
3339 u64 next_seq;3387 u64 next_seq;
3340 u32 next_idx;
3341 size_t l = 0;3388 size_t l = 0;
3342 bool ret = false;3389 bool ret = false;
3343 bool time = printk_time;3390 bool time = printk_time;
3344 3391
3345 if (!dumper->active)3392 prb_rec_init_rd(&r, &info, buf, size, NULL, 0);
3393
3394 if (!dumper->active || !buf || !size)
3346 goto out;3395 goto out;
3347 3396
3348 logbuf_lock_irqsave(flags);3397 logbuf_lock_irqsave(flags);
3349 if (dumper->cur_seq < log_first_seq) {3398 if (dumper->cur_seq < prb_first_valid_seq(prb)) {
3350 /* messages are gone, move to first available one */3399 /* messages are gone, move to first available one */
3351 dumper->cur_seq = log_first_seq;3400 dumper->cur_seq = prb_first_valid_seq(prb);
3352 dumper->cur_idx = log_first_idx;
3353 }3401 }
3354 3402
3355 /* last entry */3403 /* last entry */
@@ -3360,41 +3408,41 @@ bool kmsg_dump_get_buffer(struct kmsg_dumper *dumper, bool syslog, · 4 unmodified lines
3360 3408
3361 /* calculate length of entire buffer */3409 /* calculate length of entire buffer */
3362 seq = dumper->cur_seq;3410 seq = dumper->cur_seq;
3363 idx = dumper->cur_idx;3411 while (prb_read_valid_info(prb, seq, &info, &line_count)) {
3364 while (seq < dumper->next_seq) {3412 if (r.info->seq >= dumper->next_seq)
3365 struct printk_log *msg = log_from_idx(idx);3413 break;
3366 3414 l += get_record_print_text_size(&info, line_count, true, time);
3367 l += msg_print_text(msg, true, time, NULL, 0);3415 seq = r.info->seq + 1;
3368 idx = log_next(idx);
3369 seq++;
3370 }3416 }
3371 3417
3372 /* move first record forward until length fits into the buffer */3418 /* move first record forward until length fits into the buffer */
3373 seq = dumper->cur_seq;3419 seq = dumper->cur_seq;
3374 idx = dumper->cur_idx;3420 while (l >= size && prb_read_valid_info(prb, seq,
3375 while (l >= size && seq < dumper->next_seq) {3421 &info, &line_count)) {
3376 struct printk_log *msg = log_from_idx(idx);3422 if (r.info->seq >= dumper->next_seq)
3377 3423 break;
3378 l -= msg_print_text(msg, true, time, NULL, 0);3424 l -= get_record_print_text_size(&info, line_count, true, time);
3379 idx = log_next(idx);3425 seq = r.info->seq + 1;
3380 seq++;
3381 }3426 }
3382 3427
3383 /* last message in next interation */3428 /* last message in next interation */
3384 next_seq = seq;3429 next_seq = seq;
3385 next_idx = idx;
3386 3430
3431 /* actually read text into the buffer now */
3387 l = 0;3432 l = 0;
3388 while (seq < dumper->next_seq) {3433 while (prb_read_valid(prb, seq, &r)) {
3389 struct printk_log *msg = log_from_idx(idx);3434 if (r.info->seq >= dumper->next_seq)
3435 break;
3436
3437 l += record_print_text(&r, syslog, time);
3438
3439 /* adjust record to store to remaining buffer space */
3440 prb_rec_init_rd(&r, &info, buf + l, size - l, NULL, 0);
3390 3441
3391 l += msg_print_text(msg, syslog, time, buf + l, size - l);3442 seq = r.info->seq + 1;
3392 idx = log_next(idx);
3393 seq++;
3394 }3443 }
3395 3444
3396 dumper->next_seq = next_seq;3445 dumper->next_seq = next_seq;
3397 dumper->next_idx = next_idx;
3398 ret = true;3446 ret = true;
3399 logbuf_unlock_irqrestore(flags);3447 logbuf_unlock_irqrestore(flags);
3400out:3448out:
@@ -3417,9 +3465,7 @@ EXPORT_SYMBOL_GPL(kmsg_dump_get_buffer); · 16 unmodified lines
3417void kmsg_dump_rewind_nolock(struct kmsg_dumper *dumper)3465void kmsg_dump_rewind_nolock(struct kmsg_dumper *dumper)
3418{3466{
3419 dumper->cur_seq = clear_seq;3467 dumper->cur_seq = clear_seq;
3420 dumper->cur_idx = clear_idx;3468 dumper->next_seq = prb_next_seq(prb);
3421 dumper->next_seq = log_next_seq;
3422 dumper->next_idx = log_next_idx;
3423}3469}
3424 3470
3425/**3471/**

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t