History

printk: use the lockless ringbuffer

Replace the existing ringbuffer usage and implementation with lockless ringbuffer usage. Even though the new ringbuffer does not require locking, all existing locking is left in place. Therefore, this change is purely replacing the underlining ringbuffer. Changes that exist due to the ringbuffer replacement: - The VMCOREINFO has been updated for the new structures. - Dictionary data is now stored in a separate data buffer from the human-readable messages. The dictionary data buffer is set to the same size as the message buffer. Therefore, the total required memory for both dictionary and message data is 2 * (2 ^ CONFIG_LOG_BUF_SHIFT) for the initial static buffers and 2 * log_buf_len (the kernel parameter) for the dynamic buffers. - Record meta-data is now stored in a separate array of descriptors. This is an additional 72 * (2 ^ (CONFIG_LOG_BUF_SHIFT - 5)) bytes for the static array and 72 * (log_buf_len >> 5) bytes for the dynamic array.

John Ogness committed 6 years ago

Commit 896fbe21 parent 8749efc1 file · +493 −447Browse files

Unified · Split

kernel/printk/printk.c+493−447
@@ -55,6 +55,7 @@ · 54 unmodified lines
5555 #define CREATE_TRACE_POINTS
5656 #include <trace/events/printk.h>
5757
58+#include "printk_ringbuffer.h"
5859 #include "console_cmdline.h"
5960 #include "braille.h"
6061 #include "internal.h"
@@ -294,30 +295,24 @@ enum con_msg_format_flags { · 233 unmodified lines
294295 static int console_msg_format = MSG_FORMAT_DEFAULT;
295296
296297 /*
297− * The printk log buffer consists of a chain of concatenated variable
298− * length records. Every record starts with a record header, containing
299− * the overall length of the record.
298+ * The printk log buffer consists of a sequenced collection of records, each
299+ * containing variable length message and dictionary text. Every record
300+ * also contains its own meta-data (@info).
300301 *
301− * The heads to the first and last entry in the buffer, as well as the
302− * sequence numbers of these entries are maintained when messages are
303− * stored.
302+ * Every record meta-data carries the timestamp in microseconds, as well as
303+ * the standard userspace syslog level and syslog facility. The usual kernel
304+ * messages use LOG_KERN; userspace-injected messages always carry a matching
305+ * syslog facility, by default LOG_USER. The origin of every message can be
306+ * reliably determined that way.
304307 *
305− * If the heads indicate available messages, the length in the header
306− * tells the start next message. A length == 0 for the next message
307− * indicates a wrap-around to the beginning of the buffer.
308+ * The human readable log message of a record is available in @text, the
309+ * length of the message text in @text_len. The stored message is not
310+ * terminated.
308311 *
309− * Every record carries the monotonic timestamp in microseconds, as well as
310− * the standard userspace syslog level and syslog facility. The usual
311− * kernel messages use LOG_KERN; userspace-injected messages always carry
312− * a matching syslog facility, by default LOG_USER. The origin of every
313− * message can be reliably determined that way.
314− *
315− * The human readable log message directly follows the message header. The
316− * length of the message text is stored in the header, the stored message
317− * is not terminated.
318− *
319− * Optionally, a message can carry a dictionary of properties (key/value pairs),
320− * to provide userspace with a machine-readable message context.
312+ * Optionally, a record can carry a dictionary of properties (key/value
313+ * pairs), to provide userspace with a machine-readable message context. The
314+ * length of the dictionary is available in @dict_len. The dictionary is not
315+ * terminated.
321316 *
322317 * Examples for well-defined, commonly used property names are:
323318 * DEVICE=b12:8 device identifier
@@ -331,21 +326,19 @@ static int console_msg_format = MSG_FORMAT_DEFAULT; · 7 unmodified lines
331326 * follows directly after a '=' character. Every property is terminated by
332327 * a '\0' character. The last property is not terminated.
333328 *
334− * Example of a message structure:
335− * 0000 ff 8f 00 00 00 00 00 00 monotonic time in nsec
336− * 0008 34 00 record is 52 bytes long
337− * 000a 0b 00 text is 11 bytes long
338− * 000c 1f 00 dictionary is 23 bytes long
339− * 000e 03 00 LOG_KERN (facility) LOG_ERR (level)
340− * 0010 69 74 27 73 20 61 20 6c "it's a l"
341− * 69 6e 65 "ine"
342− * 001b 44 45 56 49 43 "DEVIC"
343− * 45 3d 62 38 3a 32 00 44 "E=b8:2\0D"
344− * 52 49 56 45 52 3d 62 75 "RIVER=bu"
345− * 67 "g"
346− * 0032 00 00 00 padding to next message header
347− *
348− * The 'struct printk_log' buffer header must never be directly exported to
329+ * Example of record values:
330+ * record.text_buf = "it's a line" (unterminated)
331+ * record.dict_buf = "DEVICE=b8:2\0DRIVER=bug" (unterminated)
332+ * record.info.seq = 56
333+ * record.info.ts_nsec = 36863
334+ * record.info.text_len = 11
335+ * record.info.dict_len = 22
336+ * record.info.facility = 0 (LOG_KERN)
337+ * record.info.flags = 0
338+ * record.info.level = 3 (LOG_ERR)
339+ * record.info.caller_id = 299 (task 299)
340+ *
341+ * The 'struct printk_info' buffer must never be directly exported to
349342 * userspace, it is a kernel-private implementation detail that might
350343 * need to be changed in the future, when the requirements change.
351344 *
@@ -365,23 +358,6 @@ enum log_flags { · 13 unmodified lines
365358 LOG_CONT = 8, /* text is a fragment of a continuation line */
366359 };
367360
368−struct printk_log {
369− u64 ts_nsec; /* timestamp in nanoseconds */
370− u16 len; /* length of entire record */
371− u16 text_len; /* length of text buffer */
372− u16 dict_len; /* length of dictionary buffer */
373− u8 facility; /* syslog facility */
374− u8 flags:5; /* internal record flags */
375− u8 level:3; /* syslog level */
376−#ifdef CONFIG_PRINTK_CALLER
377− u32 caller_id; /* thread id or processor id */
378−#endif
379−}
380−#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
381−__packed __aligned(4)
382−#endif
383−;
384−
385361 /*
386362 * The logbuf_lock protects kmsg buffer, indices, counters. This can be taken
387363 * within the scheduler's rq lock. It must be released before calling
@@ -421,26 +397,16 @@ DEFINE_RAW_SPINLOCK(logbuf_lock); · 33 unmodified lines
421397 DECLARE_WAIT_QUEUE_HEAD(log_wait);
422398 /* the next printk record to read by syslog(READ) or /proc/kmsg */
423399 static u64 syslog_seq;
424−static u32 syslog_idx;
425400 static size_t syslog_partial;
426401 static bool syslog_time;
427402
428−/* index and sequence number of the first record stored in the buffer */
429−static u64 log_first_seq;
430−static u32 log_first_idx;
431−
432−/* index and sequence number of the next record to store in the buffer */
433−static u64 log_next_seq;
434−static u32 log_next_idx;
435−
436403 /* the next printk record to write to the console */
437404 static u64 console_seq;
438−static u32 console_idx;
439405 static u64 exclusive_console_stop_seq;
406+static unsigned long console_dropped;
440407
441408 /* the next printk record to read after the last 'clear' command */
442409 static u64 clear_seq;
443−static u32 clear_idx;
444410
445411 #ifdef CONFIG_PRINTK_CALLER
446412 #define PREFIX_MAX 48
@@ -453,13 +419,31 @@ static u32 clear_idx; · 6 unmodified lines
453419 #define LOG_FACILITY(v) ((v) >> 3 & 0xff)
454420
455421 /* record buffer */
456−#define LOG_ALIGN __alignof__(struct printk_log)
422+#define LOG_ALIGN __alignof__(unsigned long)
457423 #define __LOG_BUF_LEN (1 << CONFIG_LOG_BUF_SHIFT)
458424 #define LOG_BUF_LEN_MAX (u32)(1 << 31)
459425 static char __log_buf[__LOG_BUF_LEN] __aligned(LOG_ALIGN);
460426 static char *log_buf = __log_buf;
461427 static u32 log_buf_len = __LOG_BUF_LEN;
462428
429+/*
430+ * Define the average message size. This only affects the number of
431+ * descriptors that will be available. Underestimating is better than
432+ * overestimating (too many available descriptors is better than not enough).
433+ * The dictionary buffer will be the same size as the text buffer.
434+ */
435+#define PRB_AVGBITS 5 /* 32 character average length */
436+
437+#if CONFIG_LOG_BUF_SHIFT <= PRB_AVGBITS
438+#error CONFIG_LOG_BUF_SHIFT value too small.
439+#endif
440+_DEFINE_PRINTKRB(printk_rb_static, CONFIG_LOG_BUF_SHIFT - PRB_AVGBITS,
441+ PRB_AVGBITS, PRB_AVGBITS, &__log_buf[0]);
442+
443+static struct printk_ringbuffer printk_rb_dynamic;
444+
445+static struct printk_ringbuffer *prb = &printk_rb_static;
446+
463447 /*
464448 * We cannot access per-CPU data (e.g. per-CPU flush irq_work) before
465449 * per_cpu_areas are initialised. This variable is set to true when
@@ -484,108 +468,6 @@ u32 log_buf_len_get(void) · 18 unmodified lines
484468 return log_buf_len;
485469 }
486470
487−/* human readable text of the record */
488−static char *log_text(const struct printk_log *msg)
489−{
490− return (char *)msg + sizeof(struct printk_log);
491−}
492−
493−/* optional key/value pair dictionary attached to the record */
494−static char *log_dict(const struct printk_log *msg)
495−{
496− return (char *)msg + sizeof(struct printk_log) + msg->text_len;
497−}
498−
499−/* get record by index; idx must point to valid msg */
500−static struct printk_log *log_from_idx(u32 idx)
501−{
502− struct printk_log *msg = (struct printk_log *)(log_buf + idx);
503−
504− /*
505− * A length == 0 record is the end of buffer marker. Wrap around and
506− * read the message at the start of the buffer.
507− */
508− if (!msg->len)
509− return (struct printk_log *)log_buf;
510− return msg;
511−}
512−
513−/* get next record; idx must point to valid msg */
514−static u32 log_next(u32 idx)
515−{
516− struct printk_log *msg = (struct printk_log *)(log_buf + idx);
517−
518− /* length == 0 indicates the end of the buffer; wrap */
519− /*
520− * A length == 0 record is the end of buffer marker. Wrap around and
521− * read the message at the start of the buffer as *this* one, and
522− * return the one after that.
523− */
524− if (!msg->len) {
525− msg = (struct printk_log *)log_buf;
526− return msg->len;
527− }
528− return idx + msg->len;
529−}
530−
531−/*
532− * Check whether there is enough free space for the given message.
533− *
534− * The same values of first_idx and next_idx mean that the buffer
535− * is either empty or full.
536− *
537− * If the buffer is empty, we must respect the position of the indexes.
538− * They cannot be reset to the beginning of the buffer.
539− */
540−static int logbuf_has_space(u32 msg_size, bool empty)
541−{
542− u32 free;
543−
544− if (log_next_idx > log_first_idx || empty)
545− free = max(log_buf_len - log_next_idx, log_first_idx);
546− else
547− free = log_first_idx - log_next_idx;
548−
549− /*
550− * We need space also for an empty header that signalizes wrapping
551− * of the buffer.
552− */
553− return free >= msg_size + sizeof(struct printk_log);
554−}
555−
556−static int log_make_free_space(u32 msg_size)
557−{
558− while (log_first_seq < log_next_seq &&
559− !logbuf_has_space(msg_size, false)) {
560− /* drop old messages until we have enough contiguous space */
561− log_first_idx = log_next(log_first_idx);
562− log_first_seq++;
563− }
564−
565− if (clear_seq < log_first_seq) {
566− clear_seq = log_first_seq;
567− clear_idx = log_first_idx;
568− }
569−
570− /* sequence numbers are equal, so the log buffer is empty */
571− if (logbuf_has_space(msg_size, log_first_seq == log_next_seq))
572− return 0;
573−
574− return -ENOMEM;
575−}
576−
577−/* compute the message size including the padding bytes */
578−static u32 msg_used_size(u16 text_len, u16 dict_len, u32 *pad_len)
579−{
580− u32 size;
581−
582− size = sizeof(struct printk_log) + text_len + dict_len;
583− *pad_len = (-size) & (LOG_ALIGN - 1);
584− size += *pad_len;
585−
586− return size;
587−}
588−
589471 /*
590472 * Define how much of the log buffer we could take at maximum. The value
591473 * must be greater than two. Note that only half of the buffer is available
@@ -594,22 +476,23 @@ static u32 msg_used_size(u16 text_len, u16 dict_len, u32 *pad_len) · 2 unmodified lines
594476 #define MAX_LOG_TAKE_PART 4
595477 static const char trunc_msg[] = "<truncated>";
596478
597−static u32 truncate_msg(u16 *text_len, u16 *trunc_msg_len,
598− u16 *dict_len, u32 *pad_len)
479+static void truncate_msg(u16 *text_len, u16 *trunc_msg_len)
599480 {
600481 /*
601482 * The message should not take the whole buffer. Otherwise, it might
602483 * get removed too soon.
603484 */
604485 u32 max_text_len = log_buf_len / MAX_LOG_TAKE_PART;
486+
605487 if (*text_len > max_text_len)
606488 *text_len = max_text_len;
607− /* enable the warning message */
489+
490+ /* enable the warning message (if there is room) */
608491 *trunc_msg_len = strlen(trunc_msg);
609− /* disable the "dict" completely */
610− *dict_len = 0;
611− /* compute the size again, count also the warning message */
612− return msg_used_size(*text_len + *trunc_msg_len, 0, pad_len);
492+ if (*text_len >= *trunc_msg_len)
493+ *text_len -= *trunc_msg_len;
494+ else
495+ *trunc_msg_len = 0;
613496 }
614497
615498 /* insert record into the buffer, discard old ones, update heads */
@@ -618,60 +501,40 @@ static int log_store(u32 caller_id, int facility, int level, · 2 unmodified lines
618501 const char *dict, u16 dict_len,
619502 const char *text, u16 text_len)
620503 {
621− struct printk_log *msg;
622− u32 size, pad_len;
504+ struct prb_reserved_entry e;
505+ struct printk_record r;
623506 u16 trunc_msg_len = 0;
624507
625− /* number of '\0' padding bytes to next message */
626− size = msg_used_size(text_len, dict_len, &pad_len);
508+ prb_rec_init_wr(&r, text_len, dict_len);
627509
628− if (log_make_free_space(size)) {
510+ if (!prb_reserve(&e, prb, &r)) {
629511 /* truncate the message if it is too long for empty buffer */
630− size = truncate_msg(&text_len, &trunc_msg_len,
631− &dict_len, &pad_len);
512+ truncate_msg(&text_len, &trunc_msg_len);
513+ prb_rec_init_wr(&r, text_len + trunc_msg_len, dict_len);
632514 /* survive when the log buffer is too small for trunc_msg */
633− if (log_make_free_space(size))
515+ if (!prb_reserve(&e, prb, &r))
634516 return 0;
635517 }
636518
637− if (log_next_idx + size + sizeof(struct printk_log) > log_buf_len) {
638− /*
639− * This message + an additional empty header does not fit
640− * at the end of the buffer. Add an empty header with len == 0
641− * to signify a wrap around.
642− */
643− memset(log_buf + log_next_idx, 0, sizeof(struct printk_log));
644− log_next_idx = 0;
645− }
646−
647519 /* fill message */
648− msg = (struct printk_log *)(log_buf + log_next_idx);
649− memcpy(log_text(msg), text, text_len);
650− msg->text_len = text_len;
651− if (trunc_msg_len) {
652− memcpy(log_text(msg) + text_len, trunc_msg, trunc_msg_len);
653− msg->text_len += trunc_msg_len;
654− }
655− memcpy(log_dict(msg), dict, dict_len);
656− msg->dict_len = dict_len;
657− msg->facility = facility;
658− msg->level = level & 7;
659− msg->flags = flags & 0x1f;
520+ memcpy(&r.text_buf[0], text, text_len);
521+ if (trunc_msg_len)
522+ memcpy(&r.text_buf[text_len], trunc_msg, trunc_msg_len);
523+ if (r.dict_buf)
524+ memcpy(&r.dict_buf[0], dict, dict_len);
525+ r.info->facility = facility;
526+ r.info->level = level & 7;
527+ r.info->flags = flags & 0x1f;
660528 if (ts_nsec > 0)
661− msg->ts_nsec = ts_nsec;
529+ r.info->ts_nsec = ts_nsec;
662530 else
663− msg->ts_nsec = local_clock();
664−#ifdef CONFIG_PRINTK_CALLER
665− msg->caller_id = caller_id;
666−#endif
667− memset(log_dict(msg) + dict_len, 0, pad_len);
668− msg->len = size;
531+ r.info->ts_nsec = local_clock();
532+ r.info->caller_id = caller_id;
669533
670534 /* insert message */
671− log_next_idx += msg->len;
672− log_next_seq++;
535+ prb_commit(&e);
673536
674− return msg->text_len;
537+ return (text_len + trunc_msg_len);
675538 }
676539
677540 int dmesg_restrict = IS_ENABLED(CONFIG_SECURITY_DMESG_RESTRICT);
@@ -723,13 +586,13 @@ static void append_char(char **pp, char *e, char c) · 45 unmodified lines
723586 *(*pp)++ = c;
724587 }
725588
726−static ssize_t msg_print_ext_header(char *buf, size_t size,
727− struct printk_log *msg, u64 seq)
589+static ssize_t info_print_ext_header(char *buf, size_t size,
590+ struct printk_info *info)
728591 {
729− u64 ts_usec = msg->ts_nsec;
592+ u64 ts_usec = info->ts_nsec;
730593 char caller[20];
731594 #ifdef CONFIG_PRINTK_CALLER
732− u32 id = msg->caller_id;
595+ u32 id = info->caller_id;
733596
734597 snprintf(caller, sizeof(caller), ",caller=%c%u",
735598 id & 0x80000000 ? 'C' : 'T', id & ~0x80000000);
@@ -740,8 +603,8 @@ static ssize_t msg_print_ext_header(char *buf, size_t size, · 4 unmodified lines
740603 do_div(ts_usec, 1000);
741604
742605 return scnprintf(buf, size, "%u,%llu,%llu,%c%s;",
743− (msg->facility << 3) | msg->level, seq, ts_usec,
744− msg->flags & LOG_CONT ? 'c' : '-', caller);
606+ (info->facility << 3) | info->level, info->seq,
607+ ts_usec, info->flags & LOG_CONT ? 'c' : '-', caller);
745608 }
746609
747610 static ssize_t msg_print_ext_body(char *buf, size_t size,
@@ -795,10 +658,14 @@ static ssize_t msg_print_ext_body(char *buf, size_t size, · 47 unmodified lines
795658 /* /dev/kmsg - userspace message inject/listen interface */
796659 struct devkmsg_user {
797660 u64 seq;
798− u32 idx;
799661 struct ratelimit_state rs;
800662 struct mutex lock;
801663 char buf[CONSOLE_EXT_LOG_MAX];
664+
665+ struct printk_info info;
666+ char text_buf[CONSOLE_EXT_LOG_MAX];
667+ char dict_buf[CONSOLE_EXT_LOG_MAX];
668+ struct printk_record record;
802669 };
803670
804671 static __printf(3, 4) __cold
@@ -881,7 +748,7 @@ static ssize_t devkmsg_read(struct file *file, char __user *buf, · 76 unmodified lines
881748 size_t count, loff_t *ppos)
882749 {
883750 struct devkmsg_user *user = file->private_data;
884− struct printk_log *msg;
751+ struct printk_record *r = &user->record;
885752 size_t len;
886753 ssize_t ret;
887754
@@ -893,7 +760,7 @@ static ssize_t devkmsg_read(struct file *file, char __user *buf, · 5 unmodified lines
893760 return ret;
894761
895762 logbuf_lock_irq();
896− while (user->seq == log_next_seq) {
763+ if (!prb_read_valid(prb, user->seq, r)) {
897764 if (file->f_flags & O_NONBLOCK) {
898765 ret = -EAGAIN;
899766 logbuf_unlock_irq();
@@ -902,30 +769,26 @@ static ssize_t devkmsg_read(struct file *file, char __user *buf, · 2 unmodified lines
902769
903770 logbuf_unlock_irq();
904771 ret = wait_event_interruptible(log_wait,
905− user->seq != log_next_seq);
772+ prb_read_valid(prb, user->seq, r));
906773 if (ret)
907774 goto out;
908775 logbuf_lock_irq();
909776 }
910777
911− if (user->seq < log_first_seq) {
778+ if (user->seq < prb_first_valid_seq(prb)) {
912779 /* our last seen message is gone, return error and reset */
913− user->idx = log_first_idx;
914− user->seq = log_first_seq;
780+ user->seq = prb_first_valid_seq(prb);
915781 ret = -EPIPE;
916782 logbuf_unlock_irq();
917783 goto out;
918784 }
919785
920− msg = log_from_idx(user->idx);
921− len = msg_print_ext_header(user->buf, sizeof(user->buf),
922− msg, user->seq);
786+ len = info_print_ext_header(user->buf, sizeof(user->buf), r->info);
923787 len += msg_print_ext_body(user->buf + len, sizeof(user->buf) - len,
924− log_dict(msg), msg->dict_len,
925− log_text(msg), msg->text_len);
788+ &r->dict_buf[0], r->info->dict_len,
789+ &r->text_buf[0], r->info->text_len);
926790
927− user->idx = log_next(user->idx);
928− user->seq++;
791+ user->seq = r->info->seq + 1;
929792 logbuf_unlock_irq();
930793
931794 if (len > count) {
@@ -957,8 +820,7 @@ static loff_t devkmsg_llseek(struct file *file, loff_t offset, int whence) · 25 unmodified lines
957820 switch (whence) {
958821 case SEEK_SET:
959822 /* the first record */
960− user->idx = log_first_idx;
961− user->seq = log_first_seq;
823+ user->seq = prb_first_valid_seq(prb);
962824 break;
963825 case SEEK_DATA:
964826 /*
@@ -966,13 +828,11 @@ static loff_t devkmsg_llseek(struct file *file, loff_t offset, int whence) · 1 unmodified line
966828 * like issued by 'dmesg -c'. Reading /dev/kmsg itself
967829 * changes no global state, and does not clear anything.
968830 */
969− user->idx = clear_idx;
970831 user->seq = clear_seq;
971832 break;
972833 case SEEK_END:
973834 /* after the last record */
974− user->idx = log_next_idx;
975− user->seq = log_next_seq;
835+ user->seq = prb_next_seq(prb);
976836 break;
977837 default:
978838 ret = -EINVAL;
@@ -992,9 +852,9 @@ static __poll_t devkmsg_poll(struct file *file, poll_table *wait) · 13 unmodified lines
992852 poll_wait(file, &log_wait, wait);
993853
994854 logbuf_lock_irq();
995− if (user->seq < log_next_seq) {
855+ if (prb_read_valid(prb, user->seq, NULL)) {
996856 /* return error when data has vanished underneath us */
997− if (user->seq < log_first_seq)
857+ if (user->seq < prb_first_valid_seq(prb))
998858 ret = EPOLLIN|EPOLLRDNORM|EPOLLERR|EPOLLPRI;
999859 else
1000860 ret = EPOLLIN|EPOLLRDNORM;
@@ -1029,9 +889,12 @@ static int devkmsg_open(struct inode *inode, struct file *file) · 28 unmodified lines
1029889
1030890 mutex_init(&user->lock);
1031891
892+ prb_rec_init_rd(&user->record, &user->info,
893+ &user->text_buf[0], sizeof(user->text_buf),
894+ &user->dict_buf[0], sizeof(user->dict_buf));
895+
1032896 logbuf_lock_irq();
1033− user->idx = log_first_idx;
1034− user->seq = log_first_seq;
897+ user->seq = prb_first_valid_seq(prb);
1035898 logbuf_unlock_irq();
1036899
1037900 file->private_data = user;
@@ -1072,23 +935,52 @@ const struct file_operations kmsg_fops = { · 34 unmodified lines
1072935 */
1073936 void log_buf_vmcoreinfo_setup(void)
1074937 {
1075− VMCOREINFO_SYMBOL(log_buf);
1076− VMCOREINFO_SYMBOL(log_buf_len);
1077− VMCOREINFO_SYMBOL(log_first_idx);
1078− VMCOREINFO_SYMBOL(clear_idx);
1079− VMCOREINFO_SYMBOL(log_next_idx);
938+ VMCOREINFO_SYMBOL(prb);
939+ VMCOREINFO_SYMBOL(printk_rb_static);
940+ VMCOREINFO_SYMBOL(clear_seq);
941+
1080942 /*
1081− * Export struct printk_log size and field offsets. User space tools can
943+ * Export struct size and field offsets. User space tools can
1082944 * parse it and detect any changes to structure down the line.
1083945 */
1084− VMCOREINFO_STRUCT_SIZE(printk_log);
1085− VMCOREINFO_OFFSET(printk_log, ts_nsec);
1086− VMCOREINFO_OFFSET(printk_log, len);
1087− VMCOREINFO_OFFSET(printk_log, text_len);
1088− VMCOREINFO_OFFSET(printk_log, dict_len);
1089−#ifdef CONFIG_PRINTK_CALLER
1090− VMCOREINFO_OFFSET(printk_log, caller_id);
1091−#endif
946+
947+ VMCOREINFO_STRUCT_SIZE(printk_ringbuffer);
948+ VMCOREINFO_OFFSET(printk_ringbuffer, desc_ring);
949+ VMCOREINFO_OFFSET(printk_ringbuffer, text_data_ring);
950+ VMCOREINFO_OFFSET(printk_ringbuffer, dict_data_ring);
951+ VMCOREINFO_OFFSET(printk_ringbuffer, fail);
952+
953+ VMCOREINFO_STRUCT_SIZE(prb_desc_ring);
954+ VMCOREINFO_OFFSET(prb_desc_ring, count_bits);
955+ VMCOREINFO_OFFSET(prb_desc_ring, descs);
956+ VMCOREINFO_OFFSET(prb_desc_ring, head_id);
957+ VMCOREINFO_OFFSET(prb_desc_ring, tail_id);
958+
959+ VMCOREINFO_STRUCT_SIZE(prb_desc);
960+ VMCOREINFO_OFFSET(prb_desc, info);
961+ VMCOREINFO_OFFSET(prb_desc, state_var);
962+ VMCOREINFO_OFFSET(prb_desc, text_blk_lpos);
963+ VMCOREINFO_OFFSET(prb_desc, dict_blk_lpos);
964+
965+ VMCOREINFO_STRUCT_SIZE(prb_data_blk_lpos);
966+ VMCOREINFO_OFFSET(prb_data_blk_lpos, begin);
967+ VMCOREINFO_OFFSET(prb_data_blk_lpos, next);
968+
969+ VMCOREINFO_STRUCT_SIZE(printk_info);
970+ VMCOREINFO_OFFSET(printk_info, seq);
971+ VMCOREINFO_OFFSET(printk_info, ts_nsec);
972+ VMCOREINFO_OFFSET(printk_info, text_len);
973+ VMCOREINFO_OFFSET(printk_info, dict_len);
974+ VMCOREINFO_OFFSET(printk_info, caller_id);
975+
976+ VMCOREINFO_STRUCT_SIZE(prb_data_ring);
977+ VMCOREINFO_OFFSET(prb_data_ring, size_bits);
978+ VMCOREINFO_OFFSET(prb_data_ring, data);
979+ VMCOREINFO_OFFSET(prb_data_ring, head_lpos);
980+ VMCOREINFO_OFFSET(prb_data_ring, tail_lpos);
981+
982+ VMCOREINFO_SIZE(atomic_long_t);
983+ VMCOREINFO_TYPE_OFFSET(atomic_long_t, counter);
1092984 }
1093985 #endif
1094986
@@ -1166,11 +1058,48 @@ static void __init set_percpu_data_ready(void) · 71 unmodified lines
11661058 __printk_percpu_data_ready = true;
11671059 }
11681060
1061+static unsigned int __init add_to_rb(struct printk_ringbuffer *rb,
1062+ struct printk_record *r)
1063+{
1064+ struct prb_reserved_entry e;
1065+ struct printk_record dest_r;
1066+
1067+ prb_rec_init_wr(&dest_r, r->info->text_len, r->info->dict_len);
1068+
1069+ if (!prb_reserve(&e, rb, &dest_r))
1070+ return 0;
1071+
1072+ memcpy(&dest_r.text_buf[0], &r->text_buf[0], dest_r.text_buf_size);
1073+ if (dest_r.dict_buf) {
1074+ memcpy(&dest_r.dict_buf[0], &r->dict_buf[0],
1075+ dest_r.dict_buf_size);
1076+ }
1077+ dest_r.info->facility = r->info->facility;
1078+ dest_r.info->level = r->info->level;
1079+ dest_r.info->flags = r->info->flags;
1080+ dest_r.info->ts_nsec = r->info->ts_nsec;
1081+ dest_r.info->caller_id = r->info->caller_id;
1082+
1083+ prb_commit(&e);
1084+
1085+ return prb_record_text_space(&e);
1086+}
1087+
1088+static char setup_text_buf[CONSOLE_EXT_LOG_MAX] __initdata;
1089+static char setup_dict_buf[CONSOLE_EXT_LOG_MAX] __initdata;
1090+
11691091 void __init setup_log_buf(int early)
11701092 {
1093+ unsigned int new_descs_count;
1094+ struct prb_desc *new_descs;
1095+ struct printk_info info;
1096+ struct printk_record r;
1097+ size_t new_descs_size;
11711098 unsigned long flags;
1099+ char *new_dict_buf;
11721100 char *new_log_buf;
11731101 unsigned int free;
1102+ u64 seq;
11741103
11751104 /*
11761105 * Some archs call setup_log_buf() multiple times - first is very
@@ -1189,21 +1118,70 @@ void __init setup_log_buf(int early) · 12 unmodified lines
11891118 if (!new_log_buf_len)
11901119 return;
11911120
1121+ new_descs_count = new_log_buf_len >> PRB_AVGBITS;
1122+ if (new_descs_count == 0) {
1123+ pr_err("new_log_buf_len: %lu too small\n", new_log_buf_len);
1124+ return;
1125+ }
1126+
11921127 new_log_buf = memblock_alloc(new_log_buf_len, LOG_ALIGN);
11931128 if (unlikely(!new_log_buf)) {
1194− pr_err("log_buf_len: %lu bytes not available\n",
1195− new_log_buf_len);
1129+ pr_err("log_buf_len: %lu text bytes not available\n",
1130+ new_log_buf_len);
11961131 return;
11971132 }
11981133
1134+ new_dict_buf = memblock_alloc(new_log_buf_len, LOG_ALIGN);
1135+ if (unlikely(!new_dict_buf)) {
1136+ pr_err("log_buf_len: %lu dict bytes not available\n",
1137+ new_log_buf_len);
1138+ memblock_free(__pa(new_log_buf), new_log_buf_len);
1139+ return;
1140+ }
1141+
1142+ new_descs_size = new_descs_count * sizeof(struct prb_desc);
1143+ new_descs = memblock_alloc(new_descs_size, LOG_ALIGN);
1144+ if (unlikely(!new_descs)) {
1145+ pr_err("log_buf_len: %zu desc bytes not available\n",
1146+ new_descs_size);
1147+ memblock_free(__pa(new_dict_buf), new_log_buf_len);
1148+ memblock_free(__pa(new_log_buf), new_log_buf_len);
1149+ return;
1150+ }
1151+
1152+ prb_rec_init_rd(&r, &info,
1153+ &setup_text_buf[0], sizeof(setup_text_buf),
1154+ &setup_dict_buf[0], sizeof(setup_dict_buf));
1155+
1156+ prb_init(&printk_rb_dynamic,
1157+ new_log_buf, ilog2(new_log_buf_len),
1158+ new_dict_buf, ilog2(new_log_buf_len),
1159+ new_descs, ilog2(new_descs_count));
1160+
11991161 logbuf_lock_irqsave(flags);
1162+
12001163 log_buf_len = new_log_buf_len;
12011164 log_buf = new_log_buf;
12021165 new_log_buf_len = 0;
1203− free = __LOG_BUF_LEN - log_next_idx;
1204− memcpy(log_buf, __log_buf, __LOG_BUF_LEN);
1166+
1167+ free = __LOG_BUF_LEN;
1168+ prb_for_each_record(0, &printk_rb_static, seq, &r)
1169+ free -= add_to_rb(&printk_rb_dynamic, &r);
1170+
1171+ /*
1172+ * This is early enough that everything is still running on the
1173+ * boot CPU and interrupts are disabled. So no new messages will
1174+ * appear during the transition to the dynamic buffer.
1175+ */
1176+ prb = &printk_rb_dynamic;
1177+
12051178 logbuf_unlock_irqrestore(flags);
12061179
1180+ if (seq != prb_next_seq(&printk_rb_static)) {
1181+ pr_err("dropped %llu messages\n",
1182+ prb_next_seq(&printk_rb_static) - seq);
1183+ }
1184+
12071185 pr_info("log_buf_len: %u bytes\n", log_buf_len);
12081186 pr_info("early log buf free: %u(%u%%)\n",
12091187 free, (free * 100) / __LOG_BUF_LEN);
@@ -1313,18 +1291,18 @@ static size_t print_caller(u32 id, char *buf) · 103 unmodified lines
13131291 #define print_caller(id, buf) 0
13141292 #endif
13151293
1316−static size_t print_prefix(const struct printk_log *msg, bool syslog,
1317− bool time, char *buf)
1294+static size_t info_print_prefix(const struct printk_info *info, bool syslog,
1295+ bool time, char *buf)
13181296 {
13191297 size_t len = 0;
13201298
13211299 if (syslog)
1322− len = print_syslog((msg->facility << 3) | msg->level, buf);
1300+ len = print_syslog((info->facility << 3) | info->level, buf);
13231301
13241302 if (time)
1325− len += print_time(msg->ts_nsec, buf + len);
1303+ len += print_time(info->ts_nsec, buf + len);
13261304
1327− len += print_caller(msg->caller_id, buf + len);
1305+ len += print_caller(info->caller_id, buf + len);
13281306
13291307 if (IS_ENABLED(CONFIG_PRINTK_CALLER) || time) {
13301308 buf[len++] = ' ';
@@ -1334,72 +1312,143 @@ static size_t print_prefix(const struct printk_log *msg, bool syslog, · 3 unmodified lines
13341312 return len;
13351313 }
13361314
1337−static size_t msg_print_text(const struct printk_log *msg, bool syslog,
1338− bool time, char *buf, size_t size)
1315+/*
1316+ * Prepare the record for printing. The text is shifted within the given
1317+ * buffer to avoid a need for another one. The following operations are
1318+ * done:
1319+ *
1320+ * - Add prefix for each line.
1321+ * - Add the trailing newline that has been removed in vprintk_store().
1322+ * - Drop truncated lines that do not longer fit into the buffer.
1323+ *
1324+ * Return: The length of the updated/prepared text, including the added
1325+ * prefixes and the newline. The dropped line(s) are not counted.
1326+ */
1327+static size_t record_print_text(struct printk_record *r, bool syslog,
1328+ bool time)
13391329 {
1340− const char *text = log_text(msg);
1341− size_t text_size = msg->text_len;
1342− size_t len = 0;
1330+ size_t text_len = r->info->text_len;
1331+ size_t buf_size = r->text_buf_size;
1332+ char *text = r->text_buf;
13431333 char prefix[PREFIX_MAX];
1344− const size_t prefix_len = print_prefix(msg, syslog, time, prefix);
1334+ bool truncated = false;
1335+ size_t prefix_len;
1336+ size_t line_len;
1337+ size_t len = 0;
1338+ char *next;
13451339
1346− do {
1347− const char *next = memchr(text, '\n', text_size);
1348− size_t text_len;
1340+ prefix_len = info_print_prefix(r->info, syslog, time, prefix);
13491341
1342+ /*
1343+ * @text_len: bytes of unprocessed text
1344+ * @line_len: bytes of current line _without_ newline
1345+ * @text: pointer to beginning of current line
1346+ * @len: number of bytes prepared in r->text_buf
1347+ */
1348+ for (;;) {
1349+ next = memchr(text, '\n', text_len);
13501350 if (next) {
1351− text_len = next - text;
1352− next++;
1353− text_size -= next - text;
1351+ line_len = next - text;
13541352 } else {
1355− text_len = text_size;
1353+ /* Drop truncated line(s). */
1354+ if (truncated)
1355+ break;
1356+ line_len = text_len;
13561357 }
13571358
1358− if (buf) {
1359− if (prefix_len + text_len + 1 >= size - len)
1359+ /*
1360+ * Truncate the text if there is not enough space to add the
1361+ * prefix and a trailing newline.
1362+ */
1363+ if (len + prefix_len + text_len + 1 > buf_size) {
1364+ /* Drop even the current line if no space. */
1365+ if (len + prefix_len + line_len + 1 > buf_size)
13601366 break;
13611367
1362− memcpy(buf + len, prefix, prefix_len);
1363− len += prefix_len;
1364− memcpy(buf + len, text, text_len);
1365− len += text_len;
1366− buf[len++] = '\n';
1367− } else {
1368− /* SYSLOG_ACTION_* buffer size only calculation */
1369− len += prefix_len + text_len + 1;
1368+ text_len = buf_size - len - prefix_len - 1;
1369+ truncated = true;
13701370 }
13711371
1372− text = next;
1373− } while (text);
1372+ memmove(text + prefix_len, text, text_len);
1373+ memcpy(text, prefix, prefix_len);
1374+
1375+ len += prefix_len + line_len + 1;
1376+
1377+ if (text_len == line_len) {
1378+ /*
1379+ * Add the trailing newline removed in
1380+ * vprintk_store().
1381+ */
1382+ text[prefix_len + line_len] = '\n';
1383+ break;
1384+ }
1385+
1386+ /*
1387+ * Advance beyond the added prefix and the related line with
1388+ * its newline.
1389+ */
1390+ text += prefix_len + line_len + 1;
1391+
1392+ /*
1393+ * The remaining text has only decreased by the line with its
1394+ * newline.
1395+ *
1396+ * Note that @text_len can become zero. It happens when @text
1397+ * ended with a newline (either due to truncation or the
1398+ * original string ending with "\n\n"). The loop is correctly
1399+ * repeated and (if not truncated) an empty line with a prefix
1400+ * will be prepared.
1401+ */
1402+ text_len -= line_len + 1;
1403+ }
13741404
13751405 return len;
13761406 }
13771407
1408+static size_t get_record_print_text_size(struct printk_info *info,
1409+ unsigned int line_count,
1410+ bool syslog, bool time)
1411+{
1412+ char prefix[PREFIX_MAX];
1413+ size_t prefix_len;
1414+
1415+ prefix_len = info_print_prefix(info, syslog, time, prefix);
1416+
1417+ /*
1418+ * Each line will be preceded with a prefix. The intermediate
1419+ * newlines are already within the text, but a final trailing
1420+ * newline will be added.
1421+ */
1422+ return ((prefix_len * line_count) + info->text_len + 1);
1423+}
1424+
13781425 static int syslog_print(char __user *buf, int size)
13791426 {
1427+ struct printk_info info;
1428+ struct printk_record r;
13801429 char *text;
1381− struct printk_log *msg;
13821430 int len = 0;
13831431
13841432 text = kmalloc(LOG_LINE_MAX + PREFIX_MAX, GFP_KERNEL);
13851433 if (!text)
13861434 return -ENOMEM;
13871435
1436+ prb_rec_init_rd(&r, &info, text, LOG_LINE_MAX + PREFIX_MAX, NULL, 0);
1437+
13881438 while (size > 0) {
13891439 size_t n;
13901440 size_t skip;
13911441
13921442 logbuf_lock_irq();
1393− if (syslog_seq < log_first_seq) {
1394− /* messages are gone, move to first one */
1395− syslog_seq = log_first_seq;
1396− syslog_idx = log_first_idx;
1397− syslog_partial = 0;
1398− }
1399− if (syslog_seq == log_next_seq) {
1443+ if (!prb_read_valid(prb, syslog_seq, &r)) {
14001444 logbuf_unlock_irq();
14011445 break;
14021446 }
1447+ if (r.info->seq != syslog_seq) {
1448+ /* message is gone, move to next valid one */
1449+ syslog_seq = r.info->seq;
1450+ syslog_partial = 0;
1451+ }
14031452
14041453 /*
14051454 * To keep reading/counting partial line consistent,
@@ -1409,13 +1458,10 @@ static int syslog_print(char __user *buf, int size) · 3 unmodified lines
14091458 syslog_time = printk_time;
14101459
14111460 skip = syslog_partial;
1412− msg = log_from_idx(syslog_idx);
1413− n = msg_print_text(msg, true, syslog_time, text,
1414− LOG_LINE_MAX + PREFIX_MAX);
1461+ n = record_print_text(&r, true, syslog_time);
14151462 if (n - syslog_partial <= size) {
14161463 /* message fits into buffer, move forward */
1417− syslog_idx = log_next(syslog_idx);
1418− syslog_seq++;
1464+ syslog_seq = r.info->seq + 1;
14191465 n -= syslog_partial;
14201466 syslog_partial = 0;
14211467 } else if (!len){
@@ -1446,11 +1492,12 @@ static int syslog_print(char __user *buf, int size) · 24 unmodified lines
14461492
14471493 static int syslog_print_all(char __user *buf, int size, bool clear)
14481494 {
1495+ struct printk_info info;
1496+ unsigned int line_count;
1497+ struct printk_record r;
14491498 char *text;
14501499 int len = 0;
1451− u64 next_seq;
14521500 u64 seq;
1453− u32 idx;
14541501 bool time;
14551502
14561503 text = kmalloc(LOG_LINE_MAX + PREFIX_MAX, GFP_KERNEL);
@@ -1463,38 +1510,28 @@ static int syslog_print_all(char __user *buf, int size, bool clear) · 6 unmodified lines
14631510 * Find first record that fits, including all following records,
14641511 * into the user-provided buffer for this dump.
14651512 */
1466− seq = clear_seq;
1467− idx = clear_idx;
1468− while (seq < log_next_seq) {
1469− struct printk_log *msg = log_from_idx(idx);
1470−
1471− len += msg_print_text(msg, true, time, NULL, 0);
1472− idx = log_next(idx);
1473− seq++;
1474− }
1513+ prb_for_each_info(clear_seq, prb, seq, &info, &line_count)
1514+ len += get_record_print_text_size(&info, line_count, true, time);
14751515
14761516 /* move first record forward until length fits into the buffer */
1477− seq = clear_seq;
1478− idx = clear_idx;
1479− while (len > size && seq < log_next_seq) {
1480− struct printk_log *msg = log_from_idx(idx);
1481−
1482− len -= msg_print_text(msg, true, time, NULL, 0);
1483− idx = log_next(idx);
1484− seq++;
1517+ prb_for_each_info(clear_seq, prb, seq, &info, &line_count) {
1518+ if (len <= size)
1519+ break;
1520+ len -= get_record_print_text_size(&info, line_count, true, time);
14851521 }
14861522
1487− /* last message fitting into this dump */
1488− next_seq = log_next_seq;
1523+ prb_rec_init_rd(&r, &info, text, LOG_LINE_MAX + PREFIX_MAX, NULL, 0);
14891524
14901525 len = 0;
1491− while (len >= 0 && seq < next_seq) {
1492− struct printk_log *msg = log_from_idx(idx);
1493− int textlen = msg_print_text(msg, true, time, text,
1494− LOG_LINE_MAX + PREFIX_MAX);
1526+ prb_for_each_record(seq, prb, seq, &r) {
1527+ int textlen;
14951528
1496− idx = log_next(idx);
1497− seq++;
1529+ textlen = record_print_text(&r, true, time);
1530+
1531+ if (len + textlen > size) {
1532+ seq--;
1533+ break;
1534+ }
14981535
14991536 logbuf_unlock_irq();
15001537 if (copy_to_user(buf + len, text, textlen))
@@ -1503,17 +1540,12 @@ static int syslog_print_all(char __user *buf, int size, bool clear) · 2 unmodified lines
15031540 len += textlen;
15041541 logbuf_lock_irq();
15051542
1506− if (seq < log_first_seq) {
1507− /* messages are gone, move to next one */
1508− seq = log_first_seq;
1509− idx = log_first_idx;
1510− }
1543+ if (len < 0)
1544+ break;
15111545 }
15121546
1513− if (clear) {
1514− clear_seq = log_next_seq;
1515− clear_idx = log_next_idx;
1516− }
1547+ if (clear)
1548+ clear_seq = seq;
15171549 logbuf_unlock_irq();
15181550
15191551 kfree(text);
@@ -1523,8 +1555,7 @@ static int syslog_print_all(char __user *buf, int size, bool clear) · 3 unmodified lines
15231555 static void syslog_clear(void)
15241556 {
15251557 logbuf_lock_irq();
1526− clear_seq = log_next_seq;
1527− clear_idx = log_next_idx;
1558+ clear_seq = prb_next_seq(prb);
15281559 logbuf_unlock_irq();
15291560 }
15301561
@@ -1551,7 +1582,7 @@ int do_syslog(int type, char __user *buf, int len, int source) · 20 unmodified lines
15511582 if (!access_ok(buf, len))
15521583 return -EFAULT;
15531584 error = wait_event_interruptible(log_wait,
1554− syslog_seq != log_next_seq);
1585+ prb_read_valid(prb, syslog_seq, NULL));
15551586 if (error)
15561587 return error;
15571588 error = syslog_print(buf, len);
@@ -1600,10 +1631,9 @@ int do_syslog(int type, char __user *buf, int len, int source) · 42 unmodified lines
16001631 /* Number of chars in the log buffer */
16011632 case SYSLOG_ACTION_SIZE_UNREAD:
16021633 logbuf_lock_irq();
1603− if (syslog_seq < log_first_seq) {
1634+ if (syslog_seq < prb_first_valid_seq(prb)) {
16041635 /* messages are gone, move to first one */
1605− syslog_seq = log_first_seq;
1606− syslog_idx = log_first_idx;
1636+ syslog_seq = prb_first_valid_seq(prb);
16071637 syslog_partial = 0;
16081638 }
16091639 if (source == SYSLOG_FROM_PROC) {
@@ -1612,20 +1642,18 @@ int do_syslog(int type, char __user *buf, int len, int source) · 2 unmodified lines
16121642 * for pending data, not the size; return the count of
16131643 * records, not the length.
16141644 */
1615− error = log_next_seq - syslog_seq;
1645+ error = prb_next_seq(prb) - syslog_seq;
16161646 } else {
1617− u64 seq = syslog_seq;
1618− u32 idx = syslog_idx;
16191647 bool time = syslog_partial ? syslog_time : printk_time;
1620−
1621− while (seq < log_next_seq) {
1622− struct printk_log *msg = log_from_idx(idx);
1623−
1624− error += msg_print_text(msg, true, time, NULL,
1625− 0);
1648+ struct printk_info info;
1649+ unsigned int line_count;
1650+ u64 seq;
1651+
1652+ prb_for_each_info(syslog_seq, prb, seq, &info,
1653+ &line_count) {
1654+ error += get_record_print_text_size(&info, line_count,
1655+ true, time);
16261656 time = printk_time;
1627− idx = log_next(idx);
1628− seq++;
16291657 }
16301658 error -= syslog_partial;
16311659 }
@@ -1796,10 +1824,22 @@ static int console_trylock_spinning(void) · 164 unmodified lines
17961824 static void call_console_drivers(const char *ext_text, size_t ext_len,
17971825 const char *text, size_t len)
17981826 {
1827+ static char dropped_text[64];
1828+ size_t dropped_len = 0;
17991829 struct console *con;
18001830
18011831 trace_console_rcuidle(text, len);
18021832
1833+ if (!console_drivers)
1834+ return;
1835+
1836+ if (console_dropped) {
1837+ dropped_len = snprintf(dropped_text, sizeof(dropped_text),
1838+ "** %lu printk messages dropped **\n",
1839+ console_dropped);
1840+ console_dropped = 0;
1841+ }
1842+
18031843 for_each_console(con) {
18041844 if (exclusive_console && con != exclusive_console)
18051845 continue;
@@ -1812,8 +1852,11 @@ static void call_console_drivers(const char *ext_text, size_t ext_len, · 6 unmodified lines
18121852 continue;
18131853 if (con->flags & CON_EXTENDED)
18141854 con->write(con, ext_text, ext_len);
1815− else
1855+ else {
1856+ if (dropped_len)
1857+ con->write(con, dropped_text, dropped_len);
18161858 con->write(con, text, len);
1859+ }
18171860 }
18181861 }
18191862
@@ -2076,21 +2119,24 @@ EXPORT_SYMBOL(printk); · 256 unmodified lines
20762119 #define PREFIX_MAX 0
20772120 #define printk_time false
20782121
2122+#define prb_read_valid(rb, seq, r) false
2123+#define prb_first_valid_seq(rb) 0
2124+
20792125 static u64 syslog_seq;
2080−static u32 syslog_idx;
20812126 static u64 console_seq;
2082−static u32 console_idx;
20832127 static u64 exclusive_console_stop_seq;
2084−static u64 log_first_seq;
2085−static u32 log_first_idx;
2086−static u64 log_next_seq;
2087−static char *log_text(const struct printk_log *msg) { return NULL; }
2088−static char *log_dict(const struct printk_log *msg) { return NULL; }
2089−static struct printk_log *log_from_idx(u32 idx) { return NULL; }
2090−static u32 log_next(u32 idx) { return 0; }
2091−static ssize_t msg_print_ext_header(char *buf, size_t size,
2092− struct printk_log *msg,
2093− u64 seq) { return 0; }
2128+static unsigned long console_dropped;
2129+
2130+static size_t record_print_text(const struct printk_record *r,
2131+ bool syslog, bool time)
2132+{
2133+ return 0;
2134+}
2135+static ssize_t info_print_ext_header(char *buf, size_t size,
2136+ struct printk_info *info)
2137+{
2138+ return 0;
2139+}
20942140 static ssize_t msg_print_ext_body(char *buf, size_t size,
20952141 char *dict, size_t dict_len,
20962142 char *text, size_t text_len) { return 0; }
@@ -2098,8 +2144,6 @@ static void console_lock_spinning_enable(void) { } · 1 unmodified line
20982144 static int console_lock_spinning_disable_and_check(void) { return 0; }
20992145 static void call_console_drivers(const char *ext_text, size_t ext_len,
21002146 const char *text, size_t len) {}
2101−static size_t msg_print_text(const struct printk_log *msg, bool syslog,
2102− bool time, char *buf, size_t size) { return 0; }
21032147 static bool suppress_message_printing(int level) { return false; }
21042148
21052149 #endif /* CONFIG_PRINTK */
@@ -2384,14 +2428,19 @@ void console_unlock(void) · 278 unmodified lines
23842428 {
23852429 static char ext_text[CONSOLE_EXT_LOG_MAX];
23862430 static char text[LOG_LINE_MAX + PREFIX_MAX];
2431+ static char dict[LOG_LINE_MAX];
23872432 unsigned long flags;
23882433 bool do_cond_resched, retry;
2434+ struct printk_info info;
2435+ struct printk_record r;
23892436
23902437 if (console_suspended) {
23912438 up_console_sem();
23922439 return;
23932440 }
23942441
2442+ prb_rec_init_rd(&r, &info, text, sizeof(text), dict, sizeof(dict));
2443+
23952444 /*
23962445 * Console drivers are called with interrupts disabled, so
23972446 * @console_may_schedule should be cleared before; however, we may
@@ -2422,35 +2471,26 @@ again: · 24 unmodified lines
24222471 }
24232472
24242473 for (;;) {
2425− struct printk_log *msg;
24262474 size_t ext_len = 0;
24272475 size_t len;
24282476
24292477 printk_safe_enter_irqsave(flags);
24302478 raw_spin_lock(&logbuf_lock);
2431− if (console_seq < log_first_seq) {
2432− len = snprintf(text, sizeof(text),
2433− "** %llu printk messages dropped **\n",
2434− log_first_seq - console_seq);
2435−
2436− /* messages are gone, move to first one */
2437− console_seq = log_first_seq;
2438− console_idx = log_first_idx;
2439− } else {
2440− len = 0;
2441− }
24422479 skip:
2443− if (console_seq == log_next_seq)
2480+ if (!prb_read_valid(prb, console_seq, &r))
24442481 break;
24452482
2446− msg = log_from_idx(console_idx);
2447− if (suppress_message_printing(msg->level)) {
2483+ if (console_seq != r.info->seq) {
2484+ console_dropped += r.info->seq - console_seq;
2485+ console_seq = r.info->seq;
2486+ }
2487+
2488+ if (suppress_message_printing(r.info->level)) {
24482489 /*
24492490 * Skip record we have buffered and already printed
24502491 * directly to the console when we received it, and
24512492 * record that has level above the console loglevel.
24522493 */
2453− console_idx = log_next(console_idx);
24542494 console_seq++;
24552495 goto skip;
24562496 }
@@ -2461,19 +2501,24 @@ skip: · 4 unmodified lines
24612501 exclusive_console = NULL;
24622502 }
24632503
2464− len += msg_print_text(msg,
2465− console_msg_format & MSG_FORMAT_SYSLOG,
2466− printk_time, text + len, sizeof(text) - len);
2504+ /*
2505+ * Handle extended console text first because later
2506+ * record_print_text() will modify the record buffer in-place.
2507+ */
24672508 if (nr_ext_console_drivers) {
2468− ext_len = msg_print_ext_header(ext_text,
2509+ ext_len = info_print_ext_header(ext_text,
24692510 sizeof(ext_text),
2470− msg, console_seq);
2511+ r.info);
24712512 ext_len += msg_print_ext_body(ext_text + ext_len,
24722513 sizeof(ext_text) - ext_len,
2473− log_dict(msg), msg->dict_len,
2474− log_text(msg), msg->text_len);
2514+ &r.dict_buf[0],
2515+ r.info->dict_len,
2516+ &r.text_buf[0],
2517+ r.info->text_len);
24752518 }
2476− console_idx = log_next(console_idx);
2519+ len = record_print_text(&r,
2520+ console_msg_format & MSG_FORMAT_SYSLOG,
2521+ printk_time);
24772522 console_seq++;
24782523 raw_spin_unlock(&logbuf_lock);
24792524
@@ -2513,7 +2558,7 @@ skip: · 33 unmodified lines
25132558 * flush, no worries.
25142559 */
25152560 raw_spin_lock(&logbuf_lock);
2516− retry = console_seq != log_next_seq;
2561+ retry = prb_read_valid(prb, console_seq, NULL);
25172562 raw_spin_unlock(&logbuf_lock);
25182563 printk_safe_exit_irqrestore(flags);
25192564
@@ -2582,8 +2627,7 @@ void console_flush_on_panic(enum con_flush_mode mode) · 62 unmodified lines
25822627 unsigned long flags;
25832628
25842629 logbuf_lock_irqsave(flags);
2585− console_seq = log_first_seq;
2586− console_idx = log_first_idx;
2630+ console_seq = prb_first_valid_seq(prb);
25872631 logbuf_unlock_irqrestore(flags);
25882632 }
25892633 console_unlock();
@@ -2826,7 +2870,6 @@ void register_console(struct console *newcon) · 236 unmodified lines
28262870 exclusive_console = newcon;
28272871 exclusive_console_stop_seq = console_seq;
28282872 console_seq = syslog_seq;
2829− console_idx = syslog_idx;
28302873 logbuf_unlock_irqrestore(flags);
28312874 }
28322875 console_unlock();
@@ -3215,9 +3258,7 @@ void kmsg_dump(enum kmsg_dump_reason reason) · 382 unmodified lines
32153258
32163259 logbuf_lock_irqsave(flags);
32173260 dumper->cur_seq = clear_seq;
3218− dumper->cur_idx = clear_idx;
3219− dumper->next_seq = log_next_seq;
3220− dumper->next_idx = log_next_idx;
3261+ dumper->next_seq = prb_next_seq(prb);
32213262 logbuf_unlock_irqrestore(flags);
32223263
32233264 /* invoke dumper which will iterate over records */
@@ -3251,28 +3292,33 @@ void kmsg_dump(enum kmsg_dump_reason reason) · 27 unmodified lines
32513292 bool kmsg_dump_get_line_nolock(struct kmsg_dumper *dumper, bool syslog,
32523293 char *line, size_t size, size_t *len)
32533294 {
3254− struct printk_log *msg;
3295+ struct printk_info info;
3296+ unsigned int line_count;
3297+ struct printk_record r;
32553298 size_t l = 0;
32563299 bool ret = false;
32573300
3301+ prb_rec_init_rd(&r, &info, line, size, NULL, 0);
3302+
32583303 if (!dumper->active)
32593304 goto out;
32603305
3261− if (dumper->cur_seq < log_first_seq) {
3262− /* messages are gone, move to first available one */
3263− dumper->cur_seq = log_first_seq;
3264− dumper->cur_idx = log_first_idx;
3265− }
3266−
3267− /* last entry */
3268− if (dumper->cur_seq >= log_next_seq)
3269− goto out;
3306+ /* Read text or count text lines? */
3307+ if (line) {
3308+ if (!prb_read_valid(prb, dumper->cur_seq, &r))
3309+ goto out;
3310+ l = record_print_text(&r, syslog, printk_time);
3311+ } else {
3312+ if (!prb_read_valid_info(prb, dumper->cur_seq,
3313+ &info, &line_count)) {
3314+ goto out;
3315+ }
3316+ l = get_record_print_text_size(&info, line_count, syslog,
3317+ printk_time);
32703318
3271− msg = log_from_idx(dumper->cur_idx);
3272− l = msg_print_text(msg, syslog, printk_time, line, size);
3319+ }
32733320
3274− dumper->cur_idx = log_next(dumper->cur_idx);
3275− dumper->cur_seq++;
3321+ dumper->cur_seq = r.info->seq + 1;
32763322 ret = true;
32773323 out:
32783324 if (len)
@@ -3333,23 +3379,25 @@ EXPORT_SYMBOL_GPL(kmsg_dump_get_line); · 54 unmodified lines
33333379 bool kmsg_dump_get_buffer(struct kmsg_dumper *dumper, bool syslog,
33343380 char *buf, size_t size, size_t *len)
33353381 {
3382+ struct printk_info info;
3383+ unsigned int line_count;
3384+ struct printk_record r;
33363385 unsigned long flags;
33373386 u64 seq;
3338− u32 idx;
33393387 u64 next_seq;
3340− u32 next_idx;
33413388 size_t l = 0;
33423389 bool ret = false;
33433390 bool time = printk_time;
33443391
3345− if (!dumper->active)
3392+ prb_rec_init_rd(&r, &info, buf, size, NULL, 0);
3393+
3394+ if (!dumper->active || !buf || !size)
33463395 goto out;
33473396
33483397 logbuf_lock_irqsave(flags);
3349− if (dumper->cur_seq < log_first_seq) {
3398+ if (dumper->cur_seq < prb_first_valid_seq(prb)) {
33503399 /* messages are gone, move to first available one */
3351− dumper->cur_seq = log_first_seq;
3352− dumper->cur_idx = log_first_idx;
3400+ dumper->cur_seq = prb_first_valid_seq(prb);
33533401 }
33543402
33553403 /* last entry */
@@ -3360,41 +3408,41 @@ bool kmsg_dump_get_buffer(struct kmsg_dumper *dumper, bool syslog, · 4 unmodified lines
33603408
33613409 /* calculate length of entire buffer */
33623410 seq = dumper->cur_seq;
3363− idx = dumper->cur_idx;
3364− while (seq < dumper->next_seq) {
3365− struct printk_log *msg = log_from_idx(idx);
3366−
3367− l += msg_print_text(msg, true, time, NULL, 0);
3368− idx = log_next(idx);
3369− seq++;
3411+ while (prb_read_valid_info(prb, seq, &info, &line_count)) {
3412+ if (r.info->seq >= dumper->next_seq)
3413+ break;
3414+ l += get_record_print_text_size(&info, line_count, true, time);
3415+ seq = r.info->seq + 1;
33703416 }
33713417
33723418 /* move first record forward until length fits into the buffer */
33733419 seq = dumper->cur_seq;
3374− idx = dumper->cur_idx;
3375− while (l >= size && seq < dumper->next_seq) {
3376− struct printk_log *msg = log_from_idx(idx);
3377−
3378− l -= msg_print_text(msg, true, time, NULL, 0);
3379− idx = log_next(idx);
3380− seq++;
3420+ while (l >= size && prb_read_valid_info(prb, seq,
3421+ &info, &line_count)) {
3422+ if (r.info->seq >= dumper->next_seq)
3423+ break;
3424+ l -= get_record_print_text_size(&info, line_count, true, time);
3425+ seq = r.info->seq + 1;
33813426 }
33823427
33833428 /* last message in next interation */
33843429 next_seq = seq;
3385− next_idx = idx;
33863430
3431+ /* actually read text into the buffer now */
33873432 l = 0;
3388− while (seq < dumper->next_seq) {
3389− struct printk_log *msg = log_from_idx(idx);
3433+ while (prb_read_valid(prb, seq, &r)) {
3434+ if (r.info->seq >= dumper->next_seq)
3435+ break;
3436+
3437+ l += record_print_text(&r, syslog, time);
3438+
3439+ /* adjust record to store to remaining buffer space */
3440+ prb_rec_init_rd(&r, &info, buf + l, size - l, NULL, 0);
33903441
3391− l += msg_print_text(msg, syslog, time, buf + l, size - l);
3392− idx = log_next(idx);
3393− seq++;
3442+ seq = r.info->seq + 1;
33943443 }
33953444
33963445 dumper->next_seq = next_seq;
3397− dumper->next_idx = next_idx;
33983446 ret = true;
33993447 logbuf_unlock_irqrestore(flags);
34003448 out:
@@ -3417,9 +3465,7 @@ EXPORT_SYMBOL_GPL(kmsg_dump_get_buffer); · 16 unmodified lines
34173465 void kmsg_dump_rewind_nolock(struct kmsg_dumper *dumper)
34183466 {
34193467 dumper->cur_seq = clear_seq;
3420− dumper->cur_idx = clear_idx;
3421− dumper->next_seq = log_next_seq;
3422− dumper->next_idx = log_next_idx;
3468+ dumper->next_seq = prb_next_seq(prb);
34233469 }
34243470
34253471 /**

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t