History

Merge tag 'trace-v6.8' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace

Pull tracing updates from Steven Rostedt: - Allow kernel trace instance creation to specify what events are created Inside the kernel, a subsystem may create a tracing instance that it can use to send events to user space. This sub-system may not care about the thousands of events that exist in eventfs. Allow the sub-system to specify what sub-systems of events it cares about, and only those events are exposed to this instance. - Allow the ring buffer to be broken up into bigger sub-buffers than just the architecture page size. A new tracefs file called "buffer_subbuf_size_kb" is created. The user can now specify a minimum size the sub-buffer may be in kilobytes. Note, that the implementation currently make the sub-buffer size a power of 2 pages (1, 2, 4, 8, 16, ...) but the user only writes in kilobyte size, and the sub-buffer will be updated to the next size that it will can accommodate it. If the user writes in 10, it will change the size to be 4 pages on x86 (16K), as that is the next available size that can hold 10K pages. - Update the debug output when a corrupt time is detected in the ring buffer. If the ring buffer detects inconsistent timestamps, there's a debug config options that will dump the contents of the meta data of the sub-buffer that is used for debugging. Add some more information to this dump that helps with debugging. - Add more timestamp debugging checks (only triggers when the config is enabled) - Increase the trace_seq iterator to 2 page sizes. - Allow strings written into tracefs_marker to be larger. Up to just under 2 page sizes (based on what trace_seq can hold). - Increase the trace_maker_raw write to be as big as a sub-buffer can hold. - Remove 32 bit time stamp logic, now that the rb_time_cmpxchg() has been removed. - More selftests were added. - Some code clean ups as well. * tag 'trace-v6.8' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace: (29 commits) ring-buffer: Remove stale comment from ring_buffer_size() tracing histograms: Simplify parse_actions() function tracing/selftests: Remove exec permissions from trace_marker.tc test ring-buffer: Use subbuf_order for buffer page masking tracing: Update subbuffer with kilobytes not page order ringbuffer/selftest: Add basic selftest to test changing subbuf order ring-buffer: Add documentation on the buffer_subbuf_order file ring-buffer: Just update the subbuffers when changing their allocation order ring-buffer: Keep the same size when updating the order tracing: Stop the tracing while changing the ring buffer subbuf size tracing: Update snapshot order along with main buffer order ring-buffer: Make sure the spare sub buffer used for reads has same size ring-buffer: Do no swap cpu buffers if order is different ring-buffer: Clear pages on error in ring_buffer_subbuf_order_set() failure ring-buffer: Read and write to ring buffers with custom sub buffer size ring-buffer: Set new size of the ring buffer sub page ring-buffer: Add interface for configuring trace sub buffer size ring-buffer: Page size per ring buffer ring-buffer: Have ring_buffer_print_page_header() be able to access ring_buffer_iter ring-buffer: Check if absolute timestamp goes backwards ...

Linus Torvalds committed 2 years ago

Commit a2ded782 parents 5b890ad 25742ae16 files · +999 −374Browse files

Showing the changes to kernel/trace/ring_buffer.c (1 of 16 files). Show all

Unified · Split

kernel/trace/ring_buffer.c+458−275
@@ -27,6 +27,7 @@ · 26 unmodified lines
2727 #include <linux/cpu.h>
2828 #include <linux/oom.h>
2929
30+#include <asm/local64.h>
3031 #include <asm/local.h>
3132
3233 /*
@@ -317,6 +318,11 @@ struct buffer_data_page { · 284 unmodified lines
317318 unsigned char data[] RB_ALIGN_DATA; /* data of buffer page */
318319 };
319320
321+struct buffer_data_read_page {
322+ unsigned order; /* order of the page */
323+ struct buffer_data_page *data; /* actual data, stored in this page */
324+};
325+
320326 /*
321327 * Note, the buffer_page list must be first. The buffer pages
322328 * are allocated in cache lines, which means that each buffer
@@ -331,6 +337,7 @@ struct buffer_page { · 8 unmodified lines
331337 unsigned read; /* index for next read */
332338 local_t entries; /* entries on this page */
333339 unsigned long real_end; /* real end of data */
340+ unsigned order; /* order of the page */
334341 struct buffer_data_page *page; /* Actual data page */
335342 };
336343
@@ -361,7 +368,7 @@ static __always_inline unsigned int rb_page_commit(struct buffer_page *bpage) · 24 unmodified lines
361368
362369 static void free_buffer_page(struct buffer_page *bpage)
363370 {
364− free_page((unsigned long)bpage->page);
371+ free_pages((unsigned long)bpage->page, bpage->order);
365372 kfree(bpage);
366373 }
367374
@@ -373,41 +380,6 @@ static inline bool test_time_stamp(u64 delta) · 5 unmodified lines
373380 return !!(delta & TS_DELTA_TEST);
374381 }
375382
376−#define BUF_PAGE_SIZE (PAGE_SIZE - BUF_PAGE_HDR_SIZE)
377−
378−/* Max payload is BUF_PAGE_SIZE - header (8bytes) */
379−#define BUF_MAX_DATA_SIZE (BUF_PAGE_SIZE - (sizeof(u32) * 2))
380−
381−int ring_buffer_print_page_header(struct trace_seq *s)
382−{
383− struct buffer_data_page field;
384−
385− trace_seq_printf(s, "\tfield: u64 timestamp;\t"
386− "offset:0;\tsize:%u;\tsigned:%u;\n",
387− (unsigned int)sizeof(field.time_stamp),
388− (unsigned int)is_signed_type(u64));
389−
390− trace_seq_printf(s, "\tfield: local_t commit;\t"
391− "offset:%u;\tsize:%u;\tsigned:%u;\n",
392− (unsigned int)offsetof(typeof(field), commit),
393− (unsigned int)sizeof(field.commit),
394− (unsigned int)is_signed_type(long));
395−
396− trace_seq_printf(s, "\tfield: int overwrite;\t"
397− "offset:%u;\tsize:%u;\tsigned:%u;\n",
398− (unsigned int)offsetof(typeof(field), commit),
399− 1,
400− (unsigned int)is_signed_type(long));
401−
402− trace_seq_printf(s, "\tfield: char data;\t"
403− "offset:%u;\tsize:%u;\tsigned:%u;\n",
404− (unsigned int)offsetof(typeof(field), data),
405− (unsigned int)BUF_PAGE_SIZE,
406− (unsigned int)is_signed_type(char));
407−
408− return !trace_seq_has_overflowed(s);
409−}
410−
411383 struct rb_irq_work {
412384 struct irq_work work;
413385 wait_queue_head_t waiters;
@@ -463,27 +435,9 @@ enum { · 49 unmodified lines
463435 RB_CTX_MAX
464436 };
465437
466−#if BITS_PER_LONG == 32
467−#define RB_TIME_32
468−#endif
469−
470−/* To test on 64 bit machines */
471−//#define RB_TIME_32
472−
473−#ifdef RB_TIME_32
474−
475−struct rb_time_struct {
476− local_t cnt;
477− local_t top;
478− local_t bottom;
479− local_t msb;
480−};
481−#else
482−#include <asm/local64.h>
483438 struct rb_time_struct {
484439 local64_t time;
485440 };
486−#endif
487441 typedef struct rb_time_struct rb_time_t;
488442
489443 #define MAX_NEST 5
@@ -557,6 +511,10 @@ struct trace_buffer { · 67 unmodified lines
557511
558512 struct rb_irq_work irq_work;
559513 bool time_stamp_abs;
514+
515+ unsigned int subbuf_size;
516+ unsigned int subbuf_order;
517+ unsigned int max_data_size;
560518 };
561519
562520 struct ring_buffer_iter {
@@ -570,150 +528,48 @@ struct ring_buffer_iter { · 7 unmodified lines
570528 u64 read_stamp;
571529 u64 page_stamp;
572530 struct ring_buffer_event *event;
531+ size_t event_size;
573532 int missed_events;
574533 };
575534
576−#ifdef RB_TIME_32
577−
578−/*
579− * On 32 bit machines, local64_t is very expensive. As the ring
580− * buffer doesn't need all the features of a true 64 bit atomic,
581− * on 32 bit, it uses these functions (64 still uses local64_t).
582− *
583− * For the ring buffer, 64 bit required operations for the time is
584− * the following:
585− *
586− * - Reads may fail if it interrupted a modification of the time stamp.
587− * It will succeed if it did not interrupt another write even if
588− * the read itself is interrupted by a write.
589− * It returns whether it was successful or not.
590− *
591− * - Writes always succeed and will overwrite other writes and writes
592− * that were done by events interrupting the current write.
593− *
594− * - A write followed by a read of the same time stamp will always succeed,
595− * but may not contain the same value.
596− *
597− * - A cmpxchg will fail if it interrupted another write or cmpxchg.
598− * Other than that, it acts like a normal cmpxchg.
599− *
600− * The 60 bit time stamp is broken up by 30 bits in a top and bottom half
601− * (bottom being the least significant 30 bits of the 60 bit time stamp).
602− *
603− * The two most significant bits of each half holds a 2 bit counter (0-3).
604− * Each update will increment this counter by one.
605− * When reading the top and bottom, if the two counter bits match then the
606− * top and bottom together make a valid 60 bit number.
607− */
608−#define RB_TIME_SHIFT 30
609−#define RB_TIME_VAL_MASK ((1 << RB_TIME_SHIFT) - 1)
610−#define RB_TIME_MSB_SHIFT 60
611−
612−static inline int rb_time_cnt(unsigned long val)
535+int ring_buffer_print_page_header(struct trace_buffer *buffer, struct trace_seq *s)
613536 {
614− return (val >> RB_TIME_SHIFT) & 3;
615−}
616−
617−static inline u64 rb_time_val(unsigned long top, unsigned long bottom)
618−{
619− u64 val;
620−
621− val = top & RB_TIME_VAL_MASK;
622− val <<= RB_TIME_SHIFT;
623− val |= bottom & RB_TIME_VAL_MASK;
624−
625− return val;
626−}
627−
628−static inline bool __rb_time_read(rb_time_t *t, u64 *ret, unsigned long *cnt)
629−{
630− unsigned long top, bottom, msb;
631− unsigned long c;
632−
633− /*
634− * If the read is interrupted by a write, then the cnt will
635− * be different. Loop until both top and bottom have been read
636− * without interruption.
637− */
638− do {
639− c = local_read(&t->cnt);
640− top = local_read(&t->top);
641− bottom = local_read(&t->bottom);
642− msb = local_read(&t->msb);
643− } while (c != local_read(&t->cnt));
644−
645− *cnt = rb_time_cnt(top);
646−
647− /* If top, msb or bottom counts don't match, this interrupted a write */
648− if (*cnt != rb_time_cnt(msb) || *cnt != rb_time_cnt(bottom))
649− return false;
650−
651− /* The shift to msb will lose its cnt bits */
652− *ret = rb_time_val(top, bottom) | ((u64)msb << RB_TIME_MSB_SHIFT);
653− return true;
654−}
655−
656−static bool rb_time_read(rb_time_t *t, u64 *ret)
657−{
658− unsigned long cnt;
659−
660− return __rb_time_read(t, ret, &cnt);
661−}
662−
663−static inline unsigned long rb_time_val_cnt(unsigned long val, unsigned long cnt)
664−{
665− return (val & RB_TIME_VAL_MASK) | ((cnt & 3) << RB_TIME_SHIFT);
666−}
667−
668−static inline void rb_time_split(u64 val, unsigned long *top, unsigned long *bottom,
669− unsigned long *msb)
670−{
671− *top = (unsigned long)((val >> RB_TIME_SHIFT) & RB_TIME_VAL_MASK);
672− *bottom = (unsigned long)(val & RB_TIME_VAL_MASK);
673− *msb = (unsigned long)(val >> RB_TIME_MSB_SHIFT);
674−}
537+ struct buffer_data_page field;
675538
676−static inline void rb_time_val_set(local_t *t, unsigned long val, unsigned long cnt)
677−{
678− val = rb_time_val_cnt(val, cnt);
679− local_set(t, val);
680−}
539+ trace_seq_printf(s, "\tfield: u64 timestamp;\t"
540+ "offset:0;\tsize:%u;\tsigned:%u;\n",
541+ (unsigned int)sizeof(field.time_stamp),
542+ (unsigned int)is_signed_type(u64));
681543
682−static void rb_time_set(rb_time_t *t, u64 val)
683−{
684− unsigned long cnt, top, bottom, msb;
544+ trace_seq_printf(s, "\tfield: local_t commit;\t"
545+ "offset:%u;\tsize:%u;\tsigned:%u;\n",
546+ (unsigned int)offsetof(typeof(field), commit),
547+ (unsigned int)sizeof(field.commit),
548+ (unsigned int)is_signed_type(long));
685549
686− rb_time_split(val, &top, &bottom, &msb);
550+ trace_seq_printf(s, "\tfield: int overwrite;\t"
551+ "offset:%u;\tsize:%u;\tsigned:%u;\n",
552+ (unsigned int)offsetof(typeof(field), commit),
553+ 1,
554+ (unsigned int)is_signed_type(long));
687555
688− /* Writes always succeed with a valid number even if it gets interrupted. */
689− do {
690− cnt = local_inc_return(&t->cnt);
691− rb_time_val_set(&t->top, top, cnt);
692− rb_time_val_set(&t->bottom, bottom, cnt);
693− rb_time_val_set(&t->msb, val >> RB_TIME_MSB_SHIFT, cnt);
694− } while (cnt != local_read(&t->cnt));
695−}
556+ trace_seq_printf(s, "\tfield: char data;\t"
557+ "offset:%u;\tsize:%u;\tsigned:%u;\n",
558+ (unsigned int)offsetof(typeof(field), data),
559+ (unsigned int)buffer->subbuf_size,
560+ (unsigned int)is_signed_type(char));
696561
697−static inline bool
698−rb_time_read_cmpxchg(local_t *l, unsigned long expect, unsigned long set)
699−{
700− return local_try_cmpxchg(l, &expect, set);
562+ return !trace_seq_has_overflowed(s);
701563 }
702564
703−#else /* 64 bits */
704−
705−/* local64_t always succeeds */
706−
707−static inline bool rb_time_read(rb_time_t *t, u64 *ret)
565+static inline void rb_time_read(rb_time_t *t, u64 *ret)
708566 {
709567 *ret = local64_read(&t->time);
710− return true;
711568 }
712569 static void rb_time_set(rb_time_t *t, u64 val)
713570 {
714571 local64_set(&t->time, val);
715572 }
716−#endif
717573
718574 /*
719575 * Enable this to make sure that the event passed to
@@ -820,10 +676,7 @@ u64 ring_buffer_event_time_stamp(struct trace_buffer *buffer, · 100 unmodified lines
820676 WARN_ONCE(1, "nest (%d) greater than max", nest);
821677
822678 fail:
823− /* Can only fail on 32 bit */
824− if (!rb_time_read(&cpu_buffer->write_stamp, &ts))
825− /* Screw it, just read the current time */
826− ts = rb_time_stamp(cpu_buffer->buffer);
679+ rb_time_read(&cpu_buffer->write_stamp, &ts);
827680
828681 return ts;
829682 }
@@ -1619,10 +1472,12 @@ static int __rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, · 789 unmodified lines
16191472
16201473 list_add(&bpage->list, pages);
16211474
1622− page = alloc_pages_node(cpu_to_node(cpu_buffer->cpu), mflags, 0);
1475+ page = alloc_pages_node(cpu_to_node(cpu_buffer->cpu), mflags,
1476+ cpu_buffer->buffer->subbuf_order);
16231477 if (!page)
16241478 goto free_pages;
16251479 bpage->page = page_address(page);
1480+ bpage->order = cpu_buffer->buffer->subbuf_order;
16261481 rb_init_page(bpage->page);
16271482
16281483 if (user_thread && fatal_signal_pending(current))
@@ -1701,7 +1556,8 @@ rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu) · 72 unmodified lines
17011556 rb_check_bpage(cpu_buffer, bpage);
17021557
17031558 cpu_buffer->reader_page = bpage;
1704− page = alloc_pages_node(cpu_to_node(cpu), GFP_KERNEL, 0);
1559+
1560+ page = alloc_pages_node(cpu_to_node(cpu), GFP_KERNEL, cpu_buffer->buffer->subbuf_order);
17051561 if (!page)
17061562 goto fail_free_reader;
17071563 bpage->page = page_address(page);
@@ -1784,7 +1640,14 @@ struct trace_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags, · 76 unmodified lines
17841640 if (!zalloc_cpumask_var(&buffer->cpumask, GFP_KERNEL))
17851641 goto fail_free_buffer;
17861642
1787− nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
1643+ /* Default buffer page size - one system page */
1644+ buffer->subbuf_order = 0;
1645+ buffer->subbuf_size = PAGE_SIZE - BUF_PAGE_HDR_SIZE;
1646+
1647+ /* Max payload is buffer page size - header (8bytes) */
1648+ buffer->max_data_size = buffer->subbuf_size - (sizeof(u32) * 2);
1649+
1650+ nr_pages = DIV_ROUND_UP(size, buffer->subbuf_size);
17881651 buffer->flags = flags;
17891652 buffer->clock = trace_clock_local;
17901653 buffer->reader_lock_key = key;
@@ -2103,7 +1966,7 @@ static void update_pages_handler(struct work_struct *work) · 312 unmodified lines
21031966 * @size: the new size.
21041967 * @cpu_id: the cpu buffer to resize
21051968 *
2106− * Minimum size is 2 * BUF_PAGE_SIZE.
1969+ * Minimum size is 2 * buffer->subbuf_size.
21071970 *
21081971 * Returns 0 on success and < 0 on failure.
21091972 */
@@ -2125,7 +1988,7 @@ int ring_buffer_resize(struct trace_buffer *buffer, unsigned long size, · 15 unmodified lines
21251988 !cpumask_test_cpu(cpu_id, buffer->cpumask))
21261989 return 0;
21271990
2128− nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
1991+ nr_pages = DIV_ROUND_UP(size, buffer->subbuf_size);
21291992
21301993 /* we need a minimum of two pages */
21311994 if (nr_pages < 2)
@@ -2372,7 +2235,7 @@ rb_iter_head_event(struct ring_buffer_iter *iter) · 240 unmodified lines
23722235 */
23732236 barrier();
23742237
2375− if ((iter->head + length) > commit || length > BUF_PAGE_SIZE)
2238+ if ((iter->head + length) > commit || length > iter->event_size)
23762239 /* Writer corrupted the read? */
23772240 goto reset;
23782241
@@ -2412,11 +2275,13 @@ rb_commit_index(struct ring_buffer_per_cpu *cpu_buffer) · 33 unmodified lines
24122275 }
24132276
24142277 static __always_inline unsigned
2415−rb_event_index(struct ring_buffer_event *event)
2278+rb_event_index(struct ring_buffer_per_cpu *cpu_buffer, struct ring_buffer_event *event)
24162279 {
24172280 unsigned long addr = (unsigned long)event;
24182281
2419− return (addr & ~PAGE_MASK) - BUF_PAGE_HDR_SIZE;
2282+ addr &= (PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1;
2283+
2284+ return addr - BUF_PAGE_HDR_SIZE;
24202285 }
24212286
24222287 static void rb_inc_iter(struct ring_buffer_iter *iter)
@@ -2605,6 +2470,7 @@ static inline void · 182 unmodified lines
26052470 rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer,
26062471 unsigned long tail, struct rb_event_info *info)
26072472 {
2473+ unsigned long bsize = READ_ONCE(cpu_buffer->buffer->subbuf_size);
26082474 struct buffer_page *tail_page = info->tail_page;
26092475 struct ring_buffer_event *event;
26102476 unsigned long length = info->length;
@@ -2613,13 +2479,13 @@ rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer, · 2 unmodified lines
26132479 * Only the event that crossed the page boundary
26142480 * must fill the old tail_page with padding.
26152481 */
2616− if (tail >= BUF_PAGE_SIZE) {
2482+ if (tail >= bsize) {
26172483 /*
26182484 * If the page was filled, then we still need
26192485 * to update the real_end. Reset it to zero
26202486 * and the reader will ignore it.
26212487 */
2622− if (tail == BUF_PAGE_SIZE)
2488+ if (tail == bsize)
26232489 tail_page->real_end = 0;
26242490
26252491 local_sub(length, &tail_page->write);
@@ -2647,7 +2513,7 @@ rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer, · 21 unmodified lines
26472513 * If we are less than the minimum size, we don't need to
26482514 * worry about it.
26492515 */
2650− if (tail > (BUF_PAGE_SIZE - RB_EVNT_MIN_SIZE)) {
2516+ if (tail > (bsize - RB_EVNT_MIN_SIZE)) {
26512517 /* No room for any events */
26522518
26532519 /* Mark the rest of the page with padding */
@@ -2662,19 +2528,19 @@ rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer, · 8 unmodified lines
26622528 }
26632529
26642530 /* Put in a discarded event */
2665− event->array[0] = (BUF_PAGE_SIZE - tail) - RB_EVNT_HDR_SIZE;
2531+ event->array[0] = (bsize - tail) - RB_EVNT_HDR_SIZE;
26662532 event->type_len = RINGBUF_TYPE_PADDING;
26672533 /* time delta must be non zero */
26682534 event->time_delta = 1;
26692535
26702536 /* account for padding bytes */
2671− local_add(BUF_PAGE_SIZE - tail, &cpu_buffer->entries_bytes);
2537+ local_add(bsize - tail, &cpu_buffer->entries_bytes);
26722538
26732539 /* Make sure the padding is visible before the tail_page->write update */
26742540 smp_wmb();
26752541
26762542 /* Set write to end of buffer */
2677− length = (tail + length) - BUF_PAGE_SIZE;
2543+ length = (tail + length) - bsize;
26782544 local_sub(length, &tail_page->write);
26792545 }
26802546
@@ -2788,7 +2654,8 @@ rb_move_tail(struct ring_buffer_per_cpu *cpu_buffer, · 107 unmodified lines
27882654
27892655 /* Slow path */
27902656 static struct ring_buffer_event *
2791−rb_add_time_stamp(struct ring_buffer_event *event, u64 delta, bool abs)
2657+rb_add_time_stamp(struct ring_buffer_per_cpu *cpu_buffer,
2658+ struct ring_buffer_event *event, u64 delta, bool abs)
27922659 {
27932660 if (abs)
27942661 event->type_len = RINGBUF_TYPE_TIME_STAMP;
@@ -2796,7 +2663,7 @@ rb_add_time_stamp(struct ring_buffer_event *event, u64 delta, bool abs) · 1 unmodified line
27962663 event->type_len = RINGBUF_TYPE_TIME_EXTEND;
27972664
27982665 /* Not the first event on the page, or not delta? */
2799− if (abs || rb_event_index(event)) {
2666+ if (abs || rb_event_index(cpu_buffer, event)) {
28002667 event->time_delta = delta & TS_MASK;
28012668 event->array[0] = delta >> TS_SHIFT;
28022669 } else {
@@ -2826,7 +2693,7 @@ rb_check_timestamp(struct ring_buffer_per_cpu *cpu_buffer, · 23 unmodified lines
28262693 (unsigned long long)info->ts,
28272694 (unsigned long long)info->before,
28282695 (unsigned long long)info->after,
2829− (unsigned long long)(rb_time_read(&cpu_buffer->write_stamp, &write_stamp) ? write_stamp : 0),
2696+ (unsigned long long)({rb_time_read(&cpu_buffer->write_stamp, &write_stamp); write_stamp;}),
28302697 sched_clock_stable() ? "" :
28312698 "If you just came from a suspend/resume,\n"
28322699 "please switch to the trace global clock:\n"
@@ -2870,7 +2737,7 @@ static void rb_add_timestamp(struct ring_buffer_per_cpu *cpu_buffer, · 37 unmodified lines
28702737 if (!abs)
28712738 info->delta = 0;
28722739 }
2873− *event = rb_add_time_stamp(*event, info->delta, abs);
2740+ *event = rb_add_time_stamp(cpu_buffer, *event, info->delta, abs);
28742741 *length -= RB_LEN_TIME_EXTEND;
28752742 *delta = 0;
28762743 }
@@ -2954,10 +2821,10 @@ rb_try_to_discard(struct ring_buffer_per_cpu *cpu_buffer, · 77 unmodified lines
29542821 struct buffer_page *bpage;
29552822 unsigned long addr;
29562823
2957− new_index = rb_event_index(event);
2824+ new_index = rb_event_index(cpu_buffer, event);
29582825 old_index = new_index + rb_event_ts_length(event);
29592826 addr = (unsigned long)event;
2960− addr &= PAGE_MASK;
2827+ addr &= ~((PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1);
29612828
29622829 bpage = READ_ONCE(cpu_buffer->tail_page);
29632830
@@ -3344,6 +3211,76 @@ EXPORT_SYMBOL_GPL(ring_buffer_unlock_commit); · 380 unmodified lines
33443211 #define CHECK_FULL_PAGE 1L
33453212
33463213 #ifdef CONFIG_RING_BUFFER_VALIDATE_TIME_DELTAS
3214+
3215+static const char *show_irq_str(int bits)
3216+{
3217+ const char *type[] = {
3218+ ".", // 0
3219+ "s", // 1
3220+ "h", // 2
3221+ "Hs", // 3
3222+ "n", // 4
3223+ "Ns", // 5
3224+ "Nh", // 6
3225+ "NHs", // 7
3226+ };
3227+
3228+ return type[bits];
3229+}
3230+
3231+/* Assume this is an trace event */
3232+static const char *show_flags(struct ring_buffer_event *event)
3233+{
3234+ struct trace_entry *entry;
3235+ int bits = 0;
3236+
3237+ if (rb_event_data_length(event) - RB_EVNT_HDR_SIZE < sizeof(*entry))
3238+ return "X";
3239+
3240+ entry = ring_buffer_event_data(event);
3241+
3242+ if (entry->flags & TRACE_FLAG_SOFTIRQ)
3243+ bits |= 1;
3244+
3245+ if (entry->flags & TRACE_FLAG_HARDIRQ)
3246+ bits |= 2;
3247+
3248+ if (entry->flags & TRACE_FLAG_NMI)
3249+ bits |= 4;
3250+
3251+ return show_irq_str(bits);
3252+}
3253+
3254+static const char *show_irq(struct ring_buffer_event *event)
3255+{
3256+ struct trace_entry *entry;
3257+
3258+ if (rb_event_data_length(event) - RB_EVNT_HDR_SIZE < sizeof(*entry))
3259+ return "";
3260+
3261+ entry = ring_buffer_event_data(event);
3262+ if (entry->flags & TRACE_FLAG_IRQS_OFF)
3263+ return "d";
3264+ return "";
3265+}
3266+
3267+static const char *show_interrupt_level(void)
3268+{
3269+ unsigned long pc = preempt_count();
3270+ unsigned char level = 0;
3271+
3272+ if (pc & SOFTIRQ_OFFSET)
3273+ level |= 1;
3274+
3275+ if (pc & HARDIRQ_MASK)
3276+ level |= 2;
3277+
3278+ if (pc & NMI_MASK)
3279+ level |= 4;
3280+
3281+ return show_irq_str(level);
3282+}
3283+
33473284 static void dump_buffer_page(struct buffer_data_page *bpage,
33483285 struct rb_event_info *info,
33493286 unsigned long tail)
@@ -3364,34 +3301,57 @@ static void dump_buffer_page(struct buffer_data_page *bpage, · 14 unmodified lines
33643301 case RINGBUF_TYPE_TIME_EXTEND:
33653302 delta = rb_event_time_stamp(event);
33663303 ts += delta;
3367− pr_warn(" [%lld] delta:%lld TIME EXTEND\n", ts, delta);
3304+ pr_warn(" 0x%x: [%lld] delta:%lld TIME EXTEND\n",
3305+ e, ts, delta);
33683306 break;
33693307
33703308 case RINGBUF_TYPE_TIME_STAMP:
33713309 delta = rb_event_time_stamp(event);
33723310 ts = rb_fix_abs_ts(delta, ts);
3373− pr_warn(" [%lld] absolute:%lld TIME STAMP\n", ts, delta);
3311+ pr_warn(" 0x%x: [%lld] absolute:%lld TIME STAMP\n",
3312+ e, ts, delta);
33743313 break;
33753314
33763315 case RINGBUF_TYPE_PADDING:
33773316 ts += event->time_delta;
3378− pr_warn(" [%lld] delta:%d PADDING\n", ts, event->time_delta);
3317+ pr_warn(" 0x%x: [%lld] delta:%d PADDING\n",
3318+ e, ts, event->time_delta);
33793319 break;
33803320
33813321 case RINGBUF_TYPE_DATA:
33823322 ts += event->time_delta;
3383− pr_warn(" [%lld] delta:%d\n", ts, event->time_delta);
3323+ pr_warn(" 0x%x: [%lld] delta:%d %s%s\n",
3324+ e, ts, event->time_delta,
3325+ show_flags(event), show_irq(event));
33843326 break;
33853327
33863328 default:
33873329 break;
33883330 }
33893331 }
3332+ pr_warn("expected end:0x%lx last event actually ended at:0x%x\n", tail, e);
33903333 }
33913334
33923335 static DEFINE_PER_CPU(atomic_t, checking);
33933336 static atomic_t ts_dump;
33943337
3338+#define buffer_warn_return(fmt, ...) \
3339+ do { \
3340+ /* If another report is happening, ignore this one */ \
3341+ if (atomic_inc_return(&ts_dump) != 1) { \
3342+ atomic_dec(&ts_dump); \
3343+ goto out; \
3344+ } \
3345+ atomic_inc(&cpu_buffer->record_disabled); \
3346+ pr_warn(fmt, ##__VA_ARGS__); \
3347+ dump_buffer_page(bpage, info, tail); \
3348+ atomic_dec(&ts_dump); \
3349+ /* There's some cases in boot up that this can happen */ \
3350+ if (WARN_ON_ONCE(system_state != SYSTEM_BOOTING)) \
3351+ /* Do not re-enable checking */ \
3352+ return; \
3353+ } while (0)
3354+
33953355 /*
33963356 * Check if the current event time stamp matches the deltas on
33973357 * the buffer page.
@@ -3445,7 +3405,12 @@ static void check_buffer(struct ring_buffer_per_cpu *cpu_buffer, · 47 unmodified lines
34453405
34463406 case RINGBUF_TYPE_TIME_STAMP:
34473407 delta = rb_event_time_stamp(event);
3448− ts = rb_fix_abs_ts(delta, ts);
3408+ delta = rb_fix_abs_ts(delta, ts);
3409+ if (delta < ts) {
3410+ buffer_warn_return("[CPU: %d]ABSOLUTE TIME WENT BACKWARDS: last ts: %lld absolute ts: %lld\n",
3411+ cpu_buffer->cpu, ts, delta);
3412+ }
3413+ ts = delta;
34493414 break;
34503415
34513416 case RINGBUF_TYPE_PADDING:
@@ -3462,23 +3427,11 @@ static void check_buffer(struct ring_buffer_per_cpu *cpu_buffer, · 10 unmodified lines
34623427 }
34633428 if ((full && ts > info->ts) ||
34643429 (!full && ts + info->delta != info->ts)) {
3465− /* If another report is happening, ignore this one */
3466− if (atomic_inc_return(&ts_dump) != 1) {
3467− atomic_dec(&ts_dump);
3468− goto out;
3469− }
3470− atomic_inc(&cpu_buffer->record_disabled);
3471− /* There's some cases in boot up that this can happen */
3472− WARN_ON_ONCE(system_state != SYSTEM_BOOTING);
3473− pr_warn("[CPU: %d]TIME DOES NOT MATCH expected:%lld actual:%lld delta:%lld before:%lld after:%lld%s\n",
3474− cpu_buffer->cpu,
3475− ts + info->delta, info->ts, info->delta,
3476− info->before, info->after,
3477− full ? " (full)" : "");
3478− dump_buffer_page(bpage, info, tail);
3479− atomic_dec(&ts_dump);
3480− /* Do not re-enable checking */
3481− return;
3430+ buffer_warn_return("[CPU: %d]TIME DOES NOT MATCH expected:%lld actual:%lld delta:%lld before:%lld after:%lld%s context:%s\n",
3431+ cpu_buffer->cpu,
3432+ ts + info->delta, info->ts, info->delta,
3433+ info->before, info->after,
3434+ full ? " (full)" : "", show_interrupt_level());
34823435 }
34833436 out:
34843437 atomic_dec(this_cpu_ptr(&checking));
@@ -3498,16 +3451,14 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer, · 13 unmodified lines
34983451 struct ring_buffer_event *event;
34993452 struct buffer_page *tail_page;
35003453 unsigned long tail, write, w;
3501− bool a_ok;
3502− bool b_ok;
35033454
35043455 /* Don't let the compiler play games with cpu_buffer->tail_page */
35053456 tail_page = info->tail_page = READ_ONCE(cpu_buffer->tail_page);
35063457
35073458 /*A*/ w = local_read(&tail_page->write) & RB_WRITE_MASK;
35083459 barrier();
3509− b_ok = rb_time_read(&cpu_buffer->before_stamp, &info->before);
3510− a_ok = rb_time_read(&cpu_buffer->write_stamp, &info->after);
3460+ rb_time_read(&cpu_buffer->before_stamp, &info->before);
3461+ rb_time_read(&cpu_buffer->write_stamp, &info->after);
35113462 barrier();
35123463 info->ts = rb_time_stamp(cpu_buffer->buffer);
35133464
@@ -3522,7 +3473,7 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer, · 8 unmodified lines
35223473 if (!w) {
35233474 /* Use the sub-buffer timestamp */
35243475 info->delta = 0;
3525− } else if (unlikely(!a_ok || !b_ok || info->before != info->after)) {
3476+ } else if (unlikely(info->before != info->after)) {
35263477 info->add_timestamp |= RB_ADD_STAMP_FORCE | RB_ADD_STAMP_EXTEND;
35273478 info->length += RB_LEN_TIME_EXTEND;
35283479 } else {
@@ -3544,7 +3495,7 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer, · 15 unmodified lines
35443495 tail = write - info->length;
35453496
35463497 /* See if we shot pass the end of this buffer page */
3547− if (unlikely(write > BUF_PAGE_SIZE)) {
3498+ if (unlikely(write > cpu_buffer->buffer->subbuf_size)) {
35483499 check_buffer(cpu_buffer, info, CHECK_FULL_PAGE);
35493500 return rb_move_tail(cpu_buffer, tail, info);
35503501 }
@@ -3571,8 +3522,7 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer, · 20 unmodified lines
35713522 /* SLOW PATH - Interrupted between A and C */
35723523
35733524 /* Save the old before_stamp */
3574− a_ok = rb_time_read(&cpu_buffer->before_stamp, &info->before);
3575− RB_WARN_ON(cpu_buffer, !a_ok);
3525+ rb_time_read(&cpu_buffer->before_stamp, &info->before);
35763526
35773527 /*
35783528 * Read a new timestamp and update the before_stamp to make
@@ -3584,9 +3534,7 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer, · 5 unmodified lines
35843534 rb_time_set(&cpu_buffer->before_stamp, ts);
35853535
35863536 barrier();
3587− /*E*/ a_ok = rb_time_read(&cpu_buffer->write_stamp, &info->after);
3588− /* Was interrupted before here, write_stamp must be valid */
3589− RB_WARN_ON(cpu_buffer, !a_ok);
3537+ /*E*/ rb_time_read(&cpu_buffer->write_stamp, &info->after);
35903538 barrier();
35913539 /*F*/ if (write == (local_read(&tail_page->write) & RB_WRITE_MASK) &&
35923540 info->after == info->before && info->after < ts) {
@@ -3678,7 +3626,7 @@ rb_reserve_next_event(struct trace_buffer *buffer, · 85 unmodified lines
36783626 if (ring_buffer_time_stamp_abs(cpu_buffer->buffer)) {
36793627 add_ts_default = RB_ADD_STAMP_ABSOLUTE;
36803628 info.length += RB_LEN_TIME_EXTEND;
3681− if (info.length > BUF_MAX_DATA_SIZE)
3629+ if (info.length > cpu_buffer->buffer->max_data_size)
36823630 goto out_fail;
36833631 } else {
36843632 add_ts_default = RB_ADD_STAMP_NONE;
@@ -3753,7 +3701,7 @@ ring_buffer_lock_reserve(struct trace_buffer *buffer, unsigned long length) · 68 unmodified lines
37533701 if (unlikely(atomic_read(&cpu_buffer->record_disabled)))
37543702 goto out;
37553703
3756− if (unlikely(length > BUF_MAX_DATA_SIZE))
3704+ if (unlikely(length > buffer->max_data_size))
37573705 goto out;
37583706
37593707 if (unlikely(trace_recursive_lock(cpu_buffer)))
@@ -3787,7 +3735,7 @@ rb_decrement_entry(struct ring_buffer_per_cpu *cpu_buffer, · 27 unmodified lines
37873735 struct buffer_page *bpage = cpu_buffer->commit_page;
37883736 struct buffer_page *start;
37893737
3790− addr &= PAGE_MASK;
3738+ addr &= ~((PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1);
37913739
37923740 /* Do the likely case first */
37933741 if (likely(bpage->page == (void *)addr)) {
@@ -3903,7 +3851,7 @@ int ring_buffer_write(struct trace_buffer *buffer, · 109 unmodified lines
39033851 if (atomic_read(&cpu_buffer->record_disabled))
39043852 goto out;
39053853
3906− if (length > BUF_MAX_DATA_SIZE)
3854+ if (length > buffer->max_data_size)
39073855 goto out;
39083856
39093857 if (unlikely(trace_recursive_lock(cpu_buffer)))
@@ -4483,6 +4431,7 @@ static struct buffer_page * · 573 unmodified lines
44834431 rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
44844432 {
44854433 struct buffer_page *reader = NULL;
4434+ unsigned long bsize = READ_ONCE(cpu_buffer->buffer->subbuf_size);
44864435 unsigned long overwrite;
44874436 unsigned long flags;
44884437 int nr_loops = 0;
@@ -4618,7 +4567,7 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) · 129 unmodified lines
46184567 #define USECS_WAIT 1000000
46194568 for (nr_loops = 0; nr_loops < USECS_WAIT; nr_loops++) {
46204569 /* If the write is past the end of page, a writer is still updating it */
4621− if (likely(!reader || rb_page_write(reader) <= BUF_PAGE_SIZE))
4570+ if (likely(!reader || rb_page_write(reader) <= bsize))
46224571 break;
46234572
46244573 udelay(1);
@@ -5062,7 +5011,8 @@ ring_buffer_read_prepare(struct trace_buffer *buffer, int cpu, gfp_t flags) · 437 unmodified lines
50625011 return NULL;
50635012
50645013 /* Holds the entire event: data and meta data */
5065− iter->event = kmalloc(BUF_PAGE_SIZE, flags);
5014+ iter->event_size = buffer->subbuf_size;
5015+ iter->event = kmalloc(iter->event_size, flags);
50665016 if (!iter->event) {
50675017 kfree(iter);
50685018 return NULL;
@@ -5178,19 +5128,28 @@ EXPORT_SYMBOL_GPL(ring_buffer_iter_advance); · 109 unmodified lines
51785128 */
51795129 unsigned long ring_buffer_size(struct trace_buffer *buffer, int cpu)
51805130 {
5181− /*
5182− * Earlier, this method returned
5183− * BUF_PAGE_SIZE * buffer->nr_pages
5184− * Since the nr_pages field is now removed, we have converted this to
5185− * return the per cpu buffer value.
5186− */
51875131 if (!cpumask_test_cpu(cpu, buffer->cpumask))
51885132 return 0;
51895133
5190− return BUF_PAGE_SIZE * buffer->buffers[cpu]->nr_pages;
5134+ return buffer->subbuf_size * buffer->buffers[cpu]->nr_pages;
51915135 }
51925136 EXPORT_SYMBOL_GPL(ring_buffer_size);
51935137
5138+/**
5139+ * ring_buffer_max_event_size - return the max data size of an event
5140+ * @buffer: The ring buffer.
5141+ *
5142+ * Returns the maximum size an event can be.
5143+ */
5144+unsigned long ring_buffer_max_event_size(struct trace_buffer *buffer)
5145+{
5146+ /* If abs timestamp is requested, events have a timestamp too */
5147+ if (ring_buffer_time_stamp_abs(buffer))
5148+ return buffer->max_data_size - RB_LEN_TIME_EXTEND;
5149+ return buffer->max_data_size;
5150+}
5151+EXPORT_SYMBOL_GPL(ring_buffer_max_event_size);
5152+
51945153 static void rb_clear_buffer_page(struct buffer_page *page)
51955154 {
51965155 local_set(&page->write, 0);
@@ -5461,6 +5420,9 @@ int ring_buffer_swap_cpu(struct trace_buffer *buffer_a, · 264 unmodified lines
54615420 if (cpu_buffer_a->nr_pages != cpu_buffer_b->nr_pages)
54625421 goto out;
54635422
5423+ if (buffer_a->subbuf_order != buffer_b->subbuf_order)
5424+ goto out;
5425+
54645426 ret = -EAGAIN;
54655427
54665428 if (atomic_read(&buffer_a->record_disabled))
@@ -5532,40 +5494,48 @@ EXPORT_SYMBOL_GPL(ring_buffer_swap_cpu); · 65 unmodified lines
55325494 * Returns:
55335495 * The page allocated, or ERR_PTR
55345496 */
5535−void *ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu)
5497+struct buffer_data_read_page *
5498+ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu)
55365499 {
55375500 struct ring_buffer_per_cpu *cpu_buffer;
5538− struct buffer_data_page *bpage = NULL;
5501+ struct buffer_data_read_page *bpage = NULL;
55395502 unsigned long flags;
55405503 struct page *page;
55415504
55425505 if (!cpumask_test_cpu(cpu, buffer->cpumask))
55435506 return ERR_PTR(-ENODEV);
55445507
5508+ bpage = kzalloc(sizeof(*bpage), GFP_KERNEL);
5509+ if (!bpage)
5510+ return ERR_PTR(-ENOMEM);
5511+
5512+ bpage->order = buffer->subbuf_order;
55455513 cpu_buffer = buffer->buffers[cpu];
55465514 local_irq_save(flags);
55475515 arch_spin_lock(&cpu_buffer->lock);
55485516
55495517 if (cpu_buffer->free_page) {
5550− bpage = cpu_buffer->free_page;
5518+ bpage->data = cpu_buffer->free_page;
55515519 cpu_buffer->free_page = NULL;
55525520 }
55535521
55545522 arch_spin_unlock(&cpu_buffer->lock);
55555523 local_irq_restore(flags);
55565524
5557− if (bpage)
5525+ if (bpage->data)
55585526 goto out;
55595527
5560− page = alloc_pages_node(cpu_to_node(cpu),
5561− GFP_KERNEL | __GFP_NORETRY, 0);
5562− if (!page)
5528+ page = alloc_pages_node(cpu_to_node(cpu), GFP_KERNEL | __GFP_NORETRY,
5529+ cpu_buffer->buffer->subbuf_order);
5530+ if (!page) {
5531+ kfree(bpage);
55635532 return ERR_PTR(-ENOMEM);
5533+ }
55645534
5565− bpage = page_address(page);
5535+ bpage->data = page_address(page);
55665536
55675537 out:
5568− rb_init_page(bpage);
5538+ rb_init_page(bpage->data);
55695539
55705540 return bpage;
55715541 }
@@ -5575,14 +5545,15 @@ EXPORT_SYMBOL_GPL(ring_buffer_alloc_read_page); · 3 unmodified lines
55755545 * ring_buffer_free_read_page - free an allocated read page
55765546 * @buffer: the buffer the page was allocate for
55775547 * @cpu: the cpu buffer the page came from
5578− * @data: the page to free
5548+ * @data_page: the page to free
55795549 *
55805550 * Free a page allocated from ring_buffer_alloc_read_page.
55815551 */
5582−void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, void *data)
5552+void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu,
5553+ struct buffer_data_read_page *data_page)
55835554 {
55845555 struct ring_buffer_per_cpu *cpu_buffer;
5585− struct buffer_data_page *bpage = data;
5556+ struct buffer_data_page *bpage = data_page->data;
55865557 struct page *page = virt_to_page(bpage);
55875558 unsigned long flags;
55885559
@@ -5591,8 +5562,12 @@ void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, void *data · 2 unmodified lines
55915562
55925563 cpu_buffer = buffer->buffers[cpu];
55935564
5594− /* If the page is still in use someplace else, we can't reuse it */
5595− if (page_ref_count(page) > 1)
5565+ /*
5566+ * If the page is still in use someplace else, or order of the page
5567+ * is different from the subbuffer order of the buffer -
5568+ * we can't reuse it
5569+ */
5570+ if (page_ref_count(page) > 1 || data_page->order != buffer->subbuf_order)
55965571 goto out;
55975572
55985573 local_irq_save(flags);
@@ -5607,7 +5582,8 @@ void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, void *data · 8 unmodified lines
56075582 local_irq_restore(flags);
56085583
56095584 out:
5610− free_page((unsigned long)bpage);
5585+ free_pages((unsigned long)bpage, data_page->order);
5586+ kfree(data_page);
56115587 }
56125588 EXPORT_SYMBOL_GPL(ring_buffer_free_read_page);
56135589
@@ -5628,9 +5604,10 @@ EXPORT_SYMBOL_GPL(ring_buffer_free_read_page); · 14 unmodified lines
56285604 * rpage = ring_buffer_alloc_read_page(buffer, cpu);
56295605 * if (IS_ERR(rpage))
56305606 * return PTR_ERR(rpage);
5631− * ret = ring_buffer_read_page(buffer, &rpage, len, cpu, 0);
5607+ * ret = ring_buffer_read_page(buffer, rpage, len, cpu, 0);
56325608 * if (ret >= 0)
5633− * process_page(rpage, ret);
5609+ * process_page(ring_buffer_read_page_data(rpage), ret);
5610+ * ring_buffer_free_read_page(buffer, cpu, rpage);
56345611 *
56355612 * When @full is set, the function will not return true unless
56365613 * the writer is off the reader page.
@@ -5645,7 +5622,8 @@ EXPORT_SYMBOL_GPL(ring_buffer_free_read_page); · 8 unmodified lines
56455622 * <0 if no data has been transferred.
56465623 */
56475624 int ring_buffer_read_page(struct trace_buffer *buffer,
5648− void **data_page, size_t len, int cpu, int full)
5625+ struct buffer_data_read_page *data_page,
5626+ size_t len, int cpu, int full)
56495627 {
56505628 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
56515629 struct ring_buffer_event *event;
@@ -5670,10 +5648,12 @@ int ring_buffer_read_page(struct trace_buffer *buffer, · 18 unmodified lines
56705648
56715649 len -= BUF_PAGE_HDR_SIZE;
56725650
5673− if (!data_page)
5651+ if (!data_page || !data_page->data)
5652+ goto out;
5653+ if (data_page->order != buffer->subbuf_order)
56745654 goto out;
56755655
5676− bpage = *data_page;
5656+ bpage = data_page->data;
56775657 if (!bpage)
56785658 goto out;
56795659
@@ -5767,11 +5747,11 @@ int ring_buffer_read_page(struct trace_buffer *buffer, · 87 unmodified lines
57675747 /* swap the pages */
57685748 rb_init_page(bpage);
57695749 bpage = reader->page;
5770− reader->page = *data_page;
5750+ reader->page = data_page->data;
57715751 local_set(&reader->write, 0);
57725752 local_set(&reader->entries, 0);
57735753 reader->read = 0;
5774− *data_page = bpage;
5754+ data_page->data = bpage;
57755755
57765756 /*
57775757 * Use the real_end for the data size,
@@ -5793,7 +5773,7 @@ int ring_buffer_read_page(struct trace_buffer *buffer, · 15 unmodified lines
57935773 /* If there is room at the end of the page to save the
57945774 * missed events, then record it there.
57955775 */
5796− if (BUF_PAGE_SIZE - commit >= sizeof(missed_events)) {
5776+ if (buffer->subbuf_size - commit >= sizeof(missed_events)) {
57975777 memcpy(&bpage->data[commit], &missed_events,
57985778 sizeof(missed_events));
57995779 local_add(RB_MISSED_STORED, &bpage->commit);
@@ -5805,8 +5785,8 @@ int ring_buffer_read_page(struct trace_buffer *buffer, · 5 unmodified lines
58055785 /*
58065786 * This page may be off to user land. Zero it out here.
58075787 */
5808− if (commit < BUF_PAGE_SIZE)
5809− memset(&bpage->data[commit], 0, BUF_PAGE_SIZE - commit);
5788+ if (commit < buffer->subbuf_size)
5789+ memset(&bpage->data[commit], 0, buffer->subbuf_size - commit);
58105790
58115791 out_unlock:
58125792 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
@@ -5816,6 +5796,209 @@ int ring_buffer_read_page(struct trace_buffer *buffer, · 3 unmodified lines
58165796 }
58175797 EXPORT_SYMBOL_GPL(ring_buffer_read_page);
58185798
5799+/**
5800+ * ring_buffer_read_page_data - get pointer to the data in the page.
5801+ * @page: the page to get the data from
5802+ *
5803+ * Returns pointer to the actual data in this page.
5804+ */
5805+void *ring_buffer_read_page_data(struct buffer_data_read_page *page)
5806+{
5807+ return page->data;
5808+}
5809+EXPORT_SYMBOL_GPL(ring_buffer_read_page_data);
5810+
5811+/**
5812+ * ring_buffer_subbuf_size_get - get size of the sub buffer.
5813+ * @buffer: the buffer to get the sub buffer size from
5814+ *
5815+ * Returns size of the sub buffer, in bytes.
5816+ */
5817+int ring_buffer_subbuf_size_get(struct trace_buffer *buffer)
5818+{
5819+ return buffer->subbuf_size + BUF_PAGE_HDR_SIZE;
5820+}
5821+EXPORT_SYMBOL_GPL(ring_buffer_subbuf_size_get);
5822+
5823+/**
5824+ * ring_buffer_subbuf_order_get - get order of system sub pages in one buffer page.
5825+ * @buffer: The ring_buffer to get the system sub page order from
5826+ *
5827+ * By default, one ring buffer sub page equals to one system page. This parameter
5828+ * is configurable, per ring buffer. The size of the ring buffer sub page can be
5829+ * extended, but must be an order of system page size.
5830+ *
5831+ * Returns the order of buffer sub page size, in system pages:
5832+ * 0 means the sub buffer size is 1 system page and so forth.
5833+ * In case of an error < 0 is returned.
5834+ */
5835+int ring_buffer_subbuf_order_get(struct trace_buffer *buffer)
5836+{
5837+ if (!buffer)
5838+ return -EINVAL;
5839+
5840+ return buffer->subbuf_order;
5841+}
5842+EXPORT_SYMBOL_GPL(ring_buffer_subbuf_order_get);
5843+
5844+/**
5845+ * ring_buffer_subbuf_order_set - set the size of ring buffer sub page.
5846+ * @buffer: The ring_buffer to set the new page size.
5847+ * @order: Order of the system pages in one sub buffer page
5848+ *
5849+ * By default, one ring buffer pages equals to one system page. This API can be
5850+ * used to set new size of the ring buffer page. The size must be order of
5851+ * system page size, that's why the input parameter @order is the order of
5852+ * system pages that are allocated for one ring buffer page:
5853+ * 0 - 1 system page
5854+ * 1 - 2 system pages
5855+ * 3 - 4 system pages
5856+ * ...
5857+ *
5858+ * Returns 0 on success or < 0 in case of an error.
5859+ */
5860+int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order)
5861+{
5862+ struct ring_buffer_per_cpu *cpu_buffer;
5863+ struct buffer_page *bpage, *tmp;
5864+ int old_order, old_size;
5865+ int nr_pages;
5866+ int psize;
5867+ int err;
5868+ int cpu;
5869+
5870+ if (!buffer || order < 0)
5871+ return -EINVAL;
5872+
5873+ if (buffer->subbuf_order == order)
5874+ return 0;
5875+
5876+ psize = (1 << order) * PAGE_SIZE;
5877+ if (psize <= BUF_PAGE_HDR_SIZE)
5878+ return -EINVAL;
5879+
5880+ old_order = buffer->subbuf_order;
5881+ old_size = buffer->subbuf_size;
5882+
5883+ /* prevent another thread from changing buffer sizes */
5884+ mutex_lock(&buffer->mutex);
5885+ atomic_inc(&buffer->record_disabled);
5886+
5887+ /* Make sure all commits have finished */
5888+ synchronize_rcu();
5889+
5890+ buffer->subbuf_order = order;
5891+ buffer->subbuf_size = psize - BUF_PAGE_HDR_SIZE;
5892+
5893+ /* Make sure all new buffers are allocated, before deleting the old ones */
5894+ for_each_buffer_cpu(buffer, cpu) {
5895+
5896+ if (!cpumask_test_cpu(cpu, buffer->cpumask))
5897+ continue;
5898+
5899+ cpu_buffer = buffer->buffers[cpu];
5900+
5901+ /* Update the number of pages to match the new size */
5902+ nr_pages = old_size * buffer->buffers[cpu]->nr_pages;
5903+ nr_pages = DIV_ROUND_UP(nr_pages, buffer->subbuf_size);
5904+
5905+ /* we need a minimum of two pages */
5906+ if (nr_pages < 2)
5907+ nr_pages = 2;
5908+
5909+ cpu_buffer->nr_pages_to_update = nr_pages;
5910+
5911+ /* Include the reader page */
5912+ nr_pages++;
5913+
5914+ /* Allocate the new size buffer */
5915+ INIT_LIST_HEAD(&cpu_buffer->new_pages);
5916+ if (__rb_allocate_pages(cpu_buffer, nr_pages,
5917+ &cpu_buffer->new_pages)) {
5918+ /* not enough memory for new pages */
5919+ err = -ENOMEM;
5920+ goto error;
5921+ }
5922+ }
5923+
5924+ for_each_buffer_cpu(buffer, cpu) {
5925+
5926+ if (!cpumask_test_cpu(cpu, buffer->cpumask))
5927+ continue;
5928+
5929+ cpu_buffer = buffer->buffers[cpu];
5930+
5931+ /* Clear the head bit to make the link list normal to read */
5932+ rb_head_page_deactivate(cpu_buffer);
5933+
5934+ /* Now walk the list and free all the old sub buffers */
5935+ list_for_each_entry_safe(bpage, tmp, cpu_buffer->pages, list) {
5936+ list_del_init(&bpage->list);
5937+ free_buffer_page(bpage);
5938+ }
5939+ /* The above loop stopped an the last page needing to be freed */
5940+ bpage = list_entry(cpu_buffer->pages, struct buffer_page, list);
5941+ free_buffer_page(bpage);
5942+
5943+ /* Free the current reader page */
5944+ free_buffer_page(cpu_buffer->reader_page);
5945+
5946+ /* One page was allocated for the reader page */
5947+ cpu_buffer->reader_page = list_entry(cpu_buffer->new_pages.next,
5948+ struct buffer_page, list);
5949+ list_del_init(&cpu_buffer->reader_page->list);
5950+
5951+ /* The cpu_buffer pages are a link list with no head */
5952+ cpu_buffer->pages = cpu_buffer->new_pages.next;
5953+ cpu_buffer->new_pages.next->prev = cpu_buffer->new_pages.prev;
5954+ cpu_buffer->new_pages.prev->next = cpu_buffer->new_pages.next;
5955+
5956+ /* Clear the new_pages list */
5957+ INIT_LIST_HEAD(&cpu_buffer->new_pages);
5958+
5959+ cpu_buffer->head_page
5960+ = list_entry(cpu_buffer->pages, struct buffer_page, list);
5961+ cpu_buffer->tail_page = cpu_buffer->commit_page = cpu_buffer->head_page;
5962+
5963+ cpu_buffer->nr_pages = cpu_buffer->nr_pages_to_update;
5964+ cpu_buffer->nr_pages_to_update = 0;
5965+
5966+ free_pages((unsigned long)cpu_buffer->free_page, old_order);
5967+ cpu_buffer->free_page = NULL;
5968+
5969+ rb_head_page_activate(cpu_buffer);
5970+
5971+ rb_check_pages(cpu_buffer);
5972+ }
5973+
5974+ atomic_dec(&buffer->record_disabled);
5975+ mutex_unlock(&buffer->mutex);
5976+
5977+ return 0;
5978+
5979+error:
5980+ buffer->subbuf_order = old_order;
5981+ buffer->subbuf_size = old_size;
5982+
5983+ atomic_dec(&buffer->record_disabled);
5984+ mutex_unlock(&buffer->mutex);
5985+
5986+ for_each_buffer_cpu(buffer, cpu) {
5987+ cpu_buffer = buffer->buffers[cpu];
5988+
5989+ if (!cpu_buffer->nr_pages_to_update)
5990+ continue;
5991+
5992+ list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages, list) {
5993+ list_del_init(&bpage->list);
5994+ free_buffer_page(bpage);
5995+ }
5996+ }
5997+
5998+ return err;
5999+}
6000+EXPORT_SYMBOL_GPL(ring_buffer_subbuf_order_set);
6001+
58196002 /*
58206003 * We only allocate new buffers, never free them if the CPU goes down.
58216004 * If we were to free the buffer, then the user would lose any trace that was in