History

Merge tag 'trace-v6.8' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace

Pull tracing updates from Steven Rostedt: - Allow kernel trace instance creation to specify what events are created Inside the kernel, a subsystem may create a tracing instance that it can use to send events to user space. This sub-system may not care about the thousands of events that exist in eventfs. Allow the sub-system to specify what sub-systems of events it cares about, and only those events are exposed to this instance. - Allow the ring buffer to be broken up into bigger sub-buffers than just the architecture page size. A new tracefs file called "buffer_subbuf_size_kb" is created. The user can now specify a minimum size the sub-buffer may be in kilobytes. Note, that the implementation currently make the sub-buffer size a power of 2 pages (1, 2, 4, 8, 16, ...) but the user only writes in kilobyte size, and the sub-buffer will be updated to the next size that it will can accommodate it. If the user writes in 10, it will change the size to be 4 pages on x86 (16K), as that is the next available size that can hold 10K pages. - Update the debug output when a corrupt time is detected in the ring buffer. If the ring buffer detects inconsistent timestamps, there's a debug config options that will dump the contents of the meta data of the sub-buffer that is used for debugging. Add some more information to this dump that helps with debugging. - Add more timestamp debugging checks (only triggers when the config is enabled) - Increase the trace_seq iterator to 2 page sizes. - Allow strings written into tracefs_marker to be larger. Up to just under 2 page sizes (based on what trace_seq can hold). - Increase the trace_maker_raw write to be as big as a sub-buffer can hold. - Remove 32 bit time stamp logic, now that the rb_time_cmpxchg() has been removed. - More selftests were added. - Some code clean ups as well. * tag 'trace-v6.8' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace: (29 commits) ring-buffer: Remove stale comment from ring_buffer_size() tracing histograms: Simplify parse_actions() function tracing/selftests: Remove exec permissions from trace_marker.tc test ring-buffer: Use subbuf_order for buffer page masking tracing: Update subbuffer with kilobytes not page order ringbuffer/selftest: Add basic selftest to test changing subbuf order ring-buffer: Add documentation on the buffer_subbuf_order file ring-buffer: Just update the subbuffers when changing their allocation order ring-buffer: Keep the same size when updating the order tracing: Stop the tracing while changing the ring buffer subbuf size tracing: Update snapshot order along with main buffer order ring-buffer: Make sure the spare sub buffer used for reads has same size ring-buffer: Do no swap cpu buffers if order is different ring-buffer: Clear pages on error in ring_buffer_subbuf_order_set() failure ring-buffer: Read and write to ring buffers with custom sub buffer size ring-buffer: Set new size of the ring buffer sub page ring-buffer: Add interface for configuring trace sub buffer size ring-buffer: Page size per ring buffer ring-buffer: Have ring_buffer_print_page_header() be able to access ring_buffer_iter ring-buffer: Check if absolute timestamp goes backwards ...

Linus Torvalds committed 2 years ago

Commit a2ded782 parents 5b890ad 25742ae16 files · +999 −374Browse files

Showing the changes to kernel/trace/ring_buffer.c (1 of 16 files). Show all

Unified · Split

kernel/trace/ring_buffer.c+458−275
@@ -27,6 +27,7 @@ · 26 unmodified lines
27#include <linux/cpu.h>27#include <linux/cpu.h>
28#include <linux/oom.h>28#include <linux/oom.h>
29 29
30#include <asm/local64.h>
30#include <asm/local.h>31#include <asm/local.h>
31 32
32/*33/*
@@ -317,6 +318,11 @@ struct buffer_data_page { · 284 unmodified lines
317 unsigned char data[] RB_ALIGN_DATA; /* data of buffer page */318 unsigned char data[] RB_ALIGN_DATA; /* data of buffer page */
318};319};
319 320
321struct buffer_data_read_page {
322 unsigned order; /* order of the page */
323 struct buffer_data_page *data; /* actual data, stored in this page */
324};
325
320/*326/*
321 * Note, the buffer_page list must be first. The buffer pages327 * Note, the buffer_page list must be first. The buffer pages
322 * are allocated in cache lines, which means that each buffer328 * are allocated in cache lines, which means that each buffer
@@ -331,6 +337,7 @@ struct buffer_page { · 8 unmodified lines
331 unsigned read; /* index for next read */337 unsigned read; /* index for next read */
332 local_t entries; /* entries on this page */338 local_t entries; /* entries on this page */
333 unsigned long real_end; /* real end of data */339 unsigned long real_end; /* real end of data */
340 unsigned order; /* order of the page */
334 struct buffer_data_page *page; /* Actual data page */341 struct buffer_data_page *page; /* Actual data page */
335};342};
336 343
@@ -361,7 +368,7 @@ static __always_inline unsigned int rb_page_commit(struct buffer_page *bpage) · 24 unmodified lines
361 368
362static void free_buffer_page(struct buffer_page *bpage)369static void free_buffer_page(struct buffer_page *bpage)
363{370{
364 free_page((unsigned long)bpage->page);371 free_pages((unsigned long)bpage->page, bpage->order);
365 kfree(bpage);372 kfree(bpage);
366}373}
367 374
@@ -373,41 +380,6 @@ static inline bool test_time_stamp(u64 delta) · 5 unmodified lines
373 return !!(delta & TS_DELTA_TEST);380 return !!(delta & TS_DELTA_TEST);
374}381}
375 382
376#define BUF_PAGE_SIZE (PAGE_SIZE - BUF_PAGE_HDR_SIZE)
377
378/* Max payload is BUF_PAGE_SIZE - header (8bytes) */
379#define BUF_MAX_DATA_SIZE (BUF_PAGE_SIZE - (sizeof(u32) * 2))
380
381int ring_buffer_print_page_header(struct trace_seq *s)
382{
383 struct buffer_data_page field;
384
385 trace_seq_printf(s, "\tfield: u64 timestamp;\t"
386 "offset:0;\tsize:%u;\tsigned:%u;\n",
387 (unsigned int)sizeof(field.time_stamp),
388 (unsigned int)is_signed_type(u64));
389
390 trace_seq_printf(s, "\tfield: local_t commit;\t"
391 "offset:%u;\tsize:%u;\tsigned:%u;\n",
392 (unsigned int)offsetof(typeof(field), commit),
393 (unsigned int)sizeof(field.commit),
394 (unsigned int)is_signed_type(long));
395
396 trace_seq_printf(s, "\tfield: int overwrite;\t"
397 "offset:%u;\tsize:%u;\tsigned:%u;\n",
398 (unsigned int)offsetof(typeof(field), commit),
399 1,
400 (unsigned int)is_signed_type(long));
401
402 trace_seq_printf(s, "\tfield: char data;\t"
403 "offset:%u;\tsize:%u;\tsigned:%u;\n",
404 (unsigned int)offsetof(typeof(field), data),
405 (unsigned int)BUF_PAGE_SIZE,
406 (unsigned int)is_signed_type(char));
407
408 return !trace_seq_has_overflowed(s);
409}
410
411struct rb_irq_work {383struct rb_irq_work {
412 struct irq_work work;384 struct irq_work work;
413 wait_queue_head_t waiters;385 wait_queue_head_t waiters;
@@ -463,27 +435,9 @@ enum { · 49 unmodified lines
463 RB_CTX_MAX435 RB_CTX_MAX
464};436};
465 437
466#if BITS_PER_LONG == 32
467#define RB_TIME_32
468#endif
469
470/* To test on 64 bit machines */
471//#define RB_TIME_32
472
473#ifdef RB_TIME_32
474
475struct rb_time_struct {
476 local_t cnt;
477 local_t top;
478 local_t bottom;
479 local_t msb;
480};
481#else
482#include <asm/local64.h>
483struct rb_time_struct {438struct rb_time_struct {
484 local64_t time;439 local64_t time;
485};440};
486#endif
487typedef struct rb_time_struct rb_time_t;441typedef struct rb_time_struct rb_time_t;
488 442
489#define MAX_NEST 5443#define MAX_NEST 5
@@ -557,6 +511,10 @@ struct trace_buffer { · 67 unmodified lines
557 511
558 struct rb_irq_work irq_work;512 struct rb_irq_work irq_work;
559 bool time_stamp_abs;513 bool time_stamp_abs;
514
515 unsigned int subbuf_size;
516 unsigned int subbuf_order;
517 unsigned int max_data_size;
560};518};
561 519
562struct ring_buffer_iter {520struct ring_buffer_iter {
@@ -570,150 +528,48 @@ struct ring_buffer_iter { · 7 unmodified lines
570 u64 read_stamp;528 u64 read_stamp;
571 u64 page_stamp;529 u64 page_stamp;
572 struct ring_buffer_event *event;530 struct ring_buffer_event *event;
531 size_t event_size;
573 int missed_events;532 int missed_events;
574};533};
575 534
576#ifdef RB_TIME_32535int ring_buffer_print_page_header(struct trace_buffer *buffer, struct trace_seq *s)
577
578/*
579 * On 32 bit machines, local64_t is very expensive. As the ring
580 * buffer doesn't need all the features of a true 64 bit atomic,
581 * on 32 bit, it uses these functions (64 still uses local64_t).
582 *
583 * For the ring buffer, 64 bit required operations for the time is
584 * the following:
585 *
586 * - Reads may fail if it interrupted a modification of the time stamp.
587 * It will succeed if it did not interrupt another write even if
588 * the read itself is interrupted by a write.
589 * It returns whether it was successful or not.
590 *
591 * - Writes always succeed and will overwrite other writes and writes
592 * that were done by events interrupting the current write.
593 *
594 * - A write followed by a read of the same time stamp will always succeed,
595 * but may not contain the same value.
596 *
597 * - A cmpxchg will fail if it interrupted another write or cmpxchg.
598 * Other than that, it acts like a normal cmpxchg.
599 *
600 * The 60 bit time stamp is broken up by 30 bits in a top and bottom half
601 * (bottom being the least significant 30 bits of the 60 bit time stamp).
602 *
603 * The two most significant bits of each half holds a 2 bit counter (0-3).
604 * Each update will increment this counter by one.
605 * When reading the top and bottom, if the two counter bits match then the
606 * top and bottom together make a valid 60 bit number.
607 */
608#define RB_TIME_SHIFT 30
609#define RB_TIME_VAL_MASK ((1 << RB_TIME_SHIFT) - 1)
610#define RB_TIME_MSB_SHIFT 60
611
612static inline int rb_time_cnt(unsigned long val)
613{536{
614 return (val >> RB_TIME_SHIFT) & 3;537 struct buffer_data_page field;
615}
616
617static inline u64 rb_time_val(unsigned long top, unsigned long bottom)
618{
619 u64 val;
620
621 val = top & RB_TIME_VAL_MASK;
622 val <<= RB_TIME_SHIFT;
623 val |= bottom & RB_TIME_VAL_MASK;
624
625 return val;
626}
627
628static inline bool __rb_time_read(rb_time_t *t, u64 *ret, unsigned long *cnt)
629{
630 unsigned long top, bottom, msb;
631 unsigned long c;
632
633 /*
634 * If the read is interrupted by a write, then the cnt will
635 * be different. Loop until both top and bottom have been read
636 * without interruption.
637 */
638 do {
639 c = local_read(&t->cnt);
640 top = local_read(&t->top);
641 bottom = local_read(&t->bottom);
642 msb = local_read(&t->msb);
643 } while (c != local_read(&t->cnt));
644
645 *cnt = rb_time_cnt(top);
646
647 /* If top, msb or bottom counts don't match, this interrupted a write */
648 if (*cnt != rb_time_cnt(msb) || *cnt != rb_time_cnt(bottom))
649 return false;
650
651 /* The shift to msb will lose its cnt bits */
652 *ret = rb_time_val(top, bottom) | ((u64)msb << RB_TIME_MSB_SHIFT);
653 return true;
654}
655
656static bool rb_time_read(rb_time_t *t, u64 *ret)
657{
658 unsigned long cnt;
659
660 return __rb_time_read(t, ret, &cnt);
661}
662
663static inline unsigned long rb_time_val_cnt(unsigned long val, unsigned long cnt)
664{
665 return (val & RB_TIME_VAL_MASK) | ((cnt & 3) << RB_TIME_SHIFT);
666}
667
668static inline void rb_time_split(u64 val, unsigned long *top, unsigned long *bottom,
669 unsigned long *msb)
670{
671 *top = (unsigned long)((val >> RB_TIME_SHIFT) & RB_TIME_VAL_MASK);
672 *bottom = (unsigned long)(val & RB_TIME_VAL_MASK);
673 *msb = (unsigned long)(val >> RB_TIME_MSB_SHIFT);
674}
675 538
676static inline void rb_time_val_set(local_t *t, unsigned long val, unsigned long cnt)539 trace_seq_printf(s, "\tfield: u64 timestamp;\t"
677{540 "offset:0;\tsize:%u;\tsigned:%u;\n",
678 val = rb_time_val_cnt(val, cnt);541 (unsigned int)sizeof(field.time_stamp),
679 local_set(t, val);542 (unsigned int)is_signed_type(u64));
680}
681 543
682static void rb_time_set(rb_time_t *t, u64 val)544 trace_seq_printf(s, "\tfield: local_t commit;\t"
683{545 "offset:%u;\tsize:%u;\tsigned:%u;\n",
684 unsigned long cnt, top, bottom, msb;546 (unsigned int)offsetof(typeof(field), commit),
547 (unsigned int)sizeof(field.commit),
548 (unsigned int)is_signed_type(long));
685 549
686 rb_time_split(val, &top, &bottom, &msb);550 trace_seq_printf(s, "\tfield: int overwrite;\t"
551 "offset:%u;\tsize:%u;\tsigned:%u;\n",
552 (unsigned int)offsetof(typeof(field), commit),
553 1,
554 (unsigned int)is_signed_type(long));
687 555
688 /* Writes always succeed with a valid number even if it gets interrupted. */556 trace_seq_printf(s, "\tfield: char data;\t"
689 do {557 "offset:%u;\tsize:%u;\tsigned:%u;\n",
690 cnt = local_inc_return(&t->cnt);558 (unsigned int)offsetof(typeof(field), data),
691 rb_time_val_set(&t->top, top, cnt);559 (unsigned int)buffer->subbuf_size,
692 rb_time_val_set(&t->bottom, bottom, cnt);560 (unsigned int)is_signed_type(char));
693 rb_time_val_set(&t->msb, val >> RB_TIME_MSB_SHIFT, cnt);
694 } while (cnt != local_read(&t->cnt));
695}
696 561
697static inline bool562 return !trace_seq_has_overflowed(s);
698rb_time_read_cmpxchg(local_t *l, unsigned long expect, unsigned long set)
699{
700 return local_try_cmpxchg(l, &expect, set);
701}563}
702 564
703#else /* 64 bits */565static inline void rb_time_read(rb_time_t *t, u64 *ret)
704
705/* local64_t always succeeds */
706
707static inline bool rb_time_read(rb_time_t *t, u64 *ret)
708{566{
709 *ret = local64_read(&t->time);567 *ret = local64_read(&t->time);
710 return true;
711}568}
712static void rb_time_set(rb_time_t *t, u64 val)569static void rb_time_set(rb_time_t *t, u64 val)
713{570{
714 local64_set(&t->time, val);571 local64_set(&t->time, val);
715}572}
716#endif
717 573
718/*574/*
719 * Enable this to make sure that the event passed to575 * Enable this to make sure that the event passed to
@@ -820,10 +676,7 @@ u64 ring_buffer_event_time_stamp(struct trace_buffer *buffer, · 100 unmodified lines
820 WARN_ONCE(1, "nest (%d) greater than max", nest);676 WARN_ONCE(1, "nest (%d) greater than max", nest);
821 677
822 fail:678 fail:
823 /* Can only fail on 32 bit */679 rb_time_read(&cpu_buffer->write_stamp, &ts);
824 if (!rb_time_read(&cpu_buffer->write_stamp, &ts))
825 /* Screw it, just read the current time */
826 ts = rb_time_stamp(cpu_buffer->buffer);
827 680
828 return ts;681 return ts;
829}682}
@@ -1619,10 +1472,12 @@ static int __rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, · 789 unmodified lines
1619 1472
1620 list_add(&bpage->list, pages);1473 list_add(&bpage->list, pages);
1621 1474
1622 page = alloc_pages_node(cpu_to_node(cpu_buffer->cpu), mflags, 0);1475 page = alloc_pages_node(cpu_to_node(cpu_buffer->cpu), mflags,
1476 cpu_buffer->buffer->subbuf_order);
1623 if (!page)1477 if (!page)
1624 goto free_pages;1478 goto free_pages;
1625 bpage->page = page_address(page);1479 bpage->page = page_address(page);
1480 bpage->order = cpu_buffer->buffer->subbuf_order;
1626 rb_init_page(bpage->page);1481 rb_init_page(bpage->page);
1627 1482
1628 if (user_thread && fatal_signal_pending(current))1483 if (user_thread && fatal_signal_pending(current))
@@ -1701,7 +1556,8 @@ rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu) · 72 unmodified lines
1701 rb_check_bpage(cpu_buffer, bpage);1556 rb_check_bpage(cpu_buffer, bpage);
1702 1557
1703 cpu_buffer->reader_page = bpage;1558 cpu_buffer->reader_page = bpage;
1704 page = alloc_pages_node(cpu_to_node(cpu), GFP_KERNEL, 0);1559
1560 page = alloc_pages_node(cpu_to_node(cpu), GFP_KERNEL, cpu_buffer->buffer->subbuf_order);
1705 if (!page)1561 if (!page)
1706 goto fail_free_reader;1562 goto fail_free_reader;
1707 bpage->page = page_address(page);1563 bpage->page = page_address(page);
@@ -1784,7 +1640,14 @@ struct trace_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags, · 76 unmodified lines
1784 if (!zalloc_cpumask_var(&buffer->cpumask, GFP_KERNEL))1640 if (!zalloc_cpumask_var(&buffer->cpumask, GFP_KERNEL))
1785 goto fail_free_buffer;1641 goto fail_free_buffer;
1786 1642
1787 nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);1643 /* Default buffer page size - one system page */
1644 buffer->subbuf_order = 0;
1645 buffer->subbuf_size = PAGE_SIZE - BUF_PAGE_HDR_SIZE;
1646
1647 /* Max payload is buffer page size - header (8bytes) */
1648 buffer->max_data_size = buffer->subbuf_size - (sizeof(u32) * 2);
1649
1650 nr_pages = DIV_ROUND_UP(size, buffer->subbuf_size);
1788 buffer->flags = flags;1651 buffer->flags = flags;
1789 buffer->clock = trace_clock_local;1652 buffer->clock = trace_clock_local;
1790 buffer->reader_lock_key = key;1653 buffer->reader_lock_key = key;
@@ -2103,7 +1966,7 @@ static void update_pages_handler(struct work_struct *work) · 312 unmodified lines
2103 * @size: the new size.1966 * @size: the new size.
2104 * @cpu_id: the cpu buffer to resize1967 * @cpu_id: the cpu buffer to resize
2105 *1968 *
2106 * Minimum size is 2 * BUF_PAGE_SIZE.1969 * Minimum size is 2 * buffer->subbuf_size.
2107 *1970 *
2108 * Returns 0 on success and < 0 on failure.1971 * Returns 0 on success and < 0 on failure.
2109 */1972 */
@@ -2125,7 +1988,7 @@ int ring_buffer_resize(struct trace_buffer *buffer, unsigned long size, · 15 unmodified lines
2125 !cpumask_test_cpu(cpu_id, buffer->cpumask))1988 !cpumask_test_cpu(cpu_id, buffer->cpumask))
2126 return 0;1989 return 0;
2127 1990
2128 nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);1991 nr_pages = DIV_ROUND_UP(size, buffer->subbuf_size);
2129 1992
2130 /* we need a minimum of two pages */1993 /* we need a minimum of two pages */
2131 if (nr_pages < 2)1994 if (nr_pages < 2)
@@ -2372,7 +2235,7 @@ rb_iter_head_event(struct ring_buffer_iter *iter) · 240 unmodified lines
2372 */2235 */
2373 barrier();2236 barrier();
2374 2237
2375 if ((iter->head + length) > commit || length > BUF_PAGE_SIZE)2238 if ((iter->head + length) > commit || length > iter->event_size)
2376 /* Writer corrupted the read? */2239 /* Writer corrupted the read? */
2377 goto reset;2240 goto reset;
2378 2241
@@ -2412,11 +2275,13 @@ rb_commit_index(struct ring_buffer_per_cpu *cpu_buffer) · 33 unmodified lines
2412}2275}
2413 2276
2414static __always_inline unsigned2277static __always_inline unsigned
2415rb_event_index(struct ring_buffer_event *event)2278rb_event_index(struct ring_buffer_per_cpu *cpu_buffer, struct ring_buffer_event *event)
2416{2279{
2417 unsigned long addr = (unsigned long)event;2280 unsigned long addr = (unsigned long)event;
2418 2281
2419 return (addr & ~PAGE_MASK) - BUF_PAGE_HDR_SIZE;2282 addr &= (PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1;
2283
2284 return addr - BUF_PAGE_HDR_SIZE;
2420}2285}
2421 2286
2422static void rb_inc_iter(struct ring_buffer_iter *iter)2287static void rb_inc_iter(struct ring_buffer_iter *iter)
@@ -2605,6 +2470,7 @@ static inline void · 182 unmodified lines
2605rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer,2470rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer,
2606 unsigned long tail, struct rb_event_info *info)2471 unsigned long tail, struct rb_event_info *info)
2607{2472{
2473 unsigned long bsize = READ_ONCE(cpu_buffer->buffer->subbuf_size);
2608 struct buffer_page *tail_page = info->tail_page;2474 struct buffer_page *tail_page = info->tail_page;
2609 struct ring_buffer_event *event;2475 struct ring_buffer_event *event;
2610 unsigned long length = info->length;2476 unsigned long length = info->length;
@@ -2613,13 +2479,13 @@ rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer, · 2 unmodified lines
2613 * Only the event that crossed the page boundary2479 * Only the event that crossed the page boundary
2614 * must fill the old tail_page with padding.2480 * must fill the old tail_page with padding.
2615 */2481 */
2616 if (tail >= BUF_PAGE_SIZE) {2482 if (tail >= bsize) {
2617 /*2483 /*
2618 * If the page was filled, then we still need2484 * If the page was filled, then we still need
2619 * to update the real_end. Reset it to zero2485 * to update the real_end. Reset it to zero
2620 * and the reader will ignore it.2486 * and the reader will ignore it.
2621 */2487 */
2622 if (tail == BUF_PAGE_SIZE)2488 if (tail == bsize)
2623 tail_page->real_end = 0;2489 tail_page->real_end = 0;
2624 2490
2625 local_sub(length, &tail_page->write);2491 local_sub(length, &tail_page->write);
@@ -2647,7 +2513,7 @@ rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer, · 21 unmodified lines
2647 * If we are less than the minimum size, we don't need to2513 * If we are less than the minimum size, we don't need to
2648 * worry about it.2514 * worry about it.
2649 */2515 */
2650 if (tail > (BUF_PAGE_SIZE - RB_EVNT_MIN_SIZE)) {2516 if (tail > (bsize - RB_EVNT_MIN_SIZE)) {
2651 /* No room for any events */2517 /* No room for any events */
2652 2518
2653 /* Mark the rest of the page with padding */2519 /* Mark the rest of the page with padding */
@@ -2662,19 +2528,19 @@ rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer, · 8 unmodified lines
2662 }2528 }
2663 2529
2664 /* Put in a discarded event */2530 /* Put in a discarded event */
2665 event->array[0] = (BUF_PAGE_SIZE - tail) - RB_EVNT_HDR_SIZE;2531 event->array[0] = (bsize - tail) - RB_EVNT_HDR_SIZE;
2666 event->type_len = RINGBUF_TYPE_PADDING;2532 event->type_len = RINGBUF_TYPE_PADDING;
2667 /* time delta must be non zero */2533 /* time delta must be non zero */
2668 event->time_delta = 1;2534 event->time_delta = 1;
2669 2535
2670 /* account for padding bytes */2536 /* account for padding bytes */
2671 local_add(BUF_PAGE_SIZE - tail, &cpu_buffer->entries_bytes);2537 local_add(bsize - tail, &cpu_buffer->entries_bytes);
2672 2538
2673 /* Make sure the padding is visible before the tail_page->write update */2539 /* Make sure the padding is visible before the tail_page->write update */
2674 smp_wmb();2540 smp_wmb();
2675 2541
2676 /* Set write to end of buffer */2542 /* Set write to end of buffer */
2677 length = (tail + length) - BUF_PAGE_SIZE;2543 length = (tail + length) - bsize;
2678 local_sub(length, &tail_page->write);2544 local_sub(length, &tail_page->write);
2679}2545}
2680 2546
@@ -2788,7 +2654,8 @@ rb_move_tail(struct ring_buffer_per_cpu *cpu_buffer, · 107 unmodified lines
2788 2654
2789/* Slow path */2655/* Slow path */
2790static struct ring_buffer_event *2656static struct ring_buffer_event *
2791rb_add_time_stamp(struct ring_buffer_event *event, u64 delta, bool abs)2657rb_add_time_stamp(struct ring_buffer_per_cpu *cpu_buffer,
2658 struct ring_buffer_event *event, u64 delta, bool abs)
2792{2659{
2793 if (abs)2660 if (abs)
2794 event->type_len = RINGBUF_TYPE_TIME_STAMP;2661 event->type_len = RINGBUF_TYPE_TIME_STAMP;
@@ -2796,7 +2663,7 @@ rb_add_time_stamp(struct ring_buffer_event *event, u64 delta, bool abs) · 1 unmodified line
2796 event->type_len = RINGBUF_TYPE_TIME_EXTEND;2663 event->type_len = RINGBUF_TYPE_TIME_EXTEND;
2797 2664
2798 /* Not the first event on the page, or not delta? */2665 /* Not the first event on the page, or not delta? */
2799 if (abs || rb_event_index(event)) {2666 if (abs || rb_event_index(cpu_buffer, event)) {
2800 event->time_delta = delta & TS_MASK;2667 event->time_delta = delta & TS_MASK;
2801 event->array[0] = delta >> TS_SHIFT;2668 event->array[0] = delta >> TS_SHIFT;
2802 } else {2669 } else {
@@ -2826,7 +2693,7 @@ rb_check_timestamp(struct ring_buffer_per_cpu *cpu_buffer, · 23 unmodified lines
2826 (unsigned long long)info->ts,2693 (unsigned long long)info->ts,
2827 (unsigned long long)info->before,2694 (unsigned long long)info->before,
2828 (unsigned long long)info->after,2695 (unsigned long long)info->after,
2829 (unsigned long long)(rb_time_read(&cpu_buffer->write_stamp, &write_stamp) ? write_stamp : 0),2696 (unsigned long long)({rb_time_read(&cpu_buffer->write_stamp, &write_stamp); write_stamp;}),
2830 sched_clock_stable() ? "" :2697 sched_clock_stable() ? "" :
2831 "If you just came from a suspend/resume,\n"2698 "If you just came from a suspend/resume,\n"
2832 "please switch to the trace global clock:\n"2699 "please switch to the trace global clock:\n"
@@ -2870,7 +2737,7 @@ static void rb_add_timestamp(struct ring_buffer_per_cpu *cpu_buffer, · 37 unmodified lines
2870 if (!abs)2737 if (!abs)
2871 info->delta = 0;2738 info->delta = 0;
2872 }2739 }
2873 *event = rb_add_time_stamp(*event, info->delta, abs);2740 *event = rb_add_time_stamp(cpu_buffer, *event, info->delta, abs);
2874 *length -= RB_LEN_TIME_EXTEND;2741 *length -= RB_LEN_TIME_EXTEND;
2875 *delta = 0;2742 *delta = 0;
2876}2743}
@@ -2954,10 +2821,10 @@ rb_try_to_discard(struct ring_buffer_per_cpu *cpu_buffer, · 77 unmodified lines
2954 struct buffer_page *bpage;2821 struct buffer_page *bpage;
2955 unsigned long addr;2822 unsigned long addr;
2956 2823
2957 new_index = rb_event_index(event);2824 new_index = rb_event_index(cpu_buffer, event);
2958 old_index = new_index + rb_event_ts_length(event);2825 old_index = new_index + rb_event_ts_length(event);
2959 addr = (unsigned long)event;2826 addr = (unsigned long)event;
2960 addr &= PAGE_MASK;2827 addr &= ~((PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1);
2961 2828
2962 bpage = READ_ONCE(cpu_buffer->tail_page);2829 bpage = READ_ONCE(cpu_buffer->tail_page);
2963 2830
@@ -3344,6 +3211,76 @@ EXPORT_SYMBOL_GPL(ring_buffer_unlock_commit); · 380 unmodified lines
3344#define CHECK_FULL_PAGE 1L3211#define CHECK_FULL_PAGE 1L
3345 3212
3346#ifdef CONFIG_RING_BUFFER_VALIDATE_TIME_DELTAS3213#ifdef CONFIG_RING_BUFFER_VALIDATE_TIME_DELTAS
3214
3215static const char *show_irq_str(int bits)
3216{
3217 const char *type[] = {
3218 ".", // 0
3219 "s", // 1
3220 "h", // 2
3221 "Hs", // 3
3222 "n", // 4
3223 "Ns", // 5
3224 "Nh", // 6
3225 "NHs", // 7
3226 };
3227
3228 return type[bits];
3229}
3230
3231/* Assume this is an trace event */
3232static const char *show_flags(struct ring_buffer_event *event)
3233{
3234 struct trace_entry *entry;
3235 int bits = 0;
3236
3237 if (rb_event_data_length(event) - RB_EVNT_HDR_SIZE < sizeof(*entry))
3238 return "X";
3239
3240 entry = ring_buffer_event_data(event);
3241
3242 if (entry->flags & TRACE_FLAG_SOFTIRQ)
3243 bits |= 1;
3244
3245 if (entry->flags & TRACE_FLAG_HARDIRQ)
3246 bits |= 2;
3247
3248 if (entry->flags & TRACE_FLAG_NMI)
3249 bits |= 4;
3250
3251 return show_irq_str(bits);
3252}
3253
3254static const char *show_irq(struct ring_buffer_event *event)
3255{
3256 struct trace_entry *entry;
3257
3258 if (rb_event_data_length(event) - RB_EVNT_HDR_SIZE < sizeof(*entry))
3259 return "";
3260
3261 entry = ring_buffer_event_data(event);
3262 if (entry->flags & TRACE_FLAG_IRQS_OFF)
3263 return "d";
3264 return "";
3265}
3266
3267static const char *show_interrupt_level(void)
3268{
3269 unsigned long pc = preempt_count();
3270 unsigned char level = 0;
3271
3272 if (pc & SOFTIRQ_OFFSET)
3273 level |= 1;
3274
3275 if (pc & HARDIRQ_MASK)
3276 level |= 2;
3277
3278 if (pc & NMI_MASK)
3279 level |= 4;
3280
3281 return show_irq_str(level);
3282}
3283
3347static void dump_buffer_page(struct buffer_data_page *bpage,3284static void dump_buffer_page(struct buffer_data_page *bpage,
3348 struct rb_event_info *info,3285 struct rb_event_info *info,
3349 unsigned long tail)3286 unsigned long tail)
@@ -3364,34 +3301,57 @@ static void dump_buffer_page(struct buffer_data_page *bpage, · 14 unmodified lines
3364 case RINGBUF_TYPE_TIME_EXTEND:3301 case RINGBUF_TYPE_TIME_EXTEND:
3365 delta = rb_event_time_stamp(event);3302 delta = rb_event_time_stamp(event);
3366 ts += delta;3303 ts += delta;
3367 pr_warn(" [%lld] delta:%lld TIME EXTEND\n", ts, delta);3304 pr_warn(" 0x%x: [%lld] delta:%lld TIME EXTEND\n",
3305 e, ts, delta);
3368 break;3306 break;
3369 3307
3370 case RINGBUF_TYPE_TIME_STAMP:3308 case RINGBUF_TYPE_TIME_STAMP:
3371 delta = rb_event_time_stamp(event);3309 delta = rb_event_time_stamp(event);
3372 ts = rb_fix_abs_ts(delta, ts);3310 ts = rb_fix_abs_ts(delta, ts);
3373 pr_warn(" [%lld] absolute:%lld TIME STAMP\n", ts, delta);3311 pr_warn(" 0x%x: [%lld] absolute:%lld TIME STAMP\n",
3312 e, ts, delta);
3374 break;3313 break;
3375 3314
3376 case RINGBUF_TYPE_PADDING:3315 case RINGBUF_TYPE_PADDING:
3377 ts += event->time_delta;3316 ts += event->time_delta;
3378 pr_warn(" [%lld] delta:%d PADDING\n", ts, event->time_delta);3317 pr_warn(" 0x%x: [%lld] delta:%d PADDING\n",
3318 e, ts, event->time_delta);
3379 break;3319 break;
3380 3320
3381 case RINGBUF_TYPE_DATA:3321 case RINGBUF_TYPE_DATA:
3382 ts += event->time_delta;3322 ts += event->time_delta;
3383 pr_warn(" [%lld] delta:%d\n", ts, event->time_delta);3323 pr_warn(" 0x%x: [%lld] delta:%d %s%s\n",
3324 e, ts, event->time_delta,
3325 show_flags(event), show_irq(event));
3384 break;3326 break;
3385 3327
3386 default:3328 default:
3387 break;3329 break;
3388 }3330 }
3389 }3331 }
3332 pr_warn("expected end:0x%lx last event actually ended at:0x%x\n", tail, e);
3390}3333}
3391 3334
3392static DEFINE_PER_CPU(atomic_t, checking);3335static DEFINE_PER_CPU(atomic_t, checking);
3393static atomic_t ts_dump;3336static atomic_t ts_dump;
3394 3337
3338#define buffer_warn_return(fmt, ...) \
3339 do { \
3340 /* If another report is happening, ignore this one */ \
3341 if (atomic_inc_return(&ts_dump) != 1) { \
3342 atomic_dec(&ts_dump); \
3343 goto out; \
3344 } \
3345 atomic_inc(&cpu_buffer->record_disabled); \
3346 pr_warn(fmt, ##__VA_ARGS__); \
3347 dump_buffer_page(bpage, info, tail); \
3348 atomic_dec(&ts_dump); \
3349 /* There's some cases in boot up that this can happen */ \
3350 if (WARN_ON_ONCE(system_state != SYSTEM_BOOTING)) \
3351 /* Do not re-enable checking */ \
3352 return; \
3353 } while (0)
3354
3395/*3355/*
3396 * Check if the current event time stamp matches the deltas on3356 * Check if the current event time stamp matches the deltas on
3397 * the buffer page.3357 * the buffer page.
@@ -3445,7 +3405,12 @@ static void check_buffer(struct ring_buffer_per_cpu *cpu_buffer, · 47 unmodified lines
3445 3405
3446 case RINGBUF_TYPE_TIME_STAMP:3406 case RINGBUF_TYPE_TIME_STAMP:
3447 delta = rb_event_time_stamp(event);3407 delta = rb_event_time_stamp(event);
3448 ts = rb_fix_abs_ts(delta, ts);3408 delta = rb_fix_abs_ts(delta, ts);
3409 if (delta < ts) {
3410 buffer_warn_return("[CPU: %d]ABSOLUTE TIME WENT BACKWARDS: last ts: %lld absolute ts: %lld\n",
3411 cpu_buffer->cpu, ts, delta);
3412 }
3413 ts = delta;
3449 break;3414 break;
3450 3415
3451 case RINGBUF_TYPE_PADDING:3416 case RINGBUF_TYPE_PADDING:
@@ -3462,23 +3427,11 @@ static void check_buffer(struct ring_buffer_per_cpu *cpu_buffer, · 10 unmodified lines
3462 }3427 }
3463 if ((full && ts > info->ts) ||3428 if ((full && ts > info->ts) ||
3464 (!full && ts + info->delta != info->ts)) {3429 (!full && ts + info->delta != info->ts)) {
3465 /* If another report is happening, ignore this one */3430 buffer_warn_return("[CPU: %d]TIME DOES NOT MATCH expected:%lld actual:%lld delta:%lld before:%lld after:%lld%s context:%s\n",
3466 if (atomic_inc_return(&ts_dump) != 1) {3431 cpu_buffer->cpu,
3467 atomic_dec(&ts_dump);3432 ts + info->delta, info->ts, info->delta,
3468 goto out;3433 info->before, info->after,
3469 }3434 full ? " (full)" : "", show_interrupt_level());
3470 atomic_inc(&cpu_buffer->record_disabled);
3471 /* There's some cases in boot up that this can happen */
3472 WARN_ON_ONCE(system_state != SYSTEM_BOOTING);
3473 pr_warn("[CPU: %d]TIME DOES NOT MATCH expected:%lld actual:%lld delta:%lld before:%lld after:%lld%s\n",
3474 cpu_buffer->cpu,
3475 ts + info->delta, info->ts, info->delta,
3476 info->before, info->after,
3477 full ? " (full)" : "");
3478 dump_buffer_page(bpage, info, tail);
3479 atomic_dec(&ts_dump);
3480 /* Do not re-enable checking */
3481 return;
3482 }3435 }
3483out:3436out:
3484 atomic_dec(this_cpu_ptr(&checking));3437 atomic_dec(this_cpu_ptr(&checking));
@@ -3498,16 +3451,14 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer, · 13 unmodified lines
3498 struct ring_buffer_event *event;3451 struct ring_buffer_event *event;
3499 struct buffer_page *tail_page;3452 struct buffer_page *tail_page;
3500 unsigned long tail, write, w;3453 unsigned long tail, write, w;
3501 bool a_ok;
3502 bool b_ok;
3503 3454
3504 /* Don't let the compiler play games with cpu_buffer->tail_page */3455 /* Don't let the compiler play games with cpu_buffer->tail_page */
3505 tail_page = info->tail_page = READ_ONCE(cpu_buffer->tail_page);3456 tail_page = info->tail_page = READ_ONCE(cpu_buffer->tail_page);
3506 3457
3507 /*A*/ w = local_read(&tail_page->write) & RB_WRITE_MASK;3458 /*A*/ w = local_read(&tail_page->write) & RB_WRITE_MASK;
3508 barrier();3459 barrier();
3509 b_ok = rb_time_read(&cpu_buffer->before_stamp, &info->before);3460 rb_time_read(&cpu_buffer->before_stamp, &info->before);
3510 a_ok = rb_time_read(&cpu_buffer->write_stamp, &info->after);3461 rb_time_read(&cpu_buffer->write_stamp, &info->after);
3511 barrier();3462 barrier();
3512 info->ts = rb_time_stamp(cpu_buffer->buffer);3463 info->ts = rb_time_stamp(cpu_buffer->buffer);
3513 3464
@@ -3522,7 +3473,7 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer, · 8 unmodified lines
3522 if (!w) {3473 if (!w) {
3523 /* Use the sub-buffer timestamp */3474 /* Use the sub-buffer timestamp */
3524 info->delta = 0;3475 info->delta = 0;
3525 } else if (unlikely(!a_ok || !b_ok || info->before != info->after)) {3476 } else if (unlikely(info->before != info->after)) {
3526 info->add_timestamp |= RB_ADD_STAMP_FORCE | RB_ADD_STAMP_EXTEND;3477 info->add_timestamp |= RB_ADD_STAMP_FORCE | RB_ADD_STAMP_EXTEND;
3527 info->length += RB_LEN_TIME_EXTEND;3478 info->length += RB_LEN_TIME_EXTEND;
3528 } else {3479 } else {
@@ -3544,7 +3495,7 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer, · 15 unmodified lines
3544 tail = write - info->length;3495 tail = write - info->length;
3545 3496
3546 /* See if we shot pass the end of this buffer page */3497 /* See if we shot pass the end of this buffer page */
3547 if (unlikely(write > BUF_PAGE_SIZE)) {3498 if (unlikely(write > cpu_buffer->buffer->subbuf_size)) {
3548 check_buffer(cpu_buffer, info, CHECK_FULL_PAGE);3499 check_buffer(cpu_buffer, info, CHECK_FULL_PAGE);
3549 return rb_move_tail(cpu_buffer, tail, info);3500 return rb_move_tail(cpu_buffer, tail, info);
3550 }3501 }
@@ -3571,8 +3522,7 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer, · 20 unmodified lines
3571 /* SLOW PATH - Interrupted between A and C */3522 /* SLOW PATH - Interrupted between A and C */
3572 3523
3573 /* Save the old before_stamp */3524 /* Save the old before_stamp */
3574 a_ok = rb_time_read(&cpu_buffer->before_stamp, &info->before);3525 rb_time_read(&cpu_buffer->before_stamp, &info->before);
3575 RB_WARN_ON(cpu_buffer, !a_ok);
3576 3526
3577 /*3527 /*
3578 * Read a new timestamp and update the before_stamp to make3528 * Read a new timestamp and update the before_stamp to make
@@ -3584,9 +3534,7 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer, · 5 unmodified lines
3584 rb_time_set(&cpu_buffer->before_stamp, ts);3534 rb_time_set(&cpu_buffer->before_stamp, ts);
3585 3535
3586 barrier();3536 barrier();
3587 /*E*/ a_ok = rb_time_read(&cpu_buffer->write_stamp, &info->after);3537 /*E*/ rb_time_read(&cpu_buffer->write_stamp, &info->after);
3588 /* Was interrupted before here, write_stamp must be valid */
3589 RB_WARN_ON(cpu_buffer, !a_ok);
3590 barrier();3538 barrier();
3591 /*F*/ if (write == (local_read(&tail_page->write) & RB_WRITE_MASK) &&3539 /*F*/ if (write == (local_read(&tail_page->write) & RB_WRITE_MASK) &&
3592 info->after == info->before && info->after < ts) {3540 info->after == info->before && info->after < ts) {
@@ -3678,7 +3626,7 @@ rb_reserve_next_event(struct trace_buffer *buffer, · 85 unmodified lines
3678 if (ring_buffer_time_stamp_abs(cpu_buffer->buffer)) {3626 if (ring_buffer_time_stamp_abs(cpu_buffer->buffer)) {
3679 add_ts_default = RB_ADD_STAMP_ABSOLUTE;3627 add_ts_default = RB_ADD_STAMP_ABSOLUTE;
3680 info.length += RB_LEN_TIME_EXTEND;3628 info.length += RB_LEN_TIME_EXTEND;
3681 if (info.length > BUF_MAX_DATA_SIZE)3629 if (info.length > cpu_buffer->buffer->max_data_size)
3682 goto out_fail;3630 goto out_fail;
3683 } else {3631 } else {
3684 add_ts_default = RB_ADD_STAMP_NONE;3632 add_ts_default = RB_ADD_STAMP_NONE;
@@ -3753,7 +3701,7 @@ ring_buffer_lock_reserve(struct trace_buffer *buffer, unsigned long length) · 68 unmodified lines
3753 if (unlikely(atomic_read(&cpu_buffer->record_disabled)))3701 if (unlikely(atomic_read(&cpu_buffer->record_disabled)))
3754 goto out;3702 goto out;
3755 3703
3756 if (unlikely(length > BUF_MAX_DATA_SIZE))3704 if (unlikely(length > buffer->max_data_size))
3757 goto out;3705 goto out;
3758 3706
3759 if (unlikely(trace_recursive_lock(cpu_buffer)))3707 if (unlikely(trace_recursive_lock(cpu_buffer)))
@@ -3787,7 +3735,7 @@ rb_decrement_entry(struct ring_buffer_per_cpu *cpu_buffer, · 27 unmodified lines
3787 struct buffer_page *bpage = cpu_buffer->commit_page;3735 struct buffer_page *bpage = cpu_buffer->commit_page;
3788 struct buffer_page *start;3736 struct buffer_page *start;
3789 3737
3790 addr &= PAGE_MASK;3738 addr &= ~((PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1);
3791 3739
3792 /* Do the likely case first */3740 /* Do the likely case first */
3793 if (likely(bpage->page == (void *)addr)) {3741 if (likely(bpage->page == (void *)addr)) {
@@ -3903,7 +3851,7 @@ int ring_buffer_write(struct trace_buffer *buffer, · 109 unmodified lines
3903 if (atomic_read(&cpu_buffer->record_disabled))3851 if (atomic_read(&cpu_buffer->record_disabled))
3904 goto out;3852 goto out;
3905 3853
3906 if (length > BUF_MAX_DATA_SIZE)3854 if (length > buffer->max_data_size)
3907 goto out;3855 goto out;
3908 3856
3909 if (unlikely(trace_recursive_lock(cpu_buffer)))3857 if (unlikely(trace_recursive_lock(cpu_buffer)))
@@ -4483,6 +4431,7 @@ static struct buffer_page * · 573 unmodified lines
4483rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)4431rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
4484{4432{
4485 struct buffer_page *reader = NULL;4433 struct buffer_page *reader = NULL;
4434 unsigned long bsize = READ_ONCE(cpu_buffer->buffer->subbuf_size);
4486 unsigned long overwrite;4435 unsigned long overwrite;
4487 unsigned long flags;4436 unsigned long flags;
4488 int nr_loops = 0;4437 int nr_loops = 0;
@@ -4618,7 +4567,7 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) · 129 unmodified lines
4618#define USECS_WAIT 10000004567#define USECS_WAIT 1000000
4619 for (nr_loops = 0; nr_loops < USECS_WAIT; nr_loops++) {4568 for (nr_loops = 0; nr_loops < USECS_WAIT; nr_loops++) {
4620 /* If the write is past the end of page, a writer is still updating it */4569 /* If the write is past the end of page, a writer is still updating it */
4621 if (likely(!reader || rb_page_write(reader) <= BUF_PAGE_SIZE))4570 if (likely(!reader || rb_page_write(reader) <= bsize))
4622 break;4571 break;
4623 4572
4624 udelay(1);4573 udelay(1);
@@ -5062,7 +5011,8 @@ ring_buffer_read_prepare(struct trace_buffer *buffer, int cpu, gfp_t flags) · 437 unmodified lines
5062 return NULL;5011 return NULL;
5063 5012
5064 /* Holds the entire event: data and meta data */5013 /* Holds the entire event: data and meta data */
5065 iter->event = kmalloc(BUF_PAGE_SIZE, flags);5014 iter->event_size = buffer->subbuf_size;
5015 iter->event = kmalloc(iter->event_size, flags);
5066 if (!iter->event) {5016 if (!iter->event) {
5067 kfree(iter);5017 kfree(iter);
5068 return NULL;5018 return NULL;
@@ -5178,19 +5128,28 @@ EXPORT_SYMBOL_GPL(ring_buffer_iter_advance); · 109 unmodified lines
5178 */5128 */
5179unsigned long ring_buffer_size(struct trace_buffer *buffer, int cpu)5129unsigned long ring_buffer_size(struct trace_buffer *buffer, int cpu)
5180{5130{
5181 /*
5182 * Earlier, this method returned
5183 * BUF_PAGE_SIZE * buffer->nr_pages
5184 * Since the nr_pages field is now removed, we have converted this to
5185 * return the per cpu buffer value.
5186 */
5187 if (!cpumask_test_cpu(cpu, buffer->cpumask))5131 if (!cpumask_test_cpu(cpu, buffer->cpumask))
5188 return 0;5132 return 0;
5189 5133
5190 return BUF_PAGE_SIZE * buffer->buffers[cpu]->nr_pages;5134 return buffer->subbuf_size * buffer->buffers[cpu]->nr_pages;
5191}5135}
5192EXPORT_SYMBOL_GPL(ring_buffer_size);5136EXPORT_SYMBOL_GPL(ring_buffer_size);
5193 5137
5138/**
5139 * ring_buffer_max_event_size - return the max data size of an event
5140 * @buffer: The ring buffer.
5141 *
5142 * Returns the maximum size an event can be.
5143 */
5144unsigned long ring_buffer_max_event_size(struct trace_buffer *buffer)
5145{
5146 /* If abs timestamp is requested, events have a timestamp too */
5147 if (ring_buffer_time_stamp_abs(buffer))
5148 return buffer->max_data_size - RB_LEN_TIME_EXTEND;
5149 return buffer->max_data_size;
5150}
5151EXPORT_SYMBOL_GPL(ring_buffer_max_event_size);
5152
5194static void rb_clear_buffer_page(struct buffer_page *page)5153static void rb_clear_buffer_page(struct buffer_page *page)
5195{5154{
5196 local_set(&page->write, 0);5155 local_set(&page->write, 0);
@@ -5461,6 +5420,9 @@ int ring_buffer_swap_cpu(struct trace_buffer *buffer_a, · 264 unmodified lines
5461 if (cpu_buffer_a->nr_pages != cpu_buffer_b->nr_pages)5420 if (cpu_buffer_a->nr_pages != cpu_buffer_b->nr_pages)
5462 goto out;5421 goto out;
5463 5422
5423 if (buffer_a->subbuf_order != buffer_b->subbuf_order)
5424 goto out;
5425
5464 ret = -EAGAIN;5426 ret = -EAGAIN;
5465 5427
5466 if (atomic_read(&buffer_a->record_disabled))5428 if (atomic_read(&buffer_a->record_disabled))
@@ -5532,40 +5494,48 @@ EXPORT_SYMBOL_GPL(ring_buffer_swap_cpu); · 65 unmodified lines
5532 * Returns:5494 * Returns:
5533 * The page allocated, or ERR_PTR5495 * The page allocated, or ERR_PTR
5534 */5496 */
5535void *ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu)5497struct buffer_data_read_page *
5498ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu)
5536{5499{
5537 struct ring_buffer_per_cpu *cpu_buffer;5500 struct ring_buffer_per_cpu *cpu_buffer;
5538 struct buffer_data_page *bpage = NULL;5501 struct buffer_data_read_page *bpage = NULL;
5539 unsigned long flags;5502 unsigned long flags;
5540 struct page *page;5503 struct page *page;
5541 5504
5542 if (!cpumask_test_cpu(cpu, buffer->cpumask))5505 if (!cpumask_test_cpu(cpu, buffer->cpumask))
5543 return ERR_PTR(-ENODEV);5506 return ERR_PTR(-ENODEV);
5544 5507
5508 bpage = kzalloc(sizeof(*bpage), GFP_KERNEL);
5509 if (!bpage)
5510 return ERR_PTR(-ENOMEM);
5511
5512 bpage->order = buffer->subbuf_order;
5545 cpu_buffer = buffer->buffers[cpu];5513 cpu_buffer = buffer->buffers[cpu];
5546 local_irq_save(flags);5514 local_irq_save(flags);
5547 arch_spin_lock(&cpu_buffer->lock);5515 arch_spin_lock(&cpu_buffer->lock);
5548 5516
5549 if (cpu_buffer->free_page) {5517 if (cpu_buffer->free_page) {
5550 bpage = cpu_buffer->free_page;5518 bpage->data = cpu_buffer->free_page;
5551 cpu_buffer->free_page = NULL;5519 cpu_buffer->free_page = NULL;
5552 }5520 }
5553 5521
5554 arch_spin_unlock(&cpu_buffer->lock);5522 arch_spin_unlock(&cpu_buffer->lock);
5555 local_irq_restore(flags);5523 local_irq_restore(flags);
5556 5524
5557 if (bpage)5525 if (bpage->data)
5558 goto out;5526 goto out;
5559 5527
5560 page = alloc_pages_node(cpu_to_node(cpu),5528 page = alloc_pages_node(cpu_to_node(cpu), GFP_KERNEL | __GFP_NORETRY,
5561 GFP_KERNEL | __GFP_NORETRY, 0);5529 cpu_buffer->buffer->subbuf_order);
5562 if (!page)5530 if (!page) {
5531 kfree(bpage);
5563 return ERR_PTR(-ENOMEM);5532 return ERR_PTR(-ENOMEM);
5533 }
5564 5534
5565 bpage = page_address(page);5535 bpage->data = page_address(page);
5566 5536
5567 out:5537 out:
5568 rb_init_page(bpage);5538 rb_init_page(bpage->data);
5569 5539
5570 return bpage;5540 return bpage;
5571}5541}
@@ -5575,14 +5545,15 @@ EXPORT_SYMBOL_GPL(ring_buffer_alloc_read_page); · 3 unmodified lines
5575 * ring_buffer_free_read_page - free an allocated read page5545 * ring_buffer_free_read_page - free an allocated read page
5576 * @buffer: the buffer the page was allocate for5546 * @buffer: the buffer the page was allocate for
5577 * @cpu: the cpu buffer the page came from5547 * @cpu: the cpu buffer the page came from
5578 * @data: the page to free5548 * @data_page: the page to free
5579 *5549 *
5580 * Free a page allocated from ring_buffer_alloc_read_page.5550 * Free a page allocated from ring_buffer_alloc_read_page.
5581 */5551 */
5582void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, void *data)5552void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu,
5553 struct buffer_data_read_page *data_page)
5583{5554{
5584 struct ring_buffer_per_cpu *cpu_buffer;5555 struct ring_buffer_per_cpu *cpu_buffer;
5585 struct buffer_data_page *bpage = data;5556 struct buffer_data_page *bpage = data_page->data;
5586 struct page *page = virt_to_page(bpage);5557 struct page *page = virt_to_page(bpage);
5587 unsigned long flags;5558 unsigned long flags;
5588 5559
@@ -5591,8 +5562,12 @@ void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, void *data · 2 unmodified lines
5591 5562
5592 cpu_buffer = buffer->buffers[cpu];5563 cpu_buffer = buffer->buffers[cpu];
5593 5564
5594 /* If the page is still in use someplace else, we can't reuse it */5565 /*
5595 if (page_ref_count(page) > 1)5566 * If the page is still in use someplace else, or order of the page
5567 * is different from the subbuffer order of the buffer -
5568 * we can't reuse it
5569 */
5570 if (page_ref_count(page) > 1 || data_page->order != buffer->subbuf_order)
5596 goto out;5571 goto out;
5597 5572
5598 local_irq_save(flags);5573 local_irq_save(flags);
@@ -5607,7 +5582,8 @@ void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, void *data · 8 unmodified lines
5607 local_irq_restore(flags);5582 local_irq_restore(flags);
5608 5583
5609 out:5584 out:
5610 free_page((unsigned long)bpage);5585 free_pages((unsigned long)bpage, data_page->order);
5586 kfree(data_page);
5611}5587}
5612EXPORT_SYMBOL_GPL(ring_buffer_free_read_page);5588EXPORT_SYMBOL_GPL(ring_buffer_free_read_page);
5613 5589
@@ -5628,9 +5604,10 @@ EXPORT_SYMBOL_GPL(ring_buffer_free_read_page); · 14 unmodified lines
5628 * rpage = ring_buffer_alloc_read_page(buffer, cpu);5604 * rpage = ring_buffer_alloc_read_page(buffer, cpu);
5629 * if (IS_ERR(rpage))5605 * if (IS_ERR(rpage))
5630 * return PTR_ERR(rpage);5606 * return PTR_ERR(rpage);
5631 * ret = ring_buffer_read_page(buffer, &rpage, len, cpu, 0);5607 * ret = ring_buffer_read_page(buffer, rpage, len, cpu, 0);
5632 * if (ret >= 0)5608 * if (ret >= 0)
5633 * process_page(rpage, ret);5609 * process_page(ring_buffer_read_page_data(rpage), ret);
5610 * ring_buffer_free_read_page(buffer, cpu, rpage);
5634 *5611 *
5635 * When @full is set, the function will not return true unless5612 * When @full is set, the function will not return true unless
5636 * the writer is off the reader page.5613 * the writer is off the reader page.
@@ -5645,7 +5622,8 @@ EXPORT_SYMBOL_GPL(ring_buffer_free_read_page); · 8 unmodified lines
5645 * <0 if no data has been transferred.5622 * <0 if no data has been transferred.
5646 */5623 */
5647int ring_buffer_read_page(struct trace_buffer *buffer,5624int ring_buffer_read_page(struct trace_buffer *buffer,
5648 void **data_page, size_t len, int cpu, int full)5625 struct buffer_data_read_page *data_page,
5626 size_t len, int cpu, int full)
5649{5627{
5650 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];5628 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
5651 struct ring_buffer_event *event;5629 struct ring_buffer_event *event;
@@ -5670,10 +5648,12 @@ int ring_buffer_read_page(struct trace_buffer *buffer, · 18 unmodified lines
5670 5648
5671 len -= BUF_PAGE_HDR_SIZE;5649 len -= BUF_PAGE_HDR_SIZE;
5672 5650
5673 if (!data_page)5651 if (!data_page || !data_page->data)
5652 goto out;
5653 if (data_page->order != buffer->subbuf_order)
5674 goto out;5654 goto out;
5675 5655
5676 bpage = *data_page;5656 bpage = data_page->data;
5677 if (!bpage)5657 if (!bpage)
5678 goto out;5658 goto out;
5679 5659
@@ -5767,11 +5747,11 @@ int ring_buffer_read_page(struct trace_buffer *buffer, · 87 unmodified lines
5767 /* swap the pages */5747 /* swap the pages */
5768 rb_init_page(bpage);5748 rb_init_page(bpage);
5769 bpage = reader->page;5749 bpage = reader->page;
5770 reader->page = *data_page;5750 reader->page = data_page->data;
5771 local_set(&reader->write, 0);5751 local_set(&reader->write, 0);
5772 local_set(&reader->entries, 0);5752 local_set(&reader->entries, 0);
5773 reader->read = 0;5753 reader->read = 0;
5774 *data_page = bpage;5754 data_page->data = bpage;
5775 5755
5776 /*5756 /*
5777 * Use the real_end for the data size,5757 * Use the real_end for the data size,
@@ -5793,7 +5773,7 @@ int ring_buffer_read_page(struct trace_buffer *buffer, · 15 unmodified lines
5793 /* If there is room at the end of the page to save the5773 /* If there is room at the end of the page to save the
5794 * missed events, then record it there.5774 * missed events, then record it there.
5795 */5775 */
5796 if (BUF_PAGE_SIZE - commit >= sizeof(missed_events)) {5776 if (buffer->subbuf_size - commit >= sizeof(missed_events)) {
5797 memcpy(&bpage->data[commit], &missed_events,5777 memcpy(&bpage->data[commit], &missed_events,
5798 sizeof(missed_events));5778 sizeof(missed_events));
5799 local_add(RB_MISSED_STORED, &bpage->commit);5779 local_add(RB_MISSED_STORED, &bpage->commit);
@@ -5805,8 +5785,8 @@ int ring_buffer_read_page(struct trace_buffer *buffer, · 5 unmodified lines
5805 /*5785 /*
5806 * This page may be off to user land. Zero it out here.5786 * This page may be off to user land. Zero it out here.
5807 */5787 */
5808 if (commit < BUF_PAGE_SIZE)5788 if (commit < buffer->subbuf_size)
5809 memset(&bpage->data[commit], 0, BUF_PAGE_SIZE - commit);5789 memset(&bpage->data[commit], 0, buffer->subbuf_size - commit);
5810 5790
5811 out_unlock:5791 out_unlock:
5812 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);5792 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
@@ -5816,6 +5796,209 @@ int ring_buffer_read_page(struct trace_buffer *buffer, · 3 unmodified lines
5816}5796}
5817EXPORT_SYMBOL_GPL(ring_buffer_read_page);5797EXPORT_SYMBOL_GPL(ring_buffer_read_page);
5818 5798
5799/**
5800 * ring_buffer_read_page_data - get pointer to the data in the page.
5801 * @page: the page to get the data from
5802 *
5803 * Returns pointer to the actual data in this page.
5804 */
5805void *ring_buffer_read_page_data(struct buffer_data_read_page *page)
5806{
5807 return page->data;
5808}
5809EXPORT_SYMBOL_GPL(ring_buffer_read_page_data);
5810
5811/**
5812 * ring_buffer_subbuf_size_get - get size of the sub buffer.
5813 * @buffer: the buffer to get the sub buffer size from
5814 *
5815 * Returns size of the sub buffer, in bytes.
5816 */
5817int ring_buffer_subbuf_size_get(struct trace_buffer *buffer)
5818{
5819 return buffer->subbuf_size + BUF_PAGE_HDR_SIZE;
5820}
5821EXPORT_SYMBOL_GPL(ring_buffer_subbuf_size_get);
5822
5823/**
5824 * ring_buffer_subbuf_order_get - get order of system sub pages in one buffer page.
5825 * @buffer: The ring_buffer to get the system sub page order from
5826 *
5827 * By default, one ring buffer sub page equals to one system page. This parameter
5828 * is configurable, per ring buffer. The size of the ring buffer sub page can be
5829 * extended, but must be an order of system page size.
5830 *
5831 * Returns the order of buffer sub page size, in system pages:
5832 * 0 means the sub buffer size is 1 system page and so forth.
5833 * In case of an error < 0 is returned.
5834 */
5835int ring_buffer_subbuf_order_get(struct trace_buffer *buffer)
5836{
5837 if (!buffer)
5838 return -EINVAL;
5839
5840 return buffer->subbuf_order;
5841}
5842EXPORT_SYMBOL_GPL(ring_buffer_subbuf_order_get);
5843
5844/**
5845 * ring_buffer_subbuf_order_set - set the size of ring buffer sub page.
5846 * @buffer: The ring_buffer to set the new page size.
5847 * @order: Order of the system pages in one sub buffer page
5848 *
5849 * By default, one ring buffer pages equals to one system page. This API can be
5850 * used to set new size of the ring buffer page. The size must be order of
5851 * system page size, that's why the input parameter @order is the order of
5852 * system pages that are allocated for one ring buffer page:
5853 * 0 - 1 system page
5854 * 1 - 2 system pages
5855 * 3 - 4 system pages
5856 * ...
5857 *
5858 * Returns 0 on success or < 0 in case of an error.
5859 */
5860int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order)
5861{
5862 struct ring_buffer_per_cpu *cpu_buffer;
5863 struct buffer_page *bpage, *tmp;
5864 int old_order, old_size;
5865 int nr_pages;
5866 int psize;
5867 int err;
5868 int cpu;
5869
5870 if (!buffer || order < 0)
5871 return -EINVAL;
5872
5873 if (buffer->subbuf_order == order)
5874 return 0;
5875
5876 psize = (1 << order) * PAGE_SIZE;
5877 if (psize <= BUF_PAGE_HDR_SIZE)
5878 return -EINVAL;
5879
5880 old_order = buffer->subbuf_order;
5881 old_size = buffer->subbuf_size;
5882
5883 /* prevent another thread from changing buffer sizes */
5884 mutex_lock(&buffer->mutex);
5885 atomic_inc(&buffer->record_disabled);
5886
5887 /* Make sure all commits have finished */
5888 synchronize_rcu();
5889
5890 buffer->subbuf_order = order;
5891 buffer->subbuf_size = psize - BUF_PAGE_HDR_SIZE;
5892
5893 /* Make sure all new buffers are allocated, before deleting the old ones */
5894 for_each_buffer_cpu(buffer, cpu) {
5895
5896 if (!cpumask_test_cpu(cpu, buffer->cpumask))
5897 continue;
5898
5899 cpu_buffer = buffer->buffers[cpu];
5900
5901 /* Update the number of pages to match the new size */
5902 nr_pages = old_size * buffer->buffers[cpu]->nr_pages;
5903 nr_pages = DIV_ROUND_UP(nr_pages, buffer->subbuf_size);
5904
5905 /* we need a minimum of two pages */
5906 if (nr_pages < 2)
5907 nr_pages = 2;
5908
5909 cpu_buffer->nr_pages_to_update = nr_pages;
5910
5911 /* Include the reader page */
5912 nr_pages++;
5913
5914 /* Allocate the new size buffer */
5915 INIT_LIST_HEAD(&cpu_buffer->new_pages);
5916 if (__rb_allocate_pages(cpu_buffer, nr_pages,
5917 &cpu_buffer->new_pages)) {
5918 /* not enough memory for new pages */
5919 err = -ENOMEM;
5920 goto error;
5921 }
5922 }
5923
5924 for_each_buffer_cpu(buffer, cpu) {
5925
5926 if (!cpumask_test_cpu(cpu, buffer->cpumask))
5927 continue;
5928
5929 cpu_buffer = buffer->buffers[cpu];
5930
5931 /* Clear the head bit to make the link list normal to read */
5932 rb_head_page_deactivate(cpu_buffer);
5933
5934 /* Now walk the list and free all the old sub buffers */
5935 list_for_each_entry_safe(bpage, tmp, cpu_buffer->pages, list) {
5936 list_del_init(&bpage->list);
5937 free_buffer_page(bpage);
5938 }
5939 /* The above loop stopped an the last page needing to be freed */
5940 bpage = list_entry(cpu_buffer->pages, struct buffer_page, list);
5941 free_buffer_page(bpage);
5942
5943 /* Free the current reader page */
5944 free_buffer_page(cpu_buffer->reader_page);
5945
5946 /* One page was allocated for the reader page */
5947 cpu_buffer->reader_page = list_entry(cpu_buffer->new_pages.next,
5948 struct buffer_page, list);
5949 list_del_init(&cpu_buffer->reader_page->list);
5950
5951 /* The cpu_buffer pages are a link list with no head */
5952 cpu_buffer->pages = cpu_buffer->new_pages.next;
5953 cpu_buffer->new_pages.next->prev = cpu_buffer->new_pages.prev;
5954 cpu_buffer->new_pages.prev->next = cpu_buffer->new_pages.next;
5955
5956 /* Clear the new_pages list */
5957 INIT_LIST_HEAD(&cpu_buffer->new_pages);
5958
5959 cpu_buffer->head_page
5960 = list_entry(cpu_buffer->pages, struct buffer_page, list);
5961 cpu_buffer->tail_page = cpu_buffer->commit_page = cpu_buffer->head_page;
5962
5963 cpu_buffer->nr_pages = cpu_buffer->nr_pages_to_update;
5964 cpu_buffer->nr_pages_to_update = 0;
5965
5966 free_pages((unsigned long)cpu_buffer->free_page, old_order);
5967 cpu_buffer->free_page = NULL;
5968
5969 rb_head_page_activate(cpu_buffer);
5970
5971 rb_check_pages(cpu_buffer);
5972 }
5973
5974 atomic_dec(&buffer->record_disabled);
5975 mutex_unlock(&buffer->mutex);
5976
5977 return 0;
5978
5979error:
5980 buffer->subbuf_order = old_order;
5981 buffer->subbuf_size = old_size;
5982
5983 atomic_dec(&buffer->record_disabled);
5984 mutex_unlock(&buffer->mutex);
5985
5986 for_each_buffer_cpu(buffer, cpu) {
5987 cpu_buffer = buffer->buffers[cpu];
5988
5989 if (!cpu_buffer->nr_pages_to_update)
5990 continue;
5991
5992 list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages, list) {
5993 list_del_init(&bpage->list);
5994 free_buffer_page(bpage);
5995 }
5996 }
5997
5998 return err;
5999}
6000EXPORT_SYMBOL_GPL(ring_buffer_subbuf_order_set);
6001
5819/*6002/*
5820 * We only allocate new buffers, never free them if the CPU goes down.6003 * We only allocate new buffers, never free them if the CPU goes down.
5821 * If we were to free the buffer, then the user would lose any trace that was in6004 * If we were to free the buffer, then the user would lose any trace that was in