History

Merge tag 'trace-v6.8' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace

Pull tracing updates from Steven Rostedt: - Allow kernel trace instance creation to specify what events are created Inside the kernel, a subsystem may create a tracing instance that it can use to send events to user space. This sub-system may not care about the thousands of events that exist in eventfs. Allow the sub-system to specify what sub-systems of events it cares about, and only those events are exposed to this instance. - Allow the ring buffer to be broken up into bigger sub-buffers than just the architecture page size. A new tracefs file called "buffer_subbuf_size_kb" is created. The user can now specify a minimum size the sub-buffer may be in kilobytes. Note, that the implementation currently make the sub-buffer size a power of 2 pages (1, 2, 4, 8, 16, ...) but the user only writes in kilobyte size, and the sub-buffer will be updated to the next size that it will can accommodate it. If the user writes in 10, it will change the size to be 4 pages on x86 (16K), as that is the next available size that can hold 10K pages. - Update the debug output when a corrupt time is detected in the ring buffer. If the ring buffer detects inconsistent timestamps, there's a debug config options that will dump the contents of the meta data of the sub-buffer that is used for debugging. Add some more information to this dump that helps with debugging. - Add more timestamp debugging checks (only triggers when the config is enabled) - Increase the trace_seq iterator to 2 page sizes. - Allow strings written into tracefs_marker to be larger. Up to just under 2 page sizes (based on what trace_seq can hold). - Increase the trace_maker_raw write to be as big as a sub-buffer can hold. - Remove 32 bit time stamp logic, now that the rb_time_cmpxchg() has been removed. - More selftests were added. - Some code clean ups as well. * tag 'trace-v6.8' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace: (29 commits) ring-buffer: Remove stale comment from ring_buffer_size() tracing histograms: Simplify parse_actions() function tracing/selftests: Remove exec permissions from trace_marker.tc test ring-buffer: Use subbuf_order for buffer page masking tracing: Update subbuffer with kilobytes not page order ringbuffer/selftest: Add basic selftest to test changing subbuf order ring-buffer: Add documentation on the buffer_subbuf_order file ring-buffer: Just update the subbuffers when changing their allocation order ring-buffer: Keep the same size when updating the order tracing: Stop the tracing while changing the ring buffer subbuf size tracing: Update snapshot order along with main buffer order ring-buffer: Make sure the spare sub buffer used for reads has same size ring-buffer: Do no swap cpu buffers if order is different ring-buffer: Clear pages on error in ring_buffer_subbuf_order_set() failure ring-buffer: Read and write to ring buffers with custom sub buffer size ring-buffer: Set new size of the ring buffer sub page ring-buffer: Add interface for configuring trace sub buffer size ring-buffer: Page size per ring buffer ring-buffer: Have ring_buffer_print_page_header() be able to access ring_buffer_iter ring-buffer: Check if absolute timestamp goes backwards ...

Linus Torvalds committed 2 years ago

Commit a2ded782 parents 5b890ad 25742ae16 files · +999 −374Browse files

Showing the changes to kernel/trace (8 of 16 files). Show all

Unified · Split

8 files
  1. kernel/trace/ring_buffer.c +458 −275
  2. kernel/trace/ring_buffer_benchmark.c +6 −4
  3. kernel/trace/trace.c +196 −38
  4. kernel/trace/trace.h +2 −0
  5. kernel/trace/trace_boot.c +1 −1
  6. kernel/trace/trace_events.c +90 −17
  7. kernel/trace/trace_events_hist.c +24 −25
  8. kernel/trace/trace_seq.c +0 −3
kernel/trace/ring_buffer.c+458−275
@@ -27,6 +27,7 @@ · 26 unmodified lines
27#include <linux/cpu.h>27#include <linux/cpu.h>
28#include <linux/oom.h>28#include <linux/oom.h>
29 29
30#include <asm/local64.h>
30#include <asm/local.h>31#include <asm/local.h>
31 32
32/*33/*
@@ -317,6 +318,11 @@ struct buffer_data_page { · 284 unmodified lines
317 unsigned char data[] RB_ALIGN_DATA; /* data of buffer page */318 unsigned char data[] RB_ALIGN_DATA; /* data of buffer page */
318};319};
319 320
321struct buffer_data_read_page {
322 unsigned order; /* order of the page */
323 struct buffer_data_page *data; /* actual data, stored in this page */
324};
325
320/*326/*
321 * Note, the buffer_page list must be first. The buffer pages327 * Note, the buffer_page list must be first. The buffer pages
322 * are allocated in cache lines, which means that each buffer328 * are allocated in cache lines, which means that each buffer
@@ -331,6 +337,7 @@ struct buffer_page { · 8 unmodified lines
331 unsigned read; /* index for next read */337 unsigned read; /* index for next read */
332 local_t entries; /* entries on this page */338 local_t entries; /* entries on this page */
333 unsigned long real_end; /* real end of data */339 unsigned long real_end; /* real end of data */
340 unsigned order; /* order of the page */
334 struct buffer_data_page *page; /* Actual data page */341 struct buffer_data_page *page; /* Actual data page */
335};342};
336 343
@@ -361,7 +368,7 @@ static __always_inline unsigned int rb_page_commit(struct buffer_page *bpage) · 24 unmodified lines
361 368
362static void free_buffer_page(struct buffer_page *bpage)369static void free_buffer_page(struct buffer_page *bpage)
363{370{
364 free_page((unsigned long)bpage->page);371 free_pages((unsigned long)bpage->page, bpage->order);
365 kfree(bpage);372 kfree(bpage);
366}373}
367 374
@@ -373,41 +380,6 @@ static inline bool test_time_stamp(u64 delta) · 5 unmodified lines
373 return !!(delta & TS_DELTA_TEST);380 return !!(delta & TS_DELTA_TEST);
374}381}
375 382
376#define BUF_PAGE_SIZE (PAGE_SIZE - BUF_PAGE_HDR_SIZE)
377
378/* Max payload is BUF_PAGE_SIZE - header (8bytes) */
379#define BUF_MAX_DATA_SIZE (BUF_PAGE_SIZE - (sizeof(u32) * 2))
380
381int ring_buffer_print_page_header(struct trace_seq *s)
382{
383 struct buffer_data_page field;
384
385 trace_seq_printf(s, "\tfield: u64 timestamp;\t"
386 "offset:0;\tsize:%u;\tsigned:%u;\n",
387 (unsigned int)sizeof(field.time_stamp),
388 (unsigned int)is_signed_type(u64));
389
390 trace_seq_printf(s, "\tfield: local_t commit;\t"
391 "offset:%u;\tsize:%u;\tsigned:%u;\n",
392 (unsigned int)offsetof(typeof(field), commit),
393 (unsigned int)sizeof(field.commit),
394 (unsigned int)is_signed_type(long));
395
396 trace_seq_printf(s, "\tfield: int overwrite;\t"
397 "offset:%u;\tsize:%u;\tsigned:%u;\n",
398 (unsigned int)offsetof(typeof(field), commit),
399 1,
400 (unsigned int)is_signed_type(long));
401
402 trace_seq_printf(s, "\tfield: char data;\t"
403 "offset:%u;\tsize:%u;\tsigned:%u;\n",
404 (unsigned int)offsetof(typeof(field), data),
405 (unsigned int)BUF_PAGE_SIZE,
406 (unsigned int)is_signed_type(char));
407
408 return !trace_seq_has_overflowed(s);
409}
410
411struct rb_irq_work {383struct rb_irq_work {
412 struct irq_work work;384 struct irq_work work;
413 wait_queue_head_t waiters;385 wait_queue_head_t waiters;
@@ -463,27 +435,9 @@ enum { · 49 unmodified lines
463 RB_CTX_MAX435 RB_CTX_MAX
464};436};
465 437
466#if BITS_PER_LONG == 32
467#define RB_TIME_32
468#endif
469
470/* To test on 64 bit machines */
471//#define RB_TIME_32
472
473#ifdef RB_TIME_32
474
475struct rb_time_struct {
476 local_t cnt;
477 local_t top;
478 local_t bottom;
479 local_t msb;
480};
481#else
482#include <asm/local64.h>
483struct rb_time_struct {438struct rb_time_struct {
484 local64_t time;439 local64_t time;
485};440};
486#endif
487typedef struct rb_time_struct rb_time_t;441typedef struct rb_time_struct rb_time_t;
488 442
489#define MAX_NEST 5443#define MAX_NEST 5
@@ -557,6 +511,10 @@ struct trace_buffer { · 67 unmodified lines
557 511
558 struct rb_irq_work irq_work;512 struct rb_irq_work irq_work;
559 bool time_stamp_abs;513 bool time_stamp_abs;
514
515 unsigned int subbuf_size;
516 unsigned int subbuf_order;
517 unsigned int max_data_size;
560};518};
561 519
562struct ring_buffer_iter {520struct ring_buffer_iter {
@@ -570,150 +528,48 @@ struct ring_buffer_iter { · 7 unmodified lines
570 u64 read_stamp;528 u64 read_stamp;
571 u64 page_stamp;529 u64 page_stamp;
572 struct ring_buffer_event *event;530 struct ring_buffer_event *event;
531 size_t event_size;
573 int missed_events;532 int missed_events;
574};533};
575 534
576#ifdef RB_TIME_32535int ring_buffer_print_page_header(struct trace_buffer *buffer, struct trace_seq *s)
577
578/*
579 * On 32 bit machines, local64_t is very expensive. As the ring
580 * buffer doesn't need all the features of a true 64 bit atomic,
581 * on 32 bit, it uses these functions (64 still uses local64_t).
582 *
583 * For the ring buffer, 64 bit required operations for the time is
584 * the following:
585 *
586 * - Reads may fail if it interrupted a modification of the time stamp.
587 * It will succeed if it did not interrupt another write even if
588 * the read itself is interrupted by a write.
589 * It returns whether it was successful or not.
590 *
591 * - Writes always succeed and will overwrite other writes and writes
592 * that were done by events interrupting the current write.
593 *
594 * - A write followed by a read of the same time stamp will always succeed,
595 * but may not contain the same value.
596 *
597 * - A cmpxchg will fail if it interrupted another write or cmpxchg.
598 * Other than that, it acts like a normal cmpxchg.
599 *
600 * The 60 bit time stamp is broken up by 30 bits in a top and bottom half
601 * (bottom being the least significant 30 bits of the 60 bit time stamp).
602 *
603 * The two most significant bits of each half holds a 2 bit counter (0-3).
604 * Each update will increment this counter by one.
605 * When reading the top and bottom, if the two counter bits match then the
606 * top and bottom together make a valid 60 bit number.
607 */
608#define RB_TIME_SHIFT 30
609#define RB_TIME_VAL_MASK ((1 << RB_TIME_SHIFT) - 1)
610#define RB_TIME_MSB_SHIFT 60
611
612static inline int rb_time_cnt(unsigned long val)
613{536{
614 return (val >> RB_TIME_SHIFT) & 3;537 struct buffer_data_page field;
615}
616
617static inline u64 rb_time_val(unsigned long top, unsigned long bottom)
618{
619 u64 val;
620
621 val = top & RB_TIME_VAL_MASK;
622 val <<= RB_TIME_SHIFT;
623 val |= bottom & RB_TIME_VAL_MASK;
624
625 return val;
626}
627
628static inline bool __rb_time_read(rb_time_t *t, u64 *ret, unsigned long *cnt)
629{
630 unsigned long top, bottom, msb;
631 unsigned long c;
632
633 /*
634 * If the read is interrupted by a write, then the cnt will
635 * be different. Loop until both top and bottom have been read
636 * without interruption.
637 */
638 do {
639 c = local_read(&t->cnt);
640 top = local_read(&t->top);
641 bottom = local_read(&t->bottom);
642 msb = local_read(&t->msb);
643 } while (c != local_read(&t->cnt));
644
645 *cnt = rb_time_cnt(top);
646
647 /* If top, msb or bottom counts don't match, this interrupted a write */
648 if (*cnt != rb_time_cnt(msb) || *cnt != rb_time_cnt(bottom))
649 return false;
650
651 /* The shift to msb will lose its cnt bits */
652 *ret = rb_time_val(top, bottom) | ((u64)msb << RB_TIME_MSB_SHIFT);
653 return true;
654}
655
656static bool rb_time_read(rb_time_t *t, u64 *ret)
657{
658 unsigned long cnt;
659
660 return __rb_time_read(t, ret, &cnt);
661}
662
663static inline unsigned long rb_time_val_cnt(unsigned long val, unsigned long cnt)
664{
665 return (val & RB_TIME_VAL_MASK) | ((cnt & 3) << RB_TIME_SHIFT);
666}
667
668static inline void rb_time_split(u64 val, unsigned long *top, unsigned long *bottom,
669 unsigned long *msb)
670{
671 *top = (unsigned long)((val >> RB_TIME_SHIFT) & RB_TIME_VAL_MASK);
672 *bottom = (unsigned long)(val & RB_TIME_VAL_MASK);
673 *msb = (unsigned long)(val >> RB_TIME_MSB_SHIFT);
674}
675 538
676static inline void rb_time_val_set(local_t *t, unsigned long val, unsigned long cnt)539 trace_seq_printf(s, "\tfield: u64 timestamp;\t"
677{540 "offset:0;\tsize:%u;\tsigned:%u;\n",
678 val = rb_time_val_cnt(val, cnt);541 (unsigned int)sizeof(field.time_stamp),
679 local_set(t, val);542 (unsigned int)is_signed_type(u64));
680}
681 543
682static void rb_time_set(rb_time_t *t, u64 val)544 trace_seq_printf(s, "\tfield: local_t commit;\t"
683{545 "offset:%u;\tsize:%u;\tsigned:%u;\n",
684 unsigned long cnt, top, bottom, msb;546 (unsigned int)offsetof(typeof(field), commit),
547 (unsigned int)sizeof(field.commit),
548 (unsigned int)is_signed_type(long));
685 549
686 rb_time_split(val, &top, &bottom, &msb);550 trace_seq_printf(s, "\tfield: int overwrite;\t"
551 "offset:%u;\tsize:%u;\tsigned:%u;\n",
552 (unsigned int)offsetof(typeof(field), commit),
553 1,
554 (unsigned int)is_signed_type(long));
687 555
688 /* Writes always succeed with a valid number even if it gets interrupted. */556 trace_seq_printf(s, "\tfield: char data;\t"
689 do {557 "offset:%u;\tsize:%u;\tsigned:%u;\n",
690 cnt = local_inc_return(&t->cnt);558 (unsigned int)offsetof(typeof(field), data),
691 rb_time_val_set(&t->top, top, cnt);559 (unsigned int)buffer->subbuf_size,
692 rb_time_val_set(&t->bottom, bottom, cnt);560 (unsigned int)is_signed_type(char));
693 rb_time_val_set(&t->msb, val >> RB_TIME_MSB_SHIFT, cnt);
694 } while (cnt != local_read(&t->cnt));
695}
696 561
697static inline bool562 return !trace_seq_has_overflowed(s);
698rb_time_read_cmpxchg(local_t *l, unsigned long expect, unsigned long set)
699{
700 return local_try_cmpxchg(l, &expect, set);
701}563}
702 564
703#else /* 64 bits */565static inline void rb_time_read(rb_time_t *t, u64 *ret)
704
705/* local64_t always succeeds */
706
707static inline bool rb_time_read(rb_time_t *t, u64 *ret)
708{566{
709 *ret = local64_read(&t->time);567 *ret = local64_read(&t->time);
710 return true;
711}568}
712static void rb_time_set(rb_time_t *t, u64 val)569static void rb_time_set(rb_time_t *t, u64 val)
713{570{
714 local64_set(&t->time, val);571 local64_set(&t->time, val);
715}572}
716#endif
717 573
718/*574/*
719 * Enable this to make sure that the event passed to575 * Enable this to make sure that the event passed to
@@ -820,10 +676,7 @@ u64 ring_buffer_event_time_stamp(struct trace_buffer *buffer, · 100 unmodified lines
820 WARN_ONCE(1, "nest (%d) greater than max", nest);676 WARN_ONCE(1, "nest (%d) greater than max", nest);
821 677
822 fail:678 fail:
823 /* Can only fail on 32 bit */679 rb_time_read(&cpu_buffer->write_stamp, &ts);
824 if (!rb_time_read(&cpu_buffer->write_stamp, &ts))
825 /* Screw it, just read the current time */
826 ts = rb_time_stamp(cpu_buffer->buffer);
827 680
828 return ts;681 return ts;
829}682}
@@ -1619,10 +1472,12 @@ static int __rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, · 789 unmodified lines
1619 1472
1620 list_add(&bpage->list, pages);1473 list_add(&bpage->list, pages);
1621 1474
1622 page = alloc_pages_node(cpu_to_node(cpu_buffer->cpu), mflags, 0);1475 page = alloc_pages_node(cpu_to_node(cpu_buffer->cpu), mflags,
1476 cpu_buffer->buffer->subbuf_order);
1623 if (!page)1477 if (!page)
1624 goto free_pages;1478 goto free_pages;
1625 bpage->page = page_address(page);1479 bpage->page = page_address(page);
1480 bpage->order = cpu_buffer->buffer->subbuf_order;
1626 rb_init_page(bpage->page);1481 rb_init_page(bpage->page);
1627 1482
1628 if (user_thread && fatal_signal_pending(current))1483 if (user_thread && fatal_signal_pending(current))
@@ -1701,7 +1556,8 @@ rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu) · 72 unmodified lines
1701 rb_check_bpage(cpu_buffer, bpage);1556 rb_check_bpage(cpu_buffer, bpage);
1702 1557
1703 cpu_buffer->reader_page = bpage;1558 cpu_buffer->reader_page = bpage;
1704 page = alloc_pages_node(cpu_to_node(cpu), GFP_KERNEL, 0);1559
1560 page = alloc_pages_node(cpu_to_node(cpu), GFP_KERNEL, cpu_buffer->buffer->subbuf_order);
1705 if (!page)1561 if (!page)
1706 goto fail_free_reader;1562 goto fail_free_reader;
1707 bpage->page = page_address(page);1563 bpage->page = page_address(page);
@@ -1784,7 +1640,14 @@ struct trace_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags, · 76 unmodified lines
1784 if (!zalloc_cpumask_var(&buffer->cpumask, GFP_KERNEL))1640 if (!zalloc_cpumask_var(&buffer->cpumask, GFP_KERNEL))
1785 goto fail_free_buffer;1641 goto fail_free_buffer;
1786 1642
1787 nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);1643 /* Default buffer page size - one system page */
1644 buffer->subbuf_order = 0;
1645 buffer->subbuf_size = PAGE_SIZE - BUF_PAGE_HDR_SIZE;
1646
1647 /* Max payload is buffer page size - header (8bytes) */
1648 buffer->max_data_size = buffer->subbuf_size - (sizeof(u32) * 2);
1649
1650 nr_pages = DIV_ROUND_UP(size, buffer->subbuf_size);
1788 buffer->flags = flags;1651 buffer->flags = flags;
1789 buffer->clock = trace_clock_local;1652 buffer->clock = trace_clock_local;
1790 buffer->reader_lock_key = key;1653 buffer->reader_lock_key = key;
@@ -2103,7 +1966,7 @@ static void update_pages_handler(struct work_struct *work) · 312 unmodified lines
2103 * @size: the new size.1966 * @size: the new size.
2104 * @cpu_id: the cpu buffer to resize1967 * @cpu_id: the cpu buffer to resize
2105 *1968 *
2106 * Minimum size is 2 * BUF_PAGE_SIZE.1969 * Minimum size is 2 * buffer->subbuf_size.
2107 *1970 *
2108 * Returns 0 on success and < 0 on failure.1971 * Returns 0 on success and < 0 on failure.
2109 */1972 */
@@ -2125,7 +1988,7 @@ int ring_buffer_resize(struct trace_buffer *buffer, unsigned long size, · 15 unmodified lines
2125 !cpumask_test_cpu(cpu_id, buffer->cpumask))1988 !cpumask_test_cpu(cpu_id, buffer->cpumask))
2126 return 0;1989 return 0;
2127 1990
2128 nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);1991 nr_pages = DIV_ROUND_UP(size, buffer->subbuf_size);
2129 1992
2130 /* we need a minimum of two pages */1993 /* we need a minimum of two pages */
2131 if (nr_pages < 2)1994 if (nr_pages < 2)
@@ -2372,7 +2235,7 @@ rb_iter_head_event(struct ring_buffer_iter *iter) · 240 unmodified lines
2372 */2235 */
2373 barrier();2236 barrier();
2374 2237
2375 if ((iter->head + length) > commit || length > BUF_PAGE_SIZE)2238 if ((iter->head + length) > commit || length > iter->event_size)
2376 /* Writer corrupted the read? */2239 /* Writer corrupted the read? */
2377 goto reset;2240 goto reset;
2378 2241
@@ -2412,11 +2275,13 @@ rb_commit_index(struct ring_buffer_per_cpu *cpu_buffer) · 33 unmodified lines
2412}2275}
2413 2276
2414static __always_inline unsigned2277static __always_inline unsigned
2415rb_event_index(struct ring_buffer_event *event)2278rb_event_index(struct ring_buffer_per_cpu *cpu_buffer, struct ring_buffer_event *event)
2416{2279{
2417 unsigned long addr = (unsigned long)event;2280 unsigned long addr = (unsigned long)event;
2418 2281
2419 return (addr & ~PAGE_MASK) - BUF_PAGE_HDR_SIZE;2282 addr &= (PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1;
2283
2284 return addr - BUF_PAGE_HDR_SIZE;
2420}2285}
2421 2286
2422static void rb_inc_iter(struct ring_buffer_iter *iter)2287static void rb_inc_iter(struct ring_buffer_iter *iter)
@@ -2605,6 +2470,7 @@ static inline void · 182 unmodified lines
2605rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer,2470rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer,
2606 unsigned long tail, struct rb_event_info *info)2471 unsigned long tail, struct rb_event_info *info)
2607{2472{
2473 unsigned long bsize = READ_ONCE(cpu_buffer->buffer->subbuf_size);
2608 struct buffer_page *tail_page = info->tail_page;2474 struct buffer_page *tail_page = info->tail_page;
2609 struct ring_buffer_event *event;2475 struct ring_buffer_event *event;
2610 unsigned long length = info->length;2476 unsigned long length = info->length;
@@ -2613,13 +2479,13 @@ rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer, · 2 unmodified lines
2613 * Only the event that crossed the page boundary2479 * Only the event that crossed the page boundary
2614 * must fill the old tail_page with padding.2480 * must fill the old tail_page with padding.
2615 */2481 */
2616 if (tail >= BUF_PAGE_SIZE) {2482 if (tail >= bsize) {
2617 /*2483 /*
2618 * If the page was filled, then we still need2484 * If the page was filled, then we still need
2619 * to update the real_end. Reset it to zero2485 * to update the real_end. Reset it to zero
2620 * and the reader will ignore it.2486 * and the reader will ignore it.
2621 */2487 */
2622 if (tail == BUF_PAGE_SIZE)2488 if (tail == bsize)
2623 tail_page->real_end = 0;2489 tail_page->real_end = 0;
2624 2490
2625 local_sub(length, &tail_page->write);2491 local_sub(length, &tail_page->write);
@@ -2647,7 +2513,7 @@ rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer, · 21 unmodified lines
2647 * If we are less than the minimum size, we don't need to2513 * If we are less than the minimum size, we don't need to
2648 * worry about it.2514 * worry about it.
2649 */2515 */
2650 if (tail > (BUF_PAGE_SIZE - RB_EVNT_MIN_SIZE)) {2516 if (tail > (bsize - RB_EVNT_MIN_SIZE)) {
2651 /* No room for any events */2517 /* No room for any events */
2652 2518
2653 /* Mark the rest of the page with padding */2519 /* Mark the rest of the page with padding */
@@ -2662,19 +2528,19 @@ rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer, · 8 unmodified lines
2662 }2528 }
2663 2529
2664 /* Put in a discarded event */2530 /* Put in a discarded event */
2665 event->array[0] = (BUF_PAGE_SIZE - tail) - RB_EVNT_HDR_SIZE;2531 event->array[0] = (bsize - tail) - RB_EVNT_HDR_SIZE;
2666 event->type_len = RINGBUF_TYPE_PADDING;2532 event->type_len = RINGBUF_TYPE_PADDING;
2667 /* time delta must be non zero */2533 /* time delta must be non zero */
2668 event->time_delta = 1;2534 event->time_delta = 1;
2669 2535
2670 /* account for padding bytes */2536 /* account for padding bytes */
2671 local_add(BUF_PAGE_SIZE - tail, &cpu_buffer->entries_bytes);2537 local_add(bsize - tail, &cpu_buffer->entries_bytes);
2672 2538
2673 /* Make sure the padding is visible before the tail_page->write update */2539 /* Make sure the padding is visible before the tail_page->write update */
2674 smp_wmb();2540 smp_wmb();
2675 2541
2676 /* Set write to end of buffer */2542 /* Set write to end of buffer */
2677 length = (tail + length) - BUF_PAGE_SIZE;2543 length = (tail + length) - bsize;
2678 local_sub(length, &tail_page->write);2544 local_sub(length, &tail_page->write);
2679}2545}
2680 2546
@@ -2788,7 +2654,8 @@ rb_move_tail(struct ring_buffer_per_cpu *cpu_buffer, · 107 unmodified lines
2788 2654
2789/* Slow path */2655/* Slow path */
2790static struct ring_buffer_event *2656static struct ring_buffer_event *
2791rb_add_time_stamp(struct ring_buffer_event *event, u64 delta, bool abs)2657rb_add_time_stamp(struct ring_buffer_per_cpu *cpu_buffer,
2658 struct ring_buffer_event *event, u64 delta, bool abs)
2792{2659{
2793 if (abs)2660 if (abs)
2794 event->type_len = RINGBUF_TYPE_TIME_STAMP;2661 event->type_len = RINGBUF_TYPE_TIME_STAMP;
@@ -2796,7 +2663,7 @@ rb_add_time_stamp(struct ring_buffer_event *event, u64 delta, bool abs) · 1 unmodified line
2796 event->type_len = RINGBUF_TYPE_TIME_EXTEND;2663 event->type_len = RINGBUF_TYPE_TIME_EXTEND;
2797 2664
2798 /* Not the first event on the page, or not delta? */2665 /* Not the first event on the page, or not delta? */
2799 if (abs || rb_event_index(event)) {2666 if (abs || rb_event_index(cpu_buffer, event)) {
2800 event->time_delta = delta & TS_MASK;2667 event->time_delta = delta & TS_MASK;
2801 event->array[0] = delta >> TS_SHIFT;2668 event->array[0] = delta >> TS_SHIFT;
2802 } else {2669 } else {
@@ -2826,7 +2693,7 @@ rb_check_timestamp(struct ring_buffer_per_cpu *cpu_buffer, · 23 unmodified lines
2826 (unsigned long long)info->ts,2693 (unsigned long long)info->ts,
2827 (unsigned long long)info->before,2694 (unsigned long long)info->before,
2828 (unsigned long long)info->after,2695 (unsigned long long)info->after,
2829 (unsigned long long)(rb_time_read(&cpu_buffer->write_stamp, &write_stamp) ? write_stamp : 0),2696 (unsigned long long)({rb_time_read(&cpu_buffer->write_stamp, &write_stamp); write_stamp;}),
2830 sched_clock_stable() ? "" :2697 sched_clock_stable() ? "" :
2831 "If you just came from a suspend/resume,\n"2698 "If you just came from a suspend/resume,\n"
2832 "please switch to the trace global clock:\n"2699 "please switch to the trace global clock:\n"
@@ -2870,7 +2737,7 @@ static void rb_add_timestamp(struct ring_buffer_per_cpu *cpu_buffer, · 37 unmodified lines
2870 if (!abs)2737 if (!abs)
2871 info->delta = 0;2738 info->delta = 0;
2872 }2739 }
2873 *event = rb_add_time_stamp(*event, info->delta, abs);2740 *event = rb_add_time_stamp(cpu_buffer, *event, info->delta, abs);
2874 *length -= RB_LEN_TIME_EXTEND;2741 *length -= RB_LEN_TIME_EXTEND;
2875 *delta = 0;2742 *delta = 0;
2876}2743}
@@ -2954,10 +2821,10 @@ rb_try_to_discard(struct ring_buffer_per_cpu *cpu_buffer, · 77 unmodified lines
2954 struct buffer_page *bpage;2821 struct buffer_page *bpage;
2955 unsigned long addr;2822 unsigned long addr;
2956 2823
2957 new_index = rb_event_index(event);2824 new_index = rb_event_index(cpu_buffer, event);
2958 old_index = new_index + rb_event_ts_length(event);2825 old_index = new_index + rb_event_ts_length(event);
2959 addr = (unsigned long)event;2826 addr = (unsigned long)event;
2960 addr &= PAGE_MASK;2827 addr &= ~((PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1);
2961 2828
2962 bpage = READ_ONCE(cpu_buffer->tail_page);2829 bpage = READ_ONCE(cpu_buffer->tail_page);
2963 2830
@@ -3344,6 +3211,76 @@ EXPORT_SYMBOL_GPL(ring_buffer_unlock_commit); · 380 unmodified lines
3344#define CHECK_FULL_PAGE 1L3211#define CHECK_FULL_PAGE 1L
3345 3212
3346#ifdef CONFIG_RING_BUFFER_VALIDATE_TIME_DELTAS3213#ifdef CONFIG_RING_BUFFER_VALIDATE_TIME_DELTAS
3214
3215static const char *show_irq_str(int bits)
3216{
3217 const char *type[] = {
3218 ".", // 0
3219 "s", // 1
3220 "h", // 2
3221 "Hs", // 3
3222 "n", // 4
3223 "Ns", // 5
3224 "Nh", // 6
3225 "NHs", // 7
3226 };
3227
3228 return type[bits];
3229}
3230
3231/* Assume this is an trace event */
3232static const char *show_flags(struct ring_buffer_event *event)
3233{
3234 struct trace_entry *entry;
3235 int bits = 0;
3236
3237 if (rb_event_data_length(event) - RB_EVNT_HDR_SIZE < sizeof(*entry))
3238 return "X";
3239
3240 entry = ring_buffer_event_data(event);
3241
3242 if (entry->flags & TRACE_FLAG_SOFTIRQ)
3243 bits |= 1;
3244
3245 if (entry->flags & TRACE_FLAG_HARDIRQ)
3246 bits |= 2;
3247
3248 if (entry->flags & TRACE_FLAG_NMI)
3249 bits |= 4;
3250
3251 return show_irq_str(bits);
3252}
3253
3254static const char *show_irq(struct ring_buffer_event *event)
3255{
3256 struct trace_entry *entry;
3257
3258 if (rb_event_data_length(event) - RB_EVNT_HDR_SIZE < sizeof(*entry))
3259 return "";
3260
3261 entry = ring_buffer_event_data(event);
3262 if (entry->flags & TRACE_FLAG_IRQS_OFF)
3263 return "d";
3264 return "";
3265}
3266
3267static const char *show_interrupt_level(void)
3268{
3269 unsigned long pc = preempt_count();
3270 unsigned char level = 0;
3271
3272 if (pc & SOFTIRQ_OFFSET)
3273 level |= 1;
3274
3275 if (pc & HARDIRQ_MASK)
3276 level |= 2;
3277
3278 if (pc & NMI_MASK)
3279 level |= 4;
3280
3281 return show_irq_str(level);
3282}
3283
3347static void dump_buffer_page(struct buffer_data_page *bpage,3284static void dump_buffer_page(struct buffer_data_page *bpage,
3348 struct rb_event_info *info,3285 struct rb_event_info *info,
3349 unsigned long tail)3286 unsigned long tail)
@@ -3364,34 +3301,57 @@ static void dump_buffer_page(struct buffer_data_page *bpage, · 14 unmodified lines
3364 case RINGBUF_TYPE_TIME_EXTEND:3301 case RINGBUF_TYPE_TIME_EXTEND:
3365 delta = rb_event_time_stamp(event);3302 delta = rb_event_time_stamp(event);
3366 ts += delta;3303 ts += delta;
3367 pr_warn(" [%lld] delta:%lld TIME EXTEND\n", ts, delta);3304 pr_warn(" 0x%x: [%lld] delta:%lld TIME EXTEND\n",
3305 e, ts, delta);
3368 break;3306 break;
3369 3307
3370 case RINGBUF_TYPE_TIME_STAMP:3308 case RINGBUF_TYPE_TIME_STAMP:
3371 delta = rb_event_time_stamp(event);3309 delta = rb_event_time_stamp(event);
3372 ts = rb_fix_abs_ts(delta, ts);3310 ts = rb_fix_abs_ts(delta, ts);
3373 pr_warn(" [%lld] absolute:%lld TIME STAMP\n", ts, delta);3311 pr_warn(" 0x%x: [%lld] absolute:%lld TIME STAMP\n",
3312 e, ts, delta);
3374 break;3313 break;
3375 3314
3376 case RINGBUF_TYPE_PADDING:3315 case RINGBUF_TYPE_PADDING:
3377 ts += event->time_delta;3316 ts += event->time_delta;
3378 pr_warn(" [%lld] delta:%d PADDING\n", ts, event->time_delta);3317 pr_warn(" 0x%x: [%lld] delta:%d PADDING\n",
3318 e, ts, event->time_delta);
3379 break;3319 break;
3380 3320
3381 case RINGBUF_TYPE_DATA:3321 case RINGBUF_TYPE_DATA:
3382 ts += event->time_delta;3322 ts += event->time_delta;
3383 pr_warn(" [%lld] delta:%d\n", ts, event->time_delta);3323 pr_warn(" 0x%x: [%lld] delta:%d %s%s\n",
3324 e, ts, event->time_delta,
3325 show_flags(event), show_irq(event));
3384 break;3326 break;
3385 3327
3386 default:3328 default:
3387 break;3329 break;
3388 }3330 }
3389 }3331 }
3332 pr_warn("expected end:0x%lx last event actually ended at:0x%x\n", tail, e);
3390}3333}
3391 3334
3392static DEFINE_PER_CPU(atomic_t, checking);3335static DEFINE_PER_CPU(atomic_t, checking);
3393static atomic_t ts_dump;3336static atomic_t ts_dump;
3394 3337
3338#define buffer_warn_return(fmt, ...) \
3339 do { \
3340 /* If another report is happening, ignore this one */ \
3341 if (atomic_inc_return(&ts_dump) != 1) { \
3342 atomic_dec(&ts_dump); \
3343 goto out; \
3344 } \
3345 atomic_inc(&cpu_buffer->record_disabled); \
3346 pr_warn(fmt, ##__VA_ARGS__); \
3347 dump_buffer_page(bpage, info, tail); \
3348 atomic_dec(&ts_dump); \
3349 /* There's some cases in boot up that this can happen */ \
3350 if (WARN_ON_ONCE(system_state != SYSTEM_BOOTING)) \
3351 /* Do not re-enable checking */ \
3352 return; \
3353 } while (0)
3354
3395/*3355/*
3396 * Check if the current event time stamp matches the deltas on3356 * Check if the current event time stamp matches the deltas on
3397 * the buffer page.3357 * the buffer page.
@@ -3445,7 +3405,12 @@ static void check_buffer(struct ring_buffer_per_cpu *cpu_buffer, · 47 unmodified lines
3445 3405
3446 case RINGBUF_TYPE_TIME_STAMP:3406 case RINGBUF_TYPE_TIME_STAMP:
3447 delta = rb_event_time_stamp(event);3407 delta = rb_event_time_stamp(event);
3448 ts = rb_fix_abs_ts(delta, ts);3408 delta = rb_fix_abs_ts(delta, ts);
3409 if (delta < ts) {
3410 buffer_warn_return("[CPU: %d]ABSOLUTE TIME WENT BACKWARDS: last ts: %lld absolute ts: %lld\n",
3411 cpu_buffer->cpu, ts, delta);
3412 }
3413 ts = delta;
3449 break;3414 break;
3450 3415
3451 case RINGBUF_TYPE_PADDING:3416 case RINGBUF_TYPE_PADDING:
@@ -3462,23 +3427,11 @@ static void check_buffer(struct ring_buffer_per_cpu *cpu_buffer, · 10 unmodified lines
3462 }3427 }
3463 if ((full && ts > info->ts) ||3428 if ((full && ts > info->ts) ||
3464 (!full && ts + info->delta != info->ts)) {3429 (!full && ts + info->delta != info->ts)) {
3465 /* If another report is happening, ignore this one */3430 buffer_warn_return("[CPU: %d]TIME DOES NOT MATCH expected:%lld actual:%lld delta:%lld before:%lld after:%lld%s context:%s\n",
3466 if (atomic_inc_return(&ts_dump) != 1) {3431 cpu_buffer->cpu,
3467 atomic_dec(&ts_dump);3432 ts + info->delta, info->ts, info->delta,
3468 goto out;3433 info->before, info->after,
3469 }3434 full ? " (full)" : "", show_interrupt_level());
3470 atomic_inc(&cpu_buffer->record_disabled);
3471 /* There's some cases in boot up that this can happen */
3472 WARN_ON_ONCE(system_state != SYSTEM_BOOTING);
3473 pr_warn("[CPU: %d]TIME DOES NOT MATCH expected:%lld actual:%lld delta:%lld before:%lld after:%lld%s\n",
3474 cpu_buffer->cpu,
3475 ts + info->delta, info->ts, info->delta,
3476 info->before, info->after,
3477 full ? " (full)" : "");
3478 dump_buffer_page(bpage, info, tail);
3479 atomic_dec(&ts_dump);
3480 /* Do not re-enable checking */
3481 return;
3482 }3435 }
3483out:3436out:
3484 atomic_dec(this_cpu_ptr(&checking));3437 atomic_dec(this_cpu_ptr(&checking));
@@ -3498,16 +3451,14 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer, · 13 unmodified lines
3498 struct ring_buffer_event *event;3451 struct ring_buffer_event *event;
3499 struct buffer_page *tail_page;3452 struct buffer_page *tail_page;
3500 unsigned long tail, write, w;3453 unsigned long tail, write, w;
3501 bool a_ok;
3502 bool b_ok;
3503 3454
3504 /* Don't let the compiler play games with cpu_buffer->tail_page */3455 /* Don't let the compiler play games with cpu_buffer->tail_page */
3505 tail_page = info->tail_page = READ_ONCE(cpu_buffer->tail_page);3456 tail_page = info->tail_page = READ_ONCE(cpu_buffer->tail_page);
3506 3457
3507 /*A*/ w = local_read(&tail_page->write) & RB_WRITE_MASK;3458 /*A*/ w = local_read(&tail_page->write) & RB_WRITE_MASK;
3508 barrier();3459 barrier();
3509 b_ok = rb_time_read(&cpu_buffer->before_stamp, &info->before);3460 rb_time_read(&cpu_buffer->before_stamp, &info->before);
3510 a_ok = rb_time_read(&cpu_buffer->write_stamp, &info->after);3461 rb_time_read(&cpu_buffer->write_stamp, &info->after);
3511 barrier();3462 barrier();
3512 info->ts = rb_time_stamp(cpu_buffer->buffer);3463 info->ts = rb_time_stamp(cpu_buffer->buffer);
3513 3464
@@ -3522,7 +3473,7 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer, · 8 unmodified lines
3522 if (!w) {3473 if (!w) {
3523 /* Use the sub-buffer timestamp */3474 /* Use the sub-buffer timestamp */
3524 info->delta = 0;3475 info->delta = 0;
3525 } else if (unlikely(!a_ok || !b_ok || info->before != info->after)) {3476 } else if (unlikely(info->before != info->after)) {
3526 info->add_timestamp |= RB_ADD_STAMP_FORCE | RB_ADD_STAMP_EXTEND;3477 info->add_timestamp |= RB_ADD_STAMP_FORCE | RB_ADD_STAMP_EXTEND;
3527 info->length += RB_LEN_TIME_EXTEND;3478 info->length += RB_LEN_TIME_EXTEND;
3528 } else {3479 } else {
@@ -3544,7 +3495,7 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer, · 15 unmodified lines
3544 tail = write - info->length;3495 tail = write - info->length;
3545 3496
3546 /* See if we shot pass the end of this buffer page */3497 /* See if we shot pass the end of this buffer page */
3547 if (unlikely(write > BUF_PAGE_SIZE)) {3498 if (unlikely(write > cpu_buffer->buffer->subbuf_size)) {
3548 check_buffer(cpu_buffer, info, CHECK_FULL_PAGE);3499 check_buffer(cpu_buffer, info, CHECK_FULL_PAGE);
3549 return rb_move_tail(cpu_buffer, tail, info);3500 return rb_move_tail(cpu_buffer, tail, info);
3550 }3501 }
@@ -3571,8 +3522,7 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer, · 20 unmodified lines
3571 /* SLOW PATH - Interrupted between A and C */3522 /* SLOW PATH - Interrupted between A and C */
3572 3523
3573 /* Save the old before_stamp */3524 /* Save the old before_stamp */
3574 a_ok = rb_time_read(&cpu_buffer->before_stamp, &info->before);3525 rb_time_read(&cpu_buffer->before_stamp, &info->before);
3575 RB_WARN_ON(cpu_buffer, !a_ok);
3576 3526
3577 /*3527 /*
3578 * Read a new timestamp and update the before_stamp to make3528 * Read a new timestamp and update the before_stamp to make
@@ -3584,9 +3534,7 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer, · 5 unmodified lines
3584 rb_time_set(&cpu_buffer->before_stamp, ts);3534 rb_time_set(&cpu_buffer->before_stamp, ts);
3585 3535
3586 barrier();3536 barrier();
3587 /*E*/ a_ok = rb_time_read(&cpu_buffer->write_stamp, &info->after);3537 /*E*/ rb_time_read(&cpu_buffer->write_stamp, &info->after);
3588 /* Was interrupted before here, write_stamp must be valid */
3589 RB_WARN_ON(cpu_buffer, !a_ok);
3590 barrier();3538 barrier();
3591 /*F*/ if (write == (local_read(&tail_page->write) & RB_WRITE_MASK) &&3539 /*F*/ if (write == (local_read(&tail_page->write) & RB_WRITE_MASK) &&
3592 info->after == info->before && info->after < ts) {3540 info->after == info->before && info->after < ts) {
@@ -3678,7 +3626,7 @@ rb_reserve_next_event(struct trace_buffer *buffer, · 85 unmodified lines
3678 if (ring_buffer_time_stamp_abs(cpu_buffer->buffer)) {3626 if (ring_buffer_time_stamp_abs(cpu_buffer->buffer)) {
3679 add_ts_default = RB_ADD_STAMP_ABSOLUTE;3627 add_ts_default = RB_ADD_STAMP_ABSOLUTE;
3680 info.length += RB_LEN_TIME_EXTEND;3628 info.length += RB_LEN_TIME_EXTEND;
3681 if (info.length > BUF_MAX_DATA_SIZE)3629 if (info.length > cpu_buffer->buffer->max_data_size)
3682 goto out_fail;3630 goto out_fail;
3683 } else {3631 } else {
3684 add_ts_default = RB_ADD_STAMP_NONE;3632 add_ts_default = RB_ADD_STAMP_NONE;
@@ -3753,7 +3701,7 @@ ring_buffer_lock_reserve(struct trace_buffer *buffer, unsigned long length) · 68 unmodified lines
3753 if (unlikely(atomic_read(&cpu_buffer->record_disabled)))3701 if (unlikely(atomic_read(&cpu_buffer->record_disabled)))
3754 goto out;3702 goto out;
3755 3703
3756 if (unlikely(length > BUF_MAX_DATA_SIZE))3704 if (unlikely(length > buffer->max_data_size))
3757 goto out;3705 goto out;
3758 3706
3759 if (unlikely(trace_recursive_lock(cpu_buffer)))3707 if (unlikely(trace_recursive_lock(cpu_buffer)))
@@ -3787,7 +3735,7 @@ rb_decrement_entry(struct ring_buffer_per_cpu *cpu_buffer, · 27 unmodified lines
3787 struct buffer_page *bpage = cpu_buffer->commit_page;3735 struct buffer_page *bpage = cpu_buffer->commit_page;
3788 struct buffer_page *start;3736 struct buffer_page *start;
3789 3737
3790 addr &= PAGE_MASK;3738 addr &= ~((PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1);
3791 3739
3792 /* Do the likely case first */3740 /* Do the likely case first */
3793 if (likely(bpage->page == (void *)addr)) {3741 if (likely(bpage->page == (void *)addr)) {
@@ -3903,7 +3851,7 @@ int ring_buffer_write(struct trace_buffer *buffer, · 109 unmodified lines
3903 if (atomic_read(&cpu_buffer->record_disabled))3851 if (atomic_read(&cpu_buffer->record_disabled))
3904 goto out;3852 goto out;
3905 3853
3906 if (length > BUF_MAX_DATA_SIZE)3854 if (length > buffer->max_data_size)
3907 goto out;3855 goto out;
3908 3856
3909 if (unlikely(trace_recursive_lock(cpu_buffer)))3857 if (unlikely(trace_recursive_lock(cpu_buffer)))
@@ -4483,6 +4431,7 @@ static struct buffer_page * · 573 unmodified lines
4483rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)4431rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
4484{4432{
4485 struct buffer_page *reader = NULL;4433 struct buffer_page *reader = NULL;
4434 unsigned long bsize = READ_ONCE(cpu_buffer->buffer->subbuf_size);
4486 unsigned long overwrite;4435 unsigned long overwrite;
4487 unsigned long flags;4436 unsigned long flags;
4488 int nr_loops = 0;4437 int nr_loops = 0;
@@ -4618,7 +4567,7 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) · 129 unmodified lines
4618#define USECS_WAIT 10000004567#define USECS_WAIT 1000000
4619 for (nr_loops = 0; nr_loops < USECS_WAIT; nr_loops++) {4568 for (nr_loops = 0; nr_loops < USECS_WAIT; nr_loops++) {
4620 /* If the write is past the end of page, a writer is still updating it */4569 /* If the write is past the end of page, a writer is still updating it */
4621 if (likely(!reader || rb_page_write(reader) <= BUF_PAGE_SIZE))4570 if (likely(!reader || rb_page_write(reader) <= bsize))
4622 break;4571 break;
4623 4572
4624 udelay(1);4573 udelay(1);
@@ -5062,7 +5011,8 @@ ring_buffer_read_prepare(struct trace_buffer *buffer, int cpu, gfp_t flags) · 437 unmodified lines
5062 return NULL;5011 return NULL;
5063 5012
5064 /* Holds the entire event: data and meta data */5013 /* Holds the entire event: data and meta data */
5065 iter->event = kmalloc(BUF_PAGE_SIZE, flags);5014 iter->event_size = buffer->subbuf_size;
5015 iter->event = kmalloc(iter->event_size, flags);
5066 if (!iter->event) {5016 if (!iter->event) {
5067 kfree(iter);5017 kfree(iter);
5068 return NULL;5018 return NULL;
@@ -5178,19 +5128,28 @@ EXPORT_SYMBOL_GPL(ring_buffer_iter_advance); · 109 unmodified lines
5178 */5128 */
5179unsigned long ring_buffer_size(struct trace_buffer *buffer, int cpu)5129unsigned long ring_buffer_size(struct trace_buffer *buffer, int cpu)
5180{5130{
5181 /*
5182 * Earlier, this method returned
5183 * BUF_PAGE_SIZE * buffer->nr_pages
5184 * Since the nr_pages field is now removed, we have converted this to
5185 * return the per cpu buffer value.
5186 */
5187 if (!cpumask_test_cpu(cpu, buffer->cpumask))5131 if (!cpumask_test_cpu(cpu, buffer->cpumask))
5188 return 0;5132 return 0;
5189 5133
5190 return BUF_PAGE_SIZE * buffer->buffers[cpu]->nr_pages;5134 return buffer->subbuf_size * buffer->buffers[cpu]->nr_pages;
5191}5135}
5192EXPORT_SYMBOL_GPL(ring_buffer_size);5136EXPORT_SYMBOL_GPL(ring_buffer_size);
5193 5137
5138/**
5139 * ring_buffer_max_event_size - return the max data size of an event
5140 * @buffer: The ring buffer.
5141 *
5142 * Returns the maximum size an event can be.
5143 */
5144unsigned long ring_buffer_max_event_size(struct trace_buffer *buffer)
5145{
5146 /* If abs timestamp is requested, events have a timestamp too */
5147 if (ring_buffer_time_stamp_abs(buffer))
5148 return buffer->max_data_size - RB_LEN_TIME_EXTEND;
5149 return buffer->max_data_size;
5150}
5151EXPORT_SYMBOL_GPL(ring_buffer_max_event_size);
5152
5194static void rb_clear_buffer_page(struct buffer_page *page)5153static void rb_clear_buffer_page(struct buffer_page *page)
5195{5154{
5196 local_set(&page->write, 0);5155 local_set(&page->write, 0);
@@ -5461,6 +5420,9 @@ int ring_buffer_swap_cpu(struct trace_buffer *buffer_a, · 264 unmodified lines
5461 if (cpu_buffer_a->nr_pages != cpu_buffer_b->nr_pages)5420 if (cpu_buffer_a->nr_pages != cpu_buffer_b->nr_pages)
5462 goto out;5421 goto out;
5463 5422
5423 if (buffer_a->subbuf_order != buffer_b->subbuf_order)
5424 goto out;
5425
5464 ret = -EAGAIN;5426 ret = -EAGAIN;
5465 5427
5466 if (atomic_read(&buffer_a->record_disabled))5428 if (atomic_read(&buffer_a->record_disabled))
@@ -5532,40 +5494,48 @@ EXPORT_SYMBOL_GPL(ring_buffer_swap_cpu); · 65 unmodified lines
5532 * Returns:5494 * Returns:
5533 * The page allocated, or ERR_PTR5495 * The page allocated, or ERR_PTR
5534 */5496 */
5535void *ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu)5497struct buffer_data_read_page *
5498ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu)
5536{5499{
5537 struct ring_buffer_per_cpu *cpu_buffer;5500 struct ring_buffer_per_cpu *cpu_buffer;
5538 struct buffer_data_page *bpage = NULL;5501 struct buffer_data_read_page *bpage = NULL;
5539 unsigned long flags;5502 unsigned long flags;
5540 struct page *page;5503 struct page *page;
5541 5504
5542 if (!cpumask_test_cpu(cpu, buffer->cpumask))5505 if (!cpumask_test_cpu(cpu, buffer->cpumask))
5543 return ERR_PTR(-ENODEV);5506 return ERR_PTR(-ENODEV);
5544 5507
5508 bpage = kzalloc(sizeof(*bpage), GFP_KERNEL);
5509 if (!bpage)
5510 return ERR_PTR(-ENOMEM);
5511
5512 bpage->order = buffer->subbuf_order;
5545 cpu_buffer = buffer->buffers[cpu];5513 cpu_buffer = buffer->buffers[cpu];
5546 local_irq_save(flags);5514 local_irq_save(flags);
5547 arch_spin_lock(&cpu_buffer->lock);5515 arch_spin_lock(&cpu_buffer->lock);
5548 5516
5549 if (cpu_buffer->free_page) {5517 if (cpu_buffer->free_page) {
5550 bpage = cpu_buffer->free_page;5518 bpage->data = cpu_buffer->free_page;
5551 cpu_buffer->free_page = NULL;5519 cpu_buffer->free_page = NULL;
5552 }5520 }
5553 5521
5554 arch_spin_unlock(&cpu_buffer->lock);5522 arch_spin_unlock(&cpu_buffer->lock);
5555 local_irq_restore(flags);5523 local_irq_restore(flags);
5556 5524
5557 if (bpage)5525 if (bpage->data)
5558 goto out;5526 goto out;
5559 5527
5560 page = alloc_pages_node(cpu_to_node(cpu),5528 page = alloc_pages_node(cpu_to_node(cpu), GFP_KERNEL | __GFP_NORETRY,
5561 GFP_KERNEL | __GFP_NORETRY, 0);5529 cpu_buffer->buffer->subbuf_order);
5562 if (!page)5530 if (!page) {
5531 kfree(bpage);
5563 return ERR_PTR(-ENOMEM);5532 return ERR_PTR(-ENOMEM);
5533 }
5564 5534
5565 bpage = page_address(page);5535 bpage->data = page_address(page);
5566 5536
5567 out:5537 out:
5568 rb_init_page(bpage);5538 rb_init_page(bpage->data);
5569 5539
5570 return bpage;5540 return bpage;
5571}5541}
@@ -5575,14 +5545,15 @@ EXPORT_SYMBOL_GPL(ring_buffer_alloc_read_page); · 3 unmodified lines
5575 * ring_buffer_free_read_page - free an allocated read page5545 * ring_buffer_free_read_page - free an allocated read page
5576 * @buffer: the buffer the page was allocate for5546 * @buffer: the buffer the page was allocate for
5577 * @cpu: the cpu buffer the page came from5547 * @cpu: the cpu buffer the page came from
5578 * @data: the page to free5548 * @data_page: the page to free
5579 *5549 *
5580 * Free a page allocated from ring_buffer_alloc_read_page.5550 * Free a page allocated from ring_buffer_alloc_read_page.
5581 */5551 */
5582void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, void *data)5552void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu,
5553 struct buffer_data_read_page *data_page)
5583{5554{
5584 struct ring_buffer_per_cpu *cpu_buffer;5555 struct ring_buffer_per_cpu *cpu_buffer;
5585 struct buffer_data_page *bpage = data;5556 struct buffer_data_page *bpage = data_page->data;
5586 struct page *page = virt_to_page(bpage);5557 struct page *page = virt_to_page(bpage);
5587 unsigned long flags;5558 unsigned long flags;
5588 5559
@@ -5591,8 +5562,12 @@ void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, void *data · 2 unmodified lines
5591 5562
5592 cpu_buffer = buffer->buffers[cpu];5563 cpu_buffer = buffer->buffers[cpu];
5593 5564
5594 /* If the page is still in use someplace else, we can't reuse it */5565 /*
5595 if (page_ref_count(page) > 1)5566 * If the page is still in use someplace else, or order of the page
5567 * is different from the subbuffer order of the buffer -
5568 * we can't reuse it
5569 */
5570 if (page_ref_count(page) > 1 || data_page->order != buffer->subbuf_order)
5596 goto out;5571 goto out;
5597 5572
5598 local_irq_save(flags);5573 local_irq_save(flags);
@@ -5607,7 +5582,8 @@ void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, void *data · 8 unmodified lines
5607 local_irq_restore(flags);5582 local_irq_restore(flags);
5608 5583
5609 out:5584 out:
5610 free_page((unsigned long)bpage);5585 free_pages((unsigned long)bpage, data_page->order);
5586 kfree(data_page);
5611}5587}
5612EXPORT_SYMBOL_GPL(ring_buffer_free_read_page);5588EXPORT_SYMBOL_GPL(ring_buffer_free_read_page);
5613 5589
@@ -5628,9 +5604,10 @@ EXPORT_SYMBOL_GPL(ring_buffer_free_read_page); · 14 unmodified lines
5628 * rpage = ring_buffer_alloc_read_page(buffer, cpu);5604 * rpage = ring_buffer_alloc_read_page(buffer, cpu);
5629 * if (IS_ERR(rpage))5605 * if (IS_ERR(rpage))
5630 * return PTR_ERR(rpage);5606 * return PTR_ERR(rpage);
5631 * ret = ring_buffer_read_page(buffer, &rpage, len, cpu, 0);5607 * ret = ring_buffer_read_page(buffer, rpage, len, cpu, 0);
5632 * if (ret >= 0)5608 * if (ret >= 0)
5633 * process_page(rpage, ret);5609 * process_page(ring_buffer_read_page_data(rpage), ret);
5610 * ring_buffer_free_read_page(buffer, cpu, rpage);
5634 *5611 *
5635 * When @full is set, the function will not return true unless5612 * When @full is set, the function will not return true unless
5636 * the writer is off the reader page.5613 * the writer is off the reader page.
@@ -5645,7 +5622,8 @@ EXPORT_SYMBOL_GPL(ring_buffer_free_read_page); · 8 unmodified lines
5645 * <0 if no data has been transferred.5622 * <0 if no data has been transferred.
5646 */5623 */
5647int ring_buffer_read_page(struct trace_buffer *buffer,5624int ring_buffer_read_page(struct trace_buffer *buffer,
5648 void **data_page, size_t len, int cpu, int full)5625 struct buffer_data_read_page *data_page,
5626 size_t len, int cpu, int full)
5649{5627{
5650 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];5628 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
5651 struct ring_buffer_event *event;5629 struct ring_buffer_event *event;
@@ -5670,10 +5648,12 @@ int ring_buffer_read_page(struct trace_buffer *buffer, · 18 unmodified lines
5670 5648
5671 len -= BUF_PAGE_HDR_SIZE;5649 len -= BUF_PAGE_HDR_SIZE;
5672 5650
5673 if (!data_page)5651 if (!data_page || !data_page->data)
5652 goto out;
5653 if (data_page->order != buffer->subbuf_order)
5674 goto out;5654 goto out;
5675 5655
5676 bpage = *data_page;5656 bpage = data_page->data;
5677 if (!bpage)5657 if (!bpage)
5678 goto out;5658 goto out;
5679 5659
@@ -5767,11 +5747,11 @@ int ring_buffer_read_page(struct trace_buffer *buffer, · 87 unmodified lines
5767 /* swap the pages */5747 /* swap the pages */
5768 rb_init_page(bpage);5748 rb_init_page(bpage);
5769 bpage = reader->page;5749 bpage = reader->page;
5770 reader->page = *data_page;5750 reader->page = data_page->data;
5771 local_set(&reader->write, 0);5751 local_set(&reader->write, 0);
5772 local_set(&reader->entries, 0);5752 local_set(&reader->entries, 0);
5773 reader->read = 0;5753 reader->read = 0;
5774 *data_page = bpage;5754 data_page->data = bpage;
5775 5755
5776 /*5756 /*
5777 * Use the real_end for the data size,5757 * Use the real_end for the data size,
@@ -5793,7 +5773,7 @@ int ring_buffer_read_page(struct trace_buffer *buffer, · 15 unmodified lines
5793 /* If there is room at the end of the page to save the5773 /* If there is room at the end of the page to save the
5794 * missed events, then record it there.5774 * missed events, then record it there.
5795 */5775 */
5796 if (BUF_PAGE_SIZE - commit >= sizeof(missed_events)) {5776 if (buffer->subbuf_size - commit >= sizeof(missed_events)) {
5797 memcpy(&bpage->data[commit], &missed_events,5777 memcpy(&bpage->data[commit], &missed_events,
5798 sizeof(missed_events));5778 sizeof(missed_events));
5799 local_add(RB_MISSED_STORED, &bpage->commit);5779 local_add(RB_MISSED_STORED, &bpage->commit);
@@ -5805,8 +5785,8 @@ int ring_buffer_read_page(struct trace_buffer *buffer, · 5 unmodified lines
5805 /*5785 /*
5806 * This page may be off to user land. Zero it out here.5786 * This page may be off to user land. Zero it out here.
5807 */5787 */
5808 if (commit < BUF_PAGE_SIZE)5788 if (commit < buffer->subbuf_size)
5809 memset(&bpage->data[commit], 0, BUF_PAGE_SIZE - commit);5789 memset(&bpage->data[commit], 0, buffer->subbuf_size - commit);
5810 5790
5811 out_unlock:5791 out_unlock:
5812 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);5792 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
@@ -5816,6 +5796,209 @@ int ring_buffer_read_page(struct trace_buffer *buffer, · 3 unmodified lines
5816}5796}
5817EXPORT_SYMBOL_GPL(ring_buffer_read_page);5797EXPORT_SYMBOL_GPL(ring_buffer_read_page);
5818 5798
5799/**
5800 * ring_buffer_read_page_data - get pointer to the data in the page.
5801 * @page: the page to get the data from
5802 *
5803 * Returns pointer to the actual data in this page.
5804 */
5805void *ring_buffer_read_page_data(struct buffer_data_read_page *page)
5806{
5807 return page->data;
5808}
5809EXPORT_SYMBOL_GPL(ring_buffer_read_page_data);
5810
5811/**
5812 * ring_buffer_subbuf_size_get - get size of the sub buffer.
5813 * @buffer: the buffer to get the sub buffer size from
5814 *
5815 * Returns size of the sub buffer, in bytes.
5816 */
5817int ring_buffer_subbuf_size_get(struct trace_buffer *buffer)
5818{
5819 return buffer->subbuf_size + BUF_PAGE_HDR_SIZE;
5820}
5821EXPORT_SYMBOL_GPL(ring_buffer_subbuf_size_get);
5822
5823/**
5824 * ring_buffer_subbuf_order_get - get order of system sub pages in one buffer page.
5825 * @buffer: The ring_buffer to get the system sub page order from
5826 *
5827 * By default, one ring buffer sub page equals to one system page. This parameter
5828 * is configurable, per ring buffer. The size of the ring buffer sub page can be
5829 * extended, but must be an order of system page size.
5830 *
5831 * Returns the order of buffer sub page size, in system pages:
5832 * 0 means the sub buffer size is 1 system page and so forth.
5833 * In case of an error < 0 is returned.
5834 */
5835int ring_buffer_subbuf_order_get(struct trace_buffer *buffer)
5836{
5837 if (!buffer)
5838 return -EINVAL;
5839
5840 return buffer->subbuf_order;
5841}
5842EXPORT_SYMBOL_GPL(ring_buffer_subbuf_order_get);
5843
5844/**
5845 * ring_buffer_subbuf_order_set - set the size of ring buffer sub page.
5846 * @buffer: The ring_buffer to set the new page size.
5847 * @order: Order of the system pages in one sub buffer page
5848 *
5849 * By default, one ring buffer pages equals to one system page. This API can be
5850 * used to set new size of the ring buffer page. The size must be order of
5851 * system page size, that's why the input parameter @order is the order of
5852 * system pages that are allocated for one ring buffer page:
5853 * 0 - 1 system page
5854 * 1 - 2 system pages
5855 * 3 - 4 system pages
5856 * ...
5857 *
5858 * Returns 0 on success or < 0 in case of an error.
5859 */
5860int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order)
5861{
5862 struct ring_buffer_per_cpu *cpu_buffer;
5863 struct buffer_page *bpage, *tmp;
5864 int old_order, old_size;
5865 int nr_pages;
5866 int psize;
5867 int err;
5868 int cpu;
5869
5870 if (!buffer || order < 0)
5871 return -EINVAL;
5872
5873 if (buffer->subbuf_order == order)
5874 return 0;
5875
5876 psize = (1 << order) * PAGE_SIZE;
5877 if (psize <= BUF_PAGE_HDR_SIZE)
5878 return -EINVAL;
5879
5880 old_order = buffer->subbuf_order;
5881 old_size = buffer->subbuf_size;
5882
5883 /* prevent another thread from changing buffer sizes */
5884 mutex_lock(&buffer->mutex);
5885 atomic_inc(&buffer->record_disabled);
5886
5887 /* Make sure all commits have finished */
5888 synchronize_rcu();
5889
5890 buffer->subbuf_order = order;
5891 buffer->subbuf_size = psize - BUF_PAGE_HDR_SIZE;
5892
5893 /* Make sure all new buffers are allocated, before deleting the old ones */
5894 for_each_buffer_cpu(buffer, cpu) {
5895
5896 if (!cpumask_test_cpu(cpu, buffer->cpumask))
5897 continue;
5898
5899 cpu_buffer = buffer->buffers[cpu];
5900
5901 /* Update the number of pages to match the new size */
5902 nr_pages = old_size * buffer->buffers[cpu]->nr_pages;
5903 nr_pages = DIV_ROUND_UP(nr_pages, buffer->subbuf_size);
5904
5905 /* we need a minimum of two pages */
5906 if (nr_pages < 2)
5907 nr_pages = 2;
5908
5909 cpu_buffer->nr_pages_to_update = nr_pages;
5910
5911 /* Include the reader page */
5912 nr_pages++;
5913
5914 /* Allocate the new size buffer */
5915 INIT_LIST_HEAD(&cpu_buffer->new_pages);
5916 if (__rb_allocate_pages(cpu_buffer, nr_pages,
5917 &cpu_buffer->new_pages)) {
5918 /* not enough memory for new pages */
5919 err = -ENOMEM;
5920 goto error;
5921 }
5922 }
5923
5924 for_each_buffer_cpu(buffer, cpu) {
5925
5926 if (!cpumask_test_cpu(cpu, buffer->cpumask))
5927 continue;
5928
5929 cpu_buffer = buffer->buffers[cpu];
5930
5931 /* Clear the head bit to make the link list normal to read */
5932 rb_head_page_deactivate(cpu_buffer);
5933
5934 /* Now walk the list and free all the old sub buffers */
5935 list_for_each_entry_safe(bpage, tmp, cpu_buffer->pages, list) {
5936 list_del_init(&bpage->list);
5937 free_buffer_page(bpage);
5938 }
5939 /* The above loop stopped an the last page needing to be freed */
5940 bpage = list_entry(cpu_buffer->pages, struct buffer_page, list);
5941 free_buffer_page(bpage);
5942
5943 /* Free the current reader page */
5944 free_buffer_page(cpu_buffer->reader_page);
5945
5946 /* One page was allocated for the reader page */
5947 cpu_buffer->reader_page = list_entry(cpu_buffer->new_pages.next,
5948 struct buffer_page, list);
5949 list_del_init(&cpu_buffer->reader_page->list);
5950
5951 /* The cpu_buffer pages are a link list with no head */
5952 cpu_buffer->pages = cpu_buffer->new_pages.next;
5953 cpu_buffer->new_pages.next->prev = cpu_buffer->new_pages.prev;
5954 cpu_buffer->new_pages.prev->next = cpu_buffer->new_pages.next;
5955
5956 /* Clear the new_pages list */
5957 INIT_LIST_HEAD(&cpu_buffer->new_pages);
5958
5959 cpu_buffer->head_page
5960 = list_entry(cpu_buffer->pages, struct buffer_page, list);
5961 cpu_buffer->tail_page = cpu_buffer->commit_page = cpu_buffer->head_page;
5962
5963 cpu_buffer->nr_pages = cpu_buffer->nr_pages_to_update;
5964 cpu_buffer->nr_pages_to_update = 0;
5965
5966 free_pages((unsigned long)cpu_buffer->free_page, old_order);
5967 cpu_buffer->free_page = NULL;
5968
5969 rb_head_page_activate(cpu_buffer);
5970
5971 rb_check_pages(cpu_buffer);
5972 }
5973
5974 atomic_dec(&buffer->record_disabled);
5975 mutex_unlock(&buffer->mutex);
5976
5977 return 0;
5978
5979error:
5980 buffer->subbuf_order = old_order;
5981 buffer->subbuf_size = old_size;
5982
5983 atomic_dec(&buffer->record_disabled);
5984 mutex_unlock(&buffer->mutex);
5985
5986 for_each_buffer_cpu(buffer, cpu) {
5987 cpu_buffer = buffer->buffers[cpu];
5988
5989 if (!cpu_buffer->nr_pages_to_update)
5990 continue;
5991
5992 list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages, list) {
5993 list_del_init(&bpage->list);
5994 free_buffer_page(bpage);
5995 }
5996 }
5997
5998 return err;
5999}
6000EXPORT_SYMBOL_GPL(ring_buffer_subbuf_order_set);
6001
5819/*6002/*
5820 * We only allocate new buffers, never free them if the CPU goes down.6003 * We only allocate new buffers, never free them if the CPU goes down.
5821 * If we were to free the buffer, then the user would lose any trace that was in6004 * If we were to free the buffer, then the user would lose any trace that was in
kernel/trace/ring_buffer_benchmark.c+6−4
@@ -104,10 +104,11 @@ static enum event_status read_event(int cpu) · 103 unmodified lines
104 104
105static enum event_status read_page(int cpu)105static enum event_status read_page(int cpu)
106{106{
107 struct buffer_data_read_page *bpage;
107 struct ring_buffer_event *event;108 struct ring_buffer_event *event;
108 struct rb_page *rpage;109 struct rb_page *rpage;
109 unsigned long commit;110 unsigned long commit;
110 void *bpage;111 int page_size;
111 int *entry;112 int *entry;
112 int ret;113 int ret;
113 int inc;114 int inc;
@@ -117,14 +118,15 @@ static enum event_status read_page(int cpu) · 3 unmodified lines
117 if (IS_ERR(bpage))118 if (IS_ERR(bpage))
118 return EVENT_DROPPED;119 return EVENT_DROPPED;
119 120
120 ret = ring_buffer_read_page(buffer, &bpage, PAGE_SIZE, cpu, 1);121 page_size = ring_buffer_subbuf_size_get(buffer);
122 ret = ring_buffer_read_page(buffer, bpage, page_size, cpu, 1);
121 if (ret >= 0) {123 if (ret >= 0) {
122 rpage = bpage;124 rpage = ring_buffer_read_page_data(bpage);
123 /* The commit may have missed event flags set, clear them */125 /* The commit may have missed event flags set, clear them */
124 commit = local_read(&rpage->commit) & 0xfffff;126 commit = local_read(&rpage->commit) & 0xfffff;
125 for (i = 0; i < commit && !test_error ; i += inc) {127 for (i = 0; i < commit && !test_error ; i += inc) {
126 128
127 if (i >= (PAGE_SIZE - offsetof(struct rb_page, data))) {129 if (i >= (page_size - offsetof(struct rb_page, data))) {
128 TEST_ERROR();130 TEST_ERROR();
129 break;131 break;
130 }132 }
kernel/trace/trace.c+196−38
@@ -1263,10 +1263,17 @@ static void set_buffer_entries(struct array_buffer *buf, unsigned long val); · 1262 unmodified lines
1263 1263
1264int tracing_alloc_snapshot_instance(struct trace_array *tr)1264int tracing_alloc_snapshot_instance(struct trace_array *tr)
1265{1265{
1266 int order;
1266 int ret;1267 int ret;
1267 1268
1268 if (!tr->allocated_snapshot) {1269 if (!tr->allocated_snapshot) {
1269 1270
1271 /* Make the snapshot buffer have the same order as main buffer */
1272 order = ring_buffer_subbuf_order_get(tr->array_buffer.buffer);
1273 ret = ring_buffer_subbuf_order_set(tr->max_buffer.buffer, order);
1274 if (ret < 0)
1275 return ret;
1276
1270 /* allocate spare buffer */1277 /* allocate spare buffer */
1271 ret = resize_buffer_duplicate_size(&tr->max_buffer,1278 ret = resize_buffer_duplicate_size(&tr->max_buffer,
1272 &tr->array_buffer, RING_BUFFER_ALL_CPUS);1279 &tr->array_buffer, RING_BUFFER_ALL_CPUS);
@@ -1286,6 +1293,7 @@ static void free_snapshot(struct trace_array *tr) · 13 unmodified lines
1286 * The max_tr ring buffer has some state (e.g. ring->clock) and1293 * The max_tr ring buffer has some state (e.g. ring->clock) and
1287 * we want preserve it.1294 * we want preserve it.
1288 */1295 */
1296 ring_buffer_subbuf_order_set(tr->max_buffer.buffer, 0);
1289 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);1297 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
1290 set_buffer_entries(&tr->max_buffer, 1);1298 set_buffer_entries(&tr->max_buffer, 1);
1291 tracing_reset_online_cpus(&tr->max_buffer);1299 tracing_reset_online_cpus(&tr->max_buffer);
@@ -3767,7 +3775,7 @@ static bool trace_safe_str(struct trace_iterator *iter, const char *str, · 2475 unmodified lines
3767 3775
3768 /* OK if part of the temp seq buffer */3776 /* OK if part of the temp seq buffer */
3769 if ((addr >= (unsigned long)iter->tmp_seq.buffer) &&3777 if ((addr >= (unsigned long)iter->tmp_seq.buffer) &&
3770 (addr < (unsigned long)iter->tmp_seq.buffer + PAGE_SIZE))3778 (addr < (unsigned long)iter->tmp_seq.buffer + TRACE_SEQ_BUFFER_SIZE))
3771 return true;3779 return true;
3772 3780
3773 /* Core rodata can not be freed */3781 /* Core rodata can not be freed */
@@ -5032,7 +5040,7 @@ static int tracing_release(struct inode *inode, struct file *file) · 1258 unmodified lines
5032 return 0;5040 return 0;
5033}5041}
5034 5042
5035static int tracing_release_generic_tr(struct inode *inode, struct file *file)5043int tracing_release_generic_tr(struct inode *inode, struct file *file)
5036{5044{
5037 struct trace_array *tr = inode->i_private;5045 struct trace_array *tr = inode->i_private;
5038 5046
@@ -6946,8 +6954,8 @@ waitagain: · 1907 unmodified lines
6946 goto out;6954 goto out;
6947 }6955 }
6948 6956
6949 if (cnt >= PAGE_SIZE)6957 if (cnt >= TRACE_SEQ_BUFFER_SIZE)
6950 cnt = PAGE_SIZE - 1;6958 cnt = TRACE_SEQ_BUFFER_SIZE - 1;
6951 6959
6952 /* reset all but tr, trace, and overruns */6960 /* reset all but tr, trace, and overruns */
6953 trace_iterator_reset(iter);6961 trace_iterator_reset(iter);
@@ -7292,8 +7300,9 @@ tracing_mark_write(struct file *filp, const char __user *ubuf, · 338 unmodified lines
7292 enum event_trigger_type tt = ETT_NONE;7300 enum event_trigger_type tt = ETT_NONE;
7293 struct trace_buffer *buffer;7301 struct trace_buffer *buffer;
7294 struct print_entry *entry;7302 struct print_entry *entry;
7303 int meta_size;
7295 ssize_t written;7304 ssize_t written;
7296 int size;7305 size_t size;
7297 int len;7306 int len;
7298 7307
7299/* Used in tracing_mark_raw_write() as well */7308/* Used in tracing_mark_raw_write() as well */
@@ -7306,23 +7315,44 @@ tracing_mark_write(struct file *filp, const char __user *ubuf, · 6 unmodified lines
7306 if (!(tr->trace_flags & TRACE_ITER_MARKERS))7315 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
7307 return -EINVAL;7316 return -EINVAL;
7308 7317
7309 if (cnt > TRACE_BUF_SIZE)7318 if ((ssize_t)cnt < 0)
7310 cnt = TRACE_BUF_SIZE;7319 return -EINVAL;
7311
7312 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
7313 7320
7314 size = sizeof(*entry) + cnt + 2; /* add '\0' and possible '\n' */7321 meta_size = sizeof(*entry) + 2; /* add '\0' and possible '\n' */
7322 again:
7323 size = cnt + meta_size;
7315 7324
7316 /* If less than "<faulted>", then make sure we can still add that */7325 /* If less than "<faulted>", then make sure we can still add that */
7317 if (cnt < FAULTED_SIZE)7326 if (cnt < FAULTED_SIZE)
7318 size += FAULTED_SIZE - cnt;7327 size += FAULTED_SIZE - cnt;
7319 7328
7329 if (size > TRACE_SEQ_BUFFER_SIZE) {
7330 cnt -= size - TRACE_SEQ_BUFFER_SIZE;
7331 goto again;
7332 }
7333
7320 buffer = tr->array_buffer.buffer;7334 buffer = tr->array_buffer.buffer;
7321 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,7335 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
7322 tracing_gen_ctx());7336 tracing_gen_ctx());
7323 if (unlikely(!event))7337 if (unlikely(!event)) {
7338 /*
7339 * If the size was greater than what was allowed, then
7340 * make it smaller and try again.
7341 */
7342 if (size > ring_buffer_max_event_size(buffer)) {
7343 /* cnt < FAULTED size should never be bigger than max */
7344 if (WARN_ON_ONCE(cnt < FAULTED_SIZE))
7345 return -EBADF;
7346 cnt = ring_buffer_max_event_size(buffer) - meta_size;
7347 /* The above should only happen once */
7348 if (WARN_ON_ONCE(cnt + meta_size == size))
7349 return -EBADF;
7350 goto again;
7351 }
7352
7324 /* Ring buffer disabled, return as if not open for write */7353 /* Ring buffer disabled, return as if not open for write */
7325 return -EBADF;7354 return -EBADF;
7355 }
7326 7356
7327 entry = ring_buffer_event_data(event);7357 entry = ring_buffer_event_data(event);
7328 entry->ip = _THIS_IP_;7358 entry->ip = _THIS_IP_;
@@ -7357,9 +7387,6 @@ tracing_mark_write(struct file *filp, const char __user *ubuf, · 28 unmodified lines
7357 return written;7387 return written;
7358}7388}
7359 7389
7360/* Limit it for now to 3K (including tag) */
7361#define RAW_DATA_MAX_SIZE (1024*3)
7362
7363static ssize_t7390static ssize_t
7364tracing_mark_raw_write(struct file *filp, const char __user *ubuf,7391tracing_mark_raw_write(struct file *filp, const char __user *ubuf,
7365 size_t cnt, loff_t *fpos)7392 size_t cnt, loff_t *fpos)
@@ -7381,19 +7408,18 @@ tracing_mark_raw_write(struct file *filp, const char __user *ubuf, · 15 unmodified lines
7381 return -EINVAL;7408 return -EINVAL;
7382 7409
7383 /* The marker must at least have a tag id */7410 /* The marker must at least have a tag id */
7384 if (cnt < sizeof(unsigned int) || cnt > RAW_DATA_MAX_SIZE)7411 if (cnt < sizeof(unsigned int))
7385 return -EINVAL;7412 return -EINVAL;
7386 7413
7387 if (cnt > TRACE_BUF_SIZE)
7388 cnt = TRACE_BUF_SIZE;
7389
7390 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
7391
7392 size = sizeof(*entry) + cnt;7414 size = sizeof(*entry) + cnt;
7393 if (cnt < FAULT_SIZE_ID)7415 if (cnt < FAULT_SIZE_ID)
7394 size += FAULT_SIZE_ID - cnt;7416 size += FAULT_SIZE_ID - cnt;
7395 7417
7396 buffer = tr->array_buffer.buffer;7418 buffer = tr->array_buffer.buffer;
7419
7420 if (size > ring_buffer_max_event_size(buffer))
7421 return -EINVAL;
7422
7397 event = __trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size,7423 event = __trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size,
7398 tracing_gen_ctx());7424 tracing_gen_ctx());
7399 if (!event)7425 if (!event)
@@ -7578,6 +7604,7 @@ struct ftrace_buffer_info { · 178 unmodified lines
7578 struct trace_iterator iter;7604 struct trace_iterator iter;
7579 void *spare;7605 void *spare;
7580 unsigned int spare_cpu;7606 unsigned int spare_cpu;
7607 unsigned int spare_size;
7581 unsigned int read;7608 unsigned int read;
7582};7609};
7583 7610
@@ -8282,6 +8309,8 @@ tracing_buffers_read(struct file *filp, char __user *ubuf, · 698 unmodified lines
8282{8309{
8283 struct ftrace_buffer_info *info = filp->private_data;8310 struct ftrace_buffer_info *info = filp->private_data;
8284 struct trace_iterator *iter = &info->iter;8311 struct trace_iterator *iter = &info->iter;
8312 void *trace_data;
8313 int page_size;
8285 ssize_t ret = 0;8314 ssize_t ret = 0;
8286 ssize_t size;8315 ssize_t size;
8287 8316
@@ -8293,6 +8322,17 @@ tracing_buffers_read(struct file *filp, char __user *ubuf, · 5 unmodified lines
8293 return -EBUSY;8322 return -EBUSY;
8294#endif8323#endif
8295 8324
8325 page_size = ring_buffer_subbuf_size_get(iter->array_buffer->buffer);
8326
8327 /* Make sure the spare matches the current sub buffer size */
8328 if (info->spare) {
8329 if (page_size != info->spare_size) {
8330 ring_buffer_free_read_page(iter->array_buffer->buffer,
8331 info->spare_cpu, info->spare);
8332 info->spare = NULL;
8333 }
8334 }
8335
8296 if (!info->spare) {8336 if (!info->spare) {
8297 info->spare = ring_buffer_alloc_read_page(iter->array_buffer->buffer,8337 info->spare = ring_buffer_alloc_read_page(iter->array_buffer->buffer,
8298 iter->cpu_file);8338 iter->cpu_file);
@@ -8301,19 +8341,20 @@ tracing_buffers_read(struct file *filp, char __user *ubuf, · 2 unmodified lines
8301 info->spare = NULL;8341 info->spare = NULL;
8302 } else {8342 } else {
8303 info->spare_cpu = iter->cpu_file;8343 info->spare_cpu = iter->cpu_file;
8344 info->spare_size = page_size;
8304 }8345 }
8305 }8346 }
8306 if (!info->spare)8347 if (!info->spare)
8307 return ret;8348 return ret;
8308 8349
8309 /* Do we have previous read data to read? */8350 /* Do we have previous read data to read? */
8310 if (info->read < PAGE_SIZE)8351 if (info->read < page_size)
8311 goto read;8352 goto read;
8312 8353
8313 again:8354 again:
8314 trace_access_lock(iter->cpu_file);8355 trace_access_lock(iter->cpu_file);
8315 ret = ring_buffer_read_page(iter->array_buffer->buffer,8356 ret = ring_buffer_read_page(iter->array_buffer->buffer,
8316 &info->spare,8357 info->spare,
8317 count,8358 count,
8318 iter->cpu_file, 0);8359 iter->cpu_file, 0);
8319 trace_access_unlock(iter->cpu_file);8360 trace_access_unlock(iter->cpu_file);
@@ -8334,11 +8375,11 @@ tracing_buffers_read(struct file *filp, char __user *ubuf, · 14 unmodified lines
8334 8375
8335 info->read = 0;8376 info->read = 0;
8336 read:8377 read:
8337 size = PAGE_SIZE - info->read;8378 size = page_size - info->read;
8338 if (size > count)8379 if (size > count)
8339 size = count;8380 size = count;
8340 8381 trace_data = ring_buffer_read_page_data(info->spare);
8341 ret = copy_to_user(ubuf, info->spare + info->read, size);8382 ret = copy_to_user(ubuf, trace_data + info->read, size);
8342 if (ret == size)8383 if (ret == size)
8343 return -EFAULT;8384 return -EFAULT;
8344 8385
@@ -8449,6 +8490,7 @@ tracing_buffers_splice_read(struct file *file, loff_t *ppos, · 104 unmodified lines
8449 .spd_release = buffer_spd_release,8490 .spd_release = buffer_spd_release,
8450 };8491 };
8451 struct buffer_ref *ref;8492 struct buffer_ref *ref;
8493 int page_size;
8452 int entries, i;8494 int entries, i;
8453 ssize_t ret = 0;8495 ssize_t ret = 0;
8454 8496
@@ -8457,13 +8499,14 @@ tracing_buffers_splice_read(struct file *file, loff_t *ppos, · 2 unmodified lines
8457 return -EBUSY;8499 return -EBUSY;
8458#endif8500#endif
8459 8501
8460 if (*ppos & (PAGE_SIZE - 1))8502 page_size = ring_buffer_subbuf_size_get(iter->array_buffer->buffer);
8503 if (*ppos & (page_size - 1))
8461 return -EINVAL;8504 return -EINVAL;
8462 8505
8463 if (len & (PAGE_SIZE - 1)) {8506 if (len & (page_size - 1)) {
8464 if (len < PAGE_SIZE)8507 if (len < page_size)
8465 return -EINVAL;8508 return -EINVAL;
8466 len &= PAGE_MASK;8509 len &= (~(page_size - 1));
8467 }8510 }
8468 8511
8469 if (splice_grow_spd(pipe, &spd))8512 if (splice_grow_spd(pipe, &spd))
@@ -8473,7 +8516,7 @@ tracing_buffers_splice_read(struct file *file, loff_t *ppos, · 3 unmodified lines
8473 trace_access_lock(iter->cpu_file);8516 trace_access_lock(iter->cpu_file);
8474 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file);8517 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file);
8475 8518
8476 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {8519 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= page_size) {
8477 struct page *page;8520 struct page *page;
8478 int r;8521 int r;
8479 8522
@@ -8494,7 +8537,7 @@ tracing_buffers_splice_read(struct file *file, loff_t *ppos, · 14 unmodified lines
8494 }8537 }
8495 ref->cpu = iter->cpu_file;8538 ref->cpu = iter->cpu_file;
8496 8539
8497 r = ring_buffer_read_page(ref->buffer, &ref->page,8540 r = ring_buffer_read_page(ref->buffer, ref->page,
8498 len, iter->cpu_file, 1);8541 len, iter->cpu_file, 1);
8499 if (r < 0) {8542 if (r < 0) {
8500 ring_buffer_free_read_page(ref->buffer, ref->cpu,8543 ring_buffer_free_read_page(ref->buffer, ref->cpu,
@@ -8503,14 +8546,14 @@ tracing_buffers_splice_read(struct file *file, loff_t *ppos, · 2 unmodified lines
8503 break;8546 break;
8504 }8547 }
8505 8548
8506 page = virt_to_page(ref->page);8549 page = virt_to_page(ring_buffer_read_page_data(ref->page));
8507 8550
8508 spd.pages[i] = page;8551 spd.pages[i] = page;
8509 spd.partial[i].len = PAGE_SIZE;8552 spd.partial[i].len = page_size;
8510 spd.partial[i].offset = 0;8553 spd.partial[i].offset = 0;
8511 spd.partial[i].private = (unsigned long)ref;8554 spd.partial[i].private = (unsigned long)ref;
8512 spd.nr_pages++;8555 spd.nr_pages++;
8513 *ppos += PAGE_SIZE;8556 *ppos += page_size;
8514 8557
8515 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file);8558 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file);
8516 }8559 }
@@ -9354,6 +9397,103 @@ static const struct file_operations buffer_percent_fops = { · 837 unmodified lines
9354 .llseek = default_llseek,9397 .llseek = default_llseek,
9355};9398};
9356 9399
9400static ssize_t
9401buffer_subbuf_size_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
9402{
9403 struct trace_array *tr = filp->private_data;
9404 size_t size;
9405 char buf[64];
9406 int order;
9407 int r;
9408
9409 order = ring_buffer_subbuf_order_get(tr->array_buffer.buffer);
9410 size = (PAGE_SIZE << order) / 1024;
9411
9412 r = sprintf(buf, "%zd\n", size);
9413
9414 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
9415}
9416
9417static ssize_t
9418buffer_subbuf_size_write(struct file *filp, const char __user *ubuf,
9419 size_t cnt, loff_t *ppos)
9420{
9421 struct trace_array *tr = filp->private_data;
9422 unsigned long val;
9423 int old_order;
9424 int order;
9425 int pages;
9426 int ret;
9427
9428 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
9429 if (ret)
9430 return ret;
9431
9432 val *= 1024; /* value passed in is in KB */
9433
9434 pages = DIV_ROUND_UP(val, PAGE_SIZE);
9435 order = fls(pages - 1);
9436
9437 /* limit between 1 and 128 system pages */
9438 if (order < 0 || order > 7)
9439 return -EINVAL;
9440
9441 /* Do not allow tracing while changing the order of the ring buffer */
9442 tracing_stop_tr(tr);
9443
9444 old_order = ring_buffer_subbuf_order_get(tr->array_buffer.buffer);
9445 if (old_order == order)
9446 goto out;
9447
9448 ret = ring_buffer_subbuf_order_set(tr->array_buffer.buffer, order);
9449 if (ret)
9450 goto out;
9451
9452#ifdef CONFIG_TRACER_MAX_TRACE
9453
9454 if (!tr->allocated_snapshot)
9455 goto out_max;
9456
9457 ret = ring_buffer_subbuf_order_set(tr->max_buffer.buffer, order);
9458 if (ret) {
9459 /* Put back the old order */
9460 cnt = ring_buffer_subbuf_order_set(tr->array_buffer.buffer, old_order);
9461 if (WARN_ON_ONCE(cnt)) {
9462 /*
9463 * AARGH! We are left with different orders!
9464 * The max buffer is our "snapshot" buffer.
9465 * When a tracer needs a snapshot (one of the
9466 * latency tracers), it swaps the max buffer
9467 * with the saved snap shot. We succeeded to
9468 * update the order of the main buffer, but failed to
9469 * update the order of the max buffer. But when we tried
9470 * to reset the main buffer to the original size, we
9471 * failed there too. This is very unlikely to
9472 * happen, but if it does, warn and kill all
9473 * tracing.
9474 */
9475 tracing_disabled = 1;
9476 }
9477 goto out;
9478 }
9479 out_max:
9480#endif
9481 (*ppos)++;
9482 out:
9483 if (ret)
9484 cnt = ret;
9485 tracing_start_tr(tr);
9486 return cnt;
9487}
9488
9489static const struct file_operations buffer_subbuf_size_fops = {
9490 .open = tracing_open_generic_tr,
9491 .read = buffer_subbuf_size_read,
9492 .write = buffer_subbuf_size_write,
9493 .release = tracing_release_generic_tr,
9494 .llseek = default_llseek,
9495};
9496
9357static struct dentry *trace_instance_dir;9497static struct dentry *trace_instance_dir;
9358 9498
9359static void9499static void
@@ -9504,7 +9644,8 @@ static int trace_array_create_dir(struct trace_array *tr) · 144 unmodified lines
9504 return ret;9644 return ret;
9505}9645}
9506 9646
9507static struct trace_array *trace_array_create(const char *name)9647static struct trace_array *
9648trace_array_create_systems(const char *name, const char *systems)
9508{9649{
9509 struct trace_array *tr;9650 struct trace_array *tr;
9510 int ret;9651 int ret;
@@ -9524,6 +9665,12 @@ static struct trace_array *trace_array_create(const char *name) · 13 unmodified lines
9524 if (!zalloc_cpumask_var(&tr->pipe_cpumask, GFP_KERNEL))9665 if (!zalloc_cpumask_var(&tr->pipe_cpumask, GFP_KERNEL))
9525 goto out_free_tr;9666 goto out_free_tr;
9526 9667
9668 if (systems) {
9669 tr->system_names = kstrdup_const(systems, GFP_KERNEL);
9670 if (!tr->system_names)
9671 goto out_free_tr;
9672 }
9673
9527 tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS;9674 tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS;
9528 9675
9529 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);9676 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
@@ -9570,12 +9717,18 @@ static struct trace_array *trace_array_create(const char *name) · 40 unmodified lines
9570 free_trace_buffers(tr);9717 free_trace_buffers(tr);
9571 free_cpumask_var(tr->pipe_cpumask);9718 free_cpumask_var(tr->pipe_cpumask);
9572 free_cpumask_var(tr->tracing_cpumask);9719 free_cpumask_var(tr->tracing_cpumask);
9720 kfree_const(tr->system_names);
9573 kfree(tr->name);9721 kfree(tr->name);
9574 kfree(tr);9722 kfree(tr);
9575 9723
9576 return ERR_PTR(ret);9724 return ERR_PTR(ret);
9577}9725}
9578 9726
9727static struct trace_array *trace_array_create(const char *name)
9728{
9729 return trace_array_create_systems(name, NULL);
9730}
9731
9579static int instance_mkdir(const char *name)9732static int instance_mkdir(const char *name)
9580{9733{
9581 struct trace_array *tr;9734 struct trace_array *tr;
@@ -9601,6 +9754,7 @@ out_unlock: · 19 unmodified lines
9601/**9754/**
9602 * trace_array_get_by_name - Create/Lookup a trace array, given its name.9755 * trace_array_get_by_name - Create/Lookup a trace array, given its name.
9603 * @name: The name of the trace array to be looked up/created.9756 * @name: The name of the trace array to be looked up/created.
9757 * @systems: A list of systems to create event directories for (NULL for all)
9604 *9758 *
9605 * Returns pointer to trace array with given name.9759 * Returns pointer to trace array with given name.
9606 * NULL, if it cannot be created.9760 * NULL, if it cannot be created.
@@ -9614,7 +9768,7 @@ out_unlock: · 7 unmodified lines
9614 * trace_array_put() is called, user space can not delete it.9768 * trace_array_put() is called, user space can not delete it.
9615 *9769 *
9616 */9770 */
9617struct trace_array *trace_array_get_by_name(const char *name)9771struct trace_array *trace_array_get_by_name(const char *name, const char *systems)
9618{9772{
9619 struct trace_array *tr;9773 struct trace_array *tr;
9620 9774
@@ -9626,7 +9780,7 @@ struct trace_array *trace_array_get_by_name(const char *name) · 5 unmodified lines
9626 goto out_unlock;9780 goto out_unlock;
9627 }9781 }
9628 9782
9629 tr = trace_array_create(name);9783 tr = trace_array_create_systems(name, systems);
9630 9784
9631 if (IS_ERR(tr))9785 if (IS_ERR(tr))
9632 tr = NULL;9786 tr = NULL;
@@ -9673,6 +9827,7 @@ static int __remove_instance(struct trace_array *tr) · 40 unmodified lines
9673 9827
9674 free_cpumask_var(tr->pipe_cpumask);9828 free_cpumask_var(tr->pipe_cpumask);
9675 free_cpumask_var(tr->tracing_cpumask);9829 free_cpumask_var(tr->tracing_cpumask);
9830 kfree_const(tr->system_names);
9676 kfree(tr->name);9831 kfree(tr->name);
9677 kfree(tr);9832 kfree(tr);
9678 9833
@@ -9805,6 +9960,9 @@ init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer) · 126 unmodified lines
9805 trace_create_file("buffer_percent", TRACE_MODE_WRITE, d_tracer,9960 trace_create_file("buffer_percent", TRACE_MODE_WRITE, d_tracer,
9806 tr, &buffer_percent_fops);9961 tr, &buffer_percent_fops);
9807 9962
9963 trace_create_file("buffer_subbuf_size_kb", TRACE_MODE_WRITE, d_tracer,
9964 tr, &buffer_subbuf_size_fops);
9965
9808 create_trace_options_dir(tr);9966 create_trace_options_dir(tr);
9809 9967
9810#ifdef CONFIG_TRACER_MAX_TRACE9968#ifdef CONFIG_TRACER_MAX_TRACE
@@ -10391,7 +10549,7 @@ __init static void enable_instances(void) · 580 unmodified lines
10391 if (IS_ENABLED(CONFIG_TRACER_MAX_TRACE))10549 if (IS_ENABLED(CONFIG_TRACER_MAX_TRACE))
10392 do_allocate_snapshot(tok);10550 do_allocate_snapshot(tok);
10393 10551
10394 tr = trace_array_get_by_name(tok);10552 tr = trace_array_get_by_name(tok, NULL);
10395 if (!tr) {10553 if (!tr) {
10396 pr_warn("Failed to create instance buffer %s\n", curr_str);10554 pr_warn("Failed to create instance buffer %s\n", curr_str);
10397 continue;10555 continue;
kernel/trace/trace.h+2−0
@@ -377,6 +377,7 @@ struct trace_array { · 376 unmodified lines
377 unsigned char trace_flags_index[TRACE_FLAGS_MAX_SIZE];377 unsigned char trace_flags_index[TRACE_FLAGS_MAX_SIZE];
378 unsigned int flags;378 unsigned int flags;
379 raw_spinlock_t start_lock;379 raw_spinlock_t start_lock;
380 const char *system_names;
380 struct list_head err_log;381 struct list_head err_log;
381 struct dentry *dir;382 struct dentry *dir;
382 struct dentry *options;383 struct dentry *options;
@@ -615,6 +616,7 @@ void tracing_reset_all_online_cpus(void); · 232 unmodified lines
615void tracing_reset_all_online_cpus_unlocked(void);616void tracing_reset_all_online_cpus_unlocked(void);
616int tracing_open_generic(struct inode *inode, struct file *filp);617int tracing_open_generic(struct inode *inode, struct file *filp);
617int tracing_open_generic_tr(struct inode *inode, struct file *filp);618int tracing_open_generic_tr(struct inode *inode, struct file *filp);
619int tracing_release_generic_tr(struct inode *inode, struct file *file);
618int tracing_open_file_tr(struct inode *inode, struct file *filp);620int tracing_open_file_tr(struct inode *inode, struct file *filp);
619int tracing_release_file_tr(struct inode *inode, struct file *filp);621int tracing_release_file_tr(struct inode *inode, struct file *filp);
620int tracing_single_release_file_tr(struct inode *inode, struct file *filp);622int tracing_single_release_file_tr(struct inode *inode, struct file *filp);
kernel/trace/trace_boot.c+1−1
@@ -633,7 +633,7 @@ trace_boot_init_instances(struct xbc_node *node) · 632 unmodified lines
633 if (!p || *p == '\0')633 if (!p || *p == '\0')
634 continue;634 continue;
635 635
636 tr = trace_array_get_by_name(p);636 tr = trace_array_get_by_name(p, NULL);
637 if (!tr) {637 if (!tr) {
638 pr_err("Failed to get trace instance %s\n", p);638 pr_err("Failed to get trace instance %s\n", p);
639 continue;639 continue;
kernel/trace/trace_events.c+90−17
@@ -1893,9 +1893,9 @@ subsystem_filter_write(struct file *filp, const char __user *ubuf, size_t cnt, · 1892 unmodified lines
1893}1893}
1894 1894
1895static ssize_t1895static ssize_t
1896show_header(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)1896show_header_page_file(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
1897{1897{
1898 int (*func)(struct trace_seq *s) = filp->private_data;1898 struct trace_array *tr = filp->private_data;
1899 struct trace_seq *s;1899 struct trace_seq *s;
1900 int r;1900 int r;
1901 1901
@@ -1908,7 +1908,31 @@ show_header(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) · 6 unmodified lines
1908 1908
1909 trace_seq_init(s);1909 trace_seq_init(s);
1910 1910
1911 func(s);1911 ring_buffer_print_page_header(tr->array_buffer.buffer, s);
1912 r = simple_read_from_buffer(ubuf, cnt, ppos,
1913 s->buffer, trace_seq_used(s));
1914
1915 kfree(s);
1916
1917 return r;
1918}
1919
1920static ssize_t
1921show_header_event_file(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
1922{
1923 struct trace_seq *s;
1924 int r;
1925
1926 if (*ppos)
1927 return 0;
1928
1929 s = kmalloc(sizeof(*s), GFP_KERNEL);
1930 if (!s)
1931 return -ENOMEM;
1932
1933 trace_seq_init(s);
1934
1935 ring_buffer_print_entry_header(s);
1912 r = simple_read_from_buffer(ubuf, cnt, ppos,1936 r = simple_read_from_buffer(ubuf, cnt, ppos,
1913 s->buffer, trace_seq_used(s));1937 s->buffer, trace_seq_used(s));
1914 1938
@@ -2165,10 +2189,18 @@ static const struct file_operations ftrace_tr_enable_fops = { · 250 unmodified lines
2165 .release = subsystem_release,2189 .release = subsystem_release,
2166};2190};
2167 2191
2168static const struct file_operations ftrace_show_header_fops = {2192static const struct file_operations ftrace_show_header_page_fops = {
2169 .open = tracing_open_generic,2193 .open = tracing_open_generic_tr,
2170 .read = show_header,2194 .read = show_header_page_file,
2195 .llseek = default_llseek,
2196 .release = tracing_release_generic_tr,
2197};
2198
2199static const struct file_operations ftrace_show_header_event_fops = {
2200 .open = tracing_open_generic_tr,
2201 .read = show_header_event_file,
2171 .llseek = default_llseek,2202 .llseek = default_llseek,
2203 .release = tracing_release_generic_tr,
2172};2204};
2173 2205
2174static int2206static int
@@ -2896,6 +2928,27 @@ void trace_event_eval_update(struct trace_eval_map **map, int len) · 721 unmodified lines
2896 up_write(&trace_event_sem);2928 up_write(&trace_event_sem);
2897}2929}
2898 2930
2931static bool event_in_systems(struct trace_event_call *call,
2932 const char *systems)
2933{
2934 const char *system;
2935 const char *p;
2936
2937 if (!systems)
2938 return true;
2939
2940 system = call->class->system;
2941 p = strstr(systems, system);
2942 if (!p)
2943 return false;
2944
2945 if (p != systems && !isspace(*(p - 1)) && *(p - 1) != ',')
2946 return false;
2947
2948 p += strlen(system);
2949 return !*p || isspace(*p) || *p == ',';
2950}
2951
2899static struct trace_event_file *2952static struct trace_event_file *
2900trace_create_new_event(struct trace_event_call *call,2953trace_create_new_event(struct trace_event_call *call,
2901 struct trace_array *tr)2954 struct trace_array *tr)
@@ -2905,9 +2958,12 @@ trace_create_new_event(struct trace_event_call *call, · 3 unmodified lines
2905 struct trace_event_file *file;2958 struct trace_event_file *file;
2906 unsigned int first;2959 unsigned int first;
2907 2960
2961 if (!event_in_systems(call, tr->system_names))
2962 return NULL;
2963
2908 file = kmem_cache_alloc(file_cachep, GFP_TRACE);2964 file = kmem_cache_alloc(file_cachep, GFP_TRACE);
2909 if (!file)2965 if (!file)
2910 return NULL;2966 return ERR_PTR(-ENOMEM);
2911 2967
2912 pid_list = rcu_dereference_protected(tr->filtered_pids,2968 pid_list = rcu_dereference_protected(tr->filtered_pids,
2913 lockdep_is_held(&event_mutex));2969 lockdep_is_held(&event_mutex));
@@ -2972,8 +3028,17 @@ __trace_add_new_event(struct trace_event_call *call, struct trace_array *tr) · 58 unmodified lines
2972 struct trace_event_file *file;3028 struct trace_event_file *file;
2973 3029
2974 file = trace_create_new_event(call, tr);3030 file = trace_create_new_event(call, tr);
3031 /*
3032 * trace_create_new_event() returns ERR_PTR(-ENOMEM) if failed
3033 * allocation, or NULL if the event is not part of the tr->system_names.
3034 * When the event is not part of the tr->system_names, return zero, not
3035 * an error.
3036 */
2975 if (!file)3037 if (!file)
2976 return -ENOMEM;3038 return 0;
3039
3040 if (IS_ERR(file))
3041 return PTR_ERR(file);
2977 3042
2978 if (eventdir_initialized)3043 if (eventdir_initialized)
2979 return event_create_dir(tr->event_dir, file);3044 return event_create_dir(tr->event_dir, file);
@@ -3012,8 +3077,17 @@ __trace_early_add_new_event(struct trace_event_call *call, · 32 unmodified lines
3012 int ret;3077 int ret;
3013 3078
3014 file = trace_create_new_event(call, tr);3079 file = trace_create_new_event(call, tr);
3080 /*
3081 * trace_create_new_event() returns ERR_PTR(-ENOMEM) if failed
3082 * allocation, or NULL if the event is not part of the tr->system_names.
3083 * When the event is not part of the tr->system_names, return zero, not
3084 * an error.
3085 */
3015 if (!file)3086 if (!file)
3016 return -ENOMEM;3087 return 0;
3088
3089 if (IS_ERR(file))
3090 return PTR_ERR(file);
3017 3091
3018 ret = event_define_fields(call);3092 ret = event_define_fields(call);
3019 if (ret)3093 if (ret)
@@ -3752,17 +3826,16 @@ static int events_callback(const char *name, umode_t *mode, void **data, · 732 unmodified lines
3752 return 1;3826 return 1;
3753 }3827 }
3754 3828
3755 if (strcmp(name, "header_page") == 0)3829 if (strcmp(name, "header_page") == 0) {
3756 *data = ring_buffer_print_page_header;3830 *mode = TRACE_MODE_READ;
3757 3831 *fops = &ftrace_show_header_page_fops;
3758 else if (strcmp(name, "header_event") == 0)
3759 *data = ring_buffer_print_entry_header;
3760 3832
3761 else3833 } else if (strcmp(name, "header_event") == 0) {
3834 *mode = TRACE_MODE_READ;
3835 *fops = &ftrace_show_header_event_fops;
3836 } else
3762 return 0;3837 return 0;
3763 3838
3764 *mode = TRACE_MODE_READ;
3765 *fops = &ftrace_show_header_fops;
3766 return 1;3839 return 1;
3767}3840}
3768 3841
kernel/trace/trace_events_hist.c+24−25
@@ -4805,36 +4805,35 @@ static int parse_actions(struct hist_trigger_data *hist_data) · 4804 unmodified lines
4805 int len;4805 int len;
4806 4806
4807 for (i = 0; i < hist_data->attrs->n_actions; i++) {4807 for (i = 0; i < hist_data->attrs->n_actions; i++) {
4808 enum handler_id hid = 0;
4809 char *action_str;
4810
4808 str = hist_data->attrs->action_str[i];4811 str = hist_data->attrs->action_str[i];
4809 4812
4810 if ((len = str_has_prefix(str, "onmatch("))) {4813 if ((len = str_has_prefix(str, "onmatch(")))
4811 char *action_str = str + len;4814 hid = HANDLER_ONMATCH;
4815 else if ((len = str_has_prefix(str, "onmax(")))
4816 hid = HANDLER_ONMAX;
4817 else if ((len = str_has_prefix(str, "onchange(")))
4818 hid = HANDLER_ONCHANGE;
4812 4819
4813 data = onmatch_parse(tr, action_str);4820 action_str = str + len;
4814 if (IS_ERR(data)) {
4815 ret = PTR_ERR(data);
4816 break;
4817 }
4818 } else if ((len = str_has_prefix(str, "onmax("))) {
4819 char *action_str = str + len;
4820 4821
4821 data = track_data_parse(hist_data, action_str,4822 switch (hid) {
4822 HANDLER_ONMAX);4823 case HANDLER_ONMATCH:
4823 if (IS_ERR(data)) {4824 data = onmatch_parse(tr, action_str);
4824 ret = PTR_ERR(data);4825 break;
4825 break;4826 case HANDLER_ONMAX:
4826 }4827 case HANDLER_ONCHANGE:
4827 } else if ((len = str_has_prefix(str, "onchange("))) {4828 data = track_data_parse(hist_data, action_str, hid);
4828 char *action_str = str + len;4829 break;
4830 default:
4831 data = ERR_PTR(-EINVAL);
4832 break;
4833 }
4829 4834
4830 data = track_data_parse(hist_data, action_str,4835 if (IS_ERR(data)) {
4831 HANDLER_ONCHANGE);4836 ret = PTR_ERR(data);
4832 if (IS_ERR(data)) {
4833 ret = PTR_ERR(data);
4834 break;
4835 }
4836 } else {
4837 ret = -EINVAL;
4838 break;4837 break;
4839 }4838 }
4840 4839
kernel/trace/trace_seq.c+0−3
@@ -13,9 +13,6 @@ · 12 unmodified lines
13 * trace_seq_init() more than once to reset the trace_seq to start13 * trace_seq_init() more than once to reset the trace_seq to start
14 * from scratch.14 * from scratch.
15 * 15 *
16 * The buffer size is currently PAGE_SIZE, although it may become dynamic
17 * in the future.
18 *
19 * A write to the buffer will either succeed or fail. That is, unlike16 * A write to the buffer will either succeed or fail. That is, unlike
20 * sprintf() there will not be a partial write (well it may write into17 * sprintf() there will not be a partial write (well it may write into
21 * the buffer but it wont update the pointers). This allows users to18 * the buffer but it wont update the pointers). This allows users to