/* * Generic ring buffer * * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com> */#include <linux/ring_buffer.h>#include <linux/trace_clock.h>#include <linux/ftrace_irq.h>#include <linux/spinlock.h>#include <linux/debugfs.h>#include <linux/uaccess.h>#include <linux/hardirq.h>#include <linux/kmemcheck.h>#include <linux/module.h>#include <linux/percpu.h>#include <linux/mutex.h>#include <linux/slab.h>#include <linux/init.h>#include <linux/hash.h>#include <linux/list.h>#include <linux/cpu.h>#include <linux/fs.h> #include <asm/local.h>#include "trace.h" /* * The ring buffer header is special. We must manually up keep it. */int ring_buffer_print_entry_header(struct trace_seq *s){ int ret; ret = trace_seq_printf(s, "# compressed entry header\n"); ret = trace_seq_printf(s, "\ttype_len : 5 bits\n"); ret = trace_seq_printf(s, "\ttime_delta : 27 bits\n"); ret = trace_seq_printf(s, "\tarray : 32 bits\n"); ret = trace_seq_printf(s, "\n"); ret = trace_seq_printf(s, "\tpadding : type == %d\n", RINGBUF_TYPE_PADDING); ret = trace_seq_printf(s, "\ttime_extend : type == %d\n", RINGBUF_TYPE_TIME_EXTEND); ret = trace_seq_printf(s, "\tdata max type_len == %d\n", RINGBUF_TYPE_DATA_TYPE_LEN_MAX); return ret;} /* * The ring buffer is made up of a list of pages. A separate list of pages is * allocated for each CPU. A writer may only write to a buffer that is * associated with the CPU it is currently executing on. A reader may read * from any per cpu buffer. * * The reader is special. For each per cpu buffer, the reader has its own * reader page. When a reader has read the entire reader page, this reader * page is swapped with another page in the ring buffer. * * Now, as long as the writer is off the reader page, the reader can do what * ever it wants with that page. The writer will never write to that page * again (as long as it is out of the ring buffer). * * Here's some silly ASCII art. * * +------+ * |reader| RING BUFFER * |page | * +------+ +---+ +---+ +---+ * | |-->| |-->| | * +---+ +---+ +---+ * ^ | * | | * +---------------+ * * * +------+ * |reader| RING BUFFER * |page |------------------v * +------+ +---+ +---+ +---+ * | |-->| |-->| | * +---+ +---+ +---+ * ^ | * | | * +---------------+ * * * +------+ * |reader| RING BUFFER * |page |------------------v * +------+ +---+ +---+ +---+ * ^ | |-->| |-->| | * | +---+ +---+ +---+ * | | * | | * +------------------------------+ * * * +------+ * |buffer| RING BUFFER * |page |------------------v * +------+ +---+ +---+ +---+ * ^ | | | |-->| | * | New +---+ +---+ +---+ * | Reader------^ | * | page | * +------------------------------+ * * * After we make this swap, the reader can hand this page off to the splice * code and be done with it. It can even allocate a new page if it needs to * and swap that into the ring buffer. * * We will be using cmpxchg soon to make all this lockless. * */ /* * A fast way to enable or disable all ring buffers is to * call tracing_on or tracing_off. Turning off the ring buffers * prevents all ring buffers from being recorded to. * Turning this switch on, makes it OK to write to the * ring buffer, if the ring buffer is enabled itself. * * There's three layers that must be on in order to write * to the ring buffer. * * 1) This global flag must be set. * 2) The ring buffer must be enabled for recording. * 3) The per cpu buffer must be enabled for recording. * * In case of an anomaly, this global flag has a bit set that * will permantly disable all ring buffers. */ /* * Global flag to disable all recording to ring buffers * This has two bits: ON, DISABLED * * ON DISABLED * ---- ---------- * 0 0 : ring buffers are off * 1 0 : ring buffers are on * X 1 : ring buffers are permanently disabled */ enum { RB_BUFFERS_ON_BIT = 0, RB_BUFFERS_DISABLED_BIT = 1,}; enum { RB_BUFFERS_ON = 1 << RB_BUFFERS_ON_BIT, RB_BUFFERS_DISABLED = 1 << RB_BUFFERS_DISABLED_BIT,}; static unsigned long ring_buffer_flags __read_mostly = RB_BUFFERS_ON; #define BUF_PAGE_HDR_SIZE offsetof(struct buffer_data_page, data) /** * tracing_on - enable all tracing buffers * * This function enables all tracing buffers that may have been * disabled with tracing_off. */void tracing_on(void){ set_bit(RB_BUFFERS_ON_BIT, &ring_buffer_flags);}EXPORT_SYMBOL_GPL(tracing_on); /** * tracing_off - turn off all tracing buffers * * This function stops all tracing buffers from recording data. * It does not disable any overhead the tracers themselves may * be causing. This function simply causes all recording to * the ring buffers to fail. */void tracing_off(void){ clear_bit(RB_BUFFERS_ON_BIT, &ring_buffer_flags);}EXPORT_SYMBOL_GPL(tracing_off); /** * tracing_off_permanent - permanently disable ring buffers * * This function, once called, will disable all ring buffers * permanently. */void tracing_off_permanent(void){ set_bit(RB_BUFFERS_DISABLED_BIT, &ring_buffer_flags);} /** * tracing_is_on - show state of ring buffers enabled */int tracing_is_on(void){ return ring_buffer_flags == RB_BUFFERS_ON;}EXPORT_SYMBOL_GPL(tracing_is_on); #define RB_EVNT_HDR_SIZE (offsetof(struct ring_buffer_event, array))#define RB_ALIGNMENT 4U#define RB_MAX_SMALL_DATA (RB_ALIGNMENT * RINGBUF_TYPE_DATA_TYPE_LEN_MAX)#define RB_EVNT_MIN_SIZE 8U /* two 32bit words */ #if !defined(CONFIG_64BIT) || defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)# define RB_FORCE_8BYTE_ALIGNMENT 0# define RB_ARCH_ALIGNMENT RB_ALIGNMENT#else# define RB_FORCE_8BYTE_ALIGNMENT 1# define RB_ARCH_ALIGNMENT 8U#endif /* define RINGBUF_TYPE_DATA for 'case RINGBUF_TYPE_DATA:' */#define RINGBUF_TYPE_DATA 0 ... RINGBUF_TYPE_DATA_TYPE_LEN_MAX enum { RB_LEN_TIME_EXTEND = 8, RB_LEN_TIME_STAMP = 16,}; static inline int rb_null_event(struct ring_buffer_event *event){ return event->type_len == RINGBUF_TYPE_PADDING && !event->time_delta;} static void rb_event_set_padding(struct ring_buffer_event *event){ /* padding has a NULL time_delta */ event->type_len = RINGBUF_TYPE_PADDING; event->time_delta = 0;} static unsignedrb_event_data_length(struct ring_buffer_event *event){ unsigned length; if (event->type_len) length = event->type_len * RB_ALIGNMENT; else length = event->array[0]; return length + RB_EVNT_HDR_SIZE;} /* inline for ring buffer fast paths */static unsignedrb_event_length(struct ring_buffer_event *event){ switch (event->type_len) { case RINGBUF_TYPE_PADDING: if (rb_null_event(event)) /* undefined */ return -1; return event->array[0] + RB_EVNT_HDR_SIZE; case RINGBUF_TYPE_TIME_EXTEND: return RB_LEN_TIME_EXTEND; case RINGBUF_TYPE_TIME_STAMP: return RB_LEN_TIME_STAMP; case RINGBUF_TYPE_DATA: return rb_event_data_length(event); default: BUG(); } /* not hit */ return 0;} /** * ring_buffer_event_length - return the length of the event * @event: the event to get the length of */unsigned ring_buffer_event_length(struct ring_buffer_event *event){ unsigned length = rb_event_length(event); if (event->type_len > RINGBUF_TYPE_DATA_TYPE_LEN_MAX) return length; length -= RB_EVNT_HDR_SIZE; if (length > RB_MAX_SMALL_DATA + sizeof(event->array[0])) length -= sizeof(event->array[0]); return length;}EXPORT_SYMBOL_GPL(ring_buffer_event_length); /* inline for ring buffer fast paths */static void *rb_event_data(struct ring_buffer_event *event){ BUG_ON(event->type_len > RINGBUF_TYPE_DATA_TYPE_LEN_MAX); /* If length is in len field, then array[0] has the data */ if (event->type_len) return (void *)&event->array[0]; /* Otherwise length is in array[0] and array[1] has the data */ return (void *)&event->array[1];} /** * ring_buffer_event_data - return the data of the event * @event: the event to get the data from */void *ring_buffer_event_data(struct ring_buffer_event *event){ return rb_event_data(event);}EXPORT_SYMBOL_GPL(ring_buffer_event_data); #define for_each_buffer_cpu(buffer, cpu) \ for_each_cpu(cpu, buffer->cpumask) #define TS_SHIFT 27#define TS_MASK ((1ULL << TS_SHIFT) - 1)#define TS_DELTA_TEST (~TS_MASK) /* Flag when events were overwritten */#define RB_MISSED_EVENTS (1 << 31)/* Missed count stored at end */#define RB_MISSED_STORED (1 << 30) struct buffer_data_page { u64 time_stamp; /* page time stamp */ local_t commit; /* write committed index */ unsigned char data[]; /* data of buffer page */}; /* * Note, the buffer_page list must be first. The buffer pages * are allocated in cache lines, which means that each buffer * page will be at the beginning of a cache line, and thus * the least significant bits will be zero. We use this to * add flags in the list struct pointers, to make the ring buffer * lockless. */struct buffer_page { struct list_head list; /* list of buffer pages */ local_t write; /* index for next write */ unsigned read; /* index for next read */ local_t entries; /* entries on this page */ unsigned long real_end; /* real end of data */ struct buffer_data_page *page; /* Actual data page */}; /* * The buffer page counters, write and entries, must be reset * atomically when crossing page boundaries. To synchronize this * update, two counters are inserted into the number. One is * the actual counter for the write position or count on the page. * * The other is a counter of updaters. Before an update happens * the update partition of the counter is incremented. This will * allow the updater to update the counter atomically. * * The counter is 20 bits, and the state data is 12. */#define RB_WRITE_MASK 0xfffff#define RB_WRITE_INTCNT (1 << 20) static void rb_init_page(struct buffer_data_page *bpage){ local_set(&bpage->commit, 0);} /** * ring_buffer_page_len - the size of data on the page. * @page: The page to read * * Returns the amount of data on the page, including buffer page header. */size_t ring_buffer_page_len(void *page){ return local_read(&((struct buffer_data_page *)page)->commit) + BUF_PAGE_HDR_SIZE;} /* * Also stolen from mm/slob.c. Thanks to Mathieu Desnoyers for pointing * this issue out. */static void free_buffer_page(struct buffer_page *bpage){ free_page((unsigned long)bpage->page); kfree(bpage);} /* * We need to fit the time_stamp delta into 27 bits. */static inline int test_time_stamp(u64 delta){ if (delta & TS_DELTA_TEST) return 1; return 0;} #define BUF_PAGE_SIZE (PAGE_SIZE - BUF_PAGE_HDR_SIZE) /* Max payload is BUF_PAGE_SIZE - header (8bytes) */#define BUF_MAX_DATA_SIZE (BUF_PAGE_SIZE - (sizeof(u32) * 2)) /* Max number of timestamps that can fit on a page */#define RB_TIMESTAMPS_PER_PAGE (BUF_PAGE_SIZE / RB_LEN_TIME_STAMP) int ring_buffer_print_page_header(struct trace_seq *s){ struct buffer_data_page field; int ret; ret = trace_seq_printf(s, "\tfield: u64 timestamp;\t" "offset:0;\tsize:%u;\tsigned:%u;\n", (unsigned int)sizeof(field.time_stamp), (unsigned int)is_signed_type(u64)); ret = trace_seq_printf(s, "\tfield: local_t commit;\t" "offset:%u;\tsize:%u;\tsigned:%u;\n", (unsigned int)offsetof(typeof(field), commit), (unsigned int)sizeof(field.commit), (unsigned int)is_signed_type(long)); ret = trace_seq_printf(s, "\tfield: int overwrite;\t" "offset:%u;\tsize:%u;\tsigned:%u;\n", (unsigned int)offsetof(typeof(field), commit), 1, (unsigned int)is_signed_type(long)); ret = trace_seq_printf(s, "\tfield: char data;\t" "offset:%u;\tsize:%u;\tsigned:%u;\n", (unsigned int)offsetof(typeof(field), data), (unsigned int)BUF_PAGE_SIZE, (unsigned int)is_signed_type(char)); return ret;} /* * head_page == tail_page && head == tail then buffer is empty. */struct ring_buffer_per_cpu { int cpu; atomic_t record_disabled; struct ring_buffer *buffer; spinlock_t reader_lock; /* serialize readers */ arch_spinlock_t lock; struct lock_class_key lock_key; struct list_head *pages; struct buffer_page *head_page; /* read from head */ struct buffer_page *tail_page; /* write to tail */ struct buffer_page *commit_page; /* committed pages */ struct buffer_page *reader_page; unsigned long lost_events; unsigned long last_overrun; local_t commit_overrun; local_t overrun; local_t entries; local_t committing; local_t commits; unsigned long read; u64 write_stamp; u64 read_stamp;}; struct ring_buffer { unsigned pages; unsigned flags; int cpus; atomic_t record_disabled; cpumask_var_t cpumask; struct lock_class_key *reader_lock_key; struct mutex mutex; struct ring_buffer_per_cpu **buffers; #ifdef CONFIG_HOTPLUG_CPU struct notifier_block cpu_notify;#endif u64 (*clock)(void);}; struct ring_buffer_iter { struct ring_buffer_per_cpu *cpu_buffer; unsigned long head; struct buffer_page *head_page; struct buffer_page *cache_reader_page; unsigned long cache_read; u64 read_stamp;}; /* buffer may be either ring_buffer or ring_buffer_per_cpu */#define RB_WARN_ON(b, cond) \ ({ \ int _____ret = unlikely(cond); \ if (_____ret) { \Lines 1–500 of 4023. Load more
/* * Generic ring buffer * * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com> */ #include <linux/ring_buffer.h> #include <linux/trace_clock.h> #include <linux/ftrace_irq.h> #include <linux/spinlock.h> #include <linux/debugfs.h> #include <linux/uaccess.h> #include <linux/hardirq.h> #include <linux/kmemcheck.h> #include <linux/module.h> #include <linux/percpu.h> #include <linux/mutex.h> #include <linux/slab.h> #include <linux/init.h> #include <linux/hash.h> #include <linux/list.h> #include <linux/cpu.h> #include <linux/fs.h> #include <asm/local.h> #include "trace.h" /* * The ring buffer header is special. We must manually up keep it. */ int ring_buffer_print_entry_header(struct trace_seq *s) { int ret; ret = trace_seq_printf(s, "# compressed entry header\n"); ret = trace_seq_printf(s, "\ttype_len : 5 bits\n"); ret = trace_seq_printf(s, "\ttime_delta : 27 bits\n"); ret = trace_seq_printf(s, "\tarray : 32 bits\n"); ret = trace_seq_printf(s, "\n"); ret = trace_seq_printf(s, "\tpadding : type == %d\n", RINGBUF_TYPE_PADDING); ret = trace_seq_printf(s, "\ttime_extend : type == %d\n", RINGBUF_TYPE_TIME_EXTEND); ret = trace_seq_printf(s, "\tdata max type_len == %d\n", RINGBUF_TYPE_DATA_TYPE_LEN_MAX); return ret; } /* * The ring buffer is made up of a list of pages. A separate list of pages is * allocated for each CPU. A writer may only write to a buffer that is * associated with the CPU it is currently executing on. A reader may read * from any per cpu buffer. * * The reader is special. For each per cpu buffer, the reader has its own * reader page. When a reader has read the entire reader page, this reader * page is swapped with another page in the ring buffer. * * Now, as long as the writer is off the reader page, the reader can do what * ever it wants with that page. The writer will never write to that page * again (as long as it is out of the ring buffer). * * Here's some silly ASCII art. * * +------+ * |reader| RING BUFFER * |page | * +------+ +---+ +---+ +---+ * | |-->| |-->| | * +---+ +---+ +---+ * ^ | * | | * +---------------+ * * * +------+ * |reader| RING BUFFER * |page |------------------v * +------+ +---+ +---+ +---+ * | |-->| |-->| | * +---+ +---+ +---+ * ^ | * | | * +---------------+ * * * +------+ * |reader| RING BUFFER * |page |------------------v * +------+ +---+ +---+ +---+ * ^ | |-->| |-->| | * | +---+ +---+ +---+ * | | * | | * +------------------------------+ * * * +------+ * |buffer| RING BUFFER * |page |------------------v * +------+ +---+ +---+ +---+ * ^ | | | |-->| | * | New +---+ +---+ +---+ * | Reader------^ | * | page | * +------------------------------+ * * * After we make this swap, the reader can hand this page off to the splice * code and be done with it. It can even allocate a new page if it needs to * and swap that into the ring buffer. * * We will be using cmpxchg soon to make all this lockless. * */ /* * A fast way to enable or disable all ring buffers is to * call tracing_on or tracing_off. Turning off the ring buffers * prevents all ring buffers from being recorded to. * Turning this switch on, makes it OK to write to the * ring buffer, if the ring buffer is enabled itself. * * There's three layers that must be on in order to write * to the ring buffer. * * 1) This global flag must be set. * 2) The ring buffer must be enabled for recording. * 3) The per cpu buffer must be enabled for recording. * * In case of an anomaly, this global flag has a bit set that * will permantly disable all ring buffers. */ /* * Global flag to disable all recording to ring buffers * This has two bits: ON, DISABLED * * ON DISABLED * ---- ---------- * 0 0 : ring buffers are off * 1 0 : ring buffers are on * X 1 : ring buffers are permanently disabled */ enum { RB_BUFFERS_ON_BIT = 0, RB_BUFFERS_DISABLED_BIT = 1, }; enum { RB_BUFFERS_ON = 1 << RB_BUFFERS_ON_BIT, RB_BUFFERS_DISABLED = 1 << RB_BUFFERS_DISABLED_BIT, }; static unsigned long ring_buffer_flags __read_mostly = RB_BUFFERS_ON; #define BUF_PAGE_HDR_SIZE offsetof(struct buffer_data_page, data) /** * tracing_on - enable all tracing buffers * * This function enables all tracing buffers that may have been * disabled with tracing_off. */ void tracing_on(void) { set_bit(RB_BUFFERS_ON_BIT, &ring_buffer_flags); } EXPORT_SYMBOL_GPL(tracing_on); /** * tracing_off - turn off all tracing buffers * * This function stops all tracing buffers from recording data. * It does not disable any overhead the tracers themselves may * be causing. This function simply causes all recording to * the ring buffers to fail. */ void tracing_off(void) { clear_bit(RB_BUFFERS_ON_BIT, &ring_buffer_flags); } EXPORT_SYMBOL_GPL(tracing_off); /** * tracing_off_permanent - permanently disable ring buffers * * This function, once called, will disable all ring buffers * permanently. */ void tracing_off_permanent(void) { set_bit(RB_BUFFERS_DISABLED_BIT, &ring_buffer_flags); } /** * tracing_is_on - show state of ring buffers enabled */ int tracing_is_on(void) { return ring_buffer_flags == RB_BUFFERS_ON; } EXPORT_SYMBOL_GPL(tracing_is_on); #define RB_EVNT_HDR_SIZE (offsetof(struct ring_buffer_event, array)) #define RB_ALIGNMENT 4U #define RB_MAX_SMALL_DATA (RB_ALIGNMENT * RINGBUF_TYPE_DATA_TYPE_LEN_MAX) #define RB_EVNT_MIN_SIZE 8U /* two 32bit words */ #if !defined(CONFIG_64BIT) || defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) # define RB_FORCE_8BYTE_ALIGNMENT 0 # define RB_ARCH_ALIGNMENT RB_ALIGNMENT #else # define RB_FORCE_8BYTE_ALIGNMENT 1 # define RB_ARCH_ALIGNMENT 8U #endif /* define RINGBUF_TYPE_DATA for 'case RINGBUF_TYPE_DATA:' */ #define RINGBUF_TYPE_DATA 0 ... RINGBUF_TYPE_DATA_TYPE_LEN_MAX enum { RB_LEN_TIME_EXTEND = 8, RB_LEN_TIME_STAMP = 16, }; static inline int rb_null_event(struct ring_buffer_event *event) { return event->type_len == RINGBUF_TYPE_PADDING && !event->time_delta; } static void rb_event_set_padding(struct ring_buffer_event *event) { /* padding has a NULL time_delta */ event->type_len = RINGBUF_TYPE_PADDING; event->time_delta = 0; } static unsigned rb_event_data_length(struct ring_buffer_event *event) { unsigned length; if (event->type_len) length = event->type_len * RB_ALIGNMENT; else length = event->array[0]; return length + RB_EVNT_HDR_SIZE; } /* inline for ring buffer fast paths */ static unsigned rb_event_length(struct ring_buffer_event *event) { switch (event->type_len) { case RINGBUF_TYPE_PADDING: if (rb_null_event(event)) /* undefined */ return -1; return event->array[0] + RB_EVNT_HDR_SIZE; case RINGBUF_TYPE_TIME_EXTEND: return RB_LEN_TIME_EXTEND; case RINGBUF_TYPE_TIME_STAMP: return RB_LEN_TIME_STAMP; case RINGBUF_TYPE_DATA: return rb_event_data_length(event); default: BUG(); } /* not hit */ return 0; } /** * ring_buffer_event_length - return the length of the event * @event: the event to get the length of */ unsigned ring_buffer_event_length(struct ring_buffer_event *event) { unsigned length = rb_event_length(event); if (event->type_len > RINGBUF_TYPE_DATA_TYPE_LEN_MAX) return length; length -= RB_EVNT_HDR_SIZE; if (length > RB_MAX_SMALL_DATA + sizeof(event->array[0])) length -= sizeof(event->array[0]); return length; } EXPORT_SYMBOL_GPL(ring_buffer_event_length); /* inline for ring buffer fast paths */ static void * rb_event_data(struct ring_buffer_event *event) { BUG_ON(event->type_len > RINGBUF_TYPE_DATA_TYPE_LEN_MAX); /* If length is in len field, then array[0] has the data */ if (event->type_len) return (void *)&event->array[0]; /* Otherwise length is in array[0] and array[1] has the data */ return (void *)&event->array[1]; } /** * ring_buffer_event_data - return the data of the event * @event: the event to get the data from */ void *ring_buffer_event_data(struct ring_buffer_event *event) { return rb_event_data(event); } EXPORT_SYMBOL_GPL(ring_buffer_event_data); #define for_each_buffer_cpu(buffer, cpu) \ for_each_cpu(cpu, buffer->cpumask) #define TS_SHIFT 27 #define TS_MASK ((1ULL << TS_SHIFT) - 1) #define TS_DELTA_TEST (~TS_MASK) /* Flag when events were overwritten */ #define RB_MISSED_EVENTS (1 << 31) /* Missed count stored at end */ #define RB_MISSED_STORED (1 << 30) struct buffer_data_page { u64 time_stamp; /* page time stamp */ local_t commit; /* write committed index */ unsigned char data[]; /* data of buffer page */ }; /* * Note, the buffer_page list must be first. The buffer pages * are allocated in cache lines, which means that each buffer * page will be at the beginning of a cache line, and thus * the least significant bits will be zero. We use this to * add flags in the list struct pointers, to make the ring buffer * lockless. */ struct buffer_page { struct list_head list; /* list of buffer pages */ local_t write; /* index for next write */ unsigned read; /* index for next read */ local_t entries; /* entries on this page */ unsigned long real_end; /* real end of data */ struct buffer_data_page *page; /* Actual data page */ }; /* * The buffer page counters, write and entries, must be reset * atomically when crossing page boundaries. To synchronize this * update, two counters are inserted into the number. One is * the actual counter for the write position or count on the page. * * The other is a counter of updaters. Before an update happens * the update partition of the counter is incremented. This will * allow the updater to update the counter atomically. * * The counter is 20 bits, and the state data is 12. */ #define RB_WRITE_MASK 0xfffff #define RB_WRITE_INTCNT (1 << 20) static void rb_init_page(struct buffer_data_page *bpage) { local_set(&bpage->commit, 0); } /** * ring_buffer_page_len - the size of data on the page. * @page: The page to read * * Returns the amount of data on the page, including buffer page header. */ size_t ring_buffer_page_len(void *page) { return local_read(&((struct buffer_data_page *)page)->commit) + BUF_PAGE_HDR_SIZE; } /* * Also stolen from mm/slob.c. Thanks to Mathieu Desnoyers for pointing * this issue out. */ static void free_buffer_page(struct buffer_page *bpage) { free_page((unsigned long)bpage->page); kfree(bpage); } /* * We need to fit the time_stamp delta into 27 bits. */ static inline int test_time_stamp(u64 delta) { if (delta & TS_DELTA_TEST) return 1; return 0; } #define BUF_PAGE_SIZE (PAGE_SIZE - BUF_PAGE_HDR_SIZE) /* Max payload is BUF_PAGE_SIZE - header (8bytes) */ #define BUF_MAX_DATA_SIZE (BUF_PAGE_SIZE - (sizeof(u32) * 2)) /* Max number of timestamps that can fit on a page */ #define RB_TIMESTAMPS_PER_PAGE (BUF_PAGE_SIZE / RB_LEN_TIME_STAMP) int ring_buffer_print_page_header(struct trace_seq *s) { struct buffer_data_page field; int ret; ret = trace_seq_printf(s, "\tfield: u64 timestamp;\t" "offset:0;\tsize:%u;\tsigned:%u;\n", (unsigned int)sizeof(field.time_stamp), (unsigned int)is_signed_type(u64)); ret = trace_seq_printf(s, "\tfield: local_t commit;\t" "offset:%u;\tsize:%u;\tsigned:%u;\n", (unsigned int)offsetof(typeof(field), commit), (unsigned int)sizeof(field.commit), (unsigned int)is_signed_type(long)); ret = trace_seq_printf(s, "\tfield: int overwrite;\t" "offset:%u;\tsize:%u;\tsigned:%u;\n", (unsigned int)offsetof(typeof(field), commit), 1, (unsigned int)is_signed_type(long)); ret = trace_seq_printf(s, "\tfield: char data;\t" "offset:%u;\tsize:%u;\tsigned:%u;\n", (unsigned int)offsetof(typeof(field), data), (unsigned int)BUF_PAGE_SIZE, (unsigned int)is_signed_type(char)); return ret; } /* * head_page == tail_page && head == tail then buffer is empty. */ struct ring_buffer_per_cpu { int cpu; atomic_t record_disabled; struct ring_buffer *buffer; spinlock_t reader_lock; /* serialize readers */ arch_spinlock_t lock; struct lock_class_key lock_key; struct list_head *pages; struct buffer_page *head_page; /* read from head */ struct buffer_page *tail_page; /* write to tail */ struct buffer_page *commit_page; /* committed pages */ struct buffer_page *reader_page; unsigned long lost_events; unsigned long last_overrun; local_t commit_overrun; local_t overrun; local_t entries; local_t committing; local_t commits; unsigned long read; u64 write_stamp; u64 read_stamp; }; struct ring_buffer { unsigned pages; unsigned flags; int cpus; atomic_t record_disabled; cpumask_var_t cpumask; struct lock_class_key *reader_lock_key; struct mutex mutex; struct ring_buffer_per_cpu **buffers; #ifdef CONFIG_HOTPLUG_CPU struct notifier_block cpu_notify; #endif u64 (*clock)(void); }; struct ring_buffer_iter { struct ring_buffer_per_cpu *cpu_buffer; unsigned long head; struct buffer_page *head_page; struct buffer_page *cache_reader_page; unsigned long cache_read; u64 read_stamp; }; /* buffer may be either ring_buffer or ring_buffer_per_cpu */ #define RB_WARN_ON(b, cond) \ ({ \ int _____ret = unlikely(cond); \ if (_____ret) { \