/* * Generic ring buffer * * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com> */#include <linux/ring_buffer.h>#include <linux/trace_clock.h>#include <linux/ftrace_irq.h>#include <linux/spinlock.h>#include <linux/debugfs.h>#include <linux/uaccess.h>#include <linux/hardirq.h>#include <linux/module.h>#include <linux/percpu.h>#include <linux/mutex.h>#include <linux/init.h>#include <linux/hash.h>#include <linux/list.h>#include <linux/cpu.h>#include <linux/fs.h> #include "trace.h" /* * The ring buffer is made up of a list of pages. A separate list of pages is * allocated for each CPU. A writer may only write to a buffer that is * associated with the CPU it is currently executing on.  A reader may read * from any per cpu buffer. * * The reader is special. For each per cpu buffer, the reader has its own * reader page. When a reader has read the entire reader page, this reader * page is swapped with another page in the ring buffer. * * Now, as long as the writer is off the reader page, the reader can do what * ever it wants with that page. The writer will never write to that page * again (as long as it is out of the ring buffer). * * Here's some silly ASCII art. * *   +------+ *   |reader|          RING BUFFER *   |page  | *   +------+        +---+   +---+   +---+ *                   |   |-->|   |-->|   | *                   +---+   +---+   +---+ *                     ^               | *                     |               | *                     +---------------+ * * *   +------+ *   |reader|          RING BUFFER *   |page  |------------------v *   +------+        +---+   +---+   +---+ *                   |   |-->|   |-->|   | *                   +---+   +---+   +---+ *                     ^               | *                     |               | *                     +---------------+ * * *   +------+ *   |reader|          RING BUFFER *   |page  |------------------v *   +------+        +---+   +---+   +---+ *      ^            |   |-->|   |-->|   | *      |            +---+   +---+   +---+ *      |                              | *      |                              | *      +------------------------------+ * * *   +------+ *   |buffer|          RING BUFFER *   |page  |------------------v *   +------+        +---+   +---+   +---+ *      ^            |   |   |   |-->|   | *      |   New      +---+   +---+   +---+ *      |  Reader------^               | *      |   page                       | *      +------------------------------+ * * * After we make this swap, the reader can hand this page off to the splice * code and be done with it. It can even allocate a new page if it needs to * and swap that into the ring buffer. * * We will be using cmpxchg soon to make all this lockless. * */ /* * A fast way to enable or disable all ring buffers is to * call tracing_on or tracing_off. Turning off the ring buffers * prevents all ring buffers from being recorded to. * Turning this switch on, makes it OK to write to the * ring buffer, if the ring buffer is enabled itself. * * There's three layers that must be on in order to write * to the ring buffer. * * 1) This global flag must be set. * 2) The ring buffer must be enabled for recording. * 3) The per cpu buffer must be enabled for recording. * * In case of an anomaly, this global flag has a bit set that * will permantly disable all ring buffers. */ /* * Global flag to disable all recording to ring buffers *  This has two bits: ON, DISABLED * *  ON   DISABLED * ---- ---------- *   0      0        : ring buffers are off *   1      0        : ring buffers are on *   X      1        : ring buffers are permanently disabled */ enum {	RB_BUFFERS_ON_BIT	= 0,	RB_BUFFERS_DISABLED_BIT	= 1,}; enum {	RB_BUFFERS_ON		= 1 << RB_BUFFERS_ON_BIT,	RB_BUFFERS_DISABLED	= 1 << RB_BUFFERS_DISABLED_BIT,}; static unsigned long ring_buffer_flags __read_mostly = RB_BUFFERS_ON; #define BUF_PAGE_HDR_SIZE offsetof(struct buffer_data_page, data) /** * tracing_on - enable all tracing buffers * * This function enables all tracing buffers that may have been * disabled with tracing_off. */void tracing_on(void){	set_bit(RB_BUFFERS_ON_BIT, &ring_buffer_flags);}EXPORT_SYMBOL_GPL(tracing_on); /** * tracing_off - turn off all tracing buffers * * This function stops all tracing buffers from recording data. * It does not disable any overhead the tracers themselves may * be causing. This function simply causes all recording to * the ring buffers to fail. */void tracing_off(void){	clear_bit(RB_BUFFERS_ON_BIT, &ring_buffer_flags);}EXPORT_SYMBOL_GPL(tracing_off); /** * tracing_off_permanent - permanently disable ring buffers * * This function, once called, will disable all ring buffers * permanently. */void tracing_off_permanent(void){	set_bit(RB_BUFFERS_DISABLED_BIT, &ring_buffer_flags);} /** * tracing_is_on - show state of ring buffers enabled */int tracing_is_on(void){	return ring_buffer_flags == RB_BUFFERS_ON;}EXPORT_SYMBOL_GPL(tracing_is_on); #include "trace.h" #define RB_EVNT_HDR_SIZE (offsetof(struct ring_buffer_event, array))#define RB_ALIGNMENT		4U#define RB_MAX_SMALL_DATA	28 enum {	RB_LEN_TIME_EXTEND = 8,	RB_LEN_TIME_STAMP = 16,}; static inline int rb_null_event(struct ring_buffer_event *event){	return event->type == RINGBUF_TYPE_PADDING && event->time_delta == 0;} static inline int rb_discarded_event(struct ring_buffer_event *event){	return event->type == RINGBUF_TYPE_PADDING && event->time_delta;} static void rb_event_set_padding(struct ring_buffer_event *event){	event->type = RINGBUF_TYPE_PADDING;	event->time_delta = 0;} /** * ring_buffer_event_discard - discard an event in the ring buffer * @buffer: the ring buffer * @event: the event to discard * * Sometimes a event that is in the ring buffer needs to be ignored. * This function lets the user discard an event in the ring buffer * and then that event will not be read later. * * Note, it is up to the user to be careful with this, and protect * against races. If the user discards an event that has been consumed * it is possible that it could corrupt the ring buffer. */void ring_buffer_event_discard(struct ring_buffer_event *event){	event->type = RINGBUF_TYPE_PADDING;	/* time delta must be non zero */	if (!event->time_delta)		event->time_delta = 1;} static unsignedrb_event_data_length(struct ring_buffer_event *event){	unsigned length; 	if (event->len)		length = event->len * RB_ALIGNMENT;	else		length = event->array[0];	return length + RB_EVNT_HDR_SIZE;} /* inline for ring buffer fast paths */static unsignedrb_event_length(struct ring_buffer_event *event){	switch (event->type) {	case RINGBUF_TYPE_PADDING:		if (rb_null_event(event))			/* undefined */			return -1;		return rb_event_data_length(event); 	case RINGBUF_TYPE_TIME_EXTEND:		return RB_LEN_TIME_EXTEND; 	case RINGBUF_TYPE_TIME_STAMP:		return RB_LEN_TIME_STAMP; 	case RINGBUF_TYPE_DATA:		return rb_event_data_length(event);	default:		BUG();	}	/* not hit */	return 0;} /** * ring_buffer_event_length - return the length of the event * @event: the event to get the length of */unsigned ring_buffer_event_length(struct ring_buffer_event *event){	unsigned length = rb_event_length(event);	if (event->type != RINGBUF_TYPE_DATA)		return length;	length -= RB_EVNT_HDR_SIZE;	if (length > RB_MAX_SMALL_DATA + sizeof(event->array[0]))                length -= sizeof(event->array[0]);	return length;}EXPORT_SYMBOL_GPL(ring_buffer_event_length); /* inline for ring buffer fast paths */static void *rb_event_data(struct ring_buffer_event *event){	BUG_ON(event->type != RINGBUF_TYPE_DATA);	/* If length is in len field, then array[0] has the data */	if (event->len)		return (void *)&event->array[0];	/* Otherwise length is in array[0] and array[1] has the data */	return (void *)&event->array[1];} /** * ring_buffer_event_data - return the data of the event * @event: the event to get the data from */void *ring_buffer_event_data(struct ring_buffer_event *event){	return rb_event_data(event);}EXPORT_SYMBOL_GPL(ring_buffer_event_data); #define for_each_buffer_cpu(buffer, cpu)		\	for_each_cpu(cpu, buffer->cpumask) #define TS_SHIFT	27#define TS_MASK		((1ULL << TS_SHIFT) - 1)#define TS_DELTA_TEST	(~TS_MASK) struct buffer_data_page {	u64		 time_stamp;	/* page time stamp */	local_t		 commit;	/* write committed index */	unsigned char	 data[];	/* data of buffer page */}; struct buffer_page {	local_t		 write;		/* index for next write */	unsigned	 read;		/* index for next read */	struct list_head list;		/* list of free pages */	struct buffer_data_page *page;	/* Actual data page */}; static void rb_init_page(struct buffer_data_page *bpage){	local_set(&bpage->commit, 0);} /** * ring_buffer_page_len - the size of data on the page. * @page: The page to read * * Returns the amount of data on the page, including buffer page header. */size_t ring_buffer_page_len(void *page){	return local_read(&((struct buffer_data_page *)page)->commit)		+ BUF_PAGE_HDR_SIZE;} /* * Also stolen from mm/slob.c. Thanks to Mathieu Desnoyers for pointing * this issue out. */static void free_buffer_page(struct buffer_page *bpage){	free_page((unsigned long)bpage->page);	kfree(bpage);} /* * We need to fit the time_stamp delta into 27 bits. */static inline int test_time_stamp(u64 delta){	if (delta & TS_DELTA_TEST)		return 1;	return 0;} #define BUF_PAGE_SIZE (PAGE_SIZE - BUF_PAGE_HDR_SIZE) /* * head_page == tail_page && head == tail then buffer is empty. */struct ring_buffer_per_cpu {	int				cpu;	struct ring_buffer		*buffer;	spinlock_t			reader_lock; /* serialize readers */	raw_spinlock_t			lock;	struct lock_class_key		lock_key;	struct list_head		pages;	struct buffer_page		*head_page;	/* read from head */	struct buffer_page		*tail_page;	/* write to tail */	struct buffer_page		*commit_page;	/* committed pages */	struct buffer_page		*reader_page;	unsigned long			overrun;	unsigned long			entries;	u64				write_stamp;	u64				read_stamp;	atomic_t			record_disabled;}; struct ring_buffer {	unsigned			pages;	unsigned			flags;	int				cpus;	atomic_t			record_disabled;	cpumask_var_t			cpumask; 	struct mutex			mutex; 	struct ring_buffer_per_cpu	**buffers; #ifdef CONFIG_HOTPLUG_CPU	struct notifier_block		cpu_notify;#endif	u64				(*clock)(void);}; struct ring_buffer_iter {	struct ring_buffer_per_cpu	*cpu_buffer;	unsigned long			head;	struct buffer_page		*head_page;	u64				read_stamp;}; /* buffer may be either ring_buffer or ring_buffer_per_cpu */#define RB_WARN_ON(buffer, cond)				\	({							\		int _____ret = unlikely(cond);			\		if (_____ret) {					\			atomic_inc(&buffer->record_disabled);	\			WARN_ON(1);				\		}						\		_____ret;					\	}) /* Up this if you want to test the TIME_EXTENTS and normalization */#define DEBUG_SHIFT 0 u64 ring_buffer_time_stamp(struct ring_buffer *buffer, int cpu){	u64 time; 	preempt_disable_notrace();	/* shift to debug/test normalization and TIME_EXTENTS */	time = buffer->clock() << DEBUG_SHIFT;	preempt_enable_no_resched_notrace(); 	return time;}EXPORT_SYMBOL_GPL(ring_buffer_time_stamp); void ring_buffer_normalize_time_stamp(struct ring_buffer *buffer,				      int cpu, u64 *ts){	/* Just stupid testing the normalize function and deltas */	*ts >>= DEBUG_SHIFT;}EXPORT_SYMBOL_GPL(ring_buffer_normalize_time_stamp); /** * check_pages - integrity check of buffer pages * @cpu_buffer: CPU buffer with pages to test * * As a safety measure we check to make sure the data pages have not * been corrupted. */static int rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer){	struct list_head *head = &cpu_buffer->pages;	struct buffer_page *bpage, *tmp; 	if (RB_WARN_ON(cpu_buffer, head->next->prev != head))		return -1;	if (RB_WARN_ON(cpu_buffer, head->prev->next != head))		return -1; 	list_for_each_entry_safe(bpage, tmp, head, list) {		if (RB_WARN_ON(cpu_buffer,			       bpage->list.next->prev != &bpage->list))			return -1;		if (RB_WARN_ON(cpu_buffer,			       bpage->list.prev->next != &bpage->list))			return -1;	} 	return 0;} static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer,			     unsigned nr_pages){	struct list_head *head = &cpu_buffer->pages;	struct buffer_page *bpage, *tmp;	unsigned long addr;	LIST_HEAD(pages);	unsigned i; 	for (i = 0; i < nr_pages; i++) {		bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),				    GFP_KERNEL, cpu_to_node(cpu_buffer->cpu));		if (!bpage)			goto free_pages;		list_add(&bpage->list, &pages); 		addr = __get_free_page(GFP_KERNEL);		if (!addr)			goto free_pages;		bpage->page = (void *)addr;		rb_init_page(bpage->page);	} 	list_splice(&pages, head); 	rb_check_pages(cpu_buffer); 	return 0;

Lines 1–500 of 2899. Load more