// SPDX-License-Identifier: GPL-2.0/* * Generic ring buffer * * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com> */#include <linux/trace_recursion.h>#include <linux/trace_events.h>#include <linux/ring_buffer.h>#include <linux/trace_clock.h>#include <linux/sched/clock.h>#include <linux/cacheflush.h>#include <linux/trace_seq.h>#include <linux/spinlock.h>#include <linux/irq_work.h>#include <linux/security.h>#include <linux/uaccess.h>#include <linux/hardirq.h>#include <linux/kthread.h>	/* for self test */#include <linux/module.h>#include <linux/percpu.h>#include <linux/mutex.h>#include <linux/delay.h>#include <linux/slab.h>#include <linux/init.h>#include <linux/hash.h>#include <linux/list.h>#include <linux/cpu.h>#include <linux/oom.h>#include <linux/mm.h> #include <asm/local64.h>#include <asm/local.h>#include <asm/setup.h> #include "trace.h" /* * The "absolute" timestamp in the buffer is only 59 bits. * If a clock has the 5 MSBs set, it needs to be saved and * reinserted. */#define TS_MSB		(0xf8ULL << 56)#define ABS_TS_MASK	(~TS_MSB) static void update_pages_handler(struct work_struct *work); #define RING_BUFFER_META_MAGIC	0xBADFEED struct ring_buffer_meta {	int		magic;	int		struct_sizes;	unsigned long	total_size;	unsigned long	buffers_offset;}; struct ring_buffer_cpu_meta {	unsigned long	first_buffer;	unsigned long	head_buffer;	unsigned long	commit_buffer;	__u32		subbuf_size;	__u32		nr_subbufs;	int		buffers[];}; /* * The ring buffer header is special. We must manually up keep it. */int ring_buffer_print_entry_header(struct trace_seq *s){	trace_seq_puts(s, "# compressed entry header\n");	trace_seq_puts(s, "\ttype_len    :    5 bits\n");	trace_seq_puts(s, "\ttime_delta  :   27 bits\n");	trace_seq_puts(s, "\tarray       :   32 bits\n");	trace_seq_putc(s, '\n');	trace_seq_printf(s, "\tpadding     : type == %d\n",			 RINGBUF_TYPE_PADDING);	trace_seq_printf(s, "\ttime_extend : type == %d\n",			 RINGBUF_TYPE_TIME_EXTEND);	trace_seq_printf(s, "\ttime_stamp : type == %d\n",			 RINGBUF_TYPE_TIME_STAMP);	trace_seq_printf(s, "\tdata max type_len  == %d\n",			 RINGBUF_TYPE_DATA_TYPE_LEN_MAX); 	return !trace_seq_has_overflowed(s);} /* * The ring buffer is made up of a list of pages. A separate list of pages is * allocated for each CPU. A writer may only write to a buffer that is * associated with the CPU it is currently executing on.  A reader may read * from any per cpu buffer. * * The reader is special. For each per cpu buffer, the reader has its own * reader page. When a reader has read the entire reader page, this reader * page is swapped with another page in the ring buffer. * * Now, as long as the writer is off the reader page, the reader can do what * ever it wants with that page. The writer will never write to that page * again (as long as it is out of the ring buffer). * * Here's some silly ASCII art. * *   +------+ *   |reader|          RING BUFFER *   |page  | *   +------+        +---+   +---+   +---+ *                   |   |-->|   |-->|   | *                   +---+   +---+   +---+ *                     ^               | *                     |               | *                     +---------------+ * * *   +------+ *   |reader|          RING BUFFER *   |page  |------------------v *   +------+        +---+   +---+   +---+ *                   |   |-->|   |-->|   | *                   +---+   +---+   +---+ *                     ^               | *                     |               | *                     +---------------+ * * *   +------+ *   |reader|          RING BUFFER *   |page  |------------------v *   +------+        +---+   +---+   +---+ *      ^            |   |-->|   |-->|   | *      |            +---+   +---+   +---+ *      |                              | *      |                              | *      +------------------------------+ * * *   +------+ *   |buffer|          RING BUFFER *   |page  |------------------v *   +------+        +---+   +---+   +---+ *      ^            |   |   |   |-->|   | *      |   New      +---+   +---+   +---+ *      |  Reader------^               | *      |   page                       | *      +------------------------------+ * * * After we make this swap, the reader can hand this page off to the splice * code and be done with it. It can even allocate a new page if it needs to * and swap that into the ring buffer. * * We will be using cmpxchg soon to make all this lockless. * */ /* Used for individual buffers (after the counter) */#define RB_BUFFER_OFF		(1 << 20) #define BUF_PAGE_HDR_SIZE offsetof(struct buffer_data_page, data) #define RB_EVNT_HDR_SIZE (offsetof(struct ring_buffer_event, array))#define RB_ALIGNMENT		4U#define RB_MAX_SMALL_DATA	(RB_ALIGNMENT * RINGBUF_TYPE_DATA_TYPE_LEN_MAX)#define RB_EVNT_MIN_SIZE	8U	/* two 32bit words */ #ifndef CONFIG_HAVE_64BIT_ALIGNED_ACCESS# define RB_FORCE_8BYTE_ALIGNMENT	0# define RB_ARCH_ALIGNMENT		RB_ALIGNMENT#else# define RB_FORCE_8BYTE_ALIGNMENT	1# define RB_ARCH_ALIGNMENT		8U#endif #define RB_ALIGN_DATA		__aligned(RB_ARCH_ALIGNMENT) /* define RINGBUF_TYPE_DATA for 'case RINGBUF_TYPE_DATA:' */#define RINGBUF_TYPE_DATA 0 ... RINGBUF_TYPE_DATA_TYPE_LEN_MAX enum {	RB_LEN_TIME_EXTEND = 8,	RB_LEN_TIME_STAMP =  8,}; #define skip_time_extend(event) \	((struct ring_buffer_event *)((char *)event + RB_LEN_TIME_EXTEND)) #define extended_time(event) \	(event->type_len >= RINGBUF_TYPE_TIME_EXTEND) static inline bool rb_null_event(struct ring_buffer_event *event){	return event->type_len == RINGBUF_TYPE_PADDING && !event->time_delta;} static void rb_event_set_padding(struct ring_buffer_event *event){	/* padding has a NULL time_delta */	event->type_len = RINGBUF_TYPE_PADDING;	event->time_delta = 0;} static unsignedrb_event_data_length(struct ring_buffer_event *event){	unsigned length; 	if (event->type_len)		length = event->type_len * RB_ALIGNMENT;	else		length = event->array[0];	return length + RB_EVNT_HDR_SIZE;} /* * Return the length of the given event. Will return * the length of the time extend if the event is a * time extend. */static inline unsignedrb_event_length(struct ring_buffer_event *event){	switch (event->type_len) {	case RINGBUF_TYPE_PADDING:		if (rb_null_event(event))			/* undefined */			return -1;		return  event->array[0] + RB_EVNT_HDR_SIZE; 	case RINGBUF_TYPE_TIME_EXTEND:		return RB_LEN_TIME_EXTEND; 	case RINGBUF_TYPE_TIME_STAMP:		return RB_LEN_TIME_STAMP; 	case RINGBUF_TYPE_DATA:		return rb_event_data_length(event);	default:		WARN_ON_ONCE(1);	}	/* not hit */	return 0;} /* * Return total length of time extend and data, *   or just the event length for all other events. */static inline unsignedrb_event_ts_length(struct ring_buffer_event *event){	unsigned len = 0; 	if (extended_time(event)) {		/* time extends include the data event after it */		len = RB_LEN_TIME_EXTEND;		event = skip_time_extend(event);	}	return len + rb_event_length(event);} /** * ring_buffer_event_length - return the length of the event * @event: the event to get the length of * * Returns the size of the data load of a data event. * If the event is something other than a data event, it * returns the size of the event itself. With the exception * of a TIME EXTEND, where it still returns the size of the * data load of the data event after it. */unsigned ring_buffer_event_length(struct ring_buffer_event *event){	unsigned length; 	if (extended_time(event))		event = skip_time_extend(event); 	length = rb_event_length(event);	if (event->type_len > RINGBUF_TYPE_DATA_TYPE_LEN_MAX)		return length;	length -= RB_EVNT_HDR_SIZE;	if (length > RB_MAX_SMALL_DATA + sizeof(event->array[0]))                length -= sizeof(event->array[0]);	return length;}EXPORT_SYMBOL_GPL(ring_buffer_event_length); /* inline for ring buffer fast paths */static __always_inline void *rb_event_data(struct ring_buffer_event *event){	if (extended_time(event))		event = skip_time_extend(event);	WARN_ON_ONCE(event->type_len > RINGBUF_TYPE_DATA_TYPE_LEN_MAX);	/* If length is in len field, then array[0] has the data */	if (event->type_len)		return (void *)&event->array[0];	/* Otherwise length is in array[0] and array[1] has the data */	return (void *)&event->array[1];} /** * ring_buffer_event_data - return the data of the event * @event: the event to get the data from */void *ring_buffer_event_data(struct ring_buffer_event *event){	return rb_event_data(event);}EXPORT_SYMBOL_GPL(ring_buffer_event_data); #define for_each_buffer_cpu(buffer, cpu)		\	for_each_cpu(cpu, buffer->cpumask) #define for_each_online_buffer_cpu(buffer, cpu)		\	for_each_cpu_and(cpu, buffer->cpumask, cpu_online_mask) #define TS_SHIFT	27#define TS_MASK		((1ULL << TS_SHIFT) - 1)#define TS_DELTA_TEST	(~TS_MASK) static u64 rb_event_time_stamp(struct ring_buffer_event *event){	u64 ts; 	ts = event->array[0];	ts <<= TS_SHIFT;	ts += event->time_delta; 	return ts;} /* Flag when events were overwritten */#define RB_MISSED_EVENTS	(1 << 31)/* Missed count stored at end */#define RB_MISSED_STORED	(1 << 30) #define RB_MISSED_MASK		(3 << 30) struct buffer_data_page {	u64		 time_stamp;	/* page time stamp */	local_t		 commit;	/* write committed index */	unsigned char	 data[] RB_ALIGN_DATA;	/* data of buffer page */}; struct buffer_data_read_page {	unsigned		order;	/* order of the page */	struct buffer_data_page	*data;	/* actual data, stored in this page */}; /* * Note, the buffer_page list must be first. The buffer pages * are allocated in cache lines, which means that each buffer * page will be at the beginning of a cache line, and thus * the least significant bits will be zero. We use this to * add flags in the list struct pointers, to make the ring buffer * lockless. */struct buffer_page {	struct list_head list;		/* list of buffer pages */	local_t		 write;		/* index for next write */	unsigned	 read;		/* index for next read */	local_t		 entries;	/* entries on this page */	unsigned long	 real_end;	/* real end of data */	unsigned	 order;		/* order of the page */	u32		 id:30;		/* ID for external mapping */	u32		 range:1;	/* Mapped via a range */	struct buffer_data_page *page;	/* Actual data page */}; /* * The buffer page counters, write and entries, must be reset * atomically when crossing page boundaries. To synchronize this * update, two counters are inserted into the number. One is * the actual counter for the write position or count on the page. * * The other is a counter of updaters. Before an update happens * the update partition of the counter is incremented. This will * allow the updater to update the counter atomically. * * The counter is 20 bits, and the state data is 12. */#define RB_WRITE_MASK		0xfffff#define RB_WRITE_INTCNT		(1 << 20) static void rb_init_page(struct buffer_data_page *bpage){	local_set(&bpage->commit, 0);} static __always_inline unsigned int rb_page_commit(struct buffer_page *bpage){	return local_read(&bpage->page->commit);} static void free_buffer_page(struct buffer_page *bpage){	/* Range pages are not to be freed */	if (!bpage->range)		free_pages((unsigned long)bpage->page, bpage->order);	kfree(bpage);} /* * For best performance, allocate cpu buffer data cache line sized * and per CPU. */#define alloc_cpu_buffer(cpu) (struct ring_buffer_per_cpu *)		\	kzalloc_node(ALIGN(sizeof(struct ring_buffer_per_cpu),		\			   cache_line_size()), GFP_KERNEL, cpu_to_node(cpu)); #define alloc_cpu_page(cpu) (struct buffer_page *)			\	kzalloc_node(ALIGN(sizeof(struct buffer_page),			\			   cache_line_size()), GFP_KERNEL, cpu_to_node(cpu)); static struct buffer_data_page *alloc_cpu_data(int cpu, int order){	struct buffer_data_page *dpage;	struct page *page;	gfp_t mflags; 	/*	 * __GFP_RETRY_MAYFAIL flag makes sure that the allocation fails	 * gracefully without invoking oom-killer and the system is not	 * destabilized.	 */	mflags = GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_COMP | __GFP_ZERO; 	page = alloc_pages_node(cpu_to_node(cpu), mflags, order);	if (!page)		return NULL; 	dpage = page_address(page);	rb_init_page(dpage); 	return dpage;} /* * We need to fit the time_stamp delta into 27 bits. */static inline bool test_time_stamp(u64 delta){	return !!(delta & TS_DELTA_TEST);} struct rb_irq_work {	struct irq_work			work;	wait_queue_head_t		waiters;	wait_queue_head_t		full_waiters;	atomic_t			seq;	bool				waiters_pending;	bool				full_waiters_pending;	bool				wakeup_full;}; /* * Structure to hold event state and handle nested events. */struct rb_event_info {	u64			ts;	u64			delta;	u64			before;	u64			after;	unsigned long		length;	struct buffer_page	*tail_page;	int			add_timestamp;}; /* * Used for the add_timestamp *  NONE *  EXTEND - wants a time extend *  ABSOLUTE - the buffer requests all events to have absolute time stamps *  FORCE - force a full time stamp. */enum {	RB_ADD_STAMP_NONE		= 0,	RB_ADD_STAMP_EXTEND		= BIT(1),	RB_ADD_STAMP_ABSOLUTE		= BIT(2),	RB_ADD_STAMP_FORCE		= BIT(3)};/* * Used for which event context the event is in. *  TRANSITION = 0 *  NMI     = 1 *  IRQ     = 2 *  SOFTIRQ = 3 *  NORMAL  = 4 * * See trace_recursive_lock() comment below for more details. */enum {	RB_CTX_TRANSITION,	RB_CTX_NMI,	RB_CTX_IRQ,	RB_CTX_SOFTIRQ,	RB_CTX_NORMAL,	RB_CTX_MAX};

Lines 1–500 of 7785. Load more