*/struct ring_buffer_per_cpu {	int				cpu;	atomic_t			record_disabled;	atomic_t			resize_disabled;	struct trace_buffer	*buffer;	raw_spinlock_t			reader_lock;	/* serialize readers */	arch_spinlock_t			lock;	struct lock_class_key		lock_key;	struct buffer_data_page		*free_page;	unsigned long			nr_pages;	unsigned int			current_context;	struct list_head		*pages;	/* pages generation counter, incremented when the list changes */	unsigned long			cnt;	struct buffer_page		*head_page;	/* read from head */	struct buffer_page		*tail_page;	/* write to tail */	struct buffer_page		*commit_page;	/* committed pages */	struct buffer_page		*reader_page;	unsigned long			lost_events;	unsigned long			last_overrun;	unsigned long			nest;	local_t				entries_bytes;	local_t				entries;	local_t				overrun;	local_t				commit_overrun;	local_t				dropped_events;	local_t				committing;	local_t				commits;	local_t				pages_touched;	local_t				pages_lost;	local_t				pages_read;	long				last_pages_touch;	size_t				shortest_full;	unsigned long			read;	unsigned long			read_bytes;	rb_time_t			write_stamp;	rb_time_t			before_stamp;	u64				event_stamp[MAX_NEST];	u64				read_stamp;	/* pages removed since last reset */	unsigned long			pages_removed; 	unsigned int			mapped;	unsigned int			user_mapped;	/* user space mapping */	struct mutex			mapping_lock;	struct buffer_page		**subbuf_ids;	/* ID to subbuf VA */	struct trace_buffer_meta	*meta_page;	struct ring_buffer_cpu_meta	*ring_meta; 	struct ring_buffer_remote	*remote; 	/* ring buffer pages to update, > 0 to add, < 0 to remove */	long				nr_pages_to_update;	struct list_head		new_pages; /* new pages to add */	struct work_struct		update_pages_work;	struct completion		update_done; 	struct rb_irq_work		irq_work;}; struct trace_buffer {	unsigned			flags;	int				cpus;	atomic_t			record_disabled;	atomic_t			resizing;	cpumask_var_t			cpumask; 	struct lock_class_key		*reader_lock_key; 	struct mutex			mutex; 	struct ring_buffer_per_cpu	**buffers; 	struct ring_buffer_remote	*remote; 	struct hlist_node		node;	u64				(*clock)(void); 	struct rb_irq_work		irq_work;	bool				time_stamp_abs; 	unsigned long			range_addr_start;	unsigned long			range_addr_end;	struct notifier_block		flush_nb; 	struct ring_buffer_meta		*meta; 	unsigned int			subbuf_size;	unsigned int			subbuf_order;	unsigned int			max_data_size;}; struct ring_buffer_iter {	struct ring_buffer_per_cpu	*cpu_buffer;	unsigned long			head;	unsigned long			next_event;	struct buffer_page		*head_page;	struct buffer_page		*cache_reader_page;	unsigned long			cache_read;	unsigned long			cache_pages_removed;	u64				read_stamp;	u64				page_stamp;	struct ring_buffer_event	*event;	size_t				event_size;	int				missed_events;}; int ring_buffer_print_page_header(struct trace_buffer *buffer, struct trace_seq *s){	struct buffer_data_page field; 	trace_seq_printf(s, "\tfield: u64 timestamp;\t"			 "offset:0;\tsize:%u;\tsigned:%u;\n",			 (unsigned int)sizeof(field.time_stamp),			 (unsigned int)is_signed_type(u64)); 	trace_seq_printf(s, "\tfield: local_t commit;\t"			 "offset:%u;\tsize:%u;\tsigned:%u;\n",			 (unsigned int)offsetof(typeof(field), commit),			 (unsigned int)sizeof(field.commit),			 (unsigned int)is_signed_type(long)); 	trace_seq_printf(s, "\tfield: char overwrite;\t"			 "offset:%u;\tsize:%u;\tsigned:%u;\n",			 (unsigned int)offsetof(typeof(field), commit),			 1,			 (unsigned int)is_signed_type(char)); 	trace_seq_printf(s, "\tfield: char data;\t"			 "offset:%u;\tsize:%u;\tsigned:%u;\n",			 (unsigned int)offsetof(typeof(field), data),			 (unsigned int)(buffer ? buffer->subbuf_size :						 PAGE_SIZE - BUF_PAGE_HDR_SIZE),			 (unsigned int)is_signed_type(char)); 	return !trace_seq_has_overflowed(s);} static inline void rb_time_read(rb_time_t *t, u64 *ret){	*ret = local64_read(&t->time);}static void rb_time_set(rb_time_t *t, u64 val){	local64_set(&t->time, val);} /* * Enable this to make sure that the event passed to * ring_buffer_event_time_stamp() is not committed and also * is on the buffer that it passed in. *///#define RB_VERIFY_EVENT#ifdef RB_VERIFY_EVENTstatic struct list_head *rb_list_head(struct list_head *list);static void verify_event(struct ring_buffer_per_cpu *cpu_buffer,			 void *event){	struct buffer_page *page = cpu_buffer->commit_page;	struct buffer_page *tail_page = READ_ONCE(cpu_buffer->tail_page);	struct list_head *next;	long commit, write;	unsigned long addr = (unsigned long)event;	bool done = false;	int stop = 0; 	/* Make sure the event exists and is not committed yet */	do {		if (page == tail_page || WARN_ON_ONCE(stop++ > 100))			done = true;		commit = rb_page_commit(page);		write = local_read(&page->write);		if (addr >= (unsigned long)&page->page->data[commit] &&		    addr < (unsigned long)&page->page->data[write])			return; 		next = rb_list_head(page->list.next);		page = list_entry(next, struct buffer_page, list);	} while (!done);	WARN_ON_ONCE(1);}#elsestatic inline void verify_event(struct ring_buffer_per_cpu *cpu_buffer,			 void *event){}#endif /* * The absolute time stamp drops the 5 MSBs and some clocks may * require them. The rb_fix_abs_ts() will take a previous full * time stamp, and add the 5 MSB of that time stamp on to the * saved absolute time stamp. Then they are compared in case of * the unlikely event that the latest time stamp incremented * the 5 MSB. */static inline u64 rb_fix_abs_ts(u64 abs, u64 save_ts){	if (save_ts & TS_MSB) {		abs |= save_ts & TS_MSB;		/* Check for overflow */		if (unlikely(abs < save_ts))			abs += 1ULL << 59;	}	return abs;} static inline u64 rb_time_stamp(struct trace_buffer *buffer); /** * ring_buffer_event_time_stamp - return the event's current time stamp * @buffer: The buffer that the event is on * @event: the event to get the time stamp of * * Note, this must be called after @event is reserved, and before it is * committed to the ring buffer. And must be called from the same * context where the event was reserved (normal, softirq, irq, etc). * * Returns the time stamp associated with the current event. * If the event has an extended time stamp, then that is used as * the time stamp to return. * In the highly unlikely case that the event was nested more than * the max nesting, then the write_stamp of the buffer is returned, * otherwise  current time is returned, but that really neither of * the last two cases should ever happen. */u64 ring_buffer_event_time_stamp(struct trace_buffer *buffer,				 struct ring_buffer_event *event){	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[smp_processor_id()];	unsigned int nest;	u64 ts; 	/* If the event includes an absolute time, then just use that */	if (event->type_len == RINGBUF_TYPE_TIME_STAMP) {		ts = rb_event_time_stamp(event);		return rb_fix_abs_ts(ts, cpu_buffer->tail_page->page->time_stamp);	} 	nest = local_read(&cpu_buffer->committing);	verify_event(cpu_buffer, event);	if (WARN_ON_ONCE(!nest))		goto fail; 	/* Read the current saved nesting level time stamp */	if (likely(--nest < MAX_NEST))		return cpu_buffer->event_stamp[nest]; 	/* Shouldn't happen, warn if it does */	WARN_ONCE(1, "nest (%d) greater than max", nest);  fail:	rb_time_read(&cpu_buffer->write_stamp, &ts); 	return ts;} /** * ring_buffer_nr_dirty_pages - get the number of used pages in the ring buffer * @buffer: The ring_buffer to get the number of pages from * @cpu: The cpu of the ring_buffer to get the number of pages from * * Returns the number of pages that have content in the ring buffer. */size_t ring_buffer_nr_dirty_pages(struct trace_buffer *buffer, int cpu){	size_t read;	size_t lost;	size_t cnt; 	read = local_read(&buffer->buffers[cpu]->pages_read);	lost = local_read(&buffer->buffers[cpu]->pages_lost);	cnt = local_read(&buffer->buffers[cpu]->pages_touched); 	if (WARN_ON_ONCE(cnt < lost))		return 0; 	cnt -= lost; 	/* The reader can read an empty page, but not more than that */	if (cnt < read) {		WARN_ON_ONCE(read > cnt + 1);		return 0;	} 	return cnt - read;} static __always_inline bool full_hit(struct trace_buffer *buffer, int cpu, int full){	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];	size_t nr_pages;	size_t dirty; 	nr_pages = cpu_buffer->nr_pages;	if (!nr_pages || !full)		return true; 	/*	 * Add one as dirty will never equal nr_pages, as the sub-buffer	 * that the writer is on is not counted as dirty.	 * This is needed if "buffer_percent" is set to 100.	 */	dirty = ring_buffer_nr_dirty_pages(buffer, cpu) + 1; 	return (dirty * 100) >= (full * nr_pages);} /* * rb_wake_up_waiters - wake up tasks waiting for ring buffer input * * Schedules a delayed work to wake up any task that is blocked on the * ring buffer waiters queue. */static void rb_wake_up_waiters(struct irq_work *work){	struct rb_irq_work *rbwork = container_of(work, struct rb_irq_work, work); 	/* For waiters waiting for the first wake up */	(void)atomic_fetch_inc_release(&rbwork->seq); 	wake_up_all(&rbwork->waiters);	if (rbwork->full_waiters_pending || rbwork->wakeup_full) {		/* Only cpu_buffer sets the above flags */		struct ring_buffer_per_cpu *cpu_buffer =			container_of(rbwork, struct ring_buffer_per_cpu, irq_work); 		/* Called from interrupt context */		raw_spin_lock(&cpu_buffer->reader_lock);		rbwork->wakeup_full = false;		rbwork->full_waiters_pending = false; 		/* Waking up all waiters, they will reset the shortest full */		cpu_buffer->shortest_full = 0;		raw_spin_unlock(&cpu_buffer->reader_lock); 		wake_up_all(&rbwork->full_waiters);	}} /** * ring_buffer_wake_waiters - wake up any waiters on this ring buffer * @buffer: The ring buffer to wake waiters on * @cpu: The CPU buffer to wake waiters on * * In the case of a file that represents a ring buffer is closing, * it is prudent to wake up any waiters that are on this. */void ring_buffer_wake_waiters(struct trace_buffer *buffer, int cpu){	struct ring_buffer_per_cpu *cpu_buffer;	struct rb_irq_work *rbwork; 	if (!buffer)		return; 	if (cpu == RING_BUFFER_ALL_CPUS) { 		/* Wake up individual ones too. One level recursion */		for_each_buffer_cpu(buffer, cpu)			ring_buffer_wake_waiters(buffer, cpu); 		rbwork = &buffer->irq_work;	} else {		if (WARN_ON_ONCE(!buffer->buffers))			return;		if (WARN_ON_ONCE(cpu >= nr_cpu_ids))			return; 		cpu_buffer = buffer->buffers[cpu];		/* The CPU buffer may not have been initialized yet */		if (!cpu_buffer)			return;		rbwork = &cpu_buffer->irq_work;	} 	/* This can be called in any context */	irq_work_queue(&rbwork->work);} static bool rb_watermark_hit(struct trace_buffer *buffer, int cpu, int full){	struct ring_buffer_per_cpu *cpu_buffer;	bool ret = false; 	/* Reads of all CPUs always waits for any data */	if (cpu == RING_BUFFER_ALL_CPUS)		return !ring_buffer_empty(buffer); 	cpu_buffer = buffer->buffers[cpu]; 	if (!ring_buffer_empty_cpu(buffer, cpu)) {		unsigned long flags;		bool pagebusy; 		if (!full)			return true; 		raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);		pagebusy = cpu_buffer->reader_page == cpu_buffer->commit_page;		ret = !pagebusy && full_hit(buffer, cpu, full); 		if (!ret && (!cpu_buffer->shortest_full ||			     cpu_buffer->shortest_full > full)) {		    cpu_buffer->shortest_full = full;		}		raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);	}	return ret;} static inline boolrb_wait_cond(struct rb_irq_work *rbwork, struct trace_buffer *buffer,	     int cpu, int full, ring_buffer_cond_fn cond, void *data){	if (rb_watermark_hit(buffer, cpu, full))		return true; 	if (cond(data))		return true; 	/*	 * The events can happen in critical sections where	 * checking a work queue can cause deadlocks.	 * After adding a task to the queue, this flag is set	 * only to notify events to try to wake up the queue	 * using irq_work.	 *	 * We don't clear it even if the buffer is no longer	 * empty. The flag only causes the next event to run	 * irq_work to do the work queue wake up. The worse	 * that can happen if we race with !trace_empty() is that	 * an event will cause an irq_work to try to wake up	 * an empty queue.	 *	 * There's no reason to protect this flag either, as	 * the work queue and irq_work logic will do the necessary	 * synchronization for the wake ups. The only thing	 * that is necessary is that the wake up happens after	 * a task has been queued. It's OK for spurious wake ups.	 */	if (full)		rbwork->full_waiters_pending = true;	else		rbwork->waiters_pending = true; 	return false;} struct rb_wait_data {	struct rb_irq_work		*irq_work;	int				seq;}; /* * The default wait condition for ring_buffer_wait() is to just to exit the * wait loop the first time it is woken up. */static bool rb_wait_once(void *data){	struct rb_wait_data *rdata = data;	struct rb_irq_work *rbwork = rdata->irq_work; 	return atomic_read_acquire(&rbwork->seq) != rdata->seq;} /** * ring_buffer_wait - wait for input to the ring buffer * @buffer: buffer to wait on * @cpu: the cpu buffer to wait on * @full: wait until the percentage of pages are available, if @cpu != RING_BUFFER_ALL_CPUS * @cond: condition function to break out of wait (NULL to run once) * @data: the data to pass to @cond. * * If @cpu == RING_BUFFER_ALL_CPUS then the task will wake up as soon * as data is added to any of the @buffer's cpu buffers. Otherwise * it will wait for data to be added to a specific cpu buffer. */int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full,		     ring_buffer_cond_fn cond, void *data){	struct ring_buffer_per_cpu *cpu_buffer;	struct wait_queue_head *waitq;	struct rb_irq_work *rbwork;	struct rb_wait_data rdata;	int ret = 0; 	/*	 * Depending on what the caller is waiting for, either any	 * data in any cpu buffer, or a specific buffer, put the	 * caller on the appropriate wait queue.	 */	if (cpu == RING_BUFFER_ALL_CPUS) {		rbwork = &buffer->irq_work;		/* Full only makes sense on per cpu reads */		full = 0;	} else {		if (!cpumask_test_cpu(cpu, buffer->cpumask))			return -ENODEV;

Lines 501–1000 of 8311. Previous lines Load more