5723 lines · 150.8 KB · C
RawDownload
/* * This is the slow path, force gcc not to inline it. */static noinline struct ring_buffer_event *rb_move_tail(struct ring_buffer_per_cpu *cpu_buffer,	     unsigned long tail, struct rb_event_info *info){	struct buffer_page *tail_page = info->tail_page;	struct buffer_page *commit_page = cpu_buffer->commit_page;	struct trace_buffer *buffer = cpu_buffer->buffer;	struct buffer_page *next_page;	int ret; 	next_page = tail_page; 	rb_inc_page(cpu_buffer, &next_page); 	/*	 * If for some reason, we had an interrupt storm that made	 * it all the way around the buffer, bail, and warn	 * about it.	 */	if (unlikely(next_page == commit_page)) {		local_inc(&cpu_buffer->commit_overrun);		goto out_reset;	} 	/*	 * This is where the fun begins!	 *	 * We are fighting against races between a reader that	 * could be on another CPU trying to swap its reader	 * page with the buffer head.	 *	 * We are also fighting against interrupts coming in and	 * moving the head or tail on us as well.	 *	 * If the next page is the head page then we have filled	 * the buffer, unless the commit page is still on the	 * reader page.	 */	if (rb_is_head_page(cpu_buffer, next_page, &tail_page->list)) { 		/*		 * If the commit is not on the reader page, then		 * move the header page.		 */		if (!rb_is_reader_page(cpu_buffer->commit_page)) {			/*			 * If we are not in overwrite mode,			 * this is easy, just stop here.			 */			if (!(buffer->flags & RB_FL_OVERWRITE)) {				local_inc(&cpu_buffer->dropped_events);				goto out_reset;			} 			ret = rb_handle_head_page(cpu_buffer,						  tail_page,						  next_page);			if (ret < 0)				goto out_reset;			if (ret)				goto out_again;		} else {			/*			 * We need to be careful here too. The			 * commit page could still be on the reader			 * page. We could have a small buffer, and			 * have filled up the buffer with events			 * from interrupts and such, and wrapped.			 *			 * Note, if the tail page is also the on the			 * reader_page, we let it move out.			 */			if (unlikely((cpu_buffer->commit_page !=				      cpu_buffer->tail_page) &&				     (cpu_buffer->commit_page ==				      cpu_buffer->reader_page))) {				local_inc(&cpu_buffer->commit_overrun);				goto out_reset;			}		}	} 	rb_tail_page_update(cpu_buffer, tail_page, next_page);  out_again: 	rb_reset_tail(cpu_buffer, tail, info); 	/* Commit what we have for now. */	rb_end_commit(cpu_buffer);	/* rb_end_commit() decs committing */	local_inc(&cpu_buffer->committing); 	/* fail and let the caller try again */	return ERR_PTR(-EAGAIN);  out_reset:	/* reset write */	rb_reset_tail(cpu_buffer, tail, info); 	return NULL;} /* Slow path */static struct ring_buffer_event *rb_add_time_stamp(struct ring_buffer_event *event, u64 delta, bool abs){	if (abs)		event->type_len = RINGBUF_TYPE_TIME_STAMP;	else		event->type_len = RINGBUF_TYPE_TIME_EXTEND; 	/* Not the first event on the page, or not delta? */	if (abs || rb_event_index(event)) {		event->time_delta = delta & TS_MASK;		event->array[0] = delta >> TS_SHIFT;	} else {		/* nope, just zero it */		event->time_delta = 0;		event->array[0] = 0;	} 	return skip_time_extend(event);} static inline bool rb_event_is_commit(struct ring_buffer_per_cpu *cpu_buffer,				     struct ring_buffer_event *event); #ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCKstatic inline bool sched_clock_stable(void){	return true;}#endif static voidrb_check_timestamp(struct ring_buffer_per_cpu *cpu_buffer,		   struct rb_event_info *info){	u64 write_stamp; 	WARN_ONCE(1, "Delta way too big! %llu ts=%llu before=%llu after=%llu write stamp=%llu\n%s",		  (unsigned long long)info->delta,		  (unsigned long long)info->ts,		  (unsigned long long)info->before,		  (unsigned long long)info->after,		  (unsigned long long)(rb_time_read(&cpu_buffer->write_stamp, &write_stamp) ? write_stamp : 0),		  sched_clock_stable() ? "" :		  "If you just came from a suspend/resume,\n"		  "please switch to the trace global clock:\n"		  "  echo global > /sys/kernel/debug/tracing/trace_clock\n"		  "or add trace_clock=global to the kernel command line\n");} static void rb_add_timestamp(struct ring_buffer_per_cpu *cpu_buffer,				      struct ring_buffer_event **event,				      struct rb_event_info *info,				      u64 *delta,				      unsigned int *length){	bool abs = info->add_timestamp &		(RB_ADD_STAMP_FORCE | RB_ADD_STAMP_ABSOLUTE); 	if (unlikely(info->delta > (1ULL << 59))) {		/* did the clock go backwards */		if (info->before == info->after && info->before > info->ts) {			/* not interrupted */			static int once; 			/*			 * This is possible with a recalibrating of the TSC.			 * Do not produce a call stack, but just report it.			 */			if (!once) {				once++;				pr_warn("Ring buffer clock went backwards: %llu -> %llu\n",					info->before, info->ts);			}		} else			rb_check_timestamp(cpu_buffer, info);		if (!abs)			info->delta = 0;	}	*event = rb_add_time_stamp(*event, info->delta, abs);	*length -= RB_LEN_TIME_EXTEND;	*delta = 0;} /** * rb_update_event - update event type and data * @cpu_buffer: The per cpu buffer of the @event * @event: the event to update * @info: The info to update the @event with (contains length and delta) * * Update the type and data fields of the @event. The length * is the actual size that is written to the ring buffer, * and with this, we can determine what to place into the * data field. */static voidrb_update_event(struct ring_buffer_per_cpu *cpu_buffer,		struct ring_buffer_event *event,		struct rb_event_info *info){	unsigned length = info->length;	u64 delta = info->delta; 	/*	 * If we need to add a timestamp, then we	 * add it to the start of the reserved space.	 */	if (unlikely(info->add_timestamp))		rb_add_timestamp(cpu_buffer, &event, info, &delta, &length); 	event->time_delta = delta;	length -= RB_EVNT_HDR_SIZE;	if (length > RB_MAX_SMALL_DATA) {		event->type_len = 0;		event->array[0] = length;	} else		event->type_len = DIV_ROUND_UP(length, RB_ALIGNMENT);} static unsigned rb_calculate_event_length(unsigned length){	struct ring_buffer_event event; /* Used only for sizeof array */ 	/* zero length can cause confusions */	if (!length)		length++; 	if (length > RB_MAX_SMALL_DATA)		length += sizeof(event.array[0]); 	length += RB_EVNT_HDR_SIZE;	length = ALIGN(length, RB_ALIGNMENT); 	/*	 * In case the time delta is larger than the 27 bits for it	 * in the header, we need to add a timestamp. If another	 * event comes in when trying to discard this one to increase	 * the length, then the timestamp will be added in the allocated	 * space of this event. If length is bigger than the size needed	 * for the TIME_EXTEND, then padding has to be used. The events	 * length must be either RB_LEN_TIME_EXTEND, or greater than or equal	 * to RB_LEN_TIME_EXTEND + 8, as 8 is the minimum size for padding.	 * As length is a multiple of 4, we only need to worry if it	 * is 12 (RB_LEN_TIME_EXTEND + 4).	 */	if (length == RB_LEN_TIME_EXTEND + RB_ALIGNMENT)		length += RB_ALIGNMENT; 	return length;} static __always_inline boolrb_event_is_commit(struct ring_buffer_per_cpu *cpu_buffer,		   struct ring_buffer_event *event){	unsigned long addr = (unsigned long)event;	unsigned long index; 	index = rb_event_index(event);	addr &= PAGE_MASK; 	return cpu_buffer->commit_page->page == (void *)addr &&		rb_commit_index(cpu_buffer) == index;} static u64 rb_time_delta(struct ring_buffer_event *event){	switch (event->type_len) {	case RINGBUF_TYPE_PADDING:		return 0; 	case RINGBUF_TYPE_TIME_EXTEND:		return ring_buffer_event_time_stamp(event); 	case RINGBUF_TYPE_TIME_STAMP:		return 0; 	case RINGBUF_TYPE_DATA:		return event->time_delta;	default:		return 0;	}} static inline intrb_try_to_discard(struct ring_buffer_per_cpu *cpu_buffer,		  struct ring_buffer_event *event){	unsigned long new_index, old_index;	struct buffer_page *bpage;	unsigned long index;	unsigned long addr;	u64 write_stamp;	u64 delta; 	new_index = rb_event_index(event);	old_index = new_index + rb_event_ts_length(event);	addr = (unsigned long)event;	addr &= PAGE_MASK; 	bpage = READ_ONCE(cpu_buffer->tail_page); 	delta = rb_time_delta(event); 	if (!rb_time_read(&cpu_buffer->write_stamp, &write_stamp))		return 0; 	/* Make sure the write stamp is read before testing the location */	barrier(); 	if (bpage->page == (void *)addr && rb_page_write(bpage) == old_index) {		unsigned long write_mask =			local_read(&bpage->write) & ~RB_WRITE_MASK;		unsigned long event_length = rb_event_length(event); 		/* Something came in, can't discard */		if (!rb_time_cmpxchg(&cpu_buffer->write_stamp,				       write_stamp, write_stamp - delta))			return 0; 		/*		 * If an event were to come in now, it would see that the		 * write_stamp and the before_stamp are different, and assume		 * that this event just added itself before updating		 * the write stamp. The interrupting event will fix the		 * write stamp for us, and use the before stamp as its delta.		 */ 		/*		 * This is on the tail page. It is possible that		 * a write could come in and move the tail page		 * and write to the next page. That is fine		 * because we just shorten what is on this page.		 */		old_index += write_mask;		new_index += write_mask;		index = local_cmpxchg(&bpage->write, old_index, new_index);		if (index == old_index) {			/* update counters */			local_sub(event_length, &cpu_buffer->entries_bytes);			return 1;		}	} 	/* could not discard */	return 0;} static void rb_start_commit(struct ring_buffer_per_cpu *cpu_buffer){	local_inc(&cpu_buffer->committing);	local_inc(&cpu_buffer->commits);} static __always_inline voidrb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer){	unsigned long max_count; 	/*	 * We only race with interrupts and NMIs on this CPU.	 * If we own the commit event, then we can commit	 * all others that interrupted us, since the interruptions	 * are in stack format (they finish before they come	 * back to us). This allows us to do a simple loop to	 * assign the commit to the tail.	 */ again:	max_count = cpu_buffer->nr_pages * 100; 	while (cpu_buffer->commit_page != READ_ONCE(cpu_buffer->tail_page)) {		if (RB_WARN_ON(cpu_buffer, !(--max_count)))			return;		if (RB_WARN_ON(cpu_buffer,			       rb_is_reader_page(cpu_buffer->tail_page)))			return;		local_set(&cpu_buffer->commit_page->page->commit,			  rb_page_write(cpu_buffer->commit_page));		rb_inc_page(cpu_buffer, &cpu_buffer->commit_page);		/* add barrier to keep gcc from optimizing too much */		barrier();	}	while (rb_commit_index(cpu_buffer) !=	       rb_page_write(cpu_buffer->commit_page)) { 		local_set(&cpu_buffer->commit_page->page->commit,			  rb_page_write(cpu_buffer->commit_page));		RB_WARN_ON(cpu_buffer,			   local_read(&cpu_buffer->commit_page->page->commit) &			   ~RB_WRITE_MASK);		barrier();	} 	/* again, keep gcc from optimizing */	barrier(); 	/*	 * If an interrupt came in just after the first while loop	 * and pushed the tail page forward, we will be left with	 * a dangling commit that will never go forward.	 */	if (unlikely(cpu_buffer->commit_page != READ_ONCE(cpu_buffer->tail_page)))		goto again;} static __always_inline void rb_end_commit(struct ring_buffer_per_cpu *cpu_buffer){	unsigned long commits; 	if (RB_WARN_ON(cpu_buffer,		       !local_read(&cpu_buffer->committing)))		return;  again:	commits = local_read(&cpu_buffer->commits);	/* synchronize with interrupts */	barrier();	if (local_read(&cpu_buffer->committing) == 1)		rb_set_commit_to_write(cpu_buffer); 	local_dec(&cpu_buffer->committing); 	/* synchronize with interrupts */	barrier(); 	/*	 * Need to account for interrupts coming in between the	 * updating of the commit page and the clearing of the	 * committing counter.	 */	if (unlikely(local_read(&cpu_buffer->commits) != commits) &&	    !local_read(&cpu_buffer->committing)) {		local_inc(&cpu_buffer->committing);		goto again;	}} static inline void rb_event_discard(struct ring_buffer_event *event){	if (extended_time(event))		event = skip_time_extend(event); 	/* array[0] holds the actual length for the discarded event */	event->array[0] = rb_event_data_length(event) - RB_EVNT_HDR_SIZE;	event->type_len = RINGBUF_TYPE_PADDING;	/* time delta must be non zero */	if (!event->time_delta)		event->time_delta = 1;} static void rb_commit(struct ring_buffer_per_cpu *cpu_buffer,		      struct ring_buffer_event *event){	local_inc(&cpu_buffer->entries);	rb_end_commit(cpu_buffer);} static __always_inline voidrb_wakeups(struct trace_buffer *buffer, struct ring_buffer_per_cpu *cpu_buffer){	size_t nr_pages;	size_t dirty;	size_t full; 	if (buffer->irq_work.waiters_pending) {		buffer->irq_work.waiters_pending = false;		/* irq_work_queue() supplies it's own memory barriers */		irq_work_queue(&buffer->irq_work.work);	} 	if (cpu_buffer->irq_work.waiters_pending) {		cpu_buffer->irq_work.waiters_pending = false;		/* irq_work_queue() supplies it's own memory barriers */		irq_work_queue(&cpu_buffer->irq_work.work);	} 	if (cpu_buffer->last_pages_touch == local_read(&cpu_buffer->pages_touched))		return; 	if (cpu_buffer->reader_page == cpu_buffer->commit_page)		return; 	if (!cpu_buffer->irq_work.full_waiters_pending)		return; 	cpu_buffer->last_pages_touch = local_read(&cpu_buffer->pages_touched); 	full = cpu_buffer->shortest_full;	nr_pages = cpu_buffer->nr_pages;	dirty = ring_buffer_nr_dirty_pages(buffer, cpu_buffer->cpu);	if (full && nr_pages && (dirty * 100) <= full * nr_pages)		return; 

Lines 2501–3000 of 5723. Previous lines

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t