4366 lines · 112.1 KB · C
RawDownload
	 * then it is up to us to reset it to NORMAL.	 */	if (type == RB_PAGE_HEAD) {		ret = rb_head_page_set_normal(cpu_buffer, next_page,					      tail_page,					      RB_PAGE_UPDATE);		if (RB_WARN_ON(cpu_buffer,			       ret != RB_PAGE_UPDATE))			return -1;	} 	return 0;} static unsigned rb_calculate_event_length(unsigned length){	struct ring_buffer_event event; /* Used only for sizeof array */ 	/* zero length can cause confusions */	if (!length)		length = 1; 	if (length > RB_MAX_SMALL_DATA || RB_FORCE_8BYTE_ALIGNMENT)		length += sizeof(event.array[0]); 	length += RB_EVNT_HDR_SIZE;	length = ALIGN(length, RB_ARCH_ALIGNMENT); 	return length;} static inline voidrb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer,	      struct buffer_page *tail_page,	      unsigned long tail, unsigned long length){	struct ring_buffer_event *event; 	/*	 * Only the event that crossed the page boundary	 * must fill the old tail_page with padding.	 */	if (tail >= BUF_PAGE_SIZE) {		/*		 * If the page was filled, then we still need		 * to update the real_end. Reset it to zero		 * and the reader will ignore it.		 */		if (tail == BUF_PAGE_SIZE)			tail_page->real_end = 0; 		local_sub(length, &tail_page->write);		return;	} 	event = __rb_page_index(tail_page, tail);	kmemcheck_annotate_bitfield(event, bitfield); 	/* account for padding bytes */	local_add(BUF_PAGE_SIZE - tail, &cpu_buffer->entries_bytes); 	/*	 * Save the original length to the meta data.	 * This will be used by the reader to add lost event	 * counter.	 */	tail_page->real_end = tail; 	/*	 * If this event is bigger than the minimum size, then	 * we need to be careful that we don't subtract the	 * write counter enough to allow another writer to slip	 * in on this page.	 * We put in a discarded commit instead, to make sure	 * that this space is not used again.	 *	 * If we are less than the minimum size, we don't need to	 * worry about it.	 */	if (tail > (BUF_PAGE_SIZE - RB_EVNT_MIN_SIZE)) {		/* No room for any events */ 		/* Mark the rest of the page with padding */		rb_event_set_padding(event); 		/* Set the write back to the previous setting */		local_sub(length, &tail_page->write);		return;	} 	/* Put in a discarded event */	event->array[0] = (BUF_PAGE_SIZE - tail) - RB_EVNT_HDR_SIZE;	event->type_len = RINGBUF_TYPE_PADDING;	/* time delta must be non zero */	event->time_delta = 1; 	/* Set write to end of buffer */	length = (tail + length) - BUF_PAGE_SIZE;	local_sub(length, &tail_page->write);} /* * This is the slow path, force gcc not to inline it. */static noinline struct ring_buffer_event *rb_move_tail(struct ring_buffer_per_cpu *cpu_buffer,	     unsigned long length, unsigned long tail,	     struct buffer_page *tail_page, u64 ts){	struct buffer_page *commit_page = cpu_buffer->commit_page;	struct ring_buffer *buffer = cpu_buffer->buffer;	struct buffer_page *next_page;	int ret; 	next_page = tail_page; 	rb_inc_page(cpu_buffer, &next_page); 	/*	 * If for some reason, we had an interrupt storm that made	 * it all the way around the buffer, bail, and warn	 * about it.	 */	if (unlikely(next_page == commit_page)) {		local_inc(&cpu_buffer->commit_overrun);		goto out_reset;	} 	/*	 * This is where the fun begins!	 *	 * We are fighting against races between a reader that	 * could be on another CPU trying to swap its reader	 * page with the buffer head.	 *	 * We are also fighting against interrupts coming in and	 * moving the head or tail on us as well.	 *	 * If the next page is the head page then we have filled	 * the buffer, unless the commit page is still on the	 * reader page.	 */	if (rb_is_head_page(cpu_buffer, next_page, &tail_page->list)) { 		/*		 * If the commit is not on the reader page, then		 * move the header page.		 */		if (!rb_is_reader_page(cpu_buffer->commit_page)) {			/*			 * If we are not in overwrite mode,			 * this is easy, just stop here.			 */			if (!(buffer->flags & RB_FL_OVERWRITE))				goto out_reset; 			ret = rb_handle_head_page(cpu_buffer,						  tail_page,						  next_page);			if (ret < 0)				goto out_reset;			if (ret)				goto out_again;		} else {			/*			 * We need to be careful here too. The			 * commit page could still be on the reader			 * page. We could have a small buffer, and			 * have filled up the buffer with events			 * from interrupts and such, and wrapped.			 *			 * Note, if the tail page is also the on the			 * reader_page, we let it move out.			 */			if (unlikely((cpu_buffer->commit_page !=				      cpu_buffer->tail_page) &&				     (cpu_buffer->commit_page ==				      cpu_buffer->reader_page))) {				local_inc(&cpu_buffer->commit_overrun);				goto out_reset;			}		}	} 	ret = rb_tail_page_update(cpu_buffer, tail_page, next_page);	if (ret) {		/*		 * Nested commits always have zero deltas, so		 * just reread the time stamp		 */		ts = rb_time_stamp(buffer);		next_page->page->time_stamp = ts;	}  out_again: 	rb_reset_tail(cpu_buffer, tail_page, tail, length); 	/* fail and let the caller try again */	return ERR_PTR(-EAGAIN);  out_reset:	/* reset write */	rb_reset_tail(cpu_buffer, tail_page, tail, length); 	return NULL;} static struct ring_buffer_event *__rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,		  unsigned long length, u64 ts,		  u64 delta, int add_timestamp){	struct buffer_page *tail_page;	struct ring_buffer_event *event;	unsigned long tail, write; 	/*	 * If the time delta since the last event is too big to	 * hold in the time field of the event, then we append a	 * TIME EXTEND event ahead of the data event.	 */	if (unlikely(add_timestamp))		length += RB_LEN_TIME_EXTEND; 	tail_page = cpu_buffer->tail_page;	write = local_add_return(length, &tail_page->write); 	/* set write to only the index of the write */	write &= RB_WRITE_MASK;	tail = write - length; 	/* See if we shot pass the end of this buffer page */	if (unlikely(write > BUF_PAGE_SIZE))		return rb_move_tail(cpu_buffer, length, tail,				    tail_page, ts); 	/* We reserved something on the buffer */ 	event = __rb_page_index(tail_page, tail);	kmemcheck_annotate_bitfield(event, bitfield);	rb_update_event(cpu_buffer, event, length, add_timestamp, delta); 	local_inc(&tail_page->entries); 	/*	 * If this is the first commit on the page, then update	 * its timestamp.	 */	if (!tail)		tail_page->page->time_stamp = ts; 	/* account for these added bytes */	local_add(length, &cpu_buffer->entries_bytes); 	return event;} static inline intrb_try_to_discard(struct ring_buffer_per_cpu *cpu_buffer,		  struct ring_buffer_event *event){	unsigned long new_index, old_index;	struct buffer_page *bpage;	unsigned long index;	unsigned long addr; 	new_index = rb_event_index(event);	old_index = new_index + rb_event_ts_length(event);	addr = (unsigned long)event;	addr &= PAGE_MASK; 	bpage = cpu_buffer->tail_page; 	if (bpage->page == (void *)addr && rb_page_write(bpage) == old_index) {		unsigned long write_mask =			local_read(&bpage->write) & ~RB_WRITE_MASK;		unsigned long event_length = rb_event_length(event);		/*		 * This is on the tail page. It is possible that		 * a write could come in and move the tail page		 * and write to the next page. That is fine		 * because we just shorten what is on this page.		 */		old_index += write_mask;		new_index += write_mask;		index = local_cmpxchg(&bpage->write, old_index, new_index);		if (index == old_index) {			/* update counters */			local_sub(event_length, &cpu_buffer->entries_bytes);			return 1;		}	} 	/* could not discard */	return 0;} static void rb_start_commit(struct ring_buffer_per_cpu *cpu_buffer){	local_inc(&cpu_buffer->committing);	local_inc(&cpu_buffer->commits);} static inline void rb_end_commit(struct ring_buffer_per_cpu *cpu_buffer){	unsigned long commits; 	if (RB_WARN_ON(cpu_buffer,		       !local_read(&cpu_buffer->committing)))		return;  again:	commits = local_read(&cpu_buffer->commits);	/* synchronize with interrupts */	barrier();	if (local_read(&cpu_buffer->committing) == 1)		rb_set_commit_to_write(cpu_buffer); 	local_dec(&cpu_buffer->committing); 	/* synchronize with interrupts */	barrier(); 	/*	 * Need to account for interrupts coming in between the	 * updating of the commit page and the clearing of the	 * committing counter.	 */	if (unlikely(local_read(&cpu_buffer->commits) != commits) &&	    !local_read(&cpu_buffer->committing)) {		local_inc(&cpu_buffer->committing);		goto again;	}} static struct ring_buffer_event *rb_reserve_next_event(struct ring_buffer *buffer,		      struct ring_buffer_per_cpu *cpu_buffer,		      unsigned long length){	struct ring_buffer_event *event;	u64 ts, delta;	int nr_loops = 0;	int add_timestamp;	u64 diff; 	rb_start_commit(cpu_buffer); #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP	/*	 * Due to the ability to swap a cpu buffer from a buffer	 * it is possible it was swapped before we committed.	 * (committing stops a swap). We check for it here and	 * if it happened, we have to fail the write.	 */	barrier();	if (unlikely(ACCESS_ONCE(cpu_buffer->buffer) != buffer)) {		local_dec(&cpu_buffer->committing);		local_dec(&cpu_buffer->commits);		return NULL;	}#endif 	length = rb_calculate_event_length(length); again:	add_timestamp = 0;	delta = 0; 	/*	 * We allow for interrupts to reenter here and do a trace.	 * If one does, it will cause this original code to loop	 * back here. Even with heavy interrupts happening, this	 * should only happen a few times in a row. If this happens	 * 1000 times in a row, there must be either an interrupt	 * storm or we have something buggy.	 * Bail!	 */	if (RB_WARN_ON(cpu_buffer, ++nr_loops > 1000))		goto out_fail; 	ts = rb_time_stamp(cpu_buffer->buffer);	diff = ts - cpu_buffer->write_stamp; 	/* make sure this diff is calculated here */	barrier(); 	/* Did the write stamp get updated already? */	if (likely(ts >= cpu_buffer->write_stamp)) {		delta = diff;		if (unlikely(test_time_stamp(delta))) {			int local_clock_stable = 1;#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK			local_clock_stable = sched_clock_stable;#endif			WARN_ONCE(delta > (1ULL << 59),				  KERN_WARNING "Delta way too big! %llu ts=%llu write stamp = %llu\n%s",				  (unsigned long long)delta,				  (unsigned long long)ts,				  (unsigned long long)cpu_buffer->write_stamp,				  local_clock_stable ? "" :				  "If you just came from a suspend/resume,\n"				  "please switch to the trace global clock:\n"				  "  echo global > /sys/kernel/debug/tracing/trace_clock\n");			add_timestamp = 1;		}	} 	event = __rb_reserve_next(cpu_buffer, length, ts,				  delta, add_timestamp);	if (unlikely(PTR_ERR(event) == -EAGAIN))		goto again; 	if (!event)		goto out_fail; 	return event;  out_fail:	rb_end_commit(cpu_buffer);	return NULL;} #ifdef CONFIG_TRACING #define TRACE_RECURSIVE_DEPTH 16 /* Keep this code out of the fast path cache */static noinline void trace_recursive_fail(void){	/* Disable all tracing before we do anything else */	tracing_off_permanent(); 	printk_once(KERN_WARNING "Tracing recursion: depth[%ld]:"		    "HC[%lu]:SC[%lu]:NMI[%lu]\n",		    trace_recursion_buffer(),		    hardirq_count() >> HARDIRQ_SHIFT,		    softirq_count() >> SOFTIRQ_SHIFT,		    in_nmi()); 	WARN_ON_ONCE(1);} static inline int trace_recursive_lock(void){	trace_recursion_inc(); 	if (likely(trace_recursion_buffer() < TRACE_RECURSIVE_DEPTH))		return 0; 	trace_recursive_fail(); 	return -1;} static inline void trace_recursive_unlock(void){	WARN_ON_ONCE(!trace_recursion_buffer()); 	trace_recursion_dec();} #else #define trace_recursive_lock()		(0)#define trace_recursive_unlock()	do { } while (0) #endif /** * ring_buffer_lock_reserve - reserve a part of the buffer * @buffer: the ring buffer to reserve from * @length: the length of the data to reserve (excluding event header) * * Returns a reseverd event on the ring buffer to copy directly to. * The user of this interface will need to get the body to write into * and can use the ring_buffer_event_data() interface. * * The length is the length of the data needed, not the event length * which also includes the event header. * * Must be paired with ring_buffer_unlock_commit, unless NULL is returned. * If NULL is returned, then nothing has been allocated or locked. */struct ring_buffer_event *ring_buffer_lock_reserve(struct ring_buffer *buffer, unsigned long length){	struct ring_buffer_per_cpu *cpu_buffer;	struct ring_buffer_event *event;	int cpu; 	if (ring_buffer_flags != RB_BUFFERS_ON)		return NULL; 	/* If we are tracing schedule, we don't want to recurse */	preempt_disable_notrace(); 	if (atomic_read(&buffer->record_disabled))		goto out_nocheck; 

Lines 2001–2500 of 4366. Previous lines

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t