4366 lines · 112.1 KB · C
RawDownload
	unsigned long ret; 	if (!cpumask_test_cpu(cpu, buffer->cpumask))		return 0; 	cpu_buffer = buffer->buffers[cpu];	ret = local_read(&cpu_buffer->overrun); 	return ret;}EXPORT_SYMBOL_GPL(ring_buffer_overrun_cpu); /** * ring_buffer_commit_overrun_cpu - get the number of overruns caused by commits * @buffer: The ring buffer * @cpu: The per CPU buffer to get the number of overruns from */unsigned longring_buffer_commit_overrun_cpu(struct ring_buffer *buffer, int cpu){	struct ring_buffer_per_cpu *cpu_buffer;	unsigned long ret; 	if (!cpumask_test_cpu(cpu, buffer->cpumask))		return 0; 	cpu_buffer = buffer->buffers[cpu];	ret = local_read(&cpu_buffer->commit_overrun); 	return ret;}EXPORT_SYMBOL_GPL(ring_buffer_commit_overrun_cpu); /** * ring_buffer_entries - get the number of entries in a buffer * @buffer: The ring buffer * * Returns the total number of entries in the ring buffer * (all CPU entries) */unsigned long ring_buffer_entries(struct ring_buffer *buffer){	struct ring_buffer_per_cpu *cpu_buffer;	unsigned long entries = 0;	int cpu; 	/* if you care about this being correct, lock the buffer */	for_each_buffer_cpu(buffer, cpu) {		cpu_buffer = buffer->buffers[cpu];		entries += rb_num_of_entries(cpu_buffer);	} 	return entries;}EXPORT_SYMBOL_GPL(ring_buffer_entries); /** * ring_buffer_overruns - get the number of overruns in buffer * @buffer: The ring buffer * * Returns the total number of overruns in the ring buffer * (all CPU entries) */unsigned long ring_buffer_overruns(struct ring_buffer *buffer){	struct ring_buffer_per_cpu *cpu_buffer;	unsigned long overruns = 0;	int cpu; 	/* if you care about this being correct, lock the buffer */	for_each_buffer_cpu(buffer, cpu) {		cpu_buffer = buffer->buffers[cpu];		overruns += local_read(&cpu_buffer->overrun);	} 	return overruns;}EXPORT_SYMBOL_GPL(ring_buffer_overruns); static void rb_iter_reset(struct ring_buffer_iter *iter){	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer; 	/* Iterator usage is expected to have record disabled */	if (list_empty(&cpu_buffer->reader_page->list)) {		iter->head_page = rb_set_head_page(cpu_buffer);		if (unlikely(!iter->head_page))			return;		iter->head = iter->head_page->read;	} else {		iter->head_page = cpu_buffer->reader_page;		iter->head = cpu_buffer->reader_page->read;	}	if (iter->head)		iter->read_stamp = cpu_buffer->read_stamp;	else		iter->read_stamp = iter->head_page->page->time_stamp;	iter->cache_reader_page = cpu_buffer->reader_page;	iter->cache_read = cpu_buffer->read;} /** * ring_buffer_iter_reset - reset an iterator * @iter: The iterator to reset * * Resets the iterator, so that it will start from the beginning * again. */void ring_buffer_iter_reset(struct ring_buffer_iter *iter){	struct ring_buffer_per_cpu *cpu_buffer;	unsigned long flags; 	if (!iter)		return; 	cpu_buffer = iter->cpu_buffer; 	raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);	rb_iter_reset(iter);	raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);}EXPORT_SYMBOL_GPL(ring_buffer_iter_reset); /** * ring_buffer_iter_empty - check if an iterator has no more to read * @iter: The iterator to check */int ring_buffer_iter_empty(struct ring_buffer_iter *iter){	struct ring_buffer_per_cpu *cpu_buffer; 	cpu_buffer = iter->cpu_buffer; 	return iter->head_page == cpu_buffer->commit_page &&		iter->head == rb_commit_index(cpu_buffer);}EXPORT_SYMBOL_GPL(ring_buffer_iter_empty); static voidrb_update_read_stamp(struct ring_buffer_per_cpu *cpu_buffer,		     struct ring_buffer_event *event){	u64 delta; 	switch (event->type_len) {	case RINGBUF_TYPE_PADDING:		return; 	case RINGBUF_TYPE_TIME_EXTEND:		delta = event->array[0];		delta <<= TS_SHIFT;		delta += event->time_delta;		cpu_buffer->read_stamp += delta;		return; 	case RINGBUF_TYPE_TIME_STAMP:		/* FIXME: not implemented */		return; 	case RINGBUF_TYPE_DATA:		cpu_buffer->read_stamp += event->time_delta;		return; 	default:		BUG();	}	return;} static voidrb_update_iter_read_stamp(struct ring_buffer_iter *iter,			  struct ring_buffer_event *event){	u64 delta; 	switch (event->type_len) {	case RINGBUF_TYPE_PADDING:		return; 	case RINGBUF_TYPE_TIME_EXTEND:		delta = event->array[0];		delta <<= TS_SHIFT;		delta += event->time_delta;		iter->read_stamp += delta;		return; 	case RINGBUF_TYPE_TIME_STAMP:		/* FIXME: not implemented */		return; 	case RINGBUF_TYPE_DATA:		iter->read_stamp += event->time_delta;		return; 	default:		BUG();	}	return;} static struct buffer_page *rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer){	struct buffer_page *reader = NULL;	unsigned long overwrite;	unsigned long flags;	int nr_loops = 0;	int ret; 	local_irq_save(flags);	arch_spin_lock(&cpu_buffer->lock);  again:	/*	 * This should normally only loop twice. But because the	 * start of the reader inserts an empty page, it causes	 * a case where we will loop three times. There should be no	 * reason to loop four times (that I know of).	 */	if (RB_WARN_ON(cpu_buffer, ++nr_loops > 3)) {		reader = NULL;		goto out;	} 	reader = cpu_buffer->reader_page; 	/* If there's more to read, return this page */	if (cpu_buffer->reader_page->read < rb_page_size(reader))		goto out; 	/* Never should we have an index greater than the size */	if (RB_WARN_ON(cpu_buffer,		       cpu_buffer->reader_page->read > rb_page_size(reader)))		goto out; 	/* check if we caught up to the tail */	reader = NULL;	if (cpu_buffer->commit_page == cpu_buffer->reader_page)		goto out; 	/*	 * Reset the reader page to size zero.	 */	local_set(&cpu_buffer->reader_page->write, 0);	local_set(&cpu_buffer->reader_page->entries, 0);	local_set(&cpu_buffer->reader_page->page->commit, 0);	cpu_buffer->reader_page->real_end = 0;  spin:	/*	 * Splice the empty reader page into the list around the head.	 */	reader = rb_set_head_page(cpu_buffer);	cpu_buffer->reader_page->list.next = rb_list_head(reader->list.next);	cpu_buffer->reader_page->list.prev = reader->list.prev; 	/*	 * cpu_buffer->pages just needs to point to the buffer, it	 *  has no specific buffer page to point to. Lets move it out	 *  of our way so we don't accidentally swap it.	 */	cpu_buffer->pages = reader->list.prev; 	/* The reader page will be pointing to the new head */	rb_set_list_to_head(cpu_buffer, &cpu_buffer->reader_page->list); 	/*	 * We want to make sure we read the overruns after we set up our	 * pointers to the next object. The writer side does a	 * cmpxchg to cross pages which acts as the mb on the writer	 * side. Note, the reader will constantly fail the swap	 * while the writer is updating the pointers, so this	 * guarantees that the overwrite recorded here is the one we	 * want to compare with the last_overrun.	 */	smp_mb();	overwrite = local_read(&(cpu_buffer->overrun)); 	/*	 * Here's the tricky part.	 *	 * We need to move the pointer past the header page.	 * But we can only do that if a writer is not currently	 * moving it. The page before the header page has the	 * flag bit '1' set if it is pointing to the page we want.	 * but if the writer is in the process of moving it	 * than it will be '2' or already moved '0'.	 */ 	ret = rb_head_page_replace(reader, cpu_buffer->reader_page); 	/*	 * If we did not convert it, then we must try again.	 */	if (!ret)		goto spin; 	/*	 * Yeah! We succeeded in replacing the page.	 *	 * Now make the new head point back to the reader page.	 */	rb_list_head(reader->list.next)->prev = &cpu_buffer->reader_page->list;	rb_inc_page(cpu_buffer, &cpu_buffer->head_page); 	/* Finally update the reader page to the new head */	cpu_buffer->reader_page = reader;	rb_reset_reader_page(cpu_buffer); 	if (overwrite != cpu_buffer->last_overrun) {		cpu_buffer->lost_events = overwrite - cpu_buffer->last_overrun;		cpu_buffer->last_overrun = overwrite;	} 	goto again;  out:	arch_spin_unlock(&cpu_buffer->lock);	local_irq_restore(flags); 	return reader;} static void rb_advance_reader(struct ring_buffer_per_cpu *cpu_buffer){	struct ring_buffer_event *event;	struct buffer_page *reader;	unsigned length; 	reader = rb_get_reader_page(cpu_buffer); 	/* This function should not be called when buffer is empty */	if (RB_WARN_ON(cpu_buffer, !reader))		return; 	event = rb_reader_event(cpu_buffer); 	if (event->type_len <= RINGBUF_TYPE_DATA_TYPE_LEN_MAX)		cpu_buffer->read++; 	rb_update_read_stamp(cpu_buffer, event); 	length = rb_event_length(event);	cpu_buffer->reader_page->read += length;} static void rb_advance_iter(struct ring_buffer_iter *iter){	struct ring_buffer_per_cpu *cpu_buffer;	struct ring_buffer_event *event;	unsigned length; 	cpu_buffer = iter->cpu_buffer; 	/*	 * Check if we are at the end of the buffer.	 */	if (iter->head >= rb_page_size(iter->head_page)) {		/* discarded commits can make the page empty */		if (iter->head_page == cpu_buffer->commit_page)			return;		rb_inc_iter(iter);		return;	} 	event = rb_iter_head_event(iter); 	length = rb_event_length(event); 	/*	 * This should not be called to advance the header if we are	 * at the tail of the buffer.	 */	if (RB_WARN_ON(cpu_buffer,		       (iter->head_page == cpu_buffer->commit_page) &&		       (iter->head + length > rb_commit_index(cpu_buffer))))		return; 	rb_update_iter_read_stamp(iter, event); 	iter->head += length; 	/* check for end of page padding */	if ((iter->head >= rb_page_size(iter->head_page)) &&	    (iter->head_page != cpu_buffer->commit_page))		rb_advance_iter(iter);} static int rb_lost_events(struct ring_buffer_per_cpu *cpu_buffer){	return cpu_buffer->lost_events;} static struct ring_buffer_event *rb_buffer_peek(struct ring_buffer_per_cpu *cpu_buffer, u64 *ts,	       unsigned long *lost_events){	struct ring_buffer_event *event;	struct buffer_page *reader;	int nr_loops = 0;  again:	/*	 * We repeat when a time extend is encountered.	 * Since the time extend is always attached to a data event,	 * we should never loop more than once.	 * (We never hit the following condition more than twice).	 */	if (RB_WARN_ON(cpu_buffer, ++nr_loops > 2))		return NULL; 	reader = rb_get_reader_page(cpu_buffer);	if (!reader)		return NULL; 	event = rb_reader_event(cpu_buffer); 	switch (event->type_len) {	case RINGBUF_TYPE_PADDING:		if (rb_null_event(event))			RB_WARN_ON(cpu_buffer, 1);		/*		 * Because the writer could be discarding every		 * event it creates (which would probably be bad)		 * if we were to go back to "again" then we may never		 * catch up, and will trigger the warn on, or lock		 * the box. Return the padding, and we will release		 * the current locks, and try again.		 */		return event; 	case RINGBUF_TYPE_TIME_EXTEND:		/* Internal data, OK to advance */		rb_advance_reader(cpu_buffer);		goto again; 	case RINGBUF_TYPE_TIME_STAMP:		/* FIXME: not implemented */		rb_advance_reader(cpu_buffer);		goto again; 	case RINGBUF_TYPE_DATA:		if (ts) {			*ts = cpu_buffer->read_stamp + event->time_delta;			ring_buffer_normalize_time_stamp(cpu_buffer->buffer,							 cpu_buffer->cpu, ts);		}		if (lost_events)			*lost_events = rb_lost_events(cpu_buffer);		return event; 	default:		BUG();	} 	return NULL;}EXPORT_SYMBOL_GPL(ring_buffer_peek); static struct ring_buffer_event *rb_iter_peek(struct ring_buffer_iter *iter, u64 *ts){	struct ring_buffer *buffer;	struct ring_buffer_per_cpu *cpu_buffer;	struct ring_buffer_event *event;	int nr_loops = 0; 	cpu_buffer = iter->cpu_buffer;	buffer = cpu_buffer->buffer; 	/*	 * Check if someone performed a consuming read to	 * the buffer. A consuming read invalidates the iterator	 * and we need to reset the iterator in this case.	 */	if (unlikely(iter->cache_read != cpu_buffer->read ||		     iter->cache_reader_page != cpu_buffer->reader_page))		rb_iter_reset(iter);  again:	if (ring_buffer_iter_empty(iter))		return NULL; 	/*	 * We repeat when a time extend is encountered.	 * Since the time extend is always attached to a data event,	 * we should never loop more than once.	 * (We never hit the following condition more than twice).	 */	if (RB_WARN_ON(cpu_buffer, ++nr_loops > 2))		return NULL; 	if (rb_per_cpu_empty(cpu_buffer))		return NULL; 	if (iter->head >= local_read(&iter->head_page->page->commit)) {		rb_inc_iter(iter);		goto again;	}

Lines 3001–3500 of 4366. Previous lines

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t