4366 lines · 112.1 KB · C
RawDownload
 	event = rb_iter_head_event(iter); 	switch (event->type_len) {	case RINGBUF_TYPE_PADDING:		if (rb_null_event(event)) {			rb_inc_iter(iter);			goto again;		}		rb_advance_iter(iter);		return event; 	case RINGBUF_TYPE_TIME_EXTEND:		/* Internal data, OK to advance */		rb_advance_iter(iter);		goto again; 	case RINGBUF_TYPE_TIME_STAMP:		/* FIXME: not implemented */		rb_advance_iter(iter);		goto again; 	case RINGBUF_TYPE_DATA:		if (ts) {			*ts = iter->read_stamp + event->time_delta;			ring_buffer_normalize_time_stamp(buffer,							 cpu_buffer->cpu, ts);		}		return event; 	default:		BUG();	} 	return NULL;}EXPORT_SYMBOL_GPL(ring_buffer_iter_peek); static inline int rb_ok_to_lock(void){	/*	 * If an NMI die dumps out the content of the ring buffer	 * do not grab locks. We also permanently disable the ring	 * buffer too. A one time deal is all you get from reading	 * the ring buffer from an NMI.	 */	if (likely(!in_nmi()))		return 1; 	tracing_off_permanent();	return 0;} /** * ring_buffer_peek - peek at the next event to be read * @buffer: The ring buffer to read * @cpu: The cpu to peak at * @ts: The timestamp counter of this event. * @lost_events: a variable to store if events were lost (may be NULL) * * This will return the event that will be read next, but does * not consume the data. */struct ring_buffer_event *ring_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts,		 unsigned long *lost_events){	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];	struct ring_buffer_event *event;	unsigned long flags;	int dolock; 	if (!cpumask_test_cpu(cpu, buffer->cpumask))		return NULL; 	dolock = rb_ok_to_lock(); again:	local_irq_save(flags);	if (dolock)		raw_spin_lock(&cpu_buffer->reader_lock);	event = rb_buffer_peek(cpu_buffer, ts, lost_events);	if (event && event->type_len == RINGBUF_TYPE_PADDING)		rb_advance_reader(cpu_buffer);	if (dolock)		raw_spin_unlock(&cpu_buffer->reader_lock);	local_irq_restore(flags); 	if (event && event->type_len == RINGBUF_TYPE_PADDING)		goto again; 	return event;} /** * ring_buffer_iter_peek - peek at the next event to be read * @iter: The ring buffer iterator * @ts: The timestamp counter of this event. * * This will return the event that will be read next, but does * not increment the iterator. */struct ring_buffer_event *ring_buffer_iter_peek(struct ring_buffer_iter *iter, u64 *ts){	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;	struct ring_buffer_event *event;	unsigned long flags;  again:	raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);	event = rb_iter_peek(iter, ts);	raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); 	if (event && event->type_len == RINGBUF_TYPE_PADDING)		goto again; 	return event;} /** * ring_buffer_consume - return an event and consume it * @buffer: The ring buffer to get the next event from * @cpu: the cpu to read the buffer from * @ts: a variable to store the timestamp (may be NULL) * @lost_events: a variable to store if events were lost (may be NULL) * * Returns the next event in the ring buffer, and that event is consumed. * Meaning, that sequential reads will keep returning a different event, * and eventually empty the ring buffer if the producer is slower. */struct ring_buffer_event *ring_buffer_consume(struct ring_buffer *buffer, int cpu, u64 *ts,		    unsigned long *lost_events){	struct ring_buffer_per_cpu *cpu_buffer;	struct ring_buffer_event *event = NULL;	unsigned long flags;	int dolock; 	dolock = rb_ok_to_lock();  again:	/* might be called in atomic */	preempt_disable(); 	if (!cpumask_test_cpu(cpu, buffer->cpumask))		goto out; 	cpu_buffer = buffer->buffers[cpu];	local_irq_save(flags);	if (dolock)		raw_spin_lock(&cpu_buffer->reader_lock); 	event = rb_buffer_peek(cpu_buffer, ts, lost_events);	if (event) {		cpu_buffer->lost_events = 0;		rb_advance_reader(cpu_buffer);	} 	if (dolock)		raw_spin_unlock(&cpu_buffer->reader_lock);	local_irq_restore(flags);  out:	preempt_enable(); 	if (event && event->type_len == RINGBUF_TYPE_PADDING)		goto again; 	return event;}EXPORT_SYMBOL_GPL(ring_buffer_consume); /** * ring_buffer_read_prepare - Prepare for a non consuming read of the buffer * @buffer: The ring buffer to read from * @cpu: The cpu buffer to iterate over * * This performs the initial preparations necessary to iterate * through the buffer.  Memory is allocated, buffer recording * is disabled, and the iterator pointer is returned to the caller. * * Disabling buffer recordng prevents the reading from being * corrupted. This is not a consuming read, so a producer is not * expected. * * After a sequence of ring_buffer_read_prepare calls, the user is * expected to make at least one call to ring_buffer_prepare_sync. * Afterwards, ring_buffer_read_start is invoked to get things going * for real. * * This overall must be paired with ring_buffer_finish. */struct ring_buffer_iter *ring_buffer_read_prepare(struct ring_buffer *buffer, int cpu){	struct ring_buffer_per_cpu *cpu_buffer;	struct ring_buffer_iter *iter; 	if (!cpumask_test_cpu(cpu, buffer->cpumask))		return NULL; 	iter = kmalloc(sizeof(*iter), GFP_KERNEL);	if (!iter)		return NULL; 	cpu_buffer = buffer->buffers[cpu]; 	iter->cpu_buffer = cpu_buffer; 	atomic_inc(&buffer->resize_disabled);	atomic_inc(&cpu_buffer->record_disabled); 	return iter;}EXPORT_SYMBOL_GPL(ring_buffer_read_prepare); /** * ring_buffer_read_prepare_sync - Synchronize a set of prepare calls * * All previously invoked ring_buffer_read_prepare calls to prepare * iterators will be synchronized.  Afterwards, read_buffer_read_start * calls on those iterators are allowed. */voidring_buffer_read_prepare_sync(void){	synchronize_sched();}EXPORT_SYMBOL_GPL(ring_buffer_read_prepare_sync); /** * ring_buffer_read_start - start a non consuming read of the buffer * @iter: The iterator returned by ring_buffer_read_prepare * * This finalizes the startup of an iteration through the buffer. * The iterator comes from a call to ring_buffer_read_prepare and * an intervening ring_buffer_read_prepare_sync must have been * performed. * * Must be paired with ring_buffer_finish. */voidring_buffer_read_start(struct ring_buffer_iter *iter){	struct ring_buffer_per_cpu *cpu_buffer;	unsigned long flags; 	if (!iter)		return; 	cpu_buffer = iter->cpu_buffer; 	raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);	arch_spin_lock(&cpu_buffer->lock);	rb_iter_reset(iter);	arch_spin_unlock(&cpu_buffer->lock);	raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);}EXPORT_SYMBOL_GPL(ring_buffer_read_start); /** * ring_buffer_finish - finish reading the iterator of the buffer * @iter: The iterator retrieved by ring_buffer_start * * This re-enables the recording to the buffer, and frees the * iterator. */voidring_buffer_read_finish(struct ring_buffer_iter *iter){	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer; 	/*	 * Ring buffer is disabled from recording, here's a good place	 * to check the integrity of the ring buffer. 	 */	rb_check_pages(cpu_buffer); 	atomic_dec(&cpu_buffer->record_disabled);	atomic_dec(&cpu_buffer->buffer->resize_disabled);	kfree(iter);}EXPORT_SYMBOL_GPL(ring_buffer_read_finish); /** * ring_buffer_read - read the next item in the ring buffer by the iterator * @iter: The ring buffer iterator * @ts: The time stamp of the event read. * * This reads the next event in the ring buffer and increments the iterator. */struct ring_buffer_event *ring_buffer_read(struct ring_buffer_iter *iter, u64 *ts){	struct ring_buffer_event *event;	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;	unsigned long flags; 	raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); again:	event = rb_iter_peek(iter, ts);	if (!event)		goto out; 	if (event->type_len == RINGBUF_TYPE_PADDING)		goto again; 	rb_advance_iter(iter); out:	raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); 	return event;}EXPORT_SYMBOL_GPL(ring_buffer_read); /** * ring_buffer_size - return the size of the ring buffer (in bytes) * @buffer: The ring buffer. */unsigned long ring_buffer_size(struct ring_buffer *buffer, int cpu){	/*	 * Earlier, this method returned	 *	BUF_PAGE_SIZE * buffer->nr_pages	 * Since the nr_pages field is now removed, we have converted this to	 * return the per cpu buffer value.	 */	if (!cpumask_test_cpu(cpu, buffer->cpumask))		return 0; 	return BUF_PAGE_SIZE * buffer->buffers[cpu]->nr_pages;}EXPORT_SYMBOL_GPL(ring_buffer_size); static voidrb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer){	rb_head_page_deactivate(cpu_buffer); 	cpu_buffer->head_page		= list_entry(cpu_buffer->pages, struct buffer_page, list);	local_set(&cpu_buffer->head_page->write, 0);	local_set(&cpu_buffer->head_page->entries, 0);	local_set(&cpu_buffer->head_page->page->commit, 0); 	cpu_buffer->head_page->read = 0; 	cpu_buffer->tail_page = cpu_buffer->head_page;	cpu_buffer->commit_page = cpu_buffer->head_page; 	INIT_LIST_HEAD(&cpu_buffer->reader_page->list);	INIT_LIST_HEAD(&cpu_buffer->new_pages);	local_set(&cpu_buffer->reader_page->write, 0);	local_set(&cpu_buffer->reader_page->entries, 0);	local_set(&cpu_buffer->reader_page->page->commit, 0);	cpu_buffer->reader_page->read = 0; 	local_set(&cpu_buffer->commit_overrun, 0);	local_set(&cpu_buffer->entries_bytes, 0);	local_set(&cpu_buffer->overrun, 0);	local_set(&cpu_buffer->entries, 0);	local_set(&cpu_buffer->committing, 0);	local_set(&cpu_buffer->commits, 0);	cpu_buffer->read = 0;	cpu_buffer->read_bytes = 0; 	cpu_buffer->write_stamp = 0;	cpu_buffer->read_stamp = 0; 	cpu_buffer->lost_events = 0;	cpu_buffer->last_overrun = 0; 	rb_head_page_activate(cpu_buffer);} /** * ring_buffer_reset_cpu - reset a ring buffer per CPU buffer * @buffer: The ring buffer to reset a per cpu buffer of * @cpu: The CPU buffer to be reset */void ring_buffer_reset_cpu(struct ring_buffer *buffer, int cpu){	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];	unsigned long flags; 	if (!cpumask_test_cpu(cpu, buffer->cpumask))		return; 	atomic_inc(&buffer->resize_disabled);	atomic_inc(&cpu_buffer->record_disabled); 	/* Make sure all commits have finished */	synchronize_sched(); 	raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); 	if (RB_WARN_ON(cpu_buffer, local_read(&cpu_buffer->committing)))		goto out; 	arch_spin_lock(&cpu_buffer->lock); 	rb_reset_cpu(cpu_buffer); 	arch_spin_unlock(&cpu_buffer->lock);  out:	raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); 	atomic_dec(&cpu_buffer->record_disabled);	atomic_dec(&buffer->resize_disabled);}EXPORT_SYMBOL_GPL(ring_buffer_reset_cpu); /** * ring_buffer_reset - reset a ring buffer * @buffer: The ring buffer to reset all cpu buffers */void ring_buffer_reset(struct ring_buffer *buffer){	int cpu; 	for_each_buffer_cpu(buffer, cpu)		ring_buffer_reset_cpu(buffer, cpu);}EXPORT_SYMBOL_GPL(ring_buffer_reset); /** * rind_buffer_empty - is the ring buffer empty? * @buffer: The ring buffer to test */int ring_buffer_empty(struct ring_buffer *buffer){	struct ring_buffer_per_cpu *cpu_buffer;	unsigned long flags;	int dolock;	int cpu;	int ret; 	dolock = rb_ok_to_lock(); 	/* yes this is racy, but if you don't like the race, lock the buffer */	for_each_buffer_cpu(buffer, cpu) {		cpu_buffer = buffer->buffers[cpu];		local_irq_save(flags);		if (dolock)			raw_spin_lock(&cpu_buffer->reader_lock);		ret = rb_per_cpu_empty(cpu_buffer);		if (dolock)			raw_spin_unlock(&cpu_buffer->reader_lock);		local_irq_restore(flags); 		if (!ret)			return 0;	} 	return 1;}EXPORT_SYMBOL_GPL(ring_buffer_empty); /** * ring_buffer_empty_cpu - is a cpu buffer of a ring buffer empty? * @buffer: The ring buffer * @cpu: The CPU buffer to test */int ring_buffer_empty_cpu(struct ring_buffer *buffer, int cpu){	struct ring_buffer_per_cpu *cpu_buffer;	unsigned long flags;	int dolock;	int ret; 	if (!cpumask_test_cpu(cpu, buffer->cpumask))		return 1; 	dolock = rb_ok_to_lock(); 	cpu_buffer = buffer->buffers[cpu];	local_irq_save(flags);	if (dolock)		raw_spin_lock(&cpu_buffer->reader_lock);	ret = rb_per_cpu_empty(cpu_buffer);	if (dolock)		raw_spin_unlock(&cpu_buffer->reader_lock);	local_irq_restore(flags); 	return ret;}EXPORT_SYMBOL_GPL(ring_buffer_empty_cpu); #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP/** * ring_buffer_swap_cpu - swap a CPU buffer between two ring buffers * @buffer_a: One buffer to swap with * @buffer_b: The other buffer to swap with * * This function is useful for tracers that want to take a "snapshot" * of a CPU buffer and has another back up buffer lying around. * it is expected that the tracer handles the cpu buffer not being * used at the moment.

Lines 3501–4000 of 4366. Previous lines

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t