4366 lines · 112.1 KB · C
RawDownload
	if (trace_recursive_lock())		goto out_nocheck; 	cpu = raw_smp_processor_id(); 	if (!cpumask_test_cpu(cpu, buffer->cpumask))		goto out; 	cpu_buffer = buffer->buffers[cpu]; 	if (atomic_read(&cpu_buffer->record_disabled))		goto out; 	if (length > BUF_MAX_DATA_SIZE)		goto out; 	event = rb_reserve_next_event(buffer, cpu_buffer, length);	if (!event)		goto out; 	return event;  out:	trace_recursive_unlock();  out_nocheck:	preempt_enable_notrace();	return NULL;}EXPORT_SYMBOL_GPL(ring_buffer_lock_reserve); static voidrb_update_write_stamp(struct ring_buffer_per_cpu *cpu_buffer,		      struct ring_buffer_event *event){	u64 delta; 	/*	 * The event first in the commit queue updates the	 * time stamp.	 */	if (rb_event_is_commit(cpu_buffer, event)) {		/*		 * A commit event that is first on a page		 * updates the write timestamp with the page stamp		 */		if (!rb_event_index(event))			cpu_buffer->write_stamp =				cpu_buffer->commit_page->page->time_stamp;		else if (event->type_len == RINGBUF_TYPE_TIME_EXTEND) {			delta = event->array[0];			delta <<= TS_SHIFT;			delta += event->time_delta;			cpu_buffer->write_stamp += delta;		} else			cpu_buffer->write_stamp += event->time_delta;	}} static void rb_commit(struct ring_buffer_per_cpu *cpu_buffer,		      struct ring_buffer_event *event){	local_inc(&cpu_buffer->entries);	rb_update_write_stamp(cpu_buffer, event);	rb_end_commit(cpu_buffer);} /** * ring_buffer_unlock_commit - commit a reserved * @buffer: The buffer to commit to * @event: The event pointer to commit. * * This commits the data to the ring buffer, and releases any locks held. * * Must be paired with ring_buffer_lock_reserve. */int ring_buffer_unlock_commit(struct ring_buffer *buffer,			      struct ring_buffer_event *event){	struct ring_buffer_per_cpu *cpu_buffer;	int cpu = raw_smp_processor_id(); 	cpu_buffer = buffer->buffers[cpu]; 	rb_commit(cpu_buffer, event); 	trace_recursive_unlock(); 	preempt_enable_notrace(); 	return 0;}EXPORT_SYMBOL_GPL(ring_buffer_unlock_commit); static inline void rb_event_discard(struct ring_buffer_event *event){	if (event->type_len == RINGBUF_TYPE_TIME_EXTEND)		event = skip_time_extend(event); 	/* array[0] holds the actual length for the discarded event */	event->array[0] = rb_event_data_length(event) - RB_EVNT_HDR_SIZE;	event->type_len = RINGBUF_TYPE_PADDING;	/* time delta must be non zero */	if (!event->time_delta)		event->time_delta = 1;} /* * Decrement the entries to the page that an event is on. * The event does not even need to exist, only the pointer * to the page it is on. This may only be called before the commit * takes place. */static inline voidrb_decrement_entry(struct ring_buffer_per_cpu *cpu_buffer,		   struct ring_buffer_event *event){	unsigned long addr = (unsigned long)event;	struct buffer_page *bpage = cpu_buffer->commit_page;	struct buffer_page *start; 	addr &= PAGE_MASK; 	/* Do the likely case first */	if (likely(bpage->page == (void *)addr)) {		local_dec(&bpage->entries);		return;	} 	/*	 * Because the commit page may be on the reader page we	 * start with the next page and check the end loop there.	 */	rb_inc_page(cpu_buffer, &bpage);	start = bpage;	do {		if (bpage->page == (void *)addr) {			local_dec(&bpage->entries);			return;		}		rb_inc_page(cpu_buffer, &bpage);	} while (bpage != start); 	/* commit not part of this buffer?? */	RB_WARN_ON(cpu_buffer, 1);} /** * ring_buffer_commit_discard - discard an event that has not been committed * @buffer: the ring buffer * @event: non committed event to discard * * Sometimes an event that is in the ring buffer needs to be ignored. * This function lets the user discard an event in the ring buffer * and then that event will not be read later. * * This function only works if it is called before the the item has been * committed. It will try to free the event from the ring buffer * if another event has not been added behind it. * * If another event has been added behind it, it will set the event * up as discarded, and perform the commit. * * If this function is called, do not call ring_buffer_unlock_commit on * the event. */void ring_buffer_discard_commit(struct ring_buffer *buffer,				struct ring_buffer_event *event){	struct ring_buffer_per_cpu *cpu_buffer;	int cpu; 	/* The event is discarded regardless */	rb_event_discard(event); 	cpu = smp_processor_id();	cpu_buffer = buffer->buffers[cpu]; 	/*	 * This must only be called if the event has not been	 * committed yet. Thus we can assume that preemption	 * is still disabled.	 */	RB_WARN_ON(buffer, !local_read(&cpu_buffer->committing)); 	rb_decrement_entry(cpu_buffer, event);	if (rb_try_to_discard(cpu_buffer, event))		goto out; 	/*	 * The commit is still visible by the reader, so we	 * must still update the timestamp.	 */	rb_update_write_stamp(cpu_buffer, event); out:	rb_end_commit(cpu_buffer); 	trace_recursive_unlock(); 	preempt_enable_notrace(); }EXPORT_SYMBOL_GPL(ring_buffer_discard_commit); /** * ring_buffer_write - write data to the buffer without reserving * @buffer: The ring buffer to write to. * @length: The length of the data being written (excluding the event header) * @data: The data to write to the buffer. * * This is like ring_buffer_lock_reserve and ring_buffer_unlock_commit as * one function. If you already have the data to write to the buffer, it * may be easier to simply call this function. * * Note, like ring_buffer_lock_reserve, the length is the length of the data * and not the length of the event which would hold the header. */int ring_buffer_write(struct ring_buffer *buffer,			unsigned long length,			void *data){	struct ring_buffer_per_cpu *cpu_buffer;	struct ring_buffer_event *event;	void *body;	int ret = -EBUSY;	int cpu; 	if (ring_buffer_flags != RB_BUFFERS_ON)		return -EBUSY; 	preempt_disable_notrace(); 	if (atomic_read(&buffer->record_disabled))		goto out; 	cpu = raw_smp_processor_id(); 	if (!cpumask_test_cpu(cpu, buffer->cpumask))		goto out; 	cpu_buffer = buffer->buffers[cpu]; 	if (atomic_read(&cpu_buffer->record_disabled))		goto out; 	if (length > BUF_MAX_DATA_SIZE)		goto out; 	event = rb_reserve_next_event(buffer, cpu_buffer, length);	if (!event)		goto out; 	body = rb_event_data(event); 	memcpy(body, data, length); 	rb_commit(cpu_buffer, event); 	ret = 0; out:	preempt_enable_notrace(); 	return ret;}EXPORT_SYMBOL_GPL(ring_buffer_write); static int rb_per_cpu_empty(struct ring_buffer_per_cpu *cpu_buffer){	struct buffer_page *reader = cpu_buffer->reader_page;	struct buffer_page *head = rb_set_head_page(cpu_buffer);	struct buffer_page *commit = cpu_buffer->commit_page; 	/* In case of error, head will be NULL */	if (unlikely(!head))		return 1; 	return reader->read == rb_page_commit(reader) &&		(commit == reader ||		 (commit == head &&		  head->read == rb_page_commit(commit)));} /** * ring_buffer_record_disable - stop all writes into the buffer * @buffer: The ring buffer to stop writes to. * * This prevents all writes to the buffer. Any attempt to write * to the buffer after this will fail and return NULL. * * The caller should call synchronize_sched() after this. */void ring_buffer_record_disable(struct ring_buffer *buffer){	atomic_inc(&buffer->record_disabled);}EXPORT_SYMBOL_GPL(ring_buffer_record_disable); /** * ring_buffer_record_enable - enable writes to the buffer * @buffer: The ring buffer to enable writes * * Note, multiple disables will need the same number of enables * to truly enable the writing (much like preempt_disable). */void ring_buffer_record_enable(struct ring_buffer *buffer){	atomic_dec(&buffer->record_disabled);}EXPORT_SYMBOL_GPL(ring_buffer_record_enable); /** * ring_buffer_record_off - stop all writes into the buffer * @buffer: The ring buffer to stop writes to. * * This prevents all writes to the buffer. Any attempt to write * to the buffer after this will fail and return NULL. * * This is different than ring_buffer_record_disable() as * it works like an on/off switch, where as the disable() verison * must be paired with a enable(). */void ring_buffer_record_off(struct ring_buffer *buffer){	unsigned int rd;	unsigned int new_rd; 	do {		rd = atomic_read(&buffer->record_disabled);		new_rd = rd | RB_BUFFER_OFF;	} while (atomic_cmpxchg(&buffer->record_disabled, rd, new_rd) != rd);}EXPORT_SYMBOL_GPL(ring_buffer_record_off); /** * ring_buffer_record_on - restart writes into the buffer * @buffer: The ring buffer to start writes to. * * This enables all writes to the buffer that was disabled by * ring_buffer_record_off(). * * This is different than ring_buffer_record_enable() as * it works like an on/off switch, where as the enable() verison * must be paired with a disable(). */void ring_buffer_record_on(struct ring_buffer *buffer){	unsigned int rd;	unsigned int new_rd; 	do {		rd = atomic_read(&buffer->record_disabled);		new_rd = rd & ~RB_BUFFER_OFF;	} while (atomic_cmpxchg(&buffer->record_disabled, rd, new_rd) != rd);}EXPORT_SYMBOL_GPL(ring_buffer_record_on); /** * ring_buffer_record_is_on - return true if the ring buffer can write * @buffer: The ring buffer to see if write is enabled * * Returns true if the ring buffer is in a state that it accepts writes. */int ring_buffer_record_is_on(struct ring_buffer *buffer){	return !atomic_read(&buffer->record_disabled);} /** * ring_buffer_record_disable_cpu - stop all writes into the cpu_buffer * @buffer: The ring buffer to stop writes to. * @cpu: The CPU buffer to stop * * This prevents all writes to the buffer. Any attempt to write * to the buffer after this will fail and return NULL. * * The caller should call synchronize_sched() after this. */void ring_buffer_record_disable_cpu(struct ring_buffer *buffer, int cpu){	struct ring_buffer_per_cpu *cpu_buffer; 	if (!cpumask_test_cpu(cpu, buffer->cpumask))		return; 	cpu_buffer = buffer->buffers[cpu];	atomic_inc(&cpu_buffer->record_disabled);}EXPORT_SYMBOL_GPL(ring_buffer_record_disable_cpu); /** * ring_buffer_record_enable_cpu - enable writes to the buffer * @buffer: The ring buffer to enable writes * @cpu: The CPU to enable. * * Note, multiple disables will need the same number of enables * to truly enable the writing (much like preempt_disable). */void ring_buffer_record_enable_cpu(struct ring_buffer *buffer, int cpu){	struct ring_buffer_per_cpu *cpu_buffer; 	if (!cpumask_test_cpu(cpu, buffer->cpumask))		return; 	cpu_buffer = buffer->buffers[cpu];	atomic_dec(&cpu_buffer->record_disabled);}EXPORT_SYMBOL_GPL(ring_buffer_record_enable_cpu); /* * The total entries in the ring buffer is the running counter * of entries entered into the ring buffer, minus the sum of * the entries read from the ring buffer and the number of * entries that were overwritten. */static inline unsigned longrb_num_of_entries(struct ring_buffer_per_cpu *cpu_buffer){	return local_read(&cpu_buffer->entries) -		(local_read(&cpu_buffer->overrun) + cpu_buffer->read);} /** * ring_buffer_oldest_event_ts - get the oldest event timestamp from the buffer * @buffer: The ring buffer * @cpu: The per CPU buffer to read from. */unsigned long ring_buffer_oldest_event_ts(struct ring_buffer *buffer, int cpu){	unsigned long flags;	struct ring_buffer_per_cpu *cpu_buffer;	struct buffer_page *bpage;	unsigned long ret; 	if (!cpumask_test_cpu(cpu, buffer->cpumask))		return 0; 	cpu_buffer = buffer->buffers[cpu];	raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);	/*	 * if the tail is on reader_page, oldest time stamp is on the reader	 * page	 */	if (cpu_buffer->tail_page == cpu_buffer->reader_page)		bpage = cpu_buffer->reader_page;	else		bpage = rb_set_head_page(cpu_buffer);	ret = bpage->page->time_stamp;	raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); 	return ret;}EXPORT_SYMBOL_GPL(ring_buffer_oldest_event_ts); /** * ring_buffer_bytes_cpu - get the number of bytes consumed in a cpu buffer * @buffer: The ring buffer * @cpu: The per CPU buffer to read from. */unsigned long ring_buffer_bytes_cpu(struct ring_buffer *buffer, int cpu){	struct ring_buffer_per_cpu *cpu_buffer;	unsigned long ret; 	if (!cpumask_test_cpu(cpu, buffer->cpumask))		return 0; 	cpu_buffer = buffer->buffers[cpu];	ret = local_read(&cpu_buffer->entries_bytes) - cpu_buffer->read_bytes; 	return ret;}EXPORT_SYMBOL_GPL(ring_buffer_bytes_cpu); /** * ring_buffer_entries_cpu - get the number of entries in a cpu buffer * @buffer: The ring buffer * @cpu: The per CPU buffer to get the entries from. */unsigned long ring_buffer_entries_cpu(struct ring_buffer *buffer, int cpu){	struct ring_buffer_per_cpu *cpu_buffer; 	if (!cpumask_test_cpu(cpu, buffer->cpumask))		return 0; 	cpu_buffer = buffer->buffers[cpu]; 	return rb_num_of_entries(cpu_buffer);}EXPORT_SYMBOL_GPL(ring_buffer_entries_cpu); /** * ring_buffer_overrun_cpu - get the number of overruns in a cpu_buffer * @buffer: The ring buffer * @cpu: The per CPU buffer to get the number of overruns from */unsigned long ring_buffer_overrun_cpu(struct ring_buffer *buffer, int cpu){	struct ring_buffer_per_cpu *cpu_buffer;

Lines 2501–3000 of 4366. Previous lines

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t