4366 lines · 112.1 KB · C
RawDownload
 */int ring_buffer_swap_cpu(struct ring_buffer *buffer_a,			 struct ring_buffer *buffer_b, int cpu){	struct ring_buffer_per_cpu *cpu_buffer_a;	struct ring_buffer_per_cpu *cpu_buffer_b;	int ret = -EINVAL; 	if (!cpumask_test_cpu(cpu, buffer_a->cpumask) ||	    !cpumask_test_cpu(cpu, buffer_b->cpumask))		goto out; 	cpu_buffer_a = buffer_a->buffers[cpu];	cpu_buffer_b = buffer_b->buffers[cpu]; 	/* At least make sure the two buffers are somewhat the same */	if (cpu_buffer_a->nr_pages != cpu_buffer_b->nr_pages)		goto out; 	ret = -EAGAIN; 	if (ring_buffer_flags != RB_BUFFERS_ON)		goto out; 	if (atomic_read(&buffer_a->record_disabled))		goto out; 	if (atomic_read(&buffer_b->record_disabled))		goto out; 	if (atomic_read(&cpu_buffer_a->record_disabled))		goto out; 	if (atomic_read(&cpu_buffer_b->record_disabled))		goto out; 	/*	 * We can't do a synchronize_sched here because this	 * function can be called in atomic context.	 * Normally this will be called from the same CPU as cpu.	 * If not it's up to the caller to protect this.	 */	atomic_inc(&cpu_buffer_a->record_disabled);	atomic_inc(&cpu_buffer_b->record_disabled); 	ret = -EBUSY;	if (local_read(&cpu_buffer_a->committing))		goto out_dec;	if (local_read(&cpu_buffer_b->committing))		goto out_dec; 	buffer_a->buffers[cpu] = cpu_buffer_b;	buffer_b->buffers[cpu] = cpu_buffer_a; 	cpu_buffer_b->buffer = buffer_a;	cpu_buffer_a->buffer = buffer_b; 	ret = 0; out_dec:	atomic_dec(&cpu_buffer_a->record_disabled);	atomic_dec(&cpu_buffer_b->record_disabled);out:	return ret;}EXPORT_SYMBOL_GPL(ring_buffer_swap_cpu);#endif /* CONFIG_RING_BUFFER_ALLOW_SWAP */ /** * ring_buffer_alloc_read_page - allocate a page to read from buffer * @buffer: the buffer to allocate for. * * This function is used in conjunction with ring_buffer_read_page. * When reading a full page from the ring buffer, these functions * can be used to speed up the process. The calling function should * allocate a few pages first with this function. Then when it * needs to get pages from the ring buffer, it passes the result * of this function into ring_buffer_read_page, which will swap * the page that was allocated, with the read page of the buffer. * * Returns: *  The page allocated, or NULL on error. */void *ring_buffer_alloc_read_page(struct ring_buffer *buffer, int cpu){	struct buffer_data_page *bpage;	struct page *page; 	page = alloc_pages_node(cpu_to_node(cpu),				GFP_KERNEL | __GFP_NORETRY, 0);	if (!page)		return NULL; 	bpage = page_address(page); 	rb_init_page(bpage); 	return bpage;}EXPORT_SYMBOL_GPL(ring_buffer_alloc_read_page); /** * ring_buffer_free_read_page - free an allocated read page * @buffer: the buffer the page was allocate for * @data: the page to free * * Free a page allocated from ring_buffer_alloc_read_page. */void ring_buffer_free_read_page(struct ring_buffer *buffer, void *data){	free_page((unsigned long)data);}EXPORT_SYMBOL_GPL(ring_buffer_free_read_page); /** * ring_buffer_read_page - extract a page from the ring buffer * @buffer: buffer to extract from * @data_page: the page to use allocated from ring_buffer_alloc_read_page * @len: amount to extract * @cpu: the cpu of the buffer to extract * @full: should the extraction only happen when the page is full. * * This function will pull out a page from the ring buffer and consume it. * @data_page must be the address of the variable that was returned * from ring_buffer_alloc_read_page. This is because the page might be used * to swap with a page in the ring buffer. * * for example: *	rpage = ring_buffer_alloc_read_page(buffer); *	if (!rpage) *		return error; *	ret = ring_buffer_read_page(buffer, &rpage, len, cpu, 0); *	if (ret >= 0) *		process_page(rpage, ret); * * When @full is set, the function will not return true unless * the writer is off the reader page. * * Note: it is up to the calling functions to handle sleeps and wakeups. *  The ring buffer can be used anywhere in the kernel and can not *  blindly call wake_up. The layer that uses the ring buffer must be *  responsible for that. * * Returns: *  >=0 if data has been transferred, returns the offset of consumed data. *  <0 if no data has been transferred. */int ring_buffer_read_page(struct ring_buffer *buffer,			  void **data_page, size_t len, int cpu, int full){	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];	struct ring_buffer_event *event;	struct buffer_data_page *bpage;	struct buffer_page *reader;	unsigned long missed_events;	unsigned long flags;	unsigned int commit;	unsigned int read;	u64 save_timestamp;	int ret = -1; 	if (!cpumask_test_cpu(cpu, buffer->cpumask))		goto out; 	/*	 * If len is not big enough to hold the page header, then	 * we can not copy anything.	 */	if (len <= BUF_PAGE_HDR_SIZE)		goto out; 	len -= BUF_PAGE_HDR_SIZE; 	if (!data_page)		goto out; 	bpage = *data_page;	if (!bpage)		goto out; 	raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); 	reader = rb_get_reader_page(cpu_buffer);	if (!reader)		goto out_unlock; 	event = rb_reader_event(cpu_buffer); 	read = reader->read;	commit = rb_page_commit(reader); 	/* Check if any events were dropped */	missed_events = cpu_buffer->lost_events; 	/*	 * If this page has been partially read or	 * if len is not big enough to read the rest of the page or	 * a writer is still on the page, then	 * we must copy the data from the page to the buffer.	 * Otherwise, we can simply swap the page with the one passed in.	 */	if (read || (len < (commit - read)) ||	    cpu_buffer->reader_page == cpu_buffer->commit_page) {		struct buffer_data_page *rpage = cpu_buffer->reader_page->page;		unsigned int rpos = read;		unsigned int pos = 0;		unsigned int size; 		if (full)			goto out_unlock; 		if (len > (commit - read))			len = (commit - read); 		/* Always keep the time extend and data together */		size = rb_event_ts_length(event); 		if (len < size)			goto out_unlock; 		/* save the current timestamp, since the user will need it */		save_timestamp = cpu_buffer->read_stamp; 		/* Need to copy one event at a time */		do {			/* We need the size of one event, because			 * rb_advance_reader only advances by one event,			 * whereas rb_event_ts_length may include the size of			 * one or two events.			 * We have already ensured there's enough space if this			 * is a time extend. */			size = rb_event_length(event);			memcpy(bpage->data + pos, rpage->data + rpos, size); 			len -= size; 			rb_advance_reader(cpu_buffer);			rpos = reader->read;			pos += size; 			if (rpos >= commit)				break; 			event = rb_reader_event(cpu_buffer);			/* Always keep the time extend and data together */			size = rb_event_ts_length(event);		} while (len >= size); 		/* update bpage */		local_set(&bpage->commit, pos);		bpage->time_stamp = save_timestamp; 		/* we copied everything to the beginning */		read = 0;	} else {		/* update the entry counter */		cpu_buffer->read += rb_page_entries(reader);		cpu_buffer->read_bytes += BUF_PAGE_SIZE; 		/* swap the pages */		rb_init_page(bpage);		bpage = reader->page;		reader->page = *data_page;		local_set(&reader->write, 0);		local_set(&reader->entries, 0);		reader->read = 0;		*data_page = bpage; 		/*		 * Use the real_end for the data size,		 * This gives us a chance to store the lost events		 * on the page.		 */		if (reader->real_end)			local_set(&bpage->commit, reader->real_end);	}	ret = read; 	cpu_buffer->lost_events = 0; 	commit = local_read(&bpage->commit);	/*	 * Set a flag in the commit field if we lost events	 */	if (missed_events) {		/* If there is room at the end of the page to save the		 * missed events, then record it there.		 */		if (BUF_PAGE_SIZE - commit >= sizeof(missed_events)) {			memcpy(&bpage->data[commit], &missed_events,			       sizeof(missed_events));			local_add(RB_MISSED_STORED, &bpage->commit);			commit += sizeof(missed_events);		}		local_add(RB_MISSED_EVENTS, &bpage->commit);	} 	/*	 * This page may be off to user land. Zero it out here.	 */	if (commit < BUF_PAGE_SIZE)		memset(&bpage->data[commit], 0, BUF_PAGE_SIZE - commit);  out_unlock:	raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);  out:	return ret;}EXPORT_SYMBOL_GPL(ring_buffer_read_page); #ifdef CONFIG_HOTPLUG_CPUstatic int rb_cpu_notify(struct notifier_block *self,			 unsigned long action, void *hcpu){	struct ring_buffer *buffer =		container_of(self, struct ring_buffer, cpu_notify);	long cpu = (long)hcpu;	int cpu_i, nr_pages_same;	unsigned int nr_pages; 	switch (action) {	case CPU_UP_PREPARE:	case CPU_UP_PREPARE_FROZEN:		if (cpumask_test_cpu(cpu, buffer->cpumask))			return NOTIFY_OK; 		nr_pages = 0;		nr_pages_same = 1;		/* check if all cpu sizes are same */		for_each_buffer_cpu(buffer, cpu_i) {			/* fill in the size from first enabled cpu */			if (nr_pages == 0)				nr_pages = buffer->buffers[cpu_i]->nr_pages;			if (nr_pages != buffer->buffers[cpu_i]->nr_pages) {				nr_pages_same = 0;				break;			}		}		/* allocate minimum pages, user can later expand it */		if (!nr_pages_same)			nr_pages = 2;		buffer->buffers[cpu] =			rb_allocate_cpu_buffer(buffer, nr_pages, cpu);		if (!buffer->buffers[cpu]) {			WARN(1, "failed to allocate ring buffer on CPU %ld\n",			     cpu);			return NOTIFY_OK;		}		smp_wmb();		cpumask_set_cpu(cpu, buffer->cpumask);		break;	case CPU_DOWN_PREPARE:	case CPU_DOWN_PREPARE_FROZEN:		/*		 * Do nothing.		 *  If we were to free the buffer, then the user would		 *  lose any trace that was in the buffer.		 */		break;	default:		break;	}	return NOTIFY_OK;}#endif

Lines 4001–4366 of 4366. Previous lines

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t