4366 lines · 112.1 KB · C
RawDownload
	nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE); 	/*	 * Don't succeed if resizing is disabled, as a reader might be	 * manipulating the ring buffer and is expecting a sane state while	 * this is true.	 */	if (atomic_read(&buffer->resize_disabled))		return -EBUSY; 	/* prevent another thread from changing buffer sizes */	mutex_lock(&buffer->mutex); 	if (cpu_id == RING_BUFFER_ALL_CPUS) {		/* calculate the pages to update */		for_each_buffer_cpu(buffer, cpu) {			cpu_buffer = buffer->buffers[cpu]; 			cpu_buffer->nr_pages_to_update = nr_pages -							cpu_buffer->nr_pages;			/*			 * nothing more to do for removing pages or no update			 */			if (cpu_buffer->nr_pages_to_update <= 0)				continue;			/*			 * to add pages, make sure all new pages can be			 * allocated without receiving ENOMEM			 */			INIT_LIST_HEAD(&cpu_buffer->new_pages);			if (__rb_allocate_pages(cpu_buffer->nr_pages_to_update,						&cpu_buffer->new_pages, cpu)) {				/* not enough memory for new pages */				err = -ENOMEM;				goto out_err;			}		} 		get_online_cpus();		/*		 * Fire off all the required work handlers		 * We can't schedule on offline CPUs, but it's not necessary		 * since we can change their buffer sizes without any race.		 */		for_each_buffer_cpu(buffer, cpu) {			cpu_buffer = buffer->buffers[cpu];			if (!cpu_buffer->nr_pages_to_update)				continue; 			if (cpu_online(cpu))				schedule_work_on(cpu,						&cpu_buffer->update_pages_work);			else				rb_update_pages(cpu_buffer);		} 		/* wait for all the updates to complete */		for_each_buffer_cpu(buffer, cpu) {			cpu_buffer = buffer->buffers[cpu];			if (!cpu_buffer->nr_pages_to_update)				continue; 			if (cpu_online(cpu))				wait_for_completion(&cpu_buffer->update_done);			cpu_buffer->nr_pages_to_update = 0;		} 		put_online_cpus();	} else {		cpu_buffer = buffer->buffers[cpu_id]; 		if (nr_pages == cpu_buffer->nr_pages)			goto out; 		cpu_buffer->nr_pages_to_update = nr_pages -						cpu_buffer->nr_pages; 		INIT_LIST_HEAD(&cpu_buffer->new_pages);		if (cpu_buffer->nr_pages_to_update > 0 &&			__rb_allocate_pages(cpu_buffer->nr_pages_to_update,					    &cpu_buffer->new_pages, cpu_id)) {			err = -ENOMEM;			goto out_err;		} 		get_online_cpus(); 		if (cpu_online(cpu_id)) {			schedule_work_on(cpu_id,					 &cpu_buffer->update_pages_work);			wait_for_completion(&cpu_buffer->update_done);		} else			rb_update_pages(cpu_buffer); 		cpu_buffer->nr_pages_to_update = 0;		put_online_cpus();	}  out:	/*	 * The ring buffer resize can happen with the ring buffer	 * enabled, so that the update disturbs the tracing as little	 * as possible. But if the buffer is disabled, we do not need	 * to worry about that, and we can take the time to verify	 * that the buffer is not corrupt.	 */	if (atomic_read(&buffer->record_disabled)) {		atomic_inc(&buffer->record_disabled);		/*		 * Even though the buffer was disabled, we must make sure		 * that it is truly disabled before calling rb_check_pages.		 * There could have been a race between checking		 * record_disable and incrementing it.		 */		synchronize_sched();		for_each_buffer_cpu(buffer, cpu) {			cpu_buffer = buffer->buffers[cpu];			rb_check_pages(cpu_buffer);		}		atomic_dec(&buffer->record_disabled);	} 	mutex_unlock(&buffer->mutex);	return size;  out_err:	for_each_buffer_cpu(buffer, cpu) {		struct buffer_page *bpage, *tmp; 		cpu_buffer = buffer->buffers[cpu];		cpu_buffer->nr_pages_to_update = 0; 		if (list_empty(&cpu_buffer->new_pages))			continue; 		list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages,					list) {			list_del_init(&bpage->list);			free_buffer_page(bpage);		}	}	mutex_unlock(&buffer->mutex);	return err;}EXPORT_SYMBOL_GPL(ring_buffer_resize); void ring_buffer_change_overwrite(struct ring_buffer *buffer, int val){	mutex_lock(&buffer->mutex);	if (val)		buffer->flags |= RB_FL_OVERWRITE;	else		buffer->flags &= ~RB_FL_OVERWRITE;	mutex_unlock(&buffer->mutex);}EXPORT_SYMBOL_GPL(ring_buffer_change_overwrite); static inline void *__rb_data_page_index(struct buffer_data_page *bpage, unsigned index){	return bpage->data + index;} static inline void *__rb_page_index(struct buffer_page *bpage, unsigned index){	return bpage->page->data + index;} static inline struct ring_buffer_event *rb_reader_event(struct ring_buffer_per_cpu *cpu_buffer){	return __rb_page_index(cpu_buffer->reader_page,			       cpu_buffer->reader_page->read);} static inline struct ring_buffer_event *rb_iter_head_event(struct ring_buffer_iter *iter){	return __rb_page_index(iter->head_page, iter->head);} static inline unsigned rb_page_commit(struct buffer_page *bpage){	return local_read(&bpage->page->commit);} /* Size is determined by what has been committed */static inline unsigned rb_page_size(struct buffer_page *bpage){	return rb_page_commit(bpage);} static inline unsignedrb_commit_index(struct ring_buffer_per_cpu *cpu_buffer){	return rb_page_commit(cpu_buffer->commit_page);} static inline unsignedrb_event_index(struct ring_buffer_event *event){	unsigned long addr = (unsigned long)event; 	return (addr & ~PAGE_MASK) - BUF_PAGE_HDR_SIZE;} static inline intrb_event_is_commit(struct ring_buffer_per_cpu *cpu_buffer,		   struct ring_buffer_event *event){	unsigned long addr = (unsigned long)event;	unsigned long index; 	index = rb_event_index(event);	addr &= PAGE_MASK; 	return cpu_buffer->commit_page->page == (void *)addr &&		rb_commit_index(cpu_buffer) == index;} static voidrb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer){	unsigned long max_count; 	/*	 * We only race with interrupts and NMIs on this CPU.	 * If we own the commit event, then we can commit	 * all others that interrupted us, since the interruptions	 * are in stack format (they finish before they come	 * back to us). This allows us to do a simple loop to	 * assign the commit to the tail.	 */ again:	max_count = cpu_buffer->nr_pages * 100; 	while (cpu_buffer->commit_page != cpu_buffer->tail_page) {		if (RB_WARN_ON(cpu_buffer, !(--max_count)))			return;		if (RB_WARN_ON(cpu_buffer,			       rb_is_reader_page(cpu_buffer->tail_page)))			return;		local_set(&cpu_buffer->commit_page->page->commit,			  rb_page_write(cpu_buffer->commit_page));		rb_inc_page(cpu_buffer, &cpu_buffer->commit_page);		cpu_buffer->write_stamp =			cpu_buffer->commit_page->page->time_stamp;		/* add barrier to keep gcc from optimizing too much */		barrier();	}	while (rb_commit_index(cpu_buffer) !=	       rb_page_write(cpu_buffer->commit_page)) { 		local_set(&cpu_buffer->commit_page->page->commit,			  rb_page_write(cpu_buffer->commit_page));		RB_WARN_ON(cpu_buffer,			   local_read(&cpu_buffer->commit_page->page->commit) &			   ~RB_WRITE_MASK);		barrier();	} 	/* again, keep gcc from optimizing */	barrier(); 	/*	 * If an interrupt came in just after the first while loop	 * and pushed the tail page forward, we will be left with	 * a dangling commit that will never go forward.	 */	if (unlikely(cpu_buffer->commit_page != cpu_buffer->tail_page))		goto again;} static void rb_reset_reader_page(struct ring_buffer_per_cpu *cpu_buffer){	cpu_buffer->read_stamp = cpu_buffer->reader_page->page->time_stamp;	cpu_buffer->reader_page->read = 0;} static void rb_inc_iter(struct ring_buffer_iter *iter){	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer; 	/*	 * The iterator could be on the reader page (it starts there).	 * But the head could have moved, since the reader was	 * found. Check for this case and assign the iterator	 * to the head page instead of next.	 */	if (iter->head_page == cpu_buffer->reader_page)		iter->head_page = rb_set_head_page(cpu_buffer);	else		rb_inc_page(cpu_buffer, &iter->head_page); 	iter->read_stamp = iter->head_page->page->time_stamp;	iter->head = 0;} /* Slow path, do not inline */static noinline struct ring_buffer_event *rb_add_time_stamp(struct ring_buffer_event *event, u64 delta){	event->type_len = RINGBUF_TYPE_TIME_EXTEND; 	/* Not the first event on the page? */	if (rb_event_index(event)) {		event->time_delta = delta & TS_MASK;		event->array[0] = delta >> TS_SHIFT;	} else {		/* nope, just zero it */		event->time_delta = 0;		event->array[0] = 0;	} 	return skip_time_extend(event);} /** * ring_buffer_update_event - update event type and data * @event: the even to update * @type: the type of event * @length: the size of the event field in the ring buffer * * Update the type and data fields of the event. The length * is the actual size that is written to the ring buffer, * and with this, we can determine what to place into the * data field. */static voidrb_update_event(struct ring_buffer_per_cpu *cpu_buffer,		struct ring_buffer_event *event, unsigned length,		int add_timestamp, u64 delta){	/* Only a commit updates the timestamp */	if (unlikely(!rb_event_is_commit(cpu_buffer, event)))		delta = 0; 	/*	 * If we need to add a timestamp, then we	 * add it to the start of the resevered space.	 */	if (unlikely(add_timestamp)) {		event = rb_add_time_stamp(event, delta);		length -= RB_LEN_TIME_EXTEND;		delta = 0;	} 	event->time_delta = delta;	length -= RB_EVNT_HDR_SIZE;	if (length > RB_MAX_SMALL_DATA || RB_FORCE_8BYTE_ALIGNMENT) {		event->type_len = 0;		event->array[0] = length;	} else		event->type_len = DIV_ROUND_UP(length, RB_ALIGNMENT);} /* * rb_handle_head_page - writer hit the head page * * Returns: +1 to retry page *           0 to continue *          -1 on error */static intrb_handle_head_page(struct ring_buffer_per_cpu *cpu_buffer,		    struct buffer_page *tail_page,		    struct buffer_page *next_page){	struct buffer_page *new_head;	int entries;	int type;	int ret; 	entries = rb_page_entries(next_page); 	/*	 * The hard part is here. We need to move the head	 * forward, and protect against both readers on	 * other CPUs and writers coming in via interrupts.	 */	type = rb_head_page_set_update(cpu_buffer, next_page, tail_page,				       RB_PAGE_HEAD); 	/*	 * type can be one of four:	 *  NORMAL - an interrupt already moved it for us	 *  HEAD   - we are the first to get here.	 *  UPDATE - we are the interrupt interrupting	 *           a current move.	 *  MOVED  - a reader on another CPU moved the next	 *           pointer to its reader page. Give up	 *           and try again.	 */ 	switch (type) {	case RB_PAGE_HEAD:		/*		 * We changed the head to UPDATE, thus		 * it is our responsibility to update		 * the counters.		 */		local_add(entries, &cpu_buffer->overrun);		local_sub(BUF_PAGE_SIZE, &cpu_buffer->entries_bytes); 		/*		 * The entries will be zeroed out when we move the		 * tail page.		 */ 		/* still more to do */		break; 	case RB_PAGE_UPDATE:		/*		 * This is an interrupt that interrupt the		 * previous update. Still more to do.		 */		break;	case RB_PAGE_NORMAL:		/*		 * An interrupt came in before the update		 * and processed this for us.		 * Nothing left to do.		 */		return 1;	case RB_PAGE_MOVED:		/*		 * The reader is on another CPU and just did		 * a swap with our next_page.		 * Try again.		 */		return 1;	default:		RB_WARN_ON(cpu_buffer, 1); /* WTF??? */		return -1;	} 	/*	 * Now that we are here, the old head pointer is	 * set to UPDATE. This will keep the reader from	 * swapping the head page with the reader page.	 * The reader (on another CPU) will spin till	 * we are finished.	 *	 * We just need to protect against interrupts	 * doing the job. We will set the next pointer	 * to HEAD. After that, we set the old pointer	 * to NORMAL, but only if it was HEAD before.	 * otherwise we are an interrupt, and only	 * want the outer most commit to reset it.	 */	new_head = next_page;	rb_inc_page(cpu_buffer, &new_head); 	ret = rb_head_page_set_head(cpu_buffer, new_head, next_page,				    RB_PAGE_NORMAL); 	/*	 * Valid returns are:	 *  HEAD   - an interrupt came in and already set it.	 *  NORMAL - One of two things:	 *            1) We really set it.	 *            2) A bunch of interrupts came in and moved	 *               the page forward again.	 */	switch (ret) {	case RB_PAGE_HEAD:	case RB_PAGE_NORMAL:		/* OK */		break;	default:		RB_WARN_ON(cpu_buffer, 1);		return -1;	} 	/*	 * It is possible that an interrupt came in,	 * set the head up, then more interrupts came in	 * and moved it again. When we get back here,	 * the page would have been set to NORMAL but we	 * just set it back to HEAD.	 *	 * How do you detect this? Well, if that happened	 * the tail page would have moved.	 */	if (ret == RB_PAGE_NORMAL) {		/*		 * If the tail had moved passed next, then we need		 * to reset the pointer.		 */		if (cpu_buffer->tail_page != tail_page &&		    cpu_buffer->tail_page != next_page)			rb_head_page_set_normal(cpu_buffer, new_head,						next_page,						RB_PAGE_HEAD);	} 	/*	 * If this was the outer most commit (the one that	 * changed the original pointer from HEAD to UPDATE),

Lines 1501–2000 of 4366. Previous lines

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t