5723 lines · 150.8 KB · C
RawDownload
			 * nothing more to do for removing pages or no update			 */			if (cpu_buffer->nr_pages_to_update <= 0)				continue;			/*			 * to add pages, make sure all new pages can be			 * allocated without receiving ENOMEM			 */			INIT_LIST_HEAD(&cpu_buffer->new_pages);			if (__rb_allocate_pages(cpu_buffer->nr_pages_to_update,						&cpu_buffer->new_pages, cpu)) {				/* not enough memory for new pages */				err = -ENOMEM;				goto out_err;			}		} 		get_online_cpus();		/*		 * Fire off all the required work handlers		 * We can't schedule on offline CPUs, but it's not necessary		 * since we can change their buffer sizes without any race.		 */		for_each_buffer_cpu(buffer, cpu) {			cpu_buffer = buffer->buffers[cpu];			if (!cpu_buffer->nr_pages_to_update)				continue; 			/* Can't run something on an offline CPU. */			if (!cpu_online(cpu)) {				rb_update_pages(cpu_buffer);				cpu_buffer->nr_pages_to_update = 0;			} else {				schedule_work_on(cpu,						&cpu_buffer->update_pages_work);			}		} 		/* wait for all the updates to complete */		for_each_buffer_cpu(buffer, cpu) {			cpu_buffer = buffer->buffers[cpu];			if (!cpu_buffer->nr_pages_to_update)				continue; 			if (cpu_online(cpu))				wait_for_completion(&cpu_buffer->update_done);			cpu_buffer->nr_pages_to_update = 0;		} 		put_online_cpus();	} else {		/* Make sure this CPU has been initialized */		if (!cpumask_test_cpu(cpu_id, buffer->cpumask))			goto out; 		cpu_buffer = buffer->buffers[cpu_id]; 		if (nr_pages == cpu_buffer->nr_pages)			goto out; 		/*		 * Don't succeed if resizing is disabled, as a reader might be		 * manipulating the ring buffer and is expecting a sane state while		 * this is true.		 */		if (atomic_read(&cpu_buffer->resize_disabled)) {			err = -EBUSY;			goto out_err_unlock;		} 		cpu_buffer->nr_pages_to_update = nr_pages -						cpu_buffer->nr_pages; 		INIT_LIST_HEAD(&cpu_buffer->new_pages);		if (cpu_buffer->nr_pages_to_update > 0 &&			__rb_allocate_pages(cpu_buffer->nr_pages_to_update,					    &cpu_buffer->new_pages, cpu_id)) {			err = -ENOMEM;			goto out_err;		} 		get_online_cpus(); 		/* Can't run something on an offline CPU. */		if (!cpu_online(cpu_id))			rb_update_pages(cpu_buffer);		else {			schedule_work_on(cpu_id,					 &cpu_buffer->update_pages_work);			wait_for_completion(&cpu_buffer->update_done);		} 		cpu_buffer->nr_pages_to_update = 0;		put_online_cpus();	}  out:	/*	 * The ring buffer resize can happen with the ring buffer	 * enabled, so that the update disturbs the tracing as little	 * as possible. But if the buffer is disabled, we do not need	 * to worry about that, and we can take the time to verify	 * that the buffer is not corrupt.	 */	if (atomic_read(&buffer->record_disabled)) {		atomic_inc(&buffer->record_disabled);		/*		 * Even though the buffer was disabled, we must make sure		 * that it is truly disabled before calling rb_check_pages.		 * There could have been a race between checking		 * record_disable and incrementing it.		 */		synchronize_rcu();		for_each_buffer_cpu(buffer, cpu) {			cpu_buffer = buffer->buffers[cpu];			rb_check_pages(cpu_buffer);		}		atomic_dec(&buffer->record_disabled);	} 	mutex_unlock(&buffer->mutex);	return size;  out_err:	for_each_buffer_cpu(buffer, cpu) {		struct buffer_page *bpage, *tmp; 		cpu_buffer = buffer->buffers[cpu];		cpu_buffer->nr_pages_to_update = 0; 		if (list_empty(&cpu_buffer->new_pages))			continue; 		list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages,					list) {			list_del_init(&bpage->list);			free_buffer_page(bpage);		}	} out_err_unlock:	mutex_unlock(&buffer->mutex);	return err;}EXPORT_SYMBOL_GPL(ring_buffer_resize); void ring_buffer_change_overwrite(struct trace_buffer *buffer, int val){	mutex_lock(&buffer->mutex);	if (val)		buffer->flags |= RB_FL_OVERWRITE;	else		buffer->flags &= ~RB_FL_OVERWRITE;	mutex_unlock(&buffer->mutex);}EXPORT_SYMBOL_GPL(ring_buffer_change_overwrite); static __always_inline void *__rb_page_index(struct buffer_page *bpage, unsigned index){	return bpage->page->data + index;} static __always_inline struct ring_buffer_event *rb_reader_event(struct ring_buffer_per_cpu *cpu_buffer){	return __rb_page_index(cpu_buffer->reader_page,			       cpu_buffer->reader_page->read);} static __always_inline unsigned rb_page_commit(struct buffer_page *bpage){	return local_read(&bpage->page->commit);} static struct ring_buffer_event *rb_iter_head_event(struct ring_buffer_iter *iter){	struct ring_buffer_event *event;	struct buffer_page *iter_head_page = iter->head_page;	unsigned long commit;	unsigned length; 	if (iter->head != iter->next_event)		return iter->event; 	/*	 * When the writer goes across pages, it issues a cmpxchg which	 * is a mb(), which will synchronize with the rmb here.	 * (see rb_tail_page_update() and __rb_reserve_next())	 */	commit = rb_page_commit(iter_head_page);	smp_rmb();	event = __rb_page_index(iter_head_page, iter->head);	length = rb_event_length(event); 	/*	 * READ_ONCE() doesn't work on functions and we don't want the	 * compiler doing any crazy optimizations with length.	 */	barrier(); 	if ((iter->head + length) > commit || length > BUF_MAX_DATA_SIZE)		/* Writer corrupted the read? */		goto reset; 	memcpy(iter->event, event, length);	/*	 * If the page stamp is still the same after this rmb() then the	 * event was safely copied without the writer entering the page.	 */	smp_rmb(); 	/* Make sure the page didn't change since we read this */	if (iter->page_stamp != iter_head_page->page->time_stamp ||	    commit > rb_page_commit(iter_head_page))		goto reset; 	iter->next_event = iter->head + length;	return iter->event; reset:	/* Reset to the beginning */	iter->page_stamp = iter->read_stamp = iter->head_page->page->time_stamp;	iter->head = 0;	iter->next_event = 0;	iter->missed_events = 1;	return NULL;} /* Size is determined by what has been committed */static __always_inline unsigned rb_page_size(struct buffer_page *bpage){	return rb_page_commit(bpage);} static __always_inline unsignedrb_commit_index(struct ring_buffer_per_cpu *cpu_buffer){	return rb_page_commit(cpu_buffer->commit_page);} static __always_inline unsignedrb_event_index(struct ring_buffer_event *event){	unsigned long addr = (unsigned long)event; 	return (addr & ~PAGE_MASK) - BUF_PAGE_HDR_SIZE;} static void rb_inc_iter(struct ring_buffer_iter *iter){	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer; 	/*	 * The iterator could be on the reader page (it starts there).	 * But the head could have moved, since the reader was	 * found. Check for this case and assign the iterator	 * to the head page instead of next.	 */	if (iter->head_page == cpu_buffer->reader_page)		iter->head_page = rb_set_head_page(cpu_buffer);	else		rb_inc_page(cpu_buffer, &iter->head_page); 	iter->page_stamp = iter->read_stamp = iter->head_page->page->time_stamp;	iter->head = 0;	iter->next_event = 0;} /* * rb_handle_head_page - writer hit the head page * * Returns: +1 to retry page *           0 to continue *          -1 on error */static intrb_handle_head_page(struct ring_buffer_per_cpu *cpu_buffer,		    struct buffer_page *tail_page,		    struct buffer_page *next_page){	struct buffer_page *new_head;	int entries;	int type;	int ret; 	entries = rb_page_entries(next_page); 	/*	 * The hard part is here. We need to move the head	 * forward, and protect against both readers on	 * other CPUs and writers coming in via interrupts.	 */	type = rb_head_page_set_update(cpu_buffer, next_page, tail_page,				       RB_PAGE_HEAD); 	/*	 * type can be one of four:	 *  NORMAL - an interrupt already moved it for us	 *  HEAD   - we are the first to get here.	 *  UPDATE - we are the interrupt interrupting	 *           a current move.	 *  MOVED  - a reader on another CPU moved the next	 *           pointer to its reader page. Give up	 *           and try again.	 */ 	switch (type) {	case RB_PAGE_HEAD:		/*		 * We changed the head to UPDATE, thus		 * it is our responsibility to update		 * the counters.		 */		local_add(entries, &cpu_buffer->overrun);		local_sub(BUF_PAGE_SIZE, &cpu_buffer->entries_bytes); 		/*		 * The entries will be zeroed out when we move the		 * tail page.		 */ 		/* still more to do */		break; 	case RB_PAGE_UPDATE:		/*		 * This is an interrupt that interrupt the		 * previous update. Still more to do.		 */		break;	case RB_PAGE_NORMAL:		/*		 * An interrupt came in before the update		 * and processed this for us.		 * Nothing left to do.		 */		return 1;	case RB_PAGE_MOVED:		/*		 * The reader is on another CPU and just did		 * a swap with our next_page.		 * Try again.		 */		return 1;	default:		RB_WARN_ON(cpu_buffer, 1); /* WTF??? */		return -1;	} 	/*	 * Now that we are here, the old head pointer is	 * set to UPDATE. This will keep the reader from	 * swapping the head page with the reader page.	 * The reader (on another CPU) will spin till	 * we are finished.	 *	 * We just need to protect against interrupts	 * doing the job. We will set the next pointer	 * to HEAD. After that, we set the old pointer	 * to NORMAL, but only if it was HEAD before.	 * otherwise we are an interrupt, and only	 * want the outer most commit to reset it.	 */	new_head = next_page;	rb_inc_page(cpu_buffer, &new_head); 	ret = rb_head_page_set_head(cpu_buffer, new_head, next_page,				    RB_PAGE_NORMAL); 	/*	 * Valid returns are:	 *  HEAD   - an interrupt came in and already set it.	 *  NORMAL - One of two things:	 *            1) We really set it.	 *            2) A bunch of interrupts came in and moved	 *               the page forward again.	 */	switch (ret) {	case RB_PAGE_HEAD:	case RB_PAGE_NORMAL:		/* OK */		break;	default:		RB_WARN_ON(cpu_buffer, 1);		return -1;	} 	/*	 * It is possible that an interrupt came in,	 * set the head up, then more interrupts came in	 * and moved it again. When we get back here,	 * the page would have been set to NORMAL but we	 * just set it back to HEAD.	 *	 * How do you detect this? Well, if that happened	 * the tail page would have moved.	 */	if (ret == RB_PAGE_NORMAL) {		struct buffer_page *buffer_tail_page; 		buffer_tail_page = READ_ONCE(cpu_buffer->tail_page);		/*		 * If the tail had moved passed next, then we need		 * to reset the pointer.		 */		if (buffer_tail_page != tail_page &&		    buffer_tail_page != next_page)			rb_head_page_set_normal(cpu_buffer, new_head,						next_page,						RB_PAGE_HEAD);	} 	/*	 * If this was the outer most commit (the one that	 * changed the original pointer from HEAD to UPDATE),	 * then it is up to us to reset it to NORMAL.	 */	if (type == RB_PAGE_HEAD) {		ret = rb_head_page_set_normal(cpu_buffer, next_page,					      tail_page,					      RB_PAGE_UPDATE);		if (RB_WARN_ON(cpu_buffer,			       ret != RB_PAGE_UPDATE))			return -1;	} 	return 0;} static inline voidrb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer,	      unsigned long tail, struct rb_event_info *info){	struct buffer_page *tail_page = info->tail_page;	struct ring_buffer_event *event;	unsigned long length = info->length; 	/*	 * Only the event that crossed the page boundary	 * must fill the old tail_page with padding.	 */	if (tail >= BUF_PAGE_SIZE) {		/*		 * If the page was filled, then we still need		 * to update the real_end. Reset it to zero		 * and the reader will ignore it.		 */		if (tail == BUF_PAGE_SIZE)			tail_page->real_end = 0; 		local_sub(length, &tail_page->write);		return;	} 	event = __rb_page_index(tail_page, tail); 	/* account for padding bytes */	local_add(BUF_PAGE_SIZE - tail, &cpu_buffer->entries_bytes); 	/*	 * Save the original length to the meta data.	 * This will be used by the reader to add lost event	 * counter.	 */	tail_page->real_end = tail; 	/*	 * If this event is bigger than the minimum size, then	 * we need to be careful that we don't subtract the	 * write counter enough to allow another writer to slip	 * in on this page.	 * We put in a discarded commit instead, to make sure	 * that this space is not used again.	 *	 * If we are less than the minimum size, we don't need to	 * worry about it.	 */	if (tail > (BUF_PAGE_SIZE - RB_EVNT_MIN_SIZE)) {		/* No room for any events */ 		/* Mark the rest of the page with padding */		rb_event_set_padding(event); 		/* Set the write back to the previous setting */		local_sub(length, &tail_page->write);		return;	} 	/* Put in a discarded event */	event->array[0] = (BUF_PAGE_SIZE - tail) - RB_EVNT_HDR_SIZE;	event->type_len = RINGBUF_TYPE_PADDING;	/* time delta must be non zero */	event->time_delta = 1; 	/* Set write to end of buffer */	length = (tail + length) - BUF_PAGE_SIZE;	local_sub(length, &tail_page->write);} static inline void rb_end_commit(struct ring_buffer_per_cpu *cpu_buffer); 

Lines 2001–2500 of 5723. Previous lines

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t