4366 lines · 112.1 KB · C
RawDownload
			goto free_pages;		bpage->page = page_address(page);		rb_init_page(bpage->page);	} 	return 0; free_pages:	list_for_each_entry_safe(bpage, tmp, pages, list) {		list_del_init(&bpage->list);		free_buffer_page(bpage);	} 	return -ENOMEM;} static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer,			     unsigned nr_pages){	LIST_HEAD(pages); 	WARN_ON(!nr_pages); 	if (__rb_allocate_pages(nr_pages, &pages, cpu_buffer->cpu))		return -ENOMEM; 	/*	 * The ring buffer page list is a circular list that does not	 * start and end with a list head. All page list items point to	 * other pages.	 */	cpu_buffer->pages = pages.next;	list_del(&pages); 	cpu_buffer->nr_pages = nr_pages; 	rb_check_pages(cpu_buffer); 	return 0;} static struct ring_buffer_per_cpu *rb_allocate_cpu_buffer(struct ring_buffer *buffer, int nr_pages, int cpu){	struct ring_buffer_per_cpu *cpu_buffer;	struct buffer_page *bpage;	struct page *page;	int ret; 	cpu_buffer = kzalloc_node(ALIGN(sizeof(*cpu_buffer), cache_line_size()),				  GFP_KERNEL, cpu_to_node(cpu));	if (!cpu_buffer)		return NULL; 	cpu_buffer->cpu = cpu;	cpu_buffer->buffer = buffer;	raw_spin_lock_init(&cpu_buffer->reader_lock);	lockdep_set_class(&cpu_buffer->reader_lock, buffer->reader_lock_key);	cpu_buffer->lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;	INIT_WORK(&cpu_buffer->update_pages_work, update_pages_handler);	init_completion(&cpu_buffer->update_done); 	bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),			    GFP_KERNEL, cpu_to_node(cpu));	if (!bpage)		goto fail_free_buffer; 	rb_check_bpage(cpu_buffer, bpage); 	cpu_buffer->reader_page = bpage;	page = alloc_pages_node(cpu_to_node(cpu), GFP_KERNEL, 0);	if (!page)		goto fail_free_reader;	bpage->page = page_address(page);	rb_init_page(bpage->page); 	INIT_LIST_HEAD(&cpu_buffer->reader_page->list); 	ret = rb_allocate_pages(cpu_buffer, nr_pages);	if (ret < 0)		goto fail_free_reader; 	cpu_buffer->head_page		= list_entry(cpu_buffer->pages, struct buffer_page, list);	cpu_buffer->tail_page = cpu_buffer->commit_page = cpu_buffer->head_page; 	rb_head_page_activate(cpu_buffer); 	return cpu_buffer;  fail_free_reader:	free_buffer_page(cpu_buffer->reader_page);  fail_free_buffer:	kfree(cpu_buffer);	return NULL;} static void rb_free_cpu_buffer(struct ring_buffer_per_cpu *cpu_buffer){	struct list_head *head = cpu_buffer->pages;	struct buffer_page *bpage, *tmp; 	free_buffer_page(cpu_buffer->reader_page); 	rb_head_page_deactivate(cpu_buffer); 	if (head) {		list_for_each_entry_safe(bpage, tmp, head, list) {			list_del_init(&bpage->list);			free_buffer_page(bpage);		}		bpage = list_entry(head, struct buffer_page, list);		free_buffer_page(bpage);	} 	kfree(cpu_buffer);} #ifdef CONFIG_HOTPLUG_CPUstatic int rb_cpu_notify(struct notifier_block *self,			 unsigned long action, void *hcpu);#endif /** * ring_buffer_alloc - allocate a new ring_buffer * @size: the size in bytes per cpu that is needed. * @flags: attributes to set for the ring buffer. * * Currently the only flag that is available is the RB_FL_OVERWRITE * flag. This flag means that the buffer will overwrite old data * when the buffer wraps. If this flag is not set, the buffer will * drop data when the tail hits the head. */struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags,					struct lock_class_key *key){	struct ring_buffer *buffer;	int bsize;	int cpu, nr_pages; 	/* keep it in its own cache line */	buffer = kzalloc(ALIGN(sizeof(*buffer), cache_line_size()),			 GFP_KERNEL);	if (!buffer)		return NULL; 	if (!alloc_cpumask_var(&buffer->cpumask, GFP_KERNEL))		goto fail_free_buffer; 	nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);	buffer->flags = flags;	buffer->clock = trace_clock_local;	buffer->reader_lock_key = key; 	/* need at least two pages */	if (nr_pages < 2)		nr_pages = 2; 	/*	 * In case of non-hotplug cpu, if the ring-buffer is allocated	 * in early initcall, it will not be notified of secondary cpus.	 * In that off case, we need to allocate for all possible cpus.	 */#ifdef CONFIG_HOTPLUG_CPU	get_online_cpus();	cpumask_copy(buffer->cpumask, cpu_online_mask);#else	cpumask_copy(buffer->cpumask, cpu_possible_mask);#endif	buffer->cpus = nr_cpu_ids; 	bsize = sizeof(void *) * nr_cpu_ids;	buffer->buffers = kzalloc(ALIGN(bsize, cache_line_size()),				  GFP_KERNEL);	if (!buffer->buffers)		goto fail_free_cpumask; 	for_each_buffer_cpu(buffer, cpu) {		buffer->buffers[cpu] =			rb_allocate_cpu_buffer(buffer, nr_pages, cpu);		if (!buffer->buffers[cpu])			goto fail_free_buffers;	} #ifdef CONFIG_HOTPLUG_CPU	buffer->cpu_notify.notifier_call = rb_cpu_notify;	buffer->cpu_notify.priority = 0;	register_cpu_notifier(&buffer->cpu_notify);#endif 	put_online_cpus();	mutex_init(&buffer->mutex); 	return buffer;  fail_free_buffers:	for_each_buffer_cpu(buffer, cpu) {		if (buffer->buffers[cpu])			rb_free_cpu_buffer(buffer->buffers[cpu]);	}	kfree(buffer->buffers);  fail_free_cpumask:	free_cpumask_var(buffer->cpumask);	put_online_cpus();  fail_free_buffer:	kfree(buffer);	return NULL;}EXPORT_SYMBOL_GPL(__ring_buffer_alloc); /** * ring_buffer_free - free a ring buffer. * @buffer: the buffer to free. */voidring_buffer_free(struct ring_buffer *buffer){	int cpu; 	get_online_cpus(); #ifdef CONFIG_HOTPLUG_CPU	unregister_cpu_notifier(&buffer->cpu_notify);#endif 	for_each_buffer_cpu(buffer, cpu)		rb_free_cpu_buffer(buffer->buffers[cpu]); 	put_online_cpus(); 	kfree(buffer->buffers);	free_cpumask_var(buffer->cpumask); 	kfree(buffer);}EXPORT_SYMBOL_GPL(ring_buffer_free); void ring_buffer_set_clock(struct ring_buffer *buffer,			   u64 (*clock)(void)){	buffer->clock = clock;} static void rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer); static inline unsigned long rb_page_entries(struct buffer_page *bpage){	return local_read(&bpage->entries) & RB_WRITE_MASK;} static inline unsigned long rb_page_write(struct buffer_page *bpage){	return local_read(&bpage->write) & RB_WRITE_MASK;} static intrb_remove_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned int nr_pages){	struct list_head *tail_page, *to_remove, *next_page;	struct buffer_page *to_remove_page, *tmp_iter_page;	struct buffer_page *last_page, *first_page;	unsigned int nr_removed;	unsigned long head_bit;	int page_entries; 	head_bit = 0; 	raw_spin_lock_irq(&cpu_buffer->reader_lock);	atomic_inc(&cpu_buffer->record_disabled);	/*	 * We don't race with the readers since we have acquired the reader	 * lock. We also don't race with writers after disabling recording.	 * This makes it easy to figure out the first and the last page to be	 * removed from the list. We unlink all the pages in between including	 * the first and last pages. This is done in a busy loop so that we	 * lose the least number of traces.	 * The pages are freed after we restart recording and unlock readers.	 */	tail_page = &cpu_buffer->tail_page->list; 	/*	 * tail page might be on reader page, we remove the next page	 * from the ring buffer	 */	if (cpu_buffer->tail_page == cpu_buffer->reader_page)		tail_page = rb_list_head(tail_page->next);	to_remove = tail_page; 	/* start of pages to remove */	first_page = list_entry(rb_list_head(to_remove->next),				struct buffer_page, list); 	for (nr_removed = 0; nr_removed < nr_pages; nr_removed++) {		to_remove = rb_list_head(to_remove)->next;		head_bit |= (unsigned long)to_remove & RB_PAGE_HEAD;	} 	next_page = rb_list_head(to_remove)->next; 	/*	 * Now we remove all pages between tail_page and next_page.	 * Make sure that we have head_bit value preserved for the	 * next page	 */	tail_page->next = (struct list_head *)((unsigned long)next_page |						head_bit);	next_page = rb_list_head(next_page);	next_page->prev = tail_page; 	/* make sure pages points to a valid page in the ring buffer */	cpu_buffer->pages = next_page; 	/* update head page */	if (head_bit)		cpu_buffer->head_page = list_entry(next_page,						struct buffer_page, list); 	/*	 * change read pointer to make sure any read iterators reset	 * themselves	 */	cpu_buffer->read = 0; 	/* pages are removed, resume tracing and then free the pages */	atomic_dec(&cpu_buffer->record_disabled);	raw_spin_unlock_irq(&cpu_buffer->reader_lock); 	RB_WARN_ON(cpu_buffer, list_empty(cpu_buffer->pages)); 	/* last buffer page to remove */	last_page = list_entry(rb_list_head(to_remove), struct buffer_page,				list);	tmp_iter_page = first_page; 	do {		to_remove_page = tmp_iter_page;		rb_inc_page(cpu_buffer, &tmp_iter_page); 		/* update the counters */		page_entries = rb_page_entries(to_remove_page);		if (page_entries) {			/*			 * If something was added to this page, it was full			 * since it is not the tail page. So we deduct the			 * bytes consumed in ring buffer from here.			 * No need to update overruns, since this page is			 * deleted from ring buffer and its entries are			 * already accounted for.			 */			local_sub(BUF_PAGE_SIZE, &cpu_buffer->entries_bytes);		} 		/*		 * We have already removed references to this list item, just		 * free up the buffer_page and its page		 */		free_buffer_page(to_remove_page);		nr_removed--; 	} while (to_remove_page != last_page); 	RB_WARN_ON(cpu_buffer, nr_removed); 	return nr_removed == 0;} static intrb_insert_pages(struct ring_buffer_per_cpu *cpu_buffer){	struct list_head *pages = &cpu_buffer->new_pages;	int retries, success; 	raw_spin_lock_irq(&cpu_buffer->reader_lock);	/*	 * We are holding the reader lock, so the reader page won't be swapped	 * in the ring buffer. Now we are racing with the writer trying to	 * move head page and the tail page.	 * We are going to adapt the reader page update process where:	 * 1. We first splice the start and end of list of new pages between	 *    the head page and its previous page.	 * 2. We cmpxchg the prev_page->next to point from head page to the	 *    start of new pages list.	 * 3. Finally, we update the head->prev to the end of new list.	 *	 * We will try this process 10 times, to make sure that we don't keep	 * spinning.	 */	retries = 10;	success = 0;	while (retries--) {		struct list_head *head_page, *prev_page, *r;		struct list_head *last_page, *first_page;		struct list_head *head_page_with_bit; 		head_page = &rb_set_head_page(cpu_buffer)->list;		prev_page = head_page->prev; 		first_page = pages->next;		last_page  = pages->prev; 		head_page_with_bit = (struct list_head *)				     ((unsigned long)head_page | RB_PAGE_HEAD); 		last_page->next = head_page_with_bit;		first_page->prev = prev_page; 		r = cmpxchg(&prev_page->next, head_page_with_bit, first_page); 		if (r == head_page_with_bit) {			/*			 * yay, we replaced the page pointer to our new list,			 * now, we just have to update to head page's prev			 * pointer to point to end of list			 */			head_page->prev = last_page;			success = 1;			break;		}	} 	if (success)		INIT_LIST_HEAD(pages);	/*	 * If we weren't successful in adding in new pages, warn and stop	 * tracing	 */	RB_WARN_ON(cpu_buffer, !success);	raw_spin_unlock_irq(&cpu_buffer->reader_lock); 	/* free pages if they weren't inserted */	if (!success) {		struct buffer_page *bpage, *tmp;		list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages,					 list) {			list_del_init(&bpage->list);			free_buffer_page(bpage);		}	}	return success;} static void rb_update_pages(struct ring_buffer_per_cpu *cpu_buffer){	int success; 	if (cpu_buffer->nr_pages_to_update > 0)		success = rb_insert_pages(cpu_buffer);	else		success = rb_remove_pages(cpu_buffer,					-cpu_buffer->nr_pages_to_update); 	if (success)		cpu_buffer->nr_pages += cpu_buffer->nr_pages_to_update;} static void update_pages_handler(struct work_struct *work){	struct ring_buffer_per_cpu *cpu_buffer = container_of(work,			struct ring_buffer_per_cpu, update_pages_work);	rb_update_pages(cpu_buffer);	complete(&cpu_buffer->update_done);} /** * ring_buffer_resize - resize the ring buffer * @buffer: the buffer to resize. * @size: the new size. * * Minimum size is 2 * BUF_PAGE_SIZE. * * Returns 0 on success and < 0 on failure. */int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size,			int cpu_id){	struct ring_buffer_per_cpu *cpu_buffer;	unsigned nr_pages;	int cpu, err = 0; 	/*	 * Always succeed at resizing a non-existent buffer:	 */	if (!buffer)		return size; 	/* Make sure the requested buffer exists */	if (cpu_id != RING_BUFFER_ALL_CPUS &&	    !cpumask_test_cpu(cpu_id, buffer->cpumask))		return size; 	size = DIV_ROUND_UP(size, BUF_PAGE_SIZE);	size *= BUF_PAGE_SIZE; 	/* we need a minimum of two pages */	if (size < BUF_PAGE_SIZE * 2)		size = BUF_PAGE_SIZE * 2; 

Lines 1001–1500 of 4366. Previous lines

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t