lockdep_set_class(&cpu_buffer->reader_lock, buffer->reader_lock_key);	cpu_buffer->lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;	INIT_WORK(&cpu_buffer->update_pages_work, update_pages_handler);	init_completion(&cpu_buffer->update_done);	init_irq_work(&cpu_buffer->irq_work.work, rb_wake_up_waiters);	init_waitqueue_head(&cpu_buffer->irq_work.waiters);	init_waitqueue_head(&cpu_buffer->irq_work.full_waiters);	mutex_init(&cpu_buffer->mapping_lock); 	bpage = alloc_cpu_page(cpu);	if (!bpage)		return NULL; 	rb_check_bpage(cpu_buffer, bpage); 	cpu_buffer->reader_page = bpage; 	if (buffer->range_addr_start) {		/*		 * Range mapped buffers have the same restrictions as memory		 * mapped ones do.		 */		cpu_buffer->mapped = 1;		cpu_buffer->ring_meta = rb_range_meta(buffer, nr_pages, cpu);		bpage->page = rb_range_buffer(cpu_buffer, 0);		if (!bpage->page)			goto fail_free_reader;		if (cpu_buffer->ring_meta->head_buffer)			rb_meta_buffer_update(cpu_buffer, bpage);		bpage->range = 1;	} else if (buffer->remote) {		struct ring_buffer_desc *desc = ring_buffer_desc(buffer->remote->desc, cpu); 		if (!desc)			goto fail_free_reader; 		cpu_buffer->remote = buffer->remote;		cpu_buffer->meta_page = (struct trace_buffer_meta *)(void *)desc->meta_va;		cpu_buffer->nr_pages = nr_pages;		cpu_buffer->subbuf_ids = kcalloc(cpu_buffer->nr_pages + 1,						 sizeof(*cpu_buffer->subbuf_ids), GFP_KERNEL);		if (!cpu_buffer->subbuf_ids)			goto fail_free_reader; 		/* Remote buffers are read-only and immutable */		atomic_inc(&cpu_buffer->record_disabled);		atomic_inc(&cpu_buffer->resize_disabled); 		bpage->page = ring_buffer_desc_page(desc, cpu_buffer->meta_page->reader.id);		if (!bpage->page)			goto fail_free_reader; 		bpage->range = 1;		cpu_buffer->subbuf_ids[0] = bpage;	} else {		int order = cpu_buffer->buffer->subbuf_order;		bpage->page = alloc_cpu_data(cpu, order);		if (!bpage->page)			goto fail_free_reader;	} 	INIT_LIST_HEAD(&cpu_buffer->reader_page->list);	INIT_LIST_HEAD(&cpu_buffer->new_pages); 	ret = rb_allocate_pages(cpu_buffer, nr_pages);	if (ret < 0)		goto fail_free_reader; 	rb_meta_validate_events(cpu_buffer); 	/* If the boot meta was valid then this has already been updated */	meta = cpu_buffer->ring_meta;	if (!meta || !meta->head_buffer ||	    !cpu_buffer->head_page || !cpu_buffer->commit_page || !cpu_buffer->tail_page) {		if (meta && meta->head_buffer &&		    (cpu_buffer->head_page || cpu_buffer->commit_page || cpu_buffer->tail_page)) {			pr_warn("Ring buffer meta buffers not all mapped\n");			if (!cpu_buffer->head_page)				pr_warn("   Missing head_page\n");			if (!cpu_buffer->commit_page)				pr_warn("   Missing commit_page\n");			if (!cpu_buffer->tail_page)				pr_warn("   Missing tail_page\n");		} 		cpu_buffer->head_page			= list_entry(cpu_buffer->pages, struct buffer_page, list);		cpu_buffer->tail_page = cpu_buffer->commit_page = cpu_buffer->head_page; 		rb_head_page_activate(cpu_buffer); 		if (cpu_buffer->ring_meta)			meta->commit_buffer = meta->head_buffer;	} else {		/* The valid meta buffer still needs to activate the head page */		rb_head_page_activate(cpu_buffer);	} 	return_ptr(cpu_buffer);  fail_free_reader:	free_buffer_page(cpu_buffer->reader_page); 	return NULL;} static void rb_free_cpu_buffer(struct ring_buffer_per_cpu *cpu_buffer){	struct list_head *head = cpu_buffer->pages;	struct buffer_page *bpage, *tmp; 	irq_work_sync(&cpu_buffer->irq_work.work); 	if (cpu_buffer->remote)		kfree(cpu_buffer->subbuf_ids); 	free_buffer_page(cpu_buffer->reader_page); 	if (head) {		rb_head_page_deactivate(cpu_buffer); 		list_for_each_entry_safe(bpage, tmp, head, list) {			list_del_init(&bpage->list);			free_buffer_page(bpage);		}		bpage = list_entry(head, struct buffer_page, list);		free_buffer_page(bpage);	} 	free_page((unsigned long)cpu_buffer->free_page); 	kfree(cpu_buffer);} #ifdef CONFIG_RING_BUFFER_PERSISTENT_INJECTstatic void rb_test_inject_invalid_pages(struct trace_buffer *buffer){	struct ring_buffer_per_cpu *cpu_buffer;	struct ring_buffer_cpu_meta *meta;	struct buffer_data_page *dpage;	unsigned long entry_bytes = 0;	unsigned long ptr;	int subbuf_size;	int invalid = 0;	int cpu;	int i; 	if (!(buffer->flags & RB_FL_TESTING))		return; 	guard(preempt)();	cpu = smp_processor_id(); 	cpu_buffer = buffer->buffers[cpu];	if (!cpu_buffer)		return;	meta = cpu_buffer->ring_meta;	if (!meta)		return; 	ptr = (unsigned long)rb_subbufs_from_meta(meta);	subbuf_size = meta->subbuf_size; 	for (i = 0; i < meta->nr_subbufs; i++) {		unsigned long idx = meta->buffers[i]; 		dpage = (void *)(ptr + idx * subbuf_size);		/* Skip unused pages */		if (!rb_data_page_commit(dpage))			continue; 		/*		 * Invalidate even pages or multiples of 5. This will cause 3		 * contiguous invalidated(empty) pages.		 */		if (!(i & 0x1) || !(i % 5)) {			local_add(subbuf_size + 1, &dpage->commit);			invalid++;		} else {			/* Count total commit bytes. */			entry_bytes += rb_data_page_size(dpage);		}	} 	pr_info("Inject invalidated %d pages on CPU%d, total size: %ld\n",		invalid, cpu, (long)entry_bytes);	meta->nr_invalid = invalid;	meta->entry_bytes = entry_bytes;}#else /* !CONFIG_RING_BUFFER_PERSISTENT_INJECT */#define rb_test_inject_invalid_pages(buffer)	do { } while (0)#endif /* Stop recording on a persistent buffer and flush cache if needed. */static int rb_flush_buffer_cb(struct notifier_block *nb, unsigned long event, void *data){	struct trace_buffer *buffer = container_of(nb, struct trace_buffer, flush_nb); 	ring_buffer_record_off(buffer);	rb_test_inject_invalid_pages(buffer);	arch_ring_buffer_flush_range(buffer->range_addr_start, buffer->range_addr_end);	return NOTIFY_DONE;} static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags,					 int order, unsigned long start,					 unsigned long end,					 unsigned long scratch_size,					 struct lock_class_key *key,					 struct ring_buffer_remote *remote){	struct trace_buffer *buffer __free(kfree) = NULL;	long nr_pages;	int subbuf_size;	int bsize;	int cpu;	int ret; 	/* keep it in its own cache line */	buffer = kzalloc(ALIGN(sizeof(*buffer), cache_line_size()),			 GFP_KERNEL);	if (!buffer)		return NULL; 	if (!zalloc_cpumask_var(&buffer->cpumask, GFP_KERNEL))		return NULL; 	buffer->subbuf_order = order;	subbuf_size = (PAGE_SIZE << order);	buffer->subbuf_size = subbuf_size - BUF_PAGE_HDR_SIZE; 	/* Max payload is buffer page size - header (8bytes) */	buffer->max_data_size = buffer->subbuf_size - (sizeof(u32) * 2); 	buffer->flags = flags;	buffer->clock = trace_clock_local;	buffer->reader_lock_key = key; 	init_irq_work(&buffer->irq_work.work, rb_wake_up_waiters);	init_waitqueue_head(&buffer->irq_work.waiters); 	buffer->cpus = nr_cpu_ids; 	bsize = sizeof(void *) * nr_cpu_ids;	buffer->buffers = kzalloc(ALIGN(bsize, cache_line_size()),				  GFP_KERNEL);	if (!buffer->buffers)		goto fail_free_cpumask; 	cpu = raw_smp_processor_id(); 	/* If start/end are specified, then that overrides size */	if (start && end) {		unsigned long buffers_start;		unsigned long ptr;		int n; 		/* Make sure that start is word aligned */		start = ALIGN(start, sizeof(long)); 		/* scratch_size needs to be aligned too */		scratch_size = ALIGN(scratch_size, sizeof(long)); 		/* Subtract the buffer meta data and word aligned */		buffers_start = start + sizeof(struct ring_buffer_cpu_meta);		buffers_start = ALIGN(buffers_start, sizeof(long));		buffers_start += scratch_size; 		/* Calculate the size for the per CPU data */		size = end - buffers_start;		size = size / nr_cpu_ids; 		/*		 * The number of sub-buffers (nr_pages) is determined by the		 * total size allocated minus the meta data size.		 * Then that is divided by the number of per CPU buffers		 * needed, plus account for the integer array index that		 * will be appended to the meta data.		 */		nr_pages = (size - sizeof(struct ring_buffer_cpu_meta)) /			(subbuf_size + sizeof(int));		/* Need at least two pages plus the reader page */		if (nr_pages < 3)			goto fail_free_buffers;  again:		/* Make sure that the size fits aligned */		for (n = 0, ptr = buffers_start; n < nr_cpu_ids; n++) {			ptr += sizeof(struct ring_buffer_cpu_meta) +				sizeof(int) * nr_pages;			ptr = ALIGN(ptr, subbuf_size);			ptr += subbuf_size * nr_pages;		}		if (ptr > end) {			if (nr_pages <= 3)				goto fail_free_buffers;			nr_pages--;			goto again;		} 		/* nr_pages should not count the reader page */		nr_pages--;		buffer->range_addr_start = start;		buffer->range_addr_end = end; 		rb_range_meta_init(buffer, nr_pages, scratch_size);	} else if (remote) {		struct ring_buffer_desc *desc = ring_buffer_desc(remote->desc, cpu); 		buffer->remote = remote;		/* The writer is remote. This ring-buffer is read-only */		atomic_inc(&buffer->record_disabled);		nr_pages = desc->nr_page_va - 1;		if (nr_pages < 2)			goto fail_free_buffers;	} else { 		/* need at least two pages */		nr_pages = DIV_ROUND_UP(size, buffer->subbuf_size);		if (nr_pages < 2)			nr_pages = 2;	} 	cpumask_set_cpu(cpu, buffer->cpumask);	buffer->buffers[cpu] = rb_allocate_cpu_buffer(buffer, nr_pages, cpu);	if (!buffer->buffers[cpu])		goto fail_free_buffers; 	ret = cpuhp_state_add_instance(CPUHP_TRACE_RB_PREPARE, &buffer->node);	if (ret < 0)		goto fail_free_buffers; 	mutex_init(&buffer->mutex); 	/* Persistent ring buffer needs to flush cache before reboot. */	if (start && end) {		buffer->flush_nb.notifier_call = rb_flush_buffer_cb;		atomic_notifier_chain_register(&panic_notifier_list, &buffer->flush_nb);	} 	return_ptr(buffer);  fail_free_buffers:	for_each_buffer_cpu(buffer, cpu) {		if (buffer->buffers[cpu])			rb_free_cpu_buffer(buffer->buffers[cpu]);	}	kfree(buffer->buffers);  fail_free_cpumask:	free_cpumask_var(buffer->cpumask); 	return NULL;} /** * __ring_buffer_alloc - allocate a new ring_buffer * @size: the size in bytes per cpu that is needed. * @flags: attributes to set for the ring buffer. * @key: ring buffer reader_lock_key. * * Currently the only flag that is available is the RB_FL_OVERWRITE * flag. This flag means that the buffer will overwrite old data * when the buffer wraps. If this flag is not set, the buffer will * drop data when the tail hits the head. */struct trace_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags,					struct lock_class_key *key){	/* Default buffer page size - one system page */	return alloc_buffer(size, flags, 0, 0, 0, 0, key, NULL); }EXPORT_SYMBOL_GPL(__ring_buffer_alloc); /** * __ring_buffer_alloc_range - allocate a new ring_buffer from existing memory * @size: the size in bytes per cpu that is needed. * @flags: attributes to set for the ring buffer. * @order: sub-buffer order * @start: start of allocated range * @range_size: size of allocated range * @scratch_size: size of scratch area (for preallocated memory buffers) * @key: ring buffer reader_lock_key. * * Currently the only flag that is available is the RB_FL_OVERWRITE * flag. This flag means that the buffer will overwrite old data * when the buffer wraps. If this flag is not set, the buffer will * drop data when the tail hits the head. */struct trace_buffer *__ring_buffer_alloc_range(unsigned long size, unsigned flags,					       int order, unsigned long start,					       unsigned long range_size,					       unsigned long scratch_size,					       struct lock_class_key *key){	return alloc_buffer(size, flags, order, start, start + range_size,			    scratch_size, key, NULL);} /** * __ring_buffer_alloc_remote - allocate a new ring_buffer from a remote * @remote: Contains a description of the ring-buffer pages and remote callbacks. * @key: ring buffer reader_lock_key. */struct trace_buffer *__ring_buffer_alloc_remote(struct ring_buffer_remote *remote,						struct lock_class_key *key){	return alloc_buffer(0, 0, 0, 0, 0, 0, key, remote);} void *ring_buffer_meta_scratch(struct trace_buffer *buffer, unsigned int *size){	struct ring_buffer_meta *meta;	void *ptr; 	if (!buffer || !buffer->meta)		return NULL; 	meta = buffer->meta; 	ptr = (void *)ALIGN((unsigned long)meta + sizeof(*meta), sizeof(long)); 	if (size)		*size = (void *)meta + meta->buffers_offset - ptr; 	return ptr;} /** * ring_buffer_free - free a ring buffer. * @buffer: the buffer to free. */voidring_buffer_free(struct trace_buffer *buffer){	int cpu; 	if (buffer->range_addr_start && buffer->range_addr_end)		atomic_notifier_chain_unregister(&panic_notifier_list, &buffer->flush_nb); 	cpuhp_state_remove_instance(CPUHP_TRACE_RB_PREPARE, &buffer->node); 	irq_work_sync(&buffer->irq_work.work); 	for_each_buffer_cpu(buffer, cpu)		rb_free_cpu_buffer(buffer->buffers[cpu]); 	kfree(buffer->buffers);	free_cpumask_var(buffer->cpumask); 	kfree(buffer);}EXPORT_SYMBOL_GPL(ring_buffer_free); void ring_buffer_set_clock(struct trace_buffer *buffer,			   u64 (*clock)(void)){	buffer->clock = clock;} void ring_buffer_set_time_stamp_abs(struct trace_buffer *buffer, bool abs){	buffer->time_stamp_abs = abs;} bool ring_buffer_time_stamp_abs(struct trace_buffer *buffer){	return buffer->time_stamp_abs;} static inline unsigned long rb_page_entries(struct buffer_page *bpage){	return local_read(&bpage->entries) & RB_WRITE_MASK;} static inline unsigned long rb_page_write(struct buffer_page *bpage){	return local_read(&bpage->write) & RB_WRITE_MASK;} static boolrb_remove_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned long nr_pages){	struct list_head *tail_page, *to_remove, *next_page;	struct buffer_page *to_remove_page, *tmp_iter_page;	struct buffer_page *last_page, *first_page;	unsigned long nr_removed;	unsigned long head_bit;	int page_entries; 	head_bit = 0; 	raw_spin_lock_irq(&cpu_buffer->reader_lock);	atomic_inc(&cpu_buffer->record_disabled);	/*	 * We don't race with the readers since we have acquired the reader	 * lock. We also don't race with writers after disabling recording.	 * This makes it easy to figure out the first and the last page to be	 * removed from the list. We unlink all the pages in between including

Lines 2501–3000 of 8311. Previous lines Load more