// SPDX-License-Identifier: GPL-2.0/* * Performance events ring-buffer code: * *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> *  Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar *  Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> */ #include <linux/perf_event.h>#include <linux/vmalloc.h>#include <linux/slab.h>#include <linux/circ_buf.h>#include <linux/poll.h>#include <linux/nospec.h> #include "internal.h" static void perf_output_wakeup(struct perf_output_handle *handle){	atomic_set(&handle->rb->poll, EPOLLIN); 	handle->event->pending_wakeup = 1;	irq_work_queue(&handle->event->pending_irq);} /* * We need to ensure a later event_id doesn't publish a head when a former * event isn't done writing. However since we need to deal with NMIs we * cannot fully serialize things. * * We only publish the head (and generate a wakeup) when the outer-most * event completes. */static void perf_output_get_handle(struct perf_output_handle *handle){	struct perf_buffer *rb = handle->rb; 	preempt_disable(); 	/*	 * Avoid an explicit LOAD/STORE such that architectures with memops	 * can use them.	 */	(*(volatile unsigned int *)&rb->nest)++;	handle->wakeup = local_read(&rb->wakeup);} static void perf_output_put_handle(struct perf_output_handle *handle){	struct perf_buffer *rb = handle->rb;	unsigned long head;	unsigned int nest; 	/*	 * If this isn't the outermost nesting, we don't have to update	 * @rb->user_page->data_head.	 */	nest = READ_ONCE(rb->nest);	if (nest > 1) {		WRITE_ONCE(rb->nest, nest - 1);		goto out;	} again:	/*	 * In order to avoid publishing a head value that goes backwards,	 * we must ensure the load of @rb->head happens after we've	 * incremented @rb->nest.	 *	 * Otherwise we can observe a @rb->head value before one published	 * by an IRQ/NMI happening between the load and the increment.	 */	barrier();	head = local_read(&rb->head); 	/*	 * IRQ/NMI can happen here and advance @rb->head, causing our	 * load above to be stale.	 */ 	/*	 * Since the mmap() consumer (userspace) can run on a different CPU:	 *	 *   kernel				user	 *	 *   if (LOAD ->data_tail) {		LOAD ->data_head	 *			(A)		smp_rmb()	(C)	 *	STORE $data			LOAD $data	 *	smp_wmb()	(B)		smp_mb()	(D)	 *	STORE ->data_head		STORE ->data_tail	 *   }	 *	 * Where A pairs with D, and B pairs with C.	 *	 * In our case (A) is a control dependency that separates the load of	 * the ->data_tail and the stores of $data. In case ->data_tail	 * indicates there is no room in the buffer to store $data we do not.	 *	 * D needs to be a full barrier since it separates the data READ	 * from the tail WRITE.	 *	 * For B a WMB is sufficient since it separates two WRITEs, and for C	 * an RMB is sufficient since it separates two READs.	 *	 * See perf_output_begin().	 */	smp_wmb(); /* B, matches C */	WRITE_ONCE(rb->user_page->data_head, head); 	/*	 * We must publish the head before decrementing the nest count,	 * otherwise an IRQ/NMI can publish a more recent head value and our	 * write will (temporarily) publish a stale value.	 */	barrier();	WRITE_ONCE(rb->nest, 0); 	/*	 * Ensure we decrement @rb->nest before we validate the @rb->head.	 * Otherwise we cannot be sure we caught the 'last' nested update.	 */	barrier();	if (unlikely(head != local_read(&rb->head))) {		WRITE_ONCE(rb->nest, 1);		goto again;	} 	if (handle->wakeup != local_read(&rb->wakeup))		perf_output_wakeup(handle); out:	preempt_enable();} static __always_inline boolring_buffer_has_space(unsigned long head, unsigned long tail,		      unsigned long data_size, unsigned int size,		      bool backward){	if (!backward)		return CIRC_SPACE(head, tail, data_size) >= size;	else		return CIRC_SPACE(tail, head, data_size) >= size;} static __always_inline int__perf_output_begin(struct perf_output_handle *handle,		    struct perf_sample_data *data,		    struct perf_event *event, unsigned int size,		    bool backward){	struct perf_buffer *rb;	unsigned long tail, offset, head;	int have_lost, page_shift;	struct {		struct perf_event_header header;		u64			 id;		u64			 lost;	} lost_event; 	rcu_read_lock();	/*	 * For inherited events we send all the output towards the parent.	 */	if (event->parent)		event = event->parent; 	rb = rcu_dereference(event->rb);	if (unlikely(!rb))		goto out; 	if (unlikely(rb->paused)) {		if (rb->nr_pages) {			local_inc(&rb->lost);			atomic64_inc(&event->lost_samples);		}		goto out;	} 	handle->rb    = rb;	handle->event = event; 	have_lost = local_read(&rb->lost);	if (unlikely(have_lost)) {		size += sizeof(lost_event);		if (event->attr.sample_id_all)			size += event->id_header_size;	} 	perf_output_get_handle(handle); 	offset = local_read(&rb->head);	do {		head = offset;		tail = READ_ONCE(rb->user_page->data_tail);		if (!rb->overwrite) {			if (unlikely(!ring_buffer_has_space(head, tail,							    perf_data_size(rb),							    size, backward)))				goto fail;		} 		/*		 * The above forms a control dependency barrier separating the		 * @tail load above from the data stores below. Since the @tail		 * load is required to compute the branch to fail below.		 *		 * A, matches D; the full memory barrier userspace SHOULD issue		 * after reading the data and before storing the new tail		 * position.		 *		 * See perf_output_put_handle().		 */ 		if (!backward)			head += size;		else			head -= size;	} while (!local_try_cmpxchg(&rb->head, &offset, head)); 	if (backward) {		offset = head;		head = (u64)(-head);	} 	/*	 * We rely on the implied barrier() by local_cmpxchg() to ensure	 * none of the data stores below can be lifted up by the compiler.	 */ 	if (unlikely(head - local_read(&rb->wakeup) > rb->watermark))		local_add(rb->watermark, &rb->wakeup); 	page_shift = PAGE_SHIFT + page_order(rb); 	handle->page = (offset >> page_shift) & (rb->nr_pages - 1);	offset &= (1UL << page_shift) - 1;	handle->addr = rb->data_pages[handle->page] + offset;	handle->size = (1UL << page_shift) - offset; 	if (unlikely(have_lost)) {		lost_event.header.size = sizeof(lost_event);		lost_event.header.type = PERF_RECORD_LOST;		lost_event.header.misc = 0;		lost_event.id          = event->id;		lost_event.lost        = local_xchg(&rb->lost, 0); 		/* XXX mostly redundant; @data is already fully initializes */		perf_event_header__init_id(&lost_event.header, data, event);		perf_output_put(handle, lost_event);		perf_event__output_id_sample(event, handle, data);	} 	return 0; fail:	local_inc(&rb->lost);	atomic64_inc(&event->lost_samples);	perf_output_put_handle(handle);out:	rcu_read_unlock(); 	return -ENOSPC;} int perf_output_begin_forward(struct perf_output_handle *handle,			      struct perf_sample_data *data,			      struct perf_event *event, unsigned int size){	return __perf_output_begin(handle, data, event, size, false);} int perf_output_begin_backward(struct perf_output_handle *handle,			       struct perf_sample_data *data,			       struct perf_event *event, unsigned int size){	return __perf_output_begin(handle, data, event, size, true);} int perf_output_begin(struct perf_output_handle *handle,		      struct perf_sample_data *data,		      struct perf_event *event, unsigned int size){ 	return __perf_output_begin(handle, data, event, size,				   unlikely(is_write_backward(event)));} unsigned int perf_output_copy(struct perf_output_handle *handle,		      const void *buf, unsigned int len){	return __output_copy(handle, buf, len);} unsigned int perf_output_skip(struct perf_output_handle *handle,			      unsigned int len){	return __output_skip(handle, NULL, len);} void perf_output_end(struct perf_output_handle *handle){	perf_output_put_handle(handle);	rcu_read_unlock();} static voidring_buffer_init(struct perf_buffer *rb, long watermark, int flags){	long max_size = perf_data_size(rb); 	if (watermark)		rb->watermark = min(max_size, watermark); 	if (!rb->watermark)		rb->watermark = max_size / 2; 	if (flags & RING_BUFFER_WRITABLE)		rb->overwrite = 0;	else		rb->overwrite = 1; 	refcount_set(&rb->refcount, 1); 	INIT_LIST_HEAD(&rb->event_list);	spin_lock_init(&rb->event_lock); 	/*	 * perf_output_begin() only checks rb->paused, therefore	 * rb->paused must be true if we have no pages for output.	 */	if (!rb->nr_pages)		rb->paused = 1;} void perf_aux_output_flag(struct perf_output_handle *handle, u64 flags){	/*	 * OVERWRITE is determined by perf_aux_output_end() and can't	 * be passed in directly.	 */	if (WARN_ON_ONCE(flags & PERF_AUX_FLAG_OVERWRITE))		return; 	handle->aux_flags |= flags;}EXPORT_SYMBOL_GPL(perf_aux_output_flag); /* * This is called before hardware starts writing to the AUX area to * obtain an output handle and make sure there's room in the buffer. * When the capture completes, call perf_aux_output_end() to commit * the recorded data to the buffer. * * The ordering is similar to that of perf_output_{begin,end}, with * the exception of (B), which should be taken care of by the pmu * driver, since ordering rules will differ depending on hardware. * * Call this from pmu::start(); see the comment in perf_aux_output_end() * about its use in pmu callbacks. Both can also be called from the PMI * handler if needed. */void *perf_aux_output_begin(struct perf_output_handle *handle,			    struct perf_event *event){	struct perf_event *output_event = event;	unsigned long aux_head, aux_tail;	struct perf_buffer *rb;	unsigned int nest; 	if (output_event->parent)		output_event = output_event->parent; 	/*	 * Since this will typically be open across pmu::add/pmu::del, we	 * grab ring_buffer's refcount instead of holding rcu read lock	 * to make sure it doesn't disappear under us.	 */	rb = ring_buffer_get(output_event);	if (!rb)		return NULL; 	if (!rb_has_aux(rb))		goto err; 	/*	 * If aux_mmap_count is zero, the aux buffer is in perf_mmap_close(),	 * about to get freed, so we leave immediately.	 *	 * Checking rb::aux_mmap_count and rb::refcount has to be done in	 * the same order, see perf_mmap_close. Otherwise we end up freeing	 * aux pages in this path, which is a bug, because in_atomic().	 */	if (!atomic_read(&rb->aux_mmap_count))		goto err; 	if (!refcount_inc_not_zero(&rb->aux_refcount))		goto err; 	nest = READ_ONCE(rb->aux_nest);	/*	 * Nesting is not supported for AUX area, make sure nested	 * writers are caught early	 */	if (WARN_ON_ONCE(nest))		goto err_put; 	WRITE_ONCE(rb->aux_nest, nest + 1); 	aux_head = rb->aux_head; 	handle->rb = rb;	handle->event = event;	handle->head = aux_head;	handle->size = 0;	handle->aux_flags = 0; 	/*	 * In overwrite mode, AUX data stores do not depend on aux_tail,	 * therefore (A) control dependency barrier does not exist. The	 * (B) <-> (C) ordering is still observed by the pmu driver.	 */	if (!rb->aux_overwrite) {		aux_tail = READ_ONCE(rb->user_page->aux_tail);		handle->wakeup = rb->aux_wakeup + rb->aux_watermark;		if (aux_head - aux_tail < perf_aux_size(rb))			handle->size = CIRC_SPACE(aux_head, aux_tail, perf_aux_size(rb)); 		/*		 * handle->size computation depends on aux_tail load; this forms a		 * control dependency barrier separating aux_tail load from aux data		 * store that will be enabled on successful return		 */		if (!handle->size) { /* A, matches D */			event->pending_disable = smp_processor_id();			perf_output_wakeup(handle);			WRITE_ONCE(rb->aux_nest, 0);			goto err_put;		}	} 	return handle->rb->aux_priv; err_put:	/* can't be last */	rb_free_aux(rb); err:	ring_buffer_put(rb);	handle->event = NULL; 	return NULL;}EXPORT_SYMBOL_GPL(perf_aux_output_begin); static __always_inline bool rb_need_aux_wakeup(struct perf_buffer *rb){	if (rb->aux_overwrite)		return false; 	if (rb->aux_head - rb->aux_wakeup >= rb->aux_watermark) {		rb->aux_wakeup = rounddown(rb->aux_head, rb->aux_watermark);		return true;	} 	return false;} /* * Commit the data written by hardware into the ring buffer by adjusting * aux_head and posting a PERF_RECORD_AUX into the perf buffer. It is the * pmu driver's responsibility to observe ordering rules of the hardware, * so that all the data is externally visible before this is called. * * Note: this has to be called from pmu::stop() callback, as the assumption * of the AUX buffer management code is that after pmu::stop(), the AUX * transaction must be stopped and therefore drop the AUX reference count. */void perf_aux_output_end(struct perf_output_handle *handle, unsigned long size){	bool wakeup = !!(handle->aux_flags & PERF_AUX_FLAG_TRUNCATED);	struct perf_buffer *rb = handle->rb;	unsigned long aux_head; 	/* in overwrite mode, driver provides aux_head via handle */	if (rb->aux_overwrite) {		handle->aux_flags |= PERF_AUX_FLAG_OVERWRITE; 		aux_head = handle->head;		rb->aux_head = aux_head;	} else {		handle->aux_flags &= ~PERF_AUX_FLAG_OVERWRITE; 		aux_head = rb->aux_head;		rb->aux_head += size;	} 	/*

Lines 1–500 of 969. Load more