/* * Performance events core code: * *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> *  Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar *  Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> * * For licensing details see kernel-base/COPYING */ #include <linux/fs.h>#include <linux/mm.h>#include <linux/cpu.h>#include <linux/smp.h>#include <linux/idr.h>#include <linux/file.h>#include <linux/poll.h>#include <linux/slab.h>#include <linux/hash.h>#include <linux/tick.h>#include <linux/sysfs.h>#include <linux/dcache.h>#include <linux/percpu.h>#include <linux/ptrace.h>#include <linux/reboot.h>#include <linux/vmstat.h>#include <linux/device.h>#include <linux/export.h>#include <linux/vmalloc.h>#include <linux/hardirq.h>#include <linux/rculist.h>#include <linux/uaccess.h>#include <linux/syscalls.h>#include <linux/anon_inodes.h>#include <linux/kernel_stat.h>#include <linux/cgroup.h>#include <linux/perf_event.h>#include <linux/trace_events.h>#include <linux/hw_breakpoint.h>#include <linux/mm_types.h>#include <linux/module.h>#include <linux/mman.h>#include <linux/compat.h>#include <linux/bpf.h>#include <linux/filter.h>#include <linux/namei.h>#include <linux/parser.h>#include <linux/sched/clock.h>#include <linux/sched/mm.h>#include <linux/proc_ns.h>#include <linux/mount.h> #include "internal.h" #include <asm/irq_regs.h> typedef int (*remote_function_f)(void *); struct remote_function_call {	struct task_struct	*p;	remote_function_f	func;	void			*info;	int			ret;}; static void remote_function(void *data){	struct remote_function_call *tfc = data;	struct task_struct *p = tfc->p; 	if (p) {		/* -EAGAIN */		if (task_cpu(p) != smp_processor_id())			return; 		/*		 * Now that we're on right CPU with IRQs disabled, we can test		 * if we hit the right task without races.		 */ 		tfc->ret = -ESRCH; /* No such (running) process */		if (p != current)			return;	} 	tfc->ret = tfc->func(tfc->info);} /** * task_function_call - call a function on the cpu on which a task runs * @p:		the task to evaluate * @func:	the function to be called * @info:	the function call argument * * Calls the function @func when the task is currently running. This might * be on the current CPU, which just calls the function directly * * returns: @func return value, or *	    -ESRCH  - when the process isn't running *	    -EAGAIN - when the process moved away */static inttask_function_call(struct task_struct *p, remote_function_f func, void *info){	struct remote_function_call data = {		.p	= p,		.func	= func,		.info	= info,		.ret	= -EAGAIN,	};	int ret; 	do {		ret = smp_call_function_single(task_cpu(p), remote_function, &data, 1);		if (!ret)			ret = data.ret;	} while (ret == -EAGAIN); 	return ret;} /** * cpu_function_call - call a function on the cpu * @func:	the function to be called * @info:	the function call argument * * Calls the function @func on the remote cpu. * * returns: @func return value or -ENXIO when the cpu is offline */static int cpu_function_call(int cpu, remote_function_f func, void *info){	struct remote_function_call data = {		.p	= NULL,		.func	= func,		.info	= info,		.ret	= -ENXIO, /* No such CPU */	}; 	smp_call_function_single(cpu, remote_function, &data, 1); 	return data.ret;} static inline struct perf_cpu_context *__get_cpu_context(struct perf_event_context *ctx){	return this_cpu_ptr(ctx->pmu->pmu_cpu_context);} static void perf_ctx_lock(struct perf_cpu_context *cpuctx,			  struct perf_event_context *ctx){	raw_spin_lock(&cpuctx->ctx.lock);	if (ctx)		raw_spin_lock(&ctx->lock);} static void perf_ctx_unlock(struct perf_cpu_context *cpuctx,			    struct perf_event_context *ctx){	if (ctx)		raw_spin_unlock(&ctx->lock);	raw_spin_unlock(&cpuctx->ctx.lock);} #define TASK_TOMBSTONE ((void *)-1L) static bool is_kernel_event(struct perf_event *event){	return READ_ONCE(event->owner) == TASK_TOMBSTONE;} /* * On task ctx scheduling... * * When !ctx->nr_events a task context will not be scheduled. This means * we can disable the scheduler hooks (for performance) without leaving * pending task ctx state. * * This however results in two special cases: * *  - removing the last event from a task ctx; this is relatively straight *    forward and is done in __perf_remove_from_context. * *  - adding the first event to a task ctx; this is tricky because we cannot *    rely on ctx->is_active and therefore cannot use event_function_call(). *    See perf_install_in_context(). * * If ctx->nr_events, then ctx->is_active and cpuctx->task_ctx are set. */ typedef void (*event_f)(struct perf_event *, struct perf_cpu_context *,			struct perf_event_context *, void *); struct event_function_struct {	struct perf_event *event;	event_f func;	void *data;}; static int event_function(void *info){	struct event_function_struct *efs = info;	struct perf_event *event = efs->event;	struct perf_event_context *ctx = event->ctx;	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);	struct perf_event_context *task_ctx = cpuctx->task_ctx;	int ret = 0; 	WARN_ON_ONCE(!irqs_disabled()); 	perf_ctx_lock(cpuctx, task_ctx);	/*	 * Since we do the IPI call without holding ctx->lock things can have	 * changed, double check we hit the task we set out to hit.	 */	if (ctx->task) {		if (ctx->task != current) {			ret = -ESRCH;			goto unlock;		} 		/*		 * We only use event_function_call() on established contexts,		 * and event_function() is only ever called when active (or		 * rather, we'll have bailed in task_function_call() or the		 * above ctx->task != current test), therefore we must have		 * ctx->is_active here.		 */		WARN_ON_ONCE(!ctx->is_active);		/*		 * And since we have ctx->is_active, cpuctx->task_ctx must		 * match.		 */		WARN_ON_ONCE(task_ctx != ctx);	} else {		WARN_ON_ONCE(&cpuctx->ctx != ctx);	} 	efs->func(event, cpuctx, ctx, efs->data);unlock:	perf_ctx_unlock(cpuctx, task_ctx); 	return ret;} static void event_function_call(struct perf_event *event, event_f func, void *data){	struct perf_event_context *ctx = event->ctx;	struct task_struct *task = READ_ONCE(ctx->task); /* verified in event_function */	struct event_function_struct efs = {		.event = event,		.func = func,		.data = data,	}; 	if (!event->parent) {		/*		 * If this is a !child event, we must hold ctx::mutex to		 * stabilize the the event->ctx relation. See		 * perf_event_ctx_lock().		 */		lockdep_assert_held(&ctx->mutex);	} 	if (!task) {		cpu_function_call(event->cpu, event_function, &efs);		return;	} 	if (task == TASK_TOMBSTONE)		return; again:	if (!task_function_call(task, event_function, &efs))		return; 	raw_spin_lock_irq(&ctx->lock);	/*	 * Reload the task pointer, it might have been changed by	 * a concurrent perf_event_context_sched_out().	 */	task = ctx->task;	if (task == TASK_TOMBSTONE) {		raw_spin_unlock_irq(&ctx->lock);		return;	}	if (ctx->is_active) {		raw_spin_unlock_irq(&ctx->lock);		goto again;	}	func(event, NULL, ctx, data);	raw_spin_unlock_irq(&ctx->lock);} /* * Similar to event_function_call() + event_function(), but hard assumes IRQs * are already disabled and we're on the right CPU. */static void event_function_local(struct perf_event *event, event_f func, void *data){	struct perf_event_context *ctx = event->ctx;	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);	struct task_struct *task = READ_ONCE(ctx->task);	struct perf_event_context *task_ctx = NULL; 	WARN_ON_ONCE(!irqs_disabled()); 	if (task) {		if (task == TASK_TOMBSTONE)			return; 		task_ctx = ctx;	} 	perf_ctx_lock(cpuctx, task_ctx); 	task = ctx->task;	if (task == TASK_TOMBSTONE)		goto unlock; 	if (task) {		/*		 * We must be either inactive or active and the right task,		 * otherwise we're screwed, since we cannot IPI to somewhere		 * else.		 */		if (ctx->is_active) {			if (WARN_ON_ONCE(task != current))				goto unlock; 			if (WARN_ON_ONCE(cpuctx->task_ctx != ctx))				goto unlock;		}	} else {		WARN_ON_ONCE(&cpuctx->ctx != ctx);	} 	func(event, cpuctx, ctx, data);unlock:	perf_ctx_unlock(cpuctx, task_ctx);} #define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\		       PERF_FLAG_FD_OUTPUT  |\		       PERF_FLAG_PID_CGROUP |\		       PERF_FLAG_FD_CLOEXEC) /* * branch priv levels that need permission checks */#define PERF_SAMPLE_BRANCH_PERM_PLM \	(PERF_SAMPLE_BRANCH_KERNEL |\	 PERF_SAMPLE_BRANCH_HV) enum event_type_t {	EVENT_FLEXIBLE = 0x1,	EVENT_PINNED = 0x2,	EVENT_TIME = 0x4,	/* see ctx_resched() for details */	EVENT_CPU = 0x8,	EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,}; /* * perf_sched_events : >0 events exist * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu */ static void perf_sched_delayed(struct work_struct *work);DEFINE_STATIC_KEY_FALSE(perf_sched_events);static DECLARE_DELAYED_WORK(perf_sched_work, perf_sched_delayed);static DEFINE_MUTEX(perf_sched_mutex);static atomic_t perf_sched_count; static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);static DEFINE_PER_CPU(int, perf_sched_cb_usages);static DEFINE_PER_CPU(struct pmu_event_list, pmu_sb_events); static atomic_t nr_mmap_events __read_mostly;static atomic_t nr_comm_events __read_mostly;static atomic_t nr_namespaces_events __read_mostly;static atomic_t nr_task_events __read_mostly;static atomic_t nr_freq_events __read_mostly;static atomic_t nr_switch_events __read_mostly; static LIST_HEAD(pmus);static DEFINE_MUTEX(pmus_lock);static struct srcu_struct pmus_srcu;static cpumask_var_t perf_online_mask; /* * perf event paranoia level: *  -1 - not paranoid at all *   0 - disallow raw tracepoint access for unpriv *   1 - disallow cpu events for unpriv *   2 - disallow kernel profiling for unpriv */int sysctl_perf_event_paranoid __read_mostly = 2; /* Minimum for 512 kiB + 1 user control page */int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */ /* * max perf event sample rate */#define DEFAULT_MAX_SAMPLE_RATE		100000#define DEFAULT_SAMPLE_PERIOD_NS	(NSEC_PER_SEC / DEFAULT_MAX_SAMPLE_RATE)#define DEFAULT_CPU_TIME_MAX_PERCENT	25 int sysctl_perf_event_sample_rate __read_mostly	= DEFAULT_MAX_SAMPLE_RATE; static int max_samples_per_tick __read_mostly	= DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ);static int perf_sample_period_ns __read_mostly	= DEFAULT_SAMPLE_PERIOD_NS; static int perf_sample_allowed_ns __read_mostly =	DEFAULT_SAMPLE_PERIOD_NS * DEFAULT_CPU_TIME_MAX_PERCENT / 100; static void update_perf_cpu_limits(void){	u64 tmp = perf_sample_period_ns; 	tmp *= sysctl_perf_cpu_time_max_percent;	tmp = div_u64(tmp, 100);	if (!tmp)		tmp = 1; 	WRITE_ONCE(perf_sample_allowed_ns, tmp);} static int perf_rotate_context(struct perf_cpu_context *cpuctx); int perf_proc_update_handler(struct ctl_table *table, int write,		void __user *buffer, size_t *lenp,		loff_t *ppos){	int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 	if (ret || !write)		return ret; 	/*	 * If throttling is disabled don't allow the write:	 */	if (sysctl_perf_cpu_time_max_percent == 100 ||	    sysctl_perf_cpu_time_max_percent == 0)		return -EINVAL; 	max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ);	perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;	update_perf_cpu_limits(); 	return 0;} int sysctl_perf_cpu_time_max_percent __read_mostly = DEFAULT_CPU_TIME_MAX_PERCENT; int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write,				void __user *buffer, size_t *lenp,				loff_t *ppos){	int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 	if (ret || !write)		return ret; 	if (sysctl_perf_cpu_time_max_percent == 100 ||	    sysctl_perf_cpu_time_max_percent == 0) {		printk(KERN_WARNING		       "perf: Dynamic interrupt throttling disabled, can hang your system!\n");		WRITE_ONCE(perf_sample_allowed_ns, 0);	} else {		update_perf_cpu_limits();	} 	return 0;} /* * perf samples are done in some very critical code paths (NMIs). * If they take too much CPU time, the system can lock up and not * get any real work done.  This will drop the sample rate when * we detect that events are taking too long. */#define NR_ACCUMULATED_SAMPLES 128static DEFINE_PER_CPU(u64, running_sample_length); static u64 __report_avg;static u64 __report_allowed; static void perf_duration_warn(struct irq_work *w){	printk_ratelimited(KERN_INFO		"perf: interrupt took too long (%lld > %lld), lowering "		"kernel.perf_event_max_sample_rate to %d\n",		__report_avg, __report_allowed,		sysctl_perf_event_sample_rate);}

Lines 1–500 of 11242. Load more