/* * Performance events core code: * * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> * * For licensing details see kernel-base/COPYING */ #include <linux/fs.h>#include <linux/mm.h>#include <linux/cpu.h>#include <linux/smp.h>#include <linux/idr.h>#include <linux/file.h>#include <linux/poll.h>#include <linux/slab.h>#include <linux/hash.h>#include <linux/tick.h>#include <linux/sysfs.h>#include <linux/dcache.h>#include <linux/percpu.h>#include <linux/ptrace.h>#include <linux/reboot.h>#include <linux/vmstat.h>#include <linux/device.h>#include <linux/export.h>#include <linux/vmalloc.h>#include <linux/hardirq.h>#include <linux/rculist.h>#include <linux/uaccess.h>#include <linux/syscalls.h>#include <linux/anon_inodes.h>#include <linux/kernel_stat.h>#include <linux/perf_event.h>#include <linux/ftrace_event.h>#include <linux/hw_breakpoint.h>#include <linux/mm_types.h>#include <linux/cgroup.h> #include "internal.h" #include <asm/irq_regs.h> struct remote_function_call { struct task_struct *p; int (*func)(void *info); void *info; int ret;}; static void remote_function(void *data){ struct remote_function_call *tfc = data; struct task_struct *p = tfc->p; if (p) { tfc->ret = -EAGAIN; if (task_cpu(p) != smp_processor_id() || !task_curr(p)) return; } tfc->ret = tfc->func(tfc->info);} /** * task_function_call - call a function on the cpu on which a task runs * @p: the task to evaluate * @func: the function to be called * @info: the function call argument * * Calls the function @func when the task is currently running. This might * be on the current CPU, which just calls the function directly * * returns: @func return value, or * -ESRCH - when the process isn't running * -EAGAIN - when the process moved away */static inttask_function_call(struct task_struct *p, int (*func) (void *info), void *info){ struct remote_function_call data = { .p = p, .func = func, .info = info, .ret = -ESRCH, /* No such (running) process */ }; if (task_curr(p)) smp_call_function_single(task_cpu(p), remote_function, &data, 1); return data.ret;} /** * cpu_function_call - call a function on the cpu * @func: the function to be called * @info: the function call argument * * Calls the function @func on the remote cpu. * * returns: @func return value or -ENXIO when the cpu is offline */static int cpu_function_call(int cpu, int (*func) (void *info), void *info){ struct remote_function_call data = { .p = NULL, .func = func, .info = info, .ret = -ENXIO, /* No such CPU */ }; smp_call_function_single(cpu, remote_function, &data, 1); return data.ret;} #define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\ PERF_FLAG_FD_OUTPUT |\ PERF_FLAG_PID_CGROUP) /* * branch priv levels that need permission checks */#define PERF_SAMPLE_BRANCH_PERM_PLM \ (PERF_SAMPLE_BRANCH_KERNEL |\ PERF_SAMPLE_BRANCH_HV) enum event_type_t { EVENT_FLEXIBLE = 0x1, EVENT_PINNED = 0x2, EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,}; /* * perf_sched_events : >0 events exist * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu */struct static_key_deferred perf_sched_events __read_mostly;static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);static DEFINE_PER_CPU(atomic_t, perf_branch_stack_events); static atomic_t nr_mmap_events __read_mostly;static atomic_t nr_comm_events __read_mostly;static atomic_t nr_task_events __read_mostly;static atomic_t nr_freq_events __read_mostly; static LIST_HEAD(pmus);static DEFINE_MUTEX(pmus_lock);static struct srcu_struct pmus_srcu; /* * perf event paranoia level: * -1 - not paranoid at all * 0 - disallow raw tracepoint access for unpriv * 1 - disallow cpu events for unpriv * 2 - disallow kernel profiling for unpriv */int sysctl_perf_event_paranoid __read_mostly = 1; /* Minimum for 512 kiB + 1 user control page */int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */ /* * max perf event sample rate */#define DEFAULT_MAX_SAMPLE_RATE 100000#define DEFAULT_SAMPLE_PERIOD_NS (NSEC_PER_SEC / DEFAULT_MAX_SAMPLE_RATE)#define DEFAULT_CPU_TIME_MAX_PERCENT 25 int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE; static int max_samples_per_tick __read_mostly = DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ);static int perf_sample_period_ns __read_mostly = DEFAULT_SAMPLE_PERIOD_NS; static int perf_sample_allowed_ns __read_mostly = DEFAULT_SAMPLE_PERIOD_NS * DEFAULT_CPU_TIME_MAX_PERCENT / 100; void update_perf_cpu_limits(void){ u64 tmp = perf_sample_period_ns; tmp *= sysctl_perf_cpu_time_max_percent; do_div(tmp, 100); ACCESS_ONCE(perf_sample_allowed_ns) = tmp;} static int perf_rotate_context(struct perf_cpu_context *cpuctx); int perf_proc_update_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos){ int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); if (ret || !write) return ret; max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ); perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate; update_perf_cpu_limits(); return 0;} int sysctl_perf_cpu_time_max_percent __read_mostly = DEFAULT_CPU_TIME_MAX_PERCENT; int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos){ int ret = proc_dointvec(table, write, buffer, lenp, ppos); if (ret || !write) return ret; update_perf_cpu_limits(); return 0;} /* * perf samples are done in some very critical code paths (NMIs). * If they take too much CPU time, the system can lock up and not * get any real work done. This will drop the sample rate when * we detect that events are taking too long. */#define NR_ACCUMULATED_SAMPLES 128static DEFINE_PER_CPU(u64, running_sample_length); void perf_sample_event_took(u64 sample_len_ns){ u64 avg_local_sample_len; u64 local_samples_len; u64 allowed_ns = ACCESS_ONCE(perf_sample_allowed_ns); if (allowed_ns == 0) return; /* decay the counter by 1 average sample */ local_samples_len = __get_cpu_var(running_sample_length); local_samples_len -= local_samples_len/NR_ACCUMULATED_SAMPLES; local_samples_len += sample_len_ns; __get_cpu_var(running_sample_length) = local_samples_len; /* * note: this will be biased artifically low until we have * seen NR_ACCUMULATED_SAMPLES. Doing it this way keeps us * from having to maintain a count. */ avg_local_sample_len = local_samples_len/NR_ACCUMULATED_SAMPLES; if (avg_local_sample_len <= allowed_ns) return; if (max_samples_per_tick <= 1) return; max_samples_per_tick = DIV_ROUND_UP(max_samples_per_tick, 2); sysctl_perf_event_sample_rate = max_samples_per_tick * HZ; perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate; printk_ratelimited(KERN_WARNING "perf samples too long (%lld > %lld), lowering " "kernel.perf_event_max_sample_rate to %d\n", avg_local_sample_len, allowed_ns, sysctl_perf_event_sample_rate); update_perf_cpu_limits();} static atomic64_t perf_event_id; static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx, enum event_type_t event_type); static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx, enum event_type_t event_type, struct task_struct *task); static void update_context_time(struct perf_event_context *ctx);static u64 perf_event_time(struct perf_event *event); void __weak perf_event_print_debug(void) { } extern __weak const char *perf_pmu_name(void){ return "pmu";} static inline u64 perf_clock(void){ return local_clock();} static inline struct perf_cpu_context *__get_cpu_context(struct perf_event_context *ctx){ return this_cpu_ptr(ctx->pmu->pmu_cpu_context);} static void perf_ctx_lock(struct perf_cpu_context *cpuctx, struct perf_event_context *ctx){ raw_spin_lock(&cpuctx->ctx.lock); if (ctx) raw_spin_lock(&ctx->lock);} static void perf_ctx_unlock(struct perf_cpu_context *cpuctx, struct perf_event_context *ctx){ if (ctx) raw_spin_unlock(&ctx->lock); raw_spin_unlock(&cpuctx->ctx.lock);} #ifdef CONFIG_CGROUP_PERF /* * perf_cgroup_info keeps track of time_enabled for a cgroup. * This is a per-cpu dynamically allocated data structure. */struct perf_cgroup_info { u64 time; u64 timestamp;}; struct perf_cgroup { struct cgroup_subsys_state css; struct perf_cgroup_info __percpu *info;}; /* * Must ensure cgroup is pinned (css_get) before calling * this function. In other words, we cannot call this function * if there is no cgroup event for the current CPU context. */static inline struct perf_cgroup *perf_cgroup_from_task(struct task_struct *task){ return container_of(task_css(task, perf_subsys_id), struct perf_cgroup, css);} static inline boolperf_cgroup_match(struct perf_event *event){ struct perf_event_context *ctx = event->ctx; struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); /* @event doesn't care about cgroup */ if (!event->cgrp) return true; /* wants specific cgroup scope but @cpuctx isn't associated with any */ if (!cpuctx->cgrp) return false; /* * Cgroup scoping is recursive. An event enabled for a cgroup is * also enabled for all its descendant cgroups. If @cpuctx's * cgroup is a descendant of @event's (the test covers identity * case), it's a match. */ return cgroup_is_descendant(cpuctx->cgrp->css.cgroup, event->cgrp->css.cgroup);} static inline bool perf_tryget_cgroup(struct perf_event *event){ return css_tryget(&event->cgrp->css);} static inline void perf_put_cgroup(struct perf_event *event){ css_put(&event->cgrp->css);} static inline void perf_detach_cgroup(struct perf_event *event){ perf_put_cgroup(event); event->cgrp = NULL;} static inline int is_cgroup_event(struct perf_event *event){ return event->cgrp != NULL;} static inline u64 perf_cgroup_event_time(struct perf_event *event){ struct perf_cgroup_info *t; t = per_cpu_ptr(event->cgrp->info, event->cpu); return t->time;} static inline void __update_cgrp_time(struct perf_cgroup *cgrp){ struct perf_cgroup_info *info; u64 now; now = perf_clock(); info = this_cpu_ptr(cgrp->info); info->time += now - info->timestamp; info->timestamp = now;} static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx){ struct perf_cgroup *cgrp_out = cpuctx->cgrp; if (cgrp_out) __update_cgrp_time(cgrp_out);} static inline void update_cgrp_time_from_event(struct perf_event *event){ struct perf_cgroup *cgrp; /* * ensure we access cgroup data only when needed and * when we know the cgroup is pinned (css_get) */ if (!is_cgroup_event(event)) return; cgrp = perf_cgroup_from_task(current); /* * Do not update time when cgroup is not active */ if (cgrp == event->cgrp) __update_cgrp_time(event->cgrp);} static inline voidperf_cgroup_set_timestamp(struct task_struct *task, struct perf_event_context *ctx){ struct perf_cgroup *cgrp; struct perf_cgroup_info *info; /* * ctx->lock held by caller * ensure we do not access cgroup data * unless we have the cgroup pinned (css_get) */ if (!task || !ctx->nr_cgroups) return; cgrp = perf_cgroup_from_task(task); info = this_cpu_ptr(cgrp->info); info->timestamp = ctx->timestamp;} #define PERF_CGROUP_SWOUT 0x1 /* cgroup switch out every event */#define PERF_CGROUP_SWIN 0x2 /* cgroup switch in events based on task */ /* * reschedule events based on the cgroup constraint of task. * * mode SWOUT : schedule out everything * mode SWIN : schedule in based on cgroup for next */void perf_cgroup_switch(struct task_struct *task, int mode){ struct perf_cpu_context *cpuctx; struct pmu *pmu; unsigned long flags; /* * disable interrupts to avoid geting nr_cgroup * changes via __perf_event_disable(). Also * avoids preemption. */ local_irq_save(flags); /* * we reschedule only in the presence of cgroup * constrained events. */ rcu_read_lock(); list_for_each_entry_rcu(pmu, &pmus, entry) { cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); if (cpuctx->unique_pmu != pmu) continue; /* ensure we process each cpuctx once */ /* * perf_cgroup_events says at least one * context on this CPU has cgroup events. * * ctx->nr_cgroups reports the number of cgroup * events for a context. */ if (cpuctx->ctx.nr_cgroups > 0) {Lines 1–500 of 8043. Load more
/* * Performance events core code: * * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> * * For licensing details see kernel-base/COPYING */ #include <linux/fs.h> #include <linux/mm.h> #include <linux/cpu.h> #include <linux/smp.h> #include <linux/idr.h> #include <linux/file.h> #include <linux/poll.h> #include <linux/slab.h> #include <linux/hash.h> #include <linux/tick.h> #include <linux/sysfs.h> #include <linux/dcache.h> #include <linux/percpu.h> #include <linux/ptrace.h> #include <linux/reboot.h> #include <linux/vmstat.h> #include <linux/device.h> #include <linux/export.h> #include <linux/vmalloc.h> #include <linux/hardirq.h> #include <linux/rculist.h> #include <linux/uaccess.h> #include <linux/syscalls.h> #include <linux/anon_inodes.h> #include <linux/kernel_stat.h> #include <linux/perf_event.h> #include <linux/ftrace_event.h> #include <linux/hw_breakpoint.h> #include <linux/mm_types.h> #include <linux/cgroup.h> #include "internal.h" #include <asm/irq_regs.h> struct remote_function_call { struct task_struct *p; int (*func)(void *info); void *info; int ret; }; static void remote_function(void *data) { struct remote_function_call *tfc = data; struct task_struct *p = tfc->p; if (p) { tfc->ret = -EAGAIN; if (task_cpu(p) != smp_processor_id() || !task_curr(p)) return; } tfc->ret = tfc->func(tfc->info); } /** * task_function_call - call a function on the cpu on which a task runs * @p: the task to evaluate * @func: the function to be called * @info: the function call argument * * Calls the function @func when the task is currently running. This might * be on the current CPU, which just calls the function directly * * returns: @func return value, or * -ESRCH - when the process isn't running * -EAGAIN - when the process moved away */ static int task_function_call(struct task_struct *p, int (*func) (void *info), void *info) { struct remote_function_call data = { .p = p, .func = func, .info = info, .ret = -ESRCH, /* No such (running) process */ }; if (task_curr(p)) smp_call_function_single(task_cpu(p), remote_function, &data, 1); return data.ret; } /** * cpu_function_call - call a function on the cpu * @func: the function to be called * @info: the function call argument * * Calls the function @func on the remote cpu. * * returns: @func return value or -ENXIO when the cpu is offline */ static int cpu_function_call(int cpu, int (*func) (void *info), void *info) { struct remote_function_call data = { .p = NULL, .func = func, .info = info, .ret = -ENXIO, /* No such CPU */ }; smp_call_function_single(cpu, remote_function, &data, 1); return data.ret; } #define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\ PERF_FLAG_FD_OUTPUT |\ PERF_FLAG_PID_CGROUP) /* * branch priv levels that need permission checks */ #define PERF_SAMPLE_BRANCH_PERM_PLM \ (PERF_SAMPLE_BRANCH_KERNEL |\ PERF_SAMPLE_BRANCH_HV) enum event_type_t { EVENT_FLEXIBLE = 0x1, EVENT_PINNED = 0x2, EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED, }; /* * perf_sched_events : >0 events exist * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu */ struct static_key_deferred perf_sched_events __read_mostly; static DEFINE_PER_CPU(atomic_t, perf_cgroup_events); static DEFINE_PER_CPU(atomic_t, perf_branch_stack_events); static atomic_t nr_mmap_events __read_mostly; static atomic_t nr_comm_events __read_mostly; static atomic_t nr_task_events __read_mostly; static atomic_t nr_freq_events __read_mostly; static LIST_HEAD(pmus); static DEFINE_MUTEX(pmus_lock); static struct srcu_struct pmus_srcu; /* * perf event paranoia level: * -1 - not paranoid at all * 0 - disallow raw tracepoint access for unpriv * 1 - disallow cpu events for unpriv * 2 - disallow kernel profiling for unpriv */ int sysctl_perf_event_paranoid __read_mostly = 1; /* Minimum for 512 kiB + 1 user control page */ int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */ /* * max perf event sample rate */ #define DEFAULT_MAX_SAMPLE_RATE 100000 #define DEFAULT_SAMPLE_PERIOD_NS (NSEC_PER_SEC / DEFAULT_MAX_SAMPLE_RATE) #define DEFAULT_CPU_TIME_MAX_PERCENT 25 int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE; static int max_samples_per_tick __read_mostly = DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ); static int perf_sample_period_ns __read_mostly = DEFAULT_SAMPLE_PERIOD_NS; static int perf_sample_allowed_ns __read_mostly = DEFAULT_SAMPLE_PERIOD_NS * DEFAULT_CPU_TIME_MAX_PERCENT / 100; void update_perf_cpu_limits(void) { u64 tmp = perf_sample_period_ns; tmp *= sysctl_perf_cpu_time_max_percent; do_div(tmp, 100); ACCESS_ONCE(perf_sample_allowed_ns) = tmp; } static int perf_rotate_context(struct perf_cpu_context *cpuctx); int perf_proc_update_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos) { int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); if (ret || !write) return ret; max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ); perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate; update_perf_cpu_limits(); return 0; } int sysctl_perf_cpu_time_max_percent __read_mostly = DEFAULT_CPU_TIME_MAX_PERCENT; int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos) { int ret = proc_dointvec(table, write, buffer, lenp, ppos); if (ret || !write) return ret; update_perf_cpu_limits(); return 0; } /* * perf samples are done in some very critical code paths (NMIs). * If they take too much CPU time, the system can lock up and not * get any real work done. This will drop the sample rate when * we detect that events are taking too long. */ #define NR_ACCUMULATED_SAMPLES 128 static DEFINE_PER_CPU(u64, running_sample_length); void perf_sample_event_took(u64 sample_len_ns) { u64 avg_local_sample_len; u64 local_samples_len; u64 allowed_ns = ACCESS_ONCE(perf_sample_allowed_ns); if (allowed_ns == 0) return; /* decay the counter by 1 average sample */ local_samples_len = __get_cpu_var(running_sample_length); local_samples_len -= local_samples_len/NR_ACCUMULATED_SAMPLES; local_samples_len += sample_len_ns; __get_cpu_var(running_sample_length) = local_samples_len; /* * note: this will be biased artifically low until we have * seen NR_ACCUMULATED_SAMPLES. Doing it this way keeps us * from having to maintain a count. */ avg_local_sample_len = local_samples_len/NR_ACCUMULATED_SAMPLES; if (avg_local_sample_len <= allowed_ns) return; if (max_samples_per_tick <= 1) return; max_samples_per_tick = DIV_ROUND_UP(max_samples_per_tick, 2); sysctl_perf_event_sample_rate = max_samples_per_tick * HZ; perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate; printk_ratelimited(KERN_WARNING "perf samples too long (%lld > %lld), lowering " "kernel.perf_event_max_sample_rate to %d\n", avg_local_sample_len, allowed_ns, sysctl_perf_event_sample_rate); update_perf_cpu_limits(); } static atomic64_t perf_event_id; static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx, enum event_type_t event_type); static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx, enum event_type_t event_type, struct task_struct *task); static void update_context_time(struct perf_event_context *ctx); static u64 perf_event_time(struct perf_event *event); void __weak perf_event_print_debug(void) { } extern __weak const char *perf_pmu_name(void) { return "pmu"; } static inline u64 perf_clock(void) { return local_clock(); } static inline struct perf_cpu_context * __get_cpu_context(struct perf_event_context *ctx) { return this_cpu_ptr(ctx->pmu->pmu_cpu_context); } static void perf_ctx_lock(struct perf_cpu_context *cpuctx, struct perf_event_context *ctx) { raw_spin_lock(&cpuctx->ctx.lock); if (ctx) raw_spin_lock(&ctx->lock); } static void perf_ctx_unlock(struct perf_cpu_context *cpuctx, struct perf_event_context *ctx) { if (ctx) raw_spin_unlock(&ctx->lock); raw_spin_unlock(&cpuctx->ctx.lock); } #ifdef CONFIG_CGROUP_PERF /* * perf_cgroup_info keeps track of time_enabled for a cgroup. * This is a per-cpu dynamically allocated data structure. */ struct perf_cgroup_info { u64 time; u64 timestamp; }; struct perf_cgroup { struct cgroup_subsys_state css; struct perf_cgroup_info __percpu *info; }; /* * Must ensure cgroup is pinned (css_get) before calling * this function. In other words, we cannot call this function * if there is no cgroup event for the current CPU context. */ static inline struct perf_cgroup * perf_cgroup_from_task(struct task_struct *task) { return container_of(task_css(task, perf_subsys_id), struct perf_cgroup, css); } static inline bool perf_cgroup_match(struct perf_event *event) { struct perf_event_context *ctx = event->ctx; struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); /* @event doesn't care about cgroup */ if (!event->cgrp) return true; /* wants specific cgroup scope but @cpuctx isn't associated with any */ if (!cpuctx->cgrp) return false; /* * Cgroup scoping is recursive. An event enabled for a cgroup is * also enabled for all its descendant cgroups. If @cpuctx's * cgroup is a descendant of @event's (the test covers identity * case), it's a match. */ return cgroup_is_descendant(cpuctx->cgrp->css.cgroup, event->cgrp->css.cgroup); } static inline bool perf_tryget_cgroup(struct perf_event *event) { return css_tryget(&event->cgrp->css); } static inline void perf_put_cgroup(struct perf_event *event) { css_put(&event->cgrp->css); } static inline void perf_detach_cgroup(struct perf_event *event) { perf_put_cgroup(event); event->cgrp = NULL; } static inline int is_cgroup_event(struct perf_event *event) { return event->cgrp != NULL; } static inline u64 perf_cgroup_event_time(struct perf_event *event) { struct perf_cgroup_info *t; t = per_cpu_ptr(event->cgrp->info, event->cpu); return t->time; } static inline void __update_cgrp_time(struct perf_cgroup *cgrp) { struct perf_cgroup_info *info; u64 now; now = perf_clock(); info = this_cpu_ptr(cgrp->info); info->time += now - info->timestamp; info->timestamp = now; } static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx) { struct perf_cgroup *cgrp_out = cpuctx->cgrp; if (cgrp_out) __update_cgrp_time(cgrp_out); } static inline void update_cgrp_time_from_event(struct perf_event *event) { struct perf_cgroup *cgrp; /* * ensure we access cgroup data only when needed and * when we know the cgroup is pinned (css_get) */ if (!is_cgroup_event(event)) return; cgrp = perf_cgroup_from_task(current); /* * Do not update time when cgroup is not active */ if (cgrp == event->cgrp) __update_cgrp_time(event->cgrp); } static inline void perf_cgroup_set_timestamp(struct task_struct *task, struct perf_event_context *ctx) { struct perf_cgroup *cgrp; struct perf_cgroup_info *info; /* * ctx->lock held by caller * ensure we do not access cgroup data * unless we have the cgroup pinned (css_get) */ if (!task || !ctx->nr_cgroups) return; cgrp = perf_cgroup_from_task(task); info = this_cpu_ptr(cgrp->info); info->timestamp = ctx->timestamp; } #define PERF_CGROUP_SWOUT 0x1 /* cgroup switch out every event */ #define PERF_CGROUP_SWIN 0x2 /* cgroup switch in events based on task */ /* * reschedule events based on the cgroup constraint of task. * * mode SWOUT : schedule out everything * mode SWIN : schedule in based on cgroup for next */ void perf_cgroup_switch(struct task_struct *task, int mode) { struct perf_cpu_context *cpuctx; struct pmu *pmu; unsigned long flags; /* * disable interrupts to avoid geting nr_cgroup * changes via __perf_event_disable(). Also * avoids preemption. */ local_irq_save(flags); /* * we reschedule only in the presence of cgroup * constrained events. */ rcu_read_lock(); list_for_each_entry_rcu(pmu, &pmus, entry) { cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); if (cpuctx->unique_pmu != pmu) continue; /* ensure we process each cpuctx once */ /* * perf_cgroup_events says at least one * context on this CPU has cgroup events. * * ctx->nr_cgroups reports the number of cgroup * events for a context. */ if (cpuctx->ctx.nr_cgroups > 0) {