History

Merge branch 'perf/hw-branch-sampling' into perf/core

Merge reason: The 'perf record -b' hardware branch sampling feature is ready for upstream.

Ingo Molnar committed 14 years ago

Commit bea95c12 parents f9b4eeb 24bff2d33 files · +2017 −243Browse files

Showing the changes to kernel (2 of 33 files). Show all

Unified · Split

2 files
  1. kernel/events/core.c +177 −0
  2. kernel/events/hw_breakpoint.c +6 −0
kernel/events/core.c+177−0
@@ -118,6 +118,13 @@ static int cpu_function_call(int cpu, int (*func) (void *info), void *info) · 117 unmodified lines
118118 PERF_FLAG_FD_OUTPUT |\
119119 PERF_FLAG_PID_CGROUP)
120120
121+/*
122+ * branch priv levels that need permission checks
123+ */
124+#define PERF_SAMPLE_BRANCH_PERM_PLM \
125+ (PERF_SAMPLE_BRANCH_KERNEL |\
126+ PERF_SAMPLE_BRANCH_HV)
127+
121128 enum event_type_t {
122129 EVENT_FLEXIBLE = 0x1,
123130 EVENT_PINNED = 0x2,
@@ -130,6 +137,7 @@ enum event_type_t { · 6 unmodified lines
130137 */
131138 struct static_key_deferred perf_sched_events __read_mostly;
132139 static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
140+static DEFINE_PER_CPU(atomic_t, perf_branch_stack_events);
133141
134142 static atomic_t nr_mmap_events __read_mostly;
135143 static atomic_t nr_comm_events __read_mostly;
@@ -881,6 +889,9 @@ list_add_event(struct perf_event *event, struct perf_event_context *ctx) · 745 unmodified lines
881889 if (is_cgroup_event(event))
882890 ctx->nr_cgroups++;
883891
892+ if (has_branch_stack(event))
893+ ctx->nr_branch_stack++;
894+
884895 list_add_rcu(&event->event_entry, &ctx->event_list);
885896 if (!ctx->nr_events)
886897 perf_pmu_rotate_start(ctx->pmu);
@@ -1020,6 +1031,9 @@ list_del_event(struct perf_event *event, struct perf_event_context *ctx) · 133 unmodified lines
10201031 cpuctx->cgrp = NULL;
10211032 }
10221033
1034+ if (has_branch_stack(event))
1035+ ctx->nr_branch_stack--;
1036+
10231037 ctx->nr_events--;
10241038 if (event->attr.inherit_stat)
10251039 ctx->nr_stat--;
@@ -2194,6 +2208,66 @@ static void perf_event_context_sched_in(struct perf_event_context *ctx, · 1168 unmodified lines
21942208 perf_pmu_rotate_start(ctx->pmu);
21952209 }
21962210
2211+/*
2212+ * When sampling the branck stack in system-wide, it may be necessary
2213+ * to flush the stack on context switch. This happens when the branch
2214+ * stack does not tag its entries with the pid of the current task.
2215+ * Otherwise it becomes impossible to associate a branch entry with a
2216+ * task. This ambiguity is more likely to appear when the branch stack
2217+ * supports priv level filtering and the user sets it to monitor only
2218+ * at the user level (which could be a useful measurement in system-wide
2219+ * mode). In that case, the risk is high of having a branch stack with
2220+ * branch from multiple tasks. Flushing may mean dropping the existing
2221+ * entries or stashing them somewhere in the PMU specific code layer.
2222+ *
2223+ * This function provides the context switch callback to the lower code
2224+ * layer. It is invoked ONLY when there is at least one system-wide context
2225+ * with at least one active event using taken branch sampling.
2226+ */
2227+static void perf_branch_stack_sched_in(struct task_struct *prev,
2228+ struct task_struct *task)
2229+{
2230+ struct perf_cpu_context *cpuctx;
2231+ struct pmu *pmu;
2232+ unsigned long flags;
2233+
2234+ /* no need to flush branch stack if not changing task */
2235+ if (prev == task)
2236+ return;
2237+
2238+ local_irq_save(flags);
2239+
2240+ rcu_read_lock();
2241+
2242+ list_for_each_entry_rcu(pmu, &pmus, entry) {
2243+ cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
2244+
2245+ /*
2246+ * check if the context has at least one
2247+ * event using PERF_SAMPLE_BRANCH_STACK
2248+ */
2249+ if (cpuctx->ctx.nr_branch_stack > 0
2250+ && pmu->flush_branch_stack) {
2251+
2252+ pmu = cpuctx->ctx.pmu;
2253+
2254+ perf_ctx_lock(cpuctx, cpuctx->task_ctx);
2255+
2256+ perf_pmu_disable(pmu);
2257+
2258+ pmu->flush_branch_stack();
2259+
2260+ perf_pmu_enable(pmu);
2261+
2262+ perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
2263+ }
2264+ }
2265+
2266+ rcu_read_unlock();
2267+
2268+ local_irq_restore(flags);
2269+}
2270+
21972271 /*
21982272 * Called from scheduler to add the events of the current task
21992273 * with interrupts disabled.
@@ -2225,6 +2299,10 @@ void __perf_event_task_sched_in(struct task_struct *prev, · 25 unmodified lines
22252299 */
22262300 if (atomic_read(&__get_cpu_var(perf_cgroup_events)))
22272301 perf_cgroup_sched_in(prev, task);
2302+
2303+ /* check for system-wide branch_stack events */
2304+ if (atomic_read(&__get_cpu_var(perf_branch_stack_events)))
2305+ perf_branch_stack_sched_in(prev, task);
22282306 }
22292307
22302308 static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
@@ -2791,6 +2869,14 @@ static void free_event(struct perf_event *event) · 560 unmodified lines
27912869 atomic_dec(&per_cpu(perf_cgroup_events, event->cpu));
27922870 static_key_slow_dec_deferred(&perf_sched_events);
27932871 }
2872+
2873+ if (has_branch_stack(event)) {
2874+ static_key_slow_dec_deferred(&perf_sched_events);
2875+ /* is system-wide event */
2876+ if (!(event->attach_state & PERF_ATTACH_TASK))
2877+ atomic_dec(&per_cpu(perf_branch_stack_events,
2878+ event->cpu));
2879+ }
27942880 }
27952881
27962882 if (event->rb) {
@@ -3907,6 +3993,24 @@ void perf_output_sample(struct perf_output_handle *handle, · 1110 unmodified lines
39073993 }
39083994 }
39093995 }
3996+
3997+ if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
3998+ if (data->br_stack) {
3999+ size_t size;
4000+
4001+ size = data->br_stack->nr
4002+ * sizeof(struct perf_branch_entry);
4003+
4004+ perf_output_put(handle, data->br_stack->nr);
4005+ perf_output_copy(handle, data->br_stack->entries, size);
4006+ } else {
4007+ /*
4008+ * we always store at least the value of nr
4009+ */
4010+ u64 nr = 0;
4011+ perf_output_put(handle, nr);
4012+ }
4013+ }
39104014 }
39114015
39124016 void perf_prepare_sample(struct perf_event_header *header,
@@ -3949,6 +4053,15 @@ void perf_prepare_sample(struct perf_event_header *header, · 36 unmodified lines
39494053 WARN_ON_ONCE(size & (sizeof(u64)-1));
39504054 header->size += size;
39514055 }
4056+
4057+ if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
4058+ int size = sizeof(u64); /* nr */
4059+ if (data->br_stack) {
4060+ size += data->br_stack->nr
4061+ * sizeof(struct perf_branch_entry);
4062+ }
4063+ header->size += size;
4064+ }
39524065 }
39534066
39544067 static void perf_event_output(struct perf_event *event,
@@ -5010,6 +5123,12 @@ static int perf_swevent_init(struct perf_event *event) · 1055 unmodified lines
50105123 if (event->attr.type != PERF_TYPE_SOFTWARE)
50115124 return -ENOENT;
50125125
5126+ /*
5127+ * no branch sampling for software events
5128+ */
5129+ if (has_branch_stack(event))
5130+ return -EOPNOTSUPP;
5131+
50135132 switch (event_id) {
50145133 case PERF_COUNT_SW_CPU_CLOCK:
50155134 case PERF_COUNT_SW_TASK_CLOCK:
@@ -5120,6 +5239,12 @@ static int perf_tp_event_init(struct perf_event *event) · 104 unmodified lines
51205239 if (event->attr.type != PERF_TYPE_TRACEPOINT)
51215240 return -ENOENT;
51225241
5242+ /*
5243+ * no branch sampling for tracepoint events
5244+ */
5245+ if (has_branch_stack(event))
5246+ return -EOPNOTSUPP;
5247+
51235248 err = perf_trace_init(event);
51245249 if (err)
51255250 return err;
@@ -5345,6 +5470,12 @@ static int cpu_clock_event_init(struct perf_event *event) · 219 unmodified lines
53455470 if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)
53465471 return -ENOENT;
53475472
5473+ /*
5474+ * no branch sampling for software events
5475+ */
5476+ if (has_branch_stack(event))
5477+ return -EOPNOTSUPP;
5478+
53485479 perf_swevent_init_hrtimer(event);
53495480
53505481 return 0;
@@ -5419,6 +5550,12 @@ static int task_clock_event_init(struct perf_event *event) · 68 unmodified lines
54195550 if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)
54205551 return -ENOENT;
54215552
5553+ /*
5554+ * no branch sampling for software events
5555+ */
5556+ if (has_branch_stack(event))
5557+ return -EOPNOTSUPP;
5558+
54225559 perf_swevent_init_hrtimer(event);
54235560
54245561 return 0;
@@ -5866,6 +6003,12 @@ done: · 441 unmodified lines
58666003 return ERR_PTR(err);
58676004 }
58686005 }
6006+ if (has_branch_stack(event)) {
6007+ static_key_slow_inc(&perf_sched_events.key);
6008+ if (!(event->attach_state & PERF_ATTACH_TASK))
6009+ atomic_inc(&per_cpu(perf_branch_stack_events,
6010+ event->cpu));
6011+ }
58696012 }
58706013
58716014 return event;
@@ -5935,6 +6078,40 @@ static int perf_copy_attr(struct perf_event_attr __user *uattr, · 63 unmodified lines
59356078 if (attr->read_format & ~(PERF_FORMAT_MAX-1))
59366079 return -EINVAL;
59376080
6081+ if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) {
6082+ u64 mask = attr->branch_sample_type;
6083+
6084+ /* only using defined bits */
6085+ if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1))
6086+ return -EINVAL;
6087+
6088+ /* at least one branch bit must be set */
6089+ if (!(mask & ~PERF_SAMPLE_BRANCH_PLM_ALL))
6090+ return -EINVAL;
6091+
6092+ /* kernel level capture: check permissions */
6093+ if ((mask & PERF_SAMPLE_BRANCH_PERM_PLM)
6094+ && perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
6095+ return -EACCES;
6096+
6097+ /* propagate priv level, when not set for branch */
6098+ if (!(mask & PERF_SAMPLE_BRANCH_PLM_ALL)) {
6099+
6100+ /* exclude_kernel checked on syscall entry */
6101+ if (!attr->exclude_kernel)
6102+ mask |= PERF_SAMPLE_BRANCH_KERNEL;
6103+
6104+ if (!attr->exclude_user)
6105+ mask |= PERF_SAMPLE_BRANCH_USER;
6106+
6107+ if (!attr->exclude_hv)
6108+ mask |= PERF_SAMPLE_BRANCH_HV;
6109+ /*
6110+ * adjust user setting (for HW filter setup)
6111+ */
6112+ attr->branch_sample_type = mask;
6113+ }
6114+ }
59386115 out:
59396116 return ret;
59406117
kernel/events/hw_breakpoint.c+6−0
@@ -581,6 +581,12 @@ static int hw_breakpoint_event_init(struct perf_event *bp) · 580 unmodified lines
581581 if (bp->attr.type != PERF_TYPE_BREAKPOINT)
582582 return -ENOENT;
583583
584+ /*
585+ * no branch sampling for breakpoint events
586+ */
587+ if (has_branch_stack(bp))
588+ return -EOPNOTSUPP;
589+
584590 err = register_perf_hw_breakpoint(bp);
585591 if (err)
586592 return err;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t