History

Merge branch 'perf/hw-branch-sampling' into perf/core

Merge reason: The 'perf record -b' hardware branch sampling feature is ready for upstream.

Ingo Molnar committed 14 years ago

Commit bea95c12 parents f9b4eeb 24bff2d33 files · +2017 −243Browse files

Unified · Split

33 files
  1. arch/alpha/kernel/perf_event.c +4 −0
  2. arch/arm/kernel/perf_event.c +4 −0
  3. arch/mips/kernel/perf_event_mipsxx.c +4 −0
  4. arch/powerpc/kernel/perf_event.c +4 −0
  5. arch/sh/kernel/perf_event.c +4 −0
  6. arch/sparc/kernel/perf_event.c +4 −0
  7. arch/x86/include/asm/msr-index.h +7 −0
  8. arch/x86/kernel/cpu/perf_event.c +78 −7
  9. arch/x86/kernel/cpu/perf_event.h +19 −0
  10. arch/x86/kernel/cpu/perf_event_amd.c +3 −0
  11. arch/x86/kernel/cpu/perf_event_intel.c +93 −27
  12. arch/x86/kernel/cpu/perf_event_intel_ds.c +5 −17
  13. arch/x86/kernel/cpu/perf_event_intel_lbr.c +505 −21
  14. include/linux/perf_event.h +77 −5
  15. kernel/events/core.c +177 −0
  16. kernel/events/hw_breakpoint.c +6 −0
  17. tools/perf/Documentation/perf-record.txt +30 −0
  18. tools/perf/Documentation/perf-report.txt +10 −0
  19. tools/perf/builtin-record.c +95 −0
  20. tools/perf/builtin-report.c +156 −22
  21. tools/perf/perf.h +18 −0
  22. tools/perf/util/event.h +1 −0
  23. tools/perf/util/evsel.c +14 −0
  24. tools/perf/util/header.c +164 −43
  25. tools/perf/util/header.h +1 −1
  26. tools/perf/util/hist.c +95 −27
  27. tools/perf/util/hist.h +11 −0
  28. tools/perf/util/session.c +75 −2
  29. tools/perf/util/session.h +4 −0
  30. tools/perf/util/sort.c +236 −51
  31. tools/perf/util/sort.h +11 −0
  32. tools/perf/util/symbol.h +19 −1
  33. tools/perf/util/ui/browsers/hists.c +83 −19
arch/alpha/kernel/perf_event.c+4−0
@@ -685,6 +685,10 @@ static int alpha_pmu_event_init(struct perf_event *event) · 684 unmodified lines
685685 {
686686 int err;
687687
688+ /* does not support taken branch sampling */
689+ if (has_branch_stack(event))
690+ return -EOPNOTSUPP;
691+
688692 switch (event->attr.type) {
689693 case PERF_TYPE_RAW:
690694 case PERF_TYPE_HARDWARE:
arch/arm/kernel/perf_event.c+4−0
@@ -539,6 +539,10 @@ static int armpmu_event_init(struct perf_event *event) · 538 unmodified lines
539539 int err = 0;
540540 atomic_t *active_events = &armpmu->active_events;
541541
542+ /* does not support taken branch sampling */
543+ if (has_branch_stack(event))
544+ return -EOPNOTSUPP;
545+
542546 if (armpmu->map_event(event) == -ENOENT)
543547 return -ENOENT;
544548
arch/mips/kernel/perf_event_mipsxx.c+4−0
@@ -606,6 +606,10 @@ static int mipspmu_event_init(struct perf_event *event) · 605 unmodified lines
606606 {
607607 int err = 0;
608608
609+ /* does not support taken branch sampling */
610+ if (has_branch_stack(event))
611+ return -EOPNOTSUPP;
612+
609613 switch (event->attr.type) {
610614 case PERF_TYPE_RAW:
611615 case PERF_TYPE_HARDWARE:
arch/powerpc/kernel/perf_event.c+4−0
@@ -1084,6 +1084,10 @@ static int power_pmu_event_init(struct perf_event *event) · 1083 unmodified lines
10841084 if (!ppmu)
10851085 return -ENOENT;
10861086
1087+ /* does not support taken branch sampling */
1088+ if (has_branch_stack(event))
1089+ return -EOPNOTSUPP;
1090+
10871091 switch (event->attr.type) {
10881092 case PERF_TYPE_HARDWARE:
10891093 ev = event->attr.config;
arch/sh/kernel/perf_event.c+4−0
@@ -310,6 +310,10 @@ static int sh_pmu_event_init(struct perf_event *event) · 309 unmodified lines
310310 {
311311 int err;
312312
313+ /* does not support taken branch sampling */
314+ if (has_branch_stack(event))
315+ return -EOPNOTSUPP;
316+
313317 switch (event->attr.type) {
314318 case PERF_TYPE_RAW:
315319 case PERF_TYPE_HW_CACHE:
arch/sparc/kernel/perf_event.c+4−0
@@ -1105,6 +1105,10 @@ static int sparc_pmu_event_init(struct perf_event *event) · 1104 unmodified lines
11051105 if (atomic_read(&nmi_active) < 0)
11061106 return -ENODEV;
11071107
1108+ /* does not support taken branch sampling */
1109+ if (has_branch_stack(event))
1110+ return -EOPNOTSUPP;
1111+
11081112 switch (attr->type) {
11091113 case PERF_TYPE_HARDWARE:
11101114 if (attr->config >= sparc_pmu->max_events)
arch/x86/include/asm/msr-index.h+7−0
@@ -56,6 +56,13 @@ · 55 unmodified lines
5656 #define MSR_OFFCORE_RSP_0 0x000001a6
5757 #define MSR_OFFCORE_RSP_1 0x000001a7
5858
59+#define MSR_LBR_SELECT 0x000001c8
60+#define MSR_LBR_TOS 0x000001c9
61+#define MSR_LBR_NHM_FROM 0x00000680
62+#define MSR_LBR_NHM_TO 0x000006c0
63+#define MSR_LBR_CORE_FROM 0x00000040
64+#define MSR_LBR_CORE_TO 0x00000060
65+
5966 #define MSR_IA32_PEBS_ENABLE 0x000003f1
6067 #define MSR_IA32_DS_AREA 0x00000600
6168 #define MSR_IA32_PERF_CAPABILITIES 0x00000345
arch/x86/kernel/cpu/perf_event.c+78−7
@@ -353,6 +353,36 @@ int x86_setup_perfctr(struct perf_event *event) · 352 unmodified lines
353353 return 0;
354354 }
355355
356+/*
357+ * check that branch_sample_type is compatible with
358+ * settings needed for precise_ip > 1 which implies
359+ * using the LBR to capture ALL taken branches at the
360+ * priv levels of the measurement
361+ */
362+static inline int precise_br_compat(struct perf_event *event)
363+{
364+ u64 m = event->attr.branch_sample_type;
365+ u64 b = 0;
366+
367+ /* must capture all branches */
368+ if (!(m & PERF_SAMPLE_BRANCH_ANY))
369+ return 0;
370+
371+ m &= PERF_SAMPLE_BRANCH_KERNEL | PERF_SAMPLE_BRANCH_USER;
372+
373+ if (!event->attr.exclude_user)
374+ b |= PERF_SAMPLE_BRANCH_USER;
375+
376+ if (!event->attr.exclude_kernel)
377+ b |= PERF_SAMPLE_BRANCH_KERNEL;
378+
379+ /*
380+ * ignore PERF_SAMPLE_BRANCH_HV, not supported on x86
381+ */
382+
383+ return m == b;
384+}
385+
356386 int x86_pmu_hw_config(struct perf_event *event)
357387 {
358388 if (event->attr.precise_ip) {
@@ -369,6 +399,36 @@ int x86_pmu_hw_config(struct perf_event *event) · 10 unmodified lines
369399
370400 if (event->attr.precise_ip > precise)
371401 return -EOPNOTSUPP;
402+ /*
403+ * check that PEBS LBR correction does not conflict with
404+ * whatever the user is asking with attr->branch_sample_type
405+ */
406+ if (event->attr.precise_ip > 1) {
407+ u64 *br_type = &event->attr.branch_sample_type;
408+
409+ if (has_branch_stack(event)) {
410+ if (!precise_br_compat(event))
411+ return -EOPNOTSUPP;
412+
413+ /* branch_sample_type is compatible */
414+
415+ } else {
416+ /*
417+ * user did not specify branch_sample_type
418+ *
419+ * For PEBS fixups, we capture all
420+ * the branches at the priv level of the
421+ * event.
422+ */
423+ *br_type = PERF_SAMPLE_BRANCH_ANY;
424+
425+ if (!event->attr.exclude_user)
426+ *br_type |= PERF_SAMPLE_BRANCH_USER;
427+
428+ if (!event->attr.exclude_kernel)
429+ *br_type |= PERF_SAMPLE_BRANCH_KERNEL;
430+ }
431+ }
372432 }
373433
374434 /*
@@ -426,6 +486,10 @@ static int __x86_pmu_event_init(struct perf_event *event) · 51 unmodified lines
426486 /* mark unused */
427487 event->hw.extra_reg.idx = EXTRA_REG_NONE;
428488
489+ /* mark not used */
490+ event->hw.extra_reg.idx = EXTRA_REG_NONE;
491+ event->hw.branch_reg.idx = EXTRA_REG_NONE;
492+
429493 return x86_pmu.hw_config(event);
430494 }
431495
@@ -1607,25 +1671,32 @@ static const struct attribute_group *x86_pmu_attr_groups[] = { · 1175 unmodified lines
16071671 NULL,
16081672 };
16091673
1674+static void x86_pmu_flush_branch_stack(void)
1675+{
1676+ if (x86_pmu.flush_branch_stack)
1677+ x86_pmu.flush_branch_stack();
1678+}
1679+
16101680 static struct pmu pmu = {
1611− .pmu_enable = x86_pmu_enable,
1612− .pmu_disable = x86_pmu_disable,
1681+ .pmu_enable = x86_pmu_enable,
1682+ .pmu_disable = x86_pmu_disable,
16131683
16141684 .attr_groups = x86_pmu_attr_groups,
16151685
16161686 .event_init = x86_pmu_event_init,
16171687
1618− .add = x86_pmu_add,
1619− .del = x86_pmu_del,
1620− .start = x86_pmu_start,
1621− .stop = x86_pmu_stop,
1622− .read = x86_pmu_read,
1688+ .add = x86_pmu_add,
1689+ .del = x86_pmu_del,
1690+ .start = x86_pmu_start,
1691+ .stop = x86_pmu_stop,
1692+ .read = x86_pmu_read,
16231693
16241694 .start_txn = x86_pmu_start_txn,
16251695 .cancel_txn = x86_pmu_cancel_txn,
16261696 .commit_txn = x86_pmu_commit_txn,
16271697
16281698 .event_idx = x86_pmu_event_idx,
1699+ .flush_branch_stack = x86_pmu_flush_branch_stack,
16291700 };
16301701
16311702 void perf_update_user_clock(struct perf_event_mmap_page *userpg, u64 now)
arch/x86/kernel/cpu/perf_event.h+19−0
@@ -33,6 +33,7 @@ enum extra_reg_type { · 32 unmodified lines
3333
3434 EXTRA_REG_RSP_0 = 0, /* offcore_response_0 */
3535 EXTRA_REG_RSP_1 = 1, /* offcore_response_1 */
36+ EXTRA_REG_LBR = 2, /* lbr_select */
3637
3738 EXTRA_REG_MAX /* number of entries needed */
3839 };
@@ -130,6 +131,8 @@ struct cpu_hw_events { · 91 unmodified lines
130131 void *lbr_context;
131132 struct perf_branch_stack lbr_stack;
132133 struct perf_branch_entry lbr_entries[MAX_LBR_ENTRIES];
134+ struct er_account *lbr_sel;
135+ u64 br_sel;
133136
134137 /*
135138 * Intel host/guest exclude bits
@@ -344,6 +347,7 @@ struct x86_pmu { · 208 unmodified lines
344347 void (*cpu_starting)(int cpu);
345348 void (*cpu_dying)(int cpu);
346349 void (*cpu_dead)(int cpu);
350+ void (*flush_branch_stack)(void);
347351
348352 /*
349353 * Intel Arch Perfmon v2+
@@ -365,6 +369,8 @@ struct x86_pmu { · 15 unmodified lines
365369 */
366370 unsigned long lbr_tos, lbr_from, lbr_to; /* MSR base regs */
367371 int lbr_nr; /* hardware stack size */
372+ u64 lbr_sel_mask; /* LBR_SELECT valid bits */
373+ const int *lbr_sel_map; /* lbr_select mappings */
368374
369375 /*
370376 * Extra registers for events
@@ -478,6 +484,15 @@ extern struct event_constraint emptyconstraint; · 107 unmodified lines
478484
479485 extern struct event_constraint unconstrained;
480486
487+static inline bool kernel_ip(unsigned long ip)
488+{
489+#ifdef CONFIG_X86_32
490+ return ip > PAGE_OFFSET;
491+#else
492+ return (long)ip < 0;
493+#endif
494+}
495+
481496 #ifdef CONFIG_CPU_SUP_AMD
482497
483498 int amd_pmu_init(void);
@@ -558,6 +573,10 @@ void intel_pmu_lbr_init_nhm(void); · 74 unmodified lines
558573
559574 void intel_pmu_lbr_init_atom(void);
560575
576+void intel_pmu_lbr_init_snb(void);
577+
578+int intel_pmu_setup_lbr_filter(struct perf_event *event);
579+
561580 int p4_pmu_init(void);
562581
563582 int p6_pmu_init(void);
arch/x86/kernel/cpu/perf_event_amd.c+3−0
@@ -139,6 +139,9 @@ static int amd_pmu_hw_config(struct perf_event *event) · 138 unmodified lines
139139 if (ret)
140140 return ret;
141141
142+ if (has_branch_stack(event))
143+ return -EOPNOTSUPP;
144+
142145 if (event->attr.exclude_host && event->attr.exclude_guest)
143146 /*
144147 * When HO == GO == 1 the hardware treats that as GO == HO == 0
arch/x86/kernel/cpu/perf_event_intel.c+93−27
@@ -728,6 +728,19 @@ static __initconst const u64 atom_hw_cache_event_ids · 727 unmodified lines
728728 },
729729 };
730730
731+static inline bool intel_pmu_needs_lbr_smpl(struct perf_event *event)
732+{
733+ /* user explicitly requested branch sampling */
734+ if (has_branch_stack(event))
735+ return true;
736+
737+ /* implicit branch sampling to correct PEBS skid */
738+ if (x86_pmu.intel_cap.pebs_trap && event->attr.precise_ip > 1)
739+ return true;
740+
741+ return false;
742+}
743+
731744 static void intel_pmu_disable_all(void)
732745 {
733746 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
@@ -882,6 +895,13 @@ static void intel_pmu_disable_event(struct perf_event *event) · 148 unmodified lines
882895 cpuc->intel_ctrl_guest_mask &= ~(1ull << hwc->idx);
883896 cpuc->intel_ctrl_host_mask &= ~(1ull << hwc->idx);
884897
898+ /*
899+ * must disable before any actual event
900+ * because any event may be combined with LBR
901+ */
902+ if (intel_pmu_needs_lbr_smpl(event))
903+ intel_pmu_lbr_disable(event);
904+
885905 if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
886906 intel_pmu_disable_fixed(hwc);
887907 return;
@@ -936,6 +956,12 @@ static void intel_pmu_enable_event(struct perf_event *event) · 48 unmodified lines
936956 intel_pmu_enable_bts(hwc->config);
937957 return;
938958 }
959+ /*
960+ * must enabled before any actual event
961+ * because any event may be combined with LBR
962+ */
963+ if (intel_pmu_needs_lbr_smpl(event))
964+ intel_pmu_lbr_enable(event);
939965
940966 if (event->attr.exclude_host)
941967 cpuc->intel_ctrl_guest_mask |= (1ull << hwc->idx);
@@ -1058,6 +1084,9 @@ again: · 116 unmodified lines
10581084
10591085 data.period = event->hw.last_period;
10601086
1087+ if (has_branch_stack(event))
1088+ data.br_stack = &cpuc->lbr_stack;
1089+
10611090 if (perf_event_overflow(event, &data, regs))
10621091 x86_pmu_stop(event, 0);
10631092 }
@@ -1124,17 +1153,17 @@ static bool intel_try_alt_er(struct perf_event *event, int orig_idx) · 60 unmodified lines
11241153 */
11251154 static struct event_constraint *
11261155 __intel_shared_reg_get_constraints(struct cpu_hw_events *cpuc,
1127− struct perf_event *event)
1156+ struct perf_event *event,
1157+ struct hw_perf_event_extra *reg)
11281158 {
11291159 struct event_constraint *c = &emptyconstraint;
1130− struct hw_perf_event_extra *reg = &event->hw.extra_reg;
11311160 struct er_account *era;
11321161 unsigned long flags;
11331162 int orig_idx = reg->idx;
11341163
11351164 /* already allocated shared msr */
11361165 if (reg->alloc)
1137− return &unconstrained;
1166+ return NULL; /* call x86_get_event_constraint() */
11381167
11391168 again:
11401169 era = &cpuc->shared_regs->regs[reg->idx];
@@ -1157,14 +1186,10 @@ again: · 16 unmodified lines
11571186 reg->alloc = 1;
11581187
11591188 /*
1160− * All events using extra_reg are unconstrained.
1161− * Avoids calling x86_get_event_constraints()
1162− *
1163− * Must revisit if extra_reg controlling events
1164− * ever have constraints. Worst case we go through
1165− * the regular event constraint table.
1189+ * need to call x86_get_event_constraint()
1190+ * to check if associated event has constraints
11661191 */
1167− c = &unconstrained;
1192+ c = NULL;
11681193 } else if (intel_try_alt_er(event, orig_idx)) {
11691194 raw_spin_unlock_irqrestore(&era->lock, flags);
11701195 goto again;
@@ -1201,11 +1226,23 @@ static struct event_constraint * · 30 unmodified lines
12011226 intel_shared_regs_constraints(struct cpu_hw_events *cpuc,
12021227 struct perf_event *event)
12031228 {
1204− struct event_constraint *c = NULL;
1205−
1206− if (event->hw.extra_reg.idx != EXTRA_REG_NONE)
1207− c = __intel_shared_reg_get_constraints(cpuc, event);
1208−
1229+ struct event_constraint *c = NULL, *d;
1230+ struct hw_perf_event_extra *xreg, *breg;
1231+
1232+ xreg = &event->hw.extra_reg;
1233+ if (xreg->idx != EXTRA_REG_NONE) {
1234+ c = __intel_shared_reg_get_constraints(cpuc, event, xreg);
1235+ if (c == &emptyconstraint)
1236+ return c;
1237+ }
1238+ breg = &event->hw.branch_reg;
1239+ if (breg->idx != EXTRA_REG_NONE) {
1240+ d = __intel_shared_reg_get_constraints(cpuc, event, breg);
1241+ if (d == &emptyconstraint) {
1242+ __intel_shared_reg_put_constraints(cpuc, xreg);
1243+ c = d;
1244+ }
1245+ }
12091246 return c;
12101247 }
12111248
@@ -1253,6 +1290,10 @@ intel_put_shared_regs_event_constraints(struct cpu_hw_events *cpuc, · 41 unmodified lines
12531290 reg = &event->hw.extra_reg;
12541291 if (reg->idx != EXTRA_REG_NONE)
12551292 __intel_shared_reg_put_constraints(cpuc, reg);
1293+
1294+ reg = &event->hw.branch_reg;
1295+ if (reg->idx != EXTRA_REG_NONE)
1296+ __intel_shared_reg_put_constraints(cpuc, reg);
12561297 }
12571298
12581299 static void intel_put_event_constraints(struct cpu_hw_events *cpuc,
@@ -1295,6 +1336,12 @@ static int intel_pmu_hw_config(struct perf_event *event) · 36 unmodified lines
12951336 event->hw.config = alt_config;
12961337 }
12971338
1339+ if (intel_pmu_needs_lbr_smpl(event)) {
1340+ ret = intel_pmu_setup_lbr_filter(event);
1341+ if (ret)
1342+ return ret;
1343+ }
1344+
12981345 if (event->attr.type != PERF_TYPE_RAW)
12991346 return 0;
13001347
@@ -1433,7 +1480,7 @@ static int intel_pmu_cpu_prepare(int cpu) · 132 unmodified lines
14331480 {
14341481 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
14351482
1436− if (!x86_pmu.extra_regs)
1483+ if (!(x86_pmu.extra_regs || x86_pmu.lbr_sel_map))
14371484 return NOTIFY_OK;
14381485
14391486 cpuc->shared_regs = allocate_shared_regs(cpu);
@@ -1455,22 +1502,28 @@ static void intel_pmu_cpu_starting(int cpu) · 15 unmodified lines
14551502 */
14561503 intel_pmu_lbr_reset();
14571504
1458− if (!cpuc->shared_regs || (x86_pmu.er_flags & ERF_NO_HT_SHARING))
1505+ cpuc->lbr_sel = NULL;
1506+
1507+ if (!cpuc->shared_regs)
14591508 return;
14601509
1461− for_each_cpu(i, topology_thread_cpumask(cpu)) {
1462− struct intel_shared_regs *pc;
1510+ if (!(x86_pmu.er_flags & ERF_NO_HT_SHARING)) {
1511+ for_each_cpu(i, topology_thread_cpumask(cpu)) {
1512+ struct intel_shared_regs *pc;
14631513
1464− pc = per_cpu(cpu_hw_events, i).shared_regs;
1465− if (pc && pc->core_id == core_id) {
1466− cpuc->kfree_on_online = cpuc->shared_regs;
1467− cpuc->shared_regs = pc;
1468− break;
1514+ pc = per_cpu(cpu_hw_events, i).shared_regs;
1515+ if (pc && pc->core_id == core_id) {
1516+ cpuc->kfree_on_online = cpuc->shared_regs;
1517+ cpuc->shared_regs = pc;
1518+ break;
1519+ }
14691520 }
1521+ cpuc->shared_regs->core_id = core_id;
1522+ cpuc->shared_regs->refcnt++;
14701523 }
14711524
1472− cpuc->shared_regs->core_id = core_id;
1473− cpuc->shared_regs->refcnt++;
1525+ if (x86_pmu.lbr_sel_map)
1526+ cpuc->lbr_sel = &cpuc->shared_regs->regs[EXTRA_REG_LBR];
14741527 }
14751528
14761529 static void intel_pmu_cpu_dying(int cpu)
@@ -1488,6 +1541,18 @@ static void intel_pmu_cpu_dying(int cpu) · 11 unmodified lines
14881541 fini_debug_store_on_cpu(cpu);
14891542 }
14901543
1544+static void intel_pmu_flush_branch_stack(void)
1545+{
1546+ /*
1547+ * Intel LBR does not tag entries with the
1548+ * PID of the current task, then we need to
1549+ * flush it on ctxsw
1550+ * For now, we simply reset it
1551+ */
1552+ if (x86_pmu.lbr_nr)
1553+ intel_pmu_lbr_reset();
1554+}
1555+
14911556 static __initconst const struct x86_pmu intel_pmu = {
14921557 .name = "Intel",
14931558 .handle_irq = intel_pmu_handle_irq,
@@ -1515,6 +1580,7 @@ static __initconst const struct x86_pmu intel_pmu = { · 21 unmodified lines
15151580 .cpu_starting = intel_pmu_cpu_starting,
15161581 .cpu_dying = intel_pmu_cpu_dying,
15171582 .guest_get_msrs = intel_guest_get_msrs,
1583+ .flush_branch_stack = intel_pmu_flush_branch_stack,
15181584 };
15191585
15201586 static __init void intel_clovertown_quirk(void)
@@ -1745,7 +1811,7 @@ __init int intel_pmu_init(void) · 224 unmodified lines
17451811 memcpy(hw_cache_event_ids, snb_hw_cache_event_ids,
17461812 sizeof(hw_cache_event_ids));
17471813
1748− intel_pmu_lbr_init_nhm();
1814+ intel_pmu_lbr_init_snb();
17491815
17501816 x86_pmu.event_constraints = intel_snb_event_constraints;
17511817 x86_pmu.pebs_constraints = intel_snb_pebs_event_constraints;
arch/x86/kernel/cpu/perf_event_intel_ds.c+5−17
@@ -3,6 +3,7 @@ · 2 unmodified lines
33 #include <linux/slab.h>
44
55 #include <asm/perf_event.h>
6+#include <asm/insn.h>
67
78 #include "perf_event.h"
89
@@ -439,9 +440,6 @@ void intel_pmu_pebs_enable(struct perf_event *event) · 430 unmodified lines
439440 hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT;
440441
441442 cpuc->pebs_enabled |= 1ULL << hwc->idx;
442−
443− if (x86_pmu.intel_cap.pebs_trap && event->attr.precise_ip > 1)
444− intel_pmu_lbr_enable(event);
445443 }
446444
447445 void intel_pmu_pebs_disable(struct perf_event *event)
@@ -454,9 +452,6 @@ void intel_pmu_pebs_disable(struct perf_event *event) · 6 unmodified lines
454452 wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
455453
456454 hwc->config |= ARCH_PERFMON_EVENTSEL_INT;
457−
458− if (x86_pmu.intel_cap.pebs_trap && event->attr.precise_ip > 1)
459− intel_pmu_lbr_disable(event);
460455 }
461456
462457 void intel_pmu_pebs_enable_all(void)
@@ -475,17 +470,6 @@ void intel_pmu_pebs_disable_all(void) · 12 unmodified lines
475470 wrmsrl(MSR_IA32_PEBS_ENABLE, 0);
476471 }
477472
478−#include <asm/insn.h>
479−
480−static inline bool kernel_ip(unsigned long ip)
481−{
482−#ifdef CONFIG_X86_32
483− return ip > PAGE_OFFSET;
484−#else
485− return (long)ip < 0;
486−#endif
487−}
488−
489473 static int intel_pmu_pebs_fixup_ip(struct pt_regs *regs)
490474 {
491475 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
@@ -572,6 +556,7 @@ static void __intel_pmu_pebs_event(struct perf_event *event, · 80 unmodified lines
572556 * both formats and we don't use the other fields in this
573557 * routine.
574558 */
559+ struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
575560 struct pebs_record_core *pebs = __pebs;
576561 struct perf_sample_data data;
577562 struct pt_regs regs;
@@ -602,6 +587,9 @@ static void __intel_pmu_pebs_event(struct perf_event *event, · 24 unmodified lines
602587 else
603588 regs.flags &= ~PERF_EFLAGS_EXACT;
604589
590+ if (has_branch_stack(event))
591+ data.br_stack = &cpuc->lbr_stack;
592+
605593 if (perf_event_overflow(event, &data, &regs))
606594 x86_pmu_stop(event, 0);
607595 }
arch/x86/kernel/cpu/perf_event_intel_lbr.c+505−21
@@ -3,6 +3,7 @@ · 2 unmodified lines
33
44 #include <asm/perf_event.h>
55 #include <asm/msr.h>
6+#include <asm/insn.h>
67
78 #include "perf_event.h"
89
@@ -13,6 +14,100 @@ enum { · 4 unmodified lines
1314 LBR_FORMAT_EIP_FLAGS = 0x03,
1415 };
1516
17+/*
18+ * Intel LBR_SELECT bits
19+ * Intel Vol3a, April 2011, Section 16.7 Table 16-10
20+ *
21+ * Hardware branch filter (not available on all CPUs)
22+ */
23+#define LBR_KERNEL_BIT 0 /* do not capture at ring0 */
24+#define LBR_USER_BIT 1 /* do not capture at ring > 0 */
25+#define LBR_JCC_BIT 2 /* do not capture conditional branches */
26+#define LBR_REL_CALL_BIT 3 /* do not capture relative calls */
27+#define LBR_IND_CALL_BIT 4 /* do not capture indirect calls */
28+#define LBR_RETURN_BIT 5 /* do not capture near returns */
29+#define LBR_IND_JMP_BIT 6 /* do not capture indirect jumps */
30+#define LBR_REL_JMP_BIT 7 /* do not capture relative jumps */
31+#define LBR_FAR_BIT 8 /* do not capture far branches */
32+
33+#define LBR_KERNEL (1 << LBR_KERNEL_BIT)
34+#define LBR_USER (1 << LBR_USER_BIT)
35+#define LBR_JCC (1 << LBR_JCC_BIT)
36+#define LBR_REL_CALL (1 << LBR_REL_CALL_BIT)
37+#define LBR_IND_CALL (1 << LBR_IND_CALL_BIT)
38+#define LBR_RETURN (1 << LBR_RETURN_BIT)
39+#define LBR_REL_JMP (1 << LBR_REL_JMP_BIT)
40+#define LBR_IND_JMP (1 << LBR_IND_JMP_BIT)
41+#define LBR_FAR (1 << LBR_FAR_BIT)
42+
43+#define LBR_PLM (LBR_KERNEL | LBR_USER)
44+
45+#define LBR_SEL_MASK 0x1ff /* valid bits in LBR_SELECT */
46+#define LBR_NOT_SUPP -1 /* LBR filter not supported */
47+#define LBR_IGN 0 /* ignored */
48+
49+#define LBR_ANY \
50+ (LBR_JCC |\
51+ LBR_REL_CALL |\
52+ LBR_IND_CALL |\
53+ LBR_RETURN |\
54+ LBR_REL_JMP |\
55+ LBR_IND_JMP |\
56+ LBR_FAR)
57+
58+#define LBR_FROM_FLAG_MISPRED (1ULL << 63)
59+
60+#define for_each_branch_sample_type(x) \
61+ for ((x) = PERF_SAMPLE_BRANCH_USER; \
62+ (x) < PERF_SAMPLE_BRANCH_MAX; (x) <<= 1)
63+
64+/*
65+ * x86control flow change classification
66+ * x86control flow changes include branches, interrupts, traps, faults
67+ */
68+enum {
69+ X86_BR_NONE = 0, /* unknown */
70+
71+ X86_BR_USER = 1 << 0, /* branch target is user */
72+ X86_BR_KERNEL = 1 << 1, /* branch target is kernel */
73+
74+ X86_BR_CALL = 1 << 2, /* call */
75+ X86_BR_RET = 1 << 3, /* return */
76+ X86_BR_SYSCALL = 1 << 4, /* syscall */
77+ X86_BR_SYSRET = 1 << 5, /* syscall return */
78+ X86_BR_INT = 1 << 6, /* sw interrupt */
79+ X86_BR_IRET = 1 << 7, /* return from interrupt */
80+ X86_BR_JCC = 1 << 8, /* conditional */
81+ X86_BR_JMP = 1 << 9, /* jump */
82+ X86_BR_IRQ = 1 << 10,/* hw interrupt or trap or fault */
83+ X86_BR_IND_CALL = 1 << 11,/* indirect calls */
84+};
85+
86+#define X86_BR_PLM (X86_BR_USER | X86_BR_KERNEL)
87+
88+#define X86_BR_ANY \
89+ (X86_BR_CALL |\
90+ X86_BR_RET |\
91+ X86_BR_SYSCALL |\
92+ X86_BR_SYSRET |\
93+ X86_BR_INT |\
94+ X86_BR_IRET |\
95+ X86_BR_JCC |\
96+ X86_BR_JMP |\
97+ X86_BR_IRQ |\
98+ X86_BR_IND_CALL)
99+
100+#define X86_BR_ALL (X86_BR_PLM | X86_BR_ANY)
101+
102+#define X86_BR_ANY_CALL \
103+ (X86_BR_CALL |\
104+ X86_BR_IND_CALL |\
105+ X86_BR_SYSCALL |\
106+ X86_BR_IRQ |\
107+ X86_BR_INT)
108+
109+static void intel_pmu_lbr_filter(struct cpu_hw_events *cpuc);
110+
16111 /*
17112 * We only support LBR implementations that have FREEZE_LBRS_ON_PMI
18113 * otherwise it becomes near impossible to get a reliable stack.
@@ -21,6 +116,10 @@ enum { · 2 unmodified lines
21116 static void __intel_pmu_lbr_enable(void)
22117 {
23118 u64 debugctl;
119+ struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
120+
121+ if (cpuc->lbr_sel)
122+ wrmsrl(MSR_LBR_SELECT, cpuc->lbr_sel->config);
24123
25124 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
26125 debugctl |= (DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
@@ -76,11 +175,11 @@ void intel_pmu_lbr_enable(struct perf_event *event) · 49 unmodified lines
76175 * Reset the LBR stack if we changed task context to
77176 * avoid data leaks.
78177 */
79−
80178 if (event->ctx->task && cpuc->lbr_context != event->ctx) {
81179 intel_pmu_lbr_reset();
82180 cpuc->lbr_context = event->ctx;
83181 }
182+ cpuc->br_sel = event->hw.branch_reg.reg;
84183
85184 cpuc->lbr_users++;
86185 }
@@ -95,8 +194,11 @@ void intel_pmu_lbr_disable(struct perf_event *event) · 8 unmodified lines
95194 cpuc->lbr_users--;
96195 WARN_ON_ONCE(cpuc->lbr_users < 0);
97196
98− if (cpuc->enabled && !cpuc->lbr_users)
197+ if (cpuc->enabled && !cpuc->lbr_users) {
99198 __intel_pmu_lbr_disable();
199+ /* avoid stale pointer */
200+ cpuc->lbr_context = NULL;
201+ }
100202 }
101203
102204 void intel_pmu_lbr_enable_all(void)
@@ -115,6 +217,9 @@ void intel_pmu_lbr_disable_all(void) · 12 unmodified lines
115217 __intel_pmu_lbr_disable();
116218 }
117219
220+/*
221+ * TOS = most recently recorded branch
222+ */
118223 static inline u64 intel_pmu_lbr_tos(void)
119224 {
120225 u64 tos;
@@ -142,15 +247,15 @@ static void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc) · 21 unmodified lines
142247
143248 rdmsrl(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr);
144249
145− cpuc->lbr_entries[i].from = msr_lastbranch.from;
146− cpuc->lbr_entries[i].to = msr_lastbranch.to;
147− cpuc->lbr_entries[i].flags = 0;
250+ cpuc->lbr_entries[i].from = msr_lastbranch.from;
251+ cpuc->lbr_entries[i].to = msr_lastbranch.to;
252+ cpuc->lbr_entries[i].mispred = 0;
253+ cpuc->lbr_entries[i].predicted = 0;
254+ cpuc->lbr_entries[i].reserved = 0;
148255 }
149256 cpuc->lbr_stack.nr = i;
150257 }
151258
152−#define LBR_FROM_FLAG_MISPRED (1ULL << 63)
153−
154259 /*
155260 * Due to lack of segmentation in Linux the effective address (offset)
156261 * is the same as the linear address, allowing us to merge the LIP and EIP
@@ -165,19 +270,22 @@ static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc) · 8 unmodified lines
165270
166271 for (i = 0; i < x86_pmu.lbr_nr; i++) {
167272 unsigned long lbr_idx = (tos - i) & mask;
168− u64 from, to, flags = 0;
273+ u64 from, to, mis = 0, pred = 0;
169274
170275 rdmsrl(x86_pmu.lbr_from + lbr_idx, from);
171276 rdmsrl(x86_pmu.lbr_to + lbr_idx, to);
172277
173278 if (lbr_format == LBR_FORMAT_EIP_FLAGS) {
174− flags = !!(from & LBR_FROM_FLAG_MISPRED);
279+ mis = !!(from & LBR_FROM_FLAG_MISPRED);
280+ pred = !mis;
175281 from = (u64)((((s64)from) << 1) >> 1);
176282 }
177283
178− cpuc->lbr_entries[i].from = from;
179− cpuc->lbr_entries[i].to = to;
180− cpuc->lbr_entries[i].flags = flags;
284+ cpuc->lbr_entries[i].from = from;
285+ cpuc->lbr_entries[i].to = to;
286+ cpuc->lbr_entries[i].mispred = mis;
287+ cpuc->lbr_entries[i].predicted = pred;
288+ cpuc->lbr_entries[i].reserved = 0;
181289 }
182290 cpuc->lbr_stack.nr = i;
183291 }
@@ -193,28 +301,404 @@ void intel_pmu_lbr_read(void) · 9 unmodified lines
193301 intel_pmu_lbr_read_32(cpuc);
194302 else
195303 intel_pmu_lbr_read_64(cpuc);
304+
305+ intel_pmu_lbr_filter(cpuc);
306+}
307+
308+/*
309+ * SW filter is used:
310+ * - in case there is no HW filter
311+ * - in case the HW filter has errata or limitations
312+ */
313+static void intel_pmu_setup_sw_lbr_filter(struct perf_event *event)
314+{
315+ u64 br_type = event->attr.branch_sample_type;
316+ int mask = 0;
317+
318+ if (br_type & PERF_SAMPLE_BRANCH_USER)
319+ mask |= X86_BR_USER;
320+
321+ if (br_type & PERF_SAMPLE_BRANCH_KERNEL)
322+ mask |= X86_BR_KERNEL;
323+
324+ /* we ignore BRANCH_HV here */
325+
326+ if (br_type & PERF_SAMPLE_BRANCH_ANY)
327+ mask |= X86_BR_ANY;
328+
329+ if (br_type & PERF_SAMPLE_BRANCH_ANY_CALL)
330+ mask |= X86_BR_ANY_CALL;
331+
332+ if (br_type & PERF_SAMPLE_BRANCH_ANY_RETURN)
333+ mask |= X86_BR_RET | X86_BR_IRET | X86_BR_SYSRET;
334+
335+ if (br_type & PERF_SAMPLE_BRANCH_IND_CALL)
336+ mask |= X86_BR_IND_CALL;
337+ /*
338+ * stash actual user request into reg, it may
339+ * be used by fixup code for some CPU
340+ */
341+ event->hw.branch_reg.reg = mask;
342+}
343+
344+/*
345+ * setup the HW LBR filter
346+ * Used only when available, may not be enough to disambiguate
347+ * all branches, may need the help of the SW filter
348+ */
349+static int intel_pmu_setup_hw_lbr_filter(struct perf_event *event)
350+{
351+ struct hw_perf_event_extra *reg;
352+ u64 br_type = event->attr.branch_sample_type;
353+ u64 mask = 0, m;
354+ u64 v;
355+
356+ for_each_branch_sample_type(m) {
357+ if (!(br_type & m))
358+ continue;
359+
360+ v = x86_pmu.lbr_sel_map[m];
361+ if (v == LBR_NOT_SUPP)
362+ return -EOPNOTSUPP;
363+
364+ if (v != LBR_IGN)
365+ mask |= v;
366+ }
367+ reg = &event->hw.branch_reg;
368+ reg->idx = EXTRA_REG_LBR;
369+
370+ /* LBR_SELECT operates in suppress mode so invert mask */
371+ reg->config = ~mask & x86_pmu.lbr_sel_mask;
372+
373+ return 0;
374+}
375+
376+int intel_pmu_setup_lbr_filter(struct perf_event *event)
377+{
378+ int ret = 0;
379+
380+ /*
381+ * no LBR on this PMU
382+ */
383+ if (!x86_pmu.lbr_nr)
384+ return -EOPNOTSUPP;
385+
386+ /*
387+ * setup SW LBR filter
388+ */
389+ intel_pmu_setup_sw_lbr_filter(event);
390+
391+ /*
392+ * setup HW LBR filter, if any
393+ */
394+ if (x86_pmu.lbr_sel_map)
395+ ret = intel_pmu_setup_hw_lbr_filter(event);
396+
397+ return ret;
196398 }
197399
400+/*
401+ * return the type of control flow change at address "from"
402+ * intruction is not necessarily a branch (in case of interrupt).
403+ *
404+ * The branch type returned also includes the priv level of the
405+ * target of the control flow change (X86_BR_USER, X86_BR_KERNEL).
406+ *
407+ * If a branch type is unknown OR the instruction cannot be
408+ * decoded (e.g., text page not present), then X86_BR_NONE is
409+ * returned.
410+ */
411+static int branch_type(unsigned long from, unsigned long to)
412+{
413+ struct insn insn;
414+ void *addr;
415+ int bytes, size = MAX_INSN_SIZE;
416+ int ret = X86_BR_NONE;
417+ int ext, to_plm, from_plm;
418+ u8 buf[MAX_INSN_SIZE];
419+ int is64 = 0;
420+
421+ to_plm = kernel_ip(to) ? X86_BR_KERNEL : X86_BR_USER;
422+ from_plm = kernel_ip(from) ? X86_BR_KERNEL : X86_BR_USER;
423+
424+ /*
425+ * maybe zero if lbr did not fill up after a reset by the time
426+ * we get a PMU interrupt
427+ */
428+ if (from == 0 || to == 0)
429+ return X86_BR_NONE;
430+
431+ if (from_plm == X86_BR_USER) {
432+ /*
433+ * can happen if measuring at the user level only
434+ * and we interrupt in a kernel thread, e.g., idle.
435+ */
436+ if (!current->mm)
437+ return X86_BR_NONE;
438+
439+ /* may fail if text not present */
440+ bytes = copy_from_user_nmi(buf, (void __user *)from, size);
441+ if (bytes != size)
442+ return X86_BR_NONE;
443+
444+ addr = buf;
445+ } else
446+ addr = (void *)from;
447+
448+ /*
449+ * decoder needs to know the ABI especially
450+ * on 64-bit systems running 32-bit apps
451+ */
452+#ifdef CONFIG_X86_64
453+ is64 = kernel_ip((unsigned long)addr) || !test_thread_flag(TIF_IA32);
454+#endif
455+ insn_init(&insn, addr, is64);
456+ insn_get_opcode(&insn);
457+
458+ switch (insn.opcode.bytes[0]) {
459+ case 0xf:
460+ switch (insn.opcode.bytes[1]) {
461+ case 0x05: /* syscall */
462+ case 0x34: /* sysenter */
463+ ret = X86_BR_SYSCALL;
464+ break;
465+ case 0x07: /* sysret */
466+ case 0x35: /* sysexit */
467+ ret = X86_BR_SYSRET;
468+ break;
469+ case 0x80 ... 0x8f: /* conditional */
470+ ret = X86_BR_JCC;
471+ break;
472+ default:
473+ ret = X86_BR_NONE;
474+ }
475+ break;
476+ case 0x70 ... 0x7f: /* conditional */
477+ ret = X86_BR_JCC;
478+ break;
479+ case 0xc2: /* near ret */
480+ case 0xc3: /* near ret */
481+ case 0xca: /* far ret */
482+ case 0xcb: /* far ret */
483+ ret = X86_BR_RET;
484+ break;
485+ case 0xcf: /* iret */
486+ ret = X86_BR_IRET;
487+ break;
488+ case 0xcc ... 0xce: /* int */
489+ ret = X86_BR_INT;
490+ break;
491+ case 0xe8: /* call near rel */
492+ case 0x9a: /* call far absolute */
493+ ret = X86_BR_CALL;
494+ break;
495+ case 0xe0 ... 0xe3: /* loop jmp */
496+ ret = X86_BR_JCC;
497+ break;
498+ case 0xe9 ... 0xeb: /* jmp */
499+ ret = X86_BR_JMP;
500+ break;
501+ case 0xff: /* call near absolute, call far absolute ind */
502+ insn_get_modrm(&insn);
503+ ext = (insn.modrm.bytes[0] >> 3) & 0x7;
504+ switch (ext) {
505+ case 2: /* near ind call */
506+ case 3: /* far ind call */
507+ ret = X86_BR_IND_CALL;
508+ break;
509+ case 4:
510+ case 5:
511+ ret = X86_BR_JMP;
512+ break;
513+ }
514+ break;
515+ default:
516+ ret = X86_BR_NONE;
517+ }
518+ /*
519+ * interrupts, traps, faults (and thus ring transition) may
520+ * occur on any instructions. Thus, to classify them correctly,
521+ * we need to first look at the from and to priv levels. If they
522+ * are different and to is in the kernel, then it indicates
523+ * a ring transition. If the from instruction is not a ring
524+ * transition instr (syscall, systenter, int), then it means
525+ * it was a irq, trap or fault.
526+ *
527+ * we have no way of detecting kernel to kernel faults.
528+ */
529+ if (from_plm == X86_BR_USER && to_plm == X86_BR_KERNEL
530+ && ret != X86_BR_SYSCALL && ret != X86_BR_INT)
531+ ret = X86_BR_IRQ;
532+
533+ /*
534+ * branch priv level determined by target as
535+ * is done by HW when LBR_SELECT is implemented
536+ */
537+ if (ret != X86_BR_NONE)
538+ ret |= to_plm;
539+
540+ return ret;
541+}
542+
543+/*
544+ * implement actual branch filter based on user demand.
545+ * Hardware may not exactly satisfy that request, thus
546+ * we need to inspect opcodes. Mismatched branches are
547+ * discarded. Therefore, the number of branches returned
548+ * in PERF_SAMPLE_BRANCH_STACK sample may vary.
549+ */
550+static void
551+intel_pmu_lbr_filter(struct cpu_hw_events *cpuc)
552+{
553+ u64 from, to;
554+ int br_sel = cpuc->br_sel;
555+ int i, j, type;
556+ bool compress = false;
557+
558+ /* if sampling all branches, then nothing to filter */
559+ if ((br_sel & X86_BR_ALL) == X86_BR_ALL)
560+ return;
561+
562+ for (i = 0; i < cpuc->lbr_stack.nr; i++) {
563+
564+ from = cpuc->lbr_entries[i].from;
565+ to = cpuc->lbr_entries[i].to;
566+
567+ type = branch_type(from, to);
568+
569+ /* if type does not correspond, then discard */
570+ if (type == X86_BR_NONE || (br_sel & type) != type) {
571+ cpuc->lbr_entries[i].from = 0;
572+ compress = true;
573+ }
574+ }
575+
576+ if (!compress)
577+ return;
578+
579+ /* remove all entries with from=0 */
580+ for (i = 0; i < cpuc->lbr_stack.nr; ) {
581+ if (!cpuc->lbr_entries[i].from) {
582+ j = i;
583+ while (++j < cpuc->lbr_stack.nr)
584+ cpuc->lbr_entries[j-1] = cpuc->lbr_entries[j];
585+ cpuc->lbr_stack.nr--;
586+ if (!cpuc->lbr_entries[i].from)
587+ continue;
588+ }
589+ i++;
590+ }
591+}
592+
593+/*
594+ * Map interface branch filters onto LBR filters
595+ */
596+static const int nhm_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX] = {
597+ [PERF_SAMPLE_BRANCH_ANY] = LBR_ANY,
598+ [PERF_SAMPLE_BRANCH_USER] = LBR_USER,
599+ [PERF_SAMPLE_BRANCH_KERNEL] = LBR_KERNEL,
600+ [PERF_SAMPLE_BRANCH_HV] = LBR_IGN,
601+ [PERF_SAMPLE_BRANCH_ANY_RETURN] = LBR_RETURN | LBR_REL_JMP
602+ | LBR_IND_JMP | LBR_FAR,
603+ /*
604+ * NHM/WSM erratum: must include REL_JMP+IND_JMP to get CALL branches
605+ */
606+ [PERF_SAMPLE_BRANCH_ANY_CALL] =
607+ LBR_REL_CALL | LBR_IND_CALL | LBR_REL_JMP | LBR_IND_JMP | LBR_FAR,
608+ /*
609+ * NHM/WSM erratum: must include IND_JMP to capture IND_CALL
610+ */
611+ [PERF_SAMPLE_BRANCH_IND_CALL] = LBR_IND_CALL | LBR_IND_JMP,
612+};
613+
614+static const int snb_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX] = {
615+ [PERF_SAMPLE_BRANCH_ANY] = LBR_ANY,
616+ [PERF_SAMPLE_BRANCH_USER] = LBR_USER,
617+ [PERF_SAMPLE_BRANCH_KERNEL] = LBR_KERNEL,
618+ [PERF_SAMPLE_BRANCH_HV] = LBR_IGN,
619+ [PERF_SAMPLE_BRANCH_ANY_RETURN] = LBR_RETURN | LBR_FAR,
620+ [PERF_SAMPLE_BRANCH_ANY_CALL] = LBR_REL_CALL | LBR_IND_CALL
621+ | LBR_FAR,
622+ [PERF_SAMPLE_BRANCH_IND_CALL] = LBR_IND_CALL,
623+};
624+
625+/* core */
198626 void intel_pmu_lbr_init_core(void)
199627 {
200628 x86_pmu.lbr_nr = 4;
201− x86_pmu.lbr_tos = 0x01c9;
202− x86_pmu.lbr_from = 0x40;
203− x86_pmu.lbr_to = 0x60;
629+ x86_pmu.lbr_tos = MSR_LBR_TOS;
630+ x86_pmu.lbr_from = MSR_LBR_CORE_FROM;
631+ x86_pmu.lbr_to = MSR_LBR_CORE_TO;
632+
633+ /*
634+ * SW branch filter usage:
635+ * - compensate for lack of HW filter
636+ */
637+ pr_cont("4-deep LBR, ");
204638 }
205639
640+/* nehalem/westmere */
206641 void intel_pmu_lbr_init_nhm(void)
207642 {
208643 x86_pmu.lbr_nr = 16;
209− x86_pmu.lbr_tos = 0x01c9;
210− x86_pmu.lbr_from = 0x680;
211− x86_pmu.lbr_to = 0x6c0;
644+ x86_pmu.lbr_tos = MSR_LBR_TOS;
645+ x86_pmu.lbr_from = MSR_LBR_NHM_FROM;
646+ x86_pmu.lbr_to = MSR_LBR_NHM_TO;
647+
648+ x86_pmu.lbr_sel_mask = LBR_SEL_MASK;
649+ x86_pmu.lbr_sel_map = nhm_lbr_sel_map;
650+
651+ /*
652+ * SW branch filter usage:
653+ * - workaround LBR_SEL errata (see above)
654+ * - support syscall, sysret capture.
655+ * That requires LBR_FAR but that means far
656+ * jmp need to be filtered out
657+ */
658+ pr_cont("16-deep LBR, ");
659+}
660+
661+/* sandy bridge */
662+void intel_pmu_lbr_init_snb(void)
663+{
664+ x86_pmu.lbr_nr = 16;
665+ x86_pmu.lbr_tos = MSR_LBR_TOS;
666+ x86_pmu.lbr_from = MSR_LBR_NHM_FROM;
667+ x86_pmu.lbr_to = MSR_LBR_NHM_TO;
668+
669+ x86_pmu.lbr_sel_mask = LBR_SEL_MASK;
670+ x86_pmu.lbr_sel_map = snb_lbr_sel_map;
671+
672+ /*
673+ * SW branch filter usage:
674+ * - support syscall, sysret capture.
675+ * That requires LBR_FAR but that means far
676+ * jmp need to be filtered out
677+ */
678+ pr_cont("16-deep LBR, ");
212679 }
213680
681+/* atom */
214682 void intel_pmu_lbr_init_atom(void)
215683 {
684+ /*
685+ * only models starting at stepping 10 seems
686+ * to have an operational LBR which can freeze
687+ * on PMU interrupt
688+ */
689+ if (boot_cpu_data.x86_mask < 10) {
690+ pr_cont("LBR disabled due to erratum");
691+ return;
692+ }
693+
216694 x86_pmu.lbr_nr = 8;
217− x86_pmu.lbr_tos = 0x01c9;
218− x86_pmu.lbr_from = 0x40;
219− x86_pmu.lbr_to = 0x60;
695+ x86_pmu.lbr_tos = MSR_LBR_TOS;
696+ x86_pmu.lbr_from = MSR_LBR_CORE_FROM;
697+ x86_pmu.lbr_to = MSR_LBR_CORE_TO;
698+
699+ /*
700+ * SW branch filter usage:
701+ * - compensate for lack of HW filter
702+ */
703+ pr_cont("8-deep LBR, ");
220704 }
include/linux/perf_event.h+77−5
@@ -129,10 +129,39 @@ enum perf_event_sample_format { · 128 unmodified lines
129129 PERF_SAMPLE_PERIOD = 1U << 8,
130130 PERF_SAMPLE_STREAM_ID = 1U << 9,
131131 PERF_SAMPLE_RAW = 1U << 10,
132+ PERF_SAMPLE_BRANCH_STACK = 1U << 11,
132133
133− PERF_SAMPLE_MAX = 1U << 11, /* non-ABI */
134+ PERF_SAMPLE_MAX = 1U << 12, /* non-ABI */
134135 };
135136
137+/*
138+ * values to program into branch_sample_type when PERF_SAMPLE_BRANCH is set
139+ *
140+ * If the user does not pass priv level information via branch_sample_type,
141+ * the kernel uses the event's priv level. Branch and event priv levels do
142+ * not have to match. Branch priv level is checked for permissions.
143+ *
144+ * The branch types can be combined, however BRANCH_ANY covers all types
145+ * of branches and therefore it supersedes all the other types.
146+ */
147+enum perf_branch_sample_type {
148+ PERF_SAMPLE_BRANCH_USER = 1U << 0, /* user branches */
149+ PERF_SAMPLE_BRANCH_KERNEL = 1U << 1, /* kernel branches */
150+ PERF_SAMPLE_BRANCH_HV = 1U << 2, /* hypervisor branches */
151+
152+ PERF_SAMPLE_BRANCH_ANY = 1U << 3, /* any branch types */
153+ PERF_SAMPLE_BRANCH_ANY_CALL = 1U << 4, /* any call branch */
154+ PERF_SAMPLE_BRANCH_ANY_RETURN = 1U << 5, /* any return branch */
155+ PERF_SAMPLE_BRANCH_IND_CALL = 1U << 6, /* indirect calls */
156+
157+ PERF_SAMPLE_BRANCH_MAX = 1U << 7, /* non-ABI */
158+};
159+
160+#define PERF_SAMPLE_BRANCH_PLM_ALL \
161+ (PERF_SAMPLE_BRANCH_USER|\
162+ PERF_SAMPLE_BRANCH_KERNEL|\
163+ PERF_SAMPLE_BRANCH_HV)
164+
136165 /*
137166 * The format of the data returned by read() on a perf event fd,
138167 * as specified by attr.read_format:
@@ -163,6 +192,8 @@ enum perf_event_read_format { · 24 unmodified lines
163192 };
164193
165194 #define PERF_ATTR_SIZE_VER0 64 /* sizeof first published struct */
195+#define PERF_ATTR_SIZE_VER1 72 /* add: config2 */
196+#define PERF_ATTR_SIZE_VER2 80 /* add: branch_sample_type */
166197
167198 /*
168199 * Hardware event_id to monitor via a performance monitoring event:
@@ -240,6 +271,7 @@ struct perf_event_attr { · 71 unmodified lines
240271 __u64 bp_len;
241272 __u64 config2; /* extension of config1 */
242273 };
274+ __u64 branch_sample_type; /* enum branch_sample_type */
243275 };
244276
245277 /*
@@ -458,6 +490,8 @@ enum perf_event_type { · 212 unmodified lines
458490 *
459491 * { u32 size;
460492 * char data[size];}&& PERF_SAMPLE_RAW
493+ *
494+ * { u64 from, to, flags } lbr[nr];} && PERF_SAMPLE_BRANCH_STACK
461495 * };
462496 */
463497 PERF_RECORD_SAMPLE = 9,
@@ -530,12 +564,34 @@ struct perf_raw_record { · 66 unmodified lines
530564 void *data;
531565 };
532566
567+/*
568+ * single taken branch record layout:
569+ *
570+ * from: source instruction (may not always be a branch insn)
571+ * to: branch target
572+ * mispred: branch target was mispredicted
573+ * predicted: branch target was predicted
574+ *
575+ * support for mispred, predicted is optional. In case it
576+ * is not supported mispred = predicted = 0.
577+ */
533578 struct perf_branch_entry {
534− __u64 from;
535− __u64 to;
536− __u64 flags;
579+ __u64 from;
580+ __u64 to;
581+ __u64 mispred:1, /* target mispredicted */
582+ predicted:1,/* target predicted */
583+ reserved:62;
537584 };
538585
586+/*
587+ * branch stack layout:
588+ * nr: number of taken branches stored in entries[]
589+ *
590+ * Note that nr can vary from sample to sample
591+ * branches (to, from) are stored from most recent
592+ * to least recent, i.e., entries[0] contains the most
593+ * recent branch.
594+ */
539595 struct perf_branch_stack {
540596 __u64 nr;
541597 struct perf_branch_entry entries[0];
@@ -566,7 +622,9 @@ struct hw_perf_event { · 24 unmodified lines
566622 unsigned long event_base;
567623 int idx;
568624 int last_cpu;
625+
569626 struct hw_perf_event_extra extra_reg;
627+ struct hw_perf_event_extra branch_reg;
570628 };
571629 struct { /* software */
572630 struct hrtimer hrtimer;
@@ -690,6 +748,11 @@ struct pmu { · 117 unmodified lines
690748 * if no implementation is provided it will default to: event->hw.idx + 1.
691749 */
692750 int (*event_idx) (struct perf_event *event); /*optional */
751+
752+ /*
753+ * flush branch stack on context-switches (needed in cpu-wide mode)
754+ */
755+ void (*flush_branch_stack) (void);
693756 };
694757
695758 /**
@@ -923,7 +986,8 @@ struct perf_event_context { · 227 unmodified lines
923986 u64 parent_gen;
924987 u64 generation;
925988 int pin_count;
926− int nr_cgroups; /* cgroup events present */
989+ int nr_cgroups; /* cgroup evts */
990+ int nr_branch_stack; /* branch_stack evt */
927991 struct rcu_head rcu_head;
928992 };
929993
@@ -988,6 +1052,7 @@ perf_event_create_kernel_counter(struct perf_event_attr *attr, · 58 unmodified lines
9881052 extern u64 perf_event_read_value(struct perf_event *event,
9891053 u64 *enabled, u64 *running);
9901054
1055+
9911056 struct perf_sample_data {
9921057 u64 type;
9931058
@@ -1007,12 +1072,14 @@ struct perf_sample_data { · 13 unmodified lines
10071072 u64 period;
10081073 struct perf_callchain_entry *callchain;
10091074 struct perf_raw_record *raw;
1075+ struct perf_branch_stack *br_stack;
10101076 };
10111077
10121078 static inline void perf_sample_data_init(struct perf_sample_data *data, u64 addr)
10131079 {
10141080 data->addr = addr;
10151081 data->raw = NULL;
1082+ data->br_stack = NULL;
10161083 }
10171084
10181085 extern void perf_output_sample(struct perf_output_handle *handle,
@@ -1151,6 +1218,11 @@ extern void perf_bp_event(struct perf_event *event, void *data); · 132 unmodified lines
11511218 # define perf_instruction_pointer(regs) instruction_pointer(regs)
11521219 #endif
11531220
1221+static inline bool has_branch_stack(struct perf_event *event)
1222+{
1223+ return event->attr.sample_type & PERF_SAMPLE_BRANCH_STACK;
1224+}
1225+
11541226 extern int perf_output_begin(struct perf_output_handle *handle,
11551227 struct perf_event *event, unsigned int size);
11561228 extern void perf_output_end(struct perf_output_handle *handle);
kernel/events/core.c+177−0
@@ -118,6 +118,13 @@ static int cpu_function_call(int cpu, int (*func) (void *info), void *info) · 117 unmodified lines
118118 PERF_FLAG_FD_OUTPUT |\
119119 PERF_FLAG_PID_CGROUP)
120120
121+/*
122+ * branch priv levels that need permission checks
123+ */
124+#define PERF_SAMPLE_BRANCH_PERM_PLM \
125+ (PERF_SAMPLE_BRANCH_KERNEL |\
126+ PERF_SAMPLE_BRANCH_HV)
127+
121128 enum event_type_t {
122129 EVENT_FLEXIBLE = 0x1,
123130 EVENT_PINNED = 0x2,
@@ -130,6 +137,7 @@ enum event_type_t { · 6 unmodified lines
130137 */
131138 struct static_key_deferred perf_sched_events __read_mostly;
132139 static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
140+static DEFINE_PER_CPU(atomic_t, perf_branch_stack_events);
133141
134142 static atomic_t nr_mmap_events __read_mostly;
135143 static atomic_t nr_comm_events __read_mostly;
@@ -881,6 +889,9 @@ list_add_event(struct perf_event *event, struct perf_event_context *ctx) · 745 unmodified lines
881889 if (is_cgroup_event(event))
882890 ctx->nr_cgroups++;
883891
892+ if (has_branch_stack(event))
893+ ctx->nr_branch_stack++;
894+
884895 list_add_rcu(&event->event_entry, &ctx->event_list);
885896 if (!ctx->nr_events)
886897 perf_pmu_rotate_start(ctx->pmu);
@@ -1020,6 +1031,9 @@ list_del_event(struct perf_event *event, struct perf_event_context *ctx) · 133 unmodified lines
10201031 cpuctx->cgrp = NULL;
10211032 }
10221033
1034+ if (has_branch_stack(event))
1035+ ctx->nr_branch_stack--;
1036+
10231037 ctx->nr_events--;
10241038 if (event->attr.inherit_stat)
10251039 ctx->nr_stat--;
@@ -2194,6 +2208,66 @@ static void perf_event_context_sched_in(struct perf_event_context *ctx, · 1168 unmodified lines
21942208 perf_pmu_rotate_start(ctx->pmu);
21952209 }
21962210
2211+/*
2212+ * When sampling the branck stack in system-wide, it may be necessary
2213+ * to flush the stack on context switch. This happens when the branch
2214+ * stack does not tag its entries with the pid of the current task.
2215+ * Otherwise it becomes impossible to associate a branch entry with a
2216+ * task. This ambiguity is more likely to appear when the branch stack
2217+ * supports priv level filtering and the user sets it to monitor only
2218+ * at the user level (which could be a useful measurement in system-wide
2219+ * mode). In that case, the risk is high of having a branch stack with
2220+ * branch from multiple tasks. Flushing may mean dropping the existing
2221+ * entries or stashing them somewhere in the PMU specific code layer.
2222+ *
2223+ * This function provides the context switch callback to the lower code
2224+ * layer. It is invoked ONLY when there is at least one system-wide context
2225+ * with at least one active event using taken branch sampling.
2226+ */
2227+static void perf_branch_stack_sched_in(struct task_struct *prev,
2228+ struct task_struct *task)
2229+{
2230+ struct perf_cpu_context *cpuctx;
2231+ struct pmu *pmu;
2232+ unsigned long flags;
2233+
2234+ /* no need to flush branch stack if not changing task */
2235+ if (prev == task)
2236+ return;
2237+
2238+ local_irq_save(flags);
2239+
2240+ rcu_read_lock();
2241+
2242+ list_for_each_entry_rcu(pmu, &pmus, entry) {
2243+ cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
2244+
2245+ /*
2246+ * check if the context has at least one
2247+ * event using PERF_SAMPLE_BRANCH_STACK
2248+ */
2249+ if (cpuctx->ctx.nr_branch_stack > 0
2250+ && pmu->flush_branch_stack) {
2251+
2252+ pmu = cpuctx->ctx.pmu;
2253+
2254+ perf_ctx_lock(cpuctx, cpuctx->task_ctx);
2255+
2256+ perf_pmu_disable(pmu);
2257+
2258+ pmu->flush_branch_stack();
2259+
2260+ perf_pmu_enable(pmu);
2261+
2262+ perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
2263+ }
2264+ }
2265+
2266+ rcu_read_unlock();
2267+
2268+ local_irq_restore(flags);
2269+}
2270+
21972271 /*
21982272 * Called from scheduler to add the events of the current task
21992273 * with interrupts disabled.
@@ -2225,6 +2299,10 @@ void __perf_event_task_sched_in(struct task_struct *prev, · 25 unmodified lines
22252299 */
22262300 if (atomic_read(&__get_cpu_var(perf_cgroup_events)))
22272301 perf_cgroup_sched_in(prev, task);
2302+
2303+ /* check for system-wide branch_stack events */
2304+ if (atomic_read(&__get_cpu_var(perf_branch_stack_events)))
2305+ perf_branch_stack_sched_in(prev, task);
22282306 }
22292307
22302308 static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
@@ -2791,6 +2869,14 @@ static void free_event(struct perf_event *event) · 560 unmodified lines
27912869 atomic_dec(&per_cpu(perf_cgroup_events, event->cpu));
27922870 static_key_slow_dec_deferred(&perf_sched_events);
27932871 }
2872+
2873+ if (has_branch_stack(event)) {
2874+ static_key_slow_dec_deferred(&perf_sched_events);
2875+ /* is system-wide event */
2876+ if (!(event->attach_state & PERF_ATTACH_TASK))
2877+ atomic_dec(&per_cpu(perf_branch_stack_events,
2878+ event->cpu));
2879+ }
27942880 }
27952881
27962882 if (event->rb) {
@@ -3907,6 +3993,24 @@ void perf_output_sample(struct perf_output_handle *handle, · 1110 unmodified lines
39073993 }
39083994 }
39093995 }
3996+
3997+ if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
3998+ if (data->br_stack) {
3999+ size_t size;
4000+
4001+ size = data->br_stack->nr
4002+ * sizeof(struct perf_branch_entry);
4003+
4004+ perf_output_put(handle, data->br_stack->nr);
4005+ perf_output_copy(handle, data->br_stack->entries, size);
4006+ } else {
4007+ /*
4008+ * we always store at least the value of nr
4009+ */
4010+ u64 nr = 0;
4011+ perf_output_put(handle, nr);
4012+ }
4013+ }
39104014 }
39114015
39124016 void perf_prepare_sample(struct perf_event_header *header,
@@ -3949,6 +4053,15 @@ void perf_prepare_sample(struct perf_event_header *header, · 36 unmodified lines
39494053 WARN_ON_ONCE(size & (sizeof(u64)-1));
39504054 header->size += size;
39514055 }
4056+
4057+ if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
4058+ int size = sizeof(u64); /* nr */
4059+ if (data->br_stack) {
4060+ size += data->br_stack->nr
4061+ * sizeof(struct perf_branch_entry);
4062+ }
4063+ header->size += size;
4064+ }
39524065 }
39534066
39544067 static void perf_event_output(struct perf_event *event,
@@ -5010,6 +5123,12 @@ static int perf_swevent_init(struct perf_event *event) · 1055 unmodified lines
50105123 if (event->attr.type != PERF_TYPE_SOFTWARE)
50115124 return -ENOENT;
50125125
5126+ /*
5127+ * no branch sampling for software events
5128+ */
5129+ if (has_branch_stack(event))
5130+ return -EOPNOTSUPP;
5131+
50135132 switch (event_id) {
50145133 case PERF_COUNT_SW_CPU_CLOCK:
50155134 case PERF_COUNT_SW_TASK_CLOCK:
@@ -5120,6 +5239,12 @@ static int perf_tp_event_init(struct perf_event *event) · 104 unmodified lines
51205239 if (event->attr.type != PERF_TYPE_TRACEPOINT)
51215240 return -ENOENT;
51225241
5242+ /*
5243+ * no branch sampling for tracepoint events
5244+ */
5245+ if (has_branch_stack(event))
5246+ return -EOPNOTSUPP;
5247+
51235248 err = perf_trace_init(event);
51245249 if (err)
51255250 return err;
@@ -5345,6 +5470,12 @@ static int cpu_clock_event_init(struct perf_event *event) · 219 unmodified lines
53455470 if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)
53465471 return -ENOENT;
53475472
5473+ /*
5474+ * no branch sampling for software events
5475+ */
5476+ if (has_branch_stack(event))
5477+ return -EOPNOTSUPP;
5478+
53485479 perf_swevent_init_hrtimer(event);
53495480
53505481 return 0;
@@ -5419,6 +5550,12 @@ static int task_clock_event_init(struct perf_event *event) · 68 unmodified lines
54195550 if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)
54205551 return -ENOENT;
54215552
5553+ /*
5554+ * no branch sampling for software events
5555+ */
5556+ if (has_branch_stack(event))
5557+ return -EOPNOTSUPP;
5558+
54225559 perf_swevent_init_hrtimer(event);
54235560
54245561 return 0;
@@ -5866,6 +6003,12 @@ done: · 441 unmodified lines
58666003 return ERR_PTR(err);
58676004 }
58686005 }
6006+ if (has_branch_stack(event)) {
6007+ static_key_slow_inc(&perf_sched_events.key);
6008+ if (!(event->attach_state & PERF_ATTACH_TASK))
6009+ atomic_inc(&per_cpu(perf_branch_stack_events,
6010+ event->cpu));
6011+ }
58696012 }
58706013
58716014 return event;
@@ -5935,6 +6078,40 @@ static int perf_copy_attr(struct perf_event_attr __user *uattr, · 63 unmodified lines
59356078 if (attr->read_format & ~(PERF_FORMAT_MAX-1))
59366079 return -EINVAL;
59376080
6081+ if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) {
6082+ u64 mask = attr->branch_sample_type;
6083+
6084+ /* only using defined bits */
6085+ if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1))
6086+ return -EINVAL;
6087+
6088+ /* at least one branch bit must be set */
6089+ if (!(mask & ~PERF_SAMPLE_BRANCH_PLM_ALL))
6090+ return -EINVAL;
6091+
6092+ /* kernel level capture: check permissions */
6093+ if ((mask & PERF_SAMPLE_BRANCH_PERM_PLM)
6094+ && perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
6095+ return -EACCES;
6096+
6097+ /* propagate priv level, when not set for branch */
6098+ if (!(mask & PERF_SAMPLE_BRANCH_PLM_ALL)) {
6099+
6100+ /* exclude_kernel checked on syscall entry */
6101+ if (!attr->exclude_kernel)
6102+ mask |= PERF_SAMPLE_BRANCH_KERNEL;
6103+
6104+ if (!attr->exclude_user)
6105+ mask |= PERF_SAMPLE_BRANCH_USER;
6106+
6107+ if (!attr->exclude_hv)
6108+ mask |= PERF_SAMPLE_BRANCH_HV;
6109+ /*
6110+ * adjust user setting (for HW filter setup)
6111+ */
6112+ attr->branch_sample_type = mask;
6113+ }
6114+ }
59386115 out:
59396116 return ret;
59406117
kernel/events/hw_breakpoint.c+6−0
@@ -581,6 +581,12 @@ static int hw_breakpoint_event_init(struct perf_event *bp) · 580 unmodified lines
581581 if (bp->attr.type != PERF_TYPE_BREAKPOINT)
582582 return -ENOENT;
583583
584+ /*
585+ * no branch sampling for breakpoint events
586+ */
587+ if (has_branch_stack(bp))
588+ return -EOPNOTSUPP;
589+
584590 err = register_perf_hw_breakpoint(bp);
585591 if (err)
586592 return err;
tools/perf/Documentation/perf-record.txt+30−0
@@ -152,6 +152,36 @@ an empty cgroup (monitor all the time) using, e.g., -G foo,,bar. Cgroups must ha · 151 unmodified lines
152152 corresponding events, i.e., they always refer to events defined earlier on the command
153153 line.
154154
155+-b::
156+--branch-any::
157+Enable taken branch stack sampling. Any type of taken branch may be sampled.
158+This is a shortcut for --branch-filter any. See --branch-filter for more infos.
159+
160+-j::
161+--branch-filter::
162+Enable taken branch stack sampling. Each sample captures a series of consecutive
163+taken branches. The number of branches captured with each sample depends on the
164+underlying hardware, the type of branches of interest, and the executed code.
165+It is possible to select the types of branches captured by enabling filters. The
166+following filters are defined:
167+
168+ - any: any type of branches
169+ - any_call: any function call or system call
170+ - any_ret: any function return or system call return
171+ - any_ind: any indirect branch
172+ - u: only when the branch target is at the user level
173+ - k: only when the branch target is in the kernel
174+ - hv: only when the target is at the hypervisor level
175+
176++
177+The option requires at least one branch type among any, any_call, any_ret, ind_call.
178+The privilege levels may be ommitted, in which case, the privilege levels of the associated
179+event are applied to the branch filter. Both kernel (k) and hypervisor (hv) privilege
180+levels are subject to permissions. When sampling on multiple events, branch stack sampling
181+is enabled for all the sampling events. The sampled branch type is the same for all events.
182+The various filters must be specified as a comma separated list: --branch-filter any_ret,u,k
183+Note that this feature may not be available on all processors.
184+
155185 SEE ALSO
156186 --------
157187 linkperf:perf-stat[1], linkperf:perf-list[1]
tools/perf/Documentation/perf-report.txt+10−0
@@ -153,6 +153,16 @@ OPTIONS · 152 unmodified lines
153153 information which may be very large and thus may clutter the display.
154154 It currently includes: cpu and numa topology of the host system.
155155
156+-b::
157+--branch-stack::
158+ Use the addresses of sampled taken branches instead of the instruction
159+ address to build the histograms. To generate meaningful output, the
160+ perf.data file must have been obtained using perf record -b or
161+ perf record --branch-filter xxx where xxx is a branch filter option.
162+ perf report is able to auto-detect whether a perf.data file contains
163+ branch stacks and it will automatically switch to the branch view mode,
164+ unless --no-branch-stack is used.
165+
156166 SEE ALSO
157167 --------
158168 linkperf:perf-stat[1], linkperf:perf-annotate[1]
tools/perf/builtin-record.c+95−0
@@ -473,6 +473,9 @@ static int __cmd_record(struct perf_record *rec, int argc, const char **argv) · 472 unmodified lines
473473 if (!have_tracepoints(&evsel_list->entries))
474474 perf_header__clear_feat(&session->header, HEADER_TRACE_INFO);
475475
476+ if (!rec->opts.branch_stack)
477+ perf_header__clear_feat(&session->header, HEADER_BRANCH_STACK);
478+
476479 if (!rec->file_new) {
477480 err = perf_session__read_header(session, output);
478481 if (err < 0)
@@ -638,6 +641,90 @@ out_delete_session: · 159 unmodified lines
638641 return err;
639642 }
640643
644+#define BRANCH_OPT(n, m) \
645+ { .name = n, .mode = (m) }
646+
647+#define BRANCH_END { .name = NULL }
648+
649+struct branch_mode {
650+ const char *name;
651+ int mode;
652+};
653+
654+static const struct branch_mode branch_modes[] = {
655+ BRANCH_OPT("u", PERF_SAMPLE_BRANCH_USER),
656+ BRANCH_OPT("k", PERF_SAMPLE_BRANCH_KERNEL),
657+ BRANCH_OPT("hv", PERF_SAMPLE_BRANCH_HV),
658+ BRANCH_OPT("any", PERF_SAMPLE_BRANCH_ANY),
659+ BRANCH_OPT("any_call", PERF_SAMPLE_BRANCH_ANY_CALL),
660+ BRANCH_OPT("any_ret", PERF_SAMPLE_BRANCH_ANY_RETURN),
661+ BRANCH_OPT("ind_call", PERF_SAMPLE_BRANCH_IND_CALL),
662+ BRANCH_END
663+};
664+
665+static int
666+parse_branch_stack(const struct option *opt, const char *str, int unset)
667+{
668+#define ONLY_PLM \
669+ (PERF_SAMPLE_BRANCH_USER |\
670+ PERF_SAMPLE_BRANCH_KERNEL |\
671+ PERF_SAMPLE_BRANCH_HV)
672+
673+ uint64_t *mode = (uint64_t *)opt->value;
674+ const struct branch_mode *br;
675+ char *s, *os = NULL, *p;
676+ int ret = -1;
677+
678+ if (unset)
679+ return 0;
680+
681+ /*
682+ * cannot set it twice, -b + --branch-filter for instance
683+ */
684+ if (*mode)
685+ return -1;
686+
687+ /* str may be NULL in case no arg is passed to -b */
688+ if (str) {
689+ /* because str is read-only */
690+ s = os = strdup(str);
691+ if (!s)
692+ return -1;
693+
694+ for (;;) {
695+ p = strchr(s, ',');
696+ if (p)
697+ *p = '\0';
698+
699+ for (br = branch_modes; br->name; br++) {
700+ if (!strcasecmp(s, br->name))
701+ break;
702+ }
703+ if (!br->name) {
704+ ui__warning("unknown branch filter %s,"
705+ " check man page\n", s);
706+ goto error;
707+ }
708+
709+ *mode |= br->mode;
710+
711+ if (!p)
712+ break;
713+
714+ s = p + 1;
715+ }
716+ }
717+ ret = 0;
718+
719+ /* default to any branch */
720+ if ((*mode & ~ONLY_PLM) == 0) {
721+ *mode = PERF_SAMPLE_BRANCH_ANY;
722+ }
723+error:
724+ free(os);
725+ return ret;
726+}
727+
641728 static const char * const record_usage[] = {
642729 "perf record [<options>] [<command>]",
643730 "perf record [<options>] -- <command> [<options>]",
@@ -727,6 +814,14 @@ const struct option record_options[] = { · 83 unmodified lines
727814 "monitor event in cgroup name only",
728815 parse_cgroups),
729816 OPT_STRING('u', "uid", &record.uid_str, "user", "user to profile"),
817+
818+ OPT_CALLBACK_NOOPT('b', "branch-any", &record.opts.branch_stack,
819+ "branch any", "sample any taken branches",
820+ parse_branch_stack),
821+
822+ OPT_CALLBACK('j', "branch-filter", &record.opts.branch_stack,
823+ "branch filter mask", "branch stack filter modes",
824+ parse_branch_stack),
730825 OPT_END()
731826 };
732827
tools/perf/builtin-report.c+156−22
@@ -53,6 +53,82 @@ struct perf_report { · 52 unmodified lines
5353 DECLARE_BITMAP(cpu_bitmap, MAX_NR_CPUS);
5454 };
5555
56+static int perf_report__add_branch_hist_entry(struct perf_tool *tool,
57+ struct addr_location *al,
58+ struct perf_sample *sample,
59+ struct perf_evsel *evsel,
60+ struct machine *machine)
61+{
62+ struct perf_report *rep = container_of(tool, struct perf_report, tool);
63+ struct symbol *parent = NULL;
64+ int err = 0;
65+ unsigned i;
66+ struct hist_entry *he;
67+ struct branch_info *bi, *bx;
68+
69+ if ((sort__has_parent || symbol_conf.use_callchain)
70+ && sample->callchain) {
71+ err = machine__resolve_callchain(machine, evsel, al->thread,
72+ sample->callchain, &parent);
73+ if (err)
74+ return err;
75+ }
76+
77+ bi = machine__resolve_bstack(machine, al->thread,
78+ sample->branch_stack);
79+ if (!bi)
80+ return -ENOMEM;
81+
82+ for (i = 0; i < sample->branch_stack->nr; i++) {
83+ if (rep->hide_unresolved && !(bi[i].from.sym && bi[i].to.sym))
84+ continue;
85+ /*
86+ * The report shows the percentage of total branches captured
87+ * and not events sampled. Thus we use a pseudo period of 1.
88+ */
89+ he = __hists__add_branch_entry(&evsel->hists, al, parent,
90+ &bi[i], 1);
91+ if (he) {
92+ struct annotation *notes;
93+ err = -ENOMEM;
94+ bx = he->branch_info;
95+ if (bx->from.sym && use_browser > 0) {
96+ notes = symbol__annotation(bx->from.sym);
97+ if (!notes->src
98+ && symbol__alloc_hist(bx->from.sym) < 0)
99+ goto out;
100+
101+ err = symbol__inc_addr_samples(bx->from.sym,
102+ bx->from.map,
103+ evsel->idx,
104+ bx->from.al_addr);
105+ if (err)
106+ goto out;
107+ }
108+
109+ if (bx->to.sym && use_browser > 0) {
110+ notes = symbol__annotation(bx->to.sym);
111+ if (!notes->src
112+ && symbol__alloc_hist(bx->to.sym) < 0)
113+ goto out;
114+
115+ err = symbol__inc_addr_samples(bx->to.sym,
116+ bx->to.map,
117+ evsel->idx,
118+ bx->to.al_addr);
119+ if (err)
120+ goto out;
121+ }
122+ evsel->hists.stats.total_period += 1;
123+ hists__inc_nr_events(&evsel->hists, PERF_RECORD_SAMPLE);
124+ err = 0;
125+ } else
126+ return -ENOMEM;
127+ }
128+out:
129+ return err;
130+}
131+
56132 static int perf_evsel__add_hist_entry(struct perf_evsel *evsel,
57133 struct addr_location *al,
58134 struct perf_sample *sample,
@@ -126,14 +202,21 @@ static int process_sample_event(struct perf_tool *tool, · 67 unmodified lines
126202 if (rep->cpu_list && !test_bit(sample->cpu, rep->cpu_bitmap))
127203 return 0;
128204
129− if (al.map != NULL)
130− al.map->dso->hit = 1;
205+ if (sort__branch_mode == 1) {
206+ if (perf_report__add_branch_hist_entry(tool, &al, sample,
207+ evsel, machine)) {
208+ pr_debug("problem adding lbr entry, skipping event\n");
209+ return -1;
210+ }
211+ } else {
212+ if (al.map != NULL)
213+ al.map->dso->hit = 1;
131214
132− if (perf_evsel__add_hist_entry(evsel, &al, sample, machine)) {
133− pr_debug("problem incrementing symbol period, skipping event\n");
134− return -1;
215+ if (perf_evsel__add_hist_entry(evsel, &al, sample, machine)) {
216+ pr_debug("problem incrementing symbol period, skipping event\n");
217+ return -1;
218+ }
135219 }
136−
137220 return 0;
138221 }
139222
@@ -188,6 +271,15 @@ static int perf_report__setup_sample_type(struct perf_report *rep) · 48 unmodified lines
188271 }
189272 }
190273
274+ if (sort__branch_mode == 1) {
275+ if (!(self->sample_type & PERF_SAMPLE_BRANCH_STACK)) {
276+ fprintf(stderr, "selected -b but no branch data."
277+ " Did you call perf record without"
278+ " -b?\n");
279+ return -1;
280+ }
281+ }
282+
191283 return 0;
192284 }
193285
@@ -246,7 +338,7 @@ static int __cmd_report(struct perf_report *rep) · 52 unmodified lines
246338 {
247339 int ret = -EINVAL;
248340 u64 nr_samples;
249− struct perf_session *session;
341+ struct perf_session *session = rep->session;
250342 struct perf_evsel *pos;
251343 struct map *kernel_map;
252344 struct kmap *kernel_kmap;
@@ -254,13 +346,6 @@ static int __cmd_report(struct perf_report *rep) · 1 unmodified line
254346
255347 signal(SIGINT, sig_handler);
256348
257− session = perf_session__new(rep->input_name, O_RDONLY,
258− rep->force, false, &rep->tool);
259− if (session == NULL)
260− return -ENOMEM;
261−
262− rep->session = session;
263−
264349 if (rep->cpu_list) {
265350 ret = perf_session__cpu_bitmap(session, rep->cpu_list,
266351 rep->cpu_bitmap);
@@ -427,9 +512,19 @@ setup: · 160 unmodified lines
427512 return 0;
428513 }
429514
515+static int
516+parse_branch_mode(const struct option *opt __used, const char *str __used, int unset)
517+{
518+ sort__branch_mode = !unset;
519+ return 0;
520+}
521+
430522 int cmd_report(int argc, const char **argv, const char *prefix __used)
431523 {
524+ struct perf_session *session;
432525 struct stat st;
526+ bool has_br_stack = false;
527+ int ret = -1;
433528 char callchain_default_opt[] = "fractal,0.5,callee";
434529 const char * const report_usage[] = {
435530 "perf report [<options>]",
@@ -477,7 +572,8 @@ int cmd_report(int argc, const char **argv, const char *prefix __used) · 41 unmodified lines
477572 OPT_BOOLEAN(0, "stdio", &report.use_stdio,
478573 "Use the stdio interface"),
479574 OPT_STRING('s', "sort", &sort_order, "key[,key2...]",
480− "sort by key(s): pid, comm, dso, symbol, parent"),
575+ "sort by key(s): pid, comm, dso, symbol, parent, dso_to,"
576+ " dso_from, symbol_to, symbol_from, mispredict"),
481577 OPT_BOOLEAN(0, "showcpuutilization", &symbol_conf.show_cpu_utilization,
482578 "Show sample percentage for different cpu modes"),
483579 OPT_STRING('p', "parent", &parent_pattern, "regex",
@@ -517,6 +613,8 @@ int cmd_report(int argc, const char **argv, const char *prefix __used) · 33 unmodified lines
517613 "Specify disassembler style (e.g. -M intel for intel syntax)"),
518614 OPT_BOOLEAN(0, "show-total-period", &symbol_conf.show_total_period,
519615 "Show a column with the sum of periods"),
616+ OPT_CALLBACK_NOOPT('b', "branch-stack", &sort__branch_mode, "",
617+ "use branch records for histogram filling", parse_branch_mode),
520618 OPT_END()
521619 };
522620
@@ -536,11 +634,36 @@ int cmd_report(int argc, const char **argv, const char *prefix __used) · 13 unmodified lines
536634 else
537635 report.input_name = "perf.data";
538636 }
637+ session = perf_session__new(report.input_name, O_RDONLY,
638+ report.force, false, &report.tool);
639+ if (session == NULL)
640+ return -ENOMEM;
539641
540− if (strcmp(report.input_name, "-") != 0)
642+ report.session = session;
643+
644+ has_br_stack = perf_header__has_feat(&session->header,
645+ HEADER_BRANCH_STACK);
646+
647+ if (sort__branch_mode == -1 && has_br_stack)
648+ sort__branch_mode = 1;
649+
650+ /* sort__branch_mode could be 0 if --no-branch-stack */
651+ if (sort__branch_mode == 1) {
652+ /*
653+ * if no sort_order is provided, then specify
654+ * branch-mode specific order
655+ */
656+ if (sort_order == default_sort_order)
657+ sort_order = "comm,dso_from,symbol_from,"
658+ "dso_to,symbol_to";
659+
660+ }
661+
662+ if (strcmp(report.input_name, "-") != 0) {
541663 setup_browser(true);
542− else
664+ } else {
543665 use_browser = 0;
666+ }
544667
545668 /*
546669 * Only in the newt browser we are doing integrated annotation,
@@ -568,13 +691,13 @@ int cmd_report(int argc, const char **argv, const char *prefix __used) · 21 unmodified lines
568691 }
569692
570693 if (symbol__init() < 0)
571− return -1;
694+ goto error;
572695
573696 setup_sorting(report_usage, options);
574697
575698 if (parent_pattern != default_parent_pattern) {
576699 if (sort_dimension__add("parent") < 0)
577− return -1;
700+ goto error;
578701
579702 /*
580703 * Only show the parent fields if we explicitly
@@ -592,9 +715,20 @@ int cmd_report(int argc, const char **argv, const char *prefix __used) · 11 unmodified lines
592715 if (argc)
593716 usage_with_options(report_usage, options);
594717
595− sort_entry__setup_elide(&sort_dso, symbol_conf.dso_list, "dso", stdout);
596718 sort_entry__setup_elide(&sort_comm, symbol_conf.comm_list, "comm", stdout);
597− sort_entry__setup_elide(&sort_sym, symbol_conf.sym_list, "symbol", stdout);
598719
599− return __cmd_report(&report);
720+ if (sort__branch_mode == 1) {
721+ sort_entry__setup_elide(&sort_dso_from, symbol_conf.dso_from_list, "dso_from", stdout);
722+ sort_entry__setup_elide(&sort_dso_to, symbol_conf.dso_to_list, "dso_to", stdout);
723+ sort_entry__setup_elide(&sort_sym_from, symbol_conf.sym_from_list, "sym_from", stdout);
724+ sort_entry__setup_elide(&sort_sym_to, symbol_conf.sym_to_list, "sym_to", stdout);
725+ } else {
726+ sort_entry__setup_elide(&sort_dso, symbol_conf.dso_list, "dso", stdout);
727+ sort_entry__setup_elide(&sort_sym, symbol_conf.sym_list, "symbol", stdout);
728+ }
729+
730+ ret = __cmd_report(&report);
731+error:
732+ perf_session__delete(session);
733+ return ret;
600734 }
tools/perf/perf.h+18−0
@@ -179,6 +179,23 @@ struct ip_callchain { · 178 unmodified lines
179179 u64 ips[0];
180180 };
181181
182+struct branch_flags {
183+ u64 mispred:1;
184+ u64 predicted:1;
185+ u64 reserved:62;
186+};
187+
188+struct branch_entry {
189+ u64 from;
190+ u64 to;
191+ struct branch_flags flags;
192+};
193+
194+struct branch_stack {
195+ u64 nr;
196+ struct branch_entry entries[0];
197+};
198+
182199 extern bool perf_host, perf_guest;
183200 extern const char perf_version_string[];
184201
@@ -205,6 +222,7 @@ struct perf_record_opts { · 20 unmodified lines
205222 unsigned int freq;
206223 unsigned int mmap_pages;
207224 unsigned int user_freq;
225+ int branch_stack;
208226 u64 default_interval;
209227 u64 user_interval;
210228 const char *cpu_list;
tools/perf/util/event.h+1−0
@@ -81,6 +81,7 @@ struct perf_sample { · 80 unmodified lines
8181 u32 raw_size;
8282 void *raw_data;
8383 struct ip_callchain *callchain;
84+ struct branch_stack *branch_stack;
8485 };
8586
8687 #define BUILD_ID_SIZE 20
tools/perf/util/evsel.c+14−0
@@ -126,6 +126,10 @@ void perf_evsel__config(struct perf_evsel *evsel, struct perf_record_opts *opts) · 125 unmodified lines
126126 attr->watermark = 0;
127127 attr->wakeup_events = 1;
128128 }
129+ if (opts->branch_stack) {
130+ attr->sample_type |= PERF_SAMPLE_BRANCH_STACK;
131+ attr->branch_sample_type = opts->branch_stack;
132+ }
129133
130134 attr->mmap = track;
131135 attr->comm = track;
@@ -576,6 +580,16 @@ int perf_event__parse_sample(const union perf_event *event, u64 type, · 444 unmodified lines
576580 data->raw_data = (void *) pdata;
577581 }
578582
583+ if (type & PERF_SAMPLE_BRANCH_STACK) {
584+ u64 sz;
585+
586+ data->branch_stack = (struct branch_stack *)array;
587+ array++; /* nr */
588+
589+ sz = data->branch_stack->nr * sizeof(struct branch_entry);
590+ sz /= sizeof(u64);
591+ array += sz;
592+ }
579593 return 0;
580594 }
581595
tools/perf/util/header.c+164−43
@@ -1023,6 +1023,12 @@ write_it: · 1022 unmodified lines
10231023 return do_write_string(fd, buffer);
10241024 }
10251025
1026+static int write_branch_stack(int fd __used, struct perf_header *h __used,
1027+ struct perf_evlist *evlist __used)
1028+{
1029+ return 0;
1030+}
1031+
10261032 static void print_hostname(struct perf_header *ph, int fd, FILE *fp)
10271033 {
10281034 char *str = do_read_string(fd, ph);
@@ -1144,8 +1150,9 @@ static void print_event_desc(struct perf_header *ph, int fd, FILE *fp) · 115 unmodified lines
11441150 uint64_t id;
11451151 void *buf = NULL;
11461152 char *str;
1147− u32 nre, sz, nr, i, j, msz;
1148− int ret;
1153+ u32 nre, sz, nr, i, j;
1154+ ssize_t ret;
1155+ size_t msz;
11491156
11501157 /* number of events */
11511158 ret = read(fd, &nre, sizeof(nre));
@@ -1162,25 +1169,23 @@ static void print_event_desc(struct perf_header *ph, int fd, FILE *fp) · 10 unmodified lines
11621169 if (ph->needs_swap)
11631170 sz = bswap_32(sz);
11641171
1165− /*
1166− * ensure it is at least to our ABI rev
1167− */
1168− if (sz < (u32)sizeof(attr))
1169− goto error;
1170−
11711172 memset(&attr, 0, sizeof(attr));
11721173
1173− /* read entire region to sync up to next field */
1174+ /* buffer to hold on file attr struct */
11741175 buf = malloc(sz);
11751176 if (!buf)
11761177 goto error;
11771178
11781179 msz = sizeof(attr);
1179− if (sz < msz)
1180+ if (sz < (ssize_t)msz)
11801181 msz = sz;
11811182
11821183 for (i = 0 ; i < nre; i++) {
11831184
1185+ /*
1186+ * must read entire on-file attr struct to
1187+ * sync up with layout.
1188+ */
11841189 ret = read(fd, buf, sz);
11851190 if (ret != (ssize_t)sz)
11861191 goto error;
@@ -1316,6 +1321,12 @@ static void print_cpuid(struct perf_header *ph, int fd, FILE *fp) · 129 unmodified lines
13161321 free(str);
13171322 }
13181323
1324+static void print_branch_stack(struct perf_header *ph __used, int fd __used,
1325+ FILE *fp)
1326+{
1327+ fprintf(fp, "# contains samples with branch stack\n");
1328+}
1329+
13191330 static int __event_process_build_id(struct build_id_event *bev,
13201331 char *filename,
13211332 struct perf_session *session)
@@ -1520,6 +1531,7 @@ static const struct feature_ops feat_ops[HEADER_LAST_FEATURE] = { · 198 unmodified lines
15201531 FEAT_OPA(HEADER_CMDLINE, cmdline),
15211532 FEAT_OPF(HEADER_CPU_TOPOLOGY, cpu_topology),
15221533 FEAT_OPF(HEADER_NUMA_TOPOLOGY, numa_topology),
1534+ FEAT_OPA(HEADER_BRANCH_STACK, branch_stack),
15231535 };
15241536
15251537 struct header_print_data {
@@ -1804,35 +1816,101 @@ out_free: · 278 unmodified lines
18041816 return err;
18051817 }
18061818
1807−static int check_magic_endian(u64 *magic, struct perf_file_header *header,
1808− struct perf_header *ph)
1819+static const int attr_file_abi_sizes[] = {
1820+ [0] = PERF_ATTR_SIZE_VER0,
1821+ [1] = PERF_ATTR_SIZE_VER1,
1822+ 0,
1823+};
1824+
1825+/*
1826+ * In the legacy file format, the magic number is not used to encode endianness.
1827+ * hdr_sz was used to encode endianness. But given that hdr_sz can vary based
1828+ * on ABI revisions, we need to try all combinations for all endianness to
1829+ * detect the endianness.
1830+ */
1831+static int try_all_file_abis(uint64_t hdr_sz, struct perf_header *ph)
18091832 {
1810− int ret;
1833+ uint64_t ref_size, attr_size;
1834+ int i;
18111835
1812− /* check for legacy format */
1813− ret = memcmp(magic, __perf_magic1, sizeof(*magic));
1814− if (ret == 0) {
1815− pr_debug("legacy perf.data format\n");
1816− if (!header)
1817− return -1;
1836+ for (i = 0 ; attr_file_abi_sizes[i]; i++) {
1837+ ref_size = attr_file_abi_sizes[i]
1838+ + sizeof(struct perf_file_section);
1839+ if (hdr_sz != ref_size) {
1840+ attr_size = bswap_64(hdr_sz);
1841+ if (attr_size != ref_size)
1842+ continue;
18181843
1819− if (header->attr_size != sizeof(struct perf_file_attr)) {
1820− u64 attr_size = bswap_64(header->attr_size);
1844+ ph->needs_swap = true;
1845+ }
1846+ pr_debug("ABI%d perf.data file detected, need_swap=%d\n",
1847+ i,
1848+ ph->needs_swap);
1849+ return 0;
1850+ }
1851+ /* could not determine endianness */
1852+ return -1;
1853+}
18211854
1822− if (attr_size != sizeof(struct perf_file_attr))
1823− return -1;
1855+#define PERF_PIPE_HDR_VER0 16
1856+
1857+static const size_t attr_pipe_abi_sizes[] = {
1858+ [0] = PERF_PIPE_HDR_VER0,
1859+ 0,
1860+};
1861+
1862+/*
1863+ * In the legacy pipe format, there is an implicit assumption that endiannesss
1864+ * between host recording the samples, and host parsing the samples is the
1865+ * same. This is not always the case given that the pipe output may always be
1866+ * redirected into a file and analyzed on a different machine with possibly a
1867+ * different endianness and perf_event ABI revsions in the perf tool itself.
1868+ */
1869+static int try_all_pipe_abis(uint64_t hdr_sz, struct perf_header *ph)
1870+{
1871+ u64 attr_size;
1872+ int i;
1873+
1874+ for (i = 0 ; attr_pipe_abi_sizes[i]; i++) {
1875+ if (hdr_sz != attr_pipe_abi_sizes[i]) {
1876+ attr_size = bswap_64(hdr_sz);
1877+ if (attr_size != hdr_sz)
1878+ continue;
18241879
18251880 ph->needs_swap = true;
18261881 }
1882+ pr_debug("Pipe ABI%d perf.data file detected\n", i);
18271883 return 0;
18281884 }
1885+ return -1;
1886+}
1887+
1888+static int check_magic_endian(u64 magic, uint64_t hdr_sz,
1889+ bool is_pipe, struct perf_header *ph)
1890+{
1891+ int ret;
1892+
1893+ /* check for legacy format */
1894+ ret = memcmp(&magic, __perf_magic1, sizeof(magic));
1895+ if (ret == 0) {
1896+ pr_debug("legacy perf.data format\n");
1897+ if (is_pipe)
1898+ return try_all_pipe_abis(hdr_sz, ph);
1899+
1900+ return try_all_file_abis(hdr_sz, ph);
1901+ }
1902+ /*
1903+ * the new magic number serves two purposes:
1904+ * - unique number to identify actual perf.data files
1905+ * - encode endianness of file
1906+ */
18291907
1830− /* check magic number with same endianness */
1831− if (*magic == __perf_magic2)
1908+ /* check magic number with one endianness */
1909+ if (magic == __perf_magic2)
18321910 return 0;
18331911
1834− /* check magic number but opposite endianness */
1835− if (*magic != __perf_magic2_sw)
1912+ /* check magic number with opposite endianness */
1913+ if (magic != __perf_magic2_sw)
18361914 return -1;
18371915
18381916 ph->needs_swap = true;
@@ -1851,8 +1929,11 @@ int perf_file_header__read(struct perf_file_header *header, · 12 unmodified lines
18511929 if (ret <= 0)
18521930 return -1;
18531931
1854− if (check_magic_endian(&header->magic, header, ph) < 0)
1932+ if (check_magic_endian(header->magic,
1933+ header->attr_size, false, ph) < 0) {
1934+ pr_debug("magic/endian check failed\n");
18551935 return -1;
1936+ }
18561937
18571938 if (ph->needs_swap) {
18581939 mem_bswap_64(header, offsetof(struct perf_file_header,
@@ -1939,21 +2020,17 @@ static int perf_file_header__read_pipe(struct perf_pipe_file_header *header, · 80 unmodified lines
19392020 if (ret <= 0)
19402021 return -1;
19412022
1942− if (check_magic_endian(&header->magic, NULL, ph) < 0)
2023+ if (check_magic_endian(header->magic, header->size, true, ph) < 0) {
2024+ pr_debug("endian/magic failed\n");
19432025 return -1;
2026+ }
2027+
2028+ if (ph->needs_swap)
2029+ header->size = bswap_64(header->size);
19442030
19452031 if (repipe && do_write(STDOUT_FILENO, header, sizeof(*header)) < 0)
19462032 return -1;
19472033
1948− if (header->size != sizeof(*header)) {
1949− u64 size = bswap_64(header->size);
1950−
1951− if (size != sizeof(*header))
1952− return -1;
1953−
1954− ph->needs_swap = true;
1955− }
1956−
19572034 return 0;
19582035 }
19592036
@@ -1973,6 +2050,52 @@ static int perf_header__read_pipe(struct perf_session *session, int fd) · 13 unmodified lines
19732050 return 0;
19742051 }
19752052
2053+static int read_attr(int fd, struct perf_header *ph,
2054+ struct perf_file_attr *f_attr)
2055+{
2056+ struct perf_event_attr *attr = &f_attr->attr;
2057+ size_t sz, left;
2058+ size_t our_sz = sizeof(f_attr->attr);
2059+ int ret;
2060+
2061+ memset(f_attr, 0, sizeof(*f_attr));
2062+
2063+ /* read minimal guaranteed structure */
2064+ ret = readn(fd, attr, PERF_ATTR_SIZE_VER0);
2065+ if (ret <= 0) {
2066+ pr_debug("cannot read %d bytes of header attr\n",
2067+ PERF_ATTR_SIZE_VER0);
2068+ return -1;
2069+ }
2070+
2071+ /* on file perf_event_attr size */
2072+ sz = attr->size;
2073+
2074+ if (ph->needs_swap)
2075+ sz = bswap_32(sz);
2076+
2077+ if (sz == 0) {
2078+ /* assume ABI0 */
2079+ sz = PERF_ATTR_SIZE_VER0;
2080+ } else if (sz > our_sz) {
2081+ pr_debug("file uses a more recent and unsupported ABI"
2082+ " (%zu bytes extra)\n", sz - our_sz);
2083+ return -1;
2084+ }
2085+ /* what we have not yet read and that we know about */
2086+ left = sz - PERF_ATTR_SIZE_VER0;
2087+ if (left) {
2088+ void *ptr = attr;
2089+ ptr += PERF_ATTR_SIZE_VER0;
2090+
2091+ ret = readn(fd, ptr, left);
2092+ }
2093+ /* read perf_file_section, ids are read in caller */
2094+ ret = readn(fd, &f_attr->ids, sizeof(f_attr->ids));
2095+
2096+ return ret <= 0 ? -1 : 0;
2097+}
2098+
19762099 int perf_session__read_header(struct perf_session *session, int fd)
19772100 {
19782101 struct perf_header *header = &session->header;
@@ -1988,19 +2111,17 @@ int perf_session__read_header(struct perf_session *session, int fd) · 9 unmodified lines
19882111 if (session->fd_pipe)
19892112 return perf_header__read_pipe(session, fd);
19902113
1991− if (perf_file_header__read(&f_header, header, fd) < 0) {
1992− pr_debug("incompatible file format\n");
2114+ if (perf_file_header__read(&f_header, header, fd) < 0)
19932115 return -EINVAL;
1994− }
19952116
1996− nr_attrs = f_header.attrs.size / sizeof(f_attr);
2117+ nr_attrs = f_header.attrs.size / f_header.attr_size;
19972118 lseek(fd, f_header.attrs.offset, SEEK_SET);
19982119
19992120 for (i = 0; i < nr_attrs; i++) {
20002121 struct perf_evsel *evsel;
20012122 off_t tmp;
20022123
2003− if (readn(fd, &f_attr, sizeof(f_attr)) <= 0)
2124+ if (read_attr(fd, header, &f_attr) < 0)
20042125 goto out_errno;
20052126
20062127 if (header->needs_swap)
tools/perf/util/header.h+1−1
@@ -27,7 +27,7 @@ enum { · 26 unmodified lines
2727 HEADER_EVENT_DESC,
2828 HEADER_CPU_TOPOLOGY,
2929 HEADER_NUMA_TOPOLOGY,
30−
30+ HEADER_BRANCH_STACK,
3131 HEADER_LAST_FEATURE,
3232 HEADER_FEAT_BITS = 256,
3333 };
tools/perf/util/hist.c+95−27
@@ -50,21 +50,25 @@ static void hists__reset_col_len(struct hists *hists) · 49 unmodified lines
5050 hists__set_col_len(hists, col, 0);
5151 }
5252
53+static void hists__set_unres_dso_col_len(struct hists *hists, int dso)
54+{
55+ const unsigned int unresolved_col_width = BITS_PER_LONG / 4;
56+
57+ if (hists__col_len(hists, dso) < unresolved_col_width &&
58+ !symbol_conf.col_width_list_str && !symbol_conf.field_sep &&
59+ !symbol_conf.dso_list)
60+ hists__set_col_len(hists, dso, unresolved_col_width);
61+}
62+
5363 static void hists__calc_col_len(struct hists *hists, struct hist_entry *h)
5464 {
65+ const unsigned int unresolved_col_width = BITS_PER_LONG / 4;
5566 u16 len;
5667
5768 if (h->ms.sym)
58− hists__new_col_len(hists, HISTC_SYMBOL, h->ms.sym->namelen);
59− else {
60− const unsigned int unresolved_col_width = BITS_PER_LONG / 4;
61−
62− if (hists__col_len(hists, HISTC_DSO) < unresolved_col_width &&
63− !symbol_conf.col_width_list_str && !symbol_conf.field_sep &&
64− !symbol_conf.dso_list)
65− hists__set_col_len(hists, HISTC_DSO,
66− unresolved_col_width);
67− }
69+ hists__new_col_len(hists, HISTC_SYMBOL, h->ms.sym->namelen + 4);
70+ else
71+ hists__set_unres_dso_col_len(hists, HISTC_DSO);
6872
6973 len = thread__comm_len(h->thread);
7074 if (hists__new_col_len(hists, HISTC_COMM, len))
@@ -74,6 +78,37 @@ static void hists__calc_col_len(struct hists *hists, struct hist_entry *h) · 3 unmodified lines
7478 len = dso__name_len(h->ms.map->dso);
7579 hists__new_col_len(hists, HISTC_DSO, len);
7680 }
81+
82+ if (h->branch_info) {
83+ int symlen;
84+ /*
85+ * +4 accounts for '[x] ' priv level info
86+ * +2 account of 0x prefix on raw addresses
87+ */
88+ if (h->branch_info->from.sym) {
89+ symlen = (int)h->branch_info->from.sym->namelen + 4;
90+ hists__new_col_len(hists, HISTC_SYMBOL_FROM, symlen);
91+
92+ symlen = dso__name_len(h->branch_info->from.map->dso);
93+ hists__new_col_len(hists, HISTC_DSO_FROM, symlen);
94+ } else {
95+ symlen = unresolved_col_width + 4 + 2;
96+ hists__new_col_len(hists, HISTC_SYMBOL_FROM, symlen);
97+ hists__set_unres_dso_col_len(hists, HISTC_DSO_FROM);
98+ }
99+
100+ if (h->branch_info->to.sym) {
101+ symlen = (int)h->branch_info->to.sym->namelen + 4;
102+ hists__new_col_len(hists, HISTC_SYMBOL_TO, symlen);
103+
104+ symlen = dso__name_len(h->branch_info->to.map->dso);
105+ hists__new_col_len(hists, HISTC_DSO_TO, symlen);
106+ } else {
107+ symlen = unresolved_col_width + 4 + 2;
108+ hists__new_col_len(hists, HISTC_SYMBOL_TO, symlen);
109+ hists__set_unres_dso_col_len(hists, HISTC_DSO_TO);
110+ }
111+ }
77112 }
78113
79114 static void hist_entry__add_cpumode_period(struct hist_entry *he,
@@ -195,26 +230,14 @@ static u8 symbol__parent_filter(const struct symbol *parent) · 115 unmodified lines
195230 return 0;
196231 }
197232
198−struct hist_entry *__hists__add_entry(struct hists *hists,
233+static struct hist_entry *add_hist_entry(struct hists *hists,
234+ struct hist_entry *entry,
199235 struct addr_location *al,
200− struct symbol *sym_parent, u64 period)
236+ u64 period)
201237 {
202238 struct rb_node **p;
203239 struct rb_node *parent = NULL;
204240 struct hist_entry *he;
205− struct hist_entry entry = {
206− .thread = al->thread,
207− .ms = {
208− .map = al->map,
209− .sym = al->sym,
210− },
211− .cpu = al->cpu,
212− .ip = al->addr,
213− .level = al->level,
214− .period = period,
215− .parent = sym_parent,
216− .filtered = symbol__parent_filter(sym_parent),
217− };
218241 int cmp;
219242
220243 pthread_mutex_lock(&hists->lock);
@@ -225,7 +248,7 @@ struct hist_entry *__hists__add_entry(struct hists *hists, · 4 unmodified lines
225248 parent = *p;
226249 he = rb_entry(parent, struct hist_entry, rb_node_in);
227250
228− cmp = hist_entry__cmp(&entry, he);
251+ cmp = hist_entry__cmp(entry, he);
229252
230253 if (!cmp) {
231254 he->period += period;
@@ -239,7 +262,7 @@ struct hist_entry *__hists__add_entry(struct hists *hists, · 7 unmodified lines
239262 p = &(*p)->rb_right;
240263 }
241264
242− he = hist_entry__new(&entry);
265+ he = hist_entry__new(entry);
243266 if (!he)
244267 goto out_unlock;
245268
@@ -252,6 +275,51 @@ out_unlock: · 6 unmodified lines
252275 return he;
253276 }
254277
278+struct hist_entry *__hists__add_branch_entry(struct hists *self,
279+ struct addr_location *al,
280+ struct symbol *sym_parent,
281+ struct branch_info *bi,
282+ u64 period)
283+{
284+ struct hist_entry entry = {
285+ .thread = al->thread,
286+ .ms = {
287+ .map = bi->to.map,
288+ .sym = bi->to.sym,
289+ },
290+ .cpu = al->cpu,
291+ .ip = bi->to.addr,
292+ .level = al->level,
293+ .period = period,
294+ .parent = sym_parent,
295+ .filtered = symbol__parent_filter(sym_parent),
296+ .branch_info = bi,
297+ };
298+
299+ return add_hist_entry(self, &entry, al, period);
300+}
301+
302+struct hist_entry *__hists__add_entry(struct hists *self,
303+ struct addr_location *al,
304+ struct symbol *sym_parent, u64 period)
305+{
306+ struct hist_entry entry = {
307+ .thread = al->thread,
308+ .ms = {
309+ .map = al->map,
310+ .sym = al->sym,
311+ },
312+ .cpu = al->cpu,
313+ .ip = al->addr,
314+ .level = al->level,
315+ .period = period,
316+ .parent = sym_parent,
317+ .filtered = symbol__parent_filter(sym_parent),
318+ };
319+
320+ return add_hist_entry(self, &entry, al, period);
321+}
322+
255323 int64_t
256324 hist_entry__cmp(struct hist_entry *left, struct hist_entry *right)
257325 {
tools/perf/util/hist.h+11−0
@@ -42,6 +42,11 @@ enum hist_column { · 41 unmodified lines
4242 HISTC_COMM,
4343 HISTC_PARENT,
4444 HISTC_CPU,
45+ HISTC_MISPREDICT,
46+ HISTC_SYMBOL_FROM,
47+ HISTC_SYMBOL_TO,
48+ HISTC_DSO_FROM,
49+ HISTC_DSO_TO,
4550 HISTC_NR_COLS, /* Last entry */
4651 };
4752
@@ -74,6 +79,12 @@ int hist_entry__snprintf(struct hist_entry *self, char *bf, size_t size, · 26 unmodified lines
7479 struct hists *hists);
7580 void hist_entry__free(struct hist_entry *);
7681
82+struct hist_entry *__hists__add_branch_entry(struct hists *self,
83+ struct addr_location *al,
84+ struct symbol *sym_parent,
85+ struct branch_info *bi,
86+ u64 period);
87+
7788 void hists__output_resort(struct hists *self);
7889 void hists__output_resort_threaded(struct hists *hists);
7990 void hists__collapse_resort(struct hists *self);
tools/perf/util/session.c+75−2
@@ -24,7 +24,7 @@ static int perf_session__open(struct perf_session *self, bool force) · 23 unmodified lines
2424 self->fd = STDIN_FILENO;
2525
2626 if (perf_session__read_header(self, self->fd) < 0)
27− pr_err("incompatible file format");
27+ pr_err("incompatible file format (rerun with -v to learn more)");
2828
2929 return 0;
3030 }
@@ -56,7 +56,7 @@ static int perf_session__open(struct perf_session *self, bool force) · 25 unmodified lines
5656 }
5757
5858 if (perf_session__read_header(self, self->fd) < 0) {
59− pr_err("incompatible file format");
59+ pr_err("incompatible file format (rerun with -v to learn more)");
6060 goto out_close;
6161 }
6262
@@ -229,6 +229,64 @@ static bool symbol__match_parent_regex(struct symbol *sym) · 166 unmodified lines
229229 return 0;
230230 }
231231
232+static const u8 cpumodes[] = {
233+ PERF_RECORD_MISC_USER,
234+ PERF_RECORD_MISC_KERNEL,
235+ PERF_RECORD_MISC_GUEST_USER,
236+ PERF_RECORD_MISC_GUEST_KERNEL
237+};
238+#define NCPUMODES (sizeof(cpumodes)/sizeof(u8))
239+
240+static void ip__resolve_ams(struct machine *self, struct thread *thread,
241+ struct addr_map_symbol *ams,
242+ u64 ip)
243+{
244+ struct addr_location al;
245+ size_t i;
246+ u8 m;
247+
248+ memset(&al, 0, sizeof(al));
249+
250+ for (i = 0; i < NCPUMODES; i++) {
251+ m = cpumodes[i];
252+ /*
253+ * We cannot use the header.misc hint to determine whether a
254+ * branch stack address is user, kernel, guest, hypervisor.
255+ * Branches may straddle the kernel/user/hypervisor boundaries.
256+ * Thus, we have to try consecutively until we find a match
257+ * or else, the symbol is unknown
258+ */
259+ thread__find_addr_location(thread, self, m, MAP__FUNCTION,
260+ ip, &al, NULL);
261+ if (al.sym)
262+ goto found;
263+ }
264+found:
265+ ams->addr = ip;
266+ ams->al_addr = al.addr;
267+ ams->sym = al.sym;
268+ ams->map = al.map;
269+}
270+
271+struct branch_info *machine__resolve_bstack(struct machine *self,
272+ struct thread *thr,
273+ struct branch_stack *bs)
274+{
275+ struct branch_info *bi;
276+ unsigned int i;
277+
278+ bi = calloc(bs->nr, sizeof(struct branch_info));
279+ if (!bi)
280+ return NULL;
281+
282+ for (i = 0; i < bs->nr; i++) {
283+ ip__resolve_ams(self, thr, &bi[i].to, bs->entries[i].to);
284+ ip__resolve_ams(self, thr, &bi[i].from, bs->entries[i].from);
285+ bi[i].flags = bs->entries[i].flags;
286+ }
287+ return bi;
288+}
289+
232290 int machine__resolve_callchain(struct machine *self, struct perf_evsel *evsel,
233291 struct thread *thread,
234292 struct ip_callchain *chain,
@@ -697,6 +755,18 @@ static void callchain__printf(struct perf_sample *sample) · 462 unmodified lines
697755 i, sample->callchain->ips[i]);
698756 }
699757
758+static void branch_stack__printf(struct perf_sample *sample)
759+{
760+ uint64_t i;
761+
762+ printf("... branch stack: nr:%" PRIu64 "\n", sample->branch_stack->nr);
763+
764+ for (i = 0; i < sample->branch_stack->nr; i++)
765+ printf("..... %2"PRIu64": %016" PRIx64 " -> %016" PRIx64 "\n",
766+ i, sample->branch_stack->entries[i].from,
767+ sample->branch_stack->entries[i].to);
768+}
769+
700770 static void perf_session__print_tstamp(struct perf_session *session,
701771 union perf_event *event,
702772 struct perf_sample *sample)
@@ -744,6 +814,9 @@ static void dump_sample(struct perf_session *session, union perf_event *event, · 41 unmodified lines
744814
745815 if (session->sample_type & PERF_SAMPLE_CALLCHAIN)
746816 callchain__printf(sample);
817+
818+ if (session->sample_type & PERF_SAMPLE_BRANCH_STACK)
819+ branch_stack__printf(sample);
747820 }
748821
749822 static struct machine *
tools/perf/util/session.h+4−0
@@ -73,6 +73,10 @@ int perf_session__resolve_callchain(struct perf_session *self, struct perf_evsel · 72 unmodified lines
7373 struct ip_callchain *chain,
7474 struct symbol **parent);
7575
76+struct branch_info *machine__resolve_bstack(struct machine *self,
77+ struct thread *thread,
78+ struct branch_stack *bs);
79+
7680 bool perf_session__has_traces(struct perf_session *self, const char *msg);
7781
7882 void mem_bswap_64(void *src, int byte_size);
tools/perf/util/sort.c+236−51
@@ -8,6 +8,7 @@ const char default_sort_order[] = "comm,dso,symbol"; · 7 unmodified lines
88 const char *sort_order = default_sort_order;
99 int sort__need_collapse = 0;
1010 int sort__has_parent = 0;
11+int sort__branch_mode = -1; /* -1 = means not set */
1112
1213 enum sort_type sort__first_dimension;
1314
@@ -94,6 +95,26 @@ static int hist_entry__comm_snprintf(struct hist_entry *self, char *bf, · 80 unmodified lines
9495 return repsep_snprintf(bf, size, "%*s", width, self->thread->comm);
9596 }
9697
98+static int64_t _sort__dso_cmp(struct map *map_l, struct map *map_r)
99+{
100+ struct dso *dso_l = map_l ? map_l->dso : NULL;
101+ struct dso *dso_r = map_r ? map_r->dso : NULL;
102+ const char *dso_name_l, *dso_name_r;
103+
104+ if (!dso_l || !dso_r)
105+ return cmp_null(dso_l, dso_r);
106+
107+ if (verbose) {
108+ dso_name_l = dso_l->long_name;
109+ dso_name_r = dso_r->long_name;
110+ } else {
111+ dso_name_l = dso_l->short_name;
112+ dso_name_r = dso_r->short_name;
113+ }
114+
115+ return strcmp(dso_name_l, dso_name_r);
116+}
117+
97118 struct sort_entry sort_comm = {
98119 .se_header = "Command",
99120 .se_cmp = sort__comm_cmp,
@@ -107,36 +128,74 @@ struct sort_entry sort_comm = { · 7 unmodified lines
107128 static int64_t
108129 sort__dso_cmp(struct hist_entry *left, struct hist_entry *right)
109130 {
110− struct dso *dso_l = left->ms.map ? left->ms.map->dso : NULL;
111− struct dso *dso_r = right->ms.map ? right->ms.map->dso : NULL;
112− const char *dso_name_l, *dso_name_r;
131+ return _sort__dso_cmp(left->ms.map, right->ms.map);
132+}
113133
114− if (!dso_l || !dso_r)
115− return cmp_null(dso_l, dso_r);
116134
117− if (verbose) {
118− dso_name_l = dso_l->long_name;
119− dso_name_r = dso_r->long_name;
120− } else {
121− dso_name_l = dso_l->short_name;
122− dso_name_r = dso_r->short_name;
135+static int64_t _sort__sym_cmp(struct symbol *sym_l, struct symbol *sym_r,
136+ u64 ip_l, u64 ip_r)
137+{
138+ if (!sym_l || !sym_r)
139+ return cmp_null(sym_l, sym_r);
140+
141+ if (sym_l == sym_r)
142+ return 0;
143+
144+ if (sym_l)
145+ ip_l = sym_l->start;
146+ if (sym_r)
147+ ip_r = sym_r->start;
148+
149+ return (int64_t)(ip_r - ip_l);
150+}
151+
152+static int _hist_entry__dso_snprintf(struct map *map, char *bf,
153+ size_t size, unsigned int width)
154+{
155+ if (map && map->dso) {
156+ const char *dso_name = !verbose ? map->dso->short_name :
157+ map->dso->long_name;
158+ return repsep_snprintf(bf, size, "%-*s", width, dso_name);
123159 }
124160
125− return strcmp(dso_name_l, dso_name_r);
161+ return repsep_snprintf(bf, size, "%-*s", width, "[unknown]");
126162 }
127163
128164 static int hist_entry__dso_snprintf(struct hist_entry *self, char *bf,
129165 size_t size, unsigned int width)
130166 {
131− if (self->ms.map && self->ms.map->dso) {
132− const char *dso_name = !verbose ? self->ms.map->dso->short_name :
133− self->ms.map->dso->long_name;
134− return repsep_snprintf(bf, size, "%-*s", width, dso_name);
167+ return _hist_entry__dso_snprintf(self->ms.map, bf, size, width);
168+}
169+
170+static int _hist_entry__sym_snprintf(struct map *map, struct symbol *sym,
171+ u64 ip, char level, char *bf, size_t size,
172+ unsigned int width __used)
173+{
174+ size_t ret = 0;
175+
176+ if (verbose) {
177+ char o = map ? dso__symtab_origin(map->dso) : '!';
178+ ret += repsep_snprintf(bf, size, "%-#*llx %c ",
179+ BITS_PER_LONG / 4, ip, o);
135180 }
136181
137− return repsep_snprintf(bf, size, "%-*s", width, "[unknown]");
182+ ret += repsep_snprintf(bf + ret, size - ret, "[%c] ", level);
183+ if (sym)
184+ ret += repsep_snprintf(bf + ret, size - ret, "%-*s",
185+ width - ret,
186+ sym->name);
187+ else {
188+ size_t len = BITS_PER_LONG / 4;
189+ ret += repsep_snprintf(bf + ret, size - ret, "%-#.*llx",
190+ len, ip);
191+ ret += repsep_snprintf(bf + ret, size - ret, "%-*s",
192+ width - ret, "");
193+ }
194+
195+ return ret;
138196 }
139197
198+
140199 struct sort_entry sort_dso = {
141200 .se_header = "Shared Object",
142201 .se_cmp = sort__dso_cmp,
@@ -144,8 +203,14 @@ struct sort_entry sort_dso = { · 1 unmodified line
144203 .se_width_idx = HISTC_DSO,
145204 };
146205
147−/* --sort symbol */
206+static int hist_entry__sym_snprintf(struct hist_entry *self, char *bf,
207+ size_t size, unsigned int width __used)
208+{
209+ return _hist_entry__sym_snprintf(self->ms.map, self->ms.sym, self->ip,
210+ self->level, bf, size, width);
211+}
148212
213+/* --sort symbol */
149214 static int64_t
150215 sort__sym_cmp(struct hist_entry *left, struct hist_entry *right)
151216 {
@@ -163,31 +228,7 @@ sort__sym_cmp(struct hist_entry *left, struct hist_entry *right) · 11 unmodified lines
163228 ip_l = left->ms.sym->start;
164229 ip_r = right->ms.sym->start;
165230
166− return (int64_t)(ip_r - ip_l);
167−}
168−
169−static int hist_entry__sym_snprintf(struct hist_entry *self, char *bf,
170− size_t size, unsigned int width __used)
171−{
172− size_t ret = 0;
173−
174− if (verbose) {
175− char o = self->ms.map ? dso__symtab_origin(self->ms.map->dso) : '!';
176− ret += repsep_snprintf(bf, size, "%-#*llx %c ",
177− BITS_PER_LONG / 4, self->ip, o);
178− }
179−
180− if (!sort_dso.elide)
181− ret += repsep_snprintf(bf + ret, size - ret, "[%c] ", self->level);
182−
183− if (self->ms.sym)
184− ret += repsep_snprintf(bf + ret, size - ret, "%s",
185− self->ms.sym->name);
186− else
187− ret += repsep_snprintf(bf + ret, size - ret, "%-#*llx",
188− BITS_PER_LONG / 4, self->ip);
189−
190− return ret;
231+ return _sort__sym_cmp(left->ms.sym, right->ms.sym, ip_l, ip_r);
191232 }
192233
193234 struct sort_entry sort_sym = {
@@ -246,19 +287,155 @@ struct sort_entry sort_cpu = { · 52 unmodified lines
246287 .se_width_idx = HISTC_CPU,
247288 };
248289
290+static int64_t
291+sort__dso_from_cmp(struct hist_entry *left, struct hist_entry *right)
292+{
293+ return _sort__dso_cmp(left->branch_info->from.map,
294+ right->branch_info->from.map);
295+}
296+
297+static int hist_entry__dso_from_snprintf(struct hist_entry *self, char *bf,
298+ size_t size, unsigned int width)
299+{
300+ return _hist_entry__dso_snprintf(self->branch_info->from.map,
301+ bf, size, width);
302+}
303+
304+struct sort_entry sort_dso_from = {
305+ .se_header = "Source Shared Object",
306+ .se_cmp = sort__dso_from_cmp,
307+ .se_snprintf = hist_entry__dso_from_snprintf,
308+ .se_width_idx = HISTC_DSO_FROM,
309+};
310+
311+static int64_t
312+sort__dso_to_cmp(struct hist_entry *left, struct hist_entry *right)
313+{
314+ return _sort__dso_cmp(left->branch_info->to.map,
315+ right->branch_info->to.map);
316+}
317+
318+static int hist_entry__dso_to_snprintf(struct hist_entry *self, char *bf,
319+ size_t size, unsigned int width)
320+{
321+ return _hist_entry__dso_snprintf(self->branch_info->to.map,
322+ bf, size, width);
323+}
324+
325+static int64_t
326+sort__sym_from_cmp(struct hist_entry *left, struct hist_entry *right)
327+{
328+ struct addr_map_symbol *from_l = &left->branch_info->from;
329+ struct addr_map_symbol *from_r = &right->branch_info->from;
330+
331+ if (!from_l->sym && !from_r->sym)
332+ return right->level - left->level;
333+
334+ return _sort__sym_cmp(from_l->sym, from_r->sym, from_l->addr,
335+ from_r->addr);
336+}
337+
338+static int64_t
339+sort__sym_to_cmp(struct hist_entry *left, struct hist_entry *right)
340+{
341+ struct addr_map_symbol *to_l = &left->branch_info->to;
342+ struct addr_map_symbol *to_r = &right->branch_info->to;
343+
344+ if (!to_l->sym && !to_r->sym)
345+ return right->level - left->level;
346+
347+ return _sort__sym_cmp(to_l->sym, to_r->sym, to_l->addr, to_r->addr);
348+}
349+
350+static int hist_entry__sym_from_snprintf(struct hist_entry *self, char *bf,
351+ size_t size, unsigned int width __used)
352+{
353+ struct addr_map_symbol *from = &self->branch_info->from;
354+ return _hist_entry__sym_snprintf(from->map, from->sym, from->addr,
355+ self->level, bf, size, width);
356+
357+}
358+
359+static int hist_entry__sym_to_snprintf(struct hist_entry *self, char *bf,
360+ size_t size, unsigned int width __used)
361+{
362+ struct addr_map_symbol *to = &self->branch_info->to;
363+ return _hist_entry__sym_snprintf(to->map, to->sym, to->addr,
364+ self->level, bf, size, width);
365+
366+}
367+
368+struct sort_entry sort_dso_to = {
369+ .se_header = "Target Shared Object",
370+ .se_cmp = sort__dso_to_cmp,
371+ .se_snprintf = hist_entry__dso_to_snprintf,
372+ .se_width_idx = HISTC_DSO_TO,
373+};
374+
375+struct sort_entry sort_sym_from = {
376+ .se_header = "Source Symbol",
377+ .se_cmp = sort__sym_from_cmp,
378+ .se_snprintf = hist_entry__sym_from_snprintf,
379+ .se_width_idx = HISTC_SYMBOL_FROM,
380+};
381+
382+struct sort_entry sort_sym_to = {
383+ .se_header = "Target Symbol",
384+ .se_cmp = sort__sym_to_cmp,
385+ .se_snprintf = hist_entry__sym_to_snprintf,
386+ .se_width_idx = HISTC_SYMBOL_TO,
387+};
388+
389+static int64_t
390+sort__mispredict_cmp(struct hist_entry *left, struct hist_entry *right)
391+{
392+ const unsigned char mp = left->branch_info->flags.mispred !=
393+ right->branch_info->flags.mispred;
394+ const unsigned char p = left->branch_info->flags.predicted !=
395+ right->branch_info->flags.predicted;
396+
397+ return mp || p;
398+}
399+
400+static int hist_entry__mispredict_snprintf(struct hist_entry *self, char *bf,
401+ size_t size, unsigned int width){
402+ static const char *out = "N/A";
403+
404+ if (self->branch_info->flags.predicted)
405+ out = "N";
406+ else if (self->branch_info->flags.mispred)
407+ out = "Y";
408+
409+ return repsep_snprintf(bf, size, "%-*s", width, out);
410+}
411+
412+struct sort_entry sort_mispredict = {
413+ .se_header = "Branch Mispredicted",
414+ .se_cmp = sort__mispredict_cmp,
415+ .se_snprintf = hist_entry__mispredict_snprintf,
416+ .se_width_idx = HISTC_MISPREDICT,
417+};
418+
249419 struct sort_dimension {
250420 const char *name;
251421 struct sort_entry *entry;
252422 int taken;
253423 };
254424
425+#define DIM(d, n, func) [d] = { .name = n, .entry = &(func) }
426+
255427 static struct sort_dimension sort_dimensions[] = {
256− { .name = "pid", .entry = &sort_thread, },
257− { .name = "comm", .entry = &sort_comm, },
258− { .name = "dso", .entry = &sort_dso, },
259− { .name = "symbol", .entry = &sort_sym, },
260− { .name = "parent", .entry = &sort_parent, },
261− { .name = "cpu", .entry = &sort_cpu, },
428+ DIM(SORT_PID, "pid", sort_thread),
429+ DIM(SORT_COMM, "comm", sort_comm),
430+ DIM(SORT_DSO, "dso", sort_dso),
431+ DIM(SORT_DSO_FROM, "dso_from", sort_dso_from),
432+ DIM(SORT_DSO_TO, "dso_to", sort_dso_to),
433+ DIM(SORT_SYM, "symbol", sort_sym),
434+ DIM(SORT_SYM_TO, "symbol_from", sort_sym_from),
435+ DIM(SORT_SYM_FROM, "symbol_to", sort_sym_to),
436+ DIM(SORT_PARENT, "parent", sort_parent),
437+ DIM(SORT_CPU, "cpu", sort_cpu),
438+ DIM(SORT_MISPREDICT, "mispredict", sort_mispredict),
262439 };
263440
264441 int sort_dimension__add(const char *tok)
@@ -270,7 +447,6 @@ int sort_dimension__add(const char *tok) · 5 unmodified lines
270447
271448 if (strncasecmp(tok, sd->name, strlen(tok)))
272449 continue;
273−
274450 if (sd->entry == &sort_parent) {
275451 int ret = regcomp(&parent_regex, parent_pattern, REG_EXTENDED);
276452 if (ret) {
@@ -302,6 +478,16 @@ int sort_dimension__add(const char *tok) · 25 unmodified lines
302478 sort__first_dimension = SORT_PARENT;
303479 else if (!strcmp(sd->name, "cpu"))
304480 sort__first_dimension = SORT_CPU;
481+ else if (!strcmp(sd->name, "symbol_from"))
482+ sort__first_dimension = SORT_SYM_FROM;
483+ else if (!strcmp(sd->name, "symbol_to"))
484+ sort__first_dimension = SORT_SYM_TO;
485+ else if (!strcmp(sd->name, "dso_from"))
486+ sort__first_dimension = SORT_DSO_FROM;
487+ else if (!strcmp(sd->name, "dso_to"))
488+ sort__first_dimension = SORT_DSO_TO;
489+ else if (!strcmp(sd->name, "mispredict"))
490+ sort__first_dimension = SORT_MISPREDICT;
305491 }
306492
307493 list_add_tail(&sd->entry->list, &hist_entry__sort_list);
@@ -309,7 +495,6 @@ int sort_dimension__add(const char *tok) · 1 unmodified line
309495
310496 return 0;
311497 }
312−
313498 return -ESRCH;
314499 }
315500
tools/perf/util/sort.h+11−0
@@ -31,11 +31,16 @@ extern const char *parent_pattern; · 30 unmodified lines
3131 extern const char default_sort_order[];
3232 extern int sort__need_collapse;
3333 extern int sort__has_parent;
34+extern int sort__branch_mode;
3435 extern char *field_sep;
3536 extern struct sort_entry sort_comm;
3637 extern struct sort_entry sort_dso;
3738 extern struct sort_entry sort_sym;
3839 extern struct sort_entry sort_parent;
40+extern struct sort_entry sort_dso_from;
41+extern struct sort_entry sort_dso_to;
42+extern struct sort_entry sort_sym_from;
43+extern struct sort_entry sort_sym_to;
3944 extern enum sort_type sort__first_dimension;
4045
4146 /**
@@ -72,6 +77,7 @@ struct hist_entry { · 30 unmodified lines
7277 struct hist_entry *pair;
7378 struct rb_root sorted_chain;
7479 };
80+ struct branch_info *branch_info;
7581 struct callchain_root callchain[0];
7682 };
7783
@@ -82,6 +88,11 @@ enum sort_type { · 4 unmodified lines
8288 SORT_SYM,
8389 SORT_PARENT,
8490 SORT_CPU,
91+ SORT_DSO_FROM,
92+ SORT_DSO_TO,
93+ SORT_SYM_FROM,
94+ SORT_SYM_TO,
95+ SORT_MISPREDICT,
8596 };
8697
8798 /*
tools/perf/util/symbol.h+19−1
@@ -5,6 +5,7 @@ · 4 unmodified lines
55 #include <stdbool.h>
66 #include <stdint.h>
77 #include "map.h"
8+#include "../perf.h"
89 #include <linux/list.h>
910 #include <linux/rbtree.h>
1011 #include <stdio.h>
@@ -96,7 +97,11 @@ struct symbol_conf { · 85 unmodified lines
9697 *col_width_list_str;
9798 struct strlist *dso_list,
9899 *comm_list,
99− *sym_list;
100+ *sym_list,
101+ *dso_from_list,
102+ *dso_to_list,
103+ *sym_from_list,
104+ *sym_to_list;
100105 const char *symfs;
101106 };
102107
@@ -120,6 +125,19 @@ struct map_symbol { · 17 unmodified lines
120125 bool has_children;
121126 };
122127
128+struct addr_map_symbol {
129+ struct map *map;
130+ struct symbol *sym;
131+ u64 addr;
132+ u64 al_addr;
133+};
134+
135+struct branch_info {
136+ struct addr_map_symbol from;
137+ struct addr_map_symbol to;
138+ struct branch_flags flags;
139+};
140+
123141 struct addr_location {
124142 struct thread *thread;
125143 struct map *map;
tools/perf/util/ui/browsers/hists.c+83−19
@@ -805,8 +805,11 @@ static struct hist_browser *hist_browser__new(struct hists *hists) · 804 unmodified lines
805805 self->hists = hists;
806806 self->b.refresh = hist_browser__refresh;
807807 self->b.seek = ui_browser__hists_seek;
808− self->b.use_navkeypressed = true,
809− self->has_symbols = sort_sym.list.next != NULL;
808+ self->b.use_navkeypressed = true;
809+ if (sort__branch_mode == 1)
810+ self->has_symbols = sort_sym_from.list.next != NULL;
811+ else
812+ self->has_symbols = sort_sym.list.next != NULL;
810813 }
811814
812815 return self;
@@ -853,6 +856,16 @@ static int hists__browser_title(struct hists *self, char *bf, size_t size, · 40 unmodified lines
853856 return printed;
854857 }
855858
859+static inline void free_popup_options(char **options, int n)
860+{
861+ int i;
862+
863+ for (i = 0; i < n; ++i) {
864+ free(options[i]);
865+ options[i] = NULL;
866+ }
867+}
868+
856869 static int perf_evsel__hists_browse(struct perf_evsel *evsel, int nr_events,
857870 const char *helpline, const char *ev_name,
858871 bool left_exits,
@@ -861,7 +874,10 @@ static int perf_evsel__hists_browse(struct perf_evsel *evsel, int nr_events, · 2 unmodified lines
861874 {
862875 struct hists *self = &evsel->hists;
863876 struct hist_browser *browser = hist_browser__new(self);
877+ struct branch_info *bi;
864878 struct pstack *fstack;
879+ char *options[16];
880+ int nr_options = 0;
865881 int key = -1;
866882
867883 if (browser == NULL)
@@ -873,13 +889,16 @@ static int perf_evsel__hists_browse(struct perf_evsel *evsel, int nr_events, · 5 unmodified lines
873889
874890 ui_helpline__push(helpline);
875891
892+ memset(options, 0, sizeof(options));
893+
876894 while (1) {
877895 const struct thread *thread = NULL;
878896 const struct dso *dso = NULL;
879− char *options[16];
880− int nr_options = 0, choice = 0, i,
897+ int choice = 0,
881898 annotate = -2, zoom_dso = -2, zoom_thread = -2,
882− browse_map = -2;
899+ annotate_f = -2, annotate_t = -2, browse_map = -2;
900+
901+ nr_options = 0;
883902
884903 key = hist_browser__run(browser, ev_name, timer, arg, delay_secs);
885904
@@ -887,7 +906,6 @@ static int perf_evsel__hists_browse(struct perf_evsel *evsel, int nr_events, · 1 unmodified line
887906 thread = hist_browser__selected_thread(browser);
888907 dso = browser->selection->map ? browser->selection->map->dso : NULL;
889908 }
890−
891909 switch (key) {
892910 case K_TAB:
893911 case K_UNTAB:
@@ -902,7 +920,7 @@ static int perf_evsel__hists_browse(struct perf_evsel *evsel, int nr_events, · 8 unmodified lines
902920 if (!browser->has_symbols) {
903921 ui_browser__warning(&browser->b, delay_secs * 2,
904922 "Annotation is only available for symbolic views, "
905− "include \"sym\" in --sort to use it.");
923+ "include \"sym*\" in --sort to use it.");
906924 continue;
907925 }
908926
@@ -972,12 +990,34 @@ static int perf_evsel__hists_browse(struct perf_evsel *evsel, int nr_events, · 63 unmodified lines
972990 if (!browser->has_symbols)
973991 goto add_exit_option;
974992
975− if (browser->selection != NULL &&
976− browser->selection->sym != NULL &&
977− !browser->selection->map->dso->annotate_warned &&
978− asprintf(&options[nr_options], "Annotate %s",
979− browser->selection->sym->name) > 0)
980− annotate = nr_options++;
993+ if (sort__branch_mode == 1) {
994+ bi = browser->he_selection->branch_info;
995+ if (browser->selection != NULL &&
996+ bi &&
997+ bi->from.sym != NULL &&
998+ !bi->from.map->dso->annotate_warned &&
999+ asprintf(&options[nr_options], "Annotate %s",
1000+ bi->from.sym->name) > 0)
1001+ annotate_f = nr_options++;
1002+
1003+ if (browser->selection != NULL &&
1004+ bi &&
1005+ bi->to.sym != NULL &&
1006+ !bi->to.map->dso->annotate_warned &&
1007+ (bi->to.sym != bi->from.sym ||
1008+ bi->to.map->dso != bi->from.map->dso) &&
1009+ asprintf(&options[nr_options], "Annotate %s",
1010+ bi->to.sym->name) > 0)
1011+ annotate_t = nr_options++;
1012+ } else {
1013+
1014+ if (browser->selection != NULL &&
1015+ browser->selection->sym != NULL &&
1016+ !browser->selection->map->dso->annotate_warned &&
1017+ asprintf(&options[nr_options], "Annotate %s",
1018+ browser->selection->sym->name) > 0)
1019+ annotate = nr_options++;
1020+ }
9811021
9821022 if (thread != NULL &&
9831023 asprintf(&options[nr_options], "Zoom %s %s(%d) thread",
@@ -998,25 +1038,39 @@ static int perf_evsel__hists_browse(struct perf_evsel *evsel, int nr_events, · 14 unmodified lines
9981038 browse_map = nr_options++;
9991039 add_exit_option:
10001040 options[nr_options++] = (char *)"Exit";
1001−
1041+retry_popup_menu:
10021042 choice = ui__popup_menu(nr_options, options);
10031043
1004− for (i = 0; i < nr_options - 1; ++i)
1005− free(options[i]);
1006−
10071044 if (choice == nr_options - 1)
10081045 break;
10091046
1010− if (choice == -1)
1047+ if (choice == -1) {
1048+ free_popup_options(options, nr_options - 1);
10111049 continue;
1050+ }
10121051
1013− if (choice == annotate) {
1052+ if (choice == annotate || choice == annotate_t || choice == annotate_f) {
10141053 struct hist_entry *he;
10151054 int err;
10161055 do_annotate:
10171056 he = hist_browser__selected_entry(browser);
10181057 if (he == NULL)
10191058 continue;
1059+
1060+ /*
1061+ * we stash the branch_info symbol + map into the
1062+ * the ms so we don't have to rewrite all the annotation
1063+ * code to use branch_info.
1064+ * in branch mode, the ms struct is not used
1065+ */
1066+ if (choice == annotate_f) {
1067+ he->ms.sym = he->branch_info->from.sym;
1068+ he->ms.map = he->branch_info->from.map;
1069+ } else if (choice == annotate_t) {
1070+ he->ms.sym = he->branch_info->to.sym;
1071+ he->ms.map = he->branch_info->to.map;
1072+ }
1073+
10201074 /*
10211075 * Don't let this be freed, say, by hists__decay_entry.
10221076 */
@@ -1024,9 +1078,18 @@ do_annotate: · 1 unmodified line
10241078 err = hist_entry__tui_annotate(he, evsel->idx,
10251079 timer, arg, delay_secs);
10261080 he->used = false;
1081+ /*
1082+ * offer option to annotate the other branch source or target
1083+ * (if they exists) when returning from annotate
1084+ */
1085+ if ((err == 'q' || err == CTRL('c'))
1086+ && annotate_t != -2 && annotate_f != -2)
1087+ goto retry_popup_menu;
1088+
10271089 ui_browser__update_nr_entries(&browser->b, browser->hists->nr_entries);
10281090 if (err)
10291091 ui_browser__handle_resize(&browser->b);
1092+
10301093 } else if (choice == browse_map)
10311094 map__browse(browser->selection->map);
10321095 else if (choice == zoom_dso) {
@@ -1072,6 +1135,7 @@ out_free_stack: · 39 unmodified lines
10721135 pstack__delete(fstack);
10731136 out:
10741137 hist_browser__delete(browser);
1138+ free_popup_options(options, nr_options - 1);
10751139 return key;
10761140 }
10771141

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t