perf: Add branch stack counters

Currently, the additional information of a branch entry is stored in a u64 space. With more and more information added, the space is running out. For example, the information of occurrences of events will be added for each branch. Two places were suggested to append the counters. https://lore.kernel.org/lkml/20230802215814.GH231007@hirez.programming.kicks-ass.net/ One place is right after the flags of each branch entry. It changes the existing struct perf_branch_entry. The later ARCH specific implementation has to be really careful to consistently pick the right struct. The other place is right after the entire struct perf_branch_stack. The disadvantage is that the pointer of the extra space has to be recorded. The common interface perf_sample_save_brstack() has to be updated. The latter is much straightforward, and should be easily understood and maintained. It is implemented in the patch. Add a new branch sample type, PERF_SAMPLE_BRANCH_COUNTERS, to indicate the event which is recorded in the branch info. The "u64 counters" may store the occurrences of several events. The information regarding the number of events/counters and the width of each counter should be exposed via sysfs as a reference for the perf tool. Define the branch_counter_nr and branch_counter_width ABI here. The support will be implemented later in the Intel-specific patch.

Kan Liang committed 3 years ago

Commit 571d91d1 parent 744940f9 files · +46 −7Browse files

Unified · Split

9 files

Documentation/ABI/testing/sysfs-bus-event_source-devices-caps +6 −0
arch/powerpc/perf/core-book3s.c +1 −1
arch/x86/events/amd/core.c +1 −1
arch/x86/events/core.c +1 −1
arch/x86/events/intel/core.c +1 −1
arch/x86/events/intel/ds.c +2 −2
include/linux/perf_event.h +16 −1
include/uapi/linux/perf_event.h +10 −0
kernel/events/core.c +8 −0

Documentation/ABI/testing/sysfs-bus-event_source-devices-caps+6−0

@@ -16,3 +16,9 @@ Description: · 15 unmodified lines
16	16	Example output in powerpc:
17	17	grep . /sys/bus/event_source/devices/cpu/caps/*
18	18	/sys/bus/event_source/devices/cpu/caps/pmu_name:POWER9
	19	+
	20	+ The "branch_counter_nr" in the supported platform exposes the
	21	+ maximum number of counters which can be shown in the u64 counters
	22	+ of PERF_SAMPLE_BRANCH_COUNTERS, while the "branch_counter_width"
	23	+ exposes the width of each counter. Both of them can be used by
	24	+ the perf tool to parse the logged counters in each branch.

arch/powerpc/perf/core-book3s.c+1−1

@@ -2313,7 +2313,7 @@ static void record_and_restart(struct perf_event *event, unsigned long val, · 2312 unmodified lines
2313	2313	struct cpu_hw_events *cpuhw;
2314	2314	cpuhw = this_cpu_ptr(&cpu_hw_events);
2315	2315	power_pmu_bhrb_read(event, cpuhw);
2316		− perf_sample_save_brstack(&data, event, &cpuhw->bhrb_stack);
	2316	+ perf_sample_save_brstack(&data, event, &cpuhw->bhrb_stack, NULL);
2317	2317	}
2318	2318
2319	2319	if (event->attr.sample_type & PERF_SAMPLE_DATA_SRC &&

arch/x86/events/amd/core.c+1−1

@@ -940,7 +940,7 @@ static int amd_pmu_v2_handle_irq(struct pt_regs *regs) · 939 unmodified lines
940	940	continue;
941	941
942	942	if (has_branch_stack(event))
943		− perf_sample_save_brstack(&data, event, &cpuc->lbr_stack);
	943	+ perf_sample_save_brstack(&data, event, &cpuc->lbr_stack, NULL);
944	944
945	945	if (perf_event_overflow(event, &data, regs))
946	946	x86_pmu_stop(event, 0);

arch/x86/events/core.c+1−1

@@ -1702,7 +1702,7 @@ int x86_pmu_handle_irq(struct pt_regs *regs) · 1701 unmodified lines
1702	1702	perf_sample_data_init(&data, 0, event->hw.last_period);
1703	1703
1704	1704	if (has_branch_stack(event))
1705		− perf_sample_save_brstack(&data, event, &cpuc->lbr_stack);
	1705	+ perf_sample_save_brstack(&data, event, &cpuc->lbr_stack, NULL);
1706	1706
1707	1707	if (perf_event_overflow(event, &data, regs))
1708	1708	x86_pmu_stop(event, 0);

arch/x86/events/intel/core.c+1−1

@@ -3047,7 +3047,7 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) · 3046 unmodified lines
3047	3047	perf_sample_data_init(&data, 0, event->hw.last_period);
3048	3048
3049	3049	if (has_branch_stack(event))
3050		− perf_sample_save_brstack(&data, event, &cpuc->lbr_stack);
	3050	+ perf_sample_save_brstack(&data, event, &cpuc->lbr_stack, NULL);
3051	3051
3052	3052	if (perf_event_overflow(event, &data, regs))
3053	3053	x86_pmu_stop(event, 0);

arch/x86/events/intel/ds.c+2−2

@@ -1755,7 +1755,7 @@ static void setup_pebs_fixed_sample_data(struct perf_event *event, · 1754 unmodified lines
1755	1755	setup_pebs_time(event, data, pebs->tsc);
1756	1756
1757	1757	if (has_branch_stack(event))
1758		− perf_sample_save_brstack(data, event, &cpuc->lbr_stack);
	1758	+ perf_sample_save_brstack(data, event, &cpuc->lbr_stack, NULL);
1759	1759	}
1760	1760
1761	1761	static void adaptive_pebs_save_regs(struct pt_regs *regs,
@@ -1912,7 +1912,7 @@ static void setup_pebs_adaptive_sample_data(struct perf_event *event, · 150 unmodified lines
1912	1912
1913	1913	if (has_branch_stack(event)) {
1914	1914	intel_pmu_store_pebs_lbrs(lbr);
1915		− perf_sample_save_brstack(data, event, &cpuc->lbr_stack);
	1915	+ perf_sample_save_brstack(data, event, &cpuc->lbr_stack, NULL);
1916	1916	}
1917	1917	}
1918	1918

include/linux/perf_event.h+16−1

@@ -1139,6 +1139,10 @@ static inline bool branch_sample_priv(const struct perf_event *event) · 1138 unmodified lines
1139	1139	return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_PRIV_SAVE;
1140	1140	}
1141	1141
	1142	+static inline bool branch_sample_counters(const struct perf_event *event)
	1143	+{
	1144	+ return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_COUNTERS;
	1145	+}
1142	1146
1143	1147	struct perf_sample_data {
1144	1148	/*
@@ -1173,6 +1177,7 @@ struct perf_sample_data { · 28 unmodified lines
1173	1177	struct perf_callchain_entry *callchain;
1174	1178	struct perf_raw_record *raw;
1175	1179	struct perf_branch_stack *br_stack;
	1180	+ u64 *br_stack_cntr;
1176	1181	union perf_sample_weight weight;
1177	1182	union perf_mem_data_src data_src;
1178	1183	u64 txn;
@@ -1250,7 +1255,8 @@ static inline void perf_sample_save_raw_data(struct perf_sample_data *data, · 71 unmodified lines
1250	1255
1251	1256	static inline void perf_sample_save_brstack(struct perf_sample_data *data,
1252	1257	struct perf_event *event,
1253		− struct perf_branch_stack *brs)
	1258	+ struct perf_branch_stack *brs,
	1259	+ u64 *brs_cntr)
1254	1260	{
1255	1261	int size = sizeof(u64); /* nr */
1256	1262
@@ -1258,7 +1264,16 @@ static inline void perf_sample_save_brstack(struct perf_sample_data *data, · 1 unmodified line
1258	1264	size += sizeof(u64);
1259	1265	size += brs->nr * sizeof(struct perf_branch_entry);
1260	1266
	1267	+ /*
	1268	+ * The extension space for counters is appended after the
	1269	+ * struct perf_branch_stack. It is used to store the occurrences
	1270	+ * of events of each branch.
	1271	+ */
	1272	+ if (brs_cntr)
	1273	+ size += brs->nr * sizeof(u64);
	1274	+
1261	1275	data->br_stack = brs;
	1276	+ data->br_stack_cntr = brs_cntr;
1262	1277	data->dyn_size += size;
1263	1278	data->sample_flags \|= PERF_SAMPLE_BRANCH_STACK;
1264	1279	}

include/uapi/linux/perf_event.h+10−0

@@ -204,6 +204,8 @@ enum perf_branch_sample_type_shift { · 203 unmodified lines
204	204
205	205	PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT = 18, /* save privilege mode */
206	206
	207	+ PERF_SAMPLE_BRANCH_COUNTERS_SHIFT = 19, /* save occurrences of events on a branch */
	208	+
207	209	PERF_SAMPLE_BRANCH_MAX_SHIFT /* non-ABI */
208	210	};
209	211
@@ -235,6 +237,8 @@ enum perf_branch_sample_type { · 25 unmodified lines
235	237
236	238	PERF_SAMPLE_BRANCH_PRIV_SAVE = 1U << PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT,
237	239
	240	+ PERF_SAMPLE_BRANCH_COUNTERS = 1U << PERF_SAMPLE_BRANCH_COUNTERS_SHIFT,
	241	+
238	242	PERF_SAMPLE_BRANCH_MAX = 1U << PERF_SAMPLE_BRANCH_MAX_SHIFT,
239	243	};
240	244
@@ -982,6 +986,12 @@ enum perf_event_type { · 741 unmodified lines
982	986	* { u64 nr;
983	987	* { u64 hw_idx; } && PERF_SAMPLE_BRANCH_HW_INDEX
984	988	* { u64 from, to, flags } lbr[nr];
	989	+ * #
	990	+ * # The format of the counters is decided by the
	991	+ * # "branch_counter_nr" and "branch_counter_width",
	992	+ * # which are defined in the ABI.
	993	+ * #
	994	+ * { u64 counters; } cntr[nr] && PERF_SAMPLE_BRANCH_COUNTERS
985	995	* } && PERF_SAMPLE_BRANCH_STACK
986	996	*
987	997	* { u64 abi; # enum perf_sample_regs_abi

kernel/events/core.c+8−0

@@ -7341,6 +7341,14 @@ void perf_output_sample(struct perf_output_handle *handle, · 7340 unmodified lines
7341	7341	if (branch_sample_hw_index(event))
7342	7342	perf_output_put(handle, data->br_stack->hw_idx);
7343	7343	perf_output_copy(handle, data->br_stack->entries, size);
	7344	+ /*
	7345	+ * Add the extension space which is appended
	7346	+ * right after the struct perf_branch_stack.
	7347	+ */
	7348	+ if (data->br_stack_cntr) {
	7349	+ size = data->br_stack->nr * sizeof(u64);
	7350	+ perf_output_copy(handle, data->br_stack_cntr, size);
	7351	+ }
7344	7352	} else {
7345	7353	/*
7346	7354	* we always store at least the value of nr

perf: Add branch stack counters

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code