Currently, the additional information of a branch entry is stored in a u64 space. With more and more information added, the space is running out. For example, the information of occurrences of events will be added for each branch. Two places were suggested to append the counters. https://lore.kernel.org/lkml/20230802215814.GH231007@hirez.programming.kicks-ass.net/ One place is right after the flags of each branch entry. It changes the existing struct perf_branch_entry. The later ARCH specific implementation has to be really careful to consistently pick the right struct. The other place is right after the entire struct perf_branch_stack. The disadvantage is that the pointer of the extra space has to be recorded. The common interface perf_sample_save_brstack() has to be updated. The latter is much straightforward, and should be easily understood and maintained. It is implemented in the patch. Add a new branch sample type, PERF_SAMPLE_BRANCH_COUNTERS, to indicate the event which is recorded in the branch info. The "u64 counters" may store the occurrences of several events. The information regarding the number of events/counters and the width of each counter should be exposed via sysfs as a reference for the perf tool. Define the branch_counter_nr and branch_counter_width ABI here. The support will be implemented later in the Intel-specific patch.
Unified · Split
| @@ -16,3 +16,9 @@ Description: · 15 unmodified lines | |||
| 16 | Example output in powerpc: | 16 | Example output in powerpc: |
| 17 | grep . /sys/bus/event_source/devices/cpu/caps/* | 17 | grep . /sys/bus/event_source/devices/cpu/caps/* |
| 18 | /sys/bus/event_source/devices/cpu/caps/pmu_name:POWER9 | 18 | /sys/bus/event_source/devices/cpu/caps/pmu_name:POWER9 |
| 19 | |||
| 20 | The "branch_counter_nr" in the supported platform exposes the | ||
| 21 | maximum number of counters which can be shown in the u64 counters | ||
| 22 | of PERF_SAMPLE_BRANCH_COUNTERS, while the "branch_counter_width" | ||
| 23 | exposes the width of each counter. Both of them can be used by | ||
| 24 | the perf tool to parse the logged counters in each branch. | ||
| @@ -2313,7 +2313,7 @@ static void record_and_restart(struct perf_event *event, unsigned long val, · 2312 unmodified lines | |||
| 2313 | struct cpu_hw_events *cpuhw; | 2313 | struct cpu_hw_events *cpuhw; |
| 2314 | cpuhw = this_cpu_ptr(&cpu_hw_events); | 2314 | cpuhw = this_cpu_ptr(&cpu_hw_events); |
| 2315 | power_pmu_bhrb_read(event, cpuhw); | 2315 | power_pmu_bhrb_read(event, cpuhw); |
| 2316 | perf_sample_save_brstack(&data, event, &cpuhw->bhrb_stack); | 2316 | perf_sample_save_brstack(&data, event, &cpuhw->bhrb_stack, NULL); |
| 2317 | } | 2317 | } |
| 2318 | 2318 | ||
| 2319 | if (event->attr.sample_type & PERF_SAMPLE_DATA_SRC && | 2319 | if (event->attr.sample_type & PERF_SAMPLE_DATA_SRC && |
| @@ -940,7 +940,7 @@ static int amd_pmu_v2_handle_irq(struct pt_regs *regs) · 939 unmodified lines | |||
| 940 | continue; | 940 | continue; |
| 941 | 941 | ||
| 942 | if (has_branch_stack(event)) | 942 | if (has_branch_stack(event)) |
| 943 | perf_sample_save_brstack(&data, event, &cpuc->lbr_stack); | 943 | perf_sample_save_brstack(&data, event, &cpuc->lbr_stack, NULL); |
| 944 | 944 | ||
| 945 | if (perf_event_overflow(event, &data, regs)) | 945 | if (perf_event_overflow(event, &data, regs)) |
| 946 | x86_pmu_stop(event, 0); | 946 | x86_pmu_stop(event, 0); |
| @@ -1702,7 +1702,7 @@ int x86_pmu_handle_irq(struct pt_regs *regs) · 1701 unmodified lines | |||
| 1702 | perf_sample_data_init(&data, 0, event->hw.last_period); | 1702 | perf_sample_data_init(&data, 0, event->hw.last_period); |
| 1703 | 1703 | ||
| 1704 | if (has_branch_stack(event)) | 1704 | if (has_branch_stack(event)) |
| 1705 | perf_sample_save_brstack(&data, event, &cpuc->lbr_stack); | 1705 | perf_sample_save_brstack(&data, event, &cpuc->lbr_stack, NULL); |
| 1706 | 1706 | ||
| 1707 | if (perf_event_overflow(event, &data, regs)) | 1707 | if (perf_event_overflow(event, &data, regs)) |
| 1708 | x86_pmu_stop(event, 0); | 1708 | x86_pmu_stop(event, 0); |
| @@ -3047,7 +3047,7 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) · 3046 unmodified lines | |||
| 3047 | perf_sample_data_init(&data, 0, event->hw.last_period); | 3047 | perf_sample_data_init(&data, 0, event->hw.last_period); |
| 3048 | 3048 | ||
| 3049 | if (has_branch_stack(event)) | 3049 | if (has_branch_stack(event)) |
| 3050 | perf_sample_save_brstack(&data, event, &cpuc->lbr_stack); | 3050 | perf_sample_save_brstack(&data, event, &cpuc->lbr_stack, NULL); |
| 3051 | 3051 | ||
| 3052 | if (perf_event_overflow(event, &data, regs)) | 3052 | if (perf_event_overflow(event, &data, regs)) |
| 3053 | x86_pmu_stop(event, 0); | 3053 | x86_pmu_stop(event, 0); |
| @@ -1755,7 +1755,7 @@ static void setup_pebs_fixed_sample_data(struct perf_event *event, · 1754 unmodified lines | |||
| 1755 | setup_pebs_time(event, data, pebs->tsc); | 1755 | setup_pebs_time(event, data, pebs->tsc); |
| 1756 | 1756 | ||
| 1757 | if (has_branch_stack(event)) | 1757 | if (has_branch_stack(event)) |
| 1758 | perf_sample_save_brstack(data, event, &cpuc->lbr_stack); | 1758 | perf_sample_save_brstack(data, event, &cpuc->lbr_stack, NULL); |
| 1759 | } | 1759 | } |
| 1760 | 1760 | ||
| 1761 | static void adaptive_pebs_save_regs(struct pt_regs *regs, | 1761 | static void adaptive_pebs_save_regs(struct pt_regs *regs, |
| @@ -1912,7 +1912,7 @@ static void setup_pebs_adaptive_sample_data(struct perf_event *event, · 150 unmodified lines | |||
| 1912 | 1912 | ||
| 1913 | if (has_branch_stack(event)) { | 1913 | if (has_branch_stack(event)) { |
| 1914 | intel_pmu_store_pebs_lbrs(lbr); | 1914 | intel_pmu_store_pebs_lbrs(lbr); |
| 1915 | perf_sample_save_brstack(data, event, &cpuc->lbr_stack); | 1915 | perf_sample_save_brstack(data, event, &cpuc->lbr_stack, NULL); |
| 1916 | } | 1916 | } |
| 1917 | } | 1917 | } |
| 1918 | 1918 | ||
| @@ -1139,6 +1139,10 @@ static inline bool branch_sample_priv(const struct perf_event *event) · 1138 unmodified lines | |||
| 1139 | return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_PRIV_SAVE; | 1139 | return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_PRIV_SAVE; |
| 1140 | } | 1140 | } |
| 1141 | 1141 | ||
| 1142 | static inline bool branch_sample_counters(const struct perf_event *event) | ||
| 1143 | { | ||
| 1144 | return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_COUNTERS; | ||
| 1145 | } | ||
| 1142 | 1146 | ||
| 1143 | struct perf_sample_data { | 1147 | struct perf_sample_data { |
| 1144 | /* | 1148 | /* |
| @@ -1173,6 +1177,7 @@ struct perf_sample_data { · 28 unmodified lines | |||
| 1173 | struct perf_callchain_entry *callchain; | 1177 | struct perf_callchain_entry *callchain; |
| 1174 | struct perf_raw_record *raw; | 1178 | struct perf_raw_record *raw; |
| 1175 | struct perf_branch_stack *br_stack; | 1179 | struct perf_branch_stack *br_stack; |
| 1180 | u64 *br_stack_cntr; | ||
| 1176 | union perf_sample_weight weight; | 1181 | union perf_sample_weight weight; |
| 1177 | union perf_mem_data_src data_src; | 1182 | union perf_mem_data_src data_src; |
| 1178 | u64 txn; | 1183 | u64 txn; |
| @@ -1250,7 +1255,8 @@ static inline void perf_sample_save_raw_data(struct perf_sample_data *data, · 71 unmodified lines | |||
| 1250 | 1255 | ||
| 1251 | static inline void perf_sample_save_brstack(struct perf_sample_data *data, | 1256 | static inline void perf_sample_save_brstack(struct perf_sample_data *data, |
| 1252 | struct perf_event *event, | 1257 | struct perf_event *event, |
| 1253 | struct perf_branch_stack *brs) | 1258 | struct perf_branch_stack *brs, |
| 1259 | u64 *brs_cntr) | ||
| 1254 | { | 1260 | { |
| 1255 | int size = sizeof(u64); /* nr */ | 1261 | int size = sizeof(u64); /* nr */ |
| 1256 | 1262 | ||
| @@ -1258,7 +1264,16 @@ static inline void perf_sample_save_brstack(struct perf_sample_data *data, · 1 unmodified line | |||
| 1258 | size += sizeof(u64); | 1264 | size += sizeof(u64); |
| 1259 | size += brs->nr * sizeof(struct perf_branch_entry); | 1265 | size += brs->nr * sizeof(struct perf_branch_entry); |
| 1260 | 1266 | ||
| 1267 | /* | ||
| 1268 | * The extension space for counters is appended after the | ||
| 1269 | * struct perf_branch_stack. It is used to store the occurrences | ||
| 1270 | * of events of each branch. | ||
| 1271 | */ | ||
| 1272 | if (brs_cntr) | ||
| 1273 | size += brs->nr * sizeof(u64); | ||
| 1274 | |||
| 1261 | data->br_stack = brs; | 1275 | data->br_stack = brs; |
| 1276 | data->br_stack_cntr = brs_cntr; | ||
| 1262 | data->dyn_size += size; | 1277 | data->dyn_size += size; |
| 1263 | data->sample_flags |= PERF_SAMPLE_BRANCH_STACK; | 1278 | data->sample_flags |= PERF_SAMPLE_BRANCH_STACK; |
| 1264 | } | 1279 | } |
| @@ -204,6 +204,8 @@ enum perf_branch_sample_type_shift { · 203 unmodified lines | |||
| 204 | 204 | ||
| 205 | PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT = 18, /* save privilege mode */ | 205 | PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT = 18, /* save privilege mode */ |
| 206 | 206 | ||
| 207 | PERF_SAMPLE_BRANCH_COUNTERS_SHIFT = 19, /* save occurrences of events on a branch */ | ||
| 208 | |||
| 207 | PERF_SAMPLE_BRANCH_MAX_SHIFT /* non-ABI */ | 209 | PERF_SAMPLE_BRANCH_MAX_SHIFT /* non-ABI */ |
| 208 | }; | 210 | }; |
| 209 | 211 | ||
| @@ -235,6 +237,8 @@ enum perf_branch_sample_type { · 25 unmodified lines | |||
| 235 | 237 | ||
| 236 | PERF_SAMPLE_BRANCH_PRIV_SAVE = 1U << PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT, | 238 | PERF_SAMPLE_BRANCH_PRIV_SAVE = 1U << PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT, |
| 237 | 239 | ||
| 240 | PERF_SAMPLE_BRANCH_COUNTERS = 1U << PERF_SAMPLE_BRANCH_COUNTERS_SHIFT, | ||
| 241 | |||
| 238 | PERF_SAMPLE_BRANCH_MAX = 1U << PERF_SAMPLE_BRANCH_MAX_SHIFT, | 242 | PERF_SAMPLE_BRANCH_MAX = 1U << PERF_SAMPLE_BRANCH_MAX_SHIFT, |
| 239 | }; | 243 | }; |
| 240 | 244 | ||
| @@ -982,6 +986,12 @@ enum perf_event_type { · 741 unmodified lines | |||
| 982 | * { u64 nr; | 986 | * { u64 nr; |
| 983 | * { u64 hw_idx; } && PERF_SAMPLE_BRANCH_HW_INDEX | 987 | * { u64 hw_idx; } && PERF_SAMPLE_BRANCH_HW_INDEX |
| 984 | * { u64 from, to, flags } lbr[nr]; | 988 | * { u64 from, to, flags } lbr[nr]; |
| 989 | * # | ||
| 990 | * # The format of the counters is decided by the | ||
| 991 | * # "branch_counter_nr" and "branch_counter_width", | ||
| 992 | * # which are defined in the ABI. | ||
| 993 | * # | ||
| 994 | * { u64 counters; } cntr[nr] && PERF_SAMPLE_BRANCH_COUNTERS | ||
| 985 | * } && PERF_SAMPLE_BRANCH_STACK | 995 | * } && PERF_SAMPLE_BRANCH_STACK |
| 986 | * | 996 | * |
| 987 | * { u64 abi; # enum perf_sample_regs_abi | 997 | * { u64 abi; # enum perf_sample_regs_abi |
| @@ -7341,6 +7341,14 @@ void perf_output_sample(struct perf_output_handle *handle, · 7340 unmodified lines | |||
| 7341 | if (branch_sample_hw_index(event)) | 7341 | if (branch_sample_hw_index(event)) |
| 7342 | perf_output_put(handle, data->br_stack->hw_idx); | 7342 | perf_output_put(handle, data->br_stack->hw_idx); |
| 7343 | perf_output_copy(handle, data->br_stack->entries, size); | 7343 | perf_output_copy(handle, data->br_stack->entries, size); |
| 7344 | /* | ||
| 7345 | * Add the extension space which is appended | ||
| 7346 | * right after the struct perf_branch_stack. | ||
| 7347 | */ | ||
| 7348 | if (data->br_stack_cntr) { | ||
| 7349 | size = data->br_stack->nr * sizeof(u64); | ||
| 7350 | perf_output_copy(handle, data->br_stack_cntr, size); | ||
| 7351 | } | ||
| 7344 | } else { | 7352 | } else { |
| 7345 | /* | 7353 | /* |
| 7346 | * we always store at least the value of nr | 7354 | * we always store at least the value of nr |