Merge tag 'perf-core-2025-05-25' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull perf events updates from Ingo Molnar: "Core & generic-arch updates: - Add support for dynamic constraints and propagate it to the Intel driver (Kan Liang) - Fix & enhance driver-specific throttling support (Kan Liang) - Record sample last_period before updating on the x86 and PowerPC platforms (Mark Barnett) - Make perf_pmu_unregister() usable (Peter Zijlstra) - Unify perf_event_free_task() / perf_event_exit_task_context() (Peter Zijlstra) - Simplify perf_event_release_kernel() and perf_event_free_task() (Peter Zijlstra) - Allocate non-contiguous AUX pages by default (Yabin Cui) Uprobes updates: - Add support to emulate NOP instructions (Jiri Olsa) - selftests/bpf: Add 5-byte NOP uprobe trigger benchmark (Jiri Olsa) x86 Intel PMU enhancements: - Support Intel Auto Counter Reload [ACR] (Kan Liang) - Add PMU support for Clearwater Forest (Dapeng Mi) - Arch-PEBS preparatory changes: (Dapeng Mi) - Parse CPUID archPerfmonExt leaves for non-hybrid CPUs - Decouple BTS initialization from PEBS initialization - Introduce pairs of PEBS static calls x86 AMD PMU enhancements: - Use hrtimer for handling overflows in the AMD uncore driver (Sandipan Das) - Prevent UMC counters from saturating (Sandipan Das) Fixes and cleanups: - Fix put_ctx() ordering (Frederic Weisbecker) - Fix irq work dereferencing garbage (Frederic Weisbecker) - Misc fixes and cleanups (Changbin Du, Frederic Weisbecker, Ian Rogers, Ingo Molnar, Kan Liang, Peter Zijlstra, Qing Wang, Sandipan Das, Thorsten Blum)" * tag 'perf-core-2025-05-25' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (60 commits) perf/headers: Clean up <linux/perf_event.h> a bit perf/uapi: Clean up <uapi/linux/perf_event.h> a bit perf/uapi: Fix PERF_RECORD_SAMPLE comments in <uapi/linux/perf_event.h> mips/perf: Remove driver-specific throttle support xtensa/perf: Remove driver-specific throttle support sparc/perf: Remove driver-specific throttle support loongarch/perf: Remove driver-specific throttle support csky/perf: Remove driver-specific throttle support arc/perf: Remove driver-specific throttle support alpha/perf: Remove driver-specific throttle support perf/apple_m1: Remove driver-specific throttle support perf/arm: Remove driver-specific throttle support s390/perf: Remove driver-specific throttle support powerpc/perf: Remove driver-specific throttle support perf/x86/zhaoxin: Remove driver-specific throttle support perf/x86/amd: Remove driver-specific throttle support perf/x86/intel: Remove driver-specific throttle support perf: Only dump the throttle log for the leader perf: Fix the throttle logic for a group perf/core: Add the is_event_in_freq_mode() helper to simplify the code ...

Linus Torvalds committed 1 year ago

Commit ddddf9d2 parents eaed94d e7d952c44 files · +1996 −1210Browse files

Unified · Split

44 files

arch/alpha/kernel/perf_event.c +3 −8
arch/arc/kernel/perf_event.c +2 −4
arch/csky/kernel/perf_event.c +1 −2
arch/loongarch/kernel/perf_event.c +1 −2
arch/mips/kernel/perf_event_mipsxx.c +1 −2
arch/powerpc/perf/core-book3s.c +4 −5
arch/powerpc/perf/core-fsl-emb.c +3 −3
arch/s390/kernel/perf_cpum_cf.c +0 −2
arch/s390/kernel/perf_cpum_sf.c +1 −4
arch/sparc/kernel/perf_event.c +1 −2
arch/x86/events/amd/core.c +1 −2
arch/x86/events/amd/ibs.c +1 −3
arch/x86/events/amd/uncore.c +101 −2
arch/x86/events/core.c +25 −12
arch/x86/events/intel/bts.c +77 −73
arch/x86/events/intel/core.c +319 −26
arch/x86/events/intel/ds.c +29 −26
arch/x86/events/intel/knc.c +4 −3
arch/x86/events/intel/lbr.c +1 −1
arch/x86/events/intel/p4.c +1 −2
arch/x86/events/intel/pt.c +2 −0
arch/x86/events/intel/uncore.c +2 −10
arch/x86/events/perf_event.h +43 −2
arch/x86/events/perf_event_flags.h +21 −20
arch/x86/events/zhaoxin/core.c +1 −2
arch/x86/include/asm/msr-index.h +4 −0
arch/x86/include/asm/perf_event.h +1 −0
arch/x86/kernel/uprobes.c +5 −0
arch/xtensa/kernel/perf_event.c +1 −2
drivers/perf/apple_m1_cpu_pmu.c +1 −2
drivers/perf/arm_pmuv3.c +1 −2
drivers/perf/arm_v6_pmu.c +1 −2
drivers/perf/arm_v7_pmu.c +1 −2
drivers/perf/arm_xscale_pmu.c +2 −4
include/linux/cpuhotplug.h +0 −1
include/linux/perf_event.h +167 −131
include/uapi/linux/perf_event.h +334 −323
kernel/cpu.c +0 −5
kernel/events/core.c +424 −185
kernel/events/ring_buffer.c +20 −9
tools/include/uapi/linux/perf_event.h +334 −323
tools/testing/selftests/bpf/bench.c +12 −0
tools/testing/selftests/bpf/benchs/bench_trigger.c +42 −0
tools/testing/selftests/bpf/benchs/run_bench_uprobes.sh +1 −1

arch/alpha/kernel/perf_event.c+3−8

@@ -852,14 +852,9 @@ static void alpha_perf_event_irq_handler(unsigned long la_ptr, · 851 unmodified lines
852	852	alpha_perf_event_update(event, hwc, idx, alpha_pmu->pmc_max_period[idx]+1);
853	853	perf_sample_data_init(&data, 0, hwc->last_period);
854	854
855		− if (alpha_perf_event_set_period(event, hwc, idx)) {
856		− if (perf_event_overflow(event, &data, regs)) {
857		− /* Interrupts coming too quickly; "throttle" the
858		− * counter, i.e., disable it for a little while.
859		− */
860		− alpha_pmu_stop(event, 0);
861		− }
862		− }
	855	+ if (alpha_perf_event_set_period(event, hwc, idx))
	856	+ perf_event_overflow(event, &data, regs);
	857	+
863	858	wrperfmon(PERFMON_CMD_ENABLE, cpuc->idx_mask);
864	859
865	860	return;

arch/arc/kernel/perf_event.c+2−4

@@ -599,10 +599,8 @@ static irqreturn_t arc_pmu_intr(int irq, void *dev) · 598 unmodified lines
599	599
600	600	arc_perf_event_update(event, &event->hw, event->hw.idx);
601	601	perf_sample_data_init(&data, 0, hwc->last_period);
602		− if (arc_pmu_event_set_period(event)) {
603		− if (perf_event_overflow(event, &data, regs))
604		− arc_pmu_stop(event, 0);
605		− }
	602	+ if (arc_pmu_event_set_period(event))
	603	+ perf_event_overflow(event, &data, regs);
606	604
607	605	active_ints &= ~BIT(idx);
608	606	} while (active_ints);

arch/csky/kernel/perf_event.c+1−2

@@ -1139,8 +1139,7 @@ static irqreturn_t csky_pmu_handle_irq(int irq_num, void *dev) · 1138 unmodified lines
1139	1139	perf_sample_data_init(&data, 0, hwc->last_period);
1140	1140	csky_pmu_event_set_period(event);
1141	1141
1142		− if (perf_event_overflow(event, &data, regs))
1143		− csky_pmu_stop_event(event);
	1142	+ perf_event_overflow(event, &data, regs);
1144	1143	}
1145	1144
1146	1145	csky_pmu_enable(&csky_pmu.pmu);

arch/loongarch/kernel/perf_event.c+1−2

@@ -479,8 +479,7 @@ static void handle_associated_event(struct cpu_hw_events *cpuc, int idx, · 478 unmodified lines
479	479	if (!loongarch_pmu_event_set_period(event, hwc, idx))
480	480	return;
481	481
482		− if (perf_event_overflow(event, data, regs))
483		− loongarch_pmu_disable_event(idx);
	482	+ perf_event_overflow(event, data, regs);
484	483	}
485	484
486	485	static irqreturn_t pmu_handle_irq(int irq, void *dev)

arch/mips/kernel/perf_event_mipsxx.c+1−2

@@ -791,8 +791,7 @@ static void handle_associated_event(struct cpu_hw_events *cpuc, · 790 unmodified lines
791	791	if (!mipspmu_event_set_period(event, hwc, idx))
792	792	return;
793	793
794		− if (perf_event_overflow(event, data, regs))
795		− mipsxx_pmu_disable_event(idx);
	794	+ perf_event_overflow(event, data, regs);
796	795	}
797	796
798	797

arch/powerpc/perf/core-book3s.c+4−5

@@ -2239,6 +2239,7 @@ static void record_and_restart(struct perf_event *event, unsigned long val, · 2238 unmodified lines
2239	2239	struct pt_regs *regs)
2240	2240	{
2241	2241	u64 period = event->hw.sample_period;
	2242	+ const u64 last_period = event->hw.last_period;
2242	2243	s64 prev, delta, left;
2243	2244	int record = 0;
2244	2245
@@ -2320,7 +2321,7 @@ static void record_and_restart(struct perf_event *event, unsigned long val, · 75 unmodified lines
2320	2321	if (record) {
2321	2322	struct perf_sample_data data;
2322	2323
2323		− perf_sample_data_init(&data, ~0ULL, event->hw.last_period);
	2324	+ perf_sample_data_init(&data, ~0ULL, last_period);
2324	2325
2325	2326	if (event->attr.sample_type & PERF_SAMPLE_ADDR_TYPE)
2326	2327	perf_get_data_addr(event, regs, &data.addr);
@@ -2343,12 +2344,10 @@ static void record_and_restart(struct perf_event *event, unsigned long val, · 16 unmodified lines
2343	2344	ppmu->get_mem_weight(&data.weight.full, event->attr.sample_type);
2344	2345	data.sample_flags \|= PERF_SAMPLE_WEIGHT_TYPE;
2345	2346	}
2346		− if (perf_event_overflow(event, &data, regs))
2347		− power_pmu_stop(event, 0);
	2347	+ perf_event_overflow(event, &data, regs);
2348	2348	} else if (period) {
2349	2349	/* Account for interrupt in case of invalid SIAR */
2350		− if (perf_event_account_interrupt(event))
2351		− power_pmu_stop(event, 0);
	2350	+ perf_event_account_interrupt(event);
2352	2351	}
2353	2352	}
2354	2353

arch/powerpc/perf/core-fsl-emb.c+3−3

@@ -590,6 +590,7 @@ static void record_and_restart(struct perf_event *event, unsigned long val, · 589 unmodified lines
590	590	struct pt_regs *regs)
591	591	{
592	592	u64 period = event->hw.sample_period;
	593	+ const u64 last_period = event->hw.last_period;
593	594	s64 prev, delta, left;
594	595	int record = 0;
595	596
@@ -632,10 +633,9 @@ static void record_and_restart(struct perf_event *event, unsigned long val, · 36 unmodified lines
632	633	if (record) {
633	634	struct perf_sample_data data;
634	635
635		− perf_sample_data_init(&data, 0, event->hw.last_period);
	636	+ perf_sample_data_init(&data, 0, last_period);
636	637
637		− if (perf_event_overflow(event, &data, regs))
638		− fsl_emb_pmu_stop(event, 0);
	638	+ perf_event_overflow(event, &data, regs);
639	639	}
640	640	}
641	641

arch/s390/kernel/perf_cpum_cf.c+0−2

@@ -980,8 +980,6 @@ static int cfdiag_push_sample(struct perf_event *event, · 979 unmodified lines
980	980	}
981	981
982	982	overflow = perf_event_overflow(event, &data, &regs);
983		− if (overflow)
984		− event->pmu->stop(event, 0);
985	983
986	984	perf_event_update_userpage(event);
987	985	return overflow;

arch/s390/kernel/perf_cpum_sf.c+1−4

@@ -1072,10 +1072,7 @@ static int perf_push_sample(struct perf_event *event, · 1071 unmodified lines
1072	1072	overflow = 0;
1073	1073	if (perf_event_exclude(event, &regs, sde_regs))
1074	1074	goto out;
1075		− if (perf_event_overflow(event, &data, &regs)) {
1076		− overflow = 1;
1077		− event->pmu->stop(event, 0);
1078		− }
	1075	+ overflow = perf_event_overflow(event, &data, &regs);
1079	1076	perf_event_update_userpage(event);
1080	1077	out:
1081	1078	return overflow;

arch/sparc/kernel/perf_event.c+1−2

@@ -1668,8 +1668,7 @@ static int __kprobes perf_event_nmi_handler(struct notifier_block *self, · 1667 unmodified lines
1668	1668	if (!sparc_perf_event_set_period(event, hwc, idx))
1669	1669	continue;
1670	1670
1671		− if (perf_event_overflow(event, &data, regs))
1672		− sparc_pmu_stop(event, 0);
	1671	+ perf_event_overflow(event, &data, regs);
1673	1672	}
1674	1673
1675	1674	finish_clock = sched_clock();

arch/x86/events/amd/core.c+1−2

@@ -1003,8 +1003,7 @@ static int amd_pmu_v2_handle_irq(struct pt_regs *regs) · 1002 unmodified lines
1003	1003
1004	1004	perf_sample_save_brstack(&data, event, &cpuc->lbr_stack, NULL);
1005	1005
1006		− if (perf_event_overflow(event, &data, regs))
1007		− x86_pmu_stop(event, 0);
	1006	+ perf_event_overflow(event, &data, regs);
1008	1007	}
1009	1008
1010	1009	/*

arch/x86/events/amd/ibs.c+1−3

@@ -1373,9 +1373,7 @@ fail: · 1372 unmodified lines
1373	1373	hwc->sample_period = perf_ibs->min_period;
1374	1374
1375	1375	out:
1376		− if (throttle) {
1377		− perf_ibs_stop(event, 0);
1378		− } else {
	1376	+ if (!throttle) {
1379	1377	if (perf_ibs == &perf_ibs_op) {
1380	1378	if (ibs_caps & IBS_CAPS_OPCNTEXT) {
1381	1379	new_config = period & IBS_OP_MAX_CNT_EXT_MASK;

arch/x86/events/amd/uncore.c+101−2

@@ -21,6 +21,7 @@ · 20 unmodified lines
21	21	#define NUM_COUNTERS_NB 4
22	22	#define NUM_COUNTERS_L2 4
23	23	#define NUM_COUNTERS_L3 6
	24	+#define NUM_COUNTERS_MAX 64
24	25
25	26	#define RDPMC_BASE_NB 6
26	27	#define RDPMC_BASE_LLC 10
@@ -38,7 +39,10 @@ struct amd_uncore_ctx { · 11 unmodified lines
38	39	int refcnt;
39	40	int cpu;
40	41	struct perf_event **events;
41		− struct hlist_node node;
	42	+ unsigned long active_mask[BITS_TO_LONGS(NUM_COUNTERS_MAX)];
	43	+ int nr_active;
	44	+ struct hrtimer hrtimer;
	45	+ u64 hrtimer_duration;
42	46	};
43	47
44	48	struct amd_uncore_pmu {
@@ -83,11 +87,51 @@ struct amd_uncore { · 38 unmodified lines
83	87
84	88	static struct amd_uncore uncores[UNCORE_TYPE_MAX];
85	89
	90	+/* Interval for hrtimer, defaults to 60000 milliseconds */
	91	+static unsigned int update_interval = 60 * MSEC_PER_SEC;
	92	+module_param(update_interval, uint, 0444);
	93	+
86	94	static struct amd_uncore_pmu event_to_amd_uncore_pmu(struct perf_event event)
87	95	{
88	96	return container_of(event->pmu, struct amd_uncore_pmu, pmu);
89	97	}
90	98
	99	+static enum hrtimer_restart amd_uncore_hrtimer(struct hrtimer *hrtimer)
	100	+{
	101	+ struct amd_uncore_ctx *ctx;
	102	+ struct perf_event *event;
	103	+ int bit;
	104	+
	105	+ ctx = container_of(hrtimer, struct amd_uncore_ctx, hrtimer);
	106	+
	107	+ if (!ctx->nr_active \|\| ctx->cpu != smp_processor_id())
	108	+ return HRTIMER_NORESTART;
	109	+
	110	+ for_each_set_bit(bit, ctx->active_mask, NUM_COUNTERS_MAX) {
	111	+ event = ctx->events[bit];
	112	+ event->pmu->read(event);
	113	+ }
	114	+
	115	+ hrtimer_forward_now(hrtimer, ns_to_ktime(ctx->hrtimer_duration));
	116	+ return HRTIMER_RESTART;
	117	+}
	118	+
	119	+static void amd_uncore_start_hrtimer(struct amd_uncore_ctx *ctx)
	120	+{
	121	+ hrtimer_start(&ctx->hrtimer, ns_to_ktime(ctx->hrtimer_duration),
	122	+ HRTIMER_MODE_REL_PINNED_HARD);
	123	+}
	124	+
	125	+static void amd_uncore_cancel_hrtimer(struct amd_uncore_ctx *ctx)
	126	+{
	127	+ hrtimer_cancel(&ctx->hrtimer);
	128	+}
	129	+
	130	+static void amd_uncore_init_hrtimer(struct amd_uncore_ctx *ctx)
	131	+{
	132	+ hrtimer_setup(&ctx->hrtimer, amd_uncore_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
	133	+}
	134	+
91	135	static void amd_uncore_read(struct perf_event *event)
92	136	{
93	137	struct hw_perf_event *hwc = &event->hw;
@@ -118,18 +162,26 @@ static void amd_uncore_read(struct perf_event *event) · 24 unmodified lines
118	162
119	163	static void amd_uncore_start(struct perf_event *event, int flags)
120	164	{
	165	+ struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event);
	166	+ struct amd_uncore_ctx ctx = per_cpu_ptr(pmu->ctx, event->cpu);
121	167	struct hw_perf_event *hwc = &event->hw;
122	168
	169	+ if (!ctx->nr_active++)
	170	+ amd_uncore_start_hrtimer(ctx);
	171	+
123	172	if (flags & PERF_EF_RELOAD)
124	173	wrmsrl(hwc->event_base, (u64)local64_read(&hwc->prev_count));
125	174
126	175	hwc->state = 0;
	176	+ __set_bit(hwc->idx, ctx->active_mask);
127	177	wrmsrl(hwc->config_base, (hwc->config \| ARCH_PERFMON_EVENTSEL_ENABLE));
128	178	perf_event_update_userpage(event);
129	179	}
130	180
131	181	static void amd_uncore_stop(struct perf_event *event, int flags)
132	182	{
	183	+ struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event);
	184	+ struct amd_uncore_ctx ctx = per_cpu_ptr(pmu->ctx, event->cpu);
133	185	struct hw_perf_event *hwc = &event->hw;
134	186
135	187	wrmsrl(hwc->config_base, hwc->config);
@@ -139,6 +191,11 @@ static void amd_uncore_stop(struct perf_event *event, int flags) · 3 unmodified lines
139	191	event->pmu->read(event);
140	192	hwc->state \|= PERF_HES_UPTODATE;
141	193	}
	194	+
	195	+ if (!--ctx->nr_active)
	196	+ amd_uncore_cancel_hrtimer(ctx);
	197	+
	198	+ __clear_bit(hwc->idx, ctx->active_mask);
142	199	}
143	200
144	201	static int amd_uncore_add(struct perf_event *event, int flags)
@@ -491,6 +548,9 @@ static int amd_uncore_ctx_init(struct amd_uncore *uncore, unsigned int cpu) · 346 unmodified lines
491	548	goto fail;
492	549	}
493	550
	551	+ amd_uncore_init_hrtimer(curr);
	552	+ curr->hrtimer_duration = (u64)update_interval * NSEC_PER_MSEC;
	553	+
494	554	cpumask_set_cpu(cpu, &pmu->active_mask);
495	555	}
496	556
@@ -880,16 +940,55 @@ static int amd_uncore_umc_event_init(struct perf_event *event) · 383 unmodified lines
880	940
881	941	static void amd_uncore_umc_start(struct perf_event *event, int flags)
882	942	{
	943	+ struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event);
	944	+ struct amd_uncore_ctx ctx = per_cpu_ptr(pmu->ctx, event->cpu);
883	945	struct hw_perf_event *hwc = &event->hw;
884	946
	947	+ if (!ctx->nr_active++)
	948	+ amd_uncore_start_hrtimer(ctx);
	949	+
885	950	if (flags & PERF_EF_RELOAD)
886	951	wrmsrl(hwc->event_base, (u64)local64_read(&hwc->prev_count));
887	952
888	953	hwc->state = 0;
	954	+ __set_bit(hwc->idx, ctx->active_mask);
889	955	wrmsrl(hwc->config_base, (hwc->config \| AMD64_PERFMON_V2_ENABLE_UMC));
890	956	perf_event_update_userpage(event);
891	957	}
892	958
	959	+static void amd_uncore_umc_read(struct perf_event *event)
	960	+{
	961	+ struct hw_perf_event *hwc = &event->hw;
	962	+ u64 prev, new, shift;
	963	+ s64 delta;
	964	+
	965	+ shift = COUNTER_SHIFT + 1;
	966	+ prev = local64_read(&hwc->prev_count);
	967	+
	968	+ /*
	969	+ * UMC counters do not have RDPMC assignments. Read counts directly
	970	+ * from the corresponding PERF_CTR.
	971	+ */
	972	+ rdmsrl(hwc->event_base, new);
	973	+
	974	+ /*
	975	+ * Unlike the other uncore counters, UMC counters saturate and set the
	976	+ * Overflow bit (bit 48) on overflow. Since they do not roll over,
	977	+ * proactively reset the corresponding PERF_CTR when bit 47 is set so
	978	+ * that the counter never gets a chance to saturate.
	979	+ */
	980	+ if (new & BIT_ULL(63 - COUNTER_SHIFT)) {
	981	+ wrmsrl(hwc->event_base, 0);
	982	+ local64_set(&hwc->prev_count, 0);
	983	+ } else {
	984	+ local64_set(&hwc->prev_count, new);
	985	+ }
	986	+
	987	+ delta = (new << shift) - (prev << shift);
	988	+ delta >>= shift;
	989	+ local64_add(delta, &event->count);
	990	+}
	991	+
893	992	static
894	993	void amd_uncore_umc_ctx_scan(struct amd_uncore *uncore, unsigned int cpu)
895	994	{
@@ -968,7 +1067,7 @@ int amd_uncore_umc_ctx_init(struct amd_uncore *uncore, unsigned int cpu) · 72 unmodified lines
968	1067	.del = amd_uncore_del,
969	1068	.start = amd_uncore_umc_start,
970	1069	.stop = amd_uncore_stop,
971		− .read = amd_uncore_read,
	1070	+ .read = amd_uncore_umc_read,
972	1071	.capabilities = PERF_PMU_CAP_NO_EXCLUDE \| PERF_PMU_CAP_NO_INTERRUPT,
973	1072	.module = THIS_MODULE,
974	1073	};

arch/x86/events/core.c+25−12

@@ -95,6 +95,11 @@ DEFINE_STATIC_CALL_NULL(x86_pmu_filter, *x86_pmu.filter); · 94 unmodified lines
95	95
96	96	DEFINE_STATIC_CALL_NULL(x86_pmu_late_setup, *x86_pmu.late_setup);
97	97
	98	+DEFINE_STATIC_CALL_NULL(x86_pmu_pebs_enable, *x86_pmu.pebs_enable);
	99	+DEFINE_STATIC_CALL_NULL(x86_pmu_pebs_disable, *x86_pmu.pebs_disable);
	100	+DEFINE_STATIC_CALL_NULL(x86_pmu_pebs_enable_all, *x86_pmu.pebs_enable_all);
	101	+DEFINE_STATIC_CALL_NULL(x86_pmu_pebs_disable_all, *x86_pmu.pebs_disable_all);
	102	+
98	103	/*
99	104	* This one is magic, it will get called even when PMU init fails (because
100	105	* there is no PMU), in which case it should simply return NULL.
@@ -674,6 +679,7 @@ static int __x86_pmu_event_init(struct perf_event *event) · 573 unmodified lines
674	679	event->hw.idx = -1;
675	680	event->hw.last_cpu = -1;
676	681	event->hw.last_tag = ~0ULL;
	682	+ event->hw.dyn_constraint = ~0ULL;
677	683
678	684	/* mark unused */
679	685	event->hw.extra_reg.idx = EXTRA_REG_NONE;
@@ -756,15 +762,16 @@ void x86_pmu_enable_all(int added) · 76 unmodified lines
756	762
757	763	int is_x86_event(struct perf_event *event)
758	764	{
759		− int i;
760		−
761		− if (!is_hybrid())
762		− return event->pmu == &pmu;
763		−
764		− for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) {
765		− if (event->pmu == &x86_pmu.hybrid_pmu[i].pmu)
766		− return true;
767		− }
	765	+ /*
	766	+ * For a non-hybrid platforms, the type of X86 pmu is
	767	+ * always PERF_TYPE_RAW.
	768	+ * For a hybrid platform, the PERF_PMU_CAP_EXTENDED_HW_TYPE
	769	+ * is a unique capability for the X86 PMU.
	770	+ * Use them to detect a X86 event.
	771	+ */
	772	+ if (event->pmu->type == PERF_TYPE_RAW \|\|
	773	+ event->pmu->capabilities & PERF_PMU_CAP_EXTENDED_HW_TYPE)
	774	+ return true;
768	775
769	776	return false;
770	777	}
@@ -1683,6 +1690,7 @@ int x86_pmu_handle_irq(struct pt_regs *regs) · 912 unmodified lines
1683	1690	struct cpu_hw_events *cpuc;
1684	1691	struct perf_event *event;
1685	1692	int idx, handled = 0;
	1693	+ u64 last_period;
1686	1694	u64 val;
1687	1695
1688	1696	cpuc = this_cpu_ptr(&cpu_hw_events);
@@ -1702,6 +1710,7 @@ int x86_pmu_handle_irq(struct pt_regs *regs) · 13 unmodified lines
1702	1710	continue;
1703	1711
1704	1712	event = cpuc->events[idx];
	1713	+ last_period = event->hw.last_period;
1705	1714
1706	1715	val = static_call(x86_pmu_update)(event);
1707	1716	if (val & (1ULL << (x86_pmu.cntval_bits - 1)))
@@ -1715,12 +1724,11 @@ int x86_pmu_handle_irq(struct pt_regs *regs) · 7 unmodified lines
1715	1724	if (!static_call(x86_pmu_set_period)(event))
1716	1725	continue;
1717	1726
1718		− perf_sample_data_init(&data, 0, event->hw.last_period);
	1727	+ perf_sample_data_init(&data, 0, last_period);
1719	1728
1720	1729	perf_sample_save_brstack(&data, event, &cpuc->lbr_stack, NULL);
1721	1730
1722		− if (perf_event_overflow(event, &data, regs))
1723		− x86_pmu_stop(event, 0);
	1731	+ perf_event_overflow(event, &data, regs);
1724	1732	}
1725	1733
1726	1734	if (handled)
@@ -2046,6 +2054,11 @@ static void x86_pmu_static_call_update(void) · 319 unmodified lines
2046	2054	static_call_update(x86_pmu_filter, x86_pmu.filter);
2047	2055
2048	2056	static_call_update(x86_pmu_late_setup, x86_pmu.late_setup);
	2057	+
	2058	+ static_call_update(x86_pmu_pebs_enable, x86_pmu.pebs_enable);
	2059	+ static_call_update(x86_pmu_pebs_disable, x86_pmu.pebs_disable);
	2060	+ static_call_update(x86_pmu_pebs_enable_all, x86_pmu.pebs_enable_all);
	2061	+ static_call_update(x86_pmu_pebs_disable_all, x86_pmu.pebs_disable_all);
2049	2062	}
2050	2063
2051	2064	static void _x86_pmu_read(struct perf_event *event)

arch/x86/events/intel/bts.c+77−73

@@ -80,54 +80,54 @@ static void * · 79 unmodified lines
80	80	bts_buffer_setup_aux(struct perf_event event, void *pages,
81	81	int nr_pages, bool overwrite)
82	82	{
83		− struct bts_buffer *buf;
	83	+ struct bts_buffer *bb;
84	84	struct page *page;
85	85	int cpu = event->cpu;
86	86	int node = (cpu == -1) ? cpu : cpu_to_node(cpu);
87	87	unsigned long offset;
88	88	size_t size = nr_pages << PAGE_SHIFT;
89		− int pg, nbuf, pad;
	89	+ int pg, nr_buf, pad;
90	90
91	91	/* count all the high order buffers */
92		− for (pg = 0, nbuf = 0; pg < nr_pages;) {
	92	+ for (pg = 0, nr_buf = 0; pg < nr_pages;) {
93	93	page = virt_to_page(pages[pg]);
94	94	pg += buf_nr_pages(page);
95		− nbuf++;
	95	+ nr_buf++;
96	96	}
97	97
98	98	/*
99	99	* to avoid interrupts in overwrite mode, only allow one physical
100	100	*/
101		− if (overwrite && nbuf > 1)
	101	+ if (overwrite && nr_buf > 1)
102	102	return NULL;
103	103
104		− buf = kzalloc_node(offsetof(struct bts_buffer, buf[nbuf]), GFP_KERNEL, node);
105		− if (!buf)
	104	+ bb = kzalloc_node(struct_size(bb, buf, nr_buf), GFP_KERNEL, node);
	105	+ if (!bb)
106	106	return NULL;
107	107
108		− buf->nr_pages = nr_pages;
109		− buf->nr_bufs = nbuf;
110		− buf->snapshot = overwrite;
111		− buf->data_pages = pages;
112		− buf->real_size = size - size % BTS_RECORD_SIZE;
	108	+ bb->nr_pages = nr_pages;
	109	+ bb->nr_bufs = nr_buf;
	110	+ bb->snapshot = overwrite;
	111	+ bb->data_pages = pages;
	112	+ bb->real_size = size - size % BTS_RECORD_SIZE;
113	113
114		− for (pg = 0, nbuf = 0, offset = 0, pad = 0; nbuf < buf->nr_bufs; nbuf++) {
	114	+ for (pg = 0, nr_buf = 0, offset = 0, pad = 0; nr_buf < bb->nr_bufs; nr_buf++) {
115	115	unsigned int __nr_pages;
116	116
117	117	page = virt_to_page(pages[pg]);
118	118	__nr_pages = buf_nr_pages(page);
119		− buf->buf[nbuf].page = page;
120		− buf->buf[nbuf].offset = offset;
121		− buf->buf[nbuf].displacement = (pad ? BTS_RECORD_SIZE - pad : 0);
122		− buf->buf[nbuf].size = buf_size(page) - buf->buf[nbuf].displacement;
123		− pad = buf->buf[nbuf].size % BTS_RECORD_SIZE;
124		− buf->buf[nbuf].size -= pad;
	119	+ bb->buf[nr_buf].page = page;
	120	+ bb->buf[nr_buf].offset = offset;
	121	+ bb->buf[nr_buf].displacement = (pad ? BTS_RECORD_SIZE - pad : 0);
	122	+ bb->buf[nr_buf].size = buf_size(page) - bb->buf[nr_buf].displacement;
	123	+ pad = bb->buf[nr_buf].size % BTS_RECORD_SIZE;
	124	+ bb->buf[nr_buf].size -= pad;
125	125
126	126	pg += __nr_pages;
127	127	offset += __nr_pages << PAGE_SHIFT;
128	128	}
129	129
130		− return buf;
	130	+ return bb;
131	131	}
132	132
133	133	static void bts_buffer_free_aux(void *data)
@@ -135,25 +135,25 @@ static void bts_buffer_free_aux(void *data) · 1 unmodified line
135	135	kfree(data);
136	136	}
137	137
138		−static unsigned long bts_buffer_offset(struct bts_buffer *buf, unsigned int idx)
	138	+static unsigned long bts_buffer_offset(struct bts_buffer *bb, unsigned int idx)
139	139	{
140		− return buf->buf[idx].offset + buf->buf[idx].displacement;
	140	+ return bb->buf[idx].offset + bb->buf[idx].displacement;
141	141	}
142	142
143	143	static void
144		−bts_config_buffer(struct bts_buffer *buf)
	144	+bts_config_buffer(struct bts_buffer *bb)
145	145	{
146	146	int cpu = raw_smp_processor_id();
147	147	struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
148		− struct bts_phys *phys = &buf->buf[buf->cur_buf];
	148	+ struct bts_phys *phys = &bb->buf[bb->cur_buf];
149	149	unsigned long index, thresh = 0, end = phys->size;
150	150	struct page *page = phys->page;
151	151
152		− index = local_read(&buf->head);
	152	+ index = local_read(&bb->head);
153	153
154		− if (!buf->snapshot) {
155		− if (buf->end < phys->offset + buf_size(page))
156		− end = buf->end - phys->offset - phys->displacement;
	154	+ if (!bb->snapshot) {
	155	+ if (bb->end < phys->offset + buf_size(page))
	156	+ end = bb->end - phys->offset - phys->displacement;
157	157
158	158	index -= phys->offset + phys->displacement;
159	159
@@ -168,7 +168,7 @@ bts_config_buffer(struct bts_buffer *buf) · 8 unmodified lines
168	168	ds->bts_buffer_base = (u64)(long)page_address(page) + phys->displacement;
169	169	ds->bts_index = ds->bts_buffer_base + index;
170	170	ds->bts_absolute_maximum = ds->bts_buffer_base + end;
171		− ds->bts_interrupt_threshold = !buf->snapshot
	171	+ ds->bts_interrupt_threshold = !bb->snapshot
172	172	? ds->bts_buffer_base + thresh
173	173	: ds->bts_absolute_maximum + BTS_RECORD_SIZE;
174	174	}
@@ -184,16 +184,16 @@ static void bts_update(struct bts_ctx *bts) · 9 unmodified lines
184	184	{
185	185	int cpu = raw_smp_processor_id();
186	186	struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
187		− struct bts_buffer *buf = perf_get_aux(&bts->handle);
	187	+ struct bts_buffer *bb = perf_get_aux(&bts->handle);
188	188	unsigned long index = ds->bts_index - ds->bts_buffer_base, old, head;
189	189
190		− if (!buf)
	190	+ if (!bb)
191	191	return;
192	192
193		− head = index + bts_buffer_offset(buf, buf->cur_buf);
194		− old = local_xchg(&buf->head, head);
	193	+ head = index + bts_buffer_offset(bb, bb->cur_buf);
	194	+ old = local_xchg(&bb->head, head);
195	195
196		− if (!buf->snapshot) {
	196	+ if (!bb->snapshot) {
197	197	if (old == head)
198	198	return;
199	199
@@ -205,9 +205,9 @@ static void bts_update(struct bts_ctx *bts) · 5 unmodified lines
205	205	* old and head are always in the same physical buffer, so we
206	206	* can subtract them to get the data size.
207	207	*/
208		− local_add(head - old, &buf->data_size);
	208	+ local_add(head - old, &bb->data_size);
209	209	} else {
210		− local_set(&buf->data_size, head);
	210	+ local_set(&bb->data_size, head);
211	211	}
212	212
213	213	/*
@@ -218,7 +218,7 @@ static void bts_update(struct bts_ctx *bts) · 4 unmodified lines
218	218	}
219	219
220	220	static int
221		−bts_buffer_reset(struct bts_buffer buf, struct perf_output_handle handle);
	221	+bts_buffer_reset(struct bts_buffer bb, struct perf_output_handle handle);
222	222
223	223	/*
224	224	* Ordering PMU callbacks wrt themselves and the PMI is done by means
@@ -232,17 +232,17 @@ bts_buffer_reset(struct bts_buffer buf, struct perf_output_handle handle); · 7 unmodified lines
232	232	static void __bts_event_start(struct perf_event *event)
233	233	{
234	234	struct bts_ctx *bts = this_cpu_ptr(bts_ctx);
235		− struct bts_buffer *buf = perf_get_aux(&bts->handle);
	235	+ struct bts_buffer *bb = perf_get_aux(&bts->handle);
236	236	u64 config = 0;
237	237
238		− if (!buf->snapshot)
	238	+ if (!bb->snapshot)
239	239	config \|= ARCH_PERFMON_EVENTSEL_INT;
240	240	if (!event->attr.exclude_kernel)
241	241	config \|= ARCH_PERFMON_EVENTSEL_OS;
242	242	if (!event->attr.exclude_user)
243	243	config \|= ARCH_PERFMON_EVENTSEL_USR;
244	244
245		− bts_config_buffer(buf);
	245	+ bts_config_buffer(bb);
246	246
247	247	/*
248	248	* local barrier to make sure that ds configuration made it
@@ -261,13 +261,13 @@ static void bts_event_start(struct perf_event *event, int flags) · 12 unmodified lines
261	261	{
262	262	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
263	263	struct bts_ctx *bts = this_cpu_ptr(bts_ctx);
264		− struct bts_buffer *buf;
	264	+ struct bts_buffer *bb;
265	265
266		− buf = perf_aux_output_begin(&bts->handle, event);
267		− if (!buf)
	266	+ bb = perf_aux_output_begin(&bts->handle, event);
	267	+ if (!bb)
268	268	goto fail_stop;
269	269
270		− if (bts_buffer_reset(buf, &bts->handle))
	270	+ if (bts_buffer_reset(bb, &bts->handle))
271	271	goto fail_end_stop;
272	272
273	273	bts->ds_back.bts_buffer_base = cpuc->ds->bts_buffer_base;
@@ -306,27 +306,27 @@ static void bts_event_stop(struct perf_event *event, int flags) · 32 unmodified lines
306	306	{
307	307	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
308	308	struct bts_ctx *bts = this_cpu_ptr(bts_ctx);
309		− struct bts_buffer *buf = NULL;
	309	+ struct bts_buffer *bb = NULL;
310	310	int state = READ_ONCE(bts->state);
311	311
312	312	if (state == BTS_STATE_ACTIVE)
313	313	__bts_event_stop(event, BTS_STATE_STOPPED);
314	314
315	315	if (state != BTS_STATE_STOPPED)
316		− buf = perf_get_aux(&bts->handle);
	316	+ bb = perf_get_aux(&bts->handle);
317	317
318	318	event->hw.state \|= PERF_HES_STOPPED;
319	319
320	320	if (flags & PERF_EF_UPDATE) {
321	321	bts_update(bts);
322	322
323		− if (buf) {
324		− if (buf->snapshot)
	323	+ if (bb) {
	324	+ if (bb->snapshot)
325	325	bts->handle.head =
326		− local_xchg(&buf->data_size,
327		− buf->nr_pages << PAGE_SHIFT);
	326	+ local_xchg(&bb->data_size,
	327	+ bb->nr_pages << PAGE_SHIFT);
328	328	perf_aux_output_end(&bts->handle,
329		− local_xchg(&buf->data_size, 0));
	329	+ local_xchg(&bb->data_size, 0));
330	330	}
331	331
332	332	cpuc->ds->bts_index = bts->ds_back.bts_buffer_base;
@@ -382,19 +382,19 @@ void intel_bts_disable_local(void) · 49 unmodified lines
382	382	}
383	383
384	384	static int
385		−bts_buffer_reset(struct bts_buffer buf, struct perf_output_handle handle)
	385	+bts_buffer_reset(struct bts_buffer bb, struct perf_output_handle handle)
386	386	{
387	387	unsigned long head, space, next_space, pad, gap, skip, wakeup;
388	388	unsigned int next_buf;
389	389	struct bts_phys phys, next_phys;
390	390	int ret;
391	391
392		− if (buf->snapshot)
	392	+ if (bb->snapshot)
393	393	return 0;
394	394
395		− head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1);
	395	+ head = handle->head & ((bb->nr_pages << PAGE_SHIFT) - 1);
396	396
397		− phys = &buf->buf[buf->cur_buf];
	397	+ phys = &bb->buf[bb->cur_buf];
398	398	space = phys->offset + phys->displacement + phys->size - head;
399	399	pad = space;
400	400	if (space > handle->size) {
@@ -403,10 +403,10 @@ bts_buffer_reset(struct bts_buffer buf, struct perf_output_handle handle) · 2 unmodified lines
403	403	}
404	404	if (space <= BTS_SAFETY_MARGIN) {
405	405	/* See if next phys buffer has more space */
406		− next_buf = buf->cur_buf + 1;
407		− if (next_buf >= buf->nr_bufs)
	406	+ next_buf = bb->cur_buf + 1;
	407	+ if (next_buf >= bb->nr_bufs)
408	408	next_buf = 0;
409		− next_phys = &buf->buf[next_buf];
	409	+ next_phys = &bb->buf[next_buf];
410	410	gap = buf_size(phys->page) - phys->displacement - phys->size +
411	411	next_phys->displacement;
412	412	skip = pad + gap;
@@ -431,8 +431,8 @@ bts_buffer_reset(struct bts_buffer buf, struct perf_output_handle handle) · 18 unmodified lines
431	431	* anymore, so we must not be racing with
432	432	* bts_update().
433	433	*/
434		− buf->cur_buf = next_buf;
435		− local_set(&buf->head, head);
	434	+ bb->cur_buf = next_buf;
	435	+ local_set(&bb->head, head);
436	436	}
437	437	}
438	438	}
@@ -445,7 +445,7 @@ bts_buffer_reset(struct bts_buffer buf, struct perf_output_handle handle) · 6 unmodified lines
445	445	space -= space % BTS_RECORD_SIZE;
446	446	}
447	447
448		− buf->end = head + space;
	448	+ bb->end = head + space;
449	449
450	450	/*
451	451	* If we have no space, the lost notification would have been sent when
@@ -462,7 +462,7 @@ int intel_bts_interrupt(void) · 10 unmodified lines
462	462	struct debug_store *ds = this_cpu_ptr(&cpu_hw_events)->ds;
463	463	struct bts_ctx *bts;
464	464	struct perf_event *event;
465		− struct bts_buffer *buf;
	465	+ struct bts_buffer *bb;
466	466	s64 old_head;
467	467	int err = -ENOSPC, handled = 0;
468	468
@@ -485,8 +485,8 @@ int intel_bts_interrupt(void) · 16 unmodified lines
485	485	if (READ_ONCE(bts->state) == BTS_STATE_STOPPED)
486	486	return handled;
487	487
488		− buf = perf_get_aux(&bts->handle);
489		− if (!buf)
	488	+ bb = perf_get_aux(&bts->handle);
	489	+ if (!bb)
490	490	return handled;
491	491
492	492	/*
@@ -494,26 +494,26 @@ int intel_bts_interrupt(void) · 1 unmodified line
494	494	* there's no other way of telling, because the pointer will
495	495	* keep moving
496	496	*/
497		− if (buf->snapshot)
	497	+ if (bb->snapshot)
498	498	return 0;
499	499
500		− old_head = local_read(&buf->head);
	500	+ old_head = local_read(&bb->head);
501	501	bts_update(bts);
502	502
503	503	/* no new data */
504		− if (old_head == local_read(&buf->head))
	504	+ if (old_head == local_read(&bb->head))
505	505	return handled;
506	506
507		− perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0));
	507	+ perf_aux_output_end(&bts->handle, local_xchg(&bb->data_size, 0));
508	508
509		− buf = perf_aux_output_begin(&bts->handle, event);
510		− if (buf)
511		− err = bts_buffer_reset(buf, &bts->handle);
	509	+ bb = perf_aux_output_begin(&bts->handle, event);
	510	+ if (bb)
	511	+ err = bts_buffer_reset(bb, &bts->handle);
512	512
513	513	if (err) {
514	514	WRITE_ONCE(bts->state, BTS_STATE_STOPPED);
515	515
516		− if (buf) {
	516	+ if (bb) {
517	517	/*
518	518	* BTS_STATE_STOPPED should be visible before
519	519	* cleared handle::event
@@ -599,7 +599,11 @@ static void bts_event_read(struct perf_event *event) · 79 unmodified lines
599	599
600	600	static __init int bts_init(void)
601	601	{
602		− if (!boot_cpu_has(X86_FEATURE_DTES64) \|\| !x86_pmu.bts)
	602	+ if (!boot_cpu_has(X86_FEATURE_DTES64))
	603	+ return -ENODEV;
	604	+
	605	+ x86_pmu.bts = boot_cpu_has(X86_FEATURE_BTS);
	606	+ if (!x86_pmu.bts)
603	607	return -ENODEV;
604	608
605	609	if (boot_cpu_has(X86_FEATURE_PTI)) {

arch/x86/events/intel/core.c+319−26

@@ -2224,6 +2224,18 @@ static struct extra_reg intel_cmt_extra_regs[] __read_mostly = { · 2223 unmodified lines
2224	2224	EVENT_EXTRA_END
2225	2225	};
2226	2226
	2227	+EVENT_ATTR_STR(topdown-fe-bound, td_fe_bound_skt, "event=0x9c,umask=0x01");
	2228	+EVENT_ATTR_STR(topdown-retiring, td_retiring_skt, "event=0xc2,umask=0x02");
	2229	+EVENT_ATTR_STR(topdown-be-bound, td_be_bound_skt, "event=0xa4,umask=0x02");
	2230	+
	2231	+static struct attribute *skt_events_attrs[] = {
	2232	+ EVENT_PTR(td_fe_bound_skt),
	2233	+ EVENT_PTR(td_retiring_skt),
	2234	+ EVENT_PTR(td_bad_spec_cmt),
	2235	+ EVENT_PTR(td_be_bound_skt),
	2236	+ NULL,
	2237	+};
	2238	+
2227	2239	#define KNL_OT_L2_HITE BIT_ULL(19) /* Other Tile L2 Hit */
2228	2240	#define KNL_OT_L2_HITF BIT_ULL(20) /* Other Tile L2 Hit */
2229	2241	#define KNL_MCDRAM_LOCAL BIT_ULL(21)
@@ -2294,7 +2306,7 @@ static __always_inline void __intel_pmu_disable_all(bool bts) · 64 unmodified lines
2294	2306	static __always_inline void intel_pmu_disable_all(void)
2295	2307	{
2296	2308	__intel_pmu_disable_all(true);
2297		− intel_pmu_pebs_disable_all();
	2309	+ static_call_cond(x86_pmu_pebs_disable_all)();
2298	2310	intel_pmu_lbr_disable_all();
2299	2311	}
2300	2312
@@ -2326,7 +2338,7 @@ static void __intel_pmu_enable_all(int added, bool pmi) · 25 unmodified lines
2326	2338
2327	2339	static void intel_pmu_enable_all(int added)
2328	2340	{
2329		− intel_pmu_pebs_enable_all();
	2341	+ static_call_cond(x86_pmu_pebs_enable_all)();
2330	2342	__intel_pmu_enable_all(added, false);
2331	2343	}
2332	2344
@@ -2583,7 +2595,7 @@ static void intel_pmu_disable_event(struct perf_event *event) · 250 unmodified lines
2583	2595	* so we don't trigger the event without PEBS bit set.
2584	2596	*/
2585	2597	if (unlikely(event->attr.precise_ip))
2586		− intel_pmu_pebs_disable(event);
	2598	+ static_call(x86_pmu_pebs_disable)(event);
2587	2599	}
2588	2600
2589	2601	static void intel_pmu_assign_event(struct perf_event *event, int idx)
@@ -2603,6 +2615,9 @@ static void intel_pmu_del_event(struct perf_event *event) · 13 unmodified lines
2603	2615	intel_pmu_lbr_del(event);
2604	2616	if (event->attr.precise_ip)
2605	2617	intel_pmu_pebs_del(event);
	2618	+ if (is_pebs_counter_event_group(event) \|\|
	2619	+ is_acr_event_group(event))
	2620	+ this_cpu_ptr(&cpu_hw_events)->n_late_setup--;
2606	2621	}
2607	2622
2608	2623	static int icl_set_topdown_event_period(struct perf_event *event)
@@ -2880,6 +2895,52 @@ static void intel_pmu_enable_fixed(struct perf_event *event) · 271 unmodified lines
2880	2895	cpuc->fixed_ctrl_val \|= bits;
2881	2896	}
2882	2897
	2898	+static void intel_pmu_config_acr(int idx, u64 mask, u32 reload)
	2899	+{
	2900	+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
	2901	+ int msr_b, msr_c;
	2902	+
	2903	+ if (!mask && !cpuc->acr_cfg_b[idx])
	2904	+ return;
	2905	+
	2906	+ if (idx < INTEL_PMC_IDX_FIXED) {
	2907	+ msr_b = MSR_IA32_PMC_V6_GP0_CFG_B;
	2908	+ msr_c = MSR_IA32_PMC_V6_GP0_CFG_C;
	2909	+ } else {
	2910	+ msr_b = MSR_IA32_PMC_V6_FX0_CFG_B;
	2911	+ msr_c = MSR_IA32_PMC_V6_FX0_CFG_C;
	2912	+ idx -= INTEL_PMC_IDX_FIXED;
	2913	+ }
	2914	+
	2915	+ if (cpuc->acr_cfg_b[idx] != mask) {
	2916	+ wrmsrl(msr_b + x86_pmu.addr_offset(idx, false), mask);
	2917	+ cpuc->acr_cfg_b[idx] = mask;
	2918	+ }
	2919	+ /* Only need to update the reload value when there is a valid config value. */
	2920	+ if (mask && cpuc->acr_cfg_c[idx] != reload) {
	2921	+ wrmsrl(msr_c + x86_pmu.addr_offset(idx, false), reload);
	2922	+ cpuc->acr_cfg_c[idx] = reload;
	2923	+ }
	2924	+}
	2925	+
	2926	+static void intel_pmu_enable_acr(struct perf_event *event)
	2927	+{
	2928	+ struct hw_perf_event *hwc = &event->hw;
	2929	+
	2930	+ if (!is_acr_event_group(event) \|\| !event->attr.config2) {
	2931	+ /*
	2932	+ * The disable doesn't clear the ACR CFG register.
	2933	+ * Check and clear the ACR CFG register.
	2934	+ */
	2935	+ intel_pmu_config_acr(hwc->idx, 0, 0);
	2936	+ return;
	2937	+ }
	2938	+
	2939	+ intel_pmu_config_acr(hwc->idx, hwc->config1, -hwc->sample_period);
	2940	+}
	2941	+
	2942	+DEFINE_STATIC_CALL_NULL(intel_pmu_enable_acr_event, intel_pmu_enable_acr);
	2943	+
2883	2944	static void intel_pmu_enable_event(struct perf_event *event)
2884	2945	{
2885	2946	u64 enable_mask = ARCH_PERFMON_EVENTSEL_ENABLE;
@@ -2887,16 +2948,19 @@ static void intel_pmu_enable_event(struct perf_event *event) · 1 unmodified line
2887	2948	int idx = hwc->idx;
2888	2949
2889	2950	if (unlikely(event->attr.precise_ip))
2890		− intel_pmu_pebs_enable(event);
	2951	+ static_call(x86_pmu_pebs_enable)(event);
2891	2952
2892	2953	switch (idx) {
2893	2954	case 0 ... INTEL_PMC_IDX_FIXED - 1:
2894	2955	if (branch_sample_counters(event))
2895	2956	enable_mask \|= ARCH_PERFMON_EVENTSEL_BR_CNTR;
2896	2957	intel_set_masks(event, idx);
	2958	+ static_call_cond(intel_pmu_enable_acr_event)(event);
2897	2959	__x86_pmu_enable_event(hwc, enable_mask);
2898	2960	break;
2899	2961	case INTEL_PMC_IDX_FIXED ... INTEL_PMC_IDX_FIXED_BTS - 1:
	2962	+ static_call_cond(intel_pmu_enable_acr_event)(event);
	2963	+ fallthrough;
2900	2964	case INTEL_PMC_IDX_METRIC_BASE ... INTEL_PMC_IDX_METRIC_END:
2901	2965	intel_pmu_enable_fixed(event);
2902	2966	break;
@@ -2914,12 +2978,51 @@ static void intel_pmu_enable_event(struct perf_event *event) · 11 unmodified lines
2914	2978	}
2915	2979	}
2916	2980
	2981	+static void intel_pmu_acr_late_setup(struct cpu_hw_events *cpuc)
	2982	+{
	2983	+ struct perf_event event, leader;
	2984	+ int i, j, idx;
	2985	+
	2986	+ for (i = 0; i < cpuc->n_events; i++) {
	2987	+ leader = cpuc->event_list[i];
	2988	+ if (!is_acr_event_group(leader))
	2989	+ continue;
	2990	+
	2991	+ /* The ACR events must be contiguous. */
	2992	+ for (j = i; j < cpuc->n_events; j++) {
	2993	+ event = cpuc->event_list[j];
	2994	+ if (event->group_leader != leader->group_leader)
	2995	+ break;
	2996	+ for_each_set_bit(idx, (unsigned long *)&event->attr.config2, X86_PMC_IDX_MAX) {
	2997	+ if (WARN_ON_ONCE(i + idx > cpuc->n_events))
	2998	+ return;
	2999	+ __set_bit(cpuc->assign[i + idx], (unsigned long *)&event->hw.config1);
	3000	+ }
	3001	+ }
	3002	+ i = j - 1;
	3003	+ }
	3004	+}
	3005	+
	3006	+void intel_pmu_late_setup(void)
	3007	+{
	3008	+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
	3009	+
	3010	+ if (!cpuc->n_late_setup)
	3011	+ return;
	3012	+
	3013	+ intel_pmu_pebs_late_setup(cpuc);
	3014	+ intel_pmu_acr_late_setup(cpuc);
	3015	+}
	3016	+
2917	3017	static void intel_pmu_add_event(struct perf_event *event)
2918	3018	{
2919	3019	if (event->attr.precise_ip)
2920	3020	intel_pmu_pebs_add(event);
2921	3021	if (intel_pmu_needs_branch_stack(event))
2922	3022	intel_pmu_lbr_add(event);
	3023	+ if (is_pebs_counter_event_group(event) \|\|
	3024	+ is_acr_event_group(event))
	3025	+ this_cpu_ptr(&cpu_hw_events)->n_late_setup++;
2923	3026	}
2924	3027
2925	3028	/*
@@ -3035,8 +3138,7 @@ static void x86_pmu_handle_guest_pebs(struct pt_regs *regs, · 109 unmodified lines
3035	3138	continue;
3036	3139
3037	3140	perf_sample_data_init(data, 0, event->hw.last_period);
3038		− if (perf_event_overflow(event, data, regs))
3039		− x86_pmu_stop(event, 0);
	3141	+ perf_event_overflow(event, data, regs);
3040	3142
3041	3143	/* Inject one fake event is enough. */
3042	3144	break;
@@ -3141,6 +3243,7 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) · 98 unmodified lines
3141	3243
3142	3244	for_each_set_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
3143	3245	struct perf_event *event = cpuc->events[bit];
	3246	+ u64 last_period;
3144	3247
3145	3248	handled++;
3146	3249
@@ -3168,16 +3271,17 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) · 21 unmodified lines
3168	3271	if (is_pebs_counter_event_group(event))
3169	3272	x86_pmu.drain_pebs(regs, &data);
3170	3273
	3274	+ last_period = event->hw.last_period;
	3275	+
3171	3276	if (!intel_pmu_save_and_restart(event))
3172	3277	continue;
3173	3278
3174		− perf_sample_data_init(&data, 0, event->hw.last_period);
	3279	+ perf_sample_data_init(&data, 0, last_period);
3175	3280
3176	3281	if (has_branch_stack(event))
3177	3282	intel_pmu_lbr_save_brstack(&data, cpuc, event);
3178	3283
3179		− if (perf_event_overflow(event, &data, regs))
3180		− x86_pmu_stop(event, 0);
	3284	+ perf_event_overflow(event, &data, regs);
3181	3285	}
3182	3286
3183	3287	return handled;
@@ -3739,10 +3843,9 @@ intel_get_event_constraints(struct cpu_hw_events *cpuc, int idx, · 555 unmodified lines
3739	3843	if (cpuc->excl_cntrs)
3740	3844	return intel_get_excl_constraints(cpuc, event, idx, c2);
3741	3845
3742		− /* Not all counters support the branch counter feature. */
3743		− if (branch_sample_counters(event)) {
	3846	+ if (event->hw.dyn_constraint != ~0ULL) {
3744	3847	c2 = dyn_constraint(cpuc, c2, idx);
3745		− c2->idxmsk64 &= x86_pmu.lbr_counters;
	3848	+ c2->idxmsk64 &= event->hw.dyn_constraint;
3746	3849	c2->weight = hweight64(c2->idxmsk64);
3747	3850	}
3748	3851
@@ -4083,6 +4186,39 @@ end: · 334 unmodified lines
4083	4186	return start;
4084	4187	}
4085	4188
	4189	+static inline bool intel_pmu_has_acr(struct pmu *pmu)
	4190	+{
	4191	+ return !!hybrid(pmu, acr_cause_mask64);
	4192	+}
	4193	+
	4194	+static bool intel_pmu_is_acr_group(struct perf_event *event)
	4195	+{
	4196	+ /* The group leader has the ACR flag set */
	4197	+ if (is_acr_event_group(event))
	4198	+ return true;
	4199	+
	4200	+ /* The acr_mask is set */
	4201	+ if (event->attr.config2)
	4202	+ return true;
	4203	+
	4204	+ return false;
	4205	+}
	4206	+
	4207	+static inline void intel_pmu_set_acr_cntr_constr(struct perf_event *event,
	4208	+ u64 cause_mask, int num)
	4209	+{
	4210	+ event->hw.dyn_constraint &= hybrid(event->pmu, acr_cntr_mask64);
	4211	+ *cause_mask \|= event->attr.config2;
	4212	+ *num += 1;
	4213	+}
	4214	+
	4215	+static inline void intel_pmu_set_acr_caused_constr(struct perf_event *event,
	4216	+ int idx, u64 cause_mask)
	4217	+{
	4218	+ if (test_bit(idx, (unsigned long *)&cause_mask))
	4219	+ event->hw.dyn_constraint &= hybrid(event->pmu, acr_cause_mask64);
	4220	+}
	4221	+
4086	4222	static int intel_pmu_hw_config(struct perf_event *event)
4087	4223	{
4088	4224	int ret = x86_pmu_hw_config(event);
@@ -4144,15 +4280,19 @@ static int intel_pmu_hw_config(struct perf_event *event) · 55 unmodified lines
4144	4280	leader = event->group_leader;
4145	4281	if (branch_sample_call_stack(leader))
4146	4282	return -EINVAL;
4147		− if (branch_sample_counters(leader))
	4283	+ if (branch_sample_counters(leader)) {
4148	4284	num++;
	4285	+ leader->hw.dyn_constraint &= x86_pmu.lbr_counters;
	4286	+ }
4149	4287	leader->hw.flags \|= PERF_X86_EVENT_BRANCH_COUNTERS;
4150	4288
4151	4289	for_each_sibling_event(sibling, leader) {
4152	4290	if (branch_sample_call_stack(sibling))
4153	4291	return -EINVAL;
4154		− if (branch_sample_counters(sibling))
	4292	+ if (branch_sample_counters(sibling)) {
4155	4293	num++;
	4294	+ sibling->hw.dyn_constraint &= x86_pmu.lbr_counters;
	4295	+ }
4156	4296	}
4157	4297
4158	4298	if (num > fls(x86_pmu.lbr_counters))
@@ -4207,6 +4347,94 @@ static int intel_pmu_hw_config(struct perf_event *event) · 48 unmodified lines
4207	4347	event->attr.precise_ip)
4208	4348	event->group_leader->hw.flags \|= PERF_X86_EVENT_PEBS_CNTR;
4209	4349
	4350	+ if (intel_pmu_has_acr(event->pmu) && intel_pmu_is_acr_group(event)) {
	4351	+ struct perf_event sibling, leader = event->group_leader;
	4352	+ struct pmu *pmu = event->pmu;
	4353	+ bool has_sw_event = false;
	4354	+ int num = 0, idx = 0;
	4355	+ u64 cause_mask = 0;
	4356	+
	4357	+ /* Not support perf metrics */
	4358	+ if (is_metric_event(event))
	4359	+ return -EINVAL;
	4360	+
	4361	+ /* Not support freq mode */
	4362	+ if (event->attr.freq)
	4363	+ return -EINVAL;
	4364	+
	4365	+ /* PDist is not supported */
	4366	+ if (event->attr.config2 && event->attr.precise_ip > 2)
	4367	+ return -EINVAL;
	4368	+
	4369	+ /* The reload value cannot exceeds the max period */
	4370	+ if (event->attr.sample_period > x86_pmu.max_period)
	4371	+ return -EINVAL;
	4372	+ /*
	4373	+ * The counter-constraints of each event cannot be finalized
	4374	+ * unless the whole group is scanned. However, it's hard
	4375	+ * to know whether the event is the last one of the group.
	4376	+ * Recalculate the counter-constraints for each event when
	4377	+ * adding a new event.
	4378	+ *
	4379	+ * The group is traversed twice, which may be optimized later.
	4380	+ * In the first round,
	4381	+ * - Find all events which do reload when other events
	4382	+ * overflow and set the corresponding counter-constraints
	4383	+ * - Add all events, which can cause other events reload,
	4384	+ * in the cause_mask
	4385	+ * - Error out if the number of events exceeds the HW limit
	4386	+ * - The ACR events must be contiguous.
	4387	+ * Error out if there are non-X86 events between ACR events.
	4388	+ * This is not a HW limit, but a SW limit.
	4389	+ * With the assumption, the intel_pmu_acr_late_setup() can
	4390	+ * easily convert the event idx to counter idx without
	4391	+ * traversing the whole event list.
	4392	+ */
	4393	+ if (!is_x86_event(leader))
	4394	+ return -EINVAL;
	4395	+
	4396	+ if (leader->attr.config2)
	4397	+ intel_pmu_set_acr_cntr_constr(leader, &cause_mask, &num);
	4398	+
	4399	+ if (leader->nr_siblings) {
	4400	+ for_each_sibling_event(sibling, leader) {
	4401	+ if (!is_x86_event(sibling)) {
	4402	+ has_sw_event = true;
	4403	+ continue;
	4404	+ }
	4405	+ if (!sibling->attr.config2)
	4406	+ continue;
	4407	+ if (has_sw_event)
	4408	+ return -EINVAL;
	4409	+ intel_pmu_set_acr_cntr_constr(sibling, &cause_mask, &num);
	4410	+ }
	4411	+ }
	4412	+ if (leader != event && event->attr.config2) {
	4413	+ if (has_sw_event)
	4414	+ return -EINVAL;
	4415	+ intel_pmu_set_acr_cntr_constr(event, &cause_mask, &num);
	4416	+ }
	4417	+
	4418	+ if (hweight64(cause_mask) > hweight64(hybrid(pmu, acr_cause_mask64)) \|\|
	4419	+ num > hweight64(hybrid(event->pmu, acr_cntr_mask64)))
	4420	+ return -EINVAL;
	4421	+ /*
	4422	+ * In the second round, apply the counter-constraints for
	4423	+ * the events which can cause other events reload.
	4424	+ */
	4425	+ intel_pmu_set_acr_caused_constr(leader, idx++, cause_mask);
	4426	+
	4427	+ if (leader->nr_siblings) {
	4428	+ for_each_sibling_event(sibling, leader)
	4429	+ intel_pmu_set_acr_caused_constr(sibling, idx++, cause_mask);
	4430	+ }
	4431	+
	4432	+ if (leader != event)
	4433	+ intel_pmu_set_acr_caused_constr(event, idx, cause_mask);
	4434	+
	4435	+ leader->hw.flags \|= PERF_X86_EVENT_ACR;
	4436	+ }
	4437	+
4210	4438	if ((event->attr.type == PERF_TYPE_HARDWARE) \|\|
4211	4439	(event->attr.type == PERF_TYPE_HW_CACHE))
4212	4440	return 0;
@@ -4354,7 +4582,7 @@ static struct perf_guest_switch_msr intel_guest_get_msrs(int nr, void *data) · 141 unmodified lines
4354	4582	.guest = intel_ctrl & ~cpuc->intel_ctrl_host_mask & ~pebs_mask,
4355	4583	};
4356	4584
4357		− if (!x86_pmu.pebs)
	4585	+ if (!x86_pmu.ds_pebs)
4358	4586	return arr;
4359	4587
4360	4588	/*
@@ -4952,7 +5180,7 @@ int intel_cpuc_prepare(struct cpu_hw_events *cpuc, int cpu) · 591 unmodified lines
4952	5180	goto err;
4953	5181	}
4954	5182
4955		− if (x86_pmu.flags & (PMU_FL_EXCL_CNTRS \| PMU_FL_TFA \| PMU_FL_BR_CNTR)) {
	5183	+ if (x86_pmu.flags & (PMU_FL_EXCL_CNTRS \| PMU_FL_TFA \| PMU_FL_DYN_CONSTRAINT)) {
4956	5184	size_t sz = X86_PMC_IDX_MAX * sizeof(struct event_constraint);
4957	5185
4958	5186	cpuc->constraint_list = kzalloc_node(sz, GFP_KERNEL, cpu_to_node(cpu));
@@ -5041,7 +5269,7 @@ static inline bool intel_pmu_broken_perf_cap(void) · 82 unmodified lines
5041	5269	return false;
5042	5270	}
5043	5271
5044		−static void update_pmu_cap(struct x86_hybrid_pmu *pmu)
	5272	+static void update_pmu_cap(struct pmu *pmu)
5045	5273	{
5046	5274	unsigned int cntr, fixed_cntr, ecx, edx;
5047	5275	union cpuid35_eax eax;
@@ -5050,20 +5278,30 @@ static void update_pmu_cap(struct x86_hybrid_pmu *pmu) · 2 unmodified lines
5050	5278	cpuid(ARCH_PERFMON_EXT_LEAF, &eax.full, &ebx.full, &ecx, &edx);
5051	5279
5052	5280	if (ebx.split.umask2)
5053		− pmu->config_mask \|= ARCH_PERFMON_EVENTSEL_UMASK2;
	5281	+ hybrid(pmu, config_mask) \|= ARCH_PERFMON_EVENTSEL_UMASK2;
5054	5282	if (ebx.split.eq)
5055		− pmu->config_mask \|= ARCH_PERFMON_EVENTSEL_EQ;
	5283	+ hybrid(pmu, config_mask) \|= ARCH_PERFMON_EVENTSEL_EQ;
5056	5284
5057	5285	if (eax.split.cntr_subleaf) {
5058	5286	cpuid_count(ARCH_PERFMON_EXT_LEAF, ARCH_PERFMON_NUM_COUNTER_LEAF,
5059	5287	&cntr, &fixed_cntr, &ecx, &edx);
5060		− pmu->cntr_mask64 = cntr;
5061		− pmu->fixed_cntr_mask64 = fixed_cntr;
	5288	+ hybrid(pmu, cntr_mask64) = cntr;
	5289	+ hybrid(pmu, fixed_cntr_mask64) = fixed_cntr;
	5290	+ }
	5291	+
	5292	+ if (eax.split.acr_subleaf) {
	5293	+ cpuid_count(ARCH_PERFMON_EXT_LEAF, ARCH_PERFMON_ACR_LEAF,
	5294	+ &cntr, &fixed_cntr, &ecx, &edx);
	5295	+ /* The mask of the counters which can be reloaded */
	5296	+ hybrid(pmu, acr_cntr_mask64) = cntr \| ((u64)fixed_cntr << INTEL_PMC_IDX_FIXED);
	5297	+
	5298	+ /* The mask of the counters which can cause a reload of reloadable counters */
	5299	+ hybrid(pmu, acr_cause_mask64) = ecx \| ((u64)edx << INTEL_PMC_IDX_FIXED);
5062	5300	}
5063	5301
5064	5302	if (!intel_pmu_broken_perf_cap()) {
5065	5303	/* Perf Metric (Bit 15) and PEBS via PT (Bit 16) are hybrid enumeration */
5066		− rdmsrl(MSR_IA32_PERF_CAPABILITIES, pmu->intel_cap.capabilities);
	5304	+ rdmsrl(MSR_IA32_PERF_CAPABILITIES, hybrid(pmu, intel_cap).capabilities);
5067	5305	}
5068	5306	}
5069	5307
@@ -5150,7 +5388,7 @@ static bool init_hybrid_pmu(int cpu) · 80 unmodified lines
5150	5388	goto end;
5151	5389
5152	5390	if (this_cpu_has(X86_FEATURE_ARCH_PERFMON_EXT))
5153		− update_pmu_cap(pmu);
	5391	+ update_pmu_cap(&pmu->pmu);
5154	5392
5155	5393	intel_pmu_check_hybrid_pmus(pmu);
5156	5394
@@ -5524,7 +5762,7 @@ static __init void intel_clovertown_quirk(void) · 367 unmodified lines
5524	5762	* these chips.
5525	5763	*/
5526	5764	pr_warn("PEBS disabled due to CPU errata\n");
5527		− x86_pmu.pebs = 0;
	5765	+ x86_pmu.ds_pebs = 0;
5528	5766	x86_pmu.pebs_constraints = NULL;
5529	5767	}
5530	5768
@@ -6012,7 +6250,7 @@ tsx_is_visible(struct kobject kobj, struct attribute attr, int i) · 481 unmodified lines
6012	6250	static umode_t
6013	6251	pebs_is_visible(struct kobject kobj, struct attribute attr, int i)
6014	6252	{
6015		− return x86_pmu.pebs ? attr->mode : 0;
	6253	+ return x86_pmu.ds_pebs ? attr->mode : 0;
6016	6254	}
6017	6255
6018	6256	static umode_t
@@ -6043,6 +6281,21 @@ td_is_visible(struct kobject kobj, struct attribute attr, int i) · 24 unmodified lines
6043	6281	return attr->mode;
6044	6282	}
6045	6283
	6284	+PMU_FORMAT_ATTR(acr_mask, "config2:0-63");
	6285	+
	6286	+static struct attribute *format_acr_attrs[] = {
	6287	+ &format_attr_acr_mask.attr,
	6288	+ NULL
	6289	+};
	6290	+
	6291	+static umode_t
	6292	+acr_is_visible(struct kobject kobj, struct attribute attr, int i)
	6293	+{
	6294	+ struct device *dev = kobj_to_dev(kobj);
	6295	+
	6296	+ return intel_pmu_has_acr(dev_get_drvdata(dev)) ? attr->mode : 0;
	6297	+}
	6298	+
6046	6299	static struct attribute_group group_events_td = {
6047	6300	.name = "events",
6048	6301	.is_visible = td_is_visible,
@@ -6085,6 +6338,12 @@ static struct attribute_group group_format_evtsel_ext = { · 36 unmodified lines
6085	6338	.is_visible = evtsel_ext_is_visible,
6086	6339	};
6087	6340
	6341	+static struct attribute_group group_format_acr = {
	6342	+ .name = "format",
	6343	+ .attrs = format_acr_attrs,
	6344	+ .is_visible = acr_is_visible,
	6345	+};
	6346	+
6088	6347	static struct attribute_group group_default = {
6089	6348	.attrs = intel_pmu_attrs,
6090	6349	.is_visible = default_is_visible,
@@ -6099,6 +6358,7 @@ static const struct attribute_group *attr_update[] = { · 8 unmodified lines
6099	6358	&group_format_extra,
6100	6359	&group_format_extra_skl,
6101	6360	&group_format_evtsel_ext,
	6361	+ &group_format_acr,
6102	6362	&group_default,
6103	6363	NULL,
6104	6364	};
@@ -6383,6 +6643,7 @@ static const struct attribute_group *hybrid_attr_update[] = { · 278 unmodified lines
6383	6643	&group_caps_lbr,
6384	6644	&hybrid_group_format_extra,
6385	6645	&group_format_evtsel_ext,
	6646	+ &group_format_acr,
6386	6647	&group_default,
6387	6648	&hybrid_group_cpus,
6388	6649	NULL,
@@ -6575,6 +6836,7 @@ static __always_inline void intel_pmu_init_skt(struct pmu *pmu) · 186 unmodified lines
6575	6836	intel_pmu_init_grt(pmu);
6576	6837	hybrid(pmu, event_constraints) = intel_skt_event_constraints;
6577	6838	hybrid(pmu, extra_regs) = intel_cmt_extra_regs;
	6839	+ static_call_update(intel_pmu_enable_acr_event, intel_pmu_enable_acr);
6578	6840	}
6579	6841
6580	6842	__init int intel_pmu_init(void)
@@ -6635,6 +6897,7 @@ __init int intel_pmu_init(void) · 54 unmodified lines
6635	6897
6636	6898	x86_pmu.pebs_events_mask = intel_pmu_pebs_mask(x86_pmu.cntr_mask64);
6637	6899	x86_pmu.pebs_capable = PEBS_COUNTER_MASK;
	6900	+ x86_pmu.config_mask = X86_RAW_EVENT_MASK;
6638	6901
6639	6902	/*
6640	6903	* Quirk: v2 perfmon does not report fixed-purpose events, so
@@ -6663,7 +6926,7 @@ __init int intel_pmu_init(void) · 22 unmodified lines
6663	6926	if (boot_cpu_has(X86_FEATURE_ARCH_LBR))
6664	6927	intel_pmu_arch_lbr_init();
6665	6928
6666		− intel_ds_init();
	6929	+ intel_pebs_init();
6667	6930
6668	6931	x86_add_quirk(intel_arch_events_quirk); /* Install first, so it runs last */
6669	6932
@@ -6673,6 +6936,12 @@ __init int intel_pmu_init(void) · 3 unmodified lines
6673	6936	pr_cont(" AnyThread deprecated, ");
6674	6937	}
6675	6938
	6939	+ /*
	6940	+ * Many features on and after V6 require dynamic constraint,
	6941	+ * e.g., Arch PEBS, ACR.
	6942	+ */
	6943	+ if (version >= 6)
	6944	+ x86_pmu.flags \|= PMU_FL_DYN_CONSTRAINT;
6676	6945	/*
6677	6946	* Install the hw-cache-events table:
6678	6947	*/
@@ -6884,6 +7153,18 @@ __init int intel_pmu_init(void) · 205 unmodified lines
6884	7153	name = "crestmont";
6885	7154	break;
6886	7155
	7156	+ case INTEL_ATOM_DARKMONT_X:
	7157	+ intel_pmu_init_skt(NULL);
	7158	+ intel_pmu_pebs_data_source_cmt();
	7159	+ x86_pmu.pebs_latency_data = cmt_latency_data;
	7160	+ x86_pmu.get_event_constraints = cmt_get_event_constraints;
	7161	+ td_attr = skt_events_attrs;
	7162	+ mem_attr = grt_mem_attrs;
	7163	+ extra_attr = cmt_format_attr;
	7164	+ pr_cont("Darkmont events, ");
	7165	+ name = "darkmont";
	7166	+ break;
	7167	+
6887	7168	case INTEL_WESTMERE:
6888	7169	case INTEL_WESTMERE_EP:
6889	7170	case INTEL_WESTMERE_EX:
@@ -7433,6 +7714,18 @@ __init int intel_pmu_init(void) · 543 unmodified lines
7433	7714	x86_pmu.attr_update = hybrid_attr_update;
7434	7715	}
7435	7716
	7717	+ /*
	7718	+ * The archPerfmonExt (0x23) includes an enhanced enumeration of
	7719	+ * PMU architectural features with a per-core view. For non-hybrid,
	7720	+ * each core has the same PMU capabilities. It's good enough to
	7721	+ * update the x86_pmu from the booting CPU. For hybrid, the x86_pmu
	7722	+ * is used to keep the common capabilities. Still keep the values
	7723	+ * from the leaf 0xa. The core specific update will be done later
	7724	+ * when a new type is online.
	7725	+ */
	7726	+ if (!is_hybrid() && boot_cpu_has(X86_FEATURE_ARCH_PERFMON_EXT))
	7727	+ update_pmu_cap(NULL);
	7728	+
7436	7729	intel_pmu_check_counters_mask(&x86_pmu.cntr_mask64,
7437	7730	&x86_pmu.fixed_cntr_mask64,
7438	7731	&x86_pmu.intel_ctrl);

arch/x86/events/intel/ds.c+29−26

@@ -624,7 +624,7 @@ static int alloc_pebs_buffer(int cpu) · 623 unmodified lines
624	624	int max, node = cpu_to_node(cpu);
625	625	void buffer, insn_buff, *cea;
626	626
627		− if (!x86_pmu.pebs)
	627	+ if (!x86_pmu.ds_pebs)
628	628	return 0;
629	629
630	630	buffer = dsalloc_pages(bsiz, GFP_KERNEL, cpu);
@@ -659,7 +659,7 @@ static void release_pebs_buffer(int cpu) · 28 unmodified lines
659	659	struct cpu_hw_events *hwev = per_cpu_ptr(&cpu_hw_events, cpu);
660	660	void *cea;
661	661
662		− if (!x86_pmu.pebs)
	662	+ if (!x86_pmu.ds_pebs)
663	663	return;
664	664
665	665	kfree(per_cpu(insn_buffer, cpu));
@@ -734,7 +734,7 @@ void release_ds_buffers(void) · 68 unmodified lines
734	734	{
735	735	int cpu;
736	736
737		− if (!x86_pmu.bts && !x86_pmu.pebs)
	737	+ if (!x86_pmu.bts && !x86_pmu.ds_pebs)
738	738	return;
739	739
740	740	for_each_possible_cpu(cpu)
@@ -750,7 +750,8 @@ void release_ds_buffers(void) · 9 unmodified lines
750	750	}
751	751
752	752	for_each_possible_cpu(cpu) {
753		− release_pebs_buffer(cpu);
	753	+ if (x86_pmu.ds_pebs)
	754	+ release_pebs_buffer(cpu);
754	755	release_bts_buffer(cpu);
755	756	}
756	757	}
@@ -761,15 +762,17 @@ void reserve_ds_buffers(void) · 4 unmodified lines
761	762	int cpu;
762	763
763	764	x86_pmu.bts_active = 0;
764		− x86_pmu.pebs_active = 0;
765	765
766		− if (!x86_pmu.bts && !x86_pmu.pebs)
	766	+ if (x86_pmu.ds_pebs)
	767	+ x86_pmu.pebs_active = 0;
	768	+
	769	+ if (!x86_pmu.bts && !x86_pmu.ds_pebs)
767	770	return;
768	771
769	772	if (!x86_pmu.bts)
770	773	bts_err = 1;
771	774
772		− if (!x86_pmu.pebs)
	775	+ if (!x86_pmu.ds_pebs)
773	776	pebs_err = 1;
774	777
775	778	for_each_possible_cpu(cpu) {
@@ -781,7 +784,8 @@ void reserve_ds_buffers(void) · 5 unmodified lines
781	784	if (!bts_err && alloc_bts_buffer(cpu))
782	785	bts_err = 1;
783	786
784		− if (!pebs_err && alloc_pebs_buffer(cpu))
	787	+ if (x86_pmu.ds_pebs && !pebs_err &&
	788	+ alloc_pebs_buffer(cpu))
785	789	pebs_err = 1;
786	790
787	791	if (bts_err && pebs_err)
@@ -793,7 +797,7 @@ void reserve_ds_buffers(void) · 5 unmodified lines
793	797	release_bts_buffer(cpu);
794	798	}
795	799
796		− if (pebs_err) {
	800	+ if (x86_pmu.ds_pebs && pebs_err) {
797	801	for_each_possible_cpu(cpu)
798	802	release_pebs_buffer(cpu);
799	803	}
@@ -805,7 +809,7 @@ void reserve_ds_buffers(void) · 5 unmodified lines
805	809	if (x86_pmu.bts && !bts_err)
806	810	x86_pmu.bts_active = 1;
807	811
808		− if (x86_pmu.pebs && !pebs_err)
	812	+ if (x86_pmu.ds_pebs && !pebs_err)
809	813	x86_pmu.pebs_active = 1;
810	814
811	815	for_each_possible_cpu(cpu) {
@@ -1355,9 +1359,8 @@ static void __intel_pmu_pebs_update_cfg(struct perf_event *event, · 543 unmodified lines
1355	1359	}
1356	1360
1357	1361
1358		−static void intel_pmu_late_setup(void)
	1362	+void intel_pmu_pebs_late_setup(struct cpu_hw_events *cpuc)
1359	1363	{
1360		− struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1361	1364	struct perf_event *event;
1362	1365	u64 pebs_data_cfg = 0;
1363	1366	int i;
@@ -1828,8 +1831,6 @@ static void setup_pebs_fixed_sample_data(struct perf_event *event, · 464 unmodified lines
1828	1831
1829	1832	perf_sample_data_init(data, 0, event->hw.last_period);
1830	1833
1831		− data->period = event->hw.last_period;
1832		−
1833	1834	/*
1834	1835	* Use latency for weight (only avail with PEBS-LL)
1835	1836	*/
@@ -2082,7 +2083,6 @@ static void setup_pebs_adaptive_sample_data(struct perf_event *event, · 246 unmodified lines
2082	2083	sample_type = event->attr.sample_type;
2083	2084	format_group = basic->format_group;
2084	2085	perf_sample_data_init(data, 0, event->hw.last_period);
2085		− data->period = event->hw.last_period;
2086	2086
2087	2087	setup_pebs_time(event, data, basic->tsc);
2088	2088
@@ -2359,8 +2359,7 @@ __intel_pmu_pebs_last_event(struct perf_event *event, · 270 unmodified lines
2359	2359	* All but the last records are processed.
2360	2360	* The last one is left to be able to call the overflow handler.
2361	2361	*/
2362		− if (perf_event_overflow(event, data, regs))
2363		− x86_pmu_stop(event, 0);
	2362	+ perf_event_overflow(event, data, regs);
2364	2363	}
2365	2364
2366	2365	if (hwc->flags & PERF_X86_EVENT_AUTO_RELOAD) {
@@ -2589,8 +2588,8 @@ static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, struct perf_sample_d · 222 unmodified lines
2589	2588	if (error[bit]) {
2590	2589	perf_log_lost_samples(event, error[bit]);
2591	2590
2592		− if (iregs && perf_event_account_interrupt(event))
2593		− x86_pmu_stop(event, 0);
	2591	+ if (iregs)
	2592	+ perf_event_account_interrupt(event);
2594	2593	}
2595	2594
2596	2595	if (counts[bit]) {
@@ -2670,10 +2669,10 @@ static void intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_d · 73 unmodified lines
2670	2669	}
2671	2670
2672	2671	/*
2673		− * BTS, PEBS probe and setup
	2672	+ * PEBS probe and setup
2674	2673	*/
2675	2674
2676		−void __init intel_ds_init(void)
	2675	+void __init intel_pebs_init(void)
2677	2676	{
2678	2677	/*
2679	2678	* No support for 32bit formats
@@ -2681,13 +2680,12 @@ void __init intel_ds_init(void) · 1 unmodified line
2681	2680	if (!boot_cpu_has(X86_FEATURE_DTES64))
2682	2681	return;
2683	2682
2684		− x86_pmu.bts = boot_cpu_has(X86_FEATURE_BTS);
2685		− x86_pmu.pebs = boot_cpu_has(X86_FEATURE_PEBS);
	2683	+ x86_pmu.ds_pebs = boot_cpu_has(X86_FEATURE_PEBS);
2686	2684	x86_pmu.pebs_buffer_size = PEBS_BUFFER_SIZE;
2687	2685	if (x86_pmu.version <= 4)
2688	2686	x86_pmu.pebs_no_isolation = 1;
2689	2687
2690		− if (x86_pmu.pebs) {
	2688	+ if (x86_pmu.ds_pebs) {
2691	2689	char pebs_type = x86_pmu.intel_cap.pebs_trap ? '+' : '-';
2692	2690	char *pebs_qual = "";
2693	2691	int format = x86_pmu.intel_cap.pebs_format;
@@ -2695,6 +2693,11 @@ void __init intel_ds_init(void) · 1 unmodified line
2695	2693	if (format < 4)
2696	2694	x86_pmu.intel_cap.pebs_baseline = 0;
2697	2695
	2696	+ x86_pmu.pebs_enable = intel_pmu_pebs_enable;
	2697	+ x86_pmu.pebs_disable = intel_pmu_pebs_disable;
	2698	+ x86_pmu.pebs_enable_all = intel_pmu_pebs_enable_all;
	2699	+ x86_pmu.pebs_disable_all = intel_pmu_pebs_disable_all;
	2700	+
2698	2701	switch (format) {
2699	2702	case 0:
2700	2703	pr_cont("PEBS fmt0%c, ", pebs_type);
@@ -2779,7 +2782,7 @@ void __init intel_ds_init(void) · 78 unmodified lines
2779	2782
2780	2783	default:
2781	2784	pr_cont("no PEBS fmt%d%c, ", format, pebs_type);
2782		− x86_pmu.pebs = 0;
	2785	+ x86_pmu.ds_pebs = 0;
2783	2786	}
2784	2787	}
2785	2788	}
@@ -2788,7 +2791,7 @@ void perf_restore_debug_store(void) · 2 unmodified lines
2788	2791	{
2789	2792	struct debug_store *ds = __this_cpu_read(cpu_hw_events.ds);
2790	2793
2791		− if (!x86_pmu.bts && !x86_pmu.pebs)
	2794	+ if (!x86_pmu.bts && !x86_pmu.ds_pebs)
2792	2795	return;
2793	2796
2794	2797	wrmsrl(MSR_IA32_DS_AREA, (unsigned long)ds);

arch/x86/events/intel/knc.c+4−3

@@ -241,19 +241,20 @@ again: · 240 unmodified lines
241	241
242	242	for_each_set_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
243	243	struct perf_event *event = cpuc->events[bit];
	244	+ u64 last_period;
244	245
245	246	handled++;
246	247
247	248	if (!test_bit(bit, cpuc->active_mask))
248	249	continue;
249	250
	251	+ last_period = event->hw.last_period;
250	252	if (!intel_pmu_save_and_restart(event))
251	253	continue;
252	254
253		− perf_sample_data_init(&data, 0, event->hw.last_period);
	255	+ perf_sample_data_init(&data, 0, last_period);
254	256
255		− if (perf_event_overflow(event, &data, regs))
256		− x86_pmu_stop(event, 0);
	257	+ perf_event_overflow(event, &data, regs);
257	258	}
258	259
259	260	/*

arch/x86/events/intel/lbr.c+1−1

@@ -1618,7 +1618,7 @@ void __init intel_pmu_arch_lbr_init(void) · 1617 unmodified lines
1618	1618	x86_pmu.lbr_nr = lbr_nr;
1619	1619
1620	1620	if (!!x86_pmu.lbr_counters)
1621		− x86_pmu.flags \|= PMU_FL_BR_CNTR;
	1621	+ x86_pmu.flags \|= PMU_FL_BR_CNTR \| PMU_FL_DYN_CONSTRAINT;
1622	1622
1623	1623	if (x86_pmu.lbr_mispred)
1624	1624	static_branch_enable(&x86_lbr_mispred);

arch/x86/events/intel/p4.c+1−2

@@ -1072,8 +1072,7 @@ static int p4_pmu_handle_irq(struct pt_regs *regs) · 1071 unmodified lines
1072	1072	continue;
1073	1073
1074	1074
1075		− if (perf_event_overflow(event, &data, regs))
1076		− x86_pmu_stop(event, 0);
	1075	+ perf_event_overflow(event, &data, regs);
1077	1076	}
1078	1077
1079	1078	if (handled)

arch/x86/events/intel/pt.c+2−0

@@ -1863,6 +1863,8 @@ static __init int pt_init(void) · 1862 unmodified lines
1863	1863
1864	1864	if (!intel_pt_validate_hw_cap(PT_CAP_topa_multiple_entries))
1865	1865	pt_pmu.pmu.capabilities = PERF_PMU_CAP_AUX_NO_SG;
	1866	+ else
	1867	+ pt_pmu.pmu.capabilities = PERF_PMU_CAP_AUX_PREFER_LARGE;
1866	1868
1867	1869	pt_pmu.pmu.capabilities \|= PERF_PMU_CAP_EXCLUSIVE \|
1868	1870	PERF_PMU_CAP_ITRACE \|

arch/x86/events/intel/uncore.c+2−10

@@ -305,17 +305,11 @@ static enum hrtimer_restart uncore_pmu_hrtimer(struct hrtimer *hrtimer) · 304 unmodified lines
305	305	{
306	306	struct intel_uncore_box *box;
307	307	struct perf_event *event;
308		− unsigned long flags;
309	308	int bit;
310	309
311	310	box = container_of(hrtimer, struct intel_uncore_box, hrtimer);
312	311	if (!box->n_active \|\| box->cpu != smp_processor_id())
313	312	return HRTIMER_NORESTART;
314		− /*
315		− * disable local interrupt to prevent uncore_pmu_event_start/stop
316		− * to interrupt the update process
317		− */
318		− local_irq_save(flags);
319	313
320	314	/*
321	315	* handle boxes with an active event list as opposed to active
@@ -328,8 +322,6 @@ static enum hrtimer_restart uncore_pmu_hrtimer(struct hrtimer *hrtimer) · 6 unmodified lines
328	322	for_each_set_bit(bit, box->active_mask, UNCORE_PMC_IDX_MAX)
329	323	uncore_perf_event_update(box, box->events[bit]);
330	324
331		− local_irq_restore(flags);
332		−
333	325	hrtimer_forward_now(hrtimer, ns_to_ktime(box->hrtimer_duration));
334	326	return HRTIMER_RESTART;
335	327	}
@@ -337,7 +329,7 @@ static enum hrtimer_restart uncore_pmu_hrtimer(struct hrtimer *hrtimer) · 1 unmodified line
337	329	void uncore_pmu_start_hrtimer(struct intel_uncore_box *box)
338	330	{
339	331	hrtimer_start(&box->hrtimer, ns_to_ktime(box->hrtimer_duration),
340		− HRTIMER_MODE_REL_PINNED);
	332	+ HRTIMER_MODE_REL_PINNED_HARD);
341	333	}
342	334
343	335	void uncore_pmu_cancel_hrtimer(struct intel_uncore_box *box)
@@ -347,7 +339,7 @@ void uncore_pmu_cancel_hrtimer(struct intel_uncore_box *box) · 3 unmodified lines
347	339
348	340	static void uncore_pmu_init_hrtimer(struct intel_uncore_box *box)
349	341	{
350		− hrtimer_setup(&box->hrtimer, uncore_pmu_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	342	+ hrtimer_setup(&box->hrtimer, uncore_pmu_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
351	343	}
352	344
353	345	static struct intel_uncore_box uncore_alloc_box(struct intel_uncore_type type,

arch/x86/events/perf_event.h+43−2

@@ -127,6 +127,11 @@ static inline bool is_pebs_counter_event_group(struct perf_event *event) · 126 unmodified lines
127	127	return check_leader_group(event->group_leader, PERF_X86_EVENT_PEBS_CNTR);
128	128	}
129	129
	130	+static inline bool is_acr_event_group(struct perf_event *event)
	131	+{
	132	+ return check_leader_group(event->group_leader, PERF_X86_EVENT_ACR);
	133	+}
	134	+
130	135	struct amd_nb {
131	136	int nb_id; /* NorthBridge id */
132	137	int refcnt; /* reference count */
@@ -268,6 +273,7 @@ struct cpu_hw_events { · 135 unmodified lines
268	273	struct event_constraint *event_constraint[X86_PMC_IDX_MAX];
269	274
270	275	int n_excl; /* the number of exclusive events */
	276	+ int n_late_setup; /* the num of events needs late setup */
271	277
272	278	unsigned int txn_flags;
273	279	int is_fake;
@@ -293,6 +299,10 @@ struct cpu_hw_events { · 19 unmodified lines
293	299	u64 fixed_ctrl_val;
294	300	u64 active_fixed_ctrl_val;
295	301
	302	+ /* Intel ACR configuration */
	303	+ u64 acr_cfg_b[X86_PMC_IDX_MAX];
	304	+ u64 acr_cfg_c[X86_PMC_IDX_MAX];
	305	+
296	306	/*
297	307	* Intel LBR bits
298	308	*/
@@ -714,6 +724,15 @@ struct x86_hybrid_pmu { · 415 unmodified lines
714	724	u64 fixed_cntr_mask64;
715	725	unsigned long fixed_cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
716	726	};
	727	+
	728	+ union {
	729	+ u64 acr_cntr_mask64;
	730	+ unsigned long acr_cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
	731	+ };
	732	+ union {
	733	+ u64 acr_cause_mask64;
	734	+ unsigned long acr_cause_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
	735	+ };
717	736	struct event_constraint unconstrained;
718	737
719	738	u64 hw_cache_event_ids
@@ -796,6 +815,10 @@ struct x86_pmu { · 76 unmodified lines
796	815	int (hw_config)(struct perf_event event);
797	816	int (schedule_events)(struct cpu_hw_events cpuc, int n, int *assign);
798	817	void (*late_setup)(void);
	818	+ void (pebs_enable)(struct perf_event event);
	819	+ void (pebs_disable)(struct perf_event event);
	820	+ void (*pebs_enable_all)(void);
	821	+ void (*pebs_disable_all)(void);
799	822	unsigned eventsel;
800	823	unsigned perfctr;
801	824	unsigned fixedctr;
@@ -812,6 +835,14 @@ struct x86_pmu { · 10 unmodified lines
812	835	u64 fixed_cntr_mask64;
813	836	unsigned long fixed_cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
814	837	};
	838	+ union {
	839	+ u64 acr_cntr_mask64;
	840	+ unsigned long acr_cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
	841	+ };
	842	+ union {
	843	+ u64 acr_cause_mask64;
	844	+ unsigned long acr_cause_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
	845	+ };
815	846	int cntval_bits;
816	847	u64 cntval_mask;
817	848	union {
@@ -878,7 +909,7 @@ struct x86_pmu { · 60 unmodified lines
878	909	*/
879	910	unsigned int bts :1,
880	911	bts_active :1,
881		− pebs :1,
	912	+ ds_pebs :1,
882	913	pebs_active :1,
883	914	pebs_broken :1,
884	915	pebs_prec_dist :1,
@@ -1049,6 +1080,7 @@ do { \ · 164 unmodified lines
1049	1080	#define PMU_FL_MEM_LOADS_AUX 0x100 /* Require an auxiliary event for the complete memory info */
1050	1081	#define PMU_FL_RETIRE_LATENCY 0x200 /* Support Retire Latency in PEBS */
1051	1082	#define PMU_FL_BR_CNTR 0x400 /* Support branch counter logging */
	1083	+#define PMU_FL_DYN_CONSTRAINT 0x800 /* Needs dynamic constraint */
1052	1084
1053	1085	#define EVENT_VAR(_id) event_attr_##_id
1054	1086	#define EVENT_PTR(_id) &event_attr_##_id.attr.attr
@@ -1091,6 +1123,7 @@ static struct perf_pmu_format_hybrid_attr format_attr_hybrid_##_name = {\ · 36 unmodified lines
1091	1123	.pmu_type = _pmu, \
1092	1124	}
1093	1125
	1126	+int is_x86_event(struct perf_event *event);
1094	1127	struct pmu *x86_get_pmu(unsigned int cpu);
1095	1128	extern struct x86_pmu x86_pmu __read_mostly;
1096	1129
@@ -1098,6 +1131,10 @@ DECLARE_STATIC_CALL(x86_pmu_set_period, *x86_pmu.set_period); · 1 unmodified line
1098	1131	DECLARE_STATIC_CALL(x86_pmu_update, *x86_pmu.update);
1099	1132	DECLARE_STATIC_CALL(x86_pmu_drain_pebs, *x86_pmu.drain_pebs);
1100	1133	DECLARE_STATIC_CALL(x86_pmu_late_setup, *x86_pmu.late_setup);
	1134	+DECLARE_STATIC_CALL(x86_pmu_pebs_enable, *x86_pmu.pebs_enable);
	1135	+DECLARE_STATIC_CALL(x86_pmu_pebs_disable, *x86_pmu.pebs_disable);
	1136	+DECLARE_STATIC_CALL(x86_pmu_pebs_enable_all, *x86_pmu.pebs_enable_all);
	1137	+DECLARE_STATIC_CALL(x86_pmu_pebs_disable_all, *x86_pmu.pebs_disable_all);
1101	1138
1102	1139	static __always_inline struct x86_perf_task_context_opt task_context_opt(void ctx)
1103	1140	{
@@ -1587,6 +1624,8 @@ void intel_pmu_disable_bts(void); · 483 unmodified lines
1587	1624
1588	1625	int intel_pmu_drain_bts_buffer(void);
1589	1626
	1627	+void intel_pmu_late_setup(void);
	1628	+
1590	1629	u64 grt_latency_data(struct perf_event *event, u64 status);
1591	1630
1592	1631	u64 cmt_latency_data(struct perf_event *event, u64 status);
@@ -1643,11 +1682,13 @@ void intel_pmu_pebs_disable_all(void); · 50 unmodified lines
1643	1682
1644	1683	void intel_pmu_pebs_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in);
1645	1684
	1685	+void intel_pmu_pebs_late_setup(struct cpu_hw_events *cpuc);
	1686	+
1646	1687	void intel_pmu_drain_pebs_buffer(void);
1647	1688
1648	1689	void intel_pmu_store_pebs_lbrs(struct lbr_entry *lbr);
1649	1690
1650		−void intel_ds_init(void);
	1691	+void intel_pebs_init(void);
1651	1692
1652	1693	void intel_pmu_lbr_save_brstack(struct perf_sample_data *data,
1653	1694	struct cpu_hw_events *cpuc,

arch/x86/events/perf_event_flags.h+21−20

@@ -2,23 +2,24 @@ · 1 unmodified line
2	2	/*
3	3	* struct hw_perf_event.flags flags
4	4	*/
5		−PERF_ARCH(PEBS_LDLAT, 0x00001) /* ld+ldlat data address sampling */
6		−PERF_ARCH(PEBS_ST, 0x00002) /* st data address sampling */
7		−PERF_ARCH(PEBS_ST_HSW, 0x00004) /* haswell style datala, store */
8		−PERF_ARCH(PEBS_LD_HSW, 0x00008) /* haswell style datala, load */
9		−PERF_ARCH(PEBS_NA_HSW, 0x00010) /* haswell style datala, unknown */
10		−PERF_ARCH(EXCL, 0x00020) /* HT exclusivity on counter */
11		−PERF_ARCH(DYNAMIC, 0x00040) /* dynamic alloc'd constraint */
12		−PERF_ARCH(PEBS_CNTR, 0x00080) /* PEBS counters snapshot */
13		−PERF_ARCH(EXCL_ACCT, 0x00100) /* accounted EXCL event */
14		−PERF_ARCH(AUTO_RELOAD, 0x00200) /* use PEBS auto-reload */
15		−PERF_ARCH(LARGE_PEBS, 0x00400) /* use large PEBS */
16		−PERF_ARCH(PEBS_VIA_PT, 0x00800) /* use PT buffer for PEBS */
17		−PERF_ARCH(PAIR, 0x01000) /* Large Increment per Cycle */
18		−PERF_ARCH(LBR_SELECT, 0x02000) /* Save/Restore MSR_LBR_SELECT */
19		−PERF_ARCH(TOPDOWN, 0x04000) /* Count Topdown slots/metrics events */
20		−PERF_ARCH(PEBS_STLAT, 0x08000) /* st+stlat data address sampling */
21		−PERF_ARCH(AMD_BRS, 0x10000) /* AMD Branch Sampling */
22		−PERF_ARCH(PEBS_LAT_HYBRID, 0x20000) /* ld and st lat for hybrid */
23		−PERF_ARCH(NEEDS_BRANCH_STACK, 0x40000) /* require branch stack setup */
24		−PERF_ARCH(BRANCH_COUNTERS, 0x80000) /* logs the counters in the extra space of each branch */
	5	+PERF_ARCH(PEBS_LDLAT, 0x0000001) /* ld+ldlat data address sampling */
	6	+PERF_ARCH(PEBS_ST, 0x0000002) /* st data address sampling */
	7	+PERF_ARCH(PEBS_ST_HSW, 0x0000004) /* haswell style datala, store */
	8	+PERF_ARCH(PEBS_LD_HSW, 0x0000008) /* haswell style datala, load */
	9	+PERF_ARCH(PEBS_NA_HSW, 0x0000010) /* haswell style datala, unknown */
	10	+PERF_ARCH(EXCL, 0x0000020) /* HT exclusivity on counter */
	11	+PERF_ARCH(DYNAMIC, 0x0000040) /* dynamic alloc'd constraint */
	12	+PERF_ARCH(PEBS_CNTR, 0x0000080) /* PEBS counters snapshot */
	13	+PERF_ARCH(EXCL_ACCT, 0x0000100) /* accounted EXCL event */
	14	+PERF_ARCH(AUTO_RELOAD, 0x0000200) /* use PEBS auto-reload */
	15	+PERF_ARCH(LARGE_PEBS, 0x0000400) /* use large PEBS */
	16	+PERF_ARCH(PEBS_VIA_PT, 0x0000800) /* use PT buffer for PEBS */
	17	+PERF_ARCH(PAIR, 0x0001000) /* Large Increment per Cycle */
	18	+PERF_ARCH(LBR_SELECT, 0x0002000) /* Save/Restore MSR_LBR_SELECT */
	19	+PERF_ARCH(TOPDOWN, 0x0004000) /* Count Topdown slots/metrics events */
	20	+PERF_ARCH(PEBS_STLAT, 0x0008000) /* st+stlat data address sampling */
	21	+PERF_ARCH(AMD_BRS, 0x0010000) /* AMD Branch Sampling */
	22	+PERF_ARCH(PEBS_LAT_HYBRID, 0x0020000) /* ld and st lat for hybrid */
	23	+PERF_ARCH(NEEDS_BRANCH_STACK, 0x0040000) /* require branch stack setup */
	24	+PERF_ARCH(BRANCH_COUNTERS, 0x0080000) /* logs the counters in the extra space of each branch */
	25	+PERF_ARCH(ACR, 0x0100000) /* Auto counter reload */

arch/x86/events/zhaoxin/core.c+1−2

@@ -397,8 +397,7 @@ again: · 396 unmodified lines
397	397	if (!x86_perf_event_set_period(event))
398	398	continue;
399	399
400		− if (perf_event_overflow(event, &data, regs))
401		− x86_pmu_stop(event, 0);
	400	+ perf_event_overflow(event, &data, regs);
402	401	}
403	402
404	403	/*

arch/x86/include/asm/msr-index.h+4−0

@@ -602,7 +602,11 @@ · 601 unmodified lines
602	602	/* V6 PMON MSR range */
603	603	#define MSR_IA32_PMC_V6_GP0_CTR 0x1900
604	604	#define MSR_IA32_PMC_V6_GP0_CFG_A 0x1901
	605	+#define MSR_IA32_PMC_V6_GP0_CFG_B 0x1902
	606	+#define MSR_IA32_PMC_V6_GP0_CFG_C 0x1903
605	607	#define MSR_IA32_PMC_V6_FX0_CTR 0x1980
	608	+#define MSR_IA32_PMC_V6_FX0_CFG_B 0x1982
	609	+#define MSR_IA32_PMC_V6_FX0_CFG_C 0x1983
606	610	#define MSR_IA32_PMC_V6_STEP 4
607	611
608	612	/* KeyID partitioning between MKTME and TDX */

arch/x86/include/asm/perf_event.h+1−0

@@ -195,6 +195,7 @@ union cpuid10_edx { · 194 unmodified lines
195	195	*/
196	196	#define ARCH_PERFMON_EXT_LEAF 0x00000023
197	197	#define ARCH_PERFMON_NUM_COUNTER_LEAF 0x1
	198	+#define ARCH_PERFMON_ACR_LEAF 0x2
198	199
199	200	union cpuid35_eax {
200	201	struct {

arch/x86/kernel/uprobes.c+5−0

@@ -840,6 +840,11 @@ static int branch_setup_xol_ops(struct arch_uprobe auprobe, struct insn insn) · 839 unmodified lines
840	840	insn_byte_t p;
841	841	int i;
842	842
	843	+ /* x86_nops[insn->length]; same as jmp with .offs = 0 */
	844	+ if (insn->length <= ASM_NOP_MAX &&
	845	+ !memcmp(insn->kaddr, x86_nops[insn->length], insn->length))
	846	+ goto setup;
	847	+
843	848	switch (opc1) {
844	849	case 0xeb: /* jmp 8 */
845	850	case 0xe9: /* jmp 32 */

arch/xtensa/kernel/perf_event.c+1−2

@@ -388,8 +388,7 @@ irqreturn_t xtensa_pmu_irq_handler(int irq, void *dev_id) · 387 unmodified lines
388	388	struct pt_regs *regs = get_irq_regs();
389	389
390	390	perf_sample_data_init(&data, 0, last_period);
391		− if (perf_event_overflow(event, &data, regs))
392		− xtensa_pmu_stop(event, 0);
	391	+ perf_event_overflow(event, &data, regs);
393	392	}
394	393
395	394	rc = IRQ_HANDLED;

drivers/perf/apple_m1_cpu_pmu.c+1−2

@@ -474,8 +474,7 @@ static irqreturn_t m1_pmu_handle_irq(struct arm_pmu *cpu_pmu) · 473 unmodified lines
474	474	if (!armpmu_event_set_period(event))
475	475	continue;
476	476
477		− if (perf_event_overflow(event, &data, regs))
478		− m1_pmu_disable_event(event);
	477	+ perf_event_overflow(event, &data, regs);
479	478	}
480	479
481	480	cpu_pmu->start(cpu_pmu);

drivers/perf/arm_pmuv3.c+1−2

@@ -887,8 +887,7 @@ static irqreturn_t armv8pmu_handle_irq(struct arm_pmu *cpu_pmu) · 886 unmodified lines
887	887	* an irq_work which will be taken care of in the handling of
888	888	* IPI_IRQ_WORK.
889	889	*/
890		− if (perf_event_overflow(event, &data, regs))
891		− cpu_pmu->disable(event);
	890	+ perf_event_overflow(event, &data, regs);
892	891	}
893	892	armv8pmu_start(cpu_pmu);
894	893

drivers/perf/arm_v6_pmu.c+1−2

@@ -276,8 +276,7 @@ armv6pmu_handle_irq(struct arm_pmu *cpu_pmu) · 275 unmodified lines
276	276	if (!armpmu_event_set_period(event))
277	277	continue;
278	278
279		− if (perf_event_overflow(event, &data, regs))
280		− cpu_pmu->disable(event);
	279	+ perf_event_overflow(event, &data, regs);
281	280	}
282	281
283	282	/*

drivers/perf/arm_v7_pmu.c+1−2

@@ -930,8 +930,7 @@ static irqreturn_t armv7pmu_handle_irq(struct arm_pmu *cpu_pmu) · 929 unmodified lines
930	930	if (!armpmu_event_set_period(event))
931	931	continue;
932	932
933		− if (perf_event_overflow(event, &data, regs))
934		− cpu_pmu->disable(event);
	933	+ perf_event_overflow(event, &data, regs);
935	934	}
936	935
937	936	/*

drivers/perf/arm_xscale_pmu.c+2−4

@@ -186,8 +186,7 @@ xscale1pmu_handle_irq(struct arm_pmu *cpu_pmu) · 185 unmodified lines
186	186	if (!armpmu_event_set_period(event))
187	187	continue;
188	188
189		− if (perf_event_overflow(event, &data, regs))
190		− cpu_pmu->disable(event);
	189	+ perf_event_overflow(event, &data, regs);
191	190	}
192	191
193	192	irq_work_run();
@@ -519,8 +518,7 @@ xscale2pmu_handle_irq(struct arm_pmu *cpu_pmu) · 325 unmodified lines
519	518	if (!armpmu_event_set_period(event))
520	519	continue;
521	520
522		− if (perf_event_overflow(event, &data, regs))
523		− cpu_pmu->disable(event);
	521	+ perf_event_overflow(event, &data, regs);
524	522	}
525	523
526	524	irq_work_run();

include/linux/cpuhotplug.h+0−1

@@ -60,7 +60,6 @@ enum cpuhp_state { · 59 unmodified lines
60	60	/* PREPARE section invoked on a control CPU */
61	61	CPUHP_OFFLINE = 0,
62	62	CPUHP_CREATE_THREADS,
63		− CPUHP_PERF_PREPARE,
64	63	CPUHP_PERF_X86_PREPARE,
65	64	CPUHP_PERF_X86_AMD_UNCORE_PREP,
66	65	CPUHP_PERF_POWER,

include/linux/perf_event.h+167−131

@@ -26,18 +26,9 @@ · 25 unmodified lines
26	26	# include <asm/local64.h>
27	27	#endif
28	28
29		−#define PERF_GUEST_ACTIVE 0x01
30		−#define PERF_GUEST_USER 0x02
31		−
32		−struct perf_guest_info_callbacks {
33		− unsigned int (*state)(void);
34		− unsigned long (*get_ip)(void);
35		− unsigned int (*handle_intel_pt_intr)(void);
36		−};
37		−
38	29	#ifdef CONFIG_HAVE_HW_BREAKPOINT
39		−#include <linux/rhashtable-types.h>
40		−#include <asm/hw_breakpoint.h>
	30	+# include <linux/rhashtable-types.h>
	31	+# include <asm/hw_breakpoint.h>
41	32	#endif
42	33
43	34	#include <linux/list.h>
@@ -62,19 +53,20 @@ struct perf_guest_info_callbacks { · 18 unmodified lines
62	53	#include <linux/security.h>
63	54	#include <linux/static_call.h>
64	55	#include <linux/lockdep.h>
	56	+
65	57	#include <asm/local.h>
66	58
67	59	struct perf_callchain_entry {
68		− __u64 nr;
69		− __u64 ip[]; /* /proc/sys/kernel/perf_event_max_stack */
	60	+ u64 nr;
	61	+ u64 ip[]; /* /proc/sys/kernel/perf_event_max_stack */
70	62	};
71	63
72	64	struct perf_callchain_entry_ctx {
73		− struct perf_callchain_entry *entry;
74		− u32 max_stack;
75		− u32 nr;
76		− short contexts;
77		− bool contexts_maxed;
	65	+ struct perf_callchain_entry *entry;
	66	+ u32 max_stack;
	67	+ u32 nr;
	68	+ short contexts;
	69	+ bool contexts_maxed;
78	70	};
79	71
80	72	typedef unsigned long (perf_copy_f)(void dst, const void *src,
@@ -121,8 +113,8 @@ static __always_inline bool perf_raw_frag_last(const struct perf_raw_frag *frag) · 40 unmodified lines
121	113	* already stored in age order, the hw_idx should be 0.
122	114	*/
123	115	struct perf_branch_stack {
124		− __u64 nr;
125		− __u64 hw_idx;
	116	+ u64 nr;
	117	+ u64 hw_idx;
126	118	struct perf_branch_entry entries[];
127	119	};
128	120
@@ -132,10 +124,10 @@ struct task_struct; · 3 unmodified lines
132	124	* extra PMU register associated with an event
133	125	*/
134	126	struct hw_perf_event_extra {
135		− u64 config; /* register value */
136		− unsigned int reg; /* register address or index */
137		− int alloc; /* extra register already allocated */
138		− int idx; /* index in shared_regs->regs[] */
	127	+ u64 config; /* register value */
	128	+ unsigned int reg; /* register address or index */
	129	+ int alloc; /* extra register already allocated */
	130	+ int idx; /* index in shared_regs->regs[] */
139	131	};
140	132
141	133	/**
@@ -144,8 +136,8 @@ struct hw_perf_event_extra { · 2 unmodified lines
144	136	* PERF_EVENT_FLAG_ARCH bits are reserved for architecture-specific
145	137	* usage.
146	138	*/
147		−#define PERF_EVENT_FLAG_ARCH 0x000fffff
148		−#define PERF_EVENT_FLAG_USER_READ_CNT 0x80000000
	139	+#define PERF_EVENT_FLAG_ARCH 0x0fffffff
	140	+#define PERF_EVENT_FLAG_USER_READ_CNT 0x80000000
149	141
150	142	static_assert((PERF_EVENT_FLAG_USER_READ_CNT & PERF_EVENT_FLAG_ARCH) == 0);
151	143
@@ -157,7 +149,9 @@ struct hw_perf_event { · 5 unmodified lines
157	149	union {
158	150	struct { /* hardware */
159	151	u64 config;
	152	+ u64 config1;
160	153	u64 last_tag;
	154	+ u64 dyn_constraint;
161	155	unsigned long config_base;
162	156	unsigned long event_base;
163	157	int event_base_rdpmc;
@@ -225,9 +219,14 @@ struct hw_perf_event { · 61 unmodified lines
225	219	/*
226	220	* hw_perf_event::state flags; used to track the PERF_EF_* state.
227	221	*/
228		−#define PERF_HES_STOPPED 0x01 /* the counter is stopped */
229		−#define PERF_HES_UPTODATE 0x02 /* event->count up-to-date */
230		−#define PERF_HES_ARCH 0x04
	222	+
	223	+/* the counter is stopped */
	224	+#define PERF_HES_STOPPED 0x01
	225	+
	226	+/* event->count up-to-date */
	227	+#define PERF_HES_UPTODATE 0x02
	228	+
	229	+#define PERF_HES_ARCH 0x04
231	230
232	231	int state;
233	232
@@ -276,7 +275,7 @@ struct hw_perf_event { · 42 unmodified lines
276	275	*/
277	276	u64 freq_time_stamp;
278	277	u64 freq_count_stamp;
279		−#endif
	278	+#endif /* CONFIG_PERF_EVENTS */
280	279	};
281	280
282	281	struct perf_event;
@@ -285,28 +284,33 @@ struct perf_event_pmu_context; · 2 unmodified lines
285	284	/*
286	285	* Common implementation detail of pmu::{start,commit,cancel}_txn
287	286	*/
288		−#define PERF_PMU_TXN_ADD 0x1 /* txn to add/schedule event on PMU */
289		−#define PERF_PMU_TXN_READ 0x2 /* txn to read event group from PMU */
	287	+
	288	+/* txn to add/schedule event on PMU */
	289	+#define PERF_PMU_TXN_ADD 0x1
	290	+
	291	+/* txn to read event group from PMU */
	292	+#define PERF_PMU_TXN_READ 0x2
290	293
291	294	/**
292	295	* pmu::capabilities flags
293	296	*/
294		−#define PERF_PMU_CAP_NO_INTERRUPT 0x0001
295		−#define PERF_PMU_CAP_NO_NMI 0x0002
296		−#define PERF_PMU_CAP_AUX_NO_SG 0x0004
297		−#define PERF_PMU_CAP_EXTENDED_REGS 0x0008
298		−#define PERF_PMU_CAP_EXCLUSIVE 0x0010
299		−#define PERF_PMU_CAP_ITRACE 0x0020
300		−#define PERF_PMU_CAP_NO_EXCLUDE 0x0040
301		−#define PERF_PMU_CAP_AUX_OUTPUT 0x0080
302		−#define PERF_PMU_CAP_EXTENDED_HW_TYPE 0x0100
303		−#define PERF_PMU_CAP_AUX_PAUSE 0x0200
	297	+#define PERF_PMU_CAP_NO_INTERRUPT 0x0001
	298	+#define PERF_PMU_CAP_NO_NMI 0x0002
	299	+#define PERF_PMU_CAP_AUX_NO_SG 0x0004
	300	+#define PERF_PMU_CAP_EXTENDED_REGS 0x0008
	301	+#define PERF_PMU_CAP_EXCLUSIVE 0x0010
	302	+#define PERF_PMU_CAP_ITRACE 0x0020
	303	+#define PERF_PMU_CAP_NO_EXCLUDE 0x0040
	304	+#define PERF_PMU_CAP_AUX_OUTPUT 0x0080
	305	+#define PERF_PMU_CAP_EXTENDED_HW_TYPE 0x0100
	306	+#define PERF_PMU_CAP_AUX_PAUSE 0x0200
	307	+#define PERF_PMU_CAP_AUX_PREFER_LARGE 0x0400
304	308
305	309	/**
306	310	* pmu::scope
307	311	*/
308	312	enum perf_pmu_scope {
309		− PERF_PMU_SCOPE_NONE = 0,
	313	+ PERF_PMU_SCOPE_NONE = 0,
310	314	PERF_PMU_SCOPE_CORE,
311	315	PERF_PMU_SCOPE_DIE,
312	316	PERF_PMU_SCOPE_CLUSTER,
@@ -325,6 +329,9 @@ struct perf_output_handle; · 12 unmodified lines
325	329	struct pmu {
326	330	struct list_head entry;
327	331
	332	+ spinlock_t events_lock;
	333	+ struct list_head events;
	334	+
328	335	struct module *module;
329	336	struct device *dev;
330	337	struct device *parent;
@@ -387,11 +394,21 @@ struct pmu { · 56 unmodified lines
387	394	* Flags for ->add()/->del()/ ->start()/->stop(). There are
388	395	* matching hw_perf_event::state flags.
389	396	*/
390		−#define PERF_EF_START 0x01 /* start the counter when adding */
391		−#define PERF_EF_RELOAD 0x02 /* reload the counter when starting */
392		−#define PERF_EF_UPDATE 0x04 /* update the counter when stopping */
393		−#define PERF_EF_PAUSE 0x08 /* AUX area event, pause tracing */
394		−#define PERF_EF_RESUME 0x10 /* AUX area event, resume tracing */
	397	+
	398	+/* start the counter when adding */
	399	+#define PERF_EF_START 0x01
	400	+
	401	+/* reload the counter when starting */
	402	+#define PERF_EF_RELOAD 0x02
	403	+
	404	+/* update the counter when stopping */
	405	+#define PERF_EF_UPDATE 0x04
	406	+
	407	+/* AUX area event, pause tracing */
	408	+#define PERF_EF_PAUSE 0x08
	409	+
	410	+/* AUX area event, resume tracing */
	411	+#define PERF_EF_RESUME 0x10
395	412
396	413	/*
397	414	* Adds/Removes a counter to/from the PMU, can be done inside a
@@ -590,10 +607,10 @@ enum perf_addr_filter_action_t { · 192 unmodified lines
590	607	* This is a hardware-agnostic filter configuration as specified by the user.
591	608	*/
592	609	struct perf_addr_filter {
593		− struct list_head entry;
594		− struct path path;
595		− unsigned long offset;
596		− unsigned long size;
	610	+ struct list_head entry;
	611	+ struct path path;
	612	+ unsigned long offset;
	613	+ unsigned long size;
597	614	enum perf_addr_filter_action_t action;
598	615	};
599	616
@@ -608,23 +625,24 @@ struct perf_addr_filter { · 8 unmodified lines
608	625	* bundled together; see perf_event_addr_filters().
609	626	*/
610	627	struct perf_addr_filters_head {
611		− struct list_head list;
612		− raw_spinlock_t lock;
613		− unsigned int nr_file_filters;
	628	+ struct list_head list;
	629	+ raw_spinlock_t lock;
	630	+ unsigned int nr_file_filters;
614	631	};
615	632
616	633	struct perf_addr_filter_range {
617		− unsigned long start;
618		− unsigned long size;
	634	+ unsigned long start;
	635	+ unsigned long size;
619	636	};
620	637
621	638	/**
622	639	* enum perf_event_state - the states of an event:
623	640	*/
624	641	enum perf_event_state {
625		− PERF_EVENT_STATE_DEAD = -4,
626		− PERF_EVENT_STATE_EXIT = -3,
627		− PERF_EVENT_STATE_ERROR = -2,
	642	+ PERF_EVENT_STATE_DEAD = -5,
	643	+ PERF_EVENT_STATE_REVOKED = -4, /* pmu gone, must not touch */
	644	+ PERF_EVENT_STATE_EXIT = -3, /* task died, still inherit */
	645	+ PERF_EVENT_STATE_ERROR = -2, /* scheduling error, can enable */
628	646	PERF_EVENT_STATE_OFF = -1,
629	647	PERF_EVENT_STATE_INACTIVE = 0,
630	648	PERF_EVENT_STATE_ACTIVE = 1,
@@ -662,24 +680,24 @@ struct swevent_hlist { · 31 unmodified lines
662	680	struct rcu_head rcu_head;
663	681	};
664	682
665		−#define PERF_ATTACH_CONTEXT 0x0001
666		−#define PERF_ATTACH_GROUP 0x0002
667		−#define PERF_ATTACH_TASK 0x0004
668		−#define PERF_ATTACH_TASK_DATA 0x0008
669		−#define PERF_ATTACH_GLOBAL_DATA 0x0010
670		−#define PERF_ATTACH_SCHED_CB 0x0020
671		−#define PERF_ATTACH_CHILD 0x0040
672		−#define PERF_ATTACH_EXCLUSIVE 0x0080
673		−#define PERF_ATTACH_CALLCHAIN 0x0100
674		−#define PERF_ATTACH_ITRACE 0x0200
	683	+#define PERF_ATTACH_CONTEXT 0x0001
	684	+#define PERF_ATTACH_GROUP 0x0002
	685	+#define PERF_ATTACH_TASK 0x0004
	686	+#define PERF_ATTACH_TASK_DATA 0x0008
	687	+#define PERF_ATTACH_GLOBAL_DATA 0x0010
	688	+#define PERF_ATTACH_SCHED_CB 0x0020
	689	+#define PERF_ATTACH_CHILD 0x0040
	690	+#define PERF_ATTACH_EXCLUSIVE 0x0080
	691	+#define PERF_ATTACH_CALLCHAIN 0x0100
	692	+#define PERF_ATTACH_ITRACE 0x0200
675	693
676	694	struct bpf_prog;
677	695	struct perf_cgroup;
678	696	struct perf_buffer;
679	697
680	698	struct pmu_event_list {
681		− raw_spinlock_t lock;
682		− struct list_head list;
	699	+ raw_spinlock_t lock;
	700	+ struct list_head list;
683	701	};
684	702
685	703	/*
@@ -689,12 +707,12 @@ struct pmu_event_list { · 3 unmodified lines
689	707	* disabled is sufficient since it will hold-off the IPIs.
690	708	*/
691	709	#ifdef CONFIG_PROVE_LOCKING
692		−#define lockdep_assert_event_ctx(event) \
	710	+# define lockdep_assert_event_ctx(event) \
693	711	WARN_ON_ONCE(__lockdep_enabled && \
694	712	(this_cpu_read(hardirqs_enabled) && \
695	713	lockdep_is_held(&(event)->ctx->mutex) != LOCK_STATE_HELD))
696	714	#else
697		−#define lockdep_assert_event_ctx(event)
	715	+# define lockdep_assert_event_ctx(event)
698	716	#endif
699	717
700	718	#define for_each_sibling_event(sibling, event) \
@@ -852,9 +870,9 @@ struct perf_event { · 151 unmodified lines
852	870	#ifdef CONFIG_EVENT_TRACING
853	871	struct trace_event_call *tp_event;
854	872	struct event_filter *filter;
855		−#ifdef CONFIG_FUNCTION_TRACER
	873	+# ifdef CONFIG_FUNCTION_TRACER
856	874	struct ftrace_ops ftrace_ops;
857		−#endif
	875	+# endif
858	876	#endif
859	877
860	878	#ifdef CONFIG_CGROUP_PERF
@@ -865,6 +883,7 @@ struct perf_event { · 4 unmodified lines
865	883	void *security;
866	884	#endif
867	885	struct list_head sb_list;
	886	+ struct list_head pmu_list;
868	887
869	888	/*
870	889	* Certain events gets forwarded to another pmu internally by over-
@@ -872,7 +891,7 @@ struct perf_event { · 1 unmodified line
872	891	* of it. event->orig_type contains original 'type' requested by
873	892	* user.
874	893	*/
875		− __u32 orig_type;
	894	+ u32 orig_type;
876	895	#endif /* CONFIG_PERF_EVENTS */
877	896	};
878	897
@@ -937,8 +956,8 @@ static inline bool perf_pmu_ctx_is_active(struct perf_event_pmu_context *epc) · 58 unmodified lines
937	956	}
938	957
939	958	struct perf_event_groups {
940		− struct rb_root tree;
941		− u64 index;
	959	+ struct rb_root tree;
	960	+ u64 index;
942	961	};
943	962
944	963
@@ -1155,7 +1174,7 @@ extern void perf_aux_output_flag(struct perf_output_handle *handle, u64 flags); · 210 unmodified lines
1155	1174	extern void perf_event_itrace_started(struct perf_event *event);
1156	1175
1157	1176	extern int perf_pmu_register(struct pmu pmu, const char name, int type);
1158		−extern void perf_pmu_unregister(struct pmu *pmu);
	1177	+extern int perf_pmu_unregister(struct pmu *pmu);
1159	1178
1160	1179	extern void __perf_event_task_sched_in(struct task_struct *prev,
1161	1180	struct task_struct *task);
@@ -1181,16 +1200,18 @@ extern void perf_pmu_resched(struct pmu *pmu); · 19 unmodified lines
1181	1200	extern int perf_event_refresh(struct perf_event *event, int refresh);
1182	1201	extern void perf_event_update_userpage(struct perf_event *event);
1183	1202	extern int perf_event_release_kernel(struct perf_event *event);
	1203	+
1184	1204	extern struct perf_event *
1185	1205	perf_event_create_kernel_counter(struct perf_event_attr *attr,
1186		− int cpu,
1187		− struct task_struct *task,
1188		− perf_overflow_handler_t callback,
1189		− void *context);
	1206	+ int cpu,
	1207	+ struct task_struct *task,
	1208	+ perf_overflow_handler_t callback,
	1209	+ void *context);
	1210	+
1190	1211	extern void perf_pmu_migrate_context(struct pmu *pmu,
1191		− int src_cpu, int dst_cpu);
1192		−int perf_event_read_local(struct perf_event event, u64 value,
1193		− u64 enabled, u64 running);
	1212	+ int src_cpu, int dst_cpu);
	1213	+extern int perf_event_read_local(struct perf_event event, u64 value,
	1214	+ u64 enabled, u64 running);
1194	1215	extern u64 perf_event_read_value(struct perf_event *event,
1195	1216	u64 enabled, u64 running);
1196	1217
@@ -1407,14 +1428,14 @@ static inline u32 perf_sample_data_size(struct perf_sample_data *data, · 210 unmodified lines
1407	1428	*/
1408	1429	static inline void perf_clear_branch_entry_bitfields(struct perf_branch_entry *br)
1409	1430	{
1410		− br->mispred = 0;
1411		− br->predicted = 0;
1412		− br->in_tx = 0;
1413		− br->abort = 0;
1414		− br->cycles = 0;
1415		− br->type = 0;
1416		− br->spec = PERF_BR_SPEC_NA;
1417		− br->reserved = 0;
	1431	+ br->mispred = 0;
	1432	+ br->predicted = 0;
	1433	+ br->in_tx = 0;
	1434	+ br->abort = 0;
	1435	+ br->cycles = 0;
	1436	+ br->type = 0;
	1437	+ br->spec = PERF_BR_SPEC_NA;
	1438	+ br->reserved = 0;
1418	1439	}
1419	1440
1420	1441	extern void perf_output_sample(struct perf_output_handle *handle,
@@ -1603,7 +1624,17 @@ extern void perf_event_bpf_event(struct bpf_prog *prog, · 182 unmodified lines
1603	1624	enum perf_bpf_event_type type,
1604	1625	u16 flags);
1605	1626
	1627	+#define PERF_GUEST_ACTIVE 0x01
	1628	+#define PERF_GUEST_USER 0x02
	1629	+
	1630	+struct perf_guest_info_callbacks {
	1631	+ unsigned int (*state)(void);
	1632	+ unsigned long (*get_ip)(void);
	1633	+ unsigned int (*handle_intel_pt_intr)(void);
	1634	+};
	1635	+
1606	1636	#ifdef CONFIG_GUEST_PERF_EVENTS
	1637	+
1607	1638	extern struct perf_guest_info_callbacks __rcu *perf_guest_cbs;
1608	1639
1609	1640	DECLARE_STATIC_CALL(__perf_guest_state, *perf_guest_cbs->state);
@@ -1614,21 +1645,27 @@ static inline unsigned int perf_guest_state(void) · 4 unmodified lines
1614	1645	{
1615	1646	return static_call(__perf_guest_state)();
1616	1647	}
	1648	+
1617	1649	static inline unsigned long perf_guest_get_ip(void)
1618	1650	{
1619	1651	return static_call(__perf_guest_get_ip)();
1620	1652	}
	1653	+
1621	1654	static inline unsigned int perf_guest_handle_intel_pt_intr(void)
1622	1655	{
1623	1656	return static_call(__perf_guest_handle_intel_pt_intr)();
1624	1657	}
	1658	+
1625	1659	extern void perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs);
1626	1660	extern void perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs);
1627		−#else
	1661	+
	1662	+#else /* !CONFIG_GUEST_PERF_EVENTS: */
	1663	+
1628	1664	static inline unsigned int perf_guest_state(void) { return 0; }
1629	1665	static inline unsigned long perf_guest_get_ip(void) { return 0; }
1630	1666	static inline unsigned int perf_guest_handle_intel_pt_intr(void) { return 0; }
1631		−#endif /* CONFIG_GUEST_PERF_EVENTS */
	1667	+
	1668	+#endif /* !CONFIG_GUEST_PERF_EVENTS */
1632	1669
1633	1670	extern void perf_event_exec(void);
1634	1671	extern void perf_event_comm(struct task_struct *tsk, bool exec);
@@ -1658,6 +1695,7 @@ static inline int perf_callchain_store_context(struct perf_callchain_entry_ctx * · 23 unmodified lines
1658	1695	{
1659	1696	if (ctx->contexts < sysctl_perf_event_max_contexts_per_stack) {
1660	1697	struct perf_callchain_entry *entry = ctx->entry;
	1698	+
1661	1699	entry->ip[entry->nr++] = ip;
1662	1700	++ctx->contexts;
1663	1701	return 0;
@@ -1671,6 +1709,7 @@ static inline int perf_callchain_store(struct perf_callchain_entry_ctx *ctx, u64 · 7 unmodified lines
1671	1709	{
1672	1710	if (ctx->nr < ctx->max_stack && !ctx->contexts_maxed) {
1673	1711	struct perf_callchain_entry *entry = ctx->entry;
	1712	+
1674	1713	entry->ip[entry->nr++] = ip;
1675	1714	++ctx->nr;
1676	1715	return 0;
@@ -1697,7 +1736,7 @@ static inline int perf_is_paranoid(void) · 20 unmodified lines
1697	1736	return sysctl_perf_event_paranoid > -1;
1698	1737	}
1699	1738
1700		−int perf_allow_kernel(void);
	1739	+extern int perf_allow_kernel(void);
1701	1740
1702	1741	static inline int perf_allow_cpu(void)
1703	1742	{
@@ -1760,7 +1799,7 @@ static inline bool needs_branch_stack(struct perf_event *event) · 56 unmodified lines
1760	1799
1761	1800	static inline bool has_aux(struct perf_event *event)
1762	1801	{
1763		− return event->pmu->setup_aux;
	1802	+ return event->pmu && event->pmu->setup_aux;
1764	1803	}
1765	1804
1766	1805	static inline bool has_aux_action(struct perf_event *event)
@@ -1819,7 +1858,7 @@ extern int perf_output_begin_backward(struct perf_output_handle *handle, · 52 unmodified lines
1819	1858
1820	1859	extern void perf_output_end(struct perf_output_handle *handle);
1821	1860	extern unsigned int perf_output_copy(struct perf_output_handle *handle,
1822		− const void *buf, unsigned int len);
	1861	+ const void *buf, unsigned int len);
1823	1862	extern unsigned int perf_output_skip(struct perf_output_handle *handle,
1824	1863	unsigned int len);
1825	1864	extern long perf_output_copy_aux(struct perf_output_handle *aux_handle,
@@ -1836,7 +1875,9 @@ extern void perf_event_task_tick(void); · 10 unmodified lines
1836	1875	extern int perf_event_account_interrupt(struct perf_event *event);
1837	1876	extern int perf_event_period(struct perf_event *event, u64 value);
1838	1877	extern u64 perf_event_pause(struct perf_event *event, bool reset);
	1878	+
1839	1879	#else /* !CONFIG_PERF_EVENTS: */
	1880	+
1840	1881	static inline void *
1841	1882	perf_aux_output_begin(struct perf_output_handle *handle,
1842	1883	struct perf_event *event) { return NULL; }
@@ -1914,19 +1955,14 @@ static inline void perf_event_disable(struct perf_event *event) { } · 71 unmodified lines
1914	1955	static inline int __perf_event_disable(void *info) { return -1; }
1915	1956	static inline void perf_event_task_tick(void) { }
1916	1957	static inline int perf_event_release_kernel(struct perf_event *event) { return 0; }
1917		−static inline int perf_event_period(struct perf_event *event, u64 value)
1918		−{
1919		− return -EINVAL;
1920		−}
1921		−static inline u64 perf_event_pause(struct perf_event *event, bool reset)
1922		−{
1923		− return 0;
1924		−}
1925		−static inline int perf_exclude_event(struct perf_event event, struct pt_regs regs)
1926		−{
1927		− return 0;
1928		−}
1929		−#endif
	1958	+static inline int
	1959	+perf_event_period(struct perf_event *event, u64 value) { return -EINVAL; }
	1960	+static inline u64
	1961	+perf_event_pause(struct perf_event *event, bool reset) { return 0; }
	1962	+static inline int
	1963	+perf_exclude_event(struct perf_event event, struct pt_regs regs) { return 0; }
	1964	+
	1965	+#endif /* !CONFIG_PERF_EVENTS */
1930	1966
1931	1967	#if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_INTEL)
1932	1968	extern void perf_restore_debug_store(void);
@@ -1934,31 +1970,31 @@ extern void perf_restore_debug_store(void); · 1 unmodified line
1934	1970	static inline void perf_restore_debug_store(void) { }
1935	1971	#endif
1936	1972
1937		−#define perf_output_put(handle, x) perf_output_copy((handle), &(x), sizeof(x))
	1973	+#define perf_output_put(handle, x) perf_output_copy((handle), &(x), sizeof(x))
1938	1974
1939	1975	struct perf_pmu_events_attr {
1940		− struct device_attribute attr;
1941		− u64 id;
1942		− const char *event_str;
	1976	+ struct device_attribute attr;
	1977	+ u64 id;
	1978	+ const char *event_str;
1943	1979	};
1944	1980
1945	1981	struct perf_pmu_events_ht_attr {
1946		− struct device_attribute attr;
1947		− u64 id;
1948		− const char *event_str_ht;
1949		− const char *event_str_noht;
	1982	+ struct device_attribute attr;
	1983	+ u64 id;
	1984	+ const char *event_str_ht;
	1985	+ const char *event_str_noht;
1950	1986	};
1951	1987
1952	1988	struct perf_pmu_events_hybrid_attr {
1953		− struct device_attribute attr;
1954		− u64 id;
1955		− const char *event_str;
1956		− u64 pmu_type;
	1989	+ struct device_attribute attr;
	1990	+ u64 id;
	1991	+ const char *event_str;
	1992	+ u64 pmu_type;
1957	1993	};
1958	1994
1959	1995	struct perf_pmu_format_hybrid_attr {
1960		− struct device_attribute attr;
1961		− u64 pmu_type;
	1996	+ struct device_attribute attr;
	1997	+ u64 pmu_type;
1962	1998	};
1963	1999
1964	2000	ssize_t perf_event_sysfs_show(struct device dev, struct device_attribute attr,
@@ -2000,11 +2036,11 @@ static struct device_attribute format_attr_##_name = __ATTR_RO(_name) · 35 unmodified lines
2000	2036
2001	2037	/* Performance counter hotplug functions */
2002	2038	#ifdef CONFIG_PERF_EVENTS
2003		−int perf_event_init_cpu(unsigned int cpu);
2004		−int perf_event_exit_cpu(unsigned int cpu);
	2039	+extern int perf_event_init_cpu(unsigned int cpu);
	2040	+extern int perf_event_exit_cpu(unsigned int cpu);
2005	2041	#else
2006		−#define perf_event_init_cpu NULL
2007		−#define perf_event_exit_cpu NULL
	2042	+# define perf_event_init_cpu NULL
	2043	+# define perf_event_exit_cpu NULL
2008	2044	#endif
2009	2045
2010	2046	extern void arch_perf_update_userpage(struct perf_event *event,

include/uapi/linux/perf_event.h+334−323

@@ -39,18 +39,21 @@ enum perf_type_id { · 38 unmodified lines
39	39
40	40	/*
41	41	* attr.config layout for type PERF_TYPE_HARDWARE and PERF_TYPE_HW_CACHE
	42	+ *
42	43	* PERF_TYPE_HARDWARE: 0xEEEEEEEE000000AA
43	44	* AA: hardware event ID
44	45	* EEEEEEEE: PMU type ID
	46	+ *
45	47	* PERF_TYPE_HW_CACHE: 0xEEEEEEEE00DDCCBB
46	48	* BB: hardware cache ID
47	49	* CC: hardware cache op ID
48	50	* DD: hardware cache op result ID
49	51	* EEEEEEEE: PMU type ID
50		− * If the PMU type ID is 0, the PERF_TYPE_RAW will be applied.
	52	+ *
	53	+ * If the PMU type ID is 0, PERF_TYPE_RAW will be applied.
51	54	*/
52		−#define PERF_PMU_TYPE_SHIFT 32
53		−#define PERF_HW_EVENT_MASK 0xffffffff
	55	+#define PERF_PMU_TYPE_SHIFT 32
	56	+#define PERF_HW_EVENT_MASK 0xffffffff
54	57
55	58	/*
56	59	* Generalized performance event event_id types, used by the
@@ -112,7 +115,7 @@ enum perf_hw_cache_op_result_id { · 55 unmodified lines
112	115	/*
113	116	* Special "software" events provided by the kernel, even if the hardware
114	117	* does not support performance events. These events measure various
115		− * physical and sw events of the kernel (and allow the profiling of them as
	118	+ * physical and SW events of the kernel (and allow the profiling of them as
116	119	* well):
117	120	*/
118	121	enum perf_sw_ids {
@@ -167,8 +170,9 @@ enum perf_event_sample_format { · 48 unmodified lines
167	170	};
168	171
169	172	#define PERF_SAMPLE_WEIGHT_TYPE (PERF_SAMPLE_WEIGHT \| PERF_SAMPLE_WEIGHT_STRUCT)
	173	+
170	174	/*
171		− * values to program into branch_sample_type when PERF_SAMPLE_BRANCH is set
	175	+ * Values to program into branch_sample_type when PERF_SAMPLE_BRANCH is set.
172	176	*
173	177	* If the user does not pass priv level information via branch_sample_type,
174	178	* the kernel uses the event's priv level. Branch and event priv levels do
@@ -178,20 +182,20 @@ enum perf_event_sample_format { · 3 unmodified lines
178	182	* of branches and therefore it supersedes all the other types.
179	183	*/
180	184	enum perf_branch_sample_type_shift {
181		− PERF_SAMPLE_BRANCH_USER_SHIFT = 0, /* user branches */
182		− PERF_SAMPLE_BRANCH_KERNEL_SHIFT = 1, /* kernel branches */
183		− PERF_SAMPLE_BRANCH_HV_SHIFT = 2, /* hypervisor branches */
184		−
185		− PERF_SAMPLE_BRANCH_ANY_SHIFT = 3, /* any branch types */
186		− PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT = 4, /* any call branch */
187		− PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT = 5, /* any return branch */
188		− PERF_SAMPLE_BRANCH_IND_CALL_SHIFT = 6, /* indirect calls */
189		− PERF_SAMPLE_BRANCH_ABORT_TX_SHIFT = 7, /* transaction aborts */
190		− PERF_SAMPLE_BRANCH_IN_TX_SHIFT = 8, /* in transaction */
191		− PERF_SAMPLE_BRANCH_NO_TX_SHIFT = 9, /* not in transaction */
	185	+ PERF_SAMPLE_BRANCH_USER_SHIFT = 0, /* user branches */
	186	+ PERF_SAMPLE_BRANCH_KERNEL_SHIFT = 1, /* kernel branches */
	187	+ PERF_SAMPLE_BRANCH_HV_SHIFT = 2, /* hypervisor branches */
	188	+
	189	+ PERF_SAMPLE_BRANCH_ANY_SHIFT = 3, /* any branch types */
	190	+ PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT = 4, /* any call branch */
	191	+ PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT = 5, /* any return branch */
	192	+ PERF_SAMPLE_BRANCH_IND_CALL_SHIFT = 6, /* indirect calls */
	193	+ PERF_SAMPLE_BRANCH_ABORT_TX_SHIFT = 7, /* transaction aborts */
	194	+ PERF_SAMPLE_BRANCH_IN_TX_SHIFT = 8, /* in transaction */
	195	+ PERF_SAMPLE_BRANCH_NO_TX_SHIFT = 9, /* not in transaction */
192	196	PERF_SAMPLE_BRANCH_COND_SHIFT = 10, /* conditional branches */
193	197
194		− PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT = 11, /* call/ret stack */
	198	+ PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT = 11, /* CALL/RET stack */
195	199	PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT = 12, /* indirect jumps */
196	200	PERF_SAMPLE_BRANCH_CALL_SHIFT = 13, /* direct call */
197	201
@@ -210,96 +214,95 @@ enum perf_branch_sample_type_shift { · 12 unmodified lines
210	214	};
211	215
212	216	enum perf_branch_sample_type {
213		− PERF_SAMPLE_BRANCH_USER = 1U << PERF_SAMPLE_BRANCH_USER_SHIFT,
214		− PERF_SAMPLE_BRANCH_KERNEL = 1U << PERF_SAMPLE_BRANCH_KERNEL_SHIFT,
215		− PERF_SAMPLE_BRANCH_HV = 1U << PERF_SAMPLE_BRANCH_HV_SHIFT,
	217	+ PERF_SAMPLE_BRANCH_USER = 1U << PERF_SAMPLE_BRANCH_USER_SHIFT,
	218	+ PERF_SAMPLE_BRANCH_KERNEL = 1U << PERF_SAMPLE_BRANCH_KERNEL_SHIFT,
	219	+ PERF_SAMPLE_BRANCH_HV = 1U << PERF_SAMPLE_BRANCH_HV_SHIFT,
216	220
217		− PERF_SAMPLE_BRANCH_ANY = 1U << PERF_SAMPLE_BRANCH_ANY_SHIFT,
218		− PERF_SAMPLE_BRANCH_ANY_CALL = 1U << PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT,
219		− PERF_SAMPLE_BRANCH_ANY_RETURN = 1U << PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT,
220		− PERF_SAMPLE_BRANCH_IND_CALL = 1U << PERF_SAMPLE_BRANCH_IND_CALL_SHIFT,
221		− PERF_SAMPLE_BRANCH_ABORT_TX = 1U << PERF_SAMPLE_BRANCH_ABORT_TX_SHIFT,
222		− PERF_SAMPLE_BRANCH_IN_TX = 1U << PERF_SAMPLE_BRANCH_IN_TX_SHIFT,
223		− PERF_SAMPLE_BRANCH_NO_TX = 1U << PERF_SAMPLE_BRANCH_NO_TX_SHIFT,
224		− PERF_SAMPLE_BRANCH_COND = 1U << PERF_SAMPLE_BRANCH_COND_SHIFT,
	221	+ PERF_SAMPLE_BRANCH_ANY = 1U << PERF_SAMPLE_BRANCH_ANY_SHIFT,
	222	+ PERF_SAMPLE_BRANCH_ANY_CALL = 1U << PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT,
	223	+ PERF_SAMPLE_BRANCH_ANY_RETURN = 1U << PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT,
	224	+ PERF_SAMPLE_BRANCH_IND_CALL = 1U << PERF_SAMPLE_BRANCH_IND_CALL_SHIFT,
	225	+ PERF_SAMPLE_BRANCH_ABORT_TX = 1U << PERF_SAMPLE_BRANCH_ABORT_TX_SHIFT,
	226	+ PERF_SAMPLE_BRANCH_IN_TX = 1U << PERF_SAMPLE_BRANCH_IN_TX_SHIFT,
	227	+ PERF_SAMPLE_BRANCH_NO_TX = 1U << PERF_SAMPLE_BRANCH_NO_TX_SHIFT,
	228	+ PERF_SAMPLE_BRANCH_COND = 1U << PERF_SAMPLE_BRANCH_COND_SHIFT,
225	229
226		− PERF_SAMPLE_BRANCH_CALL_STACK = 1U << PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT,
227		− PERF_SAMPLE_BRANCH_IND_JUMP = 1U << PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT,
228		− PERF_SAMPLE_BRANCH_CALL = 1U << PERF_SAMPLE_BRANCH_CALL_SHIFT,
	230	+ PERF_SAMPLE_BRANCH_CALL_STACK = 1U << PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT,
	231	+ PERF_SAMPLE_BRANCH_IND_JUMP = 1U << PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT,
	232	+ PERF_SAMPLE_BRANCH_CALL = 1U << PERF_SAMPLE_BRANCH_CALL_SHIFT,
229	233
230		− PERF_SAMPLE_BRANCH_NO_FLAGS = 1U << PERF_SAMPLE_BRANCH_NO_FLAGS_SHIFT,
231		− PERF_SAMPLE_BRANCH_NO_CYCLES = 1U << PERF_SAMPLE_BRANCH_NO_CYCLES_SHIFT,
	234	+ PERF_SAMPLE_BRANCH_NO_FLAGS = 1U << PERF_SAMPLE_BRANCH_NO_FLAGS_SHIFT,
	235	+ PERF_SAMPLE_BRANCH_NO_CYCLES = 1U << PERF_SAMPLE_BRANCH_NO_CYCLES_SHIFT,
232	236
233		− PERF_SAMPLE_BRANCH_TYPE_SAVE =
234		− 1U << PERF_SAMPLE_BRANCH_TYPE_SAVE_SHIFT,
	237	+ PERF_SAMPLE_BRANCH_TYPE_SAVE = 1U << PERF_SAMPLE_BRANCH_TYPE_SAVE_SHIFT,
235	238
236		− PERF_SAMPLE_BRANCH_HW_INDEX = 1U << PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT,
	239	+ PERF_SAMPLE_BRANCH_HW_INDEX = 1U << PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT,
237	240
238		− PERF_SAMPLE_BRANCH_PRIV_SAVE = 1U << PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT,
	241	+ PERF_SAMPLE_BRANCH_PRIV_SAVE = 1U << PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT,
239	242
240		− PERF_SAMPLE_BRANCH_COUNTERS = 1U << PERF_SAMPLE_BRANCH_COUNTERS_SHIFT,
	243	+ PERF_SAMPLE_BRANCH_COUNTERS = 1U << PERF_SAMPLE_BRANCH_COUNTERS_SHIFT,
241	244
242		− PERF_SAMPLE_BRANCH_MAX = 1U << PERF_SAMPLE_BRANCH_MAX_SHIFT,
	245	+ PERF_SAMPLE_BRANCH_MAX = 1U << PERF_SAMPLE_BRANCH_MAX_SHIFT,
243	246	};
244	247
245	248	/*
246		− * Common flow change classification
	249	+ * Common control flow change classifications:
247	250	*/
248	251	enum {
249		− PERF_BR_UNKNOWN = 0, /* unknown */
250		− PERF_BR_COND = 1, /* conditional */
251		− PERF_BR_UNCOND = 2, /* unconditional */
252		− PERF_BR_IND = 3, /* indirect */
253		− PERF_BR_CALL = 4, /* function call */
254		− PERF_BR_IND_CALL = 5, /* indirect function call */
255		− PERF_BR_RET = 6, /* function return */
256		− PERF_BR_SYSCALL = 7, /* syscall */
257		− PERF_BR_SYSRET = 8, /* syscall return */
258		− PERF_BR_COND_CALL = 9, /* conditional function call */
259		− PERF_BR_COND_RET = 10, /* conditional function return */
260		− PERF_BR_ERET = 11, /* exception return */
261		− PERF_BR_IRQ = 12, /* irq */
262		− PERF_BR_SERROR = 13, /* system error */
263		− PERF_BR_NO_TX = 14, /* not in transaction */
264		− PERF_BR_EXTEND_ABI = 15, /* extend ABI */
	252	+ PERF_BR_UNKNOWN = 0, /* Unknown */
	253	+ PERF_BR_COND = 1, /* Conditional */
	254	+ PERF_BR_UNCOND = 2, /* Unconditional */
	255	+ PERF_BR_IND = 3, /* Indirect */
	256	+ PERF_BR_CALL = 4, /* Function call */
	257	+ PERF_BR_IND_CALL = 5, /* Indirect function call */
	258	+ PERF_BR_RET = 6, /* Function return */
	259	+ PERF_BR_SYSCALL = 7, /* Syscall */
	260	+ PERF_BR_SYSRET = 8, /* Syscall return */
	261	+ PERF_BR_COND_CALL = 9, /* Conditional function call */
	262	+ PERF_BR_COND_RET = 10, /* Conditional function return */
	263	+ PERF_BR_ERET = 11, /* Exception return */
	264	+ PERF_BR_IRQ = 12, /* IRQ */
	265	+ PERF_BR_SERROR = 13, /* System error */
	266	+ PERF_BR_NO_TX = 14, /* Not in transaction */
	267	+ PERF_BR_EXTEND_ABI = 15, /* Extend ABI */
265	268	PERF_BR_MAX,
266	269	};
267	270
268	271	/*
269		− * Common branch speculation outcome classification
	272	+ * Common branch speculation outcome classifications:
270	273	*/
271	274	enum {
272		− PERF_BR_SPEC_NA = 0, /* Not available */
273		− PERF_BR_SPEC_WRONG_PATH = 1, /* Speculative but on wrong path */
274		− PERF_BR_NON_SPEC_CORRECT_PATH = 2, /* Non-speculative but on correct path */
275		− PERF_BR_SPEC_CORRECT_PATH = 3, /* Speculative and on correct path */
	275	+ PERF_BR_SPEC_NA = 0, /* Not available */
	276	+ PERF_BR_SPEC_WRONG_PATH = 1, /* Speculative but on wrong path */
	277	+ PERF_BR_NON_SPEC_CORRECT_PATH = 2, /* Non-speculative but on correct path */
	278	+ PERF_BR_SPEC_CORRECT_PATH = 3, /* Speculative and on correct path */
276	279	PERF_BR_SPEC_MAX,
277	280	};
278	281
279	282	enum {
280		− PERF_BR_NEW_FAULT_ALGN = 0, /* Alignment fault */
281		− PERF_BR_NEW_FAULT_DATA = 1, /* Data fault */
282		− PERF_BR_NEW_FAULT_INST = 2, /* Inst fault */
283		− PERF_BR_NEW_ARCH_1 = 3, /* Architecture specific */
284		− PERF_BR_NEW_ARCH_2 = 4, /* Architecture specific */
285		− PERF_BR_NEW_ARCH_3 = 5, /* Architecture specific */
286		− PERF_BR_NEW_ARCH_4 = 6, /* Architecture specific */
287		− PERF_BR_NEW_ARCH_5 = 7, /* Architecture specific */
	283	+ PERF_BR_NEW_FAULT_ALGN = 0, /* Alignment fault */
	284	+ PERF_BR_NEW_FAULT_DATA = 1, /* Data fault */
	285	+ PERF_BR_NEW_FAULT_INST = 2, /* Inst fault */
	286	+ PERF_BR_NEW_ARCH_1 = 3, /* Architecture specific */
	287	+ PERF_BR_NEW_ARCH_2 = 4, /* Architecture specific */
	288	+ PERF_BR_NEW_ARCH_3 = 5, /* Architecture specific */
	289	+ PERF_BR_NEW_ARCH_4 = 6, /* Architecture specific */
	290	+ PERF_BR_NEW_ARCH_5 = 7, /* Architecture specific */
288	291	PERF_BR_NEW_MAX,
289	292	};
290	293
291	294	enum {
292		− PERF_BR_PRIV_UNKNOWN = 0,
293		− PERF_BR_PRIV_USER = 1,
294		− PERF_BR_PRIV_KERNEL = 2,
295		− PERF_BR_PRIV_HV = 3,
	295	+ PERF_BR_PRIV_UNKNOWN = 0,
	296	+ PERF_BR_PRIV_USER = 1,
	297	+ PERF_BR_PRIV_KERNEL = 2,
	298	+ PERF_BR_PRIV_HV = 3,
296	299	};
297	300
298		−#define PERF_BR_ARM64_FIQ PERF_BR_NEW_ARCH_1
299		−#define PERF_BR_ARM64_DEBUG_HALT PERF_BR_NEW_ARCH_2
300		−#define PERF_BR_ARM64_DEBUG_EXIT PERF_BR_NEW_ARCH_3
301		−#define PERF_BR_ARM64_DEBUG_INST PERF_BR_NEW_ARCH_4
302		−#define PERF_BR_ARM64_DEBUG_DATA PERF_BR_NEW_ARCH_5
	301	+#define PERF_BR_ARM64_FIQ PERF_BR_NEW_ARCH_1
	302	+#define PERF_BR_ARM64_DEBUG_HALT PERF_BR_NEW_ARCH_2
	303	+#define PERF_BR_ARM64_DEBUG_EXIT PERF_BR_NEW_ARCH_3
	304	+#define PERF_BR_ARM64_DEBUG_INST PERF_BR_NEW_ARCH_4
	305	+#define PERF_BR_ARM64_DEBUG_DATA PERF_BR_NEW_ARCH_5
303	306
304	307	#define PERF_SAMPLE_BRANCH_PLM_ALL \
305	308	(PERF_SAMPLE_BRANCH_USER\|\
@@ -310,9 +313,9 @@ enum { · 4 unmodified lines
310	313	* Values to determine ABI of the registers dump.
311	314	*/
312	315	enum perf_sample_regs_abi {
313		− PERF_SAMPLE_REGS_ABI_NONE = 0,
314		− PERF_SAMPLE_REGS_ABI_32 = 1,
315		− PERF_SAMPLE_REGS_ABI_64 = 2,
	316	+ PERF_SAMPLE_REGS_ABI_NONE = 0,
	317	+ PERF_SAMPLE_REGS_ABI_32 = 1,
	318	+ PERF_SAMPLE_REGS_ABI_64 = 2,
316	319	};
317	320
318	321	/*
@@ -320,21 +323,21 @@ enum perf_sample_regs_abi { · 1 unmodified line
320	323	* abort events. Multiple bits can be set.
321	324	*/
322	325	enum {
323		− PERF_TXN_ELISION = (1 << 0), /* From elision */
324		− PERF_TXN_TRANSACTION = (1 << 1), /* From transaction */
325		− PERF_TXN_SYNC = (1 << 2), /* Instruction is related */
326		− PERF_TXN_ASYNC = (1 << 3), /* Instruction not related */
327		− PERF_TXN_RETRY = (1 << 4), /* Retry possible */
328		− PERF_TXN_CONFLICT = (1 << 5), /* Conflict abort */
329		− PERF_TXN_CAPACITY_WRITE = (1 << 6), /* Capacity write abort */
330		− PERF_TXN_CAPACITY_READ = (1 << 7), /* Capacity read abort */
	326	+ PERF_TXN_ELISION = (1 << 0), /* From elision */
	327	+ PERF_TXN_TRANSACTION = (1 << 1), /* From transaction */
	328	+ PERF_TXN_SYNC = (1 << 2), /* Instruction is related */
	329	+ PERF_TXN_ASYNC = (1 << 3), /* Instruction is not related */
	330	+ PERF_TXN_RETRY = (1 << 4), /* Retry possible */
	331	+ PERF_TXN_CONFLICT = (1 << 5), /* Conflict abort */
	332	+ PERF_TXN_CAPACITY_WRITE = (1 << 6), /* Capacity write abort */
	333	+ PERF_TXN_CAPACITY_READ = (1 << 7), /* Capacity read abort */
331	334
332		− PERF_TXN_MAX = (1 << 8), /* non-ABI */
	335	+ PERF_TXN_MAX = (1 << 8), /* non-ABI */
333	336
334		− /* bits 32..63 are reserved for the abort code */
	337	+ /* Bits 32..63 are reserved for the abort code */
335	338
336		− PERF_TXN_ABORT_MASK = (0xffffffffULL << 32),
337		− PERF_TXN_ABORT_SHIFT = 32,
	339	+ PERF_TXN_ABORT_MASK = (0xffffffffULL << 32),
	340	+ PERF_TXN_ABORT_SHIFT = 32,
338	341	};
339	342
340	343	/*
@@ -369,24 +372,22 @@ enum perf_event_read_format { · 28 unmodified lines
369	372	PERF_FORMAT_MAX = 1U << 5, /* non-ABI */
370	373	};
371	374
372		−#define PERF_ATTR_SIZE_VER0 64 /* sizeof first published struct */
373		−#define PERF_ATTR_SIZE_VER1 72 /* add: config2 */
374		−#define PERF_ATTR_SIZE_VER2 80 /* add: branch_sample_type */
375		−#define PERF_ATTR_SIZE_VER3 96 /* add: sample_regs_user */
376		− /* add: sample_stack_user */
377		−#define PERF_ATTR_SIZE_VER4 104 /* add: sample_regs_intr */
378		−#define PERF_ATTR_SIZE_VER5 112 /* add: aux_watermark */
379		−#define PERF_ATTR_SIZE_VER6 120 /* add: aux_sample_size */
380		−#define PERF_ATTR_SIZE_VER7 128 /* add: sig_data */
381		−#define PERF_ATTR_SIZE_VER8 136 /* add: config3 */
	375	+#define PERF_ATTR_SIZE_VER0 64 /* Size of first published 'struct perf_event_attr' */
	376	+#define PERF_ATTR_SIZE_VER1 72 /* Add: config2 */
	377	+#define PERF_ATTR_SIZE_VER2 80 /* Add: branch_sample_type */
	378	+#define PERF_ATTR_SIZE_VER3 96 /* Add: sample_regs_user */
	379	+ /* Add: sample_stack_user */
	380	+#define PERF_ATTR_SIZE_VER4 104 /* Add: sample_regs_intr */
	381	+#define PERF_ATTR_SIZE_VER5 112 /* Add: aux_watermark */
	382	+#define PERF_ATTR_SIZE_VER6 120 /* Add: aux_sample_size */
	383	+#define PERF_ATTR_SIZE_VER7 128 /* Add: sig_data */
	384	+#define PERF_ATTR_SIZE_VER8 136 /* Add: config3 */
382	385
383	386	/*
384		− * Hardware event_id to monitor via a performance monitoring event:
385		− *
386		− * @sample_max_stack: Max number of frame pointers in a callchain,
387		− * should be < /proc/sys/kernel/perf_event_max_stack
388		− * Max number of entries of branch stack
389		− * should be < hardware limit
	387	+ * 'struct perf_event_attr' contains various attributes that define
	388	+ * a performance event - most of them hardware related configuration
	389	+ * details, but also a lot of behavioral switches and values implemented
	390	+ * by the kernel.
390	391	*/
391	392	struct perf_event_attr {
392	393
@@ -396,7 +397,7 @@ struct perf_event_attr { · 3 unmodified lines
396	397	__u32 type;
397	398
398	399	/*
399		− * Size of the attr structure, for fwd/bwd compat.
	400	+ * Size of the attr structure, for forward/backwards compatibility.
400	401	*/
401	402	__u32 size;
402	403
@@ -451,21 +452,21 @@ struct perf_event_attr { · 48 unmodified lines
451	452	comm_exec : 1, /* flag comm events that are due to an exec */
452	453	use_clockid : 1, /* use @clockid for time fields */
453	454	context_switch : 1, /* context switch data */
454		− write_backward : 1, /* Write ring buffer from end to beginning */
	455	+ write_backward : 1, /* write ring buffer from end to beginning */
455	456	namespaces : 1, /* include namespaces data */
456	457	ksymbol : 1, /* include ksymbol events */
457		− bpf_event : 1, /* include bpf events */
	458	+ bpf_event : 1, /* include BPF events */
458	459	aux_output : 1, /* generate AUX records instead of events */
459	460	cgroup : 1, /* include cgroup events */
460	461	text_poke : 1, /* include text poke events */
461		− build_id : 1, /* use build id in mmap2 events */
	462	+ build_id : 1, /* use build ID in mmap2 events */
462	463	inherit_thread : 1, /* children only inherit if cloned with CLONE_THREAD */
463	464	remove_on_exec : 1, /* event is removed from task on exec */
464	465	sigtrap : 1, /* send synchronous SIGTRAP on event */
465	466	__reserved_1 : 26;
466	467
467	468	union {
468		− __u32 wakeup_events; /* wakeup every n events */
	469	+ __u32 wakeup_events; /* wake up every n events */
469	470	__u32 wakeup_watermark; /* bytes before wakeup */
470	471	};
471	472
@@ -474,13 +475,13 @@ struct perf_event_attr { · 2 unmodified lines
474	475	__u64 bp_addr;
475	476	__u64 kprobe_func; /* for perf_kprobe */
476	477	__u64 uprobe_path; /* for perf_uprobe */
477		− __u64 config1; /* extension of config */
	478	+ __u64 config1; /* extension of config */
478	479	};
479	480	union {
480	481	__u64 bp_len;
481		− __u64 kprobe_addr; /* when kprobe_func == NULL */
	482	+ __u64 kprobe_addr; /* when kprobe_func == NULL */
482	483	__u64 probe_offset; /* for perf_[k,u]probe */
483		− __u64 config2; /* extension of config1 */
	484	+ __u64 config2; /* extension of config1 */
484	485	};
485	486	__u64 branch_sample_type; /* enum perf_branch_sample_type */
486	487
@@ -510,7 +511,16 @@ struct perf_event_attr { · 23 unmodified lines
510	511	* Wakeup watermark for AUX area
511	512	*/
512	513	__u32 aux_watermark;
	514	+
	515	+ /*
	516	+ * Max number of frame pointers in a callchain, should be
	517	+ * lower than /proc/sys/kernel/perf_event_max_stack.
	518	+ *
	519	+ * Max number of entries of branch stack should be lower
	520	+ * than the hardware limit.
	521	+ */
513	522	__u16 sample_max_stack;
	523	+
514	524	__u16 __reserved_2;
515	525	__u32 aux_sample_size;
516	526
@@ -537,7 +547,7 @@ struct perf_event_attr { · 20 unmodified lines
537	547
538	548	/*
539	549	* Structure used by below PERF_EVENT_IOC_QUERY_BPF command
540		− * to query bpf programs attached to the same perf tracepoint
	550	+ * to query BPF programs attached to the same perf tracepoint
541	551	* as the given perf event.
542	552	*/
543	553	struct perf_event_query_bpf {
@@ -559,21 +569,21 @@ struct perf_event_query_bpf { · 15 unmodified lines
559	569	/*
560	570	* Ioctls that can be done on a perf event fd:
561	571	*/
562		−#define PERF_EVENT_IOC_ENABLE _IO ('$', 0)
563		−#define PERF_EVENT_IOC_DISABLE _IO ('$', 1)
564		−#define PERF_EVENT_IOC_REFRESH _IO ('$', 2)
565		−#define PERF_EVENT_IOC_RESET _IO ('$', 3)
566		−#define PERF_EVENT_IOC_PERIOD _IOW('$', 4, __u64)
567		−#define PERF_EVENT_IOC_SET_OUTPUT _IO ('$', 5)
568		−#define PERF_EVENT_IOC_SET_FILTER _IOW('$', 6, char *)
569		−#define PERF_EVENT_IOC_ID _IOR('$', 7, __u64 *)
570		−#define PERF_EVENT_IOC_SET_BPF _IOW('$', 8, __u32)
571		−#define PERF_EVENT_IOC_PAUSE_OUTPUT _IOW('$', 9, __u32)
	572	+#define PERF_EVENT_IOC_ENABLE _IO ('$', 0)
	573	+#define PERF_EVENT_IOC_DISABLE _IO ('$', 1)
	574	+#define PERF_EVENT_IOC_REFRESH _IO ('$', 2)
	575	+#define PERF_EVENT_IOC_RESET _IO ('$', 3)
	576	+#define PERF_EVENT_IOC_PERIOD _IOW ('$', 4, __u64)
	577	+#define PERF_EVENT_IOC_SET_OUTPUT _IO ('$', 5)
	578	+#define PERF_EVENT_IOC_SET_FILTER _IOW ('$', 6, char *)
	579	+#define PERF_EVENT_IOC_ID _IOR ('$', 7, __u64 *)
	580	+#define PERF_EVENT_IOC_SET_BPF _IOW ('$', 8, __u32)
	581	+#define PERF_EVENT_IOC_PAUSE_OUTPUT _IOW ('$', 9, __u32)
572	582	#define PERF_EVENT_IOC_QUERY_BPF _IOWR('$', 10, struct perf_event_query_bpf *)
573		−#define PERF_EVENT_IOC_MODIFY_ATTRIBUTES _IOW('$', 11, struct perf_event_attr *)
	583	+#define PERF_EVENT_IOC_MODIFY_ATTRIBUTES _IOW ('$', 11, struct perf_event_attr *)
574	584
575	585	enum perf_event_ioc_flags {
576		− PERF_IOC_FLAG_GROUP = 1U << 0,
	586	+ PERF_IOC_FLAG_GROUP = 1U << 0,
577	587	};
578	588
579	589	/*
@@ -584,7 +594,7 @@ struct perf_event_mmap_page { · 4 unmodified lines
584	594	__u32 compat_version; /* lowest version this is compat with */
585	595
586	596	/*
587		− * Bits needed to read the hw events in user-space.
	597	+ * Bits needed to read the HW events in user-space.
588	598	*
589	599	* u32 seq, time_mult, time_shift, index, width;
590	600	* u64 count, enabled, running;
@@ -622,7 +632,7 @@ struct perf_event_mmap_page { · 31 unmodified lines
622	632	__u32 index; /* hardware event identifier */
623	633	__s64 offset; /* add to hardware event value */
624	634	__u64 time_enabled; /* time event active */
625		− __u64 time_running; /* time event on cpu */
	635	+ __u64 time_running; /* time event on CPU */
626	636	union {
627	637	__u64 capabilities;
628	638	struct {
@@ -650,7 +660,7 @@ struct perf_event_mmap_page { · 21 unmodified lines
650	660
651	661	/*
652	662	* If cap_usr_time the below fields can be used to compute the time
653		− * delta since time_enabled (in ns) using rdtsc or similar.
	663	+ * delta since time_enabled (in ns) using RDTSC or similar.
654	664	*
655	665	* u64 quot, rem;
656	666	* u64 delta;
@@ -723,7 +733,7 @@ struct perf_event_mmap_page { · 66 unmodified lines
723	733	* after reading this value.
724	734	*
725	735	* When the mapping is PROT_WRITE the @data_tail value should be
726		− * written by userspace to reflect the last read data, after issueing
	736	+ * written by user-space to reflect the last read data, after issuing
727	737	* an smp_mb() to separate the data read from the ->data_tail store.
728	738	* In this case the kernel will not over-write unread data.
729	739	*
@@ -739,7 +749,7 @@ struct perf_event_mmap_page { · 9 unmodified lines
739	749
740	750	/*
741	751	* AUX area is defined by aux_{offset,size} fields that should be set
742		− * by the userspace, so that
	752	+ * by the user-space, so that
743	753	*
744	754	* aux_offset >= data_offset + data_size
745	755	*
@@ -813,7 +823,7 @@ struct perf_event_mmap_page { · 67 unmodified lines
813	823	* Indicates that thread was preempted in TASK_RUNNING state.
814	824	*
815	825	* PERF_RECORD_MISC_MMAP_BUILD_ID:
816		− * Indicates that mmap2 event carries build id data.
	826	+ * Indicates that mmap2 event carries build ID data.
817	827	*/
818	828	#define PERF_RECORD_MISC_EXACT_IP (1 << 14)
819	829	#define PERF_RECORD_MISC_SWITCH_OUT_PREEMPT (1 << 14)
@@ -824,26 +834,26 @@ struct perf_event_mmap_page { · 4 unmodified lines
824	834	#define PERF_RECORD_MISC_EXT_RESERVED (1 << 15)
825	835
826	836	struct perf_event_header {
827		− __u32 type;
828		− __u16 misc;
829		− __u16 size;
	837	+ __u32 type;
	838	+ __u16 misc;
	839	+ __u16 size;
830	840	};
831	841
832	842	struct perf_ns_link_info {
833		− __u64 dev;
834		− __u64 ino;
	843	+ __u64 dev;
	844	+ __u64 ino;
835	845	};
836	846
837	847	enum {
838		− NET_NS_INDEX = 0,
839		− UTS_NS_INDEX = 1,
840		− IPC_NS_INDEX = 2,
841		− PID_NS_INDEX = 3,
842		− USER_NS_INDEX = 4,
843		− MNT_NS_INDEX = 5,
844		− CGROUP_NS_INDEX = 6,
845		−
846		− NR_NAMESPACES, /* number of available namespaces */
	848	+ NET_NS_INDEX = 0,
	849	+ UTS_NS_INDEX = 1,
	850	+ IPC_NS_INDEX = 2,
	851	+ PID_NS_INDEX = 3,
	852	+ USER_NS_INDEX = 4,
	853	+ MNT_NS_INDEX = 5,
	854	+ CGROUP_NS_INDEX = 6,
	855	+
	856	+ NR_NAMESPACES, /* number of available namespaces */
847	857	};
848	858
849	859	enum perf_event_type {
@@ -859,11 +869,11 @@ enum perf_event_type { · 9 unmodified lines
859	869	* optional fields being ignored.
860	870	*
861	871	* struct sample_id {
862		− * { u32 pid, tid; } && PERF_SAMPLE_TID
863		− * { u64 time; } && PERF_SAMPLE_TIME
864		− * { u64 id; } && PERF_SAMPLE_ID
865		− * { u64 stream_id;} && PERF_SAMPLE_STREAM_ID
866		− * { u32 cpu, res; } && PERF_SAMPLE_CPU
	872	+ * { u32 pid, tid; } && PERF_SAMPLE_TID
	873	+ * { u64 time; } && PERF_SAMPLE_TIME
	874	+ * { u64 id; } && PERF_SAMPLE_ID
	875	+ * { u64 stream_id;} && PERF_SAMPLE_STREAM_ID
	876	+ * { u32 cpu, res; } && PERF_SAMPLE_CPU
867	877	* { u64 id; } && PERF_SAMPLE_IDENTIFIER
868	878	* } && perf_event_attr::sample_id_all
869	879	*
@@ -874,7 +884,7 @@ enum perf_event_type { · 4 unmodified lines
874	884
875	885	/*
876	886	* The MMAP events record the PROT_EXEC mappings so that we can
877		− * correlate userspace IPs to code. They have the following structure:
	887	+ * correlate user-space IPs to code. They have the following structure:
878	888	*
879	889	* struct {
880	890	* struct perf_event_header header;
@@ -884,7 +894,7 @@ enum perf_event_type { · 3 unmodified lines
884	894	* u64 len;
885	895	* u64 pgoff;
886	896	* char filename[];
887		− * struct sample_id sample_id;
	897	+ * struct sample_id sample_id;
888	898	* };
889	899	*/
890	900	PERF_RECORD_MMAP = 1,
@@ -894,7 +904,7 @@ enum perf_event_type { · 3 unmodified lines
894	904	* struct perf_event_header header;
895	905	* u64 id;
896	906	* u64 lost;
897		− * struct sample_id sample_id;
	907	+ * struct sample_id sample_id;
898	908	* };
899	909	*/
900	910	PERF_RECORD_LOST = 2,
@@ -905,7 +915,7 @@ enum perf_event_type { · 4 unmodified lines
905	915	*
906	916	* u32 pid, tid;
907	917	* char comm[];
908		− * struct sample_id sample_id;
	918	+ * struct sample_id sample_id;
909	919	* };
910	920	*/
911	921	PERF_RECORD_COMM = 3,
@@ -916,7 +926,7 @@ enum perf_event_type { · 4 unmodified lines
916	926	* u32 pid, ppid;
917	927	* u32 tid, ptid;
918	928	* u64 time;
919		− * struct sample_id sample_id;
	929	+ * struct sample_id sample_id;
920	930	* };
921	931	*/
922	932	PERF_RECORD_EXIT = 4,
@@ -927,7 +937,7 @@ enum perf_event_type { · 4 unmodified lines
927	937	* u64 time;
928	938	* u64 id;
929	939	* u64 stream_id;
930		− * struct sample_id sample_id;
	940	+ * struct sample_id sample_id;
931	941	* };
932	942	*/
933	943	PERF_RECORD_THROTTLE = 5,
@@ -939,7 +949,7 @@ enum perf_event_type { · 5 unmodified lines
939	949	* u32 pid, ppid;
940	950	* u32 tid, ptid;
941	951	* u64 time;
942		− * struct sample_id sample_id;
	952	+ * struct sample_id sample_id;
943	953	* };
944	954	*/
945	955	PERF_RECORD_FORK = 7,
@@ -950,7 +960,7 @@ enum perf_event_type { · 4 unmodified lines
950	960	* u32 pid, tid;
951	961	*
952	962	* struct read_format values;
953		− * struct sample_id sample_id;
	963	+ * struct sample_id sample_id;
954	964	* };
955	965	*/
956	966	PERF_RECORD_READ = 8,
@@ -1005,12 +1015,12 @@ enum perf_event_type { · 48 unmodified lines
1005	1015	* { u64 counters; } cntr[nr] && PERF_SAMPLE_BRANCH_COUNTERS
1006	1016	* } && PERF_SAMPLE_BRANCH_STACK
1007	1017	*
1008		− * { u64 abi; # enum perf_sample_regs_abi
1009		− * u64 regs[weight(mask)]; } && PERF_SAMPLE_REGS_USER
	1018	+ * { u64 abi; # enum perf_sample_regs_abi
	1019	+ * u64 regs[weight(mask)]; } && PERF_SAMPLE_REGS_USER
1010	1020	*
1011		− * { u64 size;
1012		− * char data[size];
1013		− * u64 dyn_size; } && PERF_SAMPLE_STACK_USER
	1021	+ * { u64 size;
	1022	+ * char data[size];
	1023	+ * u64 dyn_size; } && PERF_SAMPLE_STACK_USER
1014	1024	*
1015	1025	* { union perf_sample_weight
1016	1026	* {
@@ -1035,10 +1045,11 @@ enum perf_event_type { · 18 unmodified lines
1035	1045	* { u64 abi; # enum perf_sample_regs_abi
1036	1046	* u64 regs[weight(mask)]; } && PERF_SAMPLE_REGS_INTR
1037	1047	* { u64 phys_addr;} && PERF_SAMPLE_PHYS_ADDR
1038		− * { u64 size;
1039		− * char data[size]; } && PERF_SAMPLE_AUX
	1048	+ * { u64 cgroup;} && PERF_SAMPLE_CGROUP
1040	1049	* { u64 data_page_size;} && PERF_SAMPLE_DATA_PAGE_SIZE
1041	1050	* { u64 code_page_size;} && PERF_SAMPLE_CODE_PAGE_SIZE
	1051	+ * { u64 size;
	1052	+ * char data[size]; } && PERF_SAMPLE_AUX
1042	1053	* };
1043	1054	*/
1044	1055	PERF_RECORD_SAMPLE = 9,
@@ -1070,7 +1081,7 @@ enum perf_event_type { · 25 unmodified lines
1070	1081	* };
1071	1082	* u32 prot, flags;
1072	1083	* char filename[];
1073		− * struct sample_id sample_id;
	1084	+ * struct sample_id sample_id;
1074	1085	* };
1075	1086	*/
1076	1087	PERF_RECORD_MMAP2 = 10,
@@ -1079,12 +1090,12 @@ enum perf_event_type { · 2 unmodified lines
1079	1090	* Records that new data landed in the AUX buffer part.
1080	1091	*
1081	1092	* struct {
1082		− * struct perf_event_header header;
	1093	+ * struct perf_event_header header;
1083	1094	*
1084		− * u64 aux_offset;
1085		− * u64 aux_size;
	1095	+ * u64 aux_offset;
	1096	+ * u64 aux_size;
1086	1097	* u64 flags;
1087		− * struct sample_id sample_id;
	1098	+ * struct sample_id sample_id;
1088	1099	* };
1089	1100	*/
1090	1101	PERF_RECORD_AUX = 11,
@@ -1167,7 +1178,7 @@ enum perf_event_type { · 76 unmodified lines
1167	1178	PERF_RECORD_KSYMBOL = 17,
1168	1179
1169	1180	/*
1170		− * Record bpf events:
	1181	+ * Record BPF events:
1171	1182	* enum perf_bpf_event_type {
1172	1183	* PERF_BPF_EVENT_UNKNOWN = 0,
1173	1184	* PERF_BPF_EVENT_PROG_LOAD = 1,
@@ -1245,181 +1256,181 @@ enum perf_record_ksymbol_type { · 71 unmodified lines
1245	1256	#define PERF_RECORD_KSYMBOL_FLAGS_UNREGISTER (1 << 0)
1246	1257
1247	1258	enum perf_bpf_event_type {
1248		− PERF_BPF_EVENT_UNKNOWN = 0,
1249		− PERF_BPF_EVENT_PROG_LOAD = 1,
1250		− PERF_BPF_EVENT_PROG_UNLOAD = 2,
1251		− PERF_BPF_EVENT_MAX, /* non-ABI */
	1259	+ PERF_BPF_EVENT_UNKNOWN = 0,
	1260	+ PERF_BPF_EVENT_PROG_LOAD = 1,
	1261	+ PERF_BPF_EVENT_PROG_UNLOAD = 2,
	1262	+ PERF_BPF_EVENT_MAX, /* non-ABI */
1252	1263	};
1253	1264
1254		−#define PERF_MAX_STACK_DEPTH 127
1255		−#define PERF_MAX_CONTEXTS_PER_STACK 8
	1265	+#define PERF_MAX_STACK_DEPTH 127
	1266	+#define PERF_MAX_CONTEXTS_PER_STACK 8
1256	1267
1257	1268	enum perf_callchain_context {
1258		− PERF_CONTEXT_HV = (__u64)-32,
1259		− PERF_CONTEXT_KERNEL = (__u64)-128,
1260		− PERF_CONTEXT_USER = (__u64)-512,
	1269	+ PERF_CONTEXT_HV = (__u64)-32,
	1270	+ PERF_CONTEXT_KERNEL = (__u64)-128,
	1271	+ PERF_CONTEXT_USER = (__u64)-512,
1261	1272
1262		− PERF_CONTEXT_GUEST = (__u64)-2048,
1263		− PERF_CONTEXT_GUEST_KERNEL = (__u64)-2176,
1264		− PERF_CONTEXT_GUEST_USER = (__u64)-2560,
	1273	+ PERF_CONTEXT_GUEST = (__u64)-2048,
	1274	+ PERF_CONTEXT_GUEST_KERNEL = (__u64)-2176,
	1275	+ PERF_CONTEXT_GUEST_USER = (__u64)-2560,
1265	1276
1266		− PERF_CONTEXT_MAX = (__u64)-4095,
	1277	+ PERF_CONTEXT_MAX = (__u64)-4095,
1267	1278	};
1268	1279
1269	1280	/**
1270	1281	* PERF_RECORD_AUX::flags bits
1271	1282	*/
1272		−#define PERF_AUX_FLAG_TRUNCATED 0x01 /* record was truncated to fit */
1273		−#define PERF_AUX_FLAG_OVERWRITE 0x02 /* snapshot from overwrite mode */
1274		−#define PERF_AUX_FLAG_PARTIAL 0x04 /* record contains gaps */
1275		−#define PERF_AUX_FLAG_COLLISION 0x08 /* sample collided with another */
	1283	+#define PERF_AUX_FLAG_TRUNCATED 0x0001 /* Record was truncated to fit */
	1284	+#define PERF_AUX_FLAG_OVERWRITE 0x0002 /* Snapshot from overwrite mode */
	1285	+#define PERF_AUX_FLAG_PARTIAL 0x0004 /* Record contains gaps */
	1286	+#define PERF_AUX_FLAG_COLLISION 0x0008 /* Sample collided with another */
1276	1287	#define PERF_AUX_FLAG_PMU_FORMAT_TYPE_MASK 0xff00 /* PMU specific trace format type */
1277	1288
1278	1289	/* CoreSight PMU AUX buffer formats */
1279		−#define PERF_AUX_FLAG_CORESIGHT_FORMAT_CORESIGHT 0x0000 /* Default for backward compatibility */
1280		−#define PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW 0x0100 /* Raw format of the source */
	1290	+#define PERF_AUX_FLAG_CORESIGHT_FORMAT_CORESIGHT 0x0000 /* Default for backward compatibility */
	1291	+#define PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW 0x0100 /* Raw format of the source */
1281	1292
1282		−#define PERF_FLAG_FD_NO_GROUP (1UL << 0)
1283		−#define PERF_FLAG_FD_OUTPUT (1UL << 1)
1284		−#define PERF_FLAG_PID_CGROUP (1UL << 2) /* pid=cgroup id, per-cpu mode only */
1285		−#define PERF_FLAG_FD_CLOEXEC (1UL << 3) /* O_CLOEXEC */
	1293	+#define PERF_FLAG_FD_NO_GROUP (1UL << 0)
	1294	+#define PERF_FLAG_FD_OUTPUT (1UL << 1)
	1295	+#define PERF_FLAG_PID_CGROUP (1UL << 2) /* pid=cgroup ID, per-CPU mode only */
	1296	+#define PERF_FLAG_FD_CLOEXEC (1UL << 3) /* O_CLOEXEC */
1286	1297
1287	1298	#if defined(__LITTLE_ENDIAN_BITFIELD)
1288	1299	union perf_mem_data_src {
1289	1300	__u64 val;
1290	1301	struct {
1291		− __u64 mem_op:5, /* type of opcode */
1292		− mem_lvl:14, /* memory hierarchy level */
1293		− mem_snoop:5, /* snoop mode */
1294		− mem_lock:2, /* lock instr */
1295		− mem_dtlb:7, /* tlb access */
1296		− mem_lvl_num:4, /* memory hierarchy level number */
1297		− mem_remote:1, /* remote */
1298		− mem_snoopx:2, /* snoop mode, ext */
1299		− mem_blk:3, /* access blocked */
1300		− mem_hops:3, /* hop level */
1301		− mem_rsvd:18;
	1302	+ __u64 mem_op : 5, /* Type of opcode */
	1303	+ mem_lvl : 14, /* Memory hierarchy level */
	1304	+ mem_snoop : 5, /* Snoop mode */
	1305	+ mem_lock : 2, /* Lock instr */
	1306	+ mem_dtlb : 7, /* TLB access */
	1307	+ mem_lvl_num : 4, /* Memory hierarchy level number */
	1308	+ mem_remote : 1, /* Remote */
	1309	+ mem_snoopx : 2, /* Snoop mode, ext */
	1310	+ mem_blk : 3, /* Access blocked */
	1311	+ mem_hops : 3, /* Hop level */
	1312	+ mem_rsvd : 18;
1302	1313	};
1303	1314	};
1304	1315	#elif defined(__BIG_ENDIAN_BITFIELD)
1305	1316	union perf_mem_data_src {
1306	1317	__u64 val;
1307	1318	struct {
1308		− __u64 mem_rsvd:18,
1309		− mem_hops:3, /* hop level */
1310		− mem_blk:3, /* access blocked */
1311		− mem_snoopx:2, /* snoop mode, ext */
1312		− mem_remote:1, /* remote */
1313		− mem_lvl_num:4, /* memory hierarchy level number */
1314		− mem_dtlb:7, /* tlb access */
1315		− mem_lock:2, /* lock instr */
1316		− mem_snoop:5, /* snoop mode */
1317		− mem_lvl:14, /* memory hierarchy level */
1318		− mem_op:5; /* type of opcode */
	1319	+ __u64 mem_rsvd : 18,
	1320	+ mem_hops : 3, /* Hop level */
	1321	+ mem_blk : 3, /* Access blocked */
	1322	+ mem_snoopx : 2, /* Snoop mode, ext */
	1323	+ mem_remote : 1, /* Remote */
	1324	+ mem_lvl_num : 4, /* Memory hierarchy level number */
	1325	+ mem_dtlb : 7, /* TLB access */
	1326	+ mem_lock : 2, /* Lock instr */
	1327	+ mem_snoop : 5, /* Snoop mode */
	1328	+ mem_lvl : 14, /* Memory hierarchy level */
	1329	+ mem_op : 5; /* Type of opcode */
1319	1330	};
1320	1331	};
1321	1332	#else
1322		−#error "Unknown endianness"
	1333	+# error "Unknown endianness"
1323	1334	#endif
1324	1335
1325		−/* type of opcode (load/store/prefetch,code) */
1326		−#define PERF_MEM_OP_NA 0x01 /* not available */
1327		−#define PERF_MEM_OP_LOAD 0x02 /* load instruction */
1328		−#define PERF_MEM_OP_STORE 0x04 /* store instruction */
1329		−#define PERF_MEM_OP_PFETCH 0x08 /* prefetch */
1330		−#define PERF_MEM_OP_EXEC 0x10 /* code (execution) */
1331		−#define PERF_MEM_OP_SHIFT 0
	1336	+/* Type of memory opcode: */
	1337	+#define PERF_MEM_OP_NA 0x0001 /* Not available */
	1338	+#define PERF_MEM_OP_LOAD 0x0002 /* Load instruction */
	1339	+#define PERF_MEM_OP_STORE 0x0004 /* Store instruction */
	1340	+#define PERF_MEM_OP_PFETCH 0x0008 /* Prefetch */
	1341	+#define PERF_MEM_OP_EXEC 0x0010 /* Code (execution) */
	1342	+#define PERF_MEM_OP_SHIFT 0
1332	1343
1333	1344	/*
1334		− * PERF_MEM_LVL_* namespace being depricated to some extent in the
	1345	+ * The PERF_MEM_LVL_* namespace is being deprecated to some extent in
1335	1346	* favour of newer composite PERF_MEM_{LVLNUM_,REMOTE_,SNOOPX_} fields.
1336		− * Supporting this namespace inorder to not break defined ABIs.
	1347	+ * We support this namespace in order to not break defined ABIs.
1337	1348	*
1338		− * memory hierarchy (memory level, hit or miss)
	1349	+ * Memory hierarchy (memory level, hit or miss)
1339	1350	*/
1340		−#define PERF_MEM_LVL_NA 0x01 /* not available */
1341		−#define PERF_MEM_LVL_HIT 0x02 /* hit level */
1342		−#define PERF_MEM_LVL_MISS 0x04 /* miss level */
1343		−#define PERF_MEM_LVL_L1 0x08 /* L1 */
1344		−#define PERF_MEM_LVL_LFB 0x10 /* Line Fill Buffer */
1345		−#define PERF_MEM_LVL_L2 0x20 /* L2 */
1346		−#define PERF_MEM_LVL_L3 0x40 /* L3 */
1347		−#define PERF_MEM_LVL_LOC_RAM 0x80 /* Local DRAM */
1348		−#define PERF_MEM_LVL_REM_RAM1 0x100 /* Remote DRAM (1 hop) */
1349		−#define PERF_MEM_LVL_REM_RAM2 0x200 /* Remote DRAM (2 hops) */
1350		−#define PERF_MEM_LVL_REM_CCE1 0x400 /* Remote Cache (1 hop) */
1351		−#define PERF_MEM_LVL_REM_CCE2 0x800 /* Remote Cache (2 hops) */
1352		−#define PERF_MEM_LVL_IO 0x1000 /* I/O memory */
1353		−#define PERF_MEM_LVL_UNC 0x2000 /* Uncached memory */
1354		−#define PERF_MEM_LVL_SHIFT 5
1355		−
1356		−#define PERF_MEM_REMOTE_REMOTE 0x01 /* Remote */
1357		−#define PERF_MEM_REMOTE_SHIFT 37
1358		−
1359		−#define PERF_MEM_LVLNUM_L1 0x01 /* L1 */
1360		−#define PERF_MEM_LVLNUM_L2 0x02 /* L2 */
1361		−#define PERF_MEM_LVLNUM_L3 0x03 /* L3 */
1362		−#define PERF_MEM_LVLNUM_L4 0x04 /* L4 */
1363		−#define PERF_MEM_LVLNUM_L2_MHB 0x05 /* L2 Miss Handling Buffer */
1364		−#define PERF_MEM_LVLNUM_MSC 0x06 /* Memory-side Cache */
1365		−/* 0x7 available */
1366		−#define PERF_MEM_LVLNUM_UNC 0x08 /* Uncached */
1367		−#define PERF_MEM_LVLNUM_CXL 0x09 /* CXL */
1368		−#define PERF_MEM_LVLNUM_IO 0x0a /* I/O */
1369		−#define PERF_MEM_LVLNUM_ANY_CACHE 0x0b /* Any cache */
1370		−#define PERF_MEM_LVLNUM_LFB 0x0c /* LFB / L1 Miss Handling Buffer */
1371		−#define PERF_MEM_LVLNUM_RAM 0x0d /* RAM */
1372		−#define PERF_MEM_LVLNUM_PMEM 0x0e /* PMEM */
1373		−#define PERF_MEM_LVLNUM_NA 0x0f /* N/A */
1374		−
1375		−#define PERF_MEM_LVLNUM_SHIFT 33
1376		−
1377		−/* snoop mode */
1378		−#define PERF_MEM_SNOOP_NA 0x01 /* not available */
1379		−#define PERF_MEM_SNOOP_NONE 0x02 /* no snoop */
1380		−#define PERF_MEM_SNOOP_HIT 0x04 /* snoop hit */
1381		−#define PERF_MEM_SNOOP_MISS 0x08 /* snoop miss */
1382		−#define PERF_MEM_SNOOP_HITM 0x10 /* snoop hit modified */
1383		−#define PERF_MEM_SNOOP_SHIFT 19
1384		−
1385		−#define PERF_MEM_SNOOPX_FWD 0x01 /* forward */
1386		−#define PERF_MEM_SNOOPX_PEER 0x02 /* xfer from peer */
1387		−#define PERF_MEM_SNOOPX_SHIFT 38
1388		−
1389		−/* locked instruction */
1390		−#define PERF_MEM_LOCK_NA 0x01 /* not available */
1391		−#define PERF_MEM_LOCK_LOCKED 0x02 /* locked transaction */
1392		−#define PERF_MEM_LOCK_SHIFT 24
	1351	+#define PERF_MEM_LVL_NA 0x0001 /* Not available */
	1352	+#define PERF_MEM_LVL_HIT 0x0002 /* Hit level */
	1353	+#define PERF_MEM_LVL_MISS 0x0004 /* Miss level */
	1354	+#define PERF_MEM_LVL_L1 0x0008 /* L1 */
	1355	+#define PERF_MEM_LVL_LFB 0x0010 /* Line Fill Buffer */
	1356	+#define PERF_MEM_LVL_L2 0x0020 /* L2 */
	1357	+#define PERF_MEM_LVL_L3 0x0040 /* L3 */
	1358	+#define PERF_MEM_LVL_LOC_RAM 0x0080 /* Local DRAM */
	1359	+#define PERF_MEM_LVL_REM_RAM1 0x0100 /* Remote DRAM (1 hop) */
	1360	+#define PERF_MEM_LVL_REM_RAM2 0x0200 /* Remote DRAM (2 hops) */
	1361	+#define PERF_MEM_LVL_REM_CCE1 0x0400 /* Remote Cache (1 hop) */
	1362	+#define PERF_MEM_LVL_REM_CCE2 0x0800 /* Remote Cache (2 hops) */
	1363	+#define PERF_MEM_LVL_IO 0x1000 /* I/O memory */
	1364	+#define PERF_MEM_LVL_UNC 0x2000 /* Uncached memory */
	1365	+#define PERF_MEM_LVL_SHIFT 5
	1366	+
	1367	+#define PERF_MEM_REMOTE_REMOTE 0x0001 /* Remote */
	1368	+#define PERF_MEM_REMOTE_SHIFT 37
	1369	+
	1370	+#define PERF_MEM_LVLNUM_L1 0x0001 /* L1 */
	1371	+#define PERF_MEM_LVLNUM_L2 0x0002 /* L2 */
	1372	+#define PERF_MEM_LVLNUM_L3 0x0003 /* L3 */
	1373	+#define PERF_MEM_LVLNUM_L4 0x0004 /* L4 */
	1374	+#define PERF_MEM_LVLNUM_L2_MHB 0x0005 /* L2 Miss Handling Buffer */
	1375	+#define PERF_MEM_LVLNUM_MSC 0x0006 /* Memory-side Cache */
	1376	+/* 0x007 available */
	1377	+#define PERF_MEM_LVLNUM_UNC 0x0008 /* Uncached */
	1378	+#define PERF_MEM_LVLNUM_CXL 0x0009 /* CXL */
	1379	+#define PERF_MEM_LVLNUM_IO 0x000a /* I/O */
	1380	+#define PERF_MEM_LVLNUM_ANY_CACHE 0x000b /* Any cache */
	1381	+#define PERF_MEM_LVLNUM_LFB 0x000c /* LFB / L1 Miss Handling Buffer */
	1382	+#define PERF_MEM_LVLNUM_RAM 0x000d /* RAM */
	1383	+#define PERF_MEM_LVLNUM_PMEM 0x000e /* PMEM */
	1384	+#define PERF_MEM_LVLNUM_NA 0x000f /* N/A */
	1385	+
	1386	+#define PERF_MEM_LVLNUM_SHIFT 33
	1387	+
	1388	+/* Snoop mode */
	1389	+#define PERF_MEM_SNOOP_NA 0x0001 /* Not available */
	1390	+#define PERF_MEM_SNOOP_NONE 0x0002 /* No snoop */
	1391	+#define PERF_MEM_SNOOP_HIT 0x0004 /* Snoop hit */
	1392	+#define PERF_MEM_SNOOP_MISS 0x0008 /* Snoop miss */
	1393	+#define PERF_MEM_SNOOP_HITM 0x0010 /* Snoop hit modified */
	1394	+#define PERF_MEM_SNOOP_SHIFT 19
	1395	+
	1396	+#define PERF_MEM_SNOOPX_FWD 0x0001 /* Forward */
	1397	+#define PERF_MEM_SNOOPX_PEER 0x0002 /* Transfer from peer */
	1398	+#define PERF_MEM_SNOOPX_SHIFT 38
	1399	+
	1400	+/* Locked instruction */
	1401	+#define PERF_MEM_LOCK_NA 0x0001 /* Not available */
	1402	+#define PERF_MEM_LOCK_LOCKED 0x0002 /* Locked transaction */
	1403	+#define PERF_MEM_LOCK_SHIFT 24
1393	1404
1394	1405	/* TLB access */
1395		−#define PERF_MEM_TLB_NA 0x01 /* not available */
1396		−#define PERF_MEM_TLB_HIT 0x02 /* hit level */
1397		−#define PERF_MEM_TLB_MISS 0x04 /* miss level */
1398		−#define PERF_MEM_TLB_L1 0x08 /* L1 */
1399		−#define PERF_MEM_TLB_L2 0x10 /* L2 */
1400		−#define PERF_MEM_TLB_WK 0x20 /* Hardware Walker*/
1401		−#define PERF_MEM_TLB_OS 0x40 /* OS fault handler */
1402		−#define PERF_MEM_TLB_SHIFT 26
	1406	+#define PERF_MEM_TLB_NA 0x0001 /* Not available */
	1407	+#define PERF_MEM_TLB_HIT 0x0002 /* Hit level */
	1408	+#define PERF_MEM_TLB_MISS 0x0004 /* Miss level */
	1409	+#define PERF_MEM_TLB_L1 0x0008 /* L1 */
	1410	+#define PERF_MEM_TLB_L2 0x0010 /* L2 */
	1411	+#define PERF_MEM_TLB_WK 0x0020 /* Hardware Walker*/
	1412	+#define PERF_MEM_TLB_OS 0x0040 /* OS fault handler */
	1413	+#define PERF_MEM_TLB_SHIFT 26
1403	1414
1404	1415	/* Access blocked */
1405		−#define PERF_MEM_BLK_NA 0x01 /* not available */
1406		−#define PERF_MEM_BLK_DATA 0x02 /* data could not be forwarded */
1407		−#define PERF_MEM_BLK_ADDR 0x04 /* address conflict */
1408		−#define PERF_MEM_BLK_SHIFT 40
1409		−
1410		−/* hop level */
1411		−#define PERF_MEM_HOPS_0 0x01 /* remote core, same node */
1412		−#define PERF_MEM_HOPS_1 0x02 /* remote node, same socket */
1413		−#define PERF_MEM_HOPS_2 0x03 /* remote socket, same board */
1414		−#define PERF_MEM_HOPS_3 0x04 /* remote board */
	1416	+#define PERF_MEM_BLK_NA 0x0001 /* Not available */
	1417	+#define PERF_MEM_BLK_DATA 0x0002 /* Data could not be forwarded */
	1418	+#define PERF_MEM_BLK_ADDR 0x0004 /* Address conflict */
	1419	+#define PERF_MEM_BLK_SHIFT 40
	1420	+
	1421	+/* Hop level */
	1422	+#define PERF_MEM_HOPS_0 0x0001 /* Remote core, same node */
	1423	+#define PERF_MEM_HOPS_1 0x0002 /* Remote node, same socket */
	1424	+#define PERF_MEM_HOPS_2 0x0003 /* Remote socket, same board */
	1425	+#define PERF_MEM_HOPS_3 0x0004 /* Remote board */
1415	1426	/* 5-7 available */
1416		−#define PERF_MEM_HOPS_SHIFT 43
	1427	+#define PERF_MEM_HOPS_SHIFT 43
1417	1428
1418	1429	#define PERF_MEM_S(a, s) \
1419	1430	(((__u64)PERF_MEM_##a##_##s) << PERF_MEM_##a##_SHIFT)
1420	1431
1421	1432	/*
1422		− * single taken branch record layout:
	1433	+ * Layout of single taken branch records:
1423	1434	*
1424	1435	* from: source instruction (may not always be a branch insn)
1425	1436	* to: branch target
@@ -1438,37 +1449,37 @@ union perf_mem_data_src { · 12 unmodified lines
1438	1449	struct perf_branch_entry {
1439	1450	__u64 from;
1440	1451	__u64 to;
1441		− __u64 mispred:1, /* target mispredicted */
1442		− predicted:1,/* target predicted */
1443		− in_tx:1, /* in transaction */
1444		− abort:1, /* transaction abort */
1445		− cycles:16, /* cycle count to last branch */
1446		− type:4, /* branch type */
1447		− spec:2, /* branch speculation info */
1448		− new_type:4, /* additional branch type */
1449		− priv:3, /* privilege level */
1450		− reserved:31;
	1452	+ __u64 mispred : 1, /* target mispredicted */
	1453	+ predicted : 1, /* target predicted */
	1454	+ in_tx : 1, /* in transaction */
	1455	+ abort : 1, /* transaction abort */
	1456	+ cycles : 16, /* cycle count to last branch */
	1457	+ type : 4, /* branch type */
	1458	+ spec : 2, /* branch speculation info */
	1459	+ new_type : 4, /* additional branch type */
	1460	+ priv : 3, /* privilege level */
	1461	+ reserved : 31;
1451	1462	};
1452	1463
1453	1464	/* Size of used info bits in struct perf_branch_entry */
1454	1465	#define PERF_BRANCH_ENTRY_INFO_BITS_MAX 33
1455	1466
1456	1467	union perf_sample_weight {
1457		− __u64 full;
	1468	+ __u64 full;
1458	1469	#if defined(__LITTLE_ENDIAN_BITFIELD)
1459	1470	struct {
1460		− __u32 var1_dw;
1461		− __u16 var2_w;
1462		− __u16 var3_w;
	1471	+ __u32 var1_dw;
	1472	+ __u16 var2_w;
	1473	+ __u16 var3_w;
1463	1474	};
1464	1475	#elif defined(__BIG_ENDIAN_BITFIELD)
1465	1476	struct {
1466		− __u16 var3_w;
1467		− __u16 var2_w;
1468		− __u32 var1_dw;
	1477	+ __u16 var3_w;
	1478	+ __u16 var2_w;
	1479	+ __u32 var1_dw;
1469	1480	};
1470	1481	#else
1471		−#error "Unknown endianness"
	1482	+# error "Unknown endianness"
1472	1483	#endif
1473	1484	};
1474	1485

kernel/cpu.c+0−5

@@ -2069,11 +2069,6 @@ static struct cpuhp_step cpuhp_hp_states[] = { · 2068 unmodified lines
2069	2069	.teardown.single = NULL,
2070	2070	.cant_stop = true,
2071	2071	},
2072		− [CPUHP_PERF_PREPARE] = {
2073		− .name = "perf:prepare",
2074		− .startup.single = perf_event_init_cpu,
2075		− .teardown.single = perf_event_exit_cpu,
2076		− },
2077	2072	[CPUHP_RANDOM_PREPARE] = {
2078	2073	.name = "random:prepare",
2079	2074	.startup.single = random_prepare_cpu,

kernel/events/core.c+424−185

@@ -1270,6 +1270,10 @@ static void put_ctx(struct perf_event_context *ctx) · 1269 unmodified lines
1270	1270	if (ctx->task && ctx->task != TASK_TOMBSTONE)
1271	1271	put_task_struct(ctx->task);
1272	1272	call_rcu(&ctx->rcu_head, free_ctx);
	1273	+ } else {
	1274	+ smp_mb__after_atomic(); /* pairs with wait_var_event() */
	1275	+ if (ctx->task == TASK_TOMBSTONE)
	1276	+ wake_up_var(&ctx->refcount);
1273	1277	}
1274	1278	}
1275	1279
@@ -2167,7 +2171,7 @@ static void perf_put_aux_event(struct perf_event *event) · 891 unmodified lines
2167	2171	* If the event is an aux_event, tear down all links to
2168	2172	* it from other events.
2169	2173	*/
2170		− for_each_sibling_event(iter, event->group_leader) {
	2174	+ for_each_sibling_event(iter, event) {
2171	2175	if (iter->aux_event != event)
2172	2176	continue;
2173	2177
@@ -2325,7 +2329,11 @@ static void perf_child_detach(struct perf_event *event) · 151 unmodified lines
2325	2329	if (WARN_ON_ONCE(!parent_event))
2326	2330	return;
2327	2331
	2332	+ /*
	2333	+ * Can't check this from an IPI, the holder is likey another CPU.
	2334	+ *
2328	2335	lockdep_assert_held(&parent_event->child_mutex);
	2336	+ */
2329	2337
2330	2338	sync_child_event(event);
2331	2339	list_del_init(&event->child_list);
@@ -2343,6 +2351,11 @@ event_filter_match(struct perf_event *event) · 11 unmodified lines
2343	2351	perf_cgroup_match(event);
2344	2352	}
2345	2353
	2354	+static inline bool is_event_in_freq_mode(struct perf_event *event)
	2355	+{
	2356	+ return event->attr.freq && event->attr.sample_freq;
	2357	+}
	2358	+
2346	2359	static void
2347	2360	event_sched_out(struct perf_event event, struct perf_event_context ctx)
2348	2361	{
@@ -2380,7 +2393,7 @@ event_sched_out(struct perf_event event, struct perf_event_context ctx) · 31 unmodified lines
2380	2393
2381	2394	if (!is_software_event(event))
2382	2395	cpc->active_oncpu--;
2383		− if (event->attr.freq && event->attr.sample_freq) {
	2396	+ if (is_event_in_freq_mode(event)) {
2384	2397	ctx->nr_freq--;
2385	2398	epc->nr_freq--;
2386	2399	}
@@ -2450,8 +2463,9 @@ ctx_time_update_event(struct perf_event_context ctx, struct perf_event event) · 63 unmodified lines
2450	2463
2451	2464	#define DETACH_GROUP 0x01UL
2452	2465	#define DETACH_CHILD 0x02UL
2453		−#define DETACH_DEAD 0x04UL
2454		−#define DETACH_EXIT 0x08UL
	2466	+#define DETACH_EXIT 0x04UL
	2467	+#define DETACH_REVOKE 0x08UL
	2468	+#define DETACH_DEAD 0x10UL
2455	2469
2456	2470	/*
2457	2471	* Cross CPU call to remove a performance event
@@ -2477,12 +2491,15 @@ __perf_remove_from_context(struct perf_event *event, · 19 unmodified lines
2477	2491	*/
2478	2492	if (flags & DETACH_EXIT)
2479	2493	state = PERF_EVENT_STATE_EXIT;
	2494	+ if (flags & DETACH_REVOKE)
	2495	+ state = PERF_EVENT_STATE_REVOKED;
2480	2496	if (flags & DETACH_DEAD) {
2481	2497	event->pending_disable = 1;
2482	2498	state = PERF_EVENT_STATE_DEAD;
2483	2499	}
2484	2500	event_sched_out(event, ctx);
2485	2501	perf_event_set_state(event, min(event->state, state));
	2502	+
2486	2503	if (flags & DETACH_GROUP)
2487	2504	perf_group_detach(event);
2488	2505	if (flags & DETACH_CHILD)
@@ -2628,6 +2645,41 @@ void perf_event_disable_inatomic(struct perf_event *event) · 139 unmodified lines
2628	2645	static void perf_log_throttle(struct perf_event *event, int enable);
2629	2646	static void perf_log_itrace_start(struct perf_event *event);
2630	2647
	2648	+static void perf_event_unthrottle(struct perf_event *event, bool start)
	2649	+{
	2650	+ event->hw.interrupts = 0;
	2651	+ if (start)
	2652	+ event->pmu->start(event, 0);
	2653	+ if (event == event->group_leader)
	2654	+ perf_log_throttle(event, 1);
	2655	+}
	2656	+
	2657	+static void perf_event_throttle(struct perf_event *event)
	2658	+{
	2659	+ event->pmu->stop(event, 0);
	2660	+ event->hw.interrupts = MAX_INTERRUPTS;
	2661	+ if (event == event->group_leader)
	2662	+ perf_log_throttle(event, 0);
	2663	+}
	2664	+
	2665	+static void perf_event_unthrottle_group(struct perf_event *event, bool skip_start_event)
	2666	+{
	2667	+ struct perf_event sibling, leader = event->group_leader;
	2668	+
	2669	+ perf_event_unthrottle(leader, skip_start_event ? leader != event : true);
	2670	+ for_each_sibling_event(sibling, leader)
	2671	+ perf_event_unthrottle(sibling, skip_start_event ? sibling != event : true);
	2672	+}
	2673	+
	2674	+static void perf_event_throttle_group(struct perf_event *event)
	2675	+{
	2676	+ struct perf_event sibling, leader = event->group_leader;
	2677	+
	2678	+ perf_event_throttle(leader);
	2679	+ for_each_sibling_event(sibling, leader)
	2680	+ perf_event_throttle(sibling);
	2681	+}
	2682	+
2631	2683	static int
2632	2684	event_sched_in(struct perf_event event, struct perf_event_context ctx)
2633	2685	{
@@ -2656,10 +2708,8 @@ event_sched_in(struct perf_event event, struct perf_event_context ctx) · 22 unmodified lines
2656	2708	* ticks already, also for a heavily scheduling task there is little
2657	2709	* guarantee it'll get a tick in a timely manner.
2658	2710	*/
2659		− if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) {
2660		− perf_log_throttle(event, 1);
2661		− event->hw.interrupts = 0;
2662		− }
	2711	+ if (unlikely(event->hw.interrupts == MAX_INTERRUPTS))
	2712	+ perf_event_unthrottle(event, false);
2663	2713
2664	2714	perf_pmu_disable(event->pmu);
2665	2715
@@ -2674,7 +2724,7 @@ event_sched_in(struct perf_event event, struct perf_event_context ctx) · 8 unmodified lines
2674	2724
2675	2725	if (!is_software_event(event))
2676	2726	cpc->active_oncpu++;
2677		− if (event->attr.freq && event->attr.sample_freq) {
	2727	+ if (is_event_in_freq_mode(event)) {
2678	2728	ctx->nr_freq++;
2679	2729	epc->nr_freq++;
2680	2730	}
@@ -4237,14 +4287,10 @@ static void perf_adjust_freq_unthr_events(struct list_head *event_list) · 1556 unmodified lines
4237	4287
4238	4288	hwc = &event->hw;
4239	4289
4240		− if (hwc->interrupts == MAX_INTERRUPTS) {
4241		− hwc->interrupts = 0;
4242		− perf_log_throttle(event, 1);
4243		− if (!event->attr.freq \|\| !event->attr.sample_freq)
4244		− event->pmu->start(event, 0);
4245		− }
	4290	+ if (hwc->interrupts == MAX_INTERRUPTS)
	4291	+ perf_event_unthrottle_group(event, is_event_in_freq_mode(event));
4246	4292
4247		− if (!event->attr.freq \|\| !event->attr.sample_freq)
	4293	+ if (!is_event_in_freq_mode(event))
4248	4294	continue;
4249	4295
4250	4296	/*
@@ -4516,7 +4562,8 @@ out: · 265 unmodified lines
4516	4562
4517	4563	static void perf_remove_from_owner(struct perf_event *event);
4518	4564	static void perf_event_exit_event(struct perf_event *event,
4519		− struct perf_event_context *ctx);
	4565	+ struct perf_event_context *ctx,
	4566	+ bool revoke);
4520	4567
4521	4568	/*
4522	4569	* Removes all events from the current task that have been marked
@@ -4543,7 +4590,7 @@ static void perf_event_remove_on_exec(struct perf_event_context *ctx) · 20 unmodified lines
4543	4590
4544	4591	modified = true;
4545	4592
4546		− perf_event_exit_event(event, ctx);
	4593	+ perf_event_exit_event(event, ctx, false);
4547	4594	}
4548	4595
4549	4596	raw_spin_lock_irqsave(&ctx->lock, flags);
@@ -5125,6 +5172,7 @@ static bool is_sb_event(struct perf_event *event) · 575 unmodified lines
5125	5172	attr->context_switch \|\| attr->text_poke \|\|
5126	5173	attr->bpf_event)
5127	5174	return true;
	5175	+
5128	5176	return false;
5129	5177	}
5130	5178
@@ -5521,6 +5569,8 @@ static void perf_free_addr_filters(struct perf_event *event); · 390 unmodified lines
5521	5569	/* vs perf_event_alloc() error */
5522	5570	static void __free_event(struct perf_event *event)
5523	5571	{
	5572	+ struct pmu *pmu = event->pmu;
	5573	+
5524	5574	if (event->attach_state & PERF_ATTACH_CALLCHAIN)
5525	5575	put_callchain_buffers();
5526	5576
@@ -5550,6 +5600,7 @@ static void __free_event(struct perf_event *event) · 23 unmodified lines
5550	5600	* put_pmu_ctx() needs an event->ctx reference, because of
5551	5601	* epc->ctx.
5552	5602	*/
	5603	+ WARN_ON_ONCE(!pmu);
5553	5604	WARN_ON_ONCE(!event->ctx);
5554	5605	WARN_ON_ONCE(event->pmu_ctx->ctx != event->ctx);
5555	5606	put_pmu_ctx(event->pmu_ctx);
@@ -5562,8 +5613,13 @@ static void __free_event(struct perf_event *event) · 6 unmodified lines
5562	5613	if (event->ctx)
5563	5614	put_ctx(event->ctx);
5564	5615
5565		− if (event->pmu)
5566		− module_put(event->pmu->module);
	5616	+ if (pmu) {
	5617	+ module_put(pmu->module);
	5618	+ scoped_guard (spinlock, &pmu->events_lock) {
	5619	+ list_del(&event->pmu_list);
	5620	+ wake_up_var(pmu);
	5621	+ }
	5622	+ }
5567	5623
5568	5624	call_rcu(&event->rcu_head, free_event_rcu);
5569	5625	}
@@ -5600,13 +5656,13 @@ static void _free_event(struct perf_event *event) · 30 unmodified lines
5600	5656
5601	5657	/*
5602	5658	* Used to free events which have a known refcount of 1, such as in error paths
5603		− * where the event isn't exposed yet and inherited events.
	5659	+ * of inherited events.
5604	5660	*/
5605	5661	static void free_event(struct perf_event *event)
5606	5662	{
5607	5663	if (WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1,
5608		− "unexpected event refcount: %ld; ptr=%p\n",
5609		− atomic_long_read(&event->refcount), event)) {
	5664	+ "unexpected event refcount: %ld; ptr=%p\n",
	5665	+ atomic_long_read(&event->refcount), event)) {
5610	5666	/* leak to avoid use-after-free */
5611	5667	return;
5612	5668	}
@@ -5689,7 +5745,6 @@ int perf_event_release_kernel(struct perf_event *event) · 76 unmodified lines
5689	5745	{
5690	5746	struct perf_event_context *ctx = event->ctx;
5691	5747	struct perf_event child, tmp;
5692		− LIST_HEAD(free_list);
5693	5748
5694	5749	/*
5695	5750	* If we got here through err_alloc: free_event(event); we will not
@@ -5718,15 +5773,17 @@ int perf_event_release_kernel(struct perf_event *event) · 22 unmodified lines
5718	5773	* Thus this guarantees that we will in fact observe and kill _ALL_
5719	5774	* child events.
5720	5775	*/
5721		− perf_remove_from_context(event, DETACH_GROUP\|DETACH_DEAD);
	5776	+ if (event->state > PERF_EVENT_STATE_REVOKED) {
	5777	+ perf_remove_from_context(event, DETACH_GROUP\|DETACH_DEAD);
	5778	+ } else {
	5779	+ event->state = PERF_EVENT_STATE_DEAD;
	5780	+ }
5722	5781
5723	5782	perf_event_ctx_unlock(event, ctx);
5724	5783
5725	5784	again:
5726	5785	mutex_lock(&event->child_mutex);
5727	5786	list_for_each_entry(child, &event->child_list, child_list) {
5728		− void *var = NULL;
5729		−
5730	5787	/*
5731	5788	* Cannot change, child events are not migrated, see the
5732	5789	* comment with perf_event_ctx_lock_nested().
@@ -5759,44 +5816,24 @@ again: · 26 unmodified lines
5759	5816	tmp = list_first_entry_or_null(&event->child_list,
5760	5817	struct perf_event, child_list);
5761	5818	if (tmp == child) {
5762		− perf_remove_from_context(child, DETACH_GROUP);
5763		− list_move(&child->child_list, &free_list);
	5819	+ perf_remove_from_context(child, DETACH_GROUP \| DETACH_CHILD);
5764	5820	} else {
5765		− var = &ctx->refcount;
	5821	+ child = NULL;
5766	5822	}
5767	5823
5768	5824	mutex_unlock(&event->child_mutex);
5769	5825	mutex_unlock(&ctx->mutex);
5770		− put_ctx(ctx);
5771	5826
5772		− if (var) {
5773		− /*
5774		− * If perf_event_free_task() has deleted all events from the
5775		− * ctx while the child_mutex got released above, make sure to
5776		− * notify about the preceding put_ctx().
5777		− */
5778		− smp_mb(); /* pairs with wait_var_event() */
5779		− wake_up_var(var);
	5827	+ if (child) {
	5828	+ /* Last reference unless ->pending_task work is pending */
	5829	+ put_event(child);
5780	5830	}
	5831	+ put_ctx(ctx);
	5832	+
5781	5833	goto again;
5782	5834	}
5783	5835	mutex_unlock(&event->child_mutex);
5784	5836
5785		− list_for_each_entry_safe(child, tmp, &free_list, child_list) {
5786		− void *var = &child->ctx->refcount;
5787		−
5788		− list_del(&child->child_list);
5789		− /* Last reference unless ->pending_task work is pending */
5790		− put_event(child);
5791		−
5792		− /*
5793		− * Wake any perf_event_free_task() waiting for this event to be
5794		− * freed.
5795		− */
5796		− smp_mb(); /* pairs with wait_var_event() */
5797		− wake_up_var(var);
5798		− }
5799		−
5800	5837	no_ctx:
5801	5838	/*
5802	5839	* Last reference unless ->pending_task work is pending on this event
@@ -6068,8 +6105,14 @@ static __poll_t perf_poll(struct file file, poll_table wait) · 265 unmodified lines
6068	6105	struct perf_buffer *rb;
6069	6106	__poll_t events = EPOLLHUP;
6070	6107
	6108	+ if (event->state <= PERF_EVENT_STATE_REVOKED)
	6109	+ return EPOLLERR;
	6110	+
6071	6111	poll_wait(file, &event->waitq, wait);
6072	6112
	6113	+ if (event->state <= PERF_EVENT_STATE_REVOKED)
	6114	+ return EPOLLERR;
	6115	+
6073	6116	if (is_event_hup(event))
6074	6117	return events;
6075	6118
@@ -6167,14 +6210,6 @@ static void __perf_event_period(struct perf_event *event, · 91 unmodified lines
6167	6210	active = (event->state == PERF_EVENT_STATE_ACTIVE);
6168	6211	if (active) {
6169	6212	perf_pmu_disable(event->pmu);
6170		− /*
6171		− * We could be throttled; unthrottle now to avoid the tick
6172		− * trying to unthrottle while we already re-started the event.
6173		− */
6174		− if (event->hw.interrupts == MAX_INTERRUPTS) {
6175		− event->hw.interrupts = 0;
6176		− perf_log_throttle(event, 1);
6177		− }
6178	6213	event->pmu->stop(event, PERF_EF_UPDATE);
6179	6214	}
6180	6215
@@ -6182,6 +6217,14 @@ static void __perf_event_period(struct perf_event *event, · 1 unmodified line
6182	6217
6183	6218	if (active) {
6184	6219	event->pmu->start(event, PERF_EF_RELOAD);
	6220	+ /*
	6221	+ * Once the period is force-reset, the event starts immediately.
	6222	+ * But the event/group could be throttled. Unthrottle the
	6223	+ * event/group now to avoid the next tick trying to unthrottle
	6224	+ * while we already re-started the event/group.
	6225	+ */
	6226	+ if (event->hw.interrupts == MAX_INTERRUPTS)
	6227	+ perf_event_unthrottle_group(event, true);
6185	6228	perf_pmu_enable(event->pmu);
6186	6229	}
6187	6230	}
@@ -6239,12 +6282,18 @@ static int perf_event_set_output(struct perf_event *event, · 51 unmodified lines
6239	6282	static int perf_event_set_filter(struct perf_event event, void __user arg);
6240	6283	static int perf_copy_attr(struct perf_event_attr __user *uattr,
6241	6284	struct perf_event_attr *attr);
	6285	+static int __perf_event_set_bpf_prog(struct perf_event *event,
	6286	+ struct bpf_prog *prog,
	6287	+ u64 bpf_cookie);
6242	6288
6243	6289	static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned long arg)
6244	6290	{
6245	6291	void (func)(struct perf_event );
6246	6292	u32 flags = arg;
6247	6293
	6294	+ if (event->state <= PERF_EVENT_STATE_REVOKED)
	6295	+ return -ENODEV;
	6296	+
6248	6297	switch (cmd) {
6249	6298	case PERF_EVENT_IOC_ENABLE:
6250	6299	func = _perf_event_enable;
@@ -6301,7 +6350,7 @@ static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned lon · 50 unmodified lines
6301	6350	if (IS_ERR(prog))
6302	6351	return PTR_ERR(prog);
6303	6352
6304		− err = perf_event_set_bpf_prog(event, prog, 0);
	6353	+ err = __perf_event_set_bpf_prog(event, prog, 0);
6305	6354	if (err) {
6306	6355	bpf_prog_put(prog);
6307	6356	return err;
@@ -6620,9 +6669,22 @@ void ring_buffer_put(struct perf_buffer *rb) · 312 unmodified lines
6620	6669	call_rcu(&rb->rcu_head, rb_free_rcu);
6621	6670	}
6622	6671
	6672	+typedef void (mapped_f)(struct perf_event event, struct mm_struct *mm);
	6673	+
	6674	+#define get_mapped(event, func) \
	6675	+({ struct pmu *pmu; \
	6676	+ mapped_f f = NULL; \
	6677	+ guard(rcu)(); \
	6678	+ pmu = READ_ONCE(event->pmu); \
	6679	+ if (pmu) \
	6680	+ f = pmu->func; \
	6681	+ f; \
	6682	+})
	6683	+
6623	6684	static void perf_mmap_open(struct vm_area_struct *vma)
6624	6685	{
6625	6686	struct perf_event *event = vma->vm_file->private_data;
	6687	+ mapped_f mapped = get_mapped(event, event_mapped);
6626	6688
6627	6689	atomic_inc(&event->mmap_count);
6628	6690	atomic_inc(&event->rb->mmap_count);
@@ -6630,8 +6692,8 @@ static void perf_mmap_open(struct vm_area_struct *vma) · 1 unmodified line
6630	6692	if (vma->vm_pgoff)
6631	6693	atomic_inc(&event->rb->aux_mmap_count);
6632	6694
6633		− if (event->pmu->event_mapped)
6634		− event->pmu->event_mapped(event, vma->vm_mm);
	6695	+ if (mapped)
	6696	+ mapped(event, vma->vm_mm);
6635	6697	}
6636	6698
6637	6699	static void perf_pmu_output_stop(struct perf_event *event);
@@ -6647,14 +6709,16 @@ static void perf_pmu_output_stop(struct perf_event *event); · 9 unmodified lines
6647	6709	static void perf_mmap_close(struct vm_area_struct *vma)
6648	6710	{
6649	6711	struct perf_event *event = vma->vm_file->private_data;
	6712	+ mapped_f unmapped = get_mapped(event, event_unmapped);
6650	6713	struct perf_buffer *rb = ring_buffer_get(event);
6651	6714	struct user_struct *mmap_user = rb->mmap_user;
6652	6715	int mmap_locked = rb->mmap_locked;
6653	6716	unsigned long size = perf_data_size(rb);
6654	6717	bool detach_rest = false;
6655	6718
6656		− if (event->pmu->event_unmapped)
6657		− event->pmu->event_unmapped(event, vma->vm_mm);
	6719	+ /* FIXIES vs perf_pmu_unregister() */
	6720	+ if (unmapped)
	6721	+ unmapped(event, vma->vm_mm);
6658	6722
6659	6723	/*
6660	6724	* The AUX buffer is strictly a sub-buffer, serialize using aux_mutex
@@ -6847,6 +6911,7 @@ static int perf_mmap(struct file file, struct vm_area_struct vma) · 186 unmodified lines
6847	6911	unsigned long nr_pages;
6848	6912	long user_extra = 0, extra = 0;
6849	6913	int ret, flags = 0;
	6914	+ mapped_f mapped;
6850	6915
6851	6916	/*
6852	6917	* Don't allow mmap() of inherited per-task counters. This would
@@ -6877,6 +6942,16 @@ static int perf_mmap(struct file file, struct vm_area_struct vma) · 24 unmodified lines
6877	6942	mutex_lock(&event->mmap_mutex);
6878	6943	ret = -EINVAL;
6879	6944
	6945	+ /*
	6946	+ * This relies on __pmu_detach_event() taking mmap_mutex after marking
	6947	+ * the event REVOKED. Either we observe the state, or __pmu_detach_event()
	6948	+ * will detach the rb created here.
	6949	+ */
	6950	+ if (event->state <= PERF_EVENT_STATE_REVOKED) {
	6951	+ ret = -ENODEV;
	6952	+ goto unlock;
	6953	+ }
	6954	+
6880	6955	if (vma->vm_pgoff == 0) {
6881	6956	nr_pages -= 1;
6882	6957
@@ -7055,8 +7130,9 @@ aux_unlock: · 172 unmodified lines
7055	7130	if (!ret)
7056	7131	ret = map_range(rb, vma);
7057	7132
7058		− if (!ret && event->pmu->event_mapped)
7059		− event->pmu->event_mapped(event, vma->vm_mm);
	7133	+ mapped = get_mapped(event, event_mapped);
	7134	+ if (mapped)
	7135	+ mapped(event, vma->vm_mm);
7060	7136
7061	7137	return ret;
7062	7138	}
@@ -7067,6 +7143,9 @@ static int perf_fasync(int fd, struct file *filp, int on) · 4 unmodified lines
7067	7143	struct perf_event *event = filp->private_data;
7068	7144	int retval;
7069	7145
	7146	+ if (event->state <= PERF_EVENT_STATE_REVOKED)
	7147	+ return -ENODEV;
	7148	+
7070	7149	inode_lock(inode);
7071	7150	retval = fasync_helper(fd, filp, on, &event->fasync);
7072	7151	inode_unlock(inode);
@@ -9946,7 +10025,7 @@ void perf_event_text_poke(const void addr, const void old_bytes, · 2873 unmodified lines
9946	10025
9947	10026	void perf_event_itrace_started(struct perf_event *event)
9948	10027	{
9949		− event->attach_state \|= PERF_ATTACH_ITRACE;
	10028	+ WRITE_ONCE(event->attach_state, event->attach_state \| PERF_ATTACH_ITRACE);
9950	10029	}
9951	10030
9952	10031	static void perf_log_itrace_start(struct perf_event *event)
@@ -10029,14 +10108,13 @@ __perf_event_account_interrupt(struct perf_event *event, int throttle) · 76 unmodified lines
10029	10108	hwc->interrupts = 1;
10030	10109	} else {
10031	10110	hwc->interrupts++;
10032		− if (unlikely(throttle &&
10033		− hwc->interrupts > max_samples_per_tick)) {
10034		− __this_cpu_inc(perf_throttled_count);
10035		− tick_dep_set_cpu(smp_processor_id(), TICK_DEP_BIT_PERF_EVENTS);
10036		− hwc->interrupts = MAX_INTERRUPTS;
10037		− perf_log_throttle(event, 0);
10038		− ret = 1;
10039		− }
	10111	+ }
	10112	+
	10113	+ if (unlikely(throttle && hwc->interrupts >= max_samples_per_tick)) {
	10114	+ __this_cpu_inc(perf_throttled_count);
	10115	+ tick_dep_set_cpu(smp_processor_id(), TICK_DEP_BIT_PERF_EVENTS);
	10116	+ perf_event_throttle_group(event);
	10117	+ ret = 1;
10040	10118	}
10041	10119
10042	10120	if (event->attr.freq) {
@@ -11069,11 +11147,15 @@ static inline bool perf_event_is_tracing(struct perf_event *event) · 1026 unmodified lines
11069	11147	return false;
11070	11148	}
11071	11149
11072		−int perf_event_set_bpf_prog(struct perf_event event, struct bpf_prog prog,
11073		− u64 bpf_cookie)
	11150	+static int __perf_event_set_bpf_prog(struct perf_event *event,
	11151	+ struct bpf_prog *prog,
	11152	+ u64 bpf_cookie)
11074	11153	{
11075	11154	bool is_kprobe, is_uprobe, is_tracepoint, is_syscall_tp;
11076	11155
	11156	+ if (event->state <= PERF_EVENT_STATE_REVOKED)
	11157	+ return -ENODEV;
	11158	+
11077	11159	if (!perf_event_is_tracing(event))
11078	11160	return perf_event_set_bpf_handler(event, prog, bpf_cookie);
11079	11161
@@ -11108,6 +11190,20 @@ int perf_event_set_bpf_prog(struct perf_event event, struct bpf_prog prog, · 28 unmodified lines
11108	11190	return perf_event_attach_bpf_prog(event, prog, bpf_cookie);
11109	11191	}
11110	11192
	11193	+int perf_event_set_bpf_prog(struct perf_event *event,
	11194	+ struct bpf_prog *prog,
	11195	+ u64 bpf_cookie)
	11196	+{
	11197	+ struct perf_event_context *ctx;
	11198	+ int ret;
	11199	+
	11200	+ ctx = perf_event_ctx_lock(event);
	11201	+ ret = __perf_event_set_bpf_prog(event, prog, bpf_cookie);
	11202	+ perf_event_ctx_unlock(event, ctx);
	11203	+
	11204	+ return ret;
	11205	+}
	11206	+
11111	11207	void perf_event_free_bpf_prog(struct perf_event *event)
11112	11208	{
11113	11209	if (!event->prog)
@@ -11130,7 +11226,15 @@ static void perf_event_free_filter(struct perf_event *event) · 16 unmodified lines
11130	11226	{
11131	11227	}
11132	11228
11133		−int perf_event_set_bpf_prog(struct perf_event event, struct bpf_prog prog,
	11229	+static int __perf_event_set_bpf_prog(struct perf_event *event,
	11230	+ struct bpf_prog *prog,
	11231	+ u64 bpf_cookie)
	11232	+{
	11233	+ return -ENOENT;
	11234	+}
	11235	+
	11236	+int perf_event_set_bpf_prog(struct perf_event *event,
	11237	+ struct bpf_prog *prog,
11134	11238	u64 bpf_cookie)
11135	11239	{
11136	11240	return -ENOENT;
@@ -12235,6 +12339,9 @@ int perf_pmu_register(struct pmu _pmu, const char name, int type) · 1098 unmodified lines
12235	12339	if (!pmu->event_idx)
12236	12340	pmu->event_idx = perf_event_idx_default;
12237	12341
	12342	+ INIT_LIST_HEAD(&pmu->events);
	12343	+ spin_lock_init(&pmu->events_lock);
	12344	+
12238	12345	/*
12239	12346	* Now that the PMU is complete, make it visible to perf_try_init_event().
12240	12347	*/
@@ -12248,21 +12355,143 @@ int perf_pmu_register(struct pmu _pmu, const char name, int type) · 7 unmodified lines
12248	12355	}
12249	12356	EXPORT_SYMBOL_GPL(perf_pmu_register);
12250	12357
12251		−void perf_pmu_unregister(struct pmu *pmu)
	12358	+static void __pmu_detach_event(struct pmu pmu, struct perf_event event,
	12359	+ struct perf_event_context *ctx)
	12360	+{
	12361	+ /*
	12362	+ * De-schedule the event and mark it REVOKED.
	12363	+ */
	12364	+ perf_event_exit_event(event, ctx, true);
	12365	+
	12366	+ /*
	12367	+ * All _free_event() bits that rely on event->pmu:
	12368	+ *
	12369	+ * Notably, perf_mmap() relies on the ordering here.
	12370	+ */
	12371	+ scoped_guard (mutex, &event->mmap_mutex) {
	12372	+ WARN_ON_ONCE(pmu->event_unmapped);
	12373	+ /*
	12374	+ * Mostly an empty lock sequence, such that perf_mmap(), which
	12375	+ * relies on mmap_mutex, is sure to observe the state change.
	12376	+ */
	12377	+ }
	12378	+
	12379	+ perf_event_free_bpf_prog(event);
	12380	+ perf_free_addr_filters(event);
	12381	+
	12382	+ if (event->destroy) {
	12383	+ event->destroy(event);
	12384	+ event->destroy = NULL;
	12385	+ }
	12386	+
	12387	+ if (event->pmu_ctx) {
	12388	+ put_pmu_ctx(event->pmu_ctx);
	12389	+ event->pmu_ctx = NULL;
	12390	+ }
	12391	+
	12392	+ exclusive_event_destroy(event);
	12393	+ module_put(pmu->module);
	12394	+
	12395	+ event->pmu = NULL; /* force fault instead of UAF */
	12396	+}
	12397	+
	12398	+static void pmu_detach_event(struct pmu pmu, struct perf_event event)
	12399	+{
	12400	+ struct perf_event_context *ctx;
	12401	+
	12402	+ ctx = perf_event_ctx_lock(event);
	12403	+ __pmu_detach_event(pmu, event, ctx);
	12404	+ perf_event_ctx_unlock(event, ctx);
	12405	+
	12406	+ scoped_guard (spinlock, &pmu->events_lock)
	12407	+ list_del(&event->pmu_list);
	12408	+}
	12409	+
	12410	+static struct perf_event pmu_get_event(struct pmu pmu)
	12411	+{
	12412	+ struct perf_event *event;
	12413	+
	12414	+ guard(spinlock)(&pmu->events_lock);
	12415	+ list_for_each_entry(event, &pmu->events, pmu_list) {
	12416	+ if (atomic_long_inc_not_zero(&event->refcount))
	12417	+ return event;
	12418	+ }
	12419	+
	12420	+ return NULL;
	12421	+}
	12422	+
	12423	+static bool pmu_empty(struct pmu *pmu)
	12424	+{
	12425	+ guard(spinlock)(&pmu->events_lock);
	12426	+ return list_empty(&pmu->events);
	12427	+}
	12428	+
	12429	+static void pmu_detach_events(struct pmu *pmu)
	12430	+{
	12431	+ struct perf_event *event;
	12432	+
	12433	+ for (;;) {
	12434	+ event = pmu_get_event(pmu);
	12435	+ if (!event)
	12436	+ break;
	12437	+
	12438	+ pmu_detach_event(pmu, event);
	12439	+ put_event(event);
	12440	+ }
	12441	+
	12442	+ /*
	12443	+ * wait for pending _free_event()s
	12444	+ */
	12445	+ wait_var_event(pmu, pmu_empty(pmu));
	12446	+}
	12447	+
	12448	+int perf_pmu_unregister(struct pmu *pmu)
12252	12449	{
12253	12450	scoped_guard (mutex, &pmus_lock) {
	12451	+ if (!idr_cmpxchg(&pmu_idr, pmu->type, pmu, NULL))
	12452	+ return -EINVAL;
	12453	+
12254	12454	list_del_rcu(&pmu->entry);
12255		− idr_remove(&pmu_idr, pmu->type);
12256	12455	}
12257	12456
12258	12457	/*
12259	12458	* We dereference the pmu list under both SRCU and regular RCU, so
12260	12459	* synchronize against both of those.
	12460	+ *
	12461	+ * Notably, the entirety of event creation, from perf_init_event()
	12462	+ * (which will now fail, because of the above) until
	12463	+ * perf_install_in_context() should be under SRCU such that
	12464	+ * this synchronizes against event creation. This avoids trying to
	12465	+ * detach events that are not fully formed.
12261	12466	*/
12262	12467	synchronize_srcu(&pmus_srcu);
12263	12468	synchronize_rcu();
12264	12469
	12470	+ if (pmu->event_unmapped && !pmu_empty(pmu)) {
	12471	+ /*
	12472	+ * Can't force remove events when pmu::event_unmapped()
	12473	+ * is used in perf_mmap_close().
	12474	+ */
	12475	+ guard(mutex)(&pmus_lock);
	12476	+ idr_cmpxchg(&pmu_idr, pmu->type, NULL, pmu);
	12477	+ list_add_rcu(&pmu->entry, &pmus);
	12478	+ return -EBUSY;
	12479	+ }
	12480	+
	12481	+ scoped_guard (mutex, &pmus_lock)
	12482	+ idr_remove(&pmu_idr, pmu->type);
	12483	+
	12484	+ /*
	12485	+ * PMU is removed from the pmus list, so no new events will
	12486	+ * be created, now take care of the existing ones.
	12487	+ */
	12488	+ pmu_detach_events(pmu);
	12489	+
	12490	+ /*
	12491	+ * PMU is unused, make it go away.
	12492	+ */
12265	12493	perf_pmu_free(pmu);
	12494	+ return 0;
12266	12495	}
12267	12496	EXPORT_SYMBOL_GPL(perf_pmu_unregister);
12268	12497
@@ -12356,7 +12585,7 @@ static struct pmu perf_init_event(struct perf_event event) · 87 unmodified lines
12356	12585	struct pmu *pmu;
12357	12586	int type, ret;
12358	12587
12359		− guard(srcu)(&pmus_srcu);
	12588	+ guard(srcu)(&pmus_srcu); /* pmu idr/list access */
12360	12589
12361	12590	/*
12362	12591	* Save original type before calling pmu->event_init() since certain
@@ -12580,6 +12809,7 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu, · 217 unmodified lines
12580	12809	INIT_LIST_HEAD(&event->active_entry);
12581	12810	INIT_LIST_HEAD(&event->addr_filters.list);
12582	12811	INIT_HLIST_NODE(&event->hlist_entry);
	12812	+ INIT_LIST_HEAD(&event->pmu_list);
12583	12813
12584	12814
12585	12815	init_waitqueue_head(&event->waitq);
@@ -12651,7 +12881,7 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu, · 65 unmodified lines
12651	12881
12652	12882	hwc = &event->hw;
12653	12883	hwc->sample_period = attr->sample_period;
12654		− if (attr->freq && attr->sample_freq)
	12884	+ if (is_event_in_freq_mode(event))
12655	12885	hwc->sample_period = 1;
12656	12886	hwc->last_period = hwc->sample_period;
12657	12887
@@ -12758,6 +12988,13 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu, · 100 unmodified lines
12758	12988	/* symmetric to unaccount_event() in _free_event() */
12759	12989	account_event(event);
12760	12990
	12991	+ /*
	12992	+ * Event creation should be under SRCU, see perf_pmu_unregister().
	12993	+ */
	12994	+ lockdep_assert_held(&pmus_srcu);
	12995	+ scoped_guard (spinlock, &pmu->events_lock)
	12996	+ list_add(&event->pmu_list, &pmu->events);
	12997	+
12761	12998	return_ptr(event);
12762	12999	}
12763	13000
@@ -12957,6 +13194,9 @@ set: · 193 unmodified lines
12957	13194	goto unlock;
12958	13195
12959	13196	if (output_event) {
	13197	+ if (output_event->state <= PERF_EVENT_STATE_REVOKED)
	13198	+ goto unlock;
	13199	+
12960	13200	/* get the rb we want to redirect to */
12961	13201	rb = ring_buffer_get(output_event);
12962	13202	if (!rb)
@@ -13138,6 +13378,11 @@ SYSCALL_DEFINE5(perf_event_open, · 175 unmodified lines
13138	13378	if (event_fd < 0)
13139	13379	return event_fd;
13140	13380
	13381	+ /*
	13382	+ * Event creation should be under SRCU, see perf_pmu_unregister().
	13383	+ */
	13384	+ guard(srcu)(&pmus_srcu);
	13385	+
13141	13386	CLASS(fd, group)(group_fd); // group_fd == -1 => empty
13142	13387	if (group_fd != -1) {
13143	13388	if (!is_perf_file(group)) {
@@ -13145,6 +13390,10 @@ SYSCALL_DEFINE5(perf_event_open, · 1 unmodified line
13145	13390	goto err_fd;
13146	13391	}
13147	13392	group_leader = fd_file(group)->private_data;
	13393	+ if (group_leader->state <= PERF_EVENT_STATE_REVOKED) {
	13394	+ err = -ENODEV;
	13395	+ goto err_fd;
	13396	+ }
13148	13397	if (flags & PERF_FLAG_FD_OUTPUT)
13149	13398	output_event = group_leader;
13150	13399	if (flags & PERF_FLAG_FD_NO_GROUP)
@@ -13441,7 +13690,7 @@ err_cred: · 290 unmodified lines
13441	13690	if (task)
13442	13691	up_read(&task->signal->exec_update_lock);
13443	13692	err_alloc:
13444		− free_event(event);
	13693	+ put_event(event);
13445	13694	err_task:
13446	13695	if (task)
13447	13696	put_task_struct(task);
@@ -13478,6 +13727,11 @@ perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu, · 30 unmodified lines
13478	13727	if (attr->aux_output \|\| attr->aux_action)
13479	13728	return ERR_PTR(-EINVAL);
13480	13729
	13730	+ /*
	13731	+ * Event creation should be under SRCU, see perf_pmu_unregister().
	13732	+ */
	13733	+ guard(srcu)(&pmus_srcu);
	13734	+
13481	13735	event = perf_event_alloc(attr, cpu, task, NULL, NULL,
13482	13736	overflow_handler, context, -1);
13483	13737	if (IS_ERR(event)) {
@@ -13549,7 +13803,7 @@ err_unlock: · 65 unmodified lines
13549	13803	perf_unpin_context(ctx);
13550	13804	put_ctx(ctx);
13551	13805	err_alloc:
13552		− free_event(event);
	13806	+ put_event(event);
13553	13807	err:
13554	13808	return ERR_PTR(err);
13555	13809	}
@@ -13689,10 +13943,12 @@ static void sync_child_event(struct perf_event *child_event) · 133 unmodified lines
13689	13943	}
13690	13944
13691	13945	static void
13692		−perf_event_exit_event(struct perf_event event, struct perf_event_context ctx)
	13946	+perf_event_exit_event(struct perf_event *event,
	13947	+ struct perf_event_context *ctx, bool revoke)
13693	13948	{
13694	13949	struct perf_event *parent_event = event->parent;
13695		− unsigned long detach_flags = 0;
	13950	+ unsigned long detach_flags = DETACH_EXIT;
	13951	+ unsigned int attach_state;
13696	13952
13697	13953	if (parent_event) {
13698	13954	/*
@@ -13707,22 +13963,38 @@ perf_event_exit_event(struct perf_event event, struct perf_event_context ctx) · 8 unmodified lines
13707	13963	* Do destroy all inherited groups, we don't care about those
13708	13964	* and being thorough is better.
13709	13965	*/
13710		− detach_flags = DETACH_GROUP \| DETACH_CHILD;
	13966	+ detach_flags \|= DETACH_GROUP \| DETACH_CHILD;
13711	13967	mutex_lock(&parent_event->child_mutex);
	13968	+ /* PERF_ATTACH_ITRACE might be set concurrently */
	13969	+ attach_state = READ_ONCE(event->attach_state);
13712	13970	}
13713	13971
13714		− perf_remove_from_context(event, detach_flags \| DETACH_EXIT);
	13972	+ if (revoke)
	13973	+ detach_flags \|= DETACH_GROUP \| DETACH_REVOKE;
13715	13974
	13975	+ perf_remove_from_context(event, detach_flags);
13716	13976	/*
13717	13977	* Child events can be freed.
13718	13978	*/
13719	13979	if (parent_event) {
13720	13980	mutex_unlock(&parent_event->child_mutex);
	13981	+
13721	13982	/*
13722		− * Kick perf_poll() for is_event_hup();
	13983	+ * Match the refcount initialization. Make sure it doesn't happen
	13984	+ * twice if pmu_detach_event() calls it on an already exited task.
13723	13985	*/
13724		− perf_event_wakeup(parent_event);
13725		− put_event(event);
	13986	+ if (attach_state & PERF_ATTACH_CHILD) {
	13987	+ /*
	13988	+ * Kick perf_poll() for is_event_hup();
	13989	+ */
	13990	+ perf_event_wakeup(parent_event);
	13991	+ /*
	13992	+ * pmu_detach_event() will have an extra refcount.
	13993	+ * perf_pending_task() might have one too.
	13994	+ */
	13995	+ put_event(event);
	13996	+ }
	13997	+
13726	13998	return;
13727	13999	}
13728	14000
@@ -13732,15 +14004,13 @@ perf_event_exit_event(struct perf_event event, struct perf_event_context ctx) · 3 unmodified lines
13732	14004	perf_event_wakeup(event);
13733	14005	}
13734	14006
13735		−static void perf_event_exit_task_context(struct task_struct *child)
	14007	+static void perf_event_exit_task_context(struct task_struct *task, bool exit)
13736	14008	{
13737		− struct perf_event_context child_ctx, clone_ctx = NULL;
	14009	+ struct perf_event_context ctx, clone_ctx = NULL;
13738	14010	struct perf_event child_event, next;
13739	14011
13740		− WARN_ON_ONCE(child != current);
13741		−
13742		− child_ctx = perf_pin_task_context(child);
13743		− if (!child_ctx)
	14012	+ ctx = perf_pin_task_context(task);
	14013	+ if (!ctx)
13744	14014	return;
13745	14015
13746	14016	/*
@@ -13753,27 +14023,28 @@ static void perf_event_exit_task_context(struct task_struct *child) · 6 unmodified lines
13753	14023	* without ctx::mutex (it cannot because of the move_group double mutex
13754	14024	* lock thing). See the comments in perf_install_in_context().
13755	14025	*/
13756		− mutex_lock(&child_ctx->mutex);
	14026	+ mutex_lock(&ctx->mutex);
13757	14027
13758	14028	/*
13759	14029	* In a single ctx::lock section, de-schedule the events and detach the
13760	14030	* context from the task such that we cannot ever get it scheduled back
13761	14031	* in.
13762	14032	*/
13763		− raw_spin_lock_irq(&child_ctx->lock);
13764		− task_ctx_sched_out(child_ctx, NULL, EVENT_ALL);
	14033	+ raw_spin_lock_irq(&ctx->lock);
	14034	+ if (exit)
	14035	+ task_ctx_sched_out(ctx, NULL, EVENT_ALL);
13765	14036
13766	14037	/*
13767	14038	* Now that the context is inactive, destroy the task <-> ctx relation
13768	14039	* and mark the context dead.
13769	14040	*/
13770		− RCU_INIT_POINTER(child->perf_event_ctxp, NULL);
13771		− put_ctx(child_ctx); /* cannot be last */
13772		− WRITE_ONCE(child_ctx->task, TASK_TOMBSTONE);
13773		− put_task_struct(current); /* cannot be last */
	14041	+ RCU_INIT_POINTER(task->perf_event_ctxp, NULL);
	14042	+ put_ctx(ctx); /* cannot be last */
	14043	+ WRITE_ONCE(ctx->task, TASK_TOMBSTONE);
	14044	+ put_task_struct(task); /* cannot be last */
13774	14045
13775		− clone_ctx = unclone_ctx(child_ctx);
13776		− raw_spin_unlock_irq(&child_ctx->lock);
	14046	+ clone_ctx = unclone_ctx(ctx);
	14047	+ raw_spin_unlock_irq(&ctx->lock);
13777	14048
13778	14049	if (clone_ctx)
13779	14050	put_ctx(clone_ctx);
@@ -13783,28 +14054,48 @@ static void perf_event_exit_task_context(struct task_struct *child) · 3 unmodified lines
13783	14054	* won't get any samples after PERF_RECORD_EXIT. We can however still
13784	14055	* get a few PERF_RECORD_READ events.
13785	14056	*/
13786		− perf_event_task(child, child_ctx, 0);
	14057	+ if (exit)
	14058	+ perf_event_task(task, ctx, 0);
13787	14059
13788		− list_for_each_entry_safe(child_event, next, &child_ctx->event_list, event_entry)
13789		− perf_event_exit_event(child_event, child_ctx);
	14060	+ list_for_each_entry_safe(child_event, next, &ctx->event_list, event_entry)
	14061	+ perf_event_exit_event(child_event, ctx, false);
13790	14062
13791		− mutex_unlock(&child_ctx->mutex);
	14063	+ mutex_unlock(&ctx->mutex);
13792	14064
13793		− put_ctx(child_ctx);
	14065	+ if (!exit) {
	14066	+ /*
	14067	+ * perf_event_release_kernel() could still have a reference on
	14068	+ * this context. In that case we must wait for these events to
	14069	+ * have been freed (in particular all their references to this
	14070	+ * task must've been dropped).
	14071	+ *
	14072	+ * Without this copy_process() will unconditionally free this
	14073	+ * task (irrespective of its reference count) and
	14074	+ * _free_event()'s put_task_struct(event->hw.target) will be a
	14075	+ * use-after-free.
	14076	+ *
	14077	+ * Wait for all events to drop their context reference.
	14078	+ */
	14079	+ wait_var_event(&ctx->refcount,
	14080	+ refcount_read(&ctx->refcount) == 1);
	14081	+ }
	14082	+ put_ctx(ctx);
13794	14083	}
13795	14084
13796	14085	/*
13797		− * When a child task exits, feed back event values to parent events.
	14086	+ * When a task exits, feed back event values to parent events.
13798	14087	*
13799	14088	* Can be called with exec_update_lock held when called from
13800	14089	* setup_new_exec().
13801	14090	*/
13802		−void perf_event_exit_task(struct task_struct *child)
	14091	+void perf_event_exit_task(struct task_struct *task)
13803	14092	{
13804	14093	struct perf_event event, tmp;
13805	14094
13806		− mutex_lock(&child->perf_event_mutex);
13807		− list_for_each_entry_safe(event, tmp, &child->perf_event_list,
	14095	+ WARN_ON_ONCE(task != current);
	14096	+
	14097	+ mutex_lock(&task->perf_event_mutex);
	14098	+ list_for_each_entry_safe(event, tmp, &task->perf_event_list,
13808	14099	owner_entry) {
13809	14100	list_del_init(&event->owner_entry);
13810	14101
@@ -13815,42 +14106,23 @@ void perf_event_exit_task(struct task_struct *child) · 4 unmodified lines
13815	14106	*/
13816	14107	smp_store_release(&event->owner, NULL);
13817	14108	}
13818		− mutex_unlock(&child->perf_event_mutex);
	14109	+ mutex_unlock(&task->perf_event_mutex);
13819	14110
13820		− perf_event_exit_task_context(child);
	14111	+ perf_event_exit_task_context(task, true);
13821	14112
13822	14113	/*
13823	14114	* The perf_event_exit_task_context calls perf_event_task
13824		− * with child's task_ctx, which generates EXIT events for
13825		− * child contexts and sets child->perf_event_ctxp[] to NULL.
	14115	+ * with task's task_ctx, which generates EXIT events for
	14116	+ * task contexts and sets task->perf_event_ctxp[] to NULL.
13826	14117	* At this point we need to send EXIT events to cpu contexts.
13827	14118	*/
13828		− perf_event_task(child, NULL, 0);
	14119	+ perf_event_task(task, NULL, 0);
13829	14120
13830	14121	/*
13831	14122	* Detach the perf_ctx_data for the system-wide event.
13832	14123	*/
13833	14124	guard(percpu_read)(&global_ctx_data_rwsem);
13834		− detach_task_ctx_data(child);
13835		−}
13836		−
13837		−static void perf_free_event(struct perf_event *event,
13838		− struct perf_event_context *ctx)
13839		−{
13840		− struct perf_event *parent = event->parent;
13841		−
13842		− if (WARN_ON_ONCE(!parent))
13843		− return;
13844		−
13845		− mutex_lock(&parent->child_mutex);
13846		− list_del_init(&event->child_list);
13847		− mutex_unlock(&parent->child_mutex);
13848		−
13849		− raw_spin_lock_irq(&ctx->lock);
13850		− perf_group_detach(event);
13851		− list_del_event(event, ctx);
13852		− raw_spin_unlock_irq(&ctx->lock);
13853		− put_event(event);
	14125	+ detach_task_ctx_data(task);
13854	14126	}
13855	14127
13856	14128	/*
@@ -13862,48 +14134,7 @@ static void perf_free_event(struct perf_event *event, · 5 unmodified lines
13862	14134	*/
13863	14135	void perf_event_free_task(struct task_struct *task)
13864	14136	{
13865		− struct perf_event_context *ctx;
13866		− struct perf_event event, tmp;
13867		−
13868		− ctx = rcu_access_pointer(task->perf_event_ctxp);
13869		− if (!ctx)
13870		− return;
13871		−
13872		− mutex_lock(&ctx->mutex);
13873		− raw_spin_lock_irq(&ctx->lock);
13874		− /*
13875		− * Destroy the task <-> ctx relation and mark the context dead.
13876		− *
13877		− * This is important because even though the task hasn't been
13878		− * exposed yet the context has been (through child_list).
13879		− */
13880		− RCU_INIT_POINTER(task->perf_event_ctxp, NULL);
13881		− WRITE_ONCE(ctx->task, TASK_TOMBSTONE);
13882		− put_task_struct(task); /* cannot be last */
13883		− raw_spin_unlock_irq(&ctx->lock);
13884		−
13885		−
13886		− list_for_each_entry_safe(event, tmp, &ctx->event_list, event_entry)
13887		− perf_free_event(event, ctx);
13888		−
13889		− mutex_unlock(&ctx->mutex);
13890		−
13891		− /*
13892		− * perf_event_release_kernel() could've stolen some of our
13893		− * child events and still have them on its free_list. In that
13894		− * case we must wait for these events to have been freed (in
13895		− * particular all their references to this task must've been
13896		− * dropped).
13897		− *
13898		− * Without this copy_process() will unconditionally free this
13899		− * task (irrespective of its reference count) and
13900		− * _free_event()'s put_task_struct(event->hw.target) will be a
13901		− * use-after-free.
13902		− *
13903		− * Wait for all events to drop their context reference.
13904		− */
13905		− wait_var_event(&ctx->refcount, refcount_read(&ctx->refcount) == 1);
13906		− put_ctx(ctx); /* must be last */
	14137	+ perf_event_exit_task_context(task, false);
13907	14138	}
13908	14139
13909	14140	void perf_event_delayed_put(struct task_struct *task)
@@ -13980,6 +14211,14 @@ inherit_event(struct perf_event *parent_event, · 70 unmodified lines
13980	14211	if (parent_event->parent)
13981	14212	parent_event = parent_event->parent;
13982	14213
	14214	+ if (parent_event->state <= PERF_EVENT_STATE_REVOKED)
	14215	+ return NULL;
	14216	+
	14217	+ /*
	14218	+ * Event creation should be under SRCU, see perf_pmu_unregister().
	14219	+ */
	14220	+ guard(srcu)(&pmus_srcu);
	14221	+
13983	14222	child_event = perf_event_alloc(&parent_event->attr,
13984	14223	parent_event->cpu,
13985	14224	child,

kernel/events/ring_buffer.c+20−9

@@ -679,7 +679,15 @@ int rb_alloc_aux(struct perf_buffer rb, struct perf_event event, · 678 unmodified lines
679	679	{
680	680	bool overwrite = !(flags & RING_BUFFER_WRITABLE);
681	681	int node = (event->cpu == -1) ? -1 : cpu_to_node(event->cpu);
682		− int ret = -ENOMEM, max_order;
	682	+ bool use_contiguous_pages = event->pmu->capabilities & (
	683	+ PERF_PMU_CAP_AUX_NO_SG \| PERF_PMU_CAP_AUX_PREFER_LARGE);
	684	+ /*
	685	+ * Initialize max_order to 0 for page allocation. This allocates single
	686	+ * pages to minimize memory fragmentation. This is overridden if the
	687	+ * PMU needs or prefers contiguous pages (use_contiguous_pages = true).
	688	+ */
	689	+ int max_order = 0;
	690	+ int ret = -ENOMEM;
683	691
684	692	if (!has_aux(event))
685	693	return -EOPNOTSUPP;
@@ -689,8 +697,8 @@ int rb_alloc_aux(struct perf_buffer rb, struct perf_event event, · 3 unmodified lines
689	697
690	698	if (!overwrite) {
691	699	/*
692		− * Watermark defaults to half the buffer, and so does the
693		− * max_order, to aid PMU drivers in double buffering.
	700	+ * Watermark defaults to half the buffer, to aid PMU drivers
	701	+ * in double buffering.
694	702	*/
695	703	if (!watermark)
696	704	watermark = min_t(unsigned long,
@@ -698,16 +706,19 @@ int rb_alloc_aux(struct perf_buffer rb, struct perf_event event, · 1 unmodified line
698	706	(unsigned long)nr_pages << (PAGE_SHIFT - 1));
699	707
700	708	/*
701		− * Use aux_watermark as the basis for chunking to
702		− * help PMU drivers honor the watermark.
	709	+ * If using contiguous pages, use aux_watermark as the basis
	710	+ * for chunking to help PMU drivers honor the watermark.
703	711	*/
704		− max_order = get_order(watermark);
	712	+ if (use_contiguous_pages)
	713	+ max_order = get_order(watermark);
705	714	} else {
706	715	/*
707		− * We need to start with the max_order that fits in nr_pages,
708		− * not the other way around, hence ilog2() and not get_order.
	716	+ * If using contiguous pages, we need to start with the
	717	+ * max_order that fits in nr_pages, not the other way around,
	718	+ * hence ilog2() and not get_order.
709	719	*/
710		− max_order = ilog2(nr_pages);
	720	+ if (use_contiguous_pages)
	721	+ max_order = ilog2(nr_pages);
711	722	watermark = 0;
712	723	}
713	724

tools/include/uapi/linux/perf_event.h+334−323

@@ -39,18 +39,21 @@ enum perf_type_id { · 38 unmodified lines
39	39
40	40	/*
41	41	* attr.config layout for type PERF_TYPE_HARDWARE and PERF_TYPE_HW_CACHE
	42	+ *
42	43	* PERF_TYPE_HARDWARE: 0xEEEEEEEE000000AA
43	44	* AA: hardware event ID
44	45	* EEEEEEEE: PMU type ID
	46	+ *
45	47	* PERF_TYPE_HW_CACHE: 0xEEEEEEEE00DDCCBB
46	48	* BB: hardware cache ID
47	49	* CC: hardware cache op ID
48	50	* DD: hardware cache op result ID
49	51	* EEEEEEEE: PMU type ID
50		− * If the PMU type ID is 0, the PERF_TYPE_RAW will be applied.
	52	+ *
	53	+ * If the PMU type ID is 0, PERF_TYPE_RAW will be applied.
51	54	*/
52		−#define PERF_PMU_TYPE_SHIFT 32
53		−#define PERF_HW_EVENT_MASK 0xffffffff
	55	+#define PERF_PMU_TYPE_SHIFT 32
	56	+#define PERF_HW_EVENT_MASK 0xffffffff
54	57
55	58	/*
56	59	* Generalized performance event event_id types, used by the
@@ -112,7 +115,7 @@ enum perf_hw_cache_op_result_id { · 55 unmodified lines
112	115	/*
113	116	* Special "software" events provided by the kernel, even if the hardware
114	117	* does not support performance events. These events measure various
115		− * physical and sw events of the kernel (and allow the profiling of them as
	118	+ * physical and SW events of the kernel (and allow the profiling of them as
116	119	* well):
117	120	*/
118	121	enum perf_sw_ids {
@@ -167,8 +170,9 @@ enum perf_event_sample_format { · 48 unmodified lines
167	170	};
168	171
169	172	#define PERF_SAMPLE_WEIGHT_TYPE (PERF_SAMPLE_WEIGHT \| PERF_SAMPLE_WEIGHT_STRUCT)
	173	+
170	174	/*
171		− * values to program into branch_sample_type when PERF_SAMPLE_BRANCH is set
	175	+ * Values to program into branch_sample_type when PERF_SAMPLE_BRANCH is set.
172	176	*
173	177	* If the user does not pass priv level information via branch_sample_type,
174	178	* the kernel uses the event's priv level. Branch and event priv levels do
@@ -178,20 +182,20 @@ enum perf_event_sample_format { · 3 unmodified lines
178	182	* of branches and therefore it supersedes all the other types.
179	183	*/
180	184	enum perf_branch_sample_type_shift {
181		− PERF_SAMPLE_BRANCH_USER_SHIFT = 0, /* user branches */
182		− PERF_SAMPLE_BRANCH_KERNEL_SHIFT = 1, /* kernel branches */
183		− PERF_SAMPLE_BRANCH_HV_SHIFT = 2, /* hypervisor branches */
184		−
185		− PERF_SAMPLE_BRANCH_ANY_SHIFT = 3, /* any branch types */
186		− PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT = 4, /* any call branch */
187		− PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT = 5, /* any return branch */
188		− PERF_SAMPLE_BRANCH_IND_CALL_SHIFT = 6, /* indirect calls */
189		− PERF_SAMPLE_BRANCH_ABORT_TX_SHIFT = 7, /* transaction aborts */
190		− PERF_SAMPLE_BRANCH_IN_TX_SHIFT = 8, /* in transaction */
191		− PERF_SAMPLE_BRANCH_NO_TX_SHIFT = 9, /* not in transaction */
	185	+ PERF_SAMPLE_BRANCH_USER_SHIFT = 0, /* user branches */
	186	+ PERF_SAMPLE_BRANCH_KERNEL_SHIFT = 1, /* kernel branches */
	187	+ PERF_SAMPLE_BRANCH_HV_SHIFT = 2, /* hypervisor branches */
	188	+
	189	+ PERF_SAMPLE_BRANCH_ANY_SHIFT = 3, /* any branch types */
	190	+ PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT = 4, /* any call branch */
	191	+ PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT = 5, /* any return branch */
	192	+ PERF_SAMPLE_BRANCH_IND_CALL_SHIFT = 6, /* indirect calls */
	193	+ PERF_SAMPLE_BRANCH_ABORT_TX_SHIFT = 7, /* transaction aborts */
	194	+ PERF_SAMPLE_BRANCH_IN_TX_SHIFT = 8, /* in transaction */
	195	+ PERF_SAMPLE_BRANCH_NO_TX_SHIFT = 9, /* not in transaction */
192	196	PERF_SAMPLE_BRANCH_COND_SHIFT = 10, /* conditional branches */
193	197
194		− PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT = 11, /* call/ret stack */
	198	+ PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT = 11, /* CALL/RET stack */
195	199	PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT = 12, /* indirect jumps */
196	200	PERF_SAMPLE_BRANCH_CALL_SHIFT = 13, /* direct call */
197	201
@@ -210,96 +214,95 @@ enum perf_branch_sample_type_shift { · 12 unmodified lines
210	214	};
211	215
212	216	enum perf_branch_sample_type {
213		− PERF_SAMPLE_BRANCH_USER = 1U << PERF_SAMPLE_BRANCH_USER_SHIFT,
214		− PERF_SAMPLE_BRANCH_KERNEL = 1U << PERF_SAMPLE_BRANCH_KERNEL_SHIFT,
215		− PERF_SAMPLE_BRANCH_HV = 1U << PERF_SAMPLE_BRANCH_HV_SHIFT,
	217	+ PERF_SAMPLE_BRANCH_USER = 1U << PERF_SAMPLE_BRANCH_USER_SHIFT,
	218	+ PERF_SAMPLE_BRANCH_KERNEL = 1U << PERF_SAMPLE_BRANCH_KERNEL_SHIFT,
	219	+ PERF_SAMPLE_BRANCH_HV = 1U << PERF_SAMPLE_BRANCH_HV_SHIFT,
216	220
217		− PERF_SAMPLE_BRANCH_ANY = 1U << PERF_SAMPLE_BRANCH_ANY_SHIFT,
218		− PERF_SAMPLE_BRANCH_ANY_CALL = 1U << PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT,
219		− PERF_SAMPLE_BRANCH_ANY_RETURN = 1U << PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT,
220		− PERF_SAMPLE_BRANCH_IND_CALL = 1U << PERF_SAMPLE_BRANCH_IND_CALL_SHIFT,
221		− PERF_SAMPLE_BRANCH_ABORT_TX = 1U << PERF_SAMPLE_BRANCH_ABORT_TX_SHIFT,
222		− PERF_SAMPLE_BRANCH_IN_TX = 1U << PERF_SAMPLE_BRANCH_IN_TX_SHIFT,
223		− PERF_SAMPLE_BRANCH_NO_TX = 1U << PERF_SAMPLE_BRANCH_NO_TX_SHIFT,
224		− PERF_SAMPLE_BRANCH_COND = 1U << PERF_SAMPLE_BRANCH_COND_SHIFT,
	221	+ PERF_SAMPLE_BRANCH_ANY = 1U << PERF_SAMPLE_BRANCH_ANY_SHIFT,
	222	+ PERF_SAMPLE_BRANCH_ANY_CALL = 1U << PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT,
	223	+ PERF_SAMPLE_BRANCH_ANY_RETURN = 1U << PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT,
	224	+ PERF_SAMPLE_BRANCH_IND_CALL = 1U << PERF_SAMPLE_BRANCH_IND_CALL_SHIFT,
	225	+ PERF_SAMPLE_BRANCH_ABORT_TX = 1U << PERF_SAMPLE_BRANCH_ABORT_TX_SHIFT,
	226	+ PERF_SAMPLE_BRANCH_IN_TX = 1U << PERF_SAMPLE_BRANCH_IN_TX_SHIFT,
	227	+ PERF_SAMPLE_BRANCH_NO_TX = 1U << PERF_SAMPLE_BRANCH_NO_TX_SHIFT,
	228	+ PERF_SAMPLE_BRANCH_COND = 1U << PERF_SAMPLE_BRANCH_COND_SHIFT,
225	229
226		− PERF_SAMPLE_BRANCH_CALL_STACK = 1U << PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT,
227		− PERF_SAMPLE_BRANCH_IND_JUMP = 1U << PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT,
228		− PERF_SAMPLE_BRANCH_CALL = 1U << PERF_SAMPLE_BRANCH_CALL_SHIFT,
	230	+ PERF_SAMPLE_BRANCH_CALL_STACK = 1U << PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT,
	231	+ PERF_SAMPLE_BRANCH_IND_JUMP = 1U << PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT,
	232	+ PERF_SAMPLE_BRANCH_CALL = 1U << PERF_SAMPLE_BRANCH_CALL_SHIFT,
229	233
230		− PERF_SAMPLE_BRANCH_NO_FLAGS = 1U << PERF_SAMPLE_BRANCH_NO_FLAGS_SHIFT,
231		− PERF_SAMPLE_BRANCH_NO_CYCLES = 1U << PERF_SAMPLE_BRANCH_NO_CYCLES_SHIFT,
	234	+ PERF_SAMPLE_BRANCH_NO_FLAGS = 1U << PERF_SAMPLE_BRANCH_NO_FLAGS_SHIFT,
	235	+ PERF_SAMPLE_BRANCH_NO_CYCLES = 1U << PERF_SAMPLE_BRANCH_NO_CYCLES_SHIFT,
232	236
233		− PERF_SAMPLE_BRANCH_TYPE_SAVE =
234		− 1U << PERF_SAMPLE_BRANCH_TYPE_SAVE_SHIFT,
	237	+ PERF_SAMPLE_BRANCH_TYPE_SAVE = 1U << PERF_SAMPLE_BRANCH_TYPE_SAVE_SHIFT,
235	238
236		− PERF_SAMPLE_BRANCH_HW_INDEX = 1U << PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT,
	239	+ PERF_SAMPLE_BRANCH_HW_INDEX = 1U << PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT,
237	240
238		− PERF_SAMPLE_BRANCH_PRIV_SAVE = 1U << PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT,
	241	+ PERF_SAMPLE_BRANCH_PRIV_SAVE = 1U << PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT,
239	242
240		− PERF_SAMPLE_BRANCH_COUNTERS = 1U << PERF_SAMPLE_BRANCH_COUNTERS_SHIFT,
	243	+ PERF_SAMPLE_BRANCH_COUNTERS = 1U << PERF_SAMPLE_BRANCH_COUNTERS_SHIFT,
241	244
242		− PERF_SAMPLE_BRANCH_MAX = 1U << PERF_SAMPLE_BRANCH_MAX_SHIFT,
	245	+ PERF_SAMPLE_BRANCH_MAX = 1U << PERF_SAMPLE_BRANCH_MAX_SHIFT,
243	246	};
244	247
245	248	/*
246		− * Common flow change classification
	249	+ * Common control flow change classifications:
247	250	*/
248	251	enum {
249		− PERF_BR_UNKNOWN = 0, /* unknown */
250		− PERF_BR_COND = 1, /* conditional */
251		− PERF_BR_UNCOND = 2, /* unconditional */
252		− PERF_BR_IND = 3, /* indirect */
253		− PERF_BR_CALL = 4, /* function call */
254		− PERF_BR_IND_CALL = 5, /* indirect function call */
255		− PERF_BR_RET = 6, /* function return */
256		− PERF_BR_SYSCALL = 7, /* syscall */
257		− PERF_BR_SYSRET = 8, /* syscall return */
258		− PERF_BR_COND_CALL = 9, /* conditional function call */
259		− PERF_BR_COND_RET = 10, /* conditional function return */
260		− PERF_BR_ERET = 11, /* exception return */
261		− PERF_BR_IRQ = 12, /* irq */
262		− PERF_BR_SERROR = 13, /* system error */
263		− PERF_BR_NO_TX = 14, /* not in transaction */
264		− PERF_BR_EXTEND_ABI = 15, /* extend ABI */
	252	+ PERF_BR_UNKNOWN = 0, /* Unknown */
	253	+ PERF_BR_COND = 1, /* Conditional */
	254	+ PERF_BR_UNCOND = 2, /* Unconditional */
	255	+ PERF_BR_IND = 3, /* Indirect */
	256	+ PERF_BR_CALL = 4, /* Function call */
	257	+ PERF_BR_IND_CALL = 5, /* Indirect function call */
	258	+ PERF_BR_RET = 6, /* Function return */
	259	+ PERF_BR_SYSCALL = 7, /* Syscall */
	260	+ PERF_BR_SYSRET = 8, /* Syscall return */
	261	+ PERF_BR_COND_CALL = 9, /* Conditional function call */
	262	+ PERF_BR_COND_RET = 10, /* Conditional function return */
	263	+ PERF_BR_ERET = 11, /* Exception return */
	264	+ PERF_BR_IRQ = 12, /* IRQ */
	265	+ PERF_BR_SERROR = 13, /* System error */
	266	+ PERF_BR_NO_TX = 14, /* Not in transaction */
	267	+ PERF_BR_EXTEND_ABI = 15, /* Extend ABI */
265	268	PERF_BR_MAX,
266	269	};
267	270
268	271	/*
269		− * Common branch speculation outcome classification
	272	+ * Common branch speculation outcome classifications:
270	273	*/
271	274	enum {
272		− PERF_BR_SPEC_NA = 0, /* Not available */
273		− PERF_BR_SPEC_WRONG_PATH = 1, /* Speculative but on wrong path */
274		− PERF_BR_NON_SPEC_CORRECT_PATH = 2, /* Non-speculative but on correct path */
275		− PERF_BR_SPEC_CORRECT_PATH = 3, /* Speculative and on correct path */
	275	+ PERF_BR_SPEC_NA = 0, /* Not available */
	276	+ PERF_BR_SPEC_WRONG_PATH = 1, /* Speculative but on wrong path */
	277	+ PERF_BR_NON_SPEC_CORRECT_PATH = 2, /* Non-speculative but on correct path */
	278	+ PERF_BR_SPEC_CORRECT_PATH = 3, /* Speculative and on correct path */
276	279	PERF_BR_SPEC_MAX,
277	280	};
278	281
279	282	enum {
280		− PERF_BR_NEW_FAULT_ALGN = 0, /* Alignment fault */
281		− PERF_BR_NEW_FAULT_DATA = 1, /* Data fault */
282		− PERF_BR_NEW_FAULT_INST = 2, /* Inst fault */
283		− PERF_BR_NEW_ARCH_1 = 3, /* Architecture specific */
284		− PERF_BR_NEW_ARCH_2 = 4, /* Architecture specific */
285		− PERF_BR_NEW_ARCH_3 = 5, /* Architecture specific */
286		− PERF_BR_NEW_ARCH_4 = 6, /* Architecture specific */
287		− PERF_BR_NEW_ARCH_5 = 7, /* Architecture specific */
	283	+ PERF_BR_NEW_FAULT_ALGN = 0, /* Alignment fault */
	284	+ PERF_BR_NEW_FAULT_DATA = 1, /* Data fault */
	285	+ PERF_BR_NEW_FAULT_INST = 2, /* Inst fault */
	286	+ PERF_BR_NEW_ARCH_1 = 3, /* Architecture specific */
	287	+ PERF_BR_NEW_ARCH_2 = 4, /* Architecture specific */
	288	+ PERF_BR_NEW_ARCH_3 = 5, /* Architecture specific */
	289	+ PERF_BR_NEW_ARCH_4 = 6, /* Architecture specific */
	290	+ PERF_BR_NEW_ARCH_5 = 7, /* Architecture specific */
288	291	PERF_BR_NEW_MAX,
289	292	};
290	293
291	294	enum {
292		− PERF_BR_PRIV_UNKNOWN = 0,
293		− PERF_BR_PRIV_USER = 1,
294		− PERF_BR_PRIV_KERNEL = 2,
295		− PERF_BR_PRIV_HV = 3,
	295	+ PERF_BR_PRIV_UNKNOWN = 0,
	296	+ PERF_BR_PRIV_USER = 1,
	297	+ PERF_BR_PRIV_KERNEL = 2,
	298	+ PERF_BR_PRIV_HV = 3,
296	299	};
297	300
298		−#define PERF_BR_ARM64_FIQ PERF_BR_NEW_ARCH_1
299		−#define PERF_BR_ARM64_DEBUG_HALT PERF_BR_NEW_ARCH_2
300		−#define PERF_BR_ARM64_DEBUG_EXIT PERF_BR_NEW_ARCH_3
301		−#define PERF_BR_ARM64_DEBUG_INST PERF_BR_NEW_ARCH_4
302		−#define PERF_BR_ARM64_DEBUG_DATA PERF_BR_NEW_ARCH_5
	301	+#define PERF_BR_ARM64_FIQ PERF_BR_NEW_ARCH_1
	302	+#define PERF_BR_ARM64_DEBUG_HALT PERF_BR_NEW_ARCH_2
	303	+#define PERF_BR_ARM64_DEBUG_EXIT PERF_BR_NEW_ARCH_3
	304	+#define PERF_BR_ARM64_DEBUG_INST PERF_BR_NEW_ARCH_4
	305	+#define PERF_BR_ARM64_DEBUG_DATA PERF_BR_NEW_ARCH_5
303	306
304	307	#define PERF_SAMPLE_BRANCH_PLM_ALL \
305	308	(PERF_SAMPLE_BRANCH_USER\|\
@@ -310,9 +313,9 @@ enum { · 4 unmodified lines
310	313	* Values to determine ABI of the registers dump.
311	314	*/
312	315	enum perf_sample_regs_abi {
313		− PERF_SAMPLE_REGS_ABI_NONE = 0,
314		− PERF_SAMPLE_REGS_ABI_32 = 1,
315		− PERF_SAMPLE_REGS_ABI_64 = 2,
	316	+ PERF_SAMPLE_REGS_ABI_NONE = 0,
	317	+ PERF_SAMPLE_REGS_ABI_32 = 1,
	318	+ PERF_SAMPLE_REGS_ABI_64 = 2,
316	319	};
317	320
318	321	/*
@@ -320,21 +323,21 @@ enum perf_sample_regs_abi { · 1 unmodified line
320	323	* abort events. Multiple bits can be set.
321	324	*/
322	325	enum {
323		− PERF_TXN_ELISION = (1 << 0), /* From elision */
324		− PERF_TXN_TRANSACTION = (1 << 1), /* From transaction */
325		− PERF_TXN_SYNC = (1 << 2), /* Instruction is related */
326		− PERF_TXN_ASYNC = (1 << 3), /* Instruction not related */
327		− PERF_TXN_RETRY = (1 << 4), /* Retry possible */
328		− PERF_TXN_CONFLICT = (1 << 5), /* Conflict abort */
329		− PERF_TXN_CAPACITY_WRITE = (1 << 6), /* Capacity write abort */
330		− PERF_TXN_CAPACITY_READ = (1 << 7), /* Capacity read abort */
	326	+ PERF_TXN_ELISION = (1 << 0), /* From elision */
	327	+ PERF_TXN_TRANSACTION = (1 << 1), /* From transaction */
	328	+ PERF_TXN_SYNC = (1 << 2), /* Instruction is related */
	329	+ PERF_TXN_ASYNC = (1 << 3), /* Instruction is not related */
	330	+ PERF_TXN_RETRY = (1 << 4), /* Retry possible */
	331	+ PERF_TXN_CONFLICT = (1 << 5), /* Conflict abort */
	332	+ PERF_TXN_CAPACITY_WRITE = (1 << 6), /* Capacity write abort */
	333	+ PERF_TXN_CAPACITY_READ = (1 << 7), /* Capacity read abort */
331	334
332		− PERF_TXN_MAX = (1 << 8), /* non-ABI */
	335	+ PERF_TXN_MAX = (1 << 8), /* non-ABI */
333	336
334		− /* bits 32..63 are reserved for the abort code */
	337	+ /* Bits 32..63 are reserved for the abort code */
335	338
336		− PERF_TXN_ABORT_MASK = (0xffffffffULL << 32),
337		− PERF_TXN_ABORT_SHIFT = 32,
	339	+ PERF_TXN_ABORT_MASK = (0xffffffffULL << 32),
	340	+ PERF_TXN_ABORT_SHIFT = 32,
338	341	};
339	342
340	343	/*
@@ -369,24 +372,22 @@ enum perf_event_read_format { · 28 unmodified lines
369	372	PERF_FORMAT_MAX = 1U << 5, /* non-ABI */
370	373	};
371	374
372		−#define PERF_ATTR_SIZE_VER0 64 /* sizeof first published struct */
373		−#define PERF_ATTR_SIZE_VER1 72 /* add: config2 */
374		−#define PERF_ATTR_SIZE_VER2 80 /* add: branch_sample_type */
375		−#define PERF_ATTR_SIZE_VER3 96 /* add: sample_regs_user */
376		− /* add: sample_stack_user */
377		−#define PERF_ATTR_SIZE_VER4 104 /* add: sample_regs_intr */
378		−#define PERF_ATTR_SIZE_VER5 112 /* add: aux_watermark */
379		−#define PERF_ATTR_SIZE_VER6 120 /* add: aux_sample_size */
380		−#define PERF_ATTR_SIZE_VER7 128 /* add: sig_data */
381		−#define PERF_ATTR_SIZE_VER8 136 /* add: config3 */
	375	+#define PERF_ATTR_SIZE_VER0 64 /* Size of first published 'struct perf_event_attr' */
	376	+#define PERF_ATTR_SIZE_VER1 72 /* Add: config2 */
	377	+#define PERF_ATTR_SIZE_VER2 80 /* Add: branch_sample_type */
	378	+#define PERF_ATTR_SIZE_VER3 96 /* Add: sample_regs_user */
	379	+ /* Add: sample_stack_user */
	380	+#define PERF_ATTR_SIZE_VER4 104 /* Add: sample_regs_intr */
	381	+#define PERF_ATTR_SIZE_VER5 112 /* Add: aux_watermark */
	382	+#define PERF_ATTR_SIZE_VER6 120 /* Add: aux_sample_size */
	383	+#define PERF_ATTR_SIZE_VER7 128 /* Add: sig_data */
	384	+#define PERF_ATTR_SIZE_VER8 136 /* Add: config3 */
382	385
383	386	/*
384		− * Hardware event_id to monitor via a performance monitoring event:
385		− *
386		− * @sample_max_stack: Max number of frame pointers in a callchain,
387		− * should be < /proc/sys/kernel/perf_event_max_stack
388		− * Max number of entries of branch stack
389		− * should be < hardware limit
	387	+ * 'struct perf_event_attr' contains various attributes that define
	388	+ * a performance event - most of them hardware related configuration
	389	+ * details, but also a lot of behavioral switches and values implemented
	390	+ * by the kernel.
390	391	*/
391	392	struct perf_event_attr {
392	393
@@ -396,7 +397,7 @@ struct perf_event_attr { · 3 unmodified lines
396	397	__u32 type;
397	398
398	399	/*
399		− * Size of the attr structure, for fwd/bwd compat.
	400	+ * Size of the attr structure, for forward/backwards compatibility.
400	401	*/
401	402	__u32 size;
402	403
@@ -451,21 +452,21 @@ struct perf_event_attr { · 48 unmodified lines
451	452	comm_exec : 1, /* flag comm events that are due to an exec */
452	453	use_clockid : 1, /* use @clockid for time fields */
453	454	context_switch : 1, /* context switch data */
454		− write_backward : 1, /* Write ring buffer from end to beginning */
	455	+ write_backward : 1, /* write ring buffer from end to beginning */
455	456	namespaces : 1, /* include namespaces data */
456	457	ksymbol : 1, /* include ksymbol events */
457		− bpf_event : 1, /* include bpf events */
	458	+ bpf_event : 1, /* include BPF events */
458	459	aux_output : 1, /* generate AUX records instead of events */
459	460	cgroup : 1, /* include cgroup events */
460	461	text_poke : 1, /* include text poke events */
461		− build_id : 1, /* use build id in mmap2 events */
	462	+ build_id : 1, /* use build ID in mmap2 events */
462	463	inherit_thread : 1, /* children only inherit if cloned with CLONE_THREAD */
463	464	remove_on_exec : 1, /* event is removed from task on exec */
464	465	sigtrap : 1, /* send synchronous SIGTRAP on event */
465	466	__reserved_1 : 26;
466	467
467	468	union {
468		− __u32 wakeup_events; /* wakeup every n events */
	469	+ __u32 wakeup_events; /* wake up every n events */
469	470	__u32 wakeup_watermark; /* bytes before wakeup */
470	471	};
471	472
@@ -474,13 +475,13 @@ struct perf_event_attr { · 2 unmodified lines
474	475	__u64 bp_addr;
475	476	__u64 kprobe_func; /* for perf_kprobe */
476	477	__u64 uprobe_path; /* for perf_uprobe */
477		− __u64 config1; /* extension of config */
	478	+ __u64 config1; /* extension of config */
478	479	};
479	480	union {
480	481	__u64 bp_len;
481		− __u64 kprobe_addr; /* when kprobe_func == NULL */
	482	+ __u64 kprobe_addr; /* when kprobe_func == NULL */
482	483	__u64 probe_offset; /* for perf_[k,u]probe */
483		− __u64 config2; /* extension of config1 */
	484	+ __u64 config2; /* extension of config1 */
484	485	};
485	486	__u64 branch_sample_type; /* enum perf_branch_sample_type */
486	487
@@ -510,7 +511,16 @@ struct perf_event_attr { · 23 unmodified lines
510	511	* Wakeup watermark for AUX area
511	512	*/
512	513	__u32 aux_watermark;
	514	+
	515	+ /*
	516	+ * Max number of frame pointers in a callchain, should be
	517	+ * lower than /proc/sys/kernel/perf_event_max_stack.
	518	+ *
	519	+ * Max number of entries of branch stack should be lower
	520	+ * than the hardware limit.
	521	+ */
513	522	__u16 sample_max_stack;
	523	+
514	524	__u16 __reserved_2;
515	525	__u32 aux_sample_size;
516	526
@@ -537,7 +547,7 @@ struct perf_event_attr { · 20 unmodified lines
537	547
538	548	/*
539	549	* Structure used by below PERF_EVENT_IOC_QUERY_BPF command
540		− * to query bpf programs attached to the same perf tracepoint
	550	+ * to query BPF programs attached to the same perf tracepoint
541	551	* as the given perf event.
542	552	*/
543	553	struct perf_event_query_bpf {
@@ -559,21 +569,21 @@ struct perf_event_query_bpf { · 15 unmodified lines
559	569	/*
560	570	* Ioctls that can be done on a perf event fd:
561	571	*/
562		−#define PERF_EVENT_IOC_ENABLE _IO ('$', 0)
563		−#define PERF_EVENT_IOC_DISABLE _IO ('$', 1)
564		−#define PERF_EVENT_IOC_REFRESH _IO ('$', 2)
565		−#define PERF_EVENT_IOC_RESET _IO ('$', 3)
566		−#define PERF_EVENT_IOC_PERIOD _IOW('$', 4, __u64)
567		−#define PERF_EVENT_IOC_SET_OUTPUT _IO ('$', 5)
568		−#define PERF_EVENT_IOC_SET_FILTER _IOW('$', 6, char *)
569		−#define PERF_EVENT_IOC_ID _IOR('$', 7, __u64 *)
570		−#define PERF_EVENT_IOC_SET_BPF _IOW('$', 8, __u32)
571		−#define PERF_EVENT_IOC_PAUSE_OUTPUT _IOW('$', 9, __u32)
	572	+#define PERF_EVENT_IOC_ENABLE _IO ('$', 0)
	573	+#define PERF_EVENT_IOC_DISABLE _IO ('$', 1)
	574	+#define PERF_EVENT_IOC_REFRESH _IO ('$', 2)
	575	+#define PERF_EVENT_IOC_RESET _IO ('$', 3)
	576	+#define PERF_EVENT_IOC_PERIOD _IOW ('$', 4, __u64)
	577	+#define PERF_EVENT_IOC_SET_OUTPUT _IO ('$', 5)
	578	+#define PERF_EVENT_IOC_SET_FILTER _IOW ('$', 6, char *)
	579	+#define PERF_EVENT_IOC_ID _IOR ('$', 7, __u64 *)
	580	+#define PERF_EVENT_IOC_SET_BPF _IOW ('$', 8, __u32)
	581	+#define PERF_EVENT_IOC_PAUSE_OUTPUT _IOW ('$', 9, __u32)
572	582	#define PERF_EVENT_IOC_QUERY_BPF _IOWR('$', 10, struct perf_event_query_bpf *)
573		−#define PERF_EVENT_IOC_MODIFY_ATTRIBUTES _IOW('$', 11, struct perf_event_attr *)
	583	+#define PERF_EVENT_IOC_MODIFY_ATTRIBUTES _IOW ('$', 11, struct perf_event_attr *)
574	584
575	585	enum perf_event_ioc_flags {
576		− PERF_IOC_FLAG_GROUP = 1U << 0,
	586	+ PERF_IOC_FLAG_GROUP = 1U << 0,
577	587	};
578	588
579	589	/*
@@ -584,7 +594,7 @@ struct perf_event_mmap_page { · 4 unmodified lines
584	594	__u32 compat_version; /* lowest version this is compat with */
585	595
586	596	/*
587		− * Bits needed to read the hw events in user-space.
	597	+ * Bits needed to read the HW events in user-space.
588	598	*
589	599	* u32 seq, time_mult, time_shift, index, width;
590	600	* u64 count, enabled, running;
@@ -622,7 +632,7 @@ struct perf_event_mmap_page { · 31 unmodified lines
622	632	__u32 index; /* hardware event identifier */
623	633	__s64 offset; /* add to hardware event value */
624	634	__u64 time_enabled; /* time event active */
625		− __u64 time_running; /* time event on cpu */
	635	+ __u64 time_running; /* time event on CPU */
626	636	union {
627	637	__u64 capabilities;
628	638	struct {
@@ -650,7 +660,7 @@ struct perf_event_mmap_page { · 21 unmodified lines
650	660
651	661	/*
652	662	* If cap_usr_time the below fields can be used to compute the time
653		− * delta since time_enabled (in ns) using rdtsc or similar.
	663	+ * delta since time_enabled (in ns) using RDTSC or similar.
654	664	*
655	665	* u64 quot, rem;
656	666	* u64 delta;
@@ -723,7 +733,7 @@ struct perf_event_mmap_page { · 66 unmodified lines
723	733	* after reading this value.
724	734	*
725	735	* When the mapping is PROT_WRITE the @data_tail value should be
726		− * written by userspace to reflect the last read data, after issueing
	736	+ * written by user-space to reflect the last read data, after issuing
727	737	* an smp_mb() to separate the data read from the ->data_tail store.
728	738	* In this case the kernel will not over-write unread data.
729	739	*
@@ -739,7 +749,7 @@ struct perf_event_mmap_page { · 9 unmodified lines
739	749
740	750	/*
741	751	* AUX area is defined by aux_{offset,size} fields that should be set
742		− * by the userspace, so that
	752	+ * by the user-space, so that
743	753	*
744	754	* aux_offset >= data_offset + data_size
745	755	*
@@ -813,7 +823,7 @@ struct perf_event_mmap_page { · 67 unmodified lines
813	823	* Indicates that thread was preempted in TASK_RUNNING state.
814	824	*
815	825	* PERF_RECORD_MISC_MMAP_BUILD_ID:
816		− * Indicates that mmap2 event carries build id data.
	826	+ * Indicates that mmap2 event carries build ID data.
817	827	*/
818	828	#define PERF_RECORD_MISC_EXACT_IP (1 << 14)
819	829	#define PERF_RECORD_MISC_SWITCH_OUT_PREEMPT (1 << 14)
@@ -824,26 +834,26 @@ struct perf_event_mmap_page { · 4 unmodified lines
824	834	#define PERF_RECORD_MISC_EXT_RESERVED (1 << 15)
825	835
826	836	struct perf_event_header {
827		− __u32 type;
828		− __u16 misc;
829		− __u16 size;
	837	+ __u32 type;
	838	+ __u16 misc;
	839	+ __u16 size;
830	840	};
831	841
832	842	struct perf_ns_link_info {
833		− __u64 dev;
834		− __u64 ino;
	843	+ __u64 dev;
	844	+ __u64 ino;
835	845	};
836	846
837	847	enum {
838		− NET_NS_INDEX = 0,
839		− UTS_NS_INDEX = 1,
840		− IPC_NS_INDEX = 2,
841		− PID_NS_INDEX = 3,
842		− USER_NS_INDEX = 4,
843		− MNT_NS_INDEX = 5,
844		− CGROUP_NS_INDEX = 6,
845		−
846		− NR_NAMESPACES, /* number of available namespaces */
	848	+ NET_NS_INDEX = 0,
	849	+ UTS_NS_INDEX = 1,
	850	+ IPC_NS_INDEX = 2,
	851	+ PID_NS_INDEX = 3,
	852	+ USER_NS_INDEX = 4,
	853	+ MNT_NS_INDEX = 5,
	854	+ CGROUP_NS_INDEX = 6,
	855	+
	856	+ NR_NAMESPACES, /* number of available namespaces */
847	857	};
848	858
849	859	enum perf_event_type {
@@ -859,11 +869,11 @@ enum perf_event_type { · 9 unmodified lines
859	869	* optional fields being ignored.
860	870	*
861	871	* struct sample_id {
862		− * { u32 pid, tid; } && PERF_SAMPLE_TID
863		− * { u64 time; } && PERF_SAMPLE_TIME
864		− * { u64 id; } && PERF_SAMPLE_ID
865		− * { u64 stream_id;} && PERF_SAMPLE_STREAM_ID
866		− * { u32 cpu, res; } && PERF_SAMPLE_CPU
	872	+ * { u32 pid, tid; } && PERF_SAMPLE_TID
	873	+ * { u64 time; } && PERF_SAMPLE_TIME
	874	+ * { u64 id; } && PERF_SAMPLE_ID
	875	+ * { u64 stream_id;} && PERF_SAMPLE_STREAM_ID
	876	+ * { u32 cpu, res; } && PERF_SAMPLE_CPU
867	877	* { u64 id; } && PERF_SAMPLE_IDENTIFIER
868	878	* } && perf_event_attr::sample_id_all
869	879	*
@@ -874,7 +884,7 @@ enum perf_event_type { · 4 unmodified lines
874	884
875	885	/*
876	886	* The MMAP events record the PROT_EXEC mappings so that we can
877		− * correlate userspace IPs to code. They have the following structure:
	887	+ * correlate user-space IPs to code. They have the following structure:
878	888	*
879	889	* struct {
880	890	* struct perf_event_header header;
@@ -884,7 +894,7 @@ enum perf_event_type { · 3 unmodified lines
884	894	* u64 len;
885	895	* u64 pgoff;
886	896	* char filename[];
887		− * struct sample_id sample_id;
	897	+ * struct sample_id sample_id;
888	898	* };
889	899	*/
890	900	PERF_RECORD_MMAP = 1,
@@ -894,7 +904,7 @@ enum perf_event_type { · 3 unmodified lines
894	904	* struct perf_event_header header;
895	905	* u64 id;
896	906	* u64 lost;
897		− * struct sample_id sample_id;
	907	+ * struct sample_id sample_id;
898	908	* };
899	909	*/
900	910	PERF_RECORD_LOST = 2,
@@ -905,7 +915,7 @@ enum perf_event_type { · 4 unmodified lines
905	915	*
906	916	* u32 pid, tid;
907	917	* char comm[];
908		− * struct sample_id sample_id;
	918	+ * struct sample_id sample_id;
909	919	* };
910	920	*/
911	921	PERF_RECORD_COMM = 3,
@@ -916,7 +926,7 @@ enum perf_event_type { · 4 unmodified lines
916	926	* u32 pid, ppid;
917	927	* u32 tid, ptid;
918	928	* u64 time;
919		− * struct sample_id sample_id;
	929	+ * struct sample_id sample_id;
920	930	* };
921	931	*/
922	932	PERF_RECORD_EXIT = 4,
@@ -927,7 +937,7 @@ enum perf_event_type { · 4 unmodified lines
927	937	* u64 time;
928	938	* u64 id;
929	939	* u64 stream_id;
930		− * struct sample_id sample_id;
	940	+ * struct sample_id sample_id;
931	941	* };
932	942	*/
933	943	PERF_RECORD_THROTTLE = 5,
@@ -939,7 +949,7 @@ enum perf_event_type { · 5 unmodified lines
939	949	* u32 pid, ppid;
940	950	* u32 tid, ptid;
941	951	* u64 time;
942		− * struct sample_id sample_id;
	952	+ * struct sample_id sample_id;
943	953	* };
944	954	*/
945	955	PERF_RECORD_FORK = 7,
@@ -950,7 +960,7 @@ enum perf_event_type { · 4 unmodified lines
950	960	* u32 pid, tid;
951	961	*
952	962	* struct read_format values;
953		− * struct sample_id sample_id;
	963	+ * struct sample_id sample_id;
954	964	* };
955	965	*/
956	966	PERF_RECORD_READ = 8,
@@ -1005,12 +1015,12 @@ enum perf_event_type { · 48 unmodified lines
1005	1015	* { u64 counters; } cntr[nr] && PERF_SAMPLE_BRANCH_COUNTERS
1006	1016	* } && PERF_SAMPLE_BRANCH_STACK
1007	1017	*
1008		− * { u64 abi; # enum perf_sample_regs_abi
1009		− * u64 regs[weight(mask)]; } && PERF_SAMPLE_REGS_USER
	1018	+ * { u64 abi; # enum perf_sample_regs_abi
	1019	+ * u64 regs[weight(mask)]; } && PERF_SAMPLE_REGS_USER
1010	1020	*
1011		− * { u64 size;
1012		− * char data[size];
1013		− * u64 dyn_size; } && PERF_SAMPLE_STACK_USER
	1021	+ * { u64 size;
	1022	+ * char data[size];
	1023	+ * u64 dyn_size; } && PERF_SAMPLE_STACK_USER
1014	1024	*
1015	1025	* { union perf_sample_weight
1016	1026	* {
@@ -1035,10 +1045,11 @@ enum perf_event_type { · 18 unmodified lines
1035	1045	* { u64 abi; # enum perf_sample_regs_abi
1036	1046	* u64 regs[weight(mask)]; } && PERF_SAMPLE_REGS_INTR
1037	1047	* { u64 phys_addr;} && PERF_SAMPLE_PHYS_ADDR
1038		− * { u64 size;
1039		− * char data[size]; } && PERF_SAMPLE_AUX
	1048	+ * { u64 cgroup;} && PERF_SAMPLE_CGROUP
1040	1049	* { u64 data_page_size;} && PERF_SAMPLE_DATA_PAGE_SIZE
1041	1050	* { u64 code_page_size;} && PERF_SAMPLE_CODE_PAGE_SIZE
	1051	+ * { u64 size;
	1052	+ * char data[size]; } && PERF_SAMPLE_AUX
1042	1053	* };
1043	1054	*/
1044	1055	PERF_RECORD_SAMPLE = 9,
@@ -1070,7 +1081,7 @@ enum perf_event_type { · 25 unmodified lines
1070	1081	* };
1071	1082	* u32 prot, flags;
1072	1083	* char filename[];
1073		− * struct sample_id sample_id;
	1084	+ * struct sample_id sample_id;
1074	1085	* };
1075	1086	*/
1076	1087	PERF_RECORD_MMAP2 = 10,
@@ -1079,12 +1090,12 @@ enum perf_event_type { · 2 unmodified lines
1079	1090	* Records that new data landed in the AUX buffer part.
1080	1091	*
1081	1092	* struct {
1082		− * struct perf_event_header header;
	1093	+ * struct perf_event_header header;
1083	1094	*
1084		− * u64 aux_offset;
1085		− * u64 aux_size;
	1095	+ * u64 aux_offset;
	1096	+ * u64 aux_size;
1086	1097	* u64 flags;
1087		− * struct sample_id sample_id;
	1098	+ * struct sample_id sample_id;
1088	1099	* };
1089	1100	*/
1090	1101	PERF_RECORD_AUX = 11,
@@ -1167,7 +1178,7 @@ enum perf_event_type { · 76 unmodified lines
1167	1178	PERF_RECORD_KSYMBOL = 17,
1168	1179
1169	1180	/*
1170		− * Record bpf events:
	1181	+ * Record BPF events:
1171	1182	* enum perf_bpf_event_type {
1172	1183	* PERF_BPF_EVENT_UNKNOWN = 0,
1173	1184	* PERF_BPF_EVENT_PROG_LOAD = 1,
@@ -1245,181 +1256,181 @@ enum perf_record_ksymbol_type { · 71 unmodified lines
1245	1256	#define PERF_RECORD_KSYMBOL_FLAGS_UNREGISTER (1 << 0)
1246	1257
1247	1258	enum perf_bpf_event_type {
1248		− PERF_BPF_EVENT_UNKNOWN = 0,
1249		− PERF_BPF_EVENT_PROG_LOAD = 1,
1250		− PERF_BPF_EVENT_PROG_UNLOAD = 2,
1251		− PERF_BPF_EVENT_MAX, /* non-ABI */
	1259	+ PERF_BPF_EVENT_UNKNOWN = 0,
	1260	+ PERF_BPF_EVENT_PROG_LOAD = 1,
	1261	+ PERF_BPF_EVENT_PROG_UNLOAD = 2,
	1262	+ PERF_BPF_EVENT_MAX, /* non-ABI */
1252	1263	};
1253	1264
1254		−#define PERF_MAX_STACK_DEPTH 127
1255		−#define PERF_MAX_CONTEXTS_PER_STACK 8
	1265	+#define PERF_MAX_STACK_DEPTH 127
	1266	+#define PERF_MAX_CONTEXTS_PER_STACK 8
1256	1267
1257	1268	enum perf_callchain_context {
1258		− PERF_CONTEXT_HV = (__u64)-32,
1259		− PERF_CONTEXT_KERNEL = (__u64)-128,
1260		− PERF_CONTEXT_USER = (__u64)-512,
	1269	+ PERF_CONTEXT_HV = (__u64)-32,
	1270	+ PERF_CONTEXT_KERNEL = (__u64)-128,
	1271	+ PERF_CONTEXT_USER = (__u64)-512,
1261	1272
1262		− PERF_CONTEXT_GUEST = (__u64)-2048,
1263		− PERF_CONTEXT_GUEST_KERNEL = (__u64)-2176,
1264		− PERF_CONTEXT_GUEST_USER = (__u64)-2560,
	1273	+ PERF_CONTEXT_GUEST = (__u64)-2048,
	1274	+ PERF_CONTEXT_GUEST_KERNEL = (__u64)-2176,
	1275	+ PERF_CONTEXT_GUEST_USER = (__u64)-2560,
1265	1276
1266		− PERF_CONTEXT_MAX = (__u64)-4095,
	1277	+ PERF_CONTEXT_MAX = (__u64)-4095,
1267	1278	};
1268	1279
1269	1280	/**
1270	1281	* PERF_RECORD_AUX::flags bits
1271	1282	*/
1272		−#define PERF_AUX_FLAG_TRUNCATED 0x01 /* record was truncated to fit */
1273		−#define PERF_AUX_FLAG_OVERWRITE 0x02 /* snapshot from overwrite mode */
1274		−#define PERF_AUX_FLAG_PARTIAL 0x04 /* record contains gaps */
1275		−#define PERF_AUX_FLAG_COLLISION 0x08 /* sample collided with another */
	1283	+#define PERF_AUX_FLAG_TRUNCATED 0x0001 /* Record was truncated to fit */
	1284	+#define PERF_AUX_FLAG_OVERWRITE 0x0002 /* Snapshot from overwrite mode */
	1285	+#define PERF_AUX_FLAG_PARTIAL 0x0004 /* Record contains gaps */
	1286	+#define PERF_AUX_FLAG_COLLISION 0x0008 /* Sample collided with another */
1276	1287	#define PERF_AUX_FLAG_PMU_FORMAT_TYPE_MASK 0xff00 /* PMU specific trace format type */
1277	1288
1278	1289	/* CoreSight PMU AUX buffer formats */
1279		−#define PERF_AUX_FLAG_CORESIGHT_FORMAT_CORESIGHT 0x0000 /* Default for backward compatibility */
1280		−#define PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW 0x0100 /* Raw format of the source */
	1290	+#define PERF_AUX_FLAG_CORESIGHT_FORMAT_CORESIGHT 0x0000 /* Default for backward compatibility */
	1291	+#define PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW 0x0100 /* Raw format of the source */
1281	1292
1282		−#define PERF_FLAG_FD_NO_GROUP (1UL << 0)
1283		−#define PERF_FLAG_FD_OUTPUT (1UL << 1)
1284		−#define PERF_FLAG_PID_CGROUP (1UL << 2) /* pid=cgroup id, per-cpu mode only */
1285		−#define PERF_FLAG_FD_CLOEXEC (1UL << 3) /* O_CLOEXEC */
	1293	+#define PERF_FLAG_FD_NO_GROUP (1UL << 0)
	1294	+#define PERF_FLAG_FD_OUTPUT (1UL << 1)
	1295	+#define PERF_FLAG_PID_CGROUP (1UL << 2) /* pid=cgroup ID, per-CPU mode only */
	1296	+#define PERF_FLAG_FD_CLOEXEC (1UL << 3) /* O_CLOEXEC */
1286	1297
1287	1298	#if defined(__LITTLE_ENDIAN_BITFIELD)
1288	1299	union perf_mem_data_src {
1289	1300	__u64 val;
1290	1301	struct {
1291		− __u64 mem_op:5, /* type of opcode */
1292		− mem_lvl:14, /* memory hierarchy level */
1293		− mem_snoop:5, /* snoop mode */
1294		− mem_lock:2, /* lock instr */
1295		− mem_dtlb:7, /* tlb access */
1296		− mem_lvl_num:4, /* memory hierarchy level number */
1297		− mem_remote:1, /* remote */
1298		− mem_snoopx:2, /* snoop mode, ext */
1299		− mem_blk:3, /* access blocked */
1300		− mem_hops:3, /* hop level */
1301		− mem_rsvd:18;
	1302	+ __u64 mem_op : 5, /* Type of opcode */
	1303	+ mem_lvl : 14, /* Memory hierarchy level */
	1304	+ mem_snoop : 5, /* Snoop mode */
	1305	+ mem_lock : 2, /* Lock instr */
	1306	+ mem_dtlb : 7, /* TLB access */
	1307	+ mem_lvl_num : 4, /* Memory hierarchy level number */
	1308	+ mem_remote : 1, /* Remote */
	1309	+ mem_snoopx : 2, /* Snoop mode, ext */
	1310	+ mem_blk : 3, /* Access blocked */
	1311	+ mem_hops : 3, /* Hop level */
	1312	+ mem_rsvd : 18;
1302	1313	};
1303	1314	};
1304	1315	#elif defined(__BIG_ENDIAN_BITFIELD)
1305	1316	union perf_mem_data_src {
1306	1317	__u64 val;
1307	1318	struct {
1308		− __u64 mem_rsvd:18,
1309		− mem_hops:3, /* hop level */
1310		− mem_blk:3, /* access blocked */
1311		− mem_snoopx:2, /* snoop mode, ext */
1312		− mem_remote:1, /* remote */
1313		− mem_lvl_num:4, /* memory hierarchy level number */
1314		− mem_dtlb:7, /* tlb access */
1315		− mem_lock:2, /* lock instr */
1316		− mem_snoop:5, /* snoop mode */
1317		− mem_lvl:14, /* memory hierarchy level */
1318		− mem_op:5; /* type of opcode */
	1319	+ __u64 mem_rsvd : 18,
	1320	+ mem_hops : 3, /* Hop level */
	1321	+ mem_blk : 3, /* Access blocked */
	1322	+ mem_snoopx : 2, /* Snoop mode, ext */
	1323	+ mem_remote : 1, /* Remote */
	1324	+ mem_lvl_num : 4, /* Memory hierarchy level number */
	1325	+ mem_dtlb : 7, /* TLB access */
	1326	+ mem_lock : 2, /* Lock instr */
	1327	+ mem_snoop : 5, /* Snoop mode */
	1328	+ mem_lvl : 14, /* Memory hierarchy level */
	1329	+ mem_op : 5; /* Type of opcode */
1319	1330	};
1320	1331	};
1321	1332	#else
1322		−#error "Unknown endianness"
	1333	+# error "Unknown endianness"
1323	1334	#endif
1324	1335
1325		−/* type of opcode (load/store/prefetch,code) */
1326		−#define PERF_MEM_OP_NA 0x01 /* not available */
1327		−#define PERF_MEM_OP_LOAD 0x02 /* load instruction */
1328		−#define PERF_MEM_OP_STORE 0x04 /* store instruction */
1329		−#define PERF_MEM_OP_PFETCH 0x08 /* prefetch */
1330		−#define PERF_MEM_OP_EXEC 0x10 /* code (execution) */
1331		−#define PERF_MEM_OP_SHIFT 0
	1336	+/* Type of memory opcode: */
	1337	+#define PERF_MEM_OP_NA 0x0001 /* Not available */
	1338	+#define PERF_MEM_OP_LOAD 0x0002 /* Load instruction */
	1339	+#define PERF_MEM_OP_STORE 0x0004 /* Store instruction */
	1340	+#define PERF_MEM_OP_PFETCH 0x0008 /* Prefetch */
	1341	+#define PERF_MEM_OP_EXEC 0x0010 /* Code (execution) */
	1342	+#define PERF_MEM_OP_SHIFT 0
1332	1343
1333	1344	/*
1334		− * PERF_MEM_LVL_* namespace being depricated to some extent in the
	1345	+ * The PERF_MEM_LVL_* namespace is being deprecated to some extent in
1335	1346	* favour of newer composite PERF_MEM_{LVLNUM_,REMOTE_,SNOOPX_} fields.
1336		− * Supporting this namespace inorder to not break defined ABIs.
	1347	+ * We support this namespace in order to not break defined ABIs.
1337	1348	*
1338		− * memory hierarchy (memory level, hit or miss)
	1349	+ * Memory hierarchy (memory level, hit or miss)
1339	1350	*/
1340		−#define PERF_MEM_LVL_NA 0x01 /* not available */
1341		−#define PERF_MEM_LVL_HIT 0x02 /* hit level */
1342		−#define PERF_MEM_LVL_MISS 0x04 /* miss level */
1343		−#define PERF_MEM_LVL_L1 0x08 /* L1 */
1344		−#define PERF_MEM_LVL_LFB 0x10 /* Line Fill Buffer */
1345		−#define PERF_MEM_LVL_L2 0x20 /* L2 */
1346		−#define PERF_MEM_LVL_L3 0x40 /* L3 */
1347		−#define PERF_MEM_LVL_LOC_RAM 0x80 /* Local DRAM */
1348		−#define PERF_MEM_LVL_REM_RAM1 0x100 /* Remote DRAM (1 hop) */
1349		−#define PERF_MEM_LVL_REM_RAM2 0x200 /* Remote DRAM (2 hops) */
1350		−#define PERF_MEM_LVL_REM_CCE1 0x400 /* Remote Cache (1 hop) */
1351		−#define PERF_MEM_LVL_REM_CCE2 0x800 /* Remote Cache (2 hops) */
1352		−#define PERF_MEM_LVL_IO 0x1000 /* I/O memory */
1353		−#define PERF_MEM_LVL_UNC 0x2000 /* Uncached memory */
1354		−#define PERF_MEM_LVL_SHIFT 5
1355		−
1356		−#define PERF_MEM_REMOTE_REMOTE 0x01 /* Remote */
1357		−#define PERF_MEM_REMOTE_SHIFT 37
1358		−
1359		−#define PERF_MEM_LVLNUM_L1 0x01 /* L1 */
1360		−#define PERF_MEM_LVLNUM_L2 0x02 /* L2 */
1361		−#define PERF_MEM_LVLNUM_L3 0x03 /* L3 */
1362		−#define PERF_MEM_LVLNUM_L4 0x04 /* L4 */
1363		−#define PERF_MEM_LVLNUM_L2_MHB 0x05 /* L2 Miss Handling Buffer */
1364		−#define PERF_MEM_LVLNUM_MSC 0x06 /* Memory-side Cache */
1365		−/* 0x7 available */
1366		−#define PERF_MEM_LVLNUM_UNC 0x08 /* Uncached */
1367		−#define PERF_MEM_LVLNUM_CXL 0x09 /* CXL */
1368		−#define PERF_MEM_LVLNUM_IO 0x0a /* I/O */
1369		−#define PERF_MEM_LVLNUM_ANY_CACHE 0x0b /* Any cache */
1370		−#define PERF_MEM_LVLNUM_LFB 0x0c /* LFB / L1 Miss Handling Buffer */
1371		−#define PERF_MEM_LVLNUM_RAM 0x0d /* RAM */
1372		−#define PERF_MEM_LVLNUM_PMEM 0x0e /* PMEM */
1373		−#define PERF_MEM_LVLNUM_NA 0x0f /* N/A */
1374		−
1375		−#define PERF_MEM_LVLNUM_SHIFT 33
1376		−
1377		−/* snoop mode */
1378		−#define PERF_MEM_SNOOP_NA 0x01 /* not available */
1379		−#define PERF_MEM_SNOOP_NONE 0x02 /* no snoop */
1380		−#define PERF_MEM_SNOOP_HIT 0x04 /* snoop hit */
1381		−#define PERF_MEM_SNOOP_MISS 0x08 /* snoop miss */
1382		−#define PERF_MEM_SNOOP_HITM 0x10 /* snoop hit modified */
1383		−#define PERF_MEM_SNOOP_SHIFT 19
1384		−
1385		−#define PERF_MEM_SNOOPX_FWD 0x01 /* forward */
1386		−#define PERF_MEM_SNOOPX_PEER 0x02 /* xfer from peer */
1387		−#define PERF_MEM_SNOOPX_SHIFT 38
1388		−
1389		−/* locked instruction */
1390		−#define PERF_MEM_LOCK_NA 0x01 /* not available */
1391		−#define PERF_MEM_LOCK_LOCKED 0x02 /* locked transaction */
1392		−#define PERF_MEM_LOCK_SHIFT 24
	1351	+#define PERF_MEM_LVL_NA 0x0001 /* Not available */
	1352	+#define PERF_MEM_LVL_HIT 0x0002 /* Hit level */
	1353	+#define PERF_MEM_LVL_MISS 0x0004 /* Miss level */
	1354	+#define PERF_MEM_LVL_L1 0x0008 /* L1 */
	1355	+#define PERF_MEM_LVL_LFB 0x0010 /* Line Fill Buffer */
	1356	+#define PERF_MEM_LVL_L2 0x0020 /* L2 */
	1357	+#define PERF_MEM_LVL_L3 0x0040 /* L3 */
	1358	+#define PERF_MEM_LVL_LOC_RAM 0x0080 /* Local DRAM */
	1359	+#define PERF_MEM_LVL_REM_RAM1 0x0100 /* Remote DRAM (1 hop) */
	1360	+#define PERF_MEM_LVL_REM_RAM2 0x0200 /* Remote DRAM (2 hops) */
	1361	+#define PERF_MEM_LVL_REM_CCE1 0x0400 /* Remote Cache (1 hop) */
	1362	+#define PERF_MEM_LVL_REM_CCE2 0x0800 /* Remote Cache (2 hops) */
	1363	+#define PERF_MEM_LVL_IO 0x1000 /* I/O memory */
	1364	+#define PERF_MEM_LVL_UNC 0x2000 /* Uncached memory */
	1365	+#define PERF_MEM_LVL_SHIFT 5
	1366	+
	1367	+#define PERF_MEM_REMOTE_REMOTE 0x0001 /* Remote */
	1368	+#define PERF_MEM_REMOTE_SHIFT 37
	1369	+
	1370	+#define PERF_MEM_LVLNUM_L1 0x0001 /* L1 */
	1371	+#define PERF_MEM_LVLNUM_L2 0x0002 /* L2 */
	1372	+#define PERF_MEM_LVLNUM_L3 0x0003 /* L3 */
	1373	+#define PERF_MEM_LVLNUM_L4 0x0004 /* L4 */
	1374	+#define PERF_MEM_LVLNUM_L2_MHB 0x0005 /* L2 Miss Handling Buffer */
	1375	+#define PERF_MEM_LVLNUM_MSC 0x0006 /* Memory-side Cache */
	1376	+/* 0x007 available */
	1377	+#define PERF_MEM_LVLNUM_UNC 0x0008 /* Uncached */
	1378	+#define PERF_MEM_LVLNUM_CXL 0x0009 /* CXL */
	1379	+#define PERF_MEM_LVLNUM_IO 0x000a /* I/O */
	1380	+#define PERF_MEM_LVLNUM_ANY_CACHE 0x000b /* Any cache */
	1381	+#define PERF_MEM_LVLNUM_LFB 0x000c /* LFB / L1 Miss Handling Buffer */
	1382	+#define PERF_MEM_LVLNUM_RAM 0x000d /* RAM */
	1383	+#define PERF_MEM_LVLNUM_PMEM 0x000e /* PMEM */
	1384	+#define PERF_MEM_LVLNUM_NA 0x000f /* N/A */
	1385	+
	1386	+#define PERF_MEM_LVLNUM_SHIFT 33
	1387	+
	1388	+/* Snoop mode */
	1389	+#define PERF_MEM_SNOOP_NA 0x0001 /* Not available */
	1390	+#define PERF_MEM_SNOOP_NONE 0x0002 /* No snoop */
	1391	+#define PERF_MEM_SNOOP_HIT 0x0004 /* Snoop hit */
	1392	+#define PERF_MEM_SNOOP_MISS 0x0008 /* Snoop miss */
	1393	+#define PERF_MEM_SNOOP_HITM 0x0010 /* Snoop hit modified */
	1394	+#define PERF_MEM_SNOOP_SHIFT 19
	1395	+
	1396	+#define PERF_MEM_SNOOPX_FWD 0x0001 /* Forward */
	1397	+#define PERF_MEM_SNOOPX_PEER 0x0002 /* Transfer from peer */
	1398	+#define PERF_MEM_SNOOPX_SHIFT 38
	1399	+
	1400	+/* Locked instruction */
	1401	+#define PERF_MEM_LOCK_NA 0x0001 /* Not available */
	1402	+#define PERF_MEM_LOCK_LOCKED 0x0002 /* Locked transaction */
	1403	+#define PERF_MEM_LOCK_SHIFT 24
1393	1404
1394	1405	/* TLB access */
1395		−#define PERF_MEM_TLB_NA 0x01 /* not available */
1396		−#define PERF_MEM_TLB_HIT 0x02 /* hit level */
1397		−#define PERF_MEM_TLB_MISS 0x04 /* miss level */
1398		−#define PERF_MEM_TLB_L1 0x08 /* L1 */
1399		−#define PERF_MEM_TLB_L2 0x10 /* L2 */
1400		−#define PERF_MEM_TLB_WK 0x20 /* Hardware Walker*/
1401		−#define PERF_MEM_TLB_OS 0x40 /* OS fault handler */
1402		−#define PERF_MEM_TLB_SHIFT 26
	1406	+#define PERF_MEM_TLB_NA 0x0001 /* Not available */
	1407	+#define PERF_MEM_TLB_HIT 0x0002 /* Hit level */
	1408	+#define PERF_MEM_TLB_MISS 0x0004 /* Miss level */
	1409	+#define PERF_MEM_TLB_L1 0x0008 /* L1 */
	1410	+#define PERF_MEM_TLB_L2 0x0010 /* L2 */
	1411	+#define PERF_MEM_TLB_WK 0x0020 /* Hardware Walker*/
	1412	+#define PERF_MEM_TLB_OS 0x0040 /* OS fault handler */
	1413	+#define PERF_MEM_TLB_SHIFT 26
1403	1414
1404	1415	/* Access blocked */
1405		−#define PERF_MEM_BLK_NA 0x01 /* not available */
1406		−#define PERF_MEM_BLK_DATA 0x02 /* data could not be forwarded */
1407		−#define PERF_MEM_BLK_ADDR 0x04 /* address conflict */
1408		−#define PERF_MEM_BLK_SHIFT 40
1409		−
1410		−/* hop level */
1411		−#define PERF_MEM_HOPS_0 0x01 /* remote core, same node */
1412		−#define PERF_MEM_HOPS_1 0x02 /* remote node, same socket */
1413		−#define PERF_MEM_HOPS_2 0x03 /* remote socket, same board */
1414		−#define PERF_MEM_HOPS_3 0x04 /* remote board */
	1416	+#define PERF_MEM_BLK_NA 0x0001 /* Not available */
	1417	+#define PERF_MEM_BLK_DATA 0x0002 /* Data could not be forwarded */
	1418	+#define PERF_MEM_BLK_ADDR 0x0004 /* Address conflict */
	1419	+#define PERF_MEM_BLK_SHIFT 40
	1420	+
	1421	+/* Hop level */
	1422	+#define PERF_MEM_HOPS_0 0x0001 /* Remote core, same node */
	1423	+#define PERF_MEM_HOPS_1 0x0002 /* Remote node, same socket */
	1424	+#define PERF_MEM_HOPS_2 0x0003 /* Remote socket, same board */
	1425	+#define PERF_MEM_HOPS_3 0x0004 /* Remote board */
1415	1426	/* 5-7 available */
1416		−#define PERF_MEM_HOPS_SHIFT 43
	1427	+#define PERF_MEM_HOPS_SHIFT 43
1417	1428
1418	1429	#define PERF_MEM_S(a, s) \
1419	1430	(((__u64)PERF_MEM_##a##_##s) << PERF_MEM_##a##_SHIFT)
1420	1431
1421	1432	/*
1422		− * single taken branch record layout:
	1433	+ * Layout of single taken branch records:
1423	1434	*
1424	1435	* from: source instruction (may not always be a branch insn)
1425	1436	* to: branch target
@@ -1438,37 +1449,37 @@ union perf_mem_data_src { · 12 unmodified lines
1438	1449	struct perf_branch_entry {
1439	1450	__u64 from;
1440	1451	__u64 to;
1441		− __u64 mispred:1, /* target mispredicted */
1442		− predicted:1,/* target predicted */
1443		− in_tx:1, /* in transaction */
1444		− abort:1, /* transaction abort */
1445		− cycles:16, /* cycle count to last branch */
1446		− type:4, /* branch type */
1447		− spec:2, /* branch speculation info */
1448		− new_type:4, /* additional branch type */
1449		− priv:3, /* privilege level */
1450		− reserved:31;
	1452	+ __u64 mispred : 1, /* target mispredicted */
	1453	+ predicted : 1, /* target predicted */
	1454	+ in_tx : 1, /* in transaction */
	1455	+ abort : 1, /* transaction abort */
	1456	+ cycles : 16, /* cycle count to last branch */
	1457	+ type : 4, /* branch type */
	1458	+ spec : 2, /* branch speculation info */
	1459	+ new_type : 4, /* additional branch type */
	1460	+ priv : 3, /* privilege level */
	1461	+ reserved : 31;
1451	1462	};
1452	1463
1453	1464	/* Size of used info bits in struct perf_branch_entry */
1454	1465	#define PERF_BRANCH_ENTRY_INFO_BITS_MAX 33
1455	1466
1456	1467	union perf_sample_weight {
1457		− __u64 full;
	1468	+ __u64 full;
1458	1469	#if defined(__LITTLE_ENDIAN_BITFIELD)
1459	1470	struct {
1460		− __u32 var1_dw;
1461		− __u16 var2_w;
1462		− __u16 var3_w;
	1471	+ __u32 var1_dw;
	1472	+ __u16 var2_w;
	1473	+ __u16 var3_w;
1463	1474	};
1464	1475	#elif defined(__BIG_ENDIAN_BITFIELD)
1465	1476	struct {
1466		− __u16 var3_w;
1467		− __u16 var2_w;
1468		− __u32 var1_dw;
	1477	+ __u16 var3_w;
	1478	+ __u16 var2_w;
	1479	+ __u32 var1_dw;
1469	1480	};
1470	1481	#else
1471		−#error "Unknown endianness"
	1482	+# error "Unknown endianness"
1472	1483	#endif
1473	1484	};
1474	1485

tools/testing/selftests/bpf/bench.c+12−0

@@ -526,6 +526,12 @@ extern const struct bench bench_trig_uprobe_multi_push; · 525 unmodified lines
526	526	extern const struct bench bench_trig_uretprobe_multi_push;
527	527	extern const struct bench bench_trig_uprobe_multi_ret;
528	528	extern const struct bench bench_trig_uretprobe_multi_ret;
	529	+#ifdef __x86_64__
	530	+extern const struct bench bench_trig_uprobe_nop5;
	531	+extern const struct bench bench_trig_uretprobe_nop5;
	532	+extern const struct bench bench_trig_uprobe_multi_nop5;
	533	+extern const struct bench bench_trig_uretprobe_multi_nop5;
	534	+#endif
529	535
530	536	extern const struct bench bench_rb_libbpf;
531	537	extern const struct bench bench_rb_custom;
@@ -586,6 +592,12 @@ static const struct bench *benchs[] = { · 54 unmodified lines
586	592	&bench_trig_uretprobe_multi_push,
587	593	&bench_trig_uprobe_multi_ret,
588	594	&bench_trig_uretprobe_multi_ret,
	595	+#ifdef __x86_64__
	596	+ &bench_trig_uprobe_nop5,
	597	+ &bench_trig_uretprobe_nop5,
	598	+ &bench_trig_uprobe_multi_nop5,
	599	+ &bench_trig_uretprobe_multi_nop5,
	600	+#endif
589	601	/* ringbuf/perfbuf benchmarks */
590	602	&bench_rb_libbpf,
591	603	&bench_rb_custom,

tools/testing/selftests/bpf/benchs/bench_trigger.c+42−0

@@ -333,6 +333,20 @@ static void uprobe_producer_ret(void input) · 332 unmodified lines
333	333	return NULL;
334	334	}
335	335
	336	+#ifdef __x86_64__
	337	+__nocf_check __weak void uprobe_target_nop5(void)
	338	+{
	339	+ asm volatile (".byte 0x0f, 0x1f, 0x44, 0x00, 0x00");
	340	+}
	341	+
	342	+static void uprobe_producer_nop5(void input)
	343	+{
	344	+ while (true)
	345	+ uprobe_target_nop5();
	346	+ return NULL;
	347	+}
	348	+#endif
	349	+
336	350	static void usetup(bool use_retprobe, bool use_multi, void *target_addr)
337	351	{
338	352	size_t uprobe_offset;
@@ -448,6 +462,28 @@ static void uretprobe_multi_ret_setup(void) · 109 unmodified lines
448	462	usetup(true, true /* use_multi */, &uprobe_target_ret);
449	463	}
450	464
	465	+#ifdef __x86_64__
	466	+static void uprobe_nop5_setup(void)
	467	+{
	468	+ usetup(false, false /* !use_multi */, &uprobe_target_nop5);
	469	+}
	470	+
	471	+static void uretprobe_nop5_setup(void)
	472	+{
	473	+ usetup(true, false /* !use_multi */, &uprobe_target_nop5);
	474	+}
	475	+
	476	+static void uprobe_multi_nop5_setup(void)
	477	+{
	478	+ usetup(false, true /* use_multi */, &uprobe_target_nop5);
	479	+}
	480	+
	481	+static void uretprobe_multi_nop5_setup(void)
	482	+{
	483	+ usetup(true, true /* use_multi */, &uprobe_target_nop5);
	484	+}
	485	+#endif
	486	+
451	487	const struct bench bench_trig_syscall_count = {
452	488	.name = "trig-syscall-count",
453	489	.validate = trigger_validate,
@@ -506,3 +542,9 @@ BENCH_TRIG_USERMODE(uprobe_multi_ret, ret, "uprobe-multi-ret"); · 52 unmodified lines
506	542	BENCH_TRIG_USERMODE(uretprobe_multi_nop, nop, "uretprobe-multi-nop");
507	543	BENCH_TRIG_USERMODE(uretprobe_multi_push, push, "uretprobe-multi-push");
508	544	BENCH_TRIG_USERMODE(uretprobe_multi_ret, ret, "uretprobe-multi-ret");
	545	+#ifdef __x86_64__
	546	+BENCH_TRIG_USERMODE(uprobe_nop5, nop5, "uprobe-nop5");
	547	+BENCH_TRIG_USERMODE(uretprobe_nop5, nop5, "uretprobe-nop5");
	548	+BENCH_TRIG_USERMODE(uprobe_multi_nop5, nop5, "uprobe-multi-nop5");
	549	+BENCH_TRIG_USERMODE(uretprobe_multi_nop5, nop5, "uretprobe-multi-nop5");
	550	+#endif

tools/testing/selftests/bpf/benchs/run_bench_uprobes.sh+1−1

@@ -2,7 +2,7 @@ · 1 unmodified line
2	2
3	3	set -eufo pipefail
4	4
5		−for i in usermode-count syscall-count {uprobe,uretprobe}-{nop,push,ret}
	5	+for i in usermode-count syscall-count {uprobe,uretprobe}-{nop,push,ret,nop5}
6	6	do
7	7	summary=$(sudo ./bench -w2 -d5 -a trig-$i \| tail -n1 \| cut -d'(' -f1 \| cut -d' ' -f3-)
8	8	printf "%-15s: %s\n" $i "$summary"