lael-labs/linux

arch/alpha/kernel/perf_event.c+3−8

@@ -852,14 +852,9 @@ static void alpha_perf_event_irq_handler(unsigned long la_ptr, · 851 unmodified lines
852	alpha_perf_event_update(event, hwc, idx, alpha_pmu->pmc_max_period[idx]+1);	852	alpha_perf_event_update(event, hwc, idx, alpha_pmu->pmc_max_period[idx]+1);
853	perf_sample_data_init(&data, 0, hwc->last_period);	853	perf_sample_data_init(&data, 0, hwc->last_period);
854		854
855	if (alpha_perf_event_set_period(event, hwc, idx)) {	855	if (alpha_perf_event_set_period(event, hwc, idx))
856	if (perf_event_overflow(event, &data, regs)) {	856	perf_event_overflow(event, &data, regs);
857	/* Interrupts coming too quickly; "throttle" the	857
858	* counter, i.e., disable it for a little while.
859	*/
860	alpha_pmu_stop(event, 0);
861	}
862	}
863	wrperfmon(PERFMON_CMD_ENABLE, cpuc->idx_mask);	858	wrperfmon(PERFMON_CMD_ENABLE, cpuc->idx_mask);
864		859
865	return;	860	return;

arch/arc/kernel/perf_event.c+2−4

 		arc_perf_event_update(event, &event->hw, event->hw.idx);
 		perf_sample_data_init(&data, 0, hwc->last_period);
-		if (arc_pmu_event_set_period(event)) {
+		if (arc_pmu_event_set_period(event))
-			if (perf_event_overflow(event, &data, regs))
+			perf_event_overflow(event, &data, regs);
-				arc_pmu_stop(event, 0);
-		}
 		active_ints &= ~BIT(idx);
 	} while (active_ints);

arch/csky/kernel/perf_event.c+1−2

@@ -1139,8 +1139,7 @@ static irqreturn_t csky_pmu_handle_irq(int irq_num, void *dev) · 1138 unmodified lines
1139	perf_sample_data_init(&data, 0, hwc->last_period);	1139	perf_sample_data_init(&data, 0, hwc->last_period);
1140	csky_pmu_event_set_period(event);	1140	csky_pmu_event_set_period(event);
1141		1141
1142	if (perf_event_overflow(event, &data, regs))	1142	perf_event_overflow(event, &data, regs);
1143	csky_pmu_stop_event(event);
1144	}	1143	}
1145		1144
1146	csky_pmu_enable(&csky_pmu.pmu);	1145	csky_pmu_enable(&csky_pmu.pmu);

arch/loongarch/kernel/perf_event.c+1−2

@@ -479,8 +479,7 @@ static void handle_associated_event(struct cpu_hw_events *cpuc, int idx, · 478 unmodified lines
479	if (!loongarch_pmu_event_set_period(event, hwc, idx))	479	if (!loongarch_pmu_event_set_period(event, hwc, idx))
480	return;	480	return;
481		481
482	if (perf_event_overflow(event, data, regs))	482	perf_event_overflow(event, data, regs);
483	loongarch_pmu_disable_event(idx);
484	}	483	}
485		484
486	static irqreturn_t pmu_handle_irq(int irq, void *dev)	485	static irqreturn_t pmu_handle_irq(int irq, void *dev)

arch/mips/kernel/perf_event_mipsxx.c+1−2

@@ -791,8 +791,7 @@ static void handle_associated_event(struct cpu_hw_events *cpuc, · 790 unmodified lines
791	if (!mipspmu_event_set_period(event, hwc, idx))	791	if (!mipspmu_event_set_period(event, hwc, idx))
792	return;	792	return;
793		793
794	if (perf_event_overflow(event, data, regs))	794	perf_event_overflow(event, data, regs);
795	mipsxx_pmu_disable_event(idx);
796	}	795	}
797		796
798		797

arch/powerpc/perf/core-book3s.c+4−5

 			       struct pt_regs *regs)
 {
 	u64 period = event->hw.sample_period;
+	const u64 last_period = event->hw.last_period;
 	s64 prev, delta, left;
 	int record = 0;
 	if (record) {
 		struct perf_sample_data data;
-		perf_sample_data_init(&data, ~0ULL, event->hw.last_period);
+		perf_sample_data_init(&data, ~0ULL, last_period);
 		if (event->attr.sample_type & PERF_SAMPLE_ADDR_TYPE)
 			perf_get_data_addr(event, regs, &data.addr);
 			ppmu->get_mem_weight(&data.weight.full, event->attr.sample_type);
 			data.sample_flags |= PERF_SAMPLE_WEIGHT_TYPE;
 		}
-		if (perf_event_overflow(event, &data, regs))
+		perf_event_overflow(event, &data, regs);
-			power_pmu_stop(event, 0);
 	} else if (period) {
 		/* Account for interrupt in case of invalid SIAR */
-		if (perf_event_account_interrupt(event))
+		perf_event_account_interrupt(event);
-			power_pmu_stop(event, 0);
 	}
 }

arch/powerpc/perf/core-fsl-emb.c+3−3

 			       struct pt_regs *regs)
 {
 	u64 period = event->hw.sample_period;
+	const u64 last_period = event->hw.last_period;
 	s64 prev, delta, left;
 	int record = 0;
 	if (record) {
 		struct perf_sample_data data;
-		perf_sample_data_init(&data, 0, event->hw.last_period);
+		perf_sample_data_init(&data, 0, last_period);
-		if (perf_event_overflow(event, &data, regs))
+		perf_event_overflow(event, &data, regs);
-			fsl_emb_pmu_stop(event, 0);
 	}
 }

arch/s390/kernel/perf_cpum_cf.c+0−2

@@ -980,8 +980,6 @@ static int cfdiag_push_sample(struct perf_event *event, · 979 unmodified lines
980	}	980	}
981		981
982	overflow = perf_event_overflow(event, &data, &regs);	982	overflow = perf_event_overflow(event, &data, &regs);
983	if (overflow)
984	event->pmu->stop(event, 0);
985		983
986	perf_event_update_userpage(event);	984	perf_event_update_userpage(event);
987	return overflow;	985	return overflow;

arch/s390/kernel/perf_cpum_sf.c+1−4

 	overflow = 0;
 	if (perf_event_exclude(event, &regs, sde_regs))
 		goto out;
-	if (perf_event_overflow(event, &data, &regs)) {
+	overflow = perf_event_overflow(event, &data, &regs);
-		overflow = 1;
-		event->pmu->stop(event, 0);
-	}
 	perf_event_update_userpage(event);
 out:
 	return overflow;

arch/sparc/kernel/perf_event.c+1−2

@@ -1668,8 +1668,7 @@ static int __kprobes perf_event_nmi_handler(struct notifier_block *self, · 1667 unmodified lines
1668	if (!sparc_perf_event_set_period(event, hwc, idx))	1668	if (!sparc_perf_event_set_period(event, hwc, idx))
1669	continue;	1669	continue;
1670		1670
1671	if (perf_event_overflow(event, &data, regs))	1671	perf_event_overflow(event, &data, regs);
1672	sparc_pmu_stop(event, 0);
1673	}	1672	}
1674		1673
1675	finish_clock = sched_clock();	1674	finish_clock = sched_clock();

arch/x86/events/amd/core.c+1−2

@@ -1003,8 +1003,7 @@ static int amd_pmu_v2_handle_irq(struct pt_regs *regs) · 1002 unmodified lines
1003		1003
1004	perf_sample_save_brstack(&data, event, &cpuc->lbr_stack, NULL);	1004	perf_sample_save_brstack(&data, event, &cpuc->lbr_stack, NULL);
1005		1005
1006	if (perf_event_overflow(event, &data, regs))	1006	perf_event_overflow(event, &data, regs);
1007	x86_pmu_stop(event, 0);
1008	}	1007	}
1009		1008
1010	/*	1009	/*

arch/x86/events/amd/ibs.c+1−3

 		hwc->sample_period = perf_ibs->min_period;
 out:
-	if (throttle) {
+	if (!throttle) {
-		perf_ibs_stop(event, 0);
-	} else {
 		if (perf_ibs == &perf_ibs_op) {
 			if (ibs_caps & IBS_CAPS_OPCNTEXT) {
 				new_config = period & IBS_OP_MAX_CNT_EXT_MASK;

arch/x86/events/amd/uncore.c+101−2

@@ -21,6 +21,7 @@ · 20 unmodified lines
21	#define NUM_COUNTERS_NB 4	21	#define NUM_COUNTERS_NB 4
22	#define NUM_COUNTERS_L2 4	22	#define NUM_COUNTERS_L2 4
23	#define NUM_COUNTERS_L3 6	23	#define NUM_COUNTERS_L3 6
		24	#define NUM_COUNTERS_MAX 64
24		25
25	#define RDPMC_BASE_NB 6	26	#define RDPMC_BASE_NB 6
26	#define RDPMC_BASE_LLC 10	27	#define RDPMC_BASE_LLC 10
@@ -38,7 +39,10 @@ struct amd_uncore_ctx { · 11 unmodified lines
38	int refcnt;	39	int refcnt;
39	int cpu;	40	int cpu;
40	struct perf_event **events;	41	struct perf_event **events;
41	struct hlist_node node;	42	unsigned long active_mask[BITS_TO_LONGS(NUM_COUNTERS_MAX)];
		43	int nr_active;
		44	struct hrtimer hrtimer;
		45	u64 hrtimer_duration;
42	};	46	};
43		47
44	struct amd_uncore_pmu {	48	struct amd_uncore_pmu {
@@ -83,11 +87,51 @@ struct amd_uncore { · 38 unmodified lines
83		87
84	static struct amd_uncore uncores[UNCORE_TYPE_MAX];	88	static struct amd_uncore uncores[UNCORE_TYPE_MAX];
85		89
		90	/* Interval for hrtimer, defaults to 60000 milliseconds */
		91	static unsigned int update_interval = 60 * MSEC_PER_SEC;
		92	module_param(update_interval, uint, 0444);
		93
86	static struct amd_uncore_pmu event_to_amd_uncore_pmu(struct perf_event event)	94	static struct amd_uncore_pmu event_to_amd_uncore_pmu(struct perf_event event)
87	{	95	{
88	return container_of(event->pmu, struct amd_uncore_pmu, pmu);	96	return container_of(event->pmu, struct amd_uncore_pmu, pmu);
89	}	97	}
90		98
		99	static enum hrtimer_restart amd_uncore_hrtimer(struct hrtimer *hrtimer)
		100	{
		101	struct amd_uncore_ctx *ctx;
		102	struct perf_event *event;
		103	int bit;
		104
		105	ctx = container_of(hrtimer, struct amd_uncore_ctx, hrtimer);
		106
		107	if (!ctx->nr_active \|\| ctx->cpu != smp_processor_id())
		108	return HRTIMER_NORESTART;
		109
		110	for_each_set_bit(bit, ctx->active_mask, NUM_COUNTERS_MAX) {
		111	event = ctx->events[bit];
		112	event->pmu->read(event);
		113	}
		114
		115	hrtimer_forward_now(hrtimer, ns_to_ktime(ctx->hrtimer_duration));
		116	return HRTIMER_RESTART;
		117	}
		118
		119	static void amd_uncore_start_hrtimer(struct amd_uncore_ctx *ctx)
		120	{
		121	hrtimer_start(&ctx->hrtimer, ns_to_ktime(ctx->hrtimer_duration),
		122	HRTIMER_MODE_REL_PINNED_HARD);
		123	}
		124
		125	static void amd_uncore_cancel_hrtimer(struct amd_uncore_ctx *ctx)
		126	{
		127	hrtimer_cancel(&ctx->hrtimer);
		128	}
		129
		130	static void amd_uncore_init_hrtimer(struct amd_uncore_ctx *ctx)
		131	{
		132	hrtimer_setup(&ctx->hrtimer, amd_uncore_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
		133	}
		134
91	static void amd_uncore_read(struct perf_event *event)	135	static void amd_uncore_read(struct perf_event *event)
92	{	136	{
93	struct hw_perf_event *hwc = &event->hw;	137	struct hw_perf_event *hwc = &event->hw;
@@ -118,18 +162,26 @@ static void amd_uncore_read(struct perf_event *event) · 24 unmodified lines
118		162
119	static void amd_uncore_start(struct perf_event *event, int flags)	163	static void amd_uncore_start(struct perf_event *event, int flags)
120	{	164	{
		165	struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event);
		166	struct amd_uncore_ctx ctx = per_cpu_ptr(pmu->ctx, event->cpu);
121	struct hw_perf_event *hwc = &event->hw;	167	struct hw_perf_event *hwc = &event->hw;
122		168
		169	if (!ctx->nr_active++)
		170	amd_uncore_start_hrtimer(ctx);
		171
123	if (flags & PERF_EF_RELOAD)	172	if (flags & PERF_EF_RELOAD)
124	wrmsrl(hwc->event_base, (u64)local64_read(&hwc->prev_count));	173	wrmsrl(hwc->event_base, (u64)local64_read(&hwc->prev_count));
125		174
126	hwc->state = 0;	175	hwc->state = 0;
		176	__set_bit(hwc->idx, ctx->active_mask);
127	wrmsrl(hwc->config_base, (hwc->config \| ARCH_PERFMON_EVENTSEL_ENABLE));	177	wrmsrl(hwc->config_base, (hwc->config \| ARCH_PERFMON_EVENTSEL_ENABLE));
128	perf_event_update_userpage(event);	178	perf_event_update_userpage(event);
129	}	179	}
130		180
131	static void amd_uncore_stop(struct perf_event *event, int flags)	181	static void amd_uncore_stop(struct perf_event *event, int flags)
132	{	182	{
		183	struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event);
		184	struct amd_uncore_ctx ctx = per_cpu_ptr(pmu->ctx, event->cpu);
133	struct hw_perf_event *hwc = &event->hw;	185	struct hw_perf_event *hwc = &event->hw;
134		186
135	wrmsrl(hwc->config_base, hwc->config);	187	wrmsrl(hwc->config_base, hwc->config);
@@ -139,6 +191,11 @@ static void amd_uncore_stop(struct perf_event *event, int flags) · 3 unmodified lines
139	event->pmu->read(event);	191	event->pmu->read(event);
140	hwc->state \|= PERF_HES_UPTODATE;	192	hwc->state \|= PERF_HES_UPTODATE;
141	}	193	}
		194
		195	if (!--ctx->nr_active)
		196	amd_uncore_cancel_hrtimer(ctx);
		197
		198	__clear_bit(hwc->idx, ctx->active_mask);
142	}	199	}
143		200
144	static int amd_uncore_add(struct perf_event *event, int flags)	201	static int amd_uncore_add(struct perf_event *event, int flags)
@@ -491,6 +548,9 @@ static int amd_uncore_ctx_init(struct amd_uncore *uncore, unsigned int cpu) · 346 unmodified lines
491	goto fail;	548	goto fail;
492	}	549	}
493		550
		551	amd_uncore_init_hrtimer(curr);
		552	curr->hrtimer_duration = (u64)update_interval * NSEC_PER_MSEC;
		553
494	cpumask_set_cpu(cpu, &pmu->active_mask);	554	cpumask_set_cpu(cpu, &pmu->active_mask);
495	}	555	}
496		556
@@ -880,16 +940,55 @@ static int amd_uncore_umc_event_init(struct perf_event *event) · 383 unmodified lines
880		940
881	static void amd_uncore_umc_start(struct perf_event *event, int flags)	941	static void amd_uncore_umc_start(struct perf_event *event, int flags)
882	{	942	{
		943	struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event);
		944	struct amd_uncore_ctx ctx = per_cpu_ptr(pmu->ctx, event->cpu);
883	struct hw_perf_event *hwc = &event->hw;	945	struct hw_perf_event *hwc = &event->hw;
884		946
		947	if (!ctx->nr_active++)
		948	amd_uncore_start_hrtimer(ctx);
		949
885	if (flags & PERF_EF_RELOAD)	950	if (flags & PERF_EF_RELOAD)
886	wrmsrl(hwc->event_base, (u64)local64_read(&hwc->prev_count));	951	wrmsrl(hwc->event_base, (u64)local64_read(&hwc->prev_count));
887		952
888	hwc->state = 0;	953	hwc->state = 0;
		954	__set_bit(hwc->idx, ctx->active_mask);
889	wrmsrl(hwc->config_base, (hwc->config \| AMD64_PERFMON_V2_ENABLE_UMC));	955	wrmsrl(hwc->config_base, (hwc->config \| AMD64_PERFMON_V2_ENABLE_UMC));
890	perf_event_update_userpage(event);	956	perf_event_update_userpage(event);
891	}	957	}
892		958
		959	static void amd_uncore_umc_read(struct perf_event *event)
		960	{
		961	struct hw_perf_event *hwc = &event->hw;
		962	u64 prev, new, shift;
		963	s64 delta;
		964
		965	shift = COUNTER_SHIFT + 1;
		966	prev = local64_read(&hwc->prev_count);
		967
		968	/*
		969	* UMC counters do not have RDPMC assignments. Read counts directly
		970	* from the corresponding PERF_CTR.
		971	*/
		972	rdmsrl(hwc->event_base, new);
		973
		974	/*
		975	* Unlike the other uncore counters, UMC counters saturate and set the
		976	* Overflow bit (bit 48) on overflow. Since they do not roll over,
		977	* proactively reset the corresponding PERF_CTR when bit 47 is set so
		978	* that the counter never gets a chance to saturate.
		979	*/
		980	if (new & BIT_ULL(63 - COUNTER_SHIFT)) {
		981	wrmsrl(hwc->event_base, 0);
		982	local64_set(&hwc->prev_count, 0);
		983	} else {
		984	local64_set(&hwc->prev_count, new);
		985	}
		986
		987	delta = (new << shift) - (prev << shift);
		988	delta >>= shift;
		989	local64_add(delta, &event->count);
		990	}
		991
893	static	992	static
894	void amd_uncore_umc_ctx_scan(struct amd_uncore *uncore, unsigned int cpu)	993	void amd_uncore_umc_ctx_scan(struct amd_uncore *uncore, unsigned int cpu)
895	{	994	{
@@ -968,7 +1067,7 @@ int amd_uncore_umc_ctx_init(struct amd_uncore *uncore, unsigned int cpu) · 72 unmodified lines
968	.del = amd_uncore_del,	1067	.del = amd_uncore_del,
969	.start = amd_uncore_umc_start,	1068	.start = amd_uncore_umc_start,
970	.stop = amd_uncore_stop,	1069	.stop = amd_uncore_stop,
971	.read = amd_uncore_read,	1070	.read = amd_uncore_umc_read,
972	.capabilities = PERF_PMU_CAP_NO_EXCLUDE \| PERF_PMU_CAP_NO_INTERRUPT,	1071	.capabilities = PERF_PMU_CAP_NO_EXCLUDE \| PERF_PMU_CAP_NO_INTERRUPT,
973	.module = THIS_MODULE,	1072	.module = THIS_MODULE,
974	};	1073	};

arch/x86/events/core.c+25−12

@@ -95,6 +95,11 @@ DEFINE_STATIC_CALL_NULL(x86_pmu_filter, *x86_pmu.filter); · 94 unmodified lines
95		95
96	DEFINE_STATIC_CALL_NULL(x86_pmu_late_setup, *x86_pmu.late_setup);	96	DEFINE_STATIC_CALL_NULL(x86_pmu_late_setup, *x86_pmu.late_setup);
97		97
		98	DEFINE_STATIC_CALL_NULL(x86_pmu_pebs_enable, *x86_pmu.pebs_enable);
		99	DEFINE_STATIC_CALL_NULL(x86_pmu_pebs_disable, *x86_pmu.pebs_disable);
		100	DEFINE_STATIC_CALL_NULL(x86_pmu_pebs_enable_all, *x86_pmu.pebs_enable_all);
		101	DEFINE_STATIC_CALL_NULL(x86_pmu_pebs_disable_all, *x86_pmu.pebs_disable_all);
		102
98	/*	103	/*
99	* This one is magic, it will get called even when PMU init fails (because	104	* This one is magic, it will get called even when PMU init fails (because
100	* there is no PMU), in which case it should simply return NULL.	105	* there is no PMU), in which case it should simply return NULL.
@@ -674,6 +679,7 @@ static int __x86_pmu_event_init(struct perf_event *event) · 573 unmodified lines
674	event->hw.idx = -1;	679	event->hw.idx = -1;
675	event->hw.last_cpu = -1;	680	event->hw.last_cpu = -1;
676	event->hw.last_tag = ~0ULL;	681	event->hw.last_tag = ~0ULL;
		682	event->hw.dyn_constraint = ~0ULL;
677		683
678	/* mark unused */	684	/* mark unused */
679	event->hw.extra_reg.idx = EXTRA_REG_NONE;	685	event->hw.extra_reg.idx = EXTRA_REG_NONE;
@@ -756,15 +762,16 @@ void x86_pmu_enable_all(int added) · 76 unmodified lines
756		762
757	int is_x86_event(struct perf_event *event)	763	int is_x86_event(struct perf_event *event)
758	{	764	{
759	int i;	765	/*
760		766	* For a non-hybrid platforms, the type of X86 pmu is
761	if (!is_hybrid())	767	* always PERF_TYPE_RAW.
762	return event->pmu == &pmu;	768	* For a hybrid platform, the PERF_PMU_CAP_EXTENDED_HW_TYPE
763		769	* is a unique capability for the X86 PMU.
764	for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) {	770	* Use them to detect a X86 event.
765	if (event->pmu == &x86_pmu.hybrid_pmu[i].pmu)	771	*/
766	return true;	772	if (event->pmu->type == PERF_TYPE_RAW \|\|
767	}	773	event->pmu->capabilities & PERF_PMU_CAP_EXTENDED_HW_TYPE)
		774	return true;
768		775
769	return false;	776	return false;
770	}	777	}
@@ -1683,6 +1690,7 @@ int x86_pmu_handle_irq(struct pt_regs *regs) · 912 unmodified lines
1683	struct cpu_hw_events *cpuc;	1690	struct cpu_hw_events *cpuc;
1684	struct perf_event *event;	1691	struct perf_event *event;
1685	int idx, handled = 0;	1692	int idx, handled = 0;
		1693	u64 last_period;
1686	u64 val;	1694	u64 val;
1687		1695
1688	cpuc = this_cpu_ptr(&cpu_hw_events);	1696	cpuc = this_cpu_ptr(&cpu_hw_events);
@@ -1702,6 +1710,7 @@ int x86_pmu_handle_irq(struct pt_regs *regs) · 13 unmodified lines
1702	continue;	1710	continue;
1703		1711
1704	event = cpuc->events[idx];	1712	event = cpuc->events[idx];
		1713	last_period = event->hw.last_period;
1705		1714
1706	val = static_call(x86_pmu_update)(event);	1715	val = static_call(x86_pmu_update)(event);
1707	if (val & (1ULL << (x86_pmu.cntval_bits - 1)))	1716	if (val & (1ULL << (x86_pmu.cntval_bits - 1)))
@@ -1715,12 +1724,11 @@ int x86_pmu_handle_irq(struct pt_regs *regs) · 7 unmodified lines
1715	if (!static_call(x86_pmu_set_period)(event))	1724	if (!static_call(x86_pmu_set_period)(event))
1716	continue;	1725	continue;
1717		1726
1718	perf_sample_data_init(&data, 0, event->hw.last_period);	1727	perf_sample_data_init(&data, 0, last_period);
1719		1728
1720	perf_sample_save_brstack(&data, event, &cpuc->lbr_stack, NULL);	1729	perf_sample_save_brstack(&data, event, &cpuc->lbr_stack, NULL);
1721		1730
1722	if (perf_event_overflow(event, &data, regs))	1731	perf_event_overflow(event, &data, regs);
1723	x86_pmu_stop(event, 0);
1724	}	1732	}
1725		1733
1726	if (handled)	1734	if (handled)
@@ -2046,6 +2054,11 @@ static void x86_pmu_static_call_update(void) · 319 unmodified lines
2046	static_call_update(x86_pmu_filter, x86_pmu.filter);	2054	static_call_update(x86_pmu_filter, x86_pmu.filter);
2047		2055
2048	static_call_update(x86_pmu_late_setup, x86_pmu.late_setup);	2056	static_call_update(x86_pmu_late_setup, x86_pmu.late_setup);
		2057
		2058	static_call_update(x86_pmu_pebs_enable, x86_pmu.pebs_enable);
		2059	static_call_update(x86_pmu_pebs_disable, x86_pmu.pebs_disable);
		2060	static_call_update(x86_pmu_pebs_enable_all, x86_pmu.pebs_enable_all);
		2061	static_call_update(x86_pmu_pebs_disable_all, x86_pmu.pebs_disable_all);
2049	}	2062	}
2050		2063
2051	static void _x86_pmu_read(struct perf_event *event)	2064	static void _x86_pmu_read(struct perf_event *event)

arch/x86/events/intel/bts.c+77−73

 bts_buffer_setup_aux(struct perf_event *event, void **pages,
 		     int nr_pages, bool overwrite)
 {
-	struct bts_buffer *buf;
+	struct bts_buffer *bb;
 	struct page *page;
 	int cpu = event->cpu;
 	int node = (cpu == -1) ? cpu : cpu_to_node(cpu);
 	unsigned long offset;
 	size_t size = nr_pages << PAGE_SHIFT;
-	int pg, nbuf, pad;
+	int pg, nr_buf, pad;
 	/* count all the high order buffers */
-	for (pg = 0, nbuf = 0; pg < nr_pages;) {
+	for (pg = 0, nr_buf = 0; pg < nr_pages;) {
 		page = virt_to_page(pages[pg]);
 		pg += buf_nr_pages(page);
-		nbuf++;
+		nr_buf++;
 	}
 	/*
 	 * to avoid interrupts in overwrite mode, only allow one physical
 	 */
-	if (overwrite && nbuf > 1)
+	if (overwrite && nr_buf > 1)
 		return NULL;
-	buf = kzalloc_node(offsetof(struct bts_buffer, buf[nbuf]), GFP_KERNEL, node);
+	bb = kzalloc_node(struct_size(bb, buf, nr_buf), GFP_KERNEL, node);
-	if (!buf)
+	if (!bb)
 		return NULL;
-	buf->nr_pages = nr_pages;
+	bb->nr_pages = nr_pages;
-	buf->nr_bufs = nbuf;
+	bb->nr_bufs = nr_buf;
-	buf->snapshot = overwrite;
+	bb->snapshot = overwrite;
-	buf->data_pages = pages;
+	bb->data_pages = pages;
-	buf->real_size = size - size % BTS_RECORD_SIZE;
+	bb->real_size = size - size % BTS_RECORD_SIZE;
-	for (pg = 0, nbuf = 0, offset = 0, pad = 0; nbuf < buf->nr_bufs; nbuf++) {
+	for (pg = 0, nr_buf = 0, offset = 0, pad = 0; nr_buf < bb->nr_bufs; nr_buf++) {
 		unsigned int __nr_pages;
 		page = virt_to_page(pages[pg]);
 		__nr_pages = buf_nr_pages(page);
-		buf->buf[nbuf].page = page;
+		bb->buf[nr_buf].page = page;
-		buf->buf[nbuf].offset = offset;
+		bb->buf[nr_buf].offset = offset;
-		buf->buf[nbuf].displacement = (pad ? BTS_RECORD_SIZE - pad : 0);
+		bb->buf[nr_buf].displacement = (pad ? BTS_RECORD_SIZE - pad : 0);
-		buf->buf[nbuf].size = buf_size(page) - buf->buf[nbuf].displacement;
+		bb->buf[nr_buf].size = buf_size(page) - bb->buf[nr_buf].displacement;
-		pad = buf->buf[nbuf].size % BTS_RECORD_SIZE;
+		pad = bb->buf[nr_buf].size % BTS_RECORD_SIZE;
-		buf->buf[nbuf].size -= pad;
+		bb->buf[nr_buf].size -= pad;
 		pg += __nr_pages;
 		offset += __nr_pages << PAGE_SHIFT;
 	}
-	return buf;
+	return bb;
 }
 static void bts_buffer_free_aux(void *data)
 	kfree(data);
 }
-static unsigned long bts_buffer_offset(struct bts_buffer *buf, unsigned int idx)
+static unsigned long bts_buffer_offset(struct bts_buffer *bb, unsigned int idx)
 {
-	return buf->buf[idx].offset + buf->buf[idx].displacement;
+	return bb->buf[idx].offset + bb->buf[idx].displacement;
 }
 static void
-bts_config_buffer(struct bts_buffer *buf)
+bts_config_buffer(struct bts_buffer *bb)
 {
 	int cpu = raw_smp_processor_id();
 	struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
-	struct bts_phys *phys = &buf->buf[buf->cur_buf];
+	struct bts_phys *phys = &bb->buf[bb->cur_buf];
 	unsigned long index, thresh = 0, end = phys->size;
 	struct page *page = phys->page;
-	index = local_read(&buf->head);
+	index = local_read(&bb->head);
-	if (!buf->snapshot) {
+	if (!bb->snapshot) {
-		if (buf->end < phys->offset + buf_size(page))
+		if (bb->end < phys->offset + buf_size(page))
-			end = buf->end - phys->offset - phys->displacement;
+			end = bb->end - phys->offset - phys->displacement;
 		index -= phys->offset + phys->displacement;
 	ds->bts_buffer_base = (u64)(long)page_address(page) + phys->displacement;
 	ds->bts_index = ds->bts_buffer_base + index;
 	ds->bts_absolute_maximum = ds->bts_buffer_base + end;
-	ds->bts_interrupt_threshold = !buf->snapshot
+	ds->bts_interrupt_threshold = !bb->snapshot
 		? ds->bts_buffer_base + thresh
 		: ds->bts_absolute_maximum + BTS_RECORD_SIZE;
 }
 {
 	int cpu = raw_smp_processor_id();
 	struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
-	struct bts_buffer *buf = perf_get_aux(&bts->handle);
+	struct bts_buffer *bb = perf_get_aux(&bts->handle);
 	unsigned long index = ds->bts_index - ds->bts_buffer_base, old, head;
-	if (!buf)
+	if (!bb)
 		return;
-	head = index + bts_buffer_offset(buf, buf->cur_buf);
+	head = index + bts_buffer_offset(bb, bb->cur_buf);
-	old = local_xchg(&buf->head, head);
+	old = local_xchg(&bb->head, head);
-	if (!buf->snapshot) {
+	if (!bb->snapshot) {
 		if (old == head)
 			return;
 		 * old and head are always in the same physical buffer, so we
 		 * can subtract them to get the data size.
 		 */
-		local_add(head - old, &buf->data_size);
+		local_add(head - old, &bb->data_size);
 	} else {
-		local_set(&buf->data_size, head);
+		local_set(&bb->data_size, head);
 	}
 	/*
 }
 static int
-bts_buffer_reset(struct bts_buffer *buf, struct perf_output_handle *handle);
+bts_buffer_reset(struct bts_buffer *bb, struct perf_output_handle *handle);
 /*
  * Ordering PMU callbacks wrt themselves and the PMI is done by means
 static void __bts_event_start(struct perf_event *event)
 {
 	struct bts_ctx *bts = this_cpu_ptr(bts_ctx);
-	struct bts_buffer *buf = perf_get_aux(&bts->handle);
+	struct bts_buffer *bb = perf_get_aux(&bts->handle);
 	u64 config = 0;
-	if (!buf->snapshot)
+	if (!bb->snapshot)
 		config |= ARCH_PERFMON_EVENTSEL_INT;
 	if (!event->attr.exclude_kernel)
 		config |= ARCH_PERFMON_EVENTSEL_OS;
 	if (!event->attr.exclude_user)
 		config |= ARCH_PERFMON_EVENTSEL_USR;
-	bts_config_buffer(buf);
+	bts_config_buffer(bb);
 	/*
 	 * local barrier to make sure that ds configuration made it
 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 	struct bts_ctx *bts = this_cpu_ptr(bts_ctx);
-	struct bts_buffer *buf;
+	struct bts_buffer *bb;
-	buf = perf_aux_output_begin(&bts->handle, event);
+	bb = perf_aux_output_begin(&bts->handle, event);
-	if (!buf)
+	if (!bb)
 		goto fail_stop;
-	if (bts_buffer_reset(buf, &bts->handle))
+	if (bts_buffer_reset(bb, &bts->handle))
 		goto fail_end_stop;
 	bts->ds_back.bts_buffer_base = cpuc->ds->bts_buffer_base;
 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 	struct bts_ctx *bts = this_cpu_ptr(bts_ctx);
-	struct bts_buffer *buf = NULL;
+	struct bts_buffer *bb = NULL;
 	int state = READ_ONCE(bts->state);
 	if (state == BTS_STATE_ACTIVE)
 		__bts_event_stop(event, BTS_STATE_STOPPED);
 	if (state != BTS_STATE_STOPPED)
-		buf = perf_get_aux(&bts->handle);
+		bb = perf_get_aux(&bts->handle);
 	event->hw.state |= PERF_HES_STOPPED;
 	if (flags & PERF_EF_UPDATE) {
 		bts_update(bts);
-		if (buf) {
+		if (bb) {
-			if (buf->snapshot)
+			if (bb->snapshot)
 				bts->handle.head =
-					local_xchg(&buf->data_size,
+					local_xchg(&bb->data_size,
-						   buf->nr_pages << PAGE_SHIFT);
+						   bb->nr_pages << PAGE_SHIFT);
 			perf_aux_output_end(&bts->handle,
-			                    local_xchg(&buf->data_size, 0));
+					    local_xchg(&bb->data_size, 0));
 		}
 		cpuc->ds->bts_index = bts->ds_back.bts_buffer_base;
 }
 static int
-bts_buffer_reset(struct bts_buffer *buf, struct perf_output_handle *handle)
+bts_buffer_reset(struct bts_buffer *bb, struct perf_output_handle *handle)
 {
 	unsigned long head, space, next_space, pad, gap, skip, wakeup;
 	unsigned int next_buf;
 	struct bts_phys *phys, *next_phys;
 	int ret;
-	if (buf->snapshot)
+	if (bb->snapshot)
 		return 0;
-	head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1);
+	head = handle->head & ((bb->nr_pages << PAGE_SHIFT) - 1);
-	phys = &buf->buf[buf->cur_buf];
+	phys = &bb->buf[bb->cur_buf];
 	space = phys->offset + phys->displacement + phys->size - head;
 	pad = space;
 	if (space > handle->size) {
 	}
 	if (space <= BTS_SAFETY_MARGIN) {
 		/* See if next phys buffer has more space */
-		next_buf = buf->cur_buf + 1;
+		next_buf = bb->cur_buf + 1;
-		if (next_buf >= buf->nr_bufs)
+		if (next_buf >= bb->nr_bufs)
 			next_buf = 0;
-		next_phys = &buf->buf[next_buf];
+		next_phys = &bb->buf[next_buf];
 		gap = buf_size(phys->page) - phys->displacement - phys->size +
 		      next_phys->displacement;
 		skip = pad + gap;
 				 * anymore, so we must not be racing with
 				 * bts_update().
 				 */
-				buf->cur_buf = next_buf;
+				bb->cur_buf = next_buf;
-				local_set(&buf->head, head);
+				local_set(&bb->head, head);
 			}
 		}
 	}
 		space -= space % BTS_RECORD_SIZE;
 	}
-	buf->end = head + space;
+	bb->end = head + space;
 	/*
 	 * If we have no space, the lost notification would have been sent when
 	struct debug_store *ds = this_cpu_ptr(&cpu_hw_events)->ds;
 	struct bts_ctx *bts;
 	struct perf_event *event;
-	struct bts_buffer *buf;
+	struct bts_buffer *bb;
 	s64 old_head;
 	int err = -ENOSPC, handled = 0;
 	if (READ_ONCE(bts->state) == BTS_STATE_STOPPED)
 		return handled;
-	buf = perf_get_aux(&bts->handle);
+	bb = perf_get_aux(&bts->handle);
-	if (!buf)
+	if (!bb)
 		return handled;
 	/*
 	 * there's no other way of telling, because the pointer will
 	 * keep moving
 	 */
-	if (buf->snapshot)
+	if (bb->snapshot)
 		return 0;
-	old_head = local_read(&buf->head);
+	old_head = local_read(&bb->head);
 	bts_update(bts);
 	/* no new data */
-	if (old_head == local_read(&buf->head))
+	if (old_head == local_read(&bb->head))
 		return handled;
-	perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0));
+	perf_aux_output_end(&bts->handle, local_xchg(&bb->data_size, 0));
-	buf = perf_aux_output_begin(&bts->handle, event);
+	bb = perf_aux_output_begin(&bts->handle, event);
-	if (buf)
+	if (bb)
-		err = bts_buffer_reset(buf, &bts->handle);
+		err = bts_buffer_reset(bb, &bts->handle);
 	if (err) {
 		WRITE_ONCE(bts->state, BTS_STATE_STOPPED);
-		if (buf) {
+		if (bb) {
 			/*
 			 * BTS_STATE_STOPPED should be visible before
 			 * cleared handle::event
 static __init int bts_init(void)
 {
-	if (!boot_cpu_has(X86_FEATURE_DTES64) || !x86_pmu.bts)
+	if (!boot_cpu_has(X86_FEATURE_DTES64))
+		return -ENODEV;
+	x86_pmu.bts = boot_cpu_has(X86_FEATURE_BTS);
+	if (!x86_pmu.bts)
 		return -ENODEV;
 	if (boot_cpu_has(X86_FEATURE_PTI)) {

arch/x86/events/intel/core.c+319−26

@@ -2224,6 +2224,18 @@ static struct extra_reg intel_cmt_extra_regs[] __read_mostly = { · 2223 unmodified lines
2224	EVENT_EXTRA_END	2224	EVENT_EXTRA_END
2225	};	2225	};
2226		2226
		2227	EVENT_ATTR_STR(topdown-fe-bound, td_fe_bound_skt, "event=0x9c,umask=0x01");
		2228	EVENT_ATTR_STR(topdown-retiring, td_retiring_skt, "event=0xc2,umask=0x02");
		2229	EVENT_ATTR_STR(topdown-be-bound, td_be_bound_skt, "event=0xa4,umask=0x02");
		2230
		2231	static struct attribute *skt_events_attrs[] = {
		2232	EVENT_PTR(td_fe_bound_skt),
		2233	EVENT_PTR(td_retiring_skt),
		2234	EVENT_PTR(td_bad_spec_cmt),
		2235	EVENT_PTR(td_be_bound_skt),
		2236	NULL,
		2237	};
		2238
2227	#define KNL_OT_L2_HITE BIT_ULL(19) /* Other Tile L2 Hit */	2239	#define KNL_OT_L2_HITE BIT_ULL(19) /* Other Tile L2 Hit */
2228	#define KNL_OT_L2_HITF BIT_ULL(20) /* Other Tile L2 Hit */	2240	#define KNL_OT_L2_HITF BIT_ULL(20) /* Other Tile L2 Hit */
2229	#define KNL_MCDRAM_LOCAL BIT_ULL(21)	2241	#define KNL_MCDRAM_LOCAL BIT_ULL(21)
@@ -2294,7 +2306,7 @@ static __always_inline void __intel_pmu_disable_all(bool bts) · 64 unmodified lines
2294	static __always_inline void intel_pmu_disable_all(void)	2306	static __always_inline void intel_pmu_disable_all(void)
2295	{	2307	{
2296	__intel_pmu_disable_all(true);	2308	__intel_pmu_disable_all(true);
2297	intel_pmu_pebs_disable_all();	2309	static_call_cond(x86_pmu_pebs_disable_all)();
2298	intel_pmu_lbr_disable_all();	2310	intel_pmu_lbr_disable_all();
2299	}	2311	}
2300		2312
@@ -2326,7 +2338,7 @@ static void __intel_pmu_enable_all(int added, bool pmi) · 25 unmodified lines
2326		2338
2327	static void intel_pmu_enable_all(int added)	2339	static void intel_pmu_enable_all(int added)
2328	{	2340	{
2329	intel_pmu_pebs_enable_all();	2341	static_call_cond(x86_pmu_pebs_enable_all)();
2330	__intel_pmu_enable_all(added, false);	2342	__intel_pmu_enable_all(added, false);
2331	}	2343	}
2332		2344
@@ -2583,7 +2595,7 @@ static void intel_pmu_disable_event(struct perf_event *event) · 250 unmodified lines
2583	* so we don't trigger the event without PEBS bit set.	2595	* so we don't trigger the event without PEBS bit set.
2584	*/	2596	*/
2585	if (unlikely(event->attr.precise_ip))	2597	if (unlikely(event->attr.precise_ip))
2586	intel_pmu_pebs_disable(event);	2598	static_call(x86_pmu_pebs_disable)(event);
2587	}	2599	}
2588		2600
2589	static void intel_pmu_assign_event(struct perf_event *event, int idx)	2601	static void intel_pmu_assign_event(struct perf_event *event, int idx)
@@ -2603,6 +2615,9 @@ static void intel_pmu_del_event(struct perf_event *event) · 13 unmodified lines
2603	intel_pmu_lbr_del(event);	2615	intel_pmu_lbr_del(event);
2604	if (event->attr.precise_ip)	2616	if (event->attr.precise_ip)
2605	intel_pmu_pebs_del(event);	2617	intel_pmu_pebs_del(event);
		2618	if (is_pebs_counter_event_group(event) \|\|
		2619	is_acr_event_group(event))
		2620	this_cpu_ptr(&cpu_hw_events)->n_late_setup--;
2606	}	2621	}
2607		2622
2608	static int icl_set_topdown_event_period(struct perf_event *event)	2623	static int icl_set_topdown_event_period(struct perf_event *event)
@@ -2880,6 +2895,52 @@ static void intel_pmu_enable_fixed(struct perf_event *event) · 271 unmodified lines
2880	cpuc->fixed_ctrl_val \|= bits;	2895	cpuc->fixed_ctrl_val \|= bits;
2881	}	2896	}
2882		2897
		2898	static void intel_pmu_config_acr(int idx, u64 mask, u32 reload)
		2899	{
		2900	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
		2901	int msr_b, msr_c;
		2902
		2903	if (!mask && !cpuc->acr_cfg_b[idx])
		2904	return;
		2905
		2906	if (idx < INTEL_PMC_IDX_FIXED) {
		2907	msr_b = MSR_IA32_PMC_V6_GP0_CFG_B;
		2908	msr_c = MSR_IA32_PMC_V6_GP0_CFG_C;
		2909	} else {
		2910	msr_b = MSR_IA32_PMC_V6_FX0_CFG_B;
		2911	msr_c = MSR_IA32_PMC_V6_FX0_CFG_C;
		2912	idx -= INTEL_PMC_IDX_FIXED;
		2913	}
		2914
		2915	if (cpuc->acr_cfg_b[idx] != mask) {
		2916	wrmsrl(msr_b + x86_pmu.addr_offset(idx, false), mask);
		2917	cpuc->acr_cfg_b[idx] = mask;
		2918	}
		2919	/* Only need to update the reload value when there is a valid config value. */
		2920	if (mask && cpuc->acr_cfg_c[idx] != reload) {
		2921	wrmsrl(msr_c + x86_pmu.addr_offset(idx, false), reload);
		2922	cpuc->acr_cfg_c[idx] = reload;
		2923	}
		2924	}
		2925
		2926	static void intel_pmu_enable_acr(struct perf_event *event)
		2927	{
		2928	struct hw_perf_event *hwc = &event->hw;
		2929
		2930	if (!is_acr_event_group(event) \|\| !event->attr.config2) {
		2931	/*
		2932	* The disable doesn't clear the ACR CFG register.
		2933	* Check and clear the ACR CFG register.
		2934	*/
		2935	intel_pmu_config_acr(hwc->idx, 0, 0);
		2936	return;
		2937	}
		2938
		2939	intel_pmu_config_acr(hwc->idx, hwc->config1, -hwc->sample_period);
		2940	}
		2941
		2942	DEFINE_STATIC_CALL_NULL(intel_pmu_enable_acr_event, intel_pmu_enable_acr);
		2943
2883	static void intel_pmu_enable_event(struct perf_event *event)	2944	static void intel_pmu_enable_event(struct perf_event *event)
2884	{	2945	{
2885	u64 enable_mask = ARCH_PERFMON_EVENTSEL_ENABLE;	2946	u64 enable_mask = ARCH_PERFMON_EVENTSEL_ENABLE;
@@ -2887,16 +2948,19 @@ static void intel_pmu_enable_event(struct perf_event *event) · 1 unmodified line
2887	int idx = hwc->idx;	2948	int idx = hwc->idx;
2888		2949
2889	if (unlikely(event->attr.precise_ip))	2950	if (unlikely(event->attr.precise_ip))
2890	intel_pmu_pebs_enable(event);	2951	static_call(x86_pmu_pebs_enable)(event);
2891		2952
2892	switch (idx) {	2953	switch (idx) {
2893	case 0 ... INTEL_PMC_IDX_FIXED - 1:	2954	case 0 ... INTEL_PMC_IDX_FIXED - 1:
2894	if (branch_sample_counters(event))	2955	if (branch_sample_counters(event))
2895	enable_mask \|= ARCH_PERFMON_EVENTSEL_BR_CNTR;	2956	enable_mask \|= ARCH_PERFMON_EVENTSEL_BR_CNTR;
2896	intel_set_masks(event, idx);	2957	intel_set_masks(event, idx);
		2958	static_call_cond(intel_pmu_enable_acr_event)(event);
2897	__x86_pmu_enable_event(hwc, enable_mask);	2959	__x86_pmu_enable_event(hwc, enable_mask);
2898	break;	2960	break;
2899	case INTEL_PMC_IDX_FIXED ... INTEL_PMC_IDX_FIXED_BTS - 1:	2961	case INTEL_PMC_IDX_FIXED ... INTEL_PMC_IDX_FIXED_BTS - 1:
		2962	static_call_cond(intel_pmu_enable_acr_event)(event);
		2963	fallthrough;
2900	case INTEL_PMC_IDX_METRIC_BASE ... INTEL_PMC_IDX_METRIC_END:	2964	case INTEL_PMC_IDX_METRIC_BASE ... INTEL_PMC_IDX_METRIC_END:
2901	intel_pmu_enable_fixed(event);	2965	intel_pmu_enable_fixed(event);
2902	break;	2966	break;
@@ -2914,12 +2978,51 @@ static void intel_pmu_enable_event(struct perf_event *event) · 11 unmodified lines
2914	}	2978	}
2915	}	2979	}
2916		2980
		2981	static void intel_pmu_acr_late_setup(struct cpu_hw_events *cpuc)
		2982	{
		2983	struct perf_event event, leader;
		2984	int i, j, idx;
		2985
		2986	for (i = 0; i < cpuc->n_events; i++) {
		2987	leader = cpuc->event_list[i];
		2988	if (!is_acr_event_group(leader))
		2989	continue;
		2990
		2991	/* The ACR events must be contiguous. */
		2992	for (j = i; j < cpuc->n_events; j++) {
		2993	event = cpuc->event_list[j];
		2994	if (event->group_leader != leader->group_leader)
		2995	break;
		2996	for_each_set_bit(idx, (unsigned long *)&event->attr.config2, X86_PMC_IDX_MAX) {
		2997	if (WARN_ON_ONCE(i + idx > cpuc->n_events))
		2998	return;
		2999	__set_bit(cpuc->assign[i + idx], (unsigned long *)&event->hw.config1);
		3000	}
		3001	}
		3002	i = j - 1;
		3003	}
		3004	}
		3005
		3006	void intel_pmu_late_setup(void)
		3007	{
		3008	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
		3009
		3010	if (!cpuc->n_late_setup)
		3011	return;
		3012
		3013	intel_pmu_pebs_late_setup(cpuc);
		3014	intel_pmu_acr_late_setup(cpuc);
		3015	}
		3016
2917	static void intel_pmu_add_event(struct perf_event *event)	3017	static void intel_pmu_add_event(struct perf_event *event)
2918	{	3018	{
2919	if (event->attr.precise_ip)	3019	if (event->attr.precise_ip)
2920	intel_pmu_pebs_add(event);	3020	intel_pmu_pebs_add(event);
2921	if (intel_pmu_needs_branch_stack(event))	3021	if (intel_pmu_needs_branch_stack(event))
2922	intel_pmu_lbr_add(event);	3022	intel_pmu_lbr_add(event);
		3023	if (is_pebs_counter_event_group(event) \|\|
		3024	is_acr_event_group(event))
		3025	this_cpu_ptr(&cpu_hw_events)->n_late_setup++;
2923	}	3026	}
2924		3027
2925	/*	3028	/*
@@ -3035,8 +3138,7 @@ static void x86_pmu_handle_guest_pebs(struct pt_regs *regs, · 109 unmodified lines
3035	continue;	3138	continue;
3036		3139
3037	perf_sample_data_init(data, 0, event->hw.last_period);	3140	perf_sample_data_init(data, 0, event->hw.last_period);
3038	if (perf_event_overflow(event, data, regs))	3141	perf_event_overflow(event, data, regs);
3039	x86_pmu_stop(event, 0);
3040		3142
3041	/* Inject one fake event is enough. */	3143	/* Inject one fake event is enough. */
3042	break;	3144	break;
@@ -3141,6 +3243,7 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) · 98 unmodified lines
3141		3243
3142	for_each_set_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {	3244	for_each_set_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
3143	struct perf_event *event = cpuc->events[bit];	3245	struct perf_event *event = cpuc->events[bit];
		3246	u64 last_period;
3144		3247
3145	handled++;	3248	handled++;
3146		3249
@@ -3168,16 +3271,17 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) · 21 unmodified lines
3168	if (is_pebs_counter_event_group(event))	3271	if (is_pebs_counter_event_group(event))
3169	x86_pmu.drain_pebs(regs, &data);	3272	x86_pmu.drain_pebs(regs, &data);
3170		3273
		3274	last_period = event->hw.last_period;
		3275
3171	if (!intel_pmu_save_and_restart(event))	3276	if (!intel_pmu_save_and_restart(event))
3172	continue;	3277	continue;
3173		3278
3174	perf_sample_data_init(&data, 0, event->hw.last_period);	3279	perf_sample_data_init(&data, 0, last_period);
3175		3280
3176	if (has_branch_stack(event))	3281	if (has_branch_stack(event))
3177	intel_pmu_lbr_save_brstack(&data, cpuc, event);	3282	intel_pmu_lbr_save_brstack(&data, cpuc, event);
3178		3283
3179	if (perf_event_overflow(event, &data, regs))	3284	perf_event_overflow(event, &data, regs);
3180	x86_pmu_stop(event, 0);
3181	}	3285	}
3182		3286
3183	return handled;	3287	return handled;
@@ -3739,10 +3843,9 @@ intel_get_event_constraints(struct cpu_hw_events *cpuc, int idx, · 555 unmodified lines
3739	if (cpuc->excl_cntrs)	3843	if (cpuc->excl_cntrs)
3740	return intel_get_excl_constraints(cpuc, event, idx, c2);	3844	return intel_get_excl_constraints(cpuc, event, idx, c2);
3741		3845
3742	/* Not all counters support the branch counter feature. */	3846	if (event->hw.dyn_constraint != ~0ULL) {
3743	if (branch_sample_counters(event)) {
3744	c2 = dyn_constraint(cpuc, c2, idx);	3847	c2 = dyn_constraint(cpuc, c2, idx);
3745	c2->idxmsk64 &= x86_pmu.lbr_counters;	3848	c2->idxmsk64 &= event->hw.dyn_constraint;
3746	c2->weight = hweight64(c2->idxmsk64);	3849	c2->weight = hweight64(c2->idxmsk64);
3747	}	3850	}
3748		3851
@@ -4083,6 +4186,39 @@ end: · 334 unmodified lines
4083	return start;	4186	return start;
4084	}	4187	}
4085		4188
		4189	static inline bool intel_pmu_has_acr(struct pmu *pmu)
		4190	{
		4191	return !!hybrid(pmu, acr_cause_mask64);
		4192	}
		4193
		4194	static bool intel_pmu_is_acr_group(struct perf_event *event)
		4195	{
		4196	/* The group leader has the ACR flag set */
		4197	if (is_acr_event_group(event))
		4198	return true;
		4199
		4200	/* The acr_mask is set */
		4201	if (event->attr.config2)
		4202	return true;
		4203
		4204	return false;
		4205	}
		4206
		4207	static inline void intel_pmu_set_acr_cntr_constr(struct perf_event *event,
		4208	u64 cause_mask, int num)
		4209	{
		4210	event->hw.dyn_constraint &= hybrid(event->pmu, acr_cntr_mask64);
		4211	*cause_mask \|= event->attr.config2;
		4212	*num += 1;
		4213	}
		4214
		4215	static inline void intel_pmu_set_acr_caused_constr(struct perf_event *event,
		4216	int idx, u64 cause_mask)
		4217	{
		4218	if (test_bit(idx, (unsigned long *)&cause_mask))
		4219	event->hw.dyn_constraint &= hybrid(event->pmu, acr_cause_mask64);
		4220	}
		4221
4086	static int intel_pmu_hw_config(struct perf_event *event)	4222	static int intel_pmu_hw_config(struct perf_event *event)
4087	{	4223	{
4088	int ret = x86_pmu_hw_config(event);	4224	int ret = x86_pmu_hw_config(event);
@@ -4144,15 +4280,19 @@ static int intel_pmu_hw_config(struct perf_event *event) · 55 unmodified lines
4144	leader = event->group_leader;	4280	leader = event->group_leader;
4145	if (branch_sample_call_stack(leader))	4281	if (branch_sample_call_stack(leader))
4146	return -EINVAL;	4282	return -EINVAL;
4147	if (branch_sample_counters(leader))	4283	if (branch_sample_counters(leader)) {
4148	num++;	4284	num++;
		4285	leader->hw.dyn_constraint &= x86_pmu.lbr_counters;
		4286	}
4149	leader->hw.flags \|= PERF_X86_EVENT_BRANCH_COUNTERS;	4287	leader->hw.flags \|= PERF_X86_EVENT_BRANCH_COUNTERS;
4150		4288
4151	for_each_sibling_event(sibling, leader) {	4289	for_each_sibling_event(sibling, leader) {
4152	if (branch_sample_call_stack(sibling))	4290	if (branch_sample_call_stack(sibling))
4153	return -EINVAL;	4291	return -EINVAL;
4154	if (branch_sample_counters(sibling))	4292	if (branch_sample_counters(sibling)) {
4155	num++;	4293	num++;
		4294	sibling->hw.dyn_constraint &= x86_pmu.lbr_counters;
		4295	}
4156	}	4296	}
4157		4297
4158	if (num > fls(x86_pmu.lbr_counters))	4298	if (num > fls(x86_pmu.lbr_counters))
@@ -4207,6 +4347,94 @@ static int intel_pmu_hw_config(struct perf_event *event) · 48 unmodified lines
4207	event->attr.precise_ip)	4347	event->attr.precise_ip)
4208	event->group_leader->hw.flags \|= PERF_X86_EVENT_PEBS_CNTR;	4348	event->group_leader->hw.flags \|= PERF_X86_EVENT_PEBS_CNTR;
4209		4349
		4350	if (intel_pmu_has_acr(event->pmu) && intel_pmu_is_acr_group(event)) {
		4351	struct perf_event sibling, leader = event->group_leader;
		4352	struct pmu *pmu = event->pmu;
		4353	bool has_sw_event = false;
		4354	int num = 0, idx = 0;
		4355	u64 cause_mask = 0;
		4356
		4357	/* Not support perf metrics */
		4358	if (is_metric_event(event))
		4359	return -EINVAL;
		4360
		4361	/* Not support freq mode */
		4362	if (event->attr.freq)
		4363	return -EINVAL;
		4364
		4365	/* PDist is not supported */
		4366	if (event->attr.config2 && event->attr.precise_ip > 2)
		4367	return -EINVAL;
		4368
		4369	/* The reload value cannot exceeds the max period */
		4370	if (event->attr.sample_period > x86_pmu.max_period)
		4371	return -EINVAL;
		4372	/*
		4373	* The counter-constraints of each event cannot be finalized
		4374	* unless the whole group is scanned. However, it's hard
		4375	* to know whether the event is the last one of the group.
		4376	* Recalculate the counter-constraints for each event when
		4377	* adding a new event.
		4378	*
		4379	* The group is traversed twice, which may be optimized later.
		4380	* In the first round,
		4381	* - Find all events which do reload when other events
		4382	* overflow and set the corresponding counter-constraints
		4383	* - Add all events, which can cause other events reload,
		4384	* in the cause_mask
		4385	* - Error out if the number of events exceeds the HW limit
		4386	* - The ACR events must be contiguous.
		4387	* Error out if there are non-X86 events between ACR events.
		4388	* This is not a HW limit, but a SW limit.
		4389	* With the assumption, the intel_pmu_acr_late_setup() can
		4390	* easily convert the event idx to counter idx without
		4391	* traversing the whole event list.
		4392	*/
		4393	if (!is_x86_event(leader))
		4394	return -EINVAL;
		4395
		4396	if (leader->attr.config2)
		4397	intel_pmu_set_acr_cntr_constr(leader, &cause_mask, &num);
		4398
		4399	if (leader->nr_siblings) {
		4400	for_each_sibling_event(sibling, leader) {
		4401	if (!is_x86_event(sibling)) {
		4402	has_sw_event = true;
		4403	continue;
		4404	}
		4405	if (!sibling->attr.config2)
		4406	continue;
		4407	if (has_sw_event)
		4408	return -EINVAL;
		4409	intel_pmu_set_acr_cntr_constr(sibling, &cause_mask, &num);
		4410	}
		4411	}
		4412	if (leader != event && event->attr.config2) {
		4413	if (has_sw_event)
		4414	return -EINVAL;
		4415	intel_pmu_set_acr_cntr_constr(event, &cause_mask, &num);
		4416	}
		4417
		4418	if (hweight64(cause_mask) > hweight64(hybrid(pmu, acr_cause_mask64)) \|\|
		4419	num > hweight64(hybrid(event->pmu, acr_cntr_mask64)))
		4420	return -EINVAL;
		4421	/*
		4422	* In the second round, apply the counter-constraints for
		4423	* the events which can cause other events reload.
		4424	*/
		4425	intel_pmu_set_acr_caused_constr(leader, idx++, cause_mask);
		4426
		4427	if (leader->nr_siblings) {
		4428	for_each_sibling_event(sibling, leader)
		4429	intel_pmu_set_acr_caused_constr(sibling, idx++, cause_mask);
		4430	}
		4431
		4432	if (leader != event)
		4433	intel_pmu_set_acr_caused_constr(event, idx, cause_mask);
		4434
		4435	leader->hw.flags \|= PERF_X86_EVENT_ACR;
		4436	}
		4437
4210	if ((event->attr.type == PERF_TYPE_HARDWARE) \|\|	4438	if ((event->attr.type == PERF_TYPE_HARDWARE) \|\|
4211	(event->attr.type == PERF_TYPE_HW_CACHE))	4439	(event->attr.type == PERF_TYPE_HW_CACHE))
4212	return 0;	4440	return 0;
@@ -4354,7 +4582,7 @@ static struct perf_guest_switch_msr intel_guest_get_msrs(int nr, void *data) · 141 unmodified lines
4354	.guest = intel_ctrl & ~cpuc->intel_ctrl_host_mask & ~pebs_mask,	4582	.guest = intel_ctrl & ~cpuc->intel_ctrl_host_mask & ~pebs_mask,
4355	};	4583	};
4356		4584
4357	if (!x86_pmu.pebs)	4585	if (!x86_pmu.ds_pebs)
4358	return arr;	4586	return arr;
4359		4587
4360	/*	4588	/*
@@ -4952,7 +5180,7 @@ int intel_cpuc_prepare(struct cpu_hw_events *cpuc, int cpu) · 591 unmodified lines
4952	goto err;	5180	goto err;
4953	}	5181	}
4954		5182
4955	if (x86_pmu.flags & (PMU_FL_EXCL_CNTRS \| PMU_FL_TFA \| PMU_FL_BR_CNTR)) {	5183	if (x86_pmu.flags & (PMU_FL_EXCL_CNTRS \| PMU_FL_TFA \| PMU_FL_DYN_CONSTRAINT)) {
4956	size_t sz = X86_PMC_IDX_MAX * sizeof(struct event_constraint);	5184	size_t sz = X86_PMC_IDX_MAX * sizeof(struct event_constraint);
4957		5185
4958	cpuc->constraint_list = kzalloc_node(sz, GFP_KERNEL, cpu_to_node(cpu));	5186	cpuc->constraint_list = kzalloc_node(sz, GFP_KERNEL, cpu_to_node(cpu));
@@ -5041,7 +5269,7 @@ static inline bool intel_pmu_broken_perf_cap(void) · 82 unmodified lines
5041	return false;	5269	return false;
5042	}	5270	}
5043		5271
5044	static void update_pmu_cap(struct x86_hybrid_pmu *pmu)	5272	static void update_pmu_cap(struct pmu *pmu)
5045	{	5273	{
5046	unsigned int cntr, fixed_cntr, ecx, edx;	5274	unsigned int cntr, fixed_cntr, ecx, edx;
5047	union cpuid35_eax eax;	5275	union cpuid35_eax eax;
@@ -5050,20 +5278,30 @@ static void update_pmu_cap(struct x86_hybrid_pmu *pmu) · 2 unmodified lines
5050	cpuid(ARCH_PERFMON_EXT_LEAF, &eax.full, &ebx.full, &ecx, &edx);	5278	cpuid(ARCH_PERFMON_EXT_LEAF, &eax.full, &ebx.full, &ecx, &edx);
5051		5279
5052	if (ebx.split.umask2)	5280	if (ebx.split.umask2)
5053	pmu->config_mask \|= ARCH_PERFMON_EVENTSEL_UMASK2;	5281	hybrid(pmu, config_mask) \|= ARCH_PERFMON_EVENTSEL_UMASK2;
5054	if (ebx.split.eq)	5282	if (ebx.split.eq)
5055	pmu->config_mask \|= ARCH_PERFMON_EVENTSEL_EQ;	5283	hybrid(pmu, config_mask) \|= ARCH_PERFMON_EVENTSEL_EQ;
5056		5284
5057	if (eax.split.cntr_subleaf) {	5285	if (eax.split.cntr_subleaf) {
5058	cpuid_count(ARCH_PERFMON_EXT_LEAF, ARCH_PERFMON_NUM_COUNTER_LEAF,	5286	cpuid_count(ARCH_PERFMON_EXT_LEAF, ARCH_PERFMON_NUM_COUNTER_LEAF,
5059	&cntr, &fixed_cntr, &ecx, &edx);	5287	&cntr, &fixed_cntr, &ecx, &edx);
5060	pmu->cntr_mask64 = cntr;	5288	hybrid(pmu, cntr_mask64) = cntr;
5061	pmu->fixed_cntr_mask64 = fixed_cntr;	5289	hybrid(pmu, fixed_cntr_mask64) = fixed_cntr;
		5290	}
		5291
		5292	if (eax.split.acr_subleaf) {
		5293	cpuid_count(ARCH_PERFMON_EXT_LEAF, ARCH_PERFMON_ACR_LEAF,
		5294	&cntr, &fixed_cntr, &ecx, &edx);
		5295	/* The mask of the counters which can be reloaded */
		5296	hybrid(pmu, acr_cntr_mask64) = cntr \| ((u64)fixed_cntr << INTEL_PMC_IDX_FIXED);
		5297
		5298	/* The mask of the counters which can cause a reload of reloadable counters */
		5299	hybrid(pmu, acr_cause_mask64) = ecx \| ((u64)edx << INTEL_PMC_IDX_FIXED);
5062	}	5300	}
5063		5301
5064	if (!intel_pmu_broken_perf_cap()) {	5302	if (!intel_pmu_broken_perf_cap()) {
5065	/* Perf Metric (Bit 15) and PEBS via PT (Bit 16) are hybrid enumeration */	5303	/* Perf Metric (Bit 15) and PEBS via PT (Bit 16) are hybrid enumeration */
5066	rdmsrl(MSR_IA32_PERF_CAPABILITIES, pmu->intel_cap.capabilities);	5304	rdmsrl(MSR_IA32_PERF_CAPABILITIES, hybrid(pmu, intel_cap).capabilities);
5067	}	5305	}
5068	}	5306	}
5069		5307
@@ -5150,7 +5388,7 @@ static bool init_hybrid_pmu(int cpu) · 80 unmodified lines
5150	goto end;	5388	goto end;
5151		5389
5152	if (this_cpu_has(X86_FEATURE_ARCH_PERFMON_EXT))	5390	if (this_cpu_has(X86_FEATURE_ARCH_PERFMON_EXT))
5153	update_pmu_cap(pmu);	5391	update_pmu_cap(&pmu->pmu);
5154		5392
5155	intel_pmu_check_hybrid_pmus(pmu);	5393	intel_pmu_check_hybrid_pmus(pmu);
5156		5394
@@ -5524,7 +5762,7 @@ static __init void intel_clovertown_quirk(void) · 367 unmodified lines
5524	* these chips.	5762	* these chips.
5525	*/	5763	*/
5526	pr_warn("PEBS disabled due to CPU errata\n");	5764	pr_warn("PEBS disabled due to CPU errata\n");
5527	x86_pmu.pebs = 0;	5765	x86_pmu.ds_pebs = 0;
5528	x86_pmu.pebs_constraints = NULL;	5766	x86_pmu.pebs_constraints = NULL;
5529	}	5767	}
5530		5768
@@ -6012,7 +6250,7 @@ tsx_is_visible(struct kobject kobj, struct attribute attr, int i) · 481 unmodified lines
6012	static umode_t	6250	static umode_t
6013	pebs_is_visible(struct kobject kobj, struct attribute attr, int i)	6251	pebs_is_visible(struct kobject kobj, struct attribute attr, int i)
6014	{	6252	{
6015	return x86_pmu.pebs ? attr->mode : 0;	6253	return x86_pmu.ds_pebs ? attr->mode : 0;
6016	}	6254	}
6017		6255
6018	static umode_t	6256	static umode_t
@@ -6043,6 +6281,21 @@ td_is_visible(struct kobject kobj, struct attribute attr, int i) · 24 unmodified lines
6043	return attr->mode;	6281	return attr->mode;
6044	}	6282	}
6045		6283
		6284	PMU_FORMAT_ATTR(acr_mask, "config2:0-63");
		6285
		6286	static struct attribute *format_acr_attrs[] = {
		6287	&format_attr_acr_mask.attr,
		6288	NULL
		6289	};
		6290
		6291	static umode_t
		6292	acr_is_visible(struct kobject kobj, struct attribute attr, int i)
		6293	{
		6294	struct device *dev = kobj_to_dev(kobj);
		6295
		6296	return intel_pmu_has_acr(dev_get_drvdata(dev)) ? attr->mode : 0;
		6297	}
		6298
6046	static struct attribute_group group_events_td = {	6299	static struct attribute_group group_events_td = {
6047	.name = "events",	6300	.name = "events",
6048	.is_visible = td_is_visible,	6301	.is_visible = td_is_visible,
@@ -6085,6 +6338,12 @@ static struct attribute_group group_format_evtsel_ext = { · 36 unmodified lines
6085	.is_visible = evtsel_ext_is_visible,	6338	.is_visible = evtsel_ext_is_visible,
6086	};	6339	};
6087		6340
		6341	static struct attribute_group group_format_acr = {
		6342	.name = "format",
		6343	.attrs = format_acr_attrs,
		6344	.is_visible = acr_is_visible,
		6345	};
		6346
6088	static struct attribute_group group_default = {	6347	static struct attribute_group group_default = {
6089	.attrs = intel_pmu_attrs,	6348	.attrs = intel_pmu_attrs,
6090	.is_visible = default_is_visible,	6349	.is_visible = default_is_visible,
@@ -6099,6 +6358,7 @@ static const struct attribute_group *attr_update[] = { · 8 unmodified lines
6099	&group_format_extra,	6358	&group_format_extra,
6100	&group_format_extra_skl,	6359	&group_format_extra_skl,
6101	&group_format_evtsel_ext,	6360	&group_format_evtsel_ext,
		6361	&group_format_acr,
6102	&group_default,	6362	&group_default,
6103	NULL,	6363	NULL,
6104	};	6364	};
@@ -6383,6 +6643,7 @@ static const struct attribute_group *hybrid_attr_update[] = { · 278 unmodified lines
6383	&group_caps_lbr,	6643	&group_caps_lbr,
6384	&hybrid_group_format_extra,	6644	&hybrid_group_format_extra,
6385	&group_format_evtsel_ext,	6645	&group_format_evtsel_ext,
		6646	&group_format_acr,
6386	&group_default,	6647	&group_default,
6387	&hybrid_group_cpus,	6648	&hybrid_group_cpus,
6388	NULL,	6649	NULL,
@@ -6575,6 +6836,7 @@ static __always_inline void intel_pmu_init_skt(struct pmu *pmu) · 186 unmodified lines
6575	intel_pmu_init_grt(pmu);	6836	intel_pmu_init_grt(pmu);
6576	hybrid(pmu, event_constraints) = intel_skt_event_constraints;	6837	hybrid(pmu, event_constraints) = intel_skt_event_constraints;
6577	hybrid(pmu, extra_regs) = intel_cmt_extra_regs;	6838	hybrid(pmu, extra_regs) = intel_cmt_extra_regs;
		6839	static_call_update(intel_pmu_enable_acr_event, intel_pmu_enable_acr);
6578	}	6840	}
6579		6841
6580	__init int intel_pmu_init(void)	6842	__init int intel_pmu_init(void)
@@ -6635,6 +6897,7 @@ __init int intel_pmu_init(void) · 54 unmodified lines
6635		6897
6636	x86_pmu.pebs_events_mask = intel_pmu_pebs_mask(x86_pmu.cntr_mask64);	6898	x86_pmu.pebs_events_mask = intel_pmu_pebs_mask(x86_pmu.cntr_mask64);
6637	x86_pmu.pebs_capable = PEBS_COUNTER_MASK;	6899	x86_pmu.pebs_capable = PEBS_COUNTER_MASK;
		6900	x86_pmu.config_mask = X86_RAW_EVENT_MASK;
6638		6901
6639	/*	6902	/*
6640	* Quirk: v2 perfmon does not report fixed-purpose events, so	6903	* Quirk: v2 perfmon does not report fixed-purpose events, so
@@ -6663,7 +6926,7 @@ __init int intel_pmu_init(void) · 22 unmodified lines
6663	if (boot_cpu_has(X86_FEATURE_ARCH_LBR))	6926	if (boot_cpu_has(X86_FEATURE_ARCH_LBR))
6664	intel_pmu_arch_lbr_init();	6927	intel_pmu_arch_lbr_init();
6665		6928
6666	intel_ds_init();	6929	intel_pebs_init();
6667		6930
6668	x86_add_quirk(intel_arch_events_quirk); /* Install first, so it runs last */	6931	x86_add_quirk(intel_arch_events_quirk); /* Install first, so it runs last */
6669		6932
@@ -6673,6 +6936,12 @@ __init int intel_pmu_init(void) · 3 unmodified lines
6673	pr_cont(" AnyThread deprecated, ");	6936	pr_cont(" AnyThread deprecated, ");
6674	}	6937	}
6675		6938
		6939	/*
		6940	* Many features on and after V6 require dynamic constraint,
		6941	* e.g., Arch PEBS, ACR.
		6942	*/
		6943	if (version >= 6)
		6944	x86_pmu.flags \|= PMU_FL_DYN_CONSTRAINT;
6676	/*	6945	/*
6677	* Install the hw-cache-events table:	6946	* Install the hw-cache-events table:
6678	*/	6947	*/
@@ -6884,6 +7153,18 @@ __init int intel_pmu_init(void) · 205 unmodified lines
6884	name = "crestmont";	7153	name = "crestmont";
6885	break;	7154	break;
6886		7155
		7156	case INTEL_ATOM_DARKMONT_X:
		7157	intel_pmu_init_skt(NULL);
		7158	intel_pmu_pebs_data_source_cmt();
		7159	x86_pmu.pebs_latency_data = cmt_latency_data;
		7160	x86_pmu.get_event_constraints = cmt_get_event_constraints;
		7161	td_attr = skt_events_attrs;
		7162	mem_attr = grt_mem_attrs;
		7163	extra_attr = cmt_format_attr;
		7164	pr_cont("Darkmont events, ");
		7165	name = "darkmont";
		7166	break;
		7167
6887	case INTEL_WESTMERE:	7168	case INTEL_WESTMERE:
6888	case INTEL_WESTMERE_EP:	7169	case INTEL_WESTMERE_EP:
6889	case INTEL_WESTMERE_EX:	7170	case INTEL_WESTMERE_EX:
@@ -7433,6 +7714,18 @@ __init int intel_pmu_init(void) · 543 unmodified lines
7433	x86_pmu.attr_update = hybrid_attr_update;	7714	x86_pmu.attr_update = hybrid_attr_update;
7434	}	7715	}
7435		7716
		7717	/*
		7718	* The archPerfmonExt (0x23) includes an enhanced enumeration of
		7719	* PMU architectural features with a per-core view. For non-hybrid,
		7720	* each core has the same PMU capabilities. It's good enough to
		7721	* update the x86_pmu from the booting CPU. For hybrid, the x86_pmu
		7722	* is used to keep the common capabilities. Still keep the values
		7723	* from the leaf 0xa. The core specific update will be done later
		7724	* when a new type is online.
		7725	*/
		7726	if (!is_hybrid() && boot_cpu_has(X86_FEATURE_ARCH_PERFMON_EXT))
		7727	update_pmu_cap(NULL);
		7728
7436	intel_pmu_check_counters_mask(&x86_pmu.cntr_mask64,	7729	intel_pmu_check_counters_mask(&x86_pmu.cntr_mask64,
7437	&x86_pmu.fixed_cntr_mask64,	7730	&x86_pmu.fixed_cntr_mask64,
7438	&x86_pmu.intel_ctrl);	7731	&x86_pmu.intel_ctrl);

arch/x86/events/intel/ds.c+29−26

 	int max, node = cpu_to_node(cpu);
 	void *buffer, *insn_buff, *cea;
-	if (!x86_pmu.pebs)
+	if (!x86_pmu.ds_pebs)
 		return 0;
 	buffer = dsalloc_pages(bsiz, GFP_KERNEL, cpu);
 	struct cpu_hw_events *hwev = per_cpu_ptr(&cpu_hw_events, cpu);
 	void *cea;
-	if (!x86_pmu.pebs)
+	if (!x86_pmu.ds_pebs)
 		return;
 	kfree(per_cpu(insn_buffer, cpu));
 {
 	int cpu;
-	if (!x86_pmu.bts && !x86_pmu.pebs)
+	if (!x86_pmu.bts && !x86_pmu.ds_pebs)
 		return;
 	for_each_possible_cpu(cpu)
 	}
 	for_each_possible_cpu(cpu) {
-		release_pebs_buffer(cpu);
+		if (x86_pmu.ds_pebs)
+			release_pebs_buffer(cpu);
 		release_bts_buffer(cpu);
 	}
 }
 	int cpu;
 	x86_pmu.bts_active = 0;
-	x86_pmu.pebs_active = 0;
-	if (!x86_pmu.bts && !x86_pmu.pebs)
+	if (x86_pmu.ds_pebs)
+		x86_pmu.pebs_active = 0;
+	if (!x86_pmu.bts && !x86_pmu.ds_pebs)
 		return;
 	if (!x86_pmu.bts)
 		bts_err = 1;
-	if (!x86_pmu.pebs)
+	if (!x86_pmu.ds_pebs)
 		pebs_err = 1;
 	for_each_possible_cpu(cpu) {
 		if (!bts_err && alloc_bts_buffer(cpu))
 			bts_err = 1;
-		if (!pebs_err && alloc_pebs_buffer(cpu))
+		if (x86_pmu.ds_pebs && !pebs_err &&
+		    alloc_pebs_buffer(cpu))
 			pebs_err = 1;
 		if (bts_err && pebs_err)
 			release_bts_buffer(cpu);
 	}
-	if (pebs_err) {
+	if (x86_pmu.ds_pebs && pebs_err) {
 		for_each_possible_cpu(cpu)
 			release_pebs_buffer(cpu);
 	}
 		if (x86_pmu.bts && !bts_err)
 			x86_pmu.bts_active = 1;
-		if (x86_pmu.pebs && !pebs_err)
+		if (x86_pmu.ds_pebs && !pebs_err)
 			x86_pmu.pebs_active = 1;
 		for_each_possible_cpu(cpu) {
 }
-static void intel_pmu_late_setup(void)
+void intel_pmu_pebs_late_setup(struct cpu_hw_events *cpuc)
 {
-	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 	struct perf_event *event;
 	u64 pebs_data_cfg = 0;
 	int i;
 	perf_sample_data_init(data, 0, event->hw.last_period);
-	data->period = event->hw.last_period;
 	/*
 	 * Use latency for weight (only avail with PEBS-LL)
 	 */
 	sample_type = event->attr.sample_type;
 	format_group = basic->format_group;
 	perf_sample_data_init(data, 0, event->hw.last_period);
-	data->period = event->hw.last_period;
 	setup_pebs_time(event, data, basic->tsc);
 		 * All but the last records are processed.
 		 * The last one is left to be able to call the overflow handler.
 		 */
-		if (perf_event_overflow(event, data, regs))
+		perf_event_overflow(event, data, regs);
-			x86_pmu_stop(event, 0);
 	}
 	if (hwc->flags & PERF_X86_EVENT_AUTO_RELOAD) {
 		if (error[bit]) {
 			perf_log_lost_samples(event, error[bit]);
-			if (iregs && perf_event_account_interrupt(event))
+			if (iregs)
-				x86_pmu_stop(event, 0);
+				perf_event_account_interrupt(event);
 		}
 		if (counts[bit]) {
 }
 /*
- * BTS, PEBS probe and setup
+ * PEBS probe and setup
  */
-void __init intel_ds_init(void)
+void __init intel_pebs_init(void)
 {
 	/*
 	 * No support for 32bit formats
 	if (!boot_cpu_has(X86_FEATURE_DTES64))
 		return;
-	x86_pmu.bts  = boot_cpu_has(X86_FEATURE_BTS);
+	x86_pmu.ds_pebs = boot_cpu_has(X86_FEATURE_PEBS);
-	x86_pmu.pebs = boot_cpu_has(X86_FEATURE_PEBS);
 	x86_pmu.pebs_buffer_size = PEBS_BUFFER_SIZE;
 	if (x86_pmu.version <= 4)
 		x86_pmu.pebs_no_isolation = 1;
-	if (x86_pmu.pebs) {
+	if (x86_pmu.ds_pebs) {
 		char pebs_type = x86_pmu.intel_cap.pebs_trap ?  '+' : '-';
 		char *pebs_qual = "";
 		int format = x86_pmu.intel_cap.pebs_format;
 		if (format < 4)
 			x86_pmu.intel_cap.pebs_baseline = 0;
+		x86_pmu.pebs_enable = intel_pmu_pebs_enable;
+		x86_pmu.pebs_disable = intel_pmu_pebs_disable;
+		x86_pmu.pebs_enable_all = intel_pmu_pebs_enable_all;
+		x86_pmu.pebs_disable_all = intel_pmu_pebs_disable_all;
 		switch (format) {
 		case 0:
 			pr_cont("PEBS fmt0%c, ", pebs_type);
 		default:
 			pr_cont("no PEBS fmt%d%c, ", format, pebs_type);
-			x86_pmu.pebs = 0;
+			x86_pmu.ds_pebs = 0;
 		}
 	}
 }
 {
 	struct debug_store *ds = __this_cpu_read(cpu_hw_events.ds);
-	if (!x86_pmu.bts && !x86_pmu.pebs)
+	if (!x86_pmu.bts && !x86_pmu.ds_pebs)
 		return;
 	wrmsrl(MSR_IA32_DS_AREA, (unsigned long)ds);

arch/x86/events/intel/knc.c+4−3

 	for_each_set_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
 		struct perf_event *event = cpuc->events[bit];
+		u64 last_period;
 		handled++;
 		if (!test_bit(bit, cpuc->active_mask))
 			continue;
+		last_period = event->hw.last_period;
 		if (!intel_pmu_save_and_restart(event))
 			continue;
-		perf_sample_data_init(&data, 0, event->hw.last_period);
+		perf_sample_data_init(&data, 0, last_period);
-		if (perf_event_overflow(event, &data, regs))
+		perf_event_overflow(event, &data, regs);
-			x86_pmu_stop(event, 0);
 	}
 	/*

arch/x86/events/intel/lbr.c+1−1

@@ -1618,7 +1618,7 @@ void __init intel_pmu_arch_lbr_init(void) · 1617 unmodified lines
1618	x86_pmu.lbr_nr = lbr_nr;	1618	x86_pmu.lbr_nr = lbr_nr;
1619		1619
1620	if (!!x86_pmu.lbr_counters)	1620	if (!!x86_pmu.lbr_counters)
1621	x86_pmu.flags \|= PMU_FL_BR_CNTR;	1621	x86_pmu.flags \|= PMU_FL_BR_CNTR \| PMU_FL_DYN_CONSTRAINT;
1622		1622
1623	if (x86_pmu.lbr_mispred)	1623	if (x86_pmu.lbr_mispred)
1624	static_branch_enable(&x86_lbr_mispred);	1624	static_branch_enable(&x86_lbr_mispred);

arch/x86/events/intel/p4.c+1−2

@@ -1072,8 +1072,7 @@ static int p4_pmu_handle_irq(struct pt_regs *regs) · 1071 unmodified lines
1072	continue;	1072	continue;
1073		1073
1074		1074
1075	if (perf_event_overflow(event, &data, regs))	1075	perf_event_overflow(event, &data, regs);
1076	x86_pmu_stop(event, 0);
1077	}	1076	}
1078		1077
1079	if (handled)	1078	if (handled)

arch/x86/events/intel/pt.c+2−0

@@ -1863,6 +1863,8 @@ static __init int pt_init(void) · 1862 unmodified lines
1863		1863
1864	if (!intel_pt_validate_hw_cap(PT_CAP_topa_multiple_entries))	1864	if (!intel_pt_validate_hw_cap(PT_CAP_topa_multiple_entries))
1865	pt_pmu.pmu.capabilities = PERF_PMU_CAP_AUX_NO_SG;	1865	pt_pmu.pmu.capabilities = PERF_PMU_CAP_AUX_NO_SG;
		1866	else
		1867	pt_pmu.pmu.capabilities = PERF_PMU_CAP_AUX_PREFER_LARGE;
1866		1868
1867	pt_pmu.pmu.capabilities \|= PERF_PMU_CAP_EXCLUSIVE \|	1869	pt_pmu.pmu.capabilities \|= PERF_PMU_CAP_EXCLUSIVE \|
1868	PERF_PMU_CAP_ITRACE \|	1870	PERF_PMU_CAP_ITRACE \|

arch/x86/events/intel/uncore.c+2−10

 {
 	struct intel_uncore_box *box;
 	struct perf_event *event;
-	unsigned long flags;
 	int bit;
 	box = container_of(hrtimer, struct intel_uncore_box, hrtimer);
 	if (!box->n_active || box->cpu != smp_processor_id())
 		return HRTIMER_NORESTART;
-	/*
-	 * disable local interrupt to prevent uncore_pmu_event_start/stop
-	 * to interrupt the update process
-	 */
-	local_irq_save(flags);
 	/*
 	 * handle boxes with an active event list as opposed to active
 	for_each_set_bit(bit, box->active_mask, UNCORE_PMC_IDX_MAX)
 		uncore_perf_event_update(box, box->events[bit]);
-	local_irq_restore(flags);
 	hrtimer_forward_now(hrtimer, ns_to_ktime(box->hrtimer_duration));
 	return HRTIMER_RESTART;
 }
 void uncore_pmu_start_hrtimer(struct intel_uncore_box *box)
 {
 	hrtimer_start(&box->hrtimer, ns_to_ktime(box->hrtimer_duration),
-		      HRTIMER_MODE_REL_PINNED);
+		      HRTIMER_MODE_REL_PINNED_HARD);
 }
 void uncore_pmu_cancel_hrtimer(struct intel_uncore_box *box)
 static void uncore_pmu_init_hrtimer(struct intel_uncore_box *box)
 {
-	hrtimer_setup(&box->hrtimer, uncore_pmu_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+	hrtimer_setup(&box->hrtimer, uncore_pmu_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
 }
 static struct intel_uncore_box *uncore_alloc_box(struct intel_uncore_type *type,

arch/x86/events/perf_event.h+43−2

 	return check_leader_group(event->group_leader, PERF_X86_EVENT_PEBS_CNTR);
 }
+static inline bool is_acr_event_group(struct perf_event *event)
+{
+	return check_leader_group(event->group_leader, PERF_X86_EVENT_ACR);
+}
 struct amd_nb {
 	int nb_id;  /* NorthBridge id */
 	int refcnt; /* reference count */
 	struct event_constraint	*event_constraint[X86_PMC_IDX_MAX];
 	int			n_excl; /* the number of exclusive events */
+	int			n_late_setup; /* the num of events needs late setup */
 	unsigned int		txn_flags;
 	int			is_fake;
 	u64			fixed_ctrl_val;
 	u64			active_fixed_ctrl_val;
+	/* Intel ACR configuration */
+	u64			acr_cfg_b[X86_PMC_IDX_MAX];
+	u64			acr_cfg_c[X86_PMC_IDX_MAX];
 	/*
 	 * Intel LBR bits
 	 */
 			u64		fixed_cntr_mask64;
 			unsigned long	fixed_cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
 	};
+	union {
+			u64		acr_cntr_mask64;
+			unsigned long	acr_cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
+	};
+	union {
+			u64		acr_cause_mask64;
+			unsigned long	acr_cause_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
+	};
 	struct event_constraint		unconstrained;
 	u64				hw_cache_event_ids
 	int		(*hw_config)(struct perf_event *event);
 	int		(*schedule_events)(struct cpu_hw_events *cpuc, int n, int *assign);
 	void		(*late_setup)(void);
+	void		(*pebs_enable)(struct perf_event *event);
+	void		(*pebs_disable)(struct perf_event *event);
+	void		(*pebs_enable_all)(void);
+	void		(*pebs_disable_all)(void);
 	unsigned	eventsel;
 	unsigned	perfctr;
 	unsigned	fixedctr;
 			u64		fixed_cntr_mask64;
 			unsigned long	fixed_cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
 	};
+	union {
+			u64		acr_cntr_mask64;
+			unsigned long	acr_cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
+	};
+	union {
+			u64		acr_cause_mask64;
+			unsigned long	acr_cause_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
+	};
 	int		cntval_bits;
 	u64		cntval_mask;
 	union {
 	 */
 	unsigned int	bts			:1,
 			bts_active		:1,
-			pebs			:1,
+			ds_pebs			:1,
 			pebs_active		:1,
 			pebs_broken		:1,
 			pebs_prec_dist		:1,
 #define PMU_FL_MEM_LOADS_AUX	0x100 /* Require an auxiliary event for the complete memory info */
 #define PMU_FL_RETIRE_LATENCY	0x200 /* Support Retire Latency in PEBS */
 #define PMU_FL_BR_CNTR		0x400 /* Support branch counter logging */
+#define PMU_FL_DYN_CONSTRAINT	0x800 /* Needs dynamic constraint */
 #define EVENT_VAR(_id)  event_attr_##_id
 #define EVENT_PTR(_id) &event_attr_##_id.attr.attr
 	.pmu_type	= _pmu,						\
 }
+int is_x86_event(struct perf_event *event);
 struct pmu *x86_get_pmu(unsigned int cpu);
 extern struct x86_pmu x86_pmu __read_mostly;
 DECLARE_STATIC_CALL(x86_pmu_update,     *x86_pmu.update);
 DECLARE_STATIC_CALL(x86_pmu_drain_pebs,	*x86_pmu.drain_pebs);
 DECLARE_STATIC_CALL(x86_pmu_late_setup,	*x86_pmu.late_setup);
+DECLARE_STATIC_CALL(x86_pmu_pebs_enable, *x86_pmu.pebs_enable);
+DECLARE_STATIC_CALL(x86_pmu_pebs_disable, *x86_pmu.pebs_disable);
+DECLARE_STATIC_CALL(x86_pmu_pebs_enable_all, *x86_pmu.pebs_enable_all);
+DECLARE_STATIC_CALL(x86_pmu_pebs_disable_all, *x86_pmu.pebs_disable_all);
 static __always_inline struct x86_perf_task_context_opt *task_context_opt(void *ctx)
 {
 int intel_pmu_drain_bts_buffer(void);
+void intel_pmu_late_setup(void);
 u64 grt_latency_data(struct perf_event *event, u64 status);
 u64 cmt_latency_data(struct perf_event *event, u64 status);
 void intel_pmu_pebs_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in);
+void intel_pmu_pebs_late_setup(struct cpu_hw_events *cpuc);
 void intel_pmu_drain_pebs_buffer(void);
 void intel_pmu_store_pebs_lbrs(struct lbr_entry *lbr);
-void intel_ds_init(void);
+void intel_pebs_init(void);
 void intel_pmu_lbr_save_brstack(struct perf_sample_data *data,
 				struct cpu_hw_events *cpuc,

arch/x86/events/perf_event_flags.h+21−20

@@ -2,23 +2,24 @@ · 1 unmodified line
2	/*	2	/*
3	* struct hw_perf_event.flags flags	3	* struct hw_perf_event.flags flags
4	*/	4	*/
5	PERF_ARCH(PEBS_LDLAT, 0x00001) /* ld+ldlat data address sampling */	5	PERF_ARCH(PEBS_LDLAT, 0x0000001) /* ld+ldlat data address sampling */
6	PERF_ARCH(PEBS_ST, 0x00002) /* st data address sampling */	6	PERF_ARCH(PEBS_ST, 0x0000002) /* st data address sampling */
7	PERF_ARCH(PEBS_ST_HSW, 0x00004) /* haswell style datala, store */	7	PERF_ARCH(PEBS_ST_HSW, 0x0000004) /* haswell style datala, store */
8	PERF_ARCH(PEBS_LD_HSW, 0x00008) /* haswell style datala, load */	8	PERF_ARCH(PEBS_LD_HSW, 0x0000008) /* haswell style datala, load */
9	PERF_ARCH(PEBS_NA_HSW, 0x00010) /* haswell style datala, unknown */	9	PERF_ARCH(PEBS_NA_HSW, 0x0000010) /* haswell style datala, unknown */
10	PERF_ARCH(EXCL, 0x00020) /* HT exclusivity on counter */	10	PERF_ARCH(EXCL, 0x0000020) /* HT exclusivity on counter */
11	PERF_ARCH(DYNAMIC, 0x00040) /* dynamic alloc'd constraint */	11	PERF_ARCH(DYNAMIC, 0x0000040) /* dynamic alloc'd constraint */
12	PERF_ARCH(PEBS_CNTR, 0x00080) /* PEBS counters snapshot */	12	PERF_ARCH(PEBS_CNTR, 0x0000080) /* PEBS counters snapshot */
13	PERF_ARCH(EXCL_ACCT, 0x00100) /* accounted EXCL event */	13	PERF_ARCH(EXCL_ACCT, 0x0000100) /* accounted EXCL event */
14	PERF_ARCH(AUTO_RELOAD, 0x00200) /* use PEBS auto-reload */	14	PERF_ARCH(AUTO_RELOAD, 0x0000200) /* use PEBS auto-reload */
15	PERF_ARCH(LARGE_PEBS, 0x00400) /* use large PEBS */	15	PERF_ARCH(LARGE_PEBS, 0x0000400) /* use large PEBS */
16	PERF_ARCH(PEBS_VIA_PT, 0x00800) /* use PT buffer for PEBS */	16	PERF_ARCH(PEBS_VIA_PT, 0x0000800) /* use PT buffer for PEBS */
17	PERF_ARCH(PAIR, 0x01000) /* Large Increment per Cycle */	17	PERF_ARCH(PAIR, 0x0001000) /* Large Increment per Cycle */
18	PERF_ARCH(LBR_SELECT, 0x02000) /* Save/Restore MSR_LBR_SELECT */	18	PERF_ARCH(LBR_SELECT, 0x0002000) /* Save/Restore MSR_LBR_SELECT */
19	PERF_ARCH(TOPDOWN, 0x04000) /* Count Topdown slots/metrics events */	19	PERF_ARCH(TOPDOWN, 0x0004000) /* Count Topdown slots/metrics events */
20	PERF_ARCH(PEBS_STLAT, 0x08000) /* st+stlat data address sampling */	20	PERF_ARCH(PEBS_STLAT, 0x0008000) /* st+stlat data address sampling */
21	PERF_ARCH(AMD_BRS, 0x10000) /* AMD Branch Sampling */	21	PERF_ARCH(AMD_BRS, 0x0010000) /* AMD Branch Sampling */
22	PERF_ARCH(PEBS_LAT_HYBRID, 0x20000) /* ld and st lat for hybrid */	22	PERF_ARCH(PEBS_LAT_HYBRID, 0x0020000) /* ld and st lat for hybrid */
23	PERF_ARCH(NEEDS_BRANCH_STACK, 0x40000) /* require branch stack setup */	23	PERF_ARCH(NEEDS_BRANCH_STACK, 0x0040000) /* require branch stack setup */
24	PERF_ARCH(BRANCH_COUNTERS, 0x80000) /* logs the counters in the extra space of each branch */	24	PERF_ARCH(BRANCH_COUNTERS, 0x0080000) /* logs the counters in the extra space of each branch */
		25	PERF_ARCH(ACR, 0x0100000) /* Auto counter reload */

arch/x86/events/zhaoxin/core.c+1−2

@@ -397,8 +397,7 @@ again: · 396 unmodified lines
397	if (!x86_perf_event_set_period(event))	397	if (!x86_perf_event_set_period(event))
398	continue;	398	continue;
399		399
400	if (perf_event_overflow(event, &data, regs))	400	perf_event_overflow(event, &data, regs);
401	x86_pmu_stop(event, 0);
402	}	401	}
403		402
404	/*	403	/*

arch/x86/include/asm/msr-index.h+4−0

 /* V6 PMON MSR range */
 #define MSR_IA32_PMC_V6_GP0_CTR		0x1900
 #define MSR_IA32_PMC_V6_GP0_CFG_A	0x1901
+#define MSR_IA32_PMC_V6_GP0_CFG_B	0x1902
+#define MSR_IA32_PMC_V6_GP0_CFG_C	0x1903
 #define MSR_IA32_PMC_V6_FX0_CTR		0x1980
+#define MSR_IA32_PMC_V6_FX0_CFG_B	0x1982
+#define MSR_IA32_PMC_V6_FX0_CFG_C	0x1983
 #define MSR_IA32_PMC_V6_STEP		4
 /* KeyID partitioning between MKTME and TDX */

arch/x86/include/asm/perf_event.h+1−0

@@ -195,6 +195,7 @@ union cpuid10_edx { · 194 unmodified lines
195	*/	195	*/
196	#define ARCH_PERFMON_EXT_LEAF 0x00000023	196	#define ARCH_PERFMON_EXT_LEAF 0x00000023
197	#define ARCH_PERFMON_NUM_COUNTER_LEAF 0x1	197	#define ARCH_PERFMON_NUM_COUNTER_LEAF 0x1
		198	#define ARCH_PERFMON_ACR_LEAF 0x2
198		199
199	union cpuid35_eax {	200	union cpuid35_eax {
200	struct {	201	struct {

arch/x86/kernel/uprobes.c+5−0

 	insn_byte_t p;
 	int i;
+	/* x86_nops[insn->length]; same as jmp with .offs = 0 */
+	if (insn->length <= ASM_NOP_MAX &&
+	    !memcmp(insn->kaddr, x86_nops[insn->length], insn->length))
+		goto setup;
 	switch (opc1) {
 	case 0xeb:	/* jmp 8 */
 	case 0xe9:	/* jmp 32 */

arch/xtensa/kernel/perf_event.c+1−2

@@ -388,8 +388,7 @@ irqreturn_t xtensa_pmu_irq_handler(int irq, void *dev_id) · 387 unmodified lines
388	struct pt_regs *regs = get_irq_regs();	388	struct pt_regs *regs = get_irq_regs();
389		389
390	perf_sample_data_init(&data, 0, last_period);	390	perf_sample_data_init(&data, 0, last_period);
391	if (perf_event_overflow(event, &data, regs))	391	perf_event_overflow(event, &data, regs);
392	xtensa_pmu_stop(event, 0);
393	}	392	}
394		393
395	rc = IRQ_HANDLED;	394	rc = IRQ_HANDLED;

drivers/perf/apple_m1_cpu_pmu.c+1−2

@@ -474,8 +474,7 @@ static irqreturn_t m1_pmu_handle_irq(struct arm_pmu *cpu_pmu) · 473 unmodified lines
474	if (!armpmu_event_set_period(event))	474	if (!armpmu_event_set_period(event))
475	continue;	475	continue;
476		476
477	if (perf_event_overflow(event, &data, regs))	477	perf_event_overflow(event, &data, regs);
478	m1_pmu_disable_event(event);
479	}	478	}
480		479
481	cpu_pmu->start(cpu_pmu);	480	cpu_pmu->start(cpu_pmu);

drivers/perf/arm_pmuv3.c+1−2

@@ -887,8 +887,7 @@ static irqreturn_t armv8pmu_handle_irq(struct arm_pmu *cpu_pmu) · 886 unmodified lines
887	* an irq_work which will be taken care of in the handling of	887	* an irq_work which will be taken care of in the handling of
888	* IPI_IRQ_WORK.	888	* IPI_IRQ_WORK.
889	*/	889	*/
890	if (perf_event_overflow(event, &data, regs))	890	perf_event_overflow(event, &data, regs);
891	cpu_pmu->disable(event);
892	}	891	}
893	armv8pmu_start(cpu_pmu);	892	armv8pmu_start(cpu_pmu);
894		893

drivers/perf/arm_v6_pmu.c+1−2

@@ -276,8 +276,7 @@ armv6pmu_handle_irq(struct arm_pmu *cpu_pmu) · 275 unmodified lines
276	if (!armpmu_event_set_period(event))	276	if (!armpmu_event_set_period(event))
277	continue;	277	continue;
278		278
279	if (perf_event_overflow(event, &data, regs))	279	perf_event_overflow(event, &data, regs);
280	cpu_pmu->disable(event);
281	}	280	}
282		281
283	/*	282	/*

drivers/perf/arm_v7_pmu.c+1−2

@@ -930,8 +930,7 @@ static irqreturn_t armv7pmu_handle_irq(struct arm_pmu *cpu_pmu) · 929 unmodified lines
930	if (!armpmu_event_set_period(event))	930	if (!armpmu_event_set_period(event))
931	continue;	931	continue;
932		932
933	if (perf_event_overflow(event, &data, regs))	933	perf_event_overflow(event, &data, regs);
934	cpu_pmu->disable(event);
935	}	934	}
936		935
937	/*	936	/*

drivers/perf/arm_xscale_pmu.c+2−4

 		if (!armpmu_event_set_period(event))
 			continue;
-		if (perf_event_overflow(event, &data, regs))
+		perf_event_overflow(event, &data, regs);
-			cpu_pmu->disable(event);
 	}
 	irq_work_run();
 		if (!armpmu_event_set_period(event))
 			continue;
-		if (perf_event_overflow(event, &data, regs))
+		perf_event_overflow(event, &data, regs);
-			cpu_pmu->disable(event);
 	}
 	irq_work_run();

include/linux/cpuhotplug.h+0−1

@@ -60,7 +60,6 @@ enum cpuhp_state { · 59 unmodified lines
60	/* PREPARE section invoked on a control CPU */	60	/* PREPARE section invoked on a control CPU */
61	CPUHP_OFFLINE = 0,	61	CPUHP_OFFLINE = 0,
62	CPUHP_CREATE_THREADS,	62	CPUHP_CREATE_THREADS,
63	CPUHP_PERF_PREPARE,
64	CPUHP_PERF_X86_PREPARE,	63	CPUHP_PERF_X86_PREPARE,
65	CPUHP_PERF_X86_AMD_UNCORE_PREP,	64	CPUHP_PERF_X86_AMD_UNCORE_PREP,
66	CPUHP_PERF_POWER,	65	CPUHP_PERF_POWER,

include/linux/perf_event.h+167−131

@@ -26,18 +26,9 @@ · 25 unmodified lines
26	# include <asm/local64.h>	26	# include <asm/local64.h>
27	#endif	27	#endif
28		28
29	#define PERF_GUEST_ACTIVE 0x01
30	#define PERF_GUEST_USER 0x02
31
32	struct perf_guest_info_callbacks {
33	unsigned int (*state)(void);
34	unsigned long (*get_ip)(void);
35	unsigned int (*handle_intel_pt_intr)(void);
36	};
37
38	#ifdef CONFIG_HAVE_HW_BREAKPOINT	29	#ifdef CONFIG_HAVE_HW_BREAKPOINT
39	#include <linux/rhashtable-types.h>	30	# include <linux/rhashtable-types.h>
40	#include <asm/hw_breakpoint.h>	31	# include <asm/hw_breakpoint.h>
41	#endif	32	#endif
42		33
43	#include <linux/list.h>	34	#include <linux/list.h>
@@ -62,19 +53,20 @@ struct perf_guest_info_callbacks { · 18 unmodified lines
62	#include <linux/security.h>	53	#include <linux/security.h>
63	#include <linux/static_call.h>	54	#include <linux/static_call.h>
64	#include <linux/lockdep.h>	55	#include <linux/lockdep.h>
		56
65	#include <asm/local.h>	57	#include <asm/local.h>
66		58
67	struct perf_callchain_entry {	59	struct perf_callchain_entry {
68	__u64 nr;	60	u64 nr;
69	__u64 ip[]; /* /proc/sys/kernel/perf_event_max_stack */	61	u64 ip[]; /* /proc/sys/kernel/perf_event_max_stack */
70	};	62	};
71		63
72	struct perf_callchain_entry_ctx {	64	struct perf_callchain_entry_ctx {
73	struct perf_callchain_entry *entry;	65	struct perf_callchain_entry *entry;
74	u32 max_stack;	66	u32 max_stack;
75	u32 nr;	67	u32 nr;
76	short contexts;	68	short contexts;
77	bool contexts_maxed;	69	bool contexts_maxed;
78	};	70	};
79		71
80	typedef unsigned long (perf_copy_f)(void dst, const void *src,	72	typedef unsigned long (perf_copy_f)(void dst, const void *src,
@@ -121,8 +113,8 @@ static __always_inline bool perf_raw_frag_last(const struct perf_raw_frag *frag) · 40 unmodified lines
121	* already stored in age order, the hw_idx should be 0.	113	* already stored in age order, the hw_idx should be 0.
122	*/	114	*/
123	struct perf_branch_stack {	115	struct perf_branch_stack {
124	__u64 nr;	116	u64 nr;
125	__u64 hw_idx;	117	u64 hw_idx;
126	struct perf_branch_entry entries[];	118	struct perf_branch_entry entries[];
127	};	119	};
128		120
@@ -132,10 +124,10 @@ struct task_struct; · 3 unmodified lines
132	* extra PMU register associated with an event	124	* extra PMU register associated with an event
133	*/	125	*/
134	struct hw_perf_event_extra {	126	struct hw_perf_event_extra {
135	u64 config; /* register value */	127	u64 config; /* register value */
136	unsigned int reg; /* register address or index */	128	unsigned int reg; /* register address or index */
137	int alloc; /* extra register already allocated */	129	int alloc; /* extra register already allocated */
138	int idx; /* index in shared_regs->regs[] */	130	int idx; /* index in shared_regs->regs[] */
139	};	131	};
140		132
141	/**	133	/**
@@ -144,8 +136,8 @@ struct hw_perf_event_extra { · 2 unmodified lines
144	* PERF_EVENT_FLAG_ARCH bits are reserved for architecture-specific	136	* PERF_EVENT_FLAG_ARCH bits are reserved for architecture-specific
145	* usage.	137	* usage.
146	*/	138	*/
147	#define PERF_EVENT_FLAG_ARCH 0x000fffff	139	#define PERF_EVENT_FLAG_ARCH 0x0fffffff
148	#define PERF_EVENT_FLAG_USER_READ_CNT 0x80000000	140	#define PERF_EVENT_FLAG_USER_READ_CNT 0x80000000
149		141
150	static_assert((PERF_EVENT_FLAG_USER_READ_CNT & PERF_EVENT_FLAG_ARCH) == 0);	142	static_assert((PERF_EVENT_FLAG_USER_READ_CNT & PERF_EVENT_FLAG_ARCH) == 0);
151		143
@@ -157,7 +149,9 @@ struct hw_perf_event { · 5 unmodified lines
157	union {	149	union {
158	struct { /* hardware */	150	struct { /* hardware */
159	u64 config;	151	u64 config;
		152	u64 config1;
160	u64 last_tag;	153	u64 last_tag;
		154	u64 dyn_constraint;
161	unsigned long config_base;	155	unsigned long config_base;
162	unsigned long event_base;	156	unsigned long event_base;
163	int event_base_rdpmc;	157	int event_base_rdpmc;
@@ -225,9 +219,14 @@ struct hw_perf_event { · 61 unmodified lines
225	/*	219	/*
226	* hw_perf_event::state flags; used to track the PERF_EF_* state.	220	* hw_perf_event::state flags; used to track the PERF_EF_* state.
227	*/	221	*/
228	#define PERF_HES_STOPPED 0x01 /* the counter is stopped */	222
229	#define PERF_HES_UPTODATE 0x02 /* event->count up-to-date */	223	/* the counter is stopped */
230	#define PERF_HES_ARCH 0x04	224	#define PERF_HES_STOPPED 0x01
		225
		226	/* event->count up-to-date */
		227	#define PERF_HES_UPTODATE 0x02
		228
		229	#define PERF_HES_ARCH 0x04
231		230
232	int state;	231	int state;
233		232
@@ -276,7 +275,7 @@ struct hw_perf_event { · 42 unmodified lines
276	*/	275	*/
277	u64 freq_time_stamp;	276	u64 freq_time_stamp;
278	u64 freq_count_stamp;	277	u64 freq_count_stamp;
279	#endif	278	#endif /* CONFIG_PERF_EVENTS */
280	};	279	};
281		280
282	struct perf_event;	281	struct perf_event;
@@ -285,28 +284,33 @@ struct perf_event_pmu_context; · 2 unmodified lines
285	/*	284	/*
286	* Common implementation detail of pmu::{start,commit,cancel}_txn	285	* Common implementation detail of pmu::{start,commit,cancel}_txn
287	*/	286	*/
288	#define PERF_PMU_TXN_ADD 0x1 /* txn to add/schedule event on PMU */	287
289	#define PERF_PMU_TXN_READ 0x2 /* txn to read event group from PMU */	288	/* txn to add/schedule event on PMU */
		289	#define PERF_PMU_TXN_ADD 0x1
		290
		291	/* txn to read event group from PMU */
		292	#define PERF_PMU_TXN_READ 0x2
290		293
291	/**	294	/**
292	* pmu::capabilities flags	295	* pmu::capabilities flags
293	*/	296	*/
294	#define PERF_PMU_CAP_NO_INTERRUPT 0x0001	297	#define PERF_PMU_CAP_NO_INTERRUPT 0x0001
295	#define PERF_PMU_CAP_NO_NMI 0x0002	298	#define PERF_PMU_CAP_NO_NMI 0x0002
296	#define PERF_PMU_CAP_AUX_NO_SG 0x0004	299	#define PERF_PMU_CAP_AUX_NO_SG 0x0004
297	#define PERF_PMU_CAP_EXTENDED_REGS 0x0008	300	#define PERF_PMU_CAP_EXTENDED_REGS 0x0008
298	#define PERF_PMU_CAP_EXCLUSIVE 0x0010	301	#define PERF_PMU_CAP_EXCLUSIVE 0x0010
299	#define PERF_PMU_CAP_ITRACE 0x0020	302	#define PERF_PMU_CAP_ITRACE 0x0020
300	#define PERF_PMU_CAP_NO_EXCLUDE 0x0040	303	#define PERF_PMU_CAP_NO_EXCLUDE 0x0040
301	#define PERF_PMU_CAP_AUX_OUTPUT 0x0080	304	#define PERF_PMU_CAP_AUX_OUTPUT 0x0080
302	#define PERF_PMU_CAP_EXTENDED_HW_TYPE 0x0100	305	#define PERF_PMU_CAP_EXTENDED_HW_TYPE 0x0100
303	#define PERF_PMU_CAP_AUX_PAUSE 0x0200	306	#define PERF_PMU_CAP_AUX_PAUSE 0x0200
		307	#define PERF_PMU_CAP_AUX_PREFER_LARGE 0x0400
304		308
305	/**	309	/**
306	* pmu::scope	310	* pmu::scope
307	*/	311	*/
308	enum perf_pmu_scope {	312	enum perf_pmu_scope {
309	PERF_PMU_SCOPE_NONE = 0,	313	PERF_PMU_SCOPE_NONE = 0,
310	PERF_PMU_SCOPE_CORE,	314	PERF_PMU_SCOPE_CORE,
311	PERF_PMU_SCOPE_DIE,	315	PERF_PMU_SCOPE_DIE,
312	PERF_PMU_SCOPE_CLUSTER,	316	PERF_PMU_SCOPE_CLUSTER,
@@ -325,6 +329,9 @@ struct perf_output_handle; · 12 unmodified lines
325	struct pmu {	329	struct pmu {
326	struct list_head entry;	330	struct list_head entry;
327		331
		332	spinlock_t events_lock;
		333	struct list_head events;
		334
328	struct module *module;	335	struct module *module;
329	struct device *dev;	336	struct device *dev;
330	struct device *parent;	337	struct device *parent;
@@ -387,11 +394,21 @@ struct pmu { · 56 unmodified lines
387	* Flags for ->add()/->del()/ ->start()/->stop(). There are	394	* Flags for ->add()/->del()/ ->start()/->stop(). There are
388	* matching hw_perf_event::state flags.	395	* matching hw_perf_event::state flags.
389	*/	396	*/
390	#define PERF_EF_START 0x01 /* start the counter when adding */	397
391	#define PERF_EF_RELOAD 0x02 /* reload the counter when starting */	398	/* start the counter when adding */
392	#define PERF_EF_UPDATE 0x04 /* update the counter when stopping */	399	#define PERF_EF_START 0x01
393	#define PERF_EF_PAUSE 0x08 /* AUX area event, pause tracing */	400
394	#define PERF_EF_RESUME 0x10 /* AUX area event, resume tracing */	401	/* reload the counter when starting */
		402	#define PERF_EF_RELOAD 0x02
		403
		404	/* update the counter when stopping */
		405	#define PERF_EF_UPDATE 0x04
		406
		407	/* AUX area event, pause tracing */
		408	#define PERF_EF_PAUSE 0x08
		409
		410	/* AUX area event, resume tracing */
		411	#define PERF_EF_RESUME 0x10
395		412
396	/*	413	/*
397	* Adds/Removes a counter to/from the PMU, can be done inside a	414	* Adds/Removes a counter to/from the PMU, can be done inside a
@@ -590,10 +607,10 @@ enum perf_addr_filter_action_t { · 192 unmodified lines
590	* This is a hardware-agnostic filter configuration as specified by the user.	607	* This is a hardware-agnostic filter configuration as specified by the user.
591	*/	608	*/
592	struct perf_addr_filter {	609	struct perf_addr_filter {
593	struct list_head entry;	610	struct list_head entry;
594	struct path path;	611	struct path path;
595	unsigned long offset;	612	unsigned long offset;
596	unsigned long size;	613	unsigned long size;
597	enum perf_addr_filter_action_t action;	614	enum perf_addr_filter_action_t action;
598	};	615	};
599		616
@@ -608,23 +625,24 @@ struct perf_addr_filter { · 8 unmodified lines
608	* bundled together; see perf_event_addr_filters().	625	* bundled together; see perf_event_addr_filters().
609	*/	626	*/
610	struct perf_addr_filters_head {	627	struct perf_addr_filters_head {
611	struct list_head list;	628	struct list_head list;
612	raw_spinlock_t lock;	629	raw_spinlock_t lock;
613	unsigned int nr_file_filters;	630	unsigned int nr_file_filters;
614	};	631	};
615		632
616	struct perf_addr_filter_range {	633	struct perf_addr_filter_range {
617	unsigned long start;	634	unsigned long start;
618	unsigned long size;	635	unsigned long size;
619	};	636	};
620		637
621	/**	638	/**
622	* enum perf_event_state - the states of an event:	639	* enum perf_event_state - the states of an event:
623	*/	640	*/
624	enum perf_event_state {	641	enum perf_event_state {
625	PERF_EVENT_STATE_DEAD = -4,	642	PERF_EVENT_STATE_DEAD = -5,
626	PERF_EVENT_STATE_EXIT = -3,	643	PERF_EVENT_STATE_REVOKED = -4, /* pmu gone, must not touch */
627	PERF_EVENT_STATE_ERROR = -2,	644	PERF_EVENT_STATE_EXIT = -3, /* task died, still inherit */
		645	PERF_EVENT_STATE_ERROR = -2, /* scheduling error, can enable */
628	PERF_EVENT_STATE_OFF = -1,	646	PERF_EVENT_STATE_OFF = -1,
629	PERF_EVENT_STATE_INACTIVE = 0,	647	PERF_EVENT_STATE_INACTIVE = 0,
630	PERF_EVENT_STATE_ACTIVE = 1,	648	PERF_EVENT_STATE_ACTIVE = 1,
@@ -662,24 +680,24 @@ struct swevent_hlist { · 31 unmodified lines
662	struct rcu_head rcu_head;	680	struct rcu_head rcu_head;
663	};	681	};
664		682
665	#define PERF_ATTACH_CONTEXT 0x0001	683	#define PERF_ATTACH_CONTEXT 0x0001
666	#define PERF_ATTACH_GROUP 0x0002	684	#define PERF_ATTACH_GROUP 0x0002
667	#define PERF_ATTACH_TASK 0x0004	685	#define PERF_ATTACH_TASK 0x0004
668	#define PERF_ATTACH_TASK_DATA 0x0008	686	#define PERF_ATTACH_TASK_DATA 0x0008
669	#define PERF_ATTACH_GLOBAL_DATA 0x0010	687	#define PERF_ATTACH_GLOBAL_DATA 0x0010
670	#define PERF_ATTACH_SCHED_CB 0x0020	688	#define PERF_ATTACH_SCHED_CB 0x0020
671	#define PERF_ATTACH_CHILD 0x0040	689	#define PERF_ATTACH_CHILD 0x0040
672	#define PERF_ATTACH_EXCLUSIVE 0x0080	690	#define PERF_ATTACH_EXCLUSIVE 0x0080
673	#define PERF_ATTACH_CALLCHAIN 0x0100	691	#define PERF_ATTACH_CALLCHAIN 0x0100
674	#define PERF_ATTACH_ITRACE 0x0200	692	#define PERF_ATTACH_ITRACE 0x0200
675		693
676	struct bpf_prog;	694	struct bpf_prog;
677	struct perf_cgroup;	695	struct perf_cgroup;
678	struct perf_buffer;	696	struct perf_buffer;
679		697
680	struct pmu_event_list {	698	struct pmu_event_list {
681	raw_spinlock_t lock;	699	raw_spinlock_t lock;
682	struct list_head list;	700	struct list_head list;
683	};	701	};
684		702
685	/*	703	/*
@@ -689,12 +707,12 @@ struct pmu_event_list { · 3 unmodified lines
689	* disabled is sufficient since it will hold-off the IPIs.	707	* disabled is sufficient since it will hold-off the IPIs.
690	*/	708	*/
691	#ifdef CONFIG_PROVE_LOCKING	709	#ifdef CONFIG_PROVE_LOCKING
692	#define lockdep_assert_event_ctx(event) \	710	# define lockdep_assert_event_ctx(event) \
693	WARN_ON_ONCE(__lockdep_enabled && \	711	WARN_ON_ONCE(__lockdep_enabled && \
694	(this_cpu_read(hardirqs_enabled) && \	712	(this_cpu_read(hardirqs_enabled) && \
695	lockdep_is_held(&(event)->ctx->mutex) != LOCK_STATE_HELD))	713	lockdep_is_held(&(event)->ctx->mutex) != LOCK_STATE_HELD))
696	#else	714	#else
697	#define lockdep_assert_event_ctx(event)	715	# define lockdep_assert_event_ctx(event)
698	#endif	716	#endif
699		717
700	#define for_each_sibling_event(sibling, event) \	718	#define for_each_sibling_event(sibling, event) \
@@ -852,9 +870,9 @@ struct perf_event { · 151 unmodified lines
852	#ifdef CONFIG_EVENT_TRACING	870	#ifdef CONFIG_EVENT_TRACING
853	struct trace_event_call *tp_event;	871	struct trace_event_call *tp_event;
854	struct event_filter *filter;	872	struct event_filter *filter;
855	#ifdef CONFIG_FUNCTION_TRACER	873	# ifdef CONFIG_FUNCTION_TRACER
856	struct ftrace_ops ftrace_ops;	874	struct ftrace_ops ftrace_ops;
857	#endif	875	# endif
858	#endif	876	#endif
859		877
860	#ifdef CONFIG_CGROUP_PERF	878	#ifdef CONFIG_CGROUP_PERF
@@ -865,6 +883,7 @@ struct perf_event { · 4 unmodified lines
865	void *security;	883	void *security;
866	#endif	884	#endif
867	struct list_head sb_list;	885	struct list_head sb_list;
		886	struct list_head pmu_list;
868		887
869	/*	888	/*
870	* Certain events gets forwarded to another pmu internally by over-	889	* Certain events gets forwarded to another pmu internally by over-
@@ -872,7 +891,7 @@ struct perf_event { · 1 unmodified line
872	* of it. event->orig_type contains original 'type' requested by	891	* of it. event->orig_type contains original 'type' requested by
873	* user.	892	* user.
874	*/	893	*/
875	__u32 orig_type;	894	u32 orig_type;
876	#endif /* CONFIG_PERF_EVENTS */	895	#endif /* CONFIG_PERF_EVENTS */
877	};	896	};
878		897
@@ -937,8 +956,8 @@ static inline bool perf_pmu_ctx_is_active(struct perf_event_pmu_context *epc) · 58 unmodified lines
937	}	956	}
938		957
939	struct perf_event_groups {	958	struct perf_event_groups {
940	struct rb_root tree;	959	struct rb_root tree;
941	u64 index;	960	u64 index;
942	};	961	};
943		962
944		963
@@ -1155,7 +1174,7 @@ extern void perf_aux_output_flag(struct perf_output_handle *handle, u64 flags); · 210 unmodified lines
1155	extern void perf_event_itrace_started(struct perf_event *event);	1174	extern void perf_event_itrace_started(struct perf_event *event);
1156		1175
1157	extern int perf_pmu_register(struct pmu pmu, const char name, int type);	1176	extern int perf_pmu_register(struct pmu pmu, const char name, int type);
1158	extern void perf_pmu_unregister(struct pmu *pmu);	1177	extern int perf_pmu_unregister(struct pmu *pmu);
1159		1178
1160	extern void __perf_event_task_sched_in(struct task_struct *prev,	1179	extern void __perf_event_task_sched_in(struct task_struct *prev,
1161	struct task_struct *task);	1180	struct task_struct *task);
@@ -1181,16 +1200,18 @@ extern void perf_pmu_resched(struct pmu *pmu); · 19 unmodified lines
1181	extern int perf_event_refresh(struct perf_event *event, int refresh);	1200	extern int perf_event_refresh(struct perf_event *event, int refresh);
1182	extern void perf_event_update_userpage(struct perf_event *event);	1201	extern void perf_event_update_userpage(struct perf_event *event);
1183	extern int perf_event_release_kernel(struct perf_event *event);	1202	extern int perf_event_release_kernel(struct perf_event *event);
		1203
1184	extern struct perf_event *	1204	extern struct perf_event *
1185	perf_event_create_kernel_counter(struct perf_event_attr *attr,	1205	perf_event_create_kernel_counter(struct perf_event_attr *attr,
1186	int cpu,	1206	int cpu,
1187	struct task_struct *task,	1207	struct task_struct *task,
1188	perf_overflow_handler_t callback,	1208	perf_overflow_handler_t callback,
1189	void *context);	1209	void *context);
		1210
1190	extern void perf_pmu_migrate_context(struct pmu *pmu,	1211	extern void perf_pmu_migrate_context(struct pmu *pmu,
1191	int src_cpu, int dst_cpu);	1212	int src_cpu, int dst_cpu);
1192	int perf_event_read_local(struct perf_event event, u64 value,	1213	extern int perf_event_read_local(struct perf_event event, u64 value,
1193	u64 enabled, u64 running);	1214	u64 enabled, u64 running);
1194	extern u64 perf_event_read_value(struct perf_event *event,	1215	extern u64 perf_event_read_value(struct perf_event *event,
1195	u64 enabled, u64 running);	1216	u64 enabled, u64 running);
1196		1217
@@ -1407,14 +1428,14 @@ static inline u32 perf_sample_data_size(struct perf_sample_data *data, · 210 unmodified lines
1407	*/	1428	*/
1408	static inline void perf_clear_branch_entry_bitfields(struct perf_branch_entry *br)	1429	static inline void perf_clear_branch_entry_bitfields(struct perf_branch_entry *br)
1409	{	1430	{
1410	br->mispred = 0;	1431	br->mispred = 0;
1411	br->predicted = 0;	1432	br->predicted = 0;
1412	br->in_tx = 0;	1433	br->in_tx = 0;
1413	br->abort = 0;	1434	br->abort = 0;
1414	br->cycles = 0;	1435	br->cycles = 0;
1415	br->type = 0;	1436	br->type = 0;
1416	br->spec = PERF_BR_SPEC_NA;	1437	br->spec = PERF_BR_SPEC_NA;
1417	br->reserved = 0;	1438	br->reserved = 0;
1418	}	1439	}
1419		1440
1420	extern void perf_output_sample(struct perf_output_handle *handle,	1441	extern void perf_output_sample(struct perf_output_handle *handle,
@@ -1603,7 +1624,17 @@ extern void perf_event_bpf_event(struct bpf_prog *prog, · 182 unmodified lines
1603	enum perf_bpf_event_type type,	1624	enum perf_bpf_event_type type,
1604	u16 flags);	1625	u16 flags);
1605		1626
		1627	#define PERF_GUEST_ACTIVE 0x01
		1628	#define PERF_GUEST_USER 0x02
		1629
		1630	struct perf_guest_info_callbacks {
		1631	unsigned int (*state)(void);
		1632	unsigned long (*get_ip)(void);
		1633	unsigned int (*handle_intel_pt_intr)(void);
		1634	};
		1635
1606	#ifdef CONFIG_GUEST_PERF_EVENTS	1636	#ifdef CONFIG_GUEST_PERF_EVENTS
		1637
1607	extern struct perf_guest_info_callbacks __rcu *perf_guest_cbs;	1638	extern struct perf_guest_info_callbacks __rcu *perf_guest_cbs;
1608		1639
1609	DECLARE_STATIC_CALL(__perf_guest_state, *perf_guest_cbs->state);	1640	DECLARE_STATIC_CALL(__perf_guest_state, *perf_guest_cbs->state);
@@ -1614,21 +1645,27 @@ static inline unsigned int perf_guest_state(void) · 4 unmodified lines
1614	{	1645	{
1615	return static_call(__perf_guest_state)();	1646	return static_call(__perf_guest_state)();
1616	}	1647	}
		1648
1617	static inline unsigned long perf_guest_get_ip(void)	1649	static inline unsigned long perf_guest_get_ip(void)
1618	{	1650	{
1619	return static_call(__perf_guest_get_ip)();	1651	return static_call(__perf_guest_get_ip)();
1620	}	1652	}
		1653
1621	static inline unsigned int perf_guest_handle_intel_pt_intr(void)	1654	static inline unsigned int perf_guest_handle_intel_pt_intr(void)
1622	{	1655	{
1623	return static_call(__perf_guest_handle_intel_pt_intr)();	1656	return static_call(__perf_guest_handle_intel_pt_intr)();
1624	}	1657	}
		1658
1625	extern void perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs);	1659	extern void perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs);
1626	extern void perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs);	1660	extern void perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs);
1627	#else	1661
		1662	#else /* !CONFIG_GUEST_PERF_EVENTS: */
		1663
1628	static inline unsigned int perf_guest_state(void) { return 0; }	1664	static inline unsigned int perf_guest_state(void) { return 0; }
1629	static inline unsigned long perf_guest_get_ip(void) { return 0; }	1665	static inline unsigned long perf_guest_get_ip(void) { return 0; }
1630	static inline unsigned int perf_guest_handle_intel_pt_intr(void) { return 0; }	1666	static inline unsigned int perf_guest_handle_intel_pt_intr(void) { return 0; }
1631	#endif /* CONFIG_GUEST_PERF_EVENTS */	1667
		1668	#endif /* !CONFIG_GUEST_PERF_EVENTS */
1632		1669
1633	extern void perf_event_exec(void);	1670	extern void perf_event_exec(void);
1634	extern void perf_event_comm(struct task_struct *tsk, bool exec);	1671	extern void perf_event_comm(struct task_struct *tsk, bool exec);
@@ -1658,6 +1695,7 @@ static inline int perf_callchain_store_context(struct perf_callchain_entry_ctx * · 23 unmodified lines
1658	{	1695	{
1659	if (ctx->contexts < sysctl_perf_event_max_contexts_per_stack) {	1696	if (ctx->contexts < sysctl_perf_event_max_contexts_per_stack) {
1660	struct perf_callchain_entry *entry = ctx->entry;	1697	struct perf_callchain_entry *entry = ctx->entry;
		1698
1661	entry->ip[entry->nr++] = ip;	1699	entry->ip[entry->nr++] = ip;
1662	++ctx->contexts;	1700	++ctx->contexts;
1663	return 0;	1701	return 0;
@@ -1671,6 +1709,7 @@ static inline int perf_callchain_store(struct perf_callchain_entry_ctx *ctx, u64 · 7 unmodified lines
1671	{	1709	{
1672	if (ctx->nr < ctx->max_stack && !ctx->contexts_maxed) {	1710	if (ctx->nr < ctx->max_stack && !ctx->contexts_maxed) {
1673	struct perf_callchain_entry *entry = ctx->entry;	1711	struct perf_callchain_entry *entry = ctx->entry;
		1712
1674	entry->ip[entry->nr++] = ip;	1713	entry->ip[entry->nr++] = ip;
1675	++ctx->nr;	1714	++ctx->nr;
1676	return 0;	1715	return 0;
@@ -1697,7 +1736,7 @@ static inline int perf_is_paranoid(void) · 20 unmodified lines
1697	return sysctl_perf_event_paranoid > -1;	1736	return sysctl_perf_event_paranoid > -1;
1698	}	1737	}
1699		1738
1700	int perf_allow_kernel(void);	1739	extern int perf_allow_kernel(void);
1701		1740
1702	static inline int perf_allow_cpu(void)	1741	static inline int perf_allow_cpu(void)
1703	{	1742	{
@@ -1760,7 +1799,7 @@ static inline bool needs_branch_stack(struct perf_event *event) · 56 unmodified lines
1760		1799
1761	static inline bool has_aux(struct perf_event *event)	1800	static inline bool has_aux(struct perf_event *event)
1762	{	1801	{
1763	return event->pmu->setup_aux;	1802	return event->pmu && event->pmu->setup_aux;
1764	}	1803	}
1765		1804
1766	static inline bool has_aux_action(struct perf_event *event)	1805	static inline bool has_aux_action(struct perf_event *event)
@@ -1819,7 +1858,7 @@ extern int perf_output_begin_backward(struct perf_output_handle *handle, · 52 unmodified lines
1819		1858
1820	extern void perf_output_end(struct perf_output_handle *handle);	1859	extern void perf_output_end(struct perf_output_handle *handle);
1821	extern unsigned int perf_output_copy(struct perf_output_handle *handle,	1860	extern unsigned int perf_output_copy(struct perf_output_handle *handle,
1822	const void *buf, unsigned int len);	1861	const void *buf, unsigned int len);
1823	extern unsigned int perf_output_skip(struct perf_output_handle *handle,	1862	extern unsigned int perf_output_skip(struct perf_output_handle *handle,
1824	unsigned int len);	1863	unsigned int len);
1825	extern long perf_output_copy_aux(struct perf_output_handle *aux_handle,	1864	extern long perf_output_copy_aux(struct perf_output_handle *aux_handle,
@@ -1836,7 +1875,9 @@ extern void perf_event_task_tick(void); · 10 unmodified lines
1836	extern int perf_event_account_interrupt(struct perf_event *event);	1875	extern int perf_event_account_interrupt(struct perf_event *event);
1837	extern int perf_event_period(struct perf_event *event, u64 value);	1876	extern int perf_event_period(struct perf_event *event, u64 value);
1838	extern u64 perf_event_pause(struct perf_event *event, bool reset);	1877	extern u64 perf_event_pause(struct perf_event *event, bool reset);
		1878
1839	#else /* !CONFIG_PERF_EVENTS: */	1879	#else /* !CONFIG_PERF_EVENTS: */
		1880
1840	static inline void *	1881	static inline void *
1841	perf_aux_output_begin(struct perf_output_handle *handle,	1882	perf_aux_output_begin(struct perf_output_handle *handle,
1842	struct perf_event *event) { return NULL; }	1883	struct perf_event *event) { return NULL; }
@@ -1914,19 +1955,14 @@ static inline void perf_event_disable(struct perf_event *event) { } · 71 unmodified lines
1914	static inline int __perf_event_disable(void *info) { return -1; }	1955	static inline int __perf_event_disable(void *info) { return -1; }
1915	static inline void perf_event_task_tick(void) { }	1956	static inline void perf_event_task_tick(void) { }
1916	static inline int perf_event_release_kernel(struct perf_event *event) { return 0; }	1957	static inline int perf_event_release_kernel(struct perf_event *event) { return 0; }
1917	static inline int perf_event_period(struct perf_event *event, u64 value)	1958	static inline int
1918	{	1959	perf_event_period(struct perf_event *event, u64 value) { return -EINVAL; }
1919	return -EINVAL;	1960	static inline u64
1920	}	1961	perf_event_pause(struct perf_event *event, bool reset) { return 0; }
1921	static inline u64 perf_event_pause(struct perf_event *event, bool reset)	1962	static inline int
1922	{	1963	perf_exclude_event(struct perf_event event, struct pt_regs regs) { return 0; }
1923	return 0;	1964
1924	}	1965	#endif /* !CONFIG_PERF_EVENTS */
1925	static inline int perf_exclude_event(struct perf_event event, struct pt_regs regs)
1926	{
1927	return 0;
1928	}
1929	#endif
1930		1966
1931	#if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_INTEL)	1967	#if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_INTEL)
1932	extern void perf_restore_debug_store(void);	1968	extern void perf_restore_debug_store(void);
@@ -1934,31 +1970,31 @@ extern void perf_restore_debug_store(void); · 1 unmodified line
1934	static inline void perf_restore_debug_store(void) { }	1970	static inline void perf_restore_debug_store(void) { }
1935	#endif	1971	#endif
1936		1972
1937	#define perf_output_put(handle, x) perf_output_copy((handle), &(x), sizeof(x))	1973	#define perf_output_put(handle, x) perf_output_copy((handle), &(x), sizeof(x))
1938		1974
1939	struct perf_pmu_events_attr {	1975	struct perf_pmu_events_attr {
1940	struct device_attribute attr;	1976	struct device_attribute attr;
1941	u64 id;	1977	u64 id;
1942	const char *event_str;	1978	const char *event_str;
1943	};	1979	};
1944		1980
1945	struct perf_pmu_events_ht_attr {	1981	struct perf_pmu_events_ht_attr {
1946	struct device_attribute attr;	1982	struct device_attribute attr;
1947	u64 id;	1983	u64 id;
1948	const char *event_str_ht;	1984	const char *event_str_ht;
1949	const char *event_str_noht;	1985	const char *event_str_noht;
1950	};	1986	};
1951		1987
1952	struct perf_pmu_events_hybrid_attr {	1988	struct perf_pmu_events_hybrid_attr {
1953	struct device_attribute attr;	1989	struct device_attribute attr;
1954	u64 id;	1990	u64 id;
1955	const char *event_str;	1991	const char *event_str;
1956	u64 pmu_type;	1992	u64 pmu_type;
1957	};	1993	};
1958		1994
1959	struct perf_pmu_format_hybrid_attr {	1995	struct perf_pmu_format_hybrid_attr {
1960	struct device_attribute attr;	1996	struct device_attribute attr;
1961	u64 pmu_type;	1997	u64 pmu_type;
1962	};	1998	};
1963		1999
1964	ssize_t perf_event_sysfs_show(struct device dev, struct device_attribute attr,	2000	ssize_t perf_event_sysfs_show(struct device dev, struct device_attribute attr,
@@ -2000,11 +2036,11 @@ static struct device_attribute format_attr_##_name = __ATTR_RO(_name) · 35 unmodified lines
2000		2036
2001	/* Performance counter hotplug functions */	2037	/* Performance counter hotplug functions */
2002	#ifdef CONFIG_PERF_EVENTS	2038	#ifdef CONFIG_PERF_EVENTS
2003	int perf_event_init_cpu(unsigned int cpu);	2039	extern int perf_event_init_cpu(unsigned int cpu);
2004	int perf_event_exit_cpu(unsigned int cpu);	2040	extern int perf_event_exit_cpu(unsigned int cpu);
2005	#else	2041	#else
2006	#define perf_event_init_cpu NULL	2042	# define perf_event_init_cpu NULL
2007	#define perf_event_exit_cpu NULL	2043	# define perf_event_exit_cpu NULL
2008	#endif	2044	#endif
2009		2045
2010	extern void arch_perf_update_userpage(struct perf_event *event,	2046	extern void arch_perf_update_userpage(struct perf_event *event,

include/uapi/linux/perf_event.h+334−323

@@ -39,18 +39,21 @@ enum perf_type_id { · 38 unmodified lines
39		39
40	/*	40	/*
41	* attr.config layout for type PERF_TYPE_HARDWARE and PERF_TYPE_HW_CACHE	41	* attr.config layout for type PERF_TYPE_HARDWARE and PERF_TYPE_HW_CACHE
		42	*
42	* PERF_TYPE_HARDWARE: 0xEEEEEEEE000000AA	43	* PERF_TYPE_HARDWARE: 0xEEEEEEEE000000AA
43	* AA: hardware event ID	44	* AA: hardware event ID
44	* EEEEEEEE: PMU type ID	45	* EEEEEEEE: PMU type ID
		46	*
45	* PERF_TYPE_HW_CACHE: 0xEEEEEEEE00DDCCBB	47	* PERF_TYPE_HW_CACHE: 0xEEEEEEEE00DDCCBB
46	* BB: hardware cache ID	48	* BB: hardware cache ID
47	* CC: hardware cache op ID	49	* CC: hardware cache op ID
48	* DD: hardware cache op result ID	50	* DD: hardware cache op result ID
49	* EEEEEEEE: PMU type ID	51	* EEEEEEEE: PMU type ID
50	* If the PMU type ID is 0, the PERF_TYPE_RAW will be applied.	52	*
		53	* If the PMU type ID is 0, PERF_TYPE_RAW will be applied.
51	*/	54	*/
52	#define PERF_PMU_TYPE_SHIFT 32	55	#define PERF_PMU_TYPE_SHIFT 32
53	#define PERF_HW_EVENT_MASK 0xffffffff	56	#define PERF_HW_EVENT_MASK 0xffffffff
54		57
55	/*	58	/*
56	* Generalized performance event event_id types, used by the	59	* Generalized performance event event_id types, used by the
@@ -112,7 +115,7 @@ enum perf_hw_cache_op_result_id { · 55 unmodified lines
112	/*	115	/*
113	* Special "software" events provided by the kernel, even if the hardware	116	* Special "software" events provided by the kernel, even if the hardware
114	* does not support performance events. These events measure various	117	* does not support performance events. These events measure various
115	* physical and sw events of the kernel (and allow the profiling of them as	118	* physical and SW events of the kernel (and allow the profiling of them as
116	* well):	119	* well):
117	*/	120	*/
118	enum perf_sw_ids {	121	enum perf_sw_ids {
@@ -167,8 +170,9 @@ enum perf_event_sample_format { · 48 unmodified lines
167	};	170	};
168		171
169	#define PERF_SAMPLE_WEIGHT_TYPE (PERF_SAMPLE_WEIGHT \| PERF_SAMPLE_WEIGHT_STRUCT)	172	#define PERF_SAMPLE_WEIGHT_TYPE (PERF_SAMPLE_WEIGHT \| PERF_SAMPLE_WEIGHT_STRUCT)
		173
170	/*	174	/*
171	* values to program into branch_sample_type when PERF_SAMPLE_BRANCH is set	175	* Values to program into branch_sample_type when PERF_SAMPLE_BRANCH is set.
172	*	176	*
173	* If the user does not pass priv level information via branch_sample_type,	177	* If the user does not pass priv level information via branch_sample_type,
174	* the kernel uses the event's priv level. Branch and event priv levels do	178	* the kernel uses the event's priv level. Branch and event priv levels do
@@ -178,20 +182,20 @@ enum perf_event_sample_format { · 3 unmodified lines
178	* of branches and therefore it supersedes all the other types.	182	* of branches and therefore it supersedes all the other types.
179	*/	183	*/
180	enum perf_branch_sample_type_shift {	184	enum perf_branch_sample_type_shift {
181	PERF_SAMPLE_BRANCH_USER_SHIFT = 0, /* user branches */	185	PERF_SAMPLE_BRANCH_USER_SHIFT = 0, /* user branches */
182	PERF_SAMPLE_BRANCH_KERNEL_SHIFT = 1, /* kernel branches */	186	PERF_SAMPLE_BRANCH_KERNEL_SHIFT = 1, /* kernel branches */
183	PERF_SAMPLE_BRANCH_HV_SHIFT = 2, /* hypervisor branches */	187	PERF_SAMPLE_BRANCH_HV_SHIFT = 2, /* hypervisor branches */
184		188
185	PERF_SAMPLE_BRANCH_ANY_SHIFT = 3, /* any branch types */	189	PERF_SAMPLE_BRANCH_ANY_SHIFT = 3, /* any branch types */
186	PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT = 4, /* any call branch */	190	PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT = 4, /* any call branch */
187	PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT = 5, /* any return branch */	191	PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT = 5, /* any return branch */
188	PERF_SAMPLE_BRANCH_IND_CALL_SHIFT = 6, /* indirect calls */	192	PERF_SAMPLE_BRANCH_IND_CALL_SHIFT = 6, /* indirect calls */
189	PERF_SAMPLE_BRANCH_ABORT_TX_SHIFT = 7, /* transaction aborts */	193	PERF_SAMPLE_BRANCH_ABORT_TX_SHIFT = 7, /* transaction aborts */
190	PERF_SAMPLE_BRANCH_IN_TX_SHIFT = 8, /* in transaction */	194	PERF_SAMPLE_BRANCH_IN_TX_SHIFT = 8, /* in transaction */
191	PERF_SAMPLE_BRANCH_NO_TX_SHIFT = 9, /* not in transaction */	195	PERF_SAMPLE_BRANCH_NO_TX_SHIFT = 9, /* not in transaction */
192	PERF_SAMPLE_BRANCH_COND_SHIFT = 10, /* conditional branches */	196	PERF_SAMPLE_BRANCH_COND_SHIFT = 10, /* conditional branches */
193		197
194	PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT = 11, /* call/ret stack */	198	PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT = 11, /* CALL/RET stack */
195	PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT = 12, /* indirect jumps */	199	PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT = 12, /* indirect jumps */
196	PERF_SAMPLE_BRANCH_CALL_SHIFT = 13, /* direct call */	200	PERF_SAMPLE_BRANCH_CALL_SHIFT = 13, /* direct call */
197		201
@@ -210,96 +214,95 @@ enum perf_branch_sample_type_shift { · 12 unmodified lines
210	};	214	};
211		215
212	enum perf_branch_sample_type {	216	enum perf_branch_sample_type {
213	PERF_SAMPLE_BRANCH_USER = 1U << PERF_SAMPLE_BRANCH_USER_SHIFT,	217	PERF_SAMPLE_BRANCH_USER = 1U << PERF_SAMPLE_BRANCH_USER_SHIFT,
214	PERF_SAMPLE_BRANCH_KERNEL = 1U << PERF_SAMPLE_BRANCH_KERNEL_SHIFT,	218	PERF_SAMPLE_BRANCH_KERNEL = 1U << PERF_SAMPLE_BRANCH_KERNEL_SHIFT,
215	PERF_SAMPLE_BRANCH_HV = 1U << PERF_SAMPLE_BRANCH_HV_SHIFT,	219	PERF_SAMPLE_BRANCH_HV = 1U << PERF_SAMPLE_BRANCH_HV_SHIFT,
216		220
217	PERF_SAMPLE_BRANCH_ANY = 1U << PERF_SAMPLE_BRANCH_ANY_SHIFT,	221	PERF_SAMPLE_BRANCH_ANY = 1U << PERF_SAMPLE_BRANCH_ANY_SHIFT,
218	PERF_SAMPLE_BRANCH_ANY_CALL = 1U << PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT,	222	PERF_SAMPLE_BRANCH_ANY_CALL = 1U << PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT,
219	PERF_SAMPLE_BRANCH_ANY_RETURN = 1U << PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT,	223	PERF_SAMPLE_BRANCH_ANY_RETURN = 1U << PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT,
220	PERF_SAMPLE_BRANCH_IND_CALL = 1U << PERF_SAMPLE_BRANCH_IND_CALL_SHIFT,	224	PERF_SAMPLE_BRANCH_IND_CALL = 1U << PERF_SAMPLE_BRANCH_IND_CALL_SHIFT,
221	PERF_SAMPLE_BRANCH_ABORT_TX = 1U << PERF_SAMPLE_BRANCH_ABORT_TX_SHIFT,	225	PERF_SAMPLE_BRANCH_ABORT_TX = 1U << PERF_SAMPLE_BRANCH_ABORT_TX_SHIFT,
222	PERF_SAMPLE_BRANCH_IN_TX = 1U << PERF_SAMPLE_BRANCH_IN_TX_SHIFT,	226	PERF_SAMPLE_BRANCH_IN_TX = 1U << PERF_SAMPLE_BRANCH_IN_TX_SHIFT,
223	PERF_SAMPLE_BRANCH_NO_TX = 1U << PERF_SAMPLE_BRANCH_NO_TX_SHIFT,	227	PERF_SAMPLE_BRANCH_NO_TX = 1U << PERF_SAMPLE_BRANCH_NO_TX_SHIFT,
224	PERF_SAMPLE_BRANCH_COND = 1U << PERF_SAMPLE_BRANCH_COND_SHIFT,	228	PERF_SAMPLE_BRANCH_COND = 1U << PERF_SAMPLE_BRANCH_COND_SHIFT,
225		229
226	PERF_SAMPLE_BRANCH_CALL_STACK = 1U << PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT,	230	PERF_SAMPLE_BRANCH_CALL_STACK = 1U << PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT,
227	PERF_SAMPLE_BRANCH_IND_JUMP = 1U << PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT,	231	PERF_SAMPLE_BRANCH_IND_JUMP = 1U << PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT,
228	PERF_SAMPLE_BRANCH_CALL = 1U << PERF_SAMPLE_BRANCH_CALL_SHIFT,	232	PERF_SAMPLE_BRANCH_CALL = 1U << PERF_SAMPLE_BRANCH_CALL_SHIFT,
229		233
230	PERF_SAMPLE_BRANCH_NO_FLAGS = 1U << PERF_SAMPLE_BRANCH_NO_FLAGS_SHIFT,	234	PERF_SAMPLE_BRANCH_NO_FLAGS = 1U << PERF_SAMPLE_BRANCH_NO_FLAGS_SHIFT,
231	PERF_SAMPLE_BRANCH_NO_CYCLES = 1U << PERF_SAMPLE_BRANCH_NO_CYCLES_SHIFT,	235	PERF_SAMPLE_BRANCH_NO_CYCLES = 1U << PERF_SAMPLE_BRANCH_NO_CYCLES_SHIFT,
232		236
233	PERF_SAMPLE_BRANCH_TYPE_SAVE =	237	PERF_SAMPLE_BRANCH_TYPE_SAVE = 1U << PERF_SAMPLE_BRANCH_TYPE_SAVE_SHIFT,
234	1U << PERF_SAMPLE_BRANCH_TYPE_SAVE_SHIFT,
235		238
236	PERF_SAMPLE_BRANCH_HW_INDEX = 1U << PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT,	239	PERF_SAMPLE_BRANCH_HW_INDEX = 1U << PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT,
237		240
238	PERF_SAMPLE_BRANCH_PRIV_SAVE = 1U << PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT,	241	PERF_SAMPLE_BRANCH_PRIV_SAVE = 1U << PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT,
239		242
240	PERF_SAMPLE_BRANCH_COUNTERS = 1U << PERF_SAMPLE_BRANCH_COUNTERS_SHIFT,	243	PERF_SAMPLE_BRANCH_COUNTERS = 1U << PERF_SAMPLE_BRANCH_COUNTERS_SHIFT,
241		244
242	PERF_SAMPLE_BRANCH_MAX = 1U << PERF_SAMPLE_BRANCH_MAX_SHIFT,	245	PERF_SAMPLE_BRANCH_MAX = 1U << PERF_SAMPLE_BRANCH_MAX_SHIFT,
243	};	246	};
244		247
245	/*	248	/*
246	* Common flow change classification	249	* Common control flow change classifications:
247	*/	250	*/
248	enum {	251	enum {
249	PERF_BR_UNKNOWN = 0, /* unknown */	252	PERF_BR_UNKNOWN = 0, /* Unknown */
250	PERF_BR_COND = 1, /* conditional */	253	PERF_BR_COND = 1, /* Conditional */
251	PERF_BR_UNCOND = 2, /* unconditional */	254	PERF_BR_UNCOND = 2, /* Unconditional */
252	PERF_BR_IND = 3, /* indirect */	255	PERF_BR_IND = 3, /* Indirect */
253	PERF_BR_CALL = 4, /* function call */	256	PERF_BR_CALL = 4, /* Function call */
254	PERF_BR_IND_CALL = 5, /* indirect function call */	257	PERF_BR_IND_CALL = 5, /* Indirect function call */
255	PERF_BR_RET = 6, /* function return */	258	PERF_BR_RET = 6, /* Function return */
256	PERF_BR_SYSCALL = 7, /* syscall */	259	PERF_BR_SYSCALL = 7, /* Syscall */
257	PERF_BR_SYSRET = 8, /* syscall return */	260	PERF_BR_SYSRET = 8, /* Syscall return */
258	PERF_BR_COND_CALL = 9, /* conditional function call */	261	PERF_BR_COND_CALL = 9, /* Conditional function call */
259	PERF_BR_COND_RET = 10, /* conditional function return */	262	PERF_BR_COND_RET = 10, /* Conditional function return */
260	PERF_BR_ERET = 11, /* exception return */	263	PERF_BR_ERET = 11, /* Exception return */
261	PERF_BR_IRQ = 12, /* irq */	264	PERF_BR_IRQ = 12, /* IRQ */
262	PERF_BR_SERROR = 13, /* system error */	265	PERF_BR_SERROR = 13, /* System error */
263	PERF_BR_NO_TX = 14, /* not in transaction */	266	PERF_BR_NO_TX = 14, /* Not in transaction */
264	PERF_BR_EXTEND_ABI = 15, /* extend ABI */	267	PERF_BR_EXTEND_ABI = 15, /* Extend ABI */
265	PERF_BR_MAX,	268	PERF_BR_MAX,
266	};	269	};
267		270
268	/*	271	/*
269	* Common branch speculation outcome classification	272	* Common branch speculation outcome classifications:
270	*/	273	*/
271	enum {	274	enum {
272	PERF_BR_SPEC_NA = 0, /* Not available */	275	PERF_BR_SPEC_NA = 0, /* Not available */
273	PERF_BR_SPEC_WRONG_PATH = 1, /* Speculative but on wrong path */	276	PERF_BR_SPEC_WRONG_PATH = 1, /* Speculative but on wrong path */
274	PERF_BR_NON_SPEC_CORRECT_PATH = 2, /* Non-speculative but on correct path */	277	PERF_BR_NON_SPEC_CORRECT_PATH = 2, /* Non-speculative but on correct path */
275	PERF_BR_SPEC_CORRECT_PATH = 3, /* Speculative and on correct path */	278	PERF_BR_SPEC_CORRECT_PATH = 3, /* Speculative and on correct path */
276	PERF_BR_SPEC_MAX,	279	PERF_BR_SPEC_MAX,
277	};	280	};
278		281
279	enum {	282	enum {
280	PERF_BR_NEW_FAULT_ALGN = 0, /* Alignment fault */	283	PERF_BR_NEW_FAULT_ALGN = 0, /* Alignment fault */
281	PERF_BR_NEW_FAULT_DATA = 1, /* Data fault */	284	PERF_BR_NEW_FAULT_DATA = 1, /* Data fault */
282	PERF_BR_NEW_FAULT_INST = 2, /* Inst fault */	285	PERF_BR_NEW_FAULT_INST = 2, /* Inst fault */
283	PERF_BR_NEW_ARCH_1 = 3, /* Architecture specific */	286	PERF_BR_NEW_ARCH_1 = 3, /* Architecture specific */
284	PERF_BR_NEW_ARCH_2 = 4, /* Architecture specific */	287	PERF_BR_NEW_ARCH_2 = 4, /* Architecture specific */
285	PERF_BR_NEW_ARCH_3 = 5, /* Architecture specific */	288	PERF_BR_NEW_ARCH_3 = 5, /* Architecture specific */
286	PERF_BR_NEW_ARCH_4 = 6, /* Architecture specific */	289	PERF_BR_NEW_ARCH_4 = 6, /* Architecture specific */
287	PERF_BR_NEW_ARCH_5 = 7, /* Architecture specific */	290	PERF_BR_NEW_ARCH_5 = 7, /* Architecture specific */
288	PERF_BR_NEW_MAX,	291	PERF_BR_NEW_MAX,
289	};	292	};
290		293
291	enum {	294	enum {
292	PERF_BR_PRIV_UNKNOWN = 0,	295	PERF_BR_PRIV_UNKNOWN = 0,
293	PERF_BR_PRIV_USER = 1,	296	PERF_BR_PRIV_USER = 1,
294	PERF_BR_PRIV_KERNEL = 2,	297	PERF_BR_PRIV_KERNEL = 2,
295	PERF_BR_PRIV_HV = 3,	298	PERF_BR_PRIV_HV = 3,
296	};	299	};
297		300
298	#define PERF_BR_ARM64_FIQ PERF_BR_NEW_ARCH_1	301	#define PERF_BR_ARM64_FIQ PERF_BR_NEW_ARCH_1
299	#define PERF_BR_ARM64_DEBUG_HALT PERF_BR_NEW_ARCH_2	302	#define PERF_BR_ARM64_DEBUG_HALT PERF_BR_NEW_ARCH_2
300	#define PERF_BR_ARM64_DEBUG_EXIT PERF_BR_NEW_ARCH_3	303	#define PERF_BR_ARM64_DEBUG_EXIT PERF_BR_NEW_ARCH_3
301	#define PERF_BR_ARM64_DEBUG_INST PERF_BR_NEW_ARCH_4	304	#define PERF_BR_ARM64_DEBUG_INST PERF_BR_NEW_ARCH_4
302	#define PERF_BR_ARM64_DEBUG_DATA PERF_BR_NEW_ARCH_5	305	#define PERF_BR_ARM64_DEBUG_DATA PERF_BR_NEW_ARCH_5
303		306
304	#define PERF_SAMPLE_BRANCH_PLM_ALL \	307	#define PERF_SAMPLE_BRANCH_PLM_ALL \
305	(PERF_SAMPLE_BRANCH_USER\|\	308	(PERF_SAMPLE_BRANCH_USER\|\
@@ -310,9 +313,9 @@ enum { · 4 unmodified lines
310	* Values to determine ABI of the registers dump.	313	* Values to determine ABI of the registers dump.
311	*/	314	*/
312	enum perf_sample_regs_abi {	315	enum perf_sample_regs_abi {
313	PERF_SAMPLE_REGS_ABI_NONE = 0,	316	PERF_SAMPLE_REGS_ABI_NONE = 0,
314	PERF_SAMPLE_REGS_ABI_32 = 1,	317	PERF_SAMPLE_REGS_ABI_32 = 1,
315	PERF_SAMPLE_REGS_ABI_64 = 2,	318	PERF_SAMPLE_REGS_ABI_64 = 2,
316	};	319	};
317		320
318	/*	321	/*
@@ -320,21 +323,21 @@ enum perf_sample_regs_abi { · 1 unmodified line
320	* abort events. Multiple bits can be set.	323	* abort events. Multiple bits can be set.
321	*/	324	*/
322	enum {	325	enum {
323	PERF_TXN_ELISION = (1 << 0), /* From elision */	326	PERF_TXN_ELISION = (1 << 0), /* From elision */
324	PERF_TXN_TRANSACTION = (1 << 1), /* From transaction */	327	PERF_TXN_TRANSACTION = (1 << 1), /* From transaction */
325	PERF_TXN_SYNC = (1 << 2), /* Instruction is related */	328	PERF_TXN_SYNC = (1 << 2), /* Instruction is related */
326	PERF_TXN_ASYNC = (1 << 3), /* Instruction not related */	329	PERF_TXN_ASYNC = (1 << 3), /* Instruction is not related */
327	PERF_TXN_RETRY = (1 << 4), /* Retry possible */	330	PERF_TXN_RETRY = (1 << 4), /* Retry possible */
328	PERF_TXN_CONFLICT = (1 << 5), /* Conflict abort */	331	PERF_TXN_CONFLICT = (1 << 5), /* Conflict abort */
329	PERF_TXN_CAPACITY_WRITE = (1 << 6), /* Capacity write abort */	332	PERF_TXN_CAPACITY_WRITE = (1 << 6), /* Capacity write abort */
330	PERF_TXN_CAPACITY_READ = (1 << 7), /* Capacity read abort */	333	PERF_TXN_CAPACITY_READ = (1 << 7), /* Capacity read abort */
331		334
332	PERF_TXN_MAX = (1 << 8), /* non-ABI */	335	PERF_TXN_MAX = (1 << 8), /* non-ABI */
333		336
334	/* bits 32..63 are reserved for the abort code */	337	/* Bits 32..63 are reserved for the abort code */
335		338
336	PERF_TXN_ABORT_MASK = (0xffffffffULL << 32),	339	PERF_TXN_ABORT_MASK = (0xffffffffULL << 32),
337	PERF_TXN_ABORT_SHIFT = 32,	340	PERF_TXN_ABORT_SHIFT = 32,
338	};	341	};
339		342
340	/*	343	/*
@@ -369,24 +372,22 @@ enum perf_event_read_format { · 28 unmodified lines
369	PERF_FORMAT_MAX = 1U << 5, /* non-ABI */	372	PERF_FORMAT_MAX = 1U << 5, /* non-ABI */
370	};	373	};
371		374
372	#define PERF_ATTR_SIZE_VER0 64 /* sizeof first published struct */	375	#define PERF_ATTR_SIZE_VER0 64 /* Size of first published 'struct perf_event_attr' */
373	#define PERF_ATTR_SIZE_VER1 72 /* add: config2 */	376	#define PERF_ATTR_SIZE_VER1 72 /* Add: config2 */
374	#define PERF_ATTR_SIZE_VER2 80 /* add: branch_sample_type */	377	#define PERF_ATTR_SIZE_VER2 80 /* Add: branch_sample_type */
375	#define PERF_ATTR_SIZE_VER3 96 /* add: sample_regs_user */	378	#define PERF_ATTR_SIZE_VER3 96 /* Add: sample_regs_user */
376	/* add: sample_stack_user */	379	/* Add: sample_stack_user */
377	#define PERF_ATTR_SIZE_VER4 104 /* add: sample_regs_intr */	380	#define PERF_ATTR_SIZE_VER4 104 /* Add: sample_regs_intr */
378	#define PERF_ATTR_SIZE_VER5 112 /* add: aux_watermark */	381	#define PERF_ATTR_SIZE_VER5 112 /* Add: aux_watermark */
379	#define PERF_ATTR_SIZE_VER6 120 /* add: aux_sample_size */	382	#define PERF_ATTR_SIZE_VER6 120 /* Add: aux_sample_size */
380	#define PERF_ATTR_SIZE_VER7 128 /* add: sig_data */	383	#define PERF_ATTR_SIZE_VER7 128 /* Add: sig_data */
381	#define PERF_ATTR_SIZE_VER8 136 /* add: config3 */	384	#define PERF_ATTR_SIZE_VER8 136 /* Add: config3 */
382		385
383	/*	386	/*
384	* Hardware event_id to monitor via a performance monitoring event:	387	* 'struct perf_event_attr' contains various attributes that define
385	*	388	* a performance event - most of them hardware related configuration
386	* @sample_max_stack: Max number of frame pointers in a callchain,	389	* details, but also a lot of behavioral switches and values implemented
387	* should be < /proc/sys/kernel/perf_event_max_stack	390	* by the kernel.
388	* Max number of entries of branch stack
389	* should be < hardware limit
390	*/	391	*/
391	struct perf_event_attr {	392	struct perf_event_attr {
392		393
@@ -396,7 +397,7 @@ struct perf_event_attr { · 3 unmodified lines
396	__u32 type;	397	__u32 type;
397		398
398	/*	399	/*
399	* Size of the attr structure, for fwd/bwd compat.	400	* Size of the attr structure, for forward/backwards compatibility.
400	*/	401	*/
401	__u32 size;	402	__u32 size;
402		403
@@ -451,21 +452,21 @@ struct perf_event_attr { · 48 unmodified lines
451	comm_exec : 1, /* flag comm events that are due to an exec */	452	comm_exec : 1, /* flag comm events that are due to an exec */
452	use_clockid : 1, /* use @clockid for time fields */	453	use_clockid : 1, /* use @clockid for time fields */
453	context_switch : 1, /* context switch data */	454	context_switch : 1, /* context switch data */
454	write_backward : 1, /* Write ring buffer from end to beginning */	455	write_backward : 1, /* write ring buffer from end to beginning */
455	namespaces : 1, /* include namespaces data */	456	namespaces : 1, /* include namespaces data */
456	ksymbol : 1, /* include ksymbol events */	457	ksymbol : 1, /* include ksymbol events */
457	bpf_event : 1, /* include bpf events */	458	bpf_event : 1, /* include BPF events */
458	aux_output : 1, /* generate AUX records instead of events */	459	aux_output : 1, /* generate AUX records instead of events */
459	cgroup : 1, /* include cgroup events */	460	cgroup : 1, /* include cgroup events */
460	text_poke : 1, /* include text poke events */	461	text_poke : 1, /* include text poke events */
461	build_id : 1, /* use build id in mmap2 events */	462	build_id : 1, /* use build ID in mmap2 events */
462	inherit_thread : 1, /* children only inherit if cloned with CLONE_THREAD */	463	inherit_thread : 1, /* children only inherit if cloned with CLONE_THREAD */
463	remove_on_exec : 1, /* event is removed from task on exec */	464	remove_on_exec : 1, /* event is removed from task on exec */
464	sigtrap : 1, /* send synchronous SIGTRAP on event */	465	sigtrap : 1, /* send synchronous SIGTRAP on event */
465	__reserved_1 : 26;	466	__reserved_1 : 26;
466		467
467	union {	468	union {
468	__u32 wakeup_events; /* wakeup every n events */	469	__u32 wakeup_events; /* wake up every n events */
469	__u32 wakeup_watermark; /* bytes before wakeup */	470	__u32 wakeup_watermark; /* bytes before wakeup */
470	};	471	};
471		472
@@ -474,13 +475,13 @@ struct perf_event_attr { · 2 unmodified lines
474	__u64 bp_addr;	475	__u64 bp_addr;
475	__u64 kprobe_func; /* for perf_kprobe */	476	__u64 kprobe_func; /* for perf_kprobe */
476	__u64 uprobe_path; /* for perf_uprobe */	477	__u64 uprobe_path; /* for perf_uprobe */
477	__u64 config1; /* extension of config */	478	__u64 config1; /* extension of config */
478	};	479	};
479	union {	480	union {
480	__u64 bp_len;	481	__u64 bp_len;
481	__u64 kprobe_addr; /* when kprobe_func == NULL */	482	__u64 kprobe_addr; /* when kprobe_func == NULL */
482	__u64 probe_offset; /* for perf_[k,u]probe */	483	__u64 probe_offset; /* for perf_[k,u]probe */
483	__u64 config2; /* extension of config1 */	484	__u64 config2; /* extension of config1 */
484	};	485	};
485	__u64 branch_sample_type; /* enum perf_branch_sample_type */	486	__u64 branch_sample_type; /* enum perf_branch_sample_type */
486		487
@@ -510,7 +511,16 @@ struct perf_event_attr { · 23 unmodified lines
510	* Wakeup watermark for AUX area	511	* Wakeup watermark for AUX area
511	*/	512	*/
512	__u32 aux_watermark;	513	__u32 aux_watermark;
		514
		515	/*
		516	* Max number of frame pointers in a callchain, should be
		517	* lower than /proc/sys/kernel/perf_event_max_stack.
		518	*
		519	* Max number of entries of branch stack should be lower
		520	* than the hardware limit.
		521	*/
513	__u16 sample_max_stack;	522	__u16 sample_max_stack;
		523
514	__u16 __reserved_2;	524	__u16 __reserved_2;
515	__u32 aux_sample_size;	525	__u32 aux_sample_size;
516		526
@@ -537,7 +547,7 @@ struct perf_event_attr { · 20 unmodified lines
537		547
538	/*	548	/*
539	* Structure used by below PERF_EVENT_IOC_QUERY_BPF command	549	* Structure used by below PERF_EVENT_IOC_QUERY_BPF command
540	* to query bpf programs attached to the same perf tracepoint	550	* to query BPF programs attached to the same perf tracepoint
541	* as the given perf event.	551	* as the given perf event.
542	*/	552	*/
543	struct perf_event_query_bpf {	553	struct perf_event_query_bpf {
@@ -559,21 +569,21 @@ struct perf_event_query_bpf { · 15 unmodified lines
559	/*	569	/*
560	* Ioctls that can be done on a perf event fd:	570	* Ioctls that can be done on a perf event fd:
561	*/	571	*/
562	#define PERF_EVENT_IOC_ENABLE _IO ('$', 0)	572	#define PERF_EVENT_IOC_ENABLE _IO ('$', 0)
563	#define PERF_EVENT_IOC_DISABLE _IO ('$', 1)	573	#define PERF_EVENT_IOC_DISABLE _IO ('$', 1)
564	#define PERF_EVENT_IOC_REFRESH _IO ('$', 2)	574	#define PERF_EVENT_IOC_REFRESH _IO ('$', 2)
565	#define PERF_EVENT_IOC_RESET _IO ('$', 3)	575	#define PERF_EVENT_IOC_RESET _IO ('$', 3)
566	#define PERF_EVENT_IOC_PERIOD _IOW('$', 4, __u64)	576	#define PERF_EVENT_IOC_PERIOD _IOW ('$', 4, __u64)
567	#define PERF_EVENT_IOC_SET_OUTPUT _IO ('$', 5)	577	#define PERF_EVENT_IOC_SET_OUTPUT _IO ('$', 5)
568	#define PERF_EVENT_IOC_SET_FILTER _IOW('$', 6, char *)	578	#define PERF_EVENT_IOC_SET_FILTER _IOW ('$', 6, char *)
569	#define PERF_EVENT_IOC_ID _IOR('$', 7, __u64 *)	579	#define PERF_EVENT_IOC_ID _IOR ('$', 7, __u64 *)
570	#define PERF_EVENT_IOC_SET_BPF _IOW('$', 8, __u32)	580	#define PERF_EVENT_IOC_SET_BPF _IOW ('$', 8, __u32)
571	#define PERF_EVENT_IOC_PAUSE_OUTPUT _IOW('$', 9, __u32)	581	#define PERF_EVENT_IOC_PAUSE_OUTPUT _IOW ('$', 9, __u32)
572	#define PERF_EVENT_IOC_QUERY_BPF _IOWR('$', 10, struct perf_event_query_bpf *)	582	#define PERF_EVENT_IOC_QUERY_BPF _IOWR('$', 10, struct perf_event_query_bpf *)
573	#define PERF_EVENT_IOC_MODIFY_ATTRIBUTES _IOW('$', 11, struct perf_event_attr *)	583	#define PERF_EVENT_IOC_MODIFY_ATTRIBUTES _IOW ('$', 11, struct perf_event_attr *)
574		584
575	enum perf_event_ioc_flags {	585	enum perf_event_ioc_flags {
576	PERF_IOC_FLAG_GROUP = 1U << 0,	586	PERF_IOC_FLAG_GROUP = 1U << 0,
577	};	587	};
578		588
579	/*	589	/*
@@ -584,7 +594,7 @@ struct perf_event_mmap_page { · 4 unmodified lines
584	__u32 compat_version; /* lowest version this is compat with */	594	__u32 compat_version; /* lowest version this is compat with */
585		595
586	/*	596	/*
587	* Bits needed to read the hw events in user-space.	597	* Bits needed to read the HW events in user-space.
588	*	598	*
589	* u32 seq, time_mult, time_shift, index, width;	599	* u32 seq, time_mult, time_shift, index, width;
590	* u64 count, enabled, running;	600	* u64 count, enabled, running;
@@ -622,7 +632,7 @@ struct perf_event_mmap_page { · 31 unmodified lines
622	__u32 index; /* hardware event identifier */	632	__u32 index; /* hardware event identifier */
623	__s64 offset; /* add to hardware event value */	633	__s64 offset; /* add to hardware event value */
624	__u64 time_enabled; /* time event active */	634	__u64 time_enabled; /* time event active */
625	__u64 time_running; /* time event on cpu */	635	__u64 time_running; /* time event on CPU */
626	union {	636	union {
627	__u64 capabilities;	637	__u64 capabilities;
628	struct {	638	struct {
@@ -650,7 +660,7 @@ struct perf_event_mmap_page { · 21 unmodified lines
650		660
651	/*	661	/*
652	* If cap_usr_time the below fields can be used to compute the time	662	* If cap_usr_time the below fields can be used to compute the time
653	* delta since time_enabled (in ns) using rdtsc or similar.	663	* delta since time_enabled (in ns) using RDTSC or similar.
654	*	664	*
655	* u64 quot, rem;	665	* u64 quot, rem;
656	* u64 delta;	666	* u64 delta;
@@ -723,7 +733,7 @@ struct perf_event_mmap_page { · 66 unmodified lines
723	* after reading this value.	733	* after reading this value.
724	*	734	*
725	* When the mapping is PROT_WRITE the @data_tail value should be	735	* When the mapping is PROT_WRITE the @data_tail value should be
726	* written by userspace to reflect the last read data, after issueing	736	* written by user-space to reflect the last read data, after issuing
727	* an smp_mb() to separate the data read from the ->data_tail store.	737	* an smp_mb() to separate the data read from the ->data_tail store.
728	* In this case the kernel will not over-write unread data.	738	* In this case the kernel will not over-write unread data.
729	*	739	*
@@ -739,7 +749,7 @@ struct perf_event_mmap_page { · 9 unmodified lines
739		749
740	/*	750	/*
741	* AUX area is defined by aux_{offset,size} fields that should be set	751	* AUX area is defined by aux_{offset,size} fields that should be set
742	* by the userspace, so that	752	* by the user-space, so that
743	*	753	*
744	* aux_offset >= data_offset + data_size	754	* aux_offset >= data_offset + data_size
745	*	755	*
@@ -813,7 +823,7 @@ struct perf_event_mmap_page { · 67 unmodified lines
813	* Indicates that thread was preempted in TASK_RUNNING state.	823	* Indicates that thread was preempted in TASK_RUNNING state.
814	*	824	*
815	* PERF_RECORD_MISC_MMAP_BUILD_ID:	825	* PERF_RECORD_MISC_MMAP_BUILD_ID:
816	* Indicates that mmap2 event carries build id data.	826	* Indicates that mmap2 event carries build ID data.
817	*/	827	*/
818	#define PERF_RECORD_MISC_EXACT_IP (1 << 14)	828	#define PERF_RECORD_MISC_EXACT_IP (1 << 14)
819	#define PERF_RECORD_MISC_SWITCH_OUT_PREEMPT (1 << 14)	829	#define PERF_RECORD_MISC_SWITCH_OUT_PREEMPT (1 << 14)
@@ -824,26 +834,26 @@ struct perf_event_mmap_page { · 4 unmodified lines
824	#define PERF_RECORD_MISC_EXT_RESERVED (1 << 15)	834	#define PERF_RECORD_MISC_EXT_RESERVED (1 << 15)
825		835
826	struct perf_event_header {	836	struct perf_event_header {
827	__u32 type;	837	__u32 type;
828	__u16 misc;	838	__u16 misc;
829	__u16 size;	839	__u16 size;
830	};	840	};
831		841
832	struct perf_ns_link_info {	842	struct perf_ns_link_info {
833	__u64 dev;	843	__u64 dev;
834	__u64 ino;	844	__u64 ino;
835	};	845	};
836		846
837	enum {	847	enum {
838	NET_NS_INDEX = 0,	848	NET_NS_INDEX = 0,
839	UTS_NS_INDEX = 1,	849	UTS_NS_INDEX = 1,
840	IPC_NS_INDEX = 2,	850	IPC_NS_INDEX = 2,
841	PID_NS_INDEX = 3,	851	PID_NS_INDEX = 3,
842	USER_NS_INDEX = 4,	852	USER_NS_INDEX = 4,
843	MNT_NS_INDEX = 5,	853	MNT_NS_INDEX = 5,
844	CGROUP_NS_INDEX = 6,	854	CGROUP_NS_INDEX = 6,
845		855
846	NR_NAMESPACES, /* number of available namespaces */	856	NR_NAMESPACES, /* number of available namespaces */
847	};	857	};
848		858
849	enum perf_event_type {	859	enum perf_event_type {
@@ -859,11 +869,11 @@ enum perf_event_type { · 9 unmodified lines
859	* optional fields being ignored.	869	* optional fields being ignored.
860	*	870	*
861	* struct sample_id {	871	* struct sample_id {
862	* { u32 pid, tid; } && PERF_SAMPLE_TID	872	* { u32 pid, tid; } && PERF_SAMPLE_TID
863	* { u64 time; } && PERF_SAMPLE_TIME	873	* { u64 time; } && PERF_SAMPLE_TIME
864	* { u64 id; } && PERF_SAMPLE_ID	874	* { u64 id; } && PERF_SAMPLE_ID
865	* { u64 stream_id;} && PERF_SAMPLE_STREAM_ID	875	* { u64 stream_id;} && PERF_SAMPLE_STREAM_ID
866	* { u32 cpu, res; } && PERF_SAMPLE_CPU	876	* { u32 cpu, res; } && PERF_SAMPLE_CPU
867	* { u64 id; } && PERF_SAMPLE_IDENTIFIER	877	* { u64 id; } && PERF_SAMPLE_IDENTIFIER
868	* } && perf_event_attr::sample_id_all	878	* } && perf_event_attr::sample_id_all
869	*	879	*
@@ -874,7 +884,7 @@ enum perf_event_type { · 4 unmodified lines
874		884
875	/*	885	/*
876	* The MMAP events record the PROT_EXEC mappings so that we can	886	* The MMAP events record the PROT_EXEC mappings so that we can
877	* correlate userspace IPs to code. They have the following structure:	887	* correlate user-space IPs to code. They have the following structure:
878	*	888	*
879	* struct {	889	* struct {
880	* struct perf_event_header header;	890	* struct perf_event_header header;
@@ -884,7 +894,7 @@ enum perf_event_type { · 3 unmodified lines
884	* u64 len;	894	* u64 len;
885	* u64 pgoff;	895	* u64 pgoff;
886	* char filename[];	896	* char filename[];
887	* struct sample_id sample_id;	897	* struct sample_id sample_id;
888	* };	898	* };
889	*/	899	*/
890	PERF_RECORD_MMAP = 1,	900	PERF_RECORD_MMAP = 1,
@@ -894,7 +904,7 @@ enum perf_event_type { · 3 unmodified lines
894	* struct perf_event_header header;	904	* struct perf_event_header header;
895	* u64 id;	905	* u64 id;
896	* u64 lost;	906	* u64 lost;
897	* struct sample_id sample_id;	907	* struct sample_id sample_id;
898	* };	908	* };
899	*/	909	*/
900	PERF_RECORD_LOST = 2,	910	PERF_RECORD_LOST = 2,
@@ -905,7 +915,7 @@ enum perf_event_type { · 4 unmodified lines
905	*	915	*
906	* u32 pid, tid;	916	* u32 pid, tid;
907	* char comm[];	917	* char comm[];
908	* struct sample_id sample_id;	918	* struct sample_id sample_id;
909	* };	919	* };
910	*/	920	*/
911	PERF_RECORD_COMM = 3,	921	PERF_RECORD_COMM = 3,
@@ -916,7 +926,7 @@ enum perf_event_type { · 4 unmodified lines
916	* u32 pid, ppid;	926	* u32 pid, ppid;
917	* u32 tid, ptid;	927	* u32 tid, ptid;
918	* u64 time;	928	* u64 time;
919	* struct sample_id sample_id;	929	* struct sample_id sample_id;
920	* };	930	* };
921	*/	931	*/
922	PERF_RECORD_EXIT = 4,	932	PERF_RECORD_EXIT = 4,
@@ -927,7 +937,7 @@ enum perf_event_type { · 4 unmodified lines
927	* u64 time;	937	* u64 time;
928	* u64 id;	938	* u64 id;
929	* u64 stream_id;	939	* u64 stream_id;
930	* struct sample_id sample_id;	940	* struct sample_id sample_id;
931	* };	941	* };
932	*/	942	*/
933	PERF_RECORD_THROTTLE = 5,	943	PERF_RECORD_THROTTLE = 5,
@@ -939,7 +949,7 @@ enum perf_event_type { · 5 unmodified lines
939	* u32 pid, ppid;	949	* u32 pid, ppid;
940	* u32 tid, ptid;	950	* u32 tid, ptid;
941	* u64 time;	951	* u64 time;
942	* struct sample_id sample_id;	952	* struct sample_id sample_id;
943	* };	953	* };
944	*/	954	*/
945	PERF_RECORD_FORK = 7,	955	PERF_RECORD_FORK = 7,
@@ -950,7 +960,7 @@ enum perf_event_type { · 4 unmodified lines
950	* u32 pid, tid;	960	* u32 pid, tid;
951	*	961	*
952	* struct read_format values;	962	* struct read_format values;
953	* struct sample_id sample_id;	963	* struct sample_id sample_id;
954	* };	964	* };
955	*/	965	*/
956	PERF_RECORD_READ = 8,	966	PERF_RECORD_READ = 8,
@@ -1005,12 +1015,12 @@ enum perf_event_type { · 48 unmodified lines
1005	* { u64 counters; } cntr[nr] && PERF_SAMPLE_BRANCH_COUNTERS	1015	* { u64 counters; } cntr[nr] && PERF_SAMPLE_BRANCH_COUNTERS
1006	* } && PERF_SAMPLE_BRANCH_STACK	1016	* } && PERF_SAMPLE_BRANCH_STACK
1007	*	1017	*
1008	* { u64 abi; # enum perf_sample_regs_abi	1018	* { u64 abi; # enum perf_sample_regs_abi
1009	* u64 regs[weight(mask)]; } && PERF_SAMPLE_REGS_USER	1019	* u64 regs[weight(mask)]; } && PERF_SAMPLE_REGS_USER
1010	*	1020	*
1011	* { u64 size;	1021	* { u64 size;
1012	* char data[size];	1022	* char data[size];
1013	* u64 dyn_size; } && PERF_SAMPLE_STACK_USER	1023	* u64 dyn_size; } && PERF_SAMPLE_STACK_USER
1014	*	1024	*
1015	* { union perf_sample_weight	1025	* { union perf_sample_weight
1016	* {	1026	* {
@@ -1035,10 +1045,11 @@ enum perf_event_type { · 18 unmodified lines
1035	* { u64 abi; # enum perf_sample_regs_abi	1045	* { u64 abi; # enum perf_sample_regs_abi
1036	* u64 regs[weight(mask)]; } && PERF_SAMPLE_REGS_INTR	1046	* u64 regs[weight(mask)]; } && PERF_SAMPLE_REGS_INTR
1037	* { u64 phys_addr;} && PERF_SAMPLE_PHYS_ADDR	1047	* { u64 phys_addr;} && PERF_SAMPLE_PHYS_ADDR
1038	* { u64 size;	1048	* { u64 cgroup;} && PERF_SAMPLE_CGROUP
1039	* char data[size]; } && PERF_SAMPLE_AUX
1040	* { u64 data_page_size;} && PERF_SAMPLE_DATA_PAGE_SIZE	1049	* { u64 data_page_size;} && PERF_SAMPLE_DATA_PAGE_SIZE
1041	* { u64 code_page_size;} && PERF_SAMPLE_CODE_PAGE_SIZE	1050	* { u64 code_page_size;} && PERF_SAMPLE_CODE_PAGE_SIZE
		1051	* { u64 size;
		1052	* char data[size]; } && PERF_SAMPLE_AUX
1042	* };	1053	* };
1043	*/	1054	*/
1044	PERF_RECORD_SAMPLE = 9,	1055	PERF_RECORD_SAMPLE = 9,
@@ -1070,7 +1081,7 @@ enum perf_event_type { · 25 unmodified lines
1070	* };	1081	* };
1071	* u32 prot, flags;	1082	* u32 prot, flags;
1072	* char filename[];	1083	* char filename[];
1073	* struct sample_id sample_id;	1084	* struct sample_id sample_id;
1074	* };	1085	* };
1075	*/	1086	*/
1076	PERF_RECORD_MMAP2 = 10,	1087	PERF_RECORD_MMAP2 = 10,
@@ -1079,12 +1090,12 @@ enum perf_event_type { · 2 unmodified lines
1079	* Records that new data landed in the AUX buffer part.	1090	* Records that new data landed in the AUX buffer part.
1080	*	1091	*
1081	* struct {	1092	* struct {
1082	* struct perf_event_header header;	1093	* struct perf_event_header header;
1083	*	1094	*
1084	* u64 aux_offset;	1095	* u64 aux_offset;
1085	* u64 aux_size;	1096	* u64 aux_size;
1086	* u64 flags;	1097	* u64 flags;
1087	* struct sample_id sample_id;	1098	* struct sample_id sample_id;
1088	* };	1099	* };
1089	*/	1100	*/
1090	PERF_RECORD_AUX = 11,	1101	PERF_RECORD_AUX = 11,
@@ -1167,7 +1178,7 @@ enum perf_event_type { · 76 unmodified lines
1167	PERF_RECORD_KSYMBOL = 17,	1178	PERF_RECORD_KSYMBOL = 17,
1168		1179
1169	/*	1180	/*
1170	* Record bpf events:	1181	* Record BPF events:
1171	* enum perf_bpf_event_type {	1182	* enum perf_bpf_event_type {
1172	* PERF_BPF_EVENT_UNKNOWN = 0,	1183	* PERF_BPF_EVENT_UNKNOWN = 0,
1173	* PERF_BPF_EVENT_PROG_LOAD = 1,	1184	* PERF_BPF_EVENT_PROG_LOAD = 1,
@@ -1245,181 +1256,181 @@ enum perf_record_ksymbol_type { · 71 unmodified lines
1245	#define PERF_RECORD_KSYMBOL_FLAGS_UNREGISTER (1 << 0)	1256	#define PERF_RECORD_KSYMBOL_FLAGS_UNREGISTER (1 << 0)
1246		1257
1247	enum perf_bpf_event_type {	1258	enum perf_bpf_event_type {
1248	PERF_BPF_EVENT_UNKNOWN = 0,	1259	PERF_BPF_EVENT_UNKNOWN = 0,
1249	PERF_BPF_EVENT_PROG_LOAD = 1,	1260	PERF_BPF_EVENT_PROG_LOAD = 1,
1250	PERF_BPF_EVENT_PROG_UNLOAD = 2,	1261	PERF_BPF_EVENT_PROG_UNLOAD = 2,
1251	PERF_BPF_EVENT_MAX, /* non-ABI */	1262	PERF_BPF_EVENT_MAX, /* non-ABI */
1252	};	1263	};
1253		1264
1254	#define PERF_MAX_STACK_DEPTH 127	1265	#define PERF_MAX_STACK_DEPTH 127
1255	#define PERF_MAX_CONTEXTS_PER_STACK 8	1266	#define PERF_MAX_CONTEXTS_PER_STACK 8
1256		1267
1257	enum perf_callchain_context {	1268	enum perf_callchain_context {
1258	PERF_CONTEXT_HV = (__u64)-32,	1269	PERF_CONTEXT_HV = (__u64)-32,
1259	PERF_CONTEXT_KERNEL = (__u64)-128,	1270	PERF_CONTEXT_KERNEL = (__u64)-128,
1260	PERF_CONTEXT_USER = (__u64)-512,	1271	PERF_CONTEXT_USER = (__u64)-512,
1261		1272
1262	PERF_CONTEXT_GUEST = (__u64)-2048,	1273	PERF_CONTEXT_GUEST = (__u64)-2048,
1263	PERF_CONTEXT_GUEST_KERNEL = (__u64)-2176,	1274	PERF_CONTEXT_GUEST_KERNEL = (__u64)-2176,
1264	PERF_CONTEXT_GUEST_USER = (__u64)-2560,	1275	PERF_CONTEXT_GUEST_USER = (__u64)-2560,
1265		1276
1266	PERF_CONTEXT_MAX = (__u64)-4095,	1277	PERF_CONTEXT_MAX = (__u64)-4095,
1267	};	1278	};
1268		1279
1269	/**	1280	/**
1270	* PERF_RECORD_AUX::flags bits	1281	* PERF_RECORD_AUX::flags bits
1271	*/	1282	*/
1272	#define PERF_AUX_FLAG_TRUNCATED 0x01 /* record was truncated to fit */	1283	#define PERF_AUX_FLAG_TRUNCATED 0x0001 /* Record was truncated to fit */
1273	#define PERF_AUX_FLAG_OVERWRITE 0x02 /* snapshot from overwrite mode */	1284	#define PERF_AUX_FLAG_OVERWRITE 0x0002 /* Snapshot from overwrite mode */
1274	#define PERF_AUX_FLAG_PARTIAL 0x04 /* record contains gaps */	1285	#define PERF_AUX_FLAG_PARTIAL 0x0004 /* Record contains gaps */
1275	#define PERF_AUX_FLAG_COLLISION 0x08 /* sample collided with another */	1286	#define PERF_AUX_FLAG_COLLISION 0x0008 /* Sample collided with another */
1276	#define PERF_AUX_FLAG_PMU_FORMAT_TYPE_MASK 0xff00 /* PMU specific trace format type */	1287	#define PERF_AUX_FLAG_PMU_FORMAT_TYPE_MASK 0xff00 /* PMU specific trace format type */
1277		1288
1278	/* CoreSight PMU AUX buffer formats */	1289	/* CoreSight PMU AUX buffer formats */
1279	#define PERF_AUX_FLAG_CORESIGHT_FORMAT_CORESIGHT 0x0000 /* Default for backward compatibility */	1290	#define PERF_AUX_FLAG_CORESIGHT_FORMAT_CORESIGHT 0x0000 /* Default for backward compatibility */
1280	#define PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW 0x0100 /* Raw format of the source */	1291	#define PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW 0x0100 /* Raw format of the source */
1281		1292
1282	#define PERF_FLAG_FD_NO_GROUP (1UL << 0)	1293	#define PERF_FLAG_FD_NO_GROUP (1UL << 0)
1283	#define PERF_FLAG_FD_OUTPUT (1UL << 1)	1294	#define PERF_FLAG_FD_OUTPUT (1UL << 1)
1284	#define PERF_FLAG_PID_CGROUP (1UL << 2) /* pid=cgroup id, per-cpu mode only */	1295	#define PERF_FLAG_PID_CGROUP (1UL << 2) /* pid=cgroup ID, per-CPU mode only */
1285	#define PERF_FLAG_FD_CLOEXEC (1UL << 3) /* O_CLOEXEC */	1296	#define PERF_FLAG_FD_CLOEXEC (1UL << 3) /* O_CLOEXEC */
1286		1297
1287	#if defined(__LITTLE_ENDIAN_BITFIELD)	1298	#if defined(__LITTLE_ENDIAN_BITFIELD)
1288	union perf_mem_data_src {	1299	union perf_mem_data_src {
1289	__u64 val;	1300	__u64 val;
1290	struct {	1301	struct {
1291	__u64 mem_op:5, /* type of opcode */	1302	__u64 mem_op : 5, /* Type of opcode */
1292	mem_lvl:14, /* memory hierarchy level */	1303	mem_lvl : 14, /* Memory hierarchy level */
1293	mem_snoop:5, /* snoop mode */	1304	mem_snoop : 5, /* Snoop mode */
1294	mem_lock:2, /* lock instr */	1305	mem_lock : 2, /* Lock instr */
1295	mem_dtlb:7, /* tlb access */	1306	mem_dtlb : 7, /* TLB access */
1296	mem_lvl_num:4, /* memory hierarchy level number */	1307	mem_lvl_num : 4, /* Memory hierarchy level number */
1297	mem_remote:1, /* remote */	1308	mem_remote : 1, /* Remote */
1298	mem_snoopx:2, /* snoop mode, ext */	1309	mem_snoopx : 2, /* Snoop mode, ext */
1299	mem_blk:3, /* access blocked */	1310	mem_blk : 3, /* Access blocked */
1300	mem_hops:3, /* hop level */	1311	mem_hops : 3, /* Hop level */
1301	mem_rsvd:18;	1312	mem_rsvd : 18;
1302	};	1313	};
1303	};	1314	};
1304	#elif defined(__BIG_ENDIAN_BITFIELD)	1315	#elif defined(__BIG_ENDIAN_BITFIELD)
1305	union perf_mem_data_src {	1316	union perf_mem_data_src {
1306	__u64 val;	1317	__u64 val;
1307	struct {	1318	struct {
1308	__u64 mem_rsvd:18,	1319	__u64 mem_rsvd : 18,
1309	mem_hops:3, /* hop level */	1320	mem_hops : 3, /* Hop level */
1310	mem_blk:3, /* access blocked */	1321	mem_blk : 3, /* Access blocked */
1311	mem_snoopx:2, /* snoop mode, ext */	1322	mem_snoopx : 2, /* Snoop mode, ext */
1312	mem_remote:1, /* remote */	1323	mem_remote : 1, /* Remote */
1313	mem_lvl_num:4, /* memory hierarchy level number */	1324	mem_lvl_num : 4, /* Memory hierarchy level number */
1314	mem_dtlb:7, /* tlb access */	1325	mem_dtlb : 7, /* TLB access */
1315	mem_lock:2, /* lock instr */	1326	mem_lock : 2, /* Lock instr */
1316	mem_snoop:5, /* snoop mode */	1327	mem_snoop : 5, /* Snoop mode */
1317	mem_lvl:14, /* memory hierarchy level */	1328	mem_lvl : 14, /* Memory hierarchy level */
1318	mem_op:5; /* type of opcode */	1329	mem_op : 5; /* Type of opcode */
1319	};	1330	};
1320	};	1331	};
1321	#else	1332	#else
1322	#error "Unknown endianness"	1333	# error "Unknown endianness"
1323	#endif	1334	#endif
1324		1335
1325	/* type of opcode (load/store/prefetch,code) */	1336	/* Type of memory opcode: */
1326	#define PERF_MEM_OP_NA 0x01 /* not available */	1337	#define PERF_MEM_OP_NA 0x0001 /* Not available */
1327	#define PERF_MEM_OP_LOAD 0x02 /* load instruction */	1338	#define PERF_MEM_OP_LOAD 0x0002 /* Load instruction */
1328	#define PERF_MEM_OP_STORE 0x04 /* store instruction */	1339	#define PERF_MEM_OP_STORE 0x0004 /* Store instruction */
1329	#define PERF_MEM_OP_PFETCH 0x08 /* prefetch */	1340	#define PERF_MEM_OP_PFETCH 0x0008 /* Prefetch */
1330	#define PERF_MEM_OP_EXEC 0x10 /* code (execution) */	1341	#define PERF_MEM_OP_EXEC 0x0010 /* Code (execution) */
1331	#define PERF_MEM_OP_SHIFT 0	1342	#define PERF_MEM_OP_SHIFT 0
1332		1343
1333	/*	1344	/*
1334	* PERF_MEM_LVL_* namespace being depricated to some extent in the	1345	* The PERF_MEM_LVL_* namespace is being deprecated to some extent in
1335	* favour of newer composite PERF_MEM_{LVLNUM_,REMOTE_,SNOOPX_} fields.	1346	* favour of newer composite PERF_MEM_{LVLNUM_,REMOTE_,SNOOPX_} fields.
1336	* Supporting this namespace inorder to not break defined ABIs.	1347	* We support this namespace in order to not break defined ABIs.
1337	*	1348	*
1338	* memory hierarchy (memory level, hit or miss)	1349	* Memory hierarchy (memory level, hit or miss)
1339	*/	1350	*/
1340	#define PERF_MEM_LVL_NA 0x01 /* not available */	1351	#define PERF_MEM_LVL_NA 0x0001 /* Not available */
1341	#define PERF_MEM_LVL_HIT 0x02 /* hit level */	1352	#define PERF_MEM_LVL_HIT 0x0002 /* Hit level */
1342	#define PERF_MEM_LVL_MISS 0x04 /* miss level */	1353	#define PERF_MEM_LVL_MISS 0x0004 /* Miss level */
1343	#define PERF_MEM_LVL_L1 0x08 /* L1 */	1354	#define PERF_MEM_LVL_L1 0x0008 /* L1 */
1344	#define PERF_MEM_LVL_LFB 0x10 /* Line Fill Buffer */	1355	#define PERF_MEM_LVL_LFB 0x0010 /* Line Fill Buffer */
1345	#define PERF_MEM_LVL_L2 0x20 /* L2 */	1356	#define PERF_MEM_LVL_L2 0x0020 /* L2 */
1346	#define PERF_MEM_LVL_L3 0x40 /* L3 */	1357	#define PERF_MEM_LVL_L3 0x0040 /* L3 */
1347	#define PERF_MEM_LVL_LOC_RAM 0x80 /* Local DRAM */	1358	#define PERF_MEM_LVL_LOC_RAM 0x0080 /* Local DRAM */
1348	#define PERF_MEM_LVL_REM_RAM1 0x100 /* Remote DRAM (1 hop) */	1359	#define PERF_MEM_LVL_REM_RAM1 0x0100 /* Remote DRAM (1 hop) */
1349	#define PERF_MEM_LVL_REM_RAM2 0x200 /* Remote DRAM (2 hops) */	1360	#define PERF_MEM_LVL_REM_RAM2 0x0200 /* Remote DRAM (2 hops) */
1350	#define PERF_MEM_LVL_REM_CCE1 0x400 /* Remote Cache (1 hop) */	1361	#define PERF_MEM_LVL_REM_CCE1 0x0400 /* Remote Cache (1 hop) */
1351	#define PERF_MEM_LVL_REM_CCE2 0x800 /* Remote Cache (2 hops) */	1362	#define PERF_MEM_LVL_REM_CCE2 0x0800 /* Remote Cache (2 hops) */
1352	#define PERF_MEM_LVL_IO 0x1000 /* I/O memory */	1363	#define PERF_MEM_LVL_IO 0x1000 /* I/O memory */
1353	#define PERF_MEM_LVL_UNC 0x2000 /* Uncached memory */	1364	#define PERF_MEM_LVL_UNC 0x2000 /* Uncached memory */
1354	#define PERF_MEM_LVL_SHIFT 5	1365	#define PERF_MEM_LVL_SHIFT 5
1355		1366
1356	#define PERF_MEM_REMOTE_REMOTE 0x01 /* Remote */	1367	#define PERF_MEM_REMOTE_REMOTE 0x0001 /* Remote */
1357	#define PERF_MEM_REMOTE_SHIFT 37	1368	#define PERF_MEM_REMOTE_SHIFT 37
1358		1369
1359	#define PERF_MEM_LVLNUM_L1 0x01 /* L1 */	1370	#define PERF_MEM_LVLNUM_L1 0x0001 /* L1 */
1360	#define PERF_MEM_LVLNUM_L2 0x02 /* L2 */	1371	#define PERF_MEM_LVLNUM_L2 0x0002 /* L2 */
1361	#define PERF_MEM_LVLNUM_L3 0x03 /* L3 */	1372	#define PERF_MEM_LVLNUM_L3 0x0003 /* L3 */
1362	#define PERF_MEM_LVLNUM_L4 0x04 /* L4 */	1373	#define PERF_MEM_LVLNUM_L4 0x0004 /* L4 */
1363	#define PERF_MEM_LVLNUM_L2_MHB 0x05 /* L2 Miss Handling Buffer */	1374	#define PERF_MEM_LVLNUM_L2_MHB 0x0005 /* L2 Miss Handling Buffer */
1364	#define PERF_MEM_LVLNUM_MSC 0x06 /* Memory-side Cache */	1375	#define PERF_MEM_LVLNUM_MSC 0x0006 /* Memory-side Cache */
1365	/* 0x7 available */	1376	/* 0x007 available */
1366	#define PERF_MEM_LVLNUM_UNC 0x08 /* Uncached */	1377	#define PERF_MEM_LVLNUM_UNC 0x0008 /* Uncached */
1367	#define PERF_MEM_LVLNUM_CXL 0x09 /* CXL */	1378	#define PERF_MEM_LVLNUM_CXL 0x0009 /* CXL */
1368	#define PERF_MEM_LVLNUM_IO 0x0a /* I/O */	1379	#define PERF_MEM_LVLNUM_IO 0x000a /* I/O */
1369	#define PERF_MEM_LVLNUM_ANY_CACHE 0x0b /* Any cache */	1380	#define PERF_MEM_LVLNUM_ANY_CACHE 0x000b /* Any cache */
1370	#define PERF_MEM_LVLNUM_LFB 0x0c /* LFB / L1 Miss Handling Buffer */	1381	#define PERF_MEM_LVLNUM_LFB 0x000c /* LFB / L1 Miss Handling Buffer */
1371	#define PERF_MEM_LVLNUM_RAM 0x0d /* RAM */	1382	#define PERF_MEM_LVLNUM_RAM 0x000d /* RAM */
1372	#define PERF_MEM_LVLNUM_PMEM 0x0e /* PMEM */	1383	#define PERF_MEM_LVLNUM_PMEM 0x000e /* PMEM */
1373	#define PERF_MEM_LVLNUM_NA 0x0f /* N/A */	1384	#define PERF_MEM_LVLNUM_NA 0x000f /* N/A */
1374		1385
1375	#define PERF_MEM_LVLNUM_SHIFT 33	1386	#define PERF_MEM_LVLNUM_SHIFT 33
1376		1387
1377	/* snoop mode */	1388	/* Snoop mode */
1378	#define PERF_MEM_SNOOP_NA 0x01 /* not available */	1389	#define PERF_MEM_SNOOP_NA 0x0001 /* Not available */
1379	#define PERF_MEM_SNOOP_NONE 0x02 /* no snoop */	1390	#define PERF_MEM_SNOOP_NONE 0x0002 /* No snoop */
1380	#define PERF_MEM_SNOOP_HIT 0x04 /* snoop hit */	1391	#define PERF_MEM_SNOOP_HIT 0x0004 /* Snoop hit */
1381	#define PERF_MEM_SNOOP_MISS 0x08 /* snoop miss */	1392	#define PERF_MEM_SNOOP_MISS 0x0008 /* Snoop miss */
1382	#define PERF_MEM_SNOOP_HITM 0x10 /* snoop hit modified */	1393	#define PERF_MEM_SNOOP_HITM 0x0010 /* Snoop hit modified */
1383	#define PERF_MEM_SNOOP_SHIFT 19	1394	#define PERF_MEM_SNOOP_SHIFT 19
1384		1395
1385	#define PERF_MEM_SNOOPX_FWD 0x01 /* forward */	1396	#define PERF_MEM_SNOOPX_FWD 0x0001 /* Forward */
1386	#define PERF_MEM_SNOOPX_PEER 0x02 /* xfer from peer */	1397	#define PERF_MEM_SNOOPX_PEER 0x0002 /* Transfer from peer */
1387	#define PERF_MEM_SNOOPX_SHIFT 38	1398	#define PERF_MEM_SNOOPX_SHIFT 38
1388		1399
1389	/* locked instruction */	1400	/* Locked instruction */
1390	#define PERF_MEM_LOCK_NA 0x01 /* not available */	1401	#define PERF_MEM_LOCK_NA 0x0001 /* Not available */
1391	#define PERF_MEM_LOCK_LOCKED 0x02 /* locked transaction */	1402	#define PERF_MEM_LOCK_LOCKED 0x0002 /* Locked transaction */
1392	#define PERF_MEM_LOCK_SHIFT 24	1403	#define PERF_MEM_LOCK_SHIFT 24
1393		1404
1394	/* TLB access */	1405	/* TLB access */
1395	#define PERF_MEM_TLB_NA 0x01 /* not available */	1406	#define PERF_MEM_TLB_NA 0x0001 /* Not available */
1396	#define PERF_MEM_TLB_HIT 0x02 /* hit level */	1407	#define PERF_MEM_TLB_HIT 0x0002 /* Hit level */
1397	#define PERF_MEM_TLB_MISS 0x04 /* miss level */	1408	#define PERF_MEM_TLB_MISS 0x0004 /* Miss level */
1398	#define PERF_MEM_TLB_L1 0x08 /* L1 */	1409	#define PERF_MEM_TLB_L1 0x0008 /* L1 */
1399	#define PERF_MEM_TLB_L2 0x10 /* L2 */	1410	#define PERF_MEM_TLB_L2 0x0010 /* L2 */
1400	#define PERF_MEM_TLB_WK 0x20 /* Hardware Walker*/	1411	#define PERF_MEM_TLB_WK 0x0020 /* Hardware Walker*/
1401	#define PERF_MEM_TLB_OS 0x40 /* OS fault handler */	1412	#define PERF_MEM_TLB_OS 0x0040 /* OS fault handler */
1402	#define PERF_MEM_TLB_SHIFT 26	1413	#define PERF_MEM_TLB_SHIFT 26
1403		1414
1404	/* Access blocked */	1415	/* Access blocked */
1405	#define PERF_MEM_BLK_NA 0x01 /* not available */	1416	#define PERF_MEM_BLK_NA 0x0001 /* Not available */
1406	#define PERF_MEM_BLK_DATA 0x02 /* data could not be forwarded */	1417	#define PERF_MEM_BLK_DATA 0x0002 /* Data could not be forwarded */
1407	#define PERF_MEM_BLK_ADDR 0x04 /* address conflict */	1418	#define PERF_MEM_BLK_ADDR 0x0004 /* Address conflict */
1408	#define PERF_MEM_BLK_SHIFT 40	1419	#define PERF_MEM_BLK_SHIFT 40
1409		1420
1410	/* hop level */	1421	/* Hop level */
1411	#define PERF_MEM_HOPS_0 0x01 /* remote core, same node */	1422	#define PERF_MEM_HOPS_0 0x0001 /* Remote core, same node */
1412	#define PERF_MEM_HOPS_1 0x02 /* remote node, same socket */	1423	#define PERF_MEM_HOPS_1 0x0002 /* Remote node, same socket */
1413	#define PERF_MEM_HOPS_2 0x03 /* remote socket, same board */	1424	#define PERF_MEM_HOPS_2 0x0003 /* Remote socket, same board */
1414	#define PERF_MEM_HOPS_3 0x04 /* remote board */	1425	#define PERF_MEM_HOPS_3 0x0004 /* Remote board */
1415	/* 5-7 available */	1426	/* 5-7 available */
1416	#define PERF_MEM_HOPS_SHIFT 43	1427	#define PERF_MEM_HOPS_SHIFT 43
1417		1428
1418	#define PERF_MEM_S(a, s) \	1429	#define PERF_MEM_S(a, s) \
1419	(((__u64)PERF_MEM_##a##_##s) << PERF_MEM_##a##_SHIFT)	1430	(((__u64)PERF_MEM_##a##_##s) << PERF_MEM_##a##_SHIFT)
1420		1431
1421	/*	1432	/*
1422	* single taken branch record layout:	1433	* Layout of single taken branch records:
1423	*	1434	*
1424	* from: source instruction (may not always be a branch insn)	1435	* from: source instruction (may not always be a branch insn)
1425	* to: branch target	1436	* to: branch target
@@ -1438,37 +1449,37 @@ union perf_mem_data_src { · 12 unmodified lines
1438	struct perf_branch_entry {	1449	struct perf_branch_entry {
1439	__u64 from;	1450	__u64 from;
1440	__u64 to;	1451	__u64 to;
1441	__u64 mispred:1, /* target mispredicted */	1452	__u64 mispred : 1, /* target mispredicted */
1442	predicted:1,/* target predicted */	1453	predicted : 1, /* target predicted */
1443	in_tx:1, /* in transaction */	1454	in_tx : 1, /* in transaction */
1444	abort:1, /* transaction abort */	1455	abort : 1, /* transaction abort */
1445	cycles:16, /* cycle count to last branch */	1456	cycles : 16, /* cycle count to last branch */
1446	type:4, /* branch type */	1457	type : 4, /* branch type */
1447	spec:2, /* branch speculation info */	1458	spec : 2, /* branch speculation info */
1448	new_type:4, /* additional branch type */	1459	new_type : 4, /* additional branch type */
1449	priv:3, /* privilege level */	1460	priv : 3, /* privilege level */
1450	reserved:31;	1461	reserved : 31;
1451	};	1462	};
1452		1463
1453	/* Size of used info bits in struct perf_branch_entry */	1464	/* Size of used info bits in struct perf_branch_entry */
1454	#define PERF_BRANCH_ENTRY_INFO_BITS_MAX 33	1465	#define PERF_BRANCH_ENTRY_INFO_BITS_MAX 33
1455		1466
1456	union perf_sample_weight {	1467	union perf_sample_weight {
1457	__u64 full;	1468	__u64 full;
1458	#if defined(__LITTLE_ENDIAN_BITFIELD)	1469	#if defined(__LITTLE_ENDIAN_BITFIELD)
1459	struct {	1470	struct {
1460	__u32 var1_dw;	1471	__u32 var1_dw;
1461	__u16 var2_w;	1472	__u16 var2_w;
1462	__u16 var3_w;	1473	__u16 var3_w;
1463	};	1474	};
1464	#elif defined(__BIG_ENDIAN_BITFIELD)	1475	#elif defined(__BIG_ENDIAN_BITFIELD)
1465	struct {	1476	struct {
1466	__u16 var3_w;	1477	__u16 var3_w;
1467	__u16 var2_w;	1478	__u16 var2_w;
1468	__u32 var1_dw;	1479	__u32 var1_dw;
1469	};	1480	};
1470	#else	1481	#else
1471	#error "Unknown endianness"	1482	# error "Unknown endianness"
1472	#endif	1483	#endif
1473	};	1484	};
1474		1485

kernel/cpu.c+0−5

 		.teardown.single	= NULL,
 		.cant_stop		= true,
 	},
-	[CPUHP_PERF_PREPARE] = {
-		.name			= "perf:prepare",
-		.startup.single		= perf_event_init_cpu,
-		.teardown.single	= perf_event_exit_cpu,
-	},
 	[CPUHP_RANDOM_PREPARE] = {
 		.name			= "random:prepare",
 		.startup.single		= random_prepare_cpu,

kernel/events/core.c+424−185

@@ -1270,6 +1270,10 @@ static void put_ctx(struct perf_event_context *ctx) · 1269 unmodified lines
1270	if (ctx->task && ctx->task != TASK_TOMBSTONE)	1270	if (ctx->task && ctx->task != TASK_TOMBSTONE)
1271	put_task_struct(ctx->task);	1271	put_task_struct(ctx->task);
1272	call_rcu(&ctx->rcu_head, free_ctx);	1272	call_rcu(&ctx->rcu_head, free_ctx);
		1273	} else {
		1274	smp_mb__after_atomic(); /* pairs with wait_var_event() */
		1275	if (ctx->task == TASK_TOMBSTONE)
		1276	wake_up_var(&ctx->refcount);
1273	}	1277	}
1274	}	1278	}
1275		1279
@@ -2167,7 +2171,7 @@ static void perf_put_aux_event(struct perf_event *event) · 891 unmodified lines
2167	* If the event is an aux_event, tear down all links to	2171	* If the event is an aux_event, tear down all links to
2168	* it from other events.	2172	* it from other events.
2169	*/	2173	*/
2170	for_each_sibling_event(iter, event->group_leader) {	2174	for_each_sibling_event(iter, event) {
2171	if (iter->aux_event != event)	2175	if (iter->aux_event != event)
2172	continue;	2176	continue;
2173		2177
@@ -2325,7 +2329,11 @@ static void perf_child_detach(struct perf_event *event) · 151 unmodified lines
2325	if (WARN_ON_ONCE(!parent_event))	2329	if (WARN_ON_ONCE(!parent_event))
2326	return;	2330	return;
2327		2331
		2332	/*
		2333	* Can't check this from an IPI, the holder is likey another CPU.
		2334	*
2328	lockdep_assert_held(&parent_event->child_mutex);	2335	lockdep_assert_held(&parent_event->child_mutex);
		2336	*/
2329		2337
2330	sync_child_event(event);	2338	sync_child_event(event);
2331	list_del_init(&event->child_list);	2339	list_del_init(&event->child_list);
@@ -2343,6 +2351,11 @@ event_filter_match(struct perf_event *event) · 11 unmodified lines
2343	perf_cgroup_match(event);	2351	perf_cgroup_match(event);
2344	}	2352	}
2345		2353
		2354	static inline bool is_event_in_freq_mode(struct perf_event *event)
		2355	{
		2356	return event->attr.freq && event->attr.sample_freq;
		2357	}
		2358
2346	static void	2359	static void
2347	event_sched_out(struct perf_event event, struct perf_event_context ctx)	2360	event_sched_out(struct perf_event event, struct perf_event_context ctx)
2348	{	2361	{
@@ -2380,7 +2393,7 @@ event_sched_out(struct perf_event event, struct perf_event_context ctx) · 31 unmodified lines
2380		2393
2381	if (!is_software_event(event))	2394	if (!is_software_event(event))
2382	cpc->active_oncpu--;	2395	cpc->active_oncpu--;
2383	if (event->attr.freq && event->attr.sample_freq) {	2396	if (is_event_in_freq_mode(event)) {
2384	ctx->nr_freq--;	2397	ctx->nr_freq--;
2385	epc->nr_freq--;	2398	epc->nr_freq--;
2386	}	2399	}
@@ -2450,8 +2463,9 @@ ctx_time_update_event(struct perf_event_context ctx, struct perf_event event) · 63 unmodified lines
2450		2463
2451	#define DETACH_GROUP 0x01UL	2464	#define DETACH_GROUP 0x01UL
2452	#define DETACH_CHILD 0x02UL	2465	#define DETACH_CHILD 0x02UL
2453	#define DETACH_DEAD 0x04UL	2466	#define DETACH_EXIT 0x04UL
2454	#define DETACH_EXIT 0x08UL	2467	#define DETACH_REVOKE 0x08UL
		2468	#define DETACH_DEAD 0x10UL
2455		2469
2456	/*	2470	/*
2457	* Cross CPU call to remove a performance event	2471	* Cross CPU call to remove a performance event
@@ -2477,12 +2491,15 @@ __perf_remove_from_context(struct perf_event *event, · 19 unmodified lines
2477	*/	2491	*/
2478	if (flags & DETACH_EXIT)	2492	if (flags & DETACH_EXIT)
2479	state = PERF_EVENT_STATE_EXIT;	2493	state = PERF_EVENT_STATE_EXIT;
		2494	if (flags & DETACH_REVOKE)
		2495	state = PERF_EVENT_STATE_REVOKED;
2480	if (flags & DETACH_DEAD) {	2496	if (flags & DETACH_DEAD) {
2481	event->pending_disable = 1;	2497	event->pending_disable = 1;
2482	state = PERF_EVENT_STATE_DEAD;	2498	state = PERF_EVENT_STATE_DEAD;
2483	}	2499	}
2484	event_sched_out(event, ctx);	2500	event_sched_out(event, ctx);
2485	perf_event_set_state(event, min(event->state, state));	2501	perf_event_set_state(event, min(event->state, state));
		2502
2486	if (flags & DETACH_GROUP)	2503	if (flags & DETACH_GROUP)
2487	perf_group_detach(event);	2504	perf_group_detach(event);
2488	if (flags & DETACH_CHILD)	2505	if (flags & DETACH_CHILD)
@@ -2628,6 +2645,41 @@ void perf_event_disable_inatomic(struct perf_event *event) · 139 unmodified lines
2628	static void perf_log_throttle(struct perf_event *event, int enable);	2645	static void perf_log_throttle(struct perf_event *event, int enable);
2629	static void perf_log_itrace_start(struct perf_event *event);	2646	static void perf_log_itrace_start(struct perf_event *event);
2630		2647
		2648	static void perf_event_unthrottle(struct perf_event *event, bool start)
		2649	{
		2650	event->hw.interrupts = 0;
		2651	if (start)
		2652	event->pmu->start(event, 0);
		2653	if (event == event->group_leader)
		2654	perf_log_throttle(event, 1);
		2655	}
		2656
		2657	static void perf_event_throttle(struct perf_event *event)
		2658	{
		2659	event->pmu->stop(event, 0);
		2660	event->hw.interrupts = MAX_INTERRUPTS;
		2661	if (event == event->group_leader)
		2662	perf_log_throttle(event, 0);
		2663	}
		2664
		2665	static void perf_event_unthrottle_group(struct perf_event *event, bool skip_start_event)
		2666	{
		2667	struct perf_event sibling, leader = event->group_leader;
		2668
		2669	perf_event_unthrottle(leader, skip_start_event ? leader != event : true);
		2670	for_each_sibling_event(sibling, leader)
		2671	perf_event_unthrottle(sibling, skip_start_event ? sibling != event : true);
		2672	}
		2673
		2674	static void perf_event_throttle_group(struct perf_event *event)
		2675	{
		2676	struct perf_event sibling, leader = event->group_leader;
		2677
		2678	perf_event_throttle(leader);
		2679	for_each_sibling_event(sibling, leader)
		2680	perf_event_throttle(sibling);
		2681	}
		2682
2631	static int	2683	static int
2632	event_sched_in(struct perf_event event, struct perf_event_context ctx)	2684	event_sched_in(struct perf_event event, struct perf_event_context ctx)
2633	{	2685	{
@@ -2656,10 +2708,8 @@ event_sched_in(struct perf_event event, struct perf_event_context ctx) · 22 unmodified lines
2656	* ticks already, also for a heavily scheduling task there is little	2708	* ticks already, also for a heavily scheduling task there is little
2657	* guarantee it'll get a tick in a timely manner.	2709	* guarantee it'll get a tick in a timely manner.
2658	*/	2710	*/
2659	if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) {	2711	if (unlikely(event->hw.interrupts == MAX_INTERRUPTS))
2660	perf_log_throttle(event, 1);	2712	perf_event_unthrottle(event, false);
2661	event->hw.interrupts = 0;
2662	}
2663		2713
2664	perf_pmu_disable(event->pmu);	2714	perf_pmu_disable(event->pmu);
2665		2715
@@ -2674,7 +2724,7 @@ event_sched_in(struct perf_event event, struct perf_event_context ctx) · 8 unmodified lines
2674		2724
2675	if (!is_software_event(event))	2725	if (!is_software_event(event))
2676	cpc->active_oncpu++;	2726	cpc->active_oncpu++;
2677	if (event->attr.freq && event->attr.sample_freq) {	2727	if (is_event_in_freq_mode(event)) {
2678	ctx->nr_freq++;	2728	ctx->nr_freq++;
2679	epc->nr_freq++;	2729	epc->nr_freq++;
2680	}	2730	}
@@ -4237,14 +4287,10 @@ static void perf_adjust_freq_unthr_events(struct list_head *event_list) · 1556 unmodified lines
4237		4287
4238	hwc = &event->hw;	4288	hwc = &event->hw;
4239		4289
4240	if (hwc->interrupts == MAX_INTERRUPTS) {	4290	if (hwc->interrupts == MAX_INTERRUPTS)
4241	hwc->interrupts = 0;	4291	perf_event_unthrottle_group(event, is_event_in_freq_mode(event));
4242	perf_log_throttle(event, 1);
4243	if (!event->attr.freq \|\| !event->attr.sample_freq)
4244	event->pmu->start(event, 0);
4245	}
4246		4292
4247	if (!event->attr.freq \|\| !event->attr.sample_freq)	4293	if (!is_event_in_freq_mode(event))
4248	continue;	4294	continue;
4249		4295
4250	/*	4296	/*
@@ -4516,7 +4562,8 @@ out: · 265 unmodified lines
4516		4562
4517	static void perf_remove_from_owner(struct perf_event *event);	4563	static void perf_remove_from_owner(struct perf_event *event);
4518	static void perf_event_exit_event(struct perf_event *event,	4564	static void perf_event_exit_event(struct perf_event *event,
4519	struct perf_event_context *ctx);	4565	struct perf_event_context *ctx,
		4566	bool revoke);
4520		4567
4521	/*	4568	/*
4522	* Removes all events from the current task that have been marked	4569	* Removes all events from the current task that have been marked
@@ -4543,7 +4590,7 @@ static void perf_event_remove_on_exec(struct perf_event_context *ctx) · 20 unmodified lines
4543		4590
4544	modified = true;	4591	modified = true;
4545		4592
4546	perf_event_exit_event(event, ctx);	4593	perf_event_exit_event(event, ctx, false);
4547	}	4594	}
4548		4595
4549	raw_spin_lock_irqsave(&ctx->lock, flags);	4596	raw_spin_lock_irqsave(&ctx->lock, flags);
@@ -5125,6 +5172,7 @@ static bool is_sb_event(struct perf_event *event) · 575 unmodified lines
5125	attr->context_switch \|\| attr->text_poke \|\|	5172	attr->context_switch \|\| attr->text_poke \|\|
5126	attr->bpf_event)	5173	attr->bpf_event)
5127	return true;	5174	return true;
		5175
5128	return false;	5176	return false;
5129	}	5177	}
5130		5178
@@ -5521,6 +5569,8 @@ static void perf_free_addr_filters(struct perf_event *event); · 390 unmodified lines
5521	/* vs perf_event_alloc() error */	5569	/* vs perf_event_alloc() error */
5522	static void __free_event(struct perf_event *event)	5570	static void __free_event(struct perf_event *event)
5523	{	5571	{
		5572	struct pmu *pmu = event->pmu;
		5573
5524	if (event->attach_state & PERF_ATTACH_CALLCHAIN)	5574	if (event->attach_state & PERF_ATTACH_CALLCHAIN)
5525	put_callchain_buffers();	5575	put_callchain_buffers();
5526		5576
@@ -5550,6 +5600,7 @@ static void __free_event(struct perf_event *event) · 23 unmodified lines
5550	* put_pmu_ctx() needs an event->ctx reference, because of	5600	* put_pmu_ctx() needs an event->ctx reference, because of
5551	* epc->ctx.	5601	* epc->ctx.
5552	*/	5602	*/
		5603	WARN_ON_ONCE(!pmu);
5553	WARN_ON_ONCE(!event->ctx);	5604	WARN_ON_ONCE(!event->ctx);
5554	WARN_ON_ONCE(event->pmu_ctx->ctx != event->ctx);	5605	WARN_ON_ONCE(event->pmu_ctx->ctx != event->ctx);
5555	put_pmu_ctx(event->pmu_ctx);	5606	put_pmu_ctx(event->pmu_ctx);
@@ -5562,8 +5613,13 @@ static void __free_event(struct perf_event *event) · 6 unmodified lines
5562	if (event->ctx)	5613	if (event->ctx)
5563	put_ctx(event->ctx);	5614	put_ctx(event->ctx);
5564		5615
5565	if (event->pmu)	5616	if (pmu) {
5566	module_put(event->pmu->module);	5617	module_put(pmu->module);
		5618	scoped_guard (spinlock, &pmu->events_lock) {
		5619	list_del(&event->pmu_list);
		5620	wake_up_var(pmu);
		5621	}
		5622	}
5567		5623
5568	call_rcu(&event->rcu_head, free_event_rcu);	5624	call_rcu(&event->rcu_head, free_event_rcu);
5569	}	5625	}
@@ -5600,13 +5656,13 @@ static void _free_event(struct perf_event *event) · 30 unmodified lines
5600		5656
5601	/*	5657	/*
5602	* Used to free events which have a known refcount of 1, such as in error paths	5658	* Used to free events which have a known refcount of 1, such as in error paths
5603	* where the event isn't exposed yet and inherited events.	5659	* of inherited events.
5604	*/	5660	*/
5605	static void free_event(struct perf_event *event)	5661	static void free_event(struct perf_event *event)
5606	{	5662	{
5607	if (WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1,	5663	if (WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1,
5608	"unexpected event refcount: %ld; ptr=%p\n",	5664	"unexpected event refcount: %ld; ptr=%p\n",
5609	atomic_long_read(&event->refcount), event)) {	5665	atomic_long_read(&event->refcount), event)) {
5610	/* leak to avoid use-after-free */	5666	/* leak to avoid use-after-free */
5611	return;	5667	return;
5612	}	5668	}
@@ -5689,7 +5745,6 @@ int perf_event_release_kernel(struct perf_event *event) · 76 unmodified lines
5689	{	5745	{
5690	struct perf_event_context *ctx = event->ctx;	5746	struct perf_event_context *ctx = event->ctx;
5691	struct perf_event child, tmp;	5747	struct perf_event child, tmp;
5692	LIST_HEAD(free_list);
5693		5748
5694	/*	5749	/*
5695	* If we got here through err_alloc: free_event(event); we will not	5750	* If we got here through err_alloc: free_event(event); we will not
@@ -5718,15 +5773,17 @@ int perf_event_release_kernel(struct perf_event *event) · 22 unmodified lines
5718	* Thus this guarantees that we will in fact observe and kill _ALL_	5773	* Thus this guarantees that we will in fact observe and kill _ALL_
5719	* child events.	5774	* child events.
5720	*/	5775	*/
5721	perf_remove_from_context(event, DETACH_GROUP\|DETACH_DEAD);	5776	if (event->state > PERF_EVENT_STATE_REVOKED) {
		5777	perf_remove_from_context(event, DETACH_GROUP\|DETACH_DEAD);
		5778	} else {
		5779	event->state = PERF_EVENT_STATE_DEAD;
		5780	}
5722		5781
5723	perf_event_ctx_unlock(event, ctx);	5782	perf_event_ctx_unlock(event, ctx);
5724		5783
5725	again:	5784	again:
5726	mutex_lock(&event->child_mutex);	5785	mutex_lock(&event->child_mutex);
5727	list_for_each_entry(child, &event->child_list, child_list) {	5786	list_for_each_entry(child, &event->child_list, child_list) {
5728	void *var = NULL;
5729
5730	/*	5787	/*
5731	* Cannot change, child events are not migrated, see the	5788	* Cannot change, child events are not migrated, see the
5732	* comment with perf_event_ctx_lock_nested().	5789	* comment with perf_event_ctx_lock_nested().
@@ -5759,44 +5816,24 @@ again: · 26 unmodified lines
5759	tmp = list_first_entry_or_null(&event->child_list,	5816	tmp = list_first_entry_or_null(&event->child_list,
5760	struct perf_event, child_list);	5817	struct perf_event, child_list);
5761	if (tmp == child) {	5818	if (tmp == child) {
5762	perf_remove_from_context(child, DETACH_GROUP);	5819	perf_remove_from_context(child, DETACH_GROUP \| DETACH_CHILD);
5763	list_move(&child->child_list, &free_list);
5764	} else {	5820	} else {
5765	var = &ctx->refcount;	5821	child = NULL;
5766	}	5822	}
5767		5823
5768	mutex_unlock(&event->child_mutex);	5824	mutex_unlock(&event->child_mutex);
5769	mutex_unlock(&ctx->mutex);	5825	mutex_unlock(&ctx->mutex);
5770	put_ctx(ctx);
5771		5826
5772	if (var) {	5827	if (child) {
5773	/*	5828	/* Last reference unless ->pending_task work is pending */
5774	* If perf_event_free_task() has deleted all events from the	5829	put_event(child);
5775	* ctx while the child_mutex got released above, make sure to
5776	* notify about the preceding put_ctx().
5777	*/
5778	smp_mb(); /* pairs with wait_var_event() */
5779	wake_up_var(var);
5780	}	5830	}
		5831	put_ctx(ctx);
		5832
5781	goto again;	5833	goto again;
5782	}	5834	}
5783	mutex_unlock(&event->child_mutex);	5835	mutex_unlock(&event->child_mutex);
5784		5836
5785	list_for_each_entry_safe(child, tmp, &free_list, child_list) {
5786	void *var = &child->ctx->refcount;
5787
5788	list_del(&child->child_list);
5789	/* Last reference unless ->pending_task work is pending */
5790	put_event(child);
5791
5792	/*
5793	* Wake any perf_event_free_task() waiting for this event to be
5794	* freed.
5795	*/
5796	smp_mb(); /* pairs with wait_var_event() */
5797	wake_up_var(var);
5798	}
5799
5800	no_ctx:	5837	no_ctx:
5801	/*	5838	/*
5802	* Last reference unless ->pending_task work is pending on this event	5839	* Last reference unless ->pending_task work is pending on this event
@@ -6068,8 +6105,14 @@ static __poll_t perf_poll(struct file file, poll_table wait) · 265 unmodified lines
6068	struct perf_buffer *rb;	6105	struct perf_buffer *rb;
6069	__poll_t events = EPOLLHUP;	6106	__poll_t events = EPOLLHUP;
6070		6107
		6108	if (event->state <= PERF_EVENT_STATE_REVOKED)
		6109	return EPOLLERR;
		6110
6071	poll_wait(file, &event->waitq, wait);	6111	poll_wait(file, &event->waitq, wait);
6072		6112
		6113	if (event->state <= PERF_EVENT_STATE_REVOKED)
		6114	return EPOLLERR;
		6115
6073	if (is_event_hup(event))	6116	if (is_event_hup(event))
6074	return events;	6117	return events;
6075		6118
@@ -6167,14 +6210,6 @@ static void __perf_event_period(struct perf_event *event, · 91 unmodified lines
6167	active = (event->state == PERF_EVENT_STATE_ACTIVE);	6210	active = (event->state == PERF_EVENT_STATE_ACTIVE);
6168	if (active) {	6211	if (active) {
6169	perf_pmu_disable(event->pmu);	6212	perf_pmu_disable(event->pmu);
6170	/*
6171	* We could be throttled; unthrottle now to avoid the tick
6172	* trying to unthrottle while we already re-started the event.
6173	*/
6174	if (event->hw.interrupts == MAX_INTERRUPTS) {
6175	event->hw.interrupts = 0;
6176	perf_log_throttle(event, 1);
6177	}
6178	event->pmu->stop(event, PERF_EF_UPDATE);	6213	event->pmu->stop(event, PERF_EF_UPDATE);
6179	}	6214	}
6180		6215
@@ -6182,6 +6217,14 @@ static void __perf_event_period(struct perf_event *event, · 1 unmodified line
6182		6217
6183	if (active) {	6218	if (active) {
6184	event->pmu->start(event, PERF_EF_RELOAD);	6219	event->pmu->start(event, PERF_EF_RELOAD);
		6220	/*
		6221	* Once the period is force-reset, the event starts immediately.
		6222	* But the event/group could be throttled. Unthrottle the
		6223	* event/group now to avoid the next tick trying to unthrottle
		6224	* while we already re-started the event/group.
		6225	*/
		6226	if (event->hw.interrupts == MAX_INTERRUPTS)
		6227	perf_event_unthrottle_group(event, true);
6185	perf_pmu_enable(event->pmu);	6228	perf_pmu_enable(event->pmu);
6186	}	6229	}
6187	}	6230	}
@@ -6239,12 +6282,18 @@ static int perf_event_set_output(struct perf_event *event, · 51 unmodified lines
6239	static int perf_event_set_filter(struct perf_event event, void __user arg);	6282	static int perf_event_set_filter(struct perf_event event, void __user arg);
6240	static int perf_copy_attr(struct perf_event_attr __user *uattr,	6283	static int perf_copy_attr(struct perf_event_attr __user *uattr,
6241	struct perf_event_attr *attr);	6284	struct perf_event_attr *attr);
		6285	static int __perf_event_set_bpf_prog(struct perf_event *event,
		6286	struct bpf_prog *prog,
		6287	u64 bpf_cookie);
6242		6288
6243	static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned long arg)	6289	static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned long arg)
6244	{	6290	{
6245	void (func)(struct perf_event );	6291	void (func)(struct perf_event );
6246	u32 flags = arg;	6292	u32 flags = arg;
6247		6293
		6294	if (event->state <= PERF_EVENT_STATE_REVOKED)
		6295	return -ENODEV;
		6296
6248	switch (cmd) {	6297	switch (cmd) {
6249	case PERF_EVENT_IOC_ENABLE:	6298	case PERF_EVENT_IOC_ENABLE:
6250	func = _perf_event_enable;	6299	func = _perf_event_enable;
@@ -6301,7 +6350,7 @@ static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned lon · 50 unmodified lines
6301	if (IS_ERR(prog))	6350	if (IS_ERR(prog))
6302	return PTR_ERR(prog);	6351	return PTR_ERR(prog);
6303		6352
6304	err = perf_event_set_bpf_prog(event, prog, 0);	6353	err = __perf_event_set_bpf_prog(event, prog, 0);
6305	if (err) {	6354	if (err) {
6306	bpf_prog_put(prog);	6355	bpf_prog_put(prog);
6307	return err;	6356	return err;
@@ -6620,9 +6669,22 @@ void ring_buffer_put(struct perf_buffer *rb) · 312 unmodified lines
6620	call_rcu(&rb->rcu_head, rb_free_rcu);	6669	call_rcu(&rb->rcu_head, rb_free_rcu);
6621	}	6670	}
6622		6671
		6672	typedef void (mapped_f)(struct perf_event event, struct mm_struct *mm);
		6673
		6674	#define get_mapped(event, func) \
		6675	({ struct pmu *pmu; \
		6676	mapped_f f = NULL; \
		6677	guard(rcu)(); \
		6678	pmu = READ_ONCE(event->pmu); \
		6679	if (pmu) \
		6680	f = pmu->func; \
		6681	f; \
		6682	})
		6683
6623	static void perf_mmap_open(struct vm_area_struct *vma)	6684	static void perf_mmap_open(struct vm_area_struct *vma)
6624	{	6685	{
6625	struct perf_event *event = vma->vm_file->private_data;	6686	struct perf_event *event = vma->vm_file->private_data;
		6687	mapped_f mapped = get_mapped(event, event_mapped);
6626		6688
6627	atomic_inc(&event->mmap_count);	6689	atomic_inc(&event->mmap_count);
6628	atomic_inc(&event->rb->mmap_count);	6690	atomic_inc(&event->rb->mmap_count);
@@ -6630,8 +6692,8 @@ static void perf_mmap_open(struct vm_area_struct *vma) · 1 unmodified line
6630	if (vma->vm_pgoff)	6692	if (vma->vm_pgoff)
6631	atomic_inc(&event->rb->aux_mmap_count);	6693	atomic_inc(&event->rb->aux_mmap_count);
6632		6694
6633	if (event->pmu->event_mapped)	6695	if (mapped)
6634	event->pmu->event_mapped(event, vma->vm_mm);	6696	mapped(event, vma->vm_mm);
6635	}	6697	}
6636		6698
6637	static void perf_pmu_output_stop(struct perf_event *event);	6699	static void perf_pmu_output_stop(struct perf_event *event);
@@ -6647,14 +6709,16 @@ static void perf_pmu_output_stop(struct perf_event *event); · 9 unmodified lines
6647	static void perf_mmap_close(struct vm_area_struct *vma)	6709	static void perf_mmap_close(struct vm_area_struct *vma)
6648	{	6710	{
6649	struct perf_event *event = vma->vm_file->private_data;	6711	struct perf_event *event = vma->vm_file->private_data;
		6712	mapped_f unmapped = get_mapped(event, event_unmapped);
6650	struct perf_buffer *rb = ring_buffer_get(event);	6713	struct perf_buffer *rb = ring_buffer_get(event);
6651	struct user_struct *mmap_user = rb->mmap_user;	6714	struct user_struct *mmap_user = rb->mmap_user;
6652	int mmap_locked = rb->mmap_locked;	6715	int mmap_locked = rb->mmap_locked;
6653	unsigned long size = perf_data_size(rb);	6716	unsigned long size = perf_data_size(rb);
6654	bool detach_rest = false;	6717	bool detach_rest = false;
6655		6718
6656	if (event->pmu->event_unmapped)	6719	/* FIXIES vs perf_pmu_unregister() */
6657	event->pmu->event_unmapped(event, vma->vm_mm);	6720	if (unmapped)
		6721	unmapped(event, vma->vm_mm);
6658		6722
6659	/*	6723	/*
6660	* The AUX buffer is strictly a sub-buffer, serialize using aux_mutex	6724	* The AUX buffer is strictly a sub-buffer, serialize using aux_mutex
@@ -6847,6 +6911,7 @@ static int perf_mmap(struct file file, struct vm_area_struct vma) · 186 unmodified lines
6847	unsigned long nr_pages;	6911	unsigned long nr_pages;
6848	long user_extra = 0, extra = 0;	6912	long user_extra = 0, extra = 0;
6849	int ret, flags = 0;	6913	int ret, flags = 0;
		6914	mapped_f mapped;
6850		6915
6851	/*	6916	/*
6852	* Don't allow mmap() of inherited per-task counters. This would	6917	* Don't allow mmap() of inherited per-task counters. This would
@@ -6877,6 +6942,16 @@ static int perf_mmap(struct file file, struct vm_area_struct vma) · 24 unmodified lines
6877	mutex_lock(&event->mmap_mutex);	6942	mutex_lock(&event->mmap_mutex);
6878	ret = -EINVAL;	6943	ret = -EINVAL;
6879		6944
		6945	/*
		6946	* This relies on __pmu_detach_event() taking mmap_mutex after marking
		6947	* the event REVOKED. Either we observe the state, or __pmu_detach_event()
		6948	* will detach the rb created here.
		6949	*/
		6950	if (event->state <= PERF_EVENT_STATE_REVOKED) {
		6951	ret = -ENODEV;
		6952	goto unlock;
		6953	}
		6954
6880	if (vma->vm_pgoff == 0) {	6955	if (vma->vm_pgoff == 0) {
6881	nr_pages -= 1;	6956	nr_pages -= 1;
6882		6957
@@ -7055,8 +7130,9 @@ aux_unlock: · 172 unmodified lines
7055	if (!ret)	7130	if (!ret)
7056	ret = map_range(rb, vma);	7131	ret = map_range(rb, vma);
7057		7132
7058	if (!ret && event->pmu->event_mapped)	7133	mapped = get_mapped(event, event_mapped);
7059	event->pmu->event_mapped(event, vma->vm_mm);	7134	if (mapped)
		7135	mapped(event, vma->vm_mm);
7060		7136
7061	return ret;	7137	return ret;
7062	}	7138	}
@@ -7067,6 +7143,9 @@ static int perf_fasync(int fd, struct file *filp, int on) · 4 unmodified lines
7067	struct perf_event *event = filp->private_data;	7143	struct perf_event *event = filp->private_data;
7068	int retval;	7144	int retval;
7069		7145
		7146	if (event->state <= PERF_EVENT_STATE_REVOKED)
		7147	return -ENODEV;
		7148
7070	inode_lock(inode);	7149	inode_lock(inode);
7071	retval = fasync_helper(fd, filp, on, &event->fasync);	7150	retval = fasync_helper(fd, filp, on, &event->fasync);
7072	inode_unlock(inode);	7151	inode_unlock(inode);
@@ -9946,7 +10025,7 @@ void perf_event_text_poke(const void addr, const void old_bytes, · 2873 unmodified lines
9946		10025
9947	void perf_event_itrace_started(struct perf_event *event)	10026	void perf_event_itrace_started(struct perf_event *event)
9948	{	10027	{
9949	event->attach_state \|= PERF_ATTACH_ITRACE;	10028	WRITE_ONCE(event->attach_state, event->attach_state \| PERF_ATTACH_ITRACE);
9950	}	10029	}
9951		10030
9952	static void perf_log_itrace_start(struct perf_event *event)	10031	static void perf_log_itrace_start(struct perf_event *event)
@@ -10029,14 +10108,13 @@ __perf_event_account_interrupt(struct perf_event *event, int throttle) · 76 unmodified lines
10029	hwc->interrupts = 1;	10108	hwc->interrupts = 1;
10030	} else {	10109	} else {
10031	hwc->interrupts++;	10110	hwc->interrupts++;
10032	if (unlikely(throttle &&	10111	}
10033	hwc->interrupts > max_samples_per_tick)) {	10112
10034	__this_cpu_inc(perf_throttled_count);	10113	if (unlikely(throttle && hwc->interrupts >= max_samples_per_tick)) {
10035	tick_dep_set_cpu(smp_processor_id(), TICK_DEP_BIT_PERF_EVENTS);	10114	__this_cpu_inc(perf_throttled_count);
10036	hwc->interrupts = MAX_INTERRUPTS;	10115	tick_dep_set_cpu(smp_processor_id(), TICK_DEP_BIT_PERF_EVENTS);
10037	perf_log_throttle(event, 0);	10116	perf_event_throttle_group(event);
10038	ret = 1;	10117	ret = 1;
10039	}
10040	}	10118	}
10041		10119
10042	if (event->attr.freq) {	10120	if (event->attr.freq) {
@@ -11069,11 +11147,15 @@ static inline bool perf_event_is_tracing(struct perf_event *event) · 1026 unmodified lines
11069	return false;	11147	return false;
11070	}	11148	}
11071		11149
11072	int perf_event_set_bpf_prog(struct perf_event event, struct bpf_prog prog,	11150	static int __perf_event_set_bpf_prog(struct perf_event *event,
11073	u64 bpf_cookie)	11151	struct bpf_prog *prog,
		11152	u64 bpf_cookie)
11074	{	11153	{
11075	bool is_kprobe, is_uprobe, is_tracepoint, is_syscall_tp;	11154	bool is_kprobe, is_uprobe, is_tracepoint, is_syscall_tp;
11076		11155
		11156	if (event->state <= PERF_EVENT_STATE_REVOKED)
		11157	return -ENODEV;
		11158
11077	if (!perf_event_is_tracing(event))	11159	if (!perf_event_is_tracing(event))
11078	return perf_event_set_bpf_handler(event, prog, bpf_cookie);	11160	return perf_event_set_bpf_handler(event, prog, bpf_cookie);
11079		11161
@@ -11108,6 +11190,20 @@ int perf_event_set_bpf_prog(struct perf_event event, struct bpf_prog prog, · 28 unmodified lines
11108	return perf_event_attach_bpf_prog(event, prog, bpf_cookie);	11190	return perf_event_attach_bpf_prog(event, prog, bpf_cookie);
11109	}	11191	}
11110		11192
		11193	int perf_event_set_bpf_prog(struct perf_event *event,
		11194	struct bpf_prog *prog,
		11195	u64 bpf_cookie)
		11196	{
		11197	struct perf_event_context *ctx;
		11198	int ret;
		11199
		11200	ctx = perf_event_ctx_lock(event);
		11201	ret = __perf_event_set_bpf_prog(event, prog, bpf_cookie);
		11202	perf_event_ctx_unlock(event, ctx);
		11203
		11204	return ret;
		11205	}
		11206
11111	void perf_event_free_bpf_prog(struct perf_event *event)	11207	void perf_event_free_bpf_prog(struct perf_event *event)
11112	{	11208	{
11113	if (!event->prog)	11209	if (!event->prog)
@@ -11130,7 +11226,15 @@ static void perf_event_free_filter(struct perf_event *event) · 16 unmodified lines
11130	{	11226	{
11131	}	11227	}
11132		11228
11133	int perf_event_set_bpf_prog(struct perf_event event, struct bpf_prog prog,	11229	static int __perf_event_set_bpf_prog(struct perf_event *event,
		11230	struct bpf_prog *prog,
		11231	u64 bpf_cookie)
		11232	{
		11233	return -ENOENT;
		11234	}
		11235
		11236	int perf_event_set_bpf_prog(struct perf_event *event,
		11237	struct bpf_prog *prog,
11134	u64 bpf_cookie)	11238	u64 bpf_cookie)
11135	{	11239	{
11136	return -ENOENT;	11240	return -ENOENT;
@@ -12235,6 +12339,9 @@ int perf_pmu_register(struct pmu _pmu, const char name, int type) · 1098 unmodified lines
12235	if (!pmu->event_idx)	12339	if (!pmu->event_idx)
12236	pmu->event_idx = perf_event_idx_default;	12340	pmu->event_idx = perf_event_idx_default;
12237		12341
		12342	INIT_LIST_HEAD(&pmu->events);
		12343	spin_lock_init(&pmu->events_lock);
		12344
12238	/*	12345	/*
12239	* Now that the PMU is complete, make it visible to perf_try_init_event().	12346	* Now that the PMU is complete, make it visible to perf_try_init_event().
12240	*/	12347	*/
@@ -12248,21 +12355,143 @@ int perf_pmu_register(struct pmu _pmu, const char name, int type) · 7 unmodified lines
12248	}	12355	}
12249	EXPORT_SYMBOL_GPL(perf_pmu_register);	12356	EXPORT_SYMBOL_GPL(perf_pmu_register);
12250		12357
12251	void perf_pmu_unregister(struct pmu *pmu)	12358	static void __pmu_detach_event(struct pmu pmu, struct perf_event event,
		12359	struct perf_event_context *ctx)
		12360	{
		12361	/*
		12362	* De-schedule the event and mark it REVOKED.
		12363	*/
		12364	perf_event_exit_event(event, ctx, true);
		12365
		12366	/*
		12367	* All _free_event() bits that rely on event->pmu:
		12368	*
		12369	* Notably, perf_mmap() relies on the ordering here.
		12370	*/
		12371	scoped_guard (mutex, &event->mmap_mutex) {
		12372	WARN_ON_ONCE(pmu->event_unmapped);
		12373	/*
		12374	* Mostly an empty lock sequence, such that perf_mmap(), which
		12375	* relies on mmap_mutex, is sure to observe the state change.
		12376	*/
		12377	}
		12378
		12379	perf_event_free_bpf_prog(event);
		12380	perf_free_addr_filters(event);
		12381
		12382	if (event->destroy) {
		12383	event->destroy(event);
		12384	event->destroy = NULL;
		12385	}
		12386
		12387	if (event->pmu_ctx) {
		12388	put_pmu_ctx(event->pmu_ctx);
		12389	event->pmu_ctx = NULL;
		12390	}
		12391
		12392	exclusive_event_destroy(event);
		12393	module_put(pmu->module);
		12394
		12395	event->pmu = NULL; /* force fault instead of UAF */
		12396	}
		12397
		12398	static void pmu_detach_event(struct pmu pmu, struct perf_event event)
		12399	{
		12400	struct perf_event_context *ctx;
		12401
		12402	ctx = perf_event_ctx_lock(event);
		12403	__pmu_detach_event(pmu, event, ctx);
		12404	perf_event_ctx_unlock(event, ctx);
		12405
		12406	scoped_guard (spinlock, &pmu->events_lock)
		12407	list_del(&event->pmu_list);
		12408	}
		12409
		12410	static struct perf_event pmu_get_event(struct pmu pmu)
		12411	{
		12412	struct perf_event *event;
		12413
		12414	guard(spinlock)(&pmu->events_lock);
		12415	list_for_each_entry(event, &pmu->events, pmu_list) {
		12416	if (atomic_long_inc_not_zero(&event->refcount))
		12417	return event;
		12418	}
		12419
		12420	return NULL;
		12421	}
		12422
		12423	static bool pmu_empty(struct pmu *pmu)
		12424	{
		12425	guard(spinlock)(&pmu->events_lock);
		12426	return list_empty(&pmu->events);
		12427	}
		12428
		12429	static void pmu_detach_events(struct pmu *pmu)
		12430	{
		12431	struct perf_event *event;
		12432
		12433	for (;;) {
		12434	event = pmu_get_event(pmu);
		12435	if (!event)
		12436	break;
		12437
		12438	pmu_detach_event(pmu, event);
		12439	put_event(event);
		12440	}
		12441
		12442	/*
		12443	* wait for pending _free_event()s
		12444	*/
		12445	wait_var_event(pmu, pmu_empty(pmu));
		12446	}
		12447
		12448	int perf_pmu_unregister(struct pmu *pmu)
12252	{	12449	{
12253	scoped_guard (mutex, &pmus_lock) {	12450	scoped_guard (mutex, &pmus_lock) {
		12451	if (!idr_cmpxchg(&pmu_idr, pmu->type, pmu, NULL))
		12452	return -EINVAL;
		12453
12254	list_del_rcu(&pmu->entry);	12454	list_del_rcu(&pmu->entry);
12255	idr_remove(&pmu_idr, pmu->type);
12256	}	12455	}
12257		12456
12258	/*	12457	/*
12259	* We dereference the pmu list under both SRCU and regular RCU, so	12458	* We dereference the pmu list under both SRCU and regular RCU, so
12260	* synchronize against both of those.	12459	* synchronize against both of those.
		12460	*
		12461	* Notably, the entirety of event creation, from perf_init_event()
		12462	* (which will now fail, because of the above) until
		12463	* perf_install_in_context() should be under SRCU such that
		12464	* this synchronizes against event creation. This avoids trying to
		12465	* detach events that are not fully formed.
12261	*/	12466	*/
12262	synchronize_srcu(&pmus_srcu);	12467	synchronize_srcu(&pmus_srcu);
12263	synchronize_rcu();	12468	synchronize_rcu();
12264		12469
		12470	if (pmu->event_unmapped && !pmu_empty(pmu)) {
		12471	/*
		12472	* Can't force remove events when pmu::event_unmapped()
		12473	* is used in perf_mmap_close().
		12474	*/
		12475	guard(mutex)(&pmus_lock);
		12476	idr_cmpxchg(&pmu_idr, pmu->type, NULL, pmu);
		12477	list_add_rcu(&pmu->entry, &pmus);
		12478	return -EBUSY;
		12479	}
		12480
		12481	scoped_guard (mutex, &pmus_lock)
		12482	idr_remove(&pmu_idr, pmu->type);
		12483
		12484	/*
		12485	* PMU is removed from the pmus list, so no new events will
		12486	* be created, now take care of the existing ones.
		12487	*/
		12488	pmu_detach_events(pmu);
		12489
		12490	/*
		12491	* PMU is unused, make it go away.
		12492	*/
12265	perf_pmu_free(pmu);	12493	perf_pmu_free(pmu);
		12494	return 0;
12266	}	12495	}
12267	EXPORT_SYMBOL_GPL(perf_pmu_unregister);	12496	EXPORT_SYMBOL_GPL(perf_pmu_unregister);
12268		12497
@@ -12356,7 +12585,7 @@ static struct pmu perf_init_event(struct perf_event event) · 87 unmodified lines
12356	struct pmu *pmu;	12585	struct pmu *pmu;
12357	int type, ret;	12586	int type, ret;
12358		12587
12359	guard(srcu)(&pmus_srcu);	12588	guard(srcu)(&pmus_srcu); /* pmu idr/list access */
12360		12589
12361	/*	12590	/*
12362	* Save original type before calling pmu->event_init() since certain	12591	* Save original type before calling pmu->event_init() since certain
@@ -12580,6 +12809,7 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu, · 217 unmodified lines
12580	INIT_LIST_HEAD(&event->active_entry);	12809	INIT_LIST_HEAD(&event->active_entry);
12581	INIT_LIST_HEAD(&event->addr_filters.list);	12810	INIT_LIST_HEAD(&event->addr_filters.list);
12582	INIT_HLIST_NODE(&event->hlist_entry);	12811	INIT_HLIST_NODE(&event->hlist_entry);
		12812	INIT_LIST_HEAD(&event->pmu_list);
12583		12813
12584		12814
12585	init_waitqueue_head(&event->waitq);	12815	init_waitqueue_head(&event->waitq);
@@ -12651,7 +12881,7 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu, · 65 unmodified lines
12651		12881
12652	hwc = &event->hw;	12882	hwc = &event->hw;
12653	hwc->sample_period = attr->sample_period;	12883	hwc->sample_period = attr->sample_period;
12654	if (attr->freq && attr->sample_freq)	12884	if (is_event_in_freq_mode(event))
12655	hwc->sample_period = 1;	12885	hwc->sample_period = 1;
12656	hwc->last_period = hwc->sample_period;	12886	hwc->last_period = hwc->sample_period;
12657		12887
@@ -12758,6 +12988,13 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu, · 100 unmodified lines
12758	/* symmetric to unaccount_event() in _free_event() */	12988	/* symmetric to unaccount_event() in _free_event() */
12759	account_event(event);	12989	account_event(event);
12760		12990
		12991	/*
		12992	* Event creation should be under SRCU, see perf_pmu_unregister().
		12993	*/
		12994	lockdep_assert_held(&pmus_srcu);
		12995	scoped_guard (spinlock, &pmu->events_lock)
		12996	list_add(&event->pmu_list, &pmu->events);
		12997
12761	return_ptr(event);	12998	return_ptr(event);
12762	}	12999	}
12763		13000
@@ -12957,6 +13194,9 @@ set: · 193 unmodified lines
12957	goto unlock;	13194	goto unlock;
12958		13195
12959	if (output_event) {	13196	if (output_event) {
		13197	if (output_event->state <= PERF_EVENT_STATE_REVOKED)
		13198	goto unlock;
		13199
12960	/* get the rb we want to redirect to */	13200	/* get the rb we want to redirect to */
12961	rb = ring_buffer_get(output_event);	13201	rb = ring_buffer_get(output_event);
12962	if (!rb)	13202	if (!rb)
@@ -13138,6 +13378,11 @@ SYSCALL_DEFINE5(perf_event_open, · 175 unmodified lines
13138	if (event_fd < 0)	13378	if (event_fd < 0)
13139	return event_fd;	13379	return event_fd;
13140		13380
		13381	/*
		13382	* Event creation should be under SRCU, see perf_pmu_unregister().
		13383	*/
		13384	guard(srcu)(&pmus_srcu);
		13385
13141	CLASS(fd, group)(group_fd); // group_fd == -1 => empty	13386	CLASS(fd, group)(group_fd); // group_fd == -1 => empty
13142	if (group_fd != -1) {	13387	if (group_fd != -1) {
13143	if (!is_perf_file(group)) {	13388	if (!is_perf_file(group)) {
@@ -13145,6 +13390,10 @@ SYSCALL_DEFINE5(perf_event_open, · 1 unmodified line
13145	goto err_fd;	13390	goto err_fd;
13146	}	13391	}
13147	group_leader = fd_file(group)->private_data;	13392	group_leader = fd_file(group)->private_data;
		13393	if (group_leader->state <= PERF_EVENT_STATE_REVOKED) {
		13394	err = -ENODEV;
		13395	goto err_fd;
		13396	}
13148	if (flags & PERF_FLAG_FD_OUTPUT)	13397	if (flags & PERF_FLAG_FD_OUTPUT)
13149	output_event = group_leader;	13398	output_event = group_leader;
13150	if (flags & PERF_FLAG_FD_NO_GROUP)	13399	if (flags & PERF_FLAG_FD_NO_GROUP)
@@ -13441,7 +13690,7 @@ err_cred: · 290 unmodified lines
13441	if (task)	13690	if (task)
13442	up_read(&task->signal->exec_update_lock);	13691	up_read(&task->signal->exec_update_lock);
13443	err_alloc:	13692	err_alloc:
13444	free_event(event);	13693	put_event(event);
13445	err_task:	13694	err_task:
13446	if (task)	13695	if (task)
13447	put_task_struct(task);	13696	put_task_struct(task);
@@ -13478,6 +13727,11 @@ perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu, · 30 unmodified lines
13478	if (attr->aux_output \|\| attr->aux_action)	13727	if (attr->aux_output \|\| attr->aux_action)
13479	return ERR_PTR(-EINVAL);	13728	return ERR_PTR(-EINVAL);
13480		13729
		13730	/*
		13731	* Event creation should be under SRCU, see perf_pmu_unregister().
		13732	*/
		13733	guard(srcu)(&pmus_srcu);
		13734
13481	event = perf_event_alloc(attr, cpu, task, NULL, NULL,	13735	event = perf_event_alloc(attr, cpu, task, NULL, NULL,
13482	overflow_handler, context, -1);	13736	overflow_handler, context, -1);
13483	if (IS_ERR(event)) {	13737	if (IS_ERR(event)) {
@@ -13549,7 +13803,7 @@ err_unlock: · 65 unmodified lines
13549	perf_unpin_context(ctx);	13803	perf_unpin_context(ctx);
13550	put_ctx(ctx);	13804	put_ctx(ctx);
13551	err_alloc:	13805	err_alloc:
13552	free_event(event);	13806	put_event(event);
13553	err:	13807	err:
13554	return ERR_PTR(err);	13808	return ERR_PTR(err);
13555	}	13809	}
@@ -13689,10 +13943,12 @@ static void sync_child_event(struct perf_event *child_event) · 133 unmodified lines
13689	}	13943	}
13690		13944
13691	static void	13945	static void
13692	perf_event_exit_event(struct perf_event event, struct perf_event_context ctx)	13946	perf_event_exit_event(struct perf_event *event,
		13947	struct perf_event_context *ctx, bool revoke)
13693	{	13948	{
13694	struct perf_event *parent_event = event->parent;	13949	struct perf_event *parent_event = event->parent;
13695	unsigned long detach_flags = 0;	13950	unsigned long detach_flags = DETACH_EXIT;
		13951	unsigned int attach_state;
13696		13952
13697	if (parent_event) {	13953	if (parent_event) {
13698	/*	13954	/*
@@ -13707,22 +13963,38 @@ perf_event_exit_event(struct perf_event event, struct perf_event_context ctx) · 8 unmodified lines
13707	* Do destroy all inherited groups, we don't care about those	13963	* Do destroy all inherited groups, we don't care about those
13708	* and being thorough is better.	13964	* and being thorough is better.
13709	*/	13965	*/
13710	detach_flags = DETACH_GROUP \| DETACH_CHILD;	13966	detach_flags \|= DETACH_GROUP \| DETACH_CHILD;
13711	mutex_lock(&parent_event->child_mutex);	13967	mutex_lock(&parent_event->child_mutex);
		13968	/* PERF_ATTACH_ITRACE might be set concurrently */
		13969	attach_state = READ_ONCE(event->attach_state);
13712	}	13970	}
13713		13971
13714	perf_remove_from_context(event, detach_flags \| DETACH_EXIT);	13972	if (revoke)
		13973	detach_flags \|= DETACH_GROUP \| DETACH_REVOKE;
13715		13974
		13975	perf_remove_from_context(event, detach_flags);
13716	/*	13976	/*
13717	* Child events can be freed.	13977	* Child events can be freed.
13718	*/	13978	*/
13719	if (parent_event) {	13979	if (parent_event) {
13720	mutex_unlock(&parent_event->child_mutex);	13980	mutex_unlock(&parent_event->child_mutex);
		13981
13721	/*	13982	/*
13722	* Kick perf_poll() for is_event_hup();	13983	* Match the refcount initialization. Make sure it doesn't happen
		13984	* twice if pmu_detach_event() calls it on an already exited task.
13723	*/	13985	*/
13724	perf_event_wakeup(parent_event);	13986	if (attach_state & PERF_ATTACH_CHILD) {
13725	put_event(event);	13987	/*
		13988	* Kick perf_poll() for is_event_hup();
		13989	*/
		13990	perf_event_wakeup(parent_event);
		13991	/*
		13992	* pmu_detach_event() will have an extra refcount.
		13993	* perf_pending_task() might have one too.
		13994	*/
		13995	put_event(event);
		13996	}
		13997
13726	return;	13998	return;
13727	}	13999	}
13728		14000
@@ -13732,15 +14004,13 @@ perf_event_exit_event(struct perf_event event, struct perf_event_context ctx) · 3 unmodified lines
13732	perf_event_wakeup(event);	14004	perf_event_wakeup(event);
13733	}	14005	}
13734		14006
13735	static void perf_event_exit_task_context(struct task_struct *child)	14007	static void perf_event_exit_task_context(struct task_struct *task, bool exit)
13736	{	14008	{
13737	struct perf_event_context child_ctx, clone_ctx = NULL;	14009	struct perf_event_context ctx, clone_ctx = NULL;
13738	struct perf_event child_event, next;	14010	struct perf_event child_event, next;
13739		14011
13740	WARN_ON_ONCE(child != current);	14012	ctx = perf_pin_task_context(task);
13741		14013	if (!ctx)
13742	child_ctx = perf_pin_task_context(child);
13743	if (!child_ctx)
13744	return;	14014	return;
13745		14015
13746	/*	14016	/*
@@ -13753,27 +14023,28 @@ static void perf_event_exit_task_context(struct task_struct *child) · 6 unmodified lines
13753	* without ctx::mutex (it cannot because of the move_group double mutex	14023	* without ctx::mutex (it cannot because of the move_group double mutex
13754	* lock thing). See the comments in perf_install_in_context().	14024	* lock thing). See the comments in perf_install_in_context().
13755	*/	14025	*/
13756	mutex_lock(&child_ctx->mutex);	14026	mutex_lock(&ctx->mutex);
13757		14027
13758	/*	14028	/*
13759	* In a single ctx::lock section, de-schedule the events and detach the	14029	* In a single ctx::lock section, de-schedule the events and detach the
13760	* context from the task such that we cannot ever get it scheduled back	14030	* context from the task such that we cannot ever get it scheduled back
13761	* in.	14031	* in.
13762	*/	14032	*/
13763	raw_spin_lock_irq(&child_ctx->lock);	14033	raw_spin_lock_irq(&ctx->lock);
13764	task_ctx_sched_out(child_ctx, NULL, EVENT_ALL);	14034	if (exit)
		14035	task_ctx_sched_out(ctx, NULL, EVENT_ALL);
13765		14036
13766	/*	14037	/*
13767	* Now that the context is inactive, destroy the task <-> ctx relation	14038	* Now that the context is inactive, destroy the task <-> ctx relation
13768	* and mark the context dead.	14039	* and mark the context dead.
13769	*/	14040	*/
13770	RCU_INIT_POINTER(child->perf_event_ctxp, NULL);	14041	RCU_INIT_POINTER(task->perf_event_ctxp, NULL);
13771	put_ctx(child_ctx); /* cannot be last */	14042	put_ctx(ctx); /* cannot be last */
13772	WRITE_ONCE(child_ctx->task, TASK_TOMBSTONE);	14043	WRITE_ONCE(ctx->task, TASK_TOMBSTONE);
13773	put_task_struct(current); /* cannot be last */	14044	put_task_struct(task); /* cannot be last */
13774		14045
13775	clone_ctx = unclone_ctx(child_ctx);	14046	clone_ctx = unclone_ctx(ctx);
13776	raw_spin_unlock_irq(&child_ctx->lock);	14047	raw_spin_unlock_irq(&ctx->lock);
13777		14048
13778	if (clone_ctx)	14049	if (clone_ctx)
13779	put_ctx(clone_ctx);	14050	put_ctx(clone_ctx);
@@ -13783,28 +14054,48 @@ static void perf_event_exit_task_context(struct task_struct *child) · 3 unmodified lines
13783	* won't get any samples after PERF_RECORD_EXIT. We can however still	14054	* won't get any samples after PERF_RECORD_EXIT. We can however still
13784	* get a few PERF_RECORD_READ events.	14055	* get a few PERF_RECORD_READ events.
13785	*/	14056	*/
13786	perf_event_task(child, child_ctx, 0);	14057	if (exit)
		14058	perf_event_task(task, ctx, 0);
13787		14059
13788	list_for_each_entry_safe(child_event, next, &child_ctx->event_list, event_entry)	14060	list_for_each_entry_safe(child_event, next, &ctx->event_list, event_entry)
13789	perf_event_exit_event(child_event, child_ctx);	14061	perf_event_exit_event(child_event, ctx, false);
13790		14062
13791	mutex_unlock(&child_ctx->mutex);	14063	mutex_unlock(&ctx->mutex);
13792		14064
13793	put_ctx(child_ctx);	14065	if (!exit) {
		14066	/*
		14067	* perf_event_release_kernel() could still have a reference on
		14068	* this context. In that case we must wait for these events to
		14069	* have been freed (in particular all their references to this
		14070	* task must've been dropped).
		14071	*
		14072	* Without this copy_process() will unconditionally free this
		14073	* task (irrespective of its reference count) and
		14074	* _free_event()'s put_task_struct(event->hw.target) will be a
		14075	* use-after-free.
		14076	*
		14077	* Wait for all events to drop their context reference.
		14078	*/
		14079	wait_var_event(&ctx->refcount,
		14080	refcount_read(&ctx->refcount) == 1);
		14081	}
		14082	put_ctx(ctx);
13794	}	14083	}
13795		14084
13796	/*	14085	/*
13797	* When a child task exits, feed back event values to parent events.	14086	* When a task exits, feed back event values to parent events.
13798	*	14087	*
13799	* Can be called with exec_update_lock held when called from	14088	* Can be called with exec_update_lock held when called from
13800	* setup_new_exec().	14089	* setup_new_exec().
13801	*/	14090	*/
13802	void perf_event_exit_task(struct task_struct *child)	14091	void perf_event_exit_task(struct task_struct *task)
13803	{	14092	{
13804	struct perf_event event, tmp;	14093	struct perf_event event, tmp;
13805		14094
13806	mutex_lock(&child->perf_event_mutex);	14095	WARN_ON_ONCE(task != current);
13807	list_for_each_entry_safe(event, tmp, &child->perf_event_list,	14096
		14097	mutex_lock(&task->perf_event_mutex);
		14098	list_for_each_entry_safe(event, tmp, &task->perf_event_list,
13808	owner_entry) {	14099	owner_entry) {
13809	list_del_init(&event->owner_entry);	14100	list_del_init(&event->owner_entry);
13810		14101
@@ -13815,42 +14106,23 @@ void perf_event_exit_task(struct task_struct *child) · 4 unmodified lines
13815	*/	14106	*/
13816	smp_store_release(&event->owner, NULL);	14107	smp_store_release(&event->owner, NULL);
13817	}	14108	}
13818	mutex_unlock(&child->perf_event_mutex);	14109	mutex_unlock(&task->perf_event_mutex);
13819		14110
13820	perf_event_exit_task_context(child);	14111	perf_event_exit_task_context(task, true);
13821		14112
13822	/*	14113	/*
13823	* The perf_event_exit_task_context calls perf_event_task	14114	* The perf_event_exit_task_context calls perf_event_task
13824	* with child's task_ctx, which generates EXIT events for	14115	* with task's task_ctx, which generates EXIT events for
13825	* child contexts and sets child->perf_event_ctxp[] to NULL.	14116	* task contexts and sets task->perf_event_ctxp[] to NULL.
13826	* At this point we need to send EXIT events to cpu contexts.	14117	* At this point we need to send EXIT events to cpu contexts.
13827	*/	14118	*/
13828	perf_event_task(child, NULL, 0);	14119	perf_event_task(task, NULL, 0);
13829		14120
13830	/*	14121	/*
13831	* Detach the perf_ctx_data for the system-wide event.	14122	* Detach the perf_ctx_data for the system-wide event.
13832	*/	14123	*/
13833	guard(percpu_read)(&global_ctx_data_rwsem);	14124	guard(percpu_read)(&global_ctx_data_rwsem);
13834	detach_task_ctx_data(child);	14125	detach_task_ctx_data(task);
13835	}
13836
13837	static void perf_free_event(struct perf_event *event,
13838	struct perf_event_context *ctx)
13839	{
13840	struct perf_event *parent = event->parent;
13841
13842	if (WARN_ON_ONCE(!parent))
13843	return;
13844
13845	mutex_lock(&parent->child_mutex);
13846	list_del_init(&event->child_list);
13847	mutex_unlock(&parent->child_mutex);
13848
13849	raw_spin_lock_irq(&ctx->lock);
13850	perf_group_detach(event);
13851	list_del_event(event, ctx);
13852	raw_spin_unlock_irq(&ctx->lock);
13853	put_event(event);
13854	}	14126	}
13855		14127
13856	/*	14128	/*
@@ -13862,48 +14134,7 @@ static void perf_free_event(struct perf_event *event, · 5 unmodified lines
13862	*/	14134	*/
13863	void perf_event_free_task(struct task_struct *task)	14135	void perf_event_free_task(struct task_struct *task)
13864	{	14136	{
13865	struct perf_event_context *ctx;	14137	perf_event_exit_task_context(task, false);
13866	struct perf_event event, tmp;
13867
13868	ctx = rcu_access_pointer(task->perf_event_ctxp);
13869	if (!ctx)
13870	return;
13871
13872	mutex_lock(&ctx->mutex);
13873	raw_spin_lock_irq(&ctx->lock);
13874	/*
13875	* Destroy the task <-> ctx relation and mark the context dead.
13876	*
13877	* This is important because even though the task hasn't been
13878	* exposed yet the context has been (through child_list).
13879	*/
13880	RCU_INIT_POINTER(task->perf_event_ctxp, NULL);
13881	WRITE_ONCE(ctx->task, TASK_TOMBSTONE);
13882	put_task_struct(task); /* cannot be last */
13883	raw_spin_unlock_irq(&ctx->lock);
13884
13885
13886	list_for_each_entry_safe(event, tmp, &ctx->event_list, event_entry)
13887	perf_free_event(event, ctx);
13888
13889	mutex_unlock(&ctx->mutex);
13890
13891	/*
13892	* perf_event_release_kernel() could've stolen some of our
13893	* child events and still have them on its free_list. In that
13894	* case we must wait for these events to have been freed (in
13895	* particular all their references to this task must've been
13896	* dropped).
13897	*
13898	* Without this copy_process() will unconditionally free this
13899	* task (irrespective of its reference count) and
13900	* _free_event()'s put_task_struct(event->hw.target) will be a
13901	* use-after-free.
13902	*
13903	* Wait for all events to drop their context reference.
13904	*/
13905	wait_var_event(&ctx->refcount, refcount_read(&ctx->refcount) == 1);
13906	put_ctx(ctx); /* must be last */
13907	}	14138	}
13908		14139
13909	void perf_event_delayed_put(struct task_struct *task)	14140	void perf_event_delayed_put(struct task_struct *task)
@@ -13980,6 +14211,14 @@ inherit_event(struct perf_event *parent_event, · 70 unmodified lines
13980	if (parent_event->parent)	14211	if (parent_event->parent)
13981	parent_event = parent_event->parent;	14212	parent_event = parent_event->parent;
13982		14213
		14214	if (parent_event->state <= PERF_EVENT_STATE_REVOKED)
		14215	return NULL;
		14216
		14217	/*
		14218	* Event creation should be under SRCU, see perf_pmu_unregister().
		14219	*/
		14220	guard(srcu)(&pmus_srcu);
		14221
13983	child_event = perf_event_alloc(&parent_event->attr,	14222	child_event = perf_event_alloc(&parent_event->attr,
13984	parent_event->cpu,	14223	parent_event->cpu,
13985	child,	14224	child,

kernel/events/ring_buffer.c+20−9

 {
 	bool overwrite = !(flags & RING_BUFFER_WRITABLE);
 	int node = (event->cpu == -1) ? -1 : cpu_to_node(event->cpu);
-	int ret = -ENOMEM, max_order;
+	bool use_contiguous_pages = event->pmu->capabilities & (
+		PERF_PMU_CAP_AUX_NO_SG | PERF_PMU_CAP_AUX_PREFER_LARGE);
+	/*
+	 * Initialize max_order to 0 for page allocation. This allocates single
+	 * pages to minimize memory fragmentation. This is overridden if the
+	 * PMU needs or prefers contiguous pages (use_contiguous_pages = true).
+	 */
+	int max_order = 0;
+	int ret = -ENOMEM;
 	if (!has_aux(event))
 		return -EOPNOTSUPP;
 	if (!overwrite) {
 		/*
-		 * Watermark defaults to half the buffer, and so does the
+		 * Watermark defaults to half the buffer, to aid PMU drivers
-		 * max_order, to aid PMU drivers in double buffering.
+		 * in double buffering.
 		 */
 		if (!watermark)
 			watermark = min_t(unsigned long,
 					  (unsigned long)nr_pages << (PAGE_SHIFT - 1));
 		/*
-		 * Use aux_watermark as the basis for chunking to
+		 * If using contiguous pages, use aux_watermark as the basis
-		 * help PMU drivers honor the watermark.
+		 * for chunking to help PMU drivers honor the watermark.
 		 */
-		max_order = get_order(watermark);
+		if (use_contiguous_pages)
+			max_order = get_order(watermark);
 	} else {
 		/*
-		 * We need to start with the max_order that fits in nr_pages,
+		 * If using contiguous pages, we need to start with the
-		 * not the other way around, hence ilog2() and not get_order.
+		 * max_order that fits in nr_pages, not the other way around,
+		 * hence ilog2() and not get_order.
 		 */
-		max_order = ilog2(nr_pages);
+		if (use_contiguous_pages)
+			max_order = ilog2(nr_pages);
 		watermark = 0;
 	}

tools/include/uapi/linux/perf_event.h+334−323

@@ -39,18 +39,21 @@ enum perf_type_id { · 38 unmodified lines
39		39
40	/*	40	/*
41	* attr.config layout for type PERF_TYPE_HARDWARE and PERF_TYPE_HW_CACHE	41	* attr.config layout for type PERF_TYPE_HARDWARE and PERF_TYPE_HW_CACHE
		42	*
42	* PERF_TYPE_HARDWARE: 0xEEEEEEEE000000AA	43	* PERF_TYPE_HARDWARE: 0xEEEEEEEE000000AA
43	* AA: hardware event ID	44	* AA: hardware event ID
44	* EEEEEEEE: PMU type ID	45	* EEEEEEEE: PMU type ID
		46	*
45	* PERF_TYPE_HW_CACHE: 0xEEEEEEEE00DDCCBB	47	* PERF_TYPE_HW_CACHE: 0xEEEEEEEE00DDCCBB
46	* BB: hardware cache ID	48	* BB: hardware cache ID
47	* CC: hardware cache op ID	49	* CC: hardware cache op ID
48	* DD: hardware cache op result ID	50	* DD: hardware cache op result ID
49	* EEEEEEEE: PMU type ID	51	* EEEEEEEE: PMU type ID
50	* If the PMU type ID is 0, the PERF_TYPE_RAW will be applied.	52	*
		53	* If the PMU type ID is 0, PERF_TYPE_RAW will be applied.
51	*/	54	*/
52	#define PERF_PMU_TYPE_SHIFT 32	55	#define PERF_PMU_TYPE_SHIFT 32
53	#define PERF_HW_EVENT_MASK 0xffffffff	56	#define PERF_HW_EVENT_MASK 0xffffffff
54		57
55	/*	58	/*
56	* Generalized performance event event_id types, used by the	59	* Generalized performance event event_id types, used by the
@@ -112,7 +115,7 @@ enum perf_hw_cache_op_result_id { · 55 unmodified lines
112	/*	115	/*
113	* Special "software" events provided by the kernel, even if the hardware	116	* Special "software" events provided by the kernel, even if the hardware
114	* does not support performance events. These events measure various	117	* does not support performance events. These events measure various
115	* physical and sw events of the kernel (and allow the profiling of them as	118	* physical and SW events of the kernel (and allow the profiling of them as
116	* well):	119	* well):
117	*/	120	*/
118	enum perf_sw_ids {	121	enum perf_sw_ids {
@@ -167,8 +170,9 @@ enum perf_event_sample_format { · 48 unmodified lines
167	};	170	};
168		171
169	#define PERF_SAMPLE_WEIGHT_TYPE (PERF_SAMPLE_WEIGHT \| PERF_SAMPLE_WEIGHT_STRUCT)	172	#define PERF_SAMPLE_WEIGHT_TYPE (PERF_SAMPLE_WEIGHT \| PERF_SAMPLE_WEIGHT_STRUCT)
		173
170	/*	174	/*
171	* values to program into branch_sample_type when PERF_SAMPLE_BRANCH is set	175	* Values to program into branch_sample_type when PERF_SAMPLE_BRANCH is set.
172	*	176	*
173	* If the user does not pass priv level information via branch_sample_type,	177	* If the user does not pass priv level information via branch_sample_type,
174	* the kernel uses the event's priv level. Branch and event priv levels do	178	* the kernel uses the event's priv level. Branch and event priv levels do
@@ -178,20 +182,20 @@ enum perf_event_sample_format { · 3 unmodified lines
178	* of branches and therefore it supersedes all the other types.	182	* of branches and therefore it supersedes all the other types.
179	*/	183	*/
180	enum perf_branch_sample_type_shift {	184	enum perf_branch_sample_type_shift {
181	PERF_SAMPLE_BRANCH_USER_SHIFT = 0, /* user branches */	185	PERF_SAMPLE_BRANCH_USER_SHIFT = 0, /* user branches */
182	PERF_SAMPLE_BRANCH_KERNEL_SHIFT = 1, /* kernel branches */	186	PERF_SAMPLE_BRANCH_KERNEL_SHIFT = 1, /* kernel branches */
183	PERF_SAMPLE_BRANCH_HV_SHIFT = 2, /* hypervisor branches */	187	PERF_SAMPLE_BRANCH_HV_SHIFT = 2, /* hypervisor branches */
184		188
185	PERF_SAMPLE_BRANCH_ANY_SHIFT = 3, /* any branch types */	189	PERF_SAMPLE_BRANCH_ANY_SHIFT = 3, /* any branch types */
186	PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT = 4, /* any call branch */	190	PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT = 4, /* any call branch */
187	PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT = 5, /* any return branch */	191	PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT = 5, /* any return branch */
188	PERF_SAMPLE_BRANCH_IND_CALL_SHIFT = 6, /* indirect calls */	192	PERF_SAMPLE_BRANCH_IND_CALL_SHIFT = 6, /* indirect calls */
189	PERF_SAMPLE_BRANCH_ABORT_TX_SHIFT = 7, /* transaction aborts */	193	PERF_SAMPLE_BRANCH_ABORT_TX_SHIFT = 7, /* transaction aborts */
190	PERF_SAMPLE_BRANCH_IN_TX_SHIFT = 8, /* in transaction */	194	PERF_SAMPLE_BRANCH_IN_TX_SHIFT = 8, /* in transaction */
191	PERF_SAMPLE_BRANCH_NO_TX_SHIFT = 9, /* not in transaction */	195	PERF_SAMPLE_BRANCH_NO_TX_SHIFT = 9, /* not in transaction */
192	PERF_SAMPLE_BRANCH_COND_SHIFT = 10, /* conditional branches */	196	PERF_SAMPLE_BRANCH_COND_SHIFT = 10, /* conditional branches */
193		197
194	PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT = 11, /* call/ret stack */	198	PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT = 11, /* CALL/RET stack */
195	PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT = 12, /* indirect jumps */	199	PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT = 12, /* indirect jumps */
196	PERF_SAMPLE_BRANCH_CALL_SHIFT = 13, /* direct call */	200	PERF_SAMPLE_BRANCH_CALL_SHIFT = 13, /* direct call */
197		201
@@ -210,96 +214,95 @@ enum perf_branch_sample_type_shift { · 12 unmodified lines
210	};	214	};
211		215
212	enum perf_branch_sample_type {	216	enum perf_branch_sample_type {
213	PERF_SAMPLE_BRANCH_USER = 1U << PERF_SAMPLE_BRANCH_USER_SHIFT,	217	PERF_SAMPLE_BRANCH_USER = 1U << PERF_SAMPLE_BRANCH_USER_SHIFT,
214	PERF_SAMPLE_BRANCH_KERNEL = 1U << PERF_SAMPLE_BRANCH_KERNEL_SHIFT,	218	PERF_SAMPLE_BRANCH_KERNEL = 1U << PERF_SAMPLE_BRANCH_KERNEL_SHIFT,
215	PERF_SAMPLE_BRANCH_HV = 1U << PERF_SAMPLE_BRANCH_HV_SHIFT,	219	PERF_SAMPLE_BRANCH_HV = 1U << PERF_SAMPLE_BRANCH_HV_SHIFT,
216		220
217	PERF_SAMPLE_BRANCH_ANY = 1U << PERF_SAMPLE_BRANCH_ANY_SHIFT,	221	PERF_SAMPLE_BRANCH_ANY = 1U << PERF_SAMPLE_BRANCH_ANY_SHIFT,
218	PERF_SAMPLE_BRANCH_ANY_CALL = 1U << PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT,	222	PERF_SAMPLE_BRANCH_ANY_CALL = 1U << PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT,
219	PERF_SAMPLE_BRANCH_ANY_RETURN = 1U << PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT,	223	PERF_SAMPLE_BRANCH_ANY_RETURN = 1U << PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT,
220	PERF_SAMPLE_BRANCH_IND_CALL = 1U << PERF_SAMPLE_BRANCH_IND_CALL_SHIFT,	224	PERF_SAMPLE_BRANCH_IND_CALL = 1U << PERF_SAMPLE_BRANCH_IND_CALL_SHIFT,
221	PERF_SAMPLE_BRANCH_ABORT_TX = 1U << PERF_SAMPLE_BRANCH_ABORT_TX_SHIFT,	225	PERF_SAMPLE_BRANCH_ABORT_TX = 1U << PERF_SAMPLE_BRANCH_ABORT_TX_SHIFT,
222	PERF_SAMPLE_BRANCH_IN_TX = 1U << PERF_SAMPLE_BRANCH_IN_TX_SHIFT,	226	PERF_SAMPLE_BRANCH_IN_TX = 1U << PERF_SAMPLE_BRANCH_IN_TX_SHIFT,
223	PERF_SAMPLE_BRANCH_NO_TX = 1U << PERF_SAMPLE_BRANCH_NO_TX_SHIFT,	227	PERF_SAMPLE_BRANCH_NO_TX = 1U << PERF_SAMPLE_BRANCH_NO_TX_SHIFT,
224	PERF_SAMPLE_BRANCH_COND = 1U << PERF_SAMPLE_BRANCH_COND_SHIFT,	228	PERF_SAMPLE_BRANCH_COND = 1U << PERF_SAMPLE_BRANCH_COND_SHIFT,
225		229
226	PERF_SAMPLE_BRANCH_CALL_STACK = 1U << PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT,	230	PERF_SAMPLE_BRANCH_CALL_STACK = 1U << PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT,
227	PERF_SAMPLE_BRANCH_IND_JUMP = 1U << PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT,	231	PERF_SAMPLE_BRANCH_IND_JUMP = 1U << PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT,
228	PERF_SAMPLE_BRANCH_CALL = 1U << PERF_SAMPLE_BRANCH_CALL_SHIFT,	232	PERF_SAMPLE_BRANCH_CALL = 1U << PERF_SAMPLE_BRANCH_CALL_SHIFT,
229		233
230	PERF_SAMPLE_BRANCH_NO_FLAGS = 1U << PERF_SAMPLE_BRANCH_NO_FLAGS_SHIFT,	234	PERF_SAMPLE_BRANCH_NO_FLAGS = 1U << PERF_SAMPLE_BRANCH_NO_FLAGS_SHIFT,
231	PERF_SAMPLE_BRANCH_NO_CYCLES = 1U << PERF_SAMPLE_BRANCH_NO_CYCLES_SHIFT,	235	PERF_SAMPLE_BRANCH_NO_CYCLES = 1U << PERF_SAMPLE_BRANCH_NO_CYCLES_SHIFT,
232		236
233	PERF_SAMPLE_BRANCH_TYPE_SAVE =	237	PERF_SAMPLE_BRANCH_TYPE_SAVE = 1U << PERF_SAMPLE_BRANCH_TYPE_SAVE_SHIFT,
234	1U << PERF_SAMPLE_BRANCH_TYPE_SAVE_SHIFT,
235		238
236	PERF_SAMPLE_BRANCH_HW_INDEX = 1U << PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT,	239	PERF_SAMPLE_BRANCH_HW_INDEX = 1U << PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT,
237		240
238	PERF_SAMPLE_BRANCH_PRIV_SAVE = 1U << PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT,	241	PERF_SAMPLE_BRANCH_PRIV_SAVE = 1U << PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT,
239		242
240	PERF_SAMPLE_BRANCH_COUNTERS = 1U << PERF_SAMPLE_BRANCH_COUNTERS_SHIFT,	243	PERF_SAMPLE_BRANCH_COUNTERS = 1U << PERF_SAMPLE_BRANCH_COUNTERS_SHIFT,
241		244
242	PERF_SAMPLE_BRANCH_MAX = 1U << PERF_SAMPLE_BRANCH_MAX_SHIFT,	245	PERF_SAMPLE_BRANCH_MAX = 1U << PERF_SAMPLE_BRANCH_MAX_SHIFT,
243	};	246	};
244		247
245	/*	248	/*
246	* Common flow change classification	249	* Common control flow change classifications:
247	*/	250	*/
248	enum {	251	enum {
249	PERF_BR_UNKNOWN = 0, /* unknown */	252	PERF_BR_UNKNOWN = 0, /* Unknown */
250	PERF_BR_COND = 1, /* conditional */	253	PERF_BR_COND = 1, /* Conditional */
251	PERF_BR_UNCOND = 2, /* unconditional */	254	PERF_BR_UNCOND = 2, /* Unconditional */
252	PERF_BR_IND = 3, /* indirect */	255	PERF_BR_IND = 3, /* Indirect */
253	PERF_BR_CALL = 4, /* function call */	256	PERF_BR_CALL = 4, /* Function call */
254	PERF_BR_IND_CALL = 5, /* indirect function call */	257	PERF_BR_IND_CALL = 5, /* Indirect function call */
255	PERF_BR_RET = 6, /* function return */	258	PERF_BR_RET = 6, /* Function return */
256	PERF_BR_SYSCALL = 7, /* syscall */	259	PERF_BR_SYSCALL = 7, /* Syscall */
257	PERF_BR_SYSRET = 8, /* syscall return */	260	PERF_BR_SYSRET = 8, /* Syscall return */
258	PERF_BR_COND_CALL = 9, /* conditional function call */	261	PERF_BR_COND_CALL = 9, /* Conditional function call */
259	PERF_BR_COND_RET = 10, /* conditional function return */	262	PERF_BR_COND_RET = 10, /* Conditional function return */
260	PERF_BR_ERET = 11, /* exception return */	263	PERF_BR_ERET = 11, /* Exception return */
261	PERF_BR_IRQ = 12, /* irq */	264	PERF_BR_IRQ = 12, /* IRQ */
262	PERF_BR_SERROR = 13, /* system error */	265	PERF_BR_SERROR = 13, /* System error */
263	PERF_BR_NO_TX = 14, /* not in transaction */	266	PERF_BR_NO_TX = 14, /* Not in transaction */
264	PERF_BR_EXTEND_ABI = 15, /* extend ABI */	267	PERF_BR_EXTEND_ABI = 15, /* Extend ABI */
265	PERF_BR_MAX,	268	PERF_BR_MAX,
266	};	269	};
267		270
268	/*	271	/*
269	* Common branch speculation outcome classification	272	* Common branch speculation outcome classifications:
270	*/	273	*/
271	enum {	274	enum {
272	PERF_BR_SPEC_NA = 0, /* Not available */	275	PERF_BR_SPEC_NA = 0, /* Not available */
273	PERF_BR_SPEC_WRONG_PATH = 1, /* Speculative but on wrong path */	276	PERF_BR_SPEC_WRONG_PATH = 1, /* Speculative but on wrong path */
274	PERF_BR_NON_SPEC_CORRECT_PATH = 2, /* Non-speculative but on correct path */	277	PERF_BR_NON_SPEC_CORRECT_PATH = 2, /* Non-speculative but on correct path */
275	PERF_BR_SPEC_CORRECT_PATH = 3, /* Speculative and on correct path */	278	PERF_BR_SPEC_CORRECT_PATH = 3, /* Speculative and on correct path */
276	PERF_BR_SPEC_MAX,	279	PERF_BR_SPEC_MAX,
277	};	280	};
278		281
279	enum {	282	enum {
280	PERF_BR_NEW_FAULT_ALGN = 0, /* Alignment fault */	283	PERF_BR_NEW_FAULT_ALGN = 0, /* Alignment fault */
281	PERF_BR_NEW_FAULT_DATA = 1, /* Data fault */	284	PERF_BR_NEW_FAULT_DATA = 1, /* Data fault */
282	PERF_BR_NEW_FAULT_INST = 2, /* Inst fault */	285	PERF_BR_NEW_FAULT_INST = 2, /* Inst fault */
283	PERF_BR_NEW_ARCH_1 = 3, /* Architecture specific */	286	PERF_BR_NEW_ARCH_1 = 3, /* Architecture specific */
284	PERF_BR_NEW_ARCH_2 = 4, /* Architecture specific */	287	PERF_BR_NEW_ARCH_2 = 4, /* Architecture specific */
285	PERF_BR_NEW_ARCH_3 = 5, /* Architecture specific */	288	PERF_BR_NEW_ARCH_3 = 5, /* Architecture specific */
286	PERF_BR_NEW_ARCH_4 = 6, /* Architecture specific */	289	PERF_BR_NEW_ARCH_4 = 6, /* Architecture specific */
287	PERF_BR_NEW_ARCH_5 = 7, /* Architecture specific */	290	PERF_BR_NEW_ARCH_5 = 7, /* Architecture specific */
288	PERF_BR_NEW_MAX,	291	PERF_BR_NEW_MAX,
289	};	292	};
290		293
291	enum {	294	enum {
292	PERF_BR_PRIV_UNKNOWN = 0,	295	PERF_BR_PRIV_UNKNOWN = 0,
293	PERF_BR_PRIV_USER = 1,	296	PERF_BR_PRIV_USER = 1,
294	PERF_BR_PRIV_KERNEL = 2,	297	PERF_BR_PRIV_KERNEL = 2,
295	PERF_BR_PRIV_HV = 3,	298	PERF_BR_PRIV_HV = 3,
296	};	299	};
297		300
298	#define PERF_BR_ARM64_FIQ PERF_BR_NEW_ARCH_1	301	#define PERF_BR_ARM64_FIQ PERF_BR_NEW_ARCH_1
299	#define PERF_BR_ARM64_DEBUG_HALT PERF_BR_NEW_ARCH_2	302	#define PERF_BR_ARM64_DEBUG_HALT PERF_BR_NEW_ARCH_2
300	#define PERF_BR_ARM64_DEBUG_EXIT PERF_BR_NEW_ARCH_3	303	#define PERF_BR_ARM64_DEBUG_EXIT PERF_BR_NEW_ARCH_3
301	#define PERF_BR_ARM64_DEBUG_INST PERF_BR_NEW_ARCH_4	304	#define PERF_BR_ARM64_DEBUG_INST PERF_BR_NEW_ARCH_4
302	#define PERF_BR_ARM64_DEBUG_DATA PERF_BR_NEW_ARCH_5	305	#define PERF_BR_ARM64_DEBUG_DATA PERF_BR_NEW_ARCH_5
303		306
304	#define PERF_SAMPLE_BRANCH_PLM_ALL \	307	#define PERF_SAMPLE_BRANCH_PLM_ALL \
305	(PERF_SAMPLE_BRANCH_USER\|\	308	(PERF_SAMPLE_BRANCH_USER\|\
@@ -310,9 +313,9 @@ enum { · 4 unmodified lines
310	* Values to determine ABI of the registers dump.	313	* Values to determine ABI of the registers dump.
311	*/	314	*/
312	enum perf_sample_regs_abi {	315	enum perf_sample_regs_abi {
313	PERF_SAMPLE_REGS_ABI_NONE = 0,	316	PERF_SAMPLE_REGS_ABI_NONE = 0,
314	PERF_SAMPLE_REGS_ABI_32 = 1,	317	PERF_SAMPLE_REGS_ABI_32 = 1,
315	PERF_SAMPLE_REGS_ABI_64 = 2,	318	PERF_SAMPLE_REGS_ABI_64 = 2,
316	};	319	};
317		320
318	/*	321	/*
@@ -320,21 +323,21 @@ enum perf_sample_regs_abi { · 1 unmodified line
320	* abort events. Multiple bits can be set.	323	* abort events. Multiple bits can be set.
321	*/	324	*/
322	enum {	325	enum {
323	PERF_TXN_ELISION = (1 << 0), /* From elision */	326	PERF_TXN_ELISION = (1 << 0), /* From elision */
324	PERF_TXN_TRANSACTION = (1 << 1), /* From transaction */	327	PERF_TXN_TRANSACTION = (1 << 1), /* From transaction */
325	PERF_TXN_SYNC = (1 << 2), /* Instruction is related */	328	PERF_TXN_SYNC = (1 << 2), /* Instruction is related */
326	PERF_TXN_ASYNC = (1 << 3), /* Instruction not related */	329	PERF_TXN_ASYNC = (1 << 3), /* Instruction is not related */
327	PERF_TXN_RETRY = (1 << 4), /* Retry possible */	330	PERF_TXN_RETRY = (1 << 4), /* Retry possible */
328	PERF_TXN_CONFLICT = (1 << 5), /* Conflict abort */	331	PERF_TXN_CONFLICT = (1 << 5), /* Conflict abort */
329	PERF_TXN_CAPACITY_WRITE = (1 << 6), /* Capacity write abort */	332	PERF_TXN_CAPACITY_WRITE = (1 << 6), /* Capacity write abort */
330	PERF_TXN_CAPACITY_READ = (1 << 7), /* Capacity read abort */	333	PERF_TXN_CAPACITY_READ = (1 << 7), /* Capacity read abort */
331		334
332	PERF_TXN_MAX = (1 << 8), /* non-ABI */	335	PERF_TXN_MAX = (1 << 8), /* non-ABI */
333		336
334	/* bits 32..63 are reserved for the abort code */	337	/* Bits 32..63 are reserved for the abort code */
335		338
336	PERF_TXN_ABORT_MASK = (0xffffffffULL << 32),	339	PERF_TXN_ABORT_MASK = (0xffffffffULL << 32),
337	PERF_TXN_ABORT_SHIFT = 32,	340	PERF_TXN_ABORT_SHIFT = 32,
338	};	341	};
339		342
340	/*	343	/*
@@ -369,24 +372,22 @@ enum perf_event_read_format { · 28 unmodified lines
369	PERF_FORMAT_MAX = 1U << 5, /* non-ABI */	372	PERF_FORMAT_MAX = 1U << 5, /* non-ABI */
370	};	373	};
371		374
372	#define PERF_ATTR_SIZE_VER0 64 /* sizeof first published struct */	375	#define PERF_ATTR_SIZE_VER0 64 /* Size of first published 'struct perf_event_attr' */
373	#define PERF_ATTR_SIZE_VER1 72 /* add: config2 */	376	#define PERF_ATTR_SIZE_VER1 72 /* Add: config2 */
374	#define PERF_ATTR_SIZE_VER2 80 /* add: branch_sample_type */	377	#define PERF_ATTR_SIZE_VER2 80 /* Add: branch_sample_type */
375	#define PERF_ATTR_SIZE_VER3 96 /* add: sample_regs_user */	378	#define PERF_ATTR_SIZE_VER3 96 /* Add: sample_regs_user */
376	/* add: sample_stack_user */	379	/* Add: sample_stack_user */
377	#define PERF_ATTR_SIZE_VER4 104 /* add: sample_regs_intr */	380	#define PERF_ATTR_SIZE_VER4 104 /* Add: sample_regs_intr */
378	#define PERF_ATTR_SIZE_VER5 112 /* add: aux_watermark */	381	#define PERF_ATTR_SIZE_VER5 112 /* Add: aux_watermark */
379	#define PERF_ATTR_SIZE_VER6 120 /* add: aux_sample_size */	382	#define PERF_ATTR_SIZE_VER6 120 /* Add: aux_sample_size */
380	#define PERF_ATTR_SIZE_VER7 128 /* add: sig_data */	383	#define PERF_ATTR_SIZE_VER7 128 /* Add: sig_data */
381	#define PERF_ATTR_SIZE_VER8 136 /* add: config3 */	384	#define PERF_ATTR_SIZE_VER8 136 /* Add: config3 */
382		385
383	/*	386	/*
384	* Hardware event_id to monitor via a performance monitoring event:	387	* 'struct perf_event_attr' contains various attributes that define
385	*	388	* a performance event - most of them hardware related configuration
386	* @sample_max_stack: Max number of frame pointers in a callchain,	389	* details, but also a lot of behavioral switches and values implemented
387	* should be < /proc/sys/kernel/perf_event_max_stack	390	* by the kernel.
388	* Max number of entries of branch stack
389	* should be < hardware limit
390	*/	391	*/
391	struct perf_event_attr {	392	struct perf_event_attr {
392		393
@@ -396,7 +397,7 @@ struct perf_event_attr { · 3 unmodified lines
396	__u32 type;	397	__u32 type;
397		398
398	/*	399	/*
399	* Size of the attr structure, for fwd/bwd compat.	400	* Size of the attr structure, for forward/backwards compatibility.
400	*/	401	*/
401	__u32 size;	402	__u32 size;
402		403
@@ -451,21 +452,21 @@ struct perf_event_attr { · 48 unmodified lines
451	comm_exec : 1, /* flag comm events that are due to an exec */	452	comm_exec : 1, /* flag comm events that are due to an exec */
452	use_clockid : 1, /* use @clockid for time fields */	453	use_clockid : 1, /* use @clockid for time fields */
453	context_switch : 1, /* context switch data */	454	context_switch : 1, /* context switch data */
454	write_backward : 1, /* Write ring buffer from end to beginning */	455	write_backward : 1, /* write ring buffer from end to beginning */
455	namespaces : 1, /* include namespaces data */	456	namespaces : 1, /* include namespaces data */
456	ksymbol : 1, /* include ksymbol events */	457	ksymbol : 1, /* include ksymbol events */
457	bpf_event : 1, /* include bpf events */	458	bpf_event : 1, /* include BPF events */
458	aux_output : 1, /* generate AUX records instead of events */	459	aux_output : 1, /* generate AUX records instead of events */
459	cgroup : 1, /* include cgroup events */	460	cgroup : 1, /* include cgroup events */
460	text_poke : 1, /* include text poke events */	461	text_poke : 1, /* include text poke events */
461	build_id : 1, /* use build id in mmap2 events */	462	build_id : 1, /* use build ID in mmap2 events */
462	inherit_thread : 1, /* children only inherit if cloned with CLONE_THREAD */	463	inherit_thread : 1, /* children only inherit if cloned with CLONE_THREAD */
463	remove_on_exec : 1, /* event is removed from task on exec */	464	remove_on_exec : 1, /* event is removed from task on exec */
464	sigtrap : 1, /* send synchronous SIGTRAP on event */	465	sigtrap : 1, /* send synchronous SIGTRAP on event */
465	__reserved_1 : 26;	466	__reserved_1 : 26;
466		467
467	union {	468	union {
468	__u32 wakeup_events; /* wakeup every n events */	469	__u32 wakeup_events; /* wake up every n events */
469	__u32 wakeup_watermark; /* bytes before wakeup */	470	__u32 wakeup_watermark; /* bytes before wakeup */
470	};	471	};
471		472
@@ -474,13 +475,13 @@ struct perf_event_attr { · 2 unmodified lines
474	__u64 bp_addr;	475	__u64 bp_addr;
475	__u64 kprobe_func; /* for perf_kprobe */	476	__u64 kprobe_func; /* for perf_kprobe */
476	__u64 uprobe_path; /* for perf_uprobe */	477	__u64 uprobe_path; /* for perf_uprobe */
477	__u64 config1; /* extension of config */	478	__u64 config1; /* extension of config */
478	};	479	};
479	union {	480	union {
480	__u64 bp_len;	481	__u64 bp_len;
481	__u64 kprobe_addr; /* when kprobe_func == NULL */	482	__u64 kprobe_addr; /* when kprobe_func == NULL */
482	__u64 probe_offset; /* for perf_[k,u]probe */	483	__u64 probe_offset; /* for perf_[k,u]probe */
483	__u64 config2; /* extension of config1 */	484	__u64 config2; /* extension of config1 */
484	};	485	};
485	__u64 branch_sample_type; /* enum perf_branch_sample_type */	486	__u64 branch_sample_type; /* enum perf_branch_sample_type */
486		487
@@ -510,7 +511,16 @@ struct perf_event_attr { · 23 unmodified lines
510	* Wakeup watermark for AUX area	511	* Wakeup watermark for AUX area
511	*/	512	*/
512	__u32 aux_watermark;	513	__u32 aux_watermark;
		514
		515	/*
		516	* Max number of frame pointers in a callchain, should be
		517	* lower than /proc/sys/kernel/perf_event_max_stack.
		518	*
		519	* Max number of entries of branch stack should be lower
		520	* than the hardware limit.
		521	*/
513	__u16 sample_max_stack;	522	__u16 sample_max_stack;
		523
514	__u16 __reserved_2;	524	__u16 __reserved_2;
515	__u32 aux_sample_size;	525	__u32 aux_sample_size;
516		526
@@ -537,7 +547,7 @@ struct perf_event_attr { · 20 unmodified lines
537		547
538	/*	548	/*
539	* Structure used by below PERF_EVENT_IOC_QUERY_BPF command	549	* Structure used by below PERF_EVENT_IOC_QUERY_BPF command
540	* to query bpf programs attached to the same perf tracepoint	550	* to query BPF programs attached to the same perf tracepoint
541	* as the given perf event.	551	* as the given perf event.
542	*/	552	*/
543	struct perf_event_query_bpf {	553	struct perf_event_query_bpf {
@@ -559,21 +569,21 @@ struct perf_event_query_bpf { · 15 unmodified lines
559	/*	569	/*
560	* Ioctls that can be done on a perf event fd:	570	* Ioctls that can be done on a perf event fd:
561	*/	571	*/
562	#define PERF_EVENT_IOC_ENABLE _IO ('$', 0)	572	#define PERF_EVENT_IOC_ENABLE _IO ('$', 0)
563	#define PERF_EVENT_IOC_DISABLE _IO ('$', 1)	573	#define PERF_EVENT_IOC_DISABLE _IO ('$', 1)
564	#define PERF_EVENT_IOC_REFRESH _IO ('$', 2)	574	#define PERF_EVENT_IOC_REFRESH _IO ('$', 2)
565	#define PERF_EVENT_IOC_RESET _IO ('$', 3)	575	#define PERF_EVENT_IOC_RESET _IO ('$', 3)
566	#define PERF_EVENT_IOC_PERIOD _IOW('$', 4, __u64)	576	#define PERF_EVENT_IOC_PERIOD _IOW ('$', 4, __u64)
567	#define PERF_EVENT_IOC_SET_OUTPUT _IO ('$', 5)	577	#define PERF_EVENT_IOC_SET_OUTPUT _IO ('$', 5)
568	#define PERF_EVENT_IOC_SET_FILTER _IOW('$', 6, char *)	578	#define PERF_EVENT_IOC_SET_FILTER _IOW ('$', 6, char *)
569	#define PERF_EVENT_IOC_ID _IOR('$', 7, __u64 *)	579	#define PERF_EVENT_IOC_ID _IOR ('$', 7, __u64 *)
570	#define PERF_EVENT_IOC_SET_BPF _IOW('$', 8, __u32)	580	#define PERF_EVENT_IOC_SET_BPF _IOW ('$', 8, __u32)
571	#define PERF_EVENT_IOC_PAUSE_OUTPUT _IOW('$', 9, __u32)	581	#define PERF_EVENT_IOC_PAUSE_OUTPUT _IOW ('$', 9, __u32)
572	#define PERF_EVENT_IOC_QUERY_BPF _IOWR('$', 10, struct perf_event_query_bpf *)	582	#define PERF_EVENT_IOC_QUERY_BPF _IOWR('$', 10, struct perf_event_query_bpf *)
573	#define PERF_EVENT_IOC_MODIFY_ATTRIBUTES _IOW('$', 11, struct perf_event_attr *)	583	#define PERF_EVENT_IOC_MODIFY_ATTRIBUTES _IOW ('$', 11, struct perf_event_attr *)
574		584
575	enum perf_event_ioc_flags {	585	enum perf_event_ioc_flags {
576	PERF_IOC_FLAG_GROUP = 1U << 0,	586	PERF_IOC_FLAG_GROUP = 1U << 0,
577	};	587	};
578		588
579	/*	589	/*
@@ -584,7 +594,7 @@ struct perf_event_mmap_page { · 4 unmodified lines
584	__u32 compat_version; /* lowest version this is compat with */	594	__u32 compat_version; /* lowest version this is compat with */
585		595
586	/*	596	/*
587	* Bits needed to read the hw events in user-space.	597	* Bits needed to read the HW events in user-space.
588	*	598	*
589	* u32 seq, time_mult, time_shift, index, width;	599	* u32 seq, time_mult, time_shift, index, width;
590	* u64 count, enabled, running;	600	* u64 count, enabled, running;
@@ -622,7 +632,7 @@ struct perf_event_mmap_page { · 31 unmodified lines
622	__u32 index; /* hardware event identifier */	632	__u32 index; /* hardware event identifier */
623	__s64 offset; /* add to hardware event value */	633	__s64 offset; /* add to hardware event value */
624	__u64 time_enabled; /* time event active */	634	__u64 time_enabled; /* time event active */
625	__u64 time_running; /* time event on cpu */	635	__u64 time_running; /* time event on CPU */
626	union {	636	union {
627	__u64 capabilities;	637	__u64 capabilities;
628	struct {	638	struct {
@@ -650,7 +660,7 @@ struct perf_event_mmap_page { · 21 unmodified lines
650		660
651	/*	661	/*
652	* If cap_usr_time the below fields can be used to compute the time	662	* If cap_usr_time the below fields can be used to compute the time
653	* delta since time_enabled (in ns) using rdtsc or similar.	663	* delta since time_enabled (in ns) using RDTSC or similar.
654	*	664	*
655	* u64 quot, rem;	665	* u64 quot, rem;
656	* u64 delta;	666	* u64 delta;
@@ -723,7 +733,7 @@ struct perf_event_mmap_page { · 66 unmodified lines
723	* after reading this value.	733	* after reading this value.
724	*	734	*
725	* When the mapping is PROT_WRITE the @data_tail value should be	735	* When the mapping is PROT_WRITE the @data_tail value should be
726	* written by userspace to reflect the last read data, after issueing	736	* written by user-space to reflect the last read data, after issuing
727	* an smp_mb() to separate the data read from the ->data_tail store.	737	* an smp_mb() to separate the data read from the ->data_tail store.
728	* In this case the kernel will not over-write unread data.	738	* In this case the kernel will not over-write unread data.
729	*	739	*
@@ -739,7 +749,7 @@ struct perf_event_mmap_page { · 9 unmodified lines
739		749
740	/*	750	/*
741	* AUX area is defined by aux_{offset,size} fields that should be set	751	* AUX area is defined by aux_{offset,size} fields that should be set
742	* by the userspace, so that	752	* by the user-space, so that
743	*	753	*
744	* aux_offset >= data_offset + data_size	754	* aux_offset >= data_offset + data_size
745	*	755	*
@@ -813,7 +823,7 @@ struct perf_event_mmap_page { · 67 unmodified lines
813	* Indicates that thread was preempted in TASK_RUNNING state.	823	* Indicates that thread was preempted in TASK_RUNNING state.
814	*	824	*
815	* PERF_RECORD_MISC_MMAP_BUILD_ID:	825	* PERF_RECORD_MISC_MMAP_BUILD_ID:
816	* Indicates that mmap2 event carries build id data.	826	* Indicates that mmap2 event carries build ID data.
817	*/	827	*/
818	#define PERF_RECORD_MISC_EXACT_IP (1 << 14)	828	#define PERF_RECORD_MISC_EXACT_IP (1 << 14)
819	#define PERF_RECORD_MISC_SWITCH_OUT_PREEMPT (1 << 14)	829	#define PERF_RECORD_MISC_SWITCH_OUT_PREEMPT (1 << 14)
@@ -824,26 +834,26 @@ struct perf_event_mmap_page { · 4 unmodified lines
824	#define PERF_RECORD_MISC_EXT_RESERVED (1 << 15)	834	#define PERF_RECORD_MISC_EXT_RESERVED (1 << 15)
825		835
826	struct perf_event_header {	836	struct perf_event_header {
827	__u32 type;	837	__u32 type;
828	__u16 misc;	838	__u16 misc;
829	__u16 size;	839	__u16 size;
830	};	840	};
831		841
832	struct perf_ns_link_info {	842	struct perf_ns_link_info {
833	__u64 dev;	843	__u64 dev;
834	__u64 ino;	844	__u64 ino;
835	};	845	};
836		846
837	enum {	847	enum {
838	NET_NS_INDEX = 0,	848	NET_NS_INDEX = 0,
839	UTS_NS_INDEX = 1,	849	UTS_NS_INDEX = 1,
840	IPC_NS_INDEX = 2,	850	IPC_NS_INDEX = 2,
841	PID_NS_INDEX = 3,	851	PID_NS_INDEX = 3,
842	USER_NS_INDEX = 4,	852	USER_NS_INDEX = 4,
843	MNT_NS_INDEX = 5,	853	MNT_NS_INDEX = 5,
844	CGROUP_NS_INDEX = 6,	854	CGROUP_NS_INDEX = 6,
845		855
846	NR_NAMESPACES, /* number of available namespaces */	856	NR_NAMESPACES, /* number of available namespaces */
847	};	857	};
848		858
849	enum perf_event_type {	859	enum perf_event_type {
@@ -859,11 +869,11 @@ enum perf_event_type { · 9 unmodified lines
859	* optional fields being ignored.	869	* optional fields being ignored.
860	*	870	*
861	* struct sample_id {	871	* struct sample_id {
862	* { u32 pid, tid; } && PERF_SAMPLE_TID	872	* { u32 pid, tid; } && PERF_SAMPLE_TID
863	* { u64 time; } && PERF_SAMPLE_TIME	873	* { u64 time; } && PERF_SAMPLE_TIME
864	* { u64 id; } && PERF_SAMPLE_ID	874	* { u64 id; } && PERF_SAMPLE_ID
865	* { u64 stream_id;} && PERF_SAMPLE_STREAM_ID	875	* { u64 stream_id;} && PERF_SAMPLE_STREAM_ID
866	* { u32 cpu, res; } && PERF_SAMPLE_CPU	876	* { u32 cpu, res; } && PERF_SAMPLE_CPU
867	* { u64 id; } && PERF_SAMPLE_IDENTIFIER	877	* { u64 id; } && PERF_SAMPLE_IDENTIFIER
868	* } && perf_event_attr::sample_id_all	878	* } && perf_event_attr::sample_id_all
869	*	879	*
@@ -874,7 +884,7 @@ enum perf_event_type { · 4 unmodified lines
874		884
875	/*	885	/*
876	* The MMAP events record the PROT_EXEC mappings so that we can	886	* The MMAP events record the PROT_EXEC mappings so that we can
877	* correlate userspace IPs to code. They have the following structure:	887	* correlate user-space IPs to code. They have the following structure:
878	*	888	*
879	* struct {	889	* struct {
880	* struct perf_event_header header;	890	* struct perf_event_header header;
@@ -884,7 +894,7 @@ enum perf_event_type { · 3 unmodified lines
884	* u64 len;	894	* u64 len;
885	* u64 pgoff;	895	* u64 pgoff;
886	* char filename[];	896	* char filename[];
887	* struct sample_id sample_id;	897	* struct sample_id sample_id;
888	* };	898	* };
889	*/	899	*/
890	PERF_RECORD_MMAP = 1,	900	PERF_RECORD_MMAP = 1,
@@ -894,7 +904,7 @@ enum perf_event_type { · 3 unmodified lines
894	* struct perf_event_header header;	904	* struct perf_event_header header;
895	* u64 id;	905	* u64 id;
896	* u64 lost;	906	* u64 lost;
897	* struct sample_id sample_id;	907	* struct sample_id sample_id;
898	* };	908	* };
899	*/	909	*/
900	PERF_RECORD_LOST = 2,	910	PERF_RECORD_LOST = 2,
@@ -905,7 +915,7 @@ enum perf_event_type { · 4 unmodified lines
905	*	915	*
906	* u32 pid, tid;	916	* u32 pid, tid;
907	* char comm[];	917	* char comm[];
908	* struct sample_id sample_id;	918	* struct sample_id sample_id;
909	* };	919	* };
910	*/	920	*/
911	PERF_RECORD_COMM = 3,	921	PERF_RECORD_COMM = 3,
@@ -916,7 +926,7 @@ enum perf_event_type { · 4 unmodified lines
916	* u32 pid, ppid;	926	* u32 pid, ppid;
917	* u32 tid, ptid;	927	* u32 tid, ptid;
918	* u64 time;	928	* u64 time;
919	* struct sample_id sample_id;	929	* struct sample_id sample_id;
920	* };	930	* };
921	*/	931	*/
922	PERF_RECORD_EXIT = 4,	932	PERF_RECORD_EXIT = 4,
@@ -927,7 +937,7 @@ enum perf_event_type { · 4 unmodified lines
927	* u64 time;	937	* u64 time;
928	* u64 id;	938	* u64 id;
929	* u64 stream_id;	939	* u64 stream_id;
930	* struct sample_id sample_id;	940	* struct sample_id sample_id;
931	* };	941	* };
932	*/	942	*/
933	PERF_RECORD_THROTTLE = 5,	943	PERF_RECORD_THROTTLE = 5,
@@ -939,7 +949,7 @@ enum perf_event_type { · 5 unmodified lines
939	* u32 pid, ppid;	949	* u32 pid, ppid;
940	* u32 tid, ptid;	950	* u32 tid, ptid;
941	* u64 time;	951	* u64 time;
942	* struct sample_id sample_id;	952	* struct sample_id sample_id;
943	* };	953	* };
944	*/	954	*/
945	PERF_RECORD_FORK = 7,	955	PERF_RECORD_FORK = 7,
@@ -950,7 +960,7 @@ enum perf_event_type { · 4 unmodified lines
950	* u32 pid, tid;	960	* u32 pid, tid;
951	*	961	*
952	* struct read_format values;	962	* struct read_format values;
953	* struct sample_id sample_id;	963	* struct sample_id sample_id;
954	* };	964	* };
955	*/	965	*/
956	PERF_RECORD_READ = 8,	966	PERF_RECORD_READ = 8,
@@ -1005,12 +1015,12 @@ enum perf_event_type { · 48 unmodified lines
1005	* { u64 counters; } cntr[nr] && PERF_SAMPLE_BRANCH_COUNTERS	1015	* { u64 counters; } cntr[nr] && PERF_SAMPLE_BRANCH_COUNTERS
1006	* } && PERF_SAMPLE_BRANCH_STACK	1016	* } && PERF_SAMPLE_BRANCH_STACK
1007	*	1017	*
1008	* { u64 abi; # enum perf_sample_regs_abi	1018	* { u64 abi; # enum perf_sample_regs_abi
1009	* u64 regs[weight(mask)]; } && PERF_SAMPLE_REGS_USER	1019	* u64 regs[weight(mask)]; } && PERF_SAMPLE_REGS_USER
1010	*	1020	*
1011	* { u64 size;	1021	* { u64 size;
1012	* char data[size];	1022	* char data[size];
1013	* u64 dyn_size; } && PERF_SAMPLE_STACK_USER	1023	* u64 dyn_size; } && PERF_SAMPLE_STACK_USER
1014	*	1024	*
1015	* { union perf_sample_weight	1025	* { union perf_sample_weight
1016	* {	1026	* {
@@ -1035,10 +1045,11 @@ enum perf_event_type { · 18 unmodified lines
1035	* { u64 abi; # enum perf_sample_regs_abi	1045	* { u64 abi; # enum perf_sample_regs_abi
1036	* u64 regs[weight(mask)]; } && PERF_SAMPLE_REGS_INTR	1046	* u64 regs[weight(mask)]; } && PERF_SAMPLE_REGS_INTR
1037	* { u64 phys_addr;} && PERF_SAMPLE_PHYS_ADDR	1047	* { u64 phys_addr;} && PERF_SAMPLE_PHYS_ADDR
1038	* { u64 size;	1048	* { u64 cgroup;} && PERF_SAMPLE_CGROUP
1039	* char data[size]; } && PERF_SAMPLE_AUX
1040	* { u64 data_page_size;} && PERF_SAMPLE_DATA_PAGE_SIZE	1049	* { u64 data_page_size;} && PERF_SAMPLE_DATA_PAGE_SIZE
1041	* { u64 code_page_size;} && PERF_SAMPLE_CODE_PAGE_SIZE	1050	* { u64 code_page_size;} && PERF_SAMPLE_CODE_PAGE_SIZE
		1051	* { u64 size;
		1052	* char data[size]; } && PERF_SAMPLE_AUX
1042	* };	1053	* };
1043	*/	1054	*/
1044	PERF_RECORD_SAMPLE = 9,	1055	PERF_RECORD_SAMPLE = 9,
@@ -1070,7 +1081,7 @@ enum perf_event_type { · 25 unmodified lines
1070	* };	1081	* };
1071	* u32 prot, flags;	1082	* u32 prot, flags;
1072	* char filename[];	1083	* char filename[];
1073	* struct sample_id sample_id;	1084	* struct sample_id sample_id;
1074	* };	1085	* };
1075	*/	1086	*/
1076	PERF_RECORD_MMAP2 = 10,	1087	PERF_RECORD_MMAP2 = 10,
@@ -1079,12 +1090,12 @@ enum perf_event_type { · 2 unmodified lines
1079	* Records that new data landed in the AUX buffer part.	1090	* Records that new data landed in the AUX buffer part.
1080	*	1091	*
1081	* struct {	1092	* struct {
1082	* struct perf_event_header header;	1093	* struct perf_event_header header;
1083	*	1094	*
1084	* u64 aux_offset;	1095	* u64 aux_offset;
1085	* u64 aux_size;	1096	* u64 aux_size;
1086	* u64 flags;	1097	* u64 flags;
1087	* struct sample_id sample_id;	1098	* struct sample_id sample_id;
1088	* };	1099	* };
1089	*/	1100	*/
1090	PERF_RECORD_AUX = 11,	1101	PERF_RECORD_AUX = 11,
@@ -1167,7 +1178,7 @@ enum perf_event_type { · 76 unmodified lines
1167	PERF_RECORD_KSYMBOL = 17,	1178	PERF_RECORD_KSYMBOL = 17,
1168		1179
1169	/*	1180	/*
1170	* Record bpf events:	1181	* Record BPF events:
1171	* enum perf_bpf_event_type {	1182	* enum perf_bpf_event_type {
1172	* PERF_BPF_EVENT_UNKNOWN = 0,	1183	* PERF_BPF_EVENT_UNKNOWN = 0,
1173	* PERF_BPF_EVENT_PROG_LOAD = 1,	1184	* PERF_BPF_EVENT_PROG_LOAD = 1,
@@ -1245,181 +1256,181 @@ enum perf_record_ksymbol_type { · 71 unmodified lines
1245	#define PERF_RECORD_KSYMBOL_FLAGS_UNREGISTER (1 << 0)	1256	#define PERF_RECORD_KSYMBOL_FLAGS_UNREGISTER (1 << 0)
1246		1257
1247	enum perf_bpf_event_type {	1258	enum perf_bpf_event_type {
1248	PERF_BPF_EVENT_UNKNOWN = 0,	1259	PERF_BPF_EVENT_UNKNOWN = 0,
1249	PERF_BPF_EVENT_PROG_LOAD = 1,	1260	PERF_BPF_EVENT_PROG_LOAD = 1,
1250	PERF_BPF_EVENT_PROG_UNLOAD = 2,	1261	PERF_BPF_EVENT_PROG_UNLOAD = 2,
1251	PERF_BPF_EVENT_MAX, /* non-ABI */	1262	PERF_BPF_EVENT_MAX, /* non-ABI */
1252	};	1263	};
1253		1264
1254	#define PERF_MAX_STACK_DEPTH 127	1265	#define PERF_MAX_STACK_DEPTH 127
1255	#define PERF_MAX_CONTEXTS_PER_STACK 8	1266	#define PERF_MAX_CONTEXTS_PER_STACK 8
1256		1267
1257	enum perf_callchain_context {	1268	enum perf_callchain_context {
1258	PERF_CONTEXT_HV = (__u64)-32,	1269	PERF_CONTEXT_HV = (__u64)-32,
1259	PERF_CONTEXT_KERNEL = (__u64)-128,	1270	PERF_CONTEXT_KERNEL = (__u64)-128,
1260	PERF_CONTEXT_USER = (__u64)-512,	1271	PERF_CONTEXT_USER = (__u64)-512,
1261		1272
1262	PERF_CONTEXT_GUEST = (__u64)-2048,	1273	PERF_CONTEXT_GUEST = (__u64)-2048,
1263	PERF_CONTEXT_GUEST_KERNEL = (__u64)-2176,	1274	PERF_CONTEXT_GUEST_KERNEL = (__u64)-2176,
1264	PERF_CONTEXT_GUEST_USER = (__u64)-2560,	1275	PERF_CONTEXT_GUEST_USER = (__u64)-2560,
1265		1276
1266	PERF_CONTEXT_MAX = (__u64)-4095,	1277	PERF_CONTEXT_MAX = (__u64)-4095,
1267	};	1278	};
1268		1279
1269	/**	1280	/**
1270	* PERF_RECORD_AUX::flags bits	1281	* PERF_RECORD_AUX::flags bits
1271	*/	1282	*/
1272	#define PERF_AUX_FLAG_TRUNCATED 0x01 /* record was truncated to fit */	1283	#define PERF_AUX_FLAG_TRUNCATED 0x0001 /* Record was truncated to fit */
1273	#define PERF_AUX_FLAG_OVERWRITE 0x02 /* snapshot from overwrite mode */	1284	#define PERF_AUX_FLAG_OVERWRITE 0x0002 /* Snapshot from overwrite mode */
1274	#define PERF_AUX_FLAG_PARTIAL 0x04 /* record contains gaps */	1285	#define PERF_AUX_FLAG_PARTIAL 0x0004 /* Record contains gaps */
1275	#define PERF_AUX_FLAG_COLLISION 0x08 /* sample collided with another */	1286	#define PERF_AUX_FLAG_COLLISION 0x0008 /* Sample collided with another */
1276	#define PERF_AUX_FLAG_PMU_FORMAT_TYPE_MASK 0xff00 /* PMU specific trace format type */	1287	#define PERF_AUX_FLAG_PMU_FORMAT_TYPE_MASK 0xff00 /* PMU specific trace format type */
1277		1288
1278	/* CoreSight PMU AUX buffer formats */	1289	/* CoreSight PMU AUX buffer formats */
1279	#define PERF_AUX_FLAG_CORESIGHT_FORMAT_CORESIGHT 0x0000 /* Default for backward compatibility */	1290	#define PERF_AUX_FLAG_CORESIGHT_FORMAT_CORESIGHT 0x0000 /* Default for backward compatibility */
1280	#define PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW 0x0100 /* Raw format of the source */	1291	#define PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW 0x0100 /* Raw format of the source */
1281		1292
1282	#define PERF_FLAG_FD_NO_GROUP (1UL << 0)	1293	#define PERF_FLAG_FD_NO_GROUP (1UL << 0)
1283	#define PERF_FLAG_FD_OUTPUT (1UL << 1)	1294	#define PERF_FLAG_FD_OUTPUT (1UL << 1)
1284	#define PERF_FLAG_PID_CGROUP (1UL << 2) /* pid=cgroup id, per-cpu mode only */	1295	#define PERF_FLAG_PID_CGROUP (1UL << 2) /* pid=cgroup ID, per-CPU mode only */
1285	#define PERF_FLAG_FD_CLOEXEC (1UL << 3) /* O_CLOEXEC */	1296	#define PERF_FLAG_FD_CLOEXEC (1UL << 3) /* O_CLOEXEC */
1286		1297
1287	#if defined(__LITTLE_ENDIAN_BITFIELD)	1298	#if defined(__LITTLE_ENDIAN_BITFIELD)
1288	union perf_mem_data_src {	1299	union perf_mem_data_src {
1289	__u64 val;	1300	__u64 val;
1290	struct {	1301	struct {
1291	__u64 mem_op:5, /* type of opcode */	1302	__u64 mem_op : 5, /* Type of opcode */
1292	mem_lvl:14, /* memory hierarchy level */	1303	mem_lvl : 14, /* Memory hierarchy level */
1293	mem_snoop:5, /* snoop mode */	1304	mem_snoop : 5, /* Snoop mode */
1294	mem_lock:2, /* lock instr */	1305	mem_lock : 2, /* Lock instr */
1295	mem_dtlb:7, /* tlb access */	1306	mem_dtlb : 7, /* TLB access */
1296	mem_lvl_num:4, /* memory hierarchy level number */	1307	mem_lvl_num : 4, /* Memory hierarchy level number */
1297	mem_remote:1, /* remote */	1308	mem_remote : 1, /* Remote */
1298	mem_snoopx:2, /* snoop mode, ext */	1309	mem_snoopx : 2, /* Snoop mode, ext */
1299	mem_blk:3, /* access blocked */	1310	mem_blk : 3, /* Access blocked */
1300	mem_hops:3, /* hop level */	1311	mem_hops : 3, /* Hop level */
1301	mem_rsvd:18;	1312	mem_rsvd : 18;
1302	};	1313	};
1303	};	1314	};
1304	#elif defined(__BIG_ENDIAN_BITFIELD)	1315	#elif defined(__BIG_ENDIAN_BITFIELD)
1305	union perf_mem_data_src {	1316	union perf_mem_data_src {
1306	__u64 val;	1317	__u64 val;
1307	struct {	1318	struct {
1308	__u64 mem_rsvd:18,	1319	__u64 mem_rsvd : 18,
1309	mem_hops:3, /* hop level */	1320	mem_hops : 3, /* Hop level */
1310	mem_blk:3, /* access blocked */	1321	mem_blk : 3, /* Access blocked */
1311	mem_snoopx:2, /* snoop mode, ext */	1322	mem_snoopx : 2, /* Snoop mode, ext */
1312	mem_remote:1, /* remote */	1323	mem_remote : 1, /* Remote */
1313	mem_lvl_num:4, /* memory hierarchy level number */	1324	mem_lvl_num : 4, /* Memory hierarchy level number */
1314	mem_dtlb:7, /* tlb access */	1325	mem_dtlb : 7, /* TLB access */
1315	mem_lock:2, /* lock instr */	1326	mem_lock : 2, /* Lock instr */
1316	mem_snoop:5, /* snoop mode */	1327	mem_snoop : 5, /* Snoop mode */
1317	mem_lvl:14, /* memory hierarchy level */	1328	mem_lvl : 14, /* Memory hierarchy level */
1318	mem_op:5; /* type of opcode */	1329	mem_op : 5; /* Type of opcode */
1319	};	1330	};
1320	};	1331	};
1321	#else	1332	#else
1322	#error "Unknown endianness"	1333	# error "Unknown endianness"
1323	#endif	1334	#endif
1324		1335
1325	/* type of opcode (load/store/prefetch,code) */	1336	/* Type of memory opcode: */
1326	#define PERF_MEM_OP_NA 0x01 /* not available */	1337	#define PERF_MEM_OP_NA 0x0001 /* Not available */
1327	#define PERF_MEM_OP_LOAD 0x02 /* load instruction */	1338	#define PERF_MEM_OP_LOAD 0x0002 /* Load instruction */
1328	#define PERF_MEM_OP_STORE 0x04 /* store instruction */	1339	#define PERF_MEM_OP_STORE 0x0004 /* Store instruction */
1329	#define PERF_MEM_OP_PFETCH 0x08 /* prefetch */	1340	#define PERF_MEM_OP_PFETCH 0x0008 /* Prefetch */
1330	#define PERF_MEM_OP_EXEC 0x10 /* code (execution) */	1341	#define PERF_MEM_OP_EXEC 0x0010 /* Code (execution) */
1331	#define PERF_MEM_OP_SHIFT 0	1342	#define PERF_MEM_OP_SHIFT 0
1332		1343
1333	/*	1344	/*
1334	* PERF_MEM_LVL_* namespace being depricated to some extent in the	1345	* The PERF_MEM_LVL_* namespace is being deprecated to some extent in
1335	* favour of newer composite PERF_MEM_{LVLNUM_,REMOTE_,SNOOPX_} fields.	1346	* favour of newer composite PERF_MEM_{LVLNUM_,REMOTE_,SNOOPX_} fields.
1336	* Supporting this namespace inorder to not break defined ABIs.	1347	* We support this namespace in order to not break defined ABIs.
1337	*	1348	*
1338	* memory hierarchy (memory level, hit or miss)	1349	* Memory hierarchy (memory level, hit or miss)
1339	*/	1350	*/
1340	#define PERF_MEM_LVL_NA 0x01 /* not available */	1351	#define PERF_MEM_LVL_NA 0x0001 /* Not available */
1341	#define PERF_MEM_LVL_HIT 0x02 /* hit level */	1352	#define PERF_MEM_LVL_HIT 0x0002 /* Hit level */
1342	#define PERF_MEM_LVL_MISS 0x04 /* miss level */	1353	#define PERF_MEM_LVL_MISS 0x0004 /* Miss level */
1343	#define PERF_MEM_LVL_L1 0x08 /* L1 */	1354	#define PERF_MEM_LVL_L1 0x0008 /* L1 */
1344	#define PERF_MEM_LVL_LFB 0x10 /* Line Fill Buffer */	1355	#define PERF_MEM_LVL_LFB 0x0010 /* Line Fill Buffer */
1345	#define PERF_MEM_LVL_L2 0x20 /* L2 */	1356	#define PERF_MEM_LVL_L2 0x0020 /* L2 */
1346	#define PERF_MEM_LVL_L3 0x40 /* L3 */	1357	#define PERF_MEM_LVL_L3 0x0040 /* L3 */
1347	#define PERF_MEM_LVL_LOC_RAM 0x80 /* Local DRAM */	1358	#define PERF_MEM_LVL_LOC_RAM 0x0080 /* Local DRAM */
1348	#define PERF_MEM_LVL_REM_RAM1 0x100 /* Remote DRAM (1 hop) */	1359	#define PERF_MEM_LVL_REM_RAM1 0x0100 /* Remote DRAM (1 hop) */
1349	#define PERF_MEM_LVL_REM_RAM2 0x200 /* Remote DRAM (2 hops) */	1360	#define PERF_MEM_LVL_REM_RAM2 0x0200 /* Remote DRAM (2 hops) */
1350	#define PERF_MEM_LVL_REM_CCE1 0x400 /* Remote Cache (1 hop) */	1361	#define PERF_MEM_LVL_REM_CCE1 0x0400 /* Remote Cache (1 hop) */
1351	#define PERF_MEM_LVL_REM_CCE2 0x800 /* Remote Cache (2 hops) */	1362	#define PERF_MEM_LVL_REM_CCE2 0x0800 /* Remote Cache (2 hops) */
1352	#define PERF_MEM_LVL_IO 0x1000 /* I/O memory */	1363	#define PERF_MEM_LVL_IO 0x1000 /* I/O memory */
1353	#define PERF_MEM_LVL_UNC 0x2000 /* Uncached memory */	1364	#define PERF_MEM_LVL_UNC 0x2000 /* Uncached memory */
1354	#define PERF_MEM_LVL_SHIFT 5	1365	#define PERF_MEM_LVL_SHIFT 5
1355		1366
1356	#define PERF_MEM_REMOTE_REMOTE 0x01 /* Remote */	1367	#define PERF_MEM_REMOTE_REMOTE 0x0001 /* Remote */
1357	#define PERF_MEM_REMOTE_SHIFT 37	1368	#define PERF_MEM_REMOTE_SHIFT 37
1358		1369
1359	#define PERF_MEM_LVLNUM_L1 0x01 /* L1 */	1370	#define PERF_MEM_LVLNUM_L1 0x0001 /* L1 */
1360	#define PERF_MEM_LVLNUM_L2 0x02 /* L2 */	1371	#define PERF_MEM_LVLNUM_L2 0x0002 /* L2 */
1361	#define PERF_MEM_LVLNUM_L3 0x03 /* L3 */	1372	#define PERF_MEM_LVLNUM_L3 0x0003 /* L3 */
1362	#define PERF_MEM_LVLNUM_L4 0x04 /* L4 */	1373	#define PERF_MEM_LVLNUM_L4 0x0004 /* L4 */
1363	#define PERF_MEM_LVLNUM_L2_MHB 0x05 /* L2 Miss Handling Buffer */	1374	#define PERF_MEM_LVLNUM_L2_MHB 0x0005 /* L2 Miss Handling Buffer */
1364	#define PERF_MEM_LVLNUM_MSC 0x06 /* Memory-side Cache */	1375	#define PERF_MEM_LVLNUM_MSC 0x0006 /* Memory-side Cache */
1365	/* 0x7 available */	1376	/* 0x007 available */
1366	#define PERF_MEM_LVLNUM_UNC 0x08 /* Uncached */	1377	#define PERF_MEM_LVLNUM_UNC 0x0008 /* Uncached */
1367	#define PERF_MEM_LVLNUM_CXL 0x09 /* CXL */	1378	#define PERF_MEM_LVLNUM_CXL 0x0009 /* CXL */
1368	#define PERF_MEM_LVLNUM_IO 0x0a /* I/O */	1379	#define PERF_MEM_LVLNUM_IO 0x000a /* I/O */
1369	#define PERF_MEM_LVLNUM_ANY_CACHE 0x0b /* Any cache */	1380	#define PERF_MEM_LVLNUM_ANY_CACHE 0x000b /* Any cache */
1370	#define PERF_MEM_LVLNUM_LFB 0x0c /* LFB / L1 Miss Handling Buffer */	1381	#define PERF_MEM_LVLNUM_LFB 0x000c /* LFB / L1 Miss Handling Buffer */
1371	#define PERF_MEM_LVLNUM_RAM 0x0d /* RAM */	1382	#define PERF_MEM_LVLNUM_RAM 0x000d /* RAM */
1372	#define PERF_MEM_LVLNUM_PMEM 0x0e /* PMEM */	1383	#define PERF_MEM_LVLNUM_PMEM 0x000e /* PMEM */
1373	#define PERF_MEM_LVLNUM_NA 0x0f /* N/A */	1384	#define PERF_MEM_LVLNUM_NA 0x000f /* N/A */
1374		1385
1375	#define PERF_MEM_LVLNUM_SHIFT 33	1386	#define PERF_MEM_LVLNUM_SHIFT 33
1376		1387
1377	/* snoop mode */	1388	/* Snoop mode */
1378	#define PERF_MEM_SNOOP_NA 0x01 /* not available */	1389	#define PERF_MEM_SNOOP_NA 0x0001 /* Not available */
1379	#define PERF_MEM_SNOOP_NONE 0x02 /* no snoop */	1390	#define PERF_MEM_SNOOP_NONE 0x0002 /* No snoop */
1380	#define PERF_MEM_SNOOP_HIT 0x04 /* snoop hit */	1391	#define PERF_MEM_SNOOP_HIT 0x0004 /* Snoop hit */
1381	#define PERF_MEM_SNOOP_MISS 0x08 /* snoop miss */	1392	#define PERF_MEM_SNOOP_MISS 0x0008 /* Snoop miss */
1382	#define PERF_MEM_SNOOP_HITM 0x10 /* snoop hit modified */	1393	#define PERF_MEM_SNOOP_HITM 0x0010 /* Snoop hit modified */
1383	#define PERF_MEM_SNOOP_SHIFT 19	1394	#define PERF_MEM_SNOOP_SHIFT 19
1384		1395
1385	#define PERF_MEM_SNOOPX_FWD 0x01 /* forward */	1396	#define PERF_MEM_SNOOPX_FWD 0x0001 /* Forward */
1386	#define PERF_MEM_SNOOPX_PEER 0x02 /* xfer from peer */	1397	#define PERF_MEM_SNOOPX_PEER 0x0002 /* Transfer from peer */
1387	#define PERF_MEM_SNOOPX_SHIFT 38	1398	#define PERF_MEM_SNOOPX_SHIFT 38
1388		1399
1389	/* locked instruction */	1400	/* Locked instruction */
1390	#define PERF_MEM_LOCK_NA 0x01 /* not available */	1401	#define PERF_MEM_LOCK_NA 0x0001 /* Not available */
1391	#define PERF_MEM_LOCK_LOCKED 0x02 /* locked transaction */	1402	#define PERF_MEM_LOCK_LOCKED 0x0002 /* Locked transaction */
1392	#define PERF_MEM_LOCK_SHIFT 24	1403	#define PERF_MEM_LOCK_SHIFT 24
1393		1404
1394	/* TLB access */	1405	/* TLB access */
1395	#define PERF_MEM_TLB_NA 0x01 /* not available */	1406	#define PERF_MEM_TLB_NA 0x0001 /* Not available */
1396	#define PERF_MEM_TLB_HIT 0x02 /* hit level */	1407	#define PERF_MEM_TLB_HIT 0x0002 /* Hit level */
1397	#define PERF_MEM_TLB_MISS 0x04 /* miss level */	1408	#define PERF_MEM_TLB_MISS 0x0004 /* Miss level */
1398	#define PERF_MEM_TLB_L1 0x08 /* L1 */	1409	#define PERF_MEM_TLB_L1 0x0008 /* L1 */
1399	#define PERF_MEM_TLB_L2 0x10 /* L2 */	1410	#define PERF_MEM_TLB_L2 0x0010 /* L2 */
1400	#define PERF_MEM_TLB_WK 0x20 /* Hardware Walker*/	1411	#define PERF_MEM_TLB_WK 0x0020 /* Hardware Walker*/
1401	#define PERF_MEM_TLB_OS 0x40 /* OS fault handler */	1412	#define PERF_MEM_TLB_OS 0x0040 /* OS fault handler */
1402	#define PERF_MEM_TLB_SHIFT 26	1413	#define PERF_MEM_TLB_SHIFT 26
1403		1414
1404	/* Access blocked */	1415	/* Access blocked */
1405	#define PERF_MEM_BLK_NA 0x01 /* not available */	1416	#define PERF_MEM_BLK_NA 0x0001 /* Not available */
1406	#define PERF_MEM_BLK_DATA 0x02 /* data could not be forwarded */	1417	#define PERF_MEM_BLK_DATA 0x0002 /* Data could not be forwarded */
1407	#define PERF_MEM_BLK_ADDR 0x04 /* address conflict */	1418	#define PERF_MEM_BLK_ADDR 0x0004 /* Address conflict */
1408	#define PERF_MEM_BLK_SHIFT 40	1419	#define PERF_MEM_BLK_SHIFT 40
1409		1420
1410	/* hop level */	1421	/* Hop level */
1411	#define PERF_MEM_HOPS_0 0x01 /* remote core, same node */	1422	#define PERF_MEM_HOPS_0 0x0001 /* Remote core, same node */
1412	#define PERF_MEM_HOPS_1 0x02 /* remote node, same socket */	1423	#define PERF_MEM_HOPS_1 0x0002 /* Remote node, same socket */
1413	#define PERF_MEM_HOPS_2 0x03 /* remote socket, same board */	1424	#define PERF_MEM_HOPS_2 0x0003 /* Remote socket, same board */
1414	#define PERF_MEM_HOPS_3 0x04 /* remote board */	1425	#define PERF_MEM_HOPS_3 0x0004 /* Remote board */
1415	/* 5-7 available */	1426	/* 5-7 available */
1416	#define PERF_MEM_HOPS_SHIFT 43	1427	#define PERF_MEM_HOPS_SHIFT 43
1417		1428
1418	#define PERF_MEM_S(a, s) \	1429	#define PERF_MEM_S(a, s) \
1419	(((__u64)PERF_MEM_##a##_##s) << PERF_MEM_##a##_SHIFT)	1430	(((__u64)PERF_MEM_##a##_##s) << PERF_MEM_##a##_SHIFT)
1420		1431
1421	/*	1432	/*
1422	* single taken branch record layout:	1433	* Layout of single taken branch records:
1423	*	1434	*
1424	* from: source instruction (may not always be a branch insn)	1435	* from: source instruction (may not always be a branch insn)
1425	* to: branch target	1436	* to: branch target
@@ -1438,37 +1449,37 @@ union perf_mem_data_src { · 12 unmodified lines
1438	struct perf_branch_entry {	1449	struct perf_branch_entry {
1439	__u64 from;	1450	__u64 from;
1440	__u64 to;	1451	__u64 to;
1441	__u64 mispred:1, /* target mispredicted */	1452	__u64 mispred : 1, /* target mispredicted */
1442	predicted:1,/* target predicted */	1453	predicted : 1, /* target predicted */
1443	in_tx:1, /* in transaction */	1454	in_tx : 1, /* in transaction */
1444	abort:1, /* transaction abort */	1455	abort : 1, /* transaction abort */
1445	cycles:16, /* cycle count to last branch */	1456	cycles : 16, /* cycle count to last branch */
1446	type:4, /* branch type */	1457	type : 4, /* branch type */
1447	spec:2, /* branch speculation info */	1458	spec : 2, /* branch speculation info */
1448	new_type:4, /* additional branch type */	1459	new_type : 4, /* additional branch type */
1449	priv:3, /* privilege level */	1460	priv : 3, /* privilege level */
1450	reserved:31;	1461	reserved : 31;
1451	};	1462	};
1452		1463
1453	/* Size of used info bits in struct perf_branch_entry */	1464	/* Size of used info bits in struct perf_branch_entry */
1454	#define PERF_BRANCH_ENTRY_INFO_BITS_MAX 33	1465	#define PERF_BRANCH_ENTRY_INFO_BITS_MAX 33
1455		1466
1456	union perf_sample_weight {	1467	union perf_sample_weight {
1457	__u64 full;	1468	__u64 full;
1458	#if defined(__LITTLE_ENDIAN_BITFIELD)	1469	#if defined(__LITTLE_ENDIAN_BITFIELD)
1459	struct {	1470	struct {
1460	__u32 var1_dw;	1471	__u32 var1_dw;
1461	__u16 var2_w;	1472	__u16 var2_w;
1462	__u16 var3_w;	1473	__u16 var3_w;
1463	};	1474	};
1464	#elif defined(__BIG_ENDIAN_BITFIELD)	1475	#elif defined(__BIG_ENDIAN_BITFIELD)
1465	struct {	1476	struct {
1466	__u16 var3_w;	1477	__u16 var3_w;
1467	__u16 var2_w;	1478	__u16 var2_w;
1468	__u32 var1_dw;	1479	__u32 var1_dw;
1469	};	1480	};
1470	#else	1481	#else
1471	#error "Unknown endianness"	1482	# error "Unknown endianness"
1472	#endif	1483	#endif
1473	};	1484	};
1474		1485

tools/testing/selftests/bpf/bench.c+12−0

 extern const struct bench bench_trig_uretprobe_multi_push;
 extern const struct bench bench_trig_uprobe_multi_ret;
 extern const struct bench bench_trig_uretprobe_multi_ret;
+#ifdef __x86_64__
+extern const struct bench bench_trig_uprobe_nop5;
+extern const struct bench bench_trig_uretprobe_nop5;
+extern const struct bench bench_trig_uprobe_multi_nop5;
+extern const struct bench bench_trig_uretprobe_multi_nop5;
+#endif
 extern const struct bench bench_rb_libbpf;
 extern const struct bench bench_rb_custom;
 	&bench_trig_uretprobe_multi_push,
 	&bench_trig_uprobe_multi_ret,
 	&bench_trig_uretprobe_multi_ret,
+#ifdef __x86_64__
+	&bench_trig_uprobe_nop5,
+	&bench_trig_uretprobe_nop5,
+	&bench_trig_uprobe_multi_nop5,
+	&bench_trig_uretprobe_multi_nop5,
+#endif
 	/* ringbuf/perfbuf benchmarks */
 	&bench_rb_libbpf,
 	&bench_rb_custom,

tools/testing/selftests/bpf/benchs/bench_trigger.c+42−0

@@ -333,6 +333,20 @@ static void uprobe_producer_ret(void input) · 332 unmodified lines
333	return NULL;	333	return NULL;
334	}	334	}
335		335
		336	#ifdef __x86_64__
		337	__nocf_check __weak void uprobe_target_nop5(void)
		338	{
		339	asm volatile (".byte 0x0f, 0x1f, 0x44, 0x00, 0x00");
		340	}
		341
		342	static void uprobe_producer_nop5(void input)
		343	{
		344	while (true)
		345	uprobe_target_nop5();
		346	return NULL;
		347	}
		348	#endif
		349
336	static void usetup(bool use_retprobe, bool use_multi, void *target_addr)	350	static void usetup(bool use_retprobe, bool use_multi, void *target_addr)
337	{	351	{
338	size_t uprobe_offset;	352	size_t uprobe_offset;
@@ -448,6 +462,28 @@ static void uretprobe_multi_ret_setup(void) · 109 unmodified lines
448	usetup(true, true /* use_multi */, &uprobe_target_ret);	462	usetup(true, true /* use_multi */, &uprobe_target_ret);
449	}	463	}
450		464
		465	#ifdef __x86_64__
		466	static void uprobe_nop5_setup(void)
		467	{
		468	usetup(false, false /* !use_multi */, &uprobe_target_nop5);
		469	}
		470
		471	static void uretprobe_nop5_setup(void)
		472	{
		473	usetup(true, false /* !use_multi */, &uprobe_target_nop5);
		474	}
		475
		476	static void uprobe_multi_nop5_setup(void)
		477	{
		478	usetup(false, true /* use_multi */, &uprobe_target_nop5);
		479	}
		480
		481	static void uretprobe_multi_nop5_setup(void)
		482	{
		483	usetup(true, true /* use_multi */, &uprobe_target_nop5);
		484	}
		485	#endif
		486
451	const struct bench bench_trig_syscall_count = {	487	const struct bench bench_trig_syscall_count = {
452	.name = "trig-syscall-count",	488	.name = "trig-syscall-count",
453	.validate = trigger_validate,	489	.validate = trigger_validate,
@@ -506,3 +542,9 @@ BENCH_TRIG_USERMODE(uprobe_multi_ret, ret, "uprobe-multi-ret"); · 52 unmodified lines
506	BENCH_TRIG_USERMODE(uretprobe_multi_nop, nop, "uretprobe-multi-nop");	542	BENCH_TRIG_USERMODE(uretprobe_multi_nop, nop, "uretprobe-multi-nop");
507	BENCH_TRIG_USERMODE(uretprobe_multi_push, push, "uretprobe-multi-push");	543	BENCH_TRIG_USERMODE(uretprobe_multi_push, push, "uretprobe-multi-push");
508	BENCH_TRIG_USERMODE(uretprobe_multi_ret, ret, "uretprobe-multi-ret");	544	BENCH_TRIG_USERMODE(uretprobe_multi_ret, ret, "uretprobe-multi-ret");
		545	#ifdef __x86_64__
		546	BENCH_TRIG_USERMODE(uprobe_nop5, nop5, "uprobe-nop5");
		547	BENCH_TRIG_USERMODE(uretprobe_nop5, nop5, "uretprobe-nop5");
		548	BENCH_TRIG_USERMODE(uprobe_multi_nop5, nop5, "uprobe-multi-nop5");
		549	BENCH_TRIG_USERMODE(uretprobe_multi_nop5, nop5, "uretprobe-multi-nop5");
		550	#endif

tools/testing/selftests/bpf/benchs/run_bench_uprobes.sh+1−1

@@ -2,7 +2,7 @@ · 1 unmodified line
2		2
3	set -eufo pipefail	3	set -eufo pipefail
4		4
5	for i in usermode-count syscall-count {uprobe,uretprobe}-{nop,push,ret}	5	for i in usermode-count syscall-count {uprobe,uretprobe}-{nop,push,ret,nop5}
6	do	6	do
7	summary=$(sudo ./bench -w2 -d5 -a trig-$i \| tail -n1 \| cut -d'(' -f1 \| cut -d' ' -f3-)	7	summary=$(sudo ./bench -w2 -d5 -a trig-$i \| tail -n1 \| cut -d'(' -f1 \| cut -d' ' -f3-)
8	printf "%-15s: %s\n" $i "$summary"	8	printf "%-15s: %s\n" $i "$summary"