Merge tag 'perf-core-2025-09-26' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull performance events updates from Ingo Molnar: "Core perf code updates: - Convert mmap() related reference counts to refcount_t. This is in reaction to the recently fixed refcount bugs, which could have been detected earlier and could have mitigated the bug somewhat (Thomas Gleixner, Peter Zijlstra) - Clean up and simplify the callchain code, in preparation for sframes (Steven Rostedt, Josh Poimboeuf) Uprobes updates: - Add support to optimize usdt probes on x86-64, which gives a substantial speedup (Jiri Olsa) - Cleanups and fixes on x86 (Peter Zijlstra) PMU driver updates: - Various optimizations and fixes to the Intel PMU driver (Dapeng Mi) Misc cleanups and fixes: - Remove redundant __GFP_NOWARN (Qianfeng Rong)" * tag 'perf-core-2025-09-26' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (57 commits) selftests/bpf: Fix uprobe_sigill test for uprobe syscall error value uprobes/x86: Return error from uprobe syscall when not called from trampoline perf: Skip user unwind if the task is a kernel thread perf: Simplify get_perf_callchain() user logic perf: Use current->flags & PF_KTHREAD|PF_USER_WORKER instead of current->mm == NULL perf: Have get_perf_callchain() return NULL if crosstask and user are set perf: Remove get_perf_callchain() init_nr argument perf/x86: Print PMU counters bitmap in x86_pmu_show_pmu_cap() perf/x86/intel: Add ICL_FIXED_0_ADAPTIVE bit into INTEL_FIXED_BITS_MASK perf/x86/intel: Change macro GLOBAL_CTRL_EN_PERF_METRICS to BIT_ULL(48) perf/x86: Add PERF_CAP_PEBS_TIMING_INFO flag perf/x86/intel: Fix IA32_PMC_x_CFG_B MSRs access error perf/x86/intel: Use early_initcall() to hook bts_init() uprobes: Remove redundant __GFP_NOWARN selftests/seccomp: validate uprobe syscall passes through seccomp seccomp: passthrough uprobe systemcall without filtering selftests/bpf: Fix uprobe syscall shadow stack test selftests/bpf: Change test_uretprobe_regs_change for uprobe and uretprobe selftests/bpf: Add uprobe_regs_equal test selftests/bpf: Add optimized usdt variant for basic usdt test ...

Linus Torvalds committed 1 year ago

Commit e4dcbdf2 parents 6c7340a 6d4843632 files · +2221 −375Browse files

Unified · Split

32 files

arch/arm/probes/uprobes/core.c +1 −1
arch/x86/entry/syscalls/syscall_64.tbl +1 −0
arch/x86/events/core.c +9 −7
arch/x86/events/intel/bts.c +1 −1
arch/x86/events/intel/core.c +9 −12
arch/x86/include/asm/msr-index.h +8 −6
arch/x86/include/asm/perf_event.h +6 −2
arch/x86/include/asm/shstk.h +4 −0
arch/x86/include/asm/uprobes.h +7 −0
arch/x86/kernel/shstk.c +40 −0
arch/x86/kernel/uprobes.c +613 −22
arch/x86/kvm/pmu.h +1 −1
include/linux/perf_event.h +2 −2
include/linux/syscalls.h +2 −0
include/linux/uprobes.h +18 −2
kernel/bpf/stackmap.c +2 −2
kernel/events/callchain.c +19 −21
kernel/events/core.c +191 −178
kernel/events/internal.h +2 −2
kernel/events/ring_buffer.c +1 −1
kernel/events/uprobes.c +73 −29
kernel/fork.c +1 −0
kernel/seccomp.c +25 −7
kernel/sys_ni.c +1 −0
tools/arch/x86/include/asm/msr-index.h +8 −6
tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c +451 −33
tools/testing/selftests/bpf/prog_tests/usdt.c +25 −13
tools/testing/selftests/bpf/progs/uprobe_syscall.c +2 −2
tools/testing/selftests/bpf/progs/uprobe_syscall_executed.c +58 −2
tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +9 −2
tools/testing/selftests/bpf/usdt.h +545 −0
tools/testing/selftests/seccomp/seccomp_bpf.c +86 −21

arch/arm/probes/uprobes/core.c+1−1

@@ -30,7 +30,7 @@ int set_swbp(struct arch_uprobe auprobe, struct vm_area_struct vma, · 29 unmodified lines
30	30	unsigned long vaddr)
31	31	{
32	32	return uprobe_write_opcode(auprobe, vma, vaddr,
33		− __opcode_to_mem_arm(auprobe->bpinsn));
	33	+ __opcode_to_mem_arm(auprobe->bpinsn), true);
34	34	}
35	35
36	36	bool arch_uprobe_ignore(struct arch_uprobe auprobe, struct pt_regs regs)

arch/x86/entry/syscalls/syscall_64.tbl+1−0

@@ -345,6 +345,7 @@ · 344 unmodified lines
345	345	333 common io_pgetevents sys_io_pgetevents
346	346	334 common rseq sys_rseq
347	347	335 common uretprobe sys_uretprobe
	348	+336 common uprobe sys_uprobe
348	349	# don't use numbers 387 through 423, add new calls after the last
349	350	# 'common' entry
350	351	424 common pidfd_send_signal sys_pidfd_send_signal

arch/x86/events/core.c+9−7

@@ -2069,13 +2069,15 @@ static void _x86_pmu_read(struct perf_event *event) · 2068 unmodified lines
2069	2069
2070	2070	void x86_pmu_show_pmu_cap(struct pmu *pmu)
2071	2071	{
2072		− pr_info("... version: %d\n", x86_pmu.version);
2073		− pr_info("... bit width: %d\n", x86_pmu.cntval_bits);
2074		− pr_info("... generic registers: %d\n", x86_pmu_num_counters(pmu));
2075		− pr_info("... value mask: %016Lx\n", x86_pmu.cntval_mask);
2076		− pr_info("... max period: %016Lx\n", x86_pmu.max_period);
2077		− pr_info("... fixed-purpose events: %d\n", x86_pmu_num_counters_fixed(pmu));
2078		− pr_info("... event mask: %016Lx\n", hybrid(pmu, intel_ctrl));
	2072	+ pr_info("... version: %d\n", x86_pmu.version);
	2073	+ pr_info("... bit width: %d\n", x86_pmu.cntval_bits);
	2074	+ pr_info("... generic counters: %d\n", x86_pmu_num_counters(pmu));
	2075	+ pr_info("... generic bitmap: %016llx\n", hybrid(pmu, cntr_mask64));
	2076	+ pr_info("... fixed-purpose counters: %d\n", x86_pmu_num_counters_fixed(pmu));
	2077	+ pr_info("... fixed-purpose bitmap: %016llx\n", hybrid(pmu, fixed_cntr_mask64));
	2078	+ pr_info("... value mask: %016llx\n", x86_pmu.cntval_mask);
	2079	+ pr_info("... max period: %016llx\n", x86_pmu.max_period);
	2080	+ pr_info("... global_ctrl mask: %016llx\n", hybrid(pmu, intel_ctrl));
2079	2081	}
2080	2082
2081	2083	static int __init init_hw_perf_events(void)

arch/x86/events/intel/bts.c+1−1

@@ -643,4 +643,4 @@ static __init int bts_init(void) · 642 unmodified lines
643	643
644	644	return perf_pmu_register(&bts_pmu, "intel_bts", -1);
645	645	}
646		−arch_initcall(bts_init);
	646	+early_initcall(bts_init);

arch/x86/events/intel/core.c+9−12

@@ -2845,8 +2845,8 @@ static void intel_pmu_enable_fixed(struct perf_event *event) · 2844 unmodified lines
2845	2845	{
2846	2846	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
2847	2847	struct hw_perf_event *hwc = &event->hw;
2848		− u64 mask, bits = 0;
2849	2848	int idx = hwc->idx;
	2849	+ u64 bits = 0;
2850	2850
2851	2851	if (is_topdown_idx(idx)) {
2852	2852	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
@@ -2885,14 +2885,10 @@ static void intel_pmu_enable_fixed(struct perf_event *event) · 32 unmodified lines
2885	2885
2886	2886	idx -= INTEL_PMC_IDX_FIXED;
2887	2887	bits = intel_fixed_bits_by_idx(idx, bits);
2888		− mask = intel_fixed_bits_by_idx(idx, INTEL_FIXED_BITS_MASK);
2889		−
2890		− if (x86_pmu.intel_cap.pebs_baseline && event->attr.precise_ip) {
	2888	+ if (x86_pmu.intel_cap.pebs_baseline && event->attr.precise_ip)
2891	2889	bits \|= intel_fixed_bits_by_idx(idx, ICL_FIXED_0_ADAPTIVE);
2892		− mask \|= intel_fixed_bits_by_idx(idx, ICL_FIXED_0_ADAPTIVE);
2893		− }
2894	2890
2895		− cpuc->fixed_ctrl_val &= ~mask;
	2891	+ cpuc->fixed_ctrl_val &= ~intel_fixed_bits_by_idx(idx, INTEL_FIXED_BITS_MASK);
2896	2892	cpuc->fixed_ctrl_val \|= bits;
2897	2893	}
2898	2894
@@ -2997,7 +2993,8 @@ static void intel_pmu_acr_late_setup(struct cpu_hw_events *cpuc) · 98 unmodified lines
2997	2993	if (event->group_leader != leader->group_leader)
2998	2994	break;
2999	2995	for_each_set_bit(idx, (unsigned long *)&event->attr.config2, X86_PMC_IDX_MAX) {
3000		− if (WARN_ON_ONCE(i + idx > cpuc->n_events))
	2996	+ if (i + idx >= cpuc->n_events \|\|
	2997	+ !is_acr_event_group(cpuc->event_list[i + idx]))
3001	2998	return;
3002	2999	__set_bit(cpuc->assign[i + idx], (unsigned long *)&event->hw.config1);
3003	3000	}
@@ -5318,9 +5315,9 @@ static void intel_pmu_check_hybrid_pmus(struct x86_hybrid_pmu *pmu) · 2314 unmodified lines
5318	5315	0, x86_pmu_num_counters(&pmu->pmu), 0, 0);
5319	5316
5320	5317	if (pmu->intel_cap.perf_metrics)
5321		− pmu->intel_ctrl \|= 1ULL << GLOBAL_CTRL_EN_PERF_METRICS;
	5318	+ pmu->intel_ctrl \|= GLOBAL_CTRL_EN_PERF_METRICS;
5322	5319	else
5323		− pmu->intel_ctrl &= ~(1ULL << GLOBAL_CTRL_EN_PERF_METRICS);
	5320	+ pmu->intel_ctrl &= ~GLOBAL_CTRL_EN_PERF_METRICS;
5324	5321
5325	5322	intel_pmu_check_event_constraints(pmu->event_constraints,
5326	5323	pmu->cntr_mask64,
@@ -5455,7 +5452,7 @@ static void intel_pmu_cpu_starting(int cpu) · 128 unmodified lines
5455	5452	rdmsrq(MSR_IA32_PERF_CAPABILITIES, perf_cap.capabilities);
5456	5453	if (!perf_cap.perf_metrics) {
5457	5454	x86_pmu.intel_cap.perf_metrics = 0;
5458		− x86_pmu.intel_ctrl &= ~(1ULL << GLOBAL_CTRL_EN_PERF_METRICS);
	5455	+ x86_pmu.intel_ctrl &= ~GLOBAL_CTRL_EN_PERF_METRICS;
5459	5456	}
5460	5457	}
5461	5458
@@ -7789,7 +7786,7 @@ __init int intel_pmu_init(void) · 2327 unmodified lines
7789	7786	}
7790	7787
7791	7788	if (!is_hybrid() && x86_pmu.intel_cap.perf_metrics)
7792		− x86_pmu.intel_ctrl \|= 1ULL << GLOBAL_CTRL_EN_PERF_METRICS;
	7789	+ x86_pmu.intel_ctrl \|= GLOBAL_CTRL_EN_PERF_METRICS;
7793	7790
7794	7791	if (x86_pmu.intel_cap.pebs_timing_info)
7795	7792	x86_pmu.flags \|= PMU_FL_RETIRE_LATENCY;

arch/x86/include/asm/msr-index.h+8−6

@@ -315,12 +315,14 @@ · 314 unmodified lines
315	315	#define PERF_CAP_PT_IDX 16
316	316
317	317	#define MSR_PEBS_LD_LAT_THRESHOLD 0x000003f6
318		−#define PERF_CAP_PEBS_TRAP BIT_ULL(6)
319		−#define PERF_CAP_ARCH_REG BIT_ULL(7)
320		−#define PERF_CAP_PEBS_FORMAT 0xf00
321		−#define PERF_CAP_PEBS_BASELINE BIT_ULL(14)
322		−#define PERF_CAP_PEBS_MASK (PERF_CAP_PEBS_TRAP \| PERF_CAP_ARCH_REG \| \
323		− PERF_CAP_PEBS_FORMAT \| PERF_CAP_PEBS_BASELINE)
	318	+#define PERF_CAP_PEBS_TRAP BIT_ULL(6)
	319	+#define PERF_CAP_ARCH_REG BIT_ULL(7)
	320	+#define PERF_CAP_PEBS_FORMAT 0xf00
	321	+#define PERF_CAP_PEBS_BASELINE BIT_ULL(14)
	322	+#define PERF_CAP_PEBS_TIMING_INFO BIT_ULL(17)
	323	+#define PERF_CAP_PEBS_MASK (PERF_CAP_PEBS_TRAP \| PERF_CAP_ARCH_REG \| \
	324	+ PERF_CAP_PEBS_FORMAT \| PERF_CAP_PEBS_BASELINE \| \
	325	+ PERF_CAP_PEBS_TIMING_INFO)
324	326
325	327	#define MSR_IA32_RTIT_CTL 0x00000570
326	328	#define RTIT_CTL_TRACEEN BIT(0)

arch/x86/include/asm/perf_event.h+6−2

@@ -35,7 +35,6 @@ · 34 unmodified lines
35	35	#define ARCH_PERFMON_EVENTSEL_EQ (1ULL << 36)
36	36	#define ARCH_PERFMON_EVENTSEL_UMASK2 (0xFFULL << 40)
37	37
38		−#define INTEL_FIXED_BITS_MASK 0xFULL
39	38	#define INTEL_FIXED_BITS_STRIDE 4
40	39	#define INTEL_FIXED_0_KERNEL (1ULL << 0)
41	40	#define INTEL_FIXED_0_USER (1ULL << 1)
@@ -48,6 +47,11 @@ · 6 unmodified lines
48	47	#define ICL_EVENTSEL_ADAPTIVE (1ULL << 34)
49	48	#define ICL_FIXED_0_ADAPTIVE (1ULL << 32)
50	49
	50	+#define INTEL_FIXED_BITS_MASK \
	51	+ (INTEL_FIXED_0_KERNEL \| INTEL_FIXED_0_USER \| \
	52	+ INTEL_FIXED_0_ANYTHREAD \| INTEL_FIXED_0_ENABLE_PMI \| \
	53	+ ICL_FIXED_0_ADAPTIVE)
	54	+
51	55	#define intel_fixed_bits_by_idx(_idx, _bits) \
52	56	((_bits) << ((_idx) * INTEL_FIXED_BITS_STRIDE))
53	57
@@ -430,7 +434,7 @@ static inline bool is_topdown_idx(int idx) · 376 unmodified lines
430	434	#define GLOBAL_STATUS_TRACE_TOPAPMI BIT_ULL(GLOBAL_STATUS_TRACE_TOPAPMI_BIT)
431	435	#define GLOBAL_STATUS_PERF_METRICS_OVF_BIT 48
432	436
433		−#define GLOBAL_CTRL_EN_PERF_METRICS 48
	437	+#define GLOBAL_CTRL_EN_PERF_METRICS BIT_ULL(48)
434	438	/*
435	439	* We model guest LBR event tracing as another fixed-mode PMC like BTS.
436	440	*

arch/x86/include/asm/shstk.h+4−0

@@ -23,6 +23,8 @@ int setup_signal_shadow_stack(struct ksignal *ksig); · 22 unmodified lines
23	23	int restore_signal_shadow_stack(void);
24	24	int shstk_update_last_frame(unsigned long val);
25	25	bool shstk_is_enabled(void);
	26	+int shstk_pop(u64 *val);
	27	+int shstk_push(u64 val);
26	28	#else
27	29	static inline long shstk_prctl(struct task_struct *task, int option,
28	30	unsigned long arg2) { return -EINVAL; }
@@ -35,6 +37,8 @@ static inline int setup_signal_shadow_stack(struct ksignal *ksig) { return 0; } · 6 unmodified lines
35	37	static inline int restore_signal_shadow_stack(void) { return 0; }
36	38	static inline int shstk_update_last_frame(unsigned long val) { return 0; }
37	39	static inline bool shstk_is_enabled(void) { return false; }
	40	+static inline int shstk_pop(u64 *val) { return -ENOTSUPP; }
	41	+static inline int shstk_push(u64 val) { return -ENOTSUPP; }
38	42	#endif /* CONFIG_X86_USER_SHADOW_STACK */
39	43
40	44	#endif /* __ASSEMBLER__ */

arch/x86/include/asm/uprobes.h+7−0

@@ -20,6 +20,11 @@ typedef u8 uprobe_opcode_t; · 19 unmodified lines
20	20	#define UPROBE_SWBP_INSN 0xcc
21	21	#define UPROBE_SWBP_INSN_SIZE 1
22	22
	23	+enum {
	24	+ ARCH_UPROBE_FLAG_CAN_OPTIMIZE = 0,
	25	+ ARCH_UPROBE_FLAG_OPTIMIZE_FAIL = 1,
	26	+};
	27	+
23	28	struct uprobe_xol_ops;
24	29
25	30	struct arch_uprobe {
@@ -45,6 +50,8 @@ struct arch_uprobe { · 19 unmodified lines
45	50	u8 ilen;
46	51	} push;
47	52	};
	53	+
	54	+ unsigned long flags;
48	55	};
49	56
50	57	struct arch_uprobe_task {

arch/x86/kernel/shstk.c+40−0

@@ -246,6 +246,46 @@ static unsigned long get_user_shstk_addr(void) · 245 unmodified lines
246	246	return ssp;
247	247	}
248	248
	249	+int shstk_pop(u64 *val)
	250	+{
	251	+ int ret = 0;
	252	+ u64 ssp;
	253	+
	254	+ if (!features_enabled(ARCH_SHSTK_SHSTK))
	255	+ return -ENOTSUPP;
	256	+
	257	+ fpregs_lock_and_load();
	258	+
	259	+ rdmsrq(MSR_IA32_PL3_SSP, ssp);
	260	+ if (val && get_user(val, (__user u64 )ssp))
	261	+ ret = -EFAULT;
	262	+ else
	263	+ wrmsrq(MSR_IA32_PL3_SSP, ssp + SS_FRAME_SIZE);
	264	+ fpregs_unlock();
	265	+
	266	+ return ret;
	267	+}
	268	+
	269	+int shstk_push(u64 val)
	270	+{
	271	+ u64 ssp;
	272	+ int ret;
	273	+
	274	+ if (!features_enabled(ARCH_SHSTK_SHSTK))
	275	+ return -ENOTSUPP;
	276	+
	277	+ fpregs_lock_and_load();
	278	+
	279	+ rdmsrq(MSR_IA32_PL3_SSP, ssp);
	280	+ ssp -= SS_FRAME_SIZE;
	281	+ ret = write_user_shstk_64((__user void *)ssp, val);
	282	+ if (!ret)
	283	+ wrmsrq(MSR_IA32_PL3_SSP, ssp);
	284	+ fpregs_unlock();
	285	+
	286	+ return ret;
	287	+}
	288	+
249	289	#define SHSTK_DATA_BIT BIT(63)
250	290
251	291	static int put_shstk_data(u64 __user *addr, u64 data)

arch/x86/kernel/uprobes.c+613−22

@@ -18,6 +18,7 @@ · 17 unmodified lines
18	18	#include <asm/processor.h>
19	19	#include <asm/insn.h>
20	20	#include <asm/mmu_context.h>
	21	+#include <asm/nops.h>
21	22
22	23	/* Post-execution fixups. */
23	24
@@ -310,25 +311,32 @@ static int uprobe_init_insn(struct arch_uprobe auprobe, struct insn insn, bool · 286 unmodified lines
310	311
311	312	#ifdef CONFIG_X86_64
312	313
	314	+struct uretprobe_syscall_args {
	315	+ unsigned long r11;
	316	+ unsigned long cx;
	317	+ unsigned long ax;
	318	+};
	319	+
313	320	asm (
314	321	".pushsection .rodata\n"
315	322	".global uretprobe_trampoline_entry\n"
316	323	"uretprobe_trampoline_entry:\n"
317		− "pushq %rax\n"
318		− "pushq %rcx\n"
319		− "pushq %r11\n"
320		− "movq $" __stringify(__NR_uretprobe) ", %rax\n"
	324	+ "push %rax\n"
	325	+ "push %rcx\n"
	326	+ "push %r11\n"
	327	+ "mov $" __stringify(__NR_uretprobe) ", %rax\n"
321	328	"syscall\n"
322	329	".global uretprobe_syscall_check\n"
323	330	"uretprobe_syscall_check:\n"
324		− "popq %r11\n"
325		− "popq %rcx\n"
326		−
327		− /* The uretprobe syscall replaces stored %rax value with final
	331	+ "pop %r11\n"
	332	+ "pop %rcx\n"
	333	+ /*
	334	+ * The uretprobe syscall replaces stored %rax value with final
328	335	* return address, so we don't restore %rax in here and just
329	336	* call ret.
330	337	*/
331		− "retq\n"
	338	+ "ret\n"
	339	+ "int3\n"
332	340	".global uretprobe_trampoline_end\n"
333	341	"uretprobe_trampoline_end:\n"
334	342	".popsection\n"
@@ -338,7 +346,7 @@ extern u8 uretprobe_trampoline_entry[]; · 3 unmodified lines
338	346	extern u8 uretprobe_trampoline_end[];
339	347	extern u8 uretprobe_syscall_check[];
340	348
341		−void arch_uprobe_trampoline(unsigned long psize)
	349	+void arch_uretprobe_trampoline(unsigned long psize)
342	350	{
343	351	static uprobe_opcode_t insn = UPROBE_SWBP_INSN;
344	352	struct pt_regs *regs = task_pt_regs(current);
@@ -365,7 +373,8 @@ static unsigned long trampoline_check_ip(unsigned long tramp) · 20 unmodified lines
365	373	SYSCALL_DEFINE0(uretprobe)
366	374	{
367	375	struct pt_regs *regs = task_pt_regs(current);
368		− unsigned long err, ip, sp, r11_cx_ax[3], tramp;
	376	+ struct uretprobe_syscall_args args;
	377	+ unsigned long err, ip, sp, tramp;
369	378
370	379	/* If there's no trampoline, we are called from wrong place. */
371	380	tramp = uprobe_get_trampoline_vaddr();
@@ -376,15 +385,15 @@ SYSCALL_DEFINE0(uretprobe) · 4 unmodified lines
376	385	if (unlikely(regs->ip != trampoline_check_ip(tramp)))
377	386	goto sigill;
378	387
379		− err = copy_from_user(r11_cx_ax, (void __user *)regs->sp, sizeof(r11_cx_ax));
	388	+ err = copy_from_user(&args, (void __user *)regs->sp, sizeof(args));
380	389	if (err)
381	390	goto sigill;
382	391
383	392	/* expose the "right" values of r11/cx/ax/sp to uprobe_consumer/s */
384		− regs->r11 = r11_cx_ax[0];
385		− regs->cx = r11_cx_ax[1];
386		− regs->ax = r11_cx_ax[2];
387		− regs->sp += sizeof(r11_cx_ax);
	393	+ regs->r11 = args.r11;
	394	+ regs->cx = args.cx;
	395	+ regs->ax = args.ax;
	396	+ regs->sp += sizeof(args);
388	397	regs->orig_ax = -1;
389	398
390	399	ip = regs->ip;
@@ -400,21 +409,21 @@ SYSCALL_DEFINE0(uretprobe) · 9 unmodified lines
400	409	*/
401	410	if (regs->sp != sp \|\| shstk_is_enabled())
402	411	return regs->ax;
403		− regs->sp -= sizeof(r11_cx_ax);
	412	+ regs->sp -= sizeof(args);
404	413
405	414	/* for the case uprobe_consumer has changed r11/cx */
406		− r11_cx_ax[0] = regs->r11;
407		− r11_cx_ax[1] = regs->cx;
	415	+ args.r11 = regs->r11;
	416	+ args.cx = regs->cx;
408	417
409	418	/*
410	419	* ax register is passed through as return value, so we can use
411	420	* its space on stack for ip value and jump to it through the
412	421	* trampoline's ret instruction
413	422	*/
414		− r11_cx_ax[2] = regs->ip;
	423	+ args.ax = regs->ip;
415	424	regs->ip = ip;
416	425
417		− err = copy_to_user((void __user *)regs->sp, r11_cx_ax, sizeof(r11_cx_ax));
	426	+ err = copy_to_user((void __user *)regs->sp, &args, sizeof(args));
418	427	if (err)
419	428	goto sigill;
420	429
@@ -608,6 +617,581 @@ static void riprel_post_xol(struct arch_uprobe auprobe, struct pt_regs regs) · 187 unmodified lines
608	617	*sr = utask->autask.saved_scratch_register;
609	618	}
610	619	}
	620	+
	621	+static int tramp_mremap(const struct vm_special_mapping sm, struct vm_area_struct new_vma)
	622	+{
	623	+ return -EPERM;
	624	+}
	625	+
	626	+static struct page *tramp_mapping_pages[2] __ro_after_init;
	627	+
	628	+static struct vm_special_mapping tramp_mapping = {
	629	+ .name = "[uprobes-trampoline]",
	630	+ .mremap = tramp_mremap,
	631	+ .pages = tramp_mapping_pages,
	632	+};
	633	+
	634	+struct uprobe_trampoline {
	635	+ struct hlist_node node;
	636	+ unsigned long vaddr;
	637	+};
	638	+
	639	+static bool is_reachable_by_call(unsigned long vtramp, unsigned long vaddr)
	640	+{
	641	+ long delta = (long)(vaddr + 5 - vtramp);
	642	+
	643	+ return delta >= INT_MIN && delta <= INT_MAX;
	644	+}
	645	+
	646	+static unsigned long find_nearest_trampoline(unsigned long vaddr)
	647	+{
	648	+ struct vm_unmapped_area_info info = {
	649	+ .length = PAGE_SIZE,
	650	+ .align_mask = ~PAGE_MASK,
	651	+ };
	652	+ unsigned long low_limit, high_limit;
	653	+ unsigned long low_tramp, high_tramp;
	654	+ unsigned long call_end = vaddr + 5;
	655	+
	656	+ if (check_add_overflow(call_end, INT_MIN, &low_limit))
	657	+ low_limit = PAGE_SIZE;
	658	+
	659	+ high_limit = call_end + INT_MAX;
	660	+
	661	+ /* Search up from the caller address. */
	662	+ info.low_limit = call_end;
	663	+ info.high_limit = min(high_limit, TASK_SIZE);
	664	+ high_tramp = vm_unmapped_area(&info);
	665	+
	666	+ /* Search down from the caller address. */
	667	+ info.low_limit = max(low_limit, PAGE_SIZE);
	668	+ info.high_limit = call_end;
	669	+ info.flags = VM_UNMAPPED_AREA_TOPDOWN;
	670	+ low_tramp = vm_unmapped_area(&info);
	671	+
	672	+ if (IS_ERR_VALUE(high_tramp) && IS_ERR_VALUE(low_tramp))
	673	+ return -ENOMEM;
	674	+ if (IS_ERR_VALUE(high_tramp))
	675	+ return low_tramp;
	676	+ if (IS_ERR_VALUE(low_tramp))
	677	+ return high_tramp;
	678	+
	679	+ /* Return address that's closest to the caller address. */
	680	+ if (call_end - low_tramp < high_tramp - call_end)
	681	+ return low_tramp;
	682	+ return high_tramp;
	683	+}
	684	+
	685	+static struct uprobe_trampoline *create_uprobe_trampoline(unsigned long vaddr)
	686	+{
	687	+ struct pt_regs *regs = task_pt_regs(current);
	688	+ struct mm_struct *mm = current->mm;
	689	+ struct uprobe_trampoline *tramp;
	690	+ struct vm_area_struct *vma;
	691	+
	692	+ if (!user_64bit_mode(regs))
	693	+ return NULL;
	694	+
	695	+ vaddr = find_nearest_trampoline(vaddr);
	696	+ if (IS_ERR_VALUE(vaddr))
	697	+ return NULL;
	698	+
	699	+ tramp = kzalloc(sizeof(*tramp), GFP_KERNEL);
	700	+ if (unlikely(!tramp))
	701	+ return NULL;
	702	+
	703	+ tramp->vaddr = vaddr;
	704	+ vma = _install_special_mapping(mm, tramp->vaddr, PAGE_SIZE,
	705	+ VM_READ\|VM_EXEC\|VM_MAYEXEC\|VM_MAYREAD\|VM_DONTCOPY\|VM_IO,
	706	+ &tramp_mapping);
	707	+ if (IS_ERR(vma)) {
	708	+ kfree(tramp);
	709	+ return NULL;
	710	+ }
	711	+ return tramp;
	712	+}
	713	+
	714	+static struct uprobe_trampoline get_uprobe_trampoline(unsigned long vaddr, bool new)
	715	+{
	716	+ struct uprobes_state *state = &current->mm->uprobes_state;
	717	+ struct uprobe_trampoline *tramp = NULL;
	718	+
	719	+ if (vaddr > TASK_SIZE \|\| vaddr < PAGE_SIZE)
	720	+ return NULL;
	721	+
	722	+ hlist_for_each_entry(tramp, &state->head_tramps, node) {
	723	+ if (is_reachable_by_call(tramp->vaddr, vaddr)) {
	724	+ *new = false;
	725	+ return tramp;
	726	+ }
	727	+ }
	728	+
	729	+ tramp = create_uprobe_trampoline(vaddr);
	730	+ if (!tramp)
	731	+ return NULL;
	732	+
	733	+ *new = true;
	734	+ hlist_add_head(&tramp->node, &state->head_tramps);
	735	+ return tramp;
	736	+}
	737	+
	738	+static void destroy_uprobe_trampoline(struct uprobe_trampoline *tramp)
	739	+{
	740	+ /*
	741	+ * We do not unmap and release uprobe trampoline page itself,
	742	+ * because there's no easy way to make sure none of the threads
	743	+ * is still inside the trampoline.
	744	+ */
	745	+ hlist_del(&tramp->node);
	746	+ kfree(tramp);
	747	+}
	748	+
	749	+void arch_uprobe_init_state(struct mm_struct *mm)
	750	+{
	751	+ INIT_HLIST_HEAD(&mm->uprobes_state.head_tramps);
	752	+}
	753	+
	754	+void arch_uprobe_clear_state(struct mm_struct *mm)
	755	+{
	756	+ struct uprobes_state *state = &mm->uprobes_state;
	757	+ struct uprobe_trampoline *tramp;
	758	+ struct hlist_node *n;
	759	+
	760	+ hlist_for_each_entry_safe(tramp, n, &state->head_tramps, node)
	761	+ destroy_uprobe_trampoline(tramp);
	762	+}
	763	+
	764	+static bool __in_uprobe_trampoline(unsigned long ip)
	765	+{
	766	+ struct vm_area_struct *vma = vma_lookup(current->mm, ip);
	767	+
	768	+ return vma && vma_is_special_mapping(vma, &tramp_mapping);
	769	+}
	770	+
	771	+static bool in_uprobe_trampoline(unsigned long ip)
	772	+{
	773	+ struct mm_struct *mm = current->mm;
	774	+ bool found, retry = true;
	775	+ unsigned int seq;
	776	+
	777	+ rcu_read_lock();
	778	+ if (mmap_lock_speculate_try_begin(mm, &seq)) {
	779	+ found = __in_uprobe_trampoline(ip);
	780	+ retry = mmap_lock_speculate_retry(mm, seq);
	781	+ }
	782	+ rcu_read_unlock();
	783	+
	784	+ if (retry) {
	785	+ mmap_read_lock(mm);
	786	+ found = __in_uprobe_trampoline(ip);
	787	+ mmap_read_unlock(mm);
	788	+ }
	789	+ return found;
	790	+}
	791	+
	792	+/*
	793	+ * See uprobe syscall trampoline; the call to the trampoline will push
	794	+ * the return address on the stack, the trampoline itself then pushes
	795	+ * cx, r11 and ax.
	796	+ */
	797	+struct uprobe_syscall_args {
	798	+ unsigned long ax;
	799	+ unsigned long r11;
	800	+ unsigned long cx;
	801	+ unsigned long retaddr;
	802	+};
	803	+
	804	+SYSCALL_DEFINE0(uprobe)
	805	+{
	806	+ struct pt_regs *regs = task_pt_regs(current);
	807	+ struct uprobe_syscall_args args;
	808	+ unsigned long ip, sp, sret;
	809	+ int err;
	810	+
	811	+ /* Allow execution only from uprobe trampolines. */
	812	+ if (!in_uprobe_trampoline(regs->ip))
	813	+ return -ENXIO;
	814	+
	815	+ err = copy_from_user(&args, (void __user *)regs->sp, sizeof(args));
	816	+ if (err)
	817	+ goto sigill;
	818	+
	819	+ ip = regs->ip;
	820	+
	821	+ /*
	822	+ * expose the "right" values of ax/r11/cx/ip/sp to uprobe_consumer/s, plus:
	823	+ * - adjust ip to the probe address, call saved next instruction address
	824	+ * - adjust sp to the probe's stack frame (check trampoline code)
	825	+ */
	826	+ regs->ax = args.ax;
	827	+ regs->r11 = args.r11;
	828	+ regs->cx = args.cx;
	829	+ regs->ip = args.retaddr - 5;
	830	+ regs->sp += sizeof(args);
	831	+ regs->orig_ax = -1;
	832	+
	833	+ sp = regs->sp;
	834	+
	835	+ err = shstk_pop((u64 *)&sret);
	836	+ if (err == -EFAULT \|\| (!err && sret != args.retaddr))
	837	+ goto sigill;
	838	+
	839	+ handle_syscall_uprobe(regs, regs->ip);
	840	+
	841	+ /*
	842	+ * Some of the uprobe consumers has changed sp, we can do nothing,
	843	+ * just return via iret.
	844	+ */
	845	+ if (regs->sp != sp) {
	846	+ /* skip the trampoline call */
	847	+ if (args.retaddr - 5 == regs->ip)
	848	+ regs->ip += 5;
	849	+ return regs->ax;
	850	+ }
	851	+
	852	+ regs->sp -= sizeof(args);
	853	+
	854	+ /* for the case uprobe_consumer has changed ax/r11/cx */
	855	+ args.ax = regs->ax;
	856	+ args.r11 = regs->r11;
	857	+ args.cx = regs->cx;
	858	+
	859	+ /* keep return address unless we are instructed otherwise */
	860	+ if (args.retaddr - 5 != regs->ip)
	861	+ args.retaddr = regs->ip;
	862	+
	863	+ if (shstk_push(args.retaddr) == -EFAULT)
	864	+ goto sigill;
	865	+
	866	+ regs->ip = ip;
	867	+
	868	+ err = copy_to_user((void __user *)regs->sp, &args, sizeof(args));
	869	+ if (err)
	870	+ goto sigill;
	871	+
	872	+ /* ensure sysret, see do_syscall_64() */
	873	+ regs->r11 = regs->flags;
	874	+ regs->cx = regs->ip;
	875	+ return 0;
	876	+
	877	+sigill:
	878	+ force_sig(SIGILL);
	879	+ return -1;
	880	+}
	881	+
	882	+asm (
	883	+ ".pushsection .rodata\n"
	884	+ ".balign " __stringify(PAGE_SIZE) "\n"
	885	+ "uprobe_trampoline_entry:\n"
	886	+ "push %rcx\n"
	887	+ "push %r11\n"
	888	+ "push %rax\n"
	889	+ "mov $" __stringify(__NR_uprobe) ", %rax\n"
	890	+ "syscall\n"
	891	+ "pop %rax\n"
	892	+ "pop %r11\n"
	893	+ "pop %rcx\n"
	894	+ "ret\n"
	895	+ "int3\n"
	896	+ ".balign " __stringify(PAGE_SIZE) "\n"
	897	+ ".popsection\n"
	898	+);
	899	+
	900	+extern u8 uprobe_trampoline_entry[];
	901	+
	902	+static int __init arch_uprobes_init(void)
	903	+{
	904	+ tramp_mapping_pages[0] = virt_to_page(uprobe_trampoline_entry);
	905	+ return 0;
	906	+}
	907	+
	908	+late_initcall(arch_uprobes_init);
	909	+
	910	+enum {
	911	+ EXPECT_SWBP,
	912	+ EXPECT_CALL,
	913	+};
	914	+
	915	+struct write_opcode_ctx {
	916	+ unsigned long base;
	917	+ int expect;
	918	+};
	919	+
	920	+static int is_call_insn(uprobe_opcode_t *insn)
	921	+{
	922	+ return *insn == CALL_INSN_OPCODE;
	923	+}
	924	+
	925	+/*
	926	+ * Verification callback used by int3_update uprobe_write calls to make sure
	927	+ * the underlying instruction is as expected - either int3 or call.
	928	+ */
	929	+static int verify_insn(struct page page, unsigned long vaddr, uprobe_opcode_t new_opcode,
	930	+ int nbytes, void *data)
	931	+{
	932	+ struct write_opcode_ctx *ctx = data;
	933	+ uprobe_opcode_t old_opcode[5];
	934	+
	935	+ uprobe_copy_from_page(page, ctx->base, (uprobe_opcode_t *) &old_opcode, 5);
	936	+
	937	+ switch (ctx->expect) {
	938	+ case EXPECT_SWBP:
	939	+ if (is_swbp_insn(&old_opcode[0]))
	940	+ return 1;
	941	+ break;
	942	+ case EXPECT_CALL:
	943	+ if (is_call_insn(&old_opcode[0]))
	944	+ return 1;
	945	+ break;
	946	+ }
	947	+
	948	+ return -1;
	949	+}
	950	+
	951	+/*
	952	+ * Modify multi-byte instructions by using INT3 breakpoints on SMP.
	953	+ * We completely avoid using stop_machine() here, and achieve the
	954	+ * synchronization using INT3 breakpoints and SMP cross-calls.
	955	+ * (borrowed comment from smp_text_poke_batch_finish)
	956	+ *
	957	+ * The way it is done:
	958	+ * - Add an INT3 trap to the address that will be patched
	959	+ * - SMP sync all CPUs
	960	+ * - Update all but the first byte of the patched range
	961	+ * - SMP sync all CPUs
	962	+ * - Replace the first byte (INT3) by the first byte of the replacing opcode
	963	+ * - SMP sync all CPUs
	964	+ */
	965	+static int int3_update(struct arch_uprobe auprobe, struct vm_area_struct vma,
	966	+ unsigned long vaddr, char *insn, bool optimize)
	967	+{
	968	+ uprobe_opcode_t int3 = UPROBE_SWBP_INSN;
	969	+ struct write_opcode_ctx ctx = {
	970	+ .base = vaddr,
	971	+ };
	972	+ int err;
	973	+
	974	+ /*
	975	+ * Write int3 trap.
	976	+ *
	977	+ * The swbp_optimize path comes with breakpoint already installed,
	978	+ * so we can skip this step for optimize == true.
	979	+ */
	980	+ if (!optimize) {
	981	+ ctx.expect = EXPECT_CALL;
	982	+ err = uprobe_write(auprobe, vma, vaddr, &int3, 1, verify_insn,
	983	+ true /* is_register /, false / do_update_ref_ctr */,
	984	+ &ctx);
	985	+ if (err)
	986	+ return err;
	987	+ }
	988	+
	989	+ smp_text_poke_sync_each_cpu();
	990	+
	991	+ /* Write all but the first byte of the patched range. */
	992	+ ctx.expect = EXPECT_SWBP;
	993	+ err = uprobe_write(auprobe, vma, vaddr + 1, insn + 1, 4, verify_insn,
	994	+ true /* is_register /, false / do_update_ref_ctr */,
	995	+ &ctx);
	996	+ if (err)
	997	+ return err;
	998	+
	999	+ smp_text_poke_sync_each_cpu();
	1000	+
	1001	+ /*
	1002	+ * Write first byte.
	1003	+ *
	1004	+ * The swbp_unoptimize needs to finish uprobe removal together
	1005	+ * with ref_ctr update, using uprobe_write with proper flags.
	1006	+ */
	1007	+ err = uprobe_write(auprobe, vma, vaddr, insn, 1, verify_insn,
	1008	+ optimize /* is_register /, !optimize / do_update_ref_ctr */,
	1009	+ &ctx);
	1010	+ if (err)
	1011	+ return err;
	1012	+
	1013	+ smp_text_poke_sync_each_cpu();
	1014	+ return 0;
	1015	+}
	1016	+
	1017	+static int swbp_optimize(struct arch_uprobe auprobe, struct vm_area_struct vma,
	1018	+ unsigned long vaddr, unsigned long tramp)
	1019	+{
	1020	+ u8 call[5];
	1021	+
	1022	+ __text_gen_insn(call, CALL_INSN_OPCODE, (const void *) vaddr,
	1023	+ (const void *) tramp, CALL_INSN_SIZE);
	1024	+ return int3_update(auprobe, vma, vaddr, call, true /* optimize */);
	1025	+}
	1026	+
	1027	+static int swbp_unoptimize(struct arch_uprobe auprobe, struct vm_area_struct vma,
	1028	+ unsigned long vaddr)
	1029	+{
	1030	+ return int3_update(auprobe, vma, vaddr, auprobe->insn, false /* optimize */);
	1031	+}
	1032	+
	1033	+static int copy_from_vaddr(struct mm_struct mm, unsigned long vaddr, void dst, int len)
	1034	+{
	1035	+ unsigned int gup_flags = FOLL_FORCE\|FOLL_SPLIT_PMD;
	1036	+ struct vm_area_struct *vma;
	1037	+ struct page *page;
	1038	+
	1039	+ page = get_user_page_vma_remote(mm, vaddr, gup_flags, &vma);
	1040	+ if (IS_ERR(page))
	1041	+ return PTR_ERR(page);
	1042	+ uprobe_copy_from_page(page, vaddr, dst, len);
	1043	+ put_page(page);
	1044	+ return 0;
	1045	+}
	1046	+
	1047	+static bool __is_optimized(uprobe_opcode_t *insn, unsigned long vaddr)
	1048	+{
	1049	+ struct __packed __arch_relative_insn {
	1050	+ u8 op;
	1051	+ s32 raddr;
	1052	+ } call = (struct __arch_relative_insn ) insn;
	1053	+
	1054	+ if (!is_call_insn(insn))
	1055	+ return false;
	1056	+ return __in_uprobe_trampoline(vaddr + 5 + call->raddr);
	1057	+}
	1058	+
	1059	+static int is_optimized(struct mm_struct *mm, unsigned long vaddr)
	1060	+{
	1061	+ uprobe_opcode_t insn[5];
	1062	+ int err;
	1063	+
	1064	+ err = copy_from_vaddr(mm, vaddr, &insn, 5);
	1065	+ if (err)
	1066	+ return err;
	1067	+ return __is_optimized((uprobe_opcode_t *)&insn, vaddr);
	1068	+}
	1069	+
	1070	+static bool should_optimize(struct arch_uprobe *auprobe)
	1071	+{
	1072	+ return !test_bit(ARCH_UPROBE_FLAG_OPTIMIZE_FAIL, &auprobe->flags) &&
	1073	+ test_bit(ARCH_UPROBE_FLAG_CAN_OPTIMIZE, &auprobe->flags);
	1074	+}
	1075	+
	1076	+int set_swbp(struct arch_uprobe auprobe, struct vm_area_struct vma,
	1077	+ unsigned long vaddr)
	1078	+{
	1079	+ if (should_optimize(auprobe)) {
	1080	+ /*
	1081	+ * We could race with another thread that already optimized the probe,
	1082	+ * so let's not overwrite it with int3 again in this case.
	1083	+ */
	1084	+ int ret = is_optimized(vma->vm_mm, vaddr);
	1085	+ if (ret < 0)
	1086	+ return ret;
	1087	+ if (ret)
	1088	+ return 0;
	1089	+ }
	1090	+ return uprobe_write_opcode(auprobe, vma, vaddr, UPROBE_SWBP_INSN,
	1091	+ true /* is_register */);
	1092	+}
	1093	+
	1094	+int set_orig_insn(struct arch_uprobe auprobe, struct vm_area_struct vma,
	1095	+ unsigned long vaddr)
	1096	+{
	1097	+ if (test_bit(ARCH_UPROBE_FLAG_CAN_OPTIMIZE, &auprobe->flags)) {
	1098	+ int ret = is_optimized(vma->vm_mm, vaddr);
	1099	+ if (ret < 0)
	1100	+ return ret;
	1101	+ if (ret) {
	1102	+ ret = swbp_unoptimize(auprobe, vma, vaddr);
	1103	+ WARN_ON_ONCE(ret);
	1104	+ return ret;
	1105	+ }
	1106	+ }
	1107	+ return uprobe_write_opcode(auprobe, vma, vaddr, (uprobe_opcode_t )&auprobe->insn,
	1108	+ false /* is_register */);
	1109	+}
	1110	+
	1111	+static int __arch_uprobe_optimize(struct arch_uprobe auprobe, struct mm_struct mm,
	1112	+ unsigned long vaddr)
	1113	+{
	1114	+ struct uprobe_trampoline *tramp;
	1115	+ struct vm_area_struct *vma;
	1116	+ bool new = false;
	1117	+ int err = 0;
	1118	+
	1119	+ vma = find_vma(mm, vaddr);
	1120	+ if (!vma)
	1121	+ return -EINVAL;
	1122	+ tramp = get_uprobe_trampoline(vaddr, &new);
	1123	+ if (!tramp)
	1124	+ return -EINVAL;
	1125	+ err = swbp_optimize(auprobe, vma, vaddr, tramp->vaddr);
	1126	+ if (WARN_ON_ONCE(err) && new)
	1127	+ destroy_uprobe_trampoline(tramp);
	1128	+ return err;
	1129	+}
	1130	+
	1131	+void arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr)
	1132	+{
	1133	+ struct mm_struct *mm = current->mm;
	1134	+ uprobe_opcode_t insn[5];
	1135	+
	1136	+ if (!should_optimize(auprobe))
	1137	+ return;
	1138	+
	1139	+ mmap_write_lock(mm);
	1140	+
	1141	+ /*
	1142	+ * Check if some other thread already optimized the uprobe for us,
	1143	+ * if it's the case just go away silently.
	1144	+ */
	1145	+ if (copy_from_vaddr(mm, vaddr, &insn, 5))
	1146	+ goto unlock;
	1147	+ if (!is_swbp_insn((uprobe_opcode_t*) &insn))
	1148	+ goto unlock;
	1149	+
	1150	+ /*
	1151	+ * If we fail to optimize the uprobe we set the fail bit so the
	1152	+ * above should_optimize will fail from now on.
	1153	+ */
	1154	+ if (__arch_uprobe_optimize(auprobe, mm, vaddr))
	1155	+ set_bit(ARCH_UPROBE_FLAG_OPTIMIZE_FAIL, &auprobe->flags);
	1156	+
	1157	+unlock:
	1158	+ mmap_write_unlock(mm);
	1159	+}
	1160	+
	1161	+static bool insn_is_nop(struct insn *insn)
	1162	+{
	1163	+ return insn->opcode.nbytes == 1 && insn->opcode.bytes[0] == 0x90;
	1164	+}
	1165	+
	1166	+static bool insn_is_nopl(struct insn *insn)
	1167	+{
	1168	+ if (insn->opcode.nbytes != 2)
	1169	+ return false;
	1170	+
	1171	+ if (insn->opcode.bytes[0] != 0x0f \|\| insn->opcode.bytes[1] != 0x1f)
	1172	+ return false;
	1173	+
	1174	+ if (!insn->modrm.nbytes)
	1175	+ return false;
	1176	+
	1177	+ if (X86_MODRM_REG(insn->modrm.bytes[0]) != 0)
	1178	+ return false;
	1179	+
	1180	+ /* 0f 1f /0 - NOPL */
	1181	+ return true;
	1182	+}
	1183	+
	1184	+static bool can_optimize(struct insn *insn, unsigned long vaddr)
	1185	+{
	1186	+ if (!insn->x86_64 \|\| insn->length != 5)
	1187	+ return false;
	1188	+
	1189	+ if (!insn_is_nop(insn) && !insn_is_nopl(insn))
	1190	+ return false;
	1191	+
	1192	+ /* We can't do cross page atomic writes yet. */
	1193	+ return PAGE_SIZE - (vaddr & ~PAGE_MASK) >= 5;
	1194	+}
611	1195	#else /* 32-bit: */
612	1196	/*
613	1197	* No RIP-relative addressing on 32-bit
@@ -621,6 +1205,10 @@ static void riprel_pre_xol(struct arch_uprobe auprobe, struct pt_regs regs) · 7 unmodified lines
621	1205	static void riprel_post_xol(struct arch_uprobe auprobe, struct pt_regs regs)
622	1206	{
623	1207	}
	1208	+static bool can_optimize(struct insn *insn, unsigned long vaddr)
	1209	+{
	1210	+ return false;
	1211	+}
624	1212	#endif /* CONFIG_X86_64 */
625	1213
626	1214	struct uprobe_xol_ops {
@@ -979,14 +1567,17 @@ static int push_setup_xol_ops(struct arch_uprobe auprobe, struct insn insn) · 352 unmodified lines
979	1567	*/
980	1568	int arch_uprobe_analyze_insn(struct arch_uprobe auprobe, struct mm_struct mm, unsigned long addr)
981	1569	{
982		− struct insn insn;
983	1570	u8 fix_ip_or_call = UPROBE_FIX_IP;
	1571	+ struct insn insn;
984	1572	int ret;
985	1573
986	1574	ret = uprobe_init_insn(auprobe, &insn, is_64bit_mm(mm));
987	1575	if (ret)
988	1576	return ret;
989	1577
	1578	+ if (can_optimize(&insn, addr))
	1579	+ set_bit(ARCH_UPROBE_FLAG_CAN_OPTIMIZE, &auprobe->flags);
	1580	+
990	1581	ret = branch_setup_xol_ops(auprobe, &insn);
991	1582	if (ret != -ENOSYS)
992	1583	return ret;

arch/x86/kvm/pmu.h+1−1

@@ -13,7 +13,7 @@ · 12 unmodified lines
13	13	#define MSR_IA32_MISC_ENABLE_PMU_RO_MASK (MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL \| \
14	14	MSR_IA32_MISC_ENABLE_BTS_UNAVAIL)
15	15
16		−/* retrieve the 4 bits for EN and PMI out of IA32_FIXED_CTR_CTRL */
	16	+/* retrieve a fixed counter bits out of IA32_FIXED_CTR_CTRL */
17	17	#define fixed_ctrl_field(ctrl_reg, idx) \
18	18	(((ctrl_reg) >> ((idx) * INTEL_FIXED_BITS_STRIDE)) & INTEL_FIXED_BITS_MASK)
19	19

include/linux/perf_event.h+2−2

@@ -859,7 +859,7 @@ struct perf_event { · 858 unmodified lines
859	859
860	860	/* mmap bits */
861	861	struct mutex mmap_mutex;
862		− atomic_t mmap_count;
	862	+ refcount_t mmap_count;
863	863
864	864	struct perf_buffer *rb;
865	865	struct list_head rb_entry;
@@ -1719,7 +1719,7 @@ DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry); · 853 unmodified lines
1719	1719	extern void perf_callchain_user(struct perf_callchain_entry_ctx entry, struct pt_regs regs);
1720	1720	extern void perf_callchain_kernel(struct perf_callchain_entry_ctx entry, struct pt_regs regs);
1721	1721	extern struct perf_callchain_entry *
1722		−get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user,
	1722	+get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
1723	1723	u32 max_stack, bool crosstask, bool add_mark);
1724	1724	extern int get_callchain_buffers(int max_stack);
1725	1725	extern void put_callchain_buffers(void);

include/linux/syscalls.h+2−0

@@ -1005,6 +1005,8 @@ asmlinkage long sys_ioperm(unsigned long from, unsigned long num, int on); · 1004 unmodified lines
1005	1005
1006	1006	asmlinkage long sys_uretprobe(void);
1007	1007
	1008	+asmlinkage long sys_uprobe(void);
	1009	+
1008	1010	/* pciconfig: alpha, arm, arm64, ia64, sparc */
1009	1011	asmlinkage long sys_pciconfig_read(unsigned long bus, unsigned long dfn,
1010	1012	unsigned long off, unsigned long len,

include/linux/uprobes.h+18−2

@@ -17,6 +17,7 @@ · 16 unmodified lines
17	17	#include <linux/wait.h>
18	18	#include <linux/timer.h>
19	19	#include <linux/seqlock.h>
	20	+#include <linux/mutex.h>
20	21
21	22	struct uprobe;
22	23	struct vm_area_struct;
@@ -185,8 +186,14 @@ struct xol_area; · 162 unmodified lines
185	186
186	187	struct uprobes_state {
187	188	struct xol_area *xol_area;
	189	+#ifdef CONFIG_X86_64
	190	+ struct hlist_head head_tramps;
	191	+#endif
188	192	};
189	193
	194	+typedef int (uprobe_write_verify_t)(struct page page, unsigned long vaddr,
	195	+ uprobe_opcode_t insn, int nbytes, void data);
	196	+
190	197	extern void __init uprobes_init(void);
191	198	extern int set_swbp(struct arch_uprobe aup, struct vm_area_struct vma, unsigned long vaddr);
192	199	extern int set_orig_insn(struct arch_uprobe aup, struct vm_area_struct vma, unsigned long vaddr);
@@ -194,7 +201,11 @@ extern bool is_swbp_insn(uprobe_opcode_t *insn); · 1 unmodified line
194	201	extern bool is_trap_insn(uprobe_opcode_t *insn);
195	202	extern unsigned long uprobe_get_swbp_addr(struct pt_regs *regs);
196	203	extern unsigned long uprobe_get_trap_addr(struct pt_regs *regs);
197		−extern int uprobe_write_opcode(struct arch_uprobe auprobe, struct vm_area_struct vma, unsigned long vaddr, uprobe_opcode_t);
	204	+extern int uprobe_write_opcode(struct arch_uprobe auprobe, struct vm_area_struct vma, unsigned long vaddr, uprobe_opcode_t,
	205	+ bool is_register);
	206	+extern int uprobe_write(struct arch_uprobe auprobe, struct vm_area_struct vma, const unsigned long opcode_vaddr,
	207	+ uprobe_opcode_t *insn, int nbytes, uprobe_write_verify_t verify, bool is_register, bool do_update_ref_ctr,
	208	+ void *data);
198	209	extern struct uprobe uprobe_register(struct inode inode, loff_t offset, loff_t ref_ctr_offset, struct uprobe_consumer *uc);
199	210	extern int uprobe_apply(struct uprobe uprobe, struct uprobe_consumer uc, bool);
200	211	extern void uprobe_unregister_nosync(struct uprobe uprobe, struct uprobe_consumer uc);
@@ -224,8 +235,13 @@ extern bool arch_uprobe_ignore(struct arch_uprobe aup, struct pt_regs regs); · 23 unmodified lines
224	235	extern void arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr,
225	236	void *src, unsigned long len);
226	237	extern void uprobe_handle_trampoline(struct pt_regs *regs);
227		−extern void arch_uprobe_trampoline(unsigned long psize);
	238	+extern void arch_uretprobe_trampoline(unsigned long psize);
228	239	extern unsigned long uprobe_get_trampoline_vaddr(void);
	240	+extern void uprobe_copy_from_page(struct page page, unsigned long vaddr, void dst, int len);
	241	+extern void arch_uprobe_clear_state(struct mm_struct *mm);
	242	+extern void arch_uprobe_init_state(struct mm_struct *mm);
	243	+extern void handle_syscall_uprobe(struct pt_regs *regs, unsigned long bp_vaddr);
	244	+extern void arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr);
229	245	#else /* !CONFIG_UPROBES */
230	246	struct uprobes_state {
231	247	};

kernel/bpf/stackmap.c+2−2

@@ -314,7 +314,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs , regs, struct bpf_map , map, · 313 unmodified lines
314	314	if (max_depth > sysctl_perf_event_max_stack)
315	315	max_depth = sysctl_perf_event_max_stack;
316	316
317		− trace = get_perf_callchain(regs, 0, kernel, user, max_depth,
	317	+ trace = get_perf_callchain(regs, kernel, user, max_depth,
318	318	false, false);
319	319
320	320	if (unlikely(!trace))
@@ -451,7 +451,7 @@ static long __bpf_get_stack(struct pt_regs regs, struct task_struct task, · 130 unmodified lines
451	451	else if (kernel && task)
452	452	trace = get_callchain_entry_for_task(task, max_depth);
453	453	else
454		− trace = get_perf_callchain(regs, 0, kernel, user, max_depth,
	454	+ trace = get_perf_callchain(regs, kernel, user, max_depth,
455	455	crosstask, false);
456	456
457	457	if (unlikely(!trace) \|\| trace->nr < skip) {

kernel/events/callchain.c+19−21

@@ -217,22 +217,26 @@ static void fixup_uretprobe_trampoline_entries(struct perf_callchain_entry *entr · 216 unmodified lines
217	217	}
218	218
219	219	struct perf_callchain_entry *
220		−get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user,
	220	+get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
221	221	u32 max_stack, bool crosstask, bool add_mark)
222	222	{
223	223	struct perf_callchain_entry *entry;
224	224	struct perf_callchain_entry_ctx ctx;
225	225	int rctx, start_entry_idx;
226	226
	227	+ /* crosstask is not supported for user stacks */
	228	+ if (crosstask && user && !kernel)
	229	+ return NULL;
	230	+
227	231	entry = get_callchain_entry(&rctx);
228	232	if (!entry)
229	233	return NULL;
230	234
231		− ctx.entry = entry;
232		− ctx.max_stack = max_stack;
233		− ctx.nr = entry->nr = init_nr;
234		− ctx.contexts = 0;
235		− ctx.contexts_maxed = false;
	235	+ ctx.entry = entry;
	236	+ ctx.max_stack = max_stack;
	237	+ ctx.nr = entry->nr = 0;
	238	+ ctx.contexts = 0;
	239	+ ctx.contexts_maxed = false;
236	240
237	241	if (kernel && !user_mode(regs)) {
238	242	if (add_mark)
@@ -240,25 +244,19 @@ get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user, · 1 unmodified line
240	244	perf_callchain_kernel(&ctx, regs);
241	245	}
242	246
243		− if (user) {
	247	+ if (user && !crosstask) {
244	248	if (!user_mode(regs)) {
245		− if (current->mm)
246		− regs = task_pt_regs(current);
247		− else
248		− regs = NULL;
249		− }
250		−
251		− if (regs) {
252		− if (crosstask)
	249	+ if (current->flags & (PF_KTHREAD \| PF_USER_WORKER))
253	250	goto exit_put;
	251	+ regs = task_pt_regs(current);
	252	+ }
254	253
255		− if (add_mark)
256		− perf_callchain_store_context(&ctx, PERF_CONTEXT_USER);
	254	+ if (add_mark)
	255	+ perf_callchain_store_context(&ctx, PERF_CONTEXT_USER);
257	256
258		− start_entry_idx = entry->nr;
259		− perf_callchain_user(&ctx, regs);
260		− fixup_uretprobe_trampoline_entries(entry, start_entry_idx);
261		− }
	257	+ start_entry_idx = entry->nr;
	258	+ perf_callchain_user(&ctx, regs);
	259	+ fixup_uretprobe_trampoline_entries(entry, start_entry_idx);
262	260	}
263	261
264	262	exit_put:

kernel/events/core.c+191−178

@@ -3974,7 +3974,7 @@ static noinline int visit_groups_merge(struct perf_event_context *ctx, · 3973 unmodified lines
3974	3974	*/
3975	3975	static inline bool event_update_userpage(struct perf_event *event)
3976	3976	{
3977		− if (likely(!atomic_read(&event->mmap_count)))
	3977	+ if (likely(!refcount_read(&event->mmap_count)))
3978	3978	return false;
3979	3979
3980	3980	perf_event_update_time(event);
@@ -6710,11 +6710,11 @@ static void perf_mmap_open(struct vm_area_struct *vma) · 2729 unmodified lines
6710	6710	struct perf_event *event = vma->vm_file->private_data;
6711	6711	mapped_f mapped = get_mapped(event, event_mapped);
6712	6712
6713		− atomic_inc(&event->mmap_count);
6714		− atomic_inc(&event->rb->mmap_count);
	6713	+ refcount_inc(&event->mmap_count);
	6714	+ refcount_inc(&event->rb->mmap_count);
6715	6715
6716	6716	if (vma->vm_pgoff)
6717		− atomic_inc(&event->rb->aux_mmap_count);
	6717	+ refcount_inc(&event->rb->aux_mmap_count);
6718	6718
6719	6719	if (mapped)
6720	6720	mapped(event, vma->vm_mm);
@@ -6749,7 +6749,7 @@ static void perf_mmap_close(struct vm_area_struct *vma) · 28 unmodified lines
6749	6749	* to avoid complications.
6750	6750	*/
6751	6751	if (rb_has_aux(rb) && vma->vm_pgoff == rb->aux_pgoff &&
6752		− atomic_dec_and_mutex_lock(&rb->aux_mmap_count, &rb->aux_mutex)) {
	6752	+ refcount_dec_and_mutex_lock(&rb->aux_mmap_count, &rb->aux_mutex)) {
6753	6753	/*
6754	6754	* Stop all AUX events that are writing to this buffer,
6755	6755	* so that we can free its AUX pages and corresponding PMU
@@ -6769,10 +6769,10 @@ static void perf_mmap_close(struct vm_area_struct *vma) · 13 unmodified lines
6769	6769	mutex_unlock(&rb->aux_mutex);
6770	6770	}
6771	6771
6772		− if (atomic_dec_and_test(&rb->mmap_count))
	6772	+ if (refcount_dec_and_test(&rb->mmap_count))
6773	6773	detach_rest = true;
6774	6774
6775		− if (!atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex))
	6775	+ if (!refcount_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex))
6776	6776	goto out_put;
6777	6777
6778	6778	ring_buffer_attach(event, NULL);
@@ -6933,230 +6933,242 @@ static int map_range(struct perf_buffer rb, struct vm_area_struct vma) · 154 unmodified lines
6933	6933	return err;
6934	6934	}
6935	6935
6936		−static int perf_mmap(struct file file, struct vm_area_struct vma)
	6936	+static bool perf_mmap_calc_limits(struct vm_area_struct vma, long user_extra, long *extra)
6937	6937	{
6938		− struct perf_event *event = file->private_data;
6939		− unsigned long user_locked, user_lock_limit;
	6938	+ unsigned long user_locked, user_lock_limit, locked, lock_limit;
6940	6939	struct user_struct *user = current_user();
6941		− struct mutex *aux_mutex = NULL;
6942		− struct perf_buffer *rb = NULL;
6943		− unsigned long locked, lock_limit;
6944		− unsigned long vma_size;
6945		− unsigned long nr_pages;
6946		− long user_extra = 0, extra = 0;
6947		− int ret, flags = 0;
6948		− mapped_f mapped;
	6940	+
	6941	+ user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
	6942	+ /* Increase the limit linearly with more CPUs */
	6943	+ user_lock_limit *= num_online_cpus();
	6944	+
	6945	+ user_locked = atomic_long_read(&user->locked_vm);
6949	6946
6950	6947	/*
6951		− * Don't allow mmap() of inherited per-task counters. This would
6952		− * create a performance issue due to all children writing to the
6953		− * same rb.
	6948	+ * sysctl_perf_event_mlock may have changed, so that
	6949	+ * user->locked_vm > user_lock_limit
6954	6950	*/
6955		− if (event->cpu == -1 && event->attr.inherit)
6956		− return -EINVAL;
	6951	+ if (user_locked > user_lock_limit)
	6952	+ user_locked = user_lock_limit;
	6953	+ user_locked += *user_extra;
6957	6954
6958		− if (!(vma->vm_flags & VM_SHARED))
6959		− return -EINVAL;
	6955	+ if (user_locked > user_lock_limit) {
	6956	+ /*
	6957	+ * charge locked_vm until it hits user_lock_limit;
	6958	+ * charge the rest from pinned_vm
	6959	+ */
	6960	+ *extra = user_locked - user_lock_limit;
	6961	+ user_extra -= extra;
	6962	+ }
6960	6963
6961		− ret = security_perf_event_read(event);
6962		− if (ret)
6963		− return ret;
	6964	+ lock_limit = rlimit(RLIMIT_MEMLOCK);
	6965	+ lock_limit >>= PAGE_SHIFT;
	6966	+ locked = atomic64_read(&vma->vm_mm->pinned_vm) + *extra;
6964	6967
6965		− vma_size = vma->vm_end - vma->vm_start;
6966		− nr_pages = vma_size / PAGE_SIZE;
	6968	+ return locked <= lock_limit \|\| !perf_is_paranoid() \|\| capable(CAP_IPC_LOCK);
	6969	+}
6967	6970
6968		− if (nr_pages > INT_MAX)
6969		− return -ENOMEM;
	6971	+static void perf_mmap_account(struct vm_area_struct *vma, long user_extra, long extra)
	6972	+{
	6973	+ struct user_struct *user = current_user();
6970	6974
6971		− if (vma_size != PAGE_SIZE * nr_pages)
6972		− return -EINVAL;
	6975	+ atomic_long_add(user_extra, &user->locked_vm);
	6976	+ atomic64_add(extra, &vma->vm_mm->pinned_vm);
	6977	+}
6973	6978
6974		− user_extra = nr_pages;
	6979	+static int perf_mmap_rb(struct vm_area_struct vma, struct perf_event event,
	6980	+ unsigned long nr_pages)
	6981	+{
	6982	+ long extra = 0, user_extra = nr_pages;
	6983	+ struct perf_buffer *rb;
	6984	+ int rb_flags = 0;
6975	6985
6976		− mutex_lock(&event->mmap_mutex);
6977		− ret = -EINVAL;
	6986	+ nr_pages -= 1;
6978	6987
6979	6988	/*
6980		− * This relies on __pmu_detach_event() taking mmap_mutex after marking
6981		− * the event REVOKED. Either we observe the state, or __pmu_detach_event()
6982		− * will detach the rb created here.
	6989	+ * If we have rb pages ensure they're a power-of-two number, so we
	6990	+ * can do bitmasks instead of modulo.
6983	6991	*/
6984		− if (event->state <= PERF_EVENT_STATE_REVOKED) {
6985		− ret = -ENODEV;
6986		− goto unlock;
6987		− }
6988		−
6989		− if (vma->vm_pgoff == 0) {
6990		− nr_pages -= 1;
6991		−
6992		− /*
6993		− * If we have rb pages ensure they're a power-of-two number, so we
6994		− * can do bitmasks instead of modulo.
6995		− */
6996		− if (nr_pages != 0 && !is_power_of_2(nr_pages))
6997		− goto unlock;
6998		−
6999		− WARN_ON_ONCE(event->ctx->parent_ctx);
	6992	+ if (nr_pages != 0 && !is_power_of_2(nr_pages))
	6993	+ return -EINVAL;
7000	6994
7001		− if (event->rb) {
7002		− if (data_page_nr(event->rb) != nr_pages)
7003		− goto unlock;
	6995	+ WARN_ON_ONCE(event->ctx->parent_ctx);
7004	6996
7005		− if (atomic_inc_not_zero(&event->rb->mmap_count)) {
7006		− /*
7007		− * Success -- managed to mmap() the same buffer
7008		− * multiple times.
7009		− */
7010		− ret = 0;
7011		− /* We need the rb to map pages. */
7012		− rb = event->rb;
7013		− goto unlock;
7014		− }
	6997	+ if (event->rb) {
	6998	+ if (data_page_nr(event->rb) != nr_pages)
	6999	+ return -EINVAL;
7015	7000
	7001	+ if (refcount_inc_not_zero(&event->rb->mmap_count)) {
7016	7002	/*
7017		− * Raced against perf_mmap_close()'s
7018		− * atomic_dec_and_mutex_lock() remove the
7019		− * event and continue as if !event->rb
	7003	+ * Success -- managed to mmap() the same buffer
	7004	+ * multiple times.
7020	7005	*/
7021		− ring_buffer_attach(event, NULL);
	7006	+ perf_mmap_account(vma, user_extra, extra);
	7007	+ refcount_inc(&event->mmap_count);
	7008	+ return 0;
7022	7009	}
7023	7010
7024		− } else {
7025	7011	/*
7026		− * AUX area mapping: if rb->aux_nr_pages != 0, it's already
7027		− * mapped, all subsequent mappings should have the same size
7028		− * and offset. Must be above the normal perf buffer.
	7012	+ * Raced against perf_mmap_close()'s
	7013	+ * refcount_dec_and_mutex_lock() remove the
	7014	+ * event and continue as if !event->rb
7029	7015	*/
7030		− u64 aux_offset, aux_size;
	7016	+ ring_buffer_attach(event, NULL);
	7017	+ }
7031	7018
7032		− rb = event->rb;
7033		− if (!rb)
7034		− goto aux_unlock;
	7019	+ if (!perf_mmap_calc_limits(vma, &user_extra, &extra))
	7020	+ return -EPERM;
7035	7021
7036		− aux_mutex = &rb->aux_mutex;
7037		− mutex_lock(aux_mutex);
	7022	+ if (vma->vm_flags & VM_WRITE)
	7023	+ rb_flags \|= RING_BUFFER_WRITABLE;
7038	7024
7039		− aux_offset = READ_ONCE(rb->user_page->aux_offset);
7040		− aux_size = READ_ONCE(rb->user_page->aux_size);
	7025	+ rb = rb_alloc(nr_pages,
	7026	+ event->attr.watermark ? event->attr.wakeup_watermark : 0,
	7027	+ event->cpu, rb_flags);
7041	7028
7042		− if (aux_offset < perf_data_size(rb) + PAGE_SIZE)
7043		− goto aux_unlock;
	7029	+ if (!rb)
	7030	+ return -ENOMEM;
7044	7031
7045		− if (aux_offset != vma->vm_pgoff << PAGE_SHIFT)
7046		− goto aux_unlock;
	7032	+ refcount_set(&rb->mmap_count, 1);
	7033	+ rb->mmap_user = get_current_user();
	7034	+ rb->mmap_locked = extra;
7047	7035
7048		− /* already mapped with a different offset */
7049		− if (rb_has_aux(rb) && rb->aux_pgoff != vma->vm_pgoff)
7050		− goto aux_unlock;
	7036	+ ring_buffer_attach(event, rb);
7051	7037
7052		− if (aux_size != vma_size \|\| aux_size != nr_pages * PAGE_SIZE)
7053		− goto aux_unlock;
	7038	+ perf_event_update_time(event);
	7039	+ perf_event_init_userpage(event);
	7040	+ perf_event_update_userpage(event);
7054	7041
7055		− /* already mapped with a different size */
7056		− if (rb_has_aux(rb) && rb->aux_nr_pages != nr_pages)
7057		− goto aux_unlock;
	7042	+ perf_mmap_account(vma, user_extra, extra);
	7043	+ refcount_set(&event->mmap_count, 1);
7058	7044
7059		− if (!is_power_of_2(nr_pages))
7060		− goto aux_unlock;
	7045	+ return 0;
	7046	+}
7061	7047
7062		− if (!atomic_inc_not_zero(&rb->mmap_count))
7063		− goto aux_unlock;
	7048	+static int perf_mmap_aux(struct vm_area_struct vma, struct perf_event event,
	7049	+ unsigned long nr_pages)
	7050	+{
	7051	+ long extra = 0, user_extra = nr_pages;
	7052	+ u64 aux_offset, aux_size;
	7053	+ struct perf_buffer *rb;
	7054	+ int ret, rb_flags = 0;
7064	7055
7065		− if (rb_has_aux(rb)) {
7066		− atomic_inc(&rb->aux_mmap_count);
7067		− ret = 0;
7068		− goto unlock;
7069		− }
7070		− }
	7056	+ rb = event->rb;
	7057	+ if (!rb)
	7058	+ return -EINVAL;
7071	7059
7072		− user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
	7060	+ guard(mutex)(&rb->aux_mutex);
7073	7061
7074	7062	/*
7075		− * Increase the limit linearly with more CPUs:
	7063	+ * AUX area mapping: if rb->aux_nr_pages != 0, it's already
	7064	+ * mapped, all subsequent mappings should have the same size
	7065	+ * and offset. Must be above the normal perf buffer.
7076	7066	*/
7077		− user_lock_limit *= num_online_cpus();
	7067	+ aux_offset = READ_ONCE(rb->user_page->aux_offset);
	7068	+ aux_size = READ_ONCE(rb->user_page->aux_size);
7078	7069
7079		− user_locked = atomic_long_read(&user->locked_vm);
	7070	+ if (aux_offset < perf_data_size(rb) + PAGE_SIZE)
	7071	+ return -EINVAL;
7080	7072
7081		− /*
7082		− * sysctl_perf_event_mlock may have changed, so that
7083		− * user->locked_vm > user_lock_limit
7084		− */
7085		− if (user_locked > user_lock_limit)
7086		− user_locked = user_lock_limit;
7087		− user_locked += user_extra;
	7073	+ if (aux_offset != vma->vm_pgoff << PAGE_SHIFT)
	7074	+ return -EINVAL;
7088	7075
7089		− if (user_locked > user_lock_limit) {
7090		− /*
7091		− * charge locked_vm until it hits user_lock_limit;
7092		− * charge the rest from pinned_vm
7093		− */
7094		− extra = user_locked - user_lock_limit;
7095		− user_extra -= extra;
7096		− }
	7076	+ /* already mapped with a different offset */
	7077	+ if (rb_has_aux(rb) && rb->aux_pgoff != vma->vm_pgoff)
	7078	+ return -EINVAL;
7097	7079
7098		− lock_limit = rlimit(RLIMIT_MEMLOCK);
7099		− lock_limit >>= PAGE_SHIFT;
7100		− locked = atomic64_read(&vma->vm_mm->pinned_vm) + extra;
	7080	+ if (aux_size != nr_pages * PAGE_SIZE)
	7081	+ return -EINVAL;
7101	7082
7102		− if ((locked > lock_limit) && perf_is_paranoid() &&
7103		− !capable(CAP_IPC_LOCK)) {
7104		− ret = -EPERM;
7105		− goto unlock;
7106		− }
	7083	+ /* already mapped with a different size */
	7084	+ if (rb_has_aux(rb) && rb->aux_nr_pages != nr_pages)
	7085	+ return -EINVAL;
7107	7086
7108		− WARN_ON(!rb && event->rb);
	7087	+ if (!is_power_of_2(nr_pages))
	7088	+ return -EINVAL;
7109	7089
7110		− if (vma->vm_flags & VM_WRITE)
7111		− flags \|= RING_BUFFER_WRITABLE;
	7090	+ if (!refcount_inc_not_zero(&rb->mmap_count))
	7091	+ return -EINVAL;
7112	7092
7113		− if (!rb) {
7114		− rb = rb_alloc(nr_pages,
7115		− event->attr.watermark ? event->attr.wakeup_watermark : 0,
7116		− event->cpu, flags);
	7093	+ if (rb_has_aux(rb)) {
	7094	+ refcount_inc(&rb->aux_mmap_count);
7117	7095
7118		− if (!rb) {
7119		− ret = -ENOMEM;
7120		− goto unlock;
	7096	+ } else {
	7097	+ if (!perf_mmap_calc_limits(vma, &user_extra, &extra)) {
	7098	+ refcount_dec(&rb->mmap_count);
	7099	+ return -EPERM;
7121	7100	}
7122	7101
7123		− atomic_set(&rb->mmap_count, 1);
7124		− rb->mmap_user = get_current_user();
7125		− rb->mmap_locked = extra;
	7102	+ WARN_ON(!rb && event->rb);
7126	7103
7127		− ring_buffer_attach(event, rb);
	7104	+ if (vma->vm_flags & VM_WRITE)
	7105	+ rb_flags \|= RING_BUFFER_WRITABLE;
7128	7106
7129		− perf_event_update_time(event);
7130		− perf_event_init_userpage(event);
7131		− perf_event_update_userpage(event);
7132		− ret = 0;
7133		− } else {
7134	7107	ret = rb_alloc_aux(rb, event, vma->vm_pgoff, nr_pages,
7135		− event->attr.aux_watermark, flags);
7136		− if (!ret) {
7137		− atomic_set(&rb->aux_mmap_count, 1);
7138		− rb->aux_mmap_locked = extra;
	7108	+ event->attr.aux_watermark, rb_flags);
	7109	+ if (ret) {
	7110	+ refcount_dec(&rb->mmap_count);
	7111	+ return ret;
7139	7112	}
	7113	+
	7114	+ refcount_set(&rb->aux_mmap_count, 1);
	7115	+ rb->aux_mmap_locked = extra;
7140	7116	}
7141	7117
7142		−unlock:
7143		− if (!ret) {
7144		− atomic_long_add(user_extra, &user->locked_vm);
7145		− atomic64_add(extra, &vma->vm_mm->pinned_vm);
7146		−
7147		− atomic_inc(&event->mmap_count);
7148		− } else if (rb) {
7149		− /* AUX allocation failed */
7150		− atomic_dec(&rb->mmap_count);
7151		− }
7152		−aux_unlock:
7153		− if (aux_mutex)
7154		− mutex_unlock(aux_mutex);
7155		− mutex_unlock(&event->mmap_mutex);
	7118	+ perf_mmap_account(vma, user_extra, extra);
	7119	+ refcount_inc(&event->mmap_count);
7156	7120
	7121	+ return 0;
	7122	+}
	7123	+
	7124	+static int perf_mmap(struct file file, struct vm_area_struct vma)
	7125	+{
	7126	+ struct perf_event *event = file->private_data;
	7127	+ unsigned long vma_size, nr_pages;
	7128	+ mapped_f mapped;
	7129	+ int ret;
	7130	+
	7131	+ /*
	7132	+ * Don't allow mmap() of inherited per-task counters. This would
	7133	+ * create a performance issue due to all children writing to the
	7134	+ * same rb.
	7135	+ */
	7136	+ if (event->cpu == -1 && event->attr.inherit)
	7137	+ return -EINVAL;
	7138	+
	7139	+ if (!(vma->vm_flags & VM_SHARED))
	7140	+ return -EINVAL;
	7141	+
	7142	+ ret = security_perf_event_read(event);
7157	7143	if (ret)
7158	7144	return ret;
7159	7145
	7146	+ vma_size = vma->vm_end - vma->vm_start;
	7147	+ nr_pages = vma_size / PAGE_SIZE;
	7148	+
	7149	+ if (nr_pages > INT_MAX)
	7150	+ return -ENOMEM;
	7151	+
	7152	+ if (vma_size != PAGE_SIZE * nr_pages)
	7153	+ return -EINVAL;
	7154	+
	7155	+ scoped_guard (mutex, &event->mmap_mutex) {
	7156	+ /*
	7157	+ * This relies on __pmu_detach_event() taking mmap_mutex after marking
	7158	+ * the event REVOKED. Either we observe the state, or __pmu_detach_event()
	7159	+ * will detach the rb created here.
	7160	+ */
	7161	+ if (event->state <= PERF_EVENT_STATE_REVOKED)
	7162	+ return -ENODEV;
	7163	+
	7164	+ if (vma->vm_pgoff == 0)
	7165	+ ret = perf_mmap_rb(vma, event, nr_pages);
	7166	+ else
	7167	+ ret = perf_mmap_aux(vma, event, nr_pages);
	7168	+ if (ret)
	7169	+ return ret;
	7170	+ }
	7171	+
7160	7172	/*
7161	7173	* Since pinned accounting is per vm we cannot allow fork() to copy our
7162	7174	* vma.
@@ -7174,7 +7186,7 @@ aux_unlock: · 11 unmodified lines
7174	7186	* full cleanup in this case and therefore does not invoke
7175	7187	* vmops::close().
7176	7188	*/
7177		− ret = map_range(rb, vma);
	7189	+ ret = map_range(event->rb, vma);
7178	7190	if (ret)
7179	7191	perf_mmap_close(vma);
7180	7192
@@ -7440,7 +7452,7 @@ static void perf_sample_regs_user(struct perf_regs *regs_user, · 259 unmodified lines
7440	7452	if (user_mode(regs)) {
7441	7453	regs_user->abi = perf_reg_abi(current);
7442	7454	regs_user->regs = regs;
7443		− } else if (!(current->flags & PF_KTHREAD)) {
	7455	+ } else if (!(current->flags & (PF_KTHREAD \| PF_USER_WORKER))) {
7444	7456	perf_get_regs_user(regs_user, regs);
7445	7457	} else {
7446	7458	regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE;
@@ -8080,7 +8092,7 @@ static u64 perf_virt_to_phys(u64 virt) · 633 unmodified lines
8080	8092	* Try IRQ-safe get_user_page_fast_only first.
8081	8093	* If failed, leave phys_addr as 0.
8082	8094	*/
8083		− if (current->mm != NULL) {
	8095	+ if (!(current->flags & (PF_KTHREAD \| PF_USER_WORKER))) {
8084	8096	struct page *p;
8085	8097
8086	8098	pagefault_disable();
@@ -8192,7 +8204,8 @@ struct perf_callchain_entry * · 105 unmodified lines
8192	8204	perf_callchain(struct perf_event event, struct pt_regs regs)
8193	8205	{
8194	8206	bool kernel = !event->attr.exclude_callchain_kernel;
8195		− bool user = !event->attr.exclude_callchain_user;
	8207	+ bool user = !event->attr.exclude_callchain_user &&
	8208	+ !(current->flags & (PF_KTHREAD \| PF_USER_WORKER));
8196	8209	/* Disallow cross-task user callchains. */
8197	8210	bool crosstask = event->ctx->task && event->ctx->task != current;
8198	8211	const u32 max_stack = event->attr.sample_max_stack;
@@ -8204,7 +8217,7 @@ perf_callchain(struct perf_event event, struct pt_regs regs) · 5 unmodified lines
8204	8217	if (!kernel && !user)
8205	8218	return &__empty_callchain;
8206	8219
8207		− callchain = get_perf_callchain(regs, 0, kernel, user,
	8220	+ callchain = get_perf_callchain(regs, kernel, user,
8208	8221	max_stack, crosstask, true);
8209	8222	return callchain ?: &__empty_callchain;
8210	8223	}
@@ -13249,7 +13262,7 @@ perf_event_set_output(struct perf_event event, struct perf_event output_event) · 5038 unmodified lines
13249	13262	mutex_lock_double(&event->mmap_mutex, &output_event->mmap_mutex);
13250	13263	set:
13251	13264	/* Can't redirect output if we've got an active mmap() */
13252		− if (atomic_read(&event->mmap_count))
	13265	+ if (refcount_read(&event->mmap_count))
13253	13266	goto unlock;
13254	13267
13255	13268	if (output_event) {
@@ -13262,7 +13275,7 @@ set: · 6 unmodified lines
13262	13275	goto unlock;
13263	13276
13264	13277	/* did we race against perf_mmap_close() */
13265		− if (!atomic_read(&rb->mmap_count)) {
	13278	+ if (!refcount_read(&rb->mmap_count)) {
13266	13279	ring_buffer_put(rb);
13267	13280	goto unlock;
13268	13281	}

kernel/events/internal.h+2−2

@@ -35,7 +35,7 @@ struct perf_buffer { · 34 unmodified lines
35	35	spinlock_t event_lock;
36	36	struct list_head event_list;
37	37
38		− atomic_t mmap_count;
	38	+ refcount_t mmap_count;
39	39	unsigned long mmap_locked;
40	40	struct user_struct *mmap_user;
41	41
@@ -47,7 +47,7 @@ struct perf_buffer { · 5 unmodified lines
47	47	unsigned long aux_pgoff;
48	48	int aux_nr_pages;
49	49	int aux_overwrite;
50		− atomic_t aux_mmap_count;
	50	+ refcount_t aux_mmap_count;
51	51	unsigned long aux_mmap_locked;
52	52	void (free_aux)(void );
53	53	refcount_t aux_refcount;

kernel/events/ring_buffer.c+1−1

@@ -400,7 +400,7 @@ void perf_aux_output_begin(struct perf_output_handle handle, · 399 unmodified lines
400	400	* the same order, see perf_mmap_close. Otherwise we end up freeing
401	401	* aux pages in this path, which is a bug, because in_atomic().
402	402	*/
403		− if (!atomic_read(&rb->aux_mmap_count))
	403	+ if (!refcount_read(&rb->aux_mmap_count))
404	404	goto err;
405	405
406	406	if (!refcount_inc_not_zero(&rb->aux_refcount))

kernel/events/uprobes.c+73−29

@@ -177,7 +177,7 @@ bool __weak is_trap_insn(uprobe_opcode_t *insn) · 176 unmodified lines
177	177	return is_swbp_insn(insn);
178	178	}
179	179
180		−static void copy_from_page(struct page page, unsigned long vaddr, void dst, int len)
	180	+void uprobe_copy_from_page(struct page page, unsigned long vaddr, void dst, int len)
181	181	{
182	182	void *kaddr = kmap_atomic(page);
183	183	memcpy(dst, kaddr + (vaddr & ~PAGE_MASK), len);
@@ -191,7 +191,8 @@ static void copy_to_page(struct page page, unsigned long vaddr, const void src · 7 unmodified lines
191	191	kunmap_atomic(kaddr);
192	192	}
193	193
194		−static int verify_opcode(struct page page, unsigned long vaddr, uprobe_opcode_t new_opcode)
	194	+static int verify_opcode(struct page page, unsigned long vaddr, uprobe_opcode_t insn,
	195	+ int nbytes, void *data)
195	196	{
196	197	uprobe_opcode_t old_opcode;
197	198	bool is_swbp;
@@ -205,10 +206,10 @@ static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t · 7 unmodified lines
205	206	* is a trap variant; uprobes always wins over any other (gdb)
206	207	* breakpoint.
207	208	*/
208		− copy_from_page(page, vaddr, &old_opcode, UPROBE_SWBP_INSN_SIZE);
	209	+ uprobe_copy_from_page(page, vaddr, &old_opcode, UPROBE_SWBP_INSN_SIZE);
209	210	is_swbp = is_swbp_insn(&old_opcode);
210	211
211		− if (is_swbp_insn(new_opcode)) {
	212	+ if (is_swbp_insn(insn)) {
212	213	if (is_swbp) /* register: already installed? */
213	214	return 0;
214	215	} else {
@@ -399,12 +400,12 @@ static bool orig_page_is_identical(struct vm_area_struct *vma, · 184 unmodified lines
399	400	return identical;
400	401	}
401	402
402		−static int __uprobe_write_opcode(struct vm_area_struct *vma,
	403	+static int __uprobe_write(struct vm_area_struct *vma,
403	404	struct folio_walk fw, struct folio folio,
404		− unsigned long opcode_vaddr, uprobe_opcode_t opcode)
	405	+ unsigned long insn_vaddr, uprobe_opcode_t *insn, int nbytes,
	406	+ bool is_register)
405	407	{
406		− const unsigned long vaddr = opcode_vaddr & PAGE_MASK;
407		− const bool is_register = !!is_swbp_insn(&opcode);
	408	+ const unsigned long vaddr = insn_vaddr & PAGE_MASK;
408	409	bool pmd_mappable;
409	410
410	411	/* For now, we'll only handle PTE-mapped folios. */
@@ -429,7 +430,7 @@ static int __uprobe_write_opcode(struct vm_area_struct *vma, · 18 unmodified lines
429	430	*/
430	431	flush_cache_page(vma, vaddr, pte_pfn(fw->pte));
431	432	fw->pte = ptep_clear_flush(vma, vaddr, fw->ptep);
432		− copy_to_page(fw->page, opcode_vaddr, &opcode, UPROBE_SWBP_INSN_SIZE);
	433	+ copy_to_page(fw->page, insn_vaddr, insn, nbytes);
433	434
434	435	/*
435	436	* When unregistering, we may only zap a PTE if uffd is disabled and
@@ -482,23 +483,32 @@ remap: · 46 unmodified lines
482	483	* @opcode_vaddr: the virtual address to store the opcode.
483	484	* @opcode: opcode to be written at @opcode_vaddr.
484	485	*
485		− * Called with mm->mmap_lock held for read or write.
	486	+ * Called with mm->mmap_lock held for write.
486	487	* Return 0 (success) or a negative errno.
487	488	*/
488	489	int uprobe_write_opcode(struct arch_uprobe auprobe, struct vm_area_struct vma,
489		− const unsigned long opcode_vaddr, uprobe_opcode_t opcode)
	490	+ const unsigned long opcode_vaddr, uprobe_opcode_t opcode,
	491	+ bool is_register)
490	492	{
491		− const unsigned long vaddr = opcode_vaddr & PAGE_MASK;
	493	+ return uprobe_write(auprobe, vma, opcode_vaddr, &opcode, UPROBE_SWBP_INSN_SIZE,
	494	+ verify_opcode, is_register, true /* do_update_ref_ctr */, NULL);
	495	+}
	496	+
	497	+int uprobe_write(struct arch_uprobe auprobe, struct vm_area_struct vma,
	498	+ const unsigned long insn_vaddr, uprobe_opcode_t *insn, int nbytes,
	499	+ uprobe_write_verify_t verify, bool is_register, bool do_update_ref_ctr,
	500	+ void *data)
	501	+{
	502	+ const unsigned long vaddr = insn_vaddr & PAGE_MASK;
492	503	struct mm_struct *mm = vma->vm_mm;
493	504	struct uprobe *uprobe;
494		− int ret, is_register, ref_ctr_updated = 0;
	505	+ int ret, ref_ctr_updated = 0;
495	506	unsigned int gup_flags = FOLL_FORCE;
496	507	struct mmu_notifier_range range;
497	508	struct folio_walk fw;
498	509	struct folio *folio;
499	510	struct page *page;
500	511
501		− is_register = is_swbp_insn(&opcode);
502	512	uprobe = container_of(auprobe, struct uprobe, arch);
503	513
504	514	if (WARN_ON_ONCE(!is_cow_mapping(vma->vm_flags)))
@@ -509,7 +519,7 @@ int uprobe_write_opcode(struct arch_uprobe auprobe, struct vm_area_struct vma, · 4 unmodified lines
509	519	* page that we can safely modify. Use FOLL_WRITE to trigger a write
510	520	* fault if required. When unregistering, we might be lucky and the
511	521	* anon page is already gone. So defer write faults until really
512		− * required. Use FOLL_SPLIT_PMD, because __uprobe_write_opcode()
	522	+ * required. Use FOLL_SPLIT_PMD, because __uprobe_write()
513	523	* cannot deal with PMDs yet.
514	524	*/
515	525	if (is_register)
@@ -521,14 +531,14 @@ retry: · 5 unmodified lines
521	531	goto out;
522	532	folio = page_folio(page);
523	533
524		− ret = verify_opcode(page, opcode_vaddr, &opcode);
	534	+ ret = verify(page, insn_vaddr, insn, nbytes, data);
525	535	if (ret <= 0) {
526	536	folio_put(folio);
527	537	goto out;
528	538	}
529	539
530	540	/* We are going to replace instruction, update ref_ctr. */
531		− if (!ref_ctr_updated && uprobe->ref_ctr_offset) {
	541	+ if (do_update_ref_ctr && !ref_ctr_updated && uprobe->ref_ctr_offset) {
532	542	ret = update_ref_ctr(uprobe, mm, is_register ? 1 : -1);
533	543	if (ret) {
534	544	folio_put(folio);
@@ -560,7 +570,7 @@ retry: · 25 unmodified lines
560	570	/* Walk the page tables again, to perform the actual update. */
561	571	if (folio_walk_start(&fw, vma, vaddr, 0)) {
562	572	if (fw.page == page)
563		− ret = __uprobe_write_opcode(vma, &fw, folio, opcode_vaddr, opcode);
	573	+ ret = __uprobe_write(vma, &fw, folio, insn_vaddr, insn, nbytes, is_register);
564	574	folio_walk_end(&fw, vma);
565	575	}
566	576
@@ -580,7 +590,7 @@ retry: · 13 unmodified lines
580	590
581	591	out:
582	592	/* Revert back reference counter if instruction update failed. */
583		− if (ret < 0 && ref_ctr_updated)
	593	+ if (do_update_ref_ctr && ret < 0 && ref_ctr_updated)
584	594	update_ref_ctr(uprobe, mm, is_register ? -1 : 1);
585	595
586	596	/* try collapse pmd for compound page */
@@ -602,7 +612,7 @@ out: · 15 unmodified lines
602	612	int __weak set_swbp(struct arch_uprobe auprobe, struct vm_area_struct vma,
603	613	unsigned long vaddr)
604	614	{
605		− return uprobe_write_opcode(auprobe, vma, vaddr, UPROBE_SWBP_INSN);
	615	+ return uprobe_write_opcode(auprobe, vma, vaddr, UPROBE_SWBP_INSN, true);
606	616	}
607	617
608	618	/**
@@ -618,7 +628,7 @@ int __weak set_orig_insn(struct arch_uprobe *auprobe, · 9 unmodified lines
618	628	struct vm_area_struct *vma, unsigned long vaddr)
619	629	{
620	630	return uprobe_write_opcode(auprobe, vma, vaddr,
621		− (uprobe_opcode_t )&auprobe->insn);
	631	+ (uprobe_opcode_t )&auprobe->insn, false);
622	632	}
623	633
624	634	/* uprobe should have guaranteed positive refcount */
@@ -1051,7 +1061,7 @@ static int __copy_insn(struct address_space mapping, struct file filp, · 426 unmodified lines
1051	1061	if (IS_ERR(page))
1052	1062	return PTR_ERR(page);
1053	1063
1054		− copy_from_page(page, offset, insn, nbytes);
	1064	+ uprobe_copy_from_page(page, offset, insn, nbytes);
1055	1065	put_page(page);
1056	1066
1057	1067	return 0;
@@ -1210,7 +1220,7 @@ build_map_info(struct address_space *mapping, loff_t offset, bool is_register) · 152 unmodified lines
1210	1220	* reclaim. This is optimistic, no harm done if it fails.
1211	1221	*/
1212	1222	prev = kmalloc(sizeof(struct map_info),
1213		− GFP_NOWAIT \| __GFP_NOMEMALLOC \| __GFP_NOWARN);
	1223	+ GFP_NOWAIT \| __GFP_NOMEMALLOC);
1214	1224	if (prev)
1215	1225	prev->next = NULL;
1216	1226	}
@@ -1397,7 +1407,7 @@ struct uprobe uprobe_register(struct inode inode, · 180 unmodified lines
1397	1407	return ERR_PTR(-EINVAL);
1398	1408
1399	1409	/*
1400		− * This ensures that copy_from_page(), copy_to_page() and
	1410	+ * This ensures that uprobe_copy_from_page(), copy_to_page() and
1401	1411	* __update_ref_ctr() can't cross page boundary.
1402	1412	*/
1403	1413	if (!IS_ALIGNED(offset, UPROBE_SWBP_INSN_SIZE))
@@ -1463,7 +1473,7 @@ static int unapply_uprobe(struct uprobe uprobe, struct mm_struct mm) · 59 unmodified lines
1463	1473	struct vm_area_struct *vma;
1464	1474	int err = 0;
1465	1475
1466		− mmap_read_lock(mm);
	1476	+ mmap_write_lock(mm);
1467	1477	for_each_vma(vmi, vma) {
1468	1478	unsigned long vaddr;
1469	1479	loff_t offset;
@@ -1480,7 +1490,7 @@ static int unapply_uprobe(struct uprobe uprobe, struct mm_struct mm) · 10 unmodified lines
1480	1490	vaddr = offset_to_vaddr(vma, uprobe->offset);
1481	1491	err \|= remove_breakpoint(uprobe, vma, vaddr);
1482	1492	}
1483		− mmap_read_unlock(mm);
	1493	+ mmap_write_unlock(mm);
1484	1494
1485	1495	return err;
1486	1496	}
@@ -1726,7 +1736,7 @@ static int xol_add_vma(struct mm_struct mm, struct xol_area area) · 239 unmodified lines
1726	1736	return ret;
1727	1737	}
1728	1738
1729		−void * __weak arch_uprobe_trampoline(unsigned long *psize)
	1739	+void * __weak arch_uretprobe_trampoline(unsigned long *psize)
1730	1740	{
1731	1741	static uprobe_opcode_t insn = UPROBE_SWBP_INSN;
1732	1742
@@ -1758,7 +1768,7 @@ static struct xol_area *__create_xol_area(unsigned long vaddr) · 25 unmodified lines
1758	1768	init_waitqueue_head(&area->wq);
1759	1769	/* Reserve the 1st slot for get_trampoline_vaddr() */
1760	1770	set_bit(0, area->bitmap);
1761		− insns = arch_uprobe_trampoline(&insns_size);
	1771	+ insns = arch_uretprobe_trampoline(&insns_size);
1762	1772	arch_uprobe_copy_ixol(area->page, 0, insns, insns_size);
1763	1773
1764	1774	if (!xol_add_vma(mm, area))
@@ -1792,6 +1802,14 @@ static struct xol_area *get_xol_area(void) · 27 unmodified lines
1792	1802	return area;
1793	1803	}
1794	1804
	1805	+void __weak arch_uprobe_clear_state(struct mm_struct *mm)
	1806	+{
	1807	+}
	1808	+
	1809	+void __weak arch_uprobe_init_state(struct mm_struct *mm)
	1810	+{
	1811	+}
	1812	+
1795	1813	/*
1796	1814	* uprobe_clear_state - Free the area allocated for slots.
1797	1815	*/
@@ -1803,6 +1821,8 @@ void uprobe_clear_state(struct mm_struct *mm) · 5 unmodified lines
1803	1821	delayed_uprobe_remove(NULL, mm);
1804	1822	mutex_unlock(&delayed_uprobe_lock);
1805	1823
	1824	+ arch_uprobe_clear_state(mm);
	1825	+
1806	1826	if (!area)
1807	1827	return;
1808	1828
@@ -2393,7 +2413,7 @@ static int is_trap_at_addr(struct mm_struct *mm, unsigned long vaddr) · 584 unmodified lines
2393	2413	if (result < 0)
2394	2414	return result;
2395	2415
2396		− copy_from_page(page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE);
	2416	+ uprobe_copy_from_page(page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE);
2397	2417	put_page(page);
2398	2418	out:
2399	2419	/* This needs to return true for any variant of the trap insn */
@@ -2677,6 +2697,10 @@ bool __weak arch_uretprobe_is_alive(struct return_instance *ret, enum rp_check c · 277 unmodified lines
2677	2697	return true;
2678	2698	}
2679	2699
	2700	+void __weak arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr)
	2701	+{
	2702	+}
	2703	+
2680	2704	/*
2681	2705	* Run handler and ask thread to singlestep.
2682	2706	* Ensure all non-fatal signals cannot interrupt thread while it singlesteps.
@@ -2741,6 +2765,9 @@ static void handle_swbp(struct pt_regs *regs) · 58 unmodified lines
2741	2765
2742	2766	handler_chain(uprobe, regs);
2743	2767
	2768	+ /* Try to optimize after first hit. */
	2769	+ arch_uprobe_optimize(&uprobe->arch, bp_vaddr);
	2770	+
2744	2771	if (arch_uprobe_skip_sstep(&uprobe->arch, regs))
2745	2772	goto out;
2746	2773
@@ -2752,6 +2779,23 @@ out: · 5 unmodified lines
2752	2779	rcu_read_unlock_trace();
2753	2780	}
2754	2781
	2782	+void handle_syscall_uprobe(struct pt_regs *regs, unsigned long bp_vaddr)
	2783	+{
	2784	+ struct uprobe *uprobe;
	2785	+ int is_swbp;
	2786	+
	2787	+ guard(rcu_tasks_trace)();
	2788	+
	2789	+ uprobe = find_active_uprobe_rcu(bp_vaddr, &is_swbp);
	2790	+ if (!uprobe)
	2791	+ return;
	2792	+ if (!get_utask())
	2793	+ return;
	2794	+ if (arch_uprobe_ignore(&uprobe->arch, regs))
	2795	+ return;
	2796	+ handler_chain(uprobe, regs);
	2797	+}
	2798	+
2755	2799	/*
2756	2800	* Perform required fix-ups and disable singlestep.
2757	2801	* Allow pending signals to take effect.

kernel/fork.c+1−0

@@ -1014,6 +1014,7 @@ static void mm_init_uprobes_state(struct mm_struct *mm) · 1013 unmodified lines
1014	1014	{
1015	1015	#ifdef CONFIG_UPROBES
1016	1016	mm->uprobes_state.xol_area = NULL;
	1017	+ arch_uprobe_init_state(mm);
1017	1018	#endif
1018	1019	}
1019	1020

kernel/seccomp.c+25−7

@@ -741,6 +741,26 @@ out: · 740 unmodified lines
741	741	}
742	742
743	743	#ifdef SECCOMP_ARCH_NATIVE
	744	+static bool seccomp_uprobe_exception(struct seccomp_data *sd)
	745	+{
	746	+#if defined __NR_uretprobe \|\| defined __NR_uprobe
	747	+#ifdef SECCOMP_ARCH_COMPAT
	748	+ if (sd->arch == SECCOMP_ARCH_NATIVE)
	749	+#endif
	750	+ {
	751	+#ifdef __NR_uretprobe
	752	+ if (sd->nr == __NR_uretprobe)
	753	+ return true;
	754	+#endif
	755	+#ifdef __NR_uprobe
	756	+ if (sd->nr == __NR_uprobe)
	757	+ return true;
	758	+#endif
	759	+ }
	760	+#endif
	761	+ return false;
	762	+}
	763	+
744	764	/**
745	765	* seccomp_is_const_allow - check if filter is constant allow with given data
746	766	* @fprog: The BPF programs
@@ -758,13 +778,8 @@ static bool seccomp_is_const_allow(struct sock_fprog_kern *fprog, · 11 unmodified lines
758	778	return false;
759	779
760	780	/* Our single exception to filtering. */
761		−#ifdef __NR_uretprobe
762		−#ifdef SECCOMP_ARCH_COMPAT
763		− if (sd->arch == SECCOMP_ARCH_NATIVE)
764		−#endif
765		− if (sd->nr == __NR_uretprobe)
766		− return true;
767		−#endif
	781	+ if (seccomp_uprobe_exception(sd))
	782	+ return true;
768	783
769	784	for (pc = 0; pc < fprog->len; pc++) {
770	785	struct sock_filter *insn = &fprog->filter[pc];
@@ -1042,6 +1057,9 @@ static const int mode1_syscalls[] = { · 271 unmodified lines
1042	1057	__NR_seccomp_read, __NR_seccomp_write, __NR_seccomp_exit, __NR_seccomp_sigreturn,
1043	1058	#ifdef __NR_uretprobe
1044	1059	__NR_uretprobe,
	1060	+#endif
	1061	+#ifdef __NR_uprobe
	1062	+ __NR_uprobe,
1045	1063	#endif
1046	1064	-1, /* negative terminated */
1047	1065	};

kernel/sys_ni.c+1−0

@@ -392,3 +392,4 @@ COND_SYSCALL(setuid16); · 391 unmodified lines
392	392	COND_SYSCALL(rseq);
393	393
394	394	COND_SYSCALL(uretprobe);
	395	+COND_SYSCALL(uprobe);

tools/arch/x86/include/asm/msr-index.h+8−6

@@ -315,12 +315,14 @@ · 314 unmodified lines
315	315	#define PERF_CAP_PT_IDX 16
316	316
317	317	#define MSR_PEBS_LD_LAT_THRESHOLD 0x000003f6
318		−#define PERF_CAP_PEBS_TRAP BIT_ULL(6)
319		−#define PERF_CAP_ARCH_REG BIT_ULL(7)
320		−#define PERF_CAP_PEBS_FORMAT 0xf00
321		−#define PERF_CAP_PEBS_BASELINE BIT_ULL(14)
322		−#define PERF_CAP_PEBS_MASK (PERF_CAP_PEBS_TRAP \| PERF_CAP_ARCH_REG \| \
323		− PERF_CAP_PEBS_FORMAT \| PERF_CAP_PEBS_BASELINE)
	318	+#define PERF_CAP_PEBS_TRAP BIT_ULL(6)
	319	+#define PERF_CAP_ARCH_REG BIT_ULL(7)
	320	+#define PERF_CAP_PEBS_FORMAT 0xf00
	321	+#define PERF_CAP_PEBS_BASELINE BIT_ULL(14)
	322	+#define PERF_CAP_PEBS_TIMING_INFO BIT_ULL(17)
	323	+#define PERF_CAP_PEBS_MASK (PERF_CAP_PEBS_TRAP \| PERF_CAP_ARCH_REG \| \
	324	+ PERF_CAP_PEBS_FORMAT \| PERF_CAP_PEBS_BASELINE \| \
	325	+ PERF_CAP_PEBS_TIMING_INFO)
324	326
325	327	#define MSR_IA32_RTIT_CTL 0x00000570
326	328	#define RTIT_CTL_TRACEEN BIT(0)

tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c+451−33

@@ -8,22 +8,31 @@ · 7 unmodified lines
8	8	#include <asm/ptrace.h>
9	9	#include <linux/compiler.h>
10	10	#include <linux/stringify.h>
	11	+#include <linux/kernel.h>
11	12	#include <sys/wait.h>
12	13	#include <sys/syscall.h>
13	14	#include <sys/prctl.h>
14	15	#include <asm/prctl.h>
15	16	#include "uprobe_syscall.skel.h"
16	17	#include "uprobe_syscall_executed.skel.h"
	18	+#include "bpf/libbpf_internal.h"
17	19
18		−__naked unsigned long uretprobe_regs_trigger(void)
	20	+#define USDT_NOP .byte 0x0f, 0x1f, 0x44, 0x00, 0x00
	21	+#include "usdt.h"
	22	+
	23	+#pragma GCC diagnostic ignored "-Wattributes"
	24	+
	25	+__attribute__((aligned(16)))
	26	+__nocf_check __weak __naked unsigned long uprobe_regs_trigger(void)
19	27	{
20	28	asm volatile (
	29	+ ".byte 0x0f, 0x1f, 0x44, 0x00, 0x00\n" /* nop5 */
21	30	"movq $0xdeadbeef, %rax\n"
22	31	"ret\n"
23	32	);
24	33	}
25	34
26		−__naked void uretprobe_regs(struct pt_regs before, struct pt_regs after)
	35	+__naked void uprobe_regs(struct pt_regs before, struct pt_regs after)
27	36	{
28	37	asm volatile (
29	38	"movq %r15, 0(%rdi)\n"
@@ -44,15 +53,17 @@ __naked void uretprobe_regs(struct pt_regs before, struct pt_regs after) · 14 unmodified lines
44	53	"movq $0, 120(%rdi)\n" /* orig_rax */
45	54	"movq $0, 128(%rdi)\n" /* rip */
46	55	"movq $0, 136(%rdi)\n" /* cs */
	56	+ "pushq %rax\n"
47	57	"pushf\n"
48	58	"pop %rax\n"
49	59	"movq %rax, 144(%rdi)\n" /* eflags */
	60	+ "pop %rax\n"
50	61	"movq %rsp, 152(%rdi)\n" /* rsp */
51	62	"movq $0, 160(%rdi)\n" /* ss */
52	63
53	64	/* save 2nd argument */
54	65	"pushq %rsi\n"
55		− "call uretprobe_regs_trigger\n"
	66	+ "call uprobe_regs_trigger\n"
56	67
57	68	/* save return value and load 2nd argument pointer to rax */
58	69	"pushq %rax\n"
@@ -92,25 +103,37 @@ __naked void uretprobe_regs(struct pt_regs before, struct pt_regs after) · 33 unmodified lines
92	103	);
93	104	}
94	105
95		−static void test_uretprobe_regs_equal(void)
	106	+static void test_uprobe_regs_equal(bool retprobe)
96	107	{
	108	+ LIBBPF_OPTS(bpf_uprobe_opts, opts,
	109	+ .retprobe = retprobe,
	110	+ );
97	111	struct uprobe_syscall *skel = NULL;
98	112	struct pt_regs before = {}, after = {};
99	113	unsigned long pb = (unsigned long ) &before;
100	114	unsigned long pa = (unsigned long ) &after;
101	115	unsigned long *pp;
	116	+ unsigned long offset;
102	117	unsigned int i, cnt;
103		− int err;
	118	+
	119	+ offset = get_uprobe_offset(&uprobe_regs_trigger);
	120	+ if (!ASSERT_GE(offset, 0, "get_uprobe_offset"))
	121	+ return;
104	122
105	123	skel = uprobe_syscall__open_and_load();
106	124	if (!ASSERT_OK_PTR(skel, "uprobe_syscall__open_and_load"))
107	125	goto cleanup;
108	126
109		− err = uprobe_syscall__attach(skel);
110		− if (!ASSERT_OK(err, "uprobe_syscall__attach"))
	127	+ skel->links.probe = bpf_program__attach_uprobe_opts(skel->progs.probe,
	128	+ 0, "/proc/self/exe", offset, &opts);
	129	+ if (!ASSERT_OK_PTR(skel->links.probe, "bpf_program__attach_uprobe_opts"))
111	130	goto cleanup;
112	131
113		− uretprobe_regs(&before, &after);
	132	+ /* make sure uprobe gets optimized */
	133	+ if (!retprobe)
	134	+ uprobe_regs_trigger();
	135	+
	136	+ uprobe_regs(&before, &after);
114	137
115	138	pp = (unsigned long *) &skel->bss->regs;
116	139	cnt = sizeof(before)/sizeof(*pb);
@@ -119,7 +142,7 @@ static void test_uretprobe_regs_equal(void) · 2 unmodified lines
119	142	unsigned int offset = i * sizeof(unsigned long);
120	143
121	144	/*
122		− * Check register before and after uretprobe_regs_trigger call
	145	+ * Check register before and after uprobe_regs_trigger call
123	146	* that triggers the uretprobe.
124	147	*/
125	148	switch (offset) {
@@ -133,7 +156,7 @@ static void test_uretprobe_regs_equal(void) · 7 unmodified lines
133	156
134	157	/*
135	158	* Check register seen from bpf program and register after
136		− * uretprobe_regs_trigger call
	159	+ * uprobe_regs_trigger call (with rax exception, check below).
137	160	*/
138	161	switch (offset) {
139	162	/*
@@ -146,6 +169,15 @@ static void test_uretprobe_regs_equal(void) · 6 unmodified lines
146	169	case offsetof(struct pt_regs, rsp):
147	170	case offsetof(struct pt_regs, ss):
148	171	break;
	172	+ /*
	173	+ * uprobe does not see return value in rax, it needs to see the
	174	+ * original (before) rax value
	175	+ */
	176	+ case offsetof(struct pt_regs, rax):
	177	+ if (!retprobe) {
	178	+ ASSERT_EQ(pp[i], pb[i], "uprobe rax prog-before value check");
	179	+ break;
	180	+ }
149	181	default:
150	182	if (!ASSERT_EQ(pp[i], pa[i], "register prog-after value check"))
151	183	fprintf(stdout, "failed register offset %u\n", offset);
@@ -175,7 +207,7 @@ static int write_bpf_testmod_uprobe(unsigned long offset) · 23 unmodified lines
175	207	return ret != n ? (int) ret : 0;
176	208	}
177	209
178		−static void test_uretprobe_regs_change(void)
	210	+static void test_regs_change(void)
179	211	{
180	212	struct pt_regs before = {}, after = {};
181	213	unsigned long pb = (unsigned long ) &before;
@@ -183,13 +215,16 @@ static void test_uretprobe_regs_change(void) · 1 unmodified line
183	215	unsigned long cnt = sizeof(before)/sizeof(*pb);
184	216	unsigned int i, err, offset;
185	217
186		− offset = get_uprobe_offset(uretprobe_regs_trigger);
	218	+ offset = get_uprobe_offset(uprobe_regs_trigger);
187	219
188	220	err = write_bpf_testmod_uprobe(offset);
189	221	if (!ASSERT_OK(err, "register_uprobe"))
190	222	return;
191	223
192		− uretprobe_regs(&before, &after);
	224	+ /* make sure uprobe gets optimized */
	225	+ uprobe_regs_trigger();
	226	+
	227	+ uprobe_regs(&before, &after);
193	228
194	229	err = write_bpf_testmod_uprobe(0);
195	230	if (!ASSERT_OK(err, "unregister_uprobe"))
@@ -252,6 +287,7 @@ static void test_uretprobe_syscall_call(void) · 56 unmodified lines
252	287	);
253	288	struct uprobe_syscall_executed *skel;
254	289	int pid, status, err, go[2], c = 0;
	290	+ struct bpf_link *link;
255	291
256	292	if (!ASSERT_OK(pipe(go), "pipe"))
257	293	return;
@@ -277,11 +313,14 @@ static void test_uretprobe_syscall_call(void) · 19 unmodified lines
277	313	_exit(0);
278	314	}
279	315
280		− skel->links.test = bpf_program__attach_uprobe_multi(skel->progs.test, pid,
281		− "/proc/self/exe",
282		− "uretprobe_syscall_call", &opts);
283		− if (!ASSERT_OK_PTR(skel->links.test, "bpf_program__attach_uprobe_multi"))
	316	+ skel->bss->pid = pid;
	317	+
	318	+ link = bpf_program__attach_uprobe_multi(skel->progs.test_uretprobe_multi,
	319	+ pid, "/proc/self/exe",
	320	+ "uretprobe_syscall_call", &opts);
	321	+ if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi"))
284	322	goto cleanup;
	323	+ skel->links.test_uretprobe_multi = link;
285	324
286	325	/* kick the child */
287	326	write(go[1], &c, 1);
@@ -301,6 +340,256 @@ cleanup: · 13 unmodified lines
301	340	close(go[0]);
302	341	}
303	342
	343	+#define TRAMP "[uprobes-trampoline]"
	344	+
	345	+__attribute__((aligned(16)))
	346	+__nocf_check __weak __naked void uprobe_test(void)
	347	+{
	348	+ asm volatile (" \n"
	349	+ ".byte 0x0f, 0x1f, 0x44, 0x00, 0x00 \n"
	350	+ "ret \n"
	351	+ );
	352	+}
	353	+
	354	+__attribute__((aligned(16)))
	355	+__nocf_check __weak void usdt_test(void)
	356	+{
	357	+ USDT(optimized_uprobe, usdt);
	358	+}
	359	+
	360	+static int find_uprobes_trampoline(void *tramp_addr)
	361	+{
	362	+ void start, end;
	363	+ char line[128];
	364	+ int ret = -1;
	365	+ FILE *maps;
	366	+
	367	+ maps = fopen("/proc/self/maps", "r");
	368	+ if (!maps) {
	369	+ fprintf(stderr, "cannot open maps\n");
	370	+ return -1;
	371	+ }
	372	+
	373	+ while (fgets(line, sizeof(line), maps)) {
	374	+ int m = -1;
	375	+
	376	+ /* We care only about private r-x mappings. */
	377	+ if (sscanf(line, "%p-%p r-xp %x %x:%x %u %n", &start, &end, &m) != 2)
	378	+ continue;
	379	+ if (m < 0)
	380	+ continue;
	381	+ if (!strncmp(&line[m], TRAMP, sizeof(TRAMP)-1) && (start == tramp_addr)) {
	382	+ ret = 0;
	383	+ break;
	384	+ }
	385	+ }
	386	+
	387	+ fclose(maps);
	388	+ return ret;
	389	+}
	390	+
	391	+static unsigned char nop5[5] = { 0x0f, 0x1f, 0x44, 0x00, 0x00 };
	392	+
	393	+static void find_nop5(void fn)
	394	+{
	395	+ int i;
	396	+
	397	+ for (i = 0; i < 10; i++) {
	398	+ if (!memcmp(nop5, fn + i, 5))
	399	+ return fn + i;
	400	+ }
	401	+ return NULL;
	402	+}
	403	+
	404	+typedef void (__attribute__((nocf_check)) *trigger_t)(void);
	405	+
	406	+static void check_attach(struct uprobe_syscall_executed skel, trigger_t trigger,
	407	+ void *addr, int executed)
	408	+{
	409	+ struct __arch_relative_insn {
	410	+ __u8 op;
	411	+ __s32 raddr;
	412	+ } __packed *call;
	413	+ void *tramp = NULL;
	414	+
	415	+ /* Uprobe gets optimized after first trigger, so let's press twice. */
	416	+ trigger();
	417	+ trigger();
	418	+
	419	+ /* Make sure bpf program got executed.. */
	420	+ ASSERT_EQ(skel->bss->executed, executed, "executed");
	421	+
	422	+ /* .. and check the trampoline is as expected. */
	423	+ call = (struct __arch_relative_insn *) addr;
	424	+ tramp = (void *) (call + 1) + call->raddr;
	425	+ ASSERT_EQ(call->op, 0xe8, "call");
	426	+ ASSERT_OK(find_uprobes_trampoline(tramp), "uprobes_trampoline");
	427	+
	428	+ return tramp;
	429	+}
	430	+
	431	+static void check_detach(void addr, void tramp)
	432	+{
	433	+ /* [uprobes_trampoline] stays after detach */
	434	+ ASSERT_OK(find_uprobes_trampoline(tramp), "uprobes_trampoline");
	435	+ ASSERT_OK(memcmp(addr, nop5, 5), "nop5");
	436	+}
	437	+
	438	+static void check(struct uprobe_syscall_executed skel, struct bpf_link link,
	439	+ trigger_t trigger, void *addr, int executed)
	440	+{
	441	+ void *tramp;
	442	+
	443	+ tramp = check_attach(skel, trigger, addr, executed);
	444	+ bpf_link__destroy(link);
	445	+ check_detach(addr, tramp);
	446	+}
	447	+
	448	+static void test_uprobe_legacy(void)
	449	+{
	450	+ struct uprobe_syscall_executed *skel = NULL;
	451	+ LIBBPF_OPTS(bpf_uprobe_opts, opts,
	452	+ .retprobe = true,
	453	+ );
	454	+ struct bpf_link *link;
	455	+ unsigned long offset;
	456	+
	457	+ offset = get_uprobe_offset(&uprobe_test);
	458	+ if (!ASSERT_GE(offset, 0, "get_uprobe_offset"))
	459	+ goto cleanup;
	460	+
	461	+ /* uprobe */
	462	+ skel = uprobe_syscall_executed__open_and_load();
	463	+ if (!ASSERT_OK_PTR(skel, "uprobe_syscall_executed__open_and_load"))
	464	+ return;
	465	+
	466	+ skel->bss->pid = getpid();
	467	+
	468	+ link = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe,
	469	+ 0, "/proc/self/exe", offset, NULL);
	470	+ if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_opts"))
	471	+ goto cleanup;
	472	+
	473	+ check(skel, link, uprobe_test, uprobe_test, 2);
	474	+
	475	+ /* uretprobe */
	476	+ skel->bss->executed = 0;
	477	+
	478	+ link = bpf_program__attach_uprobe_opts(skel->progs.test_uretprobe,
	479	+ 0, "/proc/self/exe", offset, &opts);
	480	+ if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_opts"))
	481	+ goto cleanup;
	482	+
	483	+ check(skel, link, uprobe_test, uprobe_test, 2);
	484	+
	485	+cleanup:
	486	+ uprobe_syscall_executed__destroy(skel);
	487	+}
	488	+
	489	+static void test_uprobe_multi(void)
	490	+{
	491	+ struct uprobe_syscall_executed *skel = NULL;
	492	+ LIBBPF_OPTS(bpf_uprobe_multi_opts, opts);
	493	+ struct bpf_link *link;
	494	+ unsigned long offset;
	495	+
	496	+ offset = get_uprobe_offset(&uprobe_test);
	497	+ if (!ASSERT_GE(offset, 0, "get_uprobe_offset"))
	498	+ goto cleanup;
	499	+
	500	+ opts.offsets = &offset;
	501	+ opts.cnt = 1;
	502	+
	503	+ skel = uprobe_syscall_executed__open_and_load();
	504	+ if (!ASSERT_OK_PTR(skel, "uprobe_syscall_executed__open_and_load"))
	505	+ return;
	506	+
	507	+ skel->bss->pid = getpid();
	508	+
	509	+ /* uprobe.multi */
	510	+ link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_multi,
	511	+ 0, "/proc/self/exe", NULL, &opts);
	512	+ if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi"))
	513	+ goto cleanup;
	514	+
	515	+ check(skel, link, uprobe_test, uprobe_test, 2);
	516	+
	517	+ /* uretprobe.multi */
	518	+ skel->bss->executed = 0;
	519	+ opts.retprobe = true;
	520	+ link = bpf_program__attach_uprobe_multi(skel->progs.test_uretprobe_multi,
	521	+ 0, "/proc/self/exe", NULL, &opts);
	522	+ if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi"))
	523	+ goto cleanup;
	524	+
	525	+ check(skel, link, uprobe_test, uprobe_test, 2);
	526	+
	527	+cleanup:
	528	+ uprobe_syscall_executed__destroy(skel);
	529	+}
	530	+
	531	+static void test_uprobe_session(void)
	532	+{
	533	+ struct uprobe_syscall_executed *skel = NULL;
	534	+ LIBBPF_OPTS(bpf_uprobe_multi_opts, opts,
	535	+ .session = true,
	536	+ );
	537	+ struct bpf_link *link;
	538	+ unsigned long offset;
	539	+
	540	+ offset = get_uprobe_offset(&uprobe_test);
	541	+ if (!ASSERT_GE(offset, 0, "get_uprobe_offset"))
	542	+ goto cleanup;
	543	+
	544	+ opts.offsets = &offset;
	545	+ opts.cnt = 1;
	546	+
	547	+ skel = uprobe_syscall_executed__open_and_load();
	548	+ if (!ASSERT_OK_PTR(skel, "uprobe_syscall_executed__open_and_load"))
	549	+ return;
	550	+
	551	+ skel->bss->pid = getpid();
	552	+
	553	+ link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_session,
	554	+ 0, "/proc/self/exe", NULL, &opts);
	555	+ if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi"))
	556	+ goto cleanup;
	557	+
	558	+ check(skel, link, uprobe_test, uprobe_test, 4);
	559	+
	560	+cleanup:
	561	+ uprobe_syscall_executed__destroy(skel);
	562	+}
	563	+
	564	+static void test_uprobe_usdt(void)
	565	+{
	566	+ struct uprobe_syscall_executed *skel;
	567	+ struct bpf_link *link;
	568	+ void *addr;
	569	+
	570	+ errno = 0;
	571	+ addr = find_nop5(usdt_test);
	572	+ if (!ASSERT_OK_PTR(addr, "find_nop5"))
	573	+ return;
	574	+
	575	+ skel = uprobe_syscall_executed__open_and_load();
	576	+ if (!ASSERT_OK_PTR(skel, "uprobe_syscall_executed__open_and_load"))
	577	+ return;
	578	+
	579	+ skel->bss->pid = getpid();
	580	+
	581	+ link = bpf_program__attach_usdt(skel->progs.test_usdt,
	582	+ -1 /* all PIDs */, "/proc/self/exe",
	583	+ "optimized_uprobe", "usdt", NULL);
	584	+ if (!ASSERT_OK_PTR(link, "bpf_program__attach_usdt"))
	585	+ goto cleanup;
	586	+
	587	+ check(skel, link, usdt_test, addr, 2);
	588	+
	589	+cleanup:
	590	+ uprobe_syscall_executed__destroy(skel);
	591	+}
	592	+
304	593	/*
305	594	* Borrowed from tools/testing/selftests/x86/test_shadow_stack.c.
306	595	*
@@ -343,43 +632,172 @@ static void test_uretprobe_shadow_stack(void) · 36 unmodified lines
343	632	return;
344	633	}
345	634
346		− /* Run all of the uretprobe tests. */
347		− test_uretprobe_regs_equal();
348		− test_uretprobe_regs_change();
	635	+ /* Run all the tests with shadow stack in place. */
	636	+
	637	+ test_uprobe_regs_equal(false);
	638	+ test_uprobe_regs_equal(true);
349	639	test_uretprobe_syscall_call();
350	640
	641	+ test_uprobe_legacy();
	642	+ test_uprobe_multi();
	643	+ test_uprobe_session();
	644	+ test_uprobe_usdt();
	645	+
	646	+ test_regs_change();
	647	+
351	648	ARCH_PRCTL(ARCH_SHSTK_DISABLE, ARCH_SHSTK_SHSTK);
352	649	}
353		−#else
354		−static void test_uretprobe_regs_equal(void)
	650	+
	651	+static volatile bool race_stop;
	652	+
	653	+static USDT_DEFINE_SEMA(race);
	654	+
	655	+static void worker_trigger(void arg)
355	656	{
356		− test__skip();
	657	+ unsigned long rounds = 0;
	658	+
	659	+ while (!race_stop) {
	660	+ uprobe_test();
	661	+ rounds++;
	662	+ }
	663	+
	664	+ printf("tid %d trigger rounds: %lu\n", gettid(), rounds);
	665	+ return NULL;
357	666	}
358	667
359		−static void test_uretprobe_regs_change(void)
	668	+static void worker_attach(void arg)
360	669	{
361		− test__skip();
	670	+ LIBBPF_OPTS(bpf_uprobe_opts, opts);
	671	+ struct uprobe_syscall_executed *skel;
	672	+ unsigned long rounds = 0, offset;
	673	+ const char *sema[2] = {
	674	+ __stringify(USDT_SEMA(race)),
	675	+ NULL,
	676	+ };
	677	+ unsigned long *ref;
	678	+ int err;
	679	+
	680	+ offset = get_uprobe_offset(&uprobe_test);
	681	+ if (!ASSERT_GE(offset, 0, "get_uprobe_offset"))
	682	+ return NULL;
	683	+
	684	+ err = elf_resolve_syms_offsets("/proc/self/exe", 1, (const char **) &sema, &ref, STT_OBJECT);
	685	+ if (!ASSERT_OK(err, "elf_resolve_syms_offsets_sema"))
	686	+ return NULL;
	687	+
	688	+ opts.ref_ctr_offset = *ref;
	689	+
	690	+ skel = uprobe_syscall_executed__open_and_load();
	691	+ if (!ASSERT_OK_PTR(skel, "uprobe_syscall_executed__open_and_load"))
	692	+ return NULL;
	693	+
	694	+ skel->bss->pid = getpid();
	695	+
	696	+ while (!race_stop) {
	697	+ skel->links.test_uprobe = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe,
	698	+ 0, "/proc/self/exe", offset, &opts);
	699	+ if (!ASSERT_OK_PTR(skel->links.test_uprobe, "bpf_program__attach_uprobe_opts"))
	700	+ break;
	701	+
	702	+ bpf_link__destroy(skel->links.test_uprobe);
	703	+ skel->links.test_uprobe = NULL;
	704	+ rounds++;
	705	+ }
	706	+
	707	+ printf("tid %d attach rounds: %lu hits: %d\n", gettid(), rounds, skel->bss->executed);
	708	+ uprobe_syscall_executed__destroy(skel);
	709	+ free(ref);
	710	+ return NULL;
362	711	}
363	712
364		−static void test_uretprobe_syscall_call(void)
	713	+static useconds_t race_msec(void)
365	714	{
366		− test__skip();
	715	+ char *env;
	716	+
	717	+ env = getenv("BPF_SELFTESTS_UPROBE_SYSCALL_RACE_MSEC");
	718	+ if (env)
	719	+ return atoi(env);
	720	+
	721	+ /* default duration is 500ms */
	722	+ return 500;
367	723	}
368	724
369		−static void test_uretprobe_shadow_stack(void)
	725	+static void test_uprobe_race(void)
370	726	{
371		− test__skip();
	727	+ int err, i, nr_threads;
	728	+ pthread_t *threads;
	729	+
	730	+ nr_threads = libbpf_num_possible_cpus();
	731	+ if (!ASSERT_GT(nr_threads, 0, "libbpf_num_possible_cpus"))
	732	+ return;
	733	+ nr_threads = max(2, nr_threads);
	734	+
	735	+ threads = alloca(sizeof(threads) nr_threads);
	736	+ if (!ASSERT_OK_PTR(threads, "malloc"))
	737	+ return;
	738	+
	739	+ for (i = 0; i < nr_threads; i++) {
	740	+ err = pthread_create(&threads[i], NULL, i % 2 ? worker_trigger : worker_attach,
	741	+ NULL);
	742	+ if (!ASSERT_OK(err, "pthread_create"))
	743	+ goto cleanup;
	744	+ }
	745	+
	746	+ usleep(race_msec() * 1000);
	747	+
	748	+cleanup:
	749	+ race_stop = true;
	750	+ for (nr_threads = i, i = 0; i < nr_threads; i++)
	751	+ pthread_join(threads[i], NULL);
	752	+
	753	+ ASSERT_FALSE(USDT_SEMA_IS_ACTIVE(race), "race_semaphore");
372	754	}
	755	+
	756	+#ifndef __NR_uprobe
	757	+#define __NR_uprobe 336
373	758	#endif
374	759
375		−void test_uprobe_syscall(void)
	760	+static void test_uprobe_error(void)
	761	+{
	762	+ long err = syscall(__NR_uprobe);
	763	+
	764	+ ASSERT_EQ(err, -1, "error");
	765	+ ASSERT_EQ(errno, ENXIO, "errno");
	766	+}
	767	+
	768	+static void __test_uprobe_syscall(void)
376	769	{
377	770	if (test__start_subtest("uretprobe_regs_equal"))
378		− test_uretprobe_regs_equal();
379		− if (test__start_subtest("uretprobe_regs_change"))
380		− test_uretprobe_regs_change();
	771	+ test_uprobe_regs_equal(true);
381	772	if (test__start_subtest("uretprobe_syscall_call"))
382	773	test_uretprobe_syscall_call();
383	774	if (test__start_subtest("uretprobe_shadow_stack"))
384	775	test_uretprobe_shadow_stack();
	776	+ if (test__start_subtest("uprobe_legacy"))
	777	+ test_uprobe_legacy();
	778	+ if (test__start_subtest("uprobe_multi"))
	779	+ test_uprobe_multi();
	780	+ if (test__start_subtest("uprobe_session"))
	781	+ test_uprobe_session();
	782	+ if (test__start_subtest("uprobe_usdt"))
	783	+ test_uprobe_usdt();
	784	+ if (test__start_subtest("uprobe_race"))
	785	+ test_uprobe_race();
	786	+ if (test__start_subtest("uprobe_error"))
	787	+ test_uprobe_error();
	788	+ if (test__start_subtest("uprobe_regs_equal"))
	789	+ test_uprobe_regs_equal(false);
	790	+ if (test__start_subtest("regs_change"))
	791	+ test_regs_change();
	792	+}
	793	+#else
	794	+static void __test_uprobe_syscall(void)
	795	+{
	796	+ test__skip();
	797	+}
	798	+#endif
	799	+
	800	+void test_uprobe_syscall(void)
	801	+{
	802	+ __test_uprobe_syscall();
385	803	}

tools/testing/selftests/bpf/prog_tests/usdt.c+25−13

@@ -40,12 +40,19 @@ static void __always_inline trigger_func(int x) { · 39 unmodified lines
40	40	}
41	41	}
42	42
43		−static void subtest_basic_usdt(void)
	43	+static void subtest_basic_usdt(bool optimized)
44	44	{
45	45	LIBBPF_OPTS(bpf_usdt_opts, opts);
46	46	struct test_usdt *skel;
47	47	struct test_usdt__bss *bss;
48		− int err, i;
	48	+ int err, i, called;
	49	+
	50	+#define TRIGGER(x) ({ \
	51	+ trigger_func(x); \
	52	+ if (optimized) \
	53	+ trigger_func(x); \
	54	+ optimized ? 2 : 1; \
	55	+ })
49	56
50	57	skel = test_usdt__open_and_load();
51	58	if (!ASSERT_OK_PTR(skel, "skel_open"))
@@ -66,11 +73,11 @@ static void subtest_basic_usdt(void) · 14 unmodified lines
66	73	if (!ASSERT_OK_PTR(skel->links.usdt0, "usdt0_link"))
67	74	goto cleanup;
68	75
69		− trigger_func(1);
	76	+ called = TRIGGER(1);
70	77
71		− ASSERT_EQ(bss->usdt0_called, 1, "usdt0_called");
72		− ASSERT_EQ(bss->usdt3_called, 1, "usdt3_called");
73		− ASSERT_EQ(bss->usdt12_called, 1, "usdt12_called");
	78	+ ASSERT_EQ(bss->usdt0_called, called, "usdt0_called");
	79	+ ASSERT_EQ(bss->usdt3_called, called, "usdt3_called");
	80	+ ASSERT_EQ(bss->usdt12_called, called, "usdt12_called");
74	81
75	82	ASSERT_EQ(bss->usdt0_cookie, 0xcafedeadbeeffeed, "usdt0_cookie");
76	83	ASSERT_EQ(bss->usdt0_arg_cnt, 0, "usdt0_arg_cnt");
@@ -119,11 +126,11 @@ static void subtest_basic_usdt(void) · 42 unmodified lines
119	126	* bpf_program__attach_usdt() handles this properly and attaches to
120	127	* all possible places of USDT invocation.
121	128	*/
122		− trigger_func(2);
	129	+ called += TRIGGER(2);
123	130
124		− ASSERT_EQ(bss->usdt0_called, 2, "usdt0_called");
125		− ASSERT_EQ(bss->usdt3_called, 2, "usdt3_called");
126		− ASSERT_EQ(bss->usdt12_called, 2, "usdt12_called");
	131	+ ASSERT_EQ(bss->usdt0_called, called, "usdt0_called");
	132	+ ASSERT_EQ(bss->usdt3_called, called, "usdt3_called");
	133	+ ASSERT_EQ(bss->usdt12_called, called, "usdt12_called");
127	134
128	135	/* only check values that depend on trigger_func()'s input value */
129	136	ASSERT_EQ(bss->usdt3_args[0], 2, "usdt3_arg1");
@@ -142,9 +149,9 @@ static void subtest_basic_usdt(void) · 12 unmodified lines
142	149	if (!ASSERT_OK_PTR(skel->links.usdt3, "usdt3_reattach"))
143	150	goto cleanup;
144	151
145		− trigger_func(3);
	152	+ called += TRIGGER(3);
146	153
147		− ASSERT_EQ(bss->usdt3_called, 3, "usdt3_called");
	154	+ ASSERT_EQ(bss->usdt3_called, called, "usdt3_called");
148	155	/* this time usdt3 has custom cookie */
149	156	ASSERT_EQ(bss->usdt3_cookie, 0xBADC00C51E, "usdt3_cookie");
150	157	ASSERT_EQ(bss->usdt3_arg_cnt, 3, "usdt3_arg_cnt");
@@ -158,6 +165,7 @@ static void subtest_basic_usdt(void) · 7 unmodified lines
158	165
159	166	cleanup:
160	167	test_usdt__destroy(skel);
	168	+#undef TRIGGER
161	169	}
162	170
163	171	unsigned short test_usdt_100_semaphore SEC(".probes");
@@ -425,7 +433,11 @@ cleanup: · 261 unmodified lines
425	433	void test_usdt(void)
426	434	{
427	435	if (test__start_subtest("basic"))
428		− subtest_basic_usdt();
	436	+ subtest_basic_usdt(false);
	437	+#ifdef __x86_64__
	438	+ if (test__start_subtest("basic_optimized"))
	439	+ subtest_basic_usdt(true);
	440	+#endif
429	441	if (test__start_subtest("multispec"))
430	442	subtest_multispec_usdt();
431	443	if (test__start_subtest("urand_auto_attach"))

tools/testing/selftests/bpf/progs/uprobe_syscall.c+2−2

@@ -7,8 +7,8 @@ struct pt_regs regs; · 6 unmodified lines
7	7
8	8	char _license[] SEC("license") = "GPL";
9	9
10		−SEC("uretprobe//proc/self/exe:uretprobe_regs_trigger")
11		−int uretprobe(struct pt_regs *ctx)
	10	+SEC("uprobe")
	11	+int probe(struct pt_regs *ctx)
12	12	{
13	13	__builtin_memcpy(&regs, ctx, sizeof(regs));
14	14	return 0;

tools/testing/selftests/bpf/progs/uprobe_syscall_executed.c+58−2

@@ -1,6 +1,8 @@
1	1	// SPDX-License-Identifier: GPL-2.0
2	2	#include "vmlinux.h"
3	3	#include <bpf/bpf_helpers.h>
	4	+#include <bpf/bpf_tracing.h>
	5	+#include <bpf/usdt.bpf.h>
4	6	#include <string.h>
5	7
6	8	struct pt_regs regs;
@@ -8,10 +10,64 @@ struct pt_regs regs; · 1 unmodified line
8	10	char _license[] SEC("license") = "GPL";
9	11
10	12	int executed = 0;
	13	+int pid;
	14	+
	15	+SEC("uprobe")
	16	+int BPF_UPROBE(test_uprobe)
	17	+{
	18	+ if (bpf_get_current_pid_tgid() >> 32 != pid)
	19	+ return 0;
	20	+
	21	+ executed++;
	22	+ return 0;
	23	+}
	24	+
	25	+SEC("uretprobe")
	26	+int BPF_URETPROBE(test_uretprobe)
	27	+{
	28	+ if (bpf_get_current_pid_tgid() >> 32 != pid)
	29	+ return 0;
	30	+
	31	+ executed++;
	32	+ return 0;
	33	+}
	34	+
	35	+SEC("uprobe.multi")
	36	+int test_uprobe_multi(struct pt_regs *ctx)
	37	+{
	38	+ if (bpf_get_current_pid_tgid() >> 32 != pid)
	39	+ return 0;
	40	+
	41	+ executed++;
	42	+ return 0;
	43	+}
11	44
12	45	SEC("uretprobe.multi")
13		−int test(struct pt_regs *regs)
	46	+int test_uretprobe_multi(struct pt_regs *ctx)
	47	+{
	48	+ if (bpf_get_current_pid_tgid() >> 32 != pid)
	49	+ return 0;
	50	+
	51	+ executed++;
	52	+ return 0;
	53	+}
	54	+
	55	+SEC("uprobe.session")
	56	+int test_uprobe_session(struct pt_regs *ctx)
14	57	{
15		− executed = 1;
	58	+ if (bpf_get_current_pid_tgid() >> 32 != pid)
	59	+ return 0;
	60	+
	61	+ executed++;
	62	+ return 0;
	63	+}
	64	+
	65	+SEC("usdt")
	66	+int test_usdt(struct pt_regs *ctx)
	67	+{
	68	+ if (bpf_get_current_pid_tgid() >> 32 != pid)
	69	+ return 0;
	70	+
	71	+ executed++;
16	72	return 0;
17	73	}

tools/testing/selftests/bpf/test_kmods/bpf_testmod.c+9−2

@@ -500,15 +500,21 @@ static struct bin_attribute bin_attr_bpf_testmod_file __ro_after_init = { · 499 unmodified lines
500	500	*/
501	501	#ifdef __x86_64__
502	502
	503	+static int
	504	+uprobe_handler(struct uprobe_consumer self, struct pt_regs regs, __u64 *data)
	505	+{
	506	+ regs->cx = 0x87654321feebdaed;
	507	+ return 0;
	508	+}
	509	+
503	510	static int
504	511	uprobe_ret_handler(struct uprobe_consumer *self, unsigned long func,
505	512	struct pt_regs regs, __u64 data)
506	513
507	514	{
508	515	regs->ax = 0x12345678deadbeef;
509		− regs->cx = 0x87654321feebdaed;
510	516	regs->r11 = (u64) -1;
511		− return true;
	517	+ return 0;
512	518	}
513	519
514	520	struct testmod_uprobe {
@@ -520,6 +526,7 @@ struct testmod_uprobe { · 5 unmodified lines
520	526	static DEFINE_MUTEX(testmod_uprobe_mutex);
521	527
522	528	static struct testmod_uprobe uprobe = {
	529	+ .consumer.handler = uprobe_handler,
523	530	.consumer.ret_handler = uprobe_ret_handler,
524	531	};
525	532

tools/testing/selftests/bpf/usdt.h+545−0

@@ -0,0 +1,545 @@
	1	+// SPDX-License-Identifier: BSD-2-Clause
	2	+/*
	3	+ * This single-header library defines a collection of variadic macros for
	4	+ * defining and triggering USDTs (User Statically-Defined Tracepoints):
	5	+ *
	6	+ * - For USDTs without associated semaphore:
	7	+ * USDT(group, name, args...)
	8	+ *
	9	+ * - For USDTs with implicit (transparent to the user) semaphore:
	10	+ * USDT_WITH_SEMA(group, name, args...)
	11	+ * USDT_IS_ACTIVE(group, name)
	12	+ *
	13	+ * - For USDTs with explicit (user-defined and provided) semaphore:
	14	+ * USDT_WITH_EXPLICIT_SEMA(sema, group, name, args...)
	15	+ * USDT_SEMA_IS_ACTIVE(sema)
	16	+ *
	17	+ * all of which emit a NOP instruction into the instruction stream, and so
	18	+ * have zero overhead for the surrounding code. USDTs are identified by
	19	+ * a combination of `group` and `name` identifiers, which is used by external
	20	+ * tracing tooling (tracers) for identifying exact USDTs of interest.
	21	+ *
	22	+ * USDTs can have an associated (2-byte) activity counter (USDT semaphore),
	23	+ * automatically maintained by Linux kernel whenever any correctly written
	24	+ * BPF-based tracer is attached to the USDT. This USDT semaphore can be used
	25	+ * to check whether there is a need to do any extra data collection and
	26	+ * processing for a given USDT (if necessary), and otherwise avoid extra work
	27	+ * for a common case of USDT not being traced ("active").
	28	+ *
	29	+ * See documentation for USDT_WITH_SEMA()/USDT_IS_ACTIVE() or
	30	+ * USDT_WITH_EXPLICIT_SEMA()/USDT_SEMA_IS_ACTIVE() APIs below for details on
	31	+ * working with USDTs with implicitly or explicitly associated
	32	+ * USDT semaphores, respectively.
	33	+ *
	34	+ * There is also some additional data recorded into an auxiliary note
	35	+ * section. The data in the note section describes the operands, in terms of
	36	+ * size and location, used by tracing tooling to know where to find USDT
	37	+ * arguments. Each location is encoded as an assembler operand string.
	38	+ * Tracing tools (bpftrace and BPF-based tracers, systemtap, etc) insert
	39	+ * breakpoints on top of the nop, and decode the location operand-strings,
	40	+ * like an assembler, to find the values being passed.
	41	+ *
	42	+ * The operand strings are selected by the compiler for each operand.
	43	+ * They are constrained by inline-assembler codes.The default is:
	44	+ *
	45	+ * #define USDT_ARG_CONSTRAINT nor
	46	+ *
	47	+ * This is a good default if the operands tend to be integral and
	48	+ * moderate in number (smaller than number of registers). In other
	49	+ * cases, the compiler may report "'asm' requires impossible reload" or
	50	+ * similar. In this case, consider simplifying the macro call (fewer
	51	+ * and simpler operands), reduce optimization, or override the default
	52	+ * constraints string via:
	53	+ *
	54	+ * #define USDT_ARG_CONSTRAINT g
	55	+ * #include <usdt.h>
	56	+ *
	57	+ * For some historical description of USDT v3 format (the one used by this
	58	+ * library and generally recognized and assumed by BPF-based tracing tools)
	59	+ * see [0]. The more formal specification can be found at [1]. Additional
	60	+ * argument constraints information can be found at [2].
	61	+ *
	62	+ * Original SystemTap's sys/sdt.h implementation ([3]) was used as a base for
	63	+ * this USDT library implementation. Current implementation differs a lot in
	64	+ * terms of exposed user API and general usability, which was the main goal
	65	+ * and focus of the reimplementation work. Nevertheless, underlying recorded
	66	+ * USDT definitions are fully binary compatible and any USDT-based tooling
	67	+ * should work equally well with USDTs defined by either SystemTap's or this
	68	+ * library's USDT implementation.
	69	+ *
	70	+ * [0] https://ecos.sourceware.org/ml/systemtap/2010-q3/msg00145.html
	71	+ * [1] https://sourceware.org/systemtap/wiki/UserSpaceProbeImplementation
	72	+ * [2] https://gcc.gnu.org/onlinedocs/gcc/Constraints.html
	73	+ * [3] https://sourceware.org/git/?p=systemtap.git;a=blob;f=includes/sys/sdt.h
	74	+ */
	75	+#ifndef __USDT_H
	76	+#define __USDT_H
	77	+
	78	+/*
	79	+ * Changelog:
	80	+ *
	81	+ * 0.1.0
	82	+ * -----
	83	+ * - Initial release
	84	+ */
	85	+#define USDT_MAJOR_VERSION 0
	86	+#define USDT_MINOR_VERSION 1
	87	+#define USDT_PATCH_VERSION 0
	88	+
	89	+/* C++20 and C23 added __VA_OPT__ as a standard replacement for non-standard `##__VA_ARGS__` extension */
	90	+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ > 201710L) \|\| (defined(__cplusplus) && __cplusplus > 201703L)
	91	+#define __usdt_va_opt 1
	92	+#define __usdt_va_args(...) __VA_OPT__(,) __VA_ARGS__
	93	+#else
	94	+#define __usdt_va_args(...) , ##__VA_ARGS__
	95	+#endif
	96	+
	97	+/*
	98	+ * Trigger USDT with `group`:`name` identifier and pass through `args` as its
	99	+ * arguments. Zero arguments are acceptable as well. No USDT semaphore is
	100	+ * associated with this USDT.
	101	+ *
	102	+ * Such "semaphoreless" USDTs are commonly used when there is no extra data
	103	+ * collection or processing needed to collect and prepare USDT arguments and
	104	+ * they are just available in the surrounding code. USDT() macro will just
	105	+ * record their locations in CPU registers or in memory for tracing tooling to
	106	+ * be able to access them, if necessary.
	107	+ */
	108	+#ifdef __usdt_va_opt
	109	+#define USDT(group, name, ...) \
	110	+ __usdt_probe(group, name, __usdt_sema_none, 0 __VA_OPT__(,) __VA_ARGS__)
	111	+#else
	112	+#define USDT(group, name, ...) \
	113	+ __usdt_probe(group, name, __usdt_sema_none, 0, ##__VA_ARGS__)
	114	+#endif
	115	+
	116	+/*
	117	+ * Trigger USDT with `group`:`name` identifier and pass through `args` as its
	118	+ * arguments. Zero arguments are acceptable as well. USDT also get an
	119	+ * implicitly-defined associated USDT semaphore, which will be "activated" by
	120	+ * tracing tooling and can be used to check whether USDT is being actively
	121	+ * observed.
	122	+ *
	123	+ * USDTs with semaphore are commonly used when there is a need to perform
	124	+ * additional data collection and processing to prepare USDT arguments, which
	125	+ * otherwise might not be necessary for the rest of application logic. In such
	126	+ * case, USDT semaphore can be used to avoid unnecessary extra work. If USDT
	127	+ * is not traced (which is presumed to be a common situation), the associated
	128	+ * USDT semaphore is "inactive", and so there is no need to waste resources to
	129	+ * prepare USDT arguments. Use USDT_IS_ACTIVE(group, name) to check whether
	130	+ * USDT is "active".
	131	+ *
	132	+ * N.B. There is an inherent (albeit short) gap between checking whether USDT
	133	+ * is active and triggering corresponding USDT, in which external tracer can
	134	+ * be attached to an USDT and activate USDT semaphore after the activity check.
	135	+ * If such a race occurs, tracers might miss one USDT execution. Tracers are
	136	+ * expected to accommodate such possibility and this is expected to not be
	137	+ * a problem for applications and tracers.
	138	+ *
	139	+ * N.B. Implicit USDT semaphore defined by USDT_WITH_SEMA() is contained
	140	+ * within a single executable or shared library and is not shared outside
	141	+ * them. I.e., if you use USDT_WITH_SEMA() with the same USDT group and name
	142	+ * identifier across executable and shared library, it will work and won't
	143	+ * conflict, per se, but will define independent USDT semaphores, one for each
	144	+ * shared library/executable in which USDT_WITH_SEMA(group, name) is used.
	145	+ * That is, if you attach to this USDT in one shared library (or executable),
	146	+ * then only USDT semaphore within that shared library (or executable) will be
	147	+ * updated by the kernel, while other libraries (or executable) will not see
	148	+ * activated USDT semaphore. In short, it's best to use unique USDT group:name
	149	+ * identifiers across different shared libraries (and, equivalently, between
	150	+ * executable and shared library). This is advanced consideration and is
	151	+ * rarely (if ever) seen in practice, but just to avoid surprises this is
	152	+ * called out here. (Static libraries become a part of final executable, once
	153	+ * linked by linker, so the above considerations don't apply to them.)
	154	+ */
	155	+#ifdef __usdt_va_opt
	156	+#define USDT_WITH_SEMA(group, name, ...) \
	157	+ __usdt_probe(group, name, \
	158	+ __usdt_sema_implicit, __usdt_sema_name(group, name) \
	159	+ __VA_OPT__(,) __VA_ARGS__)
	160	+#else
	161	+#define USDT_WITH_SEMA(group, name, ...) \
	162	+ __usdt_probe(group, name, \
	163	+ __usdt_sema_implicit, __usdt_sema_name(group, name), \
	164	+ ##__VA_ARGS__)
	165	+#endif
	166	+
	167	+struct usdt_sema { volatile unsigned short active; };
	168	+
	169	+/*
	170	+ * Check if USDT with `group`:`name` identifier is "active" (i.e., whether it
	171	+ * is attached to by external tracing tooling and is actively observed).
	172	+ *
	173	+ * This macro can be used to decide whether any additional and potentially
	174	+ * expensive data collection or processing should be done to pass extra
	175	+ * information into the given USDT. It is assumed that USDT is triggered with
	176	+ * USDT_WITH_SEMA() macro which will implicitly define associated USDT
	177	+ * semaphore. (If one needs more control over USDT semaphore, see
	178	+ * USDT_DEFINE_SEMA() and USDT_WITH_EXPLICIT_SEMA() macros below.)
	179	+ *
	180	+ * N.B. Such checks are necessarily racy and speculative. Between checking
	181	+ * whether USDT is active and triggering the USDT itself, tracer can be
	182	+ * detached with no notification. This race should be extremely rare and worst
	183	+ * case should result in one-time wasted extra data collection and processing.
	184	+ */
	185	+#define USDT_IS_ACTIVE(group, name) ({ \
	186	+ extern struct usdt_sema __usdt_sema_name(group, name) \
	187	+ __usdt_asm_name(__usdt_sema_name(group, name)); \
	188	+ __usdt_sema_implicit(__usdt_sema_name(group, name)); \
	189	+ __usdt_sema_name(group, name).active > 0; \
	190	+})
	191	+
	192	+/*
	193	+ * APIs for working with user-defined explicit USDT semaphores.
	194	+ *
	195	+ * This is a less commonly used advanced API for use cases in which user needs
	196	+ * an explicit control over (potentially shared across multiple USDTs) USDT
	197	+ * semaphore instance. This can be used when there is a group of logically
	198	+ * related USDTs that all need extra data collection and processing whenever
	199	+ * any of a family of related USDTs are "activated" (i.e., traced). In such
	200	+ * a case, all such related USDTs will be associated with the same shared USDT
	201	+ * semaphore defined with USDT_DEFINE_SEMA() and the USDTs themselves will be
	202	+ * triggered with USDT_WITH_EXPLICIT_SEMA() macros, taking an explicit extra
	203	+ * USDT semaphore identifier as an extra parameter.
	204	+ */
	205	+
	206	+/**
	207	+ * Underlying C global variable name for user-defined USDT semaphore with
	208	+ * `sema` identifier. Could be useful for debugging, but normally shouldn't be
	209	+ * used explicitly.
	210	+ */
	211	+#define USDT_SEMA(sema) __usdt_sema_##sema
	212	+
	213	+/*
	214	+ * Define storage for user-defined USDT semaphore `sema`.
	215	+ *
	216	+ * Should be used only once in non-header source file to let compiler allocate
	217	+ * space for the semaphore variable. Just like with any other global variable.
	218	+ *
	219	+ * This macro can be used anywhere where global variable declaration is
	220	+ * allowed. Just like with global variable definitions, there should be only
	221	+ * one definition of user-defined USDT semaphore with given `sema` identifier,
	222	+ * otherwise compiler or linker will complain about duplicate variable
	223	+ * definition.
	224	+ *
	225	+ * For C++, it is allowed to use USDT_DEFINE_SEMA() both in global namespace
	226	+ * and inside namespaces (including nested namespaces). Just make sure that
	227	+ * USDT_DECLARE_SEMA() is placed within the namespace where this semaphore is
	228	+ * referenced, or any of its parent namespaces, so the C++ language-level
	229	+ * identifier is visible to the code that needs to reference the semaphore.
	230	+ * At the lowest layer, USDT semaphores have global naming and visibility
	231	+ * (they have a corresponding `__usdt_sema_<name>` symbol, which can be linked
	232	+ * against from C or C++ code, if necessary). To keep it simple, putting
	233	+ * USDT_DECLARE_SEMA() declarations into global namespaces is the simplest
	234	+ * no-brainer solution. All these aspects are irrelevant for plain C, because
	235	+ * C doesn't have namespaces and everything is always in the global namespace.
	236	+ *
	237	+ * N.B. Due to USDT metadata being recorded in non-allocatable ELF note
	238	+ * section, it has limitations when it comes to relocations, which, in
	239	+ * practice, means that it's not possible to correctly share USDT semaphores
	240	+ * between main executable and shared libraries, or even between multiple
	241	+ * shared libraries. USDT semaphore has to be contained to individual shared
	242	+ * library or executable to avoid unpleasant surprises with half-working USDT
	243	+ * semaphores. We enforce this by marking semaphore ELF symbols as having
	244	+ * a hidden visibility. This is quite an advanced use case and consideration
	245	+ * and for most users this should have no consequences whatsoever.
	246	+ */
	247	+#define USDT_DEFINE_SEMA(sema) \
	248	+ struct usdt_sema __usdt_sema_sec USDT_SEMA(sema) \
	249	+ __usdt_asm_name(USDT_SEMA(sema)) \
	250	+ __attribute__((visibility("hidden"))) = { 0 }
	251	+
	252	+/*
	253	+ * Declare extern reference to user-defined USDT semaphore `sema`.
	254	+ *
	255	+ * Refers to a variable defined in another compilation unit by
	256	+ * USDT_DEFINE_SEMA() and allows to use the same USDT semaphore across
	257	+ * multiple compilation units (i.e., .c and .cpp files).
	258	+ *
	259	+ * See USDT_DEFINE_SEMA() notes above for C++ language usage peculiarities.
	260	+ */
	261	+#define USDT_DECLARE_SEMA(sema) \
	262	+ extern struct usdt_sema USDT_SEMA(sema) __usdt_asm_name(USDT_SEMA(sema))
	263	+
	264	+/*
	265	+ * Check if user-defined USDT semaphore `sema` is "active" (i.e., whether it
	266	+ * is attached to by external tracing tooling and is actively observed).
	267	+ *
	268	+ * This macro can be used to decide whether any additional and potentially
	269	+ * expensive data collection or processing should be done to pass extra
	270	+ * information into USDT(s) associated with USDT semaphore `sema`.
	271	+ *
	272	+ * N.B. Such checks are necessarily racy. Between checking the state of USDT
	273	+ * semaphore and triggering associated USDT(s), the active tracer might attach
	274	+ * or detach. This race should be extremely rare and worst case should result
	275	+ * in one-time missed USDT event or wasted extra data collection and
	276	+ * processing. USDT-using tracers should be written with this in mind and is
	277	+ * not a concern of the application defining USDTs with associated semaphore.
	278	+ */
	279	+#define USDT_SEMA_IS_ACTIVE(sema) (USDT_SEMA(sema).active > 0)
	280	+
	281	+/*
	282	+ * Invoke USDT specified by `group` and `name` identifiers and associate
	283	+ * explicitly user-defined semaphore `sema` with it. Pass through `args` as
	284	+ * USDT arguments. `args` are optional and zero arguments are acceptable.
	285	+ *
	286	+ * Semaphore is defined with the help of USDT_DEFINE_SEMA() macro and can be
	287	+ * checked whether active with USDT_SEMA_IS_ACTIVE().
	288	+ */
	289	+#ifdef __usdt_va_opt
	290	+#define USDT_WITH_EXPLICIT_SEMA(sema, group, name, ...) \
	291	+ __usdt_probe(group, name, __usdt_sema_explicit, USDT_SEMA(sema), ##__VA_ARGS__)
	292	+#else
	293	+#define USDT_WITH_EXPLICIT_SEMA(sema, group, name, ...) \
	294	+ __usdt_probe(group, name, __usdt_sema_explicit, USDT_SEMA(sema) __VA_OPT__(,) __VA_ARGS__)
	295	+#endif
	296	+
	297	+/*
	298	+ * Adjustable implementation aspects
	299	+ */
	300	+#ifndef USDT_ARG_CONSTRAINT
	301	+#if defined __powerpc__
	302	+#define USDT_ARG_CONSTRAINT nZr
	303	+#elif defined __arm__
	304	+#define USDT_ARG_CONSTRAINT g
	305	+#elif defined __loongarch__
	306	+#define USDT_ARG_CONSTRAINT nmr
	307	+#else
	308	+#define USDT_ARG_CONSTRAINT nor
	309	+#endif
	310	+#endif /* USDT_ARG_CONSTRAINT */
	311	+
	312	+#ifndef USDT_NOP
	313	+#if defined(__ia64__) \|\| defined(__s390__) \|\| defined(__s390x__)
	314	+#define USDT_NOP nop 0
	315	+#else
	316	+#define USDT_NOP nop
	317	+#endif
	318	+#endif /* USDT_NOP */
	319	+
	320	+/*
	321	+ * Implementation details
	322	+ */
	323	+/* USDT name for implicitly-defined USDT semaphore, derived from group:name */
	324	+#define __usdt_sema_name(group, name) __usdt_sema_##group##__##name
	325	+/* ELF section into which USDT semaphores are put */
	326	+#define __usdt_sema_sec __attribute__((section(".probes")))
	327	+
	328	+#define __usdt_concat(a, b) a ## b
	329	+#define __usdt_apply(fn, n) __usdt_concat(fn, n)
	330	+
	331	+#ifndef __usdt_nth
	332	+#define __usdt_nth(_, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, N, ...) N
	333	+#endif
	334	+
	335	+#ifndef __usdt_narg
	336	+#ifdef __usdt_va_opt
	337	+#define __usdt_narg(...) __usdt_nth(_ __VA_OPT__(,) __VA_ARGS__, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)
	338	+#else
	339	+#define __usdt_narg(...) __usdt_nth(_, ##__VA_ARGS__, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)
	340	+#endif
	341	+#endif /* __usdt_narg */
	342	+
	343	+#define __usdt_hash #
	344	+#define __usdt_str_(x) #x
	345	+#define __usdt_str(x) __usdt_str_(x)
	346	+
	347	+#ifndef __usdt_asm_name
	348	+#define __usdt_asm_name(name) __asm__(__usdt_str(name))
	349	+#endif
	350	+
	351	+#define __usdt_asm0() "\n"
	352	+#define __usdt_asm1(x) __usdt_str(x) "\n"
	353	+#define __usdt_asm2(x, ...) __usdt_str(x) "," __usdt_asm1(__VA_ARGS__)
	354	+#define __usdt_asm3(x, ...) __usdt_str(x) "," __usdt_asm2(__VA_ARGS__)
	355	+#define __usdt_asm4(x, ...) __usdt_str(x) "," __usdt_asm3(__VA_ARGS__)
	356	+#define __usdt_asm5(x, ...) __usdt_str(x) "," __usdt_asm4(__VA_ARGS__)
	357	+#define __usdt_asm6(x, ...) __usdt_str(x) "," __usdt_asm5(__VA_ARGS__)
	358	+#define __usdt_asm7(x, ...) __usdt_str(x) "," __usdt_asm6(__VA_ARGS__)
	359	+#define __usdt_asm8(x, ...) __usdt_str(x) "," __usdt_asm7(__VA_ARGS__)
	360	+#define __usdt_asm9(x, ...) __usdt_str(x) "," __usdt_asm8(__VA_ARGS__)
	361	+#define __usdt_asm10(x, ...) __usdt_str(x) "," __usdt_asm9(__VA_ARGS__)
	362	+#define __usdt_asm11(x, ...) __usdt_str(x) "," __usdt_asm10(__VA_ARGS__)
	363	+#define __usdt_asm12(x, ...) __usdt_str(x) "," __usdt_asm11(__VA_ARGS__)
	364	+#define __usdt_asm(...) __usdt_apply(__usdt_asm, __usdt_narg(__VA_ARGS__))(__VA_ARGS__)
	365	+
	366	+#ifdef __LP64__
	367	+#define __usdt_asm_addr .8byte
	368	+#else
	369	+#define __usdt_asm_addr .4byte
	370	+#endif
	371	+
	372	+#define __usdt_asm_strz_(x) __usdt_asm1(.asciz #x)
	373	+#define __usdt_asm_strz(x) __usdt_asm_strz_(x)
	374	+#define __usdt_asm_str_(x) __usdt_asm1(.ascii #x)
	375	+#define __usdt_asm_str(x) __usdt_asm_str_(x)
	376	+
	377	+/* "semaphoreless" USDT case */
	378	+#ifndef __usdt_sema_none
	379	+#define __usdt_sema_none(sema)
	380	+#endif
	381	+
	382	+/* implicitly defined __usdt_sema__group__name semaphore (using weak symbols) */
	383	+#ifndef __usdt_sema_implicit
	384	+#define __usdt_sema_implicit(sema) \
	385	+ __asm__ __volatile__ ( \
	386	+ __usdt_asm1(.ifndef sema) \
	387	+ __usdt_asm3( .pushsection .probes, "aw", "progbits") \
	388	+ __usdt_asm1( .weak sema) \
	389	+ __usdt_asm1( .hidden sema) \
	390	+ __usdt_asm1( .align 2) \
	391	+ __usdt_asm1(sema:) \
	392	+ __usdt_asm1( .zero 2) \
	393	+ __usdt_asm2( .type sema, @object) \
	394	+ __usdt_asm2( .size sema, 2) \
	395	+ __usdt_asm1( .popsection) \
	396	+ __usdt_asm1(.endif) \
	397	+ );
	398	+#endif
	399	+
	400	+/* externally defined semaphore using USDT_DEFINE_SEMA() and passed explicitly by user */
	401	+#ifndef __usdt_sema_explicit
	402	+#define __usdt_sema_explicit(sema) \
	403	+ __asm__ __volatile__ ("" :: "m" (sema));
	404	+#endif
	405	+
	406	+/* main USDT definition (nop and .note.stapsdt metadata) */
	407	+#define __usdt_probe(group, name, sema_def, sema, ...) do { \
	408	+ sema_def(sema) \
	409	+ __asm__ __volatile__ ( \
	410	+ __usdt_asm( 990: USDT_NOP) \
	411	+ __usdt_asm3( .pushsection .note.stapsdt, "", "note") \
	412	+ __usdt_asm1( .balign 4) \
	413	+ __usdt_asm3( .4byte 992f-991f,994f-993f,3) \
	414	+ __usdt_asm1(991: .asciz "stapsdt") \
	415	+ __usdt_asm1(992: .balign 4) \
	416	+ __usdt_asm1(993: __usdt_asm_addr 990b) \
	417	+ __usdt_asm1( __usdt_asm_addr _.stapsdt.base) \
	418	+ __usdt_asm1( __usdt_asm_addr sema) \
	419	+ __usdt_asm_strz(group) \
	420	+ __usdt_asm_strz(name) \
	421	+ __usdt_asm_args(__VA_ARGS__) \
	422	+ __usdt_asm1( .ascii "\0") \
	423	+ __usdt_asm1(994: .balign 4) \
	424	+ __usdt_asm1( .popsection) \
	425	+ __usdt_asm1(.ifndef _.stapsdt.base) \
	426	+ __usdt_asm5( .pushsection .stapsdt.base,"aG","progbits",.stapsdt.base,comdat)\
	427	+ __usdt_asm1( .weak _.stapsdt.base) \
	428	+ __usdt_asm1( .hidden _.stapsdt.base) \
	429	+ __usdt_asm1(_.stapsdt.base:) \
	430	+ __usdt_asm1( .space 1) \
	431	+ __usdt_asm2( .size _.stapsdt.base, 1) \
	432	+ __usdt_asm1( .popsection) \
	433	+ __usdt_asm1(.endif) \
	434	+ :: __usdt_asm_ops(__VA_ARGS__) \
	435	+ ); \
	436	+} while (0)
	437	+
	438	+/*
	439	+ * NB: gdb PR24541 highlighted an unspecified corner of the sdt.h
	440	+ * operand note format.
	441	+ *
	442	+ * The named register may be a longer or shorter (!) alias for the
	443	+ * storage where the value in question is found. For example, on
	444	+ * i386, 64-bit value may be put in register pairs, and a register
	445	+ * name stored would identify just one of them. Previously, gcc was
	446	+ * asked to emit the %w[id] (16-bit alias of some registers holding
	447	+ * operands), even when a wider 32-bit value was used.
	448	+ *
	449	+ * Bottom line: the byte-width given before the @ sign governs. If
	450	+ * there is a mismatch between that width and that of the named
	451	+ * register, then a sys/sdt.h note consumer may need to employ
	452	+ * architecture-specific heuristics to figure out where the compiler
	453	+ * has actually put the complete value.
	454	+ */
	455	+#if defined(__powerpc__) \|\| defined(__powerpc64__)
	456	+#define __usdt_argref(id) %I[id]%[id]
	457	+#elif defined(__i386__)
	458	+#define __usdt_argref(id) %k[id] /* gcc.gnu.org/PR80115 sourceware.org/PR24541 */
	459	+#else
	460	+#define __usdt_argref(id) %[id]
	461	+#endif
	462	+
	463	+#define __usdt_asm_arg(n) __usdt_asm_str(%c[__usdt_asz##n]) \
	464	+ __usdt_asm1(.ascii "@") \
	465	+ __usdt_asm_str(__usdt_argref(__usdt_aval##n))
	466	+
	467	+#define __usdt_asm_args0 /* no arguments */
	468	+#define __usdt_asm_args1 __usdt_asm_arg(1)
	469	+#define __usdt_asm_args2 __usdt_asm_args1 __usdt_asm1(.ascii " ") __usdt_asm_arg(2)
	470	+#define __usdt_asm_args3 __usdt_asm_args2 __usdt_asm1(.ascii " ") __usdt_asm_arg(3)
	471	+#define __usdt_asm_args4 __usdt_asm_args3 __usdt_asm1(.ascii " ") __usdt_asm_arg(4)
	472	+#define __usdt_asm_args5 __usdt_asm_args4 __usdt_asm1(.ascii " ") __usdt_asm_arg(5)
	473	+#define __usdt_asm_args6 __usdt_asm_args5 __usdt_asm1(.ascii " ") __usdt_asm_arg(6)
	474	+#define __usdt_asm_args7 __usdt_asm_args6 __usdt_asm1(.ascii " ") __usdt_asm_arg(7)
	475	+#define __usdt_asm_args8 __usdt_asm_args7 __usdt_asm1(.ascii " ") __usdt_asm_arg(8)
	476	+#define __usdt_asm_args9 __usdt_asm_args8 __usdt_asm1(.ascii " ") __usdt_asm_arg(9)
	477	+#define __usdt_asm_args10 __usdt_asm_args9 __usdt_asm1(.ascii " ") __usdt_asm_arg(10)
	478	+#define __usdt_asm_args11 __usdt_asm_args10 __usdt_asm1(.ascii " ") __usdt_asm_arg(11)
	479	+#define __usdt_asm_args12 __usdt_asm_args11 __usdt_asm1(.ascii " ") __usdt_asm_arg(12)
	480	+#define __usdt_asm_args(...) __usdt_apply(__usdt_asm_args, __usdt_narg(__VA_ARGS__))
	481	+
	482	+#define __usdt_is_arr(x) (__builtin_classify_type(x) == 14 \|\| __builtin_classify_type(x) == 5)
	483	+#define __usdt_arg_size(x) (__usdt_is_arr(x) ? sizeof(void *) : sizeof(x))
	484	+
	485	+/*
	486	+ * We can't use __builtin_choose_expr() in C++, so fall back to table-based
	487	+ * signedness determination for known types, utilizing templates magic.
	488	+ */
	489	+#ifdef __cplusplus
	490	+
	491	+#define __usdt_is_signed(x) (!__usdt_is_arr(x) && __usdt_t<__typeof(x)>::is_signed)
	492	+
	493	+#include <cstddef>
	494	+
	495	+template<typename T> struct __usdt_t { static const bool is_signed = false; };
	496	+template<typename A> struct __usdt_t<A[]> : public __usdt_t<A *> {};
	497	+template<typename A, size_t N> struct __usdt_t<A[N]> : public __usdt_t<A *> {};
	498	+
	499	+#define __usdt_def_signed(T) \
	500	+template<> struct __usdt_t<T> { static const bool is_signed = true; }; \
	501	+template<> struct __usdt_t<const T> { static const bool is_signed = true; }; \
	502	+template<> struct __usdt_t<volatile T> { static const bool is_signed = true; }; \
	503	+template<> struct __usdt_t<const volatile T> { static const bool is_signed = true; }
	504	+#define __usdt_maybe_signed(T) \
	505	+template<> struct __usdt_t<T> { static const bool is_signed = (T)-1 < (T)1; }; \
	506	+template<> struct __usdt_t<const T> { static const bool is_signed = (T)-1 < (T)1; }; \
	507	+template<> struct __usdt_t<volatile T> { static const bool is_signed = (T)-1 < (T)1; }; \
	508	+template<> struct __usdt_t<const volatile T> { static const bool is_signed = (T)-1 < (T)1; }
	509	+
	510	+__usdt_def_signed(signed char);
	511	+__usdt_def_signed(short);
	512	+__usdt_def_signed(int);
	513	+__usdt_def_signed(long);
	514	+__usdt_def_signed(long long);
	515	+__usdt_maybe_signed(char);
	516	+__usdt_maybe_signed(wchar_t);
	517	+
	518	+#else /* !__cplusplus */
	519	+
	520	+#define __usdt_is_inttype(x) (__builtin_classify_type(x) >= 1 && __builtin_classify_type(x) <= 4)
	521	+#define __usdt_inttype(x) __typeof(__builtin_choose_expr(__usdt_is_inttype(x), (x), 0U))
	522	+#define __usdt_is_signed(x) ((__usdt_inttype(x))-1 < (__usdt_inttype(x))1)
	523	+
	524	+#endif /* __cplusplus */
	525	+
	526	+#define __usdt_asm_op(n, x) \
	527	+ [__usdt_asz##n] "n" ((__usdt_is_signed(x) ? (int)-1 : 1) * (int)__usdt_arg_size(x)), \
	528	+ [__usdt_aval##n] __usdt_str(USDT_ARG_CONSTRAINT)(x)
	529	+
	530	+#define __usdt_asm_ops0() [__usdt_dummy] "g" (0)
	531	+#define __usdt_asm_ops1(x) __usdt_asm_op(1, x)
	532	+#define __usdt_asm_ops2(a,x) __usdt_asm_ops1(a), __usdt_asm_op(2, x)
	533	+#define __usdt_asm_ops3(a,b,x) __usdt_asm_ops2(a,b), __usdt_asm_op(3, x)
	534	+#define __usdt_asm_ops4(a,b,c,x) __usdt_asm_ops3(a,b,c), __usdt_asm_op(4, x)
	535	+#define __usdt_asm_ops5(a,b,c,d,x) __usdt_asm_ops4(a,b,c,d), __usdt_asm_op(5, x)
	536	+#define __usdt_asm_ops6(a,b,c,d,e,x) __usdt_asm_ops5(a,b,c,d,e), __usdt_asm_op(6, x)
	537	+#define __usdt_asm_ops7(a,b,c,d,e,f,x) __usdt_asm_ops6(a,b,c,d,e,f), __usdt_asm_op(7, x)
	538	+#define __usdt_asm_ops8(a,b,c,d,e,f,g,x) __usdt_asm_ops7(a,b,c,d,e,f,g), __usdt_asm_op(8, x)
	539	+#define __usdt_asm_ops9(a,b,c,d,e,f,g,h,x) __usdt_asm_ops8(a,b,c,d,e,f,g,h), __usdt_asm_op(9, x)
	540	+#define __usdt_asm_ops10(a,b,c,d,e,f,g,h,i,x) __usdt_asm_ops9(a,b,c,d,e,f,g,h,i), __usdt_asm_op(10, x)
	541	+#define __usdt_asm_ops11(a,b,c,d,e,f,g,h,i,j,x) __usdt_asm_ops10(a,b,c,d,e,f,g,h,i,j), __usdt_asm_op(11, x)
	542	+#define __usdt_asm_ops12(a,b,c,d,e,f,g,h,i,j,k,x) __usdt_asm_ops11(a,b,c,d,e,f,g,h,i,j,k), __usdt_asm_op(12, x)
	543	+#define __usdt_asm_ops(...) __usdt_apply(__usdt_asm_ops, __usdt_narg(__VA_ARGS__))(__VA_ARGS__)
	544	+
	545	+#endif /* __USDT_H */

tools/testing/selftests/seccomp/seccomp_bpf.c+86−21

@@ -74,6 +74,14 @@ · 73 unmodified lines
74	74	#define noinline __attribute__((noinline))
75	75	#endif
76	76
	77	+#ifndef __nocf_check
	78	+#define __nocf_check __attribute__((nocf_check))
	79	+#endif
	80	+
	81	+#ifndef __naked
	82	+#define __naked __attribute__((__naked__))
	83	+#endif
	84	+
77	85	#ifndef PR_SET_NO_NEW_PRIVS
78	86	#define PR_SET_NO_NEW_PRIVS 38
79	87	#define PR_GET_NO_NEW_PRIVS 39
@@ -5027,7 +5035,36 @@ TEST(tsync_vs_dead_thread_leader) · 4947 unmodified lines
5027	5035	EXPECT_EQ(0, status);
5028	5036	}
5029	5037
5030		−noinline int probed(void)
	5038	+#ifdef __x86_64__
	5039	+
	5040	+/*
	5041	+ * We need naked probed_uprobe function. Using __nocf_check
	5042	+ * check to skip possible endbr64 instruction and ignoring
	5043	+ * -Wattributes, otherwise the compilation might fail.
	5044	+ */
	5045	+#pragma GCC diagnostic push
	5046	+#pragma GCC diagnostic ignored "-Wattributes"
	5047	+
	5048	+__naked __nocf_check noinline int probed_uprobe(void)
	5049	+{
	5050	+ /*
	5051	+ * Optimized uprobe is possible only on top of nop5 instruction.
	5052	+ */
	5053	+ asm volatile (" \n"
	5054	+ ".byte 0x0f, 0x1f, 0x44, 0x00, 0x00 \n"
	5055	+ "ret \n"
	5056	+ );
	5057	+}
	5058	+#pragma GCC diagnostic pop
	5059	+
	5060	+#else
	5061	+noinline int probed_uprobe(void)
	5062	+{
	5063	+ return 1;
	5064	+}
	5065	+#endif
	5066	+
	5067	+noinline int probed_uretprobe(void)
5031	5068	{
5032	5069	return 1;
5033	5070	}
@@ -5080,35 +5117,46 @@ static ssize_t get_uprobe_offset(const void *addr) · 46 unmodified lines
5080	5117	return found ? (uintptr_t)addr - start + base : -1;
5081	5118	}
5082	5119
5083		−FIXTURE(URETPROBE) {
	5120	+FIXTURE(UPROBE) {
5084	5121	int fd;
5085	5122	};
5086	5123
5087		−FIXTURE_VARIANT(URETPROBE) {
	5124	+FIXTURE_VARIANT(UPROBE) {
5088	5125	/*
5089		− * All of the URETPROBE behaviors can be tested with either
5090		− * uretprobe attached or not
	5126	+ * All of the U(RET)PROBE behaviors can be tested with either
	5127	+ * u(ret)probe attached or not
5091	5128	*/
5092	5129	bool attach;
	5130	+ /*
	5131	+ * Test both uprobe and uretprobe.
	5132	+ */
	5133	+ bool uretprobe;
5093	5134	};
5094	5135
5095		−FIXTURE_VARIANT_ADD(URETPROBE, attached) {
	5136	+FIXTURE_VARIANT_ADD(UPROBE, not_attached) {
	5137	+ .attach = false,
	5138	+ .uretprobe = false,
	5139	+};
	5140	+
	5141	+FIXTURE_VARIANT_ADD(UPROBE, uprobe_attached) {
5096	5142	.attach = true,
	5143	+ .uretprobe = false,
5097	5144	};
5098	5145
5099		−FIXTURE_VARIANT_ADD(URETPROBE, not_attached) {
5100		− .attach = false,
	5146	+FIXTURE_VARIANT_ADD(UPROBE, uretprobe_attached) {
	5147	+ .attach = true,
	5148	+ .uretprobe = true,
5101	5149	};
5102	5150
5103		−FIXTURE_SETUP(URETPROBE)
	5151	+FIXTURE_SETUP(UPROBE)
5104	5152	{
5105	5153	const size_t attr_sz = sizeof(struct perf_event_attr);
5106	5154	struct perf_event_attr attr;
5107	5155	ssize_t offset;
5108	5156	int type, bit;
5109	5157
5110		−#ifndef __NR_uretprobe
5111		− SKIP(return, "__NR_uretprobe syscall not defined");
	5158	+#if !defined(__NR_uprobe) \|\| !defined(__NR_uretprobe)
	5159	+ SKIP(return, "__NR_uprobe ot __NR_uretprobe syscalls not defined");
5112	5160	#endif
5113	5161
5114	5162	if (!variant->attach)
@@ -5118,12 +5166,17 @@ FIXTURE_SETUP(URETPROBE) · 3 unmodified lines
5118	5166
5119	5167	type = determine_uprobe_perf_type();
5120	5168	ASSERT_GE(type, 0);
5121		− bit = determine_uprobe_retprobe_bit();
5122		− ASSERT_GE(bit, 0);
5123		− offset = get_uprobe_offset(probed);
	5169	+
	5170	+ if (variant->uretprobe) {
	5171	+ bit = determine_uprobe_retprobe_bit();
	5172	+ ASSERT_GE(bit, 0);
	5173	+ }
	5174	+
	5175	+ offset = get_uprobe_offset(variant->uretprobe ? probed_uretprobe : probed_uprobe);
5124	5176	ASSERT_GE(offset, 0);
5125	5177
5126		− attr.config \|= 1 << bit;
	5178	+ if (variant->uretprobe)
	5179	+ attr.config \|= 1 << bit;
5127	5180	attr.size = attr_sz;
5128	5181	attr.type = type;
5129	5182	attr.config1 = ptr_to_u64("/proc/self/exe");
@@ -5134,7 +5187,7 @@ FIXTURE_SETUP(URETPROBE) · 4 unmodified lines
5134	5187	PERF_FLAG_FD_CLOEXEC);
5135	5188	}
5136	5189
5137		−FIXTURE_TEARDOWN(URETPROBE)
	5190	+FIXTURE_TEARDOWN(UPROBE)
5138	5191	{
5139	5192	/* we could call close(self->fd), but we'd need extra filter for
5140	5193	* that and since we are calling _exit right away..
@@ -5148,11 +5201,17 @@ static int run_probed_with_filter(struct sock_fprog *prog) · 7 unmodified lines
5148	5201	return -1;
5149	5202	}
5150	5203
5151		− probed();
	5204	+ /*
	5205	+ * Uprobe is optimized after first hit, so let's hit twice.
	5206	+ */
	5207	+ probed_uprobe();
	5208	+ probed_uprobe();
	5209	+
	5210	+ probed_uretprobe();
5152	5211	return 0;
5153	5212	}
5154	5213
5155		−TEST_F(URETPROBE, uretprobe_default_allow)
	5214	+TEST_F(UPROBE, uprobe_default_allow)
5156	5215	{
5157	5216	struct sock_filter filter[] = {
5158	5217	BPF_STMT(BPF_RET\|BPF_K, SECCOMP_RET_ALLOW),
@@ -5165,7 +5224,7 @@ TEST_F(URETPROBE, uretprobe_default_allow) · 6 unmodified lines
5165	5224	ASSERT_EQ(0, run_probed_with_filter(&prog));
5166	5225	}
5167	5226
5168		−TEST_F(URETPROBE, uretprobe_default_block)
	5227	+TEST_F(UPROBE, uprobe_default_block)
5169	5228	{
5170	5229	struct sock_filter filter[] = {
5171	5230	BPF_STMT(BPF_LD\|BPF_W\|BPF_ABS,
@@ -5182,11 +5241,14 @@ TEST_F(URETPROBE, uretprobe_default_block) · 10 unmodified lines
5182	5241	ASSERT_EQ(0, run_probed_with_filter(&prog));
5183	5242	}
5184	5243
5185		−TEST_F(URETPROBE, uretprobe_block_uretprobe_syscall)
	5244	+TEST_F(UPROBE, uprobe_block_syscall)
5186	5245	{
5187	5246	struct sock_filter filter[] = {
5188	5247	BPF_STMT(BPF_LD\|BPF_W\|BPF_ABS,
5189	5248	offsetof(struct seccomp_data, nr)),
	5249	+#ifdef __NR_uprobe
	5250	+ BPF_JUMP(BPF_JMP\|BPF_JEQ\|BPF_K, __NR_uprobe, 1, 2),
	5251	+#endif
5190	5252	#ifdef __NR_uretprobe
5191	5253	BPF_JUMP(BPF_JMP\|BPF_JEQ\|BPF_K, __NR_uretprobe, 0, 1),
5192	5254	#endif
@@ -5201,11 +5263,14 @@ TEST_F(URETPROBE, uretprobe_block_uretprobe_syscall) · 8 unmodified lines
5201	5263	ASSERT_EQ(0, run_probed_with_filter(&prog));
5202	5264	}
5203	5265
5204		−TEST_F(URETPROBE, uretprobe_default_block_with_uretprobe_syscall)
	5266	+TEST_F(UPROBE, uprobe_default_block_with_syscall)
5205	5267	{
5206	5268	struct sock_filter filter[] = {
5207	5269	BPF_STMT(BPF_LD\|BPF_W\|BPF_ABS,
5208	5270	offsetof(struct seccomp_data, nr)),
	5271	+#ifdef __NR_uprobe
	5272	+ BPF_JUMP(BPF_JMP\|BPF_JEQ\|BPF_K, __NR_uprobe, 3, 0),
	5273	+#endif
5209	5274	#ifdef __NR_uretprobe
5210	5275	BPF_JUMP(BPF_JMP\|BPF_JEQ\|BPF_K, __NR_uretprobe, 2, 0),
5211	5276	#endif