lael-labs/linux

arch/arm/probes/uprobes/core.c+1−1

@@ -30,7 +30,7 @@ int set_swbp(struct arch_uprobe auprobe, struct vm_area_struct vma, · 29 unmodified lines
30	unsigned long vaddr)	30	unsigned long vaddr)
31	{	31	{
32	return uprobe_write_opcode(auprobe, vma, vaddr,	32	return uprobe_write_opcode(auprobe, vma, vaddr,
33	__opcode_to_mem_arm(auprobe->bpinsn));	33	__opcode_to_mem_arm(auprobe->bpinsn), true);
34	}	34	}
35		35
36	bool arch_uprobe_ignore(struct arch_uprobe auprobe, struct pt_regs regs)	36	bool arch_uprobe_ignore(struct arch_uprobe auprobe, struct pt_regs regs)

arch/x86/entry/syscalls/syscall_64.tbl+1−0

@@ -345,6 +345,7 @@ · 344 unmodified lines
345	333 common io_pgetevents sys_io_pgetevents	345	333 common io_pgetevents sys_io_pgetevents
346	334 common rseq sys_rseq	346	334 common rseq sys_rseq
347	335 common uretprobe sys_uretprobe	347	335 common uretprobe sys_uretprobe
		348	336 common uprobe sys_uprobe
348	# don't use numbers 387 through 423, add new calls after the last	349	# don't use numbers 387 through 423, add new calls after the last
349	# 'common' entry	350	# 'common' entry
350	424 common pidfd_send_signal sys_pidfd_send_signal	351	424 common pidfd_send_signal sys_pidfd_send_signal

arch/x86/events/core.c+9−7

@@ -2069,13 +2069,15 @@ static void _x86_pmu_read(struct perf_event *event) · 2068 unmodified lines
2069		2069
2070	void x86_pmu_show_pmu_cap(struct pmu *pmu)	2070	void x86_pmu_show_pmu_cap(struct pmu *pmu)
2071	{	2071	{
2072	pr_info("... version: %d\n", x86_pmu.version);	2072	pr_info("... version: %d\n", x86_pmu.version);
2073	pr_info("... bit width: %d\n", x86_pmu.cntval_bits);	2073	pr_info("... bit width: %d\n", x86_pmu.cntval_bits);
2074	pr_info("... generic registers: %d\n", x86_pmu_num_counters(pmu));	2074	pr_info("... generic counters: %d\n", x86_pmu_num_counters(pmu));
2075	pr_info("... value mask: %016Lx\n", x86_pmu.cntval_mask);	2075	pr_info("... generic bitmap: %016llx\n", hybrid(pmu, cntr_mask64));
2076	pr_info("... max period: %016Lx\n", x86_pmu.max_period);	2076	pr_info("... fixed-purpose counters: %d\n", x86_pmu_num_counters_fixed(pmu));
2077	pr_info("... fixed-purpose events: %d\n", x86_pmu_num_counters_fixed(pmu));	2077	pr_info("... fixed-purpose bitmap: %016llx\n", hybrid(pmu, fixed_cntr_mask64));
2078	pr_info("... event mask: %016Lx\n", hybrid(pmu, intel_ctrl));	2078	pr_info("... value mask: %016llx\n", x86_pmu.cntval_mask);
		2079	pr_info("... max period: %016llx\n", x86_pmu.max_period);
		2080	pr_info("... global_ctrl mask: %016llx\n", hybrid(pmu, intel_ctrl));
2079	}	2081	}
2080		2082
2081	static int __init init_hw_perf_events(void)	2083	static int __init init_hw_perf_events(void)

arch/x86/events/intel/bts.c+1−1

@@ -643,4 +643,4 @@ static __init int bts_init(void) · 642 unmodified lines
643		643
644	return perf_pmu_register(&bts_pmu, "intel_bts", -1);	644	return perf_pmu_register(&bts_pmu, "intel_bts", -1);
645	}	645	}
646	arch_initcall(bts_init);	646	early_initcall(bts_init);

arch/x86/events/intel/core.c+9−12

 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 	struct hw_perf_event *hwc = &event->hw;
-	u64 mask, bits = 0;
 	int idx = hwc->idx;
+	u64 bits = 0;
 	if (is_topdown_idx(idx)) {
 		struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 	idx -= INTEL_PMC_IDX_FIXED;
 	bits = intel_fixed_bits_by_idx(idx, bits);
-	mask = intel_fixed_bits_by_idx(idx, INTEL_FIXED_BITS_MASK);
+	if (x86_pmu.intel_cap.pebs_baseline && event->attr.precise_ip)
-	if (x86_pmu.intel_cap.pebs_baseline && event->attr.precise_ip) {
 		bits |= intel_fixed_bits_by_idx(idx, ICL_FIXED_0_ADAPTIVE);
-		mask |= intel_fixed_bits_by_idx(idx, ICL_FIXED_0_ADAPTIVE);
-	}
-	cpuc->fixed_ctrl_val &= ~mask;
+	cpuc->fixed_ctrl_val &= ~intel_fixed_bits_by_idx(idx, INTEL_FIXED_BITS_MASK);
 	cpuc->fixed_ctrl_val |= bits;
 }
 			if (event->group_leader != leader->group_leader)
 				break;
 			for_each_set_bit(idx, (unsigned long *)&event->attr.config2, X86_PMC_IDX_MAX) {
-				if (WARN_ON_ONCE(i + idx > cpuc->n_events))
+				if (i + idx >= cpuc->n_events ||
+				    !is_acr_event_group(cpuc->event_list[i + idx]))
 					return;
 				__set_bit(cpuc->assign[i + idx], (unsigned long *)&event->hw.config1);
 			}
 						0, x86_pmu_num_counters(&pmu->pmu), 0, 0);
 	if (pmu->intel_cap.perf_metrics)
-		pmu->intel_ctrl |= 1ULL << GLOBAL_CTRL_EN_PERF_METRICS;
+		pmu->intel_ctrl |= GLOBAL_CTRL_EN_PERF_METRICS;
 	else
-		pmu->intel_ctrl &= ~(1ULL << GLOBAL_CTRL_EN_PERF_METRICS);
+		pmu->intel_ctrl &= ~GLOBAL_CTRL_EN_PERF_METRICS;
 	intel_pmu_check_event_constraints(pmu->event_constraints,
 					  pmu->cntr_mask64,
 		rdmsrq(MSR_IA32_PERF_CAPABILITIES, perf_cap.capabilities);
 		if (!perf_cap.perf_metrics) {
 			x86_pmu.intel_cap.perf_metrics = 0;
-			x86_pmu.intel_ctrl &= ~(1ULL << GLOBAL_CTRL_EN_PERF_METRICS);
+			x86_pmu.intel_ctrl &= ~GLOBAL_CTRL_EN_PERF_METRICS;
 		}
 	}
 	}
 	if (!is_hybrid() && x86_pmu.intel_cap.perf_metrics)
-		x86_pmu.intel_ctrl |= 1ULL << GLOBAL_CTRL_EN_PERF_METRICS;
+		x86_pmu.intel_ctrl |= GLOBAL_CTRL_EN_PERF_METRICS;
 	if (x86_pmu.intel_cap.pebs_timing_info)
 		x86_pmu.flags |= PMU_FL_RETIRE_LATENCY;

arch/x86/include/asm/msr-index.h+8−6

@@ -315,12 +315,14 @@ · 314 unmodified lines
315	#define PERF_CAP_PT_IDX 16	315	#define PERF_CAP_PT_IDX 16
316		316
317	#define MSR_PEBS_LD_LAT_THRESHOLD 0x000003f6	317	#define MSR_PEBS_LD_LAT_THRESHOLD 0x000003f6
318	#define PERF_CAP_PEBS_TRAP BIT_ULL(6)	318	#define PERF_CAP_PEBS_TRAP BIT_ULL(6)
319	#define PERF_CAP_ARCH_REG BIT_ULL(7)	319	#define PERF_CAP_ARCH_REG BIT_ULL(7)
320	#define PERF_CAP_PEBS_FORMAT 0xf00	320	#define PERF_CAP_PEBS_FORMAT 0xf00
321	#define PERF_CAP_PEBS_BASELINE BIT_ULL(14)	321	#define PERF_CAP_PEBS_BASELINE BIT_ULL(14)
322	#define PERF_CAP_PEBS_MASK (PERF_CAP_PEBS_TRAP \| PERF_CAP_ARCH_REG \| \	322	#define PERF_CAP_PEBS_TIMING_INFO BIT_ULL(17)
323	PERF_CAP_PEBS_FORMAT \| PERF_CAP_PEBS_BASELINE)	323	#define PERF_CAP_PEBS_MASK (PERF_CAP_PEBS_TRAP \| PERF_CAP_ARCH_REG \| \
		324	PERF_CAP_PEBS_FORMAT \| PERF_CAP_PEBS_BASELINE \| \
		325	PERF_CAP_PEBS_TIMING_INFO)
324		326
325	#define MSR_IA32_RTIT_CTL 0x00000570	327	#define MSR_IA32_RTIT_CTL 0x00000570
326	#define RTIT_CTL_TRACEEN BIT(0)	328	#define RTIT_CTL_TRACEEN BIT(0)

arch/x86/include/asm/perf_event.h+6−2

 #define ARCH_PERFMON_EVENTSEL_EQ			(1ULL << 36)
 #define ARCH_PERFMON_EVENTSEL_UMASK2			(0xFFULL << 40)
-#define INTEL_FIXED_BITS_MASK				0xFULL
 #define INTEL_FIXED_BITS_STRIDE			4
 #define INTEL_FIXED_0_KERNEL				(1ULL << 0)
 #define INTEL_FIXED_0_USER				(1ULL << 1)
 #define ICL_EVENTSEL_ADAPTIVE				(1ULL << 34)
 #define ICL_FIXED_0_ADAPTIVE				(1ULL << 32)
+#define INTEL_FIXED_BITS_MASK					\
+	(INTEL_FIXED_0_KERNEL | INTEL_FIXED_0_USER |		\
+	 INTEL_FIXED_0_ANYTHREAD | INTEL_FIXED_0_ENABLE_PMI |	\
+	 ICL_FIXED_0_ADAPTIVE)
 #define intel_fixed_bits_by_idx(_idx, _bits)			\
 	((_bits) << ((_idx) * INTEL_FIXED_BITS_STRIDE))
 #define GLOBAL_STATUS_TRACE_TOPAPMI		BIT_ULL(GLOBAL_STATUS_TRACE_TOPAPMI_BIT)
 #define GLOBAL_STATUS_PERF_METRICS_OVF_BIT	48
-#define GLOBAL_CTRL_EN_PERF_METRICS		48
+#define GLOBAL_CTRL_EN_PERF_METRICS		BIT_ULL(48)
 /*
  * We model guest LBR event tracing as another fixed-mode PMC like BTS.
  *

arch/x86/include/asm/shstk.h+4−0

 int restore_signal_shadow_stack(void);
 int shstk_update_last_frame(unsigned long val);
 bool shstk_is_enabled(void);
+int shstk_pop(u64 *val);
+int shstk_push(u64 val);
 #else
 static inline long shstk_prctl(struct task_struct *task, int option,
 			       unsigned long arg2) { return -EINVAL; }
 static inline int restore_signal_shadow_stack(void) { return 0; }
 static inline int shstk_update_last_frame(unsigned long val) { return 0; }
 static inline bool shstk_is_enabled(void) { return false; }
+static inline int shstk_pop(u64 *val) { return -ENOTSUPP; }
+static inline int shstk_push(u64 val) { return -ENOTSUPP; }
 #endif /* CONFIG_X86_USER_SHADOW_STACK */
 #endif /* __ASSEMBLER__ */

arch/x86/include/asm/uprobes.h+7−0

 #define UPROBE_SWBP_INSN		0xcc
 #define UPROBE_SWBP_INSN_SIZE		   1
+enum {
+	ARCH_UPROBE_FLAG_CAN_OPTIMIZE   = 0,
+	ARCH_UPROBE_FLAG_OPTIMIZE_FAIL  = 1,
+};
 struct uprobe_xol_ops;
 struct arch_uprobe {
 			u8	ilen;
 		}			push;
 	};
+	unsigned long flags;
 };
 struct arch_uprobe_task {

arch/x86/kernel/shstk.c+40−0

@@ -246,6 +246,46 @@ static unsigned long get_user_shstk_addr(void) · 245 unmodified lines
246	return ssp;	246	return ssp;
247	}	247	}
248		248
		249	int shstk_pop(u64 *val)
		250	{
		251	int ret = 0;
		252	u64 ssp;
		253
		254	if (!features_enabled(ARCH_SHSTK_SHSTK))
		255	return -ENOTSUPP;
		256
		257	fpregs_lock_and_load();
		258
		259	rdmsrq(MSR_IA32_PL3_SSP, ssp);
		260	if (val && get_user(val, (__user u64 )ssp))
		261	ret = -EFAULT;
		262	else
		263	wrmsrq(MSR_IA32_PL3_SSP, ssp + SS_FRAME_SIZE);
		264	fpregs_unlock();
		265
		266	return ret;
		267	}
		268
		269	int shstk_push(u64 val)
		270	{
		271	u64 ssp;
		272	int ret;
		273
		274	if (!features_enabled(ARCH_SHSTK_SHSTK))
		275	return -ENOTSUPP;
		276
		277	fpregs_lock_and_load();
		278
		279	rdmsrq(MSR_IA32_PL3_SSP, ssp);
		280	ssp -= SS_FRAME_SIZE;
		281	ret = write_user_shstk_64((__user void *)ssp, val);
		282	if (!ret)
		283	wrmsrq(MSR_IA32_PL3_SSP, ssp);
		284	fpregs_unlock();
		285
		286	return ret;
		287	}
		288
249	#define SHSTK_DATA_BIT BIT(63)	289	#define SHSTK_DATA_BIT BIT(63)
250		290
251	static int put_shstk_data(u64 __user *addr, u64 data)	291	static int put_shstk_data(u64 __user *addr, u64 data)

arch/x86/kernel/uprobes.c+613−22

@@ -18,6 +18,7 @@ · 17 unmodified lines
18	#include <asm/processor.h>	18	#include <asm/processor.h>
19	#include <asm/insn.h>	19	#include <asm/insn.h>
20	#include <asm/mmu_context.h>	20	#include <asm/mmu_context.h>
		21	#include <asm/nops.h>
21		22
22	/* Post-execution fixups. */	23	/* Post-execution fixups. */
23		24
@@ -310,25 +311,32 @@ static int uprobe_init_insn(struct arch_uprobe auprobe, struct insn insn, bool · 286 unmodified lines
310		311
311	#ifdef CONFIG_X86_64	312	#ifdef CONFIG_X86_64
312		313
		314	struct uretprobe_syscall_args {
		315	unsigned long r11;
		316	unsigned long cx;
		317	unsigned long ax;
		318	};
		319
313	asm (	320	asm (
314	".pushsection .rodata\n"	321	".pushsection .rodata\n"
315	".global uretprobe_trampoline_entry\n"	322	".global uretprobe_trampoline_entry\n"
316	"uretprobe_trampoline_entry:\n"	323	"uretprobe_trampoline_entry:\n"
317	"pushq %rax\n"	324	"push %rax\n"
318	"pushq %rcx\n"	325	"push %rcx\n"
319	"pushq %r11\n"	326	"push %r11\n"
320	"movq $" __stringify(__NR_uretprobe) ", %rax\n"	327	"mov $" __stringify(__NR_uretprobe) ", %rax\n"
321	"syscall\n"	328	"syscall\n"
322	".global uretprobe_syscall_check\n"	329	".global uretprobe_syscall_check\n"
323	"uretprobe_syscall_check:\n"	330	"uretprobe_syscall_check:\n"
324	"popq %r11\n"	331	"pop %r11\n"
325	"popq %rcx\n"	332	"pop %rcx\n"
326		333	/*
327	/* The uretprobe syscall replaces stored %rax value with final	334	* The uretprobe syscall replaces stored %rax value with final
328	* return address, so we don't restore %rax in here and just	335	* return address, so we don't restore %rax in here and just
329	* call ret.	336	* call ret.
330	*/	337	*/
331	"retq\n"	338	"ret\n"
		339	"int3\n"
332	".global uretprobe_trampoline_end\n"	340	".global uretprobe_trampoline_end\n"
333	"uretprobe_trampoline_end:\n"	341	"uretprobe_trampoline_end:\n"
334	".popsection\n"	342	".popsection\n"
@@ -338,7 +346,7 @@ extern u8 uretprobe_trampoline_entry[]; · 3 unmodified lines
338	extern u8 uretprobe_trampoline_end[];	346	extern u8 uretprobe_trampoline_end[];
339	extern u8 uretprobe_syscall_check[];	347	extern u8 uretprobe_syscall_check[];
340		348
341	void arch_uprobe_trampoline(unsigned long psize)	349	void arch_uretprobe_trampoline(unsigned long psize)
342	{	350	{
343	static uprobe_opcode_t insn = UPROBE_SWBP_INSN;	351	static uprobe_opcode_t insn = UPROBE_SWBP_INSN;
344	struct pt_regs *regs = task_pt_regs(current);	352	struct pt_regs *regs = task_pt_regs(current);
@@ -365,7 +373,8 @@ static unsigned long trampoline_check_ip(unsigned long tramp) · 20 unmodified lines
365	SYSCALL_DEFINE0(uretprobe)	373	SYSCALL_DEFINE0(uretprobe)
366	{	374	{
367	struct pt_regs *regs = task_pt_regs(current);	375	struct pt_regs *regs = task_pt_regs(current);
368	unsigned long err, ip, sp, r11_cx_ax[3], tramp;	376	struct uretprobe_syscall_args args;
		377	unsigned long err, ip, sp, tramp;
369		378
370	/* If there's no trampoline, we are called from wrong place. */	379	/* If there's no trampoline, we are called from wrong place. */
371	tramp = uprobe_get_trampoline_vaddr();	380	tramp = uprobe_get_trampoline_vaddr();
@@ -376,15 +385,15 @@ SYSCALL_DEFINE0(uretprobe) · 4 unmodified lines
376	if (unlikely(regs->ip != trampoline_check_ip(tramp)))	385	if (unlikely(regs->ip != trampoline_check_ip(tramp)))
377	goto sigill;	386	goto sigill;
378		387
379	err = copy_from_user(r11_cx_ax, (void __user *)regs->sp, sizeof(r11_cx_ax));	388	err = copy_from_user(&args, (void __user *)regs->sp, sizeof(args));
380	if (err)	389	if (err)
381	goto sigill;	390	goto sigill;
382		391
383	/* expose the "right" values of r11/cx/ax/sp to uprobe_consumer/s */	392	/* expose the "right" values of r11/cx/ax/sp to uprobe_consumer/s */
384	regs->r11 = r11_cx_ax[0];	393	regs->r11 = args.r11;
385	regs->cx = r11_cx_ax[1];	394	regs->cx = args.cx;
386	regs->ax = r11_cx_ax[2];	395	regs->ax = args.ax;
387	regs->sp += sizeof(r11_cx_ax);	396	regs->sp += sizeof(args);
388	regs->orig_ax = -1;	397	regs->orig_ax = -1;
389		398
390	ip = regs->ip;	399	ip = regs->ip;
@@ -400,21 +409,21 @@ SYSCALL_DEFINE0(uretprobe) · 9 unmodified lines
400	*/	409	*/
401	if (regs->sp != sp \|\| shstk_is_enabled())	410	if (regs->sp != sp \|\| shstk_is_enabled())
402	return regs->ax;	411	return regs->ax;
403	regs->sp -= sizeof(r11_cx_ax);	412	regs->sp -= sizeof(args);
404		413
405	/* for the case uprobe_consumer has changed r11/cx */	414	/* for the case uprobe_consumer has changed r11/cx */
406	r11_cx_ax[0] = regs->r11;	415	args.r11 = regs->r11;
407	r11_cx_ax[1] = regs->cx;	416	args.cx = regs->cx;
408		417
409	/*	418	/*
410	* ax register is passed through as return value, so we can use	419	* ax register is passed through as return value, so we can use
411	* its space on stack for ip value and jump to it through the	420	* its space on stack for ip value and jump to it through the
412	* trampoline's ret instruction	421	* trampoline's ret instruction
413	*/	422	*/
414	r11_cx_ax[2] = regs->ip;	423	args.ax = regs->ip;
415	regs->ip = ip;	424	regs->ip = ip;
416		425
417	err = copy_to_user((void __user *)regs->sp, r11_cx_ax, sizeof(r11_cx_ax));	426	err = copy_to_user((void __user *)regs->sp, &args, sizeof(args));
418	if (err)	427	if (err)
419	goto sigill;	428	goto sigill;
420		429
@@ -608,6 +617,581 @@ static void riprel_post_xol(struct arch_uprobe auprobe, struct pt_regs regs) · 187 unmodified lines
608	*sr = utask->autask.saved_scratch_register;	617	*sr = utask->autask.saved_scratch_register;
609	}	618	}
610	}	619	}
		620
		621	static int tramp_mremap(const struct vm_special_mapping sm, struct vm_area_struct new_vma)
		622	{
		623	return -EPERM;
		624	}
		625
		626	static struct page *tramp_mapping_pages[2] __ro_after_init;
		627
		628	static struct vm_special_mapping tramp_mapping = {
		629	.name = "[uprobes-trampoline]",
		630	.mremap = tramp_mremap,
		631	.pages = tramp_mapping_pages,
		632	};
		633
		634	struct uprobe_trampoline {
		635	struct hlist_node node;
		636	unsigned long vaddr;
		637	};
		638
		639	static bool is_reachable_by_call(unsigned long vtramp, unsigned long vaddr)
		640	{
		641	long delta = (long)(vaddr + 5 - vtramp);
		642
		643	return delta >= INT_MIN && delta <= INT_MAX;
		644	}
		645
		646	static unsigned long find_nearest_trampoline(unsigned long vaddr)
		647	{
		648	struct vm_unmapped_area_info info = {
		649	.length = PAGE_SIZE,
		650	.align_mask = ~PAGE_MASK,
		651	};
		652	unsigned long low_limit, high_limit;
		653	unsigned long low_tramp, high_tramp;
		654	unsigned long call_end = vaddr + 5;
		655
		656	if (check_add_overflow(call_end, INT_MIN, &low_limit))
		657	low_limit = PAGE_SIZE;
		658
		659	high_limit = call_end + INT_MAX;
		660
		661	/* Search up from the caller address. */
		662	info.low_limit = call_end;
		663	info.high_limit = min(high_limit, TASK_SIZE);
		664	high_tramp = vm_unmapped_area(&info);
		665
		666	/* Search down from the caller address. */
		667	info.low_limit = max(low_limit, PAGE_SIZE);
		668	info.high_limit = call_end;
		669	info.flags = VM_UNMAPPED_AREA_TOPDOWN;
		670	low_tramp = vm_unmapped_area(&info);
		671
		672	if (IS_ERR_VALUE(high_tramp) && IS_ERR_VALUE(low_tramp))
		673	return -ENOMEM;
		674	if (IS_ERR_VALUE(high_tramp))
		675	return low_tramp;
		676	if (IS_ERR_VALUE(low_tramp))
		677	return high_tramp;
		678
		679	/* Return address that's closest to the caller address. */
		680	if (call_end - low_tramp < high_tramp - call_end)
		681	return low_tramp;
		682	return high_tramp;
		683	}
		684
		685	static struct uprobe_trampoline *create_uprobe_trampoline(unsigned long vaddr)
		686	{
		687	struct pt_regs *regs = task_pt_regs(current);
		688	struct mm_struct *mm = current->mm;
		689	struct uprobe_trampoline *tramp;
		690	struct vm_area_struct *vma;
		691
		692	if (!user_64bit_mode(regs))
		693	return NULL;
		694
		695	vaddr = find_nearest_trampoline(vaddr);
		696	if (IS_ERR_VALUE(vaddr))
		697	return NULL;
		698
		699	tramp = kzalloc(sizeof(*tramp), GFP_KERNEL);
		700	if (unlikely(!tramp))
		701	return NULL;
		702
		703	tramp->vaddr = vaddr;
		704	vma = _install_special_mapping(mm, tramp->vaddr, PAGE_SIZE,
		705	VM_READ\|VM_EXEC\|VM_MAYEXEC\|VM_MAYREAD\|VM_DONTCOPY\|VM_IO,
		706	&tramp_mapping);
		707	if (IS_ERR(vma)) {
		708	kfree(tramp);
		709	return NULL;
		710	}
		711	return tramp;
		712	}
		713
		714	static struct uprobe_trampoline get_uprobe_trampoline(unsigned long vaddr, bool new)
		715	{
		716	struct uprobes_state *state = &current->mm->uprobes_state;
		717	struct uprobe_trampoline *tramp = NULL;
		718
		719	if (vaddr > TASK_SIZE \|\| vaddr < PAGE_SIZE)
		720	return NULL;
		721
		722	hlist_for_each_entry(tramp, &state->head_tramps, node) {
		723	if (is_reachable_by_call(tramp->vaddr, vaddr)) {
		724	*new = false;
		725	return tramp;
		726	}
		727	}
		728
		729	tramp = create_uprobe_trampoline(vaddr);
		730	if (!tramp)
		731	return NULL;
		732
		733	*new = true;
		734	hlist_add_head(&tramp->node, &state->head_tramps);
		735	return tramp;
		736	}
		737
		738	static void destroy_uprobe_trampoline(struct uprobe_trampoline *tramp)
		739	{
		740	/*
		741	* We do not unmap and release uprobe trampoline page itself,
		742	* because there's no easy way to make sure none of the threads
		743	* is still inside the trampoline.
		744	*/
		745	hlist_del(&tramp->node);
		746	kfree(tramp);
		747	}
		748
		749	void arch_uprobe_init_state(struct mm_struct *mm)
		750	{
		751	INIT_HLIST_HEAD(&mm->uprobes_state.head_tramps);
		752	}
		753
		754	void arch_uprobe_clear_state(struct mm_struct *mm)
		755	{
		756	struct uprobes_state *state = &mm->uprobes_state;
		757	struct uprobe_trampoline *tramp;
		758	struct hlist_node *n;
		759
		760	hlist_for_each_entry_safe(tramp, n, &state->head_tramps, node)
		761	destroy_uprobe_trampoline(tramp);
		762	}
		763
		764	static bool __in_uprobe_trampoline(unsigned long ip)
		765	{
		766	struct vm_area_struct *vma = vma_lookup(current->mm, ip);
		767
		768	return vma && vma_is_special_mapping(vma, &tramp_mapping);
		769	}
		770
		771	static bool in_uprobe_trampoline(unsigned long ip)
		772	{
		773	struct mm_struct *mm = current->mm;
		774	bool found, retry = true;
		775	unsigned int seq;
		776
		777	rcu_read_lock();
		778	if (mmap_lock_speculate_try_begin(mm, &seq)) {
		779	found = __in_uprobe_trampoline(ip);
		780	retry = mmap_lock_speculate_retry(mm, seq);
		781	}
		782	rcu_read_unlock();
		783
		784	if (retry) {
		785	mmap_read_lock(mm);
		786	found = __in_uprobe_trampoline(ip);
		787	mmap_read_unlock(mm);
		788	}
		789	return found;
		790	}
		791
		792	/*
		793	* See uprobe syscall trampoline; the call to the trampoline will push
		794	* the return address on the stack, the trampoline itself then pushes
		795	* cx, r11 and ax.
		796	*/
		797	struct uprobe_syscall_args {
		798	unsigned long ax;
		799	unsigned long r11;
		800	unsigned long cx;
		801	unsigned long retaddr;
		802	};
		803
		804	SYSCALL_DEFINE0(uprobe)
		805	{
		806	struct pt_regs *regs = task_pt_regs(current);
		807	struct uprobe_syscall_args args;
		808	unsigned long ip, sp, sret;
		809	int err;
		810
		811	/* Allow execution only from uprobe trampolines. */
		812	if (!in_uprobe_trampoline(regs->ip))
		813	return -ENXIO;
		814
		815	err = copy_from_user(&args, (void __user *)regs->sp, sizeof(args));
		816	if (err)
		817	goto sigill;
		818
		819	ip = regs->ip;
		820
		821	/*
		822	* expose the "right" values of ax/r11/cx/ip/sp to uprobe_consumer/s, plus:
		823	* - adjust ip to the probe address, call saved next instruction address
		824	* - adjust sp to the probe's stack frame (check trampoline code)
		825	*/
		826	regs->ax = args.ax;
		827	regs->r11 = args.r11;
		828	regs->cx = args.cx;
		829	regs->ip = args.retaddr - 5;
		830	regs->sp += sizeof(args);
		831	regs->orig_ax = -1;
		832
		833	sp = regs->sp;
		834
		835	err = shstk_pop((u64 *)&sret);
		836	if (err == -EFAULT \|\| (!err && sret != args.retaddr))
		837	goto sigill;
		838
		839	handle_syscall_uprobe(regs, regs->ip);
		840
		841	/*
		842	* Some of the uprobe consumers has changed sp, we can do nothing,
		843	* just return via iret.
		844	*/
		845	if (regs->sp != sp) {
		846	/* skip the trampoline call */
		847	if (args.retaddr - 5 == regs->ip)
		848	regs->ip += 5;
		849	return regs->ax;
		850	}
		851
		852	regs->sp -= sizeof(args);
		853
		854	/* for the case uprobe_consumer has changed ax/r11/cx */
		855	args.ax = regs->ax;
		856	args.r11 = regs->r11;
		857	args.cx = regs->cx;
		858
		859	/* keep return address unless we are instructed otherwise */
		860	if (args.retaddr - 5 != regs->ip)
		861	args.retaddr = regs->ip;
		862
		863	if (shstk_push(args.retaddr) == -EFAULT)
		864	goto sigill;
		865
		866	regs->ip = ip;
		867
		868	err = copy_to_user((void __user *)regs->sp, &args, sizeof(args));
		869	if (err)
		870	goto sigill;
		871
		872	/* ensure sysret, see do_syscall_64() */
		873	regs->r11 = regs->flags;
		874	regs->cx = regs->ip;
		875	return 0;
		876
		877	sigill:
		878	force_sig(SIGILL);
		879	return -1;
		880	}
		881
		882	asm (
		883	".pushsection .rodata\n"
		884	".balign " __stringify(PAGE_SIZE) "\n"
		885	"uprobe_trampoline_entry:\n"
		886	"push %rcx\n"
		887	"push %r11\n"
		888	"push %rax\n"
		889	"mov $" __stringify(__NR_uprobe) ", %rax\n"
		890	"syscall\n"
		891	"pop %rax\n"
		892	"pop %r11\n"
		893	"pop %rcx\n"
		894	"ret\n"
		895	"int3\n"
		896	".balign " __stringify(PAGE_SIZE) "\n"
		897	".popsection\n"
		898	);
		899
		900	extern u8 uprobe_trampoline_entry[];
		901
		902	static int __init arch_uprobes_init(void)
		903	{
		904	tramp_mapping_pages[0] = virt_to_page(uprobe_trampoline_entry);
		905	return 0;
		906	}
		907
		908	late_initcall(arch_uprobes_init);
		909
		910	enum {
		911	EXPECT_SWBP,
		912	EXPECT_CALL,
		913	};
		914
		915	struct write_opcode_ctx {
		916	unsigned long base;
		917	int expect;
		918	};
		919
		920	static int is_call_insn(uprobe_opcode_t *insn)
		921	{
		922	return *insn == CALL_INSN_OPCODE;
		923	}
		924
		925	/*
		926	* Verification callback used by int3_update uprobe_write calls to make sure
		927	* the underlying instruction is as expected - either int3 or call.
		928	*/
		929	static int verify_insn(struct page page, unsigned long vaddr, uprobe_opcode_t new_opcode,
		930	int nbytes, void *data)
		931	{
		932	struct write_opcode_ctx *ctx = data;
		933	uprobe_opcode_t old_opcode[5];
		934
		935	uprobe_copy_from_page(page, ctx->base, (uprobe_opcode_t *) &old_opcode, 5);
		936
		937	switch (ctx->expect) {
		938	case EXPECT_SWBP:
		939	if (is_swbp_insn(&old_opcode[0]))
		940	return 1;
		941	break;
		942	case EXPECT_CALL:
		943	if (is_call_insn(&old_opcode[0]))
		944	return 1;
		945	break;
		946	}
		947
		948	return -1;
		949	}
		950
		951	/*
		952	* Modify multi-byte instructions by using INT3 breakpoints on SMP.
		953	* We completely avoid using stop_machine() here, and achieve the
		954	* synchronization using INT3 breakpoints and SMP cross-calls.
		955	* (borrowed comment from smp_text_poke_batch_finish)
		956	*
		957	* The way it is done:
		958	* - Add an INT3 trap to the address that will be patched
		959	* - SMP sync all CPUs
		960	* - Update all but the first byte of the patched range
		961	* - SMP sync all CPUs
		962	* - Replace the first byte (INT3) by the first byte of the replacing opcode
		963	* - SMP sync all CPUs
		964	*/
		965	static int int3_update(struct arch_uprobe auprobe, struct vm_area_struct vma,
		966	unsigned long vaddr, char *insn, bool optimize)
		967	{
		968	uprobe_opcode_t int3 = UPROBE_SWBP_INSN;
		969	struct write_opcode_ctx ctx = {
		970	.base = vaddr,
		971	};
		972	int err;
		973
		974	/*
		975	* Write int3 trap.
		976	*
		977	* The swbp_optimize path comes with breakpoint already installed,
		978	* so we can skip this step for optimize == true.
		979	*/
		980	if (!optimize) {
		981	ctx.expect = EXPECT_CALL;
		982	err = uprobe_write(auprobe, vma, vaddr, &int3, 1, verify_insn,
		983	true /* is_register /, false / do_update_ref_ctr */,
		984	&ctx);
		985	if (err)
		986	return err;
		987	}
		988
		989	smp_text_poke_sync_each_cpu();
		990
		991	/* Write all but the first byte of the patched range. */
		992	ctx.expect = EXPECT_SWBP;
		993	err = uprobe_write(auprobe, vma, vaddr + 1, insn + 1, 4, verify_insn,
		994	true /* is_register /, false / do_update_ref_ctr */,
		995	&ctx);
		996	if (err)
		997	return err;
		998
		999	smp_text_poke_sync_each_cpu();
		1000
		1001	/*
		1002	* Write first byte.
		1003	*
		1004	* The swbp_unoptimize needs to finish uprobe removal together
		1005	* with ref_ctr update, using uprobe_write with proper flags.
		1006	*/
		1007	err = uprobe_write(auprobe, vma, vaddr, insn, 1, verify_insn,
		1008	optimize /* is_register /, !optimize / do_update_ref_ctr */,
		1009	&ctx);
		1010	if (err)
		1011	return err;
		1012
		1013	smp_text_poke_sync_each_cpu();
		1014	return 0;
		1015	}
		1016
		1017	static int swbp_optimize(struct arch_uprobe auprobe, struct vm_area_struct vma,
		1018	unsigned long vaddr, unsigned long tramp)
		1019	{
		1020	u8 call[5];
		1021
		1022	__text_gen_insn(call, CALL_INSN_OPCODE, (const void *) vaddr,
		1023	(const void *) tramp, CALL_INSN_SIZE);
		1024	return int3_update(auprobe, vma, vaddr, call, true /* optimize */);
		1025	}
		1026
		1027	static int swbp_unoptimize(struct arch_uprobe auprobe, struct vm_area_struct vma,
		1028	unsigned long vaddr)
		1029	{
		1030	return int3_update(auprobe, vma, vaddr, auprobe->insn, false /* optimize */);
		1031	}
		1032
		1033	static int copy_from_vaddr(struct mm_struct mm, unsigned long vaddr, void dst, int len)
		1034	{
		1035	unsigned int gup_flags = FOLL_FORCE\|FOLL_SPLIT_PMD;
		1036	struct vm_area_struct *vma;
		1037	struct page *page;
		1038
		1039	page = get_user_page_vma_remote(mm, vaddr, gup_flags, &vma);
		1040	if (IS_ERR(page))
		1041	return PTR_ERR(page);
		1042	uprobe_copy_from_page(page, vaddr, dst, len);
		1043	put_page(page);
		1044	return 0;
		1045	}
		1046
		1047	static bool __is_optimized(uprobe_opcode_t *insn, unsigned long vaddr)
		1048	{
		1049	struct __packed __arch_relative_insn {
		1050	u8 op;
		1051	s32 raddr;
		1052	} call = (struct __arch_relative_insn ) insn;
		1053
		1054	if (!is_call_insn(insn))
		1055	return false;
		1056	return __in_uprobe_trampoline(vaddr + 5 + call->raddr);
		1057	}
		1058
		1059	static int is_optimized(struct mm_struct *mm, unsigned long vaddr)
		1060	{
		1061	uprobe_opcode_t insn[5];
		1062	int err;
		1063
		1064	err = copy_from_vaddr(mm, vaddr, &insn, 5);
		1065	if (err)
		1066	return err;
		1067	return __is_optimized((uprobe_opcode_t *)&insn, vaddr);
		1068	}
		1069
		1070	static bool should_optimize(struct arch_uprobe *auprobe)
		1071	{
		1072	return !test_bit(ARCH_UPROBE_FLAG_OPTIMIZE_FAIL, &auprobe->flags) &&
		1073	test_bit(ARCH_UPROBE_FLAG_CAN_OPTIMIZE, &auprobe->flags);
		1074	}
		1075
		1076	int set_swbp(struct arch_uprobe auprobe, struct vm_area_struct vma,
		1077	unsigned long vaddr)
		1078	{
		1079	if (should_optimize(auprobe)) {
		1080	/*
		1081	* We could race with another thread that already optimized the probe,
		1082	* so let's not overwrite it with int3 again in this case.
		1083	*/
		1084	int ret = is_optimized(vma->vm_mm, vaddr);
		1085	if (ret < 0)
		1086	return ret;
		1087	if (ret)
		1088	return 0;
		1089	}
		1090	return uprobe_write_opcode(auprobe, vma, vaddr, UPROBE_SWBP_INSN,
		1091	true /* is_register */);
		1092	}
		1093
		1094	int set_orig_insn(struct arch_uprobe auprobe, struct vm_area_struct vma,
		1095	unsigned long vaddr)
		1096	{
		1097	if (test_bit(ARCH_UPROBE_FLAG_CAN_OPTIMIZE, &auprobe->flags)) {
		1098	int ret = is_optimized(vma->vm_mm, vaddr);
		1099	if (ret < 0)
		1100	return ret;
		1101	if (ret) {
		1102	ret = swbp_unoptimize(auprobe, vma, vaddr);
		1103	WARN_ON_ONCE(ret);
		1104	return ret;
		1105	}
		1106	}
		1107	return uprobe_write_opcode(auprobe, vma, vaddr, (uprobe_opcode_t )&auprobe->insn,
		1108	false /* is_register */);
		1109	}
		1110
		1111	static int __arch_uprobe_optimize(struct arch_uprobe auprobe, struct mm_struct mm,
		1112	unsigned long vaddr)
		1113	{
		1114	struct uprobe_trampoline *tramp;
		1115	struct vm_area_struct *vma;
		1116	bool new = false;
		1117	int err = 0;
		1118
		1119	vma = find_vma(mm, vaddr);
		1120	if (!vma)
		1121	return -EINVAL;
		1122	tramp = get_uprobe_trampoline(vaddr, &new);
		1123	if (!tramp)
		1124	return -EINVAL;
		1125	err = swbp_optimize(auprobe, vma, vaddr, tramp->vaddr);
		1126	if (WARN_ON_ONCE(err) && new)
		1127	destroy_uprobe_trampoline(tramp);
		1128	return err;
		1129	}
		1130
		1131	void arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr)
		1132	{
		1133	struct mm_struct *mm = current->mm;
		1134	uprobe_opcode_t insn[5];
		1135
		1136	if (!should_optimize(auprobe))
		1137	return;
		1138
		1139	mmap_write_lock(mm);
		1140
		1141	/*
		1142	* Check if some other thread already optimized the uprobe for us,
		1143	* if it's the case just go away silently.
		1144	*/
		1145	if (copy_from_vaddr(mm, vaddr, &insn, 5))
		1146	goto unlock;
		1147	if (!is_swbp_insn((uprobe_opcode_t*) &insn))
		1148	goto unlock;
		1149
		1150	/*
		1151	* If we fail to optimize the uprobe we set the fail bit so the
		1152	* above should_optimize will fail from now on.
		1153	*/
		1154	if (__arch_uprobe_optimize(auprobe, mm, vaddr))
		1155	set_bit(ARCH_UPROBE_FLAG_OPTIMIZE_FAIL, &auprobe->flags);
		1156
		1157	unlock:
		1158	mmap_write_unlock(mm);
		1159	}
		1160
		1161	static bool insn_is_nop(struct insn *insn)
		1162	{
		1163	return insn->opcode.nbytes == 1 && insn->opcode.bytes[0] == 0x90;
		1164	}
		1165
		1166	static bool insn_is_nopl(struct insn *insn)
		1167	{
		1168	if (insn->opcode.nbytes != 2)
		1169	return false;
		1170
		1171	if (insn->opcode.bytes[0] != 0x0f \|\| insn->opcode.bytes[1] != 0x1f)
		1172	return false;
		1173
		1174	if (!insn->modrm.nbytes)
		1175	return false;
		1176
		1177	if (X86_MODRM_REG(insn->modrm.bytes[0]) != 0)
		1178	return false;
		1179
		1180	/* 0f 1f /0 - NOPL */
		1181	return true;
		1182	}
		1183
		1184	static bool can_optimize(struct insn *insn, unsigned long vaddr)
		1185	{
		1186	if (!insn->x86_64 \|\| insn->length != 5)
		1187	return false;
		1188
		1189	if (!insn_is_nop(insn) && !insn_is_nopl(insn))
		1190	return false;
		1191
		1192	/* We can't do cross page atomic writes yet. */
		1193	return PAGE_SIZE - (vaddr & ~PAGE_MASK) >= 5;
		1194	}
611	#else /* 32-bit: */	1195	#else /* 32-bit: */
612	/*	1196	/*
613	* No RIP-relative addressing on 32-bit	1197	* No RIP-relative addressing on 32-bit
@@ -621,6 +1205,10 @@ static void riprel_pre_xol(struct arch_uprobe auprobe, struct pt_regs regs) · 7 unmodified lines
621	static void riprel_post_xol(struct arch_uprobe auprobe, struct pt_regs regs)	1205	static void riprel_post_xol(struct arch_uprobe auprobe, struct pt_regs regs)
622	{	1206	{
623	}	1207	}
		1208	static bool can_optimize(struct insn *insn, unsigned long vaddr)
		1209	{
		1210	return false;
		1211	}
624	#endif /* CONFIG_X86_64 */	1212	#endif /* CONFIG_X86_64 */
625		1213
626	struct uprobe_xol_ops {	1214	struct uprobe_xol_ops {
@@ -979,14 +1567,17 @@ static int push_setup_xol_ops(struct arch_uprobe auprobe, struct insn insn) · 352 unmodified lines
979	*/	1567	*/
980	int arch_uprobe_analyze_insn(struct arch_uprobe auprobe, struct mm_struct mm, unsigned long addr)	1568	int arch_uprobe_analyze_insn(struct arch_uprobe auprobe, struct mm_struct mm, unsigned long addr)
981	{	1569	{
982	struct insn insn;
983	u8 fix_ip_or_call = UPROBE_FIX_IP;	1570	u8 fix_ip_or_call = UPROBE_FIX_IP;
		1571	struct insn insn;
984	int ret;	1572	int ret;
985		1573
986	ret = uprobe_init_insn(auprobe, &insn, is_64bit_mm(mm));	1574	ret = uprobe_init_insn(auprobe, &insn, is_64bit_mm(mm));
987	if (ret)	1575	if (ret)
988	return ret;	1576	return ret;
989		1577
		1578	if (can_optimize(&insn, addr))
		1579	set_bit(ARCH_UPROBE_FLAG_CAN_OPTIMIZE, &auprobe->flags);
		1580
990	ret = branch_setup_xol_ops(auprobe, &insn);	1581	ret = branch_setup_xol_ops(auprobe, &insn);
991	if (ret != -ENOSYS)	1582	if (ret != -ENOSYS)
992	return ret;	1583	return ret;

arch/x86/kvm/pmu.h+1−1

@@ -13,7 +13,7 @@ · 12 unmodified lines
13	#define MSR_IA32_MISC_ENABLE_PMU_RO_MASK (MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL \| \	13	#define MSR_IA32_MISC_ENABLE_PMU_RO_MASK (MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL \| \
14	MSR_IA32_MISC_ENABLE_BTS_UNAVAIL)	14	MSR_IA32_MISC_ENABLE_BTS_UNAVAIL)
15		15
16	/* retrieve the 4 bits for EN and PMI out of IA32_FIXED_CTR_CTRL */	16	/* retrieve a fixed counter bits out of IA32_FIXED_CTR_CTRL */
17	#define fixed_ctrl_field(ctrl_reg, idx) \	17	#define fixed_ctrl_field(ctrl_reg, idx) \
18	(((ctrl_reg) >> ((idx) * INTEL_FIXED_BITS_STRIDE)) & INTEL_FIXED_BITS_MASK)	18	(((ctrl_reg) >> ((idx) * INTEL_FIXED_BITS_STRIDE)) & INTEL_FIXED_BITS_MASK)
19		19

include/linux/perf_event.h+2−2

@@ -859,7 +859,7 @@ struct perf_event { · 858 unmodified lines
859		859
860	/* mmap bits */	860	/* mmap bits */
861	struct mutex mmap_mutex;	861	struct mutex mmap_mutex;
862	atomic_t mmap_count;	862	refcount_t mmap_count;
863		863
864	struct perf_buffer *rb;	864	struct perf_buffer *rb;
865	struct list_head rb_entry;	865	struct list_head rb_entry;
@@ -1719,7 +1719,7 @@ DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry); · 853 unmodified lines
1719	extern void perf_callchain_user(struct perf_callchain_entry_ctx entry, struct pt_regs regs);	1719	extern void perf_callchain_user(struct perf_callchain_entry_ctx entry, struct pt_regs regs);
1720	extern void perf_callchain_kernel(struct perf_callchain_entry_ctx entry, struct pt_regs regs);	1720	extern void perf_callchain_kernel(struct perf_callchain_entry_ctx entry, struct pt_regs regs);
1721	extern struct perf_callchain_entry *	1721	extern struct perf_callchain_entry *
1722	get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user,	1722	get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
1723	u32 max_stack, bool crosstask, bool add_mark);	1723	u32 max_stack, bool crosstask, bool add_mark);
1724	extern int get_callchain_buffers(int max_stack);	1724	extern int get_callchain_buffers(int max_stack);
1725	extern void put_callchain_buffers(void);	1725	extern void put_callchain_buffers(void);

include/linux/syscalls.h+2−0

@@ -1005,6 +1005,8 @@ asmlinkage long sys_ioperm(unsigned long from, unsigned long num, int on); · 1004 unmodified lines
1005		1005
1006	asmlinkage long sys_uretprobe(void);	1006	asmlinkage long sys_uretprobe(void);
1007		1007
		1008	asmlinkage long sys_uprobe(void);
		1009
1008	/* pciconfig: alpha, arm, arm64, ia64, sparc */	1010	/* pciconfig: alpha, arm, arm64, ia64, sparc */
1009	asmlinkage long sys_pciconfig_read(unsigned long bus, unsigned long dfn,	1011	asmlinkage long sys_pciconfig_read(unsigned long bus, unsigned long dfn,
1010	unsigned long off, unsigned long len,	1012	unsigned long off, unsigned long len,

include/linux/uprobes.h+18−2

 #include <linux/wait.h>
 #include <linux/timer.h>
 #include <linux/seqlock.h>
+#include <linux/mutex.h>
 struct uprobe;
 struct vm_area_struct;
 struct uprobes_state {
 	struct xol_area		*xol_area;
+#ifdef CONFIG_X86_64
+	struct hlist_head	head_tramps;
+#endif
 };
+typedef int (*uprobe_write_verify_t)(struct page *page, unsigned long vaddr,
+				     uprobe_opcode_t *insn, int nbytes, void *data);
 extern void __init uprobes_init(void);
 extern int set_swbp(struct arch_uprobe *aup, struct vm_area_struct *vma, unsigned long vaddr);
 extern int set_orig_insn(struct arch_uprobe *aup, struct vm_area_struct *vma, unsigned long vaddr);
 extern bool is_trap_insn(uprobe_opcode_t *insn);
 extern unsigned long uprobe_get_swbp_addr(struct pt_regs *regs);
 extern unsigned long uprobe_get_trap_addr(struct pt_regs *regs);
-extern int uprobe_write_opcode(struct arch_uprobe *auprobe, struct vm_area_struct *vma, unsigned long vaddr, uprobe_opcode_t);
+extern int uprobe_write_opcode(struct arch_uprobe *auprobe, struct vm_area_struct *vma, unsigned long vaddr, uprobe_opcode_t,
+			       bool is_register);
+extern int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma, const unsigned long opcode_vaddr,
+			uprobe_opcode_t *insn, int nbytes, uprobe_write_verify_t verify, bool is_register, bool do_update_ref_ctr,
+			void *data);
 extern struct uprobe *uprobe_register(struct inode *inode, loff_t offset, loff_t ref_ctr_offset, struct uprobe_consumer *uc);
 extern int uprobe_apply(struct uprobe *uprobe, struct uprobe_consumer *uc, bool);
 extern void uprobe_unregister_nosync(struct uprobe *uprobe, struct uprobe_consumer *uc);
 extern void arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr,
 					 void *src, unsigned long len);
 extern void uprobe_handle_trampoline(struct pt_regs *regs);
-extern void *arch_uprobe_trampoline(unsigned long *psize);
+extern void *arch_uretprobe_trampoline(unsigned long *psize);
 extern unsigned long uprobe_get_trampoline_vaddr(void);
+extern void uprobe_copy_from_page(struct page *page, unsigned long vaddr, void *dst, int len);
+extern void arch_uprobe_clear_state(struct mm_struct *mm);
+extern void arch_uprobe_init_state(struct mm_struct *mm);
+extern void handle_syscall_uprobe(struct pt_regs *regs, unsigned long bp_vaddr);
+extern void arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr);
 #else /* !CONFIG_UPROBES */
 struct uprobes_state {
 };

kernel/bpf/stackmap.c+2−2

@@ -314,7 +314,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs , regs, struct bpf_map , map, · 313 unmodified lines
314	if (max_depth > sysctl_perf_event_max_stack)	314	if (max_depth > sysctl_perf_event_max_stack)
315	max_depth = sysctl_perf_event_max_stack;	315	max_depth = sysctl_perf_event_max_stack;
316		316
317	trace = get_perf_callchain(regs, 0, kernel, user, max_depth,	317	trace = get_perf_callchain(regs, kernel, user, max_depth,
318	false, false);	318	false, false);
319		319
320	if (unlikely(!trace))	320	if (unlikely(!trace))
@@ -451,7 +451,7 @@ static long __bpf_get_stack(struct pt_regs regs, struct task_struct task, · 130 unmodified lines
451	else if (kernel && task)	451	else if (kernel && task)
452	trace = get_callchain_entry_for_task(task, max_depth);	452	trace = get_callchain_entry_for_task(task, max_depth);
453	else	453	else
454	trace = get_perf_callchain(regs, 0, kernel, user, max_depth,	454	trace = get_perf_callchain(regs, kernel, user, max_depth,
455	crosstask, false);	455	crosstask, false);
456		456
457	if (unlikely(!trace) \|\| trace->nr < skip) {	457	if (unlikely(!trace) \|\| trace->nr < skip) {

kernel/events/callchain.c+19−21

 }
 struct perf_callchain_entry *
-get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user,
+get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
 		   u32 max_stack, bool crosstask, bool add_mark)
 {
 	struct perf_callchain_entry *entry;
 	struct perf_callchain_entry_ctx ctx;
 	int rctx, start_entry_idx;
+	/* crosstask is not supported for user stacks */
+	if (crosstask && user && !kernel)
+		return NULL;
 	entry = get_callchain_entry(&rctx);
 	if (!entry)
 		return NULL;
-	ctx.entry     = entry;
+	ctx.entry		= entry;
-	ctx.max_stack = max_stack;
+	ctx.max_stack		= max_stack;
-	ctx.nr	      = entry->nr = init_nr;
+	ctx.nr			= entry->nr = 0;
-	ctx.contexts       = 0;
+	ctx.contexts		= 0;
-	ctx.contexts_maxed = false;
+	ctx.contexts_maxed	= false;
 	if (kernel && !user_mode(regs)) {
 		if (add_mark)
 		perf_callchain_kernel(&ctx, regs);
 	}
-	if (user) {
+	if (user && !crosstask) {
 		if (!user_mode(regs)) {
-			if  (current->mm)
+			if (current->flags & (PF_KTHREAD | PF_USER_WORKER))
-				regs = task_pt_regs(current);
-			else
-				regs = NULL;
-		}
-		if (regs) {
-			if (crosstask)
 				goto exit_put;
+			regs = task_pt_regs(current);
+		}
-			if (add_mark)
+		if (add_mark)
-				perf_callchain_store_context(&ctx, PERF_CONTEXT_USER);
+			perf_callchain_store_context(&ctx, PERF_CONTEXT_USER);
-			start_entry_idx = entry->nr;
+		start_entry_idx = entry->nr;
-			perf_callchain_user(&ctx, regs);
+		perf_callchain_user(&ctx, regs);
-			fixup_uretprobe_trampoline_entries(entry, start_entry_idx);
+		fixup_uretprobe_trampoline_entries(entry, start_entry_idx);
-		}
 	}
 exit_put:

kernel/events/core.c+191−178

@@ -3974,7 +3974,7 @@ static noinline int visit_groups_merge(struct perf_event_context *ctx, · 3973 unmodified lines
3974	*/	3974	*/
3975	static inline bool event_update_userpage(struct perf_event *event)	3975	static inline bool event_update_userpage(struct perf_event *event)
3976	{	3976	{
3977	if (likely(!atomic_read(&event->mmap_count)))	3977	if (likely(!refcount_read(&event->mmap_count)))
3978	return false;	3978	return false;
3979		3979
3980	perf_event_update_time(event);	3980	perf_event_update_time(event);
@@ -6710,11 +6710,11 @@ static void perf_mmap_open(struct vm_area_struct *vma) · 2729 unmodified lines
6710	struct perf_event *event = vma->vm_file->private_data;	6710	struct perf_event *event = vma->vm_file->private_data;
6711	mapped_f mapped = get_mapped(event, event_mapped);	6711	mapped_f mapped = get_mapped(event, event_mapped);
6712		6712
6713	atomic_inc(&event->mmap_count);	6713	refcount_inc(&event->mmap_count);
6714	atomic_inc(&event->rb->mmap_count);	6714	refcount_inc(&event->rb->mmap_count);
6715		6715
6716	if (vma->vm_pgoff)	6716	if (vma->vm_pgoff)
6717	atomic_inc(&event->rb->aux_mmap_count);	6717	refcount_inc(&event->rb->aux_mmap_count);
6718		6718
6719	if (mapped)	6719	if (mapped)
6720	mapped(event, vma->vm_mm);	6720	mapped(event, vma->vm_mm);
@@ -6749,7 +6749,7 @@ static void perf_mmap_close(struct vm_area_struct *vma) · 28 unmodified lines
6749	* to avoid complications.	6749	* to avoid complications.
6750	*/	6750	*/
6751	if (rb_has_aux(rb) && vma->vm_pgoff == rb->aux_pgoff &&	6751	if (rb_has_aux(rb) && vma->vm_pgoff == rb->aux_pgoff &&
6752	atomic_dec_and_mutex_lock(&rb->aux_mmap_count, &rb->aux_mutex)) {	6752	refcount_dec_and_mutex_lock(&rb->aux_mmap_count, &rb->aux_mutex)) {
6753	/*	6753	/*
6754	* Stop all AUX events that are writing to this buffer,	6754	* Stop all AUX events that are writing to this buffer,
6755	* so that we can free its AUX pages and corresponding PMU	6755	* so that we can free its AUX pages and corresponding PMU
@@ -6769,10 +6769,10 @@ static void perf_mmap_close(struct vm_area_struct *vma) · 13 unmodified lines
6769	mutex_unlock(&rb->aux_mutex);	6769	mutex_unlock(&rb->aux_mutex);
6770	}	6770	}
6771		6771
6772	if (atomic_dec_and_test(&rb->mmap_count))	6772	if (refcount_dec_and_test(&rb->mmap_count))
6773	detach_rest = true;	6773	detach_rest = true;
6774		6774
6775	if (!atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex))	6775	if (!refcount_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex))
6776	goto out_put;	6776	goto out_put;
6777		6777
6778	ring_buffer_attach(event, NULL);	6778	ring_buffer_attach(event, NULL);
@@ -6933,230 +6933,242 @@ static int map_range(struct perf_buffer rb, struct vm_area_struct vma) · 154 unmodified lines
6933	return err;	6933	return err;
6934	}	6934	}
6935		6935
6936	static int perf_mmap(struct file file, struct vm_area_struct vma)	6936	static bool perf_mmap_calc_limits(struct vm_area_struct vma, long user_extra, long *extra)
6937	{	6937	{
6938	struct perf_event *event = file->private_data;	6938	unsigned long user_locked, user_lock_limit, locked, lock_limit;
6939	unsigned long user_locked, user_lock_limit;
6940	struct user_struct *user = current_user();	6939	struct user_struct *user = current_user();
6941	struct mutex *aux_mutex = NULL;	6940
6942	struct perf_buffer *rb = NULL;	6941	user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
6943	unsigned long locked, lock_limit;	6942	/* Increase the limit linearly with more CPUs */
6944	unsigned long vma_size;	6943	user_lock_limit *= num_online_cpus();
6945	unsigned long nr_pages;	6944
6946	long user_extra = 0, extra = 0;	6945	user_locked = atomic_long_read(&user->locked_vm);
6947	int ret, flags = 0;
6948	mapped_f mapped;
6949		6946
6950	/*	6947	/*
6951	* Don't allow mmap() of inherited per-task counters. This would	6948	* sysctl_perf_event_mlock may have changed, so that
6952	* create a performance issue due to all children writing to the	6949	* user->locked_vm > user_lock_limit
6953	* same rb.
6954	*/	6950	*/
6955	if (event->cpu == -1 && event->attr.inherit)	6951	if (user_locked > user_lock_limit)
6956	return -EINVAL;	6952	user_locked = user_lock_limit;
		6953	user_locked += *user_extra;
6957		6954
6958	if (!(vma->vm_flags & VM_SHARED))	6955	if (user_locked > user_lock_limit) {
6959	return -EINVAL;	6956	/*
		6957	* charge locked_vm until it hits user_lock_limit;
		6958	* charge the rest from pinned_vm
		6959	*/
		6960	*extra = user_locked - user_lock_limit;
		6961	user_extra -= extra;
		6962	}
6960		6963
6961	ret = security_perf_event_read(event);	6964	lock_limit = rlimit(RLIMIT_MEMLOCK);
6962	if (ret)	6965	lock_limit >>= PAGE_SHIFT;
6963	return ret;	6966	locked = atomic64_read(&vma->vm_mm->pinned_vm) + *extra;
6964		6967
6965	vma_size = vma->vm_end - vma->vm_start;	6968	return locked <= lock_limit \|\| !perf_is_paranoid() \|\| capable(CAP_IPC_LOCK);
6966	nr_pages = vma_size / PAGE_SIZE;	6969	}
6967		6970
6968	if (nr_pages > INT_MAX)	6971	static void perf_mmap_account(struct vm_area_struct *vma, long user_extra, long extra)
6969	return -ENOMEM;	6972	{
		6973	struct user_struct *user = current_user();
6970		6974
6971	if (vma_size != PAGE_SIZE * nr_pages)	6975	atomic_long_add(user_extra, &user->locked_vm);
6972	return -EINVAL;	6976	atomic64_add(extra, &vma->vm_mm->pinned_vm);
		6977	}
6973		6978
6974	user_extra = nr_pages;	6979	static int perf_mmap_rb(struct vm_area_struct vma, struct perf_event event,
		6980	unsigned long nr_pages)
		6981	{
		6982	long extra = 0, user_extra = nr_pages;
		6983	struct perf_buffer *rb;
		6984	int rb_flags = 0;
6975		6985
6976	mutex_lock(&event->mmap_mutex);	6986	nr_pages -= 1;
6977	ret = -EINVAL;
6978		6987
6979	/*	6988	/*
6980	* This relies on __pmu_detach_event() taking mmap_mutex after marking	6989	* If we have rb pages ensure they're a power-of-two number, so we
6981	* the event REVOKED. Either we observe the state, or __pmu_detach_event()	6990	* can do bitmasks instead of modulo.
6982	* will detach the rb created here.
6983	*/	6991	*/
6984	if (event->state <= PERF_EVENT_STATE_REVOKED) {	6992	if (nr_pages != 0 && !is_power_of_2(nr_pages))
6985	ret = -ENODEV;	6993	return -EINVAL;
6986	goto unlock;
6987	}
6988
6989	if (vma->vm_pgoff == 0) {
6990	nr_pages -= 1;
6991
6992	/*
6993	* If we have rb pages ensure they're a power-of-two number, so we
6994	* can do bitmasks instead of modulo.
6995	*/
6996	if (nr_pages != 0 && !is_power_of_2(nr_pages))
6997	goto unlock;
6998
6999	WARN_ON_ONCE(event->ctx->parent_ctx);
7000		6994
7001	if (event->rb) {	6995	WARN_ON_ONCE(event->ctx->parent_ctx);
7002	if (data_page_nr(event->rb) != nr_pages)
7003	goto unlock;
7004		6996
7005	if (atomic_inc_not_zero(&event->rb->mmap_count)) {	6997	if (event->rb) {
7006	/*	6998	if (data_page_nr(event->rb) != nr_pages)
7007	* Success -- managed to mmap() the same buffer	6999	return -EINVAL;
7008	* multiple times.
7009	*/
7010	ret = 0;
7011	/* We need the rb to map pages. */
7012	rb = event->rb;
7013	goto unlock;
7014	}
7015		7000
		7001	if (refcount_inc_not_zero(&event->rb->mmap_count)) {
7016	/*	7002	/*
7017	* Raced against perf_mmap_close()'s	7003	* Success -- managed to mmap() the same buffer
7018	* atomic_dec_and_mutex_lock() remove the	7004	* multiple times.
7019	* event and continue as if !event->rb
7020	*/	7005	*/
7021	ring_buffer_attach(event, NULL);	7006	perf_mmap_account(vma, user_extra, extra);
		7007	refcount_inc(&event->mmap_count);
		7008	return 0;
7022	}	7009	}
7023		7010
7024	} else {
7025	/*	7011	/*
7026	* AUX area mapping: if rb->aux_nr_pages != 0, it's already	7012	* Raced against perf_mmap_close()'s
7027	* mapped, all subsequent mappings should have the same size	7013	* refcount_dec_and_mutex_lock() remove the
7028	* and offset. Must be above the normal perf buffer.	7014	* event and continue as if !event->rb
7029	*/	7015	*/
7030	u64 aux_offset, aux_size;	7016	ring_buffer_attach(event, NULL);
		7017	}
7031		7018
7032	rb = event->rb;	7019	if (!perf_mmap_calc_limits(vma, &user_extra, &extra))
7033	if (!rb)	7020	return -EPERM;
7034	goto aux_unlock;
7035		7021
7036	aux_mutex = &rb->aux_mutex;	7022	if (vma->vm_flags & VM_WRITE)
7037	mutex_lock(aux_mutex);	7023	rb_flags \|= RING_BUFFER_WRITABLE;
7038		7024
7039	aux_offset = READ_ONCE(rb->user_page->aux_offset);	7025	rb = rb_alloc(nr_pages,
7040	aux_size = READ_ONCE(rb->user_page->aux_size);	7026	event->attr.watermark ? event->attr.wakeup_watermark : 0,
		7027	event->cpu, rb_flags);
7041		7028
7042	if (aux_offset < perf_data_size(rb) + PAGE_SIZE)	7029	if (!rb)
7043	goto aux_unlock;	7030	return -ENOMEM;
7044		7031
7045	if (aux_offset != vma->vm_pgoff << PAGE_SHIFT)	7032	refcount_set(&rb->mmap_count, 1);
7046	goto aux_unlock;	7033	rb->mmap_user = get_current_user();
		7034	rb->mmap_locked = extra;
7047		7035
7048	/* already mapped with a different offset */	7036	ring_buffer_attach(event, rb);
7049	if (rb_has_aux(rb) && rb->aux_pgoff != vma->vm_pgoff)
7050	goto aux_unlock;
7051		7037
7052	if (aux_size != vma_size \|\| aux_size != nr_pages * PAGE_SIZE)	7038	perf_event_update_time(event);
7053	goto aux_unlock;	7039	perf_event_init_userpage(event);
		7040	perf_event_update_userpage(event);
7054		7041
7055	/* already mapped with a different size */	7042	perf_mmap_account(vma, user_extra, extra);
7056	if (rb_has_aux(rb) && rb->aux_nr_pages != nr_pages)	7043	refcount_set(&event->mmap_count, 1);
7057	goto aux_unlock;
7058		7044
7059	if (!is_power_of_2(nr_pages))	7045	return 0;
7060	goto aux_unlock;	7046	}
7061		7047
7062	if (!atomic_inc_not_zero(&rb->mmap_count))	7048	static int perf_mmap_aux(struct vm_area_struct vma, struct perf_event event,
7063	goto aux_unlock;	7049	unsigned long nr_pages)
		7050	{
		7051	long extra = 0, user_extra = nr_pages;
		7052	u64 aux_offset, aux_size;
		7053	struct perf_buffer *rb;
		7054	int ret, rb_flags = 0;
7064		7055
7065	if (rb_has_aux(rb)) {	7056	rb = event->rb;
7066	atomic_inc(&rb->aux_mmap_count);	7057	if (!rb)
7067	ret = 0;	7058	return -EINVAL;
7068	goto unlock;
7069	}
7070	}
7071		7059
7072	user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);	7060	guard(mutex)(&rb->aux_mutex);
7073		7061
7074	/*	7062	/*
7075	* Increase the limit linearly with more CPUs:	7063	* AUX area mapping: if rb->aux_nr_pages != 0, it's already
		7064	* mapped, all subsequent mappings should have the same size
		7065	* and offset. Must be above the normal perf buffer.
7076	*/	7066	*/
7077	user_lock_limit *= num_online_cpus();	7067	aux_offset = READ_ONCE(rb->user_page->aux_offset);
		7068	aux_size = READ_ONCE(rb->user_page->aux_size);
7078		7069
7079	user_locked = atomic_long_read(&user->locked_vm);	7070	if (aux_offset < perf_data_size(rb) + PAGE_SIZE)
		7071	return -EINVAL;
7080		7072
7081	/*	7073	if (aux_offset != vma->vm_pgoff << PAGE_SHIFT)
7082	* sysctl_perf_event_mlock may have changed, so that	7074	return -EINVAL;
7083	* user->locked_vm > user_lock_limit
7084	*/
7085	if (user_locked > user_lock_limit)
7086	user_locked = user_lock_limit;
7087	user_locked += user_extra;
7088		7075
7089	if (user_locked > user_lock_limit) {	7076	/* already mapped with a different offset */
7090	/*	7077	if (rb_has_aux(rb) && rb->aux_pgoff != vma->vm_pgoff)
7091	* charge locked_vm until it hits user_lock_limit;	7078	return -EINVAL;
7092	* charge the rest from pinned_vm
7093	*/
7094	extra = user_locked - user_lock_limit;
7095	user_extra -= extra;
7096	}
7097		7079
7098	lock_limit = rlimit(RLIMIT_MEMLOCK);	7080	if (aux_size != nr_pages * PAGE_SIZE)
7099	lock_limit >>= PAGE_SHIFT;	7081	return -EINVAL;
7100	locked = atomic64_read(&vma->vm_mm->pinned_vm) + extra;
7101		7082
7102	if ((locked > lock_limit) && perf_is_paranoid() &&	7083	/* already mapped with a different size */
7103	!capable(CAP_IPC_LOCK)) {	7084	if (rb_has_aux(rb) && rb->aux_nr_pages != nr_pages)
7104	ret = -EPERM;	7085	return -EINVAL;
7105	goto unlock;
7106	}
7107		7086
7108	WARN_ON(!rb && event->rb);	7087	if (!is_power_of_2(nr_pages))
		7088	return -EINVAL;
7109		7089
7110	if (vma->vm_flags & VM_WRITE)	7090	if (!refcount_inc_not_zero(&rb->mmap_count))
7111	flags \|= RING_BUFFER_WRITABLE;	7091	return -EINVAL;
7112		7092
7113	if (!rb) {	7093	if (rb_has_aux(rb)) {
7114	rb = rb_alloc(nr_pages,	7094	refcount_inc(&rb->aux_mmap_count);
7115	event->attr.watermark ? event->attr.wakeup_watermark : 0,
7116	event->cpu, flags);
7117		7095
7118	if (!rb) {	7096	} else {
7119	ret = -ENOMEM;	7097	if (!perf_mmap_calc_limits(vma, &user_extra, &extra)) {
7120	goto unlock;	7098	refcount_dec(&rb->mmap_count);
		7099	return -EPERM;
7121	}	7100	}
7122		7101
7123	atomic_set(&rb->mmap_count, 1);	7102	WARN_ON(!rb && event->rb);
7124	rb->mmap_user = get_current_user();
7125	rb->mmap_locked = extra;
7126		7103
7127	ring_buffer_attach(event, rb);	7104	if (vma->vm_flags & VM_WRITE)
		7105	rb_flags \|= RING_BUFFER_WRITABLE;
7128		7106
7129	perf_event_update_time(event);
7130	perf_event_init_userpage(event);
7131	perf_event_update_userpage(event);
7132	ret = 0;
7133	} else {
7134	ret = rb_alloc_aux(rb, event, vma->vm_pgoff, nr_pages,	7107	ret = rb_alloc_aux(rb, event, vma->vm_pgoff, nr_pages,
7135	event->attr.aux_watermark, flags);	7108	event->attr.aux_watermark, rb_flags);
7136	if (!ret) {	7109	if (ret) {
7137	atomic_set(&rb->aux_mmap_count, 1);	7110	refcount_dec(&rb->mmap_count);
7138	rb->aux_mmap_locked = extra;	7111	return ret;
7139	}	7112	}
		7113
		7114	refcount_set(&rb->aux_mmap_count, 1);
		7115	rb->aux_mmap_locked = extra;
7140	}	7116	}
7141		7117
7142	unlock:	7118	perf_mmap_account(vma, user_extra, extra);
7143	if (!ret) {	7119	refcount_inc(&event->mmap_count);
7144	atomic_long_add(user_extra, &user->locked_vm);
7145	atomic64_add(extra, &vma->vm_mm->pinned_vm);
7146
7147	atomic_inc(&event->mmap_count);
7148	} else if (rb) {
7149	/* AUX allocation failed */
7150	atomic_dec(&rb->mmap_count);
7151	}
7152	aux_unlock:
7153	if (aux_mutex)
7154	mutex_unlock(aux_mutex);
7155	mutex_unlock(&event->mmap_mutex);
7156		7120
		7121	return 0;
		7122	}
		7123
		7124	static int perf_mmap(struct file file, struct vm_area_struct vma)
		7125	{
		7126	struct perf_event *event = file->private_data;
		7127	unsigned long vma_size, nr_pages;
		7128	mapped_f mapped;
		7129	int ret;
		7130
		7131	/*
		7132	* Don't allow mmap() of inherited per-task counters. This would
		7133	* create a performance issue due to all children writing to the
		7134	* same rb.
		7135	*/
		7136	if (event->cpu == -1 && event->attr.inherit)
		7137	return -EINVAL;
		7138
		7139	if (!(vma->vm_flags & VM_SHARED))
		7140	return -EINVAL;
		7141
		7142	ret = security_perf_event_read(event);
7157	if (ret)	7143	if (ret)
7158	return ret;	7144	return ret;
7159		7145
		7146	vma_size = vma->vm_end - vma->vm_start;
		7147	nr_pages = vma_size / PAGE_SIZE;
		7148
		7149	if (nr_pages > INT_MAX)
		7150	return -ENOMEM;
		7151
		7152	if (vma_size != PAGE_SIZE * nr_pages)
		7153	return -EINVAL;
		7154
		7155	scoped_guard (mutex, &event->mmap_mutex) {
		7156	/*
		7157	* This relies on __pmu_detach_event() taking mmap_mutex after marking
		7158	* the event REVOKED. Either we observe the state, or __pmu_detach_event()
		7159	* will detach the rb created here.
		7160	*/
		7161	if (event->state <= PERF_EVENT_STATE_REVOKED)
		7162	return -ENODEV;
		7163
		7164	if (vma->vm_pgoff == 0)
		7165	ret = perf_mmap_rb(vma, event, nr_pages);
		7166	else
		7167	ret = perf_mmap_aux(vma, event, nr_pages);
		7168	if (ret)
		7169	return ret;
		7170	}
		7171
7160	/*	7172	/*
7161	* Since pinned accounting is per vm we cannot allow fork() to copy our	7173	* Since pinned accounting is per vm we cannot allow fork() to copy our
7162	* vma.	7174	* vma.
@@ -7174,7 +7186,7 @@ aux_unlock: · 11 unmodified lines
7174	* full cleanup in this case and therefore does not invoke	7186	* full cleanup in this case and therefore does not invoke
7175	* vmops::close().	7187	* vmops::close().
7176	*/	7188	*/
7177	ret = map_range(rb, vma);	7189	ret = map_range(event->rb, vma);
7178	if (ret)	7190	if (ret)
7179	perf_mmap_close(vma);	7191	perf_mmap_close(vma);
7180		7192
@@ -7440,7 +7452,7 @@ static void perf_sample_regs_user(struct perf_regs *regs_user, · 259 unmodified lines
7440	if (user_mode(regs)) {	7452	if (user_mode(regs)) {
7441	regs_user->abi = perf_reg_abi(current);	7453	regs_user->abi = perf_reg_abi(current);
7442	regs_user->regs = regs;	7454	regs_user->regs = regs;
7443	} else if (!(current->flags & PF_KTHREAD)) {	7455	} else if (!(current->flags & (PF_KTHREAD \| PF_USER_WORKER))) {
7444	perf_get_regs_user(regs_user, regs);	7456	perf_get_regs_user(regs_user, regs);
7445	} else {	7457	} else {
7446	regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE;	7458	regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE;
@@ -8080,7 +8092,7 @@ static u64 perf_virt_to_phys(u64 virt) · 633 unmodified lines
8080	* Try IRQ-safe get_user_page_fast_only first.	8092	* Try IRQ-safe get_user_page_fast_only first.
8081	* If failed, leave phys_addr as 0.	8093	* If failed, leave phys_addr as 0.
8082	*/	8094	*/
8083	if (current->mm != NULL) {	8095	if (!(current->flags & (PF_KTHREAD \| PF_USER_WORKER))) {
8084	struct page *p;	8096	struct page *p;
8085		8097
8086	pagefault_disable();	8098	pagefault_disable();
@@ -8192,7 +8204,8 @@ struct perf_callchain_entry * · 105 unmodified lines
8192	perf_callchain(struct perf_event event, struct pt_regs regs)	8204	perf_callchain(struct perf_event event, struct pt_regs regs)
8193	{	8205	{
8194	bool kernel = !event->attr.exclude_callchain_kernel;	8206	bool kernel = !event->attr.exclude_callchain_kernel;
8195	bool user = !event->attr.exclude_callchain_user;	8207	bool user = !event->attr.exclude_callchain_user &&
		8208	!(current->flags & (PF_KTHREAD \| PF_USER_WORKER));
8196	/* Disallow cross-task user callchains. */	8209	/* Disallow cross-task user callchains. */
8197	bool crosstask = event->ctx->task && event->ctx->task != current;	8210	bool crosstask = event->ctx->task && event->ctx->task != current;
8198	const u32 max_stack = event->attr.sample_max_stack;	8211	const u32 max_stack = event->attr.sample_max_stack;
@@ -8204,7 +8217,7 @@ perf_callchain(struct perf_event event, struct pt_regs regs) · 5 unmodified lines
8204	if (!kernel && !user)	8217	if (!kernel && !user)
8205	return &__empty_callchain;	8218	return &__empty_callchain;
8206		8219
8207	callchain = get_perf_callchain(regs, 0, kernel, user,	8220	callchain = get_perf_callchain(regs, kernel, user,
8208	max_stack, crosstask, true);	8221	max_stack, crosstask, true);
8209	return callchain ?: &__empty_callchain;	8222	return callchain ?: &__empty_callchain;
8210	}	8223	}
@@ -13249,7 +13262,7 @@ perf_event_set_output(struct perf_event event, struct perf_event output_event) · 5038 unmodified lines
13249	mutex_lock_double(&event->mmap_mutex, &output_event->mmap_mutex);	13262	mutex_lock_double(&event->mmap_mutex, &output_event->mmap_mutex);
13250	set:	13263	set:
13251	/* Can't redirect output if we've got an active mmap() */	13264	/* Can't redirect output if we've got an active mmap() */
13252	if (atomic_read(&event->mmap_count))	13265	if (refcount_read(&event->mmap_count))
13253	goto unlock;	13266	goto unlock;
13254		13267
13255	if (output_event) {	13268	if (output_event) {
@@ -13262,7 +13275,7 @@ set: · 6 unmodified lines
13262	goto unlock;	13275	goto unlock;
13263		13276
13264	/* did we race against perf_mmap_close() */	13277	/* did we race against perf_mmap_close() */
13265	if (!atomic_read(&rb->mmap_count)) {	13278	if (!refcount_read(&rb->mmap_count)) {
13266	ring_buffer_put(rb);	13279	ring_buffer_put(rb);
13267	goto unlock;	13280	goto unlock;
13268	}	13281	}

kernel/events/internal.h+2−2

@@ -35,7 +35,7 @@ struct perf_buffer { · 34 unmodified lines
35	spinlock_t event_lock;	35	spinlock_t event_lock;
36	struct list_head event_list;	36	struct list_head event_list;
37		37
38	atomic_t mmap_count;	38	refcount_t mmap_count;
39	unsigned long mmap_locked;	39	unsigned long mmap_locked;
40	struct user_struct *mmap_user;	40	struct user_struct *mmap_user;
41		41
@@ -47,7 +47,7 @@ struct perf_buffer { · 5 unmodified lines
47	unsigned long aux_pgoff;	47	unsigned long aux_pgoff;
48	int aux_nr_pages;	48	int aux_nr_pages;
49	int aux_overwrite;	49	int aux_overwrite;
50	atomic_t aux_mmap_count;	50	refcount_t aux_mmap_count;
51	unsigned long aux_mmap_locked;	51	unsigned long aux_mmap_locked;
52	void (free_aux)(void );	52	void (free_aux)(void );
53	refcount_t aux_refcount;	53	refcount_t aux_refcount;

kernel/events/ring_buffer.c+1−1

@@ -400,7 +400,7 @@ void perf_aux_output_begin(struct perf_output_handle handle, · 399 unmodified lines
400	* the same order, see perf_mmap_close. Otherwise we end up freeing	400	* the same order, see perf_mmap_close. Otherwise we end up freeing
401	* aux pages in this path, which is a bug, because in_atomic().	401	* aux pages in this path, which is a bug, because in_atomic().
402	*/	402	*/
403	if (!atomic_read(&rb->aux_mmap_count))	403	if (!refcount_read(&rb->aux_mmap_count))
404	goto err;	404	goto err;
405		405
406	if (!refcount_inc_not_zero(&rb->aux_refcount))	406	if (!refcount_inc_not_zero(&rb->aux_refcount))

kernel/events/uprobes.c+73−29

 	return is_swbp_insn(insn);
 }
-static void copy_from_page(struct page *page, unsigned long vaddr, void *dst, int len)
+void uprobe_copy_from_page(struct page *page, unsigned long vaddr, void *dst, int len)
 {
 	void *kaddr = kmap_atomic(page);
 	memcpy(dst, kaddr + (vaddr & ~PAGE_MASK), len);
 	kunmap_atomic(kaddr);
 }
-static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t *new_opcode)
+static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t *insn,
+			 int nbytes, void *data)
 {
 	uprobe_opcode_t old_opcode;
 	bool is_swbp;
 	 * is a trap variant; uprobes always wins over any other (gdb)
 	 * breakpoint.
 	 */
-	copy_from_page(page, vaddr, &old_opcode, UPROBE_SWBP_INSN_SIZE);
+	uprobe_copy_from_page(page, vaddr, &old_opcode, UPROBE_SWBP_INSN_SIZE);
 	is_swbp = is_swbp_insn(&old_opcode);
-	if (is_swbp_insn(new_opcode)) {
+	if (is_swbp_insn(insn)) {
 		if (is_swbp)		/* register: already installed? */
 			return 0;
 	} else {
 	return identical;
 }
-static int __uprobe_write_opcode(struct vm_area_struct *vma,
+static int __uprobe_write(struct vm_area_struct *vma,
 		struct folio_walk *fw, struct folio *folio,
-		unsigned long opcode_vaddr, uprobe_opcode_t opcode)
+		unsigned long insn_vaddr, uprobe_opcode_t *insn, int nbytes,
+		bool is_register)
 {
-	const unsigned long vaddr = opcode_vaddr & PAGE_MASK;
+	const unsigned long vaddr = insn_vaddr & PAGE_MASK;
-	const bool is_register = !!is_swbp_insn(&opcode);
 	bool pmd_mappable;
 	/* For now, we'll only handle PTE-mapped folios. */
 	 */
 	flush_cache_page(vma, vaddr, pte_pfn(fw->pte));
 	fw->pte = ptep_clear_flush(vma, vaddr, fw->ptep);
-	copy_to_page(fw->page, opcode_vaddr, &opcode, UPROBE_SWBP_INSN_SIZE);
+	copy_to_page(fw->page, insn_vaddr, insn, nbytes);
 	/*
 	 * When unregistering, we may only zap a PTE if uffd is disabled and
  * @opcode_vaddr: the virtual address to store the opcode.
  * @opcode: opcode to be written at @opcode_vaddr.
  *
- * Called with mm->mmap_lock held for read or write.
+ * Called with mm->mmap_lock held for write.
  * Return 0 (success) or a negative errno.
  */
 int uprobe_write_opcode(struct arch_uprobe *auprobe, struct vm_area_struct *vma,
-		const unsigned long opcode_vaddr, uprobe_opcode_t opcode)
+		const unsigned long opcode_vaddr, uprobe_opcode_t opcode,
+		bool is_register)
 {
-	const unsigned long vaddr = opcode_vaddr & PAGE_MASK;
+	return uprobe_write(auprobe, vma, opcode_vaddr, &opcode, UPROBE_SWBP_INSN_SIZE,
+			    verify_opcode, is_register, true /* do_update_ref_ctr */, NULL);
+}
+int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma,
+		 const unsigned long insn_vaddr, uprobe_opcode_t *insn, int nbytes,
+		 uprobe_write_verify_t verify, bool is_register, bool do_update_ref_ctr,
+		 void *data)
+{
+	const unsigned long vaddr = insn_vaddr & PAGE_MASK;
 	struct mm_struct *mm = vma->vm_mm;
 	struct uprobe *uprobe;
-	int ret, is_register, ref_ctr_updated = 0;
+	int ret, ref_ctr_updated = 0;
 	unsigned int gup_flags = FOLL_FORCE;
 	struct mmu_notifier_range range;
 	struct folio_walk fw;
 	struct folio *folio;
 	struct page *page;
-	is_register = is_swbp_insn(&opcode);
 	uprobe = container_of(auprobe, struct uprobe, arch);
 	if (WARN_ON_ONCE(!is_cow_mapping(vma->vm_flags)))
 	 * page that we can safely modify. Use FOLL_WRITE to trigger a write
 	 * fault if required. When unregistering, we might be lucky and the
 	 * anon page is already gone. So defer write faults until really
-	 * required. Use FOLL_SPLIT_PMD, because __uprobe_write_opcode()
+	 * required. Use FOLL_SPLIT_PMD, because __uprobe_write()
 	 * cannot deal with PMDs yet.
 	 */
 	if (is_register)
 		goto out;
 	folio = page_folio(page);
-	ret = verify_opcode(page, opcode_vaddr, &opcode);
+	ret = verify(page, insn_vaddr, insn, nbytes, data);
 	if (ret <= 0) {
 		folio_put(folio);
 		goto out;
 	}
 	/* We are going to replace instruction, update ref_ctr. */
-	if (!ref_ctr_updated && uprobe->ref_ctr_offset) {
+	if (do_update_ref_ctr && !ref_ctr_updated && uprobe->ref_ctr_offset) {
 		ret = update_ref_ctr(uprobe, mm, is_register ? 1 : -1);
 		if (ret) {
 			folio_put(folio);
 	/* Walk the page tables again, to perform the actual update. */
 	if (folio_walk_start(&fw, vma, vaddr, 0)) {
 		if (fw.page == page)
-			ret = __uprobe_write_opcode(vma, &fw, folio, opcode_vaddr, opcode);
+			ret = __uprobe_write(vma, &fw, folio, insn_vaddr, insn, nbytes, is_register);
 		folio_walk_end(&fw, vma);
 	}
 out:
 	/* Revert back reference counter if instruction update failed. */
-	if (ret < 0 && ref_ctr_updated)
+	if (do_update_ref_ctr && ret < 0 && ref_ctr_updated)
 		update_ref_ctr(uprobe, mm, is_register ? -1 : 1);
 	/* try collapse pmd for compound page */
 int __weak set_swbp(struct arch_uprobe *auprobe, struct vm_area_struct *vma,
 		unsigned long vaddr)
 {
-	return uprobe_write_opcode(auprobe, vma, vaddr, UPROBE_SWBP_INSN);
+	return uprobe_write_opcode(auprobe, vma, vaddr, UPROBE_SWBP_INSN, true);
 }
 /**
 		struct vm_area_struct *vma, unsigned long vaddr)
 {
 	return uprobe_write_opcode(auprobe, vma, vaddr,
-			*(uprobe_opcode_t *)&auprobe->insn);
+			*(uprobe_opcode_t *)&auprobe->insn, false);
 }
 /* uprobe should have guaranteed positive refcount */
 	if (IS_ERR(page))
 		return PTR_ERR(page);
-	copy_from_page(page, offset, insn, nbytes);
+	uprobe_copy_from_page(page, offset, insn, nbytes);
 	put_page(page);
 	return 0;
 			 * reclaim. This is optimistic, no harm done if it fails.
 			 */
 			prev = kmalloc(sizeof(struct map_info),
-					GFP_NOWAIT | __GFP_NOMEMALLOC | __GFP_NOWARN);
+					GFP_NOWAIT | __GFP_NOMEMALLOC);
 			if (prev)
 				prev->next = NULL;
 		}
 		return ERR_PTR(-EINVAL);
 	/*
-	 * This ensures that copy_from_page(), copy_to_page() and
+	 * This ensures that uprobe_copy_from_page(), copy_to_page() and
 	 * __update_ref_ctr() can't cross page boundary.
 	 */
 	if (!IS_ALIGNED(offset, UPROBE_SWBP_INSN_SIZE))
 	struct vm_area_struct *vma;
 	int err = 0;
-	mmap_read_lock(mm);
+	mmap_write_lock(mm);
 	for_each_vma(vmi, vma) {
 		unsigned long vaddr;
 		loff_t offset;
 		vaddr = offset_to_vaddr(vma, uprobe->offset);
 		err |= remove_breakpoint(uprobe, vma, vaddr);
 	}
-	mmap_read_unlock(mm);
+	mmap_write_unlock(mm);
 	return err;
 }
 	return ret;
 }
-void * __weak arch_uprobe_trampoline(unsigned long *psize)
+void * __weak arch_uretprobe_trampoline(unsigned long *psize)
 {
 	static uprobe_opcode_t insn = UPROBE_SWBP_INSN;
 	init_waitqueue_head(&area->wq);
 	/* Reserve the 1st slot for get_trampoline_vaddr() */
 	set_bit(0, area->bitmap);
-	insns = arch_uprobe_trampoline(&insns_size);
+	insns = arch_uretprobe_trampoline(&insns_size);
 	arch_uprobe_copy_ixol(area->page, 0, insns, insns_size);
 	if (!xol_add_vma(mm, area))
 	return area;
 }
+void __weak arch_uprobe_clear_state(struct mm_struct *mm)
+{
+}
+void __weak arch_uprobe_init_state(struct mm_struct *mm)
+{
+}
 /*
  * uprobe_clear_state - Free the area allocated for slots.
  */
 	delayed_uprobe_remove(NULL, mm);
 	mutex_unlock(&delayed_uprobe_lock);
+	arch_uprobe_clear_state(mm);
 	if (!area)
 		return;
 	if (result < 0)
 		return result;
-	copy_from_page(page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE);
+	uprobe_copy_from_page(page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE);
 	put_page(page);
  out:
 	/* This needs to return true for any variant of the trap insn */
 	return true;
 }
+void __weak arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr)
+{
+}
 /*
  * Run handler and ask thread to singlestep.
  * Ensure all non-fatal signals cannot interrupt thread while it singlesteps.
 	handler_chain(uprobe, regs);
+	/* Try to optimize after first hit. */
+	arch_uprobe_optimize(&uprobe->arch, bp_vaddr);
 	if (arch_uprobe_skip_sstep(&uprobe->arch, regs))
 		goto out;
 	rcu_read_unlock_trace();
 }
+void handle_syscall_uprobe(struct pt_regs *regs, unsigned long bp_vaddr)
+{
+	struct uprobe *uprobe;
+	int is_swbp;
+	guard(rcu_tasks_trace)();
+	uprobe = find_active_uprobe_rcu(bp_vaddr, &is_swbp);
+	if (!uprobe)
+		return;
+	if (!get_utask())
+		return;
+	if (arch_uprobe_ignore(&uprobe->arch, regs))
+		return;
+	handler_chain(uprobe, regs);
+}
 /*
  * Perform required fix-ups and disable singlestep.
  * Allow pending signals to take effect.

kernel/fork.c+1−0

@@ -1014,6 +1014,7 @@ static void mm_init_uprobes_state(struct mm_struct *mm) · 1013 unmodified lines
1014	{	1014	{
1015	#ifdef CONFIG_UPROBES	1015	#ifdef CONFIG_UPROBES
1016	mm->uprobes_state.xol_area = NULL;	1016	mm->uprobes_state.xol_area = NULL;
		1017	arch_uprobe_init_state(mm);
1017	#endif	1018	#endif
1018	}	1019	}
1019		1020

kernel/seccomp.c+25−7

 }
 #ifdef SECCOMP_ARCH_NATIVE
+static bool seccomp_uprobe_exception(struct seccomp_data *sd)
+{
+#if defined __NR_uretprobe || defined __NR_uprobe
+#ifdef SECCOMP_ARCH_COMPAT
+	if (sd->arch == SECCOMP_ARCH_NATIVE)
+#endif
+	{
+#ifdef __NR_uretprobe
+		if (sd->nr == __NR_uretprobe)
+			return true;
+#endif
+#ifdef __NR_uprobe
+		if (sd->nr == __NR_uprobe)
+			return true;
+#endif
+	}
+#endif
+	return false;
+}
 /**
  * seccomp_is_const_allow - check if filter is constant allow with given data
  * @fprog: The BPF programs
 		return false;
 	/* Our single exception to filtering. */
-#ifdef __NR_uretprobe
+	if (seccomp_uprobe_exception(sd))
-#ifdef SECCOMP_ARCH_COMPAT
+		return true;
-	if (sd->arch == SECCOMP_ARCH_NATIVE)
-#endif
-		if (sd->nr == __NR_uretprobe)
-			return true;
-#endif
 	for (pc = 0; pc < fprog->len; pc++) {
 		struct sock_filter *insn = &fprog->filter[pc];
 	__NR_seccomp_read, __NR_seccomp_write, __NR_seccomp_exit, __NR_seccomp_sigreturn,
 #ifdef __NR_uretprobe
 	__NR_uretprobe,
+#endif
+#ifdef __NR_uprobe
+	__NR_uprobe,
 #endif
 	-1, /* negative terminated */
 };

kernel/sys_ni.c+1−0

@@ -392,3 +392,4 @@ COND_SYSCALL(setuid16); · 391 unmodified lines
392	COND_SYSCALL(rseq);	392	COND_SYSCALL(rseq);
393		393
394	COND_SYSCALL(uretprobe);	394	COND_SYSCALL(uretprobe);
		395	COND_SYSCALL(uprobe);

tools/arch/x86/include/asm/msr-index.h+8−6

@@ -315,12 +315,14 @@ · 314 unmodified lines
315	#define PERF_CAP_PT_IDX 16	315	#define PERF_CAP_PT_IDX 16
316		316
317	#define MSR_PEBS_LD_LAT_THRESHOLD 0x000003f6	317	#define MSR_PEBS_LD_LAT_THRESHOLD 0x000003f6
318	#define PERF_CAP_PEBS_TRAP BIT_ULL(6)	318	#define PERF_CAP_PEBS_TRAP BIT_ULL(6)
319	#define PERF_CAP_ARCH_REG BIT_ULL(7)	319	#define PERF_CAP_ARCH_REG BIT_ULL(7)
320	#define PERF_CAP_PEBS_FORMAT 0xf00	320	#define PERF_CAP_PEBS_FORMAT 0xf00
321	#define PERF_CAP_PEBS_BASELINE BIT_ULL(14)	321	#define PERF_CAP_PEBS_BASELINE BIT_ULL(14)
322	#define PERF_CAP_PEBS_MASK (PERF_CAP_PEBS_TRAP \| PERF_CAP_ARCH_REG \| \	322	#define PERF_CAP_PEBS_TIMING_INFO BIT_ULL(17)
323	PERF_CAP_PEBS_FORMAT \| PERF_CAP_PEBS_BASELINE)	323	#define PERF_CAP_PEBS_MASK (PERF_CAP_PEBS_TRAP \| PERF_CAP_ARCH_REG \| \
		324	PERF_CAP_PEBS_FORMAT \| PERF_CAP_PEBS_BASELINE \| \
		325	PERF_CAP_PEBS_TIMING_INFO)
324		326
325	#define MSR_IA32_RTIT_CTL 0x00000570	327	#define MSR_IA32_RTIT_CTL 0x00000570
326	#define RTIT_CTL_TRACEEN BIT(0)	328	#define RTIT_CTL_TRACEEN BIT(0)

tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c+451−33

@@ -8,22 +8,31 @@ · 7 unmodified lines
8	#include <asm/ptrace.h>	8	#include <asm/ptrace.h>
9	#include <linux/compiler.h>	9	#include <linux/compiler.h>
10	#include <linux/stringify.h>	10	#include <linux/stringify.h>
		11	#include <linux/kernel.h>
11	#include <sys/wait.h>	12	#include <sys/wait.h>
12	#include <sys/syscall.h>	13	#include <sys/syscall.h>
13	#include <sys/prctl.h>	14	#include <sys/prctl.h>
14	#include <asm/prctl.h>	15	#include <asm/prctl.h>
15	#include "uprobe_syscall.skel.h"	16	#include "uprobe_syscall.skel.h"
16	#include "uprobe_syscall_executed.skel.h"	17	#include "uprobe_syscall_executed.skel.h"
		18	#include "bpf/libbpf_internal.h"
17		19
18	__naked unsigned long uretprobe_regs_trigger(void)	20	#define USDT_NOP .byte 0x0f, 0x1f, 0x44, 0x00, 0x00
		21	#include "usdt.h"
		22
		23	#pragma GCC diagnostic ignored "-Wattributes"
		24
		25	__attribute__((aligned(16)))
		26	__nocf_check __weak __naked unsigned long uprobe_regs_trigger(void)
19	{	27	{
20	asm volatile (	28	asm volatile (
		29	".byte 0x0f, 0x1f, 0x44, 0x00, 0x00\n" /* nop5 */
21	"movq $0xdeadbeef, %rax\n"	30	"movq $0xdeadbeef, %rax\n"
22	"ret\n"	31	"ret\n"
23	);	32	);
24	}	33	}
25		34
26	__naked void uretprobe_regs(struct pt_regs before, struct pt_regs after)	35	__naked void uprobe_regs(struct pt_regs before, struct pt_regs after)
27	{	36	{
28	asm volatile (	37	asm volatile (
29	"movq %r15, 0(%rdi)\n"	38	"movq %r15, 0(%rdi)\n"
@@ -44,15 +53,17 @@ __naked void uretprobe_regs(struct pt_regs before, struct pt_regs after) · 14 unmodified lines
44	"movq $0, 120(%rdi)\n" /* orig_rax */	53	"movq $0, 120(%rdi)\n" /* orig_rax */
45	"movq $0, 128(%rdi)\n" /* rip */	54	"movq $0, 128(%rdi)\n" /* rip */
46	"movq $0, 136(%rdi)\n" /* cs */	55	"movq $0, 136(%rdi)\n" /* cs */
		56	"pushq %rax\n"
47	"pushf\n"	57	"pushf\n"
48	"pop %rax\n"	58	"pop %rax\n"
49	"movq %rax, 144(%rdi)\n" /* eflags */	59	"movq %rax, 144(%rdi)\n" /* eflags */
		60	"pop %rax\n"
50	"movq %rsp, 152(%rdi)\n" /* rsp */	61	"movq %rsp, 152(%rdi)\n" /* rsp */
51	"movq $0, 160(%rdi)\n" /* ss */	62	"movq $0, 160(%rdi)\n" /* ss */
52		63
53	/* save 2nd argument */	64	/* save 2nd argument */
54	"pushq %rsi\n"	65	"pushq %rsi\n"
55	"call uretprobe_regs_trigger\n"	66	"call uprobe_regs_trigger\n"
56		67
57	/* save return value and load 2nd argument pointer to rax */	68	/* save return value and load 2nd argument pointer to rax */
58	"pushq %rax\n"	69	"pushq %rax\n"
@@ -92,25 +103,37 @@ __naked void uretprobe_regs(struct pt_regs before, struct pt_regs after) · 33 unmodified lines
92	);	103	);
93	}	104	}
94		105
95	static void test_uretprobe_regs_equal(void)	106	static void test_uprobe_regs_equal(bool retprobe)
96	{	107	{
		108	LIBBPF_OPTS(bpf_uprobe_opts, opts,
		109	.retprobe = retprobe,
		110	);
97	struct uprobe_syscall *skel = NULL;	111	struct uprobe_syscall *skel = NULL;
98	struct pt_regs before = {}, after = {};	112	struct pt_regs before = {}, after = {};
99	unsigned long pb = (unsigned long ) &before;	113	unsigned long pb = (unsigned long ) &before;
100	unsigned long pa = (unsigned long ) &after;	114	unsigned long pa = (unsigned long ) &after;
101	unsigned long *pp;	115	unsigned long *pp;
		116	unsigned long offset;
102	unsigned int i, cnt;	117	unsigned int i, cnt;
103	int err;	118
		119	offset = get_uprobe_offset(&uprobe_regs_trigger);
		120	if (!ASSERT_GE(offset, 0, "get_uprobe_offset"))
		121	return;
104		122
105	skel = uprobe_syscall__open_and_load();	123	skel = uprobe_syscall__open_and_load();
106	if (!ASSERT_OK_PTR(skel, "uprobe_syscall__open_and_load"))	124	if (!ASSERT_OK_PTR(skel, "uprobe_syscall__open_and_load"))
107	goto cleanup;	125	goto cleanup;
108		126
109	err = uprobe_syscall__attach(skel);	127	skel->links.probe = bpf_program__attach_uprobe_opts(skel->progs.probe,
110	if (!ASSERT_OK(err, "uprobe_syscall__attach"))	128	0, "/proc/self/exe", offset, &opts);
		129	if (!ASSERT_OK_PTR(skel->links.probe, "bpf_program__attach_uprobe_opts"))
111	goto cleanup;	130	goto cleanup;
112		131
113	uretprobe_regs(&before, &after);	132	/* make sure uprobe gets optimized */
		133	if (!retprobe)
		134	uprobe_regs_trigger();
		135
		136	uprobe_regs(&before, &after);
114		137
115	pp = (unsigned long *) &skel->bss->regs;	138	pp = (unsigned long *) &skel->bss->regs;
116	cnt = sizeof(before)/sizeof(*pb);	139	cnt = sizeof(before)/sizeof(*pb);
@@ -119,7 +142,7 @@ static void test_uretprobe_regs_equal(void) · 2 unmodified lines
119	unsigned int offset = i * sizeof(unsigned long);	142	unsigned int offset = i * sizeof(unsigned long);
120		143
121	/*	144	/*
122	* Check register before and after uretprobe_regs_trigger call	145	* Check register before and after uprobe_regs_trigger call
123	* that triggers the uretprobe.	146	* that triggers the uretprobe.
124	*/	147	*/
125	switch (offset) {	148	switch (offset) {
@@ -133,7 +156,7 @@ static void test_uretprobe_regs_equal(void) · 7 unmodified lines
133		156
134	/*	157	/*
135	* Check register seen from bpf program and register after	158	* Check register seen from bpf program and register after
136	* uretprobe_regs_trigger call	159	* uprobe_regs_trigger call (with rax exception, check below).
137	*/	160	*/
138	switch (offset) {	161	switch (offset) {
139	/*	162	/*
@@ -146,6 +169,15 @@ static void test_uretprobe_regs_equal(void) · 6 unmodified lines
146	case offsetof(struct pt_regs, rsp):	169	case offsetof(struct pt_regs, rsp):
147	case offsetof(struct pt_regs, ss):	170	case offsetof(struct pt_regs, ss):
148	break;	171	break;
		172	/*
		173	* uprobe does not see return value in rax, it needs to see the
		174	* original (before) rax value
		175	*/
		176	case offsetof(struct pt_regs, rax):
		177	if (!retprobe) {
		178	ASSERT_EQ(pp[i], pb[i], "uprobe rax prog-before value check");
		179	break;
		180	}
149	default:	181	default:
150	if (!ASSERT_EQ(pp[i], pa[i], "register prog-after value check"))	182	if (!ASSERT_EQ(pp[i], pa[i], "register prog-after value check"))
151	fprintf(stdout, "failed register offset %u\n", offset);	183	fprintf(stdout, "failed register offset %u\n", offset);
@@ -175,7 +207,7 @@ static int write_bpf_testmod_uprobe(unsigned long offset) · 23 unmodified lines
175	return ret != n ? (int) ret : 0;	207	return ret != n ? (int) ret : 0;
176	}	208	}
177		209
178	static void test_uretprobe_regs_change(void)	210	static void test_regs_change(void)
179	{	211	{
180	struct pt_regs before = {}, after = {};	212	struct pt_regs before = {}, after = {};
181	unsigned long pb = (unsigned long ) &before;	213	unsigned long pb = (unsigned long ) &before;
@@ -183,13 +215,16 @@ static void test_uretprobe_regs_change(void) · 1 unmodified line
183	unsigned long cnt = sizeof(before)/sizeof(*pb);	215	unsigned long cnt = sizeof(before)/sizeof(*pb);
184	unsigned int i, err, offset;	216	unsigned int i, err, offset;
185		217
186	offset = get_uprobe_offset(uretprobe_regs_trigger);	218	offset = get_uprobe_offset(uprobe_regs_trigger);
187		219
188	err = write_bpf_testmod_uprobe(offset);	220	err = write_bpf_testmod_uprobe(offset);
189	if (!ASSERT_OK(err, "register_uprobe"))	221	if (!ASSERT_OK(err, "register_uprobe"))
190	return;	222	return;
191		223
192	uretprobe_regs(&before, &after);	224	/* make sure uprobe gets optimized */
		225	uprobe_regs_trigger();
		226
		227	uprobe_regs(&before, &after);
193		228
194	err = write_bpf_testmod_uprobe(0);	229	err = write_bpf_testmod_uprobe(0);
195	if (!ASSERT_OK(err, "unregister_uprobe"))	230	if (!ASSERT_OK(err, "unregister_uprobe"))
@@ -252,6 +287,7 @@ static void test_uretprobe_syscall_call(void) · 56 unmodified lines
252	);	287	);
253	struct uprobe_syscall_executed *skel;	288	struct uprobe_syscall_executed *skel;
254	int pid, status, err, go[2], c = 0;	289	int pid, status, err, go[2], c = 0;
		290	struct bpf_link *link;
255		291
256	if (!ASSERT_OK(pipe(go), "pipe"))	292	if (!ASSERT_OK(pipe(go), "pipe"))
257	return;	293	return;
@@ -277,11 +313,14 @@ static void test_uretprobe_syscall_call(void) · 19 unmodified lines
277	_exit(0);	313	_exit(0);
278	}	314	}
279		315
280	skel->links.test = bpf_program__attach_uprobe_multi(skel->progs.test, pid,	316	skel->bss->pid = pid;
281	"/proc/self/exe",	317
282	"uretprobe_syscall_call", &opts);	318	link = bpf_program__attach_uprobe_multi(skel->progs.test_uretprobe_multi,
283	if (!ASSERT_OK_PTR(skel->links.test, "bpf_program__attach_uprobe_multi"))	319	pid, "/proc/self/exe",
		320	"uretprobe_syscall_call", &opts);
		321	if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi"))
284	goto cleanup;	322	goto cleanup;
		323	skel->links.test_uretprobe_multi = link;
285		324
286	/* kick the child */	325	/* kick the child */
287	write(go[1], &c, 1);	326	write(go[1], &c, 1);
@@ -301,6 +340,256 @@ cleanup: · 13 unmodified lines
301	close(go[0]);	340	close(go[0]);
302	}	341	}
303		342
		343	#define TRAMP "[uprobes-trampoline]"
		344
		345	__attribute__((aligned(16)))
		346	__nocf_check __weak __naked void uprobe_test(void)
		347	{
		348	asm volatile (" \n"
		349	".byte 0x0f, 0x1f, 0x44, 0x00, 0x00 \n"
		350	"ret \n"
		351	);
		352	}
		353
		354	__attribute__((aligned(16)))
		355	__nocf_check __weak void usdt_test(void)
		356	{
		357	USDT(optimized_uprobe, usdt);
		358	}
		359
		360	static int find_uprobes_trampoline(void *tramp_addr)
		361	{
		362	void start, end;
		363	char line[128];
		364	int ret = -1;
		365	FILE *maps;
		366
		367	maps = fopen("/proc/self/maps", "r");
		368	if (!maps) {
		369	fprintf(stderr, "cannot open maps\n");
		370	return -1;
		371	}
		372
		373	while (fgets(line, sizeof(line), maps)) {
		374	int m = -1;
		375
		376	/* We care only about private r-x mappings. */
		377	if (sscanf(line, "%p-%p r-xp %x %x:%x %u %n", &start, &end, &m) != 2)
		378	continue;
		379	if (m < 0)
		380	continue;
		381	if (!strncmp(&line[m], TRAMP, sizeof(TRAMP)-1) && (start == tramp_addr)) {
		382	ret = 0;
		383	break;
		384	}
		385	}
		386
		387	fclose(maps);
		388	return ret;
		389	}
		390
		391	static unsigned char nop5[5] = { 0x0f, 0x1f, 0x44, 0x00, 0x00 };
		392
		393	static void find_nop5(void fn)
		394	{
		395	int i;
		396
		397	for (i = 0; i < 10; i++) {
		398	if (!memcmp(nop5, fn + i, 5))
		399	return fn + i;
		400	}
		401	return NULL;
		402	}
		403
		404	typedef void (__attribute__((nocf_check)) *trigger_t)(void);
		405
		406	static void check_attach(struct uprobe_syscall_executed skel, trigger_t trigger,
		407	void *addr, int executed)
		408	{
		409	struct __arch_relative_insn {
		410	__u8 op;
		411	__s32 raddr;
		412	} __packed *call;
		413	void *tramp = NULL;
		414
		415	/* Uprobe gets optimized after first trigger, so let's press twice. */
		416	trigger();
		417	trigger();
		418
		419	/* Make sure bpf program got executed.. */
		420	ASSERT_EQ(skel->bss->executed, executed, "executed");
		421
		422	/* .. and check the trampoline is as expected. */
		423	call = (struct __arch_relative_insn *) addr;
		424	tramp = (void *) (call + 1) + call->raddr;
		425	ASSERT_EQ(call->op, 0xe8, "call");
		426	ASSERT_OK(find_uprobes_trampoline(tramp), "uprobes_trampoline");
		427
		428	return tramp;
		429	}
		430
		431	static void check_detach(void addr, void tramp)
		432	{
		433	/* [uprobes_trampoline] stays after detach */
		434	ASSERT_OK(find_uprobes_trampoline(tramp), "uprobes_trampoline");
		435	ASSERT_OK(memcmp(addr, nop5, 5), "nop5");
		436	}
		437
		438	static void check(struct uprobe_syscall_executed skel, struct bpf_link link,
		439	trigger_t trigger, void *addr, int executed)
		440	{
		441	void *tramp;
		442
		443	tramp = check_attach(skel, trigger, addr, executed);
		444	bpf_link__destroy(link);
		445	check_detach(addr, tramp);
		446	}
		447
		448	static void test_uprobe_legacy(void)
		449	{
		450	struct uprobe_syscall_executed *skel = NULL;
		451	LIBBPF_OPTS(bpf_uprobe_opts, opts,
		452	.retprobe = true,
		453	);
		454	struct bpf_link *link;
		455	unsigned long offset;
		456
		457	offset = get_uprobe_offset(&uprobe_test);
		458	if (!ASSERT_GE(offset, 0, "get_uprobe_offset"))
		459	goto cleanup;
		460
		461	/* uprobe */
		462	skel = uprobe_syscall_executed__open_and_load();
		463	if (!ASSERT_OK_PTR(skel, "uprobe_syscall_executed__open_and_load"))
		464	return;
		465
		466	skel->bss->pid = getpid();
		467
		468	link = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe,
		469	0, "/proc/self/exe", offset, NULL);
		470	if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_opts"))
		471	goto cleanup;
		472
		473	check(skel, link, uprobe_test, uprobe_test, 2);
		474
		475	/* uretprobe */
		476	skel->bss->executed = 0;
		477
		478	link = bpf_program__attach_uprobe_opts(skel->progs.test_uretprobe,
		479	0, "/proc/self/exe", offset, &opts);
		480	if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_opts"))
		481	goto cleanup;
		482
		483	check(skel, link, uprobe_test, uprobe_test, 2);
		484
		485	cleanup:
		486	uprobe_syscall_executed__destroy(skel);
		487	}
		488
		489	static void test_uprobe_multi(void)
		490	{
		491	struct uprobe_syscall_executed *skel = NULL;
		492	LIBBPF_OPTS(bpf_uprobe_multi_opts, opts);
		493	struct bpf_link *link;
		494	unsigned long offset;
		495
		496	offset = get_uprobe_offset(&uprobe_test);
		497	if (!ASSERT_GE(offset, 0, "get_uprobe_offset"))
		498	goto cleanup;
		499
		500	opts.offsets = &offset;
		501	opts.cnt = 1;
		502
		503	skel = uprobe_syscall_executed__open_and_load();
		504	if (!ASSERT_OK_PTR(skel, "uprobe_syscall_executed__open_and_load"))
		505	return;
		506
		507	skel->bss->pid = getpid();
		508
		509	/* uprobe.multi */
		510	link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_multi,
		511	0, "/proc/self/exe", NULL, &opts);
		512	if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi"))
		513	goto cleanup;
		514
		515	check(skel, link, uprobe_test, uprobe_test, 2);
		516
		517	/* uretprobe.multi */
		518	skel->bss->executed = 0;
		519	opts.retprobe = true;
		520	link = bpf_program__attach_uprobe_multi(skel->progs.test_uretprobe_multi,
		521	0, "/proc/self/exe", NULL, &opts);
		522	if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi"))
		523	goto cleanup;
		524
		525	check(skel, link, uprobe_test, uprobe_test, 2);
		526
		527	cleanup:
		528	uprobe_syscall_executed__destroy(skel);
		529	}
		530
		531	static void test_uprobe_session(void)
		532	{
		533	struct uprobe_syscall_executed *skel = NULL;
		534	LIBBPF_OPTS(bpf_uprobe_multi_opts, opts,
		535	.session = true,
		536	);
		537	struct bpf_link *link;
		538	unsigned long offset;
		539
		540	offset = get_uprobe_offset(&uprobe_test);
		541	if (!ASSERT_GE(offset, 0, "get_uprobe_offset"))
		542	goto cleanup;
		543
		544	opts.offsets = &offset;
		545	opts.cnt = 1;
		546
		547	skel = uprobe_syscall_executed__open_and_load();
		548	if (!ASSERT_OK_PTR(skel, "uprobe_syscall_executed__open_and_load"))
		549	return;
		550
		551	skel->bss->pid = getpid();
		552
		553	link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_session,
		554	0, "/proc/self/exe", NULL, &opts);
		555	if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi"))
		556	goto cleanup;
		557
		558	check(skel, link, uprobe_test, uprobe_test, 4);
		559
		560	cleanup:
		561	uprobe_syscall_executed__destroy(skel);
		562	}
		563
		564	static void test_uprobe_usdt(void)
		565	{
		566	struct uprobe_syscall_executed *skel;
		567	struct bpf_link *link;
		568	void *addr;
		569
		570	errno = 0;
		571	addr = find_nop5(usdt_test);
		572	if (!ASSERT_OK_PTR(addr, "find_nop5"))
		573	return;
		574
		575	skel = uprobe_syscall_executed__open_and_load();
		576	if (!ASSERT_OK_PTR(skel, "uprobe_syscall_executed__open_and_load"))
		577	return;
		578
		579	skel->bss->pid = getpid();
		580
		581	link = bpf_program__attach_usdt(skel->progs.test_usdt,
		582	-1 /* all PIDs */, "/proc/self/exe",
		583	"optimized_uprobe", "usdt", NULL);
		584	if (!ASSERT_OK_PTR(link, "bpf_program__attach_usdt"))
		585	goto cleanup;
		586
		587	check(skel, link, usdt_test, addr, 2);
		588
		589	cleanup:
		590	uprobe_syscall_executed__destroy(skel);
		591	}
		592
304	/*	593	/*
305	* Borrowed from tools/testing/selftests/x86/test_shadow_stack.c.	594	* Borrowed from tools/testing/selftests/x86/test_shadow_stack.c.
306	*	595	*
@@ -343,43 +632,172 @@ static void test_uretprobe_shadow_stack(void) · 36 unmodified lines
343	return;	632	return;
344	}	633	}
345		634
346	/* Run all of the uretprobe tests. */	635	/* Run all the tests with shadow stack in place. */
347	test_uretprobe_regs_equal();	636
348	test_uretprobe_regs_change();	637	test_uprobe_regs_equal(false);
		638	test_uprobe_regs_equal(true);
349	test_uretprobe_syscall_call();	639	test_uretprobe_syscall_call();
350		640
		641	test_uprobe_legacy();
		642	test_uprobe_multi();
		643	test_uprobe_session();
		644	test_uprobe_usdt();
		645
		646	test_regs_change();
		647
351	ARCH_PRCTL(ARCH_SHSTK_DISABLE, ARCH_SHSTK_SHSTK);	648	ARCH_PRCTL(ARCH_SHSTK_DISABLE, ARCH_SHSTK_SHSTK);
352	}	649	}
353	#else	650
354	static void test_uretprobe_regs_equal(void)	651	static volatile bool race_stop;
		652
		653	static USDT_DEFINE_SEMA(race);
		654
		655	static void worker_trigger(void arg)
355	{	656	{
356	test__skip();	657	unsigned long rounds = 0;
		658
		659	while (!race_stop) {
		660	uprobe_test();
		661	rounds++;
		662	}
		663
		664	printf("tid %d trigger rounds: %lu\n", gettid(), rounds);
		665	return NULL;
357	}	666	}
358		667
359	static void test_uretprobe_regs_change(void)	668	static void worker_attach(void arg)
360	{	669	{
361	test__skip();	670	LIBBPF_OPTS(bpf_uprobe_opts, opts);
		671	struct uprobe_syscall_executed *skel;
		672	unsigned long rounds = 0, offset;
		673	const char *sema[2] = {
		674	__stringify(USDT_SEMA(race)),
		675	NULL,
		676	};
		677	unsigned long *ref;
		678	int err;
		679
		680	offset = get_uprobe_offset(&uprobe_test);
		681	if (!ASSERT_GE(offset, 0, "get_uprobe_offset"))
		682	return NULL;
		683
		684	err = elf_resolve_syms_offsets("/proc/self/exe", 1, (const char **) &sema, &ref, STT_OBJECT);
		685	if (!ASSERT_OK(err, "elf_resolve_syms_offsets_sema"))
		686	return NULL;
		687
		688	opts.ref_ctr_offset = *ref;
		689
		690	skel = uprobe_syscall_executed__open_and_load();
		691	if (!ASSERT_OK_PTR(skel, "uprobe_syscall_executed__open_and_load"))
		692	return NULL;
		693
		694	skel->bss->pid = getpid();
		695
		696	while (!race_stop) {
		697	skel->links.test_uprobe = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe,
		698	0, "/proc/self/exe", offset, &opts);
		699	if (!ASSERT_OK_PTR(skel->links.test_uprobe, "bpf_program__attach_uprobe_opts"))
		700	break;
		701
		702	bpf_link__destroy(skel->links.test_uprobe);
		703	skel->links.test_uprobe = NULL;
		704	rounds++;
		705	}
		706
		707	printf("tid %d attach rounds: %lu hits: %d\n", gettid(), rounds, skel->bss->executed);
		708	uprobe_syscall_executed__destroy(skel);
		709	free(ref);
		710	return NULL;
362	}	711	}
363		712
364	static void test_uretprobe_syscall_call(void)	713	static useconds_t race_msec(void)
365	{	714	{
366	test__skip();	715	char *env;
		716
		717	env = getenv("BPF_SELFTESTS_UPROBE_SYSCALL_RACE_MSEC");
		718	if (env)
		719	return atoi(env);
		720
		721	/* default duration is 500ms */
		722	return 500;
367	}	723	}
368		724
369	static void test_uretprobe_shadow_stack(void)	725	static void test_uprobe_race(void)
370	{	726	{
371	test__skip();	727	int err, i, nr_threads;
		728	pthread_t *threads;
		729
		730	nr_threads = libbpf_num_possible_cpus();
		731	if (!ASSERT_GT(nr_threads, 0, "libbpf_num_possible_cpus"))
		732	return;
		733	nr_threads = max(2, nr_threads);
		734
		735	threads = alloca(sizeof(threads) nr_threads);
		736	if (!ASSERT_OK_PTR(threads, "malloc"))
		737	return;
		738
		739	for (i = 0; i < nr_threads; i++) {
		740	err = pthread_create(&threads[i], NULL, i % 2 ? worker_trigger : worker_attach,
		741	NULL);
		742	if (!ASSERT_OK(err, "pthread_create"))
		743	goto cleanup;
		744	}
		745
		746	usleep(race_msec() * 1000);
		747
		748	cleanup:
		749	race_stop = true;
		750	for (nr_threads = i, i = 0; i < nr_threads; i++)
		751	pthread_join(threads[i], NULL);
		752
		753	ASSERT_FALSE(USDT_SEMA_IS_ACTIVE(race), "race_semaphore");
372	}	754	}
		755
		756	#ifndef __NR_uprobe
		757	#define __NR_uprobe 336
373	#endif	758	#endif
374		759
375	void test_uprobe_syscall(void)	760	static void test_uprobe_error(void)
		761	{
		762	long err = syscall(__NR_uprobe);
		763
		764	ASSERT_EQ(err, -1, "error");
		765	ASSERT_EQ(errno, ENXIO, "errno");
		766	}
		767
		768	static void __test_uprobe_syscall(void)
376	{	769	{
377	if (test__start_subtest("uretprobe_regs_equal"))	770	if (test__start_subtest("uretprobe_regs_equal"))
378	test_uretprobe_regs_equal();	771	test_uprobe_regs_equal(true);
379	if (test__start_subtest("uretprobe_regs_change"))
380	test_uretprobe_regs_change();
381	if (test__start_subtest("uretprobe_syscall_call"))	772	if (test__start_subtest("uretprobe_syscall_call"))
382	test_uretprobe_syscall_call();	773	test_uretprobe_syscall_call();
383	if (test__start_subtest("uretprobe_shadow_stack"))	774	if (test__start_subtest("uretprobe_shadow_stack"))
384	test_uretprobe_shadow_stack();	775	test_uretprobe_shadow_stack();
		776	if (test__start_subtest("uprobe_legacy"))
		777	test_uprobe_legacy();
		778	if (test__start_subtest("uprobe_multi"))
		779	test_uprobe_multi();
		780	if (test__start_subtest("uprobe_session"))
		781	test_uprobe_session();
		782	if (test__start_subtest("uprobe_usdt"))
		783	test_uprobe_usdt();
		784	if (test__start_subtest("uprobe_race"))
		785	test_uprobe_race();
		786	if (test__start_subtest("uprobe_error"))
		787	test_uprobe_error();
		788	if (test__start_subtest("uprobe_regs_equal"))
		789	test_uprobe_regs_equal(false);
		790	if (test__start_subtest("regs_change"))
		791	test_regs_change();
		792	}
		793	#else
		794	static void __test_uprobe_syscall(void)
		795	{
		796	test__skip();
		797	}
		798	#endif
		799
		800	void test_uprobe_syscall(void)
		801	{
		802	__test_uprobe_syscall();
385	}	803	}

tools/testing/selftests/bpf/prog_tests/usdt.c+25−13

 	}
 }
-static void subtest_basic_usdt(void)
+static void subtest_basic_usdt(bool optimized)
 {
 	LIBBPF_OPTS(bpf_usdt_opts, opts);
 	struct test_usdt *skel;
 	struct test_usdt__bss *bss;
-	int err, i;
+	int err, i, called;
+#define TRIGGER(x) ({			\
+	trigger_func(x);		\
+	if (optimized)			\
+		trigger_func(x);	\
+	optimized ? 2 : 1;		\
+	})
 	skel = test_usdt__open_and_load();
 	if (!ASSERT_OK_PTR(skel, "skel_open"))
 	if (!ASSERT_OK_PTR(skel->links.usdt0, "usdt0_link"))
 		goto cleanup;
-	trigger_func(1);
+	called = TRIGGER(1);
-	ASSERT_EQ(bss->usdt0_called, 1, "usdt0_called");
+	ASSERT_EQ(bss->usdt0_called, called, "usdt0_called");
-	ASSERT_EQ(bss->usdt3_called, 1, "usdt3_called");
+	ASSERT_EQ(bss->usdt3_called, called, "usdt3_called");
-	ASSERT_EQ(bss->usdt12_called, 1, "usdt12_called");
+	ASSERT_EQ(bss->usdt12_called, called, "usdt12_called");
 	ASSERT_EQ(bss->usdt0_cookie, 0xcafedeadbeeffeed, "usdt0_cookie");
 	ASSERT_EQ(bss->usdt0_arg_cnt, 0, "usdt0_arg_cnt");
 	 * bpf_program__attach_usdt() handles this properly and attaches to
 	 * all possible places of USDT invocation.
 	 */
-	trigger_func(2);
+	called += TRIGGER(2);
-	ASSERT_EQ(bss->usdt0_called, 2, "usdt0_called");
+	ASSERT_EQ(bss->usdt0_called, called, "usdt0_called");
-	ASSERT_EQ(bss->usdt3_called, 2, "usdt3_called");
+	ASSERT_EQ(bss->usdt3_called, called, "usdt3_called");
-	ASSERT_EQ(bss->usdt12_called, 2, "usdt12_called");
+	ASSERT_EQ(bss->usdt12_called, called, "usdt12_called");
 	/* only check values that depend on trigger_func()'s input value */
 	ASSERT_EQ(bss->usdt3_args[0], 2, "usdt3_arg1");
 	if (!ASSERT_OK_PTR(skel->links.usdt3, "usdt3_reattach"))
 		goto cleanup;
-	trigger_func(3);
+	called += TRIGGER(3);
-	ASSERT_EQ(bss->usdt3_called, 3, "usdt3_called");
+	ASSERT_EQ(bss->usdt3_called, called, "usdt3_called");
 	/* this time usdt3 has custom cookie */
 	ASSERT_EQ(bss->usdt3_cookie, 0xBADC00C51E, "usdt3_cookie");
 	ASSERT_EQ(bss->usdt3_arg_cnt, 3, "usdt3_arg_cnt");
 cleanup:
 	test_usdt__destroy(skel);
+#undef TRIGGER
 }
 unsigned short test_usdt_100_semaphore SEC(".probes");
 void test_usdt(void)
 {
 	if (test__start_subtest("basic"))
-		subtest_basic_usdt();
+		subtest_basic_usdt(false);
+#ifdef __x86_64__
+	if (test__start_subtest("basic_optimized"))
+		subtest_basic_usdt(true);
+#endif
 	if (test__start_subtest("multispec"))
 		subtest_multispec_usdt();
 	if (test__start_subtest("urand_auto_attach"))

tools/testing/selftests/bpf/progs/uprobe_syscall.c+2−2

@@ -7,8 +7,8 @@ struct pt_regs regs; · 6 unmodified lines
7		7
8	char _license[] SEC("license") = "GPL";	8	char _license[] SEC("license") = "GPL";
9		9
10	SEC("uretprobe//proc/self/exe:uretprobe_regs_trigger")	10	SEC("uprobe")
11	int uretprobe(struct pt_regs *ctx)	11	int probe(struct pt_regs *ctx)
12	{	12	{
13	__builtin_memcpy(&regs, ctx, sizeof(regs));	13	__builtin_memcpy(&regs, ctx, sizeof(regs));
14	return 0;	14	return 0;

tools/testing/selftests/bpf/progs/uprobe_syscall_executed.c+58−2

@@ -1,6 +1,8 @@
1	// SPDX-License-Identifier: GPL-2.0	1	// SPDX-License-Identifier: GPL-2.0
2	#include "vmlinux.h"	2	#include "vmlinux.h"
3	#include <bpf/bpf_helpers.h>	3	#include <bpf/bpf_helpers.h>
		4	#include <bpf/bpf_tracing.h>
		5	#include <bpf/usdt.bpf.h>
4	#include <string.h>	6	#include <string.h>
5		7
6	struct pt_regs regs;	8	struct pt_regs regs;
@@ -8,10 +10,64 @@ struct pt_regs regs; · 1 unmodified line
8	char _license[] SEC("license") = "GPL";	10	char _license[] SEC("license") = "GPL";
9		11
10	int executed = 0;	12	int executed = 0;
		13	int pid;
		14
		15	SEC("uprobe")
		16	int BPF_UPROBE(test_uprobe)
		17	{
		18	if (bpf_get_current_pid_tgid() >> 32 != pid)
		19	return 0;
		20
		21	executed++;
		22	return 0;
		23	}
		24
		25	SEC("uretprobe")
		26	int BPF_URETPROBE(test_uretprobe)
		27	{
		28	if (bpf_get_current_pid_tgid() >> 32 != pid)
		29	return 0;
		30
		31	executed++;
		32	return 0;
		33	}
		34
		35	SEC("uprobe.multi")
		36	int test_uprobe_multi(struct pt_regs *ctx)
		37	{
		38	if (bpf_get_current_pid_tgid() >> 32 != pid)
		39	return 0;
		40
		41	executed++;
		42	return 0;
		43	}
11		44
12	SEC("uretprobe.multi")	45	SEC("uretprobe.multi")
13	int test(struct pt_regs *regs)	46	int test_uretprobe_multi(struct pt_regs *ctx)
		47	{
		48	if (bpf_get_current_pid_tgid() >> 32 != pid)
		49	return 0;
		50
		51	executed++;
		52	return 0;
		53	}
		54
		55	SEC("uprobe.session")
		56	int test_uprobe_session(struct pt_regs *ctx)
14	{	57	{
15	executed = 1;	58	if (bpf_get_current_pid_tgid() >> 32 != pid)
		59	return 0;
		60
		61	executed++;
		62	return 0;
		63	}
		64
		65	SEC("usdt")
		66	int test_usdt(struct pt_regs *ctx)
		67	{
		68	if (bpf_get_current_pid_tgid() >> 32 != pid)
		69	return 0;
		70
		71	executed++;
16	return 0;	72	return 0;
17	}	73	}

tools/testing/selftests/bpf/test_kmods/bpf_testmod.c+9−2

  */
 #ifdef __x86_64__
+static int
+uprobe_handler(struct uprobe_consumer *self, struct pt_regs *regs, __u64 *data)
+{
+	regs->cx = 0x87654321feebdaed;
+	return 0;
+}
 static int
 uprobe_ret_handler(struct uprobe_consumer *self, unsigned long func,
 		   struct pt_regs *regs, __u64 *data)
 {
 	regs->ax  = 0x12345678deadbeef;
-	regs->cx  = 0x87654321feebdaed;
 	regs->r11 = (u64) -1;
-	return true;
+	return 0;
 }
 struct testmod_uprobe {
 static DEFINE_MUTEX(testmod_uprobe_mutex);
 static struct testmod_uprobe uprobe = {
+	.consumer.handler = uprobe_handler,
 	.consumer.ret_handler = uprobe_ret_handler,
 };

tools/testing/selftests/bpf/usdt.h+545−0

@@ -0,0 +1,545 @@
	1	// SPDX-License-Identifier: BSD-2-Clause
	2	/*
	3	* This single-header library defines a collection of variadic macros for
	4	* defining and triggering USDTs (User Statically-Defined Tracepoints):
	5	*
	6	* - For USDTs without associated semaphore:
	7	* USDT(group, name, args...)
	8	*
	9	* - For USDTs with implicit (transparent to the user) semaphore:
	10	* USDT_WITH_SEMA(group, name, args...)
	11	* USDT_IS_ACTIVE(group, name)
	12	*
	13	* - For USDTs with explicit (user-defined and provided) semaphore:
	14	* USDT_WITH_EXPLICIT_SEMA(sema, group, name, args...)
	15	* USDT_SEMA_IS_ACTIVE(sema)
	16	*
	17	* all of which emit a NOP instruction into the instruction stream, and so
	18	* have zero overhead for the surrounding code. USDTs are identified by
	19	* a combination of `group` and `name` identifiers, which is used by external
	20	* tracing tooling (tracers) for identifying exact USDTs of interest.
	21	*
	22	* USDTs can have an associated (2-byte) activity counter (USDT semaphore),
	23	* automatically maintained by Linux kernel whenever any correctly written
	24	* BPF-based tracer is attached to the USDT. This USDT semaphore can be used
	25	* to check whether there is a need to do any extra data collection and
	26	* processing for a given USDT (if necessary), and otherwise avoid extra work
	27	* for a common case of USDT not being traced ("active").
	28	*
	29	* See documentation for USDT_WITH_SEMA()/USDT_IS_ACTIVE() or
	30	* USDT_WITH_EXPLICIT_SEMA()/USDT_SEMA_IS_ACTIVE() APIs below for details on
	31	* working with USDTs with implicitly or explicitly associated
	32	* USDT semaphores, respectively.
	33	*
	34	* There is also some additional data recorded into an auxiliary note
	35	* section. The data in the note section describes the operands, in terms of
	36	* size and location, used by tracing tooling to know where to find USDT
	37	* arguments. Each location is encoded as an assembler operand string.
	38	* Tracing tools (bpftrace and BPF-based tracers, systemtap, etc) insert
	39	* breakpoints on top of the nop, and decode the location operand-strings,
	40	* like an assembler, to find the values being passed.
	41	*
	42	* The operand strings are selected by the compiler for each operand.
	43	* They are constrained by inline-assembler codes.The default is:
	44	*
	45	* #define USDT_ARG_CONSTRAINT nor
	46	*
	47	* This is a good default if the operands tend to be integral and
	48	* moderate in number (smaller than number of registers). In other
	49	* cases, the compiler may report "'asm' requires impossible reload" or
	50	* similar. In this case, consider simplifying the macro call (fewer
	51	* and simpler operands), reduce optimization, or override the default
	52	* constraints string via:
	53	*
	54	* #define USDT_ARG_CONSTRAINT g
	55	* #include <usdt.h>
	56	*
	57	* For some historical description of USDT v3 format (the one used by this
	58	* library and generally recognized and assumed by BPF-based tracing tools)
	59	* see [0]. The more formal specification can be found at [1]. Additional
	60	* argument constraints information can be found at [2].
	61	*
	62	* Original SystemTap's sys/sdt.h implementation ([3]) was used as a base for
	63	* this USDT library implementation. Current implementation differs a lot in
	64	* terms of exposed user API and general usability, which was the main goal
	65	* and focus of the reimplementation work. Nevertheless, underlying recorded
	66	* USDT definitions are fully binary compatible and any USDT-based tooling
	67	* should work equally well with USDTs defined by either SystemTap's or this
	68	* library's USDT implementation.
	69	*
	70	* [0] https://ecos.sourceware.org/ml/systemtap/2010-q3/msg00145.html
	71	* [1] https://sourceware.org/systemtap/wiki/UserSpaceProbeImplementation
	72	* [2] https://gcc.gnu.org/onlinedocs/gcc/Constraints.html
	73	* [3] https://sourceware.org/git/?p=systemtap.git;a=blob;f=includes/sys/sdt.h
	74	*/
	75	#ifndef __USDT_H
	76	#define __USDT_H
	77
	78	/*
	79	* Changelog:
	80	*
	81	* 0.1.0
	82	* -----
	83	* - Initial release
	84	*/
	85	#define USDT_MAJOR_VERSION 0
	86	#define USDT_MINOR_VERSION 1
	87	#define USDT_PATCH_VERSION 0
	88
	89	/* C++20 and C23 added __VA_OPT__ as a standard replacement for non-standard `##__VA_ARGS__` extension */
	90	#if (defined(__STDC_VERSION__) && __STDC_VERSION__ > 201710L) \|\| (defined(__cplusplus) && __cplusplus > 201703L)
	91	#define __usdt_va_opt 1
	92	#define __usdt_va_args(...) __VA_OPT__(,) __VA_ARGS__
	93	#else
	94	#define __usdt_va_args(...) , ##__VA_ARGS__
	95	#endif
	96
	97	/*
	98	* Trigger USDT with `group`:`name` identifier and pass through `args` as its
	99	* arguments. Zero arguments are acceptable as well. No USDT semaphore is
	100	* associated with this USDT.
	101	*
	102	* Such "semaphoreless" USDTs are commonly used when there is no extra data
	103	* collection or processing needed to collect and prepare USDT arguments and
	104	* they are just available in the surrounding code. USDT() macro will just
	105	* record their locations in CPU registers or in memory for tracing tooling to
	106	* be able to access them, if necessary.
	107	*/
	108	#ifdef __usdt_va_opt
	109	#define USDT(group, name, ...) \
	110	__usdt_probe(group, name, __usdt_sema_none, 0 __VA_OPT__(,) __VA_ARGS__)
	111	#else
	112	#define USDT(group, name, ...) \
	113	__usdt_probe(group, name, __usdt_sema_none, 0, ##__VA_ARGS__)
	114	#endif
	115
	116	/*
	117	* Trigger USDT with `group`:`name` identifier and pass through `args` as its
	118	* arguments. Zero arguments are acceptable as well. USDT also get an
	119	* implicitly-defined associated USDT semaphore, which will be "activated" by
	120	* tracing tooling and can be used to check whether USDT is being actively
	121	* observed.
	122	*
	123	* USDTs with semaphore are commonly used when there is a need to perform
	124	* additional data collection and processing to prepare USDT arguments, which
	125	* otherwise might not be necessary for the rest of application logic. In such
	126	* case, USDT semaphore can be used to avoid unnecessary extra work. If USDT
	127	* is not traced (which is presumed to be a common situation), the associated
	128	* USDT semaphore is "inactive", and so there is no need to waste resources to
	129	* prepare USDT arguments. Use USDT_IS_ACTIVE(group, name) to check whether
	130	* USDT is "active".
	131	*
	132	* N.B. There is an inherent (albeit short) gap between checking whether USDT
	133	* is active and triggering corresponding USDT, in which external tracer can
	134	* be attached to an USDT and activate USDT semaphore after the activity check.
	135	* If such a race occurs, tracers might miss one USDT execution. Tracers are
	136	* expected to accommodate such possibility and this is expected to not be
	137	* a problem for applications and tracers.
	138	*
	139	* N.B. Implicit USDT semaphore defined by USDT_WITH_SEMA() is contained
	140	* within a single executable or shared library and is not shared outside
	141	* them. I.e., if you use USDT_WITH_SEMA() with the same USDT group and name
	142	* identifier across executable and shared library, it will work and won't
	143	* conflict, per se, but will define independent USDT semaphores, one for each
	144	* shared library/executable in which USDT_WITH_SEMA(group, name) is used.
	145	* That is, if you attach to this USDT in one shared library (or executable),
	146	* then only USDT semaphore within that shared library (or executable) will be
	147	* updated by the kernel, while other libraries (or executable) will not see
	148	* activated USDT semaphore. In short, it's best to use unique USDT group:name
	149	* identifiers across different shared libraries (and, equivalently, between
	150	* executable and shared library). This is advanced consideration and is
	151	* rarely (if ever) seen in practice, but just to avoid surprises this is
	152	* called out here. (Static libraries become a part of final executable, once
	153	* linked by linker, so the above considerations don't apply to them.)
	154	*/
	155	#ifdef __usdt_va_opt
	156	#define USDT_WITH_SEMA(group, name, ...) \
	157	__usdt_probe(group, name, \
	158	__usdt_sema_implicit, __usdt_sema_name(group, name) \
	159	__VA_OPT__(,) __VA_ARGS__)
	160	#else
	161	#define USDT_WITH_SEMA(group, name, ...) \
	162	__usdt_probe(group, name, \
	163	__usdt_sema_implicit, __usdt_sema_name(group, name), \
	164	##__VA_ARGS__)
	165	#endif
	166
	167	struct usdt_sema { volatile unsigned short active; };
	168
	169	/*
	170	* Check if USDT with `group`:`name` identifier is "active" (i.e., whether it
	171	* is attached to by external tracing tooling and is actively observed).
	172	*
	173	* This macro can be used to decide whether any additional and potentially
	174	* expensive data collection or processing should be done to pass extra
	175	* information into the given USDT. It is assumed that USDT is triggered with
	176	* USDT_WITH_SEMA() macro which will implicitly define associated USDT
	177	* semaphore. (If one needs more control over USDT semaphore, see
	178	* USDT_DEFINE_SEMA() and USDT_WITH_EXPLICIT_SEMA() macros below.)
	179	*
	180	* N.B. Such checks are necessarily racy and speculative. Between checking
	181	* whether USDT is active and triggering the USDT itself, tracer can be
	182	* detached with no notification. This race should be extremely rare and worst
	183	* case should result in one-time wasted extra data collection and processing.
	184	*/
	185	#define USDT_IS_ACTIVE(group, name) ({ \
	186	extern struct usdt_sema __usdt_sema_name(group, name) \
	187	__usdt_asm_name(__usdt_sema_name(group, name)); \
	188	__usdt_sema_implicit(__usdt_sema_name(group, name)); \
	189	__usdt_sema_name(group, name).active > 0; \
	190	})
	191
	192	/*
	193	* APIs for working with user-defined explicit USDT semaphores.
	194	*
	195	* This is a less commonly used advanced API for use cases in which user needs
	196	* an explicit control over (potentially shared across multiple USDTs) USDT
	197	* semaphore instance. This can be used when there is a group of logically
	198	* related USDTs that all need extra data collection and processing whenever
	199	* any of a family of related USDTs are "activated" (i.e., traced). In such
	200	* a case, all such related USDTs will be associated with the same shared USDT
	201	* semaphore defined with USDT_DEFINE_SEMA() and the USDTs themselves will be
	202	* triggered with USDT_WITH_EXPLICIT_SEMA() macros, taking an explicit extra
	203	* USDT semaphore identifier as an extra parameter.
	204	*/
	205
	206	/**
	207	* Underlying C global variable name for user-defined USDT semaphore with
	208	* `sema` identifier. Could be useful for debugging, but normally shouldn't be
	209	* used explicitly.
	210	*/
	211	#define USDT_SEMA(sema) __usdt_sema_##sema
	212
	213	/*
	214	* Define storage for user-defined USDT semaphore `sema`.
	215	*
	216	* Should be used only once in non-header source file to let compiler allocate
	217	* space for the semaphore variable. Just like with any other global variable.
	218	*
	219	* This macro can be used anywhere where global variable declaration is
	220	* allowed. Just like with global variable definitions, there should be only
	221	* one definition of user-defined USDT semaphore with given `sema` identifier,
	222	* otherwise compiler or linker will complain about duplicate variable
	223	* definition.
	224	*
	225	* For C++, it is allowed to use USDT_DEFINE_SEMA() both in global namespace
	226	* and inside namespaces (including nested namespaces). Just make sure that
	227	* USDT_DECLARE_SEMA() is placed within the namespace where this semaphore is
	228	* referenced, or any of its parent namespaces, so the C++ language-level
	229	* identifier is visible to the code that needs to reference the semaphore.
	230	* At the lowest layer, USDT semaphores have global naming and visibility
	231	* (they have a corresponding `__usdt_sema_<name>` symbol, which can be linked
	232	* against from C or C++ code, if necessary). To keep it simple, putting
	233	* USDT_DECLARE_SEMA() declarations into global namespaces is the simplest
	234	* no-brainer solution. All these aspects are irrelevant for plain C, because
	235	* C doesn't have namespaces and everything is always in the global namespace.
	236	*
	237	* N.B. Due to USDT metadata being recorded in non-allocatable ELF note
	238	* section, it has limitations when it comes to relocations, which, in
	239	* practice, means that it's not possible to correctly share USDT semaphores
	240	* between main executable and shared libraries, or even between multiple
	241	* shared libraries. USDT semaphore has to be contained to individual shared
	242	* library or executable to avoid unpleasant surprises with half-working USDT
	243	* semaphores. We enforce this by marking semaphore ELF symbols as having
	244	* a hidden visibility. This is quite an advanced use case and consideration
	245	* and for most users this should have no consequences whatsoever.
	246	*/
	247	#define USDT_DEFINE_SEMA(sema) \
	248	struct usdt_sema __usdt_sema_sec USDT_SEMA(sema) \
	249	__usdt_asm_name(USDT_SEMA(sema)) \
	250	__attribute__((visibility("hidden"))) = { 0 }
	251
	252	/*
	253	* Declare extern reference to user-defined USDT semaphore `sema`.
	254	*
	255	* Refers to a variable defined in another compilation unit by
	256	* USDT_DEFINE_SEMA() and allows to use the same USDT semaphore across
	257	* multiple compilation units (i.e., .c and .cpp files).
	258	*
	259	* See USDT_DEFINE_SEMA() notes above for C++ language usage peculiarities.
	260	*/
	261	#define USDT_DECLARE_SEMA(sema) \
	262	extern struct usdt_sema USDT_SEMA(sema) __usdt_asm_name(USDT_SEMA(sema))
	263
	264	/*
	265	* Check if user-defined USDT semaphore `sema` is "active" (i.e., whether it
	266	* is attached to by external tracing tooling and is actively observed).
	267	*
	268	* This macro can be used to decide whether any additional and potentially
	269	* expensive data collection or processing should be done to pass extra
	270	* information into USDT(s) associated with USDT semaphore `sema`.
	271	*
	272	* N.B. Such checks are necessarily racy. Between checking the state of USDT
	273	* semaphore and triggering associated USDT(s), the active tracer might attach
	274	* or detach. This race should be extremely rare and worst case should result
	275	* in one-time missed USDT event or wasted extra data collection and
	276	* processing. USDT-using tracers should be written with this in mind and is
	277	* not a concern of the application defining USDTs with associated semaphore.
	278	*/
	279	#define USDT_SEMA_IS_ACTIVE(sema) (USDT_SEMA(sema).active > 0)
	280
	281	/*
	282	* Invoke USDT specified by `group` and `name` identifiers and associate
	283	* explicitly user-defined semaphore `sema` with it. Pass through `args` as
	284	* USDT arguments. `args` are optional and zero arguments are acceptable.
	285	*
	286	* Semaphore is defined with the help of USDT_DEFINE_SEMA() macro and can be
	287	* checked whether active with USDT_SEMA_IS_ACTIVE().
	288	*/
	289	#ifdef __usdt_va_opt
	290	#define USDT_WITH_EXPLICIT_SEMA(sema, group, name, ...) \
	291	__usdt_probe(group, name, __usdt_sema_explicit, USDT_SEMA(sema), ##__VA_ARGS__)
	292	#else
	293	#define USDT_WITH_EXPLICIT_SEMA(sema, group, name, ...) \
	294	__usdt_probe(group, name, __usdt_sema_explicit, USDT_SEMA(sema) __VA_OPT__(,) __VA_ARGS__)
	295	#endif
	296
	297	/*
	298	* Adjustable implementation aspects
	299	*/
	300	#ifndef USDT_ARG_CONSTRAINT
	301	#if defined __powerpc__
	302	#define USDT_ARG_CONSTRAINT nZr
	303	#elif defined __arm__
	304	#define USDT_ARG_CONSTRAINT g
	305	#elif defined __loongarch__
	306	#define USDT_ARG_CONSTRAINT nmr
	307	#else
	308	#define USDT_ARG_CONSTRAINT nor
	309	#endif
	310	#endif /* USDT_ARG_CONSTRAINT */
	311
	312	#ifndef USDT_NOP
	313	#if defined(__ia64__) \|\| defined(__s390__) \|\| defined(__s390x__)
	314	#define USDT_NOP nop 0
	315	#else
	316	#define USDT_NOP nop
	317	#endif
	318	#endif /* USDT_NOP */
	319
	320	/*
	321	* Implementation details
	322	*/
	323	/* USDT name for implicitly-defined USDT semaphore, derived from group:name */
	324	#define __usdt_sema_name(group, name) __usdt_sema_##group##__##name
	325	/* ELF section into which USDT semaphores are put */
	326	#define __usdt_sema_sec __attribute__((section(".probes")))
	327
	328	#define __usdt_concat(a, b) a ## b
	329	#define __usdt_apply(fn, n) __usdt_concat(fn, n)
	330
	331	#ifndef __usdt_nth
	332	#define __usdt_nth(_, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, N, ...) N
	333	#endif
	334
	335	#ifndef __usdt_narg
	336	#ifdef __usdt_va_opt
	337	#define __usdt_narg(...) __usdt_nth(_ __VA_OPT__(,) __VA_ARGS__, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)
	338	#else
	339	#define __usdt_narg(...) __usdt_nth(_, ##__VA_ARGS__, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)
	340	#endif
	341	#endif /* __usdt_narg */
	342
	343	#define __usdt_hash #
	344	#define __usdt_str_(x) #x
	345	#define __usdt_str(x) __usdt_str_(x)
	346
	347	#ifndef __usdt_asm_name
	348	#define __usdt_asm_name(name) __asm__(__usdt_str(name))
	349	#endif
	350
	351	#define __usdt_asm0() "\n"
	352	#define __usdt_asm1(x) __usdt_str(x) "\n"
	353	#define __usdt_asm2(x, ...) __usdt_str(x) "," __usdt_asm1(__VA_ARGS__)
	354	#define __usdt_asm3(x, ...) __usdt_str(x) "," __usdt_asm2(__VA_ARGS__)
	355	#define __usdt_asm4(x, ...) __usdt_str(x) "," __usdt_asm3(__VA_ARGS__)
	356	#define __usdt_asm5(x, ...) __usdt_str(x) "," __usdt_asm4(__VA_ARGS__)
	357	#define __usdt_asm6(x, ...) __usdt_str(x) "," __usdt_asm5(__VA_ARGS__)
	358	#define __usdt_asm7(x, ...) __usdt_str(x) "," __usdt_asm6(__VA_ARGS__)
	359	#define __usdt_asm8(x, ...) __usdt_str(x) "," __usdt_asm7(__VA_ARGS__)
	360	#define __usdt_asm9(x, ...) __usdt_str(x) "," __usdt_asm8(__VA_ARGS__)
	361	#define __usdt_asm10(x, ...) __usdt_str(x) "," __usdt_asm9(__VA_ARGS__)
	362	#define __usdt_asm11(x, ...) __usdt_str(x) "," __usdt_asm10(__VA_ARGS__)
	363	#define __usdt_asm12(x, ...) __usdt_str(x) "," __usdt_asm11(__VA_ARGS__)
	364	#define __usdt_asm(...) __usdt_apply(__usdt_asm, __usdt_narg(__VA_ARGS__))(__VA_ARGS__)
	365
	366	#ifdef __LP64__
	367	#define __usdt_asm_addr .8byte
	368	#else
	369	#define __usdt_asm_addr .4byte
	370	#endif
	371
	372	#define __usdt_asm_strz_(x) __usdt_asm1(.asciz #x)
	373	#define __usdt_asm_strz(x) __usdt_asm_strz_(x)
	374	#define __usdt_asm_str_(x) __usdt_asm1(.ascii #x)
	375	#define __usdt_asm_str(x) __usdt_asm_str_(x)
	376
	377	/* "semaphoreless" USDT case */
	378	#ifndef __usdt_sema_none
	379	#define __usdt_sema_none(sema)
	380	#endif
	381
	382	/* implicitly defined __usdt_sema__group__name semaphore (using weak symbols) */
	383	#ifndef __usdt_sema_implicit
	384	#define __usdt_sema_implicit(sema) \
	385	__asm__ __volatile__ ( \
	386	__usdt_asm1(.ifndef sema) \
	387	__usdt_asm3( .pushsection .probes, "aw", "progbits") \
	388	__usdt_asm1( .weak sema) \
	389	__usdt_asm1( .hidden sema) \
	390	__usdt_asm1( .align 2) \
	391	__usdt_asm1(sema:) \
	392	__usdt_asm1( .zero 2) \
	393	__usdt_asm2( .type sema, @object) \
	394	__usdt_asm2( .size sema, 2) \
	395	__usdt_asm1( .popsection) \
	396	__usdt_asm1(.endif) \
	397	);
	398	#endif
	399
	400	/* externally defined semaphore using USDT_DEFINE_SEMA() and passed explicitly by user */
	401	#ifndef __usdt_sema_explicit
	402	#define __usdt_sema_explicit(sema) \
	403	__asm__ __volatile__ ("" :: "m" (sema));
	404	#endif
	405
	406	/* main USDT definition (nop and .note.stapsdt metadata) */
	407	#define __usdt_probe(group, name, sema_def, sema, ...) do { \
	408	sema_def(sema) \
	409	__asm__ __volatile__ ( \
	410	__usdt_asm( 990: USDT_NOP) \
	411	__usdt_asm3( .pushsection .note.stapsdt, "", "note") \
	412	__usdt_asm1( .balign 4) \
	413	__usdt_asm3( .4byte 992f-991f,994f-993f,3) \
	414	__usdt_asm1(991: .asciz "stapsdt") \
	415	__usdt_asm1(992: .balign 4) \
	416	__usdt_asm1(993: __usdt_asm_addr 990b) \
	417	__usdt_asm1( __usdt_asm_addr _.stapsdt.base) \
	418	__usdt_asm1( __usdt_asm_addr sema) \
	419	__usdt_asm_strz(group) \
	420	__usdt_asm_strz(name) \
	421	__usdt_asm_args(__VA_ARGS__) \
	422	__usdt_asm1( .ascii "\0") \
	423	__usdt_asm1(994: .balign 4) \
	424	__usdt_asm1( .popsection) \
	425	__usdt_asm1(.ifndef _.stapsdt.base) \
	426	__usdt_asm5( .pushsection .stapsdt.base,"aG","progbits",.stapsdt.base,comdat)\
	427	__usdt_asm1( .weak _.stapsdt.base) \
	428	__usdt_asm1( .hidden _.stapsdt.base) \
	429	__usdt_asm1(_.stapsdt.base:) \
	430	__usdt_asm1( .space 1) \
	431	__usdt_asm2( .size _.stapsdt.base, 1) \
	432	__usdt_asm1( .popsection) \
	433	__usdt_asm1(.endif) \
	434	:: __usdt_asm_ops(__VA_ARGS__) \
	435	); \
	436	} while (0)
	437
	438	/*
	439	* NB: gdb PR24541 highlighted an unspecified corner of the sdt.h
	440	* operand note format.
	441	*
	442	* The named register may be a longer or shorter (!) alias for the
	443	* storage where the value in question is found. For example, on
	444	* i386, 64-bit value may be put in register pairs, and a register
	445	* name stored would identify just one of them. Previously, gcc was
	446	* asked to emit the %w[id] (16-bit alias of some registers holding
	447	* operands), even when a wider 32-bit value was used.
	448	*
	449	* Bottom line: the byte-width given before the @ sign governs. If
	450	* there is a mismatch between that width and that of the named
	451	* register, then a sys/sdt.h note consumer may need to employ
	452	* architecture-specific heuristics to figure out where the compiler
	453	* has actually put the complete value.
	454	*/
	455	#if defined(__powerpc__) \|\| defined(__powerpc64__)
	456	#define __usdt_argref(id) %I[id]%[id]
	457	#elif defined(__i386__)
	458	#define __usdt_argref(id) %k[id] /* gcc.gnu.org/PR80115 sourceware.org/PR24541 */
	459	#else
	460	#define __usdt_argref(id) %[id]
	461	#endif
	462
	463	#define __usdt_asm_arg(n) __usdt_asm_str(%c[__usdt_asz##n]) \
	464	__usdt_asm1(.ascii "@") \
	465	__usdt_asm_str(__usdt_argref(__usdt_aval##n))
	466
	467	#define __usdt_asm_args0 /* no arguments */
	468	#define __usdt_asm_args1 __usdt_asm_arg(1)
	469	#define __usdt_asm_args2 __usdt_asm_args1 __usdt_asm1(.ascii " ") __usdt_asm_arg(2)
	470	#define __usdt_asm_args3 __usdt_asm_args2 __usdt_asm1(.ascii " ") __usdt_asm_arg(3)
	471	#define __usdt_asm_args4 __usdt_asm_args3 __usdt_asm1(.ascii " ") __usdt_asm_arg(4)
	472	#define __usdt_asm_args5 __usdt_asm_args4 __usdt_asm1(.ascii " ") __usdt_asm_arg(5)
	473	#define __usdt_asm_args6 __usdt_asm_args5 __usdt_asm1(.ascii " ") __usdt_asm_arg(6)
	474	#define __usdt_asm_args7 __usdt_asm_args6 __usdt_asm1(.ascii " ") __usdt_asm_arg(7)
	475	#define __usdt_asm_args8 __usdt_asm_args7 __usdt_asm1(.ascii " ") __usdt_asm_arg(8)
	476	#define __usdt_asm_args9 __usdt_asm_args8 __usdt_asm1(.ascii " ") __usdt_asm_arg(9)
	477	#define __usdt_asm_args10 __usdt_asm_args9 __usdt_asm1(.ascii " ") __usdt_asm_arg(10)
	478	#define __usdt_asm_args11 __usdt_asm_args10 __usdt_asm1(.ascii " ") __usdt_asm_arg(11)
	479	#define __usdt_asm_args12 __usdt_asm_args11 __usdt_asm1(.ascii " ") __usdt_asm_arg(12)
	480	#define __usdt_asm_args(...) __usdt_apply(__usdt_asm_args, __usdt_narg(__VA_ARGS__))
	481
	482	#define __usdt_is_arr(x) (__builtin_classify_type(x) == 14 \|\| __builtin_classify_type(x) == 5)
	483	#define __usdt_arg_size(x) (__usdt_is_arr(x) ? sizeof(void *) : sizeof(x))
	484
	485	/*
	486	* We can't use __builtin_choose_expr() in C++, so fall back to table-based
	487	* signedness determination for known types, utilizing templates magic.
	488	*/
	489	#ifdef __cplusplus
	490
	491	#define __usdt_is_signed(x) (!__usdt_is_arr(x) && __usdt_t<__typeof(x)>::is_signed)
	492
	493	#include <cstddef>
	494
	495	template<typename T> struct __usdt_t { static const bool is_signed = false; };
	496	template<typename A> struct __usdt_t<A[]> : public __usdt_t<A *> {};
	497	template<typename A, size_t N> struct __usdt_t<A[N]> : public __usdt_t<A *> {};
	498
	499	#define __usdt_def_signed(T) \
	500	template<> struct __usdt_t<T> { static const bool is_signed = true; }; \
	501	template<> struct __usdt_t<const T> { static const bool is_signed = true; }; \
	502	template<> struct __usdt_t<volatile T> { static const bool is_signed = true; }; \
	503	template<> struct __usdt_t<const volatile T> { static const bool is_signed = true; }
	504	#define __usdt_maybe_signed(T) \
	505	template<> struct __usdt_t<T> { static const bool is_signed = (T)-1 < (T)1; }; \
	506	template<> struct __usdt_t<const T> { static const bool is_signed = (T)-1 < (T)1; }; \
	507	template<> struct __usdt_t<volatile T> { static const bool is_signed = (T)-1 < (T)1; }; \
	508	template<> struct __usdt_t<const volatile T> { static const bool is_signed = (T)-1 < (T)1; }
	509
	510	__usdt_def_signed(signed char);
	511	__usdt_def_signed(short);
	512	__usdt_def_signed(int);
	513	__usdt_def_signed(long);
	514	__usdt_def_signed(long long);
	515	__usdt_maybe_signed(char);
	516	__usdt_maybe_signed(wchar_t);
	517
	518	#else /* !__cplusplus */
	519
	520	#define __usdt_is_inttype(x) (__builtin_classify_type(x) >= 1 && __builtin_classify_type(x) <= 4)
	521	#define __usdt_inttype(x) __typeof(__builtin_choose_expr(__usdt_is_inttype(x), (x), 0U))
	522	#define __usdt_is_signed(x) ((__usdt_inttype(x))-1 < (__usdt_inttype(x))1)
	523
	524	#endif /* __cplusplus */
	525
	526	#define __usdt_asm_op(n, x) \
	527	[__usdt_asz##n] "n" ((__usdt_is_signed(x) ? (int)-1 : 1) * (int)__usdt_arg_size(x)), \
	528	[__usdt_aval##n] __usdt_str(USDT_ARG_CONSTRAINT)(x)
	529
	530	#define __usdt_asm_ops0() [__usdt_dummy] "g" (0)
	531	#define __usdt_asm_ops1(x) __usdt_asm_op(1, x)
	532	#define __usdt_asm_ops2(a,x) __usdt_asm_ops1(a), __usdt_asm_op(2, x)
	533	#define __usdt_asm_ops3(a,b,x) __usdt_asm_ops2(a,b), __usdt_asm_op(3, x)
	534	#define __usdt_asm_ops4(a,b,c,x) __usdt_asm_ops3(a,b,c), __usdt_asm_op(4, x)
	535	#define __usdt_asm_ops5(a,b,c,d,x) __usdt_asm_ops4(a,b,c,d), __usdt_asm_op(5, x)
	536	#define __usdt_asm_ops6(a,b,c,d,e,x) __usdt_asm_ops5(a,b,c,d,e), __usdt_asm_op(6, x)
	537	#define __usdt_asm_ops7(a,b,c,d,e,f,x) __usdt_asm_ops6(a,b,c,d,e,f), __usdt_asm_op(7, x)
	538	#define __usdt_asm_ops8(a,b,c,d,e,f,g,x) __usdt_asm_ops7(a,b,c,d,e,f,g), __usdt_asm_op(8, x)
	539	#define __usdt_asm_ops9(a,b,c,d,e,f,g,h,x) __usdt_asm_ops8(a,b,c,d,e,f,g,h), __usdt_asm_op(9, x)
	540	#define __usdt_asm_ops10(a,b,c,d,e,f,g,h,i,x) __usdt_asm_ops9(a,b,c,d,e,f,g,h,i), __usdt_asm_op(10, x)
	541	#define __usdt_asm_ops11(a,b,c,d,e,f,g,h,i,j,x) __usdt_asm_ops10(a,b,c,d,e,f,g,h,i,j), __usdt_asm_op(11, x)
	542	#define __usdt_asm_ops12(a,b,c,d,e,f,g,h,i,j,k,x) __usdt_asm_ops11(a,b,c,d,e,f,g,h,i,j,k), __usdt_asm_op(12, x)
	543	#define __usdt_asm_ops(...) __usdt_apply(__usdt_asm_ops, __usdt_narg(__VA_ARGS__))(__VA_ARGS__)
	544
	545	#endif /* __USDT_H */

tools/testing/selftests/seccomp/seccomp_bpf.c+86−21

@@ -74,6 +74,14 @@ · 73 unmodified lines
74	#define noinline __attribute__((noinline))	74	#define noinline __attribute__((noinline))
75	#endif	75	#endif
76		76
		77	#ifndef __nocf_check
		78	#define __nocf_check __attribute__((nocf_check))
		79	#endif
		80
		81	#ifndef __naked
		82	#define __naked __attribute__((__naked__))
		83	#endif
		84
77	#ifndef PR_SET_NO_NEW_PRIVS	85	#ifndef PR_SET_NO_NEW_PRIVS
78	#define PR_SET_NO_NEW_PRIVS 38	86	#define PR_SET_NO_NEW_PRIVS 38
79	#define PR_GET_NO_NEW_PRIVS 39	87	#define PR_GET_NO_NEW_PRIVS 39
@@ -5027,7 +5035,36 @@ TEST(tsync_vs_dead_thread_leader) · 4947 unmodified lines
5027	EXPECT_EQ(0, status);	5035	EXPECT_EQ(0, status);
5028	}	5036	}
5029		5037
5030	noinline int probed(void)	5038	#ifdef __x86_64__
		5039
		5040	/*
		5041	* We need naked probed_uprobe function. Using __nocf_check
		5042	* check to skip possible endbr64 instruction and ignoring
		5043	* -Wattributes, otherwise the compilation might fail.
		5044	*/
		5045	#pragma GCC diagnostic push
		5046	#pragma GCC diagnostic ignored "-Wattributes"
		5047
		5048	__naked __nocf_check noinline int probed_uprobe(void)
		5049	{
		5050	/*
		5051	* Optimized uprobe is possible only on top of nop5 instruction.
		5052	*/
		5053	asm volatile (" \n"
		5054	".byte 0x0f, 0x1f, 0x44, 0x00, 0x00 \n"
		5055	"ret \n"
		5056	);
		5057	}
		5058	#pragma GCC diagnostic pop
		5059
		5060	#else
		5061	noinline int probed_uprobe(void)
		5062	{
		5063	return 1;
		5064	}
		5065	#endif
		5066
		5067	noinline int probed_uretprobe(void)
5031	{	5068	{
5032	return 1;	5069	return 1;
5033	}	5070	}
@@ -5080,35 +5117,46 @@ static ssize_t get_uprobe_offset(const void *addr) · 46 unmodified lines
5080	return found ? (uintptr_t)addr - start + base : -1;	5117	return found ? (uintptr_t)addr - start + base : -1;
5081	}	5118	}
5082		5119
5083	FIXTURE(URETPROBE) {	5120	FIXTURE(UPROBE) {
5084	int fd;	5121	int fd;
5085	};	5122	};
5086		5123
5087	FIXTURE_VARIANT(URETPROBE) {	5124	FIXTURE_VARIANT(UPROBE) {
5088	/*	5125	/*
5089	* All of the URETPROBE behaviors can be tested with either	5126	* All of the U(RET)PROBE behaviors can be tested with either
5090	* uretprobe attached or not	5127	* u(ret)probe attached or not
5091	*/	5128	*/
5092	bool attach;	5129	bool attach;
		5130	/*
		5131	* Test both uprobe and uretprobe.
		5132	*/
		5133	bool uretprobe;
5093	};	5134	};
5094		5135
5095	FIXTURE_VARIANT_ADD(URETPROBE, attached) {	5136	FIXTURE_VARIANT_ADD(UPROBE, not_attached) {
		5137	.attach = false,
		5138	.uretprobe = false,
		5139	};
		5140
		5141	FIXTURE_VARIANT_ADD(UPROBE, uprobe_attached) {
5096	.attach = true,	5142	.attach = true,
		5143	.uretprobe = false,
5097	};	5144	};
5098		5145
5099	FIXTURE_VARIANT_ADD(URETPROBE, not_attached) {	5146	FIXTURE_VARIANT_ADD(UPROBE, uretprobe_attached) {
5100	.attach = false,	5147	.attach = true,
		5148	.uretprobe = true,
5101	};	5149	};
5102		5150
5103	FIXTURE_SETUP(URETPROBE)	5151	FIXTURE_SETUP(UPROBE)
5104	{	5152	{
5105	const size_t attr_sz = sizeof(struct perf_event_attr);	5153	const size_t attr_sz = sizeof(struct perf_event_attr);
5106	struct perf_event_attr attr;	5154	struct perf_event_attr attr;
5107	ssize_t offset;	5155	ssize_t offset;
5108	int type, bit;	5156	int type, bit;
5109		5157
5110	#ifndef __NR_uretprobe	5158	#if !defined(__NR_uprobe) \|\| !defined(__NR_uretprobe)
5111	SKIP(return, "__NR_uretprobe syscall not defined");	5159	SKIP(return, "__NR_uprobe ot __NR_uretprobe syscalls not defined");
5112	#endif	5160	#endif
5113		5161
5114	if (!variant->attach)	5162	if (!variant->attach)
@@ -5118,12 +5166,17 @@ FIXTURE_SETUP(URETPROBE) · 3 unmodified lines
5118		5166
5119	type = determine_uprobe_perf_type();	5167	type = determine_uprobe_perf_type();
5120	ASSERT_GE(type, 0);	5168	ASSERT_GE(type, 0);
5121	bit = determine_uprobe_retprobe_bit();	5169
5122	ASSERT_GE(bit, 0);	5170	if (variant->uretprobe) {
5123	offset = get_uprobe_offset(probed);	5171	bit = determine_uprobe_retprobe_bit();
		5172	ASSERT_GE(bit, 0);
		5173	}
		5174
		5175	offset = get_uprobe_offset(variant->uretprobe ? probed_uretprobe : probed_uprobe);
5124	ASSERT_GE(offset, 0);	5176	ASSERT_GE(offset, 0);
5125		5177
5126	attr.config \|= 1 << bit;	5178	if (variant->uretprobe)
		5179	attr.config \|= 1 << bit;
5127	attr.size = attr_sz;	5180	attr.size = attr_sz;
5128	attr.type = type;	5181	attr.type = type;
5129	attr.config1 = ptr_to_u64("/proc/self/exe");	5182	attr.config1 = ptr_to_u64("/proc/self/exe");
@@ -5134,7 +5187,7 @@ FIXTURE_SETUP(URETPROBE) · 4 unmodified lines
5134	PERF_FLAG_FD_CLOEXEC);	5187	PERF_FLAG_FD_CLOEXEC);
5135	}	5188	}
5136		5189
5137	FIXTURE_TEARDOWN(URETPROBE)	5190	FIXTURE_TEARDOWN(UPROBE)
5138	{	5191	{
5139	/* we could call close(self->fd), but we'd need extra filter for	5192	/* we could call close(self->fd), but we'd need extra filter for
5140	* that and since we are calling _exit right away..	5193	* that and since we are calling _exit right away..
@@ -5148,11 +5201,17 @@ static int run_probed_with_filter(struct sock_fprog *prog) · 7 unmodified lines
5148	return -1;	5201	return -1;
5149	}	5202	}
5150		5203
5151	probed();	5204	/*
		5205	* Uprobe is optimized after first hit, so let's hit twice.
		5206	*/
		5207	probed_uprobe();
		5208	probed_uprobe();
		5209
		5210	probed_uretprobe();
5152	return 0;	5211	return 0;
5153	}	5212	}
5154		5213
5155	TEST_F(URETPROBE, uretprobe_default_allow)	5214	TEST_F(UPROBE, uprobe_default_allow)
5156	{	5215	{
5157	struct sock_filter filter[] = {	5216	struct sock_filter filter[] = {
5158	BPF_STMT(BPF_RET\|BPF_K, SECCOMP_RET_ALLOW),	5217	BPF_STMT(BPF_RET\|BPF_K, SECCOMP_RET_ALLOW),
@@ -5165,7 +5224,7 @@ TEST_F(URETPROBE, uretprobe_default_allow) · 6 unmodified lines
5165	ASSERT_EQ(0, run_probed_with_filter(&prog));	5224	ASSERT_EQ(0, run_probed_with_filter(&prog));
5166	}	5225	}
5167		5226
5168	TEST_F(URETPROBE, uretprobe_default_block)	5227	TEST_F(UPROBE, uprobe_default_block)
5169	{	5228	{
5170	struct sock_filter filter[] = {	5229	struct sock_filter filter[] = {
5171	BPF_STMT(BPF_LD\|BPF_W\|BPF_ABS,	5230	BPF_STMT(BPF_LD\|BPF_W\|BPF_ABS,
@@ -5182,11 +5241,14 @@ TEST_F(URETPROBE, uretprobe_default_block) · 10 unmodified lines
5182	ASSERT_EQ(0, run_probed_with_filter(&prog));	5241	ASSERT_EQ(0, run_probed_with_filter(&prog));
5183	}	5242	}
5184		5243
5185	TEST_F(URETPROBE, uretprobe_block_uretprobe_syscall)	5244	TEST_F(UPROBE, uprobe_block_syscall)
5186	{	5245	{
5187	struct sock_filter filter[] = {	5246	struct sock_filter filter[] = {
5188	BPF_STMT(BPF_LD\|BPF_W\|BPF_ABS,	5247	BPF_STMT(BPF_LD\|BPF_W\|BPF_ABS,
5189	offsetof(struct seccomp_data, nr)),	5248	offsetof(struct seccomp_data, nr)),
		5249	#ifdef __NR_uprobe
		5250	BPF_JUMP(BPF_JMP\|BPF_JEQ\|BPF_K, __NR_uprobe, 1, 2),
		5251	#endif
5190	#ifdef __NR_uretprobe	5252	#ifdef __NR_uretprobe
5191	BPF_JUMP(BPF_JMP\|BPF_JEQ\|BPF_K, __NR_uretprobe, 0, 1),	5253	BPF_JUMP(BPF_JMP\|BPF_JEQ\|BPF_K, __NR_uretprobe, 0, 1),
5192	#endif	5254	#endif
@@ -5201,11 +5263,14 @@ TEST_F(URETPROBE, uretprobe_block_uretprobe_syscall) · 8 unmodified lines
5201	ASSERT_EQ(0, run_probed_with_filter(&prog));	5263	ASSERT_EQ(0, run_probed_with_filter(&prog));
5202	}	5264	}
5203		5265
5204	TEST_F(URETPROBE, uretprobe_default_block_with_uretprobe_syscall)	5266	TEST_F(UPROBE, uprobe_default_block_with_syscall)
5205	{	5267	{
5206	struct sock_filter filter[] = {	5268	struct sock_filter filter[] = {
5207	BPF_STMT(BPF_LD\|BPF_W\|BPF_ABS,	5269	BPF_STMT(BPF_LD\|BPF_W\|BPF_ABS,
5208	offsetof(struct seccomp_data, nr)),	5270	offsetof(struct seccomp_data, nr)),
		5271	#ifdef __NR_uprobe
		5272	BPF_JUMP(BPF_JMP\|BPF_JEQ\|BPF_K, __NR_uprobe, 3, 0),
		5273	#endif
5209	#ifdef __NR_uretprobe	5274	#ifdef __NR_uretprobe
5210	BPF_JUMP(BPF_JMP\|BPF_JEQ\|BPF_K, __NR_uretprobe, 2, 0),	5275	BPF_JUMP(BPF_JMP\|BPF_JEQ\|BPF_K, __NR_uretprobe, 2, 0),
5211	#endif	5276	#endif