Merge tag 'probes-v6.11' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace

Pull probes updates from Masami Hiramatsu: "Uprobes: - x86/shstk: Make return uprobe work with shadow stack - Add uretprobe syscall which speeds up the uretprobe 10-30% faster. This syscall is automatically used from user-space trampolines which are generated by the uretprobe. If this syscall is used by normal user program, it will cause SIGILL. Note that this is currently only implemented on x86_64. (This also has two fixes for adjusting the syscall number to avoid conflict with new *attrat syscalls.) - uprobes/perf: fix user stack traces in the presence of pending uretprobe. This corrects the uretprobe's trampoline address in the stacktrace with correct return address - selftests/x86: Add a return uprobe with shadow stack test - selftests/bpf: Add uretprobe syscall related tests. - test case for register integrity check - test case with register changing case - test case for uretprobe syscall without uprobes (expected to fail) - test case for uretprobe with shadow stack - selftests/bpf: add test validating uprobe/uretprobe stack traces - MAINTAINERS: Add uprobes entry. This does not specify the tree but to clarify who maintains and reviews the uprobes Kprobes: - tracing/kprobes: Test case cleanups. Replace redundant WARN_ON_ONCE() + pr_warn() with WARN_ONCE() and remove unnecessary code from selftest - tracing/kprobes: Add symbol counting check when module loads. This checks the uniqueness of the probed symbol on modules. The same check has already done for kernel symbols (This also has a fix for build error with CONFIG_MODULES=n) Cleanup: - Add MODULE_DESCRIPTION() macros for fprobe and kprobe examples" * tag 'probes-v6.11' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace: MAINTAINERS: Add uprobes entry selftests/bpf: Change uretprobe syscall number in uprobe_syscall test uprobe: Change uretprobe syscall scope and number tracing/kprobes: Fix build error when find_module() is not available tracing/kprobes: Add symbol counting check when module loads selftests/bpf: add test validating uprobe/uretprobe stack traces perf,uprobes: fix user stack traces in the presence of pending uretprobes tracing/kprobe: Remove cleanup code unrelated to selftest tracing/kprobe: Integrate test warnings into WARN_ONCE selftests/bpf: Add uretprobe shadow stack test selftests/bpf: Add uretprobe syscall call from user space test selftests/bpf: Add uretprobe syscall test for regs changes selftests/bpf: Add uretprobe syscall test for regs integrity selftests/x86: Add return uprobe shadow stack test uprobe: Add uretprobe syscall to speed up return probe uprobe: Wire up uretprobe system call x86/shstk: Make return uprobe work with shadow stack samples: kprobes: add missing MODULE_DESCRIPTION() macros fprobe: add missing MODULE_DESCRIPTION() macro

Linus Torvalds committed 2 years ago

Commit 91bd0082 parents cb273eb c26b1b823 files · +1320 −92Browse files

Unified · Split

23 files

MAINTAINERS +13 −0
arch/x86/entry/syscalls/syscall_64.tbl +1 −0
arch/x86/include/asm/shstk.h +4 −0
arch/x86/kernel/shstk.c +16 −0
arch/x86/kernel/uprobes.c +123 −1
include/linux/syscalls.h +2 −0
include/linux/uprobes.h +3 −0
include/uapi/asm-generic/unistd.h +4 −1
kernel/events/callchain.c +42 −1
kernel/events/uprobes.c +26 −7
kernel/sys_ni.c +2 −0
kernel/trace/trace_kprobe.c +111 −81
samples/fprobe/fprobe_example.c +1 −0
samples/kprobes/kprobe_example.c +1 −0
samples/kprobes/kretprobe_example.c +1 −0
tools/include/linux/compiler.h +4 −0
tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c +122 −1
tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c +385 −0
tools/testing/selftests/bpf/prog_tests/uretprobe_stack.c +186 −0
tools/testing/selftests/bpf/progs/uprobe_syscall.c +15 −0
tools/testing/selftests/bpf/progs/uprobe_syscall_executed.c +17 −0
tools/testing/selftests/bpf/progs/uretprobe_stack.c +96 −0
tools/testing/selftests/x86/test_shadow_stack.c +145 −0

MAINTAINERS+13−0

@@ -23367,6 +23367,19 @@ F: drivers/mtd/ubi/ · 23366 unmodified lines
23367	23367	F: include/linux/mtd/ubi.h
23368	23368	F: include/uapi/mtd/ubi-user.h
23369	23369
	23370	+UPROBES
	23371	+M: Masami Hiramatsu <mhiramat@kernel.org>
	23372	+M: Oleg Nesterov <oleg@redhat.com>
	23373	+M: Peter Zijlstra <peterz@infradead.org>
	23374	+L: linux-kernel@vger.kernel.org
	23375	+L: linux-trace-kernel@vger.kernel.org
	23376	+S: Maintained
	23377	+F: arch/*/include/asm/uprobes.h
	23378	+F: arch/*/kernel/probes/uprobes.c
	23379	+F: arch/*/kernel/uprobes.c
	23380	+F: include/linux/uprobes.h
	23381	+F: kernel/events/uprobes.c
	23382	+
23370	23383	USB "USBNET" DRIVER FRAMEWORK
23371	23384	M: Oliver Neukum <oneukum@suse.com>
23372	23385	L: netdev@vger.kernel.org

arch/x86/entry/syscalls/syscall_64.tbl+1−0

@@ -385,6 +385,7 @@ · 384 unmodified lines
385	385	460 common lsm_set_self_attr sys_lsm_set_self_attr
386	386	461 common lsm_list_modules sys_lsm_list_modules
387	387	462 common mseal sys_mseal
	388	+467 common uretprobe sys_uretprobe
388	389
389	390	#
390	391	# Due to a historical design error, certain syscalls are numbered differently

arch/x86/include/asm/shstk.h+4−0

@@ -21,6 +21,8 @@ unsigned long shstk_alloc_thread_stack(struct task_struct *p, unsigned long clon · 20 unmodified lines
21	21	void shstk_free(struct task_struct *p);
22	22	int setup_signal_shadow_stack(struct ksignal *ksig);
23	23	int restore_signal_shadow_stack(void);
	24	+int shstk_update_last_frame(unsigned long val);
	25	+bool shstk_is_enabled(void);
24	26	#else
25	27	static inline long shstk_prctl(struct task_struct *task, int option,
26	28	unsigned long arg2) { return -EINVAL; }
@@ -31,6 +33,8 @@ static inline unsigned long shstk_alloc_thread_stack(struct task_struct *p, · 4 unmodified lines
31	33	static inline void shstk_free(struct task_struct *p) {}
32	34	static inline int setup_signal_shadow_stack(struct ksignal *ksig) { return 0; }
33	35	static inline int restore_signal_shadow_stack(void) { return 0; }
	36	+static inline int shstk_update_last_frame(unsigned long val) { return 0; }
	37	+static inline bool shstk_is_enabled(void) { return false; }
34	38	#endif /* CONFIG_X86_USER_SHADOW_STACK */
35	39
36	40	#endif /* __ASSEMBLY__ */

arch/x86/kernel/shstk.c+16−0

@@ -577,3 +577,19 @@ long shstk_prctl(struct task_struct *task, int option, unsigned long arg2) · 576 unmodified lines
577	577	return wrss_control(true);
578	578	return -EINVAL;
579	579	}
	580	+
	581	+int shstk_update_last_frame(unsigned long val)
	582	+{
	583	+ unsigned long ssp;
	584	+
	585	+ if (!features_enabled(ARCH_SHSTK_SHSTK))
	586	+ return 0;
	587	+
	588	+ ssp = get_user_shstk_addr();
	589	+ return write_user_shstk_64((u64 __user *)ssp, (u64)val);
	590	+}
	591	+
	592	+bool shstk_is_enabled(void)
	593	+{
	594	+ return features_enabled(ARCH_SHSTK_SHSTK);
	595	+}

arch/x86/kernel/uprobes.c+123−1

@@ -12,6 +12,7 @@ · 11 unmodified lines
12	12	#include <linux/ptrace.h>
13	13	#include <linux/uprobes.h>
14	14	#include <linux/uaccess.h>
	15	+#include <linux/syscalls.h>
15	16
16	17	#include <linux/kdebug.h>
17	18	#include <asm/processor.h>
@@ -308,6 +309,122 @@ static int uprobe_init_insn(struct arch_uprobe auprobe, struct insn insn, bool · 290 unmodified lines
308	309	}
309	310
310	311	#ifdef CONFIG_X86_64
	312	+
	313	+asm (
	314	+ ".pushsection .rodata\n"
	315	+ ".global uretprobe_trampoline_entry\n"
	316	+ "uretprobe_trampoline_entry:\n"
	317	+ "pushq %rax\n"
	318	+ "pushq %rcx\n"
	319	+ "pushq %r11\n"
	320	+ "movq $" __stringify(__NR_uretprobe) ", %rax\n"
	321	+ "syscall\n"
	322	+ ".global uretprobe_syscall_check\n"
	323	+ "uretprobe_syscall_check:\n"
	324	+ "popq %r11\n"
	325	+ "popq %rcx\n"
	326	+
	327	+ /* The uretprobe syscall replaces stored %rax value with final
	328	+ * return address, so we don't restore %rax in here and just
	329	+ * call ret.
	330	+ */
	331	+ "retq\n"
	332	+ ".global uretprobe_trampoline_end\n"
	333	+ "uretprobe_trampoline_end:\n"
	334	+ ".popsection\n"
	335	+);
	336	+
	337	+extern u8 uretprobe_trampoline_entry[];
	338	+extern u8 uretprobe_trampoline_end[];
	339	+extern u8 uretprobe_syscall_check[];
	340	+
	341	+void arch_uprobe_trampoline(unsigned long psize)
	342	+{
	343	+ static uprobe_opcode_t insn = UPROBE_SWBP_INSN;
	344	+ struct pt_regs *regs = task_pt_regs(current);
	345	+
	346	+ /*
	347	+ * At the moment the uretprobe syscall trampoline is supported
	348	+ * only for native 64-bit process, the compat process still uses
	349	+ * standard breakpoint.
	350	+ */
	351	+ if (user_64bit_mode(regs)) {
	352	+ *psize = uretprobe_trampoline_end - uretprobe_trampoline_entry;
	353	+ return uretprobe_trampoline_entry;
	354	+ }
	355	+
	356	+ *psize = UPROBE_SWBP_INSN_SIZE;
	357	+ return &insn;
	358	+}
	359	+
	360	+static unsigned long trampoline_check_ip(void)
	361	+{
	362	+ unsigned long tramp = uprobe_get_trampoline_vaddr();
	363	+
	364	+ return tramp + (uretprobe_syscall_check - uretprobe_trampoline_entry);
	365	+}
	366	+
	367	+SYSCALL_DEFINE0(uretprobe)
	368	+{
	369	+ struct pt_regs *regs = task_pt_regs(current);
	370	+ unsigned long err, ip, sp, r11_cx_ax[3];
	371	+
	372	+ if (regs->ip != trampoline_check_ip())
	373	+ goto sigill;
	374	+
	375	+ err = copy_from_user(r11_cx_ax, (void __user *)regs->sp, sizeof(r11_cx_ax));
	376	+ if (err)
	377	+ goto sigill;
	378	+
	379	+ /* expose the "right" values of r11/cx/ax/sp to uprobe_consumer/s */
	380	+ regs->r11 = r11_cx_ax[0];
	381	+ regs->cx = r11_cx_ax[1];
	382	+ regs->ax = r11_cx_ax[2];
	383	+ regs->sp += sizeof(r11_cx_ax);
	384	+ regs->orig_ax = -1;
	385	+
	386	+ ip = regs->ip;
	387	+ sp = regs->sp;
	388	+
	389	+ uprobe_handle_trampoline(regs);
	390	+
	391	+ /*
	392	+ * Some of the uprobe consumers has changed sp, we can do nothing,
	393	+ * just return via iret.
	394	+ * .. or shadow stack is enabled, in which case we need to skip
	395	+ * return through the user space stack address.
	396	+ */
	397	+ if (regs->sp != sp \|\| shstk_is_enabled())
	398	+ return regs->ax;
	399	+ regs->sp -= sizeof(r11_cx_ax);
	400	+
	401	+ /* for the case uprobe_consumer has changed r11/cx */
	402	+ r11_cx_ax[0] = regs->r11;
	403	+ r11_cx_ax[1] = regs->cx;
	404	+
	405	+ /*
	406	+ * ax register is passed through as return value, so we can use
	407	+ * its space on stack for ip value and jump to it through the
	408	+ * trampoline's ret instruction
	409	+ */
	410	+ r11_cx_ax[2] = regs->ip;
	411	+ regs->ip = ip;
	412	+
	413	+ err = copy_to_user((void __user *)regs->sp, r11_cx_ax, sizeof(r11_cx_ax));
	414	+ if (err)
	415	+ goto sigill;
	416	+
	417	+ /* ensure sysret, see do_syscall_64() */
	418	+ regs->r11 = regs->flags;
	419	+ regs->cx = regs->ip;
	420	+
	421	+ return regs->ax;
	422	+
	423	+sigill:
	424	+ force_sig(SIGILL);
	425	+ return -1;
	426	+}
	427	+
311	428	/*
312	429	* If arch_uprobe->insn doesn't use rip-relative addressing, return
313	430	* immediately. Otherwise, rewrite the instruction so that it accesses
@@ -1076,8 +1193,13 @@ arch_uretprobe_hijack_return_addr(unsigned long trampoline_vaddr, struct pt_regs · 762 unmodified lines
1076	1193	return orig_ret_vaddr;
1077	1194
1078	1195	nleft = copy_to_user((void __user *)regs->sp, &trampoline_vaddr, rasize);
1079		− if (likely(!nleft))
	1196	+ if (likely(!nleft)) {
	1197	+ if (shstk_update_last_frame(trampoline_vaddr)) {
	1198	+ force_sig(SIGSEGV);
	1199	+ return -1;
	1200	+ }
1080	1201	return orig_ret_vaddr;
	1202	+ }
1081	1203
1082	1204	if (nleft != rasize) {
1083	1205	pr_err("return address clobbered: pid=%d, %%sp=%#lx, %%ip=%#lx\n",

include/linux/syscalls.h+2−0

@@ -979,6 +979,8 @@ asmlinkage long sys_lsm_list_modules(u64 __user ids, u32 __user size, u32 flag · 978 unmodified lines
979	979	/* x86 */
980	980	asmlinkage long sys_ioperm(unsigned long from, unsigned long num, int on);
981	981
	982	+asmlinkage long sys_uretprobe(void);
	983	+
982	984	/* pciconfig: alpha, arm, arm64, ia64, sparc */
983	985	asmlinkage long sys_pciconfig_read(unsigned long bus, unsigned long dfn,
984	986	unsigned long off, unsigned long len,

include/linux/uprobes.h+3−0

@@ -138,6 +138,9 @@ extern bool arch_uretprobe_is_alive(struct return_instance *ret, enum rp_check c · 137 unmodified lines
138	138	extern bool arch_uprobe_ignore(struct arch_uprobe aup, struct pt_regs regs);
139	139	extern void arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr,
140	140	void *src, unsigned long len);
	141	+extern void uprobe_handle_trampoline(struct pt_regs *regs);
	142	+extern void arch_uprobe_trampoline(unsigned long psize);
	143	+extern unsigned long uprobe_get_trampoline_vaddr(void);
141	144	#else /* !CONFIG_UPROBES */
142	145	struct uprobes_state {
143	146	};

include/uapi/asm-generic/unistd.h+4−1

@@ -841,8 +841,11 @@ __SYSCALL(__NR_lsm_list_modules, sys_lsm_list_modules) · 840 unmodified lines
841	841	#define __NR_mseal 462
842	842	__SYSCALL(__NR_mseal, sys_mseal)
843	843
	844	+#define __NR_uretprobe 463
	845	+__SYSCALL(__NR_uretprobe, sys_uretprobe)
	846	+
844	847	#undef __NR_syscalls
845		−#define __NR_syscalls 463
	848	+#define __NR_syscalls 464
846	849
847	850	/*
848	851	* 32 bit systems traditionally used different

kernel/events/callchain.c+42−1

@@ -11,6 +11,7 @@ · 10 unmodified lines
11	11	#include <linux/perf_event.h>
12	12	#include <linux/slab.h>
13	13	#include <linux/sched/task_stack.h>
	14	+#include <linux/uprobes.h>
14	15
15	16	#include "internal.h"
16	17
@@ -176,13 +177,51 @@ put_callchain_entry(int rctx) · 159 unmodified lines
176	177	put_recursion_context(this_cpu_ptr(callchain_recursion), rctx);
177	178	}
178	179
	180	+static void fixup_uretprobe_trampoline_entries(struct perf_callchain_entry *entry,
	181	+ int start_entry_idx)
	182	+{
	183	+#ifdef CONFIG_UPROBES
	184	+ struct uprobe_task *utask = current->utask;
	185	+ struct return_instance *ri;
	186	+ __u64 cur_ip, last_ip, tramp_addr;
	187	+
	188	+ if (likely(!utask \|\| !utask->return_instances))
	189	+ return;
	190	+
	191	+ cur_ip = &entry->ip[start_entry_idx];
	192	+ last_ip = &entry->ip[entry->nr - 1];
	193	+ ri = utask->return_instances;
	194	+ tramp_addr = uprobe_get_trampoline_vaddr();
	195	+
	196	+ /*
	197	+ * If there are pending uretprobes for the current thread, they are
	198	+ * recorded in a list inside utask->return_instances; each such
	199	+ * pending uretprobe replaces traced user function's return address on
	200	+ * the stack, so when stack trace is captured, instead of seeing
	201	+ * actual function's return address, we'll have one or many uretprobe
	202	+ * trampoline addresses in the stack trace, which are not helpful and
	203	+ * misleading to users.
	204	+ * So here we go over the pending list of uretprobes, and each
	205	+ * encountered trampoline address is replaced with actual return
	206	+ * address.
	207	+ */
	208	+ while (ri && cur_ip <= last_ip) {
	209	+ if (*cur_ip == tramp_addr) {
	210	+ *cur_ip = ri->orig_ret_vaddr;
	211	+ ri = ri->next;
	212	+ }
	213	+ cur_ip++;
	214	+ }
	215	+#endif
	216	+}
	217	+
179	218	struct perf_callchain_entry *
180	219	get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user,
181	220	u32 max_stack, bool crosstask, bool add_mark)
182	221	{
183	222	struct perf_callchain_entry *entry;
184	223	struct perf_callchain_entry_ctx ctx;
185		− int rctx;
	224	+ int rctx, start_entry_idx;
186	225
187	226	entry = get_callchain_entry(&rctx);
188	227	if (!entry)
@@ -215,7 +254,9 @@ get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user, · 26 unmodified lines
215	254	if (add_mark)
216	255	perf_callchain_store_context(&ctx, PERF_CONTEXT_USER);
217	256
	257	+ start_entry_idx = entry->nr;
218	258	perf_callchain_user(&ctx, regs);
	259	+ fixup_uretprobe_trampoline_entries(entry, start_entry_idx);
219	260	}
220	261	}
221	262

kernel/events/uprobes.c+26−7

@@ -1474,11 +1474,20 @@ static int xol_add_vma(struct mm_struct mm, struct xol_area area) · 1473 unmodified lines
1474	1474	return ret;
1475	1475	}
1476	1476
	1477	+void * __weak arch_uprobe_trampoline(unsigned long *psize)
	1478	+{
	1479	+ static uprobe_opcode_t insn = UPROBE_SWBP_INSN;
	1480	+
	1481	+ *psize = UPROBE_SWBP_INSN_SIZE;
	1482	+ return &insn;
	1483	+}
	1484	+
1477	1485	static struct xol_area *__create_xol_area(unsigned long vaddr)
1478	1486	{
1479	1487	struct mm_struct *mm = current->mm;
1480		− uprobe_opcode_t insn = UPROBE_SWBP_INSN;
	1488	+ unsigned long insns_size;
1481	1489	struct xol_area *area;
	1490	+ void *insns;
1482	1491
1483	1492	area = kmalloc(sizeof(*area), GFP_KERNEL);
1484	1493	if (unlikely(!area))
@@ -1502,7 +1511,8 @@ static struct xol_area *__create_xol_area(unsigned long vaddr) · 17 unmodified lines
1502	1511	/* Reserve the 1st slot for get_trampoline_vaddr() */
1503	1512	set_bit(0, area->bitmap);
1504	1513	atomic_set(&area->slot_count, 1);
1505		− arch_uprobe_copy_ixol(area->pages[0], 0, &insn, UPROBE_SWBP_INSN_SIZE);
	1514	+ insns = arch_uprobe_trampoline(&insns_size);
	1515	+ arch_uprobe_copy_ixol(area->pages[0], 0, insns, insns_size);
1506	1516
1507	1517	if (!xol_add_vma(mm, area))
1508	1518	return area;
@@ -1827,7 +1837,7 @@ void uprobe_copy_process(struct task_struct *t, unsigned long flags) · 318 unmodified lines
1827	1837	*
1828	1838	* Returns -1 in case the xol_area is not allocated.
1829	1839	*/
1830		−static unsigned long get_trampoline_vaddr(void)
	1840	+unsigned long uprobe_get_trampoline_vaddr(void)
1831	1841	{
1832	1842	struct xol_area *area;
1833	1843	unsigned long trampoline_vaddr = -1;
@@ -1878,7 +1888,7 @@ static void prepare_uretprobe(struct uprobe uprobe, struct pt_regs regs) · 44 unmodified lines
1878	1888	if (!ri)
1879	1889	return;
1880	1890
1881		− trampoline_vaddr = get_trampoline_vaddr();
	1891	+ trampoline_vaddr = uprobe_get_trampoline_vaddr();
1882	1892	orig_ret_vaddr = arch_uretprobe_hijack_return_addr(trampoline_vaddr, regs);
1883	1893	if (orig_ret_vaddr == -1)
1884	1894	goto fail;
@@ -2123,7 +2133,7 @@ static struct return_instance find_next_ret_chain(struct return_instance ri) · 238 unmodified lines
2123	2133	return ri;
2124	2134	}
2125	2135
2126		−static void handle_trampoline(struct pt_regs *regs)
	2136	+void uprobe_handle_trampoline(struct pt_regs *regs)
2127	2137	{
2128	2138	struct uprobe_task *utask;
2129	2139	struct return_instance ri, next;
@@ -2149,6 +2159,15 @@ static void handle_trampoline(struct pt_regs *regs) · 19 unmodified lines
2149	2159
2150	2160	instruction_pointer_set(regs, ri->orig_ret_vaddr);
2151	2161	do {
	2162	+ /* pop current instance from the stack of pending return instances,
	2163	+ * as it's not pending anymore: we just fixed up original
	2164	+ * instruction pointer in regs and are about to call handlers;
	2165	+ * this allows fixup_uretprobe_trampoline_entries() to properly fix up
	2166	+ * captured stack traces from uretprobe handlers, in which pending
	2167	+ * trampoline addresses on the stack are replaced with correct
	2168	+ * original return addresses
	2169	+ */
	2170	+ utask->return_instances = ri->next;
2152	2171	if (valid)
2153	2172	handle_uretprobe_chain(ri, regs);
2154	2173	ri = free_ret_instance(ri);
@@ -2187,8 +2206,8 @@ static void handle_swbp(struct pt_regs *regs) · 32 unmodified lines
2187	2206	int is_swbp;
2188	2207
2189	2208	bp_vaddr = uprobe_get_swbp_addr(regs);
2190		− if (bp_vaddr == get_trampoline_vaddr())
2191		− return handle_trampoline(regs);
	2209	+ if (bp_vaddr == uprobe_get_trampoline_vaddr())
	2210	+ return uprobe_handle_trampoline(regs);
2192	2211
2193	2212	uprobe = find_active_uprobe(bp_vaddr, &is_swbp);
2194	2213	if (!uprobe) {

kernel/sys_ni.c+2−0

@@ -390,3 +390,5 @@ COND_SYSCALL(setuid16); · 389 unmodified lines
390	390
391	391	/* restartable sequence */
392	392	COND_SYSCALL(rseq);
	393	+
	394	+COND_SYSCALL(uretprobe);

kernel/trace/trace_kprobe.c+111−81

@@ -678,6 +678,21 @@ end: · 677 unmodified lines
678	678	}
679	679
680	680	#ifdef CONFIG_MODULES
	681	+static int validate_module_probe_symbol(const char modname, const char symbol);
	682	+
	683	+static int register_module_trace_kprobe(struct module mod, struct trace_kprobe tk)
	684	+{
	685	+ const char *p;
	686	+ int ret = 0;
	687	+
	688	+ p = strchr(trace_kprobe_symbol(tk), ':');
	689	+ if (p)
	690	+ ret = validate_module_probe_symbol(module_name(mod), p + 1);
	691	+ if (!ret)
	692	+ ret = __register_trace_kprobe(tk);
	693	+ return ret;
	694	+}
	695	+
681	696	/* Module notifier call back, checking event on the module */
682	697	static int trace_kprobe_module_callback(struct notifier_block *nb,
683	698	unsigned long val, void *data)
@@ -696,7 +711,7 @@ static int trace_kprobe_module_callback(struct notifier_block *nb, · 12 unmodified lines
696	711	if (trace_kprobe_within_module(tk, mod)) {
697	712	/* Don't need to check busy - this should have gone. */
698	713	__unregister_trace_kprobe(tk);
699		− ret = __register_trace_kprobe(tk);
	714	+ ret = register_module_trace_kprobe(mod, tk);
700	715	if (ret)
701	716	pr_warn("Failed to re-register probe %s on %s: %d\n",
702	717	trace_probe_name(&tk->tp),
@@ -747,17 +762,81 @@ static int count_mod_symbols(void data, const char name, unsigned long unused) · 44 unmodified lines
747	762	return 0;
748	763	}
749	764
750		−static unsigned int number_of_same_symbols(char *func_name)
	765	+static unsigned int number_of_same_symbols(const char mod, const char func_name)
751	766	{
752	767	struct sym_count_ctx ctx = { .count = 0, .name = func_name };
753	768
754		− kallsyms_on_each_match_symbol(count_symbols, func_name, &ctx.count);
	769	+ if (!mod)
	770	+ kallsyms_on_each_match_symbol(count_symbols, func_name, &ctx.count);
755	771
756		− module_kallsyms_on_each_symbol(NULL, count_mod_symbols, &ctx);
	772	+ module_kallsyms_on_each_symbol(mod, count_mod_symbols, &ctx);
757	773
758	774	return ctx.count;
759	775	}
760	776
	777	+static int validate_module_probe_symbol(const char modname, const char symbol)
	778	+{
	779	+ unsigned int count = number_of_same_symbols(modname, symbol);
	780	+
	781	+ if (count > 1) {
	782	+ /*
	783	+ * Users should use ADDR to remove the ambiguity of
	784	+ * using KSYM only.
	785	+ */
	786	+ return -EADDRNOTAVAIL;
	787	+ } else if (count == 0) {
	788	+ /*
	789	+ * We can return ENOENT earlier than when register the
	790	+ * kprobe.
	791	+ */
	792	+ return -ENOENT;
	793	+ }
	794	+ return 0;
	795	+}
	796	+
	797	+#ifdef CONFIG_MODULES
	798	+/* Return NULL if the module is not loaded or under unloading. */
	799	+static struct module try_module_get_by_name(const char name)
	800	+{
	801	+ struct module *mod;
	802	+
	803	+ rcu_read_lock_sched();
	804	+ mod = find_module(name);
	805	+ if (mod && !try_module_get(mod))
	806	+ mod = NULL;
	807	+ rcu_read_unlock_sched();
	808	+
	809	+ return mod;
	810	+}
	811	+#else
	812	+#define try_module_get_by_name(name) (NULL)
	813	+#endif
	814	+
	815	+static int validate_probe_symbol(char *symbol)
	816	+{
	817	+ struct module *mod = NULL;
	818	+ char modname = NULL, p;
	819	+ int ret = 0;
	820	+
	821	+ p = strchr(symbol, ':');
	822	+ if (p) {
	823	+ modname = symbol;
	824	+ symbol = p + 1;
	825	+ *p = '\0';
	826	+ mod = try_module_get_by_name(modname);
	827	+ if (!mod)
	828	+ goto out;
	829	+ }
	830	+
	831	+ ret = validate_module_probe_symbol(modname, symbol);
	832	+out:
	833	+ if (p)
	834	+ *p = ':';
	835	+ if (mod)
	836	+ module_put(mod);
	837	+ return ret;
	838	+}
	839	+
761	840	static int trace_kprobe_entry_handler(struct kretprobe_instance *ri,
762	841	struct pt_regs *regs);
763	842
@@ -881,6 +960,14 @@ static int __trace_kprobe_create(int argc, const char *argv[]) · 117 unmodified lines
881	960	trace_probe_log_err(0, BAD_PROBE_ADDR);
882	961	goto parse_error;
883	962	}
	963	+ ret = validate_probe_symbol(symbol);
	964	+ if (ret) {
	965	+ if (ret == -EADDRNOTAVAIL)
	966	+ trace_probe_log_err(0, NON_UNIQ_SYMBOL);
	967	+ else
	968	+ trace_probe_log_err(0, BAD_PROBE_ADDR);
	969	+ goto parse_error;
	970	+ }
884	971	if (is_return)
885	972	ctx.flags \|= TPARG_FL_RETURN;
886	973	ret = kprobe_on_func_entry(NULL, symbol, offset);
@@ -893,31 +980,6 @@ static int __trace_kprobe_create(int argc, const char *argv[]) · 6 unmodified lines
893	980	}
894	981	}
895	982
896		− if (symbol && !strchr(symbol, ':')) {
897		− unsigned int count;
898		−
899		− count = number_of_same_symbols(symbol);
900		− if (count > 1) {
901		− /*
902		− * Users should use ADDR to remove the ambiguity of
903		− * using KSYM only.
904		− */
905		− trace_probe_log_err(0, NON_UNIQ_SYMBOL);
906		− ret = -EADDRNOTAVAIL;
907		−
908		− goto error;
909		− } else if (count == 0) {
910		− /*
911		− * We can return ENOENT earlier than when register the
912		− * kprobe.
913		− */
914		− trace_probe_log_err(0, BAD_PROBE_ADDR);
915		− ret = -ENOENT;
916		−
917		− goto error;
918		− }
919		− }
920		−
921	983	trace_probe_log_set_index(0);
922	984	if (event) {
923	985	ret = traceprobe_parse_event_name(&event, &group, gbuf,
@@ -1835,21 +1897,9 @@ create_local_trace_kprobe(char func, void addr, unsigned long offs, · 911 unmodified lines
1835	1897	char *event;
1836	1898
1837	1899	if (func) {
1838		− unsigned int count;
1839		−
1840		− count = number_of_same_symbols(func);
1841		− if (count > 1)
1842		− /*
1843		− * Users should use addr to remove the ambiguity of
1844		− * using func only.
1845		− */
1846		− return ERR_PTR(-EADDRNOTAVAIL);
1847		− else if (count == 0)
1848		− /*
1849		− * We can return ENOENT earlier than when register the
1850		− * kprobe.
1851		− */
1852		− return ERR_PTR(-ENOENT);
	1900	+ ret = validate_probe_symbol(func);
	1901	+ if (ret)
	1902	+ return ERR_PTR(ret);
1853	1903	}
1854	1904
1855	1905	/*
@@ -2023,19 +2073,16 @@ static __init int kprobe_trace_self_tests_init(void) · 167 unmodified lines
2023	2073	pr_info("Testing kprobe tracing: ");
2024	2074
2025	2075	ret = create_or_delete_trace_kprobe("p:testprobe kprobe_trace_selftest_target $stack $stack0 +0($stack)");
2026		− if (WARN_ON_ONCE(ret)) {
2027		− pr_warn("error on probing function entry.\n");
	2076	+ if (WARN_ONCE(ret, "error on probing function entry.")) {
2028	2077	warn++;
2029	2078	} else {
2030	2079	/* Enable trace point */
2031	2080	tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);
2032		− if (WARN_ON_ONCE(tk == NULL)) {
2033		− pr_warn("error on getting new probe.\n");
	2081	+ if (WARN_ONCE(tk == NULL, "error on probing function entry.")) {
2034	2082	warn++;
2035	2083	} else {
2036	2084	file = find_trace_probe_file(tk, top_trace_array());
2037		− if (WARN_ON_ONCE(file == NULL)) {
2038		− pr_warn("error on getting probe file.\n");
	2085	+ if (WARN_ONCE(file == NULL, "error on getting probe file.")) {
2039	2086	warn++;
2040	2087	} else
2041	2088	enable_trace_kprobe(
@@ -2044,19 +2091,16 @@ static __init int kprobe_trace_self_tests_init(void) · 2 unmodified lines
2044	2091	}
2045	2092
2046	2093	ret = create_or_delete_trace_kprobe("r:testprobe2 kprobe_trace_selftest_target $retval");
2047		− if (WARN_ON_ONCE(ret)) {
2048		− pr_warn("error on probing function return.\n");
	2094	+ if (WARN_ONCE(ret, "error on probing function return.")) {
2049	2095	warn++;
2050	2096	} else {
2051	2097	/* Enable trace point */
2052	2098	tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);
2053		− if (WARN_ON_ONCE(tk == NULL)) {
2054		− pr_warn("error on getting 2nd new probe.\n");
	2099	+ if (WARN_ONCE(tk == NULL, "error on getting 2nd new probe.")) {
2055	2100	warn++;
2056	2101	} else {
2057	2102	file = find_trace_probe_file(tk, top_trace_array());
2058		− if (WARN_ON_ONCE(file == NULL)) {
2059		− pr_warn("error on getting probe file.\n");
	2103	+ if (WARN_ONCE(file == NULL, "error on getting probe file.")) {
2060	2104	warn++;
2061	2105	} else
2062	2106	enable_trace_kprobe(
@@ -2079,18 +2123,15 @@ static __init int kprobe_trace_self_tests_init(void) · 16 unmodified lines
2079	2123
2080	2124	/* Disable trace points before removing it */
2081	2125	tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);
2082		− if (WARN_ON_ONCE(tk == NULL)) {
2083		− pr_warn("error on getting test probe.\n");
	2126	+ if (WARN_ONCE(tk == NULL, "error on getting test probe.")) {
2084	2127	warn++;
2085	2128	} else {
2086		− if (trace_kprobe_nhit(tk) != 1) {
2087		− pr_warn("incorrect number of testprobe hits\n");
	2129	+ if (WARN_ONCE(trace_kprobe_nhit(tk) != 1,
	2130	+ "incorrect number of testprobe hits."))
2088	2131	warn++;
2089		− }
2090	2132
2091	2133	file = find_trace_probe_file(tk, top_trace_array());
2092		− if (WARN_ON_ONCE(file == NULL)) {
2093		− pr_warn("error on getting probe file.\n");
	2134	+ if (WARN_ONCE(file == NULL, "error on getting probe file.")) {
2094	2135	warn++;
2095	2136	} else
2096	2137	disable_trace_kprobe(
@@ -2098,18 +2139,15 @@ static __init int kprobe_trace_self_tests_init(void) · 1 unmodified line
2098	2139	}
2099	2140
2100	2141	tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);
2101		− if (WARN_ON_ONCE(tk == NULL)) {
2102		− pr_warn("error on getting 2nd test probe.\n");
	2142	+ if (WARN_ONCE(tk == NULL, "error on getting 2nd test probe.")) {
2103	2143	warn++;
2104	2144	} else {
2105		− if (trace_kprobe_nhit(tk) != 1) {
2106		− pr_warn("incorrect number of testprobe2 hits\n");
	2145	+ if (WARN_ONCE(trace_kprobe_nhit(tk) != 1,
	2146	+ "incorrect number of testprobe2 hits."))
2107	2147	warn++;
2108		− }
2109	2148
2110	2149	file = find_trace_probe_file(tk, top_trace_array());
2111		− if (WARN_ON_ONCE(file == NULL)) {
2112		− pr_warn("error on getting probe file.\n");
	2150	+ if (WARN_ONCE(file == NULL, "error on getting probe file.")) {
2113	2151	warn++;
2114	2152	} else
2115	2153	disable_trace_kprobe(
@@ -2117,23 +2155,15 @@ static __init int kprobe_trace_self_tests_init(void) · 1 unmodified line
2117	2155	}
2118	2156
2119	2157	ret = create_or_delete_trace_kprobe("-:testprobe");
2120		− if (WARN_ON_ONCE(ret)) {
2121		− pr_warn("error on deleting a probe.\n");
	2158	+ if (WARN_ONCE(ret, "error on deleting a probe."))
2122	2159	warn++;
2123		− }
2124	2160
2125	2161	ret = create_or_delete_trace_kprobe("-:testprobe2");
2126		− if (WARN_ON_ONCE(ret)) {
2127		− pr_warn("error on deleting a probe.\n");
	2162	+ if (WARN_ONCE(ret, "error on deleting a probe."))
2128	2163	warn++;
2129		− }
	2164	+
2130	2165
2131	2166	end:
2132		− ret = dyn_events_release_all(&trace_kprobe_ops);
2133		− if (WARN_ON_ONCE(ret)) {
2134		− pr_warn("error on cleaning up probes.\n");
2135		− warn++;
2136		− }
2137	2167	/*
2138	2168	* Wait for the optimizer work to finish. Otherwise it might fiddle
2139	2169	* with probes in already freed __init text.

samples/fprobe/fprobe_example.c+1−0

@@ -150,4 +150,5 @@ static void __exit fprobe_exit(void) · 149 unmodified lines
150	150
151	151	module_init(fprobe_init)
152	152	module_exit(fprobe_exit)
	153	+MODULE_DESCRIPTION("sample kernel module showing the use of fprobe");
153	154	MODULE_LICENSE("GPL");

samples/kprobes/kprobe_example.c+1−0

@@ -125,4 +125,5 @@ static void __exit kprobe_exit(void) · 124 unmodified lines
125	125
126	126	module_init(kprobe_init)
127	127	module_exit(kprobe_exit)
	128	+MODULE_DESCRIPTION("sample kernel module showing the use of kprobes");
128	129	MODULE_LICENSE("GPL");

samples/kprobes/kretprobe_example.c+1−0

@@ -104,4 +104,5 @@ static void __exit kretprobe_exit(void) · 103 unmodified lines
104	104
105	105	module_init(kretprobe_init)
106	106	module_exit(kretprobe_exit)
	107	+MODULE_DESCRIPTION("sample kernel module showing the use of return probes");
107	108	MODULE_LICENSE("GPL");

tools/include/linux/compiler.h+4−0

@@ -62,6 +62,10 @@ · 61 unmodified lines
62	62	#define __nocf_check __attribute__((nocf_check))
63	63	#endif
64	64
	65	+#ifndef __naked
	66	+#define __naked __attribute__((__naked__))
	67	+#endif
	68	+
65	69	/* Are two types/vars the same type (ignoring qualifiers)? */
66	70	#ifndef __same_type
67	71	# define __same_type(a, b) __builtin_types_compatible_p(typeof(a), typeof(b))

tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c+122−1

@@ -18,6 +18,7 @@ · 17 unmodified lines
18	18	#include <linux/in6.h>
19	19	#include <linux/un.h>
20	20	#include <net/sock.h>
	21	+#include <linux/namei.h>
21	22	#include "bpf_testmod.h"
22	23	#include "bpf_testmod_kfunc.h"
23	24
@@ -413,6 +414,119 @@ static struct bin_attribute bin_attr_bpf_testmod_file __ro_after_init = { · 389 unmodified lines
413	414	.write = bpf_testmod_test_write,
414	415	};
415	416
	417	+/* bpf_testmod_uprobe sysfs attribute is so far enabled for x86_64 only,
	418	+ * please see test_uretprobe_regs_change test
	419	+ */
	420	+#ifdef __x86_64__
	421	+
	422	+static int
	423	+uprobe_ret_handler(struct uprobe_consumer *self, unsigned long func,
	424	+ struct pt_regs *regs)
	425	+
	426	+{
	427	+ regs->ax = 0x12345678deadbeef;
	428	+ regs->cx = 0x87654321feebdaed;
	429	+ regs->r11 = (u64) -1;
	430	+ return true;
	431	+}
	432	+
	433	+struct testmod_uprobe {
	434	+ struct path path;
	435	+ loff_t offset;
	436	+ struct uprobe_consumer consumer;
	437	+};
	438	+
	439	+static DEFINE_MUTEX(testmod_uprobe_mutex);
	440	+
	441	+static struct testmod_uprobe uprobe = {
	442	+ .consumer.ret_handler = uprobe_ret_handler,
	443	+};
	444	+
	445	+static int testmod_register_uprobe(loff_t offset)
	446	+{
	447	+ int err = -EBUSY;
	448	+
	449	+ if (uprobe.offset)
	450	+ return -EBUSY;
	451	+
	452	+ mutex_lock(&testmod_uprobe_mutex);
	453	+
	454	+ if (uprobe.offset)
	455	+ goto out;
	456	+
	457	+ err = kern_path("/proc/self/exe", LOOKUP_FOLLOW, &uprobe.path);
	458	+ if (err)
	459	+ goto out;
	460	+
	461	+ err = uprobe_register_refctr(d_real_inode(uprobe.path.dentry),
	462	+ offset, 0, &uprobe.consumer);
	463	+ if (err)
	464	+ path_put(&uprobe.path);
	465	+ else
	466	+ uprobe.offset = offset;
	467	+
	468	+out:
	469	+ mutex_unlock(&testmod_uprobe_mutex);
	470	+ return err;
	471	+}
	472	+
	473	+static void testmod_unregister_uprobe(void)
	474	+{
	475	+ mutex_lock(&testmod_uprobe_mutex);
	476	+
	477	+ if (uprobe.offset) {
	478	+ uprobe_unregister(d_real_inode(uprobe.path.dentry),
	479	+ uprobe.offset, &uprobe.consumer);
	480	+ uprobe.offset = 0;
	481	+ }
	482	+
	483	+ mutex_unlock(&testmod_uprobe_mutex);
	484	+}
	485	+
	486	+static ssize_t
	487	+bpf_testmod_uprobe_write(struct file file, struct kobject kobj,
	488	+ struct bin_attribute *bin_attr,
	489	+ char *buf, loff_t off, size_t len)
	490	+{
	491	+ unsigned long offset = 0;
	492	+ int err = 0;
	493	+
	494	+ if (kstrtoul(buf, 0, &offset))
	495	+ return -EINVAL;
	496	+
	497	+ if (offset)
	498	+ err = testmod_register_uprobe(offset);
	499	+ else
	500	+ testmod_unregister_uprobe();
	501	+
	502	+ return err ?: strlen(buf);
	503	+}
	504	+
	505	+static struct bin_attribute bin_attr_bpf_testmod_uprobe_file __ro_after_init = {
	506	+ .attr = { .name = "bpf_testmod_uprobe", .mode = 0666, },
	507	+ .write = bpf_testmod_uprobe_write,
	508	+};
	509	+
	510	+static int register_bpf_testmod_uprobe(void)
	511	+{
	512	+ return sysfs_create_bin_file(kernel_kobj, &bin_attr_bpf_testmod_uprobe_file);
	513	+}
	514	+
	515	+static void unregister_bpf_testmod_uprobe(void)
	516	+{
	517	+ testmod_unregister_uprobe();
	518	+ sysfs_remove_bin_file(kernel_kobj, &bin_attr_bpf_testmod_uprobe_file);
	519	+}
	520	+
	521	+#else
	522	+static int register_bpf_testmod_uprobe(void)
	523	+{
	524	+ return 0;
	525	+}
	526	+
	527	+static void unregister_bpf_testmod_uprobe(void) { }
	528	+#endif
	529	+
416	530	BTF_KFUNCS_START(bpf_testmod_common_kfunc_ids)
417	531	BTF_ID_FLAGS(func, bpf_iter_testmod_seq_new, KF_ITER_NEW)
418	532	BTF_ID_FLAGS(func, bpf_iter_testmod_seq_next, KF_ITER_NEXT \| KF_RET_NULL)
@@ -983,7 +1097,13 @@ static int bpf_testmod_init(void) · 564 unmodified lines
983	1097	return -EINVAL;
984	1098	sock = NULL;
985	1099	mutex_init(&sock_lock);
986		− return sysfs_create_bin_file(kernel_kobj, &bin_attr_bpf_testmod_file);
	1100	+ ret = sysfs_create_bin_file(kernel_kobj, &bin_attr_bpf_testmod_file);
	1101	+ if (ret < 0)
	1102	+ return ret;
	1103	+ ret = register_bpf_testmod_uprobe();
	1104	+ if (ret < 0)
	1105	+ return ret;
	1106	+ return 0;
987	1107	}
988	1108
989	1109	static void bpf_testmod_exit(void)
@@ -998,6 +1118,7 @@ static void bpf_testmod_exit(void) · 8 unmodified lines
998	1118
999	1119	bpf_kfunc_close_sock();
1000	1120	sysfs_remove_bin_file(kernel_kobj, &bin_attr_bpf_testmod_file);
	1121	+ unregister_bpf_testmod_uprobe();
1001	1122	}
1002	1123
1003	1124	module_init(bpf_testmod_init);

tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c+385−0

@@ -0,0 +1,385 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+
	3	+#include <test_progs.h>
	4	+
	5	+#ifdef __x86_64__
	6	+
	7	+#include <unistd.h>
	8	+#include <asm/ptrace.h>
	9	+#include <linux/compiler.h>
	10	+#include <linux/stringify.h>
	11	+#include <sys/wait.h>
	12	+#include <sys/syscall.h>
	13	+#include <sys/prctl.h>
	14	+#include <asm/prctl.h>
	15	+#include "uprobe_syscall.skel.h"
	16	+#include "uprobe_syscall_executed.skel.h"
	17	+
	18	+__naked unsigned long uretprobe_regs_trigger(void)
	19	+{
	20	+ asm volatile (
	21	+ "movq $0xdeadbeef, %rax\n"
	22	+ "ret\n"
	23	+ );
	24	+}
	25	+
	26	+__naked void uretprobe_regs(struct pt_regs before, struct pt_regs after)
	27	+{
	28	+ asm volatile (
	29	+ "movq %r15, 0(%rdi)\n"
	30	+ "movq %r14, 8(%rdi)\n"
	31	+ "movq %r13, 16(%rdi)\n"
	32	+ "movq %r12, 24(%rdi)\n"
	33	+ "movq %rbp, 32(%rdi)\n"
	34	+ "movq %rbx, 40(%rdi)\n"
	35	+ "movq %r11, 48(%rdi)\n"
	36	+ "movq %r10, 56(%rdi)\n"
	37	+ "movq %r9, 64(%rdi)\n"
	38	+ "movq %r8, 72(%rdi)\n"
	39	+ "movq %rax, 80(%rdi)\n"
	40	+ "movq %rcx, 88(%rdi)\n"
	41	+ "movq %rdx, 96(%rdi)\n"
	42	+ "movq %rsi, 104(%rdi)\n"
	43	+ "movq %rdi, 112(%rdi)\n"
	44	+ "movq $0, 120(%rdi)\n" /* orig_rax */
	45	+ "movq $0, 128(%rdi)\n" /* rip */
	46	+ "movq $0, 136(%rdi)\n" /* cs */
	47	+ "pushf\n"
	48	+ "pop %rax\n"
	49	+ "movq %rax, 144(%rdi)\n" /* eflags */
	50	+ "movq %rsp, 152(%rdi)\n" /* rsp */
	51	+ "movq $0, 160(%rdi)\n" /* ss */
	52	+
	53	+ /* save 2nd argument */
	54	+ "pushq %rsi\n"
	55	+ "call uretprobe_regs_trigger\n"
	56	+
	57	+ /* save return value and load 2nd argument pointer to rax */
	58	+ "pushq %rax\n"
	59	+ "movq 8(%rsp), %rax\n"
	60	+
	61	+ "movq %r15, 0(%rax)\n"
	62	+ "movq %r14, 8(%rax)\n"
	63	+ "movq %r13, 16(%rax)\n"
	64	+ "movq %r12, 24(%rax)\n"
	65	+ "movq %rbp, 32(%rax)\n"
	66	+ "movq %rbx, 40(%rax)\n"
	67	+ "movq %r11, 48(%rax)\n"
	68	+ "movq %r10, 56(%rax)\n"
	69	+ "movq %r9, 64(%rax)\n"
	70	+ "movq %r8, 72(%rax)\n"
	71	+ "movq %rcx, 88(%rax)\n"
	72	+ "movq %rdx, 96(%rax)\n"
	73	+ "movq %rsi, 104(%rax)\n"
	74	+ "movq %rdi, 112(%rax)\n"
	75	+ "movq $0, 120(%rax)\n" /* orig_rax */
	76	+ "movq $0, 128(%rax)\n" /* rip */
	77	+ "movq $0, 136(%rax)\n" /* cs */
	78	+
	79	+ /* restore return value and 2nd argument */
	80	+ "pop %rax\n"
	81	+ "pop %rsi\n"
	82	+
	83	+ "movq %rax, 80(%rsi)\n"
	84	+
	85	+ "pushf\n"
	86	+ "pop %rax\n"
	87	+
	88	+ "movq %rax, 144(%rsi)\n" /* eflags */
	89	+ "movq %rsp, 152(%rsi)\n" /* rsp */
	90	+ "movq $0, 160(%rsi)\n" /* ss */
	91	+ "ret\n"
	92	+);
	93	+}
	94	+
	95	+static void test_uretprobe_regs_equal(void)
	96	+{
	97	+ struct uprobe_syscall *skel = NULL;
	98	+ struct pt_regs before = {}, after = {};
	99	+ unsigned long pb = (unsigned long ) &before;
	100	+ unsigned long pa = (unsigned long ) &after;
	101	+ unsigned long *pp;
	102	+ unsigned int i, cnt;
	103	+ int err;
	104	+
	105	+ skel = uprobe_syscall__open_and_load();
	106	+ if (!ASSERT_OK_PTR(skel, "uprobe_syscall__open_and_load"))
	107	+ goto cleanup;
	108	+
	109	+ err = uprobe_syscall__attach(skel);
	110	+ if (!ASSERT_OK(err, "uprobe_syscall__attach"))
	111	+ goto cleanup;
	112	+
	113	+ uretprobe_regs(&before, &after);
	114	+
	115	+ pp = (unsigned long *) &skel->bss->regs;
	116	+ cnt = sizeof(before)/sizeof(*pb);
	117	+
	118	+ for (i = 0; i < cnt; i++) {
	119	+ unsigned int offset = i * sizeof(unsigned long);
	120	+
	121	+ /*
	122	+ * Check register before and after uretprobe_regs_trigger call
	123	+ * that triggers the uretprobe.
	124	+ */
	125	+ switch (offset) {
	126	+ case offsetof(struct pt_regs, rax):
	127	+ ASSERT_EQ(pa[i], 0xdeadbeef, "return value");
	128	+ break;
	129	+ default:
	130	+ if (!ASSERT_EQ(pb[i], pa[i], "register before-after value check"))
	131	+ fprintf(stdout, "failed register offset %u\n", offset);
	132	+ }
	133	+
	134	+ /*
	135	+ * Check register seen from bpf program and register after
	136	+ * uretprobe_regs_trigger call
	137	+ */
	138	+ switch (offset) {
	139	+ /*
	140	+ * These values will be different (not set in uretprobe_regs),
	141	+ * we don't care.
	142	+ */
	143	+ case offsetof(struct pt_regs, orig_rax):
	144	+ case offsetof(struct pt_regs, rip):
	145	+ case offsetof(struct pt_regs, cs):
	146	+ case offsetof(struct pt_regs, rsp):
	147	+ case offsetof(struct pt_regs, ss):
	148	+ break;
	149	+ default:
	150	+ if (!ASSERT_EQ(pp[i], pa[i], "register prog-after value check"))
	151	+ fprintf(stdout, "failed register offset %u\n", offset);
	152	+ }
	153	+ }
	154	+
	155	+cleanup:
	156	+ uprobe_syscall__destroy(skel);
	157	+}
	158	+
	159	+#define BPF_TESTMOD_UPROBE_TEST_FILE "/sys/kernel/bpf_testmod_uprobe"
	160	+
	161	+static int write_bpf_testmod_uprobe(unsigned long offset)
	162	+{
	163	+ size_t n, ret;
	164	+ char buf[30];
	165	+ int fd;
	166	+
	167	+ n = sprintf(buf, "%lu", offset);
	168	+
	169	+ fd = open(BPF_TESTMOD_UPROBE_TEST_FILE, O_WRONLY);
	170	+ if (fd < 0)
	171	+ return -errno;
	172	+
	173	+ ret = write(fd, buf, n);
	174	+ close(fd);
	175	+ return ret != n ? (int) ret : 0;
	176	+}
	177	+
	178	+static void test_uretprobe_regs_change(void)
	179	+{
	180	+ struct pt_regs before = {}, after = {};
	181	+ unsigned long pb = (unsigned long ) &before;
	182	+ unsigned long pa = (unsigned long ) &after;
	183	+ unsigned long cnt = sizeof(before)/sizeof(*pb);
	184	+ unsigned int i, err, offset;
	185	+
	186	+ offset = get_uprobe_offset(uretprobe_regs_trigger);
	187	+
	188	+ err = write_bpf_testmod_uprobe(offset);
	189	+ if (!ASSERT_OK(err, "register_uprobe"))
	190	+ return;
	191	+
	192	+ uretprobe_regs(&before, &after);
	193	+
	194	+ err = write_bpf_testmod_uprobe(0);
	195	+ if (!ASSERT_OK(err, "unregister_uprobe"))
	196	+ return;
	197	+
	198	+ for (i = 0; i < cnt; i++) {
	199	+ unsigned int offset = i * sizeof(unsigned long);
	200	+
	201	+ switch (offset) {
	202	+ case offsetof(struct pt_regs, rax):
	203	+ ASSERT_EQ(pa[i], 0x12345678deadbeef, "rax");
	204	+ break;
	205	+ case offsetof(struct pt_regs, rcx):
	206	+ ASSERT_EQ(pa[i], 0x87654321feebdaed, "rcx");
	207	+ break;
	208	+ case offsetof(struct pt_regs, r11):
	209	+ ASSERT_EQ(pa[i], (__u64) -1, "r11");
	210	+ break;
	211	+ default:
	212	+ if (!ASSERT_EQ(pa[i], pb[i], "register before-after value check"))
	213	+ fprintf(stdout, "failed register offset %u\n", offset);
	214	+ }
	215	+ }
	216	+}
	217	+
	218	+#ifndef __NR_uretprobe
	219	+#define __NR_uretprobe 467
	220	+#endif
	221	+
	222	+__naked unsigned long uretprobe_syscall_call_1(void)
	223	+{
	224	+ /*
	225	+ * Pretend we are uretprobe trampoline to trigger the return
	226	+ * probe invocation in order to verify we get SIGILL.
	227	+ */
	228	+ asm volatile (
	229	+ "pushq %rax\n"
	230	+ "pushq %rcx\n"
	231	+ "pushq %r11\n"
	232	+ "movq $" __stringify(__NR_uretprobe) ", %rax\n"
	233	+ "syscall\n"
	234	+ "popq %r11\n"
	235	+ "popq %rcx\n"
	236	+ "retq\n"
	237	+ );
	238	+}
	239	+
	240	+__naked unsigned long uretprobe_syscall_call(void)
	241	+{
	242	+ asm volatile (
	243	+ "call uretprobe_syscall_call_1\n"
	244	+ "retq\n"
	245	+ );
	246	+}
	247	+
	248	+static void test_uretprobe_syscall_call(void)
	249	+{
	250	+ LIBBPF_OPTS(bpf_uprobe_multi_opts, opts,
	251	+ .retprobe = true,
	252	+ );
	253	+ struct uprobe_syscall_executed *skel;
	254	+ int pid, status, err, go[2], c;
	255	+
	256	+ if (ASSERT_OK(pipe(go), "pipe"))
	257	+ return;
	258	+
	259	+ skel = uprobe_syscall_executed__open_and_load();
	260	+ if (!ASSERT_OK_PTR(skel, "uprobe_syscall_executed__open_and_load"))
	261	+ goto cleanup;
	262	+
	263	+ pid = fork();
	264	+ if (!ASSERT_GE(pid, 0, "fork"))
	265	+ goto cleanup;
	266	+
	267	+ /* child */
	268	+ if (pid == 0) {
	269	+ close(go[1]);
	270	+
	271	+ /* wait for parent's kick */
	272	+ err = read(go[0], &c, 1);
	273	+ if (err != 1)
	274	+ exit(-1);
	275	+
	276	+ uretprobe_syscall_call();
	277	+ _exit(0);
	278	+ }
	279	+
	280	+ skel->links.test = bpf_program__attach_uprobe_multi(skel->progs.test, pid,
	281	+ "/proc/self/exe",
	282	+ "uretprobe_syscall_call", &opts);
	283	+ if (!ASSERT_OK_PTR(skel->links.test, "bpf_program__attach_uprobe_multi"))
	284	+ goto cleanup;
	285	+
	286	+ /* kick the child */
	287	+ write(go[1], &c, 1);
	288	+ err = waitpid(pid, &status, 0);
	289	+ ASSERT_EQ(err, pid, "waitpid");
	290	+
	291	+ /* verify the child got killed with SIGILL */
	292	+ ASSERT_EQ(WIFSIGNALED(status), 1, "WIFSIGNALED");
	293	+ ASSERT_EQ(WTERMSIG(status), SIGILL, "WTERMSIG");
	294	+
	295	+ /* verify the uretprobe program wasn't called */
	296	+ ASSERT_EQ(skel->bss->executed, 0, "executed");
	297	+
	298	+cleanup:
	299	+ uprobe_syscall_executed__destroy(skel);
	300	+ close(go[1]);
	301	+ close(go[0]);
	302	+}
	303	+
	304	+/*
	305	+ * Borrowed from tools/testing/selftests/x86/test_shadow_stack.c.
	306	+ *
	307	+ * For use in inline enablement of shadow stack.
	308	+ *
	309	+ * The program can't return from the point where shadow stack gets enabled
	310	+ * because there will be no address on the shadow stack. So it can't use
	311	+ * syscall() for enablement, since it is a function.
	312	+ *
	313	+ * Based on code from nolibc.h. Keep a copy here because this can't pull
	314	+ * in all of nolibc.h.
	315	+ */
	316	+#define ARCH_PRCTL(arg1, arg2) \
	317	+({ \
	318	+ long _ret; \
	319	+ register long _num asm("eax") = __NR_arch_prctl; \
	320	+ register long _arg1 asm("rdi") = (long)(arg1); \
	321	+ register long _arg2 asm("rsi") = (long)(arg2); \
	322	+ \
	323	+ asm volatile ( \
	324	+ "syscall\n" \
	325	+ : "=a"(_ret) \
	326	+ : "r"(_arg1), "r"(_arg2), \
	327	+ "0"(_num) \
	328	+ : "rcx", "r11", "memory", "cc" \
	329	+ ); \
	330	+ _ret; \
	331	+})
	332	+
	333	+#ifndef ARCH_SHSTK_ENABLE
	334	+#define ARCH_SHSTK_ENABLE 0x5001
	335	+#define ARCH_SHSTK_DISABLE 0x5002
	336	+#define ARCH_SHSTK_SHSTK (1ULL << 0)
	337	+#endif
	338	+
	339	+static void test_uretprobe_shadow_stack(void)
	340	+{
	341	+ if (ARCH_PRCTL(ARCH_SHSTK_ENABLE, ARCH_SHSTK_SHSTK)) {
	342	+ test__skip();
	343	+ return;
	344	+ }
	345	+
	346	+ /* Run all of the uretprobe tests. */
	347	+ test_uretprobe_regs_equal();
	348	+ test_uretprobe_regs_change();
	349	+ test_uretprobe_syscall_call();
	350	+
	351	+ ARCH_PRCTL(ARCH_SHSTK_DISABLE, ARCH_SHSTK_SHSTK);
	352	+}
	353	+#else
	354	+static void test_uretprobe_regs_equal(void)
	355	+{
	356	+ test__skip();
	357	+}
	358	+
	359	+static void test_uretprobe_regs_change(void)
	360	+{
	361	+ test__skip();
	362	+}
	363	+
	364	+static void test_uretprobe_syscall_call(void)
	365	+{
	366	+ test__skip();
	367	+}
	368	+
	369	+static void test_uretprobe_shadow_stack(void)
	370	+{
	371	+ test__skip();
	372	+}
	373	+#endif
	374	+
	375	+void test_uprobe_syscall(void)
	376	+{
	377	+ if (test__start_subtest("uretprobe_regs_equal"))
	378	+ test_uretprobe_regs_equal();
	379	+ if (test__start_subtest("uretprobe_regs_change"))
	380	+ test_uretprobe_regs_change();
	381	+ if (test__start_subtest("uretprobe_syscall_call"))
	382	+ test_uretprobe_syscall_call();
	383	+ if (test__start_subtest("uretprobe_shadow_stack"))
	384	+ test_uretprobe_shadow_stack();
	385	+}

tools/testing/selftests/bpf/prog_tests/uretprobe_stack.c+186−0

@@ -0,0 +1,186 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
	3	+
	4	+#include <test_progs.h>
	5	+#include "uretprobe_stack.skel.h"
	6	+#include "../sdt.h"
	7	+
	8	+/* We set up target_1() -> target_2() -> target_3() -> target_4() -> USDT()
	9	+ * call chain, each being traced by our BPF program. On entry or return from
	10	+ * each target_*() we are capturing user stack trace and recording it in
	11	+ * global variable, so that user space part of the test can validate it.
	12	+ *
	13	+ * Note, we put each target function into a custom section to get those
	14	+ * __start_XXX/__stop_XXX symbols, generated by linker for us, which allow us
	15	+ * to know address range of those functions
	16	+ */
	17	+__attribute__((section("uprobe__target_4")))
	18	+__weak int target_4(void)
	19	+{
	20	+ STAP_PROBE1(uretprobe_stack, target, 42);
	21	+ return 42;
	22	+}
	23	+
	24	+extern const void *__start_uprobe__target_4;
	25	+extern const void *__stop_uprobe__target_4;
	26	+
	27	+__attribute__((section("uprobe__target_3")))
	28	+__weak int target_3(void)
	29	+{
	30	+ return target_4();
	31	+}
	32	+
	33	+extern const void *__start_uprobe__target_3;
	34	+extern const void *__stop_uprobe__target_3;
	35	+
	36	+__attribute__((section("uprobe__target_2")))
	37	+__weak int target_2(void)
	38	+{
	39	+ return target_3();
	40	+}
	41	+
	42	+extern const void *__start_uprobe__target_2;
	43	+extern const void *__stop_uprobe__target_2;
	44	+
	45	+__attribute__((section("uprobe__target_1")))
	46	+__weak int target_1(int depth)
	47	+{
	48	+ if (depth < 1)
	49	+ return 1 + target_1(depth + 1);
	50	+ else
	51	+ return target_2();
	52	+}
	53	+
	54	+extern const void *__start_uprobe__target_1;
	55	+extern const void *__stop_uprobe__target_1;
	56	+
	57	+extern const void *__start_uretprobe_stack_sec;
	58	+extern const void *__stop_uretprobe_stack_sec;
	59	+
	60	+struct range {
	61	+ long start;
	62	+ long stop;
	63	+};
	64	+
	65	+static struct range targets[] = {
	66	+ {}, /* we want target_1 to map to target[1], so need 1-based indexing */
	67	+ { (long)&__start_uprobe__target_1, (long)&__stop_uprobe__target_1 },
	68	+ { (long)&__start_uprobe__target_2, (long)&__stop_uprobe__target_2 },
	69	+ { (long)&__start_uprobe__target_3, (long)&__stop_uprobe__target_3 },
	70	+ { (long)&__start_uprobe__target_4, (long)&__stop_uprobe__target_4 },
	71	+};
	72	+
	73	+static struct range caller = {
	74	+ (long)&__start_uretprobe_stack_sec,
	75	+ (long)&__stop_uretprobe_stack_sec,
	76	+};
	77	+
	78	+static void validate_stack(__u64 *ips, int stack_len, int cnt, ...)
	79	+{
	80	+ int i, j;
	81	+ va_list args;
	82	+
	83	+ if (!ASSERT_GT(stack_len, 0, "stack_len"))
	84	+ return;
	85	+
	86	+ stack_len /= 8;
	87	+
	88	+ /* check if we have enough entries to satisfy test expectations */
	89	+ if (!ASSERT_GE(stack_len, cnt, "stack_len2"))
	90	+ return;
	91	+
	92	+ if (env.verbosity >= VERBOSE_NORMAL) {
	93	+ printf("caller: %#lx - %#lx\n", caller.start, caller.stop);
	94	+ for (i = 1; i < ARRAY_SIZE(targets); i++)
	95	+ printf("target_%d: %#lx - %#lx\n", i, targets[i].start, targets[i].stop);
	96	+ for (i = 0; i < stack_len; i++) {
	97	+ for (j = 1; j < ARRAY_SIZE(targets); j++) {
	98	+ if (ips[i] >= targets[j].start && ips[i] < targets[j].stop)
	99	+ break;
	100	+ }
	101	+ if (j < ARRAY_SIZE(targets)) { /* found target match */
	102	+ printf("ENTRY #%d: %#lx (in target_%d)\n", i, (long)ips[i], j);
	103	+ } else if (ips[i] >= caller.start && ips[i] < caller.stop) {
	104	+ printf("ENTRY #%d: %#lx (in caller)\n", i, (long)ips[i]);
	105	+ } else {
	106	+ printf("ENTRY #%d: %#lx\n", i, (long)ips[i]);
	107	+ }
	108	+ }
	109	+ }
	110	+
	111	+ va_start(args, cnt);
	112	+
	113	+ for (i = cnt - 1; i >= 0; i--) {
	114	+ /* most recent entry is the deepest target function */
	115	+ const struct range t = va_arg(args, const struct range );
	116	+
	117	+ ASSERT_GE(ips[i], t->start, "addr_start");
	118	+ ASSERT_LT(ips[i], t->stop, "addr_stop");
	119	+ }
	120	+
	121	+ va_end(args);
	122	+}
	123	+
	124	+/* __weak prevents inlining */
	125	+__attribute__((section("uretprobe_stack_sec")))
	126	+__weak void test_uretprobe_stack(void)
	127	+{
	128	+ LIBBPF_OPTS(bpf_uprobe_opts, uprobe_opts);
	129	+ struct uretprobe_stack *skel;
	130	+ int err;
	131	+
	132	+ skel = uretprobe_stack__open_and_load();
	133	+ if (!ASSERT_OK_PTR(skel, "skel_open"))
	134	+ return;
	135	+
	136	+ err = uretprobe_stack__attach(skel);
	137	+ if (!ASSERT_OK(err, "skel_attach"))
	138	+ goto cleanup;
	139	+
	140	+ /* trigger */
	141	+ ASSERT_EQ(target_1(0), 42 + 1, "trigger_return");
	142	+
	143	+ /*
	144	+ * Stacks captured on ENTRY uprobes
	145	+ */
	146	+
	147	+ /* (uprobe 1) target_1 in stack trace*/
	148	+ validate_stack(skel->bss->entry_stack1, skel->bss->entry1_len,
	149	+ 2, &caller, &targets[1]);
	150	+ /* (uprobe 1, recursed) */
	151	+ validate_stack(skel->bss->entry_stack1_recur, skel->bss->entry1_recur_len,
	152	+ 3, &caller, &targets[1], &targets[1]);
	153	+ /* (uprobe 2) caller -> target_1 -> target_1 -> target_2 */
	154	+ validate_stack(skel->bss->entry_stack2, skel->bss->entry2_len,
	155	+ 4, &caller, &targets[1], &targets[1], &targets[2]);
	156	+ /* (uprobe 3) */
	157	+ validate_stack(skel->bss->entry_stack3, skel->bss->entry3_len,
	158	+ 5, &caller, &targets[1], &targets[1], &targets[2], &targets[3]);
	159	+ /* (uprobe 4) caller -> target_1 -> target_1 -> target_2 -> target_3 -> target_4 */
	160	+ validate_stack(skel->bss->entry_stack4, skel->bss->entry4_len,
	161	+ 6, &caller, &targets[1], &targets[1], &targets[2], &targets[3], &targets[4]);
	162	+
	163	+ /* (USDT): full caller -> target_1 -> target_1 -> target_2 (uretprobed)
	164	+ * -> target_3 -> target_4 (uretprobes) chain
	165	+ */
	166	+ validate_stack(skel->bss->usdt_stack, skel->bss->usdt_len,
	167	+ 6, &caller, &targets[1], &targets[1], &targets[2], &targets[3], &targets[4]);
	168	+
	169	+ /*
	170	+ * Now stacks captured on the way out in EXIT uprobes
	171	+ */
	172	+
	173	+ /* (uretprobe 4) everything up to target_4, but excluding it */
	174	+ validate_stack(skel->bss->exit_stack4, skel->bss->exit4_len,
	175	+ 5, &caller, &targets[1], &targets[1], &targets[2], &targets[3]);
	176	+ /* we didn't install uretprobes on target_2 and target_3 */
	177	+ /* (uretprobe 1, recur) first target_1 call only */
	178	+ validate_stack(skel->bss->exit_stack1_recur, skel->bss->exit1_recur_len,
	179	+ 2, &caller, &targets[1]);
	180	+ /* (uretprobe 1) just a caller in the stack trace */
	181	+ validate_stack(skel->bss->exit_stack1, skel->bss->exit1_len,
	182	+ 1, &caller);
	183	+
	184	+cleanup:
	185	+ uretprobe_stack__destroy(skel);
	186	+}

tools/testing/selftests/bpf/progs/uprobe_syscall.c+15−0

@@ -0,0 +1,15 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+#include "vmlinux.h"
	3	+#include <bpf/bpf_helpers.h>
	4	+#include <string.h>
	5	+
	6	+struct pt_regs regs;
	7	+
	8	+char _license[] SEC("license") = "GPL";
	9	+
	10	+SEC("uretprobe//proc/self/exe:uretprobe_regs_trigger")
	11	+int uretprobe(struct pt_regs *ctx)
	12	+{
	13	+ __builtin_memcpy(&regs, ctx, sizeof(regs));
	14	+ return 0;
	15	+}

tools/testing/selftests/bpf/progs/uprobe_syscall_executed.c+17−0

@@ -0,0 +1,17 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+#include "vmlinux.h"
	3	+#include <bpf/bpf_helpers.h>
	4	+#include <string.h>
	5	+
	6	+struct pt_regs regs;
	7	+
	8	+char _license[] SEC("license") = "GPL";
	9	+
	10	+int executed = 0;
	11	+
	12	+SEC("uretprobe.multi")
	13	+int test(struct pt_regs *regs)
	14	+{
	15	+ executed = 1;
	16	+ return 0;
	17	+}

tools/testing/selftests/bpf/progs/uretprobe_stack.c+96−0

@@ -0,0 +1,96 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
	3	+#include <vmlinux.h>
	4	+#include <bpf/bpf_helpers.h>
	5	+#include <bpf/bpf_tracing.h>
	6	+#include <bpf/usdt.bpf.h>
	7	+
	8	+char _license[] SEC("license") = "GPL";
	9	+
	10	+__u64 entry_stack1[32], exit_stack1[32];
	11	+__u64 entry_stack1_recur[32], exit_stack1_recur[32];
	12	+__u64 entry_stack2[32];
	13	+__u64 entry_stack3[32];
	14	+__u64 entry_stack4[32], exit_stack4[32];
	15	+__u64 usdt_stack[32];
	16	+
	17	+int entry1_len, exit1_len;
	18	+int entry1_recur_len, exit1_recur_len;
	19	+int entry2_len, exit2_len;
	20	+int entry3_len, exit3_len;
	21	+int entry4_len, exit4_len;
	22	+int usdt_len;
	23	+
	24	+#define SZ sizeof(usdt_stack)
	25	+
	26	+SEC("uprobe//proc/self/exe:target_1")
	27	+int BPF_UPROBE(uprobe_1)
	28	+{
	29	+ /* target_1 is recursive wit depth of 2, so we capture two separate
	30	+ * stack traces, depending on which occurence it is
	31	+ */
	32	+ static bool recur = false;
	33	+
	34	+ if (!recur)
	35	+ entry1_len = bpf_get_stack(ctx, &entry_stack1, SZ, BPF_F_USER_STACK);
	36	+ else
	37	+ entry1_recur_len = bpf_get_stack(ctx, &entry_stack1_recur, SZ, BPF_F_USER_STACK);
	38	+
	39	+ recur = true;
	40	+ return 0;
	41	+}
	42	+
	43	+SEC("uretprobe//proc/self/exe:target_1")
	44	+int BPF_URETPROBE(uretprobe_1)
	45	+{
	46	+ /* see above, target_1 is recursive */
	47	+ static bool recur = false;
	48	+
	49	+ /* NOTE: order of returns is reversed to order of entries */
	50	+ if (!recur)
	51	+ exit1_recur_len = bpf_get_stack(ctx, &exit_stack1_recur, SZ, BPF_F_USER_STACK);
	52	+ else
	53	+ exit1_len = bpf_get_stack(ctx, &exit_stack1, SZ, BPF_F_USER_STACK);
	54	+
	55	+ recur = true;
	56	+ return 0;
	57	+}
	58	+
	59	+SEC("uprobe//proc/self/exe:target_2")
	60	+int BPF_UPROBE(uprobe_2)
	61	+{
	62	+ entry2_len = bpf_get_stack(ctx, &entry_stack2, SZ, BPF_F_USER_STACK);
	63	+ return 0;
	64	+}
	65	+
	66	+/* no uretprobe for target_2 */
	67	+
	68	+SEC("uprobe//proc/self/exe:target_3")
	69	+int BPF_UPROBE(uprobe_3)
	70	+{
	71	+ entry3_len = bpf_get_stack(ctx, &entry_stack3, SZ, BPF_F_USER_STACK);
	72	+ return 0;
	73	+}
	74	+
	75	+/* no uretprobe for target_3 */
	76	+
	77	+SEC("uprobe//proc/self/exe:target_4")
	78	+int BPF_UPROBE(uprobe_4)
	79	+{
	80	+ entry4_len = bpf_get_stack(ctx, &entry_stack4, SZ, BPF_F_USER_STACK);
	81	+ return 0;
	82	+}
	83	+
	84	+SEC("uretprobe//proc/self/exe:target_4")
	85	+int BPF_URETPROBE(uretprobe_4)
	86	+{
	87	+ exit4_len = bpf_get_stack(ctx, &exit_stack4, SZ, BPF_F_USER_STACK);
	88	+ return 0;
	89	+}
	90	+
	91	+SEC("usdt//proc/self/exe:uretprobe_stack:target")
	92	+int BPF_USDT(usdt_probe)
	93	+{
	94	+ usdt_len = bpf_get_stack(ctx, &usdt_stack, SZ, BPF_F_USER_STACK);
	95	+ return 0;
	96	+}

tools/testing/selftests/x86/test_shadow_stack.c+145−0

@@ -34,6 +34,7 @@ · 33 unmodified lines
34	34	#include <sys/ptrace.h>
35	35	#include <sys/signal.h>
36	36	#include <linux/elf.h>
	37	+#include <linux/perf_event.h>
37	38
38	39	/*
39	40	* Define the ABI defines if needed, so people can run the tests
@@ -734,6 +735,144 @@ int test_32bit(void) · 694 unmodified lines
734	735	return !segv_triggered;
735	736	}
736	737
	738	+static int parse_uint_from_file(const char file, const char fmt)
	739	+{
	740	+ int err, ret;
	741	+ FILE *f;
	742	+
	743	+ f = fopen(file, "re");
	744	+ if (!f) {
	745	+ err = -errno;
	746	+ printf("failed to open '%s': %d\n", file, err);
	747	+ return err;
	748	+ }
	749	+ err = fscanf(f, fmt, &ret);
	750	+ if (err != 1) {
	751	+ err = err == EOF ? -EIO : -errno;
	752	+ printf("failed to parse '%s': %d\n", file, err);
	753	+ fclose(f);
	754	+ return err;
	755	+ }
	756	+ fclose(f);
	757	+ return ret;
	758	+}
	759	+
	760	+static int determine_uprobe_perf_type(void)
	761	+{
	762	+ const char *file = "/sys/bus/event_source/devices/uprobe/type";
	763	+
	764	+ return parse_uint_from_file(file, "%d\n");
	765	+}
	766	+
	767	+static int determine_uprobe_retprobe_bit(void)
	768	+{
	769	+ const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe";
	770	+
	771	+ return parse_uint_from_file(file, "config:%d\n");
	772	+}
	773	+
	774	+static ssize_t get_uprobe_offset(const void *addr)
	775	+{
	776	+ size_t start, end, base;
	777	+ char buf[256];
	778	+ bool found = false;
	779	+ FILE *f;
	780	+
	781	+ f = fopen("/proc/self/maps", "r");
	782	+ if (!f)
	783	+ return -errno;
	784	+
	785	+ while (fscanf(f, "%zx-%zx %s %zx %*[^\n]\n", &start, &end, buf, &base) == 4) {
	786	+ if (buf[2] == 'x' && (uintptr_t)addr >= start && (uintptr_t)addr < end) {
	787	+ found = true;
	788	+ break;
	789	+ }
	790	+ }
	791	+
	792	+ fclose(f);
	793	+
	794	+ if (!found)
	795	+ return -ESRCH;
	796	+
	797	+ return (uintptr_t)addr - start + base;
	798	+}
	799	+
	800	+static __attribute__((noinline)) void uretprobe_trigger(void)
	801	+{
	802	+ asm volatile ("");
	803	+}
	804	+
	805	+/*
	806	+ * This test setups return uprobe, which is sensitive to shadow stack
	807	+ * (crashes without extra fix). After executing the uretprobe we fail
	808	+ * the test if we receive SIGSEGV, no crash means we're good.
	809	+ *
	810	+ * Helper functions above borrowed from bpf selftests.
	811	+ */
	812	+static int test_uretprobe(void)
	813	+{
	814	+ const size_t attr_sz = sizeof(struct perf_event_attr);
	815	+ const char *file = "/proc/self/exe";
	816	+ int bit, fd = 0, type, err = 1;
	817	+ struct perf_event_attr attr;
	818	+ struct sigaction sa = {};
	819	+ ssize_t offset;
	820	+
	821	+ type = determine_uprobe_perf_type();
	822	+ if (type < 0) {
	823	+ if (type == -ENOENT)
	824	+ printf("[SKIP]\tUretprobe test, uprobes are not available\n");
	825	+ return 0;
	826	+ }
	827	+
	828	+ offset = get_uprobe_offset(uretprobe_trigger);
	829	+ if (offset < 0)
	830	+ return 1;
	831	+
	832	+ bit = determine_uprobe_retprobe_bit();
	833	+ if (bit < 0)
	834	+ return 1;
	835	+
	836	+ sa.sa_sigaction = segv_gp_handler;
	837	+ sa.sa_flags = SA_SIGINFO;
	838	+ if (sigaction(SIGSEGV, &sa, NULL))
	839	+ return 1;
	840	+
	841	+ /* Setup return uprobe through perf event interface. */
	842	+ memset(&attr, 0, attr_sz);
	843	+ attr.size = attr_sz;
	844	+ attr.type = type;
	845	+ attr.config = 1 << bit;
	846	+ attr.config1 = (__u64) (unsigned long) file;
	847	+ attr.config2 = offset;
	848	+
	849	+ fd = syscall(__NR_perf_event_open, &attr, 0 /* pid /, -1 / cpu */,
	850	+ -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
	851	+ if (fd < 0)
	852	+ goto out;
	853	+
	854	+ if (sigsetjmp(jmp_buffer, 1))
	855	+ goto out;
	856	+
	857	+ ARCH_PRCTL(ARCH_SHSTK_ENABLE, ARCH_SHSTK_SHSTK);
	858	+
	859	+ /*
	860	+ * This either segfaults and goes through sigsetjmp above
	861	+ * or succeeds and we're good.
	862	+ */
	863	+ uretprobe_trigger();
	864	+
	865	+ printf("[OK]\tUretprobe test\n");
	866	+ err = 0;
	867	+
	868	+out:
	869	+ ARCH_PRCTL(ARCH_SHSTK_DISABLE, ARCH_SHSTK_SHSTK);
	870	+ signal(SIGSEGV, SIG_DFL);
	871	+ if (fd)
	872	+ close(fd);
	873	+ return err;
	874	+}
	875	+
737	876	void segv_handler_ptrace(int signum, siginfo_t si, void uc)
738	877	{
739	878	/* The SSP adjustment caused a segfault. */
@@ -926,6 +1065,12 @@ int main(int argc, char *argv[]) · 186 unmodified lines
926	1065	goto out;
927	1066	}
928	1067
	1068	+ if (test_uretprobe()) {
	1069	+ ret = 1;
	1070	+ printf("[FAIL]\turetprobe test\n");
	1071	+ goto out;
	1072	+ }
	1073	+
929	1074	return ret;
930	1075
931	1076	out: