History

Merge tag 'probes-v6.11' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace

Pull probes updates from Masami Hiramatsu: "Uprobes: - x86/shstk: Make return uprobe work with shadow stack - Add uretprobe syscall which speeds up the uretprobe 10-30% faster. This syscall is automatically used from user-space trampolines which are generated by the uretprobe. If this syscall is used by normal user program, it will cause SIGILL. Note that this is currently only implemented on x86_64. (This also has two fixes for adjusting the syscall number to avoid conflict with new *attrat syscalls.) - uprobes/perf: fix user stack traces in the presence of pending uretprobe. This corrects the uretprobe's trampoline address in the stacktrace with correct return address - selftests/x86: Add a return uprobe with shadow stack test - selftests/bpf: Add uretprobe syscall related tests. - test case for register integrity check - test case with register changing case - test case for uretprobe syscall without uprobes (expected to fail) - test case for uretprobe with shadow stack - selftests/bpf: add test validating uprobe/uretprobe stack traces - MAINTAINERS: Add uprobes entry. This does not specify the tree but to clarify who maintains and reviews the uprobes Kprobes: - tracing/kprobes: Test case cleanups. Replace redundant WARN_ON_ONCE() + pr_warn() with WARN_ONCE() and remove unnecessary code from selftest - tracing/kprobes: Add symbol counting check when module loads. This checks the uniqueness of the probed symbol on modules. The same check has already done for kernel symbols (This also has a fix for build error with CONFIG_MODULES=n) Cleanup: - Add MODULE_DESCRIPTION() macros for fprobe and kprobe examples" * tag 'probes-v6.11' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace: MAINTAINERS: Add uprobes entry selftests/bpf: Change uretprobe syscall number in uprobe_syscall test uprobe: Change uretprobe syscall scope and number tracing/kprobes: Fix build error when find_module() is not available tracing/kprobes: Add symbol counting check when module loads selftests/bpf: add test validating uprobe/uretprobe stack traces perf,uprobes: fix user stack traces in the presence of pending uretprobes tracing/kprobe: Remove cleanup code unrelated to selftest tracing/kprobe: Integrate test warnings into WARN_ONCE selftests/bpf: Add uretprobe shadow stack test selftests/bpf: Add uretprobe syscall call from user space test selftests/bpf: Add uretprobe syscall test for regs changes selftests/bpf: Add uretprobe syscall test for regs integrity selftests/x86: Add return uprobe shadow stack test uprobe: Add uretprobe syscall to speed up return probe uprobe: Wire up uretprobe system call x86/shstk: Make return uprobe work with shadow stack samples: kprobes: add missing MODULE_DESCRIPTION() macros fprobe: add missing MODULE_DESCRIPTION() macro

Linus Torvalds committed 2 years ago

Commit 91bd0082 parents cb273eb c26b1b823 files · +1320 −92Browse files

Unified · Split

23 files
  1. MAINTAINERS +13 −0
  2. arch/x86/entry/syscalls/syscall_64.tbl +1 −0
  3. arch/x86/include/asm/shstk.h +4 −0
  4. arch/x86/kernel/shstk.c +16 −0
  5. arch/x86/kernel/uprobes.c +123 −1
  6. include/linux/syscalls.h +2 −0
  7. include/linux/uprobes.h +3 −0
  8. include/uapi/asm-generic/unistd.h +4 −1
  9. kernel/events/callchain.c +42 −1
  10. kernel/events/uprobes.c +26 −7
  11. kernel/sys_ni.c +2 −0
  12. kernel/trace/trace_kprobe.c +111 −81
  13. samples/fprobe/fprobe_example.c +1 −0
  14. samples/kprobes/kprobe_example.c +1 −0
  15. samples/kprobes/kretprobe_example.c +1 −0
  16. tools/include/linux/compiler.h +4 −0
  17. tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c +122 −1
  18. tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c +385 −0
  19. tools/testing/selftests/bpf/prog_tests/uretprobe_stack.c +186 −0
  20. tools/testing/selftests/bpf/progs/uprobe_syscall.c +15 −0
  21. tools/testing/selftests/bpf/progs/uprobe_syscall_executed.c +17 −0
  22. tools/testing/selftests/bpf/progs/uretprobe_stack.c +96 −0
  23. tools/testing/selftests/x86/test_shadow_stack.c +145 −0
MAINTAINERS+13−0
@@ -23367,6 +23367,19 @@ F: drivers/mtd/ubi/ · 23366 unmodified lines
23367F: include/linux/mtd/ubi.h23367F: include/linux/mtd/ubi.h
23368F: include/uapi/mtd/ubi-user.h23368F: include/uapi/mtd/ubi-user.h
23369 23369
23370UPROBES
23371M: Masami Hiramatsu <mhiramat@kernel.org>
23372M: Oleg Nesterov <oleg@redhat.com>
23373M: Peter Zijlstra <peterz@infradead.org>
23374L: linux-kernel@vger.kernel.org
23375L: linux-trace-kernel@vger.kernel.org
23376S: Maintained
23377F: arch/*/include/asm/uprobes.h
23378F: arch/*/kernel/probes/uprobes.c
23379F: arch/*/kernel/uprobes.c
23380F: include/linux/uprobes.h
23381F: kernel/events/uprobes.c
23382
23370USB "USBNET" DRIVER FRAMEWORK23383USB "USBNET" DRIVER FRAMEWORK
23371M: Oliver Neukum <oneukum@suse.com>23384M: Oliver Neukum <oneukum@suse.com>
23372L: netdev@vger.kernel.org23385L: netdev@vger.kernel.org
arch/x86/entry/syscalls/syscall_64.tbl+1−0
@@ -385,6 +385,7 @@ · 384 unmodified lines
385460 common lsm_set_self_attr sys_lsm_set_self_attr385460 common lsm_set_self_attr sys_lsm_set_self_attr
386461 common lsm_list_modules sys_lsm_list_modules386461 common lsm_list_modules sys_lsm_list_modules
387462 common mseal sys_mseal387462 common mseal sys_mseal
388467 common uretprobe sys_uretprobe
388 389
389#390#
390# Due to a historical design error, certain syscalls are numbered differently391# Due to a historical design error, certain syscalls are numbered differently
arch/x86/include/asm/shstk.h+4−0
@@ -21,6 +21,8 @@ unsigned long shstk_alloc_thread_stack(struct task_struct *p, unsigned long clon · 20 unmodified lines
21void shstk_free(struct task_struct *p);21void shstk_free(struct task_struct *p);
22int setup_signal_shadow_stack(struct ksignal *ksig);22int setup_signal_shadow_stack(struct ksignal *ksig);
23int restore_signal_shadow_stack(void);23int restore_signal_shadow_stack(void);
24int shstk_update_last_frame(unsigned long val);
25bool shstk_is_enabled(void);
24#else26#else
25static inline long shstk_prctl(struct task_struct *task, int option,27static inline long shstk_prctl(struct task_struct *task, int option,
26 unsigned long arg2) { return -EINVAL; }28 unsigned long arg2) { return -EINVAL; }
@@ -31,6 +33,8 @@ static inline unsigned long shstk_alloc_thread_stack(struct task_struct *p, · 4 unmodified lines
31static inline void shstk_free(struct task_struct *p) {}33static inline void shstk_free(struct task_struct *p) {}
32static inline int setup_signal_shadow_stack(struct ksignal *ksig) { return 0; }34static inline int setup_signal_shadow_stack(struct ksignal *ksig) { return 0; }
33static inline int restore_signal_shadow_stack(void) { return 0; }35static inline int restore_signal_shadow_stack(void) { return 0; }
36static inline int shstk_update_last_frame(unsigned long val) { return 0; }
37static inline bool shstk_is_enabled(void) { return false; }
34#endif /* CONFIG_X86_USER_SHADOW_STACK */38#endif /* CONFIG_X86_USER_SHADOW_STACK */
35 39
36#endif /* __ASSEMBLY__ */40#endif /* __ASSEMBLY__ */
arch/x86/kernel/shstk.c+16−0
@@ -577,3 +577,19 @@ long shstk_prctl(struct task_struct *task, int option, unsigned long arg2) · 576 unmodified lines
577 return wrss_control(true);577 return wrss_control(true);
578 return -EINVAL;578 return -EINVAL;
579}579}
580
581int shstk_update_last_frame(unsigned long val)
582{
583 unsigned long ssp;
584
585 if (!features_enabled(ARCH_SHSTK_SHSTK))
586 return 0;
587
588 ssp = get_user_shstk_addr();
589 return write_user_shstk_64((u64 __user *)ssp, (u64)val);
590}
591
592bool shstk_is_enabled(void)
593{
594 return features_enabled(ARCH_SHSTK_SHSTK);
595}
arch/x86/kernel/uprobes.c+123−1
@@ -12,6 +12,7 @@ · 11 unmodified lines
12#include <linux/ptrace.h>12#include <linux/ptrace.h>
13#include <linux/uprobes.h>13#include <linux/uprobes.h>
14#include <linux/uaccess.h>14#include <linux/uaccess.h>
15#include <linux/syscalls.h>
15 16
16#include <linux/kdebug.h>17#include <linux/kdebug.h>
17#include <asm/processor.h>18#include <asm/processor.h>
@@ -308,6 +309,122 @@ static int uprobe_init_insn(struct arch_uprobe *auprobe, struct insn *insn, bool · 290 unmodified lines
308}309}
309 310
310#ifdef CONFIG_X86_64311#ifdef CONFIG_X86_64
312
313asm (
314 ".pushsection .rodata\n"
315 ".global uretprobe_trampoline_entry\n"
316 "uretprobe_trampoline_entry:\n"
317 "pushq %rax\n"
318 "pushq %rcx\n"
319 "pushq %r11\n"
320 "movq $" __stringify(__NR_uretprobe) ", %rax\n"
321 "syscall\n"
322 ".global uretprobe_syscall_check\n"
323 "uretprobe_syscall_check:\n"
324 "popq %r11\n"
325 "popq %rcx\n"
326
327 /* The uretprobe syscall replaces stored %rax value with final
328 * return address, so we don't restore %rax in here and just
329 * call ret.
330 */
331 "retq\n"
332 ".global uretprobe_trampoline_end\n"
333 "uretprobe_trampoline_end:\n"
334 ".popsection\n"
335);
336
337extern u8 uretprobe_trampoline_entry[];
338extern u8 uretprobe_trampoline_end[];
339extern u8 uretprobe_syscall_check[];
340
341void *arch_uprobe_trampoline(unsigned long *psize)
342{
343 static uprobe_opcode_t insn = UPROBE_SWBP_INSN;
344 struct pt_regs *regs = task_pt_regs(current);
345
346 /*
347 * At the moment the uretprobe syscall trampoline is supported
348 * only for native 64-bit process, the compat process still uses
349 * standard breakpoint.
350 */
351 if (user_64bit_mode(regs)) {
352 *psize = uretprobe_trampoline_end - uretprobe_trampoline_entry;
353 return uretprobe_trampoline_entry;
354 }
355
356 *psize = UPROBE_SWBP_INSN_SIZE;
357 return &insn;
358}
359
360static unsigned long trampoline_check_ip(void)
361{
362 unsigned long tramp = uprobe_get_trampoline_vaddr();
363
364 return tramp + (uretprobe_syscall_check - uretprobe_trampoline_entry);
365}
366
367SYSCALL_DEFINE0(uretprobe)
368{
369 struct pt_regs *regs = task_pt_regs(current);
370 unsigned long err, ip, sp, r11_cx_ax[3];
371
372 if (regs->ip != trampoline_check_ip())
373 goto sigill;
374
375 err = copy_from_user(r11_cx_ax, (void __user *)regs->sp, sizeof(r11_cx_ax));
376 if (err)
377 goto sigill;
378
379 /* expose the "right" values of r11/cx/ax/sp to uprobe_consumer/s */
380 regs->r11 = r11_cx_ax[0];
381 regs->cx = r11_cx_ax[1];
382 regs->ax = r11_cx_ax[2];
383 regs->sp += sizeof(r11_cx_ax);
384 regs->orig_ax = -1;
385
386 ip = regs->ip;
387 sp = regs->sp;
388
389 uprobe_handle_trampoline(regs);
390
391 /*
392 * Some of the uprobe consumers has changed sp, we can do nothing,
393 * just return via iret.
394 * .. or shadow stack is enabled, in which case we need to skip
395 * return through the user space stack address.
396 */
397 if (regs->sp != sp || shstk_is_enabled())
398 return regs->ax;
399 regs->sp -= sizeof(r11_cx_ax);
400
401 /* for the case uprobe_consumer has changed r11/cx */
402 r11_cx_ax[0] = regs->r11;
403 r11_cx_ax[1] = regs->cx;
404
405 /*
406 * ax register is passed through as return value, so we can use
407 * its space on stack for ip value and jump to it through the
408 * trampoline's ret instruction
409 */
410 r11_cx_ax[2] = regs->ip;
411 regs->ip = ip;
412
413 err = copy_to_user((void __user *)regs->sp, r11_cx_ax, sizeof(r11_cx_ax));
414 if (err)
415 goto sigill;
416
417 /* ensure sysret, see do_syscall_64() */
418 regs->r11 = regs->flags;
419 regs->cx = regs->ip;
420
421 return regs->ax;
422
423sigill:
424 force_sig(SIGILL);
425 return -1;
426}
427
311/*428/*
312 * If arch_uprobe->insn doesn't use rip-relative addressing, return429 * If arch_uprobe->insn doesn't use rip-relative addressing, return
313 * immediately. Otherwise, rewrite the instruction so that it accesses430 * immediately. Otherwise, rewrite the instruction so that it accesses
@@ -1076,8 +1193,13 @@ arch_uretprobe_hijack_return_addr(unsigned long trampoline_vaddr, struct pt_regs · 762 unmodified lines
1076 return orig_ret_vaddr;1193 return orig_ret_vaddr;
1077 1194
1078 nleft = copy_to_user((void __user *)regs->sp, &trampoline_vaddr, rasize);1195 nleft = copy_to_user((void __user *)regs->sp, &trampoline_vaddr, rasize);
1079 if (likely(!nleft))1196 if (likely(!nleft)) {
1197 if (shstk_update_last_frame(trampoline_vaddr)) {
1198 force_sig(SIGSEGV);
1199 return -1;
1200 }
1080 return orig_ret_vaddr;1201 return orig_ret_vaddr;
1202 }
1081 1203
1082 if (nleft != rasize) {1204 if (nleft != rasize) {
1083 pr_err("return address clobbered: pid=%d, %%sp=%#lx, %%ip=%#lx\n",1205 pr_err("return address clobbered: pid=%d, %%sp=%#lx, %%ip=%#lx\n",
include/linux/syscalls.h+2−0
@@ -979,6 +979,8 @@ asmlinkage long sys_lsm_list_modules(u64 __user *ids, u32 __user *size, u32 flag · 978 unmodified lines
979/* x86 */979/* x86 */
980asmlinkage long sys_ioperm(unsigned long from, unsigned long num, int on);980asmlinkage long sys_ioperm(unsigned long from, unsigned long num, int on);
981 981
982asmlinkage long sys_uretprobe(void);
983
982/* pciconfig: alpha, arm, arm64, ia64, sparc */984/* pciconfig: alpha, arm, arm64, ia64, sparc */
983asmlinkage long sys_pciconfig_read(unsigned long bus, unsigned long dfn,985asmlinkage long sys_pciconfig_read(unsigned long bus, unsigned long dfn,
984 unsigned long off, unsigned long len,986 unsigned long off, unsigned long len,
include/linux/uprobes.h+3−0
@@ -138,6 +138,9 @@ extern bool arch_uretprobe_is_alive(struct return_instance *ret, enum rp_check c · 137 unmodified lines
138extern bool arch_uprobe_ignore(struct arch_uprobe *aup, struct pt_regs *regs);138extern bool arch_uprobe_ignore(struct arch_uprobe *aup, struct pt_regs *regs);
139extern void arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr,139extern void arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr,
140 void *src, unsigned long len);140 void *src, unsigned long len);
141extern void uprobe_handle_trampoline(struct pt_regs *regs);
142extern void *arch_uprobe_trampoline(unsigned long *psize);
143extern unsigned long uprobe_get_trampoline_vaddr(void);
141#else /* !CONFIG_UPROBES */144#else /* !CONFIG_UPROBES */
142struct uprobes_state {145struct uprobes_state {
143};146};
include/uapi/asm-generic/unistd.h+4−1
@@ -841,8 +841,11 @@ __SYSCALL(__NR_lsm_list_modules, sys_lsm_list_modules) · 840 unmodified lines
841#define __NR_mseal 462841#define __NR_mseal 462
842__SYSCALL(__NR_mseal, sys_mseal)842__SYSCALL(__NR_mseal, sys_mseal)
843 843
844#define __NR_uretprobe 463
845__SYSCALL(__NR_uretprobe, sys_uretprobe)
846
844#undef __NR_syscalls847#undef __NR_syscalls
845#define __NR_syscalls 463848#define __NR_syscalls 464
846 849
847/*850/*
848 * 32 bit systems traditionally used different851 * 32 bit systems traditionally used different
kernel/events/callchain.c+42−1
@@ -11,6 +11,7 @@ · 10 unmodified lines
11#include <linux/perf_event.h>11#include <linux/perf_event.h>
12#include <linux/slab.h>12#include <linux/slab.h>
13#include <linux/sched/task_stack.h>13#include <linux/sched/task_stack.h>
14#include <linux/uprobes.h>
14 15
15#include "internal.h"16#include "internal.h"
16 17
@@ -176,13 +177,51 @@ put_callchain_entry(int rctx) · 159 unmodified lines
176 put_recursion_context(this_cpu_ptr(callchain_recursion), rctx);177 put_recursion_context(this_cpu_ptr(callchain_recursion), rctx);
177}178}
178 179
180static void fixup_uretprobe_trampoline_entries(struct perf_callchain_entry *entry,
181 int start_entry_idx)
182{
183#ifdef CONFIG_UPROBES
184 struct uprobe_task *utask = current->utask;
185 struct return_instance *ri;
186 __u64 *cur_ip, *last_ip, tramp_addr;
187
188 if (likely(!utask || !utask->return_instances))
189 return;
190
191 cur_ip = &entry->ip[start_entry_idx];
192 last_ip = &entry->ip[entry->nr - 1];
193 ri = utask->return_instances;
194 tramp_addr = uprobe_get_trampoline_vaddr();
195
196 /*
197 * If there are pending uretprobes for the current thread, they are
198 * recorded in a list inside utask->return_instances; each such
199 * pending uretprobe replaces traced user function's return address on
200 * the stack, so when stack trace is captured, instead of seeing
201 * actual function's return address, we'll have one or many uretprobe
202 * trampoline addresses in the stack trace, which are not helpful and
203 * misleading to users.
204 * So here we go over the pending list of uretprobes, and each
205 * encountered trampoline address is replaced with actual return
206 * address.
207 */
208 while (ri && cur_ip <= last_ip) {
209 if (*cur_ip == tramp_addr) {
210 *cur_ip = ri->orig_ret_vaddr;
211 ri = ri->next;
212 }
213 cur_ip++;
214 }
215#endif
216}
217
179struct perf_callchain_entry *218struct perf_callchain_entry *
180get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user,219get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user,
181 u32 max_stack, bool crosstask, bool add_mark)220 u32 max_stack, bool crosstask, bool add_mark)
182{221{
183 struct perf_callchain_entry *entry;222 struct perf_callchain_entry *entry;
184 struct perf_callchain_entry_ctx ctx;223 struct perf_callchain_entry_ctx ctx;
185 int rctx;224 int rctx, start_entry_idx;
186 225
187 entry = get_callchain_entry(&rctx);226 entry = get_callchain_entry(&rctx);
188 if (!entry)227 if (!entry)
@@ -215,7 +254,9 @@ get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user, · 26 unmodified lines
215 if (add_mark)254 if (add_mark)
216 perf_callchain_store_context(&ctx, PERF_CONTEXT_USER);255 perf_callchain_store_context(&ctx, PERF_CONTEXT_USER);
217 256
257 start_entry_idx = entry->nr;
218 perf_callchain_user(&ctx, regs);258 perf_callchain_user(&ctx, regs);
259 fixup_uretprobe_trampoline_entries(entry, start_entry_idx);
219 }260 }
220 }261 }
221 262
kernel/events/uprobes.c+26−7
@@ -1474,11 +1474,20 @@ static int xol_add_vma(struct mm_struct *mm, struct xol_area *area) · 1473 unmodified lines
1474 return ret;1474 return ret;
1475}1475}
1476 1476
1477void * __weak arch_uprobe_trampoline(unsigned long *psize)
1478{
1479 static uprobe_opcode_t insn = UPROBE_SWBP_INSN;
1480
1481 *psize = UPROBE_SWBP_INSN_SIZE;
1482 return &insn;
1483}
1484
1477static struct xol_area *__create_xol_area(unsigned long vaddr)1485static struct xol_area *__create_xol_area(unsigned long vaddr)
1478{1486{
1479 struct mm_struct *mm = current->mm;1487 struct mm_struct *mm = current->mm;
1480 uprobe_opcode_t insn = UPROBE_SWBP_INSN;1488 unsigned long insns_size;
1481 struct xol_area *area;1489 struct xol_area *area;
1490 void *insns;
1482 1491
1483 area = kmalloc(sizeof(*area), GFP_KERNEL);1492 area = kmalloc(sizeof(*area), GFP_KERNEL);
1484 if (unlikely(!area))1493 if (unlikely(!area))
@@ -1502,7 +1511,8 @@ static struct xol_area *__create_xol_area(unsigned long vaddr) · 17 unmodified lines
1502 /* Reserve the 1st slot for get_trampoline_vaddr() */1511 /* Reserve the 1st slot for get_trampoline_vaddr() */
1503 set_bit(0, area->bitmap);1512 set_bit(0, area->bitmap);
1504 atomic_set(&area->slot_count, 1);1513 atomic_set(&area->slot_count, 1);
1505 arch_uprobe_copy_ixol(area->pages[0], 0, &insn, UPROBE_SWBP_INSN_SIZE);1514 insns = arch_uprobe_trampoline(&insns_size);
1515 arch_uprobe_copy_ixol(area->pages[0], 0, insns, insns_size);
1506 1516
1507 if (!xol_add_vma(mm, area))1517 if (!xol_add_vma(mm, area))
1508 return area;1518 return area;
@@ -1827,7 +1837,7 @@ void uprobe_copy_process(struct task_struct *t, unsigned long flags) · 318 unmodified lines
1827 *1837 *
1828 * Returns -1 in case the xol_area is not allocated.1838 * Returns -1 in case the xol_area is not allocated.
1829 */1839 */
1830static unsigned long get_trampoline_vaddr(void)1840unsigned long uprobe_get_trampoline_vaddr(void)
1831{1841{
1832 struct xol_area *area;1842 struct xol_area *area;
1833 unsigned long trampoline_vaddr = -1;1843 unsigned long trampoline_vaddr = -1;
@@ -1878,7 +1888,7 @@ static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs) · 44 unmodified lines
1878 if (!ri)1888 if (!ri)
1879 return;1889 return;
1880 1890
1881 trampoline_vaddr = get_trampoline_vaddr();1891 trampoline_vaddr = uprobe_get_trampoline_vaddr();
1882 orig_ret_vaddr = arch_uretprobe_hijack_return_addr(trampoline_vaddr, regs);1892 orig_ret_vaddr = arch_uretprobe_hijack_return_addr(trampoline_vaddr, regs);
1883 if (orig_ret_vaddr == -1)1893 if (orig_ret_vaddr == -1)
1884 goto fail;1894 goto fail;
@@ -2123,7 +2133,7 @@ static struct return_instance *find_next_ret_chain(struct return_instance *ri) · 238 unmodified lines
2123 return ri;2133 return ri;
2124}2134}
2125 2135
2126static void handle_trampoline(struct pt_regs *regs)2136void uprobe_handle_trampoline(struct pt_regs *regs)
2127{2137{
2128 struct uprobe_task *utask;2138 struct uprobe_task *utask;
2129 struct return_instance *ri, *next;2139 struct return_instance *ri, *next;
@@ -2149,6 +2159,15 @@ static void handle_trampoline(struct pt_regs *regs) · 19 unmodified lines
2149 2159
2150 instruction_pointer_set(regs, ri->orig_ret_vaddr);2160 instruction_pointer_set(regs, ri->orig_ret_vaddr);
2151 do {2161 do {
2162 /* pop current instance from the stack of pending return instances,
2163 * as it's not pending anymore: we just fixed up original
2164 * instruction pointer in regs and are about to call handlers;
2165 * this allows fixup_uretprobe_trampoline_entries() to properly fix up
2166 * captured stack traces from uretprobe handlers, in which pending
2167 * trampoline addresses on the stack are replaced with correct
2168 * original return addresses
2169 */
2170 utask->return_instances = ri->next;
2152 if (valid)2171 if (valid)
2153 handle_uretprobe_chain(ri, regs);2172 handle_uretprobe_chain(ri, regs);
2154 ri = free_ret_instance(ri);2173 ri = free_ret_instance(ri);
@@ -2187,8 +2206,8 @@ static void handle_swbp(struct pt_regs *regs) · 32 unmodified lines
2187 int is_swbp;2206 int is_swbp;
2188 2207
2189 bp_vaddr = uprobe_get_swbp_addr(regs);2208 bp_vaddr = uprobe_get_swbp_addr(regs);
2190 if (bp_vaddr == get_trampoline_vaddr())2209 if (bp_vaddr == uprobe_get_trampoline_vaddr())
2191 return handle_trampoline(regs);2210 return uprobe_handle_trampoline(regs);
2192 2211
2193 uprobe = find_active_uprobe(bp_vaddr, &is_swbp);2212 uprobe = find_active_uprobe(bp_vaddr, &is_swbp);
2194 if (!uprobe) {2213 if (!uprobe) {
kernel/sys_ni.c+2−0
@@ -390,3 +390,5 @@ COND_SYSCALL(setuid16); · 389 unmodified lines
390 390
391/* restartable sequence */391/* restartable sequence */
392COND_SYSCALL(rseq);392COND_SYSCALL(rseq);
393
394COND_SYSCALL(uretprobe);
kernel/trace/trace_kprobe.c+111−81
@@ -678,6 +678,21 @@ end: · 677 unmodified lines
678}678}
679 679
680#ifdef CONFIG_MODULES680#ifdef CONFIG_MODULES
681static int validate_module_probe_symbol(const char *modname, const char *symbol);
682
683static int register_module_trace_kprobe(struct module *mod, struct trace_kprobe *tk)
684{
685 const char *p;
686 int ret = 0;
687
688 p = strchr(trace_kprobe_symbol(tk), ':');
689 if (p)
690 ret = validate_module_probe_symbol(module_name(mod), p + 1);
691 if (!ret)
692 ret = __register_trace_kprobe(tk);
693 return ret;
694}
695
681/* Module notifier call back, checking event on the module */696/* Module notifier call back, checking event on the module */
682static int trace_kprobe_module_callback(struct notifier_block *nb,697static int trace_kprobe_module_callback(struct notifier_block *nb,
683 unsigned long val, void *data)698 unsigned long val, void *data)
@@ -696,7 +711,7 @@ static int trace_kprobe_module_callback(struct notifier_block *nb, · 12 unmodified lines
696 if (trace_kprobe_within_module(tk, mod)) {711 if (trace_kprobe_within_module(tk, mod)) {
697 /* Don't need to check busy - this should have gone. */712 /* Don't need to check busy - this should have gone. */
698 __unregister_trace_kprobe(tk);713 __unregister_trace_kprobe(tk);
699 ret = __register_trace_kprobe(tk);714 ret = register_module_trace_kprobe(mod, tk);
700 if (ret)715 if (ret)
701 pr_warn("Failed to re-register probe %s on %s: %d\n",716 pr_warn("Failed to re-register probe %s on %s: %d\n",
702 trace_probe_name(&tk->tp),717 trace_probe_name(&tk->tp),
@@ -747,17 +762,81 @@ static int count_mod_symbols(void *data, const char *name, unsigned long unused) · 44 unmodified lines
747 return 0;762 return 0;
748}763}
749 764
750static unsigned int number_of_same_symbols(char *func_name)765static unsigned int number_of_same_symbols(const char *mod, const char *func_name)
751{766{
752 struct sym_count_ctx ctx = { .count = 0, .name = func_name };767 struct sym_count_ctx ctx = { .count = 0, .name = func_name };
753 768
754 kallsyms_on_each_match_symbol(count_symbols, func_name, &ctx.count);769 if (!mod)
770 kallsyms_on_each_match_symbol(count_symbols, func_name, &ctx.count);
755 771
756 module_kallsyms_on_each_symbol(NULL, count_mod_symbols, &ctx);772 module_kallsyms_on_each_symbol(mod, count_mod_symbols, &ctx);
757 773
758 return ctx.count;774 return ctx.count;
759}775}
760 776
777static int validate_module_probe_symbol(const char *modname, const char *symbol)
778{
779 unsigned int count = number_of_same_symbols(modname, symbol);
780
781 if (count > 1) {
782 /*
783 * Users should use ADDR to remove the ambiguity of
784 * using KSYM only.
785 */
786 return -EADDRNOTAVAIL;
787 } else if (count == 0) {
788 /*
789 * We can return ENOENT earlier than when register the
790 * kprobe.
791 */
792 return -ENOENT;
793 }
794 return 0;
795}
796
797#ifdef CONFIG_MODULES
798/* Return NULL if the module is not loaded or under unloading. */
799static struct module *try_module_get_by_name(const char *name)
800{
801 struct module *mod;
802
803 rcu_read_lock_sched();
804 mod = find_module(name);
805 if (mod && !try_module_get(mod))
806 mod = NULL;
807 rcu_read_unlock_sched();
808
809 return mod;
810}
811#else
812#define try_module_get_by_name(name) (NULL)
813#endif
814
815static int validate_probe_symbol(char *symbol)
816{
817 struct module *mod = NULL;
818 char *modname = NULL, *p;
819 int ret = 0;
820
821 p = strchr(symbol, ':');
822 if (p) {
823 modname = symbol;
824 symbol = p + 1;
825 *p = '\0';
826 mod = try_module_get_by_name(modname);
827 if (!mod)
828 goto out;
829 }
830
831 ret = validate_module_probe_symbol(modname, symbol);
832out:
833 if (p)
834 *p = ':';
835 if (mod)
836 module_put(mod);
837 return ret;
838}
839
761static int trace_kprobe_entry_handler(struct kretprobe_instance *ri,840static int trace_kprobe_entry_handler(struct kretprobe_instance *ri,
762 struct pt_regs *regs);841 struct pt_regs *regs);
763 842
@@ -881,6 +960,14 @@ static int __trace_kprobe_create(int argc, const char *argv[]) · 117 unmodified lines
881 trace_probe_log_err(0, BAD_PROBE_ADDR);960 trace_probe_log_err(0, BAD_PROBE_ADDR);
882 goto parse_error;961 goto parse_error;
883 }962 }
963 ret = validate_probe_symbol(symbol);
964 if (ret) {
965 if (ret == -EADDRNOTAVAIL)
966 trace_probe_log_err(0, NON_UNIQ_SYMBOL);
967 else
968 trace_probe_log_err(0, BAD_PROBE_ADDR);
969 goto parse_error;
970 }
884 if (is_return)971 if (is_return)
885 ctx.flags |= TPARG_FL_RETURN;972 ctx.flags |= TPARG_FL_RETURN;
886 ret = kprobe_on_func_entry(NULL, symbol, offset);973 ret = kprobe_on_func_entry(NULL, symbol, offset);
@@ -893,31 +980,6 @@ static int __trace_kprobe_create(int argc, const char *argv[]) · 6 unmodified lines
893 }980 }
894 }981 }
895 982
896 if (symbol && !strchr(symbol, ':')) {
897 unsigned int count;
898
899 count = number_of_same_symbols(symbol);
900 if (count > 1) {
901 /*
902 * Users should use ADDR to remove the ambiguity of
903 * using KSYM only.
904 */
905 trace_probe_log_err(0, NON_UNIQ_SYMBOL);
906 ret = -EADDRNOTAVAIL;
907
908 goto error;
909 } else if (count == 0) {
910 /*
911 * We can return ENOENT earlier than when register the
912 * kprobe.
913 */
914 trace_probe_log_err(0, BAD_PROBE_ADDR);
915 ret = -ENOENT;
916
917 goto error;
918 }
919 }
920
921 trace_probe_log_set_index(0);983 trace_probe_log_set_index(0);
922 if (event) {984 if (event) {
923 ret = traceprobe_parse_event_name(&event, &group, gbuf,985 ret = traceprobe_parse_event_name(&event, &group, gbuf,
@@ -1835,21 +1897,9 @@ create_local_trace_kprobe(char *func, void *addr, unsigned long offs, · 911 unmodified lines
1835 char *event;1897 char *event;
1836 1898
1837 if (func) {1899 if (func) {
1838 unsigned int count;1900 ret = validate_probe_symbol(func);
1839 1901 if (ret)
1840 count = number_of_same_symbols(func);1902 return ERR_PTR(ret);
1841 if (count > 1)
1842 /*
1843 * Users should use addr to remove the ambiguity of
1844 * using func only.
1845 */
1846 return ERR_PTR(-EADDRNOTAVAIL);
1847 else if (count == 0)
1848 /*
1849 * We can return ENOENT earlier than when register the
1850 * kprobe.
1851 */
1852 return ERR_PTR(-ENOENT);
1853 }1903 }
1854 1904
1855 /*1905 /*
@@ -2023,19 +2073,16 @@ static __init int kprobe_trace_self_tests_init(void) · 167 unmodified lines
2023 pr_info("Testing kprobe tracing: ");2073 pr_info("Testing kprobe tracing: ");
2024 2074
2025 ret = create_or_delete_trace_kprobe("p:testprobe kprobe_trace_selftest_target $stack $stack0 +0($stack)");2075 ret = create_or_delete_trace_kprobe("p:testprobe kprobe_trace_selftest_target $stack $stack0 +0($stack)");
2026 if (WARN_ON_ONCE(ret)) {2076 if (WARN_ONCE(ret, "error on probing function entry.")) {
2027 pr_warn("error on probing function entry.\n");
2028 warn++;2077 warn++;
2029 } else {2078 } else {
2030 /* Enable trace point */2079 /* Enable trace point */
2031 tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);2080 tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);
2032 if (WARN_ON_ONCE(tk == NULL)) {2081 if (WARN_ONCE(tk == NULL, "error on probing function entry.")) {
2033 pr_warn("error on getting new probe.\n");
2034 warn++;2082 warn++;
2035 } else {2083 } else {
2036 file = find_trace_probe_file(tk, top_trace_array());2084 file = find_trace_probe_file(tk, top_trace_array());
2037 if (WARN_ON_ONCE(file == NULL)) {2085 if (WARN_ONCE(file == NULL, "error on getting probe file.")) {
2038 pr_warn("error on getting probe file.\n");
2039 warn++;2086 warn++;
2040 } else2087 } else
2041 enable_trace_kprobe(2088 enable_trace_kprobe(
@@ -2044,19 +2091,16 @@ static __init int kprobe_trace_self_tests_init(void) · 2 unmodified lines
2044 }2091 }
2045 2092
2046 ret = create_or_delete_trace_kprobe("r:testprobe2 kprobe_trace_selftest_target $retval");2093 ret = create_or_delete_trace_kprobe("r:testprobe2 kprobe_trace_selftest_target $retval");
2047 if (WARN_ON_ONCE(ret)) {2094 if (WARN_ONCE(ret, "error on probing function return.")) {
2048 pr_warn("error on probing function return.\n");
2049 warn++;2095 warn++;
2050 } else {2096 } else {
2051 /* Enable trace point */2097 /* Enable trace point */
2052 tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);2098 tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);
2053 if (WARN_ON_ONCE(tk == NULL)) {2099 if (WARN_ONCE(tk == NULL, "error on getting 2nd new probe.")) {
2054 pr_warn("error on getting 2nd new probe.\n");
2055 warn++;2100 warn++;
2056 } else {2101 } else {
2057 file = find_trace_probe_file(tk, top_trace_array());2102 file = find_trace_probe_file(tk, top_trace_array());
2058 if (WARN_ON_ONCE(file == NULL)) {2103 if (WARN_ONCE(file == NULL, "error on getting probe file.")) {
2059 pr_warn("error on getting probe file.\n");
2060 warn++;2104 warn++;
2061 } else2105 } else
2062 enable_trace_kprobe(2106 enable_trace_kprobe(
@@ -2079,18 +2123,15 @@ static __init int kprobe_trace_self_tests_init(void) · 16 unmodified lines
2079 2123
2080 /* Disable trace points before removing it */2124 /* Disable trace points before removing it */
2081 tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);2125 tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);
2082 if (WARN_ON_ONCE(tk == NULL)) {2126 if (WARN_ONCE(tk == NULL, "error on getting test probe.")) {
2083 pr_warn("error on getting test probe.\n");
2084 warn++;2127 warn++;
2085 } else {2128 } else {
2086 if (trace_kprobe_nhit(tk) != 1) {2129 if (WARN_ONCE(trace_kprobe_nhit(tk) != 1,
2087 pr_warn("incorrect number of testprobe hits\n");2130 "incorrect number of testprobe hits."))
2088 warn++;2131 warn++;
2089 }
2090 2132
2091 file = find_trace_probe_file(tk, top_trace_array());2133 file = find_trace_probe_file(tk, top_trace_array());
2092 if (WARN_ON_ONCE(file == NULL)) {2134 if (WARN_ONCE(file == NULL, "error on getting probe file.")) {
2093 pr_warn("error on getting probe file.\n");
2094 warn++;2135 warn++;
2095 } else2136 } else
2096 disable_trace_kprobe(2137 disable_trace_kprobe(
@@ -2098,18 +2139,15 @@ static __init int kprobe_trace_self_tests_init(void) · 1 unmodified line
2098 }2139 }
2099 2140
2100 tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);2141 tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);
2101 if (WARN_ON_ONCE(tk == NULL)) {2142 if (WARN_ONCE(tk == NULL, "error on getting 2nd test probe.")) {
2102 pr_warn("error on getting 2nd test probe.\n");
2103 warn++;2143 warn++;
2104 } else {2144 } else {
2105 if (trace_kprobe_nhit(tk) != 1) {2145 if (WARN_ONCE(trace_kprobe_nhit(tk) != 1,
2106 pr_warn("incorrect number of testprobe2 hits\n");2146 "incorrect number of testprobe2 hits."))
2107 warn++;2147 warn++;
2108 }
2109 2148
2110 file = find_trace_probe_file(tk, top_trace_array());2149 file = find_trace_probe_file(tk, top_trace_array());
2111 if (WARN_ON_ONCE(file == NULL)) {2150 if (WARN_ONCE(file == NULL, "error on getting probe file.")) {
2112 pr_warn("error on getting probe file.\n");
2113 warn++;2151 warn++;
2114 } else2152 } else
2115 disable_trace_kprobe(2153 disable_trace_kprobe(
@@ -2117,23 +2155,15 @@ static __init int kprobe_trace_self_tests_init(void) · 1 unmodified line
2117 }2155 }
2118 2156
2119 ret = create_or_delete_trace_kprobe("-:testprobe");2157 ret = create_or_delete_trace_kprobe("-:testprobe");
2120 if (WARN_ON_ONCE(ret)) {2158 if (WARN_ONCE(ret, "error on deleting a probe."))
2121 pr_warn("error on deleting a probe.\n");
2122 warn++;2159 warn++;
2123 }
2124 2160
2125 ret = create_or_delete_trace_kprobe("-:testprobe2");2161 ret = create_or_delete_trace_kprobe("-:testprobe2");
2126 if (WARN_ON_ONCE(ret)) {2162 if (WARN_ONCE(ret, "error on deleting a probe."))
2127 pr_warn("error on deleting a probe.\n");
2128 warn++;2163 warn++;
2129 }2164
2130 2165
2131end:2166end:
2132 ret = dyn_events_release_all(&trace_kprobe_ops);
2133 if (WARN_ON_ONCE(ret)) {
2134 pr_warn("error on cleaning up probes.\n");
2135 warn++;
2136 }
2137 /*2167 /*
2138 * Wait for the optimizer work to finish. Otherwise it might fiddle2168 * Wait for the optimizer work to finish. Otherwise it might fiddle
2139 * with probes in already freed __init text.2169 * with probes in already freed __init text.
samples/fprobe/fprobe_example.c+1−0
@@ -150,4 +150,5 @@ static void __exit fprobe_exit(void) · 149 unmodified lines
150 150
151module_init(fprobe_init)151module_init(fprobe_init)
152module_exit(fprobe_exit)152module_exit(fprobe_exit)
153MODULE_DESCRIPTION("sample kernel module showing the use of fprobe");
153MODULE_LICENSE("GPL");154MODULE_LICENSE("GPL");
samples/kprobes/kprobe_example.c+1−0
@@ -125,4 +125,5 @@ static void __exit kprobe_exit(void) · 124 unmodified lines
125 125
126module_init(kprobe_init)126module_init(kprobe_init)
127module_exit(kprobe_exit)127module_exit(kprobe_exit)
128MODULE_DESCRIPTION("sample kernel module showing the use of kprobes");
128MODULE_LICENSE("GPL");129MODULE_LICENSE("GPL");
samples/kprobes/kretprobe_example.c+1−0
@@ -104,4 +104,5 @@ static void __exit kretprobe_exit(void) · 103 unmodified lines
104 104
105module_init(kretprobe_init)105module_init(kretprobe_init)
106module_exit(kretprobe_exit)106module_exit(kretprobe_exit)
107MODULE_DESCRIPTION("sample kernel module showing the use of return probes");
107MODULE_LICENSE("GPL");108MODULE_LICENSE("GPL");
tools/include/linux/compiler.h+4−0
@@ -62,6 +62,10 @@ · 61 unmodified lines
62#define __nocf_check __attribute__((nocf_check))62#define __nocf_check __attribute__((nocf_check))
63#endif63#endif
64 64
65#ifndef __naked
66#define __naked __attribute__((__naked__))
67#endif
68
65/* Are two types/vars the same type (ignoring qualifiers)? */69/* Are two types/vars the same type (ignoring qualifiers)? */
66#ifndef __same_type70#ifndef __same_type
67# define __same_type(a, b) __builtin_types_compatible_p(typeof(a), typeof(b))71# define __same_type(a, b) __builtin_types_compatible_p(typeof(a), typeof(b))
tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c+122−1
@@ -18,6 +18,7 @@ · 17 unmodified lines
18#include <linux/in6.h>18#include <linux/in6.h>
19#include <linux/un.h>19#include <linux/un.h>
20#include <net/sock.h>20#include <net/sock.h>
21#include <linux/namei.h>
21#include "bpf_testmod.h"22#include "bpf_testmod.h"
22#include "bpf_testmod_kfunc.h"23#include "bpf_testmod_kfunc.h"
23 24
@@ -413,6 +414,119 @@ static struct bin_attribute bin_attr_bpf_testmod_file __ro_after_init = { · 389 unmodified lines
413 .write = bpf_testmod_test_write,414 .write = bpf_testmod_test_write,
414};415};
415 416
417/* bpf_testmod_uprobe sysfs attribute is so far enabled for x86_64 only,
418 * please see test_uretprobe_regs_change test
419 */
420#ifdef __x86_64__
421
422static int
423uprobe_ret_handler(struct uprobe_consumer *self, unsigned long func,
424 struct pt_regs *regs)
425
426{
427 regs->ax = 0x12345678deadbeef;
428 regs->cx = 0x87654321feebdaed;
429 regs->r11 = (u64) -1;
430 return true;
431}
432
433struct testmod_uprobe {
434 struct path path;
435 loff_t offset;
436 struct uprobe_consumer consumer;
437};
438
439static DEFINE_MUTEX(testmod_uprobe_mutex);
440
441static struct testmod_uprobe uprobe = {
442 .consumer.ret_handler = uprobe_ret_handler,
443};
444
445static int testmod_register_uprobe(loff_t offset)
446{
447 int err = -EBUSY;
448
449 if (uprobe.offset)
450 return -EBUSY;
451
452 mutex_lock(&testmod_uprobe_mutex);
453
454 if (uprobe.offset)
455 goto out;
456
457 err = kern_path("/proc/self/exe", LOOKUP_FOLLOW, &uprobe.path);
458 if (err)
459 goto out;
460
461 err = uprobe_register_refctr(d_real_inode(uprobe.path.dentry),
462 offset, 0, &uprobe.consumer);
463 if (err)
464 path_put(&uprobe.path);
465 else
466 uprobe.offset = offset;
467
468out:
469 mutex_unlock(&testmod_uprobe_mutex);
470 return err;
471}
472
473static void testmod_unregister_uprobe(void)
474{
475 mutex_lock(&testmod_uprobe_mutex);
476
477 if (uprobe.offset) {
478 uprobe_unregister(d_real_inode(uprobe.path.dentry),
479 uprobe.offset, &uprobe.consumer);
480 uprobe.offset = 0;
481 }
482
483 mutex_unlock(&testmod_uprobe_mutex);
484}
485
486static ssize_t
487bpf_testmod_uprobe_write(struct file *file, struct kobject *kobj,
488 struct bin_attribute *bin_attr,
489 char *buf, loff_t off, size_t len)
490{
491 unsigned long offset = 0;
492 int err = 0;
493
494 if (kstrtoul(buf, 0, &offset))
495 return -EINVAL;
496
497 if (offset)
498 err = testmod_register_uprobe(offset);
499 else
500 testmod_unregister_uprobe();
501
502 return err ?: strlen(buf);
503}
504
505static struct bin_attribute bin_attr_bpf_testmod_uprobe_file __ro_after_init = {
506 .attr = { .name = "bpf_testmod_uprobe", .mode = 0666, },
507 .write = bpf_testmod_uprobe_write,
508};
509
510static int register_bpf_testmod_uprobe(void)
511{
512 return sysfs_create_bin_file(kernel_kobj, &bin_attr_bpf_testmod_uprobe_file);
513}
514
515static void unregister_bpf_testmod_uprobe(void)
516{
517 testmod_unregister_uprobe();
518 sysfs_remove_bin_file(kernel_kobj, &bin_attr_bpf_testmod_uprobe_file);
519}
520
521#else
522static int register_bpf_testmod_uprobe(void)
523{
524 return 0;
525}
526
527static void unregister_bpf_testmod_uprobe(void) { }
528#endif
529
416BTF_KFUNCS_START(bpf_testmod_common_kfunc_ids)530BTF_KFUNCS_START(bpf_testmod_common_kfunc_ids)
417BTF_ID_FLAGS(func, bpf_iter_testmod_seq_new, KF_ITER_NEW)531BTF_ID_FLAGS(func, bpf_iter_testmod_seq_new, KF_ITER_NEW)
418BTF_ID_FLAGS(func, bpf_iter_testmod_seq_next, KF_ITER_NEXT | KF_RET_NULL)532BTF_ID_FLAGS(func, bpf_iter_testmod_seq_next, KF_ITER_NEXT | KF_RET_NULL)
@@ -983,7 +1097,13 @@ static int bpf_testmod_init(void) · 564 unmodified lines
983 return -EINVAL;1097 return -EINVAL;
984 sock = NULL;1098 sock = NULL;
985 mutex_init(&sock_lock);1099 mutex_init(&sock_lock);
986 return sysfs_create_bin_file(kernel_kobj, &bin_attr_bpf_testmod_file);1100 ret = sysfs_create_bin_file(kernel_kobj, &bin_attr_bpf_testmod_file);
1101 if (ret < 0)
1102 return ret;
1103 ret = register_bpf_testmod_uprobe();
1104 if (ret < 0)
1105 return ret;
1106 return 0;
987}1107}
988 1108
989static void bpf_testmod_exit(void)1109static void bpf_testmod_exit(void)
@@ -998,6 +1118,7 @@ static void bpf_testmod_exit(void) · 8 unmodified lines
998 1118
999 bpf_kfunc_close_sock();1119 bpf_kfunc_close_sock();
1000 sysfs_remove_bin_file(kernel_kobj, &bin_attr_bpf_testmod_file);1120 sysfs_remove_bin_file(kernel_kobj, &bin_attr_bpf_testmod_file);
1121 unregister_bpf_testmod_uprobe();
1001}1122}
1002 1123
1003module_init(bpf_testmod_init);1124module_init(bpf_testmod_init);
tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c+385−0
@@ -0,0 +1,385 @@
1// SPDX-License-Identifier: GPL-2.0
2
3#include <test_progs.h>
4
5#ifdef __x86_64__
6
7#include <unistd.h>
8#include <asm/ptrace.h>
9#include <linux/compiler.h>
10#include <linux/stringify.h>
11#include <sys/wait.h>
12#include <sys/syscall.h>
13#include <sys/prctl.h>
14#include <asm/prctl.h>
15#include "uprobe_syscall.skel.h"
16#include "uprobe_syscall_executed.skel.h"
17
18__naked unsigned long uretprobe_regs_trigger(void)
19{
20 asm volatile (
21 "movq $0xdeadbeef, %rax\n"
22 "ret\n"
23 );
24}
25
26__naked void uretprobe_regs(struct pt_regs *before, struct pt_regs *after)
27{
28 asm volatile (
29 "movq %r15, 0(%rdi)\n"
30 "movq %r14, 8(%rdi)\n"
31 "movq %r13, 16(%rdi)\n"
32 "movq %r12, 24(%rdi)\n"
33 "movq %rbp, 32(%rdi)\n"
34 "movq %rbx, 40(%rdi)\n"
35 "movq %r11, 48(%rdi)\n"
36 "movq %r10, 56(%rdi)\n"
37 "movq %r9, 64(%rdi)\n"
38 "movq %r8, 72(%rdi)\n"
39 "movq %rax, 80(%rdi)\n"
40 "movq %rcx, 88(%rdi)\n"
41 "movq %rdx, 96(%rdi)\n"
42 "movq %rsi, 104(%rdi)\n"
43 "movq %rdi, 112(%rdi)\n"
44 "movq $0, 120(%rdi)\n" /* orig_rax */
45 "movq $0, 128(%rdi)\n" /* rip */
46 "movq $0, 136(%rdi)\n" /* cs */
47 "pushf\n"
48 "pop %rax\n"
49 "movq %rax, 144(%rdi)\n" /* eflags */
50 "movq %rsp, 152(%rdi)\n" /* rsp */
51 "movq $0, 160(%rdi)\n" /* ss */
52
53 /* save 2nd argument */
54 "pushq %rsi\n"
55 "call uretprobe_regs_trigger\n"
56
57 /* save return value and load 2nd argument pointer to rax */
58 "pushq %rax\n"
59 "movq 8(%rsp), %rax\n"
60
61 "movq %r15, 0(%rax)\n"
62 "movq %r14, 8(%rax)\n"
63 "movq %r13, 16(%rax)\n"
64 "movq %r12, 24(%rax)\n"
65 "movq %rbp, 32(%rax)\n"
66 "movq %rbx, 40(%rax)\n"
67 "movq %r11, 48(%rax)\n"
68 "movq %r10, 56(%rax)\n"
69 "movq %r9, 64(%rax)\n"
70 "movq %r8, 72(%rax)\n"
71 "movq %rcx, 88(%rax)\n"
72 "movq %rdx, 96(%rax)\n"
73 "movq %rsi, 104(%rax)\n"
74 "movq %rdi, 112(%rax)\n"
75 "movq $0, 120(%rax)\n" /* orig_rax */
76 "movq $0, 128(%rax)\n" /* rip */
77 "movq $0, 136(%rax)\n" /* cs */
78
79 /* restore return value and 2nd argument */
80 "pop %rax\n"
81 "pop %rsi\n"
82
83 "movq %rax, 80(%rsi)\n"
84
85 "pushf\n"
86 "pop %rax\n"
87
88 "movq %rax, 144(%rsi)\n" /* eflags */
89 "movq %rsp, 152(%rsi)\n" /* rsp */
90 "movq $0, 160(%rsi)\n" /* ss */
91 "ret\n"
92);
93}
94
95static void test_uretprobe_regs_equal(void)
96{
97 struct uprobe_syscall *skel = NULL;
98 struct pt_regs before = {}, after = {};
99 unsigned long *pb = (unsigned long *) &before;
100 unsigned long *pa = (unsigned long *) &after;
101 unsigned long *pp;
102 unsigned int i, cnt;
103 int err;
104
105 skel = uprobe_syscall__open_and_load();
106 if (!ASSERT_OK_PTR(skel, "uprobe_syscall__open_and_load"))
107 goto cleanup;
108
109 err = uprobe_syscall__attach(skel);
110 if (!ASSERT_OK(err, "uprobe_syscall__attach"))
111 goto cleanup;
112
113 uretprobe_regs(&before, &after);
114
115 pp = (unsigned long *) &skel->bss->regs;
116 cnt = sizeof(before)/sizeof(*pb);
117
118 for (i = 0; i < cnt; i++) {
119 unsigned int offset = i * sizeof(unsigned long);
120
121 /*
122 * Check register before and after uretprobe_regs_trigger call
123 * that triggers the uretprobe.
124 */
125 switch (offset) {
126 case offsetof(struct pt_regs, rax):
127 ASSERT_EQ(pa[i], 0xdeadbeef, "return value");
128 break;
129 default:
130 if (!ASSERT_EQ(pb[i], pa[i], "register before-after value check"))
131 fprintf(stdout, "failed register offset %u\n", offset);
132 }
133
134 /*
135 * Check register seen from bpf program and register after
136 * uretprobe_regs_trigger call
137 */
138 switch (offset) {
139 /*
140 * These values will be different (not set in uretprobe_regs),
141 * we don't care.
142 */
143 case offsetof(struct pt_regs, orig_rax):
144 case offsetof(struct pt_regs, rip):
145 case offsetof(struct pt_regs, cs):
146 case offsetof(struct pt_regs, rsp):
147 case offsetof(struct pt_regs, ss):
148 break;
149 default:
150 if (!ASSERT_EQ(pp[i], pa[i], "register prog-after value check"))
151 fprintf(stdout, "failed register offset %u\n", offset);
152 }
153 }
154
155cleanup:
156 uprobe_syscall__destroy(skel);
157}
158
159#define BPF_TESTMOD_UPROBE_TEST_FILE "/sys/kernel/bpf_testmod_uprobe"
160
161static int write_bpf_testmod_uprobe(unsigned long offset)
162{
163 size_t n, ret;
164 char buf[30];
165 int fd;
166
167 n = sprintf(buf, "%lu", offset);
168
169 fd = open(BPF_TESTMOD_UPROBE_TEST_FILE, O_WRONLY);
170 if (fd < 0)
171 return -errno;
172
173 ret = write(fd, buf, n);
174 close(fd);
175 return ret != n ? (int) ret : 0;
176}
177
178static void test_uretprobe_regs_change(void)
179{
180 struct pt_regs before = {}, after = {};
181 unsigned long *pb = (unsigned long *) &before;
182 unsigned long *pa = (unsigned long *) &after;
183 unsigned long cnt = sizeof(before)/sizeof(*pb);
184 unsigned int i, err, offset;
185
186 offset = get_uprobe_offset(uretprobe_regs_trigger);
187
188 err = write_bpf_testmod_uprobe(offset);
189 if (!ASSERT_OK(err, "register_uprobe"))
190 return;
191
192 uretprobe_regs(&before, &after);
193
194 err = write_bpf_testmod_uprobe(0);
195 if (!ASSERT_OK(err, "unregister_uprobe"))
196 return;
197
198 for (i = 0; i < cnt; i++) {
199 unsigned int offset = i * sizeof(unsigned long);
200
201 switch (offset) {
202 case offsetof(struct pt_regs, rax):
203 ASSERT_EQ(pa[i], 0x12345678deadbeef, "rax");
204 break;
205 case offsetof(struct pt_regs, rcx):
206 ASSERT_EQ(pa[i], 0x87654321feebdaed, "rcx");
207 break;
208 case offsetof(struct pt_regs, r11):
209 ASSERT_EQ(pa[i], (__u64) -1, "r11");
210 break;
211 default:
212 if (!ASSERT_EQ(pa[i], pb[i], "register before-after value check"))
213 fprintf(stdout, "failed register offset %u\n", offset);
214 }
215 }
216}
217
218#ifndef __NR_uretprobe
219#define __NR_uretprobe 467
220#endif
221
222__naked unsigned long uretprobe_syscall_call_1(void)
223{
224 /*
225 * Pretend we are uretprobe trampoline to trigger the return
226 * probe invocation in order to verify we get SIGILL.
227 */
228 asm volatile (
229 "pushq %rax\n"
230 "pushq %rcx\n"
231 "pushq %r11\n"
232 "movq $" __stringify(__NR_uretprobe) ", %rax\n"
233 "syscall\n"
234 "popq %r11\n"
235 "popq %rcx\n"
236 "retq\n"
237 );
238}
239
240__naked unsigned long uretprobe_syscall_call(void)
241{
242 asm volatile (
243 "call uretprobe_syscall_call_1\n"
244 "retq\n"
245 );
246}
247
248static void test_uretprobe_syscall_call(void)
249{
250 LIBBPF_OPTS(bpf_uprobe_multi_opts, opts,
251 .retprobe = true,
252 );
253 struct uprobe_syscall_executed *skel;
254 int pid, status, err, go[2], c;
255
256 if (ASSERT_OK(pipe(go), "pipe"))
257 return;
258
259 skel = uprobe_syscall_executed__open_and_load();
260 if (!ASSERT_OK_PTR(skel, "uprobe_syscall_executed__open_and_load"))
261 goto cleanup;
262
263 pid = fork();
264 if (!ASSERT_GE(pid, 0, "fork"))
265 goto cleanup;
266
267 /* child */
268 if (pid == 0) {
269 close(go[1]);
270
271 /* wait for parent's kick */
272 err = read(go[0], &c, 1);
273 if (err != 1)
274 exit(-1);
275
276 uretprobe_syscall_call();
277 _exit(0);
278 }
279
280 skel->links.test = bpf_program__attach_uprobe_multi(skel->progs.test, pid,
281 "/proc/self/exe",
282 "uretprobe_syscall_call", &opts);
283 if (!ASSERT_OK_PTR(skel->links.test, "bpf_program__attach_uprobe_multi"))
284 goto cleanup;
285
286 /* kick the child */
287 write(go[1], &c, 1);
288 err = waitpid(pid, &status, 0);
289 ASSERT_EQ(err, pid, "waitpid");
290
291 /* verify the child got killed with SIGILL */
292 ASSERT_EQ(WIFSIGNALED(status), 1, "WIFSIGNALED");
293 ASSERT_EQ(WTERMSIG(status), SIGILL, "WTERMSIG");
294
295 /* verify the uretprobe program wasn't called */
296 ASSERT_EQ(skel->bss->executed, 0, "executed");
297
298cleanup:
299 uprobe_syscall_executed__destroy(skel);
300 close(go[1]);
301 close(go[0]);
302}
303
304/*
305 * Borrowed from tools/testing/selftests/x86/test_shadow_stack.c.
306 *
307 * For use in inline enablement of shadow stack.
308 *
309 * The program can't return from the point where shadow stack gets enabled
310 * because there will be no address on the shadow stack. So it can't use
311 * syscall() for enablement, since it is a function.
312 *
313 * Based on code from nolibc.h. Keep a copy here because this can't pull
314 * in all of nolibc.h.
315 */
316#define ARCH_PRCTL(arg1, arg2) \
317({ \
318 long _ret; \
319 register long _num asm("eax") = __NR_arch_prctl; \
320 register long _arg1 asm("rdi") = (long)(arg1); \
321 register long _arg2 asm("rsi") = (long)(arg2); \
322 \
323 asm volatile ( \
324 "syscall\n" \
325 : "=a"(_ret) \
326 : "r"(_arg1), "r"(_arg2), \
327 "0"(_num) \
328 : "rcx", "r11", "memory", "cc" \
329 ); \
330 _ret; \
331})
332
333#ifndef ARCH_SHSTK_ENABLE
334#define ARCH_SHSTK_ENABLE 0x5001
335#define ARCH_SHSTK_DISABLE 0x5002
336#define ARCH_SHSTK_SHSTK (1ULL << 0)
337#endif
338
339static void test_uretprobe_shadow_stack(void)
340{
341 if (ARCH_PRCTL(ARCH_SHSTK_ENABLE, ARCH_SHSTK_SHSTK)) {
342 test__skip();
343 return;
344 }
345
346 /* Run all of the uretprobe tests. */
347 test_uretprobe_regs_equal();
348 test_uretprobe_regs_change();
349 test_uretprobe_syscall_call();
350
351 ARCH_PRCTL(ARCH_SHSTK_DISABLE, ARCH_SHSTK_SHSTK);
352}
353#else
354static void test_uretprobe_regs_equal(void)
355{
356 test__skip();
357}
358
359static void test_uretprobe_regs_change(void)
360{
361 test__skip();
362}
363
364static void test_uretprobe_syscall_call(void)
365{
366 test__skip();
367}
368
369static void test_uretprobe_shadow_stack(void)
370{
371 test__skip();
372}
373#endif
374
375void test_uprobe_syscall(void)
376{
377 if (test__start_subtest("uretprobe_regs_equal"))
378 test_uretprobe_regs_equal();
379 if (test__start_subtest("uretprobe_regs_change"))
380 test_uretprobe_regs_change();
381 if (test__start_subtest("uretprobe_syscall_call"))
382 test_uretprobe_syscall_call();
383 if (test__start_subtest("uretprobe_shadow_stack"))
384 test_uretprobe_shadow_stack();
385}
tools/testing/selftests/bpf/prog_tests/uretprobe_stack.c+186−0
@@ -0,0 +1,186 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
3
4#include <test_progs.h>
5#include "uretprobe_stack.skel.h"
6#include "../sdt.h"
7
8/* We set up target_1() -> target_2() -> target_3() -> target_4() -> USDT()
9 * call chain, each being traced by our BPF program. On entry or return from
10 * each target_*() we are capturing user stack trace and recording it in
11 * global variable, so that user space part of the test can validate it.
12 *
13 * Note, we put each target function into a custom section to get those
14 * __start_XXX/__stop_XXX symbols, generated by linker for us, which allow us
15 * to know address range of those functions
16 */
17__attribute__((section("uprobe__target_4")))
18__weak int target_4(void)
19{
20 STAP_PROBE1(uretprobe_stack, target, 42);
21 return 42;
22}
23
24extern const void *__start_uprobe__target_4;
25extern const void *__stop_uprobe__target_4;
26
27__attribute__((section("uprobe__target_3")))
28__weak int target_3(void)
29{
30 return target_4();
31}
32
33extern const void *__start_uprobe__target_3;
34extern const void *__stop_uprobe__target_3;
35
36__attribute__((section("uprobe__target_2")))
37__weak int target_2(void)
38{
39 return target_3();
40}
41
42extern const void *__start_uprobe__target_2;
43extern const void *__stop_uprobe__target_2;
44
45__attribute__((section("uprobe__target_1")))
46__weak int target_1(int depth)
47{
48 if (depth < 1)
49 return 1 + target_1(depth + 1);
50 else
51 return target_2();
52}
53
54extern const void *__start_uprobe__target_1;
55extern const void *__stop_uprobe__target_1;
56
57extern const void *__start_uretprobe_stack_sec;
58extern const void *__stop_uretprobe_stack_sec;
59
60struct range {
61 long start;
62 long stop;
63};
64
65static struct range targets[] = {
66 {}, /* we want target_1 to map to target[1], so need 1-based indexing */
67 { (long)&__start_uprobe__target_1, (long)&__stop_uprobe__target_1 },
68 { (long)&__start_uprobe__target_2, (long)&__stop_uprobe__target_2 },
69 { (long)&__start_uprobe__target_3, (long)&__stop_uprobe__target_3 },
70 { (long)&__start_uprobe__target_4, (long)&__stop_uprobe__target_4 },
71};
72
73static struct range caller = {
74 (long)&__start_uretprobe_stack_sec,
75 (long)&__stop_uretprobe_stack_sec,
76};
77
78static void validate_stack(__u64 *ips, int stack_len, int cnt, ...)
79{
80 int i, j;
81 va_list args;
82
83 if (!ASSERT_GT(stack_len, 0, "stack_len"))
84 return;
85
86 stack_len /= 8;
87
88 /* check if we have enough entries to satisfy test expectations */
89 if (!ASSERT_GE(stack_len, cnt, "stack_len2"))
90 return;
91
92 if (env.verbosity >= VERBOSE_NORMAL) {
93 printf("caller: %#lx - %#lx\n", caller.start, caller.stop);
94 for (i = 1; i < ARRAY_SIZE(targets); i++)
95 printf("target_%d: %#lx - %#lx\n", i, targets[i].start, targets[i].stop);
96 for (i = 0; i < stack_len; i++) {
97 for (j = 1; j < ARRAY_SIZE(targets); j++) {
98 if (ips[i] >= targets[j].start && ips[i] < targets[j].stop)
99 break;
100 }
101 if (j < ARRAY_SIZE(targets)) { /* found target match */
102 printf("ENTRY #%d: %#lx (in target_%d)\n", i, (long)ips[i], j);
103 } else if (ips[i] >= caller.start && ips[i] < caller.stop) {
104 printf("ENTRY #%d: %#lx (in caller)\n", i, (long)ips[i]);
105 } else {
106 printf("ENTRY #%d: %#lx\n", i, (long)ips[i]);
107 }
108 }
109 }
110
111 va_start(args, cnt);
112
113 for (i = cnt - 1; i >= 0; i--) {
114 /* most recent entry is the deepest target function */
115 const struct range *t = va_arg(args, const struct range *);
116
117 ASSERT_GE(ips[i], t->start, "addr_start");
118 ASSERT_LT(ips[i], t->stop, "addr_stop");
119 }
120
121 va_end(args);
122}
123
124/* __weak prevents inlining */
125__attribute__((section("uretprobe_stack_sec")))
126__weak void test_uretprobe_stack(void)
127{
128 LIBBPF_OPTS(bpf_uprobe_opts, uprobe_opts);
129 struct uretprobe_stack *skel;
130 int err;
131
132 skel = uretprobe_stack__open_and_load();
133 if (!ASSERT_OK_PTR(skel, "skel_open"))
134 return;
135
136 err = uretprobe_stack__attach(skel);
137 if (!ASSERT_OK(err, "skel_attach"))
138 goto cleanup;
139
140 /* trigger */
141 ASSERT_EQ(target_1(0), 42 + 1, "trigger_return");
142
143 /*
144 * Stacks captured on ENTRY uprobes
145 */
146
147 /* (uprobe 1) target_1 in stack trace*/
148 validate_stack(skel->bss->entry_stack1, skel->bss->entry1_len,
149 2, &caller, &targets[1]);
150 /* (uprobe 1, recursed) */
151 validate_stack(skel->bss->entry_stack1_recur, skel->bss->entry1_recur_len,
152 3, &caller, &targets[1], &targets[1]);
153 /* (uprobe 2) caller -> target_1 -> target_1 -> target_2 */
154 validate_stack(skel->bss->entry_stack2, skel->bss->entry2_len,
155 4, &caller, &targets[1], &targets[1], &targets[2]);
156 /* (uprobe 3) */
157 validate_stack(skel->bss->entry_stack3, skel->bss->entry3_len,
158 5, &caller, &targets[1], &targets[1], &targets[2], &targets[3]);
159 /* (uprobe 4) caller -> target_1 -> target_1 -> target_2 -> target_3 -> target_4 */
160 validate_stack(skel->bss->entry_stack4, skel->bss->entry4_len,
161 6, &caller, &targets[1], &targets[1], &targets[2], &targets[3], &targets[4]);
162
163 /* (USDT): full caller -> target_1 -> target_1 -> target_2 (uretprobed)
164 * -> target_3 -> target_4 (uretprobes) chain
165 */
166 validate_stack(skel->bss->usdt_stack, skel->bss->usdt_len,
167 6, &caller, &targets[1], &targets[1], &targets[2], &targets[3], &targets[4]);
168
169 /*
170 * Now stacks captured on the way out in EXIT uprobes
171 */
172
173 /* (uretprobe 4) everything up to target_4, but excluding it */
174 validate_stack(skel->bss->exit_stack4, skel->bss->exit4_len,
175 5, &caller, &targets[1], &targets[1], &targets[2], &targets[3]);
176 /* we didn't install uretprobes on target_2 and target_3 */
177 /* (uretprobe 1, recur) first target_1 call only */
178 validate_stack(skel->bss->exit_stack1_recur, skel->bss->exit1_recur_len,
179 2, &caller, &targets[1]);
180 /* (uretprobe 1) just a caller in the stack trace */
181 validate_stack(skel->bss->exit_stack1, skel->bss->exit1_len,
182 1, &caller);
183
184cleanup:
185 uretprobe_stack__destroy(skel);
186}
tools/testing/selftests/bpf/progs/uprobe_syscall.c+15−0
@@ -0,0 +1,15 @@
1// SPDX-License-Identifier: GPL-2.0
2#include "vmlinux.h"
3#include <bpf/bpf_helpers.h>
4#include <string.h>
5
6struct pt_regs regs;
7
8char _license[] SEC("license") = "GPL";
9
10SEC("uretprobe//proc/self/exe:uretprobe_regs_trigger")
11int uretprobe(struct pt_regs *ctx)
12{
13 __builtin_memcpy(&regs, ctx, sizeof(regs));
14 return 0;
15}
tools/testing/selftests/bpf/progs/uprobe_syscall_executed.c+17−0
@@ -0,0 +1,17 @@
1// SPDX-License-Identifier: GPL-2.0
2#include "vmlinux.h"
3#include <bpf/bpf_helpers.h>
4#include <string.h>
5
6struct pt_regs regs;
7
8char _license[] SEC("license") = "GPL";
9
10int executed = 0;
11
12SEC("uretprobe.multi")
13int test(struct pt_regs *regs)
14{
15 executed = 1;
16 return 0;
17}
tools/testing/selftests/bpf/progs/uretprobe_stack.c+96−0
@@ -0,0 +1,96 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
3#include <vmlinux.h>
4#include <bpf/bpf_helpers.h>
5#include <bpf/bpf_tracing.h>
6#include <bpf/usdt.bpf.h>
7
8char _license[] SEC("license") = "GPL";
9
10__u64 entry_stack1[32], exit_stack1[32];
11__u64 entry_stack1_recur[32], exit_stack1_recur[32];
12__u64 entry_stack2[32];
13__u64 entry_stack3[32];
14__u64 entry_stack4[32], exit_stack4[32];
15__u64 usdt_stack[32];
16
17int entry1_len, exit1_len;
18int entry1_recur_len, exit1_recur_len;
19int entry2_len, exit2_len;
20int entry3_len, exit3_len;
21int entry4_len, exit4_len;
22int usdt_len;
23
24#define SZ sizeof(usdt_stack)
25
26SEC("uprobe//proc/self/exe:target_1")
27int BPF_UPROBE(uprobe_1)
28{
29 /* target_1 is recursive wit depth of 2, so we capture two separate
30 * stack traces, depending on which occurence it is
31 */
32 static bool recur = false;
33
34 if (!recur)
35 entry1_len = bpf_get_stack(ctx, &entry_stack1, SZ, BPF_F_USER_STACK);
36 else
37 entry1_recur_len = bpf_get_stack(ctx, &entry_stack1_recur, SZ, BPF_F_USER_STACK);
38
39 recur = true;
40 return 0;
41}
42
43SEC("uretprobe//proc/self/exe:target_1")
44int BPF_URETPROBE(uretprobe_1)
45{
46 /* see above, target_1 is recursive */
47 static bool recur = false;
48
49 /* NOTE: order of returns is reversed to order of entries */
50 if (!recur)
51 exit1_recur_len = bpf_get_stack(ctx, &exit_stack1_recur, SZ, BPF_F_USER_STACK);
52 else
53 exit1_len = bpf_get_stack(ctx, &exit_stack1, SZ, BPF_F_USER_STACK);
54
55 recur = true;
56 return 0;
57}
58
59SEC("uprobe//proc/self/exe:target_2")
60int BPF_UPROBE(uprobe_2)
61{
62 entry2_len = bpf_get_stack(ctx, &entry_stack2, SZ, BPF_F_USER_STACK);
63 return 0;
64}
65
66/* no uretprobe for target_2 */
67
68SEC("uprobe//proc/self/exe:target_3")
69int BPF_UPROBE(uprobe_3)
70{
71 entry3_len = bpf_get_stack(ctx, &entry_stack3, SZ, BPF_F_USER_STACK);
72 return 0;
73}
74
75/* no uretprobe for target_3 */
76
77SEC("uprobe//proc/self/exe:target_4")
78int BPF_UPROBE(uprobe_4)
79{
80 entry4_len = bpf_get_stack(ctx, &entry_stack4, SZ, BPF_F_USER_STACK);
81 return 0;
82}
83
84SEC("uretprobe//proc/self/exe:target_4")
85int BPF_URETPROBE(uretprobe_4)
86{
87 exit4_len = bpf_get_stack(ctx, &exit_stack4, SZ, BPF_F_USER_STACK);
88 return 0;
89}
90
91SEC("usdt//proc/self/exe:uretprobe_stack:target")
92int BPF_USDT(usdt_probe)
93{
94 usdt_len = bpf_get_stack(ctx, &usdt_stack, SZ, BPF_F_USER_STACK);
95 return 0;
96}
tools/testing/selftests/x86/test_shadow_stack.c+145−0
@@ -34,6 +34,7 @@ · 33 unmodified lines
34#include <sys/ptrace.h>34#include <sys/ptrace.h>
35#include <sys/signal.h>35#include <sys/signal.h>
36#include <linux/elf.h>36#include <linux/elf.h>
37#include <linux/perf_event.h>
37 38
38/*39/*
39 * Define the ABI defines if needed, so people can run the tests40 * Define the ABI defines if needed, so people can run the tests
@@ -734,6 +735,144 @@ int test_32bit(void) · 694 unmodified lines
734 return !segv_triggered;735 return !segv_triggered;
735}736}
736 737
738static int parse_uint_from_file(const char *file, const char *fmt)
739{
740 int err, ret;
741 FILE *f;
742
743 f = fopen(file, "re");
744 if (!f) {
745 err = -errno;
746 printf("failed to open '%s': %d\n", file, err);
747 return err;
748 }
749 err = fscanf(f, fmt, &ret);
750 if (err != 1) {
751 err = err == EOF ? -EIO : -errno;
752 printf("failed to parse '%s': %d\n", file, err);
753 fclose(f);
754 return err;
755 }
756 fclose(f);
757 return ret;
758}
759
760static int determine_uprobe_perf_type(void)
761{
762 const char *file = "/sys/bus/event_source/devices/uprobe/type";
763
764 return parse_uint_from_file(file, "%d\n");
765}
766
767static int determine_uprobe_retprobe_bit(void)
768{
769 const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe";
770
771 return parse_uint_from_file(file, "config:%d\n");
772}
773
774static ssize_t get_uprobe_offset(const void *addr)
775{
776 size_t start, end, base;
777 char buf[256];
778 bool found = false;
779 FILE *f;
780
781 f = fopen("/proc/self/maps", "r");
782 if (!f)
783 return -errno;
784
785 while (fscanf(f, "%zx-%zx %s %zx %*[^\n]\n", &start, &end, buf, &base) == 4) {
786 if (buf[2] == 'x' && (uintptr_t)addr >= start && (uintptr_t)addr < end) {
787 found = true;
788 break;
789 }
790 }
791
792 fclose(f);
793
794 if (!found)
795 return -ESRCH;
796
797 return (uintptr_t)addr - start + base;
798}
799
800static __attribute__((noinline)) void uretprobe_trigger(void)
801{
802 asm volatile ("");
803}
804
805/*
806 * This test setups return uprobe, which is sensitive to shadow stack
807 * (crashes without extra fix). After executing the uretprobe we fail
808 * the test if we receive SIGSEGV, no crash means we're good.
809 *
810 * Helper functions above borrowed from bpf selftests.
811 */
812static int test_uretprobe(void)
813{
814 const size_t attr_sz = sizeof(struct perf_event_attr);
815 const char *file = "/proc/self/exe";
816 int bit, fd = 0, type, err = 1;
817 struct perf_event_attr attr;
818 struct sigaction sa = {};
819 ssize_t offset;
820
821 type = determine_uprobe_perf_type();
822 if (type < 0) {
823 if (type == -ENOENT)
824 printf("[SKIP]\tUretprobe test, uprobes are not available\n");
825 return 0;
826 }
827
828 offset = get_uprobe_offset(uretprobe_trigger);
829 if (offset < 0)
830 return 1;
831
832 bit = determine_uprobe_retprobe_bit();
833 if (bit < 0)
834 return 1;
835
836 sa.sa_sigaction = segv_gp_handler;
837 sa.sa_flags = SA_SIGINFO;
838 if (sigaction(SIGSEGV, &sa, NULL))
839 return 1;
840
841 /* Setup return uprobe through perf event interface. */
842 memset(&attr, 0, attr_sz);
843 attr.size = attr_sz;
844 attr.type = type;
845 attr.config = 1 << bit;
846 attr.config1 = (__u64) (unsigned long) file;
847 attr.config2 = offset;
848
849 fd = syscall(__NR_perf_event_open, &attr, 0 /* pid */, -1 /* cpu */,
850 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
851 if (fd < 0)
852 goto out;
853
854 if (sigsetjmp(jmp_buffer, 1))
855 goto out;
856
857 ARCH_PRCTL(ARCH_SHSTK_ENABLE, ARCH_SHSTK_SHSTK);
858
859 /*
860 * This either segfaults and goes through sigsetjmp above
861 * or succeeds and we're good.
862 */
863 uretprobe_trigger();
864
865 printf("[OK]\tUretprobe test\n");
866 err = 0;
867
868out:
869 ARCH_PRCTL(ARCH_SHSTK_DISABLE, ARCH_SHSTK_SHSTK);
870 signal(SIGSEGV, SIG_DFL);
871 if (fd)
872 close(fd);
873 return err;
874}
875
737void segv_handler_ptrace(int signum, siginfo_t *si, void *uc)876void segv_handler_ptrace(int signum, siginfo_t *si, void *uc)
738{877{
739 /* The SSP adjustment caused a segfault. */878 /* The SSP adjustment caused a segfault. */
@@ -926,6 +1065,12 @@ int main(int argc, char *argv[]) · 186 unmodified lines
926 goto out;1065 goto out;
927 }1066 }
928 1067
1068 if (test_uretprobe()) {
1069 ret = 1;
1070 printf("[FAIL]\turetprobe test\n");
1071 goto out;
1072 }
1073
929 return ret;1074 return ret;
930 1075
931out:1076out: