History

uprobes/core: Handle breakpoint and singlestep exceptions

Uprobes uses exception notifiers to get to know if a thread hit a breakpoint or a singlestep exception. When a thread hits a uprobe or is singlestepping post a uprobe hit, the uprobe exception notifier sets its TIF_UPROBE bit, which will then be checked on its return to userspace path (do_notify_resume() ->uprobe_notify_resume()), where the consumers handlers are run (in task context) based on the defined filters. Uprobe hits are thread specific and hence we need to maintain information about if a task hit a uprobe, what uprobe was hit, the slot where the original instruction was copied for xol so that it can be singlestepped with appropriate fixups. In some cases, special care is needed for instructions that are executed out of line (xol). These are architecture specific artefacts, such as handling RIP relative instructions on x86_64. Since the instruction at which the uprobe was inserted is executed out of line, architecture specific fixups are added so that the thread continues normal execution in the presence of a uprobe. Postpone the signals until we execute the probed insn. post_xol() path does a recalc_sigpending() before return to user-mode, this ensures the signal can't be lost. Uprobes relies on DIE_DEBUG notification to notify if a singlestep is complete. Adds x86 specific uprobe exception notifiers and appropriate hooks needed to determine a uprobe hit and subsequent post processing. Add requisite x86 fixups for xol for uprobes. Specific cases needing fixups include relative jumps (x86_64), calls, etc. Where possible, we check and skip singlestepping the breakpointed instructions. For now we skip single byte as well as few multibyte nop instructions. However this can be extended to other instructions too. Credits to Oleg Nesterov for suggestions/patches related to signal, breakpoint, singlestep handling code.

Srikar Dronamraju committed 14 years ago

Commit 0326f5a1 parent ef15eda9 files · +664 −15Browse files

Unified · Split

9 files
  1. arch/x86/include/asm/thread_info.h +2 −0
  2. arch/x86/include/asm/uprobes.h +15 −1
  3. arch/x86/kernel/signal.c +6 −0
  4. arch/x86/kernel/uprobes.c +258 −7
  5. include/linux/sched.h +4 −0
  6. include/linux/uprobes.h +52 −3
  7. kernel/events/uprobes.c +319 −4
  8. kernel/fork.c +4 −0
  9. kernel/signal.c +4 −0
arch/x86/include/asm/thread_info.h+2−0
@@ -85,6 +85,7 @@ struct thread_info { · 84 unmodified lines
8585 #define TIF_SECCOMP 8 /* secure computing */
8686 #define TIF_MCE_NOTIFY 10 /* notify userspace of an MCE */
8787 #define TIF_USER_RETURN_NOTIFY 11 /* notify kernel of userspace return */
88+#define TIF_UPROBE 12 /* breakpointed or singlestepping */
8889 #define TIF_NOTSC 16 /* TSC is not accessible in userland */
8990 #define TIF_IA32 17 /* IA32 compatibility process */
9091 #define TIF_FORK 18 /* ret_from_fork */
@@ -109,6 +110,7 @@ struct thread_info { · 18 unmodified lines
109110 #define _TIF_SECCOMP (1 << TIF_SECCOMP)
110111 #define _TIF_MCE_NOTIFY (1 << TIF_MCE_NOTIFY)
111112 #define _TIF_USER_RETURN_NOTIFY (1 << TIF_USER_RETURN_NOTIFY)
113+#define _TIF_UPROBE (1 << TIF_UPROBE)
112114 #define _TIF_NOTSC (1 << TIF_NOTSC)
113115 #define _TIF_IA32 (1 << TIF_IA32)
114116 #define _TIF_FORK (1 << TIF_FORK)
arch/x86/include/asm/uprobes.h+15−1
@@ -23,6 +23,8 @@ · 22 unmodified lines
2323 * Jim Keniston
2424 */
2525
26+#include <linux/notifier.h>
27+
2628 typedef u8 uprobe_opcode_t;
2729
2830 #define MAX_UINSN_BYTES 16
@@ -39,5 +41,17 @@ struct arch_uprobe { · 10 unmodified lines
3941 #endif
4042 };
4143
42−extern int arch_uprobes_analyze_insn(struct arch_uprobe *aup, struct mm_struct *mm);
44+struct arch_uprobe_task {
45+ unsigned long saved_trap_nr;
46+#ifdef CONFIG_X86_64
47+ unsigned long saved_scratch_register;
48+#endif
49+};
50+
51+extern int arch_uprobe_analyze_insn(struct arch_uprobe *aup, struct mm_struct *mm);
52+extern int arch_uprobe_pre_xol(struct arch_uprobe *aup, struct pt_regs *regs);
53+extern int arch_uprobe_post_xol(struct arch_uprobe *aup, struct pt_regs *regs);
54+extern bool arch_uprobe_xol_was_trapped(struct task_struct *tsk);
55+extern int arch_uprobe_exception_notify(struct notifier_block *self, unsigned long val, void *data);
56+extern void arch_uprobe_abort_xol(struct arch_uprobe *aup, struct pt_regs *regs);
4357 #endif /* _ASM_UPROBES_H */
arch/x86/kernel/signal.c+6−0
@@ -18,6 +18,7 @@ · 17 unmodified lines
1818 #include <linux/personality.h>
1919 #include <linux/uaccess.h>
2020 #include <linux/user-return-notifier.h>
21+#include <linux/uprobes.h>
2122
2223 #include <asm/processor.h>
2324 #include <asm/ucontext.h>
@@ -823,6 +824,11 @@ do_notify_resume(struct pt_regs *regs, void *unused, __u32 thread_info_flags) · 799 unmodified lines
823824 mce_notify_process();
824825 #endif /* CONFIG_X86_64 && CONFIG_X86_MCE */
825826
827+ if (thread_info_flags & _TIF_UPROBE) {
828+ clear_thread_flag(TIF_UPROBE);
829+ uprobe_notify_resume(regs);
830+ }
831+
826832 /* deal with pending signal delivery */
827833 if (thread_info_flags & _TIF_SIGPENDING)
828834 do_signal(regs);
arch/x86/kernel/uprobes.c+258−7
@@ -24,22 +24,28 @@ · 23 unmodified lines
2424 #include <linux/sched.h>
2525 #include <linux/ptrace.h>
2626 #include <linux/uprobes.h>
27+#include <linux/uaccess.h>
2728
2829 #include <linux/kdebug.h>
30+#include <asm/processor.h>
2931 #include <asm/insn.h>
3032
3133 /* Post-execution fixups. */
3234
3335 /* No fixup needed */
34−#define UPROBE_FIX_NONE 0x0
36+#define UPROBE_FIX_NONE 0x0
37+
3538 /* Adjust IP back to vicinity of actual insn */
3639 #define UPROBE_FIX_IP 0x1
40+
3741 /* Adjust the return address of a call insn */
3842 #define UPROBE_FIX_CALL 0x2
3943
4044 #define UPROBE_FIX_RIP_AX 0x8000
4145 #define UPROBE_FIX_RIP_CX 0x4000
4246
47+#define UPROBE_TRAP_NR UINT_MAX
48+
4349 /* Adaptations for mhiramat x86 decoder v14. */
4450 #define OPCODE1(insn) ((insn)->opcode.bytes[0])
4551 #define OPCODE2(insn) ((insn)->opcode.bytes[1])
@@ -221,10 +227,9 @@ static int validate_insn_32bits(struct arch_uprobe *auprobe, struct insn *insn) · 175 unmodified lines
221227 }
222228
223229 /*
224− * Figure out which fixups post_xol() will need to perform, and annotate
225− * arch_uprobe->fixups accordingly. To start with,
226− * arch_uprobe->fixups is either zero or it reflects rip-related
227− * fixups.
230+ * Figure out which fixups arch_uprobe_post_xol() will need to perform, and
231+ * annotate arch_uprobe->fixups accordingly. To start with,
232+ * arch_uprobe->fixups is either zero or it reflects rip-related fixups.
228233 */
229234 static void prepare_fixups(struct arch_uprobe *auprobe, struct insn *insn)
230235 {
@@ -401,12 +406,12 @@ static int validate_insn_bits(struct arch_uprobe *auprobe, struct mm_struct *mm, · 170 unmodified lines
401406 #endif /* CONFIG_X86_64 */
402407
403408 /**
404− * arch_uprobes_analyze_insn - instruction analysis including validity and fixups.
409+ * arch_uprobe_analyze_insn - instruction analysis including validity and fixups.
405410 * @mm: the probed address space.
406411 * @arch_uprobe: the probepoint information.
407412 * Return 0 on success or a -ve number on error.
408413 */
409−int arch_uprobes_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm)
414+int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm)
410415 {
411416 int ret;
412417 struct insn insn;
@@ -421,3 +426,249 @@ int arch_uprobes_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm) · 8 unmodified lines
421426
422427 return 0;
423428 }
429+
430+#ifdef CONFIG_X86_64
431+/*
432+ * If we're emulating a rip-relative instruction, save the contents
433+ * of the scratch register and store the target address in that register.
434+ */
435+static void
436+pre_xol_rip_insn(struct arch_uprobe *auprobe, struct pt_regs *regs,
437+ struct arch_uprobe_task *autask)
438+{
439+ if (auprobe->fixups & UPROBE_FIX_RIP_AX) {
440+ autask->saved_scratch_register = regs->ax;
441+ regs->ax = current->utask->vaddr;
442+ regs->ax += auprobe->rip_rela_target_address;
443+ } else if (auprobe->fixups & UPROBE_FIX_RIP_CX) {
444+ autask->saved_scratch_register = regs->cx;
445+ regs->cx = current->utask->vaddr;
446+ regs->cx += auprobe->rip_rela_target_address;
447+ }
448+}
449+#else
450+static void
451+pre_xol_rip_insn(struct arch_uprobe *auprobe, struct pt_regs *regs,
452+ struct arch_uprobe_task *autask)
453+{
454+ /* No RIP-relative addressing on 32-bit */
455+}
456+#endif
457+
458+/*
459+ * arch_uprobe_pre_xol - prepare to execute out of line.
460+ * @auprobe: the probepoint information.
461+ * @regs: reflects the saved user state of current task.
462+ */
463+int arch_uprobe_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
464+{
465+ struct arch_uprobe_task *autask;
466+
467+ autask = &current->utask->autask;
468+ autask->saved_trap_nr = current->thread.trap_nr;
469+ current->thread.trap_nr = UPROBE_TRAP_NR;
470+ regs->ip = current->utask->xol_vaddr;
471+ pre_xol_rip_insn(auprobe, regs, autask);
472+
473+ return 0;
474+}
475+
476+/*
477+ * This function is called by arch_uprobe_post_xol() to adjust the return
478+ * address pushed by a call instruction executed out of line.
479+ */
480+static int adjust_ret_addr(unsigned long sp, long correction)
481+{
482+ int rasize, ncopied;
483+ long ra = 0;
484+
485+ if (is_ia32_task())
486+ rasize = 4;
487+ else
488+ rasize = 8;
489+
490+ ncopied = copy_from_user(&ra, (void __user *)sp, rasize);
491+ if (unlikely(ncopied))
492+ return -EFAULT;
493+
494+ ra += correction;
495+ ncopied = copy_to_user((void __user *)sp, &ra, rasize);
496+ if (unlikely(ncopied))
497+ return -EFAULT;
498+
499+ return 0;
500+}
501+
502+#ifdef CONFIG_X86_64
503+static bool is_riprel_insn(struct arch_uprobe *auprobe)
504+{
505+ return ((auprobe->fixups & (UPROBE_FIX_RIP_AX | UPROBE_FIX_RIP_CX)) != 0);
506+}
507+
508+static void
509+handle_riprel_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs, long *correction)
510+{
511+ if (is_riprel_insn(auprobe)) {
512+ struct arch_uprobe_task *autask;
513+
514+ autask = &current->utask->autask;
515+ if (auprobe->fixups & UPROBE_FIX_RIP_AX)
516+ regs->ax = autask->saved_scratch_register;
517+ else
518+ regs->cx = autask->saved_scratch_register;
519+
520+ /*
521+ * The original instruction includes a displacement, and so
522+ * is 4 bytes longer than what we've just single-stepped.
523+ * Fall through to handle stuff like "jmpq *...(%rip)" and
524+ * "callq *...(%rip)".
525+ */
526+ if (correction)
527+ *correction += 4;
528+ }
529+}
530+#else
531+static void
532+handle_riprel_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs, long *correction)
533+{
534+ /* No RIP-relative addressing on 32-bit */
535+}
536+#endif
537+
538+/*
539+ * If xol insn itself traps and generates a signal(Say,
540+ * SIGILL/SIGSEGV/etc), then detect the case where a singlestepped
541+ * instruction jumps back to its own address. It is assumed that anything
542+ * like do_page_fault/do_trap/etc sets thread.trap_nr != -1.
543+ *
544+ * arch_uprobe_pre_xol/arch_uprobe_post_xol save/restore thread.trap_nr,
545+ * arch_uprobe_xol_was_trapped() simply checks that ->trap_nr is not equal to
546+ * UPROBE_TRAP_NR == -1 set by arch_uprobe_pre_xol().
547+ */
548+bool arch_uprobe_xol_was_trapped(struct task_struct *t)
549+{
550+ if (t->thread.trap_nr != UPROBE_TRAP_NR)
551+ return true;
552+
553+ return false;
554+}
555+
556+/*
557+ * Called after single-stepping. To avoid the SMP problems that can
558+ * occur when we temporarily put back the original opcode to
559+ * single-step, we single-stepped a copy of the instruction.
560+ *
561+ * This function prepares to resume execution after the single-step.
562+ * We have to fix things up as follows:
563+ *
564+ * Typically, the new ip is relative to the copied instruction. We need
565+ * to make it relative to the original instruction (FIX_IP). Exceptions
566+ * are return instructions and absolute or indirect jump or call instructions.
567+ *
568+ * If the single-stepped instruction was a call, the return address that
569+ * is atop the stack is the address following the copied instruction. We
570+ * need to make it the address following the original instruction (FIX_CALL).
571+ *
572+ * If the original instruction was a rip-relative instruction such as
573+ * "movl %edx,0xnnnn(%rip)", we have instead executed an equivalent
574+ * instruction using a scratch register -- e.g., "movl %edx,(%rax)".
575+ * We need to restore the contents of the scratch register and adjust
576+ * the ip, keeping in mind that the instruction we executed is 4 bytes
577+ * shorter than the original instruction (since we squeezed out the offset
578+ * field). (FIX_RIP_AX or FIX_RIP_CX)
579+ */
580+int arch_uprobe_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
581+{
582+ struct uprobe_task *utask;
583+ long correction;
584+ int result = 0;
585+
586+ WARN_ON_ONCE(current->thread.trap_nr != UPROBE_TRAP_NR);
587+
588+ utask = current->utask;
589+ current->thread.trap_nr = utask->autask.saved_trap_nr;
590+ correction = (long)(utask->vaddr - utask->xol_vaddr);
591+ handle_riprel_post_xol(auprobe, regs, &correction);
592+ if (auprobe->fixups & UPROBE_FIX_IP)
593+ regs->ip += correction;
594+
595+ if (auprobe->fixups & UPROBE_FIX_CALL)
596+ result = adjust_ret_addr(regs->sp, correction);
597+
598+ return result;
599+}
600+
601+/* callback routine for handling exceptions. */
602+int arch_uprobe_exception_notify(struct notifier_block *self, unsigned long val, void *data)
603+{
604+ struct die_args *args = data;
605+ struct pt_regs *regs = args->regs;
606+ int ret = NOTIFY_DONE;
607+
608+ /* We are only interested in userspace traps */
609+ if (regs && !user_mode_vm(regs))
610+ return NOTIFY_DONE;
611+
612+ switch (val) {
613+ case DIE_INT3:
614+ if (uprobe_pre_sstep_notifier(regs))
615+ ret = NOTIFY_STOP;
616+
617+ break;
618+
619+ case DIE_DEBUG:
620+ if (uprobe_post_sstep_notifier(regs))
621+ ret = NOTIFY_STOP;
622+
623+ default:
624+ break;
625+ }
626+
627+ return ret;
628+}
629+
630+/*
631+ * This function gets called when XOL instruction either gets trapped or
632+ * the thread has a fatal signal, so reset the instruction pointer to its
633+ * probed address.
634+ */
635+void arch_uprobe_abort_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
636+{
637+ struct uprobe_task *utask = current->utask;
638+
639+ current->thread.trap_nr = utask->autask.saved_trap_nr;
640+ handle_riprel_post_xol(auprobe, regs, NULL);
641+ instruction_pointer_set(regs, utask->vaddr);
642+}
643+
644+/*
645+ * Skip these instructions as per the currently known x86 ISA.
646+ * 0x66* { 0x90 | 0x0f 0x1f | 0x0f 0x19 | 0x87 0xc0 }
647+ */
648+bool arch_uprobe_skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs)
649+{
650+ int i;
651+
652+ for (i = 0; i < MAX_UINSN_BYTES; i++) {
653+ if ((auprobe->insn[i] == 0x66))
654+ continue;
655+
656+ if (auprobe->insn[i] == 0x90)
657+ return true;
658+
659+ if (i == (MAX_UINSN_BYTES - 1))
660+ break;
661+
662+ if ((auprobe->insn[i] == 0x0f) && (auprobe->insn[i+1] == 0x1f))
663+ return true;
664+
665+ if ((auprobe->insn[i] == 0x0f) && (auprobe->insn[i+1] == 0x19))
666+ return true;
667+
668+ if ((auprobe->insn[i] == 0x87) && (auprobe->insn[i+1] == 0xc0))
669+ return true;
670+
671+ break;
672+ }
673+ return false;
674+}
include/linux/sched.h+4−0
@@ -1590,6 +1590,10 @@ struct task_struct { · 1589 unmodified lines
15901590 #ifdef CONFIG_HAVE_HW_BREAKPOINT
15911591 atomic_t ptrace_bp_refcnt;
15921592 #endif
1593+#ifdef CONFIG_UPROBES
1594+ struct uprobe_task *utask;
1595+ int uprobe_srcu_id;
1596+#endif
15931597 };
15941598
15951599 /* Future-safe accessor for struct task_struct's cpus_allowed. */
include/linux/uprobes.h+52−3
@@ -28,8 +28,9 @@ · 27 unmodified lines
2828 #include <linux/rbtree.h>
2929
3030 struct vm_area_struct;
31+
3132 #ifdef CONFIG_ARCH_SUPPORTS_UPROBES
32−#include <asm/uprobes.h>
33+# include <asm/uprobes.h>
3334 #endif
3435
3536 /* flags that denote/change uprobes behaviour */
@@ -39,6 +40,8 @@ struct vm_area_struct; · 3 unmodified lines
3940
4041 /* Dont run handlers when first register/ last unregister in progress*/
4142 #define UPROBE_RUN_HANDLER 0x2
43+/* Can skip singlestep */
44+#define UPROBE_SKIP_SSTEP 0x4
4245
4346 struct uprobe_consumer {
4447 int (*handler)(struct uprobe_consumer *self, struct pt_regs *regs);
@@ -52,13 +55,42 @@ struct uprobe_consumer { · 7 unmodified lines
5255 };
5356
5457 #ifdef CONFIG_UPROBES
58+enum uprobe_task_state {
59+ UTASK_RUNNING,
60+ UTASK_BP_HIT,
61+ UTASK_SSTEP,
62+ UTASK_SSTEP_ACK,
63+ UTASK_SSTEP_TRAPPED,
64+};
65+
66+/*
67+ * uprobe_task: Metadata of a task while it singlesteps.
68+ */
69+struct uprobe_task {
70+ enum uprobe_task_state state;
71+ struct arch_uprobe_task autask;
72+
73+ struct uprobe *active_uprobe;
74+
75+ unsigned long xol_vaddr;
76+ unsigned long vaddr;
77+};
78+
5579 extern int __weak set_swbp(struct arch_uprobe *aup, struct mm_struct *mm, unsigned long vaddr);
5680 extern int __weak set_orig_insn(struct arch_uprobe *aup, struct mm_struct *mm, unsigned long vaddr, bool verify);
5781 extern bool __weak is_swbp_insn(uprobe_opcode_t *insn);
5882 extern int uprobe_register(struct inode *inode, loff_t offset, struct uprobe_consumer *uc);
5983 extern void uprobe_unregister(struct inode *inode, loff_t offset, struct uprobe_consumer *uc);
6084 extern int uprobe_mmap(struct vm_area_struct *vma);
61−#else /* CONFIG_UPROBES is not defined */
85+extern void uprobe_free_utask(struct task_struct *t);
86+extern void uprobe_copy_process(struct task_struct *t);
87+extern unsigned long __weak uprobe_get_swbp_addr(struct pt_regs *regs);
88+extern int uprobe_post_sstep_notifier(struct pt_regs *regs);
89+extern int uprobe_pre_sstep_notifier(struct pt_regs *regs);
90+extern void uprobe_notify_resume(struct pt_regs *regs);
91+extern bool uprobe_deny_signal(void);
92+extern bool __weak arch_uprobe_skip_sstep(struct arch_uprobe *aup, struct pt_regs *regs);
93+#else /* !CONFIG_UPROBES */
6294 static inline int
6395 uprobe_register(struct inode *inode, loff_t offset, struct uprobe_consumer *uc)
6496 {
@@ -72,5 +104,22 @@ static inline int uprobe_mmap(struct vm_area_struct *vma) · 7 unmodified lines
72104 {
73105 return 0;
74106 }
75−#endif /* CONFIG_UPROBES */
107+static inline void uprobe_notify_resume(struct pt_regs *regs)
108+{
109+}
110+static inline bool uprobe_deny_signal(void)
111+{
112+ return false;
113+}
114+static inline unsigned long uprobe_get_swbp_addr(struct pt_regs *regs)
115+{
116+ return 0;
117+}
118+static inline void uprobe_free_utask(struct task_struct *t)
119+{
120+}
121+static inline void uprobe_copy_process(struct task_struct *t)
122+{
123+}
124+#endif /* !CONFIG_UPROBES */
76125 #endif /* _LINUX_UPROBES_H */
kernel/events/uprobes.c+319−4
@@ -30,9 +30,12 @@ · 29 unmodified lines
3030 #include <linux/rmap.h> /* anon_vma_prepare */
3131 #include <linux/mmu_notifier.h> /* set_pte_at_notify */
3232 #include <linux/swap.h> /* try_to_free_swap */
33+#include <linux/ptrace.h> /* user_enable_single_step */
34+#include <linux/kdebug.h> /* notifier mechanism */
3335
3436 #include <linux/uprobes.h>
3537
38+static struct srcu_struct uprobes_srcu;
3639 static struct rb_root uprobes_tree = RB_ROOT;
3740
3841 static DEFINE_SPINLOCK(uprobes_treelock); /* serialize rbtree access */
@@ -486,6 +489,9 @@ static struct uprobe *insert_uprobe(struct uprobe *uprobe) · 447 unmodified lines
486489 u = __insert_uprobe(uprobe);
487490 spin_unlock_irqrestore(&uprobes_treelock, flags);
488491
492+ /* For now assume that the instruction need not be single-stepped */
493+ uprobe->flags |= UPROBE_SKIP_SSTEP;
494+
489495 return u;
490496 }
491497
@@ -523,6 +529,21 @@ static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset) · 31 unmodified lines
523529 return uprobe;
524530 }
525531
532+static void handler_chain(struct uprobe *uprobe, struct pt_regs *regs)
533+{
534+ struct uprobe_consumer *uc;
535+
536+ if (!(uprobe->flags & UPROBE_RUN_HANDLER))
537+ return;
538+
539+ down_read(&uprobe->consumer_rwsem);
540+ for (uc = uprobe->consumers; uc; uc = uc->next) {
541+ if (!uc->filter || uc->filter(uc, current))
542+ uc->handler(uc, regs);
543+ }
544+ up_read(&uprobe->consumer_rwsem);
545+}
546+
526547 /* Returns the previous consumer */
527548 static struct uprobe_consumer *
528549 consumer_add(struct uprobe *uprobe, struct uprobe_consumer *uc)
@@ -645,7 +666,7 @@ install_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, · 116 unmodified lines
645666 if (is_swbp_insn((uprobe_opcode_t *)uprobe->arch.insn))
646667 return -EEXIST;
647668
648− ret = arch_uprobes_analyze_insn(&uprobe->arch, mm);
669+ ret = arch_uprobe_analyze_insn(&uprobe->arch, mm);
649670 if (ret)
650671 return ret;
651672
@@ -662,10 +683,21 @@ remove_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, loff_t vaddr) · 10 unmodified lines
662683 set_orig_insn(&uprobe->arch, mm, (unsigned long)vaddr, true);
663684 }
664685
686+/*
687+ * There could be threads that have hit the breakpoint and are entering the
688+ * notifier code and trying to acquire the uprobes_treelock. The thread
689+ * calling delete_uprobe() that is removing the uprobe from the rb_tree can
690+ * race with these threads and might acquire the uprobes_treelock compared
691+ * to some of the breakpoint hit threads. In such a case, the breakpoint
692+ * hit threads will not find the uprobe. The current unregistering thread
693+ * waits till all other threads have hit a breakpoint, to acquire the
694+ * uprobes_treelock before the uprobe is removed from the rbtree.
695+ */
665696 static void delete_uprobe(struct uprobe *uprobe)
666697 {
667698 unsigned long flags;
668699
700+ synchronize_srcu(&uprobes_srcu);
669701 spin_lock_irqsave(&uprobes_treelock, flags);
670702 rb_erase(&uprobe->rb_node, &uprobes_tree);
671703 spin_unlock_irqrestore(&uprobes_treelock, flags);
@@ -1010,6 +1042,288 @@ int uprobe_mmap(struct vm_area_struct *vma) · 338 unmodified lines
10101042 return ret;
10111043 }
10121044
1045+/**
1046+ * uprobe_get_swbp_addr - compute address of swbp given post-swbp regs
1047+ * @regs: Reflects the saved state of the task after it has hit a breakpoint
1048+ * instruction.
1049+ * Return the address of the breakpoint instruction.
1050+ */
1051+unsigned long __weak uprobe_get_swbp_addr(struct pt_regs *regs)
1052+{
1053+ return instruction_pointer(regs) - UPROBE_SWBP_INSN_SIZE;
1054+}
1055+
1056+/*
1057+ * Called with no locks held.
1058+ * Called in context of a exiting or a exec-ing thread.
1059+ */
1060+void uprobe_free_utask(struct task_struct *t)
1061+{
1062+ struct uprobe_task *utask = t->utask;
1063+
1064+ if (t->uprobe_srcu_id != -1)
1065+ srcu_read_unlock_raw(&uprobes_srcu, t->uprobe_srcu_id);
1066+
1067+ if (!utask)
1068+ return;
1069+
1070+ if (utask->active_uprobe)
1071+ put_uprobe(utask->active_uprobe);
1072+
1073+ kfree(utask);
1074+ t->utask = NULL;
1075+}
1076+
1077+/*
1078+ * Called in context of a new clone/fork from copy_process.
1079+ */
1080+void uprobe_copy_process(struct task_struct *t)
1081+{
1082+ t->utask = NULL;
1083+ t->uprobe_srcu_id = -1;
1084+}
1085+
1086+/*
1087+ * Allocate a uprobe_task object for the task.
1088+ * Called when the thread hits a breakpoint for the first time.
1089+ *
1090+ * Returns:
1091+ * - pointer to new uprobe_task on success
1092+ * - NULL otherwise
1093+ */
1094+static struct uprobe_task *add_utask(void)
1095+{
1096+ struct uprobe_task *utask;
1097+
1098+ utask = kzalloc(sizeof *utask, GFP_KERNEL);
1099+ if (unlikely(!utask))
1100+ return NULL;
1101+
1102+ utask->active_uprobe = NULL;
1103+ current->utask = utask;
1104+ return utask;
1105+}
1106+
1107+/* Prepare to single-step probed instruction out of line. */
1108+static int
1109+pre_ssout(struct uprobe *uprobe, struct pt_regs *regs, unsigned long vaddr)
1110+{
1111+ return -EFAULT;
1112+}
1113+
1114+/*
1115+ * If we are singlestepping, then ensure this thread is not connected to
1116+ * non-fatal signals until completion of singlestep. When xol insn itself
1117+ * triggers the signal, restart the original insn even if the task is
1118+ * already SIGKILL'ed (since coredump should report the correct ip). This
1119+ * is even more important if the task has a handler for SIGSEGV/etc, The
1120+ * _same_ instruction should be repeated again after return from the signal
1121+ * handler, and SSTEP can never finish in this case.
1122+ */
1123+bool uprobe_deny_signal(void)
1124+{
1125+ struct task_struct *t = current;
1126+ struct uprobe_task *utask = t->utask;
1127+
1128+ if (likely(!utask || !utask->active_uprobe))
1129+ return false;
1130+
1131+ WARN_ON_ONCE(utask->state != UTASK_SSTEP);
1132+
1133+ if (signal_pending(t)) {
1134+ spin_lock_irq(&t->sighand->siglock);
1135+ clear_tsk_thread_flag(t, TIF_SIGPENDING);
1136+ spin_unlock_irq(&t->sighand->siglock);
1137+
1138+ if (__fatal_signal_pending(t) || arch_uprobe_xol_was_trapped(t)) {
1139+ utask->state = UTASK_SSTEP_TRAPPED;
1140+ set_tsk_thread_flag(t, TIF_UPROBE);
1141+ set_tsk_thread_flag(t, TIF_NOTIFY_RESUME);
1142+ }
1143+ }
1144+
1145+ return true;
1146+}
1147+
1148+/*
1149+ * Avoid singlestepping the original instruction if the original instruction
1150+ * is a NOP or can be emulated.
1151+ */
1152+static bool can_skip_sstep(struct uprobe *uprobe, struct pt_regs *regs)
1153+{
1154+ if (arch_uprobe_skip_sstep(&uprobe->arch, regs))
1155+ return true;
1156+
1157+ uprobe->flags &= ~UPROBE_SKIP_SSTEP;
1158+ return false;
1159+}
1160+
1161+/*
1162+ * Run handler and ask thread to singlestep.
1163+ * Ensure all non-fatal signals cannot interrupt thread while it singlesteps.
1164+ */
1165+static void handle_swbp(struct pt_regs *regs)
1166+{
1167+ struct vm_area_struct *vma;
1168+ struct uprobe_task *utask;
1169+ struct uprobe *uprobe;
1170+ struct mm_struct *mm;
1171+ unsigned long bp_vaddr;
1172+
1173+ uprobe = NULL;
1174+ bp_vaddr = uprobe_get_swbp_addr(regs);
1175+ mm = current->mm;
1176+ down_read(&mm->mmap_sem);
1177+ vma = find_vma(mm, bp_vaddr);
1178+
1179+ if (vma && vma->vm_start <= bp_vaddr && valid_vma(vma, false)) {
1180+ struct inode *inode;
1181+ loff_t offset;
1182+
1183+ inode = vma->vm_file->f_mapping->host;
1184+ offset = bp_vaddr - vma->vm_start;
1185+ offset += (vma->vm_pgoff << PAGE_SHIFT);
1186+ uprobe = find_uprobe(inode, offset);
1187+ }
1188+
1189+ srcu_read_unlock_raw(&uprobes_srcu, current->uprobe_srcu_id);
1190+ current->uprobe_srcu_id = -1;
1191+ up_read(&mm->mmap_sem);
1192+
1193+ if (!uprobe) {
1194+ /* No matching uprobe; signal SIGTRAP. */
1195+ send_sig(SIGTRAP, current, 0);
1196+ return;
1197+ }
1198+
1199+ utask = current->utask;
1200+ if (!utask) {
1201+ utask = add_utask();
1202+ /* Cannot allocate; re-execute the instruction. */
1203+ if (!utask)
1204+ goto cleanup_ret;
1205+ }
1206+ utask->active_uprobe = uprobe;
1207+ handler_chain(uprobe, regs);
1208+ if (uprobe->flags & UPROBE_SKIP_SSTEP && can_skip_sstep(uprobe, regs))
1209+ goto cleanup_ret;
1210+
1211+ utask->state = UTASK_SSTEP;
1212+ if (!pre_ssout(uprobe, regs, bp_vaddr)) {
1213+ user_enable_single_step(current);
1214+ return;
1215+ }
1216+
1217+cleanup_ret:
1218+ if (utask) {
1219+ utask->active_uprobe = NULL;
1220+ utask->state = UTASK_RUNNING;
1221+ }
1222+ if (uprobe) {
1223+ if (!(uprobe->flags & UPROBE_SKIP_SSTEP))
1224+
1225+ /*
1226+ * cannot singlestep; cannot skip instruction;
1227+ * re-execute the instruction.
1228+ */
1229+ instruction_pointer_set(regs, bp_vaddr);
1230+
1231+ put_uprobe(uprobe);
1232+ }
1233+}
1234+
1235+/*
1236+ * Perform required fix-ups and disable singlestep.
1237+ * Allow pending signals to take effect.
1238+ */
1239+static void handle_singlestep(struct uprobe_task *utask, struct pt_regs *regs)
1240+{
1241+ struct uprobe *uprobe;
1242+
1243+ uprobe = utask->active_uprobe;
1244+ if (utask->state == UTASK_SSTEP_ACK)
1245+ arch_uprobe_post_xol(&uprobe->arch, regs);
1246+ else if (utask->state == UTASK_SSTEP_TRAPPED)
1247+ arch_uprobe_abort_xol(&uprobe->arch, regs);
1248+ else
1249+ WARN_ON_ONCE(1);
1250+
1251+ put_uprobe(uprobe);
1252+ utask->active_uprobe = NULL;
1253+ utask->state = UTASK_RUNNING;
1254+ user_disable_single_step(current);
1255+
1256+ spin_lock_irq(&current->sighand->siglock);
1257+ recalc_sigpending(); /* see uprobe_deny_signal() */
1258+ spin_unlock_irq(&current->sighand->siglock);
1259+}
1260+
1261+/*
1262+ * On breakpoint hit, breakpoint notifier sets the TIF_UPROBE flag. (and on
1263+ * subsequent probe hits on the thread sets the state to UTASK_BP_HIT) and
1264+ * allows the thread to return from interrupt.
1265+ *
1266+ * On singlestep exception, singlestep notifier sets the TIF_UPROBE flag and
1267+ * also sets the state to UTASK_SSTEP_ACK and allows the thread to return from
1268+ * interrupt.
1269+ *
1270+ * While returning to userspace, thread notices the TIF_UPROBE flag and calls
1271+ * uprobe_notify_resume().
1272+ */
1273+void uprobe_notify_resume(struct pt_regs *regs)
1274+{
1275+ struct uprobe_task *utask;
1276+
1277+ utask = current->utask;
1278+ if (!utask || utask->state == UTASK_BP_HIT)
1279+ handle_swbp(regs);
1280+ else
1281+ handle_singlestep(utask, regs);
1282+}
1283+
1284+/*
1285+ * uprobe_pre_sstep_notifier gets called from interrupt context as part of
1286+ * notifier mechanism. Set TIF_UPROBE flag and indicate breakpoint hit.
1287+ */
1288+int uprobe_pre_sstep_notifier(struct pt_regs *regs)
1289+{
1290+ struct uprobe_task *utask;
1291+
1292+ if (!current->mm)
1293+ return 0;
1294+
1295+ utask = current->utask;
1296+ if (utask)
1297+ utask->state = UTASK_BP_HIT;
1298+
1299+ set_thread_flag(TIF_UPROBE);
1300+ current->uprobe_srcu_id = srcu_read_lock_raw(&uprobes_srcu);
1301+
1302+ return 1;
1303+}
1304+
1305+/*
1306+ * uprobe_post_sstep_notifier gets called in interrupt context as part of notifier
1307+ * mechanism. Set TIF_UPROBE flag and indicate completion of singlestep.
1308+ */
1309+int uprobe_post_sstep_notifier(struct pt_regs *regs)
1310+{
1311+ struct uprobe_task *utask = current->utask;
1312+
1313+ if (!current->mm || !utask || !utask->active_uprobe)
1314+ /* task is currently not uprobed */
1315+ return 0;
1316+
1317+ utask->state = UTASK_SSTEP_ACK;
1318+ set_thread_flag(TIF_UPROBE);
1319+ return 1;
1320+}
1321+
1322+static struct notifier_block uprobe_exception_nb = {
1323+ .notifier_call = arch_uprobe_exception_notify,
1324+ .priority = INT_MAX-1, /* notified after kprobes, kgdb */
1325+};
1326+
10131327 static int __init init_uprobes(void)
10141328 {
10151329 int i;
@@ -1018,12 +1332,13 @@ static int __init init_uprobes(void) · 2 unmodified lines
10181332 mutex_init(&uprobes_mutex[i]);
10191333 mutex_init(&uprobes_mmap_mutex[i]);
10201334 }
1021− return 0;
1335+ init_srcu_struct(&uprobes_srcu);
1336+
1337+ return register_die_notifier(&uprobe_exception_nb);
10221338 }
1339+module_init(init_uprobes);
10231340
10241341 static void __exit exit_uprobes(void)
10251342 {
10261343 }
1027−
1028−module_init(init_uprobes);
10291344 module_exit(exit_uprobes);
kernel/fork.c+4−0
@@ -67,6 +67,7 @@ · 66 unmodified lines
6767 #include <linux/oom.h>
6868 #include <linux/khugepaged.h>
6969 #include <linux/signalfd.h>
70+#include <linux/uprobes.h>
7071
7172 #include <asm/pgtable.h>
7273 #include <asm/pgalloc.h>
@@ -701,6 +702,8 @@ void mm_release(struct task_struct *tsk, struct mm_struct *mm) · 628 unmodified lines
701702 exit_pi_state_list(tsk);
702703 #endif
703704
705+ uprobe_free_utask(tsk);
706+
704707 /* Get rid of any cached register state */
705708 deactivate_mm(tsk, mm);
706709
@@ -1295,6 +1298,7 @@ static struct task_struct *copy_process(unsigned long clone_flags, · 588 unmodified lines
12951298 INIT_LIST_HEAD(&p->pi_state_list);
12961299 p->pi_state_cache = NULL;
12971300 #endif
1301+ uprobe_copy_process(p);
12981302 /*
12991303 * sigaltstack should be cleared when sharing the same VM
13001304 */
kernel/signal.c+4−0
@@ -29,6 +29,7 @@ · 28 unmodified lines
2929 #include <linux/pid_namespace.h>
3030 #include <linux/nsproxy.h>
3131 #include <linux/user_namespace.h>
32+#include <linux/uprobes.h>
3233 #define CREATE_TRACE_POINTS
3334 #include <trace/events/signal.h>
3435
@@ -2192,6 +2193,9 @@ int get_signal_to_deliver(siginfo_t *info, struct k_sigaction *return_ka, · 2157 unmodified lines
21922193 struct signal_struct *signal = current->signal;
21932194 int signr;
21942195
2196+ if (unlikely(uprobe_deny_signal()))
2197+ return 0;
2198+
21952199 relock:
21962200 /*
21972201 * We'll jump back here after any time we were stopped in TASK_STOPPED.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t