[PATCH] kprobes: fix single-step out of line - take2

Now that PPC64 has no-execute support, here is a second try to fix the single step out of line during kprobe execution. Kprobes on x86_64 already solved this problem by allocating an executable page and using it as the scratch area for stepping out of line. Reuse that.

Ananth N Mavinakayanahalli committed 21 years ago

Commit 9ec4b1f1 parent d3b8a1a6 files · +128 −117Browse files

Unified · Split

6 files

arch/ppc64/kernel/kprobes.c +22 −4
arch/x86_64/kernel/kprobes.c +1 −112
include/asm-ia64/kprobes.h +1 −0
include/asm-ppc64/kprobes.h +1 −1
include/linux/kprobes.h +2 −0
kernel/kprobes.c +101 −0

arch/ppc64/kernel/kprobes.c+22−4

@@ -36,6 +36,8 @@ · 35 unmodified lines
36	36	#include <asm/kdebug.h>
37	37	#include <asm/sstep.h>
38	38
	39	+static DECLARE_MUTEX(kprobe_mutex);
	40	+
39	41	static struct kprobe *current_kprobe;
40	42	static unsigned long kprobe_status, kprobe_saved_msr;
41	43	static struct kprobe *kprobe_prev;
@@ -54,6 +56,15 @@ int arch_prepare_kprobe(struct kprobe *p) · 12 unmodified lines
54	56	printk("Cannot register a kprobe on rfid or mtmsrd\n");
55	57	ret = -EINVAL;
56	58	}
	59	+
	60	+ /* insn must be on a special executable page on ppc64 */
	61	+ if (!ret) {
	62	+ up(&kprobe_mutex);
	63	+ p->ainsn.insn = get_insn_slot();
	64	+ down(&kprobe_mutex);
	65	+ if (!p->ainsn.insn)
	66	+ ret = -ENOMEM;
	67	+ }
57	68	return ret;
58	69	}
59	70
@@ -79,16 +90,22 @@ void arch_disarm_kprobe(struct kprobe *p) · 19 unmodified lines
79	90
80	91	void arch_remove_kprobe(struct kprobe *p)
81	92	{
	93	+ up(&kprobe_mutex);
	94	+ free_insn_slot(p->ainsn.insn);
	95	+ down(&kprobe_mutex);
82	96	}
83	97
84	98	static inline void prepare_singlestep(struct kprobe p, struct pt_regs regs)
85	99	{
	100	+ kprobe_opcode_t insn = *p->ainsn.insn;
	101	+
86	102	regs->msr \|= MSR_SE;
87		− /single step inline if it a breakpoint instruction/
88		− if (p->opcode == BREAKPOINT_INSTRUCTION)
	103	+
	104	+ /* single step inline if it is a trap variant */
	105	+ if (IS_TW(insn) \|\| IS_TD(insn) \|\| IS_TWI(insn) \|\| IS_TDI(insn))
89	106	regs->nip = (unsigned long)p->addr;
90	107	else
91		− regs->nip = (unsigned long)&p->ainsn.insn;
	108	+ regs->nip = (unsigned long)p->ainsn.insn;
92	109	}
93	110
94	111	static inline void save_previous_kprobe(void)
@@ -205,9 +222,10 @@ no_kprobe: · 110 unmodified lines
205	222	static void resume_execution(struct kprobe p, struct pt_regs regs)
206	223	{
207	224	int ret;
	225	+ unsigned int insn = *p->ainsn.insn;
208	226
209	227	regs->nip = (unsigned long)p->addr;
210		− ret = emulate_step(regs, p->ainsn.insn[0]);
	228	+ ret = emulate_step(regs, insn);
211	229	if (ret == 0)
212	230	regs->nip = (unsigned long)p->addr + 4;
213	231	}

arch/x86_64/kernel/kprobes.c+1−112

@@ -38,7 +38,7 @@ · 37 unmodified lines
38	38	#include <linux/string.h>
39	39	#include <linux/slab.h>
40	40	#include <linux/preempt.h>
41		−#include <linux/moduleloader.h>
	41	+
42	42	#include <asm/cacheflush.h>
43	43	#include <asm/pgtable.h>
44	44	#include <asm/kdebug.h>
@@ -51,8 +51,6 @@ static struct kprobe *kprobe_prev; · 6 unmodified lines
51	51	static unsigned long kprobe_status_prev, kprobe_old_rflags_prev, kprobe_saved_rflags_prev;
52	52	static struct pt_regs jprobe_saved_regs;
53	53	static long *jprobe_saved_rsp;
54		−static kprobe_opcode_t *get_insn_slot(void);
55		−static void free_insn_slot(kprobe_opcode_t *slot);
56	54	void jprobe_return_end(void);
57	55
58	56	/* copy of the kernel stack at the probe fire time */
@@ -681,112 +679,3 @@ int longjmp_break_handler(struct kprobe p, struct pt_regs regs) · 622 unmodified lines
681	679	}
682	680	return 0;
683	681	}
684		−
685		−/*
686		− * kprobe->ainsn.insn points to the copy of the instruction to be single-stepped.
687		− * By default on x86_64, pages we get from kmalloc or vmalloc are not
688		− * executable. Single-stepping an instruction on such a page yields an
689		− * oops. So instead of storing the instruction copies in their respective
690		− * kprobe objects, we allocate a page, map it executable, and store all the
691		− * instruction copies there. (We can allocate additional pages if somebody
692		− * inserts a huge number of probes.) Each page can hold up to INSNS_PER_PAGE
693		− * instruction slots, each of which is MAX_INSN_SIZE*sizeof(kprobe_opcode_t)
694		− * bytes.
695		− */
696		−#define INSNS_PER_PAGE (PAGE_SIZE/(MAX_INSN_SIZE*sizeof(kprobe_opcode_t)))
697		−struct kprobe_insn_page {
698		− struct hlist_node hlist;
699		− kprobe_opcode_t insns; / page of instruction slots */
700		− char slot_used[INSNS_PER_PAGE];
701		− int nused;
702		−};
703		−
704		−static struct hlist_head kprobe_insn_pages;
705		−
706		−/**
707		− * get_insn_slot() - Find a slot on an executable page for an instruction.
708		− * We allocate an executable page if there's no room on existing ones.
709		− */
710		−static kprobe_opcode_t *get_insn_slot(void)
711		−{
712		− struct kprobe_insn_page *kip;
713		− struct hlist_node *pos;
714		−
715		− hlist_for_each(pos, &kprobe_insn_pages) {
716		− kip = hlist_entry(pos, struct kprobe_insn_page, hlist);
717		− if (kip->nused < INSNS_PER_PAGE) {
718		− int i;
719		− for (i = 0; i < INSNS_PER_PAGE; i++) {
720		− if (!kip->slot_used[i]) {
721		− kip->slot_used[i] = 1;
722		− kip->nused++;
723		− return kip->insns + (i*MAX_INSN_SIZE);
724		− }
725		− }
726		− /* Surprise! No unused slots. Fix kip->nused. */
727		− kip->nused = INSNS_PER_PAGE;
728		− }
729		− }
730		−
731		− /* All out of space. Need to allocate a new page. Use slot 0.*/
732		− kip = kmalloc(sizeof(struct kprobe_insn_page), GFP_KERNEL);
733		− if (!kip) {
734		− return NULL;
735		− }
736		−
737		− /*
738		− * For the %rip-relative displacement fixups to be doable, we
739		− * need our instruction copy to be within +/- 2GB of any data it
740		− * might access via %rip. That is, within 2GB of where the
741		− * kernel image and loaded module images reside. So we allocate
742		− * a page in the module loading area.
743		− */
744		− kip->insns = module_alloc(PAGE_SIZE);
745		− if (!kip->insns) {
746		− kfree(kip);
747		− return NULL;
748		− }
749		− INIT_HLIST_NODE(&kip->hlist);
750		− hlist_add_head(&kip->hlist, &kprobe_insn_pages);
751		− memset(kip->slot_used, 0, INSNS_PER_PAGE);
752		− kip->slot_used[0] = 1;
753		− kip->nused = 1;
754		− return kip->insns;
755		−}
756		−
757		−/**
758		− * free_insn_slot() - Free instruction slot obtained from get_insn_slot().
759		− */
760		−static void free_insn_slot(kprobe_opcode_t *slot)
761		−{
762		− struct kprobe_insn_page *kip;
763		− struct hlist_node *pos;
764		−
765		− hlist_for_each(pos, &kprobe_insn_pages) {
766		− kip = hlist_entry(pos, struct kprobe_insn_page, hlist);
767		− if (kip->insns <= slot
768		− && slot < kip->insns+(INSNS_PER_PAGE*MAX_INSN_SIZE)) {
769		− int i = (slot - kip->insns) / MAX_INSN_SIZE;
770		− kip->slot_used[i] = 0;
771		− kip->nused--;
772		− if (kip->nused == 0) {
773		− /*
774		− * Page is no longer in use. Free it unless
775		− * it's the last one. We keep the last one
776		− * so as not to have to set it up again the
777		− * next time somebody inserts a probe.
778		− */
779		− hlist_del(&kip->hlist);
780		− if (hlist_empty(&kprobe_insn_pages)) {
781		− INIT_HLIST_NODE(&kip->hlist);
782		− hlist_add_head(&kip->hlist,
783		− &kprobe_insn_pages);
784		− } else {
785		− module_free(NULL, kip->insns);
786		− kfree(kip);
787		− }
788		− }
789		− return;
790		− }
791		− }
792		−}

include/asm-ia64/kprobes.h+1−0

@@ -28,6 +28,7 @@ · 27 unmodified lines
28	28	#include <linux/ptrace.h>
29	29	#include <asm/break.h>
30	30
	31	+#define MAX_INSN_SIZE 16
31	32	#define BREAK_INST (long)(__IA64_BREAK_KPROBE << 6)
32	33
33	34	typedef union cmp_inst {

include/asm-ppc64/kprobes.h+1−1

@@ -45,7 +45,7 @@ typedef unsigned int kprobe_opcode_t; · 44 unmodified lines
45	45	/* Architecture specific copy of original instruction */
46	46	struct arch_specific_insn {
47	47	/* copy of original instruction */
48		− kprobe_opcode_t insn[MAX_INSN_SIZE];
	48	+ kprobe_opcode_t *insn;
49	49	};
50	50
51	51	#ifdef CONFIG_KPROBES

include/linux/kprobes.h+2−0

@@ -177,6 +177,8 @@ extern void arch_arm_kprobe(struct kprobe *p); · 176 unmodified lines
177	177	extern void arch_disarm_kprobe(struct kprobe *p);
178	178	extern void arch_remove_kprobe(struct kprobe *p);
179	179	extern void show_registers(struct pt_regs *regs);
	180	+extern kprobe_opcode_t *get_insn_slot(void);
	181	+extern void free_insn_slot(kprobe_opcode_t *slot);
180	182
181	183	/* Get the kprobe at this addr (if any). Must have called lock_kprobes */
182	184	struct kprobe get_kprobe(void addr);

kernel/kprobes.c+101−0

@@ -36,6 +36,7 @@ · 35 unmodified lines
36	36	#include <linux/hash.h>
37	37	#include <linux/init.h>
38	38	#include <linux/module.h>
	39	+#include <linux/moduleloader.h>
39	40	#include <asm/cacheflush.h>
40	41	#include <asm/errno.h>
41	42	#include <asm/kdebug.h>
@@ -50,6 +51,106 @@ unsigned int kprobe_cpu = NR_CPUS; · 8 unmodified lines
50	51	static DEFINE_SPINLOCK(kprobe_lock);
51	52	static struct kprobe *curr_kprobe;
52	53
	54	+/*
	55	+ * kprobe->ainsn.insn points to the copy of the instruction to be
	56	+ * single-stepped. x86_64, POWER4 and above have no-exec support and
	57	+ * stepping on the instruction on a vmalloced/kmalloced/data page
	58	+ * is a recipe for disaster
	59	+ */
	60	+#define INSNS_PER_PAGE (PAGE_SIZE/(MAX_INSN_SIZE * sizeof(kprobe_opcode_t)))
	61	+
	62	+struct kprobe_insn_page {
	63	+ struct hlist_node hlist;
	64	+ kprobe_opcode_t insns; / Page of instruction slots */
	65	+ char slot_used[INSNS_PER_PAGE];
	66	+ int nused;
	67	+};
	68	+
	69	+static struct hlist_head kprobe_insn_pages;
	70	+
	71	+/**
	72	+ * get_insn_slot() - Find a slot on an executable page for an instruction.
	73	+ * We allocate an executable page if there's no room on existing ones.
	74	+ */
	75	+kprobe_opcode_t *get_insn_slot(void)
	76	+{
	77	+ struct kprobe_insn_page *kip;
	78	+ struct hlist_node *pos;
	79	+
	80	+ hlist_for_each(pos, &kprobe_insn_pages) {
	81	+ kip = hlist_entry(pos, struct kprobe_insn_page, hlist);
	82	+ if (kip->nused < INSNS_PER_PAGE) {
	83	+ int i;
	84	+ for (i = 0; i < INSNS_PER_PAGE; i++) {
	85	+ if (!kip->slot_used[i]) {
	86	+ kip->slot_used[i] = 1;
	87	+ kip->nused++;
	88	+ return kip->insns + (i * MAX_INSN_SIZE);
	89	+ }
	90	+ }
	91	+ /* Surprise! No unused slots. Fix kip->nused. */
	92	+ kip->nused = INSNS_PER_PAGE;
	93	+ }
	94	+ }
	95	+
	96	+ /* All out of space. Need to allocate a new page. Use slot 0.*/
	97	+ kip = kmalloc(sizeof(struct kprobe_insn_page), GFP_KERNEL);
	98	+ if (!kip) {
	99	+ return NULL;
	100	+ }
	101	+
	102	+ /*
	103	+ * Use module_alloc so this page is within +/- 2GB of where the
	104	+ * kernel image and loaded module images reside. This is required
	105	+ * so x86_64 can correctly handle the %rip-relative fixups.
	106	+ */
	107	+ kip->insns = module_alloc(PAGE_SIZE);
	108	+ if (!kip->insns) {
	109	+ kfree(kip);
	110	+ return NULL;
	111	+ }
	112	+ INIT_HLIST_NODE(&kip->hlist);
	113	+ hlist_add_head(&kip->hlist, &kprobe_insn_pages);
	114	+ memset(kip->slot_used, 0, INSNS_PER_PAGE);
	115	+ kip->slot_used[0] = 1;
	116	+ kip->nused = 1;
	117	+ return kip->insns;
	118	+}
	119	+
	120	+void free_insn_slot(kprobe_opcode_t *slot)
	121	+{
	122	+ struct kprobe_insn_page *kip;
	123	+ struct hlist_node *pos;
	124	+
	125	+ hlist_for_each(pos, &kprobe_insn_pages) {
	126	+ kip = hlist_entry(pos, struct kprobe_insn_page, hlist);
	127	+ if (kip->insns <= slot &&
	128	+ slot < kip->insns + (INSNS_PER_PAGE * MAX_INSN_SIZE)) {
	129	+ int i = (slot - kip->insns) / MAX_INSN_SIZE;
	130	+ kip->slot_used[i] = 0;
	131	+ kip->nused--;
	132	+ if (kip->nused == 0) {
	133	+ /*
	134	+ * Page is no longer in use. Free it unless
	135	+ * it's the last one. We keep the last one
	136	+ * so as not to have to set it up again the
	137	+ * next time somebody inserts a probe.
	138	+ */
	139	+ hlist_del(&kip->hlist);
	140	+ if (hlist_empty(&kprobe_insn_pages)) {
	141	+ INIT_HLIST_NODE(&kip->hlist);
	142	+ hlist_add_head(&kip->hlist,
	143	+ &kprobe_insn_pages);
	144	+ } else {
	145	+ module_free(NULL, kip->insns);
	146	+ kfree(kip);
	147	+ }
	148	+ }
	149	+ return;
	150	+ }
	151	+ }
	152	+}
	153	+
53	154	/* Locks kprobe: irqs must be disabled */
54	155	void lock_kprobes(void)
55	156	{

[PATCH] kprobes: fix single-step out of line - take2

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code