Merge branch 'perf-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull more perf updates from Ingo Molnar: "A second round of perf updates: - wide reaching kprobes sanitization and robustization, with the hope of fixing all 'probe this function crashes the kernel' bugs, by Masami Hiramatsu. - uprobes updates from Oleg Nesterov: tmpfs support, corner case fixes and robustization work. - perf tooling updates and fixes from Jiri Olsa, Namhyung Ki, Arnaldo et al: * Add support to accumulate hist periods (Namhyung Kim) * various fixes, refactorings and enhancements" * 'perf-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (101 commits) perf: Differentiate exec() and non-exec() comm events perf: Fix perf_event_comm() vs. exec() assumption uprobes/x86: Rename arch_uprobe->def to ->defparam, minor comment updates perf/documentation: Add description for conditional branch filter perf/x86: Add conditional branch filtering support perf/tool: Add conditional branch filter 'cond' to perf record perf: Add new conditional branch filter 'PERF_SAMPLE_BRANCH_COND' uprobes: Teach copy_insn() to support tmpfs uprobes: Shift ->readpage check from __copy_insn() to uprobe_register() perf/x86: Use common PMU interrupt disabled code perf/ARM: Use common PMU interrupt disabled code perf: Disable sampled events if no PMU interrupt perf: Fix use after free in perf_remove_from_context() perf tools: Fix 'make help' message error perf record: Fix poll return value propagation perf tools: Move elide bool into perf_hpp_fmt struct perf tools: Remove elide setup for SORT_MODE__MEMORY mode perf tools: Fix "==" into "=" in ui_browser__warning assignment perf tools: Allow overriding sysfs and proc finding with env var perf tools: Consider header files outside perf directory in tags target ...

Linus Torvalds committed 12 years ago

Commit 3737a122 parents c29deef 82b897781 files · +2979 −1182Browse files

Showing the changes to kernel (10 of 81 files). Show all

Unified · Split

10 files

kernel/events/core.c +27 −16
kernel/events/uprobes.c +33 −19
kernel/kprobes.c +229 −163
kernel/notifier.c +13 −9
kernel/sched/core.c +5 −2
kernel/trace/trace_event_perf.c +3 −2
kernel/trace/trace_kprobe.c +41 −30
kernel/trace/trace_probe.c +35 −30
kernel/trace/trace_probe.h +7 −8
kernel/trace/trace_uprobe.c +35 −31

kernel/events/core.c+27−16

 	local_irq_restore(flags);
 }
+void perf_event_exec(void)
+{
+	struct perf_event_context *ctx;
+	int ctxn;
+	rcu_read_lock();
+	for_each_task_context_nr(ctxn) {
+		ctx = current->perf_event_ctxp[ctxn];
+		if (!ctx)
+			continue;
+		perf_event_enable_on_exec(ctx);
+	}
+	rcu_read_unlock();
+}
 /*
  * Cross CPU call to read the hardware event
  */
 		       NULL);
 }
-void perf_event_comm(struct task_struct *task)
+void perf_event_comm(struct task_struct *task, bool exec)
 {
 	struct perf_comm_event comm_event;
-	struct perf_event_context *ctx;
-	int ctxn;
-	rcu_read_lock();
-	for_each_task_context_nr(ctxn) {
-		ctx = task->perf_event_ctxp[ctxn];
-		if (!ctx)
-			continue;
-		perf_event_enable_on_exec(ctx);
-	}
-	rcu_read_unlock();
 	if (!atomic_read(&nr_comm_events))
 		return;
 		.event_id  = {
 			.header = {
 				.type = PERF_RECORD_COMM,
-				.misc = 0,
+				.misc = exec ? PERF_RECORD_MISC_COMM_EXEC : 0,
 				/* .size */
 			},
 			/* .pid */
 		}
 	}
+	if (is_sampling_event(event)) {
+		if (event->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT) {
+			err = -ENOTSUPP;
+			goto err_alloc;
+		}
+	}
 	account_event(event);
 	/*
 static void perf_event_exit_task_context(struct task_struct *child, int ctxn)
 {
-	struct perf_event *child_event;
+	struct perf_event *child_event, *next;
 	struct perf_event_context *child_ctx;
 	unsigned long flags;
 	 */
 	mutex_lock(&child_ctx->mutex);
-	list_for_each_entry_rcu(child_event, &child_ctx->event_list, event_entry)
+	list_for_each_entry_safe(child_event, next, &child_ctx->event_list, event_entry)
 		__perf_event_exit_task(child_event, child_ctx, child);
 	mutex_unlock(&child_ctx->mutex);

kernel/events/uprobes.c+33−19

 #include "../../mm/internal.h"	/* munlock_vma_page */
 #include <linux/percpu-rwsem.h>
 #include <linux/task_work.h>
+#include <linux/shmem_fs.h>
 #include <linux/uprobes.h>
  */
 static bool valid_vma(struct vm_area_struct *vma, bool is_register)
 {
-	vm_flags_t flags = VM_HUGETLB | VM_MAYEXEC | VM_SHARED;
+	vm_flags_t flags = VM_HUGETLB | VM_MAYEXEC | VM_MAYSHARE;
 	if (is_register)
 		flags |= VM_WRITE;
  * supported by that architecture then we need to modify is_trap_at_addr and
  * uprobe_write_opcode accordingly. This would never be a problem for archs
  * that have fixed length instructions.
- */
+ *
-/*
  * uprobe_write_opcode - write the opcode at a given virtual address.
  * @mm: the probed process address space.
  * @vaddr: the virtual address to store the opcode.
  * @opcode: opcode to be written at @vaddr.
  *
- * Called with mm->mmap_sem held (for read and with a reference to
+ * Called with mm->mmap_sem held for write.
- * mm).
- *
- * For mm @mm, write the opcode at @vaddr.
  * Return 0 (success) or a negative errno.
  */
 int uprobe_write_opcode(struct mm_struct *mm, unsigned long vaddr,
 	if (ret <= 0)
 		goto put_old;
+	ret = anon_vma_prepare(vma);
+	if (ret)
+		goto put_old;
 	ret = -ENOMEM;
 	new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vaddr);
 	if (!new_page)
 		goto put_old;
-	__SetPageUptodate(new_page);
+	if (mem_cgroup_charge_anon(new_page, mm, GFP_KERNEL))
+		goto put_new;
+	__SetPageUptodate(new_page);
 	copy_highpage(new_page, old_page);
 	copy_to_page(new_page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE);
-	ret = anon_vma_prepare(vma);
-	if (ret)
-		goto put_new;
 	ret = __replace_page(vma, vaddr, old_page, new_page);
+	if (ret)
+		mem_cgroup_uncharge_page(new_page);
 put_new:
 	page_cache_release(new_page);
 			void *insn, int nbytes, loff_t offset)
 {
 	struct page *page;
-	if (!mapping->a_ops->readpage)
-		return -EIO;
 	/*
-	 * Ensure that the page that has the original instruction is
+	 * Ensure that the page that has the original instruction is populated
-	 * populated and in page-cache.
+	 * and in page-cache. If ->readpage == NULL it must be shmem_mapping(),
+	 * see uprobe_register().
 	 */
-	page = read_mapping_page(mapping, offset >> PAGE_CACHE_SHIFT, filp);
+	if (mapping->a_ops->readpage)
+		page = read_mapping_page(mapping, offset >> PAGE_CACHE_SHIFT, filp);
+	else
+		page = shmem_read_mapping_page(mapping, offset >> PAGE_CACHE_SHIFT);
 	if (IS_ERR(page))
 		return PTR_ERR(page);
 	if (!uc->handler && !uc->ret_handler)
 		return -EINVAL;
+	/* copy_insn() uses read_mapping_page() or shmem_read_mapping_page() */
+	if (!inode->i_mapping->a_ops->readpage && !shmem_mapping(inode->i_mapping))
+		return -EIO;
 	/* Racy, just to catch the obvious mistakes */
 	if (offset > i_size_read(inode))
 		return -EINVAL;
 	return instruction_pointer(regs) - UPROBE_SWBP_INSN_SIZE;
 }
+unsigned long uprobe_get_trap_addr(struct pt_regs *regs)
+{
+	struct uprobe_task *utask = current->utask;
+	if (unlikely(utask && utask->active_uprobe))
+		return utask->vaddr;
+	return instruction_pointer(regs);
+}
 /*
  * Called with no locks held.
  * Called in context of a exiting or a exec-ing thread.

kernel/kprobes.c+229−163

@@ -86,21 +86,8 @@ static raw_spinlock_t *kretprobe_table_lock_ptr(unsigned long hash) · 85 unmodified lines
86	return &(kretprobe_table_locks[hash].lock);	86	return &(kretprobe_table_locks[hash].lock);
87	}	87	}
88		88
89	/*	89	/* Blacklist -- list of struct kprobe_blacklist_entry */
90	* Normally, functions that we'd want to prohibit kprobes in, are marked	90	static LIST_HEAD(kprobe_blacklist);
91	* __kprobes. But, there are cases where such functions already belong to
92	* a different section (__sched for preempt_schedule)
93	*
94	* For such cases, we now have a blacklist
95	*/
96	static struct kprobe_blackpoint kprobe_blacklist[] = {
97	{"preempt_schedule",},
98	{"native_get_debugreg",},
99	{"irq_entries_start",},
100	{"common_interrupt",},
101	{"mcount",}, /* mcount can be called from everywhere */
102	{NULL} /* Terminator */
103	};
104		91
105	#ifdef __ARCH_WANT_KPROBES_INSN_SLOT	92	#ifdef __ARCH_WANT_KPROBES_INSN_SLOT
106	/*	93	/*
@@ -151,13 +138,13 @@ struct kprobe_insn_cache kprobe_insn_slots = { · 44 unmodified lines
151	.insn_size = MAX_INSN_SIZE,	138	.insn_size = MAX_INSN_SIZE,
152	.nr_garbage = 0,	139	.nr_garbage = 0,
153	};	140	};
154	static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c);	141	static int collect_garbage_slots(struct kprobe_insn_cache *c);
155		142
156	/**	143	/**
157	* __get_insn_slot() - Find a slot on an executable page for an instruction.	144	* __get_insn_slot() - Find a slot on an executable page for an instruction.
158	* We allocate an executable page if there's no room on existing ones.	145	* We allocate an executable page if there's no room on existing ones.
159	*/	146	*/
160	kprobe_opcode_t __kprobes __get_insn_slot(struct kprobe_insn_cache c)	147	kprobe_opcode_t __get_insn_slot(struct kprobe_insn_cache c)
161	{	148	{
162	struct kprobe_insn_page *kip;	149	struct kprobe_insn_page *kip;
163	kprobe_opcode_t *slot = NULL;	150	kprobe_opcode_t *slot = NULL;
@@ -214,7 +201,7 @@ out: · 50 unmodified lines
214	}	201	}
215		202
216	/* Return 1 if all garbages are collected, otherwise 0. */	203	/* Return 1 if all garbages are collected, otherwise 0. */
217	static int __kprobes collect_one_slot(struct kprobe_insn_page *kip, int idx)	204	static int collect_one_slot(struct kprobe_insn_page *kip, int idx)
218	{	205	{
219	kip->slot_used[idx] = SLOT_CLEAN;	206	kip->slot_used[idx] = SLOT_CLEAN;
220	kip->nused--;	207	kip->nused--;
@@ -235,7 +222,7 @@ static int __kprobes collect_one_slot(struct kprobe_insn_page *kip, int idx) · 14 unmodified lines
235	return 0;	222	return 0;
236	}	223	}
237		224
238	static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c)	225	static int collect_garbage_slots(struct kprobe_insn_cache *c)
239	{	226	{
240	struct kprobe_insn_page kip, next;	227	struct kprobe_insn_page kip, next;
241		228
@@ -257,8 +244,8 @@ static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c) · 15 unmodified lines
257	return 0;	244	return 0;
258	}	245	}
259		246
260	void __kprobes __free_insn_slot(struct kprobe_insn_cache *c,	247	void __free_insn_slot(struct kprobe_insn_cache *c,
261	kprobe_opcode_t *slot, int dirty)	248	kprobe_opcode_t *slot, int dirty)
262	{	249	{
263	struct kprobe_insn_page *kip;	250	struct kprobe_insn_page *kip;
264		251
@@ -314,7 +301,7 @@ static inline void reset_kprobe_instance(void) · 49 unmodified lines
314	* OR	301	* OR
315	* - with preemption disabled - from arch/xxx/kernel/kprobes.c	302	* - with preemption disabled - from arch/xxx/kernel/kprobes.c
316	*/	303	*/
317	struct kprobe __kprobes get_kprobe(void addr)	304	struct kprobe get_kprobe(void addr)
318	{	305	{
319	struct hlist_head *head;	306	struct hlist_head *head;
320	struct kprobe *p;	307	struct kprobe *p;
@@ -327,8 +314,9 @@ struct kprobe __kprobes get_kprobe(void addr) · 6 unmodified lines
327		314
328	return NULL;	315	return NULL;
329	}	316	}
		317	NOKPROBE_SYMBOL(get_kprobe);
330		318
331	static int __kprobes aggr_pre_handler(struct kprobe p, struct pt_regs regs);	319	static int aggr_pre_handler(struct kprobe p, struct pt_regs regs);
332		320
333	/* Return true if the kprobe is an aggregator */	321	/* Return true if the kprobe is an aggregator */
334	static inline int kprobe_aggrprobe(struct kprobe *p)	322	static inline int kprobe_aggrprobe(struct kprobe *p)
@@ -360,7 +348,7 @@ static bool kprobes_allow_optimization; · 25 unmodified lines
360	* Call all pre_handler on the list, but ignores its return value.	348	* Call all pre_handler on the list, but ignores its return value.
361	* This must be called from arch-dep optimized caller.	349	* This must be called from arch-dep optimized caller.
362	*/	350	*/
363	void __kprobes opt_pre_handler(struct kprobe p, struct pt_regs regs)	351	void opt_pre_handler(struct kprobe p, struct pt_regs regs)
364	{	352	{
365	struct kprobe *kp;	353	struct kprobe *kp;
366		354
@@ -372,9 +360,10 @@ void __kprobes opt_pre_handler(struct kprobe p, struct pt_regs regs) · 5 unmodified lines
372	reset_kprobe_instance();	360	reset_kprobe_instance();
373	}	361	}
374	}	362	}
		363	NOKPROBE_SYMBOL(opt_pre_handler);
375		364
376	/* Free optimized instructions and optimized_kprobe */	365	/* Free optimized instructions and optimized_kprobe */
377	static __kprobes void free_aggr_kprobe(struct kprobe *p)	366	static void free_aggr_kprobe(struct kprobe *p)
378	{	367	{
379	struct optimized_kprobe *op;	368	struct optimized_kprobe *op;
380		369
@@ -412,7 +401,7 @@ static inline int kprobe_disarmed(struct kprobe *p) · 31 unmodified lines
412	}	401	}
413		402
414	/* Return true(!0) if the probe is queued on (un)optimizing lists */	403	/* Return true(!0) if the probe is queued on (un)optimizing lists */
415	static int __kprobes kprobe_queued(struct kprobe *p)	404	static int kprobe_queued(struct kprobe *p)
416	{	405	{
417	struct optimized_kprobe *op;	406	struct optimized_kprobe *op;
418		407
@@ -428,7 +417,7 @@ static int __kprobes kprobe_queued(struct kprobe *p) · 9 unmodified lines
428	* Return an optimized kprobe whose optimizing code replaces	417	* Return an optimized kprobe whose optimizing code replaces
429	* instructions including addr (exclude breakpoint).	418	* instructions including addr (exclude breakpoint).
430	*/	419	*/
431	static struct kprobe *__kprobes get_optimized_kprobe(unsigned long addr)	420	static struct kprobe *get_optimized_kprobe(unsigned long addr)
432	{	421	{
433	int i;	422	int i;
434	struct kprobe *p = NULL;	423	struct kprobe *p = NULL;
@@ -460,7 +449,7 @@ static DECLARE_DELAYED_WORK(optimizing_work, kprobe_optimizer); · 25 unmodified lines
460	* Optimize (replace a breakpoint with a jump) kprobes listed on	449	* Optimize (replace a breakpoint with a jump) kprobes listed on
461	* optimizing_list.	450	* optimizing_list.
462	*/	451	*/
463	static __kprobes void do_optimize_kprobes(void)	452	static void do_optimize_kprobes(void)
464	{	453	{
465	/* Optimization never be done when disarmed */	454	/* Optimization never be done when disarmed */
466	if (kprobes_all_disarmed \|\| !kprobes_allow_optimization \|\|	455	if (kprobes_all_disarmed \|\| !kprobes_allow_optimization \|\|
@@ -488,7 +477,7 @@ static __kprobes void do_optimize_kprobes(void) · 21 unmodified lines
488	* Unoptimize (replace a jump with a breakpoint and remove the breakpoint	477	* Unoptimize (replace a jump with a breakpoint and remove the breakpoint
489	* if need) kprobes listed on unoptimizing_list.	478	* if need) kprobes listed on unoptimizing_list.
490	*/	479	*/
491	static __kprobes void do_unoptimize_kprobes(void)	480	static void do_unoptimize_kprobes(void)
492	{	481	{
493	struct optimized_kprobe op, tmp;	482	struct optimized_kprobe op, tmp;
494		483
@@ -520,7 +509,7 @@ static __kprobes void do_unoptimize_kprobes(void) · 25 unmodified lines
520	}	509	}
521		510
522	/* Reclaim all kprobes on the free_list */	511	/* Reclaim all kprobes on the free_list */
523	static __kprobes void do_free_cleaned_kprobes(void)	512	static void do_free_cleaned_kprobes(void)
524	{	513	{
525	struct optimized_kprobe op, tmp;	514	struct optimized_kprobe op, tmp;
526		515
@@ -532,13 +521,13 @@ static __kprobes void do_free_cleaned_kprobes(void) · 5 unmodified lines
532	}	521	}
533		522
534	/* Start optimizer after OPTIMIZE_DELAY passed */	523	/* Start optimizer after OPTIMIZE_DELAY passed */
535	static __kprobes void kick_kprobe_optimizer(void)	524	static void kick_kprobe_optimizer(void)
536	{	525	{
537	schedule_delayed_work(&optimizing_work, OPTIMIZE_DELAY);	526	schedule_delayed_work(&optimizing_work, OPTIMIZE_DELAY);
538	}	527	}
539		528
540	/* Kprobe jump optimizer */	529	/* Kprobe jump optimizer */
541	static __kprobes void kprobe_optimizer(struct work_struct *work)	530	static void kprobe_optimizer(struct work_struct *work)
542	{	531	{
543	mutex_lock(&kprobe_mutex);	532	mutex_lock(&kprobe_mutex);
544	/* Lock modules while optimizing kprobes */	533	/* Lock modules while optimizing kprobes */
@@ -574,7 +563,7 @@ static __kprobes void kprobe_optimizer(struct work_struct *work) · 29 unmodified lines
574	}	563	}
575		564
576	/* Wait for completing optimization and unoptimization */	565	/* Wait for completing optimization and unoptimization */
577	static __kprobes void wait_for_kprobe_optimizer(void)	566	static void wait_for_kprobe_optimizer(void)
578	{	567	{
579	mutex_lock(&kprobe_mutex);	568	mutex_lock(&kprobe_mutex);
580		569
@@ -593,7 +582,7 @@ static __kprobes void wait_for_kprobe_optimizer(void) · 12 unmodified lines
593	}	582	}
594		583
595	/* Optimize kprobe if p is ready to be optimized */	584	/* Optimize kprobe if p is ready to be optimized */
596	static __kprobes void optimize_kprobe(struct kprobe *p)	585	static void optimize_kprobe(struct kprobe *p)
597	{	586	{
598	struct optimized_kprobe *op;	587	struct optimized_kprobe *op;
599		588
@@ -627,7 +616,7 @@ static __kprobes void optimize_kprobe(struct kprobe *p) · 27 unmodified lines
627	}	616	}
628		617
629	/* Short cut to direct unoptimizing */	618	/* Short cut to direct unoptimizing */
630	static __kprobes void force_unoptimize_kprobe(struct optimized_kprobe *op)	619	static void force_unoptimize_kprobe(struct optimized_kprobe *op)
631	{	620	{
632	get_online_cpus();	621	get_online_cpus();
633	arch_unoptimize_kprobe(op);	622	arch_unoptimize_kprobe(op);
@@ -637,7 +626,7 @@ static __kprobes void force_unoptimize_kprobe(struct optimized_kprobe *op) · 3 unmodified lines
637	}	626	}
638		627
639	/* Unoptimize a kprobe if p is optimized */	628	/* Unoptimize a kprobe if p is optimized */
640	static __kprobes void unoptimize_kprobe(struct kprobe *p, bool force)	629	static void unoptimize_kprobe(struct kprobe *p, bool force)
641	{	630	{
642	struct optimized_kprobe *op;	631	struct optimized_kprobe *op;
643		632
@@ -697,7 +686,7 @@ static void reuse_unused_kprobe(struct kprobe *ap) · 53 unmodified lines
697	}	686	}
698		687
699	/* Remove optimized instructions */	688	/* Remove optimized instructions */
700	static void __kprobes kill_optimized_kprobe(struct kprobe *p)	689	static void kill_optimized_kprobe(struct kprobe *p)
701	{	690	{
702	struct optimized_kprobe *op;	691	struct optimized_kprobe *op;
703		692
@@ -723,7 +712,7 @@ static void __kprobes kill_optimized_kprobe(struct kprobe *p) · 19 unmodified lines
723	}	712	}
724		713
725	/* Try to prepare optimized instructions */	714	/* Try to prepare optimized instructions */
726	static __kprobes void prepare_optimized_kprobe(struct kprobe *p)	715	static void prepare_optimized_kprobe(struct kprobe *p)
727	{	716	{
728	struct optimized_kprobe *op;	717	struct optimized_kprobe *op;
729		718
@@ -732,7 +721,7 @@ static __kprobes void prepare_optimized_kprobe(struct kprobe *p) · 2 unmodified lines
732	}	721	}
733		722
734	/* Allocate new optimized_kprobe and try to prepare optimized instructions */	723	/* Allocate new optimized_kprobe and try to prepare optimized instructions */
735	static __kprobes struct kprobe alloc_aggr_kprobe(struct kprobe p)	724	static struct kprobe alloc_aggr_kprobe(struct kprobe p)
736	{	725	{
737	struct optimized_kprobe *op;	726	struct optimized_kprobe *op;
738		727
@@ -747,13 +736,13 @@ static __kprobes struct kprobe alloc_aggr_kprobe(struct kprobe p) · 8 unmodified lines
747	return &op->kp;	736	return &op->kp;
748	}	737	}
749		738
750	static void __kprobes init_aggr_kprobe(struct kprobe ap, struct kprobe p);	739	static void init_aggr_kprobe(struct kprobe ap, struct kprobe p);
751		740
752	/*	741	/*
753	* Prepare an optimized_kprobe and optimize it	742	* Prepare an optimized_kprobe and optimize it
754	* NOTE: p must be a normal registered kprobe	743	* NOTE: p must be a normal registered kprobe
755	*/	744	*/
756	static __kprobes void try_to_optimize_kprobe(struct kprobe *p)	745	static void try_to_optimize_kprobe(struct kprobe *p)
757	{	746	{
758	struct kprobe *ap;	747	struct kprobe *ap;
759	struct optimized_kprobe *op;	748	struct optimized_kprobe *op;
@@ -787,7 +776,7 @@ out: · 27 unmodified lines
787	}	776	}
788		777
789	#ifdef CONFIG_SYSCTL	778	#ifdef CONFIG_SYSCTL
790	static void __kprobes optimize_all_kprobes(void)	779	static void optimize_all_kprobes(void)
791	{	780	{
792	struct hlist_head *head;	781	struct hlist_head *head;
793	struct kprobe *p;	782	struct kprobe *p;
@@ -810,7 +799,7 @@ out: · 16 unmodified lines
810	mutex_unlock(&kprobe_mutex);	799	mutex_unlock(&kprobe_mutex);
811	}	800	}
812		801
813	static void __kprobes unoptimize_all_kprobes(void)	802	static void unoptimize_all_kprobes(void)
814	{	803	{
815	struct hlist_head *head;	804	struct hlist_head *head;
816	struct kprobe *p;	805	struct kprobe *p;
@@ -861,7 +850,7 @@ int proc_kprobes_optimization_handler(struct ctl_table *table, int write, · 44 unmodified lines
861	#endif /* CONFIG_SYSCTL */	850	#endif /* CONFIG_SYSCTL */
862		851
863	/* Put a breakpoint for a probe. Must be called with text_mutex locked */	852	/* Put a breakpoint for a probe. Must be called with text_mutex locked */
864	static void __kprobes __arm_kprobe(struct kprobe *p)	853	static void __arm_kprobe(struct kprobe *p)
865	{	854	{
866	struct kprobe *_p;	855	struct kprobe *_p;
867		856
@@ -876,7 +865,7 @@ static void __kprobes __arm_kprobe(struct kprobe *p) · 8 unmodified lines
876	}	865	}
877		866
878	/* Remove the breakpoint of a probe. Must be called with text_mutex locked */	867	/* Remove the breakpoint of a probe. Must be called with text_mutex locked */
879	static void __kprobes __disarm_kprobe(struct kprobe *p, bool reopt)	868	static void __disarm_kprobe(struct kprobe *p, bool reopt)
880	{	869	{
881	struct kprobe *_p;	870	struct kprobe *_p;
882		871
@@ -911,13 +900,13 @@ static void reuse_unused_kprobe(struct kprobe *ap) · 28 unmodified lines
911	BUG_ON(kprobe_unused(ap));	900	BUG_ON(kprobe_unused(ap));
912	}	901	}
913		902
914	static __kprobes void free_aggr_kprobe(struct kprobe *p)	903	static void free_aggr_kprobe(struct kprobe *p)
915	{	904	{
916	arch_remove_kprobe(p);	905	arch_remove_kprobe(p);
917	kfree(p);	906	kfree(p);
918	}	907	}
919		908
920	static __kprobes struct kprobe alloc_aggr_kprobe(struct kprobe p)	909	static struct kprobe alloc_aggr_kprobe(struct kprobe p)
921	{	910	{
922	return kzalloc(sizeof(struct kprobe), GFP_KERNEL);	911	return kzalloc(sizeof(struct kprobe), GFP_KERNEL);
923	}	912	}
@@ -931,7 +920,7 @@ static struct ftrace_ops kprobe_ftrace_ops __read_mostly = { · 7 unmodified lines
931	static int kprobe_ftrace_enabled;	920	static int kprobe_ftrace_enabled;
932		921
933	/* Must ensure p->addr is really on ftrace */	922	/* Must ensure p->addr is really on ftrace */
934	static int __kprobes prepare_kprobe(struct kprobe *p)	923	static int prepare_kprobe(struct kprobe *p)
935	{	924	{
936	if (!kprobe_ftrace(p))	925	if (!kprobe_ftrace(p))
937	return arch_prepare_kprobe(p);	926	return arch_prepare_kprobe(p);
@@ -940,7 +929,7 @@ static int __kprobes prepare_kprobe(struct kprobe *p) · 2 unmodified lines
940	}	929	}
941		930
942	/* Caller must lock kprobe_mutex */	931	/* Caller must lock kprobe_mutex */
943	static void __kprobes arm_kprobe_ftrace(struct kprobe *p)	932	static void arm_kprobe_ftrace(struct kprobe *p)
944	{	933	{
945	int ret;	934	int ret;
946		935
@@ -955,7 +944,7 @@ static void __kprobes arm_kprobe_ftrace(struct kprobe *p) · 8 unmodified lines
955	}	944	}
956		945
957	/* Caller must lock kprobe_mutex */	946	/* Caller must lock kprobe_mutex */
958	static void __kprobes disarm_kprobe_ftrace(struct kprobe *p)	947	static void disarm_kprobe_ftrace(struct kprobe *p)
959	{	948	{
960	int ret;	949	int ret;
961		950
@@ -975,7 +964,7 @@ static void __kprobes disarm_kprobe_ftrace(struct kprobe *p) · 13 unmodified lines
975	#endif	964	#endif
976		965
977	/* Arm a kprobe with text_mutex */	966	/* Arm a kprobe with text_mutex */
978	static void __kprobes arm_kprobe(struct kprobe *kp)	967	static void arm_kprobe(struct kprobe *kp)
979	{	968	{
980	if (unlikely(kprobe_ftrace(kp))) {	969	if (unlikely(kprobe_ftrace(kp))) {
981	arm_kprobe_ftrace(kp);	970	arm_kprobe_ftrace(kp);
@@ -992,7 +981,7 @@ static void __kprobes arm_kprobe(struct kprobe *kp) · 10 unmodified lines
992	}	981	}
993		982
994	/* Disarm a kprobe with text_mutex */	983	/* Disarm a kprobe with text_mutex */
995	static void __kprobes disarm_kprobe(struct kprobe *kp, bool reopt)	984	static void disarm_kprobe(struct kprobe *kp, bool reopt)
996	{	985	{
997	if (unlikely(kprobe_ftrace(kp))) {	986	if (unlikely(kprobe_ftrace(kp))) {
998	disarm_kprobe_ftrace(kp);	987	disarm_kprobe_ftrace(kp);
@@ -1008,7 +997,7 @@ static void __kprobes disarm_kprobe(struct kprobe *kp, bool reopt) · 9 unmodified lines
1008	* Aggregate handlers for multiple kprobes support - these handlers	997	* Aggregate handlers for multiple kprobes support - these handlers
1009	* take care of invoking the individual kprobe handlers on p->list	998	* take care of invoking the individual kprobe handlers on p->list
1010	*/	999	*/
1011	static int __kprobes aggr_pre_handler(struct kprobe p, struct pt_regs regs)	1000	static int aggr_pre_handler(struct kprobe p, struct pt_regs regs)
1012	{	1001	{
1013	struct kprobe *kp;	1002	struct kprobe *kp;
1014		1003
@@ -1022,9 +1011,10 @@ static int __kprobes aggr_pre_handler(struct kprobe p, struct pt_regs regs) · 7 unmodified lines
1022	}	1011	}
1023	return 0;	1012	return 0;
1024	}	1013	}
		1014	NOKPROBE_SYMBOL(aggr_pre_handler);
1025		1015
1026	static void __kprobes aggr_post_handler(struct kprobe p, struct pt_regs regs,	1016	static void aggr_post_handler(struct kprobe p, struct pt_regs regs,
1027	unsigned long flags)	1017	unsigned long flags)
1028	{	1018	{
1029	struct kprobe *kp;	1019	struct kprobe *kp;
1030		1020
@@ -1036,9 +1026,10 @@ static void __kprobes aggr_post_handler(struct kprobe p, struct pt_regs regs, · 5 unmodified lines
1036	}	1026	}
1037	}	1027	}
1038	}	1028	}
		1029	NOKPROBE_SYMBOL(aggr_post_handler);
1039		1030
1040	static int __kprobes aggr_fault_handler(struct kprobe p, struct pt_regs regs,	1031	static int aggr_fault_handler(struct kprobe p, struct pt_regs regs,
1041	int trapnr)	1032	int trapnr)
1042	{	1033	{
1043	struct kprobe *cur = __this_cpu_read(kprobe_instance);	1034	struct kprobe *cur = __this_cpu_read(kprobe_instance);
1044		1035
@@ -1052,8 +1043,9 @@ static int __kprobes aggr_fault_handler(struct kprobe p, struct pt_regs regs, · 7 unmodified lines
1052	}	1043	}
1053	return 0;	1044	return 0;
1054	}	1045	}
		1046	NOKPROBE_SYMBOL(aggr_fault_handler);
1055		1047
1056	static int __kprobes aggr_break_handler(struct kprobe p, struct pt_regs regs)	1048	static int aggr_break_handler(struct kprobe p, struct pt_regs regs)
1057	{	1049	{
1058	struct kprobe *cur = __this_cpu_read(kprobe_instance);	1050	struct kprobe *cur = __this_cpu_read(kprobe_instance);
1059	int ret = 0;	1051	int ret = 0;
@@ -1065,9 +1057,10 @@ static int __kprobes aggr_break_handler(struct kprobe p, struct pt_regs regs) · 5 unmodified lines
1065	reset_kprobe_instance();	1057	reset_kprobe_instance();
1066	return ret;	1058	return ret;
1067	}	1059	}
		1060	NOKPROBE_SYMBOL(aggr_break_handler);
1068		1061
1069	/* Walks the list and increments nmissed count for multiprobe case */	1062	/* Walks the list and increments nmissed count for multiprobe case */
1070	void __kprobes kprobes_inc_nmissed_count(struct kprobe *p)	1063	void kprobes_inc_nmissed_count(struct kprobe *p)
1071	{	1064	{
1072	struct kprobe *kp;	1065	struct kprobe *kp;
1073	if (!kprobe_aggrprobe(p)) {	1066	if (!kprobe_aggrprobe(p)) {
@@ -1078,9 +1071,10 @@ void __kprobes kprobes_inc_nmissed_count(struct kprobe *p) · 4 unmodified lines
1078	}	1071	}
1079	return;	1072	return;
1080	}	1073	}
		1074	NOKPROBE_SYMBOL(kprobes_inc_nmissed_count);
1081		1075
1082	void __kprobes recycle_rp_inst(struct kretprobe_instance *ri,	1076	void recycle_rp_inst(struct kretprobe_instance *ri,
1083	struct hlist_head *head)	1077	struct hlist_head *head)
1084	{	1078	{
1085	struct kretprobe *rp = ri->rp;	1079	struct kretprobe *rp = ri->rp;
1086		1080
@@ -1095,8 +1089,9 @@ void __kprobes recycle_rp_inst(struct kretprobe_instance *ri, · 8 unmodified lines
1095	/* Unregistering */	1089	/* Unregistering */
1096	hlist_add_head(&ri->hlist, head);	1090	hlist_add_head(&ri->hlist, head);
1097	}	1091	}
		1092	NOKPROBE_SYMBOL(recycle_rp_inst);
1098		1093
1099	void __kprobes kretprobe_hash_lock(struct task_struct *tsk,	1094	void kretprobe_hash_lock(struct task_struct *tsk,
1100	struct hlist_head *head, unsigned long flags)	1095	struct hlist_head *head, unsigned long flags)
1101	__acquires(hlist_lock)	1096	__acquires(hlist_lock)
1102	{	1097	{
@@ -1107,17 +1102,19 @@ __acquires(hlist_lock) · 4 unmodified lines
1107	hlist_lock = kretprobe_table_lock_ptr(hash);	1102	hlist_lock = kretprobe_table_lock_ptr(hash);
1108	raw_spin_lock_irqsave(hlist_lock, *flags);	1103	raw_spin_lock_irqsave(hlist_lock, *flags);
1109	}	1104	}
		1105	NOKPROBE_SYMBOL(kretprobe_hash_lock);
1110		1106
1111	static void __kprobes kretprobe_table_lock(unsigned long hash,	1107	static void kretprobe_table_lock(unsigned long hash,
1112	unsigned long *flags)	1108	unsigned long *flags)
1113	__acquires(hlist_lock)	1109	__acquires(hlist_lock)
1114	{	1110	{
1115	raw_spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);	1111	raw_spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
1116	raw_spin_lock_irqsave(hlist_lock, *flags);	1112	raw_spin_lock_irqsave(hlist_lock, *flags);
1117	}	1113	}
		1114	NOKPROBE_SYMBOL(kretprobe_table_lock);
1118		1115
1119	void __kprobes kretprobe_hash_unlock(struct task_struct *tsk,	1116	void kretprobe_hash_unlock(struct task_struct *tsk,
1120	unsigned long *flags)	1117	unsigned long *flags)
1121	__releases(hlist_lock)	1118	__releases(hlist_lock)
1122	{	1119	{
1123	unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);	1120	unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
@@ -1126,14 +1123,16 @@ __releases(hlist_lock) · 2 unmodified lines
1126	hlist_lock = kretprobe_table_lock_ptr(hash);	1123	hlist_lock = kretprobe_table_lock_ptr(hash);
1127	raw_spin_unlock_irqrestore(hlist_lock, *flags);	1124	raw_spin_unlock_irqrestore(hlist_lock, *flags);
1128	}	1125	}
		1126	NOKPROBE_SYMBOL(kretprobe_hash_unlock);
1129		1127
1130	static void __kprobes kretprobe_table_unlock(unsigned long hash,	1128	static void kretprobe_table_unlock(unsigned long hash,
1131	unsigned long *flags)	1129	unsigned long *flags)
1132	__releases(hlist_lock)	1130	__releases(hlist_lock)
1133	{	1131	{
1134	raw_spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);	1132	raw_spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
1135	raw_spin_unlock_irqrestore(hlist_lock, *flags);	1133	raw_spin_unlock_irqrestore(hlist_lock, *flags);
1136	}	1134	}
		1135	NOKPROBE_SYMBOL(kretprobe_table_unlock);
1137		1136
1138	/*	1137	/*
1139	* This function is called from finish_task_switch when task tk becomes dead,	1138	* This function is called from finish_task_switch when task tk becomes dead,
@@ -1141,7 +1140,7 @@ __releases(hlist_lock) · 1 unmodified line
1141	* with this task. These left over instances represent probed functions	1140	* with this task. These left over instances represent probed functions
1142	* that have been called but will never return.	1141	* that have been called but will never return.
1143	*/	1142	*/
1144	void __kprobes kprobe_flush_task(struct task_struct *tk)	1143	void kprobe_flush_task(struct task_struct *tk)
1145	{	1144	{
1146	struct kretprobe_instance *ri;	1145	struct kretprobe_instance *ri;
1147	struct hlist_head *head, empty_rp;	1146	struct hlist_head *head, empty_rp;
@@ -1166,6 +1165,7 @@ void __kprobes kprobe_flush_task(struct task_struct *tk) · 18 unmodified lines
1166	kfree(ri);	1165	kfree(ri);
1167	}	1166	}
1168	}	1167	}
		1168	NOKPROBE_SYMBOL(kprobe_flush_task);
1169		1169
1170	static inline void free_rp_inst(struct kretprobe *rp)	1170	static inline void free_rp_inst(struct kretprobe *rp)
1171	{	1171	{
@@ -1178,7 +1178,7 @@ static inline void free_rp_inst(struct kretprobe *rp) · 6 unmodified lines
1178	}	1178	}
1179	}	1179	}
1180		1180
1181	static void __kprobes cleanup_rp_inst(struct kretprobe *rp)	1181	static void cleanup_rp_inst(struct kretprobe *rp)
1182	{	1182	{
1183	unsigned long flags, hash;	1183	unsigned long flags, hash;
1184	struct kretprobe_instance *ri;	1184	struct kretprobe_instance *ri;
@@ -1197,12 +1197,13 @@ static void __kprobes cleanup_rp_inst(struct kretprobe *rp) · 12 unmodified lines
1197	}	1197	}
1198	free_rp_inst(rp);	1198	free_rp_inst(rp);
1199	}	1199	}
		1200	NOKPROBE_SYMBOL(cleanup_rp_inst);
1200		1201
1201	/*	1202	/*
1202	* Add the new probe to ap->list. Fail if this is the	1203	* Add the new probe to ap->list. Fail if this is the
1203	* second jprobe at the address - two jprobes can't coexist	1204	* second jprobe at the address - two jprobes can't coexist
1204	*/	1205	*/
1205	static int __kprobes add_new_kprobe(struct kprobe ap, struct kprobe p)	1206	static int add_new_kprobe(struct kprobe ap, struct kprobe p)
1206	{	1207	{
1207	BUG_ON(kprobe_gone(ap) \|\| kprobe_gone(p));	1208	BUG_ON(kprobe_gone(ap) \|\| kprobe_gone(p));
1208		1209
@@ -1226,7 +1227,7 @@ static int __kprobes add_new_kprobe(struct kprobe ap, struct kprobe p) · 17 unmodified lines
1226	* Fill in the required fields of the "manager kprobe". Replace the	1227	* Fill in the required fields of the "manager kprobe". Replace the
1227	* earlier kprobe in the hlist with the manager kprobe	1228	* earlier kprobe in the hlist with the manager kprobe
1228	*/	1229	*/
1229	static void __kprobes init_aggr_kprobe(struct kprobe ap, struct kprobe p)	1230	static void init_aggr_kprobe(struct kprobe ap, struct kprobe p)
1230	{	1231	{
1231	/* Copy p's insn slot to ap */	1232	/* Copy p's insn slot to ap */
1232	copy_kprobe(p, ap);	1233	copy_kprobe(p, ap);
@@ -1252,8 +1253,7 @@ static void __kprobes init_aggr_kprobe(struct kprobe ap, struct kprobe p) · 19 unmodified lines
1252	* This is the second or subsequent kprobe at the address - handle	1253	* This is the second or subsequent kprobe at the address - handle
1253	* the intricacies	1254	* the intricacies
1254	*/	1255	*/
1255	static int __kprobes register_aggr_kprobe(struct kprobe *orig_p,	1256	static int register_aggr_kprobe(struct kprobe orig_p, struct kprobe p)
1256	struct kprobe *p)
1257	{	1257	{
1258	int ret = 0;	1258	int ret = 0;
1259	struct kprobe *ap = orig_p;	1259	struct kprobe *ap = orig_p;
@@ -1324,25 +1324,29 @@ out: · 64 unmodified lines
1324	return ret;	1324	return ret;
1325	}	1325	}
1326		1326
1327	static int __kprobes in_kprobes_functions(unsigned long addr)	1327	bool __weak arch_within_kprobe_blacklist(unsigned long addr)
1328	{	1328	{
1329	struct kprobe_blackpoint *kb;	1329	/* The __kprobes marked functions and entry code must not be probed */
		1330	return addr >= (unsigned long)__kprobes_text_start &&
		1331	addr < (unsigned long)__kprobes_text_end;
		1332	}
1330		1333
1331	if (addr >= (unsigned long)__kprobes_text_start &&	1334	static bool within_kprobe_blacklist(unsigned long addr)
1332	addr < (unsigned long)__kprobes_text_end)	1335	{
1333	return -EINVAL;	1336	struct kprobe_blacklist_entry *ent;
		1337
		1338	if (arch_within_kprobe_blacklist(addr))
		1339	return true;
1334	/*	1340	/*
1335	* If there exists a kprobe_blacklist, verify and	1341	* If there exists a kprobe_blacklist, verify and
1336	* fail any probe registration in the prohibited area	1342	* fail any probe registration in the prohibited area
1337	*/	1343	*/
1338	for (kb = kprobe_blacklist; kb->name != NULL; kb++) {	1344	list_for_each_entry(ent, &kprobe_blacklist, list) {
1339	if (kb->start_addr) {	1345	if (addr >= ent->start_addr && addr < ent->end_addr)
1340	if (addr >= kb->start_addr &&	1346	return true;
1341	addr < (kb->start_addr + kb->range))
1342	return -EINVAL;
1343	}
1344	}	1347	}
1345	return 0;	1348
		1349	return false;
1346	}	1350	}
1347		1351
1348	/*	1352	/*
@@ -1351,7 +1355,7 @@ static int __kprobes in_kprobes_functions(unsigned long addr) · 2 unmodified lines
1351	* This returns encoded errors if it fails to look up symbol or invalid	1355	* This returns encoded errors if it fails to look up symbol or invalid
1352	* combination of parameters.	1356	* combination of parameters.
1353	*/	1357	*/
1354	static kprobe_opcode_t __kprobes kprobe_addr(struct kprobe p)	1358	static kprobe_opcode_t kprobe_addr(struct kprobe p)
1355	{	1359	{
1356	kprobe_opcode_t *addr = p->addr;	1360	kprobe_opcode_t *addr = p->addr;
1357		1361
@@ -1374,7 +1378,7 @@ invalid: · 16 unmodified lines
1374	}	1378	}
1375		1379
1376	/* Check passed kprobe is valid and return kprobe in kprobe_table. */	1380	/* Check passed kprobe is valid and return kprobe in kprobe_table. */
1377	static struct kprobe * __kprobes __get_valid_kprobe(struct kprobe *p)	1381	static struct kprobe __get_valid_kprobe(struct kprobe p)
1378	{	1382	{
1379	struct kprobe ap, list_p;	1383	struct kprobe ap, list_p;
1380		1384
@@ -1406,8 +1410,8 @@ static inline int check_kprobe_rereg(struct kprobe *p) · 25 unmodified lines
1406	return ret;	1410	return ret;
1407	}	1411	}
1408		1412
1409	static __kprobes int check_kprobe_address_safe(struct kprobe *p,	1413	static int check_kprobe_address_safe(struct kprobe *p,
1410	struct module **probed_mod)	1414	struct module **probed_mod)
1411	{	1415	{
1412	int ret = 0;	1416	int ret = 0;
1413	unsigned long ftrace_addr;	1417	unsigned long ftrace_addr;
@@ -1433,7 +1437,7 @@ static __kprobes int check_kprobe_address_safe(struct kprobe *p, · 19 unmodified lines
1433		1437
1434	/* Ensure it is not in reserved area nor out of text */	1438	/* Ensure it is not in reserved area nor out of text */
1435	if (!kernel_text_address((unsigned long) p->addr) \|\|	1439	if (!kernel_text_address((unsigned long) p->addr) \|\|
1436	in_kprobes_functions((unsigned long) p->addr) \|\|	1440	within_kprobe_blacklist((unsigned long) p->addr) \|\|
1437	jump_label_text_reserved(p->addr, p->addr)) {	1441	jump_label_text_reserved(p->addr, p->addr)) {
1438	ret = -EINVAL;	1442	ret = -EINVAL;
1439	goto out;	1443	goto out;
@@ -1469,7 +1473,7 @@ out: · 29 unmodified lines
1469	return ret;	1473	return ret;
1470	}	1474	}
1471		1475
1472	int __kprobes register_kprobe(struct kprobe *p)	1476	int register_kprobe(struct kprobe *p)
1473	{	1477	{
1474	int ret;	1478	int ret;
1475	struct kprobe *old_p;	1479	struct kprobe *old_p;
@@ -1531,7 +1535,7 @@ out: · 55 unmodified lines
1531	EXPORT_SYMBOL_GPL(register_kprobe);	1535	EXPORT_SYMBOL_GPL(register_kprobe);
1532		1536
1533	/* Check if all probes on the aggrprobe are disabled */	1537	/* Check if all probes on the aggrprobe are disabled */
1534	static int __kprobes aggr_kprobe_disabled(struct kprobe *ap)	1538	static int aggr_kprobe_disabled(struct kprobe *ap)
1535	{	1539	{
1536	struct kprobe *kp;	1540	struct kprobe *kp;
1537		1541
@@ -1547,7 +1551,7 @@ static int __kprobes aggr_kprobe_disabled(struct kprobe *ap) · 9 unmodified lines
1547	}	1551	}
1548		1552
1549	/* Disable one kprobe: Make sure called under kprobe_mutex is locked */	1553	/* Disable one kprobe: Make sure called under kprobe_mutex is locked */
1550	static struct kprobe __kprobes __disable_kprobe(struct kprobe p)	1554	static struct kprobe __disable_kprobe(struct kprobe p)
1551	{	1555	{
1552	struct kprobe *orig_p;	1556	struct kprobe *orig_p;
1553		1557
@@ -1574,7 +1578,7 @@ static struct kprobe __kprobes __disable_kprobe(struct kprobe p) · 20 unmodified lines
1574	/*	1578	/*
1575	* Unregister a kprobe without a scheduler synchronization.	1579	* Unregister a kprobe without a scheduler synchronization.
1576	*/	1580	*/
1577	static int __kprobes __unregister_kprobe_top(struct kprobe *p)	1581	static int __unregister_kprobe_top(struct kprobe *p)
1578	{	1582	{
1579	struct kprobe ap, list_p;	1583	struct kprobe ap, list_p;
1580		1584
@@ -1631,7 +1635,7 @@ disarmed: · 50 unmodified lines
1631	return 0;	1635	return 0;
1632	}	1636	}
1633		1637
1634	static void __kprobes __unregister_kprobe_bottom(struct kprobe *p)	1638	static void __unregister_kprobe_bottom(struct kprobe *p)
1635	{	1639	{
1636	struct kprobe *ap;	1640	struct kprobe *ap;
1637		1641
@@ -1647,7 +1651,7 @@ static void __kprobes __unregister_kprobe_bottom(struct kprobe *p) · 9 unmodified lines
1647	/* Otherwise, do nothing. */	1651	/* Otherwise, do nothing. */
1648	}	1652	}
1649		1653
1650	int __kprobes register_kprobes(struct kprobe **kps, int num)	1654	int register_kprobes(struct kprobe **kps, int num)
1651	{	1655	{
1652	int i, ret = 0;	1656	int i, ret = 0;
1653		1657
@@ -1665,13 +1669,13 @@ int __kprobes register_kprobes(struct kprobe **kps, int num) · 11 unmodified lines
1665	}	1669	}
1666	EXPORT_SYMBOL_GPL(register_kprobes);	1670	EXPORT_SYMBOL_GPL(register_kprobes);
1667		1671
1668	void __kprobes unregister_kprobe(struct kprobe *p)	1672	void unregister_kprobe(struct kprobe *p)
1669	{	1673	{
1670	unregister_kprobes(&p, 1);	1674	unregister_kprobes(&p, 1);
1671	}	1675	}
1672	EXPORT_SYMBOL_GPL(unregister_kprobe);	1676	EXPORT_SYMBOL_GPL(unregister_kprobe);
1673		1677
1674	void __kprobes unregister_kprobes(struct kprobe **kps, int num)	1678	void unregister_kprobes(struct kprobe **kps, int num)
1675	{	1679	{
1676	int i;	1680	int i;
1677		1681
@@ -1700,7 +1704,7 @@ unsigned long __weak arch_deref_entry_point(void *entry) · 22 unmodified lines
1700	return (unsigned long)entry;	1704	return (unsigned long)entry;
1701	}	1705	}
1702		1706
1703	int __kprobes register_jprobes(struct jprobe **jps, int num)	1707	int register_jprobes(struct jprobe **jps, int num)
1704	{	1708	{
1705	struct jprobe *jp;	1709	struct jprobe *jp;
1706	int ret = 0, i;	1710	int ret = 0, i;
@@ -1731,19 +1735,19 @@ int __kprobes register_jprobes(struct jprobe **jps, int num) · 24 unmodified lines
1731	}	1735	}
1732	EXPORT_SYMBOL_GPL(register_jprobes);	1736	EXPORT_SYMBOL_GPL(register_jprobes);
1733		1737
1734	int __kprobes register_jprobe(struct jprobe *jp)	1738	int register_jprobe(struct jprobe *jp)
1735	{	1739	{
1736	return register_jprobes(&jp, 1);	1740	return register_jprobes(&jp, 1);
1737	}	1741	}
1738	EXPORT_SYMBOL_GPL(register_jprobe);	1742	EXPORT_SYMBOL_GPL(register_jprobe);
1739		1743
1740	void __kprobes unregister_jprobe(struct jprobe *jp)	1744	void unregister_jprobe(struct jprobe *jp)
1741	{	1745	{
1742	unregister_jprobes(&jp, 1);	1746	unregister_jprobes(&jp, 1);
1743	}	1747	}
1744	EXPORT_SYMBOL_GPL(unregister_jprobe);	1748	EXPORT_SYMBOL_GPL(unregister_jprobe);
1745		1749
1746	void __kprobes unregister_jprobes(struct jprobe **jps, int num)	1750	void unregister_jprobes(struct jprobe **jps, int num)
1747	{	1751	{
1748	int i;	1752	int i;
1749		1753
@@ -1768,8 +1772,7 @@ EXPORT_SYMBOL_GPL(unregister_jprobes); · 18 unmodified lines
1768	* This kprobe pre_handler is registered with every kretprobe. When probe	1772	* This kprobe pre_handler is registered with every kretprobe. When probe
1769	* hits it will set up the return probe.	1773	* hits it will set up the return probe.
1770	*/	1774	*/
1771	static int __kprobes pre_handler_kretprobe(struct kprobe *p,	1775	static int pre_handler_kretprobe(struct kprobe p, struct pt_regs regs)
1772	struct pt_regs *regs)
1773	{	1776	{
1774	struct kretprobe *rp = container_of(p, struct kretprobe, kp);	1777	struct kretprobe *rp = container_of(p, struct kretprobe, kp);
1775	unsigned long hash, flags = 0;	1778	unsigned long hash, flags = 0;
@@ -1807,8 +1810,9 @@ static int __kprobes pre_handler_kretprobe(struct kprobe *p, · 31 unmodified lines
1807	}	1810	}
1808	return 0;	1811	return 0;
1809	}	1812	}
		1813	NOKPROBE_SYMBOL(pre_handler_kretprobe);
1810		1814
1811	int __kprobes register_kretprobe(struct kretprobe *rp)	1815	int register_kretprobe(struct kretprobe *rp)
1812	{	1816	{
1813	int ret = 0;	1817	int ret = 0;
1814	struct kretprobe_instance *inst;	1818	struct kretprobe_instance *inst;
@@ -1861,7 +1865,7 @@ int __kprobes register_kretprobe(struct kretprobe *rp) · 46 unmodified lines
1861	}	1865	}
1862	EXPORT_SYMBOL_GPL(register_kretprobe);	1866	EXPORT_SYMBOL_GPL(register_kretprobe);
1863		1867
1864	int __kprobes register_kretprobes(struct kretprobe **rps, int num)	1868	int register_kretprobes(struct kretprobe **rps, int num)
1865	{	1869	{
1866	int ret = 0, i;	1870	int ret = 0, i;
1867		1871
@@ -1879,13 +1883,13 @@ int __kprobes register_kretprobes(struct kretprobe **rps, int num) · 11 unmodified lines
1879	}	1883	}
1880	EXPORT_SYMBOL_GPL(register_kretprobes);	1884	EXPORT_SYMBOL_GPL(register_kretprobes);
1881		1885
1882	void __kprobes unregister_kretprobe(struct kretprobe *rp)	1886	void unregister_kretprobe(struct kretprobe *rp)
1883	{	1887	{
1884	unregister_kretprobes(&rp, 1);	1888	unregister_kretprobes(&rp, 1);
1885	}	1889	}
1886	EXPORT_SYMBOL_GPL(unregister_kretprobe);	1890	EXPORT_SYMBOL_GPL(unregister_kretprobe);
1887		1891
1888	void __kprobes unregister_kretprobes(struct kretprobe **rps, int num)	1892	void unregister_kretprobes(struct kretprobe **rps, int num)
1889	{	1893	{
1890	int i;	1894	int i;
1891		1895
@@ -1908,38 +1912,38 @@ void __kprobes unregister_kretprobes(struct kretprobe **rps, int num) · 16 unmodified lines
1908	EXPORT_SYMBOL_GPL(unregister_kretprobes);	1912	EXPORT_SYMBOL_GPL(unregister_kretprobes);
1909		1913
1910	#else /* CONFIG_KRETPROBES */	1914	#else /* CONFIG_KRETPROBES */
1911	int __kprobes register_kretprobe(struct kretprobe *rp)	1915	int register_kretprobe(struct kretprobe *rp)
1912	{	1916	{
1913	return -ENOSYS;	1917	return -ENOSYS;
1914	}	1918	}
1915	EXPORT_SYMBOL_GPL(register_kretprobe);	1919	EXPORT_SYMBOL_GPL(register_kretprobe);
1916		1920
1917	int __kprobes register_kretprobes(struct kretprobe **rps, int num)	1921	int register_kretprobes(struct kretprobe **rps, int num)
1918	{	1922	{
1919	return -ENOSYS;	1923	return -ENOSYS;
1920	}	1924	}
1921	EXPORT_SYMBOL_GPL(register_kretprobes);	1925	EXPORT_SYMBOL_GPL(register_kretprobes);
1922		1926
1923	void __kprobes unregister_kretprobe(struct kretprobe *rp)	1927	void unregister_kretprobe(struct kretprobe *rp)
1924	{	1928	{
1925	}	1929	}
1926	EXPORT_SYMBOL_GPL(unregister_kretprobe);	1930	EXPORT_SYMBOL_GPL(unregister_kretprobe);
1927		1931
1928	void __kprobes unregister_kretprobes(struct kretprobe **rps, int num)	1932	void unregister_kretprobes(struct kretprobe **rps, int num)
1929	{	1933	{
1930	}	1934	}
1931	EXPORT_SYMBOL_GPL(unregister_kretprobes);	1935	EXPORT_SYMBOL_GPL(unregister_kretprobes);
1932		1936
1933	static int __kprobes pre_handler_kretprobe(struct kprobe *p,	1937	static int pre_handler_kretprobe(struct kprobe p, struct pt_regs regs)
1934	struct pt_regs *regs)
1935	{	1938	{
1936	return 0;	1939	return 0;
1937	}	1940	}
		1941	NOKPROBE_SYMBOL(pre_handler_kretprobe);
1938		1942
1939	#endif /* CONFIG_KRETPROBES */	1943	#endif /* CONFIG_KRETPROBES */
1940		1944
1941	/* Set the kprobe gone and remove its instruction buffer. */	1945	/* Set the kprobe gone and remove its instruction buffer. */
1942	static void __kprobes kill_kprobe(struct kprobe *p)	1946	static void kill_kprobe(struct kprobe *p)
1943	{	1947	{
1944	struct kprobe *kp;	1948	struct kprobe *kp;
1945		1949
@@ -1963,7 +1967,7 @@ static void __kprobes kill_kprobe(struct kprobe *p) · 17 unmodified lines
1963	}	1967	}
1964		1968
1965	/* Disable one kprobe */	1969	/* Disable one kprobe */
1966	int __kprobes disable_kprobe(struct kprobe *kp)	1970	int disable_kprobe(struct kprobe *kp)
1967	{	1971	{
1968	int ret = 0;	1972	int ret = 0;
1969		1973
@@ -1979,7 +1983,7 @@ int __kprobes disable_kprobe(struct kprobe *kp) · 9 unmodified lines
1979	EXPORT_SYMBOL_GPL(disable_kprobe);	1983	EXPORT_SYMBOL_GPL(disable_kprobe);
1980		1984
1981	/* Enable one kprobe */	1985	/* Enable one kprobe */
1982	int __kprobes enable_kprobe(struct kprobe *kp)	1986	int enable_kprobe(struct kprobe *kp)
1983	{	1987	{
1984	int ret = 0;	1988	int ret = 0;
1985	struct kprobe *p;	1989	struct kprobe *p;
@@ -2012,16 +2016,49 @@ out: · 26 unmodified lines
2012	}	2016	}
2013	EXPORT_SYMBOL_GPL(enable_kprobe);	2017	EXPORT_SYMBOL_GPL(enable_kprobe);
2014		2018
2015	void __kprobes dump_kprobe(struct kprobe *kp)	2019	void dump_kprobe(struct kprobe *kp)
2016	{	2020	{
2017	printk(KERN_WARNING "Dumping kprobe:\n");	2021	printk(KERN_WARNING "Dumping kprobe:\n");
2018	printk(KERN_WARNING "Name: %s\nAddress: %p\nOffset: %x\n",	2022	printk(KERN_WARNING "Name: %s\nAddress: %p\nOffset: %x\n",
2019	kp->symbol_name, kp->addr, kp->offset);	2023	kp->symbol_name, kp->addr, kp->offset);
2020	}	2024	}
		2025	NOKPROBE_SYMBOL(dump_kprobe);
		2026
		2027	/*
		2028	* Lookup and populate the kprobe_blacklist.
		2029	*
		2030	* Unlike the kretprobe blacklist, we'll need to determine
		2031	* the range of addresses that belong to the said functions,
		2032	* since a kprobe need not necessarily be at the beginning
		2033	* of a function.
		2034	*/
		2035	static int __init populate_kprobe_blacklist(unsigned long *start,
		2036	unsigned long *end)
		2037	{
		2038	unsigned long *iter;
		2039	struct kprobe_blacklist_entry *ent;
		2040	unsigned long offset = 0, size = 0;
		2041
		2042	for (iter = start; iter < end; iter++) {
		2043	if (!kallsyms_lookup_size_offset(*iter, &size, &offset)) {
		2044	pr_err("Failed to find blacklist %p\n", (void )iter);
		2045	continue;
		2046	}
		2047
		2048	ent = kmalloc(sizeof(*ent), GFP_KERNEL);
		2049	if (!ent)
		2050	return -ENOMEM;
		2051	ent->start_addr = *iter;
		2052	ent->end_addr = *iter + size;
		2053	INIT_LIST_HEAD(&ent->list);
		2054	list_add_tail(&ent->list, &kprobe_blacklist);
		2055	}
		2056	return 0;
		2057	}
2021		2058
2022	/* Module notifier call back, checking kprobes on the module */	2059	/* Module notifier call back, checking kprobes on the module */
2023	static int __kprobes kprobes_module_callback(struct notifier_block *nb,	2060	static int kprobes_module_callback(struct notifier_block *nb,
2024	unsigned long val, void *data)	2061	unsigned long val, void *data)
2025	{	2062	{
2026	struct module *mod = data;	2063	struct module *mod = data;
2027	struct hlist_head *head;	2064	struct hlist_head *head;
@@ -2062,14 +2099,13 @@ static struct notifier_block kprobe_module_nb = { · 34 unmodified lines
2062	.priority = 0	2099	.priority = 0
2063	};	2100	};
2064		2101
		2102	/* Markers of _kprobe_blacklist section */
		2103	extern unsigned long __start_kprobe_blacklist[];
		2104	extern unsigned long __stop_kprobe_blacklist[];
		2105
2065	static int __init init_kprobes(void)	2106	static int __init init_kprobes(void)
2066	{	2107	{
2067	int i, err = 0;	2108	int i, err = 0;
2068	unsigned long offset = 0, size = 0;
2069	char *modname, namebuf[KSYM_NAME_LEN];
2070	const char *symbol_name;
2071	void *addr;
2072	struct kprobe_blackpoint *kb;
2073		2109
2074	/* FIXME allocate the probe table, currently defined statically */	2110	/* FIXME allocate the probe table, currently defined statically */
2075	/* initialize all list heads */	2111	/* initialize all list heads */
@@ -2079,26 +2115,11 @@ static int __init init_kprobes(void) · 3 unmodified lines
2079	raw_spin_lock_init(&(kretprobe_table_locks[i].lock));	2115	raw_spin_lock_init(&(kretprobe_table_locks[i].lock));
2080	}	2116	}
2081		2117
2082	/*	2118	err = populate_kprobe_blacklist(__start_kprobe_blacklist,
2083	* Lookup and populate the kprobe_blacklist.	2119	__stop_kprobe_blacklist);
2084	*	2120	if (err) {
2085	* Unlike the kretprobe blacklist, we'll need to determine	2121	pr_err("kprobes: failed to populate blacklist: %d\n", err);
2086	* the range of addresses that belong to the said functions,	2122	pr_err("Please take care of using kprobes.\n");
2087	* since a kprobe need not necessarily be at the beginning
2088	* of a function.
2089	*/
2090	for (kb = kprobe_blacklist; kb->name != NULL; kb++) {
2091	kprobe_lookup_name(kb->name, addr);
2092	if (!addr)
2093	continue;
2094
2095	kb->start_addr = (unsigned long)addr;
2096	symbol_name = kallsyms_lookup(kb->start_addr,
2097	&size, &offset, &modname, namebuf);
2098	if (!symbol_name)
2099	kb->range = 0;
2100	else
2101	kb->range = size;
2102	}	2123	}
2103		2124
2104	if (kretprobe_blacklist_size) {	2125	if (kretprobe_blacklist_size) {
@@ -2138,7 +2159,7 @@ static int __init init_kprobes(void) · 33 unmodified lines
2138	}	2159	}
2139		2160
2140	#ifdef CONFIG_DEBUG_FS	2161	#ifdef CONFIG_DEBUG_FS
2141	static void __kprobes report_probe(struct seq_file pi, struct kprobe p,	2162	static void report_probe(struct seq_file pi, struct kprobe p,
2142	const char sym, int offset, char modname, struct kprobe *pp)	2163	const char sym, int offset, char modname, struct kprobe *pp)
2143	{	2164	{
2144	char *kprobe_type;	2165	char *kprobe_type;
@@ -2167,12 +2188,12 @@ static void __kprobes report_probe(struct seq_file pi, struct kprobe p, · 22 unmodified lines
2167	(kprobe_ftrace(pp) ? "[FTRACE]" : ""));	2188	(kprobe_ftrace(pp) ? "[FTRACE]" : ""));
2168	}	2189	}
2169		2190
2170	static void __kprobes kprobe_seq_start(struct seq_file f, loff_t *pos)	2191	static void kprobe_seq_start(struct seq_file f, loff_t *pos)
2171	{	2192	{
2172	return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL;	2193	return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL;
2173	}	2194	}
2174		2195
2175	static void __kprobes kprobe_seq_next(struct seq_file f, void v, loff_t pos)	2196	static void kprobe_seq_next(struct seq_file f, void v, loff_t pos)
2176	{	2197	{
2177	(*pos)++;	2198	(*pos)++;
2178	if (*pos >= KPROBE_TABLE_SIZE)	2199	if (*pos >= KPROBE_TABLE_SIZE)
@@ -2180,12 +2201,12 @@ static void __kprobes kprobe_seq_next(struct seq_file f, void v, loff_t pos) · 1 unmodified line
2180	return pos;	2201	return pos;
2181	}	2202	}
2182		2203
2183	static void __kprobes kprobe_seq_stop(struct seq_file f, void v)	2204	static void kprobe_seq_stop(struct seq_file f, void v)
2184	{	2205	{
2185	/* Nothing to do */	2206	/* Nothing to do */
2186	}	2207	}
2187		2208
2188	static int __kprobes show_kprobe_addr(struct seq_file pi, void v)	2209	static int show_kprobe_addr(struct seq_file pi, void v)
2189	{	2210	{
2190	struct hlist_head *head;	2211	struct hlist_head *head;
2191	struct kprobe p, kp;	2212	struct kprobe p, kp;
@@ -2216,7 +2237,7 @@ static const struct seq_operations kprobes_seq_ops = { · 24 unmodified lines
2216	.show = show_kprobe_addr	2237	.show = show_kprobe_addr
2217	};	2238	};
2218		2239
2219	static int __kprobes kprobes_open(struct inode inode, struct file filp)	2240	static int kprobes_open(struct inode inode, struct file filp)
2220	{	2241	{
2221	return seq_open(filp, &kprobes_seq_ops);	2242	return seq_open(filp, &kprobes_seq_ops);
2222	}	2243	}
@@ -2228,7 +2249,47 @@ static const struct file_operations debugfs_kprobes_operations = { · 5 unmodified lines
2228	.release = seq_release,	2249	.release = seq_release,
2229	};	2250	};
2230		2251
2231	static void __kprobes arm_all_kprobes(void)	2252	/* kprobes/blacklist -- shows which functions can not be probed */
		2253	static void kprobe_blacklist_seq_start(struct seq_file m, loff_t *pos)
		2254	{
		2255	return seq_list_start(&kprobe_blacklist, *pos);
		2256	}
		2257
		2258	static void kprobe_blacklist_seq_next(struct seq_file m, void v, loff_t pos)
		2259	{
		2260	return seq_list_next(v, &kprobe_blacklist, pos);
		2261	}
		2262
		2263	static int kprobe_blacklist_seq_show(struct seq_file m, void v)
		2264	{
		2265	struct kprobe_blacklist_entry *ent =
		2266	list_entry(v, struct kprobe_blacklist_entry, list);
		2267
		2268	seq_printf(m, "0x%p-0x%p\t%ps\n", (void *)ent->start_addr,
		2269	(void )ent->end_addr, (void )ent->start_addr);
		2270	return 0;
		2271	}
		2272
		2273	static const struct seq_operations kprobe_blacklist_seq_ops = {
		2274	.start = kprobe_blacklist_seq_start,
		2275	.next = kprobe_blacklist_seq_next,
		2276	.stop = kprobe_seq_stop, /* Reuse void function */
		2277	.show = kprobe_blacklist_seq_show,
		2278	};
		2279
		2280	static int kprobe_blacklist_open(struct inode inode, struct file filp)
		2281	{
		2282	return seq_open(filp, &kprobe_blacklist_seq_ops);
		2283	}
		2284
		2285	static const struct file_operations debugfs_kprobe_blacklist_ops = {
		2286	.open = kprobe_blacklist_open,
		2287	.read = seq_read,
		2288	.llseek = seq_lseek,
		2289	.release = seq_release,
		2290	};
		2291
		2292	static void arm_all_kprobes(void)
2232	{	2293	{
2233	struct hlist_head *head;	2294	struct hlist_head *head;
2234	struct kprobe *p;	2295	struct kprobe *p;
@@ -2256,7 +2317,7 @@ already_enabled: · 21 unmodified lines
2256	return;	2317	return;
2257	}	2318	}
2258		2319
2259	static void __kprobes disarm_all_kprobes(void)	2320	static void disarm_all_kprobes(void)
2260	{	2321	{
2261	struct hlist_head *head;	2322	struct hlist_head *head;
2262	struct kprobe *p;	2323	struct kprobe *p;
@@ -2340,7 +2401,7 @@ static const struct file_operations fops_kp = { · 77 unmodified lines
2340	.llseek = default_llseek,	2401	.llseek = default_llseek,
2341	};	2402	};
2342		2403
2343	static int __kprobes debugfs_kprobe_init(void)	2404	static int __init debugfs_kprobe_init(void)
2344	{	2405	{
2345	struct dentry dir, file;	2406	struct dentry dir, file;
2346	unsigned int value = 1;	2407	unsigned int value = 1;
@@ -2351,19 +2412,24 @@ static int __kprobes debugfs_kprobe_init(void) · 4 unmodified lines
2351		2412
2352	file = debugfs_create_file("list", 0444, dir, NULL,	2413	file = debugfs_create_file("list", 0444, dir, NULL,
2353	&debugfs_kprobes_operations);	2414	&debugfs_kprobes_operations);
2354	if (!file) {	2415	if (!file)
2355	debugfs_remove(dir);	2416	goto error;
2356	return -ENOMEM;
2357	}
2358		2417
2359	file = debugfs_create_file("enabled", 0600, dir,	2418	file = debugfs_create_file("enabled", 0600, dir,
2360	&value, &fops_kp);	2419	&value, &fops_kp);
2361	if (!file) {	2420	if (!file)
2362	debugfs_remove(dir);	2421	goto error;
2363	return -ENOMEM;	2422
2364	}	2423	file = debugfs_create_file("blacklist", 0444, dir, NULL,
		2424	&debugfs_kprobe_blacklist_ops);
		2425	if (!file)
		2426	goto error;
2365		2427
2366	return 0;	2428	return 0;
		2429
		2430	error:
		2431	debugfs_remove(dir);
		2432	return -ENOMEM;
2367	}	2433	}
2368		2434
2369	late_initcall(debugfs_kprobe_init);	2435	late_initcall(debugfs_kprobe_init);

kernel/notifier.c+13−9

  *	@returns:	notifier_call_chain returns the value returned by the
  *			last notifier function called.
  */
-static int __kprobes notifier_call_chain(struct notifier_block **nl,
+static int notifier_call_chain(struct notifier_block **nl,
-					unsigned long val, void *v,
+			       unsigned long val, void *v,
-					int nr_to_call,	int *nr_calls)
+			       int nr_to_call, int *nr_calls)
 {
 	int ret = NOTIFY_DONE;
 	struct notifier_block *nb, *next_nb;
 	}
 	return ret;
 }
+NOKPROBE_SYMBOL(notifier_call_chain);
 /*
  *	Atomic notifier chain routines.  Registration and unregistration
  *	Otherwise the return value is the return value
  *	of the last notifier function called.
  */
-int __kprobes __atomic_notifier_call_chain(struct atomic_notifier_head *nh,
+int __atomic_notifier_call_chain(struct atomic_notifier_head *nh,
-					unsigned long val, void *v,
+				 unsigned long val, void *v,
-					int nr_to_call, int *nr_calls)
+				 int nr_to_call, int *nr_calls)
 {
 	int ret;
 	return ret;
 }
 EXPORT_SYMBOL_GPL(__atomic_notifier_call_chain);
+NOKPROBE_SYMBOL(__atomic_notifier_call_chain);
-int __kprobes atomic_notifier_call_chain(struct atomic_notifier_head *nh,
+int atomic_notifier_call_chain(struct atomic_notifier_head *nh,
-		unsigned long val, void *v)
+			       unsigned long val, void *v)
 {
 	return __atomic_notifier_call_chain(nh, val, v, -1, NULL);
 }
 EXPORT_SYMBOL_GPL(atomic_notifier_call_chain);
+NOKPROBE_SYMBOL(atomic_notifier_call_chain);
 /*
  *	Blocking notifier chain routines.  All access to the chain is
 static ATOMIC_NOTIFIER_HEAD(die_chain);
-int notrace __kprobes notify_die(enum die_val val, const char *str,
+int notrace notify_die(enum die_val val, const char *str,
 	       struct pt_regs *regs, long err, int trap, int sig)
 {
 	struct die_args args = {
 	};
 	return atomic_notifier_call_chain(&die_chain, val, &args);
 }
+NOKPROBE_SYMBOL(notify_die);
 int register_die_notifier(struct notifier_block *nb)
 {

kernel/sched/core.c+5−2

 #if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
 				defined(CONFIG_PREEMPT_TRACER))
-void __kprobes preempt_count_add(int val)
+void preempt_count_add(int val)
 {
 #ifdef CONFIG_DEBUG_PREEMPT
 	/*
 	}
 }
 EXPORT_SYMBOL(preempt_count_add);
+NOKPROBE_SYMBOL(preempt_count_add);
-void __kprobes preempt_count_sub(int val)
+void preempt_count_sub(int val)
 {
 #ifdef CONFIG_DEBUG_PREEMPT
 	/*
 	__preempt_count_sub(val);
 }
 EXPORT_SYMBOL(preempt_count_sub);
+NOKPROBE_SYMBOL(preempt_count_sub);
 #endif
 		barrier();
 	} while (need_resched());
 }
+NOKPROBE_SYMBOL(preempt_schedule);
 EXPORT_SYMBOL(preempt_schedule);
 #endif /* CONFIG_PREEMPT */

kernel/trace/trace_event_perf.c+3−2

 	tp_event->class->reg(tp_event, TRACE_REG_PERF_DEL, p_event);
 }
-__kprobes void *perf_trace_buf_prepare(int size, unsigned short type,
+void *perf_trace_buf_prepare(int size, unsigned short type,
-				       struct pt_regs *regs, int *rctxp)
+			     struct pt_regs *regs, int *rctxp)
 {
 	struct trace_entry *entry;
 	unsigned long flags;
 	return raw_data;
 }
 EXPORT_SYMBOL_GPL(perf_trace_buf_prepare);
+NOKPROBE_SYMBOL(perf_trace_buf_prepare);
 #ifdef CONFIG_FUNCTION_TRACER
 static void

kernel/trace/trace_kprobe.c+41−30

 	(sizeof(struct probe_arg) * (n)))
-static __kprobes bool trace_kprobe_is_return(struct trace_kprobe *tk)
+static nokprobe_inline bool trace_kprobe_is_return(struct trace_kprobe *tk)
 {
 	return tk->rp.handler != NULL;
 }
-static __kprobes const char *trace_kprobe_symbol(struct trace_kprobe *tk)
+static nokprobe_inline const char *trace_kprobe_symbol(struct trace_kprobe *tk)
 {
 	return tk->symbol ? tk->symbol : "unknown";
 }
-static __kprobes unsigned long trace_kprobe_offset(struct trace_kprobe *tk)
+static nokprobe_inline unsigned long trace_kprobe_offset(struct trace_kprobe *tk)
 {
 	return tk->rp.kp.offset;
 }
-static __kprobes bool trace_kprobe_has_gone(struct trace_kprobe *tk)
+static nokprobe_inline bool trace_kprobe_has_gone(struct trace_kprobe *tk)
 {
 	return !!(kprobe_gone(&tk->rp.kp));
 }
-static __kprobes bool trace_kprobe_within_module(struct trace_kprobe *tk,
+static nokprobe_inline bool trace_kprobe_within_module(struct trace_kprobe *tk,
 						 struct module *mod)
 {
 	int len = strlen(mod->name);
 	return strncmp(mod->name, name, len) == 0 && name[len] == ':';
 }
-static __kprobes bool trace_kprobe_is_on_module(struct trace_kprobe *tk)
+static nokprobe_inline bool trace_kprobe_is_on_module(struct trace_kprobe *tk)
 {
 	return !!strchr(trace_kprobe_symbol(tk), ':');
 }
  * Kprobes-specific fetch functions
  */
 #define DEFINE_FETCH_stack(type)					\
-static __kprobes void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs,\
+static void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs,		\
 					  void *offset, void *dest)	\
 {									\
 	*(type *)dest = (type)regs_get_kernel_stack_nth(regs,		\
 				(unsigned int)((unsigned long)offset));	\
-}
+}									\
+NOKPROBE_SYMBOL(FETCH_FUNC_NAME(stack, type));
 DEFINE_BASIC_FETCH_FUNCS(stack)
 /* No string on the stack entry */
 #define fetch_stack_string	NULL
 #define fetch_stack_string_size	NULL
 #define DEFINE_FETCH_memory(type)					\
-static __kprobes void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs,\
+static void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs,		\
 					  void *addr, void *dest)	\
 {									\
 	type retval;							\
 		*(type *)dest = 0;					\
 	else								\
 		*(type *)dest = retval;					\
-}
+}									\
+NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, type));
 DEFINE_BASIC_FETCH_FUNCS(memory)
 /*
  * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max
  * length and relative data location.
  */
-static __kprobes void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs,
+static void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs,
-						      void *addr, void *dest)
+					    void *addr, void *dest)
 {
 	long ret;
 	int maxlen = get_rloc_len(*(u32 *)dest);
 					      get_rloc_offs(*(u32 *)dest));
 	}
 }
+NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, string));
 /* Return the length of string -- including null terminal byte */
-static __kprobes void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs,
+static void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs,
-							void *addr, void *dest)
+						 void *addr, void *dest)
 {
 	mm_segment_t old_fs;
 	int ret, len = 0;
 	else
 		*(u32 *)dest = len;
 }
+NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, string_size));
 #define DEFINE_FETCH_symbol(type)					\
-__kprobes void FETCH_FUNC_NAME(symbol, type)(struct pt_regs *regs,	\
+void FETCH_FUNC_NAME(symbol, type)(struct pt_regs *regs, void *data, void *dest)\
-					  void *data, void *dest)	\
 {									\
 	struct symbol_cache *sc = data;					\
 	if (sc->addr)							\
 		fetch_memory_##type(regs, (void *)sc->addr, dest);	\
 	else								\
 		*(type *)dest = 0;					\
-}
+}									\
+NOKPROBE_SYMBOL(FETCH_FUNC_NAME(symbol, type));
 DEFINE_BASIC_FETCH_FUNCS(symbol)
 DEFINE_FETCH_symbol(string)
 DEFINE_FETCH_symbol(string_size)
 };
 /* Kprobe handler */
-static __kprobes void
+static nokprobe_inline void
 __kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs,
 		    struct ftrace_event_file *ftrace_file)
 {
 					 entry, irq_flags, pc, regs);
 }
-static __kprobes void
+static void
 kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs)
 {
 	struct event_file_link *link;
 	list_for_each_entry_rcu(link, &tk->tp.files, list)
 		__kprobe_trace_func(tk, regs, link->file);
 }
+NOKPROBE_SYMBOL(kprobe_trace_func);
 /* Kretprobe handler */
-static __kprobes void
+static nokprobe_inline void
 __kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
 		       struct pt_regs *regs,
 		       struct ftrace_event_file *ftrace_file)
 					 entry, irq_flags, pc, regs);
 }
-static __kprobes void
+static void
 kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
 		     struct pt_regs *regs)
 {
 	list_for_each_entry_rcu(link, &tk->tp.files, list)
 		__kretprobe_trace_func(tk, ri, regs, link->file);
 }
+NOKPROBE_SYMBOL(kretprobe_trace_func);
 /* Event entry printers */
 static enum print_line_t
 #ifdef CONFIG_PERF_EVENTS
 /* Kprobe profile handler */
-static __kprobes void
+static void
 kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs)
 {
 	struct ftrace_event_call *call = &tk->tp.call;
 	store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
 	perf_trace_buf_submit(entry, size, rctx, 0, 1, regs, head, NULL);
 }
+NOKPROBE_SYMBOL(kprobe_perf_func);
 /* Kretprobe profile handler */
-static __kprobes void
+static void
 kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
 		    struct pt_regs *regs)
 {
 	store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
 	perf_trace_buf_submit(entry, size, rctx, 0, 1, regs, head, NULL);
 }
+NOKPROBE_SYMBOL(kretprobe_perf_func);
 #endif	/* CONFIG_PERF_EVENTS */
 /*
  * kprobe_trace_self_tests_init() does enable_trace_probe/disable_trace_probe
  * lockless, but we can't race with this __init function.
  */
-static __kprobes
+static int kprobe_register(struct ftrace_event_call *event,
-int kprobe_register(struct ftrace_event_call *event,
+			   enum trace_reg type, void *data)
-		    enum trace_reg type, void *data)
 {
 	struct trace_kprobe *tk = (struct trace_kprobe *)event->data;
 	struct ftrace_event_file *file = data;
 	return 0;
 }
-static __kprobes
+static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs)
-int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs)
 {
 	struct trace_kprobe *tk = container_of(kp, struct trace_kprobe, rp.kp);
 #endif
 	return 0;	/* We don't tweek kernel, so just return 0 */
 }
+NOKPROBE_SYMBOL(kprobe_dispatcher);
-static __kprobes
+static int
-int kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs)
+kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs)
 {
 	struct trace_kprobe *tk = container_of(ri->rp, struct trace_kprobe, rp);
 #endif
 	return 0;	/* We don't tweek kernel, so just return 0 */
 }
+NOKPROBE_SYMBOL(kretprobe_dispatcher);
 static struct trace_event_functions kretprobe_funcs = {
 	.trace		= print_kretprobe_event

kernel/trace/trace_probe.c+35−30

@@ -37,13 +37,13 @@ const char *reserved_field_names[] = { · 36 unmodified lines
37		37
38	/* Printing in basic type function template */	38	/* Printing in basic type function template */
39	#define DEFINE_BASIC_PRINT_TYPE_FUNC(type, fmt) \	39	#define DEFINE_BASIC_PRINT_TYPE_FUNC(type, fmt) \
40	__kprobes int PRINT_TYPE_FUNC_NAME(type)(struct trace_seq *s, \	40	int PRINT_TYPE_FUNC_NAME(type)(struct trace_seq s, const char name, \
41	const char *name, \	41	void data, void ent) \
42	void data, void ent) \
43	{ \	42	{ \
44	return trace_seq_printf(s, " %s=" fmt, name, (type )data); \	43	return trace_seq_printf(s, " %s=" fmt, name, (type )data); \
45	} \	44	} \
46	const char PRINT_TYPE_FMT_NAME(type)[] = fmt;	45	const char PRINT_TYPE_FMT_NAME(type)[] = fmt; \
		46	NOKPROBE_SYMBOL(PRINT_TYPE_FUNC_NAME(type));
47		47
48	DEFINE_BASIC_PRINT_TYPE_FUNC(u8 , "0x%x")	48	DEFINE_BASIC_PRINT_TYPE_FUNC(u8 , "0x%x")
49	DEFINE_BASIC_PRINT_TYPE_FUNC(u16, "0x%x")	49	DEFINE_BASIC_PRINT_TYPE_FUNC(u16, "0x%x")
@@ -55,9 +55,8 @@ DEFINE_BASIC_PRINT_TYPE_FUNC(s32, "%d") · 5 unmodified lines
55	DEFINE_BASIC_PRINT_TYPE_FUNC(s64, "%Ld")	55	DEFINE_BASIC_PRINT_TYPE_FUNC(s64, "%Ld")
56		56
57	/* Print type function for string type */	57	/* Print type function for string type */
58	__kprobes int PRINT_TYPE_FUNC_NAME(string)(struct trace_seq *s,	58	int PRINT_TYPE_FUNC_NAME(string)(struct trace_seq s, const char name,
59	const char *name,	59	void data, void ent)
60	void data, void ent)
61	{	60	{
62	int len = (u32 )data >> 16;	61	int len = (u32 )data >> 16;
63		62
@@ -67,6 +66,7 @@ __kprobes int PRINT_TYPE_FUNC_NAME(string)(struct trace_seq *s, · 3 unmodified lines
67	return trace_seq_printf(s, " %s=\"%s\"", name,	66	return trace_seq_printf(s, " %s=\"%s\"", name,
68	(const char *)get_loc_data(data, ent));	67	(const char *)get_loc_data(data, ent));
69	}	68	}
		69	NOKPROBE_SYMBOL(PRINT_TYPE_FUNC_NAME(string));
70		70
71	const char PRINT_TYPE_FMT_NAME(string)[] = "\\\"%s\\\"";	71	const char PRINT_TYPE_FMT_NAME(string)[] = "\\\"%s\\\"";
72		72
@@ -81,23 +81,24 @@ const char PRINT_TYPE_FMT_NAME(string)[] = "\\\"%s\\\""; · 8 unmodified lines
81		81
82	/* Data fetch function templates */	82	/* Data fetch function templates */
83	#define DEFINE_FETCH_reg(type) \	83	#define DEFINE_FETCH_reg(type) \
84	__kprobes void FETCH_FUNC_NAME(reg, type)(struct pt_regs *regs, \	84	void FETCH_FUNC_NAME(reg, type)(struct pt_regs regs, void offset, void *dest) \
85	void offset, void dest) \
86	{ \	85	{ \
87	(type )dest = (type)regs_get_register(regs, \	86	(type )dest = (type)regs_get_register(regs, \
88	(unsigned int)((unsigned long)offset)); \	87	(unsigned int)((unsigned long)offset)); \
89	}	88	} \
		89	NOKPROBE_SYMBOL(FETCH_FUNC_NAME(reg, type));
90	DEFINE_BASIC_FETCH_FUNCS(reg)	90	DEFINE_BASIC_FETCH_FUNCS(reg)
91	/* No string on the register */	91	/* No string on the register */
92	#define fetch_reg_string NULL	92	#define fetch_reg_string NULL
93	#define fetch_reg_string_size NULL	93	#define fetch_reg_string_size NULL
94		94
95	#define DEFINE_FETCH_retval(type) \	95	#define DEFINE_FETCH_retval(type) \
96	__kprobes void FETCH_FUNC_NAME(retval, type)(struct pt_regs *regs, \	96	void FETCH_FUNC_NAME(retval, type)(struct pt_regs *regs, \
97	void dummy, void dest) \	97	void dummy, void dest) \
98	{ \	98	{ \
99	(type )dest = (type)regs_return_value(regs); \	99	(type )dest = (type)regs_return_value(regs); \
100	}	100	} \
		101	NOKPROBE_SYMBOL(FETCH_FUNC_NAME(retval, type));
101	DEFINE_BASIC_FETCH_FUNCS(retval)	102	DEFINE_BASIC_FETCH_FUNCS(retval)
102	/* No string on the retval */	103	/* No string on the retval */
103	#define fetch_retval_string NULL	104	#define fetch_retval_string NULL
@@ -112,8 +113,8 @@ struct deref_fetch_param { · 8 unmodified lines
112	};	113	};
113		114
114	#define DEFINE_FETCH_deref(type) \	115	#define DEFINE_FETCH_deref(type) \
115	__kprobes void FETCH_FUNC_NAME(deref, type)(struct pt_regs *regs, \	116	void FETCH_FUNC_NAME(deref, type)(struct pt_regs *regs, \
116	void data, void dest) \	117	void data, void dest) \
117	{ \	118	{ \
118	struct deref_fetch_param *dprm = data; \	119	struct deref_fetch_param *dprm = data; \
119	unsigned long addr; \	120	unsigned long addr; \
@@ -123,12 +124,13 @@ __kprobes void FETCH_FUNC_NAME(deref, type)(struct pt_regs *regs, \ · 3 unmodified lines
123	dprm->fetch(regs, (void *)addr, dest); \	124	dprm->fetch(regs, (void *)addr, dest); \
124	} else \	125	} else \
125	(type )dest = 0; \	126	(type )dest = 0; \
126	}	127	} \
		128	NOKPROBE_SYMBOL(FETCH_FUNC_NAME(deref, type));
127	DEFINE_BASIC_FETCH_FUNCS(deref)	129	DEFINE_BASIC_FETCH_FUNCS(deref)
128	DEFINE_FETCH_deref(string)	130	DEFINE_FETCH_deref(string)
129		131
130	__kprobes void FETCH_FUNC_NAME(deref, string_size)(struct pt_regs *regs,	132	void FETCH_FUNC_NAME(deref, string_size)(struct pt_regs *regs,
131	void data, void dest)	133	void data, void dest)
132	{	134	{
133	struct deref_fetch_param *dprm = data;	135	struct deref_fetch_param *dprm = data;
134	unsigned long addr;	136	unsigned long addr;
@@ -140,16 +142,18 @@ __kprobes void FETCH_FUNC_NAME(deref, string_size)(struct pt_regs *regs, · 5 unmodified lines
140	} else	142	} else
141	(string_size )dest = 0;	143	(string_size )dest = 0;
142	}	144	}
		145	NOKPROBE_SYMBOL(FETCH_FUNC_NAME(deref, string_size));
143		146
144	static __kprobes void update_deref_fetch_param(struct deref_fetch_param *data)	147	static void update_deref_fetch_param(struct deref_fetch_param *data)
145	{	148	{
146	if (CHECK_FETCH_FUNCS(deref, data->orig.fn))	149	if (CHECK_FETCH_FUNCS(deref, data->orig.fn))
147	update_deref_fetch_param(data->orig.data);	150	update_deref_fetch_param(data->orig.data);
148	else if (CHECK_FETCH_FUNCS(symbol, data->orig.fn))	151	else if (CHECK_FETCH_FUNCS(symbol, data->orig.fn))
149	update_symbol_cache(data->orig.data);	152	update_symbol_cache(data->orig.data);
150	}	153	}
		154	NOKPROBE_SYMBOL(update_deref_fetch_param);
151		155
152	static __kprobes void free_deref_fetch_param(struct deref_fetch_param *data)	156	static void free_deref_fetch_param(struct deref_fetch_param *data)
153	{	157	{
154	if (CHECK_FETCH_FUNCS(deref, data->orig.fn))	158	if (CHECK_FETCH_FUNCS(deref, data->orig.fn))
155	free_deref_fetch_param(data->orig.data);	159	free_deref_fetch_param(data->orig.data);
@@ -157,6 +161,7 @@ static __kprobes void free_deref_fetch_param(struct deref_fetch_param *data) · 1 unmodified line
157	free_symbol_cache(data->orig.data);	161	free_symbol_cache(data->orig.data);
158	kfree(data);	162	kfree(data);
159	}	163	}
		164	NOKPROBE_SYMBOL(free_deref_fetch_param);
160		165
161	/* Bitfield fetch function */	166	/* Bitfield fetch function */
162	struct bitfield_fetch_param {	167	struct bitfield_fetch_param {
@@ -166,8 +171,8 @@ struct bitfield_fetch_param { · 3 unmodified lines
166	};	171	};
167		172
168	#define DEFINE_FETCH_bitfield(type) \	173	#define DEFINE_FETCH_bitfield(type) \
169	__kprobes void FETCH_FUNC_NAME(bitfield, type)(struct pt_regs *regs, \	174	void FETCH_FUNC_NAME(bitfield, type)(struct pt_regs *regs, \
170	void data, void dest) \	175	void data, void dest) \
171	{ \	176	{ \
172	struct bitfield_fetch_param *bprm = data; \	177	struct bitfield_fetch_param *bprm = data; \
173	type buf = 0; \	178	type buf = 0; \
@@ -177,13 +182,13 @@ __kprobes void FETCH_FUNC_NAME(bitfield, type)(struct pt_regs *regs, \ · 3 unmodified lines
177	buf >>= bprm->low_shift; \	182	buf >>= bprm->low_shift; \
178	} \	183	} \
179	(type )dest = buf; \	184	(type )dest = buf; \
180	}	185	} \
181		186	NOKPROBE_SYMBOL(FETCH_FUNC_NAME(bitfield, type));
182	DEFINE_BASIC_FETCH_FUNCS(bitfield)	187	DEFINE_BASIC_FETCH_FUNCS(bitfield)
183	#define fetch_bitfield_string NULL	188	#define fetch_bitfield_string NULL
184	#define fetch_bitfield_string_size NULL	189	#define fetch_bitfield_string_size NULL
185		190
186	static __kprobes void	191	static void
187	update_bitfield_fetch_param(struct bitfield_fetch_param *data)	192	update_bitfield_fetch_param(struct bitfield_fetch_param *data)
188	{	193	{
189	/*	194	/*
@@ -196,7 +201,7 @@ update_bitfield_fetch_param(struct bitfield_fetch_param *data) · 6 unmodified lines
196	update_symbol_cache(data->orig.data);	201	update_symbol_cache(data->orig.data);
197	}	202	}
198		203
199	static __kprobes void	204	static void
200	free_bitfield_fetch_param(struct bitfield_fetch_param *data)	205	free_bitfield_fetch_param(struct bitfield_fetch_param *data)
201	{	206	{
202	/*	207	/*
@@ -255,17 +260,17 @@ fail: · 52 unmodified lines
255	}	260	}
256		261
257	/* Special function : only accept unsigned long */	262	/* Special function : only accept unsigned long */
258	static __kprobes void fetch_kernel_stack_address(struct pt_regs *regs,	263	static void fetch_kernel_stack_address(struct pt_regs regs, void dummy, void *dest)
259	void dummy, void dest)
260	{	264	{
261	(unsigned long )dest = kernel_stack_pointer(regs);	265	(unsigned long )dest = kernel_stack_pointer(regs);
262	}	266	}
		267	NOKPROBE_SYMBOL(fetch_kernel_stack_address);
263		268
264	static __kprobes void fetch_user_stack_address(struct pt_regs *regs,	269	static void fetch_user_stack_address(struct pt_regs regs, void dummy, void *dest)
265	void dummy, void dest)
266	{	270	{
267	(unsigned long )dest = user_stack_pointer(regs);	271	(unsigned long )dest = user_stack_pointer(regs);
268	}	272	}
		273	NOKPROBE_SYMBOL(fetch_user_stack_address);
269		274
270	static fetch_func_t get_fetch_size_function(const struct fetch_type *type,	275	static fetch_func_t get_fetch_size_function(const struct fetch_type *type,
271	fetch_func_t orig_fn,	276	fetch_func_t orig_fn,

kernel/trace/trace_probe.h+7−8

  */
 #define convert_rloc_to_loc(dl, offs)	((u32)(dl) + (offs))
-static inline void *get_rloc_data(u32 *dl)
+static nokprobe_inline void *get_rloc_data(u32 *dl)
 {
 	return (u8 *)dl + get_rloc_offs(*dl);
 }
 /* For data_loc conversion */
-static inline void *get_loc_data(u32 *dl, void *ent)
+static nokprobe_inline void *get_loc_data(u32 *dl, void *ent)
 {
 	return (u8 *)ent + get_rloc_offs(*dl);
 }
 /* Printing  in basic type function template */
 #define DECLARE_BASIC_PRINT_TYPE_FUNC(type)				\
-__kprobes int PRINT_TYPE_FUNC_NAME(type)(struct trace_seq *s,		\
+int PRINT_TYPE_FUNC_NAME(type)(struct trace_seq *s, const char *name,	\
-					 const char *name,		\
+				void *data, void *ent);			\
-					 void *data, void *ent);	\
 extern const char PRINT_TYPE_FMT_NAME(type)[]
 DECLARE_BASIC_PRINT_TYPE_FUNC(u8);
 	return !!(tp->flags & TP_FLAG_REGISTERED);
 }
-static inline __kprobes void call_fetch(struct fetch_param *fprm,
+static nokprobe_inline void call_fetch(struct fetch_param *fprm,
 				 struct pt_regs *regs, void *dest)
 {
 	return fprm->fn(regs, fprm->data, dest);
 extern int traceprobe_command(const char *buf, int (*createfn)(int, char**));
 /* Sum up total data length for dynamic arraies (strings) */
-static inline __kprobes int
+static nokprobe_inline int
 __get_data_size(struct trace_probe *tp, struct pt_regs *regs)
 {
 	int i, ret = 0;
 }
 /* Store the value of each argument */
-static inline __kprobes void
+static nokprobe_inline void
 store_trace_args(int ent_size, struct trace_probe *tp, struct pt_regs *regs,
 		 u8 *data, int maxlen)
 {

kernel/trace/trace_uprobe.c+35−31

@@ -108,8 +108,8 @@ static unsigned long get_user_stack_nth(struct pt_regs *regs, unsigned int n) · 107 unmodified lines
108	* Uprobes-specific fetch functions	108	* Uprobes-specific fetch functions
109	*/	109	*/
110	#define DEFINE_FETCH_stack(type) \	110	#define DEFINE_FETCH_stack(type) \
111	static __kprobes void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs,\	111	static void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs, \
112	void offset, void dest) \	112	void offset, void dest) \
113	{ \	113	{ \
114	(type )dest = (type)get_user_stack_nth(regs, \	114	(type )dest = (type)get_user_stack_nth(regs, \
115	((unsigned long)offset)); \	115	((unsigned long)offset)); \
@@ -120,8 +120,8 @@ DEFINE_BASIC_FETCH_FUNCS(stack) · 4 unmodified lines
120	#define fetch_stack_string_size NULL	120	#define fetch_stack_string_size NULL
121		121
122	#define DEFINE_FETCH_memory(type) \	122	#define DEFINE_FETCH_memory(type) \
123	static __kprobes void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs,\	123	static void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs, \
124	void addr, void dest) \	124	void addr, void dest) \
125	{ \	125	{ \
126	type retval; \	126	type retval; \
127	void __user vaddr = (void __force __user ) addr; \	127	void __user vaddr = (void __force __user ) addr; \
@@ -136,8 +136,8 @@ DEFINE_BASIC_FETCH_FUNCS(memory) · 8 unmodified lines
136	* Fetch a null-terminated string. Caller MUST set (u32 )dest with max	136	* Fetch a null-terminated string. Caller MUST set (u32 )dest with max
137	* length and relative data location.	137	* length and relative data location.
138	*/	138	*/
139	static __kprobes void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs,	139	static void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs,
140	void addr, void dest)	140	void addr, void dest)
141	{	141	{
142	long ret;	142	long ret;
143	u32 rloc = (u32 )dest;	143	u32 rloc = (u32 )dest;
@@ -158,8 +158,8 @@ static __kprobes void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs, · 14 unmodified lines
158	}	158	}
159	}	159	}
160		160
161	static __kprobes void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs,	161	static void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs,
162	void addr, void dest)	162	void addr, void dest)
163	{	163	{
164	int len;	164	int len;
165	void __user vaddr = (void __force __user ) addr;	165	void __user vaddr = (void __force __user ) addr;
@@ -184,8 +184,8 @@ static unsigned long translate_user_vaddr(void *file_offset) · 18 unmodified lines
184	}	184	}
185		185
186	#define DEFINE_FETCH_file_offset(type) \	186	#define DEFINE_FETCH_file_offset(type) \
187	static __kprobes void FETCH_FUNC_NAME(file_offset, type)(struct pt_regs *regs,\	187	static void FETCH_FUNC_NAME(file_offset, type)(struct pt_regs *regs, \
188	void offset, void dest) \	188	void offset, void dest)\
189	{ \	189	{ \
190	void vaddr = (void )translate_user_vaddr(offset); \	190	void vaddr = (void )translate_user_vaddr(offset); \
191	\	191	\
@@ -1009,56 +1009,60 @@ uprobe_filter_event(struct trace_uprobe tu, struct perf_event event) · 817 unmodified lines
1009	return __uprobe_perf_filter(&tu->filter, event->hw.tp_target->mm);	1009	return __uprobe_perf_filter(&tu->filter, event->hw.tp_target->mm);
1010	}	1010	}
1011		1011
1012	static int uprobe_perf_open(struct trace_uprobe tu, struct perf_event event)	1012	static int uprobe_perf_close(struct trace_uprobe tu, struct perf_event event)
1013	{	1013	{
1014	bool done;	1014	bool done;
1015		1015
1016	write_lock(&tu->filter.rwlock);	1016	write_lock(&tu->filter.rwlock);
1017	if (event->hw.tp_target) {	1017	if (event->hw.tp_target) {
1018	/*	1018	list_del(&event->hw.tp_list);
1019	* event->parent != NULL means copy_process(), we can avoid
1020	* uprobe_apply(). current->mm must be probed and we can rely
1021	* on dup_mmap() which preserves the already installed bp's.
1022	*
1023	* attr.enable_on_exec means that exec/mmap will install the
1024	* breakpoints we need.
1025	*/
1026	done = tu->filter.nr_systemwide \|\|	1019	done = tu->filter.nr_systemwide \|\|
1027	event->parent \|\| event->attr.enable_on_exec \|\|	1020	(event->hw.tp_target->flags & PF_EXITING) \|\|
1028	uprobe_filter_event(tu, event);	1021	uprobe_filter_event(tu, event);
1029	list_add(&event->hw.tp_list, &tu->filter.perf_events);
1030	} else {	1022	} else {
		1023	tu->filter.nr_systemwide--;
1031	done = tu->filter.nr_systemwide;	1024	done = tu->filter.nr_systemwide;
1032	tu->filter.nr_systemwide++;
1033	}	1025	}
1034	write_unlock(&tu->filter.rwlock);	1026	write_unlock(&tu->filter.rwlock);
1035		1027
1036	if (!done)	1028	if (!done)
1037	uprobe_apply(tu->inode, tu->offset, &tu->consumer, true);	1029	return uprobe_apply(tu->inode, tu->offset, &tu->consumer, false);
1038		1030
1039	return 0;	1031	return 0;
1040	}	1032	}
1041		1033
1042	static int uprobe_perf_close(struct trace_uprobe tu, struct perf_event event)	1034	static int uprobe_perf_open(struct trace_uprobe tu, struct perf_event event)
1043	{	1035	{
1044	bool done;	1036	bool done;
		1037	int err;
1045		1038
1046	write_lock(&tu->filter.rwlock);	1039	write_lock(&tu->filter.rwlock);
1047	if (event->hw.tp_target) {	1040	if (event->hw.tp_target) {
1048	list_del(&event->hw.tp_list);	1041	/*
		1042	* event->parent != NULL means copy_process(), we can avoid
		1043	* uprobe_apply(). current->mm must be probed and we can rely
		1044	* on dup_mmap() which preserves the already installed bp's.
		1045	*
		1046	* attr.enable_on_exec means that exec/mmap will install the
		1047	* breakpoints we need.
		1048	*/
1049	done = tu->filter.nr_systemwide \|\|	1049	done = tu->filter.nr_systemwide \|\|
1050	(event->hw.tp_target->flags & PF_EXITING) \|\|	1050	event->parent \|\| event->attr.enable_on_exec \|\|
1051	uprobe_filter_event(tu, event);	1051	uprobe_filter_event(tu, event);
		1052	list_add(&event->hw.tp_list, &tu->filter.perf_events);
1052	} else {	1053	} else {
1053	tu->filter.nr_systemwide--;
1054	done = tu->filter.nr_systemwide;	1054	done = tu->filter.nr_systemwide;
		1055	tu->filter.nr_systemwide++;
1055	}	1056	}
1056	write_unlock(&tu->filter.rwlock);	1057	write_unlock(&tu->filter.rwlock);
1057		1058
1058	if (!done)	1059	err = 0;
1059	uprobe_apply(tu->inode, tu->offset, &tu->consumer, false);	1060	if (!done) {
1060		1061	err = uprobe_apply(tu->inode, tu->offset, &tu->consumer, true);
1061	return 0;	1062	if (err)
		1063	uprobe_perf_close(tu, event);
		1064	}
		1065	return err;
1062	}	1066	}
1063		1067
1064	static bool uprobe_perf_filter(struct uprobe_consumer *uc,	1068	static bool uprobe_perf_filter(struct uprobe_consumer *uc,