Merge branch 'perf-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull more perf updates from Ingo Molnar: "A second round of perf updates: - wide reaching kprobes sanitization and robustization, with the hope of fixing all 'probe this function crashes the kernel' bugs, by Masami Hiramatsu. - uprobes updates from Oleg Nesterov: tmpfs support, corner case fixes and robustization work. - perf tooling updates and fixes from Jiri Olsa, Namhyung Ki, Arnaldo et al: * Add support to accumulate hist periods (Namhyung Kim) * various fixes, refactorings and enhancements" * 'perf-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (101 commits) perf: Differentiate exec() and non-exec() comm events perf: Fix perf_event_comm() vs. exec() assumption uprobes/x86: Rename arch_uprobe->def to ->defparam, minor comment updates perf/documentation: Add description for conditional branch filter perf/x86: Add conditional branch filtering support perf/tool: Add conditional branch filter 'cond' to perf record perf: Add new conditional branch filter 'PERF_SAMPLE_BRANCH_COND' uprobes: Teach copy_insn() to support tmpfs uprobes: Shift ->readpage check from __copy_insn() to uprobe_register() perf/x86: Use common PMU interrupt disabled code perf/ARM: Use common PMU interrupt disabled code perf: Disable sampled events if no PMU interrupt perf: Fix use after free in perf_remove_from_context() perf tools: Fix 'make help' message error perf record: Fix poll return value propagation perf tools: Move elide bool into perf_hpp_fmt struct perf tools: Remove elide setup for SORT_MODE__MEMORY mode perf tools: Fix "==" into "=" in ui_browser__warning assignment perf tools: Allow overriding sysfs and proc finding with env var perf tools: Consider header files outside perf directory in tags target ...

Linus Torvalds committed 12 years ago

Commit 3737a122 parents c29deef 82b897781 files · +2979 −1182Browse files

Showing the changes to kernel (10 of 81 files). Show all

Unified · Split

10 files

kernel/events/core.c +27 −16
kernel/events/uprobes.c +33 −19
kernel/kprobes.c +229 −163
kernel/notifier.c +13 −9
kernel/sched/core.c +5 −2
kernel/trace/trace_event_perf.c +3 −2
kernel/trace/trace_kprobe.c +41 −30
kernel/trace/trace_probe.c +35 −30
kernel/trace/trace_probe.h +7 −8
kernel/trace/trace_uprobe.c +35 −31

kernel/events/core.c+27−16

@@ -2974,6 +2974,22 @@ out: · 2973 unmodified lines
2974	2974	local_irq_restore(flags);
2975	2975	}
2976	2976
	2977	+void perf_event_exec(void)
	2978	+{
	2979	+ struct perf_event_context *ctx;
	2980	+ int ctxn;
	2981	+
	2982	+ rcu_read_lock();
	2983	+ for_each_task_context_nr(ctxn) {
	2984	+ ctx = current->perf_event_ctxp[ctxn];
	2985	+ if (!ctx)
	2986	+ continue;
	2987	+
	2988	+ perf_event_enable_on_exec(ctx);
	2989	+ }
	2990	+ rcu_read_unlock();
	2991	+}
	2992	+
2977	2993	/*
2978	2994	* Cross CPU call to read the hardware event
2979	2995	*/
@@ -5075,21 +5091,9 @@ static void perf_event_comm_event(struct perf_comm_event *comm_event) · 2095 unmodified lines
5075	5091	NULL);
5076	5092	}
5077	5093
5078		−void perf_event_comm(struct task_struct *task)
	5094	+void perf_event_comm(struct task_struct *task, bool exec)
5079	5095	{
5080	5096	struct perf_comm_event comm_event;
5081		− struct perf_event_context *ctx;
5082		− int ctxn;
5083		−
5084		− rcu_read_lock();
5085		− for_each_task_context_nr(ctxn) {
5086		− ctx = task->perf_event_ctxp[ctxn];
5087		− if (!ctx)
5088		− continue;
5089		−
5090		− perf_event_enable_on_exec(ctx);
5091		− }
5092		− rcu_read_unlock();
5093	5097
5094	5098	if (!atomic_read(&nr_comm_events))
5095	5099	return;
@@ -5101,7 +5105,7 @@ void perf_event_comm(struct task_struct *task) · 5 unmodified lines
5101	5105	.event_id = {
5102	5106	.header = {
5103	5107	.type = PERF_RECORD_COMM,
5104		− .misc = 0,
	5108	+ .misc = exec ? PERF_RECORD_MISC_COMM_EXEC : 0,
5105	5109	/* .size */
5106	5110	},
5107	5111	/* .pid */
@@ -7122,6 +7126,13 @@ SYSCALL_DEFINE5(perf_event_open, · 2014 unmodified lines
7122	7126	}
7123	7127	}
7124	7128
	7129	+ if (is_sampling_event(event)) {
	7130	+ if (event->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT) {
	7131	+ err = -ENOTSUPP;
	7132	+ goto err_alloc;
	7133	+ }
	7134	+ }
	7135	+
7125	7136	account_event(event);
7126	7137
7127	7138	/*
@@ -7433,7 +7444,7 @@ __perf_event_exit_task(struct perf_event *child_event, · 305 unmodified lines
7433	7444
7434	7445	static void perf_event_exit_task_context(struct task_struct *child, int ctxn)
7435	7446	{
7436		− struct perf_event *child_event;
	7447	+ struct perf_event child_event, next;
7437	7448	struct perf_event_context *child_ctx;
7438	7449	unsigned long flags;
7439	7450
@@ -7487,7 +7498,7 @@ static void perf_event_exit_task_context(struct task_struct *child, int ctxn) · 47 unmodified lines
7487	7498	*/
7488	7499	mutex_lock(&child_ctx->mutex);
7489	7500
7490		− list_for_each_entry_rcu(child_event, &child_ctx->event_list, event_entry)
	7501	+ list_for_each_entry_safe(child_event, next, &child_ctx->event_list, event_entry)
7491	7502	__perf_event_exit_task(child_event, child_ctx, child);
7492	7503
7493	7504	mutex_unlock(&child_ctx->mutex);

kernel/events/uprobes.c+33−19

@@ -36,6 +36,7 @@ · 35 unmodified lines
36	36	#include "../../mm/internal.h" /* munlock_vma_page */
37	37	#include <linux/percpu-rwsem.h>
38	38	#include <linux/task_work.h>
	39	+#include <linux/shmem_fs.h>
39	40
40	41	#include <linux/uprobes.h>
41	42
@@ -127,7 +128,7 @@ struct xol_area { · 85 unmodified lines
127	128	*/
128	129	static bool valid_vma(struct vm_area_struct *vma, bool is_register)
129	130	{
130		− vm_flags_t flags = VM_HUGETLB \| VM_MAYEXEC \| VM_SHARED;
	131	+ vm_flags_t flags = VM_HUGETLB \| VM_MAYEXEC \| VM_MAYSHARE;
131	132
132	133	if (is_register)
133	134	flags \|= VM_WRITE;
@@ -279,18 +280,13 @@ static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t · 145 unmodified lines
279	280	* supported by that architecture then we need to modify is_trap_at_addr and
280	281	* uprobe_write_opcode accordingly. This would never be a problem for archs
281	282	* that have fixed length instructions.
282		− */
283		−
284		−/*
	283	+ *
285	284	* uprobe_write_opcode - write the opcode at a given virtual address.
286	285	* @mm: the probed process address space.
287	286	* @vaddr: the virtual address to store the opcode.
288	287	* @opcode: opcode to be written at @vaddr.
289	288	*
290		− * Called with mm->mmap_sem held (for read and with a reference to
291		− * mm).
292		− *
293		− * For mm @mm, write the opcode at @vaddr.
	289	+ * Called with mm->mmap_sem held for write.
294	290	* Return 0 (success) or a negative errno.
295	291	*/
296	292	int uprobe_write_opcode(struct mm_struct *mm, unsigned long vaddr,
@@ -310,21 +306,25 @@ retry: · 13 unmodified lines
310	306	if (ret <= 0)
311	307	goto put_old;
312	308
	309	+ ret = anon_vma_prepare(vma);
	310	+ if (ret)
	311	+ goto put_old;
	312	+
313	313	ret = -ENOMEM;
314	314	new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vaddr);
315	315	if (!new_page)
316	316	goto put_old;
317	317
318		− __SetPageUptodate(new_page);
	318	+ if (mem_cgroup_charge_anon(new_page, mm, GFP_KERNEL))
	319	+ goto put_new;
319	320
	321	+ __SetPageUptodate(new_page);
320	322	copy_highpage(new_page, old_page);
321	323	copy_to_page(new_page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE);
322	324
323		− ret = anon_vma_prepare(vma);
324		− if (ret)
325		− goto put_new;
326		−
327	325	ret = __replace_page(vma, vaddr, old_page, new_page);
	326	+ if (ret)
	327	+ mem_cgroup_uncharge_page(new_page);
328	328
329	329	put_new:
330	330	page_cache_release(new_page);
@@ -537,14 +537,15 @@ static int __copy_insn(struct address_space mapping, struct file filp, · 206 unmodified lines
537	537	void *insn, int nbytes, loff_t offset)
538	538	{
539	539	struct page *page;
540		−
541		− if (!mapping->a_ops->readpage)
542		− return -EIO;
543	540	/*
544		− * Ensure that the page that has the original instruction is
545		− * populated and in page-cache.
	541	+ * Ensure that the page that has the original instruction is populated
	542	+ * and in page-cache. If ->readpage == NULL it must be shmem_mapping(),
	543	+ * see uprobe_register().
546	544	*/
547		− page = read_mapping_page(mapping, offset >> PAGE_CACHE_SHIFT, filp);
	545	+ if (mapping->a_ops->readpage)
	546	+ page = read_mapping_page(mapping, offset >> PAGE_CACHE_SHIFT, filp);
	547	+ else
	548	+ page = shmem_read_mapping_page(mapping, offset >> PAGE_CACHE_SHIFT);
548	549	if (IS_ERR(page))
549	550	return PTR_ERR(page);
550	551
@@ -880,6 +881,9 @@ int uprobe_register(struct inode inode, loff_t offset, struct uprobe_consumer · 329 unmodified lines
880	881	if (!uc->handler && !uc->ret_handler)
881	882	return -EINVAL;
882	883
	884	+ /* copy_insn() uses read_mapping_page() or shmem_read_mapping_page() */
	885	+ if (!inode->i_mapping->a_ops->readpage && !shmem_mapping(inode->i_mapping))
	886	+ return -EIO;
883	887	/* Racy, just to catch the obvious mistakes */
884	888	if (offset > i_size_read(inode))
885	889	return -EINVAL;
@@ -1361,6 +1365,16 @@ unsigned long __weak uprobe_get_swbp_addr(struct pt_regs *regs) · 475 unmodified lines
1361	1365	return instruction_pointer(regs) - UPROBE_SWBP_INSN_SIZE;
1362	1366	}
1363	1367
	1368	+unsigned long uprobe_get_trap_addr(struct pt_regs *regs)
	1369	+{
	1370	+ struct uprobe_task *utask = current->utask;
	1371	+
	1372	+ if (unlikely(utask && utask->active_uprobe))
	1373	+ return utask->vaddr;
	1374	+
	1375	+ return instruction_pointer(regs);
	1376	+}
	1377	+
1364	1378	/*
1365	1379	* Called with no locks held.
1366	1380	* Called in context of a exiting or a exec-ing thread.

kernel/kprobes.c+229−163

@@ -86,21 +86,8 @@ static raw_spinlock_t *kretprobe_table_lock_ptr(unsigned long hash) · 85 unmodified lines
86	86	return &(kretprobe_table_locks[hash].lock);
87	87	}
88	88
89		−/*
90		− * Normally, functions that we'd want to prohibit kprobes in, are marked
91		− * __kprobes. But, there are cases where such functions already belong to
92		− * a different section (__sched for preempt_schedule)
93		− *
94		− * For such cases, we now have a blacklist
95		− */
96		−static struct kprobe_blackpoint kprobe_blacklist[] = {
97		− {"preempt_schedule",},
98		− {"native_get_debugreg",},
99		− {"irq_entries_start",},
100		− {"common_interrupt",},
101		− {"mcount",}, /* mcount can be called from everywhere */
102		− {NULL} /* Terminator */
103		−};
	89	+/* Blacklist -- list of struct kprobe_blacklist_entry */
	90	+static LIST_HEAD(kprobe_blacklist);
104	91
105	92	#ifdef __ARCH_WANT_KPROBES_INSN_SLOT
106	93	/*
@@ -151,13 +138,13 @@ struct kprobe_insn_cache kprobe_insn_slots = { · 44 unmodified lines
151	138	.insn_size = MAX_INSN_SIZE,
152	139	.nr_garbage = 0,
153	140	};
154		−static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c);
	141	+static int collect_garbage_slots(struct kprobe_insn_cache *c);
155	142
156	143	/**
157	144	* __get_insn_slot() - Find a slot on an executable page for an instruction.
158	145	* We allocate an executable page if there's no room on existing ones.
159	146	*/
160		−kprobe_opcode_t __kprobes __get_insn_slot(struct kprobe_insn_cache c)
	147	+kprobe_opcode_t __get_insn_slot(struct kprobe_insn_cache c)
161	148	{
162	149	struct kprobe_insn_page *kip;
163	150	kprobe_opcode_t *slot = NULL;
@@ -214,7 +201,7 @@ out: · 50 unmodified lines
214	201	}
215	202
216	203	/* Return 1 if all garbages are collected, otherwise 0. */
217		−static int __kprobes collect_one_slot(struct kprobe_insn_page *kip, int idx)
	204	+static int collect_one_slot(struct kprobe_insn_page *kip, int idx)
218	205	{
219	206	kip->slot_used[idx] = SLOT_CLEAN;
220	207	kip->nused--;
@@ -235,7 +222,7 @@ static int __kprobes collect_one_slot(struct kprobe_insn_page *kip, int idx) · 14 unmodified lines
235	222	return 0;
236	223	}
237	224
238		−static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c)
	225	+static int collect_garbage_slots(struct kprobe_insn_cache *c)
239	226	{
240	227	struct kprobe_insn_page kip, next;
241	228
@@ -257,8 +244,8 @@ static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c) · 15 unmodified lines
257	244	return 0;
258	245	}
259	246
260		−void __kprobes __free_insn_slot(struct kprobe_insn_cache *c,
261		− kprobe_opcode_t *slot, int dirty)
	247	+void __free_insn_slot(struct kprobe_insn_cache *c,
	248	+ kprobe_opcode_t *slot, int dirty)
262	249	{
263	250	struct kprobe_insn_page *kip;
264	251
@@ -314,7 +301,7 @@ static inline void reset_kprobe_instance(void) · 49 unmodified lines
314	301	* OR
315	302	* - with preemption disabled - from arch/xxx/kernel/kprobes.c
316	303	*/
317		−struct kprobe __kprobes get_kprobe(void addr)
	304	+struct kprobe get_kprobe(void addr)
318	305	{
319	306	struct hlist_head *head;
320	307	struct kprobe *p;
@@ -327,8 +314,9 @@ struct kprobe __kprobes get_kprobe(void addr) · 6 unmodified lines
327	314
328	315	return NULL;
329	316	}
	317	+NOKPROBE_SYMBOL(get_kprobe);
330	318
331		−static int __kprobes aggr_pre_handler(struct kprobe p, struct pt_regs regs);
	319	+static int aggr_pre_handler(struct kprobe p, struct pt_regs regs);
332	320
333	321	/* Return true if the kprobe is an aggregator */
334	322	static inline int kprobe_aggrprobe(struct kprobe *p)
@@ -360,7 +348,7 @@ static bool kprobes_allow_optimization; · 25 unmodified lines
360	348	* Call all pre_handler on the list, but ignores its return value.
361	349	* This must be called from arch-dep optimized caller.
362	350	*/
363		−void __kprobes opt_pre_handler(struct kprobe p, struct pt_regs regs)
	351	+void opt_pre_handler(struct kprobe p, struct pt_regs regs)
364	352	{
365	353	struct kprobe *kp;
366	354
@@ -372,9 +360,10 @@ void __kprobes opt_pre_handler(struct kprobe p, struct pt_regs regs) · 5 unmodified lines
372	360	reset_kprobe_instance();
373	361	}
374	362	}
	363	+NOKPROBE_SYMBOL(opt_pre_handler);
375	364
376	365	/* Free optimized instructions and optimized_kprobe */
377		−static __kprobes void free_aggr_kprobe(struct kprobe *p)
	366	+static void free_aggr_kprobe(struct kprobe *p)
378	367	{
379	368	struct optimized_kprobe *op;
380	369
@@ -412,7 +401,7 @@ static inline int kprobe_disarmed(struct kprobe *p) · 31 unmodified lines
412	401	}
413	402
414	403	/* Return true(!0) if the probe is queued on (un)optimizing lists */
415		−static int __kprobes kprobe_queued(struct kprobe *p)
	404	+static int kprobe_queued(struct kprobe *p)
416	405	{
417	406	struct optimized_kprobe *op;
418	407
@@ -428,7 +417,7 @@ static int __kprobes kprobe_queued(struct kprobe *p) · 9 unmodified lines
428	417	* Return an optimized kprobe whose optimizing code replaces
429	418	* instructions including addr (exclude breakpoint).
430	419	*/
431		−static struct kprobe *__kprobes get_optimized_kprobe(unsigned long addr)
	420	+static struct kprobe *get_optimized_kprobe(unsigned long addr)
432	421	{
433	422	int i;
434	423	struct kprobe *p = NULL;
@@ -460,7 +449,7 @@ static DECLARE_DELAYED_WORK(optimizing_work, kprobe_optimizer); · 25 unmodified lines
460	449	* Optimize (replace a breakpoint with a jump) kprobes listed on
461	450	* optimizing_list.
462	451	*/
463		−static __kprobes void do_optimize_kprobes(void)
	452	+static void do_optimize_kprobes(void)
464	453	{
465	454	/* Optimization never be done when disarmed */
466	455	if (kprobes_all_disarmed \|\| !kprobes_allow_optimization \|\|
@@ -488,7 +477,7 @@ static __kprobes void do_optimize_kprobes(void) · 21 unmodified lines
488	477	* Unoptimize (replace a jump with a breakpoint and remove the breakpoint
489	478	* if need) kprobes listed on unoptimizing_list.
490	479	*/
491		−static __kprobes void do_unoptimize_kprobes(void)
	480	+static void do_unoptimize_kprobes(void)
492	481	{
493	482	struct optimized_kprobe op, tmp;
494	483
@@ -520,7 +509,7 @@ static __kprobes void do_unoptimize_kprobes(void) · 25 unmodified lines
520	509	}
521	510
522	511	/* Reclaim all kprobes on the free_list */
523		−static __kprobes void do_free_cleaned_kprobes(void)
	512	+static void do_free_cleaned_kprobes(void)
524	513	{
525	514	struct optimized_kprobe op, tmp;
526	515
@@ -532,13 +521,13 @@ static __kprobes void do_free_cleaned_kprobes(void) · 5 unmodified lines
532	521	}
533	522
534	523	/* Start optimizer after OPTIMIZE_DELAY passed */
535		−static __kprobes void kick_kprobe_optimizer(void)
	524	+static void kick_kprobe_optimizer(void)
536	525	{
537	526	schedule_delayed_work(&optimizing_work, OPTIMIZE_DELAY);
538	527	}
539	528
540	529	/* Kprobe jump optimizer */
541		−static __kprobes void kprobe_optimizer(struct work_struct *work)
	530	+static void kprobe_optimizer(struct work_struct *work)
542	531	{
543	532	mutex_lock(&kprobe_mutex);
544	533	/* Lock modules while optimizing kprobes */
@@ -574,7 +563,7 @@ static __kprobes void kprobe_optimizer(struct work_struct *work) · 29 unmodified lines
574	563	}
575	564
576	565	/* Wait for completing optimization and unoptimization */
577		−static __kprobes void wait_for_kprobe_optimizer(void)
	566	+static void wait_for_kprobe_optimizer(void)
578	567	{
579	568	mutex_lock(&kprobe_mutex);
580	569
@@ -593,7 +582,7 @@ static __kprobes void wait_for_kprobe_optimizer(void) · 12 unmodified lines
593	582	}
594	583
595	584	/* Optimize kprobe if p is ready to be optimized */
596		−static __kprobes void optimize_kprobe(struct kprobe *p)
	585	+static void optimize_kprobe(struct kprobe *p)
597	586	{
598	587	struct optimized_kprobe *op;
599	588
@@ -627,7 +616,7 @@ static __kprobes void optimize_kprobe(struct kprobe *p) · 27 unmodified lines
627	616	}
628	617
629	618	/* Short cut to direct unoptimizing */
630		−static __kprobes void force_unoptimize_kprobe(struct optimized_kprobe *op)
	619	+static void force_unoptimize_kprobe(struct optimized_kprobe *op)
631	620	{
632	621	get_online_cpus();
633	622	arch_unoptimize_kprobe(op);
@@ -637,7 +626,7 @@ static __kprobes void force_unoptimize_kprobe(struct optimized_kprobe *op) · 3 unmodified lines
637	626	}
638	627
639	628	/* Unoptimize a kprobe if p is optimized */
640		−static __kprobes void unoptimize_kprobe(struct kprobe *p, bool force)
	629	+static void unoptimize_kprobe(struct kprobe *p, bool force)
641	630	{
642	631	struct optimized_kprobe *op;
643	632
@@ -697,7 +686,7 @@ static void reuse_unused_kprobe(struct kprobe *ap) · 53 unmodified lines
697	686	}
698	687
699	688	/* Remove optimized instructions */
700		−static void __kprobes kill_optimized_kprobe(struct kprobe *p)
	689	+static void kill_optimized_kprobe(struct kprobe *p)
701	690	{
702	691	struct optimized_kprobe *op;
703	692
@@ -723,7 +712,7 @@ static void __kprobes kill_optimized_kprobe(struct kprobe *p) · 19 unmodified lines
723	712	}
724	713
725	714	/* Try to prepare optimized instructions */
726		−static __kprobes void prepare_optimized_kprobe(struct kprobe *p)
	715	+static void prepare_optimized_kprobe(struct kprobe *p)
727	716	{
728	717	struct optimized_kprobe *op;
729	718
@@ -732,7 +721,7 @@ static __kprobes void prepare_optimized_kprobe(struct kprobe *p) · 2 unmodified lines
732	721	}
733	722
734	723	/* Allocate new optimized_kprobe and try to prepare optimized instructions */
735		−static __kprobes struct kprobe alloc_aggr_kprobe(struct kprobe p)
	724	+static struct kprobe alloc_aggr_kprobe(struct kprobe p)
736	725	{
737	726	struct optimized_kprobe *op;
738	727
@@ -747,13 +736,13 @@ static __kprobes struct kprobe alloc_aggr_kprobe(struct kprobe p) · 8 unmodified lines
747	736	return &op->kp;
748	737	}
749	738
750		−static void __kprobes init_aggr_kprobe(struct kprobe ap, struct kprobe p);
	739	+static void init_aggr_kprobe(struct kprobe ap, struct kprobe p);
751	740
752	741	/*
753	742	* Prepare an optimized_kprobe and optimize it
754	743	* NOTE: p must be a normal registered kprobe
755	744	*/
756		−static __kprobes void try_to_optimize_kprobe(struct kprobe *p)
	745	+static void try_to_optimize_kprobe(struct kprobe *p)
757	746	{
758	747	struct kprobe *ap;
759	748	struct optimized_kprobe *op;
@@ -787,7 +776,7 @@ out: · 27 unmodified lines
787	776	}
788	777
789	778	#ifdef CONFIG_SYSCTL
790		−static void __kprobes optimize_all_kprobes(void)
	779	+static void optimize_all_kprobes(void)
791	780	{
792	781	struct hlist_head *head;
793	782	struct kprobe *p;
@@ -810,7 +799,7 @@ out: · 16 unmodified lines
810	799	mutex_unlock(&kprobe_mutex);
811	800	}
812	801
813		−static void __kprobes unoptimize_all_kprobes(void)
	802	+static void unoptimize_all_kprobes(void)
814	803	{
815	804	struct hlist_head *head;
816	805	struct kprobe *p;
@@ -861,7 +850,7 @@ int proc_kprobes_optimization_handler(struct ctl_table *table, int write, · 44 unmodified lines
861	850	#endif /* CONFIG_SYSCTL */
862	851
863	852	/* Put a breakpoint for a probe. Must be called with text_mutex locked */
864		−static void __kprobes __arm_kprobe(struct kprobe *p)
	853	+static void __arm_kprobe(struct kprobe *p)
865	854	{
866	855	struct kprobe *_p;
867	856
@@ -876,7 +865,7 @@ static void __kprobes __arm_kprobe(struct kprobe *p) · 8 unmodified lines
876	865	}
877	866
878	867	/* Remove the breakpoint of a probe. Must be called with text_mutex locked */
879		−static void __kprobes __disarm_kprobe(struct kprobe *p, bool reopt)
	868	+static void __disarm_kprobe(struct kprobe *p, bool reopt)
880	869	{
881	870	struct kprobe *_p;
882	871
@@ -911,13 +900,13 @@ static void reuse_unused_kprobe(struct kprobe *ap) · 28 unmodified lines
911	900	BUG_ON(kprobe_unused(ap));
912	901	}
913	902
914		−static __kprobes void free_aggr_kprobe(struct kprobe *p)
	903	+static void free_aggr_kprobe(struct kprobe *p)
915	904	{
916	905	arch_remove_kprobe(p);
917	906	kfree(p);
918	907	}
919	908
920		−static __kprobes struct kprobe alloc_aggr_kprobe(struct kprobe p)
	909	+static struct kprobe alloc_aggr_kprobe(struct kprobe p)
921	910	{
922	911	return kzalloc(sizeof(struct kprobe), GFP_KERNEL);
923	912	}
@@ -931,7 +920,7 @@ static struct ftrace_ops kprobe_ftrace_ops __read_mostly = { · 7 unmodified lines
931	920	static int kprobe_ftrace_enabled;
932	921
933	922	/* Must ensure p->addr is really on ftrace */
934		−static int __kprobes prepare_kprobe(struct kprobe *p)
	923	+static int prepare_kprobe(struct kprobe *p)
935	924	{
936	925	if (!kprobe_ftrace(p))
937	926	return arch_prepare_kprobe(p);
@@ -940,7 +929,7 @@ static int __kprobes prepare_kprobe(struct kprobe *p) · 2 unmodified lines
940	929	}
941	930
942	931	/* Caller must lock kprobe_mutex */
943		−static void __kprobes arm_kprobe_ftrace(struct kprobe *p)
	932	+static void arm_kprobe_ftrace(struct kprobe *p)
944	933	{
945	934	int ret;
946	935
@@ -955,7 +944,7 @@ static void __kprobes arm_kprobe_ftrace(struct kprobe *p) · 8 unmodified lines
955	944	}
956	945
957	946	/* Caller must lock kprobe_mutex */
958		−static void __kprobes disarm_kprobe_ftrace(struct kprobe *p)
	947	+static void disarm_kprobe_ftrace(struct kprobe *p)
959	948	{
960	949	int ret;
961	950
@@ -975,7 +964,7 @@ static void __kprobes disarm_kprobe_ftrace(struct kprobe *p) · 13 unmodified lines
975	964	#endif
976	965
977	966	/* Arm a kprobe with text_mutex */
978		−static void __kprobes arm_kprobe(struct kprobe *kp)
	967	+static void arm_kprobe(struct kprobe *kp)
979	968	{
980	969	if (unlikely(kprobe_ftrace(kp))) {
981	970	arm_kprobe_ftrace(kp);
@@ -992,7 +981,7 @@ static void __kprobes arm_kprobe(struct kprobe *kp) · 10 unmodified lines
992	981	}
993	982
994	983	/* Disarm a kprobe with text_mutex */
995		−static void __kprobes disarm_kprobe(struct kprobe *kp, bool reopt)
	984	+static void disarm_kprobe(struct kprobe *kp, bool reopt)
996	985	{
997	986	if (unlikely(kprobe_ftrace(kp))) {
998	987	disarm_kprobe_ftrace(kp);
@@ -1008,7 +997,7 @@ static void __kprobes disarm_kprobe(struct kprobe *kp, bool reopt) · 9 unmodified lines
1008	997	* Aggregate handlers for multiple kprobes support - these handlers
1009	998	* take care of invoking the individual kprobe handlers on p->list
1010	999	*/
1011		−static int __kprobes aggr_pre_handler(struct kprobe p, struct pt_regs regs)
	1000	+static int aggr_pre_handler(struct kprobe p, struct pt_regs regs)
1012	1001	{
1013	1002	struct kprobe *kp;
1014	1003
@@ -1022,9 +1011,10 @@ static int __kprobes aggr_pre_handler(struct kprobe p, struct pt_regs regs) · 7 unmodified lines
1022	1011	}
1023	1012	return 0;
1024	1013	}
	1014	+NOKPROBE_SYMBOL(aggr_pre_handler);
1025	1015
1026		−static void __kprobes aggr_post_handler(struct kprobe p, struct pt_regs regs,
1027		− unsigned long flags)
	1016	+static void aggr_post_handler(struct kprobe p, struct pt_regs regs,
	1017	+ unsigned long flags)
1028	1018	{
1029	1019	struct kprobe *kp;
1030	1020
@@ -1036,9 +1026,10 @@ static void __kprobes aggr_post_handler(struct kprobe p, struct pt_regs regs, · 5 unmodified lines
1036	1026	}
1037	1027	}
1038	1028	}
	1029	+NOKPROBE_SYMBOL(aggr_post_handler);
1039	1030
1040		−static int __kprobes aggr_fault_handler(struct kprobe p, struct pt_regs regs,
1041		− int trapnr)
	1031	+static int aggr_fault_handler(struct kprobe p, struct pt_regs regs,
	1032	+ int trapnr)
1042	1033	{
1043	1034	struct kprobe *cur = __this_cpu_read(kprobe_instance);
1044	1035
@@ -1052,8 +1043,9 @@ static int __kprobes aggr_fault_handler(struct kprobe p, struct pt_regs regs, · 7 unmodified lines
1052	1043	}
1053	1044	return 0;
1054	1045	}
	1046	+NOKPROBE_SYMBOL(aggr_fault_handler);
1055	1047
1056		−static int __kprobes aggr_break_handler(struct kprobe p, struct pt_regs regs)
	1048	+static int aggr_break_handler(struct kprobe p, struct pt_regs regs)
1057	1049	{
1058	1050	struct kprobe *cur = __this_cpu_read(kprobe_instance);
1059	1051	int ret = 0;
@@ -1065,9 +1057,10 @@ static int __kprobes aggr_break_handler(struct kprobe p, struct pt_regs regs) · 5 unmodified lines
1065	1057	reset_kprobe_instance();
1066	1058	return ret;
1067	1059	}
	1060	+NOKPROBE_SYMBOL(aggr_break_handler);
1068	1061
1069	1062	/* Walks the list and increments nmissed count for multiprobe case */
1070		−void __kprobes kprobes_inc_nmissed_count(struct kprobe *p)
	1063	+void kprobes_inc_nmissed_count(struct kprobe *p)
1071	1064	{
1072	1065	struct kprobe *kp;
1073	1066	if (!kprobe_aggrprobe(p)) {
@@ -1078,9 +1071,10 @@ void __kprobes kprobes_inc_nmissed_count(struct kprobe *p) · 4 unmodified lines
1078	1071	}
1079	1072	return;
1080	1073	}
	1074	+NOKPROBE_SYMBOL(kprobes_inc_nmissed_count);
1081	1075
1082		−void __kprobes recycle_rp_inst(struct kretprobe_instance *ri,
1083		− struct hlist_head *head)
	1076	+void recycle_rp_inst(struct kretprobe_instance *ri,
	1077	+ struct hlist_head *head)
1084	1078	{
1085	1079	struct kretprobe *rp = ri->rp;
1086	1080
@@ -1095,8 +1089,9 @@ void __kprobes recycle_rp_inst(struct kretprobe_instance *ri, · 8 unmodified lines
1095	1089	/* Unregistering */
1096	1090	hlist_add_head(&ri->hlist, head);
1097	1091	}
	1092	+NOKPROBE_SYMBOL(recycle_rp_inst);
1098	1093
1099		−void __kprobes kretprobe_hash_lock(struct task_struct *tsk,
	1094	+void kretprobe_hash_lock(struct task_struct *tsk,
1100	1095	struct hlist_head *head, unsigned long flags)
1101	1096	__acquires(hlist_lock)
1102	1097	{
@@ -1107,17 +1102,19 @@ __acquires(hlist_lock) · 4 unmodified lines
1107	1102	hlist_lock = kretprobe_table_lock_ptr(hash);
1108	1103	raw_spin_lock_irqsave(hlist_lock, *flags);
1109	1104	}
	1105	+NOKPROBE_SYMBOL(kretprobe_hash_lock);
1110	1106
1111		−static void __kprobes kretprobe_table_lock(unsigned long hash,
1112		− unsigned long *flags)
	1107	+static void kretprobe_table_lock(unsigned long hash,
	1108	+ unsigned long *flags)
1113	1109	__acquires(hlist_lock)
1114	1110	{
1115	1111	raw_spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
1116	1112	raw_spin_lock_irqsave(hlist_lock, *flags);
1117	1113	}
	1114	+NOKPROBE_SYMBOL(kretprobe_table_lock);
1118	1115
1119		−void __kprobes kretprobe_hash_unlock(struct task_struct *tsk,
1120		− unsigned long *flags)
	1116	+void kretprobe_hash_unlock(struct task_struct *tsk,
	1117	+ unsigned long *flags)
1121	1118	__releases(hlist_lock)
1122	1119	{
1123	1120	unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
@@ -1126,14 +1123,16 @@ __releases(hlist_lock) · 2 unmodified lines
1126	1123	hlist_lock = kretprobe_table_lock_ptr(hash);
1127	1124	raw_spin_unlock_irqrestore(hlist_lock, *flags);
1128	1125	}
	1126	+NOKPROBE_SYMBOL(kretprobe_hash_unlock);
1129	1127
1130		−static void __kprobes kretprobe_table_unlock(unsigned long hash,
1131		− unsigned long *flags)
	1128	+static void kretprobe_table_unlock(unsigned long hash,
	1129	+ unsigned long *flags)
1132	1130	__releases(hlist_lock)
1133	1131	{
1134	1132	raw_spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
1135	1133	raw_spin_unlock_irqrestore(hlist_lock, *flags);
1136	1134	}
	1135	+NOKPROBE_SYMBOL(kretprobe_table_unlock);
1137	1136
1138	1137	/*
1139	1138	* This function is called from finish_task_switch when task tk becomes dead,
@@ -1141,7 +1140,7 @@ __releases(hlist_lock) · 1 unmodified line
1141	1140	* with this task. These left over instances represent probed functions
1142	1141	* that have been called but will never return.
1143	1142	*/
1144		−void __kprobes kprobe_flush_task(struct task_struct *tk)
	1143	+void kprobe_flush_task(struct task_struct *tk)
1145	1144	{
1146	1145	struct kretprobe_instance *ri;
1147	1146	struct hlist_head *head, empty_rp;
@@ -1166,6 +1165,7 @@ void __kprobes kprobe_flush_task(struct task_struct *tk) · 18 unmodified lines
1166	1165	kfree(ri);
1167	1166	}
1168	1167	}
	1168	+NOKPROBE_SYMBOL(kprobe_flush_task);
1169	1169
1170	1170	static inline void free_rp_inst(struct kretprobe *rp)
1171	1171	{
@@ -1178,7 +1178,7 @@ static inline void free_rp_inst(struct kretprobe *rp) · 6 unmodified lines
1178	1178	}
1179	1179	}
1180	1180
1181		−static void __kprobes cleanup_rp_inst(struct kretprobe *rp)
	1181	+static void cleanup_rp_inst(struct kretprobe *rp)
1182	1182	{
1183	1183	unsigned long flags, hash;
1184	1184	struct kretprobe_instance *ri;
@@ -1197,12 +1197,13 @@ static void __kprobes cleanup_rp_inst(struct kretprobe *rp) · 12 unmodified lines
1197	1197	}
1198	1198	free_rp_inst(rp);
1199	1199	}
	1200	+NOKPROBE_SYMBOL(cleanup_rp_inst);
1200	1201
1201	1202	/*
1202	1203	* Add the new probe to ap->list. Fail if this is the
1203	1204	* second jprobe at the address - two jprobes can't coexist
1204	1205	*/
1205		−static int __kprobes add_new_kprobe(struct kprobe ap, struct kprobe p)
	1206	+static int add_new_kprobe(struct kprobe ap, struct kprobe p)
1206	1207	{
1207	1208	BUG_ON(kprobe_gone(ap) \|\| kprobe_gone(p));
1208	1209
@@ -1226,7 +1227,7 @@ static int __kprobes add_new_kprobe(struct kprobe ap, struct kprobe p) · 17 unmodified lines
1226	1227	* Fill in the required fields of the "manager kprobe". Replace the
1227	1228	* earlier kprobe in the hlist with the manager kprobe
1228	1229	*/
1229		−static void __kprobes init_aggr_kprobe(struct kprobe ap, struct kprobe p)
	1230	+static void init_aggr_kprobe(struct kprobe ap, struct kprobe p)
1230	1231	{
1231	1232	/* Copy p's insn slot to ap */
1232	1233	copy_kprobe(p, ap);
@@ -1252,8 +1253,7 @@ static void __kprobes init_aggr_kprobe(struct kprobe ap, struct kprobe p) · 19 unmodified lines
1252	1253	* This is the second or subsequent kprobe at the address - handle
1253	1254	* the intricacies
1254	1255	*/
1255		−static int __kprobes register_aggr_kprobe(struct kprobe *orig_p,
1256		− struct kprobe *p)
	1256	+static int register_aggr_kprobe(struct kprobe orig_p, struct kprobe p)
1257	1257	{
1258	1258	int ret = 0;
1259	1259	struct kprobe *ap = orig_p;
@@ -1324,25 +1324,29 @@ out: · 64 unmodified lines
1324	1324	return ret;
1325	1325	}
1326	1326
1327		−static int __kprobes in_kprobes_functions(unsigned long addr)
	1327	+bool __weak arch_within_kprobe_blacklist(unsigned long addr)
1328	1328	{
1329		− struct kprobe_blackpoint *kb;
	1329	+ /* The __kprobes marked functions and entry code must not be probed */
	1330	+ return addr >= (unsigned long)__kprobes_text_start &&
	1331	+ addr < (unsigned long)__kprobes_text_end;
	1332	+}
1330	1333
1331		− if (addr >= (unsigned long)__kprobes_text_start &&
1332		− addr < (unsigned long)__kprobes_text_end)
1333		− return -EINVAL;
	1334	+static bool within_kprobe_blacklist(unsigned long addr)
	1335	+{
	1336	+ struct kprobe_blacklist_entry *ent;
	1337	+
	1338	+ if (arch_within_kprobe_blacklist(addr))
	1339	+ return true;
1334	1340	/*
1335	1341	* If there exists a kprobe_blacklist, verify and
1336	1342	* fail any probe registration in the prohibited area
1337	1343	*/
1338		− for (kb = kprobe_blacklist; kb->name != NULL; kb++) {
1339		− if (kb->start_addr) {
1340		− if (addr >= kb->start_addr &&
1341		− addr < (kb->start_addr + kb->range))
1342		− return -EINVAL;
1343		− }
	1344	+ list_for_each_entry(ent, &kprobe_blacklist, list) {
	1345	+ if (addr >= ent->start_addr && addr < ent->end_addr)
	1346	+ return true;
1344	1347	}
1345		− return 0;
	1348	+
	1349	+ return false;
1346	1350	}
1347	1351
1348	1352	/*
@@ -1351,7 +1355,7 @@ static int __kprobes in_kprobes_functions(unsigned long addr) · 2 unmodified lines
1351	1355	* This returns encoded errors if it fails to look up symbol or invalid
1352	1356	* combination of parameters.
1353	1357	*/
1354		−static kprobe_opcode_t __kprobes kprobe_addr(struct kprobe p)
	1358	+static kprobe_opcode_t kprobe_addr(struct kprobe p)
1355	1359	{
1356	1360	kprobe_opcode_t *addr = p->addr;
1357	1361
@@ -1374,7 +1378,7 @@ invalid: · 16 unmodified lines
1374	1378	}
1375	1379
1376	1380	/* Check passed kprobe is valid and return kprobe in kprobe_table. */
1377		−static struct kprobe * __kprobes __get_valid_kprobe(struct kprobe *p)
	1381	+static struct kprobe __get_valid_kprobe(struct kprobe p)
1378	1382	{
1379	1383	struct kprobe ap, list_p;
1380	1384
@@ -1406,8 +1410,8 @@ static inline int check_kprobe_rereg(struct kprobe *p) · 25 unmodified lines
1406	1410	return ret;
1407	1411	}
1408	1412
1409		−static __kprobes int check_kprobe_address_safe(struct kprobe *p,
1410		− struct module **probed_mod)
	1413	+static int check_kprobe_address_safe(struct kprobe *p,
	1414	+ struct module **probed_mod)
1411	1415	{
1412	1416	int ret = 0;
1413	1417	unsigned long ftrace_addr;
@@ -1433,7 +1437,7 @@ static __kprobes int check_kprobe_address_safe(struct kprobe *p, · 19 unmodified lines
1433	1437
1434	1438	/* Ensure it is not in reserved area nor out of text */
1435	1439	if (!kernel_text_address((unsigned long) p->addr) \|\|
1436		− in_kprobes_functions((unsigned long) p->addr) \|\|
	1440	+ within_kprobe_blacklist((unsigned long) p->addr) \|\|
1437	1441	jump_label_text_reserved(p->addr, p->addr)) {
1438	1442	ret = -EINVAL;
1439	1443	goto out;
@@ -1469,7 +1473,7 @@ out: · 29 unmodified lines
1469	1473	return ret;
1470	1474	}
1471	1475
1472		−int __kprobes register_kprobe(struct kprobe *p)
	1476	+int register_kprobe(struct kprobe *p)
1473	1477	{
1474	1478	int ret;
1475	1479	struct kprobe *old_p;
@@ -1531,7 +1535,7 @@ out: · 55 unmodified lines
1531	1535	EXPORT_SYMBOL_GPL(register_kprobe);
1532	1536
1533	1537	/* Check if all probes on the aggrprobe are disabled */
1534		−static int __kprobes aggr_kprobe_disabled(struct kprobe *ap)
	1538	+static int aggr_kprobe_disabled(struct kprobe *ap)
1535	1539	{
1536	1540	struct kprobe *kp;
1537	1541
@@ -1547,7 +1551,7 @@ static int __kprobes aggr_kprobe_disabled(struct kprobe *ap) · 9 unmodified lines
1547	1551	}
1548	1552
1549	1553	/* Disable one kprobe: Make sure called under kprobe_mutex is locked */
1550		−static struct kprobe __kprobes __disable_kprobe(struct kprobe p)
	1554	+static struct kprobe __disable_kprobe(struct kprobe p)
1551	1555	{
1552	1556	struct kprobe *orig_p;
1553	1557
@@ -1574,7 +1578,7 @@ static struct kprobe __kprobes __disable_kprobe(struct kprobe p) · 20 unmodified lines
1574	1578	/*
1575	1579	* Unregister a kprobe without a scheduler synchronization.
1576	1580	*/
1577		−static int __kprobes __unregister_kprobe_top(struct kprobe *p)
	1581	+static int __unregister_kprobe_top(struct kprobe *p)
1578	1582	{
1579	1583	struct kprobe ap, list_p;
1580	1584
@@ -1631,7 +1635,7 @@ disarmed: · 50 unmodified lines
1631	1635	return 0;
1632	1636	}
1633	1637
1634		−static void __kprobes __unregister_kprobe_bottom(struct kprobe *p)
	1638	+static void __unregister_kprobe_bottom(struct kprobe *p)
1635	1639	{
1636	1640	struct kprobe *ap;
1637	1641
@@ -1647,7 +1651,7 @@ static void __kprobes __unregister_kprobe_bottom(struct kprobe *p) · 9 unmodified lines
1647	1651	/* Otherwise, do nothing. */
1648	1652	}
1649	1653
1650		−int __kprobes register_kprobes(struct kprobe **kps, int num)
	1654	+int register_kprobes(struct kprobe **kps, int num)
1651	1655	{
1652	1656	int i, ret = 0;
1653	1657
@@ -1665,13 +1669,13 @@ int __kprobes register_kprobes(struct kprobe **kps, int num) · 11 unmodified lines
1665	1669	}
1666	1670	EXPORT_SYMBOL_GPL(register_kprobes);
1667	1671
1668		−void __kprobes unregister_kprobe(struct kprobe *p)
	1672	+void unregister_kprobe(struct kprobe *p)
1669	1673	{
1670	1674	unregister_kprobes(&p, 1);
1671	1675	}
1672	1676	EXPORT_SYMBOL_GPL(unregister_kprobe);
1673	1677
1674		−void __kprobes unregister_kprobes(struct kprobe **kps, int num)
	1678	+void unregister_kprobes(struct kprobe **kps, int num)
1675	1679	{
1676	1680	int i;
1677	1681
@@ -1700,7 +1704,7 @@ unsigned long __weak arch_deref_entry_point(void *entry) · 22 unmodified lines
1700	1704	return (unsigned long)entry;
1701	1705	}
1702	1706
1703		−int __kprobes register_jprobes(struct jprobe **jps, int num)
	1707	+int register_jprobes(struct jprobe **jps, int num)
1704	1708	{
1705	1709	struct jprobe *jp;
1706	1710	int ret = 0, i;
@@ -1731,19 +1735,19 @@ int __kprobes register_jprobes(struct jprobe **jps, int num) · 24 unmodified lines
1731	1735	}
1732	1736	EXPORT_SYMBOL_GPL(register_jprobes);
1733	1737
1734		−int __kprobes register_jprobe(struct jprobe *jp)
	1738	+int register_jprobe(struct jprobe *jp)
1735	1739	{
1736	1740	return register_jprobes(&jp, 1);
1737	1741	}
1738	1742	EXPORT_SYMBOL_GPL(register_jprobe);
1739	1743
1740		−void __kprobes unregister_jprobe(struct jprobe *jp)
	1744	+void unregister_jprobe(struct jprobe *jp)
1741	1745	{
1742	1746	unregister_jprobes(&jp, 1);
1743	1747	}
1744	1748	EXPORT_SYMBOL_GPL(unregister_jprobe);
1745	1749
1746		−void __kprobes unregister_jprobes(struct jprobe **jps, int num)
	1750	+void unregister_jprobes(struct jprobe **jps, int num)
1747	1751	{
1748	1752	int i;
1749	1753
@@ -1768,8 +1772,7 @@ EXPORT_SYMBOL_GPL(unregister_jprobes); · 18 unmodified lines
1768	1772	* This kprobe pre_handler is registered with every kretprobe. When probe
1769	1773	* hits it will set up the return probe.
1770	1774	*/
1771		−static int __kprobes pre_handler_kretprobe(struct kprobe *p,
1772		− struct pt_regs *regs)
	1775	+static int pre_handler_kretprobe(struct kprobe p, struct pt_regs regs)
1773	1776	{
1774	1777	struct kretprobe *rp = container_of(p, struct kretprobe, kp);
1775	1778	unsigned long hash, flags = 0;
@@ -1807,8 +1810,9 @@ static int __kprobes pre_handler_kretprobe(struct kprobe *p, · 31 unmodified lines
1807	1810	}
1808	1811	return 0;
1809	1812	}
	1813	+NOKPROBE_SYMBOL(pre_handler_kretprobe);
1810	1814
1811		−int __kprobes register_kretprobe(struct kretprobe *rp)
	1815	+int register_kretprobe(struct kretprobe *rp)
1812	1816	{
1813	1817	int ret = 0;
1814	1818	struct kretprobe_instance *inst;
@@ -1861,7 +1865,7 @@ int __kprobes register_kretprobe(struct kretprobe *rp) · 46 unmodified lines
1861	1865	}
1862	1866	EXPORT_SYMBOL_GPL(register_kretprobe);
1863	1867
1864		−int __kprobes register_kretprobes(struct kretprobe **rps, int num)
	1868	+int register_kretprobes(struct kretprobe **rps, int num)
1865	1869	{
1866	1870	int ret = 0, i;
1867	1871
@@ -1879,13 +1883,13 @@ int __kprobes register_kretprobes(struct kretprobe **rps, int num) · 11 unmodified lines
1879	1883	}
1880	1884	EXPORT_SYMBOL_GPL(register_kretprobes);
1881	1885
1882		−void __kprobes unregister_kretprobe(struct kretprobe *rp)
	1886	+void unregister_kretprobe(struct kretprobe *rp)
1883	1887	{
1884	1888	unregister_kretprobes(&rp, 1);
1885	1889	}
1886	1890	EXPORT_SYMBOL_GPL(unregister_kretprobe);
1887	1891
1888		−void __kprobes unregister_kretprobes(struct kretprobe **rps, int num)
	1892	+void unregister_kretprobes(struct kretprobe **rps, int num)
1889	1893	{
1890	1894	int i;
1891	1895
@@ -1908,38 +1912,38 @@ void __kprobes unregister_kretprobes(struct kretprobe **rps, int num) · 16 unmodified lines
1908	1912	EXPORT_SYMBOL_GPL(unregister_kretprobes);
1909	1913
1910	1914	#else /* CONFIG_KRETPROBES */
1911		−int __kprobes register_kretprobe(struct kretprobe *rp)
	1915	+int register_kretprobe(struct kretprobe *rp)
1912	1916	{
1913	1917	return -ENOSYS;
1914	1918	}
1915	1919	EXPORT_SYMBOL_GPL(register_kretprobe);
1916	1920
1917		−int __kprobes register_kretprobes(struct kretprobe **rps, int num)
	1921	+int register_kretprobes(struct kretprobe **rps, int num)
1918	1922	{
1919	1923	return -ENOSYS;
1920	1924	}
1921	1925	EXPORT_SYMBOL_GPL(register_kretprobes);
1922	1926
1923		−void __kprobes unregister_kretprobe(struct kretprobe *rp)
	1927	+void unregister_kretprobe(struct kretprobe *rp)
1924	1928	{
1925	1929	}
1926	1930	EXPORT_SYMBOL_GPL(unregister_kretprobe);
1927	1931
1928		−void __kprobes unregister_kretprobes(struct kretprobe **rps, int num)
	1932	+void unregister_kretprobes(struct kretprobe **rps, int num)
1929	1933	{
1930	1934	}
1931	1935	EXPORT_SYMBOL_GPL(unregister_kretprobes);
1932	1936
1933		−static int __kprobes pre_handler_kretprobe(struct kprobe *p,
1934		− struct pt_regs *regs)
	1937	+static int pre_handler_kretprobe(struct kprobe p, struct pt_regs regs)
1935	1938	{
1936	1939	return 0;
1937	1940	}
	1941	+NOKPROBE_SYMBOL(pre_handler_kretprobe);
1938	1942
1939	1943	#endif /* CONFIG_KRETPROBES */
1940	1944
1941	1945	/* Set the kprobe gone and remove its instruction buffer. */
1942		−static void __kprobes kill_kprobe(struct kprobe *p)
	1946	+static void kill_kprobe(struct kprobe *p)
1943	1947	{
1944	1948	struct kprobe *kp;
1945	1949
@@ -1963,7 +1967,7 @@ static void __kprobes kill_kprobe(struct kprobe *p) · 17 unmodified lines
1963	1967	}
1964	1968
1965	1969	/* Disable one kprobe */
1966		−int __kprobes disable_kprobe(struct kprobe *kp)
	1970	+int disable_kprobe(struct kprobe *kp)
1967	1971	{
1968	1972	int ret = 0;
1969	1973
@@ -1979,7 +1983,7 @@ int __kprobes disable_kprobe(struct kprobe *kp) · 9 unmodified lines
1979	1983	EXPORT_SYMBOL_GPL(disable_kprobe);
1980	1984
1981	1985	/* Enable one kprobe */
1982		−int __kprobes enable_kprobe(struct kprobe *kp)
	1986	+int enable_kprobe(struct kprobe *kp)
1983	1987	{
1984	1988	int ret = 0;
1985	1989	struct kprobe *p;
@@ -2012,16 +2016,49 @@ out: · 26 unmodified lines
2012	2016	}
2013	2017	EXPORT_SYMBOL_GPL(enable_kprobe);
2014	2018
2015		−void __kprobes dump_kprobe(struct kprobe *kp)
	2019	+void dump_kprobe(struct kprobe *kp)
2016	2020	{
2017	2021	printk(KERN_WARNING "Dumping kprobe:\n");
2018	2022	printk(KERN_WARNING "Name: %s\nAddress: %p\nOffset: %x\n",
2019	2023	kp->symbol_name, kp->addr, kp->offset);
2020	2024	}
	2025	+NOKPROBE_SYMBOL(dump_kprobe);
	2026	+
	2027	+/*
	2028	+ * Lookup and populate the kprobe_blacklist.
	2029	+ *
	2030	+ * Unlike the kretprobe blacklist, we'll need to determine
	2031	+ * the range of addresses that belong to the said functions,
	2032	+ * since a kprobe need not necessarily be at the beginning
	2033	+ * of a function.
	2034	+ */
	2035	+static int __init populate_kprobe_blacklist(unsigned long *start,
	2036	+ unsigned long *end)
	2037	+{
	2038	+ unsigned long *iter;
	2039	+ struct kprobe_blacklist_entry *ent;
	2040	+ unsigned long offset = 0, size = 0;
	2041	+
	2042	+ for (iter = start; iter < end; iter++) {
	2043	+ if (!kallsyms_lookup_size_offset(*iter, &size, &offset)) {
	2044	+ pr_err("Failed to find blacklist %p\n", (void )iter);
	2045	+ continue;
	2046	+ }
	2047	+
	2048	+ ent = kmalloc(sizeof(*ent), GFP_KERNEL);
	2049	+ if (!ent)
	2050	+ return -ENOMEM;
	2051	+ ent->start_addr = *iter;
	2052	+ ent->end_addr = *iter + size;
	2053	+ INIT_LIST_HEAD(&ent->list);
	2054	+ list_add_tail(&ent->list, &kprobe_blacklist);
	2055	+ }
	2056	+ return 0;
	2057	+}
2021	2058
2022	2059	/* Module notifier call back, checking kprobes on the module */
2023		−static int __kprobes kprobes_module_callback(struct notifier_block *nb,
2024		− unsigned long val, void *data)
	2060	+static int kprobes_module_callback(struct notifier_block *nb,
	2061	+ unsigned long val, void *data)
2025	2062	{
2026	2063	struct module *mod = data;
2027	2064	struct hlist_head *head;
@@ -2062,14 +2099,13 @@ static struct notifier_block kprobe_module_nb = { · 34 unmodified lines
2062	2099	.priority = 0
2063	2100	};
2064	2101
	2102	+/* Markers of _kprobe_blacklist section */
	2103	+extern unsigned long __start_kprobe_blacklist[];
	2104	+extern unsigned long __stop_kprobe_blacklist[];
	2105	+
2065	2106	static int __init init_kprobes(void)
2066	2107	{
2067	2108	int i, err = 0;
2068		− unsigned long offset = 0, size = 0;
2069		− char *modname, namebuf[KSYM_NAME_LEN];
2070		− const char *symbol_name;
2071		− void *addr;
2072		− struct kprobe_blackpoint *kb;
2073	2109
2074	2110	/* FIXME allocate the probe table, currently defined statically */
2075	2111	/* initialize all list heads */
@@ -2079,26 +2115,11 @@ static int __init init_kprobes(void) · 3 unmodified lines
2079	2115	raw_spin_lock_init(&(kretprobe_table_locks[i].lock));
2080	2116	}
2081	2117
2082		− /*
2083		− * Lookup and populate the kprobe_blacklist.
2084		− *
2085		− * Unlike the kretprobe blacklist, we'll need to determine
2086		− * the range of addresses that belong to the said functions,
2087		− * since a kprobe need not necessarily be at the beginning
2088		− * of a function.
2089		− */
2090		− for (kb = kprobe_blacklist; kb->name != NULL; kb++) {
2091		− kprobe_lookup_name(kb->name, addr);
2092		− if (!addr)
2093		− continue;
2094		−
2095		− kb->start_addr = (unsigned long)addr;
2096		− symbol_name = kallsyms_lookup(kb->start_addr,
2097		− &size, &offset, &modname, namebuf);
2098		− if (!symbol_name)
2099		− kb->range = 0;
2100		− else
2101		− kb->range = size;
	2118	+ err = populate_kprobe_blacklist(__start_kprobe_blacklist,
	2119	+ __stop_kprobe_blacklist);
	2120	+ if (err) {
	2121	+ pr_err("kprobes: failed to populate blacklist: %d\n", err);
	2122	+ pr_err("Please take care of using kprobes.\n");
2102	2123	}
2103	2124
2104	2125	if (kretprobe_blacklist_size) {
@@ -2138,7 +2159,7 @@ static int __init init_kprobes(void) · 33 unmodified lines
2138	2159	}
2139	2160
2140	2161	#ifdef CONFIG_DEBUG_FS
2141		−static void __kprobes report_probe(struct seq_file pi, struct kprobe p,
	2162	+static void report_probe(struct seq_file pi, struct kprobe p,
2142	2163	const char sym, int offset, char modname, struct kprobe *pp)
2143	2164	{
2144	2165	char *kprobe_type;
@@ -2167,12 +2188,12 @@ static void __kprobes report_probe(struct seq_file pi, struct kprobe p, · 22 unmodified lines
2167	2188	(kprobe_ftrace(pp) ? "[FTRACE]" : ""));
2168	2189	}
2169	2190
2170		−static void __kprobes kprobe_seq_start(struct seq_file f, loff_t *pos)
	2191	+static void kprobe_seq_start(struct seq_file f, loff_t *pos)
2171	2192	{
2172	2193	return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL;
2173	2194	}
2174	2195
2175		−static void __kprobes kprobe_seq_next(struct seq_file f, void v, loff_t pos)
	2196	+static void kprobe_seq_next(struct seq_file f, void v, loff_t pos)
2176	2197	{
2177	2198	(*pos)++;
2178	2199	if (*pos >= KPROBE_TABLE_SIZE)
@@ -2180,12 +2201,12 @@ static void __kprobes kprobe_seq_next(struct seq_file f, void v, loff_t pos) · 1 unmodified line
2180	2201	return pos;
2181	2202	}
2182	2203
2183		−static void __kprobes kprobe_seq_stop(struct seq_file f, void v)
	2204	+static void kprobe_seq_stop(struct seq_file f, void v)
2184	2205	{
2185	2206	/* Nothing to do */
2186	2207	}
2187	2208
2188		−static int __kprobes show_kprobe_addr(struct seq_file pi, void v)
	2209	+static int show_kprobe_addr(struct seq_file pi, void v)
2189	2210	{
2190	2211	struct hlist_head *head;
2191	2212	struct kprobe p, kp;
@@ -2216,7 +2237,7 @@ static const struct seq_operations kprobes_seq_ops = { · 24 unmodified lines
2216	2237	.show = show_kprobe_addr
2217	2238	};
2218	2239
2219		−static int __kprobes kprobes_open(struct inode inode, struct file filp)
	2240	+static int kprobes_open(struct inode inode, struct file filp)
2220	2241	{
2221	2242	return seq_open(filp, &kprobes_seq_ops);
2222	2243	}
@@ -2228,7 +2249,47 @@ static const struct file_operations debugfs_kprobes_operations = { · 5 unmodified lines
2228	2249	.release = seq_release,
2229	2250	};
2230	2251
2231		−static void __kprobes arm_all_kprobes(void)
	2252	+/* kprobes/blacklist -- shows which functions can not be probed */
	2253	+static void kprobe_blacklist_seq_start(struct seq_file m, loff_t *pos)
	2254	+{
	2255	+ return seq_list_start(&kprobe_blacklist, *pos);
	2256	+}
	2257	+
	2258	+static void kprobe_blacklist_seq_next(struct seq_file m, void v, loff_t pos)
	2259	+{
	2260	+ return seq_list_next(v, &kprobe_blacklist, pos);
	2261	+}
	2262	+
	2263	+static int kprobe_blacklist_seq_show(struct seq_file m, void v)
	2264	+{
	2265	+ struct kprobe_blacklist_entry *ent =
	2266	+ list_entry(v, struct kprobe_blacklist_entry, list);
	2267	+
	2268	+ seq_printf(m, "0x%p-0x%p\t%ps\n", (void *)ent->start_addr,
	2269	+ (void )ent->end_addr, (void )ent->start_addr);
	2270	+ return 0;
	2271	+}
	2272	+
	2273	+static const struct seq_operations kprobe_blacklist_seq_ops = {
	2274	+ .start = kprobe_blacklist_seq_start,
	2275	+ .next = kprobe_blacklist_seq_next,
	2276	+ .stop = kprobe_seq_stop, /* Reuse void function */
	2277	+ .show = kprobe_blacklist_seq_show,
	2278	+};
	2279	+
	2280	+static int kprobe_blacklist_open(struct inode inode, struct file filp)
	2281	+{
	2282	+ return seq_open(filp, &kprobe_blacklist_seq_ops);
	2283	+}
	2284	+
	2285	+static const struct file_operations debugfs_kprobe_blacklist_ops = {
	2286	+ .open = kprobe_blacklist_open,
	2287	+ .read = seq_read,
	2288	+ .llseek = seq_lseek,
	2289	+ .release = seq_release,
	2290	+};
	2291	+
	2292	+static void arm_all_kprobes(void)
2232	2293	{
2233	2294	struct hlist_head *head;
2234	2295	struct kprobe *p;
@@ -2256,7 +2317,7 @@ already_enabled: · 21 unmodified lines
2256	2317	return;
2257	2318	}
2258	2319
2259		−static void __kprobes disarm_all_kprobes(void)
	2320	+static void disarm_all_kprobes(void)
2260	2321	{
2261	2322	struct hlist_head *head;
2262	2323	struct kprobe *p;
@@ -2340,7 +2401,7 @@ static const struct file_operations fops_kp = { · 77 unmodified lines
2340	2401	.llseek = default_llseek,
2341	2402	};
2342	2403
2343		−static int __kprobes debugfs_kprobe_init(void)
	2404	+static int __init debugfs_kprobe_init(void)
2344	2405	{
2345	2406	struct dentry dir, file;
2346	2407	unsigned int value = 1;
@@ -2351,19 +2412,24 @@ static int __kprobes debugfs_kprobe_init(void) · 4 unmodified lines
2351	2412
2352	2413	file = debugfs_create_file("list", 0444, dir, NULL,
2353	2414	&debugfs_kprobes_operations);
2354		− if (!file) {
2355		− debugfs_remove(dir);
2356		− return -ENOMEM;
2357		− }
	2415	+ if (!file)
	2416	+ goto error;
2358	2417
2359	2418	file = debugfs_create_file("enabled", 0600, dir,
2360	2419	&value, &fops_kp);
2361		− if (!file) {
2362		− debugfs_remove(dir);
2363		− return -ENOMEM;
2364		− }
	2420	+ if (!file)
	2421	+ goto error;
	2422	+
	2423	+ file = debugfs_create_file("blacklist", 0444, dir, NULL,
	2424	+ &debugfs_kprobe_blacklist_ops);
	2425	+ if (!file)
	2426	+ goto error;
2365	2427
2366	2428	return 0;
	2429	+
	2430	+error:
	2431	+ debugfs_remove(dir);
	2432	+ return -ENOMEM;
2367	2433	}
2368	2434
2369	2435	late_initcall(debugfs_kprobe_init);

kernel/notifier.c+13−9

@@ -71,9 +71,9 @@ static int notifier_chain_unregister(struct notifier_block **nl, · 70 unmodified lines
71	71	* @returns: notifier_call_chain returns the value returned by the
72	72	* last notifier function called.
73	73	*/
74		−static int __kprobes notifier_call_chain(struct notifier_block **nl,
75		− unsigned long val, void *v,
76		− int nr_to_call, int *nr_calls)
	74	+static int notifier_call_chain(struct notifier_block **nl,
	75	+ unsigned long val, void *v,
	76	+ int nr_to_call, int *nr_calls)
77	77	{
78	78	int ret = NOTIFY_DONE;
79	79	struct notifier_block nb, next_nb;
@@ -102,6 +102,7 @@ static int __kprobes notifier_call_chain(struct notifier_block **nl, · 22 unmodified lines
102	102	}
103	103	return ret;
104	104	}
	105	+NOKPROBE_SYMBOL(notifier_call_chain);
105	106
106	107	/*
107	108	* Atomic notifier chain routines. Registration and unregistration
@@ -172,9 +173,9 @@ EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister); · 64 unmodified lines
172	173	* Otherwise the return value is the return value
173	174	* of the last notifier function called.
174	175	*/
175		−int __kprobes __atomic_notifier_call_chain(struct atomic_notifier_head *nh,
176		− unsigned long val, void *v,
177		− int nr_to_call, int *nr_calls)
	176	+int __atomic_notifier_call_chain(struct atomic_notifier_head *nh,
	177	+ unsigned long val, void *v,
	178	+ int nr_to_call, int *nr_calls)
178	179	{
179	180	int ret;
180	181
@@ -184,13 +185,15 @@ int __kprobes __atomic_notifier_call_chain(struct atomic_notifier_head *nh, · 3 unmodified lines
184	185	return ret;
185	186	}
186	187	EXPORT_SYMBOL_GPL(__atomic_notifier_call_chain);
	188	+NOKPROBE_SYMBOL(__atomic_notifier_call_chain);
187	189
188		−int __kprobes atomic_notifier_call_chain(struct atomic_notifier_head *nh,
189		− unsigned long val, void *v)
	190	+int atomic_notifier_call_chain(struct atomic_notifier_head *nh,
	191	+ unsigned long val, void *v)
190	192	{
191	193	return __atomic_notifier_call_chain(nh, val, v, -1, NULL);
192	194	}
193	195	EXPORT_SYMBOL_GPL(atomic_notifier_call_chain);
	196	+NOKPROBE_SYMBOL(atomic_notifier_call_chain);
194	197
195	198	/*
196	199	* Blocking notifier chain routines. All access to the chain is
@@ -527,7 +530,7 @@ EXPORT_SYMBOL_GPL(srcu_init_notifier_head); · 330 unmodified lines
527	530
528	531	static ATOMIC_NOTIFIER_HEAD(die_chain);
529	532
530		−int notrace __kprobes notify_die(enum die_val val, const char *str,
	533	+int notrace notify_die(enum die_val val, const char *str,
531	534	struct pt_regs *regs, long err, int trap, int sig)
532	535	{
533	536	struct die_args args = {
@@ -540,6 +543,7 @@ int notrace __kprobes notify_die(enum die_val val, const char *str, · 6 unmodified lines
540	543	};
541	544	return atomic_notifier_call_chain(&die_chain, val, &args);
542	545	}
	546	+NOKPROBE_SYMBOL(notify_die);
543	547
544	548	int register_die_notifier(struct notifier_block *nb)
545	549	{

kernel/sched/core.c+5−2

@@ -2527,7 +2527,7 @@ notrace unsigned long get_parent_ip(unsigned long addr) · 2526 unmodified lines
2527	2527	#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) \|\| \
2528	2528	defined(CONFIG_PREEMPT_TRACER))
2529	2529
2530		−void __kprobes preempt_count_add(int val)
	2530	+void preempt_count_add(int val)
2531	2531	{
2532	2532	#ifdef CONFIG_DEBUG_PREEMPT
2533	2533	/*
@@ -2553,8 +2553,9 @@ void __kprobes preempt_count_add(int val) · 19 unmodified lines
2553	2553	}
2554	2554	}
2555	2555	EXPORT_SYMBOL(preempt_count_add);
	2556	+NOKPROBE_SYMBOL(preempt_count_add);
2556	2557
2557		−void __kprobes preempt_count_sub(int val)
	2558	+void preempt_count_sub(int val)
2558	2559	{
2559	2560	#ifdef CONFIG_DEBUG_PREEMPT
2560	2561	/*
@@ -2575,6 +2576,7 @@ void __kprobes preempt_count_sub(int val) · 14 unmodified lines
2575	2576	__preempt_count_sub(val);
2576	2577	}
2577	2578	EXPORT_SYMBOL(preempt_count_sub);
	2579	+NOKPROBE_SYMBOL(preempt_count_sub);
2578	2580
2579	2581	#endif
2580	2582
@@ -2857,6 +2859,7 @@ asmlinkage __visible void __sched notrace preempt_schedule(void) · 276 unmodified lines
2857	2859	barrier();
2858	2860	} while (need_resched());
2859	2861	}
	2862	+NOKPROBE_SYMBOL(preempt_schedule);
2860	2863	EXPORT_SYMBOL(preempt_schedule);
2861	2864	#endif /* CONFIG_PREEMPT */
2862	2865

kernel/trace/trace_event_perf.c+3−2

@@ -248,8 +248,8 @@ void perf_trace_del(struct perf_event *p_event, int flags) · 247 unmodified lines
248	248	tp_event->class->reg(tp_event, TRACE_REG_PERF_DEL, p_event);
249	249	}
250	250
251		−__kprobes void *perf_trace_buf_prepare(int size, unsigned short type,
252		− struct pt_regs regs, int rctxp)
	251	+void *perf_trace_buf_prepare(int size, unsigned short type,
	252	+ struct pt_regs regs, int rctxp)
253	253	{
254	254	struct trace_entry *entry;
255	255	unsigned long flags;
@@ -281,6 +281,7 @@ __kprobes void *perf_trace_buf_prepare(int size, unsigned short type, · 25 unmodified lines
281	281	return raw_data;
282	282	}
283	283	EXPORT_SYMBOL_GPL(perf_trace_buf_prepare);
	284	+NOKPROBE_SYMBOL(perf_trace_buf_prepare);
284	285
285	286	#ifdef CONFIG_FUNCTION_TRACER
286	287	static void

kernel/trace/trace_kprobe.c+41−30

@@ -40,27 +40,27 @@ struct trace_kprobe { · 39 unmodified lines
40	40	(sizeof(struct probe_arg) * (n)))
41	41
42	42
43		−static __kprobes bool trace_kprobe_is_return(struct trace_kprobe *tk)
	43	+static nokprobe_inline bool trace_kprobe_is_return(struct trace_kprobe *tk)
44	44	{
45	45	return tk->rp.handler != NULL;
46	46	}
47	47
48		−static __kprobes const char trace_kprobe_symbol(struct trace_kprobe tk)
	48	+static nokprobe_inline const char trace_kprobe_symbol(struct trace_kprobe tk)
49	49	{
50	50	return tk->symbol ? tk->symbol : "unknown";
51	51	}
52	52
53		−static __kprobes unsigned long trace_kprobe_offset(struct trace_kprobe *tk)
	53	+static nokprobe_inline unsigned long trace_kprobe_offset(struct trace_kprobe *tk)
54	54	{
55	55	return tk->rp.kp.offset;
56	56	}
57	57
58		−static __kprobes bool trace_kprobe_has_gone(struct trace_kprobe *tk)
	58	+static nokprobe_inline bool trace_kprobe_has_gone(struct trace_kprobe *tk)
59	59	{
60	60	return !!(kprobe_gone(&tk->rp.kp));
61	61	}
62	62
63		−static __kprobes bool trace_kprobe_within_module(struct trace_kprobe *tk,
	63	+static nokprobe_inline bool trace_kprobe_within_module(struct trace_kprobe *tk,
64	64	struct module *mod)
65	65	{
66	66	int len = strlen(mod->name);
@@ -68,7 +68,7 @@ static __kprobes bool trace_kprobe_within_module(struct trace_kprobe *tk, · 1 unmodified line
68	68	return strncmp(mod->name, name, len) == 0 && name[len] == ':';
69	69	}
70	70
71		−static __kprobes bool trace_kprobe_is_on_module(struct trace_kprobe *tk)
	71	+static nokprobe_inline bool trace_kprobe_is_on_module(struct trace_kprobe *tk)
72	72	{
73	73	return !!strchr(trace_kprobe_symbol(tk), ':');
74	74	}
@@ -132,19 +132,21 @@ struct symbol_cache alloc_symbol_cache(const char sym, long offset) · 57 unmodified lines
132	132	* Kprobes-specific fetch functions
133	133	*/
134	134	#define DEFINE_FETCH_stack(type) \
135		−static __kprobes void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs,\
	135	+static void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs, \
136	136	void offset, void dest) \
137	137	{ \
138	138	(type )dest = (type)regs_get_kernel_stack_nth(regs, \
139	139	(unsigned int)((unsigned long)offset)); \
140		−}
	140	+} \
	141	+NOKPROBE_SYMBOL(FETCH_FUNC_NAME(stack, type));
	142	+
141	143	DEFINE_BASIC_FETCH_FUNCS(stack)
142	144	/* No string on the stack entry */
143	145	#define fetch_stack_string NULL
144	146	#define fetch_stack_string_size NULL
145	147
146	148	#define DEFINE_FETCH_memory(type) \
147		−static __kprobes void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs,\
	149	+static void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs, \
148	150	void addr, void dest) \
149	151	{ \
150	152	type retval; \
@@ -152,14 +154,16 @@ static __kprobes void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs,\ · 1 unmodified line
152	154	(type )dest = 0; \
153	155	else \
154	156	(type )dest = retval; \
155		−}
	157	+} \
	158	+NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, type));
	159	+
156	160	DEFINE_BASIC_FETCH_FUNCS(memory)
157	161	/*
158	162	* Fetch a null-terminated string. Caller MUST set (u32 )dest with max
159	163	* length and relative data location.
160	164	*/
161		−static __kprobes void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs,
162		− void addr, void dest)
	165	+static void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs,
	166	+ void addr, void dest)
163	167	{
164	168	long ret;
165	169	int maxlen = get_rloc_len((u32 )dest);
@@ -193,10 +197,11 @@ static __kprobes void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs, · 27 unmodified lines
193	197	get_rloc_offs((u32 )dest));
194	198	}
195	199	}
	200	+NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, string));
196	201
197	202	/* Return the length of string -- including null terminal byte */
198		−static __kprobes void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs,
199		− void addr, void dest)
	203	+static void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs,
	204	+ void addr, void dest)
200	205	{
201	206	mm_segment_t old_fs;
202	207	int ret, len = 0;
@@ -219,17 +224,19 @@ static __kprobes void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs, · 16 unmodified lines
219	224	else
220	225	(u32 )dest = len;
221	226	}
	227	+NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, string_size));
222	228
223	229	#define DEFINE_FETCH_symbol(type) \
224		−__kprobes void FETCH_FUNC_NAME(symbol, type)(struct pt_regs *regs, \
225		− void data, void dest) \
	230	+void FETCH_FUNC_NAME(symbol, type)(struct pt_regs regs, void data, void *dest)\
226	231	{ \
227	232	struct symbol_cache *sc = data; \
228	233	if (sc->addr) \
229	234	fetch_memory_##type(regs, (void *)sc->addr, dest); \
230	235	else \
231	236	(type )dest = 0; \
232		−}
	237	+} \
	238	+NOKPROBE_SYMBOL(FETCH_FUNC_NAME(symbol, type));
	239	+
233	240	DEFINE_BASIC_FETCH_FUNCS(symbol)
234	241	DEFINE_FETCH_symbol(string)
235	242	DEFINE_FETCH_symbol(string_size)
@@ -907,7 +914,7 @@ static const struct file_operations kprobe_profile_ops = { · 671 unmodified lines
907	914	};
908	915
909	916	/* Kprobe handler */
910		−static __kprobes void
	917	+static nokprobe_inline void
911	918	__kprobe_trace_func(struct trace_kprobe tk, struct pt_regs regs,
912	919	struct ftrace_event_file *ftrace_file)
913	920	{
@@ -943,7 +950,7 @@ __kprobe_trace_func(struct trace_kprobe tk, struct pt_regs regs, · 29 unmodified lines
943	950	entry, irq_flags, pc, regs);
944	951	}
945	952
946		−static __kprobes void
	953	+static void
947	954	kprobe_trace_func(struct trace_kprobe tk, struct pt_regs regs)
948	955	{
949	956	struct event_file_link *link;
@@ -951,9 +958,10 @@ kprobe_trace_func(struct trace_kprobe tk, struct pt_regs regs) · 1 unmodified line
951	958	list_for_each_entry_rcu(link, &tk->tp.files, list)
952	959	__kprobe_trace_func(tk, regs, link->file);
953	960	}
	961	+NOKPROBE_SYMBOL(kprobe_trace_func);
954	962
955	963	/* Kretprobe handler */
956		−static __kprobes void
	964	+static nokprobe_inline void
957	965	__kretprobe_trace_func(struct trace_kprobe tk, struct kretprobe_instance ri,
958	966	struct pt_regs *regs,
959	967	struct ftrace_event_file *ftrace_file)
@@ -991,7 +999,7 @@ __kretprobe_trace_func(struct trace_kprobe tk, struct kretprobe_instance ri, · 31 unmodified lines
991	999	entry, irq_flags, pc, regs);
992	1000	}
993	1001
994		−static __kprobes void
	1002	+static void
995	1003	kretprobe_trace_func(struct trace_kprobe tk, struct kretprobe_instance ri,
996	1004	struct pt_regs *regs)
997	1005	{
@@ -1000,6 +1008,7 @@ kretprobe_trace_func(struct trace_kprobe tk, struct kretprobe_instance ri, · 2 unmodified lines
1000	1008	list_for_each_entry_rcu(link, &tk->tp.files, list)
1001	1009	__kretprobe_trace_func(tk, ri, regs, link->file);
1002	1010	}
	1011	+NOKPROBE_SYMBOL(kretprobe_trace_func);
1003	1012
1004	1013	/* Event entry printers */
1005	1014	static enum print_line_t
@@ -1131,7 +1140,7 @@ static int kretprobe_event_define_fields(struct ftrace_event_call *event_call) · 125 unmodified lines
1131	1140	#ifdef CONFIG_PERF_EVENTS
1132	1141
1133	1142	/* Kprobe profile handler */
1134		−static __kprobes void
	1143	+static void
1135	1144	kprobe_perf_func(struct trace_kprobe tk, struct pt_regs regs)
1136	1145	{
1137	1146	struct ftrace_event_call *call = &tk->tp.call;
@@ -1158,9 +1167,10 @@ kprobe_perf_func(struct trace_kprobe tk, struct pt_regs regs) · 20 unmodified lines
1158	1167	store_trace_args(sizeof(entry), &tk->tp, regs, (u8 )&entry[1], dsize);
1159	1168	perf_trace_buf_submit(entry, size, rctx, 0, 1, regs, head, NULL);
1160	1169	}
	1170	+NOKPROBE_SYMBOL(kprobe_perf_func);
1161	1171
1162	1172	/* Kretprobe profile handler */
1163		−static __kprobes void
	1173	+static void
1164	1174	kretprobe_perf_func(struct trace_kprobe tk, struct kretprobe_instance ri,
1165	1175	struct pt_regs *regs)
1166	1176	{
@@ -1188,6 +1198,7 @@ kretprobe_perf_func(struct trace_kprobe tk, struct kretprobe_instance ri, · 21 unmodified lines
1188	1198	store_trace_args(sizeof(entry), &tk->tp, regs, (u8 )&entry[1], dsize);
1189	1199	perf_trace_buf_submit(entry, size, rctx, 0, 1, regs, head, NULL);
1190	1200	}
	1201	+NOKPROBE_SYMBOL(kretprobe_perf_func);
1191	1202	#endif /* CONFIG_PERF_EVENTS */
1192	1203
1193	1204	/*
@@ -1196,9 +1207,8 @@ kretprobe_perf_func(struct trace_kprobe tk, struct kretprobe_instance ri, · 2 unmodified lines
1196	1207	* kprobe_trace_self_tests_init() does enable_trace_probe/disable_trace_probe
1197	1208	* lockless, but we can't race with this __init function.
1198	1209	*/
1199		−static __kprobes
1200		−int kprobe_register(struct ftrace_event_call *event,
1201		− enum trace_reg type, void *data)
	1210	+static int kprobe_register(struct ftrace_event_call *event,
	1211	+ enum trace_reg type, void *data)
1202	1212	{
1203	1213	struct trace_kprobe tk = (struct trace_kprobe )event->data;
1204	1214	struct ftrace_event_file *file = data;
@@ -1224,8 +1234,7 @@ int kprobe_register(struct ftrace_event_call *event, · 19 unmodified lines
1224	1234	return 0;
1225	1235	}
1226	1236
1227		−static __kprobes
1228		−int kprobe_dispatcher(struct kprobe kp, struct pt_regs regs)
	1237	+static int kprobe_dispatcher(struct kprobe kp, struct pt_regs regs)
1229	1238	{
1230	1239	struct trace_kprobe *tk = container_of(kp, struct trace_kprobe, rp.kp);
1231	1240
@@ -1239,9 +1248,10 @@ int kprobe_dispatcher(struct kprobe kp, struct pt_regs regs) · 7 unmodified lines
1239	1248	#endif
1240	1249	return 0; /* We don't tweek kernel, so just return 0 */
1241	1250	}
	1251	+NOKPROBE_SYMBOL(kprobe_dispatcher);
1242	1252
1243		−static __kprobes
1244		−int kretprobe_dispatcher(struct kretprobe_instance ri, struct pt_regs regs)
	1253	+static int
	1254	+kretprobe_dispatcher(struct kretprobe_instance ri, struct pt_regs regs)
1245	1255	{
1246	1256	struct trace_kprobe *tk = container_of(ri->rp, struct trace_kprobe, rp);
1247	1257
@@ -1255,6 +1265,7 @@ int kretprobe_dispatcher(struct kretprobe_instance ri, struct pt_regs regs) · 7 unmodified lines
1255	1265	#endif
1256	1266	return 0; /* We don't tweek kernel, so just return 0 */
1257	1267	}
	1268	+NOKPROBE_SYMBOL(kretprobe_dispatcher);
1258	1269
1259	1270	static struct trace_event_functions kretprobe_funcs = {
1260	1271	.trace = print_kretprobe_event

kernel/trace/trace_probe.c+35−30

@@ -37,13 +37,13 @@ const char *reserved_field_names[] = { · 36 unmodified lines
37	37
38	38	/* Printing in basic type function template */
39	39	#define DEFINE_BASIC_PRINT_TYPE_FUNC(type, fmt) \
40		−__kprobes int PRINT_TYPE_FUNC_NAME(type)(struct trace_seq *s, \
41		− const char *name, \
42		− void data, void ent) \
	40	+int PRINT_TYPE_FUNC_NAME(type)(struct trace_seq s, const char name, \
	41	+ void data, void ent) \
43	42	{ \
44	43	return trace_seq_printf(s, " %s=" fmt, name, (type )data); \
45	44	} \
46		−const char PRINT_TYPE_FMT_NAME(type)[] = fmt;
	45	+const char PRINT_TYPE_FMT_NAME(type)[] = fmt; \
	46	+NOKPROBE_SYMBOL(PRINT_TYPE_FUNC_NAME(type));
47	47
48	48	DEFINE_BASIC_PRINT_TYPE_FUNC(u8 , "0x%x")
49	49	DEFINE_BASIC_PRINT_TYPE_FUNC(u16, "0x%x")
@@ -55,9 +55,8 @@ DEFINE_BASIC_PRINT_TYPE_FUNC(s32, "%d") · 5 unmodified lines
55	55	DEFINE_BASIC_PRINT_TYPE_FUNC(s64, "%Ld")
56	56
57	57	/* Print type function for string type */
58		−__kprobes int PRINT_TYPE_FUNC_NAME(string)(struct trace_seq *s,
59		− const char *name,
60		− void data, void ent)
	58	+int PRINT_TYPE_FUNC_NAME(string)(struct trace_seq s, const char name,
	59	+ void data, void ent)
61	60	{
62	61	int len = (u32 )data >> 16;
63	62
@@ -67,6 +66,7 @@ __kprobes int PRINT_TYPE_FUNC_NAME(string)(struct trace_seq *s, · 3 unmodified lines
67	66	return trace_seq_printf(s, " %s=\"%s\"", name,
68	67	(const char *)get_loc_data(data, ent));
69	68	}
	69	+NOKPROBE_SYMBOL(PRINT_TYPE_FUNC_NAME(string));
70	70
71	71	const char PRINT_TYPE_FMT_NAME(string)[] = "\\\"%s\\\"";
72	72
@@ -81,23 +81,24 @@ const char PRINT_TYPE_FMT_NAME(string)[] = "\\\"%s\\\""; · 8 unmodified lines
81	81
82	82	/* Data fetch function templates */
83	83	#define DEFINE_FETCH_reg(type) \
84		−__kprobes void FETCH_FUNC_NAME(reg, type)(struct pt_regs *regs, \
85		− void offset, void dest) \
	84	+void FETCH_FUNC_NAME(reg, type)(struct pt_regs regs, void offset, void *dest) \
86	85	{ \
87	86	(type )dest = (type)regs_get_register(regs, \
88	87	(unsigned int)((unsigned long)offset)); \
89		−}
	88	+} \
	89	+NOKPROBE_SYMBOL(FETCH_FUNC_NAME(reg, type));
90	90	DEFINE_BASIC_FETCH_FUNCS(reg)
91	91	/* No string on the register */
92	92	#define fetch_reg_string NULL
93	93	#define fetch_reg_string_size NULL
94	94
95	95	#define DEFINE_FETCH_retval(type) \
96		−__kprobes void FETCH_FUNC_NAME(retval, type)(struct pt_regs *regs, \
97		− void dummy, void dest) \
	96	+void FETCH_FUNC_NAME(retval, type)(struct pt_regs *regs, \
	97	+ void dummy, void dest) \
98	98	{ \
99	99	(type )dest = (type)regs_return_value(regs); \
100		−}
	100	+} \
	101	+NOKPROBE_SYMBOL(FETCH_FUNC_NAME(retval, type));
101	102	DEFINE_BASIC_FETCH_FUNCS(retval)
102	103	/* No string on the retval */
103	104	#define fetch_retval_string NULL
@@ -112,8 +113,8 @@ struct deref_fetch_param { · 8 unmodified lines
112	113	};
113	114
114	115	#define DEFINE_FETCH_deref(type) \
115		−__kprobes void FETCH_FUNC_NAME(deref, type)(struct pt_regs *regs, \
116		− void data, void dest) \
	116	+void FETCH_FUNC_NAME(deref, type)(struct pt_regs *regs, \
	117	+ void data, void dest) \
117	118	{ \
118	119	struct deref_fetch_param *dprm = data; \
119	120	unsigned long addr; \
@@ -123,12 +124,13 @@ __kprobes void FETCH_FUNC_NAME(deref, type)(struct pt_regs *regs, \ · 3 unmodified lines
123	124	dprm->fetch(regs, (void *)addr, dest); \
124	125	} else \
125	126	(type )dest = 0; \
126		−}
	127	+} \
	128	+NOKPROBE_SYMBOL(FETCH_FUNC_NAME(deref, type));
127	129	DEFINE_BASIC_FETCH_FUNCS(deref)
128	130	DEFINE_FETCH_deref(string)
129	131
130		−__kprobes void FETCH_FUNC_NAME(deref, string_size)(struct pt_regs *regs,
131		− void data, void dest)
	132	+void FETCH_FUNC_NAME(deref, string_size)(struct pt_regs *regs,
	133	+ void data, void dest)
132	134	{
133	135	struct deref_fetch_param *dprm = data;
134	136	unsigned long addr;
@@ -140,16 +142,18 @@ __kprobes void FETCH_FUNC_NAME(deref, string_size)(struct pt_regs *regs, · 5 unmodified lines
140	142	} else
141	143	(string_size )dest = 0;
142	144	}
	145	+NOKPROBE_SYMBOL(FETCH_FUNC_NAME(deref, string_size));
143	146
144		−static __kprobes void update_deref_fetch_param(struct deref_fetch_param *data)
	147	+static void update_deref_fetch_param(struct deref_fetch_param *data)
145	148	{
146	149	if (CHECK_FETCH_FUNCS(deref, data->orig.fn))
147	150	update_deref_fetch_param(data->orig.data);
148	151	else if (CHECK_FETCH_FUNCS(symbol, data->orig.fn))
149	152	update_symbol_cache(data->orig.data);
150	153	}
	154	+NOKPROBE_SYMBOL(update_deref_fetch_param);
151	155
152		−static __kprobes void free_deref_fetch_param(struct deref_fetch_param *data)
	156	+static void free_deref_fetch_param(struct deref_fetch_param *data)
153	157	{
154	158	if (CHECK_FETCH_FUNCS(deref, data->orig.fn))
155	159	free_deref_fetch_param(data->orig.data);
@@ -157,6 +161,7 @@ static __kprobes void free_deref_fetch_param(struct deref_fetch_param *data) · 1 unmodified line
157	161	free_symbol_cache(data->orig.data);
158	162	kfree(data);
159	163	}
	164	+NOKPROBE_SYMBOL(free_deref_fetch_param);
160	165
161	166	/* Bitfield fetch function */
162	167	struct bitfield_fetch_param {
@@ -166,8 +171,8 @@ struct bitfield_fetch_param { · 3 unmodified lines
166	171	};
167	172
168	173	#define DEFINE_FETCH_bitfield(type) \
169		−__kprobes void FETCH_FUNC_NAME(bitfield, type)(struct pt_regs *regs, \
170		− void data, void dest) \
	174	+void FETCH_FUNC_NAME(bitfield, type)(struct pt_regs *regs, \
	175	+ void data, void dest) \
171	176	{ \
172	177	struct bitfield_fetch_param *bprm = data; \
173	178	type buf = 0; \
@@ -177,13 +182,13 @@ __kprobes void FETCH_FUNC_NAME(bitfield, type)(struct pt_regs *regs, \ · 3 unmodified lines
177	182	buf >>= bprm->low_shift; \
178	183	} \
179	184	(type )dest = buf; \
180		−}
181		−
	185	+} \
	186	+NOKPROBE_SYMBOL(FETCH_FUNC_NAME(bitfield, type));
182	187	DEFINE_BASIC_FETCH_FUNCS(bitfield)
183	188	#define fetch_bitfield_string NULL
184	189	#define fetch_bitfield_string_size NULL
185	190
186		−static __kprobes void
	191	+static void
187	192	update_bitfield_fetch_param(struct bitfield_fetch_param *data)
188	193	{
189	194	/*
@@ -196,7 +201,7 @@ update_bitfield_fetch_param(struct bitfield_fetch_param *data) · 6 unmodified lines
196	201	update_symbol_cache(data->orig.data);
197	202	}
198	203
199		−static __kprobes void
	204	+static void
200	205	free_bitfield_fetch_param(struct bitfield_fetch_param *data)
201	206	{
202	207	/*
@@ -255,17 +260,17 @@ fail: · 52 unmodified lines
255	260	}
256	261
257	262	/* Special function : only accept unsigned long */
258		−static __kprobes void fetch_kernel_stack_address(struct pt_regs *regs,
259		− void dummy, void dest)
	263	+static void fetch_kernel_stack_address(struct pt_regs regs, void dummy, void *dest)
260	264	{
261	265	(unsigned long )dest = kernel_stack_pointer(regs);
262	266	}
	267	+NOKPROBE_SYMBOL(fetch_kernel_stack_address);
263	268
264		−static __kprobes void fetch_user_stack_address(struct pt_regs *regs,
265		− void dummy, void dest)
	269	+static void fetch_user_stack_address(struct pt_regs regs, void dummy, void *dest)
266	270	{
267	271	(unsigned long )dest = user_stack_pointer(regs);
268	272	}
	273	+NOKPROBE_SYMBOL(fetch_user_stack_address);
269	274
270	275	static fetch_func_t get_fetch_size_function(const struct fetch_type *type,
271	276	fetch_func_t orig_fn,

kernel/trace/trace_probe.h+7−8

@@ -81,13 +81,13 @@ · 80 unmodified lines
81	81	*/
82	82	#define convert_rloc_to_loc(dl, offs) ((u32)(dl) + (offs))
83	83
84		−static inline void get_rloc_data(u32 dl)
	84	+static nokprobe_inline void get_rloc_data(u32 dl)
85	85	{
86	86	return (u8 )dl + get_rloc_offs(dl);
87	87	}
88	88
89	89	/* For data_loc conversion */
90		−static inline void get_loc_data(u32 dl, void *ent)
	90	+static nokprobe_inline void get_loc_data(u32 dl, void *ent)
91	91	{
92	92	return (u8 )ent + get_rloc_offs(dl);
93	93	}
@@ -136,9 +136,8 @@ typedef u32 string_size; · 42 unmodified lines
136	136
137	137	/* Printing in basic type function template */
138	138	#define DECLARE_BASIC_PRINT_TYPE_FUNC(type) \
139		−__kprobes int PRINT_TYPE_FUNC_NAME(type)(struct trace_seq *s, \
140		− const char *name, \
141		− void data, void ent); \
	139	+int PRINT_TYPE_FUNC_NAME(type)(struct trace_seq s, const char name, \
	140	+ void data, void ent); \
142	141	extern const char PRINT_TYPE_FMT_NAME(type)[]
143	142
144	143	DECLARE_BASIC_PRINT_TYPE_FUNC(u8);
@@ -303,7 +302,7 @@ static inline bool trace_probe_is_registered(struct trace_probe *tp) · 158 unmodified lines
303	302	return !!(tp->flags & TP_FLAG_REGISTERED);
304	303	}
305	304
306		−static inline __kprobes void call_fetch(struct fetch_param *fprm,
	305	+static nokprobe_inline void call_fetch(struct fetch_param *fprm,
307	306	struct pt_regs regs, void dest)
308	307	{
309	308	return fprm->fn(regs, fprm->data, dest);
@@ -351,7 +350,7 @@ extern ssize_t traceprobe_probes_write(struct file *file, · 41 unmodified lines
351	350	extern int traceprobe_command(const char buf, int (createfn)(int, char**));
352	351
353	352	/* Sum up total data length for dynamic arraies (strings) */
354		−static inline __kprobes int
	353	+static nokprobe_inline int
355	354	__get_data_size(struct trace_probe tp, struct pt_regs regs)
356	355	{
357	356	int i, ret = 0;
@@ -367,7 +366,7 @@ __get_data_size(struct trace_probe tp, struct pt_regs regs) · 9 unmodified lines
367	366	}
368	367
369	368	/* Store the value of each argument */
370		−static inline __kprobes void
	369	+static nokprobe_inline void
371	370	store_trace_args(int ent_size, struct trace_probe tp, struct pt_regs regs,
372	371	u8 *data, int maxlen)
373	372	{

kernel/trace/trace_uprobe.c+35−31

@@ -108,8 +108,8 @@ static unsigned long get_user_stack_nth(struct pt_regs *regs, unsigned int n) · 107 unmodified lines
108	108	* Uprobes-specific fetch functions
109	109	*/
110	110	#define DEFINE_FETCH_stack(type) \
111		−static __kprobes void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs,\
112		− void offset, void dest) \
	111	+static void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs, \
	112	+ void offset, void dest) \
113	113	{ \
114	114	(type )dest = (type)get_user_stack_nth(regs, \
115	115	((unsigned long)offset)); \
@@ -120,8 +120,8 @@ DEFINE_BASIC_FETCH_FUNCS(stack) · 4 unmodified lines
120	120	#define fetch_stack_string_size NULL
121	121
122	122	#define DEFINE_FETCH_memory(type) \
123		−static __kprobes void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs,\
124		− void addr, void dest) \
	123	+static void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs, \
	124	+ void addr, void dest) \
125	125	{ \
126	126	type retval; \
127	127	void __user vaddr = (void __force __user ) addr; \
@@ -136,8 +136,8 @@ DEFINE_BASIC_FETCH_FUNCS(memory) · 8 unmodified lines
136	136	* Fetch a null-terminated string. Caller MUST set (u32 )dest with max
137	137	* length and relative data location.
138	138	*/
139		−static __kprobes void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs,
140		− void addr, void dest)
	139	+static void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs,
	140	+ void addr, void dest)
141	141	{
142	142	long ret;
143	143	u32 rloc = (u32 )dest;
@@ -158,8 +158,8 @@ static __kprobes void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs, · 14 unmodified lines
158	158	}
159	159	}
160	160
161		−static __kprobes void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs,
162		− void addr, void dest)
	161	+static void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs,
	162	+ void addr, void dest)
163	163	{
164	164	int len;
165	165	void __user vaddr = (void __force __user ) addr;
@@ -184,8 +184,8 @@ static unsigned long translate_user_vaddr(void *file_offset) · 18 unmodified lines
184	184	}
185	185
186	186	#define DEFINE_FETCH_file_offset(type) \
187		−static __kprobes void FETCH_FUNC_NAME(file_offset, type)(struct pt_regs *regs,\
188		− void offset, void dest) \
	187	+static void FETCH_FUNC_NAME(file_offset, type)(struct pt_regs *regs, \
	188	+ void offset, void dest)\
189	189	{ \
190	190	void vaddr = (void )translate_user_vaddr(offset); \
191	191	\
@@ -1009,56 +1009,60 @@ uprobe_filter_event(struct trace_uprobe tu, struct perf_event event) · 817 unmodified lines
1009	1009	return __uprobe_perf_filter(&tu->filter, event->hw.tp_target->mm);
1010	1010	}
1011	1011
1012		−static int uprobe_perf_open(struct trace_uprobe tu, struct perf_event event)
	1012	+static int uprobe_perf_close(struct trace_uprobe tu, struct perf_event event)
1013	1013	{
1014	1014	bool done;
1015	1015
1016	1016	write_lock(&tu->filter.rwlock);
1017	1017	if (event->hw.tp_target) {
1018		− /*
1019		− * event->parent != NULL means copy_process(), we can avoid
1020		− * uprobe_apply(). current->mm must be probed and we can rely
1021		− * on dup_mmap() which preserves the already installed bp's.
1022		− *
1023		− * attr.enable_on_exec means that exec/mmap will install the
1024		− * breakpoints we need.
1025		− */
	1018	+ list_del(&event->hw.tp_list);
1026	1019	done = tu->filter.nr_systemwide \|\|
1027		− event->parent \|\| event->attr.enable_on_exec \|\|
	1020	+ (event->hw.tp_target->flags & PF_EXITING) \|\|
1028	1021	uprobe_filter_event(tu, event);
1029		− list_add(&event->hw.tp_list, &tu->filter.perf_events);
1030	1022	} else {
	1023	+ tu->filter.nr_systemwide--;
1031	1024	done = tu->filter.nr_systemwide;
1032		− tu->filter.nr_systemwide++;
1033	1025	}
1034	1026	write_unlock(&tu->filter.rwlock);
1035	1027
1036	1028	if (!done)
1037		− uprobe_apply(tu->inode, tu->offset, &tu->consumer, true);
	1029	+ return uprobe_apply(tu->inode, tu->offset, &tu->consumer, false);
1038	1030
1039	1031	return 0;
1040	1032	}
1041	1033
1042		−static int uprobe_perf_close(struct trace_uprobe tu, struct perf_event event)
	1034	+static int uprobe_perf_open(struct trace_uprobe tu, struct perf_event event)
1043	1035	{
1044	1036	bool done;
	1037	+ int err;
1045	1038
1046	1039	write_lock(&tu->filter.rwlock);
1047	1040	if (event->hw.tp_target) {
1048		− list_del(&event->hw.tp_list);
	1041	+ /*
	1042	+ * event->parent != NULL means copy_process(), we can avoid
	1043	+ * uprobe_apply(). current->mm must be probed and we can rely
	1044	+ * on dup_mmap() which preserves the already installed bp's.
	1045	+ *
	1046	+ * attr.enable_on_exec means that exec/mmap will install the
	1047	+ * breakpoints we need.
	1048	+ */
1049	1049	done = tu->filter.nr_systemwide \|\|
1050		− (event->hw.tp_target->flags & PF_EXITING) \|\|
	1050	+ event->parent \|\| event->attr.enable_on_exec \|\|
1051	1051	uprobe_filter_event(tu, event);
	1052	+ list_add(&event->hw.tp_list, &tu->filter.perf_events);
1052	1053	} else {
1053		− tu->filter.nr_systemwide--;
1054	1054	done = tu->filter.nr_systemwide;
	1055	+ tu->filter.nr_systemwide++;
1055	1056	}
1056	1057	write_unlock(&tu->filter.rwlock);
1057	1058
1058		− if (!done)
1059		− uprobe_apply(tu->inode, tu->offset, &tu->consumer, false);
1060		−
1061		− return 0;
	1059	+ err = 0;
	1060	+ if (!done) {
	1061	+ err = uprobe_apply(tu->inode, tu->offset, &tu->consumer, true);
	1062	+ if (err)
	1063	+ uprobe_perf_close(tu, event);
	1064	+ }
	1065	+ return err;
1062	1066	}
1063	1067
1064	1068	static bool uprobe_perf_filter(struct uprobe_consumer *uc,