Merge tag 'trace-v6.1' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace

Pull tracing updates from Steven Rostedt: "Major changes: - Changed location of tracing repo from personal git repo to: git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git - Added Masami Hiramatsu as co-maintainer - Updated MAINTAINERS file to separate out FTRACE as it is more than just TRACING. Minor changes: - Added Mark Rutland as FTRACE reviewer - Updated user_events to make it on its way to remove the BROKEN tag. The changes should now be acceptable but will run it through a cycle and hopefully we can remove the BROKEN tag next release. - Added filtering to eprobes - Added a delta time to the benchmark trace event - Have the histogram and filter callbacks called via a switch statement instead of indirect functions. This speeds it up to avoid retpolines. - Add a way to wake up ring buffer waiters waiting for the ring buffer to fill up to its watermark. - New ioctl() on the trace_pipe_raw file to wake up ring buffer waiters. - Wake up waiters when the ring buffer is disabled. A reader may block when the ring buffer is disabled, but if it was blocked when the ring buffer is disabled it should then wake up. Fixes: - Allow splice to read partially read ring buffer pages. This fixes splice never moving forward. - Fix inverted compare that made the "shortest" ring buffer wait queue actually the longest. - Fix a race in the ring buffer between resetting a page when a writer goes to another page, and the reader. - Fix ftrace accounting bug when function hooks are added at boot up before the weak functions are set to "disabled". - Fix bug that freed a user allocated snapshot buffer when enabling a tracer. - Fix possible recursive locks in osnoise tracer - Fix recursive locking direct functions - Other minor clean ups and fixes" * tag 'trace-v6.1' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace: (44 commits) ftrace: Create separate entry in MAINTAINERS for function hooks tracing: Update MAINTAINERS to reflect new tracing git repo tracing: Do not free snapshot if tracer is on cmdline ftrace: Still disable enabled records marked as disabled tracing/user_events: Move pages/locks into groups to prepare for namespaces tracing: Add Masami Hiramatsu as co-maintainer tracing: Remove unused variable 'dups' MAINTAINERS: add myself as a tracing reviewer ring-buffer: Fix race between reset page and reading page tracing/user_events: Update ABI documentation to align to bits vs bytes tracing/user_events: Use bits vs bytes for enabled status page data tracing/user_events: Use refcount instead of atomic for ref tracking tracing/user_events: Ensure user provided strings are safely formatted tracing/user_events: Use WRITE instead of READ for io vector import tracing/user_events: Use NULL for strstr checks tracing: Fix spelling mistake "preapre" -> "prepare" tracing: Wake up waiters when tracing is disabled tracing: Add ioctl() to force ring buffer waiters to wake up tracing: Wake up ring buffer waiters on closing of the file ring-buffer: Add ring_buffer_wake_waiters() ...

Linus Torvalds committed 4 years ago

Commit cdf072a2 parents dc55342 4f881a634 files · +1299 −486Browse files

Showing the changes to kernel (17 of 34 files). Show all

Unified · Split

17 files

kernel/trace/ftrace.c +27 −7
kernel/trace/kprobe_event_gen_test.c +44 −5
kernel/trace/ring_buffer.c +84 −3
kernel/trace/rv/monitors/wip/wip.c +4 −4
kernel/trace/rv/monitors/wwnr/wwnr.c +4 −4
kernel/trace/trace.c +73 −5
kernel/trace/trace.h +0 −13
kernel/trace/trace_benchmark.c +1 −1
kernel/trace/trace_benchmark.h +5 −3
kernel/trace/trace_eprobe.c +97 −10
kernel/trace/trace_events_filter.c +168 −71
kernel/trace/trace_events_hist.c +169 −77
kernel/trace/trace_events_user.c +426 −142
kernel/trace/trace_osnoise.c +2 −1
kernel/trace/trace_probe.h +2 −1
kernel/trace/tracing_map.c +2 −3
kernel/tracepoint.c +6 −8

kernel/trace/ftrace.c+27−7

 static struct ftrace_ops *
 ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, struct ftrace_ops *ops);
+static bool skip_record(struct dyn_ftrace *rec)
+{
+	/*
+	 * At boot up, weak functions are set to disable. Function tracing
+	 * can be enabled before they are, and they still need to be disabled now.
+	 * If the record is disabled, still continue if it is marked as already
+	 * enabled (this is needed to keep the accounting working).
+	 */
+	return rec->flags & FTRACE_FL_DISABLED &&
+		!(rec->flags & FTRACE_FL_ENABLED);
+}
 static bool __ftrace_hash_rec_update(struct ftrace_ops *ops,
 				     int filter_hash,
 				     bool inc)
 		int in_hash = 0;
 		int match = 0;
-		if (rec->flags & FTRACE_FL_DISABLED)
+		if (skip_record(rec))
 			continue;
 		if (all) {
 	ftrace_bug_type = FTRACE_BUG_UNKNOWN;
-	if (rec->flags & FTRACE_FL_DISABLED)
+	if (skip_record(rec))
 		return FTRACE_UPDATE_IGNORE;
 	/*
 	if (update) {
 		/* If there's no more users, clear all flags */
 		if (!ftrace_rec_count(rec))
-			rec->flags = 0;
+			rec->flags &= FTRACE_FL_DISABLED;
 		else
 			/*
 			 * Just disable the record, but keep the ops TRAMP
 	do_for_each_ftrace_rec(pg, rec) {
-		if (rec->flags & FTRACE_FL_DISABLED)
+		if (skip_record(rec))
 			continue;
 		failed = __ftrace_replace_code(rec, enable);
  * it is safe to modify the ftrace record, where it should be
  * currently calling @old_addr directly, to call @new_addr.
  *
+ * This is called with direct_mutex locked.
+ *
  * Safety checks should be made to make sure that the code at
  * @rec->ip is currently calling @old_addr. And this must
  * also update entry->direct to @new_addr.
 	unsigned long ip = rec->ip;
 	int ret;
+	lockdep_assert_held(&direct_mutex);
 	/*
 	 * The ftrace_lock was used to determine if the record
 	 * had more than one registered user to it. If it did,
 	if (ret)
 		goto out_lock;
-	ret = register_ftrace_function(&stub_ops);
+	ret = register_ftrace_function_nolock(&stub_ops);
 	if (ret) {
 		ftrace_set_filter_ip(&stub_ops, ip, 1, 0);
 		goto out_lock;
 		if (filter_hash) {
 			orig_hash = &iter->ops->func_hash->filter_hash;
-			if (iter->tr && !list_empty(&iter->tr->mod_trace))
+			if (iter->tr) {
-				iter->hash->flags |= FTRACE_HASH_FL_MOD;
+				if (list_empty(&iter->tr->mod_trace))
+					iter->hash->flags &= ~FTRACE_HASH_FL_MOD;
+				else
+					iter->hash->flags |= FTRACE_HASH_FL_MOD;
+			}
 		} else
 			orig_hash = &iter->ops->func_hash->notrace_hash;

kernel/trace/kprobe_event_gen_test.c+44−5

@@ -35,6 +35,45 @@ · 34 unmodified lines
35	static struct trace_event_file *gen_kprobe_test;	35	static struct trace_event_file *gen_kprobe_test;
36	static struct trace_event_file *gen_kretprobe_test;	36	static struct trace_event_file *gen_kretprobe_test;
37		37
		38	#define KPROBE_GEN_TEST_FUNC "do_sys_open"
		39
		40	/* X86 */
		41	#if defined(CONFIG_X86_64) \|\| defined(CONFIG_X86_32)
		42	#define KPROBE_GEN_TEST_ARG0 "dfd=%ax"
		43	#define KPROBE_GEN_TEST_ARG1 "filename=%dx"
		44	#define KPROBE_GEN_TEST_ARG2 "flags=%cx"
		45	#define KPROBE_GEN_TEST_ARG3 "mode=+4($stack)"
		46
		47	/* ARM64 */
		48	#elif defined(CONFIG_ARM64)
		49	#define KPROBE_GEN_TEST_ARG0 "dfd=%x0"
		50	#define KPROBE_GEN_TEST_ARG1 "filename=%x1"
		51	#define KPROBE_GEN_TEST_ARG2 "flags=%x2"
		52	#define KPROBE_GEN_TEST_ARG3 "mode=%x3"
		53
		54	/* ARM */
		55	#elif defined(CONFIG_ARM)
		56	#define KPROBE_GEN_TEST_ARG0 "dfd=%r0"
		57	#define KPROBE_GEN_TEST_ARG1 "filename=%r1"
		58	#define KPROBE_GEN_TEST_ARG2 "flags=%r2"
		59	#define KPROBE_GEN_TEST_ARG3 "mode=%r3"
		60
		61	/* RISCV */
		62	#elif defined(CONFIG_RISCV)
		63	#define KPROBE_GEN_TEST_ARG0 "dfd=%a0"
		64	#define KPROBE_GEN_TEST_ARG1 "filename=%a1"
		65	#define KPROBE_GEN_TEST_ARG2 "flags=%a2"
		66	#define KPROBE_GEN_TEST_ARG3 "mode=%a3"
		67
		68	/* others */
		69	#else
		70	#define KPROBE_GEN_TEST_ARG0 NULL
		71	#define KPROBE_GEN_TEST_ARG1 NULL
		72	#define KPROBE_GEN_TEST_ARG2 NULL
		73	#define KPROBE_GEN_TEST_ARG3 NULL
		74	#endif
		75
		76
38	/*	77	/*
39	* Test to make sure we can create a kprobe event, then add more	78	* Test to make sure we can create a kprobe event, then add more
40	* fields.	79	* fields.
@@ -58,14 +97,14 @@ static int __init test_gen_kprobe_cmd(void) · 17 unmodified lines
58	* fields.	97	* fields.
59	*/	98	*/
60	ret = kprobe_event_gen_cmd_start(&cmd, "gen_kprobe_test",	99	ret = kprobe_event_gen_cmd_start(&cmd, "gen_kprobe_test",
61	"do_sys_open",	100	KPROBE_GEN_TEST_FUNC,
62	"dfd=%ax", "filename=%dx");	101	KPROBE_GEN_TEST_ARG0, KPROBE_GEN_TEST_ARG1);
63	if (ret)	102	if (ret)
64	goto free;	103	goto free;
65		104
66	/* Use kprobe_event_add_fields to add the rest of the fields */	105	/* Use kprobe_event_add_fields to add the rest of the fields */
67		106
68	ret = kprobe_event_add_fields(&cmd, "flags=%cx", "mode=+4($stack)");	107	ret = kprobe_event_add_fields(&cmd, KPROBE_GEN_TEST_ARG2, KPROBE_GEN_TEST_ARG3);
69	if (ret)	108	if (ret)
70	goto free;	109	goto free;
71		110
@@ -128,7 +167,7 @@ static int __init test_gen_kretprobe_cmd(void) · 56 unmodified lines
128	* Define the kretprobe event.	167	* Define the kretprobe event.
129	*/	168	*/
130	ret = kretprobe_event_gen_cmd_start(&cmd, "gen_kretprobe_test",	169	ret = kretprobe_event_gen_cmd_start(&cmd, "gen_kretprobe_test",
131	"do_sys_open",	170	KPROBE_GEN_TEST_FUNC,
132	"$retval");	171	"$retval");
133	if (ret)	172	if (ret)
134	goto free;	173	goto free;
@@ -206,7 +245,7 @@ static void __exit kprobe_event_gen_test_exit(void) · 71 unmodified lines
206	WARN_ON(kprobe_event_delete("gen_kprobe_test"));	245	WARN_ON(kprobe_event_delete("gen_kprobe_test"));
207		246
208	/* Disable the event or you can't remove it */	247	/* Disable the event or you can't remove it */
209	WARN_ON(trace_array_set_clr_event(gen_kprobe_test->tr,	248	WARN_ON(trace_array_set_clr_event(gen_kretprobe_test->tr,
210	"kprobes",	249	"kprobes",
211	"gen_kretprobe_test", false));	250	"gen_kretprobe_test", false));
212		251

kernel/trace/ring_buffer.c+84−3

@@ -413,6 +413,7 @@ struct rb_irq_work { · 412 unmodified lines
413	struct irq_work work;	413	struct irq_work work;
414	wait_queue_head_t waiters;	414	wait_queue_head_t waiters;
415	wait_queue_head_t full_waiters;	415	wait_queue_head_t full_waiters;
		416	long wait_index;
416	bool waiters_pending;	417	bool waiters_pending;
417	bool full_waiters_pending;	418	bool full_waiters_pending;
418	bool wakeup_full;	419	bool wakeup_full;
@@ -917,12 +918,44 @@ static void rb_wake_up_waiters(struct irq_work *work) · 498 unmodified lines
917	struct rb_irq_work *rbwork = container_of(work, struct rb_irq_work, work);	918	struct rb_irq_work *rbwork = container_of(work, struct rb_irq_work, work);
918		919
919	wake_up_all(&rbwork->waiters);	920	wake_up_all(&rbwork->waiters);
920	if (rbwork->wakeup_full) {	921	if (rbwork->full_waiters_pending \|\| rbwork->wakeup_full) {
921	rbwork->wakeup_full = false;	922	rbwork->wakeup_full = false;
		923	rbwork->full_waiters_pending = false;
922	wake_up_all(&rbwork->full_waiters);	924	wake_up_all(&rbwork->full_waiters);
923	}	925	}
924	}	926	}
925		927
		928	/**
		929	* ring_buffer_wake_waiters - wake up any waiters on this ring buffer
		930	* @buffer: The ring buffer to wake waiters on
		931	*
		932	* In the case of a file that represents a ring buffer is closing,
		933	* it is prudent to wake up any waiters that are on this.
		934	*/
		935	void ring_buffer_wake_waiters(struct trace_buffer *buffer, int cpu)
		936	{
		937	struct ring_buffer_per_cpu *cpu_buffer;
		938	struct rb_irq_work *rbwork;
		939
		940	if (cpu == RING_BUFFER_ALL_CPUS) {
		941
		942	/* Wake up individual ones too. One level recursion */
		943	for_each_buffer_cpu(buffer, cpu)
		944	ring_buffer_wake_waiters(buffer, cpu);
		945
		946	rbwork = &buffer->irq_work;
		947	} else {
		948	cpu_buffer = buffer->buffers[cpu];
		949	rbwork = &cpu_buffer->irq_work;
		950	}
		951
		952	rbwork->wait_index++;
		953	/* make sure the waiters see the new index */
		954	smp_wmb();
		955
		956	rb_wake_up_waiters(&rbwork->work);
		957	}
		958
926	/**	959	/**
927	* ring_buffer_wait - wait for input to the ring buffer	960	* ring_buffer_wait - wait for input to the ring buffer
928	* @buffer: buffer to wait on	961	* @buffer: buffer to wait on
@@ -938,6 +971,7 @@ int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full) · 9 unmodified lines
938	struct ring_buffer_per_cpu *cpu_buffer;	971	struct ring_buffer_per_cpu *cpu_buffer;
939	DEFINE_WAIT(wait);	972	DEFINE_WAIT(wait);
940	struct rb_irq_work *work;	973	struct rb_irq_work *work;
		974	long wait_index;
941	int ret = 0;	975	int ret = 0;
942		976
943	/*	977	/*
@@ -956,6 +990,7 @@ int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full) · 12 unmodified lines
956	work = &cpu_buffer->irq_work;	990	work = &cpu_buffer->irq_work;
957	}	991	}
958		992
		993	wait_index = READ_ONCE(work->wait_index);
959		994
960	while (true) {	995	while (true) {
961	if (full)	996	if (full)
@@ -1011,7 +1046,7 @@ int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full) · 49 unmodified lines
1011	nr_pages = cpu_buffer->nr_pages;	1046	nr_pages = cpu_buffer->nr_pages;
1012	dirty = ring_buffer_nr_dirty_pages(buffer, cpu);	1047	dirty = ring_buffer_nr_dirty_pages(buffer, cpu);
1013	if (!cpu_buffer->shortest_full \|\|	1048	if (!cpu_buffer->shortest_full \|\|
1014	cpu_buffer->shortest_full < full)	1049	cpu_buffer->shortest_full > full)
1015	cpu_buffer->shortest_full = full;	1050	cpu_buffer->shortest_full = full;
1016	raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);	1051	raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
1017	if (!pagebusy &&	1052	if (!pagebusy &&
@@ -1020,6 +1055,11 @@ int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full) · 2 unmodified lines
1020	}	1055	}
1021		1056
1022	schedule();	1057	schedule();
		1058
		1059	/* Make sure to see the new wait index */
		1060	smp_rmb();
		1061	if (wait_index != work->wait_index)
		1062	break;
1023	}	1063	}
1024		1064
1025	if (full)	1065	if (full)
@@ -2608,6 +2648,9 @@ rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer, · 1582 unmodified lines
2608	/* Mark the rest of the page with padding */	2648	/* Mark the rest of the page with padding */
2609	rb_event_set_padding(event);	2649	rb_event_set_padding(event);
2610		2650
		2651	/* Make sure the padding is visible before the write update */
		2652	smp_wmb();
		2653
2611	/* Set the write back to the previous setting */	2654	/* Set the write back to the previous setting */
2612	local_sub(length, &tail_page->write);	2655	local_sub(length, &tail_page->write);
2613	return;	2656	return;
@@ -2619,6 +2662,9 @@ rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer, · 5 unmodified lines
2619	/* time delta must be non zero */	2662	/* time delta must be non zero */
2620	event->time_delta = 1;	2663	event->time_delta = 1;
2621		2664
		2665	/* Make sure the padding is visible before the tail_page->write update */
		2666	smp_wmb();
		2667
2622	/* Set write to end of buffer */	2668	/* Set write to end of buffer */
2623	length = (tail + length) - BUF_PAGE_SIZE;	2669	length = (tail + length) - BUF_PAGE_SIZE;
2624	local_sub(length, &tail_page->write);	2670	local_sub(length, &tail_page->write);
@@ -4587,6 +4633,33 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) · 1962 unmodified lines
4587	arch_spin_unlock(&cpu_buffer->lock);	4633	arch_spin_unlock(&cpu_buffer->lock);
4588	local_irq_restore(flags);	4634	local_irq_restore(flags);
4589		4635
		4636	/*
		4637	* The writer has preempt disable, wait for it. But not forever
		4638	* Although, 1 second is pretty much "forever"
		4639	*/
		4640	#define USECS_WAIT 1000000
		4641	for (nr_loops = 0; nr_loops < USECS_WAIT; nr_loops++) {
		4642	/* If the write is past the end of page, a writer is still updating it */
		4643	if (likely(!reader \|\| rb_page_write(reader) <= BUF_PAGE_SIZE))
		4644	break;
		4645
		4646	udelay(1);
		4647
		4648	/* Get the latest version of the reader write value */
		4649	smp_rmb();
		4650	}
		4651
		4652	/* The writer is not moving forward? Something is wrong */
		4653	if (RB_WARN_ON(cpu_buffer, nr_loops == USECS_WAIT))
		4654	reader = NULL;
		4655
		4656	/*
		4657	* Make sure we see any padding after the write update
		4658	* (see rb_reset_tail())
		4659	*/
		4660	smp_rmb();
		4661
		4662
4590	return reader;	4663	return reader;
4591	}	4664	}
4592		4665
@@ -5616,7 +5689,15 @@ int ring_buffer_read_page(struct trace_buffer *buffer, · 1023 unmodified lines
5616	unsigned int pos = 0;	5689	unsigned int pos = 0;
5617	unsigned int size;	5690	unsigned int size;
5618		5691
5619	if (full)	5692	/*
		5693	* If a full page is expected, this can still be returned
		5694	* if there's been a previous partial read and the
		5695	* rest of the page can be read and the commit page is off
		5696	* the reader page.
		5697	*/
		5698	if (full &&
		5699	(!read \|\| (len < (commit - read)) \|\|
		5700	cpu_buffer->reader_page == cpu_buffer->commit_page))
5620	goto out_unlock;	5701	goto out_unlock;
5621		5702
5622	if (len > (commit - read))	5703	if (len > (commit - read))

kernel/trace/rv/monitors/wip/wip.c+4−4

@@ -16,7 +16,7 @@ · 15 unmodified lines
16		16
17	#include "wip.h"	17	#include "wip.h"
18		18
19	struct rv_monitor rv_wip;	19	static struct rv_monitor rv_wip;
20	DECLARE_DA_MON_PER_CPU(wip, unsigned char);	20	DECLARE_DA_MON_PER_CPU(wip, unsigned char);
21		21
22	static void handle_preempt_disable(void *data, unsigned long ip, unsigned long parent_ip)	22	static void handle_preempt_disable(void *data, unsigned long ip, unsigned long parent_ip)
@@ -60,7 +60,7 @@ static void disable_wip(void) · 37 unmodified lines
60	da_monitor_destroy_wip();	60	da_monitor_destroy_wip();
61	}	61	}
62		62
63	struct rv_monitor rv_wip = {	63	static struct rv_monitor rv_wip = {
64	.name = "wip",	64	.name = "wip",
65	.description = "wakeup in preemptive per-cpu testing monitor.",	65	.description = "wakeup in preemptive per-cpu testing monitor.",
66	.enable = enable_wip,	66	.enable = enable_wip,
@@ -69,13 +69,13 @@ struct rv_monitor rv_wip = { · 2 unmodified lines
69	.enabled = 0,	69	.enabled = 0,
70	};	70	};
71		71
72	static int register_wip(void)	72	static int __init register_wip(void)
73	{	73	{
74	rv_register_monitor(&rv_wip);	74	rv_register_monitor(&rv_wip);
75	return 0;	75	return 0;
76	}	76	}
77		77
78	static void unregister_wip(void)	78	static void __exit unregister_wip(void)
79	{	79	{
80	rv_unregister_monitor(&rv_wip);	80	rv_unregister_monitor(&rv_wip);
81	}	81	}

kernel/trace/rv/monitors/wwnr/wwnr.c+4−4

@@ -15,7 +15,7 @@ · 14 unmodified lines
15		15
16	#include "wwnr.h"	16	#include "wwnr.h"
17		17
18	struct rv_monitor rv_wwnr;	18	static struct rv_monitor rv_wwnr;
19	DECLARE_DA_MON_PER_TASK(wwnr, unsigned char);	19	DECLARE_DA_MON_PER_TASK(wwnr, unsigned char);
20		20
21	static void handle_switch(void data, bool preempt, struct task_struct p,	21	static void handle_switch(void data, bool preempt, struct task_struct p,
@@ -59,7 +59,7 @@ static void disable_wwnr(void) · 37 unmodified lines
59	da_monitor_destroy_wwnr();	59	da_monitor_destroy_wwnr();
60	}	60	}
61		61
62	struct rv_monitor rv_wwnr = {	62	static struct rv_monitor rv_wwnr = {
63	.name = "wwnr",	63	.name = "wwnr",
64	.description = "wakeup while not running per-task testing model.",	64	.description = "wakeup while not running per-task testing model.",
65	.enable = enable_wwnr,	65	.enable = enable_wwnr,
@@ -68,13 +68,13 @@ struct rv_monitor rv_wwnr = { · 2 unmodified lines
68	.enabled = 0,	68	.enabled = 0,
69	};	69	};
70		70
71	static int register_wwnr(void)	71	static int __init register_wwnr(void)
72	{	72	{
73	rv_register_monitor(&rv_wwnr);	73	rv_register_monitor(&rv_wwnr);
74	return 0;	74	return 0;
75	}	75	}
76		76
77	static void unregister_wwnr(void)	77	static void __exit unregister_wwnr(void)
78	{	78	{
79	rv_unregister_monitor(&rv_wwnr);	79	rv_unregister_monitor(&rv_wwnr);
80	}	80	}

kernel/trace/trace.c+73−5

 {
 	void *cond_data = NULL;
+	local_irq_disable();
 	arch_spin_lock(&tr->max_lock);
 	if (tr->cond_snapshot)
 		cond_data = tr->cond_snapshot->cond_data;
 	arch_spin_unlock(&tr->max_lock);
+	local_irq_enable();
 	return cond_data;
 }
 		goto fail_unlock;
 	}
+	local_irq_disable();
 	arch_spin_lock(&tr->max_lock);
 	tr->cond_snapshot = cond_snapshot;
 	arch_spin_unlock(&tr->max_lock);
+	local_irq_enable();
 	mutex_unlock(&trace_types_lock);
 {
 	int ret = 0;
+	local_irq_disable();
 	arch_spin_lock(&tr->max_lock);
 	if (!tr->cond_snapshot)
 	}
 	arch_spin_unlock(&tr->max_lock);
+	local_irq_enable();
 	return ret;
 }
 #define SAVED_CMDLINES_DEFAULT 128
 #define NO_CMDLINE_MAP UINT_MAX
+/*
+ * Preemption must be disabled before acquiring trace_cmdline_lock.
+ * The various trace_arrays' max_lock must be acquired in a context
+ * where interrupt is disabled.
+ */
 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
 struct saved_cmdlines_buffer {
 	unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
 	 * the lock, but we also don't want to spin
 	 * nor do we want to disable interrupts,
 	 * so if we miss here, then better luck next time.
+	 *
+	 * This is called within the scheduler and wake up, so interrupts
+	 * had better been disabled and run queue lock been held.
 	 */
+	lockdep_assert_preemption_disabled();
 	if (!arch_spin_trylock(&trace_cmdline_lock))
 		return 0;
 	char buf[64];
 	int r;
+	preempt_disable();
 	arch_spin_lock(&trace_cmdline_lock);
 	r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
 	arch_spin_unlock(&trace_cmdline_lock);
+	preempt_enable();
 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
 }
 		return -ENOMEM;
 	}
+	preempt_disable();
 	arch_spin_lock(&trace_cmdline_lock);
 	savedcmd_temp = savedcmd;
 	savedcmd = s;
 	arch_spin_unlock(&trace_cmdline_lock);
+	preempt_enable();
 	free_saved_cmdlines_buffer(savedcmd_temp);
 	return 0;
 #ifdef CONFIG_TRACER_SNAPSHOT
 	if (t->use_max_tr) {
+		local_irq_disable();
 		arch_spin_lock(&tr->max_lock);
 		if (tr->cond_snapshot)
 			ret = -EBUSY;
 		arch_spin_unlock(&tr->max_lock);
+		local_irq_enable();
 		if (ret)
 			goto out;
 	}
 	if (tr->current_trace->reset)
 		tr->current_trace->reset(tr);
+#ifdef CONFIG_TRACER_MAX_TRACE
+	had_max_tr = tr->current_trace->use_max_tr;
 	/* Current trace needs to be nop_trace before synchronize_rcu */
 	tr->current_trace = &nop_trace;
-#ifdef CONFIG_TRACER_MAX_TRACE
-	had_max_tr = tr->allocated_snapshot;
 	if (had_max_tr && !t->use_max_tr) {
 		/*
 		 * We need to make sure that the update_max_tr sees that
 		free_snapshot(tr);
 	}
-	if (t->use_max_tr && !had_max_tr) {
+	if (t->use_max_tr && !tr->allocated_snapshot) {
 		ret = tracing_alloc_snapshot_instance(tr);
 		if (ret < 0)
 			goto out;
 	}
+#else
+	tr->current_trace = &nop_trace;
 #endif
 	if (t->init) {
 		goto out;
 	}
+	local_irq_disable();
 	arch_spin_lock(&tr->max_lock);
 	if (tr->cond_snapshot)
 		ret = -EBUSY;
 	arch_spin_unlock(&tr->max_lock);
+	local_irq_enable();
 	if (ret)
 		goto out;
 	__trace_array_put(iter->tr);
+	iter->wait_index++;
+	/* Make sure the waiters see the new wait_index */
+	smp_wmb();
+	ring_buffer_wake_waiters(iter->array_buffer->buffer, iter->cpu_file);
 	if (info->spare)
 		ring_buffer_free_read_page(iter->array_buffer->buffer,
 					   info->spare_cpu, info->spare);
 	/* did we read anything? */
 	if (!spd.nr_pages) {
+		long wait_index;
 		if (ret)
 			goto out;
 		if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
 			goto out;
+		wait_index = READ_ONCE(iter->wait_index);
 		ret = wait_on_pipe(iter, iter->tr->buffer_percent);
 		if (ret)
 			goto out;
+		/* No need to wait after waking up when tracing is off */
+		if (!tracer_tracing_is_on(iter->tr))
+			goto out;
+		/* Make sure we see the new wait_index */
+		smp_rmb();
+		if (wait_index != iter->wait_index)
+			goto out;
 		goto again;
 	}
 	return ret;
 }
+/* An ioctl call with cmd 0 to the ring buffer file will wake up all waiters */
+static long tracing_buffers_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+{
+	struct ftrace_buffer_info *info = file->private_data;
+	struct trace_iterator *iter = &info->iter;
+	if (cmd)
+		return -ENOIOCTLCMD;
+	mutex_lock(&trace_types_lock);
+	iter->wait_index++;
+	/* Make sure the waiters see the new wait_index */
+	smp_wmb();
+	ring_buffer_wake_waiters(iter->array_buffer->buffer, iter->cpu_file);
+	mutex_unlock(&trace_types_lock);
+	return 0;
+}
 static const struct file_operations tracing_buffers_fops = {
 	.open		= tracing_buffers_open,
 	.read		= tracing_buffers_read,
 	.poll		= tracing_buffers_poll,
 	.release	= tracing_buffers_release,
 	.splice_read	= tracing_buffers_splice_read,
+	.unlocked_ioctl = tracing_buffers_ioctl,
 	.llseek		= no_llseek,
 };
 			tracer_tracing_off(tr);
 			if (tr->current_trace->stop)
 				tr->current_trace->stop(tr);
+			/* Wake up any waiters */
+			ring_buffer_wake_waiters(buffer, RING_BUFFER_ALL_CPUS);
 		}
 		mutex_unlock(&trace_types_lock);
 	}
 	 * buffer. The memory will be removed once the "instance" is removed.
 	 */
 	ret = cpuhp_setup_state_multi(CPUHP_TRACE_RB_PREPARE,
-				      "trace/RB:preapre", trace_rb_cpu_prepare,
+				      "trace/RB:prepare", trace_rb_cpu_prepare,
 				      NULL);
 	if (ret < 0)
 		goto out_free_cpumask;

kernel/trace/trace.h+0−13

@@ -1435,8 +1435,6 @@ event_trigger_unlock_commit(struct trace_event_file *file, · 1434 unmodified lines
1435	struct filter_pred;	1435	struct filter_pred;
1436	struct regex;	1436	struct regex;
1437		1437
1438	typedef int (filter_pred_fn_t) (struct filter_pred pred, void *event);
1439
1440	typedef int (regex_match_func)(char str, struct regex *r, int len);	1438	typedef int (regex_match_func)(char str, struct regex *r, int len);
1441		1439
1442	enum regex_type {	1440	enum regex_type {
@@ -1455,17 +1453,6 @@ struct regex { · 12 unmodified lines
1455	regex_match_func match;	1453	regex_match_func match;
1456	};	1454	};
1457		1455
1458	struct filter_pred {
1459	filter_pred_fn_t fn;
1460	u64 val;
1461	struct regex regex;
1462	unsigned short *ops;
1463	struct ftrace_event_field *field;
1464	int offset;
1465	int not;
1466	int op;
1467	};
1468
1469	static inline bool is_string_field(struct ftrace_event_field *field)	1456	static inline bool is_string_field(struct ftrace_event_field *field)
1470	{	1457	{
1471	return field->filter_type == FILTER_DYN_STRING \|\|	1458	return field->filter_type == FILTER_DYN_STRING \|\|

kernel/trace/trace_benchmark.c+1−1

@@ -51,7 +51,7 @@ static void trace_do_benchmark(void) · 50 unmodified lines
51		51
52	local_irq_disable();	52	local_irq_disable();
53	start = trace_clock_local();	53	start = trace_clock_local();
54	trace_benchmark_event(bm_str);	54	trace_benchmark_event(bm_str, bm_last);
55	stop = trace_clock_local();	55	stop = trace_clock_local();
56	local_irq_enable();	56	local_irq_enable();
57		57

kernel/trace/trace_benchmark.h+5−3

 TRACE_EVENT_FN(benchmark_event,
-	TP_PROTO(const char *str),
+	TP_PROTO(const char *str, u64 delta),
-	TP_ARGS(str),
+	TP_ARGS(str, delta),
 	TP_STRUCT__entry(
 		__array(	char,	str,	BENCHMARK_EVENT_STRLEN	)
+		__field(	u64,	delta)
 	),
 	TP_fast_assign(
 		memcpy(__entry->str, str, BENCHMARK_EVENT_STRLEN);
+		__entry->delta = delta;
 	),
-	TP_printk("%s", __entry->str),
+	TP_printk("%s delta=%llu", __entry->str, __entry->delta),
 	trace_benchmark_reg, trace_benchmark_unreg
 );

kernel/trace/trace_eprobe.c+97−10

@@ -26,6 +26,9 @@ struct trace_eprobe { · 25 unmodified lines
26	/* tracepoint event */	26	/* tracepoint event */
27	const char *event_name;	27	const char *event_name;
28		28
		29	/* filter string for the tracepoint */
		30	char *filter_str;
		31
29	struct trace_event_call *event;	32	struct trace_event_call *event;
30		33
31	struct dyn_event devent;	34	struct dyn_event devent;
@@ -664,14 +667,15 @@ static struct event_trigger_data * · 632 unmodified lines
664	new_eprobe_trigger(struct trace_eprobe ep, struct trace_event_file file)	667	new_eprobe_trigger(struct trace_eprobe ep, struct trace_event_file file)
665	{	668	{
666	struct event_trigger_data *trigger;	669	struct event_trigger_data *trigger;
		670	struct event_filter *filter = NULL;
667	struct eprobe_data *edata;	671	struct eprobe_data *edata;
		672	int ret;
668		673
669	edata = kzalloc(sizeof(*edata), GFP_KERNEL);	674	edata = kzalloc(sizeof(*edata), GFP_KERNEL);
670	trigger = kzalloc(sizeof(*trigger), GFP_KERNEL);	675	trigger = kzalloc(sizeof(*trigger), GFP_KERNEL);
671	if (!trigger \|\| !edata) {	676	if (!trigger \|\| !edata) {
672	kfree(edata);	677	ret = -ENOMEM;
673	kfree(trigger);	678	goto error;
674	return ERR_PTR(-ENOMEM);
675	}	679	}
676		680
677	trigger->flags = EVENT_TRIGGER_FL_PROBE;	681	trigger->flags = EVENT_TRIGGER_FL_PROBE;
@@ -686,13 +690,25 @@ new_eprobe_trigger(struct trace_eprobe ep, struct trace_event_file file) · 8 unmodified lines
686	trigger->cmd_ops = &event_trigger_cmd;	690	trigger->cmd_ops = &event_trigger_cmd;
687		691
688	INIT_LIST_HEAD(&trigger->list);	692	INIT_LIST_HEAD(&trigger->list);
689	RCU_INIT_POINTER(trigger->filter, NULL);	693
		694	if (ep->filter_str) {
		695	ret = create_event_filter(file->tr, file->event_call,
		696	ep->filter_str, false, &filter);
		697	if (ret)
		698	goto error;
		699	}
		700	RCU_INIT_POINTER(trigger->filter, filter);
690		701
691	edata->file = file;	702	edata->file = file;
692	edata->ep = ep;	703	edata->ep = ep;
693	trigger->private_data = edata;	704	trigger->private_data = edata;
694		705
695	return trigger;	706	return trigger;
		707	error:
		708	free_event_filter(filter);
		709	kfree(edata);
		710	kfree(trigger);
		711	return ERR_PTR(ret);
696	}	712	}
697		713
698	static int enable_eprobe(struct trace_eprobe *ep,	714	static int enable_eprobe(struct trace_eprobe *ep,
@@ -726,6 +742,7 @@ static int disable_eprobe(struct trace_eprobe *ep, · 27 unmodified lines
726	{	742	{
727	struct event_trigger_data trigger = NULL, iter;	743	struct event_trigger_data trigger = NULL, iter;
728	struct trace_event_file *file;	744	struct trace_event_file *file;
		745	struct event_filter *filter;
729	struct eprobe_data *edata;	746	struct eprobe_data *edata;
730		747
731	file = find_event_file(tr, ep->event_system, ep->event_name);	748	file = find_event_file(tr, ep->event_system, ep->event_name);
@@ -752,6 +769,10 @@ static int disable_eprobe(struct trace_eprobe *ep, · 20 unmodified lines
752	/* Make sure nothing is using the edata or trigger */	769	/* Make sure nothing is using the edata or trigger */
753	tracepoint_synchronize_unregister();	770	tracepoint_synchronize_unregister();
754		771
		772	filter = rcu_access_pointer(trigger->filter);
		773
		774	if (filter)
		775	free_event_filter(filter);
755	kfree(edata);	776	kfree(edata);
756	kfree(trigger);	777	kfree(trigger);
757		778
@@ -927,12 +948,62 @@ static int trace_eprobe_tp_update_arg(struct trace_eprobe ep, const char argv[ · 169 unmodified lines
927	return ret;	948	return ret;
928	}	949	}
929		950
		951	static int trace_eprobe_parse_filter(struct trace_eprobe ep, int argc, const char argv[])
		952	{
		953	struct event_filter *dummy;
		954	int i, ret, len = 0;
		955	char *p;
		956
		957	if (argc == 0) {
		958	trace_probe_log_err(0, NO_EP_FILTER);
		959	return -EINVAL;
		960	}
		961
		962	/* Recover the filter string */
		963	for (i = 0; i < argc; i++)
		964	len += strlen(argv[i]) + 1;
		965
		966	ep->filter_str = kzalloc(len, GFP_KERNEL);
		967	if (!ep->filter_str)
		968	return -ENOMEM;
		969
		970	p = ep->filter_str;
		971	for (i = 0; i < argc; i++) {
		972	ret = snprintf(p, len, "%s ", argv[i]);
		973	if (ret < 0)
		974	goto error;
		975	if (ret > len) {
		976	ret = -E2BIG;
		977	goto error;
		978	}
		979	p += ret;
		980	len -= ret;
		981	}
		982	p[-1] = '\0';
		983
		984	/*
		985	* Ensure the filter string can be parsed correctly. Note, this
		986	* filter string is for the original event, not for the eprobe.
		987	*/
		988	ret = create_event_filter(top_trace_array(), ep->event, ep->filter_str,
		989	true, &dummy);
		990	free_event_filter(dummy);
		991	if (ret)
		992	goto error;
		993
		994	return 0;
		995	error:
		996	kfree(ep->filter_str);
		997	ep->filter_str = NULL;
		998	return ret;
		999	}
		1000
930	static int __trace_eprobe_create(int argc, const char *argv[])	1001	static int __trace_eprobe_create(int argc, const char *argv[])
931	{	1002	{
932	/*	1003	/*
933	* Argument syntax:	1004	* Argument syntax:
934	* e[:[GRP/][ENAME]] SYSTEM.EVENT [FETCHARGS]	1005	* e[:[GRP/][ENAME]] SYSTEM.EVENT [FETCHARGS] [if FILTER]
935	* Fetch args:	1006	* Fetch args (no space):
936	* <name>=$<field>[:TYPE]	1007	* <name>=$<field>[:TYPE]
937	*/	1008	*/
938	const char event = NULL, group = EPROBE_EVENT_SYSTEM;	1009	const char event = NULL, group = EPROBE_EVENT_SYSTEM;
@@ -942,8 +1013,8 @@ static int __trace_eprobe_create(int argc, const char *argv[]) · 3 unmodified lines
942	char buf1[MAX_EVENT_NAME_LEN];	1013	char buf1[MAX_EVENT_NAME_LEN];
943	char buf2[MAX_EVENT_NAME_LEN];	1014	char buf2[MAX_EVENT_NAME_LEN];
944	char gbuf[MAX_EVENT_NAME_LEN];	1015	char gbuf[MAX_EVENT_NAME_LEN];
945	int ret = 0;	1016	int ret = 0, filter_idx = 0;
946	int i;	1017	int i, filter_cnt;
947		1018
948	if (argc < 2 \|\| argv[0][0] != 'e')	1019	if (argc < 2 \|\| argv[0][0] != 'e')
949	return -ECANCELED;	1020	return -ECANCELED;
@@ -968,11 +1039,19 @@ static int __trace_eprobe_create(int argc, const char *argv[]) · 18 unmodified lines
968	}	1039	}
969		1040
970	if (!event) {	1041	if (!event) {
971	strscpy(buf1, argv[1], MAX_EVENT_NAME_LEN);	1042	strscpy(buf1, sys_event, MAX_EVENT_NAME_LEN);
972	sanitize_event_name(buf1);
973	event = buf1;	1043	event = buf1;
974	}	1044	}
975		1045
		1046	for (i = 2; i < argc; i++) {
		1047	if (!strcmp(argv[i], "if")) {
		1048	filter_idx = i + 1;
		1049	filter_cnt = argc - filter_idx;
		1050	argc = i;
		1051	break;
		1052	}
		1053	}
		1054
976	mutex_lock(&event_mutex);	1055	mutex_lock(&event_mutex);
977	event_call = find_and_get_event(sys_name, sys_event);	1056	event_call = find_and_get_event(sys_name, sys_event);
978	ep = alloc_event_probe(group, event, event_call, argc - 2);	1057	ep = alloc_event_probe(group, event, event_call, argc - 2);
@@ -988,6 +1067,14 @@ static int __trace_eprobe_create(int argc, const char *argv[]) · 9 unmodified lines
988	goto error;	1067	goto error;
989	}	1068	}
990		1069
		1070	if (filter_idx) {
		1071	trace_probe_log_set_index(filter_idx);
		1072	ret = trace_eprobe_parse_filter(ep, filter_cnt, argv + filter_idx);
		1073	if (ret)
		1074	goto parse_error;
		1075	} else
		1076	ep->filter_str = NULL;
		1077
991	argc -= 2; argv += 2;	1078	argc -= 2; argv += 2;
992	/* parse arguments */	1079	/* parse arguments */
993	for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) {	1080	for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) {

kernel/trace/trace_events_filter.c+168−71

 static const char * ops[] = { OPS };
+enum filter_pred_fn {
+	FILTER_PRED_FN_NOP,
+	FILTER_PRED_FN_64,
+	FILTER_PRED_FN_S64,
+	FILTER_PRED_FN_U64,
+	FILTER_PRED_FN_32,
+	FILTER_PRED_FN_S32,
+	FILTER_PRED_FN_U32,
+	FILTER_PRED_FN_16,
+	FILTER_PRED_FN_S16,
+	FILTER_PRED_FN_U16,
+	FILTER_PRED_FN_8,
+	FILTER_PRED_FN_S8,
+	FILTER_PRED_FN_U8,
+	FILTER_PRED_FN_COMM,
+	FILTER_PRED_FN_STRING,
+	FILTER_PRED_FN_STRLOC,
+	FILTER_PRED_FN_STRRELLOC,
+	FILTER_PRED_FN_PCHAR_USER,
+	FILTER_PRED_FN_PCHAR,
+	FILTER_PRED_FN_CPU,
+	FILTER_PRED_FN_,
+	FILTER_PRED_TEST_VISITED,
+};
+struct filter_pred {
+	enum filter_pred_fn 	fn_num;
+	u64 			val;
+	struct regex		regex;
+	unsigned short		*ops;
+	struct ftrace_event_field *field;
+	int 			offset;
+	int			not;
+	int 			op;
+};
 /*
  * pred functions are OP_LE, OP_LT, OP_GE, OP_GT, and OP_BAND
  * pred_funcs_##type below must match the order of them above.
 	return ERR_PTR(ret);
 }
+enum pred_cmp_types {
+	PRED_CMP_TYPE_NOP,
+	PRED_CMP_TYPE_LT,
+	PRED_CMP_TYPE_LE,
+	PRED_CMP_TYPE_GT,
+	PRED_CMP_TYPE_GE,
+	PRED_CMP_TYPE_BAND,
+};
 #define DEFINE_COMPARISON_PRED(type)					\
-static int filter_pred_LT_##type(struct filter_pred *pred, void *event)	\
+static int filter_pred_##type(struct filter_pred *pred, void *event)	\
-{									\
-	type *addr = (type *)(event + pred->offset);			\
-	type val = (type)pred->val;					\
-	return *addr < val;						\
-}									\
-static int filter_pred_LE_##type(struct filter_pred *pred, void *event)	\
-{									\
-	type *addr = (type *)(event + pred->offset);			\
-	type val = (type)pred->val;					\
-	return *addr <= val;						\
-}									\
-static int filter_pred_GT_##type(struct filter_pred *pred, void *event)	\
 {									\
-	type *addr = (type *)(event + pred->offset);			\
+	switch (pred->op) {						\
-	type val = (type)pred->val;					\
+	case OP_LT: {							\
-	return *addr > val;					\
+		type *addr = (type *)(event + pred->offset);		\
-}									\
+		type val = (type)pred->val;				\
-static int filter_pred_GE_##type(struct filter_pred *pred, void *event)	\
+		return *addr < val;					\
-{									\
+	}								\
-	type *addr = (type *)(event + pred->offset);			\
+	case OP_LE: {					\
-	type val = (type)pred->val;					\
+		type *addr = (type *)(event + pred->offset);		\
-	return *addr >= val;						\
+		type val = (type)pred->val;				\
-}									\
+		return *addr <= val;					\
-static int filter_pred_BAND_##type(struct filter_pred *pred, void *event) \
+	}								\
-{									\
+	case OP_GT: {					\
-	type *addr = (type *)(event + pred->offset);			\
+		type *addr = (type *)(event + pred->offset);		\
-	type val = (type)pred->val;					\
+		type val = (type)pred->val;				\
-	return !!(*addr & val);						\
+		return *addr > val;					\
-}									\
+	}								\
-static const filter_pred_fn_t pred_funcs_##type[] = {			\
+	case OP_GE: {					\
-	filter_pred_LE_##type,						\
+		type *addr = (type *)(event + pred->offset);		\
-	filter_pred_LT_##type,						\
+		type val = (type)pred->val;				\
-	filter_pred_GE_##type,						\
+		return *addr >= val;					\
-	filter_pred_GT_##type,						\
+	}								\
-	filter_pred_BAND_##type,					\
+	case OP_BAND: {					\
-};
+		type *addr = (type *)(event + pred->offset);		\
+		type val = (type)pred->val;				\
+		return !!(*addr & val);					\
+	}								\
+	default:							\
+		return 0;						\
+	}								\
+}
 #define DEFINE_EQUALITY_PRED(size)					\
 static int filter_pred_##size(struct filter_pred *pred, void *event)	\
 	return cmp ^ pred->not;
 }
-static int filter_pred_none(struct filter_pred *pred, void *event)
-{
-	return 0;
-}
 /*
  * regex_match_foo - Basic regex callbacks
  *
 	}
 }
+#ifdef CONFIG_FTRACE_STARTUP_TEST
+static int test_pred_visited_fn(struct filter_pred *pred, void *event);
+#else
+static int test_pred_visited_fn(struct filter_pred *pred, void *event)
+{
+	return 0;
+}
+#endif
+static int filter_pred_fn_call(struct filter_pred *pred, void *event);
 /* return 1 if event matches, 0 otherwise (discard) */
 int filter_match_preds(struct event_filter *filter, void *rec)
 {
 	for (i = 0; prog[i].pred; i++) {
 		struct filter_pred *pred = prog[i].pred;
-		int match = pred->fn(pred, rec);
+		int match = filter_pred_fn_call(pred, rec);
 		if (match == prog[i].when_to_branch)
 			i = prog[i].target;
 	}
 	return FILTER_OTHER;
 }
-static filter_pred_fn_t select_comparison_fn(enum filter_op_ids op,
+static enum filter_pred_fn select_comparison_fn(enum filter_op_ids op,
-					    int field_size, int field_is_signed)
+						int field_size, int field_is_signed)
 {
-	filter_pred_fn_t fn = NULL;
+	enum filter_pred_fn fn = FILTER_PRED_FN_NOP;
 	int pred_func_index = -1;
 	switch (op) {
 		break;
 	default:
 		if (WARN_ON_ONCE(op < PRED_FUNC_START))
-			return NULL;
+			return fn;
 		pred_func_index = op - PRED_FUNC_START;
 		if (WARN_ON_ONCE(pred_func_index > PRED_FUNC_MAX))
-			return NULL;
+			return fn;
 	}
 	switch (field_size) {
 	case 8:
 		if (pred_func_index < 0)
-			fn = filter_pred_64;
+			fn = FILTER_PRED_FN_64;
 		else if (field_is_signed)
-			fn = pred_funcs_s64[pred_func_index];
+			fn = FILTER_PRED_FN_S64;
 		else
-			fn = pred_funcs_u64[pred_func_index];
+			fn = FILTER_PRED_FN_U64;
 		break;
 	case 4:
 		if (pred_func_index < 0)
-			fn = filter_pred_32;
+			fn = FILTER_PRED_FN_32;
 		else if (field_is_signed)
-			fn = pred_funcs_s32[pred_func_index];
+			fn = FILTER_PRED_FN_S32;
 		else
-			fn = pred_funcs_u32[pred_func_index];
+			fn = FILTER_PRED_FN_U32;
 		break;
 	case 2:
 		if (pred_func_index < 0)
-			fn = filter_pred_16;
+			fn = FILTER_PRED_FN_16;
 		else if (field_is_signed)
-			fn = pred_funcs_s16[pred_func_index];
+			fn = FILTER_PRED_FN_S16;
 		else
-			fn = pred_funcs_u16[pred_func_index];
+			fn = FILTER_PRED_FN_U16;
 		break;
 	case 1:
 		if (pred_func_index < 0)
-			fn = filter_pred_8;
+			fn = FILTER_PRED_FN_8;
 		else if (field_is_signed)
-			fn = pred_funcs_s8[pred_func_index];
+			fn = FILTER_PRED_FN_S8;
 		else
-			fn = pred_funcs_u8[pred_func_index];
+			fn = FILTER_PRED_FN_U8;
 		break;
 	}
 	return fn;
 }
+static int filter_pred_fn_call(struct filter_pred *pred, void *event)
+{
+	switch (pred->fn_num) {
+	case FILTER_PRED_FN_64:
+		return filter_pred_64(pred, event);
+	case FILTER_PRED_FN_S64:
+		return filter_pred_s64(pred, event);
+	case FILTER_PRED_FN_U64:
+		return filter_pred_u64(pred, event);
+	case FILTER_PRED_FN_32:
+		return filter_pred_32(pred, event);
+	case FILTER_PRED_FN_S32:
+		return filter_pred_s32(pred, event);
+	case FILTER_PRED_FN_U32:
+		return filter_pred_u32(pred, event);
+	case FILTER_PRED_FN_16:
+		return filter_pred_16(pred, event);
+	case FILTER_PRED_FN_S16:
+		return filter_pred_s16(pred, event);
+	case FILTER_PRED_FN_U16:
+		return filter_pred_u16(pred, event);
+	case FILTER_PRED_FN_8:
+		return filter_pred_8(pred, event);
+	case FILTER_PRED_FN_S8:
+		return filter_pred_s8(pred, event);
+	case FILTER_PRED_FN_U8:
+		return filter_pred_u8(pred, event);
+	case FILTER_PRED_FN_COMM:
+		return filter_pred_comm(pred, event);
+	case FILTER_PRED_FN_STRING:
+		return filter_pred_string(pred, event);
+	case FILTER_PRED_FN_STRLOC:
+		return filter_pred_strloc(pred, event);
+	case FILTER_PRED_FN_STRRELLOC:
+		return filter_pred_strrelloc(pred, event);
+	case FILTER_PRED_FN_PCHAR_USER:
+		return filter_pred_pchar_user(pred, event);
+	case FILTER_PRED_FN_PCHAR:
+		return filter_pred_pchar(pred, event);
+	case FILTER_PRED_FN_CPU:
+		return filter_pred_cpu(pred, event);
+	case FILTER_PRED_TEST_VISITED:
+		return test_pred_visited_fn(pred, event);
+	default:
+		return 0;
+	}
+}
 /* Called when a predicate is encountered by predicate_parse() */
 static int parse_pred(const char *str, void *data,
 		      int pos, struct filter_parse_error *pe,
 			parse_error(pe, FILT_ERR_IP_FIELD_ONLY, pos + i);
 			goto err_free;
 		}
-		pred->fn = filter_pred_none;
+		pred->fn_num = FILTER_PRED_FN_NOP;
 		/*
 		 * Quotes are not required, but if they exist then we need
 		filter_build_regex(pred);
 		if (field->filter_type == FILTER_COMM) {
-			pred->fn = filter_pred_comm;
+			pred->fn_num = FILTER_PRED_FN_COMM;
 		} else if (field->filter_type == FILTER_STATIC_STRING) {
-			pred->fn = filter_pred_string;
+			pred->fn_num = FILTER_PRED_FN_STRING;
 			pred->regex.field_len = field->size;
 		} else if (field->filter_type == FILTER_DYN_STRING) {
-			pred->fn = filter_pred_strloc;
+			pred->fn_num = FILTER_PRED_FN_STRLOC;
 		} else if (field->filter_type == FILTER_RDYN_STRING)
-			pred->fn = filter_pred_strrelloc;
+			pred->fn_num = FILTER_PRED_FN_STRRELLOC;
 		else {
 			if (!ustring_per_cpu) {
 			}
 			if (ustring)
-				pred->fn = filter_pred_pchar_user;
+				pred->fn_num = FILTER_PRED_FN_PCHAR_USER;
 			else
-				pred->fn = filter_pred_pchar;
+				pred->fn_num = FILTER_PRED_FN_PCHAR;
 		}
 		/* go past the last quote */
 		i++;
 		pred->val = val;
 		if (field->filter_type == FILTER_CPU)
-			pred->fn = filter_pred_cpu;
+			pred->fn_num = FILTER_PRED_FN_CPU;
 		else {
-			pred->fn = select_comparison_fn(pred->op, field->size,
+			pred->fn_num = select_comparison_fn(pred->op, field->size,
-							field->is_signed);
+							    field->is_signed);
 			if (pred->op == OP_NE)
 				pred->not = 1;
 		}
 		struct filter_pred *pred = prog[i].pred;
 		struct ftrace_event_field *field = pred->field;
-		WARN_ON_ONCE(!pred->fn);
+		WARN_ON_ONCE(pred->fn_num == FILTER_PRED_FN_NOP);
 		if (!field) {
 			WARN_ONCE(1, "all leafs should have field defined %d", i);
 		if (!strchr(fields, *field->name))
 			continue;
-		pred->fn = test_pred_visited_fn;
+		pred->fn_num = FILTER_PRED_TEST_VISITED;
 	}
 }

kernel/trace/trace_events_hist.c+169−77

 	FIELD_OP_MULT,
 };
+enum hist_field_fn {
+	HIST_FIELD_FN_NOP,
+	HIST_FIELD_FN_VAR_REF,
+	HIST_FIELD_FN_COUNTER,
+	HIST_FIELD_FN_CONST,
+	HIST_FIELD_FN_LOG2,
+	HIST_FIELD_FN_BUCKET,
+	HIST_FIELD_FN_TIMESTAMP,
+	HIST_FIELD_FN_CPU,
+	HIST_FIELD_FN_STRING,
+	HIST_FIELD_FN_DYNSTRING,
+	HIST_FIELD_FN_RELDYNSTRING,
+	HIST_FIELD_FN_PSTRING,
+	HIST_FIELD_FN_S64,
+	HIST_FIELD_FN_U64,
+	HIST_FIELD_FN_S32,
+	HIST_FIELD_FN_U32,
+	HIST_FIELD_FN_S16,
+	HIST_FIELD_FN_U16,
+	HIST_FIELD_FN_S8,
+	HIST_FIELD_FN_U8,
+	HIST_FIELD_FN_UMINUS,
+	HIST_FIELD_FN_MINUS,
+	HIST_FIELD_FN_PLUS,
+	HIST_FIELD_FN_DIV,
+	HIST_FIELD_FN_MULT,
+	HIST_FIELD_FN_DIV_POWER2,
+	HIST_FIELD_FN_DIV_NOT_POWER2,
+	HIST_FIELD_FN_DIV_MULT_SHIFT,
+	HIST_FIELD_FN_EXECNAME,
+};
 /*
  * A hist_var (histogram variable) contains variable information for
  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 struct hist_field {
 	struct ftrace_event_field	*field;
 	unsigned long			flags;
-	hist_field_fn_t			fn;
-	unsigned int			ref;
-	unsigned int			size;
-	unsigned int			offset;
-	unsigned int                    is_signed;
 	unsigned long			buckets;
 	const char			*type;
 	struct hist_field		*operands[HIST_FIELD_OPERANDS_MAX];
 	struct hist_trigger_data	*hist_data;
+	enum hist_field_fn		fn_num;
+	unsigned int			ref;
+	unsigned int			size;
+	unsigned int			offset;
+	unsigned int                    is_signed;
 	/*
 	 * Variable fields contain variable-specific info in var.
 	u64				div_multiplier;
 };
-static u64 hist_field_none(struct hist_field *field,
+static u64 hist_fn_call(struct hist_field *hist_field,
-			   struct tracing_map_elt *elt,
+			struct tracing_map_elt *elt,
-			   struct trace_buffer *buffer,
+			struct trace_buffer *buffer,
-			   struct ring_buffer_event *rbe,
+			struct ring_buffer_event *rbe,
-			   void *event)
+			void *event);
-{
-	return 0;
-}
 static u64 hist_field_const(struct hist_field *field,
 			   struct tracing_map_elt *elt,
 {
 	struct hist_field *operand = hist_field->operands[0];
-	u64 val = operand->fn(operand, elt, buffer, rbe, event);
+	u64 val = hist_fn_call(operand, elt, buffer, rbe, event);
 	return (u64) ilog2(roundup_pow_of_two(val));
 }
 	struct hist_field *operand = hist_field->operands[0];
 	unsigned long buckets = hist_field->buckets;
-	u64 val = operand->fn(operand, elt, buffer, rbe, event);
+	u64 val = hist_fn_call(operand, elt, buffer, rbe, event);
 	if (WARN_ON_ONCE(!buckets))
 		return val;
 	struct hist_field *operand1 = hist_field->operands[0];
 	struct hist_field *operand2 = hist_field->operands[1];
-	u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
+	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
-	u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
+	u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 	return val1 + val2;
 }
 	struct hist_field *operand1 = hist_field->operands[0];
 	struct hist_field *operand2 = hist_field->operands[1];
-	u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
+	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
-	u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
+	u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 	return val1 - val2;
 }
 	struct hist_field *operand1 = hist_field->operands[0];
 	struct hist_field *operand2 = hist_field->operands[1];
-	u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
+	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
-	u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
+	u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 	/* Return -1 for the undefined case */
 	if (!val2)
 	struct hist_field *operand1 = hist_field->operands[0];
 	struct hist_field *operand2 = hist_field->operands[1];
-	u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
+	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 	return val1 >> __ffs64(operand2->constant);
 }
 	struct hist_field *operand1 = hist_field->operands[0];
 	struct hist_field *operand2 = hist_field->operands[1];
-	u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
+	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 	return div64_u64(val1, operand2->constant);
 }
 	struct hist_field *operand1 = hist_field->operands[0];
 	struct hist_field *operand2 = hist_field->operands[1];
-	u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
+	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 	/*
 	 * If the divisor is a constant, do a multiplication and shift instead.
 	struct hist_field *operand1 = hist_field->operands[0];
 	struct hist_field *operand2 = hist_field->operands[1];
-	u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
+	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
-	u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
+	u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 	return val1 * val2;
 }
 {
 	struct hist_field *operand = hist_field->operands[0];
-	s64 sval = (s64)operand->fn(operand, elt, buffer, rbe, event);
+	s64 sval = (s64)hist_fn_call(operand, elt, buffer, rbe, event);
 	u64 val = (u64)-sval;
 	return val;
  * Returns the specific division function to use if the divisor
  * is constant. This avoids extra branches when the trigger is hit.
  */
-static hist_field_fn_t hist_field_get_div_fn(struct hist_field *divisor)
+static enum hist_field_fn hist_field_get_div_fn(struct hist_field *divisor)
 {
 	u64 div = divisor->constant;
 	if (!(div & (div - 1)))
-		return div_by_power_of_two;
+		return HIST_FIELD_FN_DIV_POWER2;
 	/* If the divisor is too large, do a regular division */
 	if (div > (1 << HIST_DIV_SHIFT))
-		return div_by_not_power_of_two;
+		return HIST_FIELD_FN_DIV_NOT_POWER2;
 	divisor->div_multiplier = div64_u64((u64)(1 << HIST_DIV_SHIFT), div);
-	return div_by_mult_and_shift;
+	return HIST_FIELD_FN_DIV_MULT_SHIFT;
 }
 static void track_data_free(struct track_data *track_data)
 	return field_name;
 }
-static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
+static enum hist_field_fn select_value_fn(int field_size, int field_is_signed)
 {
-	hist_field_fn_t fn = NULL;
 	switch (field_size) {
 	case 8:
 		if (field_is_signed)
-			fn = hist_field_s64;
+			return HIST_FIELD_FN_S64;
 		else
-			fn = hist_field_u64;
+			return HIST_FIELD_FN_U64;
-		break;
 	case 4:
 		if (field_is_signed)
-			fn = hist_field_s32;
+			return HIST_FIELD_FN_S32;
 		else
-			fn = hist_field_u32;
+			return HIST_FIELD_FN_U32;
-		break;
 	case 2:
 		if (field_is_signed)
-			fn = hist_field_s16;
+			return HIST_FIELD_FN_S16;
 		else
-			fn = hist_field_u16;
+			return HIST_FIELD_FN_U16;
-		break;
 	case 1:
 		if (field_is_signed)
-			fn = hist_field_s8;
+			return HIST_FIELD_FN_S8;
 		else
-			fn = hist_field_u8;
+			return HIST_FIELD_FN_U8;
-		break;
 	}
-	return fn;
+	return HIST_FIELD_FN_NOP;
 }
 static int parse_map_size(char *str)
 		goto out; /* caller will populate */
 	if (flags & HIST_FIELD_FL_VAR_REF) {
-		hist_field->fn = hist_field_var_ref;
+		hist_field->fn_num = HIST_FIELD_FN_VAR_REF;
 		goto out;
 	}
 	if (flags & HIST_FIELD_FL_HITCOUNT) {
-		hist_field->fn = hist_field_counter;
+		hist_field->fn_num = HIST_FIELD_FN_COUNTER;
 		hist_field->size = sizeof(u64);
 		hist_field->type = "u64";
 		goto out;
 	}
 	if (flags & HIST_FIELD_FL_CONST) {
-		hist_field->fn = hist_field_const;
+		hist_field->fn_num = HIST_FIELD_FN_CONST;
 		hist_field->size = sizeof(u64);
 		hist_field->type = kstrdup("u64", GFP_KERNEL);
 		if (!hist_field->type)
 	}
 	if (flags & HIST_FIELD_FL_STACKTRACE) {
-		hist_field->fn = hist_field_none;
+		hist_field->fn_num = HIST_FIELD_FN_NOP;
 		goto out;
 	}
 	if (flags & (HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET)) {
 		unsigned long fl = flags & ~(HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET);
-		hist_field->fn = flags & HIST_FIELD_FL_LOG2 ? hist_field_log2 :
+		hist_field->fn_num = flags & HIST_FIELD_FL_LOG2 ? HIST_FIELD_FN_LOG2 :
-			hist_field_bucket;
+			HIST_FIELD_FN_BUCKET;
 		hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
 		hist_field->size = hist_field->operands[0]->size;
 		hist_field->type = kstrdup_const(hist_field->operands[0]->type, GFP_KERNEL);
 	}
 	if (flags & HIST_FIELD_FL_TIMESTAMP) {
-		hist_field->fn = hist_field_timestamp;
+		hist_field->fn_num = HIST_FIELD_FN_TIMESTAMP;
 		hist_field->size = sizeof(u64);
 		hist_field->type = "u64";
 		goto out;
 	}
 	if (flags & HIST_FIELD_FL_CPU) {
-		hist_field->fn = hist_field_cpu;
+		hist_field->fn_num = HIST_FIELD_FN_CPU;
 		hist_field->size = sizeof(int);
 		hist_field->type = "unsigned int";
 		goto out;
 			goto free;
 		if (field->filter_type == FILTER_STATIC_STRING) {
-			hist_field->fn = hist_field_string;
+			hist_field->fn_num = HIST_FIELD_FN_STRING;
 			hist_field->size = field->size;
 		} else if (field->filter_type == FILTER_DYN_STRING) {
-			hist_field->fn = hist_field_dynstring;
+			hist_field->fn_num = HIST_FIELD_FN_DYNSTRING;
 		} else if (field->filter_type == FILTER_RDYN_STRING)
-			hist_field->fn = hist_field_reldynstring;
+			hist_field->fn_num = HIST_FIELD_FN_RELDYNSTRING;
 		else
-			hist_field->fn = hist_field_pstring;
+			hist_field->fn_num = HIST_FIELD_FN_PSTRING;
 	} else {
 		hist_field->size = field->size;
 		hist_field->is_signed = field->is_signed;
 		if (!hist_field->type)
 			goto free;
-		hist_field->fn = select_value_fn(field->size,
+		hist_field->fn_num = select_value_fn(field->size,
-						 field->is_signed);
+						     field->is_signed);
-		if (!hist_field->fn) {
+		if (hist_field->fn_num == HIST_FIELD_FN_NOP) {
 			destroy_hist_field(hist_field, 0);
 			return NULL;
 		}
 	if (!alias)
 		return NULL;
-	alias->fn = var_ref->fn;
+	alias->fn_num = var_ref->fn_num;
 	alias->operands[0] = var_ref;
 	if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
 	expr->flags |= operand1->flags &
 		(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
-	expr->fn = hist_field_unary_minus;
+	expr->fn_num = HIST_FIELD_FN_UMINUS;
 	expr->operands[0] = operand1;
 	expr->size = operand1->size;
 	expr->is_signed = operand1->is_signed;
 	unsigned long operand_flags, operand2_flags;
 	int field_op, ret = -EINVAL;
 	char *sep, *operand1_str;
-	hist_field_fn_t op_fn;
+	enum hist_field_fn op_fn;
 	bool combine_consts;
 	if (*n_subexprs > 3) {
 	switch (field_op) {
 	case FIELD_OP_MINUS:
-		op_fn = hist_field_minus;
+		op_fn = HIST_FIELD_FN_MINUS;
 		break;
 	case FIELD_OP_PLUS:
-		op_fn = hist_field_plus;
+		op_fn = HIST_FIELD_FN_PLUS;
 		break;
 	case FIELD_OP_DIV:
-		op_fn = hist_field_div;
+		op_fn = HIST_FIELD_FN_DIV;
 		break;
 	case FIELD_OP_MULT:
-		op_fn = hist_field_mult;
+		op_fn = HIST_FIELD_FN_MULT;
 		break;
 	default:
 		ret = -EINVAL;
 		op_fn = hist_field_get_div_fn(operand2);
 	}
+	expr->fn_num = op_fn;
 	if (combine_consts) {
 		if (var1)
 			expr->operands[0] = var1;
 		if (var2)
 			expr->operands[1] = var2;
-		expr->constant = op_fn(expr, NULL, NULL, NULL, NULL);
+		expr->constant = hist_fn_call(expr, NULL, NULL, NULL, NULL);
+		expr->fn_num = HIST_FIELD_FN_CONST;
 		expr->operands[0] = NULL;
 		expr->operands[1] = NULL;
 		expr->name = expr_str(expr, 0);
 	} else {
-		expr->fn = op_fn;
 		/* The operand sizes should be the same, so just pick one */
 		expr->size = operand1->size;
 		expr->is_signed = operand1->is_signed;
 		struct hist_field *var = field_var->var;
 		struct hist_field *val = field_var->val;
-		var_val = val->fn(val, elt, buffer, rbe, rec);
+		var_val = hist_fn_call(val, elt, buffer, rbe, rec);
 		var_idx = var->var.idx;
 		if (val->flags & HIST_FIELD_FL_STRING) {
 	return (u64)(unsigned long)(elt_data->comm);
 }
+static u64 hist_fn_call(struct hist_field *hist_field,
+			struct tracing_map_elt *elt,
+			struct trace_buffer *buffer,
+			struct ring_buffer_event *rbe,
+			void *event)
+{
+	switch (hist_field->fn_num) {
+	case HIST_FIELD_FN_VAR_REF:
+		return hist_field_var_ref(hist_field, elt, buffer, rbe, event);
+	case HIST_FIELD_FN_COUNTER:
+		return hist_field_counter(hist_field, elt, buffer, rbe, event);
+	case HIST_FIELD_FN_CONST:
+		return hist_field_const(hist_field, elt, buffer, rbe, event);
+	case HIST_FIELD_FN_LOG2:
+		return hist_field_log2(hist_field, elt, buffer, rbe, event);
+	case HIST_FIELD_FN_BUCKET:
+		return hist_field_bucket(hist_field, elt, buffer, rbe, event);
+	case HIST_FIELD_FN_TIMESTAMP:
+		return hist_field_timestamp(hist_field, elt, buffer, rbe, event);
+	case HIST_FIELD_FN_CPU:
+		return hist_field_cpu(hist_field, elt, buffer, rbe, event);
+	case HIST_FIELD_FN_STRING:
+		return hist_field_string(hist_field, elt, buffer, rbe, event);
+	case HIST_FIELD_FN_DYNSTRING:
+		return hist_field_dynstring(hist_field, elt, buffer, rbe, event);
+	case HIST_FIELD_FN_RELDYNSTRING:
+		return hist_field_reldynstring(hist_field, elt, buffer, rbe, event);
+	case HIST_FIELD_FN_PSTRING:
+		return hist_field_pstring(hist_field, elt, buffer, rbe, event);
+	case HIST_FIELD_FN_S64:
+		return hist_field_s64(hist_field, elt, buffer, rbe, event);
+	case HIST_FIELD_FN_U64:
+		return hist_field_u64(hist_field, elt, buffer, rbe, event);
+	case HIST_FIELD_FN_S32:
+		return hist_field_s32(hist_field, elt, buffer, rbe, event);
+	case HIST_FIELD_FN_U32:
+		return hist_field_u32(hist_field, elt, buffer, rbe, event);
+	case HIST_FIELD_FN_S16:
+		return hist_field_s16(hist_field, elt, buffer, rbe, event);
+	case HIST_FIELD_FN_U16:
+		return hist_field_u16(hist_field, elt, buffer, rbe, event);
+	case HIST_FIELD_FN_S8:
+		return hist_field_s8(hist_field, elt, buffer, rbe, event);
+	case HIST_FIELD_FN_U8:
+		return hist_field_u8(hist_field, elt, buffer, rbe, event);
+	case HIST_FIELD_FN_UMINUS:
+		return hist_field_unary_minus(hist_field, elt, buffer, rbe, event);
+	case HIST_FIELD_FN_MINUS:
+		return hist_field_minus(hist_field, elt, buffer, rbe, event);
+	case HIST_FIELD_FN_PLUS:
+		return hist_field_plus(hist_field, elt, buffer, rbe, event);
+	case HIST_FIELD_FN_DIV:
+		return hist_field_div(hist_field, elt, buffer, rbe, event);
+	case HIST_FIELD_FN_MULT:
+		return hist_field_mult(hist_field, elt, buffer, rbe, event);
+	case HIST_FIELD_FN_DIV_POWER2:
+		return div_by_power_of_two(hist_field, elt, buffer, rbe, event);
+	case HIST_FIELD_FN_DIV_NOT_POWER2:
+		return div_by_not_power_of_two(hist_field, elt, buffer, rbe, event);
+	case HIST_FIELD_FN_DIV_MULT_SHIFT:
+		return div_by_mult_and_shift(hist_field, elt, buffer, rbe, event);
+	case HIST_FIELD_FN_EXECNAME:
+		return hist_field_execname(hist_field, elt, buffer, rbe, event);
+	default:
+		return 0;
+	}
+}
 /* Convert a var that points to common_pid.execname to a string */
 static void update_var_execname(struct hist_field *hist_field)
 {
 	kfree_const(hist_field->type);
 	hist_field->type = "char[]";
-	hist_field->fn = hist_field_execname;
+	hist_field->fn_num = HIST_FIELD_FN_EXECNAME;
 }
 static int create_var_field(struct hist_trigger_data *hist_data,
 	for_each_hist_val_field(i, hist_data) {
 		hist_field = hist_data->fields[i];
-		hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
+		hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec);
 		if (hist_field->flags & HIST_FIELD_FL_VAR) {
 			var_idx = hist_field->var.idx;
 	for_each_hist_key_field(i, hist_data) {
 		hist_field = hist_data->fields[i];
 		if (hist_field->flags & HIST_FIELD_FL_VAR) {
-			hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
+			hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec);
 			var_idx = hist_field->var.idx;
 			tracing_map_set_var(elt, var_idx, hist_val);
 		}
 					 HIST_STACKTRACE_SKIP);
 			key = entries;
 		} else {
-			field_contents = key_field->fn(key_field, elt, buffer, rbe, rec);
+			field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec);
 			if (key_field->flags & HIST_FIELD_FL_STRING) {
 				key = (void *)(unsigned long)field_contents;
 				use_compound_key = true;

kernel/trace/trace_events_user.c+426−142

 #include <linux/uio.h>
 #include <linux/ioctl.h>
 #include <linux/jhash.h>
+#include <linux/refcount.h>
 #include <linux/trace_events.h>
 #include <linux/tracefs.h>
 #include <linux/types.h>
  */
 #define MAX_PAGE_ORDER 0
 #define MAX_PAGES (1 << MAX_PAGE_ORDER)
-#define MAX_EVENTS (MAX_PAGES * PAGE_SIZE)
+#define MAX_BYTES (MAX_PAGES * PAGE_SIZE)
+#define MAX_EVENTS (MAX_BYTES * 8)
 /* Limit how long of an event name plus args within the subsystem. */
 #define MAX_EVENT_DESC 512
 #define EVENT_NAME(user_event) ((user_event)->tracepoint.name)
 #define MAX_FIELD_ARRAY_SIZE 1024
-#define MAX_FIELD_ARG_NAME 256
-static char *register_page_data;
+/*
+ * The MAP_STATUS_* macros are used for taking a index and determining the
+ * appropriate byte and the bit in the byte to set/reset for an event.
+ *
+ * The lower 3 bits of the index decide which bit to set.
+ * The remaining upper bits of the index decide which byte to use for the bit.
+ *
+ * This is used when an event has a probe attached/removed to reflect live
+ * status of the event wanting tracing or not to user-programs via shared
+ * memory maps.
+ */
+#define MAP_STATUS_BYTE(index) ((index) >> 3)
+#define MAP_STATUS_MASK(index) BIT((index) & 7)
+/*
+ * Internal bits (kernel side only) to keep track of connected probes:
+ * These are used when status is requested in text form about an event. These
+ * bits are compared against an internal byte on the event to determine which
+ * probes to print out to the user.
+ *
+ * These do not reflect the mapped bytes between the user and kernel space.
+ */
+#define EVENT_STATUS_FTRACE BIT(0)
+#define EVENT_STATUS_PERF BIT(1)
+#define EVENT_STATUS_OTHER BIT(7)
+/*
+ * Stores the pages, tables, and locks for a group of events.
+ * Each logical grouping of events has its own group, with a
+ * matching page for status checks within user programs. This
+ * allows for isolation of events to user programs by various
+ * means.
+ */
+struct user_event_group {
+	struct page *pages;
+	char *register_page_data;
+	char *system_name;
+	struct hlist_node node;
+	struct mutex reg_mutex;
+	DECLARE_HASHTABLE(register_table, 8);
+	DECLARE_BITMAP(page_bitmap, MAX_EVENTS);
+};
-static DEFINE_MUTEX(reg_mutex);
+/* Group for init_user_ns mapping, top-most group */
-static DEFINE_HASHTABLE(register_table, 4);
+static struct user_event_group *init_group;
-static DECLARE_BITMAP(page_bitmap, MAX_EVENTS);
 /*
  * Stores per-event properties, as users register events
  * within a file a user_event might be created if it does not
  * already exist. These are globally used and their lifetime
  * is tied to the refcnt member. These cannot go away until the
- * refcnt reaches zero.
+ * refcnt reaches one.
  */
 struct user_event {
+	struct user_event_group *group;
 	struct tracepoint tracepoint;
 	struct trace_event_call call;
 	struct trace_event_class class;
 	struct hlist_node node;
 	struct list_head fields;
 	struct list_head validators;
-	atomic_t refcnt;
+	refcount_t refcnt;
 	int index;
 	int flags;
 	int min_size;
+	char status;
 };
 /*
 	struct user_event *events[];
 };
+struct user_event_file_info {
+	struct user_event_group *group;
+	struct user_event_refs *refs;
+};
 #define VALIDATOR_ENSURE_NULL (1 << 0)
 #define VALIDATOR_REL (1 << 1)
 typedef void (*user_event_func_t) (struct user_event *user, struct iov_iter *i,
 				   void *tpdata, bool *faulted);
-static int user_event_parse(char *name, char *args, char *flags,
+static int user_event_parse(struct user_event_group *group, char *name,
+			    char *args, char *flags,
 			    struct user_event **newuser);
 static u32 user_event_key(char *name)
 	return jhash(name, strlen(name), 0);
 }
+static void set_page_reservations(char *pages, bool set)
+{
+	int page;
+	for (page = 0; page < MAX_PAGES; ++page) {
+		void *addr = pages + (PAGE_SIZE * page);
+		if (set)
+			SetPageReserved(virt_to_page(addr));
+		else
+			ClearPageReserved(virt_to_page(addr));
+	}
+}
+static void user_event_group_destroy(struct user_event_group *group)
+{
+	if (group->register_page_data)
+		set_page_reservations(group->register_page_data, false);
+	if (group->pages)
+		__free_pages(group->pages, MAX_PAGE_ORDER);
+	kfree(group->system_name);
+	kfree(group);
+}
+static char *user_event_group_system_name(struct user_namespace *user_ns)
+{
+	char *system_name;
+	int len = sizeof(USER_EVENTS_SYSTEM) + 1;
+	if (user_ns != &init_user_ns) {
+		/*
+		 * Unexpected at this point:
+		 * We only currently support init_user_ns.
+		 * When we enable more, this will trigger a failure so log.
+		 */
+		pr_warn("user_events: Namespace other than init_user_ns!\n");
+		return NULL;
+	}
+	system_name = kmalloc(len, GFP_KERNEL);
+	if (!system_name)
+		return NULL;
+	snprintf(system_name, len, "%s", USER_EVENTS_SYSTEM);
+	return system_name;
+}
+static inline struct user_event_group
+*user_event_group_from_user_ns(struct user_namespace *user_ns)
+{
+	if (user_ns == &init_user_ns)
+		return init_group;
+	return NULL;
+}
+static struct user_event_group *current_user_event_group(void)
+{
+	struct user_namespace *user_ns = current_user_ns();
+	struct user_event_group *group = NULL;
+	while (user_ns) {
+		group = user_event_group_from_user_ns(user_ns);
+		if (group)
+			break;
+		user_ns = user_ns->parent;
+	}
+	return group;
+}
+static struct user_event_group
+*user_event_group_create(struct user_namespace *user_ns)
+{
+	struct user_event_group *group;
+	group = kzalloc(sizeof(*group), GFP_KERNEL);
+	if (!group)
+		return NULL;
+	group->system_name = user_event_group_system_name(user_ns);
+	if (!group->system_name)
+		goto error;
+	group->pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, MAX_PAGE_ORDER);
+	if (!group->pages)
+		goto error;
+	group->register_page_data = page_address(group->pages);
+	set_page_reservations(group->register_page_data, true);
+	/* Zero all bits beside 0 (which is reserved for failures) */
+	bitmap_zero(group->page_bitmap, MAX_EVENTS);
+	set_bit(0, group->page_bitmap);
+	mutex_init(&group->reg_mutex);
+	hash_init(group->register_table);
+	return group;
+error:
+	if (group)
+		user_event_group_destroy(group);
+	return NULL;
+};
+static __always_inline
+void user_event_register_set(struct user_event *user)
+{
+	int i = user->index;
+	user->group->register_page_data[MAP_STATUS_BYTE(i)] |= MAP_STATUS_MASK(i);
+}
+static __always_inline
+void user_event_register_clear(struct user_event *user)
+{
+	int i = user->index;
+	user->group->register_page_data[MAP_STATUS_BYTE(i)] &= ~MAP_STATUS_MASK(i);
+}
+static __always_inline __must_check
+bool user_event_last_ref(struct user_event *user)
+{
+	return refcount_read(&user->refcnt) == 1;
+}
 static __always_inline __must_check
 size_t copy_nofault(void *addr, size_t bytes, struct iov_iter *i)
 {
  *
  * Upon success user_event has its ref count increased by 1.
  */
-static int user_event_parse_cmd(char *raw_command, struct user_event **newuser)
+static int user_event_parse_cmd(struct user_event_group *group,
+				char *raw_command, struct user_event **newuser)
 {
 	char *name = raw_command;
 	char *args = strpbrk(name, " ");
 	if (flags)
 		*flags++ = '\0';
-	return user_event_parse(name, args, flags, newuser);
+	return user_event_parse(group, name, args, flags, newuser);
 }
 static int user_field_array_size(const char *type)
 	goto add_field;
 add_validator:
-	if (strstr(type, "char") != 0)
+	if (strstr(type, "char") != NULL)
 		validator_flags |= VALIDATOR_ENSURE_NULL;
 	validator = kmalloc(sizeof(*validator), GFP_KERNEL);
 		return "%d";
 	if (strcmp(type, "unsigned char") == 0)
 		return "%u";
-	if (strstr(type, "char[") != 0)
+	if (strstr(type, "char[") != NULL)
 		return "%s";
 	/* Unknown, likely struct, allowed treat as 64-bit */
 	return false;
 check:
-	return strstr(type, "char") != 0;
+	return strstr(type, "char") != NULL;
 }
 #define LEN_OR_ZERO (len ? len - pos : 0)
+static int user_dyn_field_set_string(int argc, const char **argv, int *iout,
+				     char *buf, int len, bool *colon)
+{
+	int pos = 0, i = *iout;
+	*colon = false;
+	for (; i < argc; ++i) {
+		if (i != *iout)
+			pos += snprintf(buf + pos, LEN_OR_ZERO, " ");
+		pos += snprintf(buf + pos, LEN_OR_ZERO, "%s", argv[i]);
+		if (strchr(argv[i], ';')) {
+			++i;
+			*colon = true;
+			break;
+		}
+	}
+	/* Actual set, advance i */
+	if (len != 0)
+		*iout = i;
+	return pos + 1;
+}
+static int user_field_set_string(struct ftrace_event_field *field,
+				 char *buf, int len, bool colon)
+{
+	int pos = 0;
+	pos += snprintf(buf + pos, LEN_OR_ZERO, "%s", field->type);
+	pos += snprintf(buf + pos, LEN_OR_ZERO, " ");
+	pos += snprintf(buf + pos, LEN_OR_ZERO, "%s", field->name);
+	if (colon)
+		pos += snprintf(buf + pos, LEN_OR_ZERO, ";");
+	return pos + 1;
+}
 static int user_event_set_print_fmt(struct user_event *user, char *buf, int len)
 {
 	struct ftrace_event_field *field, *next;
 	dyn_event_remove(&user->devent);
-	register_page_data[user->index] = 0;
+	user_event_register_clear(user);
-	clear_bit(user->index, page_bitmap);
+	clear_bit(user->index, user->group->page_bitmap);
 	hash_del(&user->node);
 	user_event_destroy_validators(user);
 	return ret;
 }
-static struct user_event *find_user_event(char *name, u32 *outkey)
+static struct user_event *find_user_event(struct user_event_group *group,
+					  char *name, u32 *outkey)
 {
 	struct user_event *user;
 	u32 key = user_event_key(name);
 	*outkey = key;
-	hash_for_each_possible(register_table, user, node, key)
+	hash_for_each_possible(group->register_table, user, node, key)
 		if (!strcmp(EVENT_NAME(user), name)) {
-			atomic_inc(&user->refcnt);
+			refcount_inc(&user->refcnt);
 			return user;
 		}
 		rcu_read_unlock_sched();
 	}
-	register_page_data[user->index] = status;
+	if (status)
+		user_event_register_set(user);
+	else
+		user_event_register_clear(user);
+	user->status = status;
 }
 /*
 	return ret;
 inc:
-	atomic_inc(&user->refcnt);
+	refcount_inc(&user->refcnt);
 	update_reg_page_for(user);
 	return 0;
 dec:
 	update_reg_page_for(user);
-	atomic_dec(&user->refcnt);
+	refcount_dec(&user->refcnt);
 	return 0;
 }
 static int user_event_create(const char *raw_command)
 {
+	struct user_event_group *group;
 	struct user_event *user;
 	char *name;
 	int ret;
 	if (!name)
 		return -ENOMEM;
-	mutex_lock(&reg_mutex);
+	group = current_user_event_group();
-	ret = user_event_parse_cmd(name, &user);
+	if (!group)
+		return -ENOENT;
+	mutex_lock(&group->reg_mutex);
+	ret = user_event_parse_cmd(group, name, &user);
 	if (!ret)
-		atomic_dec(&user->refcnt);
+		refcount_dec(&user->refcnt);
-	mutex_unlock(&reg_mutex);
+	mutex_unlock(&group->reg_mutex);
 	if (ret)
 		kfree(name);
 {
 	struct user_event *user = container_of(ev, struct user_event, devent);
-	return atomic_read(&user->refcnt) != 0;
+	return !user_event_last_ref(user);
 }
 static int user_event_free(struct dyn_event *ev)
 {
 	struct user_event *user = container_of(ev, struct user_event, devent);
-	if (atomic_read(&user->refcnt) != 0)
+	if (!user_event_last_ref(user))
 		return -EBUSY;
 	return destroy_user_event(user);
 static bool user_field_match(struct ftrace_event_field *field, int argc,
 			     const char **argv, int *iout)
 {
-	char *field_name, *arg_name;
+	char *field_name = NULL, *dyn_field_name = NULL;
-	int len, pos, i = *iout;
 	bool colon = false, match = false;
+	int dyn_len, len;
-	if (i >= argc)
+	if (*iout >= argc)
 		return false;
-	len = MAX_FIELD_ARG_NAME;
+	dyn_len = user_dyn_field_set_string(argc, argv, iout, dyn_field_name,
-	field_name = kmalloc(len, GFP_KERNEL);
+					    0, &colon);
-	arg_name = kmalloc(len, GFP_KERNEL);
-	if (!arg_name || !field_name)
-		goto out;
-	pos = 0;
-	for (; i < argc; ++i) {
-		if (i != *iout)
-			pos += snprintf(arg_name + pos, len - pos, " ");
-		pos += snprintf(arg_name + pos, len - pos, argv[i]);
+	len = user_field_set_string(field, field_name, 0, colon);
-		if (strchr(argv[i], ';')) {
+	if (dyn_len != len)
-			++i;
+		return false;
-			colon = true;
-			break;
-		}
-	}
-	pos = 0;
+	dyn_field_name = kmalloc(dyn_len, GFP_KERNEL);
+	field_name = kmalloc(len, GFP_KERNEL);
-	pos += snprintf(field_name + pos, len - pos, field->type);
+	if (!dyn_field_name || !field_name)
-	pos += snprintf(field_name + pos, len - pos, " ");
+		goto out;
-	pos += snprintf(field_name + pos, len - pos, field->name);
-	if (colon)
+	user_dyn_field_set_string(argc, argv, iout, dyn_field_name,
-		pos += snprintf(field_name + pos, len - pos, ";");
+				  dyn_len, &colon);
-	*iout = i;
+	user_field_set_string(field, field_name, len, colon);
-	match = strcmp(arg_name, field_name) == 0;
+	match = strcmp(dyn_field_name, field_name) == 0;
 out:
-	kfree(arg_name);
+	kfree(dyn_field_name);
 	kfree(field_name);
 	return match;
  * The name buffer lifetime is owned by this method for success cases only.
  * Upon success the returned user_event has its ref count increased by 1.
  */
-static int user_event_parse(char *name, char *args, char *flags,
+static int user_event_parse(struct user_event_group *group, char *name,
+			    char *args, char *flags,
 			    struct user_event **newuser)
 {
 	int ret;
 	/* Prevent dyn_event from racing */
 	mutex_lock(&event_mutex);
-	user = find_user_event(name, &key);
+	user = find_user_event(group, name, &key);
 	mutex_unlock(&event_mutex);
 	if (user) {
 		return 0;
 	}
-	index = find_first_zero_bit(page_bitmap, MAX_EVENTS);
+	index = find_first_zero_bit(group->page_bitmap, MAX_EVENTS);
 	if (index == MAX_EVENTS)
 		return -EMFILE;
 	INIT_LIST_HEAD(&user->fields);
 	INIT_LIST_HEAD(&user->validators);
+	user->group = group;
 	user->tracepoint.name = name;
 	ret = user_event_parse_fields(user, args);
 	user->call.flags = TRACE_EVENT_FL_TRACEPOINT;
 	user->call.tp = &user->tracepoint;
 	user->call.event.funcs = &user_event_funcs;
+	user->class.system = group->system_name;
-	user->class.system = USER_EVENTS_SYSTEM;
 	user->class.fields_array = user_event_fields_array;
 	user->class.get_fields = user_event_get_fields;
 	user->class.reg = user_event_reg;
 	user->index = index;
-	/* Ensure we track ref */
+	/* Ensure we track self ref and caller ref (2) */
-	atomic_inc(&user->refcnt);
+	refcount_set(&user->refcnt, 2);
 	dyn_event_init(&user->devent, &user_event_dops);
 	dyn_event_add(&user->devent, &user->call);
-	set_bit(user->index, page_bitmap);
+	set_bit(user->index, group->page_bitmap);
-	hash_add(register_table, &user->node, key);
+	hash_add(group->register_table, &user->node, key);
 	mutex_unlock(&event_mutex);
 /*
  * Deletes a previously created event if it is no longer being used.
  */
-static int delete_user_event(char *name)
+static int delete_user_event(struct user_event_group *group, char *name)
 {
 	u32 key;
-	int ret;
+	struct user_event *user = find_user_event(group, name, &key);
-	struct user_event *user = find_user_event(name, &key);
 	if (!user)
 		return -ENOENT;
-	/* Ensure we are the last ref */
+	refcount_dec(&user->refcnt);
-	if (atomic_read(&user->refcnt) != 1) {
-		ret = -EBUSY;
-		goto put_ref;
-	}
-	ret = destroy_user_event(user);
-	if (ret)
-		goto put_ref;
-	return ret;
+	if (!user_event_last_ref(user))
-put_ref:
+		return -EBUSY;
-	/* No longer have this ref */
-	atomic_dec(&user->refcnt);
-	return ret;
+	return destroy_user_event(user);
 }
 /*
  */
 static ssize_t user_events_write_core(struct file *file, struct iov_iter *i)
 {
+	struct user_event_file_info *info = file->private_data;
 	struct user_event_refs *refs;
 	struct user_event *user = NULL;
 	struct tracepoint *tp;
 	rcu_read_lock_sched();
-	refs = rcu_dereference_sched(file->private_data);
+	refs = rcu_dereference_sched(info->refs);
 	/*
 	 * The refs->events array is protected by RCU, and new items may be
 	return ret;
 }
+static int user_events_open(struct inode *node, struct file *file)
+{
+	struct user_event_group *group;
+	struct user_event_file_info *info;
+	group = current_user_event_group();
+	if (!group)
+		return -ENOENT;
+	info = kzalloc(sizeof(*info), GFP_KERNEL);
+	if (!info)
+		return -ENOMEM;
+	info->group = group;
+	file->private_data = info;
+	return 0;
+}
 static ssize_t user_events_write(struct file *file, const char __user *ubuf,
 				 size_t count, loff_t *ppos)
 {
 	if (unlikely(*ppos != 0))
 		return -EFAULT;
-	if (unlikely(import_single_range(READ, (char *)ubuf, count, &iov, &i)))
+	if (unlikely(import_single_range(WRITE, (char __user *)ubuf,
+					 count, &iov, &i)))
 		return -EFAULT;
 	return user_events_write_core(file, &i);
 	return user_events_write_core(kp->ki_filp, i);
 }
-static int user_events_ref_add(struct file *file, struct user_event *user)
+static int user_events_ref_add(struct user_event_file_info *info,
+			       struct user_event *user)
 {
+	struct user_event_group *group = info->group;
 	struct user_event_refs *refs, *new_refs;
 	int i, size, count = 0;
-	refs = rcu_dereference_protected(file->private_data,
+	refs = rcu_dereference_protected(info->refs,
-					 lockdep_is_held(&reg_mutex));
+					 lockdep_is_held(&group->reg_mutex));
 	if (refs) {
 		count = refs->count;
 	new_refs->events[i] = user;
-	atomic_inc(&user->refcnt);
+	refcount_inc(&user->refcnt);
-	rcu_assign_pointer(file->private_data, new_refs);
+	rcu_assign_pointer(info->refs, new_refs);
 	if (refs)
 		kfree_rcu(refs, rcu);
 	if (size > PAGE_SIZE)
 		return -E2BIG;
-	return copy_struct_from_user(kreg, sizeof(*kreg), ureg, size);
+	if (size < offsetofend(struct user_reg, write_index))
+		return -EINVAL;
+	ret = copy_struct_from_user(kreg, sizeof(*kreg), ureg, size);
+	if (ret)
+		return ret;
+	kreg->size = size;
+	return 0;
 }
 /*
  * Registers a user_event on behalf of a user process.
  */
-static long user_events_ioctl_reg(struct file *file, unsigned long uarg)
+static long user_events_ioctl_reg(struct user_event_file_info *info,
+				  unsigned long uarg)
 {
 	struct user_reg __user *ureg = (struct user_reg __user *)uarg;
 	struct user_reg reg;
 		return ret;
 	}
-	ret = user_event_parse_cmd(name, &user);
+	ret = user_event_parse_cmd(info->group, name, &user);
 	if (ret) {
 		kfree(name);
 		return ret;
 	}
-	ret = user_events_ref_add(file, user);
+	ret = user_events_ref_add(info, user);
 	/* No longer need parse ref, ref_add either worked or not */
-	atomic_dec(&user->refcnt);
+	refcount_dec(&user->refcnt);
 	/* Positive number is index and valid */
 	if (ret < 0)
 		return ret;
 	put_user((u32)ret, &ureg->write_index);
-	put_user(user->index, &ureg->status_index);
+	put_user(user->index, &ureg->status_bit);
 	return 0;
 }
 /*
  * Deletes a user_event on behalf of a user process.
  */
-static long user_events_ioctl_del(struct file *file, unsigned long uarg)
+static long user_events_ioctl_del(struct user_event_file_info *info,
+				  unsigned long uarg)
 {
 	void __user *ubuf = (void __user *)uarg;
 	char *name;
 	/* event_mutex prevents dyn_event from racing */
 	mutex_lock(&event_mutex);
-	ret = delete_user_event(name);
+	ret = delete_user_event(info->group, name);
 	mutex_unlock(&event_mutex);
 	kfree(name);
 static long user_events_ioctl(struct file *file, unsigned int cmd,
 			      unsigned long uarg)
 {
+	struct user_event_file_info *info = file->private_data;
+	struct user_event_group *group = info->group;
 	long ret = -ENOTTY;
 	switch (cmd) {
 	case DIAG_IOCSREG:
-		mutex_lock(&reg_mutex);
+		mutex_lock(&group->reg_mutex);
-		ret = user_events_ioctl_reg(file, uarg);
+		ret = user_events_ioctl_reg(info, uarg);
-		mutex_unlock(&reg_mutex);
+		mutex_unlock(&group->reg_mutex);
 		break;
 	case DIAG_IOCSDEL:
-		mutex_lock(&reg_mutex);
+		mutex_lock(&group->reg_mutex);
-		ret = user_events_ioctl_del(file, uarg);
+		ret = user_events_ioctl_del(info, uarg);
-		mutex_unlock(&reg_mutex);
+		mutex_unlock(&group->reg_mutex);
 		break;
 	}
  */
 static int user_events_release(struct inode *node, struct file *file)
 {
+	struct user_event_file_info *info = file->private_data;
+	struct user_event_group *group;
 	struct user_event_refs *refs;
 	struct user_event *user;
 	int i;
+	if (!info)
+		return -EINVAL;
+	group = info->group;
 	/*
 	 * Ensure refs cannot change under any situation by taking the
 	 * register mutex during the final freeing of the references.
 	 */
-	mutex_lock(&reg_mutex);
+	mutex_lock(&group->reg_mutex);
-	refs = file->private_data;
+	refs = info->refs;
 	if (!refs)
 		goto out;
 		user = refs->events[i];
 		if (user)
-			atomic_dec(&user->refcnt);
+			refcount_dec(&user->refcnt);
 	}
 out:
 	file->private_data = NULL;
-	mutex_unlock(&reg_mutex);
+	mutex_unlock(&group->reg_mutex);
 	kfree(refs);
+	kfree(info);
 	return 0;
 }
 static const struct file_operations user_data_fops = {
+	.open = user_events_open,
 	.write = user_events_write,
 	.write_iter = user_events_write_iter,
 	.unlocked_ioctl	= user_events_ioctl,
 	.release = user_events_release,
 };
+static struct user_event_group *user_status_group(struct file *file)
+{
+	struct seq_file *m = file->private_data;
+	if (!m)
+		return NULL;
+	return m->private;
+}
 /*
  * Maps the shared page into the user process for checking if event is enabled.
  */
 static int user_status_mmap(struct file *file, struct vm_area_struct *vma)
 {
+	char *pages;
+	struct user_event_group *group = user_status_group(file);
 	unsigned long size = vma->vm_end - vma->vm_start;
-	if (size != MAX_EVENTS)
+	if (size != MAX_BYTES)
+		return -EINVAL;
+	if (!group)
 		return -EINVAL;
+	pages = group->register_page_data;
 	return remap_pfn_range(vma, vma->vm_start,
-			       virt_to_phys(register_page_data) >> PAGE_SHIFT,
+			       virt_to_phys(pages) >> PAGE_SHIFT,
 			       size, vm_get_page_prot(VM_READ));
 }
 static int user_seq_show(struct seq_file *m, void *p)
 {
+	struct user_event_group *group = m->private;
 	struct user_event *user;
 	char status;
 	int i, active = 0, busy = 0, flags;
-	mutex_lock(&reg_mutex);
+	if (!group)
+		return -EINVAL;
+	mutex_lock(&group->reg_mutex);
-	hash_for_each(register_table, i, user, node) {
+	hash_for_each(group->register_table, i, user, node) {
-		status = register_page_data[user->index];
+		status = user->status;
 		flags = user->flags;
 		seq_printf(m, "%d:%s", user->index, EVENT_NAME(user));
 		active++;
 	}
-	mutex_unlock(&reg_mutex);
+	mutex_unlock(&group->reg_mutex);
 	seq_puts(m, "\n");
 	seq_printf(m, "Active: %d\n", active);
 static int user_status_open(struct inode *node, struct file *file)
 {
-	return seq_open(file, &user_seq_ops);
+	struct user_event_group *group;
+	int ret;
+	group = current_user_event_group();
+	if (!group)
+		return -ENOENT;
+	ret = seq_open(file, &user_seq_ops);
+	if (!ret) {
+		/* Chain group to seq_file */
+		struct seq_file *m = file->private_data;
+		m->private = group;
+	}
+	return ret;
 }
 static const struct file_operations user_status_fops = {
 	return -ENODEV;
 }
-static void set_page_reservations(bool set)
-{
-	int page;
-	for (page = 0; page < MAX_PAGES; ++page) {
-		void *addr = register_page_data + (PAGE_SIZE * page);
-		if (set)
-			SetPageReserved(virt_to_page(addr));
-		else
-			ClearPageReserved(virt_to_page(addr));
-	}
-}
 static int __init trace_events_user_init(void)
 {
-	struct page *pages;
 	int ret;
-	/* Zero all bits beside 0 (which is reserved for failures) */
+	init_group = user_event_group_create(&init_user_ns);
-	bitmap_zero(page_bitmap, MAX_EVENTS);
-	set_bit(0, page_bitmap);
-	pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, MAX_PAGE_ORDER);
+	if (!init_group)
-	if (!pages)
 		return -ENOMEM;
-	register_page_data = page_address(pages);
-	set_page_reservations(true);
 	ret = create_user_tracefs();
 	if (ret) {
 		pr_warn("user_events could not register with tracefs\n");
-		set_page_reservations(false);
+		user_event_group_destroy(init_group);
-		__free_pages(pages, MAX_PAGE_ORDER);
+		init_group = NULL;
 		return ret;
 	}

kernel/trace/trace_osnoise.c+2−1

 	for_each_cpu(cpu, current_mask) {
 		retval = start_kthread(cpu);
 		if (retval) {
+			cpus_read_unlock();
 			stop_per_cpu_kthreads();
-			break;
+			return retval;
 		}
 	}

kernel/trace/trace_probe.h+2−1

@@ -445,7 +445,8 @@ extern int traceprobe_define_arg_fields(struct trace_event_call *event_call, · 444 unmodified lines
445	C(SAME_PROBE, "There is already the exact same probe event"),\	445	C(SAME_PROBE, "There is already the exact same probe event"),\
446	C(NO_EVENT_INFO, "This requires both group and event name to attach"),\	446	C(NO_EVENT_INFO, "This requires both group and event name to attach"),\
447	C(BAD_ATTACH_EVENT, "Attached event does not exist"),\	447	C(BAD_ATTACH_EVENT, "Attached event does not exist"),\
448	C(BAD_ATTACH_ARG, "Attached event does not have this field"),	448	C(BAD_ATTACH_ARG, "Attached event does not have this field"),\
		449	C(NO_EP_FILTER, "No filter rule after 'if'"),
449		450
450	#undef C	451	#undef C
451	#define C(a, b) TP_ERR_##a	452	#define C(a, b) TP_ERR_##a

kernel/trace/tracing_map.c+2−3

 static void detect_dups(struct tracing_map_sort_entry **sort_entries,
 		      int n_entries, unsigned int key_size)
 {
-	unsigned int dups = 0, total_dups = 0;
+	unsigned int total_dups = 0;
 	int i;
 	void *key;
 	key = sort_entries[0]->key;
 	for (i = 1; i < n_entries; i++) {
 		if (!memcmp(sort_entries[i]->key, key, key_size)) {
-			dups++; total_dups++;
+			total_dups++;
 			continue;
 		}
 		key = sort_entries[i]->key;
-		dups = 0;
 	}
 	WARN_ONCE(total_dups > 0,

kernel/tracepoint.c+6−8

@@ -640,7 +640,6 @@ static void tp_module_going_check_quiescent(struct tracepoint tp, void priv) · 639 unmodified lines
640	static int tracepoint_module_coming(struct module *mod)	640	static int tracepoint_module_coming(struct module *mod)
641	{	641	{
642	struct tp_module *tp_mod;	642	struct tp_module *tp_mod;
643	int ret = 0;
644		643
645	if (!mod->num_tracepoints)	644	if (!mod->num_tracepoints)
646	return 0;	645	return 0;
@@ -652,19 +651,18 @@ static int tracepoint_module_coming(struct module *mod) · 5 unmodified lines
652	*/	651	*/
653	if (trace_module_has_bad_taint(mod))	652	if (trace_module_has_bad_taint(mod))
654	return 0;	653	return 0;
655	mutex_lock(&tracepoint_module_list_mutex);	654
656	tp_mod = kmalloc(sizeof(struct tp_module), GFP_KERNEL);	655	tp_mod = kmalloc(sizeof(struct tp_module), GFP_KERNEL);
657	if (!tp_mod) {	656	if (!tp_mod)
658	ret = -ENOMEM;	657	return -ENOMEM;
659	goto end;
660	}
661	tp_mod->mod = mod;	658	tp_mod->mod = mod;
		659
		660	mutex_lock(&tracepoint_module_list_mutex);
662	list_add_tail(&tp_mod->list, &tracepoint_module_list);	661	list_add_tail(&tp_mod->list, &tracepoint_module_list);
663	blocking_notifier_call_chain(&tracepoint_notify_list,	662	blocking_notifier_call_chain(&tracepoint_notify_list,
664	MODULE_STATE_COMING, tp_mod);	663	MODULE_STATE_COMING, tp_mod);
665	end:
666	mutex_unlock(&tracepoint_module_list_mutex);	664	mutex_unlock(&tracepoint_module_list_mutex);
667	return ret;	665	return 0;
668	}	666	}
669		667
670	static void tracepoint_module_going(struct module *mod)	668	static void tracepoint_module_going(struct module *mod)