Merge tag 'trace-v6.1' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace

Pull tracing updates from Steven Rostedt: "Major changes: - Changed location of tracing repo from personal git repo to: git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git - Added Masami Hiramatsu as co-maintainer - Updated MAINTAINERS file to separate out FTRACE as it is more than just TRACING. Minor changes: - Added Mark Rutland as FTRACE reviewer - Updated user_events to make it on its way to remove the BROKEN tag. The changes should now be acceptable but will run it through a cycle and hopefully we can remove the BROKEN tag next release. - Added filtering to eprobes - Added a delta time to the benchmark trace event - Have the histogram and filter callbacks called via a switch statement instead of indirect functions. This speeds it up to avoid retpolines. - Add a way to wake up ring buffer waiters waiting for the ring buffer to fill up to its watermark. - New ioctl() on the trace_pipe_raw file to wake up ring buffer waiters. - Wake up waiters when the ring buffer is disabled. A reader may block when the ring buffer is disabled, but if it was blocked when the ring buffer is disabled it should then wake up. Fixes: - Allow splice to read partially read ring buffer pages. This fixes splice never moving forward. - Fix inverted compare that made the "shortest" ring buffer wait queue actually the longest. - Fix a race in the ring buffer between resetting a page when a writer goes to another page, and the reader. - Fix ftrace accounting bug when function hooks are added at boot up before the weak functions are set to "disabled". - Fix bug that freed a user allocated snapshot buffer when enabling a tracer. - Fix possible recursive locks in osnoise tracer - Fix recursive locking direct functions - Other minor clean ups and fixes" * tag 'trace-v6.1' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace: (44 commits) ftrace: Create separate entry in MAINTAINERS for function hooks tracing: Update MAINTAINERS to reflect new tracing git repo tracing: Do not free snapshot if tracer is on cmdline ftrace: Still disable enabled records marked as disabled tracing/user_events: Move pages/locks into groups to prepare for namespaces tracing: Add Masami Hiramatsu as co-maintainer tracing: Remove unused variable 'dups' MAINTAINERS: add myself as a tracing reviewer ring-buffer: Fix race between reset page and reading page tracing/user_events: Update ABI documentation to align to bits vs bytes tracing/user_events: Use bits vs bytes for enabled status page data tracing/user_events: Use refcount instead of atomic for ref tracking tracing/user_events: Ensure user provided strings are safely formatted tracing/user_events: Use WRITE instead of READ for io vector import tracing/user_events: Use NULL for strstr checks tracing: Fix spelling mistake "preapre" -> "prepare" tracing: Wake up waiters when tracing is disabled tracing: Add ioctl() to force ring buffer waiters to wake up tracing: Wake up ring buffer waiters on closing of the file ring-buffer: Add ring_buffer_wake_waiters() ...

Linus Torvalds committed 4 years ago

Commit cdf072a2 parents dc55342 4f881a634 files · +1299 −486Browse files

Showing the changes to kernel (17 of 34 files). Show all

Unified · Split

17 files

kernel/trace/ftrace.c +27 −7
kernel/trace/kprobe_event_gen_test.c +44 −5
kernel/trace/ring_buffer.c +84 −3
kernel/trace/rv/monitors/wip/wip.c +4 −4
kernel/trace/rv/monitors/wwnr/wwnr.c +4 −4
kernel/trace/trace.c +73 −5
kernel/trace/trace.h +0 −13
kernel/trace/trace_benchmark.c +1 −1
kernel/trace/trace_benchmark.h +5 −3
kernel/trace/trace_eprobe.c +97 −10
kernel/trace/trace_events_filter.c +168 −71
kernel/trace/trace_events_hist.c +169 −77
kernel/trace/trace_events_user.c +426 −142
kernel/trace/trace_osnoise.c +2 −1
kernel/trace/trace_probe.h +2 −1
kernel/trace/tracing_map.c +2 −3
kernel/tracepoint.c +6 −8

kernel/trace/ftrace.c+27−7

@@ -1644,6 +1644,18 @@ ftrace_find_tramp_ops_any_other(struct dyn_ftrace rec, struct ftrace_ops op_ex · 1643 unmodified lines
1644	1644	static struct ftrace_ops *
1645	1645	ftrace_find_tramp_ops_next(struct dyn_ftrace rec, struct ftrace_ops ops);
1646	1646
	1647	+static bool skip_record(struct dyn_ftrace *rec)
	1648	+{
	1649	+ /*
	1650	+ * At boot up, weak functions are set to disable. Function tracing
	1651	+ * can be enabled before they are, and they still need to be disabled now.
	1652	+ * If the record is disabled, still continue if it is marked as already
	1653	+ * enabled (this is needed to keep the accounting working).
	1654	+ */
	1655	+ return rec->flags & FTRACE_FL_DISABLED &&
	1656	+ !(rec->flags & FTRACE_FL_ENABLED);
	1657	+}
	1658	+
1647	1659	static bool __ftrace_hash_rec_update(struct ftrace_ops *ops,
1648	1660	int filter_hash,
1649	1661	bool inc)
@@ -1693,7 +1705,7 @@ static bool __ftrace_hash_rec_update(struct ftrace_ops *ops, · 43 unmodified lines
1693	1705	int in_hash = 0;
1694	1706	int match = 0;
1695	1707
1696		− if (rec->flags & FTRACE_FL_DISABLED)
	1708	+ if (skip_record(rec))
1697	1709	continue;
1698	1710
1699	1711	if (all) {
@@ -2126,7 +2138,7 @@ static int ftrace_check_record(struct dyn_ftrace *rec, bool enable, bool update) · 426 unmodified lines
2126	2138
2127	2139	ftrace_bug_type = FTRACE_BUG_UNKNOWN;
2128	2140
2129		− if (rec->flags & FTRACE_FL_DISABLED)
	2141	+ if (skip_record(rec))
2130	2142	return FTRACE_UPDATE_IGNORE;
2131	2143
2132	2144	/*
@@ -2241,7 +2253,7 @@ static int ftrace_check_record(struct dyn_ftrace *rec, bool enable, bool update) · 108 unmodified lines
2241	2253	if (update) {
2242	2254	/* If there's no more users, clear all flags */
2243	2255	if (!ftrace_rec_count(rec))
2244		− rec->flags = 0;
	2256	+ rec->flags &= FTRACE_FL_DISABLED;
2245	2257	else
2246	2258	/*
2247	2259	* Just disable the record, but keep the ops TRAMP
@@ -2634,7 +2646,7 @@ void __weak ftrace_replace_code(int mod_flags) · 386 unmodified lines
2634	2646
2635	2647	do_for_each_ftrace_rec(pg, rec) {
2636	2648
2637		− if (rec->flags & FTRACE_FL_DISABLED)
	2649	+ if (skip_record(rec))
2638	2650	continue;
2639	2651
2640	2652	failed = __ftrace_replace_code(rec, enable);
@@ -5427,6 +5439,8 @@ static struct ftrace_ops stub_ops = { · 2786 unmodified lines
5427	5439	* it is safe to modify the ftrace record, where it should be
5428	5440	* currently calling @old_addr directly, to call @new_addr.
5429	5441	*
	5442	+ * This is called with direct_mutex locked.
	5443	+ *
5430	5444	* Safety checks should be made to make sure that the code at
5431	5445	* @rec->ip is currently calling @old_addr. And this must
5432	5446	* also update entry->direct to @new_addr.
@@ -5439,6 +5453,8 @@ int __weak ftrace_modify_direct_caller(struct ftrace_func_entry *entry, · 6 unmodified lines
5439	5453	unsigned long ip = rec->ip;
5440	5454	int ret;
5441	5455
	5456	+ lockdep_assert_held(&direct_mutex);
	5457	+
5442	5458	/*
5443	5459	* The ftrace_lock was used to determine if the record
5444	5460	* had more than one registered user to it. If it did,
@@ -5461,7 +5477,7 @@ int __weak ftrace_modify_direct_caller(struct ftrace_func_entry *entry, · 16 unmodified lines
5461	5477	if (ret)
5462	5478	goto out_lock;
5463	5479
5464		− ret = register_ftrace_function(&stub_ops);
	5480	+ ret = register_ftrace_function_nolock(&stub_ops);
5465	5481	if (ret) {
5466	5482	ftrace_set_filter_ip(&stub_ops, ip, 1, 0);
5467	5483	goto out_lock;
@@ -6081,8 +6097,12 @@ int ftrace_regex_release(struct inode inode, struct file file) · 613 unmodified lines
6081	6097
6082	6098	if (filter_hash) {
6083	6099	orig_hash = &iter->ops->func_hash->filter_hash;
6084		− if (iter->tr && !list_empty(&iter->tr->mod_trace))
6085		− iter->hash->flags \|= FTRACE_HASH_FL_MOD;
	6100	+ if (iter->tr) {
	6101	+ if (list_empty(&iter->tr->mod_trace))
	6102	+ iter->hash->flags &= ~FTRACE_HASH_FL_MOD;
	6103	+ else
	6104	+ iter->hash->flags \|= FTRACE_HASH_FL_MOD;
	6105	+ }
6086	6106	} else
6087	6107	orig_hash = &iter->ops->func_hash->notrace_hash;
6088	6108

kernel/trace/kprobe_event_gen_test.c+44−5

@@ -35,6 +35,45 @@ · 34 unmodified lines
35	35	static struct trace_event_file *gen_kprobe_test;
36	36	static struct trace_event_file *gen_kretprobe_test;
37	37
	38	+#define KPROBE_GEN_TEST_FUNC "do_sys_open"
	39	+
	40	+/* X86 */
	41	+#if defined(CONFIG_X86_64) \|\| defined(CONFIG_X86_32)
	42	+#define KPROBE_GEN_TEST_ARG0 "dfd=%ax"
	43	+#define KPROBE_GEN_TEST_ARG1 "filename=%dx"
	44	+#define KPROBE_GEN_TEST_ARG2 "flags=%cx"
	45	+#define KPROBE_GEN_TEST_ARG3 "mode=+4($stack)"
	46	+
	47	+/* ARM64 */
	48	+#elif defined(CONFIG_ARM64)
	49	+#define KPROBE_GEN_TEST_ARG0 "dfd=%x0"
	50	+#define KPROBE_GEN_TEST_ARG1 "filename=%x1"
	51	+#define KPROBE_GEN_TEST_ARG2 "flags=%x2"
	52	+#define KPROBE_GEN_TEST_ARG3 "mode=%x3"
	53	+
	54	+/* ARM */
	55	+#elif defined(CONFIG_ARM)
	56	+#define KPROBE_GEN_TEST_ARG0 "dfd=%r0"
	57	+#define KPROBE_GEN_TEST_ARG1 "filename=%r1"
	58	+#define KPROBE_GEN_TEST_ARG2 "flags=%r2"
	59	+#define KPROBE_GEN_TEST_ARG3 "mode=%r3"
	60	+
	61	+/* RISCV */
	62	+#elif defined(CONFIG_RISCV)
	63	+#define KPROBE_GEN_TEST_ARG0 "dfd=%a0"
	64	+#define KPROBE_GEN_TEST_ARG1 "filename=%a1"
	65	+#define KPROBE_GEN_TEST_ARG2 "flags=%a2"
	66	+#define KPROBE_GEN_TEST_ARG3 "mode=%a3"
	67	+
	68	+/* others */
	69	+#else
	70	+#define KPROBE_GEN_TEST_ARG0 NULL
	71	+#define KPROBE_GEN_TEST_ARG1 NULL
	72	+#define KPROBE_GEN_TEST_ARG2 NULL
	73	+#define KPROBE_GEN_TEST_ARG3 NULL
	74	+#endif
	75	+
	76	+
38	77	/*
39	78	* Test to make sure we can create a kprobe event, then add more
40	79	* fields.
@@ -58,14 +97,14 @@ static int __init test_gen_kprobe_cmd(void) · 17 unmodified lines
58	97	* fields.
59	98	*/
60	99	ret = kprobe_event_gen_cmd_start(&cmd, "gen_kprobe_test",
61		− "do_sys_open",
62		− "dfd=%ax", "filename=%dx");
	100	+ KPROBE_GEN_TEST_FUNC,
	101	+ KPROBE_GEN_TEST_ARG0, KPROBE_GEN_TEST_ARG1);
63	102	if (ret)
64	103	goto free;
65	104
66	105	/* Use kprobe_event_add_fields to add the rest of the fields */
67	106
68		− ret = kprobe_event_add_fields(&cmd, "flags=%cx", "mode=+4($stack)");
	107	+ ret = kprobe_event_add_fields(&cmd, KPROBE_GEN_TEST_ARG2, KPROBE_GEN_TEST_ARG3);
69	108	if (ret)
70	109	goto free;
71	110
@@ -128,7 +167,7 @@ static int __init test_gen_kretprobe_cmd(void) · 56 unmodified lines
128	167	* Define the kretprobe event.
129	168	*/
130	169	ret = kretprobe_event_gen_cmd_start(&cmd, "gen_kretprobe_test",
131		− "do_sys_open",
	170	+ KPROBE_GEN_TEST_FUNC,
132	171	"$retval");
133	172	if (ret)
134	173	goto free;
@@ -206,7 +245,7 @@ static void __exit kprobe_event_gen_test_exit(void) · 71 unmodified lines
206	245	WARN_ON(kprobe_event_delete("gen_kprobe_test"));
207	246
208	247	/* Disable the event or you can't remove it */
209		− WARN_ON(trace_array_set_clr_event(gen_kprobe_test->tr,
	248	+ WARN_ON(trace_array_set_clr_event(gen_kretprobe_test->tr,
210	249	"kprobes",
211	250	"gen_kretprobe_test", false));
212	251

kernel/trace/ring_buffer.c+84−3

@@ -413,6 +413,7 @@ struct rb_irq_work { · 412 unmodified lines
413	413	struct irq_work work;
414	414	wait_queue_head_t waiters;
415	415	wait_queue_head_t full_waiters;
	416	+ long wait_index;
416	417	bool waiters_pending;
417	418	bool full_waiters_pending;
418	419	bool wakeup_full;
@@ -917,12 +918,44 @@ static void rb_wake_up_waiters(struct irq_work *work) · 498 unmodified lines
917	918	struct rb_irq_work *rbwork = container_of(work, struct rb_irq_work, work);
918	919
919	920	wake_up_all(&rbwork->waiters);
920		− if (rbwork->wakeup_full) {
	921	+ if (rbwork->full_waiters_pending \|\| rbwork->wakeup_full) {
921	922	rbwork->wakeup_full = false;
	923	+ rbwork->full_waiters_pending = false;
922	924	wake_up_all(&rbwork->full_waiters);
923	925	}
924	926	}
925	927
	928	+/**
	929	+ * ring_buffer_wake_waiters - wake up any waiters on this ring buffer
	930	+ * @buffer: The ring buffer to wake waiters on
	931	+ *
	932	+ * In the case of a file that represents a ring buffer is closing,
	933	+ * it is prudent to wake up any waiters that are on this.
	934	+ */
	935	+void ring_buffer_wake_waiters(struct trace_buffer *buffer, int cpu)
	936	+{
	937	+ struct ring_buffer_per_cpu *cpu_buffer;
	938	+ struct rb_irq_work *rbwork;
	939	+
	940	+ if (cpu == RING_BUFFER_ALL_CPUS) {
	941	+
	942	+ /* Wake up individual ones too. One level recursion */
	943	+ for_each_buffer_cpu(buffer, cpu)
	944	+ ring_buffer_wake_waiters(buffer, cpu);
	945	+
	946	+ rbwork = &buffer->irq_work;
	947	+ } else {
	948	+ cpu_buffer = buffer->buffers[cpu];
	949	+ rbwork = &cpu_buffer->irq_work;
	950	+ }
	951	+
	952	+ rbwork->wait_index++;
	953	+ /* make sure the waiters see the new index */
	954	+ smp_wmb();
	955	+
	956	+ rb_wake_up_waiters(&rbwork->work);
	957	+}
	958	+
926	959	/**
927	960	* ring_buffer_wait - wait for input to the ring buffer
928	961	* @buffer: buffer to wait on
@@ -938,6 +971,7 @@ int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full) · 9 unmodified lines
938	971	struct ring_buffer_per_cpu *cpu_buffer;
939	972	DEFINE_WAIT(wait);
940	973	struct rb_irq_work *work;
	974	+ long wait_index;
941	975	int ret = 0;
942	976
943	977	/*
@@ -956,6 +990,7 @@ int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full) · 12 unmodified lines
956	990	work = &cpu_buffer->irq_work;
957	991	}
958	992
	993	+ wait_index = READ_ONCE(work->wait_index);
959	994
960	995	while (true) {
961	996	if (full)
@@ -1011,7 +1046,7 @@ int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full) · 49 unmodified lines
1011	1046	nr_pages = cpu_buffer->nr_pages;
1012	1047	dirty = ring_buffer_nr_dirty_pages(buffer, cpu);
1013	1048	if (!cpu_buffer->shortest_full \|\|
1014		− cpu_buffer->shortest_full < full)
	1049	+ cpu_buffer->shortest_full > full)
1015	1050	cpu_buffer->shortest_full = full;
1016	1051	raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
1017	1052	if (!pagebusy &&
@@ -1020,6 +1055,11 @@ int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full) · 2 unmodified lines
1020	1055	}
1021	1056
1022	1057	schedule();
	1058	+
	1059	+ /* Make sure to see the new wait index */
	1060	+ smp_rmb();
	1061	+ if (wait_index != work->wait_index)
	1062	+ break;
1023	1063	}
1024	1064
1025	1065	if (full)
@@ -2608,6 +2648,9 @@ rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer, · 1582 unmodified lines
2608	2648	/* Mark the rest of the page with padding */
2609	2649	rb_event_set_padding(event);
2610	2650
	2651	+ /* Make sure the padding is visible before the write update */
	2652	+ smp_wmb();
	2653	+
2611	2654	/* Set the write back to the previous setting */
2612	2655	local_sub(length, &tail_page->write);
2613	2656	return;
@@ -2619,6 +2662,9 @@ rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer, · 5 unmodified lines
2619	2662	/* time delta must be non zero */
2620	2663	event->time_delta = 1;
2621	2664
	2665	+ /* Make sure the padding is visible before the tail_page->write update */
	2666	+ smp_wmb();
	2667	+
2622	2668	/* Set write to end of buffer */
2623	2669	length = (tail + length) - BUF_PAGE_SIZE;
2624	2670	local_sub(length, &tail_page->write);
@@ -4587,6 +4633,33 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) · 1962 unmodified lines
4587	4633	arch_spin_unlock(&cpu_buffer->lock);
4588	4634	local_irq_restore(flags);
4589	4635
	4636	+ /*
	4637	+ * The writer has preempt disable, wait for it. But not forever
	4638	+ * Although, 1 second is pretty much "forever"
	4639	+ */
	4640	+#define USECS_WAIT 1000000
	4641	+ for (nr_loops = 0; nr_loops < USECS_WAIT; nr_loops++) {
	4642	+ /* If the write is past the end of page, a writer is still updating it */
	4643	+ if (likely(!reader \|\| rb_page_write(reader) <= BUF_PAGE_SIZE))
	4644	+ break;
	4645	+
	4646	+ udelay(1);
	4647	+
	4648	+ /* Get the latest version of the reader write value */
	4649	+ smp_rmb();
	4650	+ }
	4651	+
	4652	+ /* The writer is not moving forward? Something is wrong */
	4653	+ if (RB_WARN_ON(cpu_buffer, nr_loops == USECS_WAIT))
	4654	+ reader = NULL;
	4655	+
	4656	+ /*
	4657	+ * Make sure we see any padding after the write update
	4658	+ * (see rb_reset_tail())
	4659	+ */
	4660	+ smp_rmb();
	4661	+
	4662	+
4590	4663	return reader;
4591	4664	}
4592	4665
@@ -5616,7 +5689,15 @@ int ring_buffer_read_page(struct trace_buffer *buffer, · 1023 unmodified lines
5616	5689	unsigned int pos = 0;
5617	5690	unsigned int size;
5618	5691
5619		− if (full)
	5692	+ /*
	5693	+ * If a full page is expected, this can still be returned
	5694	+ * if there's been a previous partial read and the
	5695	+ * rest of the page can be read and the commit page is off
	5696	+ * the reader page.
	5697	+ */
	5698	+ if (full &&
	5699	+ (!read \|\| (len < (commit - read)) \|\|
	5700	+ cpu_buffer->reader_page == cpu_buffer->commit_page))
5620	5701	goto out_unlock;
5621	5702
5622	5703	if (len > (commit - read))

kernel/trace/rv/monitors/wip/wip.c+4−4

@@ -16,7 +16,7 @@ · 15 unmodified lines
16	16
17	17	#include "wip.h"
18	18
19		−struct rv_monitor rv_wip;
	19	+static struct rv_monitor rv_wip;
20	20	DECLARE_DA_MON_PER_CPU(wip, unsigned char);
21	21
22	22	static void handle_preempt_disable(void *data, unsigned long ip, unsigned long parent_ip)
@@ -60,7 +60,7 @@ static void disable_wip(void) · 37 unmodified lines
60	60	da_monitor_destroy_wip();
61	61	}
62	62
63		−struct rv_monitor rv_wip = {
	63	+static struct rv_monitor rv_wip = {
64	64	.name = "wip",
65	65	.description = "wakeup in preemptive per-cpu testing monitor.",
66	66	.enable = enable_wip,
@@ -69,13 +69,13 @@ struct rv_monitor rv_wip = { · 2 unmodified lines
69	69	.enabled = 0,
70	70	};
71	71
72		−static int register_wip(void)
	72	+static int __init register_wip(void)
73	73	{
74	74	rv_register_monitor(&rv_wip);
75	75	return 0;
76	76	}
77	77
78		−static void unregister_wip(void)
	78	+static void __exit unregister_wip(void)
79	79	{
80	80	rv_unregister_monitor(&rv_wip);
81	81	}

kernel/trace/rv/monitors/wwnr/wwnr.c+4−4

@@ -15,7 +15,7 @@ · 14 unmodified lines
15	15
16	16	#include "wwnr.h"
17	17
18		−struct rv_monitor rv_wwnr;
	18	+static struct rv_monitor rv_wwnr;
19	19	DECLARE_DA_MON_PER_TASK(wwnr, unsigned char);
20	20
21	21	static void handle_switch(void data, bool preempt, struct task_struct p,
@@ -59,7 +59,7 @@ static void disable_wwnr(void) · 37 unmodified lines
59	59	da_monitor_destroy_wwnr();
60	60	}
61	61
62		−struct rv_monitor rv_wwnr = {
	62	+static struct rv_monitor rv_wwnr = {
63	63	.name = "wwnr",
64	64	.description = "wakeup while not running per-task testing model.",
65	65	.enable = enable_wwnr,
@@ -68,13 +68,13 @@ struct rv_monitor rv_wwnr = { · 2 unmodified lines
68	68	.enabled = 0,
69	69	};
70	70
71		−static int register_wwnr(void)
	71	+static int __init register_wwnr(void)
72	72	{
73	73	rv_register_monitor(&rv_wwnr);
74	74	return 0;
75	75	}
76	76
77		−static void unregister_wwnr(void)
	77	+static void __exit unregister_wwnr(void)
78	78	{
79	79	rv_unregister_monitor(&rv_wwnr);
80	80	}

kernel/trace/trace.c+73−5

@@ -1193,12 +1193,14 @@ void tracing_cond_snapshot_data(struct trace_array tr) · 1192 unmodified lines
1193	1193	{
1194	1194	void *cond_data = NULL;
1195	1195
	1196	+ local_irq_disable();
1196	1197	arch_spin_lock(&tr->max_lock);
1197	1198
1198	1199	if (tr->cond_snapshot)
1199	1200	cond_data = tr->cond_snapshot->cond_data;
1200	1201
1201	1202	arch_spin_unlock(&tr->max_lock);
	1203	+ local_irq_enable();
1202	1204
1203	1205	return cond_data;
1204	1206	}
@@ -1334,9 +1336,11 @@ int tracing_snapshot_cond_enable(struct trace_array tr, void cond_data, · 129 unmodified lines
1334	1336	goto fail_unlock;
1335	1337	}
1336	1338
	1339	+ local_irq_disable();
1337	1340	arch_spin_lock(&tr->max_lock);
1338	1341	tr->cond_snapshot = cond_snapshot;
1339	1342	arch_spin_unlock(&tr->max_lock);
	1343	+ local_irq_enable();
1340	1344
1341	1345	mutex_unlock(&trace_types_lock);
1342	1346
@@ -1363,6 +1367,7 @@ int tracing_snapshot_cond_disable(struct trace_array *tr) · 20 unmodified lines
1363	1367	{
1364	1368	int ret = 0;
1365	1369
	1370	+ local_irq_disable();
1366	1371	arch_spin_lock(&tr->max_lock);
1367	1372
1368	1373	if (!tr->cond_snapshot)
@@ -1373,6 +1378,7 @@ int tracing_snapshot_cond_disable(struct trace_array *tr) · 4 unmodified lines
1373	1378	}
1374	1379
1375	1380	arch_spin_unlock(&tr->max_lock);
	1381	+ local_irq_enable();
1376	1382
1377	1383	return ret;
1378	1384	}
@@ -2200,6 +2206,11 @@ static size_t tgid_map_max; · 821 unmodified lines
2200	2206
2201	2207	#define SAVED_CMDLINES_DEFAULT 128
2202	2208	#define NO_CMDLINE_MAP UINT_MAX
	2209	+/*
	2210	+ * Preemption must be disabled before acquiring trace_cmdline_lock.
	2211	+ * The various trace_arrays' max_lock must be acquired in a context
	2212	+ * where interrupt is disabled.
	2213	+ */
2203	2214	static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
2204	2215	struct saved_cmdlines_buffer {
2205	2216	unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
@@ -2412,7 +2423,11 @@ static int trace_save_cmdline(struct task_struct *tsk) · 206 unmodified lines
2412	2423	* the lock, but we also don't want to spin
2413	2424	* nor do we want to disable interrupts,
2414	2425	* so if we miss here, then better luck next time.
	2426	+ *
	2427	+ * This is called within the scheduler and wake up, so interrupts
	2428	+ * had better been disabled and run queue lock been held.
2415	2429	*/
	2430	+ lockdep_assert_preemption_disabled();
2416	2431	if (!arch_spin_trylock(&trace_cmdline_lock))
2417	2432	return 0;
2418	2433
@@ -5890,9 +5905,11 @@ tracing_saved_cmdlines_size_read(struct file filp, char __user ubuf, · 3471 unmodified lines
5890	5905	char buf[64];
5891	5906	int r;
5892	5907
	5908	+ preempt_disable();
5893	5909	arch_spin_lock(&trace_cmdline_lock);
5894	5910	r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
5895	5911	arch_spin_unlock(&trace_cmdline_lock);
	5912	+ preempt_enable();
5896	5913
5897	5914	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5898	5915	}
@@ -5917,10 +5934,12 @@ static int tracing_resize_saved_cmdlines(unsigned int val) · 18 unmodified lines
5917	5934	return -ENOMEM;
5918	5935	}
5919	5936
	5937	+ preempt_disable();
5920	5938	arch_spin_lock(&trace_cmdline_lock);
5921	5939	savedcmd_temp = savedcmd;
5922	5940	savedcmd = s;
5923	5941	arch_spin_unlock(&trace_cmdline_lock);
	5942	+ preempt_enable();
5924	5943	free_saved_cmdlines_buffer(savedcmd_temp);
5925	5944
5926	5945	return 0;
@@ -6373,10 +6392,12 @@ int tracing_set_tracer(struct trace_array tr, const char buf) · 446 unmodified lines
6373	6392
6374	6393	#ifdef CONFIG_TRACER_SNAPSHOT
6375	6394	if (t->use_max_tr) {
	6395	+ local_irq_disable();
6376	6396	arch_spin_lock(&tr->max_lock);
6377	6397	if (tr->cond_snapshot)
6378	6398	ret = -EBUSY;
6379	6399	arch_spin_unlock(&tr->max_lock);
	6400	+ local_irq_enable();
6380	6401	if (ret)
6381	6402	goto out;
6382	6403	}
@@ -6407,12 +6428,12 @@ int tracing_set_tracer(struct trace_array tr, const char buf) · 24 unmodified lines
6407	6428	if (tr->current_trace->reset)
6408	6429	tr->current_trace->reset(tr);
6409	6430
	6431	+#ifdef CONFIG_TRACER_MAX_TRACE
	6432	+ had_max_tr = tr->current_trace->use_max_tr;
	6433	+
6410	6434	/* Current trace needs to be nop_trace before synchronize_rcu */
6411	6435	tr->current_trace = &nop_trace;
6412	6436
6413		−#ifdef CONFIG_TRACER_MAX_TRACE
6414		− had_max_tr = tr->allocated_snapshot;
6415		−
6416	6437	if (had_max_tr && !t->use_max_tr) {
6417	6438	/*
6418	6439	* We need to make sure that the update_max_tr sees that
@@ -6425,11 +6446,13 @@ int tracing_set_tracer(struct trace_array tr, const char buf) · 6 unmodified lines
6425	6446	free_snapshot(tr);
6426	6447	}
6427	6448
6428		− if (t->use_max_tr && !had_max_tr) {
	6449	+ if (t->use_max_tr && !tr->allocated_snapshot) {
6429	6450	ret = tracing_alloc_snapshot_instance(tr);
6430	6451	if (ret < 0)
6431	6452	goto out;
6432	6453	}
	6454	+#else
	6455	+ tr->current_trace = &nop_trace;
6433	6456	#endif
6434	6457
6435	6458	if (t->init) {
@@ -7436,10 +7459,12 @@ tracing_snapshot_write(struct file filp, const char __user ubuf, size_t cnt, · 1000 unmodified lines
7436	7459	goto out;
7437	7460	}
7438	7461
	7462	+ local_irq_disable();
7439	7463	arch_spin_lock(&tr->max_lock);
7440	7464	if (tr->cond_snapshot)
7441	7465	ret = -EBUSY;
7442	7466	arch_spin_unlock(&tr->max_lock);
	7467	+ local_irq_enable();
7443	7468	if (ret)
7444	7469	goto out;
7445	7470
@@ -8137,6 +8162,12 @@ static int tracing_buffers_release(struct inode inode, struct file file) · 691 unmodified lines
8137	8162
8138	8163	__trace_array_put(iter->tr);
8139	8164
	8165	+ iter->wait_index++;
	8166	+ /* Make sure the waiters see the new wait_index */
	8167	+ smp_wmb();
	8168	+
	8169	+ ring_buffer_wake_waiters(iter->array_buffer->buffer, iter->cpu_file);
	8170	+
8140	8171	if (info->spare)
8141	8172	ring_buffer_free_read_page(iter->array_buffer->buffer,
8142	8173	info->spare_cpu, info->spare);
@@ -8290,6 +8321,8 @@ tracing_buffers_splice_read(struct file file, loff_t ppos, · 147 unmodified lines
8290	8321
8291	8322	/* did we read anything? */
8292	8323	if (!spd.nr_pages) {
	8324	+ long wait_index;
	8325	+
8293	8326	if (ret)
8294	8327	goto out;
8295	8328
@@ -8297,10 +8330,21 @@ tracing_buffers_splice_read(struct file file, loff_t ppos, · 1 unmodified line
8297	8330	if ((file->f_flags & O_NONBLOCK) \|\| (flags & SPLICE_F_NONBLOCK))
8298	8331	goto out;
8299	8332
	8333	+ wait_index = READ_ONCE(iter->wait_index);
	8334	+
8300	8335	ret = wait_on_pipe(iter, iter->tr->buffer_percent);
8301	8336	if (ret)
8302	8337	goto out;
8303	8338
	8339	+ /* No need to wait after waking up when tracing is off */
	8340	+ if (!tracer_tracing_is_on(iter->tr))
	8341	+ goto out;
	8342	+
	8343	+ /* Make sure we see the new wait_index */
	8344	+ smp_rmb();
	8345	+ if (wait_index != iter->wait_index)
	8346	+ goto out;
	8347	+
8304	8348	goto again;
8305	8349	}
8306	8350
@@ -8311,12 +8355,34 @@ out: · 4 unmodified lines
8311	8355	return ret;
8312	8356	}
8313	8357
	8358	+/* An ioctl call with cmd 0 to the ring buffer file will wake up all waiters */
	8359	+static long tracing_buffers_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
	8360	+{
	8361	+ struct ftrace_buffer_info *info = file->private_data;
	8362	+ struct trace_iterator *iter = &info->iter;
	8363	+
	8364	+ if (cmd)
	8365	+ return -ENOIOCTLCMD;
	8366	+
	8367	+ mutex_lock(&trace_types_lock);
	8368	+
	8369	+ iter->wait_index++;
	8370	+ /* Make sure the waiters see the new wait_index */
	8371	+ smp_wmb();
	8372	+
	8373	+ ring_buffer_wake_waiters(iter->array_buffer->buffer, iter->cpu_file);
	8374	+
	8375	+ mutex_unlock(&trace_types_lock);
	8376	+ return 0;
	8377	+}
	8378	+
8314	8379	static const struct file_operations tracing_buffers_fops = {
8315	8380	.open = tracing_buffers_open,
8316	8381	.read = tracing_buffers_read,
8317	8382	.poll = tracing_buffers_poll,
8318	8383	.release = tracing_buffers_release,
8319	8384	.splice_read = tracing_buffers_splice_read,
	8385	+ .unlocked_ioctl = tracing_buffers_ioctl,
8320	8386	.llseek = no_llseek,
8321	8387	};
8322	8388
@@ -9005,6 +9071,8 @@ rb_simple_write(struct file filp, const char __user ubuf, · 682 unmodified lines
9005	9071	tracer_tracing_off(tr);
9006	9072	if (tr->current_trace->stop)
9007	9073	tr->current_trace->stop(tr);
	9074	+ /* Wake up any waiters */
	9075	+ ring_buffer_wake_waiters(buffer, RING_BUFFER_ALL_CPUS);
9008	9076	}
9009	9077	mutex_unlock(&trace_types_lock);
9010	9078	}
@@ -10091,7 +10159,7 @@ __init static int tracer_alloc_buffers(void) · 1080 unmodified lines
10091	10159	* buffer. The memory will be removed once the "instance" is removed.
10092	10160	*/
10093	10161	ret = cpuhp_setup_state_multi(CPUHP_TRACE_RB_PREPARE,
10094		− "trace/RB:preapre", trace_rb_cpu_prepare,
	10162	+ "trace/RB:prepare", trace_rb_cpu_prepare,
10095	10163	NULL);
10096	10164	if (ret < 0)
10097	10165	goto out_free_cpumask;

kernel/trace/trace.h+0−13

@@ -1435,8 +1435,6 @@ event_trigger_unlock_commit(struct trace_event_file *file, · 1434 unmodified lines
1435	1435	struct filter_pred;
1436	1436	struct regex;
1437	1437
1438		−typedef int (filter_pred_fn_t) (struct filter_pred pred, void *event);
1439		−
1440	1438	typedef int (regex_match_func)(char str, struct regex *r, int len);
1441	1439
1442	1440	enum regex_type {
@@ -1455,17 +1453,6 @@ struct regex { · 12 unmodified lines
1455	1453	regex_match_func match;
1456	1454	};
1457	1455
1458		−struct filter_pred {
1459		− filter_pred_fn_t fn;
1460		− u64 val;
1461		− struct regex regex;
1462		− unsigned short *ops;
1463		− struct ftrace_event_field *field;
1464		− int offset;
1465		− int not;
1466		− int op;
1467		−};
1468		−
1469	1456	static inline bool is_string_field(struct ftrace_event_field *field)
1470	1457	{
1471	1458	return field->filter_type == FILTER_DYN_STRING \|\|

kernel/trace/trace_benchmark.c+1−1

@@ -51,7 +51,7 @@ static void trace_do_benchmark(void) · 50 unmodified lines
51	51
52	52	local_irq_disable();
53	53	start = trace_clock_local();
54		− trace_benchmark_event(bm_str);
	54	+ trace_benchmark_event(bm_str, bm_last);
55	55	stop = trace_clock_local();
56	56	local_irq_enable();
57	57

kernel/trace/trace_benchmark.h+5−3

@@ -14,19 +14,21 @@ extern void trace_benchmark_unreg(void); · 13 unmodified lines
14	14
15	15	TRACE_EVENT_FN(benchmark_event,
16	16
17		− TP_PROTO(const char *str),
	17	+ TP_PROTO(const char *str, u64 delta),
18	18
19		− TP_ARGS(str),
	19	+ TP_ARGS(str, delta),
20	20
21	21	TP_STRUCT__entry(
22	22	__array( char, str, BENCHMARK_EVENT_STRLEN )
	23	+ __field( u64, delta)
23	24	),
24	25
25	26	TP_fast_assign(
26	27	memcpy(__entry->str, str, BENCHMARK_EVENT_STRLEN);
	28	+ __entry->delta = delta;
27	29	),
28	30
29		− TP_printk("%s", __entry->str),
	31	+ TP_printk("%s delta=%llu", __entry->str, __entry->delta),
30	32
31	33	trace_benchmark_reg, trace_benchmark_unreg
32	34	);

kernel/trace/trace_eprobe.c+97−10

@@ -26,6 +26,9 @@ struct trace_eprobe { · 25 unmodified lines
26	26	/* tracepoint event */
27	27	const char *event_name;
28	28
	29	+ /* filter string for the tracepoint */
	30	+ char *filter_str;
	31	+
29	32	struct trace_event_call *event;
30	33
31	34	struct dyn_event devent;
@@ -664,14 +667,15 @@ static struct event_trigger_data * · 632 unmodified lines
664	667	new_eprobe_trigger(struct trace_eprobe ep, struct trace_event_file file)
665	668	{
666	669	struct event_trigger_data *trigger;
	670	+ struct event_filter *filter = NULL;
667	671	struct eprobe_data *edata;
	672	+ int ret;
668	673
669	674	edata = kzalloc(sizeof(*edata), GFP_KERNEL);
670	675	trigger = kzalloc(sizeof(*trigger), GFP_KERNEL);
671	676	if (!trigger \|\| !edata) {
672		− kfree(edata);
673		− kfree(trigger);
674		− return ERR_PTR(-ENOMEM);
	677	+ ret = -ENOMEM;
	678	+ goto error;
675	679	}
676	680
677	681	trigger->flags = EVENT_TRIGGER_FL_PROBE;
@@ -686,13 +690,25 @@ new_eprobe_trigger(struct trace_eprobe ep, struct trace_event_file file) · 8 unmodified lines
686	690	trigger->cmd_ops = &event_trigger_cmd;
687	691
688	692	INIT_LIST_HEAD(&trigger->list);
689		− RCU_INIT_POINTER(trigger->filter, NULL);
	693	+
	694	+ if (ep->filter_str) {
	695	+ ret = create_event_filter(file->tr, file->event_call,
	696	+ ep->filter_str, false, &filter);
	697	+ if (ret)
	698	+ goto error;
	699	+ }
	700	+ RCU_INIT_POINTER(trigger->filter, filter);
690	701
691	702	edata->file = file;
692	703	edata->ep = ep;
693	704	trigger->private_data = edata;
694	705
695	706	return trigger;
	707	+error:
	708	+ free_event_filter(filter);
	709	+ kfree(edata);
	710	+ kfree(trigger);
	711	+ return ERR_PTR(ret);
696	712	}
697	713
698	714	static int enable_eprobe(struct trace_eprobe *ep,
@@ -726,6 +742,7 @@ static int disable_eprobe(struct trace_eprobe *ep, · 27 unmodified lines
726	742	{
727	743	struct event_trigger_data trigger = NULL, iter;
728	744	struct trace_event_file *file;
	745	+ struct event_filter *filter;
729	746	struct eprobe_data *edata;
730	747
731	748	file = find_event_file(tr, ep->event_system, ep->event_name);
@@ -752,6 +769,10 @@ static int disable_eprobe(struct trace_eprobe *ep, · 20 unmodified lines
752	769	/* Make sure nothing is using the edata or trigger */
753	770	tracepoint_synchronize_unregister();
754	771
	772	+ filter = rcu_access_pointer(trigger->filter);
	773	+
	774	+ if (filter)
	775	+ free_event_filter(filter);
755	776	kfree(edata);
756	777	kfree(trigger);
757	778
@@ -927,12 +948,62 @@ static int trace_eprobe_tp_update_arg(struct trace_eprobe ep, const char argv[ · 169 unmodified lines
927	948	return ret;
928	949	}
929	950
	951	+static int trace_eprobe_parse_filter(struct trace_eprobe ep, int argc, const char argv[])
	952	+{
	953	+ struct event_filter *dummy;
	954	+ int i, ret, len = 0;
	955	+ char *p;
	956	+
	957	+ if (argc == 0) {
	958	+ trace_probe_log_err(0, NO_EP_FILTER);
	959	+ return -EINVAL;
	960	+ }
	961	+
	962	+ /* Recover the filter string */
	963	+ for (i = 0; i < argc; i++)
	964	+ len += strlen(argv[i]) + 1;
	965	+
	966	+ ep->filter_str = kzalloc(len, GFP_KERNEL);
	967	+ if (!ep->filter_str)
	968	+ return -ENOMEM;
	969	+
	970	+ p = ep->filter_str;
	971	+ for (i = 0; i < argc; i++) {
	972	+ ret = snprintf(p, len, "%s ", argv[i]);
	973	+ if (ret < 0)
	974	+ goto error;
	975	+ if (ret > len) {
	976	+ ret = -E2BIG;
	977	+ goto error;
	978	+ }
	979	+ p += ret;
	980	+ len -= ret;
	981	+ }
	982	+ p[-1] = '\0';
	983	+
	984	+ /*
	985	+ * Ensure the filter string can be parsed correctly. Note, this
	986	+ * filter string is for the original event, not for the eprobe.
	987	+ */
	988	+ ret = create_event_filter(top_trace_array(), ep->event, ep->filter_str,
	989	+ true, &dummy);
	990	+ free_event_filter(dummy);
	991	+ if (ret)
	992	+ goto error;
	993	+
	994	+ return 0;
	995	+error:
	996	+ kfree(ep->filter_str);
	997	+ ep->filter_str = NULL;
	998	+ return ret;
	999	+}
	1000	+
930	1001	static int __trace_eprobe_create(int argc, const char *argv[])
931	1002	{
932	1003	/*
933	1004	* Argument syntax:
934		− * e[:[GRP/][ENAME]] SYSTEM.EVENT [FETCHARGS]
935		− * Fetch args:
	1005	+ * e[:[GRP/][ENAME]] SYSTEM.EVENT [FETCHARGS] [if FILTER]
	1006	+ * Fetch args (no space):
936	1007	* <name>=$<field>[:TYPE]
937	1008	*/
938	1009	const char event = NULL, group = EPROBE_EVENT_SYSTEM;
@@ -942,8 +1013,8 @@ static int __trace_eprobe_create(int argc, const char *argv[]) · 3 unmodified lines
942	1013	char buf1[MAX_EVENT_NAME_LEN];
943	1014	char buf2[MAX_EVENT_NAME_LEN];
944	1015	char gbuf[MAX_EVENT_NAME_LEN];
945		− int ret = 0;
946		− int i;
	1016	+ int ret = 0, filter_idx = 0;
	1017	+ int i, filter_cnt;
947	1018
948	1019	if (argc < 2 \|\| argv[0][0] != 'e')
949	1020	return -ECANCELED;
@@ -968,11 +1039,19 @@ static int __trace_eprobe_create(int argc, const char *argv[]) · 18 unmodified lines
968	1039	}
969	1040
970	1041	if (!event) {
971		− strscpy(buf1, argv[1], MAX_EVENT_NAME_LEN);
972		− sanitize_event_name(buf1);
	1042	+ strscpy(buf1, sys_event, MAX_EVENT_NAME_LEN);
973	1043	event = buf1;
974	1044	}
975	1045
	1046	+ for (i = 2; i < argc; i++) {
	1047	+ if (!strcmp(argv[i], "if")) {
	1048	+ filter_idx = i + 1;
	1049	+ filter_cnt = argc - filter_idx;
	1050	+ argc = i;
	1051	+ break;
	1052	+ }
	1053	+ }
	1054	+
976	1055	mutex_lock(&event_mutex);
977	1056	event_call = find_and_get_event(sys_name, sys_event);
978	1057	ep = alloc_event_probe(group, event, event_call, argc - 2);
@@ -988,6 +1067,14 @@ static int __trace_eprobe_create(int argc, const char *argv[]) · 9 unmodified lines
988	1067	goto error;
989	1068	}
990	1069
	1070	+ if (filter_idx) {
	1071	+ trace_probe_log_set_index(filter_idx);
	1072	+ ret = trace_eprobe_parse_filter(ep, filter_cnt, argv + filter_idx);
	1073	+ if (ret)
	1074	+ goto parse_error;
	1075	+ } else
	1076	+ ep->filter_str = NULL;
	1077	+
991	1078	argc -= 2; argv += 2;
992	1079	/* parse arguments */
993	1080	for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) {

kernel/trace/trace_events_filter.c+168−71

@@ -43,6 +43,42 @@ enum filter_op_ids { OPS }; · 42 unmodified lines
43	43
44	44	static const char * ops[] = { OPS };
45	45
	46	+enum filter_pred_fn {
	47	+ FILTER_PRED_FN_NOP,
	48	+ FILTER_PRED_FN_64,
	49	+ FILTER_PRED_FN_S64,
	50	+ FILTER_PRED_FN_U64,
	51	+ FILTER_PRED_FN_32,
	52	+ FILTER_PRED_FN_S32,
	53	+ FILTER_PRED_FN_U32,
	54	+ FILTER_PRED_FN_16,
	55	+ FILTER_PRED_FN_S16,
	56	+ FILTER_PRED_FN_U16,
	57	+ FILTER_PRED_FN_8,
	58	+ FILTER_PRED_FN_S8,
	59	+ FILTER_PRED_FN_U8,
	60	+ FILTER_PRED_FN_COMM,
	61	+ FILTER_PRED_FN_STRING,
	62	+ FILTER_PRED_FN_STRLOC,
	63	+ FILTER_PRED_FN_STRRELLOC,
	64	+ FILTER_PRED_FN_PCHAR_USER,
	65	+ FILTER_PRED_FN_PCHAR,
	66	+ FILTER_PRED_FN_CPU,
	67	+ FILTER_PRED_FN_,
	68	+ FILTER_PRED_TEST_VISITED,
	69	+};
	70	+
	71	+struct filter_pred {
	72	+ enum filter_pred_fn fn_num;
	73	+ u64 val;
	74	+ struct regex regex;
	75	+ unsigned short *ops;
	76	+ struct ftrace_event_field *field;
	77	+ int offset;
	78	+ int not;
	79	+ int op;
	80	+};
	81	+
46	82	/*
47	83	* pred functions are OP_LE, OP_LT, OP_GE, OP_GT, and OP_BAND
48	84	* pred_funcs_##type below must match the order of them above.
@@ -590,44 +626,48 @@ out_free: · 541 unmodified lines
590	626	return ERR_PTR(ret);
591	627	}
592	628
	629	+enum pred_cmp_types {
	630	+ PRED_CMP_TYPE_NOP,
	631	+ PRED_CMP_TYPE_LT,
	632	+ PRED_CMP_TYPE_LE,
	633	+ PRED_CMP_TYPE_GT,
	634	+ PRED_CMP_TYPE_GE,
	635	+ PRED_CMP_TYPE_BAND,
	636	+};
	637	+
593	638	#define DEFINE_COMPARISON_PRED(type) \
594		−static int filter_pred_LT_##type(struct filter_pred pred, void event) \
595		−{ \
596		− type addr = (type )(event + pred->offset); \
597		− type val = (type)pred->val; \
598		− return *addr < val; \
599		−} \
600		−static int filter_pred_LE_##type(struct filter_pred pred, void event) \
601		−{ \
602		− type addr = (type )(event + pred->offset); \
603		− type val = (type)pred->val; \
604		− return *addr <= val; \
605		−} \
606		−static int filter_pred_GT_##type(struct filter_pred pred, void event) \
	639	+static int filter_pred_##type(struct filter_pred pred, void event) \
607	640	{ \
608		− type addr = (type )(event + pred->offset); \
609		− type val = (type)pred->val; \
610		− return *addr > val; \
611		−} \
612		−static int filter_pred_GE_##type(struct filter_pred pred, void event) \
613		−{ \
614		− type addr = (type )(event + pred->offset); \
615		− type val = (type)pred->val; \
616		− return *addr >= val; \
617		−} \
618		−static int filter_pred_BAND_##type(struct filter_pred pred, void event) \
619		−{ \
620		− type addr = (type )(event + pred->offset); \
621		− type val = (type)pred->val; \
622		− return !!(*addr & val); \
623		−} \
624		−static const filter_pred_fn_t pred_funcs_##type[] = { \
625		− filter_pred_LE_##type, \
626		− filter_pred_LT_##type, \
627		− filter_pred_GE_##type, \
628		− filter_pred_GT_##type, \
629		− filter_pred_BAND_##type, \
630		−};
	641	+ switch (pred->op) { \
	642	+ case OP_LT: { \
	643	+ type addr = (type )(event + pred->offset); \
	644	+ type val = (type)pred->val; \
	645	+ return *addr < val; \
	646	+ } \
	647	+ case OP_LE: { \
	648	+ type addr = (type )(event + pred->offset); \
	649	+ type val = (type)pred->val; \
	650	+ return *addr <= val; \
	651	+ } \
	652	+ case OP_GT: { \
	653	+ type addr = (type )(event + pred->offset); \
	654	+ type val = (type)pred->val; \
	655	+ return *addr > val; \
	656	+ } \
	657	+ case OP_GE: { \
	658	+ type addr = (type )(event + pred->offset); \
	659	+ type val = (type)pred->val; \
	660	+ return *addr >= val; \
	661	+ } \
	662	+ case OP_BAND: { \
	663	+ type addr = (type )(event + pred->offset); \
	664	+ type val = (type)pred->val; \
	665	+ return !!(*addr & val); \
	666	+ } \
	667	+ default: \
	668	+ return 0; \
	669	+ } \
	670	+}
631	671
632	672	#define DEFINE_EQUALITY_PRED(size) \
633	673	static int filter_pred_##size(struct filter_pred pred, void event) \
@@ -836,11 +876,6 @@ static int filter_pred_comm(struct filter_pred pred, void event) · 202 unmodified lines
836	876	return cmp ^ pred->not;
837	877	}
838	878
839		−static int filter_pred_none(struct filter_pred pred, void event)
840		−{
841		− return 0;
842		−}
843		−
844	879	/*
845	880	* regex_match_foo - Basic regex callbacks
846	881	*
@@ -986,6 +1021,19 @@ static void filter_build_regex(struct filter_pred *pred) · 139 unmodified lines
986	1021	}
987	1022	}
988	1023
	1024	+
	1025	+#ifdef CONFIG_FTRACE_STARTUP_TEST
	1026	+static int test_pred_visited_fn(struct filter_pred pred, void event);
	1027	+#else
	1028	+static int test_pred_visited_fn(struct filter_pred pred, void event)
	1029	+{
	1030	+ return 0;
	1031	+}
	1032	+#endif
	1033	+
	1034	+
	1035	+static int filter_pred_fn_call(struct filter_pred pred, void event);
	1036	+
989	1037	/* return 1 if event matches, 0 otherwise (discard) */
990	1038	int filter_match_preds(struct event_filter filter, void rec)
991	1039	{
@@ -1003,7 +1051,7 @@ int filter_match_preds(struct event_filter filter, void rec) · 11 unmodified lines
1003	1051
1004	1052	for (i = 0; prog[i].pred; i++) {
1005	1053	struct filter_pred *pred = prog[i].pred;
1006		− int match = pred->fn(pred, rec);
	1054	+ int match = filter_pred_fn_call(pred, rec);
1007	1055	if (match == prog[i].when_to_branch)
1008	1056	i = prog[i].target;
1009	1057	}
@@ -1189,10 +1237,10 @@ int filter_assign_type(const char *type) · 179 unmodified lines
1189	1237	return FILTER_OTHER;
1190	1238	}
1191	1239
1192		−static filter_pred_fn_t select_comparison_fn(enum filter_op_ids op,
1193		− int field_size, int field_is_signed)
	1240	+static enum filter_pred_fn select_comparison_fn(enum filter_op_ids op,
	1241	+ int field_size, int field_is_signed)
1194	1242	{
1195		− filter_pred_fn_t fn = NULL;
	1243	+ enum filter_pred_fn fn = FILTER_PRED_FN_NOP;
1196	1244	int pred_func_index = -1;
1197	1245
1198	1246	switch (op) {
@@ -1201,50 +1249,99 @@ static filter_pred_fn_t select_comparison_fn(enum filter_op_ids op, · 2 unmodified lines
1201	1249	break;
1202	1250	default:
1203	1251	if (WARN_ON_ONCE(op < PRED_FUNC_START))
1204		− return NULL;
	1252	+ return fn;
1205	1253	pred_func_index = op - PRED_FUNC_START;
1206	1254	if (WARN_ON_ONCE(pred_func_index > PRED_FUNC_MAX))
1207		− return NULL;
	1255	+ return fn;
1208	1256	}
1209	1257
1210	1258	switch (field_size) {
1211	1259	case 8:
1212	1260	if (pred_func_index < 0)
1213		− fn = filter_pred_64;
	1261	+ fn = FILTER_PRED_FN_64;
1214	1262	else if (field_is_signed)
1215		− fn = pred_funcs_s64[pred_func_index];
	1263	+ fn = FILTER_PRED_FN_S64;
1216	1264	else
1217		− fn = pred_funcs_u64[pred_func_index];
	1265	+ fn = FILTER_PRED_FN_U64;
1218	1266	break;
1219	1267	case 4:
1220	1268	if (pred_func_index < 0)
1221		− fn = filter_pred_32;
	1269	+ fn = FILTER_PRED_FN_32;
1222	1270	else if (field_is_signed)
1223		− fn = pred_funcs_s32[pred_func_index];
	1271	+ fn = FILTER_PRED_FN_S32;
1224	1272	else
1225		− fn = pred_funcs_u32[pred_func_index];
	1273	+ fn = FILTER_PRED_FN_U32;
1226	1274	break;
1227	1275	case 2:
1228	1276	if (pred_func_index < 0)
1229		− fn = filter_pred_16;
	1277	+ fn = FILTER_PRED_FN_16;
1230	1278	else if (field_is_signed)
1231		− fn = pred_funcs_s16[pred_func_index];
	1279	+ fn = FILTER_PRED_FN_S16;
1232	1280	else
1233		− fn = pred_funcs_u16[pred_func_index];
	1281	+ fn = FILTER_PRED_FN_U16;
1234	1282	break;
1235	1283	case 1:
1236	1284	if (pred_func_index < 0)
1237		− fn = filter_pred_8;
	1285	+ fn = FILTER_PRED_FN_8;
1238	1286	else if (field_is_signed)
1239		− fn = pred_funcs_s8[pred_func_index];
	1287	+ fn = FILTER_PRED_FN_S8;
1240	1288	else
1241		− fn = pred_funcs_u8[pred_func_index];
	1289	+ fn = FILTER_PRED_FN_U8;
1242	1290	break;
1243	1291	}
1244	1292
1245	1293	return fn;
1246	1294	}
1247	1295
	1296	+
	1297	+static int filter_pred_fn_call(struct filter_pred pred, void event)
	1298	+{
	1299	+ switch (pred->fn_num) {
	1300	+ case FILTER_PRED_FN_64:
	1301	+ return filter_pred_64(pred, event);
	1302	+ case FILTER_PRED_FN_S64:
	1303	+ return filter_pred_s64(pred, event);
	1304	+ case FILTER_PRED_FN_U64:
	1305	+ return filter_pred_u64(pred, event);
	1306	+ case FILTER_PRED_FN_32:
	1307	+ return filter_pred_32(pred, event);
	1308	+ case FILTER_PRED_FN_S32:
	1309	+ return filter_pred_s32(pred, event);
	1310	+ case FILTER_PRED_FN_U32:
	1311	+ return filter_pred_u32(pred, event);
	1312	+ case FILTER_PRED_FN_16:
	1313	+ return filter_pred_16(pred, event);
	1314	+ case FILTER_PRED_FN_S16:
	1315	+ return filter_pred_s16(pred, event);
	1316	+ case FILTER_PRED_FN_U16:
	1317	+ return filter_pred_u16(pred, event);
	1318	+ case FILTER_PRED_FN_8:
	1319	+ return filter_pred_8(pred, event);
	1320	+ case FILTER_PRED_FN_S8:
	1321	+ return filter_pred_s8(pred, event);
	1322	+ case FILTER_PRED_FN_U8:
	1323	+ return filter_pred_u8(pred, event);
	1324	+ case FILTER_PRED_FN_COMM:
	1325	+ return filter_pred_comm(pred, event);
	1326	+ case FILTER_PRED_FN_STRING:
	1327	+ return filter_pred_string(pred, event);
	1328	+ case FILTER_PRED_FN_STRLOC:
	1329	+ return filter_pred_strloc(pred, event);
	1330	+ case FILTER_PRED_FN_STRRELLOC:
	1331	+ return filter_pred_strrelloc(pred, event);
	1332	+ case FILTER_PRED_FN_PCHAR_USER:
	1333	+ return filter_pred_pchar_user(pred, event);
	1334	+ case FILTER_PRED_FN_PCHAR:
	1335	+ return filter_pred_pchar(pred, event);
	1336	+ case FILTER_PRED_FN_CPU:
	1337	+ return filter_pred_cpu(pred, event);
	1338	+ case FILTER_PRED_TEST_VISITED:
	1339	+ return test_pred_visited_fn(pred, event);
	1340	+ default:
	1341	+ return 0;
	1342	+ }
	1343	+}
	1344	+
1248	1345	/* Called when a predicate is encountered by predicate_parse() */
1249	1346	static int parse_pred(const char str, void data,
1250	1347	int pos, struct filter_parse_error *pe,
@@ -1338,7 +1435,7 @@ static int parse_pred(const char str, void data, · 87 unmodified lines
1338	1435	parse_error(pe, FILT_ERR_IP_FIELD_ONLY, pos + i);
1339	1436	goto err_free;
1340	1437	}
1341		− pred->fn = filter_pred_none;
	1438	+ pred->fn_num = FILTER_PRED_FN_NOP;
1342	1439
1343	1440	/*
1344	1441	* Quotes are not required, but if they exist then we need
@@ -1416,16 +1513,16 @@ static int parse_pred(const char str, void data, · 71 unmodified lines
1416	1513	filter_build_regex(pred);
1417	1514
1418	1515	if (field->filter_type == FILTER_COMM) {
1419		− pred->fn = filter_pred_comm;
	1516	+ pred->fn_num = FILTER_PRED_FN_COMM;
1420	1517
1421	1518	} else if (field->filter_type == FILTER_STATIC_STRING) {
1422		− pred->fn = filter_pred_string;
	1519	+ pred->fn_num = FILTER_PRED_FN_STRING;
1423	1520	pred->regex.field_len = field->size;
1424	1521
1425	1522	} else if (field->filter_type == FILTER_DYN_STRING) {
1426		− pred->fn = filter_pred_strloc;
	1523	+ pred->fn_num = FILTER_PRED_FN_STRLOC;
1427	1524	} else if (field->filter_type == FILTER_RDYN_STRING)
1428		− pred->fn = filter_pred_strrelloc;
	1525	+ pred->fn_num = FILTER_PRED_FN_STRRELLOC;
1429	1526	else {
1430	1527
1431	1528	if (!ustring_per_cpu) {
@@ -1436,9 +1533,9 @@ static int parse_pred(const char str, void data, · 4 unmodified lines
1436	1533	}
1437	1534
1438	1535	if (ustring)
1439		− pred->fn = filter_pred_pchar_user;
	1536	+ pred->fn_num = FILTER_PRED_FN_PCHAR_USER;
1440	1537	else
1441		− pred->fn = filter_pred_pchar;
	1538	+ pred->fn_num = FILTER_PRED_FN_PCHAR;
1442	1539	}
1443	1540	/* go past the last quote */
1444	1541	i++;
@@ -1486,10 +1583,10 @@ static int parse_pred(const char str, void data, · 41 unmodified lines
1486	1583	pred->val = val;
1487	1584
1488	1585	if (field->filter_type == FILTER_CPU)
1489		− pred->fn = filter_pred_cpu;
	1586	+ pred->fn_num = FILTER_PRED_FN_CPU;
1490	1587	else {
1491		− pred->fn = select_comparison_fn(pred->op, field->size,
1492		− field->is_signed);
	1588	+ pred->fn_num = select_comparison_fn(pred->op, field->size,
	1589	+ field->is_signed);
1493	1590	if (pred->op == OP_NE)
1494	1591	pred->not = 1;
1495	1592	}
@@ -2296,7 +2393,7 @@ static void update_pred_fn(struct event_filter filter, char fields) · 800 unmodified lines
2296	2393	struct filter_pred *pred = prog[i].pred;
2297	2394	struct ftrace_event_field *field = pred->field;
2298	2395
2299		− WARN_ON_ONCE(!pred->fn);
	2396	+ WARN_ON_ONCE(pred->fn_num == FILTER_PRED_FN_NOP);
2300	2397
2301	2398	if (!field) {
2302	2399	WARN_ONCE(1, "all leafs should have field defined %d", i);
@@ -2306,7 +2403,7 @@ static void update_pred_fn(struct event_filter filter, char fields) · 3 unmodified lines
2306	2403	if (!strchr(fields, *field->name))
2307	2404	continue;
2308	2405
2309		− pred->fn = test_pred_visited_fn;
	2406	+ pred->fn_num = FILTER_PRED_TEST_VISITED;
2310	2407	}
2311	2408	}
2312	2409

kernel/trace/trace_events_hist.c+169−77

@@ -104,6 +104,38 @@ enum field_op_id { · 103 unmodified lines
104	104	FIELD_OP_MULT,
105	105	};
106	106
	107	+enum hist_field_fn {
	108	+ HIST_FIELD_FN_NOP,
	109	+ HIST_FIELD_FN_VAR_REF,
	110	+ HIST_FIELD_FN_COUNTER,
	111	+ HIST_FIELD_FN_CONST,
	112	+ HIST_FIELD_FN_LOG2,
	113	+ HIST_FIELD_FN_BUCKET,
	114	+ HIST_FIELD_FN_TIMESTAMP,
	115	+ HIST_FIELD_FN_CPU,
	116	+ HIST_FIELD_FN_STRING,
	117	+ HIST_FIELD_FN_DYNSTRING,
	118	+ HIST_FIELD_FN_RELDYNSTRING,
	119	+ HIST_FIELD_FN_PSTRING,
	120	+ HIST_FIELD_FN_S64,
	121	+ HIST_FIELD_FN_U64,
	122	+ HIST_FIELD_FN_S32,
	123	+ HIST_FIELD_FN_U32,
	124	+ HIST_FIELD_FN_S16,
	125	+ HIST_FIELD_FN_U16,
	126	+ HIST_FIELD_FN_S8,
	127	+ HIST_FIELD_FN_U8,
	128	+ HIST_FIELD_FN_UMINUS,
	129	+ HIST_FIELD_FN_MINUS,
	130	+ HIST_FIELD_FN_PLUS,
	131	+ HIST_FIELD_FN_DIV,
	132	+ HIST_FIELD_FN_MULT,
	133	+ HIST_FIELD_FN_DIV_POWER2,
	134	+ HIST_FIELD_FN_DIV_NOT_POWER2,
	135	+ HIST_FIELD_FN_DIV_MULT_SHIFT,
	136	+ HIST_FIELD_FN_EXECNAME,
	137	+};
	138	+
107	139	/*
108	140	* A hist_var (histogram variable) contains variable information for
109	141	* hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
@@ -123,15 +155,15 @@ struct hist_var { · 13 unmodified lines
123	155	struct hist_field {
124	156	struct ftrace_event_field *field;
125	157	unsigned long flags;
126		− hist_field_fn_t fn;
127		− unsigned int ref;
128		− unsigned int size;
129		− unsigned int offset;
130		− unsigned int is_signed;
131	158	unsigned long buckets;
132	159	const char *type;
133	160	struct hist_field *operands[HIST_FIELD_OPERANDS_MAX];
134	161	struct hist_trigger_data *hist_data;
	162	+ enum hist_field_fn fn_num;
	163	+ unsigned int ref;
	164	+ unsigned int size;
	165	+ unsigned int offset;
	166	+ unsigned int is_signed;
135	167
136	168	/*
137	169	* Variable fields contain variable-specific info in var.
@@ -166,14 +198,11 @@ struct hist_field { · 28 unmodified lines
166	198	u64 div_multiplier;
167	199	};
168	200
169		−static u64 hist_field_none(struct hist_field *field,
170		− struct tracing_map_elt *elt,
171		− struct trace_buffer *buffer,
172		− struct ring_buffer_event *rbe,
173		− void *event)
174		−{
175		− return 0;
176		−}
	201	+static u64 hist_fn_call(struct hist_field *hist_field,
	202	+ struct tracing_map_elt *elt,
	203	+ struct trace_buffer *buffer,
	204	+ struct ring_buffer_event *rbe,
	205	+ void *event);
177	206
178	207	static u64 hist_field_const(struct hist_field *field,
179	208	struct tracing_map_elt *elt,
@@ -250,7 +279,7 @@ static u64 hist_field_log2(struct hist_field *hist_field, · 70 unmodified lines
250	279	{
251	280	struct hist_field *operand = hist_field->operands[0];
252	281
253		− u64 val = operand->fn(operand, elt, buffer, rbe, event);
	282	+ u64 val = hist_fn_call(operand, elt, buffer, rbe, event);
254	283
255	284	return (u64) ilog2(roundup_pow_of_two(val));
256	285	}
@@ -264,7 +293,7 @@ static u64 hist_field_bucket(struct hist_field *hist_field, · 7 unmodified lines
264	293	struct hist_field *operand = hist_field->operands[0];
265	294	unsigned long buckets = hist_field->buckets;
266	295
267		− u64 val = operand->fn(operand, elt, buffer, rbe, event);
	296	+ u64 val = hist_fn_call(operand, elt, buffer, rbe, event);
268	297
269	298	if (WARN_ON_ONCE(!buckets))
270	299	return val;
@@ -285,8 +314,8 @@ static u64 hist_field_plus(struct hist_field *hist_field, · 14 unmodified lines
285	314	struct hist_field *operand1 = hist_field->operands[0];
286	315	struct hist_field *operand2 = hist_field->operands[1];
287	316
288		− u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
289		− u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
	317	+ u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
	318	+ u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
290	319
291	320	return val1 + val2;
292	321	}
@@ -300,8 +329,8 @@ static u64 hist_field_minus(struct hist_field *hist_field, · 7 unmodified lines
300	329	struct hist_field *operand1 = hist_field->operands[0];
301	330	struct hist_field *operand2 = hist_field->operands[1];
302	331
303		− u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
304		− u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
	332	+ u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
	333	+ u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
305	334
306	335	return val1 - val2;
307	336	}
@@ -315,8 +344,8 @@ static u64 hist_field_div(struct hist_field *hist_field, · 7 unmodified lines
315	344	struct hist_field *operand1 = hist_field->operands[0];
316	345	struct hist_field *operand2 = hist_field->operands[1];
317	346
318		− u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
319		− u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
	347	+ u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
	348	+ u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
320	349
321	350	/* Return -1 for the undefined case */
322	351	if (!val2)
@@ -338,7 +367,7 @@ static u64 div_by_power_of_two(struct hist_field *hist_field, · 15 unmodified lines
338	367	struct hist_field *operand1 = hist_field->operands[0];
339	368	struct hist_field *operand2 = hist_field->operands[1];
340	369
341		− u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
	370	+ u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
342	371
343	372	return val1 >> __ffs64(operand2->constant);
344	373	}
@@ -352,7 +381,7 @@ static u64 div_by_not_power_of_two(struct hist_field *hist_field, · 7 unmodified lines
352	381	struct hist_field *operand1 = hist_field->operands[0];
353	382	struct hist_field *operand2 = hist_field->operands[1];
354	383
355		− u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
	384	+ u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
356	385
357	386	return div64_u64(val1, operand2->constant);
358	387	}
@@ -366,7 +395,7 @@ static u64 div_by_mult_and_shift(struct hist_field *hist_field, · 7 unmodified lines
366	395	struct hist_field *operand1 = hist_field->operands[0];
367	396	struct hist_field *operand2 = hist_field->operands[1];
368	397
369		− u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
	398	+ u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
370	399
371	400	/*
372	401	* If the divisor is a constant, do a multiplication and shift instead.
@@ -400,8 +429,8 @@ static u64 hist_field_mult(struct hist_field *hist_field, · 27 unmodified lines
400	429	struct hist_field *operand1 = hist_field->operands[0];
401	430	struct hist_field *operand2 = hist_field->operands[1];
402	431
403		− u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
404		− u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
	432	+ u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
	433	+ u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
405	434
406	435	return val1 * val2;
407	436	}
@@ -414,7 +443,7 @@ static u64 hist_field_unary_minus(struct hist_field *hist_field, · 6 unmodified lines
414	443	{
415	444	struct hist_field *operand = hist_field->operands[0];
416	445
417		− s64 sval = (s64)operand->fn(operand, elt, buffer, rbe, event);
	446	+ s64 sval = (s64)hist_fn_call(operand, elt, buffer, rbe, event);
418	447	u64 val = (u64)-sval;
419	448
420	449	return val;
@@ -657,19 +686,19 @@ struct snapshot_context { · 236 unmodified lines
657	686	* Returns the specific division function to use if the divisor
658	687	* is constant. This avoids extra branches when the trigger is hit.
659	688	*/
660		−static hist_field_fn_t hist_field_get_div_fn(struct hist_field *divisor)
	689	+static enum hist_field_fn hist_field_get_div_fn(struct hist_field *divisor)
661	690	{
662	691	u64 div = divisor->constant;
663	692
664	693	if (!(div & (div - 1)))
665		− return div_by_power_of_two;
	694	+ return HIST_FIELD_FN_DIV_POWER2;
666	695
667	696	/* If the divisor is too large, do a regular division */
668	697	if (div > (1 << HIST_DIV_SHIFT))
669		− return div_by_not_power_of_two;
	698	+ return HIST_FIELD_FN_DIV_NOT_POWER2;
670	699
671	700	divisor->div_multiplier = div64_u64((u64)(1 << HIST_DIV_SHIFT), div);
672		− return div_by_mult_and_shift;
	701	+ return HIST_FIELD_FN_DIV_MULT_SHIFT;
673	702	}
674	703
675	704	static void track_data_free(struct track_data *track_data)
@@ -1334,38 +1363,32 @@ static const char hist_field_name(struct hist_field field, · 658 unmodified lines
1334	1363	return field_name;
1335	1364	}
1336	1365
1337		−static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
	1366	+static enum hist_field_fn select_value_fn(int field_size, int field_is_signed)
1338	1367	{
1339		− hist_field_fn_t fn = NULL;
1340		−
1341	1368	switch (field_size) {
1342	1369	case 8:
1343	1370	if (field_is_signed)
1344		− fn = hist_field_s64;
	1371	+ return HIST_FIELD_FN_S64;
1345	1372	else
1346		− fn = hist_field_u64;
1347		− break;
	1373	+ return HIST_FIELD_FN_U64;
1348	1374	case 4:
1349	1375	if (field_is_signed)
1350		− fn = hist_field_s32;
	1376	+ return HIST_FIELD_FN_S32;
1351	1377	else
1352		− fn = hist_field_u32;
1353		− break;
	1378	+ return HIST_FIELD_FN_U32;
1354	1379	case 2:
1355	1380	if (field_is_signed)
1356		− fn = hist_field_s16;
	1381	+ return HIST_FIELD_FN_S16;
1357	1382	else
1358		− fn = hist_field_u16;
1359		− break;
	1383	+ return HIST_FIELD_FN_U16;
1360	1384	case 1:
1361	1385	if (field_is_signed)
1362		− fn = hist_field_s8;
	1386	+ return HIST_FIELD_FN_S8;
1363	1387	else
1364		− fn = hist_field_u8;
1365		− break;
	1388	+ return HIST_FIELD_FN_U8;
1366	1389	}
1367	1390
1368		− return fn;
	1391	+ return HIST_FIELD_FN_NOP;
1369	1392	}
1370	1393
1371	1394	static int parse_map_size(char *str)
@@ -1922,19 +1945,19 @@ static struct hist_field create_hist_field(struct hist_trigger_data hist_data, · 550 unmodified lines
1922	1945	goto out; /* caller will populate */
1923	1946
1924	1947	if (flags & HIST_FIELD_FL_VAR_REF) {
1925		− hist_field->fn = hist_field_var_ref;
	1948	+ hist_field->fn_num = HIST_FIELD_FN_VAR_REF;
1926	1949	goto out;
1927	1950	}
1928	1951
1929	1952	if (flags & HIST_FIELD_FL_HITCOUNT) {
1930		− hist_field->fn = hist_field_counter;
	1953	+ hist_field->fn_num = HIST_FIELD_FN_COUNTER;
1931	1954	hist_field->size = sizeof(u64);
1932	1955	hist_field->type = "u64";
1933	1956	goto out;
1934	1957	}
1935	1958
1936	1959	if (flags & HIST_FIELD_FL_CONST) {
1937		− hist_field->fn = hist_field_const;
	1960	+ hist_field->fn_num = HIST_FIELD_FN_CONST;
1938	1961	hist_field->size = sizeof(u64);
1939	1962	hist_field->type = kstrdup("u64", GFP_KERNEL);
1940	1963	if (!hist_field->type)
@@ -1943,14 +1966,14 @@ static struct hist_field create_hist_field(struct hist_trigger_data hist_data, · 2 unmodified lines
1943	1966	}
1944	1967
1945	1968	if (flags & HIST_FIELD_FL_STACKTRACE) {
1946		− hist_field->fn = hist_field_none;
	1969	+ hist_field->fn_num = HIST_FIELD_FN_NOP;
1947	1970	goto out;
1948	1971	}
1949	1972
1950	1973	if (flags & (HIST_FIELD_FL_LOG2 \| HIST_FIELD_FL_BUCKET)) {
1951	1974	unsigned long fl = flags & ~(HIST_FIELD_FL_LOG2 \| HIST_FIELD_FL_BUCKET);
1952		− hist_field->fn = flags & HIST_FIELD_FL_LOG2 ? hist_field_log2 :
1953		− hist_field_bucket;
	1975	+ hist_field->fn_num = flags & HIST_FIELD_FL_LOG2 ? HIST_FIELD_FN_LOG2 :
	1976	+ HIST_FIELD_FN_BUCKET;
1954	1977	hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
1955	1978	hist_field->size = hist_field->operands[0]->size;
1956	1979	hist_field->type = kstrdup_const(hist_field->operands[0]->type, GFP_KERNEL);
@@ -1960,14 +1983,14 @@ static struct hist_field create_hist_field(struct hist_trigger_data hist_data, · 3 unmodified lines
1960	1983	}
1961	1984
1962	1985	if (flags & HIST_FIELD_FL_TIMESTAMP) {
1963		− hist_field->fn = hist_field_timestamp;
	1986	+ hist_field->fn_num = HIST_FIELD_FN_TIMESTAMP;
1964	1987	hist_field->size = sizeof(u64);
1965	1988	hist_field->type = "u64";
1966	1989	goto out;
1967	1990	}
1968	1991
1969	1992	if (flags & HIST_FIELD_FL_CPU) {
1970		− hist_field->fn = hist_field_cpu;
	1993	+ hist_field->fn_num = HIST_FIELD_FN_CPU;
1971	1994	hist_field->size = sizeof(int);
1972	1995	hist_field->type = "unsigned int";
1973	1996	goto out;
@@ -1987,14 +2010,14 @@ static struct hist_field create_hist_field(struct hist_trigger_data hist_data, · 13 unmodified lines
1987	2010	goto free;
1988	2011
1989	2012	if (field->filter_type == FILTER_STATIC_STRING) {
1990		− hist_field->fn = hist_field_string;
	2013	+ hist_field->fn_num = HIST_FIELD_FN_STRING;
1991	2014	hist_field->size = field->size;
1992	2015	} else if (field->filter_type == FILTER_DYN_STRING) {
1993		− hist_field->fn = hist_field_dynstring;
	2016	+ hist_field->fn_num = HIST_FIELD_FN_DYNSTRING;
1994	2017	} else if (field->filter_type == FILTER_RDYN_STRING)
1995		− hist_field->fn = hist_field_reldynstring;
	2018	+ hist_field->fn_num = HIST_FIELD_FN_RELDYNSTRING;
1996	2019	else
1997		− hist_field->fn = hist_field_pstring;
	2020	+ hist_field->fn_num = HIST_FIELD_FN_PSTRING;
1998	2021	} else {
1999	2022	hist_field->size = field->size;
2000	2023	hist_field->is_signed = field->is_signed;
@@ -2002,9 +2025,9 @@ static struct hist_field create_hist_field(struct hist_trigger_data hist_data, · 1 unmodified line
2002	2025	if (!hist_field->type)
2003	2026	goto free;
2004	2027
2005		− hist_field->fn = select_value_fn(field->size,
2006		− field->is_signed);
2007		− if (!hist_field->fn) {
	2028	+ hist_field->fn_num = select_value_fn(field->size,
	2029	+ field->is_signed);
	2030	+ if (hist_field->fn_num == HIST_FIELD_FN_NOP) {
2008	2031	destroy_hist_field(hist_field, 0);
2009	2032	return NULL;
2010	2033	}
@@ -2340,7 +2363,7 @@ static struct hist_field create_alias(struct hist_trigger_data hist_data, · 329 unmodified lines
2340	2363	if (!alias)
2341	2364	return NULL;
2342	2365
2343		− alias->fn = var_ref->fn;
	2366	+ alias->fn_num = var_ref->fn_num;
2344	2367	alias->operands[0] = var_ref;
2345	2368
2346	2369	if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
@@ -2523,7 +2546,7 @@ static struct hist_field parse_unary(struct hist_trigger_data hist_data, · 176 unmodified lines
2523	2546
2524	2547	expr->flags \|= operand1->flags &
2525	2548	(HIST_FIELD_FL_TIMESTAMP \| HIST_FIELD_FL_TIMESTAMP_USECS);
2526		− expr->fn = hist_field_unary_minus;
	2549	+ expr->fn_num = HIST_FIELD_FN_UMINUS;
2527	2550	expr->operands[0] = operand1;
2528	2551	expr->size = operand1->size;
2529	2552	expr->is_signed = operand1->is_signed;
@@ -2595,7 +2618,7 @@ static struct hist_field parse_expr(struct hist_trigger_data hist_data, · 65 unmodified lines
2595	2618	unsigned long operand_flags, operand2_flags;
2596	2619	int field_op, ret = -EINVAL;
2597	2620	char sep, operand1_str;
2598		− hist_field_fn_t op_fn;
	2621	+ enum hist_field_fn op_fn;
2599	2622	bool combine_consts;
2600	2623
2601	2624	if (*n_subexprs > 3) {
@@ -2654,16 +2677,16 @@ static struct hist_field parse_expr(struct hist_trigger_data hist_data, · 52 unmodified lines
2654	2677
2655	2678	switch (field_op) {
2656	2679	case FIELD_OP_MINUS:
2657		− op_fn = hist_field_minus;
	2680	+ op_fn = HIST_FIELD_FN_MINUS;
2658	2681	break;
2659	2682	case FIELD_OP_PLUS:
2660		− op_fn = hist_field_plus;
	2683	+ op_fn = HIST_FIELD_FN_PLUS;
2661	2684	break;
2662	2685	case FIELD_OP_DIV:
2663		− op_fn = hist_field_div;
	2686	+ op_fn = HIST_FIELD_FN_DIV;
2664	2687	break;
2665	2688	case FIELD_OP_MULT:
2666		− op_fn = hist_field_mult;
	2689	+ op_fn = HIST_FIELD_FN_MULT;
2667	2690	break;
2668	2691	default:
2669	2692	ret = -EINVAL;
@@ -2719,13 +2742,16 @@ static struct hist_field parse_expr(struct hist_trigger_data hist_data, · 49 unmodified lines
2719	2742	op_fn = hist_field_get_div_fn(operand2);
2720	2743	}
2721	2744
	2745	+ expr->fn_num = op_fn;
	2746	+
2722	2747	if (combine_consts) {
2723	2748	if (var1)
2724	2749	expr->operands[0] = var1;
2725	2750	if (var2)
2726	2751	expr->operands[1] = var2;
2727	2752
2728		− expr->constant = op_fn(expr, NULL, NULL, NULL, NULL);
	2753	+ expr->constant = hist_fn_call(expr, NULL, NULL, NULL, NULL);
	2754	+ expr->fn_num = HIST_FIELD_FN_CONST;
2729	2755
2730	2756	expr->operands[0] = NULL;
2731	2757	expr->operands[1] = NULL;
@@ -2739,8 +2765,6 @@ static struct hist_field parse_expr(struct hist_trigger_data hist_data, · 7 unmodified lines
2739	2765
2740	2766	expr->name = expr_str(expr, 0);
2741	2767	} else {
2742		− expr->fn = op_fn;
2743		−
2744	2768	/* The operand sizes should be the same, so just pick one */
2745	2769	expr->size = operand1->size;
2746	2770	expr->is_signed = operand1->is_signed;
@@ -3065,7 +3089,7 @@ static inline void __update_field_vars(struct tracing_map_elt *elt, · 318 unmodified lines
3065	3089	struct hist_field *var = field_var->var;
3066	3090	struct hist_field *val = field_var->val;
3067	3091
3068		− var_val = val->fn(val, elt, buffer, rbe, rec);
	3092	+ var_val = hist_fn_call(val, elt, buffer, rbe, rec);
3069	3093	var_idx = var->var.idx;
3070	3094
3071	3095	if (val->flags & HIST_FIELD_FL_STRING) {
@@ -4186,6 +4210,74 @@ static u64 hist_field_execname(struct hist_field *hist_field, · 1114 unmodified lines
4186	4210	return (u64)(unsigned long)(elt_data->comm);
4187	4211	}
4188	4212
	4213	+static u64 hist_fn_call(struct hist_field *hist_field,
	4214	+ struct tracing_map_elt *elt,
	4215	+ struct trace_buffer *buffer,
	4216	+ struct ring_buffer_event *rbe,
	4217	+ void *event)
	4218	+{
	4219	+ switch (hist_field->fn_num) {
	4220	+ case HIST_FIELD_FN_VAR_REF:
	4221	+ return hist_field_var_ref(hist_field, elt, buffer, rbe, event);
	4222	+ case HIST_FIELD_FN_COUNTER:
	4223	+ return hist_field_counter(hist_field, elt, buffer, rbe, event);
	4224	+ case HIST_FIELD_FN_CONST:
	4225	+ return hist_field_const(hist_field, elt, buffer, rbe, event);
	4226	+ case HIST_FIELD_FN_LOG2:
	4227	+ return hist_field_log2(hist_field, elt, buffer, rbe, event);
	4228	+ case HIST_FIELD_FN_BUCKET:
	4229	+ return hist_field_bucket(hist_field, elt, buffer, rbe, event);
	4230	+ case HIST_FIELD_FN_TIMESTAMP:
	4231	+ return hist_field_timestamp(hist_field, elt, buffer, rbe, event);
	4232	+ case HIST_FIELD_FN_CPU:
	4233	+ return hist_field_cpu(hist_field, elt, buffer, rbe, event);
	4234	+ case HIST_FIELD_FN_STRING:
	4235	+ return hist_field_string(hist_field, elt, buffer, rbe, event);
	4236	+ case HIST_FIELD_FN_DYNSTRING:
	4237	+ return hist_field_dynstring(hist_field, elt, buffer, rbe, event);
	4238	+ case HIST_FIELD_FN_RELDYNSTRING:
	4239	+ return hist_field_reldynstring(hist_field, elt, buffer, rbe, event);
	4240	+ case HIST_FIELD_FN_PSTRING:
	4241	+ return hist_field_pstring(hist_field, elt, buffer, rbe, event);
	4242	+ case HIST_FIELD_FN_S64:
	4243	+ return hist_field_s64(hist_field, elt, buffer, rbe, event);
	4244	+ case HIST_FIELD_FN_U64:
	4245	+ return hist_field_u64(hist_field, elt, buffer, rbe, event);
	4246	+ case HIST_FIELD_FN_S32:
	4247	+ return hist_field_s32(hist_field, elt, buffer, rbe, event);
	4248	+ case HIST_FIELD_FN_U32:
	4249	+ return hist_field_u32(hist_field, elt, buffer, rbe, event);
	4250	+ case HIST_FIELD_FN_S16:
	4251	+ return hist_field_s16(hist_field, elt, buffer, rbe, event);
	4252	+ case HIST_FIELD_FN_U16:
	4253	+ return hist_field_u16(hist_field, elt, buffer, rbe, event);
	4254	+ case HIST_FIELD_FN_S8:
	4255	+ return hist_field_s8(hist_field, elt, buffer, rbe, event);
	4256	+ case HIST_FIELD_FN_U8:
	4257	+ return hist_field_u8(hist_field, elt, buffer, rbe, event);
	4258	+ case HIST_FIELD_FN_UMINUS:
	4259	+ return hist_field_unary_minus(hist_field, elt, buffer, rbe, event);
	4260	+ case HIST_FIELD_FN_MINUS:
	4261	+ return hist_field_minus(hist_field, elt, buffer, rbe, event);
	4262	+ case HIST_FIELD_FN_PLUS:
	4263	+ return hist_field_plus(hist_field, elt, buffer, rbe, event);
	4264	+ case HIST_FIELD_FN_DIV:
	4265	+ return hist_field_div(hist_field, elt, buffer, rbe, event);
	4266	+ case HIST_FIELD_FN_MULT:
	4267	+ return hist_field_mult(hist_field, elt, buffer, rbe, event);
	4268	+ case HIST_FIELD_FN_DIV_POWER2:
	4269	+ return div_by_power_of_two(hist_field, elt, buffer, rbe, event);
	4270	+ case HIST_FIELD_FN_DIV_NOT_POWER2:
	4271	+ return div_by_not_power_of_two(hist_field, elt, buffer, rbe, event);
	4272	+ case HIST_FIELD_FN_DIV_MULT_SHIFT:
	4273	+ return div_by_mult_and_shift(hist_field, elt, buffer, rbe, event);
	4274	+ case HIST_FIELD_FN_EXECNAME:
	4275	+ return hist_field_execname(hist_field, elt, buffer, rbe, event);
	4276	+ default:
	4277	+ return 0;
	4278	+ }
	4279	+}
	4280	+
4189	4281	/* Convert a var that points to common_pid.execname to a string */
4190	4282	static void update_var_execname(struct hist_field *hist_field)
4191	4283	{
@@ -4197,7 +4289,7 @@ static void update_var_execname(struct hist_field *hist_field) · 5 unmodified lines
4197	4289	kfree_const(hist_field->type);
4198	4290	hist_field->type = "char[]";
4199	4291
4200		− hist_field->fn = hist_field_execname;
	4292	+ hist_field->fn_num = HIST_FIELD_FN_EXECNAME;
4201	4293	}
4202	4294
4203	4295	static int create_var_field(struct hist_trigger_data *hist_data,
@@ -4956,7 +5048,7 @@ static void hist_trigger_elt_update(struct hist_trigger_data *hist_data, · 752 unmodified lines
4956	5048
4957	5049	for_each_hist_val_field(i, hist_data) {
4958	5050	hist_field = hist_data->fields[i];
4959		− hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
	5051	+ hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec);
4960	5052	if (hist_field->flags & HIST_FIELD_FL_VAR) {
4961	5053	var_idx = hist_field->var.idx;
4962	5054
@@ -4987,7 +5079,7 @@ static void hist_trigger_elt_update(struct hist_trigger_data *hist_data, · 24 unmodified lines
4987	5079	for_each_hist_key_field(i, hist_data) {
4988	5080	hist_field = hist_data->fields[i];
4989	5081	if (hist_field->flags & HIST_FIELD_FL_VAR) {
4990		− hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
	5082	+ hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec);
4991	5083	var_idx = hist_field->var.idx;
4992	5084	tracing_map_set_var(elt, var_idx, hist_val);
4993	5085	}
@@ -5062,7 +5154,7 @@ static void event_hist_trigger(struct event_trigger_data *data, · 68 unmodified lines
5062	5154	HIST_STACKTRACE_SKIP);
5063	5155	key = entries;
5064	5156	} else {
5065		− field_contents = key_field->fn(key_field, elt, buffer, rbe, rec);
	5157	+ field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec);
5066	5158	if (key_field->flags & HIST_FIELD_FL_STRING) {
5067	5159	key = (void *)(unsigned long)field_contents;
5068	5160	use_compound_key = true;

kernel/trace/trace_events_user.c+426−142

@@ -14,6 +14,7 @@ · 13 unmodified lines
14	14	#include <linux/uio.h>
15	15	#include <linux/ioctl.h>
16	16	#include <linux/jhash.h>
	17	+#include <linux/refcount.h>
17	18	#include <linux/trace_events.h>
18	19	#include <linux/tracefs.h>
19	20	#include <linux/types.h>
@@ -39,28 +40,69 @@ · 19 unmodified lines
39	40	*/
40	41	#define MAX_PAGE_ORDER 0
41	42	#define MAX_PAGES (1 << MAX_PAGE_ORDER)
42		−#define MAX_EVENTS (MAX_PAGES * PAGE_SIZE)
	43	+#define MAX_BYTES (MAX_PAGES * PAGE_SIZE)
	44	+#define MAX_EVENTS (MAX_BYTES * 8)
43	45
44	46	/* Limit how long of an event name plus args within the subsystem. */
45	47	#define MAX_EVENT_DESC 512
46	48	#define EVENT_NAME(user_event) ((user_event)->tracepoint.name)
47	49	#define MAX_FIELD_ARRAY_SIZE 1024
48		−#define MAX_FIELD_ARG_NAME 256
49	50
50		−static char *register_page_data;
	51	+/*
	52	+ * The MAP_STATUS_* macros are used for taking a index and determining the
	53	+ * appropriate byte and the bit in the byte to set/reset for an event.
	54	+ *
	55	+ * The lower 3 bits of the index decide which bit to set.
	56	+ * The remaining upper bits of the index decide which byte to use for the bit.
	57	+ *
	58	+ * This is used when an event has a probe attached/removed to reflect live
	59	+ * status of the event wanting tracing or not to user-programs via shared
	60	+ * memory maps.
	61	+ */
	62	+#define MAP_STATUS_BYTE(index) ((index) >> 3)
	63	+#define MAP_STATUS_MASK(index) BIT((index) & 7)
	64	+
	65	+/*
	66	+ * Internal bits (kernel side only) to keep track of connected probes:
	67	+ * These are used when status is requested in text form about an event. These
	68	+ * bits are compared against an internal byte on the event to determine which
	69	+ * probes to print out to the user.
	70	+ *
	71	+ * These do not reflect the mapped bytes between the user and kernel space.
	72	+ */
	73	+#define EVENT_STATUS_FTRACE BIT(0)
	74	+#define EVENT_STATUS_PERF BIT(1)
	75	+#define EVENT_STATUS_OTHER BIT(7)
	76	+
	77	+/*
	78	+ * Stores the pages, tables, and locks for a group of events.
	79	+ * Each logical grouping of events has its own group, with a
	80	+ * matching page for status checks within user programs. This
	81	+ * allows for isolation of events to user programs by various
	82	+ * means.
	83	+ */
	84	+struct user_event_group {
	85	+ struct page *pages;
	86	+ char *register_page_data;
	87	+ char *system_name;
	88	+ struct hlist_node node;
	89	+ struct mutex reg_mutex;
	90	+ DECLARE_HASHTABLE(register_table, 8);
	91	+ DECLARE_BITMAP(page_bitmap, MAX_EVENTS);
	92	+};
51	93
52		−static DEFINE_MUTEX(reg_mutex);
53		−static DEFINE_HASHTABLE(register_table, 4);
54		−static DECLARE_BITMAP(page_bitmap, MAX_EVENTS);
	94	+/* Group for init_user_ns mapping, top-most group */
	95	+static struct user_event_group *init_group;
55	96
56	97	/*
57	98	* Stores per-event properties, as users register events
58	99	* within a file a user_event might be created if it does not
59	100	* already exist. These are globally used and their lifetime
60	101	* is tied to the refcnt member. These cannot go away until the
61		− * refcnt reaches zero.
	102	+ * refcnt reaches one.
62	103	*/
63	104	struct user_event {
	105	+ struct user_event_group *group;
64	106	struct tracepoint tracepoint;
65	107	struct trace_event_call call;
66	108	struct trace_event_class class;
@@ -68,10 +110,11 @@ struct user_event { · 1 unmodified line
68	110	struct hlist_node node;
69	111	struct list_head fields;
70	112	struct list_head validators;
71		− atomic_t refcnt;
	113	+ refcount_t refcnt;
72	114	int index;
73	115	int flags;
74	116	int min_size;
	117	+ char status;
75	118	};
76	119
77	120	/*
@@ -86,6 +129,11 @@ struct user_event_refs { · 8 unmodified lines
86	129	struct user_event *events[];
87	130	};
88	131
	132	+struct user_event_file_info {
	133	+ struct user_event_group *group;
	134	+ struct user_event_refs *refs;
	135	+};
	136	+
89	137	#define VALIDATOR_ENSURE_NULL (1 << 0)
90	138	#define VALIDATOR_REL (1 << 1)
91	139
@@ -98,7 +146,8 @@ struct user_event_validator { · 6 unmodified lines
98	146	typedef void (user_event_func_t) (struct user_event user, struct iov_iter *i,
99	147	void tpdata, bool faulted);
100	148
101		−static int user_event_parse(char name, char args, char *flags,
	149	+static int user_event_parse(struct user_event_group group, char name,
	150	+ char args, char flags,
102	151	struct user_event **newuser);
103	152
104	153	static u32 user_event_key(char *name)
@@ -106,6 +155,144 @@ static u32 user_event_key(char *name) · 1 unmodified line
106	155	return jhash(name, strlen(name), 0);
107	156	}
108	157
	158	+static void set_page_reservations(char *pages, bool set)
	159	+{
	160	+ int page;
	161	+
	162	+ for (page = 0; page < MAX_PAGES; ++page) {
	163	+ void addr = pages + (PAGE_SIZE page);
	164	+
	165	+ if (set)
	166	+ SetPageReserved(virt_to_page(addr));
	167	+ else
	168	+ ClearPageReserved(virt_to_page(addr));
	169	+ }
	170	+}
	171	+
	172	+static void user_event_group_destroy(struct user_event_group *group)
	173	+{
	174	+ if (group->register_page_data)
	175	+ set_page_reservations(group->register_page_data, false);
	176	+
	177	+ if (group->pages)
	178	+ __free_pages(group->pages, MAX_PAGE_ORDER);
	179	+
	180	+ kfree(group->system_name);
	181	+ kfree(group);
	182	+}
	183	+
	184	+static char user_event_group_system_name(struct user_namespace user_ns)
	185	+{
	186	+ char *system_name;
	187	+ int len = sizeof(USER_EVENTS_SYSTEM) + 1;
	188	+
	189	+ if (user_ns != &init_user_ns) {
	190	+ /*
	191	+ * Unexpected at this point:
	192	+ * We only currently support init_user_ns.
	193	+ * When we enable more, this will trigger a failure so log.
	194	+ */
	195	+ pr_warn("user_events: Namespace other than init_user_ns!\n");
	196	+ return NULL;
	197	+ }
	198	+
	199	+ system_name = kmalloc(len, GFP_KERNEL);
	200	+
	201	+ if (!system_name)
	202	+ return NULL;
	203	+
	204	+ snprintf(system_name, len, "%s", USER_EVENTS_SYSTEM);
	205	+
	206	+ return system_name;
	207	+}
	208	+
	209	+static inline struct user_event_group
	210	+user_event_group_from_user_ns(struct user_namespace user_ns)
	211	+{
	212	+ if (user_ns == &init_user_ns)
	213	+ return init_group;
	214	+
	215	+ return NULL;
	216	+}
	217	+
	218	+static struct user_event_group *current_user_event_group(void)
	219	+{
	220	+ struct user_namespace *user_ns = current_user_ns();
	221	+ struct user_event_group *group = NULL;
	222	+
	223	+ while (user_ns) {
	224	+ group = user_event_group_from_user_ns(user_ns);
	225	+
	226	+ if (group)
	227	+ break;
	228	+
	229	+ user_ns = user_ns->parent;
	230	+ }
	231	+
	232	+ return group;
	233	+}
	234	+
	235	+static struct user_event_group
	236	+user_event_group_create(struct user_namespace user_ns)
	237	+{
	238	+ struct user_event_group *group;
	239	+
	240	+ group = kzalloc(sizeof(*group), GFP_KERNEL);
	241	+
	242	+ if (!group)
	243	+ return NULL;
	244	+
	245	+ group->system_name = user_event_group_system_name(user_ns);
	246	+
	247	+ if (!group->system_name)
	248	+ goto error;
	249	+
	250	+ group->pages = alloc_pages(GFP_KERNEL \| __GFP_ZERO, MAX_PAGE_ORDER);
	251	+
	252	+ if (!group->pages)
	253	+ goto error;
	254	+
	255	+ group->register_page_data = page_address(group->pages);
	256	+
	257	+ set_page_reservations(group->register_page_data, true);
	258	+
	259	+ /* Zero all bits beside 0 (which is reserved for failures) */
	260	+ bitmap_zero(group->page_bitmap, MAX_EVENTS);
	261	+ set_bit(0, group->page_bitmap);
	262	+
	263	+ mutex_init(&group->reg_mutex);
	264	+ hash_init(group->register_table);
	265	+
	266	+ return group;
	267	+error:
	268	+ if (group)
	269	+ user_event_group_destroy(group);
	270	+
	271	+ return NULL;
	272	+};
	273	+
	274	+static __always_inline
	275	+void user_event_register_set(struct user_event *user)
	276	+{
	277	+ int i = user->index;
	278	+
	279	+ user->group->register_page_data[MAP_STATUS_BYTE(i)] \|= MAP_STATUS_MASK(i);
	280	+}
	281	+
	282	+static __always_inline
	283	+void user_event_register_clear(struct user_event *user)
	284	+{
	285	+ int i = user->index;
	286	+
	287	+ user->group->register_page_data[MAP_STATUS_BYTE(i)] &= ~MAP_STATUS_MASK(i);
	288	+}
	289	+
	290	+static __always_inline __must_check
	291	+bool user_event_last_ref(struct user_event *user)
	292	+{
	293	+ return refcount_read(&user->refcnt) == 1;
	294	+}
	295	+
109	296	static __always_inline __must_check
110	297	size_t copy_nofault(void addr, size_t bytes, struct iov_iter i)
111	298	{
@@ -141,7 +328,8 @@ static struct list_head user_event_get_fields(struct trace_event_call call) · 29 unmodified lines
141	328	*
142	329	* Upon success user_event has its ref count increased by 1.
143	330	*/
144		−static int user_event_parse_cmd(char raw_command, struct user_event *newuser)
	331	+static int user_event_parse_cmd(struct user_event_group *group,
	332	+ char raw_command, struct user_event *newuser)
145	333	{
146	334	char *name = raw_command;
147	335	char *args = strpbrk(name, " ");
@@ -155,7 +343,7 @@ static int user_event_parse_cmd(char raw_command, struct user_event *newuser) · 7 unmodified lines
155	343	if (flags)
156	344	*flags++ = '\0';
157	345
158		− return user_event_parse(name, args, flags, newuser);
	346	+ return user_event_parse(group, name, args, flags, newuser);
159	347	}
160	348
161	349	static int user_field_array_size(const char *type)
@@ -277,7 +465,7 @@ static int user_event_add_field(struct user_event user, const char type, · 115 unmodified lines
277	465	goto add_field;
278	466
279	467	add_validator:
280		− if (strstr(type, "char") != 0)
	468	+ if (strstr(type, "char") != NULL)
281	469	validator_flags \|= VALIDATOR_ENSURE_NULL;
282	470
283	471	validator = kmalloc(sizeof(*validator), GFP_KERNEL);
@@ -458,7 +646,7 @@ static const char user_field_format(const char type) · 174 unmodified lines
458	646	return "%d";
459	647	if (strcmp(type, "unsigned char") == 0)
460	648	return "%u";
461		− if (strstr(type, "char[") != 0)
	649	+ if (strstr(type, "char[") != NULL)
462	650	return "%s";
463	651
464	652	/* Unknown, likely struct, allowed treat as 64-bit */
@@ -479,10 +667,52 @@ static bool user_field_is_dyn_string(const char type, const char *str_func) · 14 unmodified lines
479	667
480	668	return false;
481	669	check:
482		− return strstr(type, "char") != 0;
	670	+ return strstr(type, "char") != NULL;
483	671	}
484	672
485	673	#define LEN_OR_ZERO (len ? len - pos : 0)
	674	+static int user_dyn_field_set_string(int argc, const char *argv, int iout,
	675	+ char buf, int len, bool colon)
	676	+{
	677	+ int pos = 0, i = *iout;
	678	+
	679	+ *colon = false;
	680	+
	681	+ for (; i < argc; ++i) {
	682	+ if (i != *iout)
	683	+ pos += snprintf(buf + pos, LEN_OR_ZERO, " ");
	684	+
	685	+ pos += snprintf(buf + pos, LEN_OR_ZERO, "%s", argv[i]);
	686	+
	687	+ if (strchr(argv[i], ';')) {
	688	+ ++i;
	689	+ *colon = true;
	690	+ break;
	691	+ }
	692	+ }
	693	+
	694	+ /* Actual set, advance i */
	695	+ if (len != 0)
	696	+ *iout = i;
	697	+
	698	+ return pos + 1;
	699	+}
	700	+
	701	+static int user_field_set_string(struct ftrace_event_field *field,
	702	+ char *buf, int len, bool colon)
	703	+{
	704	+ int pos = 0;
	705	+
	706	+ pos += snprintf(buf + pos, LEN_OR_ZERO, "%s", field->type);
	707	+ pos += snprintf(buf + pos, LEN_OR_ZERO, " ");
	708	+ pos += snprintf(buf + pos, LEN_OR_ZERO, "%s", field->name);
	709	+
	710	+ if (colon)
	711	+ pos += snprintf(buf + pos, LEN_OR_ZERO, ";");
	712	+
	713	+ return pos + 1;
	714	+}
	715	+
486	716	static int user_event_set_print_fmt(struct user_event user, char buf, int len)
487	717	{
488	718	struct ftrace_event_field field, next;
@@ -600,8 +830,8 @@ static int destroy_user_event(struct user_event *user) · 111 unmodified lines
600	830
601	831	dyn_event_remove(&user->devent);
602	832
603		− register_page_data[user->index] = 0;
604		− clear_bit(user->index, page_bitmap);
	833	+ user_event_register_clear(user);
	834	+ clear_bit(user->index, user->group->page_bitmap);
605	835	hash_del(&user->node);
606	836
607	837	user_event_destroy_validators(user);
@@ -612,16 +842,17 @@ static int destroy_user_event(struct user_event *user) · 4 unmodified lines
612	842	return ret;
613	843	}
614	844
615		−static struct user_event find_user_event(char name, u32 *outkey)
	845	+static struct user_event find_user_event(struct user_event_group group,
	846	+ char name, u32 outkey)
616	847	{
617	848	struct user_event *user;
618	849	u32 key = user_event_key(name);
619	850
620	851	*outkey = key;
621	852
622		− hash_for_each_possible(register_table, user, node, key)
	853	+ hash_for_each_possible(group->register_table, user, node, key)
623	854	if (!strcmp(EVENT_NAME(user), name)) {
624		− atomic_inc(&user->refcnt);
	855	+ refcount_inc(&user->refcnt);
625	856	return user;
626	857	}
627	858
@@ -779,7 +1010,12 @@ static void update_reg_page_for(struct user_event *user) · 151 unmodified lines
779	1010	rcu_read_unlock_sched();
780	1011	}
781	1012
782		− register_page_data[user->index] = status;
	1013	+ if (status)
	1014	+ user_event_register_set(user);
	1015	+ else
	1016	+ user_event_register_clear(user);
	1017	+
	1018	+ user->status = status;
783	1019	}
784	1020
785	1021	/*
@@ -835,17 +1071,18 @@ static int user_event_reg(struct trace_event_call *call, · 49 unmodified lines
835	1071
836	1072	return ret;
837	1073	inc:
838		− atomic_inc(&user->refcnt);
	1074	+ refcount_inc(&user->refcnt);
839	1075	update_reg_page_for(user);
840	1076	return 0;
841	1077	dec:
842	1078	update_reg_page_for(user);
843		− atomic_dec(&user->refcnt);
	1079	+ refcount_dec(&user->refcnt);
844	1080	return 0;
845	1081	}
846	1082
847	1083	static int user_event_create(const char *raw_command)
848	1084	{
	1085	+ struct user_event_group *group;
849	1086	struct user_event *user;
850	1087	char *name;
851	1088	int ret;
@@ -861,14 +1098,19 @@ static int user_event_create(const char *raw_command) · 9 unmodified lines
861	1098	if (!name)
862	1099	return -ENOMEM;
863	1100
864		− mutex_lock(&reg_mutex);
	1101	+ group = current_user_event_group();
865	1102
866		− ret = user_event_parse_cmd(name, &user);
	1103	+ if (!group)
	1104	+ return -ENOENT;
	1105	+
	1106	+ mutex_lock(&group->reg_mutex);
	1107	+
	1108	+ ret = user_event_parse_cmd(group, name, &user);
867	1109
868	1110	if (!ret)
869		− atomic_dec(&user->refcnt);
	1111	+ refcount_dec(&user->refcnt);
870	1112
871		− mutex_unlock(&reg_mutex);
	1113	+ mutex_unlock(&group->reg_mutex);
872	1114
873	1115	if (ret)
874	1116	kfree(name);
@@ -910,14 +1152,14 @@ static bool user_event_is_busy(struct dyn_event *ev) · 35 unmodified lines
910	1152	{
911	1153	struct user_event *user = container_of(ev, struct user_event, devent);
912	1154
913		− return atomic_read(&user->refcnt) != 0;
	1155	+ return !user_event_last_ref(user);
914	1156	}
915	1157
916	1158	static int user_event_free(struct dyn_event *ev)
917	1159	{
918	1160	struct user_event *user = container_of(ev, struct user_event, devent);
919	1161
920		− if (atomic_read(&user->refcnt) != 0)
	1162	+ if (!user_event_last_ref(user))
921	1163	return -EBUSY;
922	1164
923	1165	return destroy_user_event(user);
@@ -926,49 +1168,35 @@ static int user_event_free(struct dyn_event *ev) · 2 unmodified lines
926	1168	static bool user_field_match(struct ftrace_event_field *field, int argc,
927	1169	const char *argv, int iout)
928	1170	{
929		− char field_name, arg_name;
930		− int len, pos, i = *iout;
	1171	+ char field_name = NULL, dyn_field_name = NULL;
931	1172	bool colon = false, match = false;
	1173	+ int dyn_len, len;
932	1174
933		− if (i >= argc)
	1175	+ if (*iout >= argc)
934	1176	return false;
935	1177
936		− len = MAX_FIELD_ARG_NAME;
937		− field_name = kmalloc(len, GFP_KERNEL);
938		− arg_name = kmalloc(len, GFP_KERNEL);
939		−
940		− if (!arg_name \|\| !field_name)
941		− goto out;
942		−
943		− pos = 0;
944		−
945		− for (; i < argc; ++i) {
946		− if (i != *iout)
947		− pos += snprintf(arg_name + pos, len - pos, " ");
	1178	+ dyn_len = user_dyn_field_set_string(argc, argv, iout, dyn_field_name,
	1179	+ 0, &colon);
948	1180
949		− pos += snprintf(arg_name + pos, len - pos, argv[i]);
	1181	+ len = user_field_set_string(field, field_name, 0, colon);
950	1182
951		− if (strchr(argv[i], ';')) {
952		− ++i;
953		− colon = true;
954		− break;
955		− }
956		− }
	1183	+ if (dyn_len != len)
	1184	+ return false;
957	1185
958		− pos = 0;
	1186	+ dyn_field_name = kmalloc(dyn_len, GFP_KERNEL);
	1187	+ field_name = kmalloc(len, GFP_KERNEL);
959	1188
960		− pos += snprintf(field_name + pos, len - pos, field->type);
961		− pos += snprintf(field_name + pos, len - pos, " ");
962		− pos += snprintf(field_name + pos, len - pos, field->name);
	1189	+ if (!dyn_field_name \|\| !field_name)
	1190	+ goto out;
963	1191
964		− if (colon)
965		− pos += snprintf(field_name + pos, len - pos, ";");
	1192	+ user_dyn_field_set_string(argc, argv, iout, dyn_field_name,
	1193	+ dyn_len, &colon);
966	1194
967		− *iout = i;
	1195	+ user_field_set_string(field, field_name, len, colon);
968	1196
969		− match = strcmp(arg_name, field_name) == 0;
	1197	+ match = strcmp(dyn_field_name, field_name) == 0;
970	1198	out:
971		− kfree(arg_name);
	1199	+ kfree(dyn_field_name);
972	1200	kfree(field_name);
973	1201
974	1202	return match;
@@ -1036,7 +1264,8 @@ static int user_event_trace_register(struct user_event *user) · 61 unmodified lines
1036	1264	* The name buffer lifetime is owned by this method for success cases only.
1037	1265	* Upon success the returned user_event has its ref count increased by 1.
1038	1266	*/
1039		−static int user_event_parse(char name, char args, char *flags,
	1267	+static int user_event_parse(struct user_event_group group, char name,
	1268	+ char args, char flags,
1040	1269	struct user_event **newuser)
1041	1270	{
1042	1271	int ret;
@@ -1046,7 +1275,7 @@ static int user_event_parse(char name, char args, char *flags, · 3 unmodified lines
1046	1275
1047	1276	/* Prevent dyn_event from racing */
1048	1277	mutex_lock(&event_mutex);
1049		− user = find_user_event(name, &key);
	1278	+ user = find_user_event(group, name, &key);
1050	1279	mutex_unlock(&event_mutex);
1051	1280
1052	1281	if (user) {
@@ -1059,7 +1288,7 @@ static int user_event_parse(char name, char args, char *flags, · 6 unmodified lines
1059	1288	return 0;
1060	1289	}
1061	1290
1062		− index = find_first_zero_bit(page_bitmap, MAX_EVENTS);
	1291	+ index = find_first_zero_bit(group->page_bitmap, MAX_EVENTS);
1063	1292
1064	1293	if (index == MAX_EVENTS)
1065	1294	return -EMFILE;
@@ -1073,6 +1302,7 @@ static int user_event_parse(char name, char args, char *flags, · 7 unmodified lines
1073	1302	INIT_LIST_HEAD(&user->fields);
1074	1303	INIT_LIST_HEAD(&user->validators);
1075	1304
	1305	+ user->group = group;
1076	1306	user->tracepoint.name = name;
1077	1307
1078	1308	ret = user_event_parse_fields(user, args);
@@ -1091,8 +1321,8 @@ static int user_event_parse(char name, char args, char *flags, · 12 unmodified lines
1091	1321	user->call.flags = TRACE_EVENT_FL_TRACEPOINT;
1092	1322	user->call.tp = &user->tracepoint;
1093	1323	user->call.event.funcs = &user_event_funcs;
	1324	+ user->class.system = group->system_name;
1094	1325
1095		− user->class.system = USER_EVENTS_SYSTEM;
1096	1326	user->class.fields_array = user_event_fields_array;
1097	1327	user->class.get_fields = user_event_get_fields;
1098	1328	user->class.reg = user_event_reg;
@@ -1110,13 +1340,13 @@ static int user_event_parse(char name, char args, char *flags, · 11 unmodified lines
1110	1340
1111	1341	user->index = index;
1112	1342
1113		− /* Ensure we track ref */
1114		− atomic_inc(&user->refcnt);
	1343	+ /* Ensure we track self ref and caller ref (2) */
	1344	+ refcount_set(&user->refcnt, 2);
1115	1345
1116	1346	dyn_event_init(&user->devent, &user_event_dops);
1117	1347	dyn_event_add(&user->devent, &user->call);
1118		− set_bit(user->index, page_bitmap);
1119		− hash_add(register_table, &user->node, key);
	1348	+ set_bit(user->index, group->page_bitmap);
	1349	+ hash_add(group->register_table, &user->node, key);
1120	1350
1121	1351	mutex_unlock(&event_mutex);
1122	1352
@@ -1134,32 +1364,20 @@ put_user: · 11 unmodified lines
1134	1364	/*
1135	1365	* Deletes a previously created event if it is no longer being used.
1136	1366	*/
1137		−static int delete_user_event(char *name)
	1367	+static int delete_user_event(struct user_event_group group, char name)
1138	1368	{
1139	1369	u32 key;
1140		− int ret;
1141		− struct user_event *user = find_user_event(name, &key);
	1370	+ struct user_event *user = find_user_event(group, name, &key);
1142	1371
1143	1372	if (!user)
1144	1373	return -ENOENT;
1145	1374
1146		− /* Ensure we are the last ref */
1147		− if (atomic_read(&user->refcnt) != 1) {
1148		− ret = -EBUSY;
1149		− goto put_ref;
1150		− }
1151		−
1152		− ret = destroy_user_event(user);
1153		−
1154		− if (ret)
1155		− goto put_ref;
	1375	+ refcount_dec(&user->refcnt);
1156	1376
1157		− return ret;
1158		−put_ref:
1159		− /* No longer have this ref */
1160		− atomic_dec(&user->refcnt);
	1377	+ if (!user_event_last_ref(user))
	1378	+ return -EBUSY;
1161	1379
1162		− return ret;
	1380	+ return destroy_user_event(user);
1163	1381	}
1164	1382
1165	1383	/*
@@ -1167,6 +1385,7 @@ put_ref: · 1 unmodified line
1167	1385	*/
1168	1386	static ssize_t user_events_write_core(struct file file, struct iov_iter i)
1169	1387	{
	1388	+ struct user_event_file_info *info = file->private_data;
1170	1389	struct user_event_refs *refs;
1171	1390	struct user_event *user = NULL;
1172	1391	struct tracepoint *tp;
@@ -1178,7 +1397,7 @@ static ssize_t user_events_write_core(struct file file, struct iov_iter i) · 5 unmodified lines
1178	1397
1179	1398	rcu_read_lock_sched();
1180	1399
1181		− refs = rcu_dereference_sched(file->private_data);
	1400	+ refs = rcu_dereference_sched(info->refs);
1182	1401
1183	1402	/*
1184	1403	* The refs->events array is protected by RCU, and new items may be
@@ -1236,6 +1455,28 @@ static ssize_t user_events_write_core(struct file file, struct iov_iter i) · 51 unmodified lines
1236	1455	return ret;
1237	1456	}
1238	1457
	1458	+static int user_events_open(struct inode node, struct file file)
	1459	+{
	1460	+ struct user_event_group *group;
	1461	+ struct user_event_file_info *info;
	1462	+
	1463	+ group = current_user_event_group();
	1464	+
	1465	+ if (!group)
	1466	+ return -ENOENT;
	1467	+
	1468	+ info = kzalloc(sizeof(*info), GFP_KERNEL);
	1469	+
	1470	+ if (!info)
	1471	+ return -ENOMEM;
	1472	+
	1473	+ info->group = group;
	1474	+
	1475	+ file->private_data = info;
	1476	+
	1477	+ return 0;
	1478	+}
	1479	+
1239	1480	static ssize_t user_events_write(struct file file, const char __user ubuf,
1240	1481	size_t count, loff_t *ppos)
1241	1482	{
@@ -1245,7 +1486,8 @@ static ssize_t user_events_write(struct file file, const char __user ubuf, · 3 unmodified lines
1245	1486	if (unlikely(*ppos != 0))
1246	1487	return -EFAULT;
1247	1488
1248		− if (unlikely(import_single_range(READ, (char *)ubuf, count, &iov, &i)))
	1489	+ if (unlikely(import_single_range(WRITE, (char __user *)ubuf,
	1490	+ count, &iov, &i)))
1249	1491	return -EFAULT;
1250	1492
1251	1493	return user_events_write_core(file, &i);
@@ -1256,13 +1498,15 @@ static ssize_t user_events_write_iter(struct kiocb kp, struct iov_iter i) · 4 unmodified lines
1256	1498	return user_events_write_core(kp->ki_filp, i);
1257	1499	}
1258	1500
1259		−static int user_events_ref_add(struct file file, struct user_event user)
	1501	+static int user_events_ref_add(struct user_event_file_info *info,
	1502	+ struct user_event *user)
1260	1503	{
	1504	+ struct user_event_group *group = info->group;
1261	1505	struct user_event_refs refs, new_refs;
1262	1506	int i, size, count = 0;
1263	1507
1264		− refs = rcu_dereference_protected(file->private_data,
1265		− lockdep_is_held(&reg_mutex));
	1508	+ refs = rcu_dereference_protected(info->refs,
	1509	+ lockdep_is_held(&group->reg_mutex));
1266	1510
1267	1511	if (refs) {
1268	1512	count = refs->count;
@@ -1286,9 +1530,9 @@ static int user_events_ref_add(struct file file, struct user_event user) · 17 unmodified lines
1286	1530
1287	1531	new_refs->events[i] = user;
1288	1532
1289		− atomic_inc(&user->refcnt);
	1533	+ refcount_inc(&user->refcnt);
1290	1534
1291		− rcu_assign_pointer(file->private_data, new_refs);
	1535	+ rcu_assign_pointer(info->refs, new_refs);
1292	1536
1293	1537	if (refs)
1294	1538	kfree_rcu(refs, rcu);
@@ -1309,13 +1553,24 @@ static long user_reg_get(struct user_reg __user ureg, struct user_reg kreg) · 14 unmodified lines
1309	1553	if (size > PAGE_SIZE)
1310	1554	return -E2BIG;
1311	1555
1312		− return copy_struct_from_user(kreg, sizeof(*kreg), ureg, size);
	1556	+ if (size < offsetofend(struct user_reg, write_index))
	1557	+ return -EINVAL;
	1558	+
	1559	+ ret = copy_struct_from_user(kreg, sizeof(*kreg), ureg, size);
	1560	+
	1561	+ if (ret)
	1562	+ return ret;
	1563	+
	1564	+ kreg->size = size;
	1565	+
	1566	+ return 0;
1313	1567	}
1314	1568
1315	1569	/*
1316	1570	* Registers a user_event on behalf of a user process.
1317	1571	*/
1318		−static long user_events_ioctl_reg(struct file *file, unsigned long uarg)
	1572	+static long user_events_ioctl_reg(struct user_event_file_info *info,
	1573	+ unsigned long uarg)
1319	1574	{
1320	1575	struct user_reg __user ureg = (struct user_reg __user )uarg;
1321	1576	struct user_reg reg;
@@ -1336,24 +1591,24 @@ static long user_events_ioctl_reg(struct file *file, unsigned long uarg) · 14 unmodified lines
1336	1591	return ret;
1337	1592	}
1338	1593
1339		− ret = user_event_parse_cmd(name, &user);
	1594	+ ret = user_event_parse_cmd(info->group, name, &user);
1340	1595
1341	1596	if (ret) {
1342	1597	kfree(name);
1343	1598	return ret;
1344	1599	}
1345	1600
1346		− ret = user_events_ref_add(file, user);
	1601	+ ret = user_events_ref_add(info, user);
1347	1602
1348	1603	/* No longer need parse ref, ref_add either worked or not */
1349		− atomic_dec(&user->refcnt);
	1604	+ refcount_dec(&user->refcnt);
1350	1605
1351	1606	/* Positive number is index and valid */
1352	1607	if (ret < 0)
1353	1608	return ret;
1354	1609
1355	1610	put_user((u32)ret, &ureg->write_index);
1356		− put_user(user->index, &ureg->status_index);
	1611	+ put_user(user->index, &ureg->status_bit);
1357	1612
1358	1613	return 0;
1359	1614	}
@@ -1361,7 +1616,8 @@ static long user_events_ioctl_reg(struct file *file, unsigned long uarg) · 1 unmodified line
1361	1616	/*
1362	1617	* Deletes a user_event on behalf of a user process.
1363	1618	*/
1364		−static long user_events_ioctl_del(struct file *file, unsigned long uarg)
	1619	+static long user_events_ioctl_del(struct user_event_file_info *info,
	1620	+ unsigned long uarg)
1365	1621	{
1366	1622	void __user ubuf = (void __user )uarg;
1367	1623	char *name;
@@ -1374,7 +1630,7 @@ static long user_events_ioctl_del(struct file *file, unsigned long uarg) · 6 unmodified lines
1374	1630
1375	1631	/* event_mutex prevents dyn_event from racing */
1376	1632	mutex_lock(&event_mutex);
1377		− ret = delete_user_event(name);
	1633	+ ret = delete_user_event(info->group, name);
1378	1634	mutex_unlock(&event_mutex);
1379	1635
1380	1636	kfree(name);
@@ -1388,19 +1644,21 @@ static long user_events_ioctl_del(struct file *file, unsigned long uarg) · 7 unmodified lines
1388	1644	static long user_events_ioctl(struct file *file, unsigned int cmd,
1389	1645	unsigned long uarg)
1390	1646	{
	1647	+ struct user_event_file_info *info = file->private_data;
	1648	+ struct user_event_group *group = info->group;
1391	1649	long ret = -ENOTTY;
1392	1650
1393	1651	switch (cmd) {
1394	1652	case DIAG_IOCSREG:
1395		− mutex_lock(&reg_mutex);
1396		− ret = user_events_ioctl_reg(file, uarg);
1397		− mutex_unlock(&reg_mutex);
	1653	+ mutex_lock(&group->reg_mutex);
	1654	+ ret = user_events_ioctl_reg(info, uarg);
	1655	+ mutex_unlock(&group->reg_mutex);
1398	1656	break;
1399	1657
1400	1658	case DIAG_IOCSDEL:
1401		− mutex_lock(&reg_mutex);
1402		− ret = user_events_ioctl_del(file, uarg);
1403		− mutex_unlock(&reg_mutex);
	1659	+ mutex_lock(&group->reg_mutex);
	1660	+ ret = user_events_ioctl_del(info, uarg);
	1661	+ mutex_unlock(&group->reg_mutex);
1404	1662	break;
1405	1663	}
1406	1664
@@ -1412,17 +1670,24 @@ static long user_events_ioctl(struct file *file, unsigned int cmd, · 5 unmodified lines
1412	1670	*/
1413	1671	static int user_events_release(struct inode node, struct file file)
1414	1672	{
	1673	+ struct user_event_file_info *info = file->private_data;
	1674	+ struct user_event_group *group;
1415	1675	struct user_event_refs *refs;
1416	1676	struct user_event *user;
1417	1677	int i;
1418	1678
	1679	+ if (!info)
	1680	+ return -EINVAL;
	1681	+
	1682	+ group = info->group;
	1683	+
1419	1684	/*
1420	1685	* Ensure refs cannot change under any situation by taking the
1421	1686	* register mutex during the final freeing of the references.
1422	1687	*/
1423		− mutex_lock(&reg_mutex);
	1688	+ mutex_lock(&group->reg_mutex);
1424	1689
1425		− refs = file->private_data;
	1690	+ refs = info->refs;
1426	1691
1427	1692	if (!refs)
1428	1693	goto out;
@@ -1436,37 +1701,56 @@ static int user_events_release(struct inode node, struct file file) · 7 unmodified lines
1436	1701	user = refs->events[i];
1437	1702
1438	1703	if (user)
1439		− atomic_dec(&user->refcnt);
	1704	+ refcount_dec(&user->refcnt);
1440	1705	}
1441	1706	out:
1442	1707	file->private_data = NULL;
1443	1708
1444		− mutex_unlock(&reg_mutex);
	1709	+ mutex_unlock(&group->reg_mutex);
1445	1710
1446	1711	kfree(refs);
	1712	+ kfree(info);
1447	1713
1448	1714	return 0;
1449	1715	}
1450	1716
1451	1717	static const struct file_operations user_data_fops = {
	1718	+ .open = user_events_open,
1452	1719	.write = user_events_write,
1453	1720	.write_iter = user_events_write_iter,
1454	1721	.unlocked_ioctl = user_events_ioctl,
1455	1722	.release = user_events_release,
1456	1723	};
1457	1724
	1725	+static struct user_event_group user_status_group(struct file file)
	1726	+{
	1727	+ struct seq_file *m = file->private_data;
	1728	+
	1729	+ if (!m)
	1730	+ return NULL;
	1731	+
	1732	+ return m->private;
	1733	+}
	1734	+
1458	1735	/*
1459	1736	* Maps the shared page into the user process for checking if event is enabled.
1460	1737	*/
1461	1738	static int user_status_mmap(struct file file, struct vm_area_struct vma)
1462	1739	{
	1740	+ char *pages;
	1741	+ struct user_event_group *group = user_status_group(file);
1463	1742	unsigned long size = vma->vm_end - vma->vm_start;
1464	1743
1465		− if (size != MAX_EVENTS)
	1744	+ if (size != MAX_BYTES)
	1745	+ return -EINVAL;
	1746	+
	1747	+ if (!group)
1466	1748	return -EINVAL;
1467	1749
	1750	+ pages = group->register_page_data;
	1751	+
1468	1752	return remap_pfn_range(vma, vma->vm_start,
1469		− virt_to_phys(register_page_data) >> PAGE_SHIFT,
	1753	+ virt_to_phys(pages) >> PAGE_SHIFT,
1470	1754	size, vm_get_page_prot(VM_READ));
1471	1755	}
1472	1756
@@ -1490,14 +1774,18 @@ static void user_seq_stop(struct seq_file m, void p) · 17 unmodified lines
1490	1774
1491	1775	static int user_seq_show(struct seq_file m, void p)
1492	1776	{
	1777	+ struct user_event_group *group = m->private;
1493	1778	struct user_event *user;
1494	1779	char status;
1495	1780	int i, active = 0, busy = 0, flags;
1496	1781
1497		− mutex_lock(&reg_mutex);
	1782	+ if (!group)
	1783	+ return -EINVAL;
	1784	+
	1785	+ mutex_lock(&group->reg_mutex);
1498	1786
1499		− hash_for_each(register_table, i, user, node) {
1500		− status = register_page_data[user->index];
	1787	+ hash_for_each(group->register_table, i, user, node) {
	1788	+ status = user->status;
1501	1789	flags = user->flags;
1502	1790
1503	1791	seq_printf(m, "%d:%s", user->index, EVENT_NAME(user));
@@ -1520,7 +1808,7 @@ static int user_seq_show(struct seq_file m, void p) · 16 unmodified lines
1520	1808	active++;
1521	1809	}
1522	1810
1523		− mutex_unlock(&reg_mutex);
	1811	+ mutex_unlock(&group->reg_mutex);
1524	1812
1525	1813	seq_puts(m, "\n");
1526	1814	seq_printf(m, "Active: %d\n", active);
@@ -1539,7 +1827,24 @@ static const struct seq_operations user_seq_ops = { · 12 unmodified lines
1539	1827
1540	1828	static int user_status_open(struct inode node, struct file file)
1541	1829	{
1542		− return seq_open(file, &user_seq_ops);
	1830	+ struct user_event_group *group;
	1831	+ int ret;
	1832	+
	1833	+ group = current_user_event_group();
	1834	+
	1835	+ if (!group)
	1836	+ return -ENOENT;
	1837	+
	1838	+ ret = seq_open(file, &user_seq_ops);
	1839	+
	1840	+ if (!ret) {
	1841	+ /* Chain group to seq_file */
	1842	+ struct seq_file *m = file->private_data;
	1843	+
	1844	+ m->private = group;
	1845	+ }
	1846	+
	1847	+ return ret;
1543	1848	}
1544	1849
1545	1850	static const struct file_operations user_status_fops = {
@@ -1580,42 +1885,21 @@ err: · 34 unmodified lines
1580	1885	return -ENODEV;
1581	1886	}
1582	1887
1583		−static void set_page_reservations(bool set)
1584		−{
1585		− int page;
1586		−
1587		− for (page = 0; page < MAX_PAGES; ++page) {
1588		− void addr = register_page_data + (PAGE_SIZE page);
1589		−
1590		− if (set)
1591		− SetPageReserved(virt_to_page(addr));
1592		− else
1593		− ClearPageReserved(virt_to_page(addr));
1594		− }
1595		−}
1596		−
1597	1888	static int __init trace_events_user_init(void)
1598	1889	{
1599		− struct page *pages;
1600	1890	int ret;
1601	1891
1602		− /* Zero all bits beside 0 (which is reserved for failures) */
1603		− bitmap_zero(page_bitmap, MAX_EVENTS);
1604		− set_bit(0, page_bitmap);
	1892	+ init_group = user_event_group_create(&init_user_ns);
1605	1893
1606		− pages = alloc_pages(GFP_KERNEL \| __GFP_ZERO, MAX_PAGE_ORDER);
1607		− if (!pages)
	1894	+ if (!init_group)
1608	1895	return -ENOMEM;
1609		− register_page_data = page_address(pages);
1610		−
1611		− set_page_reservations(true);
1612	1896
1613	1897	ret = create_user_tracefs();
1614	1898
1615	1899	if (ret) {
1616	1900	pr_warn("user_events could not register with tracefs\n");
1617		− set_page_reservations(false);
1618		− __free_pages(pages, MAX_PAGE_ORDER);
	1901	+ user_event_group_destroy(init_group);
	1902	+ init_group = NULL;
1619	1903	return ret;
1620	1904	}
1621	1905

kernel/trace/trace_osnoise.c+2−1

@@ -1786,8 +1786,9 @@ static int start_per_cpu_kthreads(void) · 1785 unmodified lines
1786	1786	for_each_cpu(cpu, current_mask) {
1787	1787	retval = start_kthread(cpu);
1788	1788	if (retval) {
	1789	+ cpus_read_unlock();
1789	1790	stop_per_cpu_kthreads();
1790		− break;
	1791	+ return retval;
1791	1792	}
1792	1793	}
1793	1794

kernel/trace/trace_probe.h+2−1

@@ -445,7 +445,8 @@ extern int traceprobe_define_arg_fields(struct trace_event_call *event_call, · 444 unmodified lines
445	445	C(SAME_PROBE, "There is already the exact same probe event"),\
446	446	C(NO_EVENT_INFO, "This requires both group and event name to attach"),\
447	447	C(BAD_ATTACH_EVENT, "Attached event does not exist"),\
448		− C(BAD_ATTACH_ARG, "Attached event does not have this field"),
	448	+ C(BAD_ATTACH_ARG, "Attached event does not have this field"),\
	449	+ C(NO_EP_FILTER, "No filter rule after 'if'"),
449	450
450	451	#undef C
451	452	#define C(a, b) TP_ERR_##a

kernel/trace/tracing_map.c+2−3

@@ -961,7 +961,7 @@ create_sort_entry(void key, struct tracing_map_elt elt) · 960 unmodified lines
961	961	static void detect_dups(struct tracing_map_sort_entry **sort_entries,
962	962	int n_entries, unsigned int key_size)
963	963	{
964		− unsigned int dups = 0, total_dups = 0;
	964	+ unsigned int total_dups = 0;
965	965	int i;
966	966	void *key;
967	967
@@ -974,11 +974,10 @@ static void detect_dups(struct tracing_map_sort_entry **sort_entries, · 6 unmodified lines
974	974	key = sort_entries[0]->key;
975	975	for (i = 1; i < n_entries; i++) {
976	976	if (!memcmp(sort_entries[i]->key, key, key_size)) {
977		− dups++; total_dups++;
	977	+ total_dups++;
978	978	continue;
979	979	}
980	980	key = sort_entries[i]->key;
981		− dups = 0;
982	981	}
983	982
984	983	WARN_ONCE(total_dups > 0,

kernel/tracepoint.c+6−8

@@ -640,7 +640,6 @@ static void tp_module_going_check_quiescent(struct tracepoint tp, void priv) · 639 unmodified lines
640	640	static int tracepoint_module_coming(struct module *mod)
641	641	{
642	642	struct tp_module *tp_mod;
643		− int ret = 0;
644	643
645	644	if (!mod->num_tracepoints)
646	645	return 0;
@@ -652,19 +651,18 @@ static int tracepoint_module_coming(struct module *mod) · 5 unmodified lines
652	651	*/
653	652	if (trace_module_has_bad_taint(mod))
654	653	return 0;
655		− mutex_lock(&tracepoint_module_list_mutex);
	654	+
656	655	tp_mod = kmalloc(sizeof(struct tp_module), GFP_KERNEL);
657		− if (!tp_mod) {
658		− ret = -ENOMEM;
659		− goto end;
660		− }
	656	+ if (!tp_mod)
	657	+ return -ENOMEM;
661	658	tp_mod->mod = mod;
	659	+
	660	+ mutex_lock(&tracepoint_module_list_mutex);
662	661	list_add_tail(&tp_mod->list, &tracepoint_module_list);
663	662	blocking_notifier_call_chain(&tracepoint_notify_list,
664	663	MODULE_STATE_COMING, tp_mod);
665		−end:
666	664	mutex_unlock(&tracepoint_module_list_mutex);
667		− return ret;
	665	+ return 0;
668	666	}
669	667
670	668	static void tracepoint_module_going(struct module *mod)