Merge tag 'trace-v5.18' of git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-trace

Pull tracing updates from Steven Rostedt: - New user_events interface. User space can register an event with the kernel describing the format of the event. Then it will receive a byte in a page mapping that it can check against. A privileged task can then enable that event like any other event, which will change the mapped byte to true, telling the user space application to start writing the event to the tracing buffer. - Add new "ftrace_boot_snapshot" kernel command line parameter. When set, the tracing buffer will be saved in the snapshot buffer at boot up when the kernel hands things over to user space. This will keep the traces that happened at boot up available even if user space boot up has tracing as well. - Have TRACE_EVENT_ENUM() also update trace event field type descriptions. Thus if a static array defines its size with an enum, the user space trace event parsers can still know how to parse that array. - Add new TRACE_CUSTOM_EVENT() macro. This acts the same as the TRACE_EVENT() macro, but will attach to an existing tracepoint. This will make one tracepoint be able to trace different content and not be stuck at only what the original TRACE_EVENT() macro exports. - Fixes to tracing error logging. - Better saving of cmdlines to PIDs when tracing (use the wakeup events for mapping). * tag 'trace-v5.18' of git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-trace: (30 commits) tracing: Have type enum modifications copy the strings user_events: Add trace event call as root for low permission cases tracing/user_events: Use alloc_pages instead of kzalloc() for register pages tracing: Add snapshot at end of kernel boot up tracing: Have TRACE_DEFINE_ENUM affect trace event types as well tracing: Fix strncpy warning in trace_events_synth.c user_events: Prevent dyn_event delete racing with ioctl add/delete tracing: Add TRACE_CUSTOM_EVENT() macro tracing: Move the defines to create TRACE_EVENTS into their own files tracing: Add sample code for custom trace events tracing: Allow custom events to be added to the tracefs directory tracing: Fix last_cmd_set() string management in histogram code user_events: Fix potential uninitialized pointer while parsing field tracing: Fix allocation of last_cmd in last_cmd_set() user_events: Add documentation file user_events: Add sample code for typical usage user_events: Add self-test for validator boundaries user_events: Add self-test for perf_event integration user_events: Add self-test for dynamic_events integration user_events: Add self-test for ftrace integration ...

Linus Torvalds committed 4 years ago

Commit 1bc19102 parents 20f463f 795301d39 files · +4169 −570Browse files

Unified · Split

39 files

Documentation/admin-guide/kernel-parameters.txt +8 −0
Documentation/trace/index.rst +1 −0
Documentation/trace/user_events.rst +216 −0
include/linux/ftrace.h +10 −1
include/linux/trace_events.h +23 −1
include/trace/define_custom_trace.h +77 −0
include/trace/stages/init.h +37 −0
include/trace/stages/stage1_defines.h +51 −0
include/trace/stages/stage2_defines.h +54 −0
include/trace/stages/stage3_defines.h +135 −0
include/trace/stages/stage4_defines.h +63 −0
include/trace/stages/stage5_defines.h +89 −0
include/trace/stages/stage6_defines.h +106 −0
include/trace/stages/stage7_defines.h +36 −0
include/trace/trace_custom_events.h +221 −0
include/trace/trace_events.h +8 −542
include/uapi/linux/user_events.h +116 −0
kernel/trace/Kconfig +14 −0
kernel/trace/Makefile +1 −0
kernel/trace/ftrace.c +2 −0
kernel/trace/trace.c +60 −13
kernel/trace/trace.h +1 −1
kernel/trace/trace_events.c +90 −0
kernel/trace/trace_events_hist.c +26 −7
kernel/trace/trace_events_synth.c +11 −3
kernel/trace/trace_events_user.c +1690 −0
kernel/trace/trace_sched_switch.c +1 −1
samples/Kconfig +7 −1
samples/Makefile +1 −0
samples/trace_events/Makefile +2 −0
samples/trace_events/trace_custom_sched.c +60 −0
samples/trace_events/trace_custom_sched.h +96 −0
samples/user_events/Makefile +5 −0
samples/user_events/example.c +91 −0
tools/testing/selftests/user_events/Makefile +9 −0
tools/testing/selftests/user_events/dyn_test.c +130 −0
tools/testing/selftests/user_events/ftrace_test.c +452 −0
tools/testing/selftests/user_events/perf_test.c +168 −0
tools/testing/selftests/user_events/settings +1 −0

Documentation/admin-guide/kernel-parameters.txt+8−0

@@ -1465,6 +1465,14 @@ · 1464 unmodified lines
1465	as early as possible in order to facilitate early	1465	as early as possible in order to facilitate early
1466	boot debugging.	1466	boot debugging.
1467		1467
		1468	ftrace_boot_snapshot
		1469	[FTRACE] On boot up, a snapshot will be taken of the
		1470	ftrace ring buffer that can be read at:
		1471	/sys/kernel/tracing/snapshot.
		1472	This is useful if you need tracing information from kernel
		1473	boot up that is likely to be overridden by user space
		1474	start up functionality.
		1475
1468	ftrace_dump_on_oops[=orig_cpu]	1476	ftrace_dump_on_oops[=orig_cpu]
1469	[FTRACE] will dump the trace buffers on oops.	1477	[FTRACE] will dump the trace buffers on oops.
1470	If no parameter is passed, ftrace will dump	1478	If no parameter is passed, ftrace will dump

Documentation/trace/index.rst+1−0

@@ -30,3 +30,4 @@ Linux Tracing Technologies · 29 unmodified lines
30	stm	30	stm
31	sys-t	31	sys-t
32	coresight/index	32	coresight/index
		33	user_events

Documentation/trace/user_events.rst+216−0

@@ -0,0 +1,216 @@
	1	=========================================
	2	user_events: User-based Event Tracing
	3	=========================================
	4
	5	:Author: Beau Belgrave
	6
	7	Overview
	8	--------
	9	User based trace events allow user processes to create events and trace data
	10	that can be viewed via existing tools, such as ftrace, perf and eBPF.
	11	To enable this feature, build your kernel with CONFIG_USER_EVENTS=y.
	12
	13	Programs can view status of the events via
	14	/sys/kernel/debug/tracing/user_events_status and can both register and write
	15	data out via /sys/kernel/debug/tracing/user_events_data.
	16
	17	Programs can also use /sys/kernel/debug/tracing/dynamic_events to register and
	18	delete user based events via the u: prefix. The format of the command to
	19	dynamic_events is the same as the ioctl with the u: prefix applied.
	20
	21	Typically programs will register a set of events that they wish to expose to
	22	tools that can read trace_events (such as ftrace and perf). The registration
	23	process gives back two ints to the program for each event. The first int is the
	24	status index. This index describes which byte in the
	25	/sys/kernel/debug/tracing/user_events_status file represents this event. The
	26	second int is the write index. This index describes the data when a write() or
	27	writev() is called on the /sys/kernel/debug/tracing/user_events_data file.
	28
	29	The structures referenced in this document are contained with the
	30	/include/uap/linux/user_events.h file in the source tree.
	31
	32	NOTE: *Both user_events_status and user_events_data are under the tracefs
	33	filesystem and may be mounted at different paths than above.*
	34
	35	Registering
	36	-----------
	37	Registering within a user process is done via ioctl() out to the
	38	/sys/kernel/debug/tracing/user_events_data file. The command to issue is
	39	DIAG_IOCSREG.
	40
	41	This command takes a struct user_reg as an argument::
	42
	43	struct user_reg {
	44	u32 size;
	45	u64 name_args;
	46	u32 status_index;
	47	u32 write_index;
	48	};
	49
	50	The struct user_reg requires two inputs, the first is the size of the structure
	51	to ensure forward and backward compatibility. The second is the command string
	52	to issue for registering. Upon success two outputs are set, the status index
	53	and the write index.
	54
	55	User based events show up under tracefs like any other event under the
	56	subsystem named "user_events". This means tools that wish to attach to the
	57	events need to use /sys/kernel/debug/tracing/events/user_events/[name]/enable
	58	or perf record -e user_events:[name] when attaching/recording.
	59
	60	NOTE: The write_index returned is only valid for the FD that was used
	61
	62	Command Format
	63	^^^^^^^^^^^^^^
	64	The command string format is as follows::
	65
	66	name[:FLAG1[,FLAG2...]] [Field1[;Field2...]]
	67
	68	Supported Flags
	69	^^^^^^^^^^^^^^^
	70	BPF_ITER - EBPF programs attached to this event will get the raw iovec
	71	struct instead of any data copies for max performance.
	72
	73	Field Format
	74	^^^^^^^^^^^^
	75	::
	76
	77	type name [size]
	78
	79	Basic types are supported (__data_loc, u32, u64, int, char, char[20], etc).
	80	User programs are encouraged to use clearly sized types like u32.
	81
	82	NOTE: Long is not supported since size can vary between user and kernel.
	83
	84	The size is only valid for types that start with a struct prefix.
	85	This allows user programs to describe custom structs out to tools, if required.
	86
	87	For example, a struct in C that looks like this::
	88
	89	struct mytype {
	90	char data[20];
	91	};
	92
	93	Would be represented by the following field::
	94
	95	struct mytype myname 20
	96
	97	Deleting
	98	-----------
	99	Deleting an event from within a user process is done via ioctl() out to the
	100	/sys/kernel/debug/tracing/user_events_data file. The command to issue is
	101	DIAG_IOCSDEL.
	102
	103	This command only requires a single string specifying the event to delete by
	104	its name. Delete will only succeed if there are no references left to the
	105	event (in both user and kernel space). User programs should use a separate file
	106	to request deletes than the one used for registration due to this.
	107
	108	Status
	109	------
	110	When tools attach/record user based events the status of the event is updated
	111	in realtime. This allows user programs to only incur the cost of the write() or
	112	writev() calls when something is actively attached to the event.
	113
	114	User programs call mmap() on /sys/kernel/debug/tracing/user_events_status to
	115	check the status for each event that is registered. The byte to check in the
	116	file is given back after the register ioctl() via user_reg.status_index.
	117	Currently the size of user_events_status is a single page, however, custom
	118	kernel configurations can change this size to allow more user based events. In
	119	all cases the size of the file is a multiple of a page size.
	120
	121	For example, if the register ioctl() gives back a status_index of 3 you would
	122	check byte 3 of the returned mmap data to see if anything is attached to that
	123	event.
	124
	125	Administrators can easily check the status of all registered events by reading
	126	the user_events_status file directly via a terminal. The output is as follows::
	127
	128	Byte:Name [# Comments]
	129	...
	130
	131	Active: ActiveCount
	132	Busy: BusyCount
	133	Max: MaxCount
	134
	135	For example, on a system that has a single event the output looks like this::
	136
	137	1:test
	138
	139	Active: 1
	140	Busy: 0
	141	Max: 4096
	142
	143	If a user enables the user event via ftrace, the output would change to this::
	144
	145	1:test # Used by ftrace
	146
	147	Active: 1
	148	Busy: 1
	149	Max: 4096
	150
	151	NOTE: *A status index of 0 will never be returned. This allows user
	152	programs to have an index that can be used on error cases.*
	153
	154	Status Bits
	155	^^^^^^^^^^^
	156	The byte being checked will be non-zero if anything is attached. Programs can
	157	check specific bits in the byte to see what mechanism has been attached.
	158
	159	The following values are defined to aid in checking what has been attached:
	160
	161	EVENT_STATUS_FTRACE - Bit set if ftrace has been attached (Bit 0).
	162
	163	EVENT_STATUS_PERF - Bit set if perf/eBPF has been attached (Bit 1).
	164
	165	Writing Data
	166	------------
	167	After registering an event the same fd that was used to register can be used
	168	to write an entry for that event. The write_index returned must be at the start
	169	of the data, then the remaining data is treated as the payload of the event.
	170
	171	For example, if write_index returned was 1 and I wanted to write out an int
	172	payload of the event. Then the data would have to be 8 bytes (2 ints) in size,
	173	with the first 4 bytes being equal to 1 and the last 4 bytes being equal to the
	174	value I want as the payload.
	175
	176	In memory this would look like this::
	177
	178	int index;
	179	int payload;
	180
	181	User programs might have well known structs that they wish to use to emit out
	182	as payloads. In those cases writev() can be used, with the first vector being
	183	the index and the following vector(s) being the actual event payload.
	184
	185	For example, if I have a struct like this::
	186
	187	struct payload {
	188	int src;
	189	int dst;
	190	int flags;
	191	};
	192
	193	It's advised for user programs to do the following::
	194
	195	struct iovec io[2];
	196	struct payload e;
	197
	198	io[0].iov_base = &write_index;
	199	io[0].iov_len = sizeof(write_index);
	200	io[1].iov_base = &e;
	201	io[1].iov_len = sizeof(e);
	202
	203	writev(fd, (const struct iovec*)io, 2);
	204
	205	NOTE: The write_index is not emitted out into the trace being recorded.
	206
	207	EBPF
	208	----
	209	EBPF programs that attach to a user-based event tracepoint are given a pointer
	210	to a struct user_bpf_context. The bpf context contains the data type (which can
	211	be a user or kernel buffer, or can be a pointer to the iovec) and the data
	212	length that was emitted (minus the write_index).
	213
	214	Example Code
	215	------------
	216	See sample code in samples/user_events.

include/linux/ftrace.h+10−1

 #define ARCH_SUPPORTS_FTRACE_OPS 0
 #endif
+#ifdef CONFIG_TRACING
+extern void ftrace_boot_snapshot(void);
+#else
+static inline void ftrace_boot_snapshot(void) { }
+#endif
 #ifdef CONFIG_FUNCTION_TRACER
 struct ftrace_ops;
 struct ftrace_regs;
 void ftrace_free_init_mem(void);
 void ftrace_free_mem(struct module *mod, void *start, void *end);
 #else
-static inline void ftrace_free_init_mem(void) { }
+static inline void ftrace_free_init_mem(void)
+{
+	ftrace_boot_snapshot();
+}
 static inline void ftrace_free_mem(struct module *mod, void *start, void *end) { }
 #endif

include/linux/trace_events.h+23−1

 	TRACE_EVENT_FL_KPROBE_BIT,
 	TRACE_EVENT_FL_UPROBE_BIT,
 	TRACE_EVENT_FL_EPROBE_BIT,
+	TRACE_EVENT_FL_CUSTOM_BIT,
 };
 /*
  *  KPROBE        - Event is a kprobe
  *  UPROBE        - Event is a uprobe
  *  EPROBE        - Event is an event probe
+ *  CUSTOM        - Event is a custom event (to be attached to an exsiting tracepoint)
+ *                   This is set when the custom event has not been attached
+ *                   to a tracepoint yet, then it is cleared when it is.
  */
 enum {
 	TRACE_EVENT_FL_FILTERED		= (1 << TRACE_EVENT_FL_FILTERED_BIT),
 	TRACE_EVENT_FL_KPROBE		= (1 << TRACE_EVENT_FL_KPROBE_BIT),
 	TRACE_EVENT_FL_UPROBE		= (1 << TRACE_EVENT_FL_UPROBE_BIT),
 	TRACE_EVENT_FL_EPROBE		= (1 << TRACE_EVENT_FL_EPROBE_BIT),
+	TRACE_EVENT_FL_CUSTOM		= (1 << TRACE_EVENT_FL_CUSTOM_BIT),
 };
 #define TRACE_EVENT_FL_UKPROBE (TRACE_EVENT_FL_KPROBE | TRACE_EVENT_FL_UPROBE)
 static inline const char *
 trace_event_name(struct trace_event_call *call)
 {
-	if (call->flags & TRACE_EVENT_FL_TRACEPOINT)
+	if (call->flags & TRACE_EVENT_FL_CUSTOM)
+		return call->name;
+	else if (call->flags & TRACE_EVENT_FL_TRACEPOINT)
 		return call->tp ? call->tp->name : NULL;
 	else
 		return call->name;
 #endif
 #endif /* _LINUX_TRACE_EVENT_H */
+/*
+ * Note: we keep the TRACE_CUSTOM_EVENT outside the include file ifdef protection.
+ *  This is due to the way trace custom events work. If a file includes two
+ *  trace event headers under one "CREATE_CUSTOM_TRACE_EVENTS" the first include
+ *  will override the TRACE_CUSTOM_EVENT and break the second include.
+ */
+#ifndef TRACE_CUSTOM_EVENT
+#define DECLARE_CUSTOM_EVENT_CLASS(name, proto, args, tstruct, assign, print)
+#define DEFINE_CUSTOM_EVENT(template, name, proto, args)
+#define TRACE_CUSTOM_EVENT(name, proto, args, struct, assign, print)
+#endif /* ifdef TRACE_CUSTOM_EVENT (see note above) */

include/trace/define_custom_trace.h+77−0

@@ -0,0 +1,77 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	/*
	3	* Trace files that want to automate creation of all tracepoints defined
	4	* in their file should include this file. The following are macros that the
	5	* trace file may define:
	6	*
	7	* TRACE_SYSTEM defines the system the tracepoint is for
	8	*
	9	* TRACE_INCLUDE_FILE if the file name is something other than TRACE_SYSTEM.h
	10	* This macro may be defined to tell define_trace.h what file to include.
	11	* Note, leave off the ".h".
	12	*
	13	* TRACE_INCLUDE_PATH if the path is something other than core kernel include/trace
	14	* then this macro can define the path to use. Note, the path is relative to
	15	* define_trace.h, not the file including it. Full path names for out of tree
	16	* modules must be used.
	17	*/
	18
	19	#ifdef CREATE_CUSTOM_TRACE_EVENTS
	20
	21	/* Prevent recursion */
	22	#undef CREATE_CUSTOM_TRACE_EVENTS
	23
	24	#include <linux/stringify.h>
	25
	26	#undef TRACE_CUSTOM_EVENT
	27	#define TRACE_CUSTOM_EVENT(name, proto, args, tstruct, assign, print)
	28
	29	#undef DEFINE_CUSTOM_EVENT
	30	#define DEFINE_CUSTOM_EVENT(template, name, proto, args)
	31
	32	#undef TRACE_INCLUDE
	33	#undef __TRACE_INCLUDE
	34
	35	#ifndef TRACE_INCLUDE_FILE
	36	# define TRACE_INCLUDE_FILE TRACE_SYSTEM
	37	# define UNDEF_TRACE_INCLUDE_FILE
	38	#endif
	39
	40	#ifndef TRACE_INCLUDE_PATH
	41	# define __TRACE_INCLUDE(system) <trace/events/system.h>
	42	# define UNDEF_TRACE_INCLUDE_PATH
	43	#else
	44	# define __TRACE_INCLUDE(system) __stringify(TRACE_INCLUDE_PATH/system.h)
	45	#endif
	46
	47	# define TRACE_INCLUDE(system) __TRACE_INCLUDE(system)
	48
	49	/* Let the trace headers be reread */
	50	#define TRACE_CUSTOM_MULTI_READ
	51
	52	#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
	53
	54	#ifdef TRACEPOINTS_ENABLED
	55	#include <trace/trace_custom_events.h>
	56	#endif
	57
	58	#undef TRACE_CUSTOM_EVENT
	59	#undef DECLARE_CUSTOM_EVENT_CLASS
	60	#undef DEFINE_CUSTOM_EVENT
	61	#undef TRACE_CUSTOM_MULTI_READ
	62
	63	/* Only undef what we defined in this file */
	64	#ifdef UNDEF_TRACE_INCLUDE_FILE
	65	# undef TRACE_INCLUDE_FILE
	66	# undef UNDEF_TRACE_INCLUDE_FILE
	67	#endif
	68
	69	#ifdef UNDEF_TRACE_INCLUDE_PATH
	70	# undef TRACE_INCLUDE_PATH
	71	# undef UNDEF_TRACE_INCLUDE_PATH
	72	#endif
	73
	74	/* We may be processing more files */
	75	#define CREATE_CUSTOM_TRACE_POINTS
	76
	77	#endif /* CREATE_CUSTOM_TRACE_POINTS */

include/trace/stages/init.h+37−0

@@ -0,0 +1,37 @@
	1
	2	#define __app__(x, y) str__##x##y
	3	#define __app(x, y) __app__(x, y)
	4
	5	#define TRACE_SYSTEM_STRING __app(TRACE_SYSTEM_VAR,__trace_system_name)
	6
	7	#define TRACE_MAKE_SYSTEM_STR() \
	8	static const char TRACE_SYSTEM_STRING[] = \
	9	__stringify(TRACE_SYSTEM)
	10
	11	TRACE_MAKE_SYSTEM_STR();
	12
	13	#undef TRACE_DEFINE_ENUM
	14	#define TRACE_DEFINE_ENUM(a) \
	15	static struct trace_eval_map __used __initdata \
	16	__##TRACE_SYSTEM##_##a = \
	17	{ \
	18	.system = TRACE_SYSTEM_STRING, \
	19	.eval_string = #a, \
	20	.eval_value = a \
	21	}; \
	22	static struct trace_eval_map __used \
	23	__section("_ftrace_eval_map") \
	24	*TRACE_SYSTEM##_##a = &__##TRACE_SYSTEM##_##a
	25
	26	#undef TRACE_DEFINE_SIZEOF
	27	#define TRACE_DEFINE_SIZEOF(a) \
	28	static struct trace_eval_map __used __initdata \
	29	__##TRACE_SYSTEM##_##a = \
	30	{ \
	31	.system = TRACE_SYSTEM_STRING, \
	32	.eval_string = "sizeof(" #a ")", \
	33	.eval_value = sizeof(a) \
	34	}; \
	35	static struct trace_eval_map __used \
	36	__section("_ftrace_eval_map") \
	37	*TRACE_SYSTEM##_##a = &__##TRACE_SYSTEM##_##a

include/trace/stages/stage1_defines.h+51−0

@@ -0,0 +1,51 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2
	3	/* Stage 1 definitions for creating trace events */
	4
	5	#undef __field
	6	#define __field(type, item) type item;
	7
	8	#undef __field_ext
	9	#define __field_ext(type, item, filter_type) type item;
	10
	11	#undef __field_struct
	12	#define __field_struct(type, item) type item;
	13
	14	#undef __field_struct_ext
	15	#define __field_struct_ext(type, item, filter_type) type item;
	16
	17	#undef __array
	18	#define __array(type, item, len) type item[len];
	19
	20	#undef __dynamic_array
	21	#define __dynamic_array(type, item, len) u32 __data_loc_##item;
	22
	23	#undef __string
	24	#define __string(item, src) __dynamic_array(char, item, -1)
	25
	26	#undef __string_len
	27	#define __string_len(item, src, len) __dynamic_array(char, item, -1)
	28
	29	#undef __bitmask
	30	#define __bitmask(item, nr_bits) __dynamic_array(char, item, -1)
	31
	32	#undef __sockaddr
	33	#define __sockaddr(field, len) __dynamic_array(u8, field, len)
	34
	35	#undef __rel_dynamic_array
	36	#define __rel_dynamic_array(type, item, len) u32 __rel_loc_##item;
	37
	38	#undef __rel_string
	39	#define __rel_string(item, src) __rel_dynamic_array(char, item, -1)
	40
	41	#undef __rel_string_len
	42	#define __rel_string_len(item, src, len) __rel_dynamic_array(char, item, -1)
	43
	44	#undef __rel_bitmask
	45	#define __rel_bitmask(item, nr_bits) __rel_dynamic_array(char, item, -1)
	46
	47	#undef __rel_sockaddr
	48	#define __rel_sockaddr(field, len) __rel_dynamic_array(u8, field, len)
	49
	50	#undef TP_STRUCT__entry
	51	#define TP_STRUCT__entry(args...) args

include/trace/stages/stage2_defines.h+54−0

@@ -0,0 +1,54 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2
	3	/* Stage 2 definitions for creating trace events */
	4
	5	#undef TRACE_DEFINE_ENUM
	6	#define TRACE_DEFINE_ENUM(a)
	7
	8	#undef TRACE_DEFINE_SIZEOF
	9	#define TRACE_DEFINE_SIZEOF(a)
	10
	11	#undef __field
	12	#define __field(type, item)
	13
	14	#undef __field_ext
	15	#define __field_ext(type, item, filter_type)
	16
	17	#undef __field_struct
	18	#define __field_struct(type, item)
	19
	20	#undef __field_struct_ext
	21	#define __field_struct_ext(type, item, filter_type)
	22
	23	#undef __array
	24	#define __array(type, item, len)
	25
	26	#undef __dynamic_array
	27	#define __dynamic_array(type, item, len) u32 item;
	28
	29	#undef __string
	30	#define __string(item, src) __dynamic_array(char, item, -1)
	31
	32	#undef __string_len
	33	#define __string_len(item, src, len) __dynamic_array(char, item, -1)
	34
	35	#undef __bitmask
	36	#define __bitmask(item, nr_bits) __dynamic_array(unsigned long, item, -1)
	37
	38	#undef __sockaddr
	39	#define __sockaddr(field, len) __dynamic_array(u8, field, len)
	40
	41	#undef __rel_dynamic_array
	42	#define __rel_dynamic_array(type, item, len) u32 item;
	43
	44	#undef __rel_string
	45	#define __rel_string(item, src) __rel_dynamic_array(char, item, -1)
	46
	47	#undef __rel_string_len
	48	#define __rel_string_len(item, src, len) __rel_dynamic_array(char, item, -1)
	49
	50	#undef __rel_bitmask
	51	#define __rel_bitmask(item, nr_bits) __rel_dynamic_array(unsigned long, item, -1)
	52
	53	#undef __rel_sockaddr
	54	#define __rel_sockaddr(field, len) __rel_dynamic_array(u8, field, len)

include/trace/stages/stage3_defines.h+135−0

@@ -0,0 +1,135 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2
	3	/* Stage 3 definitions for creating trace events */
	4
	5	#undef __entry
	6	#define __entry field
	7
	8	#undef TP_printk
	9	#define TP_printk(fmt, args...) fmt "\n", args
	10
	11	#undef __get_dynamic_array
	12	#define __get_dynamic_array(field) \
	13	((void *)__entry + (__entry->__data_loc_##field & 0xffff))
	14
	15	#undef __get_dynamic_array_len
	16	#define __get_dynamic_array_len(field) \
	17	((__entry->__data_loc_##field >> 16) & 0xffff)
	18
	19	#undef __get_str
	20	#define __get_str(field) ((char *)__get_dynamic_array(field))
	21
	22	#undef __get_rel_dynamic_array
	23	#define __get_rel_dynamic_array(field) \
	24	((void *)__entry + \
	25	offsetof(typeof(*__entry), __rel_loc_##field) + \
	26	sizeof(__entry->__rel_loc_##field) + \
	27	(__entry->__rel_loc_##field & 0xffff))
	28
	29	#undef __get_rel_dynamic_array_len
	30	#define __get_rel_dynamic_array_len(field) \
	31	((__entry->__rel_loc_##field >> 16) & 0xffff)
	32
	33	#undef __get_rel_str
	34	#define __get_rel_str(field) ((char *)__get_rel_dynamic_array(field))
	35
	36	#undef __get_bitmask
	37	#define __get_bitmask(field) \
	38	({ \
	39	void *__bitmask = __get_dynamic_array(field); \
	40	unsigned int __bitmask_size; \
	41	__bitmask_size = __get_dynamic_array_len(field); \
	42	trace_print_bitmask_seq(p, __bitmask, __bitmask_size); \
	43	})
	44
	45	#undef __get_rel_bitmask
	46	#define __get_rel_bitmask(field) \
	47	({ \
	48	void *__bitmask = __get_rel_dynamic_array(field); \
	49	unsigned int __bitmask_size; \
	50	__bitmask_size = __get_rel_dynamic_array_len(field); \
	51	trace_print_bitmask_seq(p, __bitmask, __bitmask_size); \
	52	})
	53
	54	#undef __get_sockaddr
	55	#define __get_sockaddr(field) ((struct sockaddr *)__get_dynamic_array(field))
	56
	57	#undef __get_rel_sockaddr
	58	#define __get_rel_sockaddr(field) ((struct sockaddr *)__get_rel_dynamic_array(field))
	59
	60	#undef __print_flags
	61	#define __print_flags(flag, delim, flag_array...) \
	62	({ \
	63	static const struct trace_print_flags __flags[] = \
	64	{ flag_array, { -1, NULL }}; \
	65	trace_print_flags_seq(p, delim, flag, __flags); \
	66	})
	67
	68	#undef __print_symbolic
	69	#define __print_symbolic(value, symbol_array...) \
	70	({ \
	71	static const struct trace_print_flags symbols[] = \
	72	{ symbol_array, { -1, NULL }}; \
	73	trace_print_symbols_seq(p, value, symbols); \
	74	})
	75
	76	#undef __print_flags_u64
	77	#undef __print_symbolic_u64
	78	#if BITS_PER_LONG == 32
	79	#define __print_flags_u64(flag, delim, flag_array...) \
	80	({ \
	81	static const struct trace_print_flags_u64 __flags[] = \
	82	{ flag_array, { -1, NULL } }; \
	83	trace_print_flags_seq_u64(p, delim, flag, __flags); \
	84	})
	85
	86	#define __print_symbolic_u64(value, symbol_array...) \
	87	({ \
	88	static const struct trace_print_flags_u64 symbols[] = \
	89	{ symbol_array, { -1, NULL } }; \
	90	trace_print_symbols_seq_u64(p, value, symbols); \
	91	})
	92	#else
	93	#define __print_flags_u64(flag, delim, flag_array...) \
	94	__print_flags(flag, delim, flag_array)
	95
	96	#define __print_symbolic_u64(value, symbol_array...) \
	97	__print_symbolic(value, symbol_array)
	98	#endif
	99
	100	#undef __print_hex
	101	#define __print_hex(buf, buf_len) \
	102	trace_print_hex_seq(p, buf, buf_len, false)
	103
	104	#undef __print_hex_str
	105	#define __print_hex_str(buf, buf_len) \
	106	trace_print_hex_seq(p, buf, buf_len, true)
	107
	108	#undef __print_array
	109	#define __print_array(array, count, el_size) \
	110	({ \
	111	BUILD_BUG_ON(el_size != 1 && el_size != 2 && \
	112	el_size != 4 && el_size != 8); \
	113	trace_print_array_seq(p, array, count, el_size); \
	114	})
	115
	116	#undef __print_hex_dump
	117	#define __print_hex_dump(prefix_str, prefix_type, \
	118	rowsize, groupsize, buf, len, ascii) \
	119	trace_print_hex_dump_seq(p, prefix_str, prefix_type, \
	120	rowsize, groupsize, buf, len, ascii)
	121
	122	#undef __print_ns_to_secs
	123	#define __print_ns_to_secs(value) \
	124	({ \
	125	u64 ____val = (u64)(value); \
	126	do_div(____val, NSEC_PER_SEC); \
	127	____val; \
	128	})
	129
	130	#undef __print_ns_without_secs
	131	#define __print_ns_without_secs(value) \
	132	({ \
	133	u64 ____val = (u64)(value); \
	134	(u32) do_div(____val, NSEC_PER_SEC); \
	135	})

include/trace/stages/stage4_defines.h+63−0

@@ -0,0 +1,63 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2
	3	/* Stage 4 definitions for creating trace events */
	4
	5	#undef __field_ext
	6	#define __field_ext(_type, _item, _filter_type) { \
	7	.type = #_type, .name = #_item, \
	8	.size = sizeof(_type), .align = __alignof__(_type), \
	9	.is_signed = is_signed_type(_type), .filter_type = _filter_type },
	10
	11	#undef __field_struct_ext
	12	#define __field_struct_ext(_type, _item, _filter_type) { \
	13	.type = #_type, .name = #_item, \
	14	.size = sizeof(_type), .align = __alignof__(_type), \
	15	0, .filter_type = _filter_type },
	16
	17	#undef __field
	18	#define __field(type, item) __field_ext(type, item, FILTER_OTHER)
	19
	20	#undef __field_struct
	21	#define __field_struct(type, item) __field_struct_ext(type, item, FILTER_OTHER)
	22
	23	#undef __array
	24	#define __array(_type, _item, _len) { \
	25	.type = #_type"["__stringify(_len)"]", .name = #_item, \
	26	.size = sizeof(_type[_len]), .align = __alignof__(_type), \
	27	.is_signed = is_signed_type(_type), .filter_type = FILTER_OTHER },
	28
	29	#undef __dynamic_array
	30	#define __dynamic_array(_type, _item, _len) { \
	31	.type = "__data_loc " #_type "[]", .name = #_item, \
	32	.size = 4, .align = 4, \
	33	.is_signed = is_signed_type(_type), .filter_type = FILTER_OTHER },
	34
	35	#undef __string
	36	#define __string(item, src) __dynamic_array(char, item, -1)
	37
	38	#undef __string_len
	39	#define __string_len(item, src, len) __dynamic_array(char, item, -1)
	40
	41	#undef __bitmask
	42	#define __bitmask(item, nr_bits) __dynamic_array(unsigned long, item, -1)
	43
	44	#undef __sockaddr
	45	#define __sockaddr(field, len) __dynamic_array(u8, field, len)
	46
	47	#undef __rel_dynamic_array
	48	#define __rel_dynamic_array(_type, _item, _len) { \
	49	.type = "__rel_loc " #_type "[]", .name = #_item, \
	50	.size = 4, .align = 4, \
	51	.is_signed = is_signed_type(_type), .filter_type = FILTER_OTHER },
	52
	53	#undef __rel_string
	54	#define __rel_string(item, src) __rel_dynamic_array(char, item, -1)
	55
	56	#undef __rel_string_len
	57	#define __rel_string_len(item, src, len) __rel_dynamic_array(char, item, -1)
	58
	59	#undef __rel_bitmask
	60	#define __rel_bitmask(item, nr_bits) __rel_dynamic_array(unsigned long, item, -1)
	61
	62	#undef __rel_sockaddr
	63	#define __rel_sockaddr(field, len) __rel_dynamic_array(u8, field, len)

include/trace/stages/stage5_defines.h+89−0

@@ -0,0 +1,89 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2
	3	/* Stage 5 definitions for creating trace events */
	4
	5	/*
	6	* remember the offset of each array from the beginning of the event.
	7	*/
	8
	9	#undef __entry
	10	#define __entry entry
	11
	12	#undef __field
	13	#define __field(type, item)
	14
	15	#undef __field_ext
	16	#define __field_ext(type, item, filter_type)
	17
	18	#undef __field_struct
	19	#define __field_struct(type, item)
	20
	21	#undef __field_struct_ext
	22	#define __field_struct_ext(type, item, filter_type)
	23
	24	#undef __array
	25	#define __array(type, item, len)
	26
	27	#undef __dynamic_array
	28	#define __dynamic_array(type, item, len) \
	29	__item_length = (len) * sizeof(type); \
	30	__data_offsets->item = __data_size + \
	31	offsetof(typeof(*entry), __data); \
	32	__data_offsets->item \|= __item_length << 16; \
	33	__data_size += __item_length;
	34
	35	#undef __string
	36	#define __string(item, src) __dynamic_array(char, item, \
	37	strlen((src) ? (const char *)(src) : "(null)") + 1)
	38
	39	#undef __string_len
	40	#define __string_len(item, src, len) __dynamic_array(char, item, (len) + 1)
	41
	42	#undef __rel_dynamic_array
	43	#define __rel_dynamic_array(type, item, len) \
	44	__item_length = (len) * sizeof(type); \
	45	__data_offsets->item = __data_size + \
	46	offsetof(typeof(*entry), __data) - \
	47	offsetof(typeof(*entry), __rel_loc_##item) - \
	48	sizeof(u32); \
	49	__data_offsets->item \|= __item_length << 16; \
	50	__data_size += __item_length;
	51
	52	#undef __rel_string
	53	#define __rel_string(item, src) __rel_dynamic_array(char, item, \
	54	strlen((src) ? (const char *)(src) : "(null)") + 1)
	55
	56	#undef __rel_string_len
	57	#define __rel_string_len(item, src, len) __rel_dynamic_array(char, item, (len) + 1)
	58	/*
	59	* __bitmask_size_in_bytes_raw is the number of bytes needed to hold
	60	* num_possible_cpus().
	61	*/
	62	#define __bitmask_size_in_bytes_raw(nr_bits) \
	63	(((nr_bits) + 7) / 8)
	64
	65	#define __bitmask_size_in_longs(nr_bits) \
	66	((__bitmask_size_in_bytes_raw(nr_bits) + \
	67	((BITS_PER_LONG / 8) - 1)) / (BITS_PER_LONG / 8))
	68
	69	/*
	70	* __bitmask_size_in_bytes is the number of bytes needed to hold
	71	* num_possible_cpus() padded out to the nearest long. This is what
	72	* is saved in the buffer, just to be consistent.
	73	*/
	74	#define __bitmask_size_in_bytes(nr_bits) \
	75	(__bitmask_size_in_longs(nr_bits) * (BITS_PER_LONG / 8))
	76
	77	#undef __bitmask
	78	#define __bitmask(item, nr_bits) __dynamic_array(unsigned long, item, \
	79	__bitmask_size_in_longs(nr_bits))
	80
	81	#undef __rel_bitmask
	82	#define __rel_bitmask(item, nr_bits) __rel_dynamic_array(unsigned long, item, \
	83	__bitmask_size_in_longs(nr_bits))
	84
	85	#undef __sockaddr
	86	#define __sockaddr(field, len) __dynamic_array(u8, field, len)
	87
	88	#undef __rel_sockaddr
	89	#define __rel_sockaddr(field, len) __rel_dynamic_array(u8, field, len)

include/trace/stages/stage6_defines.h+106−0

@@ -0,0 +1,106 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2
	3	/* Stage 6 definitions for creating trace events */
	4
	5	#undef __entry
	6	#define __entry entry
	7
	8	#undef __field
	9	#define __field(type, item)
	10
	11	#undef __field_struct
	12	#define __field_struct(type, item)
	13
	14	#undef __array
	15	#define __array(type, item, len)
	16
	17	#undef __dynamic_array
	18	#define __dynamic_array(type, item, len) \
	19	__entry->__data_loc_##item = __data_offsets.item;
	20
	21	#undef __string
	22	#define __string(item, src) __dynamic_array(char, item, -1)
	23
	24	#undef __string_len
	25	#define __string_len(item, src, len) __dynamic_array(char, item, -1)
	26
	27	#undef __assign_str
	28	#define __assign_str(dst, src) \
	29	strcpy(__get_str(dst), (src) ? (const char *)(src) : "(null)");
	30
	31	#undef __assign_str_len
	32	#define __assign_str_len(dst, src, len) \
	33	do { \
	34	memcpy(__get_str(dst), (src), (len)); \
	35	__get_str(dst)[len] = '\0'; \
	36	} while(0)
	37
	38	#undef __bitmask
	39	#define __bitmask(item, nr_bits) __dynamic_array(unsigned long, item, -1)
	40
	41	#undef __get_bitmask
	42	#define __get_bitmask(field) (char *)__get_dynamic_array(field)
	43
	44	#undef __assign_bitmask
	45	#define __assign_bitmask(dst, src, nr_bits) \
	46	memcpy(__get_bitmask(dst), (src), __bitmask_size_in_bytes(nr_bits))
	47
	48	#undef __sockaddr
	49	#define __sockaddr(field, len) __dynamic_array(u8, field, len)
	50
	51	#undef __get_sockaddr
	52	#define __get_sockaddr(field) ((struct sockaddr *)__get_dynamic_array(field))
	53
	54	#undef __assign_sockaddr
	55	#define __assign_sockaddr(dest, src, len) \
	56	memcpy(__get_dynamic_array(dest), src, len)
	57
	58	#undef __rel_dynamic_array
	59	#define __rel_dynamic_array(type, item, len) \
	60	__entry->__rel_loc_##item = __data_offsets.item;
	61
	62	#undef __rel_string
	63	#define __rel_string(item, src) __rel_dynamic_array(char, item, -1)
	64
	65	#undef __rel_string_len
	66	#define __rel_string_len(item, src, len) __rel_dynamic_array(char, item, -1)
	67
	68	#undef __assign_rel_str
	69	#define __assign_rel_str(dst, src) \
	70	strcpy(__get_rel_str(dst), (src) ? (const char *)(src) : "(null)");
	71
	72	#undef __assign_rel_str_len
	73	#define __assign_rel_str_len(dst, src, len) \
	74	do { \
	75	memcpy(__get_rel_str(dst), (src), (len)); \
	76	__get_rel_str(dst)[len] = '\0'; \
	77	} while (0)
	78
	79	#undef __rel_bitmask
	80	#define __rel_bitmask(item, nr_bits) __rel_dynamic_array(unsigned long, item, -1)
	81
	82	#undef __get_rel_bitmask
	83	#define __get_rel_bitmask(field) (char *)__get_rel_dynamic_array(field)
	84
	85	#undef __assign_rel_bitmask
	86	#define __assign_rel_bitmask(dst, src, nr_bits) \
	87	memcpy(__get_rel_bitmask(dst), (src), __bitmask_size_in_bytes(nr_bits))
	88
	89	#undef __rel_sockaddr
	90	#define __rel_sockaddr(field, len) __rel_dynamic_array(u8, field, len)
	91
	92	#undef __get_rel_sockaddr
	93	#define __get_rel_sockaddr(field) ((struct sockaddr *)__get_rel_dynamic_array(field))
	94
	95	#undef __assign_rel_sockaddr
	96	#define __assign_rel_sockaddr(dest, src, len) \
	97	memcpy(__get_rel_dynamic_array(dest), src, len)
	98
	99	#undef TP_fast_assign
	100	#define TP_fast_assign(args...) args
	101
	102	#undef __perf_count
	103	#define __perf_count(c) (c)
	104
	105	#undef __perf_task
	106	#define __perf_task(t) (t)

include/trace/stages/stage7_defines.h+36−0

@@ -0,0 +1,36 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2
	3	/* Stage 7 definitions for creating trace events */
	4
	5	#undef __entry
	6	#define __entry REC
	7
	8	#undef __print_flags
	9	#undef __print_symbolic
	10	#undef __print_hex
	11	#undef __print_hex_str
	12	#undef __get_dynamic_array
	13	#undef __get_dynamic_array_len
	14	#undef __get_str
	15	#undef __get_bitmask
	16	#undef __get_sockaddr
	17	#undef __get_rel_dynamic_array
	18	#undef __get_rel_dynamic_array_len
	19	#undef __get_rel_str
	20	#undef __get_rel_bitmask
	21	#undef __get_rel_sockaddr
	22	#undef __print_array
	23	#undef __print_hex_dump
	24
	25	/*
	26	* The below is not executed in the kernel. It is only what is
	27	* displayed in the print format for userspace to parse.
	28	*/
	29	#undef __print_ns_to_secs
	30	#define __print_ns_to_secs(val) (val) / 1000000000UL
	31
	32	#undef __print_ns_without_secs
	33	#define __print_ns_without_secs(val) (val) % 1000000000UL
	34
	35	#undef TP_printk
	36	#define TP_printk(fmt, args...) "\"" fmt "\", " __stringify(args)

include/trace/trace_custom_events.h+221−0

@@ -0,0 +1,221 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	/*
	3	* This is similar to the trace_events.h file, but is to only
	4	* create custom trace events to be attached to existing tracepoints.
	5	* Where as the TRACE_EVENT() macro (from trace_events.h) will create
	6	* both the trace event and the tracepoint it will attach the event to,
	7	* TRACE_CUSTOM_EVENT() is to create only a custom version of an existing
	8	* trace event (created by TRACE_EVENT() or DEFINE_EVENT()), and will
	9	* be placed in the "custom" system.
	10	*/
	11
	12	#include <linux/trace_events.h>
	13
	14	/* All custom events are placed in the custom group */
	15	#undef TRACE_SYSTEM
	16	#define TRACE_SYSTEM custom
	17
	18	#ifndef TRACE_SYSTEM_VAR
	19	#define TRACE_SYSTEM_VAR TRACE_SYSTEM
	20	#endif
	21
	22	/* The init stage creates the system string and enum mappings */
	23
	24	#include "stages/init.h"
	25
	26	#undef TRACE_CUSTOM_EVENT
	27	#define TRACE_CUSTOM_EVENT(name, proto, args, tstruct, assign, print) \
	28	DECLARE_CUSTOM_EVENT_CLASS(name, \
	29	PARAMS(proto), \
	30	PARAMS(args), \
	31	PARAMS(tstruct), \
	32	PARAMS(assign), \
	33	PARAMS(print)); \
	34	DEFINE_CUSTOM_EVENT(name, name, PARAMS(proto), PARAMS(args));
	35
	36	/* Stage 1 creates the structure of the recorded event layout */
	37
	38	#include "stages/stage1_defines.h"
	39
	40	#undef DECLARE_CUSTOM_EVENT_CLASS
	41	#define DECLARE_CUSTOM_EVENT_CLASS(name, proto, args, tstruct, assign, print) \
	42	struct trace_custom_event_raw_##name { \
	43	struct trace_entry ent; \
	44	tstruct \
	45	char __data[]; \
	46	}; \
	47	\
	48	static struct trace_event_class custom_event_class_##name;
	49
	50	#undef DEFINE_CUSTOM_EVENT
	51	#define DEFINE_CUSTOM_EVENT(template, name, proto, args) \
	52	static struct trace_event_call __used \
	53	__attribute__((__aligned__(4))) custom_event_##name
	54
	55	#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
	56
	57	/* Stage 2 creates the custom class */
	58
	59	#include "stages/stage2_defines.h"
	60
	61	#undef DECLARE_CUSTOM_EVENT_CLASS
	62	#define DECLARE_CUSTOM_EVENT_CLASS(call, proto, args, tstruct, assign, print) \
	63	struct trace_custom_event_data_offsets_##call { \
	64	tstruct; \
	65	};
	66
	67	#undef DEFINE_CUSTOM_EVENT
	68	#define DEFINE_CUSTOM_EVENT(template, name, proto, args)
	69
	70	#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
	71
	72	/* Stage 3 create the way to print the custom event */
	73
	74	#include "stages/stage3_defines.h"
	75
	76	#undef DECLARE_CUSTOM_EVENT_CLASS
	77	#define DECLARE_CUSTOM_EVENT_CLASS(call, proto, args, tstruct, assign, print) \
	78	static notrace enum print_line_t \
	79	trace_custom_raw_output_##call(struct trace_iterator *iter, int flags, \
	80	struct trace_event *trace_event) \
	81	{ \
	82	struct trace_seq *s = &iter->seq; \
	83	struct trace_seq __maybe_unused *p = &iter->tmp_seq; \
	84	struct trace_custom_event_raw_##call *field; \
	85	int ret; \
	86	\
	87	field = (typeof(field))iter->ent; \
	88	\
	89	ret = trace_raw_output_prep(iter, trace_event); \
	90	if (ret != TRACE_TYPE_HANDLED) \
	91	return ret; \
	92	\
	93	trace_event_printf(iter, print); \
	94	\
	95	return trace_handle_return(s); \
	96	} \
	97	static struct trace_event_functions trace_custom_event_type_funcs_##call = { \
	98	.trace = trace_custom_raw_output_##call, \
	99	};
	100
	101	#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
	102
	103	/* Stage 4 creates the offset layout for the fields */
	104
	105	#include "stages/stage4_defines.h"
	106
	107	#undef DECLARE_CUSTOM_EVENT_CLASS
	108	#define DECLARE_CUSTOM_EVENT_CLASS(call, proto, args, tstruct, func, print) \
	109	static struct trace_event_fields trace_custom_event_fields_##call[] = { \
	110	tstruct \
	111	{} };
	112
	113	#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
	114
	115	/* Stage 5 creates the helper function for dynamic fields */
	116
	117	#include "stages/stage5_defines.h"
	118
	119	#undef DECLARE_CUSTOM_EVENT_CLASS
	120	#define DECLARE_CUSTOM_EVENT_CLASS(call, proto, args, tstruct, assign, print) \
	121	static inline notrace int trace_custom_event_get_offsets_##call( \
	122	struct trace_custom_event_data_offsets_##call *__data_offsets, proto) \
	123	{ \
	124	int __data_size = 0; \
	125	int __maybe_unused __item_length; \
	126	struct trace_custom_event_raw_##call __maybe_unused *entry; \
	127	\
	128	tstruct; \
	129	\
	130	return __data_size; \
	131	}
	132
	133	#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
	134
	135	/* Stage 6 creates the probe function that records the event */
	136
	137	#include "stages/stage6_defines.h"
	138
	139	#undef DECLARE_CUSTOM_EVENT_CLASS
	140	#define DECLARE_CUSTOM_EVENT_CLASS(call, proto, args, tstruct, assign, print) \
	141	\
	142	static notrace void \
	143	trace_custom_event_raw_event_##call(void *__data, proto) \
	144	{ \
	145	struct trace_event_file *trace_file = __data; \
	146	struct trace_custom_event_data_offsets_##call __maybe_unused __data_offsets; \
	147	struct trace_event_buffer fbuffer; \
	148	struct trace_custom_event_raw_##call *entry; \
	149	int __data_size; \
	150	\
	151	if (trace_trigger_soft_disabled(trace_file)) \
	152	return; \
	153	\
	154	__data_size = trace_custom_event_get_offsets_##call(&__data_offsets, args); \
	155	\
	156	entry = trace_event_buffer_reserve(&fbuffer, trace_file, \
	157	sizeof(*entry) + __data_size); \
	158	\
	159	if (!entry) \
	160	return; \
	161	\
	162	tstruct \
	163	\
	164	{ assign; } \
	165	\
	166	trace_event_buffer_commit(&fbuffer); \
	167	}
	168	/*
	169	* The ftrace_test_custom_probe is compiled out, it is only here as a build time check
	170	* to make sure that if the tracepoint handling changes, the ftrace probe will
	171	* fail to compile unless it too is updated.
	172	*/
	173
	174	#undef DEFINE_CUSTOM_EVENT
	175	#define DEFINE_CUSTOM_EVENT(template, call, proto, args) \
	176	static inline void ftrace_test_custom_probe_##call(void) \
	177	{ \
	178	check_trace_callback_type_##call(trace_custom_event_raw_event_##template); \
	179	}
	180
	181	#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
	182
	183	/* Stage 7 creates the actual class and event structure for the custom event */
	184
	185	#include "stages/stage7_defines.h"
	186
	187	#undef DECLARE_CUSTOM_EVENT_CLASS
	188	#define DECLARE_CUSTOM_EVENT_CLASS(call, proto, args, tstruct, assign, print) \
	189	static char custom_print_fmt_##call[] = print; \
	190	static struct trace_event_class __used __refdata custom_event_class_##call = { \
	191	.system = TRACE_SYSTEM_STRING, \
	192	.fields_array = trace_custom_event_fields_##call, \
	193	.fields = LIST_HEAD_INIT(custom_event_class_##call.fields),\
	194	.raw_init = trace_event_raw_init, \
	195	.probe = trace_custom_event_raw_event_##call, \
	196	.reg = trace_event_reg, \
	197	};
	198
	199	#undef DEFINE_CUSTOM_EVENT
	200	#define DEFINE_CUSTOM_EVENT(template, call, proto, args) \
	201	\
	202	static struct trace_event_call __used custom_event_##call = { \
	203	.name = #call, \
	204	.class = &custom_event_class_##template, \
	205	.event.funcs = &trace_custom_event_type_funcs_##template, \
	206	.print_fmt = custom_print_fmt_##template, \
	207	.flags = TRACE_EVENT_FL_CUSTOM, \
	208	}; \
	209	static inline int trace_custom_event_##call##_update(struct tracepoint *tp) \
	210	{ \
	211	if (tp->name && strcmp(tp->name, #call) == 0) { \
	212	custom_event_##call.tp = tp; \
	213	custom_event_##call.flags = TRACE_EVENT_FL_TRACEPOINT; \
	214	return 1; \
	215	} \
	216	return 0; \
	217	} \
	218	static struct trace_event_call __used \
	219	__section("_ftrace_events") *__custom_event_##call = &custom_event_##call
	220
	221	#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)

include/trace/trace_events.h+8−542

@@ -24,42 +24,7 @@ · 23 unmodified lines
24	#define TRACE_SYSTEM_VAR TRACE_SYSTEM	24	#define TRACE_SYSTEM_VAR TRACE_SYSTEM
25	#endif	25	#endif
26		26
27	#define __app__(x, y) str__##x##y	27	#include "stages/init.h"
28	#define __app(x, y) __app__(x, y)
29
30	#define TRACE_SYSTEM_STRING __app(TRACE_SYSTEM_VAR,__trace_system_name)
31
32	#define TRACE_MAKE_SYSTEM_STR() \
33	static const char TRACE_SYSTEM_STRING[] = \
34	__stringify(TRACE_SYSTEM)
35
36	TRACE_MAKE_SYSTEM_STR();
37
38	#undef TRACE_DEFINE_ENUM
39	#define TRACE_DEFINE_ENUM(a) \
40	static struct trace_eval_map __used __initdata \
41	__##TRACE_SYSTEM##_##a = \
42	{ \
43	.system = TRACE_SYSTEM_STRING, \
44	.eval_string = #a, \
45	.eval_value = a \
46	}; \
47	static struct trace_eval_map __used \
48	__section("_ftrace_eval_map") \
49	*TRACE_SYSTEM##_##a = &__##TRACE_SYSTEM##_##a
50
51	#undef TRACE_DEFINE_SIZEOF
52	#define TRACE_DEFINE_SIZEOF(a) \
53	static struct trace_eval_map __used __initdata \
54	__##TRACE_SYSTEM##_##a = \
55	{ \
56	.system = TRACE_SYSTEM_STRING, \
57	.eval_string = "sizeof(" #a ")", \
58	.eval_value = sizeof(a) \
59	}; \
60	static struct trace_eval_map __used \
61	__section("_ftrace_eval_map") \
62	*TRACE_SYSTEM##_##a = &__##TRACE_SYSTEM##_##a
63		28
64	/*	29	/*
65	* DECLARE_EVENT_CLASS can be used to add a generic function	30	* DECLARE_EVENT_CLASS can be used to add a generic function
@@ -80,54 +45,7 @@ TRACE_MAKE_SYSTEM_STR(); · 14 unmodified lines
80	PARAMS(print)); \	45	PARAMS(print)); \
81	DEFINE_EVENT(name, name, PARAMS(proto), PARAMS(args));	46	DEFINE_EVENT(name, name, PARAMS(proto), PARAMS(args));
82		47
83		48	#include "stages/stage1_defines.h"
84	#undef __field
85	#define __field(type, item) type item;
86
87	#undef __field_ext
88	#define __field_ext(type, item, filter_type) type item;
89
90	#undef __field_struct
91	#define __field_struct(type, item) type item;
92
93	#undef __field_struct_ext
94	#define __field_struct_ext(type, item, filter_type) type item;
95
96	#undef __array
97	#define __array(type, item, len) type item[len];
98
99	#undef __dynamic_array
100	#define __dynamic_array(type, item, len) u32 __data_loc_##item;
101
102	#undef __string
103	#define __string(item, src) __dynamic_array(char, item, -1)
104
105	#undef __string_len
106	#define __string_len(item, src, len) __dynamic_array(char, item, -1)
107
108	#undef __bitmask
109	#define __bitmask(item, nr_bits) __dynamic_array(char, item, -1)
110
111	#undef __sockaddr
112	#define __sockaddr(field, len) __dynamic_array(u8, field, len)
113
114	#undef __rel_dynamic_array
115	#define __rel_dynamic_array(type, item, len) u32 __rel_loc_##item;
116
117	#undef __rel_string
118	#define __rel_string(item, src) __rel_dynamic_array(char, item, -1)
119
120	#undef __rel_string_len
121	#define __rel_string_len(item, src, len) __rel_dynamic_array(char, item, -1)
122
123	#undef __rel_bitmask
124	#define __rel_bitmask(item, nr_bits) __rel_dynamic_array(char, item, -1)
125
126	#undef __rel_sockaddr
127	#define __rel_sockaddr(field, len) __rel_dynamic_array(u8, field, len)
128
129	#undef TP_STRUCT__entry
130	#define TP_STRUCT__entry(args...) args
131		49
132	#undef DECLARE_EVENT_CLASS	50	#undef DECLARE_EVENT_CLASS
133	#define DECLARE_EVENT_CLASS(name, proto, args, tstruct, assign, print) \	51	#define DECLARE_EVENT_CLASS(name, proto, args, tstruct, assign, print) \
@@ -191,56 +109,7 @@ TRACE_MAKE_SYSTEM_STR(); · 57 unmodified lines
191	* The size of an array is also encoded, in the higher 16 bits of <item>.	109	* The size of an array is also encoded, in the higher 16 bits of <item>.
192	*/	110	*/
193		111
194	#undef TRACE_DEFINE_ENUM	112	#include "stages/stage2_defines.h"
195	#define TRACE_DEFINE_ENUM(a)
196
197	#undef TRACE_DEFINE_SIZEOF
198	#define TRACE_DEFINE_SIZEOF(a)
199
200	#undef __field
201	#define __field(type, item)
202
203	#undef __field_ext
204	#define __field_ext(type, item, filter_type)
205
206	#undef __field_struct
207	#define __field_struct(type, item)
208
209	#undef __field_struct_ext
210	#define __field_struct_ext(type, item, filter_type)
211
212	#undef __array
213	#define __array(type, item, len)
214
215	#undef __dynamic_array
216	#define __dynamic_array(type, item, len) u32 item;
217
218	#undef __string
219	#define __string(item, src) __dynamic_array(char, item, -1)
220
221	#undef __string_len
222	#define __string_len(item, src, len) __dynamic_array(char, item, -1)
223
224	#undef __bitmask
225	#define __bitmask(item, nr_bits) __dynamic_array(unsigned long, item, -1)
226
227	#undef __sockaddr
228	#define __sockaddr(field, len) __dynamic_array(u8, field, len)
229
230	#undef __rel_dynamic_array
231	#define __rel_dynamic_array(type, item, len) u32 item;
232
233	#undef __rel_string
234	#define __rel_string(item, src) __rel_dynamic_array(char, item, -1)
235
236	#undef __rel_string_len
237	#define __rel_string_len(item, src, len) __rel_dynamic_array(char, item, -1)
238
239	#undef __rel_bitmask
240	#define __rel_bitmask(item, nr_bits) __rel_dynamic_array(unsigned long, item, -1)
241
242	#undef __rel_sockaddr
243	#define __rel_sockaddr(field, len) __rel_dynamic_array(u8, field, len)
244		113
245	#undef DECLARE_EVENT_CLASS	114	#undef DECLARE_EVENT_CLASS
246	#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \	115	#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \
@@ -312,137 +181,7 @@ TRACE_MAKE_SYSTEM_STR(); · 65 unmodified lines
312	* in binary.	181	* in binary.
313	*/	182	*/
314		183
315	#undef __entry	184	#include "stages/stage3_defines.h"
316	#define __entry field
317
318	#undef TP_printk
319	#define TP_printk(fmt, args...) fmt "\n", args
320
321	#undef __get_dynamic_array
322	#define __get_dynamic_array(field) \
323	((void *)__entry + (__entry->__data_loc_##field & 0xffff))
324
325	#undef __get_dynamic_array_len
326	#define __get_dynamic_array_len(field) \
327	((__entry->__data_loc_##field >> 16) & 0xffff)
328
329	#undef __get_str
330	#define __get_str(field) ((char *)__get_dynamic_array(field))
331
332	#undef __get_rel_dynamic_array
333	#define __get_rel_dynamic_array(field) \
334	((void *)__entry + \
335	offsetof(typeof(*__entry), __rel_loc_##field) + \
336	sizeof(__entry->__rel_loc_##field) + \
337	(__entry->__rel_loc_##field & 0xffff))
338
339	#undef __get_rel_dynamic_array_len
340	#define __get_rel_dynamic_array_len(field) \
341	((__entry->__rel_loc_##field >> 16) & 0xffff)
342
343	#undef __get_rel_str
344	#define __get_rel_str(field) ((char *)__get_rel_dynamic_array(field))
345
346	#undef __get_bitmask
347	#define __get_bitmask(field) \
348	({ \
349	void *__bitmask = __get_dynamic_array(field); \
350	unsigned int __bitmask_size; \
351	__bitmask_size = __get_dynamic_array_len(field); \
352	trace_print_bitmask_seq(p, __bitmask, __bitmask_size); \
353	})
354
355	#undef __get_rel_bitmask
356	#define __get_rel_bitmask(field) \
357	({ \
358	void *__bitmask = __get_rel_dynamic_array(field); \
359	unsigned int __bitmask_size; \
360	__bitmask_size = __get_rel_dynamic_array_len(field); \
361	trace_print_bitmask_seq(p, __bitmask, __bitmask_size); \
362	})
363
364	#undef __get_sockaddr
365	#define __get_sockaddr(field) ((struct sockaddr *)__get_dynamic_array(field))
366
367	#undef __get_rel_sockaddr
368	#define __get_rel_sockaddr(field) ((struct sockaddr *)__get_rel_dynamic_array(field))
369
370	#undef __print_flags
371	#define __print_flags(flag, delim, flag_array...) \
372	({ \
373	static const struct trace_print_flags __flags[] = \
374	{ flag_array, { -1, NULL }}; \
375	trace_print_flags_seq(p, delim, flag, __flags); \
376	})
377
378	#undef __print_symbolic
379	#define __print_symbolic(value, symbol_array...) \
380	({ \
381	static const struct trace_print_flags symbols[] = \
382	{ symbol_array, { -1, NULL }}; \
383	trace_print_symbols_seq(p, value, symbols); \
384	})
385
386	#undef __print_flags_u64
387	#undef __print_symbolic_u64
388	#if BITS_PER_LONG == 32
389	#define __print_flags_u64(flag, delim, flag_array...) \
390	({ \
391	static const struct trace_print_flags_u64 __flags[] = \
392	{ flag_array, { -1, NULL } }; \
393	trace_print_flags_seq_u64(p, delim, flag, __flags); \
394	})
395
396	#define __print_symbolic_u64(value, symbol_array...) \
397	({ \
398	static const struct trace_print_flags_u64 symbols[] = \
399	{ symbol_array, { -1, NULL } }; \
400	trace_print_symbols_seq_u64(p, value, symbols); \
401	})
402	#else
403	#define __print_flags_u64(flag, delim, flag_array...) \
404	__print_flags(flag, delim, flag_array)
405
406	#define __print_symbolic_u64(value, symbol_array...) \
407	__print_symbolic(value, symbol_array)
408	#endif
409
410	#undef __print_hex
411	#define __print_hex(buf, buf_len) \
412	trace_print_hex_seq(p, buf, buf_len, false)
413
414	#undef __print_hex_str
415	#define __print_hex_str(buf, buf_len) \
416	trace_print_hex_seq(p, buf, buf_len, true)
417
418	#undef __print_array
419	#define __print_array(array, count, el_size) \
420	({ \
421	BUILD_BUG_ON(el_size != 1 && el_size != 2 && \
422	el_size != 4 && el_size != 8); \
423	trace_print_array_seq(p, array, count, el_size); \
424	})
425
426	#undef __print_hex_dump
427	#define __print_hex_dump(prefix_str, prefix_type, \
428	rowsize, groupsize, buf, len, ascii) \
429	trace_print_hex_dump_seq(p, prefix_str, prefix_type, \
430	rowsize, groupsize, buf, len, ascii)
431
432	#undef __print_ns_to_secs
433	#define __print_ns_to_secs(value) \
434	({ \
435	u64 ____val = (u64)(value); \
436	do_div(____val, NSEC_PER_SEC); \
437	____val; \
438	})
439
440	#undef __print_ns_without_secs
441	#define __print_ns_without_secs(value) \
442	({ \
443	u64 ____val = (u64)(value); \
444	(u32) do_div(____val, NSEC_PER_SEC); \
445	})
446		185
447	#undef DECLARE_EVENT_CLASS	186	#undef DECLARE_EVENT_CLASS
448	#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \	187	#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \
@@ -497,65 +236,7 @@ static struct trace_event_functions trace_event_type_funcs_##call = { \ · 48 unmodified lines
497		236
498	#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)	237	#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
499		238
500	#undef __field_ext	239	#include "stages/stage4_defines.h"
501	#define __field_ext(_type, _item, _filter_type) { \
502	.type = #_type, .name = #_item, \
503	.size = sizeof(_type), .align = __alignof__(_type), \
504	.is_signed = is_signed_type(_type), .filter_type = _filter_type },
505
506	#undef __field_struct_ext
507	#define __field_struct_ext(_type, _item, _filter_type) { \
508	.type = #_type, .name = #_item, \
509	.size = sizeof(_type), .align = __alignof__(_type), \
510	0, .filter_type = _filter_type },
511
512	#undef __field
513	#define __field(type, item) __field_ext(type, item, FILTER_OTHER)
514
515	#undef __field_struct
516	#define __field_struct(type, item) __field_struct_ext(type, item, FILTER_OTHER)
517
518	#undef __array
519	#define __array(_type, _item, _len) { \
520	.type = #_type"["__stringify(_len)"]", .name = #_item, \
521	.size = sizeof(_type[_len]), .align = __alignof__(_type), \
522	.is_signed = is_signed_type(_type), .filter_type = FILTER_OTHER },
523
524	#undef __dynamic_array
525	#define __dynamic_array(_type, _item, _len) { \
526	.type = "__data_loc " #_type "[]", .name = #_item, \
527	.size = 4, .align = 4, \
528	.is_signed = is_signed_type(_type), .filter_type = FILTER_OTHER },
529
530	#undef __string
531	#define __string(item, src) __dynamic_array(char, item, -1)
532
533	#undef __string_len
534	#define __string_len(item, src, len) __dynamic_array(char, item, -1)
535
536	#undef __bitmask
537	#define __bitmask(item, nr_bits) __dynamic_array(unsigned long, item, -1)
538
539	#undef __sockaddr
540	#define __sockaddr(field, len) __dynamic_array(u8, field, len)
541
542	#undef __rel_dynamic_array
543	#define __rel_dynamic_array(_type, _item, _len) { \
544	.type = "__rel_loc " #_type "[]", .name = #_item, \
545	.size = 4, .align = 4, \
546	.is_signed = is_signed_type(_type), .filter_type = FILTER_OTHER },
547
548	#undef __rel_string
549	#define __rel_string(item, src) __rel_dynamic_array(char, item, -1)
550
551	#undef __rel_string_len
552	#define __rel_string_len(item, src, len) __rel_dynamic_array(char, item, -1)
553
554	#undef __rel_bitmask
555	#define __rel_bitmask(item, nr_bits) __rel_dynamic_array(unsigned long, item, -1)
556
557	#undef __rel_sockaddr
558	#define __rel_sockaddr(field, len) __rel_dynamic_array(u8, field, len)
559		240
560	#undef DECLARE_EVENT_CLASS	241	#undef DECLARE_EVENT_CLASS
561	#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, func, print) \	242	#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, func, print) \
@@ -568,91 +249,7 @@ static struct trace_event_fields trace_event_fields_##call[] = { \ · 6 unmodified lines
568		249
569	#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)	250	#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
570		251
571	/*	252	#include "stages/stage5_defines.h"
572	* remember the offset of each array from the beginning of the event.
573	*/
574
575	#undef __entry
576	#define __entry entry
577
578	#undef __field
579	#define __field(type, item)
580
581	#undef __field_ext
582	#define __field_ext(type, item, filter_type)
583
584	#undef __field_struct
585	#define __field_struct(type, item)
586
587	#undef __field_struct_ext
588	#define __field_struct_ext(type, item, filter_type)
589
590	#undef __array
591	#define __array(type, item, len)
592
593	#undef __dynamic_array
594	#define __dynamic_array(type, item, len) \
595	__item_length = (len) * sizeof(type); \
596	__data_offsets->item = __data_size + \
597	offsetof(typeof(*entry), __data); \
598	__data_offsets->item \|= __item_length << 16; \
599	__data_size += __item_length;
600
601	#undef __string
602	#define __string(item, src) __dynamic_array(char, item, \
603	strlen((src) ? (const char *)(src) : "(null)") + 1)
604
605	#undef __string_len
606	#define __string_len(item, src, len) __dynamic_array(char, item, (len) + 1)
607
608	#undef __rel_dynamic_array
609	#define __rel_dynamic_array(type, item, len) \
610	__item_length = (len) * sizeof(type); \
611	__data_offsets->item = __data_size + \
612	offsetof(typeof(*entry), __data) - \
613	offsetof(typeof(*entry), __rel_loc_##item) - \
614	sizeof(u32); \
615	__data_offsets->item \|= __item_length << 16; \
616	__data_size += __item_length;
617
618	#undef __rel_string
619	#define __rel_string(item, src) __rel_dynamic_array(char, item, \
620	strlen((src) ? (const char *)(src) : "(null)") + 1)
621
622	#undef __rel_string_len
623	#define __rel_string_len(item, src, len) __rel_dynamic_array(char, item, (len) + 1)
624	/*
625	* __bitmask_size_in_bytes_raw is the number of bytes needed to hold
626	* num_possible_cpus().
627	*/
628	#define __bitmask_size_in_bytes_raw(nr_bits) \
629	(((nr_bits) + 7) / 8)
630
631	#define __bitmask_size_in_longs(nr_bits) \
632	((__bitmask_size_in_bytes_raw(nr_bits) + \
633	((BITS_PER_LONG / 8) - 1)) / (BITS_PER_LONG / 8))
634
635	/*
636	* __bitmask_size_in_bytes is the number of bytes needed to hold
637	* num_possible_cpus() padded out to the nearest long. This is what
638	* is saved in the buffer, just to be consistent.
639	*/
640	#define __bitmask_size_in_bytes(nr_bits) \
641	(__bitmask_size_in_longs(nr_bits) * (BITS_PER_LONG / 8))
642
643	#undef __bitmask
644	#define __bitmask(item, nr_bits) __dynamic_array(unsigned long, item, \
645	__bitmask_size_in_longs(nr_bits))
646
647	#undef __rel_bitmask
648	#define __rel_bitmask(item, nr_bits) __rel_dynamic_array(unsigned long, item, \
649	__bitmask_size_in_longs(nr_bits))
650
651	#undef __sockaddr
652	#define __sockaddr(field, len) __dynamic_array(u8, field, len)
653
654	#undef __rel_sockaddr
655	#define __rel_sockaddr(field, len) __rel_dynamic_array(u8, field, len)
656		253
657	#undef DECLARE_EVENT_CLASS	254	#undef DECLARE_EVENT_CLASS
658	#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \	255	#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \
@@ -775,107 +372,7 @@ static inline notrace int trace_event_get_offsets_##call( \ · 116 unmodified lines
775	#define _TRACE_PERF_INIT(call)	372	#define _TRACE_PERF_INIT(call)
776	#endif /* CONFIG_PERF_EVENTS */	373	#endif /* CONFIG_PERF_EVENTS */
777		374
778	#undef __entry	375	#include "stages/stage6_defines.h"
779	#define __entry entry
780
781	#undef __field
782	#define __field(type, item)
783
784	#undef __field_struct
785	#define __field_struct(type, item)
786
787	#undef __array
788	#define __array(type, item, len)
789
790	#undef __dynamic_array
791	#define __dynamic_array(type, item, len) \
792	__entry->__data_loc_##item = __data_offsets.item;
793
794	#undef __string
795	#define __string(item, src) __dynamic_array(char, item, -1)
796
797	#undef __string_len
798	#define __string_len(item, src, len) __dynamic_array(char, item, -1)
799
800	#undef __assign_str
801	#define __assign_str(dst, src) \
802	strcpy(__get_str(dst), (src) ? (const char *)(src) : "(null)");
803
804	#undef __assign_str_len
805	#define __assign_str_len(dst, src, len) \
806	do { \
807	memcpy(__get_str(dst), (src), (len)); \
808	__get_str(dst)[len] = '\0'; \
809	} while(0)
810
811	#undef __bitmask
812	#define __bitmask(item, nr_bits) __dynamic_array(unsigned long, item, -1)
813
814	#undef __get_bitmask
815	#define __get_bitmask(field) (char *)__get_dynamic_array(field)
816
817	#undef __assign_bitmask
818	#define __assign_bitmask(dst, src, nr_bits) \
819	memcpy(__get_bitmask(dst), (src), __bitmask_size_in_bytes(nr_bits))
820
821	#undef __sockaddr
822	#define __sockaddr(field, len) __dynamic_array(u8, field, len)
823
824	#undef __get_sockaddr
825	#define __get_sockaddr(field) ((struct sockaddr *)__get_dynamic_array(field))
826
827	#define __assign_sockaddr(dest, src, len) \
828	memcpy(__get_dynamic_array(dest), src, len)
829
830	#undef __rel_dynamic_array
831	#define __rel_dynamic_array(type, item, len) \
832	__entry->__rel_loc_##item = __data_offsets.item;
833
834	#undef __rel_string
835	#define __rel_string(item, src) __rel_dynamic_array(char, item, -1)
836
837	#undef __rel_string_len
838	#define __rel_string_len(item, src, len) __rel_dynamic_array(char, item, -1)
839
840	#undef __assign_rel_str
841	#define __assign_rel_str(dst, src) \
842	strcpy(__get_rel_str(dst), (src) ? (const char *)(src) : "(null)");
843
844	#undef __assign_rel_str_len
845	#define __assign_rel_str_len(dst, src, len) \
846	do { \
847	memcpy(__get_rel_str(dst), (src), (len)); \
848	__get_rel_str(dst)[len] = '\0'; \
849	} while (0)
850
851	#undef __rel_bitmask
852	#define __rel_bitmask(item, nr_bits) __rel_dynamic_array(unsigned long, item, -1)
853
854	#undef __get_rel_bitmask
855	#define __get_rel_bitmask(field) (char *)__get_rel_dynamic_array(field)
856
857	#undef __assign_rel_bitmask
858	#define __assign_rel_bitmask(dst, src, nr_bits) \
859	memcpy(__get_rel_bitmask(dst), (src), __bitmask_size_in_bytes(nr_bits))
860
861	#undef __rel_sockaddr
862	#define __rel_sockaddr(field, len) __rel_dynamic_array(u8, field, len)
863
864	#undef __get_rel_sockaddr
865	#define __get_rel_sockaddr(field) ((struct sockaddr *)__get_rel_dynamic_array(field))
866
867	#define __assign_rel_sockaddr(dest, src, len) \
868	memcpy(__get_rel_dynamic_array(dest), src, len)
869
870
871	#undef TP_fast_assign
872	#define TP_fast_assign(args...) args
873
874	#undef __perf_count
875	#define __perf_count(c) (c)
876
877	#undef __perf_task
878	#define __perf_task(t) (t)
879		376
880	#undef DECLARE_EVENT_CLASS	377	#undef DECLARE_EVENT_CLASS
881	#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \	378	#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \
@@ -921,38 +418,7 @@ static inline void ftrace_test_probe_##call(void) \ · 39 unmodified lines
921		418
922	#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)	419	#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
923		420
924	#undef __entry	421	#include "stages/stage7_defines.h"
925	#define __entry REC
926
927	#undef __print_flags
928	#undef __print_symbolic
929	#undef __print_hex
930	#undef __print_hex_str
931	#undef __get_dynamic_array
932	#undef __get_dynamic_array_len
933	#undef __get_str
934	#undef __get_bitmask
935	#undef __get_sockaddr
936	#undef __get_rel_dynamic_array
937	#undef __get_rel_dynamic_array_len
938	#undef __get_rel_str
939	#undef __get_rel_bitmask
940	#undef __get_rel_sockaddr
941	#undef __print_array
942	#undef __print_hex_dump
943
944	/*
945	* The below is not executed in the kernel. It is only what is
946	* displayed in the print format for userspace to parse.
947	*/
948	#undef __print_ns_to_secs
949	#define __print_ns_to_secs(val) (val) / 1000000000UL
950
951	#undef __print_ns_without_secs
952	#define __print_ns_without_secs(val) (val) % 1000000000UL
953
954	#undef TP_printk
955	#define TP_printk(fmt, args...) "\"" fmt "\", " __stringify(args)
956		422
957	#undef DECLARE_EVENT_CLASS	423	#undef DECLARE_EVENT_CLASS
958	#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \	424	#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \

include/uapi/linux/user_events.h+116−0

@@ -0,0 +1,116 @@
	1	/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
	2	/*
	3	* Copyright (c) 2021, Microsoft Corporation.
	4	*
	5	* Authors:
	6	* Beau Belgrave <beaub@linux.microsoft.com>
	7	*/
	8	#ifndef _UAPI_LINUX_USER_EVENTS_H
	9	#define _UAPI_LINUX_USER_EVENTS_H
	10
	11	#include <linux/types.h>
	12	#include <linux/ioctl.h>
	13
	14	#ifdef __KERNEL__
	15	#include <linux/uio.h>
	16	#else
	17	#include <sys/uio.h>
	18	#endif
	19
	20	#define USER_EVENTS_SYSTEM "user_events"
	21	#define USER_EVENTS_PREFIX "u:"
	22
	23	/* Bits 0-6 are for known probe types, Bit 7 is for unknown probes */
	24	#define EVENT_BIT_FTRACE 0
	25	#define EVENT_BIT_PERF 1
	26	#define EVENT_BIT_OTHER 7
	27
	28	#define EVENT_STATUS_FTRACE (1 << EVENT_BIT_FTRACE)
	29	#define EVENT_STATUS_PERF (1 << EVENT_BIT_PERF)
	30	#define EVENT_STATUS_OTHER (1 << EVENT_BIT_OTHER)
	31
	32	/* Create dynamic location entry within a 32-bit value */
	33	#define DYN_LOC(offset, size) ((size) << 16 \| (offset))
	34
	35	/* Use raw iterator for attached BPF program(s), no affect on ftrace/perf */
	36	#define FLAG_BPF_ITER (1 << 0)
	37
	38	/*
	39	* Describes an event registration and stores the results of the registration.
	40	* This structure is passed to the DIAG_IOCSREG ioctl, callers at a minimum
	41	* must set the size and name_args before invocation.
	42	*/
	43	struct user_reg {
	44
	45	/* Input: Size of the user_reg structure being used */
	46	__u32 size;
	47
	48	/* Input: Pointer to string with event name, description and flags */
	49	__u64 name_args;
	50
	51	/* Output: Byte index of the event within the status page */
	52	__u32 status_index;
	53
	54	/* Output: Index of the event to use when writing data */
	55	__u32 write_index;
	56	};
	57
	58	#define DIAG_IOC_MAGIC '*'
	59
	60	/* Requests to register a user_event */
	61	#define DIAG_IOCSREG _IOWR(DIAG_IOC_MAGIC, 0, struct user_reg*)
	62
	63	/* Requests to delete a user_event */
	64	#define DIAG_IOCSDEL _IOW(DIAG_IOC_MAGIC, 1, char*)
	65
	66	/* Data type that was passed to the BPF program */
	67	enum {
	68	/* Data resides in kernel space */
	69	USER_BPF_DATA_KERNEL,
	70
	71	/* Data resides in user space */
	72	USER_BPF_DATA_USER,
	73
	74	/* Data is a pointer to a user_bpf_iter structure */
	75	USER_BPF_DATA_ITER,
	76	};
	77
	78	/*
	79	* Describes an iovec iterator that BPF programs can use to access data for
	80	* a given user_event write() / writev() call.
	81	*/
	82	struct user_bpf_iter {
	83
	84	/* Offset of the data within the first iovec */
	85	__u32 iov_offset;
	86
	87	/* Number of iovec structures */
	88	__u32 nr_segs;
	89
	90	/* Pointer to iovec structures */
	91	const struct iovec *iov;
	92	};
	93
	94	/* Context that BPF programs receive when attached to a user_event */
	95	struct user_bpf_context {
	96
	97	/* Data type being passed (see union below) */
	98	__u32 data_type;
	99
	100	/* Length of the data */
	101	__u32 data_len;
	102
	103	/* Pointer to data, varies by data type */
	104	union {
	105	/* Kernel data (data_type == USER_BPF_DATA_KERNEL) */
	106	void *kdata;
	107
	108	/* User data (data_type == USER_BPF_DATA_USER) */
	109	void *udata;
	110
	111	/* Direct iovec (data_type == USER_BPF_DATA_ITER) */
	112	struct user_bpf_iter *iter;
	113	};
	114	};
	115
	116	#endif /* _UAPI_LINUX_USER_EVENTS_H */

kernel/trace/Kconfig+14−0

@@ -737,6 +737,20 @@ config SYNTH_EVENTS · 736 unmodified lines
737		737
738	If in doubt, say N.	738	If in doubt, say N.
739		739
		740	config USER_EVENTS
		741	bool "User trace events"
		742	select TRACING
		743	select DYNAMIC_EVENTS
		744	help
		745	User trace events are user-defined trace events that
		746	can be used like an existing kernel trace event. User trace
		747	events are generated by writing to a tracefs file. User
		748	processes can determine if their tracing events should be
		749	generated by memory mapping a tracefs file and checking for
		750	an associated byte being non-zero.
		751
		752	If in doubt, say N.
		753
740	config HIST_TRIGGERS	754	config HIST_TRIGGERS
741	bool "Histogram triggers"	755	bool "Histogram triggers"
742	depends on ARCH_HAVE_NMI_SAFE_CMPXCHG	756	depends on ARCH_HAVE_NMI_SAFE_CMPXCHG

kernel/trace/Makefile+1−0

@@ -82,6 +82,7 @@ obj-$(CONFIG_PROBE_EVENTS) += trace_eprobe.o · 81 unmodified lines
82	obj-$(CONFIG_TRACE_EVENT_INJECT) += trace_events_inject.o	82	obj-$(CONFIG_TRACE_EVENT_INJECT) += trace_events_inject.o
83	obj-$(CONFIG_SYNTH_EVENTS) += trace_events_synth.o	83	obj-$(CONFIG_SYNTH_EVENTS) += trace_events_synth.o
84	obj-$(CONFIG_HIST_TRIGGERS) += trace_events_hist.o	84	obj-$(CONFIG_HIST_TRIGGERS) += trace_events_hist.o
		85	obj-$(CONFIG_USER_EVENTS) += trace_events_user.o
85	obj-$(CONFIG_BPF_EVENTS) += bpf_trace.o	86	obj-$(CONFIG_BPF_EVENTS) += bpf_trace.o
86	obj-$(CONFIG_KPROBE_EVENTS) += trace_kprobe.o	87	obj-$(CONFIG_KPROBE_EVENTS) += trace_kprobe.o
87	obj-$(CONFIG_TRACEPOINTS) += error_report-traces.o	88	obj-$(CONFIG_TRACEPOINTS) += error_report-traces.o

kernel/trace/ftrace.c+2−0

@@ -7096,6 +7096,8 @@ void __init ftrace_free_init_mem(void) · 7095 unmodified lines
7096	void start = (void )(&__init_begin);	7096	void start = (void )(&__init_begin);
7097	void end = (void )(&__init_end);	7097	void end = (void )(&__init_end);
7098		7098
		7099	ftrace_boot_snapshot();
		7100
7099	ftrace_free_mem(NULL, start, end);	7101	ftrace_free_mem(NULL, start, end);
7100	}	7102	}
7101		7103

kernel/trace/trace.c+60−13

 static char *default_bootup_tracer;
 static bool allocate_snapshot;
+static bool snapshot_at_boot;
 static int __init set_cmdline_ftrace(char *str)
 {
 __setup("alloc_snapshot", boot_alloc_snapshot);
+static int __init boot_snapshot(char *str)
+{
+	snapshot_at_boot = true;
+	boot_alloc_snapshot(str);
+	return 1;
+}
+__setup("ftrace_boot_snapshot", boot_snapshot);
 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
 static int __init set_trace_boot_options(char *str)
 struct err_info {
 	const char	**errs;	/* ptr to loc-specific array of err strings */
 	u8		type;	/* index into errs -> specific err string */
-	u8		pos;	/* MAX_FILTER_STR_VAL = 256 */
+	u16		pos;	/* caret position */
 	u64		ts;
 };
 	struct list_head	list;
 	struct err_info		info;
 	char			loc[TRACING_LOG_LOC_MAX]; /* err location */
-	char			cmd[MAX_FILTER_STR_VAL]; /* what caused err */
+	char			*cmd;                     /* what caused err */
 };
 static DEFINE_MUTEX(tracing_err_log_lock);
-static struct tracing_log_err *get_tracing_log_err(struct trace_array *tr)
+static struct tracing_log_err *alloc_tracing_log_err(int len)
+{
+	struct tracing_log_err *err;
+	err = kzalloc(sizeof(*err), GFP_KERNEL);
+	if (!err)
+		return ERR_PTR(-ENOMEM);
+	err->cmd = kzalloc(len, GFP_KERNEL);
+	if (!err->cmd) {
+		kfree(err);
+		return ERR_PTR(-ENOMEM);
+	}
+	return err;
+}
+static void free_tracing_log_err(struct tracing_log_err *err)
+{
+	kfree(err->cmd);
+	kfree(err);
+}
+static struct tracing_log_err *get_tracing_log_err(struct trace_array *tr,
+						   int len)
 {
 	struct tracing_log_err *err;
 	if (tr->n_err_log_entries < TRACING_LOG_ERRS_MAX) {
-		err = kzalloc(sizeof(*err), GFP_KERNEL);
+		err = alloc_tracing_log_err(len);
-		if (!err)
+		if (PTR_ERR(err) != -ENOMEM)
-			err = ERR_PTR(-ENOMEM);
-		else
 			tr->n_err_log_entries++;
 		return err;
 	}
 	err = list_first_entry(&tr->err_log, struct tracing_log_err, list);
+	kfree(err->cmd);
+	err->cmd = kzalloc(len, GFP_KERNEL);
+	if (!err->cmd)
+		return ERR_PTR(-ENOMEM);
 	list_del(&err->list);
 	return err;
  */
 void tracing_log_err(struct trace_array *tr,
 		     const char *loc, const char *cmd,
-		     const char **errs, u8 type, u8 pos)
+		     const char **errs, u8 type, u16 pos)
 {
 	struct tracing_log_err *err;
+	int len = 0;
 	if (!tr)
 		tr = &global_trace;
+	len += sizeof(CMD_PREFIX) + 2 * sizeof("\n") + strlen(cmd) + 1;
 	mutex_lock(&tracing_err_log_lock);
-	err = get_tracing_log_err(tr);
+	err = get_tracing_log_err(tr, len);
 	if (PTR_ERR(err) == -ENOMEM) {
 		mutex_unlock(&tracing_err_log_lock);
 		return;
 	}
 	snprintf(err->loc, TRACING_LOG_LOC_MAX, "%s: error: ", loc);
-	snprintf(err->cmd, MAX_FILTER_STR_VAL,"\n" CMD_PREFIX "%s\n", cmd);
+	snprintf(err->cmd, len, "\n" CMD_PREFIX "%s\n", cmd);
 	err->info.errs = errs;
 	err->info.type = type;
 	mutex_lock(&tracing_err_log_lock);
 	list_for_each_entry_safe(err, next, &tr->err_log, list) {
 		list_del(&err->list);
-		kfree(err);
+		free_tracing_log_err(err);
 	}
 	tr->n_err_log_entries = 0;
 	mutex_unlock(&tracing_err_log_lock);
 }
-static void tracing_err_log_show_pos(struct seq_file *m, u8 pos)
+static void tracing_err_log_show_pos(struct seq_file *m, u16 pos)
 {
-	u8 i;
+	u16 i;
 	for (i = 0; i < sizeof(CMD_PREFIX) - 1; i++)
 		seq_putc(m, ' ');
 	return ret;
 }
+void __init ftrace_boot_snapshot(void)
+{
+	if (snapshot_at_boot) {
+		tracing_snapshot();
+		internal_trace_puts("** Boot snapshot taken **\n");
+	}
+}
 void __init early_trace_init(void)
 {
 	if (tracepoint_printk) {

kernel/trace/trace.h+1−1

@@ -1877,7 +1877,7 @@ extern ssize_t trace_parse_run_command(struct file *file, · 1876 unmodified lines
1877	extern unsigned int err_pos(char cmd, const char str);	1877	extern unsigned int err_pos(char cmd, const char str);
1878	extern void tracing_log_err(struct trace_array *tr,	1878	extern void tracing_log_err(struct trace_array *tr,
1879	const char loc, const char cmd,	1879	const char loc, const char cmd,
1880	const char **errs, u8 type, u8 pos);	1880	const char **errs, u8 type, u16 pos);
1881		1881
1882	/*	1882	/*
1883	* Normal trace_printk() and friends allocates special buffers	1883	* Normal trace_printk() and friends allocates special buffers

kernel/trace/trace_events.c+90−0

@@ -40,6 +40,14 @@ static LIST_HEAD(ftrace_generic_fields); · 39 unmodified lines
40	static LIST_HEAD(ftrace_common_fields);	40	static LIST_HEAD(ftrace_common_fields);
41	static bool eventdir_initialized;	41	static bool eventdir_initialized;
42		42
		43	static LIST_HEAD(module_strings);
		44
		45	struct module_string {
		46	struct list_head next;
		47	struct module *module;
		48	char *str;
		49	};
		50
43	#define GFP_TRACE (GFP_KERNEL \| __GFP_ZERO)	51	#define GFP_TRACE (GFP_KERNEL \| __GFP_ZERO)
44		52
45	static struct kmem_cache *field_cachep;	53	static struct kmem_cache *field_cachep;
@@ -2643,6 +2651,76 @@ static void update_event_printk(struct trace_event_call *call, · 2597 unmodified lines
2643	}	2651	}
2644	}	2652	}
2645		2653
		2654	static void add_str_to_module(struct module module, char str)
		2655	{
		2656	struct module_string *modstr;
		2657
		2658	modstr = kmalloc(sizeof(*modstr), GFP_KERNEL);
		2659
		2660	/*
		2661	* If we failed to allocate memory here, then we'll just
		2662	* let the str memory leak when the module is removed.
		2663	* If this fails to allocate, there's worse problems than
		2664	* a leaked string on module removal.
		2665	*/
		2666	if (WARN_ON_ONCE(!modstr))
		2667	return;
		2668
		2669	modstr->module = module;
		2670	modstr->str = str;
		2671
		2672	list_add(&modstr->next, &module_strings);
		2673	}
		2674
		2675	static void update_event_fields(struct trace_event_call *call,
		2676	struct trace_eval_map *map)
		2677	{
		2678	struct ftrace_event_field *field;
		2679	struct list_head *head;
		2680	char *ptr;
		2681	char *str;
		2682	int len = strlen(map->eval_string);
		2683
		2684	/* Dynamic events should never have field maps */
		2685	if (WARN_ON_ONCE(call->flags & TRACE_EVENT_FL_DYNAMIC))
		2686	return;
		2687
		2688	head = trace_get_fields(call);
		2689	list_for_each_entry(field, head, link) {
		2690	ptr = strchr(field->type, '[');
		2691	if (!ptr)
		2692	continue;
		2693	ptr++;
		2694
		2695	if (!isalpha(ptr) && ptr != '_')
		2696	continue;
		2697
		2698	if (strncmp(map->eval_string, ptr, len) != 0)
		2699	continue;
		2700
		2701	str = kstrdup(field->type, GFP_KERNEL);
		2702	if (WARN_ON_ONCE(!str))
		2703	return;
		2704	ptr = str + (ptr - field->type);
		2705	ptr = eval_replace(ptr, map, len);
		2706	/* enum/sizeof string smaller than value */
		2707	if (WARN_ON_ONCE(!ptr)) {
		2708	kfree(str);
		2709	continue;
		2710	}
		2711
		2712	/*
		2713	* If the event is part of a module, then we need to free the string
		2714	* when the module is removed. Otherwise, it will stay allocated
		2715	* until a reboot.
		2716	*/
		2717	if (call->module)
		2718	add_str_to_module(call->module, str);
		2719
		2720	field->type = str;
		2721	}
		2722	}
		2723
2646	void trace_event_eval_update(struct trace_eval_map **map, int len)	2724	void trace_event_eval_update(struct trace_eval_map **map, int len)
2647	{	2725	{
2648	struct trace_event_call call, p;	2726	struct trace_event_call call, p;
@@ -2678,6 +2756,7 @@ void trace_event_eval_update(struct trace_eval_map **map, int len) · 29 unmodified lines
2678	first = false;	2756	first = false;
2679	}	2757	}
2680	update_event_printk(call, map[i]);	2758	update_event_printk(call, map[i]);
		2759	update_event_fields(call, map[i]);
2681	}	2760	}
2682	}	2761	}
2683	}	2762	}
@@ -2768,6 +2847,7 @@ int trace_add_event_call(struct trace_event_call *call) · 84 unmodified lines
2768	mutex_unlock(&trace_types_lock);	2847	mutex_unlock(&trace_types_lock);
2769	return ret;	2848	return ret;
2770	}	2849	}
		2850	EXPORT_SYMBOL_GPL(trace_add_event_call);
2771		2851
2772	/*	2852	/*
2773	* Must be called under locking of trace_types_lock, event_mutex and	2853	* Must be called under locking of trace_types_lock, event_mutex and
@@ -2829,6 +2909,7 @@ int trace_remove_event_call(struct trace_event_call *call) · 55 unmodified lines
2829		2909
2830	return ret;	2910	return ret;
2831	}	2911	}
		2912	EXPORT_SYMBOL_GPL(trace_remove_event_call);
2832		2913
2833	#define for_each_event(event, start, end) \	2914	#define for_each_event(event, start, end) \
2834	for (event = start; \	2915	for (event = start; \
@@ -2863,6 +2944,7 @@ static void trace_module_add_events(struct module *mod) · 28 unmodified lines
2863	static void trace_module_remove_events(struct module *mod)	2944	static void trace_module_remove_events(struct module *mod)
2864	{	2945	{
2865	struct trace_event_call call, p;	2946	struct trace_event_call call, p;
		2947	struct module_string modstr, m;
2866		2948
2867	down_write(&trace_event_sem);	2949	down_write(&trace_event_sem);
2868	list_for_each_entry_safe(call, p, &ftrace_events, list) {	2950	list_for_each_entry_safe(call, p, &ftrace_events, list) {
@@ -2871,6 +2953,14 @@ static void trace_module_remove_events(struct module *mod) · 2 unmodified lines
2871	if (call->module == mod)	2953	if (call->module == mod)
2872	__trace_remove_event_call(call);	2954	__trace_remove_event_call(call);
2873	}	2955	}
		2956	/* Check for any strings allocade for this module */
		2957	list_for_each_entry_safe(modstr, m, &module_strings, next) {
		2958	if (modstr->module != mod)
		2959	continue;
		2960	list_del(&modstr->next);
		2961	kfree(modstr->str);
		2962	kfree(modstr);
		2963	}
2874	up_write(&trace_event_sem);	2964	up_write(&trace_event_sem);
2875		2965
2876	/*	2966	/*

kernel/trace/trace_events_hist.c+26−7

 	return data;
 }
-static char last_cmd[MAX_FILTER_STR_VAL];
+#define HIST_PREFIX "hist:"
+static char *last_cmd;
 static char last_cmd_loc[MAX_FILTER_STR_VAL];
 static int errpos(char *str)
 {
+	if (!str || !last_cmd)
+		return 0;
 	return err_pos(last_cmd, str);
 }
 {
 	const char *system = NULL, *name = NULL;
 	struct trace_event_call *call;
+	int len;
 	if (!str)
 		return;
-	strcpy(last_cmd, "hist:");
+	/* sizeof() contains the nul byte */
-	strncat(last_cmd, str, MAX_FILTER_STR_VAL - 1 - sizeof("hist:"));
+	len = sizeof(HIST_PREFIX) + strlen(str);
+	kfree(last_cmd);
+	last_cmd = kzalloc(len, GFP_KERNEL);
+	if (!last_cmd)
+		return;
+	strcpy(last_cmd, HIST_PREFIX);
+	/* Again, sizeof() contains the nul byte */
+	len -= sizeof(HIST_PREFIX);
+	strncat(last_cmd, str, len);
 	if (file) {
 		call = file->event_call;
 	}
 	if (system)
-		snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name);
+		snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, HIST_PREFIX "%s:%s", system, name);
 }
-static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos)
+static void hist_err(struct trace_array *tr, u8 err_type, u16 err_pos)
 {
+	if (!last_cmd)
+		return;
 	tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 			err_type, err_pos);
 }
 static void hist_err_clear(void)
 {
-	last_cmd[0] = '\0';
+	if (last_cmd)
+		last_cmd[0] = '\0';
 	last_cmd_loc[0] = '\0';
 }
 	bool have_var = false;
 	unsigned int i;
-	seq_puts(m, "hist:");
+	seq_puts(m, HIST_PREFIX);
 	if (data->name)
 		seq_printf(m, "%s:", data->name);

kernel/trace/trace_events_synth.c+11−3

 static const char *err_text[] = { ERRORS };
-static char last_cmd[MAX_FILTER_STR_VAL];
+static char *last_cmd;
 static int errpos(const char *str)
 {
+	if (!str || !last_cmd)
+		return 0;
 	return err_pos(last_cmd, str);
 }
 	if (!str)
 		return;
-	strncpy(last_cmd, str, MAX_FILTER_STR_VAL - 1);
+	kfree(last_cmd);
+	last_cmd = kstrdup(str, GFP_KERNEL);
 }
-static void synth_err(u8 err_type, u8 err_pos)
+static void synth_err(u8 err_type, u16 err_pos)
 {
+	if (!last_cmd)
+		return;
 	tracing_log_err(NULL, "synthetic_events", last_cmd, err_text,
 			err_type, err_pos);
 }

kernel/trace/trace_events_user.c+1690−0

@@ -0,0 +1,1690 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* Copyright (c) 2021, Microsoft Corporation.
	4	*
	5	* Authors:
	6	* Beau Belgrave <beaub@linux.microsoft.com>
	7	*/
	8
	9	#include <linux/bitmap.h>
	10	#include <linux/cdev.h>
	11	#include <linux/hashtable.h>
	12	#include <linux/list.h>
	13	#include <linux/io.h>
	14	#include <linux/uio.h>
	15	#include <linux/ioctl.h>
	16	#include <linux/jhash.h>
	17	#include <linux/trace_events.h>
	18	#include <linux/tracefs.h>
	19	#include <linux/types.h>
	20	#include <linux/uaccess.h>
	21	#include <uapi/linux/user_events.h>
	22	#include "trace.h"
	23	#include "trace_dynevent.h"
	24
	25	#define USER_EVENTS_PREFIX_LEN (sizeof(USER_EVENTS_PREFIX)-1)
	26
	27	#define FIELD_DEPTH_TYPE 0
	28	#define FIELD_DEPTH_NAME 1
	29	#define FIELD_DEPTH_SIZE 2
	30
	31	/*
	32	* Limits how many trace_event calls user processes can create:
	33	* Must be a power of two of PAGE_SIZE.
	34	*/
	35	#define MAX_PAGE_ORDER 0
	36	#define MAX_PAGES (1 << MAX_PAGE_ORDER)
	37	#define MAX_EVENTS (MAX_PAGES * PAGE_SIZE)
	38
	39	/* Limit how long of an event name plus args within the subsystem. */
	40	#define MAX_EVENT_DESC 512
	41	#define EVENT_NAME(user_event) ((user_event)->tracepoint.name)
	42	#define MAX_FIELD_ARRAY_SIZE 1024
	43	#define MAX_FIELD_ARG_NAME 256
	44
	45	#define MAX_BPF_COPY_SIZE PAGE_SIZE
	46	#define MAX_STACK_BPF_DATA 512
	47
	48	static char *register_page_data;
	49
	50	static DEFINE_MUTEX(reg_mutex);
	51	static DEFINE_HASHTABLE(register_table, 4);
	52	static DECLARE_BITMAP(page_bitmap, MAX_EVENTS);
	53
	54	/*
	55	* Stores per-event properties, as users register events
	56	* within a file a user_event might be created if it does not
	57	* already exist. These are globally used and their lifetime
	58	* is tied to the refcnt member. These cannot go away until the
	59	* refcnt reaches zero.
	60	*/
	61	struct user_event {
	62	struct tracepoint tracepoint;
	63	struct trace_event_call call;
	64	struct trace_event_class class;
	65	struct dyn_event devent;
	66	struct hlist_node node;
	67	struct list_head fields;
	68	struct list_head validators;
	69	atomic_t refcnt;
	70	int index;
	71	int flags;
	72	int min_size;
	73	};
	74
	75	/*
	76	* Stores per-file events references, as users register events
	77	* within a file this structure is modified and freed via RCU.
	78	* The lifetime of this struct is tied to the lifetime of the file.
	79	* These are not shared and only accessible by the file that created it.
	80	*/
	81	struct user_event_refs {
	82	struct rcu_head rcu;
	83	int count;
	84	struct user_event *events[];
	85	};
	86
	87	#define VALIDATOR_ENSURE_NULL (1 << 0)
	88	#define VALIDATOR_REL (1 << 1)
	89
	90	struct user_event_validator {
	91	struct list_head link;
	92	int offset;
	93	int flags;
	94	};
	95
	96	typedef void (user_event_func_t) (struct user_event user, struct iov_iter *i,
	97	void tpdata, bool faulted);
	98
	99	static int user_event_parse(char name, char args, char *flags,
	100	struct user_event **newuser);
	101
	102	static u32 user_event_key(char *name)
	103	{
	104	return jhash(name, strlen(name), 0);
	105	}
	106
	107	static __always_inline __must_check
	108	size_t copy_nofault(void addr, size_t bytes, struct iov_iter i)
	109	{
	110	size_t ret;
	111
	112	pagefault_disable();
	113
	114	ret = copy_from_iter_nocache(addr, bytes, i);
	115
	116	pagefault_enable();
	117
	118	return ret;
	119	}
	120
	121	static struct list_head user_event_get_fields(struct trace_event_call call)
	122	{
	123	struct user_event user = (struct user_event )call->data;
	124
	125	return &user->fields;
	126	}
	127
	128	/*
	129	* Parses a register command for user_events
	130	* Format: event_name[:FLAG1[,FLAG2...]] [field1[;field2...]]
	131	*
	132	* Example event named 'test' with a 20 char 'msg' field with an unsigned int
	133	* 'id' field after:
	134	* test char[20] msg;unsigned int id
	135	*
	136	* NOTE: Offsets are from the user data perspective, they are not from the
	137	* trace_entry/buffer perspective. We automatically add the common properties
	138	* sizes to the offset for the user.
	139	*
	140	* Upon success user_event has its ref count increased by 1.
	141	*/
	142	static int user_event_parse_cmd(char raw_command, struct user_event *newuser)
	143	{
	144	char *name = raw_command;
	145	char *args = strpbrk(name, " ");
	146	char *flags;
	147
	148	if (args)
	149	*args++ = '\0';
	150
	151	flags = strpbrk(name, ":");
	152
	153	if (flags)
	154	*flags++ = '\0';
	155
	156	return user_event_parse(name, args, flags, newuser);
	157	}
	158
	159	static int user_field_array_size(const char *type)
	160	{
	161	const char *start = strchr(type, '[');
	162	char val[8];
	163	char *bracket;
	164	int size = 0;
	165
	166	if (start == NULL)
	167	return -EINVAL;
	168
	169	if (strscpy(val, start + 1, sizeof(val)) <= 0)
	170	return -EINVAL;
	171
	172	bracket = strchr(val, ']');
	173
	174	if (!bracket)
	175	return -EINVAL;
	176
	177	*bracket = '\0';
	178
	179	if (kstrtouint(val, 0, &size))
	180	return -EINVAL;
	181
	182	if (size > MAX_FIELD_ARRAY_SIZE)
	183	return -EINVAL;
	184
	185	return size;
	186	}
	187
	188	static int user_field_size(const char *type)
	189	{
	190	/* long is not allowed from a user, since it's ambigious in size */
	191	if (strcmp(type, "s64") == 0)
	192	return sizeof(s64);
	193	if (strcmp(type, "u64") == 0)
	194	return sizeof(u64);
	195	if (strcmp(type, "s32") == 0)
	196	return sizeof(s32);
	197	if (strcmp(type, "u32") == 0)
	198	return sizeof(u32);
	199	if (strcmp(type, "int") == 0)
	200	return sizeof(int);
	201	if (strcmp(type, "unsigned int") == 0)
	202	return sizeof(unsigned int);
	203	if (strcmp(type, "s16") == 0)
	204	return sizeof(s16);
	205	if (strcmp(type, "u16") == 0)
	206	return sizeof(u16);
	207	if (strcmp(type, "short") == 0)
	208	return sizeof(short);
	209	if (strcmp(type, "unsigned short") == 0)
	210	return sizeof(unsigned short);
	211	if (strcmp(type, "s8") == 0)
	212	return sizeof(s8);
	213	if (strcmp(type, "u8") == 0)
	214	return sizeof(u8);
	215	if (strcmp(type, "char") == 0)
	216	return sizeof(char);
	217	if (strcmp(type, "unsigned char") == 0)
	218	return sizeof(unsigned char);
	219	if (str_has_prefix(type, "char["))
	220	return user_field_array_size(type);
	221	if (str_has_prefix(type, "unsigned char["))
	222	return user_field_array_size(type);
	223	if (str_has_prefix(type, "__data_loc "))
	224	return sizeof(u32);
	225	if (str_has_prefix(type, "__rel_loc "))
	226	return sizeof(u32);
	227
	228	/* Uknown basic type, error */
	229	return -EINVAL;
	230	}
	231
	232	static void user_event_destroy_validators(struct user_event *user)
	233	{
	234	struct user_event_validator validator, next;
	235	struct list_head *head = &user->validators;
	236
	237	list_for_each_entry_safe(validator, next, head, link) {
	238	list_del(&validator->link);
	239	kfree(validator);
	240	}
	241	}
	242
	243	static void user_event_destroy_fields(struct user_event *user)
	244	{
	245	struct ftrace_event_field field, next;
	246	struct list_head *head = &user->fields;
	247
	248	list_for_each_entry_safe(field, next, head, link) {
	249	list_del(&field->link);
	250	kfree(field);
	251	}
	252	}
	253
	254	static int user_event_add_field(struct user_event user, const char type,
	255	const char *name, int offset, int size,
	256	int is_signed, int filter_type)
	257	{
	258	struct user_event_validator *validator;
	259	struct ftrace_event_field *field;
	260	int validator_flags = 0;
	261
	262	field = kmalloc(sizeof(*field), GFP_KERNEL);
	263
	264	if (!field)
	265	return -ENOMEM;
	266
	267	if (str_has_prefix(type, "__data_loc "))
	268	goto add_validator;
	269
	270	if (str_has_prefix(type, "__rel_loc ")) {
	271	validator_flags \|= VALIDATOR_REL;
	272	goto add_validator;
	273	}
	274
	275	goto add_field;
	276
	277	add_validator:
	278	if (strstr(type, "char") != 0)
	279	validator_flags \|= VALIDATOR_ENSURE_NULL;
	280
	281	validator = kmalloc(sizeof(*validator), GFP_KERNEL);
	282
	283	if (!validator) {
	284	kfree(field);
	285	return -ENOMEM;
	286	}
	287
	288	validator->flags = validator_flags;
	289	validator->offset = offset;
	290
	291	/* Want sequential access when validating */
	292	list_add_tail(&validator->link, &user->validators);
	293
	294	add_field:
	295	field->type = type;
	296	field->name = name;
	297	field->offset = offset;
	298	field->size = size;
	299	field->is_signed = is_signed;
	300	field->filter_type = filter_type;
	301
	302	list_add(&field->link, &user->fields);
	303
	304	/*
	305	* Min size from user writes that are required, this does not include
	306	* the size of trace_entry (common fields).
	307	*/
	308	user->min_size = (offset + size) - sizeof(struct trace_entry);
	309
	310	return 0;
	311	}
	312
	313	/*
	314	* Parses the values of a field within the description
	315	* Format: type name [size]
	316	*/
	317	static int user_event_parse_field(char field, struct user_event user,
	318	u32 *offset)
	319	{
	320	char part, type, *name;
	321	u32 depth = 0, saved_offset = *offset;
	322	int len, size = -EINVAL;
	323	bool is_struct = false;
	324
	325	field = skip_spaces(field);
	326
	327	if (*field == '\0')
	328	return 0;
	329
	330	/* Handle types that have a space within */
	331	len = str_has_prefix(field, "unsigned ");
	332	if (len)
	333	goto skip_next;
	334
	335	len = str_has_prefix(field, "struct ");
	336	if (len) {
	337	is_struct = true;
	338	goto skip_next;
	339	}
	340
	341	len = str_has_prefix(field, "__data_loc unsigned ");
	342	if (len)
	343	goto skip_next;
	344
	345	len = str_has_prefix(field, "__data_loc ");
	346	if (len)
	347	goto skip_next;
	348
	349	len = str_has_prefix(field, "__rel_loc unsigned ");
	350	if (len)
	351	goto skip_next;
	352
	353	len = str_has_prefix(field, "__rel_loc ");
	354	if (len)
	355	goto skip_next;
	356
	357	goto parse;
	358	skip_next:
	359	type = field;
	360	field = strpbrk(field + len, " ");
	361
	362	if (field == NULL)
	363	return -EINVAL;
	364
	365	*field++ = '\0';
	366	depth++;
	367	parse:
	368	name = NULL;
	369
	370	while ((part = strsep(&field, " ")) != NULL) {
	371	switch (depth++) {
	372	case FIELD_DEPTH_TYPE:
	373	type = part;
	374	break;
	375	case FIELD_DEPTH_NAME:
	376	name = part;
	377	break;
	378	case FIELD_DEPTH_SIZE:
	379	if (!is_struct)
	380	return -EINVAL;
	381
	382	if (kstrtou32(part, 10, &size))
	383	return -EINVAL;
	384	break;
	385	default:
	386	return -EINVAL;
	387	}
	388	}
	389
	390	if (depth < FIELD_DEPTH_SIZE \|\| !name)
	391	return -EINVAL;
	392
	393	if (depth == FIELD_DEPTH_SIZE)
	394	size = user_field_size(type);
	395
	396	if (size == 0)
	397	return -EINVAL;
	398
	399	if (size < 0)
	400	return size;
	401
	402	*offset = saved_offset + size;
	403
	404	return user_event_add_field(user, type, name, saved_offset, size,
	405	type[0] != 'u', FILTER_OTHER);
	406	}
	407
	408	static void user_event_parse_flags(struct user_event user, char flags)
	409	{
	410	char *flag;
	411
	412	if (flags == NULL)
	413	return;
	414
	415	while ((flag = strsep(&flags, ",")) != NULL) {
	416	if (strcmp(flag, "BPF_ITER") == 0)
	417	user->flags \|= FLAG_BPF_ITER;
	418	}
	419	}
	420
	421	static int user_event_parse_fields(struct user_event user, char args)
	422	{
	423	char *field;
	424	u32 offset = sizeof(struct trace_entry);
	425	int ret = -EINVAL;
	426
	427	if (args == NULL)
	428	return 0;
	429
	430	while ((field = strsep(&args, ";")) != NULL) {
	431	ret = user_event_parse_field(field, user, &offset);
	432
	433	if (ret)
	434	break;
	435	}
	436
	437	return ret;
	438	}
	439
	440	static struct trace_event_fields user_event_fields_array[1];
	441
	442	static const char user_field_format(const char type)
	443	{
	444	if (strcmp(type, "s64") == 0)
	445	return "%lld";
	446	if (strcmp(type, "u64") == 0)
	447	return "%llu";
	448	if (strcmp(type, "s32") == 0)
	449	return "%d";
	450	if (strcmp(type, "u32") == 0)
	451	return "%u";
	452	if (strcmp(type, "int") == 0)
	453	return "%d";
	454	if (strcmp(type, "unsigned int") == 0)
	455	return "%u";
	456	if (strcmp(type, "s16") == 0)
	457	return "%d";
	458	if (strcmp(type, "u16") == 0)
	459	return "%u";
	460	if (strcmp(type, "short") == 0)
	461	return "%d";
	462	if (strcmp(type, "unsigned short") == 0)
	463	return "%u";
	464	if (strcmp(type, "s8") == 0)
	465	return "%d";
	466	if (strcmp(type, "u8") == 0)
	467	return "%u";
	468	if (strcmp(type, "char") == 0)
	469	return "%d";
	470	if (strcmp(type, "unsigned char") == 0)
	471	return "%u";
	472	if (strstr(type, "char[") != 0)
	473	return "%s";
	474
	475	/* Unknown, likely struct, allowed treat as 64-bit */
	476	return "%llu";
	477	}
	478
	479	static bool user_field_is_dyn_string(const char type, const char *str_func)
	480	{
	481	if (str_has_prefix(type, "__data_loc ")) {
	482	*str_func = "__get_str";
	483	goto check;
	484	}
	485
	486	if (str_has_prefix(type, "__rel_loc ")) {
	487	*str_func = "__get_rel_str";
	488	goto check;
	489	}
	490
	491	return false;
	492	check:
	493	return strstr(type, "char") != 0;
	494	}
	495
	496	#define LEN_OR_ZERO (len ? len - pos : 0)
	497	static int user_event_set_print_fmt(struct user_event user, char buf, int len)
	498	{
	499	struct ftrace_event_field field, next;
	500	struct list_head *head = &user->fields;
	501	int pos = 0, depth = 0;
	502	const char *str_func;
	503
	504	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
	505
	506	list_for_each_entry_safe_reverse(field, next, head, link) {
	507	if (depth != 0)
	508	pos += snprintf(buf + pos, LEN_OR_ZERO, " ");
	509
	510	pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s",
	511	field->name, user_field_format(field->type));
	512
	513	depth++;
	514	}
	515
	516	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
	517
	518	list_for_each_entry_safe_reverse(field, next, head, link) {
	519	if (user_field_is_dyn_string(field->type, &str_func))
	520	pos += snprintf(buf + pos, LEN_OR_ZERO,
	521	", %s(%s)", str_func, field->name);
	522	else
	523	pos += snprintf(buf + pos, LEN_OR_ZERO,
	524	", REC->%s", field->name);
	525	}
	526
	527	return pos + 1;
	528	}
	529	#undef LEN_OR_ZERO
	530
	531	static int user_event_create_print_fmt(struct user_event *user)
	532	{
	533	char *print_fmt;
	534	int len;
	535
	536	len = user_event_set_print_fmt(user, NULL, 0);
	537
	538	print_fmt = kmalloc(len, GFP_KERNEL);
	539
	540	if (!print_fmt)
	541	return -ENOMEM;
	542
	543	user_event_set_print_fmt(user, print_fmt, len);
	544
	545	user->call.print_fmt = print_fmt;
	546
	547	return 0;
	548	}
	549
	550	static enum print_line_t user_event_print_trace(struct trace_iterator *iter,
	551	int flags,
	552	struct trace_event *event)
	553	{
	554	/* Unsafe to try to decode user provided print_fmt, use hex */
	555	trace_print_hex_dump_seq(&iter->seq, "", DUMP_PREFIX_OFFSET, 16,
	556	1, iter->ent, iter->ent_size, true);
	557
	558	return trace_handle_return(&iter->seq);
	559	}
	560
	561	static struct trace_event_functions user_event_funcs = {
	562	.trace = user_event_print_trace,
	563	};
	564
	565	static int user_event_set_call_visible(struct user_event *user, bool visible)
	566	{
	567	int ret;
	568	const struct cred *old_cred;
	569	struct cred *cred;
	570
	571	cred = prepare_creds();
	572
	573	if (!cred)
	574	return -ENOMEM;
	575
	576	/*
	577	* While by default tracefs is locked down, systems can be configured
	578	* to allow user_event files to be less locked down. The extreme case
	579	* being "other" has read/write access to user_events_data/status.
	580	*
	581	* When not locked down, processes may not have have permissions to
	582	* add/remove calls themselves to tracefs. We need to temporarily
	583	* switch to root file permission to allow for this scenario.
	584	*/
	585	cred->fsuid = GLOBAL_ROOT_UID;
	586
	587	old_cred = override_creds(cred);
	588
	589	if (visible)
	590	ret = trace_add_event_call(&user->call);
	591	else
	592	ret = trace_remove_event_call(&user->call);
	593
	594	revert_creds(old_cred);
	595	put_cred(cred);
	596
	597	return ret;
	598	}
	599
	600	static int destroy_user_event(struct user_event *user)
	601	{
	602	int ret = 0;
	603
	604	/* Must destroy fields before call removal */
	605	user_event_destroy_fields(user);
	606
	607	ret = user_event_set_call_visible(user, false);
	608
	609	if (ret)
	610	return ret;
	611
	612	dyn_event_remove(&user->devent);
	613
	614	register_page_data[user->index] = 0;
	615	clear_bit(user->index, page_bitmap);
	616	hash_del(&user->node);
	617
	618	user_event_destroy_validators(user);
	619	kfree(user->call.print_fmt);
	620	kfree(EVENT_NAME(user));
	621	kfree(user);
	622
	623	return ret;
	624	}
	625
	626	static struct user_event find_user_event(char name, u32 *outkey)
	627	{
	628	struct user_event *user;
	629	u32 key = user_event_key(name);
	630
	631	*outkey = key;
	632
	633	hash_for_each_possible(register_table, user, node, key)
	634	if (!strcmp(EVENT_NAME(user), name)) {
	635	atomic_inc(&user->refcnt);
	636	return user;
	637	}
	638
	639	return NULL;
	640	}
	641
	642	static int user_event_validate(struct user_event user, void data, int len)
	643	{
	644	struct list_head *head = &user->validators;
	645	struct user_event_validator *validator;
	646	void pos, end = data + len;
	647	u32 loc, offset, size;
	648
	649	list_for_each_entry(validator, head, link) {
	650	pos = data + validator->offset;
	651
	652	/* Already done min_size check, no bounds check here */
	653	loc = (u32 )pos;
	654	offset = loc & 0xffff;
	655	size = loc >> 16;
	656
	657	if (likely(validator->flags & VALIDATOR_REL))
	658	pos += offset + sizeof(loc);
	659	else
	660	pos = data + offset;
	661
	662	pos += size;
	663
	664	if (unlikely(pos > end))
	665	return -EFAULT;
	666
	667	if (likely(validator->flags & VALIDATOR_ENSURE_NULL))
	668	if (unlikely((char )(pos - 1) != '\0'))
	669	return -EFAULT;
	670	}
	671
	672	return 0;
	673	}
	674
	675	/*
	676	* Writes the user supplied payload out to a trace file.
	677	*/
	678	static void user_event_ftrace(struct user_event user, struct iov_iter i,
	679	void tpdata, bool faulted)
	680	{
	681	struct trace_event_file *file;
	682	struct trace_entry *entry;
	683	struct trace_event_buffer event_buffer;
	684	size_t size = sizeof(*entry) + i->count;
	685
	686	file = (struct trace_event_file *)tpdata;
	687
	688	if (!file \|\|
	689	!(file->flags & EVENT_FILE_FL_ENABLED) \|\|
	690	trace_trigger_soft_disabled(file))
	691	return;
	692
	693	/* Allocates and fills trace_entry, + 1 of this is data payload */
	694	entry = trace_event_buffer_reserve(&event_buffer, file, size);
	695
	696	if (unlikely(!entry))
	697	return;
	698
	699	if (unlikely(!copy_nofault(entry + 1, i->count, i)))
	700	goto discard;
	701
	702	if (!list_empty(&user->validators) &&
	703	unlikely(user_event_validate(user, entry, size)))
	704	goto discard;
	705
	706	trace_event_buffer_commit(&event_buffer);
	707
	708	return;
	709	discard:
	710	*faulted = true;
	711	__trace_event_discard_commit(event_buffer.buffer,
	712	event_buffer.event);
	713	}
	714
	715	#ifdef CONFIG_PERF_EVENTS
	716	static void user_event_bpf(struct user_event user, struct iov_iter i)
	717	{
	718	struct user_bpf_context context;
	719	struct user_bpf_iter bpf_i;
	720	char fast_data[MAX_STACK_BPF_DATA];
	721	void *temp = NULL;
	722
	723	if ((user->flags & FLAG_BPF_ITER) && iter_is_iovec(i)) {
	724	/* Raw iterator */
	725	context.data_type = USER_BPF_DATA_ITER;
	726	context.data_len = i->count;
	727	context.iter = &bpf_i;
	728
	729	bpf_i.iov_offset = i->iov_offset;
	730	bpf_i.iov = i->iov;
	731	bpf_i.nr_segs = i->nr_segs;
	732	} else if (i->nr_segs == 1 && iter_is_iovec(i)) {
	733	/* Single buffer from user */
	734	context.data_type = USER_BPF_DATA_USER;
	735	context.data_len = i->count;
	736	context.udata = i->iov->iov_base + i->iov_offset;
	737	} else {
	738	/* Multi buffer from user */
	739	struct iov_iter copy = *i;
	740	size_t copy_size = min_t(size_t, i->count, MAX_BPF_COPY_SIZE);
	741
	742	context.data_type = USER_BPF_DATA_KERNEL;
	743	context.kdata = fast_data;
	744
	745	if (unlikely(copy_size > sizeof(fast_data))) {
	746	temp = kmalloc(copy_size, GFP_NOWAIT);
	747
	748	if (temp)
	749	context.kdata = temp;
	750	else
	751	copy_size = sizeof(fast_data);
	752	}
	753
	754	context.data_len = copy_nofault(context.kdata,
	755	copy_size, &copy);
	756	}
	757
	758	trace_call_bpf(&user->call, &context);
	759
	760	kfree(temp);
	761	}
	762
	763	/*
	764	* Writes the user supplied payload out to perf ring buffer or eBPF program.
	765	*/
	766	static void user_event_perf(struct user_event user, struct iov_iter i,
	767	void tpdata, bool faulted)
	768	{
	769	struct hlist_head *perf_head;
	770
	771	if (bpf_prog_array_valid(&user->call))
	772	user_event_bpf(user, i);
	773
	774	perf_head = this_cpu_ptr(user->call.perf_events);
	775
	776	if (perf_head && !hlist_empty(perf_head)) {
	777	struct trace_entry *perf_entry;
	778	struct pt_regs *regs;
	779	size_t size = sizeof(*perf_entry) + i->count;
	780	int context;
	781
	782	perf_entry = perf_trace_buf_alloc(ALIGN(size, 8),
	783	&regs, &context);
	784
	785	if (unlikely(!perf_entry))
	786	return;
	787
	788	perf_fetch_caller_regs(regs);
	789
	790	if (unlikely(!copy_nofault(perf_entry + 1, i->count, i)))
	791	goto discard;
	792
	793	if (!list_empty(&user->validators) &&
	794	unlikely(user_event_validate(user, perf_entry, size)))
	795	goto discard;
	796
	797	perf_trace_buf_submit(perf_entry, size, context,
	798	user->call.event.type, 1, regs,
	799	perf_head, NULL);
	800
	801	return;
	802	discard:
	803	*faulted = true;
	804	perf_swevent_put_recursion_context(context);
	805	}
	806	}
	807	#endif
	808
	809	/*
	810	* Update the register page that is shared between user processes.
	811	*/
	812	static void update_reg_page_for(struct user_event *user)
	813	{
	814	struct tracepoint *tp = &user->tracepoint;
	815	char status = 0;
	816
	817	if (atomic_read(&tp->key.enabled) > 0) {
	818	struct tracepoint_func *probe_func_ptr;
	819	user_event_func_t probe_func;
	820
	821	rcu_read_lock_sched();
	822
	823	probe_func_ptr = rcu_dereference_sched(tp->funcs);
	824
	825	if (probe_func_ptr) {
	826	do {
	827	probe_func = probe_func_ptr->func;
	828
	829	if (probe_func == user_event_ftrace)
	830	status \|= EVENT_STATUS_FTRACE;
	831	#ifdef CONFIG_PERF_EVENTS
	832	else if (probe_func == user_event_perf)
	833	status \|= EVENT_STATUS_PERF;
	834	#endif
	835	else
	836	status \|= EVENT_STATUS_OTHER;
	837	} while ((++probe_func_ptr)->func);
	838	}
	839
	840	rcu_read_unlock_sched();
	841	}
	842
	843	register_page_data[user->index] = status;
	844	}
	845
	846	/*
	847	* Register callback for our events from tracing sub-systems.
	848	*/
	849	static int user_event_reg(struct trace_event_call *call,
	850	enum trace_reg type,
	851	void *data)
	852	{
	853	struct user_event user = (struct user_event )call->data;
	854	int ret = 0;
	855
	856	if (!user)
	857	return -ENOENT;
	858
	859	switch (type) {
	860	case TRACE_REG_REGISTER:
	861	ret = tracepoint_probe_register(call->tp,
	862	call->class->probe,
	863	data);
	864	if (!ret)
	865	goto inc;
	866	break;
	867
	868	case TRACE_REG_UNREGISTER:
	869	tracepoint_probe_unregister(call->tp,
	870	call->class->probe,
	871	data);
	872	goto dec;
	873
	874	#ifdef CONFIG_PERF_EVENTS
	875	case TRACE_REG_PERF_REGISTER:
	876	ret = tracepoint_probe_register(call->tp,
	877	call->class->perf_probe,
	878	data);
	879	if (!ret)
	880	goto inc;
	881	break;
	882
	883	case TRACE_REG_PERF_UNREGISTER:
	884	tracepoint_probe_unregister(call->tp,
	885	call->class->perf_probe,
	886	data);
	887	goto dec;
	888
	889	case TRACE_REG_PERF_OPEN:
	890	case TRACE_REG_PERF_CLOSE:
	891	case TRACE_REG_PERF_ADD:
	892	case TRACE_REG_PERF_DEL:
	893	break;
	894	#endif
	895	}
	896
	897	return ret;
	898	inc:
	899	atomic_inc(&user->refcnt);
	900	update_reg_page_for(user);
	901	return 0;
	902	dec:
	903	update_reg_page_for(user);
	904	atomic_dec(&user->refcnt);
	905	return 0;
	906	}
	907
	908	static int user_event_create(const char *raw_command)
	909	{
	910	struct user_event *user;
	911	char *name;
	912	int ret;
	913
	914	if (!str_has_prefix(raw_command, USER_EVENTS_PREFIX))
	915	return -ECANCELED;
	916
	917	raw_command += USER_EVENTS_PREFIX_LEN;
	918	raw_command = skip_spaces(raw_command);
	919
	920	name = kstrdup(raw_command, GFP_KERNEL);
	921
	922	if (!name)
	923	return -ENOMEM;
	924
	925	mutex_lock(&reg_mutex);
	926
	927	ret = user_event_parse_cmd(name, &user);
	928
	929	if (!ret)
	930	atomic_dec(&user->refcnt);
	931
	932	mutex_unlock(&reg_mutex);
	933
	934	if (ret)
	935	kfree(name);
	936
	937	return ret;
	938	}
	939
	940	static int user_event_show(struct seq_file m, struct dyn_event ev)
	941	{
	942	struct user_event *user = container_of(ev, struct user_event, devent);
	943	struct ftrace_event_field field, next;
	944	struct list_head *head;
	945	int depth = 0;
	946
	947	seq_printf(m, "%s%s", USER_EVENTS_PREFIX, EVENT_NAME(user));
	948
	949	head = trace_get_fields(&user->call);
	950
	951	list_for_each_entry_safe_reverse(field, next, head, link) {
	952	if (depth == 0)
	953	seq_puts(m, " ");
	954	else
	955	seq_puts(m, "; ");
	956
	957	seq_printf(m, "%s %s", field->type, field->name);
	958
	959	if (str_has_prefix(field->type, "struct "))
	960	seq_printf(m, " %d", field->size);
	961
	962	depth++;
	963	}
	964
	965	seq_puts(m, "\n");
	966
	967	return 0;
	968	}
	969
	970	static bool user_event_is_busy(struct dyn_event *ev)
	971	{
	972	struct user_event *user = container_of(ev, struct user_event, devent);
	973
	974	return atomic_read(&user->refcnt) != 0;
	975	}
	976
	977	static int user_event_free(struct dyn_event *ev)
	978	{
	979	struct user_event *user = container_of(ev, struct user_event, devent);
	980
	981	if (atomic_read(&user->refcnt) != 0)
	982	return -EBUSY;
	983
	984	return destroy_user_event(user);
	985	}
	986
	987	static bool user_field_match(struct ftrace_event_field *field, int argc,
	988	const char *argv, int iout)
	989	{
	990	char field_name, arg_name;
	991	int len, pos, i = *iout;
	992	bool colon = false, match = false;
	993
	994	if (i >= argc)
	995	return false;
	996
	997	len = MAX_FIELD_ARG_NAME;
	998	field_name = kmalloc(len, GFP_KERNEL);
	999	arg_name = kmalloc(len, GFP_KERNEL);
	1000
	1001	if (!arg_name \|\| !field_name)
	1002	goto out;
	1003
	1004	pos = 0;
	1005
	1006	for (; i < argc; ++i) {
	1007	if (i != *iout)
	1008	pos += snprintf(arg_name + pos, len - pos, " ");
	1009
	1010	pos += snprintf(arg_name + pos, len - pos, argv[i]);
	1011
	1012	if (strchr(argv[i], ';')) {
	1013	++i;
	1014	colon = true;
	1015	break;
	1016	}
	1017	}
	1018
	1019	pos = 0;
	1020
	1021	pos += snprintf(field_name + pos, len - pos, field->type);
	1022	pos += snprintf(field_name + pos, len - pos, " ");
	1023	pos += snprintf(field_name + pos, len - pos, field->name);
	1024
	1025	if (colon)
	1026	pos += snprintf(field_name + pos, len - pos, ";");
	1027
	1028	*iout = i;
	1029
	1030	match = strcmp(arg_name, field_name) == 0;
	1031	out:
	1032	kfree(arg_name);
	1033	kfree(field_name);
	1034
	1035	return match;
	1036	}
	1037
	1038	static bool user_fields_match(struct user_event *user, int argc,
	1039	const char **argv)
	1040	{
	1041	struct ftrace_event_field field, next;
	1042	struct list_head *head = &user->fields;
	1043	int i = 0;
	1044
	1045	list_for_each_entry_safe_reverse(field, next, head, link)
	1046	if (!user_field_match(field, argc, argv, &i))
	1047	return false;
	1048
	1049	if (i != argc)
	1050	return false;
	1051
	1052	return true;
	1053	}
	1054
	1055	static bool user_event_match(const char system, const char event,
	1056	int argc, const char *argv, struct dyn_event ev)
	1057	{
	1058	struct user_event *user = container_of(ev, struct user_event, devent);
	1059	bool match;
	1060
	1061	match = strcmp(EVENT_NAME(user), event) == 0 &&
	1062	(!system \|\| strcmp(system, USER_EVENTS_SYSTEM) == 0);
	1063
	1064	if (match && argc > 0)
	1065	match = user_fields_match(user, argc, argv);
	1066
	1067	return match;
	1068	}
	1069
	1070	static struct dyn_event_operations user_event_dops = {
	1071	.create = user_event_create,
	1072	.show = user_event_show,
	1073	.is_busy = user_event_is_busy,
	1074	.free = user_event_free,
	1075	.match = user_event_match,
	1076	};
	1077
	1078	static int user_event_trace_register(struct user_event *user)
	1079	{
	1080	int ret;
	1081
	1082	ret = register_trace_event(&user->call.event);
	1083
	1084	if (!ret)
	1085	return -ENODEV;
	1086
	1087	ret = user_event_set_call_visible(user, true);
	1088
	1089	if (ret)
	1090	unregister_trace_event(&user->call.event);
	1091
	1092	return ret;
	1093	}
	1094
	1095	/*
	1096	* Parses the event name, arguments and flags then registers if successful.
	1097	* The name buffer lifetime is owned by this method for success cases only.
	1098	* Upon success the returned user_event has its ref count increased by 1.
	1099	*/
	1100	static int user_event_parse(char name, char args, char *flags,
	1101	struct user_event **newuser)
	1102	{
	1103	int ret;
	1104	int index;
	1105	u32 key;
	1106	struct user_event *user;
	1107
	1108	/* Prevent dyn_event from racing */
	1109	mutex_lock(&event_mutex);
	1110	user = find_user_event(name, &key);
	1111	mutex_unlock(&event_mutex);
	1112
	1113	if (user) {
	1114	*newuser = user;
	1115	/*
	1116	* Name is allocated by caller, free it since it already exists.
	1117	* Caller only worries about failure cases for freeing.
	1118	*/
	1119	kfree(name);
	1120	return 0;
	1121	}
	1122
	1123	index = find_first_zero_bit(page_bitmap, MAX_EVENTS);
	1124
	1125	if (index == MAX_EVENTS)
	1126	return -EMFILE;
	1127
	1128	user = kzalloc(sizeof(*user), GFP_KERNEL);
	1129
	1130	if (!user)
	1131	return -ENOMEM;
	1132
	1133	INIT_LIST_HEAD(&user->class.fields);
	1134	INIT_LIST_HEAD(&user->fields);
	1135	INIT_LIST_HEAD(&user->validators);
	1136
	1137	user->tracepoint.name = name;
	1138
	1139	user_event_parse_flags(user, flags);
	1140
	1141	ret = user_event_parse_fields(user, args);
	1142
	1143	if (ret)
	1144	goto put_user;
	1145
	1146	ret = user_event_create_print_fmt(user);
	1147
	1148	if (ret)
	1149	goto put_user;
	1150
	1151	user->call.data = user;
	1152	user->call.class = &user->class;
	1153	user->call.name = name;
	1154	user->call.flags = TRACE_EVENT_FL_TRACEPOINT;
	1155	user->call.tp = &user->tracepoint;
	1156	user->call.event.funcs = &user_event_funcs;
	1157
	1158	user->class.system = USER_EVENTS_SYSTEM;
	1159	user->class.fields_array = user_event_fields_array;
	1160	user->class.get_fields = user_event_get_fields;
	1161	user->class.reg = user_event_reg;
	1162	user->class.probe = user_event_ftrace;
	1163	#ifdef CONFIG_PERF_EVENTS
	1164	user->class.perf_probe = user_event_perf;
	1165	#endif
	1166
	1167	mutex_lock(&event_mutex);
	1168	ret = user_event_trace_register(user);
	1169	mutex_unlock(&event_mutex);
	1170
	1171	if (ret)
	1172	goto put_user;
	1173
	1174	user->index = index;
	1175
	1176	/* Ensure we track ref */
	1177	atomic_inc(&user->refcnt);
	1178
	1179	dyn_event_init(&user->devent, &user_event_dops);
	1180	dyn_event_add(&user->devent, &user->call);
	1181	set_bit(user->index, page_bitmap);
	1182	hash_add(register_table, &user->node, key);
	1183
	1184	*newuser = user;
	1185	return 0;
	1186	put_user:
	1187	user_event_destroy_fields(user);
	1188	user_event_destroy_validators(user);
	1189	kfree(user);
	1190	return ret;
	1191	}
	1192
	1193	/*
	1194	* Deletes a previously created event if it is no longer being used.
	1195	*/
	1196	static int delete_user_event(char *name)
	1197	{
	1198	u32 key;
	1199	int ret;
	1200	struct user_event *user = find_user_event(name, &key);
	1201
	1202	if (!user)
	1203	return -ENOENT;
	1204
	1205	/* Ensure we are the last ref */
	1206	if (atomic_read(&user->refcnt) != 1) {
	1207	ret = -EBUSY;
	1208	goto put_ref;
	1209	}
	1210
	1211	ret = destroy_user_event(user);
	1212
	1213	if (ret)
	1214	goto put_ref;
	1215
	1216	return ret;
	1217	put_ref:
	1218	/* No longer have this ref */
	1219	atomic_dec(&user->refcnt);
	1220
	1221	return ret;
	1222	}
	1223
	1224	/*
	1225	* Validates the user payload and writes via iterator.
	1226	*/
	1227	static ssize_t user_events_write_core(struct file file, struct iov_iter i)
	1228	{
	1229	struct user_event_refs *refs;
	1230	struct user_event *user = NULL;
	1231	struct tracepoint *tp;
	1232	ssize_t ret = i->count;
	1233	int idx;
	1234
	1235	if (unlikely(copy_from_iter(&idx, sizeof(idx), i) != sizeof(idx)))
	1236	return -EFAULT;
	1237
	1238	rcu_read_lock_sched();
	1239
	1240	refs = rcu_dereference_sched(file->private_data);
	1241
	1242	/*
	1243	* The refs->events array is protected by RCU, and new items may be
	1244	* added. But the user retrieved from indexing into the events array
	1245	* shall be immutable while the file is opened.
	1246	*/
	1247	if (likely(refs && idx < refs->count))
	1248	user = refs->events[idx];
	1249
	1250	rcu_read_unlock_sched();
	1251
	1252	if (unlikely(user == NULL))
	1253	return -ENOENT;
	1254
	1255	if (unlikely(i->count < user->min_size))
	1256	return -EINVAL;
	1257
	1258	tp = &user->tracepoint;
	1259
	1260	/*
	1261	* It's possible key.enabled disables after this check, however
	1262	* we don't mind if a few events are included in this condition.
	1263	*/
	1264	if (likely(atomic_read(&tp->key.enabled) > 0)) {
	1265	struct tracepoint_func *probe_func_ptr;
	1266	user_event_func_t probe_func;
	1267	struct iov_iter copy;
	1268	void *tpdata;
	1269	bool faulted;
	1270
	1271	if (unlikely(fault_in_iov_iter_readable(i, i->count)))
	1272	return -EFAULT;
	1273
	1274	faulted = false;
	1275
	1276	rcu_read_lock_sched();
	1277
	1278	probe_func_ptr = rcu_dereference_sched(tp->funcs);
	1279
	1280	if (probe_func_ptr) {
	1281	do {
	1282	copy = *i;
	1283	probe_func = probe_func_ptr->func;
	1284	tpdata = probe_func_ptr->data;
	1285	probe_func(user, &copy, tpdata, &faulted);
	1286	} while ((++probe_func_ptr)->func);
	1287	}
	1288
	1289	rcu_read_unlock_sched();
	1290
	1291	if (unlikely(faulted))
	1292	return -EFAULT;
	1293	}
	1294
	1295	return ret;
	1296	}
	1297
	1298	static ssize_t user_events_write(struct file file, const char __user ubuf,
	1299	size_t count, loff_t *ppos)
	1300	{
	1301	struct iovec iov;
	1302	struct iov_iter i;
	1303
	1304	if (unlikely(*ppos != 0))
	1305	return -EFAULT;
	1306
	1307	if (unlikely(import_single_range(READ, (char *)ubuf, count, &iov, &i)))
	1308	return -EFAULT;
	1309
	1310	return user_events_write_core(file, &i);
	1311	}
	1312
	1313	static ssize_t user_events_write_iter(struct kiocb kp, struct iov_iter i)
	1314	{
	1315	return user_events_write_core(kp->ki_filp, i);
	1316	}
	1317
	1318	static int user_events_ref_add(struct file file, struct user_event user)
	1319	{
	1320	struct user_event_refs refs, new_refs;
	1321	int i, size, count = 0;
	1322
	1323	refs = rcu_dereference_protected(file->private_data,
	1324	lockdep_is_held(&reg_mutex));
	1325
	1326	if (refs) {
	1327	count = refs->count;
	1328
	1329	for (i = 0; i < count; ++i)
	1330	if (refs->events[i] == user)
	1331	return i;
	1332	}
	1333
	1334	size = struct_size(refs, events, count + 1);
	1335
	1336	new_refs = kzalloc(size, GFP_KERNEL);
	1337
	1338	if (!new_refs)
	1339	return -ENOMEM;
	1340
	1341	new_refs->count = count + 1;
	1342
	1343	for (i = 0; i < count; ++i)
	1344	new_refs->events[i] = refs->events[i];
	1345
	1346	new_refs->events[i] = user;
	1347
	1348	atomic_inc(&user->refcnt);
	1349
	1350	rcu_assign_pointer(file->private_data, new_refs);
	1351
	1352	if (refs)
	1353	kfree_rcu(refs, rcu);
	1354
	1355	return i;
	1356	}
	1357
	1358	static long user_reg_get(struct user_reg __user ureg, struct user_reg kreg)
	1359	{
	1360	u32 size;
	1361	long ret;
	1362
	1363	ret = get_user(size, &ureg->size);
	1364
	1365	if (ret)
	1366	return ret;
	1367
	1368	if (size > PAGE_SIZE)
	1369	return -E2BIG;
	1370
	1371	return copy_struct_from_user(kreg, sizeof(*kreg), ureg, size);
	1372	}
	1373
	1374	/*
	1375	* Registers a user_event on behalf of a user process.
	1376	*/
	1377	static long user_events_ioctl_reg(struct file *file, unsigned long uarg)
	1378	{
	1379	struct user_reg __user ureg = (struct user_reg __user )uarg;
	1380	struct user_reg reg;
	1381	struct user_event *user;
	1382	char *name;
	1383	long ret;
	1384
	1385	ret = user_reg_get(ureg, &reg);
	1386
	1387	if (ret)
	1388	return ret;
	1389
	1390	name = strndup_user((const char __user *)(uintptr_t)reg.name_args,
	1391	MAX_EVENT_DESC);
	1392
	1393	if (IS_ERR(name)) {
	1394	ret = PTR_ERR(name);
	1395	return ret;
	1396	}
	1397
	1398	ret = user_event_parse_cmd(name, &user);
	1399
	1400	if (ret) {
	1401	kfree(name);
	1402	return ret;
	1403	}
	1404
	1405	ret = user_events_ref_add(file, user);
	1406
	1407	/* No longer need parse ref, ref_add either worked or not */
	1408	atomic_dec(&user->refcnt);
	1409
	1410	/* Positive number is index and valid */
	1411	if (ret < 0)
	1412	return ret;
	1413
	1414	put_user((u32)ret, &ureg->write_index);
	1415	put_user(user->index, &ureg->status_index);
	1416
	1417	return 0;
	1418	}
	1419
	1420	/*
	1421	* Deletes a user_event on behalf of a user process.
	1422	*/
	1423	static long user_events_ioctl_del(struct file *file, unsigned long uarg)
	1424	{
	1425	void __user ubuf = (void __user )uarg;
	1426	char *name;
	1427	long ret;
	1428
	1429	name = strndup_user(ubuf, MAX_EVENT_DESC);
	1430
	1431	if (IS_ERR(name))
	1432	return PTR_ERR(name);
	1433
	1434	/* event_mutex prevents dyn_event from racing */
	1435	mutex_lock(&event_mutex);
	1436	ret = delete_user_event(name);
	1437	mutex_unlock(&event_mutex);
	1438
	1439	kfree(name);
	1440
	1441	return ret;
	1442	}
	1443
	1444	/*
	1445	* Handles the ioctl from user mode to register or alter operations.
	1446	*/
	1447	static long user_events_ioctl(struct file *file, unsigned int cmd,
	1448	unsigned long uarg)
	1449	{
	1450	long ret = -ENOTTY;
	1451
	1452	switch (cmd) {
	1453	case DIAG_IOCSREG:
	1454	mutex_lock(&reg_mutex);
	1455	ret = user_events_ioctl_reg(file, uarg);
	1456	mutex_unlock(&reg_mutex);
	1457	break;
	1458
	1459	case DIAG_IOCSDEL:
	1460	mutex_lock(&reg_mutex);
	1461	ret = user_events_ioctl_del(file, uarg);
	1462	mutex_unlock(&reg_mutex);
	1463	break;
	1464	}
	1465
	1466	return ret;
	1467	}
	1468
	1469	/*
	1470	* Handles the final close of the file from user mode.
	1471	*/
	1472	static int user_events_release(struct inode node, struct file file)
	1473	{
	1474	struct user_event_refs *refs;
	1475	struct user_event *user;
	1476	int i;
	1477
	1478	/*
	1479	* Ensure refs cannot change under any situation by taking the
	1480	* register mutex during the final freeing of the references.
	1481	*/
	1482	mutex_lock(&reg_mutex);
	1483
	1484	refs = file->private_data;
	1485
	1486	if (!refs)
	1487	goto out;
	1488
	1489	/*
	1490	* The lifetime of refs has reached an end, it's tied to this file.
	1491	* The underlying user_events are ref counted, and cannot be freed.
	1492	* After this decrement, the user_events may be freed elsewhere.
	1493	*/
	1494	for (i = 0; i < refs->count; ++i) {
	1495	user = refs->events[i];
	1496
	1497	if (user)
	1498	atomic_dec(&user->refcnt);
	1499	}
	1500	out:
	1501	file->private_data = NULL;
	1502
	1503	mutex_unlock(&reg_mutex);
	1504
	1505	kfree(refs);
	1506
	1507	return 0;
	1508	}
	1509
	1510	static const struct file_operations user_data_fops = {
	1511	.write = user_events_write,
	1512	.write_iter = user_events_write_iter,
	1513	.unlocked_ioctl = user_events_ioctl,
	1514	.release = user_events_release,
	1515	};
	1516
	1517	/*
	1518	* Maps the shared page into the user process for checking if event is enabled.
	1519	*/
	1520	static int user_status_mmap(struct file file, struct vm_area_struct vma)
	1521	{
	1522	unsigned long size = vma->vm_end - vma->vm_start;
	1523
	1524	if (size != MAX_EVENTS)
	1525	return -EINVAL;
	1526
	1527	return remap_pfn_range(vma, vma->vm_start,
	1528	virt_to_phys(register_page_data) >> PAGE_SHIFT,
	1529	size, vm_get_page_prot(VM_READ));
	1530	}
	1531
	1532	static void user_seq_start(struct seq_file m, loff_t *pos)
	1533	{
	1534	if (*pos)
	1535	return NULL;
	1536
	1537	return (void *)1;
	1538	}
	1539
	1540	static void user_seq_next(struct seq_file m, void p, loff_t pos)
	1541	{
	1542	++*pos;
	1543	return NULL;
	1544	}
	1545
	1546	static void user_seq_stop(struct seq_file m, void p)
	1547	{
	1548	}
	1549
	1550	static int user_seq_show(struct seq_file m, void p)
	1551	{
	1552	struct user_event *user;
	1553	char status;
	1554	int i, active = 0, busy = 0, flags;
	1555
	1556	mutex_lock(&reg_mutex);
	1557
	1558	hash_for_each(register_table, i, user, node) {
	1559	status = register_page_data[user->index];
	1560	flags = user->flags;
	1561
	1562	seq_printf(m, "%d:%s", user->index, EVENT_NAME(user));
	1563
	1564	if (flags != 0 \|\| status != 0)
	1565	seq_puts(m, " #");
	1566
	1567	if (status != 0) {
	1568	seq_puts(m, " Used by");
	1569	if (status & EVENT_STATUS_FTRACE)
	1570	seq_puts(m, " ftrace");
	1571	if (status & EVENT_STATUS_PERF)
	1572	seq_puts(m, " perf");
	1573	if (status & EVENT_STATUS_OTHER)
	1574	seq_puts(m, " other");
	1575	busy++;
	1576	}
	1577
	1578	if (flags & FLAG_BPF_ITER)
	1579	seq_puts(m, " FLAG:BPF_ITER");
	1580
	1581	seq_puts(m, "\n");
	1582	active++;
	1583	}
	1584
	1585	mutex_unlock(&reg_mutex);
	1586
	1587	seq_puts(m, "\n");
	1588	seq_printf(m, "Active: %d\n", active);
	1589	seq_printf(m, "Busy: %d\n", busy);
	1590	seq_printf(m, "Max: %ld\n", MAX_EVENTS);
	1591
	1592	return 0;
	1593	}
	1594
	1595	static const struct seq_operations user_seq_ops = {
	1596	.start = user_seq_start,
	1597	.next = user_seq_next,
	1598	.stop = user_seq_stop,
	1599	.show = user_seq_show,
	1600	};
	1601
	1602	static int user_status_open(struct inode node, struct file file)
	1603	{
	1604	return seq_open(file, &user_seq_ops);
	1605	}
	1606
	1607	static const struct file_operations user_status_fops = {
	1608	.open = user_status_open,
	1609	.mmap = user_status_mmap,
	1610	.read = seq_read,
	1611	.llseek = seq_lseek,
	1612	.release = seq_release,
	1613	};
	1614
	1615	/*
	1616	* Creates a set of tracefs files to allow user mode interactions.
	1617	*/
	1618	static int create_user_tracefs(void)
	1619	{
	1620	struct dentry edata, emmap;
	1621
	1622	edata = tracefs_create_file("user_events_data", TRACE_MODE_WRITE,
	1623	NULL, NULL, &user_data_fops);
	1624
	1625	if (!edata) {
	1626	pr_warn("Could not create tracefs 'user_events_data' entry\n");
	1627	goto err;
	1628	}
	1629
	1630	/* mmap with MAP_SHARED requires writable fd */
	1631	emmap = tracefs_create_file("user_events_status", TRACE_MODE_WRITE,
	1632	NULL, NULL, &user_status_fops);
	1633
	1634	if (!emmap) {
	1635	tracefs_remove(edata);
	1636	pr_warn("Could not create tracefs 'user_events_mmap' entry\n");
	1637	goto err;
	1638	}
	1639
	1640	return 0;
	1641	err:
	1642	return -ENODEV;
	1643	}
	1644
	1645	static void set_page_reservations(bool set)
	1646	{
	1647	int page;
	1648
	1649	for (page = 0; page < MAX_PAGES; ++page) {
	1650	void addr = register_page_data + (PAGE_SIZE page);
	1651
	1652	if (set)
	1653	SetPageReserved(virt_to_page(addr));
	1654	else
	1655	ClearPageReserved(virt_to_page(addr));
	1656	}
	1657	}
	1658
	1659	static int __init trace_events_user_init(void)
	1660	{
	1661	struct page *pages;
	1662	int ret;
	1663
	1664	/* Zero all bits beside 0 (which is reserved for failures) */
	1665	bitmap_zero(page_bitmap, MAX_EVENTS);
	1666	set_bit(0, page_bitmap);
	1667
	1668	pages = alloc_pages(GFP_KERNEL \| __GFP_ZERO, MAX_PAGE_ORDER);
	1669	if (!pages)
	1670	return -ENOMEM;
	1671	register_page_data = page_address(pages);
	1672
	1673	set_page_reservations(true);
	1674
	1675	ret = create_user_tracefs();
	1676
	1677	if (ret) {
	1678	pr_warn("user_events could not register with tracefs\n");
	1679	set_page_reservations(false);
	1680	__free_pages(pages, MAX_PAGE_ORDER);
	1681	return ret;
	1682	}
	1683
	1684	if (dyn_event_register(&user_event_dops))
	1685	pr_warn("user_events could not register with dyn_events\n");
	1686
	1687	return 0;
	1688	}
	1689
	1690	fs_initcall(trace_events_user_init);

kernel/trace/trace_sched_switch.c+1−1

@@ -45,7 +45,7 @@ probe_sched_wakeup(void ignore, struct task_struct wakee) · 44 unmodified lines
45		45
46	if (!flags)	46	if (!flags)
47	return;	47	return;
48	tracing_record_taskinfo(current, flags);	48	tracing_record_taskinfo_sched_switch(current, wakee, flags);
49	}	49	}
50		50
51	static int tracing_sched_register(void)	51	static int tracing_sched_register(void)

samples/Kconfig+7−1

 	tristate "Build trace_events examples -- loadable modules only"
 	depends on EVENT_TRACING && m
 	help
-	  This build trace event example modules.
+	  This builds the trace event example module.
+config SAMPLE_TRACE_CUSTOM_EVENTS
+	tristate "Build custom trace event example -- loadable modules only"
+	depends on EVENT_TRACING && m
+	help
+	  This builds the custom trace event example module.
 config SAMPLE_TRACE_PRINTK
         tristate "Build trace_printk module - tests various trace_printk formats"

samples/Makefile+1−0

@@ -20,6 +20,7 @@ obj-$(CONFIG_SAMPLE_RPMSG_CLIENT) += rpmsg/ · 19 unmodified lines
20	subdir-$(CONFIG_SAMPLE_SECCOMP) += seccomp	20	subdir-$(CONFIG_SAMPLE_SECCOMP) += seccomp
21	subdir-$(CONFIG_SAMPLE_TIMER) += timers	21	subdir-$(CONFIG_SAMPLE_TIMER) += timers
22	obj-$(CONFIG_SAMPLE_TRACE_EVENTS) += trace_events/	22	obj-$(CONFIG_SAMPLE_TRACE_EVENTS) += trace_events/
		23	obj-$(CONFIG_SAMPLE_TRACE_CUSTOM_EVENTS) += trace_events/
23	obj-$(CONFIG_SAMPLE_TRACE_PRINTK) += trace_printk/	24	obj-$(CONFIG_SAMPLE_TRACE_PRINTK) += trace_printk/
24	obj-$(CONFIG_SAMPLE_FTRACE_DIRECT) += ftrace/	25	obj-$(CONFIG_SAMPLE_FTRACE_DIRECT) += ftrace/
25	obj-$(CONFIG_SAMPLE_FTRACE_DIRECT_MULTI) += ftrace/	26	obj-$(CONFIG_SAMPLE_FTRACE_DIRECT_MULTI) += ftrace/

samples/trace_events/Makefile+2−0

@@ -11,5 +11,7 @@ · 10 unmodified lines
11	# Here trace-events-sample.c does the CREATE_TRACE_POINTS.	11	# Here trace-events-sample.c does the CREATE_TRACE_POINTS.
12	#	12	#
13	CFLAGS_trace-events-sample.o := -I$(src)	13	CFLAGS_trace-events-sample.o := -I$(src)
		14	CFLAGS_trace_custom_sched.o := -I$(src)
14		15
15	obj-$(CONFIG_SAMPLE_TRACE_EVENTS) += trace-events-sample.o	16	obj-$(CONFIG_SAMPLE_TRACE_EVENTS) += trace-events-sample.o
		17	obj-$(CONFIG_SAMPLE_TRACE_CUSTOM_EVENTS) += trace_custom_sched.o

samples/trace_events/trace_custom_sched.c+60−0

@@ -0,0 +1,60 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* event tracer
	4	*
	5	* Copyright (C) 2022 Google Inc, Steven Rostedt <rostedt@goodmis.org>
	6	*/
	7
	8	#define pr_fmt(fmt) fmt
	9
	10	#include <linux/trace_events.h>
	11	#include <linux/version.h>
	12	#include <linux/module.h>
	13	#include <linux/sched.h>
	14
	15	/*
	16	* Must include the event header that the custom event will attach to,
	17	* from the C file, and not in the custom header file.
	18	*/
	19	#include <trace/events/sched.h>
	20
	21	/* Declare CREATE_CUSTOM_TRACE_EVENTS before including custom header */
	22	#define CREATE_CUSTOM_TRACE_EVENTS
	23
	24	#include "trace_custom_sched.h"
	25
	26	/*
	27	* As the trace events are not exported to modules, the use of
	28	* for_each_kernel_tracepoint() is needed to find the trace event
	29	* to attach to. The fct() function below, is a callback that
	30	* will be called for every event.
	31	*
	32	* Helper functions are created by the TRACE_CUSTOM_EVENT() macro
	33	* update the event. Those are of the form:
	34	*
	35	* trace_custom_event_<event>_update()
	36	*
	37	* Where <event> is the event to attach.
	38	*/
	39	static void fct(struct tracepoint tp, void priv)
	40	{
	41	trace_custom_event_sched_switch_update(tp);
	42	trace_custom_event_sched_waking_update(tp);
	43	}
	44
	45	static int __init trace_sched_init(void)
	46	{
	47	for_each_kernel_tracepoint(fct, NULL);
	48	return 0;
	49	}
	50
	51	static void __exit trace_sched_exit(void)
	52	{
	53	}
	54
	55	module_init(trace_sched_init);
	56	module_exit(trace_sched_exit);
	57
	58	MODULE_AUTHOR("Steven Rostedt");
	59	MODULE_DESCRIPTION("Custom scheduling events");
	60	MODULE_LICENSE("GPL");

samples/trace_events/trace_custom_sched.h+96−0

@@ -0,0 +1,96 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2
	3	/*
	4	* Like the headers that use TRACE_EVENT(), the TRACE_CUSTOM_EVENT()
	5	* needs a header that allows for multiple inclusions.
	6	*
	7	* Test for a unique name (here we have _TRACE_CUSTOM_SCHED_H),
	8	* also allowing to continue if TRACE_CUSTOM_MULTI_READ is defined.
	9	*/
	10	#if !defined(_TRACE_CUSTOM_SCHED_H) \|\| defined(TRACE_CUSTOM_MULTI_READ)
	11	#define _TRACE_CUSTOM_SCHED_H
	12
	13	/* Include linux/trace_events.h for initial defines of TRACE_CUSTOM_EVENT() */
	14	#include <linux/trace_events.h>
	15
	16	/*
	17	* TRACE_CUSTOM_EVENT() is just like TRACE_EVENT(). The first parameter
	18	* is the event name of an existing event where the TRACE_EVENT has been included
	19	* in the C file before including this file.
	20	*/
	21	TRACE_CUSTOM_EVENT(sched_switch,
	22
	23	/*
	24	* The TP_PROTO() and TP_ARGS must match the trace event
	25	* that the custom event is using.
	26	*/
	27	TP_PROTO(bool preempt,
	28	unsigned int prev_state,
	29	struct task_struct *prev,
	30	struct task_struct *next),
	31
	32	TP_ARGS(preempt, prev_state, prev, next),
	33
	34	/*
	35	* The next fields are where the customization happens.
	36	* The TP_STRUCT__entry() defines what will be recorded
	37	* in the ring buffer when the custom event triggers.
	38	*
	39	* The rest is just like the TRACE_EVENT() macro except that
	40	* it uses the custom entry.
	41	*/
	42	TP_STRUCT__entry(
	43	__field( unsigned short, prev_prio )
	44	__field( unsigned short, next_prio )
	45	__field( pid_t, next_pid )
	46	),
	47
	48	TP_fast_assign(
	49	__entry->prev_prio = prev->prio;
	50	__entry->next_pid = next->pid;
	51	__entry->next_prio = next->prio;
	52	),
	53
	54	TP_printk("prev_prio=%d next_pid=%d next_prio=%d",
	55	__entry->prev_prio, __entry->next_pid, __entry->next_prio)
	56	)
	57
	58
	59	TRACE_CUSTOM_EVENT(sched_waking,
	60
	61	TP_PROTO(struct task_struct *p),
	62
	63	TP_ARGS(p),
	64
	65	TP_STRUCT__entry(
	66	__field( pid_t, pid )
	67	__field( unsigned short, prio )
	68	),
	69
	70	TP_fast_assign(
	71	__entry->pid = p->pid;
	72	__entry->prio = p->prio;
	73	),
	74
	75	TP_printk("pid=%d prio=%d", __entry->pid, __entry->prio)
	76	)
	77	#endif
	78	/*
	79	* Just like the headers that create TRACE_EVENTs, the below must
	80	* be outside the protection of the above #if block.
	81	*/
	82
	83	/*
	84	* It is required that the Makefile includes:
	85	* CFLAGS_<c_file>.o := -I$(src)
	86	*/
	87	#undef TRACE_INCLUDE_PATH
	88	#undef TRACE_INCLUDE_FILE
	89	#define TRACE_INCLUDE_PATH .
	90
	91	/*
	92	* It is requred that the TRACE_INCLUDE_FILE be the same
	93	* as this file without the ".h".
	94	*/
	95	#define TRACE_INCLUDE_FILE trace_custom_sched
	96	#include <trace/define_custom_trace.h>

samples/user_events/Makefile+5−0

@@ -0,0 +1,5 @@
	1	# SPDX-License-Identifier: GPL-2.0
	2	CFLAGS += -Wl,-no-as-needed -Wall -I../../usr/include
	3
	4	example: example.o
	5	example.o: example.c

samples/user_events/example.c+91−0

@@ -0,0 +1,91 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* Copyright (c) 2021, Microsoft Corporation.
	4	*
	5	* Authors:
	6	* Beau Belgrave <beaub@linux.microsoft.com>
	7	*/
	8
	9	#include <errno.h>
	10	#include <sys/ioctl.h>
	11	#include <sys/mman.h>
	12	#include <fcntl.h>
	13	#include <stdio.h>
	14	#include <unistd.h>
	15	#include <linux/user_events.h>
	16
	17	/* Assumes debugfs is mounted */
	18	const char *data_file = "/sys/kernel/debug/tracing/user_events_data";
	19	const char *status_file = "/sys/kernel/debug/tracing/user_events_status";
	20
	21	static int event_status(char **status)
	22	{
	23	int fd = open(status_file, O_RDONLY);
	24
	25	*status = mmap(NULL, sysconf(_SC_PAGESIZE), PROT_READ,
	26	MAP_SHARED, fd, 0);
	27
	28	close(fd);
	29
	30	if (*status == MAP_FAILED)
	31	return -1;
	32
	33	return 0;
	34	}
	35
	36	static int event_reg(int fd, const char command, int status, int *write)
	37	{
	38	struct user_reg reg = {0};
	39
	40	reg.size = sizeof(reg);
	41	reg.name_args = (__u64)command;
	42
	43	if (ioctl(fd, DIAG_IOCSREG, &reg) == -1)
	44	return -1;
	45
	46	*status = reg.status_index;
	47	*write = reg.write_index;
	48
	49	return 0;
	50	}
	51
	52	int main(int argc, char **argv)
	53	{
	54	int data_fd, status, write;
	55	char *status_page;
	56	struct iovec io[2];
	57	__u32 count = 0;
	58
	59	if (event_status(&status_page) == -1)
	60	return errno;
	61
	62	data_fd = open(data_file, O_RDWR);
	63
	64	if (event_reg(data_fd, "test u32 count", &status, &write) == -1)
	65	return errno;
	66
	67	/* Setup iovec */
	68	io[0].iov_base = &write;
	69	io[0].iov_len = sizeof(write);
	70	io[1].iov_base = &count;
	71	io[1].iov_len = sizeof(count);
	72
	73	ask:
	74	printf("Press enter to check status...\n");
	75	getchar();
	76
	77	/* Check if anyone is listening */
	78	if (status_page[status]) {
	79	/* Yep, trace out our data */
	80	writev(data_fd, (const struct iovec *)io, 2);
	81
	82	/* Increase the count */
	83	count++;
	84
	85	printf("Something was attached, wrote data\n");
	86	}
	87
	88	goto ask;
	89
	90	return 0;
	91	}

tools/testing/selftests/user_events/Makefile+9−0

@@ -0,0 +1,9 @@
	1	# SPDX-License-Identifier: GPL-2.0
	2	CFLAGS += -Wl,-no-as-needed -Wall -I../../../../usr/include
	3	LDLIBS += -lrt -lpthread -lm
	4
	5	TEST_GEN_PROGS = ftrace_test dyn_test perf_test
	6
	7	TEST_FILES := settings
	8
	9	include ../lib.mk

tools/testing/selftests/user_events/dyn_test.c+130−0

@@ -0,0 +1,130 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* User Events Dyn Events Test Program
	4	*
	5	* Copyright (c) 2021 Beau Belgrave <beaub@linux.microsoft.com>
	6	*/
	7
	8	#include <errno.h>
	9	#include <linux/user_events.h>
	10	#include <stdio.h>
	11	#include <stdlib.h>
	12	#include <fcntl.h>
	13	#include <sys/ioctl.h>
	14	#include <sys/stat.h>
	15	#include <unistd.h>
	16
	17	#include "../kselftest_harness.h"
	18
	19	const char *dyn_file = "/sys/kernel/debug/tracing/dynamic_events";
	20	const char *clear = "!u:__test_event";
	21
	22	static int Append(const char *value)
	23	{
	24	int fd = open(dyn_file, O_RDWR \| O_APPEND);
	25	int ret = write(fd, value, strlen(value));
	26
	27	close(fd);
	28	return ret;
	29	}
	30
	31	#define CLEAR() \
	32	do { \
	33	int ret = Append(clear); \
	34	if (ret == -1) \
	35	ASSERT_EQ(ENOENT, errno); \
	36	} while (0)
	37
	38	#define TEST_PARSE(x) \
	39	do { \
	40	ASSERT_NE(-1, Append(x)); \
	41	CLEAR(); \
	42	} while (0)
	43
	44	#define TEST_NPARSE(x) ASSERT_EQ(-1, Append(x))
	45
	46	FIXTURE(user) {
	47	};
	48
	49	FIXTURE_SETUP(user) {
	50	CLEAR();
	51	}
	52
	53	FIXTURE_TEARDOWN(user) {
	54	CLEAR();
	55	}
	56
	57	TEST_F(user, basic_types) {
	58	/* All should work */
	59	TEST_PARSE("u:__test_event u64 a");
	60	TEST_PARSE("u:__test_event u32 a");
	61	TEST_PARSE("u:__test_event u16 a");
	62	TEST_PARSE("u:__test_event u8 a");
	63	TEST_PARSE("u:__test_event char a");
	64	TEST_PARSE("u:__test_event unsigned char a");
	65	TEST_PARSE("u:__test_event int a");
	66	TEST_PARSE("u:__test_event unsigned int a");
	67	TEST_PARSE("u:__test_event short a");
	68	TEST_PARSE("u:__test_event unsigned short a");
	69	TEST_PARSE("u:__test_event char[20] a");
	70	TEST_PARSE("u:__test_event unsigned char[20] a");
	71	TEST_PARSE("u:__test_event char[0x14] a");
	72	TEST_PARSE("u:__test_event unsigned char[0x14] a");
	73	/* Bad size format should fail */
	74	TEST_NPARSE("u:__test_event char[aa] a");
	75	/* Large size should fail */
	76	TEST_NPARSE("u:__test_event char[9999] a");
	77	/* Long size string should fail */
	78	TEST_NPARSE("u:__test_event char[0x0000000000001] a");
	79	}
	80
	81	TEST_F(user, loc_types) {
	82	/* All should work */
	83	TEST_PARSE("u:__test_event __data_loc char[] a");
	84	TEST_PARSE("u:__test_event __data_loc unsigned char[] a");
	85	TEST_PARSE("u:__test_event __rel_loc char[] a");
	86	TEST_PARSE("u:__test_event __rel_loc unsigned char[] a");
	87	}
	88
	89	TEST_F(user, size_types) {
	90	/* Should work */
	91	TEST_PARSE("u:__test_event struct custom a 20");
	92	/* Size not specified on struct should fail */
	93	TEST_NPARSE("u:__test_event struct custom a");
	94	/* Size specified on non-struct should fail */
	95	TEST_NPARSE("u:__test_event char a 20");
	96	}
	97
	98	TEST_F(user, flags) {
	99	/* Should work */
	100	TEST_PARSE("u:__test_event:BPF_ITER u32 a");
	101	/* Forward compat */
	102	TEST_PARSE("u:__test_event:BPF_ITER,FLAG_FUTURE u32 a");
	103	}
	104
	105	TEST_F(user, matching) {
	106	/* Register */
	107	ASSERT_NE(-1, Append("u:__test_event struct custom a 20"));
	108	/* Should not match */
	109	TEST_NPARSE("!u:__test_event struct custom b");
	110	/* Should match */
	111	TEST_PARSE("!u:__test_event struct custom a");
	112	/* Multi field reg */
	113	ASSERT_NE(-1, Append("u:__test_event u32 a; u32 b"));
	114	/* Non matching cases */
	115	TEST_NPARSE("!u:__test_event u32 a");
	116	TEST_NPARSE("!u:__test_event u32 b");
	117	TEST_NPARSE("!u:__test_event u32 a; u32 ");
	118	TEST_NPARSE("!u:__test_event u32 a; u32 a");
	119	/* Matching case */
	120	TEST_PARSE("!u:__test_event u32 a; u32 b");
	121	/* Register */
	122	ASSERT_NE(-1, Append("u:__test_event u32 a; u32 b"));
	123	/* Ensure trailing semi-colon case */
	124	TEST_PARSE("!u:__test_event u32 a; u32 b;");
	125	}
	126
	127	int main(int argc, char **argv)
	128	{
	129	return test_harness_run(argc, argv);
	130	}

tools/testing/selftests/user_events/ftrace_test.c+452−0

@@ -0,0 +1,452 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* User Events FTrace Test Program
	4	*
	5	* Copyright (c) 2021 Beau Belgrave <beaub@linux.microsoft.com>
	6	*/
	7
	8	#include <errno.h>
	9	#include <linux/user_events.h>
	10	#include <stdio.h>
	11	#include <stdlib.h>
	12	#include <fcntl.h>
	13	#include <sys/ioctl.h>
	14	#include <sys/stat.h>
	15	#include <unistd.h>
	16
	17	#include "../kselftest_harness.h"
	18
	19	const char *data_file = "/sys/kernel/debug/tracing/user_events_data";
	20	const char *status_file = "/sys/kernel/debug/tracing/user_events_status";
	21	const char *enable_file = "/sys/kernel/debug/tracing/events/user_events/__test_event/enable";
	22	const char *trace_file = "/sys/kernel/debug/tracing/trace";
	23	const char *fmt_file = "/sys/kernel/debug/tracing/events/user_events/__test_event/format";
	24
	25	static int trace_bytes(void)
	26	{
	27	int fd = open(trace_file, O_RDONLY);
	28	char buf[256];
	29	int bytes = 0, got;
	30
	31	if (fd == -1)
	32	return -1;
	33
	34	while (true) {
	35	got = read(fd, buf, sizeof(buf));
	36
	37	if (got == -1)
	38	return -1;
	39
	40	if (got == 0)
	41	break;
	42
	43	bytes += got;
	44	}
	45
	46	close(fd);
	47
	48	return bytes;
	49	}
	50
	51	static int skip_until_empty_line(FILE *fp)
	52	{
	53	int c, last = 0;
	54
	55	while (true) {
	56	c = getc(fp);
	57
	58	if (c == EOF)
	59	break;
	60
	61	if (last == '\n' && c == '\n')
	62	return 0;
	63
	64	last = c;
	65	}
	66
	67	return -1;
	68	}
	69
	70	static int get_print_fmt(char *buffer, int len)
	71	{
	72	FILE *fp = fopen(fmt_file, "r");
	73	char *newline;
	74
	75	if (!fp)
	76	return -1;
	77
	78	/* Read until empty line (Skip Common) */
	79	if (skip_until_empty_line(fp) < 0)
	80	goto err;
	81
	82	/* Read until empty line (Skip Properties) */
	83	if (skip_until_empty_line(fp) < 0)
	84	goto err;
	85
	86	/* Read in print_fmt: */
	87	if (fgets(buffer, len, fp) == NULL)
	88	goto err;
	89
	90	newline = strchr(buffer, '\n');
	91
	92	if (newline)
	93	*newline = '\0';
	94
	95	fclose(fp);
	96
	97	return 0;
	98	err:
	99	fclose(fp);
	100
	101	return -1;
	102	}
	103
	104	static int clear(void)
	105	{
	106	int fd = open(data_file, O_RDWR);
	107
	108	if (fd == -1)
	109	return -1;
	110
	111	if (ioctl(fd, DIAG_IOCSDEL, "__test_event") == -1)
	112	if (errno != ENOENT)
	113	return -1;
	114
	115	close(fd);
	116
	117	return 0;
	118	}
	119
	120	static int check_print_fmt(const char event, const char expected)
	121	{
	122	struct user_reg reg = {0};
	123	char print_fmt[256];
	124	int ret;
	125	int fd;
	126
	127	/* Ensure cleared */
	128	ret = clear();
	129
	130	if (ret != 0)
	131	return ret;
	132
	133	fd = open(data_file, O_RDWR);
	134
	135	if (fd == -1)
	136	return fd;
	137
	138	reg.size = sizeof(reg);
	139	reg.name_args = (__u64)event;
	140
	141	/* Register should work */
	142	ret = ioctl(fd, DIAG_IOCSREG, &reg);
	143
	144	close(fd);
	145
	146	if (ret != 0)
	147	return ret;
	148
	149	/* Ensure correct print_fmt */
	150	ret = get_print_fmt(print_fmt, sizeof(print_fmt));
	151
	152	if (ret != 0)
	153	return ret;
	154
	155	return strcmp(print_fmt, expected);
	156	}
	157
	158	FIXTURE(user) {
	159	int status_fd;
	160	int data_fd;
	161	int enable_fd;
	162	};
	163
	164	FIXTURE_SETUP(user) {
	165	self->status_fd = open(status_file, O_RDONLY);
	166	ASSERT_NE(-1, self->status_fd);
	167
	168	self->data_fd = open(data_file, O_RDWR);
	169	ASSERT_NE(-1, self->data_fd);
	170
	171	self->enable_fd = -1;
	172	}
	173
	174	FIXTURE_TEARDOWN(user) {
	175	close(self->status_fd);
	176	close(self->data_fd);
	177
	178	if (self->enable_fd != -1) {
	179	write(self->enable_fd, "0", sizeof("0"));
	180	close(self->enable_fd);
	181	}
	182
	183	ASSERT_EQ(0, clear());
	184	}
	185
	186	TEST_F(user, register_events) {
	187	struct user_reg reg = {0};
	188	int page_size = sysconf(_SC_PAGESIZE);
	189	char *status_page;
	190
	191	reg.size = sizeof(reg);
	192	reg.name_args = (__u64)"__test_event u32 field1; u32 field2";
	193
	194	status_page = mmap(NULL, page_size, PROT_READ, MAP_SHARED,
	195	self->status_fd, 0);
	196
	197	/* Register should work */
	198	ASSERT_EQ(0, ioctl(self->data_fd, DIAG_IOCSREG, &reg));
	199	ASSERT_EQ(0, reg.write_index);
	200	ASSERT_NE(0, reg.status_index);
	201
	202	/* Multiple registers should result in same index */
	203	ASSERT_EQ(0, ioctl(self->data_fd, DIAG_IOCSREG, &reg));
	204	ASSERT_EQ(0, reg.write_index);
	205	ASSERT_NE(0, reg.status_index);
	206
	207	/* Ensure disabled */
	208	self->enable_fd = open(enable_file, O_RDWR);
	209	ASSERT_NE(-1, self->enable_fd);
	210	ASSERT_NE(-1, write(self->enable_fd, "0", sizeof("0")))
	211
	212	/* MMAP should work and be zero'd */
	213	ASSERT_NE(MAP_FAILED, status_page);
	214	ASSERT_NE(NULL, status_page);
	215	ASSERT_EQ(0, status_page[reg.status_index]);
	216
	217	/* Enable event and ensure bits updated in status */
	218	ASSERT_NE(-1, write(self->enable_fd, "1", sizeof("1")))
	219	ASSERT_EQ(EVENT_STATUS_FTRACE, status_page[reg.status_index]);
	220
	221	/* Disable event and ensure bits updated in status */
	222	ASSERT_NE(-1, write(self->enable_fd, "0", sizeof("0")))
	223	ASSERT_EQ(0, status_page[reg.status_index]);
	224
	225	/* File still open should return -EBUSY for delete */
	226	ASSERT_EQ(-1, ioctl(self->data_fd, DIAG_IOCSDEL, "__test_event"));
	227	ASSERT_EQ(EBUSY, errno);
	228
	229	/* Delete should work only after close */
	230	close(self->data_fd);
	231	self->data_fd = open(data_file, O_RDWR);
	232	ASSERT_EQ(0, ioctl(self->data_fd, DIAG_IOCSDEL, "__test_event"));
	233
	234	/* Unmap should work */
	235	ASSERT_EQ(0, munmap(status_page, page_size));
	236	}
	237
	238	TEST_F(user, write_events) {
	239	struct user_reg reg = {0};
	240	struct iovec io[3];
	241	__u32 field1, field2;
	242	int before = 0, after = 0;
	243
	244	reg.size = sizeof(reg);
	245	reg.name_args = (__u64)"__test_event u32 field1; u32 field2";
	246
	247	field1 = 1;
	248	field2 = 2;
	249
	250	io[0].iov_base = &reg.write_index;
	251	io[0].iov_len = sizeof(reg.write_index);
	252	io[1].iov_base = &field1;
	253	io[1].iov_len = sizeof(field1);
	254	io[2].iov_base = &field2;
	255	io[2].iov_len = sizeof(field2);
	256
	257	/* Register should work */
	258	ASSERT_EQ(0, ioctl(self->data_fd, DIAG_IOCSREG, &reg));
	259	ASSERT_EQ(0, reg.write_index);
	260	ASSERT_NE(0, reg.status_index);
	261
	262	/* Write should fail on invalid slot with ENOENT */
	263	io[0].iov_base = &field2;
	264	io[0].iov_len = sizeof(field2);
	265	ASSERT_EQ(-1, writev(self->data_fd, (const struct iovec *)io, 3));
	266	ASSERT_EQ(ENOENT, errno);
	267	io[0].iov_base = &reg.write_index;
	268	io[0].iov_len = sizeof(reg.write_index);
	269
	270	/* Enable event */
	271	self->enable_fd = open(enable_file, O_RDWR);
	272	ASSERT_NE(-1, write(self->enable_fd, "1", sizeof("1")))
	273
	274	/* Write should make it out to ftrace buffers */
	275	before = trace_bytes();
	276	ASSERT_NE(-1, writev(self->data_fd, (const struct iovec *)io, 3));
	277	after = trace_bytes();
	278	ASSERT_GT(after, before);
	279	}
	280
	281	TEST_F(user, write_fault) {
	282	struct user_reg reg = {0};
	283	struct iovec io[2];
	284	int l = sizeof(__u64);
	285	void *anon;
	286
	287	reg.size = sizeof(reg);
	288	reg.name_args = (__u64)"__test_event u64 anon";
	289
	290	anon = mmap(NULL, l, PROT_READ, MAP_PRIVATE \| MAP_ANONYMOUS, -1, 0);
	291	ASSERT_NE(MAP_FAILED, anon);
	292
	293	io[0].iov_base = &reg.write_index;
	294	io[0].iov_len = sizeof(reg.write_index);
	295	io[1].iov_base = anon;
	296	io[1].iov_len = l;
	297
	298	/* Register should work */
	299	ASSERT_EQ(0, ioctl(self->data_fd, DIAG_IOCSREG, &reg));
	300	ASSERT_EQ(0, reg.write_index);
	301	ASSERT_NE(0, reg.status_index);
	302
	303	/* Write should work normally */
	304	ASSERT_NE(-1, writev(self->data_fd, (const struct iovec *)io, 2));
	305
	306	/* Faulted data should zero fill and work */
	307	ASSERT_EQ(0, madvise(anon, l, MADV_DONTNEED));
	308	ASSERT_NE(-1, writev(self->data_fd, (const struct iovec *)io, 2));
	309	ASSERT_EQ(0, munmap(anon, l));
	310	}
	311
	312	TEST_F(user, write_validator) {
	313	struct user_reg reg = {0};
	314	struct iovec io[3];
	315	int loc, bytes;
	316	char data[8];
	317	int before = 0, after = 0;
	318
	319	reg.size = sizeof(reg);
	320	reg.name_args = (__u64)"__test_event __rel_loc char[] data";
	321
	322	/* Register should work */
	323	ASSERT_EQ(0, ioctl(self->data_fd, DIAG_IOCSREG, &reg));
	324	ASSERT_EQ(0, reg.write_index);
	325	ASSERT_NE(0, reg.status_index);
	326
	327	io[0].iov_base = &reg.write_index;
	328	io[0].iov_len = sizeof(reg.write_index);
	329	io[1].iov_base = &loc;
	330	io[1].iov_len = sizeof(loc);
	331	io[2].iov_base = data;
	332	bytes = snprintf(data, sizeof(data), "Test") + 1;
	333	io[2].iov_len = bytes;
	334
	335	/* Undersized write should fail */
	336	ASSERT_EQ(-1, writev(self->data_fd, (const struct iovec *)io, 1));
	337	ASSERT_EQ(EINVAL, errno);
	338
	339	/* Enable event */
	340	self->enable_fd = open(enable_file, O_RDWR);
	341	ASSERT_NE(-1, write(self->enable_fd, "1", sizeof("1")))
	342
	343	/* Full in-bounds write should work */
	344	before = trace_bytes();
	345	loc = DYN_LOC(0, bytes);
	346	ASSERT_NE(-1, writev(self->data_fd, (const struct iovec *)io, 3));
	347	after = trace_bytes();
	348	ASSERT_GT(after, before);
	349
	350	/* Out of bounds write should fault (offset way out) */
	351	loc = DYN_LOC(1024, bytes);
	352	ASSERT_EQ(-1, writev(self->data_fd, (const struct iovec *)io, 3));
	353	ASSERT_EQ(EFAULT, errno);
	354
	355	/* Out of bounds write should fault (offset 1 byte out) */
	356	loc = DYN_LOC(1, bytes);
	357	ASSERT_EQ(-1, writev(self->data_fd, (const struct iovec *)io, 3));
	358	ASSERT_EQ(EFAULT, errno);
	359
	360	/* Out of bounds write should fault (size way out) */
	361	loc = DYN_LOC(0, bytes + 1024);
	362	ASSERT_EQ(-1, writev(self->data_fd, (const struct iovec *)io, 3));
	363	ASSERT_EQ(EFAULT, errno);
	364
	365	/* Out of bounds write should fault (size 1 byte out) */
	366	loc = DYN_LOC(0, bytes + 1);
	367	ASSERT_EQ(-1, writev(self->data_fd, (const struct iovec *)io, 3));
	368	ASSERT_EQ(EFAULT, errno);
	369
	370	/* Non-Null should fault */
	371	memset(data, 'A', sizeof(data));
	372	loc = DYN_LOC(0, bytes);
	373	ASSERT_EQ(-1, writev(self->data_fd, (const struct iovec *)io, 3));
	374	ASSERT_EQ(EFAULT, errno);
	375	}
	376
	377	TEST_F(user, print_fmt) {
	378	int ret;
	379
	380	ret = check_print_fmt("__test_event __rel_loc char[] data",
	381	"print fmt: \"data=%s\", __get_rel_str(data)");
	382	ASSERT_EQ(0, ret);
	383
	384	ret = check_print_fmt("__test_event __data_loc char[] data",
	385	"print fmt: \"data=%s\", __get_str(data)");
	386	ASSERT_EQ(0, ret);
	387
	388	ret = check_print_fmt("__test_event s64 data",
	389	"print fmt: \"data=%lld\", REC->data");
	390	ASSERT_EQ(0, ret);
	391
	392	ret = check_print_fmt("__test_event u64 data",
	393	"print fmt: \"data=%llu\", REC->data");
	394	ASSERT_EQ(0, ret);
	395
	396	ret = check_print_fmt("__test_event s32 data",
	397	"print fmt: \"data=%d\", REC->data");
	398	ASSERT_EQ(0, ret);
	399
	400	ret = check_print_fmt("__test_event u32 data",
	401	"print fmt: \"data=%u\", REC->data");
	402	ASSERT_EQ(0, ret);
	403
	404	ret = check_print_fmt("__test_event int data",
	405	"print fmt: \"data=%d\", REC->data");
	406	ASSERT_EQ(0, ret);
	407
	408	ret = check_print_fmt("__test_event unsigned int data",
	409	"print fmt: \"data=%u\", REC->data");
	410	ASSERT_EQ(0, ret);
	411
	412	ret = check_print_fmt("__test_event s16 data",
	413	"print fmt: \"data=%d\", REC->data");
	414	ASSERT_EQ(0, ret);
	415
	416	ret = check_print_fmt("__test_event u16 data",
	417	"print fmt: \"data=%u\", REC->data");
	418	ASSERT_EQ(0, ret);
	419
	420	ret = check_print_fmt("__test_event short data",
	421	"print fmt: \"data=%d\", REC->data");
	422	ASSERT_EQ(0, ret);
	423
	424	ret = check_print_fmt("__test_event unsigned short data",
	425	"print fmt: \"data=%u\", REC->data");
	426	ASSERT_EQ(0, ret);
	427
	428	ret = check_print_fmt("__test_event s8 data",
	429	"print fmt: \"data=%d\", REC->data");
	430	ASSERT_EQ(0, ret);
	431
	432	ret = check_print_fmt("__test_event u8 data",
	433	"print fmt: \"data=%u\", REC->data");
	434	ASSERT_EQ(0, ret);
	435
	436	ret = check_print_fmt("__test_event char data",
	437	"print fmt: \"data=%d\", REC->data");
	438	ASSERT_EQ(0, ret);
	439
	440	ret = check_print_fmt("__test_event unsigned char data",
	441	"print fmt: \"data=%u\", REC->data");
	442	ASSERT_EQ(0, ret);
	443
	444	ret = check_print_fmt("__test_event char[4] data",
	445	"print fmt: \"data=%s\", REC->data");
	446	ASSERT_EQ(0, ret);
	447	}
	448
	449	int main(int argc, char **argv)
	450	{
	451	return test_harness_run(argc, argv);
	452	}

tools/testing/selftests/user_events/perf_test.c+168−0

@@ -0,0 +1,168 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* User Events Perf Events Test Program
	4	*
	5	* Copyright (c) 2021 Beau Belgrave <beaub@linux.microsoft.com>
	6	*/
	7
	8	#include <errno.h>
	9	#include <linux/user_events.h>
	10	#include <linux/perf_event.h>
	11	#include <stdio.h>
	12	#include <stdlib.h>
	13	#include <fcntl.h>
	14	#include <sys/ioctl.h>
	15	#include <sys/stat.h>
	16	#include <unistd.h>
	17	#include <asm/unistd.h>
	18
	19	#include "../kselftest_harness.h"
	20
	21	const char *data_file = "/sys/kernel/debug/tracing/user_events_data";
	22	const char *status_file = "/sys/kernel/debug/tracing/user_events_status";
	23	const char *id_file = "/sys/kernel/debug/tracing/events/user_events/__test_event/id";
	24	const char *fmt_file = "/sys/kernel/debug/tracing/events/user_events/__test_event/format";
	25
	26	struct event {
	27	__u32 index;
	28	__u32 field1;
	29	__u32 field2;
	30	};
	31
	32	static long perf_event_open(struct perf_event_attr *pe, pid_t pid,
	33	int cpu, int group_fd, unsigned long flags)
	34	{
	35	return syscall(__NR_perf_event_open, pe, pid, cpu, group_fd, flags);
	36	}
	37
	38	static int get_id(void)
	39	{
	40	FILE *fp = fopen(id_file, "r");
	41	int ret, id = 0;
	42
	43	if (!fp)
	44	return -1;
	45
	46	ret = fscanf(fp, "%d", &id);
	47	fclose(fp);
	48
	49	if (ret != 1)
	50	return -1;
	51
	52	return id;
	53	}
	54
	55	static int get_offset(void)
	56	{
	57	FILE *fp = fopen(fmt_file, "r");
	58	int ret, c, last = 0, offset = 0;
	59
	60	if (!fp)
	61	return -1;
	62
	63	/* Read until empty line */
	64	while (true) {
	65	c = getc(fp);
	66
	67	if (c == EOF)
	68	break;
	69
	70	if (last == '\n' && c == '\n')
	71	break;
	72
	73	last = c;
	74	}
	75
	76	ret = fscanf(fp, "\tfield:u32 field1;\toffset:%d;", &offset);
	77	fclose(fp);
	78
	79	if (ret != 1)
	80	return -1;
	81
	82	return offset;
	83	}
	84
	85	FIXTURE(user) {
	86	int status_fd;
	87	int data_fd;
	88	};
	89
	90	FIXTURE_SETUP(user) {
	91	self->status_fd = open(status_file, O_RDONLY);
	92	ASSERT_NE(-1, self->status_fd);
	93
	94	self->data_fd = open(data_file, O_RDWR);
	95	ASSERT_NE(-1, self->data_fd);
	96	}
	97
	98	FIXTURE_TEARDOWN(user) {
	99	close(self->status_fd);
	100	close(self->data_fd);
	101	}
	102
	103	TEST_F(user, perf_write) {
	104	struct perf_event_attr pe = {0};
	105	struct user_reg reg = {0};
	106	int page_size = sysconf(_SC_PAGESIZE);
	107	char *status_page;
	108	struct event event;
	109	struct perf_event_mmap_page *perf_page;
	110	int id, fd, offset;
	111	__u32 *val;
	112
	113	reg.size = sizeof(reg);
	114	reg.name_args = (__u64)"__test_event u32 field1; u32 field2";
	115
	116	status_page = mmap(NULL, page_size, PROT_READ, MAP_SHARED,
	117	self->status_fd, 0);
	118	ASSERT_NE(MAP_FAILED, status_page);
	119
	120	/* Register should work */
	121	ASSERT_EQ(0, ioctl(self->data_fd, DIAG_IOCSREG, &reg));
	122	ASSERT_EQ(0, reg.write_index);
	123	ASSERT_NE(0, reg.status_index);
	124	ASSERT_EQ(0, status_page[reg.status_index]);
	125
	126	/* Id should be there */
	127	id = get_id();
	128	ASSERT_NE(-1, id);
	129	offset = get_offset();
	130	ASSERT_NE(-1, offset);
	131
	132	pe.type = PERF_TYPE_TRACEPOINT;
	133	pe.size = sizeof(pe);
	134	pe.config = id;
	135	pe.sample_type = PERF_SAMPLE_RAW;
	136	pe.sample_period = 1;
	137	pe.wakeup_events = 1;
	138
	139	/* Tracepoint attach should work */
	140	fd = perf_event_open(&pe, 0, -1, -1, 0);
	141	ASSERT_NE(-1, fd);
	142
	143	perf_page = mmap(NULL, page_size * 2, PROT_READ, MAP_SHARED, fd, 0);
	144	ASSERT_NE(MAP_FAILED, perf_page);
	145
	146	/* Status should be updated */
	147	ASSERT_EQ(EVENT_STATUS_PERF, status_page[reg.status_index]);
	148
	149	event.index = reg.write_index;
	150	event.field1 = 0xc001;
	151	event.field2 = 0xc01a;
	152
	153	/* Ensure write shows up at correct offset */
	154	ASSERT_NE(-1, write(self->data_fd, &event, sizeof(event)));
	155	val = (void )(((char )perf_page) + perf_page->data_offset);
	156	ASSERT_EQ(PERF_RECORD_SAMPLE, *val);
	157	/* Skip over header and size, move to offset */
	158	val += 3;
	159	val = (void )((char )val) + offset;
	160	/* Ensure correct */
	161	ASSERT_EQ(event.field1, *val++);
	162	ASSERT_EQ(event.field2, *val++);
	163	}
	164
	165	int main(int argc, char **argv)
	166	{
	167	return test_harness_run(argc, argv);
	168	}

tools/testing/selftests/user_events/settings+1−0

@@ -0,0 +1 @@
		1	timeout=90