History

Merge tag 'trace-v6.1' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace

Pull tracing updates from Steven Rostedt: "Major changes: - Changed location of tracing repo from personal git repo to: git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git - Added Masami Hiramatsu as co-maintainer - Updated MAINTAINERS file to separate out FTRACE as it is more than just TRACING. Minor changes: - Added Mark Rutland as FTRACE reviewer - Updated user_events to make it on its way to remove the BROKEN tag. The changes should now be acceptable but will run it through a cycle and hopefully we can remove the BROKEN tag next release. - Added filtering to eprobes - Added a delta time to the benchmark trace event - Have the histogram and filter callbacks called via a switch statement instead of indirect functions. This speeds it up to avoid retpolines. - Add a way to wake up ring buffer waiters waiting for the ring buffer to fill up to its watermark. - New ioctl() on the trace_pipe_raw file to wake up ring buffer waiters. - Wake up waiters when the ring buffer is disabled. A reader may block when the ring buffer is disabled, but if it was blocked when the ring buffer is disabled it should then wake up. Fixes: - Allow splice to read partially read ring buffer pages. This fixes splice never moving forward. - Fix inverted compare that made the "shortest" ring buffer wait queue actually the longest. - Fix a race in the ring buffer between resetting a page when a writer goes to another page, and the reader. - Fix ftrace accounting bug when function hooks are added at boot up before the weak functions are set to "disabled". - Fix bug that freed a user allocated snapshot buffer when enabling a tracer. - Fix possible recursive locks in osnoise tracer - Fix recursive locking direct functions - Other minor clean ups and fixes" * tag 'trace-v6.1' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace: (44 commits) ftrace: Create separate entry in MAINTAINERS for function hooks tracing: Update MAINTAINERS to reflect new tracing git repo tracing: Do not free snapshot if tracer is on cmdline ftrace: Still disable enabled records marked as disabled tracing/user_events: Move pages/locks into groups to prepare for namespaces tracing: Add Masami Hiramatsu as co-maintainer tracing: Remove unused variable 'dups' MAINTAINERS: add myself as a tracing reviewer ring-buffer: Fix race between reset page and reading page tracing/user_events: Update ABI documentation to align to bits vs bytes tracing/user_events: Use bits vs bytes for enabled status page data tracing/user_events: Use refcount instead of atomic for ref tracking tracing/user_events: Ensure user provided strings are safely formatted tracing/user_events: Use WRITE instead of READ for io vector import tracing/user_events: Use NULL for strstr checks tracing: Fix spelling mistake "preapre" -> "prepare" tracing: Wake up waiters when tracing is disabled tracing: Add ioctl() to force ring buffer waiters to wake up tracing: Wake up ring buffer waiters on closing of the file ring-buffer: Add ring_buffer_wake_waiters() ...

Linus Torvalds committed 4 years ago

Commit cdf072a2 parents dc55342 4f881a634 files · +1299 −486Browse files

Unified · Split

34 files
  1. Documentation/trace/user_events.rst +58 −28
  2. MAINTAINERS +18 −8
  3. arch/x86/include/asm/ftrace.h +0 −1
  4. arch/x86/include/asm/kprobes.h +0 −2
  5. arch/x86/kernel/kprobes/core.c +0 −2
  6. include/linux/ftrace.h +0 −41
  7. include/linux/ring_buffer.h +1 −1
  8. include/linux/sched.h +0 −3
  9. include/linux/trace_events.h +1 −0
  10. include/linux/user_events.h +3 −12
  11. kernel/trace/ftrace.c +27 −7
  12. kernel/trace/kprobe_event_gen_test.c +44 −5
  13. kernel/trace/ring_buffer.c +84 −3
  14. kernel/trace/rv/monitors/wip/wip.c +4 −4
  15. kernel/trace/rv/monitors/wwnr/wwnr.c +4 −4
  16. kernel/trace/trace.c +73 −5
  17. kernel/trace/trace.h +0 −13
  18. kernel/trace/trace_benchmark.c +1 −1
  19. kernel/trace/trace_benchmark.h +5 −3
  20. kernel/trace/trace_eprobe.c +97 −10
  21. kernel/trace/trace_events_filter.c +168 −71
  22. kernel/trace/trace_events_hist.c +169 −77
  23. kernel/trace/trace_events_user.c +426 −142
  24. kernel/trace/trace_osnoise.c +2 −1
  25. kernel/trace/trace_probe.h +2 −1
  26. kernel/trace/tracing_map.c +2 −3
  27. kernel/tracepoint.c +6 −8
  28. samples/user_events/example.c +18 −7
  29. tools/testing/selftests/ftrace/test.d/dynevent/eprobes_syntax_errors.tc +27 −0
  30. tools/testing/selftests/user_events/ftrace_test.c +39 −8
  31. tools/testing/selftests/user_events/perf_test.c +8 −3
  32. tools/verification/dot2/dot2k_templates/main_global.c +4 −4
  33. tools/verification/dot2/dot2k_templates/main_per_cpu.c +4 −4
  34. tools/verification/dot2/dot2k_templates/main_per_task.c +4 −4
Documentation/trace/user_events.rst+58−28
@@ -20,14 +20,14 @@ dynamic_events is the same as the ioctl with the u: prefix applied. · 19 unmodified lines
2020
2121 Typically programs will register a set of events that they wish to expose to
2222 tools that can read trace_events (such as ftrace and perf). The registration
23−process gives back two ints to the program for each event. The first int is the
24−status index. This index describes which byte in the
23+process gives back two ints to the program for each event. The first int is
24+the status bit. This describes which bit in little-endian format in the
2525 /sys/kernel/debug/tracing/user_events_status file represents this event. The
26−second int is the write index. This index describes the data when a write() or
26+second int is the write index which describes the data when a write() or
2727 writev() is called on the /sys/kernel/debug/tracing/user_events_data file.
2828
29−The structures referenced in this document are contained with the
30−/include/uap/linux/user_events.h file in the source tree.
29+The structures referenced in this document are contained within the
30+/include/uapi/linux/user_events.h file in the source tree.
3131
3232 **NOTE:** *Both user_events_status and user_events_data are under the tracefs
3333 filesystem and may be mounted at different paths than above.*
@@ -38,18 +38,18 @@ Registering within a user process is done via ioctl() out to the · 4 unmodified lines
3838 /sys/kernel/debug/tracing/user_events_data file. The command to issue is
3939 DIAG_IOCSREG.
4040
41−This command takes a struct user_reg as an argument::
41+This command takes a packed struct user_reg as an argument::
4242
4343 struct user_reg {
4444 u32 size;
4545 u64 name_args;
46− u32 status_index;
46+ u32 status_bit;
4747 u32 write_index;
4848 };
4949
5050 The struct user_reg requires two inputs, the first is the size of the structure
5151 to ensure forward and backward compatibility. The second is the command string
52−to issue for registering. Upon success two outputs are set, the status index
52+to issue for registering. Upon success two outputs are set, the status bit
5353 and the write index.
5454
5555 User based events show up under tracefs like any other event under the
@@ -111,15 +111,56 @@ in realtime. This allows user programs to only incur the cost of the write() or · 55 unmodified lines
111111 writev() calls when something is actively attached to the event.
112112
113113 User programs call mmap() on /sys/kernel/debug/tracing/user_events_status to
114−check the status for each event that is registered. The byte to check in the
115−file is given back after the register ioctl() via user_reg.status_index.
114+check the status for each event that is registered. The bit to check in the
115+file is given back after the register ioctl() via user_reg.status_bit. The bit
116+is always in little-endian format. Programs can check if the bit is set either
117+using a byte-wise index with a mask or a long-wise index with a little-endian
118+mask.
119+
116120 Currently the size of user_events_status is a single page, however, custom
117121 kernel configurations can change this size to allow more user based events. In
118122 all cases the size of the file is a multiple of a page size.
119123
120−For example, if the register ioctl() gives back a status_index of 3 you would
121−check byte 3 of the returned mmap data to see if anything is attached to that
122−event.
124+For example, if the register ioctl() gives back a status_bit of 3 you would
125+check byte 0 (3 / 8) of the returned mmap data and then AND the result with 8
126+(1 << (3 % 8)) to see if anything is attached to that event.
127+
128+A byte-wise index check is performed as follows::
129+
130+ int index, mask;
131+ char *status_page;
132+
133+ index = status_bit / 8;
134+ mask = 1 << (status_bit % 8);
135+
136+ ...
137+
138+ if (status_page[index] & mask) {
139+ /* Enabled */
140+ }
141+
142+A long-wise index check is performed as follows::
143+
144+ #include <asm/bitsperlong.h>
145+ #include <endian.h>
146+
147+ #if __BITS_PER_LONG == 64
148+ #define endian_swap(x) htole64(x)
149+ #else
150+ #define endian_swap(x) htole32(x)
151+ #endif
152+
153+ long index, mask, *status_page;
154+
155+ index = status_bit / __BITS_PER_LONG;
156+ mask = 1L << (status_bit % __BITS_PER_LONG);
157+ mask = endian_swap(mask);
158+
159+ ...
160+
161+ if (status_page[index] & mask) {
162+ /* Enabled */
163+ }
123164
124165 Administrators can easily check the status of all registered events by reading
125166 the user_events_status file directly via a terminal. The output is as follows::
@@ -137,7 +178,7 @@ For example, on a system that has a single event the output looks like this:: · 11 unmodified lines
137178
138179 Active: 1
139180 Busy: 0
140− Max: 4096
181+ Max: 32768
141182
142183 If a user enables the user event via ftrace, the output would change to this::
143184
@@ -145,21 +186,10 @@ If a user enables the user event via ftrace, the output would change to this:: · 1 unmodified line
145186
146187 Active: 1
147188 Busy: 1
148− Max: 4096
149−
150−**NOTE:** *A status index of 0 will never be returned. This allows user
151−programs to have an index that can be used on error cases.*
152−
153−Status Bits
154−^^^^^^^^^^^
155−The byte being checked will be non-zero if anything is attached. Programs can
156−check specific bits in the byte to see what mechanism has been attached.
157−
158−The following values are defined to aid in checking what has been attached:
159−
160−**EVENT_STATUS_FTRACE** - Bit set if ftrace has been attached (Bit 0).
189+ Max: 32768
161190
162−**EVENT_STATUS_PERF** - Bit set if perf has been attached (Bit 1).
191+**NOTE:** *A status bit of 0 will never be returned. This allows user programs
192+to have a bit that can be used on error cases.*
163193
164194 Writing Data
165195 ------------
MAINTAINERS+18−8
@@ -8433,6 +8433,19 @@ L: platform-driver-x86@vger.kernel.org · 8432 unmodified lines
84338433 S: Maintained
84348434 F: drivers/platform/x86/fujitsu-tablet.c
84358435
8436+FUNCTION HOOKS (FTRACE)
8437+M: Steven Rostedt <rostedt@goodmis.org>
8438+M: Masami Hiramatsu <mhiramat@kernel.org>
8439+R: Mark Rutland <mark.rutland@arm.com>
8440+S: Maintained
8441+T: git git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git
8442+F: Documentation/trace/ftrace*
8443+F: kernel/trace/ftrace*
8444+F: kernel/trace/fgraph.c
8445+F: arch/*/*/*/*ftrace*
8446+F: arch/*/*/*ftrace*
8447+F: include/*/ftrace.h
8448+
84368449 FUNGIBLE ETHERNET DRIVERS
84378450 M: Dimitris Michailidis <dmichail@fungible.com>
84388451 L: netdev@vger.kernel.org
@@ -11422,7 +11435,7 @@ M: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com> · 2983 unmodified lines
1142211435 M: "David S. Miller" <davem@davemloft.net>
1142311436 M: Masami Hiramatsu <mhiramat@kernel.org>
1142411437 S: Maintained
11425−T: git git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-trace.git
11438+T: git git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git
1142611439 F: Documentation/trace/kprobes.rst
1142711440 F: include/asm-generic/kprobes.h
1142811441 F: include/linux/kprobes.h
@@ -20771,14 +20784,11 @@ F: drivers/hwmon/pmbus/tps546d24.c · 9342 unmodified lines
2077120784
2077220785 TRACING
2077320786 M: Steven Rostedt <rostedt@goodmis.org>
20774−M: Ingo Molnar <mingo@redhat.com>
20787+M: Masami Hiramatsu <mhiramat@kernel.org>
2077520788 S: Maintained
20776−T: git git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-trace.git
20777−F: Documentation/trace/ftrace.rst
20778−F: arch/*/*/*/*ftrace*
20779−F: arch/*/*/*ftrace*
20789+T: git git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git
20790+F: Documentation/trace/*
2078020791 F: fs/tracefs/
20781−F: include/*/ftrace.h
2078220792 F: include/linux/trace*.h
2078320793 F: include/trace/
2078420794 F: kernel/trace/
@@ -20787,7 +20797,7 @@ F: tools/testing/selftests/ftrace/ · 2 unmodified lines
2078720797
2078820798 TRACING MMIO ACCESSES (MMIOTRACE)
2078920799 M: Steven Rostedt <rostedt@goodmis.org>
20790−M: Ingo Molnar <mingo@kernel.org>
20800+M: Masami Hiramatsu <mhiramat@kernel.org>
2079120801 R: Karol Herbst <karolherbst@gmail.com>
2079220802 R: Pekka Paalanen <ppaalanen@gmail.com>
2079320803 L: linux-kernel@vger.kernel.org
arch/x86/include/asm/ftrace.h+0−1
@@ -23,7 +23,6 @@ · 22 unmodified lines
2323 #define HAVE_FUNCTION_GRAPH_RET_ADDR_PTR
2424
2525 #ifndef __ASSEMBLY__
26−extern atomic_t modifying_ftrace_code;
2726 extern void __fentry__(void);
2827
2928 static inline unsigned long ftrace_call_adjust(unsigned long addr)
arch/x86/include/asm/kprobes.h+0−2
@@ -50,8 +50,6 @@ extern const int kretprobe_blacklist_size; · 49 unmodified lines
5050
5151 void arch_remove_kprobe(struct kprobe *p);
5252
53−extern void arch_kprobe_override_function(struct pt_regs *regs);
54−
5553 /* Architecture specific copy of original instruction*/
5654 struct arch_specific_insn {
5755 /* copy of the original instruction */
arch/x86/kernel/kprobes/core.c+0−2
@@ -59,8 +59,6 @@ · 58 unmodified lines
5959 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
6060 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
6161
62−#define stack_addr(regs) ((unsigned long *)regs->sp)
63−
6462 #define W(row, b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, ba, bb, bc, bd, be, bf)\
6563 (((b0##UL << 0x0)|(b1##UL << 0x1)|(b2##UL << 0x2)|(b3##UL << 0x3) | \
6664 (b4##UL << 0x4)|(b5##UL << 0x5)|(b6##UL << 0x6)|(b7##UL << 0x7) | \
include/linux/ftrace.h+0−41
@@ -1122,47 +1122,6 @@ static inline void unpause_graph_tracing(void) { } · 1121 unmodified lines
11221122 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
11231123
11241124 #ifdef CONFIG_TRACING
1125−
1126−/* flags for current->trace */
1127−enum {
1128− TSK_TRACE_FL_TRACE_BIT = 0,
1129− TSK_TRACE_FL_GRAPH_BIT = 1,
1130−};
1131−enum {
1132− TSK_TRACE_FL_TRACE = 1 << TSK_TRACE_FL_TRACE_BIT,
1133− TSK_TRACE_FL_GRAPH = 1 << TSK_TRACE_FL_GRAPH_BIT,
1134−};
1135−
1136−static inline void set_tsk_trace_trace(struct task_struct *tsk)
1137−{
1138− set_bit(TSK_TRACE_FL_TRACE_BIT, &tsk->trace);
1139−}
1140−
1141−static inline void clear_tsk_trace_trace(struct task_struct *tsk)
1142−{
1143− clear_bit(TSK_TRACE_FL_TRACE_BIT, &tsk->trace);
1144−}
1145−
1146−static inline int test_tsk_trace_trace(struct task_struct *tsk)
1147−{
1148− return tsk->trace & TSK_TRACE_FL_TRACE;
1149−}
1150−
1151−static inline void set_tsk_trace_graph(struct task_struct *tsk)
1152−{
1153− set_bit(TSK_TRACE_FL_GRAPH_BIT, &tsk->trace);
1154−}
1155−
1156−static inline void clear_tsk_trace_graph(struct task_struct *tsk)
1157−{
1158− clear_bit(TSK_TRACE_FL_GRAPH_BIT, &tsk->trace);
1159−}
1160−
1161−static inline int test_tsk_trace_graph(struct task_struct *tsk)
1162−{
1163− return tsk->trace & TSK_TRACE_FL_GRAPH;
1164−}
1165−
11661125 enum ftrace_dump_mode;
11671126
11681127 extern enum ftrace_dump_mode ftrace_dump_on_oops;
include/linux/ring_buffer.h+1−1
@@ -101,7 +101,7 @@ __ring_buffer_alloc(unsigned long size, unsigned flags, struct lock_class_key *k · 100 unmodified lines
101101 int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full);
102102 __poll_t ring_buffer_poll_wait(struct trace_buffer *buffer, int cpu,
103103 struct file *filp, poll_table *poll_table);
104−
104+void ring_buffer_wake_waiters(struct trace_buffer *buffer, int cpu);
105105
106106 #define RING_BUFFER_ALL_CPUS -1
107107
include/linux/sched.h+0−3
@@ -1390,9 +1390,6 @@ struct task_struct { · 1389 unmodified lines
13901390 #endif
13911391
13921392 #ifdef CONFIG_TRACING
1393− /* State flags for use by tracers: */
1394− unsigned long trace;
1395−
13961393 /* Bitmask and counter of trace recursion: */
13971394 unsigned long trace_recursion;
13981395 #endif /* CONFIG_TRACING */
include/linux/trace_events.h+1−0
@@ -92,6 +92,7 @@ struct trace_iterator { · 91 unmodified lines
9292 unsigned int temp_size;
9393 char *fmt; /* modified format holder */
9494 unsigned int fmt_size;
95+ long wait_index;
9596
9697 /* trace_seq for __print_flags() and __print_symbolic() etc. */
9798 struct trace_seq tmp_seq;
include/linux/user_events.h+3−12
@@ -20,15 +20,6 @@ · 19 unmodified lines
2020 #define USER_EVENTS_SYSTEM "user_events"
2121 #define USER_EVENTS_PREFIX "u:"
2222
23−/* Bits 0-6 are for known probe types, Bit 7 is for unknown probes */
24−#define EVENT_BIT_FTRACE 0
25−#define EVENT_BIT_PERF 1
26−#define EVENT_BIT_OTHER 7
27−
28−#define EVENT_STATUS_FTRACE (1 << EVENT_BIT_FTRACE)
29−#define EVENT_STATUS_PERF (1 << EVENT_BIT_PERF)
30−#define EVENT_STATUS_OTHER (1 << EVENT_BIT_OTHER)
31−
3223 /* Create dynamic location entry within a 32-bit value */
3324 #define DYN_LOC(offset, size) ((size) << 16 | (offset))
3425
@@ -45,12 +36,12 @@ struct user_reg { · 10 unmodified lines
4536 /* Input: Pointer to string with event name, description and flags */
4637 __u64 name_args;
4738
48− /* Output: Byte index of the event within the status page */
49− __u32 status_index;
39+ /* Output: Bitwise index of the event within the status page */
40+ __u32 status_bit;
5041
5142 /* Output: Index of the event to use when writing data */
5243 __u32 write_index;
53−};
44+} __attribute__((__packed__));
5445
5546 #define DIAG_IOC_MAGIC '*'
5647
kernel/trace/ftrace.c+27−7
@@ -1644,6 +1644,18 @@ ftrace_find_tramp_ops_any_other(struct dyn_ftrace *rec, struct ftrace_ops *op_ex · 1643 unmodified lines
16441644 static struct ftrace_ops *
16451645 ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, struct ftrace_ops *ops);
16461646
1647+static bool skip_record(struct dyn_ftrace *rec)
1648+{
1649+ /*
1650+ * At boot up, weak functions are set to disable. Function tracing
1651+ * can be enabled before they are, and they still need to be disabled now.
1652+ * If the record is disabled, still continue if it is marked as already
1653+ * enabled (this is needed to keep the accounting working).
1654+ */
1655+ return rec->flags & FTRACE_FL_DISABLED &&
1656+ !(rec->flags & FTRACE_FL_ENABLED);
1657+}
1658+
16471659 static bool __ftrace_hash_rec_update(struct ftrace_ops *ops,
16481660 int filter_hash,
16491661 bool inc)
@@ -1693,7 +1705,7 @@ static bool __ftrace_hash_rec_update(struct ftrace_ops *ops, · 43 unmodified lines
16931705 int in_hash = 0;
16941706 int match = 0;
16951707
1696− if (rec->flags & FTRACE_FL_DISABLED)
1708+ if (skip_record(rec))
16971709 continue;
16981710
16991711 if (all) {
@@ -2126,7 +2138,7 @@ static int ftrace_check_record(struct dyn_ftrace *rec, bool enable, bool update) · 426 unmodified lines
21262138
21272139 ftrace_bug_type = FTRACE_BUG_UNKNOWN;
21282140
2129− if (rec->flags & FTRACE_FL_DISABLED)
2141+ if (skip_record(rec))
21302142 return FTRACE_UPDATE_IGNORE;
21312143
21322144 /*
@@ -2241,7 +2253,7 @@ static int ftrace_check_record(struct dyn_ftrace *rec, bool enable, bool update) · 108 unmodified lines
22412253 if (update) {
22422254 /* If there's no more users, clear all flags */
22432255 if (!ftrace_rec_count(rec))
2244− rec->flags = 0;
2256+ rec->flags &= FTRACE_FL_DISABLED;
22452257 else
22462258 /*
22472259 * Just disable the record, but keep the ops TRAMP
@@ -2634,7 +2646,7 @@ void __weak ftrace_replace_code(int mod_flags) · 386 unmodified lines
26342646
26352647 do_for_each_ftrace_rec(pg, rec) {
26362648
2637− if (rec->flags & FTRACE_FL_DISABLED)
2649+ if (skip_record(rec))
26382650 continue;
26392651
26402652 failed = __ftrace_replace_code(rec, enable);
@@ -5427,6 +5439,8 @@ static struct ftrace_ops stub_ops = { · 2786 unmodified lines
54275439 * it is safe to modify the ftrace record, where it should be
54285440 * currently calling @old_addr directly, to call @new_addr.
54295441 *
5442+ * This is called with direct_mutex locked.
5443+ *
54305444 * Safety checks should be made to make sure that the code at
54315445 * @rec->ip is currently calling @old_addr. And this must
54325446 * also update entry->direct to @new_addr.
@@ -5439,6 +5453,8 @@ int __weak ftrace_modify_direct_caller(struct ftrace_func_entry *entry, · 6 unmodified lines
54395453 unsigned long ip = rec->ip;
54405454 int ret;
54415455
5456+ lockdep_assert_held(&direct_mutex);
5457+
54425458 /*
54435459 * The ftrace_lock was used to determine if the record
54445460 * had more than one registered user to it. If it did,
@@ -5461,7 +5477,7 @@ int __weak ftrace_modify_direct_caller(struct ftrace_func_entry *entry, · 16 unmodified lines
54615477 if (ret)
54625478 goto out_lock;
54635479
5464− ret = register_ftrace_function(&stub_ops);
5480+ ret = register_ftrace_function_nolock(&stub_ops);
54655481 if (ret) {
54665482 ftrace_set_filter_ip(&stub_ops, ip, 1, 0);
54675483 goto out_lock;
@@ -6081,8 +6097,12 @@ int ftrace_regex_release(struct inode *inode, struct file *file) · 613 unmodified lines
60816097
60826098 if (filter_hash) {
60836099 orig_hash = &iter->ops->func_hash->filter_hash;
6084− if (iter->tr && !list_empty(&iter->tr->mod_trace))
6085− iter->hash->flags |= FTRACE_HASH_FL_MOD;
6100+ if (iter->tr) {
6101+ if (list_empty(&iter->tr->mod_trace))
6102+ iter->hash->flags &= ~FTRACE_HASH_FL_MOD;
6103+ else
6104+ iter->hash->flags |= FTRACE_HASH_FL_MOD;
6105+ }
60866106 } else
60876107 orig_hash = &iter->ops->func_hash->notrace_hash;
60886108
kernel/trace/kprobe_event_gen_test.c+44−5
@@ -35,6 +35,45 @@ · 34 unmodified lines
3535 static struct trace_event_file *gen_kprobe_test;
3636 static struct trace_event_file *gen_kretprobe_test;
3737
38+#define KPROBE_GEN_TEST_FUNC "do_sys_open"
39+
40+/* X86 */
41+#if defined(CONFIG_X86_64) || defined(CONFIG_X86_32)
42+#define KPROBE_GEN_TEST_ARG0 "dfd=%ax"
43+#define KPROBE_GEN_TEST_ARG1 "filename=%dx"
44+#define KPROBE_GEN_TEST_ARG2 "flags=%cx"
45+#define KPROBE_GEN_TEST_ARG3 "mode=+4($stack)"
46+
47+/* ARM64 */
48+#elif defined(CONFIG_ARM64)
49+#define KPROBE_GEN_TEST_ARG0 "dfd=%x0"
50+#define KPROBE_GEN_TEST_ARG1 "filename=%x1"
51+#define KPROBE_GEN_TEST_ARG2 "flags=%x2"
52+#define KPROBE_GEN_TEST_ARG3 "mode=%x3"
53+
54+/* ARM */
55+#elif defined(CONFIG_ARM)
56+#define KPROBE_GEN_TEST_ARG0 "dfd=%r0"
57+#define KPROBE_GEN_TEST_ARG1 "filename=%r1"
58+#define KPROBE_GEN_TEST_ARG2 "flags=%r2"
59+#define KPROBE_GEN_TEST_ARG3 "mode=%r3"
60+
61+/* RISCV */
62+#elif defined(CONFIG_RISCV)
63+#define KPROBE_GEN_TEST_ARG0 "dfd=%a0"
64+#define KPROBE_GEN_TEST_ARG1 "filename=%a1"
65+#define KPROBE_GEN_TEST_ARG2 "flags=%a2"
66+#define KPROBE_GEN_TEST_ARG3 "mode=%a3"
67+
68+/* others */
69+#else
70+#define KPROBE_GEN_TEST_ARG0 NULL
71+#define KPROBE_GEN_TEST_ARG1 NULL
72+#define KPROBE_GEN_TEST_ARG2 NULL
73+#define KPROBE_GEN_TEST_ARG3 NULL
74+#endif
75+
76+
3877 /*
3978 * Test to make sure we can create a kprobe event, then add more
4079 * fields.
@@ -58,14 +97,14 @@ static int __init test_gen_kprobe_cmd(void) · 17 unmodified lines
5897 * fields.
5998 */
6099 ret = kprobe_event_gen_cmd_start(&cmd, "gen_kprobe_test",
61− "do_sys_open",
62− "dfd=%ax", "filename=%dx");
100+ KPROBE_GEN_TEST_FUNC,
101+ KPROBE_GEN_TEST_ARG0, KPROBE_GEN_TEST_ARG1);
63102 if (ret)
64103 goto free;
65104
66105 /* Use kprobe_event_add_fields to add the rest of the fields */
67106
68− ret = kprobe_event_add_fields(&cmd, "flags=%cx", "mode=+4($stack)");
107+ ret = kprobe_event_add_fields(&cmd, KPROBE_GEN_TEST_ARG2, KPROBE_GEN_TEST_ARG3);
69108 if (ret)
70109 goto free;
71110
@@ -128,7 +167,7 @@ static int __init test_gen_kretprobe_cmd(void) · 56 unmodified lines
128167 * Define the kretprobe event.
129168 */
130169 ret = kretprobe_event_gen_cmd_start(&cmd, "gen_kretprobe_test",
131− "do_sys_open",
170+ KPROBE_GEN_TEST_FUNC,
132171 "$retval");
133172 if (ret)
134173 goto free;
@@ -206,7 +245,7 @@ static void __exit kprobe_event_gen_test_exit(void) · 71 unmodified lines
206245 WARN_ON(kprobe_event_delete("gen_kprobe_test"));
207246
208247 /* Disable the event or you can't remove it */
209− WARN_ON(trace_array_set_clr_event(gen_kprobe_test->tr,
248+ WARN_ON(trace_array_set_clr_event(gen_kretprobe_test->tr,
210249 "kprobes",
211250 "gen_kretprobe_test", false));
212251
kernel/trace/ring_buffer.c+84−3
@@ -413,6 +413,7 @@ struct rb_irq_work { · 412 unmodified lines
413413 struct irq_work work;
414414 wait_queue_head_t waiters;
415415 wait_queue_head_t full_waiters;
416+ long wait_index;
416417 bool waiters_pending;
417418 bool full_waiters_pending;
418419 bool wakeup_full;
@@ -917,12 +918,44 @@ static void rb_wake_up_waiters(struct irq_work *work) · 498 unmodified lines
917918 struct rb_irq_work *rbwork = container_of(work, struct rb_irq_work, work);
918919
919920 wake_up_all(&rbwork->waiters);
920− if (rbwork->wakeup_full) {
921+ if (rbwork->full_waiters_pending || rbwork->wakeup_full) {
921922 rbwork->wakeup_full = false;
923+ rbwork->full_waiters_pending = false;
922924 wake_up_all(&rbwork->full_waiters);
923925 }
924926 }
925927
928+/**
929+ * ring_buffer_wake_waiters - wake up any waiters on this ring buffer
930+ * @buffer: The ring buffer to wake waiters on
931+ *
932+ * In the case of a file that represents a ring buffer is closing,
933+ * it is prudent to wake up any waiters that are on this.
934+ */
935+void ring_buffer_wake_waiters(struct trace_buffer *buffer, int cpu)
936+{
937+ struct ring_buffer_per_cpu *cpu_buffer;
938+ struct rb_irq_work *rbwork;
939+
940+ if (cpu == RING_BUFFER_ALL_CPUS) {
941+
942+ /* Wake up individual ones too. One level recursion */
943+ for_each_buffer_cpu(buffer, cpu)
944+ ring_buffer_wake_waiters(buffer, cpu);
945+
946+ rbwork = &buffer->irq_work;
947+ } else {
948+ cpu_buffer = buffer->buffers[cpu];
949+ rbwork = &cpu_buffer->irq_work;
950+ }
951+
952+ rbwork->wait_index++;
953+ /* make sure the waiters see the new index */
954+ smp_wmb();
955+
956+ rb_wake_up_waiters(&rbwork->work);
957+}
958+
926959 /**
927960 * ring_buffer_wait - wait for input to the ring buffer
928961 * @buffer: buffer to wait on
@@ -938,6 +971,7 @@ int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full) · 9 unmodified lines
938971 struct ring_buffer_per_cpu *cpu_buffer;
939972 DEFINE_WAIT(wait);
940973 struct rb_irq_work *work;
974+ long wait_index;
941975 int ret = 0;
942976
943977 /*
@@ -956,6 +990,7 @@ int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full) · 12 unmodified lines
956990 work = &cpu_buffer->irq_work;
957991 }
958992
993+ wait_index = READ_ONCE(work->wait_index);
959994
960995 while (true) {
961996 if (full)
@@ -1011,7 +1046,7 @@ int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full) · 49 unmodified lines
10111046 nr_pages = cpu_buffer->nr_pages;
10121047 dirty = ring_buffer_nr_dirty_pages(buffer, cpu);
10131048 if (!cpu_buffer->shortest_full ||
1014− cpu_buffer->shortest_full < full)
1049+ cpu_buffer->shortest_full > full)
10151050 cpu_buffer->shortest_full = full;
10161051 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
10171052 if (!pagebusy &&
@@ -1020,6 +1055,11 @@ int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full) · 2 unmodified lines
10201055 }
10211056
10221057 schedule();
1058+
1059+ /* Make sure to see the new wait index */
1060+ smp_rmb();
1061+ if (wait_index != work->wait_index)
1062+ break;
10231063 }
10241064
10251065 if (full)
@@ -2608,6 +2648,9 @@ rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer, · 1582 unmodified lines
26082648 /* Mark the rest of the page with padding */
26092649 rb_event_set_padding(event);
26102650
2651+ /* Make sure the padding is visible before the write update */
2652+ smp_wmb();
2653+
26112654 /* Set the write back to the previous setting */
26122655 local_sub(length, &tail_page->write);
26132656 return;
@@ -2619,6 +2662,9 @@ rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer, · 5 unmodified lines
26192662 /* time delta must be non zero */
26202663 event->time_delta = 1;
26212664
2665+ /* Make sure the padding is visible before the tail_page->write update */
2666+ smp_wmb();
2667+
26222668 /* Set write to end of buffer */
26232669 length = (tail + length) - BUF_PAGE_SIZE;
26242670 local_sub(length, &tail_page->write);
@@ -4587,6 +4633,33 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) · 1962 unmodified lines
45874633 arch_spin_unlock(&cpu_buffer->lock);
45884634 local_irq_restore(flags);
45894635
4636+ /*
4637+ * The writer has preempt disable, wait for it. But not forever
4638+ * Although, 1 second is pretty much "forever"
4639+ */
4640+#define USECS_WAIT 1000000
4641+ for (nr_loops = 0; nr_loops < USECS_WAIT; nr_loops++) {
4642+ /* If the write is past the end of page, a writer is still updating it */
4643+ if (likely(!reader || rb_page_write(reader) <= BUF_PAGE_SIZE))
4644+ break;
4645+
4646+ udelay(1);
4647+
4648+ /* Get the latest version of the reader write value */
4649+ smp_rmb();
4650+ }
4651+
4652+ /* The writer is not moving forward? Something is wrong */
4653+ if (RB_WARN_ON(cpu_buffer, nr_loops == USECS_WAIT))
4654+ reader = NULL;
4655+
4656+ /*
4657+ * Make sure we see any padding after the write update
4658+ * (see rb_reset_tail())
4659+ */
4660+ smp_rmb();
4661+
4662+
45904663 return reader;
45914664 }
45924665
@@ -5616,7 +5689,15 @@ int ring_buffer_read_page(struct trace_buffer *buffer, · 1023 unmodified lines
56165689 unsigned int pos = 0;
56175690 unsigned int size;
56185691
5619− if (full)
5692+ /*
5693+ * If a full page is expected, this can still be returned
5694+ * if there's been a previous partial read and the
5695+ * rest of the page can be read and the commit page is off
5696+ * the reader page.
5697+ */
5698+ if (full &&
5699+ (!read || (len < (commit - read)) ||
5700+ cpu_buffer->reader_page == cpu_buffer->commit_page))
56205701 goto out_unlock;
56215702
56225703 if (len > (commit - read))
kernel/trace/rv/monitors/wip/wip.c+4−4
@@ -16,7 +16,7 @@ · 15 unmodified lines
1616
1717 #include "wip.h"
1818
19−struct rv_monitor rv_wip;
19+static struct rv_monitor rv_wip;
2020 DECLARE_DA_MON_PER_CPU(wip, unsigned char);
2121
2222 static void handle_preempt_disable(void *data, unsigned long ip, unsigned long parent_ip)
@@ -60,7 +60,7 @@ static void disable_wip(void) · 37 unmodified lines
6060 da_monitor_destroy_wip();
6161 }
6262
63−struct rv_monitor rv_wip = {
63+static struct rv_monitor rv_wip = {
6464 .name = "wip",
6565 .description = "wakeup in preemptive per-cpu testing monitor.",
6666 .enable = enable_wip,
@@ -69,13 +69,13 @@ struct rv_monitor rv_wip = { · 2 unmodified lines
6969 .enabled = 0,
7070 };
7171
72−static int register_wip(void)
72+static int __init register_wip(void)
7373 {
7474 rv_register_monitor(&rv_wip);
7575 return 0;
7676 }
7777
78−static void unregister_wip(void)
78+static void __exit unregister_wip(void)
7979 {
8080 rv_unregister_monitor(&rv_wip);
8181 }
kernel/trace/rv/monitors/wwnr/wwnr.c+4−4
@@ -15,7 +15,7 @@ · 14 unmodified lines
1515
1616 #include "wwnr.h"
1717
18−struct rv_monitor rv_wwnr;
18+static struct rv_monitor rv_wwnr;
1919 DECLARE_DA_MON_PER_TASK(wwnr, unsigned char);
2020
2121 static void handle_switch(void *data, bool preempt, struct task_struct *p,
@@ -59,7 +59,7 @@ static void disable_wwnr(void) · 37 unmodified lines
5959 da_monitor_destroy_wwnr();
6060 }
6161
62−struct rv_monitor rv_wwnr = {
62+static struct rv_monitor rv_wwnr = {
6363 .name = "wwnr",
6464 .description = "wakeup while not running per-task testing model.",
6565 .enable = enable_wwnr,
@@ -68,13 +68,13 @@ struct rv_monitor rv_wwnr = { · 2 unmodified lines
6868 .enabled = 0,
6969 };
7070
71−static int register_wwnr(void)
71+static int __init register_wwnr(void)
7272 {
7373 rv_register_monitor(&rv_wwnr);
7474 return 0;
7575 }
7676
77−static void unregister_wwnr(void)
77+static void __exit unregister_wwnr(void)
7878 {
7979 rv_unregister_monitor(&rv_wwnr);
8080 }
kernel/trace/trace.c+73−5
@@ -1193,12 +1193,14 @@ void *tracing_cond_snapshot_data(struct trace_array *tr) · 1192 unmodified lines
11931193 {
11941194 void *cond_data = NULL;
11951195
1196+ local_irq_disable();
11961197 arch_spin_lock(&tr->max_lock);
11971198
11981199 if (tr->cond_snapshot)
11991200 cond_data = tr->cond_snapshot->cond_data;
12001201
12011202 arch_spin_unlock(&tr->max_lock);
1203+ local_irq_enable();
12021204
12031205 return cond_data;
12041206 }
@@ -1334,9 +1336,11 @@ int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data, · 129 unmodified lines
13341336 goto fail_unlock;
13351337 }
13361338
1339+ local_irq_disable();
13371340 arch_spin_lock(&tr->max_lock);
13381341 tr->cond_snapshot = cond_snapshot;
13391342 arch_spin_unlock(&tr->max_lock);
1343+ local_irq_enable();
13401344
13411345 mutex_unlock(&trace_types_lock);
13421346
@@ -1363,6 +1367,7 @@ int tracing_snapshot_cond_disable(struct trace_array *tr) · 20 unmodified lines
13631367 {
13641368 int ret = 0;
13651369
1370+ local_irq_disable();
13661371 arch_spin_lock(&tr->max_lock);
13671372
13681373 if (!tr->cond_snapshot)
@@ -1373,6 +1378,7 @@ int tracing_snapshot_cond_disable(struct trace_array *tr) · 4 unmodified lines
13731378 }
13741379
13751380 arch_spin_unlock(&tr->max_lock);
1381+ local_irq_enable();
13761382
13771383 return ret;
13781384 }
@@ -2200,6 +2206,11 @@ static size_t tgid_map_max; · 821 unmodified lines
22002206
22012207 #define SAVED_CMDLINES_DEFAULT 128
22022208 #define NO_CMDLINE_MAP UINT_MAX
2209+/*
2210+ * Preemption must be disabled before acquiring trace_cmdline_lock.
2211+ * The various trace_arrays' max_lock must be acquired in a context
2212+ * where interrupt is disabled.
2213+ */
22032214 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
22042215 struct saved_cmdlines_buffer {
22052216 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
@@ -2412,7 +2423,11 @@ static int trace_save_cmdline(struct task_struct *tsk) · 206 unmodified lines
24122423 * the lock, but we also don't want to spin
24132424 * nor do we want to disable interrupts,
24142425 * so if we miss here, then better luck next time.
2426+ *
2427+ * This is called within the scheduler and wake up, so interrupts
2428+ * had better been disabled and run queue lock been held.
24152429 */
2430+ lockdep_assert_preemption_disabled();
24162431 if (!arch_spin_trylock(&trace_cmdline_lock))
24172432 return 0;
24182433
@@ -5890,9 +5905,11 @@ tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf, · 3471 unmodified lines
58905905 char buf[64];
58915906 int r;
58925907
5908+ preempt_disable();
58935909 arch_spin_lock(&trace_cmdline_lock);
58945910 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
58955911 arch_spin_unlock(&trace_cmdline_lock);
5912+ preempt_enable();
58965913
58975914 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
58985915 }
@@ -5917,10 +5934,12 @@ static int tracing_resize_saved_cmdlines(unsigned int val) · 18 unmodified lines
59175934 return -ENOMEM;
59185935 }
59195936
5937+ preempt_disable();
59205938 arch_spin_lock(&trace_cmdline_lock);
59215939 savedcmd_temp = savedcmd;
59225940 savedcmd = s;
59235941 arch_spin_unlock(&trace_cmdline_lock);
5942+ preempt_enable();
59245943 free_saved_cmdlines_buffer(savedcmd_temp);
59255944
59265945 return 0;
@@ -6373,10 +6392,12 @@ int tracing_set_tracer(struct trace_array *tr, const char *buf) · 446 unmodified lines
63736392
63746393 #ifdef CONFIG_TRACER_SNAPSHOT
63756394 if (t->use_max_tr) {
6395+ local_irq_disable();
63766396 arch_spin_lock(&tr->max_lock);
63776397 if (tr->cond_snapshot)
63786398 ret = -EBUSY;
63796399 arch_spin_unlock(&tr->max_lock);
6400+ local_irq_enable();
63806401 if (ret)
63816402 goto out;
63826403 }
@@ -6407,12 +6428,12 @@ int tracing_set_tracer(struct trace_array *tr, const char *buf) · 24 unmodified lines
64076428 if (tr->current_trace->reset)
64086429 tr->current_trace->reset(tr);
64096430
6431+#ifdef CONFIG_TRACER_MAX_TRACE
6432+ had_max_tr = tr->current_trace->use_max_tr;
6433+
64106434 /* Current trace needs to be nop_trace before synchronize_rcu */
64116435 tr->current_trace = &nop_trace;
64126436
6413−#ifdef CONFIG_TRACER_MAX_TRACE
6414− had_max_tr = tr->allocated_snapshot;
6415−
64166437 if (had_max_tr && !t->use_max_tr) {
64176438 /*
64186439 * We need to make sure that the update_max_tr sees that
@@ -6425,11 +6446,13 @@ int tracing_set_tracer(struct trace_array *tr, const char *buf) · 6 unmodified lines
64256446 free_snapshot(tr);
64266447 }
64276448
6428− if (t->use_max_tr && !had_max_tr) {
6449+ if (t->use_max_tr && !tr->allocated_snapshot) {
64296450 ret = tracing_alloc_snapshot_instance(tr);
64306451 if (ret < 0)
64316452 goto out;
64326453 }
6454+#else
6455+ tr->current_trace = &nop_trace;
64336456 #endif
64346457
64356458 if (t->init) {
@@ -7436,10 +7459,12 @@ tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt, · 1000 unmodified lines
74367459 goto out;
74377460 }
74387461
7462+ local_irq_disable();
74397463 arch_spin_lock(&tr->max_lock);
74407464 if (tr->cond_snapshot)
74417465 ret = -EBUSY;
74427466 arch_spin_unlock(&tr->max_lock);
7467+ local_irq_enable();
74437468 if (ret)
74447469 goto out;
74457470
@@ -8137,6 +8162,12 @@ static int tracing_buffers_release(struct inode *inode, struct file *file) · 691 unmodified lines
81378162
81388163 __trace_array_put(iter->tr);
81398164
8165+ iter->wait_index++;
8166+ /* Make sure the waiters see the new wait_index */
8167+ smp_wmb();
8168+
8169+ ring_buffer_wake_waiters(iter->array_buffer->buffer, iter->cpu_file);
8170+
81408171 if (info->spare)
81418172 ring_buffer_free_read_page(iter->array_buffer->buffer,
81428173 info->spare_cpu, info->spare);
@@ -8290,6 +8321,8 @@ tracing_buffers_splice_read(struct file *file, loff_t *ppos, · 147 unmodified lines
82908321
82918322 /* did we read anything? */
82928323 if (!spd.nr_pages) {
8324+ long wait_index;
8325+
82938326 if (ret)
82948327 goto out;
82958328
@@ -8297,10 +8330,21 @@ tracing_buffers_splice_read(struct file *file, loff_t *ppos, · 1 unmodified line
82978330 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
82988331 goto out;
82998332
8333+ wait_index = READ_ONCE(iter->wait_index);
8334+
83008335 ret = wait_on_pipe(iter, iter->tr->buffer_percent);
83018336 if (ret)
83028337 goto out;
83038338
8339+ /* No need to wait after waking up when tracing is off */
8340+ if (!tracer_tracing_is_on(iter->tr))
8341+ goto out;
8342+
8343+ /* Make sure we see the new wait_index */
8344+ smp_rmb();
8345+ if (wait_index != iter->wait_index)
8346+ goto out;
8347+
83048348 goto again;
83058349 }
83068350
@@ -8311,12 +8355,34 @@ out: · 4 unmodified lines
83118355 return ret;
83128356 }
83138357
8358+/* An ioctl call with cmd 0 to the ring buffer file will wake up all waiters */
8359+static long tracing_buffers_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
8360+{
8361+ struct ftrace_buffer_info *info = file->private_data;
8362+ struct trace_iterator *iter = &info->iter;
8363+
8364+ if (cmd)
8365+ return -ENOIOCTLCMD;
8366+
8367+ mutex_lock(&trace_types_lock);
8368+
8369+ iter->wait_index++;
8370+ /* Make sure the waiters see the new wait_index */
8371+ smp_wmb();
8372+
8373+ ring_buffer_wake_waiters(iter->array_buffer->buffer, iter->cpu_file);
8374+
8375+ mutex_unlock(&trace_types_lock);
8376+ return 0;
8377+}
8378+
83148379 static const struct file_operations tracing_buffers_fops = {
83158380 .open = tracing_buffers_open,
83168381 .read = tracing_buffers_read,
83178382 .poll = tracing_buffers_poll,
83188383 .release = tracing_buffers_release,
83198384 .splice_read = tracing_buffers_splice_read,
8385+ .unlocked_ioctl = tracing_buffers_ioctl,
83208386 .llseek = no_llseek,
83218387 };
83228388
@@ -9005,6 +9071,8 @@ rb_simple_write(struct file *filp, const char __user *ubuf, · 682 unmodified lines
90059071 tracer_tracing_off(tr);
90069072 if (tr->current_trace->stop)
90079073 tr->current_trace->stop(tr);
9074+ /* Wake up any waiters */
9075+ ring_buffer_wake_waiters(buffer, RING_BUFFER_ALL_CPUS);
90089076 }
90099077 mutex_unlock(&trace_types_lock);
90109078 }
@@ -10091,7 +10159,7 @@ __init static int tracer_alloc_buffers(void) · 1080 unmodified lines
1009110159 * buffer. The memory will be removed once the "instance" is removed.
1009210160 */
1009310161 ret = cpuhp_setup_state_multi(CPUHP_TRACE_RB_PREPARE,
10094− "trace/RB:preapre", trace_rb_cpu_prepare,
10162+ "trace/RB:prepare", trace_rb_cpu_prepare,
1009510163 NULL);
1009610164 if (ret < 0)
1009710165 goto out_free_cpumask;
kernel/trace/trace.h+0−13
@@ -1435,8 +1435,6 @@ event_trigger_unlock_commit(struct trace_event_file *file, · 1434 unmodified lines
14351435 struct filter_pred;
14361436 struct regex;
14371437
1438−typedef int (*filter_pred_fn_t) (struct filter_pred *pred, void *event);
1439−
14401438 typedef int (*regex_match_func)(char *str, struct regex *r, int len);
14411439
14421440 enum regex_type {
@@ -1455,17 +1453,6 @@ struct regex { · 12 unmodified lines
14551453 regex_match_func match;
14561454 };
14571455
1458−struct filter_pred {
1459− filter_pred_fn_t fn;
1460− u64 val;
1461− struct regex regex;
1462− unsigned short *ops;
1463− struct ftrace_event_field *field;
1464− int offset;
1465− int not;
1466− int op;
1467−};
1468−
14691456 static inline bool is_string_field(struct ftrace_event_field *field)
14701457 {
14711458 return field->filter_type == FILTER_DYN_STRING ||
kernel/trace/trace_benchmark.c+1−1
@@ -51,7 +51,7 @@ static void trace_do_benchmark(void) · 50 unmodified lines
5151
5252 local_irq_disable();
5353 start = trace_clock_local();
54− trace_benchmark_event(bm_str);
54+ trace_benchmark_event(bm_str, bm_last);
5555 stop = trace_clock_local();
5656 local_irq_enable();
5757
kernel/trace/trace_benchmark.h+5−3
@@ -14,19 +14,21 @@ extern void trace_benchmark_unreg(void); · 13 unmodified lines
1414
1515 TRACE_EVENT_FN(benchmark_event,
1616
17− TP_PROTO(const char *str),
17+ TP_PROTO(const char *str, u64 delta),
1818
19− TP_ARGS(str),
19+ TP_ARGS(str, delta),
2020
2121 TP_STRUCT__entry(
2222 __array( char, str, BENCHMARK_EVENT_STRLEN )
23+ __field( u64, delta)
2324 ),
2425
2526 TP_fast_assign(
2627 memcpy(__entry->str, str, BENCHMARK_EVENT_STRLEN);
28+ __entry->delta = delta;
2729 ),
2830
29− TP_printk("%s", __entry->str),
31+ TP_printk("%s delta=%llu", __entry->str, __entry->delta),
3032
3133 trace_benchmark_reg, trace_benchmark_unreg
3234 );
kernel/trace/trace_eprobe.c+97−10
@@ -26,6 +26,9 @@ struct trace_eprobe { · 25 unmodified lines
2626 /* tracepoint event */
2727 const char *event_name;
2828
29+ /* filter string for the tracepoint */
30+ char *filter_str;
31+
2932 struct trace_event_call *event;
3033
3134 struct dyn_event devent;
@@ -664,14 +667,15 @@ static struct event_trigger_data * · 632 unmodified lines
664667 new_eprobe_trigger(struct trace_eprobe *ep, struct trace_event_file *file)
665668 {
666669 struct event_trigger_data *trigger;
670+ struct event_filter *filter = NULL;
667671 struct eprobe_data *edata;
672+ int ret;
668673
669674 edata = kzalloc(sizeof(*edata), GFP_KERNEL);
670675 trigger = kzalloc(sizeof(*trigger), GFP_KERNEL);
671676 if (!trigger || !edata) {
672− kfree(edata);
673− kfree(trigger);
674− return ERR_PTR(-ENOMEM);
677+ ret = -ENOMEM;
678+ goto error;
675679 }
676680
677681 trigger->flags = EVENT_TRIGGER_FL_PROBE;
@@ -686,13 +690,25 @@ new_eprobe_trigger(struct trace_eprobe *ep, struct trace_event_file *file) · 8 unmodified lines
686690 trigger->cmd_ops = &event_trigger_cmd;
687691
688692 INIT_LIST_HEAD(&trigger->list);
689− RCU_INIT_POINTER(trigger->filter, NULL);
693+
694+ if (ep->filter_str) {
695+ ret = create_event_filter(file->tr, file->event_call,
696+ ep->filter_str, false, &filter);
697+ if (ret)
698+ goto error;
699+ }
700+ RCU_INIT_POINTER(trigger->filter, filter);
690701
691702 edata->file = file;
692703 edata->ep = ep;
693704 trigger->private_data = edata;
694705
695706 return trigger;
707+error:
708+ free_event_filter(filter);
709+ kfree(edata);
710+ kfree(trigger);
711+ return ERR_PTR(ret);
696712 }
697713
698714 static int enable_eprobe(struct trace_eprobe *ep,
@@ -726,6 +742,7 @@ static int disable_eprobe(struct trace_eprobe *ep, · 27 unmodified lines
726742 {
727743 struct event_trigger_data *trigger = NULL, *iter;
728744 struct trace_event_file *file;
745+ struct event_filter *filter;
729746 struct eprobe_data *edata;
730747
731748 file = find_event_file(tr, ep->event_system, ep->event_name);
@@ -752,6 +769,10 @@ static int disable_eprobe(struct trace_eprobe *ep, · 20 unmodified lines
752769 /* Make sure nothing is using the edata or trigger */
753770 tracepoint_synchronize_unregister();
754771
772+ filter = rcu_access_pointer(trigger->filter);
773+
774+ if (filter)
775+ free_event_filter(filter);
755776 kfree(edata);
756777 kfree(trigger);
757778
@@ -927,12 +948,62 @@ static int trace_eprobe_tp_update_arg(struct trace_eprobe *ep, const char *argv[ · 169 unmodified lines
927948 return ret;
928949 }
929950
951+static int trace_eprobe_parse_filter(struct trace_eprobe *ep, int argc, const char *argv[])
952+{
953+ struct event_filter *dummy;
954+ int i, ret, len = 0;
955+ char *p;
956+
957+ if (argc == 0) {
958+ trace_probe_log_err(0, NO_EP_FILTER);
959+ return -EINVAL;
960+ }
961+
962+ /* Recover the filter string */
963+ for (i = 0; i < argc; i++)
964+ len += strlen(argv[i]) + 1;
965+
966+ ep->filter_str = kzalloc(len, GFP_KERNEL);
967+ if (!ep->filter_str)
968+ return -ENOMEM;
969+
970+ p = ep->filter_str;
971+ for (i = 0; i < argc; i++) {
972+ ret = snprintf(p, len, "%s ", argv[i]);
973+ if (ret < 0)
974+ goto error;
975+ if (ret > len) {
976+ ret = -E2BIG;
977+ goto error;
978+ }
979+ p += ret;
980+ len -= ret;
981+ }
982+ p[-1] = '\0';
983+
984+ /*
985+ * Ensure the filter string can be parsed correctly. Note, this
986+ * filter string is for the original event, not for the eprobe.
987+ */
988+ ret = create_event_filter(top_trace_array(), ep->event, ep->filter_str,
989+ true, &dummy);
990+ free_event_filter(dummy);
991+ if (ret)
992+ goto error;
993+
994+ return 0;
995+error:
996+ kfree(ep->filter_str);
997+ ep->filter_str = NULL;
998+ return ret;
999+}
1000+
9301001 static int __trace_eprobe_create(int argc, const char *argv[])
9311002 {
9321003 /*
9331004 * Argument syntax:
934− * e[:[GRP/][ENAME]] SYSTEM.EVENT [FETCHARGS]
935− * Fetch args:
1005+ * e[:[GRP/][ENAME]] SYSTEM.EVENT [FETCHARGS] [if FILTER]
1006+ * Fetch args (no space):
9361007 * <name>=$<field>[:TYPE]
9371008 */
9381009 const char *event = NULL, *group = EPROBE_EVENT_SYSTEM;
@@ -942,8 +1013,8 @@ static int __trace_eprobe_create(int argc, const char *argv[]) · 3 unmodified lines
9421013 char buf1[MAX_EVENT_NAME_LEN];
9431014 char buf2[MAX_EVENT_NAME_LEN];
9441015 char gbuf[MAX_EVENT_NAME_LEN];
945− int ret = 0;
946− int i;
1016+ int ret = 0, filter_idx = 0;
1017+ int i, filter_cnt;
9471018
9481019 if (argc < 2 || argv[0][0] != 'e')
9491020 return -ECANCELED;
@@ -968,11 +1039,19 @@ static int __trace_eprobe_create(int argc, const char *argv[]) · 18 unmodified lines
9681039 }
9691040
9701041 if (!event) {
971− strscpy(buf1, argv[1], MAX_EVENT_NAME_LEN);
972− sanitize_event_name(buf1);
1042+ strscpy(buf1, sys_event, MAX_EVENT_NAME_LEN);
9731043 event = buf1;
9741044 }
9751045
1046+ for (i = 2; i < argc; i++) {
1047+ if (!strcmp(argv[i], "if")) {
1048+ filter_idx = i + 1;
1049+ filter_cnt = argc - filter_idx;
1050+ argc = i;
1051+ break;
1052+ }
1053+ }
1054+
9761055 mutex_lock(&event_mutex);
9771056 event_call = find_and_get_event(sys_name, sys_event);
9781057 ep = alloc_event_probe(group, event, event_call, argc - 2);
@@ -988,6 +1067,14 @@ static int __trace_eprobe_create(int argc, const char *argv[]) · 9 unmodified lines
9881067 goto error;
9891068 }
9901069
1070+ if (filter_idx) {
1071+ trace_probe_log_set_index(filter_idx);
1072+ ret = trace_eprobe_parse_filter(ep, filter_cnt, argv + filter_idx);
1073+ if (ret)
1074+ goto parse_error;
1075+ } else
1076+ ep->filter_str = NULL;
1077+
9911078 argc -= 2; argv += 2;
9921079 /* parse arguments */
9931080 for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) {
kernel/trace/trace_events_filter.c+168−71
@@ -43,6 +43,42 @@ enum filter_op_ids { OPS }; · 42 unmodified lines
4343
4444 static const char * ops[] = { OPS };
4545
46+enum filter_pred_fn {
47+ FILTER_PRED_FN_NOP,
48+ FILTER_PRED_FN_64,
49+ FILTER_PRED_FN_S64,
50+ FILTER_PRED_FN_U64,
51+ FILTER_PRED_FN_32,
52+ FILTER_PRED_FN_S32,
53+ FILTER_PRED_FN_U32,
54+ FILTER_PRED_FN_16,
55+ FILTER_PRED_FN_S16,
56+ FILTER_PRED_FN_U16,
57+ FILTER_PRED_FN_8,
58+ FILTER_PRED_FN_S8,
59+ FILTER_PRED_FN_U8,
60+ FILTER_PRED_FN_COMM,
61+ FILTER_PRED_FN_STRING,
62+ FILTER_PRED_FN_STRLOC,
63+ FILTER_PRED_FN_STRRELLOC,
64+ FILTER_PRED_FN_PCHAR_USER,
65+ FILTER_PRED_FN_PCHAR,
66+ FILTER_PRED_FN_CPU,
67+ FILTER_PRED_FN_,
68+ FILTER_PRED_TEST_VISITED,
69+};
70+
71+struct filter_pred {
72+ enum filter_pred_fn fn_num;
73+ u64 val;
74+ struct regex regex;
75+ unsigned short *ops;
76+ struct ftrace_event_field *field;
77+ int offset;
78+ int not;
79+ int op;
80+};
81+
4682 /*
4783 * pred functions are OP_LE, OP_LT, OP_GE, OP_GT, and OP_BAND
4884 * pred_funcs_##type below must match the order of them above.
@@ -590,44 +626,48 @@ out_free: · 541 unmodified lines
590626 return ERR_PTR(ret);
591627 }
592628
629+enum pred_cmp_types {
630+ PRED_CMP_TYPE_NOP,
631+ PRED_CMP_TYPE_LT,
632+ PRED_CMP_TYPE_LE,
633+ PRED_CMP_TYPE_GT,
634+ PRED_CMP_TYPE_GE,
635+ PRED_CMP_TYPE_BAND,
636+};
637+
593638 #define DEFINE_COMPARISON_PRED(type) \
594−static int filter_pred_LT_##type(struct filter_pred *pred, void *event) \
595−{ \
596− type *addr = (type *)(event + pred->offset); \
597− type val = (type)pred->val; \
598− return *addr < val; \
599−} \
600−static int filter_pred_LE_##type(struct filter_pred *pred, void *event) \
601−{ \
602− type *addr = (type *)(event + pred->offset); \
603− type val = (type)pred->val; \
604− return *addr <= val; \
605−} \
606−static int filter_pred_GT_##type(struct filter_pred *pred, void *event) \
639+static int filter_pred_##type(struct filter_pred *pred, void *event) \
607640 { \
608− type *addr = (type *)(event + pred->offset); \
609− type val = (type)pred->val; \
610− return *addr > val; \
611−} \
612−static int filter_pred_GE_##type(struct filter_pred *pred, void *event) \
613−{ \
614− type *addr = (type *)(event + pred->offset); \
615− type val = (type)pred->val; \
616− return *addr >= val; \
617−} \
618−static int filter_pred_BAND_##type(struct filter_pred *pred, void *event) \
619−{ \
620− type *addr = (type *)(event + pred->offset); \
621− type val = (type)pred->val; \
622− return !!(*addr & val); \
623−} \
624−static const filter_pred_fn_t pred_funcs_##type[] = { \
625− filter_pred_LE_##type, \
626− filter_pred_LT_##type, \
627− filter_pred_GE_##type, \
628− filter_pred_GT_##type, \
629− filter_pred_BAND_##type, \
630−};
641+ switch (pred->op) { \
642+ case OP_LT: { \
643+ type *addr = (type *)(event + pred->offset); \
644+ type val = (type)pred->val; \
645+ return *addr < val; \
646+ } \
647+ case OP_LE: { \
648+ type *addr = (type *)(event + pred->offset); \
649+ type val = (type)pred->val; \
650+ return *addr <= val; \
651+ } \
652+ case OP_GT: { \
653+ type *addr = (type *)(event + pred->offset); \
654+ type val = (type)pred->val; \
655+ return *addr > val; \
656+ } \
657+ case OP_GE: { \
658+ type *addr = (type *)(event + pred->offset); \
659+ type val = (type)pred->val; \
660+ return *addr >= val; \
661+ } \
662+ case OP_BAND: { \
663+ type *addr = (type *)(event + pred->offset); \
664+ type val = (type)pred->val; \
665+ return !!(*addr & val); \
666+ } \
667+ default: \
668+ return 0; \
669+ } \
670+}
631671
632672 #define DEFINE_EQUALITY_PRED(size) \
633673 static int filter_pred_##size(struct filter_pred *pred, void *event) \
@@ -836,11 +876,6 @@ static int filter_pred_comm(struct filter_pred *pred, void *event) · 202 unmodified lines
836876 return cmp ^ pred->not;
837877 }
838878
839−static int filter_pred_none(struct filter_pred *pred, void *event)
840−{
841− return 0;
842−}
843−
844879 /*
845880 * regex_match_foo - Basic regex callbacks
846881 *
@@ -986,6 +1021,19 @@ static void filter_build_regex(struct filter_pred *pred) · 139 unmodified lines
9861021 }
9871022 }
9881023
1024+
1025+#ifdef CONFIG_FTRACE_STARTUP_TEST
1026+static int test_pred_visited_fn(struct filter_pred *pred, void *event);
1027+#else
1028+static int test_pred_visited_fn(struct filter_pred *pred, void *event)
1029+{
1030+ return 0;
1031+}
1032+#endif
1033+
1034+
1035+static int filter_pred_fn_call(struct filter_pred *pred, void *event);
1036+
9891037 /* return 1 if event matches, 0 otherwise (discard) */
9901038 int filter_match_preds(struct event_filter *filter, void *rec)
9911039 {
@@ -1003,7 +1051,7 @@ int filter_match_preds(struct event_filter *filter, void *rec) · 11 unmodified lines
10031051
10041052 for (i = 0; prog[i].pred; i++) {
10051053 struct filter_pred *pred = prog[i].pred;
1006− int match = pred->fn(pred, rec);
1054+ int match = filter_pred_fn_call(pred, rec);
10071055 if (match == prog[i].when_to_branch)
10081056 i = prog[i].target;
10091057 }
@@ -1189,10 +1237,10 @@ int filter_assign_type(const char *type) · 179 unmodified lines
11891237 return FILTER_OTHER;
11901238 }
11911239
1192−static filter_pred_fn_t select_comparison_fn(enum filter_op_ids op,
1193− int field_size, int field_is_signed)
1240+static enum filter_pred_fn select_comparison_fn(enum filter_op_ids op,
1241+ int field_size, int field_is_signed)
11941242 {
1195− filter_pred_fn_t fn = NULL;
1243+ enum filter_pred_fn fn = FILTER_PRED_FN_NOP;
11961244 int pred_func_index = -1;
11971245
11981246 switch (op) {
@@ -1201,50 +1249,99 @@ static filter_pred_fn_t select_comparison_fn(enum filter_op_ids op, · 2 unmodified lines
12011249 break;
12021250 default:
12031251 if (WARN_ON_ONCE(op < PRED_FUNC_START))
1204− return NULL;
1252+ return fn;
12051253 pred_func_index = op - PRED_FUNC_START;
12061254 if (WARN_ON_ONCE(pred_func_index > PRED_FUNC_MAX))
1207− return NULL;
1255+ return fn;
12081256 }
12091257
12101258 switch (field_size) {
12111259 case 8:
12121260 if (pred_func_index < 0)
1213− fn = filter_pred_64;
1261+ fn = FILTER_PRED_FN_64;
12141262 else if (field_is_signed)
1215− fn = pred_funcs_s64[pred_func_index];
1263+ fn = FILTER_PRED_FN_S64;
12161264 else
1217− fn = pred_funcs_u64[pred_func_index];
1265+ fn = FILTER_PRED_FN_U64;
12181266 break;
12191267 case 4:
12201268 if (pred_func_index < 0)
1221− fn = filter_pred_32;
1269+ fn = FILTER_PRED_FN_32;
12221270 else if (field_is_signed)
1223− fn = pred_funcs_s32[pred_func_index];
1271+ fn = FILTER_PRED_FN_S32;
12241272 else
1225− fn = pred_funcs_u32[pred_func_index];
1273+ fn = FILTER_PRED_FN_U32;
12261274 break;
12271275 case 2:
12281276 if (pred_func_index < 0)
1229− fn = filter_pred_16;
1277+ fn = FILTER_PRED_FN_16;
12301278 else if (field_is_signed)
1231− fn = pred_funcs_s16[pred_func_index];
1279+ fn = FILTER_PRED_FN_S16;
12321280 else
1233− fn = pred_funcs_u16[pred_func_index];
1281+ fn = FILTER_PRED_FN_U16;
12341282 break;
12351283 case 1:
12361284 if (pred_func_index < 0)
1237− fn = filter_pred_8;
1285+ fn = FILTER_PRED_FN_8;
12381286 else if (field_is_signed)
1239− fn = pred_funcs_s8[pred_func_index];
1287+ fn = FILTER_PRED_FN_S8;
12401288 else
1241− fn = pred_funcs_u8[pred_func_index];
1289+ fn = FILTER_PRED_FN_U8;
12421290 break;
12431291 }
12441292
12451293 return fn;
12461294 }
12471295
1296+
1297+static int filter_pred_fn_call(struct filter_pred *pred, void *event)
1298+{
1299+ switch (pred->fn_num) {
1300+ case FILTER_PRED_FN_64:
1301+ return filter_pred_64(pred, event);
1302+ case FILTER_PRED_FN_S64:
1303+ return filter_pred_s64(pred, event);
1304+ case FILTER_PRED_FN_U64:
1305+ return filter_pred_u64(pred, event);
1306+ case FILTER_PRED_FN_32:
1307+ return filter_pred_32(pred, event);
1308+ case FILTER_PRED_FN_S32:
1309+ return filter_pred_s32(pred, event);
1310+ case FILTER_PRED_FN_U32:
1311+ return filter_pred_u32(pred, event);
1312+ case FILTER_PRED_FN_16:
1313+ return filter_pred_16(pred, event);
1314+ case FILTER_PRED_FN_S16:
1315+ return filter_pred_s16(pred, event);
1316+ case FILTER_PRED_FN_U16:
1317+ return filter_pred_u16(pred, event);
1318+ case FILTER_PRED_FN_8:
1319+ return filter_pred_8(pred, event);
1320+ case FILTER_PRED_FN_S8:
1321+ return filter_pred_s8(pred, event);
1322+ case FILTER_PRED_FN_U8:
1323+ return filter_pred_u8(pred, event);
1324+ case FILTER_PRED_FN_COMM:
1325+ return filter_pred_comm(pred, event);
1326+ case FILTER_PRED_FN_STRING:
1327+ return filter_pred_string(pred, event);
1328+ case FILTER_PRED_FN_STRLOC:
1329+ return filter_pred_strloc(pred, event);
1330+ case FILTER_PRED_FN_STRRELLOC:
1331+ return filter_pred_strrelloc(pred, event);
1332+ case FILTER_PRED_FN_PCHAR_USER:
1333+ return filter_pred_pchar_user(pred, event);
1334+ case FILTER_PRED_FN_PCHAR:
1335+ return filter_pred_pchar(pred, event);
1336+ case FILTER_PRED_FN_CPU:
1337+ return filter_pred_cpu(pred, event);
1338+ case FILTER_PRED_TEST_VISITED:
1339+ return test_pred_visited_fn(pred, event);
1340+ default:
1341+ return 0;
1342+ }
1343+}
1344+
12481345 /* Called when a predicate is encountered by predicate_parse() */
12491346 static int parse_pred(const char *str, void *data,
12501347 int pos, struct filter_parse_error *pe,
@@ -1338,7 +1435,7 @@ static int parse_pred(const char *str, void *data, · 87 unmodified lines
13381435 parse_error(pe, FILT_ERR_IP_FIELD_ONLY, pos + i);
13391436 goto err_free;
13401437 }
1341− pred->fn = filter_pred_none;
1438+ pred->fn_num = FILTER_PRED_FN_NOP;
13421439
13431440 /*
13441441 * Quotes are not required, but if they exist then we need
@@ -1416,16 +1513,16 @@ static int parse_pred(const char *str, void *data, · 71 unmodified lines
14161513 filter_build_regex(pred);
14171514
14181515 if (field->filter_type == FILTER_COMM) {
1419− pred->fn = filter_pred_comm;
1516+ pred->fn_num = FILTER_PRED_FN_COMM;
14201517
14211518 } else if (field->filter_type == FILTER_STATIC_STRING) {
1422− pred->fn = filter_pred_string;
1519+ pred->fn_num = FILTER_PRED_FN_STRING;
14231520 pred->regex.field_len = field->size;
14241521
14251522 } else if (field->filter_type == FILTER_DYN_STRING) {
1426− pred->fn = filter_pred_strloc;
1523+ pred->fn_num = FILTER_PRED_FN_STRLOC;
14271524 } else if (field->filter_type == FILTER_RDYN_STRING)
1428− pred->fn = filter_pred_strrelloc;
1525+ pred->fn_num = FILTER_PRED_FN_STRRELLOC;
14291526 else {
14301527
14311528 if (!ustring_per_cpu) {
@@ -1436,9 +1533,9 @@ static int parse_pred(const char *str, void *data, · 4 unmodified lines
14361533 }
14371534
14381535 if (ustring)
1439− pred->fn = filter_pred_pchar_user;
1536+ pred->fn_num = FILTER_PRED_FN_PCHAR_USER;
14401537 else
1441− pred->fn = filter_pred_pchar;
1538+ pred->fn_num = FILTER_PRED_FN_PCHAR;
14421539 }
14431540 /* go past the last quote */
14441541 i++;
@@ -1486,10 +1583,10 @@ static int parse_pred(const char *str, void *data, · 41 unmodified lines
14861583 pred->val = val;
14871584
14881585 if (field->filter_type == FILTER_CPU)
1489− pred->fn = filter_pred_cpu;
1586+ pred->fn_num = FILTER_PRED_FN_CPU;
14901587 else {
1491− pred->fn = select_comparison_fn(pred->op, field->size,
1492− field->is_signed);
1588+ pred->fn_num = select_comparison_fn(pred->op, field->size,
1589+ field->is_signed);
14931590 if (pred->op == OP_NE)
14941591 pred->not = 1;
14951592 }
@@ -2296,7 +2393,7 @@ static void update_pred_fn(struct event_filter *filter, char *fields) · 800 unmodified lines
22962393 struct filter_pred *pred = prog[i].pred;
22972394 struct ftrace_event_field *field = pred->field;
22982395
2299− WARN_ON_ONCE(!pred->fn);
2396+ WARN_ON_ONCE(pred->fn_num == FILTER_PRED_FN_NOP);
23002397
23012398 if (!field) {
23022399 WARN_ONCE(1, "all leafs should have field defined %d", i);
@@ -2306,7 +2403,7 @@ static void update_pred_fn(struct event_filter *filter, char *fields) · 3 unmodified lines
23062403 if (!strchr(fields, *field->name))
23072404 continue;
23082405
2309− pred->fn = test_pred_visited_fn;
2406+ pred->fn_num = FILTER_PRED_TEST_VISITED;
23102407 }
23112408 }
23122409
kernel/trace/trace_events_hist.c+169−77
@@ -104,6 +104,38 @@ enum field_op_id { · 103 unmodified lines
104104 FIELD_OP_MULT,
105105 };
106106
107+enum hist_field_fn {
108+ HIST_FIELD_FN_NOP,
109+ HIST_FIELD_FN_VAR_REF,
110+ HIST_FIELD_FN_COUNTER,
111+ HIST_FIELD_FN_CONST,
112+ HIST_FIELD_FN_LOG2,
113+ HIST_FIELD_FN_BUCKET,
114+ HIST_FIELD_FN_TIMESTAMP,
115+ HIST_FIELD_FN_CPU,
116+ HIST_FIELD_FN_STRING,
117+ HIST_FIELD_FN_DYNSTRING,
118+ HIST_FIELD_FN_RELDYNSTRING,
119+ HIST_FIELD_FN_PSTRING,
120+ HIST_FIELD_FN_S64,
121+ HIST_FIELD_FN_U64,
122+ HIST_FIELD_FN_S32,
123+ HIST_FIELD_FN_U32,
124+ HIST_FIELD_FN_S16,
125+ HIST_FIELD_FN_U16,
126+ HIST_FIELD_FN_S8,
127+ HIST_FIELD_FN_U8,
128+ HIST_FIELD_FN_UMINUS,
129+ HIST_FIELD_FN_MINUS,
130+ HIST_FIELD_FN_PLUS,
131+ HIST_FIELD_FN_DIV,
132+ HIST_FIELD_FN_MULT,
133+ HIST_FIELD_FN_DIV_POWER2,
134+ HIST_FIELD_FN_DIV_NOT_POWER2,
135+ HIST_FIELD_FN_DIV_MULT_SHIFT,
136+ HIST_FIELD_FN_EXECNAME,
137+};
138+
107139 /*
108140 * A hist_var (histogram variable) contains variable information for
109141 * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
@@ -123,15 +155,15 @@ struct hist_var { · 13 unmodified lines
123155 struct hist_field {
124156 struct ftrace_event_field *field;
125157 unsigned long flags;
126− hist_field_fn_t fn;
127− unsigned int ref;
128− unsigned int size;
129− unsigned int offset;
130− unsigned int is_signed;
131158 unsigned long buckets;
132159 const char *type;
133160 struct hist_field *operands[HIST_FIELD_OPERANDS_MAX];
134161 struct hist_trigger_data *hist_data;
162+ enum hist_field_fn fn_num;
163+ unsigned int ref;
164+ unsigned int size;
165+ unsigned int offset;
166+ unsigned int is_signed;
135167
136168 /*
137169 * Variable fields contain variable-specific info in var.
@@ -166,14 +198,11 @@ struct hist_field { · 28 unmodified lines
166198 u64 div_multiplier;
167199 };
168200
169−static u64 hist_field_none(struct hist_field *field,
170− struct tracing_map_elt *elt,
171− struct trace_buffer *buffer,
172− struct ring_buffer_event *rbe,
173− void *event)
174−{
175− return 0;
176−}
201+static u64 hist_fn_call(struct hist_field *hist_field,
202+ struct tracing_map_elt *elt,
203+ struct trace_buffer *buffer,
204+ struct ring_buffer_event *rbe,
205+ void *event);
177206
178207 static u64 hist_field_const(struct hist_field *field,
179208 struct tracing_map_elt *elt,
@@ -250,7 +279,7 @@ static u64 hist_field_log2(struct hist_field *hist_field, · 70 unmodified lines
250279 {
251280 struct hist_field *operand = hist_field->operands[0];
252281
253− u64 val = operand->fn(operand, elt, buffer, rbe, event);
282+ u64 val = hist_fn_call(operand, elt, buffer, rbe, event);
254283
255284 return (u64) ilog2(roundup_pow_of_two(val));
256285 }
@@ -264,7 +293,7 @@ static u64 hist_field_bucket(struct hist_field *hist_field, · 7 unmodified lines
264293 struct hist_field *operand = hist_field->operands[0];
265294 unsigned long buckets = hist_field->buckets;
266295
267− u64 val = operand->fn(operand, elt, buffer, rbe, event);
296+ u64 val = hist_fn_call(operand, elt, buffer, rbe, event);
268297
269298 if (WARN_ON_ONCE(!buckets))
270299 return val;
@@ -285,8 +314,8 @@ static u64 hist_field_plus(struct hist_field *hist_field, · 14 unmodified lines
285314 struct hist_field *operand1 = hist_field->operands[0];
286315 struct hist_field *operand2 = hist_field->operands[1];
287316
288− u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
289− u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
317+ u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
318+ u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
290319
291320 return val1 + val2;
292321 }
@@ -300,8 +329,8 @@ static u64 hist_field_minus(struct hist_field *hist_field, · 7 unmodified lines
300329 struct hist_field *operand1 = hist_field->operands[0];
301330 struct hist_field *operand2 = hist_field->operands[1];
302331
303− u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
304− u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
332+ u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
333+ u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
305334
306335 return val1 - val2;
307336 }
@@ -315,8 +344,8 @@ static u64 hist_field_div(struct hist_field *hist_field, · 7 unmodified lines
315344 struct hist_field *operand1 = hist_field->operands[0];
316345 struct hist_field *operand2 = hist_field->operands[1];
317346
318− u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
319− u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
347+ u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
348+ u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
320349
321350 /* Return -1 for the undefined case */
322351 if (!val2)
@@ -338,7 +367,7 @@ static u64 div_by_power_of_two(struct hist_field *hist_field, · 15 unmodified lines
338367 struct hist_field *operand1 = hist_field->operands[0];
339368 struct hist_field *operand2 = hist_field->operands[1];
340369
341− u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
370+ u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
342371
343372 return val1 >> __ffs64(operand2->constant);
344373 }
@@ -352,7 +381,7 @@ static u64 div_by_not_power_of_two(struct hist_field *hist_field, · 7 unmodified lines
352381 struct hist_field *operand1 = hist_field->operands[0];
353382 struct hist_field *operand2 = hist_field->operands[1];
354383
355− u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
384+ u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
356385
357386 return div64_u64(val1, operand2->constant);
358387 }
@@ -366,7 +395,7 @@ static u64 div_by_mult_and_shift(struct hist_field *hist_field, · 7 unmodified lines
366395 struct hist_field *operand1 = hist_field->operands[0];
367396 struct hist_field *operand2 = hist_field->operands[1];
368397
369− u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
398+ u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
370399
371400 /*
372401 * If the divisor is a constant, do a multiplication and shift instead.
@@ -400,8 +429,8 @@ static u64 hist_field_mult(struct hist_field *hist_field, · 27 unmodified lines
400429 struct hist_field *operand1 = hist_field->operands[0];
401430 struct hist_field *operand2 = hist_field->operands[1];
402431
403− u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
404− u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
432+ u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
433+ u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
405434
406435 return val1 * val2;
407436 }
@@ -414,7 +443,7 @@ static u64 hist_field_unary_minus(struct hist_field *hist_field, · 6 unmodified lines
414443 {
415444 struct hist_field *operand = hist_field->operands[0];
416445
417− s64 sval = (s64)operand->fn(operand, elt, buffer, rbe, event);
446+ s64 sval = (s64)hist_fn_call(operand, elt, buffer, rbe, event);
418447 u64 val = (u64)-sval;
419448
420449 return val;
@@ -657,19 +686,19 @@ struct snapshot_context { · 236 unmodified lines
657686 * Returns the specific division function to use if the divisor
658687 * is constant. This avoids extra branches when the trigger is hit.
659688 */
660−static hist_field_fn_t hist_field_get_div_fn(struct hist_field *divisor)
689+static enum hist_field_fn hist_field_get_div_fn(struct hist_field *divisor)
661690 {
662691 u64 div = divisor->constant;
663692
664693 if (!(div & (div - 1)))
665− return div_by_power_of_two;
694+ return HIST_FIELD_FN_DIV_POWER2;
666695
667696 /* If the divisor is too large, do a regular division */
668697 if (div > (1 << HIST_DIV_SHIFT))
669− return div_by_not_power_of_two;
698+ return HIST_FIELD_FN_DIV_NOT_POWER2;
670699
671700 divisor->div_multiplier = div64_u64((u64)(1 << HIST_DIV_SHIFT), div);
672− return div_by_mult_and_shift;
701+ return HIST_FIELD_FN_DIV_MULT_SHIFT;
673702 }
674703
675704 static void track_data_free(struct track_data *track_data)
@@ -1334,38 +1363,32 @@ static const char *hist_field_name(struct hist_field *field, · 658 unmodified lines
13341363 return field_name;
13351364 }
13361365
1337−static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1366+static enum hist_field_fn select_value_fn(int field_size, int field_is_signed)
13381367 {
1339− hist_field_fn_t fn = NULL;
1340−
13411368 switch (field_size) {
13421369 case 8:
13431370 if (field_is_signed)
1344− fn = hist_field_s64;
1371+ return HIST_FIELD_FN_S64;
13451372 else
1346− fn = hist_field_u64;
1347− break;
1373+ return HIST_FIELD_FN_U64;
13481374 case 4:
13491375 if (field_is_signed)
1350− fn = hist_field_s32;
1376+ return HIST_FIELD_FN_S32;
13511377 else
1352− fn = hist_field_u32;
1353− break;
1378+ return HIST_FIELD_FN_U32;
13541379 case 2:
13551380 if (field_is_signed)
1356− fn = hist_field_s16;
1381+ return HIST_FIELD_FN_S16;
13571382 else
1358− fn = hist_field_u16;
1359− break;
1383+ return HIST_FIELD_FN_U16;
13601384 case 1:
13611385 if (field_is_signed)
1362− fn = hist_field_s8;
1386+ return HIST_FIELD_FN_S8;
13631387 else
1364− fn = hist_field_u8;
1365− break;
1388+ return HIST_FIELD_FN_U8;
13661389 }
13671390
1368− return fn;
1391+ return HIST_FIELD_FN_NOP;
13691392 }
13701393
13711394 static int parse_map_size(char *str)
@@ -1922,19 +1945,19 @@ static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, · 550 unmodified lines
19221945 goto out; /* caller will populate */
19231946
19241947 if (flags & HIST_FIELD_FL_VAR_REF) {
1925− hist_field->fn = hist_field_var_ref;
1948+ hist_field->fn_num = HIST_FIELD_FN_VAR_REF;
19261949 goto out;
19271950 }
19281951
19291952 if (flags & HIST_FIELD_FL_HITCOUNT) {
1930− hist_field->fn = hist_field_counter;
1953+ hist_field->fn_num = HIST_FIELD_FN_COUNTER;
19311954 hist_field->size = sizeof(u64);
19321955 hist_field->type = "u64";
19331956 goto out;
19341957 }
19351958
19361959 if (flags & HIST_FIELD_FL_CONST) {
1937− hist_field->fn = hist_field_const;
1960+ hist_field->fn_num = HIST_FIELD_FN_CONST;
19381961 hist_field->size = sizeof(u64);
19391962 hist_field->type = kstrdup("u64", GFP_KERNEL);
19401963 if (!hist_field->type)
@@ -1943,14 +1966,14 @@ static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, · 2 unmodified lines
19431966 }
19441967
19451968 if (flags & HIST_FIELD_FL_STACKTRACE) {
1946− hist_field->fn = hist_field_none;
1969+ hist_field->fn_num = HIST_FIELD_FN_NOP;
19471970 goto out;
19481971 }
19491972
19501973 if (flags & (HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET)) {
19511974 unsigned long fl = flags & ~(HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET);
1952− hist_field->fn = flags & HIST_FIELD_FL_LOG2 ? hist_field_log2 :
1953− hist_field_bucket;
1975+ hist_field->fn_num = flags & HIST_FIELD_FL_LOG2 ? HIST_FIELD_FN_LOG2 :
1976+ HIST_FIELD_FN_BUCKET;
19541977 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
19551978 hist_field->size = hist_field->operands[0]->size;
19561979 hist_field->type = kstrdup_const(hist_field->operands[0]->type, GFP_KERNEL);
@@ -1960,14 +1983,14 @@ static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, · 3 unmodified lines
19601983 }
19611984
19621985 if (flags & HIST_FIELD_FL_TIMESTAMP) {
1963− hist_field->fn = hist_field_timestamp;
1986+ hist_field->fn_num = HIST_FIELD_FN_TIMESTAMP;
19641987 hist_field->size = sizeof(u64);
19651988 hist_field->type = "u64";
19661989 goto out;
19671990 }
19681991
19691992 if (flags & HIST_FIELD_FL_CPU) {
1970− hist_field->fn = hist_field_cpu;
1993+ hist_field->fn_num = HIST_FIELD_FN_CPU;
19711994 hist_field->size = sizeof(int);
19721995 hist_field->type = "unsigned int";
19731996 goto out;
@@ -1987,14 +2010,14 @@ static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, · 13 unmodified lines
19872010 goto free;
19882011
19892012 if (field->filter_type == FILTER_STATIC_STRING) {
1990− hist_field->fn = hist_field_string;
2013+ hist_field->fn_num = HIST_FIELD_FN_STRING;
19912014 hist_field->size = field->size;
19922015 } else if (field->filter_type == FILTER_DYN_STRING) {
1993− hist_field->fn = hist_field_dynstring;
2016+ hist_field->fn_num = HIST_FIELD_FN_DYNSTRING;
19942017 } else if (field->filter_type == FILTER_RDYN_STRING)
1995− hist_field->fn = hist_field_reldynstring;
2018+ hist_field->fn_num = HIST_FIELD_FN_RELDYNSTRING;
19962019 else
1997− hist_field->fn = hist_field_pstring;
2020+ hist_field->fn_num = HIST_FIELD_FN_PSTRING;
19982021 } else {
19992022 hist_field->size = field->size;
20002023 hist_field->is_signed = field->is_signed;
@@ -2002,9 +2025,9 @@ static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, · 1 unmodified line
20022025 if (!hist_field->type)
20032026 goto free;
20042027
2005− hist_field->fn = select_value_fn(field->size,
2006− field->is_signed);
2007− if (!hist_field->fn) {
2028+ hist_field->fn_num = select_value_fn(field->size,
2029+ field->is_signed);
2030+ if (hist_field->fn_num == HIST_FIELD_FN_NOP) {
20082031 destroy_hist_field(hist_field, 0);
20092032 return NULL;
20102033 }
@@ -2340,7 +2363,7 @@ static struct hist_field *create_alias(struct hist_trigger_data *hist_data, · 329 unmodified lines
23402363 if (!alias)
23412364 return NULL;
23422365
2343− alias->fn = var_ref->fn;
2366+ alias->fn_num = var_ref->fn_num;
23442367 alias->operands[0] = var_ref;
23452368
23462369 if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
@@ -2523,7 +2546,7 @@ static struct hist_field *parse_unary(struct hist_trigger_data *hist_data, · 176 unmodified lines
25232546
25242547 expr->flags |= operand1->flags &
25252548 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2526− expr->fn = hist_field_unary_minus;
2549+ expr->fn_num = HIST_FIELD_FN_UMINUS;
25272550 expr->operands[0] = operand1;
25282551 expr->size = operand1->size;
25292552 expr->is_signed = operand1->is_signed;
@@ -2595,7 +2618,7 @@ static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, · 65 unmodified lines
25952618 unsigned long operand_flags, operand2_flags;
25962619 int field_op, ret = -EINVAL;
25972620 char *sep, *operand1_str;
2598− hist_field_fn_t op_fn;
2621+ enum hist_field_fn op_fn;
25992622 bool combine_consts;
26002623
26012624 if (*n_subexprs > 3) {
@@ -2654,16 +2677,16 @@ static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, · 52 unmodified lines
26542677
26552678 switch (field_op) {
26562679 case FIELD_OP_MINUS:
2657− op_fn = hist_field_minus;
2680+ op_fn = HIST_FIELD_FN_MINUS;
26582681 break;
26592682 case FIELD_OP_PLUS:
2660− op_fn = hist_field_plus;
2683+ op_fn = HIST_FIELD_FN_PLUS;
26612684 break;
26622685 case FIELD_OP_DIV:
2663− op_fn = hist_field_div;
2686+ op_fn = HIST_FIELD_FN_DIV;
26642687 break;
26652688 case FIELD_OP_MULT:
2666− op_fn = hist_field_mult;
2689+ op_fn = HIST_FIELD_FN_MULT;
26672690 break;
26682691 default:
26692692 ret = -EINVAL;
@@ -2719,13 +2742,16 @@ static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, · 49 unmodified lines
27192742 op_fn = hist_field_get_div_fn(operand2);
27202743 }
27212744
2745+ expr->fn_num = op_fn;
2746+
27222747 if (combine_consts) {
27232748 if (var1)
27242749 expr->operands[0] = var1;
27252750 if (var2)
27262751 expr->operands[1] = var2;
27272752
2728− expr->constant = op_fn(expr, NULL, NULL, NULL, NULL);
2753+ expr->constant = hist_fn_call(expr, NULL, NULL, NULL, NULL);
2754+ expr->fn_num = HIST_FIELD_FN_CONST;
27292755
27302756 expr->operands[0] = NULL;
27312757 expr->operands[1] = NULL;
@@ -2739,8 +2765,6 @@ static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, · 7 unmodified lines
27392765
27402766 expr->name = expr_str(expr, 0);
27412767 } else {
2742− expr->fn = op_fn;
2743−
27442768 /* The operand sizes should be the same, so just pick one */
27452769 expr->size = operand1->size;
27462770 expr->is_signed = operand1->is_signed;
@@ -3065,7 +3089,7 @@ static inline void __update_field_vars(struct tracing_map_elt *elt, · 318 unmodified lines
30653089 struct hist_field *var = field_var->var;
30663090 struct hist_field *val = field_var->val;
30673091
3068− var_val = val->fn(val, elt, buffer, rbe, rec);
3092+ var_val = hist_fn_call(val, elt, buffer, rbe, rec);
30693093 var_idx = var->var.idx;
30703094
30713095 if (val->flags & HIST_FIELD_FL_STRING) {
@@ -4186,6 +4210,74 @@ static u64 hist_field_execname(struct hist_field *hist_field, · 1114 unmodified lines
41864210 return (u64)(unsigned long)(elt_data->comm);
41874211 }
41884212
4213+static u64 hist_fn_call(struct hist_field *hist_field,
4214+ struct tracing_map_elt *elt,
4215+ struct trace_buffer *buffer,
4216+ struct ring_buffer_event *rbe,
4217+ void *event)
4218+{
4219+ switch (hist_field->fn_num) {
4220+ case HIST_FIELD_FN_VAR_REF:
4221+ return hist_field_var_ref(hist_field, elt, buffer, rbe, event);
4222+ case HIST_FIELD_FN_COUNTER:
4223+ return hist_field_counter(hist_field, elt, buffer, rbe, event);
4224+ case HIST_FIELD_FN_CONST:
4225+ return hist_field_const(hist_field, elt, buffer, rbe, event);
4226+ case HIST_FIELD_FN_LOG2:
4227+ return hist_field_log2(hist_field, elt, buffer, rbe, event);
4228+ case HIST_FIELD_FN_BUCKET:
4229+ return hist_field_bucket(hist_field, elt, buffer, rbe, event);
4230+ case HIST_FIELD_FN_TIMESTAMP:
4231+ return hist_field_timestamp(hist_field, elt, buffer, rbe, event);
4232+ case HIST_FIELD_FN_CPU:
4233+ return hist_field_cpu(hist_field, elt, buffer, rbe, event);
4234+ case HIST_FIELD_FN_STRING:
4235+ return hist_field_string(hist_field, elt, buffer, rbe, event);
4236+ case HIST_FIELD_FN_DYNSTRING:
4237+ return hist_field_dynstring(hist_field, elt, buffer, rbe, event);
4238+ case HIST_FIELD_FN_RELDYNSTRING:
4239+ return hist_field_reldynstring(hist_field, elt, buffer, rbe, event);
4240+ case HIST_FIELD_FN_PSTRING:
4241+ return hist_field_pstring(hist_field, elt, buffer, rbe, event);
4242+ case HIST_FIELD_FN_S64:
4243+ return hist_field_s64(hist_field, elt, buffer, rbe, event);
4244+ case HIST_FIELD_FN_U64:
4245+ return hist_field_u64(hist_field, elt, buffer, rbe, event);
4246+ case HIST_FIELD_FN_S32:
4247+ return hist_field_s32(hist_field, elt, buffer, rbe, event);
4248+ case HIST_FIELD_FN_U32:
4249+ return hist_field_u32(hist_field, elt, buffer, rbe, event);
4250+ case HIST_FIELD_FN_S16:
4251+ return hist_field_s16(hist_field, elt, buffer, rbe, event);
4252+ case HIST_FIELD_FN_U16:
4253+ return hist_field_u16(hist_field, elt, buffer, rbe, event);
4254+ case HIST_FIELD_FN_S8:
4255+ return hist_field_s8(hist_field, elt, buffer, rbe, event);
4256+ case HIST_FIELD_FN_U8:
4257+ return hist_field_u8(hist_field, elt, buffer, rbe, event);
4258+ case HIST_FIELD_FN_UMINUS:
4259+ return hist_field_unary_minus(hist_field, elt, buffer, rbe, event);
4260+ case HIST_FIELD_FN_MINUS:
4261+ return hist_field_minus(hist_field, elt, buffer, rbe, event);
4262+ case HIST_FIELD_FN_PLUS:
4263+ return hist_field_plus(hist_field, elt, buffer, rbe, event);
4264+ case HIST_FIELD_FN_DIV:
4265+ return hist_field_div(hist_field, elt, buffer, rbe, event);
4266+ case HIST_FIELD_FN_MULT:
4267+ return hist_field_mult(hist_field, elt, buffer, rbe, event);
4268+ case HIST_FIELD_FN_DIV_POWER2:
4269+ return div_by_power_of_two(hist_field, elt, buffer, rbe, event);
4270+ case HIST_FIELD_FN_DIV_NOT_POWER2:
4271+ return div_by_not_power_of_two(hist_field, elt, buffer, rbe, event);
4272+ case HIST_FIELD_FN_DIV_MULT_SHIFT:
4273+ return div_by_mult_and_shift(hist_field, elt, buffer, rbe, event);
4274+ case HIST_FIELD_FN_EXECNAME:
4275+ return hist_field_execname(hist_field, elt, buffer, rbe, event);
4276+ default:
4277+ return 0;
4278+ }
4279+}
4280+
41894281 /* Convert a var that points to common_pid.execname to a string */
41904282 static void update_var_execname(struct hist_field *hist_field)
41914283 {
@@ -4197,7 +4289,7 @@ static void update_var_execname(struct hist_field *hist_field) · 5 unmodified lines
41974289 kfree_const(hist_field->type);
41984290 hist_field->type = "char[]";
41994291
4200− hist_field->fn = hist_field_execname;
4292+ hist_field->fn_num = HIST_FIELD_FN_EXECNAME;
42014293 }
42024294
42034295 static int create_var_field(struct hist_trigger_data *hist_data,
@@ -4956,7 +5048,7 @@ static void hist_trigger_elt_update(struct hist_trigger_data *hist_data, · 752 unmodified lines
49565048
49575049 for_each_hist_val_field(i, hist_data) {
49585050 hist_field = hist_data->fields[i];
4959− hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
5051+ hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec);
49605052 if (hist_field->flags & HIST_FIELD_FL_VAR) {
49615053 var_idx = hist_field->var.idx;
49625054
@@ -4987,7 +5079,7 @@ static void hist_trigger_elt_update(struct hist_trigger_data *hist_data, · 24 unmodified lines
49875079 for_each_hist_key_field(i, hist_data) {
49885080 hist_field = hist_data->fields[i];
49895081 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4990− hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
5082+ hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec);
49915083 var_idx = hist_field->var.idx;
49925084 tracing_map_set_var(elt, var_idx, hist_val);
49935085 }
@@ -5062,7 +5154,7 @@ static void event_hist_trigger(struct event_trigger_data *data, · 68 unmodified lines
50625154 HIST_STACKTRACE_SKIP);
50635155 key = entries;
50645156 } else {
5065− field_contents = key_field->fn(key_field, elt, buffer, rbe, rec);
5157+ field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec);
50665158 if (key_field->flags & HIST_FIELD_FL_STRING) {
50675159 key = (void *)(unsigned long)field_contents;
50685160 use_compound_key = true;
kernel/trace/trace_events_user.c+426−142
@@ -14,6 +14,7 @@ · 13 unmodified lines
1414 #include <linux/uio.h>
1515 #include <linux/ioctl.h>
1616 #include <linux/jhash.h>
17+#include <linux/refcount.h>
1718 #include <linux/trace_events.h>
1819 #include <linux/tracefs.h>
1920 #include <linux/types.h>
@@ -39,28 +40,69 @@ · 19 unmodified lines
3940 */
4041 #define MAX_PAGE_ORDER 0
4142 #define MAX_PAGES (1 << MAX_PAGE_ORDER)
42−#define MAX_EVENTS (MAX_PAGES * PAGE_SIZE)
43+#define MAX_BYTES (MAX_PAGES * PAGE_SIZE)
44+#define MAX_EVENTS (MAX_BYTES * 8)
4345
4446 /* Limit how long of an event name plus args within the subsystem. */
4547 #define MAX_EVENT_DESC 512
4648 #define EVENT_NAME(user_event) ((user_event)->tracepoint.name)
4749 #define MAX_FIELD_ARRAY_SIZE 1024
48−#define MAX_FIELD_ARG_NAME 256
4950
50−static char *register_page_data;
51+/*
52+ * The MAP_STATUS_* macros are used for taking a index and determining the
53+ * appropriate byte and the bit in the byte to set/reset for an event.
54+ *
55+ * The lower 3 bits of the index decide which bit to set.
56+ * The remaining upper bits of the index decide which byte to use for the bit.
57+ *
58+ * This is used when an event has a probe attached/removed to reflect live
59+ * status of the event wanting tracing or not to user-programs via shared
60+ * memory maps.
61+ */
62+#define MAP_STATUS_BYTE(index) ((index) >> 3)
63+#define MAP_STATUS_MASK(index) BIT((index) & 7)
64+
65+/*
66+ * Internal bits (kernel side only) to keep track of connected probes:
67+ * These are used when status is requested in text form about an event. These
68+ * bits are compared against an internal byte on the event to determine which
69+ * probes to print out to the user.
70+ *
71+ * These do not reflect the mapped bytes between the user and kernel space.
72+ */
73+#define EVENT_STATUS_FTRACE BIT(0)
74+#define EVENT_STATUS_PERF BIT(1)
75+#define EVENT_STATUS_OTHER BIT(7)
76+
77+/*
78+ * Stores the pages, tables, and locks for a group of events.
79+ * Each logical grouping of events has its own group, with a
80+ * matching page for status checks within user programs. This
81+ * allows for isolation of events to user programs by various
82+ * means.
83+ */
84+struct user_event_group {
85+ struct page *pages;
86+ char *register_page_data;
87+ char *system_name;
88+ struct hlist_node node;
89+ struct mutex reg_mutex;
90+ DECLARE_HASHTABLE(register_table, 8);
91+ DECLARE_BITMAP(page_bitmap, MAX_EVENTS);
92+};
5193
52−static DEFINE_MUTEX(reg_mutex);
53−static DEFINE_HASHTABLE(register_table, 4);
54−static DECLARE_BITMAP(page_bitmap, MAX_EVENTS);
94+/* Group for init_user_ns mapping, top-most group */
95+static struct user_event_group *init_group;
5596
5697 /*
5798 * Stores per-event properties, as users register events
5899 * within a file a user_event might be created if it does not
59100 * already exist. These are globally used and their lifetime
60101 * is tied to the refcnt member. These cannot go away until the
61− * refcnt reaches zero.
102+ * refcnt reaches one.
62103 */
63104 struct user_event {
105+ struct user_event_group *group;
64106 struct tracepoint tracepoint;
65107 struct trace_event_call call;
66108 struct trace_event_class class;
@@ -68,10 +110,11 @@ struct user_event { · 1 unmodified line
68110 struct hlist_node node;
69111 struct list_head fields;
70112 struct list_head validators;
71− atomic_t refcnt;
113+ refcount_t refcnt;
72114 int index;
73115 int flags;
74116 int min_size;
117+ char status;
75118 };
76119
77120 /*
@@ -86,6 +129,11 @@ struct user_event_refs { · 8 unmodified lines
86129 struct user_event *events[];
87130 };
88131
132+struct user_event_file_info {
133+ struct user_event_group *group;
134+ struct user_event_refs *refs;
135+};
136+
89137 #define VALIDATOR_ENSURE_NULL (1 << 0)
90138 #define VALIDATOR_REL (1 << 1)
91139
@@ -98,7 +146,8 @@ struct user_event_validator { · 6 unmodified lines
98146 typedef void (*user_event_func_t) (struct user_event *user, struct iov_iter *i,
99147 void *tpdata, bool *faulted);
100148
101−static int user_event_parse(char *name, char *args, char *flags,
149+static int user_event_parse(struct user_event_group *group, char *name,
150+ char *args, char *flags,
102151 struct user_event **newuser);
103152
104153 static u32 user_event_key(char *name)
@@ -106,6 +155,144 @@ static u32 user_event_key(char *name) · 1 unmodified line
106155 return jhash(name, strlen(name), 0);
107156 }
108157
158+static void set_page_reservations(char *pages, bool set)
159+{
160+ int page;
161+
162+ for (page = 0; page < MAX_PAGES; ++page) {
163+ void *addr = pages + (PAGE_SIZE * page);
164+
165+ if (set)
166+ SetPageReserved(virt_to_page(addr));
167+ else
168+ ClearPageReserved(virt_to_page(addr));
169+ }
170+}
171+
172+static void user_event_group_destroy(struct user_event_group *group)
173+{
174+ if (group->register_page_data)
175+ set_page_reservations(group->register_page_data, false);
176+
177+ if (group->pages)
178+ __free_pages(group->pages, MAX_PAGE_ORDER);
179+
180+ kfree(group->system_name);
181+ kfree(group);
182+}
183+
184+static char *user_event_group_system_name(struct user_namespace *user_ns)
185+{
186+ char *system_name;
187+ int len = sizeof(USER_EVENTS_SYSTEM) + 1;
188+
189+ if (user_ns != &init_user_ns) {
190+ /*
191+ * Unexpected at this point:
192+ * We only currently support init_user_ns.
193+ * When we enable more, this will trigger a failure so log.
194+ */
195+ pr_warn("user_events: Namespace other than init_user_ns!\n");
196+ return NULL;
197+ }
198+
199+ system_name = kmalloc(len, GFP_KERNEL);
200+
201+ if (!system_name)
202+ return NULL;
203+
204+ snprintf(system_name, len, "%s", USER_EVENTS_SYSTEM);
205+
206+ return system_name;
207+}
208+
209+static inline struct user_event_group
210+*user_event_group_from_user_ns(struct user_namespace *user_ns)
211+{
212+ if (user_ns == &init_user_ns)
213+ return init_group;
214+
215+ return NULL;
216+}
217+
218+static struct user_event_group *current_user_event_group(void)
219+{
220+ struct user_namespace *user_ns = current_user_ns();
221+ struct user_event_group *group = NULL;
222+
223+ while (user_ns) {
224+ group = user_event_group_from_user_ns(user_ns);
225+
226+ if (group)
227+ break;
228+
229+ user_ns = user_ns->parent;
230+ }
231+
232+ return group;
233+}
234+
235+static struct user_event_group
236+*user_event_group_create(struct user_namespace *user_ns)
237+{
238+ struct user_event_group *group;
239+
240+ group = kzalloc(sizeof(*group), GFP_KERNEL);
241+
242+ if (!group)
243+ return NULL;
244+
245+ group->system_name = user_event_group_system_name(user_ns);
246+
247+ if (!group->system_name)
248+ goto error;
249+
250+ group->pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, MAX_PAGE_ORDER);
251+
252+ if (!group->pages)
253+ goto error;
254+
255+ group->register_page_data = page_address(group->pages);
256+
257+ set_page_reservations(group->register_page_data, true);
258+
259+ /* Zero all bits beside 0 (which is reserved for failures) */
260+ bitmap_zero(group->page_bitmap, MAX_EVENTS);
261+ set_bit(0, group->page_bitmap);
262+
263+ mutex_init(&group->reg_mutex);
264+ hash_init(group->register_table);
265+
266+ return group;
267+error:
268+ if (group)
269+ user_event_group_destroy(group);
270+
271+ return NULL;
272+};
273+
274+static __always_inline
275+void user_event_register_set(struct user_event *user)
276+{
277+ int i = user->index;
278+
279+ user->group->register_page_data[MAP_STATUS_BYTE(i)] |= MAP_STATUS_MASK(i);
280+}
281+
282+static __always_inline
283+void user_event_register_clear(struct user_event *user)
284+{
285+ int i = user->index;
286+
287+ user->group->register_page_data[MAP_STATUS_BYTE(i)] &= ~MAP_STATUS_MASK(i);
288+}
289+
290+static __always_inline __must_check
291+bool user_event_last_ref(struct user_event *user)
292+{
293+ return refcount_read(&user->refcnt) == 1;
294+}
295+
109296 static __always_inline __must_check
110297 size_t copy_nofault(void *addr, size_t bytes, struct iov_iter *i)
111298 {
@@ -141,7 +328,8 @@ static struct list_head *user_event_get_fields(struct trace_event_call *call) · 29 unmodified lines
141328 *
142329 * Upon success user_event has its ref count increased by 1.
143330 */
144−static int user_event_parse_cmd(char *raw_command, struct user_event **newuser)
331+static int user_event_parse_cmd(struct user_event_group *group,
332+ char *raw_command, struct user_event **newuser)
145333 {
146334 char *name = raw_command;
147335 char *args = strpbrk(name, " ");
@@ -155,7 +343,7 @@ static int user_event_parse_cmd(char *raw_command, struct user_event **newuser) · 7 unmodified lines
155343 if (flags)
156344 *flags++ = '\0';
157345
158− return user_event_parse(name, args, flags, newuser);
346+ return user_event_parse(group, name, args, flags, newuser);
159347 }
160348
161349 static int user_field_array_size(const char *type)
@@ -277,7 +465,7 @@ static int user_event_add_field(struct user_event *user, const char *type, · 115 unmodified lines
277465 goto add_field;
278466
279467 add_validator:
280− if (strstr(type, "char") != 0)
468+ if (strstr(type, "char") != NULL)
281469 validator_flags |= VALIDATOR_ENSURE_NULL;
282470
283471 validator = kmalloc(sizeof(*validator), GFP_KERNEL);
@@ -458,7 +646,7 @@ static const char *user_field_format(const char *type) · 174 unmodified lines
458646 return "%d";
459647 if (strcmp(type, "unsigned char") == 0)
460648 return "%u";
461− if (strstr(type, "char[") != 0)
649+ if (strstr(type, "char[") != NULL)
462650 return "%s";
463651
464652 /* Unknown, likely struct, allowed treat as 64-bit */
@@ -479,10 +667,52 @@ static bool user_field_is_dyn_string(const char *type, const char **str_func) · 14 unmodified lines
479667
480668 return false;
481669 check:
482− return strstr(type, "char") != 0;
670+ return strstr(type, "char") != NULL;
483671 }
484672
485673 #define LEN_OR_ZERO (len ? len - pos : 0)
674+static int user_dyn_field_set_string(int argc, const char **argv, int *iout,
675+ char *buf, int len, bool *colon)
676+{
677+ int pos = 0, i = *iout;
678+
679+ *colon = false;
680+
681+ for (; i < argc; ++i) {
682+ if (i != *iout)
683+ pos += snprintf(buf + pos, LEN_OR_ZERO, " ");
684+
685+ pos += snprintf(buf + pos, LEN_OR_ZERO, "%s", argv[i]);
686+
687+ if (strchr(argv[i], ';')) {
688+ ++i;
689+ *colon = true;
690+ break;
691+ }
692+ }
693+
694+ /* Actual set, advance i */
695+ if (len != 0)
696+ *iout = i;
697+
698+ return pos + 1;
699+}
700+
701+static int user_field_set_string(struct ftrace_event_field *field,
702+ char *buf, int len, bool colon)
703+{
704+ int pos = 0;
705+
706+ pos += snprintf(buf + pos, LEN_OR_ZERO, "%s", field->type);
707+ pos += snprintf(buf + pos, LEN_OR_ZERO, " ");
708+ pos += snprintf(buf + pos, LEN_OR_ZERO, "%s", field->name);
709+
710+ if (colon)
711+ pos += snprintf(buf + pos, LEN_OR_ZERO, ";");
712+
713+ return pos + 1;
714+}
715+
486716 static int user_event_set_print_fmt(struct user_event *user, char *buf, int len)
487717 {
488718 struct ftrace_event_field *field, *next;
@@ -600,8 +830,8 @@ static int destroy_user_event(struct user_event *user) · 111 unmodified lines
600830
601831 dyn_event_remove(&user->devent);
602832
603− register_page_data[user->index] = 0;
604− clear_bit(user->index, page_bitmap);
833+ user_event_register_clear(user);
834+ clear_bit(user->index, user->group->page_bitmap);
605835 hash_del(&user->node);
606836
607837 user_event_destroy_validators(user);
@@ -612,16 +842,17 @@ static int destroy_user_event(struct user_event *user) · 4 unmodified lines
612842 return ret;
613843 }
614844
615−static struct user_event *find_user_event(char *name, u32 *outkey)
845+static struct user_event *find_user_event(struct user_event_group *group,
846+ char *name, u32 *outkey)
616847 {
617848 struct user_event *user;
618849 u32 key = user_event_key(name);
619850
620851 *outkey = key;
621852
622− hash_for_each_possible(register_table, user, node, key)
853+ hash_for_each_possible(group->register_table, user, node, key)
623854 if (!strcmp(EVENT_NAME(user), name)) {
624− atomic_inc(&user->refcnt);
855+ refcount_inc(&user->refcnt);
625856 return user;
626857 }
627858
@@ -779,7 +1010,12 @@ static void update_reg_page_for(struct user_event *user) · 151 unmodified lines
7791010 rcu_read_unlock_sched();
7801011 }
7811012
782− register_page_data[user->index] = status;
1013+ if (status)
1014+ user_event_register_set(user);
1015+ else
1016+ user_event_register_clear(user);
1017+
1018+ user->status = status;
7831019 }
7841020
7851021 /*
@@ -835,17 +1071,18 @@ static int user_event_reg(struct trace_event_call *call, · 49 unmodified lines
8351071
8361072 return ret;
8371073 inc:
838− atomic_inc(&user->refcnt);
1074+ refcount_inc(&user->refcnt);
8391075 update_reg_page_for(user);
8401076 return 0;
8411077 dec:
8421078 update_reg_page_for(user);
843− atomic_dec(&user->refcnt);
1079+ refcount_dec(&user->refcnt);
8441080 return 0;
8451081 }
8461082
8471083 static int user_event_create(const char *raw_command)
8481084 {
1085+ struct user_event_group *group;
8491086 struct user_event *user;
8501087 char *name;
8511088 int ret;
@@ -861,14 +1098,19 @@ static int user_event_create(const char *raw_command) · 9 unmodified lines
8611098 if (!name)
8621099 return -ENOMEM;
8631100
864− mutex_lock(&reg_mutex);
1101+ group = current_user_event_group();
8651102
866− ret = user_event_parse_cmd(name, &user);
1103+ if (!group)
1104+ return -ENOENT;
1105+
1106+ mutex_lock(&group->reg_mutex);
1107+
1108+ ret = user_event_parse_cmd(group, name, &user);
8671109
8681110 if (!ret)
869− atomic_dec(&user->refcnt);
1111+ refcount_dec(&user->refcnt);
8701112
871− mutex_unlock(&reg_mutex);
1113+ mutex_unlock(&group->reg_mutex);
8721114
8731115 if (ret)
8741116 kfree(name);
@@ -910,14 +1152,14 @@ static bool user_event_is_busy(struct dyn_event *ev) · 35 unmodified lines
9101152 {
9111153 struct user_event *user = container_of(ev, struct user_event, devent);
9121154
913− return atomic_read(&user->refcnt) != 0;
1155+ return !user_event_last_ref(user);
9141156 }
9151157
9161158 static int user_event_free(struct dyn_event *ev)
9171159 {
9181160 struct user_event *user = container_of(ev, struct user_event, devent);
9191161
920− if (atomic_read(&user->refcnt) != 0)
1162+ if (!user_event_last_ref(user))
9211163 return -EBUSY;
9221164
9231165 return destroy_user_event(user);
@@ -926,49 +1168,35 @@ static int user_event_free(struct dyn_event *ev) · 2 unmodified lines
9261168 static bool user_field_match(struct ftrace_event_field *field, int argc,
9271169 const char **argv, int *iout)
9281170 {
929− char *field_name, *arg_name;
930− int len, pos, i = *iout;
1171+ char *field_name = NULL, *dyn_field_name = NULL;
9311172 bool colon = false, match = false;
1173+ int dyn_len, len;
9321174
933− if (i >= argc)
1175+ if (*iout >= argc)
9341176 return false;
9351177
936− len = MAX_FIELD_ARG_NAME;
937− field_name = kmalloc(len, GFP_KERNEL);
938− arg_name = kmalloc(len, GFP_KERNEL);
939−
940− if (!arg_name || !field_name)
941− goto out;
942−
943− pos = 0;
944−
945− for (; i < argc; ++i) {
946− if (i != *iout)
947− pos += snprintf(arg_name + pos, len - pos, " ");
1178+ dyn_len = user_dyn_field_set_string(argc, argv, iout, dyn_field_name,
1179+ 0, &colon);
9481180
949− pos += snprintf(arg_name + pos, len - pos, argv[i]);
1181+ len = user_field_set_string(field, field_name, 0, colon);
9501182
951− if (strchr(argv[i], ';')) {
952− ++i;
953− colon = true;
954− break;
955− }
956− }
1183+ if (dyn_len != len)
1184+ return false;
9571185
958− pos = 0;
1186+ dyn_field_name = kmalloc(dyn_len, GFP_KERNEL);
1187+ field_name = kmalloc(len, GFP_KERNEL);
9591188
960− pos += snprintf(field_name + pos, len - pos, field->type);
961− pos += snprintf(field_name + pos, len - pos, " ");
962− pos += snprintf(field_name + pos, len - pos, field->name);
1189+ if (!dyn_field_name || !field_name)
1190+ goto out;
9631191
964− if (colon)
965− pos += snprintf(field_name + pos, len - pos, ";");
1192+ user_dyn_field_set_string(argc, argv, iout, dyn_field_name,
1193+ dyn_len, &colon);
9661194
967− *iout = i;
1195+ user_field_set_string(field, field_name, len, colon);
9681196
969− match = strcmp(arg_name, field_name) == 0;
1197+ match = strcmp(dyn_field_name, field_name) == 0;
9701198 out:
971− kfree(arg_name);
1199+ kfree(dyn_field_name);
9721200 kfree(field_name);
9731201
9741202 return match;
@@ -1036,7 +1264,8 @@ static int user_event_trace_register(struct user_event *user) · 61 unmodified lines
10361264 * The name buffer lifetime is owned by this method for success cases only.
10371265 * Upon success the returned user_event has its ref count increased by 1.
10381266 */
1039−static int user_event_parse(char *name, char *args, char *flags,
1267+static int user_event_parse(struct user_event_group *group, char *name,
1268+ char *args, char *flags,
10401269 struct user_event **newuser)
10411270 {
10421271 int ret;
@@ -1046,7 +1275,7 @@ static int user_event_parse(char *name, char *args, char *flags, · 3 unmodified lines
10461275
10471276 /* Prevent dyn_event from racing */
10481277 mutex_lock(&event_mutex);
1049− user = find_user_event(name, &key);
1278+ user = find_user_event(group, name, &key);
10501279 mutex_unlock(&event_mutex);
10511280
10521281 if (user) {
@@ -1059,7 +1288,7 @@ static int user_event_parse(char *name, char *args, char *flags, · 6 unmodified lines
10591288 return 0;
10601289 }
10611290
1062− index = find_first_zero_bit(page_bitmap, MAX_EVENTS);
1291+ index = find_first_zero_bit(group->page_bitmap, MAX_EVENTS);
10631292
10641293 if (index == MAX_EVENTS)
10651294 return -EMFILE;
@@ -1073,6 +1302,7 @@ static int user_event_parse(char *name, char *args, char *flags, · 7 unmodified lines
10731302 INIT_LIST_HEAD(&user->fields);
10741303 INIT_LIST_HEAD(&user->validators);
10751304
1305+ user->group = group;
10761306 user->tracepoint.name = name;
10771307
10781308 ret = user_event_parse_fields(user, args);
@@ -1091,8 +1321,8 @@ static int user_event_parse(char *name, char *args, char *flags, · 12 unmodified lines
10911321 user->call.flags = TRACE_EVENT_FL_TRACEPOINT;
10921322 user->call.tp = &user->tracepoint;
10931323 user->call.event.funcs = &user_event_funcs;
1324+ user->class.system = group->system_name;
10941325
1095− user->class.system = USER_EVENTS_SYSTEM;
10961326 user->class.fields_array = user_event_fields_array;
10971327 user->class.get_fields = user_event_get_fields;
10981328 user->class.reg = user_event_reg;
@@ -1110,13 +1340,13 @@ static int user_event_parse(char *name, char *args, char *flags, · 11 unmodified lines
11101340
11111341 user->index = index;
11121342
1113− /* Ensure we track ref */
1114− atomic_inc(&user->refcnt);
1343+ /* Ensure we track self ref and caller ref (2) */
1344+ refcount_set(&user->refcnt, 2);
11151345
11161346 dyn_event_init(&user->devent, &user_event_dops);
11171347 dyn_event_add(&user->devent, &user->call);
1118− set_bit(user->index, page_bitmap);
1119− hash_add(register_table, &user->node, key);
1348+ set_bit(user->index, group->page_bitmap);
1349+ hash_add(group->register_table, &user->node, key);
11201350
11211351 mutex_unlock(&event_mutex);
11221352
@@ -1134,32 +1364,20 @@ put_user: · 11 unmodified lines
11341364 /*
11351365 * Deletes a previously created event if it is no longer being used.
11361366 */
1137−static int delete_user_event(char *name)
1367+static int delete_user_event(struct user_event_group *group, char *name)
11381368 {
11391369 u32 key;
1140− int ret;
1141− struct user_event *user = find_user_event(name, &key);
1370+ struct user_event *user = find_user_event(group, name, &key);
11421371
11431372 if (!user)
11441373 return -ENOENT;
11451374
1146− /* Ensure we are the last ref */
1147− if (atomic_read(&user->refcnt) != 1) {
1148− ret = -EBUSY;
1149− goto put_ref;
1150− }
1151−
1152− ret = destroy_user_event(user);
1153−
1154− if (ret)
1155− goto put_ref;
1375+ refcount_dec(&user->refcnt);
11561376
1157− return ret;
1158−put_ref:
1159− /* No longer have this ref */
1160− atomic_dec(&user->refcnt);
1377+ if (!user_event_last_ref(user))
1378+ return -EBUSY;
11611379
1162− return ret;
1380+ return destroy_user_event(user);
11631381 }
11641382
11651383 /*
@@ -1167,6 +1385,7 @@ put_ref: · 1 unmodified line
11671385 */
11681386 static ssize_t user_events_write_core(struct file *file, struct iov_iter *i)
11691387 {
1388+ struct user_event_file_info *info = file->private_data;
11701389 struct user_event_refs *refs;
11711390 struct user_event *user = NULL;
11721391 struct tracepoint *tp;
@@ -1178,7 +1397,7 @@ static ssize_t user_events_write_core(struct file *file, struct iov_iter *i) · 5 unmodified lines
11781397
11791398 rcu_read_lock_sched();
11801399
1181− refs = rcu_dereference_sched(file->private_data);
1400+ refs = rcu_dereference_sched(info->refs);
11821401
11831402 /*
11841403 * The refs->events array is protected by RCU, and new items may be
@@ -1236,6 +1455,28 @@ static ssize_t user_events_write_core(struct file *file, struct iov_iter *i) · 51 unmodified lines
12361455 return ret;
12371456 }
12381457
1458+static int user_events_open(struct inode *node, struct file *file)
1459+{
1460+ struct user_event_group *group;
1461+ struct user_event_file_info *info;
1462+
1463+ group = current_user_event_group();
1464+
1465+ if (!group)
1466+ return -ENOENT;
1467+
1468+ info = kzalloc(sizeof(*info), GFP_KERNEL);
1469+
1470+ if (!info)
1471+ return -ENOMEM;
1472+
1473+ info->group = group;
1474+
1475+ file->private_data = info;
1476+
1477+ return 0;
1478+}
1479+
12391480 static ssize_t user_events_write(struct file *file, const char __user *ubuf,
12401481 size_t count, loff_t *ppos)
12411482 {
@@ -1245,7 +1486,8 @@ static ssize_t user_events_write(struct file *file, const char __user *ubuf, · 3 unmodified lines
12451486 if (unlikely(*ppos != 0))
12461487 return -EFAULT;
12471488
1248− if (unlikely(import_single_range(READ, (char *)ubuf, count, &iov, &i)))
1489+ if (unlikely(import_single_range(WRITE, (char __user *)ubuf,
1490+ count, &iov, &i)))
12491491 return -EFAULT;
12501492
12511493 return user_events_write_core(file, &i);
@@ -1256,13 +1498,15 @@ static ssize_t user_events_write_iter(struct kiocb *kp, struct iov_iter *i) · 4 unmodified lines
12561498 return user_events_write_core(kp->ki_filp, i);
12571499 }
12581500
1259−static int user_events_ref_add(struct file *file, struct user_event *user)
1501+static int user_events_ref_add(struct user_event_file_info *info,
1502+ struct user_event *user)
12601503 {
1504+ struct user_event_group *group = info->group;
12611505 struct user_event_refs *refs, *new_refs;
12621506 int i, size, count = 0;
12631507
1264− refs = rcu_dereference_protected(file->private_data,
1265− lockdep_is_held(&reg_mutex));
1508+ refs = rcu_dereference_protected(info->refs,
1509+ lockdep_is_held(&group->reg_mutex));
12661510
12671511 if (refs) {
12681512 count = refs->count;
@@ -1286,9 +1530,9 @@ static int user_events_ref_add(struct file *file, struct user_event *user) · 17 unmodified lines
12861530
12871531 new_refs->events[i] = user;
12881532
1289− atomic_inc(&user->refcnt);
1533+ refcount_inc(&user->refcnt);
12901534
1291− rcu_assign_pointer(file->private_data, new_refs);
1535+ rcu_assign_pointer(info->refs, new_refs);
12921536
12931537 if (refs)
12941538 kfree_rcu(refs, rcu);
@@ -1309,13 +1553,24 @@ static long user_reg_get(struct user_reg __user *ureg, struct user_reg *kreg) · 14 unmodified lines
13091553 if (size > PAGE_SIZE)
13101554 return -E2BIG;
13111555
1312− return copy_struct_from_user(kreg, sizeof(*kreg), ureg, size);
1556+ if (size < offsetofend(struct user_reg, write_index))
1557+ return -EINVAL;
1558+
1559+ ret = copy_struct_from_user(kreg, sizeof(*kreg), ureg, size);
1560+
1561+ if (ret)
1562+ return ret;
1563+
1564+ kreg->size = size;
1565+
1566+ return 0;
13131567 }
13141568
13151569 /*
13161570 * Registers a user_event on behalf of a user process.
13171571 */
1318−static long user_events_ioctl_reg(struct file *file, unsigned long uarg)
1572+static long user_events_ioctl_reg(struct user_event_file_info *info,
1573+ unsigned long uarg)
13191574 {
13201575 struct user_reg __user *ureg = (struct user_reg __user *)uarg;
13211576 struct user_reg reg;
@@ -1336,24 +1591,24 @@ static long user_events_ioctl_reg(struct file *file, unsigned long uarg) · 14 unmodified lines
13361591 return ret;
13371592 }
13381593
1339− ret = user_event_parse_cmd(name, &user);
1594+ ret = user_event_parse_cmd(info->group, name, &user);
13401595
13411596 if (ret) {
13421597 kfree(name);
13431598 return ret;
13441599 }
13451600
1346− ret = user_events_ref_add(file, user);
1601+ ret = user_events_ref_add(info, user);
13471602
13481603 /* No longer need parse ref, ref_add either worked or not */
1349− atomic_dec(&user->refcnt);
1604+ refcount_dec(&user->refcnt);
13501605
13511606 /* Positive number is index and valid */
13521607 if (ret < 0)
13531608 return ret;
13541609
13551610 put_user((u32)ret, &ureg->write_index);
1356− put_user(user->index, &ureg->status_index);
1611+ put_user(user->index, &ureg->status_bit);
13571612
13581613 return 0;
13591614 }
@@ -1361,7 +1616,8 @@ static long user_events_ioctl_reg(struct file *file, unsigned long uarg) · 1 unmodified line
13611616 /*
13621617 * Deletes a user_event on behalf of a user process.
13631618 */
1364−static long user_events_ioctl_del(struct file *file, unsigned long uarg)
1619+static long user_events_ioctl_del(struct user_event_file_info *info,
1620+ unsigned long uarg)
13651621 {
13661622 void __user *ubuf = (void __user *)uarg;
13671623 char *name;
@@ -1374,7 +1630,7 @@ static long user_events_ioctl_del(struct file *file, unsigned long uarg) · 6 unmodified lines
13741630
13751631 /* event_mutex prevents dyn_event from racing */
13761632 mutex_lock(&event_mutex);
1377− ret = delete_user_event(name);
1633+ ret = delete_user_event(info->group, name);
13781634 mutex_unlock(&event_mutex);
13791635
13801636 kfree(name);
@@ -1388,19 +1644,21 @@ static long user_events_ioctl_del(struct file *file, unsigned long uarg) · 7 unmodified lines
13881644 static long user_events_ioctl(struct file *file, unsigned int cmd,
13891645 unsigned long uarg)
13901646 {
1647+ struct user_event_file_info *info = file->private_data;
1648+ struct user_event_group *group = info->group;
13911649 long ret = -ENOTTY;
13921650
13931651 switch (cmd) {
13941652 case DIAG_IOCSREG:
1395− mutex_lock(&reg_mutex);
1396− ret = user_events_ioctl_reg(file, uarg);
1397− mutex_unlock(&reg_mutex);
1653+ mutex_lock(&group->reg_mutex);
1654+ ret = user_events_ioctl_reg(info, uarg);
1655+ mutex_unlock(&group->reg_mutex);
13981656 break;
13991657
14001658 case DIAG_IOCSDEL:
1401− mutex_lock(&reg_mutex);
1402− ret = user_events_ioctl_del(file, uarg);
1403− mutex_unlock(&reg_mutex);
1659+ mutex_lock(&group->reg_mutex);
1660+ ret = user_events_ioctl_del(info, uarg);
1661+ mutex_unlock(&group->reg_mutex);
14041662 break;
14051663 }
14061664
@@ -1412,17 +1670,24 @@ static long user_events_ioctl(struct file *file, unsigned int cmd, · 5 unmodified lines
14121670 */
14131671 static int user_events_release(struct inode *node, struct file *file)
14141672 {
1673+ struct user_event_file_info *info = file->private_data;
1674+ struct user_event_group *group;
14151675 struct user_event_refs *refs;
14161676 struct user_event *user;
14171677 int i;
14181678
1679+ if (!info)
1680+ return -EINVAL;
1681+
1682+ group = info->group;
1683+
14191684 /*
14201685 * Ensure refs cannot change under any situation by taking the
14211686 * register mutex during the final freeing of the references.
14221687 */
1423− mutex_lock(&reg_mutex);
1688+ mutex_lock(&group->reg_mutex);
14241689
1425− refs = file->private_data;
1690+ refs = info->refs;
14261691
14271692 if (!refs)
14281693 goto out;
@@ -1436,37 +1701,56 @@ static int user_events_release(struct inode *node, struct file *file) · 7 unmodified lines
14361701 user = refs->events[i];
14371702
14381703 if (user)
1439− atomic_dec(&user->refcnt);
1704+ refcount_dec(&user->refcnt);
14401705 }
14411706 out:
14421707 file->private_data = NULL;
14431708
1444− mutex_unlock(&reg_mutex);
1709+ mutex_unlock(&group->reg_mutex);
14451710
14461711 kfree(refs);
1712+ kfree(info);
14471713
14481714 return 0;
14491715 }
14501716
14511717 static const struct file_operations user_data_fops = {
1718+ .open = user_events_open,
14521719 .write = user_events_write,
14531720 .write_iter = user_events_write_iter,
14541721 .unlocked_ioctl = user_events_ioctl,
14551722 .release = user_events_release,
14561723 };
14571724
1725+static struct user_event_group *user_status_group(struct file *file)
1726+{
1727+ struct seq_file *m = file->private_data;
1728+
1729+ if (!m)
1730+ return NULL;
1731+
1732+ return m->private;
1733+}
1734+
14581735 /*
14591736 * Maps the shared page into the user process for checking if event is enabled.
14601737 */
14611738 static int user_status_mmap(struct file *file, struct vm_area_struct *vma)
14621739 {
1740+ char *pages;
1741+ struct user_event_group *group = user_status_group(file);
14631742 unsigned long size = vma->vm_end - vma->vm_start;
14641743
1465− if (size != MAX_EVENTS)
1744+ if (size != MAX_BYTES)
1745+ return -EINVAL;
1746+
1747+ if (!group)
14661748 return -EINVAL;
14671749
1750+ pages = group->register_page_data;
1751+
14681752 return remap_pfn_range(vma, vma->vm_start,
1469− virt_to_phys(register_page_data) >> PAGE_SHIFT,
1753+ virt_to_phys(pages) >> PAGE_SHIFT,
14701754 size, vm_get_page_prot(VM_READ));
14711755 }
14721756
@@ -1490,14 +1774,18 @@ static void user_seq_stop(struct seq_file *m, void *p) · 17 unmodified lines
14901774
14911775 static int user_seq_show(struct seq_file *m, void *p)
14921776 {
1777+ struct user_event_group *group = m->private;
14931778 struct user_event *user;
14941779 char status;
14951780 int i, active = 0, busy = 0, flags;
14961781
1497− mutex_lock(&reg_mutex);
1782+ if (!group)
1783+ return -EINVAL;
1784+
1785+ mutex_lock(&group->reg_mutex);
14981786
1499− hash_for_each(register_table, i, user, node) {
1500− status = register_page_data[user->index];
1787+ hash_for_each(group->register_table, i, user, node) {
1788+ status = user->status;
15011789 flags = user->flags;
15021790
15031791 seq_printf(m, "%d:%s", user->index, EVENT_NAME(user));
@@ -1520,7 +1808,7 @@ static int user_seq_show(struct seq_file *m, void *p) · 16 unmodified lines
15201808 active++;
15211809 }
15221810
1523− mutex_unlock(&reg_mutex);
1811+ mutex_unlock(&group->reg_mutex);
15241812
15251813 seq_puts(m, "\n");
15261814 seq_printf(m, "Active: %d\n", active);
@@ -1539,7 +1827,24 @@ static const struct seq_operations user_seq_ops = { · 12 unmodified lines
15391827
15401828 static int user_status_open(struct inode *node, struct file *file)
15411829 {
1542− return seq_open(file, &user_seq_ops);
1830+ struct user_event_group *group;
1831+ int ret;
1832+
1833+ group = current_user_event_group();
1834+
1835+ if (!group)
1836+ return -ENOENT;
1837+
1838+ ret = seq_open(file, &user_seq_ops);
1839+
1840+ if (!ret) {
1841+ /* Chain group to seq_file */
1842+ struct seq_file *m = file->private_data;
1843+
1844+ m->private = group;
1845+ }
1846+
1847+ return ret;
15431848 }
15441849
15451850 static const struct file_operations user_status_fops = {
@@ -1580,42 +1885,21 @@ err: · 34 unmodified lines
15801885 return -ENODEV;
15811886 }
15821887
1583−static void set_page_reservations(bool set)
1584−{
1585− int page;
1586−
1587− for (page = 0; page < MAX_PAGES; ++page) {
1588− void *addr = register_page_data + (PAGE_SIZE * page);
1589−
1590− if (set)
1591− SetPageReserved(virt_to_page(addr));
1592− else
1593− ClearPageReserved(virt_to_page(addr));
1594− }
1595−}
1596−
15971888 static int __init trace_events_user_init(void)
15981889 {
1599− struct page *pages;
16001890 int ret;
16011891
1602− /* Zero all bits beside 0 (which is reserved for failures) */
1603− bitmap_zero(page_bitmap, MAX_EVENTS);
1604− set_bit(0, page_bitmap);
1892+ init_group = user_event_group_create(&init_user_ns);
16051893
1606− pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, MAX_PAGE_ORDER);
1607− if (!pages)
1894+ if (!init_group)
16081895 return -ENOMEM;
1609− register_page_data = page_address(pages);
1610−
1611− set_page_reservations(true);
16121896
16131897 ret = create_user_tracefs();
16141898
16151899 if (ret) {
16161900 pr_warn("user_events could not register with tracefs\n");
1617− set_page_reservations(false);
1618− __free_pages(pages, MAX_PAGE_ORDER);
1901+ user_event_group_destroy(init_group);
1902+ init_group = NULL;
16191903 return ret;
16201904 }
16211905
kernel/trace/trace_osnoise.c+2−1
@@ -1786,8 +1786,9 @@ static int start_per_cpu_kthreads(void) · 1785 unmodified lines
17861786 for_each_cpu(cpu, current_mask) {
17871787 retval = start_kthread(cpu);
17881788 if (retval) {
1789+ cpus_read_unlock();
17891790 stop_per_cpu_kthreads();
1790− break;
1791+ return retval;
17911792 }
17921793 }
17931794
kernel/trace/trace_probe.h+2−1
@@ -445,7 +445,8 @@ extern int traceprobe_define_arg_fields(struct trace_event_call *event_call, · 444 unmodified lines
445445 C(SAME_PROBE, "There is already the exact same probe event"),\
446446 C(NO_EVENT_INFO, "This requires both group and event name to attach"),\
447447 C(BAD_ATTACH_EVENT, "Attached event does not exist"),\
448− C(BAD_ATTACH_ARG, "Attached event does not have this field"),
448+ C(BAD_ATTACH_ARG, "Attached event does not have this field"),\
449+ C(NO_EP_FILTER, "No filter rule after 'if'"),
449450
450451 #undef C
451452 #define C(a, b) TP_ERR_##a
kernel/trace/tracing_map.c+2−3
@@ -961,7 +961,7 @@ create_sort_entry(void *key, struct tracing_map_elt *elt) · 960 unmodified lines
961961 static void detect_dups(struct tracing_map_sort_entry **sort_entries,
962962 int n_entries, unsigned int key_size)
963963 {
964− unsigned int dups = 0, total_dups = 0;
964+ unsigned int total_dups = 0;
965965 int i;
966966 void *key;
967967
@@ -974,11 +974,10 @@ static void detect_dups(struct tracing_map_sort_entry **sort_entries, · 6 unmodified lines
974974 key = sort_entries[0]->key;
975975 for (i = 1; i < n_entries; i++) {
976976 if (!memcmp(sort_entries[i]->key, key, key_size)) {
977− dups++; total_dups++;
977+ total_dups++;
978978 continue;
979979 }
980980 key = sort_entries[i]->key;
981− dups = 0;
982981 }
983982
984983 WARN_ONCE(total_dups > 0,
kernel/tracepoint.c+6−8
@@ -640,7 +640,6 @@ static void tp_module_going_check_quiescent(struct tracepoint *tp, void *priv) · 639 unmodified lines
640640 static int tracepoint_module_coming(struct module *mod)
641641 {
642642 struct tp_module *tp_mod;
643− int ret = 0;
644643
645644 if (!mod->num_tracepoints)
646645 return 0;
@@ -652,19 +651,18 @@ static int tracepoint_module_coming(struct module *mod) · 5 unmodified lines
652651 */
653652 if (trace_module_has_bad_taint(mod))
654653 return 0;
655− mutex_lock(&tracepoint_module_list_mutex);
654+
656655 tp_mod = kmalloc(sizeof(struct tp_module), GFP_KERNEL);
657− if (!tp_mod) {
658− ret = -ENOMEM;
659− goto end;
660− }
656+ if (!tp_mod)
657+ return -ENOMEM;
661658 tp_mod->mod = mod;
659+
660+ mutex_lock(&tracepoint_module_list_mutex);
662661 list_add_tail(&tp_mod->list, &tracepoint_module_list);
663662 blocking_notifier_call_chain(&tracepoint_notify_list,
664663 MODULE_STATE_COMING, tp_mod);
665−end:
666664 mutex_unlock(&tracepoint_module_list_mutex);
667− return ret;
665+ return 0;
668666 }
669667
670668 static void tracepoint_module_going(struct module *mod)
samples/user_events/example.c+18−7
@@ -12,13 +12,21 @@ · 11 unmodified lines
1212 #include <fcntl.h>
1313 #include <stdio.h>
1414 #include <unistd.h>
15+#include <asm/bitsperlong.h>
16+#include <endian.h>
1517 #include <linux/user_events.h>
1618
19+#if __BITS_PER_LONG == 64
20+#define endian_swap(x) htole64(x)
21+#else
22+#define endian_swap(x) htole32(x)
23+#endif
24+
1725 /* Assumes debugfs is mounted */
1826 const char *data_file = "/sys/kernel/debug/tracing/user_events_data";
1927 const char *status_file = "/sys/kernel/debug/tracing/user_events_status";
2028
21−static int event_status(char **status)
29+static int event_status(long **status)
2230 {
2331 int fd = open(status_file, O_RDONLY);
2432
@@ -33,7 +41,8 @@ static int event_status(char **status) · 8 unmodified lines
3341 return 0;
3442 }
3543
36−static int event_reg(int fd, const char *command, int *status, int *write)
44+static int event_reg(int fd, const char *command, long *index, long *mask,
45+ int *write)
3746 {
3847 struct user_reg reg = {0};
3948
@@ -43,7 +52,8 @@ static int event_reg(int fd, const char *command, int *status, int *write) · 3 unmodified lines
4352 if (ioctl(fd, DIAG_IOCSREG, &reg) == -1)
4453 return -1;
4554
46− *status = reg.status_index;
55+ *index = reg.status_bit / __BITS_PER_LONG;
56+ *mask = endian_swap(1L << (reg.status_bit % __BITS_PER_LONG));
4757 *write = reg.write_index;
4858
4959 return 0;
@@ -51,8 +61,9 @@ static int event_reg(int fd, const char *command, int *status, int *write) · 1 unmodified line
5161
5262 int main(int argc, char **argv)
5363 {
54− int data_fd, status, write;
55− char *status_page;
64+ int data_fd, write;
65+ long index, mask;
66+ long *status_page;
5667 struct iovec io[2];
5768 __u32 count = 0;
5869
@@ -61,7 +72,7 @@ int main(int argc, char **argv) · 2 unmodified lines
6172
6273 data_fd = open(data_file, O_RDWR);
6374
64− if (event_reg(data_fd, "test u32 count", &status, &write) == -1)
75+ if (event_reg(data_fd, "test u32 count", &index, &mask, &write) == -1)
6576 return errno;
6677
6778 /* Setup iovec */
@@ -75,7 +86,7 @@ ask: · 7 unmodified lines
7586 getchar();
7687
7788 /* Check if anyone is listening */
78− if (status_page[status]) {
89+ if (status_page[index] & mask) {
7990 /* Yep, trace out our data */
8091 writev(data_fd, (const struct iovec *)io, 2);
8192
tools/testing/selftests/ftrace/test.d/dynevent/eprobes_syntax_errors.tc+27−0
@@ -0,0 +1,27 @@
1+#!/bin/sh
2+# SPDX-License-Identifier: GPL-2.0
3+# description: Event probe event parser error log check
4+# requires: dynamic_events events/syscalls/sys_enter_openat "<attached-group>.<attached-event> [<args>]":README error_log
5+
6+check_error() { # command-with-error-pos-by-^
7+ ftrace_errlog_check 'event_probe' "$1" 'dynamic_events'
8+}
9+
10+check_error 'e ^a.' # NO_EVENT_INFO
11+check_error 'e ^.b' # NO_EVENT_INFO
12+check_error 'e ^a.b' # BAD_ATTACH_EVENT
13+check_error 'e syscalls/sys_enter_openat ^foo' # BAD_ATTACH_ARG
14+check_error 'e:^/bar syscalls/sys_enter_openat' # NO_GROUP_NAME
15+check_error 'e:^12345678901234567890123456789012345678901234567890123456789012345/bar syscalls/sys_enter_openat' # GROUP_TOO_LONG
16+
17+check_error 'e:^foo.1/bar syscalls/sys_enter_openat' # BAD_GROUP_NAME
18+check_error 'e:^ syscalls/sys_enter_openat' # NO_EVENT_NAME
19+check_error 'e:foo/^12345678901234567890123456789012345678901234567890123456789012345 syscalls/sys_enter_openat' # EVENT_TOO_LONG
20+check_error 'e:foo/^bar.1 syscalls/sys_enter_openat' # BAD_EVENT_NAME
21+
22+check_error 'e:foo/bar syscalls/sys_enter_openat arg=^dfd' # BAD_FETCH_ARG
23+check_error 'e:foo/bar syscalls/sys_enter_openat ^arg=$foo' # BAD_ATTACH_ARG
24+
25+check_error 'e:foo/bar syscalls/sys_enter_openat if ^' # NO_EP_FILTER
26+
27+exit 0
tools/testing/selftests/user_events/ftrace_test.c+39−8
@@ -22,6 +22,11 @@ const char *enable_file = "/sys/kernel/debug/tracing/events/user_events/__test_e · 21 unmodified lines
2222 const char *trace_file = "/sys/kernel/debug/tracing/trace";
2323 const char *fmt_file = "/sys/kernel/debug/tracing/events/user_events/__test_event/format";
2424
25+static inline int status_check(char *status_page, int status_bit)
26+{
27+ return status_page[status_bit >> 3] & (1 << (status_bit & 7));
28+}
29+
2530 static int trace_bytes(void)
2631 {
2732 int fd = open(trace_file, O_RDONLY);
@@ -197,12 +202,12 @@ TEST_F(user, register_events) { · 169 unmodified lines
197202 /* Register should work */
198203 ASSERT_EQ(0, ioctl(self->data_fd, DIAG_IOCSREG, &reg));
199204 ASSERT_EQ(0, reg.write_index);
200− ASSERT_NE(0, reg.status_index);
205+ ASSERT_NE(0, reg.status_bit);
201206
202207 /* Multiple registers should result in same index */
203208 ASSERT_EQ(0, ioctl(self->data_fd, DIAG_IOCSREG, &reg));
204209 ASSERT_EQ(0, reg.write_index);
205− ASSERT_NE(0, reg.status_index);
210+ ASSERT_NE(0, reg.status_bit);
206211
207212 /* Ensure disabled */
208213 self->enable_fd = open(enable_file, O_RDWR);
@@ -212,15 +217,15 @@ TEST_F(user, register_events) { · 3 unmodified lines
212217 /* MMAP should work and be zero'd */
213218 ASSERT_NE(MAP_FAILED, status_page);
214219 ASSERT_NE(NULL, status_page);
215− ASSERT_EQ(0, status_page[reg.status_index]);
220+ ASSERT_EQ(0, status_check(status_page, reg.status_bit));
216221
217222 /* Enable event and ensure bits updated in status */
218223 ASSERT_NE(-1, write(self->enable_fd, "1", sizeof("1")))
219− ASSERT_EQ(EVENT_STATUS_FTRACE, status_page[reg.status_index]);
224+ ASSERT_NE(0, status_check(status_page, reg.status_bit));
220225
221226 /* Disable event and ensure bits updated in status */
222227 ASSERT_NE(-1, write(self->enable_fd, "0", sizeof("0")))
223− ASSERT_EQ(0, status_page[reg.status_index]);
228+ ASSERT_EQ(0, status_check(status_page, reg.status_bit));
224229
225230 /* File still open should return -EBUSY for delete */
226231 ASSERT_EQ(-1, ioctl(self->data_fd, DIAG_IOCSDEL, "__test_event"));
@@ -240,6 +245,8 @@ TEST_F(user, write_events) { · 13 unmodified lines
240245 struct iovec io[3];
241246 __u32 field1, field2;
242247 int before = 0, after = 0;
248+ int page_size = sysconf(_SC_PAGESIZE);
249+ char *status_page;
243250
244251 reg.size = sizeof(reg);
245252 reg.name_args = (__u64)"__test_event u32 field1; u32 field2";
@@ -254,10 +261,18 @@ TEST_F(user, write_events) { · 8 unmodified lines
254261 io[2].iov_base = &field2;
255262 io[2].iov_len = sizeof(field2);
256263
264+ status_page = mmap(NULL, page_size, PROT_READ, MAP_SHARED,
265+ self->status_fd, 0);
266+
257267 /* Register should work */
258268 ASSERT_EQ(0, ioctl(self->data_fd, DIAG_IOCSREG, &reg));
259269 ASSERT_EQ(0, reg.write_index);
260− ASSERT_NE(0, reg.status_index);
270+ ASSERT_NE(0, reg.status_bit);
271+
272+ /* MMAP should work and be zero'd */
273+ ASSERT_NE(MAP_FAILED, status_page);
274+ ASSERT_NE(NULL, status_page);
275+ ASSERT_EQ(0, status_check(status_page, reg.status_bit));
261276
262277 /* Write should fail on invalid slot with ENOENT */
263278 io[0].iov_base = &field2;
@@ -271,6 +286,9 @@ TEST_F(user, write_events) { · 7 unmodified lines
271286 self->enable_fd = open(enable_file, O_RDWR);
272287 ASSERT_NE(-1, write(self->enable_fd, "1", sizeof("1")))
273288
289+ /* Event should now be enabled */
290+ ASSERT_NE(0, status_check(status_page, reg.status_bit));
291+
274292 /* Write should make it out to ftrace buffers */
275293 before = trace_bytes();
276294 ASSERT_NE(-1, writev(self->data_fd, (const struct iovec *)io, 3));
@@ -298,7 +316,7 @@ TEST_F(user, write_fault) { · 21 unmodified lines
298316 /* Register should work */
299317 ASSERT_EQ(0, ioctl(self->data_fd, DIAG_IOCSREG, &reg));
300318 ASSERT_EQ(0, reg.write_index);
301− ASSERT_NE(0, reg.status_index);
319+ ASSERT_NE(0, reg.status_bit);
302320
303321 /* Write should work normally */
304322 ASSERT_NE(-1, writev(self->data_fd, (const struct iovec *)io, 2));
@@ -315,6 +333,11 @@ TEST_F(user, write_validator) { · 10 unmodified lines
315333 int loc, bytes;
316334 char data[8];
317335 int before = 0, after = 0;
336+ int page_size = sysconf(_SC_PAGESIZE);
337+ char *status_page;
338+
339+ status_page = mmap(NULL, page_size, PROT_READ, MAP_SHARED,
340+ self->status_fd, 0);
318341
319342 reg.size = sizeof(reg);
320343 reg.name_args = (__u64)"__test_event __rel_loc char[] data";
@@ -322,7 +345,12 @@ TEST_F(user, write_validator) { · 1 unmodified line
322345 /* Register should work */
323346 ASSERT_EQ(0, ioctl(self->data_fd, DIAG_IOCSREG, &reg));
324347 ASSERT_EQ(0, reg.write_index);
325− ASSERT_NE(0, reg.status_index);
348+ ASSERT_NE(0, reg.status_bit);
349+
350+ /* MMAP should work and be zero'd */
351+ ASSERT_NE(MAP_FAILED, status_page);
352+ ASSERT_NE(NULL, status_page);
353+ ASSERT_EQ(0, status_check(status_page, reg.status_bit));
326354
327355 io[0].iov_base = &reg.write_index;
328356 io[0].iov_len = sizeof(reg.write_index);
@@ -340,6 +368,9 @@ TEST_F(user, write_validator) { · 11 unmodified lines
340368 self->enable_fd = open(enable_file, O_RDWR);
341369 ASSERT_NE(-1, write(self->enable_fd, "1", sizeof("1")))
342370
371+ /* Event should now be enabled */
372+ ASSERT_NE(0, status_check(status_page, reg.status_bit));
373+
343374 /* Full in-bounds write should work */
344375 before = trace_bytes();
345376 loc = DYN_LOC(0, bytes);
tools/testing/selftests/user_events/perf_test.c+8−3
@@ -35,6 +35,11 @@ static long perf_event_open(struct perf_event_attr *pe, pid_t pid, · 34 unmodified lines
3535 return syscall(__NR_perf_event_open, pe, pid, cpu, group_fd, flags);
3636 }
3737
38+static inline int status_check(char *status_page, int status_bit)
39+{
40+ return status_page[status_bit >> 3] & (1 << (status_bit & 7));
41+}
42+
3843 static int get_id(void)
3944 {
4045 FILE *fp = fopen(id_file, "r");
@@ -120,8 +125,8 @@ TEST_F(user, perf_write) { · 79 unmodified lines
120125 /* Register should work */
121126 ASSERT_EQ(0, ioctl(self->data_fd, DIAG_IOCSREG, &reg));
122127 ASSERT_EQ(0, reg.write_index);
123− ASSERT_NE(0, reg.status_index);
124− ASSERT_EQ(0, status_page[reg.status_index]);
128+ ASSERT_NE(0, reg.status_bit);
129+ ASSERT_EQ(0, status_check(status_page, reg.status_bit));
125130
126131 /* Id should be there */
127132 id = get_id();
@@ -144,7 +149,7 @@ TEST_F(user, perf_write) { · 16 unmodified lines
144149 ASSERT_NE(MAP_FAILED, perf_page);
145150
146151 /* Status should be updated */
147− ASSERT_EQ(EVENT_STATUS_PERF, status_page[reg.status_index]);
152+ ASSERT_NE(0, status_check(status_page, reg.status_bit));
148153
149154 event.index = reg.write_index;
150155 event.field1 = 0xc001;
tools/verification/dot2/dot2k_templates/main_global.c+4−4
@@ -27,7 +27,7 @@ · 26 unmodified lines
2727 *
2828 * The rv monitor reference is needed for the monitor declaration.
2929 */
30−struct rv_monitor rv_MODEL_NAME;
30+static struct rv_monitor rv_MODEL_NAME;
3131 DECLARE_DA_MON_GLOBAL(MODEL_NAME, MIN_TYPE);
3232
3333 /*
@@ -63,7 +63,7 @@ TRACEPOINT_DETACH · 29 unmodified lines
6363 /*
6464 * This is the monitor register section.
6565 */
66−struct rv_monitor rv_MODEL_NAME = {
66+static struct rv_monitor rv_MODEL_NAME = {
6767 .name = "MODEL_NAME",
6868 .description = "auto-generated MODEL_NAME",
6969 .enable = enable_MODEL_NAME,
@@ -72,13 +72,13 @@ struct rv_monitor rv_MODEL_NAME = { · 2 unmodified lines
7272 .enabled = 0,
7373 };
7474
75−static int register_MODEL_NAME(void)
75+static int __init register_MODEL_NAME(void)
7676 {
7777 rv_register_monitor(&rv_MODEL_NAME);
7878 return 0;
7979 }
8080
81−static void unregister_MODEL_NAME(void)
81+static void __exit unregister_MODEL_NAME(void)
8282 {
8383 rv_unregister_monitor(&rv_MODEL_NAME);
8484 }
tools/verification/dot2/dot2k_templates/main_per_cpu.c+4−4
@@ -27,7 +27,7 @@ · 26 unmodified lines
2727 *
2828 * The rv monitor reference is needed for the monitor declaration.
2929 */
30−struct rv_monitor rv_MODEL_NAME;
30+static struct rv_monitor rv_MODEL_NAME;
3131 DECLARE_DA_MON_PER_CPU(MODEL_NAME, MIN_TYPE);
3232
3333 /*
@@ -63,7 +63,7 @@ TRACEPOINT_DETACH · 29 unmodified lines
6363 /*
6464 * This is the monitor register section.
6565 */
66−struct rv_monitor rv_MODEL_NAME = {
66+static struct rv_monitor rv_MODEL_NAME = {
6767 .name = "MODEL_NAME",
6868 .description = "auto-generated MODEL_NAME",
6969 .enable = enable_MODEL_NAME,
@@ -72,13 +72,13 @@ struct rv_monitor rv_MODEL_NAME = { · 2 unmodified lines
7272 .enabled = 0,
7373 };
7474
75−static int register_MODEL_NAME(void)
75+static int __init register_MODEL_NAME(void)
7676 {
7777 rv_register_monitor(&rv_MODEL_NAME);
7878 return 0;
7979 }
8080
81−static void unregister_MODEL_NAME(void)
81+static void __exit unregister_MODEL_NAME(void)
8282 {
8383 rv_unregister_monitor(&rv_MODEL_NAME);
8484 }
tools/verification/dot2/dot2k_templates/main_per_task.c+4−4
@@ -27,7 +27,7 @@ · 26 unmodified lines
2727 *
2828 * The rv monitor reference is needed for the monitor declaration.
2929 */
30−struct rv_monitor rv_MODEL_NAME;
30+static struct rv_monitor rv_MODEL_NAME;
3131 DECLARE_DA_MON_PER_TASK(MODEL_NAME, MIN_TYPE);
3232
3333 /*
@@ -63,7 +63,7 @@ TRACEPOINT_DETACH · 29 unmodified lines
6363 /*
6464 * This is the monitor register section.
6565 */
66−struct rv_monitor rv_MODEL_NAME = {
66+static struct rv_monitor rv_MODEL_NAME = {
6767 .name = "MODEL_NAME",
6868 .description = "auto-generated MODEL_NAME",
6969 .enable = enable_MODEL_NAME,
@@ -72,13 +72,13 @@ struct rv_monitor rv_MODEL_NAME = { · 2 unmodified lines
7272 .enabled = 0,
7373 };
7474
75−static int register_MODEL_NAME(void)
75+static int __init register_MODEL_NAME(void)
7676 {
7777 rv_register_monitor(&rv_MODEL_NAME);
7878 return 0;
7979 }
8080
81−static void unregister_MODEL_NAME(void)
81+static void __exit unregister_MODEL_NAME(void)
8282 {
8383 rv_unregister_monitor(&rv_MODEL_NAME);
8484 }