// SPDX-License-Identifier: GPL-2.0/* * Fprobe-based tracing events * Copyright (C) 2022 Google LLC. */#define pr_fmt(fmt) "trace_fprobe: " fmt #include <linux/fprobe.h>#include <linux/list.h>#include <linux/module.h>#include <linux/mutex.h>#include <linux/rculist.h>#include <linux/security.h>#include <linux/tracepoint.h>#include <linux/uaccess.h> #include <asm/ptrace.h> #include "trace_dynevent.h"#include "trace_probe.h"#include "trace_probe_kernel.h"#include "trace_probe_tmpl.h" #define FPROBE_EVENT_SYSTEM "fprobes"#define TRACEPOINT_EVENT_SYSTEM "tracepoints"#define RETHOOK_MAXACTIVE_MAX 4096 static int trace_fprobe_create(const char *raw_command);static int trace_fprobe_show(struct seq_file *m, struct dyn_event *ev);static int trace_fprobe_release(struct dyn_event *ev);static bool trace_fprobe_is_busy(struct dyn_event *ev);static bool trace_fprobe_match(const char *system, const char *event, int argc, const char **argv, struct dyn_event *ev); static struct dyn_event_operations trace_fprobe_ops = { .create = trace_fprobe_create, .show = trace_fprobe_show, .is_busy = trace_fprobe_is_busy, .free = trace_fprobe_release, .match = trace_fprobe_match,}; /* List of tracepoint_user */static LIST_HEAD(tracepoint_user_list);static DEFINE_MUTEX(tracepoint_user_mutex); /* While living tracepoint_user, @tpoint can be NULL and @refcount != 0. */struct tracepoint_user { struct list_head list; const char *name; struct tracepoint *tpoint; unsigned int refcount;}; /* NOTE: you must lock tracepoint_user_mutex. */#define for_each_tracepoint_user(tuser) \ list_for_each_entry(tuser, &tracepoint_user_list, list) static int tracepoint_user_register(struct tracepoint_user *tuser){ struct tracepoint *tpoint = tuser->tpoint; if (!tpoint) return 0; return tracepoint_probe_register_prio_may_exist(tpoint, tpoint->probestub, NULL, 0);} static void tracepoint_user_unregister(struct tracepoint_user *tuser){ if (!tuser->tpoint) return; WARN_ON_ONCE(tracepoint_probe_unregister(tuser->tpoint, tuser->tpoint->probestub, NULL)); tuser->tpoint = NULL;} static unsigned long tracepoint_user_ip(struct tracepoint_user *tuser){ if (!tuser->tpoint) return 0UL; return (unsigned long)tuser->tpoint->probestub;} static void __tracepoint_user_free(struct tracepoint_user *tuser){ if (!tuser) return; kfree(tuser->name); kfree(tuser);} DEFINE_FREE(tuser_free, struct tracepoint_user *, __tracepoint_user_free(_T)) static struct tracepoint_user *__tracepoint_user_init(const char *name, struct tracepoint *tpoint){ struct tracepoint_user *tuser __free(tuser_free) = NULL; int ret; tuser = kzalloc_obj(*tuser); if (!tuser) return NULL; tuser->name = kstrdup(name, GFP_KERNEL); if (!tuser->name) return NULL; /* Register tracepoint if it is loaded. */ if (tpoint) { tuser->tpoint = tpoint; ret = tracepoint_user_register(tuser); if (ret) return ERR_PTR(ret); } tuser->refcount = 1; INIT_LIST_HEAD(&tuser->list); list_add(&tuser->list, &tracepoint_user_list); return_ptr(tuser);} static struct tracepoint *find_tracepoint(const char *tp_name, struct module **tp_mod); /* * Get tracepoint_user if exist, or allocate new one and register it. * If tracepoint is on a module, get its refcounter too. * This returns errno or NULL (not loaded yet) or tracepoint_user. */static struct tracepoint_user *tracepoint_user_find_get(const char *name, struct module **pmod){ struct module *mod __free(module_put) = NULL; struct tracepoint_user *tuser; struct tracepoint *tpoint; if (!name || !pmod) return ERR_PTR(-EINVAL); /* Get and lock the module which has tracepoint. */ tpoint = find_tracepoint(name, &mod); guard(mutex)(&tracepoint_user_mutex); /* Search existing tracepoint_user */ for_each_tracepoint_user(tuser) { if (!strcmp(tuser->name, name)) { tuser->refcount++; *pmod = no_free_ptr(mod); return tuser; } } /* The corresponding tracepoint_user is not found. */ tuser = __tracepoint_user_init(name, tpoint); if (!IS_ERR_OR_NULL(tuser)) *pmod = no_free_ptr(mod); return tuser;} static void tracepoint_user_put(struct tracepoint_user *tuser){ scoped_guard(mutex, &tracepoint_user_mutex) { if (--tuser->refcount > 0) return; list_del(&tuser->list); tracepoint_user_unregister(tuser); } __tracepoint_user_free(tuser);} DEFINE_FREE(tuser_put, struct tracepoint_user *, if (!IS_ERR_OR_NULL(_T)) tracepoint_user_put(_T)) /* * Fprobe event core functions */ /* * @tprobe is true for tracepoint probe. * @tuser can be NULL if the trace_fprobe is disabled or the tracepoint is not * loaded with a module. If @tuser != NULL, this trace_fprobe is enabled. */struct trace_fprobe { struct dyn_event devent; struct fprobe fp; const char *symbol; bool tprobe; struct tracepoint_user *tuser; struct trace_probe tp;}; static bool is_trace_fprobe(struct dyn_event *ev){ return ev->ops == &trace_fprobe_ops;} static struct trace_fprobe *to_trace_fprobe(struct dyn_event *ev){ return container_of(ev, struct trace_fprobe, devent);} /** * for_each_trace_fprobe - iterate over the trace_fprobe list * @pos: the struct trace_fprobe * for each entry * @dpos: the struct dyn_event * to use as a loop cursor */#define for_each_trace_fprobe(pos, dpos) \ for_each_dyn_event(dpos) \ if (is_trace_fprobe(dpos) && (pos = to_trace_fprobe(dpos))) static bool trace_fprobe_is_return(struct trace_fprobe *tf){ return tf->fp.exit_handler != NULL;} static bool trace_fprobe_is_tracepoint(struct trace_fprobe *tf){ return tf->tprobe;} static const char *trace_fprobe_symbol(struct trace_fprobe *tf){ return tf->symbol ? tf->symbol : "unknown";} static bool trace_fprobe_is_busy(struct dyn_event *ev){ struct trace_fprobe *tf = to_trace_fprobe(ev); return trace_probe_is_enabled(&tf->tp);} static bool trace_fprobe_match_command_head(struct trace_fprobe *tf, int argc, const char **argv){ char buf[MAX_ARGSTR_LEN + 1]; if (!argc) return true; snprintf(buf, sizeof(buf), "%s", trace_fprobe_symbol(tf)); if (strcmp(buf, argv[0])) return false; argc--; argv++; return trace_probe_match_command_args(&tf->tp, argc, argv);} static bool trace_fprobe_match(const char *system, const char *event, int argc, const char **argv, struct dyn_event *ev){ struct trace_fprobe *tf = to_trace_fprobe(ev); if (event[0] != '\0' && strcmp(trace_probe_name(&tf->tp), event)) return false; if (system && strcmp(trace_probe_group_name(&tf->tp), system)) return false; return trace_fprobe_match_command_head(tf, argc, argv);} static bool trace_fprobe_is_registered(struct trace_fprobe *tf){ return fprobe_is_registered(&tf->fp);} /* * Note that we don't verify the fetch_insn code, since it does not come * from user space. */static intprocess_fetch_insn(struct fetch_insn *code, void *rec, void *edata, void *dest, void *base){ struct ftrace_regs *fregs = rec; unsigned long val; int ret; retry: /* 1st stage: get value from context */ switch (code->op) { case FETCH_OP_STACK: val = ftrace_regs_get_kernel_stack_nth(fregs, code->param); break; case FETCH_OP_STACKP: val = ftrace_regs_get_stack_pointer(fregs); break; case FETCH_OP_RETVAL: val = ftrace_regs_get_return_value(fregs); break;#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API case FETCH_OP_ARG: val = ftrace_regs_get_argument(fregs, code->param); break; case FETCH_OP_EDATA: val = *(unsigned long *)((unsigned long)edata + code->offset); break;#endif case FETCH_NOP_SYMBOL: /* Ignore a place holder */ code++; goto retry; default: ret = process_common_fetch_insn(code, &val); if (ret < 0) return ret; } code++; return process_fetch_insn_bottom(code, val, dest, base);}NOKPROBE_SYMBOL(process_fetch_insn) /* function entry handler */static nokprobe_inline void__fentry_trace_func(struct trace_fprobe *tf, unsigned long entry_ip, struct ftrace_regs *fregs, struct trace_event_file *trace_file){ struct fentry_trace_entry_head *entry; struct trace_event_call *call = trace_probe_event_call(&tf->tp); struct trace_event_buffer fbuffer; int dsize; if (WARN_ON_ONCE(call != trace_file->event_call)) return; if (trace_trigger_soft_disabled(trace_file)) return; dsize = __get_data_size(&tf->tp, fregs, NULL); entry = trace_event_buffer_reserve(&fbuffer, trace_file, sizeof(*entry) + tf->tp.size + dsize); if (!entry) return; fbuffer.regs = ftrace_get_regs(fregs); entry = fbuffer.entry = ring_buffer_event_data(fbuffer.event); entry->ip = entry_ip; store_trace_args(&entry[1], &tf->tp, fregs, NULL, sizeof(*entry), dsize); trace_event_buffer_commit(&fbuffer);} static voidfentry_trace_func(struct trace_fprobe *tf, unsigned long entry_ip, struct ftrace_regs *fregs){ struct event_file_link *link; trace_probe_for_each_link_rcu(link, &tf->tp) __fentry_trace_func(tf, entry_ip, fregs, link->file);}NOKPROBE_SYMBOL(fentry_trace_func); static nokprobe_inlinevoid store_fprobe_entry_data(void *edata, struct trace_probe *tp, struct ftrace_regs *fregs){ struct probe_entry_arg *earg = tp->entry_arg; unsigned long val = 0; int i; if (!earg) return; for (i = 0; i < earg->size; i++) { struct fetch_insn *code = &earg->code[i]; switch (code->op) { case FETCH_OP_ARG: val = ftrace_regs_get_argument(fregs, code->param); break; case FETCH_OP_ST_EDATA: *(unsigned long *)((unsigned long)edata + code->offset) = val; break; case FETCH_OP_END: goto end; default: break; } }end: return;} /* function exit handler */static int trace_fprobe_entry_handler(struct fprobe *fp, unsigned long entry_ip, unsigned long ret_ip, struct ftrace_regs *fregs, void *entry_data){ struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp); if (tf->tp.entry_arg) store_fprobe_entry_data(entry_data, &tf->tp, fregs); return 0;}NOKPROBE_SYMBOL(trace_fprobe_entry_handler) static nokprobe_inline void__fexit_trace_func(struct trace_fprobe *tf, unsigned long entry_ip, unsigned long ret_ip, struct ftrace_regs *fregs, void *entry_data, struct trace_event_file *trace_file){ struct fexit_trace_entry_head *entry; struct trace_event_buffer fbuffer; struct trace_event_call *call = trace_probe_event_call(&tf->tp); int dsize; if (WARN_ON_ONCE(call != trace_file->event_call)) return; if (trace_trigger_soft_disabled(trace_file)) return; dsize = __get_data_size(&tf->tp, fregs, entry_data); entry = trace_event_buffer_reserve(&fbuffer, trace_file, sizeof(*entry) + tf->tp.size + dsize); if (!entry) return; fbuffer.regs = ftrace_get_regs(fregs); entry = fbuffer.entry = ring_buffer_event_data(fbuffer.event); entry->func = entry_ip; entry->ret_ip = ret_ip; store_trace_args(&entry[1], &tf->tp, fregs, entry_data, sizeof(*entry), dsize); trace_event_buffer_commit(&fbuffer);} static voidfexit_trace_func(struct trace_fprobe *tf, unsigned long entry_ip, unsigned long ret_ip, struct ftrace_regs *fregs, void *entry_data){ struct event_file_link *link; trace_probe_for_each_link_rcu(link, &tf->tp) __fexit_trace_func(tf, entry_ip, ret_ip, fregs, entry_data, link->file);}NOKPROBE_SYMBOL(fexit_trace_func); #ifdef CONFIG_PERF_EVENTS static int fentry_perf_func(struct trace_fprobe *tf, unsigned long entry_ip, struct ftrace_regs *fregs){ struct trace_event_call *call = trace_probe_event_call(&tf->tp); struct fentry_trace_entry_head *entry; struct hlist_head *head; int size, __size, dsize; struct pt_regs *regs; int rctx; head = this_cpu_ptr(call->perf_events); if (hlist_empty(head)) return 0; dsize = __get_data_size(&tf->tp, fregs, NULL); __size = sizeof(*entry) + tf->tp.size + dsize; size = ALIGN(__size + sizeof(u32), sizeof(u64)); size -= sizeof(u32); entry = perf_trace_buf_alloc(size, ®s, &rctx); if (!entry) return 0; regs = ftrace_fill_perf_regs(fregs, regs); entry->ip = entry_ip; memset(&entry[1], 0, dsize); store_trace_args(&entry[1], &tf->tp, fregs, NULL, sizeof(*entry), dsize); perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, head, NULL); return 0;}NOKPROBE_SYMBOL(fentry_perf_func); static voidfexit_perf_func(struct trace_fprobe *tf, unsigned long entry_ip, unsigned long ret_ip, struct ftrace_regs *fregs, void *entry_data){ struct trace_event_call *call = trace_probe_event_call(&tf->tp); struct fexit_trace_entry_head *entry; struct hlist_head *head; int size, __size, dsize; struct pt_regs *regs; int rctx; head = this_cpu_ptr(call->perf_events); if (hlist_empty(head)) return; Lines 1–500 of 1589. Load more
// SPDX-License-Identifier: GPL-2.0 /* * Fprobe-based tracing events * Copyright (C) 2022 Google LLC. */ #define pr_fmt(fmt) "trace_fprobe: " fmt #include <linux/fprobe.h> #include <linux/list.h> #include <linux/module.h> #include <linux/mutex.h> #include <linux/rculist.h> #include <linux/security.h> #include <linux/tracepoint.h> #include <linux/uaccess.h> #include <asm/ptrace.h> #include "trace_dynevent.h" #include "trace_probe.h" #include "trace_probe_kernel.h" #include "trace_probe_tmpl.h" #define FPROBE_EVENT_SYSTEM "fprobes" #define TRACEPOINT_EVENT_SYSTEM "tracepoints" #define RETHOOK_MAXACTIVE_MAX 4096 static int trace_fprobe_create(const char *raw_command); static int trace_fprobe_show(struct seq_file *m, struct dyn_event *ev); static int trace_fprobe_release(struct dyn_event *ev); static bool trace_fprobe_is_busy(struct dyn_event *ev); static bool trace_fprobe_match(const char *system, const char *event, int argc, const char **argv, struct dyn_event *ev); static struct dyn_event_operations trace_fprobe_ops = { .create = trace_fprobe_create, .show = trace_fprobe_show, .is_busy = trace_fprobe_is_busy, .free = trace_fprobe_release, .match = trace_fprobe_match, }; /* List of tracepoint_user */ static LIST_HEAD(tracepoint_user_list); static DEFINE_MUTEX(tracepoint_user_mutex); /* While living tracepoint_user, @tpoint can be NULL and @refcount != 0. */ struct tracepoint_user { struct list_head list; const char *name; struct tracepoint *tpoint; unsigned int refcount; }; /* NOTE: you must lock tracepoint_user_mutex. */ #define for_each_tracepoint_user(tuser) \ list_for_each_entry(tuser, &tracepoint_user_list, list) static int tracepoint_user_register(struct tracepoint_user *tuser) { struct tracepoint *tpoint = tuser->tpoint; if (!tpoint) return 0; return tracepoint_probe_register_prio_may_exist(tpoint, tpoint->probestub, NULL, 0); } static void tracepoint_user_unregister(struct tracepoint_user *tuser) { if (!tuser->tpoint) return; WARN_ON_ONCE(tracepoint_probe_unregister(tuser->tpoint, tuser->tpoint->probestub, NULL)); tuser->tpoint = NULL; } static unsigned long tracepoint_user_ip(struct tracepoint_user *tuser) { if (!tuser->tpoint) return 0UL; return (unsigned long)tuser->tpoint->probestub; } static void __tracepoint_user_free(struct tracepoint_user *tuser) { if (!tuser) return; kfree(tuser->name); kfree(tuser); } DEFINE_FREE(tuser_free, struct tracepoint_user *, __tracepoint_user_free(_T)) static struct tracepoint_user *__tracepoint_user_init(const char *name, struct tracepoint *tpoint) { struct tracepoint_user *tuser __free(tuser_free) = NULL; int ret; tuser = kzalloc_obj(*tuser); if (!tuser) return NULL; tuser->name = kstrdup(name, GFP_KERNEL); if (!tuser->name) return NULL; /* Register tracepoint if it is loaded. */ if (tpoint) { tuser->tpoint = tpoint; ret = tracepoint_user_register(tuser); if (ret) return ERR_PTR(ret); } tuser->refcount = 1; INIT_LIST_HEAD(&tuser->list); list_add(&tuser->list, &tracepoint_user_list); return_ptr(tuser); } static struct tracepoint *find_tracepoint(const char *tp_name, struct module **tp_mod); /* * Get tracepoint_user if exist, or allocate new one and register it. * If tracepoint is on a module, get its refcounter too. * This returns errno or NULL (not loaded yet) or tracepoint_user. */ static struct tracepoint_user *tracepoint_user_find_get(const char *name, struct module **pmod) { struct module *mod __free(module_put) = NULL; struct tracepoint_user *tuser; struct tracepoint *tpoint; if (!name || !pmod) return ERR_PTR(-EINVAL); /* Get and lock the module which has tracepoint. */ tpoint = find_tracepoint(name, &mod); guard(mutex)(&tracepoint_user_mutex); /* Search existing tracepoint_user */ for_each_tracepoint_user(tuser) { if (!strcmp(tuser->name, name)) { tuser->refcount++; *pmod = no_free_ptr(mod); return tuser; } } /* The corresponding tracepoint_user is not found. */ tuser = __tracepoint_user_init(name, tpoint); if (!IS_ERR_OR_NULL(tuser)) *pmod = no_free_ptr(mod); return tuser; } static void tracepoint_user_put(struct tracepoint_user *tuser) { scoped_guard(mutex, &tracepoint_user_mutex) { if (--tuser->refcount > 0) return; list_del(&tuser->list); tracepoint_user_unregister(tuser); } __tracepoint_user_free(tuser); } DEFINE_FREE(tuser_put, struct tracepoint_user *, if (!IS_ERR_OR_NULL(_T)) tracepoint_user_put(_T)) /* * Fprobe event core functions */ /* * @tprobe is true for tracepoint probe. * @tuser can be NULL if the trace_fprobe is disabled or the tracepoint is not * loaded with a module. If @tuser != NULL, this trace_fprobe is enabled. */ struct trace_fprobe { struct dyn_event devent; struct fprobe fp; const char *symbol; bool tprobe; struct tracepoint_user *tuser; struct trace_probe tp; }; static bool is_trace_fprobe(struct dyn_event *ev) { return ev->ops == &trace_fprobe_ops; } static struct trace_fprobe *to_trace_fprobe(struct dyn_event *ev) { return container_of(ev, struct trace_fprobe, devent); } /** * for_each_trace_fprobe - iterate over the trace_fprobe list * @pos: the struct trace_fprobe * for each entry * @dpos: the struct dyn_event * to use as a loop cursor */ #define for_each_trace_fprobe(pos, dpos) \ for_each_dyn_event(dpos) \ if (is_trace_fprobe(dpos) && (pos = to_trace_fprobe(dpos))) static bool trace_fprobe_is_return(struct trace_fprobe *tf) { return tf->fp.exit_handler != NULL; } static bool trace_fprobe_is_tracepoint(struct trace_fprobe *tf) { return tf->tprobe; } static const char *trace_fprobe_symbol(struct trace_fprobe *tf) { return tf->symbol ? tf->symbol : "unknown"; } static bool trace_fprobe_is_busy(struct dyn_event *ev) { struct trace_fprobe *tf = to_trace_fprobe(ev); return trace_probe_is_enabled(&tf->tp); } static bool trace_fprobe_match_command_head(struct trace_fprobe *tf, int argc, const char **argv) { char buf[MAX_ARGSTR_LEN + 1]; if (!argc) return true; snprintf(buf, sizeof(buf), "%s", trace_fprobe_symbol(tf)); if (strcmp(buf, argv[0])) return false; argc--; argv++; return trace_probe_match_command_args(&tf->tp, argc, argv); } static bool trace_fprobe_match(const char *system, const char *event, int argc, const char **argv, struct dyn_event *ev) { struct trace_fprobe *tf = to_trace_fprobe(ev); if (event[0] != '\0' && strcmp(trace_probe_name(&tf->tp), event)) return false; if (system && strcmp(trace_probe_group_name(&tf->tp), system)) return false; return trace_fprobe_match_command_head(tf, argc, argv); } static bool trace_fprobe_is_registered(struct trace_fprobe *tf) { return fprobe_is_registered(&tf->fp); } /* * Note that we don't verify the fetch_insn code, since it does not come * from user space. */ static int process_fetch_insn(struct fetch_insn *code, void *rec, void *edata, void *dest, void *base) { struct ftrace_regs *fregs = rec; unsigned long val; int ret; retry: /* 1st stage: get value from context */ switch (code->op) { case FETCH_OP_STACK: val = ftrace_regs_get_kernel_stack_nth(fregs, code->param); break; case FETCH_OP_STACKP: val = ftrace_regs_get_stack_pointer(fregs); break; case FETCH_OP_RETVAL: val = ftrace_regs_get_return_value(fregs); break; #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API case FETCH_OP_ARG: val = ftrace_regs_get_argument(fregs, code->param); break; case FETCH_OP_EDATA: val = *(unsigned long *)((unsigned long)edata + code->offset); break; #endif case FETCH_NOP_SYMBOL: /* Ignore a place holder */ code++; goto retry; default: ret = process_common_fetch_insn(code, &val); if (ret < 0) return ret; } code++; return process_fetch_insn_bottom(code, val, dest, base); } NOKPROBE_SYMBOL(process_fetch_insn) /* function entry handler */ static nokprobe_inline void __fentry_trace_func(struct trace_fprobe *tf, unsigned long entry_ip, struct ftrace_regs *fregs, struct trace_event_file *trace_file) { struct fentry_trace_entry_head *entry; struct trace_event_call *call = trace_probe_event_call(&tf->tp); struct trace_event_buffer fbuffer; int dsize; if (WARN_ON_ONCE(call != trace_file->event_call)) return; if (trace_trigger_soft_disabled(trace_file)) return; dsize = __get_data_size(&tf->tp, fregs, NULL); entry = trace_event_buffer_reserve(&fbuffer, trace_file, sizeof(*entry) + tf->tp.size + dsize); if (!entry) return; fbuffer.regs = ftrace_get_regs(fregs); entry = fbuffer.entry = ring_buffer_event_data(fbuffer.event); entry->ip = entry_ip; store_trace_args(&entry[1], &tf->tp, fregs, NULL, sizeof(*entry), dsize); trace_event_buffer_commit(&fbuffer); } static void fentry_trace_func(struct trace_fprobe *tf, unsigned long entry_ip, struct ftrace_regs *fregs) { struct event_file_link *link; trace_probe_for_each_link_rcu(link, &tf->tp) __fentry_trace_func(tf, entry_ip, fregs, link->file); } NOKPROBE_SYMBOL(fentry_trace_func); static nokprobe_inline void store_fprobe_entry_data(void *edata, struct trace_probe *tp, struct ftrace_regs *fregs) { struct probe_entry_arg *earg = tp->entry_arg; unsigned long val = 0; int i; if (!earg) return; for (i = 0; i < earg->size; i++) { struct fetch_insn *code = &earg->code[i]; switch (code->op) { case FETCH_OP_ARG: val = ftrace_regs_get_argument(fregs, code->param); break; case FETCH_OP_ST_EDATA: *(unsigned long *)((unsigned long)edata + code->offset) = val; break; case FETCH_OP_END: goto end; default: break; } } end: return; } /* function exit handler */ static int trace_fprobe_entry_handler(struct fprobe *fp, unsigned long entry_ip, unsigned long ret_ip, struct ftrace_regs *fregs, void *entry_data) { struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp); if (tf->tp.entry_arg) store_fprobe_entry_data(entry_data, &tf->tp, fregs); return 0; } NOKPROBE_SYMBOL(trace_fprobe_entry_handler) static nokprobe_inline void __fexit_trace_func(struct trace_fprobe *tf, unsigned long entry_ip, unsigned long ret_ip, struct ftrace_regs *fregs, void *entry_data, struct trace_event_file *trace_file) { struct fexit_trace_entry_head *entry; struct trace_event_buffer fbuffer; struct trace_event_call *call = trace_probe_event_call(&tf->tp); int dsize; if (WARN_ON_ONCE(call != trace_file->event_call)) return; if (trace_trigger_soft_disabled(trace_file)) return; dsize = __get_data_size(&tf->tp, fregs, entry_data); entry = trace_event_buffer_reserve(&fbuffer, trace_file, sizeof(*entry) + tf->tp.size + dsize); if (!entry) return; fbuffer.regs = ftrace_get_regs(fregs); entry = fbuffer.entry = ring_buffer_event_data(fbuffer.event); entry->func = entry_ip; entry->ret_ip = ret_ip; store_trace_args(&entry[1], &tf->tp, fregs, entry_data, sizeof(*entry), dsize); trace_event_buffer_commit(&fbuffer); } static void fexit_trace_func(struct trace_fprobe *tf, unsigned long entry_ip, unsigned long ret_ip, struct ftrace_regs *fregs, void *entry_data) { struct event_file_link *link; trace_probe_for_each_link_rcu(link, &tf->tp) __fexit_trace_func(tf, entry_ip, ret_ip, fregs, entry_data, link->file); } NOKPROBE_SYMBOL(fexit_trace_func); #ifdef CONFIG_PERF_EVENTS static int fentry_perf_func(struct trace_fprobe *tf, unsigned long entry_ip, struct ftrace_regs *fregs) { struct trace_event_call *call = trace_probe_event_call(&tf->tp); struct fentry_trace_entry_head *entry; struct hlist_head *head; int size, __size, dsize; struct pt_regs *regs; int rctx; head = this_cpu_ptr(call->perf_events); if (hlist_empty(head)) return 0; dsize = __get_data_size(&tf->tp, fregs, NULL); __size = sizeof(*entry) + tf->tp.size + dsize; size = ALIGN(__size + sizeof(u32), sizeof(u64)); size -= sizeof(u32); entry = perf_trace_buf_alloc(size, ®s, &rctx); if (!entry) return 0; regs = ftrace_fill_perf_regs(fregs, regs); entry->ip = entry_ip; memset(&entry[1], 0, dsize); store_trace_args(&entry[1], &tf->tp, fregs, NULL, sizeof(*entry), dsize); perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, head, NULL); return 0; } NOKPROBE_SYMBOL(fentry_perf_func); static void fexit_perf_func(struct trace_fprobe *tf, unsigned long entry_ip, unsigned long ret_ip, struct ftrace_regs *fregs, void *entry_data) { struct trace_event_call *call = trace_probe_event_call(&tf->tp); struct fexit_trace_entry_head *entry; struct hlist_head *head; int size, __size, dsize; struct pt_regs *regs; int rctx; head = this_cpu_ptr(call->perf_events); if (hlist_empty(head)) return;