// SPDX-License-Identifier: GPL-2.0/* * fprobe - Simple ftrace probe wrapper for function entry. */#define pr_fmt(fmt) "fprobe: " fmt #include <linux/cleanup.h>#include <linux/err.h>#include <linux/fprobe.h>#include <linux/kallsyms.h>#include <linux/kprobes.h>#include <linux/list.h>#include <linux/mutex.h>#include <linux/rhashtable.h>#include <linux/slab.h>#include <linux/sort.h> #include <asm/fprobe.h> #include "trace.h" #define FPROBE_IP_HASH_BITS 8#define FPROBE_IP_TABLE_SIZE (1 << FPROBE_IP_HASH_BITS) #define FPROBE_HASH_BITS 6#define FPROBE_TABLE_SIZE (1 << FPROBE_HASH_BITS) #define SIZE_IN_LONG(x) ((x + sizeof(long) - 1) >> (sizeof(long) == 8 ? 3 : 2)) /* * fprobe_table: hold 'fprobe_hlist::hlist' for checking the fprobe still * exists. The key is the address of fprobe instance. * fprobe_ip_table: hold 'fprobe_hlist::array[*]' for searching the fprobe * instance related to the function address. The key is the ftrace IP * address. * * When unregistering the fprobe, fprobe_hlist::fp and fprobe_hlist::array[*].fp * are set NULL and delete those from both hash tables (by hlist_del_rcu). * After an RCU grace period, the fprobe_hlist itself will be released. * * fprobe_table and fprobe_ip_table can be accessed from either * - Normal hlist traversal and RCU add/del under 'fprobe_mutex' is held. * - RCU hlist traversal under disabling preempt */static struct hlist_head fprobe_table[FPROBE_TABLE_SIZE];static struct rhltable fprobe_ip_table;static DEFINE_MUTEX(fprobe_mutex);static struct fgraph_ops fprobe_graph_ops; static u32 fprobe_node_hashfn(const void *data, u32 len, u32 seed){ return hash_ptr(*(unsigned long **)data, 32);} static int fprobe_node_cmp(struct rhashtable_compare_arg *arg, const void *ptr){ unsigned long key = *(unsigned long *)arg->key; const struct fprobe_hlist_node *n = ptr; return n->addr != key;} static u32 fprobe_node_obj_hashfn(const void *data, u32 len, u32 seed){ const struct fprobe_hlist_node *n = data; return hash_ptr((void *)n->addr, 32);} static const struct rhashtable_params fprobe_rht_params = { .head_offset = offsetof(struct fprobe_hlist_node, hlist), .key_offset = offsetof(struct fprobe_hlist_node, addr), .key_len = sizeof_field(struct fprobe_hlist_node, addr), .hashfn = fprobe_node_hashfn, .obj_hashfn = fprobe_node_obj_hashfn, .obj_cmpfn = fprobe_node_cmp, .automatic_shrinking = true,}; /* Node insertion and deletion requires the fprobe_mutex */static int __insert_fprobe_node(struct fprobe_hlist_node *node, struct fprobe *fp){ int ret; lockdep_assert_held(&fprobe_mutex); ret = rhltable_insert(&fprobe_ip_table, &node->hlist, fprobe_rht_params); /* Set the fprobe pointer if insertion was successful. */ if (!ret) WRITE_ONCE(node->fp, fp); return ret;} static void __delete_fprobe_node(struct fprobe_hlist_node *node){ lockdep_assert_held(&fprobe_mutex); /* Avoid double deleting and non-inserted nodes */ if (READ_ONCE(node->fp) != NULL) { WRITE_ONCE(node->fp, NULL); rhltable_remove(&fprobe_ip_table, &node->hlist, fprobe_rht_params); }} /* Check existence of the fprobe */static bool fprobe_registered(struct fprobe *fp){ struct hlist_head *head; struct fprobe_hlist *fph; head = &fprobe_table[hash_ptr(fp, FPROBE_HASH_BITS)]; hlist_for_each_entry_rcu(fph, head, hlist, lockdep_is_held(&fprobe_mutex)) { if (fph->fp == fp) return true; } return false;}NOKPROBE_SYMBOL(fprobe_registered); static int add_fprobe_hash(struct fprobe *fp){ struct fprobe_hlist *fph = fp->hlist_array; struct hlist_head *head; lockdep_assert_held(&fprobe_mutex); if (WARN_ON_ONCE(!fph)) return -EINVAL; head = &fprobe_table[hash_ptr(fp, FPROBE_HASH_BITS)]; hlist_add_head_rcu(&fp->hlist_array->hlist, head); return 0;} static int del_fprobe_hash(struct fprobe *fp){ struct fprobe_hlist *fph = fp->hlist_array; lockdep_assert_held(&fprobe_mutex); if (WARN_ON_ONCE(!fph)) return -EINVAL; if (!fprobe_registered(fp)) return -ENOENT; fph->fp = NULL; hlist_del_rcu(&fph->hlist); return 0;} #ifdef ARCH_DEFINE_ENCODE_FPROBE_HEADER /* The arch should encode fprobe_header info into one unsigned long */#define FPROBE_HEADER_SIZE_IN_LONG 1 static inline bool write_fprobe_header(unsigned long *stack, struct fprobe *fp, unsigned int size_words){ if (WARN_ON_ONCE(size_words > MAX_FPROBE_DATA_SIZE_WORD || !arch_fprobe_header_encodable(fp))) return false; *stack = arch_encode_fprobe_header(fp, size_words); return true;} static inline void read_fprobe_header(unsigned long *stack, struct fprobe **fp, unsigned int *size_words){ *fp = arch_decode_fprobe_header_fp(*stack); *size_words = arch_decode_fprobe_header_size(*stack);} #else /* Generic fprobe_header */struct __fprobe_header { struct fprobe *fp; unsigned long size_words;} __packed; #define FPROBE_HEADER_SIZE_IN_LONG SIZE_IN_LONG(sizeof(struct __fprobe_header)) static inline bool write_fprobe_header(unsigned long *stack, struct fprobe *fp, unsigned int size_words){ struct __fprobe_header *fph = (struct __fprobe_header *)stack; if (WARN_ON_ONCE(size_words > MAX_FPROBE_DATA_SIZE_WORD)) return false; fph->fp = fp; fph->size_words = size_words; return true;} static inline void read_fprobe_header(unsigned long *stack, struct fprobe **fp, unsigned int *size_words){ struct __fprobe_header *fph = (struct __fprobe_header *)stack; *fp = fph->fp; *size_words = fph->size_words;} #endif /* * fprobe shadow stack management: * Since fprobe shares a single fgraph_ops, it needs to share the stack entry * among the probes on the same function exit. Note that a new probe can be * registered before a target function is returning, we can not use the hash * table to find the corresponding probes. Thus the probe address is stored on * the shadow stack with its entry data size. * */static inline int __fprobe_handler(unsigned long ip, unsigned long parent_ip, struct fprobe *fp, struct ftrace_regs *fregs, void *data){ if (!fp->entry_handler) return 0; return fp->entry_handler(fp, ip, parent_ip, fregs, data);} static inline int __fprobe_kprobe_handler(unsigned long ip, unsigned long parent_ip, struct fprobe *fp, struct ftrace_regs *fregs, void *data){ int ret; /* * This user handler is shared with other kprobes and is not expected to be * called recursively. So if any other kprobe handler is running, this will * exit as kprobe does. See the section 'Share the callbacks with kprobes' * in Documentation/trace/fprobe.rst for more information. */ if (unlikely(kprobe_running())) { fp->nmissed++; return 0; } kprobe_busy_begin(); ret = __fprobe_handler(ip, parent_ip, fp, fregs, data); kprobe_busy_end(); return ret;} static int fprobe_fgraph_entry(struct ftrace_graph_ent *trace, struct fgraph_ops *gops, struct ftrace_regs *fregs);static void fprobe_return(struct ftrace_graph_ret *trace, struct fgraph_ops *gops, struct ftrace_regs *fregs); static struct fgraph_ops fprobe_graph_ops = { .entryfunc = fprobe_fgraph_entry, .retfunc = fprobe_return,};/* Number of fgraph fprobe nodes */static int nr_fgraph_fprobes;/* Is fprobe_graph_ops registered? */static bool fprobe_graph_registered; /* Add @addrs to the ftrace filter and register fgraph if needed. */static int fprobe_graph_add_ips(unsigned long *addrs, int num){ int ret; lockdep_assert_held(&fprobe_mutex); ret = ftrace_set_filter_ips(&fprobe_graph_ops.ops, addrs, num, 0, 0); if (ret) return ret; if (!fprobe_graph_registered) { ret = register_ftrace_graph(&fprobe_graph_ops); if (WARN_ON_ONCE(ret)) { ftrace_free_filter(&fprobe_graph_ops.ops); return ret; } fprobe_graph_registered = true; } return 0;} static void __fprobe_graph_unregister(void){ if (fprobe_graph_registered) { unregister_ftrace_graph(&fprobe_graph_ops); ftrace_free_filter(&fprobe_graph_ops.ops); fprobe_graph_registered = false; }} /* Remove @addrs from the ftrace filter and unregister fgraph if possible. */static void fprobe_graph_remove_ips(unsigned long *addrs, int num){ lockdep_assert_held(&fprobe_mutex); if (!nr_fgraph_fprobes) __fprobe_graph_unregister(); else if (num) ftrace_set_filter_ips(&fprobe_graph_ops.ops, addrs, num, 1, 0);} #if defined(CONFIG_DYNAMIC_FTRACE_WITH_ARGS) || defined(CONFIG_DYNAMIC_FTRACE_WITH_REGS) /* ftrace_ops callback, this processes fprobes which have only entry_handler. */static void fprobe_ftrace_entry(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *ops, struct ftrace_regs *fregs){ struct fprobe_hlist_node *node; struct rhlist_head *head, *pos; struct fprobe *fp; int bit; bit = ftrace_test_recursion_trylock(ip, parent_ip); if (bit < 0) return; /* * ftrace_test_recursion_trylock() disables preemption, but * rhltable_lookup() checks whether rcu_read_lcok is held. * So we take rcu_read_lock() here. */ rcu_read_lock(); head = rhltable_lookup(&fprobe_ip_table, &ip, fprobe_rht_params); rhl_for_each_entry_rcu(node, pos, head, hlist) { if (node->addr != ip) break; fp = READ_ONCE(node->fp); if (unlikely(!fp || fprobe_disabled(fp) || fp->exit_handler)) continue; if (fprobe_shared_with_kprobes(fp)) __fprobe_kprobe_handler(ip, parent_ip, fp, fregs, NULL); else __fprobe_handler(ip, parent_ip, fp, fregs, NULL); } rcu_read_unlock(); ftrace_test_recursion_unlock(bit);}NOKPROBE_SYMBOL(fprobe_ftrace_entry); static struct ftrace_ops fprobe_ftrace_ops = { .func = fprobe_ftrace_entry, .flags = FTRACE_OPS_FL_SAVE_ARGS,};/* Number of ftrace fprobe nodes */static int nr_ftrace_fprobes;/* Is fprobe_ftrace_ops registered? */static bool fprobe_ftrace_registered; static int fprobe_ftrace_add_ips(unsigned long *addrs, int num){ int ret; lockdep_assert_held(&fprobe_mutex); ret = ftrace_set_filter_ips(&fprobe_ftrace_ops, addrs, num, 0, 0); if (ret) return ret; if (!fprobe_ftrace_registered) { ret = register_ftrace_function(&fprobe_ftrace_ops); if (ret) { ftrace_free_filter(&fprobe_ftrace_ops); return ret; } fprobe_ftrace_registered = true; } return 0;} static void __fprobe_ftrace_unregister(void){ if (fprobe_ftrace_registered) { unregister_ftrace_function(&fprobe_ftrace_ops); ftrace_free_filter(&fprobe_ftrace_ops); fprobe_ftrace_registered = false; }} static void fprobe_ftrace_remove_ips(unsigned long *addrs, int num){ lockdep_assert_held(&fprobe_mutex); if (!nr_ftrace_fprobes) __fprobe_ftrace_unregister(); else if (num) ftrace_set_filter_ips(&fprobe_ftrace_ops, addrs, num, 1, 0);} static bool fprobe_is_ftrace(struct fprobe *fp){ return !fp->exit_handler;} /* Node insertion and deletion requires the fprobe_mutex */static int insert_fprobe_node(struct fprobe_hlist_node *node, struct fprobe *fp){ int ret; lockdep_assert_held(&fprobe_mutex); ret = __insert_fprobe_node(node, fp); if (!ret) { if (fprobe_is_ftrace(fp)) nr_ftrace_fprobes++; else nr_fgraph_fprobes++; } return ret;} static void delete_fprobe_node(struct fprobe_hlist_node *node){ struct fprobe *fp; lockdep_assert_held(&fprobe_mutex); fp = READ_ONCE(node->fp); if (fp) { if (fprobe_is_ftrace(fp)) nr_ftrace_fprobes--; else nr_fgraph_fprobes--; } __delete_fprobe_node(node);} static bool fprobe_exists_on_hash(unsigned long ip, bool ftrace){ struct rhlist_head *head, *pos; struct fprobe_hlist_node *node; struct fprobe *fp; guard(rcu)(); head = rhltable_lookup(&fprobe_ip_table, &ip, fprobe_rht_params); if (!head) return false; /* We have to check the same type on the list. */ rhl_for_each_entry_rcu(node, pos, head, hlist) { if (node->addr != ip) break; fp = READ_ONCE(node->fp); if (likely(fp)) { if ((!ftrace && fp->exit_handler) || (ftrace && !fp->exit_handler)) return true; } } return false;} #ifdef CONFIG_MODULESstatic void fprobe_remove_ips(unsigned long *ips, unsigned int cnt){ if (!nr_fgraph_fprobes) __fprobe_graph_unregister(); else if (cnt) ftrace_set_filter_ips(&fprobe_graph_ops.ops, ips, cnt, 1, 0); if (!nr_ftrace_fprobes) __fprobe_ftrace_unregister(); else if (cnt) ftrace_set_filter_ips(&fprobe_ftrace_ops, ips, cnt, 1, 0);}#endif#elsestatic int fprobe_ftrace_add_ips(unsigned long *addrs, int num){ return -ENOENT;} static void fprobe_ftrace_remove_ips(unsigned long *addrs, int num){} static bool fprobe_is_ftrace(struct fprobe *fp){ return false;} /* Node insertion and deletion requires the fprobe_mutex */static int insert_fprobe_node(struct fprobe_hlist_node *node, struct fprobe *fp){ int ret; lockdep_assert_held(&fprobe_mutex); ret = __insert_fprobe_node(node, fp);Lines 1–500 of 1137. Load more
// SPDX-License-Identifier: GPL-2.0 /* * fprobe - Simple ftrace probe wrapper for function entry. */ #define pr_fmt(fmt) "fprobe: " fmt #include <linux/cleanup.h> #include <linux/err.h> #include <linux/fprobe.h> #include <linux/kallsyms.h> #include <linux/kprobes.h> #include <linux/list.h> #include <linux/mutex.h> #include <linux/rhashtable.h> #include <linux/slab.h> #include <linux/sort.h> #include <asm/fprobe.h> #include "trace.h" #define FPROBE_IP_HASH_BITS 8 #define FPROBE_IP_TABLE_SIZE (1 << FPROBE_IP_HASH_BITS) #define FPROBE_HASH_BITS 6 #define FPROBE_TABLE_SIZE (1 << FPROBE_HASH_BITS) #define SIZE_IN_LONG(x) ((x + sizeof(long) - 1) >> (sizeof(long) == 8 ? 3 : 2)) /* * fprobe_table: hold 'fprobe_hlist::hlist' for checking the fprobe still * exists. The key is the address of fprobe instance. * fprobe_ip_table: hold 'fprobe_hlist::array[*]' for searching the fprobe * instance related to the function address. The key is the ftrace IP * address. * * When unregistering the fprobe, fprobe_hlist::fp and fprobe_hlist::array[*].fp * are set NULL and delete those from both hash tables (by hlist_del_rcu). * After an RCU grace period, the fprobe_hlist itself will be released. * * fprobe_table and fprobe_ip_table can be accessed from either * - Normal hlist traversal and RCU add/del under 'fprobe_mutex' is held. * - RCU hlist traversal under disabling preempt */ static struct hlist_head fprobe_table[FPROBE_TABLE_SIZE]; static struct rhltable fprobe_ip_table; static DEFINE_MUTEX(fprobe_mutex); static struct fgraph_ops fprobe_graph_ops; static u32 fprobe_node_hashfn(const void *data, u32 len, u32 seed) { return hash_ptr(*(unsigned long **)data, 32); } static int fprobe_node_cmp(struct rhashtable_compare_arg *arg, const void *ptr) { unsigned long key = *(unsigned long *)arg->key; const struct fprobe_hlist_node *n = ptr; return n->addr != key; } static u32 fprobe_node_obj_hashfn(const void *data, u32 len, u32 seed) { const struct fprobe_hlist_node *n = data; return hash_ptr((void *)n->addr, 32); } static const struct rhashtable_params fprobe_rht_params = { .head_offset = offsetof(struct fprobe_hlist_node, hlist), .key_offset = offsetof(struct fprobe_hlist_node, addr), .key_len = sizeof_field(struct fprobe_hlist_node, addr), .hashfn = fprobe_node_hashfn, .obj_hashfn = fprobe_node_obj_hashfn, .obj_cmpfn = fprobe_node_cmp, .automatic_shrinking = true, }; /* Node insertion and deletion requires the fprobe_mutex */ static int __insert_fprobe_node(struct fprobe_hlist_node *node, struct fprobe *fp) { int ret; lockdep_assert_held(&fprobe_mutex); ret = rhltable_insert(&fprobe_ip_table, &node->hlist, fprobe_rht_params); /* Set the fprobe pointer if insertion was successful. */ if (!ret) WRITE_ONCE(node->fp, fp); return ret; } static void __delete_fprobe_node(struct fprobe_hlist_node *node) { lockdep_assert_held(&fprobe_mutex); /* Avoid double deleting and non-inserted nodes */ if (READ_ONCE(node->fp) != NULL) { WRITE_ONCE(node->fp, NULL); rhltable_remove(&fprobe_ip_table, &node->hlist, fprobe_rht_params); } } /* Check existence of the fprobe */ static bool fprobe_registered(struct fprobe *fp) { struct hlist_head *head; struct fprobe_hlist *fph; head = &fprobe_table[hash_ptr(fp, FPROBE_HASH_BITS)]; hlist_for_each_entry_rcu(fph, head, hlist, lockdep_is_held(&fprobe_mutex)) { if (fph->fp == fp) return true; } return false; } NOKPROBE_SYMBOL(fprobe_registered); static int add_fprobe_hash(struct fprobe *fp) { struct fprobe_hlist *fph = fp->hlist_array; struct hlist_head *head; lockdep_assert_held(&fprobe_mutex); if (WARN_ON_ONCE(!fph)) return -EINVAL; head = &fprobe_table[hash_ptr(fp, FPROBE_HASH_BITS)]; hlist_add_head_rcu(&fp->hlist_array->hlist, head); return 0; } static int del_fprobe_hash(struct fprobe *fp) { struct fprobe_hlist *fph = fp->hlist_array; lockdep_assert_held(&fprobe_mutex); if (WARN_ON_ONCE(!fph)) return -EINVAL; if (!fprobe_registered(fp)) return -ENOENT; fph->fp = NULL; hlist_del_rcu(&fph->hlist); return 0; } #ifdef ARCH_DEFINE_ENCODE_FPROBE_HEADER /* The arch should encode fprobe_header info into one unsigned long */ #define FPROBE_HEADER_SIZE_IN_LONG 1 static inline bool write_fprobe_header(unsigned long *stack, struct fprobe *fp, unsigned int size_words) { if (WARN_ON_ONCE(size_words > MAX_FPROBE_DATA_SIZE_WORD || !arch_fprobe_header_encodable(fp))) return false; *stack = arch_encode_fprobe_header(fp, size_words); return true; } static inline void read_fprobe_header(unsigned long *stack, struct fprobe **fp, unsigned int *size_words) { *fp = arch_decode_fprobe_header_fp(*stack); *size_words = arch_decode_fprobe_header_size(*stack); } #else /* Generic fprobe_header */ struct __fprobe_header { struct fprobe *fp; unsigned long size_words; } __packed; #define FPROBE_HEADER_SIZE_IN_LONG SIZE_IN_LONG(sizeof(struct __fprobe_header)) static inline bool write_fprobe_header(unsigned long *stack, struct fprobe *fp, unsigned int size_words) { struct __fprobe_header *fph = (struct __fprobe_header *)stack; if (WARN_ON_ONCE(size_words > MAX_FPROBE_DATA_SIZE_WORD)) return false; fph->fp = fp; fph->size_words = size_words; return true; } static inline void read_fprobe_header(unsigned long *stack, struct fprobe **fp, unsigned int *size_words) { struct __fprobe_header *fph = (struct __fprobe_header *)stack; *fp = fph->fp; *size_words = fph->size_words; } #endif /* * fprobe shadow stack management: * Since fprobe shares a single fgraph_ops, it needs to share the stack entry * among the probes on the same function exit. Note that a new probe can be * registered before a target function is returning, we can not use the hash * table to find the corresponding probes. Thus the probe address is stored on * the shadow stack with its entry data size. * */ static inline int __fprobe_handler(unsigned long ip, unsigned long parent_ip, struct fprobe *fp, struct ftrace_regs *fregs, void *data) { if (!fp->entry_handler) return 0; return fp->entry_handler(fp, ip, parent_ip, fregs, data); } static inline int __fprobe_kprobe_handler(unsigned long ip, unsigned long parent_ip, struct fprobe *fp, struct ftrace_regs *fregs, void *data) { int ret; /* * This user handler is shared with other kprobes and is not expected to be * called recursively. So if any other kprobe handler is running, this will * exit as kprobe does. See the section 'Share the callbacks with kprobes' * in Documentation/trace/fprobe.rst for more information. */ if (unlikely(kprobe_running())) { fp->nmissed++; return 0; } kprobe_busy_begin(); ret = __fprobe_handler(ip, parent_ip, fp, fregs, data); kprobe_busy_end(); return ret; } static int fprobe_fgraph_entry(struct ftrace_graph_ent *trace, struct fgraph_ops *gops, struct ftrace_regs *fregs); static void fprobe_return(struct ftrace_graph_ret *trace, struct fgraph_ops *gops, struct ftrace_regs *fregs); static struct fgraph_ops fprobe_graph_ops = { .entryfunc = fprobe_fgraph_entry, .retfunc = fprobe_return, }; /* Number of fgraph fprobe nodes */ static int nr_fgraph_fprobes; /* Is fprobe_graph_ops registered? */ static bool fprobe_graph_registered; /* Add @addrs to the ftrace filter and register fgraph if needed. */ static int fprobe_graph_add_ips(unsigned long *addrs, int num) { int ret; lockdep_assert_held(&fprobe_mutex); ret = ftrace_set_filter_ips(&fprobe_graph_ops.ops, addrs, num, 0, 0); if (ret) return ret; if (!fprobe_graph_registered) { ret = register_ftrace_graph(&fprobe_graph_ops); if (WARN_ON_ONCE(ret)) { ftrace_free_filter(&fprobe_graph_ops.ops); return ret; } fprobe_graph_registered = true; } return 0; } static void __fprobe_graph_unregister(void) { if (fprobe_graph_registered) { unregister_ftrace_graph(&fprobe_graph_ops); ftrace_free_filter(&fprobe_graph_ops.ops); fprobe_graph_registered = false; } } /* Remove @addrs from the ftrace filter and unregister fgraph if possible. */ static void fprobe_graph_remove_ips(unsigned long *addrs, int num) { lockdep_assert_held(&fprobe_mutex); if (!nr_fgraph_fprobes) __fprobe_graph_unregister(); else if (num) ftrace_set_filter_ips(&fprobe_graph_ops.ops, addrs, num, 1, 0); } #if defined(CONFIG_DYNAMIC_FTRACE_WITH_ARGS) || defined(CONFIG_DYNAMIC_FTRACE_WITH_REGS) /* ftrace_ops callback, this processes fprobes which have only entry_handler. */ static void fprobe_ftrace_entry(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *ops, struct ftrace_regs *fregs) { struct fprobe_hlist_node *node; struct rhlist_head *head, *pos; struct fprobe *fp; int bit; bit = ftrace_test_recursion_trylock(ip, parent_ip); if (bit < 0) return; /* * ftrace_test_recursion_trylock() disables preemption, but * rhltable_lookup() checks whether rcu_read_lcok is held. * So we take rcu_read_lock() here. */ rcu_read_lock(); head = rhltable_lookup(&fprobe_ip_table, &ip, fprobe_rht_params); rhl_for_each_entry_rcu(node, pos, head, hlist) { if (node->addr != ip) break; fp = READ_ONCE(node->fp); if (unlikely(!fp || fprobe_disabled(fp) || fp->exit_handler)) continue; if (fprobe_shared_with_kprobes(fp)) __fprobe_kprobe_handler(ip, parent_ip, fp, fregs, NULL); else __fprobe_handler(ip, parent_ip, fp, fregs, NULL); } rcu_read_unlock(); ftrace_test_recursion_unlock(bit); } NOKPROBE_SYMBOL(fprobe_ftrace_entry); static struct ftrace_ops fprobe_ftrace_ops = { .func = fprobe_ftrace_entry, .flags = FTRACE_OPS_FL_SAVE_ARGS, }; /* Number of ftrace fprobe nodes */ static int nr_ftrace_fprobes; /* Is fprobe_ftrace_ops registered? */ static bool fprobe_ftrace_registered; static int fprobe_ftrace_add_ips(unsigned long *addrs, int num) { int ret; lockdep_assert_held(&fprobe_mutex); ret = ftrace_set_filter_ips(&fprobe_ftrace_ops, addrs, num, 0, 0); if (ret) return ret; if (!fprobe_ftrace_registered) { ret = register_ftrace_function(&fprobe_ftrace_ops); if (ret) { ftrace_free_filter(&fprobe_ftrace_ops); return ret; } fprobe_ftrace_registered = true; } return 0; } static void __fprobe_ftrace_unregister(void) { if (fprobe_ftrace_registered) { unregister_ftrace_function(&fprobe_ftrace_ops); ftrace_free_filter(&fprobe_ftrace_ops); fprobe_ftrace_registered = false; } } static void fprobe_ftrace_remove_ips(unsigned long *addrs, int num) { lockdep_assert_held(&fprobe_mutex); if (!nr_ftrace_fprobes) __fprobe_ftrace_unregister(); else if (num) ftrace_set_filter_ips(&fprobe_ftrace_ops, addrs, num, 1, 0); } static bool fprobe_is_ftrace(struct fprobe *fp) { return !fp->exit_handler; } /* Node insertion and deletion requires the fprobe_mutex */ static int insert_fprobe_node(struct fprobe_hlist_node *node, struct fprobe *fp) { int ret; lockdep_assert_held(&fprobe_mutex); ret = __insert_fprobe_node(node, fp); if (!ret) { if (fprobe_is_ftrace(fp)) nr_ftrace_fprobes++; else nr_fgraph_fprobes++; } return ret; } static void delete_fprobe_node(struct fprobe_hlist_node *node) { struct fprobe *fp; lockdep_assert_held(&fprobe_mutex); fp = READ_ONCE(node->fp); if (fp) { if (fprobe_is_ftrace(fp)) nr_ftrace_fprobes--; else nr_fgraph_fprobes--; } __delete_fprobe_node(node); } static bool fprobe_exists_on_hash(unsigned long ip, bool ftrace) { struct rhlist_head *head, *pos; struct fprobe_hlist_node *node; struct fprobe *fp; guard(rcu)(); head = rhltable_lookup(&fprobe_ip_table, &ip, fprobe_rht_params); if (!head) return false; /* We have to check the same type on the list. */ rhl_for_each_entry_rcu(node, pos, head, hlist) { if (node->addr != ip) break; fp = READ_ONCE(node->fp); if (likely(fp)) { if ((!ftrace && fp->exit_handler) || (ftrace && !fp->exit_handler)) return true; } } return false; } #ifdef CONFIG_MODULES static void fprobe_remove_ips(unsigned long *ips, unsigned int cnt) { if (!nr_fgraph_fprobes) __fprobe_graph_unregister(); else if (cnt) ftrace_set_filter_ips(&fprobe_graph_ops.ops, ips, cnt, 1, 0); if (!nr_ftrace_fprobes) __fprobe_ftrace_unregister(); else if (cnt) ftrace_set_filter_ips(&fprobe_ftrace_ops, ips, cnt, 1, 0); } #endif #else static int fprobe_ftrace_add_ips(unsigned long *addrs, int num) { return -ENOENT; } static void fprobe_ftrace_remove_ips(unsigned long *addrs, int num) { } static bool fprobe_is_ftrace(struct fprobe *fp) { return false; } /* Node insertion and deletion requires the fprobe_mutex */ static int insert_fprobe_node(struct fprobe_hlist_node *node, struct fprobe *fp) { int ret; lockdep_assert_held(&fprobe_mutex); ret = __insert_fprobe_node(node, fp);