// SPDX-License-Identifier: GPL-2.0/* * fprobe - Simple ftrace probe wrapper for function entry. */#define pr_fmt(fmt) "fprobe: " fmt #include <linux/err.h>#include <linux/fprobe.h>#include <linux/kallsyms.h>#include <linux/kprobes.h>#include <linux/list.h>#include <linux/mutex.h>#include <linux/rhashtable.h>#include <linux/slab.h>#include <linux/sort.h> #include <asm/fprobe.h> #include "trace.h" #define FPROBE_IP_HASH_BITS 8#define FPROBE_IP_TABLE_SIZE (1 << FPROBE_IP_HASH_BITS) #define FPROBE_HASH_BITS 6#define FPROBE_TABLE_SIZE (1 << FPROBE_HASH_BITS) #define SIZE_IN_LONG(x) ((x + sizeof(long) - 1) >> (sizeof(long) == 8 ? 3 : 2)) /* * fprobe_table: hold 'fprobe_hlist::hlist' for checking the fprobe still * exists. The key is the address of fprobe instance. * fprobe_ip_table: hold 'fprobe_hlist::array[*]' for searching the fprobe * instance related to the function address. The key is the ftrace IP * address. * * When unregistering the fprobe, fprobe_hlist::fp and fprobe_hlist::array[*].fp * are set NULL and delete those from both hash tables (by hlist_del_rcu). * After an RCU grace period, the fprobe_hlist itself will be released. * * fprobe_table and fprobe_ip_table can be accessed from either * - Normal hlist traversal and RCU add/del under 'fprobe_mutex' is held. * - RCU hlist traversal under disabling preempt */static struct hlist_head fprobe_table[FPROBE_TABLE_SIZE];static struct rhltable fprobe_ip_table;static DEFINE_MUTEX(fprobe_mutex);static struct fgraph_ops fprobe_graph_ops; static u32 fprobe_node_hashfn(const void *data, u32 len, u32 seed){ return hash_ptr(*(unsigned long **)data, 32);} static int fprobe_node_cmp(struct rhashtable_compare_arg *arg, const void *ptr){ unsigned long key = *(unsigned long *)arg->key; const struct fprobe_hlist_node *n = ptr; return n->addr != key;} static u32 fprobe_node_obj_hashfn(const void *data, u32 len, u32 seed){ const struct fprobe_hlist_node *n = data; return hash_ptr((void *)n->addr, 32);} static const struct rhashtable_params fprobe_rht_params = { .head_offset = offsetof(struct fprobe_hlist_node, hlist), .key_offset = offsetof(struct fprobe_hlist_node, addr), .key_len = sizeof_field(struct fprobe_hlist_node, addr), .hashfn = fprobe_node_hashfn, .obj_hashfn = fprobe_node_obj_hashfn, .obj_cmpfn = fprobe_node_cmp, .automatic_shrinking = true,}; /* Node insertion and deletion requires the fprobe_mutex */static int insert_fprobe_node(struct fprobe_hlist_node *node){ lockdep_assert_held(&fprobe_mutex); return rhltable_insert(&fprobe_ip_table, &node->hlist, fprobe_rht_params);} /* Return true if there are synonims */static bool delete_fprobe_node(struct fprobe_hlist_node *node){ lockdep_assert_held(&fprobe_mutex); bool ret; /* Avoid double deleting */ if (READ_ONCE(node->fp) != NULL) { WRITE_ONCE(node->fp, NULL); rhltable_remove(&fprobe_ip_table, &node->hlist, fprobe_rht_params); } rcu_read_lock(); ret = !!rhltable_lookup(&fprobe_ip_table, &node->addr, fprobe_rht_params); rcu_read_unlock(); return ret;} /* Check existence of the fprobe */static bool is_fprobe_still_exist(struct fprobe *fp){ struct hlist_head *head; struct fprobe_hlist *fph; head = &fprobe_table[hash_ptr(fp, FPROBE_HASH_BITS)]; hlist_for_each_entry_rcu(fph, head, hlist, lockdep_is_held(&fprobe_mutex)) { if (fph->fp == fp) return true; } return false;}NOKPROBE_SYMBOL(is_fprobe_still_exist); static int add_fprobe_hash(struct fprobe *fp){ struct fprobe_hlist *fph = fp->hlist_array; struct hlist_head *head; lockdep_assert_held(&fprobe_mutex); if (WARN_ON_ONCE(!fph)) return -EINVAL; if (is_fprobe_still_exist(fp)) return -EEXIST; head = &fprobe_table[hash_ptr(fp, FPROBE_HASH_BITS)]; hlist_add_head_rcu(&fp->hlist_array->hlist, head); return 0;} static int del_fprobe_hash(struct fprobe *fp){ struct fprobe_hlist *fph = fp->hlist_array; lockdep_assert_held(&fprobe_mutex); if (WARN_ON_ONCE(!fph)) return -EINVAL; if (!is_fprobe_still_exist(fp)) return -ENOENT; fph->fp = NULL; hlist_del_rcu(&fph->hlist); return 0;} #ifdef ARCH_DEFINE_ENCODE_FPROBE_HEADER /* The arch should encode fprobe_header info into one unsigned long */#define FPROBE_HEADER_SIZE_IN_LONG 1 static inline bool write_fprobe_header(unsigned long *stack, struct fprobe *fp, unsigned int size_words){ if (WARN_ON_ONCE(size_words > MAX_FPROBE_DATA_SIZE_WORD || !arch_fprobe_header_encodable(fp))) return false; *stack = arch_encode_fprobe_header(fp, size_words); return true;} static inline void read_fprobe_header(unsigned long *stack, struct fprobe **fp, unsigned int *size_words){ *fp = arch_decode_fprobe_header_fp(*stack); *size_words = arch_decode_fprobe_header_size(*stack);} #else /* Generic fprobe_header */struct __fprobe_header { struct fprobe *fp; unsigned long size_words;} __packed; #define FPROBE_HEADER_SIZE_IN_LONG SIZE_IN_LONG(sizeof(struct __fprobe_header)) static inline bool write_fprobe_header(unsigned long *stack, struct fprobe *fp, unsigned int size_words){ struct __fprobe_header *fph = (struct __fprobe_header *)stack; if (WARN_ON_ONCE(size_words > MAX_FPROBE_DATA_SIZE_WORD)) return false; fph->fp = fp; fph->size_words = size_words; return true;} static inline void read_fprobe_header(unsigned long *stack, struct fprobe **fp, unsigned int *size_words){ struct __fprobe_header *fph = (struct __fprobe_header *)stack; *fp = fph->fp; *size_words = fph->size_words;} #endif /* * fprobe shadow stack management: * Since fprobe shares a single fgraph_ops, it needs to share the stack entry * among the probes on the same function exit. Note that a new probe can be * registered before a target function is returning, we can not use the hash * table to find the corresponding probes. Thus the probe address is stored on * the shadow stack with its entry data size. * */static inline int __fprobe_handler(unsigned long ip, unsigned long parent_ip, struct fprobe *fp, struct ftrace_regs *fregs, void *data){ if (!fp->entry_handler) return 0; return fp->entry_handler(fp, ip, parent_ip, fregs, data);} static inline int __fprobe_kprobe_handler(unsigned long ip, unsigned long parent_ip, struct fprobe *fp, struct ftrace_regs *fregs, void *data){ int ret; /* * This user handler is shared with other kprobes and is not expected to be * called recursively. So if any other kprobe handler is running, this will * exit as kprobe does. See the section 'Share the callbacks with kprobes' * in Documentation/trace/fprobe.rst for more information. */ if (unlikely(kprobe_running())) { fp->nmissed++; return 0; } kprobe_busy_begin(); ret = __fprobe_handler(ip, parent_ip, fp, fregs, data); kprobe_busy_end(); return ret;} #if defined(CONFIG_DYNAMIC_FTRACE_WITH_ARGS) || defined(CONFIG_DYNAMIC_FTRACE_WITH_REGS)/* ftrace_ops callback, this processes fprobes which have only entry_handler. */static void fprobe_ftrace_entry(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *ops, struct ftrace_regs *fregs){ struct fprobe_hlist_node *node; struct rhlist_head *head, *pos; struct fprobe *fp; int bit; bit = ftrace_test_recursion_trylock(ip, parent_ip); if (bit < 0) return; /* * ftrace_test_recursion_trylock() disables preemption, but * rhltable_lookup() checks whether rcu_read_lcok is held. * So we take rcu_read_lock() here. */ rcu_read_lock(); head = rhltable_lookup(&fprobe_ip_table, &ip, fprobe_rht_params); rhl_for_each_entry_rcu(node, pos, head, hlist) { if (node->addr != ip) break; fp = READ_ONCE(node->fp); if (unlikely(!fp || fprobe_disabled(fp) || fp->exit_handler)) continue; if (fprobe_shared_with_kprobes(fp)) __fprobe_kprobe_handler(ip, parent_ip, fp, fregs, NULL); else __fprobe_handler(ip, parent_ip, fp, fregs, NULL); } rcu_read_unlock(); ftrace_test_recursion_unlock(bit);}NOKPROBE_SYMBOL(fprobe_ftrace_entry); static struct ftrace_ops fprobe_ftrace_ops = { .func = fprobe_ftrace_entry, .flags = FTRACE_OPS_FL_SAVE_ARGS,};static int fprobe_ftrace_active; static int fprobe_ftrace_add_ips(unsigned long *addrs, int num){ int ret; lockdep_assert_held(&fprobe_mutex); ret = ftrace_set_filter_ips(&fprobe_ftrace_ops, addrs, num, 0, 0); if (ret) return ret; if (!fprobe_ftrace_active) { ret = register_ftrace_function(&fprobe_ftrace_ops); if (ret) { ftrace_free_filter(&fprobe_ftrace_ops); return ret; } } fprobe_ftrace_active++; return 0;} static void fprobe_ftrace_remove_ips(unsigned long *addrs, int num){ lockdep_assert_held(&fprobe_mutex); fprobe_ftrace_active--; if (!fprobe_ftrace_active) unregister_ftrace_function(&fprobe_ftrace_ops); if (num) ftrace_set_filter_ips(&fprobe_ftrace_ops, addrs, num, 1, 0);} static bool fprobe_is_ftrace(struct fprobe *fp){ return !fp->exit_handler;} #ifdef CONFIG_MODULESstatic void fprobe_set_ips(unsigned long *ips, unsigned int cnt, int remove, int reset){ ftrace_set_filter_ips(&fprobe_graph_ops.ops, ips, cnt, remove, reset); ftrace_set_filter_ips(&fprobe_ftrace_ops, ips, cnt, remove, reset);}#endif#elsestatic int fprobe_ftrace_add_ips(unsigned long *addrs, int num){ return -ENOENT;} static void fprobe_ftrace_remove_ips(unsigned long *addrs, int num){} static bool fprobe_is_ftrace(struct fprobe *fp){ return false;} #ifdef CONFIG_MODULESstatic void fprobe_set_ips(unsigned long *ips, unsigned int cnt, int remove, int reset){ ftrace_set_filter_ips(&fprobe_graph_ops.ops, ips, cnt, remove, reset);}#endif#endif /* !CONFIG_DYNAMIC_FTRACE_WITH_ARGS && !CONFIG_DYNAMIC_FTRACE_WITH_REGS */ /* fgraph_ops callback, this processes fprobes which have exit_handler. */static int fprobe_fgraph_entry(struct ftrace_graph_ent *trace, struct fgraph_ops *gops, struct ftrace_regs *fregs){ unsigned long *fgraph_data = NULL; unsigned long func = trace->func; struct fprobe_hlist_node *node; struct rhlist_head *head, *pos; unsigned long ret_ip; int reserved_words; struct fprobe *fp; int used, ret; if (WARN_ON_ONCE(!fregs)) return 0; guard(rcu)(); head = rhltable_lookup(&fprobe_ip_table, &func, fprobe_rht_params); reserved_words = 0; rhl_for_each_entry_rcu(node, pos, head, hlist) { if (node->addr != func) continue; fp = READ_ONCE(node->fp); if (!fp || !fp->exit_handler) continue; /* * Since fprobe can be enabled until the next loop, we ignore the * fprobe's disabled flag in this loop. */ reserved_words += FPROBE_HEADER_SIZE_IN_LONG + SIZE_IN_LONG(fp->entry_data_size); } if (reserved_words) { fgraph_data = fgraph_reserve_data(gops->idx, reserved_words * sizeof(long)); if (unlikely(!fgraph_data)) { rhl_for_each_entry_rcu(node, pos, head, hlist) { if (node->addr != func) continue; fp = READ_ONCE(node->fp); if (fp && !fprobe_disabled(fp) && !fprobe_is_ftrace(fp)) fp->nmissed++; } return 0; } } /* * TODO: recursion detection has been done in the fgraph. Thus we need * to add a callback to increment missed counter. */ ret_ip = ftrace_regs_get_return_address(fregs); used = 0; rhl_for_each_entry_rcu(node, pos, head, hlist) { int data_size; void *data; if (node->addr != func) continue; fp = READ_ONCE(node->fp); if (unlikely(!fp || fprobe_disabled(fp) || fprobe_is_ftrace(fp))) continue; data_size = fp->entry_data_size; if (data_size && fp->exit_handler) data = fgraph_data + used + FPROBE_HEADER_SIZE_IN_LONG; else data = NULL; if (fprobe_shared_with_kprobes(fp)) ret = __fprobe_kprobe_handler(func, ret_ip, fp, fregs, data); else ret = __fprobe_handler(func, ret_ip, fp, fregs, data); /* If entry_handler returns !0, nmissed is not counted but skips exit_handler. */ if (!ret && fp->exit_handler) { int size_words = SIZE_IN_LONG(data_size); if (write_fprobe_header(&fgraph_data[used], fp, size_words)) used += FPROBE_HEADER_SIZE_IN_LONG + size_words; } } if (used < reserved_words) memset(fgraph_data + used, 0, reserved_words - used); /* If any exit_handler is set, data must be used. */ return used != 0;}NOKPROBE_SYMBOL(fprobe_fgraph_entry); static void fprobe_return(struct ftrace_graph_ret *trace, struct fgraph_ops *gops, struct ftrace_regs *fregs){ unsigned long *fgraph_data = NULL; unsigned long ret_ip; struct fprobe *fp; int size, curr; int size_words; fgraph_data = (unsigned long *)fgraph_retrieve_data(gops->idx, &size); if (WARN_ON_ONCE(!fgraph_data)) return; size_words = SIZE_IN_LONG(size); ret_ip = ftrace_regs_get_instruction_pointer(fregs); preempt_disable_notrace(); curr = 0; while (size_words > curr) { read_fprobe_header(&fgraph_data[curr], &fp, &size); if (!fp) break; curr += FPROBE_HEADER_SIZE_IN_LONG; if (is_fprobe_still_exist(fp) && !fprobe_disabled(fp)) { if (WARN_ON_ONCE(curr + size > size_words)) break; fp->exit_handler(fp, trace->func, ret_ip, fregs, size ? fgraph_data + curr : NULL); } curr += size; } preempt_enable_notrace();}NOKPROBE_SYMBOL(fprobe_return); static struct fgraph_ops fprobe_graph_ops = { .entryfunc = fprobe_fgraph_entry, .retfunc = fprobe_return,};Lines 1–500 of 972. Load more
// SPDX-License-Identifier: GPL-2.0 /* * fprobe - Simple ftrace probe wrapper for function entry. */ #define pr_fmt(fmt) "fprobe: " fmt #include <linux/err.h> #include <linux/fprobe.h> #include <linux/kallsyms.h> #include <linux/kprobes.h> #include <linux/list.h> #include <linux/mutex.h> #include <linux/rhashtable.h> #include <linux/slab.h> #include <linux/sort.h> #include <asm/fprobe.h> #include "trace.h" #define FPROBE_IP_HASH_BITS 8 #define FPROBE_IP_TABLE_SIZE (1 << FPROBE_IP_HASH_BITS) #define FPROBE_HASH_BITS 6 #define FPROBE_TABLE_SIZE (1 << FPROBE_HASH_BITS) #define SIZE_IN_LONG(x) ((x + sizeof(long) - 1) >> (sizeof(long) == 8 ? 3 : 2)) /* * fprobe_table: hold 'fprobe_hlist::hlist' for checking the fprobe still * exists. The key is the address of fprobe instance. * fprobe_ip_table: hold 'fprobe_hlist::array[*]' for searching the fprobe * instance related to the function address. The key is the ftrace IP * address. * * When unregistering the fprobe, fprobe_hlist::fp and fprobe_hlist::array[*].fp * are set NULL and delete those from both hash tables (by hlist_del_rcu). * After an RCU grace period, the fprobe_hlist itself will be released. * * fprobe_table and fprobe_ip_table can be accessed from either * - Normal hlist traversal and RCU add/del under 'fprobe_mutex' is held. * - RCU hlist traversal under disabling preempt */ static struct hlist_head fprobe_table[FPROBE_TABLE_SIZE]; static struct rhltable fprobe_ip_table; static DEFINE_MUTEX(fprobe_mutex); static struct fgraph_ops fprobe_graph_ops; static u32 fprobe_node_hashfn(const void *data, u32 len, u32 seed) { return hash_ptr(*(unsigned long **)data, 32); } static int fprobe_node_cmp(struct rhashtable_compare_arg *arg, const void *ptr) { unsigned long key = *(unsigned long *)arg->key; const struct fprobe_hlist_node *n = ptr; return n->addr != key; } static u32 fprobe_node_obj_hashfn(const void *data, u32 len, u32 seed) { const struct fprobe_hlist_node *n = data; return hash_ptr((void *)n->addr, 32); } static const struct rhashtable_params fprobe_rht_params = { .head_offset = offsetof(struct fprobe_hlist_node, hlist), .key_offset = offsetof(struct fprobe_hlist_node, addr), .key_len = sizeof_field(struct fprobe_hlist_node, addr), .hashfn = fprobe_node_hashfn, .obj_hashfn = fprobe_node_obj_hashfn, .obj_cmpfn = fprobe_node_cmp, .automatic_shrinking = true, }; /* Node insertion and deletion requires the fprobe_mutex */ static int insert_fprobe_node(struct fprobe_hlist_node *node) { lockdep_assert_held(&fprobe_mutex); return rhltable_insert(&fprobe_ip_table, &node->hlist, fprobe_rht_params); } /* Return true if there are synonims */ static bool delete_fprobe_node(struct fprobe_hlist_node *node) { lockdep_assert_held(&fprobe_mutex); bool ret; /* Avoid double deleting */ if (READ_ONCE(node->fp) != NULL) { WRITE_ONCE(node->fp, NULL); rhltable_remove(&fprobe_ip_table, &node->hlist, fprobe_rht_params); } rcu_read_lock(); ret = !!rhltable_lookup(&fprobe_ip_table, &node->addr, fprobe_rht_params); rcu_read_unlock(); return ret; } /* Check existence of the fprobe */ static bool is_fprobe_still_exist(struct fprobe *fp) { struct hlist_head *head; struct fprobe_hlist *fph; head = &fprobe_table[hash_ptr(fp, FPROBE_HASH_BITS)]; hlist_for_each_entry_rcu(fph, head, hlist, lockdep_is_held(&fprobe_mutex)) { if (fph->fp == fp) return true; } return false; } NOKPROBE_SYMBOL(is_fprobe_still_exist); static int add_fprobe_hash(struct fprobe *fp) { struct fprobe_hlist *fph = fp->hlist_array; struct hlist_head *head; lockdep_assert_held(&fprobe_mutex); if (WARN_ON_ONCE(!fph)) return -EINVAL; if (is_fprobe_still_exist(fp)) return -EEXIST; head = &fprobe_table[hash_ptr(fp, FPROBE_HASH_BITS)]; hlist_add_head_rcu(&fp->hlist_array->hlist, head); return 0; } static int del_fprobe_hash(struct fprobe *fp) { struct fprobe_hlist *fph = fp->hlist_array; lockdep_assert_held(&fprobe_mutex); if (WARN_ON_ONCE(!fph)) return -EINVAL; if (!is_fprobe_still_exist(fp)) return -ENOENT; fph->fp = NULL; hlist_del_rcu(&fph->hlist); return 0; } #ifdef ARCH_DEFINE_ENCODE_FPROBE_HEADER /* The arch should encode fprobe_header info into one unsigned long */ #define FPROBE_HEADER_SIZE_IN_LONG 1 static inline bool write_fprobe_header(unsigned long *stack, struct fprobe *fp, unsigned int size_words) { if (WARN_ON_ONCE(size_words > MAX_FPROBE_DATA_SIZE_WORD || !arch_fprobe_header_encodable(fp))) return false; *stack = arch_encode_fprobe_header(fp, size_words); return true; } static inline void read_fprobe_header(unsigned long *stack, struct fprobe **fp, unsigned int *size_words) { *fp = arch_decode_fprobe_header_fp(*stack); *size_words = arch_decode_fprobe_header_size(*stack); } #else /* Generic fprobe_header */ struct __fprobe_header { struct fprobe *fp; unsigned long size_words; } __packed; #define FPROBE_HEADER_SIZE_IN_LONG SIZE_IN_LONG(sizeof(struct __fprobe_header)) static inline bool write_fprobe_header(unsigned long *stack, struct fprobe *fp, unsigned int size_words) { struct __fprobe_header *fph = (struct __fprobe_header *)stack; if (WARN_ON_ONCE(size_words > MAX_FPROBE_DATA_SIZE_WORD)) return false; fph->fp = fp; fph->size_words = size_words; return true; } static inline void read_fprobe_header(unsigned long *stack, struct fprobe **fp, unsigned int *size_words) { struct __fprobe_header *fph = (struct __fprobe_header *)stack; *fp = fph->fp; *size_words = fph->size_words; } #endif /* * fprobe shadow stack management: * Since fprobe shares a single fgraph_ops, it needs to share the stack entry * among the probes on the same function exit. Note that a new probe can be * registered before a target function is returning, we can not use the hash * table to find the corresponding probes. Thus the probe address is stored on * the shadow stack with its entry data size. * */ static inline int __fprobe_handler(unsigned long ip, unsigned long parent_ip, struct fprobe *fp, struct ftrace_regs *fregs, void *data) { if (!fp->entry_handler) return 0; return fp->entry_handler(fp, ip, parent_ip, fregs, data); } static inline int __fprobe_kprobe_handler(unsigned long ip, unsigned long parent_ip, struct fprobe *fp, struct ftrace_regs *fregs, void *data) { int ret; /* * This user handler is shared with other kprobes and is not expected to be * called recursively. So if any other kprobe handler is running, this will * exit as kprobe does. See the section 'Share the callbacks with kprobes' * in Documentation/trace/fprobe.rst for more information. */ if (unlikely(kprobe_running())) { fp->nmissed++; return 0; } kprobe_busy_begin(); ret = __fprobe_handler(ip, parent_ip, fp, fregs, data); kprobe_busy_end(); return ret; } #if defined(CONFIG_DYNAMIC_FTRACE_WITH_ARGS) || defined(CONFIG_DYNAMIC_FTRACE_WITH_REGS) /* ftrace_ops callback, this processes fprobes which have only entry_handler. */ static void fprobe_ftrace_entry(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *ops, struct ftrace_regs *fregs) { struct fprobe_hlist_node *node; struct rhlist_head *head, *pos; struct fprobe *fp; int bit; bit = ftrace_test_recursion_trylock(ip, parent_ip); if (bit < 0) return; /* * ftrace_test_recursion_trylock() disables preemption, but * rhltable_lookup() checks whether rcu_read_lcok is held. * So we take rcu_read_lock() here. */ rcu_read_lock(); head = rhltable_lookup(&fprobe_ip_table, &ip, fprobe_rht_params); rhl_for_each_entry_rcu(node, pos, head, hlist) { if (node->addr != ip) break; fp = READ_ONCE(node->fp); if (unlikely(!fp || fprobe_disabled(fp) || fp->exit_handler)) continue; if (fprobe_shared_with_kprobes(fp)) __fprobe_kprobe_handler(ip, parent_ip, fp, fregs, NULL); else __fprobe_handler(ip, parent_ip, fp, fregs, NULL); } rcu_read_unlock(); ftrace_test_recursion_unlock(bit); } NOKPROBE_SYMBOL(fprobe_ftrace_entry); static struct ftrace_ops fprobe_ftrace_ops = { .func = fprobe_ftrace_entry, .flags = FTRACE_OPS_FL_SAVE_ARGS, }; static int fprobe_ftrace_active; static int fprobe_ftrace_add_ips(unsigned long *addrs, int num) { int ret; lockdep_assert_held(&fprobe_mutex); ret = ftrace_set_filter_ips(&fprobe_ftrace_ops, addrs, num, 0, 0); if (ret) return ret; if (!fprobe_ftrace_active) { ret = register_ftrace_function(&fprobe_ftrace_ops); if (ret) { ftrace_free_filter(&fprobe_ftrace_ops); return ret; } } fprobe_ftrace_active++; return 0; } static void fprobe_ftrace_remove_ips(unsigned long *addrs, int num) { lockdep_assert_held(&fprobe_mutex); fprobe_ftrace_active--; if (!fprobe_ftrace_active) unregister_ftrace_function(&fprobe_ftrace_ops); if (num) ftrace_set_filter_ips(&fprobe_ftrace_ops, addrs, num, 1, 0); } static bool fprobe_is_ftrace(struct fprobe *fp) { return !fp->exit_handler; } #ifdef CONFIG_MODULES static void fprobe_set_ips(unsigned long *ips, unsigned int cnt, int remove, int reset) { ftrace_set_filter_ips(&fprobe_graph_ops.ops, ips, cnt, remove, reset); ftrace_set_filter_ips(&fprobe_ftrace_ops, ips, cnt, remove, reset); } #endif #else static int fprobe_ftrace_add_ips(unsigned long *addrs, int num) { return -ENOENT; } static void fprobe_ftrace_remove_ips(unsigned long *addrs, int num) { } static bool fprobe_is_ftrace(struct fprobe *fp) { return false; } #ifdef CONFIG_MODULES static void fprobe_set_ips(unsigned long *ips, unsigned int cnt, int remove, int reset) { ftrace_set_filter_ips(&fprobe_graph_ops.ops, ips, cnt, remove, reset); } #endif #endif /* !CONFIG_DYNAMIC_FTRACE_WITH_ARGS && !CONFIG_DYNAMIC_FTRACE_WITH_REGS */ /* fgraph_ops callback, this processes fprobes which have exit_handler. */ static int fprobe_fgraph_entry(struct ftrace_graph_ent *trace, struct fgraph_ops *gops, struct ftrace_regs *fregs) { unsigned long *fgraph_data = NULL; unsigned long func = trace->func; struct fprobe_hlist_node *node; struct rhlist_head *head, *pos; unsigned long ret_ip; int reserved_words; struct fprobe *fp; int used, ret; if (WARN_ON_ONCE(!fregs)) return 0; guard(rcu)(); head = rhltable_lookup(&fprobe_ip_table, &func, fprobe_rht_params); reserved_words = 0; rhl_for_each_entry_rcu(node, pos, head, hlist) { if (node->addr != func) continue; fp = READ_ONCE(node->fp); if (!fp || !fp->exit_handler) continue; /* * Since fprobe can be enabled until the next loop, we ignore the * fprobe's disabled flag in this loop. */ reserved_words += FPROBE_HEADER_SIZE_IN_LONG + SIZE_IN_LONG(fp->entry_data_size); } if (reserved_words) { fgraph_data = fgraph_reserve_data(gops->idx, reserved_words * sizeof(long)); if (unlikely(!fgraph_data)) { rhl_for_each_entry_rcu(node, pos, head, hlist) { if (node->addr != func) continue; fp = READ_ONCE(node->fp); if (fp && !fprobe_disabled(fp) && !fprobe_is_ftrace(fp)) fp->nmissed++; } return 0; } } /* * TODO: recursion detection has been done in the fgraph. Thus we need * to add a callback to increment missed counter. */ ret_ip = ftrace_regs_get_return_address(fregs); used = 0; rhl_for_each_entry_rcu(node, pos, head, hlist) { int data_size; void *data; if (node->addr != func) continue; fp = READ_ONCE(node->fp); if (unlikely(!fp || fprobe_disabled(fp) || fprobe_is_ftrace(fp))) continue; data_size = fp->entry_data_size; if (data_size && fp->exit_handler) data = fgraph_data + used + FPROBE_HEADER_SIZE_IN_LONG; else data = NULL; if (fprobe_shared_with_kprobes(fp)) ret = __fprobe_kprobe_handler(func, ret_ip, fp, fregs, data); else ret = __fprobe_handler(func, ret_ip, fp, fregs, data); /* If entry_handler returns !0, nmissed is not counted but skips exit_handler. */ if (!ret && fp->exit_handler) { int size_words = SIZE_IN_LONG(data_size); if (write_fprobe_header(&fgraph_data[used], fp, size_words)) used += FPROBE_HEADER_SIZE_IN_LONG + size_words; } } if (used < reserved_words) memset(fgraph_data + used, 0, reserved_words - used); /* If any exit_handler is set, data must be used. */ return used != 0; } NOKPROBE_SYMBOL(fprobe_fgraph_entry); static void fprobe_return(struct ftrace_graph_ret *trace, struct fgraph_ops *gops, struct ftrace_regs *fregs) { unsigned long *fgraph_data = NULL; unsigned long ret_ip; struct fprobe *fp; int size, curr; int size_words; fgraph_data = (unsigned long *)fgraph_retrieve_data(gops->idx, &size); if (WARN_ON_ONCE(!fgraph_data)) return; size_words = SIZE_IN_LONG(size); ret_ip = ftrace_regs_get_instruction_pointer(fregs); preempt_disable_notrace(); curr = 0; while (size_words > curr) { read_fprobe_header(&fgraph_data[curr], &fp, &size); if (!fp) break; curr += FPROBE_HEADER_SIZE_IN_LONG; if (is_fprobe_still_exist(fp) && !fprobe_disabled(fp)) { if (WARN_ON_ONCE(curr + size > size_words)) break; fp->exit_handler(fp, trace->func, ret_ip, fregs, size ? fgraph_data + curr : NULL); } curr += size; } preempt_enable_notrace(); } NOKPROBE_SYMBOL(fprobe_return); static struct fgraph_ops fprobe_graph_ops = { .entryfunc = fprobe_fgraph_entry, .retfunc = fprobe_return, };