// SPDX-License-Identifier: GPL-2.0-only/* Copyright (c) 2019 Facebook */#include <linux/hash.h>#include <linux/bpf.h>#include <linux/filter.h>#include <linux/ftrace.h>#include <linux/rbtree_latch.h>#include <linux/perf_event.h>#include <linux/btf.h>#include <linux/rcupdate_trace.h>#include <linux/rcupdate_wait.h>#include <linux/static_call.h>#include <linux/bpf_verifier.h>#include <linux/bpf_lsm.h>#include <linux/delay.h> /* dummy _ops. The verifier will operate on target program's ops. */const struct bpf_verifier_ops bpf_extension_verifier_ops = {};const struct bpf_prog_ops bpf_extension_prog_ops = {}; /* btf_vmlinux has ~22k attachable functions. 1k htab is enough. */#define TRAMPOLINE_HASH_BITS 10#define TRAMPOLINE_TABLE_SIZE (1 << TRAMPOLINE_HASH_BITS) static struct hlist_head trampoline_key_table[TRAMPOLINE_TABLE_SIZE];static struct hlist_head trampoline_ip_table[TRAMPOLINE_TABLE_SIZE]; /* serializes access to trampoline tables */static DEFINE_MUTEX(trampoline_mutex); /* * Keep 32 trampoline locks (5 bits) in the pool so trampoline_lock_all() * stays below MAX_LOCK_DEPTH. Each pool slot has a distinct lockdep * class because trampoline_lock_all() takes all pool mutexes at once; * otherwise lockdep would report recursive locking on same-class mutexes. */#define TRAMPOLINE_LOCKS_BITS 5#define TRAMPOLINE_LOCKS_TABLE_SIZE (1 << TRAMPOLINE_LOCKS_BITS) static struct { struct mutex mutex; struct lock_class_key key;} trampoline_locks[TRAMPOLINE_LOCKS_TABLE_SIZE]; static struct mutex *select_trampoline_lock(struct bpf_trampoline *tr){ return &trampoline_locks[hash_ptr(tr, TRAMPOLINE_LOCKS_BITS)].mutex;} static void trampoline_lock(struct bpf_trampoline *tr){ mutex_lock(select_trampoline_lock(tr));} static void trampoline_unlock(struct bpf_trampoline *tr){ mutex_unlock(select_trampoline_lock(tr));} struct bpf_trampoline_ops { int (*register_fentry)(struct bpf_trampoline *tr, struct bpf_tramp_image *im, void *data); int (*unregister_fentry)(struct bpf_trampoline *tr, u32 orig_flags, void *data); int (*modify_fentry)(struct bpf_trampoline *tr, u32 orig_flags, struct bpf_tramp_image *im, bool lock_direct_mutex, void *data);}; #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLSstatic int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex, const struct bpf_trampoline_ops *ops, void *data);static const struct bpf_trampoline_ops trampoline_ops; #ifdef CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPSstatic struct bpf_trampoline *direct_ops_ip_lookup(struct ftrace_ops *ops, unsigned long ip){ struct hlist_head *head_ip; struct bpf_trampoline *tr; mutex_lock(&trampoline_mutex); head_ip = &trampoline_ip_table[hash_64(ip, TRAMPOLINE_HASH_BITS)]; hlist_for_each_entry(tr, head_ip, hlist_ip) { if (tr->ip == ip) goto out; } tr = NULL;out: mutex_unlock(&trampoline_mutex); return tr;}#elsestatic struct bpf_trampoline *direct_ops_ip_lookup(struct ftrace_ops *ops, unsigned long ip){ return ops->private;}#endif /* CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS */ static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, unsigned long ip, enum ftrace_ops_cmd cmd){ struct bpf_trampoline *tr; int ret = 0; tr = direct_ops_ip_lookup(ops, ip); if (!tr) return -EINVAL; if (cmd == FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_SELF) { /* This is called inside register_ftrace_direct_multi(), so * trampoline's mutex is already locked. */ lockdep_assert_held_once(select_trampoline_lock(tr)); /* Instead of updating the trampoline here, we propagate * -EAGAIN to register_ftrace_direct(). Then we can * retry register_ftrace_direct() after updating the * trampoline. */ if ((tr->flags & BPF_TRAMP_F_CALL_ORIG) && !(tr->flags & BPF_TRAMP_F_ORIG_STACK)) { if (WARN_ON_ONCE(tr->flags & BPF_TRAMP_F_SHARE_IPMODIFY)) return -EBUSY; tr->flags |= BPF_TRAMP_F_SHARE_IPMODIFY; return -EAGAIN; } return 0; } /* The normal locking order is * select_trampoline_lock(tr) => direct_mutex (ftrace.c) => ftrace_lock (ftrace.c) * * The following two commands are called from * * prepare_direct_functions_for_ipmodify * cleanup_direct_functions_after_ipmodify * * In both cases, direct_mutex is already locked. Use * mutex_trylock(select_trampoline_lock(tr)) to avoid deadlock in race condition * (something else holds the same pool lock). */ if (!mutex_trylock(select_trampoline_lock(tr))) { /* sleep 1 ms to make sure whatever holding select_trampoline_lock(tr) * makes some progress. */ msleep(1); return -EAGAIN; } switch (cmd) { case FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_PEER: tr->flags |= BPF_TRAMP_F_SHARE_IPMODIFY; if ((tr->flags & BPF_TRAMP_F_CALL_ORIG) && !(tr->flags & BPF_TRAMP_F_ORIG_STACK)) ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */, &trampoline_ops, NULL); break; case FTRACE_OPS_CMD_DISABLE_SHARE_IPMODIFY_PEER: tr->flags &= ~BPF_TRAMP_F_SHARE_IPMODIFY; if (tr->flags & BPF_TRAMP_F_ORIG_STACK) ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */, &trampoline_ops, NULL); break; default: ret = -EINVAL; break; } trampoline_unlock(tr); return ret;}#endif bool bpf_prog_has_trampoline(const struct bpf_prog *prog){ enum bpf_attach_type eatype = prog->expected_attach_type; enum bpf_prog_type ptype = prog->type; switch (ptype) { case BPF_PROG_TYPE_TRACING: if (eatype == BPF_TRACE_FENTRY || eatype == BPF_TRACE_FEXIT || eatype == BPF_MODIFY_RETURN || eatype == BPF_TRACE_FSESSION || eatype == BPF_TRACE_FENTRY_MULTI || eatype == BPF_TRACE_FEXIT_MULTI || eatype == BPF_TRACE_FSESSION_MULTI) return true; return false; case BPF_PROG_TYPE_LSM: return eatype == BPF_LSM_MAC; default: return false; }} void bpf_image_ksym_init(void *data, unsigned int size, struct bpf_ksym *ksym){ ksym->start = (unsigned long) data; ksym->end = ksym->start + size;} void bpf_image_ksym_add(struct bpf_ksym *ksym){ bpf_ksym_add(ksym); perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_BPF, ksym->start, PAGE_SIZE, false, ksym->name);} void bpf_image_ksym_del(struct bpf_ksym *ksym){ bpf_ksym_del(ksym); perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_BPF, ksym->start, PAGE_SIZE, true, ksym->name);} #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS#ifdef CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS/* * We have only single direct_ops which contains all the direct call * sites and is the only global ftrace_ops for all trampolines. * * We use 'update_ftrace_direct_*' api for attachment. */struct ftrace_ops direct_ops = { .ops_func = bpf_tramp_ftrace_ops_func,}; static int direct_ops_alloc(struct bpf_trampoline *tr){ tr->fops = &direct_ops; return 0;} static void direct_ops_free(struct bpf_trampoline *tr) { } static struct ftrace_hash *hash_from_ip(struct bpf_trampoline *tr, void *ptr){ unsigned long ip, addr = (unsigned long) ptr; struct ftrace_hash *hash; ip = ftrace_location(tr->ip); if (!ip) return NULL; hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS); if (!hash) return NULL; if (bpf_trampoline_use_jmp(tr->flags)) addr = ftrace_jmp_set(addr); if (!add_ftrace_hash_entry_direct(hash, ip, addr)) { free_ftrace_hash(hash); return NULL; } return hash;} static int direct_ops_add(struct bpf_trampoline *tr, void *addr){ struct ftrace_hash *hash = hash_from_ip(tr, addr); int err; if (!hash) return -ENOMEM; err = update_ftrace_direct_add(tr->fops, hash); free_ftrace_hash(hash); return err;} static int direct_ops_del(struct bpf_trampoline *tr, void *addr){ struct ftrace_hash *hash = hash_from_ip(tr, addr); int err; if (!hash) return -ENOMEM; err = update_ftrace_direct_del(tr->fops, hash); free_ftrace_hash(hash); return err;} static int direct_ops_mod(struct bpf_trampoline *tr, void *addr, bool lock_direct_mutex){ struct ftrace_hash *hash = hash_from_ip(tr, addr); int err; if (!hash) return -ENOMEM; err = update_ftrace_direct_mod(tr->fops, hash, lock_direct_mutex); free_ftrace_hash(hash); return err;}#else/* * We allocate ftrace_ops object for each trampoline and it contains * call site specific for that trampoline. * * We use *_ftrace_direct api for attachment. */static int direct_ops_alloc(struct bpf_trampoline *tr){ tr->fops = kzalloc_obj(struct ftrace_ops); if (!tr->fops) return -ENOMEM; tr->fops->private = tr; tr->fops->ops_func = bpf_tramp_ftrace_ops_func; return 0;} static void direct_ops_free(struct bpf_trampoline *tr){ if (!tr->fops) return; ftrace_free_filter(tr->fops); kfree(tr->fops);} static int direct_ops_add(struct bpf_trampoline *tr, void *ptr){ unsigned long addr = (unsigned long) ptr; struct ftrace_ops *ops = tr->fops; int ret; if (bpf_trampoline_use_jmp(tr->flags)) addr = ftrace_jmp_set(addr); ret = ftrace_set_filter_ip(ops, tr->ip, 0, 1); if (ret) return ret; return register_ftrace_direct(ops, addr);} static int direct_ops_del(struct bpf_trampoline *tr, void *addr){ return unregister_ftrace_direct(tr->fops, (long)addr, false);} static int direct_ops_mod(struct bpf_trampoline *tr, void *ptr, bool lock_direct_mutex){ unsigned long addr = (unsigned long) ptr; struct ftrace_ops *ops = tr->fops; if (bpf_trampoline_use_jmp(tr->flags)) addr = ftrace_jmp_set(addr); if (lock_direct_mutex) return modify_ftrace_direct(ops, addr); return modify_ftrace_direct_nolock(ops, addr);}#endif /* CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS */#elsestatic void direct_ops_free(struct bpf_trampoline *tr) { } static int direct_ops_alloc(struct bpf_trampoline *tr){ return 0;} static int direct_ops_add(struct bpf_trampoline *tr, void *addr){ return -ENODEV;} static int direct_ops_del(struct bpf_trampoline *tr, void *addr){ return -ENODEV;} static int direct_ops_mod(struct bpf_trampoline *tr, void *ptr, bool lock_direct_mutex){ return -ENODEV;}#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ static struct bpf_trampoline *bpf_trampoline_lookup(u64 key, unsigned long ip){ struct bpf_trampoline *tr; struct hlist_head *head; int i; mutex_lock(&trampoline_mutex); head = &trampoline_key_table[hash_64(key, TRAMPOLINE_HASH_BITS)]; hlist_for_each_entry(tr, head, hlist_key) { if (tr->key == key) { refcount_inc(&tr->refcnt); goto out; } } tr = kzalloc_obj(*tr); if (!tr) goto out; if (direct_ops_alloc(tr)) { kfree(tr); tr = NULL; goto out; } tr->key = key; tr->ip = ftrace_location(ip); INIT_HLIST_NODE(&tr->hlist_key); INIT_HLIST_NODE(&tr->hlist_ip); hlist_add_head(&tr->hlist_key, head); head = &trampoline_ip_table[hash_64(tr->ip, TRAMPOLINE_HASH_BITS)]; hlist_add_head(&tr->hlist_ip, head); refcount_set(&tr->refcnt, 1); for (i = 0; i < BPF_TRAMP_MAX; i++) INIT_HLIST_HEAD(&tr->progs_hlist[i]);out: mutex_unlock(&trampoline_mutex); return tr;} static int bpf_trampoline_update_fentry(struct bpf_trampoline *tr, u32 orig_flags, void *old_addr, void *new_addr){ enum bpf_text_poke_type new_t = BPF_MOD_CALL, old_t = BPF_MOD_CALL; void *ip = tr->func.addr; if (!new_addr) new_t = BPF_MOD_NOP; else if (bpf_trampoline_use_jmp(tr->flags)) new_t = BPF_MOD_JUMP; if (!old_addr) old_t = BPF_MOD_NOP; else if (bpf_trampoline_use_jmp(orig_flags)) old_t = BPF_MOD_JUMP; return bpf_arch_text_poke(ip, old_t, new_t, old_addr, new_addr);} static void bpf_tramp_image_put(struct bpf_tramp_image *im); static int unregister_fentry(struct bpf_trampoline *tr, u32 orig_flags, void *data __maybe_unused){ void *old_addr = tr->cur_image->image; int ret; if (tr->func.ftrace_managed) ret = direct_ops_del(tr, old_addr); else ret = bpf_trampoline_update_fentry(tr, orig_flags, old_addr, NULL); if (ret) return ret; bpf_tramp_image_put(tr->cur_image); tr->cur_image = NULL; return 0;} static int modify_fentry(struct bpf_trampoline *tr, u32 orig_flags, struct bpf_tramp_image *im, bool lock_direct_mutex, void *data __maybe_unused){ void *old_addr = tr->cur_image->image; void *new_addr = im->image; int ret; if (tr->func.ftrace_managed) { ret = direct_ops_mod(tr, new_addr, lock_direct_mutex); } else { ret = bpf_trampoline_update_fentry(tr, orig_flags, old_addr, new_addr); } if (ret) return ret; bpf_tramp_image_put(tr->cur_image); tr->cur_image = im; return 0;} /* first time registering */static int register_fentry(struct bpf_trampoline *tr, struct bpf_tramp_image *im, void *data __maybe_unused){ void *new_addr = im->image; void *ip = tr->func.addr; unsigned long faddr; int ret; faddr = ftrace_location((unsigned long)ip); if (faddr) { if (!tr->fops) return -ENOTSUPP; tr->func.ftrace_managed = true; } if (tr->func.ftrace_managed) { ret = direct_ops_add(tr, new_addr); } else { ret = bpf_trampoline_update_fentry(tr, 0, NULL, new_addr); } if (ret) return ret; tr->cur_image = im; return 0;} Lines 1–500 of 1821. Load more
// SPDX-License-Identifier: GPL-2.0-only /* Copyright (c) 2019 Facebook */ #include <linux/hash.h> #include <linux/bpf.h> #include <linux/filter.h> #include <linux/ftrace.h> #include <linux/rbtree_latch.h> #include <linux/perf_event.h> #include <linux/btf.h> #include <linux/rcupdate_trace.h> #include <linux/rcupdate_wait.h> #include <linux/static_call.h> #include <linux/bpf_verifier.h> #include <linux/bpf_lsm.h> #include <linux/delay.h> /* dummy _ops. The verifier will operate on target program's ops. */ const struct bpf_verifier_ops bpf_extension_verifier_ops = { }; const struct bpf_prog_ops bpf_extension_prog_ops = { }; /* btf_vmlinux has ~22k attachable functions. 1k htab is enough. */ #define TRAMPOLINE_HASH_BITS 10 #define TRAMPOLINE_TABLE_SIZE (1 << TRAMPOLINE_HASH_BITS) static struct hlist_head trampoline_key_table[TRAMPOLINE_TABLE_SIZE]; static struct hlist_head trampoline_ip_table[TRAMPOLINE_TABLE_SIZE]; /* serializes access to trampoline tables */ static DEFINE_MUTEX(trampoline_mutex); /* * Keep 32 trampoline locks (5 bits) in the pool so trampoline_lock_all() * stays below MAX_LOCK_DEPTH. Each pool slot has a distinct lockdep * class because trampoline_lock_all() takes all pool mutexes at once; * otherwise lockdep would report recursive locking on same-class mutexes. */ #define TRAMPOLINE_LOCKS_BITS 5 #define TRAMPOLINE_LOCKS_TABLE_SIZE (1 << TRAMPOLINE_LOCKS_BITS) static struct { struct mutex mutex; struct lock_class_key key; } trampoline_locks[TRAMPOLINE_LOCKS_TABLE_SIZE]; static struct mutex *select_trampoline_lock(struct bpf_trampoline *tr) { return &trampoline_locks[hash_ptr(tr, TRAMPOLINE_LOCKS_BITS)].mutex; } static void trampoline_lock(struct bpf_trampoline *tr) { mutex_lock(select_trampoline_lock(tr)); } static void trampoline_unlock(struct bpf_trampoline *tr) { mutex_unlock(select_trampoline_lock(tr)); } struct bpf_trampoline_ops { int (*register_fentry)(struct bpf_trampoline *tr, struct bpf_tramp_image *im, void *data); int (*unregister_fentry)(struct bpf_trampoline *tr, u32 orig_flags, void *data); int (*modify_fentry)(struct bpf_trampoline *tr, u32 orig_flags, struct bpf_tramp_image *im, bool lock_direct_mutex, void *data); }; #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex, const struct bpf_trampoline_ops *ops, void *data); static const struct bpf_trampoline_ops trampoline_ops; #ifdef CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS static struct bpf_trampoline *direct_ops_ip_lookup(struct ftrace_ops *ops, unsigned long ip) { struct hlist_head *head_ip; struct bpf_trampoline *tr; mutex_lock(&trampoline_mutex); head_ip = &trampoline_ip_table[hash_64(ip, TRAMPOLINE_HASH_BITS)]; hlist_for_each_entry(tr, head_ip, hlist_ip) { if (tr->ip == ip) goto out; } tr = NULL; out: mutex_unlock(&trampoline_mutex); return tr; } #else static struct bpf_trampoline *direct_ops_ip_lookup(struct ftrace_ops *ops, unsigned long ip) { return ops->private; } #endif /* CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS */ static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, unsigned long ip, enum ftrace_ops_cmd cmd) { struct bpf_trampoline *tr; int ret = 0; tr = direct_ops_ip_lookup(ops, ip); if (!tr) return -EINVAL; if (cmd == FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_SELF) { /* This is called inside register_ftrace_direct_multi(), so * trampoline's mutex is already locked. */ lockdep_assert_held_once(select_trampoline_lock(tr)); /* Instead of updating the trampoline here, we propagate * -EAGAIN to register_ftrace_direct(). Then we can * retry register_ftrace_direct() after updating the * trampoline. */ if ((tr->flags & BPF_TRAMP_F_CALL_ORIG) && !(tr->flags & BPF_TRAMP_F_ORIG_STACK)) { if (WARN_ON_ONCE(tr->flags & BPF_TRAMP_F_SHARE_IPMODIFY)) return -EBUSY; tr->flags |= BPF_TRAMP_F_SHARE_IPMODIFY; return -EAGAIN; } return 0; } /* The normal locking order is * select_trampoline_lock(tr) => direct_mutex (ftrace.c) => ftrace_lock (ftrace.c) * * The following two commands are called from * * prepare_direct_functions_for_ipmodify * cleanup_direct_functions_after_ipmodify * * In both cases, direct_mutex is already locked. Use * mutex_trylock(select_trampoline_lock(tr)) to avoid deadlock in race condition * (something else holds the same pool lock). */ if (!mutex_trylock(select_trampoline_lock(tr))) { /* sleep 1 ms to make sure whatever holding select_trampoline_lock(tr) * makes some progress. */ msleep(1); return -EAGAIN; } switch (cmd) { case FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_PEER: tr->flags |= BPF_TRAMP_F_SHARE_IPMODIFY; if ((tr->flags & BPF_TRAMP_F_CALL_ORIG) && !(tr->flags & BPF_TRAMP_F_ORIG_STACK)) ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */, &trampoline_ops, NULL); break; case FTRACE_OPS_CMD_DISABLE_SHARE_IPMODIFY_PEER: tr->flags &= ~BPF_TRAMP_F_SHARE_IPMODIFY; if (tr->flags & BPF_TRAMP_F_ORIG_STACK) ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */, &trampoline_ops, NULL); break; default: ret = -EINVAL; break; } trampoline_unlock(tr); return ret; } #endif bool bpf_prog_has_trampoline(const struct bpf_prog *prog) { enum bpf_attach_type eatype = prog->expected_attach_type; enum bpf_prog_type ptype = prog->type; switch (ptype) { case BPF_PROG_TYPE_TRACING: if (eatype == BPF_TRACE_FENTRY || eatype == BPF_TRACE_FEXIT || eatype == BPF_MODIFY_RETURN || eatype == BPF_TRACE_FSESSION || eatype == BPF_TRACE_FENTRY_MULTI || eatype == BPF_TRACE_FEXIT_MULTI || eatype == BPF_TRACE_FSESSION_MULTI) return true; return false; case BPF_PROG_TYPE_LSM: return eatype == BPF_LSM_MAC; default: return false; } } void bpf_image_ksym_init(void *data, unsigned int size, struct bpf_ksym *ksym) { ksym->start = (unsigned long) data; ksym->end = ksym->start + size; } void bpf_image_ksym_add(struct bpf_ksym *ksym) { bpf_ksym_add(ksym); perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_BPF, ksym->start, PAGE_SIZE, false, ksym->name); } void bpf_image_ksym_del(struct bpf_ksym *ksym) { bpf_ksym_del(ksym); perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_BPF, ksym->start, PAGE_SIZE, true, ksym->name); } #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS #ifdef CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS /* * We have only single direct_ops which contains all the direct call * sites and is the only global ftrace_ops for all trampolines. * * We use 'update_ftrace_direct_*' api for attachment. */ struct ftrace_ops direct_ops = { .ops_func = bpf_tramp_ftrace_ops_func, }; static int direct_ops_alloc(struct bpf_trampoline *tr) { tr->fops = &direct_ops; return 0; } static void direct_ops_free(struct bpf_trampoline *tr) { } static struct ftrace_hash *hash_from_ip(struct bpf_trampoline *tr, void *ptr) { unsigned long ip, addr = (unsigned long) ptr; struct ftrace_hash *hash; ip = ftrace_location(tr->ip); if (!ip) return NULL; hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS); if (!hash) return NULL; if (bpf_trampoline_use_jmp(tr->flags)) addr = ftrace_jmp_set(addr); if (!add_ftrace_hash_entry_direct(hash, ip, addr)) { free_ftrace_hash(hash); return NULL; } return hash; } static int direct_ops_add(struct bpf_trampoline *tr, void *addr) { struct ftrace_hash *hash = hash_from_ip(tr, addr); int err; if (!hash) return -ENOMEM; err = update_ftrace_direct_add(tr->fops, hash); free_ftrace_hash(hash); return err; } static int direct_ops_del(struct bpf_trampoline *tr, void *addr) { struct ftrace_hash *hash = hash_from_ip(tr, addr); int err; if (!hash) return -ENOMEM; err = update_ftrace_direct_del(tr->fops, hash); free_ftrace_hash(hash); return err; } static int direct_ops_mod(struct bpf_trampoline *tr, void *addr, bool lock_direct_mutex) { struct ftrace_hash *hash = hash_from_ip(tr, addr); int err; if (!hash) return -ENOMEM; err = update_ftrace_direct_mod(tr->fops, hash, lock_direct_mutex); free_ftrace_hash(hash); return err; } #else /* * We allocate ftrace_ops object for each trampoline and it contains * call site specific for that trampoline. * * We use *_ftrace_direct api for attachment. */ static int direct_ops_alloc(struct bpf_trampoline *tr) { tr->fops = kzalloc_obj(struct ftrace_ops); if (!tr->fops) return -ENOMEM; tr->fops->private = tr; tr->fops->ops_func = bpf_tramp_ftrace_ops_func; return 0; } static void direct_ops_free(struct bpf_trampoline *tr) { if (!tr->fops) return; ftrace_free_filter(tr->fops); kfree(tr->fops); } static int direct_ops_add(struct bpf_trampoline *tr, void *ptr) { unsigned long addr = (unsigned long) ptr; struct ftrace_ops *ops = tr->fops; int ret; if (bpf_trampoline_use_jmp(tr->flags)) addr = ftrace_jmp_set(addr); ret = ftrace_set_filter_ip(ops, tr->ip, 0, 1); if (ret) return ret; return register_ftrace_direct(ops, addr); } static int direct_ops_del(struct bpf_trampoline *tr, void *addr) { return unregister_ftrace_direct(tr->fops, (long)addr, false); } static int direct_ops_mod(struct bpf_trampoline *tr, void *ptr, bool lock_direct_mutex) { unsigned long addr = (unsigned long) ptr; struct ftrace_ops *ops = tr->fops; if (bpf_trampoline_use_jmp(tr->flags)) addr = ftrace_jmp_set(addr); if (lock_direct_mutex) return modify_ftrace_direct(ops, addr); return modify_ftrace_direct_nolock(ops, addr); } #endif /* CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS */ #else static void direct_ops_free(struct bpf_trampoline *tr) { } static int direct_ops_alloc(struct bpf_trampoline *tr) { return 0; } static int direct_ops_add(struct bpf_trampoline *tr, void *addr) { return -ENODEV; } static int direct_ops_del(struct bpf_trampoline *tr, void *addr) { return -ENODEV; } static int direct_ops_mod(struct bpf_trampoline *tr, void *ptr, bool lock_direct_mutex) { return -ENODEV; } #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ static struct bpf_trampoline *bpf_trampoline_lookup(u64 key, unsigned long ip) { struct bpf_trampoline *tr; struct hlist_head *head; int i; mutex_lock(&trampoline_mutex); head = &trampoline_key_table[hash_64(key, TRAMPOLINE_HASH_BITS)]; hlist_for_each_entry(tr, head, hlist_key) { if (tr->key == key) { refcount_inc(&tr->refcnt); goto out; } } tr = kzalloc_obj(*tr); if (!tr) goto out; if (direct_ops_alloc(tr)) { kfree(tr); tr = NULL; goto out; } tr->key = key; tr->ip = ftrace_location(ip); INIT_HLIST_NODE(&tr->hlist_key); INIT_HLIST_NODE(&tr->hlist_ip); hlist_add_head(&tr->hlist_key, head); head = &trampoline_ip_table[hash_64(tr->ip, TRAMPOLINE_HASH_BITS)]; hlist_add_head(&tr->hlist_ip, head); refcount_set(&tr->refcnt, 1); for (i = 0; i < BPF_TRAMP_MAX; i++) INIT_HLIST_HEAD(&tr->progs_hlist[i]); out: mutex_unlock(&trampoline_mutex); return tr; } static int bpf_trampoline_update_fentry(struct bpf_trampoline *tr, u32 orig_flags, void *old_addr, void *new_addr) { enum bpf_text_poke_type new_t = BPF_MOD_CALL, old_t = BPF_MOD_CALL; void *ip = tr->func.addr; if (!new_addr) new_t = BPF_MOD_NOP; else if (bpf_trampoline_use_jmp(tr->flags)) new_t = BPF_MOD_JUMP; if (!old_addr) old_t = BPF_MOD_NOP; else if (bpf_trampoline_use_jmp(orig_flags)) old_t = BPF_MOD_JUMP; return bpf_arch_text_poke(ip, old_t, new_t, old_addr, new_addr); } static void bpf_tramp_image_put(struct bpf_tramp_image *im); static int unregister_fentry(struct bpf_trampoline *tr, u32 orig_flags, void *data __maybe_unused) { void *old_addr = tr->cur_image->image; int ret; if (tr->func.ftrace_managed) ret = direct_ops_del(tr, old_addr); else ret = bpf_trampoline_update_fentry(tr, orig_flags, old_addr, NULL); if (ret) return ret; bpf_tramp_image_put(tr->cur_image); tr->cur_image = NULL; return 0; } static int modify_fentry(struct bpf_trampoline *tr, u32 orig_flags, struct bpf_tramp_image *im, bool lock_direct_mutex, void *data __maybe_unused) { void *old_addr = tr->cur_image->image; void *new_addr = im->image; int ret; if (tr->func.ftrace_managed) { ret = direct_ops_mod(tr, new_addr, lock_direct_mutex); } else { ret = bpf_trampoline_update_fentry(tr, orig_flags, old_addr, new_addr); } if (ret) return ret; bpf_tramp_image_put(tr->cur_image); tr->cur_image = im; return 0; } /* first time registering */ static int register_fentry(struct bpf_trampoline *tr, struct bpf_tramp_image *im, void *data __maybe_unused) { void *new_addr = im->image; void *ip = tr->func.addr; unsigned long faddr; int ret; faddr = ftrace_location((unsigned long)ip); if (faddr) { if (!tr->fops) return -ENOTSUPP; tr->func.ftrace_managed = true; } if (tr->func.ftrace_managed) { ret = direct_ops_add(tr, new_addr); } else { ret = bpf_trampoline_update_fentry(tr, 0, NULL, new_addr); } if (ret) return ret; tr->cur_image = im; return 0; }