Make BPF trampoline attach its generated assembly code to kernel functions via register_ftrace_direct() API. It helps ftrace-based tracers co-exist with BPF trampoline on the same kernel function. It also switches attaching logic from arch specific text_poke to generic ftrace that is available on many architectures. text_poke is still necessary for bpf-to-bpf attach and for bpf_tail_call optimization.
Unified · Split
| @@ -461,6 +461,7 @@ struct bpf_trampoline { · 460 unmodified lines | ||
| 461 | 461 | struct { |
| 462 | 462 | struct btf_func_model model; |
| 463 | 463 | void *addr; |
| 464 | + bool ftrace_managed; | |
| 464 | 465 | } func; |
| 465 | 466 | /* list of BPF programs using this trampoline */ |
| 466 | 467 | struct hlist_head progs_hlist[BPF_TRAMP_MAX]; |
| @@ -3,6 +3,7 @@ · 2 unmodified lines | ||
| 3 | 3 | #include <linux/hash.h> |
| 4 | 4 | #include <linux/bpf.h> |
| 5 | 5 | #include <linux/filter.h> |
| 6 | +#include <linux/ftrace.h> | |
| 6 | 7 | |
| 7 | 8 | /* btf_vmlinux has ~22k attachable functions. 1k htab is enough. */ |
| 8 | 9 | #define TRAMPOLINE_HASH_BITS 10 |
| @@ -59,6 +60,60 @@ out: · 50 unmodified lines | ||
| 59 | 60 | return tr; |
| 60 | 61 | } |
| 61 | 62 | |
| 63 | +static int is_ftrace_location(void *ip) | |
| 64 | +{ | |
| 65 | + long addr; | |
| 66 | + | |
| 67 | + addr = ftrace_location((long)ip); | |
| 68 | + if (!addr) | |
| 69 | + return 0; | |
| 70 | + if (WARN_ON_ONCE(addr != (long)ip)) | |
| 71 | + return -EFAULT; | |
| 72 | + return 1; | |
| 73 | +} | |
| 74 | + | |
| 75 | +static int unregister_fentry(struct bpf_trampoline *tr, void *old_addr) | |
| 76 | +{ | |
| 77 | + void *ip = tr->func.addr; | |
| 78 | + int ret; | |
| 79 | + | |
| 80 | + if (tr->func.ftrace_managed) | |
| 81 | + ret = unregister_ftrace_direct((long)ip, (long)old_addr); | |
| 82 | + else | |
| 83 | + ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, old_addr, NULL); | |
| 84 | + return ret; | |
| 85 | +} | |
| 86 | + | |
| 87 | +static int modify_fentry(struct bpf_trampoline *tr, void *old_addr, void *new_addr) | |
| 88 | +{ | |
| 89 | + void *ip = tr->func.addr; | |
| 90 | + int ret; | |
| 91 | + | |
| 92 | + if (tr->func.ftrace_managed) | |
| 93 | + ret = modify_ftrace_direct((long)ip, (long)old_addr, (long)new_addr); | |
| 94 | + else | |
| 95 | + ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, old_addr, new_addr); | |
| 96 | + return ret; | |
| 97 | +} | |
| 98 | + | |
| 99 | +/* first time registering */ | |
| 100 | +static int register_fentry(struct bpf_trampoline *tr, void *new_addr) | |
| 101 | +{ | |
| 102 | + void *ip = tr->func.addr; | |
| 103 | + int ret; | |
| 104 | + | |
| 105 | + ret = is_ftrace_location(ip); | |
| 106 | + if (ret < 0) | |
| 107 | + return ret; | |
| 108 | + tr->func.ftrace_managed = ret; | |
| 109 | + | |
| 110 | + if (tr->func.ftrace_managed) | |
| 111 | + ret = register_ftrace_direct((long)ip, (long)new_addr); | |
| 112 | + else | |
| 113 | + ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, NULL, new_addr); | |
| 114 | + return ret; | |
| 115 | +} | |
| 116 | + | |
| 62 | 117 | /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50 |
| 63 | 118 | * bytes on x86. Pick a number to fit into PAGE_SIZE / 2 |
| 64 | 119 | */ |
| @@ -77,8 +132,7 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr) · 12 unmodified lines | ||
| 77 | 132 | int err; |
| 78 | 133 | |
| 79 | 134 | if (fentry_cnt + fexit_cnt == 0) { |
| 80 | − err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_CALL, | |
| 81 | − old_image, NULL); | |
| 135 | + err = unregister_fentry(tr, old_image); | |
| 82 | 136 | tr->selector = 0; |
| 83 | 137 | goto out; |
| 84 | 138 | } |
| @@ -105,12 +159,10 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr) · 20 unmodified lines | ||
| 105 | 159 | |
| 106 | 160 | if (tr->selector) |
| 107 | 161 | /* progs already running at this address */ |
| 108 | − err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_CALL, | |
| 109 | − old_image, new_image); | |
| 162 | + err = modify_fentry(tr, old_image, new_image); | |
| 110 | 163 | else |
| 111 | 164 | /* first time registering */ |
| 112 | − err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_CALL, NULL, | |
| 113 | − new_image); | |
| 165 | + err = register_fentry(tr, new_image); | |
| 114 | 166 | if (err) |
| 115 | 167 | goto out; |
| 116 | 168 | tr->selector++; |