History

bpf: Add bpf_arch_text_poke() helper

Add bpf_arch_text_poke() helper that is used by BPF trampoline logic to patch nops/calls in kernel text into calls into BPF trampoline and to patch calls/nops inside BPF programs too.

Alexei Starovoitov committed 7 years ago

Commit 5964b201 parent 3b2744e3 files · +65 −0Browse files

Unified · Split

3 files
  1. arch/x86/net/bpf_jit_comp.c +51 −0
  2. include/linux/bpf.h +8 −0
  3. kernel/bpf/core.c +6 −0
arch/x86/net/bpf_jit_comp.c+51−0
@@ -9,9 +9,11 @@ · 8 unmodified lines
99 #include <linux/filter.h>
1010 #include <linux/if_vlan.h>
1111 #include <linux/bpf.h>
12+#include <linux/memory.h>
1213 #include <asm/extable.h>
1314 #include <asm/set_memory.h>
1415 #include <asm/nospec-branch.h>
16+#include <asm/text-patching.h>
1517
1618 static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len)
1719 {
@@ -486,6 +488,55 @@ static int emit_call(u8 **pprog, void *func, void *ip) · 468 unmodified lines
486488 return 0;
487489 }
488490
491+int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,
492+ void *old_addr, void *new_addr)
493+{
494+ u8 old_insn[X86_CALL_SIZE] = {};
495+ u8 new_insn[X86_CALL_SIZE] = {};
496+ u8 *prog;
497+ int ret;
498+
499+ if (!is_kernel_text((long)ip))
500+ /* BPF trampoline in modules is not supported */
501+ return -EINVAL;
502+
503+ if (old_addr) {
504+ prog = old_insn;
505+ ret = emit_call(&prog, old_addr, (void *)ip);
506+ if (ret)
507+ return ret;
508+ }
509+ if (new_addr) {
510+ prog = new_insn;
511+ ret = emit_call(&prog, new_addr, (void *)ip);
512+ if (ret)
513+ return ret;
514+ }
515+ ret = -EBUSY;
516+ mutex_lock(&text_mutex);
517+ switch (t) {
518+ case BPF_MOD_NOP_TO_CALL:
519+ if (memcmp(ip, ideal_nops[NOP_ATOMIC5], X86_CALL_SIZE))
520+ goto out;
521+ text_poke_bp(ip, new_insn, X86_CALL_SIZE, NULL);
522+ break;
523+ case BPF_MOD_CALL_TO_CALL:
524+ if (memcmp(ip, old_insn, X86_CALL_SIZE))
525+ goto out;
526+ text_poke_bp(ip, new_insn, X86_CALL_SIZE, NULL);
527+ break;
528+ case BPF_MOD_CALL_TO_NOP:
529+ if (memcmp(ip, old_insn, X86_CALL_SIZE))
530+ goto out;
531+ text_poke_bp(ip, ideal_nops[NOP_ATOMIC5], X86_CALL_SIZE, NULL);
532+ break;
533+ }
534+ ret = 0;
535+out:
536+ mutex_unlock(&text_mutex);
537+ return ret;
538+}
539+
489540 static bool ex_handler_bpf(const struct exception_table_entry *x,
490541 struct pt_regs *regs, int trapnr,
491542 unsigned long error_code, unsigned long fault_addr)
include/linux/bpf.h+8−0
@@ -1157,4 +1157,12 @@ static inline u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type, · 1156 unmodified lines
11571157 }
11581158 #endif /* CONFIG_INET */
11591159
1160+enum bpf_text_poke_type {
1161+ BPF_MOD_NOP_TO_CALL,
1162+ BPF_MOD_CALL_TO_CALL,
1163+ BPF_MOD_CALL_TO_NOP,
1164+};
1165+int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,
1166+ void *addr1, void *addr2);
1167+
11601168 #endif /* _LINUX_BPF_H */
kernel/bpf/core.c+6−0
@@ -2144,6 +2144,12 @@ int __weak skb_copy_bits(const struct sk_buff *skb, int offset, void *to, · 2143 unmodified lines
21442144 return -EFAULT;
21452145 }
21462146
2147+int __weak bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,
2148+ void *addr1, void *addr2)
2149+{
2150+ return -ENOTSUPP;
2151+}
2152+
21472153 DEFINE_STATIC_KEY_FALSE(bpf_stats_enabled_key);
21482154 EXPORT_SYMBOL(bpf_stats_enabled_key);
21492155