bpf: Add bpf_arch_text_poke() helper

Add bpf_arch_text_poke() helper that is used by BPF trampoline logic to patch nops/calls in kernel text into calls into BPF trampoline and to patch calls/nops inside BPF programs too.

Alexei Starovoitov committed 7 years ago

Commit 5964b201 parent 3b2744e3 files · +65 −0Browse files

Unified · Split

3 files

arch/x86/net/bpf_jit_comp.c +51 −0
include/linux/bpf.h +8 −0
kernel/bpf/core.c +6 −0

arch/x86/net/bpf_jit_comp.c+51−0

@@ -9,9 +9,11 @@ · 8 unmodified lines
9	#include <linux/filter.h>	9	#include <linux/filter.h>
10	#include <linux/if_vlan.h>	10	#include <linux/if_vlan.h>
11	#include <linux/bpf.h>	11	#include <linux/bpf.h>
		12	#include <linux/memory.h>
12	#include <asm/extable.h>	13	#include <asm/extable.h>
13	#include <asm/set_memory.h>	14	#include <asm/set_memory.h>
14	#include <asm/nospec-branch.h>	15	#include <asm/nospec-branch.h>
		16	#include <asm/text-patching.h>
15		17
16	static u8 emit_code(u8 ptr, u32 bytes, unsigned int len)	18	static u8 emit_code(u8 ptr, u32 bytes, unsigned int len)
17	{	19	{
@@ -486,6 +488,55 @@ static int emit_call(u8 *pprog, void func, void *ip) · 468 unmodified lines
486	return 0;	488	return 0;
487	}	489	}
488		490
		491	int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,
		492	void old_addr, void new_addr)
		493	{
		494	u8 old_insn[X86_CALL_SIZE] = {};
		495	u8 new_insn[X86_CALL_SIZE] = {};
		496	u8 *prog;
		497	int ret;
		498
		499	if (!is_kernel_text((long)ip))
		500	/* BPF trampoline in modules is not supported */
		501	return -EINVAL;
		502
		503	if (old_addr) {
		504	prog = old_insn;
		505	ret = emit_call(&prog, old_addr, (void *)ip);
		506	if (ret)
		507	return ret;
		508	}
		509	if (new_addr) {
		510	prog = new_insn;
		511	ret = emit_call(&prog, new_addr, (void *)ip);
		512	if (ret)
		513	return ret;
		514	}
		515	ret = -EBUSY;
		516	mutex_lock(&text_mutex);
		517	switch (t) {
		518	case BPF_MOD_NOP_TO_CALL:
		519	if (memcmp(ip, ideal_nops[NOP_ATOMIC5], X86_CALL_SIZE))
		520	goto out;
		521	text_poke_bp(ip, new_insn, X86_CALL_SIZE, NULL);
		522	break;
		523	case BPF_MOD_CALL_TO_CALL:
		524	if (memcmp(ip, old_insn, X86_CALL_SIZE))
		525	goto out;
		526	text_poke_bp(ip, new_insn, X86_CALL_SIZE, NULL);
		527	break;
		528	case BPF_MOD_CALL_TO_NOP:
		529	if (memcmp(ip, old_insn, X86_CALL_SIZE))
		530	goto out;
		531	text_poke_bp(ip, ideal_nops[NOP_ATOMIC5], X86_CALL_SIZE, NULL);
		532	break;
		533	}
		534	ret = 0;
		535	out:
		536	mutex_unlock(&text_mutex);
		537	return ret;
		538	}
		539
489	static bool ex_handler_bpf(const struct exception_table_entry *x,	540	static bool ex_handler_bpf(const struct exception_table_entry *x,
490	struct pt_regs *regs, int trapnr,	541	struct pt_regs *regs, int trapnr,
491	unsigned long error_code, unsigned long fault_addr)	542	unsigned long error_code, unsigned long fault_addr)

include/linux/bpf.h+8−0

@@ -1157,4 +1157,12 @@ static inline u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type, · 1156 unmodified lines
1157	}	1157	}
1158	#endif /* CONFIG_INET */	1158	#endif /* CONFIG_INET */
1159		1159
		1160	enum bpf_text_poke_type {
		1161	BPF_MOD_NOP_TO_CALL,
		1162	BPF_MOD_CALL_TO_CALL,
		1163	BPF_MOD_CALL_TO_NOP,
		1164	};
		1165	int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,
		1166	void addr1, void addr2);
		1167
1160	#endif /* _LINUX_BPF_H */	1168	#endif /* _LINUX_BPF_H */

kernel/bpf/core.c+6−0

 	return -EFAULT;
 }
+int __weak bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,
+			      void *addr1, void *addr2)
+{
+	return -ENOTSUPP;
+}
 DEFINE_STATIC_KEY_FALSE(bpf_stats_enabled_key);
 EXPORT_SYMBOL(bpf_stats_enabled_key);