bpf: Make BPF trampoline use register_ftrace_direct() API

Make BPF trampoline attach its generated assembly code to kernel functions via register_ftrace_direct() API. It helps ftrace-based tracers co-exist with BPF trampoline on the same kernel function. It also switches attaching logic from arch specific text_poke to generic ftrace that is available on many architectures. text_poke is still necessary for bpf-to-bpf attach and for bpf_tail_call optimization.

Alexei Starovoitov committed 6 years ago

Commit b91e0141 parent 5b79bcd2 files · +59 −6Browse files

Unified · Split

2 files

include/linux/bpf.h +1 −0
kernel/bpf/trampoline.c +58 −6

include/linux/bpf.h+1−0

@@ -461,6 +461,7 @@ struct bpf_trampoline { · 460 unmodified lines
461	struct {	461	struct {
462	struct btf_func_model model;	462	struct btf_func_model model;
463	void *addr;	463	void *addr;
		464	bool ftrace_managed;
464	} func;	465	} func;
465	/* list of BPF programs using this trampoline */	466	/* list of BPF programs using this trampoline */
466	struct hlist_head progs_hlist[BPF_TRAMP_MAX];	467	struct hlist_head progs_hlist[BPF_TRAMP_MAX];

kernel/bpf/trampoline.c+58−6

@@ -3,6 +3,7 @@ · 2 unmodified lines
3	#include <linux/hash.h>	3	#include <linux/hash.h>
4	#include <linux/bpf.h>	4	#include <linux/bpf.h>
5	#include <linux/filter.h>	5	#include <linux/filter.h>
		6	#include <linux/ftrace.h>
6		7
7	/* btf_vmlinux has ~22k attachable functions. 1k htab is enough. */	8	/* btf_vmlinux has ~22k attachable functions. 1k htab is enough. */
8	#define TRAMPOLINE_HASH_BITS 10	9	#define TRAMPOLINE_HASH_BITS 10
@@ -59,6 +60,60 @@ out: · 50 unmodified lines
59	return tr;	60	return tr;
60	}	61	}
61		62
		63	static int is_ftrace_location(void *ip)
		64	{
		65	long addr;
		66
		67	addr = ftrace_location((long)ip);
		68	if (!addr)
		69	return 0;
		70	if (WARN_ON_ONCE(addr != (long)ip))
		71	return -EFAULT;
		72	return 1;
		73	}
		74
		75	static int unregister_fentry(struct bpf_trampoline tr, void old_addr)
		76	{
		77	void *ip = tr->func.addr;
		78	int ret;
		79
		80	if (tr->func.ftrace_managed)
		81	ret = unregister_ftrace_direct((long)ip, (long)old_addr);
		82	else
		83	ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, old_addr, NULL);
		84	return ret;
		85	}
		86
		87	static int modify_fentry(struct bpf_trampoline tr, void old_addr, void *new_addr)
		88	{
		89	void *ip = tr->func.addr;
		90	int ret;
		91
		92	if (tr->func.ftrace_managed)
		93	ret = modify_ftrace_direct((long)ip, (long)old_addr, (long)new_addr);
		94	else
		95	ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, old_addr, new_addr);
		96	return ret;
		97	}
		98
		99	/* first time registering */
		100	static int register_fentry(struct bpf_trampoline tr, void new_addr)
		101	{
		102	void *ip = tr->func.addr;
		103	int ret;
		104
		105	ret = is_ftrace_location(ip);
		106	if (ret < 0)
		107	return ret;
		108	tr->func.ftrace_managed = ret;
		109
		110	if (tr->func.ftrace_managed)
		111	ret = register_ftrace_direct((long)ip, (long)new_addr);
		112	else
		113	ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, NULL, new_addr);
		114	return ret;
		115	}
		116
62	/* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50	117	/* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50
63	* bytes on x86. Pick a number to fit into PAGE_SIZE / 2	118	* bytes on x86. Pick a number to fit into PAGE_SIZE / 2
64	*/	119	*/
@@ -77,8 +132,7 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr) · 12 unmodified lines
77	int err;	132	int err;
78		133
79	if (fentry_cnt + fexit_cnt == 0) {	134	if (fentry_cnt + fexit_cnt == 0) {
80	err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_CALL,	135	err = unregister_fentry(tr, old_image);
81	old_image, NULL);
82	tr->selector = 0;	136	tr->selector = 0;
83	goto out;	137	goto out;
84	}	138	}
@@ -105,12 +159,10 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr) · 20 unmodified lines
105		159
106	if (tr->selector)	160	if (tr->selector)
107	/* progs already running at this address */	161	/* progs already running at this address */
108	err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_CALL,	162	err = modify_fentry(tr, old_image, new_image);
109	old_image, new_image);
110	else	163	else
111	/* first time registering */	164	/* first time registering */
112	err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_CALL, NULL,	165	err = register_fentry(tr, new_image);
113	new_image);
114	if (err)	166	if (err)
115	goto out;	167	goto out;
116	tr->selector++;	168	tr->selector++;

bpf: Make BPF trampoline use register_ftrace_direct() API

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code