bpf: Add per-program recursion prevention mechanism

Since both sleepable and non-sleepable programs execute under migrate_disable add recursion prevention mechanism to both types of programs when they're executed via bpf trampoline.

Alexei Starovoitov committed 5 years ago

Commit ca06f551 parent f2dd3b37 files · +50 −11Browse files

Unified · Split

7 files

arch/x86/net/bpf_jit_comp.c +15 −0
include/linux/bpf.h +3 −3
include/linux/filter.h +1 −0
kernel/bpf/core.c +8 −0
kernel/bpf/trampoline.c +19 −4
tools/testing/selftests/bpf/prog_tests/fexit_stress.c +2 −2
tools/testing/selftests/bpf/prog_tests/trampoline_count.c +2 −2

arch/x86/net/bpf_jit_comp.c+15−0

@@ -1740,8 +1740,11 @@ static int invoke_bpf_prog(const struct btf_func_model m, u8 *pprog, · 1739 unmodified lines
1740	1740	struct bpf_prog *p, int stack_size, bool mod_ret)
1741	1741	{
1742	1742	u8 prog = pprog;
	1743	+ u8 *jmp_insn;
1743	1744	int cnt = 0;
1744	1745
	1746	+ /* arg1: mov rdi, progs[i] */
	1747	+ emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p);
1745	1748	if (emit_call(&prog,
1746	1749	p->aux->sleepable ? __bpf_prog_enter_sleepable :
1747	1750	__bpf_prog_enter, prog))
@@ -1749,6 +1752,14 @@ static int invoke_bpf_prog(const struct btf_func_model m, u8 *pprog, · 1 unmodified line
1749	1752	/* remember prog start time returned by __bpf_prog_enter */
1750	1753	emit_mov_reg(&prog, true, BPF_REG_6, BPF_REG_0);
1751	1754
	1755	+ /* if (__bpf_prog_enter*(prog) == 0)
	1756	+ * goto skip_exec_of_prog;
	1757	+ */
	1758	+ EMIT3(0x48, 0x85, 0xC0); /* test rax,rax */
	1759	+ /* emit 2 nops that will be replaced with JE insn */
	1760	+ jmp_insn = prog;
	1761	+ emit_nops(&prog, 2);
	1762	+
1752	1763	/* arg1: lea rdi, [rbp - stack_size] */
1753	1764	EMIT4(0x48, 0x8D, 0x7D, -stack_size);
1754	1765	/* arg2: progs[i]->insnsi for interpreter */
@@ -1767,6 +1778,10 @@ static int invoke_bpf_prog(const struct btf_func_model m, u8 *pprog, · 12 unmodified lines
1767	1778	if (mod_ret)
1768	1779	emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);
1769	1780
	1781	+ /* replace 2 nops with JE insn, since jmp target is known */
	1782	+ jmp_insn[0] = X86_JE;
	1783	+ jmp_insn[1] = prog - jmp_insn - 2;
	1784	+
1770	1785	/* arg1: mov rdi, progs[i] */
1771	1786	emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p);
1772	1787	/* arg2: mov rsi, rbx <- start time in nsec */

include/linux/bpf.h+3−3

@@ -529,7 +529,7 @@ struct btf_func_model { · 528 unmodified lines
529	529	/* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50
530	530	* bytes on x86. Pick a number to fit into BPF_IMAGE_SIZE / 2
531	531	*/
532		−#define BPF_MAX_TRAMP_PROGS 40
	532	+#define BPF_MAX_TRAMP_PROGS 38
533	533
534	534	struct bpf_tramp_progs {
535	535	struct bpf_prog *progs[BPF_MAX_TRAMP_PROGS];
@@ -561,9 +561,9 @@ int arch_prepare_bpf_trampoline(void image, void image_end, · 25 unmodified lines
561	561	struct bpf_tramp_progs *tprogs,
562	562	void *orig_call);
563	563	/* these two functions are called from generated trampoline */
564		−u64 notrace __bpf_prog_enter(void);
	564	+u64 notrace __bpf_prog_enter(struct bpf_prog *prog);
565	565	void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start);
566		−u64 notrace __bpf_prog_enter_sleepable(void);
	566	+u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog);
567	567	void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start);
568	568
569	569	struct bpf_ksym {

include/linux/filter.h+1−0

@@ -565,6 +565,7 @@ struct bpf_prog { · 564 unmodified lines
565	565	u32 jited_len; /* Size of jited insns in bytes */
566	566	u8 tag[BPF_TAG_SIZE];
567	567	struct bpf_prog_stats __percpu *stats;
	568	+ int __percpu *active;
568	569	unsigned int (bpf_func)(const void ctx,
569	570	const struct bpf_insn *insn);
570	571	struct bpf_prog_aux aux; / Auxiliary fields */

kernel/bpf/core.c+8−0

@@ -91,6 +91,12 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag · 90 unmodified lines
91	91	vfree(fp);
92	92	return NULL;
93	93	}
	94	+ fp->active = alloc_percpu_gfp(int, GFP_KERNEL_ACCOUNT \| gfp_extra_flags);
	95	+ if (!fp->active) {
	96	+ vfree(fp);
	97	+ kfree(aux);
	98	+ return NULL;
	99	+ }
94	100
95	101	fp->pages = size / PAGE_SIZE;
96	102	fp->aux = aux;
@@ -116,6 +122,7 @@ struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags) · 19 unmodified lines
116	122
117	123	prog->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags);
118	124	if (!prog->stats) {
	125	+ free_percpu(prog->active);
119	126	kfree(prog->aux);
120	127	vfree(prog);
121	128	return NULL;
@@ -253,6 +260,7 @@ void __bpf_prog_free(struct bpf_prog *fp) · 131 unmodified lines
253	260	kfree(fp->aux);
254	261	}
255	262	free_percpu(fp->stats);
	263	+ free_percpu(fp->active);
256	264	vfree(fp);
257	265	}
258	266

kernel/bpf/trampoline.c+19−4

@@ -381,13 +381,16 @@ out: · 380 unmodified lines
381	381	mutex_unlock(&trampoline_mutex);
382	382	}
383	383
384		−#define NO_START_TIME 0
	384	+#define NO_START_TIME 1
385	385	static u64 notrace bpf_prog_start_time(void)
386	386	{
387	387	u64 start = NO_START_TIME;
388	388
389		− if (static_branch_unlikely(&bpf_stats_enabled_key))
	389	+ if (static_branch_unlikely(&bpf_stats_enabled_key)) {
390	390	start = sched_clock();
	391	+ if (unlikely(!start))
	392	+ start = NO_START_TIME;
	393	+ }
391	394	return start;
392	395	}
393	396
@@ -397,12 +400,20 @@ static u64 notrace bpf_prog_start_time(void) · 3 unmodified lines
397	400	* call __bpf_prog_enter
398	401	* call prog->bpf_func
399	402	* call __bpf_prog_exit
	403	+ *
	404	+ * __bpf_prog_enter returns:
	405	+ * 0 - skip execution of the bpf prog
	406	+ * 1 - execute bpf prog
	407	+ * [2..MAX_U64] - excute bpf prog and record execution time.
	408	+ * This is start time.
400	409	*/
401		−u64 notrace __bpf_prog_enter(void)
	410	+u64 notrace __bpf_prog_enter(struct bpf_prog *prog)
402	411	__acquires(RCU)
403	412	{
404	413	rcu_read_lock();
405	414	migrate_disable();
	415	+ if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1))
	416	+ return 0;
406	417	return bpf_prog_start_time();
407	418	}
408	419
@@ -430,21 +441,25 @@ void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start) · 21 unmodified lines
430	441	__releases(RCU)
431	442	{
432	443	update_prog_stats(prog, start);
	444	+ __this_cpu_dec(*(prog->active));
433	445	migrate_enable();
434	446	rcu_read_unlock();
435	447	}
436	448
437		−u64 notrace __bpf_prog_enter_sleepable(void)
	449	+u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog)
438	450	{
439	451	rcu_read_lock_trace();
440	452	migrate_disable();
441	453	might_fault();
	454	+ if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1))
	455	+ return 0;
442	456	return bpf_prog_start_time();
443	457	}
444	458
445	459	void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start)
446	460	{
447	461	update_prog_stats(prog, start);
	462	+ __this_cpu_dec(*(prog->active));
448	463	migrate_enable();
449	464	rcu_read_unlock_trace();
450	465	}

tools/testing/selftests/bpf/prog_tests/fexit_stress.c+2−2

@@ -2,8 +2,8 @@ · 1 unmodified line
2	2	/* Copyright (c) 2019 Facebook */
3	3	#include <test_progs.h>
4	4
5		−/* x86-64 fits 55 JITed and 43 interpreted progs into half page */
6		−#define CNT 40
	5	+/* that's kernel internal BPF_MAX_TRAMP_PROGS define */
	6	+#define CNT 38
7	7
8	8	void test_fexit_stress(void)
9	9	{

tools/testing/selftests/bpf/prog_tests/trampoline_count.c+2−2

@@ -4,7 +4,7 @@ · 3 unmodified lines
4	4	#include <sys/prctl.h>
5	5	#include <test_progs.h>
6	6
7		−#define MAX_TRAMP_PROGS 40
	7	+#define MAX_TRAMP_PROGS 38
8	8
9	9	struct inst {
10	10	struct bpf_object *obj;
@@ -52,7 +52,7 @@ void test_trampoline_count(void) · 41 unmodified lines
52	52	struct bpf_link *link;
53	53	char comm[16] = {};
54	54
55		− /* attach 'allowed' 40 trampoline programs */
	55	+ /* attach 'allowed' trampoline programs */
56	56	for (i = 0; i < MAX_TRAMP_PROGS; i++) {
57	57	obj = bpf_object__open_file(object, NULL);
58	58	if (CHECK(IS_ERR(obj), "obj_open_file", "err %ld\n", PTR_ERR(obj))) {

bpf: Add per-program recursion prevention mechanism

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code