History

bpf: Add per-program recursion prevention mechanism

Since both sleepable and non-sleepable programs execute under migrate_disable add recursion prevention mechanism to both types of programs when they're executed via bpf trampoline.

Alexei Starovoitov committed 5 years ago

Commit ca06f551 parent f2dd3b37 files · +50 −11Browse files

Unified · Split

7 files
  1. arch/x86/net/bpf_jit_comp.c +15 −0
  2. include/linux/bpf.h +3 −3
  3. include/linux/filter.h +1 −0
  4. kernel/bpf/core.c +8 −0
  5. kernel/bpf/trampoline.c +19 −4
  6. tools/testing/selftests/bpf/prog_tests/fexit_stress.c +2 −2
  7. tools/testing/selftests/bpf/prog_tests/trampoline_count.c +2 −2
arch/x86/net/bpf_jit_comp.c+15−0
@@ -1740,8 +1740,11 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, · 1739 unmodified lines
1740 struct bpf_prog *p, int stack_size, bool mod_ret)1740 struct bpf_prog *p, int stack_size, bool mod_ret)
1741{1741{
1742 u8 *prog = *pprog;1742 u8 *prog = *pprog;
1743 u8 *jmp_insn;
1743 int cnt = 0;1744 int cnt = 0;
1744 1745
1746 /* arg1: mov rdi, progs[i] */
1747 emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p);
1745 if (emit_call(&prog,1748 if (emit_call(&prog,
1746 p->aux->sleepable ? __bpf_prog_enter_sleepable :1749 p->aux->sleepable ? __bpf_prog_enter_sleepable :
1747 __bpf_prog_enter, prog))1750 __bpf_prog_enter, prog))
@@ -1749,6 +1752,14 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, · 1 unmodified line
1749 /* remember prog start time returned by __bpf_prog_enter */1752 /* remember prog start time returned by __bpf_prog_enter */
1750 emit_mov_reg(&prog, true, BPF_REG_6, BPF_REG_0);1753 emit_mov_reg(&prog, true, BPF_REG_6, BPF_REG_0);
1751 1754
1755 /* if (__bpf_prog_enter*(prog) == 0)
1756 * goto skip_exec_of_prog;
1757 */
1758 EMIT3(0x48, 0x85, 0xC0); /* test rax,rax */
1759 /* emit 2 nops that will be replaced with JE insn */
1760 jmp_insn = prog;
1761 emit_nops(&prog, 2);
1762
1752 /* arg1: lea rdi, [rbp - stack_size] */1763 /* arg1: lea rdi, [rbp - stack_size] */
1753 EMIT4(0x48, 0x8D, 0x7D, -stack_size);1764 EMIT4(0x48, 0x8D, 0x7D, -stack_size);
1754 /* arg2: progs[i]->insnsi for interpreter */1765 /* arg2: progs[i]->insnsi for interpreter */
@@ -1767,6 +1778,10 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, · 12 unmodified lines
1767 if (mod_ret)1778 if (mod_ret)
1768 emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);1779 emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);
1769 1780
1781 /* replace 2 nops with JE insn, since jmp target is known */
1782 jmp_insn[0] = X86_JE;
1783 jmp_insn[1] = prog - jmp_insn - 2;
1784
1770 /* arg1: mov rdi, progs[i] */1785 /* arg1: mov rdi, progs[i] */
1771 emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p);1786 emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p);
1772 /* arg2: mov rsi, rbx <- start time in nsec */1787 /* arg2: mov rsi, rbx <- start time in nsec */
include/linux/bpf.h+3−3
@@ -529,7 +529,7 @@ struct btf_func_model { · 528 unmodified lines
529/* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50529/* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50
530 * bytes on x86. Pick a number to fit into BPF_IMAGE_SIZE / 2530 * bytes on x86. Pick a number to fit into BPF_IMAGE_SIZE / 2
531 */531 */
532#define BPF_MAX_TRAMP_PROGS 40532#define BPF_MAX_TRAMP_PROGS 38
533 533
534struct bpf_tramp_progs {534struct bpf_tramp_progs {
535 struct bpf_prog *progs[BPF_MAX_TRAMP_PROGS];535 struct bpf_prog *progs[BPF_MAX_TRAMP_PROGS];
@@ -561,9 +561,9 @@ int arch_prepare_bpf_trampoline(void *image, void *image_end, · 25 unmodified lines
561 struct bpf_tramp_progs *tprogs,561 struct bpf_tramp_progs *tprogs,
562 void *orig_call);562 void *orig_call);
563/* these two functions are called from generated trampoline */563/* these two functions are called from generated trampoline */
564u64 notrace __bpf_prog_enter(void);564u64 notrace __bpf_prog_enter(struct bpf_prog *prog);
565void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start);565void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start);
566u64 notrace __bpf_prog_enter_sleepable(void);566u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog);
567void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start);567void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start);
568 568
569struct bpf_ksym {569struct bpf_ksym {
include/linux/filter.h+1−0
@@ -565,6 +565,7 @@ struct bpf_prog { · 564 unmodified lines
565 u32 jited_len; /* Size of jited insns in bytes */565 u32 jited_len; /* Size of jited insns in bytes */
566 u8 tag[BPF_TAG_SIZE];566 u8 tag[BPF_TAG_SIZE];
567 struct bpf_prog_stats __percpu *stats;567 struct bpf_prog_stats __percpu *stats;
568 int __percpu *active;
568 unsigned int (*bpf_func)(const void *ctx,569 unsigned int (*bpf_func)(const void *ctx,
569 const struct bpf_insn *insn);570 const struct bpf_insn *insn);
570 struct bpf_prog_aux *aux; /* Auxiliary fields */571 struct bpf_prog_aux *aux; /* Auxiliary fields */
kernel/bpf/core.c+8−0
@@ -91,6 +91,12 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag · 90 unmodified lines
91 vfree(fp);91 vfree(fp);
92 return NULL;92 return NULL;
93 }93 }
94 fp->active = alloc_percpu_gfp(int, GFP_KERNEL_ACCOUNT | gfp_extra_flags);
95 if (!fp->active) {
96 vfree(fp);
97 kfree(aux);
98 return NULL;
99 }
94 100
95 fp->pages = size / PAGE_SIZE;101 fp->pages = size / PAGE_SIZE;
96 fp->aux = aux;102 fp->aux = aux;
@@ -116,6 +122,7 @@ struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags) · 19 unmodified lines
116 122
117 prog->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags);123 prog->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags);
118 if (!prog->stats) {124 if (!prog->stats) {
125 free_percpu(prog->active);
119 kfree(prog->aux);126 kfree(prog->aux);
120 vfree(prog);127 vfree(prog);
121 return NULL;128 return NULL;
@@ -253,6 +260,7 @@ void __bpf_prog_free(struct bpf_prog *fp) · 131 unmodified lines
253 kfree(fp->aux);260 kfree(fp->aux);
254 }261 }
255 free_percpu(fp->stats);262 free_percpu(fp->stats);
263 free_percpu(fp->active);
256 vfree(fp);264 vfree(fp);
257}265}
258 266
kernel/bpf/trampoline.c+19−4
@@ -381,13 +381,16 @@ out: · 380 unmodified lines
381 mutex_unlock(&trampoline_mutex);381 mutex_unlock(&trampoline_mutex);
382}382}
383 383
384#define NO_START_TIME 0384#define NO_START_TIME 1
385static u64 notrace bpf_prog_start_time(void)385static u64 notrace bpf_prog_start_time(void)
386{386{
387 u64 start = NO_START_TIME;387 u64 start = NO_START_TIME;
388 388
389 if (static_branch_unlikely(&bpf_stats_enabled_key))389 if (static_branch_unlikely(&bpf_stats_enabled_key)) {
390 start = sched_clock();390 start = sched_clock();
391 if (unlikely(!start))
392 start = NO_START_TIME;
393 }
391 return start;394 return start;
392}395}
393 396
@@ -397,12 +400,20 @@ static u64 notrace bpf_prog_start_time(void) · 3 unmodified lines
397 * call __bpf_prog_enter400 * call __bpf_prog_enter
398 * call prog->bpf_func401 * call prog->bpf_func
399 * call __bpf_prog_exit402 * call __bpf_prog_exit
403 *
404 * __bpf_prog_enter returns:
405 * 0 - skip execution of the bpf prog
406 * 1 - execute bpf prog
407 * [2..MAX_U64] - excute bpf prog and record execution time.
408 * This is start time.
400 */409 */
401u64 notrace __bpf_prog_enter(void)410u64 notrace __bpf_prog_enter(struct bpf_prog *prog)
402 __acquires(RCU)411 __acquires(RCU)
403{412{
404 rcu_read_lock();413 rcu_read_lock();
405 migrate_disable();414 migrate_disable();
415 if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1))
416 return 0;
406 return bpf_prog_start_time();417 return bpf_prog_start_time();
407}418}
408 419
@@ -430,21 +441,25 @@ void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start) · 21 unmodified lines
430 __releases(RCU)441 __releases(RCU)
431{442{
432 update_prog_stats(prog, start);443 update_prog_stats(prog, start);
444 __this_cpu_dec(*(prog->active));
433 migrate_enable();445 migrate_enable();
434 rcu_read_unlock();446 rcu_read_unlock();
435}447}
436 448
437u64 notrace __bpf_prog_enter_sleepable(void)449u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog)
438{450{
439 rcu_read_lock_trace();451 rcu_read_lock_trace();
440 migrate_disable();452 migrate_disable();
441 might_fault();453 might_fault();
454 if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1))
455 return 0;
442 return bpf_prog_start_time();456 return bpf_prog_start_time();
443}457}
444 458
445void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start)459void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start)
446{460{
447 update_prog_stats(prog, start);461 update_prog_stats(prog, start);
462 __this_cpu_dec(*(prog->active));
448 migrate_enable();463 migrate_enable();
449 rcu_read_unlock_trace();464 rcu_read_unlock_trace();
450}465}
tools/testing/selftests/bpf/prog_tests/fexit_stress.c+2−2
@@ -2,8 +2,8 @@ · 1 unmodified line
2/* Copyright (c) 2019 Facebook */2/* Copyright (c) 2019 Facebook */
3#include <test_progs.h>3#include <test_progs.h>
4 4
5/* x86-64 fits 55 JITed and 43 interpreted progs into half page */5/* that's kernel internal BPF_MAX_TRAMP_PROGS define */
6#define CNT 406#define CNT 38
7 7
8void test_fexit_stress(void)8void test_fexit_stress(void)
9{9{
tools/testing/selftests/bpf/prog_tests/trampoline_count.c+2−2
@@ -4,7 +4,7 @@ · 3 unmodified lines
4#include <sys/prctl.h>4#include <sys/prctl.h>
5#include <test_progs.h>5#include <test_progs.h>
6 6
7#define MAX_TRAMP_PROGS 407#define MAX_TRAMP_PROGS 38
8 8
9struct inst {9struct inst {
10 struct bpf_object *obj;10 struct bpf_object *obj;
@@ -52,7 +52,7 @@ void test_trampoline_count(void) · 41 unmodified lines
52 struct bpf_link *link;52 struct bpf_link *link;
53 char comm[16] = {};53 char comm[16] = {};
54 54
55 /* attach 'allowed' 40 trampoline programs */55 /* attach 'allowed' trampoline programs */
56 for (i = 0; i < MAX_TRAMP_PROGS; i++) {56 for (i = 0; i < MAX_TRAMP_PROGS; i++) {
57 obj = bpf_object__open_file(object, NULL);57 obj = bpf_object__open_file(object, NULL);
58 if (CHECK(IS_ERR(obj), "obj_open_file", "err %ld\n", PTR_ERR(obj))) {58 if (CHECK(IS_ERR(obj), "obj_open_file", "err %ld\n", PTR_ERR(obj))) {

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t