History

bpf: introduce function calls (function boundaries)

Allow arbitrary function calls from bpf function to another bpf function. Since the beginning of bpf all bpf programs were represented as a single function and program authors were forced to use always_inline for all functions in their C code. That was causing llvm to unnecessary inflate the code size and forcing developers to move code to header files with little code reuse. With a bit of additional complexity teach verifier to recognize arbitrary function calls from one bpf function to another as long as all of functions are presented to the verifier as a single bpf program. New program layout: r6 = r1 // some code .. r1 = .. // arg1 r2 = .. // arg2 call pc+1 // function call pc-relative exit .. = r1 // access arg1 .. = r2 // access arg2 .. call pc+20 // second level of function call ... It allows for better optimized code and finally allows to introduce the core bpf libraries that can be reused in different projects, since programs are no longer limited by single elf file. With function calls bpf can be compiled into multiple .o files. This patch is the first step. It detects programs that contain multiple functions and checks that calls between them are valid. It splits the sequence of bpf instructions (one program) into a set of bpf functions that call each other. Calls to only known functions are allowed. In the future the verifier may allow calls to unresolved functions and will do dynamic linking. This logic supports statically linked bpf functions only. Such function boundary detection could have been done as part of control flow graph building in check_cfg(), but it's cleaner to separate function boundary detection vs control flow checks within a subprogram (function) into logically indepedent steps. Follow up patches may split check_cfg() further, but not check_subprogs(). Only allow bpf-to-bpf calls for root only and for non-hw-offloaded programs. These restrictions can be relaxed in the future.

Alexei Starovoitov committed 8 years ago

Commit cc8b0b91 parent 0bce7c94 files · +155 −5Browse files

Unified · Split

4 files
  1. include/linux/bpf_verifier.h +4 −1
  2. include/uapi/linux/bpf.h +6 −0
  3. kernel/bpf/disasm.c +6 −2
  4. kernel/bpf/verifier.c +139 −2
include/linux/bpf_verifier.h+4−1
@@ -141,6 +141,8 @@ struct bpf_ext_analyzer_ops { · 140 unmodified lines
141 int insn_idx, int prev_insn_idx);141 int insn_idx, int prev_insn_idx);
142};142};
143 143
144#define BPF_MAX_SUBPROGS 256
145
144/* single container for all structs146/* single container for all structs
145 * one verifier_env per bpf_check() call147 * one verifier_env per bpf_check() call
146 */148 */
@@ -159,8 +161,9 @@ struct bpf_verifier_env { · 12 unmodified lines
159 bool allow_ptr_leaks;161 bool allow_ptr_leaks;
160 bool seen_direct_write;162 bool seen_direct_write;
161 struct bpf_insn_aux_data *insn_aux_data; /* array of per-insn state */163 struct bpf_insn_aux_data *insn_aux_data; /* array of per-insn state */
162
163 struct bpf_verifer_log log;164 struct bpf_verifer_log log;
165 u32 subprog_starts[BPF_MAX_SUBPROGS];
166 u32 subprog_cnt;
164};167};
165 168
166static inline struct bpf_reg_state *cur_regs(struct bpf_verifier_env *env)169static inline struct bpf_reg_state *cur_regs(struct bpf_verifier_env *env)
include/uapi/linux/bpf.h+6−0
@@ -197,8 +197,14 @@ enum bpf_attach_type { · 196 unmodified lines
197 */197 */
198#define BPF_F_STRICT_ALIGNMENT (1U << 0)198#define BPF_F_STRICT_ALIGNMENT (1U << 0)
199 199
200/* when bpf_ldimm64->src_reg == BPF_PSEUDO_MAP_FD, bpf_ldimm64->imm == fd */
200#define BPF_PSEUDO_MAP_FD 1201#define BPF_PSEUDO_MAP_FD 1
201 202
203/* when bpf_call->src_reg == BPF_PSEUDO_CALL, bpf_call->imm == pc-relative
204 * offset to another bpf function
205 */
206#define BPF_PSEUDO_CALL 1
207
202/* flags for BPF_MAP_UPDATE_ELEM command */208/* flags for BPF_MAP_UPDATE_ELEM command */
203#define BPF_ANY 0 /* create new element or update existing */209#define BPF_ANY 0 /* create new element or update existing */
204#define BPF_NOEXIST 1 /* create new element if it didn't exist */210#define BPF_NOEXIST 1 /* create new element if it didn't exist */
kernel/bpf/disasm.c+6−2
@@ -189,8 +189,12 @@ void print_bpf_insn(bpf_insn_print_cb verbose, struct bpf_verifier_env *env, · 188 unmodified lines
189 u8 opcode = BPF_OP(insn->code);189 u8 opcode = BPF_OP(insn->code);
190 190
191 if (opcode == BPF_CALL) {191 if (opcode == BPF_CALL) {
192 verbose(env, "(%02x) call %s#%d\n", insn->code,192 if (insn->src_reg == BPF_PSEUDO_CALL)
193 func_id_name(insn->imm), insn->imm);193 verbose(env, "(%02x) call pc%+d\n", insn->code,
194 insn->imm);
195 else
196 verbose(env, "(%02x) call %s#%d\n", insn->code,
197 func_id_name(insn->imm), insn->imm);
194 } else if (insn->code == (BPF_JMP | BPF_JA)) {198 } else if (insn->code == (BPF_JMP | BPF_JA)) {
195 verbose(env, "(%02x) goto pc%+d\n",199 verbose(env, "(%02x) goto pc%+d\n",
196 insn->code, insn->off);200 insn->code, insn->off);
kernel/bpf/verifier.c+139−2
@@ -20,6 +20,8 @@ · 19 unmodified lines
20#include <linux/file.h>20#include <linux/file.h>
21#include <linux/vmalloc.h>21#include <linux/vmalloc.h>
22#include <linux/stringify.h>22#include <linux/stringify.h>
23#include <linux/bsearch.h>
24#include <linux/sort.h>
23 25
24#include "disasm.h"26#include "disasm.h"
25 27
@@ -636,6 +638,113 @@ enum reg_arg_type { · 610 unmodified lines
636 DST_OP_NO_MARK /* same as above, check only, don't mark */638 DST_OP_NO_MARK /* same as above, check only, don't mark */
637};639};
638 640
641static int cmp_subprogs(const void *a, const void *b)
642{
643 return *(int *)a - *(int *)b;
644}
645
646static int find_subprog(struct bpf_verifier_env *env, int off)
647{
648 u32 *p;
649
650 p = bsearch(&off, env->subprog_starts, env->subprog_cnt,
651 sizeof(env->subprog_starts[0]), cmp_subprogs);
652 if (!p)
653 return -ENOENT;
654 return p - env->subprog_starts;
655
656}
657
658static int add_subprog(struct bpf_verifier_env *env, int off)
659{
660 int insn_cnt = env->prog->len;
661 int ret;
662
663 if (off >= insn_cnt || off < 0) {
664 verbose(env, "call to invalid destination\n");
665 return -EINVAL;
666 }
667 ret = find_subprog(env, off);
668 if (ret >= 0)
669 return 0;
670 if (env->subprog_cnt >= BPF_MAX_SUBPROGS) {
671 verbose(env, "too many subprograms\n");
672 return -E2BIG;
673 }
674 env->subprog_starts[env->subprog_cnt++] = off;
675 sort(env->subprog_starts, env->subprog_cnt,
676 sizeof(env->subprog_starts[0]), cmp_subprogs, NULL);
677 return 0;
678}
679
680static int check_subprogs(struct bpf_verifier_env *env)
681{
682 int i, ret, subprog_start, subprog_end, off, cur_subprog = 0;
683 struct bpf_insn *insn = env->prog->insnsi;
684 int insn_cnt = env->prog->len;
685
686 /* determine subprog starts. The end is one before the next starts */
687 for (i = 0; i < insn_cnt; i++) {
688 if (insn[i].code != (BPF_JMP | BPF_CALL))
689 continue;
690 if (insn[i].src_reg != BPF_PSEUDO_CALL)
691 continue;
692 if (!env->allow_ptr_leaks) {
693 verbose(env, "function calls to other bpf functions are allowed for root only\n");
694 return -EPERM;
695 }
696 if (bpf_prog_is_dev_bound(env->prog->aux)) {
697 verbose(env, "funcation calls in offloaded programs are not supported yet\n");
698 return -EINVAL;
699 }
700 ret = add_subprog(env, i + insn[i].imm + 1);
701 if (ret < 0)
702 return ret;
703 }
704
705 if (env->log.level > 1)
706 for (i = 0; i < env->subprog_cnt; i++)
707 verbose(env, "func#%d @%d\n", i, env->subprog_starts[i]);
708
709 /* now check that all jumps are within the same subprog */
710 subprog_start = 0;
711 if (env->subprog_cnt == cur_subprog)
712 subprog_end = insn_cnt;
713 else
714 subprog_end = env->subprog_starts[cur_subprog++];
715 for (i = 0; i < insn_cnt; i++) {
716 u8 code = insn[i].code;
717
718 if (BPF_CLASS(code) != BPF_JMP)
719 goto next;
720 if (BPF_OP(code) == BPF_EXIT || BPF_OP(code) == BPF_CALL)
721 goto next;
722 off = i + insn[i].off + 1;
723 if (off < subprog_start || off >= subprog_end) {
724 verbose(env, "jump out of range from insn %d to %d\n", i, off);
725 return -EINVAL;
726 }
727next:
728 if (i == subprog_end - 1) {
729 /* to avoid fall-through from one subprog into another
730 * the last insn of the subprog should be either exit
731 * or unconditional jump back
732 */
733 if (code != (BPF_JMP | BPF_EXIT) &&
734 code != (BPF_JMP | BPF_JA)) {
735 verbose(env, "last insn is not an exit or jmp\n");
736 return -EINVAL;
737 }
738 subprog_start = subprog_end;
739 if (env->subprog_cnt == cur_subprog)
740 subprog_end = insn_cnt;
741 else
742 subprog_end = env->subprog_starts[cur_subprog++];
743 }
744 }
745 return 0;
746}
747
639static void mark_reg_read(const struct bpf_verifier_state *state, u32 regno)748static void mark_reg_read(const struct bpf_verifier_state *state, u32 regno)
640{749{
641 struct bpf_verifier_state *parent = state->parent;750 struct bpf_verifier_state *parent = state->parent;
@@ -3284,6 +3393,10 @@ static int check_cfg(struct bpf_verifier_env *env) · 2642 unmodified lines
3284 int ret = 0;3393 int ret = 0;
3285 int i, t;3394 int i, t;
3286 3395
3396 ret = check_subprogs(env);
3397 if (ret < 0)
3398 return ret;
3399
3287 insn_state = kcalloc(insn_cnt, sizeof(int), GFP_KERNEL);3400 insn_state = kcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
3288 if (!insn_state)3401 if (!insn_state)
3289 return -ENOMEM;3402 return -ENOMEM;
@@ -3316,6 +3429,14 @@ peek_stack: · 26 unmodified lines
3316 goto err_free;3429 goto err_free;
3317 if (t + 1 < insn_cnt)3430 if (t + 1 < insn_cnt)
3318 env->explored_states[t + 1] = STATE_LIST_MARK;3431 env->explored_states[t + 1] = STATE_LIST_MARK;
3432 if (insns[t].src_reg == BPF_PSEUDO_CALL) {
3433 env->explored_states[t] = STATE_LIST_MARK;
3434 ret = push_insn(t, t + insns[t].imm + 1, BRANCH, env);
3435 if (ret == 1)
3436 goto peek_stack;
3437 else if (ret < 0)
3438 goto err_free;
3439 }
3319 } else if (opcode == BPF_JA) {3440 } else if (opcode == BPF_JA) {
3320 if (BPF_SRC(insns[t].code) != BPF_K) {3441 if (BPF_SRC(insns[t].code) != BPF_K) {
3321 ret = -EINVAL;3442 ret = -EINVAL;
@@ -4245,6 +4366,19 @@ static int adjust_insn_aux_data(struct bpf_verifier_env *env, u32 prog_len, · 923 unmodified lines
4245 return 0;4366 return 0;
4246}4367}
4247 4368
4369static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len)
4370{
4371 int i;
4372
4373 if (len == 1)
4374 return;
4375 for (i = 0; i < env->subprog_cnt; i++) {
4376 if (env->subprog_starts[i] < off)
4377 continue;
4378 env->subprog_starts[i] += len - 1;
4379 }
4380}
4381
4248static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,4382static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
4249 const struct bpf_insn *patch, u32 len)4383 const struct bpf_insn *patch, u32 len)
4250{4384{
@@ -4255,6 +4389,7 @@ static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 of · 4 unmodified lines
4255 return NULL;4389 return NULL;
4256 if (adjust_insn_aux_data(env, new_prog->len, off, len))4390 if (adjust_insn_aux_data(env, new_prog->len, off, len))
4257 return NULL;4391 return NULL;
4392 adjust_subprog_starts(env, off, len);
4258 return new_prog;4393 return new_prog;
4259}4394}
4260 4395
@@ -4408,6 +4543,8 @@ static int fixup_bpf_calls(struct bpf_verifier_env *env) · 147 unmodified lines
4408 for (i = 0; i < insn_cnt; i++, insn++) {4543 for (i = 0; i < insn_cnt; i++, insn++) {
4409 if (insn->code != (BPF_JMP | BPF_CALL))4544 if (insn->code != (BPF_JMP | BPF_CALL))
4410 continue;4545 continue;
4546 if (insn->src_reg == BPF_PSEUDO_CALL)
4547 continue;
4411 4548
4412 if (insn->imm == BPF_FUNC_get_route_realm)4549 if (insn->imm == BPF_FUNC_get_route_realm)
4413 prog->dst_needed = 1;4550 prog->dst_needed = 1;
@@ -4589,12 +4726,12 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr) · 175 unmodified lines
4589 if (!env->explored_states)4726 if (!env->explored_states)
4590 goto skip_full_check;4727 goto skip_full_check;
4591 4728
4729 env->allow_ptr_leaks = capable(CAP_SYS_ADMIN);
4730
4592 ret = check_cfg(env);4731 ret = check_cfg(env);
4593 if (ret < 0)4732 if (ret < 0)
4594 goto skip_full_check;4733 goto skip_full_check;
4595 4734
4596 env->allow_ptr_leaks = capable(CAP_SYS_ADMIN);
4597
4598 ret = do_check(env);4735 ret = do_check(env);
4599 if (env->cur_state) {4736 if (env->cur_state) {
4600 free_verifier_state(env->cur_state, true);4737 free_verifier_state(env->cur_state, true);