History

bpf: introduce function calls (function boundaries)

Allow arbitrary function calls from bpf function to another bpf function. Since the beginning of bpf all bpf programs were represented as a single function and program authors were forced to use always_inline for all functions in their C code. That was causing llvm to unnecessary inflate the code size and forcing developers to move code to header files with little code reuse. With a bit of additional complexity teach verifier to recognize arbitrary function calls from one bpf function to another as long as all of functions are presented to the verifier as a single bpf program. New program layout: r6 = r1 // some code .. r1 = .. // arg1 r2 = .. // arg2 call pc+1 // function call pc-relative exit .. = r1 // access arg1 .. = r2 // access arg2 .. call pc+20 // second level of function call ... It allows for better optimized code and finally allows to introduce the core bpf libraries that can be reused in different projects, since programs are no longer limited by single elf file. With function calls bpf can be compiled into multiple .o files. This patch is the first step. It detects programs that contain multiple functions and checks that calls between them are valid. It splits the sequence of bpf instructions (one program) into a set of bpf functions that call each other. Calls to only known functions are allowed. In the future the verifier may allow calls to unresolved functions and will do dynamic linking. This logic supports statically linked bpf functions only. Such function boundary detection could have been done as part of control flow graph building in check_cfg(), but it's cleaner to separate function boundary detection vs control flow checks within a subprogram (function) into logically indepedent steps. Follow up patches may split check_cfg() further, but not check_subprogs(). Only allow bpf-to-bpf calls for root only and for non-hw-offloaded programs. These restrictions can be relaxed in the future.

Alexei Starovoitov committed 8 years ago

Commit cc8b0b91 parent 0bce7c94 files · +155 −5Browse files

Unified · Split

4 files
  1. include/linux/bpf_verifier.h +4 −1
  2. include/uapi/linux/bpf.h +6 −0
  3. kernel/bpf/disasm.c +6 −2
  4. kernel/bpf/verifier.c +139 −2
include/linux/bpf_verifier.h+4−1
@@ -141,6 +141,8 @@ struct bpf_ext_analyzer_ops { · 140 unmodified lines
141141 int insn_idx, int prev_insn_idx);
142142 };
143143
144+#define BPF_MAX_SUBPROGS 256
145+
144146 /* single container for all structs
145147 * one verifier_env per bpf_check() call
146148 */
@@ -159,8 +161,9 @@ struct bpf_verifier_env { · 12 unmodified lines
159161 bool allow_ptr_leaks;
160162 bool seen_direct_write;
161163 struct bpf_insn_aux_data *insn_aux_data; /* array of per-insn state */
162−
163164 struct bpf_verifer_log log;
165+ u32 subprog_starts[BPF_MAX_SUBPROGS];
166+ u32 subprog_cnt;
164167 };
165168
166169 static inline struct bpf_reg_state *cur_regs(struct bpf_verifier_env *env)
include/uapi/linux/bpf.h+6−0
@@ -197,8 +197,14 @@ enum bpf_attach_type { · 196 unmodified lines
197197 */
198198 #define BPF_F_STRICT_ALIGNMENT (1U << 0)
199199
200+/* when bpf_ldimm64->src_reg == BPF_PSEUDO_MAP_FD, bpf_ldimm64->imm == fd */
200201 #define BPF_PSEUDO_MAP_FD 1
201202
203+/* when bpf_call->src_reg == BPF_PSEUDO_CALL, bpf_call->imm == pc-relative
204+ * offset to another bpf function
205+ */
206+#define BPF_PSEUDO_CALL 1
207+
202208 /* flags for BPF_MAP_UPDATE_ELEM command */
203209 #define BPF_ANY 0 /* create new element or update existing */
204210 #define BPF_NOEXIST 1 /* create new element if it didn't exist */
kernel/bpf/disasm.c+6−2
@@ -189,8 +189,12 @@ void print_bpf_insn(bpf_insn_print_cb verbose, struct bpf_verifier_env *env, · 188 unmodified lines
189189 u8 opcode = BPF_OP(insn->code);
190190
191191 if (opcode == BPF_CALL) {
192− verbose(env, "(%02x) call %s#%d\n", insn->code,
193− func_id_name(insn->imm), insn->imm);
192+ if (insn->src_reg == BPF_PSEUDO_CALL)
193+ verbose(env, "(%02x) call pc%+d\n", insn->code,
194+ insn->imm);
195+ else
196+ verbose(env, "(%02x) call %s#%d\n", insn->code,
197+ func_id_name(insn->imm), insn->imm);
194198 } else if (insn->code == (BPF_JMP | BPF_JA)) {
195199 verbose(env, "(%02x) goto pc%+d\n",
196200 insn->code, insn->off);
kernel/bpf/verifier.c+139−2
@@ -20,6 +20,8 @@ · 19 unmodified lines
2020 #include <linux/file.h>
2121 #include <linux/vmalloc.h>
2222 #include <linux/stringify.h>
23+#include <linux/bsearch.h>
24+#include <linux/sort.h>
2325
2426 #include "disasm.h"
2527
@@ -636,6 +638,113 @@ enum reg_arg_type { · 610 unmodified lines
636638 DST_OP_NO_MARK /* same as above, check only, don't mark */
637639 };
638640
641+static int cmp_subprogs(const void *a, const void *b)
642+{
643+ return *(int *)a - *(int *)b;
644+}
645+
646+static int find_subprog(struct bpf_verifier_env *env, int off)
647+{
648+ u32 *p;
649+
650+ p = bsearch(&off, env->subprog_starts, env->subprog_cnt,
651+ sizeof(env->subprog_starts[0]), cmp_subprogs);
652+ if (!p)
653+ return -ENOENT;
654+ return p - env->subprog_starts;
655+
656+}
657+
658+static int add_subprog(struct bpf_verifier_env *env, int off)
659+{
660+ int insn_cnt = env->prog->len;
661+ int ret;
662+
663+ if (off >= insn_cnt || off < 0) {
664+ verbose(env, "call to invalid destination\n");
665+ return -EINVAL;
666+ }
667+ ret = find_subprog(env, off);
668+ if (ret >= 0)
669+ return 0;
670+ if (env->subprog_cnt >= BPF_MAX_SUBPROGS) {
671+ verbose(env, "too many subprograms\n");
672+ return -E2BIG;
673+ }
674+ env->subprog_starts[env->subprog_cnt++] = off;
675+ sort(env->subprog_starts, env->subprog_cnt,
676+ sizeof(env->subprog_starts[0]), cmp_subprogs, NULL);
677+ return 0;
678+}
679+
680+static int check_subprogs(struct bpf_verifier_env *env)
681+{
682+ int i, ret, subprog_start, subprog_end, off, cur_subprog = 0;
683+ struct bpf_insn *insn = env->prog->insnsi;
684+ int insn_cnt = env->prog->len;
685+
686+ /* determine subprog starts. The end is one before the next starts */
687+ for (i = 0; i < insn_cnt; i++) {
688+ if (insn[i].code != (BPF_JMP | BPF_CALL))
689+ continue;
690+ if (insn[i].src_reg != BPF_PSEUDO_CALL)
691+ continue;
692+ if (!env->allow_ptr_leaks) {
693+ verbose(env, "function calls to other bpf functions are allowed for root only\n");
694+ return -EPERM;
695+ }
696+ if (bpf_prog_is_dev_bound(env->prog->aux)) {
697+ verbose(env, "funcation calls in offloaded programs are not supported yet\n");
698+ return -EINVAL;
699+ }
700+ ret = add_subprog(env, i + insn[i].imm + 1);
701+ if (ret < 0)
702+ return ret;
703+ }
704+
705+ if (env->log.level > 1)
706+ for (i = 0; i < env->subprog_cnt; i++)
707+ verbose(env, "func#%d @%d\n", i, env->subprog_starts[i]);
708+
709+ /* now check that all jumps are within the same subprog */
710+ subprog_start = 0;
711+ if (env->subprog_cnt == cur_subprog)
712+ subprog_end = insn_cnt;
713+ else
714+ subprog_end = env->subprog_starts[cur_subprog++];
715+ for (i = 0; i < insn_cnt; i++) {
716+ u8 code = insn[i].code;
717+
718+ if (BPF_CLASS(code) != BPF_JMP)
719+ goto next;
720+ if (BPF_OP(code) == BPF_EXIT || BPF_OP(code) == BPF_CALL)
721+ goto next;
722+ off = i + insn[i].off + 1;
723+ if (off < subprog_start || off >= subprog_end) {
724+ verbose(env, "jump out of range from insn %d to %d\n", i, off);
725+ return -EINVAL;
726+ }
727+next:
728+ if (i == subprog_end - 1) {
729+ /* to avoid fall-through from one subprog into another
730+ * the last insn of the subprog should be either exit
731+ * or unconditional jump back
732+ */
733+ if (code != (BPF_JMP | BPF_EXIT) &&
734+ code != (BPF_JMP | BPF_JA)) {
735+ verbose(env, "last insn is not an exit or jmp\n");
736+ return -EINVAL;
737+ }
738+ subprog_start = subprog_end;
739+ if (env->subprog_cnt == cur_subprog)
740+ subprog_end = insn_cnt;
741+ else
742+ subprog_end = env->subprog_starts[cur_subprog++];
743+ }
744+ }
745+ return 0;
746+}
747+
639748 static void mark_reg_read(const struct bpf_verifier_state *state, u32 regno)
640749 {
641750 struct bpf_verifier_state *parent = state->parent;
@@ -3284,6 +3393,10 @@ static int check_cfg(struct bpf_verifier_env *env) · 2642 unmodified lines
32843393 int ret = 0;
32853394 int i, t;
32863395
3396+ ret = check_subprogs(env);
3397+ if (ret < 0)
3398+ return ret;
3399+
32873400 insn_state = kcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
32883401 if (!insn_state)
32893402 return -ENOMEM;
@@ -3316,6 +3429,14 @@ peek_stack: · 26 unmodified lines
33163429 goto err_free;
33173430 if (t + 1 < insn_cnt)
33183431 env->explored_states[t + 1] = STATE_LIST_MARK;
3432+ if (insns[t].src_reg == BPF_PSEUDO_CALL) {
3433+ env->explored_states[t] = STATE_LIST_MARK;
3434+ ret = push_insn(t, t + insns[t].imm + 1, BRANCH, env);
3435+ if (ret == 1)
3436+ goto peek_stack;
3437+ else if (ret < 0)
3438+ goto err_free;
3439+ }
33193440 } else if (opcode == BPF_JA) {
33203441 if (BPF_SRC(insns[t].code) != BPF_K) {
33213442 ret = -EINVAL;
@@ -4245,6 +4366,19 @@ static int adjust_insn_aux_data(struct bpf_verifier_env *env, u32 prog_len, · 923 unmodified lines
42454366 return 0;
42464367 }
42474368
4369+static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len)
4370+{
4371+ int i;
4372+
4373+ if (len == 1)
4374+ return;
4375+ for (i = 0; i < env->subprog_cnt; i++) {
4376+ if (env->subprog_starts[i] < off)
4377+ continue;
4378+ env->subprog_starts[i] += len - 1;
4379+ }
4380+}
4381+
42484382 static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
42494383 const struct bpf_insn *patch, u32 len)
42504384 {
@@ -4255,6 +4389,7 @@ static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 of · 4 unmodified lines
42554389 return NULL;
42564390 if (adjust_insn_aux_data(env, new_prog->len, off, len))
42574391 return NULL;
4392+ adjust_subprog_starts(env, off, len);
42584393 return new_prog;
42594394 }
42604395
@@ -4408,6 +4543,8 @@ static int fixup_bpf_calls(struct bpf_verifier_env *env) · 147 unmodified lines
44084543 for (i = 0; i < insn_cnt; i++, insn++) {
44094544 if (insn->code != (BPF_JMP | BPF_CALL))
44104545 continue;
4546+ if (insn->src_reg == BPF_PSEUDO_CALL)
4547+ continue;
44114548
44124549 if (insn->imm == BPF_FUNC_get_route_realm)
44134550 prog->dst_needed = 1;
@@ -4589,12 +4726,12 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr) · 175 unmodified lines
45894726 if (!env->explored_states)
45904727 goto skip_full_check;
45914728
4729+ env->allow_ptr_leaks = capable(CAP_SYS_ADMIN);
4730+
45924731 ret = check_cfg(env);
45934732 if (ret < 0)
45944733 goto skip_full_check;
45954734
4596− env->allow_ptr_leaks = capable(CAP_SYS_ADMIN);
4597−
45984735 ret = do_check(env);
45994736 if (env->cur_state) {
46004737 free_verifier_state(env->cur_state, true);