LD_[ABS|IND] instructions may return from the function early. bpf_tail_call pseudo instruction is either fallthrough or return. Allow them in the subprograms only when subprograms are BTF annotated and have scalar return types. Allow ld_abs and tail_call in the main program even if it calls into subprograms. In the past that was not ok to do for ld_abs, since it was JITed with special exit sequence. Since bpf_gen_ld_abs() was introduced the ld_abs looks like normal exit insn from JIT point of view, so it's safe to allow them in the main program.
Showing the changes to kernel (1 of 3 files). Show all
Unified · Split
| @@ -1494,6 +1494,9 @@ static int check_subprogs(struct bpf_verifier_env *env) · 1493 unmodified lines | ||
| 1494 | 1494 | insn[i].imm == BPF_FUNC_tail_call && |
| 1495 | 1495 | insn[i].src_reg != BPF_PSEUDO_CALL) |
| 1496 | 1496 | subprog[cur_subprog].has_tail_call = true; |
| 1497 | + if (BPF_CLASS(code) == BPF_LD && | |
| 1498 | + (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND)) | |
| 1499 | + subprog[cur_subprog].has_ld_abs = true; | |
| 1497 | 1500 | if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) |
| 1498 | 1501 | goto next; |
| 1499 | 1502 | if (BPF_OP(code) == BPF_EXIT || BPF_OP(code) == BPF_CALL) |
| @@ -7514,18 +7517,6 @@ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) · 6014 unmodified lines | ||
| 7514 | 7517 | return -EINVAL; |
| 7515 | 7518 | } |
| 7516 | 7519 | |
| 7517 | − if (env->subprog_cnt > 1) { | |
| 7518 | − /* when program has LD_ABS insn JITs and interpreter assume | |
| 7519 | − * that r1 == ctx == skb which is not the case for callees | |
| 7520 | − * that can have arbitrary arguments. It's problematic | |
| 7521 | − * for main prog as well since JITs would need to analyze | |
| 7522 | − * all functions in order to make proper register save/restore | |
| 7523 | − * decisions in the main prog. Hence disallow LD_ABS with calls | |
| 7524 | − */ | |
| 7525 | − verbose(env, "BPF_LD_[ABS|IND] instructions cannot be mixed with bpf-to-bpf calls\n"); | |
| 7526 | − return -EINVAL; | |
| 7527 | − } | |
| 7528 | − | |
| 7529 | 7520 | if (insn->dst_reg != BPF_REG_0 || insn->off != 0 || |
| 7530 | 7521 | BPF_SIZE(insn->code) == BPF_DW || |
| 7531 | 7522 | (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) { |
| @@ -7936,6 +7927,23 @@ err_free: · 404 unmodified lines | ||
| 7936 | 7927 | return ret; |
| 7937 | 7928 | } |
| 7938 | 7929 | |
| 7930 | +static int check_abnormal_return(struct bpf_verifier_env *env) | |
| 7931 | +{ | |
| 7932 | + int i; | |
| 7933 | + | |
| 7934 | + for (i = 1; i < env->subprog_cnt; i++) { | |
| 7935 | + if (env->subprog_info[i].has_ld_abs) { | |
| 7936 | + verbose(env, "LD_ABS is not allowed in subprogs without BTF\n"); | |
| 7937 | + return -EINVAL; | |
| 7938 | + } | |
| 7939 | + if (env->subprog_info[i].has_tail_call) { | |
| 7940 | + verbose(env, "tail_call is not allowed in subprogs without BTF\n"); | |
| 7941 | + return -EINVAL; | |
| 7942 | + } | |
| 7943 | + } | |
| 7944 | + return 0; | |
| 7945 | +} | |
| 7946 | + | |
| 7939 | 7947 | /* The minimum supported BTF func info size */ |
| 7940 | 7948 | #define MIN_BPF_FUNCINFO_SIZE 8 |
| 7941 | 7949 | #define MAX_FUNCINFO_REC_SIZE 252 |
| @@ -7944,20 +7952,24 @@ static int check_btf_func(struct bpf_verifier_env *env, · 2 unmodified lines | ||
| 7944 | 7952 | const union bpf_attr *attr, |
| 7945 | 7953 | union bpf_attr __user *uattr) |
| 7946 | 7954 | { |
| 7955 | + const struct btf_type *type, *func_proto, *ret_type; | |
| 7947 | 7956 | u32 i, nfuncs, urec_size, min_size; |
| 7948 | 7957 | u32 krec_size = sizeof(struct bpf_func_info); |
| 7949 | 7958 | struct bpf_func_info *krecord; |
| 7950 | 7959 | struct bpf_func_info_aux *info_aux = NULL; |
| 7951 | − const struct btf_type *type; | |
| 7952 | 7960 | struct bpf_prog *prog; |
| 7953 | 7961 | const struct btf *btf; |
| 7954 | 7962 | void __user *urecord; |
| 7955 | 7963 | u32 prev_offset = 0; |
| 7964 | + bool scalar_return; | |
| 7956 | 7965 | int ret = -ENOMEM; |
| 7957 | 7966 | |
| 7958 | 7967 | nfuncs = attr->func_info_cnt; |
| 7959 | − if (!nfuncs) | |
| 7968 | + if (!nfuncs) { | |
| 7969 | + if (check_abnormal_return(env)) | |
| 7970 | + return -EINVAL; | |
| 7960 | 7971 | return 0; |
| 7972 | + } | |
| 7961 | 7973 | |
| 7962 | 7974 | if (nfuncs != env->subprog_cnt) { |
| 7963 | 7975 | verbose(env, "number of funcs in func_info doesn't match number of subprogs\n"); |
| @@ -8005,25 +8017,23 @@ static int check_btf_func(struct bpf_verifier_env *env, · 41 unmodified lines | ||
| 8005 | 8017 | } |
| 8006 | 8018 | |
| 8007 | 8019 | /* check insn_off */ |
| 8020 | + ret = -EINVAL; | |
| 8008 | 8021 | if (i == 0) { |
| 8009 | 8022 | if (krecord[i].insn_off) { |
| 8010 | 8023 | verbose(env, |
| 8011 | 8024 | "nonzero insn_off %u for the first func info record", |
| 8012 | 8025 | krecord[i].insn_off); |
| 8013 | − ret = -EINVAL; | |
| 8014 | 8026 | goto err_free; |
| 8015 | 8027 | } |
| 8016 | 8028 | } else if (krecord[i].insn_off <= prev_offset) { |
| 8017 | 8029 | verbose(env, |
| 8018 | 8030 | "same or smaller insn offset (%u) than previous func info record (%u)", |
| 8019 | 8031 | krecord[i].insn_off, prev_offset); |
| 8020 | − ret = -EINVAL; | |
| 8021 | 8032 | goto err_free; |
| 8022 | 8033 | } |
| 8023 | 8034 | |
| 8024 | 8035 | if (env->subprog_info[i].start != krecord[i].insn_off) { |
| 8025 | 8036 | verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n"); |
| 8026 | − ret = -EINVAL; | |
| 8027 | 8037 | goto err_free; |
| 8028 | 8038 | } |
| 8029 | 8039 | |
| @@ -8032,10 +8042,26 @@ static int check_btf_func(struct bpf_verifier_env *env, · 2 unmodified lines | ||
| 8032 | 8042 | if (!type || !btf_type_is_func(type)) { |
| 8033 | 8043 | verbose(env, "invalid type id %d in func info", |
| 8034 | 8044 | krecord[i].type_id); |
| 8035 | − ret = -EINVAL; | |
| 8036 | 8045 | goto err_free; |
| 8037 | 8046 | } |
| 8038 | 8047 | info_aux[i].linkage = BTF_INFO_VLEN(type->info); |
| 8048 | + | |
| 8049 | + func_proto = btf_type_by_id(btf, type->type); | |
| 8050 | + if (unlikely(!func_proto || !btf_type_is_func_proto(func_proto))) | |
| 8051 | + /* btf_func_check() already verified it during BTF load */ | |
| 8052 | + goto err_free; | |
| 8053 | + ret_type = btf_type_skip_modifiers(btf, func_proto->type, NULL); | |
| 8054 | + scalar_return = | |
| 8055 | + btf_type_is_small_int(ret_type) || btf_type_is_enum(ret_type); | |
| 8056 | + if (i && !scalar_return && env->subprog_info[i].has_ld_abs) { | |
| 8057 | + verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n"); | |
| 8058 | + goto err_free; | |
| 8059 | + } | |
| 8060 | + if (i && !scalar_return && env->subprog_info[i].has_tail_call) { | |
| 8061 | + verbose(env, "tail_call is only allowed in functions that return 'int'.\n"); | |
| 8062 | + goto err_free; | |
| 8063 | + } | |
| 8064 | + | |
| 8039 | 8065 | prev_offset = krecord[i].insn_off; |
| 8040 | 8066 | urecord += urec_size; |
| 8041 | 8067 | } |
| @@ -8196,8 +8222,11 @@ static int check_btf_info(struct bpf_verifier_env *env, · 154 unmodified lines | ||
| 8196 | 8222 | struct btf *btf; |
| 8197 | 8223 | int err; |
| 8198 | 8224 | |
| 8199 | − if (!attr->func_info_cnt && !attr->line_info_cnt) | |
| 8225 | + if (!attr->func_info_cnt && !attr->line_info_cnt) { | |
| 8226 | + if (check_abnormal_return(env)) | |
| 8227 | + return -EINVAL; | |
| 8200 | 8228 | return 0; |
| 8229 | + } | |
| 8201 | 8230 | |
| 8202 | 8231 | btf = btf_get_by_fd(attr->prog_btf_fd); |
| 8203 | 8232 | if (IS_ERR(btf)) |