LD_[ABS|IND] instructions may return from the function early. bpf_tail_call pseudo instruction is either fallthrough or return. Allow them in the subprograms only when subprograms are BTF annotated and have scalar return types. Allow ld_abs and tail_call in the main program even if it calls into subprograms. In the past that was not ok to do for ld_abs, since it was JITed with special exit sequence. Since bpf_gen_ld_abs() was introduced the ld_abs looks like normal exit insn from JIT point of view, so it's safe to allow them in the main program.
Showing the changes to kernel (1 of 3 files). Show all
Unified · Split
| @@ -1494,6 +1494,9 @@ static int check_subprogs(struct bpf_verifier_env *env) · 1493 unmodified lines | |||
| 1494 | insn[i].imm == BPF_FUNC_tail_call && | 1494 | insn[i].imm == BPF_FUNC_tail_call && |
| 1495 | insn[i].src_reg != BPF_PSEUDO_CALL) | 1495 | insn[i].src_reg != BPF_PSEUDO_CALL) |
| 1496 | subprog[cur_subprog].has_tail_call = true; | 1496 | subprog[cur_subprog].has_tail_call = true; |
| 1497 | if (BPF_CLASS(code) == BPF_LD && | ||
| 1498 | (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND)) | ||
| 1499 | subprog[cur_subprog].has_ld_abs = true; | ||
| 1497 | if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) | 1500 | if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) |
| 1498 | goto next; | 1501 | goto next; |
| 1499 | if (BPF_OP(code) == BPF_EXIT || BPF_OP(code) == BPF_CALL) | 1502 | if (BPF_OP(code) == BPF_EXIT || BPF_OP(code) == BPF_CALL) |
| @@ -7514,18 +7517,6 @@ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) · 6014 unmodified lines | |||
| 7514 | return -EINVAL; | 7517 | return -EINVAL; |
| 7515 | } | 7518 | } |
| 7516 | 7519 | ||
| 7517 | if (env->subprog_cnt > 1) { | ||
| 7518 | /* when program has LD_ABS insn JITs and interpreter assume | ||
| 7519 | * that r1 == ctx == skb which is not the case for callees | ||
| 7520 | * that can have arbitrary arguments. It's problematic | ||
| 7521 | * for main prog as well since JITs would need to analyze | ||
| 7522 | * all functions in order to make proper register save/restore | ||
| 7523 | * decisions in the main prog. Hence disallow LD_ABS with calls | ||
| 7524 | */ | ||
| 7525 | verbose(env, "BPF_LD_[ABS|IND] instructions cannot be mixed with bpf-to-bpf calls\n"); | ||
| 7526 | return -EINVAL; | ||
| 7527 | } | ||
| 7528 | |||
| 7529 | if (insn->dst_reg != BPF_REG_0 || insn->off != 0 || | 7520 | if (insn->dst_reg != BPF_REG_0 || insn->off != 0 || |
| 7530 | BPF_SIZE(insn->code) == BPF_DW || | 7521 | BPF_SIZE(insn->code) == BPF_DW || |
| 7531 | (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) { | 7522 | (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) { |
| @@ -7936,6 +7927,23 @@ err_free: · 404 unmodified lines | |||
| 7936 | return ret; | 7927 | return ret; |
| 7937 | } | 7928 | } |
| 7938 | 7929 | ||
| 7930 | static int check_abnormal_return(struct bpf_verifier_env *env) | ||
| 7931 | { | ||
| 7932 | int i; | ||
| 7933 | |||
| 7934 | for (i = 1; i < env->subprog_cnt; i++) { | ||
| 7935 | if (env->subprog_info[i].has_ld_abs) { | ||
| 7936 | verbose(env, "LD_ABS is not allowed in subprogs without BTF\n"); | ||
| 7937 | return -EINVAL; | ||
| 7938 | } | ||
| 7939 | if (env->subprog_info[i].has_tail_call) { | ||
| 7940 | verbose(env, "tail_call is not allowed in subprogs without BTF\n"); | ||
| 7941 | return -EINVAL; | ||
| 7942 | } | ||
| 7943 | } | ||
| 7944 | return 0; | ||
| 7945 | } | ||
| 7946 | |||
| 7939 | /* The minimum supported BTF func info size */ | 7947 | /* The minimum supported BTF func info size */ |
| 7940 | #define MIN_BPF_FUNCINFO_SIZE 8 | 7948 | #define MIN_BPF_FUNCINFO_SIZE 8 |
| 7941 | #define MAX_FUNCINFO_REC_SIZE 252 | 7949 | #define MAX_FUNCINFO_REC_SIZE 252 |
| @@ -7944,20 +7952,24 @@ static int check_btf_func(struct bpf_verifier_env *env, · 2 unmodified lines | |||
| 7944 | const union bpf_attr *attr, | 7952 | const union bpf_attr *attr, |
| 7945 | union bpf_attr __user *uattr) | 7953 | union bpf_attr __user *uattr) |
| 7946 | { | 7954 | { |
| 7955 | const struct btf_type *type, *func_proto, *ret_type; | ||
| 7947 | u32 i, nfuncs, urec_size, min_size; | 7956 | u32 i, nfuncs, urec_size, min_size; |
| 7948 | u32 krec_size = sizeof(struct bpf_func_info); | 7957 | u32 krec_size = sizeof(struct bpf_func_info); |
| 7949 | struct bpf_func_info *krecord; | 7958 | struct bpf_func_info *krecord; |
| 7950 | struct bpf_func_info_aux *info_aux = NULL; | 7959 | struct bpf_func_info_aux *info_aux = NULL; |
| 7951 | const struct btf_type *type; | ||
| 7952 | struct bpf_prog *prog; | 7960 | struct bpf_prog *prog; |
| 7953 | const struct btf *btf; | 7961 | const struct btf *btf; |
| 7954 | void __user *urecord; | 7962 | void __user *urecord; |
| 7955 | u32 prev_offset = 0; | 7963 | u32 prev_offset = 0; |
| 7964 | bool scalar_return; | ||
| 7956 | int ret = -ENOMEM; | 7965 | int ret = -ENOMEM; |
| 7957 | 7966 | ||
| 7958 | nfuncs = attr->func_info_cnt; | 7967 | nfuncs = attr->func_info_cnt; |
| 7959 | if (!nfuncs) | 7968 | if (!nfuncs) { |
| 7969 | if (check_abnormal_return(env)) | ||
| 7970 | return -EINVAL; | ||
| 7960 | return 0; | 7971 | return 0; |
| 7972 | } | ||
| 7961 | 7973 | ||
| 7962 | if (nfuncs != env->subprog_cnt) { | 7974 | if (nfuncs != env->subprog_cnt) { |
| 7963 | verbose(env, "number of funcs in func_info doesn't match number of subprogs\n"); | 7975 | verbose(env, "number of funcs in func_info doesn't match number of subprogs\n"); |
| @@ -8005,25 +8017,23 @@ static int check_btf_func(struct bpf_verifier_env *env, · 41 unmodified lines | |||
| 8005 | } | 8017 | } |
| 8006 | 8018 | ||
| 8007 | /* check insn_off */ | 8019 | /* check insn_off */ |
| 8020 | ret = -EINVAL; | ||
| 8008 | if (i == 0) { | 8021 | if (i == 0) { |
| 8009 | if (krecord[i].insn_off) { | 8022 | if (krecord[i].insn_off) { |
| 8010 | verbose(env, | 8023 | verbose(env, |
| 8011 | "nonzero insn_off %u for the first func info record", | 8024 | "nonzero insn_off %u for the first func info record", |
| 8012 | krecord[i].insn_off); | 8025 | krecord[i].insn_off); |
| 8013 | ret = -EINVAL; | ||
| 8014 | goto err_free; | 8026 | goto err_free; |
| 8015 | } | 8027 | } |
| 8016 | } else if (krecord[i].insn_off <= prev_offset) { | 8028 | } else if (krecord[i].insn_off <= prev_offset) { |
| 8017 | verbose(env, | 8029 | verbose(env, |
| 8018 | "same or smaller insn offset (%u) than previous func info record (%u)", | 8030 | "same or smaller insn offset (%u) than previous func info record (%u)", |
| 8019 | krecord[i].insn_off, prev_offset); | 8031 | krecord[i].insn_off, prev_offset); |
| 8020 | ret = -EINVAL; | ||
| 8021 | goto err_free; | 8032 | goto err_free; |
| 8022 | } | 8033 | } |
| 8023 | 8034 | ||
| 8024 | if (env->subprog_info[i].start != krecord[i].insn_off) { | 8035 | if (env->subprog_info[i].start != krecord[i].insn_off) { |
| 8025 | verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n"); | 8036 | verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n"); |
| 8026 | ret = -EINVAL; | ||
| 8027 | goto err_free; | 8037 | goto err_free; |
| 8028 | } | 8038 | } |
| 8029 | 8039 | ||
| @@ -8032,10 +8042,26 @@ static int check_btf_func(struct bpf_verifier_env *env, · 2 unmodified lines | |||
| 8032 | if (!type || !btf_type_is_func(type)) { | 8042 | if (!type || !btf_type_is_func(type)) { |
| 8033 | verbose(env, "invalid type id %d in func info", | 8043 | verbose(env, "invalid type id %d in func info", |
| 8034 | krecord[i].type_id); | 8044 | krecord[i].type_id); |
| 8035 | ret = -EINVAL; | ||
| 8036 | goto err_free; | 8045 | goto err_free; |
| 8037 | } | 8046 | } |
| 8038 | info_aux[i].linkage = BTF_INFO_VLEN(type->info); | 8047 | info_aux[i].linkage = BTF_INFO_VLEN(type->info); |
| 8048 | |||
| 8049 | func_proto = btf_type_by_id(btf, type->type); | ||
| 8050 | if (unlikely(!func_proto || !btf_type_is_func_proto(func_proto))) | ||
| 8051 | /* btf_func_check() already verified it during BTF load */ | ||
| 8052 | goto err_free; | ||
| 8053 | ret_type = btf_type_skip_modifiers(btf, func_proto->type, NULL); | ||
| 8054 | scalar_return = | ||
| 8055 | btf_type_is_small_int(ret_type) || btf_type_is_enum(ret_type); | ||
| 8056 | if (i && !scalar_return && env->subprog_info[i].has_ld_abs) { | ||
| 8057 | verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n"); | ||
| 8058 | goto err_free; | ||
| 8059 | } | ||
| 8060 | if (i && !scalar_return && env->subprog_info[i].has_tail_call) { | ||
| 8061 | verbose(env, "tail_call is only allowed in functions that return 'int'.\n"); | ||
| 8062 | goto err_free; | ||
| 8063 | } | ||
| 8064 | |||
| 8039 | prev_offset = krecord[i].insn_off; | 8065 | prev_offset = krecord[i].insn_off; |
| 8040 | urecord += urec_size; | 8066 | urecord += urec_size; |
| 8041 | } | 8067 | } |
| @@ -8196,8 +8222,11 @@ static int check_btf_info(struct bpf_verifier_env *env, · 154 unmodified lines | |||
| 8196 | struct btf *btf; | 8222 | struct btf *btf; |
| 8197 | int err; | 8223 | int err; |
| 8198 | 8224 | ||
| 8199 | if (!attr->func_info_cnt && !attr->line_info_cnt) | 8225 | if (!attr->func_info_cnt && !attr->line_info_cnt) { |
| 8226 | if (check_abnormal_return(env)) | ||
| 8227 | return -EINVAL; | ||
| 8200 | return 0; | 8228 | return 0; |
| 8229 | } | ||
| 8201 | 8230 | ||
| 8202 | btf = btf_get_by_fd(attr->prog_btf_fd); | 8231 | btf = btf_get_by_fd(attr->prog_btf_fd); |
| 8203 | if (IS_ERR(btf)) | 8232 | if (IS_ERR(btf)) |