Pull bpf fixes from Alexei Starovoitov: - Fix bpf_throw() and global subprog combination (Kumar Kartikeya Dwivedi) - Fix out of bounds access in BPF interpreter (Yazhou Tang) - Fix potential out of bounds access in inner per-cpu array map (Guannan Wang) - Reject NULL data/sig in bpf_verify_pkcs7_signature (KP Singh) * tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf: libbpf: fix off-by-one in emit_signature_match jump offset bpf: Reject NULL data/sig in bpf_verify_pkcs7_signature selftests/bpf: Cover global subprog exception leaks bpf: Check global subprog exception paths bpf: make bpf_session_is_return() reference optional bpf: Use array_map_meta_equal for percpu array inner map replacement selftests/bpf: Add test for large offset bpf-to-bpf call bpf: Fix s16 truncation for large bpf-to-bpf call offsets bpf: Fix out-of-bounds read in bpf_patch_call_args()
Unified · Split
| @@ -2917,7 +2917,13 @@ int bpf_check_uarg_tail_zero(bpfptr_t uaddr, size_t expected_size, · 2916 unmodified lines | |||
| 2917 | int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size); | 2917 | int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size); |
| 2918 | 2918 | ||
| 2919 | #ifndef CONFIG_BPF_JIT_ALWAYS_ON | 2919 | #ifndef CONFIG_BPF_JIT_ALWAYS_ON |
| 2920 | void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth); | 2920 | int bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth); |
| 2921 | s32 bpf_call_args_imm(s16 idx); | ||
| 2922 | #else | ||
| 2923 | static inline s32 bpf_call_args_imm(s16 idx) | ||
| 2924 | { | ||
| 2925 | return 0; | ||
| 2926 | } | ||
| 2921 | #endif | 2927 | #endif |
| 2922 | 2928 | ||
| 2923 | struct btf *bpf_get_btf_vmlinux(void); | 2929 | struct btf *bpf_get_btf_vmlinux(void); |
| @@ -729,6 +729,7 @@ struct bpf_subprog_info { · 728 unmodified lines | |||
| 729 | */ | 729 | */ |
| 730 | s16 fastcall_stack_off; | 730 | s16 fastcall_stack_off; |
| 731 | bool has_tail_call: 1; | 731 | bool has_tail_call: 1; |
| 732 | bool might_throw: 1; | ||
| 732 | bool tail_call_reachable: 1; | 733 | bool tail_call_reachable: 1; |
| 733 | bool has_ld_abs: 1; | 734 | bool has_ld_abs: 1; |
| 734 | bool is_cb: 1; | 735 | bool is_cb: 1; |
| @@ -1308,6 +1309,7 @@ void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask); · 573 unmodified lines | |||
| 1308 | bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog); | 1309 | bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog); |
| 1309 | 1310 | ||
| 1310 | int bpf_find_subprog(struct bpf_verifier_env *env, int off); | 1311 | int bpf_find_subprog(struct bpf_verifier_env *env, int off); |
| 1312 | bool bpf_is_throw_kfunc(struct bpf_insn *insn); | ||
| 1311 | int bpf_compute_const_regs(struct bpf_verifier_env *env); | 1313 | int bpf_compute_const_regs(struct bpf_verifier_env *env); |
| 1312 | int bpf_prune_dead_branches(struct bpf_verifier_env *env); | 1314 | int bpf_prune_dead_branches(struct bpf_verifier_env *env); |
| 1313 | int bpf_check_cfg(struct bpf_verifier_env *env); | 1315 | int bpf_check_cfg(struct bpf_verifier_env *env); |
| @@ -1151,9 +1151,6 @@ bool sk_filter_charge(struct sock *sk, struct sk_filter *fp); · 1150 unmodified lines | |||
| 1151 | void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp); | 1151 | void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp); |
| 1152 | 1152 | ||
| 1153 | u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); | 1153 | u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); |
| 1154 | #define __bpf_call_base_args \ | ||
| 1155 | ((u64 (*)(u64, u64, u64, u64, u64, const struct bpf_insn *)) \ | ||
| 1156 | (void *)__bpf_call_base) | ||
| 1157 | 1154 | ||
| 1158 | struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog); | 1155 | struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog); |
| 1159 | void bpf_jit_compile(struct bpf_prog *prog); | 1156 | void bpf_jit_compile(struct bpf_prog *prog); |
| @@ -827,7 +827,7 @@ const struct bpf_map_ops array_map_ops = { · 826 unmodified lines | |||
| 827 | }; | 827 | }; |
| 828 | 828 | ||
| 829 | const struct bpf_map_ops percpu_array_map_ops = { | 829 | const struct bpf_map_ops percpu_array_map_ops = { |
| 830 | .map_meta_equal = bpf_map_meta_equal, | 830 | .map_meta_equal = array_map_meta_equal, |
| 831 | .map_alloc_check = array_map_alloc_check, | 831 | .map_alloc_check = array_map_alloc_check, |
| 832 | .map_alloc = array_map_alloc, | 832 | .map_alloc = array_map_alloc, |
| 833 | .map_free = array_map_free, | 833 | .map_free = array_map_free, |
| @@ -64,11 +64,19 @@ static void mark_subprog_might_sleep(struct bpf_verifier_env *env, int off) · 63 unmodified lines | |||
| 64 | subprog->might_sleep = true; | 64 | subprog->might_sleep = true; |
| 65 | } | 65 | } |
| 66 | 66 | ||
| 67 | static void mark_subprog_might_throw(struct bpf_verifier_env *env, int off) | ||
| 68 | { | ||
| 69 | struct bpf_subprog_info *subprog; | ||
| 70 | |||
| 71 | subprog = bpf_find_containing_subprog(env, off); | ||
| 72 | subprog->might_throw = true; | ||
| 73 | } | ||
| 74 | |||
| 67 | /* 't' is an index of a call-site. | 75 | /* 't' is an index of a call-site. |
| 68 | * 'w' is a callee entry point. | 76 | * 'w' is a callee entry point. |
| 69 | * Eventually this function would be called when env->cfg.insn_state[w] == EXPLORED. | 77 | * Eventually this function would be called when env->cfg.insn_state[w] == EXPLORED. |
| 70 | * Rely on DFS traversal order and absence of recursive calls to guarantee that | 78 | * Rely on DFS traversal order and absence of recursive calls to guarantee that |
| 71 | * callee's change_pkt_data marks would be correct at that moment. | 79 | * callee's effect marks would be correct at that moment. |
| 72 | */ | 80 | */ |
| 73 | static void merge_callee_effects(struct bpf_verifier_env *env, int t, int w) | 81 | static void merge_callee_effects(struct bpf_verifier_env *env, int t, int w) |
| 74 | { | 82 | { |
| @@ -78,6 +86,7 @@ static void merge_callee_effects(struct bpf_verifier_env *env, int t, int w) · 3 unmodified lines | |||
| 78 | callee = bpf_find_containing_subprog(env, w); | 86 | callee = bpf_find_containing_subprog(env, w); |
| 79 | caller->changes_pkt_data |= callee->changes_pkt_data; | 87 | caller->changes_pkt_data |= callee->changes_pkt_data; |
| 80 | caller->might_sleep |= callee->might_sleep; | 88 | caller->might_sleep |= callee->might_sleep; |
| 89 | caller->might_throw |= callee->might_throw; | ||
| 81 | } | 90 | } |
| 82 | 91 | ||
| 83 | enum { | 92 | enum { |
| @@ -509,6 +518,8 @@ static int visit_insn(int t, struct bpf_verifier_env *env) · 425 unmodified lines | |||
| 509 | mark_subprog_might_sleep(env, t); | 518 | mark_subprog_might_sleep(env, t); |
| 510 | if (ret == 0 && bpf_is_kfunc_pkt_changing(&meta)) | 519 | if (ret == 0 && bpf_is_kfunc_pkt_changing(&meta)) |
| 511 | mark_subprog_changes_pkt_data(env, t); | 520 | mark_subprog_changes_pkt_data(env, t); |
| 521 | if (ret == 0 && bpf_is_throw_kfunc(insn)) | ||
| 522 | mark_subprog_might_throw(env, t); | ||
| 512 | } | 523 | } |
| 513 | return visit_func_call_insn(t, insns, env, insn->src_reg == BPF_PSEUDO_CALL); | 524 | return visit_func_call_insn(t, insns, env, insn->src_reg == BPF_PSEUDO_CALL); |
| 514 | 525 | ||
| @@ -1771,6 +1771,9 @@ static u32 abs_s32(s32 x) · 1770 unmodified lines | |||
| 1771 | return x >= 0 ? (u32)x : -(u32)x; | 1771 | return x >= 0 ? (u32)x : -(u32)x; |
| 1772 | } | 1772 | } |
| 1773 | 1773 | ||
| 1774 | static u64 (*interpreters_args[])(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5, | ||
| 1775 | const struct bpf_insn *insn); | ||
| 1776 | |||
| 1774 | /** | 1777 | /** |
| 1775 | * ___bpf_prog_run - run eBPF program on a given context | 1778 | * ___bpf_prog_run - run eBPF program on a given context |
| 1776 | * @regs: is the array of MAX_BPF_EXT_REG eBPF pseudo-registers | 1779 | * @regs: is the array of MAX_BPF_EXT_REG eBPF pseudo-registers |
| @@ -2077,10 +2080,9 @@ select_insn: · 300 unmodified lines | |||
| 2077 | CONT; | 2080 | CONT; |
| 2078 | 2081 | ||
| 2079 | JMP_CALL_ARGS: | 2082 | JMP_CALL_ARGS: |
| 2080 | BPF_R0 = (__bpf_call_base_args + insn->imm)(BPF_R1, BPF_R2, | 2083 | BPF_R0 = interpreters_args[insn->off](BPF_R1, BPF_R2, BPF_R3, |
| 2081 | BPF_R3, BPF_R4, | 2084 | BPF_R4, BPF_R5, |
| 2082 | BPF_R5, | 2085 | insn + insn->imm + 1); |
| 2083 | insn + insn->off + 1); | ||
| 2084 | CONT; | 2086 | CONT; |
| 2085 | 2087 | ||
| 2086 | JMP_TAIL_CALL: { | 2088 | JMP_TAIL_CALL: { |
| @@ -2394,13 +2396,22 @@ EVAL4(PROG_NAME_LIST, 416, 448, 480, 512) · 307 unmodified lines | |||
| 2394 | #undef PROG_NAME_LIST | 2396 | #undef PROG_NAME_LIST |
| 2395 | 2397 | ||
| 2396 | #ifdef CONFIG_BPF_SYSCALL | 2398 | #ifdef CONFIG_BPF_SYSCALL |
| 2397 | void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth) | 2399 | int bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth) |
| 2398 | { | 2400 | { |
| 2399 | stack_depth = max_t(u32, stack_depth, 1); | 2401 | stack_depth = max_t(u32, stack_depth, 1); |
| 2400 | insn->off = (s16) insn->imm; | 2402 | /* Prevent out-of-bounds read to interpreters_args */ |
| 2401 | insn->imm = interpreters_args[(round_up(stack_depth, 32) / 32) - 1] - | 2403 | if (stack_depth > MAX_BPF_STACK) |
| 2402 | __bpf_call_base_args; | 2404 | return -EINVAL; |
| 2405 | insn->off = (round_up(stack_depth, 32) / 32) - 1; | ||
| 2403 | insn->code = BPF_JMP | BPF_CALL_ARGS; | 2406 | insn->code = BPF_JMP | BPF_CALL_ARGS; |
| 2407 | return 0; | ||
| 2408 | } | ||
| 2409 | |||
| 2410 | s32 bpf_call_args_imm(s16 idx) | ||
| 2411 | { | ||
| 2412 | if (WARN_ON_ONCE(idx < 0 || idx >= ARRAY_SIZE(interpreters_args))) | ||
| 2413 | return 0; | ||
| 2414 | return BPF_CALL_IMM(interpreters_args[idx]); | ||
| 2404 | } | 2415 | } |
| 2405 | #endif | 2416 | #endif |
| 2406 | #endif | 2417 | #endif |
| @@ -1250,9 +1250,9 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 1249 unmodified lines | |||
| 1250 | } | 1250 | } |
| 1251 | if (!bpf_pseudo_call(insn)) | 1251 | if (!bpf_pseudo_call(insn)) |
| 1252 | continue; | 1252 | continue; |
| 1253 | insn->off = env->insn_aux_data[i].call_imm; | 1253 | insn->imm = env->insn_aux_data[i].call_imm; |
| 1254 | subprog = bpf_find_subprog(env, i + insn->off + 1); | 1254 | subprog = bpf_find_subprog(env, i + insn->imm + 1); |
| 1255 | insn->imm = subprog; | 1255 | insn->off = subprog; |
| 1256 | } | 1256 | } |
| 1257 | 1257 | ||
| 1258 | prog->jited = 1; | 1258 | prog->jited = 1; |
| @@ -1416,7 +1416,12 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env) · 157 unmodified lines | |||
| 1416 | depth = get_callee_stack_depth(env, insn, i); | 1416 | depth = get_callee_stack_depth(env, insn, i); |
| 1417 | if (depth < 0) | 1417 | if (depth < 0) |
| 1418 | return depth; | 1418 | return depth; |
| 1419 | bpf_patch_call_args(insn, depth); | 1419 | err = bpf_patch_call_args(insn, depth); |
| 1420 | if (err) { | ||
| 1421 | verbose(env, "stack depth %d exceeds interpreter stack depth limit\n", | ||
| 1422 | depth); | ||
| 1423 | return err; | ||
| 1424 | } | ||
| 1420 | } | 1425 | } |
| 1421 | err = 0; | 1426 | err = 0; |
| 1422 | #endif | 1427 | #endif |
| @@ -4241,8 +4241,13 @@ __bpf_kfunc int bpf_verify_pkcs7_signature(struct bpf_dynptr *data_p, · 4240 unmodified lines | |||
| 4241 | 4241 | ||
| 4242 | data_len = __bpf_dynptr_size(data_ptr); | 4242 | data_len = __bpf_dynptr_size(data_ptr); |
| 4243 | data = __bpf_dynptr_data(data_ptr, data_len); | 4243 | data = __bpf_dynptr_data(data_ptr, data_len); |
| 4244 | if (!data) | ||
| 4245 | return -EINVAL; | ||
| 4246 | |||
| 4244 | sig_len = __bpf_dynptr_size(sig_ptr); | 4247 | sig_len = __bpf_dynptr_size(sig_ptr); |
| 4245 | sig = __bpf_dynptr_data(sig_ptr, sig_len); | 4248 | sig = __bpf_dynptr_data(sig_ptr, sig_len); |
| 4249 | if (!sig) | ||
| 4250 | return -EINVAL; | ||
| 4246 | 4251 | ||
| 4247 | return verify_pkcs7_signature(data, data_len, sig, sig_len, | 4252 | return verify_pkcs7_signature(data, data_len, sig, sig_len, |
| 4248 | trusted_keyring->key, | 4253 | trusted_keyring->key, |
| @@ -4919,6 +4919,29 @@ out: · 4918 unmodified lines | |||
| 4919 | return map; | 4919 | return map; |
| 4920 | } | 4920 | } |
| 4921 | 4921 | ||
| 4922 | static void prepare_dump_pseudo_call(struct bpf_insn *insn) | ||
| 4923 | { | ||
| 4924 | s32 call_off = insn->imm; | ||
| 4925 | |||
| 4926 | /* | ||
| 4927 | * BPF_CALL_ARGS only exists for interpreter fallback. | ||
| 4928 | * 1. For interpreter (BPF_CALL_ARGS): insn->off is the index of | ||
| 4929 | * interpreters_args array, so here using bpf_call_args_imm() | ||
| 4930 | * to get the real address offset. | ||
| 4931 | * 2. For JIT (BPF_CALL): insn->off is the subprog id. | ||
| 4932 | */ | ||
| 4933 | if (insn->code == (BPF_JMP | BPF_CALL_ARGS)) | ||
| 4934 | insn->imm = bpf_call_args_imm(insn->off); | ||
| 4935 | else | ||
| 4936 | insn->imm = insn->off; | ||
| 4937 | |||
| 4938 | /* Avoid dumping a truncated and misleading pc-relative offset. */ | ||
| 4939 | if (call_off > S16_MAX || call_off < S16_MIN) | ||
| 4940 | insn->off = 0; | ||
| 4941 | else | ||
| 4942 | insn->off = call_off; | ||
| 4943 | } | ||
| 4944 | |||
| 4922 | static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog, | 4945 | static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog, |
| 4923 | const struct cred *f_cred) | 4946 | const struct cred *f_cred) |
| 4924 | { | 4947 | { |
| @@ -4944,6 +4967,9 @@ static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog, · 19 unmodified lines | |||
| 4944 | } | 4967 | } |
| 4945 | if (code == (BPF_JMP | BPF_CALL) || | 4968 | if (code == (BPF_JMP | BPF_CALL) || |
| 4946 | code == (BPF_JMP | BPF_CALL_ARGS)) { | 4969 | code == (BPF_JMP | BPF_CALL_ARGS)) { |
| 4970 | /* Restore the legacy xlated dump layout. */ | ||
| 4971 | if (insns[i].src_reg == BPF_PSEUDO_CALL) | ||
| 4972 | prepare_dump_pseudo_call(&insns[i]); | ||
| 4947 | if (code == (BPF_JMP | BPF_CALL_ARGS)) | 4973 | if (code == (BPF_JMP | BPF_CALL_ARGS)) |
| 4948 | insns[i].code = BPF_JMP | BPF_CALL; | 4974 | insns[i].code = BPF_JMP | BPF_CALL; |
| 4949 | if (!bpf_dump_raw_ok(f_cred)) | 4975 | if (!bpf_dump_raw_ok(f_cred)) |
| @@ -442,7 +442,6 @@ static bool is_dynptr_ref_function(enum bpf_func_id func_id) · 441 unmodified lines | |||
| 442 | static bool is_sync_callback_calling_kfunc(u32 btf_id); | 442 | static bool is_sync_callback_calling_kfunc(u32 btf_id); |
| 443 | static bool is_async_callback_calling_kfunc(u32 btf_id); | 443 | static bool is_async_callback_calling_kfunc(u32 btf_id); |
| 444 | static bool is_callback_calling_kfunc(u32 btf_id); | 444 | static bool is_callback_calling_kfunc(u32 btf_id); |
| 445 | static bool is_bpf_throw_kfunc(struct bpf_insn *insn); | ||
| 446 | 445 | ||
| 447 | static bool is_bpf_wq_set_callback_kfunc(u32 btf_id); | 446 | static bool is_bpf_wq_set_callback_kfunc(u32 btf_id); |
| 448 | static bool is_task_work_add_kfunc(u32 func_id); | 447 | static bool is_task_work_add_kfunc(u32 func_id); |
| @@ -5405,7 +5404,7 @@ continue_func: · 4956 unmodified lines | |||
| 5405 | if (bpf_pseudo_kfunc_call(insn + i) && !insn[i].off) { | 5404 | if (bpf_pseudo_kfunc_call(insn + i) && !insn[i].off) { |
| 5406 | bool err = false; | 5405 | bool err = false; |
| 5407 | 5406 | ||
| 5408 | if (!is_bpf_throw_kfunc(insn + i)) | 5407 | if (!bpf_is_throw_kfunc(insn + i)) |
| 5409 | continue; | 5408 | continue; |
| 5410 | for (tmp = idx; tmp >= 0 && !err; tmp = dinfo[tmp].caller) { | 5409 | for (tmp = idx; tmp >= 0 && !err; tmp = dinfo[tmp].caller) { |
| 5411 | if (subprog[tmp].is_cb) { | 5410 | if (subprog[tmp].is_cb) { |
| @@ -9499,6 +9498,9 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins · 4087 unmodified lines | |||
| 9499 | return 0; | 9498 | return 0; |
| 9500 | } | 9499 | } |
| 9501 | 9500 | ||
| 9501 | static int process_bpf_exit_full(struct bpf_verifier_env *env, | ||
| 9502 | bool *do_print_state, bool exception_exit); | ||
| 9503 | |||
| 9502 | static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, | 9504 | static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, |
| 9503 | int *insn_idx) | 9505 | int *insn_idx) |
| 9504 | { | 9506 | { |
| @@ -9552,6 +9554,17 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · 47 unmodified lines | |||
| 9552 | caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; | 9554 | caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; |
| 9553 | } | 9555 | } |
| 9554 | 9556 | ||
| 9557 | if (env->subprog_info[subprog].might_throw) { | ||
| 9558 | struct bpf_verifier_state *branch; | ||
| 9559 | |||
| 9560 | branch = push_stack(env, *insn_idx + 1, *insn_idx, false); | ||
| 9561 | if (IS_ERR(branch)) { | ||
| 9562 | verbose(env, "failed to push state for global subprog exception path\n"); | ||
| 9563 | return PTR_ERR(branch); | ||
| 9564 | } | ||
| 9565 | return process_bpf_exit_full(env, NULL, true); | ||
| 9566 | } | ||
| 9567 | |||
| 9555 | /* continue with next insn after call */ | 9568 | /* continue with next insn after call */ |
| 9556 | return 0; | 9569 | return 0; |
| 9557 | } | 9570 | } |
| @@ -11263,7 +11276,11 @@ BTF_ID(func, bpf_task_work_schedule_resume) · 1705 unmodified lines | |||
| 11263 | BTF_ID(func, bpf_arena_alloc_pages) | 11276 | BTF_ID(func, bpf_arena_alloc_pages) |
| 11264 | BTF_ID(func, bpf_arena_free_pages) | 11277 | BTF_ID(func, bpf_arena_free_pages) |
| 11265 | BTF_ID(func, bpf_arena_reserve_pages) | 11278 | BTF_ID(func, bpf_arena_reserve_pages) |
| 11279 | #ifdef CONFIG_BPF_EVENTS | ||
| 11266 | BTF_ID(func, bpf_session_is_return) | 11280 | BTF_ID(func, bpf_session_is_return) |
| 11281 | #else | ||
| 11282 | BTF_ID_UNUSED | ||
| 11283 | #endif | ||
| 11267 | BTF_ID(func, bpf_stream_vprintk) | 11284 | BTF_ID(func, bpf_stream_vprintk) |
| 11268 | BTF_ID(func, bpf_stream_print_stack) | 11285 | BTF_ID(func, bpf_stream_print_stack) |
| 11269 | 11286 | ||
| @@ -11778,7 +11795,7 @@ static bool is_async_callback_calling_kfunc(u32 btf_id) · 508 unmodified lines | |||
| 11778 | is_task_work_add_kfunc(btf_id); | 11795 | is_task_work_add_kfunc(btf_id); |
| 11779 | } | 11796 | } |
| 11780 | 11797 | ||
| 11781 | static bool is_bpf_throw_kfunc(struct bpf_insn *insn) | 11798 | bool bpf_is_throw_kfunc(struct bpf_insn *insn) |
| 11782 | { | 11799 | { |
| 11783 | return bpf_pseudo_kfunc_call(insn) && insn->off == 0 && | 11800 | return bpf_pseudo_kfunc_call(insn) && insn->off == 0 && |
| 11784 | insn->imm == special_kfunc_list[KF_bpf_throw]; | 11801 | insn->imm == special_kfunc_list[KF_bpf_throw]; |
| @@ -12968,8 +12985,6 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_ca · 1183 unmodified lines | |||
| 12968 | } | 12985 | } |
| 12969 | 12986 | ||
| 12970 | static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name); | 12987 | static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name); |
| 12971 | static int process_bpf_exit_full(struct bpf_verifier_env *env, | ||
| 12972 | bool *do_print_state, bool exception_exit); | ||
| 12973 | 12988 | ||
| 12974 | static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, | 12989 | static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, |
| 12975 | int *insn_idx_p) | 12990 | int *insn_idx_p) |
| @@ -13350,7 +13365,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · 374 unmodified lines | |||
| 13350 | if (meta.func_id == special_kfunc_list[KF_bpf_session_cookie]) | 13365 | if (meta.func_id == special_kfunc_list[KF_bpf_session_cookie]) |
| 13351 | env->prog->call_session_cookie = true; | 13366 | env->prog->call_session_cookie = true; |
| 13352 | 13367 | ||
| 13353 | if (is_bpf_throw_kfunc(insn)) | 13368 | if (bpf_is_throw_kfunc(insn)) |
| 13354 | return process_bpf_exit_full(env, NULL, true); | 13369 | return process_bpf_exit_full(env, NULL, true); |
| 13355 | 13370 | ||
| 13356 | return 0; | 13371 | return 0; |
| @@ -592,13 +592,12 @@ static void emit_signature_match(struct bpf_gen *gen) · 591 unmodified lines | |||
| 592 | gen->hash_insn_offset[i] = gen->insn_cur - gen->insn_start; | 592 | gen->hash_insn_offset[i] = gen->insn_cur - gen->insn_start; |
| 593 | emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_3, 0, 0, 0, 0, 0)); | 593 | emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_3, 0, 0, 0, 0, 0)); |
| 594 | 594 | ||
| 595 | off = -(gen->insn_cur - gen->insn_start - gen->cleanup_label) / 8 - 1; | 595 | off = -(gen->insn_cur - gen->insn_start - gen->cleanup_label) / 8 - 2; |
| 596 | if (is_simm16(off)) { | 596 | if (is_simm16(off)) { |
| 597 | emit(gen, BPF_MOV64_IMM(BPF_REG_7, -EINVAL)); | 597 | emit(gen, BPF_MOV64_IMM(BPF_REG_7, -EINVAL)); |
| 598 | emit(gen, BPF_JMP_REG(BPF_JNE, BPF_REG_2, BPF_REG_3, off)); | 598 | emit(gen, BPF_JMP_REG(BPF_JNE, BPF_REG_2, BPF_REG_3, off)); |
| 599 | } else { | 599 | } else { |
| 600 | gen->error = -ERANGE; | 600 | gen->error = -ERANGE; |
| 601 | emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, -1)); | ||
| 602 | } | 601 | } |
| 603 | } | 602 | } |
| 604 | } | 603 | } |
| @@ -14,7 +14,7 @@ static struct { · 13 unmodified lines | |||
| 14 | const char *prog_name; | 14 | const char *prog_name; |
| 15 | int expected_runtime_err; | 15 | int expected_runtime_err; |
| 16 | } kfunc_dynptr_tests[] = { | 16 | } kfunc_dynptr_tests[] = { |
| 17 | {"dynptr_data_null", -EBADMSG}, | 17 | {"dynptr_data_null", -EINVAL}, |
| 18 | }; | 18 | }; |
| 19 | 19 | ||
| 20 | static bool kfunc_not_supported; | 20 | static bool kfunc_not_supported; |
| @@ -0,0 +1,57 @@ | |||
| 1 | // SPDX-License-Identifier: GPL-2.0 | ||
| 2 | #include <test_progs.h> | ||
| 3 | |||
| 4 | /* | ||
| 5 | * Test that replacing an inner percpu array map with one that has different | ||
| 6 | * max_entries is rejected. percpu_array_map_gen_lookup() inlines the | ||
| 7 | * template's index_mask, so allowing a smaller replacement would cause OOB. | ||
| 8 | */ | ||
| 9 | void test_percpu_array_inner_map(void) | ||
| 10 | { | ||
| 11 | LIBBPF_OPTS(bpf_map_create_opts, opts); | ||
| 12 | int outer_fd, tmpl_fd, good_fd, bad_fd, err; | ||
| 13 | int zero = 0; | ||
| 14 | |||
| 15 | /* Create template: percpu array with 8 entries */ | ||
| 16 | tmpl_fd = bpf_map_create(BPF_MAP_TYPE_PERCPU_ARRAY, "tmpl", | ||
| 17 | sizeof(int), sizeof(long), 8, NULL); | ||
| 18 | if (!ASSERT_OK_FD(tmpl_fd, "create_tmpl")) | ||
| 19 | return; | ||
| 20 | |||
| 21 | /* Create outer array-of-maps using template */ | ||
| 22 | opts.inner_map_fd = tmpl_fd; | ||
| 23 | outer_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY_OF_MAPS, "outer", | ||
| 24 | sizeof(int), sizeof(int), 1, &opts); | ||
| 25 | if (!ASSERT_OK_FD(outer_fd, "create_outer")) | ||
| 26 | goto close_tmpl; | ||
| 27 | |||
| 28 | /* Insert template as initial inner map */ | ||
| 29 | err = bpf_map_update_elem(outer_fd, &zero, &tmpl_fd, 0); | ||
| 30 | if (!ASSERT_OK(err, "insert_tmpl")) | ||
| 31 | goto close_outer; | ||
| 32 | |||
| 33 | /* Replacement with same max_entries should succeed */ | ||
| 34 | good_fd = bpf_map_create(BPF_MAP_TYPE_PERCPU_ARRAY, "good", | ||
| 35 | sizeof(int), sizeof(long), 8, NULL); | ||
| 36 | if (!ASSERT_OK_FD(good_fd, "create_good")) | ||
| 37 | goto close_outer; | ||
| 38 | |||
| 39 | err = bpf_map_update_elem(outer_fd, &zero, &good_fd, 0); | ||
| 40 | ASSERT_OK(err, "replace_same_max_entries"); | ||
| 41 | close(good_fd); | ||
| 42 | |||
| 43 | /* Replacement with fewer max_entries must fail */ | ||
| 44 | bad_fd = bpf_map_create(BPF_MAP_TYPE_PERCPU_ARRAY, "bad", | ||
| 45 | sizeof(int), sizeof(long), 2, NULL); | ||
| 46 | if (!ASSERT_OK_FD(bad_fd, "create_bad")) | ||
| 47 | goto close_outer; | ||
| 48 | |||
| 49 | err = bpf_map_update_elem(outer_fd, &zero, &bad_fd, 0); | ||
| 50 | ASSERT_ERR(err, "replace_smaller_max_entries"); | ||
| 51 | close(bad_fd); | ||
| 52 | |||
| 53 | close_outer: | ||
| 54 | close(outer_fd); | ||
| 55 | close_tmpl: | ||
| 56 | close(tmpl_fd); | ||
| 57 | } | ||
| @@ -22,6 +22,7 @@ · 21 unmodified lines | |||
| 22 | #include "verifier_bswap.skel.h" | 22 | #include "verifier_bswap.skel.h" |
| 23 | #include "verifier_btf_ctx_access.skel.h" | 23 | #include "verifier_btf_ctx_access.skel.h" |
| 24 | #include "verifier_btf_unreliable_prog.skel.h" | 24 | #include "verifier_btf_unreliable_prog.skel.h" |
| 25 | #include "verifier_call_large_imm.skel.h" | ||
| 25 | #include "verifier_cfg.skel.h" | 26 | #include "verifier_cfg.skel.h" |
| 26 | #include "verifier_cgroup_inv_retcode.skel.h" | 27 | #include "verifier_cgroup_inv_retcode.skel.h" |
| 27 | #include "verifier_cgroup_skb.skel.h" | 28 | #include "verifier_cgroup_skb.skel.h" |
| @@ -170,6 +171,7 @@ void test_verifier_bpf_trap(void) { RUN(verifier_bpf_trap); } · 142 unmodified lines | |||
| 170 | void test_verifier_bswap(void) { RUN(verifier_bswap); } | 171 | void test_verifier_bswap(void) { RUN(verifier_bswap); } |
| 171 | void test_verifier_btf_ctx_access(void) { RUN(verifier_btf_ctx_access); } | 172 | void test_verifier_btf_ctx_access(void) { RUN(verifier_btf_ctx_access); } |
| 172 | void test_verifier_btf_unreliable_prog(void) { RUN(verifier_btf_unreliable_prog); } | 173 | void test_verifier_btf_unreliable_prog(void) { RUN(verifier_btf_unreliable_prog); } |
| 174 | void test_verifier_call_large_imm(void) { RUN(verifier_call_large_imm); } | ||
| 173 | void test_verifier_cfg(void) { RUN(verifier_cfg); } | 175 | void test_verifier_cfg(void) { RUN(verifier_cfg); } |
| 174 | void test_verifier_cgroup_inv_retcode(void) { RUN(verifier_cgroup_inv_retcode); } | 176 | void test_verifier_cgroup_inv_retcode(void) { RUN(verifier_cgroup_inv_retcode); } |
| 175 | void test_verifier_cgroup_skb(void) { RUN(verifier_cgroup_skb); } | 177 | void test_verifier_cgroup_skb(void) { RUN(verifier_cgroup_skb); } |
| @@ -208,6 +208,28 @@ int reject_with_reference(void *ctx) · 207 unmodified lines | |||
| 208 | return 0; | 208 | return 0; |
| 209 | } | 209 | } |
| 210 | 210 | ||
| 211 | __noinline int global_subprog_may_throw(struct __sk_buff *ctx) | ||
| 212 | { | ||
| 213 | if (ctx->len) | ||
| 214 | bpf_throw(0); | ||
| 215 | return 0; | ||
| 216 | } | ||
| 217 | |||
| 218 | SEC("?tc") | ||
| 219 | __failure __msg("Unreleased reference") | ||
| 220 | int reject_global_subprog_throw_with_reference(struct __sk_buff *ctx) | ||
| 221 | { | ||
| 222 | struct foo *f; | ||
| 223 | |||
| 224 | f = bpf_obj_new(typeof(*f)); | ||
| 225 | if (!f) | ||
| 226 | return 0; | ||
| 227 | if (ctx->protocol) | ||
| 228 | global_subprog_may_throw(ctx); | ||
| 229 | bpf_obj_drop(f); | ||
| 230 | return 0; | ||
| 231 | } | ||
| 232 | |||
| 211 | __noinline static int subprog_ref(struct __sk_buff *ctx) | 233 | __noinline static int subprog_ref(struct __sk_buff *ctx) |
| 212 | { | 234 | { |
| 213 | struct foo *f; | 235 | struct foo *f; |
| @@ -0,0 +1,66 @@ | |||
| 1 | // SPDX-License-Identifier: GPL-2.0 | ||
| 2 | #include <linux/bpf.h> | ||
| 3 | #include <bpf/bpf_helpers.h> | ||
| 4 | #include "bpf_misc.h" | ||
| 5 | |||
| 6 | int call_happened = 0; | ||
| 7 | |||
| 8 | /* | ||
| 9 | * 32765 is the exact minimum number of padding instructions needed to | ||
| 10 | * trigger the verifier failure, because: | ||
| 11 | * 1. Counting the wrapper instructions around the padding block (one | ||
| 12 | * "r0=0" and two "exit" instructions), the actual jump distance | ||
| 13 | * evaluates to N + 3. | ||
| 14 | * 2. To overflow the s16 max bound (32767), we need N + 3 > 32767. | ||
| 15 | * Thus, N = 32765 is the exact minimum padding size required. | ||
| 16 | */ | ||
| 17 | static __attribute__((noinline)) void padding_subprog(void) | ||
| 18 | { | ||
| 19 | asm volatile ( | ||
| 20 | "r0 = 0;" | ||
| 21 | ".rept 32765;" | ||
| 22 | "r0 += 0;" | ||
| 23 | ".endr;" | ||
| 24 | ::: __clobber_all); | ||
| 25 | } | ||
| 26 | |||
| 27 | static __attribute__((noinline)) int target_subprog(void) | ||
| 28 | { | ||
| 29 | /* Use volatile variable here to prevent optimization. */ | ||
| 30 | volatile int magic_ret = 3; | ||
| 31 | return magic_ret; | ||
| 32 | } | ||
| 33 | |||
| 34 | SEC("syscall") | ||
| 35 | __success __retval(3) | ||
| 36 | int call_large_imm_test(void *ctx) | ||
| 37 | { | ||
| 38 | /* | ||
| 39 | * Landing pad to handle call error on kernel without the fix, | ||
| 40 | * preventing kernel panic. | ||
| 41 | */ | ||
| 42 | asm volatile ( | ||
| 43 | "r0 = 0;" | ||
| 44 | ".rept 32768;" | ||
| 45 | "r0 += 0;" | ||
| 46 | ".endr;" | ||
| 47 | ::: __clobber_all); | ||
| 48 | |||
| 49 | /* | ||
| 50 | * The call_happened variable is 1 only when the call insn wrongly | ||
| 51 | * go back to the landing pad above. | ||
| 52 | */ | ||
| 53 | if (call_happened == 1) { | ||
| 54 | /* Use volatile variable here to prevent optimization. */ | ||
| 55 | volatile int flag = -1; | ||
| 56 | return flag; | ||
| 57 | } | ||
| 58 | |||
| 59 | call_happened = 1; | ||
| 60 | |||
| 61 | padding_subprog(); | ||
| 62 | |||
| 63 | return target_subprog(); | ||
| 64 | } | ||
| 65 | |||
| 66 | char LICENSE[] SEC("license") = "GPL"; | ||