Global subprogs are verified independently and are not descended into when their callers are symbolically executed. This means a caller can hold references or locks across a global subprog call that may throw, while the verifier only checks the non-exceptional return path at the call site. Record whether a subprog might throw in the CFG summary pass, alongside the existing might_sleep and packet-data-changing summaries, and propagate that effect through reachable callees. When a global subprog is marked as possibly throwing, push the normal continuation and validate the exceptional path immediately at the call site, avoiding a synthetic exception state and associated special case in the pruning checks.
Unified · Split
| @@ -729,6 +729,7 @@ struct bpf_subprog_info { · 728 unmodified lines | |||
| 729 | */ | 729 | */ |
| 730 | s16 fastcall_stack_off; | 730 | s16 fastcall_stack_off; |
| 731 | bool has_tail_call: 1; | 731 | bool has_tail_call: 1; |
| 732 | bool might_throw: 1; | ||
| 732 | bool tail_call_reachable: 1; | 733 | bool tail_call_reachable: 1; |
| 733 | bool has_ld_abs: 1; | 734 | bool has_ld_abs: 1; |
| 734 | bool is_cb: 1; | 735 | bool is_cb: 1; |
| @@ -1308,6 +1309,7 @@ void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask); · 573 unmodified lines | |||
| 1308 | bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog); | 1309 | bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog); |
| 1309 | 1310 | ||
| 1310 | int bpf_find_subprog(struct bpf_verifier_env *env, int off); | 1311 | int bpf_find_subprog(struct bpf_verifier_env *env, int off); |
| 1312 | bool bpf_is_throw_kfunc(struct bpf_insn *insn); | ||
| 1311 | int bpf_compute_const_regs(struct bpf_verifier_env *env); | 1313 | int bpf_compute_const_regs(struct bpf_verifier_env *env); |
| 1312 | int bpf_prune_dead_branches(struct bpf_verifier_env *env); | 1314 | int bpf_prune_dead_branches(struct bpf_verifier_env *env); |
| 1313 | int bpf_check_cfg(struct bpf_verifier_env *env); | 1315 | int bpf_check_cfg(struct bpf_verifier_env *env); |
| @@ -64,11 +64,19 @@ static void mark_subprog_might_sleep(struct bpf_verifier_env *env, int off) · 63 unmodified lines | |||
| 64 | subprog->might_sleep = true; | 64 | subprog->might_sleep = true; |
| 65 | } | 65 | } |
| 66 | 66 | ||
| 67 | static void mark_subprog_might_throw(struct bpf_verifier_env *env, int off) | ||
| 68 | { | ||
| 69 | struct bpf_subprog_info *subprog; | ||
| 70 | |||
| 71 | subprog = bpf_find_containing_subprog(env, off); | ||
| 72 | subprog->might_throw = true; | ||
| 73 | } | ||
| 74 | |||
| 67 | /* 't' is an index of a call-site. | 75 | /* 't' is an index of a call-site. |
| 68 | * 'w' is a callee entry point. | 76 | * 'w' is a callee entry point. |
| 69 | * Eventually this function would be called when env->cfg.insn_state[w] == EXPLORED. | 77 | * Eventually this function would be called when env->cfg.insn_state[w] == EXPLORED. |
| 70 | * Rely on DFS traversal order and absence of recursive calls to guarantee that | 78 | * Rely on DFS traversal order and absence of recursive calls to guarantee that |
| 71 | * callee's change_pkt_data marks would be correct at that moment. | 79 | * callee's effect marks would be correct at that moment. |
| 72 | */ | 80 | */ |
| 73 | static void merge_callee_effects(struct bpf_verifier_env *env, int t, int w) | 81 | static void merge_callee_effects(struct bpf_verifier_env *env, int t, int w) |
| 74 | { | 82 | { |
| @@ -78,6 +86,7 @@ static void merge_callee_effects(struct bpf_verifier_env *env, int t, int w) · 3 unmodified lines | |||
| 78 | callee = bpf_find_containing_subprog(env, w); | 86 | callee = bpf_find_containing_subprog(env, w); |
| 79 | caller->changes_pkt_data |= callee->changes_pkt_data; | 87 | caller->changes_pkt_data |= callee->changes_pkt_data; |
| 80 | caller->might_sleep |= callee->might_sleep; | 88 | caller->might_sleep |= callee->might_sleep; |
| 89 | caller->might_throw |= callee->might_throw; | ||
| 81 | } | 90 | } |
| 82 | 91 | ||
| 83 | enum { | 92 | enum { |
| @@ -509,6 +518,8 @@ static int visit_insn(int t, struct bpf_verifier_env *env) · 425 unmodified lines | |||
| 509 | mark_subprog_might_sleep(env, t); | 518 | mark_subprog_might_sleep(env, t); |
| 510 | if (ret == 0 && bpf_is_kfunc_pkt_changing(&meta)) | 519 | if (ret == 0 && bpf_is_kfunc_pkt_changing(&meta)) |
| 511 | mark_subprog_changes_pkt_data(env, t); | 520 | mark_subprog_changes_pkt_data(env, t); |
| 521 | if (ret == 0 && bpf_is_throw_kfunc(insn)) | ||
| 522 | mark_subprog_might_throw(env, t); | ||
| 512 | } | 523 | } |
| 513 | return visit_func_call_insn(t, insns, env, insn->src_reg == BPF_PSEUDO_CALL); | 524 | return visit_func_call_insn(t, insns, env, insn->src_reg == BPF_PSEUDO_CALL); |
| 514 | 525 | ||
| @@ -442,7 +442,6 @@ static bool is_dynptr_ref_function(enum bpf_func_id func_id) · 441 unmodified lines | |||
| 442 | static bool is_sync_callback_calling_kfunc(u32 btf_id); | 442 | static bool is_sync_callback_calling_kfunc(u32 btf_id); |
| 443 | static bool is_async_callback_calling_kfunc(u32 btf_id); | 443 | static bool is_async_callback_calling_kfunc(u32 btf_id); |
| 444 | static bool is_callback_calling_kfunc(u32 btf_id); | 444 | static bool is_callback_calling_kfunc(u32 btf_id); |
| 445 | static bool is_bpf_throw_kfunc(struct bpf_insn *insn); | ||
| 446 | 445 | ||
| 447 | static bool is_bpf_wq_set_callback_kfunc(u32 btf_id); | 446 | static bool is_bpf_wq_set_callback_kfunc(u32 btf_id); |
| 448 | static bool is_task_work_add_kfunc(u32 func_id); | 447 | static bool is_task_work_add_kfunc(u32 func_id); |
| @@ -5405,7 +5404,7 @@ continue_func: · 4956 unmodified lines | |||
| 5405 | if (bpf_pseudo_kfunc_call(insn + i) && !insn[i].off) { | 5404 | if (bpf_pseudo_kfunc_call(insn + i) && !insn[i].off) { |
| 5406 | bool err = false; | 5405 | bool err = false; |
| 5407 | 5406 | ||
| 5408 | if (!is_bpf_throw_kfunc(insn + i)) | 5407 | if (!bpf_is_throw_kfunc(insn + i)) |
| 5409 | continue; | 5408 | continue; |
| 5410 | for (tmp = idx; tmp >= 0 && !err; tmp = dinfo[tmp].caller) { | 5409 | for (tmp = idx; tmp >= 0 && !err; tmp = dinfo[tmp].caller) { |
| 5411 | if (subprog[tmp].is_cb) { | 5410 | if (subprog[tmp].is_cb) { |
| @@ -9499,6 +9498,9 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins · 4087 unmodified lines | |||
| 9499 | return 0; | 9498 | return 0; |
| 9500 | } | 9499 | } |
| 9501 | 9500 | ||
| 9501 | static int process_bpf_exit_full(struct bpf_verifier_env *env, | ||
| 9502 | bool *do_print_state, bool exception_exit); | ||
| 9503 | |||
| 9502 | static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, | 9504 | static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, |
| 9503 | int *insn_idx) | 9505 | int *insn_idx) |
| 9504 | { | 9506 | { |
| @@ -9552,6 +9554,17 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · 47 unmodified lines | |||
| 9552 | caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; | 9554 | caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; |
| 9553 | } | 9555 | } |
| 9554 | 9556 | ||
| 9557 | if (env->subprog_info[subprog].might_throw) { | ||
| 9558 | struct bpf_verifier_state *branch; | ||
| 9559 | |||
| 9560 | branch = push_stack(env, *insn_idx + 1, *insn_idx, false); | ||
| 9561 | if (IS_ERR(branch)) { | ||
| 9562 | verbose(env, "failed to push state for global subprog exception path\n"); | ||
| 9563 | return PTR_ERR(branch); | ||
| 9564 | } | ||
| 9565 | return process_bpf_exit_full(env, NULL, true); | ||
| 9566 | } | ||
| 9567 | |||
| 9555 | /* continue with next insn after call */ | 9568 | /* continue with next insn after call */ |
| 9556 | return 0; | 9569 | return 0; |
| 9557 | } | 9570 | } |
| @@ -11782,7 +11795,7 @@ static bool is_async_callback_calling_kfunc(u32 btf_id) · 2224 unmodified lines | |||
| 11782 | is_task_work_add_kfunc(btf_id); | 11795 | is_task_work_add_kfunc(btf_id); |
| 11783 | } | 11796 | } |
| 11784 | 11797 | ||
| 11785 | static bool is_bpf_throw_kfunc(struct bpf_insn *insn) | 11798 | bool bpf_is_throw_kfunc(struct bpf_insn *insn) |
| 11786 | { | 11799 | { |
| 11787 | return bpf_pseudo_kfunc_call(insn) && insn->off == 0 && | 11800 | return bpf_pseudo_kfunc_call(insn) && insn->off == 0 && |
| 11788 | insn->imm == special_kfunc_list[KF_bpf_throw]; | 11801 | insn->imm == special_kfunc_list[KF_bpf_throw]; |
| @@ -12972,8 +12985,6 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_ca · 1183 unmodified lines | |||
| 12972 | } | 12985 | } |
| 12973 | 12986 | ||
| 12974 | static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name); | 12987 | static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name); |
| 12975 | static int process_bpf_exit_full(struct bpf_verifier_env *env, | ||
| 12976 | bool *do_print_state, bool exception_exit); | ||
| 12977 | 12988 | ||
| 12978 | static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, | 12989 | static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, |
| 12979 | int *insn_idx_p) | 12990 | int *insn_idx_p) |
| @@ -13354,7 +13365,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · 374 unmodified lines | |||
| 13354 | if (meta.func_id == special_kfunc_list[KF_bpf_session_cookie]) | 13365 | if (meta.func_id == special_kfunc_list[KF_bpf_session_cookie]) |
| 13355 | env->prog->call_session_cookie = true; | 13366 | env->prog->call_session_cookie = true; |
| 13356 | 13367 | ||
| 13357 | if (is_bpf_throw_kfunc(insn)) | 13368 | if (bpf_is_throw_kfunc(insn)) |
| 13358 | return process_bpf_exit_full(env, NULL, true); | 13369 | return process_bpf_exit_full(env, NULL, true); |
| 13359 | 13370 | ||
| 13360 | return 0; | 13371 | return 0; |