Global subprogs are verified independently and are not descended into when their callers are symbolically executed. This means a caller can hold references or locks across a global subprog call that may throw, while the verifier only checks the non-exceptional return path at the call site. Record whether a subprog might throw in the CFG summary pass, alongside the existing might_sleep and packet-data-changing summaries, and propagate that effect through reachable callees. When a global subprog is marked as possibly throwing, push the normal continuation and validate the exceptional path immediately at the call site, avoiding a synthetic exception state and associated special case in the pruning checks.
Unified · Split
| @@ -729,6 +729,7 @@ struct bpf_subprog_info { · 728 unmodified lines | ||
| 729 | 729 | */ |
| 730 | 730 | s16 fastcall_stack_off; |
| 731 | 731 | bool has_tail_call: 1; |
| 732 | + bool might_throw: 1; | |
| 732 | 733 | bool tail_call_reachable: 1; |
| 733 | 734 | bool has_ld_abs: 1; |
| 734 | 735 | bool is_cb: 1; |
| @@ -1308,6 +1309,7 @@ void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask); · 573 unmodified lines | ||
| 1308 | 1309 | bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog); |
| 1309 | 1310 | |
| 1310 | 1311 | int bpf_find_subprog(struct bpf_verifier_env *env, int off); |
| 1312 | +bool bpf_is_throw_kfunc(struct bpf_insn *insn); | |
| 1311 | 1313 | int bpf_compute_const_regs(struct bpf_verifier_env *env); |
| 1312 | 1314 | int bpf_prune_dead_branches(struct bpf_verifier_env *env); |
| 1313 | 1315 | int bpf_check_cfg(struct bpf_verifier_env *env); |
| @@ -64,11 +64,19 @@ static void mark_subprog_might_sleep(struct bpf_verifier_env *env, int off) · 63 unmodified lines | ||
| 64 | 64 | subprog->might_sleep = true; |
| 65 | 65 | } |
| 66 | 66 | |
| 67 | +static void mark_subprog_might_throw(struct bpf_verifier_env *env, int off) | |
| 68 | +{ | |
| 69 | + struct bpf_subprog_info *subprog; | |
| 70 | + | |
| 71 | + subprog = bpf_find_containing_subprog(env, off); | |
| 72 | + subprog->might_throw = true; | |
| 73 | +} | |
| 74 | + | |
| 67 | 75 | /* 't' is an index of a call-site. |
| 68 | 76 | * 'w' is a callee entry point. |
| 69 | 77 | * Eventually this function would be called when env->cfg.insn_state[w] == EXPLORED. |
| 70 | 78 | * Rely on DFS traversal order and absence of recursive calls to guarantee that |
| 71 | − * callee's change_pkt_data marks would be correct at that moment. | |
| 79 | + * callee's effect marks would be correct at that moment. | |
| 72 | 80 | */ |
| 73 | 81 | static void merge_callee_effects(struct bpf_verifier_env *env, int t, int w) |
| 74 | 82 | { |
| @@ -78,6 +86,7 @@ static void merge_callee_effects(struct bpf_verifier_env *env, int t, int w) · 3 unmodified lines | ||
| 78 | 86 | callee = bpf_find_containing_subprog(env, w); |
| 79 | 87 | caller->changes_pkt_data |= callee->changes_pkt_data; |
| 80 | 88 | caller->might_sleep |= callee->might_sleep; |
| 89 | + caller->might_throw |= callee->might_throw; | |
| 81 | 90 | } |
| 82 | 91 | |
| 83 | 92 | enum { |
| @@ -509,6 +518,8 @@ static int visit_insn(int t, struct bpf_verifier_env *env) · 425 unmodified lines | ||
| 509 | 518 | mark_subprog_might_sleep(env, t); |
| 510 | 519 | if (ret == 0 && bpf_is_kfunc_pkt_changing(&meta)) |
| 511 | 520 | mark_subprog_changes_pkt_data(env, t); |
| 521 | + if (ret == 0 && bpf_is_throw_kfunc(insn)) | |
| 522 | + mark_subprog_might_throw(env, t); | |
| 512 | 523 | } |
| 513 | 524 | return visit_func_call_insn(t, insns, env, insn->src_reg == BPF_PSEUDO_CALL); |
| 514 | 525 | |
| @@ -442,7 +442,6 @@ static bool is_dynptr_ref_function(enum bpf_func_id func_id) · 441 unmodified lines | ||
| 442 | 442 | static bool is_sync_callback_calling_kfunc(u32 btf_id); |
| 443 | 443 | static bool is_async_callback_calling_kfunc(u32 btf_id); |
| 444 | 444 | static bool is_callback_calling_kfunc(u32 btf_id); |
| 445 | −static bool is_bpf_throw_kfunc(struct bpf_insn *insn); | |
| 446 | 445 | |
| 447 | 446 | static bool is_bpf_wq_set_callback_kfunc(u32 btf_id); |
| 448 | 447 | static bool is_task_work_add_kfunc(u32 func_id); |
| @@ -5405,7 +5404,7 @@ continue_func: · 4956 unmodified lines | ||
| 5405 | 5404 | if (bpf_pseudo_kfunc_call(insn + i) && !insn[i].off) { |
| 5406 | 5405 | bool err = false; |
| 5407 | 5406 | |
| 5408 | − if (!is_bpf_throw_kfunc(insn + i)) | |
| 5407 | + if (!bpf_is_throw_kfunc(insn + i)) | |
| 5409 | 5408 | continue; |
| 5410 | 5409 | for (tmp = idx; tmp >= 0 && !err; tmp = dinfo[tmp].caller) { |
| 5411 | 5410 | if (subprog[tmp].is_cb) { |
| @@ -9499,6 +9498,9 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins · 4087 unmodified lines | ||
| 9499 | 9498 | return 0; |
| 9500 | 9499 | } |
| 9501 | 9500 | |
| 9501 | +static int process_bpf_exit_full(struct bpf_verifier_env *env, | |
| 9502 | + bool *do_print_state, bool exception_exit); | |
| 9503 | + | |
| 9502 | 9504 | static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, |
| 9503 | 9505 | int *insn_idx) |
| 9504 | 9506 | { |
| @@ -9552,6 +9554,17 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · 47 unmodified lines | ||
| 9552 | 9554 | caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; |
| 9553 | 9555 | } |
| 9554 | 9556 | |
| 9557 | + if (env->subprog_info[subprog].might_throw) { | |
| 9558 | + struct bpf_verifier_state *branch; | |
| 9559 | + | |
| 9560 | + branch = push_stack(env, *insn_idx + 1, *insn_idx, false); | |
| 9561 | + if (IS_ERR(branch)) { | |
| 9562 | + verbose(env, "failed to push state for global subprog exception path\n"); | |
| 9563 | + return PTR_ERR(branch); | |
| 9564 | + } | |
| 9565 | + return process_bpf_exit_full(env, NULL, true); | |
| 9566 | + } | |
| 9567 | + | |
| 9555 | 9568 | /* continue with next insn after call */ |
| 9556 | 9569 | return 0; |
| 9557 | 9570 | } |
| @@ -11782,7 +11795,7 @@ static bool is_async_callback_calling_kfunc(u32 btf_id) · 2224 unmodified lines | ||
| 11782 | 11795 | is_task_work_add_kfunc(btf_id); |
| 11783 | 11796 | } |
| 11784 | 11797 | |
| 11785 | −static bool is_bpf_throw_kfunc(struct bpf_insn *insn) | |
| 11798 | +bool bpf_is_throw_kfunc(struct bpf_insn *insn) | |
| 11786 | 11799 | { |
| 11787 | 11800 | return bpf_pseudo_kfunc_call(insn) && insn->off == 0 && |
| 11788 | 11801 | insn->imm == special_kfunc_list[KF_bpf_throw]; |
| @@ -12972,8 +12985,6 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_ca · 1183 unmodified lines | ||
| 12972 | 12985 | } |
| 12973 | 12986 | |
| 12974 | 12987 | static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name); |
| 12975 | −static int process_bpf_exit_full(struct bpf_verifier_env *env, | |
| 12976 | − bool *do_print_state, bool exception_exit); | |
| 12977 | 12988 | |
| 12978 | 12989 | static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, |
| 12979 | 12990 | int *insn_idx_p) |
| @@ -13354,7 +13365,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · 374 unmodified lines | ||
| 13354 | 13365 | if (meta.func_id == special_kfunc_list[KF_bpf_session_cookie]) |
| 13355 | 13366 | env->prog->call_session_cookie = true; |
| 13356 | 13367 | |
| 13357 | − if (is_bpf_throw_kfunc(insn)) | |
| 13368 | + if (bpf_is_throw_kfunc(insn)) | |
| 13358 | 13369 | return process_bpf_exit_full(env, NULL, true); |
| 13359 | 13370 | |
| 13360 | 13371 | return 0; |