bpf_convert_ctx_accesses() turns a BPF_LDX into a BPF_PROBE_MEM one by matching the type recorded for the insn against a list of exact pointer types. The list cannot keep up with the flag combinations the verifier produces, and a type which is missing from it ends up as a plain load without an exception table entry, so a bad address panics the kernel instead of being handled. Two such types exist today and are reachable: - PTR_TO_BTF_ID | PTR_UNTRUSTED | MEM_ALLOC | NON_OWN_REF - PTR_TO_BTF_ID | PTR_UNTRUSTED | MEM_RCU Rather than adding the two, just drop the list and state the property itself in the default case of the switch. This is a superset of what the list matched, the untrusted PTR_TO_MEM does not have to carry MEM_RDONLY for it anymore, and it stays in sync with the verifier side which uses the same match in save_aux_ptr_type() and reg_type_mismatch_ok(). Assert that a fault prone type which does not get the rewrite for whatever reason is rejected at load time rather than left to fault at runtime to catch any future cases.
Unified · Split
| @@ -1308,6 +1308,17 @@ static inline u32 type_flag(u32 type) · 1307 unmodified lines | ||
| 1308 | 1308 | return type & ~BPF_BASE_TYPE_MASK; |
| 1309 | 1309 | } |
| 1310 | 1310 | |
| 1311 | +static inline bool bpf_is_ptr_to_mem_or_btf_id(enum bpf_reg_type type) | |
| 1312 | +{ | |
| 1313 | + switch (base_type(type)) { | |
| 1314 | + case PTR_TO_MEM: | |
| 1315 | + case PTR_TO_BTF_ID: | |
| 1316 | + return true; | |
| 1317 | + default: | |
| 1318 | + return false; | |
| 1319 | + } | |
| 1320 | +} | |
| 1321 | + | |
| 1311 | 1322 | static inline bool bpf_may_fault_on_deref(enum bpf_reg_type type) |
| 1312 | 1323 | { |
| 1313 | 1324 | /* |
| @@ -812,6 +812,7 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) · 811 unmodified lines | ||
| 812 | 812 | |
| 813 | 813 | for (i = 0; i < insn_cnt; i++, insn++) { |
| 814 | 814 | bpf_convert_ctx_access_t convert_ctx_access; |
| 815 | + enum bpf_reg_type ptr_type; | |
| 815 | 816 | u8 mode; |
| 816 | 817 | |
| 817 | 818 | if (env->insn_aux_data[i + delta].nospec) { |
| @@ -904,7 +905,8 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) · 86 unmodified lines | ||
| 904 | 905 | continue; |
| 905 | 906 | } |
| 906 | 907 | |
| 907 | − switch ((int)env->insn_aux_data[i + delta].ptr_type) { | |
| 908 | + ptr_type = env->insn_aux_data[i + delta].ptr_type; | |
| 909 | + switch ((int)ptr_type) { | |
| 908 | 910 | case PTR_TO_CTX: |
| 909 | 911 | if (!ops->convert_ctx_access) |
| 910 | 912 | continue; |
| @@ -920,26 +922,6 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) · 9 unmodified lines | ||
| 920 | 922 | case PTR_TO_XDP_SOCK: |
| 921 | 923 | convert_ctx_access = bpf_xdp_sock_convert_ctx_access; |
| 922 | 924 | break; |
| 923 | − case PTR_TO_BTF_ID: | |
| 924 | − case PTR_TO_BTF_ID | PTR_UNTRUSTED: | |
| 925 | − /* PTR_TO_BTF_ID | MEM_ALLOC always has a valid lifetime, unlike | |
| 926 | − * PTR_TO_BTF_ID, and an active referenced id, but the same cannot | |
| 927 | − * be said once it is marked PTR_UNTRUSTED, hence we must handle | |
| 928 | − * any faults for loads into such types. BPF_WRITE is disallowed | |
| 929 | − * for this case. | |
| 930 | − */ | |
| 931 | − case PTR_TO_BTF_ID | MEM_ALLOC | PTR_UNTRUSTED: | |
| 932 | − case PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED: | |
| 933 | − if (type == BPF_READ) { | |
| 934 | − if (BPF_MODE(insn->code) == BPF_MEM) | |
| 935 | − insn->code = BPF_LDX | BPF_PROBE_MEM | | |
| 936 | − BPF_SIZE((insn)->code); | |
| 937 | − else | |
| 938 | − insn->code = BPF_LDX | BPF_PROBE_MEMSX | | |
| 939 | − BPF_SIZE((insn)->code); | |
| 940 | − env->prog->aux->num_exentries++; | |
| 941 | − } | |
| 942 | − continue; | |
| 943 | 925 | case PTR_TO_ARENA: |
| 944 | 926 | if (BPF_MODE(insn->code) == BPF_MEMSX) { |
| 945 | 927 | if (!bpf_jit_supports_insn(insn, true)) { |
| @@ -953,6 +935,29 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) · 7 unmodified lines | ||
| 953 | 935 | env->prog->aux->num_exentries++; |
| 954 | 936 | continue; |
| 955 | 937 | default: |
| 938 | + /* | |
| 939 | + * A pointer which may fault on a dereference must not | |
| 940 | + * be loaded from without fault protection, hence turn | |
| 941 | + * the BPF_LDX into a BPF_PROBE_MEM one so that a bad | |
| 942 | + * address is handled rather than panicking the kernel. | |
| 943 | + * A store through one is rejected earlier, there is no | |
| 944 | + * probed counterpart to rewrite it into. | |
| 945 | + */ | |
| 946 | + if (bpf_is_ptr_to_mem_or_btf_id(ptr_type) && | |
| 947 | + bpf_may_fault_on_deref(ptr_type) && | |
| 948 | + type == BPF_READ) { | |
| 949 | + if (BPF_MODE(insn->code) == BPF_MEM) | |
| 950 | + insn->code = BPF_LDX | BPF_PROBE_MEM | | |
| 951 | + BPF_SIZE(insn->code); | |
| 952 | + else | |
| 953 | + insn->code = BPF_LDX | BPF_PROBE_MEMSX | | |
| 954 | + BPF_SIZE(insn->code); | |
| 955 | + env->prog->aux->num_exentries++; | |
| 956 | + continue; | |
| 957 | + } | |
| 958 | + if (verifier_bug_if(bpf_may_fault_on_deref(ptr_type), env, | |
| 959 | + "access to a fault prone pointer is not rewritten as a probed one")) | |
| 960 | + return -EFAULT; | |
| 956 | 961 | continue; |
| 957 | 962 | } |
| 958 | 963 | |
| @@ -17840,17 +17840,6 @@ static bool reg_type_mismatch(enum bpf_reg_type src, enum bpf_reg_type prev) · 17839 unmodified lines | ||
| 17840 | 17840 | !reg_type_mismatch_ok(prev)); |
| 17841 | 17841 | } |
| 17842 | 17842 | |
| 17843 | −static bool is_ptr_to_mem_or_btf_id(enum bpf_reg_type type) | |
| 17844 | −{ | |
| 17845 | − switch (base_type(type)) { | |
| 17846 | − case PTR_TO_MEM: | |
| 17847 | − case PTR_TO_BTF_ID: | |
| 17848 | − return true; | |
| 17849 | − default: | |
| 17850 | − return false; | |
| 17851 | − } | |
| 17852 | −} | |
| 17853 | − | |
| 17854 | 17843 | static bool is_ptr_to_mem(enum bpf_reg_type type) |
| 17855 | 17844 | { |
| 17856 | 17845 | return base_type(type) == PTR_TO_MEM; |
| @@ -17890,8 +17879,8 @@ static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type typ · 33 unmodified lines | ||
| 17890 | 17879 | * Reject it. |
| 17891 | 17880 | */ |
| 17892 | 17881 | if (allow_trust_mismatch && |
| 17893 | − is_ptr_to_mem_or_btf_id(type) && | |
| 17894 | − is_ptr_to_mem_or_btf_id(*prev_type)) { | |
| 17882 | + bpf_is_ptr_to_mem_or_btf_id(type) && | |
| 17883 | + bpf_is_ptr_to_mem_or_btf_id(*prev_type)) { | |
| 17895 | 17884 | /* |
| 17896 | 17885 | * Have to support a use case when one path through |
| 17897 | 17886 | * the program yields a TRUSTED pointer while another |