This patch introduces non-owning reference semantics to the verifier, specifically linked_list API kfunc handling. release_on_unlock logic for refs is refactored - with small functional changes - to implement these semantics, and bpf_list_push_{front,back} are migrated to use them. When a list node is pushed to a list, the program still has a pointer to the node: n = bpf_obj_new(typeof(*n)); bpf_spin_lock(&l); bpf_list_push_back(&l, n); /* n still points to the just-added node */ bpf_spin_unlock(&l); What the verifier considers n to be after the push, and thus what can be done with n, are changed by this patch. Common properties both before/after this patch: * After push, n is only a valid reference to the node until end of critical section * After push, n cannot be pushed to any list * After push, the program can read the node's fields using n Before: * After push, n retains the ref_obj_id which it received on bpf_obj_new, but the associated bpf_reference_state's release_on_unlock field is set to true * release_on_unlock field and associated logic is used to implement "n is only a valid ref until end of critical section" * After push, n cannot be written to, the node must be removed from the list before writing to its fields * After push, n is marked PTR_UNTRUSTED After: * After push, n's ref is released and ref_obj_id set to 0. NON_OWN_REF type flag is added to reg's type, indicating that it's a non-owning reference. * NON_OWN_REF flag and logic is used to implement "n is only a valid ref until end of critical section" * n can be written to (except for special fields e.g. bpf_list_node, timer, ...) Summary of specific implementation changes to achieve the above: * release_on_unlock field, ref_set_release_on_unlock helper, and logic to "release on unlock" based on that field are removed * The anonymous active_lock struct used by bpf_verifier_state is pulled out into a named struct bpf_active_lock. * NON_OWN_REF type flag is introduced along with verifier logic changes to handle non-owning refs * Helpers are added to use NON_OWN_REF flag to implement non-owning ref semantics as described above * invalidate_non_owning_refs - helper to clobber all non-owning refs matching a particular bpf_active_lock identity. Replaces release_on_unlock logic in process_spin_lock. * ref_set_non_owning - set NON_OWN_REF type flag after doing some sanity checking * ref_convert_owning_non_owning - convert owning reference w/ specified ref_obj_id to non-owning references. Set NON_OWN_REF flag for each reg with that ref_obj_id and 0-out its ref_obj_id * Update linked_list selftests to account for minor semantic differences introduced by this patch * Writes to a release_on_unlock node ref are not allowed, while writes to non-owning reference pointees are. As a result the linked_list "write after push" failure tests are no longer scenarios that should fail. * The test##missing_lock##op and test##incorrect_lock##op macro-generated failure tests need to have a valid node argument in order to have the same error output as before. Otherwise verification will fail early and the expected error output won't be seen.
Unified · Split
| @@ -181,6 +181,7 @@ enum btf_field_type { · 180 unmodified lines | ||
| 181 | 181 | BPF_KPTR = BPF_KPTR_UNREF | BPF_KPTR_REF, |
| 182 | 182 | BPF_LIST_HEAD = (1 << 4), |
| 183 | 183 | BPF_LIST_NODE = (1 << 5), |
| 184 | + BPF_GRAPH_NODE_OR_ROOT = BPF_LIST_NODE | BPF_LIST_HEAD, | |
| 184 | 185 | }; |
| 185 | 186 | |
| 186 | 187 | struct btf_field_kptr { |
| @@ -576,6 +577,11 @@ enum bpf_type_flag { · 389 unmodified lines | ||
| 576 | 577 | /* MEM is tagged with rcu and memory access needs rcu_read_lock protection. */ |
| 577 | 578 | MEM_RCU = BIT(13 + BPF_BASE_TYPE_BITS), |
| 578 | 579 | |
| 580 | + /* Used to tag PTR_TO_BTF_ID | MEM_ALLOC references which are non-owning. | |
| 581 | + * Currently only valid for linked-list and rbtree nodes. | |
| 582 | + */ | |
| 583 | + NON_OWN_REF = BIT(14 + BPF_BASE_TYPE_BITS), | |
| 584 | + | |
| 579 | 585 | __BPF_TYPE_FLAG_MAX, |
| 580 | 586 | __BPF_TYPE_LAST_FLAG = __BPF_TYPE_FLAG_MAX - 1, |
| 581 | 587 | }; |
| @@ -43,6 +43,22 @@ enum bpf_reg_liveness { · 42 unmodified lines | ||
| 43 | 43 | REG_LIVE_DONE = 0x8, /* liveness won't be updating this register anymore */ |
| 44 | 44 | }; |
| 45 | 45 | |
| 46 | +/* For every reg representing a map value or allocated object pointer, | |
| 47 | + * we consider the tuple of (ptr, id) for them to be unique in verifier | |
| 48 | + * context and conside them to not alias each other for the purposes of | |
| 49 | + * tracking lock state. | |
| 50 | + */ | |
| 51 | +struct bpf_active_lock { | |
| 52 | + /* This can either be reg->map_ptr or reg->btf. If ptr is NULL, | |
| 53 | + * there's no active lock held, and other fields have no | |
| 54 | + * meaning. If non-NULL, it indicates that a lock is held and | |
| 55 | + * id member has the reg->id of the register which can be >= 0. | |
| 56 | + */ | |
| 57 | + void *ptr; | |
| 58 | + /* This will be reg->id */ | |
| 59 | + u32 id; | |
| 60 | +}; | |
| 61 | + | |
| 46 | 62 | struct bpf_reg_state { |
| 47 | 63 | /* Ordering of fields matters. See states_equal() */ |
| 48 | 64 | enum bpf_reg_type type; |
| @@ -226,11 +242,6 @@ struct bpf_reference_state { · 177 unmodified lines | ||
| 226 | 242 | * exiting a callback function. |
| 227 | 243 | */ |
| 228 | 244 | int callback_ref; |
| 229 | − /* Mark the reference state to release the registers sharing the same id | |
| 230 | − * on bpf_spin_unlock (for nodes that we will lose ownership to but are | |
| 231 | − * safe to access inside the critical section). | |
| 232 | − */ | |
| 233 | − bool release_on_unlock; | |
| 234 | 245 | }; |
| 235 | 246 | |
| 236 | 247 | /* state of the program: |
| @@ -331,21 +342,8 @@ struct bpf_verifier_state { · 94 unmodified lines | ||
| 331 | 342 | u32 branches; |
| 332 | 343 | u32 insn_idx; |
| 333 | 344 | u32 curframe; |
| 334 | − /* For every reg representing a map value or allocated object pointer, | |
| 335 | − * we consider the tuple of (ptr, id) for them to be unique in verifier | |
| 336 | − * context and conside them to not alias each other for the purposes of | |
| 337 | − * tracking lock state. | |
| 338 | − */ | |
| 339 | − struct { | |
| 340 | − /* This can either be reg->map_ptr or reg->btf. If ptr is NULL, | |
| 341 | − * there's no active lock held, and other fields have no | |
| 342 | − * meaning. If non-NULL, it indicates that a lock is held and | |
| 343 | − * id member has the reg->id of the register which can be >= 0. | |
| 344 | − */ | |
| 345 | − void *ptr; | |
| 346 | − /* This will be reg->id */ | |
| 347 | − u32 id; | |
| 348 | − } active_lock; | |
| 345 | + | |
| 346 | + struct bpf_active_lock active_lock; | |
| 349 | 347 | bool speculative; |
| 350 | 348 | bool active_rcu_lock; |
| 351 | 349 | |
| @@ -190,6 +190,9 @@ struct bpf_verifier_stack_elem { · 189 unmodified lines | ||
| 190 | 190 | |
| 191 | 191 | static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx); |
| 192 | 192 | static int release_reference(struct bpf_verifier_env *env, int ref_obj_id); |
| 193 | +static void invalidate_non_owning_refs(struct bpf_verifier_env *env); | |
| 194 | +static int ref_set_non_owning(struct bpf_verifier_env *env, | |
| 195 | + struct bpf_reg_state *reg); | |
| 193 | 196 | |
| 194 | 197 | static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux) |
| 195 | 198 | { |
| @@ -457,6 +460,11 @@ static bool type_is_ptr_alloc_obj(u32 type) · 261 unmodified lines | ||
| 457 | 460 | return base_type(type) == PTR_TO_BTF_ID && type_flag(type) & MEM_ALLOC; |
| 458 | 461 | } |
| 459 | 462 | |
| 463 | +static bool type_is_non_owning_ref(u32 type) | |
| 464 | +{ | |
| 465 | + return type_is_ptr_alloc_obj(type) && type_flag(type) & NON_OWN_REF; | |
| 466 | +} | |
| 467 | + | |
| 460 | 468 | static struct btf_record *reg_btf_record(const struct bpf_reg_state *reg) |
| 461 | 469 | { |
| 462 | 470 | struct btf_record *rec = NULL; |
| @@ -1073,6 +1081,8 @@ static void print_verifier_state(struct bpf_verifier_env *env, · 610 unmodified lines | ||
| 1073 | 1081 | verbose_a("id=%d", reg->id); |
| 1074 | 1082 | if (reg->ref_obj_id) |
| 1075 | 1083 | verbose_a("ref_obj_id=%d", reg->ref_obj_id); |
| 1084 | + if (type_is_non_owning_ref(reg->type)) | |
| 1085 | + verbose_a("%s", "non_own_ref"); | |
| 1076 | 1086 | if (t != SCALAR_VALUE) |
| 1077 | 1087 | verbose_a("off=%d", reg->off); |
| 1078 | 1088 | if (type_is_pkt_pointer(t)) |
| @@ -5052,7 +5062,8 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, · 3973 unmodified lines | ||
| 5052 | 5062 | return -EACCES; |
| 5053 | 5063 | } |
| 5054 | 5064 | |
| 5055 | − if (type_is_alloc(reg->type) && !reg->ref_obj_id) { | |
| 5065 | + if (type_is_alloc(reg->type) && !type_is_non_owning_ref(reg->type) && | |
| 5066 | + !reg->ref_obj_id) { | |
| 5056 | 5067 | verbose(env, "verifier internal error: ref_obj_id for allocated object must be non-zero\n"); |
| 5057 | 5068 | return -EFAULT; |
| 5058 | 5069 | } |
| @@ -6042,9 +6053,7 @@ static int process_spin_lock(struct bpf_verifier_env *env, int regno, · 983 unmodified lines | ||
| 6042 | 6053 | cur->active_lock.ptr = btf; |
| 6043 | 6054 | cur->active_lock.id = reg->id; |
| 6044 | 6055 | } else { |
| 6045 | − struct bpf_func_state *fstate = cur_func(env); | |
| 6046 | 6056 | void *ptr; |
| 6047 | − int i; | |
| 6048 | 6057 | |
| 6049 | 6058 | if (map) |
| 6050 | 6059 | ptr = map; |
| @@ -6060,25 +6069,11 @@ static int process_spin_lock(struct bpf_verifier_env *env, int regno, · 9 unmodified lines | ||
| 6060 | 6069 | verbose(env, "bpf_spin_unlock of different lock\n"); |
| 6061 | 6070 | return -EINVAL; |
| 6062 | 6071 | } |
| 6063 | − cur->active_lock.ptr = NULL; | |
| 6064 | − cur->active_lock.id = 0; | |
| 6065 | 6072 | |
| 6066 | − for (i = fstate->acquired_refs - 1; i >= 0; i--) { | |
| 6067 | − int err; | |
| 6073 | + invalidate_non_owning_refs(env); | |
| 6068 | 6074 | |
| 6069 | − /* Complain on error because this reference state cannot | |
| 6070 | − * be freed before this point, as bpf_spin_lock critical | |
| 6071 | − * section does not allow functions that release the | |
| 6072 | − * allocated object immediately. | |
| 6073 | − */ | |
| 6074 | − if (!fstate->refs[i].release_on_unlock) | |
| 6075 | − continue; | |
| 6076 | − err = release_reference(env, fstate->refs[i].id); | |
| 6077 | − if (err) { | |
| 6078 | − verbose(env, "failed to release release_on_unlock reference"); | |
| 6079 | − return err; | |
| 6080 | − } | |
| 6081 | − } | |
| 6075 | + cur->active_lock.ptr = NULL; | |
| 6076 | + cur->active_lock.id = 0; | |
| 6082 | 6077 | } |
| 6083 | 6078 | return 0; |
| 6084 | 6079 | } |
| @@ -6546,6 +6541,23 @@ found: · 461 unmodified lines | ||
| 6546 | 6541 | return 0; |
| 6547 | 6542 | } |
| 6548 | 6543 | |
| 6544 | +static struct btf_field * | |
| 6545 | +reg_find_field_offset(const struct bpf_reg_state *reg, s32 off, u32 fields) | |
| 6546 | +{ | |
| 6547 | + struct btf_field *field; | |
| 6548 | + struct btf_record *rec; | |
| 6549 | + | |
| 6550 | + rec = reg_btf_record(reg); | |
| 6551 | + if (!rec) | |
| 6552 | + return NULL; | |
| 6553 | + | |
| 6554 | + field = btf_record_find(rec, off, fields); | |
| 6555 | + if (!field) | |
| 6556 | + return NULL; | |
| 6557 | + | |
| 6558 | + return field; | |
| 6559 | +} | |
| 6560 | + | |
| 6549 | 6561 | int check_func_arg_reg_off(struct bpf_verifier_env *env, |
| 6550 | 6562 | const struct bpf_reg_state *reg, int regno, |
| 6551 | 6563 | enum bpf_arg_type arg_type) |
| @@ -6567,6 +6579,18 @@ int check_func_arg_reg_off(struct bpf_verifier_env *env, · 15 unmodified lines | ||
| 6567 | 6579 | */ |
| 6568 | 6580 | if (arg_type_is_dynptr(arg_type) && type == PTR_TO_STACK) |
| 6569 | 6581 | return 0; |
| 6582 | + | |
| 6583 | + if ((type_is_ptr_alloc_obj(type) || type_is_non_owning_ref(type)) && reg->off) { | |
| 6584 | + if (reg_find_field_offset(reg, reg->off, BPF_GRAPH_NODE_OR_ROOT)) | |
| 6585 | + return __check_ptr_off_reg(env, reg, regno, true); | |
| 6586 | + | |
| 6587 | + verbose(env, "R%d must have zero offset when passed to release func\n", | |
| 6588 | + regno); | |
| 6589 | + verbose(env, "No graph node or root found at R%d type:%s off:%d\n", regno, | |
| 6590 | + kernel_type_name(reg->btf, reg->btf_id), reg->off); | |
| 6591 | + return -EINVAL; | |
| 6592 | + } | |
| 6593 | + | |
| 6570 | 6594 | /* Doing check_ptr_off_reg check for the offset will catch this |
| 6571 | 6595 | * because fixed_off_ok is false, but checking here allows us |
| 6572 | 6596 | * to give the user a better error message. |
| @@ -6601,6 +6625,7 @@ int check_func_arg_reg_off(struct bpf_verifier_env *env, · 28 unmodified lines | ||
| 6601 | 6625 | case PTR_TO_BTF_ID | PTR_TRUSTED: |
| 6602 | 6626 | case PTR_TO_BTF_ID | MEM_RCU: |
| 6603 | 6627 | case PTR_TO_BTF_ID | MEM_ALLOC | PTR_TRUSTED: |
| 6628 | + case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF: | |
| 6604 | 6629 | /* When referenced PTR_TO_BTF_ID is passed to release function, |
| 6605 | 6630 | * its fixed offset must be 0. In the other cases, fixed offset |
| 6606 | 6631 | * can be non-zero. This was already checked above. So pass |
| @@ -7363,6 +7388,17 @@ static int release_reference(struct bpf_verifier_env *env, · 756 unmodified lines | ||
| 7363 | 7388 | return 0; |
| 7364 | 7389 | } |
| 7365 | 7390 | |
| 7391 | +static void invalidate_non_owning_refs(struct bpf_verifier_env *env) | |
| 7392 | +{ | |
| 7393 | + struct bpf_func_state *unused; | |
| 7394 | + struct bpf_reg_state *reg; | |
| 7395 | + | |
| 7396 | + bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({ | |
| 7397 | + if (type_is_non_owning_ref(reg->type)) | |
| 7398 | + __mark_reg_unknown(env, reg); | |
| 7399 | + })); | |
| 7400 | +} | |
| 7401 | + | |
| 7366 | 7402 | static void clear_caller_saved_regs(struct bpf_verifier_env *env, |
| 7367 | 7403 | struct bpf_reg_state *regs) |
| 7368 | 7404 | { |
| @@ -8915,38 +8951,54 @@ static int process_kf_arg_ptr_to_kptr(struct bpf_verifier_env *env, · 1546 unmodified lines | ||
| 8915 | 8951 | return 0; |
| 8916 | 8952 | } |
| 8917 | 8953 | |
| 8918 | −static int ref_set_release_on_unlock(struct bpf_verifier_env *env, u32 ref_obj_id) | |
| 8954 | +static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state *reg) | |
| 8919 | 8955 | { |
| 8920 | − struct bpf_func_state *state = cur_func(env); | |
| 8956 | + struct bpf_verifier_state *state = env->cur_state; | |
| 8957 | + | |
| 8958 | + if (!state->active_lock.ptr) { | |
| 8959 | + verbose(env, "verifier internal error: ref_set_non_owning w/o active lock\n"); | |
| 8960 | + return -EFAULT; | |
| 8961 | + } | |
| 8962 | + | |
| 8963 | + if (type_flag(reg->type) & NON_OWN_REF) { | |
| 8964 | + verbose(env, "verifier internal error: NON_OWN_REF already set\n"); | |
| 8965 | + return -EFAULT; | |
| 8966 | + } | |
| 8967 | + | |
| 8968 | + reg->type |= NON_OWN_REF; | |
| 8969 | + return 0; | |
| 8970 | +} | |
| 8971 | + | |
| 8972 | +static int ref_convert_owning_non_owning(struct bpf_verifier_env *env, u32 ref_obj_id) | |
| 8973 | +{ | |
| 8974 | + struct bpf_func_state *state, *unused; | |
| 8921 | 8975 | struct bpf_reg_state *reg; |
| 8922 | 8976 | int i; |
| 8923 | 8977 | |
| 8924 | − /* bpf_spin_lock only allows calling list_push and list_pop, no BPF | |
| 8925 | − * subprogs, no global functions. This means that the references would | |
| 8926 | − * not be released inside the critical section but they may be added to | |
| 8927 | − * the reference state, and the acquired_refs are never copied out for a | |
| 8928 | − * different frame as BPF to BPF calls don't work in bpf_spin_lock | |
| 8929 | − * critical sections. | |
| 8930 | − */ | |
| 8978 | + state = cur_func(env); | |
| 8979 | + | |
| 8931 | 8980 | if (!ref_obj_id) { |
| 8932 | − verbose(env, "verifier internal error: ref_obj_id is zero for release_on_unlock\n"); | |
| 8981 | + verbose(env, "verifier internal error: ref_obj_id is zero for " | |
| 8982 | + "owning -> non-owning conversion\n"); | |
| 8933 | 8983 | return -EFAULT; |
| 8934 | 8984 | } |
| 8985 | + | |
| 8935 | 8986 | for (i = 0; i < state->acquired_refs; i++) { |
| 8936 | − if (state->refs[i].id == ref_obj_id) { | |
| 8937 | − if (state->refs[i].release_on_unlock) { | |
| 8938 | − verbose(env, "verifier internal error: expected false release_on_unlock"); | |
| 8939 | − return -EFAULT; | |
| 8987 | + if (state->refs[i].id != ref_obj_id) | |
| 8988 | + continue; | |
| 8989 | + | |
| 8990 | + /* Clear ref_obj_id here so release_reference doesn't clobber | |
| 8991 | + * the whole reg | |
| 8992 | + */ | |
| 8993 | + bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({ | |
| 8994 | + if (reg->ref_obj_id == ref_obj_id) { | |
| 8995 | + reg->ref_obj_id = 0; | |
| 8996 | + ref_set_non_owning(env, reg); | |
| 8940 | 8997 | } |
| 8941 | − state->refs[i].release_on_unlock = true; | |
| 8942 | − /* Now mark everyone sharing same ref_obj_id as untrusted */ | |
| 8943 | − bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ | |
| 8944 | − if (reg->ref_obj_id == ref_obj_id) | |
| 8945 | − reg->type |= PTR_UNTRUSTED; | |
| 8946 | − })); | |
| 8947 | − return 0; | |
| 8948 | − } | |
| 8998 | + })); | |
| 8999 | + return 0; | |
| 8949 | 9000 | } |
| 9001 | + | |
| 8950 | 9002 | verbose(env, "verifier internal error: ref state missing for ref_obj_id\n"); |
| 8951 | 9003 | return -EFAULT; |
| 8952 | 9004 | } |
| @@ -9081,7 +9133,6 @@ static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env, · 128 unmodified lines | ||
| 9081 | 9133 | { |
| 9082 | 9134 | const struct btf_type *et, *t; |
| 9083 | 9135 | struct btf_field *field; |
| 9084 | − struct btf_record *rec; | |
| 9085 | 9136 | u32 list_node_off; |
| 9086 | 9137 | |
| 9087 | 9138 | if (meta->btf != btf_vmlinux || |
| @@ -9098,9 +9149,8 @@ static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env, · 10 unmodified lines | ||
| 9098 | 9149 | return -EINVAL; |
| 9099 | 9150 | } |
| 9100 | 9151 | |
| 9101 | − rec = reg_btf_record(reg); | |
| 9102 | 9152 | list_node_off = reg->off + reg->var_off.value; |
| 9103 | − field = btf_record_find(rec, list_node_off, BPF_LIST_NODE); | |
| 9153 | + field = reg_find_field_offset(reg, list_node_off, BPF_LIST_NODE); | |
| 9104 | 9154 | if (!field || field->offset != list_node_off) { |
| 9105 | 9155 | verbose(env, "bpf_list_node not found at offset=%u\n", list_node_off); |
| 9106 | 9156 | return -EINVAL; |
| @@ -9126,8 +9176,8 @@ static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env, · 19 unmodified lines | ||
| 9126 | 9176 | btf_name_by_offset(field->graph_root.btf, et->name_off)); |
| 9127 | 9177 | return -EINVAL; |
| 9128 | 9178 | } |
| 9129 | − /* Set arg#1 for expiration after unlock */ | |
| 9130 | − return ref_set_release_on_unlock(env, reg->ref_obj_id); | |
| 9179 | + | |
| 9180 | + return 0; | |
| 9131 | 9181 | } |
| 9132 | 9182 | |
| 9133 | 9183 | static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta) |
| @@ -9406,11 +9456,11 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · 272 unmodified lines | ||
| 9406 | 9456 | int *insn_idx_p) |
| 9407 | 9457 | { |
| 9408 | 9458 | const struct btf_type *t, *func, *func_proto, *ptr_type; |
| 9459 | + u32 i, nargs, func_id, ptr_type_id, release_ref_obj_id; | |
| 9409 | 9460 | struct bpf_reg_state *regs = cur_regs(env); |
| 9410 | 9461 | const char *func_name, *ptr_type_name; |
| 9411 | 9462 | bool sleepable, rcu_lock, rcu_unlock; |
| 9412 | 9463 | struct bpf_kfunc_call_arg_meta meta; |
| 9413 | − u32 i, nargs, func_id, ptr_type_id; | |
| 9414 | 9464 | int err, insn_idx = *insn_idx_p; |
| 9415 | 9465 | const struct btf_param *args; |
| 9416 | 9466 | const struct btf_type *ret_t; |
| @@ -9505,6 +9555,24 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · 88 unmodified lines | ||
| 9505 | 9555 | } |
| 9506 | 9556 | } |
| 9507 | 9557 | |
| 9558 | + if (meta.func_id == special_kfunc_list[KF_bpf_list_push_front] || | |
| 9559 | + meta.func_id == special_kfunc_list[KF_bpf_list_push_back]) { | |
| 9560 | + release_ref_obj_id = regs[BPF_REG_2].ref_obj_id; | |
| 9561 | + err = ref_convert_owning_non_owning(env, release_ref_obj_id); | |
| 9562 | + if (err) { | |
| 9563 | + verbose(env, "kfunc %s#%d conversion of owning ref to non-owning failed\n", | |
| 9564 | + func_name, func_id); | |
| 9565 | + return err; | |
| 9566 | + } | |
| 9567 | + | |
| 9568 | + err = release_reference(env, release_ref_obj_id); | |
| 9569 | + if (err) { | |
| 9570 | + verbose(env, "kfunc %s#%d reference has not been acquired before\n", | |
| 9571 | + func_name, func_id); | |
| 9572 | + return err; | |
| 9573 | + } | |
| 9574 | + } | |
| 9575 | + | |
| 9508 | 9576 | for (i = 0; i < CALLER_SAVED_REGS; i++) |
| 9509 | 9577 | mark_reg_not_init(env, regs, caller_saved[i]); |
| 9510 | 9578 | |
| @@ -11825,8 +11893,10 @@ static void mark_ptr_or_null_reg(struct bpf_func_state *state, · 2314 unmodified lines | ||
| 11825 | 11893 | */ |
| 11826 | 11894 | if (WARN_ON_ONCE(reg->smin_value || reg->smax_value || !tnum_equals_const(reg->var_off, 0))) |
| 11827 | 11895 | return; |
| 11828 | − if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC | PTR_MAYBE_NULL) && WARN_ON_ONCE(reg->off)) | |
| 11896 | + if (!(type_is_ptr_alloc_obj(reg->type) || type_is_non_owning_ref(reg->type)) && | |
| 11897 | + WARN_ON_ONCE(reg->off)) | |
| 11829 | 11898 | return; |
| 11899 | + | |
| 11830 | 11900 | if (is_null) { |
| 11831 | 11901 | reg->type = SCALAR_VALUE; |
| 11832 | 11902 | /* We don't need id and ref_obj_id from this point |
| @@ -78,8 +78,6 @@ static struct { · 77 unmodified lines | ||
| 78 | 78 | { "direct_write_head", "direct access to bpf_list_head is disallowed" }, |
| 79 | 79 | { "direct_read_node", "direct access to bpf_list_node is disallowed" }, |
| 80 | 80 | { "direct_write_node", "direct access to bpf_list_node is disallowed" }, |
| 81 | − { "write_after_push_front", "only read is supported" }, | |
| 82 | − { "write_after_push_back", "only read is supported" }, | |
| 83 | 81 | { "use_after_unlock_push_front", "invalid mem access 'scalar'" }, |
| 84 | 82 | { "use_after_unlock_push_back", "invalid mem access 'scalar'" }, |
| 85 | 83 | { "double_push_front", "arg#1 expected pointer to allocated object" }, |
| @@ -260,7 +260,7 @@ int test_list_push_pop_multiple(struct bpf_spin_lock *lock, struct bpf_list_head · 259 unmodified lines | ||
| 260 | 260 | { |
| 261 | 261 | int ret; |
| 262 | 262 | |
| 263 | − ret = list_push_pop_multiple(lock ,head, false); | |
| 263 | + ret = list_push_pop_multiple(lock, head, false); | |
| 264 | 264 | if (ret) |
| 265 | 265 | return ret; |
| 266 | 266 | return list_push_pop_multiple(lock, head, true); |
| @@ -54,28 +54,44 @@ · 53 unmodified lines | ||
| 54 | 54 | return 0; \ |
| 55 | 55 | } |
| 56 | 56 | |
| 57 | −CHECK(kptr, push_front, &f->head); | |
| 58 | −CHECK(kptr, push_back, &f->head); | |
| 59 | 57 | CHECK(kptr, pop_front, &f->head); |
| 60 | 58 | CHECK(kptr, pop_back, &f->head); |
| 61 | 59 | |
| 62 | −CHECK(global, push_front, &ghead); | |
| 63 | −CHECK(global, push_back, &ghead); | |
| 64 | 60 | CHECK(global, pop_front, &ghead); |
| 65 | 61 | CHECK(global, pop_back, &ghead); |
| 66 | 62 | |
| 67 | −CHECK(map, push_front, &v->head); | |
| 68 | −CHECK(map, push_back, &v->head); | |
| 69 | 63 | CHECK(map, pop_front, &v->head); |
| 70 | 64 | CHECK(map, pop_back, &v->head); |
| 71 | 65 | |
| 72 | −CHECK(inner_map, push_front, &iv->head); | |
| 73 | −CHECK(inner_map, push_back, &iv->head); | |
| 74 | 66 | CHECK(inner_map, pop_front, &iv->head); |
| 75 | 67 | CHECK(inner_map, pop_back, &iv->head); |
| 76 | 68 | |
| 77 | 69 | #undef CHECK |
| 78 | 70 | |
| 71 | +#define CHECK(test, op, hexpr, nexpr) \ | |
| 72 | + SEC("?tc") \ | |
| 73 | + int test##_missing_lock_##op(void *ctx) \ | |
| 74 | + { \ | |
| 75 | + INIT; \ | |
| 76 | + void (*p)(void *, void *) = (void *)&bpf_list_##op; \ | |
| 77 | + p(hexpr, nexpr); \ | |
| 78 | + return 0; \ | |
| 79 | + } | |
| 80 | + | |
| 81 | +CHECK(kptr, push_front, &f->head, b); | |
| 82 | +CHECK(kptr, push_back, &f->head, b); | |
| 83 | + | |
| 84 | +CHECK(global, push_front, &ghead, f); | |
| 85 | +CHECK(global, push_back, &ghead, f); | |
| 86 | + | |
| 87 | +CHECK(map, push_front, &v->head, f); | |
| 88 | +CHECK(map, push_back, &v->head, f); | |
| 89 | + | |
| 90 | +CHECK(inner_map, push_front, &iv->head, f); | |
| 91 | +CHECK(inner_map, push_back, &iv->head, f); | |
| 92 | + | |
| 93 | +#undef CHECK | |
| 94 | + | |
| 79 | 95 | #define CHECK(test, op, lexpr, hexpr) \ |
| 80 | 96 | SEC("?tc") \ |
| 81 | 97 | int test##_incorrect_lock_##op(void *ctx) \ |
| @@ -108,11 +124,47 @@ CHECK(inner_map, pop_back, &iv->head); · 26 unmodified lines | ||
| 108 | 124 | CHECK(inner_map_global, op, &iv->lock, &ghead); \ |
| 109 | 125 | CHECK(inner_map_map, op, &iv->lock, &v->head); |
| 110 | 126 | |
| 111 | −CHECK_OP(push_front); | |
| 112 | −CHECK_OP(push_back); | |
| 113 | 127 | CHECK_OP(pop_front); |
| 114 | 128 | CHECK_OP(pop_back); |
| 115 | 129 | |
| 130 | +#undef CHECK | |
| 131 | +#undef CHECK_OP | |
| 132 | + | |
| 133 | +#define CHECK(test, op, lexpr, hexpr, nexpr) \ | |
| 134 | + SEC("?tc") \ | |
| 135 | + int test##_incorrect_lock_##op(void *ctx) \ | |
| 136 | + { \ | |
| 137 | + INIT; \ | |
| 138 | + void (*p)(void *, void*) = (void *)&bpf_list_##op; \ | |
| 139 | + bpf_spin_lock(lexpr); \ | |
| 140 | + p(hexpr, nexpr); \ | |
| 141 | + return 0; \ | |
| 142 | + } | |
| 143 | + | |
| 144 | +#define CHECK_OP(op) \ | |
| 145 | + CHECK(kptr_kptr, op, &f1->lock, &f2->head, b); \ | |
| 146 | + CHECK(kptr_global, op, &f1->lock, &ghead, f); \ | |
| 147 | + CHECK(kptr_map, op, &f1->lock, &v->head, f); \ | |
| 148 | + CHECK(kptr_inner_map, op, &f1->lock, &iv->head, f); \ | |
| 149 | + \ | |
| 150 | + CHECK(global_global, op, &glock2, &ghead, f); \ | |
| 151 | + CHECK(global_kptr, op, &glock, &f1->head, b); \ | |
| 152 | + CHECK(global_map, op, &glock, &v->head, f); \ | |
| 153 | + CHECK(global_inner_map, op, &glock, &iv->head, f); \ | |
| 154 | + \ | |
| 155 | + CHECK(map_map, op, &v->lock, &v2->head, f); \ | |
| 156 | + CHECK(map_kptr, op, &v->lock, &f2->head, b); \ | |
| 157 | + CHECK(map_global, op, &v->lock, &ghead, f); \ | |
| 158 | + CHECK(map_inner_map, op, &v->lock, &iv->head, f); \ | |
| 159 | + \ | |
| 160 | + CHECK(inner_map_inner_map, op, &iv->lock, &iv2->head, f); \ | |
| 161 | + CHECK(inner_map_kptr, op, &iv->lock, &f2->head, b); \ | |
| 162 | + CHECK(inner_map_global, op, &iv->lock, &ghead, f); \ | |
| 163 | + CHECK(inner_map_map, op, &iv->lock, &v->head, f); | |
| 164 | + | |
| 165 | +CHECK_OP(push_front); | |
| 166 | +CHECK_OP(push_back); | |
| 167 | + | |
| 116 | 168 | #undef CHECK |
| 117 | 169 | #undef CHECK_OP |
| 118 | 170 | #undef INIT |
| @@ -303,34 +355,6 @@ int direct_write_node(void *ctx) · 184 unmodified lines | ||
| 303 | 355 | return 0; |
| 304 | 356 | } |
| 305 | 357 | |
| 306 | −static __always_inline | |
| 307 | −int write_after_op(void (*push_op)(void *head, void *node)) | |
| 308 | −{ | |
| 309 | − struct foo *f; | |
| 310 | − | |
| 311 | − f = bpf_obj_new(typeof(*f)); | |
| 312 | − if (!f) | |
| 313 | − return 0; | |
| 314 | − bpf_spin_lock(&glock); | |
| 315 | − push_op(&ghead, &f->node); | |
| 316 | − f->data = 42; | |
| 317 | − bpf_spin_unlock(&glock); | |
| 318 | − | |
| 319 | − return 0; | |
| 320 | −} | |
| 321 | − | |
| 322 | −SEC("?tc") | |
| 323 | −int write_after_push_front(void *ctx) | |
| 324 | −{ | |
| 325 | − return write_after_op((void *)bpf_list_push_front); | |
| 326 | −} | |
| 327 | − | |
| 328 | −SEC("?tc") | |
| 329 | −int write_after_push_back(void *ctx) | |
| 330 | −{ | |
| 331 | − return write_after_op((void *)bpf_list_push_back); | |
| 332 | −} | |
| 333 | − | |
| 334 | 358 | static __always_inline |
| 335 | 359 | int use_after_unlock(void (*op)(void *head, void *node)) |
| 336 | 360 | { |