bpf_refcount_acquire() increments the refcount at the caller-supplied pointer plus the refcount field offset, then returns the caller-supplied pointer unchanged. The verifier records the return value as a base pointer to the refcounted object. bpf_list_pop_front() and bpf_rbtree_remove() can return embedded graph-node pointers as PTR_TO_BTF_ID | MEM_ALLOC with a fixed offset equal to the node field offset. Passing such a pointer directly to bpf_refcount_acquire() currently passes the refcounted-kptr type check. That makes the runtime operation start from base + node_off while the verifier models the returned pointer as the object base. Require refcount-acquire arguments to have zero fixed offset by carrying the requirement through check_func_arg_reg_off() to __check_ptr_off_reg(). Programs can still acquire a refcount from a graph-node-derived pointer after normalizing it with container_of().
Showing the changes to kernel (1 of 1 file). Show all
Unified · Split
| @@ -7996,9 +7996,10 @@ reg_find_field_offset(const struct bpf_reg_state *reg, s32 off, u32 fields) · 7995 unmodified lines | ||
| 7996 | 7996 | return field; |
| 7997 | 7997 | } |
| 7998 | 7998 | |
| 7999 | −static int check_func_arg_reg_off(struct bpf_verifier_env *env, | |
| 8000 | − const struct bpf_reg_state *reg, argno_t argno, | |
| 8001 | − enum bpf_arg_type arg_type) | |
| 7999 | +static int __check_func_arg_reg_off(struct bpf_verifier_env *env, | |
| 8000 | + const struct bpf_reg_state *reg, argno_t argno, | |
| 8001 | + enum bpf_arg_type arg_type, | |
| 8002 | + bool btf_id_fixed_off_ok) | |
| 8002 | 8003 | { |
| 8003 | 8004 | u32 type = reg->type; |
| 8004 | 8005 | |
| @@ -8055,12 +8056,11 @@ static int check_func_arg_reg_off(struct bpf_verifier_env *env, · 50 unmodified lines | ||
| 8055 | 8056 | case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU: |
| 8056 | 8057 | /* When referenced PTR_TO_BTF_ID is passed to release function, |
| 8057 | 8058 | * its fixed offset must be 0. In the other cases, fixed offset |
| 8058 | − * can be non-zero. This was already checked above. So pass | |
| 8059 | − * fixed_off_ok as true to allow fixed offset for all other | |
| 8060 | − * cases. var_off always must be 0 for PTR_TO_BTF_ID, hence we | |
| 8061 | − * still need to do checks instead of returning. | |
| 8059 | + * can be non-zero unless the caller requires otherwise. | |
| 8060 | + * var_off always must be 0 for PTR_TO_BTF_ID, hence we still | |
| 8061 | + * need to do checks instead of returning. | |
| 8062 | 8062 | */ |
| 8063 | − return __check_ptr_off_reg(env, reg, argno, true); | |
| 8063 | + return __check_ptr_off_reg(env, reg, argno, btf_id_fixed_off_ok); | |
| 8064 | 8064 | case PTR_TO_CTX: |
| 8065 | 8065 | /* |
| 8066 | 8066 | * Allow fixed and variable offsets for syscall context, but |
| @@ -8076,6 +8076,13 @@ static int check_func_arg_reg_off(struct bpf_verifier_env *env, · 9 unmodified lines | ||
| 8076 | 8076 | } |
| 8077 | 8077 | } |
| 8078 | 8078 | |
| 8079 | +static int check_func_arg_reg_off(struct bpf_verifier_env *env, | |
| 8080 | + const struct bpf_reg_state *reg, argno_t argno, | |
| 8081 | + enum bpf_arg_type arg_type) | |
| 8082 | +{ | |
| 8083 | + return __check_func_arg_reg_off(env, reg, argno, arg_type, true); | |
| 8084 | +} | |
| 8085 | + | |
| 8079 | 8086 | static int check_arg_const_str(struct bpf_verifier_env *env, |
| 8080 | 8087 | struct bpf_reg_state *reg, argno_t argno) |
| 8081 | 8088 | { |
| @@ -11947,6 +11954,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ · 3865 unmodified lines | ||
| 11947 | 11954 | enum bpf_arg_type arg_type = ARG_DONTCARE; |
| 11948 | 11955 | argno_t argno = argno_from_arg(i + 1); |
| 11949 | 11956 | int regno = reg_from_argno(argno); |
| 11957 | + bool btf_id_fixed_off_ok = true; | |
| 11950 | 11958 | u32 ref_id, type_size; |
| 11951 | 11959 | bool is_ret_buf_sz = false; |
| 11952 | 11960 | int kf_arg_type; |
| @@ -12120,7 +12128,6 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ · 167 unmodified lines | ||
| 12120 | 12128 | case KF_ARG_PTR_TO_MEM: |
| 12121 | 12129 | case KF_ARG_PTR_TO_MEM_SIZE: |
| 12122 | 12130 | case KF_ARG_PTR_TO_CALLBACK: |
| 12123 | − case KF_ARG_PTR_TO_REFCOUNTED_KPTR: | |
| 12124 | 12131 | case KF_ARG_PTR_TO_CONST_STR: |
| 12125 | 12132 | case KF_ARG_PTR_TO_WORKQUEUE: |
| 12126 | 12133 | case KF_ARG_PTR_TO_TIMER: |
| @@ -12134,6 +12141,10 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ · 7 unmodified lines | ||
| 12134 | 12141 | case KF_ARG_PTR_TO_CTX: |
| 12135 | 12142 | arg_type = ARG_PTR_TO_CTX; |
| 12136 | 12143 | break; |
| 12144 | + case KF_ARG_PTR_TO_REFCOUNTED_KPTR: | |
| 12145 | + arg_type = ARG_PTR_TO_BTF_ID; | |
| 12146 | + btf_id_fixed_off_ok = false; | |
| 12147 | + break; | |
| 12137 | 12148 | default: |
| 12138 | 12149 | verifier_bug(env, "unknown kfunc arg type %d", kf_arg_type); |
| 12139 | 12150 | return -EFAULT; |
| @@ -12141,7 +12152,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ · 1 unmodified line | ||
| 12141 | 12152 | |
| 12142 | 12153 | if (regno == meta->release_regno) |
| 12143 | 12154 | arg_type |= OBJ_RELEASE; |
| 12144 | − ret = check_func_arg_reg_off(env, reg, argno, arg_type); | |
| 12155 | + ret = __check_func_arg_reg_off(env, reg, argno, arg_type, | |
| 12156 | + btf_id_fixed_off_ok); | |
| 12145 | 12157 | if (ret < 0) |
| 12146 | 12158 | return ret; |
| 12147 | 12159 | |