bpf: Migrate release_on_unlock logic to non-owning ref semantics

This patch introduces non-owning reference semantics to the verifier, specifically linked_list API kfunc handling. release_on_unlock logic for refs is refactored - with small functional changes - to implement these semantics, and bpf_list_push_{front,back} are migrated to use them. When a list node is pushed to a list, the program still has a pointer to the node: n = bpf_obj_new(typeof(*n)); bpf_spin_lock(&l); bpf_list_push_back(&l, n); /* n still points to the just-added node */ bpf_spin_unlock(&l); What the verifier considers n to be after the push, and thus what can be done with n, are changed by this patch. Common properties both before/after this patch: * After push, n is only a valid reference to the node until end of critical section * After push, n cannot be pushed to any list * After push, the program can read the node's fields using n Before: * After push, n retains the ref_obj_id which it received on bpf_obj_new, but the associated bpf_reference_state's release_on_unlock field is set to true * release_on_unlock field and associated logic is used to implement "n is only a valid ref until end of critical section" * After push, n cannot be written to, the node must be removed from the list before writing to its fields * After push, n is marked PTR_UNTRUSTED After: * After push, n's ref is released and ref_obj_id set to 0. NON_OWN_REF type flag is added to reg's type, indicating that it's a non-owning reference. * NON_OWN_REF flag and logic is used to implement "n is only a valid ref until end of critical section" * n can be written to (except for special fields e.g. bpf_list_node, timer, ...) Summary of specific implementation changes to achieve the above: * release_on_unlock field, ref_set_release_on_unlock helper, and logic to "release on unlock" based on that field are removed * The anonymous active_lock struct used by bpf_verifier_state is pulled out into a named struct bpf_active_lock. * NON_OWN_REF type flag is introduced along with verifier logic changes to handle non-owning refs * Helpers are added to use NON_OWN_REF flag to implement non-owning ref semantics as described above * invalidate_non_owning_refs - helper to clobber all non-owning refs matching a particular bpf_active_lock identity. Replaces release_on_unlock logic in process_spin_lock. * ref_set_non_owning - set NON_OWN_REF type flag after doing some sanity checking * ref_convert_owning_non_owning - convert owning reference w/ specified ref_obj_id to non-owning references. Set NON_OWN_REF flag for each reg with that ref_obj_id and 0-out its ref_obj_id * Update linked_list selftests to account for minor semantic differences introduced by this patch * Writes to a release_on_unlock node ref are not allowed, while writes to non-owning reference pointees are. As a result the linked_list "write after push" failure tests are no longer scenarios that should fail. * The test##missing_lock##op and test##incorrect_lock##op macro-generated failure tests need to have a valid node argument in order to have the same error output as before. Otherwise verification will fail early and the expected error output won't be seen.

Dave Marchevsky committed 3 years ago

Commit 6a3cd331 parent 39c536a6 files · +206 −110Browse files

Unified · Split

6 files

include/linux/bpf.h +6 −0
include/linux/bpf_verifier.h +18 −20
kernel/bpf/verifier.c +119 −49
tools/testing/selftests/bpf/prog_tests/linked_list.c +0 −2
tools/testing/selftests/bpf/progs/linked_list.c +1 −1
tools/testing/selftests/bpf/progs/linked_list_fail.c +62 −38

include/linux/bpf.h+6−0

 	BPF_KPTR       = BPF_KPTR_UNREF | BPF_KPTR_REF,
 	BPF_LIST_HEAD  = (1 << 4),
 	BPF_LIST_NODE  = (1 << 5),
+	BPF_GRAPH_NODE_OR_ROOT = BPF_LIST_NODE | BPF_LIST_HEAD,
 };
 struct btf_field_kptr {
 	/* MEM is tagged with rcu and memory access needs rcu_read_lock protection. */
 	MEM_RCU			= BIT(13 + BPF_BASE_TYPE_BITS),
+	/* Used to tag PTR_TO_BTF_ID | MEM_ALLOC references which are non-owning.
+	 * Currently only valid for linked-list and rbtree nodes.
+	 */
+	NON_OWN_REF		= BIT(14 + BPF_BASE_TYPE_BITS),
 	__BPF_TYPE_FLAG_MAX,
 	__BPF_TYPE_LAST_FLAG	= __BPF_TYPE_FLAG_MAX - 1,
 };

include/linux/bpf_verifier.h+18−20

@@ -43,6 +43,22 @@ enum bpf_reg_liveness { · 42 unmodified lines
43	REG_LIVE_DONE = 0x8, /* liveness won't be updating this register anymore */	43	REG_LIVE_DONE = 0x8, /* liveness won't be updating this register anymore */
44	};	44	};
45		45
		46	/* For every reg representing a map value or allocated object pointer,
		47	* we consider the tuple of (ptr, id) for them to be unique in verifier
		48	* context and conside them to not alias each other for the purposes of
		49	* tracking lock state.
		50	*/
		51	struct bpf_active_lock {
		52	/* This can either be reg->map_ptr or reg->btf. If ptr is NULL,
		53	* there's no active lock held, and other fields have no
		54	* meaning. If non-NULL, it indicates that a lock is held and
		55	* id member has the reg->id of the register which can be >= 0.
		56	*/
		57	void *ptr;
		58	/* This will be reg->id */
		59	u32 id;
		60	};
		61
46	struct bpf_reg_state {	62	struct bpf_reg_state {
47	/* Ordering of fields matters. See states_equal() */	63	/* Ordering of fields matters. See states_equal() */
48	enum bpf_reg_type type;	64	enum bpf_reg_type type;
@@ -226,11 +242,6 @@ struct bpf_reference_state { · 177 unmodified lines
226	* exiting a callback function.	242	* exiting a callback function.
227	*/	243	*/
228	int callback_ref;	244	int callback_ref;
229	/* Mark the reference state to release the registers sharing the same id
230	* on bpf_spin_unlock (for nodes that we will lose ownership to but are
231	* safe to access inside the critical section).
232	*/
233	bool release_on_unlock;
234	};	245	};
235		246
236	/* state of the program:	247	/* state of the program:
@@ -331,21 +342,8 @@ struct bpf_verifier_state { · 94 unmodified lines
331	u32 branches;	342	u32 branches;
332	u32 insn_idx;	343	u32 insn_idx;
333	u32 curframe;	344	u32 curframe;
334	/* For every reg representing a map value or allocated object pointer,	345
335	* we consider the tuple of (ptr, id) for them to be unique in verifier	346	struct bpf_active_lock active_lock;
336	* context and conside them to not alias each other for the purposes of
337	* tracking lock state.
338	*/
339	struct {
340	/* This can either be reg->map_ptr or reg->btf. If ptr is NULL,
341	* there's no active lock held, and other fields have no
342	* meaning. If non-NULL, it indicates that a lock is held and
343	* id member has the reg->id of the register which can be >= 0.
344	*/
345	void *ptr;
346	/* This will be reg->id */
347	u32 id;
348	} active_lock;
349	bool speculative;	347	bool speculative;
350	bool active_rcu_lock;	348	bool active_rcu_lock;
351		349

kernel/bpf/verifier.c+119−49

@@ -190,6 +190,9 @@ struct bpf_verifier_stack_elem { · 189 unmodified lines
190		190
191	static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx);	191	static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx);
192	static int release_reference(struct bpf_verifier_env *env, int ref_obj_id);	192	static int release_reference(struct bpf_verifier_env *env, int ref_obj_id);
		193	static void invalidate_non_owning_refs(struct bpf_verifier_env *env);
		194	static int ref_set_non_owning(struct bpf_verifier_env *env,
		195	struct bpf_reg_state *reg);
193		196
194	static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux)	197	static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux)
195	{	198	{
@@ -457,6 +460,11 @@ static bool type_is_ptr_alloc_obj(u32 type) · 261 unmodified lines
457	return base_type(type) == PTR_TO_BTF_ID && type_flag(type) & MEM_ALLOC;	460	return base_type(type) == PTR_TO_BTF_ID && type_flag(type) & MEM_ALLOC;
458	}	461	}
459		462
		463	static bool type_is_non_owning_ref(u32 type)
		464	{
		465	return type_is_ptr_alloc_obj(type) && type_flag(type) & NON_OWN_REF;
		466	}
		467
460	static struct btf_record reg_btf_record(const struct bpf_reg_state reg)	468	static struct btf_record reg_btf_record(const struct bpf_reg_state reg)
461	{	469	{
462	struct btf_record *rec = NULL;	470	struct btf_record *rec = NULL;
@@ -1073,6 +1081,8 @@ static void print_verifier_state(struct bpf_verifier_env *env, · 610 unmodified lines
1073	verbose_a("id=%d", reg->id);	1081	verbose_a("id=%d", reg->id);
1074	if (reg->ref_obj_id)	1082	if (reg->ref_obj_id)
1075	verbose_a("ref_obj_id=%d", reg->ref_obj_id);	1083	verbose_a("ref_obj_id=%d", reg->ref_obj_id);
		1084	if (type_is_non_owning_ref(reg->type))
		1085	verbose_a("%s", "non_own_ref");
1076	if (t != SCALAR_VALUE)	1086	if (t != SCALAR_VALUE)
1077	verbose_a("off=%d", reg->off);	1087	verbose_a("off=%d", reg->off);
1078	if (type_is_pkt_pointer(t))	1088	if (type_is_pkt_pointer(t))
@@ -5052,7 +5062,8 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, · 3973 unmodified lines
5052	return -EACCES;	5062	return -EACCES;
5053	}	5063	}
5054		5064
5055	if (type_is_alloc(reg->type) && !reg->ref_obj_id) {	5065	if (type_is_alloc(reg->type) && !type_is_non_owning_ref(reg->type) &&
		5066	!reg->ref_obj_id) {
5056	verbose(env, "verifier internal error: ref_obj_id for allocated object must be non-zero\n");	5067	verbose(env, "verifier internal error: ref_obj_id for allocated object must be non-zero\n");
5057	return -EFAULT;	5068	return -EFAULT;
5058	}	5069	}
@@ -6042,9 +6053,7 @@ static int process_spin_lock(struct bpf_verifier_env *env, int regno, · 983 unmodified lines
6042	cur->active_lock.ptr = btf;	6053	cur->active_lock.ptr = btf;
6043	cur->active_lock.id = reg->id;	6054	cur->active_lock.id = reg->id;
6044	} else {	6055	} else {
6045	struct bpf_func_state *fstate = cur_func(env);
6046	void *ptr;	6056	void *ptr;
6047	int i;
6048		6057
6049	if (map)	6058	if (map)
6050	ptr = map;	6059	ptr = map;
@@ -6060,25 +6069,11 @@ static int process_spin_lock(struct bpf_verifier_env *env, int regno, · 9 unmodified lines
6060	verbose(env, "bpf_spin_unlock of different lock\n");	6069	verbose(env, "bpf_spin_unlock of different lock\n");
6061	return -EINVAL;	6070	return -EINVAL;
6062	}	6071	}
6063	cur->active_lock.ptr = NULL;
6064	cur->active_lock.id = 0;
6065		6072
6066	for (i = fstate->acquired_refs - 1; i >= 0; i--) {	6073	invalidate_non_owning_refs(env);
6067	int err;
6068		6074
6069	/* Complain on error because this reference state cannot	6075	cur->active_lock.ptr = NULL;
6070	* be freed before this point, as bpf_spin_lock critical	6076	cur->active_lock.id = 0;
6071	* section does not allow functions that release the
6072	* allocated object immediately.
6073	*/
6074	if (!fstate->refs[i].release_on_unlock)
6075	continue;
6076	err = release_reference(env, fstate->refs[i].id);
6077	if (err) {
6078	verbose(env, "failed to release release_on_unlock reference");
6079	return err;
6080	}
6081	}
6082	}	6077	}
6083	return 0;	6078	return 0;
6084	}	6079	}
@@ -6546,6 +6541,23 @@ found: · 461 unmodified lines
6546	return 0;	6541	return 0;
6547	}	6542	}
6548		6543
		6544	static struct btf_field *
		6545	reg_find_field_offset(const struct bpf_reg_state *reg, s32 off, u32 fields)
		6546	{
		6547	struct btf_field *field;
		6548	struct btf_record *rec;
		6549
		6550	rec = reg_btf_record(reg);
		6551	if (!rec)
		6552	return NULL;
		6553
		6554	field = btf_record_find(rec, off, fields);
		6555	if (!field)
		6556	return NULL;
		6557
		6558	return field;
		6559	}
		6560
6549	int check_func_arg_reg_off(struct bpf_verifier_env *env,	6561	int check_func_arg_reg_off(struct bpf_verifier_env *env,
6550	const struct bpf_reg_state *reg, int regno,	6562	const struct bpf_reg_state *reg, int regno,
6551	enum bpf_arg_type arg_type)	6563	enum bpf_arg_type arg_type)
@@ -6567,6 +6579,18 @@ int check_func_arg_reg_off(struct bpf_verifier_env *env, · 15 unmodified lines
6567	*/	6579	*/
6568	if (arg_type_is_dynptr(arg_type) && type == PTR_TO_STACK)	6580	if (arg_type_is_dynptr(arg_type) && type == PTR_TO_STACK)
6569	return 0;	6581	return 0;
		6582
		6583	if ((type_is_ptr_alloc_obj(type) \|\| type_is_non_owning_ref(type)) && reg->off) {
		6584	if (reg_find_field_offset(reg, reg->off, BPF_GRAPH_NODE_OR_ROOT))
		6585	return __check_ptr_off_reg(env, reg, regno, true);
		6586
		6587	verbose(env, "R%d must have zero offset when passed to release func\n",
		6588	regno);
		6589	verbose(env, "No graph node or root found at R%d type:%s off:%d\n", regno,
		6590	kernel_type_name(reg->btf, reg->btf_id), reg->off);
		6591	return -EINVAL;
		6592	}
		6593
6570	/* Doing check_ptr_off_reg check for the offset will catch this	6594	/* Doing check_ptr_off_reg check for the offset will catch this
6571	* because fixed_off_ok is false, but checking here allows us	6595	* because fixed_off_ok is false, but checking here allows us
6572	* to give the user a better error message.	6596	* to give the user a better error message.
@@ -6601,6 +6625,7 @@ int check_func_arg_reg_off(struct bpf_verifier_env *env, · 28 unmodified lines
6601	case PTR_TO_BTF_ID \| PTR_TRUSTED:	6625	case PTR_TO_BTF_ID \| PTR_TRUSTED:
6602	case PTR_TO_BTF_ID \| MEM_RCU:	6626	case PTR_TO_BTF_ID \| MEM_RCU:
6603	case PTR_TO_BTF_ID \| MEM_ALLOC \| PTR_TRUSTED:	6627	case PTR_TO_BTF_ID \| MEM_ALLOC \| PTR_TRUSTED:
		6628	case PTR_TO_BTF_ID \| MEM_ALLOC \| NON_OWN_REF:
6604	/* When referenced PTR_TO_BTF_ID is passed to release function,	6629	/* When referenced PTR_TO_BTF_ID is passed to release function,
6605	* its fixed offset must be 0. In the other cases, fixed offset	6630	* its fixed offset must be 0. In the other cases, fixed offset
6606	* can be non-zero. This was already checked above. So pass	6631	* can be non-zero. This was already checked above. So pass
@@ -7363,6 +7388,17 @@ static int release_reference(struct bpf_verifier_env *env, · 756 unmodified lines
7363	return 0;	7388	return 0;
7364	}	7389	}
7365		7390
		7391	static void invalidate_non_owning_refs(struct bpf_verifier_env *env)
		7392	{
		7393	struct bpf_func_state *unused;
		7394	struct bpf_reg_state *reg;
		7395
		7396	bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({
		7397	if (type_is_non_owning_ref(reg->type))
		7398	__mark_reg_unknown(env, reg);
		7399	}));
		7400	}
		7401
7366	static void clear_caller_saved_regs(struct bpf_verifier_env *env,	7402	static void clear_caller_saved_regs(struct bpf_verifier_env *env,
7367	struct bpf_reg_state *regs)	7403	struct bpf_reg_state *regs)
7368	{	7404	{
@@ -8915,38 +8951,54 @@ static int process_kf_arg_ptr_to_kptr(struct bpf_verifier_env *env, · 1546 unmodified lines
8915	return 0;	8951	return 0;
8916	}	8952	}
8917		8953
8918	static int ref_set_release_on_unlock(struct bpf_verifier_env *env, u32 ref_obj_id)	8954	static int ref_set_non_owning(struct bpf_verifier_env env, struct bpf_reg_state reg)
8919	{	8955	{
8920	struct bpf_func_state *state = cur_func(env);	8956	struct bpf_verifier_state *state = env->cur_state;
		8957
		8958	if (!state->active_lock.ptr) {
		8959	verbose(env, "verifier internal error: ref_set_non_owning w/o active lock\n");
		8960	return -EFAULT;
		8961	}
		8962
		8963	if (type_flag(reg->type) & NON_OWN_REF) {
		8964	verbose(env, "verifier internal error: NON_OWN_REF already set\n");
		8965	return -EFAULT;
		8966	}
		8967
		8968	reg->type \|= NON_OWN_REF;
		8969	return 0;
		8970	}
		8971
		8972	static int ref_convert_owning_non_owning(struct bpf_verifier_env *env, u32 ref_obj_id)
		8973	{
		8974	struct bpf_func_state state, unused;
8921	struct bpf_reg_state *reg;	8975	struct bpf_reg_state *reg;
8922	int i;	8976	int i;
8923		8977
8924	/* bpf_spin_lock only allows calling list_push and list_pop, no BPF	8978	state = cur_func(env);
8925	* subprogs, no global functions. This means that the references would	8979
8926	* not be released inside the critical section but they may be added to
8927	* the reference state, and the acquired_refs are never copied out for a
8928	* different frame as BPF to BPF calls don't work in bpf_spin_lock
8929	* critical sections.
8930	*/
8931	if (!ref_obj_id) {	8980	if (!ref_obj_id) {
8932	verbose(env, "verifier internal error: ref_obj_id is zero for release_on_unlock\n");	8981	verbose(env, "verifier internal error: ref_obj_id is zero for "
		8982	"owning -> non-owning conversion\n");
8933	return -EFAULT;	8983	return -EFAULT;
8934	}	8984	}
		8985
8935	for (i = 0; i < state->acquired_refs; i++) {	8986	for (i = 0; i < state->acquired_refs; i++) {
8936	if (state->refs[i].id == ref_obj_id) {	8987	if (state->refs[i].id != ref_obj_id)
8937	if (state->refs[i].release_on_unlock) {	8988	continue;
8938	verbose(env, "verifier internal error: expected false release_on_unlock");	8989
8939	return -EFAULT;	8990	/* Clear ref_obj_id here so release_reference doesn't clobber
		8991	* the whole reg
		8992	*/
		8993	bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({
		8994	if (reg->ref_obj_id == ref_obj_id) {
		8995	reg->ref_obj_id = 0;
		8996	ref_set_non_owning(env, reg);
8940	}	8997	}
8941	state->refs[i].release_on_unlock = true;	8998	}));
8942	/* Now mark everyone sharing same ref_obj_id as untrusted */	8999	return 0;
8943	bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({
8944	if (reg->ref_obj_id == ref_obj_id)
8945	reg->type \|= PTR_UNTRUSTED;
8946	}));
8947	return 0;
8948	}
8949	}	9000	}
		9001
8950	verbose(env, "verifier internal error: ref state missing for ref_obj_id\n");	9002	verbose(env, "verifier internal error: ref state missing for ref_obj_id\n");
8951	return -EFAULT;	9003	return -EFAULT;
8952	}	9004	}
@@ -9081,7 +9133,6 @@ static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env, · 128 unmodified lines
9081	{	9133	{
9082	const struct btf_type et, t;	9134	const struct btf_type et, t;
9083	struct btf_field *field;	9135	struct btf_field *field;
9084	struct btf_record *rec;
9085	u32 list_node_off;	9136	u32 list_node_off;
9086		9137
9087	if (meta->btf != btf_vmlinux \|\|	9138	if (meta->btf != btf_vmlinux \|\|
@@ -9098,9 +9149,8 @@ static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env, · 10 unmodified lines
9098	return -EINVAL;	9149	return -EINVAL;
9099	}	9150	}
9100		9151
9101	rec = reg_btf_record(reg);
9102	list_node_off = reg->off + reg->var_off.value;	9152	list_node_off = reg->off + reg->var_off.value;
9103	field = btf_record_find(rec, list_node_off, BPF_LIST_NODE);	9153	field = reg_find_field_offset(reg, list_node_off, BPF_LIST_NODE);
9104	if (!field \|\| field->offset != list_node_off) {	9154	if (!field \|\| field->offset != list_node_off) {
9105	verbose(env, "bpf_list_node not found at offset=%u\n", list_node_off);	9155	verbose(env, "bpf_list_node not found at offset=%u\n", list_node_off);
9106	return -EINVAL;	9156	return -EINVAL;
@@ -9126,8 +9176,8 @@ static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env, · 19 unmodified lines
9126	btf_name_by_offset(field->graph_root.btf, et->name_off));	9176	btf_name_by_offset(field->graph_root.btf, et->name_off));
9127	return -EINVAL;	9177	return -EINVAL;
9128	}	9178	}
9129	/* Set arg#1 for expiration after unlock */	9179
9130	return ref_set_release_on_unlock(env, reg->ref_obj_id);	9180	return 0;
9131	}	9181	}
9132		9182
9133	static int check_kfunc_args(struct bpf_verifier_env env, struct bpf_kfunc_call_arg_meta meta)	9183	static int check_kfunc_args(struct bpf_verifier_env env, struct bpf_kfunc_call_arg_meta meta)
@@ -9406,11 +9456,11 @@ static int check_kfunc_call(struct bpf_verifier_env env, struct bpf_insn insn, · 272 unmodified lines
9406	int *insn_idx_p)	9456	int *insn_idx_p)
9407	{	9457	{
9408	const struct btf_type t, func, func_proto, ptr_type;	9458	const struct btf_type t, func, func_proto, ptr_type;
		9459	u32 i, nargs, func_id, ptr_type_id, release_ref_obj_id;
9409	struct bpf_reg_state *regs = cur_regs(env);	9460	struct bpf_reg_state *regs = cur_regs(env);
9410	const char func_name, ptr_type_name;	9461	const char func_name, ptr_type_name;
9411	bool sleepable, rcu_lock, rcu_unlock;	9462	bool sleepable, rcu_lock, rcu_unlock;
9412	struct bpf_kfunc_call_arg_meta meta;	9463	struct bpf_kfunc_call_arg_meta meta;
9413	u32 i, nargs, func_id, ptr_type_id;
9414	int err, insn_idx = *insn_idx_p;	9464	int err, insn_idx = *insn_idx_p;
9415	const struct btf_param *args;	9465	const struct btf_param *args;
9416	const struct btf_type *ret_t;	9466	const struct btf_type *ret_t;
@@ -9505,6 +9555,24 @@ static int check_kfunc_call(struct bpf_verifier_env env, struct bpf_insn insn, · 88 unmodified lines
9505	}	9555	}
9506	}	9556	}
9507		9557
		9558	if (meta.func_id == special_kfunc_list[KF_bpf_list_push_front] \|\|
		9559	meta.func_id == special_kfunc_list[KF_bpf_list_push_back]) {
		9560	release_ref_obj_id = regs[BPF_REG_2].ref_obj_id;
		9561	err = ref_convert_owning_non_owning(env, release_ref_obj_id);
		9562	if (err) {
		9563	verbose(env, "kfunc %s#%d conversion of owning ref to non-owning failed\n",
		9564	func_name, func_id);
		9565	return err;
		9566	}
		9567
		9568	err = release_reference(env, release_ref_obj_id);
		9569	if (err) {
		9570	verbose(env, "kfunc %s#%d reference has not been acquired before\n",
		9571	func_name, func_id);
		9572	return err;
		9573	}
		9574	}
		9575
9508	for (i = 0; i < CALLER_SAVED_REGS; i++)	9576	for (i = 0; i < CALLER_SAVED_REGS; i++)
9509	mark_reg_not_init(env, regs, caller_saved[i]);	9577	mark_reg_not_init(env, regs, caller_saved[i]);
9510		9578
@@ -11825,8 +11893,10 @@ static void mark_ptr_or_null_reg(struct bpf_func_state *state, · 2314 unmodified lines
11825	*/	11893	*/
11826	if (WARN_ON_ONCE(reg->smin_value \|\| reg->smax_value \|\| !tnum_equals_const(reg->var_off, 0)))	11894	if (WARN_ON_ONCE(reg->smin_value \|\| reg->smax_value \|\| !tnum_equals_const(reg->var_off, 0)))
11827	return;	11895	return;
11828	if (reg->type != (PTR_TO_BTF_ID \| MEM_ALLOC \| PTR_MAYBE_NULL) && WARN_ON_ONCE(reg->off))	11896	if (!(type_is_ptr_alloc_obj(reg->type) \|\| type_is_non_owning_ref(reg->type)) &&
		11897	WARN_ON_ONCE(reg->off))
11829	return;	11898	return;
		11899
11830	if (is_null) {	11900	if (is_null) {
11831	reg->type = SCALAR_VALUE;	11901	reg->type = SCALAR_VALUE;
11832	/* We don't need id and ref_obj_id from this point	11902	/* We don't need id and ref_obj_id from this point

tools/testing/selftests/bpf/prog_tests/linked_list.c+0−2

 	{ "direct_write_head", "direct access to bpf_list_head is disallowed" },
 	{ "direct_read_node", "direct access to bpf_list_node is disallowed" },
 	{ "direct_write_node", "direct access to bpf_list_node is disallowed" },
-	{ "write_after_push_front", "only read is supported" },
-	{ "write_after_push_back", "only read is supported" },
 	{ "use_after_unlock_push_front", "invalid mem access 'scalar'" },
 	{ "use_after_unlock_push_back", "invalid mem access 'scalar'" },
 	{ "double_push_front", "arg#1 expected pointer to allocated object" },

tools/testing/selftests/bpf/progs/linked_list.c+1−1

@@ -260,7 +260,7 @@ int test_list_push_pop_multiple(struct bpf_spin_lock *lock, struct bpf_list_head · 259 unmodified lines
260	{	260	{
261	int ret;	261	int ret;
262		262
263	ret = list_push_pop_multiple(lock ,head, false);	263	ret = list_push_pop_multiple(lock, head, false);
264	if (ret)	264	if (ret)
265	return ret;	265	return ret;
266	return list_push_pop_multiple(lock, head, true);	266	return list_push_pop_multiple(lock, head, true);

tools/testing/selftests/bpf/progs/linked_list_fail.c+62−38

 		return 0;                                   \
 	}
-CHECK(kptr, push_front, &f->head);
-CHECK(kptr, push_back, &f->head);
 CHECK(kptr, pop_front, &f->head);
 CHECK(kptr, pop_back, &f->head);
-CHECK(global, push_front, &ghead);
-CHECK(global, push_back, &ghead);
 CHECK(global, pop_front, &ghead);
 CHECK(global, pop_back, &ghead);
-CHECK(map, push_front, &v->head);
-CHECK(map, push_back, &v->head);
 CHECK(map, pop_front, &v->head);
 CHECK(map, pop_back, &v->head);
-CHECK(inner_map, push_front, &iv->head);
-CHECK(inner_map, push_back, &iv->head);
 CHECK(inner_map, pop_front, &iv->head);
 CHECK(inner_map, pop_back, &iv->head);
 #undef CHECK
+#define CHECK(test, op, hexpr, nexpr)					\
+	SEC("?tc")							\
+	int test##_missing_lock_##op(void *ctx)				\
+	{								\
+		INIT;							\
+		void (*p)(void *, void *) = (void *)&bpf_list_##op;	\
+		p(hexpr, nexpr);					\
+		return 0;						\
+	}
+CHECK(kptr, push_front, &f->head, b);
+CHECK(kptr, push_back, &f->head, b);
+CHECK(global, push_front, &ghead, f);
+CHECK(global, push_back, &ghead, f);
+CHECK(map, push_front, &v->head, f);
+CHECK(map, push_back, &v->head, f);
+CHECK(inner_map, push_front, &iv->head, f);
+CHECK(inner_map, push_back, &iv->head, f);
+#undef CHECK
 #define CHECK(test, op, lexpr, hexpr)                       \
 	SEC("?tc")                                          \
 	int test##_incorrect_lock_##op(void *ctx)           \
 	CHECK(inner_map_global, op, &iv->lock, &ghead);        \
 	CHECK(inner_map_map, op, &iv->lock, &v->head);
-CHECK_OP(push_front);
-CHECK_OP(push_back);
 CHECK_OP(pop_front);
 CHECK_OP(pop_back);
+#undef CHECK
+#undef CHECK_OP
+#define CHECK(test, op, lexpr, hexpr, nexpr)				\
+	SEC("?tc")							\
+	int test##_incorrect_lock_##op(void *ctx)			\
+	{								\
+		INIT;							\
+		void (*p)(void *, void*) = (void *)&bpf_list_##op;	\
+		bpf_spin_lock(lexpr);					\
+		p(hexpr, nexpr);					\
+		return 0;						\
+	}
+#define CHECK_OP(op)							\
+	CHECK(kptr_kptr, op, &f1->lock, &f2->head, b);			\
+	CHECK(kptr_global, op, &f1->lock, &ghead, f);			\
+	CHECK(kptr_map, op, &f1->lock, &v->head, f);			\
+	CHECK(kptr_inner_map, op, &f1->lock, &iv->head, f);		\
+									\
+	CHECK(global_global, op, &glock2, &ghead, f);			\
+	CHECK(global_kptr, op, &glock, &f1->head, b);			\
+	CHECK(global_map, op, &glock, &v->head, f);			\
+	CHECK(global_inner_map, op, &glock, &iv->head, f);		\
+									\
+	CHECK(map_map, op, &v->lock, &v2->head, f);			\
+	CHECK(map_kptr, op, &v->lock, &f2->head, b);			\
+	CHECK(map_global, op, &v->lock, &ghead, f);			\
+	CHECK(map_inner_map, op, &v->lock, &iv->head, f);		\
+									\
+	CHECK(inner_map_inner_map, op, &iv->lock, &iv2->head, f);	\
+	CHECK(inner_map_kptr, op, &iv->lock, &f2->head, b);		\
+	CHECK(inner_map_global, op, &iv->lock, &ghead, f);		\
+	CHECK(inner_map_map, op, &iv->lock, &v->head, f);
+CHECK_OP(push_front);
+CHECK_OP(push_back);
 #undef CHECK
 #undef CHECK_OP
 #undef INIT
 	return 0;
 }
-static __always_inline
-int write_after_op(void (*push_op)(void *head, void *node))
-{
-	struct foo *f;
-	f = bpf_obj_new(typeof(*f));
-	if (!f)
-		return 0;
-	bpf_spin_lock(&glock);
-	push_op(&ghead, &f->node);
-	f->data = 42;
-	bpf_spin_unlock(&glock);
-	return 0;
-}
-SEC("?tc")
-int write_after_push_front(void *ctx)
-{
-	return write_after_op((void *)bpf_list_push_front);
-}
-SEC("?tc")
-int write_after_push_back(void *ctx)
-{
-	return write_after_op((void *)bpf_list_push_back);
-}
 static __always_inline
 int use_after_unlock(void (*op)(void *head, void *node))
 {

bpf: Migrate release_on_unlock logic to non-owning ref semantics

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code