bpf: Register dtor for freeing special fields

There is a race window where BPF hash map elements can leak special fields if the program with access to the map value recreates these special fields between the check_and_free_fields done on the map value and its eventual return to the memory allocator. Several ways were explored prior to this patch, most notably [0] tried to use a poison value to reject attempts to recreate special fields for map values that have been logically deleted but still accessible to BPF programs (either while sitting in the free list or when reused). While this approach works well for task work, timers, wq, etc., it is harder to apply the idea to kptrs, which have a similar race and failure mode. Instead, we change bpf_mem_alloc to allow registering destructor for allocated elements, such that when they are returned to the allocator, any special fields created while they were accessible to programs in the mean time will be freed. If these values get reused, we do not free the fields again before handing the element back. The special fields thus may remain initialized while the map value sits in a free list. When bpf_mem_alloc is retired in the future, a similar concept can be introduced to kmalloc_nolock-backed kmem_cache, paired with the existing idea of a constructor. Note that the destructor registration happens in map_check_btf, after the BTF record is populated and (at that point) avaiable for inspection and duplication. Duplication is necessary since the freeing of embedded bpf_mem_alloc can be decoupled from actual map lifetime due to logic introduced to reduce the cost of rcu_barrier()s in mem alloc free path in 9f2c6e96c65e ("bpf: Optimize rcu_barrier usage between hash map and bpf_mem_alloc."). As such, once all callbacks are done, we must also free the duplicated record. To remove dependency on the bpf_map itself, also stash the key size of the map to obtain value from htab_elem long after the map is gone. [0]: https://lore.kernel.org/bpf/20260216131341.1285427-1-mykyta.yatsenko5@gmail.com

Kumar Kartikeya Dwivedi committed 8 months ago

Commit 1df97a71 parent 6881af23 files · +140 −11Browse files

Unified · Split

3 files

include/linux/bpf_mem_alloc.h +6 −0
kernel/bpf/hashtab.c +87 −0
kernel/bpf/memalloc.c +47 −11

include/linux/bpf_mem_alloc.h+6−0

@@ -14,6 +14,8 @@ struct bpf_mem_alloc { · 13 unmodified lines
14	14	struct obj_cgroup *objcg;
15	15	bool percpu;
16	16	struct work_struct work;
	17	+ void (dtor_ctx_free)(void ctx);
	18	+ void *dtor_ctx;
17	19	};
18	20
19	21	/* 'size != 0' is for bpf_mem_alloc which manages fixed-size objects.
@@ -32,6 +34,10 @@ int bpf_mem_alloc_percpu_init(struct bpf_mem_alloc ma, struct obj_cgroup objcg · 12 unmodified lines
32	34	/* The percpu allocation with a specific unit size. */
33	35	int bpf_mem_alloc_percpu_unit_init(struct bpf_mem_alloc *ma, int size);
34	36	void bpf_mem_alloc_destroy(struct bpf_mem_alloc *ma);
	37	+void bpf_mem_alloc_set_dtor(struct bpf_mem_alloc *ma,
	38	+ void (dtor)(void obj, void *ctx),
	39	+ void (dtor_ctx_free)(void ctx),
	40	+ void *ctx);
35	41
36	42	/* Check the allocation size for kmalloc equivalent allocator */
37	43	int bpf_mem_alloc_check_size(bool percpu, size_t size);

kernel/bpf/hashtab.c+87−0

@@ -125,6 +125,11 @@ struct htab_elem { · 124 unmodified lines
125	125	char key[] __aligned(8);
126	126	};
127	127
	128	+struct htab_btf_record {
	129	+ struct btf_record *record;
	130	+ u32 key_size;
	131	+};
	132	+
128	133	static inline bool htab_is_prealloc(const struct bpf_htab *htab)
129	134	{
130	135	return !(htab->map.map_flags & BPF_F_NO_PREALLOC);
@@ -457,6 +462,84 @@ static int htab_map_alloc_check(union bpf_attr *attr) · 326 unmodified lines
457	462	return 0;
458	463	}
459	464
	465	+static void htab_mem_dtor(void obj, void ctx)
	466	+{
	467	+ struct htab_btf_record *hrec = ctx;
	468	+ struct htab_elem *elem = obj;
	469	+ void *map_value;
	470	+
	471	+ if (IS_ERR_OR_NULL(hrec->record))
	472	+ return;
	473	+
	474	+ map_value = htab_elem_value(elem, hrec->key_size);
	475	+ bpf_obj_free_fields(hrec->record, map_value);
	476	+}
	477	+
	478	+static void htab_pcpu_mem_dtor(void obj, void ctx)
	479	+{
	480	+ void __percpu pptr = (void __percpu **)obj;
	481	+ struct htab_btf_record *hrec = ctx;
	482	+ int cpu;
	483	+
	484	+ if (IS_ERR_OR_NULL(hrec->record))
	485	+ return;
	486	+
	487	+ for_each_possible_cpu(cpu)
	488	+ bpf_obj_free_fields(hrec->record, per_cpu_ptr(pptr, cpu));
	489	+}
	490	+
	491	+static void htab_dtor_ctx_free(void *ctx)
	492	+{
	493	+ struct htab_btf_record *hrec = ctx;
	494	+
	495	+ btf_record_free(hrec->record);
	496	+ kfree(ctx);
	497	+}
	498	+
	499	+static int htab_set_dtor(const struct bpf_htab htab, void (dtor)(void , void ))
	500	+{
	501	+ u32 key_size = htab->map.key_size;
	502	+ const struct bpf_mem_alloc *ma;
	503	+ struct htab_btf_record *hrec;
	504	+ int err;
	505	+
	506	+ /* No need for dtors. */
	507	+ if (IS_ERR_OR_NULL(htab->map.record))
	508	+ return 0;
	509	+
	510	+ hrec = kzalloc(sizeof(*hrec), GFP_KERNEL);
	511	+ if (!hrec)
	512	+ return -ENOMEM;
	513	+ hrec->key_size = key_size;
	514	+ hrec->record = btf_record_dup(htab->map.record);
	515	+ if (IS_ERR(hrec->record)) {
	516	+ err = PTR_ERR(hrec->record);
	517	+ kfree(hrec);
	518	+ return err;
	519	+ }
	520	+ ma = htab_is_percpu(htab) ? &htab->pcpu_ma : &htab->ma;
	521	+ /* Kinda sad, but cast away const-ness since we change ma->dtor. */
	522	+ bpf_mem_alloc_set_dtor((struct bpf_mem_alloc *)ma, dtor, htab_dtor_ctx_free, hrec);
	523	+ return 0;
	524	+}
	525	+
	526	+static int htab_map_check_btf(const struct bpf_map map, const struct btf btf,
	527	+ const struct btf_type key_type, const struct btf_type value_type)
	528	+{
	529	+ struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
	530	+
	531	+ if (htab_is_prealloc(htab))
	532	+ return 0;
	533	+ /*
	534	+ * We must set the dtor using this callback, as map's BTF record is not
	535	+ * populated in htab_map_alloc(), so it will always appear as NULL.
	536	+ */
	537	+ if (htab_is_percpu(htab))
	538	+ return htab_set_dtor(htab, htab_pcpu_mem_dtor);
	539	+ else
	540	+ return htab_set_dtor(htab, htab_mem_dtor);
	541	+}
	542	+
460	543	static struct bpf_map htab_map_alloc(union bpf_attr attr)
461	544	{
462	545	bool percpu = (attr->map_type == BPF_MAP_TYPE_PERCPU_HASH \|\|
@@ -2281,6 +2364,7 @@ const struct bpf_map_ops htab_map_ops = { · 1818 unmodified lines
2281	2364	.map_seq_show_elem = htab_map_seq_show_elem,
2282	2365	.map_set_for_each_callback_args = map_set_for_each_callback_args,
2283	2366	.map_for_each_callback = bpf_for_each_hash_elem,
	2367	+ .map_check_btf = htab_map_check_btf,
2284	2368	.map_mem_usage = htab_map_mem_usage,
2285	2369	BATCH_OPS(htab),
2286	2370	.map_btf_id = &htab_map_btf_ids[0],
@@ -2303,6 +2387,7 @@ const struct bpf_map_ops htab_lru_map_ops = { · 16 unmodified lines
2303	2387	.map_seq_show_elem = htab_map_seq_show_elem,
2304	2388	.map_set_for_each_callback_args = map_set_for_each_callback_args,
2305	2389	.map_for_each_callback = bpf_for_each_hash_elem,
	2390	+ .map_check_btf = htab_map_check_btf,
2306	2391	.map_mem_usage = htab_map_mem_usage,
2307	2392	BATCH_OPS(htab_lru),
2308	2393	.map_btf_id = &htab_map_btf_ids[0],
@@ -2482,6 +2567,7 @@ const struct bpf_map_ops htab_percpu_map_ops = { · 173 unmodified lines
2482	2567	.map_seq_show_elem = htab_percpu_map_seq_show_elem,
2483	2568	.map_set_for_each_callback_args = map_set_for_each_callback_args,
2484	2569	.map_for_each_callback = bpf_for_each_hash_elem,
	2570	+ .map_check_btf = htab_map_check_btf,
2485	2571	.map_mem_usage = htab_map_mem_usage,
2486	2572	BATCH_OPS(htab_percpu),
2487	2573	.map_btf_id = &htab_map_btf_ids[0],
@@ -2502,6 +2588,7 @@ const struct bpf_map_ops htab_lru_percpu_map_ops = { · 14 unmodified lines
2502	2588	.map_seq_show_elem = htab_percpu_map_seq_show_elem,
2503	2589	.map_set_for_each_callback_args = map_set_for_each_callback_args,
2504	2590	.map_for_each_callback = bpf_for_each_hash_elem,
	2591	+ .map_check_btf = htab_map_check_btf,
2505	2592	.map_mem_usage = htab_map_mem_usage,
2506	2593	BATCH_OPS(htab_lru_percpu),
2507	2594	.map_btf_id = &htab_map_btf_ids[0],

kernel/bpf/memalloc.c+47−11

@@ -102,6 +102,8 @@ struct bpf_mem_cache { · 101 unmodified lines
102	102	int percpu_size;
103	103	bool draining;
104	104	struct bpf_mem_cache *tgt;
	105	+ void (dtor)(void obj, void *ctx);
	106	+ void *dtor_ctx;
105	107
106	108	/* list of objects to be freed after RCU GP */
107	109	struct llist_head free_by_rcu;
@@ -260,12 +262,14 @@ static void free_one(void *obj, bool percpu) · 152 unmodified lines
260	262	kfree(obj);
261	263	}
262	264
263		−static int free_all(struct llist_node *llnode, bool percpu)
	265	+static int free_all(struct bpf_mem_cache c, struct llist_node llnode, bool percpu)
264	266	{
265	267	struct llist_node pos, t;
266	268	int cnt = 0;
267	269
268	270	llist_for_each_safe(pos, t, llnode) {
	271	+ if (c->dtor)
	272	+ c->dtor((void *)pos + LLIST_NODE_SZ, c->dtor_ctx);
269	273	free_one(pos, percpu);
270	274	cnt++;
271	275	}
@@ -276,7 +280,7 @@ static void __free_rcu(struct rcu_head *head) · 4 unmodified lines
276	280	{
277	281	struct bpf_mem_cache *c = container_of(head, struct bpf_mem_cache, rcu_ttrace);
278	282
279		− free_all(llist_del_all(&c->waiting_for_gp_ttrace), !!c->percpu_size);
	283	+ free_all(c, llist_del_all(&c->waiting_for_gp_ttrace), !!c->percpu_size);
280	284	atomic_set(&c->call_rcu_ttrace_in_progress, 0);
281	285	}
282	286
@@ -308,7 +312,7 @@ static void do_call_rcu_ttrace(struct bpf_mem_cache *c) · 25 unmodified lines
308	312	if (atomic_xchg(&c->call_rcu_ttrace_in_progress, 1)) {
309	313	if (unlikely(READ_ONCE(c->draining))) {
310	314	llnode = llist_del_all(&c->free_by_rcu_ttrace);
311		− free_all(llnode, !!c->percpu_size);
	315	+ free_all(c, llnode, !!c->percpu_size);
312	316	}
313	317	return;
314	318	}
@@ -417,7 +421,7 @@ static void check_free_by_rcu(struct bpf_mem_cache *c) · 102 unmodified lines
417	421	dec_active(c, &flags);
418	422
419	423	if (unlikely(READ_ONCE(c->draining))) {
420		− free_all(llist_del_all(&c->waiting_for_gp), !!c->percpu_size);
	424	+ free_all(c, llist_del_all(&c->waiting_for_gp), !!c->percpu_size);
421	425	atomic_set(&c->call_rcu_in_progress, 0);
422	426	} else {
423	427	call_rcu_hurry(&c->rcu, __free_by_rcu);
@@ -635,13 +639,13 @@ static void drain_mem_cache(struct bpf_mem_cache *c) · 211 unmodified lines
635	639	* Except for waiting_for_gp_ttrace list, there are no concurrent operations
636	640	* on these lists, so it is safe to use __llist_del_all().
637	641	*/
638		− free_all(llist_del_all(&c->free_by_rcu_ttrace), percpu);
639		− free_all(llist_del_all(&c->waiting_for_gp_ttrace), percpu);
640		− free_all(__llist_del_all(&c->free_llist), percpu);
641		− free_all(__llist_del_all(&c->free_llist_extra), percpu);
642		− free_all(__llist_del_all(&c->free_by_rcu), percpu);
643		− free_all(__llist_del_all(&c->free_llist_extra_rcu), percpu);
644		− free_all(llist_del_all(&c->waiting_for_gp), percpu);
	642	+ free_all(c, llist_del_all(&c->free_by_rcu_ttrace), percpu);
	643	+ free_all(c, llist_del_all(&c->waiting_for_gp_ttrace), percpu);
	644	+ free_all(c, __llist_del_all(&c->free_llist), percpu);
	645	+ free_all(c, __llist_del_all(&c->free_llist_extra), percpu);
	646	+ free_all(c, __llist_del_all(&c->free_by_rcu), percpu);
	647	+ free_all(c, __llist_del_all(&c->free_llist_extra_rcu), percpu);
	648	+ free_all(c, llist_del_all(&c->waiting_for_gp), percpu);
645	649	}
646	650
647	651	static void check_mem_cache(struct bpf_mem_cache *c)
@@ -680,6 +684,9 @@ static void check_leaked_objs(struct bpf_mem_alloc *ma) · 32 unmodified lines
680	684
681	685	static void free_mem_alloc_no_barrier(struct bpf_mem_alloc *ma)
682	686	{
	687	+ /* We can free dtor ctx only once all callbacks are done using it. */
	688	+ if (ma->dtor_ctx_free)
	689	+ ma->dtor_ctx_free(ma->dtor_ctx);
683	690	check_leaked_objs(ma);
684	691	free_percpu(ma->cache);
685	692	free_percpu(ma->caches);
@@ -1014,3 +1021,32 @@ int bpf_mem_alloc_check_size(bool percpu, size_t size) · 328 unmodified lines
1014	1021
1015	1022	return 0;
1016	1023	}
	1024	+
	1025	+void bpf_mem_alloc_set_dtor(struct bpf_mem_alloc ma, void (dtor)(void obj, void ctx),
	1026	+ void (dtor_ctx_free)(void ctx), void *ctx)
	1027	+{
	1028	+ struct bpf_mem_caches *cc;
	1029	+ struct bpf_mem_cache *c;
	1030	+ int cpu, i;
	1031	+
	1032	+ ma->dtor_ctx_free = dtor_ctx_free;
	1033	+ ma->dtor_ctx = ctx;
	1034	+
	1035	+ if (ma->cache) {
	1036	+ for_each_possible_cpu(cpu) {
	1037	+ c = per_cpu_ptr(ma->cache, cpu);
	1038	+ c->dtor = dtor;
	1039	+ c->dtor_ctx = ctx;
	1040	+ }
	1041	+ }
	1042	+ if (ma->caches) {
	1043	+ for_each_possible_cpu(cpu) {
	1044	+ cc = per_cpu_ptr(ma->caches, cpu);
	1045	+ for (i = 0; i < NUM_CACHES; i++) {
	1046	+ c = &cc->cache[i];
	1047	+ c->dtor = dtor;
	1048	+ c->dtor_ctx = ctx;
	1049	+ }
	1050	+ }
	1051	+ }
	1052	+}