bpf: Fix percpu address space issues

In arraymap.c: In bpf_array_map_seq_start() and bpf_array_map_seq_next() cast return values from the __percpu address space to the generic address space via uintptr_t [1]. Correct the declaration of pptr pointer in __bpf_array_map_seq_show() to void __percpu * and cast the value from the generic address space to the __percpu address space via uintptr_t [1]. In hashtab.c: Assign the return value from bpf_mem_cache_alloc() to void pointer and cast the value to void __percpu ** (void pointer to percpu void pointer) before dereferencing. In memalloc.c: Explicitly declare __percpu variables. Cast obj to void __percpu **. In helpers.c: Cast ptr in BPF_CALL_1 and BPF_CALL_2 from generic address space to __percpu address space via const uintptr_t [1]. Found by GCC's named address space checks. There were no changes in the resulting object files. [1] https://sparse.docs.kernel.org/en/latest/annotations.html#address-space-name

Uros Bizjak committed 2 years ago

Commit 6d641ca1 parent 5148f194 files · +17 −16Browse files

Unified · Split

4 files

kernel/bpf/arraymap.c +4 −4
kernel/bpf/hashtab.c +5 −4
kernel/bpf/helpers.c +2 −2
kernel/bpf/memalloc.c +6 −6

kernel/bpf/arraymap.c+4−4

@@ -600,7 +600,7 @@ static void bpf_array_map_seq_start(struct seq_file seq, loff_t *pos) · 599 unmodified lines
600	600	array = container_of(map, struct bpf_array, map);
601	601	index = info->index & array->index_mask;
602	602	if (info->percpu_value_buf)
603		− return array->pptrs[index];
	603	+ return (void *)(uintptr_t)array->pptrs[index];
604	604	return array_map_elem_ptr(array, index);
605	605	}
606	606
@@ -619,7 +619,7 @@ static void bpf_array_map_seq_next(struct seq_file seq, void v, loff_t pos) · 12 unmodified lines
619	619	array = container_of(map, struct bpf_array, map);
620	620	index = info->index & array->index_mask;
621	621	if (info->percpu_value_buf)
622		− return array->pptrs[index];
	622	+ return (void *)(uintptr_t)array->pptrs[index];
623	623	return array_map_elem_ptr(array, index);
624	624	}
625	625
@@ -632,7 +632,7 @@ static int __bpf_array_map_seq_show(struct seq_file seq, void v) · 6 unmodified lines
632	632	struct bpf_iter_meta meta;
633	633	struct bpf_prog *prog;
634	634	int off = 0, cpu = 0;
635		− void __percpu **pptr;
	635	+ void __percpu *pptr;
636	636	u32 size;
637	637
638	638	meta.seq = seq;
@@ -648,7 +648,7 @@ static int __bpf_array_map_seq_show(struct seq_file seq, void v) · 9 unmodified lines
648	648	if (!info->percpu_value_buf) {
649	649	ctx.value = v;
650	650	} else {
651		− pptr = v;
	651	+ pptr = (void __percpu *)(uintptr_t)v;
652	652	size = array->elem_size;
653	653	for_each_possible_cpu(cpu) {
654	654	copy_map_value_long(map, info->percpu_value_buf + off,

kernel/bpf/hashtab.c+5−4

@@ -1049,14 +1049,15 @@ static struct htab_elem alloc_htab_elem(struct bpf_htab htab, void *key, · 1048 unmodified lines
1049	1049	pptr = htab_elem_get_ptr(l_new, key_size);
1050	1050	} else {
1051	1051	/* alloc_percpu zero-fills */
1052		− pptr = bpf_mem_cache_alloc(&htab->pcpu_ma);
1053		− if (!pptr) {
	1052	+ void *ptr = bpf_mem_cache_alloc(&htab->pcpu_ma);
	1053	+
	1054	+ if (!ptr) {
1054	1055	bpf_mem_cache_free(&htab->ma, l_new);
1055	1056	l_new = ERR_PTR(-ENOMEM);
1056	1057	goto dec_count;
1057	1058	}
1058		− l_new->ptr_to_pptr = pptr;
1059		− pptr = (void *)pptr;
	1059	+ l_new->ptr_to_pptr = ptr;
	1060	+ pptr = (void __percpu *)ptr;
1060	1061	}
1061	1062
1062	1063	pcpu_init_value(htab, pptr, value, onallcpus);

kernel/bpf/helpers.c+2−2

@@ -715,7 +715,7 @@ BPF_CALL_2(bpf_per_cpu_ptr, const void *, ptr, u32, cpu) · 714 unmodified lines
715	715	if (cpu >= nr_cpu_ids)
716	716	return (unsigned long)NULL;
717	717
718		− return (unsigned long)per_cpu_ptr((const void __percpu *)ptr, cpu);
	718	+ return (unsigned long)per_cpu_ptr((const void __percpu *)(const uintptr_t)ptr, cpu);
719	719	}
720	720
721	721	const struct bpf_func_proto bpf_per_cpu_ptr_proto = {
@@ -728,7 +728,7 @@ const struct bpf_func_proto bpf_per_cpu_ptr_proto = { · 6 unmodified lines
728	728
729	729	BPF_CALL_1(bpf_this_cpu_ptr, const void *, percpu_ptr)
730	730	{
731		− return (unsigned long)this_cpu_ptr((const void __percpu *)percpu_ptr);
	731	+ return (unsigned long)this_cpu_ptr((const void __percpu *)(const uintptr_t)percpu_ptr);
732	732	}
733	733
734	734	const struct bpf_func_proto bpf_this_cpu_ptr_proto = {

kernel/bpf/memalloc.c+6−6

@@ -138,8 +138,8 @@ static struct llist_node notrace __llist_del_first(struct llist_head head) · 137 unmodified lines
138	138	static void __alloc(struct bpf_mem_cache c, int node, gfp_t flags)
139	139	{
140	140	if (c->percpu_size) {
141		− void **obj = kmalloc_node(c->percpu_size, flags, node);
142		− void *pptr = __alloc_percpu_gfp(c->unit_size, 8, flags);
	141	+ void __percpu **obj = kmalloc_node(c->percpu_size, flags, node);
	142	+ void __percpu *pptr = __alloc_percpu_gfp(c->unit_size, 8, flags);
143	143
144	144	if (!obj \|\| !pptr) {
145	145	free_percpu(pptr);
@@ -253,7 +253,7 @@ static void alloc_bulk(struct bpf_mem_cache *c, int cnt, int node, bool atomic) · 107 unmodified lines
253	253	static void free_one(void *obj, bool percpu)
254	254	{
255	255	if (percpu) {
256		− free_percpu(((void **)obj)[1]);
	256	+ free_percpu(((void __percpu **)obj)[1]);
257	257	kfree(obj);
258	258	return;
259	259	}
@@ -509,8 +509,8 @@ static void prefill_mem_cache(struct bpf_mem_cache *c, int cpu) · 249 unmodified lines
509	509	*/
510	510	int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu)
511	511	{
512		− struct bpf_mem_caches cc, __percpu pcc;
513		− struct bpf_mem_cache c, __percpu pc;
	512	+ struct bpf_mem_caches cc; struct bpf_mem_caches __percpu pcc;
	513	+ struct bpf_mem_cache c; struct bpf_mem_cache __percpu pc;
514	514	struct obj_cgroup *objcg = NULL;
515	515	int cpu, i, unit_size, percpu_size = 0;
516	516
@@ -591,7 +591,7 @@ int bpf_mem_alloc_percpu_init(struct bpf_mem_alloc ma, struct obj_cgroup objcg · 74 unmodified lines
591	591
592	592	int bpf_mem_alloc_percpu_unit_init(struct bpf_mem_alloc *ma, int size)
593	593	{
594		− struct bpf_mem_caches cc, __percpu pcc;
	594	+ struct bpf_mem_caches cc; struct bpf_mem_caches __percpu pcc;
595	595	int cpu, i, unit_size, percpu_size;
596	596	struct obj_cgroup *objcg;
597	597	struct bpf_mem_cache *c;