bpf: Fix percpu address space issues

In arraymap.c: In bpf_array_map_seq_start() and bpf_array_map_seq_next() cast return values from the __percpu address space to the generic address space via uintptr_t [1]. Correct the declaration of pptr pointer in __bpf_array_map_seq_show() to void __percpu * and cast the value from the generic address space to the __percpu address space via uintptr_t [1]. In hashtab.c: Assign the return value from bpf_mem_cache_alloc() to void pointer and cast the value to void __percpu ** (void pointer to percpu void pointer) before dereferencing. In memalloc.c: Explicitly declare __percpu variables. Cast obj to void __percpu **. In helpers.c: Cast ptr in BPF_CALL_1 and BPF_CALL_2 from generic address space to __percpu address space via const uintptr_t [1]. Found by GCC's named address space checks. There were no changes in the resulting object files. [1] https://sparse.docs.kernel.org/en/latest/annotations.html#address-space-name

Uros Bizjak committed 2 years ago

Commit 6d641ca1 parent 5148f194 files · +17 −16Browse files

Unified · Split

4 files

kernel/bpf/arraymap.c +4 −4
kernel/bpf/hashtab.c +5 −4
kernel/bpf/helpers.c +2 −2
kernel/bpf/memalloc.c +6 −6

kernel/bpf/arraymap.c+4−4

@@ -600,7 +600,7 @@ static void bpf_array_map_seq_start(struct seq_file seq, loff_t *pos) · 599 unmodified lines
600	array = container_of(map, struct bpf_array, map);	600	array = container_of(map, struct bpf_array, map);
601	index = info->index & array->index_mask;	601	index = info->index & array->index_mask;
602	if (info->percpu_value_buf)	602	if (info->percpu_value_buf)
603	return array->pptrs[index];	603	return (void *)(uintptr_t)array->pptrs[index];
604	return array_map_elem_ptr(array, index);	604	return array_map_elem_ptr(array, index);
605	}	605	}
606		606
@@ -619,7 +619,7 @@ static void bpf_array_map_seq_next(struct seq_file seq, void v, loff_t pos) · 12 unmodified lines
619	array = container_of(map, struct bpf_array, map);	619	array = container_of(map, struct bpf_array, map);
620	index = info->index & array->index_mask;	620	index = info->index & array->index_mask;
621	if (info->percpu_value_buf)	621	if (info->percpu_value_buf)
622	return array->pptrs[index];	622	return (void *)(uintptr_t)array->pptrs[index];
623	return array_map_elem_ptr(array, index);	623	return array_map_elem_ptr(array, index);
624	}	624	}
625		625
@@ -632,7 +632,7 @@ static int __bpf_array_map_seq_show(struct seq_file seq, void v) · 6 unmodified lines
632	struct bpf_iter_meta meta;	632	struct bpf_iter_meta meta;
633	struct bpf_prog *prog;	633	struct bpf_prog *prog;
634	int off = 0, cpu = 0;	634	int off = 0, cpu = 0;
635	void __percpu **pptr;	635	void __percpu *pptr;
636	u32 size;	636	u32 size;
637		637
638	meta.seq = seq;	638	meta.seq = seq;
@@ -648,7 +648,7 @@ static int __bpf_array_map_seq_show(struct seq_file seq, void v) · 9 unmodified lines
648	if (!info->percpu_value_buf) {	648	if (!info->percpu_value_buf) {
649	ctx.value = v;	649	ctx.value = v;
650	} else {	650	} else {
651	pptr = v;	651	pptr = (void __percpu *)(uintptr_t)v;
652	size = array->elem_size;	652	size = array->elem_size;
653	for_each_possible_cpu(cpu) {	653	for_each_possible_cpu(cpu) {
654	copy_map_value_long(map, info->percpu_value_buf + off,	654	copy_map_value_long(map, info->percpu_value_buf + off,

kernel/bpf/hashtab.c+5−4

@@ -1049,14 +1049,15 @@ static struct htab_elem alloc_htab_elem(struct bpf_htab htab, void *key, · 1048 unmodified lines
1049	pptr = htab_elem_get_ptr(l_new, key_size);	1049	pptr = htab_elem_get_ptr(l_new, key_size);
1050	} else {	1050	} else {
1051	/* alloc_percpu zero-fills */	1051	/* alloc_percpu zero-fills */
1052	pptr = bpf_mem_cache_alloc(&htab->pcpu_ma);	1052	void *ptr = bpf_mem_cache_alloc(&htab->pcpu_ma);
1053	if (!pptr) {	1053
		1054	if (!ptr) {
1054	bpf_mem_cache_free(&htab->ma, l_new);	1055	bpf_mem_cache_free(&htab->ma, l_new);
1055	l_new = ERR_PTR(-ENOMEM);	1056	l_new = ERR_PTR(-ENOMEM);
1056	goto dec_count;	1057	goto dec_count;
1057	}	1058	}
1058	l_new->ptr_to_pptr = pptr;	1059	l_new->ptr_to_pptr = ptr;
1059	pptr = (void *)pptr;	1060	pptr = (void __percpu *)ptr;
1060	}	1061	}
1061		1062
1062	pcpu_init_value(htab, pptr, value, onallcpus);	1063	pcpu_init_value(htab, pptr, value, onallcpus);

kernel/bpf/helpers.c+2−2

@@ -715,7 +715,7 @@ BPF_CALL_2(bpf_per_cpu_ptr, const void *, ptr, u32, cpu) · 714 unmodified lines
715	if (cpu >= nr_cpu_ids)	715	if (cpu >= nr_cpu_ids)
716	return (unsigned long)NULL;	716	return (unsigned long)NULL;
717		717
718	return (unsigned long)per_cpu_ptr((const void __percpu *)ptr, cpu);	718	return (unsigned long)per_cpu_ptr((const void __percpu *)(const uintptr_t)ptr, cpu);
719	}	719	}
720		720
721	const struct bpf_func_proto bpf_per_cpu_ptr_proto = {	721	const struct bpf_func_proto bpf_per_cpu_ptr_proto = {
@@ -728,7 +728,7 @@ const struct bpf_func_proto bpf_per_cpu_ptr_proto = { · 6 unmodified lines
728		728
729	BPF_CALL_1(bpf_this_cpu_ptr, const void *, percpu_ptr)	729	BPF_CALL_1(bpf_this_cpu_ptr, const void *, percpu_ptr)
730	{	730	{
731	return (unsigned long)this_cpu_ptr((const void __percpu *)percpu_ptr);	731	return (unsigned long)this_cpu_ptr((const void __percpu *)(const uintptr_t)percpu_ptr);
732	}	732	}
733		733
734	const struct bpf_func_proto bpf_this_cpu_ptr_proto = {	734	const struct bpf_func_proto bpf_this_cpu_ptr_proto = {

kernel/bpf/memalloc.c+6−6

@@ -138,8 +138,8 @@ static struct llist_node notrace __llist_del_first(struct llist_head head) · 137 unmodified lines
138	static void __alloc(struct bpf_mem_cache c, int node, gfp_t flags)	138	static void __alloc(struct bpf_mem_cache c, int node, gfp_t flags)
139	{	139	{
140	if (c->percpu_size) {	140	if (c->percpu_size) {
141	void **obj = kmalloc_node(c->percpu_size, flags, node);	141	void __percpu **obj = kmalloc_node(c->percpu_size, flags, node);
142	void *pptr = __alloc_percpu_gfp(c->unit_size, 8, flags);	142	void __percpu *pptr = __alloc_percpu_gfp(c->unit_size, 8, flags);
143		143
144	if (!obj \|\| !pptr) {	144	if (!obj \|\| !pptr) {
145	free_percpu(pptr);	145	free_percpu(pptr);
@@ -253,7 +253,7 @@ static void alloc_bulk(struct bpf_mem_cache *c, int cnt, int node, bool atomic) · 107 unmodified lines
253	static void free_one(void *obj, bool percpu)	253	static void free_one(void *obj, bool percpu)
254	{	254	{
255	if (percpu) {	255	if (percpu) {
256	free_percpu(((void **)obj)[1]);	256	free_percpu(((void __percpu **)obj)[1]);
257	kfree(obj);	257	kfree(obj);
258	return;	258	return;
259	}	259	}
@@ -509,8 +509,8 @@ static void prefill_mem_cache(struct bpf_mem_cache *c, int cpu) · 249 unmodified lines
509	*/	509	*/
510	int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu)	510	int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu)
511	{	511	{
512	struct bpf_mem_caches cc, __percpu pcc;	512	struct bpf_mem_caches cc; struct bpf_mem_caches __percpu pcc;
513	struct bpf_mem_cache c, __percpu pc;	513	struct bpf_mem_cache c; struct bpf_mem_cache __percpu pc;
514	struct obj_cgroup *objcg = NULL;	514	struct obj_cgroup *objcg = NULL;
515	int cpu, i, unit_size, percpu_size = 0;	515	int cpu, i, unit_size, percpu_size = 0;
516		516
@@ -591,7 +591,7 @@ int bpf_mem_alloc_percpu_init(struct bpf_mem_alloc ma, struct obj_cgroup objcg · 74 unmodified lines
591		591
592	int bpf_mem_alloc_percpu_unit_init(struct bpf_mem_alloc *ma, int size)	592	int bpf_mem_alloc_percpu_unit_init(struct bpf_mem_alloc *ma, int size)
593	{	593	{
594	struct bpf_mem_caches cc, __percpu pcc;	594	struct bpf_mem_caches cc; struct bpf_mem_caches __percpu pcc;
595	int cpu, i, unit_size, percpu_size;	595	int cpu, i, unit_size, percpu_size;
596	struct obj_cgroup *objcg;	596	struct obj_cgroup *objcg;
597	struct bpf_mem_cache *c;	597	struct bpf_mem_cache *c;