bpf: Add mmap() support for BPF_MAP_TYPE_ARRAY

Add ability to memory-map contents of BPF array map. This is extremely useful for working with BPF global data from userspace programs. It allows to avoid typical bpf_map_{lookup,update}_elem operations, improving both performance and usability. There had to be special considerations for map freezing, to avoid having writable memory view into a frozen map. To solve this issue, map freezing and mmap-ing is happening under mutex now: - if map is already frozen, no writable mapping is allowed; - if map has writable memory mappings active (accounted in map->writecnt), map freezing will keep failing with -EBUSY; - once number of writable memory mappings drops to zero, map freezing can be performed again. Only non-per-CPU plain arrays are supported right now. Maps with spinlocks can't be memory mapped either. For BPF_F_MMAPABLE array, memory allocation has to be done through vmalloc() to be mmap()'able. We also need to make sure that array data memory is page-sized and page-aligned, so we over-allocate memory in such a way that struct bpf_array is at the end of a single page of memory with array->value being aligned with the start of the second page. On deallocation we need to accomodate this memory arrangement to free vmalloc()'ed memory correctly. One important consideration regarding how memory-mapping subsystem functions. Memory-mapping subsystem provides few optional callbacks, among them open() and close(). close() is called for each memory region that is unmapped, so that users can decrease their reference counters and free up resources, if necessary. open() is *almost* symmetrical: it's called for each memory region that is being mapped, **except** the very first one. So bpf_map_mmap does initial refcnt bump, while open() will do any extra ones after that. Thus number of close() calls is equal to number of open() calls plus one more.

Andrii Nakryiko committed 6 years ago

Commit fc970221 parent 85192db7 files · +183 −12Browse files

Unified · Split

7 files

include/linux/bpf.h +8 −3
include/linux/vmalloc.h +1 −0
include/uapi/linux/bpf.h +3 −0
kernel/bpf/arraymap.c +52 −6
kernel/bpf/syscall.c +96 −3
mm/vmalloc.c +20 −0
tools/include/uapi/linux/bpf.h +3 −0

include/linux/bpf.h+8−3

@@ -12,6 +12,7 @@ · 11 unmodified lines
12	12	#include <linux/err.h>
13	13	#include <linux/rbtree_latch.h>
14	14	#include <linux/numa.h>
	15	+#include <linux/mm_types.h>
15	16	#include <linux/wait.h>
16	17	#include <linux/u64_stats_sync.h>
17	18	#include <linux/refcount.h>
@@ -68,6 +69,7 @@ struct bpf_map_ops { · 50 unmodified lines
68	69	u64 *imm, u32 off);
69	70	int (map_direct_value_meta)(const struct bpf_map map,
70	71	u64 imm, u32 *off);
	72	+ int (map_mmap)(struct bpf_map map, struct vm_area_struct *vma);
71	73	};
72	74
73	75	struct bpf_map_memory {
@@ -96,9 +98,10 @@ struct bpf_map { · 22 unmodified lines
96	98	u32 btf_value_type_id;
97	99	struct btf *btf;
98	100	struct bpf_map_memory memory;
	101	+ char name[BPF_OBJ_NAME_LEN];
99	102	bool unpriv_array;
100		− bool frozen; /* write-once */
101		− /* 48 bytes hole */
	103	+ bool frozen; /* write-once; write-protected by freeze_mutex */
	104	+ /* 22 bytes hole */
102	105
103	106	/* The 3rd and 4th cacheline with misc members to avoid false sharing
104	107	* particularly with refcounting.
@@ -106,7 +109,8 @@ struct bpf_map { · 1 unmodified line
106	109	atomic64_t refcnt ____cacheline_aligned;
107	110	atomic64_t usercnt;
108	111	struct work_struct work;
109		− char name[BPF_OBJ_NAME_LEN];
	112	+ struct mutex freeze_mutex;
	113	+ u64 writecnt; /* writable mmap cnt; protected by freeze_mutex */
110	114	};
111	115
112	116	static inline bool map_value_has_spin_lock(const struct bpf_map *map)
@@ -795,6 +799,7 @@ void bpf_map_charge_finish(struct bpf_map_memory *mem); · 682 unmodified lines
795	799	void bpf_map_charge_move(struct bpf_map_memory *dst,
796	800	struct bpf_map_memory *src);
797	801	void *bpf_map_area_alloc(size_t size, int numa_node);
	802	+void *bpf_map_area_mmapable_alloc(size_t size, int numa_node);
798	803	void bpf_map_area_free(void *base);
799	804	void bpf_map_init_from_attr(struct bpf_map map, union bpf_attr attr);
800	805

include/linux/vmalloc.h+1−0

@@ -93,6 +93,7 @@ extern void *vzalloc(unsigned long size); · 92 unmodified lines
93	93	extern void *vmalloc_user(unsigned long size);
94	94	extern void *vmalloc_node(unsigned long size, int node);
95	95	extern void *vzalloc_node(unsigned long size, int node);
	96	+extern void *vmalloc_user_node_flags(unsigned long size, int node, gfp_t flags);
96	97	extern void *vmalloc_exec(unsigned long size);
97	98	extern void *vmalloc_32(unsigned long size);
98	99	extern void *vmalloc_32_user(unsigned long size);

include/uapi/linux/bpf.h+3−0

@@ -348,6 +348,9 @@ enum bpf_attach_type { · 347 unmodified lines
348	348	/* Clone map from listener for newly accepted socket */
349	349	#define BPF_F_CLONE (1U << 9)
350	350
	351	+/* Enable memory-mapping BPF map */
	352	+#define BPF_F_MMAPABLE (1U << 10)
	353	+
351	354	/* flags for BPF_PROG_QUERY */
352	355	#define BPF_F_QUERY_EFFECTIVE (1U << 0)
353	356

kernel/bpf/arraymap.c+52−6

@@ -14,7 +14,7 @@ · 13 unmodified lines
14	14	#include "map_in_map.h"
15	15
16	16	#define ARRAY_CREATE_FLAG_MASK \
17		− (BPF_F_NUMA_NODE \| BPF_F_ACCESS_MASK)
	17	+ (BPF_F_NUMA_NODE \| BPF_F_MMAPABLE \| BPF_F_ACCESS_MASK)
18	18
19	19	static void bpf_array_free_percpu(struct bpf_array *array)
20	20	{
@@ -59,6 +59,10 @@ int array_map_alloc_check(union bpf_attr *attr) · 38 unmodified lines
59	59	(percpu && numa_node != NUMA_NO_NODE))
60	60	return -EINVAL;
61	61
	62	+ if (attr->map_type != BPF_MAP_TYPE_ARRAY &&
	63	+ attr->map_flags & BPF_F_MMAPABLE)
	64	+ return -EINVAL;
	65	+
62	66	if (attr->value_size > KMALLOC_MAX_SIZE)
63	67	/* if value_size is bigger, the user space won't be able to
64	68	* access the elements.
@@ -102,10 +106,19 @@ static struct bpf_map array_map_alloc(union bpf_attr attr) · 37 unmodified lines
102	106	}
103	107
104	108	array_size = sizeof(*array);
105		− if (percpu)
	109	+ if (percpu) {
106	110	array_size += (u64) max_entries * sizeof(void *);
107		− else
108		− array_size += (u64) max_entries * elem_size;
	111	+ } else {
	112	+ /* rely on vmalloc() to return page-aligned memory and
	113	+ * ensure array->value is exactly page-aligned
	114	+ */
	115	+ if (attr->map_flags & BPF_F_MMAPABLE) {
	116	+ array_size = PAGE_ALIGN(array_size);
	117	+ array_size += PAGE_ALIGN((u64) max_entries * elem_size);
	118	+ } else {
	119	+ array_size += (u64) max_entries * elem_size;
	120	+ }
	121	+ }
109	122
110	123	/* make sure there is no u32 overflow later in round_up() */
111	124	cost = array_size;
@@ -117,7 +130,20 @@ static struct bpf_map array_map_alloc(union bpf_attr attr) · 5 unmodified lines
117	130	return ERR_PTR(ret);
118	131
119	132	/* allocate all map elements and zero-initialize them */
120		− array = bpf_map_area_alloc(array_size, numa_node);
	133	+ if (attr->map_flags & BPF_F_MMAPABLE) {
	134	+ void *data;
	135	+
	136	+ /* kmalloc'ed memory can't be mmap'ed, use explicit vmalloc */
	137	+ data = bpf_map_area_mmapable_alloc(array_size, numa_node);
	138	+ if (!data) {
	139	+ bpf_map_charge_finish(&mem);
	140	+ return ERR_PTR(-ENOMEM);
	141	+ }
	142	+ array = data + PAGE_ALIGN(sizeof(struct bpf_array))
	143	+ - offsetof(struct bpf_array, value);
	144	+ } else {
	145	+ array = bpf_map_area_alloc(array_size, numa_node);
	146	+ }
121	147	if (!array) {
122	148	bpf_map_charge_finish(&mem);
123	149	return ERR_PTR(-ENOMEM);
@@ -350,6 +376,11 @@ static int array_map_delete_elem(struct bpf_map map, void key) · 226 unmodified lines
350	376	return -EINVAL;
351	377	}
352	378
	379	+static void array_map_vmalloc_addr(struct bpf_array array)
	380	+{
	381	+ return (void *)round_down((unsigned long)array, PAGE_SIZE);
	382	+}
	383	+
353	384	/* Called when map->refcnt goes to zero, either from workqueue or from syscall */
354	385	static void array_map_free(struct bpf_map *map)
355	386	{
@@ -365,7 +396,10 @@ static void array_map_free(struct bpf_map *map) · 9 unmodified lines
365	396	if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
366	397	bpf_array_free_percpu(array);
367	398
368		− bpf_map_area_free(array);
	399	+ if (array->map.map_flags & BPF_F_MMAPABLE)
	400	+ bpf_map_area_free(array_map_vmalloc_addr(array));
	401	+ else
	402	+ bpf_map_area_free(array);
369	403	}
370	404
371	405	static void array_map_seq_show_elem(struct bpf_map map, void key,
@@ -444,6 +478,17 @@ static int array_map_check_btf(const struct bpf_map *map, · 72 unmodified lines
444	478	return 0;
445	479	}
446	480
	481	+int array_map_mmap(struct bpf_map map, struct vm_area_struct vma)
	482	+{
	483	+ struct bpf_array *array = container_of(map, struct bpf_array, map);
	484	+ pgoff_t pgoff = PAGE_ALIGN(sizeof(*array)) >> PAGE_SHIFT;
	485	+
	486	+ if (!(map->map_flags & BPF_F_MMAPABLE))
	487	+ return -EINVAL;
	488	+
	489	+ return remap_vmalloc_range(vma, array_map_vmalloc_addr(array), pgoff);
	490	+}
	491	+
447	492	const struct bpf_map_ops array_map_ops = {
448	493	.map_alloc_check = array_map_alloc_check,
449	494	.map_alloc = array_map_alloc,
@@ -455,6 +500,7 @@ const struct bpf_map_ops array_map_ops = { · 5 unmodified lines
455	500	.map_gen_lookup = array_map_gen_lookup,
456	501	.map_direct_value_addr = array_map_direct_value_addr,
457	502	.map_direct_value_meta = array_map_direct_value_meta,
	503	+ .map_mmap = array_map_mmap,
458	504	.map_seq_show_elem = array_map_seq_show_elem,
459	505	.map_check_btf = array_map_check_btf,
460	506	};

kernel/bpf/syscall.c+96−3

@@ -127,7 +127,7 @@ static struct bpf_map find_and_alloc_map(union bpf_attr attr) · 126 unmodified lines
127	127	return map;
128	128	}
129	129
130		−void *bpf_map_area_alloc(size_t size, int numa_node)
	130	+static void *__bpf_map_area_alloc(size_t size, int numa_node, bool mmapable)
131	131	{
132	132	/* We really just want to fail instead of triggering OOM killer
133	133	* under memory pressure, therefore we set __GFP_NORETRY to kmalloc,
@@ -142,18 +142,33 @@ void *bpf_map_area_alloc(size_t size, int numa_node) · 8 unmodified lines
142	142	const gfp_t flags = __GFP_NOWARN \| __GFP_ZERO;
143	143	void *area;
144	144
145		− if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) {
	145	+ /* kmalloc()'ed memory can't be mmap()'ed */
	146	+ if (!mmapable && size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) {
146	147	area = kmalloc_node(size, GFP_USER \| __GFP_NORETRY \| flags,
147	148	numa_node);
148	149	if (area != NULL)
149	150	return area;
150	151	}
151		−
	152	+ if (mmapable) {
	153	+ BUG_ON(!PAGE_ALIGNED(size));
	154	+ return vmalloc_user_node_flags(size, numa_node, GFP_KERNEL \|
	155	+ __GFP_RETRY_MAYFAIL \| flags);
	156	+ }
152	157	return __vmalloc_node_flags_caller(size, numa_node,
153	158	GFP_KERNEL \| __GFP_RETRY_MAYFAIL \|
154	159	flags, __builtin_return_address(0));
155	160	}
156	161
	162	+void *bpf_map_area_alloc(size_t size, int numa_node)
	163	+{
	164	+ return __bpf_map_area_alloc(size, numa_node, false);
	165	+}
	166	+
	167	+void *bpf_map_area_mmapable_alloc(size_t size, int numa_node)
	168	+{
	169	+ return __bpf_map_area_alloc(size, numa_node, true);
	170	+}
	171	+
157	172	void bpf_map_area_free(void *area)
158	173	{
159	174	kvfree(area);
@@ -425,6 +440,74 @@ static ssize_t bpf_dummy_write(struct file filp, const char __user buf, · 265 unmodified lines
425	440	return -EINVAL;
426	441	}
427	442
	443	+/* called for any extra memory-mapped regions (except initial) */
	444	+static void bpf_map_mmap_open(struct vm_area_struct *vma)
	445	+{
	446	+ struct bpf_map *map = vma->vm_file->private_data;
	447	+
	448	+ bpf_map_inc_with_uref(map);
	449	+
	450	+ if (vma->vm_flags & VM_WRITE) {
	451	+ mutex_lock(&map->freeze_mutex);
	452	+ map->writecnt++;
	453	+ mutex_unlock(&map->freeze_mutex);
	454	+ }
	455	+}
	456	+
	457	+/* called for all unmapped memory region (including initial) */
	458	+static void bpf_map_mmap_close(struct vm_area_struct *vma)
	459	+{
	460	+ struct bpf_map *map = vma->vm_file->private_data;
	461	+
	462	+ if (vma->vm_flags & VM_WRITE) {
	463	+ mutex_lock(&map->freeze_mutex);
	464	+ map->writecnt--;
	465	+ mutex_unlock(&map->freeze_mutex);
	466	+ }
	467	+
	468	+ bpf_map_put_with_uref(map);
	469	+}
	470	+
	471	+static const struct vm_operations_struct bpf_map_default_vmops = {
	472	+ .open = bpf_map_mmap_open,
	473	+ .close = bpf_map_mmap_close,
	474	+};
	475	+
	476	+static int bpf_map_mmap(struct file filp, struct vm_area_struct vma)
	477	+{
	478	+ struct bpf_map *map = filp->private_data;
	479	+ int err;
	480	+
	481	+ if (!map->ops->map_mmap \|\| map_value_has_spin_lock(map))
	482	+ return -ENOTSUPP;
	483	+
	484	+ if (!(vma->vm_flags & VM_SHARED))
	485	+ return -EINVAL;
	486	+
	487	+ mutex_lock(&map->freeze_mutex);
	488	+
	489	+ if ((vma->vm_flags & VM_WRITE) && map->frozen) {
	490	+ err = -EPERM;
	491	+ goto out;
	492	+ }
	493	+
	494	+ /* set default open/close callbacks */
	495	+ vma->vm_ops = &bpf_map_default_vmops;
	496	+ vma->vm_private_data = map;
	497	+
	498	+ err = map->ops->map_mmap(map, vma);
	499	+ if (err)
	500	+ goto out;
	501	+
	502	+ bpf_map_inc_with_uref(map);
	503	+
	504	+ if (vma->vm_flags & VM_WRITE)
	505	+ map->writecnt++;
	506	+out:
	507	+ mutex_unlock(&map->freeze_mutex);
	508	+ return err;
	509	+}
	510	+
428	511	const struct file_operations bpf_map_fops = {
429	512	#ifdef CONFIG_PROC_FS
430	513	.show_fdinfo = bpf_map_show_fdinfo,
@@ -432,6 +515,7 @@ const struct file_operations bpf_map_fops = { · 1 unmodified line
432	515	.release = bpf_map_release,
433	516	.read = bpf_dummy_read,
434	517	.write = bpf_dummy_write,
	518	+ .mmap = bpf_map_mmap,
435	519	};
436	520
437	521	int bpf_map_new_fd(struct bpf_map *map, int flags)
@@ -577,6 +661,7 @@ static int map_create(union bpf_attr *attr) · 139 unmodified lines
577	661
578	662	atomic64_set(&map->refcnt, 1);
579	663	atomic64_set(&map->usercnt, 1);
	664	+ mutex_init(&map->freeze_mutex);
580	665
581	666	if (attr->btf_key_type_id \|\| attr->btf_value_type_id) {
582	667	struct btf *btf;
@@ -1163,6 +1248,13 @@ static int map_freeze(const union bpf_attr *attr) · 580 unmodified lines
1163	1248	map = __bpf_map_get(f);
1164	1249	if (IS_ERR(map))
1165	1250	return PTR_ERR(map);
	1251	+
	1252	+ mutex_lock(&map->freeze_mutex);
	1253	+
	1254	+ if (map->writecnt) {
	1255	+ err = -EBUSY;
	1256	+ goto err_put;
	1257	+ }
1166	1258	if (READ_ONCE(map->frozen)) {
1167	1259	err = -EBUSY;
1168	1260	goto err_put;
@@ -1174,6 +1266,7 @@ static int map_freeze(const union bpf_attr *attr) · 5 unmodified lines
1174	1266
1175	1267	WRITE_ONCE(map->frozen, true);
1176	1268	err_put:
	1269	+ mutex_unlock(&map->freeze_mutex);
1177	1270	fdput(f);
1178	1271	return err;
1179	1272	}

mm/vmalloc.c+20−0

@@ -2671,6 +2671,26 @@ void *vzalloc_node(unsigned long size, int node) · 2670 unmodified lines
2671	2671	}
2672	2672	EXPORT_SYMBOL(vzalloc_node);
2673	2673
	2674	+/**
	2675	+ * vmalloc_user_node_flags - allocate memory for userspace on a specific node
	2676	+ * @size: allocation size
	2677	+ * @node: numa node
	2678	+ * @flags: flags for the page level allocator
	2679	+ *
	2680	+ * The resulting memory area is zeroed so it can be mapped to userspace
	2681	+ * without leaking data.
	2682	+ *
	2683	+ * Return: pointer to the allocated memory or %NULL on error
	2684	+ */
	2685	+void *vmalloc_user_node_flags(unsigned long size, int node, gfp_t flags)
	2686	+{
	2687	+ return __vmalloc_node_range(size, SHMLBA, VMALLOC_START, VMALLOC_END,
	2688	+ flags \| __GFP_ZERO, PAGE_KERNEL,
	2689	+ VM_USERMAP, node,
	2690	+ __builtin_return_address(0));
	2691	+}
	2692	+EXPORT_SYMBOL(vmalloc_user_node_flags);
	2693	+
2674	2694	/**
2675	2695	* vmalloc_exec - allocate virtually contiguous, executable memory
2676	2696	* @size: allocation size

tools/include/uapi/linux/bpf.h+3−0

@@ -348,6 +348,9 @@ enum bpf_attach_type { · 347 unmodified lines
348	348	/* Clone map from listener for newly accepted socket */
349	349	#define BPF_F_CLONE (1U << 9)
350	350
	351	+/* Enable memory-mapping BPF map */
	352	+#define BPF_F_MMAPABLE (1U << 10)
	353	+
351	354	/* flags for BPF_PROG_QUERY */
352	355	#define BPF_F_QUERY_EFFECTIVE (1U << 0)
353	356