History

bpf: Add mmap() support for BPF_MAP_TYPE_ARRAY

Add ability to memory-map contents of BPF array map. This is extremely useful for working with BPF global data from userspace programs. It allows to avoid typical bpf_map_{lookup,update}_elem operations, improving both performance and usability. There had to be special considerations for map freezing, to avoid having writable memory view into a frozen map. To solve this issue, map freezing and mmap-ing is happening under mutex now: - if map is already frozen, no writable mapping is allowed; - if map has writable memory mappings active (accounted in map->writecnt), map freezing will keep failing with -EBUSY; - once number of writable memory mappings drops to zero, map freezing can be performed again. Only non-per-CPU plain arrays are supported right now. Maps with spinlocks can't be memory mapped either. For BPF_F_MMAPABLE array, memory allocation has to be done through vmalloc() to be mmap()'able. We also need to make sure that array data memory is page-sized and page-aligned, so we over-allocate memory in such a way that struct bpf_array is at the end of a single page of memory with array->value being aligned with the start of the second page. On deallocation we need to accomodate this memory arrangement to free vmalloc()'ed memory correctly. One important consideration regarding how memory-mapping subsystem functions. Memory-mapping subsystem provides few optional callbacks, among them open() and close(). close() is called for each memory region that is unmapped, so that users can decrease their reference counters and free up resources, if necessary. open() is *almost* symmetrical: it's called for each memory region that is being mapped, **except** the very first one. So bpf_map_mmap does initial refcnt bump, while open() will do any extra ones after that. Thus number of close() calls is equal to number of open() calls plus one more.

Andrii Nakryiko committed 6 years ago

Commit fc970221 parent 85192db7 files · +183 −12Browse files

Unified · Split

7 files
  1. include/linux/bpf.h +8 −3
  2. include/linux/vmalloc.h +1 −0
  3. include/uapi/linux/bpf.h +3 −0
  4. kernel/bpf/arraymap.c +52 −6
  5. kernel/bpf/syscall.c +96 −3
  6. mm/vmalloc.c +20 −0
  7. tools/include/uapi/linux/bpf.h +3 −0
include/linux/bpf.h+8−3
@@ -12,6 +12,7 @@ · 11 unmodified lines
12#include <linux/err.h>12#include <linux/err.h>
13#include <linux/rbtree_latch.h>13#include <linux/rbtree_latch.h>
14#include <linux/numa.h>14#include <linux/numa.h>
15#include <linux/mm_types.h>
15#include <linux/wait.h>16#include <linux/wait.h>
16#include <linux/u64_stats_sync.h>17#include <linux/u64_stats_sync.h>
17#include <linux/refcount.h>18#include <linux/refcount.h>
@@ -68,6 +69,7 @@ struct bpf_map_ops { · 50 unmodified lines
68 u64 *imm, u32 off);69 u64 *imm, u32 off);
69 int (*map_direct_value_meta)(const struct bpf_map *map,70 int (*map_direct_value_meta)(const struct bpf_map *map,
70 u64 imm, u32 *off);71 u64 imm, u32 *off);
72 int (*map_mmap)(struct bpf_map *map, struct vm_area_struct *vma);
71};73};
72 74
73struct bpf_map_memory {75struct bpf_map_memory {
@@ -96,9 +98,10 @@ struct bpf_map { · 22 unmodified lines
96 u32 btf_value_type_id;98 u32 btf_value_type_id;
97 struct btf *btf;99 struct btf *btf;
98 struct bpf_map_memory memory;100 struct bpf_map_memory memory;
101 char name[BPF_OBJ_NAME_LEN];
99 bool unpriv_array;102 bool unpriv_array;
100 bool frozen; /* write-once */103 bool frozen; /* write-once; write-protected by freeze_mutex */
101 /* 48 bytes hole */104 /* 22 bytes hole */
102 105
103 /* The 3rd and 4th cacheline with misc members to avoid false sharing106 /* The 3rd and 4th cacheline with misc members to avoid false sharing
104 * particularly with refcounting.107 * particularly with refcounting.
@@ -106,7 +109,8 @@ struct bpf_map { · 1 unmodified line
106 atomic64_t refcnt ____cacheline_aligned;109 atomic64_t refcnt ____cacheline_aligned;
107 atomic64_t usercnt;110 atomic64_t usercnt;
108 struct work_struct work;111 struct work_struct work;
109 char name[BPF_OBJ_NAME_LEN];112 struct mutex freeze_mutex;
113 u64 writecnt; /* writable mmap cnt; protected by freeze_mutex */
110};114};
111 115
112static inline bool map_value_has_spin_lock(const struct bpf_map *map)116static inline bool map_value_has_spin_lock(const struct bpf_map *map)
@@ -795,6 +799,7 @@ void bpf_map_charge_finish(struct bpf_map_memory *mem); · 682 unmodified lines
795void bpf_map_charge_move(struct bpf_map_memory *dst,799void bpf_map_charge_move(struct bpf_map_memory *dst,
796 struct bpf_map_memory *src);800 struct bpf_map_memory *src);
797void *bpf_map_area_alloc(size_t size, int numa_node);801void *bpf_map_area_alloc(size_t size, int numa_node);
802void *bpf_map_area_mmapable_alloc(size_t size, int numa_node);
798void bpf_map_area_free(void *base);803void bpf_map_area_free(void *base);
799void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr);804void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr);
800 805
include/linux/vmalloc.h+1−0
@@ -93,6 +93,7 @@ extern void *vzalloc(unsigned long size); · 92 unmodified lines
93extern void *vmalloc_user(unsigned long size);93extern void *vmalloc_user(unsigned long size);
94extern void *vmalloc_node(unsigned long size, int node);94extern void *vmalloc_node(unsigned long size, int node);
95extern void *vzalloc_node(unsigned long size, int node);95extern void *vzalloc_node(unsigned long size, int node);
96extern void *vmalloc_user_node_flags(unsigned long size, int node, gfp_t flags);
96extern void *vmalloc_exec(unsigned long size);97extern void *vmalloc_exec(unsigned long size);
97extern void *vmalloc_32(unsigned long size);98extern void *vmalloc_32(unsigned long size);
98extern void *vmalloc_32_user(unsigned long size);99extern void *vmalloc_32_user(unsigned long size);
include/uapi/linux/bpf.h+3−0
@@ -348,6 +348,9 @@ enum bpf_attach_type { · 347 unmodified lines
348/* Clone map from listener for newly accepted socket */348/* Clone map from listener for newly accepted socket */
349#define BPF_F_CLONE (1U << 9)349#define BPF_F_CLONE (1U << 9)
350 350
351/* Enable memory-mapping BPF map */
352#define BPF_F_MMAPABLE (1U << 10)
353
351/* flags for BPF_PROG_QUERY */354/* flags for BPF_PROG_QUERY */
352#define BPF_F_QUERY_EFFECTIVE (1U << 0)355#define BPF_F_QUERY_EFFECTIVE (1U << 0)
353 356
kernel/bpf/arraymap.c+52−6
@@ -14,7 +14,7 @@ · 13 unmodified lines
14#include "map_in_map.h"14#include "map_in_map.h"
15 15
16#define ARRAY_CREATE_FLAG_MASK \16#define ARRAY_CREATE_FLAG_MASK \
17 (BPF_F_NUMA_NODE | BPF_F_ACCESS_MASK)17 (BPF_F_NUMA_NODE | BPF_F_MMAPABLE | BPF_F_ACCESS_MASK)
18 18
19static void bpf_array_free_percpu(struct bpf_array *array)19static void bpf_array_free_percpu(struct bpf_array *array)
20{20{
@@ -59,6 +59,10 @@ int array_map_alloc_check(union bpf_attr *attr) · 38 unmodified lines
59 (percpu && numa_node != NUMA_NO_NODE))59 (percpu && numa_node != NUMA_NO_NODE))
60 return -EINVAL;60 return -EINVAL;
61 61
62 if (attr->map_type != BPF_MAP_TYPE_ARRAY &&
63 attr->map_flags & BPF_F_MMAPABLE)
64 return -EINVAL;
65
62 if (attr->value_size > KMALLOC_MAX_SIZE)66 if (attr->value_size > KMALLOC_MAX_SIZE)
63 /* if value_size is bigger, the user space won't be able to67 /* if value_size is bigger, the user space won't be able to
64 * access the elements.68 * access the elements.
@@ -102,10 +106,19 @@ static struct bpf_map *array_map_alloc(union bpf_attr *attr) · 37 unmodified lines
102 }106 }
103 107
104 array_size = sizeof(*array);108 array_size = sizeof(*array);
105 if (percpu)109 if (percpu) {
106 array_size += (u64) max_entries * sizeof(void *);110 array_size += (u64) max_entries * sizeof(void *);
107 else111 } else {
108 array_size += (u64) max_entries * elem_size;112 /* rely on vmalloc() to return page-aligned memory and
113 * ensure array->value is exactly page-aligned
114 */
115 if (attr->map_flags & BPF_F_MMAPABLE) {
116 array_size = PAGE_ALIGN(array_size);
117 array_size += PAGE_ALIGN((u64) max_entries * elem_size);
118 } else {
119 array_size += (u64) max_entries * elem_size;
120 }
121 }
109 122
110 /* make sure there is no u32 overflow later in round_up() */123 /* make sure there is no u32 overflow later in round_up() */
111 cost = array_size;124 cost = array_size;
@@ -117,7 +130,20 @@ static struct bpf_map *array_map_alloc(union bpf_attr *attr) · 5 unmodified lines
117 return ERR_PTR(ret);130 return ERR_PTR(ret);
118 131
119 /* allocate all map elements and zero-initialize them */132 /* allocate all map elements and zero-initialize them */
120 array = bpf_map_area_alloc(array_size, numa_node);133 if (attr->map_flags & BPF_F_MMAPABLE) {
134 void *data;
135
136 /* kmalloc'ed memory can't be mmap'ed, use explicit vmalloc */
137 data = bpf_map_area_mmapable_alloc(array_size, numa_node);
138 if (!data) {
139 bpf_map_charge_finish(&mem);
140 return ERR_PTR(-ENOMEM);
141 }
142 array = data + PAGE_ALIGN(sizeof(struct bpf_array))
143 - offsetof(struct bpf_array, value);
144 } else {
145 array = bpf_map_area_alloc(array_size, numa_node);
146 }
121 if (!array) {147 if (!array) {
122 bpf_map_charge_finish(&mem);148 bpf_map_charge_finish(&mem);
123 return ERR_PTR(-ENOMEM);149 return ERR_PTR(-ENOMEM);
@@ -350,6 +376,11 @@ static int array_map_delete_elem(struct bpf_map *map, void *key) · 226 unmodified lines
350 return -EINVAL;376 return -EINVAL;
351}377}
352 378
379static void *array_map_vmalloc_addr(struct bpf_array *array)
380{
381 return (void *)round_down((unsigned long)array, PAGE_SIZE);
382}
383
353/* Called when map->refcnt goes to zero, either from workqueue or from syscall */384/* Called when map->refcnt goes to zero, either from workqueue or from syscall */
354static void array_map_free(struct bpf_map *map)385static void array_map_free(struct bpf_map *map)
355{386{
@@ -365,7 +396,10 @@ static void array_map_free(struct bpf_map *map) · 9 unmodified lines
365 if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY)396 if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
366 bpf_array_free_percpu(array);397 bpf_array_free_percpu(array);
367 398
368 bpf_map_area_free(array);399 if (array->map.map_flags & BPF_F_MMAPABLE)
400 bpf_map_area_free(array_map_vmalloc_addr(array));
401 else
402 bpf_map_area_free(array);
369}403}
370 404
371static void array_map_seq_show_elem(struct bpf_map *map, void *key,405static void array_map_seq_show_elem(struct bpf_map *map, void *key,
@@ -444,6 +478,17 @@ static int array_map_check_btf(const struct bpf_map *map, · 72 unmodified lines
444 return 0;478 return 0;
445}479}
446 480
481int array_map_mmap(struct bpf_map *map, struct vm_area_struct *vma)
482{
483 struct bpf_array *array = container_of(map, struct bpf_array, map);
484 pgoff_t pgoff = PAGE_ALIGN(sizeof(*array)) >> PAGE_SHIFT;
485
486 if (!(map->map_flags & BPF_F_MMAPABLE))
487 return -EINVAL;
488
489 return remap_vmalloc_range(vma, array_map_vmalloc_addr(array), pgoff);
490}
491
447const struct bpf_map_ops array_map_ops = {492const struct bpf_map_ops array_map_ops = {
448 .map_alloc_check = array_map_alloc_check,493 .map_alloc_check = array_map_alloc_check,
449 .map_alloc = array_map_alloc,494 .map_alloc = array_map_alloc,
@@ -455,6 +500,7 @@ const struct bpf_map_ops array_map_ops = { · 5 unmodified lines
455 .map_gen_lookup = array_map_gen_lookup,500 .map_gen_lookup = array_map_gen_lookup,
456 .map_direct_value_addr = array_map_direct_value_addr,501 .map_direct_value_addr = array_map_direct_value_addr,
457 .map_direct_value_meta = array_map_direct_value_meta,502 .map_direct_value_meta = array_map_direct_value_meta,
503 .map_mmap = array_map_mmap,
458 .map_seq_show_elem = array_map_seq_show_elem,504 .map_seq_show_elem = array_map_seq_show_elem,
459 .map_check_btf = array_map_check_btf,505 .map_check_btf = array_map_check_btf,
460};506};
kernel/bpf/syscall.c+96−3
@@ -127,7 +127,7 @@ static struct bpf_map *find_and_alloc_map(union bpf_attr *attr) · 126 unmodified lines
127 return map;127 return map;
128}128}
129 129
130void *bpf_map_area_alloc(size_t size, int numa_node)130static void *__bpf_map_area_alloc(size_t size, int numa_node, bool mmapable)
131{131{
132 /* We really just want to fail instead of triggering OOM killer132 /* We really just want to fail instead of triggering OOM killer
133 * under memory pressure, therefore we set __GFP_NORETRY to kmalloc,133 * under memory pressure, therefore we set __GFP_NORETRY to kmalloc,
@@ -142,18 +142,33 @@ void *bpf_map_area_alloc(size_t size, int numa_node) · 8 unmodified lines
142 const gfp_t flags = __GFP_NOWARN | __GFP_ZERO;142 const gfp_t flags = __GFP_NOWARN | __GFP_ZERO;
143 void *area;143 void *area;
144 144
145 if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) {145 /* kmalloc()'ed memory can't be mmap()'ed */
146 if (!mmapable && size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) {
146 area = kmalloc_node(size, GFP_USER | __GFP_NORETRY | flags,147 area = kmalloc_node(size, GFP_USER | __GFP_NORETRY | flags,
147 numa_node);148 numa_node);
148 if (area != NULL)149 if (area != NULL)
149 return area;150 return area;
150 }151 }
151 152 if (mmapable) {
153 BUG_ON(!PAGE_ALIGNED(size));
154 return vmalloc_user_node_flags(size, numa_node, GFP_KERNEL |
155 __GFP_RETRY_MAYFAIL | flags);
156 }
152 return __vmalloc_node_flags_caller(size, numa_node,157 return __vmalloc_node_flags_caller(size, numa_node,
153 GFP_KERNEL | __GFP_RETRY_MAYFAIL |158 GFP_KERNEL | __GFP_RETRY_MAYFAIL |
154 flags, __builtin_return_address(0));159 flags, __builtin_return_address(0));
155}160}
156 161
162void *bpf_map_area_alloc(size_t size, int numa_node)
163{
164 return __bpf_map_area_alloc(size, numa_node, false);
165}
166
167void *bpf_map_area_mmapable_alloc(size_t size, int numa_node)
168{
169 return __bpf_map_area_alloc(size, numa_node, true);
170}
171
157void bpf_map_area_free(void *area)172void bpf_map_area_free(void *area)
158{173{
159 kvfree(area);174 kvfree(area);
@@ -425,6 +440,74 @@ static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf, · 265 unmodified lines
425 return -EINVAL;440 return -EINVAL;
426}441}
427 442
443/* called for any extra memory-mapped regions (except initial) */
444static void bpf_map_mmap_open(struct vm_area_struct *vma)
445{
446 struct bpf_map *map = vma->vm_file->private_data;
447
448 bpf_map_inc_with_uref(map);
449
450 if (vma->vm_flags & VM_WRITE) {
451 mutex_lock(&map->freeze_mutex);
452 map->writecnt++;
453 mutex_unlock(&map->freeze_mutex);
454 }
455}
456
457/* called for all unmapped memory region (including initial) */
458static void bpf_map_mmap_close(struct vm_area_struct *vma)
459{
460 struct bpf_map *map = vma->vm_file->private_data;
461
462 if (vma->vm_flags & VM_WRITE) {
463 mutex_lock(&map->freeze_mutex);
464 map->writecnt--;
465 mutex_unlock(&map->freeze_mutex);
466 }
467
468 bpf_map_put_with_uref(map);
469}
470
471static const struct vm_operations_struct bpf_map_default_vmops = {
472 .open = bpf_map_mmap_open,
473 .close = bpf_map_mmap_close,
474};
475
476static int bpf_map_mmap(struct file *filp, struct vm_area_struct *vma)
477{
478 struct bpf_map *map = filp->private_data;
479 int err;
480
481 if (!map->ops->map_mmap || map_value_has_spin_lock(map))
482 return -ENOTSUPP;
483
484 if (!(vma->vm_flags & VM_SHARED))
485 return -EINVAL;
486
487 mutex_lock(&map->freeze_mutex);
488
489 if ((vma->vm_flags & VM_WRITE) && map->frozen) {
490 err = -EPERM;
491 goto out;
492 }
493
494 /* set default open/close callbacks */
495 vma->vm_ops = &bpf_map_default_vmops;
496 vma->vm_private_data = map;
497
498 err = map->ops->map_mmap(map, vma);
499 if (err)
500 goto out;
501
502 bpf_map_inc_with_uref(map);
503
504 if (vma->vm_flags & VM_WRITE)
505 map->writecnt++;
506out:
507 mutex_unlock(&map->freeze_mutex);
508 return err;
509}
510
428const struct file_operations bpf_map_fops = {511const struct file_operations bpf_map_fops = {
429#ifdef CONFIG_PROC_FS512#ifdef CONFIG_PROC_FS
430 .show_fdinfo = bpf_map_show_fdinfo,513 .show_fdinfo = bpf_map_show_fdinfo,
@@ -432,6 +515,7 @@ const struct file_operations bpf_map_fops = { · 1 unmodified line
432 .release = bpf_map_release,515 .release = bpf_map_release,
433 .read = bpf_dummy_read,516 .read = bpf_dummy_read,
434 .write = bpf_dummy_write,517 .write = bpf_dummy_write,
518 .mmap = bpf_map_mmap,
435};519};
436 520
437int bpf_map_new_fd(struct bpf_map *map, int flags)521int bpf_map_new_fd(struct bpf_map *map, int flags)
@@ -577,6 +661,7 @@ static int map_create(union bpf_attr *attr) · 139 unmodified lines
577 661
578 atomic64_set(&map->refcnt, 1);662 atomic64_set(&map->refcnt, 1);
579 atomic64_set(&map->usercnt, 1);663 atomic64_set(&map->usercnt, 1);
664 mutex_init(&map->freeze_mutex);
580 665
581 if (attr->btf_key_type_id || attr->btf_value_type_id) {666 if (attr->btf_key_type_id || attr->btf_value_type_id) {
582 struct btf *btf;667 struct btf *btf;
@@ -1163,6 +1248,13 @@ static int map_freeze(const union bpf_attr *attr) · 580 unmodified lines
1163 map = __bpf_map_get(f);1248 map = __bpf_map_get(f);
1164 if (IS_ERR(map))1249 if (IS_ERR(map))
1165 return PTR_ERR(map);1250 return PTR_ERR(map);
1251
1252 mutex_lock(&map->freeze_mutex);
1253
1254 if (map->writecnt) {
1255 err = -EBUSY;
1256 goto err_put;
1257 }
1166 if (READ_ONCE(map->frozen)) {1258 if (READ_ONCE(map->frozen)) {
1167 err = -EBUSY;1259 err = -EBUSY;
1168 goto err_put;1260 goto err_put;
@@ -1174,6 +1266,7 @@ static int map_freeze(const union bpf_attr *attr) · 5 unmodified lines
1174 1266
1175 WRITE_ONCE(map->frozen, true);1267 WRITE_ONCE(map->frozen, true);
1176err_put:1268err_put:
1269 mutex_unlock(&map->freeze_mutex);
1177 fdput(f);1270 fdput(f);
1178 return err;1271 return err;
1179}1272}
mm/vmalloc.c+20−0
@@ -2671,6 +2671,26 @@ void *vzalloc_node(unsigned long size, int node) · 2670 unmodified lines
2671}2671}
2672EXPORT_SYMBOL(vzalloc_node);2672EXPORT_SYMBOL(vzalloc_node);
2673 2673
2674/**
2675 * vmalloc_user_node_flags - allocate memory for userspace on a specific node
2676 * @size: allocation size
2677 * @node: numa node
2678 * @flags: flags for the page level allocator
2679 *
2680 * The resulting memory area is zeroed so it can be mapped to userspace
2681 * without leaking data.
2682 *
2683 * Return: pointer to the allocated memory or %NULL on error
2684 */
2685void *vmalloc_user_node_flags(unsigned long size, int node, gfp_t flags)
2686{
2687 return __vmalloc_node_range(size, SHMLBA, VMALLOC_START, VMALLOC_END,
2688 flags | __GFP_ZERO, PAGE_KERNEL,
2689 VM_USERMAP, node,
2690 __builtin_return_address(0));
2691}
2692EXPORT_SYMBOL(vmalloc_user_node_flags);
2693
2674/**2694/**
2675 * vmalloc_exec - allocate virtually contiguous, executable memory2695 * vmalloc_exec - allocate virtually contiguous, executable memory
2676 * @size: allocation size2696 * @size: allocation size
tools/include/uapi/linux/bpf.h+3−0
@@ -348,6 +348,9 @@ enum bpf_attach_type { · 347 unmodified lines
348/* Clone map from listener for newly accepted socket */348/* Clone map from listener for newly accepted socket */
349#define BPF_F_CLONE (1U << 9)349#define BPF_F_CLONE (1U << 9)
350 350
351/* Enable memory-mapping BPF map */
352#define BPF_F_MMAPABLE (1U << 10)
353
351/* flags for BPF_PROG_QUERY */354/* flags for BPF_PROG_QUERY */
352#define BPF_F_QUERY_EFFECTIVE (1U << 0)355#define BPF_F_QUERY_EFFECTIVE (1U << 0)
353 356