History

bpf: Add mmap() support for BPF_MAP_TYPE_ARRAY

Add ability to memory-map contents of BPF array map. This is extremely useful for working with BPF global data from userspace programs. It allows to avoid typical bpf_map_{lookup,update}_elem operations, improving both performance and usability. There had to be special considerations for map freezing, to avoid having writable memory view into a frozen map. To solve this issue, map freezing and mmap-ing is happening under mutex now: - if map is already frozen, no writable mapping is allowed; - if map has writable memory mappings active (accounted in map->writecnt), map freezing will keep failing with -EBUSY; - once number of writable memory mappings drops to zero, map freezing can be performed again. Only non-per-CPU plain arrays are supported right now. Maps with spinlocks can't be memory mapped either. For BPF_F_MMAPABLE array, memory allocation has to be done through vmalloc() to be mmap()'able. We also need to make sure that array data memory is page-sized and page-aligned, so we over-allocate memory in such a way that struct bpf_array is at the end of a single page of memory with array->value being aligned with the start of the second page. On deallocation we need to accomodate this memory arrangement to free vmalloc()'ed memory correctly. One important consideration regarding how memory-mapping subsystem functions. Memory-mapping subsystem provides few optional callbacks, among them open() and close(). close() is called for each memory region that is unmapped, so that users can decrease their reference counters and free up resources, if necessary. open() is *almost* symmetrical: it's called for each memory region that is being mapped, **except** the very first one. So bpf_map_mmap does initial refcnt bump, while open() will do any extra ones after that. Thus number of close() calls is equal to number of open() calls plus one more.

Andrii Nakryiko committed 6 years ago

Commit fc970221 parent 85192db7 files · +183 −12Browse files

Showing the changes to include/linux (2 of 7 files). Show all

Unified · Split

2 files
  1. include/linux/bpf.h +8 −3
  2. include/linux/vmalloc.h +1 −0
include/linux/bpf.h+8−3
@@ -12,6 +12,7 @@ · 11 unmodified lines
12#include <linux/err.h>12#include <linux/err.h>
13#include <linux/rbtree_latch.h>13#include <linux/rbtree_latch.h>
14#include <linux/numa.h>14#include <linux/numa.h>
15#include <linux/mm_types.h>
15#include <linux/wait.h>16#include <linux/wait.h>
16#include <linux/u64_stats_sync.h>17#include <linux/u64_stats_sync.h>
17#include <linux/refcount.h>18#include <linux/refcount.h>
@@ -68,6 +69,7 @@ struct bpf_map_ops { · 50 unmodified lines
68 u64 *imm, u32 off);69 u64 *imm, u32 off);
69 int (*map_direct_value_meta)(const struct bpf_map *map,70 int (*map_direct_value_meta)(const struct bpf_map *map,
70 u64 imm, u32 *off);71 u64 imm, u32 *off);
72 int (*map_mmap)(struct bpf_map *map, struct vm_area_struct *vma);
71};73};
72 74
73struct bpf_map_memory {75struct bpf_map_memory {
@@ -96,9 +98,10 @@ struct bpf_map { · 22 unmodified lines
96 u32 btf_value_type_id;98 u32 btf_value_type_id;
97 struct btf *btf;99 struct btf *btf;
98 struct bpf_map_memory memory;100 struct bpf_map_memory memory;
101 char name[BPF_OBJ_NAME_LEN];
99 bool unpriv_array;102 bool unpriv_array;
100 bool frozen; /* write-once */103 bool frozen; /* write-once; write-protected by freeze_mutex */
101 /* 48 bytes hole */104 /* 22 bytes hole */
102 105
103 /* The 3rd and 4th cacheline with misc members to avoid false sharing106 /* The 3rd and 4th cacheline with misc members to avoid false sharing
104 * particularly with refcounting.107 * particularly with refcounting.
@@ -106,7 +109,8 @@ struct bpf_map { · 1 unmodified line
106 atomic64_t refcnt ____cacheline_aligned;109 atomic64_t refcnt ____cacheline_aligned;
107 atomic64_t usercnt;110 atomic64_t usercnt;
108 struct work_struct work;111 struct work_struct work;
109 char name[BPF_OBJ_NAME_LEN];112 struct mutex freeze_mutex;
113 u64 writecnt; /* writable mmap cnt; protected by freeze_mutex */
110};114};
111 115
112static inline bool map_value_has_spin_lock(const struct bpf_map *map)116static inline bool map_value_has_spin_lock(const struct bpf_map *map)
@@ -795,6 +799,7 @@ void bpf_map_charge_finish(struct bpf_map_memory *mem); · 682 unmodified lines
795void bpf_map_charge_move(struct bpf_map_memory *dst,799void bpf_map_charge_move(struct bpf_map_memory *dst,
796 struct bpf_map_memory *src);800 struct bpf_map_memory *src);
797void *bpf_map_area_alloc(size_t size, int numa_node);801void *bpf_map_area_alloc(size_t size, int numa_node);
802void *bpf_map_area_mmapable_alloc(size_t size, int numa_node);
798void bpf_map_area_free(void *base);803void bpf_map_area_free(void *base);
799void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr);804void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr);
800 805
include/linux/vmalloc.h+1−0
@@ -93,6 +93,7 @@ extern void *vzalloc(unsigned long size); · 92 unmodified lines
93extern void *vmalloc_user(unsigned long size);93extern void *vmalloc_user(unsigned long size);
94extern void *vmalloc_node(unsigned long size, int node);94extern void *vmalloc_node(unsigned long size, int node);
95extern void *vzalloc_node(unsigned long size, int node);95extern void *vzalloc_node(unsigned long size, int node);
96extern void *vmalloc_user_node_flags(unsigned long size, int node, gfp_t flags);
96extern void *vmalloc_exec(unsigned long size);97extern void *vmalloc_exec(unsigned long size);
97extern void *vmalloc_32(unsigned long size);98extern void *vmalloc_32(unsigned long size);
98extern void *vmalloc_32_user(unsigned long size);99extern void *vmalloc_32_user(unsigned long size);

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t