bpf: Allow selecting numa node during map creation

The current map creation API does not allow to provide the numa-node preference. The memory usually comes from where the map-creation-process is running. The performance is not ideal if the bpf_prog is known to always run in a numa node different from the map-creation-process. One of the use case is sharding on CPU to different LRU maps (i.e. an array of LRU maps). Here is the test result of map_perf_test on the INNER_LRU_HASH_PREALLOC test if we force the lru map used by CPU0 to be allocated from a remote numa node: [ The machine has 20 cores. CPU0-9 at node 0. CPU10-19 at node 1 ] ># taskset -c 10 ./map_perf_test 512 8 1260000 8000000 5:inner_lru_hash_map_perf pre-alloc 1628380 events per sec 4:inner_lru_hash_map_perf pre-alloc 1626396 events per sec 3:inner_lru_hash_map_perf pre-alloc 1626144 events per sec 6:inner_lru_hash_map_perf pre-alloc 1621657 events per sec 2:inner_lru_hash_map_perf pre-alloc 1621534 events per sec 1:inner_lru_hash_map_perf pre-alloc 1620292 events per sec 7:inner_lru_hash_map_perf pre-alloc 1613305 events per sec 0:inner_lru_hash_map_perf pre-alloc 1239150 events per sec #<<< After specifying numa node: ># taskset -c 10 ./map_perf_test 512 8 1260000 8000000 5:inner_lru_hash_map_perf pre-alloc 1629627 events per sec 3:inner_lru_hash_map_perf pre-alloc 1628057 events per sec 1:inner_lru_hash_map_perf pre-alloc 1623054 events per sec 6:inner_lru_hash_map_perf pre-alloc 1616033 events per sec 2:inner_lru_hash_map_perf pre-alloc 1614630 events per sec 4:inner_lru_hash_map_perf pre-alloc 1612651 events per sec 7:inner_lru_hash_map_perf pre-alloc 1609337 events per sec 0:inner_lru_hash_map_perf pre-alloc 1619340 events per sec #<<< This patch adds one field, numa_node, to the bpf_attr. Since numa node 0 is a valid node, a new flag BPF_F_NUMA_NODE is also added. The numa_node field is honored if and only if the BPF_F_NUMA_NODE flag is set. Numa node selection is not supported for percpu map. This patch does not change all the kmalloc. F.e. 'htab = kzalloc()' is not changed since the object is small enough to stay in the cache.

Martin KaFai Lau committed 9 years ago

Commit 96eabe71 parent bd76b879 files · +73 −23Browse files

Unified · Split

9 files

include/linux/bpf.h +9 −1
include/uapi/linux/bpf.h +9 −1
kernel/bpf/arraymap.c +5 −2
kernel/bpf/devmap.c +6 −3
kernel/bpf/hashtab.c +15 −4
kernel/bpf/lpm_trie.c +7 −2
kernel/bpf/sockmap.c +7 −3
kernel/bpf/stackmap.c +5 −3
kernel/bpf/syscall.c +10 −4

include/linux/bpf.h+9−1

@@ -51,6 +51,7 @@ struct bpf_map { · 50 unmodified lines
51	51	u32 map_flags;
52	52	u32 pages;
53	53	u32 id;
	54	+ int numa_node;
54	55	struct user_struct *user;
55	56	const struct bpf_map_ops *ops;
56	57	struct work_struct work;
@@ -264,7 +265,7 @@ struct bpf_map * __must_check bpf_map_inc(struct bpf_map *map, bool uref); · 207 unmodified lines
264	265	void bpf_map_put_with_uref(struct bpf_map *map);
265	266	void bpf_map_put(struct bpf_map *map);
266	267	int bpf_map_precharge_memlock(u32 pages);
267		−void *bpf_map_area_alloc(size_t size);
	268	+void *bpf_map_area_alloc(size_t size, int numa_node);
268	269	void bpf_map_area_free(void *base);
269	270
270	271	extern int sysctl_unprivileged_bpf_disabled;
@@ -316,6 +317,13 @@ struct net_device __dev_map_lookup_elem(struct bpf_map map, u32 key); · 45 unmodified lines
316	317	void __dev_map_insert_ctx(struct bpf_map *map, u32 index);
317	318	void __dev_map_flush(struct bpf_map *map);
318	319
	320	+/* Return map's numa specified by userspace */
	321	+static inline int bpf_map_attr_numa_node(const union bpf_attr *attr)
	322	+{
	323	+ return (attr->map_flags & BPF_F_NUMA_NODE) ?
	324	+ attr->numa_node : NUMA_NO_NODE;
	325	+}
	326	+
319	327	#else
320	328	static inline struct bpf_prog *bpf_prog_get(u32 ufd)
321	329	{

include/uapi/linux/bpf.h+9−1

@@ -165,6 +165,7 @@ enum bpf_attach_type { · 164 unmodified lines
165	165	#define BPF_NOEXIST 1 /* create new element if it didn't exist */
166	166	#define BPF_EXIST 2 /* update existing element */
167	167
	168	+/* flags for BPF_MAP_CREATE command */
168	169	#define BPF_F_NO_PREALLOC (1U << 0)
169	170	/* Instead of having one common LRU list in the
170	171	* BPF_MAP_TYPE_LRU_[PERCPU_]HASH map, use a percpu LRU list
@@ -173,6 +174,8 @@ enum bpf_attach_type { · 2 unmodified lines
173	174	* across different LRU lists.
174	175	*/
175	176	#define BPF_F_NO_COMMON_LRU (1U << 1)
	177	+/* Specify numa node during map creation */
	178	+#define BPF_F_NUMA_NODE (1U << 2)
176	179
177	180	union bpf_attr {
178	181	struct { /* anonymous struct used by BPF_MAP_CREATE command */
@@ -180,8 +183,13 @@ union bpf_attr { · 1 unmodified line
180	183	__u32 key_size; /* size of key in bytes */
181	184	__u32 value_size; /* size of value in bytes */
182	185	__u32 max_entries; /* max number of entries in a map */
183		− __u32 map_flags; /* prealloc or not */
	186	+ __u32 map_flags; /* BPF_MAP_CREATE related
	187	+ * flags defined above.
	188	+ */
184	189	__u32 inner_map_fd; /* fd pointing to the inner map */
	190	+ __u32 numa_node; /* numa node (effective only if
	191	+ * BPF_F_NUMA_NODE is set).
	192	+ */
185	193	};
186	194
187	195	struct { /* anonymous struct used by BPF_MAP__ELEM commands /

kernel/bpf/arraymap.c+5−2

@@ -49,13 +49,15 @@ static int bpf_array_alloc_percpu(struct bpf_array *array) · 48 unmodified lines
49	49	static struct bpf_map array_map_alloc(union bpf_attr attr)
50	50	{
51	51	bool percpu = attr->map_type == BPF_MAP_TYPE_PERCPU_ARRAY;
	52	+ int numa_node = bpf_map_attr_numa_node(attr);
52	53	struct bpf_array *array;
53	54	u64 array_size;
54	55	u32 elem_size;
55	56
56	57	/* check sanity of attributes */
57	58	if (attr->max_entries == 0 \|\| attr->key_size != 4 \|\|
58		− attr->value_size == 0 \|\| attr->map_flags)
	59	+ attr->value_size == 0 \|\| attr->map_flags & ~BPF_F_NUMA_NODE \|\|
	60	+ (percpu && numa_node != NUMA_NO_NODE))
59	61	return ERR_PTR(-EINVAL);
60	62
61	63	if (attr->value_size > KMALLOC_MAX_SIZE)
@@ -77,7 +79,7 @@ static struct bpf_map array_map_alloc(union bpf_attr attr) · 15 unmodified lines
77	79	return ERR_PTR(-ENOMEM);
78	80
79	81	/* allocate all map elements and zero-initialize them */
80		− array = bpf_map_area_alloc(array_size);
	82	+ array = bpf_map_area_alloc(array_size, numa_node);
81	83	if (!array)
82	84	return ERR_PTR(-ENOMEM);
83	85
@@ -87,6 +89,7 @@ static struct bpf_map array_map_alloc(union bpf_attr attr) · 3 unmodified lines
87	89	array->map.value_size = attr->value_size;
88	90	array->map.max_entries = attr->max_entries;
89	91	array->map.map_flags = attr->map_flags;
	92	+ array->map.numa_node = numa_node;
90	93	array->elem_size = elem_size;
91	94
92	95	if (!percpu)

kernel/bpf/devmap.c+6−3

@@ -80,7 +80,7 @@ static struct bpf_map dev_map_alloc(union bpf_attr attr) · 79 unmodified lines
80	80
81	81	/* check sanity of attributes */
82	82	if (attr->max_entries == 0 \|\| attr->key_size != 4 \|\|
83		− attr->value_size != 4 \|\| attr->map_flags)
	83	+ attr->value_size != 4 \|\| attr->map_flags & ~BPF_F_NUMA_NODE)
84	84	return ERR_PTR(-EINVAL);
85	85
86	86	dtab = kzalloc(sizeof(*dtab), GFP_USER);
@@ -93,6 +93,7 @@ static struct bpf_map dev_map_alloc(union bpf_attr attr) · 6 unmodified lines
93	93	dtab->map.value_size = attr->value_size;
94	94	dtab->map.max_entries = attr->max_entries;
95	95	dtab->map.map_flags = attr->map_flags;
	96	+ dtab->map.numa_node = bpf_map_attr_numa_node(attr);
96	97
97	98	err = -ENOMEM;
98	99
@@ -119,7 +120,8 @@ static struct bpf_map dev_map_alloc(union bpf_attr attr) · 20 unmodified lines
119	120	goto free_dtab;
120	121
121	122	dtab->netdev_map = bpf_map_area_alloc(dtab->map.max_entries *
122		− sizeof(struct bpf_dtab_netdev *));
	123	+ sizeof(struct bpf_dtab_netdev *),
	124	+ dtab->map.numa_node);
123	125	if (!dtab->netdev_map)
124	126	goto free_dtab;
125	127
@@ -344,7 +346,8 @@ static int dev_map_update_elem(struct bpf_map map, void key, void *value, · 218 unmodified lines
344	346	if (!ifindex) {
345	347	dev = NULL;
346	348	} else {
347		− dev = kmalloc(sizeof(*dev), GFP_ATOMIC \| __GFP_NOWARN);
	349	+ dev = kmalloc_node(sizeof(*dev), GFP_ATOMIC \| __GFP_NOWARN,
	350	+ map->numa_node);
348	351	if (!dev)
349	352	return -ENOMEM;
350	353

kernel/bpf/hashtab.c+15−4

@@ -18,6 +18,9 @@ · 17 unmodified lines
18	18	#include "bpf_lru_list.h"
19	19	#include "map_in_map.h"
20	20
	21	+#define HTAB_CREATE_FLAG_MASK \
	22	+ (BPF_F_NO_PREALLOC \| BPF_F_NO_COMMON_LRU \| BPF_F_NUMA_NODE)
	23	+
21	24	struct bucket {
22	25	struct hlist_nulls_head head;
23	26	raw_spinlock_t lock;
@@ -138,7 +141,8 @@ static int prealloc_init(struct bpf_htab *htab) · 114 unmodified lines
138	141	if (!htab_is_percpu(htab) && !htab_is_lru(htab))
139	142	num_entries += num_possible_cpus();
140	143
141		− htab->elems = bpf_map_area_alloc(htab->elem_size * num_entries);
	144	+ htab->elems = bpf_map_area_alloc(htab->elem_size * num_entries,
	145	+ htab->map.numa_node);
142	146	if (!htab->elems)
143	147	return -ENOMEM;
144	148
@@ -233,6 +237,7 @@ static struct bpf_map htab_map_alloc(union bpf_attr attr) · 88 unmodified lines
233	237	*/
234	238	bool percpu_lru = (attr->map_flags & BPF_F_NO_COMMON_LRU);
235	239	bool prealloc = !(attr->map_flags & BPF_F_NO_PREALLOC);
	240	+ int numa_node = bpf_map_attr_numa_node(attr);
236	241	struct bpf_htab *htab;
237	242	int err, i;
238	243	u64 cost;
@@ -248,7 +253,7 @@ static struct bpf_map htab_map_alloc(union bpf_attr attr) · 9 unmodified lines
248	253	*/
249	254	return ERR_PTR(-EPERM);
250	255
251		− if (attr->map_flags & ~(BPF_F_NO_PREALLOC \| BPF_F_NO_COMMON_LRU))
	256	+ if (attr->map_flags & ~HTAB_CREATE_FLAG_MASK)
252	257	/* reserved bits should not be used */
253	258	return ERR_PTR(-EINVAL);
254	259
@@ -258,6 +263,9 @@ static struct bpf_map htab_map_alloc(union bpf_attr attr) · 3 unmodified lines
258	263	if (lru && !prealloc)
259	264	return ERR_PTR(-ENOTSUPP);
260	265
	266	+ if (numa_node != NUMA_NO_NODE && (percpu \|\| percpu_lru))
	267	+ return ERR_PTR(-EINVAL);
	268	+
261	269	htab = kzalloc(sizeof(*htab), GFP_USER);
262	270	if (!htab)
263	271	return ERR_PTR(-ENOMEM);
@@ -268,6 +276,7 @@ static struct bpf_map htab_map_alloc(union bpf_attr attr) · 4 unmodified lines
268	276	htab->map.value_size = attr->value_size;
269	277	htab->map.max_entries = attr->max_entries;
270	278	htab->map.map_flags = attr->map_flags;
	279	+ htab->map.numa_node = numa_node;
271	280
272	281	/* check sanity of attributes.
273	282	* value_size == 0 may be allowed in the future to use map as a set
@@ -346,7 +355,8 @@ static struct bpf_map htab_map_alloc(union bpf_attr attr) · 72 unmodified lines
346	355
347	356	err = -ENOMEM;
348	357	htab->buckets = bpf_map_area_alloc(htab->n_buckets *
349		− sizeof(struct bucket));
	358	+ sizeof(struct bucket),
	359	+ htab->map.numa_node);
350	360	if (!htab->buckets)
351	361	goto free_htab;
352	362
@@ -689,7 +699,8 @@ static struct htab_elem alloc_htab_elem(struct bpf_htab htab, void *key, · 336 unmodified lines
689	699	atomic_dec(&htab->count);
690	700	return ERR_PTR(-E2BIG);
691	701	}
692		− l_new = kmalloc(htab->elem_size, GFP_ATOMIC \| __GFP_NOWARN);
	702	+ l_new = kmalloc_node(htab->elem_size, GFP_ATOMIC \| __GFP_NOWARN,
	703	+ htab->map.numa_node);
693	704	if (!l_new)
694	705	return ERR_PTR(-ENOMEM);
695	706	}

kernel/bpf/lpm_trie.c+7−2

@@ -244,7 +244,8 @@ static struct lpm_trie_node lpm_trie_node_alloc(const struct lpm_trie trie, · 243 unmodified lines
244	244	if (value)
245	245	size += trie->map.value_size;
246	246
247		− node = kmalloc(size, GFP_ATOMIC \| __GFP_NOWARN);
	247	+ node = kmalloc_node(size, GFP_ATOMIC \| __GFP_NOWARN,
	248	+ trie->map.numa_node);
248	249	if (!node)
249	250	return NULL;
250	251
@@ -405,6 +406,8 @@ static int trie_delete_elem(struct bpf_map map, void key) · 154 unmodified lines
405	406	#define LPM_KEY_SIZE_MAX LPM_KEY_SIZE(LPM_DATA_SIZE_MAX)
406	407	#define LPM_KEY_SIZE_MIN LPM_KEY_SIZE(LPM_DATA_SIZE_MIN)
407	408
	409	+#define LPM_CREATE_FLAG_MASK (BPF_F_NO_PREALLOC \| BPF_F_NUMA_NODE)
	410	+
408	411	static struct bpf_map trie_alloc(union bpf_attr attr)
409	412	{
410	413	struct lpm_trie *trie;
@@ -416,7 +419,8 @@ static struct bpf_map trie_alloc(union bpf_attr attr) · 5 unmodified lines
416	419
417	420	/* check sanity of attributes */
418	421	if (attr->max_entries == 0 \|\|
419		− attr->map_flags != BPF_F_NO_PREALLOC \|\|
	422	+ !(attr->map_flags & BPF_F_NO_PREALLOC) \|\|
	423	+ attr->map_flags & ~LPM_CREATE_FLAG_MASK \|\|
420	424	attr->key_size < LPM_KEY_SIZE_MIN \|\|
421	425	attr->key_size > LPM_KEY_SIZE_MAX \|\|
422	426	attr->value_size < LPM_VAL_SIZE_MIN \|\|
@@ -433,6 +437,7 @@ static struct bpf_map trie_alloc(union bpf_attr attr) · 10 unmodified lines
433	437	trie->map.value_size = attr->value_size;
434	438	trie->map.max_entries = attr->max_entries;
435	439	trie->map.map_flags = attr->map_flags;
	440	+ trie->map.numa_node = bpf_map_attr_numa_node(attr);
436	441	trie->data_size = attr->key_size -
437	442	offsetof(struct bpf_lpm_trie_key, data);
438	443	trie->max_prefixlen = trie->data_size * 8;

kernel/bpf/sockmap.c+7−3

@@ -443,7 +443,9 @@ static struct smap_psock smap_init_psock(struct sock sock, · 442 unmodified lines
443	443	{
444	444	struct smap_psock *psock;
445	445
446		− psock = kzalloc(sizeof(struct smap_psock), GFP_ATOMIC \| __GFP_NOWARN);
	446	+ psock = kzalloc_node(sizeof(struct smap_psock),
	447	+ GFP_ATOMIC \| __GFP_NOWARN,
	448	+ stab->map.numa_node);
447	449	if (!psock)
448	450	return ERR_PTR(-ENOMEM);
449	451
@@ -465,7 +467,7 @@ static struct bpf_map sock_map_alloc(union bpf_attr attr) · 15 unmodified lines
465	467
466	468	/* check sanity of attributes */
467	469	if (attr->max_entries == 0 \|\| attr->key_size != 4 \|\|
468		− attr->value_size != 4 \|\| attr->map_flags)
	470	+ attr->value_size != 4 \|\| attr->map_flags & ~BPF_F_NUMA_NODE)
469	471	return ERR_PTR(-EINVAL);
470	472
471	473	if (attr->value_size > KMALLOC_MAX_SIZE)
@@ -481,6 +483,7 @@ static struct bpf_map sock_map_alloc(union bpf_attr attr) · 9 unmodified lines
481	483	stab->map.value_size = attr->value_size;
482	484	stab->map.max_entries = attr->max_entries;
483	485	stab->map.map_flags = attr->map_flags;
	486	+ stab->map.numa_node = bpf_map_attr_numa_node(attr);
484	487
485	488	/* make sure page count doesn't overflow */
486	489	cost = (u64) stab->map.max_entries * sizeof(struct sock *);
@@ -495,7 +498,8 @@ static struct bpf_map sock_map_alloc(union bpf_attr attr) · 8 unmodified lines
495	498	goto free_stab;
496	499
497	500	stab->sock_map = bpf_map_area_alloc(stab->map.max_entries *
498		− sizeof(struct sock *));
	501	+ sizeof(struct sock *),
	502	+ stab->map.numa_node);
499	503	if (!stab->sock_map)
500	504	goto free_stab;
501	505

kernel/bpf/stackmap.c+5−3

@@ -31,7 +31,8 @@ static int prealloc_elems_and_freelist(struct bpf_stack_map *smap) · 30 unmodified lines
31	31	u32 elem_size = sizeof(struct stack_map_bucket) + smap->map.value_size;
32	32	int err;
33	33
34		− smap->elems = bpf_map_area_alloc(elem_size * smap->map.max_entries);
	34	+ smap->elems = bpf_map_area_alloc(elem_size * smap->map.max_entries,
	35	+ smap->map.numa_node);
35	36	if (!smap->elems)
36	37	return -ENOMEM;
37	38
@@ -59,7 +60,7 @@ static struct bpf_map stack_map_alloc(union bpf_attr attr) · 21 unmodified lines
59	60	if (!capable(CAP_SYS_ADMIN))
60	61	return ERR_PTR(-EPERM);
61	62
62		− if (attr->map_flags)
	63	+ if (attr->map_flags & ~BPF_F_NUMA_NODE)
63	64	return ERR_PTR(-EINVAL);
64	65
65	66	/* check sanity of attributes */
@@ -75,7 +76,7 @@ static struct bpf_map stack_map_alloc(union bpf_attr attr) · 9 unmodified lines
75	76	if (cost >= U32_MAX - PAGE_SIZE)
76	77	return ERR_PTR(-E2BIG);
77	78
78		− smap = bpf_map_area_alloc(cost);
	79	+ smap = bpf_map_area_alloc(cost, bpf_map_attr_numa_node(attr));
79	80	if (!smap)
80	81	return ERR_PTR(-ENOMEM);
81	82
@@ -91,6 +92,7 @@ static struct bpf_map stack_map_alloc(union bpf_attr attr) · 9 unmodified lines
91	92	smap->map.map_flags = attr->map_flags;
92	93	smap->n_buckets = n_buckets;
93	94	smap->map.pages = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;
	95	+ smap->map.numa_node = bpf_map_attr_numa_node(attr);
94	96
95	97	err = bpf_map_precharge_memlock(smap->map.pages);
96	98	if (err)

kernel/bpf/syscall.c+10−4

@@ -105,7 +105,7 @@ static struct bpf_map find_and_alloc_map(union bpf_attr attr) · 104 unmodified lines
105	105	return map;
106	106	}
107	107
108		−void *bpf_map_area_alloc(size_t size)
	108	+void *bpf_map_area_alloc(size_t size, int numa_node)
109	109	{
110	110	/* We definitely need __GFP_NORETRY, so OOM killer doesn't
111	111	* trigger under memory pressure as we really just want to
@@ -115,12 +115,13 @@ void *bpf_map_area_alloc(size_t size) · 3 unmodified lines
115	115	void *area;
116	116
117	117	if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) {
118		− area = kmalloc(size, GFP_USER \| flags);
	118	+ area = kmalloc_node(size, GFP_USER \| flags, numa_node);
119	119	if (area != NULL)
120	120	return area;
121	121	}
122	122
123		− return __vmalloc(size, GFP_KERNEL \| flags, PAGE_KERNEL);
	123	+ return __vmalloc_node_flags_caller(size, numa_node, GFP_KERNEL \| flags,
	124	+ __builtin_return_address(0));
124	125	}
125	126
126	127	void bpf_map_area_free(void *area)
@@ -309,10 +310,11 @@ int bpf_map_new_fd(struct bpf_map *map) · 182 unmodified lines
309	310	offsetof(union bpf_attr, CMD##_LAST_FIELD) - \
310	311	sizeof(attr->CMD##_LAST_FIELD)) != NULL
311	312
312		−#define BPF_MAP_CREATE_LAST_FIELD inner_map_fd
	313	+#define BPF_MAP_CREATE_LAST_FIELD numa_node
313	314	/* called via syscall */
314	315	static int map_create(union bpf_attr *attr)
315	316	{
	317	+ int numa_node = bpf_map_attr_numa_node(attr);
316	318	struct bpf_map *map;
317	319	int err;
318	320
@@ -320,6 +322,10 @@ static int map_create(union bpf_attr *attr) · 1 unmodified line
320	322	if (err)
321	323	return -EINVAL;
322	324
	325	+ if (numa_node != NUMA_NO_NODE &&
	326	+ (numa_node >= nr_node_ids \|\| !node_online(numa_node)))
	327	+ return -EINVAL;
	328	+
323	329	/* find map type and init map: hashtable vs rbtree vs bloom vs ... */
324	330	map = find_and_alloc_map(attr);
325	331	if (IS_ERR(map))