Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Daniel Borkmann says: ==================== pull-request: bpf-next 2021-06-28 The following pull-request contains BPF updates for your *net-next* tree. We've added 37 non-merge commits during the last 12 day(s) which contain a total of 56 files changed, 394 insertions(+), 380 deletions(-). The main changes are: 1) XDP driver RCU cleanups, from Toke Høiland-Jørgensen and Paul E. McKenney. 2) Fix bpf_skb_change_proto() IPv4/v6 GSO handling, from Maciej Żenczykowski. 3) Fix false positive kmemleak report for BPF ringbuf alloc, from Rustam Kovhaev. 4) Fix x86 JIT's extable offset calculation for PROBE_LDX NULL, from Ravi Bangoria. 5) Enable libbpf fallback probing with tracing under RHEL7, from Jonathan Edwards. 6) Clean up x86 JIT to remove unused cnt tracking from EMIT macro, from Jiri Olsa. 7) Netlink cleanups for libbpf to please Coverity, from Kumar Kartikeya Dwivedi. 8) Allow to retrieve ancestor cgroup id in tracing programs, from Namhyung Kim. 9) Fix lirc BPF program query to use user-provided prog_cnt, from Sean Young. 10) Add initial libbpf doc including generated kdoc for its API, from Grant Seltzer. 11) Make xdp_rxq_info_unreg_mem_model() more robust, from Jakub Kicinski. 12) Fix up bpfilter startup log-level to info level, from Gary Lin. ====================

David S. Miller committed 5 years ago

Commit e1289cf2 parents 1fd07f3 a78cae256 files · +394 −380Browse files

Showing the changes to kernel (7 of 56 files). Show all

Unified · Split

7 files

kernel/bpf/cpumap.c +9 −4
kernel/bpf/devmap.c +21 −28
kernel/bpf/hashtab.c +14 −7
kernel/bpf/helpers.c +3 −3
kernel/bpf/lpm_trie.c +4 −2
kernel/bpf/ringbuf.c +2 −0
kernel/trace/bpf_trace.c +2 −0

kernel/bpf/cpumap.c+9−4

@@ -74,7 +74,7 @@ struct bpf_cpu_map_entry { · 73 unmodified lines
74	74	struct bpf_cpu_map {
75	75	struct bpf_map map;
76	76	/* Below members specific for map type */
77		− struct bpf_cpu_map_entry **cpu_map;
	77	+ struct bpf_cpu_map_entry __rcu **cpu_map;
78	78	};
79	79
80	80	static DEFINE_PER_CPU(struct list_head, cpu_map_flush_list);
@@ -469,7 +469,7 @@ static void __cpu_map_entry_replace(struct bpf_cpu_map *cmap, · 388 unmodified lines
469	469	{
470	470	struct bpf_cpu_map_entry *old_rcpu;
471	471
472		− old_rcpu = xchg(&cmap->cpu_map[key_cpu], rcpu);
	472	+ old_rcpu = unrcu_pointer(xchg(&cmap->cpu_map[key_cpu], RCU_INITIALIZER(rcpu)));
473	473	if (old_rcpu) {
474	474	call_rcu(&old_rcpu->rcu, __cpu_map_entry_free);
475	475	INIT_WORK(&old_rcpu->kthread_stop_wq, cpu_map_kthread_stop);
@@ -551,7 +551,7 @@ static void cpu_map_free(struct bpf_map *map) · 75 unmodified lines
551	551	for (i = 0; i < cmap->map.max_entries; i++) {
552	552	struct bpf_cpu_map_entry *rcpu;
553	553
554		− rcpu = READ_ONCE(cmap->cpu_map[i]);
	554	+ rcpu = rcu_dereference_raw(cmap->cpu_map[i]);
555	555	if (!rcpu)
556	556	continue;
557	557
@@ -562,6 +562,10 @@ static void cpu_map_free(struct bpf_map *map) · 4 unmodified lines
562	562	kfree(cmap);
563	563	}
564	564
	565	+/* Elements are kept alive by RCU; either by rcu_read_lock() (from syscall) or
	566	+ * by local_bh_disable() (from XDP calls inside NAPI). The
	567	+ * rcu_read_lock_bh_held() below makes lockdep accept both.
	568	+ */
565	569	static void __cpu_map_lookup_elem(struct bpf_map map, u32 key)
566	570	{
567	571	struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
@@ -570,7 +574,8 @@ static void __cpu_map_lookup_elem(struct bpf_map map, u32 key) · 2 unmodified lines
570	574	if (key >= map->max_entries)
571	575	return NULL;
572	576
573		− rcpu = READ_ONCE(cmap->cpu_map[key]);
	577	+ rcpu = rcu_dereference_check(cmap->cpu_map[key],
	578	+ rcu_read_lock_bh_held());
574	579	return rcpu;
575	580	}
576	581

kernel/bpf/devmap.c+21−28

@@ -73,7 +73,7 @@ struct bpf_dtab_netdev { · 72 unmodified lines
73	73
74	74	struct bpf_dtab {
75	75	struct bpf_map map;
76		− struct bpf_dtab_netdev *netdev_map; / DEVMAP type only */
	76	+ struct bpf_dtab_netdev __rcu *netdev_map; / DEVMAP type only */
77	77	struct list_head list;
78	78
79	79	/* these are only used for DEVMAP_HASH type maps */
@@ -226,7 +226,7 @@ static void dev_map_free(struct bpf_map *map) · 146 unmodified lines
226	226	for (i = 0; i < dtab->map.max_entries; i++) {
227	227	struct bpf_dtab_netdev *dev;
228	228
229		− dev = dtab->netdev_map[i];
	229	+ dev = rcu_dereference_raw(dtab->netdev_map[i]);
230	230	if (!dev)
231	231	continue;
232	232
@@ -259,6 +259,10 @@ static int dev_map_get_next_key(struct bpf_map map, void key, void *next_key) · 26 unmodified lines
259	259	return 0;
260	260	}
261	261
	262	+/* Elements are kept alive by RCU; either by rcu_read_lock() (from syscall) or
	263	+ * by local_bh_disable() (from XDP calls inside NAPI). The
	264	+ * rcu_read_lock_bh_held() below makes lockdep accept both.
	265	+ */
262	266	static void __dev_map_hash_lookup_elem(struct bpf_map map, u32 key)
263	267	{
264	268	struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);
@@ -410,15 +414,9 @@ out: · 145 unmodified lines
410	414	trace_xdp_devmap_xmit(bq->dev_rx, dev, sent, cnt - sent, err);
411	415	}
412	416
413		−/* __dev_flush is called from xdp_do_flush() which _must_ be signaled
414		− * from the driver before returning from its napi->poll() routine. The poll()
415		− * routine is called either from busy_poll context or net_rx_action signaled
416		− * from NET_RX_SOFTIRQ. Either way the poll routine must complete before the
417		− * net device can be torn down. On devmap tear down we ensure the flush list
418		− * is empty before completing to ensure all flush operations have completed.
419		− * When drivers update the bpf program they may need to ensure any flush ops
420		− * are also complete. Using synchronize_rcu or call_rcu will suffice for this
421		− * because both wait for napi context to exit.
	417	+/* __dev_flush is called from xdp_do_flush() which _must_ be signalled from the
	418	+ * driver before returning from its napi->poll() routine. See the comment above
	419	+ * xdp_do_flush() in filter.c.
422	420	*/
423	421	void __dev_flush(void)
424	422	{
@@ -433,9 +431,9 @@ void __dev_flush(void) · 8 unmodified lines
433	431	}
434	432	}
435	433
436		−/* rcu_read_lock (from syscall and BPF contexts) ensures that if a delete and/or
437		− * update happens in parallel here a dev_put won't happen until after reading
438		− * the ifindex.
	434	+/* Elements are kept alive by RCU; either by rcu_read_lock() (from syscall) or
	435	+ * by local_bh_disable() (from XDP calls inside NAPI). The
	436	+ * rcu_read_lock_bh_held() below makes lockdep accept both.
439	437	*/
440	438	static void __dev_map_lookup_elem(struct bpf_map map, u32 key)
441	439	{
@@ -445,12 +443,14 @@ static void __dev_map_lookup_elem(struct bpf_map map, u32 key) · 3 unmodified lines
445	443	if (key >= map->max_entries)
446	444	return NULL;
447	445
448		− obj = READ_ONCE(dtab->netdev_map[key]);
	446	+ obj = rcu_dereference_check(dtab->netdev_map[key],
	447	+ rcu_read_lock_bh_held());
449	448	return obj;
450	449	}
451	450
452		−/* Runs under RCU-read-side, plus in softirq under NAPI protection.
453		− * Thus, safe percpu variable access.
	451	+/* Runs in NAPI, i.e., softirq under local_bh_disable(). Thus, safe percpu
	452	+ * variable access, and map elements stick around. See comment above
	453	+ * xdp_do_flush() in filter.c.
454	454	*/
455	455	static void bq_enqueue(struct net_device dev, struct xdp_frame xdpf,
456	456	struct net_device dev_rx, struct bpf_prog xdp_prog)
@@ -735,14 +735,7 @@ static int dev_map_delete_elem(struct bpf_map map, void key) · 278 unmodified lines
735	735	if (k >= map->max_entries)
736	736	return -EINVAL;
737	737
738		− /* Use call_rcu() here to ensure any rcu critical sections have
739		− * completed as well as any flush operations because call_rcu
740		− * will wait for preempt-disable region to complete, NAPI in this
741		− * context. And additionally, the driver tear down ensures all
742		− * soft irqs are complete before removing the net device in the
743		− * case of dev_put equals zero.
744		− */
745		− old_dev = xchg(&dtab->netdev_map[k], NULL);
	738	+ old_dev = unrcu_pointer(xchg(&dtab->netdev_map[k], NULL));
746	739	if (old_dev)
747	740	call_rcu(&old_dev->rcu, __dev_map_entry_free);
748	741	return 0;
@@ -851,7 +844,7 @@ static int __dev_map_update_elem(struct net net, struct bpf_map map, · 102 unmodified lines
851	844	* Remembering the driver side flush operation will happen before the
852	845	* net device is removed.
853	846	*/
854		− old_dev = xchg(&dtab->netdev_map[i], dev);
	847	+ old_dev = unrcu_pointer(xchg(&dtab->netdev_map[i], RCU_INITIALIZER(dev)));
855	848	if (old_dev)
856	849	call_rcu(&old_dev->rcu, __dev_map_entry_free);
857	850
@@ -1031,10 +1024,10 @@ static int dev_map_notification(struct notifier_block *notifier, · 173 unmodified lines
1031	1024	for (i = 0; i < dtab->map.max_entries; i++) {
1032	1025	struct bpf_dtab_netdev dev, odev;
1033	1026
1034		− dev = READ_ONCE(dtab->netdev_map[i]);
	1027	+ dev = rcu_dereference(dtab->netdev_map[i]);
1035	1028	if (!dev \|\| netdev != dev->dev)
1036	1029	continue;
1037		− odev = cmpxchg(&dtab->netdev_map[i], dev, NULL);
	1030	+ odev = unrcu_pointer(cmpxchg(&dtab->netdev_map[i], RCU_INITIALIZER(dev), NULL));
1038	1031	if (dev == odev)
1039	1032	call_rcu(&dev->rcu,
1040	1033	__dev_map_entry_free);

kernel/bpf/hashtab.c+14−7

@@ -596,7 +596,8 @@ static void __htab_map_lookup_elem(struct bpf_map map, void *key) · 595 unmodified lines
596	596	struct htab_elem *l;
597	597	u32 hash, key_size;
598	598
599		− WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
	599	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
	600	+ !rcu_read_lock_bh_held());
600	601
601	602	key_size = map->key_size;
602	603
@@ -989,7 +990,8 @@ static int htab_map_update_elem(struct bpf_map map, void key, void *value, · 386 unmodified lines
989	990	/* unknown flags */
990	991	return -EINVAL;
991	992
992		− WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
	993	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
	994	+ !rcu_read_lock_bh_held());
993	995
994	996	key_size = map->key_size;
995	997
@@ -1082,7 +1084,8 @@ static int htab_lru_map_update_elem(struct bpf_map map, void key, void *value, · 86 unmodified lines
1082	1084	/* unknown flags */
1083	1085	return -EINVAL;
1084	1086
1085		− WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
	1087	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
	1088	+ !rcu_read_lock_bh_held());
1086	1089
1087	1090	key_size = map->key_size;
1088	1091
@@ -1148,7 +1151,8 @@ static int __htab_percpu_map_update_elem(struct bpf_map map, void key, · 59 unmodified lines
1148	1151	/* unknown flags */
1149	1152	return -EINVAL;
1150	1153
1151		− WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
	1154	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
	1155	+ !rcu_read_lock_bh_held());
1152	1156
1153	1157	key_size = map->key_size;
1154	1158
@@ -1202,7 +1206,8 @@ static int __htab_lru_percpu_map_update_elem(struct bpf_map map, void key, · 47 unmodified lines
1202	1206	/* unknown flags */
1203	1207	return -EINVAL;
1204	1208
1205		− WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
	1209	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
	1210	+ !rcu_read_lock_bh_held());
1206	1211
1207	1212	key_size = map->key_size;
1208	1213
@@ -1276,7 +1281,8 @@ static int htab_map_delete_elem(struct bpf_map map, void key) · 67 unmodified lines
1276	1281	u32 hash, key_size;
1277	1282	int ret;
1278	1283
1279		− WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
	1284	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
	1285	+ !rcu_read_lock_bh_held());
1280	1286
1281	1287	key_size = map->key_size;
1282	1288
@@ -1311,7 +1317,8 @@ static int htab_lru_map_delete_elem(struct bpf_map map, void key) · 28 unmodified lines
1311	1317	u32 hash, key_size;
1312	1318	int ret;
1313	1319
1314		− WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
	1320	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
	1321	+ !rcu_read_lock_bh_held());
1315	1322
1316	1323	key_size = map->key_size;
1317	1324

kernel/bpf/helpers.c+3−3

@@ -29,7 +29,7 @@ · 28 unmodified lines
29	29	*/
30	30	BPF_CALL_2(bpf_map_lookup_elem, struct bpf_map , map, void , key)
31	31	{
32		− WARN_ON_ONCE(!rcu_read_lock_held());
	32	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
33	33	return (unsigned long) map->ops->map_lookup_elem(map, key);
34	34	}
35	35
@@ -45,7 +45,7 @@ const struct bpf_func_proto bpf_map_lookup_elem_proto = { · 9 unmodified lines
45	45	BPF_CALL_4(bpf_map_update_elem, struct bpf_map , map, void , key,
46	46	void *, value, u64, flags)
47	47	{
48		− WARN_ON_ONCE(!rcu_read_lock_held());
	48	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
49	49	return map->ops->map_update_elem(map, key, value, flags);
50	50	}
51	51
@@ -62,7 +62,7 @@ const struct bpf_func_proto bpf_map_update_elem_proto = { · 10 unmodified lines
62	62
63	63	BPF_CALL_2(bpf_map_delete_elem, struct bpf_map , map, void , key)
64	64	{
65		− WARN_ON_ONCE(!rcu_read_lock_held());
	65	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
66	66	return map->ops->map_delete_elem(map, key);
67	67	}
68	68

kernel/bpf/lpm_trie.c+4−2

@@ -232,7 +232,8 @@ static void trie_lookup_elem(struct bpf_map map, void *_key) · 231 unmodified lines
232	232
233	233	/* Start walking the trie from the root node ... */
234	234
235		− for (node = rcu_dereference(trie->root); node;) {
	235	+ for (node = rcu_dereference_check(trie->root, rcu_read_lock_bh_held());
	236	+ node;) {
236	237	unsigned int next_bit;
237	238	size_t matchlen;
238	239
@@ -264,7 +265,8 @@ static void trie_lookup_elem(struct bpf_map map, void *_key) · 25 unmodified lines
264	265	* traverse down.
265	266	*/
266	267	next_bit = extract_bit(key->data, node->prefixlen);
267		− node = rcu_dereference(node->child[next_bit]);
	268	+ node = rcu_dereference_check(node->child[next_bit],
	269	+ rcu_read_lock_bh_held());
268	270	}
269	271
270	272	if (!found)

kernel/bpf/ringbuf.c+2−0

@@ -8,6 +8,7 @@ · 7 unmodified lines
8	8	#include <linux/vmalloc.h>
9	9	#include <linux/wait.h>
10	10	#include <linux/poll.h>
	11	+#include <linux/kmemleak.h>
11	12	#include <uapi/linux/btf.h>
12	13
13	14	#define RINGBUF_CREATE_FLAG_MASK (BPF_F_NUMA_NODE)
@@ -105,6 +106,7 @@ static struct bpf_ringbuf *bpf_ringbuf_area_alloc(size_t data_sz, int numa_node) · 91 unmodified lines
105	106	rb = vmap(pages, nr_meta_pages + 2 * nr_data_pages,
106	107	VM_ALLOC \| VM_USERMAP, PAGE_KERNEL);
107	108	if (rb) {
	109	+ kmemleak_not_leak(pages);
108	110	rb->pages = pages;
109	111	rb->nr_pages = nr_pages;
110	112	return rb;

kernel/trace/bpf_trace.c+2−0

@@ -1017,6 +1017,8 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 1016 unmodified lines
1017	1017	#ifdef CONFIG_CGROUPS
1018	1018	case BPF_FUNC_get_current_cgroup_id:
1019	1019	return &bpf_get_current_cgroup_id_proto;
	1020	+ case BPF_FUNC_get_current_ancestor_cgroup_id:
	1021	+ return &bpf_get_current_ancestor_cgroup_id_proto;
1020	1022	#endif
1021	1023	case BPF_FUNC_send_signal:
1022	1024	return &bpf_send_signal_proto;

Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code