Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Daniel Borkmann says: ==================== pull-request: bpf-next 2021-06-28 The following pull-request contains BPF updates for your *net-next* tree. We've added 37 non-merge commits during the last 12 day(s) which contain a total of 56 files changed, 394 insertions(+), 380 deletions(-). The main changes are: 1) XDP driver RCU cleanups, from Toke Høiland-Jørgensen and Paul E. McKenney. 2) Fix bpf_skb_change_proto() IPv4/v6 GSO handling, from Maciej Żenczykowski. 3) Fix false positive kmemleak report for BPF ringbuf alloc, from Rustam Kovhaev. 4) Fix x86 JIT's extable offset calculation for PROBE_LDX NULL, from Ravi Bangoria. 5) Enable libbpf fallback probing with tracing under RHEL7, from Jonathan Edwards. 6) Clean up x86 JIT to remove unused cnt tracking from EMIT macro, from Jiri Olsa. 7) Netlink cleanups for libbpf to please Coverity, from Kumar Kartikeya Dwivedi. 8) Allow to retrieve ancestor cgroup id in tracing programs, from Namhyung Kim. 9) Fix lirc BPF program query to use user-provided prog_cnt, from Sean Young. 10) Add initial libbpf doc including generated kdoc for its API, from Grant Seltzer. 11) Make xdp_rxq_info_unreg_mem_model() more robust, from Jakub Kicinski. 12) Fix up bpfilter startup log-level to info level, from Gary Lin. ====================

David S. Miller committed 5 years ago

Commit e1289cf2 parents 1fd07f3 a78cae256 files · +394 −380Browse files

Showing the changes to kernel (7 of 56 files). Show all

Unified · Split

7 files

kernel/bpf/cpumap.c +9 −4
kernel/bpf/devmap.c +21 −28
kernel/bpf/hashtab.c +14 −7
kernel/bpf/helpers.c +3 −3
kernel/bpf/lpm_trie.c +4 −2
kernel/bpf/ringbuf.c +2 −0
kernel/trace/bpf_trace.c +2 −0

kernel/bpf/cpumap.c+9−4

 struct bpf_cpu_map {
 	struct bpf_map map;
 	/* Below members specific for map type */
-	struct bpf_cpu_map_entry **cpu_map;
+	struct bpf_cpu_map_entry __rcu **cpu_map;
 };
 static DEFINE_PER_CPU(struct list_head, cpu_map_flush_list);
 {
 	struct bpf_cpu_map_entry *old_rcpu;
-	old_rcpu = xchg(&cmap->cpu_map[key_cpu], rcpu);
+	old_rcpu = unrcu_pointer(xchg(&cmap->cpu_map[key_cpu], RCU_INITIALIZER(rcpu)));
 	if (old_rcpu) {
 		call_rcu(&old_rcpu->rcu, __cpu_map_entry_free);
 		INIT_WORK(&old_rcpu->kthread_stop_wq, cpu_map_kthread_stop);
 	for (i = 0; i < cmap->map.max_entries; i++) {
 		struct bpf_cpu_map_entry *rcpu;
-		rcpu = READ_ONCE(cmap->cpu_map[i]);
+		rcpu = rcu_dereference_raw(cmap->cpu_map[i]);
 		if (!rcpu)
 			continue;
 	kfree(cmap);
 }
+/* Elements are kept alive by RCU; either by rcu_read_lock() (from syscall) or
+ * by local_bh_disable() (from XDP calls inside NAPI). The
+ * rcu_read_lock_bh_held() below makes lockdep accept both.
+ */
 static void *__cpu_map_lookup_elem(struct bpf_map *map, u32 key)
 {
 	struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
 	if (key >= map->max_entries)
 		return NULL;
-	rcpu = READ_ONCE(cmap->cpu_map[key]);
+	rcpu = rcu_dereference_check(cmap->cpu_map[key],
+				     rcu_read_lock_bh_held());
 	return rcpu;
 }

kernel/bpf/devmap.c+21−28

 struct bpf_dtab {
 	struct bpf_map map;
-	struct bpf_dtab_netdev **netdev_map; /* DEVMAP type only */
+	struct bpf_dtab_netdev __rcu **netdev_map; /* DEVMAP type only */
 	struct list_head list;
 	/* these are only used for DEVMAP_HASH type maps */
 		for (i = 0; i < dtab->map.max_entries; i++) {
 			struct bpf_dtab_netdev *dev;
-			dev = dtab->netdev_map[i];
+			dev = rcu_dereference_raw(dtab->netdev_map[i]);
 			if (!dev)
 				continue;
 	return 0;
 }
+/* Elements are kept alive by RCU; either by rcu_read_lock() (from syscall) or
+ * by local_bh_disable() (from XDP calls inside NAPI). The
+ * rcu_read_lock_bh_held() below makes lockdep accept both.
+ */
 static void *__dev_map_hash_lookup_elem(struct bpf_map *map, u32 key)
 {
 	struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);
 	trace_xdp_devmap_xmit(bq->dev_rx, dev, sent, cnt - sent, err);
 }
-/* __dev_flush is called from xdp_do_flush() which _must_ be signaled
+/* __dev_flush is called from xdp_do_flush() which _must_ be signalled from the
- * from the driver before returning from its napi->poll() routine. The poll()
+ * driver before returning from its napi->poll() routine. See the comment above
- * routine is called either from busy_poll context or net_rx_action signaled
+ * xdp_do_flush() in filter.c.
- * from NET_RX_SOFTIRQ. Either way the poll routine must complete before the
- * net device can be torn down. On devmap tear down we ensure the flush list
- * is empty before completing to ensure all flush operations have completed.
- * When drivers update the bpf program they may need to ensure any flush ops
- * are also complete. Using synchronize_rcu or call_rcu will suffice for this
- * because both wait for napi context to exit.
  */
 void __dev_flush(void)
 {
 	}
 }
-/* rcu_read_lock (from syscall and BPF contexts) ensures that if a delete and/or
+/* Elements are kept alive by RCU; either by rcu_read_lock() (from syscall) or
- * update happens in parallel here a dev_put won't happen until after reading
+ * by local_bh_disable() (from XDP calls inside NAPI). The
- * the ifindex.
+ * rcu_read_lock_bh_held() below makes lockdep accept both.
  */
 static void *__dev_map_lookup_elem(struct bpf_map *map, u32 key)
 {
 	if (key >= map->max_entries)
 		return NULL;
-	obj = READ_ONCE(dtab->netdev_map[key]);
+	obj = rcu_dereference_check(dtab->netdev_map[key],
+				    rcu_read_lock_bh_held());
 	return obj;
 }
-/* Runs under RCU-read-side, plus in softirq under NAPI protection.
+/* Runs in NAPI, i.e., softirq under local_bh_disable(). Thus, safe percpu
- * Thus, safe percpu variable access.
+ * variable access, and map elements stick around. See comment above
+ * xdp_do_flush() in filter.c.
  */
 static void bq_enqueue(struct net_device *dev, struct xdp_frame *xdpf,
 		       struct net_device *dev_rx, struct bpf_prog *xdp_prog)
 	if (k >= map->max_entries)
 		return -EINVAL;
-	/* Use call_rcu() here to ensure any rcu critical sections have
+	old_dev = unrcu_pointer(xchg(&dtab->netdev_map[k], NULL));
-	 * completed as well as any flush operations because call_rcu
-	 * will wait for preempt-disable region to complete, NAPI in this
-	 * context.  And additionally, the driver tear down ensures all
-	 * soft irqs are complete before removing the net device in the
-	 * case of dev_put equals zero.
-	 */
-	old_dev = xchg(&dtab->netdev_map[k], NULL);
 	if (old_dev)
 		call_rcu(&old_dev->rcu, __dev_map_entry_free);
 	return 0;
 	 * Remembering the driver side flush operation will happen before the
 	 * net device is removed.
 	 */
-	old_dev = xchg(&dtab->netdev_map[i], dev);
+	old_dev = unrcu_pointer(xchg(&dtab->netdev_map[i], RCU_INITIALIZER(dev)));
 	if (old_dev)
 		call_rcu(&old_dev->rcu, __dev_map_entry_free);
 			for (i = 0; i < dtab->map.max_entries; i++) {
 				struct bpf_dtab_netdev *dev, *odev;
-				dev = READ_ONCE(dtab->netdev_map[i]);
+				dev = rcu_dereference(dtab->netdev_map[i]);
 				if (!dev || netdev != dev->dev)
 					continue;
-				odev = cmpxchg(&dtab->netdev_map[i], dev, NULL);
+				odev = unrcu_pointer(cmpxchg(&dtab->netdev_map[i], RCU_INITIALIZER(dev), NULL));
 				if (dev == odev)
 					call_rcu(&dev->rcu,
 						 __dev_map_entry_free);

kernel/bpf/hashtab.c+14−7

 	struct htab_elem *l;
 	u32 hash, key_size;
-	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
+	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
+		     !rcu_read_lock_bh_held());
 	key_size = map->key_size;
 		/* unknown flags */
 		return -EINVAL;
-	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
+	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
+		     !rcu_read_lock_bh_held());
 	key_size = map->key_size;
 		/* unknown flags */
 		return -EINVAL;
-	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
+	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
+		     !rcu_read_lock_bh_held());
 	key_size = map->key_size;
 		/* unknown flags */
 		return -EINVAL;
-	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
+	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
+		     !rcu_read_lock_bh_held());
 	key_size = map->key_size;
 		/* unknown flags */
 		return -EINVAL;
-	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
+	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
+		     !rcu_read_lock_bh_held());
 	key_size = map->key_size;
 	u32 hash, key_size;
 	int ret;
-	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
+	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
+		     !rcu_read_lock_bh_held());
 	key_size = map->key_size;
 	u32 hash, key_size;
 	int ret;
-	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
+	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
+		     !rcu_read_lock_bh_held());
 	key_size = map->key_size;

kernel/bpf/helpers.c+3−3

@@ -29,7 +29,7 @@ · 28 unmodified lines
29	*/	29	*/
30	BPF_CALL_2(bpf_map_lookup_elem, struct bpf_map , map, void , key)	30	BPF_CALL_2(bpf_map_lookup_elem, struct bpf_map , map, void , key)
31	{	31	{
32	WARN_ON_ONCE(!rcu_read_lock_held());	32	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
33	return (unsigned long) map->ops->map_lookup_elem(map, key);	33	return (unsigned long) map->ops->map_lookup_elem(map, key);
34	}	34	}
35		35
@@ -45,7 +45,7 @@ const struct bpf_func_proto bpf_map_lookup_elem_proto = { · 9 unmodified lines
45	BPF_CALL_4(bpf_map_update_elem, struct bpf_map , map, void , key,	45	BPF_CALL_4(bpf_map_update_elem, struct bpf_map , map, void , key,
46	void *, value, u64, flags)	46	void *, value, u64, flags)
47	{	47	{
48	WARN_ON_ONCE(!rcu_read_lock_held());	48	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
49	return map->ops->map_update_elem(map, key, value, flags);	49	return map->ops->map_update_elem(map, key, value, flags);
50	}	50	}
51		51
@@ -62,7 +62,7 @@ const struct bpf_func_proto bpf_map_update_elem_proto = { · 10 unmodified lines
62		62
63	BPF_CALL_2(bpf_map_delete_elem, struct bpf_map , map, void , key)	63	BPF_CALL_2(bpf_map_delete_elem, struct bpf_map , map, void , key)
64	{	64	{
65	WARN_ON_ONCE(!rcu_read_lock_held());	65	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
66	return map->ops->map_delete_elem(map, key);	66	return map->ops->map_delete_elem(map, key);
67	}	67	}
68		68

kernel/bpf/lpm_trie.c+4−2

 	/* Start walking the trie from the root node ... */
-	for (node = rcu_dereference(trie->root); node;) {
+	for (node = rcu_dereference_check(trie->root, rcu_read_lock_bh_held());
+	     node;) {
 		unsigned int next_bit;
 		size_t matchlen;
 		 * traverse down.
 		 */
 		next_bit = extract_bit(key->data, node->prefixlen);
-		node = rcu_dereference(node->child[next_bit]);
+		node = rcu_dereference_check(node->child[next_bit],
+					     rcu_read_lock_bh_held());
 	}
 	if (!found)

kernel/bpf/ringbuf.c+2−0

 #include <linux/vmalloc.h>
 #include <linux/wait.h>
 #include <linux/poll.h>
+#include <linux/kmemleak.h>
 #include <uapi/linux/btf.h>
 #define RINGBUF_CREATE_FLAG_MASK (BPF_F_NUMA_NODE)
 	rb = vmap(pages, nr_meta_pages + 2 * nr_data_pages,
 		  VM_ALLOC | VM_USERMAP, PAGE_KERNEL);
 	if (rb) {
+		kmemleak_not_leak(pages);
 		rb->pages = pages;
 		rb->nr_pages = nr_pages;
 		return rb;

kernel/trace/bpf_trace.c+2−0

 #ifdef CONFIG_CGROUPS
 	case BPF_FUNC_get_current_cgroup_id:
 		return &bpf_get_current_cgroup_id_proto;
+	case BPF_FUNC_get_current_ancestor_cgroup_id:
+		return &bpf_get_current_ancestor_cgroup_id_proto;
 #endif
 	case BPF_FUNC_send_signal:
 		return &bpf_send_signal_proto;

Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code