Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Daniel Borkmann says: ==================== pull-request: bpf-next 2020-09-01 The following pull-request contains BPF updates for your *net-next* tree. There are two small conflicts when pulling, resolve as follows: 1) Merge conflict in tools/lib/bpf/libbpf.c between 88a82120282b ("libbpf: Factor out common ELF operations and improve logging") in bpf-next and 1e891e513e16 ("libbpf: Fix map index used in error message") in net-next. Resolve by taking the hunk in bpf-next: [...] scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx); data = elf_sec_data(obj, scn); if (!scn || !data) { pr_warn("elf: failed to get %s map definitions for %s\n", MAPS_ELF_SEC, obj->path); return -EINVAL; } [...] 2) Merge conflict in drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c between 9647c57b11e5 ("xsk: i40e: ice: ixgbe: mlx5: Test for dma_need_sync earlier for better performance") in bpf-next and e20f0dbf204f ("net/mlx5e: RX, Add a prefetch command for small L1_CACHE_BYTES") in net-next. Resolve the two locations by retaining net_prefetch() and taking xsk_buff_dma_sync_for_cpu() from bpf-next. Should look like: [...] xdp_set_data_meta_invalid(xdp); xsk_buff_dma_sync_for_cpu(xdp, rq->xsk_pool); net_prefetch(xdp->data); [...] We've added 133 non-merge commits during the last 14 day(s) which contain a total of 246 files changed, 13832 insertions(+), 3105 deletions(-). The main changes are: 1) Initial support for sleepable BPF programs along with bpf_copy_from_user() helper for tracing to reliably access user memory, from Alexei Starovoitov. 2) Add BPF infra for writing and parsing TCP header options, from Martin KaFai Lau. 3) bpf_d_path() helper for returning full path for given 'struct path', from Jiri Olsa. 4) AF_XDP support for shared umems between devices and queues, from Magnus Karlsson. 5) Initial prep work for full BPF-to-BPF call support in libbpf, from Andrii Nakryiko. 6) Generalize bpf_sk_storage map & add local storage for inodes, from KP Singh. 7) Implement sockmap/hash updates from BPF context, from Lorenz Bauer. 8) BPF xor verification for scalar types & add BPF link iterator, from Yonghong Song. 9) Use target's prog type for BPF_PROG_TYPE_EXT prog verification, from Udip Pant. 10) Rework BPF tracing samples to use libbpf loader, from Daniel T. Lee. 11) Fix xdpsock sample to really cycle through all buffers, from Weqaar Janjua. 12) Improve type safety for tun/veth XDP frame handling, from Maciej Żenczykowski. 13) Various smaller cleanups and improvements all over the place. ====================

David S. Miller committed 6 years ago

Commit 150f29f2 parents 8aa639e ebc4ecd246 files · +13832 −3105Browse files

Showing the changes to kernel (38 of 246 files). Show all

Unified · Split

38 files

kernel/Makefile +1 −1
kernel/bpf/Makefile +3 −0
kernel/bpf/arraymap.c +17 −0
kernel/bpf/bpf_inode_storage.c +274 −0
kernel/bpf/bpf_iter.c +58 −0
kernel/bpf/bpf_local_storage.c +600 −0
kernel/bpf/bpf_lsm.c +20 −1
kernel/bpf/bpf_struct_ops.c +2 −4
kernel/bpf/btf.c +139 −24
kernel/bpf/cpumap.c +4 −8
kernel/bpf/devmap.c +9 −8
kernel/bpf/hashtab.c +10 −6
kernel/bpf/helpers.c +22 −0
kernel/bpf/inode.c +113 −3
kernel/bpf/lpm_trie.c +1 −0
kernel/bpf/map_in_map.c +9 −15
kernel/bpf/map_in_map.h +0 −2
kernel/bpf/map_iter.c +15 −0
kernel/bpf/preload/Kconfig +26 −0
kernel/bpf/preload/Makefile +23 −0
kernel/bpf/preload/bpf_preload.h +16 −0
kernel/bpf/preload/bpf_preload_kern.c +91 −0
kernel/bpf/preload/bpf_preload_umd_blob.S +7 −0
kernel/bpf/preload/iterators/.gitignore +2 −0
kernel/bpf/preload/iterators/Makefile +57 −0
kernel/bpf/preload/iterators/README +4 −0
kernel/bpf/preload/iterators/bpf_preload_common.h +13 −0
kernel/bpf/preload/iterators/iterators.bpf.c +114 −0
kernel/bpf/preload/iterators/iterators.c +94 −0
kernel/bpf/preload/iterators/iterators.skel.h +410 −0
kernel/bpf/queue_stack_maps.c +2 −0
kernel/bpf/reuseport_array.c +1 −0
kernel/bpf/ringbuf.c +1 −0
kernel/bpf/stackmap.c +1 −0
kernel/bpf/syscall.c +44 −24
kernel/bpf/trampoline.c +26 −3
kernel/bpf/verifier.c +263 −20
kernel/trace/bpf_trace.c +50 −0

kernel/Makefile+1−1

@@ -12,7 +12,7 @@ obj-y = fork.o exec_domain.o panic.o \ · 11 unmodified lines
12	12	notifier.o ksysfs.o cred.o reboot.o \
13	13	async.o range.o smpboot.o ucount.o regset.o
14	14
15		−obj-$(CONFIG_BPFILTER) += usermode_driver.o
	15	+obj-$(CONFIG_USERMODE_DRIVER) += usermode_driver.o
16	16	obj-$(CONFIG_MODULES) += kmod.o
17	17	obj-$(CONFIG_MULTIUSER) += groups.o
18	18

kernel/bpf/Makefile+3−0

@@ -5,6 +5,7 @@ CFLAGS_core.o += $(call cc-disable-warning, override-init) · 4 unmodified lines
5	5	obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o bpf_iter.o map_iter.o task_iter.o prog_iter.o
6	6	obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o
7	7	obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o
	8	+obj-${CONFIG_BPF_LSM} += bpf_inode_storage.o
8	9	obj-$(CONFIG_BPF_SYSCALL) += disasm.o
9	10	obj-$(CONFIG_BPF_JIT) += trampoline.o
10	11	obj-$(CONFIG_BPF_SYSCALL) += btf.o
@@ -12,6 +13,7 @@ obj-$(CONFIG_BPF_JIT) += dispatcher.o · 1 unmodified line
12	13	ifeq ($(CONFIG_NET),y)
13	14	obj-$(CONFIG_BPF_SYSCALL) += devmap.o
14	15	obj-$(CONFIG_BPF_SYSCALL) += cpumap.o
	16	+obj-$(CONFIG_BPF_SYSCALL) += bpf_local_storage.o
15	17	obj-$(CONFIG_BPF_SYSCALL) += offload.o
16	18	obj-$(CONFIG_BPF_SYSCALL) += net_namespace.o
17	19	endif
@@ -29,3 +31,4 @@ ifeq ($(CONFIG_BPF_JIT),y) · 11 unmodified lines
29	31	obj-$(CONFIG_BPF_SYSCALL) += bpf_struct_ops.o
30	32	obj-${CONFIG_BPF_LSM} += bpf_lsm.o
31	33	endif
	34	+obj-$(CONFIG_BPF_PRELOAD) += preload/

kernel/bpf/arraymap.c+17−0

@@ -10,6 +10,7 @@ · 9 unmodified lines
10	10	#include <linux/filter.h>
11	11	#include <linux/perf_event.h>
12	12	#include <uapi/linux/btf.h>
	13	+#include <linux/rcupdate_trace.h>
13	14
14	15	#include "map_in_map.h"
15	16
@@ -487,6 +488,13 @@ static int array_map_mmap(struct bpf_map map, struct vm_area_struct vma) · 471 unmodified lines
487	488	vma->vm_pgoff + pgoff);
488	489	}
489	490
	491	+static bool array_map_meta_equal(const struct bpf_map *meta0,
	492	+ const struct bpf_map *meta1)
	493	+{
	494	+ return meta0->max_entries == meta1->max_entries &&
	495	+ bpf_map_meta_equal(meta0, meta1);
	496	+}
	497	+
490	498	struct bpf_iter_seq_array_map_info {
491	499	struct bpf_map *map;
492	500	void *percpu_value_buf;
@@ -625,6 +633,7 @@ static const struct bpf_iter_seq_info iter_seq_info = { · 132 unmodified lines
625	633
626	634	static int array_map_btf_id;
627	635	const struct bpf_map_ops array_map_ops = {
	636	+ .map_meta_equal = array_map_meta_equal,
628	637	.map_alloc_check = array_map_alloc_check,
629	638	.map_alloc = array_map_alloc,
630	639	.map_free = array_map_free,
@@ -647,6 +656,7 @@ const struct bpf_map_ops array_map_ops = { · 16 unmodified lines
647	656
648	657	static int percpu_array_map_btf_id;
649	658	const struct bpf_map_ops percpu_array_map_ops = {
	659	+ .map_meta_equal = bpf_map_meta_equal,
650	660	.map_alloc_check = array_map_alloc_check,
651	661	.map_alloc = array_map_alloc,
652	662	.map_free = array_map_free,
@@ -1003,6 +1013,11 @@ static void prog_array_map_free(struct bpf_map *map) · 350 unmodified lines
1003	1013	fd_array_map_free(map);
1004	1014	}
1005	1015
	1016	+/* prog_array->aux->{type,jited} is a runtime binding.
	1017	+ * Doing static check alone in the verifier is not enough.
	1018	+ * Thus, prog_array_map cannot be used as an inner_map
	1019	+ * and map_meta_equal is not implemented.
	1020	+ */
1006	1021	static int prog_array_map_btf_id;
1007	1022	const struct bpf_map_ops prog_array_map_ops = {
1008	1023	.map_alloc_check = fd_array_map_alloc_check,
@@ -1101,6 +1116,7 @@ static void perf_event_fd_array_release(struct bpf_map *map, · 92 unmodified lines
1101	1116
1102	1117	static int perf_event_array_map_btf_id;
1103	1118	const struct bpf_map_ops perf_event_array_map_ops = {
	1119	+ .map_meta_equal = bpf_map_meta_equal,
1104	1120	.map_alloc_check = fd_array_map_alloc_check,
1105	1121	.map_alloc = array_map_alloc,
1106	1122	.map_free = fd_array_map_free,
@@ -1137,6 +1153,7 @@ static void cgroup_fd_array_free(struct bpf_map *map) · 30 unmodified lines
1137	1153
1138	1154	static int cgroup_array_map_btf_id;
1139	1155	const struct bpf_map_ops cgroup_array_map_ops = {
	1156	+ .map_meta_equal = bpf_map_meta_equal,
1140	1157	.map_alloc_check = fd_array_map_alloc_check,
1141	1158	.map_alloc = array_map_alloc,
1142	1159	.map_free = cgroup_fd_array_free,

kernel/bpf/bpf_inode_storage.c+274−0

@@ -0,0 +1,274 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/*
	3	+ * Copyright (c) 2019 Facebook
	4	+ * Copyright 2020 Google LLC.
	5	+ */
	6	+
	7	+#include <linux/rculist.h>
	8	+#include <linux/list.h>
	9	+#include <linux/hash.h>
	10	+#include <linux/types.h>
	11	+#include <linux/spinlock.h>
	12	+#include <linux/bpf.h>
	13	+#include <linux/bpf_local_storage.h>
	14	+#include <net/sock.h>
	15	+#include <uapi/linux/sock_diag.h>
	16	+#include <uapi/linux/btf.h>
	17	+#include <linux/bpf_lsm.h>
	18	+#include <linux/btf_ids.h>
	19	+#include <linux/fdtable.h>
	20	+
	21	+DEFINE_BPF_STORAGE_CACHE(inode_cache);
	22	+
	23	+static struct bpf_local_storage __rcu **
	24	+inode_storage_ptr(void *owner)
	25	+{
	26	+ struct inode *inode = owner;
	27	+ struct bpf_storage_blob *bsb;
	28	+
	29	+ bsb = bpf_inode(inode);
	30	+ if (!bsb)
	31	+ return NULL;
	32	+ return &bsb->storage;
	33	+}
	34	+
	35	+static struct bpf_local_storage_data inode_storage_lookup(struct inode inode,
	36	+ struct bpf_map *map,
	37	+ bool cacheit_lockit)
	38	+{
	39	+ struct bpf_local_storage *inode_storage;
	40	+ struct bpf_local_storage_map *smap;
	41	+ struct bpf_storage_blob *bsb;
	42	+
	43	+ bsb = bpf_inode(inode);
	44	+ if (!bsb)
	45	+ return NULL;
	46	+
	47	+ inode_storage = rcu_dereference(bsb->storage);
	48	+ if (!inode_storage)
	49	+ return NULL;
	50	+
	51	+ smap = (struct bpf_local_storage_map *)map;
	52	+ return bpf_local_storage_lookup(inode_storage, smap, cacheit_lockit);
	53	+}
	54	+
	55	+void bpf_inode_storage_free(struct inode *inode)
	56	+{
	57	+ struct bpf_local_storage_elem *selem;
	58	+ struct bpf_local_storage *local_storage;
	59	+ bool free_inode_storage = false;
	60	+ struct bpf_storage_blob *bsb;
	61	+ struct hlist_node *n;
	62	+
	63	+ bsb = bpf_inode(inode);
	64	+ if (!bsb)
	65	+ return;
	66	+
	67	+ rcu_read_lock();
	68	+
	69	+ local_storage = rcu_dereference(bsb->storage);
	70	+ if (!local_storage) {
	71	+ rcu_read_unlock();
	72	+ return;
	73	+ }
	74	+
	75	+ /* Netiher the bpf_prog nor the bpf-map's syscall
	76	+ * could be modifying the local_storage->list now.
	77	+ * Thus, no elem can be added-to or deleted-from the
	78	+ * local_storage->list by the bpf_prog or by the bpf-map's syscall.
	79	+ *
	80	+ * It is racing with bpf_local_storage_map_free() alone
	81	+ * when unlinking elem from the local_storage->list and
	82	+ * the map's bucket->list.
	83	+ */
	84	+ raw_spin_lock_bh(&local_storage->lock);
	85	+ hlist_for_each_entry_safe(selem, n, &local_storage->list, snode) {
	86	+ /* Always unlink from map before unlinking from
	87	+ * local_storage.
	88	+ */
	89	+ bpf_selem_unlink_map(selem);
	90	+ free_inode_storage = bpf_selem_unlink_storage_nolock(
	91	+ local_storage, selem, false);
	92	+ }
	93	+ raw_spin_unlock_bh(&local_storage->lock);
	94	+ rcu_read_unlock();
	95	+
	96	+ /* free_inoode_storage should always be true as long as
	97	+ * local_storage->list was non-empty.
	98	+ */
	99	+ if (free_inode_storage)
	100	+ kfree_rcu(local_storage, rcu);
	101	+}
	102	+
	103	+static void bpf_fd_inode_storage_lookup_elem(struct bpf_map map, void *key)
	104	+{
	105	+ struct bpf_local_storage_data *sdata;
	106	+ struct file *f;
	107	+ int fd;
	108	+
	109	+ fd = (int )key;
	110	+ f = fget_raw(fd);
	111	+ if (!f)
	112	+ return NULL;
	113	+
	114	+ sdata = inode_storage_lookup(f->f_inode, map, true);
	115	+ fput(f);
	116	+ return sdata ? sdata->data : NULL;
	117	+}
	118	+
	119	+static int bpf_fd_inode_storage_update_elem(struct bpf_map map, void key,
	120	+ void *value, u64 map_flags)
	121	+{
	122	+ struct bpf_local_storage_data *sdata;
	123	+ struct file *f;
	124	+ int fd;
	125	+
	126	+ fd = (int )key;
	127	+ f = fget_raw(fd);
	128	+ if (!f \|\| !inode_storage_ptr(f->f_inode))
	129	+ return -EBADF;
	130	+
	131	+ sdata = bpf_local_storage_update(f->f_inode,
	132	+ (struct bpf_local_storage_map *)map,
	133	+ value, map_flags);
	134	+ fput(f);
	135	+ return PTR_ERR_OR_ZERO(sdata);
	136	+}
	137	+
	138	+static int inode_storage_delete(struct inode inode, struct bpf_map map)
	139	+{
	140	+ struct bpf_local_storage_data *sdata;
	141	+
	142	+ sdata = inode_storage_lookup(inode, map, false);
	143	+ if (!sdata)
	144	+ return -ENOENT;
	145	+
	146	+ bpf_selem_unlink(SELEM(sdata));
	147	+
	148	+ return 0;
	149	+}
	150	+
	151	+static int bpf_fd_inode_storage_delete_elem(struct bpf_map map, void key)
	152	+{
	153	+ struct file *f;
	154	+ int fd, err;
	155	+
	156	+ fd = (int )key;
	157	+ f = fget_raw(fd);
	158	+ if (!f)
	159	+ return -EBADF;
	160	+
	161	+ err = inode_storage_delete(f->f_inode, map);
	162	+ fput(f);
	163	+ return err;
	164	+}
	165	+
	166	+BPF_CALL_4(bpf_inode_storage_get, struct bpf_map , map, struct inode , inode,
	167	+ void *, value, u64, flags)
	168	+{
	169	+ struct bpf_local_storage_data *sdata;
	170	+
	171	+ if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
	172	+ return (unsigned long)NULL;
	173	+
	174	+ /* explicitly check that the inode_storage_ptr is not
	175	+ * NULL as inode_storage_lookup returns NULL in this case and
	176	+ * bpf_local_storage_update expects the owner to have a
	177	+ * valid storage pointer.
	178	+ */
	179	+ if (!inode_storage_ptr(inode))
	180	+ return (unsigned long)NULL;
	181	+
	182	+ sdata = inode_storage_lookup(inode, map, true);
	183	+ if (sdata)
	184	+ return (unsigned long)sdata->data;
	185	+
	186	+ /* This helper must only called from where the inode is gurranteed
	187	+ * to have a refcount and cannot be freed.
	188	+ */
	189	+ if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) {
	190	+ sdata = bpf_local_storage_update(
	191	+ inode, (struct bpf_local_storage_map *)map, value,
	192	+ BPF_NOEXIST);
	193	+ return IS_ERR(sdata) ? (unsigned long)NULL :
	194	+ (unsigned long)sdata->data;
	195	+ }
	196	+
	197	+ return (unsigned long)NULL;
	198	+}
	199	+
	200	+BPF_CALL_2(bpf_inode_storage_delete,
	201	+ struct bpf_map , map, struct inode , inode)
	202	+{
	203	+ /* This helper must only called from where the inode is gurranteed
	204	+ * to have a refcount and cannot be freed.
	205	+ */
	206	+ return inode_storage_delete(inode, map);
	207	+}
	208	+
	209	+static int notsupp_get_next_key(struct bpf_map map, void key,
	210	+ void *next_key)
	211	+{
	212	+ return -ENOTSUPP;
	213	+}
	214	+
	215	+static struct bpf_map inode_storage_map_alloc(union bpf_attr attr)
	216	+{
	217	+ struct bpf_local_storage_map *smap;
	218	+
	219	+ smap = bpf_local_storage_map_alloc(attr);
	220	+ if (IS_ERR(smap))
	221	+ return ERR_CAST(smap);
	222	+
	223	+ smap->cache_idx = bpf_local_storage_cache_idx_get(&inode_cache);
	224	+ return &smap->map;
	225	+}
	226	+
	227	+static void inode_storage_map_free(struct bpf_map *map)
	228	+{
	229	+ struct bpf_local_storage_map *smap;
	230	+
	231	+ smap = (struct bpf_local_storage_map *)map;
	232	+ bpf_local_storage_cache_idx_free(&inode_cache, smap->cache_idx);
	233	+ bpf_local_storage_map_free(smap);
	234	+}
	235	+
	236	+static int inode_storage_map_btf_id;
	237	+const struct bpf_map_ops inode_storage_map_ops = {
	238	+ .map_meta_equal = bpf_map_meta_equal,
	239	+ .map_alloc_check = bpf_local_storage_map_alloc_check,
	240	+ .map_alloc = inode_storage_map_alloc,
	241	+ .map_free = inode_storage_map_free,
	242	+ .map_get_next_key = notsupp_get_next_key,
	243	+ .map_lookup_elem = bpf_fd_inode_storage_lookup_elem,
	244	+ .map_update_elem = bpf_fd_inode_storage_update_elem,
	245	+ .map_delete_elem = bpf_fd_inode_storage_delete_elem,
	246	+ .map_check_btf = bpf_local_storage_map_check_btf,
	247	+ .map_btf_name = "bpf_local_storage_map",
	248	+ .map_btf_id = &inode_storage_map_btf_id,
	249	+ .map_owner_storage_ptr = inode_storage_ptr,
	250	+};
	251	+
	252	+BTF_ID_LIST(bpf_inode_storage_btf_ids)
	253	+BTF_ID_UNUSED
	254	+BTF_ID(struct, inode)
	255	+
	256	+const struct bpf_func_proto bpf_inode_storage_get_proto = {
	257	+ .func = bpf_inode_storage_get,
	258	+ .gpl_only = false,
	259	+ .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
	260	+ .arg1_type = ARG_CONST_MAP_PTR,
	261	+ .arg2_type = ARG_PTR_TO_BTF_ID,
	262	+ .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL,
	263	+ .arg4_type = ARG_ANYTHING,
	264	+ .btf_id = bpf_inode_storage_btf_ids,
	265	+};
	266	+
	267	+const struct bpf_func_proto bpf_inode_storage_delete_proto = {
	268	+ .func = bpf_inode_storage_delete,
	269	+ .gpl_only = false,
	270	+ .ret_type = RET_INTEGER,
	271	+ .arg1_type = ARG_CONST_MAP_PTR,
	272	+ .arg2_type = ARG_PTR_TO_BTF_ID,
	273	+ .btf_id = bpf_inode_storage_btf_ids,
	274	+};

kernel/bpf/bpf_iter.c+58−0

@@ -390,10 +390,68 @@ out_unlock: · 389 unmodified lines
390	390	return ret;
391	391	}
392	392
	393	+static void bpf_iter_link_show_fdinfo(const struct bpf_link *link,
	394	+ struct seq_file *seq)
	395	+{
	396	+ struct bpf_iter_link *iter_link =
	397	+ container_of(link, struct bpf_iter_link, link);
	398	+ bpf_iter_show_fdinfo_t show_fdinfo;
	399	+
	400	+ seq_printf(seq,
	401	+ "target_name:\t%s\n",
	402	+ iter_link->tinfo->reg_info->target);
	403	+
	404	+ show_fdinfo = iter_link->tinfo->reg_info->show_fdinfo;
	405	+ if (show_fdinfo)
	406	+ show_fdinfo(&iter_link->aux, seq);
	407	+}
	408	+
	409	+static int bpf_iter_link_fill_link_info(const struct bpf_link *link,
	410	+ struct bpf_link_info *info)
	411	+{
	412	+ struct bpf_iter_link *iter_link =
	413	+ container_of(link, struct bpf_iter_link, link);
	414	+ char __user *ubuf = u64_to_user_ptr(info->iter.target_name);
	415	+ bpf_iter_fill_link_info_t fill_link_info;
	416	+ u32 ulen = info->iter.target_name_len;
	417	+ const char *target_name;
	418	+ u32 target_len;
	419	+
	420	+ if (!ulen ^ !ubuf)
	421	+ return -EINVAL;
	422	+
	423	+ target_name = iter_link->tinfo->reg_info->target;
	424	+ target_len = strlen(target_name);
	425	+ info->iter.target_name_len = target_len + 1;
	426	+
	427	+ if (ubuf) {
	428	+ if (ulen >= target_len + 1) {
	429	+ if (copy_to_user(ubuf, target_name, target_len + 1))
	430	+ return -EFAULT;
	431	+ } else {
	432	+ char zero = '\0';
	433	+
	434	+ if (copy_to_user(ubuf, target_name, ulen - 1))
	435	+ return -EFAULT;
	436	+ if (put_user(zero, ubuf + ulen - 1))
	437	+ return -EFAULT;
	438	+ return -ENOSPC;
	439	+ }
	440	+ }
	441	+
	442	+ fill_link_info = iter_link->tinfo->reg_info->fill_link_info;
	443	+ if (fill_link_info)
	444	+ return fill_link_info(&iter_link->aux, info);
	445	+
	446	+ return 0;
	447	+}
	448	+
393	449	static const struct bpf_link_ops bpf_iter_link_lops = {
394	450	.release = bpf_iter_link_release,
395	451	.dealloc = bpf_iter_link_dealloc,
396	452	.update_prog = bpf_iter_link_replace,
	453	+ .show_fdinfo = bpf_iter_link_show_fdinfo,
	454	+ .fill_link_info = bpf_iter_link_fill_link_info,
397	455	};
398	456
399	457	bool bpf_link_is_iter(struct bpf_link *link)

kernel/bpf/bpf_local_storage.c+600−0

@@ -0,0 +1,600 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2019 Facebook */
	3	+#include <linux/rculist.h>
	4	+#include <linux/list.h>
	5	+#include <linux/hash.h>
	6	+#include <linux/types.h>
	7	+#include <linux/spinlock.h>
	8	+#include <linux/bpf.h>
	9	+#include <linux/btf_ids.h>
	10	+#include <linux/bpf_local_storage.h>
	11	+#include <net/sock.h>
	12	+#include <uapi/linux/sock_diag.h>
	13	+#include <uapi/linux/btf.h>
	14	+
	15	+#define BPF_LOCAL_STORAGE_CREATE_FLAG_MASK (BPF_F_NO_PREALLOC \| BPF_F_CLONE)
	16	+
	17	+static struct bpf_local_storage_map_bucket *
	18	+select_bucket(struct bpf_local_storage_map *smap,
	19	+ struct bpf_local_storage_elem *selem)
	20	+{
	21	+ return &smap->buckets[hash_ptr(selem, smap->bucket_log)];
	22	+}
	23	+
	24	+static int mem_charge(struct bpf_local_storage_map smap, void owner, u32 size)
	25	+{
	26	+ struct bpf_map *map = &smap->map;
	27	+
	28	+ if (!map->ops->map_local_storage_charge)
	29	+ return 0;
	30	+
	31	+ return map->ops->map_local_storage_charge(smap, owner, size);
	32	+}
	33	+
	34	+static void mem_uncharge(struct bpf_local_storage_map smap, void owner,
	35	+ u32 size)
	36	+{
	37	+ struct bpf_map *map = &smap->map;
	38	+
	39	+ if (map->ops->map_local_storage_uncharge)
	40	+ map->ops->map_local_storage_uncharge(smap, owner, size);
	41	+}
	42	+
	43	+static struct bpf_local_storage __rcu **
	44	+owner_storage(struct bpf_local_storage_map smap, void owner)
	45	+{
	46	+ struct bpf_map *map = &smap->map;
	47	+
	48	+ return map->ops->map_owner_storage_ptr(owner);
	49	+}
	50	+
	51	+static bool selem_linked_to_storage(const struct bpf_local_storage_elem *selem)
	52	+{
	53	+ return !hlist_unhashed(&selem->snode);
	54	+}
	55	+
	56	+static bool selem_linked_to_map(const struct bpf_local_storage_elem *selem)
	57	+{
	58	+ return !hlist_unhashed(&selem->map_node);
	59	+}
	60	+
	61	+struct bpf_local_storage_elem *
	62	+bpf_selem_alloc(struct bpf_local_storage_map smap, void owner,
	63	+ void *value, bool charge_mem)
	64	+{
	65	+ struct bpf_local_storage_elem *selem;
	66	+
	67	+ if (charge_mem && mem_charge(smap, owner, smap->elem_size))
	68	+ return NULL;
	69	+
	70	+ selem = kzalloc(smap->elem_size, GFP_ATOMIC \| __GFP_NOWARN);
	71	+ if (selem) {
	72	+ if (value)
	73	+ memcpy(SDATA(selem)->data, value, smap->map.value_size);
	74	+ return selem;
	75	+ }
	76	+
	77	+ if (charge_mem)
	78	+ mem_uncharge(smap, owner, smap->elem_size);
	79	+
	80	+ return NULL;
	81	+}
	82	+
	83	+/* local_storage->lock must be held and selem->local_storage == local_storage.
	84	+ * The caller must ensure selem->smap is still valid to be
	85	+ * dereferenced for its smap->elem_size and smap->cache_idx.
	86	+ */
	87	+bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage,
	88	+ struct bpf_local_storage_elem *selem,
	89	+ bool uncharge_mem)
	90	+{
	91	+ struct bpf_local_storage_map *smap;
	92	+ bool free_local_storage;
	93	+ void *owner;
	94	+
	95	+ smap = rcu_dereference(SDATA(selem)->smap);
	96	+ owner = local_storage->owner;
	97	+
	98	+ /* All uncharging on the owner must be done first.
	99	+ * The owner may be freed once the last selem is unlinked
	100	+ * from local_storage.
	101	+ */
	102	+ if (uncharge_mem)
	103	+ mem_uncharge(smap, owner, smap->elem_size);
	104	+
	105	+ free_local_storage = hlist_is_singular_node(&selem->snode,
	106	+ &local_storage->list);
	107	+ if (free_local_storage) {
	108	+ mem_uncharge(smap, owner, sizeof(struct bpf_local_storage));
	109	+ local_storage->owner = NULL;
	110	+
	111	+ /* After this RCU_INIT, owner may be freed and cannot be used */
	112	+ RCU_INIT_POINTER(*owner_storage(smap, owner), NULL);
	113	+
	114	+ /* local_storage is not freed now. local_storage->lock is
	115	+ * still held and raw_spin_unlock_bh(&local_storage->lock)
	116	+ * will be done by the caller.
	117	+ *
	118	+ * Although the unlock will be done under
	119	+ * rcu_read_lock(), it is more intutivie to
	120	+ * read if kfree_rcu(local_storage, rcu) is done
	121	+ * after the raw_spin_unlock_bh(&local_storage->lock).
	122	+ *
	123	+ * Hence, a "bool free_local_storage" is returned
	124	+ * to the caller which then calls the kfree_rcu()
	125	+ * after unlock.
	126	+ */
	127	+ }
	128	+ hlist_del_init_rcu(&selem->snode);
	129	+ if (rcu_access_pointer(local_storage->cache[smap->cache_idx]) ==
	130	+ SDATA(selem))
	131	+ RCU_INIT_POINTER(local_storage->cache[smap->cache_idx], NULL);
	132	+
	133	+ kfree_rcu(selem, rcu);
	134	+
	135	+ return free_local_storage;
	136	+}
	137	+
	138	+static void __bpf_selem_unlink_storage(struct bpf_local_storage_elem *selem)
	139	+{
	140	+ struct bpf_local_storage *local_storage;
	141	+ bool free_local_storage = false;
	142	+
	143	+ if (unlikely(!selem_linked_to_storage(selem)))
	144	+ /* selem has already been unlinked from sk */
	145	+ return;
	146	+
	147	+ local_storage = rcu_dereference(selem->local_storage);
	148	+ raw_spin_lock_bh(&local_storage->lock);
	149	+ if (likely(selem_linked_to_storage(selem)))
	150	+ free_local_storage = bpf_selem_unlink_storage_nolock(
	151	+ local_storage, selem, true);
	152	+ raw_spin_unlock_bh(&local_storage->lock);
	153	+
	154	+ if (free_local_storage)
	155	+ kfree_rcu(local_storage, rcu);
	156	+}
	157	+
	158	+void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,
	159	+ struct bpf_local_storage_elem *selem)
	160	+{
	161	+ RCU_INIT_POINTER(selem->local_storage, local_storage);
	162	+ hlist_add_head(&selem->snode, &local_storage->list);
	163	+}
	164	+
	165	+void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem)
	166	+{
	167	+ struct bpf_local_storage_map *smap;
	168	+ struct bpf_local_storage_map_bucket *b;
	169	+
	170	+ if (unlikely(!selem_linked_to_map(selem)))
	171	+ /* selem has already be unlinked from smap */
	172	+ return;
	173	+
	174	+ smap = rcu_dereference(SDATA(selem)->smap);
	175	+ b = select_bucket(smap, selem);
	176	+ raw_spin_lock_bh(&b->lock);
	177	+ if (likely(selem_linked_to_map(selem)))
	178	+ hlist_del_init_rcu(&selem->map_node);
	179	+ raw_spin_unlock_bh(&b->lock);
	180	+}
	181	+
	182	+void bpf_selem_link_map(struct bpf_local_storage_map *smap,
	183	+ struct bpf_local_storage_elem *selem)
	184	+{
	185	+ struct bpf_local_storage_map_bucket *b = select_bucket(smap, selem);
	186	+
	187	+ raw_spin_lock_bh(&b->lock);
	188	+ RCU_INIT_POINTER(SDATA(selem)->smap, smap);
	189	+ hlist_add_head_rcu(&selem->map_node, &b->list);
	190	+ raw_spin_unlock_bh(&b->lock);
	191	+}
	192	+
	193	+void bpf_selem_unlink(struct bpf_local_storage_elem *selem)
	194	+{
	195	+ /* Always unlink from map before unlinking from local_storage
	196	+ * because selem will be freed after successfully unlinked from
	197	+ * the local_storage.
	198	+ */
	199	+ bpf_selem_unlink_map(selem);
	200	+ __bpf_selem_unlink_storage(selem);
	201	+}
	202	+
	203	+struct bpf_local_storage_data *
	204	+bpf_local_storage_lookup(struct bpf_local_storage *local_storage,
	205	+ struct bpf_local_storage_map *smap,
	206	+ bool cacheit_lockit)
	207	+{
	208	+ struct bpf_local_storage_data *sdata;
	209	+ struct bpf_local_storage_elem *selem;
	210	+
	211	+ /* Fast path (cache hit) */
	212	+ sdata = rcu_dereference(local_storage->cache[smap->cache_idx]);
	213	+ if (sdata && rcu_access_pointer(sdata->smap) == smap)
	214	+ return sdata;
	215	+
	216	+ /* Slow path (cache miss) */
	217	+ hlist_for_each_entry_rcu(selem, &local_storage->list, snode)
	218	+ if (rcu_access_pointer(SDATA(selem)->smap) == smap)
	219	+ break;
	220	+
	221	+ if (!selem)
	222	+ return NULL;
	223	+
	224	+ sdata = SDATA(selem);
	225	+ if (cacheit_lockit) {
	226	+ /* spinlock is needed to avoid racing with the
	227	+ * parallel delete. Otherwise, publishing an already
	228	+ * deleted sdata to the cache will become a use-after-free
	229	+ * problem in the next bpf_local_storage_lookup().
	230	+ */
	231	+ raw_spin_lock_bh(&local_storage->lock);
	232	+ if (selem_linked_to_storage(selem))
	233	+ rcu_assign_pointer(local_storage->cache[smap->cache_idx],
	234	+ sdata);
	235	+ raw_spin_unlock_bh(&local_storage->lock);
	236	+ }
	237	+
	238	+ return sdata;
	239	+}
	240	+
	241	+static int check_flags(const struct bpf_local_storage_data *old_sdata,
	242	+ u64 map_flags)
	243	+{
	244	+ if (old_sdata && (map_flags & ~BPF_F_LOCK) == BPF_NOEXIST)
	245	+ /* elem already exists */
	246	+ return -EEXIST;
	247	+
	248	+ if (!old_sdata && (map_flags & ~BPF_F_LOCK) == BPF_EXIST)
	249	+ /* elem doesn't exist, cannot update it */
	250	+ return -ENOENT;
	251	+
	252	+ return 0;
	253	+}
	254	+
	255	+int bpf_local_storage_alloc(void *owner,
	256	+ struct bpf_local_storage_map *smap,
	257	+ struct bpf_local_storage_elem *first_selem)
	258	+{
	259	+ struct bpf_local_storage prev_storage, storage;
	260	+ struct bpf_local_storage **owner_storage_ptr;
	261	+ int err;
	262	+
	263	+ err = mem_charge(smap, owner, sizeof(*storage));
	264	+ if (err)
	265	+ return err;
	266	+
	267	+ storage = kzalloc(sizeof(*storage), GFP_ATOMIC \| __GFP_NOWARN);
	268	+ if (!storage) {
	269	+ err = -ENOMEM;
	270	+ goto uncharge;
	271	+ }
	272	+
	273	+ INIT_HLIST_HEAD(&storage->list);
	274	+ raw_spin_lock_init(&storage->lock);
	275	+ storage->owner = owner;
	276	+
	277	+ bpf_selem_link_storage_nolock(storage, first_selem);
	278	+ bpf_selem_link_map(smap, first_selem);
	279	+
	280	+ owner_storage_ptr =
	281	+ (struct bpf_local_storage **)owner_storage(smap, owner);
	282	+ /* Publish storage to the owner.
	283	+ * Instead of using any lock of the kernel object (i.e. owner),
	284	+ * cmpxchg will work with any kernel object regardless what
	285	+ * the running context is, bh, irq...etc.
	286	+ *
	287	+ * From now on, the owner->storage pointer (e.g. sk->sk_bpf_storage)
	288	+ * is protected by the storage->lock. Hence, when freeing
	289	+ * the owner->storage, the storage->lock must be held before
	290	+ * setting owner->storage ptr to NULL.
	291	+ */
	292	+ prev_storage = cmpxchg(owner_storage_ptr, NULL, storage);
	293	+ if (unlikely(prev_storage)) {
	294	+ bpf_selem_unlink_map(first_selem);
	295	+ err = -EAGAIN;
	296	+ goto uncharge;
	297	+
	298	+ /* Note that even first_selem was linked to smap's
	299	+ * bucket->list, first_selem can be freed immediately
	300	+ * (instead of kfree_rcu) because
	301	+ * bpf_local_storage_map_free() does a
	302	+ * synchronize_rcu() before walking the bucket->list.
	303	+ * Hence, no one is accessing selem from the
	304	+ * bucket->list under rcu_read_lock().
	305	+ */
	306	+ }
	307	+
	308	+ return 0;
	309	+
	310	+uncharge:
	311	+ kfree(storage);
	312	+ mem_uncharge(smap, owner, sizeof(*storage));
	313	+ return err;
	314	+}
	315	+
	316	+/* sk cannot be going away because it is linking new elem
	317	+ * to sk->sk_bpf_storage. (i.e. sk->sk_refcnt cannot be 0).
	318	+ * Otherwise, it will become a leak (and other memory issues
	319	+ * during map destruction).
	320	+ */
	321	+struct bpf_local_storage_data *
	322	+bpf_local_storage_update(void owner, struct bpf_local_storage_map smap,
	323	+ void *value, u64 map_flags)
	324	+{
	325	+ struct bpf_local_storage_data *old_sdata = NULL;
	326	+ struct bpf_local_storage_elem *selem;
	327	+ struct bpf_local_storage *local_storage;
	328	+ int err;
	329	+
	330	+ /* BPF_EXIST and BPF_NOEXIST cannot be both set */
	331	+ if (unlikely((map_flags & ~BPF_F_LOCK) > BPF_EXIST) \|\|
	332	+ /* BPF_F_LOCK can only be used in a value with spin_lock */
	333	+ unlikely((map_flags & BPF_F_LOCK) &&
	334	+ !map_value_has_spin_lock(&smap->map)))
	335	+ return ERR_PTR(-EINVAL);
	336	+
	337	+ local_storage = rcu_dereference(*owner_storage(smap, owner));
	338	+ if (!local_storage \|\| hlist_empty(&local_storage->list)) {
	339	+ /* Very first elem for the owner */
	340	+ err = check_flags(NULL, map_flags);
	341	+ if (err)
	342	+ return ERR_PTR(err);
	343	+
	344	+ selem = bpf_selem_alloc(smap, owner, value, true);
	345	+ if (!selem)
	346	+ return ERR_PTR(-ENOMEM);
	347	+
	348	+ err = bpf_local_storage_alloc(owner, smap, selem);
	349	+ if (err) {
	350	+ kfree(selem);
	351	+ mem_uncharge(smap, owner, smap->elem_size);
	352	+ return ERR_PTR(err);
	353	+ }
	354	+
	355	+ return SDATA(selem);
	356	+ }
	357	+
	358	+ if ((map_flags & BPF_F_LOCK) && !(map_flags & BPF_NOEXIST)) {
	359	+ /* Hoping to find an old_sdata to do inline update
	360	+ * such that it can avoid taking the local_storage->lock
	361	+ * and changing the lists.
	362	+ */
	363	+ old_sdata =
	364	+ bpf_local_storage_lookup(local_storage, smap, false);
	365	+ err = check_flags(old_sdata, map_flags);
	366	+ if (err)
	367	+ return ERR_PTR(err);
	368	+ if (old_sdata && selem_linked_to_storage(SELEM(old_sdata))) {
	369	+ copy_map_value_locked(&smap->map, old_sdata->data,
	370	+ value, false);
	371	+ return old_sdata;
	372	+ }
	373	+ }
	374	+
	375	+ raw_spin_lock_bh(&local_storage->lock);
	376	+
	377	+ /* Recheck local_storage->list under local_storage->lock */
	378	+ if (unlikely(hlist_empty(&local_storage->list))) {
	379	+ /* A parallel del is happening and local_storage is going
	380	+ * away. It has just been checked before, so very
	381	+ * unlikely. Return instead of retry to keep things
	382	+ * simple.
	383	+ */
	384	+ err = -EAGAIN;
	385	+ goto unlock_err;
	386	+ }
	387	+
	388	+ old_sdata = bpf_local_storage_lookup(local_storage, smap, false);
	389	+ err = check_flags(old_sdata, map_flags);
	390	+ if (err)
	391	+ goto unlock_err;
	392	+
	393	+ if (old_sdata && (map_flags & BPF_F_LOCK)) {
	394	+ copy_map_value_locked(&smap->map, old_sdata->data, value,
	395	+ false);
	396	+ selem = SELEM(old_sdata);
	397	+ goto unlock;
	398	+ }
	399	+
	400	+ /* local_storage->lock is held. Hence, we are sure
	401	+ * we can unlink and uncharge the old_sdata successfully
	402	+ * later. Hence, instead of charging the new selem now
	403	+ * and then uncharge the old selem later (which may cause
	404	+ * a potential but unnecessary charge failure), avoid taking
	405	+ * a charge at all here (the "!old_sdata" check) and the
	406	+ * old_sdata will not be uncharged later during
	407	+ * bpf_selem_unlink_storage_nolock().
	408	+ */
	409	+ selem = bpf_selem_alloc(smap, owner, value, !old_sdata);
	410	+ if (!selem) {
	411	+ err = -ENOMEM;
	412	+ goto unlock_err;
	413	+ }
	414	+
	415	+ /* First, link the new selem to the map */
	416	+ bpf_selem_link_map(smap, selem);
	417	+
	418	+ /* Second, link (and publish) the new selem to local_storage */
	419	+ bpf_selem_link_storage_nolock(local_storage, selem);
	420	+
	421	+ /* Third, remove old selem, SELEM(old_sdata) */
	422	+ if (old_sdata) {
	423	+ bpf_selem_unlink_map(SELEM(old_sdata));
	424	+ bpf_selem_unlink_storage_nolock(local_storage, SELEM(old_sdata),
	425	+ false);
	426	+ }
	427	+
	428	+unlock:
	429	+ raw_spin_unlock_bh(&local_storage->lock);
	430	+ return SDATA(selem);
	431	+
	432	+unlock_err:
	433	+ raw_spin_unlock_bh(&local_storage->lock);
	434	+ return ERR_PTR(err);
	435	+}
	436	+
	437	+u16 bpf_local_storage_cache_idx_get(struct bpf_local_storage_cache *cache)
	438	+{
	439	+ u64 min_usage = U64_MAX;
	440	+ u16 i, res = 0;
	441	+
	442	+ spin_lock(&cache->idx_lock);
	443	+
	444	+ for (i = 0; i < BPF_LOCAL_STORAGE_CACHE_SIZE; i++) {
	445	+ if (cache->idx_usage_counts[i] < min_usage) {
	446	+ min_usage = cache->idx_usage_counts[i];
	447	+ res = i;
	448	+
	449	+ /* Found a free cache_idx */
	450	+ if (!min_usage)
	451	+ break;
	452	+ }
	453	+ }
	454	+ cache->idx_usage_counts[res]++;
	455	+
	456	+ spin_unlock(&cache->idx_lock);
	457	+
	458	+ return res;
	459	+}
	460	+
	461	+void bpf_local_storage_cache_idx_free(struct bpf_local_storage_cache *cache,
	462	+ u16 idx)
	463	+{
	464	+ spin_lock(&cache->idx_lock);
	465	+ cache->idx_usage_counts[idx]--;
	466	+ spin_unlock(&cache->idx_lock);
	467	+}
	468	+
	469	+void bpf_local_storage_map_free(struct bpf_local_storage_map *smap)
	470	+{
	471	+ struct bpf_local_storage_elem *selem;
	472	+ struct bpf_local_storage_map_bucket *b;
	473	+ unsigned int i;
	474	+
	475	+ /* Note that this map might be concurrently cloned from
	476	+ * bpf_sk_storage_clone. Wait for any existing bpf_sk_storage_clone
	477	+ * RCU read section to finish before proceeding. New RCU
	478	+ * read sections should be prevented via bpf_map_inc_not_zero.
	479	+ */
	480	+ synchronize_rcu();
	481	+
	482	+ /* bpf prog and the userspace can no longer access this map
	483	+ * now. No new selem (of this map) can be added
	484	+ * to the owner->storage or to the map bucket's list.
	485	+ *
	486	+ * The elem of this map can be cleaned up here
	487	+ * or when the storage is freed e.g.
	488	+ * by bpf_sk_storage_free() during __sk_destruct().
	489	+ */
	490	+ for (i = 0; i < (1U << smap->bucket_log); i++) {
	491	+ b = &smap->buckets[i];
	492	+
	493	+ rcu_read_lock();
	494	+ /* No one is adding to b->list now */
	495	+ while ((selem = hlist_entry_safe(
	496	+ rcu_dereference_raw(hlist_first_rcu(&b->list)),
	497	+ struct bpf_local_storage_elem, map_node))) {
	498	+ bpf_selem_unlink(selem);
	499	+ cond_resched_rcu();
	500	+ }
	501	+ rcu_read_unlock();
	502	+ }
	503	+
	504	+ /* While freeing the storage we may still need to access the map.
	505	+ *
	506	+ * e.g. when bpf_sk_storage_free() has unlinked selem from the map
	507	+ * which then made the above while((selem = ...)) loop
	508	+ * exit immediately.
	509	+ *
	510	+ * However, while freeing the storage one still needs to access the
	511	+ * smap->elem_size to do the uncharging in
	512	+ * bpf_selem_unlink_storage_nolock().
	513	+ *
	514	+ * Hence, wait another rcu grace period for the storage to be freed.
	515	+ */
	516	+ synchronize_rcu();
	517	+
	518	+ kvfree(smap->buckets);
	519	+ kfree(smap);
	520	+}
	521	+
	522	+int bpf_local_storage_map_alloc_check(union bpf_attr *attr)
	523	+{
	524	+ if (attr->map_flags & ~BPF_LOCAL_STORAGE_CREATE_FLAG_MASK \|\|
	525	+ !(attr->map_flags & BPF_F_NO_PREALLOC) \|\|
	526	+ attr->max_entries \|\|
	527	+ attr->key_size != sizeof(int) \|\| !attr->value_size \|\|
	528	+ /* Enforce BTF for userspace sk dumping */
	529	+ !attr->btf_key_type_id \|\| !attr->btf_value_type_id)
	530	+ return -EINVAL;
	531	+
	532	+ if (!bpf_capable())
	533	+ return -EPERM;
	534	+
	535	+ if (attr->value_size > BPF_LOCAL_STORAGE_MAX_VALUE_SIZE)
	536	+ return -E2BIG;
	537	+
	538	+ return 0;
	539	+}
	540	+
	541	+struct bpf_local_storage_map bpf_local_storage_map_alloc(union bpf_attr attr)
	542	+{
	543	+ struct bpf_local_storage_map *smap;
	544	+ unsigned int i;
	545	+ u32 nbuckets;
	546	+ u64 cost;
	547	+ int ret;
	548	+
	549	+ smap = kzalloc(sizeof(*smap), GFP_USER \| __GFP_NOWARN);
	550	+ if (!smap)
	551	+ return ERR_PTR(-ENOMEM);
	552	+ bpf_map_init_from_attr(&smap->map, attr);
	553	+
	554	+ nbuckets = roundup_pow_of_two(num_possible_cpus());
	555	+ /* Use at least 2 buckets, select_bucket() is undefined behavior with 1 bucket */
	556	+ nbuckets = max_t(u32, 2, nbuckets);
	557	+ smap->bucket_log = ilog2(nbuckets);
	558	+ cost = sizeof(smap->buckets) nbuckets + sizeof(*smap);
	559	+
	560	+ ret = bpf_map_charge_init(&smap->map.memory, cost);
	561	+ if (ret < 0) {
	562	+ kfree(smap);
	563	+ return ERR_PTR(ret);
	564	+ }
	565	+
	566	+ smap->buckets = kvcalloc(sizeof(*smap->buckets), nbuckets,
	567	+ GFP_USER \| __GFP_NOWARN);
	568	+ if (!smap->buckets) {
	569	+ bpf_map_charge_finish(&smap->map.memory);
	570	+ kfree(smap);
	571	+ return ERR_PTR(-ENOMEM);
	572	+ }
	573	+
	574	+ for (i = 0; i < nbuckets; i++) {
	575	+ INIT_HLIST_HEAD(&smap->buckets[i].list);
	576	+ raw_spin_lock_init(&smap->buckets[i].lock);
	577	+ }
	578	+
	579	+ smap->elem_size =
	580	+ sizeof(struct bpf_local_storage_elem) + attr->value_size;
	581	+
	582	+ return smap;
	583	+}
	584	+
	585	+int bpf_local_storage_map_check_btf(const struct bpf_map *map,
	586	+ const struct btf *btf,
	587	+ const struct btf_type *key_type,
	588	+ const struct btf_type *value_type)
	589	+{
	590	+ u32 int_data;
	591	+
	592	+ if (BTF_INFO_KIND(key_type->info) != BTF_KIND_INT)
	593	+ return -EINVAL;
	594	+
	595	+ int_data = (u32 )(key_type + 1);
	596	+ if (BTF_INT_BITS(int_data) != 32 \|\| BTF_INT_OFFSET(int_data))
	597	+ return -EINVAL;
	598	+
	599	+ return 0;
	600	+}

kernel/bpf/bpf_lsm.c+20−1

@@ -11,6 +11,8 @@ · 10 unmodified lines
11	11	#include <linux/bpf_lsm.h>
12	12	#include <linux/kallsyms.h>
13	13	#include <linux/bpf_verifier.h>
	14	+#include <net/bpf_sk_storage.h>
	15	+#include <linux/bpf_local_storage.h>
14	16
15	17	/* For every LSM hook that allows attachment of BPF programs, declare a nop
16	18	* function where a BPF program can be attached.
@@ -45,10 +47,27 @@ int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog, · 28 unmodified lines
45	47	return 0;
46	48	}
47	49
	50	+static const struct bpf_func_proto *
	51	+bpf_lsm_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
	52	+{
	53	+ switch (func_id) {
	54	+ case BPF_FUNC_inode_storage_get:
	55	+ return &bpf_inode_storage_get_proto;
	56	+ case BPF_FUNC_inode_storage_delete:
	57	+ return &bpf_inode_storage_delete_proto;
	58	+ case BPF_FUNC_sk_storage_get:
	59	+ return &sk_storage_get_btf_proto;
	60	+ case BPF_FUNC_sk_storage_delete:
	61	+ return &sk_storage_delete_btf_proto;
	62	+ default:
	63	+ return tracing_prog_func_proto(func_id, prog);
	64	+ }
	65	+}
	66	+
48	67	const struct bpf_prog_ops lsm_prog_ops = {
49	68	};
50	69
51	70	const struct bpf_verifier_ops lsm_verifier_ops = {
52		− .get_func_proto = tracing_prog_func_proto,
	71	+ .get_func_proto = bpf_lsm_func_proto,
53	72	.is_valid_access = btf_ctx_access,
54	73	};

kernel/bpf/bpf_struct_ops.c+2−4

@@ -298,8 +298,7 @@ static int check_zero_holes(const struct btf_type t, void data) · 297 unmodified lines
298	298	return -EINVAL;
299	299
300	300	mtype = btf_type_by_id(btf_vmlinux, member->type);
301		− mtype = btf_resolve_size(btf_vmlinux, mtype, &msize,
302		− NULL, NULL);
	301	+ mtype = btf_resolve_size(btf_vmlinux, mtype, &msize);
303	302	if (IS_ERR(mtype))
304	303	return PTR_ERR(mtype);
305	304	prev_mend = moff + msize;
@@ -396,8 +395,7 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map map, void key, · 90 unmodified lines
396	395	u32 msize;
397	396
398	397	mtype = btf_type_by_id(btf_vmlinux, member->type);
399		− mtype = btf_resolve_size(btf_vmlinux, mtype, &msize,
400		− NULL, NULL);
	398	+ mtype = btf_resolve_size(btf_vmlinux, mtype, &msize);
401	399	if (IS_ERR(mtype)) {
402	400	err = PTR_ERR(mtype);
403	401	goto reset_unlock;

kernel/bpf/btf.c+139−24

@@ -21,6 +21,8 @@ · 20 unmodified lines
21	21	#include <linux/btf_ids.h>
22	22	#include <linux/skmsg.h>
23	23	#include <linux/perf_event.h>
	24	+#include <linux/bsearch.h>
	25	+#include <linux/btf_ids.h>
24	26	#include <net/sock.h>
25	27
26	28	/* BTF (BPF Type Format) is the meta data format which describes
@@ -1079,23 +1081,27 @@ static const struct resolve_vertex env_stack_peak(struct btf_verifier_env env) · 1052 unmodified lines
1079	1081	* type_size: (x y * sizeof(u32)). Hence, *type_size always
1080	1082	* corresponds to the return type.
1081	1083	* *elem_type: u32
	1084	+ * *elem_id: id of u32
1082	1085	* total_nelems: (x y). Hence, individual elem size is
1083	1086	* (type_size / total_nelems)
	1087	+ * *type_id: id of type if it's changed within the function, 0 if not
1084	1088	*
1085	1089	* type: is not an array (e.g. const struct X)
1086	1090	* return type: type "struct X"
1087	1091	* *type_size: sizeof(struct X)
1088	1092	* *elem_type: same as return type ("struct X")
	1093	+ * *elem_id: 0
1089	1094	* *total_nelems: 1
	1095	+ * *type_id: id of type if it's changed within the function, 0 if not
1090	1096	*/
1091		−const struct btf_type *
1092		−btf_resolve_size(const struct btf btf, const struct btf_type type,
1093		− u32 type_size, const struct btf_type *elem_type,
1094		− u32 *total_nelems)
	1097	+static const struct btf_type *
	1098	+__btf_resolve_size(const struct btf btf, const struct btf_type type,
	1099	+ u32 type_size, const struct btf_type *elem_type,
	1100	+ u32 elem_id, u32 total_nelems, u32 *type_id)
1095	1101	{
1096	1102	const struct btf_type *array_type = NULL;
1097		− const struct btf_array *array;
1098		− u32 i, size, nelems = 1;
	1103	+ const struct btf_array *array = NULL;
	1104	+ u32 i, size, nelems = 1, id = 0;
1099	1105
1100	1106	for (i = 0; i < MAX_RESOLVE_DEPTH; i++) {
1101	1107	switch (BTF_INFO_KIND(type->info)) {
@@ -1116,6 +1122,7 @@ btf_resolve_size(const struct btf btf, const struct btf_type type, · 14 unmodified lines
1116	1122	case BTF_KIND_VOLATILE:
1117	1123	case BTF_KIND_CONST:
1118	1124	case BTF_KIND_RESTRICT:
	1125	+ id = type->type;
1119	1126	type = btf_type_by_id(btf, type->type);
1120	1127	break;
1121	1128
@@ -1146,10 +1153,21 @@ resolved: · 24 unmodified lines
1146	1153	*total_nelems = nelems;
1147	1154	if (elem_type)
1148	1155	*elem_type = type;
	1156	+ if (elem_id)
	1157	+ *elem_id = array ? array->type : 0;
	1158	+ if (type_id && id)
	1159	+ *type_id = id;
1149	1160
1150	1161	return array_type ? : type;
1151	1162	}
1152	1163
	1164	+const struct btf_type *
	1165	+btf_resolve_size(const struct btf btf, const struct btf_type type,
	1166	+ u32 *type_size)
	1167	+{
	1168	+ return __btf_resolve_size(btf, type, type_size, NULL, NULL, NULL, NULL);
	1169	+}
	1170	+
1153	1171	/* The input param "type_id" must point to a needs_resolve type */
1154	1172	static const struct btf_type btf_type_id_resolve(const struct btf btf,
1155	1173	u32 *type_id)
@@ -3870,16 +3888,22 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, · 2714 unmodified lines
3870	3888	return true;
3871	3889	}
3872	3890
3873		−int btf_struct_access(struct bpf_verifier_log *log,
3874		− const struct btf_type *t, int off, int size,
3875		− enum bpf_access_type atype,
3876		− u32 *next_btf_id)
	3891	+enum bpf_struct_walk_result {
	3892	+ /* < 0 error */
	3893	+ WALK_SCALAR = 0,
	3894	+ WALK_PTR,
	3895	+ WALK_STRUCT,
	3896	+};
	3897	+
	3898	+static int btf_struct_walk(struct bpf_verifier_log *log,
	3899	+ const struct btf_type *t, int off, int size,
	3900	+ u32 *next_btf_id)
3877	3901	{
3878	3902	u32 i, moff, mtrue_end, msize = 0, total_nelems = 0;
3879	3903	const struct btf_type mtype, elem_type = NULL;
3880	3904	const struct btf_member *member;
3881	3905	const char tname, mname;
3882		− u32 vlen;
	3906	+ u32 vlen, elem_id, mid;
3883	3907
3884	3908	again:
3885	3909	tname = __btf_name_by_offset(btf_vmlinux, t->name_off);
@@ -3915,14 +3939,13 @@ again: · 29 unmodified lines
3915	3939	/* Only allow structure for now, can be relaxed for
3916	3940	* other types later.
3917	3941	*/
3918		− elem_type = btf_type_skip_modifiers(btf_vmlinux,
3919		− array_elem->type, NULL);
3920		− if (!btf_type_is_struct(elem_type))
	3942	+ t = btf_type_skip_modifiers(btf_vmlinux, array_elem->type,
	3943	+ NULL);
	3944	+ if (!btf_type_is_struct(t))
3921	3945	goto error;
3922	3946
3923		− off = (off - moff) % elem_type->size;
3924		− return btf_struct_access(log, elem_type, off, size, atype,
3925		− next_btf_id);
	3947	+ off = (off - moff) % t->size;
	3948	+ goto again;
3926	3949
3927	3950	error:
3928	3951	bpf_log(log, "access beyond struct %s at off %u size %u\n",
@@ -3951,7 +3974,7 @@ error: · 22 unmodified lines
3951	3974	*/
3952	3975	if (off <= moff &&
3953	3976	BITS_ROUNDUP_BYTES(end_bit) <= off + size)
3954		− return SCALAR_VALUE;
	3977	+ return WALK_SCALAR;
3955	3978
3956	3979	/* off may be accessing a following member
3957	3980	*
@@ -3973,11 +3996,13 @@ error: · 15 unmodified lines
3973	3996	break;
3974	3997
3975	3998	/* type of the field */
	3999	+ mid = member->type;
3976	4000	mtype = btf_type_by_id(btf_vmlinux, member->type);
3977	4001	mname = __btf_name_by_offset(btf_vmlinux, member->name_off);
3978	4002
3979		− mtype = btf_resolve_size(btf_vmlinux, mtype, &msize,
3980		− &elem_type, &total_nelems);
	4003	+ mtype = __btf_resolve_size(btf_vmlinux, mtype, &msize,
	4004	+ &elem_type, &elem_id, &total_nelems,
	4005	+ &mid);
3981	4006	if (IS_ERR(mtype)) {
3982	4007	bpf_log(log, "field %s doesn't have size\n", mname);
3983	4008	return -EFAULT;
@@ -3991,7 +4016,7 @@ error: · 7 unmodified lines
3991	4016	if (btf_type_is_array(mtype)) {
3992	4017	u32 elem_idx;
3993	4018
3994		− /* btf_resolve_size() above helps to
	4019	+ /* __btf_resolve_size() above helps to
3995	4020	* linearize a multi-dimensional array.
3996	4021	*
3997	4022	* The logic here is treating an array
@@ -4039,6 +4064,7 @@ error: · 41 unmodified lines
4039	4064	elem_idx = (off - moff) / msize;
4040	4065	moff += elem_idx * msize;
4041	4066	mtype = elem_type;
	4067	+ mid = elem_id;
4042	4068	}
4043	4069
4044	4070	/* the 'off' we're looking for is either equal to start
@@ -4048,6 +4074,12 @@ error: · 3 unmodified lines
4048	4074	/* our field must be inside that union or struct */
4049	4075	t = mtype;
4050	4076
	4077	+ /* return if the offset matches the member offset */
	4078	+ if (off == moff) {
	4079	+ *next_btf_id = mid;
	4080	+ return WALK_STRUCT;
	4081	+ }
	4082	+
4051	4083	/* adjust offset we're looking for */
4052	4084	off -= moff;
4053	4085	goto again;
@@ -4063,11 +4095,10 @@ error: · 9 unmodified lines
4063	4095	mname, moff, tname, off, size);
4064	4096	return -EACCES;
4065	4097	}
4066		−
4067	4098	stype = btf_type_skip_modifiers(btf_vmlinux, mtype->type, &id);
4068	4099	if (btf_type_is_struct(stype)) {
4069	4100	*next_btf_id = id;
4070		− return PTR_TO_BTF_ID;
	4101	+ return WALK_PTR;
4071	4102	}
4072	4103	}
4073	4104
@@ -4084,12 +4115,84 @@ error: · 10 unmodified lines
4084	4115	return -EACCES;
4085	4116	}
4086	4117
4087		− return SCALAR_VALUE;
	4118	+ return WALK_SCALAR;
4088	4119	}
4089	4120	bpf_log(log, "struct %s doesn't have field at offset %d\n", tname, off);
4090	4121	return -EINVAL;
4091	4122	}
4092	4123
	4124	+int btf_struct_access(struct bpf_verifier_log *log,
	4125	+ const struct btf_type *t, int off, int size,
	4126	+ enum bpf_access_type atype __maybe_unused,
	4127	+ u32 *next_btf_id)
	4128	+{
	4129	+ int err;
	4130	+ u32 id;
	4131	+
	4132	+ do {
	4133	+ err = btf_struct_walk(log, t, off, size, &id);
	4134	+
	4135	+ switch (err) {
	4136	+ case WALK_PTR:
	4137	+ /* If we found the pointer or scalar on t+off,
	4138	+ * we're done.
	4139	+ */
	4140	+ *next_btf_id = id;
	4141	+ return PTR_TO_BTF_ID;
	4142	+ case WALK_SCALAR:
	4143	+ return SCALAR_VALUE;
	4144	+ case WALK_STRUCT:
	4145	+ /* We found nested struct, so continue the search
	4146	+ * by diving in it. At this point the offset is
	4147	+ * aligned with the new type, so set it to 0.
	4148	+ */
	4149	+ t = btf_type_by_id(btf_vmlinux, id);
	4150	+ off = 0;
	4151	+ break;
	4152	+ default:
	4153	+ /* It's either error or unknown return value..
	4154	+ * scream and leave.
	4155	+ */
	4156	+ if (WARN_ONCE(err > 0, "unknown btf_struct_walk return value"))
	4157	+ return -EINVAL;
	4158	+ return err;
	4159	+ }
	4160	+ } while (t);
	4161	+
	4162	+ return -EINVAL;
	4163	+}
	4164	+
	4165	+bool btf_struct_ids_match(struct bpf_verifier_log *log,
	4166	+ int off, u32 id, u32 need_type_id)
	4167	+{
	4168	+ const struct btf_type *type;
	4169	+ int err;
	4170	+
	4171	+ /* Are we already done? */
	4172	+ if (need_type_id == id && off == 0)
	4173	+ return true;
	4174	+
	4175	+again:
	4176	+ type = btf_type_by_id(btf_vmlinux, id);
	4177	+ if (!type)
	4178	+ return false;
	4179	+ err = btf_struct_walk(log, type, off, 1, &id);
	4180	+ if (err != WALK_STRUCT)
	4181	+ return false;
	4182	+
	4183	+ /* We found nested struct object. If it matches
	4184	+ * the requested ID, we're done. Otherwise let's
	4185	+ * continue the search with offset 0 in the new
	4186	+ * type.
	4187	+ */
	4188	+ if (need_type_id != id) {
	4189	+ off = 0;
	4190	+ goto again;
	4191	+ }
	4192	+
	4193	+ return true;
	4194	+}
	4195	+
4093	4196	int btf_resolve_helper_id(struct bpf_verifier_log *log,
4094	4197	const struct bpf_func_proto *fn, int arg)
4095	4198	{
@@ -4661,3 +4764,15 @@ u32 btf_id(const struct btf *btf) · 565 unmodified lines
4661	4764	{
4662	4765	return btf->id;
4663	4766	}
	4767	+
	4768	+static int btf_id_cmp_func(const void a, const void b)
	4769	+{
	4770	+ const int pa = a, pb = b;
	4771	+
	4772	+ return pa - pb;
	4773	+}
	4774	+
	4775	+bool btf_id_set_contains(struct btf_id_set *set, u32 id)
	4776	+{
	4777	+ return bsearch(&id, set->ids, set->cnt, sizeof(u32), btf_id_cmp_func) != NULL;
	4778	+}

kernel/bpf/cpumap.c+4−8

@@ -79,8 +79,6 @@ struct bpf_cpu_map { · 78 unmodified lines
79	79
80	80	static DEFINE_PER_CPU(struct list_head, cpu_map_flush_list);
81	81
82		−static int bq_flush_to_queue(struct xdp_bulk_queue *bq);
83		−
84	82	static struct bpf_map cpu_map_alloc(union bpf_attr attr)
85	83	{
86	84	u32 value_size = attr->value_size;
@@ -658,6 +656,7 @@ static int cpu_map_get_next_key(struct bpf_map map, void key, void *next_key) · 571 unmodified lines
658	656
659	657	static int cpu_map_btf_id;
660	658	const struct bpf_map_ops cpu_map_ops = {
	659	+ .map_meta_equal = bpf_map_meta_equal,
661	660	.map_alloc = cpu_map_alloc,
662	661	.map_free = cpu_map_free,
663	662	.map_delete_elem = cpu_map_delete_elem,
@@ -669,7 +668,7 @@ const struct bpf_map_ops cpu_map_ops = { · 5 unmodified lines
669	668	.map_btf_id = &cpu_map_btf_id,
670	669	};
671	670
672		−static int bq_flush_to_queue(struct xdp_bulk_queue *bq)
	671	+static void bq_flush_to_queue(struct xdp_bulk_queue *bq)
673	672	{
674	673	struct bpf_cpu_map_entry *rcpu = bq->obj;
675	674	unsigned int processed = 0, drops = 0;
@@ -678,7 +677,7 @@ static int bq_flush_to_queue(struct xdp_bulk_queue *bq) · 2 unmodified lines
678	677	int i;
679	678
680	679	if (unlikely(!bq->count))
681		− return 0;
	680	+ return;
682	681
683	682	q = rcpu->queue;
684	683	spin_lock(&q->producer_lock);
@@ -701,13 +700,12 @@ static int bq_flush_to_queue(struct xdp_bulk_queue *bq) · 16 unmodified lines
701	700
702	701	/* Feedback loop via tracepoints */
703	702	trace_xdp_cpumap_enqueue(rcpu->map_id, processed, drops, to_cpu);
704		− return 0;
705	703	}
706	704
707	705	/* Runs under RCU-read-side, plus in softirq under NAPI protection.
708	706	* Thus, safe percpu variable access.
709	707	*/
710		−static int bq_enqueue(struct bpf_cpu_map_entry rcpu, struct xdp_frame xdpf)
	708	+static void bq_enqueue(struct bpf_cpu_map_entry rcpu, struct xdp_frame xdpf)
711	709	{
712	710	struct list_head *flush_list = this_cpu_ptr(&cpu_map_flush_list);
713	711	struct xdp_bulk_queue *bq = this_cpu_ptr(rcpu->bulkq);
@@ -728,8 +726,6 @@ static int bq_enqueue(struct bpf_cpu_map_entry rcpu, struct xdp_frame xdpf) · 14 unmodified lines
728	726
729	727	if (!bq->flush_node.prev)
730	728	list_add(&bq->flush_node, flush_list);
731		−
732		− return 0;
733	729	}
734	730
735	731	int cpu_map_enqueue(struct bpf_cpu_map_entry rcpu, struct xdp_buff xdp,

kernel/bpf/devmap.c+9−8

@@ -341,14 +341,14 @@ bool dev_map_can_have_prog(struct bpf_map *map) · 340 unmodified lines
341	341	return false;
342	342	}
343	343
344		−static int bq_xmit_all(struct xdp_dev_bulk_queue *bq, u32 flags)
	344	+static void bq_xmit_all(struct xdp_dev_bulk_queue *bq, u32 flags)
345	345	{
346	346	struct net_device *dev = bq->dev;
347	347	int sent = 0, drops = 0, err = 0;
348	348	int i;
349	349
350	350	if (unlikely(!bq->count))
351		− return 0;
	351	+ return;
352	352
353	353	for (i = 0; i < bq->count; i++) {
354	354	struct xdp_frame *xdpf = bq->q[i];
@@ -369,7 +369,7 @@ out: · 14 unmodified lines
369	369	trace_xdp_devmap_xmit(bq->dev_rx, dev, sent, drops, err);
370	370	bq->dev_rx = NULL;
371	371	__list_del_clearprev(&bq->flush_node);
372		− return 0;
	372	+ return;
373	373	error:
374	374	/* If ndo_xdp_xmit fails with an errno, no frames have been
375	375	* xmit'ed and it's our responsibility to them free all.
@@ -421,8 +421,8 @@ struct bpf_dtab_netdev __dev_map_lookup_elem(struct bpf_map map, u32 key) · 45 unmodified lines
421	421	/* Runs under RCU-read-side, plus in softirq under NAPI protection.
422	422	* Thus, safe percpu variable access.
423	423	*/
424		−static int bq_enqueue(struct net_device dev, struct xdp_frame xdpf,
425		− struct net_device *dev_rx)
	424	+static void bq_enqueue(struct net_device dev, struct xdp_frame xdpf,
	425	+ struct net_device *dev_rx)
426	426	{
427	427	struct list_head *flush_list = this_cpu_ptr(&dev_flush_list);
428	428	struct xdp_dev_bulk_queue *bq = this_cpu_ptr(dev->xdp_bulkq);
@@ -441,8 +441,6 @@ static int bq_enqueue(struct net_device dev, struct xdp_frame xdpf, · 12 unmodified lines
441	441
442	442	if (!bq->flush_node.prev)
443	443	list_add(&bq->flush_node, flush_list);
444		−
445		− return 0;
446	444	}
447	445
448	446	static inline int __xdp_enqueue(struct net_device dev, struct xdp_buff xdp,
@@ -462,7 +460,8 @@ static inline int __xdp_enqueue(struct net_device dev, struct xdp_buff xdp, · 13 unmodified lines
462	460	if (unlikely(!xdpf))
463	461	return -EOVERFLOW;
464	462
465		− return bq_enqueue(dev, xdpf, dev_rx);
	463	+ bq_enqueue(dev, xdpf, dev_rx);
	464	+ return 0;
466	465	}
467	466
468	467	static struct xdp_buff dev_map_run_prog(struct net_device dev,
@@ -751,6 +750,7 @@ static int dev_map_hash_update_elem(struct bpf_map map, void key, void *value, · 282 unmodified lines
751	750
752	751	static int dev_map_btf_id;
753	752	const struct bpf_map_ops dev_map_ops = {
	753	+ .map_meta_equal = bpf_map_meta_equal,
754	754	.map_alloc = dev_map_alloc,
755	755	.map_free = dev_map_free,
756	756	.map_get_next_key = dev_map_get_next_key,
@@ -764,6 +764,7 @@ const struct bpf_map_ops dev_map_ops = { · 7 unmodified lines
764	764
765	765	static int dev_map_hash_map_btf_id;
766	766	const struct bpf_map_ops dev_map_hash_ops = {
	767	+ .map_meta_equal = bpf_map_meta_equal,
767	768	.map_alloc = dev_map_alloc,
768	769	.map_free = dev_map_free,
769	770	.map_get_next_key = dev_map_hash_get_next_key,

kernel/bpf/hashtab.c+10−6

@@ -9,6 +9,7 @@ · 8 unmodified lines
9	9	#include <linux/rculist_nulls.h>
10	10	#include <linux/random.h>
11	11	#include <uapi/linux/btf.h>
	12	+#include <linux/rcupdate_trace.h>
12	13	#include "percpu_freelist.h"
13	14	#include "bpf_lru_list.h"
14	15	#include "map_in_map.h"
@@ -577,8 +578,7 @@ static void __htab_map_lookup_elem(struct bpf_map map, void *key) · 562 unmodified lines
577	578	struct htab_elem *l;
578	579	u32 hash, key_size;
579	580
580		− /* Must be called with rcu_read_lock. */
581		− WARN_ON_ONCE(!rcu_read_lock_held());
	581	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
582	582
583	583	key_size = map->key_size;
584	584
@@ -941,7 +941,7 @@ static int htab_map_update_elem(struct bpf_map map, void key, void *value, · 356 unmodified lines
941	941	/* unknown flags */
942	942	return -EINVAL;
943	943
944		− WARN_ON_ONCE(!rcu_read_lock_held());
	944	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
945	945
946	946	key_size = map->key_size;
947	947
@@ -1032,7 +1032,7 @@ static int htab_lru_map_update_elem(struct bpf_map map, void key, void *value, · 84 unmodified lines
1032	1032	/* unknown flags */
1033	1033	return -EINVAL;
1034	1034
1035		− WARN_ON_ONCE(!rcu_read_lock_held());
	1035	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
1036	1036
1037	1037	key_size = map->key_size;
1038	1038
@@ -1220,7 +1220,7 @@ static int htab_map_delete_elem(struct bpf_map map, void key) · 181 unmodified lines
1220	1220	u32 hash, key_size;
1221	1221	int ret = -ENOENT;
1222	1222
1223		− WARN_ON_ONCE(!rcu_read_lock_held());
	1223	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
1224	1224
1225	1225	key_size = map->key_size;
1226	1226
@@ -1252,7 +1252,7 @@ static int htab_lru_map_delete_elem(struct bpf_map map, void key) · 25 unmodified lines
1252	1252	u32 hash, key_size;
1253	1253	int ret = -ENOENT;
1254	1254
1255		− WARN_ON_ONCE(!rcu_read_lock_held());
	1255	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
1256	1256
1257	1257	key_size = map->key_size;
1258	1258
@@ -1810,6 +1810,7 @@ static const struct bpf_iter_seq_info iter_seq_info = { · 551 unmodified lines
1810	1810
1811	1811	static int htab_map_btf_id;
1812	1812	const struct bpf_map_ops htab_map_ops = {
	1813	+ .map_meta_equal = bpf_map_meta_equal,
1813	1814	.map_alloc_check = htab_map_alloc_check,
1814	1815	.map_alloc = htab_map_alloc,
1815	1816	.map_free = htab_map_free,
@@ -1827,6 +1828,7 @@ const struct bpf_map_ops htab_map_ops = { · 11 unmodified lines
1827	1828
1828	1829	static int htab_lru_map_btf_id;
1829	1830	const struct bpf_map_ops htab_lru_map_ops = {
	1831	+ .map_meta_equal = bpf_map_meta_equal,
1830	1832	.map_alloc_check = htab_map_alloc_check,
1831	1833	.map_alloc = htab_map_alloc,
1832	1834	.map_free = htab_map_free,
@@ -1947,6 +1949,7 @@ static void htab_percpu_map_seq_show_elem(struct bpf_map map, void key, · 114 unmodified lines
1947	1949
1948	1950	static int htab_percpu_map_btf_id;
1949	1951	const struct bpf_map_ops htab_percpu_map_ops = {
	1952	+ .map_meta_equal = bpf_map_meta_equal,
1950	1953	.map_alloc_check = htab_map_alloc_check,
1951	1954	.map_alloc = htab_map_alloc,
1952	1955	.map_free = htab_map_free,
@@ -1963,6 +1966,7 @@ const struct bpf_map_ops htab_percpu_map_ops = { · 10 unmodified lines
1963	1966
1964	1967	static int htab_lru_percpu_map_btf_id;
1965	1968	const struct bpf_map_ops htab_lru_percpu_map_ops = {
	1969	+ .map_meta_equal = bpf_map_meta_equal,
1966	1970	.map_alloc_check = htab_map_alloc_check,
1967	1971	.map_alloc = htab_map_alloc,
1968	1972	.map_free = htab_map_free,

kernel/bpf/helpers.c+22−0

@@ -601,6 +601,28 @@ const struct bpf_func_proto bpf_event_output_data_proto = { · 600 unmodified lines
601	601	.arg5_type = ARG_CONST_SIZE_OR_ZERO,
602	602	};
603	603
	604	+BPF_CALL_3(bpf_copy_from_user, void *, dst, u32, size,
	605	+ const void __user *, user_ptr)
	606	+{
	607	+ int ret = copy_from_user(dst, user_ptr, size);
	608	+
	609	+ if (unlikely(ret)) {
	610	+ memset(dst, 0, size);
	611	+ ret = -EFAULT;
	612	+ }
	613	+
	614	+ return ret;
	615	+}
	616	+
	617	+const struct bpf_func_proto bpf_copy_from_user_proto = {
	618	+ .func = bpf_copy_from_user,
	619	+ .gpl_only = false,
	620	+ .ret_type = RET_INTEGER,
	621	+ .arg1_type = ARG_PTR_TO_UNINIT_MEM,
	622	+ .arg2_type = ARG_CONST_SIZE_OR_ZERO,
	623	+ .arg3_type = ARG_ANYTHING,
	624	+};
	625	+
604	626	const struct bpf_func_proto bpf_get_current_task_proto __weak;
605	627	const struct bpf_func_proto bpf_probe_read_user_proto __weak;
606	628	const struct bpf_func_proto bpf_probe_read_user_str_proto __weak;

kernel/bpf/inode.c+113−3

@@ -20,6 +20,7 @@ · 19 unmodified lines
20	20	#include <linux/filter.h>
21	21	#include <linux/bpf.h>
22	22	#include <linux/bpf_trace.h>
	23	+#include "preload/bpf_preload.h"
23	24
24	25	enum bpf_type {
25	26	BPF_TYPE_UNSPEC = 0,
@@ -369,9 +370,10 @@ static struct dentry * · 343 unmodified lines
369	370	bpf_lookup(struct inode dir, struct dentry dentry, unsigned flags)
370	371	{
371	372	/* Dots in names (e.g. "/sys/fs/bpf/foo.bar") are reserved for future
372		− * extensions.
	373	+ * extensions. That allows popoulate_bpffs() create special files.
373	374	*/
374		− if (strchr(dentry->d_name.name, '.'))
	375	+ if ((dir->i_mode & S_IALLUGO) &&
	376	+ strchr(dentry->d_name.name, '.'))
375	377	return ERR_PTR(-EPERM);
376	378
377	379	return simple_lookup(dir, dentry, flags);
@@ -409,6 +411,27 @@ static const struct inode_operations bpf_dir_iops = { · 31 unmodified lines
409	411	.unlink = simple_unlink,
410	412	};
411	413
	414	+/* pin iterator link into bpffs */
	415	+static int bpf_iter_link_pin_kernel(struct dentry *parent,
	416	+ const char name, struct bpf_link link)
	417	+{
	418	+ umode_t mode = S_IFREG \| S_IRUSR;
	419	+ struct dentry *dentry;
	420	+ int ret;
	421	+
	422	+ inode_lock(parent->d_inode);
	423	+ dentry = lookup_one_len(name, parent, strlen(name));
	424	+ if (IS_ERR(dentry)) {
	425	+ inode_unlock(parent->d_inode);
	426	+ return PTR_ERR(dentry);
	427	+ }
	428	+ ret = bpf_mkobj_ops(dentry, mode, link, &bpf_link_iops,
	429	+ &bpf_iter_fops);
	430	+ dput(dentry);
	431	+ inode_unlock(parent->d_inode);
	432	+ return ret;
	433	+}
	434	+
412	435	static int bpf_obj_do_pin(const char __user pathname, void raw,
413	436	enum bpf_type type)
414	437	{
@@ -638,6 +661,91 @@ static int bpf_parse_param(struct fs_context fc, struct fs_parameter param) · 223 unmodified lines
638	661	return 0;
639	662	}
640	663
	664	+struct bpf_preload_ops *bpf_preload_ops;
	665	+EXPORT_SYMBOL_GPL(bpf_preload_ops);
	666	+
	667	+static bool bpf_preload_mod_get(void)
	668	+{
	669	+ /* If bpf_preload.ko wasn't loaded earlier then load it now.
	670	+ * When bpf_preload is built into vmlinux the module's __init
	671	+ * function will populate it.
	672	+ */
	673	+ if (!bpf_preload_ops) {
	674	+ request_module("bpf_preload");
	675	+ if (!bpf_preload_ops)
	676	+ return false;
	677	+ }
	678	+ /* And grab the reference, so the module doesn't disappear while the
	679	+ * kernel is interacting with the kernel module and its UMD.
	680	+ */
	681	+ if (!try_module_get(bpf_preload_ops->owner)) {
	682	+ pr_err("bpf_preload module get failed.\n");
	683	+ return false;
	684	+ }
	685	+ return true;
	686	+}
	687	+
	688	+static void bpf_preload_mod_put(void)
	689	+{
	690	+ if (bpf_preload_ops)
	691	+ /* now user can "rmmod bpf_preload" if necessary */
	692	+ module_put(bpf_preload_ops->owner);
	693	+}
	694	+
	695	+static DEFINE_MUTEX(bpf_preload_lock);
	696	+
	697	+static int populate_bpffs(struct dentry *parent)
	698	+{
	699	+ struct bpf_preload_info objs[BPF_PRELOAD_LINKS] = {};
	700	+ struct bpf_link *links[BPF_PRELOAD_LINKS] = {};
	701	+ int err = 0, i;
	702	+
	703	+ /* grab the mutex to make sure the kernel interactions with bpf_preload
	704	+ * UMD are serialized
	705	+ */
	706	+ mutex_lock(&bpf_preload_lock);
	707	+
	708	+ /* if bpf_preload.ko wasn't built into vmlinux then load it */
	709	+ if (!bpf_preload_mod_get())
	710	+ goto out;
	711	+
	712	+ if (!bpf_preload_ops->info.tgid) {
	713	+ /* preload() will start UMD that will load BPF iterator programs */
	714	+ err = bpf_preload_ops->preload(objs);
	715	+ if (err)
	716	+ goto out_put;
	717	+ for (i = 0; i < BPF_PRELOAD_LINKS; i++) {
	718	+ links[i] = bpf_link_by_id(objs[i].link_id);
	719	+ if (IS_ERR(links[i])) {
	720	+ err = PTR_ERR(links[i]);
	721	+ goto out_put;
	722	+ }
	723	+ }
	724	+ for (i = 0; i < BPF_PRELOAD_LINKS; i++) {
	725	+ err = bpf_iter_link_pin_kernel(parent,
	726	+ objs[i].link_name, links[i]);
	727	+ if (err)
	728	+ goto out_put;
	729	+ /* do not unlink successfully pinned links even
	730	+ * if later link fails to pin
	731	+ */
	732	+ links[i] = NULL;
	733	+ }
	734	+ /* finish() will tell UMD process to exit */
	735	+ err = bpf_preload_ops->finish();
	736	+ if (err)
	737	+ goto out_put;
	738	+ }
	739	+out_put:
	740	+ bpf_preload_mod_put();
	741	+out:
	742	+ mutex_unlock(&bpf_preload_lock);
	743	+ for (i = 0; i < BPF_PRELOAD_LINKS && err; i++)
	744	+ if (!IS_ERR_OR_NULL(links[i]))
	745	+ bpf_link_put(links[i]);
	746	+ return err;
	747	+}
	748	+
641	749	static int bpf_fill_super(struct super_block sb, struct fs_context fc)
642	750	{
643	751	static const struct tree_descr bpf_rfiles[] = { { "" } };
@@ -654,8 +762,8 @@ static int bpf_fill_super(struct super_block sb, struct fs_context fc) · 10 unmodified lines
654	762	inode = sb->s_root->d_inode;
655	763	inode->i_op = &bpf_dir_iops;
656	764	inode->i_mode &= ~S_IALLUGO;
	765	+ populate_bpffs(sb->s_root);
657	766	inode->i_mode \|= S_ISVTX \| opts->mode;
658		−
659	767	return 0;
660	768	}
661	769
@@ -705,6 +813,8 @@ static int __init bpf_init(void) · 43 unmodified lines
705	813	{
706	814	int ret;
707	815
	816	+ mutex_init(&bpf_preload_lock);
	817	+
708	818	ret = sysfs_create_mount_point(fs_kobj, "bpf");
709	819	if (ret)
710	820	return ret;

kernel/bpf/lpm_trie.c+1−0

@@ -732,6 +732,7 @@ static int trie_check_btf(const struct bpf_map *map, · 731 unmodified lines
732	732
733	733	static int trie_map_btf_id;
734	734	const struct bpf_map_ops trie_map_ops = {
	735	+ .map_meta_equal = bpf_map_meta_equal,
735	736	.map_alloc = trie_alloc,
736	737	.map_free = trie_free,
737	738	.map_get_next_key = trie_get_next_key,

kernel/bpf/map_in_map.c+9−15

@@ -17,23 +17,17 @@ struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd) · 16 unmodified lines
17	17	if (IS_ERR(inner_map))
18	18	return inner_map;
19	19
20		− /* prog_array->aux->{type,jited} is a runtime binding.
21		− * Doing static check alone in the verifier is not enough.
22		− */
23		− if (inner_map->map_type == BPF_MAP_TYPE_PROG_ARRAY \|\|
24		− inner_map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE \|\|
25		− inner_map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE \|\|
26		− inner_map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
27		− fdput(f);
28		− return ERR_PTR(-ENOTSUPP);
29		− }
30		−
31	20	/* Does not support >1 level map-in-map */
32	21	if (inner_map->inner_map_meta) {
33	22	fdput(f);
34	23	return ERR_PTR(-EINVAL);
35	24	}
36	25
	26	+ if (!inner_map->ops->map_meta_equal) {
	27	+ fdput(f);
	28	+ return ERR_PTR(-ENOTSUPP);
	29	+ }
	30	+
37	31	if (map_value_has_spin_lock(inner_map)) {
38	32	fdput(f);
39	33	return ERR_PTR(-ENOTSUPP);
@@ -81,15 +75,14 @@ bool bpf_map_meta_equal(const struct bpf_map *meta0, · 41 unmodified lines
81	75	return meta0->map_type == meta1->map_type &&
82	76	meta0->key_size == meta1->key_size &&
83	77	meta0->value_size == meta1->value_size &&
84		− meta0->map_flags == meta1->map_flags &&
85		− meta0->max_entries == meta1->max_entries;
	78	+ meta0->map_flags == meta1->map_flags;
86	79	}
87	80
88	81	void bpf_map_fd_get_ptr(struct bpf_map map,
89	82	struct file map_file / not used */,
90	83	int ufd)
91	84	{
92		− struct bpf_map *inner_map;
	85	+ struct bpf_map inner_map, inner_map_meta;
93	86	struct fd f;
94	87
95	88	f = fdget(ufd);
@@ -97,7 +90,8 @@ void bpf_map_fd_get_ptr(struct bpf_map map, · 1 unmodified line
97	90	if (IS_ERR(inner_map))
98	91	return inner_map;
99	92
100		− if (bpf_map_meta_equal(map->inner_map_meta, inner_map))
	93	+ inner_map_meta = map->inner_map_meta;
	94	+ if (inner_map_meta->ops->map_meta_equal(inner_map_meta, inner_map))
101	95	bpf_map_inc(inner_map);
102	96	else
103	97	inner_map = ERR_PTR(-EINVAL);

kernel/bpf/map_in_map.h+0−2

@@ -11,8 +11,6 @@ struct bpf_map; · 10 unmodified lines
11	11
12	12	struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd);
13	13	void bpf_map_meta_free(struct bpf_map *map_meta);
14		−bool bpf_map_meta_equal(const struct bpf_map *meta0,
15		− const struct bpf_map *meta1);
16	14	void bpf_map_fd_get_ptr(struct bpf_map map, struct file *map_file,
17	15	int ufd);
18	16	void bpf_map_fd_put_ptr(void *ptr);

kernel/bpf/map_iter.c+15−0

@@ -149,6 +149,19 @@ static void bpf_iter_detach_map(struct bpf_iter_aux_info *aux) · 148 unmodified lines
149	149	bpf_map_put_with_uref(aux->map);
150	150	}
151	151
	152	+void bpf_iter_map_show_fdinfo(const struct bpf_iter_aux_info *aux,
	153	+ struct seq_file *seq)
	154	+{
	155	+ seq_printf(seq, "map_id:\t%u\n", aux->map->id);
	156	+}
	157	+
	158	+int bpf_iter_map_fill_link_info(const struct bpf_iter_aux_info *aux,
	159	+ struct bpf_link_info *info)
	160	+{
	161	+ info->iter.map.map_id = aux->map->id;
	162	+ return 0;
	163	+}
	164	+
152	165	DEFINE_BPF_ITER_FUNC(bpf_map_elem, struct bpf_iter_meta *meta,
153	166	struct bpf_map map, void key, void *value)
154	167
@@ -156,6 +169,8 @@ static const struct bpf_iter_reg bpf_map_elem_reg_info = { · 1 unmodified line
156	169	.target = "bpf_map_elem",
157	170	.attach_target = bpf_iter_attach_map,
158	171	.detach_target = bpf_iter_detach_map,
	172	+ .show_fdinfo = bpf_iter_map_show_fdinfo,
	173	+ .fill_link_info = bpf_iter_map_fill_link_info,
159	174	.ctx_arg_info_size = 2,
160	175	.ctx_arg_info = {
161	176	{ offsetof(struct bpf_iter__bpf_map_elem, key),

kernel/bpf/preload/Kconfig+26−0

@@ -0,0 +1,26 @@
	1	+# SPDX-License-Identifier: GPL-2.0-only
	2	+config USERMODE_DRIVER
	3	+ bool
	4	+ default n
	5	+
	6	+menuconfig BPF_PRELOAD
	7	+ bool "Preload BPF file system with kernel specific program and map iterators"
	8	+ depends on BPF
	9	+ # The dependency on !COMPILE_TEST prevents it from being enabled
	10	+ # in allmodconfig or allyesconfig configurations
	11	+ depends on !COMPILE_TEST
	12	+ select USERMODE_DRIVER
	13	+ help
	14	+ This builds kernel module with several embedded BPF programs that are
	15	+ pinned into BPF FS mount point as human readable files that are
	16	+ useful in debugging and introspection of BPF programs and maps.
	17	+
	18	+if BPF_PRELOAD
	19	+config BPF_PRELOAD_UMD
	20	+ tristate "bpf_preload kernel module with user mode driver"
	21	+ depends on CC_CAN_LINK
	22	+ depends on m \|\| CC_CAN_LINK_STATIC
	23	+ default m
	24	+ help
	25	+ This builds bpf_preload kernel module with embedded user mode driver.
	26	+endif

kernel/bpf/preload/Makefile+23−0

@@ -0,0 +1,23 @@
	1	+# SPDX-License-Identifier: GPL-2.0
	2	+
	3	+LIBBPF_SRCS = $(srctree)/tools/lib/bpf/
	4	+LIBBPF_A = $(obj)/libbpf.a
	5	+LIBBPF_OUT = $(abspath $(obj))
	6	+
	7	+$(LIBBPF_A):
	8	+ $(Q)$(MAKE) -C $(LIBBPF_SRCS) OUTPUT=$(LIBBPF_OUT)/ $(LIBBPF_OUT)/libbpf.a
	9	+
	10	+userccflags += -I $(srctree)/tools/include/ -I $(srctree)/tools/include/uapi \
	11	+ -I $(srctree)/tools/lib/ -Wno-unused-result
	12	+
	13	+userprogs := bpf_preload_umd
	14	+
	15	+bpf_preload_umd-objs := iterators/iterators.o
	16	+bpf_preload_umd-userldlibs := $(LIBBPF_A) -lelf -lz
	17	+
	18	+$(obj)/bpf_preload_umd: $(LIBBPF_A)
	19	+
	20	+$(obj)/bpf_preload_umd_blob.o: $(obj)/bpf_preload_umd
	21	+
	22	+obj-$(CONFIG_BPF_PRELOAD_UMD) += bpf_preload.o
	23	+bpf_preload-objs += bpf_preload_kern.o bpf_preload_umd_blob.o

kernel/bpf/preload/bpf_preload.h+16−0

@@ -0,0 +1,16 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+#ifndef _BPF_PRELOAD_H
	3	+#define _BPF_PRELOAD_H
	4	+
	5	+#include <linux/usermode_driver.h>
	6	+#include "iterators/bpf_preload_common.h"
	7	+
	8	+struct bpf_preload_ops {
	9	+ struct umd_info info;
	10	+ int (preload)(struct bpf_preload_info );
	11	+ int (*finish)(void);
	12	+ struct module *owner;
	13	+};
	14	+extern struct bpf_preload_ops *bpf_preload_ops;
	15	+#define BPF_PRELOAD_LINKS 2
	16	+#endif

kernel/bpf/preload/bpf_preload_kern.c+91−0

@@ -0,0 +1,91 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
	3	+#include <linux/init.h>
	4	+#include <linux/module.h>
	5	+#include <linux/pid.h>
	6	+#include <linux/fs.h>
	7	+#include <linux/sched/signal.h>
	8	+#include "bpf_preload.h"
	9	+
	10	+extern char bpf_preload_umd_start;
	11	+extern char bpf_preload_umd_end;
	12	+
	13	+static int preload(struct bpf_preload_info *obj);
	14	+static int finish(void);
	15	+
	16	+static struct bpf_preload_ops umd_ops = {
	17	+ .info.driver_name = "bpf_preload",
	18	+ .preload = preload,
	19	+ .finish = finish,
	20	+ .owner = THIS_MODULE,
	21	+};
	22	+
	23	+static int preload(struct bpf_preload_info *obj)
	24	+{
	25	+ int magic = BPF_PRELOAD_START;
	26	+ loff_t pos = 0;
	27	+ int i, err;
	28	+ ssize_t n;
	29	+
	30	+ err = fork_usermode_driver(&umd_ops.info);
	31	+ if (err)
	32	+ return err;
	33	+
	34	+ /* send the start magic to let UMD proceed with loading BPF progs */
	35	+ n = kernel_write(umd_ops.info.pipe_to_umh,
	36	+ &magic, sizeof(magic), &pos);
	37	+ if (n != sizeof(magic))
	38	+ return -EPIPE;
	39	+
	40	+ /* receive bpf_link IDs and names from UMD */
	41	+ pos = 0;
	42	+ for (i = 0; i < BPF_PRELOAD_LINKS; i++) {
	43	+ n = kernel_read(umd_ops.info.pipe_from_umh,
	44	+ &obj[i], sizeof(*obj), &pos);
	45	+ if (n != sizeof(*obj))
	46	+ return -EPIPE;
	47	+ }
	48	+ return 0;
	49	+}
	50	+
	51	+static int finish(void)
	52	+{
	53	+ int magic = BPF_PRELOAD_END;
	54	+ struct pid *tgid;
	55	+ loff_t pos = 0;
	56	+ ssize_t n;
	57	+
	58	+ /* send the last magic to UMD. It will do a normal exit. */
	59	+ n = kernel_write(umd_ops.info.pipe_to_umh,
	60	+ &magic, sizeof(magic), &pos);
	61	+ if (n != sizeof(magic))
	62	+ return -EPIPE;
	63	+ tgid = umd_ops.info.tgid;
	64	+ wait_event(tgid->wait_pidfd, thread_group_exited(tgid));
	65	+ umd_ops.info.tgid = NULL;
	66	+ return 0;
	67	+}
	68	+
	69	+static int __init load_umd(void)
	70	+{
	71	+ int err;
	72	+
	73	+ err = umd_load_blob(&umd_ops.info, &bpf_preload_umd_start,
	74	+ &bpf_preload_umd_end - &bpf_preload_umd_start);
	75	+ if (err)
	76	+ return err;
	77	+ bpf_preload_ops = &umd_ops;
	78	+ return err;
	79	+}
	80	+
	81	+static void __exit fini_umd(void)
	82	+{
	83	+ bpf_preload_ops = NULL;
	84	+ /* kill UMD in case it's still there due to earlier error */
	85	+ kill_pid(umd_ops.info.tgid, SIGKILL, 1);
	86	+ umd_ops.info.tgid = NULL;
	87	+ umd_unload_blob(&umd_ops.info);
	88	+}
	89	+late_initcall(load_umd);
	90	+module_exit(fini_umd);
	91	+MODULE_LICENSE("GPL");

kernel/bpf/preload/bpf_preload_umd_blob.S+7−0

@@ -0,0 +1,7 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+ .section .init.rodata, "a"
	3	+ .global bpf_preload_umd_start
	4	+bpf_preload_umd_start:
	5	+ .incbin "kernel/bpf/preload/bpf_preload_umd"
	6	+ .global bpf_preload_umd_end
	7	+bpf_preload_umd_end:

kernel/bpf/preload/iterators/.gitignore+2−0

@@ -0,0 +1,2 @@
	1	+# SPDX-License-Identifier: GPL-2.0-only
	2	+/.output

kernel/bpf/preload/iterators/Makefile+57−0

@@ -0,0 +1,57 @@
	1	+# SPDX-License-Identifier: GPL-2.0
	2	+OUTPUT := .output
	3	+CLANG ?= clang
	4	+LLC ?= llc
	5	+LLVM_STRIP ?= llvm-strip
	6	+DEFAULT_BPFTOOL := $(OUTPUT)/sbin/bpftool
	7	+BPFTOOL ?= $(DEFAULT_BPFTOOL)
	8	+LIBBPF_SRC := $(abspath ../../../../tools/lib/bpf)
	9	+BPFOBJ := $(OUTPUT)/libbpf.a
	10	+BPF_INCLUDE := $(OUTPUT)
	11	+INCLUDES := -I$(OUTPUT) -I$(BPF_INCLUDE) -I$(abspath ../../../../tools/lib) \
	12	+ -I$(abspath ../../../../tools/include/uapi)
	13	+CFLAGS := -g -Wall
	14	+
	15	+abs_out := $(abspath $(OUTPUT))
	16	+ifeq ($(V),1)
	17	+Q =
	18	+msg =
	19	+else
	20	+Q = @
	21	+msg = @printf ' %-8s %s%s\n' "$(1)" "$(notdir $(2))" "$(if $(3), $(3))";
	22	+MAKEFLAGS += --no-print-directory
	23	+submake_extras := feature_display=0
	24	+endif
	25	+
	26	+.DELETE_ON_ERROR:
	27	+
	28	+.PHONY: all clean
	29	+
	30	+all: iterators.skel.h
	31	+
	32	+clean:
	33	+ $(call msg,CLEAN)
	34	+ $(Q)rm -rf $(OUTPUT) iterators
	35	+
	36	+iterators.skel.h: $(OUTPUT)/iterators.bpf.o \| $(BPFTOOL)
	37	+ $(call msg,GEN-SKEL,$@)
	38	+ $(Q)$(BPFTOOL) gen skeleton $< > $@
	39	+
	40	+
	41	+$(OUTPUT)/iterators.bpf.o: iterators.bpf.c $(BPFOBJ) \| $(OUTPUT)
	42	+ $(call msg,BPF,$@)
	43	+ $(Q)$(CLANG) -g -O2 -target bpf $(INCLUDES) \
	44	+ -c $(filter %.c,$^) -o $@ && \
	45	+ $(LLVM_STRIP) -g $@
	46	+
	47	+$(OUTPUT):
	48	+ $(call msg,MKDIR,$@)
	49	+ $(Q)mkdir -p $(OUTPUT)
	50	+
	51	+$(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) \| $(OUTPUT)
	52	+ $(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) \
	53	+ OUTPUT=$(abspath $(dir $@))/ $(abspath $@)
	54	+
	55	+$(DEFAULT_BPFTOOL):
	56	+ $(Q)$(MAKE) $(submake_extras) -C ../../../../tools/bpf/bpftool \
	57	+ prefix= OUTPUT=$(abs_out)/ DESTDIR=$(abs_out) install

kernel/bpf/preload/iterators/README+4−0

@@ -0,0 +1,4 @@
	1	+WARNING:
	2	+If you change "iterators.bpf.c" do "make -j" in this directory to rebuild "iterators.skel.h".
	3	+Make sure to have clang 10 installed.
	4	+See Documentation/bpf/bpf_devel_QA.rst

kernel/bpf/preload/iterators/bpf_preload_common.h+13−0

@@ -0,0 +1,13 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+#ifndef _BPF_PRELOAD_COMMON_H
	3	+#define _BPF_PRELOAD_COMMON_H
	4	+
	5	+#define BPF_PRELOAD_START 0x5555
	6	+#define BPF_PRELOAD_END 0xAAAA
	7	+
	8	+struct bpf_preload_info {
	9	+ char link_name[16];
	10	+ int link_id;
	11	+};
	12	+
	13	+#endif

kernel/bpf/preload/iterators/iterators.bpf.c+114−0

@@ -0,0 +1,114 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2020 Facebook */
	3	+#include <linux/bpf.h>
	4	+#include <bpf/bpf_helpers.h>
	5	+#include <bpf/bpf_tracing.h>
	6	+#include <bpf/bpf_core_read.h>
	7	+
	8	+#pragma clang attribute push (__attribute__((preserve_access_index)), apply_to = record)
	9	+struct seq_file;
	10	+struct bpf_iter_meta {
	11	+ struct seq_file *seq;
	12	+ __u64 session_id;
	13	+ __u64 seq_num;
	14	+};
	15	+
	16	+struct bpf_map {
	17	+ __u32 id;
	18	+ char name[16];
	19	+ __u32 max_entries;
	20	+};
	21	+
	22	+struct bpf_iter__bpf_map {
	23	+ struct bpf_iter_meta *meta;
	24	+ struct bpf_map *map;
	25	+};
	26	+
	27	+struct btf_type {
	28	+ __u32 name_off;
	29	+};
	30	+
	31	+struct btf_header {
	32	+ __u32 str_len;
	33	+};
	34	+
	35	+struct btf {
	36	+ const char *strings;
	37	+ struct btf_type **types;
	38	+ struct btf_header hdr;
	39	+};
	40	+
	41	+struct bpf_prog_aux {
	42	+ __u32 id;
	43	+ char name[16];
	44	+ const char *attach_func_name;
	45	+ struct bpf_prog *linked_prog;
	46	+ struct bpf_func_info *func_info;
	47	+ struct btf *btf;
	48	+};
	49	+
	50	+struct bpf_prog {
	51	+ struct bpf_prog_aux *aux;
	52	+};
	53	+
	54	+struct bpf_iter__bpf_prog {
	55	+ struct bpf_iter_meta *meta;
	56	+ struct bpf_prog *prog;
	57	+};
	58	+#pragma clang attribute pop
	59	+
	60	+static const char get_name(struct btf btf, long btf_id, const char *fallback)
	61	+{
	62	+ struct btf_type *types, t;
	63	+ unsigned int name_off;
	64	+ const char *str;
	65	+
	66	+ if (!btf)
	67	+ return fallback;
	68	+ str = btf->strings;
	69	+ types = btf->types;
	70	+ bpf_probe_read_kernel(&t, sizeof(t), types + btf_id);
	71	+ name_off = BPF_CORE_READ(t, name_off);
	72	+ if (name_off >= btf->hdr.str_len)
	73	+ return fallback;
	74	+ return str + name_off;
	75	+}
	76	+
	77	+SEC("iter/bpf_map")
	78	+int dump_bpf_map(struct bpf_iter__bpf_map *ctx)
	79	+{
	80	+ struct seq_file *seq = ctx->meta->seq;
	81	+ __u64 seq_num = ctx->meta->seq_num;
	82	+ struct bpf_map *map = ctx->map;
	83	+
	84	+ if (!map)
	85	+ return 0;
	86	+
	87	+ if (seq_num == 0)
	88	+ BPF_SEQ_PRINTF(seq, " id name max_entries\n");
	89	+
	90	+ BPF_SEQ_PRINTF(seq, "%4u %-16s%6d\n", map->id, map->name, map->max_entries);
	91	+ return 0;
	92	+}
	93	+
	94	+SEC("iter/bpf_prog")
	95	+int dump_bpf_prog(struct bpf_iter__bpf_prog *ctx)
	96	+{
	97	+ struct seq_file *seq = ctx->meta->seq;
	98	+ __u64 seq_num = ctx->meta->seq_num;
	99	+ struct bpf_prog *prog = ctx->prog;
	100	+ struct bpf_prog_aux *aux;
	101	+
	102	+ if (!prog)
	103	+ return 0;
	104	+
	105	+ aux = prog->aux;
	106	+ if (seq_num == 0)
	107	+ BPF_SEQ_PRINTF(seq, " id name attached\n");
	108	+
	109	+ BPF_SEQ_PRINTF(seq, "%4u %-16s %s %s\n", aux->id,
	110	+ get_name(aux->btf, aux->func_info[0].type_id, aux->name),
	111	+ aux->attach_func_name, aux->linked_prog->aux->name);
	112	+ return 0;
	113	+}
	114	+char LICENSE[] SEC("license") = "GPL";

kernel/bpf/preload/iterators/iterators.c+94−0

@@ -0,0 +1,94 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2020 Facebook */
	3	+#include <argp.h>
	4	+#include <stdio.h>
	5	+#include <stdlib.h>
	6	+#include <string.h>
	7	+#include <unistd.h>
	8	+#include <fcntl.h>
	9	+#include <sys/resource.h>
	10	+#include <bpf/libbpf.h>
	11	+#include <bpf/bpf.h>
	12	+#include <sys/mount.h>
	13	+#include "iterators.skel.h"
	14	+#include "bpf_preload_common.h"
	15	+
	16	+int to_kernel = -1;
	17	+int from_kernel = 0;
	18	+
	19	+static int send_link_to_kernel(struct bpf_link link, const char link_name)
	20	+{
	21	+ struct bpf_preload_info obj = {};
	22	+ struct bpf_link_info info = {};
	23	+ __u32 info_len = sizeof(info);
	24	+ int err;
	25	+
	26	+ err = bpf_obj_get_info_by_fd(bpf_link__fd(link), &info, &info_len);
	27	+ if (err)
	28	+ return err;
	29	+ obj.link_id = info.id;
	30	+ if (strlen(link_name) >= sizeof(obj.link_name))
	31	+ return -E2BIG;
	32	+ strcpy(obj.link_name, link_name);
	33	+ if (write(to_kernel, &obj, sizeof(obj)) != sizeof(obj))
	34	+ return -EPIPE;
	35	+ return 0;
	36	+}
	37	+
	38	+int main(int argc, char **argv)
	39	+{
	40	+ struct rlimit rlim = { RLIM_INFINITY, RLIM_INFINITY };
	41	+ struct iterators_bpf *skel;
	42	+ int err, magic;
	43	+ int debug_fd;
	44	+
	45	+ debug_fd = open("/dev/console", O_WRONLY \| O_NOCTTY \| O_CLOEXEC);
	46	+ if (debug_fd < 0)
	47	+ return 1;
	48	+ to_kernel = dup(1);
	49	+ close(1);
	50	+ dup(debug_fd);
	51	+ /* now stdin and stderr point to /dev/console */
	52	+
	53	+ read(from_kernel, &magic, sizeof(magic));
	54	+ if (magic != BPF_PRELOAD_START) {
	55	+ printf("bad start magic %d\n", magic);
	56	+ return 1;
	57	+ }
	58	+ setrlimit(RLIMIT_MEMLOCK, &rlim);
	59	+ /* libbpf opens BPF object and loads it into the kernel */
	60	+ skel = iterators_bpf__open_and_load();
	61	+ if (!skel) {
	62	+ /* iterators.skel.h is little endian.
	63	+ * libbpf doesn't support automatic little->big conversion
	64	+ * of BPF bytecode yet.
	65	+ * The program load will fail in such case.
	66	+ */
	67	+ printf("Failed load could be due to wrong endianness\n");
	68	+ return 1;
	69	+ }
	70	+ err = iterators_bpf__attach(skel);
	71	+ if (err)
	72	+ goto cleanup;
	73	+
	74	+ /* send two bpf_link IDs with names to the kernel */
	75	+ err = send_link_to_kernel(skel->links.dump_bpf_map, "maps.debug");
	76	+ if (err)
	77	+ goto cleanup;
	78	+ err = send_link_to_kernel(skel->links.dump_bpf_prog, "progs.debug");
	79	+ if (err)
	80	+ goto cleanup;
	81	+
	82	+ /* The kernel will proceed with pinnging the links in bpffs.
	83	+ * UMD will wait on read from pipe.
	84	+ */
	85	+ read(from_kernel, &magic, sizeof(magic));
	86	+ if (magic != BPF_PRELOAD_END) {
	87	+ printf("bad final magic %d\n", magic);
	88	+ err = -EINVAL;
	89	+ }
	90	+cleanup:
	91	+ iterators_bpf__destroy(skel);
	92	+
	93	+ return err != 0;
	94	+}

kernel/bpf/preload/iterators/iterators.skel.h+410−0

@@ -0,0 +1,410 @@
	1	+/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
	2	+
	3	+/* THIS FILE IS AUTOGENERATED! */
	4	+#ifndef __ITERATORS_BPF_SKEL_H__
	5	+#define __ITERATORS_BPF_SKEL_H__
	6	+
	7	+#include <stdlib.h>
	8	+#include <bpf/libbpf.h>
	9	+
	10	+struct iterators_bpf {
	11	+ struct bpf_object_skeleton *skeleton;
	12	+ struct bpf_object *obj;
	13	+ struct {
	14	+ struct bpf_map *rodata;
	15	+ } maps;
	16	+ struct {
	17	+ struct bpf_program *dump_bpf_map;
	18	+ struct bpf_program *dump_bpf_prog;
	19	+ } progs;
	20	+ struct {
	21	+ struct bpf_link *dump_bpf_map;
	22	+ struct bpf_link *dump_bpf_prog;
	23	+ } links;
	24	+ struct iterators_bpf__rodata {
	25	+ char dump_bpf_map____fmt[35];
	26	+ char dump_bpf_map____fmt_1[14];
	27	+ char dump_bpf_prog____fmt[32];
	28	+ char dump_bpf_prog____fmt_2[17];
	29	+ } *rodata;
	30	+};
	31	+
	32	+static void
	33	+iterators_bpf__destroy(struct iterators_bpf *obj)
	34	+{
	35	+ if (!obj)
	36	+ return;
	37	+ if (obj->skeleton)
	38	+ bpf_object__destroy_skeleton(obj->skeleton);
	39	+ free(obj);
	40	+}
	41	+
	42	+static inline int
	43	+iterators_bpf__create_skeleton(struct iterators_bpf *obj);
	44	+
	45	+static inline struct iterators_bpf *
	46	+iterators_bpf__open_opts(const struct bpf_object_open_opts *opts)
	47	+{
	48	+ struct iterators_bpf *obj;
	49	+
	50	+ obj = (typeof(obj))calloc(1, sizeof(*obj));
	51	+ if (!obj)
	52	+ return NULL;
	53	+ if (iterators_bpf__create_skeleton(obj))
	54	+ goto err;
	55	+ if (bpf_object__open_skeleton(obj->skeleton, opts))
	56	+ goto err;
	57	+
	58	+ return obj;
	59	+err:
	60	+ iterators_bpf__destroy(obj);
	61	+ return NULL;
	62	+}
	63	+
	64	+static inline struct iterators_bpf *
	65	+iterators_bpf__open(void)
	66	+{
	67	+ return iterators_bpf__open_opts(NULL);
	68	+}
	69	+
	70	+static inline int
	71	+iterators_bpf__load(struct iterators_bpf *obj)
	72	+{
	73	+ return bpf_object__load_skeleton(obj->skeleton);
	74	+}
	75	+
	76	+static inline struct iterators_bpf *
	77	+iterators_bpf__open_and_load(void)
	78	+{
	79	+ struct iterators_bpf *obj;
	80	+
	81	+ obj = iterators_bpf__open();
	82	+ if (!obj)
	83	+ return NULL;
	84	+ if (iterators_bpf__load(obj)) {
	85	+ iterators_bpf__destroy(obj);
	86	+ return NULL;
	87	+ }
	88	+ return obj;
	89	+}
	90	+
	91	+static inline int
	92	+iterators_bpf__attach(struct iterators_bpf *obj)
	93	+{
	94	+ return bpf_object__attach_skeleton(obj->skeleton);
	95	+}
	96	+
	97	+static inline void
	98	+iterators_bpf__detach(struct iterators_bpf *obj)
	99	+{
	100	+ return bpf_object__detach_skeleton(obj->skeleton);
	101	+}
	102	+
	103	+static inline int
	104	+iterators_bpf__create_skeleton(struct iterators_bpf *obj)
	105	+{
	106	+ struct bpf_object_skeleton *s;
	107	+
	108	+ s = (typeof(s))calloc(1, sizeof(*s));
	109	+ if (!s)
	110	+ return -1;
	111	+ obj->skeleton = s;
	112	+
	113	+ s->sz = sizeof(*s);
	114	+ s->name = "iterators_bpf";
	115	+ s->obj = &obj->obj;
	116	+
	117	+ /* maps */
	118	+ s->map_cnt = 1;
	119	+ s->map_skel_sz = sizeof(*s->maps);
	120	+ s->maps = (typeof(s->maps))calloc(s->map_cnt, s->map_skel_sz);
	121	+ if (!s->maps)
	122	+ goto err;
	123	+
	124	+ s->maps[0].name = "iterator.rodata";
	125	+ s->maps[0].map = &obj->maps.rodata;
	126	+ s->maps[0].mmaped = (void **)&obj->rodata;
	127	+
	128	+ /* programs */
	129	+ s->prog_cnt = 2;
	130	+ s->prog_skel_sz = sizeof(*s->progs);
	131	+ s->progs = (typeof(s->progs))calloc(s->prog_cnt, s->prog_skel_sz);
	132	+ if (!s->progs)
	133	+ goto err;
	134	+
	135	+ s->progs[0].name = "dump_bpf_map";
	136	+ s->progs[0].prog = &obj->progs.dump_bpf_map;
	137	+ s->progs[0].link = &obj->links.dump_bpf_map;
	138	+
	139	+ s->progs[1].name = "dump_bpf_prog";
	140	+ s->progs[1].prog = &obj->progs.dump_bpf_prog;
	141	+ s->progs[1].link = &obj->links.dump_bpf_prog;
	142	+
	143	+ s->data_sz = 7128;
	144	+ s->data = (void *)"\
	145	+\x7f\x45\x4c\x46\x02\x01\x01\0\0\0\0\0\0\0\0\0\x01\0\xf7\0\x01\0\0\0\0\0\0\0\0\
	146	+\0\0\0\0\0\0\0\0\0\0\0\x18\x18\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\x40\0\x0f\0\
	147	+\x0e\0\x79\x12\0\0\0\0\0\0\x79\x26\0\0\0\0\0\0\x79\x17\x08\0\0\0\0\0\x15\x07\
	148	+\x1a\0\0\0\0\0\x79\x21\x10\0\0\0\0\0\x55\x01\x08\0\0\0\0\0\xbf\xa4\0\0\0\0\0\0\
	149	+\x07\x04\0\0\xe8\xff\xff\xff\xbf\x61\0\0\0\0\0\0\x18\x02\0\0\0\0\0\0\0\0\0\0\0\
	150	+\0\0\0\xb7\x03\0\0\x23\0\0\0\xb7\x05\0\0\0\0\0\0\x85\0\0\0\x7e\0\0\0\x61\x71\0\
	151	+\0\0\0\0\0\x7b\x1a\xe8\xff\0\0\0\0\xb7\x01\0\0\x04\0\0\0\xbf\x72\0\0\0\0\0\0\
	152	+\x0f\x12\0\0\0\0\0\0\x7b\x2a\xf0\xff\0\0\0\0\x61\x71\x14\0\0\0\0\0\x7b\x1a\xf8\
	153	+\xff\0\0\0\0\xbf\xa4\0\0\0\0\0\0\x07\x04\0\0\xe8\xff\xff\xff\xbf\x61\0\0\0\0\0\
	154	+\0\x18\x02\0\0\x23\0\0\0\0\0\0\0\0\0\0\0\xb7\x03\0\0\x0e\0\0\0\xb7\x05\0\0\x18\
	155	+\0\0\0\x85\0\0\0\x7e\0\0\0\xb7\0\0\0\0\0\0\0\x95\0\0\0\0\0\0\0\x79\x12\0\0\0\0\
	156	+\0\0\x79\x26\0\0\0\0\0\0\x79\x11\x08\0\0\0\0\0\x15\x01\x3b\0\0\0\0\0\x79\x17\0\
	157	+\0\0\0\0\0\x79\x21\x10\0\0\0\0\0\x55\x01\x08\0\0\0\0\0\xbf\xa4\0\0\0\0\0\0\x07\
	158	+\x04\0\0\xd0\xff\xff\xff\xbf\x61\0\0\0\0\0\0\x18\x02\0\0\x31\0\0\0\0\0\0\0\0\0\
	159	+\0\0\xb7\x03\0\0\x20\0\0\0\xb7\x05\0\0\0\0\0\0\x85\0\0\0\x7e\0\0\0\x7b\x6a\xc8\
	160	+\xff\0\0\0\0\x61\x71\0\0\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\xb7\x03\0\0\x04\0\0\0\
	161	+\xbf\x79\0\0\0\0\0\0\x0f\x39\0\0\0\0\0\0\x79\x71\x28\0\0\0\0\0\x79\x78\x30\0\0\
	162	+\0\0\0\x15\x08\x18\0\0\0\0\0\xb7\x02\0\0\0\0\0\0\x0f\x21\0\0\0\0\0\0\x61\x11\
	163	+\x04\0\0\0\0\0\x79\x83\x08\0\0\0\0\0\x67\x01\0\0\x03\0\0\0\x0f\x13\0\0\0\0\0\0\
	164	+\x79\x86\0\0\0\0\0\0\xbf\xa1\0\0\0\0\0\0\x07\x01\0\0\xf8\xff\xff\xff\xb7\x02\0\
	165	+\0\x08\0\0\0\x85\0\0\0\x71\0\0\0\xb7\x01\0\0\0\0\0\0\x79\xa3\xf8\xff\0\0\0\0\
	166	+\x0f\x13\0\0\0\0\0\0\xbf\xa1\0\0\0\0\0\0\x07\x01\0\0\xf4\xff\xff\xff\xb7\x02\0\
	167	+\0\x04\0\0\0\x85\0\0\0\x04\0\0\0\xb7\x03\0\0\x04\0\0\0\x61\xa1\xf4\xff\0\0\0\0\
	168	+\x61\x82\x10\0\0\0\0\0\x3d\x21\x02\0\0\0\0\0\x0f\x16\0\0\0\0\0\0\xbf\x69\0\0\0\
	169	+\0\0\0\x7b\x9a\xd8\xff\0\0\0\0\x79\x71\x18\0\0\0\0\0\x7b\x1a\xe0\xff\0\0\0\0\
	170	+\x79\x71\x20\0\0\0\0\0\x79\x11\0\0\0\0\0\0\x0f\x31\0\0\0\0\0\0\x7b\x1a\xe8\xff\
	171	+\0\0\0\0\xbf\xa4\0\0\0\0\0\0\x07\x04\0\0\xd0\xff\xff\xff\x79\xa1\xc8\xff\0\0\0\
	172	+\0\x18\x02\0\0\x51\0\0\0\0\0\0\0\0\0\0\0\xb7\x03\0\0\x11\0\0\0\xb7\x05\0\0\x20\
	173	+\0\0\0\x85\0\0\0\x7e\0\0\0\xb7\0\0\0\0\0\0\0\x95\0\0\0\0\0\0\0\x20\x20\x69\x64\
	174	+\x20\x6e\x61\x6d\x65\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x6d\
	175	+\x61\x78\x5f\x65\x6e\x74\x72\x69\x65\x73\x0a\0\x25\x34\x75\x20\x25\x2d\x31\x36\
	176	+\x73\x25\x36\x64\x0a\0\x20\x20\x69\x64\x20\x6e\x61\x6d\x65\x20\x20\x20\x20\x20\
	177	+\x20\x20\x20\x20\x20\x20\x20\x20\x61\x74\x74\x61\x63\x68\x65\x64\x0a\0\x25\x34\
	178	+\x75\x20\x25\x2d\x31\x36\x73\x20\x25\x73\x20\x25\x73\x0a\0\x47\x50\x4c\0\x9f\
	179	+\xeb\x01\0\x18\0\0\0\0\0\0\0\x1c\x04\0\0\x1c\x04\0\0\0\x05\0\0\0\0\0\0\0\0\0\
	180	+\x02\x02\0\0\0\x01\0\0\0\x02\0\0\x04\x10\0\0\0\x13\0\0\0\x03\0\0\0\0\0\0\0\x18\
	181	+\0\0\0\x04\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\x02\x08\0\0\0\0\0\0\0\0\0\0\x02\x0d\0\
	182	+\0\0\0\0\0\0\x01\0\0\x0d\x06\0\0\0\x1c\0\0\0\x01\0\0\0\x20\0\0\0\0\0\0\x01\x04\
	183	+\0\0\0\x20\0\0\x01\x24\0\0\0\x01\0\0\x0c\x05\0\0\0\xa3\0\0\0\x03\0\0\x04\x18\0\
	184	+\0\0\xb1\0\0\0\x09\0\0\0\0\0\0\0\xb5\0\0\0\x0b\0\0\0\x40\0\0\0\xc0\0\0\0\x0b\0\
	185	+\0\0\x80\0\0\0\0\0\0\0\0\0\0\x02\x0a\0\0\0\xc8\0\0\0\0\0\0\x07\0\0\0\0\xd1\0\0\
	186	+\0\0\0\0\x08\x0c\0\0\0\xd7\0\0\0\0\0\0\x01\x08\0\0\0\x40\0\0\0\x98\x01\0\0\x03\
	187	+\0\0\x04\x18\0\0\0\xa0\x01\0\0\x0e\0\0\0\0\0\0\0\xa3\x01\0\0\x11\0\0\0\x20\0\0\
	188	+\0\xa8\x01\0\0\x0e\0\0\0\xa0\0\0\0\xb4\x01\0\0\0\0\0\x08\x0f\0\0\0\xba\x01\0\0\
	189	+\0\0\0\x01\x04\0\0\0\x20\0\0\0\xc7\x01\0\0\0\0\0\x01\x01\0\0\0\x08\0\0\x01\0\0\
	190	+\0\0\0\0\0\x03\0\0\0\0\x10\0\0\0\x12\0\0\0\x10\0\0\0\xcc\x01\0\0\0\0\0\x01\x04\
	191	+\0\0\0\x20\0\0\0\0\0\0\0\0\0\0\x02\x14\0\0\0\x30\x02\0\0\x02\0\0\x04\x10\0\0\0\
	192	+\x13\0\0\0\x03\0\0\0\0\0\0\0\x43\x02\0\0\x15\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\x02\
	193	+\x18\0\0\0\0\0\0\0\x01\0\0\x0d\x06\0\0\0\x1c\0\0\0\x13\0\0\0\x48\x02\0\0\x01\0\
	194	+\0\x0c\x16\0\0\0\x94\x02\0\0\x01\0\0\x04\x08\0\0\0\x9d\x02\0\0\x19\0\0\0\0\0\0\
	195	+\0\0\0\0\0\0\0\0\x02\x1a\0\0\0\xee\x02\0\0\x06\0\0\x04\x38\0\0\0\xa0\x01\0\0\
	196	+\x0e\0\0\0\0\0\0\0\xa3\x01\0\0\x11\0\0\0\x20\0\0\0\xfb\x02\0\0\x1b\0\0\0\xc0\0\
	197	+\0\0\x0c\x03\0\0\x15\0\0\0\0\x01\0\0\x18\x03\0\0\x1d\0\0\0\x40\x01\0\0\x22\x03\
	198	+\0\0\x1e\0\0\0\x80\x01\0\0\0\0\0\0\0\0\0\x02\x1c\0\0\0\0\0\0\0\0\0\0\x0a\x10\0\
	199	+\0\0\0\0\0\0\0\0\0\x02\x1f\0\0\0\0\0\0\0\0\0\0\x02\x20\0\0\0\x6c\x03\0\0\x02\0\
	200	+\0\x04\x08\0\0\0\x7a\x03\0\0\x0e\0\0\0\0\0\0\0\x83\x03\0\0\x0e\0\0\0\x20\0\0\0\
	201	+\x22\x03\0\0\x03\0\0\x04\x18\0\0\0\x8d\x03\0\0\x1b\0\0\0\0\0\0\0\x95\x03\0\0\
	202	+\x21\0\0\0\x40\0\0\0\x9b\x03\0\0\x23\0\0\0\x80\0\0\0\0\0\0\0\0\0\0\x02\x22\0\0\
	203	+\0\0\0\0\0\0\0\0\x02\x24\0\0\0\x9f\x03\0\0\x01\0\0\x04\x04\0\0\0\xaa\x03\0\0\
	204	+\x0e\0\0\0\0\0\0\0\x13\x04\0\0\x01\0\0\x04\x04\0\0\0\x1c\x04\0\0\x0e\0\0\0\0\0\
	205	+\0\0\0\0\0\0\0\0\0\x03\0\0\0\0\x1c\0\0\0\x12\0\0\0\x23\0\0\0\x92\x04\0\0\0\0\0\
	206	+\x0e\x25\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\0\0\0\x1c\0\0\0\x12\0\0\0\x0e\0\0\0\
	207	+\xa6\x04\0\0\0\0\0\x0e\x27\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\0\0\0\x1c\0\0\0\
	208	+\x12\0\0\0\x20\0\0\0\xbc\x04\0\0\0\0\0\x0e\x29\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\
	209	+\0\0\0\0\x1c\0\0\0\x12\0\0\0\x11\0\0\0\xd1\x04\0\0\0\0\0\x0e\x2b\0\0\0\0\0\0\0\
	210	+\0\0\0\0\0\0\0\x03\0\0\0\0\x10\0\0\0\x12\0\0\0\x04\0\0\0\xe8\x04\0\0\0\0\0\x0e\
	211	+\x2d\0\0\0\x01\0\0\0\xf0\x04\0\0\x04\0\0\x0f\0\0\0\0\x26\0\0\0\0\0\0\0\x23\0\0\
	212	+\0\x28\0\0\0\x23\0\0\0\x0e\0\0\0\x2a\0\0\0\x31\0\0\0\x20\0\0\0\x2c\0\0\0\x51\0\
	213	+\0\0\x11\0\0\0\xf8\x04\0\0\x01\0\0\x0f\0\0\0\0\x2e\0\0\0\0\0\0\0\x04\0\0\0\0\
	214	+\x62\x70\x66\x5f\x69\x74\x65\x72\x5f\x5f\x62\x70\x66\x5f\x6d\x61\x70\0\x6d\x65\
	215	+\x74\x61\0\x6d\x61\x70\0\x63\x74\x78\0\x69\x6e\x74\0\x64\x75\x6d\x70\x5f\x62\
	216	+\x70\x66\x5f\x6d\x61\x70\0\x69\x74\x65\x72\x2f\x62\x70\x66\x5f\x6d\x61\x70\0\
	217	+\x30\x3a\x30\0\x2f\x77\x2f\x6e\x65\x74\x2d\x6e\x65\x78\x74\x2f\x6b\x65\x72\x6e\
	218	+\x65\x6c\x2f\x62\x70\x66\x2f\x70\x72\x65\x6c\x6f\x61\x64\x2f\x69\x74\x65\x72\
	219	+\x61\x74\x6f\x72\x73\x2f\x69\x74\x65\x72\x61\x74\x6f\x72\x73\x2e\x62\x70\x66\
	220	+\x2e\x63\0\x09\x73\x74\x72\x75\x63\x74\x20\x73\x65\x71\x5f\x66\x69\x6c\x65\x20\
	221	+\x2a\x73\x65\x71\x20\x3d\x20\x63\x74\x78\x2d\x3e\x6d\x65\x74\x61\x2d\x3e\x73\
	222	+\x65\x71\x3b\0\x62\x70\x66\x5f\x69\x74\x65\x72\x5f\x6d\x65\x74\x61\0\x73\x65\
	223	+\x71\0\x73\x65\x73\x73\x69\x6f\x6e\x5f\x69\x64\0\x73\x65\x71\x5f\x6e\x75\x6d\0\
	224	+\x73\x65\x71\x5f\x66\x69\x6c\x65\0\x5f\x5f\x75\x36\x34\0\x6c\x6f\x6e\x67\x20\
	225	+\x6c\x6f\x6e\x67\x20\x75\x6e\x73\x69\x67\x6e\x65\x64\x20\x69\x6e\x74\0\x30\x3a\
	226	+\x31\0\x09\x73\x74\x72\x75\x63\x74\x20\x62\x70\x66\x5f\x6d\x61\x70\x20\x2a\x6d\
	227	+\x61\x70\x20\x3d\x20\x63\x74\x78\x2d\x3e\x6d\x61\x70\x3b\0\x09\x69\x66\x20\x28\
	228	+\x21\x6d\x61\x70\x29\0\x30\x3a\x32\0\x09\x5f\x5f\x75\x36\x34\x20\x73\x65\x71\
	229	+\x5f\x6e\x75\x6d\x20\x3d\x20\x63\x74\x78\x2d\x3e\x6d\x65\x74\x61\x2d\x3e\x73\
	230	+\x65\x71\x5f\x6e\x75\x6d\x3b\0\x09\x69\x66\x20\x28\x73\x65\x71\x5f\x6e\x75\x6d\
	231	+\x20\x3d\x3d\x20\x30\x29\0\x09\x09\x42\x50\x46\x5f\x53\x45\x51\x5f\x50\x52\x49\
	232	+\x4e\x54\x46\x28\x73\x65\x71\x2c\x20\x22\x20\x20\x69\x64\x20\x6e\x61\x6d\x65\
	233	+\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x6d\x61\x78\x5f\x65\x6e\
	234	+\x74\x72\x69\x65\x73\x5c\x6e\x22\x29\x3b\0\x62\x70\x66\x5f\x6d\x61\x70\0\x69\
	235	+\x64\0\x6e\x61\x6d\x65\0\x6d\x61\x78\x5f\x65\x6e\x74\x72\x69\x65\x73\0\x5f\x5f\
	236	+\x75\x33\x32\0\x75\x6e\x73\x69\x67\x6e\x65\x64\x20\x69\x6e\x74\0\x63\x68\x61\
	237	+\x72\0\x5f\x5f\x41\x52\x52\x41\x59\x5f\x53\x49\x5a\x45\x5f\x54\x59\x50\x45\x5f\
	238	+\x5f\0\x09\x42\x50\x46\x5f\x53\x45\x51\x5f\x50\x52\x49\x4e\x54\x46\x28\x73\x65\
	239	+\x71\x2c\x20\x22\x25\x34\x75\x20\x25\x2d\x31\x36\x73\x25\x36\x64\x5c\x6e\x22\
	240	+\x2c\x20\x6d\x61\x70\x2d\x3e\x69\x64\x2c\x20\x6d\x61\x70\x2d\x3e\x6e\x61\x6d\
	241	+\x65\x2c\x20\x6d\x61\x70\x2d\x3e\x6d\x61\x78\x5f\x65\x6e\x74\x72\x69\x65\x73\
	242	+\x29\x3b\0\x7d\0\x62\x70\x66\x5f\x69\x74\x65\x72\x5f\x5f\x62\x70\x66\x5f\x70\
	243	+\x72\x6f\x67\0\x70\x72\x6f\x67\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\
	244	+\x6f\x67\0\x69\x74\x65\x72\x2f\x62\x70\x66\x5f\x70\x72\x6f\x67\0\x09\x73\x74\
	245	+\x72\x75\x63\x74\x20\x62\x70\x66\x5f\x70\x72\x6f\x67\x20\x2a\x70\x72\x6f\x67\
	246	+\x20\x3d\x20\x63\x74\x78\x2d\x3e\x70\x72\x6f\x67\x3b\0\x09\x69\x66\x20\x28\x21\
	247	+\x70\x72\x6f\x67\x29\0\x62\x70\x66\x5f\x70\x72\x6f\x67\0\x61\x75\x78\0\x09\x61\
	248	+\x75\x78\x20\x3d\x20\x70\x72\x6f\x67\x2d\x3e\x61\x75\x78\x3b\0\x09\x09\x42\x50\
	249	+\x46\x5f\x53\x45\x51\x5f\x50\x52\x49\x4e\x54\x46\x28\x73\x65\x71\x2c\x20\x22\
	250	+\x20\x20\x69\x64\x20\x6e\x61\x6d\x65\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
	251	+\x20\x20\x20\x61\x74\x74\x61\x63\x68\x65\x64\x5c\x6e\x22\x29\x3b\0\x62\x70\x66\
	252	+\x5f\x70\x72\x6f\x67\x5f\x61\x75\x78\0\x61\x74\x74\x61\x63\x68\x5f\x66\x75\x6e\
	253	+\x63\x5f\x6e\x61\x6d\x65\0\x6c\x69\x6e\x6b\x65\x64\x5f\x70\x72\x6f\x67\0\x66\
	254	+\x75\x6e\x63\x5f\x69\x6e\x66\x6f\0\x62\x74\x66\0\x09\x42\x50\x46\x5f\x53\x45\
	255	+\x51\x5f\x50\x52\x49\x4e\x54\x46\x28\x73\x65\x71\x2c\x20\x22\x25\x34\x75\x20\
	256	+\x25\x2d\x31\x36\x73\x20\x25\x73\x20\x25\x73\x5c\x6e\x22\x2c\x20\x61\x75\x78\
	257	+\x2d\x3e\x69\x64\x2c\0\x30\x3a\x34\0\x30\x3a\x35\0\x09\x69\x66\x20\x28\x21\x62\
	258	+\x74\x66\x29\0\x62\x70\x66\x5f\x66\x75\x6e\x63\x5f\x69\x6e\x66\x6f\0\x69\x6e\
	259	+\x73\x6e\x5f\x6f\x66\x66\0\x74\x79\x70\x65\x5f\x69\x64\0\x30\0\x73\x74\x72\x69\
	260	+\x6e\x67\x73\0\x74\x79\x70\x65\x73\0\x68\x64\x72\0\x62\x74\x66\x5f\x68\x65\x61\
	261	+\x64\x65\x72\0\x73\x74\x72\x5f\x6c\x65\x6e\0\x09\x74\x79\x70\x65\x73\x20\x3d\
	262	+\x20\x62\x74\x66\x2d\x3e\x74\x79\x70\x65\x73\x3b\0\x09\x62\x70\x66\x5f\x70\x72\
	263	+\x6f\x62\x65\x5f\x72\x65\x61\x64\x5f\x6b\x65\x72\x6e\x65\x6c\x28\x26\x74\x2c\
	264	+\x20\x73\x69\x7a\x65\x6f\x66\x28\x74\x29\x2c\x20\x74\x79\x70\x65\x73\x20\x2b\
	265	+\x20\x62\x74\x66\x5f\x69\x64\x29\x3b\0\x09\x73\x74\x72\x20\x3d\x20\x62\x74\x66\
	266	+\x2d\x3e\x73\x74\x72\x69\x6e\x67\x73\x3b\0\x62\x74\x66\x5f\x74\x79\x70\x65\0\
	267	+\x6e\x61\x6d\x65\x5f\x6f\x66\x66\0\x09\x6e\x61\x6d\x65\x5f\x6f\x66\x66\x20\x3d\
	268	+\x20\x42\x50\x46\x5f\x43\x4f\x52\x45\x5f\x52\x45\x41\x44\x28\x74\x2c\x20\x6e\
	269	+\x61\x6d\x65\x5f\x6f\x66\x66\x29\x3b\0\x30\x3a\x32\x3a\x30\0\x09\x69\x66\x20\
	270	+\x28\x6e\x61\x6d\x65\x5f\x6f\x66\x66\x20\x3e\x3d\x20\x62\x74\x66\x2d\x3e\x68\
	271	+\x64\x72\x2e\x73\x74\x72\x5f\x6c\x65\x6e\x29\0\x09\x72\x65\x74\x75\x72\x6e\x20\
	272	+\x73\x74\x72\x20\x2b\x20\x6e\x61\x6d\x65\x5f\x6f\x66\x66\x3b\0\x30\x3a\x33\0\
	273	+\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\x61\x70\x2e\x5f\x5f\x5f\x66\x6d\x74\0\
	274	+\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\x61\x70\x2e\x5f\x5f\x5f\x66\x6d\x74\
	275	+\x2e\x31\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\x2e\x5f\x5f\x5f\
	276	+\x66\x6d\x74\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\x2e\x5f\x5f\
	277	+\x5f\x66\x6d\x74\x2e\x32\0\x4c\x49\x43\x45\x4e\x53\x45\0\x2e\x72\x6f\x64\x61\
	278	+\x74\x61\0\x6c\x69\x63\x65\x6e\x73\x65\0\x9f\xeb\x01\0\x20\0\0\0\0\0\0\0\x24\0\
	279	+\0\0\x24\0\0\0\x44\x02\0\0\x68\x02\0\0\xa4\x01\0\0\x08\0\0\0\x31\0\0\0\x01\0\0\
	280	+\0\0\0\0\0\x07\0\0\0\x56\x02\0\0\x01\0\0\0\0\0\0\0\x17\0\0\0\x10\0\0\0\x31\0\0\
	281	+\0\x09\0\0\0\0\0\0\0\x42\0\0\0\x7b\0\0\0\x1e\x40\x01\0\x08\0\0\0\x42\0\0\0\x7b\
	282	+\0\0\0\x24\x40\x01\0\x10\0\0\0\x42\0\0\0\xf2\0\0\0\x1d\x48\x01\0\x18\0\0\0\x42\
	283	+\0\0\0\x13\x01\0\0\x06\x50\x01\0\x20\0\0\0\x42\0\0\0\x22\x01\0\0\x1d\x44\x01\0\
	284	+\x28\0\0\0\x42\0\0\0\x47\x01\0\0\x06\x5c\x01\0\x38\0\0\0\x42\0\0\0\x5a\x01\0\0\
	285	+\x03\x60\x01\0\x70\0\0\0\x42\0\0\0\xe0\x01\0\0\x02\x68\x01\0\xf0\0\0\0\x42\0\0\
	286	+\0\x2e\x02\0\0\x01\x70\x01\0\x56\x02\0\0\x1a\0\0\0\0\0\0\0\x42\0\0\0\x7b\0\0\0\
	287	+\x1e\x84\x01\0\x08\0\0\0\x42\0\0\0\x7b\0\0\0\x24\x84\x01\0\x10\0\0\0\x42\0\0\0\
	288	+\x64\x02\0\0\x1f\x8c\x01\0\x18\0\0\0\x42\0\0\0\x88\x02\0\0\x06\x98\x01\0\x20\0\
	289	+\0\0\x42\0\0\0\xa1\x02\0\0\x0e\xa4\x01\0\x28\0\0\0\x42\0\0\0\x22\x01\0\0\x1d\
	290	+\x88\x01\0\x30\0\0\0\x42\0\0\0\x47\x01\0\0\x06\xa8\x01\0\x40\0\0\0\x42\0\0\0\
	291	+\xb3\x02\0\0\x03\xac\x01\0\x80\0\0\0\x42\0\0\0\x26\x03\0\0\x02\xb4\x01\0\xb8\0\
	292	+\0\0\x42\0\0\0\x61\x03\0\0\x06\x08\x01\0\xd0\0\0\0\x42\0\0\0\0\0\0\0\0\0\0\0\
	293	+\xd8\0\0\0\x42\0\0\0\xb2\x03\0\0\x0f\x14\x01\0\xe0\0\0\0\x42\0\0\0\xc7\x03\0\0\
	294	+\x2d\x18\x01\0\xf0\0\0\0\x42\0\0\0\xfe\x03\0\0\x0d\x10\x01\0\0\x01\0\0\x42\0\0\
	295	+\0\0\0\0\0\0\0\0\0\x08\x01\0\0\x42\0\0\0\xc7\x03\0\0\x02\x18\x01\0\x20\x01\0\0\
	296	+\x42\0\0\0\x25\x04\0\0\x0d\x1c\x01\0\x38\x01\0\0\x42\0\0\0\0\0\0\0\0\0\0\0\x40\
	297	+\x01\0\0\x42\0\0\0\x25\x04\0\0\x0d\x1c\x01\0\x58\x01\0\0\x42\0\0\0\x25\x04\0\0\
	298	+\x0d\x1c\x01\0\x60\x01\0\0\x42\0\0\0\x53\x04\0\0\x1b\x20\x01\0\x68\x01\0\0\x42\
	299	+\0\0\0\x53\x04\0\0\x06\x20\x01\0\x70\x01\0\0\x42\0\0\0\x76\x04\0\0\x0d\x28\x01\
	300	+\0\x78\x01\0\0\x42\0\0\0\0\0\0\0\0\0\0\0\x80\x01\0\0\x42\0\0\0\x26\x03\0\0\x02\
	301	+\xb4\x01\0\xf8\x01\0\0\x42\0\0\0\x2e\x02\0\0\x01\xc4\x01\0\x10\0\0\0\x31\0\0\0\
	302	+\x07\0\0\0\0\0\0\0\x02\0\0\0\x3e\0\0\0\0\0\0\0\x08\0\0\0\x08\0\0\0\x3e\0\0\0\0\
	303	+\0\0\0\x10\0\0\0\x02\0\0\0\xee\0\0\0\0\0\0\0\x20\0\0\0\x08\0\0\0\x1e\x01\0\0\0\
	304	+\0\0\0\x70\0\0\0\x0d\0\0\0\x3e\0\0\0\0\0\0\0\x80\0\0\0\x0d\0\0\0\xee\0\0\0\0\0\
	305	+\0\0\xa0\0\0\0\x0d\0\0\0\x1e\x01\0\0\0\0\0\0\x56\x02\0\0\x12\0\0\0\0\0\0\0\x14\
	306	+\0\0\0\x3e\0\0\0\0\0\0\0\x08\0\0\0\x08\0\0\0\x3e\0\0\0\0\0\0\0\x10\0\0\0\x14\0\
	307	+\0\0\xee\0\0\0\0\0\0\0\x20\0\0\0\x18\0\0\0\x3e\0\0\0\0\0\0\0\x28\0\0\0\x08\0\0\
	308	+\0\x1e\x01\0\0\0\0\0\0\x80\0\0\0\x1a\0\0\0\x3e\0\0\0\0\0\0\0\x90\0\0\0\x1a\0\0\
	309	+\0\xee\0\0\0\0\0\0\0\xa8\0\0\0\x1a\0\0\0\x59\x03\0\0\0\0\0\0\xb0\0\0\0\x1a\0\0\
	310	+\0\x5d\x03\0\0\0\0\0\0\xc0\0\0\0\x1f\0\0\0\x8b\x03\0\0\0\0\0\0\xd8\0\0\0\x20\0\
	311	+\0\0\xee\0\0\0\0\0\0\0\xf0\0\0\0\x20\0\0\0\x3e\0\0\0\0\0\0\0\x18\x01\0\0\x24\0\
	312	+\0\0\x3e\0\0\0\0\0\0\0\x50\x01\0\0\x1a\0\0\0\xee\0\0\0\0\0\0\0\x60\x01\0\0\x20\
	313	+\0\0\0\x4d\x04\0\0\0\0\0\0\x88\x01\0\0\x1a\0\0\0\x1e\x01\0\0\0\0\0\0\x98\x01\0\
	314	+\0\x1a\0\0\0\x8e\x04\0\0\0\0\0\0\xa0\x01\0\0\x18\0\0\0\x3e\0\0\0\0\0\0\0\0\0\0\
	315	+\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd6\0\0\0\0\0\x02\0\x70\0\0\0\0\
	316	+\0\0\0\0\0\0\0\0\0\0\0\xc8\0\0\0\0\0\x02\0\xf0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
	317	+\xcf\0\0\0\0\0\x03\0\x78\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc1\0\0\0\0\0\x03\0\x80\
	318	+\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xba\0\0\0\0\0\x03\0\xf8\x01\0\0\0\0\0\0\0\0\0\
	319	+\0\0\0\0\0\x14\0\0\0\x01\0\x04\0\0\0\0\0\0\0\0\0\x23\0\0\0\0\0\0\0\xf4\0\0\0\
	320	+\x01\0\x04\0\x23\0\0\0\0\0\0\0\x0e\0\0\0\0\0\0\0\x28\0\0\0\x01\0\x04\0\x31\0\0\
	321	+\0\0\0\0\0\x20\0\0\0\0\0\0\0\xdd\0\0\0\x01\0\x04\0\x51\0\0\0\0\0\0\0\x11\0\0\0\
	322	+\0\0\0\0\0\0\0\0\x03\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\x03\
	323	+\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\
	324	+\0\0\0\0\xb2\0\0\0\x11\0\x05\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\x3d\0\0\0\x12\
	325	+\0\x02\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\x5b\0\0\0\x12\0\x03\0\0\0\0\0\0\0\0\
	326	+\0\x08\x02\0\0\0\0\0\0\x48\0\0\0\0\0\0\0\x01\0\0\0\x0c\0\0\0\xc8\0\0\0\0\0\0\0\
	327	+\x01\0\0\0\x0c\0\0\0\x50\0\0\0\0\0\0\0\x01\0\0\0\x0c\0\0\0\xd0\x01\0\0\0\0\0\0\
	328	+\x01\0\0\0\x0c\0\0\0\xf0\x03\0\0\0\0\0\0\x0a\0\0\0\x0c\0\0\0\xfc\x03\0\0\0\0\0\
	329	+\0\x0a\0\0\0\x0c\0\0\0\x08\x04\0\0\0\0\0\0\x0a\0\0\0\x0c\0\0\0\x14\x04\0\0\0\0\
	330	+\0\0\x0a\0\0\0\x0c\0\0\0\x2c\x04\0\0\0\0\0\0\0\0\0\0\x0d\0\0\0\x2c\0\0\0\0\0\0\
	331	+\0\0\0\0\0\x0a\0\0\0\x3c\0\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x50\0\0\0\0\0\0\0\0\0\
	332	+\0\0\x0a\0\0\0\x60\0\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\x70\0\0\0\0\0\0\0\0\0\0\0\
	333	+\x0a\0\0\0\x80\0\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\x90\0\0\0\0\0\0\0\0\0\0\0\x0a\0\
	334	+\0\0\xa0\0\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\xb0\0\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\
	335	+\xc0\0\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\xd0\0\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\xe8\0\
	336	+\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xf8\0\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x08\x01\0\0\
	337	+\0\0\0\0\0\0\0\0\x0b\0\0\0\x18\x01\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x28\x01\0\0\0\
	338	+\0\0\0\0\0\0\0\x0b\0\0\0\x38\x01\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x48\x01\0\0\0\0\
	339	+\0\0\0\0\0\0\x0b\0\0\0\x58\x01\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x68\x01\0\0\0\0\0\
	340	+\0\0\0\0\0\x0b\0\0\0\x78\x01\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x88\x01\0\0\0\0\0\0\
	341	+\0\0\0\0\x0b\0\0\0\x98\x01\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xa8\x01\0\0\0\0\0\0\0\
	342	+\0\0\0\x0b\0\0\0\xb8\x01\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xc8\x01\0\0\0\0\0\0\0\0\
	343	+\0\0\x0b\0\0\0\xd8\x01\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xe8\x01\0\0\0\0\0\0\0\0\0\
	344	+\0\x0b\0\0\0\xf8\x01\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x08\x02\0\0\0\0\0\0\0\0\0\0\
	345	+\x0b\0\0\0\x18\x02\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x28\x02\0\0\0\0\0\0\0\0\0\0\
	346	+\x0b\0\0\0\x38\x02\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x48\x02\0\0\0\0\0\0\0\0\0\0\
	347	+\x0b\0\0\0\x58\x02\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x68\x02\0\0\0\0\0\0\0\0\0\0\
	348	+\x0b\0\0\0\x78\x02\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x94\x02\0\0\0\0\0\0\0\0\0\0\
	349	+\x0a\0\0\0\xa4\x02\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\xb4\x02\0\0\0\0\0\0\0\0\0\0\
	350	+\x0a\0\0\0\xc4\x02\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\xd4\x02\0\0\0\0\0\0\0\0\0\0\
	351	+\x0a\0\0\0\xe4\x02\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\xf4\x02\0\0\0\0\0\0\0\0\0\0\
	352	+\x0a\0\0\0\x0c\x03\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x1c\x03\0\0\0\0\0\0\0\0\0\0\
	353	+\x0b\0\0\0\x2c\x03\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x3c\x03\0\0\0\0\0\0\0\0\0\0\
	354	+\x0b\0\0\0\x4c\x03\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x5c\x03\0\0\0\0\0\0\0\0\0\0\
	355	+\x0b\0\0\0\x6c\x03\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x7c\x03\0\0\0\0\0\0\0\0\0\0\
	356	+\x0b\0\0\0\x8c\x03\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x9c\x03\0\0\0\0\0\0\0\0\0\0\
	357	+\x0b\0\0\0\xac\x03\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xbc\x03\0\0\0\0\0\0\0\0\0\0\
	358	+\x0b\0\0\0\xcc\x03\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xdc\x03\0\0\0\0\0\0\0\0\0\0\
	359	+\x0b\0\0\0\xec\x03\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xfc\x03\0\0\0\0\0\0\0\0\0\0\
	360	+\x0b\0\0\0\x0c\x04\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x1c\x04\0\0\0\0\0\0\0\0\0\0\
	361	+\x0b\0\0\0\x4e\x4f\x41\x42\x43\x44\x4d\0\x2e\x74\x65\x78\x74\0\x2e\x72\x65\x6c\
	362	+\x2e\x42\x54\x46\x2e\x65\x78\x74\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\x61\
	363	+\x70\x2e\x5f\x5f\x5f\x66\x6d\x74\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\
	364	+\x6f\x67\x2e\x5f\x5f\x5f\x66\x6d\x74\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\
	365	+\x61\x70\0\x2e\x72\x65\x6c\x69\x74\x65\x72\x2f\x62\x70\x66\x5f\x6d\x61\x70\0\
	366	+\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\0\x2e\x72\x65\x6c\x69\x74\
	367	+\x65\x72\x2f\x62\x70\x66\x5f\x70\x72\x6f\x67\0\x2e\x6c\x6c\x76\x6d\x5f\x61\x64\
	368	+\x64\x72\x73\x69\x67\0\x6c\x69\x63\x65\x6e\x73\x65\0\x2e\x73\x74\x72\x74\x61\
	369	+\x62\0\x2e\x73\x79\x6d\x74\x61\x62\0\x2e\x72\x6f\x64\x61\x74\x61\0\x2e\x72\x65\
	370	+\x6c\x2e\x42\x54\x46\0\x4c\x49\x43\x45\x4e\x53\x45\0\x4c\x42\x42\x31\x5f\x37\0\
	371	+\x4c\x42\x42\x31\x5f\x36\0\x4c\x42\x42\x30\x5f\x34\0\x4c\x42\x42\x31\x5f\x33\0\
	372	+\x4c\x42\x42\x30\x5f\x33\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\
	373	+\x2e\x5f\x5f\x5f\x66\x6d\x74\x2e\x32\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\
	374	+\x61\x70\x2e\x5f\x5f\x5f\x66\x6d\x74\x2e\x31\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
	375	+\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
	376	+\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\0\
	377	+\0\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\0\
	378	+\0\0\0\0\0\0\0\x4e\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\
	379	+\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
	380	+\x6d\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x40\x01\0\0\0\0\0\0\x08\
	381	+\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa1\0\0\0\
	382	+\x01\0\0\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x48\x03\0\0\0\0\0\0\x62\0\0\0\0\0\
	383	+\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x89\0\0\0\x01\0\0\0\x03\
	384	+\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xaa\x03\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\
	385	+\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xad\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\
	386	+\0\0\0\0\0\0\0\xae\x03\0\0\0\0\0\0\x34\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\
	387	+\0\0\0\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
	388	+\xe2\x0c\0\0\0\0\0\0\x2c\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\
	389	+\0\0\0\0\0\0\x99\0\0\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x10\x11\0\0\0\
	390	+\0\0\0\x80\x01\0\0\0\0\0\0\x0e\0\0\0\x0d\0\0\0\x08\0\0\0\0\0\0\0\x18\0\0\0\0\0\
	391	+\0\0\x4a\0\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x90\x12\0\0\0\0\0\0\
	392	+\x20\0\0\0\0\0\0\0\x08\0\0\0\x02\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x69\
	393	+\0\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb0\x12\0\0\0\0\0\0\x20\0\0\0\
	394	+\0\0\0\0\x08\0\0\0\x03\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\xa9\0\0\0\x09\
	395	+\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd0\x12\0\0\0\0\0\0\x50\0\0\0\0\0\0\0\
	396	+\x08\0\0\0\x06\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x07\0\0\0\x09\0\0\0\0\
	397	+\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x20\x13\0\0\0\0\0\0\xe0\x03\0\0\0\0\0\0\x08\0\0\
	398	+\0\x07\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x7b\0\0\0\x03\x4c\xff\x6f\0\0\
	399	+\0\x80\0\0\0\0\0\0\0\0\0\0\0\0\0\x17\0\0\0\0\0\0\x07\0\0\0\0\0\0\0\0\0\0\0\0\0\
	400	+\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x91\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\
	401	+\0\0\0\0\0\0\x07\x17\0\0\0\0\0\0\x0a\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\
	402	+\0\0\0\0\0\0\0\0\0\0\0\0";
	403	+
	404	+ return 0;
	405	+err:
	406	+ bpf_object__destroy_skeleton(s);
	407	+ return -1;
	408	+}
	409	+
	410	+#endif /* __ITERATORS_BPF_SKEL_H__ */

kernel/bpf/queue_stack_maps.c+2−0

@@ -257,6 +257,7 @@ static int queue_stack_map_get_next_key(struct bpf_map map, void key, · 256 unmodified lines
257	257
258	258	static int queue_map_btf_id;
259	259	const struct bpf_map_ops queue_map_ops = {
	260	+ .map_meta_equal = bpf_map_meta_equal,
260	261	.map_alloc_check = queue_stack_map_alloc_check,
261	262	.map_alloc = queue_stack_map_alloc,
262	263	.map_free = queue_stack_map_free,
@@ -273,6 +274,7 @@ const struct bpf_map_ops queue_map_ops = { · 10 unmodified lines
273	274
274	275	static int stack_map_btf_id;
275	276	const struct bpf_map_ops stack_map_ops = {
	277	+ .map_meta_equal = bpf_map_meta_equal,
276	278	.map_alloc_check = queue_stack_map_alloc_check,
277	279	.map_alloc = queue_stack_map_alloc,
278	280	.map_free = queue_stack_map_free,

kernel/bpf/reuseport_array.c+1−0

@@ -351,6 +351,7 @@ static int reuseport_array_get_next_key(struct bpf_map map, void key, · 350 unmodified lines
351	351
352	352	static int reuseport_array_map_btf_id;
353	353	const struct bpf_map_ops reuseport_array_ops = {
	354	+ .map_meta_equal = bpf_map_meta_equal,
354	355	.map_alloc_check = reuseport_array_alloc_check,
355	356	.map_alloc = reuseport_array_alloc,
356	357	.map_free = reuseport_array_free,

kernel/bpf/ringbuf.c+1−0

@@ -287,6 +287,7 @@ static __poll_t ringbuf_map_poll(struct bpf_map map, struct file filp, · 286 unmodified lines
287	287
288	288	static int ringbuf_map_btf_id;
289	289	const struct bpf_map_ops ringbuf_map_ops = {
	290	+ .map_meta_equal = bpf_map_meta_equal,
290	291	.map_alloc = ringbuf_map_alloc,
291	292	.map_free = ringbuf_map_free,
292	293	.map_mmap = ringbuf_map_mmap,

kernel/bpf/stackmap.c+1−0

@@ -839,6 +839,7 @@ static void stack_map_free(struct bpf_map *map) · 838 unmodified lines
839	839
840	840	static int stack_trace_map_btf_id;
841	841	const struct bpf_map_ops stack_trace_map_ops = {
	842	+ .map_meta_equal = bpf_map_meta_equal,
842	843	.map_alloc = stack_map_alloc,
843	844	.map_free = stack_map_free,
844	845	.map_get_next_key = stack_map_get_next_key,

kernel/bpf/syscall.c+44−24

@@ -29,6 +29,7 @@ · 28 unmodified lines
29	29	#include <linux/bpf_lsm.h>
30	30	#include <linux/poll.h>
31	31	#include <linux/bpf-netns.h>
	32	+#include <linux/rcupdate_trace.h>
32	33
33	34	#define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY \|\| \
34	35	(map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY \|\| \
@@ -90,6 +91,7 @@ int bpf_check_uarg_tail_zero(void __user *uaddr, · 55 unmodified lines
90	91	}
91	92
92	93	const struct bpf_map_ops bpf_map_offload_ops = {
	94	+ .map_meta_equal = bpf_map_meta_equal,
93	95	.map_alloc = bpf_map_offload_map_alloc,
94	96	.map_free = bpf_map_offload_map_free,
95	97	.map_check_btf = map_check_no_btf,
@@ -157,10 +159,11 @@ static int bpf_map_update_value(struct bpf_map map, struct fd f, void key, · 61 unmodified lines
157	159	if (bpf_map_is_dev_bound(map)) {
158	160	return bpf_map_offload_update_elem(map, key, value, flags);
159	161	} else if (map->map_type == BPF_MAP_TYPE_CPUMAP \|\|
160		− map->map_type == BPF_MAP_TYPE_SOCKHASH \|\|
161		− map->map_type == BPF_MAP_TYPE_SOCKMAP \|\|
162	162	map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
163	163	return map->ops->map_update_elem(map, key, value, flags);
	164	+ } else if (map->map_type == BPF_MAP_TYPE_SOCKHASH \|\|
	165	+ map->map_type == BPF_MAP_TYPE_SOCKMAP) {
	166	+ return sock_map_update_elem_sys(map, key, value, flags);
164	167	} else if (IS_FD_PROG_ARRAY(map)) {
165	168	return bpf_fd_array_map_update_elem(map, f.file, key, value,
166	169	flags);
@@ -768,7 +771,8 @@ static int map_check_btf(struct bpf_map map, const struct btf btf, · 601 unmodified lines
768	771	if (map->map_type != BPF_MAP_TYPE_HASH &&
769	772	map->map_type != BPF_MAP_TYPE_ARRAY &&
770	773	map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE &&
771		− map->map_type != BPF_MAP_TYPE_SK_STORAGE)
	774	+ map->map_type != BPF_MAP_TYPE_SK_STORAGE &&
	775	+ map->map_type != BPF_MAP_TYPE_INODE_STORAGE)
772	776	return -ENOTSUPP;
773	777	if (map->spin_lock_off + sizeof(struct bpf_spin_lock) >
774	778	map->value_size) {
@@ -1728,10 +1732,14 @@ static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) · 953 unmodified lines
1728	1732	btf_put(prog->aux->btf);
1729	1733	bpf_prog_free_linfo(prog);
1730	1734
1731		− if (deferred)
1732		− call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);
1733		− else
	1735	+ if (deferred) {
	1736	+ if (prog->aux->sleepable)
	1737	+ call_rcu_tasks_trace(&prog->aux->rcu, __bpf_prog_put_rcu);
	1738	+ else
	1739	+ call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);
	1740	+ } else {
1734	1741	__bpf_prog_put_rcu(&prog->aux->rcu);
	1742	+ }
1735	1743	}
1736	1744
1737	1745	static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock)
@@ -2101,6 +2109,7 @@ static int bpf_prog_load(union bpf_attr attr, union bpf_attr __user uattr) · 363 unmodified lines
2101	2109	if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT \|
2102	2110	BPF_F_ANY_ALIGNMENT \|
2103	2111	BPF_F_TEST_STATE_FREQ \|
	2112	+ BPF_F_SLEEPABLE \|
2104	2113	BPF_F_TEST_RND_HI32))
2105	2114	return -EINVAL;
2106	2115
@@ -2156,6 +2165,7 @@ static int bpf_prog_load(union bpf_attr attr, union bpf_attr __user uattr) · 49 unmodified lines
2156	2165	}
2157	2166
2158	2167	prog->aux->offload_requested = !!attr->prog_ifindex;
	2168	+ prog->aux->sleepable = attr->prog_flags & BPF_F_SLEEPABLE;
2159	2169
2160	2170	err = security_bpf_prog_alloc(prog->aux);
2161	2171	if (err)
@@ -4014,40 +4024,50 @@ static int link_detach(union bpf_attr *attr) · 1852 unmodified lines
4014	4024	return ret;
4015	4025	}
4016	4026
4017		−static int bpf_link_inc_not_zero(struct bpf_link *link)
	4027	+static struct bpf_link bpf_link_inc_not_zero(struct bpf_link link)
4018	4028	{
4019		− return atomic64_fetch_add_unless(&link->refcnt, 1, 0) ? 0 : -ENOENT;
	4029	+ return atomic64_fetch_add_unless(&link->refcnt, 1, 0) ? link : ERR_PTR(-ENOENT);
4020	4030	}
4021	4031
4022		−#define BPF_LINK_GET_FD_BY_ID_LAST_FIELD link_id
4023		−
4024		−static int bpf_link_get_fd_by_id(const union bpf_attr *attr)
	4032	+struct bpf_link *bpf_link_by_id(u32 id)
4025	4033	{
4026	4034	struct bpf_link *link;
4027		− u32 id = attr->link_id;
4028		− int fd, err;
4029	4035
4030		− if (CHECK_ATTR(BPF_LINK_GET_FD_BY_ID))
4031		− return -EINVAL;
4032		−
4033		− if (!capable(CAP_SYS_ADMIN))
4034		− return -EPERM;
	4036	+ if (!id)
	4037	+ return ERR_PTR(-ENOENT);
4035	4038
4036	4039	spin_lock_bh(&link_idr_lock);
4037		− link = idr_find(&link_idr, id);
4038	4040	/* before link is "settled", ID is 0, pretend it doesn't exist yet */
	4041	+ link = idr_find(&link_idr, id);
4039	4042	if (link) {
4040	4043	if (link->id)
4041		− err = bpf_link_inc_not_zero(link);
	4044	+ link = bpf_link_inc_not_zero(link);
4042	4045	else
4043		− err = -EAGAIN;
	4046	+ link = ERR_PTR(-EAGAIN);
4044	4047	} else {
4045		− err = -ENOENT;
	4048	+ link = ERR_PTR(-ENOENT);
4046	4049	}
4047	4050	spin_unlock_bh(&link_idr_lock);
	4051	+ return link;
	4052	+}
4048	4053
4049		− if (err)
4050		− return err;
	4054	+#define BPF_LINK_GET_FD_BY_ID_LAST_FIELD link_id
	4055	+
	4056	+static int bpf_link_get_fd_by_id(const union bpf_attr *attr)
	4057	+{
	4058	+ struct bpf_link *link;
	4059	+ u32 id = attr->link_id;
	4060	+ int fd;
	4061	+
	4062	+ if (CHECK_ATTR(BPF_LINK_GET_FD_BY_ID))
	4063	+ return -EINVAL;
	4064	+
	4065	+ if (!capable(CAP_SYS_ADMIN))
	4066	+ return -EPERM;
	4067	+
	4068	+ link = bpf_link_by_id(id);
	4069	+ if (IS_ERR(link))
	4070	+ return PTR_ERR(link);
4051	4071
4052	4072	fd = bpf_link_new_fd(link);
4053	4073	if (fd < 0)

kernel/bpf/trampoline.c+26−3

@@ -7,6 +7,8 @@ · 6 unmodified lines
7	7	#include <linux/rbtree_latch.h>
8	8	#include <linux/perf_event.h>
9	9	#include <linux/btf.h>
	10	+#include <linux/rcupdate_trace.h>
	11	+#include <linux/rcupdate_wait.h>
10	12
11	13	/* dummy _ops. The verifier will operate on target program's ops. */
12	14	const struct bpf_verifier_ops bpf_extension_verifier_ops = {
@@ -210,9 +212,12 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr) · 197 unmodified lines
210	212	* updates to trampoline would change the code from underneath the
211	213	* preempted task. Hence wait for tasks to voluntarily schedule or go
212	214	* to userspace.
	215	+ * The same trampoline can hold both sleepable and non-sleepable progs.
	216	+ * synchronize_rcu_tasks_trace() is needed to make sure all sleepable
	217	+ * programs finish executing.
	218	+ * Wait for these two grace periods together.
213	219	*/
214		−
215		− synchronize_rcu_tasks();
	220	+ synchronize_rcu_mult(call_rcu_tasks, call_rcu_tasks_trace);
216	221
217	222	err = arch_prepare_bpf_trampoline(new_image, new_image + PAGE_SIZE / 2,
218	223	&tr->func.model, flags, tprogs,
@@ -344,7 +349,14 @@ void bpf_trampoline_put(struct bpf_trampoline *tr) · 125 unmodified lines
344	349	if (WARN_ON_ONCE(!hlist_empty(&tr->progs_hlist[BPF_TRAMP_FEXIT])))
345	350	goto out;
346	351	bpf_image_ksym_del(&tr->ksym);
347		− /* wait for tasks to get out of trampoline before freeing it */
	352	+ /* This code will be executed when all bpf progs (both sleepable and
	353	+ * non-sleepable) went through
	354	+ * bpf_prog_put()->call_rcu[_tasks_trace]()->bpf_prog_free_deferred().
	355	+ * Hence no need for another synchronize_rcu_tasks_trace() here,
	356	+ * but synchronize_rcu_tasks() is still needed, since trampoline
	357	+ * may not have had any sleepable programs and we need to wait
	358	+ * for tasks to get out of trampoline code before freeing it.
	359	+ */
348	360	synchronize_rcu_tasks();
349	361	bpf_jit_free_exec(tr->image);
350	362	hlist_del(&tr->hlist);
@@ -394,6 +406,17 @@ void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start) · 43 unmodified lines
394	406	rcu_read_unlock();
395	407	}
396	408
	409	+void notrace __bpf_prog_enter_sleepable(void)
	410	+{
	411	+ rcu_read_lock_trace();
	412	+ might_fault();
	413	+}
	414	+
	415	+void notrace __bpf_prog_exit_sleepable(void)
	416	+{
	417	+ rcu_read_unlock_trace();
	418	+}
	419	+
397	420	int __weak
398	421	arch_prepare_bpf_trampoline(void image, void image_end,
399	422	const struct btf_func_model *m, u32 flags,

kernel/bpf/verifier.c+263−20

@@ -21,6 +21,7 @@ · 20 unmodified lines
21	21	#include <linux/ctype.h>
22	22	#include <linux/error-injection.h>
23	23	#include <linux/bpf_lsm.h>
	24	+#include <linux/btf_ids.h>
24	25
25	26	#include "disasm.h"
26	27
@@ -2625,11 +2626,19 @@ static int check_map_access(struct bpf_verifier_env *env, u32 regno, · 2598 unmodified lines
2625	2626
2626	2627	#define MAX_PACKET_OFF 0xffff
2627	2628
	2629	+static enum bpf_prog_type resolve_prog_type(struct bpf_prog *prog)
	2630	+{
	2631	+ return prog->aux->linked_prog ? prog->aux->linked_prog->type
	2632	+ : prog->type;
	2633	+}
	2634	+
2628	2635	static bool may_access_direct_pkt_data(struct bpf_verifier_env *env,
2629	2636	const struct bpf_call_arg_meta *meta,
2630	2637	enum bpf_access_type t)
2631	2638	{
2632		− switch (env->prog->type) {
	2639	+ enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
	2640	+
	2641	+ switch (prog_type) {
2633	2642	/* Program types only with direct read access go here! */
2634	2643	case BPF_PROG_TYPE_LWT_IN:
2635	2644	case BPF_PROG_TYPE_LWT_OUT:
@@ -3872,6 +3881,33 @@ static int int_ptr_type_to_size(enum bpf_arg_type type) · 1236 unmodified lines
3872	3881	return -EINVAL;
3873	3882	}
3874	3883
	3884	+static int resolve_map_arg_type(struct bpf_verifier_env *env,
	3885	+ const struct bpf_call_arg_meta *meta,
	3886	+ enum bpf_arg_type *arg_type)
	3887	+{
	3888	+ if (!meta->map_ptr) {
	3889	+ /* kernel subsystem misconfigured verifier */
	3890	+ verbose(env, "invalid map_ptr to access map->type\n");
	3891	+ return -EACCES;
	3892	+ }
	3893	+
	3894	+ switch (meta->map_ptr->map_type) {
	3895	+ case BPF_MAP_TYPE_SOCKMAP:
	3896	+ case BPF_MAP_TYPE_SOCKHASH:
	3897	+ if (*arg_type == ARG_PTR_TO_MAP_VALUE) {
	3898	+ *arg_type = ARG_PTR_TO_SOCKET;
	3899	+ } else {
	3900	+ verbose(env, "invalid arg_type for sockmap/sockhash\n");
	3901	+ return -EINVAL;
	3902	+ }
	3903	+ break;
	3904	+
	3905	+ default:
	3906	+ break;
	3907	+ }
	3908	+ return 0;
	3909	+}
	3910	+
3875	3911	static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
3876	3912	struct bpf_call_arg_meta *meta,
3877	3913	const struct bpf_func_proto *fn)
@@ -3904,6 +3940,14 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, · 26 unmodified lines
3904	3940	return -EACCES;
3905	3941	}
3906	3942
	3943	+ if (arg_type == ARG_PTR_TO_MAP_VALUE \|\|
	3944	+ arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE \|\|
	3945	+ arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL) {
	3946	+ err = resolve_map_arg_type(env, meta, &arg_type);
	3947	+ if (err)
	3948	+ return err;
	3949	+ }
	3950	+
3907	3951	if (arg_type == ARG_PTR_TO_MAP_KEY \|\|
3908	3952	arg_type == ARG_PTR_TO_MAP_VALUE \|\|
3909	3953	arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE \|\|
@@ -3960,16 +4004,21 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, · 50 unmodified lines
3960	4004	goto err_type;
3961	4005	}
3962	4006	} else if (arg_type == ARG_PTR_TO_BTF_ID) {
	4007	+ bool ids_match = false;
	4008	+
3963	4009	expected_type = PTR_TO_BTF_ID;
3964	4010	if (type != expected_type)
3965	4011	goto err_type;
3966	4012	if (!fn->check_btf_id) {
3967	4013	if (reg->btf_id != meta->btf_id) {
3968		− verbose(env, "Helper has type %s got %s in R%d\n",
3969		− kernel_type_name(meta->btf_id),
3970		− kernel_type_name(reg->btf_id), regno);
3971		−
3972		− return -EACCES;
	4014	+ ids_match = btf_struct_ids_match(&env->log, reg->off, reg->btf_id,
	4015	+ meta->btf_id);
	4016	+ if (!ids_match) {
	4017	+ verbose(env, "Helper has type %s got %s in R%d\n",
	4018	+ kernel_type_name(meta->btf_id),
	4019	+ kernel_type_name(reg->btf_id), regno);
	4020	+ return -EACCES;
	4021	+ }
3973	4022	}
3974	4023	} else if (!fn->check_btf_id(reg->btf_id, arg)) {
3975	4024	verbose(env, "Helper does not support %s in R%d\n",
@@ -3977,7 +4026,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, · 1 unmodified line
3977	4026
3978	4027	return -EACCES;
3979	4028	}
3980		− if (!tnum_is_const(reg->var_off) \|\| reg->var_off.value \|\| reg->off) {
	4029	+ if ((reg->off && !ids_match) \|\| !tnum_is_const(reg->var_off) \|\| reg->var_off.value) {
3981	4030	verbose(env, "R%d is a pointer to in-kernel struct with non-zero offset\n",
3982	4031	regno);
3983	4032	return -EACCES;
@@ -4143,6 +4192,38 @@ err_type: · 159 unmodified lines
4143	4192	return -EACCES;
4144	4193	}
4145	4194
	4195	+static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id)
	4196	+{
	4197	+ enum bpf_attach_type eatype = env->prog->expected_attach_type;
	4198	+ enum bpf_prog_type type = resolve_prog_type(env->prog);
	4199	+
	4200	+ if (func_id != BPF_FUNC_map_update_elem)
	4201	+ return false;
	4202	+
	4203	+ /* It's not possible to get access to a locked struct sock in these
	4204	+ * contexts, so updating is safe.
	4205	+ */
	4206	+ switch (type) {
	4207	+ case BPF_PROG_TYPE_TRACING:
	4208	+ if (eatype == BPF_TRACE_ITER)
	4209	+ return true;
	4210	+ break;
	4211	+ case BPF_PROG_TYPE_SOCKET_FILTER:
	4212	+ case BPF_PROG_TYPE_SCHED_CLS:
	4213	+ case BPF_PROG_TYPE_SCHED_ACT:
	4214	+ case BPF_PROG_TYPE_XDP:
	4215	+ case BPF_PROG_TYPE_SK_REUSEPORT:
	4216	+ case BPF_PROG_TYPE_FLOW_DISSECTOR:
	4217	+ case BPF_PROG_TYPE_SK_LOOKUP:
	4218	+ return true;
	4219	+ default:
	4220	+ break;
	4221	+ }
	4222	+
	4223	+ verbose(env, "cannot update sockmap in this context\n");
	4224	+ return false;
	4225	+}
	4226	+
4146	4227	static int check_map_func_compatibility(struct bpf_verifier_env *env,
4147	4228	struct bpf_map *map, int func_id)
4148	4229	{
@@ -4214,7 +4295,8 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env, · 65 unmodified lines
4214	4295	func_id != BPF_FUNC_map_delete_elem &&
4215	4296	func_id != BPF_FUNC_msg_redirect_map &&
4216	4297	func_id != BPF_FUNC_sk_select_reuseport &&
4217		− func_id != BPF_FUNC_map_lookup_elem)
	4298	+ func_id != BPF_FUNC_map_lookup_elem &&
	4299	+ !may_update_sockmap(env, func_id))
4218	4300	goto error;
4219	4301	break;
4220	4302	case BPF_MAP_TYPE_SOCKHASH:
@@ -4223,7 +4305,8 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env, · 2 unmodified lines
4223	4305	func_id != BPF_FUNC_map_delete_elem &&
4224	4306	func_id != BPF_FUNC_msg_redirect_hash &&
4225	4307	func_id != BPF_FUNC_sk_select_reuseport &&
4226		− func_id != BPF_FUNC_map_lookup_elem)
	4308	+ func_id != BPF_FUNC_map_lookup_elem &&
	4309	+ !may_update_sockmap(env, func_id))
4227	4310	goto error;
4228	4311	break;
4229	4312	case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY:
@@ -4242,6 +4325,11 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env, · 12 unmodified lines
4242	4325	func_id != BPF_FUNC_sk_storage_delete)
4243	4326	goto error;
4244	4327	break;
	4328	+ case BPF_MAP_TYPE_INODE_STORAGE:
	4329	+ if (func_id != BPF_FUNC_inode_storage_get &&
	4330	+ func_id != BPF_FUNC_inode_storage_delete)
	4331	+ goto error;
	4332	+ break;
4245	4333	default:
4246	4334	break;
4247	4335	}
@@ -4315,6 +4403,11 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env, · 67 unmodified lines
4315	4403	if (map->map_type != BPF_MAP_TYPE_SK_STORAGE)
4316	4404	goto error;
4317	4405	break;
	4406	+ case BPF_FUNC_inode_storage_get:
	4407	+ case BPF_FUNC_inode_storage_delete:
	4408	+ if (map->map_type != BPF_MAP_TYPE_INODE_STORAGE)
	4409	+ goto error;
	4410	+ break;
4318	4411	default:
4319	4412	break;
4320	4413	}
@@ -4775,6 +4868,11 @@ static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn · 454 unmodified lines
4775	4868	return -EINVAL;
4776	4869	}
4777	4870
	4871	+ if (fn->allowed && !fn->allowed(env->prog)) {
	4872	+ verbose(env, "helper call is not allowed in probe\n");
	4873	+ return -EINVAL;
	4874	+ }
	4875	+
4778	4876	/* With LD_ABS/IND some JITs save/restore skb from r1. */
4779	4877	changes_data = bpf_helper_changes_pkt_data(fn->func);
4780	4878	if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) {
@@ -5732,6 +5830,67 @@ static void scalar_min_max_or(struct bpf_reg_state *dst_reg, · 951 unmodified lines
5732	5830	__update_reg_bounds(dst_reg);
5733	5831	}
5734	5832
	5833	+static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg,
	5834	+ struct bpf_reg_state *src_reg)
	5835	+{
	5836	+ bool src_known = tnum_subreg_is_const(src_reg->var_off);
	5837	+ bool dst_known = tnum_subreg_is_const(dst_reg->var_off);
	5838	+ struct tnum var32_off = tnum_subreg(dst_reg->var_off);
	5839	+ s32 smin_val = src_reg->s32_min_value;
	5840	+
	5841	+ /* Assuming scalar64_min_max_xor will be called so it is safe
	5842	+ * to skip updating register for known case.
	5843	+ */
	5844	+ if (src_known && dst_known)
	5845	+ return;
	5846	+
	5847	+ /* We get both minimum and maximum from the var32_off. */
	5848	+ dst_reg->u32_min_value = var32_off.value;
	5849	+ dst_reg->u32_max_value = var32_off.value \| var32_off.mask;
	5850	+
	5851	+ if (dst_reg->s32_min_value >= 0 && smin_val >= 0) {
	5852	+ /* XORing two positive sign numbers gives a positive,
	5853	+ * so safe to cast u32 result into s32.
	5854	+ */
	5855	+ dst_reg->s32_min_value = dst_reg->u32_min_value;
	5856	+ dst_reg->s32_max_value = dst_reg->u32_max_value;
	5857	+ } else {
	5858	+ dst_reg->s32_min_value = S32_MIN;
	5859	+ dst_reg->s32_max_value = S32_MAX;
	5860	+ }
	5861	+}
	5862	+
	5863	+static void scalar_min_max_xor(struct bpf_reg_state *dst_reg,
	5864	+ struct bpf_reg_state *src_reg)
	5865	+{
	5866	+ bool src_known = tnum_is_const(src_reg->var_off);
	5867	+ bool dst_known = tnum_is_const(dst_reg->var_off);
	5868	+ s64 smin_val = src_reg->smin_value;
	5869	+
	5870	+ if (src_known && dst_known) {
	5871	+ /* dst_reg->var_off.value has been updated earlier */
	5872	+ __mark_reg_known(dst_reg, dst_reg->var_off.value);
	5873	+ return;
	5874	+ }
	5875	+
	5876	+ /* We get both minimum and maximum from the var_off. */
	5877	+ dst_reg->umin_value = dst_reg->var_off.value;
	5878	+ dst_reg->umax_value = dst_reg->var_off.value \| dst_reg->var_off.mask;
	5879	+
	5880	+ if (dst_reg->smin_value >= 0 && smin_val >= 0) {
	5881	+ /* XORing two positive sign numbers gives a positive,
	5882	+ * so safe to cast u64 result into s64.
	5883	+ */
	5884	+ dst_reg->smin_value = dst_reg->umin_value;
	5885	+ dst_reg->smax_value = dst_reg->umax_value;
	5886	+ } else {
	5887	+ dst_reg->smin_value = S64_MIN;
	5888	+ dst_reg->smax_value = S64_MAX;
	5889	+ }
	5890	+
	5891	+ __update_reg_bounds(dst_reg);
	5892	+}
	5893	+
5735	5894	static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg,
5736	5895	u64 umin_val, u64 umax_val)
5737	5896	{
@@ -6040,6 +6199,11 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, · 302 unmodified lines
6040	6199	scalar32_min_max_or(dst_reg, &src_reg);
6041	6200	scalar_min_max_or(dst_reg, &src_reg);
6042	6201	break;
	6202	+ case BPF_XOR:
	6203	+ dst_reg->var_off = tnum_xor(dst_reg->var_off, src_reg.var_off);
	6204	+ scalar32_min_max_xor(dst_reg, &src_reg);
	6205	+ scalar_min_max_xor(dst_reg, &src_reg);
	6206	+ break;
6043	6207	case BPF_LSH:
6044	6208	if (umax_val >= insn_bitness) {
6045	6209	/* Shifts greater than 31 or 63 are undefined.
@@ -7287,7 +7451,7 @@ static int check_ld_abs(struct bpf_verifier_env env, struct bpf_insn insn) · 1241 unmodified lines
7287	7451	u8 mode = BPF_MODE(insn->code);
7288	7452	int i, err;
7289	7453
7290		− if (!may_access_skb(env->prog->type)) {
	7454	+ if (!may_access_skb(resolve_prog_type(env->prog))) {
7291	7455	verbose(env, "BPF_LD_[ABS\|IND] instructions not allowed for this program type\n");
7292	7456	return -EINVAL;
7293	7457	}
@@ -7375,11 +7539,12 @@ static int check_return_code(struct bpf_verifier_env *env) · 81 unmodified lines
7375	7539	const struct bpf_prog *prog = env->prog;
7376	7540	struct bpf_reg_state *reg;
7377	7541	struct tnum range = tnum_range(0, 1);
	7542	+ enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
7378	7543	int err;
7379	7544
7380	7545	/* LSM and struct_ops func-ptr's return type could be "void" */
7381		− if ((env->prog->type == BPF_PROG_TYPE_STRUCT_OPS \|\|
7382		− env->prog->type == BPF_PROG_TYPE_LSM) &&
	7546	+ if ((prog_type == BPF_PROG_TYPE_STRUCT_OPS \|\|
	7547	+ prog_type == BPF_PROG_TYPE_LSM) &&
7383	7548	!prog->aux->attach_func_proto->type)
7384	7549	return 0;
7385	7550
@@ -7398,7 +7563,7 @@ static int check_return_code(struct bpf_verifier_env *env) · 12 unmodified lines
7398	7563	return -EACCES;
7399	7564	}
7400	7565
7401		− switch (env->prog->type) {
	7566	+ switch (prog_type) {
7402	7567	case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
7403	7568	if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG \|\|
7404	7569	env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG \|\|
@@ -9154,6 +9319,7 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, · 1749 unmodified lines
9154	9319	struct bpf_prog *prog)
9155	9320
9156	9321	{
	9322	+ enum bpf_prog_type prog_type = resolve_prog_type(prog);
9157	9323	/*
9158	9324	* Validate that trace type programs use preallocated hash maps.
9159	9325	*
@@ -9171,8 +9337,8 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, · 11 unmodified lines
9171	9337	* now, but warnings are emitted so developers are made aware of
9172	9338	* the unsafety and can fix their programs before this is enforced.
9173	9339	*/
9174		− if (is_tracing_prog_type(prog->type) && !is_preallocated_map(map)) {
9175		− if (prog->type == BPF_PROG_TYPE_PERF_EVENT) {
	9340	+ if (is_tracing_prog_type(prog_type) && !is_preallocated_map(map)) {
	9341	+ if (prog_type == BPF_PROG_TYPE_PERF_EVENT) {
9176	9342	verbose(env, "perf_event programs can only use preallocated hash map\n");
9177	9343	return -EINVAL;
9178	9344	}
@@ -9184,8 +9350,8 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, · 5 unmodified lines
9184	9350	verbose(env, "trace type programs with run-time allocated hash maps are unsafe. Switch to preallocated hash maps.\n");
9185	9351	}
9186	9352
9187		− if ((is_tracing_prog_type(prog->type) \|\|
9188		− prog->type == BPF_PROG_TYPE_SOCKET_FILTER) &&
	9353	+ if ((is_tracing_prog_type(prog_type) \|\|
	9354	+ prog_type == BPF_PROG_TYPE_SOCKET_FILTER) &&
9189	9355	map_value_has_spin_lock(map)) {
9190	9356	verbose(env, "tracing progs cannot use bpf_spin_lock yet\n");
9191	9357	return -EINVAL;
@@ -9202,6 +9368,23 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, · 10 unmodified lines
9202	9368	return -EINVAL;
9203	9369	}
9204	9370
	9371	+ if (prog->aux->sleepable)
	9372	+ switch (map->map_type) {
	9373	+ case BPF_MAP_TYPE_HASH:
	9374	+ case BPF_MAP_TYPE_LRU_HASH:
	9375	+ case BPF_MAP_TYPE_ARRAY:
	9376	+ if (!is_preallocated_map(map)) {
	9377	+ verbose(env,
	9378	+ "Sleepable programs can only use preallocated hash maps\n");
	9379	+ return -EINVAL;
	9380	+ }
	9381	+ break;
	9382	+ default:
	9383	+ verbose(env,
	9384	+ "Sleepable programs can only use array and hash maps\n");
	9385	+ return -EINVAL;
	9386	+ }
	9387	+
9205	9388	return 0;
9206	9389	}
9207	9390
@@ -9897,7 +10080,7 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env) · 689 unmodified lines
9897	10080	insn->code = BPF_LDX \| BPF_PROBE_MEM \|
9898	10081	BPF_SIZE((insn)->code);
9899	10082	env->prog->aux->num_exentries++;
9900		− } else if (env->prog->type != BPF_PROG_TYPE_STRUCT_OPS) {
	10083	+ } else if (resolve_prog_type(env->prog) != BPF_PROG_TYPE_STRUCT_OPS) {
9901	10084	verbose(env, "Writes through BTF pointers are not allowed\n");
9902	10085	return -EINVAL;
9903	10086	}
@@ -10820,6 +11003,37 @@ static int check_attach_modify_return(struct bpf_prog *prog, unsigned long addr) · 916 unmodified lines
10820	11003	return -EINVAL;
10821	11004	}
10822	11005
	11006	+/* non exhaustive list of sleepable bpf_lsm_() functions /
	11007	+BTF_SET_START(btf_sleepable_lsm_hooks)
	11008	+#ifdef CONFIG_BPF_LSM
	11009	+BTF_ID(func, bpf_lsm_bprm_committed_creds)
	11010	+#else
	11011	+BTF_ID_UNUSED
	11012	+#endif
	11013	+BTF_SET_END(btf_sleepable_lsm_hooks)
	11014	+
	11015	+static int check_sleepable_lsm_hook(u32 btf_id)
	11016	+{
	11017	+ return btf_id_set_contains(&btf_sleepable_lsm_hooks, btf_id);
	11018	+}
	11019	+
	11020	+/* list of non-sleepable functions that are otherwise on
	11021	+ * ALLOW_ERROR_INJECTION list
	11022	+ */
	11023	+BTF_SET_START(btf_non_sleepable_error_inject)
	11024	+/* Three functions below can be called from sleepable and non-sleepable context.
	11025	+ * Assume non-sleepable from bpf safety point of view.
	11026	+ */
	11027	+BTF_ID(func, __add_to_page_cache_locked)
	11028	+BTF_ID(func, should_fail_alloc_page)
	11029	+BTF_ID(func, should_failslab)
	11030	+BTF_SET_END(btf_non_sleepable_error_inject)
	11031	+
	11032	+static int check_non_sleepable_error_inject(u32 btf_id)
	11033	+{
	11034	+ return btf_id_set_contains(&btf_non_sleepable_error_inject, btf_id);
	11035	+}
	11036	+
10823	11037	static int check_attach_btf_id(struct bpf_verifier_env *env)
10824	11038	{
10825	11039	struct bpf_prog *prog = env->prog;
@@ -10837,6 +11051,12 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) · 11 unmodified lines
10837	11051	long addr;
10838	11052	u64 key;
10839	11053
	11054	+ if (prog->aux->sleepable && prog->type != BPF_PROG_TYPE_TRACING &&
	11055	+ prog->type != BPF_PROG_TYPE_LSM) {
	11056	+ verbose(env, "Only fentry/fexit/fmod_ret and lsm programs can be sleepable\n");
	11057	+ return -EINVAL;
	11058	+ }
	11059	+
10840	11060	if (prog->type == BPF_PROG_TYPE_STRUCT_OPS)
10841	11061	return check_struct_ops_btf_id(env);
10842	11062
@@ -11045,13 +11265,36 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) · 202 unmodified lines
11045	11265	}
11046	11266	}
11047	11267
11048		− if (prog->expected_attach_type == BPF_MODIFY_RETURN) {
	11268	+ if (prog->aux->sleepable) {
	11269	+ ret = -EINVAL;
	11270	+ switch (prog->type) {
	11271	+ case BPF_PROG_TYPE_TRACING:
	11272	+ /* fentry/fexit/fmod_ret progs can be sleepable only if they are
	11273	+ * attached to ALLOW_ERROR_INJECTION and are not in denylist.
	11274	+ */
	11275	+ if (!check_non_sleepable_error_inject(btf_id) &&
	11276	+ within_error_injection_list(addr))
	11277	+ ret = 0;
	11278	+ break;
	11279	+ case BPF_PROG_TYPE_LSM:
	11280	+ /* LSM progs check that they are attached to bpf_lsm_*() funcs.
	11281	+ * Only some of them are sleepable.
	11282	+ */
	11283	+ if (check_sleepable_lsm_hook(btf_id))
	11284	+ ret = 0;
	11285	+ break;
	11286	+ default:
	11287	+ break;
	11288	+ }
	11289	+ if (ret)
	11290	+ verbose(env, "%s is not sleepable\n",
	11291	+ prog->aux->attach_func_name);
	11292	+ } else if (prog->expected_attach_type == BPF_MODIFY_RETURN) {
11049	11293	ret = check_attach_modify_return(prog, addr);
11050	11294	if (ret)
11051	11295	verbose(env, "%s() is not modifiable\n",
11052	11296	prog->aux->attach_func_name);
11053	11297	}
11054		−
11055	11298	if (ret)
11056	11299	goto out;
11057	11300	tr->func.addr = (void *)addr;

kernel/trace/bpf_trace.c+50−0

@@ -1098,6 +1098,52 @@ static const struct bpf_func_proto bpf_send_signal_thread_proto = { · 1097 unmodified lines
1098	1098	.arg1_type = ARG_ANYTHING,
1099	1099	};
1100	1100
	1101	+BPF_CALL_3(bpf_d_path, struct path , path, char , buf, u32, sz)
	1102	+{
	1103	+ long len;
	1104	+ char *p;
	1105	+
	1106	+ if (!sz)
	1107	+ return 0;
	1108	+
	1109	+ p = d_path(path, buf, sz);
	1110	+ if (IS_ERR(p)) {
	1111	+ len = PTR_ERR(p);
	1112	+ } else {
	1113	+ len = buf + sz - p;
	1114	+ memmove(buf, p, len);
	1115	+ }
	1116	+
	1117	+ return len;
	1118	+}
	1119	+
	1120	+BTF_SET_START(btf_allowlist_d_path)
	1121	+BTF_ID(func, vfs_truncate)
	1122	+BTF_ID(func, vfs_fallocate)
	1123	+BTF_ID(func, dentry_open)
	1124	+BTF_ID(func, vfs_getattr)
	1125	+BTF_ID(func, filp_close)
	1126	+BTF_SET_END(btf_allowlist_d_path)
	1127	+
	1128	+static bool bpf_d_path_allowed(const struct bpf_prog *prog)
	1129	+{
	1130	+ return btf_id_set_contains(&btf_allowlist_d_path, prog->aux->attach_btf_id);
	1131	+}
	1132	+
	1133	+BTF_ID_LIST(bpf_d_path_btf_ids)
	1134	+BTF_ID(struct, path)
	1135	+
	1136	+static const struct bpf_func_proto bpf_d_path_proto = {
	1137	+ .func = bpf_d_path,
	1138	+ .gpl_only = false,
	1139	+ .ret_type = RET_INTEGER,
	1140	+ .arg1_type = ARG_PTR_TO_BTF_ID,
	1141	+ .arg2_type = ARG_PTR_TO_MEM,
	1142	+ .arg3_type = ARG_CONST_SIZE_OR_ZERO,
	1143	+ .btf_id = bpf_d_path_btf_ids,
	1144	+ .allowed = bpf_d_path_allowed,
	1145	+};
	1146	+
1101	1147	const struct bpf_func_proto *
1102	1148	bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
1103	1149	{
@@ -1182,6 +1228,8 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 78 unmodified lines
1182	1228	return &bpf_jiffies64_proto;
1183	1229	case BPF_FUNC_get_task_stack:
1184	1230	return &bpf_get_task_stack_proto;
	1231	+ case BPF_FUNC_copy_from_user:
	1232	+ return prog->aux->sleepable ? &bpf_copy_from_user_proto : NULL;
1185	1233	default:
1186	1234	return NULL;
1187	1235	}
@@ -1579,6 +1627,8 @@ tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 391 unmodified lines
1579	1627	return prog->expected_attach_type == BPF_TRACE_ITER ?
1580	1628	&bpf_seq_write_proto :
1581	1629	NULL;
	1630	+ case BPF_FUNC_d_path:
	1631	+ return &bpf_d_path_proto;
1582	1632	default:
1583	1633	return raw_tp_prog_func_proto(func_id, prog);
1584	1634	}

Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code