lael-labs/linux

kernel/Makefile+1−1

@@ -12,7 +12,7 @@ obj-y = fork.o exec_domain.o panic.o \ · 11 unmodified lines
12	notifier.o ksysfs.o cred.o reboot.o \	12	notifier.o ksysfs.o cred.o reboot.o \
13	async.o range.o smpboot.o ucount.o regset.o	13	async.o range.o smpboot.o ucount.o regset.o
14		14
15	obj-$(CONFIG_BPFILTER) += usermode_driver.o	15	obj-$(CONFIG_USERMODE_DRIVER) += usermode_driver.o
16	obj-$(CONFIG_MODULES) += kmod.o	16	obj-$(CONFIG_MODULES) += kmod.o
17	obj-$(CONFIG_MULTIUSER) += groups.o	17	obj-$(CONFIG_MULTIUSER) += groups.o
18		18

kernel/bpf/Makefile+3−0

 obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o bpf_iter.o map_iter.o task_iter.o prog_iter.o
 obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o
 obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o
+obj-${CONFIG_BPF_LSM}	  += bpf_inode_storage.o
 obj-$(CONFIG_BPF_SYSCALL) += disasm.o
 obj-$(CONFIG_BPF_JIT) += trampoline.o
 obj-$(CONFIG_BPF_SYSCALL) += btf.o
 ifeq ($(CONFIG_NET),y)
 obj-$(CONFIG_BPF_SYSCALL) += devmap.o
 obj-$(CONFIG_BPF_SYSCALL) += cpumap.o
+obj-$(CONFIG_BPF_SYSCALL) += bpf_local_storage.o
 obj-$(CONFIG_BPF_SYSCALL) += offload.o
 obj-$(CONFIG_BPF_SYSCALL) += net_namespace.o
 endif
 obj-$(CONFIG_BPF_SYSCALL) += bpf_struct_ops.o
 obj-${CONFIG_BPF_LSM} += bpf_lsm.o
 endif
+obj-$(CONFIG_BPF_PRELOAD) += preload/

kernel/bpf/arraymap.c+17−0

 #include <linux/filter.h>
 #include <linux/perf_event.h>
 #include <uapi/linux/btf.h>
+#include <linux/rcupdate_trace.h>
 #include "map_in_map.h"
 				   vma->vm_pgoff + pgoff);
 }
+static bool array_map_meta_equal(const struct bpf_map *meta0,
+				 const struct bpf_map *meta1)
+{
+	return meta0->max_entries == meta1->max_entries &&
+		bpf_map_meta_equal(meta0, meta1);
+}
 struct bpf_iter_seq_array_map_info {
 	struct bpf_map *map;
 	void *percpu_value_buf;
 static int array_map_btf_id;
 const struct bpf_map_ops array_map_ops = {
+	.map_meta_equal = array_map_meta_equal,
 	.map_alloc_check = array_map_alloc_check,
 	.map_alloc = array_map_alloc,
 	.map_free = array_map_free,
 static int percpu_array_map_btf_id;
 const struct bpf_map_ops percpu_array_map_ops = {
+	.map_meta_equal = bpf_map_meta_equal,
 	.map_alloc_check = array_map_alloc_check,
 	.map_alloc = array_map_alloc,
 	.map_free = array_map_free,
 	fd_array_map_free(map);
 }
+/* prog_array->aux->{type,jited} is a runtime binding.
+ * Doing static check alone in the verifier is not enough.
+ * Thus, prog_array_map cannot be used as an inner_map
+ * and map_meta_equal is not implemented.
+ */
 static int prog_array_map_btf_id;
 const struct bpf_map_ops prog_array_map_ops = {
 	.map_alloc_check = fd_array_map_alloc_check,
 static int perf_event_array_map_btf_id;
 const struct bpf_map_ops perf_event_array_map_ops = {
+	.map_meta_equal = bpf_map_meta_equal,
 	.map_alloc_check = fd_array_map_alloc_check,
 	.map_alloc = array_map_alloc,
 	.map_free = fd_array_map_free,
 static int cgroup_array_map_btf_id;
 const struct bpf_map_ops cgroup_array_map_ops = {
+	.map_meta_equal = bpf_map_meta_equal,
 	.map_alloc_check = fd_array_map_alloc_check,
 	.map_alloc = array_map_alloc,
 	.map_free = cgroup_fd_array_free,

kernel/bpf/bpf_inode_storage.c+274−0

@@ -0,0 +1,274 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* Copyright (c) 2019 Facebook
	4	* Copyright 2020 Google LLC.
	5	*/
	6
	7	#include <linux/rculist.h>
	8	#include <linux/list.h>
	9	#include <linux/hash.h>
	10	#include <linux/types.h>
	11	#include <linux/spinlock.h>
	12	#include <linux/bpf.h>
	13	#include <linux/bpf_local_storage.h>
	14	#include <net/sock.h>
	15	#include <uapi/linux/sock_diag.h>
	16	#include <uapi/linux/btf.h>
	17	#include <linux/bpf_lsm.h>
	18	#include <linux/btf_ids.h>
	19	#include <linux/fdtable.h>
	20
	21	DEFINE_BPF_STORAGE_CACHE(inode_cache);
	22
	23	static struct bpf_local_storage __rcu **
	24	inode_storage_ptr(void *owner)
	25	{
	26	struct inode *inode = owner;
	27	struct bpf_storage_blob *bsb;
	28
	29	bsb = bpf_inode(inode);
	30	if (!bsb)
	31	return NULL;
	32	return &bsb->storage;
	33	}
	34
	35	static struct bpf_local_storage_data inode_storage_lookup(struct inode inode,
	36	struct bpf_map *map,
	37	bool cacheit_lockit)
	38	{
	39	struct bpf_local_storage *inode_storage;
	40	struct bpf_local_storage_map *smap;
	41	struct bpf_storage_blob *bsb;
	42
	43	bsb = bpf_inode(inode);
	44	if (!bsb)
	45	return NULL;
	46
	47	inode_storage = rcu_dereference(bsb->storage);
	48	if (!inode_storage)
	49	return NULL;
	50
	51	smap = (struct bpf_local_storage_map *)map;
	52	return bpf_local_storage_lookup(inode_storage, smap, cacheit_lockit);
	53	}
	54
	55	void bpf_inode_storage_free(struct inode *inode)
	56	{
	57	struct bpf_local_storage_elem *selem;
	58	struct bpf_local_storage *local_storage;
	59	bool free_inode_storage = false;
	60	struct bpf_storage_blob *bsb;
	61	struct hlist_node *n;
	62
	63	bsb = bpf_inode(inode);
	64	if (!bsb)
	65	return;
	66
	67	rcu_read_lock();
	68
	69	local_storage = rcu_dereference(bsb->storage);
	70	if (!local_storage) {
	71	rcu_read_unlock();
	72	return;
	73	}
	74
	75	/* Netiher the bpf_prog nor the bpf-map's syscall
	76	* could be modifying the local_storage->list now.
	77	* Thus, no elem can be added-to or deleted-from the
	78	* local_storage->list by the bpf_prog or by the bpf-map's syscall.
	79	*
	80	* It is racing with bpf_local_storage_map_free() alone
	81	* when unlinking elem from the local_storage->list and
	82	* the map's bucket->list.
	83	*/
	84	raw_spin_lock_bh(&local_storage->lock);
	85	hlist_for_each_entry_safe(selem, n, &local_storage->list, snode) {
	86	/* Always unlink from map before unlinking from
	87	* local_storage.
	88	*/
	89	bpf_selem_unlink_map(selem);
	90	free_inode_storage = bpf_selem_unlink_storage_nolock(
	91	local_storage, selem, false);
	92	}
	93	raw_spin_unlock_bh(&local_storage->lock);
	94	rcu_read_unlock();
	95
	96	/* free_inoode_storage should always be true as long as
	97	* local_storage->list was non-empty.
	98	*/
	99	if (free_inode_storage)
	100	kfree_rcu(local_storage, rcu);
	101	}
	102
	103	static void bpf_fd_inode_storage_lookup_elem(struct bpf_map map, void *key)
	104	{
	105	struct bpf_local_storage_data *sdata;
	106	struct file *f;
	107	int fd;
	108
	109	fd = (int )key;
	110	f = fget_raw(fd);
	111	if (!f)
	112	return NULL;
	113
	114	sdata = inode_storage_lookup(f->f_inode, map, true);
	115	fput(f);
	116	return sdata ? sdata->data : NULL;
	117	}
	118
	119	static int bpf_fd_inode_storage_update_elem(struct bpf_map map, void key,
	120	void *value, u64 map_flags)
	121	{
	122	struct bpf_local_storage_data *sdata;
	123	struct file *f;
	124	int fd;
	125
	126	fd = (int )key;
	127	f = fget_raw(fd);
	128	if (!f \|\| !inode_storage_ptr(f->f_inode))
	129	return -EBADF;
	130
	131	sdata = bpf_local_storage_update(f->f_inode,
	132	(struct bpf_local_storage_map *)map,
	133	value, map_flags);
	134	fput(f);
	135	return PTR_ERR_OR_ZERO(sdata);
	136	}
	137
	138	static int inode_storage_delete(struct inode inode, struct bpf_map map)
	139	{
	140	struct bpf_local_storage_data *sdata;
	141
	142	sdata = inode_storage_lookup(inode, map, false);
	143	if (!sdata)
	144	return -ENOENT;
	145
	146	bpf_selem_unlink(SELEM(sdata));
	147
	148	return 0;
	149	}
	150
	151	static int bpf_fd_inode_storage_delete_elem(struct bpf_map map, void key)
	152	{
	153	struct file *f;
	154	int fd, err;
	155
	156	fd = (int )key;
	157	f = fget_raw(fd);
	158	if (!f)
	159	return -EBADF;
	160
	161	err = inode_storage_delete(f->f_inode, map);
	162	fput(f);
	163	return err;
	164	}
	165
	166	BPF_CALL_4(bpf_inode_storage_get, struct bpf_map , map, struct inode , inode,
	167	void *, value, u64, flags)
	168	{
	169	struct bpf_local_storage_data *sdata;
	170
	171	if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
	172	return (unsigned long)NULL;
	173
	174	/* explicitly check that the inode_storage_ptr is not
	175	* NULL as inode_storage_lookup returns NULL in this case and
	176	* bpf_local_storage_update expects the owner to have a
	177	* valid storage pointer.
	178	*/
	179	if (!inode_storage_ptr(inode))
	180	return (unsigned long)NULL;
	181
	182	sdata = inode_storage_lookup(inode, map, true);
	183	if (sdata)
	184	return (unsigned long)sdata->data;
	185
	186	/* This helper must only called from where the inode is gurranteed
	187	* to have a refcount and cannot be freed.
	188	*/
	189	if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) {
	190	sdata = bpf_local_storage_update(
	191	inode, (struct bpf_local_storage_map *)map, value,
	192	BPF_NOEXIST);
	193	return IS_ERR(sdata) ? (unsigned long)NULL :
	194	(unsigned long)sdata->data;
	195	}
	196
	197	return (unsigned long)NULL;
	198	}
	199
	200	BPF_CALL_2(bpf_inode_storage_delete,
	201	struct bpf_map , map, struct inode , inode)
	202	{
	203	/* This helper must only called from where the inode is gurranteed
	204	* to have a refcount and cannot be freed.
	205	*/
	206	return inode_storage_delete(inode, map);
	207	}
	208
	209	static int notsupp_get_next_key(struct bpf_map map, void key,
	210	void *next_key)
	211	{
	212	return -ENOTSUPP;
	213	}
	214
	215	static struct bpf_map inode_storage_map_alloc(union bpf_attr attr)
	216	{
	217	struct bpf_local_storage_map *smap;
	218
	219	smap = bpf_local_storage_map_alloc(attr);
	220	if (IS_ERR(smap))
	221	return ERR_CAST(smap);
	222
	223	smap->cache_idx = bpf_local_storage_cache_idx_get(&inode_cache);
	224	return &smap->map;
	225	}
	226
	227	static void inode_storage_map_free(struct bpf_map *map)
	228	{
	229	struct bpf_local_storage_map *smap;
	230
	231	smap = (struct bpf_local_storage_map *)map;
	232	bpf_local_storage_cache_idx_free(&inode_cache, smap->cache_idx);
	233	bpf_local_storage_map_free(smap);
	234	}
	235
	236	static int inode_storage_map_btf_id;
	237	const struct bpf_map_ops inode_storage_map_ops = {
	238	.map_meta_equal = bpf_map_meta_equal,
	239	.map_alloc_check = bpf_local_storage_map_alloc_check,
	240	.map_alloc = inode_storage_map_alloc,
	241	.map_free = inode_storage_map_free,
	242	.map_get_next_key = notsupp_get_next_key,
	243	.map_lookup_elem = bpf_fd_inode_storage_lookup_elem,
	244	.map_update_elem = bpf_fd_inode_storage_update_elem,
	245	.map_delete_elem = bpf_fd_inode_storage_delete_elem,
	246	.map_check_btf = bpf_local_storage_map_check_btf,
	247	.map_btf_name = "bpf_local_storage_map",
	248	.map_btf_id = &inode_storage_map_btf_id,
	249	.map_owner_storage_ptr = inode_storage_ptr,
	250	};
	251
	252	BTF_ID_LIST(bpf_inode_storage_btf_ids)
	253	BTF_ID_UNUSED
	254	BTF_ID(struct, inode)
	255
	256	const struct bpf_func_proto bpf_inode_storage_get_proto = {
	257	.func = bpf_inode_storage_get,
	258	.gpl_only = false,
	259	.ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
	260	.arg1_type = ARG_CONST_MAP_PTR,
	261	.arg2_type = ARG_PTR_TO_BTF_ID,
	262	.arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL,
	263	.arg4_type = ARG_ANYTHING,
	264	.btf_id = bpf_inode_storage_btf_ids,
	265	};
	266
	267	const struct bpf_func_proto bpf_inode_storage_delete_proto = {
	268	.func = bpf_inode_storage_delete,
	269	.gpl_only = false,
	270	.ret_type = RET_INTEGER,
	271	.arg1_type = ARG_CONST_MAP_PTR,
	272	.arg2_type = ARG_PTR_TO_BTF_ID,
	273	.btf_id = bpf_inode_storage_btf_ids,
	274	};

kernel/bpf/bpf_iter.c+58−0

@@ -390,10 +390,68 @@ out_unlock: · 389 unmodified lines
390	return ret;	390	return ret;
391	}	391	}
392		392
		393	static void bpf_iter_link_show_fdinfo(const struct bpf_link *link,
		394	struct seq_file *seq)
		395	{
		396	struct bpf_iter_link *iter_link =
		397	container_of(link, struct bpf_iter_link, link);
		398	bpf_iter_show_fdinfo_t show_fdinfo;
		399
		400	seq_printf(seq,
		401	"target_name:\t%s\n",
		402	iter_link->tinfo->reg_info->target);
		403
		404	show_fdinfo = iter_link->tinfo->reg_info->show_fdinfo;
		405	if (show_fdinfo)
		406	show_fdinfo(&iter_link->aux, seq);
		407	}
		408
		409	static int bpf_iter_link_fill_link_info(const struct bpf_link *link,
		410	struct bpf_link_info *info)
		411	{
		412	struct bpf_iter_link *iter_link =
		413	container_of(link, struct bpf_iter_link, link);
		414	char __user *ubuf = u64_to_user_ptr(info->iter.target_name);
		415	bpf_iter_fill_link_info_t fill_link_info;
		416	u32 ulen = info->iter.target_name_len;
		417	const char *target_name;
		418	u32 target_len;
		419
		420	if (!ulen ^ !ubuf)
		421	return -EINVAL;
		422
		423	target_name = iter_link->tinfo->reg_info->target;
		424	target_len = strlen(target_name);
		425	info->iter.target_name_len = target_len + 1;
		426
		427	if (ubuf) {
		428	if (ulen >= target_len + 1) {
		429	if (copy_to_user(ubuf, target_name, target_len + 1))
		430	return -EFAULT;
		431	} else {
		432	char zero = '\0';
		433
		434	if (copy_to_user(ubuf, target_name, ulen - 1))
		435	return -EFAULT;
		436	if (put_user(zero, ubuf + ulen - 1))
		437	return -EFAULT;
		438	return -ENOSPC;
		439	}
		440	}
		441
		442	fill_link_info = iter_link->tinfo->reg_info->fill_link_info;
		443	if (fill_link_info)
		444	return fill_link_info(&iter_link->aux, info);
		445
		446	return 0;
		447	}
		448
393	static const struct bpf_link_ops bpf_iter_link_lops = {	449	static const struct bpf_link_ops bpf_iter_link_lops = {
394	.release = bpf_iter_link_release,	450	.release = bpf_iter_link_release,
395	.dealloc = bpf_iter_link_dealloc,	451	.dealloc = bpf_iter_link_dealloc,
396	.update_prog = bpf_iter_link_replace,	452	.update_prog = bpf_iter_link_replace,
		453	.show_fdinfo = bpf_iter_link_show_fdinfo,
		454	.fill_link_info = bpf_iter_link_fill_link_info,
397	};	455	};
398		456
399	bool bpf_link_is_iter(struct bpf_link *link)	457	bool bpf_link_is_iter(struct bpf_link *link)

kernel/bpf/bpf_local_storage.c+600−0

@@ -0,0 +1,600 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2019 Facebook */
	3	#include <linux/rculist.h>
	4	#include <linux/list.h>
	5	#include <linux/hash.h>
	6	#include <linux/types.h>
	7	#include <linux/spinlock.h>
	8	#include <linux/bpf.h>
	9	#include <linux/btf_ids.h>
	10	#include <linux/bpf_local_storage.h>
	11	#include <net/sock.h>
	12	#include <uapi/linux/sock_diag.h>
	13	#include <uapi/linux/btf.h>
	14
	15	#define BPF_LOCAL_STORAGE_CREATE_FLAG_MASK (BPF_F_NO_PREALLOC \| BPF_F_CLONE)
	16
	17	static struct bpf_local_storage_map_bucket *
	18	select_bucket(struct bpf_local_storage_map *smap,
	19	struct bpf_local_storage_elem *selem)
	20	{
	21	return &smap->buckets[hash_ptr(selem, smap->bucket_log)];
	22	}
	23
	24	static int mem_charge(struct bpf_local_storage_map smap, void owner, u32 size)
	25	{
	26	struct bpf_map *map = &smap->map;
	27
	28	if (!map->ops->map_local_storage_charge)
	29	return 0;
	30
	31	return map->ops->map_local_storage_charge(smap, owner, size);
	32	}
	33
	34	static void mem_uncharge(struct bpf_local_storage_map smap, void owner,
	35	u32 size)
	36	{
	37	struct bpf_map *map = &smap->map;
	38
	39	if (map->ops->map_local_storage_uncharge)
	40	map->ops->map_local_storage_uncharge(smap, owner, size);
	41	}
	42
	43	static struct bpf_local_storage __rcu **
	44	owner_storage(struct bpf_local_storage_map smap, void owner)
	45	{
	46	struct bpf_map *map = &smap->map;
	47
	48	return map->ops->map_owner_storage_ptr(owner);
	49	}
	50
	51	static bool selem_linked_to_storage(const struct bpf_local_storage_elem *selem)
	52	{
	53	return !hlist_unhashed(&selem->snode);
	54	}
	55
	56	static bool selem_linked_to_map(const struct bpf_local_storage_elem *selem)
	57	{
	58	return !hlist_unhashed(&selem->map_node);
	59	}
	60
	61	struct bpf_local_storage_elem *
	62	bpf_selem_alloc(struct bpf_local_storage_map smap, void owner,
	63	void *value, bool charge_mem)
	64	{
	65	struct bpf_local_storage_elem *selem;
	66
	67	if (charge_mem && mem_charge(smap, owner, smap->elem_size))
	68	return NULL;
	69
	70	selem = kzalloc(smap->elem_size, GFP_ATOMIC \| __GFP_NOWARN);
	71	if (selem) {
	72	if (value)
	73	memcpy(SDATA(selem)->data, value, smap->map.value_size);
	74	return selem;
	75	}
	76
	77	if (charge_mem)
	78	mem_uncharge(smap, owner, smap->elem_size);
	79
	80	return NULL;
	81	}
	82
	83	/* local_storage->lock must be held and selem->local_storage == local_storage.
	84	* The caller must ensure selem->smap is still valid to be
	85	* dereferenced for its smap->elem_size and smap->cache_idx.
	86	*/
	87	bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage,
	88	struct bpf_local_storage_elem *selem,
	89	bool uncharge_mem)
	90	{
	91	struct bpf_local_storage_map *smap;
	92	bool free_local_storage;
	93	void *owner;
	94
	95	smap = rcu_dereference(SDATA(selem)->smap);
	96	owner = local_storage->owner;
	97
	98	/* All uncharging on the owner must be done first.
	99	* The owner may be freed once the last selem is unlinked
	100	* from local_storage.
	101	*/
	102	if (uncharge_mem)
	103	mem_uncharge(smap, owner, smap->elem_size);
	104
	105	free_local_storage = hlist_is_singular_node(&selem->snode,
	106	&local_storage->list);
	107	if (free_local_storage) {
	108	mem_uncharge(smap, owner, sizeof(struct bpf_local_storage));
	109	local_storage->owner = NULL;
	110
	111	/* After this RCU_INIT, owner may be freed and cannot be used */
	112	RCU_INIT_POINTER(*owner_storage(smap, owner), NULL);
	113
	114	/* local_storage is not freed now. local_storage->lock is
	115	* still held and raw_spin_unlock_bh(&local_storage->lock)
	116	* will be done by the caller.
	117	*
	118	* Although the unlock will be done under
	119	* rcu_read_lock(), it is more intutivie to
	120	* read if kfree_rcu(local_storage, rcu) is done
	121	* after the raw_spin_unlock_bh(&local_storage->lock).
	122	*
	123	* Hence, a "bool free_local_storage" is returned
	124	* to the caller which then calls the kfree_rcu()
	125	* after unlock.
	126	*/
	127	}
	128	hlist_del_init_rcu(&selem->snode);
	129	if (rcu_access_pointer(local_storage->cache[smap->cache_idx]) ==
	130	SDATA(selem))
	131	RCU_INIT_POINTER(local_storage->cache[smap->cache_idx], NULL);
	132
	133	kfree_rcu(selem, rcu);
	134
	135	return free_local_storage;
	136	}
	137
	138	static void __bpf_selem_unlink_storage(struct bpf_local_storage_elem *selem)
	139	{
	140	struct bpf_local_storage *local_storage;
	141	bool free_local_storage = false;
	142
	143	if (unlikely(!selem_linked_to_storage(selem)))
	144	/* selem has already been unlinked from sk */
	145	return;
	146
	147	local_storage = rcu_dereference(selem->local_storage);
	148	raw_spin_lock_bh(&local_storage->lock);
	149	if (likely(selem_linked_to_storage(selem)))
	150	free_local_storage = bpf_selem_unlink_storage_nolock(
	151	local_storage, selem, true);
	152	raw_spin_unlock_bh(&local_storage->lock);
	153
	154	if (free_local_storage)
	155	kfree_rcu(local_storage, rcu);
	156	}
	157
	158	void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,
	159	struct bpf_local_storage_elem *selem)
	160	{
	161	RCU_INIT_POINTER(selem->local_storage, local_storage);
	162	hlist_add_head(&selem->snode, &local_storage->list);
	163	}
	164
	165	void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem)
	166	{
	167	struct bpf_local_storage_map *smap;
	168	struct bpf_local_storage_map_bucket *b;
	169
	170	if (unlikely(!selem_linked_to_map(selem)))
	171	/* selem has already be unlinked from smap */
	172	return;
	173
	174	smap = rcu_dereference(SDATA(selem)->smap);
	175	b = select_bucket(smap, selem);
	176	raw_spin_lock_bh(&b->lock);
	177	if (likely(selem_linked_to_map(selem)))
	178	hlist_del_init_rcu(&selem->map_node);
	179	raw_spin_unlock_bh(&b->lock);
	180	}
	181
	182	void bpf_selem_link_map(struct bpf_local_storage_map *smap,
	183	struct bpf_local_storage_elem *selem)
	184	{
	185	struct bpf_local_storage_map_bucket *b = select_bucket(smap, selem);
	186
	187	raw_spin_lock_bh(&b->lock);
	188	RCU_INIT_POINTER(SDATA(selem)->smap, smap);
	189	hlist_add_head_rcu(&selem->map_node, &b->list);
	190	raw_spin_unlock_bh(&b->lock);
	191	}
	192
	193	void bpf_selem_unlink(struct bpf_local_storage_elem *selem)
	194	{
	195	/* Always unlink from map before unlinking from local_storage
	196	* because selem will be freed after successfully unlinked from
	197	* the local_storage.
	198	*/
	199	bpf_selem_unlink_map(selem);
	200	__bpf_selem_unlink_storage(selem);
	201	}
	202
	203	struct bpf_local_storage_data *
	204	bpf_local_storage_lookup(struct bpf_local_storage *local_storage,
	205	struct bpf_local_storage_map *smap,
	206	bool cacheit_lockit)
	207	{
	208	struct bpf_local_storage_data *sdata;
	209	struct bpf_local_storage_elem *selem;
	210
	211	/* Fast path (cache hit) */
	212	sdata = rcu_dereference(local_storage->cache[smap->cache_idx]);
	213	if (sdata && rcu_access_pointer(sdata->smap) == smap)
	214	return sdata;
	215
	216	/* Slow path (cache miss) */
	217	hlist_for_each_entry_rcu(selem, &local_storage->list, snode)
	218	if (rcu_access_pointer(SDATA(selem)->smap) == smap)
	219	break;
	220
	221	if (!selem)
	222	return NULL;
	223
	224	sdata = SDATA(selem);
	225	if (cacheit_lockit) {
	226	/* spinlock is needed to avoid racing with the
	227	* parallel delete. Otherwise, publishing an already
	228	* deleted sdata to the cache will become a use-after-free
	229	* problem in the next bpf_local_storage_lookup().
	230	*/
	231	raw_spin_lock_bh(&local_storage->lock);
	232	if (selem_linked_to_storage(selem))
	233	rcu_assign_pointer(local_storage->cache[smap->cache_idx],
	234	sdata);
	235	raw_spin_unlock_bh(&local_storage->lock);
	236	}
	237
	238	return sdata;
	239	}
	240
	241	static int check_flags(const struct bpf_local_storage_data *old_sdata,
	242	u64 map_flags)
	243	{
	244	if (old_sdata && (map_flags & ~BPF_F_LOCK) == BPF_NOEXIST)
	245	/* elem already exists */
	246	return -EEXIST;
	247
	248	if (!old_sdata && (map_flags & ~BPF_F_LOCK) == BPF_EXIST)
	249	/* elem doesn't exist, cannot update it */
	250	return -ENOENT;
	251
	252	return 0;
	253	}
	254
	255	int bpf_local_storage_alloc(void *owner,
	256	struct bpf_local_storage_map *smap,
	257	struct bpf_local_storage_elem *first_selem)
	258	{
	259	struct bpf_local_storage prev_storage, storage;
	260	struct bpf_local_storage **owner_storage_ptr;
	261	int err;
	262
	263	err = mem_charge(smap, owner, sizeof(*storage));
	264	if (err)
	265	return err;
	266
	267	storage = kzalloc(sizeof(*storage), GFP_ATOMIC \| __GFP_NOWARN);
	268	if (!storage) {
	269	err = -ENOMEM;
	270	goto uncharge;
	271	}
	272
	273	INIT_HLIST_HEAD(&storage->list);
	274	raw_spin_lock_init(&storage->lock);
	275	storage->owner = owner;
	276
	277	bpf_selem_link_storage_nolock(storage, first_selem);
	278	bpf_selem_link_map(smap, first_selem);
	279
	280	owner_storage_ptr =
	281	(struct bpf_local_storage **)owner_storage(smap, owner);
	282	/* Publish storage to the owner.
	283	* Instead of using any lock of the kernel object (i.e. owner),
	284	* cmpxchg will work with any kernel object regardless what
	285	* the running context is, bh, irq...etc.
	286	*
	287	* From now on, the owner->storage pointer (e.g. sk->sk_bpf_storage)
	288	* is protected by the storage->lock. Hence, when freeing
	289	* the owner->storage, the storage->lock must be held before
	290	* setting owner->storage ptr to NULL.
	291	*/
	292	prev_storage = cmpxchg(owner_storage_ptr, NULL, storage);
	293	if (unlikely(prev_storage)) {
	294	bpf_selem_unlink_map(first_selem);
	295	err = -EAGAIN;
	296	goto uncharge;
	297
	298	/* Note that even first_selem was linked to smap's
	299	* bucket->list, first_selem can be freed immediately
	300	* (instead of kfree_rcu) because
	301	* bpf_local_storage_map_free() does a
	302	* synchronize_rcu() before walking the bucket->list.
	303	* Hence, no one is accessing selem from the
	304	* bucket->list under rcu_read_lock().
	305	*/
	306	}
	307
	308	return 0;
	309
	310	uncharge:
	311	kfree(storage);
	312	mem_uncharge(smap, owner, sizeof(*storage));
	313	return err;
	314	}
	315
	316	/* sk cannot be going away because it is linking new elem
	317	* to sk->sk_bpf_storage. (i.e. sk->sk_refcnt cannot be 0).
	318	* Otherwise, it will become a leak (and other memory issues
	319	* during map destruction).
	320	*/
	321	struct bpf_local_storage_data *
	322	bpf_local_storage_update(void owner, struct bpf_local_storage_map smap,
	323	void *value, u64 map_flags)
	324	{
	325	struct bpf_local_storage_data *old_sdata = NULL;
	326	struct bpf_local_storage_elem *selem;
	327	struct bpf_local_storage *local_storage;
	328	int err;
	329
	330	/* BPF_EXIST and BPF_NOEXIST cannot be both set */
	331	if (unlikely((map_flags & ~BPF_F_LOCK) > BPF_EXIST) \|\|
	332	/* BPF_F_LOCK can only be used in a value with spin_lock */
	333	unlikely((map_flags & BPF_F_LOCK) &&
	334	!map_value_has_spin_lock(&smap->map)))
	335	return ERR_PTR(-EINVAL);
	336
	337	local_storage = rcu_dereference(*owner_storage(smap, owner));
	338	if (!local_storage \|\| hlist_empty(&local_storage->list)) {
	339	/* Very first elem for the owner */
	340	err = check_flags(NULL, map_flags);
	341	if (err)
	342	return ERR_PTR(err);
	343
	344	selem = bpf_selem_alloc(smap, owner, value, true);
	345	if (!selem)
	346	return ERR_PTR(-ENOMEM);
	347
	348	err = bpf_local_storage_alloc(owner, smap, selem);
	349	if (err) {
	350	kfree(selem);
	351	mem_uncharge(smap, owner, smap->elem_size);
	352	return ERR_PTR(err);
	353	}
	354
	355	return SDATA(selem);
	356	}
	357
	358	if ((map_flags & BPF_F_LOCK) && !(map_flags & BPF_NOEXIST)) {
	359	/* Hoping to find an old_sdata to do inline update
	360	* such that it can avoid taking the local_storage->lock
	361	* and changing the lists.
	362	*/
	363	old_sdata =
	364	bpf_local_storage_lookup(local_storage, smap, false);
	365	err = check_flags(old_sdata, map_flags);
	366	if (err)
	367	return ERR_PTR(err);
	368	if (old_sdata && selem_linked_to_storage(SELEM(old_sdata))) {
	369	copy_map_value_locked(&smap->map, old_sdata->data,
	370	value, false);
	371	return old_sdata;
	372	}
	373	}
	374
	375	raw_spin_lock_bh(&local_storage->lock);
	376
	377	/* Recheck local_storage->list under local_storage->lock */
	378	if (unlikely(hlist_empty(&local_storage->list))) {
	379	/* A parallel del is happening and local_storage is going
	380	* away. It has just been checked before, so very
	381	* unlikely. Return instead of retry to keep things
	382	* simple.
	383	*/
	384	err = -EAGAIN;
	385	goto unlock_err;
	386	}
	387
	388	old_sdata = bpf_local_storage_lookup(local_storage, smap, false);
	389	err = check_flags(old_sdata, map_flags);
	390	if (err)
	391	goto unlock_err;
	392
	393	if (old_sdata && (map_flags & BPF_F_LOCK)) {
	394	copy_map_value_locked(&smap->map, old_sdata->data, value,
	395	false);
	396	selem = SELEM(old_sdata);
	397	goto unlock;
	398	}
	399
	400	/* local_storage->lock is held. Hence, we are sure
	401	* we can unlink and uncharge the old_sdata successfully
	402	* later. Hence, instead of charging the new selem now
	403	* and then uncharge the old selem later (which may cause
	404	* a potential but unnecessary charge failure), avoid taking
	405	* a charge at all here (the "!old_sdata" check) and the
	406	* old_sdata will not be uncharged later during
	407	* bpf_selem_unlink_storage_nolock().
	408	*/
	409	selem = bpf_selem_alloc(smap, owner, value, !old_sdata);
	410	if (!selem) {
	411	err = -ENOMEM;
	412	goto unlock_err;
	413	}
	414
	415	/* First, link the new selem to the map */
	416	bpf_selem_link_map(smap, selem);
	417
	418	/* Second, link (and publish) the new selem to local_storage */
	419	bpf_selem_link_storage_nolock(local_storage, selem);
	420
	421	/* Third, remove old selem, SELEM(old_sdata) */
	422	if (old_sdata) {
	423	bpf_selem_unlink_map(SELEM(old_sdata));
	424	bpf_selem_unlink_storage_nolock(local_storage, SELEM(old_sdata),
	425	false);
	426	}
	427
	428	unlock:
	429	raw_spin_unlock_bh(&local_storage->lock);
	430	return SDATA(selem);
	431
	432	unlock_err:
	433	raw_spin_unlock_bh(&local_storage->lock);
	434	return ERR_PTR(err);
	435	}
	436
	437	u16 bpf_local_storage_cache_idx_get(struct bpf_local_storage_cache *cache)
	438	{
	439	u64 min_usage = U64_MAX;
	440	u16 i, res = 0;
	441
	442	spin_lock(&cache->idx_lock);
	443
	444	for (i = 0; i < BPF_LOCAL_STORAGE_CACHE_SIZE; i++) {
	445	if (cache->idx_usage_counts[i] < min_usage) {
	446	min_usage = cache->idx_usage_counts[i];
	447	res = i;
	448
	449	/* Found a free cache_idx */
	450	if (!min_usage)
	451	break;
	452	}
	453	}
	454	cache->idx_usage_counts[res]++;
	455
	456	spin_unlock(&cache->idx_lock);
	457
	458	return res;
	459	}
	460
	461	void bpf_local_storage_cache_idx_free(struct bpf_local_storage_cache *cache,
	462	u16 idx)
	463	{
	464	spin_lock(&cache->idx_lock);
	465	cache->idx_usage_counts[idx]--;
	466	spin_unlock(&cache->idx_lock);
	467	}
	468
	469	void bpf_local_storage_map_free(struct bpf_local_storage_map *smap)
	470	{
	471	struct bpf_local_storage_elem *selem;
	472	struct bpf_local_storage_map_bucket *b;
	473	unsigned int i;
	474
	475	/* Note that this map might be concurrently cloned from
	476	* bpf_sk_storage_clone. Wait for any existing bpf_sk_storage_clone
	477	* RCU read section to finish before proceeding. New RCU
	478	* read sections should be prevented via bpf_map_inc_not_zero.
	479	*/
	480	synchronize_rcu();
	481
	482	/* bpf prog and the userspace can no longer access this map
	483	* now. No new selem (of this map) can be added
	484	* to the owner->storage or to the map bucket's list.
	485	*
	486	* The elem of this map can be cleaned up here
	487	* or when the storage is freed e.g.
	488	* by bpf_sk_storage_free() during __sk_destruct().
	489	*/
	490	for (i = 0; i < (1U << smap->bucket_log); i++) {
	491	b = &smap->buckets[i];
	492
	493	rcu_read_lock();
	494	/* No one is adding to b->list now */
	495	while ((selem = hlist_entry_safe(
	496	rcu_dereference_raw(hlist_first_rcu(&b->list)),
	497	struct bpf_local_storage_elem, map_node))) {
	498	bpf_selem_unlink(selem);
	499	cond_resched_rcu();
	500	}
	501	rcu_read_unlock();
	502	}
	503
	504	/* While freeing the storage we may still need to access the map.
	505	*
	506	* e.g. when bpf_sk_storage_free() has unlinked selem from the map
	507	* which then made the above while((selem = ...)) loop
	508	* exit immediately.
	509	*
	510	* However, while freeing the storage one still needs to access the
	511	* smap->elem_size to do the uncharging in
	512	* bpf_selem_unlink_storage_nolock().
	513	*
	514	* Hence, wait another rcu grace period for the storage to be freed.
	515	*/
	516	synchronize_rcu();
	517
	518	kvfree(smap->buckets);
	519	kfree(smap);
	520	}
	521
	522	int bpf_local_storage_map_alloc_check(union bpf_attr *attr)
	523	{
	524	if (attr->map_flags & ~BPF_LOCAL_STORAGE_CREATE_FLAG_MASK \|\|
	525	!(attr->map_flags & BPF_F_NO_PREALLOC) \|\|
	526	attr->max_entries \|\|
	527	attr->key_size != sizeof(int) \|\| !attr->value_size \|\|
	528	/* Enforce BTF for userspace sk dumping */
	529	!attr->btf_key_type_id \|\| !attr->btf_value_type_id)
	530	return -EINVAL;
	531
	532	if (!bpf_capable())
	533	return -EPERM;
	534
	535	if (attr->value_size > BPF_LOCAL_STORAGE_MAX_VALUE_SIZE)
	536	return -E2BIG;
	537
	538	return 0;
	539	}
	540
	541	struct bpf_local_storage_map bpf_local_storage_map_alloc(union bpf_attr attr)
	542	{
	543	struct bpf_local_storage_map *smap;
	544	unsigned int i;
	545	u32 nbuckets;
	546	u64 cost;
	547	int ret;
	548
	549	smap = kzalloc(sizeof(*smap), GFP_USER \| __GFP_NOWARN);
	550	if (!smap)
	551	return ERR_PTR(-ENOMEM);
	552	bpf_map_init_from_attr(&smap->map, attr);
	553
	554	nbuckets = roundup_pow_of_two(num_possible_cpus());
	555	/* Use at least 2 buckets, select_bucket() is undefined behavior with 1 bucket */
	556	nbuckets = max_t(u32, 2, nbuckets);
	557	smap->bucket_log = ilog2(nbuckets);
	558	cost = sizeof(smap->buckets) nbuckets + sizeof(*smap);
	559
	560	ret = bpf_map_charge_init(&smap->map.memory, cost);
	561	if (ret < 0) {
	562	kfree(smap);
	563	return ERR_PTR(ret);
	564	}
	565
	566	smap->buckets = kvcalloc(sizeof(*smap->buckets), nbuckets,
	567	GFP_USER \| __GFP_NOWARN);
	568	if (!smap->buckets) {
	569	bpf_map_charge_finish(&smap->map.memory);
	570	kfree(smap);
	571	return ERR_PTR(-ENOMEM);
	572	}
	573
	574	for (i = 0; i < nbuckets; i++) {
	575	INIT_HLIST_HEAD(&smap->buckets[i].list);
	576	raw_spin_lock_init(&smap->buckets[i].lock);
	577	}
	578
	579	smap->elem_size =
	580	sizeof(struct bpf_local_storage_elem) + attr->value_size;
	581
	582	return smap;
	583	}
	584
	585	int bpf_local_storage_map_check_btf(const struct bpf_map *map,
	586	const struct btf *btf,
	587	const struct btf_type *key_type,
	588	const struct btf_type *value_type)
	589	{
	590	u32 int_data;
	591
	592	if (BTF_INFO_KIND(key_type->info) != BTF_KIND_INT)
	593	return -EINVAL;
	594
	595	int_data = (u32 )(key_type + 1);
	596	if (BTF_INT_BITS(int_data) != 32 \|\| BTF_INT_OFFSET(int_data))
	597	return -EINVAL;
	598
	599	return 0;
	600	}

kernel/bpf/bpf_lsm.c+20−1

@@ -11,6 +11,8 @@ · 10 unmodified lines
11	#include <linux/bpf_lsm.h>	11	#include <linux/bpf_lsm.h>
12	#include <linux/kallsyms.h>	12	#include <linux/kallsyms.h>
13	#include <linux/bpf_verifier.h>	13	#include <linux/bpf_verifier.h>
		14	#include <net/bpf_sk_storage.h>
		15	#include <linux/bpf_local_storage.h>
14		16
15	/* For every LSM hook that allows attachment of BPF programs, declare a nop	17	/* For every LSM hook that allows attachment of BPF programs, declare a nop
16	* function where a BPF program can be attached.	18	* function where a BPF program can be attached.
@@ -45,10 +47,27 @@ int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog, · 28 unmodified lines
45	return 0;	47	return 0;
46	}	48	}
47		49
		50	static const struct bpf_func_proto *
		51	bpf_lsm_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
		52	{
		53	switch (func_id) {
		54	case BPF_FUNC_inode_storage_get:
		55	return &bpf_inode_storage_get_proto;
		56	case BPF_FUNC_inode_storage_delete:
		57	return &bpf_inode_storage_delete_proto;
		58	case BPF_FUNC_sk_storage_get:
		59	return &sk_storage_get_btf_proto;
		60	case BPF_FUNC_sk_storage_delete:
		61	return &sk_storage_delete_btf_proto;
		62	default:
		63	return tracing_prog_func_proto(func_id, prog);
		64	}
		65	}
		66
48	const struct bpf_prog_ops lsm_prog_ops = {	67	const struct bpf_prog_ops lsm_prog_ops = {
49	};	68	};
50		69
51	const struct bpf_verifier_ops lsm_verifier_ops = {	70	const struct bpf_verifier_ops lsm_verifier_ops = {
52	.get_func_proto = tracing_prog_func_proto,	71	.get_func_proto = bpf_lsm_func_proto,
53	.is_valid_access = btf_ctx_access,	72	.is_valid_access = btf_ctx_access,
54	};	73	};

kernel/bpf/bpf_struct_ops.c+2−4

 			return -EINVAL;
 		mtype = btf_type_by_id(btf_vmlinux, member->type);
-		mtype = btf_resolve_size(btf_vmlinux, mtype, &msize,
+		mtype = btf_resolve_size(btf_vmlinux, mtype, &msize);
-					 NULL, NULL);
 		if (IS_ERR(mtype))
 			return PTR_ERR(mtype);
 		prev_mend = moff + msize;
 			u32 msize;
 			mtype = btf_type_by_id(btf_vmlinux, member->type);
-			mtype = btf_resolve_size(btf_vmlinux, mtype, &msize,
+			mtype = btf_resolve_size(btf_vmlinux, mtype, &msize);
-						 NULL, NULL);
 			if (IS_ERR(mtype)) {
 				err = PTR_ERR(mtype);
 				goto reset_unlock;

kernel/bpf/btf.c+139−24

 #include <linux/btf_ids.h>
 #include <linux/skmsg.h>
 #include <linux/perf_event.h>
+#include <linux/bsearch.h>
+#include <linux/btf_ids.h>
 #include <net/sock.h>
 /* BTF (BPF Type Format) is the meta data format which describes
  * *type_size: (x * y * sizeof(u32)).  Hence, *type_size always
  *             corresponds to the return type.
  * *elem_type: u32
+ * *elem_id: id of u32
  * *total_nelems: (x * y).  Hence, individual elem size is
  *                (*type_size / *total_nelems)
+ * *type_id: id of type if it's changed within the function, 0 if not
  *
  * type: is not an array (e.g. const struct X)
  * return type: type "struct X"
  * *type_size: sizeof(struct X)
  * *elem_type: same as return type ("struct X")
+ * *elem_id: 0
  * *total_nelems: 1
+ * *type_id: id of type if it's changed within the function, 0 if not
  */
-const struct btf_type *
+static const struct btf_type *
-btf_resolve_size(const struct btf *btf, const struct btf_type *type,
+__btf_resolve_size(const struct btf *btf, const struct btf_type *type,
-		 u32 *type_size, const struct btf_type **elem_type,
+		   u32 *type_size, const struct btf_type **elem_type,
-		 u32 *total_nelems)
+		   u32 *elem_id, u32 *total_nelems, u32 *type_id)
 {
 	const struct btf_type *array_type = NULL;
-	const struct btf_array *array;
+	const struct btf_array *array = NULL;
-	u32 i, size, nelems = 1;
+	u32 i, size, nelems = 1, id = 0;
 	for (i = 0; i < MAX_RESOLVE_DEPTH; i++) {
 		switch (BTF_INFO_KIND(type->info)) {
 		case BTF_KIND_VOLATILE:
 		case BTF_KIND_CONST:
 		case BTF_KIND_RESTRICT:
+			id = type->type;
 			type = btf_type_by_id(btf, type->type);
 			break;
 		*total_nelems = nelems;
 	if (elem_type)
 		*elem_type = type;
+	if (elem_id)
+		*elem_id = array ? array->type : 0;
+	if (type_id && id)
+		*type_id = id;
 	return array_type ? : type;
 }
+const struct btf_type *
+btf_resolve_size(const struct btf *btf, const struct btf_type *type,
+		 u32 *type_size)
+{
+	return __btf_resolve_size(btf, type, type_size, NULL, NULL, NULL, NULL);
+}
 /* The input param "type_id" must point to a needs_resolve type */
 static const struct btf_type *btf_type_id_resolve(const struct btf *btf,
 						  u32 *type_id)
 	return true;
 }
-int btf_struct_access(struct bpf_verifier_log *log,
+enum bpf_struct_walk_result {
-		      const struct btf_type *t, int off, int size,
+	/* < 0 error */
-		      enum bpf_access_type atype,
+	WALK_SCALAR = 0,
-		      u32 *next_btf_id)
+	WALK_PTR,
+	WALK_STRUCT,
+};
+static int btf_struct_walk(struct bpf_verifier_log *log,
+			   const struct btf_type *t, int off, int size,
+			   u32 *next_btf_id)
 {
 	u32 i, moff, mtrue_end, msize = 0, total_nelems = 0;
 	const struct btf_type *mtype, *elem_type = NULL;
 	const struct btf_member *member;
 	const char *tname, *mname;
-	u32 vlen;
+	u32 vlen, elem_id, mid;
 again:
 	tname = __btf_name_by_offset(btf_vmlinux, t->name_off);
 		/* Only allow structure for now, can be relaxed for
 		 * other types later.
 		 */
-		elem_type = btf_type_skip_modifiers(btf_vmlinux,
+		t = btf_type_skip_modifiers(btf_vmlinux, array_elem->type,
-						    array_elem->type, NULL);
+					    NULL);
-		if (!btf_type_is_struct(elem_type))
+		if (!btf_type_is_struct(t))
 			goto error;
-		off = (off - moff) % elem_type->size;
+		off = (off - moff) % t->size;
-		return btf_struct_access(log, elem_type, off, size, atype,
+		goto again;
-					 next_btf_id);
 error:
 		bpf_log(log, "access beyond struct %s at off %u size %u\n",
 			 */
 			if (off <= moff &&
 			    BITS_ROUNDUP_BYTES(end_bit) <= off + size)
-				return SCALAR_VALUE;
+				return WALK_SCALAR;
 			/* off may be accessing a following member
 			 *
 			break;
 		/* type of the field */
+		mid = member->type;
 		mtype = btf_type_by_id(btf_vmlinux, member->type);
 		mname = __btf_name_by_offset(btf_vmlinux, member->name_off);
-		mtype = btf_resolve_size(btf_vmlinux, mtype, &msize,
+		mtype = __btf_resolve_size(btf_vmlinux, mtype, &msize,
-					 &elem_type, &total_nelems);
+					   &elem_type, &elem_id, &total_nelems,
+					   &mid);
 		if (IS_ERR(mtype)) {
 			bpf_log(log, "field %s doesn't have size\n", mname);
 			return -EFAULT;
 		if (btf_type_is_array(mtype)) {
 			u32 elem_idx;
-			/* btf_resolve_size() above helps to
+			/* __btf_resolve_size() above helps to
 			 * linearize a multi-dimensional array.
 			 *
 			 * The logic here is treating an array
 			elem_idx = (off - moff) / msize;
 			moff += elem_idx * msize;
 			mtype = elem_type;
+			mid = elem_id;
 		}
 		/* the 'off' we're looking for is either equal to start
 			/* our field must be inside that union or struct */
 			t = mtype;
+			/* return if the offset matches the member offset */
+			if (off == moff) {
+				*next_btf_id = mid;
+				return WALK_STRUCT;
+			}
 			/* adjust offset we're looking for */
 			off -= moff;
 			goto again;
 					mname, moff, tname, off, size);
 				return -EACCES;
 			}
 			stype = btf_type_skip_modifiers(btf_vmlinux, mtype->type, &id);
 			if (btf_type_is_struct(stype)) {
 				*next_btf_id = id;
-				return PTR_TO_BTF_ID;
+				return WALK_PTR;
 			}
 		}
 			return -EACCES;
 		}
-		return SCALAR_VALUE;
+		return WALK_SCALAR;
 	}
 	bpf_log(log, "struct %s doesn't have field at offset %d\n", tname, off);
 	return -EINVAL;
 }
+int btf_struct_access(struct bpf_verifier_log *log,
+		      const struct btf_type *t, int off, int size,
+		      enum bpf_access_type atype __maybe_unused,
+		      u32 *next_btf_id)
+{
+	int err;
+	u32 id;
+	do {
+		err = btf_struct_walk(log, t, off, size, &id);
+		switch (err) {
+		case WALK_PTR:
+			/* If we found the pointer or scalar on t+off,
+			 * we're done.
+			 */
+			*next_btf_id = id;
+			return PTR_TO_BTF_ID;
+		case WALK_SCALAR:
+			return SCALAR_VALUE;
+		case WALK_STRUCT:
+			/* We found nested struct, so continue the search
+			 * by diving in it. At this point the offset is
+			 * aligned with the new type, so set it to 0.
+			 */
+			t = btf_type_by_id(btf_vmlinux, id);
+			off = 0;
+			break;
+		default:
+			/* It's either error or unknown return value..
+			 * scream and leave.
+			 */
+			if (WARN_ONCE(err > 0, "unknown btf_struct_walk return value"))
+				return -EINVAL;
+			return err;
+		}
+	} while (t);
+	return -EINVAL;
+}
+bool btf_struct_ids_match(struct bpf_verifier_log *log,
+			  int off, u32 id, u32 need_type_id)
+{
+	const struct btf_type *type;
+	int err;
+	/* Are we already done? */
+	if (need_type_id == id && off == 0)
+		return true;
+again:
+	type = btf_type_by_id(btf_vmlinux, id);
+	if (!type)
+		return false;
+	err = btf_struct_walk(log, type, off, 1, &id);
+	if (err != WALK_STRUCT)
+		return false;
+	/* We found nested struct object. If it matches
+	 * the requested ID, we're done. Otherwise let's
+	 * continue the search with offset 0 in the new
+	 * type.
+	 */
+	if (need_type_id != id) {
+		off = 0;
+		goto again;
+	}
+	return true;
+}
 int btf_resolve_helper_id(struct bpf_verifier_log *log,
 			  const struct bpf_func_proto *fn, int arg)
 {
 {
 	return btf->id;
 }
+static int btf_id_cmp_func(const void *a, const void *b)
+{
+	const int *pa = a, *pb = b;
+	return *pa - *pb;
+}
+bool btf_id_set_contains(struct btf_id_set *set, u32 id)
+{
+	return bsearch(&id, set->ids, set->cnt, sizeof(u32), btf_id_cmp_func) != NULL;
+}

kernel/bpf/cpumap.c+4−8

 static DEFINE_PER_CPU(struct list_head, cpu_map_flush_list);
-static int bq_flush_to_queue(struct xdp_bulk_queue *bq);
 static struct bpf_map *cpu_map_alloc(union bpf_attr *attr)
 {
 	u32 value_size = attr->value_size;
 static int cpu_map_btf_id;
 const struct bpf_map_ops cpu_map_ops = {
+	.map_meta_equal		= bpf_map_meta_equal,
 	.map_alloc		= cpu_map_alloc,
 	.map_free		= cpu_map_free,
 	.map_delete_elem	= cpu_map_delete_elem,
 	.map_btf_id		= &cpu_map_btf_id,
 };
-static int bq_flush_to_queue(struct xdp_bulk_queue *bq)
+static void bq_flush_to_queue(struct xdp_bulk_queue *bq)
 {
 	struct bpf_cpu_map_entry *rcpu = bq->obj;
 	unsigned int processed = 0, drops = 0;
 	int i;
 	if (unlikely(!bq->count))
-		return 0;
+		return;
 	q = rcpu->queue;
 	spin_lock(&q->producer_lock);
 	/* Feedback loop via tracepoints */
 	trace_xdp_cpumap_enqueue(rcpu->map_id, processed, drops, to_cpu);
-	return 0;
 }
 /* Runs under RCU-read-side, plus in softirq under NAPI protection.
  * Thus, safe percpu variable access.
  */
-static int bq_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf)
+static void bq_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf)
 {
 	struct list_head *flush_list = this_cpu_ptr(&cpu_map_flush_list);
 	struct xdp_bulk_queue *bq = this_cpu_ptr(rcpu->bulkq);
 	if (!bq->flush_node.prev)
 		list_add(&bq->flush_node, flush_list);
-	return 0;
 }
 int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp,

kernel/bpf/devmap.c+9−8

 	return false;
 }
-static int bq_xmit_all(struct xdp_dev_bulk_queue *bq, u32 flags)
+static void bq_xmit_all(struct xdp_dev_bulk_queue *bq, u32 flags)
 {
 	struct net_device *dev = bq->dev;
 	int sent = 0, drops = 0, err = 0;
 	int i;
 	if (unlikely(!bq->count))
-		return 0;
+		return;
 	for (i = 0; i < bq->count; i++) {
 		struct xdp_frame *xdpf = bq->q[i];
 	trace_xdp_devmap_xmit(bq->dev_rx, dev, sent, drops, err);
 	bq->dev_rx = NULL;
 	__list_del_clearprev(&bq->flush_node);
-	return 0;
+	return;
 error:
 	/* If ndo_xdp_xmit fails with an errno, no frames have been
 	 * xmit'ed and it's our responsibility to them free all.
 /* Runs under RCU-read-side, plus in softirq under NAPI protection.
  * Thus, safe percpu variable access.
  */
-static int bq_enqueue(struct net_device *dev, struct xdp_frame *xdpf,
+static void bq_enqueue(struct net_device *dev, struct xdp_frame *xdpf,
-		      struct net_device *dev_rx)
+		       struct net_device *dev_rx)
 {
 	struct list_head *flush_list = this_cpu_ptr(&dev_flush_list);
 	struct xdp_dev_bulk_queue *bq = this_cpu_ptr(dev->xdp_bulkq);
 	if (!bq->flush_node.prev)
 		list_add(&bq->flush_node, flush_list);
-	return 0;
 }
 static inline int __xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp,
 	if (unlikely(!xdpf))
 		return -EOVERFLOW;
-	return bq_enqueue(dev, xdpf, dev_rx);
+	bq_enqueue(dev, xdpf, dev_rx);
+	return 0;
 }
 static struct xdp_buff *dev_map_run_prog(struct net_device *dev,
 static int dev_map_btf_id;
 const struct bpf_map_ops dev_map_ops = {
+	.map_meta_equal = bpf_map_meta_equal,
 	.map_alloc = dev_map_alloc,
 	.map_free = dev_map_free,
 	.map_get_next_key = dev_map_get_next_key,
 static int dev_map_hash_map_btf_id;
 const struct bpf_map_ops dev_map_hash_ops = {
+	.map_meta_equal = bpf_map_meta_equal,
 	.map_alloc = dev_map_alloc,
 	.map_free = dev_map_free,
 	.map_get_next_key = dev_map_hash_get_next_key,

kernel/bpf/hashtab.c+10−6

 #include <linux/rculist_nulls.h>
 #include <linux/random.h>
 #include <uapi/linux/btf.h>
+#include <linux/rcupdate_trace.h>
 #include "percpu_freelist.h"
 #include "bpf_lru_list.h"
 #include "map_in_map.h"
 	struct htab_elem *l;
 	u32 hash, key_size;
-	/* Must be called with rcu_read_lock. */
+	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
-	WARN_ON_ONCE(!rcu_read_lock_held());
 	key_size = map->key_size;
 		/* unknown flags */
 		return -EINVAL;
-	WARN_ON_ONCE(!rcu_read_lock_held());
+	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
 	key_size = map->key_size;
 		/* unknown flags */
 		return -EINVAL;
-	WARN_ON_ONCE(!rcu_read_lock_held());
+	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
 	key_size = map->key_size;
 	u32 hash, key_size;
 	int ret = -ENOENT;
-	WARN_ON_ONCE(!rcu_read_lock_held());
+	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
 	key_size = map->key_size;
 	u32 hash, key_size;
 	int ret = -ENOENT;
-	WARN_ON_ONCE(!rcu_read_lock_held());
+	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
 	key_size = map->key_size;
 static int htab_map_btf_id;
 const struct bpf_map_ops htab_map_ops = {
+	.map_meta_equal = bpf_map_meta_equal,
 	.map_alloc_check = htab_map_alloc_check,
 	.map_alloc = htab_map_alloc,
 	.map_free = htab_map_free,
 static int htab_lru_map_btf_id;
 const struct bpf_map_ops htab_lru_map_ops = {
+	.map_meta_equal = bpf_map_meta_equal,
 	.map_alloc_check = htab_map_alloc_check,
 	.map_alloc = htab_map_alloc,
 	.map_free = htab_map_free,
 static int htab_percpu_map_btf_id;
 const struct bpf_map_ops htab_percpu_map_ops = {
+	.map_meta_equal = bpf_map_meta_equal,
 	.map_alloc_check = htab_map_alloc_check,
 	.map_alloc = htab_map_alloc,
 	.map_free = htab_map_free,
 static int htab_lru_percpu_map_btf_id;
 const struct bpf_map_ops htab_lru_percpu_map_ops = {
+	.map_meta_equal = bpf_map_meta_equal,
 	.map_alloc_check = htab_map_alloc_check,
 	.map_alloc = htab_map_alloc,
 	.map_free = htab_map_free,

kernel/bpf/helpers.c+22−0

@@ -601,6 +601,28 @@ const struct bpf_func_proto bpf_event_output_data_proto = { · 600 unmodified lines
601	.arg5_type = ARG_CONST_SIZE_OR_ZERO,	601	.arg5_type = ARG_CONST_SIZE_OR_ZERO,
602	};	602	};
603		603
		604	BPF_CALL_3(bpf_copy_from_user, void *, dst, u32, size,
		605	const void __user *, user_ptr)
		606	{
		607	int ret = copy_from_user(dst, user_ptr, size);
		608
		609	if (unlikely(ret)) {
		610	memset(dst, 0, size);
		611	ret = -EFAULT;
		612	}
		613
		614	return ret;
		615	}
		616
		617	const struct bpf_func_proto bpf_copy_from_user_proto = {
		618	.func = bpf_copy_from_user,
		619	.gpl_only = false,
		620	.ret_type = RET_INTEGER,
		621	.arg1_type = ARG_PTR_TO_UNINIT_MEM,
		622	.arg2_type = ARG_CONST_SIZE_OR_ZERO,
		623	.arg3_type = ARG_ANYTHING,
		624	};
		625
604	const struct bpf_func_proto bpf_get_current_task_proto __weak;	626	const struct bpf_func_proto bpf_get_current_task_proto __weak;
605	const struct bpf_func_proto bpf_probe_read_user_proto __weak;	627	const struct bpf_func_proto bpf_probe_read_user_proto __weak;
606	const struct bpf_func_proto bpf_probe_read_user_str_proto __weak;	628	const struct bpf_func_proto bpf_probe_read_user_str_proto __weak;

kernel/bpf/inode.c+113−3

@@ -20,6 +20,7 @@ · 19 unmodified lines
20	#include <linux/filter.h>	20	#include <linux/filter.h>
21	#include <linux/bpf.h>	21	#include <linux/bpf.h>
22	#include <linux/bpf_trace.h>	22	#include <linux/bpf_trace.h>
		23	#include "preload/bpf_preload.h"
23		24
24	enum bpf_type {	25	enum bpf_type {
25	BPF_TYPE_UNSPEC = 0,	26	BPF_TYPE_UNSPEC = 0,
@@ -369,9 +370,10 @@ static struct dentry * · 343 unmodified lines
369	bpf_lookup(struct inode dir, struct dentry dentry, unsigned flags)	370	bpf_lookup(struct inode dir, struct dentry dentry, unsigned flags)
370	{	371	{
371	/* Dots in names (e.g. "/sys/fs/bpf/foo.bar") are reserved for future	372	/* Dots in names (e.g. "/sys/fs/bpf/foo.bar") are reserved for future
372	* extensions.	373	* extensions. That allows popoulate_bpffs() create special files.
373	*/	374	*/
374	if (strchr(dentry->d_name.name, '.'))	375	if ((dir->i_mode & S_IALLUGO) &&
		376	strchr(dentry->d_name.name, '.'))
375	return ERR_PTR(-EPERM);	377	return ERR_PTR(-EPERM);
376		378
377	return simple_lookup(dir, dentry, flags);	379	return simple_lookup(dir, dentry, flags);
@@ -409,6 +411,27 @@ static const struct inode_operations bpf_dir_iops = { · 31 unmodified lines
409	.unlink = simple_unlink,	411	.unlink = simple_unlink,
410	};	412	};
411		413
		414	/* pin iterator link into bpffs */
		415	static int bpf_iter_link_pin_kernel(struct dentry *parent,
		416	const char name, struct bpf_link link)
		417	{
		418	umode_t mode = S_IFREG \| S_IRUSR;
		419	struct dentry *dentry;
		420	int ret;
		421
		422	inode_lock(parent->d_inode);
		423	dentry = lookup_one_len(name, parent, strlen(name));
		424	if (IS_ERR(dentry)) {
		425	inode_unlock(parent->d_inode);
		426	return PTR_ERR(dentry);
		427	}
		428	ret = bpf_mkobj_ops(dentry, mode, link, &bpf_link_iops,
		429	&bpf_iter_fops);
		430	dput(dentry);
		431	inode_unlock(parent->d_inode);
		432	return ret;
		433	}
		434
412	static int bpf_obj_do_pin(const char __user pathname, void raw,	435	static int bpf_obj_do_pin(const char __user pathname, void raw,
413	enum bpf_type type)	436	enum bpf_type type)
414	{	437	{
@@ -638,6 +661,91 @@ static int bpf_parse_param(struct fs_context fc, struct fs_parameter param) · 223 unmodified lines
638	return 0;	661	return 0;
639	}	662	}
640		663
		664	struct bpf_preload_ops *bpf_preload_ops;
		665	EXPORT_SYMBOL_GPL(bpf_preload_ops);
		666
		667	static bool bpf_preload_mod_get(void)
		668	{
		669	/* If bpf_preload.ko wasn't loaded earlier then load it now.
		670	* When bpf_preload is built into vmlinux the module's __init
		671	* function will populate it.
		672	*/
		673	if (!bpf_preload_ops) {
		674	request_module("bpf_preload");
		675	if (!bpf_preload_ops)
		676	return false;
		677	}
		678	/* And grab the reference, so the module doesn't disappear while the
		679	* kernel is interacting with the kernel module and its UMD.
		680	*/
		681	if (!try_module_get(bpf_preload_ops->owner)) {
		682	pr_err("bpf_preload module get failed.\n");
		683	return false;
		684	}
		685	return true;
		686	}
		687
		688	static void bpf_preload_mod_put(void)
		689	{
		690	if (bpf_preload_ops)
		691	/* now user can "rmmod bpf_preload" if necessary */
		692	module_put(bpf_preload_ops->owner);
		693	}
		694
		695	static DEFINE_MUTEX(bpf_preload_lock);
		696
		697	static int populate_bpffs(struct dentry *parent)
		698	{
		699	struct bpf_preload_info objs[BPF_PRELOAD_LINKS] = {};
		700	struct bpf_link *links[BPF_PRELOAD_LINKS] = {};
		701	int err = 0, i;
		702
		703	/* grab the mutex to make sure the kernel interactions with bpf_preload
		704	* UMD are serialized
		705	*/
		706	mutex_lock(&bpf_preload_lock);
		707
		708	/* if bpf_preload.ko wasn't built into vmlinux then load it */
		709	if (!bpf_preload_mod_get())
		710	goto out;
		711
		712	if (!bpf_preload_ops->info.tgid) {
		713	/* preload() will start UMD that will load BPF iterator programs */
		714	err = bpf_preload_ops->preload(objs);
		715	if (err)
		716	goto out_put;
		717	for (i = 0; i < BPF_PRELOAD_LINKS; i++) {
		718	links[i] = bpf_link_by_id(objs[i].link_id);
		719	if (IS_ERR(links[i])) {
		720	err = PTR_ERR(links[i]);
		721	goto out_put;
		722	}
		723	}
		724	for (i = 0; i < BPF_PRELOAD_LINKS; i++) {
		725	err = bpf_iter_link_pin_kernel(parent,
		726	objs[i].link_name, links[i]);
		727	if (err)
		728	goto out_put;
		729	/* do not unlink successfully pinned links even
		730	* if later link fails to pin
		731	*/
		732	links[i] = NULL;
		733	}
		734	/* finish() will tell UMD process to exit */
		735	err = bpf_preload_ops->finish();
		736	if (err)
		737	goto out_put;
		738	}
		739	out_put:
		740	bpf_preload_mod_put();
		741	out:
		742	mutex_unlock(&bpf_preload_lock);
		743	for (i = 0; i < BPF_PRELOAD_LINKS && err; i++)
		744	if (!IS_ERR_OR_NULL(links[i]))
		745	bpf_link_put(links[i]);
		746	return err;
		747	}
		748
641	static int bpf_fill_super(struct super_block sb, struct fs_context fc)	749	static int bpf_fill_super(struct super_block sb, struct fs_context fc)
642	{	750	{
643	static const struct tree_descr bpf_rfiles[] = { { "" } };	751	static const struct tree_descr bpf_rfiles[] = { { "" } };
@@ -654,8 +762,8 @@ static int bpf_fill_super(struct super_block sb, struct fs_context fc) · 10 unmodified lines
654	inode = sb->s_root->d_inode;	762	inode = sb->s_root->d_inode;
655	inode->i_op = &bpf_dir_iops;	763	inode->i_op = &bpf_dir_iops;
656	inode->i_mode &= ~S_IALLUGO;	764	inode->i_mode &= ~S_IALLUGO;
		765	populate_bpffs(sb->s_root);
657	inode->i_mode \|= S_ISVTX \| opts->mode;	766	inode->i_mode \|= S_ISVTX \| opts->mode;
658
659	return 0;	767	return 0;
660	}	768	}
661		769
@@ -705,6 +813,8 @@ static int __init bpf_init(void) · 43 unmodified lines
705	{	813	{
706	int ret;	814	int ret;
707		815
		816	mutex_init(&bpf_preload_lock);
		817
708	ret = sysfs_create_mount_point(fs_kobj, "bpf");	818	ret = sysfs_create_mount_point(fs_kobj, "bpf");
709	if (ret)	819	if (ret)
710	return ret;	820	return ret;

kernel/bpf/lpm_trie.c+1−0

@@ -732,6 +732,7 @@ static int trie_check_btf(const struct bpf_map *map, · 731 unmodified lines
732		732
733	static int trie_map_btf_id;	733	static int trie_map_btf_id;
734	const struct bpf_map_ops trie_map_ops = {	734	const struct bpf_map_ops trie_map_ops = {
		735	.map_meta_equal = bpf_map_meta_equal,
735	.map_alloc = trie_alloc,	736	.map_alloc = trie_alloc,
736	.map_free = trie_free,	737	.map_free = trie_free,
737	.map_get_next_key = trie_get_next_key,	738	.map_get_next_key = trie_get_next_key,

kernel/bpf/map_in_map.c+9−15

 	if (IS_ERR(inner_map))
 		return inner_map;
-	/* prog_array->aux->{type,jited} is a runtime binding.
-	 * Doing static check alone in the verifier is not enough.
-	 */
-	if (inner_map->map_type == BPF_MAP_TYPE_PROG_ARRAY ||
-	    inner_map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE ||
-	    inner_map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE ||
-	    inner_map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
-		fdput(f);
-		return ERR_PTR(-ENOTSUPP);
-	}
 	/* Does not support >1 level map-in-map */
 	if (inner_map->inner_map_meta) {
 		fdput(f);
 		return ERR_PTR(-EINVAL);
 	}
+	if (!inner_map->ops->map_meta_equal) {
+		fdput(f);
+		return ERR_PTR(-ENOTSUPP);
+	}
 	if (map_value_has_spin_lock(inner_map)) {
 		fdput(f);
 		return ERR_PTR(-ENOTSUPP);
 	return meta0->map_type == meta1->map_type &&
 		meta0->key_size == meta1->key_size &&
 		meta0->value_size == meta1->value_size &&
-		meta0->map_flags == meta1->map_flags &&
+		meta0->map_flags == meta1->map_flags;
-		meta0->max_entries == meta1->max_entries;
 }
 void *bpf_map_fd_get_ptr(struct bpf_map *map,
 			 struct file *map_file /* not used */,
 			 int ufd)
 {
-	struct bpf_map *inner_map;
+	struct bpf_map *inner_map, *inner_map_meta;
 	struct fd f;
 	f = fdget(ufd);
 	if (IS_ERR(inner_map))
 		return inner_map;
-	if (bpf_map_meta_equal(map->inner_map_meta, inner_map))
+	inner_map_meta = map->inner_map_meta;
+	if (inner_map_meta->ops->map_meta_equal(inner_map_meta, inner_map))
 		bpf_map_inc(inner_map);
 	else
 		inner_map = ERR_PTR(-EINVAL);

kernel/bpf/map_in_map.h+0−2

@@ -11,8 +11,6 @@ struct bpf_map; · 10 unmodified lines
11		11
12	struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd);	12	struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd);
13	void bpf_map_meta_free(struct bpf_map *map_meta);	13	void bpf_map_meta_free(struct bpf_map *map_meta);
14	bool bpf_map_meta_equal(const struct bpf_map *meta0,
15	const struct bpf_map *meta1);
16	void bpf_map_fd_get_ptr(struct bpf_map map, struct file *map_file,	14	void bpf_map_fd_get_ptr(struct bpf_map map, struct file *map_file,
17	int ufd);	15	int ufd);
18	void bpf_map_fd_put_ptr(void *ptr);	16	void bpf_map_fd_put_ptr(void *ptr);

kernel/bpf/map_iter.c+15−0

@@ -149,6 +149,19 @@ static void bpf_iter_detach_map(struct bpf_iter_aux_info *aux) · 148 unmodified lines
149	bpf_map_put_with_uref(aux->map);	149	bpf_map_put_with_uref(aux->map);
150	}	150	}
151		151
		152	void bpf_iter_map_show_fdinfo(const struct bpf_iter_aux_info *aux,
		153	struct seq_file *seq)
		154	{
		155	seq_printf(seq, "map_id:\t%u\n", aux->map->id);
		156	}
		157
		158	int bpf_iter_map_fill_link_info(const struct bpf_iter_aux_info *aux,
		159	struct bpf_link_info *info)
		160	{
		161	info->iter.map.map_id = aux->map->id;
		162	return 0;
		163	}
		164
152	DEFINE_BPF_ITER_FUNC(bpf_map_elem, struct bpf_iter_meta *meta,	165	DEFINE_BPF_ITER_FUNC(bpf_map_elem, struct bpf_iter_meta *meta,
153	struct bpf_map map, void key, void *value)	166	struct bpf_map map, void key, void *value)
154		167
@@ -156,6 +169,8 @@ static const struct bpf_iter_reg bpf_map_elem_reg_info = { · 1 unmodified line
156	.target = "bpf_map_elem",	169	.target = "bpf_map_elem",
157	.attach_target = bpf_iter_attach_map,	170	.attach_target = bpf_iter_attach_map,
158	.detach_target = bpf_iter_detach_map,	171	.detach_target = bpf_iter_detach_map,
		172	.show_fdinfo = bpf_iter_map_show_fdinfo,
		173	.fill_link_info = bpf_iter_map_fill_link_info,
159	.ctx_arg_info_size = 2,	174	.ctx_arg_info_size = 2,
160	.ctx_arg_info = {	175	.ctx_arg_info = {
161	{ offsetof(struct bpf_iter__bpf_map_elem, key),	176	{ offsetof(struct bpf_iter__bpf_map_elem, key),

kernel/bpf/preload/Kconfig+26−0

@@ -0,0 +1,26 @@
	1	# SPDX-License-Identifier: GPL-2.0-only
	2	config USERMODE_DRIVER
	3	bool
	4	default n
	5
	6	menuconfig BPF_PRELOAD
	7	bool "Preload BPF file system with kernel specific program and map iterators"
	8	depends on BPF
	9	# The dependency on !COMPILE_TEST prevents it from being enabled
	10	# in allmodconfig or allyesconfig configurations
	11	depends on !COMPILE_TEST
	12	select USERMODE_DRIVER
	13	help
	14	This builds kernel module with several embedded BPF programs that are
	15	pinned into BPF FS mount point as human readable files that are
	16	useful in debugging and introspection of BPF programs and maps.
	17
	18	if BPF_PRELOAD
	19	config BPF_PRELOAD_UMD
	20	tristate "bpf_preload kernel module with user mode driver"
	21	depends on CC_CAN_LINK
	22	depends on m \|\| CC_CAN_LINK_STATIC
	23	default m
	24	help
	25	This builds bpf_preload kernel module with embedded user mode driver.
	26	endif

kernel/bpf/preload/Makefile+23−0

@@ -0,0 +1,23 @@
	1	# SPDX-License-Identifier: GPL-2.0
	2
	3	LIBBPF_SRCS = $(srctree)/tools/lib/bpf/
	4	LIBBPF_A = $(obj)/libbpf.a
	5	LIBBPF_OUT = $(abspath $(obj))
	6
	7	$(LIBBPF_A):
	8	$(Q)$(MAKE) -C $(LIBBPF_SRCS) OUTPUT=$(LIBBPF_OUT)/ $(LIBBPF_OUT)/libbpf.a
	9
	10	userccflags += -I $(srctree)/tools/include/ -I $(srctree)/tools/include/uapi \
	11	-I $(srctree)/tools/lib/ -Wno-unused-result
	12
	13	userprogs := bpf_preload_umd
	14
	15	bpf_preload_umd-objs := iterators/iterators.o
	16	bpf_preload_umd-userldlibs := $(LIBBPF_A) -lelf -lz
	17
	18	$(obj)/bpf_preload_umd: $(LIBBPF_A)
	19
	20	$(obj)/bpf_preload_umd_blob.o: $(obj)/bpf_preload_umd
	21
	22	obj-$(CONFIG_BPF_PRELOAD_UMD) += bpf_preload.o
	23	bpf_preload-objs += bpf_preload_kern.o bpf_preload_umd_blob.o

kernel/bpf/preload/bpf_preload.h+16−0

@@ -0,0 +1,16 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	#ifndef _BPF_PRELOAD_H
	3	#define _BPF_PRELOAD_H
	4
	5	#include <linux/usermode_driver.h>
	6	#include "iterators/bpf_preload_common.h"
	7
	8	struct bpf_preload_ops {
	9	struct umd_info info;
	10	int (preload)(struct bpf_preload_info );
	11	int (*finish)(void);
	12	struct module *owner;
	13	};
	14	extern struct bpf_preload_ops *bpf_preload_ops;
	15	#define BPF_PRELOAD_LINKS 2
	16	#endif

kernel/bpf/preload/bpf_preload_kern.c+91−0

@@ -0,0 +1,91 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
	3	#include <linux/init.h>
	4	#include <linux/module.h>
	5	#include <linux/pid.h>
	6	#include <linux/fs.h>
	7	#include <linux/sched/signal.h>
	8	#include "bpf_preload.h"
	9
	10	extern char bpf_preload_umd_start;
	11	extern char bpf_preload_umd_end;
	12
	13	static int preload(struct bpf_preload_info *obj);
	14	static int finish(void);
	15
	16	static struct bpf_preload_ops umd_ops = {
	17	.info.driver_name = "bpf_preload",
	18	.preload = preload,
	19	.finish = finish,
	20	.owner = THIS_MODULE,
	21	};
	22
	23	static int preload(struct bpf_preload_info *obj)
	24	{
	25	int magic = BPF_PRELOAD_START;
	26	loff_t pos = 0;
	27	int i, err;
	28	ssize_t n;
	29
	30	err = fork_usermode_driver(&umd_ops.info);
	31	if (err)
	32	return err;
	33
	34	/* send the start magic to let UMD proceed with loading BPF progs */
	35	n = kernel_write(umd_ops.info.pipe_to_umh,
	36	&magic, sizeof(magic), &pos);
	37	if (n != sizeof(magic))
	38	return -EPIPE;
	39
	40	/* receive bpf_link IDs and names from UMD */
	41	pos = 0;
	42	for (i = 0; i < BPF_PRELOAD_LINKS; i++) {
	43	n = kernel_read(umd_ops.info.pipe_from_umh,
	44	&obj[i], sizeof(*obj), &pos);
	45	if (n != sizeof(*obj))
	46	return -EPIPE;
	47	}
	48	return 0;
	49	}
	50
	51	static int finish(void)
	52	{
	53	int magic = BPF_PRELOAD_END;
	54	struct pid *tgid;
	55	loff_t pos = 0;
	56	ssize_t n;
	57
	58	/* send the last magic to UMD. It will do a normal exit. */
	59	n = kernel_write(umd_ops.info.pipe_to_umh,
	60	&magic, sizeof(magic), &pos);
	61	if (n != sizeof(magic))
	62	return -EPIPE;
	63	tgid = umd_ops.info.tgid;
	64	wait_event(tgid->wait_pidfd, thread_group_exited(tgid));
	65	umd_ops.info.tgid = NULL;
	66	return 0;
	67	}
	68
	69	static int __init load_umd(void)
	70	{
	71	int err;
	72
	73	err = umd_load_blob(&umd_ops.info, &bpf_preload_umd_start,
	74	&bpf_preload_umd_end - &bpf_preload_umd_start);
	75	if (err)
	76	return err;
	77	bpf_preload_ops = &umd_ops;
	78	return err;
	79	}
	80
	81	static void __exit fini_umd(void)
	82	{
	83	bpf_preload_ops = NULL;
	84	/* kill UMD in case it's still there due to earlier error */
	85	kill_pid(umd_ops.info.tgid, SIGKILL, 1);
	86	umd_ops.info.tgid = NULL;
	87	umd_unload_blob(&umd_ops.info);
	88	}
	89	late_initcall(load_umd);
	90	module_exit(fini_umd);
	91	MODULE_LICENSE("GPL");

kernel/bpf/preload/bpf_preload_umd_blob.S+7−0

@@ -0,0 +1,7 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	.section .init.rodata, "a"
	3	.global bpf_preload_umd_start
	4	bpf_preload_umd_start:
	5	.incbin "kernel/bpf/preload/bpf_preload_umd"
	6	.global bpf_preload_umd_end
	7	bpf_preload_umd_end:

kernel/bpf/preload/iterators/.gitignore+2−0

@@ -0,0 +1,2 @@
	1	# SPDX-License-Identifier: GPL-2.0-only
	2	/.output

kernel/bpf/preload/iterators/Makefile+57−0

@@ -0,0 +1,57 @@
	1	# SPDX-License-Identifier: GPL-2.0
	2	OUTPUT := .output
	3	CLANG ?= clang
	4	LLC ?= llc
	5	LLVM_STRIP ?= llvm-strip
	6	DEFAULT_BPFTOOL := $(OUTPUT)/sbin/bpftool
	7	BPFTOOL ?= $(DEFAULT_BPFTOOL)
	8	LIBBPF_SRC := $(abspath ../../../../tools/lib/bpf)
	9	BPFOBJ := $(OUTPUT)/libbpf.a
	10	BPF_INCLUDE := $(OUTPUT)
	11	INCLUDES := -I$(OUTPUT) -I$(BPF_INCLUDE) -I$(abspath ../../../../tools/lib) \
	12	-I$(abspath ../../../../tools/include/uapi)
	13	CFLAGS := -g -Wall
	14
	15	abs_out := $(abspath $(OUTPUT))
	16	ifeq ($(V),1)
	17	Q =
	18	msg =
	19	else
	20	Q = @
	21	msg = @printf ' %-8s %s%s\n' "$(1)" "$(notdir $(2))" "$(if $(3), $(3))";
	22	MAKEFLAGS += --no-print-directory
	23	submake_extras := feature_display=0
	24	endif
	25
	26	.DELETE_ON_ERROR:
	27
	28	.PHONY: all clean
	29
	30	all: iterators.skel.h
	31
	32	clean:
	33	$(call msg,CLEAN)
	34	$(Q)rm -rf $(OUTPUT) iterators
	35
	36	iterators.skel.h: $(OUTPUT)/iterators.bpf.o \| $(BPFTOOL)
	37	$(call msg,GEN-SKEL,$@)
	38	$(Q)$(BPFTOOL) gen skeleton $< > $@
	39
	40
	41	$(OUTPUT)/iterators.bpf.o: iterators.bpf.c $(BPFOBJ) \| $(OUTPUT)
	42	$(call msg,BPF,$@)
	43	$(Q)$(CLANG) -g -O2 -target bpf $(INCLUDES) \
	44	-c $(filter %.c,$^) -o $@ && \
	45	$(LLVM_STRIP) -g $@
	46
	47	$(OUTPUT):
	48	$(call msg,MKDIR,$@)
	49	$(Q)mkdir -p $(OUTPUT)
	50
	51	$(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) \| $(OUTPUT)
	52	$(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) \
	53	OUTPUT=$(abspath $(dir $@))/ $(abspath $@)
	54
	55	$(DEFAULT_BPFTOOL):
	56	$(Q)$(MAKE) $(submake_extras) -C ../../../../tools/bpf/bpftool \
	57	prefix= OUTPUT=$(abs_out)/ DESTDIR=$(abs_out) install

kernel/bpf/preload/iterators/README+4−0

@@ -0,0 +1,4 @@
	1	WARNING:
	2	If you change "iterators.bpf.c" do "make -j" in this directory to rebuild "iterators.skel.h".
	3	Make sure to have clang 10 installed.
	4	See Documentation/bpf/bpf_devel_QA.rst

kernel/bpf/preload/iterators/bpf_preload_common.h+13−0

@@ -0,0 +1,13 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	#ifndef _BPF_PRELOAD_COMMON_H
	3	#define _BPF_PRELOAD_COMMON_H
	4
	5	#define BPF_PRELOAD_START 0x5555
	6	#define BPF_PRELOAD_END 0xAAAA
	7
	8	struct bpf_preload_info {
	9	char link_name[16];
	10	int link_id;
	11	};
	12
	13	#endif

kernel/bpf/preload/iterators/iterators.bpf.c+114−0

@@ -0,0 +1,114 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2020 Facebook */
	3	#include <linux/bpf.h>
	4	#include <bpf/bpf_helpers.h>
	5	#include <bpf/bpf_tracing.h>
	6	#include <bpf/bpf_core_read.h>
	7
	8	#pragma clang attribute push (__attribute__((preserve_access_index)), apply_to = record)
	9	struct seq_file;
	10	struct bpf_iter_meta {
	11	struct seq_file *seq;
	12	__u64 session_id;
	13	__u64 seq_num;
	14	};
	15
	16	struct bpf_map {
	17	__u32 id;
	18	char name[16];
	19	__u32 max_entries;
	20	};
	21
	22	struct bpf_iter__bpf_map {
	23	struct bpf_iter_meta *meta;
	24	struct bpf_map *map;
	25	};
	26
	27	struct btf_type {
	28	__u32 name_off;
	29	};
	30
	31	struct btf_header {
	32	__u32 str_len;
	33	};
	34
	35	struct btf {
	36	const char *strings;
	37	struct btf_type **types;
	38	struct btf_header hdr;
	39	};
	40
	41	struct bpf_prog_aux {
	42	__u32 id;
	43	char name[16];
	44	const char *attach_func_name;
	45	struct bpf_prog *linked_prog;
	46	struct bpf_func_info *func_info;
	47	struct btf *btf;
	48	};
	49
	50	struct bpf_prog {
	51	struct bpf_prog_aux *aux;
	52	};
	53
	54	struct bpf_iter__bpf_prog {
	55	struct bpf_iter_meta *meta;
	56	struct bpf_prog *prog;
	57	};
	58	#pragma clang attribute pop
	59
	60	static const char get_name(struct btf btf, long btf_id, const char *fallback)
	61	{
	62	struct btf_type *types, t;
	63	unsigned int name_off;
	64	const char *str;
	65
	66	if (!btf)
	67	return fallback;
	68	str = btf->strings;
	69	types = btf->types;
	70	bpf_probe_read_kernel(&t, sizeof(t), types + btf_id);
	71	name_off = BPF_CORE_READ(t, name_off);
	72	if (name_off >= btf->hdr.str_len)
	73	return fallback;
	74	return str + name_off;
	75	}
	76
	77	SEC("iter/bpf_map")
	78	int dump_bpf_map(struct bpf_iter__bpf_map *ctx)
	79	{
	80	struct seq_file *seq = ctx->meta->seq;
	81	__u64 seq_num = ctx->meta->seq_num;
	82	struct bpf_map *map = ctx->map;
	83
	84	if (!map)
	85	return 0;
	86
	87	if (seq_num == 0)
	88	BPF_SEQ_PRINTF(seq, " id name max_entries\n");
	89
	90	BPF_SEQ_PRINTF(seq, "%4u %-16s%6d\n", map->id, map->name, map->max_entries);
	91	return 0;
	92	}
	93
	94	SEC("iter/bpf_prog")
	95	int dump_bpf_prog(struct bpf_iter__bpf_prog *ctx)
	96	{
	97	struct seq_file *seq = ctx->meta->seq;
	98	__u64 seq_num = ctx->meta->seq_num;
	99	struct bpf_prog *prog = ctx->prog;
	100	struct bpf_prog_aux *aux;
	101
	102	if (!prog)
	103	return 0;
	104
	105	aux = prog->aux;
	106	if (seq_num == 0)
	107	BPF_SEQ_PRINTF(seq, " id name attached\n");
	108
	109	BPF_SEQ_PRINTF(seq, "%4u %-16s %s %s\n", aux->id,
	110	get_name(aux->btf, aux->func_info[0].type_id, aux->name),
	111	aux->attach_func_name, aux->linked_prog->aux->name);
	112	return 0;
	113	}
	114	char LICENSE[] SEC("license") = "GPL";

kernel/bpf/preload/iterators/iterators.c+94−0

@@ -0,0 +1,94 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2020 Facebook */
	3	#include <argp.h>
	4	#include <stdio.h>
	5	#include <stdlib.h>
	6	#include <string.h>
	7	#include <unistd.h>
	8	#include <fcntl.h>
	9	#include <sys/resource.h>
	10	#include <bpf/libbpf.h>
	11	#include <bpf/bpf.h>
	12	#include <sys/mount.h>
	13	#include "iterators.skel.h"
	14	#include "bpf_preload_common.h"
	15
	16	int to_kernel = -1;
	17	int from_kernel = 0;
	18
	19	static int send_link_to_kernel(struct bpf_link link, const char link_name)
	20	{
	21	struct bpf_preload_info obj = {};
	22	struct bpf_link_info info = {};
	23	__u32 info_len = sizeof(info);
	24	int err;
	25
	26	err = bpf_obj_get_info_by_fd(bpf_link__fd(link), &info, &info_len);
	27	if (err)
	28	return err;
	29	obj.link_id = info.id;
	30	if (strlen(link_name) >= sizeof(obj.link_name))
	31	return -E2BIG;
	32	strcpy(obj.link_name, link_name);
	33	if (write(to_kernel, &obj, sizeof(obj)) != sizeof(obj))
	34	return -EPIPE;
	35	return 0;
	36	}
	37
	38	int main(int argc, char **argv)
	39	{
	40	struct rlimit rlim = { RLIM_INFINITY, RLIM_INFINITY };
	41	struct iterators_bpf *skel;
	42	int err, magic;
	43	int debug_fd;
	44
	45	debug_fd = open("/dev/console", O_WRONLY \| O_NOCTTY \| O_CLOEXEC);
	46	if (debug_fd < 0)
	47	return 1;
	48	to_kernel = dup(1);
	49	close(1);
	50	dup(debug_fd);
	51	/* now stdin and stderr point to /dev/console */
	52
	53	read(from_kernel, &magic, sizeof(magic));
	54	if (magic != BPF_PRELOAD_START) {
	55	printf("bad start magic %d\n", magic);
	56	return 1;
	57	}
	58	setrlimit(RLIMIT_MEMLOCK, &rlim);
	59	/* libbpf opens BPF object and loads it into the kernel */
	60	skel = iterators_bpf__open_and_load();
	61	if (!skel) {
	62	/* iterators.skel.h is little endian.
	63	* libbpf doesn't support automatic little->big conversion
	64	* of BPF bytecode yet.
	65	* The program load will fail in such case.
	66	*/
	67	printf("Failed load could be due to wrong endianness\n");
	68	return 1;
	69	}
	70	err = iterators_bpf__attach(skel);
	71	if (err)
	72	goto cleanup;
	73
	74	/* send two bpf_link IDs with names to the kernel */
	75	err = send_link_to_kernel(skel->links.dump_bpf_map, "maps.debug");
	76	if (err)
	77	goto cleanup;
	78	err = send_link_to_kernel(skel->links.dump_bpf_prog, "progs.debug");
	79	if (err)
	80	goto cleanup;
	81
	82	/* The kernel will proceed with pinnging the links in bpffs.
	83	* UMD will wait on read from pipe.
	84	*/
	85	read(from_kernel, &magic, sizeof(magic));
	86	if (magic != BPF_PRELOAD_END) {
	87	printf("bad final magic %d\n", magic);
	88	err = -EINVAL;
	89	}
	90	cleanup:
	91	iterators_bpf__destroy(skel);
	92
	93	return err != 0;
	94	}

kernel/bpf/preload/iterators/iterators.skel.h+410−0

@@ -0,0 +1,410 @@
	1	/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
	2
	3	/* THIS FILE IS AUTOGENERATED! */
	4	#ifndef __ITERATORS_BPF_SKEL_H__
	5	#define __ITERATORS_BPF_SKEL_H__
	6
	7	#include <stdlib.h>
	8	#include <bpf/libbpf.h>
	9
	10	struct iterators_bpf {
	11	struct bpf_object_skeleton *skeleton;
	12	struct bpf_object *obj;
	13	struct {
	14	struct bpf_map *rodata;
	15	} maps;
	16	struct {
	17	struct bpf_program *dump_bpf_map;
	18	struct bpf_program *dump_bpf_prog;
	19	} progs;
	20	struct {
	21	struct bpf_link *dump_bpf_map;
	22	struct bpf_link *dump_bpf_prog;
	23	} links;
	24	struct iterators_bpf__rodata {
	25	char dump_bpf_map____fmt[35];
	26	char dump_bpf_map____fmt_1[14];
	27	char dump_bpf_prog____fmt[32];
	28	char dump_bpf_prog____fmt_2[17];
	29	} *rodata;
	30	};
	31
	32	static void
	33	iterators_bpf__destroy(struct iterators_bpf *obj)
	34	{
	35	if (!obj)
	36	return;
	37	if (obj->skeleton)
	38	bpf_object__destroy_skeleton(obj->skeleton);
	39	free(obj);
	40	}
	41
	42	static inline int
	43	iterators_bpf__create_skeleton(struct iterators_bpf *obj);
	44
	45	static inline struct iterators_bpf *
	46	iterators_bpf__open_opts(const struct bpf_object_open_opts *opts)
	47	{
	48	struct iterators_bpf *obj;
	49
	50	obj = (typeof(obj))calloc(1, sizeof(*obj));
	51	if (!obj)
	52	return NULL;
	53	if (iterators_bpf__create_skeleton(obj))
	54	goto err;
	55	if (bpf_object__open_skeleton(obj->skeleton, opts))
	56	goto err;
	57
	58	return obj;
	59	err:
	60	iterators_bpf__destroy(obj);
	61	return NULL;
	62	}
	63
	64	static inline struct iterators_bpf *
	65	iterators_bpf__open(void)
	66	{
	67	return iterators_bpf__open_opts(NULL);
	68	}
	69
	70	static inline int
	71	iterators_bpf__load(struct iterators_bpf *obj)
	72	{
	73	return bpf_object__load_skeleton(obj->skeleton);
	74	}
	75
	76	static inline struct iterators_bpf *
	77	iterators_bpf__open_and_load(void)
	78	{
	79	struct iterators_bpf *obj;
	80
	81	obj = iterators_bpf__open();
	82	if (!obj)
	83	return NULL;
	84	if (iterators_bpf__load(obj)) {
	85	iterators_bpf__destroy(obj);
	86	return NULL;
	87	}
	88	return obj;
	89	}
	90
	91	static inline int
	92	iterators_bpf__attach(struct iterators_bpf *obj)
	93	{
	94	return bpf_object__attach_skeleton(obj->skeleton);
	95	}
	96
	97	static inline void
	98	iterators_bpf__detach(struct iterators_bpf *obj)
	99	{
	100	return bpf_object__detach_skeleton(obj->skeleton);
	101	}
	102
	103	static inline int
	104	iterators_bpf__create_skeleton(struct iterators_bpf *obj)
	105	{
	106	struct bpf_object_skeleton *s;
	107
	108	s = (typeof(s))calloc(1, sizeof(*s));
	109	if (!s)
	110	return -1;
	111	obj->skeleton = s;
	112
	113	s->sz = sizeof(*s);
	114	s->name = "iterators_bpf";
	115	s->obj = &obj->obj;
	116
	117	/* maps */
	118	s->map_cnt = 1;
	119	s->map_skel_sz = sizeof(*s->maps);
	120	s->maps = (typeof(s->maps))calloc(s->map_cnt, s->map_skel_sz);
	121	if (!s->maps)
	122	goto err;
	123
	124	s->maps[0].name = "iterator.rodata";
	125	s->maps[0].map = &obj->maps.rodata;
	126	s->maps[0].mmaped = (void **)&obj->rodata;
	127
	128	/* programs */
	129	s->prog_cnt = 2;
	130	s->prog_skel_sz = sizeof(*s->progs);
	131	s->progs = (typeof(s->progs))calloc(s->prog_cnt, s->prog_skel_sz);
	132	if (!s->progs)
	133	goto err;
	134
	135	s->progs[0].name = "dump_bpf_map";
	136	s->progs[0].prog = &obj->progs.dump_bpf_map;
	137	s->progs[0].link = &obj->links.dump_bpf_map;
	138
	139	s->progs[1].name = "dump_bpf_prog";
	140	s->progs[1].prog = &obj->progs.dump_bpf_prog;
	141	s->progs[1].link = &obj->links.dump_bpf_prog;
	142
	143	s->data_sz = 7128;
	144	s->data = (void *)"\
	145	\x7f\x45\x4c\x46\x02\x01\x01\0\0\0\0\0\0\0\0\0\x01\0\xf7\0\x01\0\0\0\0\0\0\0\0\
	146	\0\0\0\0\0\0\0\0\0\0\0\x18\x18\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\x40\0\x0f\0\
	147	\x0e\0\x79\x12\0\0\0\0\0\0\x79\x26\0\0\0\0\0\0\x79\x17\x08\0\0\0\0\0\x15\x07\
	148	\x1a\0\0\0\0\0\x79\x21\x10\0\0\0\0\0\x55\x01\x08\0\0\0\0\0\xbf\xa4\0\0\0\0\0\0\
	149	\x07\x04\0\0\xe8\xff\xff\xff\xbf\x61\0\0\0\0\0\0\x18\x02\0\0\0\0\0\0\0\0\0\0\0\
	150	\0\0\0\xb7\x03\0\0\x23\0\0\0\xb7\x05\0\0\0\0\0\0\x85\0\0\0\x7e\0\0\0\x61\x71\0\
	151	\0\0\0\0\0\x7b\x1a\xe8\xff\0\0\0\0\xb7\x01\0\0\x04\0\0\0\xbf\x72\0\0\0\0\0\0\
	152	\x0f\x12\0\0\0\0\0\0\x7b\x2a\xf0\xff\0\0\0\0\x61\x71\x14\0\0\0\0\0\x7b\x1a\xf8\
	153	\xff\0\0\0\0\xbf\xa4\0\0\0\0\0\0\x07\x04\0\0\xe8\xff\xff\xff\xbf\x61\0\0\0\0\0\
	154	\0\x18\x02\0\0\x23\0\0\0\0\0\0\0\0\0\0\0\xb7\x03\0\0\x0e\0\0\0\xb7\x05\0\0\x18\
	155	\0\0\0\x85\0\0\0\x7e\0\0\0\xb7\0\0\0\0\0\0\0\x95\0\0\0\0\0\0\0\x79\x12\0\0\0\0\
	156	\0\0\x79\x26\0\0\0\0\0\0\x79\x11\x08\0\0\0\0\0\x15\x01\x3b\0\0\0\0\0\x79\x17\0\
	157	\0\0\0\0\0\x79\x21\x10\0\0\0\0\0\x55\x01\x08\0\0\0\0\0\xbf\xa4\0\0\0\0\0\0\x07\
	158	\x04\0\0\xd0\xff\xff\xff\xbf\x61\0\0\0\0\0\0\x18\x02\0\0\x31\0\0\0\0\0\0\0\0\0\
	159	\0\0\xb7\x03\0\0\x20\0\0\0\xb7\x05\0\0\0\0\0\0\x85\0\0\0\x7e\0\0\0\x7b\x6a\xc8\
	160	\xff\0\0\0\0\x61\x71\0\0\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\xb7\x03\0\0\x04\0\0\0\
	161	\xbf\x79\0\0\0\0\0\0\x0f\x39\0\0\0\0\0\0\x79\x71\x28\0\0\0\0\0\x79\x78\x30\0\0\
	162	\0\0\0\x15\x08\x18\0\0\0\0\0\xb7\x02\0\0\0\0\0\0\x0f\x21\0\0\0\0\0\0\x61\x11\
	163	\x04\0\0\0\0\0\x79\x83\x08\0\0\0\0\0\x67\x01\0\0\x03\0\0\0\x0f\x13\0\0\0\0\0\0\
	164	\x79\x86\0\0\0\0\0\0\xbf\xa1\0\0\0\0\0\0\x07\x01\0\0\xf8\xff\xff\xff\xb7\x02\0\
	165	\0\x08\0\0\0\x85\0\0\0\x71\0\0\0\xb7\x01\0\0\0\0\0\0\x79\xa3\xf8\xff\0\0\0\0\
	166	\x0f\x13\0\0\0\0\0\0\xbf\xa1\0\0\0\0\0\0\x07\x01\0\0\xf4\xff\xff\xff\xb7\x02\0\
	167	\0\x04\0\0\0\x85\0\0\0\x04\0\0\0\xb7\x03\0\0\x04\0\0\0\x61\xa1\xf4\xff\0\0\0\0\
	168	\x61\x82\x10\0\0\0\0\0\x3d\x21\x02\0\0\0\0\0\x0f\x16\0\0\0\0\0\0\xbf\x69\0\0\0\
	169	\0\0\0\x7b\x9a\xd8\xff\0\0\0\0\x79\x71\x18\0\0\0\0\0\x7b\x1a\xe0\xff\0\0\0\0\
	170	\x79\x71\x20\0\0\0\0\0\x79\x11\0\0\0\0\0\0\x0f\x31\0\0\0\0\0\0\x7b\x1a\xe8\xff\
	171	\0\0\0\0\xbf\xa4\0\0\0\0\0\0\x07\x04\0\0\xd0\xff\xff\xff\x79\xa1\xc8\xff\0\0\0\
	172	\0\x18\x02\0\0\x51\0\0\0\0\0\0\0\0\0\0\0\xb7\x03\0\0\x11\0\0\0\xb7\x05\0\0\x20\
	173	\0\0\0\x85\0\0\0\x7e\0\0\0\xb7\0\0\0\0\0\0\0\x95\0\0\0\0\0\0\0\x20\x20\x69\x64\
	174	\x20\x6e\x61\x6d\x65\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x6d\
	175	\x61\x78\x5f\x65\x6e\x74\x72\x69\x65\x73\x0a\0\x25\x34\x75\x20\x25\x2d\x31\x36\
	176	\x73\x25\x36\x64\x0a\0\x20\x20\x69\x64\x20\x6e\x61\x6d\x65\x20\x20\x20\x20\x20\
	177	\x20\x20\x20\x20\x20\x20\x20\x20\x61\x74\x74\x61\x63\x68\x65\x64\x0a\0\x25\x34\
	178	\x75\x20\x25\x2d\x31\x36\x73\x20\x25\x73\x20\x25\x73\x0a\0\x47\x50\x4c\0\x9f\
	179	\xeb\x01\0\x18\0\0\0\0\0\0\0\x1c\x04\0\0\x1c\x04\0\0\0\x05\0\0\0\0\0\0\0\0\0\
	180	\x02\x02\0\0\0\x01\0\0\0\x02\0\0\x04\x10\0\0\0\x13\0\0\0\x03\0\0\0\0\0\0\0\x18\
	181	\0\0\0\x04\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\x02\x08\0\0\0\0\0\0\0\0\0\0\x02\x0d\0\
	182	\0\0\0\0\0\0\x01\0\0\x0d\x06\0\0\0\x1c\0\0\0\x01\0\0\0\x20\0\0\0\0\0\0\x01\x04\
	183	\0\0\0\x20\0\0\x01\x24\0\0\0\x01\0\0\x0c\x05\0\0\0\xa3\0\0\0\x03\0\0\x04\x18\0\
	184	\0\0\xb1\0\0\0\x09\0\0\0\0\0\0\0\xb5\0\0\0\x0b\0\0\0\x40\0\0\0\xc0\0\0\0\x0b\0\
	185	\0\0\x80\0\0\0\0\0\0\0\0\0\0\x02\x0a\0\0\0\xc8\0\0\0\0\0\0\x07\0\0\0\0\xd1\0\0\
	186	\0\0\0\0\x08\x0c\0\0\0\xd7\0\0\0\0\0\0\x01\x08\0\0\0\x40\0\0\0\x98\x01\0\0\x03\
	187	\0\0\x04\x18\0\0\0\xa0\x01\0\0\x0e\0\0\0\0\0\0\0\xa3\x01\0\0\x11\0\0\0\x20\0\0\
	188	\0\xa8\x01\0\0\x0e\0\0\0\xa0\0\0\0\xb4\x01\0\0\0\0\0\x08\x0f\0\0\0\xba\x01\0\0\
	189	\0\0\0\x01\x04\0\0\0\x20\0\0\0\xc7\x01\0\0\0\0\0\x01\x01\0\0\0\x08\0\0\x01\0\0\
	190	\0\0\0\0\0\x03\0\0\0\0\x10\0\0\0\x12\0\0\0\x10\0\0\0\xcc\x01\0\0\0\0\0\x01\x04\
	191	\0\0\0\x20\0\0\0\0\0\0\0\0\0\0\x02\x14\0\0\0\x30\x02\0\0\x02\0\0\x04\x10\0\0\0\
	192	\x13\0\0\0\x03\0\0\0\0\0\0\0\x43\x02\0\0\x15\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\x02\
	193	\x18\0\0\0\0\0\0\0\x01\0\0\x0d\x06\0\0\0\x1c\0\0\0\x13\0\0\0\x48\x02\0\0\x01\0\
	194	\0\x0c\x16\0\0\0\x94\x02\0\0\x01\0\0\x04\x08\0\0\0\x9d\x02\0\0\x19\0\0\0\0\0\0\
	195	\0\0\0\0\0\0\0\0\x02\x1a\0\0\0\xee\x02\0\0\x06\0\0\x04\x38\0\0\0\xa0\x01\0\0\
	196	\x0e\0\0\0\0\0\0\0\xa3\x01\0\0\x11\0\0\0\x20\0\0\0\xfb\x02\0\0\x1b\0\0\0\xc0\0\
	197	\0\0\x0c\x03\0\0\x15\0\0\0\0\x01\0\0\x18\x03\0\0\x1d\0\0\0\x40\x01\0\0\x22\x03\
	198	\0\0\x1e\0\0\0\x80\x01\0\0\0\0\0\0\0\0\0\x02\x1c\0\0\0\0\0\0\0\0\0\0\x0a\x10\0\
	199	\0\0\0\0\0\0\0\0\0\x02\x1f\0\0\0\0\0\0\0\0\0\0\x02\x20\0\0\0\x6c\x03\0\0\x02\0\
	200	\0\x04\x08\0\0\0\x7a\x03\0\0\x0e\0\0\0\0\0\0\0\x83\x03\0\0\x0e\0\0\0\x20\0\0\0\
	201	\x22\x03\0\0\x03\0\0\x04\x18\0\0\0\x8d\x03\0\0\x1b\0\0\0\0\0\0\0\x95\x03\0\0\
	202	\x21\0\0\0\x40\0\0\0\x9b\x03\0\0\x23\0\0\0\x80\0\0\0\0\0\0\0\0\0\0\x02\x22\0\0\
	203	\0\0\0\0\0\0\0\0\x02\x24\0\0\0\x9f\x03\0\0\x01\0\0\x04\x04\0\0\0\xaa\x03\0\0\
	204	\x0e\0\0\0\0\0\0\0\x13\x04\0\0\x01\0\0\x04\x04\0\0\0\x1c\x04\0\0\x0e\0\0\0\0\0\
	205	\0\0\0\0\0\0\0\0\0\x03\0\0\0\0\x1c\0\0\0\x12\0\0\0\x23\0\0\0\x92\x04\0\0\0\0\0\
	206	\x0e\x25\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\0\0\0\x1c\0\0\0\x12\0\0\0\x0e\0\0\0\
	207	\xa6\x04\0\0\0\0\0\x0e\x27\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\0\0\0\x1c\0\0\0\
	208	\x12\0\0\0\x20\0\0\0\xbc\x04\0\0\0\0\0\x0e\x29\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\
	209	\0\0\0\0\x1c\0\0\0\x12\0\0\0\x11\0\0\0\xd1\x04\0\0\0\0\0\x0e\x2b\0\0\0\0\0\0\0\
	210	\0\0\0\0\0\0\0\x03\0\0\0\0\x10\0\0\0\x12\0\0\0\x04\0\0\0\xe8\x04\0\0\0\0\0\x0e\
	211	\x2d\0\0\0\x01\0\0\0\xf0\x04\0\0\x04\0\0\x0f\0\0\0\0\x26\0\0\0\0\0\0\0\x23\0\0\
	212	\0\x28\0\0\0\x23\0\0\0\x0e\0\0\0\x2a\0\0\0\x31\0\0\0\x20\0\0\0\x2c\0\0\0\x51\0\
	213	\0\0\x11\0\0\0\xf8\x04\0\0\x01\0\0\x0f\0\0\0\0\x2e\0\0\0\0\0\0\0\x04\0\0\0\0\
	214	\x62\x70\x66\x5f\x69\x74\x65\x72\x5f\x5f\x62\x70\x66\x5f\x6d\x61\x70\0\x6d\x65\
	215	\x74\x61\0\x6d\x61\x70\0\x63\x74\x78\0\x69\x6e\x74\0\x64\x75\x6d\x70\x5f\x62\
	216	\x70\x66\x5f\x6d\x61\x70\0\x69\x74\x65\x72\x2f\x62\x70\x66\x5f\x6d\x61\x70\0\
	217	\x30\x3a\x30\0\x2f\x77\x2f\x6e\x65\x74\x2d\x6e\x65\x78\x74\x2f\x6b\x65\x72\x6e\
	218	\x65\x6c\x2f\x62\x70\x66\x2f\x70\x72\x65\x6c\x6f\x61\x64\x2f\x69\x74\x65\x72\
	219	\x61\x74\x6f\x72\x73\x2f\x69\x74\x65\x72\x61\x74\x6f\x72\x73\x2e\x62\x70\x66\
	220	\x2e\x63\0\x09\x73\x74\x72\x75\x63\x74\x20\x73\x65\x71\x5f\x66\x69\x6c\x65\x20\
	221	\x2a\x73\x65\x71\x20\x3d\x20\x63\x74\x78\x2d\x3e\x6d\x65\x74\x61\x2d\x3e\x73\
	222	\x65\x71\x3b\0\x62\x70\x66\x5f\x69\x74\x65\x72\x5f\x6d\x65\x74\x61\0\x73\x65\
	223	\x71\0\x73\x65\x73\x73\x69\x6f\x6e\x5f\x69\x64\0\x73\x65\x71\x5f\x6e\x75\x6d\0\
	224	\x73\x65\x71\x5f\x66\x69\x6c\x65\0\x5f\x5f\x75\x36\x34\0\x6c\x6f\x6e\x67\x20\
	225	\x6c\x6f\x6e\x67\x20\x75\x6e\x73\x69\x67\x6e\x65\x64\x20\x69\x6e\x74\0\x30\x3a\
	226	\x31\0\x09\x73\x74\x72\x75\x63\x74\x20\x62\x70\x66\x5f\x6d\x61\x70\x20\x2a\x6d\
	227	\x61\x70\x20\x3d\x20\x63\x74\x78\x2d\x3e\x6d\x61\x70\x3b\0\x09\x69\x66\x20\x28\
	228	\x21\x6d\x61\x70\x29\0\x30\x3a\x32\0\x09\x5f\x5f\x75\x36\x34\x20\x73\x65\x71\
	229	\x5f\x6e\x75\x6d\x20\x3d\x20\x63\x74\x78\x2d\x3e\x6d\x65\x74\x61\x2d\x3e\x73\
	230	\x65\x71\x5f\x6e\x75\x6d\x3b\0\x09\x69\x66\x20\x28\x73\x65\x71\x5f\x6e\x75\x6d\
	231	\x20\x3d\x3d\x20\x30\x29\0\x09\x09\x42\x50\x46\x5f\x53\x45\x51\x5f\x50\x52\x49\
	232	\x4e\x54\x46\x28\x73\x65\x71\x2c\x20\x22\x20\x20\x69\x64\x20\x6e\x61\x6d\x65\
	233	\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x6d\x61\x78\x5f\x65\x6e\
	234	\x74\x72\x69\x65\x73\x5c\x6e\x22\x29\x3b\0\x62\x70\x66\x5f\x6d\x61\x70\0\x69\
	235	\x64\0\x6e\x61\x6d\x65\0\x6d\x61\x78\x5f\x65\x6e\x74\x72\x69\x65\x73\0\x5f\x5f\
	236	\x75\x33\x32\0\x75\x6e\x73\x69\x67\x6e\x65\x64\x20\x69\x6e\x74\0\x63\x68\x61\
	237	\x72\0\x5f\x5f\x41\x52\x52\x41\x59\x5f\x53\x49\x5a\x45\x5f\x54\x59\x50\x45\x5f\
	238	\x5f\0\x09\x42\x50\x46\x5f\x53\x45\x51\x5f\x50\x52\x49\x4e\x54\x46\x28\x73\x65\
	239	\x71\x2c\x20\x22\x25\x34\x75\x20\x25\x2d\x31\x36\x73\x25\x36\x64\x5c\x6e\x22\
	240	\x2c\x20\x6d\x61\x70\x2d\x3e\x69\x64\x2c\x20\x6d\x61\x70\x2d\x3e\x6e\x61\x6d\
	241	\x65\x2c\x20\x6d\x61\x70\x2d\x3e\x6d\x61\x78\x5f\x65\x6e\x74\x72\x69\x65\x73\
	242	\x29\x3b\0\x7d\0\x62\x70\x66\x5f\x69\x74\x65\x72\x5f\x5f\x62\x70\x66\x5f\x70\
	243	\x72\x6f\x67\0\x70\x72\x6f\x67\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\
	244	\x6f\x67\0\x69\x74\x65\x72\x2f\x62\x70\x66\x5f\x70\x72\x6f\x67\0\x09\x73\x74\
	245	\x72\x75\x63\x74\x20\x62\x70\x66\x5f\x70\x72\x6f\x67\x20\x2a\x70\x72\x6f\x67\
	246	\x20\x3d\x20\x63\x74\x78\x2d\x3e\x70\x72\x6f\x67\x3b\0\x09\x69\x66\x20\x28\x21\
	247	\x70\x72\x6f\x67\x29\0\x62\x70\x66\x5f\x70\x72\x6f\x67\0\x61\x75\x78\0\x09\x61\
	248	\x75\x78\x20\x3d\x20\x70\x72\x6f\x67\x2d\x3e\x61\x75\x78\x3b\0\x09\x09\x42\x50\
	249	\x46\x5f\x53\x45\x51\x5f\x50\x52\x49\x4e\x54\x46\x28\x73\x65\x71\x2c\x20\x22\
	250	\x20\x20\x69\x64\x20\x6e\x61\x6d\x65\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
	251	\x20\x20\x20\x61\x74\x74\x61\x63\x68\x65\x64\x5c\x6e\x22\x29\x3b\0\x62\x70\x66\
	252	\x5f\x70\x72\x6f\x67\x5f\x61\x75\x78\0\x61\x74\x74\x61\x63\x68\x5f\x66\x75\x6e\
	253	\x63\x5f\x6e\x61\x6d\x65\0\x6c\x69\x6e\x6b\x65\x64\x5f\x70\x72\x6f\x67\0\x66\
	254	\x75\x6e\x63\x5f\x69\x6e\x66\x6f\0\x62\x74\x66\0\x09\x42\x50\x46\x5f\x53\x45\
	255	\x51\x5f\x50\x52\x49\x4e\x54\x46\x28\x73\x65\x71\x2c\x20\x22\x25\x34\x75\x20\
	256	\x25\x2d\x31\x36\x73\x20\x25\x73\x20\x25\x73\x5c\x6e\x22\x2c\x20\x61\x75\x78\
	257	\x2d\x3e\x69\x64\x2c\0\x30\x3a\x34\0\x30\x3a\x35\0\x09\x69\x66\x20\x28\x21\x62\
	258	\x74\x66\x29\0\x62\x70\x66\x5f\x66\x75\x6e\x63\x5f\x69\x6e\x66\x6f\0\x69\x6e\
	259	\x73\x6e\x5f\x6f\x66\x66\0\x74\x79\x70\x65\x5f\x69\x64\0\x30\0\x73\x74\x72\x69\
	260	\x6e\x67\x73\0\x74\x79\x70\x65\x73\0\x68\x64\x72\0\x62\x74\x66\x5f\x68\x65\x61\
	261	\x64\x65\x72\0\x73\x74\x72\x5f\x6c\x65\x6e\0\x09\x74\x79\x70\x65\x73\x20\x3d\
	262	\x20\x62\x74\x66\x2d\x3e\x74\x79\x70\x65\x73\x3b\0\x09\x62\x70\x66\x5f\x70\x72\
	263	\x6f\x62\x65\x5f\x72\x65\x61\x64\x5f\x6b\x65\x72\x6e\x65\x6c\x28\x26\x74\x2c\
	264	\x20\x73\x69\x7a\x65\x6f\x66\x28\x74\x29\x2c\x20\x74\x79\x70\x65\x73\x20\x2b\
	265	\x20\x62\x74\x66\x5f\x69\x64\x29\x3b\0\x09\x73\x74\x72\x20\x3d\x20\x62\x74\x66\
	266	\x2d\x3e\x73\x74\x72\x69\x6e\x67\x73\x3b\0\x62\x74\x66\x5f\x74\x79\x70\x65\0\
	267	\x6e\x61\x6d\x65\x5f\x6f\x66\x66\0\x09\x6e\x61\x6d\x65\x5f\x6f\x66\x66\x20\x3d\
	268	\x20\x42\x50\x46\x5f\x43\x4f\x52\x45\x5f\x52\x45\x41\x44\x28\x74\x2c\x20\x6e\
	269	\x61\x6d\x65\x5f\x6f\x66\x66\x29\x3b\0\x30\x3a\x32\x3a\x30\0\x09\x69\x66\x20\
	270	\x28\x6e\x61\x6d\x65\x5f\x6f\x66\x66\x20\x3e\x3d\x20\x62\x74\x66\x2d\x3e\x68\
	271	\x64\x72\x2e\x73\x74\x72\x5f\x6c\x65\x6e\x29\0\x09\x72\x65\x74\x75\x72\x6e\x20\
	272	\x73\x74\x72\x20\x2b\x20\x6e\x61\x6d\x65\x5f\x6f\x66\x66\x3b\0\x30\x3a\x33\0\
	273	\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\x61\x70\x2e\x5f\x5f\x5f\x66\x6d\x74\0\
	274	\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\x61\x70\x2e\x5f\x5f\x5f\x66\x6d\x74\
	275	\x2e\x31\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\x2e\x5f\x5f\x5f\
	276	\x66\x6d\x74\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\x2e\x5f\x5f\
	277	\x5f\x66\x6d\x74\x2e\x32\0\x4c\x49\x43\x45\x4e\x53\x45\0\x2e\x72\x6f\x64\x61\
	278	\x74\x61\0\x6c\x69\x63\x65\x6e\x73\x65\0\x9f\xeb\x01\0\x20\0\0\0\0\0\0\0\x24\0\
	279	\0\0\x24\0\0\0\x44\x02\0\0\x68\x02\0\0\xa4\x01\0\0\x08\0\0\0\x31\0\0\0\x01\0\0\
	280	\0\0\0\0\0\x07\0\0\0\x56\x02\0\0\x01\0\0\0\0\0\0\0\x17\0\0\0\x10\0\0\0\x31\0\0\
	281	\0\x09\0\0\0\0\0\0\0\x42\0\0\0\x7b\0\0\0\x1e\x40\x01\0\x08\0\0\0\x42\0\0\0\x7b\
	282	\0\0\0\x24\x40\x01\0\x10\0\0\0\x42\0\0\0\xf2\0\0\0\x1d\x48\x01\0\x18\0\0\0\x42\
	283	\0\0\0\x13\x01\0\0\x06\x50\x01\0\x20\0\0\0\x42\0\0\0\x22\x01\0\0\x1d\x44\x01\0\
	284	\x28\0\0\0\x42\0\0\0\x47\x01\0\0\x06\x5c\x01\0\x38\0\0\0\x42\0\0\0\x5a\x01\0\0\
	285	\x03\x60\x01\0\x70\0\0\0\x42\0\0\0\xe0\x01\0\0\x02\x68\x01\0\xf0\0\0\0\x42\0\0\
	286	\0\x2e\x02\0\0\x01\x70\x01\0\x56\x02\0\0\x1a\0\0\0\0\0\0\0\x42\0\0\0\x7b\0\0\0\
	287	\x1e\x84\x01\0\x08\0\0\0\x42\0\0\0\x7b\0\0\0\x24\x84\x01\0\x10\0\0\0\x42\0\0\0\
	288	\x64\x02\0\0\x1f\x8c\x01\0\x18\0\0\0\x42\0\0\0\x88\x02\0\0\x06\x98\x01\0\x20\0\
	289	\0\0\x42\0\0\0\xa1\x02\0\0\x0e\xa4\x01\0\x28\0\0\0\x42\0\0\0\x22\x01\0\0\x1d\
	290	\x88\x01\0\x30\0\0\0\x42\0\0\0\x47\x01\0\0\x06\xa8\x01\0\x40\0\0\0\x42\0\0\0\
	291	\xb3\x02\0\0\x03\xac\x01\0\x80\0\0\0\x42\0\0\0\x26\x03\0\0\x02\xb4\x01\0\xb8\0\
	292	\0\0\x42\0\0\0\x61\x03\0\0\x06\x08\x01\0\xd0\0\0\0\x42\0\0\0\0\0\0\0\0\0\0\0\
	293	\xd8\0\0\0\x42\0\0\0\xb2\x03\0\0\x0f\x14\x01\0\xe0\0\0\0\x42\0\0\0\xc7\x03\0\0\
	294	\x2d\x18\x01\0\xf0\0\0\0\x42\0\0\0\xfe\x03\0\0\x0d\x10\x01\0\0\x01\0\0\x42\0\0\
	295	\0\0\0\0\0\0\0\0\0\x08\x01\0\0\x42\0\0\0\xc7\x03\0\0\x02\x18\x01\0\x20\x01\0\0\
	296	\x42\0\0\0\x25\x04\0\0\x0d\x1c\x01\0\x38\x01\0\0\x42\0\0\0\0\0\0\0\0\0\0\0\x40\
	297	\x01\0\0\x42\0\0\0\x25\x04\0\0\x0d\x1c\x01\0\x58\x01\0\0\x42\0\0\0\x25\x04\0\0\
	298	\x0d\x1c\x01\0\x60\x01\0\0\x42\0\0\0\x53\x04\0\0\x1b\x20\x01\0\x68\x01\0\0\x42\
	299	\0\0\0\x53\x04\0\0\x06\x20\x01\0\x70\x01\0\0\x42\0\0\0\x76\x04\0\0\x0d\x28\x01\
	300	\0\x78\x01\0\0\x42\0\0\0\0\0\0\0\0\0\0\0\x80\x01\0\0\x42\0\0\0\x26\x03\0\0\x02\
	301	\xb4\x01\0\xf8\x01\0\0\x42\0\0\0\x2e\x02\0\0\x01\xc4\x01\0\x10\0\0\0\x31\0\0\0\
	302	\x07\0\0\0\0\0\0\0\x02\0\0\0\x3e\0\0\0\0\0\0\0\x08\0\0\0\x08\0\0\0\x3e\0\0\0\0\
	303	\0\0\0\x10\0\0\0\x02\0\0\0\xee\0\0\0\0\0\0\0\x20\0\0\0\x08\0\0\0\x1e\x01\0\0\0\
	304	\0\0\0\x70\0\0\0\x0d\0\0\0\x3e\0\0\0\0\0\0\0\x80\0\0\0\x0d\0\0\0\xee\0\0\0\0\0\
	305	\0\0\xa0\0\0\0\x0d\0\0\0\x1e\x01\0\0\0\0\0\0\x56\x02\0\0\x12\0\0\0\0\0\0\0\x14\
	306	\0\0\0\x3e\0\0\0\0\0\0\0\x08\0\0\0\x08\0\0\0\x3e\0\0\0\0\0\0\0\x10\0\0\0\x14\0\
	307	\0\0\xee\0\0\0\0\0\0\0\x20\0\0\0\x18\0\0\0\x3e\0\0\0\0\0\0\0\x28\0\0\0\x08\0\0\
	308	\0\x1e\x01\0\0\0\0\0\0\x80\0\0\0\x1a\0\0\0\x3e\0\0\0\0\0\0\0\x90\0\0\0\x1a\0\0\
	309	\0\xee\0\0\0\0\0\0\0\xa8\0\0\0\x1a\0\0\0\x59\x03\0\0\0\0\0\0\xb0\0\0\0\x1a\0\0\
	310	\0\x5d\x03\0\0\0\0\0\0\xc0\0\0\0\x1f\0\0\0\x8b\x03\0\0\0\0\0\0\xd8\0\0\0\x20\0\
	311	\0\0\xee\0\0\0\0\0\0\0\xf0\0\0\0\x20\0\0\0\x3e\0\0\0\0\0\0\0\x18\x01\0\0\x24\0\
	312	\0\0\x3e\0\0\0\0\0\0\0\x50\x01\0\0\x1a\0\0\0\xee\0\0\0\0\0\0\0\x60\x01\0\0\x20\
	313	\0\0\0\x4d\x04\0\0\0\0\0\0\x88\x01\0\0\x1a\0\0\0\x1e\x01\0\0\0\0\0\0\x98\x01\0\
	314	\0\x1a\0\0\0\x8e\x04\0\0\0\0\0\0\xa0\x01\0\0\x18\0\0\0\x3e\0\0\0\0\0\0\0\0\0\0\
	315	\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd6\0\0\0\0\0\x02\0\x70\0\0\0\0\
	316	\0\0\0\0\0\0\0\0\0\0\0\xc8\0\0\0\0\0\x02\0\xf0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
	317	\xcf\0\0\0\0\0\x03\0\x78\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc1\0\0\0\0\0\x03\0\x80\
	318	\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xba\0\0\0\0\0\x03\0\xf8\x01\0\0\0\0\0\0\0\0\0\
	319	\0\0\0\0\0\x14\0\0\0\x01\0\x04\0\0\0\0\0\0\0\0\0\x23\0\0\0\0\0\0\0\xf4\0\0\0\
	320	\x01\0\x04\0\x23\0\0\0\0\0\0\0\x0e\0\0\0\0\0\0\0\x28\0\0\0\x01\0\x04\0\x31\0\0\
	321	\0\0\0\0\0\x20\0\0\0\0\0\0\0\xdd\0\0\0\x01\0\x04\0\x51\0\0\0\0\0\0\0\x11\0\0\0\
	322	\0\0\0\0\0\0\0\0\x03\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\x03\
	323	\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\
	324	\0\0\0\0\xb2\0\0\0\x11\0\x05\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\x3d\0\0\0\x12\
	325	\0\x02\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\x5b\0\0\0\x12\0\x03\0\0\0\0\0\0\0\0\
	326	\0\x08\x02\0\0\0\0\0\0\x48\0\0\0\0\0\0\0\x01\0\0\0\x0c\0\0\0\xc8\0\0\0\0\0\0\0\
	327	\x01\0\0\0\x0c\0\0\0\x50\0\0\0\0\0\0\0\x01\0\0\0\x0c\0\0\0\xd0\x01\0\0\0\0\0\0\
	328	\x01\0\0\0\x0c\0\0\0\xf0\x03\0\0\0\0\0\0\x0a\0\0\0\x0c\0\0\0\xfc\x03\0\0\0\0\0\
	329	\0\x0a\0\0\0\x0c\0\0\0\x08\x04\0\0\0\0\0\0\x0a\0\0\0\x0c\0\0\0\x14\x04\0\0\0\0\
	330	\0\0\x0a\0\0\0\x0c\0\0\0\x2c\x04\0\0\0\0\0\0\0\0\0\0\x0d\0\0\0\x2c\0\0\0\0\0\0\
	331	\0\0\0\0\0\x0a\0\0\0\x3c\0\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x50\0\0\0\0\0\0\0\0\0\
	332	\0\0\x0a\0\0\0\x60\0\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\x70\0\0\0\0\0\0\0\0\0\0\0\
	333	\x0a\0\0\0\x80\0\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\x90\0\0\0\0\0\0\0\0\0\0\0\x0a\0\
	334	\0\0\xa0\0\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\xb0\0\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\
	335	\xc0\0\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\xd0\0\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\xe8\0\
	336	\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xf8\0\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x08\x01\0\0\
	337	\0\0\0\0\0\0\0\0\x0b\0\0\0\x18\x01\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x28\x01\0\0\0\
	338	\0\0\0\0\0\0\0\x0b\0\0\0\x38\x01\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x48\x01\0\0\0\0\
	339	\0\0\0\0\0\0\x0b\0\0\0\x58\x01\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x68\x01\0\0\0\0\0\
	340	\0\0\0\0\0\x0b\0\0\0\x78\x01\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x88\x01\0\0\0\0\0\0\
	341	\0\0\0\0\x0b\0\0\0\x98\x01\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xa8\x01\0\0\0\0\0\0\0\
	342	\0\0\0\x0b\0\0\0\xb8\x01\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xc8\x01\0\0\0\0\0\0\0\0\
	343	\0\0\x0b\0\0\0\xd8\x01\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xe8\x01\0\0\0\0\0\0\0\0\0\
	344	\0\x0b\0\0\0\xf8\x01\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x08\x02\0\0\0\0\0\0\0\0\0\0\
	345	\x0b\0\0\0\x18\x02\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x28\x02\0\0\0\0\0\0\0\0\0\0\
	346	\x0b\0\0\0\x38\x02\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x48\x02\0\0\0\0\0\0\0\0\0\0\
	347	\x0b\0\0\0\x58\x02\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x68\x02\0\0\0\0\0\0\0\0\0\0\
	348	\x0b\0\0\0\x78\x02\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x94\x02\0\0\0\0\0\0\0\0\0\0\
	349	\x0a\0\0\0\xa4\x02\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\xb4\x02\0\0\0\0\0\0\0\0\0\0\
	350	\x0a\0\0\0\xc4\x02\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\xd4\x02\0\0\0\0\0\0\0\0\0\0\
	351	\x0a\0\0\0\xe4\x02\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\xf4\x02\0\0\0\0\0\0\0\0\0\0\
	352	\x0a\0\0\0\x0c\x03\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x1c\x03\0\0\0\0\0\0\0\0\0\0\
	353	\x0b\0\0\0\x2c\x03\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x3c\x03\0\0\0\0\0\0\0\0\0\0\
	354	\x0b\0\0\0\x4c\x03\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x5c\x03\0\0\0\0\0\0\0\0\0\0\
	355	\x0b\0\0\0\x6c\x03\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x7c\x03\0\0\0\0\0\0\0\0\0\0\
	356	\x0b\0\0\0\x8c\x03\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x9c\x03\0\0\0\0\0\0\0\0\0\0\
	357	\x0b\0\0\0\xac\x03\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xbc\x03\0\0\0\0\0\0\0\0\0\0\
	358	\x0b\0\0\0\xcc\x03\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xdc\x03\0\0\0\0\0\0\0\0\0\0\
	359	\x0b\0\0\0\xec\x03\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xfc\x03\0\0\0\0\0\0\0\0\0\0\
	360	\x0b\0\0\0\x0c\x04\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x1c\x04\0\0\0\0\0\0\0\0\0\0\
	361	\x0b\0\0\0\x4e\x4f\x41\x42\x43\x44\x4d\0\x2e\x74\x65\x78\x74\0\x2e\x72\x65\x6c\
	362	\x2e\x42\x54\x46\x2e\x65\x78\x74\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\x61\
	363	\x70\x2e\x5f\x5f\x5f\x66\x6d\x74\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\
	364	\x6f\x67\x2e\x5f\x5f\x5f\x66\x6d\x74\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\
	365	\x61\x70\0\x2e\x72\x65\x6c\x69\x74\x65\x72\x2f\x62\x70\x66\x5f\x6d\x61\x70\0\
	366	\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\0\x2e\x72\x65\x6c\x69\x74\
	367	\x65\x72\x2f\x62\x70\x66\x5f\x70\x72\x6f\x67\0\x2e\x6c\x6c\x76\x6d\x5f\x61\x64\
	368	\x64\x72\x73\x69\x67\0\x6c\x69\x63\x65\x6e\x73\x65\0\x2e\x73\x74\x72\x74\x61\
	369	\x62\0\x2e\x73\x79\x6d\x74\x61\x62\0\x2e\x72\x6f\x64\x61\x74\x61\0\x2e\x72\x65\
	370	\x6c\x2e\x42\x54\x46\0\x4c\x49\x43\x45\x4e\x53\x45\0\x4c\x42\x42\x31\x5f\x37\0\
	371	\x4c\x42\x42\x31\x5f\x36\0\x4c\x42\x42\x30\x5f\x34\0\x4c\x42\x42\x31\x5f\x33\0\
	372	\x4c\x42\x42\x30\x5f\x33\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\
	373	\x2e\x5f\x5f\x5f\x66\x6d\x74\x2e\x32\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\
	374	\x61\x70\x2e\x5f\x5f\x5f\x66\x6d\x74\x2e\x31\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
	375	\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
	376	\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\0\
	377	\0\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\0\
	378	\0\0\0\0\0\0\0\x4e\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\
	379	\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
	380	\x6d\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x40\x01\0\0\0\0\0\0\x08\
	381	\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa1\0\0\0\
	382	\x01\0\0\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x48\x03\0\0\0\0\0\0\x62\0\0\0\0\0\
	383	\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x89\0\0\0\x01\0\0\0\x03\
	384	\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xaa\x03\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\
	385	\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xad\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\
	386	\0\0\0\0\0\0\0\xae\x03\0\0\0\0\0\0\x34\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\
	387	\0\0\0\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
	388	\xe2\x0c\0\0\0\0\0\0\x2c\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\
	389	\0\0\0\0\0\0\x99\0\0\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x10\x11\0\0\0\
	390	\0\0\0\x80\x01\0\0\0\0\0\0\x0e\0\0\0\x0d\0\0\0\x08\0\0\0\0\0\0\0\x18\0\0\0\0\0\
	391	\0\0\x4a\0\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x90\x12\0\0\0\0\0\0\
	392	\x20\0\0\0\0\0\0\0\x08\0\0\0\x02\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x69\
	393	\0\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb0\x12\0\0\0\0\0\0\x20\0\0\0\
	394	\0\0\0\0\x08\0\0\0\x03\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\xa9\0\0\0\x09\
	395	\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd0\x12\0\0\0\0\0\0\x50\0\0\0\0\0\0\0\
	396	\x08\0\0\0\x06\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x07\0\0\0\x09\0\0\0\0\
	397	\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x20\x13\0\0\0\0\0\0\xe0\x03\0\0\0\0\0\0\x08\0\0\
	398	\0\x07\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x7b\0\0\0\x03\x4c\xff\x6f\0\0\
	399	\0\x80\0\0\0\0\0\0\0\0\0\0\0\0\0\x17\0\0\0\0\0\0\x07\0\0\0\0\0\0\0\0\0\0\0\0\0\
	400	\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x91\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\
	401	\0\0\0\0\0\0\x07\x17\0\0\0\0\0\0\x0a\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\
	402	\0\0\0\0\0\0\0\0\0\0\0\0";
	403
	404	return 0;
	405	err:
	406	bpf_object__destroy_skeleton(s);
	407	return -1;
	408	}
	409
	410	#endif /* __ITERATORS_BPF_SKEL_H__ */

kernel/bpf/queue_stack_maps.c+2−0

 static int queue_map_btf_id;
 const struct bpf_map_ops queue_map_ops = {
+	.map_meta_equal = bpf_map_meta_equal,
 	.map_alloc_check = queue_stack_map_alloc_check,
 	.map_alloc = queue_stack_map_alloc,
 	.map_free = queue_stack_map_free,
 static int stack_map_btf_id;
 const struct bpf_map_ops stack_map_ops = {
+	.map_meta_equal = bpf_map_meta_equal,
 	.map_alloc_check = queue_stack_map_alloc_check,
 	.map_alloc = queue_stack_map_alloc,
 	.map_free = queue_stack_map_free,

kernel/bpf/reuseport_array.c+1−0

@@ -351,6 +351,7 @@ static int reuseport_array_get_next_key(struct bpf_map map, void key, · 350 unmodified lines
351		351
352	static int reuseport_array_map_btf_id;	352	static int reuseport_array_map_btf_id;
353	const struct bpf_map_ops reuseport_array_ops = {	353	const struct bpf_map_ops reuseport_array_ops = {
		354	.map_meta_equal = bpf_map_meta_equal,
354	.map_alloc_check = reuseport_array_alloc_check,	355	.map_alloc_check = reuseport_array_alloc_check,
355	.map_alloc = reuseport_array_alloc,	356	.map_alloc = reuseport_array_alloc,
356	.map_free = reuseport_array_free,	357	.map_free = reuseport_array_free,

kernel/bpf/ringbuf.c+1−0

@@ -287,6 +287,7 @@ static __poll_t ringbuf_map_poll(struct bpf_map map, struct file filp, · 286 unmodified lines
287		287
288	static int ringbuf_map_btf_id;	288	static int ringbuf_map_btf_id;
289	const struct bpf_map_ops ringbuf_map_ops = {	289	const struct bpf_map_ops ringbuf_map_ops = {
		290	.map_meta_equal = bpf_map_meta_equal,
290	.map_alloc = ringbuf_map_alloc,	291	.map_alloc = ringbuf_map_alloc,
291	.map_free = ringbuf_map_free,	292	.map_free = ringbuf_map_free,
292	.map_mmap = ringbuf_map_mmap,	293	.map_mmap = ringbuf_map_mmap,

kernel/bpf/stackmap.c+1−0

@@ -839,6 +839,7 @@ static void stack_map_free(struct bpf_map *map) · 838 unmodified lines
839		839
840	static int stack_trace_map_btf_id;	840	static int stack_trace_map_btf_id;
841	const struct bpf_map_ops stack_trace_map_ops = {	841	const struct bpf_map_ops stack_trace_map_ops = {
		842	.map_meta_equal = bpf_map_meta_equal,
842	.map_alloc = stack_map_alloc,	843	.map_alloc = stack_map_alloc,
843	.map_free = stack_map_free,	844	.map_free = stack_map_free,
844	.map_get_next_key = stack_map_get_next_key,	845	.map_get_next_key = stack_map_get_next_key,

kernel/bpf/syscall.c+44−24

@@ -29,6 +29,7 @@ · 28 unmodified lines
29	#include <linux/bpf_lsm.h>	29	#include <linux/bpf_lsm.h>
30	#include <linux/poll.h>	30	#include <linux/poll.h>
31	#include <linux/bpf-netns.h>	31	#include <linux/bpf-netns.h>
		32	#include <linux/rcupdate_trace.h>
32		33
33	#define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY \|\| \	34	#define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY \|\| \
34	(map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY \|\| \	35	(map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY \|\| \
@@ -90,6 +91,7 @@ int bpf_check_uarg_tail_zero(void __user *uaddr, · 55 unmodified lines
90	}	91	}
91		92
92	const struct bpf_map_ops bpf_map_offload_ops = {	93	const struct bpf_map_ops bpf_map_offload_ops = {
		94	.map_meta_equal = bpf_map_meta_equal,
93	.map_alloc = bpf_map_offload_map_alloc,	95	.map_alloc = bpf_map_offload_map_alloc,
94	.map_free = bpf_map_offload_map_free,	96	.map_free = bpf_map_offload_map_free,
95	.map_check_btf = map_check_no_btf,	97	.map_check_btf = map_check_no_btf,
@@ -157,10 +159,11 @@ static int bpf_map_update_value(struct bpf_map map, struct fd f, void key, · 61 unmodified lines
157	if (bpf_map_is_dev_bound(map)) {	159	if (bpf_map_is_dev_bound(map)) {
158	return bpf_map_offload_update_elem(map, key, value, flags);	160	return bpf_map_offload_update_elem(map, key, value, flags);
159	} else if (map->map_type == BPF_MAP_TYPE_CPUMAP \|\|	161	} else if (map->map_type == BPF_MAP_TYPE_CPUMAP \|\|
160	map->map_type == BPF_MAP_TYPE_SOCKHASH \|\|
161	map->map_type == BPF_MAP_TYPE_SOCKMAP \|\|
162	map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {	162	map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
163	return map->ops->map_update_elem(map, key, value, flags);	163	return map->ops->map_update_elem(map, key, value, flags);
		164	} else if (map->map_type == BPF_MAP_TYPE_SOCKHASH \|\|
		165	map->map_type == BPF_MAP_TYPE_SOCKMAP) {
		166	return sock_map_update_elem_sys(map, key, value, flags);
164	} else if (IS_FD_PROG_ARRAY(map)) {	167	} else if (IS_FD_PROG_ARRAY(map)) {
165	return bpf_fd_array_map_update_elem(map, f.file, key, value,	168	return bpf_fd_array_map_update_elem(map, f.file, key, value,
166	flags);	169	flags);
@@ -768,7 +771,8 @@ static int map_check_btf(struct bpf_map map, const struct btf btf, · 601 unmodified lines
768	if (map->map_type != BPF_MAP_TYPE_HASH &&	771	if (map->map_type != BPF_MAP_TYPE_HASH &&
769	map->map_type != BPF_MAP_TYPE_ARRAY &&	772	map->map_type != BPF_MAP_TYPE_ARRAY &&
770	map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE &&	773	map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE &&
771	map->map_type != BPF_MAP_TYPE_SK_STORAGE)	774	map->map_type != BPF_MAP_TYPE_SK_STORAGE &&
		775	map->map_type != BPF_MAP_TYPE_INODE_STORAGE)
772	return -ENOTSUPP;	776	return -ENOTSUPP;
773	if (map->spin_lock_off + sizeof(struct bpf_spin_lock) >	777	if (map->spin_lock_off + sizeof(struct bpf_spin_lock) >
774	map->value_size) {	778	map->value_size) {
@@ -1728,10 +1732,14 @@ static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) · 953 unmodified lines
1728	btf_put(prog->aux->btf);	1732	btf_put(prog->aux->btf);
1729	bpf_prog_free_linfo(prog);	1733	bpf_prog_free_linfo(prog);
1730		1734
1731	if (deferred)	1735	if (deferred) {
1732	call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);	1736	if (prog->aux->sleepable)
1733	else	1737	call_rcu_tasks_trace(&prog->aux->rcu, __bpf_prog_put_rcu);
		1738	else
		1739	call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);
		1740	} else {
1734	__bpf_prog_put_rcu(&prog->aux->rcu);	1741	__bpf_prog_put_rcu(&prog->aux->rcu);
		1742	}
1735	}	1743	}
1736		1744
1737	static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock)	1745	static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock)
@@ -2101,6 +2109,7 @@ static int bpf_prog_load(union bpf_attr attr, union bpf_attr __user uattr) · 363 unmodified lines
2101	if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT \|	2109	if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT \|
2102	BPF_F_ANY_ALIGNMENT \|	2110	BPF_F_ANY_ALIGNMENT \|
2103	BPF_F_TEST_STATE_FREQ \|	2111	BPF_F_TEST_STATE_FREQ \|
		2112	BPF_F_SLEEPABLE \|
2104	BPF_F_TEST_RND_HI32))	2113	BPF_F_TEST_RND_HI32))
2105	return -EINVAL;	2114	return -EINVAL;
2106		2115
@@ -2156,6 +2165,7 @@ static int bpf_prog_load(union bpf_attr attr, union bpf_attr __user uattr) · 49 unmodified lines
2156	}	2165	}
2157		2166
2158	prog->aux->offload_requested = !!attr->prog_ifindex;	2167	prog->aux->offload_requested = !!attr->prog_ifindex;
		2168	prog->aux->sleepable = attr->prog_flags & BPF_F_SLEEPABLE;
2159		2169
2160	err = security_bpf_prog_alloc(prog->aux);	2170	err = security_bpf_prog_alloc(prog->aux);
2161	if (err)	2171	if (err)
@@ -4014,40 +4024,50 @@ static int link_detach(union bpf_attr *attr) · 1852 unmodified lines
4014	return ret;	4024	return ret;
4015	}	4025	}
4016		4026
4017	static int bpf_link_inc_not_zero(struct bpf_link *link)	4027	static struct bpf_link bpf_link_inc_not_zero(struct bpf_link link)
4018	{	4028	{
4019	return atomic64_fetch_add_unless(&link->refcnt, 1, 0) ? 0 : -ENOENT;	4029	return atomic64_fetch_add_unless(&link->refcnt, 1, 0) ? link : ERR_PTR(-ENOENT);
4020	}	4030	}
4021		4031
4022	#define BPF_LINK_GET_FD_BY_ID_LAST_FIELD link_id	4032	struct bpf_link *bpf_link_by_id(u32 id)
4023
4024	static int bpf_link_get_fd_by_id(const union bpf_attr *attr)
4025	{	4033	{
4026	struct bpf_link *link;	4034	struct bpf_link *link;
4027	u32 id = attr->link_id;
4028	int fd, err;
4029		4035
4030	if (CHECK_ATTR(BPF_LINK_GET_FD_BY_ID))	4036	if (!id)
4031	return -EINVAL;	4037	return ERR_PTR(-ENOENT);
4032
4033	if (!capable(CAP_SYS_ADMIN))
4034	return -EPERM;
4035		4038
4036	spin_lock_bh(&link_idr_lock);	4039	spin_lock_bh(&link_idr_lock);
4037	link = idr_find(&link_idr, id);
4038	/* before link is "settled", ID is 0, pretend it doesn't exist yet */	4040	/* before link is "settled", ID is 0, pretend it doesn't exist yet */
		4041	link = idr_find(&link_idr, id);
4039	if (link) {	4042	if (link) {
4040	if (link->id)	4043	if (link->id)
4041	err = bpf_link_inc_not_zero(link);	4044	link = bpf_link_inc_not_zero(link);
4042	else	4045	else
4043	err = -EAGAIN;	4046	link = ERR_PTR(-EAGAIN);
4044	} else {	4047	} else {
4045	err = -ENOENT;	4048	link = ERR_PTR(-ENOENT);
4046	}	4049	}
4047	spin_unlock_bh(&link_idr_lock);	4050	spin_unlock_bh(&link_idr_lock);
		4051	return link;
		4052	}
4048		4053
4049	if (err)	4054	#define BPF_LINK_GET_FD_BY_ID_LAST_FIELD link_id
4050	return err;	4055
		4056	static int bpf_link_get_fd_by_id(const union bpf_attr *attr)
		4057	{
		4058	struct bpf_link *link;
		4059	u32 id = attr->link_id;
		4060	int fd;
		4061
		4062	if (CHECK_ATTR(BPF_LINK_GET_FD_BY_ID))
		4063	return -EINVAL;
		4064
		4065	if (!capable(CAP_SYS_ADMIN))
		4066	return -EPERM;
		4067
		4068	link = bpf_link_by_id(id);
		4069	if (IS_ERR(link))
		4070	return PTR_ERR(link);
4051		4071
4052	fd = bpf_link_new_fd(link);	4072	fd = bpf_link_new_fd(link);
4053	if (fd < 0)	4073	if (fd < 0)

kernel/bpf/trampoline.c+26−3

@@ -7,6 +7,8 @@ · 6 unmodified lines
7	#include <linux/rbtree_latch.h>	7	#include <linux/rbtree_latch.h>
8	#include <linux/perf_event.h>	8	#include <linux/perf_event.h>
9	#include <linux/btf.h>	9	#include <linux/btf.h>
		10	#include <linux/rcupdate_trace.h>
		11	#include <linux/rcupdate_wait.h>
10		12
11	/* dummy _ops. The verifier will operate on target program's ops. */	13	/* dummy _ops. The verifier will operate on target program's ops. */
12	const struct bpf_verifier_ops bpf_extension_verifier_ops = {	14	const struct bpf_verifier_ops bpf_extension_verifier_ops = {
@@ -210,9 +212,12 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr) · 197 unmodified lines
210	* updates to trampoline would change the code from underneath the	212	* updates to trampoline would change the code from underneath the
211	* preempted task. Hence wait for tasks to voluntarily schedule or go	213	* preempted task. Hence wait for tasks to voluntarily schedule or go
212	* to userspace.	214	* to userspace.
		215	* The same trampoline can hold both sleepable and non-sleepable progs.
		216	* synchronize_rcu_tasks_trace() is needed to make sure all sleepable
		217	* programs finish executing.
		218	* Wait for these two grace periods together.
213	*/	219	*/
214		220	synchronize_rcu_mult(call_rcu_tasks, call_rcu_tasks_trace);
215	synchronize_rcu_tasks();
216		221
217	err = arch_prepare_bpf_trampoline(new_image, new_image + PAGE_SIZE / 2,	222	err = arch_prepare_bpf_trampoline(new_image, new_image + PAGE_SIZE / 2,
218	&tr->func.model, flags, tprogs,	223	&tr->func.model, flags, tprogs,
@@ -344,7 +349,14 @@ void bpf_trampoline_put(struct bpf_trampoline *tr) · 125 unmodified lines
344	if (WARN_ON_ONCE(!hlist_empty(&tr->progs_hlist[BPF_TRAMP_FEXIT])))	349	if (WARN_ON_ONCE(!hlist_empty(&tr->progs_hlist[BPF_TRAMP_FEXIT])))
345	goto out;	350	goto out;
346	bpf_image_ksym_del(&tr->ksym);	351	bpf_image_ksym_del(&tr->ksym);
347	/* wait for tasks to get out of trampoline before freeing it */	352	/* This code will be executed when all bpf progs (both sleepable and
		353	* non-sleepable) went through
		354	* bpf_prog_put()->call_rcu[_tasks_trace]()->bpf_prog_free_deferred().
		355	* Hence no need for another synchronize_rcu_tasks_trace() here,
		356	* but synchronize_rcu_tasks() is still needed, since trampoline
		357	* may not have had any sleepable programs and we need to wait
		358	* for tasks to get out of trampoline code before freeing it.
		359	*/
348	synchronize_rcu_tasks();	360	synchronize_rcu_tasks();
349	bpf_jit_free_exec(tr->image);	361	bpf_jit_free_exec(tr->image);
350	hlist_del(&tr->hlist);	362	hlist_del(&tr->hlist);
@@ -394,6 +406,17 @@ void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start) · 43 unmodified lines
394	rcu_read_unlock();	406	rcu_read_unlock();
395	}	407	}
396		408
		409	void notrace __bpf_prog_enter_sleepable(void)
		410	{
		411	rcu_read_lock_trace();
		412	might_fault();
		413	}
		414
		415	void notrace __bpf_prog_exit_sleepable(void)
		416	{
		417	rcu_read_unlock_trace();
		418	}
		419
397	int __weak	420	int __weak
398	arch_prepare_bpf_trampoline(void image, void image_end,	421	arch_prepare_bpf_trampoline(void image, void image_end,
399	const struct btf_func_model *m, u32 flags,	422	const struct btf_func_model *m, u32 flags,

kernel/bpf/verifier.c+263−20

@@ -21,6 +21,7 @@ · 20 unmodified lines
21	#include <linux/ctype.h>	21	#include <linux/ctype.h>
22	#include <linux/error-injection.h>	22	#include <linux/error-injection.h>
23	#include <linux/bpf_lsm.h>	23	#include <linux/bpf_lsm.h>
		24	#include <linux/btf_ids.h>
24		25
25	#include "disasm.h"	26	#include "disasm.h"
26		27
@@ -2625,11 +2626,19 @@ static int check_map_access(struct bpf_verifier_env *env, u32 regno, · 2598 unmodified lines
2625		2626
2626	#define MAX_PACKET_OFF 0xffff	2627	#define MAX_PACKET_OFF 0xffff
2627		2628
		2629	static enum bpf_prog_type resolve_prog_type(struct bpf_prog *prog)
		2630	{
		2631	return prog->aux->linked_prog ? prog->aux->linked_prog->type
		2632	: prog->type;
		2633	}
		2634
2628	static bool may_access_direct_pkt_data(struct bpf_verifier_env *env,	2635	static bool may_access_direct_pkt_data(struct bpf_verifier_env *env,
2629	const struct bpf_call_arg_meta *meta,	2636	const struct bpf_call_arg_meta *meta,
2630	enum bpf_access_type t)	2637	enum bpf_access_type t)
2631	{	2638	{
2632	switch (env->prog->type) {	2639	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
		2640
		2641	switch (prog_type) {
2633	/* Program types only with direct read access go here! */	2642	/* Program types only with direct read access go here! */
2634	case BPF_PROG_TYPE_LWT_IN:	2643	case BPF_PROG_TYPE_LWT_IN:
2635	case BPF_PROG_TYPE_LWT_OUT:	2644	case BPF_PROG_TYPE_LWT_OUT:
@@ -3872,6 +3881,33 @@ static int int_ptr_type_to_size(enum bpf_arg_type type) · 1236 unmodified lines
3872	return -EINVAL;	3881	return -EINVAL;
3873	}	3882	}
3874		3883
		3884	static int resolve_map_arg_type(struct bpf_verifier_env *env,
		3885	const struct bpf_call_arg_meta *meta,
		3886	enum bpf_arg_type *arg_type)
		3887	{
		3888	if (!meta->map_ptr) {
		3889	/* kernel subsystem misconfigured verifier */
		3890	verbose(env, "invalid map_ptr to access map->type\n");
		3891	return -EACCES;
		3892	}
		3893
		3894	switch (meta->map_ptr->map_type) {
		3895	case BPF_MAP_TYPE_SOCKMAP:
		3896	case BPF_MAP_TYPE_SOCKHASH:
		3897	if (*arg_type == ARG_PTR_TO_MAP_VALUE) {
		3898	*arg_type = ARG_PTR_TO_SOCKET;
		3899	} else {
		3900	verbose(env, "invalid arg_type for sockmap/sockhash\n");
		3901	return -EINVAL;
		3902	}
		3903	break;
		3904
		3905	default:
		3906	break;
		3907	}
		3908	return 0;
		3909	}
		3910
3875	static int check_func_arg(struct bpf_verifier_env *env, u32 arg,	3911	static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
3876	struct bpf_call_arg_meta *meta,	3912	struct bpf_call_arg_meta *meta,
3877	const struct bpf_func_proto *fn)	3913	const struct bpf_func_proto *fn)
@@ -3904,6 +3940,14 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, · 26 unmodified lines
3904	return -EACCES;	3940	return -EACCES;
3905	}	3941	}
3906		3942
		3943	if (arg_type == ARG_PTR_TO_MAP_VALUE \|\|
		3944	arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE \|\|
		3945	arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL) {
		3946	err = resolve_map_arg_type(env, meta, &arg_type);
		3947	if (err)
		3948	return err;
		3949	}
		3950
3907	if (arg_type == ARG_PTR_TO_MAP_KEY \|\|	3951	if (arg_type == ARG_PTR_TO_MAP_KEY \|\|
3908	arg_type == ARG_PTR_TO_MAP_VALUE \|\|	3952	arg_type == ARG_PTR_TO_MAP_VALUE \|\|
3909	arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE \|\|	3953	arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE \|\|
@@ -3960,16 +4004,21 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, · 50 unmodified lines
3960	goto err_type;	4004	goto err_type;
3961	}	4005	}
3962	} else if (arg_type == ARG_PTR_TO_BTF_ID) {	4006	} else if (arg_type == ARG_PTR_TO_BTF_ID) {
		4007	bool ids_match = false;
		4008
3963	expected_type = PTR_TO_BTF_ID;	4009	expected_type = PTR_TO_BTF_ID;
3964	if (type != expected_type)	4010	if (type != expected_type)
3965	goto err_type;	4011	goto err_type;
3966	if (!fn->check_btf_id) {	4012	if (!fn->check_btf_id) {
3967	if (reg->btf_id != meta->btf_id) {	4013	if (reg->btf_id != meta->btf_id) {
3968	verbose(env, "Helper has type %s got %s in R%d\n",	4014	ids_match = btf_struct_ids_match(&env->log, reg->off, reg->btf_id,
3969	kernel_type_name(meta->btf_id),	4015	meta->btf_id);
3970	kernel_type_name(reg->btf_id), regno);	4016	if (!ids_match) {
3971		4017	verbose(env, "Helper has type %s got %s in R%d\n",
3972	return -EACCES;	4018	kernel_type_name(meta->btf_id),
		4019	kernel_type_name(reg->btf_id), regno);
		4020	return -EACCES;
		4021	}
3973	}	4022	}
3974	} else if (!fn->check_btf_id(reg->btf_id, arg)) {	4023	} else if (!fn->check_btf_id(reg->btf_id, arg)) {
3975	verbose(env, "Helper does not support %s in R%d\n",	4024	verbose(env, "Helper does not support %s in R%d\n",
@@ -3977,7 +4026,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, · 1 unmodified line
3977		4026
3978	return -EACCES;	4027	return -EACCES;
3979	}	4028	}
3980	if (!tnum_is_const(reg->var_off) \|\| reg->var_off.value \|\| reg->off) {	4029	if ((reg->off && !ids_match) \|\| !tnum_is_const(reg->var_off) \|\| reg->var_off.value) {
3981	verbose(env, "R%d is a pointer to in-kernel struct with non-zero offset\n",	4030	verbose(env, "R%d is a pointer to in-kernel struct with non-zero offset\n",
3982	regno);	4031	regno);
3983	return -EACCES;	4032	return -EACCES;
@@ -4143,6 +4192,38 @@ err_type: · 159 unmodified lines
4143	return -EACCES;	4192	return -EACCES;
4144	}	4193	}
4145		4194
		4195	static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id)
		4196	{
		4197	enum bpf_attach_type eatype = env->prog->expected_attach_type;
		4198	enum bpf_prog_type type = resolve_prog_type(env->prog);
		4199
		4200	if (func_id != BPF_FUNC_map_update_elem)
		4201	return false;
		4202
		4203	/* It's not possible to get access to a locked struct sock in these
		4204	* contexts, so updating is safe.
		4205	*/
		4206	switch (type) {
		4207	case BPF_PROG_TYPE_TRACING:
		4208	if (eatype == BPF_TRACE_ITER)
		4209	return true;
		4210	break;
		4211	case BPF_PROG_TYPE_SOCKET_FILTER:
		4212	case BPF_PROG_TYPE_SCHED_CLS:
		4213	case BPF_PROG_TYPE_SCHED_ACT:
		4214	case BPF_PROG_TYPE_XDP:
		4215	case BPF_PROG_TYPE_SK_REUSEPORT:
		4216	case BPF_PROG_TYPE_FLOW_DISSECTOR:
		4217	case BPF_PROG_TYPE_SK_LOOKUP:
		4218	return true;
		4219	default:
		4220	break;
		4221	}
		4222
		4223	verbose(env, "cannot update sockmap in this context\n");
		4224	return false;
		4225	}
		4226
4146	static int check_map_func_compatibility(struct bpf_verifier_env *env,	4227	static int check_map_func_compatibility(struct bpf_verifier_env *env,
4147	struct bpf_map *map, int func_id)	4228	struct bpf_map *map, int func_id)
4148	{	4229	{
@@ -4214,7 +4295,8 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env, · 65 unmodified lines
4214	func_id != BPF_FUNC_map_delete_elem &&	4295	func_id != BPF_FUNC_map_delete_elem &&
4215	func_id != BPF_FUNC_msg_redirect_map &&	4296	func_id != BPF_FUNC_msg_redirect_map &&
4216	func_id != BPF_FUNC_sk_select_reuseport &&	4297	func_id != BPF_FUNC_sk_select_reuseport &&
4217	func_id != BPF_FUNC_map_lookup_elem)	4298	func_id != BPF_FUNC_map_lookup_elem &&
		4299	!may_update_sockmap(env, func_id))
4218	goto error;	4300	goto error;
4219	break;	4301	break;
4220	case BPF_MAP_TYPE_SOCKHASH:	4302	case BPF_MAP_TYPE_SOCKHASH:
@@ -4223,7 +4305,8 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env, · 2 unmodified lines
4223	func_id != BPF_FUNC_map_delete_elem &&	4305	func_id != BPF_FUNC_map_delete_elem &&
4224	func_id != BPF_FUNC_msg_redirect_hash &&	4306	func_id != BPF_FUNC_msg_redirect_hash &&
4225	func_id != BPF_FUNC_sk_select_reuseport &&	4307	func_id != BPF_FUNC_sk_select_reuseport &&
4226	func_id != BPF_FUNC_map_lookup_elem)	4308	func_id != BPF_FUNC_map_lookup_elem &&
		4309	!may_update_sockmap(env, func_id))
4227	goto error;	4310	goto error;
4228	break;	4311	break;
4229	case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY:	4312	case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY:
@@ -4242,6 +4325,11 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env, · 12 unmodified lines
4242	func_id != BPF_FUNC_sk_storage_delete)	4325	func_id != BPF_FUNC_sk_storage_delete)
4243	goto error;	4326	goto error;
4244	break;	4327	break;
		4328	case BPF_MAP_TYPE_INODE_STORAGE:
		4329	if (func_id != BPF_FUNC_inode_storage_get &&
		4330	func_id != BPF_FUNC_inode_storage_delete)
		4331	goto error;
		4332	break;
4245	default:	4333	default:
4246	break;	4334	break;
4247	}	4335	}
@@ -4315,6 +4403,11 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env, · 67 unmodified lines
4315	if (map->map_type != BPF_MAP_TYPE_SK_STORAGE)	4403	if (map->map_type != BPF_MAP_TYPE_SK_STORAGE)
4316	goto error;	4404	goto error;
4317	break;	4405	break;
		4406	case BPF_FUNC_inode_storage_get:
		4407	case BPF_FUNC_inode_storage_delete:
		4408	if (map->map_type != BPF_MAP_TYPE_INODE_STORAGE)
		4409	goto error;
		4410	break;
4318	default:	4411	default:
4319	break;	4412	break;
4320	}	4413	}
@@ -4775,6 +4868,11 @@ static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn · 454 unmodified lines
4775	return -EINVAL;	4868	return -EINVAL;
4776	}	4869	}
4777		4870
		4871	if (fn->allowed && !fn->allowed(env->prog)) {
		4872	verbose(env, "helper call is not allowed in probe\n");
		4873	return -EINVAL;
		4874	}
		4875
4778	/* With LD_ABS/IND some JITs save/restore skb from r1. */	4876	/* With LD_ABS/IND some JITs save/restore skb from r1. */
4779	changes_data = bpf_helper_changes_pkt_data(fn->func);	4877	changes_data = bpf_helper_changes_pkt_data(fn->func);
4780	if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) {	4878	if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) {
@@ -5732,6 +5830,67 @@ static void scalar_min_max_or(struct bpf_reg_state *dst_reg, · 951 unmodified lines
5732	__update_reg_bounds(dst_reg);	5830	__update_reg_bounds(dst_reg);
5733	}	5831	}
5734		5832
		5833	static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg,
		5834	struct bpf_reg_state *src_reg)
		5835	{
		5836	bool src_known = tnum_subreg_is_const(src_reg->var_off);
		5837	bool dst_known = tnum_subreg_is_const(dst_reg->var_off);
		5838	struct tnum var32_off = tnum_subreg(dst_reg->var_off);
		5839	s32 smin_val = src_reg->s32_min_value;
		5840
		5841	/* Assuming scalar64_min_max_xor will be called so it is safe
		5842	* to skip updating register for known case.
		5843	*/
		5844	if (src_known && dst_known)
		5845	return;
		5846
		5847	/* We get both minimum and maximum from the var32_off. */
		5848	dst_reg->u32_min_value = var32_off.value;
		5849	dst_reg->u32_max_value = var32_off.value \| var32_off.mask;
		5850
		5851	if (dst_reg->s32_min_value >= 0 && smin_val >= 0) {
		5852	/* XORing two positive sign numbers gives a positive,
		5853	* so safe to cast u32 result into s32.
		5854	*/
		5855	dst_reg->s32_min_value = dst_reg->u32_min_value;
		5856	dst_reg->s32_max_value = dst_reg->u32_max_value;
		5857	} else {
		5858	dst_reg->s32_min_value = S32_MIN;
		5859	dst_reg->s32_max_value = S32_MAX;
		5860	}
		5861	}
		5862
		5863	static void scalar_min_max_xor(struct bpf_reg_state *dst_reg,
		5864	struct bpf_reg_state *src_reg)
		5865	{
		5866	bool src_known = tnum_is_const(src_reg->var_off);
		5867	bool dst_known = tnum_is_const(dst_reg->var_off);
		5868	s64 smin_val = src_reg->smin_value;
		5869
		5870	if (src_known && dst_known) {
		5871	/* dst_reg->var_off.value has been updated earlier */
		5872	__mark_reg_known(dst_reg, dst_reg->var_off.value);
		5873	return;
		5874	}
		5875
		5876	/* We get both minimum and maximum from the var_off. */
		5877	dst_reg->umin_value = dst_reg->var_off.value;
		5878	dst_reg->umax_value = dst_reg->var_off.value \| dst_reg->var_off.mask;
		5879
		5880	if (dst_reg->smin_value >= 0 && smin_val >= 0) {
		5881	/* XORing two positive sign numbers gives a positive,
		5882	* so safe to cast u64 result into s64.
		5883	*/
		5884	dst_reg->smin_value = dst_reg->umin_value;
		5885	dst_reg->smax_value = dst_reg->umax_value;
		5886	} else {
		5887	dst_reg->smin_value = S64_MIN;
		5888	dst_reg->smax_value = S64_MAX;
		5889	}
		5890
		5891	__update_reg_bounds(dst_reg);
		5892	}
		5893
5735	static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg,	5894	static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg,
5736	u64 umin_val, u64 umax_val)	5895	u64 umin_val, u64 umax_val)
5737	{	5896	{
@@ -6040,6 +6199,11 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, · 302 unmodified lines
6040	scalar32_min_max_or(dst_reg, &src_reg);	6199	scalar32_min_max_or(dst_reg, &src_reg);
6041	scalar_min_max_or(dst_reg, &src_reg);	6200	scalar_min_max_or(dst_reg, &src_reg);
6042	break;	6201	break;
		6202	case BPF_XOR:
		6203	dst_reg->var_off = tnum_xor(dst_reg->var_off, src_reg.var_off);
		6204	scalar32_min_max_xor(dst_reg, &src_reg);
		6205	scalar_min_max_xor(dst_reg, &src_reg);
		6206	break;
6043	case BPF_LSH:	6207	case BPF_LSH:
6044	if (umax_val >= insn_bitness) {	6208	if (umax_val >= insn_bitness) {
6045	/* Shifts greater than 31 or 63 are undefined.	6209	/* Shifts greater than 31 or 63 are undefined.
@@ -7287,7 +7451,7 @@ static int check_ld_abs(struct bpf_verifier_env env, struct bpf_insn insn) · 1241 unmodified lines
7287	u8 mode = BPF_MODE(insn->code);	7451	u8 mode = BPF_MODE(insn->code);
7288	int i, err;	7452	int i, err;
7289		7453
7290	if (!may_access_skb(env->prog->type)) {	7454	if (!may_access_skb(resolve_prog_type(env->prog))) {
7291	verbose(env, "BPF_LD_[ABS\|IND] instructions not allowed for this program type\n");	7455	verbose(env, "BPF_LD_[ABS\|IND] instructions not allowed for this program type\n");
7292	return -EINVAL;	7456	return -EINVAL;
7293	}	7457	}
@@ -7375,11 +7539,12 @@ static int check_return_code(struct bpf_verifier_env *env) · 81 unmodified lines
7375	const struct bpf_prog *prog = env->prog;	7539	const struct bpf_prog *prog = env->prog;
7376	struct bpf_reg_state *reg;	7540	struct bpf_reg_state *reg;
7377	struct tnum range = tnum_range(0, 1);	7541	struct tnum range = tnum_range(0, 1);
		7542	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
7378	int err;	7543	int err;
7379		7544
7380	/* LSM and struct_ops func-ptr's return type could be "void" */	7545	/* LSM and struct_ops func-ptr's return type could be "void" */
7381	if ((env->prog->type == BPF_PROG_TYPE_STRUCT_OPS \|\|	7546	if ((prog_type == BPF_PROG_TYPE_STRUCT_OPS \|\|
7382	env->prog->type == BPF_PROG_TYPE_LSM) &&	7547	prog_type == BPF_PROG_TYPE_LSM) &&
7383	!prog->aux->attach_func_proto->type)	7548	!prog->aux->attach_func_proto->type)
7384	return 0;	7549	return 0;
7385		7550
@@ -7398,7 +7563,7 @@ static int check_return_code(struct bpf_verifier_env *env) · 12 unmodified lines
7398	return -EACCES;	7563	return -EACCES;
7399	}	7564	}
7400		7565
7401	switch (env->prog->type) {	7566	switch (prog_type) {
7402	case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:	7567	case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
7403	if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG \|\|	7568	if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG \|\|
7404	env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG \|\|	7569	env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG \|\|
@@ -9154,6 +9319,7 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, · 1749 unmodified lines
9154	struct bpf_prog *prog)	9319	struct bpf_prog *prog)
9155		9320
9156	{	9321	{
		9322	enum bpf_prog_type prog_type = resolve_prog_type(prog);
9157	/*	9323	/*
9158	* Validate that trace type programs use preallocated hash maps.	9324	* Validate that trace type programs use preallocated hash maps.
9159	*	9325	*
@@ -9171,8 +9337,8 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, · 11 unmodified lines
9171	* now, but warnings are emitted so developers are made aware of	9337	* now, but warnings are emitted so developers are made aware of
9172	* the unsafety and can fix their programs before this is enforced.	9338	* the unsafety and can fix their programs before this is enforced.
9173	*/	9339	*/
9174	if (is_tracing_prog_type(prog->type) && !is_preallocated_map(map)) {	9340	if (is_tracing_prog_type(prog_type) && !is_preallocated_map(map)) {
9175	if (prog->type == BPF_PROG_TYPE_PERF_EVENT) {	9341	if (prog_type == BPF_PROG_TYPE_PERF_EVENT) {
9176	verbose(env, "perf_event programs can only use preallocated hash map\n");	9342	verbose(env, "perf_event programs can only use preallocated hash map\n");
9177	return -EINVAL;	9343	return -EINVAL;
9178	}	9344	}
@@ -9184,8 +9350,8 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, · 5 unmodified lines
9184	verbose(env, "trace type programs with run-time allocated hash maps are unsafe. Switch to preallocated hash maps.\n");	9350	verbose(env, "trace type programs with run-time allocated hash maps are unsafe. Switch to preallocated hash maps.\n");
9185	}	9351	}
9186		9352
9187	if ((is_tracing_prog_type(prog->type) \|\|	9353	if ((is_tracing_prog_type(prog_type) \|\|
9188	prog->type == BPF_PROG_TYPE_SOCKET_FILTER) &&	9354	prog_type == BPF_PROG_TYPE_SOCKET_FILTER) &&
9189	map_value_has_spin_lock(map)) {	9355	map_value_has_spin_lock(map)) {
9190	verbose(env, "tracing progs cannot use bpf_spin_lock yet\n");	9356	verbose(env, "tracing progs cannot use bpf_spin_lock yet\n");
9191	return -EINVAL;	9357	return -EINVAL;
@@ -9202,6 +9368,23 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, · 10 unmodified lines
9202	return -EINVAL;	9368	return -EINVAL;
9203	}	9369	}
9204		9370
		9371	if (prog->aux->sleepable)
		9372	switch (map->map_type) {
		9373	case BPF_MAP_TYPE_HASH:
		9374	case BPF_MAP_TYPE_LRU_HASH:
		9375	case BPF_MAP_TYPE_ARRAY:
		9376	if (!is_preallocated_map(map)) {
		9377	verbose(env,
		9378	"Sleepable programs can only use preallocated hash maps\n");
		9379	return -EINVAL;
		9380	}
		9381	break;
		9382	default:
		9383	verbose(env,
		9384	"Sleepable programs can only use array and hash maps\n");
		9385	return -EINVAL;
		9386	}
		9387
9205	return 0;	9388	return 0;
9206	}	9389	}
9207		9390
@@ -9897,7 +10080,7 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env) · 689 unmodified lines
9897	insn->code = BPF_LDX \| BPF_PROBE_MEM \|	10080	insn->code = BPF_LDX \| BPF_PROBE_MEM \|
9898	BPF_SIZE((insn)->code);	10081	BPF_SIZE((insn)->code);
9899	env->prog->aux->num_exentries++;	10082	env->prog->aux->num_exentries++;
9900	} else if (env->prog->type != BPF_PROG_TYPE_STRUCT_OPS) {	10083	} else if (resolve_prog_type(env->prog) != BPF_PROG_TYPE_STRUCT_OPS) {
9901	verbose(env, "Writes through BTF pointers are not allowed\n");	10084	verbose(env, "Writes through BTF pointers are not allowed\n");
9902	return -EINVAL;	10085	return -EINVAL;
9903	}	10086	}
@@ -10820,6 +11003,37 @@ static int check_attach_modify_return(struct bpf_prog *prog, unsigned long addr) · 916 unmodified lines
10820	return -EINVAL;	11003	return -EINVAL;
10821	}	11004	}
10822		11005
		11006	/* non exhaustive list of sleepable bpf_lsm_() functions /
		11007	BTF_SET_START(btf_sleepable_lsm_hooks)
		11008	#ifdef CONFIG_BPF_LSM
		11009	BTF_ID(func, bpf_lsm_bprm_committed_creds)
		11010	#else
		11011	BTF_ID_UNUSED
		11012	#endif
		11013	BTF_SET_END(btf_sleepable_lsm_hooks)
		11014
		11015	static int check_sleepable_lsm_hook(u32 btf_id)
		11016	{
		11017	return btf_id_set_contains(&btf_sleepable_lsm_hooks, btf_id);
		11018	}
		11019
		11020	/* list of non-sleepable functions that are otherwise on
		11021	* ALLOW_ERROR_INJECTION list
		11022	*/
		11023	BTF_SET_START(btf_non_sleepable_error_inject)
		11024	/* Three functions below can be called from sleepable and non-sleepable context.
		11025	* Assume non-sleepable from bpf safety point of view.
		11026	*/
		11027	BTF_ID(func, __add_to_page_cache_locked)
		11028	BTF_ID(func, should_fail_alloc_page)
		11029	BTF_ID(func, should_failslab)
		11030	BTF_SET_END(btf_non_sleepable_error_inject)
		11031
		11032	static int check_non_sleepable_error_inject(u32 btf_id)
		11033	{
		11034	return btf_id_set_contains(&btf_non_sleepable_error_inject, btf_id);
		11035	}
		11036
10823	static int check_attach_btf_id(struct bpf_verifier_env *env)	11037	static int check_attach_btf_id(struct bpf_verifier_env *env)
10824	{	11038	{
10825	struct bpf_prog *prog = env->prog;	11039	struct bpf_prog *prog = env->prog;
@@ -10837,6 +11051,12 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) · 11 unmodified lines
10837	long addr;	11051	long addr;
10838	u64 key;	11052	u64 key;
10839		11053
		11054	if (prog->aux->sleepable && prog->type != BPF_PROG_TYPE_TRACING &&
		11055	prog->type != BPF_PROG_TYPE_LSM) {
		11056	verbose(env, "Only fentry/fexit/fmod_ret and lsm programs can be sleepable\n");
		11057	return -EINVAL;
		11058	}
		11059
10840	if (prog->type == BPF_PROG_TYPE_STRUCT_OPS)	11060	if (prog->type == BPF_PROG_TYPE_STRUCT_OPS)
10841	return check_struct_ops_btf_id(env);	11061	return check_struct_ops_btf_id(env);
10842		11062
@@ -11045,13 +11265,36 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) · 202 unmodified lines
11045	}	11265	}
11046	}	11266	}
11047		11267
11048	if (prog->expected_attach_type == BPF_MODIFY_RETURN) {	11268	if (prog->aux->sleepable) {
		11269	ret = -EINVAL;
		11270	switch (prog->type) {
		11271	case BPF_PROG_TYPE_TRACING:
		11272	/* fentry/fexit/fmod_ret progs can be sleepable only if they are
		11273	* attached to ALLOW_ERROR_INJECTION and are not in denylist.
		11274	*/
		11275	if (!check_non_sleepable_error_inject(btf_id) &&
		11276	within_error_injection_list(addr))
		11277	ret = 0;
		11278	break;
		11279	case BPF_PROG_TYPE_LSM:
		11280	/* LSM progs check that they are attached to bpf_lsm_*() funcs.
		11281	* Only some of them are sleepable.
		11282	*/
		11283	if (check_sleepable_lsm_hook(btf_id))
		11284	ret = 0;
		11285	break;
		11286	default:
		11287	break;
		11288	}
		11289	if (ret)
		11290	verbose(env, "%s is not sleepable\n",
		11291	prog->aux->attach_func_name);
		11292	} else if (prog->expected_attach_type == BPF_MODIFY_RETURN) {
11049	ret = check_attach_modify_return(prog, addr);	11293	ret = check_attach_modify_return(prog, addr);
11050	if (ret)	11294	if (ret)
11051	verbose(env, "%s() is not modifiable\n",	11295	verbose(env, "%s() is not modifiable\n",
11052	prog->aux->attach_func_name);	11296	prog->aux->attach_func_name);
11053	}	11297	}
11054
11055	if (ret)	11298	if (ret)
11056	goto out;	11299	goto out;
11057	tr->func.addr = (void *)addr;	11300	tr->func.addr = (void *)addr;

kernel/trace/bpf_trace.c+50−0

@@ -1098,6 +1098,52 @@ static const struct bpf_func_proto bpf_send_signal_thread_proto = { · 1097 unmodified lines
1098	.arg1_type = ARG_ANYTHING,	1098	.arg1_type = ARG_ANYTHING,
1099	};	1099	};
1100		1100
		1101	BPF_CALL_3(bpf_d_path, struct path , path, char , buf, u32, sz)
		1102	{
		1103	long len;
		1104	char *p;
		1105
		1106	if (!sz)
		1107	return 0;
		1108
		1109	p = d_path(path, buf, sz);
		1110	if (IS_ERR(p)) {
		1111	len = PTR_ERR(p);
		1112	} else {
		1113	len = buf + sz - p;
		1114	memmove(buf, p, len);
		1115	}
		1116
		1117	return len;
		1118	}
		1119
		1120	BTF_SET_START(btf_allowlist_d_path)
		1121	BTF_ID(func, vfs_truncate)
		1122	BTF_ID(func, vfs_fallocate)
		1123	BTF_ID(func, dentry_open)
		1124	BTF_ID(func, vfs_getattr)
		1125	BTF_ID(func, filp_close)
		1126	BTF_SET_END(btf_allowlist_d_path)
		1127
		1128	static bool bpf_d_path_allowed(const struct bpf_prog *prog)
		1129	{
		1130	return btf_id_set_contains(&btf_allowlist_d_path, prog->aux->attach_btf_id);
		1131	}
		1132
		1133	BTF_ID_LIST(bpf_d_path_btf_ids)
		1134	BTF_ID(struct, path)
		1135
		1136	static const struct bpf_func_proto bpf_d_path_proto = {
		1137	.func = bpf_d_path,
		1138	.gpl_only = false,
		1139	.ret_type = RET_INTEGER,
		1140	.arg1_type = ARG_PTR_TO_BTF_ID,
		1141	.arg2_type = ARG_PTR_TO_MEM,
		1142	.arg3_type = ARG_CONST_SIZE_OR_ZERO,
		1143	.btf_id = bpf_d_path_btf_ids,
		1144	.allowed = bpf_d_path_allowed,
		1145	};
		1146
1101	const struct bpf_func_proto *	1147	const struct bpf_func_proto *
1102	bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)	1148	bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
1103	{	1149	{
@@ -1182,6 +1228,8 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 78 unmodified lines
1182	return &bpf_jiffies64_proto;	1228	return &bpf_jiffies64_proto;
1183	case BPF_FUNC_get_task_stack:	1229	case BPF_FUNC_get_task_stack:
1184	return &bpf_get_task_stack_proto;	1230	return &bpf_get_task_stack_proto;
		1231	case BPF_FUNC_copy_from_user:
		1232	return prog->aux->sleepable ? &bpf_copy_from_user_proto : NULL;
1185	default:	1233	default:
1186	return NULL;	1234	return NULL;
1187	}	1235	}
@@ -1579,6 +1627,8 @@ tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 391 unmodified lines
1579	return prog->expected_attach_type == BPF_TRACE_ITER ?	1627	return prog->expected_attach_type == BPF_TRACE_ITER ?
1580	&bpf_seq_write_proto :	1628	&bpf_seq_write_proto :
1581	NULL;	1629	NULL;
		1630	case BPF_FUNC_d_path:
		1631	return &bpf_d_path_proto;
1582	default:	1632	default:
1583	return raw_tp_prog_func_proto(func_id, prog);	1633	return raw_tp_prog_func_proto(func_id, prog);
1584	}	1634	}

Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code