Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Alexei Starovoitov says: ==================== pull-request: bpf-next 2021-12-30 The following pull-request contains BPF updates for your *net-next* tree. We've added 72 non-merge commits during the last 20 day(s) which contain a total of 223 files changed, 3510 insertions(+), 1591 deletions(-). The main changes are: 1) Automatic setrlimit in libbpf when bpf is memcg's in the kernel, from Andrii. 2) Beautify and de-verbose verifier logs, from Christy. 3) Composable verifier types, from Hao. 4) bpf_strncmp helper, from Hou. 5) bpf.h header dependency cleanup, from Jakub. 6) get_func_[arg|ret|arg_cnt] helpers, from Jiri. 7) Sleepable local storage, from KP. 8) Extend kfunc with PTR_TO_CTX, PTR_TO_MEM argument support, from Kumar. ====================

David S. Miller committed 4 years ago

Commit e63a0232 parents ce2b6eb 9e6b19a221 files · +3506 −1591Browse files

Showing the changes to kernel (23 of 221 files). Show all

Unified · Split

23 files

kernel/bpf/bloom_filter.c +6 −0
kernel/bpf/bpf_inode_storage.c +5 −1
kernel/bpf/bpf_local_storage.c +37 −13
kernel/bpf/bpf_task_storage.c +5 −1
kernel/bpf/btf.c +88 −36
kernel/bpf/cgroup.c +1 −1
kernel/bpf/cpumap.c +2 −2
kernel/bpf/devmap.c +2 −2
kernel/bpf/helpers.c +23 −6
kernel/bpf/local_storage.c +1 −2
kernel/bpf/lpm_trie.c +1 −1
kernel/bpf/map_iter.c +2 −2
kernel/bpf/net_namespace.c +1 −0
kernel/bpf/reuseport_array.c +1 −5
kernel/bpf/ringbuf.c +1 −1
kernel/bpf/syscall.c +2 −1
kernel/bpf/trampoline.c +8 −0
kernel/bpf/verifier.c +415 −281
kernel/cgroup/cgroup.c +1 −0
kernel/sysctl.c +1 −0
kernel/trace/bpf_trace.c +67 −14
kernel/trace/trace_kprobe.c +1 −0
kernel/trace/trace_uprobe.c +1 −0

kernel/bpf/bloom_filter.c+6−0

@@ -82,6 +82,11 @@ static int bloom_map_delete_elem(struct bpf_map map, void value) · 81 unmodified lines
82	82	return -EOPNOTSUPP;
83	83	}
84	84
	85	+static int bloom_map_get_next_key(struct bpf_map map, void key, void *next_key)
	86	+{
	87	+ return -EOPNOTSUPP;
	88	+}
	89	+
85	90	static struct bpf_map bloom_map_alloc(union bpf_attr attr)
86	91	{
87	92	u32 bitset_bytes, bitset_mask, nr_hash_funcs, nr_bits;
@@ -192,6 +197,7 @@ const struct bpf_map_ops bloom_filter_map_ops = { · 104 unmodified lines
192	197	.map_meta_equal = bpf_map_meta_equal,
193	198	.map_alloc = bloom_map_alloc,
194	199	.map_free = bloom_map_free,
	200	+ .map_get_next_key = bloom_map_get_next_key,
195	201	.map_push_elem = bloom_map_push_elem,
196	202	.map_peek_elem = bloom_map_peek_elem,
197	203	.map_pop_elem = bloom_map_pop_elem,

kernel/bpf/bpf_inode_storage.c+5−1

@@ -17,6 +17,7 @@ · 16 unmodified lines
17	17	#include <linux/bpf_lsm.h>
18	18	#include <linux/btf_ids.h>
19	19	#include <linux/fdtable.h>
	20	+#include <linux/rcupdate_trace.h>
20	21
21	22	DEFINE_BPF_STORAGE_CACHE(inode_cache);
22	23
@@ -44,7 +45,8 @@ static struct bpf_local_storage_data inode_storage_lookup(struct inode inode, · 21 unmodified lines
44	45	if (!bsb)
45	46	return NULL;
46	47
47		− inode_storage = rcu_dereference(bsb->storage);
	48	+ inode_storage =
	49	+ rcu_dereference_check(bsb->storage, bpf_rcu_lock_held());
48	50	if (!inode_storage)
49	51	return NULL;
50	52
@@ -172,6 +174,7 @@ BPF_CALL_4(bpf_inode_storage_get, struct bpf_map , map, struct inode , inode, · 121 unmodified lines
172	174	{
173	175	struct bpf_local_storage_data *sdata;
174	176
	177	+ WARN_ON_ONCE(!bpf_rcu_lock_held());
175	178	if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
176	179	return (unsigned long)NULL;
177	180
@@ -204,6 +207,7 @@ BPF_CALL_4(bpf_inode_storage_get, struct bpf_map , map, struct inode , inode, · 26 unmodified lines
204	207	BPF_CALL_2(bpf_inode_storage_delete,
205	208	struct bpf_map , map, struct inode , inode)
206	209	{
	210	+ WARN_ON_ONCE(!bpf_rcu_lock_held());
207	211	if (!inode)
208	212	return -EINVAL;
209	213

kernel/bpf/bpf_local_storage.c+37−13

@@ -11,6 +11,9 @@ · 10 unmodified lines
11	11	#include <net/sock.h>
12	12	#include <uapi/linux/sock_diag.h>
13	13	#include <uapi/linux/btf.h>
	14	+#include <linux/rcupdate.h>
	15	+#include <linux/rcupdate_trace.h>
	16	+#include <linux/rcupdate_wait.h>
14	17
15	18	#define BPF_LOCAL_STORAGE_CREATE_FLAG_MASK (BPF_F_NO_PREALLOC \| BPF_F_CLONE)
16	19
@@ -81,6 +84,22 @@ bpf_selem_alloc(struct bpf_local_storage_map smap, void owner, · 64 unmodified lines
81	84	return NULL;
82	85	}
83	86
	87	+void bpf_local_storage_free_rcu(struct rcu_head *rcu)
	88	+{
	89	+ struct bpf_local_storage *local_storage;
	90	+
	91	+ local_storage = container_of(rcu, struct bpf_local_storage, rcu);
	92	+ kfree_rcu(local_storage, rcu);
	93	+}
	94	+
	95	+static void bpf_selem_free_rcu(struct rcu_head *rcu)
	96	+{
	97	+ struct bpf_local_storage_elem *selem;
	98	+
	99	+ selem = container_of(rcu, struct bpf_local_storage_elem, rcu);
	100	+ kfree_rcu(selem, rcu);
	101	+}
	102	+
84	103	/* local_storage->lock must be held and selem->local_storage == local_storage.
85	104	* The caller must ensure selem->smap is still valid to be
86	105	* dereferenced for its smap->elem_size and smap->cache_idx.
@@ -93,7 +112,7 @@ bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage, · 6 unmodified lines
93	112	bool free_local_storage;
94	113	void *owner;
95	114
96		− smap = rcu_dereference(SDATA(selem)->smap);
	115	+ smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held());
97	116	owner = local_storage->owner;
98	117
99	118	/* All uncharging on the owner must be done first.
@@ -118,12 +137,12 @@ bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage, · 18 unmodified lines
118	137	*
119	138	* Although the unlock will be done under
120	139	* rcu_read_lock(), it is more intutivie to
121		− * read if kfree_rcu(local_storage, rcu) is done
	140	+ * read if the freeing of the storage is done
122	141	* after the raw_spin_unlock_bh(&local_storage->lock).
123	142	*
124	143	* Hence, a "bool free_local_storage" is returned
125		− * to the caller which then calls the kfree_rcu()
126		− * after unlock.
	144	+ * to the caller which then calls then frees the storage after
	145	+ * all the RCU grace periods have expired.
127	146	*/
128	147	}
129	148	hlist_del_init_rcu(&selem->snode);
@@ -131,8 +150,7 @@ bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage, · 1 unmodified line
131	150	SDATA(selem))
132	151	RCU_INIT_POINTER(local_storage->cache[smap->cache_idx], NULL);
133	152
134		− kfree_rcu(selem, rcu);
135		−
	153	+ call_rcu_tasks_trace(&selem->rcu, bpf_selem_free_rcu);
136	154	return free_local_storage;
137	155	}
138	156
@@ -146,7 +164,8 @@ static void __bpf_selem_unlink_storage(struct bpf_local_storage_elem *selem) · 7 unmodified lines
146	164	/* selem has already been unlinked from sk */
147	165	return;
148	166
149		− local_storage = rcu_dereference(selem->local_storage);
	167	+ local_storage = rcu_dereference_check(selem->local_storage,
	168	+ bpf_rcu_lock_held());
150	169	raw_spin_lock_irqsave(&local_storage->lock, flags);
151	170	if (likely(selem_linked_to_storage(selem)))
152	171	free_local_storage = bpf_selem_unlink_storage_nolock(
@@ -154,7 +173,8 @@ static void __bpf_selem_unlink_storage(struct bpf_local_storage_elem *selem) · 1 unmodified line
154	173	raw_spin_unlock_irqrestore(&local_storage->lock, flags);
155	174
156	175	if (free_local_storage)
157		− kfree_rcu(local_storage, rcu);
	176	+ call_rcu_tasks_trace(&local_storage->rcu,
	177	+ bpf_local_storage_free_rcu);
158	178	}
159	179
160	180	void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,
@@ -174,7 +194,7 @@ void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem) · 13 unmodified lines
174	194	/* selem has already be unlinked from smap */
175	195	return;
176	196
177		− smap = rcu_dereference(SDATA(selem)->smap);
	197	+ smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held());
178	198	b = select_bucket(smap, selem);
179	199	raw_spin_lock_irqsave(&b->lock, flags);
180	200	if (likely(selem_linked_to_map(selem)))
@@ -213,12 +233,14 @@ bpf_local_storage_lookup(struct bpf_local_storage *local_storage, · 32 unmodified lines
213	233	struct bpf_local_storage_elem *selem;
214	234
215	235	/* Fast path (cache hit) */
216		− sdata = rcu_dereference(local_storage->cache[smap->cache_idx]);
	236	+ sdata = rcu_dereference_check(local_storage->cache[smap->cache_idx],
	237	+ bpf_rcu_lock_held());
217	238	if (sdata && rcu_access_pointer(sdata->smap) == smap)
218	239	return sdata;
219	240
220	241	/* Slow path (cache miss) */
221		− hlist_for_each_entry_rcu(selem, &local_storage->list, snode)
	242	+ hlist_for_each_entry_rcu(selem, &local_storage->list, snode,
	243	+ rcu_read_lock_trace_held())
222	244	if (rcu_access_pointer(SDATA(selem)->smap) == smap)
223	245	break;
224	246
@@ -306,7 +328,8 @@ int bpf_local_storage_alloc(void *owner, · 81 unmodified lines
306	328	* bucket->list, first_selem can be freed immediately
307	329	* (instead of kfree_rcu) because
308	330	* bpf_local_storage_map_free() does a
309		− * synchronize_rcu() before walking the bucket->list.
	331	+ * synchronize_rcu_mult (waiting for both sleepable and
	332	+ * normal programs) before walking the bucket->list.
310	333	* Hence, no one is accessing selem from the
311	334	* bucket->list under rcu_read_lock().
312	335	*/
@@ -342,7 +365,8 @@ bpf_local_storage_update(void owner, struct bpf_local_storage_map smap, · 29 unmodified lines
342	365	!map_value_has_spin_lock(&smap->map)))
343	366	return ERR_PTR(-EINVAL);
344	367
345		− local_storage = rcu_dereference(*owner_storage(smap, owner));
	368	+ local_storage = rcu_dereference_check(*owner_storage(smap, owner),
	369	+ bpf_rcu_lock_held());
346	370	if (!local_storage \|\| hlist_empty(&local_storage->list)) {
347	371	/* Very first elem for the owner */
348	372	err = check_flags(NULL, map_flags);

kernel/bpf/bpf_task_storage.c+5−1

@@ -17,6 +17,7 @@ · 16 unmodified lines
17	17	#include <uapi/linux/btf.h>
18	18	#include <linux/btf_ids.h>
19	19	#include <linux/fdtable.h>
	20	+#include <linux/rcupdate_trace.h>
20	21
21	22	DEFINE_BPF_STORAGE_CACHE(task_cache);
22	23
@@ -59,7 +60,8 @@ task_storage_lookup(struct task_struct task, struct bpf_map map, · 36 unmodified lines
59	60	struct bpf_local_storage *task_storage;
60	61	struct bpf_local_storage_map *smap;
61	62
62		− task_storage = rcu_dereference(task->bpf_storage);
	63	+ task_storage =
	64	+ rcu_dereference_check(task->bpf_storage, bpf_rcu_lock_held());
63	65	if (!task_storage)
64	66	return NULL;
65	67
@@ -229,6 +231,7 @@ BPF_CALL_4(bpf_task_storage_get, struct bpf_map , map, struct task_struct , · 163 unmodified lines
229	231	{
230	232	struct bpf_local_storage_data *sdata;
231	233
	234	+ WARN_ON_ONCE(!bpf_rcu_lock_held());
232	235	if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
233	236	return (unsigned long)NULL;
234	237
@@ -260,6 +263,7 @@ BPF_CALL_2(bpf_task_storage_delete, struct bpf_map , map, struct task_struct , · 25 unmodified lines
260	263	{
261	264	int ret;
262	265
	266	+ WARN_ON_ONCE(!bpf_rcu_lock_held());
263	267	if (!task)
264	268	return -EINVAL;
265	269

kernel/bpf/btf.c+88−36

@@ -4826,7 +4826,7 @@ struct btf bpf_prog_get_target_btf(const struct bpf_prog prog) · 4825 unmodified lines
4826	4826	return prog->aux->attach_btf;
4827	4827	}
4828	4828
4829		−static bool is_string_ptr(struct btf btf, const struct btf_type t)
	4829	+static bool is_int_ptr(struct btf btf, const struct btf_type t)
4830	4830	{
4831	4831	/* t comes in already as a pointer */
4832	4832	t = btf_type_by_id(btf, t->type);
@@ -4835,8 +4835,7 @@ static bool is_string_ptr(struct btf btf, const struct btf_type t) · 2 unmodified lines
4835	4835	if (BTF_INFO_KIND(t->info) == BTF_KIND_CONST)
4836	4836	t = btf_type_by_id(btf, t->type);
4837	4837
4838		− /* char, signed char, unsigned char */
4839		− return btf_type_is_int(t) && t->size == 1;
	4838	+ return btf_type_is_int(t);
4840	4839	}
4841	4840
4842	4841	bool btf_ctx_access(int off, int size, enum bpf_access_type type,
@@ -4941,10 +4940,12 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, · 98 unmodified lines
4941	4940	/* check for PTR_TO_RDONLY_BUF_OR_NULL or PTR_TO_RDWR_BUF_OR_NULL */
4942	4941	for (i = 0; i < prog->aux->ctx_arg_info_size; i++) {
4943	4942	const struct bpf_ctx_arg_aux *ctx_arg_info = &prog->aux->ctx_arg_info[i];
	4943	+ u32 type, flag;
4944	4944
4945		− if (ctx_arg_info->offset == off &&
4946		− (ctx_arg_info->reg_type == PTR_TO_RDONLY_BUF_OR_NULL \|\|
4947		− ctx_arg_info->reg_type == PTR_TO_RDWR_BUF_OR_NULL)) {
	4945	+ type = base_type(ctx_arg_info->reg_type);
	4946	+ flag = type_flag(ctx_arg_info->reg_type);
	4947	+ if (ctx_arg_info->offset == off && type == PTR_TO_BUF &&
	4948	+ (flag & PTR_MAYBE_NULL)) {
4948	4949	info->reg_type = ctx_arg_info->reg_type;
4949	4950	return true;
4950	4951	}
@@ -4957,7 +4958,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, · 6 unmodified lines
4957	4958	*/
4958	4959	return true;
4959	4960
4960		− if (is_string_ptr(btf, t))
	4961	+ if (is_int_ptr(btf, t))
4961	4962	return true;
4962	4963
4963	4964	/* this is a pointer to another type */
@@ -5575,12 +5576,53 @@ static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = { · 611 unmodified lines
5575	5576	#endif
5576	5577	};
5577	5578
	5579	+/* Returns true if struct is composed of scalars, 4 levels of nesting allowed */
	5580	+static bool __btf_type_is_scalar_struct(struct bpf_verifier_log *log,
	5581	+ const struct btf *btf,
	5582	+ const struct btf_type *t, int rec)
	5583	+{
	5584	+ const struct btf_type *member_type;
	5585	+ const struct btf_member *member;
	5586	+ u32 i;
	5587	+
	5588	+ if (!btf_type_is_struct(t))
	5589	+ return false;
	5590	+
	5591	+ for_each_member(i, t, member) {
	5592	+ const struct btf_array *array;
	5593	+
	5594	+ member_type = btf_type_skip_modifiers(btf, member->type, NULL);
	5595	+ if (btf_type_is_struct(member_type)) {
	5596	+ if (rec >= 3) {
	5597	+ bpf_log(log, "max struct nesting depth exceeded\n");
	5598	+ return false;
	5599	+ }
	5600	+ if (!__btf_type_is_scalar_struct(log, btf, member_type, rec + 1))
	5601	+ return false;
	5602	+ continue;
	5603	+ }
	5604	+ if (btf_type_is_array(member_type)) {
	5605	+ array = btf_type_array(member_type);
	5606	+ if (!array->nelems)
	5607	+ return false;
	5608	+ member_type = btf_type_skip_modifiers(btf, array->type, NULL);
	5609	+ if (!btf_type_is_scalar(member_type))
	5610	+ return false;
	5611	+ continue;
	5612	+ }
	5613	+ if (!btf_type_is_scalar(member_type))
	5614	+ return false;
	5615	+ }
	5616	+ return true;
	5617	+}
	5618	+
5578	5619	static int btf_check_func_arg_match(struct bpf_verifier_env *env,
5579	5620	const struct btf *btf, u32 func_id,
5580	5621	struct bpf_reg_state *regs,
5581	5622	bool ptr_to_mem_ok)
5582	5623	{
5583	5624	struct bpf_verifier_log *log = &env->log;
	5625	+ bool is_kfunc = btf_is_kernel(btf);
5584	5626	const char func_name, ref_tname;
5585	5627	const struct btf_type t, ref_t;
5586	5628	const struct btf_param *args;
@@ -5633,7 +5675,20 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, · 46 unmodified lines
5633	5675
5634	5676	ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id);
5635	5677	ref_tname = btf_name_by_offset(btf, ref_t->name_off);
5636		− if (btf_is_kernel(btf)) {
	5678	+ if (btf_get_prog_ctx_type(log, btf, t,
	5679	+ env->prog->type, i)) {
	5680	+ /* If function expects ctx type in BTF check that caller
	5681	+ * is passing PTR_TO_CTX.
	5682	+ */
	5683	+ if (reg->type != PTR_TO_CTX) {
	5684	+ bpf_log(log,
	5685	+ "arg#%d expected pointer to ctx, but got %s\n",
	5686	+ i, btf_type_str(t));
	5687	+ return -EINVAL;
	5688	+ }
	5689	+ if (check_ctx_reg(env, reg, regno))
	5690	+ return -EINVAL;
	5691	+ } else if (is_kfunc && (reg->type == PTR_TO_BTF_ID \|\| reg2btf_ids[reg->type])) {
5637	5692	const struct btf_type *reg_ref_t;
5638	5693	const struct btf *reg_btf;
5639	5694	const char *reg_ref_tname;
@@ -5649,14 +5704,9 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, · 9 unmodified lines
5649	5704	if (reg->type == PTR_TO_BTF_ID) {
5650	5705	reg_btf = reg->btf;
5651	5706	reg_ref_id = reg->btf_id;
5652		− } else if (reg2btf_ids[reg->type]) {
	5707	+ } else {
5653	5708	reg_btf = btf_vmlinux;
5654	5709	reg_ref_id = *reg2btf_ids[reg->type];
5655		− } else {
5656		− bpf_log(log, "kernel function %s args#%d expected pointer to %s %s but R%d is not a pointer to btf_id\n",
5657		− func_name, i,
5658		− btf_type_str(ref_t), ref_tname, regno);
5659		− return -EINVAL;
5660	5710	}
5661	5711
5662	5712	reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id,
@@ -5672,23 +5722,24 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, · 9 unmodified lines
5672	5722	reg_ref_tname);
5673	5723	return -EINVAL;
5674	5724	}
5675		− } else if (btf_get_prog_ctx_type(log, btf, t,
5676		− env->prog->type, i)) {
5677		− /* If function expects ctx type in BTF check that caller
5678		− * is passing PTR_TO_CTX.
5679		− */
5680		− if (reg->type != PTR_TO_CTX) {
5681		− bpf_log(log,
5682		− "arg#%d expected pointer to ctx, but got %s\n",
5683		− i, btf_type_str(t));
5684		− return -EINVAL;
5685		− }
5686		− if (check_ctx_reg(env, reg, regno))
5687		− return -EINVAL;
5688	5725	} else if (ptr_to_mem_ok) {
5689	5726	const struct btf_type *resolve_ret;
5690	5727	u32 type_size;
5691	5728
	5729	+ if (is_kfunc) {
	5730	+ /* Permit pointer to mem, but only when argument
	5731	+ * type is pointer to scalar, or struct composed
	5732	+ * (recursively) of scalars.
	5733	+ */
	5734	+ if (!btf_type_is_scalar(ref_t) &&
	5735	+ !__btf_type_is_scalar_struct(log, btf, ref_t, 0)) {
	5736	+ bpf_log(log,
	5737	+ "arg#%d pointer type %s %s must point to scalar or struct with scalar\n",
	5738	+ i, btf_type_str(ref_t), ref_tname);
	5739	+ return -EINVAL;
	5740	+ }
	5741	+ }
	5742	+
5692	5743	resolve_ret = btf_resolve_size(btf, ref_t, &type_size);
5693	5744	if (IS_ERR(resolve_ret)) {
5694	5745	bpf_log(log,
@@ -5701,6 +5752,8 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, · 6 unmodified lines
5701	5752	if (check_mem_reg(env, reg, regno, type_size))
5702	5753	return -EINVAL;
5703	5754	} else {
	5755	+ bpf_log(log, "reg type unsupported for arg#%d %sfunction %s#%d\n", i,
	5756	+ is_kfunc ? "kernel " : "", func_name, func_id);
5704	5757	return -EINVAL;
5705	5758	}
5706	5759	}
@@ -5750,7 +5803,7 @@ int btf_check_kfunc_arg_match(struct bpf_verifier_env *env, · 43 unmodified lines
5750	5803	const struct btf *btf, u32 func_id,
5751	5804	struct bpf_reg_state *regs)
5752	5805	{
5753		− return btf_check_func_arg_match(env, btf, func_id, regs, false);
	5806	+ return btf_check_func_arg_match(env, btf, func_id, regs, true);
5754	5807	}
5755	5808
5756	5809	/* Convert BTF of a function into bpf_reg_state if possible
@@ -5858,7 +5911,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, · 101 unmodified lines
5858	5911	return -EINVAL;
5859	5912	}
5860	5913
5861		− reg->type = PTR_TO_MEM_OR_NULL;
	5914	+ reg->type = PTR_TO_MEM \| PTR_MAYBE_NULL;
5862	5915	reg->id = ++env->id_gen;
5863	5916
5864	5917	continue;
@@ -6352,7 +6405,7 @@ const struct bpf_func_proto bpf_btf_find_by_name_kind_proto = { · 487 unmodified lines
6352	6405	.func = bpf_btf_find_by_name_kind,
6353	6406	.gpl_only = false,
6354	6407	.ret_type = RET_INTEGER,
6355		− .arg1_type = ARG_PTR_TO_MEM,
	6408	+ .arg1_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
6356	6409	.arg2_type = ARG_CONST_SIZE,
6357	6410	.arg3_type = ARG_ANYTHING,
6358	6411	.arg4_type = ARG_ANYTHING,
@@ -6534,12 +6587,11 @@ static struct bpf_cand_cache populate_cand_cache(struct bpf_cand_cache cands, · 175 unmodified lines
6534	6587	bpf_free_cands_from_cache(*cc);
6535	6588	*cc = NULL;
6536	6589	}
6537		− new_cands = kmalloc(sizeof_cands(cands->cnt), GFP_KERNEL);
	6590	+ new_cands = kmemdup(cands, sizeof_cands(cands->cnt), GFP_KERNEL);
6538	6591	if (!new_cands) {
6539	6592	bpf_free_cands(cands);
6540	6593	return ERR_PTR(-ENOMEM);
6541	6594	}
6542		− memcpy(new_cands, cands, sizeof_cands(cands->cnt));
6543	6595	/* strdup the name, since it will stay in cache.
6544	6596	* the cands->name points to strings in prog's BTF and the prog can be unloaded.
6545	6597	*/
@@ -6657,7 +6709,7 @@ bpf_core_find_cands(struct bpf_core_ctx *ctx, u32 local_type_id) · 111 unmodified lines
6657	6709
6658	6710	main_btf = bpf_get_btf_vmlinux();
6659	6711	if (IS_ERR(main_btf))
6660		− return (void *)main_btf;
	6712	+ return ERR_CAST(main_btf);
6661	6713
6662	6714	local_type = btf_type_by_id(local_btf, local_type_id);
6663	6715	if (!local_type)
@@ -6684,14 +6736,14 @@ bpf_core_find_cands(struct bpf_core_ctx *ctx, u32 local_type_id) · 20 unmodified lines
6684	6736	/* Attempt to find target candidates in vmlinux BTF first */
6685	6737	cands = bpf_core_add_cands(cands, main_btf, 1);
6686	6738	if (IS_ERR(cands))
6687		− return cands;
	6739	+ return ERR_CAST(cands);
6688	6740
6689	6741	/* cands is a pointer to kmalloced memory here if cands->cnt > 0 */
6690	6742
6691	6743	/* populate cache even when cands->cnt == 0 */
6692	6744	cc = populate_cand_cache(cands, vmlinux_cand_cache, VMLINUX_CAND_CACHE_SIZE);
6693	6745	if (IS_ERR(cc))
6694		− return cc;
	6746	+ return ERR_CAST(cc);
6695	6747
6696	6748	/* if vmlinux BTF has any candidate, don't go for module BTFs */
6697	6749	if (cc->cnt)
@@ -6717,7 +6769,7 @@ check_modules: · 19 unmodified lines
6717	6769	cands = bpf_core_add_cands(cands, mod_btf, btf_nr_types(main_btf));
6718	6770	if (IS_ERR(cands)) {
6719	6771	btf_put(mod_btf);
6720		− return cands;
	6772	+ return ERR_CAST(cands);
6721	6773	}
6722	6774	spin_lock_bh(&btf_idr_lock);
6723	6775	btf_put(mod_btf);

kernel/bpf/cgroup.c+1−1

@@ -1789,7 +1789,7 @@ static const struct bpf_func_proto bpf_sysctl_set_new_value_proto = { · 1788 unmodified lines
1789	1789	.gpl_only = false,
1790	1790	.ret_type = RET_INTEGER,
1791	1791	.arg1_type = ARG_PTR_TO_CTX,
1792		− .arg2_type = ARG_PTR_TO_MEM,
	1792	+ .arg2_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
1793	1793	.arg3_type = ARG_CONST_SIZE,
1794	1794	};
1795	1795

kernel/bpf/cpumap.c+2−2

@@ -195,7 +195,7 @@ static void cpu_map_bpf_prog_run_skb(struct bpf_cpu_map_entry *rcpu, · 194 unmodified lines
195	195	}
196	196	return;
197	197	default:
198		− bpf_warn_invalid_xdp_action(act);
	198	+ bpf_warn_invalid_xdp_action(NULL, rcpu->prog, act);
199	199	fallthrough;
200	200	case XDP_ABORTED:
201	201	trace_xdp_exception(skb->dev, rcpu->prog, act);
@@ -254,7 +254,7 @@ static int cpu_map_bpf_prog_run_xdp(struct bpf_cpu_map_entry *rcpu, · 52 unmodified lines
254	254	}
255	255	break;
256	256	default:
257		− bpf_warn_invalid_xdp_action(act);
	257	+ bpf_warn_invalid_xdp_action(NULL, rcpu->prog, act);
258	258	fallthrough;
259	259	case XDP_DROP:
260	260	xdp_return_frame(xdpf);

kernel/bpf/devmap.c+2−2

@@ -348,7 +348,7 @@ static int dev_map_bpf_prog_run(struct bpf_prog *xdp_prog, · 347 unmodified lines
348	348	frames[nframes++] = xdpf;
349	349	break;
350	350	default:
351		− bpf_warn_invalid_xdp_action(act);
	351	+ bpf_warn_invalid_xdp_action(NULL, xdp_prog, act);
352	352	fallthrough;
353	353	case XDP_ABORTED:
354	354	trace_xdp_exception(dev, xdp_prog, act);
@@ -507,7 +507,7 @@ static u32 dev_map_bpf_prog_run_skb(struct sk_buff *skb, struct bpf_dtab_netdev · 152 unmodified lines
507	507	__skb_push(skb, skb->mac_len);
508	508	break;
509	509	default:
510		− bpf_warn_invalid_xdp_action(act);
	510	+ bpf_warn_invalid_xdp_action(NULL, dst->xdp_prog, act);
511	511	fallthrough;
512	512	case XDP_ABORTED:
513	513	trace_xdp_exception(dst->dev, dst->xdp_prog, act);

kernel/bpf/helpers.c+23−6

@@ -2,6 +2,7 @@ · 1 unmodified line
2	2	/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
3	3	*/
4	4	#include <linux/bpf.h>
	5	+#include <linux/bpf-cgroup.h>
5	6	#include <linux/rcupdate.h>
6	7	#include <linux/random.h>
7	8	#include <linux/smp.h>
@@ -530,7 +531,7 @@ const struct bpf_func_proto bpf_strtol_proto = { · 522 unmodified lines
530	531	.func = bpf_strtol,
531	532	.gpl_only = false,
532	533	.ret_type = RET_INTEGER,
533		− .arg1_type = ARG_PTR_TO_MEM,
	534	+ .arg1_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
534	535	.arg2_type = ARG_CONST_SIZE,
535	536	.arg3_type = ARG_ANYTHING,
536	537	.arg4_type = ARG_PTR_TO_LONG,
@@ -558,13 +559,27 @@ const struct bpf_func_proto bpf_strtoul_proto = { · 21 unmodified lines
558	559	.func = bpf_strtoul,
559	560	.gpl_only = false,
560	561	.ret_type = RET_INTEGER,
561		− .arg1_type = ARG_PTR_TO_MEM,
	562	+ .arg1_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
562	563	.arg2_type = ARG_CONST_SIZE,
563	564	.arg3_type = ARG_ANYTHING,
564	565	.arg4_type = ARG_PTR_TO_LONG,
565	566	};
566	567	#endif
567	568
	569	+BPF_CALL_3(bpf_strncmp, const char , s1, u32, s1_sz, const char , s2)
	570	+{
	571	+ return strncmp(s1, s2, s1_sz);
	572	+}
	573	+
	574	+const struct bpf_func_proto bpf_strncmp_proto = {
	575	+ .func = bpf_strncmp,
	576	+ .gpl_only = false,
	577	+ .ret_type = RET_INTEGER,
	578	+ .arg1_type = ARG_PTR_TO_MEM,
	579	+ .arg2_type = ARG_CONST_SIZE,
	580	+ .arg3_type = ARG_PTR_TO_CONST_STR,
	581	+};
	582	+
568	583	BPF_CALL_4(bpf_get_ns_current_pid_tgid, u64, dev, u64, ino,
569	584	struct bpf_pidns_info *, nsdata, u32, size)
570	585	{
@@ -630,7 +645,7 @@ const struct bpf_func_proto bpf_event_output_data_proto = { · 59 unmodified lines
630	645	.arg1_type = ARG_PTR_TO_CTX,
631	646	.arg2_type = ARG_CONST_MAP_PTR,
632	647	.arg3_type = ARG_ANYTHING,
633		− .arg4_type = ARG_PTR_TO_MEM,
	648	+ .arg4_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
634	649	.arg5_type = ARG_CONST_SIZE_OR_ZERO,
635	650	};
636	651
@@ -667,7 +682,7 @@ BPF_CALL_2(bpf_per_cpu_ptr, const void *, ptr, u32, cpu) · 30 unmodified lines
667	682	const struct bpf_func_proto bpf_per_cpu_ptr_proto = {
668	683	.func = bpf_per_cpu_ptr,
669	684	.gpl_only = false,
670		− .ret_type = RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL,
	685	+ .ret_type = RET_PTR_TO_MEM_OR_BTF_ID \| PTR_MAYBE_NULL \| MEM_RDONLY,
671	686	.arg1_type = ARG_PTR_TO_PERCPU_BTF_ID,
672	687	.arg2_type = ARG_ANYTHING,
673	688	};
@@ -680,7 +695,7 @@ BPF_CALL_1(bpf_this_cpu_ptr, const void *, percpu_ptr) · 6 unmodified lines
680	695	const struct bpf_func_proto bpf_this_cpu_ptr_proto = {
681	696	.func = bpf_this_cpu_ptr,
682	697	.gpl_only = false,
683		− .ret_type = RET_PTR_TO_MEM_OR_BTF_ID,
	698	+ .ret_type = RET_PTR_TO_MEM_OR_BTF_ID \| MEM_RDONLY,
684	699	.arg1_type = ARG_PTR_TO_PERCPU_BTF_ID,
685	700	};
686	701
@@ -1011,7 +1026,7 @@ const struct bpf_func_proto bpf_snprintf_proto = { · 324 unmodified lines
1011	1026	.arg1_type = ARG_PTR_TO_MEM_OR_NULL,
1012	1027	.arg2_type = ARG_CONST_SIZE_OR_ZERO,
1013	1028	.arg3_type = ARG_PTR_TO_CONST_STR,
1014		− .arg4_type = ARG_PTR_TO_MEM_OR_NULL,
	1029	+ .arg4_type = ARG_PTR_TO_MEM \| PTR_MAYBE_NULL \| MEM_RDONLY,
1015	1030	.arg5_type = ARG_CONST_SIZE_OR_ZERO,
1016	1031	};
1017	1032
@@ -1378,6 +1393,8 @@ bpf_base_func_proto(enum bpf_func_id func_id) · 360 unmodified lines
1378	1393	return &bpf_for_each_map_elem_proto;
1379	1394	case BPF_FUNC_loop:
1380	1395	return &bpf_loop_proto;
	1396	+ case BPF_FUNC_strncmp:
	1397	+ return &bpf_strncmp_proto;
1381	1398	default:
1382	1399	break;
1383	1400	}

kernel/bpf/local_storage.c+1−2

@@ -163,8 +163,7 @@ static int cgroup_storage_update_elem(struct bpf_map map, void key, · 162 unmodified lines
163	163	return 0;
164	164	}
165	165
166		− new = bpf_map_kmalloc_node(map, sizeof(struct bpf_storage_buffer) +
167		− map->value_size,
	166	+ new = bpf_map_kmalloc_node(map, struct_size(new, data, map->value_size),
168	167	__GFP_ZERO \| GFP_ATOMIC \| __GFP_NOWARN,
169	168	map->numa_node);
170	169	if (!new)

kernel/bpf/lpm_trie.c+1−1

@@ -412,7 +412,7 @@ static int trie_update_elem(struct bpf_map *map, · 411 unmodified lines
412	412	rcu_assign_pointer(im_node->child[1], node);
413	413	}
414	414
415		− /* Finally, assign the intermediate node to the determined spot */
	415	+ /* Finally, assign the intermediate node to the determined slot */
416	416	rcu_assign_pointer(*slot, im_node);
417	417
418	418	out:

kernel/bpf/map_iter.c+2−2

@@ -174,9 +174,9 @@ static const struct bpf_iter_reg bpf_map_elem_reg_info = { · 173 unmodified lines
174	174	.ctx_arg_info_size = 2,
175	175	.ctx_arg_info = {
176	176	{ offsetof(struct bpf_iter__bpf_map_elem, key),
177		− PTR_TO_RDONLY_BUF_OR_NULL },
	177	+ PTR_TO_BUF \| PTR_MAYBE_NULL \| MEM_RDONLY },
178	178	{ offsetof(struct bpf_iter__bpf_map_elem, value),
179		− PTR_TO_RDWR_BUF_OR_NULL },
	179	+ PTR_TO_BUF \| PTR_MAYBE_NULL },
180	180	},
181	181	};
182	182

kernel/bpf/net_namespace.c+1−0

@@ -1,6 +1,7 @@
1	1	// SPDX-License-Identifier: GPL-2.0
2	2
3	3	#include <linux/bpf.h>
	4	+#include <linux/bpf-netns.h>
4	5	#include <linux/filter.h>
5	6	#include <net/net_namespace.h>
6	7

kernel/bpf/reuseport_array.c+1−5

@@ -152,16 +152,12 @@ static struct bpf_map reuseport_array_alloc(union bpf_attr attr) · 151 unmodified lines
152	152	{
153	153	int numa_node = bpf_map_attr_numa_node(attr);
154	154	struct reuseport_array *array;
155		− u64 array_size;
156	155
157	156	if (!bpf_capable())
158	157	return ERR_PTR(-EPERM);
159	158
160		− array_size = sizeof(*array);
161		− array_size += (u64)attr->max_entries * sizeof(struct sock *);
162		−
163	159	/* allocate all map elements and zero-initialize them */
164		− array = bpf_map_area_alloc(array_size, numa_node);
	160	+ array = bpf_map_area_alloc(struct_size(array, ptrs, attr->max_entries), numa_node);
165	161	if (!array)
166	162	return ERR_PTR(-ENOMEM);
167	163

kernel/bpf/ringbuf.c+1−1

@@ -444,7 +444,7 @@ const struct bpf_func_proto bpf_ringbuf_output_proto = { · 443 unmodified lines
444	444	.func = bpf_ringbuf_output,
445	445	.ret_type = RET_INTEGER,
446	446	.arg1_type = ARG_CONST_MAP_PTR,
447		− .arg2_type = ARG_PTR_TO_MEM,
	447	+ .arg2_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
448	448	.arg3_type = ARG_CONST_SIZE_OR_ZERO,
449	449	.arg4_type = ARG_ANYTHING,
450	450	};

kernel/bpf/syscall.c+2−1

@@ -2,6 +2,7 @@ · 1 unmodified line
2	2	/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
3	3	*/
4	4	#include <linux/bpf.h>
	5	+#include <linux/bpf-cgroup.h>
5	6	#include <linux/bpf_trace.h>
6	7	#include <linux/bpf_lirc.h>
7	8	#include <linux/bpf_verifier.h>
@@ -4772,7 +4773,7 @@ static const struct bpf_func_proto bpf_sys_bpf_proto = { · 4764 unmodified lines
4772	4773	.gpl_only = false,
4773	4774	.ret_type = RET_INTEGER,
4774	4775	.arg1_type = ARG_ANYTHING,
4775		− .arg2_type = ARG_PTR_TO_MEM,
	4776	+ .arg2_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
4776	4777	.arg3_type = ARG_CONST_SIZE,
4777	4778	};
4778	4779

kernel/bpf/trampoline.c+8−0

@@ -27,6 +27,14 @@ static struct hlist_head trampoline_table[TRAMPOLINE_TABLE_SIZE]; · 26 unmodified lines
27	27	/* serializes access to trampoline_table */
28	28	static DEFINE_MUTEX(trampoline_mutex);
29	29
	30	+bool bpf_prog_has_trampoline(const struct bpf_prog *prog)
	31	+{
	32	+ enum bpf_attach_type eatype = prog->expected_attach_type;
	33	+
	34	+ return eatype == BPF_TRACE_FENTRY \|\| eatype == BPF_TRACE_FEXIT \|\|
	35	+ eatype == BPF_MODIFY_RETURN;
	36	+}
	37	+
30	38	void *bpf_jit_alloc_exec_page(void)
31	39	{
32	40	void *image;

kernel/bpf/verifier.c+415−281

@@ -4,6 +4,7 @@ · 3 unmodified lines
4	4	* Copyright (c) 2018 Covalent IO, Inc. http://covalent.io
5	5	*/
6	6	#include <uapi/linux/btf.h>
	7	+#include <linux/bpf-cgroup.h>
7	8	#include <linux/kernel.h>
8	9	#include <linux/types.h>
9	10	#include <linux/slab.h>
@@ -441,18 +442,6 @@ static bool reg_type_not_null(enum bpf_reg_type type) · 431 unmodified lines
441	442	type == PTR_TO_SOCK_COMMON;
442	443	}
443	444
444		−static bool reg_type_may_be_null(enum bpf_reg_type type)
445		−{
446		− return type == PTR_TO_MAP_VALUE_OR_NULL \|\|
447		− type == PTR_TO_SOCKET_OR_NULL \|\|
448		− type == PTR_TO_SOCK_COMMON_OR_NULL \|\|
449		− type == PTR_TO_TCP_SOCK_OR_NULL \|\|
450		− type == PTR_TO_BTF_ID_OR_NULL \|\|
451		− type == PTR_TO_MEM_OR_NULL \|\|
452		− type == PTR_TO_RDONLY_BUF_OR_NULL \|\|
453		− type == PTR_TO_RDWR_BUF_OR_NULL;
454		−}
455		−
456	445	static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg)
457	446	{
458	447	return reg->type == PTR_TO_MAP_VALUE &&
@@ -461,12 +450,14 @@ static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg) · 2 unmodified lines
461	450
462	451	static bool reg_type_may_be_refcounted_or_null(enum bpf_reg_type type)
463	452	{
464		− return type == PTR_TO_SOCKET \|\|
465		− type == PTR_TO_SOCKET_OR_NULL \|\|
466		− type == PTR_TO_TCP_SOCK \|\|
467		− type == PTR_TO_TCP_SOCK_OR_NULL \|\|
468		− type == PTR_TO_MEM \|\|
469		− type == PTR_TO_MEM_OR_NULL;
	453	+ return base_type(type) == PTR_TO_SOCKET \|\|
	454	+ base_type(type) == PTR_TO_TCP_SOCK \|\|
	455	+ base_type(type) == PTR_TO_MEM;
	456	+}
	457	+
	458	+static bool type_is_rdonly_mem(u32 type)
	459	+{
	460	+ return type & MEM_RDONLY;
470	461	}
471	462
472	463	static bool arg_type_may_be_refcounted(enum bpf_arg_type type)
@@ -474,14 +465,9 @@ static bool arg_type_may_be_refcounted(enum bpf_arg_type type) · 1 unmodified line
474	465	return type == ARG_PTR_TO_SOCK_COMMON;
475	466	}
476	467
477		−static bool arg_type_may_be_null(enum bpf_arg_type type)
	468	+static bool type_may_be_null(u32 type)
478	469	{
479		− return type == ARG_PTR_TO_MAP_VALUE_OR_NULL \|\|
480		− type == ARG_PTR_TO_MEM_OR_NULL \|\|
481		− type == ARG_PTR_TO_CTX_OR_NULL \|\|
482		− type == ARG_PTR_TO_SOCKET_OR_NULL \|\|
483		− type == ARG_PTR_TO_ALLOC_MEM_OR_NULL \|\|
484		− type == ARG_PTR_TO_STACK_OR_NULL;
	470	+ return type & PTR_MAYBE_NULL;
485	471	}
486	472
487	473	/* Determine whether the function releases some resources allocated by another
@@ -541,39 +527,54 @@ static bool is_cmpxchg_insn(const struct bpf_insn *insn) · 53 unmodified lines
541	527	insn->imm == BPF_CMPXCHG;
542	528	}
543	529
544		−/* string representation of 'enum bpf_reg_type' */
545		−static const char * const reg_type_str[] = {
546		− [NOT_INIT] = "?",
547		− [SCALAR_VALUE] = "inv",
548		− [PTR_TO_CTX] = "ctx",
549		− [CONST_PTR_TO_MAP] = "map_ptr",
550		− [PTR_TO_MAP_VALUE] = "map_value",
551		− [PTR_TO_MAP_VALUE_OR_NULL] = "map_value_or_null",
552		− [PTR_TO_STACK] = "fp",
553		− [PTR_TO_PACKET] = "pkt",
554		− [PTR_TO_PACKET_META] = "pkt_meta",
555		− [PTR_TO_PACKET_END] = "pkt_end",
556		− [PTR_TO_FLOW_KEYS] = "flow_keys",
557		− [PTR_TO_SOCKET] = "sock",
558		− [PTR_TO_SOCKET_OR_NULL] = "sock_or_null",
559		− [PTR_TO_SOCK_COMMON] = "sock_common",
560		− [PTR_TO_SOCK_COMMON_OR_NULL] = "sock_common_or_null",
561		− [PTR_TO_TCP_SOCK] = "tcp_sock",
562		− [PTR_TO_TCP_SOCK_OR_NULL] = "tcp_sock_or_null",
563		− [PTR_TO_TP_BUFFER] = "tp_buffer",
564		− [PTR_TO_XDP_SOCK] = "xdp_sock",
565		− [PTR_TO_BTF_ID] = "ptr_",
566		− [PTR_TO_BTF_ID_OR_NULL] = "ptr_or_null_",
567		− [PTR_TO_PERCPU_BTF_ID] = "percpu_ptr_",
568		− [PTR_TO_MEM] = "mem",
569		− [PTR_TO_MEM_OR_NULL] = "mem_or_null",
570		− [PTR_TO_RDONLY_BUF] = "rdonly_buf",
571		− [PTR_TO_RDONLY_BUF_OR_NULL] = "rdonly_buf_or_null",
572		− [PTR_TO_RDWR_BUF] = "rdwr_buf",
573		− [PTR_TO_RDWR_BUF_OR_NULL] = "rdwr_buf_or_null",
574		− [PTR_TO_FUNC] = "func",
575		− [PTR_TO_MAP_KEY] = "map_key",
576		−};
	530	+/* string representation of 'enum bpf_reg_type'
	531	+ *
	532	+ * Note that reg_type_str() can not appear more than once in a single verbose()
	533	+ * statement.
	534	+ */
	535	+static const char reg_type_str(struct bpf_verifier_env env,
	536	+ enum bpf_reg_type type)
	537	+{
	538	+ char postfix[16] = {0}, prefix[16] = {0};
	539	+ static const char * const str[] = {
	540	+ [NOT_INIT] = "?",
	541	+ [SCALAR_VALUE] = "inv",
	542	+ [PTR_TO_CTX] = "ctx",
	543	+ [CONST_PTR_TO_MAP] = "map_ptr",
	544	+ [PTR_TO_MAP_VALUE] = "map_value",
	545	+ [PTR_TO_STACK] = "fp",
	546	+ [PTR_TO_PACKET] = "pkt",
	547	+ [PTR_TO_PACKET_META] = "pkt_meta",
	548	+ [PTR_TO_PACKET_END] = "pkt_end",
	549	+ [PTR_TO_FLOW_KEYS] = "flow_keys",
	550	+ [PTR_TO_SOCKET] = "sock",
	551	+ [PTR_TO_SOCK_COMMON] = "sock_common",
	552	+ [PTR_TO_TCP_SOCK] = "tcp_sock",
	553	+ [PTR_TO_TP_BUFFER] = "tp_buffer",
	554	+ [PTR_TO_XDP_SOCK] = "xdp_sock",
	555	+ [PTR_TO_BTF_ID] = "ptr_",
	556	+ [PTR_TO_PERCPU_BTF_ID] = "percpu_ptr_",
	557	+ [PTR_TO_MEM] = "mem",
	558	+ [PTR_TO_BUF] = "buf",
	559	+ [PTR_TO_FUNC] = "func",
	560	+ [PTR_TO_MAP_KEY] = "map_key",
	561	+ };
	562	+
	563	+ if (type & PTR_MAYBE_NULL) {
	564	+ if (base_type(type) == PTR_TO_BTF_ID \|\|
	565	+ base_type(type) == PTR_TO_PERCPU_BTF_ID)
	566	+ strncpy(postfix, "or_null_", 16);
	567	+ else
	568	+ strncpy(postfix, "_or_null", 16);
	569	+ }
	570	+
	571	+ if (type & MEM_RDONLY)
	572	+ strncpy(prefix, "rdonly_", 16);
	573	+
	574	+ snprintf(env->type_str_buf, TYPE_STR_BUF_LEN, "%s%s%s",
	575	+ prefix, str[base_type(type)], postfix);
	576	+ return env->type_str_buf;
	577	+}
577	578
578	579	static char slot_type_char[] = {
579	580	[STACK_INVALID] = '?',
@@ -608,6 +609,44 @@ static const char kernel_type_name(const struct btf btf, u32 id) · 28 unmodified lines
608	609	return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off);
609	610	}
610	611
	612	+static void mark_reg_scratched(struct bpf_verifier_env *env, u32 regno)
	613	+{
	614	+ env->scratched_regs \|= 1U << regno;
	615	+}
	616	+
	617	+static void mark_stack_slot_scratched(struct bpf_verifier_env *env, u32 spi)
	618	+{
	619	+ env->scratched_stack_slots \|= 1UL << spi;
	620	+}
	621	+
	622	+static bool reg_scratched(const struct bpf_verifier_env *env, u32 regno)
	623	+{
	624	+ return (env->scratched_regs >> regno) & 1;
	625	+}
	626	+
	627	+static bool stack_slot_scratched(const struct bpf_verifier_env *env, u64 regno)
	628	+{
	629	+ return (env->scratched_stack_slots >> regno) & 1;
	630	+}
	631	+
	632	+static bool verifier_state_scratched(const struct bpf_verifier_env *env)
	633	+{
	634	+ return env->scratched_regs \|\| env->scratched_stack_slots;
	635	+}
	636	+
	637	+static void mark_verifier_state_clean(struct bpf_verifier_env *env)
	638	+{
	639	+ env->scratched_regs = 0U;
	640	+ env->scratched_stack_slots = 0UL;
	641	+}
	642	+
	643	+/* Used for printing the entire verifier state. */
	644	+static void mark_verifier_state_scratched(struct bpf_verifier_env *env)
	645	+{
	646	+ env->scratched_regs = ~0U;
	647	+ env->scratched_stack_slots = ~0UL;
	648	+}
	649	+
611	650	/* The reg state of a pointer or a bounded scalar was saved when
612	651	* it was spilled to the stack.
613	652	*/
@@ -623,7 +662,8 @@ static void scrub_spilled_slot(u8 *stype) · 9 unmodified lines
623	662	}
624	663
625	664	static void print_verifier_state(struct bpf_verifier_env *env,
626		− const struct bpf_func_state *state)
	665	+ const struct bpf_func_state *state,
	666	+ bool print_all)
627	667	{
628	668	const struct bpf_reg_state *reg;
629	669	enum bpf_reg_type t;
@@ -636,9 +676,11 @@ static void print_verifier_state(struct bpf_verifier_env *env, · 6 unmodified lines
636	676	t = reg->type;
637	677	if (t == NOT_INIT)
638	678	continue;
	679	+ if (!print_all && !reg_scratched(env, i))
	680	+ continue;
639	681	verbose(env, " R%d", i);
640	682	print_liveness(env, reg->live);
641		− verbose(env, "=%s", reg_type_str[t]);
	683	+ verbose(env, "=%s", reg_type_str(env, t));
642	684	if (t == SCALAR_VALUE && reg->precise)
643	685	verbose(env, "P");
644	686	if ((t == SCALAR_VALUE \|\| t == PTR_TO_STACK) &&
@@ -646,9 +688,8 @@ static void print_verifier_state(struct bpf_verifier_env *env, · 1 unmodified line
646	688	/* reg->off should be 0 for SCALAR_VALUE */
647	689	verbose(env, "%lld", reg->var_off.value + reg->off);
648	690	} else {
649		− if (t == PTR_TO_BTF_ID \|\|
650		− t == PTR_TO_BTF_ID_OR_NULL \|\|
651		− t == PTR_TO_PERCPU_BTF_ID)
	691	+ if (base_type(t) == PTR_TO_BTF_ID \|\|
	692	+ base_type(t) == PTR_TO_PERCPU_BTF_ID)
652	693	verbose(env, "%s", kernel_type_name(reg->btf, reg->btf_id));
653	694	verbose(env, "(id=%d", reg->id);
654	695	if (reg_type_may_be_refcounted_or_null(t))
@@ -657,10 +698,9 @@ static void print_verifier_state(struct bpf_verifier_env *env, · 2 unmodified lines
657	698	verbose(env, ",off=%d", reg->off);
658	699	if (type_is_pkt_pointer(t))
659	700	verbose(env, ",r=%d", reg->range);
660		− else if (t == CONST_PTR_TO_MAP \|\|
661		− t == PTR_TO_MAP_KEY \|\|
662		− t == PTR_TO_MAP_VALUE \|\|
663		− t == PTR_TO_MAP_VALUE_OR_NULL)
	701	+ else if (base_type(t) == CONST_PTR_TO_MAP \|\|
	702	+ base_type(t) == PTR_TO_MAP_KEY \|\|
	703	+ base_type(t) == PTR_TO_MAP_VALUE)
664	704	verbose(env, ",ks=%d,vs=%d",
665	705	reg->map_ptr->key_size,
666	706	reg->map_ptr->value_size);
@@ -725,12 +765,14 @@ static void print_verifier_state(struct bpf_verifier_env *env, · 58 unmodified lines
725	765	types_buf[BPF_REG_SIZE] = 0;
726	766	if (!valid)
727	767	continue;
	768	+ if (!print_all && !stack_slot_scratched(env, i))
	769	+ continue;
728	770	verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE);
729	771	print_liveness(env, state->stack[i].spilled_ptr.live);
730	772	if (is_spilled_reg(&state->stack[i])) {
731	773	reg = &state->stack[i].spilled_ptr;
732	774	t = reg->type;
733		− verbose(env, "=%s", reg_type_str[t]);
	775	+ verbose(env, "=%s", reg_type_str(env, t));
734	776	if (t == SCALAR_VALUE && reg->precise)
735	777	verbose(env, "P");
736	778	if (t == SCALAR_VALUE && tnum_is_const(reg->var_off))
@@ -750,6 +792,26 @@ static void print_verifier_state(struct bpf_verifier_env *env, · 13 unmodified lines
750	792	if (state->in_async_callback_fn)
751	793	verbose(env, " async_cb");
752	794	verbose(env, "\n");
	795	+ mark_verifier_state_clean(env);
	796	+}
	797	+
	798	+static inline u32 vlog_alignment(u32 pos)
	799	+{
	800	+ return round_up(max(pos + BPF_LOG_MIN_ALIGNMENT / 2, BPF_LOG_ALIGNMENT),
	801	+ BPF_LOG_MIN_ALIGNMENT) - pos - 1;
	802	+}
	803	+
	804	+static void print_insn_state(struct bpf_verifier_env *env,
	805	+ const struct bpf_func_state *state)
	806	+{
	807	+ if (env->prev_log_len && env->prev_log_len == env->log.len_used) {
	808	+ /* remove new line character */
	809	+ bpf_vlog_reset(&env->log, env->prev_log_len - 1);
	810	+ verbose(env, "%*c;", vlog_alignment(env->prev_insn_print_len), ' ');
	811	+ } else {
	812	+ verbose(env, "%d:", env->insn_idx);
	813	+ }
	814	+ print_verifier_state(env, state, false);
753	815	}
754	816
755	817	/* copy array src of length n * size bytes to dst. dst is reallocated if it's too
@@ -1143,8 +1205,7 @@ static void mark_reg_known_zero(struct bpf_verifier_env *env, · 387 unmodified lines
1143	1205
1144	1206	static void mark_ptr_not_null_reg(struct bpf_reg_state *reg)
1145	1207	{
1146		− switch (reg->type) {
1147		− case PTR_TO_MAP_VALUE_OR_NULL: {
	1208	+ if (base_type(reg->type) == PTR_TO_MAP_VALUE) {
1148	1209	const struct bpf_map *map = reg->map_ptr;
1149	1210
1150	1211	if (map->inner_map_meta) {
@@ -1163,32 +1224,10 @@ static void mark_ptr_not_null_reg(struct bpf_reg_state *reg) · 12 unmodified lines
1163	1224	} else {
1164	1225	reg->type = PTR_TO_MAP_VALUE;
1165	1226	}
1166		− break;
1167		− }
1168		− case PTR_TO_SOCKET_OR_NULL:
1169		− reg->type = PTR_TO_SOCKET;
1170		− break;
1171		− case PTR_TO_SOCK_COMMON_OR_NULL:
1172		− reg->type = PTR_TO_SOCK_COMMON;
1173		− break;
1174		− case PTR_TO_TCP_SOCK_OR_NULL:
1175		− reg->type = PTR_TO_TCP_SOCK;
1176		− break;
1177		− case PTR_TO_BTF_ID_OR_NULL:
1178		− reg->type = PTR_TO_BTF_ID;
1179		− break;
1180		− case PTR_TO_MEM_OR_NULL:
1181		− reg->type = PTR_TO_MEM;
1182		− break;
1183		− case PTR_TO_RDONLY_BUF_OR_NULL:
1184		− reg->type = PTR_TO_RDONLY_BUF;
1185		− break;
1186		− case PTR_TO_RDWR_BUF_OR_NULL:
1187		− reg->type = PTR_TO_RDWR_BUF;
1188		− break;
1189		− default:
1190		− WARN_ONCE(1, "unknown nullable register type");
	1227	+ return;
1191	1228	}
	1229	+
	1230	+ reg->type &= ~PTR_MAYBE_NULL;
1192	1231	}
1193	1232
1194	1233	static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg)
@@ -1546,6 +1585,7 @@ static void init_func_state(struct bpf_verifier_env *env, · 351 unmodified lines
1546	1585	state->frameno = frameno;
1547	1586	state->subprogno = subprogno;
1548	1587	init_reg_state(env, state);
	1588	+ mark_verifier_state_scratched(env);
1549	1589	}
1550	1590
1551	1591	/* Similar to push_stack(), but for async callbacks */
@@ -2049,7 +2089,7 @@ static int mark_reg_read(struct bpf_verifier_env *env, · 497 unmodified lines
2049	2089	break;
2050	2090	if (parent->live & REG_LIVE_DONE) {
2051	2091	verbose(env, "verifier BUG type %s var_off %lld off %d\n",
2052		− reg_type_str[parent->type],
	2092	+ reg_type_str(env, parent->type),
2053	2093	parent->var_off.value, parent->off);
2054	2094	return -EFAULT;
2055	2095	}
@@ -2233,6 +2273,8 @@ static int check_reg_arg(struct bpf_verifier_env *env, u32 regno, · 177 unmodified lines
2233	2273	return -EINVAL;
2234	2274	}
2235	2275
	2276	+ mark_reg_scratched(env, regno);
	2277	+
2236	2278	reg = &regs[regno];
2237	2279	rw64 = is_reg64(env, insn, regno, reg, t);
2238	2280	if (t == SRC_OP) {
@@ -2337,7 +2379,7 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, · 98 unmodified lines
2337	2379
2338	2380	if (insn->code == 0)
2339	2381	return 0;
2340		− if (env->log.level & BPF_LOG_LEVEL) {
	2382	+ if (env->log.level & BPF_LOG_LEVEL2) {
2341	2383	verbose(env, "regs=%x stack=%llx before ", reg_mask, stack_mask);
2342	2384	verbose(env, "%d: ", idx);
2343	2385	print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
@@ -2591,7 +2633,7 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int regno, · 247 unmodified lines
2591	2633	DECLARE_BITMAP(mask, 64);
2592	2634	u32 history = st->jmp_history_cnt;
2593	2635
2594		− if (env->log.level & BPF_LOG_LEVEL)
	2636	+ if (env->log.level & BPF_LOG_LEVEL2)
2595	2637	verbose(env, "last_idx %d first_idx %d\n", last_idx, first_idx);
2596	2638	for (i = last_idx;;) {
2597	2639	if (skip_first) {
@@ -2678,11 +2720,11 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int regno, · 80 unmodified lines
2678	2720	new_marks = true;
2679	2721	reg->precise = true;
2680	2722	}
2681		− if (env->log.level & BPF_LOG_LEVEL) {
2682		− print_verifier_state(env, func);
2683		− verbose(env, "parent %s regs=%x stack=%llx marks\n",
	2723	+ if (env->log.level & BPF_LOG_LEVEL2) {
	2724	+ verbose(env, "parent %s regs=%x stack=%llx marks:",
2684	2725	new_marks ? "didn't have" : "already had",
2685	2726	reg_mask, stack_mask);
	2727	+ print_verifier_state(env, func, true);
2686	2728	}
2687	2729
2688	2730	if (!reg_mask && !stack_mask)
@@ -2708,9 +2750,8 @@ static int mark_chain_precision_stack(struct bpf_verifier_env *env, int spi) · 19 unmodified lines
2708	2750
2709	2751	static bool is_spillable_regtype(enum bpf_reg_type type)
2710	2752	{
2711		− switch (type) {
	2753	+ switch (base_type(type)) {
2712	2754	case PTR_TO_MAP_VALUE:
2713		− case PTR_TO_MAP_VALUE_OR_NULL:
2714	2755	case PTR_TO_STACK:
2715	2756	case PTR_TO_CTX:
2716	2757	case PTR_TO_PACKET:
@@ -2719,21 +2760,13 @@ static bool is_spillable_regtype(enum bpf_reg_type type) · 2 unmodified lines
2719	2760	case PTR_TO_FLOW_KEYS:
2720	2761	case CONST_PTR_TO_MAP:
2721	2762	case PTR_TO_SOCKET:
2722		− case PTR_TO_SOCKET_OR_NULL:
2723	2763	case PTR_TO_SOCK_COMMON:
2724		− case PTR_TO_SOCK_COMMON_OR_NULL:
2725	2764	case PTR_TO_TCP_SOCK:
2726		− case PTR_TO_TCP_SOCK_OR_NULL:
2727	2765	case PTR_TO_XDP_SOCK:
2728	2766	case PTR_TO_BTF_ID:
2729		− case PTR_TO_BTF_ID_OR_NULL:
2730		− case PTR_TO_RDONLY_BUF:
2731		− case PTR_TO_RDONLY_BUF_OR_NULL:
2732		− case PTR_TO_RDWR_BUF:
2733		− case PTR_TO_RDWR_BUF_OR_NULL:
	2767	+ case PTR_TO_BUF:
2734	2768	case PTR_TO_PERCPU_BTF_ID:
2735	2769	case PTR_TO_MEM:
2736		− case PTR_TO_MEM_OR_NULL:
2737	2770	case PTR_TO_FUNC:
2738	2771	case PTR_TO_MAP_KEY:
2739	2772	return true;
@@ -2838,6 +2871,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, · 98 unmodified lines
2838	2871	env->insn_aux_data[insn_idx].sanitize_stack_spill = true;
2839	2872	}
2840	2873
	2874	+ mark_stack_slot_scratched(env, spi);
2841	2875	if (reg && !(off % BPF_REG_SIZE) && register_is_bounded(reg) &&
2842	2876	!register_is_null(reg) && env->bpf_capable) {
2843	2877	if (dst_reg != BPF_REG_FP) {
@@ -2959,6 +2993,7 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env, · 115 unmodified lines
2959	2993	slot = -i - 1;
2960	2994	spi = slot / BPF_REG_SIZE;
2961	2995	stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE];
	2996	+ mark_stack_slot_scratched(env, spi);
2962	2997
2963	2998	if (!env->allow_ptr_leaks
2964	2999	&& *stype != NOT_INIT
@@ -3375,11 +3410,8 @@ static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno, · 410 unmodified lines
3375	3410	/* We may have adjusted the register pointing to memory region, so we
3376	3411	* need to try adding each of min_value and max_value to off
3377	3412	* to make sure our theoretical access will be safe.
3378		− */
3379		− if (env->log.level & BPF_LOG_LEVEL)
3380		− print_verifier_state(env, state);
3381		−
3382		− /* The minimum value is only important with signed
	3413	+ *
	3414	+ * The minimum value is only important with signed
3383	3415	* comparisons where we can't assume the floor of a
3384	3416	* value is 0. If we are using signed variables for our
3385	3417	* index'es we need to make sure that whatever we use
@@ -3574,7 +3606,7 @@ static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, · 188 unmodified lines
3574	3606	*/
3575	3607	*reg_type = info.reg_type;
3576	3608
3577		− if (reg_type == PTR_TO_BTF_ID \|\| reg_type == PTR_TO_BTF_ID_OR_NULL) {
	3609	+ if (base_type(*reg_type) == PTR_TO_BTF_ID) {
3578	3610	*btf = info.btf;
3579	3611	*btf_id = info.btf_id;
3580	3612	} else {
@@ -3642,7 +3674,7 @@ static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, · 61 unmodified lines
3642	3674	}
3643	3675
3644	3676	verbose(env, "R%d invalid %s access off=%d size=%d\n",
3645		− regno, reg_type_str[reg->type], off, size);
	3677	+ regno, reg_type_str(env, reg->type), off, size);
3646	3678
3647	3679	return -EACCES;
3648	3680	}
@@ -4369,15 +4401,30 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 720 unmodified lines
4369	4401	mark_reg_unknown(env, regs, value_regno);
4370	4402	}
4371	4403	}
4372		− } else if (reg->type == PTR_TO_MEM) {
	4404	+ } else if (base_type(reg->type) == PTR_TO_MEM) {
	4405	+ bool rdonly_mem = type_is_rdonly_mem(reg->type);
	4406	+
	4407	+ if (type_may_be_null(reg->type)) {
	4408	+ verbose(env, "R%d invalid mem access '%s'\n", regno,
	4409	+ reg_type_str(env, reg->type));
	4410	+ return -EACCES;
	4411	+ }
	4412	+
	4413	+ if (t == BPF_WRITE && rdonly_mem) {
	4414	+ verbose(env, "R%d cannot write into %s\n",
	4415	+ regno, reg_type_str(env, reg->type));
	4416	+ return -EACCES;
	4417	+ }
	4418	+
4373	4419	if (t == BPF_WRITE && value_regno >= 0 &&
4374	4420	is_pointer_value(env, value_regno)) {
4375	4421	verbose(env, "R%d leaks addr into mem\n", value_regno);
4376	4422	return -EACCES;
4377	4423	}
	4424	+
4378	4425	err = check_mem_region_access(env, regno, off, size,
4379	4426	reg->mem_size, false);
4380		− if (!err && t == BPF_READ && value_regno >= 0)
	4427	+ if (!err && value_regno >= 0 && (t == BPF_READ \|\| rdonly_mem))
4381	4428	mark_reg_unknown(env, regs, value_regno);
4382	4429	} else if (reg->type == PTR_TO_CTX) {
4383	4430	enum bpf_reg_type reg_type = SCALAR_VALUE;
@@ -4407,7 +4454,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 23 unmodified lines
4407	4454	} else {
4408	4455	mark_reg_known_zero(env, regs,
4409	4456	value_regno);
4410		− if (reg_type_may_be_null(reg_type))
	4457	+ if (type_may_be_null(reg_type))
4411	4458	regs[value_regno].id = ++env->id_gen;
4412	4459	/* A load of ctx field could have different
4413	4460	* actual load size with the one encoded in the
@@ -4415,8 +4462,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 1 unmodified line
4415	4462	* a sub-register.
4416	4463	*/
4417	4464	regs[value_regno].subreg_def = DEF_NOT_SUBREG;
4418		− if (reg_type == PTR_TO_BTF_ID \|\|
4419		− reg_type == PTR_TO_BTF_ID_OR_NULL) {
	4465	+ if (base_type(reg_type) == PTR_TO_BTF_ID) {
4420	4466	regs[value_regno].btf = btf;
4421	4467	regs[value_regno].btf_id = btf_id;
4422	4468	}
@@ -4469,7 +4515,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 46 unmodified lines
4469	4515	} else if (type_is_sk_pointer(reg->type)) {
4470	4516	if (t == BPF_WRITE) {
4471	4517	verbose(env, "R%d cannot write into %s\n",
4472		− regno, reg_type_str[reg->type]);
	4518	+ regno, reg_type_str(env, reg->type));
4473	4519	return -EACCES;
4474	4520	}
4475	4521	err = check_sock_access(env, insn_idx, regno, off, size, t);
@@ -4485,26 +4531,32 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 9 unmodified lines
4485	4531	} else if (reg->type == CONST_PTR_TO_MAP) {
4486	4532	err = check_ptr_to_map_access(env, regs, regno, off, size, t,
4487	4533	value_regno);
4488		− } else if (reg->type == PTR_TO_RDONLY_BUF) {
4489		− if (t == BPF_WRITE) {
4490		− verbose(env, "R%d cannot write into %s\n",
4491		− regno, reg_type_str[reg->type]);
4492		− return -EACCES;
	4534	+ } else if (base_type(reg->type) == PTR_TO_BUF) {
	4535	+ bool rdonly_mem = type_is_rdonly_mem(reg->type);
	4536	+ const char *buf_info;
	4537	+ u32 *max_access;
	4538	+
	4539	+ if (rdonly_mem) {
	4540	+ if (t == BPF_WRITE) {
	4541	+ verbose(env, "R%d cannot write into %s\n",
	4542	+ regno, reg_type_str(env, reg->type));
	4543	+ return -EACCES;
	4544	+ }
	4545	+ buf_info = "rdonly";
	4546	+ max_access = &env->prog->aux->max_rdonly_access;
	4547	+ } else {
	4548	+ buf_info = "rdwr";
	4549	+ max_access = &env->prog->aux->max_rdwr_access;
4493	4550	}
	4551	+
4494	4552	err = check_buffer_access(env, reg, regno, off, size, false,
4495		− "rdonly",
4496		− &env->prog->aux->max_rdonly_access);
4497		− if (!err && value_regno >= 0)
4498		− mark_reg_unknown(env, regs, value_regno);
4499		− } else if (reg->type == PTR_TO_RDWR_BUF) {
4500		− err = check_buffer_access(env, reg, regno, off, size, false,
4501		− "rdwr",
4502		− &env->prog->aux->max_rdwr_access);
4503		− if (!err && t == BPF_READ && value_regno >= 0)
	4553	+ buf_info, max_access);
	4554	+
	4555	+ if (!err && value_regno >= 0 && (rdonly_mem \|\| t == BPF_READ))
4504	4556	mark_reg_unknown(env, regs, value_regno);
4505	4557	} else {
4506	4558	verbose(env, "R%d invalid mem access '%s'\n", regno,
4507		− reg_type_str[reg->type]);
	4559	+ reg_type_str(env, reg->type));
4508	4560	return -EACCES;
4509	4561	}
4510	4562
@@ -4578,7 +4630,7 @@ static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_i · 67 unmodified lines
4578	4630	is_sk_reg(env, insn->dst_reg)) {
4579	4631	verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n",
4580	4632	insn->dst_reg,
4581		− reg_type_str[reg_state(env, insn->dst_reg)->type]);
	4633	+ reg_type_str(env, reg_state(env, insn->dst_reg)->type));
4582	4634	return -EACCES;
4583	4635	}
4584	4636
@@ -4761,8 +4813,10 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, · 176 unmodified lines
4761	4813	struct bpf_call_arg_meta *meta)
4762	4814	{
4763	4815	struct bpf_reg_state regs = cur_regs(env), reg = &regs[regno];
	4816	+ const char *buf_info;
	4817	+ u32 *max_access;
4764	4818
4765		− switch (reg->type) {
	4819	+ switch (base_type(reg->type)) {
4766	4820	case PTR_TO_PACKET:
4767	4821	case PTR_TO_PACKET_META:
4768	4822	return check_packet_access(env, regno, reg->off, access_size,
@@ -4781,18 +4835,20 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, · 12 unmodified lines
4781	4835	return check_mem_region_access(env, regno, reg->off,
4782	4836	access_size, reg->mem_size,
4783	4837	zero_size_allowed);
4784		− case PTR_TO_RDONLY_BUF:
4785		− if (meta && meta->raw_mode)
4786		− return -EACCES;
4787		− return check_buffer_access(env, reg, regno, reg->off,
4788		− access_size, zero_size_allowed,
4789		− "rdonly",
4790		− &env->prog->aux->max_rdonly_access);
4791		− case PTR_TO_RDWR_BUF:
	4838	+ case PTR_TO_BUF:
	4839	+ if (type_is_rdonly_mem(reg->type)) {
	4840	+ if (meta && meta->raw_mode)
	4841	+ return -EACCES;
	4842	+
	4843	+ buf_info = "rdonly";
	4844	+ max_access = &env->prog->aux->max_rdonly_access;
	4845	+ } else {
	4846	+ buf_info = "rdwr";
	4847	+ max_access = &env->prog->aux->max_rdwr_access;
	4848	+ }
4792	4849	return check_buffer_access(env, reg, regno, reg->off,
4793	4850	access_size, zero_size_allowed,
4794		− "rdwr",
4795		− &env->prog->aux->max_rdwr_access);
	4851	+ buf_info, max_access);
4796	4852	case PTR_TO_STACK:
4797	4853	return check_stack_range_initialized(
4798	4854	env,
@@ -4804,9 +4860,9 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, · 5 unmodified lines
4804	4860	register_is_null(reg))
4805	4861	return 0;
4806	4862
4807		− verbose(env, "R%d type=%s expected=%s\n", regno,
4808		− reg_type_str[reg->type],
4809		− reg_type_str[PTR_TO_STACK]);
	4863	+ verbose(env, "R%d type=%s ", regno,
	4864	+ reg_type_str(env, reg->type));
	4865	+ verbose(env, "expected=%s\n", reg_type_str(env, PTR_TO_STACK));
4810	4866	return -EACCES;
4811	4867	}
4812	4868	}
@@ -4817,7 +4873,7 @@ int check_mem_reg(struct bpf_verifier_env env, struct bpf_reg_state reg, · 4 unmodified lines
4817	4873	if (register_is_null(reg))
4818	4874	return 0;
4819	4875
4820		− if (reg_type_may_be_null(reg->type)) {
	4876	+ if (type_may_be_null(reg->type)) {
4821	4877	/* Assuming that the register contains a value check if the memory
4822	4878	* access is safe. Temporarily save and restore the register's state as
4823	4879	* the conversion shouldn't be visible to a caller.
@@ -4965,9 +5021,8 @@ static int process_timer_func(struct bpf_verifier_env *env, int regno, · 141 unmodified lines
4965	5021
4966	5022	static bool arg_type_is_mem_ptr(enum bpf_arg_type type)
4967	5023	{
4968		− return type == ARG_PTR_TO_MEM \|\|
4969		− type == ARG_PTR_TO_MEM_OR_NULL \|\|
4970		− type == ARG_PTR_TO_UNINIT_MEM;
	5024	+ return base_type(type) == ARG_PTR_TO_MEM \|\|
	5025	+ base_type(type) == ARG_PTR_TO_UNINIT_MEM;
4971	5026	}
4972	5027
4973	5028	static bool arg_type_is_mem_size(enum bpf_arg_type type)
@@ -5072,8 +5127,7 @@ static const struct bpf_reg_types mem_types = { · 98 unmodified lines
5072	5127	PTR_TO_MAP_KEY,
5073	5128	PTR_TO_MAP_VALUE,
5074	5129	PTR_TO_MEM,
5075		− PTR_TO_RDONLY_BUF,
5076		− PTR_TO_RDWR_BUF,
	5130	+ PTR_TO_BUF,
5077	5131	},
5078	5132	};
5079	5133
@@ -5104,31 +5158,26 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { · 24 unmodified lines
5104	5158	[ARG_PTR_TO_MAP_KEY] = &map_key_value_types,
5105	5159	[ARG_PTR_TO_MAP_VALUE] = &map_key_value_types,
5106	5160	[ARG_PTR_TO_UNINIT_MAP_VALUE] = &map_key_value_types,
5107		− [ARG_PTR_TO_MAP_VALUE_OR_NULL] = &map_key_value_types,
5108	5161	[ARG_CONST_SIZE] = &scalar_types,
5109	5162	[ARG_CONST_SIZE_OR_ZERO] = &scalar_types,
5110	5163	[ARG_CONST_ALLOC_SIZE_OR_ZERO] = &scalar_types,
5111	5164	[ARG_CONST_MAP_PTR] = &const_map_ptr_types,
5112	5165	[ARG_PTR_TO_CTX] = &context_types,
5113		− [ARG_PTR_TO_CTX_OR_NULL] = &context_types,
5114	5166	[ARG_PTR_TO_SOCK_COMMON] = &sock_types,
5115	5167	#ifdef CONFIG_NET
5116	5168	[ARG_PTR_TO_BTF_ID_SOCK_COMMON] = &btf_id_sock_common_types,
5117	5169	#endif
5118	5170	[ARG_PTR_TO_SOCKET] = &fullsock_types,
5119		− [ARG_PTR_TO_SOCKET_OR_NULL] = &fullsock_types,
5120	5171	[ARG_PTR_TO_BTF_ID] = &btf_ptr_types,
5121	5172	[ARG_PTR_TO_SPIN_LOCK] = &spin_lock_types,
5122	5173	[ARG_PTR_TO_MEM] = &mem_types,
5123		− [ARG_PTR_TO_MEM_OR_NULL] = &mem_types,
5124	5174	[ARG_PTR_TO_UNINIT_MEM] = &mem_types,
5125	5175	[ARG_PTR_TO_ALLOC_MEM] = &alloc_mem_types,
5126		− [ARG_PTR_TO_ALLOC_MEM_OR_NULL] = &alloc_mem_types,
5127	5176	[ARG_PTR_TO_INT] = &int_ptr_types,
5128	5177	[ARG_PTR_TO_LONG] = &int_ptr_types,
5129	5178	[ARG_PTR_TO_PERCPU_BTF_ID] = &percpu_btf_ptr_types,
5130	5179	[ARG_PTR_TO_FUNC] = &func_ptr_types,
5131		− [ARG_PTR_TO_STACK_OR_NULL] = &stack_ptr_types,
	5180	+ [ARG_PTR_TO_STACK] = &stack_ptr_types,
5132	5181	[ARG_PTR_TO_CONST_STR] = &const_str_ptr_types,
5133	5182	[ARG_PTR_TO_TIMER] = &timer_types,
5134	5183	};
@@ -5142,12 +5191,27 @@ static int check_reg_type(struct bpf_verifier_env *env, u32 regno, · 7 unmodified lines
5142	5191	const struct bpf_reg_types *compatible;
5143	5192	int i, j;
5144	5193
5145		− compatible = compatible_reg_types[arg_type];
	5194	+ compatible = compatible_reg_types[base_type(arg_type)];
5146	5195	if (!compatible) {
5147	5196	verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type);
5148	5197	return -EFAULT;
5149	5198	}
5150	5199
	5200	+ /* ARG_PTR_TO_MEM + RDONLY is compatible with PTR_TO_MEM and PTR_TO_MEM + RDONLY,
	5201	+ * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM and NOT with PTR_TO_MEM + RDONLY
	5202	+ *
	5203	+ * Same for MAYBE_NULL:
	5204	+ *
	5205	+ * ARG_PTR_TO_MEM + MAYBE_NULL is compatible with PTR_TO_MEM and PTR_TO_MEM + MAYBE_NULL,
	5206	+ * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM but NOT with PTR_TO_MEM + MAYBE_NULL
	5207	+ *
	5208	+ * Therefore we fold these flags depending on the arg_type before comparison.
	5209	+ */
	5210	+ if (arg_type & MEM_RDONLY)
	5211	+ type &= ~MEM_RDONLY;
	5212	+ if (arg_type & PTR_MAYBE_NULL)
	5213	+ type &= ~PTR_MAYBE_NULL;
	5214	+
5151	5215	for (i = 0; i < ARRAY_SIZE(compatible->types); i++) {
5152	5216	expected = compatible->types[i];
5153	5217	if (expected == NOT_INIT)
@@ -5157,14 +5221,14 @@ static int check_reg_type(struct bpf_verifier_env *env, u32 regno, · 3 unmodified lines
5157	5221	goto found;
5158	5222	}
5159	5223
5160		− verbose(env, "R%d type=%s expected=", regno, reg_type_str[type]);
	5224	+ verbose(env, "R%d type=%s expected=", regno, reg_type_str(env, reg->type));
5161	5225	for (j = 0; j + 1 < i; j++)
5162		− verbose(env, "%s, ", reg_type_str[compatible->types[j]]);
5163		− verbose(env, "%s\n", reg_type_str[compatible->types[j]]);
	5226	+ verbose(env, "%s, ", reg_type_str(env, compatible->types[j]));
	5227	+ verbose(env, "%s\n", reg_type_str(env, compatible->types[j]));
5164	5228	return -EACCES;
5165	5229
5166	5230	found:
5167		− if (type == PTR_TO_BTF_ID) {
	5231	+ if (reg->type == PTR_TO_BTF_ID) {
5168	5232	if (!arg_btf_id) {
5169	5233	if (!compatible->btf_id) {
5170	5234	verbose(env, "verifier internal error: missing arg compatible BTF ID\n");
@@ -5223,15 +5287,14 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, · 52 unmodified lines
5223	5287	return -EACCES;
5224	5288	}
5225	5289
5226		− if (arg_type == ARG_PTR_TO_MAP_VALUE \|\|
5227		− arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE \|\|
5228		− arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL) {
	5290	+ if (base_type(arg_type) == ARG_PTR_TO_MAP_VALUE \|\|
	5291	+ base_type(arg_type) == ARG_PTR_TO_UNINIT_MAP_VALUE) {
5229	5292	err = resolve_map_arg_type(env, meta, &arg_type);
5230	5293	if (err)
5231	5294	return err;
5232	5295	}
5233	5296
5234		− if (register_is_null(reg) && arg_type_may_be_null(arg_type))
	5297	+ if (register_is_null(reg) && type_may_be_null(arg_type))
5235	5298	/* A NULL register has a SCALAR_VALUE type, so skip
5236	5299	* type checking.
5237	5300	*/
@@ -5300,10 +5363,11 @@ skip_type_check: · 62 unmodified lines
5300	5363	err = check_helper_mem_access(env, regno,
5301	5364	meta->map_ptr->key_size, false,
5302	5365	NULL);
5303		− } else if (arg_type == ARG_PTR_TO_MAP_VALUE \|\|
5304		− (arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL &&
5305		− !register_is_null(reg)) \|\|
5306		− arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE) {
	5366	+ } else if (base_type(arg_type) == ARG_PTR_TO_MAP_VALUE \|\|
	5367	+ base_type(arg_type) == ARG_PTR_TO_UNINIT_MAP_VALUE) {
	5368	+ if (type_may_be_null(arg_type) && register_is_null(reg))
	5369	+ return 0;
	5370	+
5307	5371	/* bpf_map_xxx(..., map_ptr, ..., value) call:
5308	5372	* check [value, value + map->value_size) validity
5309	5373	*/
@@ -6025,9 +6089,9 @@ static int __check_func_call(struct bpf_verifier_env env, struct bpf_insn insn · 715 unmodified lines
6025	6089
6026	6090	if (env->log.level & BPF_LOG_LEVEL) {
6027	6091	verbose(env, "caller:\n");
6028		− print_verifier_state(env, caller);
	6092	+ print_verifier_state(env, caller, true);
6029	6093	verbose(env, "callee:\n");
6030		− print_verifier_state(env, callee);
	6094	+ print_verifier_state(env, callee, true);
6031	6095	}
6032	6096	return 0;
6033	6097	}
@@ -6242,9 +6306,9 @@ static int prepare_func_exit(struct bpf_verifier_env env, int insn_idx) · 208 unmodified lines
6242	6306	*insn_idx = callee->callsite + 1;
6243	6307	if (env->log.level & BPF_LOG_LEVEL) {
6244	6308	verbose(env, "returning from callee:\n");
6245		− print_verifier_state(env, callee);
	6309	+ print_verifier_state(env, callee, true);
6246	6310	verbose(env, "to caller at %d:\n", *insn_idx);
6247		− print_verifier_state(env, caller);
	6311	+ print_verifier_state(env, caller, true);
6248	6312	}
6249	6313	/* clear everything in the callee */
6250	6314	free_func_state(callee);
@@ -6410,13 +6474,11 @@ static int check_bpf_snprintf_call(struct bpf_verifier_env *env, · 159 unmodified lines
6410	6474
6411	6475	static int check_get_func_ip(struct bpf_verifier_env *env)
6412	6476	{
6413		− enum bpf_attach_type eatype = env->prog->expected_attach_type;
6414	6477	enum bpf_prog_type type = resolve_prog_type(env->prog);
6415	6478	int func_id = BPF_FUNC_get_func_ip;
6416	6479
6417	6480	if (type == BPF_PROG_TYPE_TRACING) {
6418		− if (eatype != BPF_TRACE_FENTRY && eatype != BPF_TRACE_FEXIT &&
6419		− eatype != BPF_MODIFY_RETURN) {
	6481	+ if (!bpf_prog_has_trampoline(env->prog)) {
6420	6482	verbose(env, "func %s#%d supported only for fentry/fexit/fmod_ret programs\n",
6421	6483	func_id_name(func_id), func_id);
6422	6484	return -ENOTSUPP;
@@ -6435,6 +6497,8 @@ static int check_helper_call(struct bpf_verifier_env env, struct bpf_insn insn · 12 unmodified lines
6435	6497	int *insn_idx_p)
6436	6498	{
6437	6499	const struct bpf_func_proto *fn = NULL;
	6500	+ enum bpf_return_type ret_type;
	6501	+ enum bpf_type_flag ret_flag;
6438	6502	struct bpf_reg_state *regs;
6439	6503	struct bpf_call_arg_meta meta;
6440	6504	int insn_idx = *insn_idx_p;
@@ -6574,13 +6638,14 @@ static int check_helper_call(struct bpf_verifier_env env, struct bpf_insn insn · 133 unmodified lines
6574	6638	regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;
6575	6639
6576	6640	/* update return register (already marked as written above) */
6577		− if (fn->ret_type == RET_INTEGER) {
	6641	+ ret_type = fn->ret_type;
	6642	+ ret_flag = type_flag(fn->ret_type);
	6643	+ if (ret_type == RET_INTEGER) {
6578	6644	/* sets type to SCALAR_VALUE */
6579	6645	mark_reg_unknown(env, regs, BPF_REG_0);
6580		− } else if (fn->ret_type == RET_VOID) {
	6646	+ } else if (ret_type == RET_VOID) {
6581	6647	regs[BPF_REG_0].type = NOT_INIT;
6582		− } else if (fn->ret_type == RET_PTR_TO_MAP_VALUE_OR_NULL \|\|
6583		− fn->ret_type == RET_PTR_TO_MAP_VALUE) {
	6648	+ } else if (base_type(ret_type) == RET_PTR_TO_MAP_VALUE) {
6584	6649	/* There is no offset yet applied, variable or fixed */
6585	6650	mark_reg_known_zero(env, regs, BPF_REG_0);
6586	6651	/* remember map_ptr, so that check_map_access()
@@ -6594,28 +6659,25 @@ static int check_helper_call(struct bpf_verifier_env env, struct bpf_insn insn · 7 unmodified lines
6594	6659	}
6595	6660	regs[BPF_REG_0].map_ptr = meta.map_ptr;
6596	6661	regs[BPF_REG_0].map_uid = meta.map_uid;
6597		− if (fn->ret_type == RET_PTR_TO_MAP_VALUE) {
6598		− regs[BPF_REG_0].type = PTR_TO_MAP_VALUE;
6599		− if (map_value_has_spin_lock(meta.map_ptr))
6600		− regs[BPF_REG_0].id = ++env->id_gen;
6601		− } else {
6602		− regs[BPF_REG_0].type = PTR_TO_MAP_VALUE_OR_NULL;
	6662	+ regs[BPF_REG_0].type = PTR_TO_MAP_VALUE \| ret_flag;
	6663	+ if (!type_may_be_null(ret_type) &&
	6664	+ map_value_has_spin_lock(meta.map_ptr)) {
	6665	+ regs[BPF_REG_0].id = ++env->id_gen;
6603	6666	}
6604		− } else if (fn->ret_type == RET_PTR_TO_SOCKET_OR_NULL) {
	6667	+ } else if (base_type(ret_type) == RET_PTR_TO_SOCKET) {
6605	6668	mark_reg_known_zero(env, regs, BPF_REG_0);
6606		− regs[BPF_REG_0].type = PTR_TO_SOCKET_OR_NULL;
6607		− } else if (fn->ret_type == RET_PTR_TO_SOCK_COMMON_OR_NULL) {
	6669	+ regs[BPF_REG_0].type = PTR_TO_SOCKET \| ret_flag;
	6670	+ } else if (base_type(ret_type) == RET_PTR_TO_SOCK_COMMON) {
6608	6671	mark_reg_known_zero(env, regs, BPF_REG_0);
6609		− regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON_OR_NULL;
6610		− } else if (fn->ret_type == RET_PTR_TO_TCP_SOCK_OR_NULL) {
	6672	+ regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON \| ret_flag;
	6673	+ } else if (base_type(ret_type) == RET_PTR_TO_TCP_SOCK) {
6611	6674	mark_reg_known_zero(env, regs, BPF_REG_0);
6612		− regs[BPF_REG_0].type = PTR_TO_TCP_SOCK_OR_NULL;
6613		− } else if (fn->ret_type == RET_PTR_TO_ALLOC_MEM_OR_NULL) {
	6675	+ regs[BPF_REG_0].type = PTR_TO_TCP_SOCK \| ret_flag;
	6676	+ } else if (base_type(ret_type) == RET_PTR_TO_ALLOC_MEM) {
6614	6677	mark_reg_known_zero(env, regs, BPF_REG_0);
6615		− regs[BPF_REG_0].type = PTR_TO_MEM_OR_NULL;
	6678	+ regs[BPF_REG_0].type = PTR_TO_MEM \| ret_flag;
6616	6679	regs[BPF_REG_0].mem_size = meta.mem_size;
6617		− } else if (fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL \|\|
6618		− fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID) {
	6680	+ } else if (base_type(ret_type) == RET_PTR_TO_MEM_OR_BTF_ID) {
6619	6681	const struct btf_type *t;
6620	6682
6621	6683	mark_reg_known_zero(env, regs, BPF_REG_0);
@@ -6633,29 +6695,30 @@ static int check_helper_call(struct bpf_verifier_env env, struct bpf_insn insn · 11 unmodified lines
6633	6695	tname, PTR_ERR(ret));
6634	6696	return -EINVAL;
6635	6697	}
6636		− regs[BPF_REG_0].type =
6637		− fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID ?
6638		− PTR_TO_MEM : PTR_TO_MEM_OR_NULL;
	6698	+ regs[BPF_REG_0].type = PTR_TO_MEM \| ret_flag;
6639	6699	regs[BPF_REG_0].mem_size = tsize;
6640	6700	} else {
6641		− regs[BPF_REG_0].type =
6642		− fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID ?
6643		− PTR_TO_BTF_ID : PTR_TO_BTF_ID_OR_NULL;
	6701	+ /* MEM_RDONLY may be carried from ret_flag, but it
	6702	+ * doesn't apply on PTR_TO_BTF_ID. Fold it, otherwise
	6703	+ * it will confuse the check of PTR_TO_BTF_ID in
	6704	+ * check_mem_access().
	6705	+ */
	6706	+ ret_flag &= ~MEM_RDONLY;
	6707	+
	6708	+ regs[BPF_REG_0].type = PTR_TO_BTF_ID \| ret_flag;
6644	6709	regs[BPF_REG_0].btf = meta.ret_btf;
6645	6710	regs[BPF_REG_0].btf_id = meta.ret_btf_id;
6646	6711	}
6647		− } else if (fn->ret_type == RET_PTR_TO_BTF_ID_OR_NULL \|\|
6648		− fn->ret_type == RET_PTR_TO_BTF_ID) {
	6712	+ } else if (base_type(ret_type) == RET_PTR_TO_BTF_ID) {
6649	6713	int ret_btf_id;
6650	6714
6651	6715	mark_reg_known_zero(env, regs, BPF_REG_0);
6652		− regs[BPF_REG_0].type = fn->ret_type == RET_PTR_TO_BTF_ID ?
6653		− PTR_TO_BTF_ID :
6654		− PTR_TO_BTF_ID_OR_NULL;
	6716	+ regs[BPF_REG_0].type = PTR_TO_BTF_ID \| ret_flag;
6655	6717	ret_btf_id = *fn->ret_btf_id;
6656	6718	if (ret_btf_id == 0) {
6657		− verbose(env, "invalid return type %d of func %s#%d\n",
6658		− fn->ret_type, func_id_name(func_id), func_id);
	6719	+ verbose(env, "invalid return type %u of func %s#%d\n",
	6720	+ base_type(ret_type), func_id_name(func_id),
	6721	+ func_id);
6659	6722	return -EINVAL;
6660	6723	}
6661	6724	/* current BPF helper definitions are only coming from
@@ -6664,12 +6727,12 @@ static int check_helper_call(struct bpf_verifier_env env, struct bpf_insn insn · 2 unmodified lines
6664	6727	regs[BPF_REG_0].btf = btf_vmlinux;
6665	6728	regs[BPF_REG_0].btf_id = ret_btf_id;
6666	6729	} else {
6667		− verbose(env, "unknown return type %d of func %s#%d\n",
6668		− fn->ret_type, func_id_name(func_id), func_id);
	6730	+ verbose(env, "unknown return type %u of func %s#%d\n",
	6731	+ base_type(ret_type), func_id_name(func_id), func_id);
6669	6732	return -EINVAL;
6670	6733	}
6671	6734
6672		− if (reg_type_may_be_null(regs[BPF_REG_0].type))
	6735	+ if (type_may_be_null(regs[BPF_REG_0].type))
6673	6736	regs[BPF_REG_0].id = ++env->id_gen;
6674	6737
6675	6738	if (is_ptr_cast_function(func_id)) {
@@ -6878,25 +6941,25 @@ static bool check_reg_sane_offset(struct bpf_verifier_env *env, · 202 unmodified lines
6878	6941
6879	6942	if (known && (val >= BPF_MAX_VAR_OFF \|\| val <= -BPF_MAX_VAR_OFF)) {
6880	6943	verbose(env, "math between %s pointer and %lld is not allowed\n",
6881		− reg_type_str[type], val);
	6944	+ reg_type_str(env, type), val);
6882	6945	return false;
6883	6946	}
6884	6947
6885	6948	if (reg->off >= BPF_MAX_VAR_OFF \|\| reg->off <= -BPF_MAX_VAR_OFF) {
6886	6949	verbose(env, "%s pointer offset %d is not allowed\n",
6887		− reg_type_str[type], reg->off);
	6950	+ reg_type_str(env, type), reg->off);
6888	6951	return false;
6889	6952	}
6890	6953
6891	6954	if (smin == S64_MIN) {
6892	6955	verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n",
6893		− reg_type_str[type]);
	6956	+ reg_type_str(env, type));
6894	6957	return false;
6895	6958	}
6896	6959
6897	6960	if (smin >= BPF_MAX_VAR_OFF \|\| smin <= -BPF_MAX_VAR_OFF) {
6898	6961	verbose(env, "value %lld makes %s pointer be out of bounds\n",
6899		− smin, reg_type_str[type]);
	6962	+ smin, reg_type_str(env, type));
6900	6963	return false;
6901	6964	}
6902	6965
@@ -7273,11 +7336,13 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, · 370 unmodified lines
7273	7336	return -EACCES;
7274	7337	}
7275	7338
7276		− switch (ptr_reg->type) {
7277		− case PTR_TO_MAP_VALUE_OR_NULL:
	7339	+ if (ptr_reg->type & PTR_MAYBE_NULL) {
7278	7340	verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n",
7279		− dst, reg_type_str[ptr_reg->type]);
	7341	+ dst, reg_type_str(env, ptr_reg->type));
7280	7342	return -EACCES;
	7343	+ }
	7344	+
	7345	+ switch (base_type(ptr_reg->type)) {
7281	7346	case CONST_PTR_TO_MAP:
7282	7347	/* smin_val represents the known value */
7283	7348	if (known && smin_val == 0 && opcode == BPF_ADD)
@@ -7285,14 +7350,11 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, · 1 unmodified line
7285	7350	fallthrough;
7286	7351	case PTR_TO_PACKET_END:
7287	7352	case PTR_TO_SOCKET:
7288		− case PTR_TO_SOCKET_OR_NULL:
7289	7353	case PTR_TO_SOCK_COMMON:
7290		− case PTR_TO_SOCK_COMMON_OR_NULL:
7291	7354	case PTR_TO_TCP_SOCK:
7292		− case PTR_TO_TCP_SOCK_OR_NULL:
7293	7355	case PTR_TO_XDP_SOCK:
7294	7356	verbose(env, "R%d pointer arithmetic on %s prohibited\n",
7295		− dst, reg_type_str[ptr_reg->type]);
	7357	+ dst, reg_type_str(env, ptr_reg->type));
7296	7358	return -EACCES;
7297	7359	default:
7298	7360	break;
@@ -8265,12 +8327,12 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, · 966 unmodified lines
8265	8327
8266	8328	/* Got here implies adding two SCALAR_VALUEs */
8267	8329	if (WARN_ON_ONCE(ptr_reg)) {
8268		− print_verifier_state(env, state);
	8330	+ print_verifier_state(env, state, true);
8269	8331	verbose(env, "verifier internal error: unexpected ptr_reg\n");
8270	8332	return -EINVAL;
8271	8333	}
8272	8334	if (WARN_ON(!src_reg)) {
8273		− print_verifier_state(env, state);
	8335	+ print_verifier_state(env, state, true);
8274	8336	verbose(env, "verifier internal error: no src_reg\n");
8275	8337	return -EINVAL;
8276	8338	}
@@ -9015,7 +9077,7 @@ static void mark_ptr_or_null_reg(struct bpf_func_state *state, · 738 unmodified lines
9015	9077	struct bpf_reg_state *reg, u32 id,
9016	9078	bool is_null)
9017	9079	{
9018		− if (reg_type_may_be_null(reg->type) && reg->id == id &&
	9080	+ if (type_may_be_null(reg->type) && reg->id == id &&
9019	9081	!WARN_ON_ONCE(!reg->id)) {
9020	9082	/* Old offset (both fixed and variable parts) should
9021	9083	* have been known-zero, because we don't allow pointer
@@ -9393,7 +9455,7 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, · 371 unmodified lines
9393	9455	*/
9394	9456	if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K &&
9395	9457	insn->imm == 0 && (opcode == BPF_JEQ \|\| opcode == BPF_JNE) &&
9396		− reg_type_may_be_null(dst_reg->type)) {
	9458	+ type_may_be_null(dst_reg->type)) {
9397	9459	/* Mark all identical registers in each branch as either
9398	9460	* safe or unknown depending R == 0 or R != 0 conditional.
9399	9461	*/
@@ -9409,7 +9471,7 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, · 9 unmodified lines
9409	9471	return -EACCES;
9410	9472	}
9411	9473	if (env->log.level & BPF_LOG_LEVEL)
9412		− print_verifier_state(env, this_branch->frame[this_branch->curframe]);
	9474	+ print_insn_state(env, this_branch->frame[this_branch->curframe]);
9413	9475	return 0;
9414	9476	}
9415	9477
@@ -9448,7 +9510,7 @@ static int check_ld_imm(struct bpf_verifier_env env, struct bpf_insn insn) · 32 unmodified lines
9448	9510	mark_reg_known_zero(env, regs, insn->dst_reg);
9449	9511
9450	9512	dst_reg->type = aux->btf_var.reg_type;
9451		− switch (dst_reg->type) {
	9513	+ switch (base_type(dst_reg->type)) {
9452	9514	case PTR_TO_MEM:
9453	9515	dst_reg->mem_size = aux->btf_var.mem_size;
9454	9516	break;
@@ -9647,7 +9709,7 @@ static int check_return_code(struct bpf_verifier_env *env) · 192 unmodified lines
9647	9709	/* enforce return zero from async callbacks like timer */
9648	9710	if (reg->type != SCALAR_VALUE) {
9649	9711	verbose(env, "In async callback the register R0 is not a known value (%s)\n",
9650		− reg_type_str[reg->type]);
	9712	+ reg_type_str(env, reg->type));
9651	9713	return -EINVAL;
9652	9714	}
9653	9715
@@ -9661,7 +9723,7 @@ static int check_return_code(struct bpf_verifier_env *env) · 7 unmodified lines
9661	9723	if (is_subprog) {
9662	9724	if (reg->type != SCALAR_VALUE) {
9663	9725	verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n",
9664		− reg_type_str[reg->type]);
	9726	+ reg_type_str(env, reg->type));
9665	9727	return -EINVAL;
9666	9728	}
9667	9729	return 0;
@@ -9725,7 +9787,7 @@ static int check_return_code(struct bpf_verifier_env *env) · 57 unmodified lines
9725	9787
9726	9788	if (reg->type != SCALAR_VALUE) {
9727	9789	verbose(env, "At program exit the register R0 is not a known value (%s)\n",
9728		− reg_type_str[reg->type]);
	9790	+ reg_type_str(env, reg->type));
9729	9791	return -EINVAL;
9730	9792	}
9731	9793
@@ -10582,7 +10644,7 @@ static bool regsafe(struct bpf_verifier_env env, struct bpf_reg_state rold, · 850 unmodified lines
10582	10644	return true;
10583	10645	if (rcur->type == NOT_INIT)
10584	10646	return false;
10585		− switch (rold->type) {
	10647	+ switch (base_type(rold->type)) {
10586	10648	case SCALAR_VALUE:
10587	10649	if (env->explore_alu_limits)
10588	10650	return false;
@@ -10604,6 +10666,22 @@ static bool regsafe(struct bpf_verifier_env env, struct bpf_reg_state rold, · 15 unmodified lines
10604	10666	}
10605	10667	case PTR_TO_MAP_KEY:
10606	10668	case PTR_TO_MAP_VALUE:
	10669	+ /* a PTR_TO_MAP_VALUE could be safe to use as a
	10670	+ * PTR_TO_MAP_VALUE_OR_NULL into the same map.
	10671	+ * However, if the old PTR_TO_MAP_VALUE_OR_NULL then got NULL-
	10672	+ * checked, doing so could have affected others with the same
	10673	+ * id, and we can't check for that because we lost the id when
	10674	+ * we converted to a PTR_TO_MAP_VALUE.
	10675	+ */
	10676	+ if (type_may_be_null(rold->type)) {
	10677	+ if (!type_may_be_null(rcur->type))
	10678	+ return false;
	10679	+ if (memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)))
	10680	+ return false;
	10681	+ /* Check our ids match any regs they're supposed to */
	10682	+ return check_ids(rold->id, rcur->id, idmap);
	10683	+ }
	10684	+
10607	10685	/* If the new min/max/var_off satisfy the old ones and
10608	10686	* everything else matches, we are OK.
10609	10687	* 'id' is not compared, since it's only used for maps with
@@ -10615,20 +10693,6 @@ static bool regsafe(struct bpf_verifier_env env, struct bpf_reg_state rold, · 5 unmodified lines
10615	10693	return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 &&
10616	10694	range_within(rold, rcur) &&
10617	10695	tnum_in(rold->var_off, rcur->var_off);
10618		− case PTR_TO_MAP_VALUE_OR_NULL:
10619		− /* a PTR_TO_MAP_VALUE could be safe to use as a
10620		− * PTR_TO_MAP_VALUE_OR_NULL into the same map.
10621		− * However, if the old PTR_TO_MAP_VALUE_OR_NULL then got NULL-
10622		− * checked, doing so could have affected others with the same
10623		− * id, and we can't check for that because we lost the id when
10624		− * we converted to a PTR_TO_MAP_VALUE.
10625		− */
10626		− if (rcur->type != PTR_TO_MAP_VALUE_OR_NULL)
10627		− return false;
10628		− if (memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)))
10629		− return false;
10630		− /* Check our ids match any regs they're supposed to */
10631		− return check_ids(rold->id, rcur->id, idmap);
10632	10696	case PTR_TO_PACKET_META:
10633	10697	case PTR_TO_PACKET:
10634	10698	if (rcur->type != rold->type)
@@ -10657,11 +10721,8 @@ static bool regsafe(struct bpf_verifier_env env, struct bpf_reg_state rold, · 22 unmodified lines
10657	10721	case PTR_TO_PACKET_END:
10658	10722	case PTR_TO_FLOW_KEYS:
10659	10723	case PTR_TO_SOCKET:
10660		− case PTR_TO_SOCKET_OR_NULL:
10661	10724	case PTR_TO_SOCK_COMMON:
10662		− case PTR_TO_SOCK_COMMON_OR_NULL:
10663	10725	case PTR_TO_TCP_SOCK:
10664		− case PTR_TO_TCP_SOCK_OR_NULL:
10665	10726	case PTR_TO_XDP_SOCK:
10666	10727	/* Only valid matches are exact, which memcmp() above
10667	10728	* would have accepted
@@ -11187,17 +11248,13 @@ next: · 519 unmodified lines
11187	11248	/* Return true if it's OK to have the same insn return a different type. */
11188	11249	static bool reg_type_mismatch_ok(enum bpf_reg_type type)
11189	11250	{
11190		− switch (type) {
	11251	+ switch (base_type(type)) {
11191	11252	case PTR_TO_CTX:
11192	11253	case PTR_TO_SOCKET:
11193		− case PTR_TO_SOCKET_OR_NULL:
11194	11254	case PTR_TO_SOCK_COMMON:
11195		− case PTR_TO_SOCK_COMMON_OR_NULL:
11196	11255	case PTR_TO_TCP_SOCK:
11197		− case PTR_TO_TCP_SOCK_OR_NULL:
11198	11256	case PTR_TO_XDP_SOCK:
11199	11257	case PTR_TO_BTF_ID:
11200		− case PTR_TO_BTF_ID_OR_NULL:
11201	11258	return false;
11202	11259	default:
11203	11260	return true;
@@ -11277,16 +11334,12 @@ static int do_check(struct bpf_verifier_env *env) · 73 unmodified lines
11277	11334	if (need_resched())
11278	11335	cond_resched();
11279	11336
11280		− if (env->log.level & BPF_LOG_LEVEL2 \|\|
11281		− (env->log.level & BPF_LOG_LEVEL && do_print_state)) {
11282		− if (env->log.level & BPF_LOG_LEVEL2)
11283		− verbose(env, "%d:", env->insn_idx);
11284		− else
11285		− verbose(env, "\nfrom %d to %d%s:",
11286		− env->prev_insn_idx, env->insn_idx,
11287		− env->cur_state->speculative ?
11288		− " (speculative execution)" : "");
11289		− print_verifier_state(env, state->frame[state->curframe]);
	11337	+ if (env->log.level & BPF_LOG_LEVEL2 && do_print_state) {
	11338	+ verbose(env, "\nfrom %d to %d%s:",
	11339	+ env->prev_insn_idx, env->insn_idx,
	11340	+ env->cur_state->speculative ?
	11341	+ " (speculative execution)" : "");
	11342	+ print_verifier_state(env, state->frame[state->curframe], true);
11290	11343	do_print_state = false;
11291	11344	}
11292	11345
@@ -11297,9 +11350,15 @@ static int do_check(struct bpf_verifier_env *env) · 4 unmodified lines
11297	11350	.private_data = env,
11298	11351	};
11299	11352
	11353	+ if (verifier_state_scratched(env))
	11354	+ print_insn_state(env, state->frame[state->curframe]);
	11355	+
11300	11356	verbose_linfo(env, env->insn_idx, "; ");
	11357	+ env->prev_log_len = env->log.len_used;
11301	11358	verbose(env, "%d: ", env->insn_idx);
11302	11359	print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
	11360	+ env->prev_insn_print_len = env->log.len_used - env->prev_log_len;
	11361	+ env->prev_log_len = env->log.len_used;
11303	11362	}
11304	11363
11305	11364	if (bpf_prog_is_dev_bound(env->prog->aux)) {
@@ -11421,7 +11480,7 @@ static int do_check(struct bpf_verifier_env *env) · 115 unmodified lines
11421	11480	if (is_ctx_reg(env, insn->dst_reg)) {
11422	11481	verbose(env, "BPF_ST stores into R%d %s is not allowed\n",
11423	11482	insn->dst_reg,
11424		− reg_type_str[reg_state(env, insn->dst_reg)->type]);
	11483	+ reg_type_str(env, reg_state(env, insn->dst_reg)->type));
11425	11484	return -EACCES;
11426	11485	}
11427	11486
@@ -11508,6 +11567,7 @@ static int do_check(struct bpf_verifier_env *env) · 80 unmodified lines
11508	11567	if (err)
11509	11568	return err;
11510	11569	process_bpf_exit:
	11570	+ mark_verifier_state_scratched(env);
11511	11571	update_branch_counts(env, env->cur_state);
11512	11572	err = pop_stack(env, &prev_insn_idx,
11513	11573	&env->insn_idx, pop_log);
@@ -11673,7 +11733,7 @@ static int check_pseudo_btf_id(struct bpf_verifier_env *env, · 159 unmodified lines
11673	11733	err = -EINVAL;
11674	11734	goto err_put;
11675	11735	}
11676		− aux->btf_var.reg_type = PTR_TO_MEM;
	11736	+ aux->btf_var.reg_type = PTR_TO_MEM \| MEM_RDONLY;
11677	11737	aux->btf_var.mem_size = tsize;
11678	11738	} else {
11679	11739	aux->btf_var.reg_type = PTR_TO_BTF_ID;
@@ -11833,6 +11893,9 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, · 153 unmodified lines
11833	11893	}
11834	11894	break;
11835	11895	case BPF_MAP_TYPE_RINGBUF:
	11896	+ case BPF_MAP_TYPE_INODE_STORAGE:
	11897	+ case BPF_MAP_TYPE_SK_STORAGE:
	11898	+ case BPF_MAP_TYPE_TASK_STORAGE:
11836	11899	break;
11837	11900	default:
11838	11901	verbose(env,
@@ -13016,6 +13079,7 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, · 1177 unmodified lines
13016	13079	static int do_misc_fixups(struct bpf_verifier_env *env)
13017	13080	{
13018	13081	struct bpf_prog *prog = env->prog;
	13082	+ enum bpf_attach_type eatype = prog->expected_attach_type;
13019	13083	bool expect_blinding = bpf_jit_blinding_enabled(prog);
13020	13084	enum bpf_prog_type prog_type = resolve_prog_type(prog);
13021	13085	struct bpf_insn *insn = prog->insnsi;
@@ -13386,11 +13450,79 @@ patch_map_ops_generic: · 364 unmodified lines
13386	13450	continue;
13387	13451	}
13388	13452
	13453	+ /* Implement bpf_get_func_arg inline. */
	13454	+ if (prog_type == BPF_PROG_TYPE_TRACING &&
	13455	+ insn->imm == BPF_FUNC_get_func_arg) {
	13456	+ /* Load nr_args from ctx - 8 */
	13457	+ insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
	13458	+ insn_buf[1] = BPF_JMP32_REG(BPF_JGE, BPF_REG_2, BPF_REG_0, 6);
	13459	+ insn_buf[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 3);
	13460	+ insn_buf[3] = BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_1);
	13461	+ insn_buf[4] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0);
	13462	+ insn_buf[5] = BPF_STX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0);
	13463	+ insn_buf[6] = BPF_MOV64_IMM(BPF_REG_0, 0);
	13464	+ insn_buf[7] = BPF_JMP_A(1);
	13465	+ insn_buf[8] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL);
	13466	+ cnt = 9;
	13467	+
	13468	+ new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
	13469	+ if (!new_prog)
	13470	+ return -ENOMEM;
	13471	+
	13472	+ delta += cnt - 1;
	13473	+ env->prog = prog = new_prog;
	13474	+ insn = new_prog->insnsi + i + delta;
	13475	+ continue;
	13476	+ }
	13477	+
	13478	+ /* Implement bpf_get_func_ret inline. */
	13479	+ if (prog_type == BPF_PROG_TYPE_TRACING &&
	13480	+ insn->imm == BPF_FUNC_get_func_ret) {
	13481	+ if (eatype == BPF_TRACE_FEXIT \|\|
	13482	+ eatype == BPF_MODIFY_RETURN) {
	13483	+ /* Load nr_args from ctx - 8 */
	13484	+ insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
	13485	+ insn_buf[1] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_0, 3);
	13486	+ insn_buf[2] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1);
	13487	+ insn_buf[3] = BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0);
	13488	+ insn_buf[4] = BPF_STX_MEM(BPF_DW, BPF_REG_2, BPF_REG_3, 0);
	13489	+ insn_buf[5] = BPF_MOV64_IMM(BPF_REG_0, 0);
	13490	+ cnt = 6;
	13491	+ } else {
	13492	+ insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, -EOPNOTSUPP);
	13493	+ cnt = 1;
	13494	+ }
	13495	+
	13496	+ new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
	13497	+ if (!new_prog)
	13498	+ return -ENOMEM;
	13499	+
	13500	+ delta += cnt - 1;
	13501	+ env->prog = prog = new_prog;
	13502	+ insn = new_prog->insnsi + i + delta;
	13503	+ continue;
	13504	+ }
	13505	+
	13506	+ /* Implement get_func_arg_cnt inline. */
	13507	+ if (prog_type == BPF_PROG_TYPE_TRACING &&
	13508	+ insn->imm == BPF_FUNC_get_func_arg_cnt) {
	13509	+ /* Load nr_args from ctx - 8 */
	13510	+ insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
	13511	+
	13512	+ new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1);
	13513	+ if (!new_prog)
	13514	+ return -ENOMEM;
	13515	+
	13516	+ env->prog = prog = new_prog;
	13517	+ insn = new_prog->insnsi + i + delta;
	13518	+ continue;
	13519	+ }
	13520	+
13389	13521	/* Implement bpf_get_func_ip inline. */
13390	13522	if (prog_type == BPF_PROG_TYPE_TRACING &&
13391	13523	insn->imm == BPF_FUNC_get_func_ip) {
13392		− /* Load IP address from ctx - 8 */
13393		− insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
	13524	+ /* Load IP address from ctx - 16 */
	13525	+ insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -16);
13394	13526
13395	13527	new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1);
13396	13528	if (!new_prog)
@@ -13504,7 +13636,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) · 107 unmodified lines
13504	13636	mark_reg_known_zero(env, regs, i);
13505	13637	else if (regs[i].type == SCALAR_VALUE)
13506	13638	mark_reg_unknown(env, regs, i);
13507		− else if (regs[i].type == PTR_TO_MEM_OR_NULL) {
	13639	+ else if (base_type(regs[i].type) == PTR_TO_MEM) {
13508	13640	const u32 mem_size = regs[i].mem_size;
13509	13641
13510	13642	mark_reg_known_zero(env, regs, i);
@@ -14099,6 +14231,8 @@ int bpf_check(struct bpf_prog *prog, union bpf_attr attr, bpfptr_t uattr) · 588 unmodified lines
14099	14231	}
14100	14232	}
14101	14233
	14234	+ mark_verifier_state_clean(env);
	14235	+
14102	14236	if (IS_ERR(btf_vmlinux)) {
14103	14237	/* Either gcc or pahole or kernel are broken. */
14104	14238	verbose(env, "in-kernel BTF is malformed\n");

kernel/cgroup/cgroup.c+1−0

@@ -30,6 +30,7 @@ · 29 unmodified lines
30	30
31	31	#include "cgroup-internal.h"
32	32
	33	+#include <linux/bpf-cgroup.h>
33	34	#include <linux/cred.h>
34	35	#include <linux/errno.h>
35	36	#include <linux/init_task.h>

kernel/sysctl.c+1−0

@@ -33,6 +33,7 @@ · 32 unmodified lines
33	33	#include <linux/security.h>
34	34	#include <linux/ctype.h>
35	35	#include <linux/kmemleak.h>
	36	+#include <linux/filter.h>
36	37	#include <linux/fs.h>
37	38	#include <linux/init.h>
38	39	#include <linux/kernel.h>

kernel/trace/bpf_trace.c+67−14

@@ -345,7 +345,7 @@ static const struct bpf_func_proto bpf_probe_write_user_proto = { · 344 unmodified lines
345	345	.gpl_only = true,
346	346	.ret_type = RET_INTEGER,
347	347	.arg1_type = ARG_ANYTHING,
348		− .arg2_type = ARG_PTR_TO_MEM,
	348	+ .arg2_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
349	349	.arg3_type = ARG_CONST_SIZE,
350	350	};
351	351
@@ -394,7 +394,7 @@ static const struct bpf_func_proto bpf_trace_printk_proto = { · 42 unmodified lines
394	394	.func = bpf_trace_printk,
395	395	.gpl_only = true,
396	396	.ret_type = RET_INTEGER,
397		− .arg1_type = ARG_PTR_TO_MEM,
	397	+ .arg1_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
398	398	.arg2_type = ARG_CONST_SIZE,
399	399	};
400	400
@@ -450,9 +450,9 @@ static const struct bpf_func_proto bpf_trace_vprintk_proto = { · 49 unmodified lines
450	450	.func = bpf_trace_vprintk,
451	451	.gpl_only = true,
452	452	.ret_type = RET_INTEGER,
453		− .arg1_type = ARG_PTR_TO_MEM,
	453	+ .arg1_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
454	454	.arg2_type = ARG_CONST_SIZE,
455		− .arg3_type = ARG_PTR_TO_MEM_OR_NULL,
	455	+ .arg3_type = ARG_PTR_TO_MEM \| PTR_MAYBE_NULL \| MEM_RDONLY,
456	456	.arg4_type = ARG_CONST_SIZE_OR_ZERO,
457	457	};
458	458
@@ -492,9 +492,9 @@ static const struct bpf_func_proto bpf_seq_printf_proto = { · 33 unmodified lines
492	492	.ret_type = RET_INTEGER,
493	493	.arg1_type = ARG_PTR_TO_BTF_ID,
494	494	.arg1_btf_id = &btf_seq_file_ids[0],
495		− .arg2_type = ARG_PTR_TO_MEM,
	495	+ .arg2_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
496	496	.arg3_type = ARG_CONST_SIZE,
497		− .arg4_type = ARG_PTR_TO_MEM_OR_NULL,
	497	+ .arg4_type = ARG_PTR_TO_MEM \| PTR_MAYBE_NULL \| MEM_RDONLY,
498	498	.arg5_type = ARG_CONST_SIZE_OR_ZERO,
499	499	};
500	500
@@ -509,7 +509,7 @@ static const struct bpf_func_proto bpf_seq_write_proto = { · 8 unmodified lines
509	509	.ret_type = RET_INTEGER,
510	510	.arg1_type = ARG_PTR_TO_BTF_ID,
511	511	.arg1_btf_id = &btf_seq_file_ids[0],
512		− .arg2_type = ARG_PTR_TO_MEM,
	512	+ .arg2_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
513	513	.arg3_type = ARG_CONST_SIZE_OR_ZERO,
514	514	};
515	515
@@ -533,7 +533,7 @@ static const struct bpf_func_proto bpf_seq_printf_btf_proto = { · 17 unmodified lines
533	533	.ret_type = RET_INTEGER,
534	534	.arg1_type = ARG_PTR_TO_BTF_ID,
535	535	.arg1_btf_id = &btf_seq_file_ids[0],
536		− .arg2_type = ARG_PTR_TO_MEM,
	536	+ .arg2_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
537	537	.arg3_type = ARG_CONST_SIZE_OR_ZERO,
538	538	.arg4_type = ARG_ANYTHING,
539	539	};
@@ -694,7 +694,7 @@ static const struct bpf_func_proto bpf_perf_event_output_proto = { · 154 unmodified lines
694	694	.arg1_type = ARG_PTR_TO_CTX,
695	695	.arg2_type = ARG_CONST_MAP_PTR,
696	696	.arg3_type = ARG_ANYTHING,
697		− .arg4_type = ARG_PTR_TO_MEM,
	697	+ .arg4_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
698	698	.arg5_type = ARG_CONST_SIZE_OR_ZERO,
699	699	};
700	700
@@ -1004,7 +1004,7 @@ const struct bpf_func_proto bpf_snprintf_btf_proto = { · 303 unmodified lines
1004	1004	.ret_type = RET_INTEGER,
1005	1005	.arg1_type = ARG_PTR_TO_MEM,
1006	1006	.arg2_type = ARG_CONST_SIZE,
1007		− .arg3_type = ARG_PTR_TO_MEM,
	1007	+ .arg3_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
1008	1008	.arg4_type = ARG_CONST_SIZE,
1009	1009	.arg5_type = ARG_ANYTHING,
1010	1010	};
@@ -1012,7 +1012,7 @@ const struct bpf_func_proto bpf_snprintf_btf_proto = { · 1 unmodified line
1012	1012	BPF_CALL_1(bpf_get_func_ip_tracing, void *, ctx)
1013	1013	{
1014	1014	/* This helper call is inlined by verifier. */
1015		− return ((u64 *)ctx)[-1];
	1015	+ return ((u64 *)ctx)[-2];
1016	1016	}
1017	1017
1018	1018	static const struct bpf_func_proto bpf_get_func_ip_proto_tracing = {
@@ -1091,6 +1091,53 @@ static const struct bpf_func_proto bpf_get_branch_snapshot_proto = { · 72 unmodified lines
1091	1091	.arg2_type = ARG_CONST_SIZE_OR_ZERO,
1092	1092	};
1093	1093
	1094	+BPF_CALL_3(get_func_arg, void , ctx, u32, n, u64 , value)
	1095	+{
	1096	+ /* This helper call is inlined by verifier. */
	1097	+ u64 nr_args = ((u64 *)ctx)[-1];
	1098	+
	1099	+ if ((u64) n >= nr_args)
	1100	+ return -EINVAL;
	1101	+ value = ((u64 )ctx)[n];
	1102	+ return 0;
	1103	+}
	1104	+
	1105	+static const struct bpf_func_proto bpf_get_func_arg_proto = {
	1106	+ .func = get_func_arg,
	1107	+ .ret_type = RET_INTEGER,
	1108	+ .arg1_type = ARG_PTR_TO_CTX,
	1109	+ .arg2_type = ARG_ANYTHING,
	1110	+ .arg3_type = ARG_PTR_TO_LONG,
	1111	+};
	1112	+
	1113	+BPF_CALL_2(get_func_ret, void , ctx, u64 , value)
	1114	+{
	1115	+ /* This helper call is inlined by verifier. */
	1116	+ u64 nr_args = ((u64 *)ctx)[-1];
	1117	+
	1118	+ value = ((u64 )ctx)[nr_args];
	1119	+ return 0;
	1120	+}
	1121	+
	1122	+static const struct bpf_func_proto bpf_get_func_ret_proto = {
	1123	+ .func = get_func_ret,
	1124	+ .ret_type = RET_INTEGER,
	1125	+ .arg1_type = ARG_PTR_TO_CTX,
	1126	+ .arg2_type = ARG_PTR_TO_LONG,
	1127	+};
	1128	+
	1129	+BPF_CALL_1(get_func_arg_cnt, void *, ctx)
	1130	+{
	1131	+ /* This helper call is inlined by verifier. */
	1132	+ return ((u64 *)ctx)[-1];
	1133	+}
	1134	+
	1135	+static const struct bpf_func_proto bpf_get_func_arg_cnt_proto = {
	1136	+ .func = get_func_arg_cnt,
	1137	+ .ret_type = RET_INTEGER,
	1138	+ .arg1_type = ARG_PTR_TO_CTX,
	1139	+};
	1140	+
1094	1141	static const struct bpf_func_proto *
1095	1142	bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
1096	1143	{
@@ -1287,7 +1334,7 @@ static const struct bpf_func_proto bpf_perf_event_output_proto_tp = { · 190 unmodified lines
1287	1334	.arg1_type = ARG_PTR_TO_CTX,
1288	1335	.arg2_type = ARG_CONST_MAP_PTR,
1289	1336	.arg3_type = ARG_ANYTHING,
1290		− .arg4_type = ARG_PTR_TO_MEM,
	1337	+ .arg4_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
1291	1338	.arg5_type = ARG_CONST_SIZE_OR_ZERO,
1292	1339	};
1293	1340
@@ -1509,7 +1556,7 @@ static const struct bpf_func_proto bpf_perf_event_output_proto_raw_tp = { · 215 unmodified lines
1509	1556	.arg1_type = ARG_PTR_TO_CTX,
1510	1557	.arg2_type = ARG_CONST_MAP_PTR,
1511	1558	.arg3_type = ARG_ANYTHING,
1512		− .arg4_type = ARG_PTR_TO_MEM,
	1559	+ .arg4_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
1513	1560	.arg5_type = ARG_CONST_SIZE_OR_ZERO,
1514	1561	};
1515	1562
@@ -1563,7 +1610,7 @@ static const struct bpf_func_proto bpf_get_stack_proto_raw_tp = { · 47 unmodified lines
1563	1610	.gpl_only = true,
1564	1611	.ret_type = RET_INTEGER,
1565	1612	.arg1_type = ARG_PTR_TO_CTX,
1566		− .arg2_type = ARG_PTR_TO_MEM,
	1613	+ .arg2_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
1567	1614	.arg3_type = ARG_CONST_SIZE_OR_ZERO,
1568	1615	.arg4_type = ARG_ANYTHING,
1569	1616	};
@@ -1629,6 +1676,12 @@ tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 59 unmodified lines
1629	1676	NULL;
1630	1677	case BPF_FUNC_d_path:
1631	1678	return &bpf_d_path_proto;
	1679	+ case BPF_FUNC_get_func_arg:
	1680	+ return bpf_prog_has_trampoline(prog) ? &bpf_get_func_arg_proto : NULL;
	1681	+ case BPF_FUNC_get_func_ret:
	1682	+ return bpf_prog_has_trampoline(prog) ? &bpf_get_func_ret_proto : NULL;
	1683	+ case BPF_FUNC_get_func_arg_cnt:
	1684	+ return bpf_prog_has_trampoline(prog) ? &bpf_get_func_arg_cnt_proto : NULL;
1632	1685	default:
1633	1686	fn = raw_tp_prog_func_proto(func_id, prog);
1634	1687	if (!fn && prog->expected_attach_type == BPF_TRACE_ITER)

kernel/trace/trace_kprobe.c+1−0

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	7	*/
8	8	#define pr_fmt(fmt) "trace_kprobe: " fmt
9	9
	10	+#include <linux/bpf-cgroup.h>
10	11	#include <linux/security.h>
11	12	#include <linux/module.h>
12	13	#include <linux/uaccess.h>

kernel/trace/trace_uprobe.c+1−0

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	7	*/
8	8	#define pr_fmt(fmt) "trace_uprobe: " fmt
9	9
	10	+#include <linux/bpf-cgroup.h>
10	11	#include <linux/security.h>
11	12	#include <linux/ctype.h>
12	13	#include <linux/module.h>