Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Alexei Starovoitov says: ==================== pull-request: bpf-next 2021-12-30 The following pull-request contains BPF updates for your *net-next* tree. We've added 72 non-merge commits during the last 20 day(s) which contain a total of 223 files changed, 3510 insertions(+), 1591 deletions(-). The main changes are: 1) Automatic setrlimit in libbpf when bpf is memcg's in the kernel, from Andrii. 2) Beautify and de-verbose verifier logs, from Christy. 3) Composable verifier types, from Hao. 4) bpf_strncmp helper, from Hou. 5) bpf.h header dependency cleanup, from Jakub. 6) get_func_[arg|ret|arg_cnt] helpers, from Jiri. 7) Sleepable local storage, from KP. 8) Extend kfunc with PTR_TO_CTX, PTR_TO_MEM argument support, from Kumar. ====================

David S. Miller committed 4 years ago

Commit e63a0232 parents ce2b6eb 9e6b19a221 files · +3506 −1591Browse files

Showing the changes to kernel (23 of 221 files). Show all

Unified · Split

23 files

kernel/bpf/bloom_filter.c +6 −0
kernel/bpf/bpf_inode_storage.c +5 −1
kernel/bpf/bpf_local_storage.c +37 −13
kernel/bpf/bpf_task_storage.c +5 −1
kernel/bpf/btf.c +88 −36
kernel/bpf/cgroup.c +1 −1
kernel/bpf/cpumap.c +2 −2
kernel/bpf/devmap.c +2 −2
kernel/bpf/helpers.c +23 −6
kernel/bpf/local_storage.c +1 −2
kernel/bpf/lpm_trie.c +1 −1
kernel/bpf/map_iter.c +2 −2
kernel/bpf/net_namespace.c +1 −0
kernel/bpf/reuseport_array.c +1 −5
kernel/bpf/ringbuf.c +1 −1
kernel/bpf/syscall.c +2 −1
kernel/bpf/trampoline.c +8 −0
kernel/bpf/verifier.c +415 −281
kernel/cgroup/cgroup.c +1 −0
kernel/sysctl.c +1 −0
kernel/trace/bpf_trace.c +67 −14
kernel/trace/trace_kprobe.c +1 −0
kernel/trace/trace_uprobe.c +1 −0

kernel/bpf/bloom_filter.c+6−0

 	return -EOPNOTSUPP;
 }
+static int bloom_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
+{
+	return -EOPNOTSUPP;
+}
 static struct bpf_map *bloom_map_alloc(union bpf_attr *attr)
 {
 	u32 bitset_bytes, bitset_mask, nr_hash_funcs, nr_bits;
 	.map_meta_equal = bpf_map_meta_equal,
 	.map_alloc = bloom_map_alloc,
 	.map_free = bloom_map_free,
+	.map_get_next_key = bloom_map_get_next_key,
 	.map_push_elem = bloom_map_push_elem,
 	.map_peek_elem = bloom_map_peek_elem,
 	.map_pop_elem = bloom_map_pop_elem,

kernel/bpf/bpf_inode_storage.c+5−1

 #include <linux/bpf_lsm.h>
 #include <linux/btf_ids.h>
 #include <linux/fdtable.h>
+#include <linux/rcupdate_trace.h>
 DEFINE_BPF_STORAGE_CACHE(inode_cache);
 	if (!bsb)
 		return NULL;
-	inode_storage = rcu_dereference(bsb->storage);
+	inode_storage =
+		rcu_dereference_check(bsb->storage, bpf_rcu_lock_held());
 	if (!inode_storage)
 		return NULL;
 {
 	struct bpf_local_storage_data *sdata;
+	WARN_ON_ONCE(!bpf_rcu_lock_held());
 	if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
 		return (unsigned long)NULL;
 BPF_CALL_2(bpf_inode_storage_delete,
 	   struct bpf_map *, map, struct inode *, inode)
 {
+	WARN_ON_ONCE(!bpf_rcu_lock_held());
 	if (!inode)
 		return -EINVAL;

kernel/bpf/bpf_local_storage.c+37−13

 #include <net/sock.h>
 #include <uapi/linux/sock_diag.h>
 #include <uapi/linux/btf.h>
+#include <linux/rcupdate.h>
+#include <linux/rcupdate_trace.h>
+#include <linux/rcupdate_wait.h>
 #define BPF_LOCAL_STORAGE_CREATE_FLAG_MASK (BPF_F_NO_PREALLOC | BPF_F_CLONE)
 	return NULL;
 }
+void bpf_local_storage_free_rcu(struct rcu_head *rcu)
+{
+	struct bpf_local_storage *local_storage;
+	local_storage = container_of(rcu, struct bpf_local_storage, rcu);
+	kfree_rcu(local_storage, rcu);
+}
+static void bpf_selem_free_rcu(struct rcu_head *rcu)
+{
+	struct bpf_local_storage_elem *selem;
+	selem = container_of(rcu, struct bpf_local_storage_elem, rcu);
+	kfree_rcu(selem, rcu);
+}
 /* local_storage->lock must be held and selem->local_storage == local_storage.
  * The caller must ensure selem->smap is still valid to be
  * dereferenced for its smap->elem_size and smap->cache_idx.
 	bool free_local_storage;
 	void *owner;
-	smap = rcu_dereference(SDATA(selem)->smap);
+	smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held());
 	owner = local_storage->owner;
 	/* All uncharging on the owner must be done first.
 		 *
 		 * Although the unlock will be done under
 		 * rcu_read_lock(),  it is more intutivie to
-		 * read if kfree_rcu(local_storage, rcu) is done
+		 * read if the freeing of the storage is done
 		 * after the raw_spin_unlock_bh(&local_storage->lock).
 		 *
 		 * Hence, a "bool free_local_storage" is returned
-		 * to the caller which then calls the kfree_rcu()
+		 * to the caller which then calls then frees the storage after
-		 * after unlock.
+		 * all the RCU grace periods have expired.
 		 */
 	}
 	hlist_del_init_rcu(&selem->snode);
 	    SDATA(selem))
 		RCU_INIT_POINTER(local_storage->cache[smap->cache_idx], NULL);
-	kfree_rcu(selem, rcu);
+	call_rcu_tasks_trace(&selem->rcu, bpf_selem_free_rcu);
 	return free_local_storage;
 }
 		/* selem has already been unlinked from sk */
 		return;
-	local_storage = rcu_dereference(selem->local_storage);
+	local_storage = rcu_dereference_check(selem->local_storage,
+					      bpf_rcu_lock_held());
 	raw_spin_lock_irqsave(&local_storage->lock, flags);
 	if (likely(selem_linked_to_storage(selem)))
 		free_local_storage = bpf_selem_unlink_storage_nolock(
 	raw_spin_unlock_irqrestore(&local_storage->lock, flags);
 	if (free_local_storage)
-		kfree_rcu(local_storage, rcu);
+		call_rcu_tasks_trace(&local_storage->rcu,
+				     bpf_local_storage_free_rcu);
 }
 void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,
 		/* selem has already be unlinked from smap */
 		return;
-	smap = rcu_dereference(SDATA(selem)->smap);
+	smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held());
 	b = select_bucket(smap, selem);
 	raw_spin_lock_irqsave(&b->lock, flags);
 	if (likely(selem_linked_to_map(selem)))
 	struct bpf_local_storage_elem *selem;
 	/* Fast path (cache hit) */
-	sdata = rcu_dereference(local_storage->cache[smap->cache_idx]);
+	sdata = rcu_dereference_check(local_storage->cache[smap->cache_idx],
+				      bpf_rcu_lock_held());
 	if (sdata && rcu_access_pointer(sdata->smap) == smap)
 		return sdata;
 	/* Slow path (cache miss) */
-	hlist_for_each_entry_rcu(selem, &local_storage->list, snode)
+	hlist_for_each_entry_rcu(selem, &local_storage->list, snode,
+				  rcu_read_lock_trace_held())
 		if (rcu_access_pointer(SDATA(selem)->smap) == smap)
 			break;
 		 * bucket->list, first_selem can be freed immediately
 		 * (instead of kfree_rcu) because
 		 * bpf_local_storage_map_free() does a
-		 * synchronize_rcu() before walking the bucket->list.
+		 * synchronize_rcu_mult (waiting for both sleepable and
+		 * normal programs) before walking the bucket->list.
 		 * Hence, no one is accessing selem from the
 		 * bucket->list under rcu_read_lock().
 		 */
 		     !map_value_has_spin_lock(&smap->map)))
 		return ERR_PTR(-EINVAL);
-	local_storage = rcu_dereference(*owner_storage(smap, owner));
+	local_storage = rcu_dereference_check(*owner_storage(smap, owner),
+					      bpf_rcu_lock_held());
 	if (!local_storage || hlist_empty(&local_storage->list)) {
 		/* Very first elem for the owner */
 		err = check_flags(NULL, map_flags);

kernel/bpf/bpf_task_storage.c+5−1

 #include <uapi/linux/btf.h>
 #include <linux/btf_ids.h>
 #include <linux/fdtable.h>
+#include <linux/rcupdate_trace.h>
 DEFINE_BPF_STORAGE_CACHE(task_cache);
 	struct bpf_local_storage *task_storage;
 	struct bpf_local_storage_map *smap;
-	task_storage = rcu_dereference(task->bpf_storage);
+	task_storage =
+		rcu_dereference_check(task->bpf_storage, bpf_rcu_lock_held());
 	if (!task_storage)
 		return NULL;
 {
 	struct bpf_local_storage_data *sdata;
+	WARN_ON_ONCE(!bpf_rcu_lock_held());
 	if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
 		return (unsigned long)NULL;
 {
 	int ret;
+	WARN_ON_ONCE(!bpf_rcu_lock_held());
 	if (!task)
 		return -EINVAL;

kernel/bpf/btf.c+88−36

 		return prog->aux->attach_btf;
 }
-static bool is_string_ptr(struct btf *btf, const struct btf_type *t)
+static bool is_int_ptr(struct btf *btf, const struct btf_type *t)
 {
 	/* t comes in already as a pointer */
 	t = btf_type_by_id(btf, t->type);
 	if (BTF_INFO_KIND(t->info) == BTF_KIND_CONST)
 		t = btf_type_by_id(btf, t->type);
-	/* char, signed char, unsigned char */
+	return btf_type_is_int(t);
-	return btf_type_is_int(t) && t->size == 1;
 }
 bool btf_ctx_access(int off, int size, enum bpf_access_type type,
 	/* check for PTR_TO_RDONLY_BUF_OR_NULL or PTR_TO_RDWR_BUF_OR_NULL */
 	for (i = 0; i < prog->aux->ctx_arg_info_size; i++) {
 		const struct bpf_ctx_arg_aux *ctx_arg_info = &prog->aux->ctx_arg_info[i];
+		u32 type, flag;
-		if (ctx_arg_info->offset == off &&
+		type = base_type(ctx_arg_info->reg_type);
-		    (ctx_arg_info->reg_type == PTR_TO_RDONLY_BUF_OR_NULL ||
+		flag = type_flag(ctx_arg_info->reg_type);
-		     ctx_arg_info->reg_type == PTR_TO_RDWR_BUF_OR_NULL)) {
+		if (ctx_arg_info->offset == off && type == PTR_TO_BUF &&
+		    (flag & PTR_MAYBE_NULL)) {
 			info->reg_type = ctx_arg_info->reg_type;
 			return true;
 		}
 		 */
 		return true;
-	if (is_string_ptr(btf, t))
+	if (is_int_ptr(btf, t))
 		return true;
 	/* this is a pointer to another type */
 #endif
 };
+/* Returns true if struct is composed of scalars, 4 levels of nesting allowed */
+static bool __btf_type_is_scalar_struct(struct bpf_verifier_log *log,
+					const struct btf *btf,
+					const struct btf_type *t, int rec)
+{
+	const struct btf_type *member_type;
+	const struct btf_member *member;
+	u32 i;
+	if (!btf_type_is_struct(t))
+		return false;
+	for_each_member(i, t, member) {
+		const struct btf_array *array;
+		member_type = btf_type_skip_modifiers(btf, member->type, NULL);
+		if (btf_type_is_struct(member_type)) {
+			if (rec >= 3) {
+				bpf_log(log, "max struct nesting depth exceeded\n");
+				return false;
+			}
+			if (!__btf_type_is_scalar_struct(log, btf, member_type, rec + 1))
+				return false;
+			continue;
+		}
+		if (btf_type_is_array(member_type)) {
+			array = btf_type_array(member_type);
+			if (!array->nelems)
+				return false;
+			member_type = btf_type_skip_modifiers(btf, array->type, NULL);
+			if (!btf_type_is_scalar(member_type))
+				return false;
+			continue;
+		}
+		if (!btf_type_is_scalar(member_type))
+			return false;
+	}
+	return true;
+}
 static int btf_check_func_arg_match(struct bpf_verifier_env *env,
 				    const struct btf *btf, u32 func_id,
 				    struct bpf_reg_state *regs,
 				    bool ptr_to_mem_ok)
 {
 	struct bpf_verifier_log *log = &env->log;
+	bool is_kfunc = btf_is_kernel(btf);
 	const char *func_name, *ref_tname;
 	const struct btf_type *t, *ref_t;
 	const struct btf_param *args;
 		ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id);
 		ref_tname = btf_name_by_offset(btf, ref_t->name_off);
-		if (btf_is_kernel(btf)) {
+		if (btf_get_prog_ctx_type(log, btf, t,
+					  env->prog->type, i)) {
+			/* If function expects ctx type in BTF check that caller
+			 * is passing PTR_TO_CTX.
+			 */
+			if (reg->type != PTR_TO_CTX) {
+				bpf_log(log,
+					"arg#%d expected pointer to ctx, but got %s\n",
+					i, btf_type_str(t));
+				return -EINVAL;
+			}
+			if (check_ctx_reg(env, reg, regno))
+				return -EINVAL;
+		} else if (is_kfunc && (reg->type == PTR_TO_BTF_ID || reg2btf_ids[reg->type])) {
 			const struct btf_type *reg_ref_t;
 			const struct btf *reg_btf;
 			const char *reg_ref_tname;
 			if (reg->type == PTR_TO_BTF_ID) {
 				reg_btf = reg->btf;
 				reg_ref_id = reg->btf_id;
-			} else if (reg2btf_ids[reg->type]) {
+			} else {
 				reg_btf = btf_vmlinux;
 				reg_ref_id = *reg2btf_ids[reg->type];
-			} else {
-				bpf_log(log, "kernel function %s args#%d expected pointer to %s %s but R%d is not a pointer to btf_id\n",
-					func_name, i,
-					btf_type_str(ref_t), ref_tname, regno);
-				return -EINVAL;
 			}
 			reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id,
 					reg_ref_tname);
 				return -EINVAL;
 			}
-		} else if (btf_get_prog_ctx_type(log, btf, t,
-						 env->prog->type, i)) {
-			/* If function expects ctx type in BTF check that caller
-			 * is passing PTR_TO_CTX.
-			 */
-			if (reg->type != PTR_TO_CTX) {
-				bpf_log(log,
-					"arg#%d expected pointer to ctx, but got %s\n",
-					i, btf_type_str(t));
-				return -EINVAL;
-			}
-			if (check_ctx_reg(env, reg, regno))
-				return -EINVAL;
 		} else if (ptr_to_mem_ok) {
 			const struct btf_type *resolve_ret;
 			u32 type_size;
+			if (is_kfunc) {
+				/* Permit pointer to mem, but only when argument
+				 * type is pointer to scalar, or struct composed
+				 * (recursively) of scalars.
+				 */
+				if (!btf_type_is_scalar(ref_t) &&
+				    !__btf_type_is_scalar_struct(log, btf, ref_t, 0)) {
+					bpf_log(log,
+						"arg#%d pointer type %s %s must point to scalar or struct with scalar\n",
+						i, btf_type_str(ref_t), ref_tname);
+					return -EINVAL;
+				}
+			}
 			resolve_ret = btf_resolve_size(btf, ref_t, &type_size);
 			if (IS_ERR(resolve_ret)) {
 				bpf_log(log,
 			if (check_mem_reg(env, reg, regno, type_size))
 				return -EINVAL;
 		} else {
+			bpf_log(log, "reg type unsupported for arg#%d %sfunction %s#%d\n", i,
+				is_kfunc ? "kernel " : "", func_name, func_id);
 			return -EINVAL;
 		}
 	}
 			      const struct btf *btf, u32 func_id,
 			      struct bpf_reg_state *regs)
 {
-	return btf_check_func_arg_match(env, btf, func_id, regs, false);
+	return btf_check_func_arg_match(env, btf, func_id, regs, true);
 }
 /* Convert BTF of a function into bpf_reg_state if possible
 				return -EINVAL;
 			}
-			reg->type = PTR_TO_MEM_OR_NULL;
+			reg->type = PTR_TO_MEM | PTR_MAYBE_NULL;
 			reg->id = ++env->id_gen;
 			continue;
 	.func		= bpf_btf_find_by_name_kind,
 	.gpl_only	= false,
 	.ret_type	= RET_INTEGER,
-	.arg1_type	= ARG_PTR_TO_MEM,
+	.arg1_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
 	.arg2_type	= ARG_CONST_SIZE,
 	.arg3_type	= ARG_ANYTHING,
 	.arg4_type	= ARG_ANYTHING,
 		bpf_free_cands_from_cache(*cc);
 		*cc = NULL;
 	}
-	new_cands = kmalloc(sizeof_cands(cands->cnt), GFP_KERNEL);
+	new_cands = kmemdup(cands, sizeof_cands(cands->cnt), GFP_KERNEL);
 	if (!new_cands) {
 		bpf_free_cands(cands);
 		return ERR_PTR(-ENOMEM);
 	}
-	memcpy(new_cands, cands, sizeof_cands(cands->cnt));
 	/* strdup the name, since it will stay in cache.
 	 * the cands->name points to strings in prog's BTF and the prog can be unloaded.
 	 */
 	main_btf = bpf_get_btf_vmlinux();
 	if (IS_ERR(main_btf))
-		return (void *)main_btf;
+		return ERR_CAST(main_btf);
 	local_type = btf_type_by_id(local_btf, local_type_id);
 	if (!local_type)
 	/* Attempt to find target candidates in vmlinux BTF first */
 	cands = bpf_core_add_cands(cands, main_btf, 1);
 	if (IS_ERR(cands))
-		return cands;
+		return ERR_CAST(cands);
 	/* cands is a pointer to kmalloced memory here if cands->cnt > 0 */
 	/* populate cache even when cands->cnt == 0 */
 	cc = populate_cand_cache(cands, vmlinux_cand_cache, VMLINUX_CAND_CACHE_SIZE);
 	if (IS_ERR(cc))
-		return cc;
+		return ERR_CAST(cc);
 	/* if vmlinux BTF has any candidate, don't go for module BTFs */
 	if (cc->cnt)
 		cands = bpf_core_add_cands(cands, mod_btf, btf_nr_types(main_btf));
 		if (IS_ERR(cands)) {
 			btf_put(mod_btf);
-			return cands;
+			return ERR_CAST(cands);
 		}
 		spin_lock_bh(&btf_idr_lock);
 		btf_put(mod_btf);

kernel/bpf/cgroup.c+1−1

@@ -1789,7 +1789,7 @@ static const struct bpf_func_proto bpf_sysctl_set_new_value_proto = { · 1788 unmodified lines
1789	.gpl_only = false,	1789	.gpl_only = false,
1790	.ret_type = RET_INTEGER,	1790	.ret_type = RET_INTEGER,
1791	.arg1_type = ARG_PTR_TO_CTX,	1791	.arg1_type = ARG_PTR_TO_CTX,
1792	.arg2_type = ARG_PTR_TO_MEM,	1792	.arg2_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
1793	.arg3_type = ARG_CONST_SIZE,	1793	.arg3_type = ARG_CONST_SIZE,
1794	};	1794	};
1795		1795

kernel/bpf/cpumap.c+2−2

@@ -195,7 +195,7 @@ static void cpu_map_bpf_prog_run_skb(struct bpf_cpu_map_entry *rcpu, · 194 unmodified lines
195	}	195	}
196	return;	196	return;
197	default:	197	default:
198	bpf_warn_invalid_xdp_action(act);	198	bpf_warn_invalid_xdp_action(NULL, rcpu->prog, act);
199	fallthrough;	199	fallthrough;
200	case XDP_ABORTED:	200	case XDP_ABORTED:
201	trace_xdp_exception(skb->dev, rcpu->prog, act);	201	trace_xdp_exception(skb->dev, rcpu->prog, act);
@@ -254,7 +254,7 @@ static int cpu_map_bpf_prog_run_xdp(struct bpf_cpu_map_entry *rcpu, · 52 unmodified lines
254	}	254	}
255	break;	255	break;
256	default:	256	default:
257	bpf_warn_invalid_xdp_action(act);	257	bpf_warn_invalid_xdp_action(NULL, rcpu->prog, act);
258	fallthrough;	258	fallthrough;
259	case XDP_DROP:	259	case XDP_DROP:
260	xdp_return_frame(xdpf);	260	xdp_return_frame(xdpf);

kernel/bpf/devmap.c+2−2

@@ -348,7 +348,7 @@ static int dev_map_bpf_prog_run(struct bpf_prog *xdp_prog, · 347 unmodified lines
348	frames[nframes++] = xdpf;	348	frames[nframes++] = xdpf;
349	break;	349	break;
350	default:	350	default:
351	bpf_warn_invalid_xdp_action(act);	351	bpf_warn_invalid_xdp_action(NULL, xdp_prog, act);
352	fallthrough;	352	fallthrough;
353	case XDP_ABORTED:	353	case XDP_ABORTED:
354	trace_xdp_exception(dev, xdp_prog, act);	354	trace_xdp_exception(dev, xdp_prog, act);
@@ -507,7 +507,7 @@ static u32 dev_map_bpf_prog_run_skb(struct sk_buff *skb, struct bpf_dtab_netdev · 152 unmodified lines
507	__skb_push(skb, skb->mac_len);	507	__skb_push(skb, skb->mac_len);
508	break;	508	break;
509	default:	509	default:
510	bpf_warn_invalid_xdp_action(act);	510	bpf_warn_invalid_xdp_action(NULL, dst->xdp_prog, act);
511	fallthrough;	511	fallthrough;
512	case XDP_ABORTED:	512	case XDP_ABORTED:
513	trace_xdp_exception(dst->dev, dst->xdp_prog, act);	513	trace_xdp_exception(dst->dev, dst->xdp_prog, act);

kernel/bpf/helpers.c+23−6

 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
  */
 #include <linux/bpf.h>
+#include <linux/bpf-cgroup.h>
 #include <linux/rcupdate.h>
 #include <linux/random.h>
 #include <linux/smp.h>
 	.func		= bpf_strtol,
 	.gpl_only	= false,
 	.ret_type	= RET_INTEGER,
-	.arg1_type	= ARG_PTR_TO_MEM,
+	.arg1_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
 	.arg2_type	= ARG_CONST_SIZE,
 	.arg3_type	= ARG_ANYTHING,
 	.arg4_type	= ARG_PTR_TO_LONG,
 	.func		= bpf_strtoul,
 	.gpl_only	= false,
 	.ret_type	= RET_INTEGER,
-	.arg1_type	= ARG_PTR_TO_MEM,
+	.arg1_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
 	.arg2_type	= ARG_CONST_SIZE,
 	.arg3_type	= ARG_ANYTHING,
 	.arg4_type	= ARG_PTR_TO_LONG,
 };
 #endif
+BPF_CALL_3(bpf_strncmp, const char *, s1, u32, s1_sz, const char *, s2)
+{
+	return strncmp(s1, s2, s1_sz);
+}
+const struct bpf_func_proto bpf_strncmp_proto = {
+	.func		= bpf_strncmp,
+	.gpl_only	= false,
+	.ret_type	= RET_INTEGER,
+	.arg1_type	= ARG_PTR_TO_MEM,
+	.arg2_type	= ARG_CONST_SIZE,
+	.arg3_type	= ARG_PTR_TO_CONST_STR,
+};
 BPF_CALL_4(bpf_get_ns_current_pid_tgid, u64, dev, u64, ino,
 	   struct bpf_pidns_info *, nsdata, u32, size)
 {
 	.arg1_type      = ARG_PTR_TO_CTX,
 	.arg2_type      = ARG_CONST_MAP_PTR,
 	.arg3_type      = ARG_ANYTHING,
-	.arg4_type      = ARG_PTR_TO_MEM,
+	.arg4_type      = ARG_PTR_TO_MEM | MEM_RDONLY,
 	.arg5_type      = ARG_CONST_SIZE_OR_ZERO,
 };
 const struct bpf_func_proto bpf_per_cpu_ptr_proto = {
 	.func		= bpf_per_cpu_ptr,
 	.gpl_only	= false,
-	.ret_type	= RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL,
+	.ret_type	= RET_PTR_TO_MEM_OR_BTF_ID | PTR_MAYBE_NULL | MEM_RDONLY,
 	.arg1_type	= ARG_PTR_TO_PERCPU_BTF_ID,
 	.arg2_type	= ARG_ANYTHING,
 };
 const struct bpf_func_proto bpf_this_cpu_ptr_proto = {
 	.func		= bpf_this_cpu_ptr,
 	.gpl_only	= false,
-	.ret_type	= RET_PTR_TO_MEM_OR_BTF_ID,
+	.ret_type	= RET_PTR_TO_MEM_OR_BTF_ID | MEM_RDONLY,
 	.arg1_type	= ARG_PTR_TO_PERCPU_BTF_ID,
 };
 	.arg1_type	= ARG_PTR_TO_MEM_OR_NULL,
 	.arg2_type	= ARG_CONST_SIZE_OR_ZERO,
 	.arg3_type	= ARG_PTR_TO_CONST_STR,
-	.arg4_type	= ARG_PTR_TO_MEM_OR_NULL,
+	.arg4_type	= ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY,
 	.arg5_type	= ARG_CONST_SIZE_OR_ZERO,
 };
 		return &bpf_for_each_map_elem_proto;
 	case BPF_FUNC_loop:
 		return &bpf_loop_proto;
+	case BPF_FUNC_strncmp:
+		return &bpf_strncmp_proto;
 	default:
 		break;
 	}

kernel/bpf/local_storage.c+1−2

@@ -163,8 +163,7 @@ static int cgroup_storage_update_elem(struct bpf_map map, void key, · 162 unmodified lines
163	return 0;	163	return 0;
164	}	164	}
165		165
166	new = bpf_map_kmalloc_node(map, sizeof(struct bpf_storage_buffer) +	166	new = bpf_map_kmalloc_node(map, struct_size(new, data, map->value_size),
167	map->value_size,
168	__GFP_ZERO \| GFP_ATOMIC \| __GFP_NOWARN,	167	__GFP_ZERO \| GFP_ATOMIC \| __GFP_NOWARN,
169	map->numa_node);	168	map->numa_node);
170	if (!new)	169	if (!new)

kernel/bpf/lpm_trie.c+1−1

@@ -412,7 +412,7 @@ static int trie_update_elem(struct bpf_map *map, · 411 unmodified lines
412	rcu_assign_pointer(im_node->child[1], node);	412	rcu_assign_pointer(im_node->child[1], node);
413	}	413	}
414		414
415	/* Finally, assign the intermediate node to the determined spot */	415	/* Finally, assign the intermediate node to the determined slot */
416	rcu_assign_pointer(*slot, im_node);	416	rcu_assign_pointer(*slot, im_node);
417		417
418	out:	418	out:

kernel/bpf/map_iter.c+2−2

@@ -174,9 +174,9 @@ static const struct bpf_iter_reg bpf_map_elem_reg_info = { · 173 unmodified lines
174	.ctx_arg_info_size = 2,	174	.ctx_arg_info_size = 2,
175	.ctx_arg_info = {	175	.ctx_arg_info = {
176	{ offsetof(struct bpf_iter__bpf_map_elem, key),	176	{ offsetof(struct bpf_iter__bpf_map_elem, key),
177	PTR_TO_RDONLY_BUF_OR_NULL },	177	PTR_TO_BUF \| PTR_MAYBE_NULL \| MEM_RDONLY },
178	{ offsetof(struct bpf_iter__bpf_map_elem, value),	178	{ offsetof(struct bpf_iter__bpf_map_elem, value),
179	PTR_TO_RDWR_BUF_OR_NULL },	179	PTR_TO_BUF \| PTR_MAYBE_NULL },
180	},	180	},
181	};	181	};
182		182

kernel/bpf/net_namespace.c+1−0

@@ -1,6 +1,7 @@
1	// SPDX-License-Identifier: GPL-2.0	1	// SPDX-License-Identifier: GPL-2.0
2		2
3	#include <linux/bpf.h>	3	#include <linux/bpf.h>
		4	#include <linux/bpf-netns.h>
4	#include <linux/filter.h>	5	#include <linux/filter.h>
5	#include <net/net_namespace.h>	6	#include <net/net_namespace.h>
6		7

kernel/bpf/reuseport_array.c+1−5

 {
 	int numa_node = bpf_map_attr_numa_node(attr);
 	struct reuseport_array *array;
-	u64 array_size;
 	if (!bpf_capable())
 		return ERR_PTR(-EPERM);
-	array_size = sizeof(*array);
-	array_size += (u64)attr->max_entries * sizeof(struct sock *);
 	/* allocate all map elements and zero-initialize them */
-	array = bpf_map_area_alloc(array_size, numa_node);
+	array = bpf_map_area_alloc(struct_size(array, ptrs, attr->max_entries), numa_node);
 	if (!array)
 		return ERR_PTR(-ENOMEM);

kernel/bpf/ringbuf.c+1−1

@@ -444,7 +444,7 @@ const struct bpf_func_proto bpf_ringbuf_output_proto = { · 443 unmodified lines
444	.func = bpf_ringbuf_output,	444	.func = bpf_ringbuf_output,
445	.ret_type = RET_INTEGER,	445	.ret_type = RET_INTEGER,
446	.arg1_type = ARG_CONST_MAP_PTR,	446	.arg1_type = ARG_CONST_MAP_PTR,
447	.arg2_type = ARG_PTR_TO_MEM,	447	.arg2_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
448	.arg3_type = ARG_CONST_SIZE_OR_ZERO,	448	.arg3_type = ARG_CONST_SIZE_OR_ZERO,
449	.arg4_type = ARG_ANYTHING,	449	.arg4_type = ARG_ANYTHING,
450	};	450	};

kernel/bpf/syscall.c+2−1

 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
  */
 #include <linux/bpf.h>
+#include <linux/bpf-cgroup.h>
 #include <linux/bpf_trace.h>
 #include <linux/bpf_lirc.h>
 #include <linux/bpf_verifier.h>
 	.gpl_only	= false,
 	.ret_type	= RET_INTEGER,
 	.arg1_type	= ARG_ANYTHING,
-	.arg2_type	= ARG_PTR_TO_MEM,
+	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
 	.arg3_type	= ARG_CONST_SIZE,
 };

kernel/bpf/trampoline.c+8−0

@@ -27,6 +27,14 @@ static struct hlist_head trampoline_table[TRAMPOLINE_TABLE_SIZE]; · 26 unmodified lines
27	/* serializes access to trampoline_table */	27	/* serializes access to trampoline_table */
28	static DEFINE_MUTEX(trampoline_mutex);	28	static DEFINE_MUTEX(trampoline_mutex);
29		29
		30	bool bpf_prog_has_trampoline(const struct bpf_prog *prog)
		31	{
		32	enum bpf_attach_type eatype = prog->expected_attach_type;
		33
		34	return eatype == BPF_TRACE_FENTRY \|\| eatype == BPF_TRACE_FEXIT \|\|
		35	eatype == BPF_MODIFY_RETURN;
		36	}
		37
30	void *bpf_jit_alloc_exec_page(void)	38	void *bpf_jit_alloc_exec_page(void)
31	{	39	{
32	void *image;	40	void *image;

kernel/bpf/verifier.c+415−281

@@ -4,6 +4,7 @@ · 3 unmodified lines
4	* Copyright (c) 2018 Covalent IO, Inc. http://covalent.io	4	* Copyright (c) 2018 Covalent IO, Inc. http://covalent.io
5	*/	5	*/
6	#include <uapi/linux/btf.h>	6	#include <uapi/linux/btf.h>
		7	#include <linux/bpf-cgroup.h>
7	#include <linux/kernel.h>	8	#include <linux/kernel.h>
8	#include <linux/types.h>	9	#include <linux/types.h>
9	#include <linux/slab.h>	10	#include <linux/slab.h>
@@ -441,18 +442,6 @@ static bool reg_type_not_null(enum bpf_reg_type type) · 431 unmodified lines
441	type == PTR_TO_SOCK_COMMON;	442	type == PTR_TO_SOCK_COMMON;
442	}	443	}
443		444
444	static bool reg_type_may_be_null(enum bpf_reg_type type)
445	{
446	return type == PTR_TO_MAP_VALUE_OR_NULL \|\|
447	type == PTR_TO_SOCKET_OR_NULL \|\|
448	type == PTR_TO_SOCK_COMMON_OR_NULL \|\|
449	type == PTR_TO_TCP_SOCK_OR_NULL \|\|
450	type == PTR_TO_BTF_ID_OR_NULL \|\|
451	type == PTR_TO_MEM_OR_NULL \|\|
452	type == PTR_TO_RDONLY_BUF_OR_NULL \|\|
453	type == PTR_TO_RDWR_BUF_OR_NULL;
454	}
455
456	static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg)	445	static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg)
457	{	446	{
458	return reg->type == PTR_TO_MAP_VALUE &&	447	return reg->type == PTR_TO_MAP_VALUE &&
@@ -461,12 +450,14 @@ static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg) · 2 unmodified lines
461		450
462	static bool reg_type_may_be_refcounted_or_null(enum bpf_reg_type type)	451	static bool reg_type_may_be_refcounted_or_null(enum bpf_reg_type type)
463	{	452	{
464	return type == PTR_TO_SOCKET \|\|	453	return base_type(type) == PTR_TO_SOCKET \|\|
465	type == PTR_TO_SOCKET_OR_NULL \|\|	454	base_type(type) == PTR_TO_TCP_SOCK \|\|
466	type == PTR_TO_TCP_SOCK \|\|	455	base_type(type) == PTR_TO_MEM;
467	type == PTR_TO_TCP_SOCK_OR_NULL \|\|	456	}
468	type == PTR_TO_MEM \|\|	457
469	type == PTR_TO_MEM_OR_NULL;	458	static bool type_is_rdonly_mem(u32 type)
		459	{
		460	return type & MEM_RDONLY;
470	}	461	}
471		462
472	static bool arg_type_may_be_refcounted(enum bpf_arg_type type)	463	static bool arg_type_may_be_refcounted(enum bpf_arg_type type)
@@ -474,14 +465,9 @@ static bool arg_type_may_be_refcounted(enum bpf_arg_type type) · 1 unmodified line
474	return type == ARG_PTR_TO_SOCK_COMMON;	465	return type == ARG_PTR_TO_SOCK_COMMON;
475	}	466	}
476		467
477	static bool arg_type_may_be_null(enum bpf_arg_type type)	468	static bool type_may_be_null(u32 type)
478	{	469	{
479	return type == ARG_PTR_TO_MAP_VALUE_OR_NULL \|\|	470	return type & PTR_MAYBE_NULL;
480	type == ARG_PTR_TO_MEM_OR_NULL \|\|
481	type == ARG_PTR_TO_CTX_OR_NULL \|\|
482	type == ARG_PTR_TO_SOCKET_OR_NULL \|\|
483	type == ARG_PTR_TO_ALLOC_MEM_OR_NULL \|\|
484	type == ARG_PTR_TO_STACK_OR_NULL;
485	}	471	}
486		472
487	/* Determine whether the function releases some resources allocated by another	473	/* Determine whether the function releases some resources allocated by another
@@ -541,39 +527,54 @@ static bool is_cmpxchg_insn(const struct bpf_insn *insn) · 53 unmodified lines
541	insn->imm == BPF_CMPXCHG;	527	insn->imm == BPF_CMPXCHG;
542	}	528	}
543		529
544	/* string representation of 'enum bpf_reg_type' */	530	/* string representation of 'enum bpf_reg_type'
545	static const char * const reg_type_str[] = {	531	*
546	[NOT_INIT] = "?",	532	* Note that reg_type_str() can not appear more than once in a single verbose()
547	[SCALAR_VALUE] = "inv",	533	* statement.
548	[PTR_TO_CTX] = "ctx",	534	*/
549	[CONST_PTR_TO_MAP] = "map_ptr",	535	static const char reg_type_str(struct bpf_verifier_env env,
550	[PTR_TO_MAP_VALUE] = "map_value",	536	enum bpf_reg_type type)
551	[PTR_TO_MAP_VALUE_OR_NULL] = "map_value_or_null",	537	{
552	[PTR_TO_STACK] = "fp",	538	char postfix[16] = {0}, prefix[16] = {0};
553	[PTR_TO_PACKET] = "pkt",	539	static const char * const str[] = {
554	[PTR_TO_PACKET_META] = "pkt_meta",	540	[NOT_INIT] = "?",
555	[PTR_TO_PACKET_END] = "pkt_end",	541	[SCALAR_VALUE] = "inv",
556	[PTR_TO_FLOW_KEYS] = "flow_keys",	542	[PTR_TO_CTX] = "ctx",
557	[PTR_TO_SOCKET] = "sock",	543	[CONST_PTR_TO_MAP] = "map_ptr",
558	[PTR_TO_SOCKET_OR_NULL] = "sock_or_null",	544	[PTR_TO_MAP_VALUE] = "map_value",
559	[PTR_TO_SOCK_COMMON] = "sock_common",	545	[PTR_TO_STACK] = "fp",
560	[PTR_TO_SOCK_COMMON_OR_NULL] = "sock_common_or_null",	546	[PTR_TO_PACKET] = "pkt",
561	[PTR_TO_TCP_SOCK] = "tcp_sock",	547	[PTR_TO_PACKET_META] = "pkt_meta",
562	[PTR_TO_TCP_SOCK_OR_NULL] = "tcp_sock_or_null",	548	[PTR_TO_PACKET_END] = "pkt_end",
563	[PTR_TO_TP_BUFFER] = "tp_buffer",	549	[PTR_TO_FLOW_KEYS] = "flow_keys",
564	[PTR_TO_XDP_SOCK] = "xdp_sock",	550	[PTR_TO_SOCKET] = "sock",
565	[PTR_TO_BTF_ID] = "ptr_",	551	[PTR_TO_SOCK_COMMON] = "sock_common",
566	[PTR_TO_BTF_ID_OR_NULL] = "ptr_or_null_",	552	[PTR_TO_TCP_SOCK] = "tcp_sock",
567	[PTR_TO_PERCPU_BTF_ID] = "percpu_ptr_",	553	[PTR_TO_TP_BUFFER] = "tp_buffer",
568	[PTR_TO_MEM] = "mem",	554	[PTR_TO_XDP_SOCK] = "xdp_sock",
569	[PTR_TO_MEM_OR_NULL] = "mem_or_null",	555	[PTR_TO_BTF_ID] = "ptr_",
570	[PTR_TO_RDONLY_BUF] = "rdonly_buf",	556	[PTR_TO_PERCPU_BTF_ID] = "percpu_ptr_",
571	[PTR_TO_RDONLY_BUF_OR_NULL] = "rdonly_buf_or_null",	557	[PTR_TO_MEM] = "mem",
572	[PTR_TO_RDWR_BUF] = "rdwr_buf",	558	[PTR_TO_BUF] = "buf",
573	[PTR_TO_RDWR_BUF_OR_NULL] = "rdwr_buf_or_null",	559	[PTR_TO_FUNC] = "func",
574	[PTR_TO_FUNC] = "func",	560	[PTR_TO_MAP_KEY] = "map_key",
575	[PTR_TO_MAP_KEY] = "map_key",	561	};
576	};	562
		563	if (type & PTR_MAYBE_NULL) {
		564	if (base_type(type) == PTR_TO_BTF_ID \|\|
		565	base_type(type) == PTR_TO_PERCPU_BTF_ID)
		566	strncpy(postfix, "or_null_", 16);
		567	else
		568	strncpy(postfix, "_or_null", 16);
		569	}
		570
		571	if (type & MEM_RDONLY)
		572	strncpy(prefix, "rdonly_", 16);
		573
		574	snprintf(env->type_str_buf, TYPE_STR_BUF_LEN, "%s%s%s",
		575	prefix, str[base_type(type)], postfix);
		576	return env->type_str_buf;
		577	}
577		578
578	static char slot_type_char[] = {	579	static char slot_type_char[] = {
579	[STACK_INVALID] = '?',	580	[STACK_INVALID] = '?',
@@ -608,6 +609,44 @@ static const char kernel_type_name(const struct btf btf, u32 id) · 28 unmodified lines
608	return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off);	609	return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off);
609	}	610	}
610		611
		612	static void mark_reg_scratched(struct bpf_verifier_env *env, u32 regno)
		613	{
		614	env->scratched_regs \|= 1U << regno;
		615	}
		616
		617	static void mark_stack_slot_scratched(struct bpf_verifier_env *env, u32 spi)
		618	{
		619	env->scratched_stack_slots \|= 1UL << spi;
		620	}
		621
		622	static bool reg_scratched(const struct bpf_verifier_env *env, u32 regno)
		623	{
		624	return (env->scratched_regs >> regno) & 1;
		625	}
		626
		627	static bool stack_slot_scratched(const struct bpf_verifier_env *env, u64 regno)
		628	{
		629	return (env->scratched_stack_slots >> regno) & 1;
		630	}
		631
		632	static bool verifier_state_scratched(const struct bpf_verifier_env *env)
		633	{
		634	return env->scratched_regs \|\| env->scratched_stack_slots;
		635	}
		636
		637	static void mark_verifier_state_clean(struct bpf_verifier_env *env)
		638	{
		639	env->scratched_regs = 0U;
		640	env->scratched_stack_slots = 0UL;
		641	}
		642
		643	/* Used for printing the entire verifier state. */
		644	static void mark_verifier_state_scratched(struct bpf_verifier_env *env)
		645	{
		646	env->scratched_regs = ~0U;
		647	env->scratched_stack_slots = ~0UL;
		648	}
		649
611	/* The reg state of a pointer or a bounded scalar was saved when	650	/* The reg state of a pointer or a bounded scalar was saved when
612	* it was spilled to the stack.	651	* it was spilled to the stack.
613	*/	652	*/
@@ -623,7 +662,8 @@ static void scrub_spilled_slot(u8 *stype) · 9 unmodified lines
623	}	662	}
624		663
625	static void print_verifier_state(struct bpf_verifier_env *env,	664	static void print_verifier_state(struct bpf_verifier_env *env,
626	const struct bpf_func_state *state)	665	const struct bpf_func_state *state,
		666	bool print_all)
627	{	667	{
628	const struct bpf_reg_state *reg;	668	const struct bpf_reg_state *reg;
629	enum bpf_reg_type t;	669	enum bpf_reg_type t;
@@ -636,9 +676,11 @@ static void print_verifier_state(struct bpf_verifier_env *env, · 6 unmodified lines
636	t = reg->type;	676	t = reg->type;
637	if (t == NOT_INIT)	677	if (t == NOT_INIT)
638	continue;	678	continue;
		679	if (!print_all && !reg_scratched(env, i))
		680	continue;
639	verbose(env, " R%d", i);	681	verbose(env, " R%d", i);
640	print_liveness(env, reg->live);	682	print_liveness(env, reg->live);
641	verbose(env, "=%s", reg_type_str[t]);	683	verbose(env, "=%s", reg_type_str(env, t));
642	if (t == SCALAR_VALUE && reg->precise)	684	if (t == SCALAR_VALUE && reg->precise)
643	verbose(env, "P");	685	verbose(env, "P");
644	if ((t == SCALAR_VALUE \|\| t == PTR_TO_STACK) &&	686	if ((t == SCALAR_VALUE \|\| t == PTR_TO_STACK) &&
@@ -646,9 +688,8 @@ static void print_verifier_state(struct bpf_verifier_env *env, · 1 unmodified line
646	/* reg->off should be 0 for SCALAR_VALUE */	688	/* reg->off should be 0 for SCALAR_VALUE */
647	verbose(env, "%lld", reg->var_off.value + reg->off);	689	verbose(env, "%lld", reg->var_off.value + reg->off);
648	} else {	690	} else {
649	if (t == PTR_TO_BTF_ID \|\|	691	if (base_type(t) == PTR_TO_BTF_ID \|\|
650	t == PTR_TO_BTF_ID_OR_NULL \|\|	692	base_type(t) == PTR_TO_PERCPU_BTF_ID)
651	t == PTR_TO_PERCPU_BTF_ID)
652	verbose(env, "%s", kernel_type_name(reg->btf, reg->btf_id));	693	verbose(env, "%s", kernel_type_name(reg->btf, reg->btf_id));
653	verbose(env, "(id=%d", reg->id);	694	verbose(env, "(id=%d", reg->id);
654	if (reg_type_may_be_refcounted_or_null(t))	695	if (reg_type_may_be_refcounted_or_null(t))
@@ -657,10 +698,9 @@ static void print_verifier_state(struct bpf_verifier_env *env, · 2 unmodified lines
657	verbose(env, ",off=%d", reg->off);	698	verbose(env, ",off=%d", reg->off);
658	if (type_is_pkt_pointer(t))	699	if (type_is_pkt_pointer(t))
659	verbose(env, ",r=%d", reg->range);	700	verbose(env, ",r=%d", reg->range);
660	else if (t == CONST_PTR_TO_MAP \|\|	701	else if (base_type(t) == CONST_PTR_TO_MAP \|\|
661	t == PTR_TO_MAP_KEY \|\|	702	base_type(t) == PTR_TO_MAP_KEY \|\|
662	t == PTR_TO_MAP_VALUE \|\|	703	base_type(t) == PTR_TO_MAP_VALUE)
663	t == PTR_TO_MAP_VALUE_OR_NULL)
664	verbose(env, ",ks=%d,vs=%d",	704	verbose(env, ",ks=%d,vs=%d",
665	reg->map_ptr->key_size,	705	reg->map_ptr->key_size,
666	reg->map_ptr->value_size);	706	reg->map_ptr->value_size);
@@ -725,12 +765,14 @@ static void print_verifier_state(struct bpf_verifier_env *env, · 58 unmodified lines
725	types_buf[BPF_REG_SIZE] = 0;	765	types_buf[BPF_REG_SIZE] = 0;
726	if (!valid)	766	if (!valid)
727	continue;	767	continue;
		768	if (!print_all && !stack_slot_scratched(env, i))
		769	continue;
728	verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE);	770	verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE);
729	print_liveness(env, state->stack[i].spilled_ptr.live);	771	print_liveness(env, state->stack[i].spilled_ptr.live);
730	if (is_spilled_reg(&state->stack[i])) {	772	if (is_spilled_reg(&state->stack[i])) {
731	reg = &state->stack[i].spilled_ptr;	773	reg = &state->stack[i].spilled_ptr;
732	t = reg->type;	774	t = reg->type;
733	verbose(env, "=%s", reg_type_str[t]);	775	verbose(env, "=%s", reg_type_str(env, t));
734	if (t == SCALAR_VALUE && reg->precise)	776	if (t == SCALAR_VALUE && reg->precise)
735	verbose(env, "P");	777	verbose(env, "P");
736	if (t == SCALAR_VALUE && tnum_is_const(reg->var_off))	778	if (t == SCALAR_VALUE && tnum_is_const(reg->var_off))
@@ -750,6 +792,26 @@ static void print_verifier_state(struct bpf_verifier_env *env, · 13 unmodified lines
750	if (state->in_async_callback_fn)	792	if (state->in_async_callback_fn)
751	verbose(env, " async_cb");	793	verbose(env, " async_cb");
752	verbose(env, "\n");	794	verbose(env, "\n");
		795	mark_verifier_state_clean(env);
		796	}
		797
		798	static inline u32 vlog_alignment(u32 pos)
		799	{
		800	return round_up(max(pos + BPF_LOG_MIN_ALIGNMENT / 2, BPF_LOG_ALIGNMENT),
		801	BPF_LOG_MIN_ALIGNMENT) - pos - 1;
		802	}
		803
		804	static void print_insn_state(struct bpf_verifier_env *env,
		805	const struct bpf_func_state *state)
		806	{
		807	if (env->prev_log_len && env->prev_log_len == env->log.len_used) {
		808	/* remove new line character */
		809	bpf_vlog_reset(&env->log, env->prev_log_len - 1);
		810	verbose(env, "%*c;", vlog_alignment(env->prev_insn_print_len), ' ');
		811	} else {
		812	verbose(env, "%d:", env->insn_idx);
		813	}
		814	print_verifier_state(env, state, false);
753	}	815	}
754		816
755	/* copy array src of length n * size bytes to dst. dst is reallocated if it's too	817	/* copy array src of length n * size bytes to dst. dst is reallocated if it's too
@@ -1143,8 +1205,7 @@ static void mark_reg_known_zero(struct bpf_verifier_env *env, · 387 unmodified lines
1143		1205
1144	static void mark_ptr_not_null_reg(struct bpf_reg_state *reg)	1206	static void mark_ptr_not_null_reg(struct bpf_reg_state *reg)
1145	{	1207	{
1146	switch (reg->type) {	1208	if (base_type(reg->type) == PTR_TO_MAP_VALUE) {
1147	case PTR_TO_MAP_VALUE_OR_NULL: {
1148	const struct bpf_map *map = reg->map_ptr;	1209	const struct bpf_map *map = reg->map_ptr;
1149		1210
1150	if (map->inner_map_meta) {	1211	if (map->inner_map_meta) {
@@ -1163,32 +1224,10 @@ static void mark_ptr_not_null_reg(struct bpf_reg_state *reg) · 12 unmodified lines
1163	} else {	1224	} else {
1164	reg->type = PTR_TO_MAP_VALUE;	1225	reg->type = PTR_TO_MAP_VALUE;
1165	}	1226	}
1166	break;	1227	return;
1167	}
1168	case PTR_TO_SOCKET_OR_NULL:
1169	reg->type = PTR_TO_SOCKET;
1170	break;
1171	case PTR_TO_SOCK_COMMON_OR_NULL:
1172	reg->type = PTR_TO_SOCK_COMMON;
1173	break;
1174	case PTR_TO_TCP_SOCK_OR_NULL:
1175	reg->type = PTR_TO_TCP_SOCK;
1176	break;
1177	case PTR_TO_BTF_ID_OR_NULL:
1178	reg->type = PTR_TO_BTF_ID;
1179	break;
1180	case PTR_TO_MEM_OR_NULL:
1181	reg->type = PTR_TO_MEM;
1182	break;
1183	case PTR_TO_RDONLY_BUF_OR_NULL:
1184	reg->type = PTR_TO_RDONLY_BUF;
1185	break;
1186	case PTR_TO_RDWR_BUF_OR_NULL:
1187	reg->type = PTR_TO_RDWR_BUF;
1188	break;
1189	default:
1190	WARN_ONCE(1, "unknown nullable register type");
1191	}	1228	}
		1229
		1230	reg->type &= ~PTR_MAYBE_NULL;
1192	}	1231	}
1193		1232
1194	static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg)	1233	static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg)
@@ -1546,6 +1585,7 @@ static void init_func_state(struct bpf_verifier_env *env, · 351 unmodified lines
1546	state->frameno = frameno;	1585	state->frameno = frameno;
1547	state->subprogno = subprogno;	1586	state->subprogno = subprogno;
1548	init_reg_state(env, state);	1587	init_reg_state(env, state);
		1588	mark_verifier_state_scratched(env);
1549	}	1589	}
1550		1590
1551	/* Similar to push_stack(), but for async callbacks */	1591	/* Similar to push_stack(), but for async callbacks */
@@ -2049,7 +2089,7 @@ static int mark_reg_read(struct bpf_verifier_env *env, · 497 unmodified lines
2049	break;	2089	break;
2050	if (parent->live & REG_LIVE_DONE) {	2090	if (parent->live & REG_LIVE_DONE) {
2051	verbose(env, "verifier BUG type %s var_off %lld off %d\n",	2091	verbose(env, "verifier BUG type %s var_off %lld off %d\n",
2052	reg_type_str[parent->type],	2092	reg_type_str(env, parent->type),
2053	parent->var_off.value, parent->off);	2093	parent->var_off.value, parent->off);
2054	return -EFAULT;	2094	return -EFAULT;
2055	}	2095	}
@@ -2233,6 +2273,8 @@ static int check_reg_arg(struct bpf_verifier_env *env, u32 regno, · 177 unmodified lines
2233	return -EINVAL;	2273	return -EINVAL;
2234	}	2274	}
2235		2275
		2276	mark_reg_scratched(env, regno);
		2277
2236	reg = &regs[regno];	2278	reg = &regs[regno];
2237	rw64 = is_reg64(env, insn, regno, reg, t);	2279	rw64 = is_reg64(env, insn, regno, reg, t);
2238	if (t == SRC_OP) {	2280	if (t == SRC_OP) {
@@ -2337,7 +2379,7 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, · 98 unmodified lines
2337		2379
2338	if (insn->code == 0)	2380	if (insn->code == 0)
2339	return 0;	2381	return 0;
2340	if (env->log.level & BPF_LOG_LEVEL) {	2382	if (env->log.level & BPF_LOG_LEVEL2) {
2341	verbose(env, "regs=%x stack=%llx before ", reg_mask, stack_mask);	2383	verbose(env, "regs=%x stack=%llx before ", reg_mask, stack_mask);
2342	verbose(env, "%d: ", idx);	2384	verbose(env, "%d: ", idx);
2343	print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);	2385	print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
@@ -2591,7 +2633,7 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int regno, · 247 unmodified lines
2591	DECLARE_BITMAP(mask, 64);	2633	DECLARE_BITMAP(mask, 64);
2592	u32 history = st->jmp_history_cnt;	2634	u32 history = st->jmp_history_cnt;
2593		2635
2594	if (env->log.level & BPF_LOG_LEVEL)	2636	if (env->log.level & BPF_LOG_LEVEL2)
2595	verbose(env, "last_idx %d first_idx %d\n", last_idx, first_idx);	2637	verbose(env, "last_idx %d first_idx %d\n", last_idx, first_idx);
2596	for (i = last_idx;;) {	2638	for (i = last_idx;;) {
2597	if (skip_first) {	2639	if (skip_first) {
@@ -2678,11 +2720,11 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int regno, · 80 unmodified lines
2678	new_marks = true;	2720	new_marks = true;
2679	reg->precise = true;	2721	reg->precise = true;
2680	}	2722	}
2681	if (env->log.level & BPF_LOG_LEVEL) {	2723	if (env->log.level & BPF_LOG_LEVEL2) {
2682	print_verifier_state(env, func);	2724	verbose(env, "parent %s regs=%x stack=%llx marks:",
2683	verbose(env, "parent %s regs=%x stack=%llx marks\n",
2684	new_marks ? "didn't have" : "already had",	2725	new_marks ? "didn't have" : "already had",
2685	reg_mask, stack_mask);	2726	reg_mask, stack_mask);
		2727	print_verifier_state(env, func, true);
2686	}	2728	}
2687		2729
2688	if (!reg_mask && !stack_mask)	2730	if (!reg_mask && !stack_mask)
@@ -2708,9 +2750,8 @@ static int mark_chain_precision_stack(struct bpf_verifier_env *env, int spi) · 19 unmodified lines
2708		2750
2709	static bool is_spillable_regtype(enum bpf_reg_type type)	2751	static bool is_spillable_regtype(enum bpf_reg_type type)
2710	{	2752	{
2711	switch (type) {	2753	switch (base_type(type)) {
2712	case PTR_TO_MAP_VALUE:	2754	case PTR_TO_MAP_VALUE:
2713	case PTR_TO_MAP_VALUE_OR_NULL:
2714	case PTR_TO_STACK:	2755	case PTR_TO_STACK:
2715	case PTR_TO_CTX:	2756	case PTR_TO_CTX:
2716	case PTR_TO_PACKET:	2757	case PTR_TO_PACKET:
@@ -2719,21 +2760,13 @@ static bool is_spillable_regtype(enum bpf_reg_type type) · 2 unmodified lines
2719	case PTR_TO_FLOW_KEYS:	2760	case PTR_TO_FLOW_KEYS:
2720	case CONST_PTR_TO_MAP:	2761	case CONST_PTR_TO_MAP:
2721	case PTR_TO_SOCKET:	2762	case PTR_TO_SOCKET:
2722	case PTR_TO_SOCKET_OR_NULL:
2723	case PTR_TO_SOCK_COMMON:	2763	case PTR_TO_SOCK_COMMON:
2724	case PTR_TO_SOCK_COMMON_OR_NULL:
2725	case PTR_TO_TCP_SOCK:	2764	case PTR_TO_TCP_SOCK:
2726	case PTR_TO_TCP_SOCK_OR_NULL:
2727	case PTR_TO_XDP_SOCK:	2765	case PTR_TO_XDP_SOCK:
2728	case PTR_TO_BTF_ID:	2766	case PTR_TO_BTF_ID:
2729	case PTR_TO_BTF_ID_OR_NULL:	2767	case PTR_TO_BUF:
2730	case PTR_TO_RDONLY_BUF:
2731	case PTR_TO_RDONLY_BUF_OR_NULL:
2732	case PTR_TO_RDWR_BUF:
2733	case PTR_TO_RDWR_BUF_OR_NULL:
2734	case PTR_TO_PERCPU_BTF_ID:	2768	case PTR_TO_PERCPU_BTF_ID:
2735	case PTR_TO_MEM:	2769	case PTR_TO_MEM:
2736	case PTR_TO_MEM_OR_NULL:
2737	case PTR_TO_FUNC:	2770	case PTR_TO_FUNC:
2738	case PTR_TO_MAP_KEY:	2771	case PTR_TO_MAP_KEY:
2739	return true;	2772	return true;
@@ -2838,6 +2871,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, · 98 unmodified lines
2838	env->insn_aux_data[insn_idx].sanitize_stack_spill = true;	2871	env->insn_aux_data[insn_idx].sanitize_stack_spill = true;
2839	}	2872	}
2840		2873
		2874	mark_stack_slot_scratched(env, spi);
2841	if (reg && !(off % BPF_REG_SIZE) && register_is_bounded(reg) &&	2875	if (reg && !(off % BPF_REG_SIZE) && register_is_bounded(reg) &&
2842	!register_is_null(reg) && env->bpf_capable) {	2876	!register_is_null(reg) && env->bpf_capable) {
2843	if (dst_reg != BPF_REG_FP) {	2877	if (dst_reg != BPF_REG_FP) {
@@ -2959,6 +2993,7 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env, · 115 unmodified lines
2959	slot = -i - 1;	2993	slot = -i - 1;
2960	spi = slot / BPF_REG_SIZE;	2994	spi = slot / BPF_REG_SIZE;
2961	stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE];	2995	stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE];
		2996	mark_stack_slot_scratched(env, spi);
2962		2997
2963	if (!env->allow_ptr_leaks	2998	if (!env->allow_ptr_leaks
2964	&& *stype != NOT_INIT	2999	&& *stype != NOT_INIT
@@ -3375,11 +3410,8 @@ static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno, · 410 unmodified lines
3375	/* We may have adjusted the register pointing to memory region, so we	3410	/* We may have adjusted the register pointing to memory region, so we
3376	* need to try adding each of min_value and max_value to off	3411	* need to try adding each of min_value and max_value to off
3377	* to make sure our theoretical access will be safe.	3412	* to make sure our theoretical access will be safe.
3378	*/	3413	*
3379	if (env->log.level & BPF_LOG_LEVEL)	3414	* The minimum value is only important with signed
3380	print_verifier_state(env, state);
3381
3382	/* The minimum value is only important with signed
3383	* comparisons where we can't assume the floor of a	3415	* comparisons where we can't assume the floor of a
3384	* value is 0. If we are using signed variables for our	3416	* value is 0. If we are using signed variables for our
3385	* index'es we need to make sure that whatever we use	3417	* index'es we need to make sure that whatever we use
@@ -3574,7 +3606,7 @@ static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, · 188 unmodified lines
3574	*/	3606	*/
3575	*reg_type = info.reg_type;	3607	*reg_type = info.reg_type;
3576		3608
3577	if (reg_type == PTR_TO_BTF_ID \|\| reg_type == PTR_TO_BTF_ID_OR_NULL) {	3609	if (base_type(*reg_type) == PTR_TO_BTF_ID) {
3578	*btf = info.btf;	3610	*btf = info.btf;
3579	*btf_id = info.btf_id;	3611	*btf_id = info.btf_id;
3580	} else {	3612	} else {
@@ -3642,7 +3674,7 @@ static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, · 61 unmodified lines
3642	}	3674	}
3643		3675
3644	verbose(env, "R%d invalid %s access off=%d size=%d\n",	3676	verbose(env, "R%d invalid %s access off=%d size=%d\n",
3645	regno, reg_type_str[reg->type], off, size);	3677	regno, reg_type_str(env, reg->type), off, size);
3646		3678
3647	return -EACCES;	3679	return -EACCES;
3648	}	3680	}
@@ -4369,15 +4401,30 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 720 unmodified lines
4369	mark_reg_unknown(env, regs, value_regno);	4401	mark_reg_unknown(env, regs, value_regno);
4370	}	4402	}
4371	}	4403	}
4372	} else if (reg->type == PTR_TO_MEM) {	4404	} else if (base_type(reg->type) == PTR_TO_MEM) {
		4405	bool rdonly_mem = type_is_rdonly_mem(reg->type);
		4406
		4407	if (type_may_be_null(reg->type)) {
		4408	verbose(env, "R%d invalid mem access '%s'\n", regno,
		4409	reg_type_str(env, reg->type));
		4410	return -EACCES;
		4411	}
		4412
		4413	if (t == BPF_WRITE && rdonly_mem) {
		4414	verbose(env, "R%d cannot write into %s\n",
		4415	regno, reg_type_str(env, reg->type));
		4416	return -EACCES;
		4417	}
		4418
4373	if (t == BPF_WRITE && value_regno >= 0 &&	4419	if (t == BPF_WRITE && value_regno >= 0 &&
4374	is_pointer_value(env, value_regno)) {	4420	is_pointer_value(env, value_regno)) {
4375	verbose(env, "R%d leaks addr into mem\n", value_regno);	4421	verbose(env, "R%d leaks addr into mem\n", value_regno);
4376	return -EACCES;	4422	return -EACCES;
4377	}	4423	}
		4424
4378	err = check_mem_region_access(env, regno, off, size,	4425	err = check_mem_region_access(env, regno, off, size,
4379	reg->mem_size, false);	4426	reg->mem_size, false);
4380	if (!err && t == BPF_READ && value_regno >= 0)	4427	if (!err && value_regno >= 0 && (t == BPF_READ \|\| rdonly_mem))
4381	mark_reg_unknown(env, regs, value_regno);	4428	mark_reg_unknown(env, regs, value_regno);
4382	} else if (reg->type == PTR_TO_CTX) {	4429	} else if (reg->type == PTR_TO_CTX) {
4383	enum bpf_reg_type reg_type = SCALAR_VALUE;	4430	enum bpf_reg_type reg_type = SCALAR_VALUE;
@@ -4407,7 +4454,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 23 unmodified lines
4407	} else {	4454	} else {
4408	mark_reg_known_zero(env, regs,	4455	mark_reg_known_zero(env, regs,
4409	value_regno);	4456	value_regno);
4410	if (reg_type_may_be_null(reg_type))	4457	if (type_may_be_null(reg_type))
4411	regs[value_regno].id = ++env->id_gen;	4458	regs[value_regno].id = ++env->id_gen;
4412	/* A load of ctx field could have different	4459	/* A load of ctx field could have different
4413	* actual load size with the one encoded in the	4460	* actual load size with the one encoded in the
@@ -4415,8 +4462,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 1 unmodified line
4415	* a sub-register.	4462	* a sub-register.
4416	*/	4463	*/
4417	regs[value_regno].subreg_def = DEF_NOT_SUBREG;	4464	regs[value_regno].subreg_def = DEF_NOT_SUBREG;
4418	if (reg_type == PTR_TO_BTF_ID \|\|	4465	if (base_type(reg_type) == PTR_TO_BTF_ID) {
4419	reg_type == PTR_TO_BTF_ID_OR_NULL) {
4420	regs[value_regno].btf = btf;	4466	regs[value_regno].btf = btf;
4421	regs[value_regno].btf_id = btf_id;	4467	regs[value_regno].btf_id = btf_id;
4422	}	4468	}
@@ -4469,7 +4515,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 46 unmodified lines
4469	} else if (type_is_sk_pointer(reg->type)) {	4515	} else if (type_is_sk_pointer(reg->type)) {
4470	if (t == BPF_WRITE) {	4516	if (t == BPF_WRITE) {
4471	verbose(env, "R%d cannot write into %s\n",	4517	verbose(env, "R%d cannot write into %s\n",
4472	regno, reg_type_str[reg->type]);	4518	regno, reg_type_str(env, reg->type));
4473	return -EACCES;	4519	return -EACCES;
4474	}	4520	}
4475	err = check_sock_access(env, insn_idx, regno, off, size, t);	4521	err = check_sock_access(env, insn_idx, regno, off, size, t);
@@ -4485,26 +4531,32 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 9 unmodified lines
4485	} else if (reg->type == CONST_PTR_TO_MAP) {	4531	} else if (reg->type == CONST_PTR_TO_MAP) {
4486	err = check_ptr_to_map_access(env, regs, regno, off, size, t,	4532	err = check_ptr_to_map_access(env, regs, regno, off, size, t,
4487	value_regno);	4533	value_regno);
4488	} else if (reg->type == PTR_TO_RDONLY_BUF) {	4534	} else if (base_type(reg->type) == PTR_TO_BUF) {
4489	if (t == BPF_WRITE) {	4535	bool rdonly_mem = type_is_rdonly_mem(reg->type);
4490	verbose(env, "R%d cannot write into %s\n",	4536	const char *buf_info;
4491	regno, reg_type_str[reg->type]);	4537	u32 *max_access;
4492	return -EACCES;	4538
		4539	if (rdonly_mem) {
		4540	if (t == BPF_WRITE) {
		4541	verbose(env, "R%d cannot write into %s\n",
		4542	regno, reg_type_str(env, reg->type));
		4543	return -EACCES;
		4544	}
		4545	buf_info = "rdonly";
		4546	max_access = &env->prog->aux->max_rdonly_access;
		4547	} else {
		4548	buf_info = "rdwr";
		4549	max_access = &env->prog->aux->max_rdwr_access;
4493	}	4550	}
		4551
4494	err = check_buffer_access(env, reg, regno, off, size, false,	4552	err = check_buffer_access(env, reg, regno, off, size, false,
4495	"rdonly",	4553	buf_info, max_access);
4496	&env->prog->aux->max_rdonly_access);	4554
4497	if (!err && value_regno >= 0)	4555	if (!err && value_regno >= 0 && (rdonly_mem \|\| t == BPF_READ))
4498	mark_reg_unknown(env, regs, value_regno);
4499	} else if (reg->type == PTR_TO_RDWR_BUF) {
4500	err = check_buffer_access(env, reg, regno, off, size, false,
4501	"rdwr",
4502	&env->prog->aux->max_rdwr_access);
4503	if (!err && t == BPF_READ && value_regno >= 0)
4504	mark_reg_unknown(env, regs, value_regno);	4556	mark_reg_unknown(env, regs, value_regno);
4505	} else {	4557	} else {
4506	verbose(env, "R%d invalid mem access '%s'\n", regno,	4558	verbose(env, "R%d invalid mem access '%s'\n", regno,
4507	reg_type_str[reg->type]);	4559	reg_type_str(env, reg->type));
4508	return -EACCES;	4560	return -EACCES;
4509	}	4561	}
4510		4562
@@ -4578,7 +4630,7 @@ static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_i · 67 unmodified lines
4578	is_sk_reg(env, insn->dst_reg)) {	4630	is_sk_reg(env, insn->dst_reg)) {
4579	verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n",	4631	verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n",
4580	insn->dst_reg,	4632	insn->dst_reg,
4581	reg_type_str[reg_state(env, insn->dst_reg)->type]);	4633	reg_type_str(env, reg_state(env, insn->dst_reg)->type));
4582	return -EACCES;	4634	return -EACCES;
4583	}	4635	}
4584		4636
@@ -4761,8 +4813,10 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, · 176 unmodified lines
4761	struct bpf_call_arg_meta *meta)	4813	struct bpf_call_arg_meta *meta)
4762	{	4814	{
4763	struct bpf_reg_state regs = cur_regs(env), reg = &regs[regno];	4815	struct bpf_reg_state regs = cur_regs(env), reg = &regs[regno];
		4816	const char *buf_info;
		4817	u32 *max_access;
4764		4818
4765	switch (reg->type) {	4819	switch (base_type(reg->type)) {
4766	case PTR_TO_PACKET:	4820	case PTR_TO_PACKET:
4767	case PTR_TO_PACKET_META:	4821	case PTR_TO_PACKET_META:
4768	return check_packet_access(env, regno, reg->off, access_size,	4822	return check_packet_access(env, regno, reg->off, access_size,
@@ -4781,18 +4835,20 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, · 12 unmodified lines
4781	return check_mem_region_access(env, regno, reg->off,	4835	return check_mem_region_access(env, regno, reg->off,
4782	access_size, reg->mem_size,	4836	access_size, reg->mem_size,
4783	zero_size_allowed);	4837	zero_size_allowed);
4784	case PTR_TO_RDONLY_BUF:	4838	case PTR_TO_BUF:
4785	if (meta && meta->raw_mode)	4839	if (type_is_rdonly_mem(reg->type)) {
4786	return -EACCES;	4840	if (meta && meta->raw_mode)
4787	return check_buffer_access(env, reg, regno, reg->off,	4841	return -EACCES;
4788	access_size, zero_size_allowed,	4842
4789	"rdonly",	4843	buf_info = "rdonly";
4790	&env->prog->aux->max_rdonly_access);	4844	max_access = &env->prog->aux->max_rdonly_access;
4791	case PTR_TO_RDWR_BUF:	4845	} else {
		4846	buf_info = "rdwr";
		4847	max_access = &env->prog->aux->max_rdwr_access;
		4848	}
4792	return check_buffer_access(env, reg, regno, reg->off,	4849	return check_buffer_access(env, reg, regno, reg->off,
4793	access_size, zero_size_allowed,	4850	access_size, zero_size_allowed,
4794	"rdwr",	4851	buf_info, max_access);
4795	&env->prog->aux->max_rdwr_access);
4796	case PTR_TO_STACK:	4852	case PTR_TO_STACK:
4797	return check_stack_range_initialized(	4853	return check_stack_range_initialized(
4798	env,	4854	env,
@@ -4804,9 +4860,9 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, · 5 unmodified lines
4804	register_is_null(reg))	4860	register_is_null(reg))
4805	return 0;	4861	return 0;
4806		4862
4807	verbose(env, "R%d type=%s expected=%s\n", regno,	4863	verbose(env, "R%d type=%s ", regno,
4808	reg_type_str[reg->type],	4864	reg_type_str(env, reg->type));
4809	reg_type_str[PTR_TO_STACK]);	4865	verbose(env, "expected=%s\n", reg_type_str(env, PTR_TO_STACK));
4810	return -EACCES;	4866	return -EACCES;
4811	}	4867	}
4812	}	4868	}
@@ -4817,7 +4873,7 @@ int check_mem_reg(struct bpf_verifier_env env, struct bpf_reg_state reg, · 4 unmodified lines
4817	if (register_is_null(reg))	4873	if (register_is_null(reg))
4818	return 0;	4874	return 0;
4819		4875
4820	if (reg_type_may_be_null(reg->type)) {	4876	if (type_may_be_null(reg->type)) {
4821	/* Assuming that the register contains a value check if the memory	4877	/* Assuming that the register contains a value check if the memory
4822	* access is safe. Temporarily save and restore the register's state as	4878	* access is safe. Temporarily save and restore the register's state as
4823	* the conversion shouldn't be visible to a caller.	4879	* the conversion shouldn't be visible to a caller.
@@ -4965,9 +5021,8 @@ static int process_timer_func(struct bpf_verifier_env *env, int regno, · 141 unmodified lines
4965		5021
4966	static bool arg_type_is_mem_ptr(enum bpf_arg_type type)	5022	static bool arg_type_is_mem_ptr(enum bpf_arg_type type)
4967	{	5023	{
4968	return type == ARG_PTR_TO_MEM \|\|	5024	return base_type(type) == ARG_PTR_TO_MEM \|\|
4969	type == ARG_PTR_TO_MEM_OR_NULL \|\|	5025	base_type(type) == ARG_PTR_TO_UNINIT_MEM;
4970	type == ARG_PTR_TO_UNINIT_MEM;
4971	}	5026	}
4972		5027
4973	static bool arg_type_is_mem_size(enum bpf_arg_type type)	5028	static bool arg_type_is_mem_size(enum bpf_arg_type type)
@@ -5072,8 +5127,7 @@ static const struct bpf_reg_types mem_types = { · 98 unmodified lines
5072	PTR_TO_MAP_KEY,	5127	PTR_TO_MAP_KEY,
5073	PTR_TO_MAP_VALUE,	5128	PTR_TO_MAP_VALUE,
5074	PTR_TO_MEM,	5129	PTR_TO_MEM,
5075	PTR_TO_RDONLY_BUF,	5130	PTR_TO_BUF,
5076	PTR_TO_RDWR_BUF,
5077	},	5131	},
5078	};	5132	};
5079		5133
@@ -5104,31 +5158,26 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { · 24 unmodified lines
5104	[ARG_PTR_TO_MAP_KEY] = &map_key_value_types,	5158	[ARG_PTR_TO_MAP_KEY] = &map_key_value_types,
5105	[ARG_PTR_TO_MAP_VALUE] = &map_key_value_types,	5159	[ARG_PTR_TO_MAP_VALUE] = &map_key_value_types,
5106	[ARG_PTR_TO_UNINIT_MAP_VALUE] = &map_key_value_types,	5160	[ARG_PTR_TO_UNINIT_MAP_VALUE] = &map_key_value_types,
5107	[ARG_PTR_TO_MAP_VALUE_OR_NULL] = &map_key_value_types,
5108	[ARG_CONST_SIZE] = &scalar_types,	5161	[ARG_CONST_SIZE] = &scalar_types,
5109	[ARG_CONST_SIZE_OR_ZERO] = &scalar_types,	5162	[ARG_CONST_SIZE_OR_ZERO] = &scalar_types,
5110	[ARG_CONST_ALLOC_SIZE_OR_ZERO] = &scalar_types,	5163	[ARG_CONST_ALLOC_SIZE_OR_ZERO] = &scalar_types,
5111	[ARG_CONST_MAP_PTR] = &const_map_ptr_types,	5164	[ARG_CONST_MAP_PTR] = &const_map_ptr_types,
5112	[ARG_PTR_TO_CTX] = &context_types,	5165	[ARG_PTR_TO_CTX] = &context_types,
5113	[ARG_PTR_TO_CTX_OR_NULL] = &context_types,
5114	[ARG_PTR_TO_SOCK_COMMON] = &sock_types,	5166	[ARG_PTR_TO_SOCK_COMMON] = &sock_types,
5115	#ifdef CONFIG_NET	5167	#ifdef CONFIG_NET
5116	[ARG_PTR_TO_BTF_ID_SOCK_COMMON] = &btf_id_sock_common_types,	5168	[ARG_PTR_TO_BTF_ID_SOCK_COMMON] = &btf_id_sock_common_types,
5117	#endif	5169	#endif
5118	[ARG_PTR_TO_SOCKET] = &fullsock_types,	5170	[ARG_PTR_TO_SOCKET] = &fullsock_types,
5119	[ARG_PTR_TO_SOCKET_OR_NULL] = &fullsock_types,
5120	[ARG_PTR_TO_BTF_ID] = &btf_ptr_types,	5171	[ARG_PTR_TO_BTF_ID] = &btf_ptr_types,
5121	[ARG_PTR_TO_SPIN_LOCK] = &spin_lock_types,	5172	[ARG_PTR_TO_SPIN_LOCK] = &spin_lock_types,
5122	[ARG_PTR_TO_MEM] = &mem_types,	5173	[ARG_PTR_TO_MEM] = &mem_types,
5123	[ARG_PTR_TO_MEM_OR_NULL] = &mem_types,
5124	[ARG_PTR_TO_UNINIT_MEM] = &mem_types,	5174	[ARG_PTR_TO_UNINIT_MEM] = &mem_types,
5125	[ARG_PTR_TO_ALLOC_MEM] = &alloc_mem_types,	5175	[ARG_PTR_TO_ALLOC_MEM] = &alloc_mem_types,
5126	[ARG_PTR_TO_ALLOC_MEM_OR_NULL] = &alloc_mem_types,
5127	[ARG_PTR_TO_INT] = &int_ptr_types,	5176	[ARG_PTR_TO_INT] = &int_ptr_types,
5128	[ARG_PTR_TO_LONG] = &int_ptr_types,	5177	[ARG_PTR_TO_LONG] = &int_ptr_types,
5129	[ARG_PTR_TO_PERCPU_BTF_ID] = &percpu_btf_ptr_types,	5178	[ARG_PTR_TO_PERCPU_BTF_ID] = &percpu_btf_ptr_types,
5130	[ARG_PTR_TO_FUNC] = &func_ptr_types,	5179	[ARG_PTR_TO_FUNC] = &func_ptr_types,
5131	[ARG_PTR_TO_STACK_OR_NULL] = &stack_ptr_types,	5180	[ARG_PTR_TO_STACK] = &stack_ptr_types,
5132	[ARG_PTR_TO_CONST_STR] = &const_str_ptr_types,	5181	[ARG_PTR_TO_CONST_STR] = &const_str_ptr_types,
5133	[ARG_PTR_TO_TIMER] = &timer_types,	5182	[ARG_PTR_TO_TIMER] = &timer_types,
5134	};	5183	};
@@ -5142,12 +5191,27 @@ static int check_reg_type(struct bpf_verifier_env *env, u32 regno, · 7 unmodified lines
5142	const struct bpf_reg_types *compatible;	5191	const struct bpf_reg_types *compatible;
5143	int i, j;	5192	int i, j;
5144		5193
5145	compatible = compatible_reg_types[arg_type];	5194	compatible = compatible_reg_types[base_type(arg_type)];
5146	if (!compatible) {	5195	if (!compatible) {
5147	verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type);	5196	verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type);
5148	return -EFAULT;	5197	return -EFAULT;
5149	}	5198	}
5150		5199
		5200	/* ARG_PTR_TO_MEM + RDONLY is compatible with PTR_TO_MEM and PTR_TO_MEM + RDONLY,
		5201	* but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM and NOT with PTR_TO_MEM + RDONLY
		5202	*
		5203	* Same for MAYBE_NULL:
		5204	*
		5205	* ARG_PTR_TO_MEM + MAYBE_NULL is compatible with PTR_TO_MEM and PTR_TO_MEM + MAYBE_NULL,
		5206	* but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM but NOT with PTR_TO_MEM + MAYBE_NULL
		5207	*
		5208	* Therefore we fold these flags depending on the arg_type before comparison.
		5209	*/
		5210	if (arg_type & MEM_RDONLY)
		5211	type &= ~MEM_RDONLY;
		5212	if (arg_type & PTR_MAYBE_NULL)
		5213	type &= ~PTR_MAYBE_NULL;
		5214
5151	for (i = 0; i < ARRAY_SIZE(compatible->types); i++) {	5215	for (i = 0; i < ARRAY_SIZE(compatible->types); i++) {
5152	expected = compatible->types[i];	5216	expected = compatible->types[i];
5153	if (expected == NOT_INIT)	5217	if (expected == NOT_INIT)
@@ -5157,14 +5221,14 @@ static int check_reg_type(struct bpf_verifier_env *env, u32 regno, · 3 unmodified lines
5157	goto found;	5221	goto found;
5158	}	5222	}
5159		5223
5160	verbose(env, "R%d type=%s expected=", regno, reg_type_str[type]);	5224	verbose(env, "R%d type=%s expected=", regno, reg_type_str(env, reg->type));
5161	for (j = 0; j + 1 < i; j++)	5225	for (j = 0; j + 1 < i; j++)
5162	verbose(env, "%s, ", reg_type_str[compatible->types[j]]);	5226	verbose(env, "%s, ", reg_type_str(env, compatible->types[j]));
5163	verbose(env, "%s\n", reg_type_str[compatible->types[j]]);	5227	verbose(env, "%s\n", reg_type_str(env, compatible->types[j]));
5164	return -EACCES;	5228	return -EACCES;
5165		5229
5166	found:	5230	found:
5167	if (type == PTR_TO_BTF_ID) {	5231	if (reg->type == PTR_TO_BTF_ID) {
5168	if (!arg_btf_id) {	5232	if (!arg_btf_id) {
5169	if (!compatible->btf_id) {	5233	if (!compatible->btf_id) {
5170	verbose(env, "verifier internal error: missing arg compatible BTF ID\n");	5234	verbose(env, "verifier internal error: missing arg compatible BTF ID\n");
@@ -5223,15 +5287,14 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, · 52 unmodified lines
5223	return -EACCES;	5287	return -EACCES;
5224	}	5288	}
5225		5289
5226	if (arg_type == ARG_PTR_TO_MAP_VALUE \|\|	5290	if (base_type(arg_type) == ARG_PTR_TO_MAP_VALUE \|\|
5227	arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE \|\|	5291	base_type(arg_type) == ARG_PTR_TO_UNINIT_MAP_VALUE) {
5228	arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL) {
5229	err = resolve_map_arg_type(env, meta, &arg_type);	5292	err = resolve_map_arg_type(env, meta, &arg_type);
5230	if (err)	5293	if (err)
5231	return err;	5294	return err;
5232	}	5295	}
5233		5296
5234	if (register_is_null(reg) && arg_type_may_be_null(arg_type))	5297	if (register_is_null(reg) && type_may_be_null(arg_type))
5235	/* A NULL register has a SCALAR_VALUE type, so skip	5298	/* A NULL register has a SCALAR_VALUE type, so skip
5236	* type checking.	5299	* type checking.
5237	*/	5300	*/
@@ -5300,10 +5363,11 @@ skip_type_check: · 62 unmodified lines
5300	err = check_helper_mem_access(env, regno,	5363	err = check_helper_mem_access(env, regno,
5301	meta->map_ptr->key_size, false,	5364	meta->map_ptr->key_size, false,
5302	NULL);	5365	NULL);
5303	} else if (arg_type == ARG_PTR_TO_MAP_VALUE \|\|	5366	} else if (base_type(arg_type) == ARG_PTR_TO_MAP_VALUE \|\|
5304	(arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL &&	5367	base_type(arg_type) == ARG_PTR_TO_UNINIT_MAP_VALUE) {
5305	!register_is_null(reg)) \|\|	5368	if (type_may_be_null(arg_type) && register_is_null(reg))
5306	arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE) {	5369	return 0;
		5370
5307	/* bpf_map_xxx(..., map_ptr, ..., value) call:	5371	/* bpf_map_xxx(..., map_ptr, ..., value) call:
5308	* check [value, value + map->value_size) validity	5372	* check [value, value + map->value_size) validity
5309	*/	5373	*/
@@ -6025,9 +6089,9 @@ static int __check_func_call(struct bpf_verifier_env env, struct bpf_insn insn · 715 unmodified lines
6025		6089
6026	if (env->log.level & BPF_LOG_LEVEL) {	6090	if (env->log.level & BPF_LOG_LEVEL) {
6027	verbose(env, "caller:\n");	6091	verbose(env, "caller:\n");
6028	print_verifier_state(env, caller);	6092	print_verifier_state(env, caller, true);
6029	verbose(env, "callee:\n");	6093	verbose(env, "callee:\n");
6030	print_verifier_state(env, callee);	6094	print_verifier_state(env, callee, true);
6031	}	6095	}
6032	return 0;	6096	return 0;
6033	}	6097	}
@@ -6242,9 +6306,9 @@ static int prepare_func_exit(struct bpf_verifier_env env, int insn_idx) · 208 unmodified lines
6242	*insn_idx = callee->callsite + 1;	6306	*insn_idx = callee->callsite + 1;
6243	if (env->log.level & BPF_LOG_LEVEL) {	6307	if (env->log.level & BPF_LOG_LEVEL) {
6244	verbose(env, "returning from callee:\n");	6308	verbose(env, "returning from callee:\n");
6245	print_verifier_state(env, callee);	6309	print_verifier_state(env, callee, true);
6246	verbose(env, "to caller at %d:\n", *insn_idx);	6310	verbose(env, "to caller at %d:\n", *insn_idx);
6247	print_verifier_state(env, caller);	6311	print_verifier_state(env, caller, true);
6248	}	6312	}
6249	/* clear everything in the callee */	6313	/* clear everything in the callee */
6250	free_func_state(callee);	6314	free_func_state(callee);
@@ -6410,13 +6474,11 @@ static int check_bpf_snprintf_call(struct bpf_verifier_env *env, · 159 unmodified lines
6410		6474
6411	static int check_get_func_ip(struct bpf_verifier_env *env)	6475	static int check_get_func_ip(struct bpf_verifier_env *env)
6412	{	6476	{
6413	enum bpf_attach_type eatype = env->prog->expected_attach_type;
6414	enum bpf_prog_type type = resolve_prog_type(env->prog);	6477	enum bpf_prog_type type = resolve_prog_type(env->prog);
6415	int func_id = BPF_FUNC_get_func_ip;	6478	int func_id = BPF_FUNC_get_func_ip;
6416		6479
6417	if (type == BPF_PROG_TYPE_TRACING) {	6480	if (type == BPF_PROG_TYPE_TRACING) {
6418	if (eatype != BPF_TRACE_FENTRY && eatype != BPF_TRACE_FEXIT &&	6481	if (!bpf_prog_has_trampoline(env->prog)) {
6419	eatype != BPF_MODIFY_RETURN) {
6420	verbose(env, "func %s#%d supported only for fentry/fexit/fmod_ret programs\n",	6482	verbose(env, "func %s#%d supported only for fentry/fexit/fmod_ret programs\n",
6421	func_id_name(func_id), func_id);	6483	func_id_name(func_id), func_id);
6422	return -ENOTSUPP;	6484	return -ENOTSUPP;
@@ -6435,6 +6497,8 @@ static int check_helper_call(struct bpf_verifier_env env, struct bpf_insn insn · 12 unmodified lines
6435	int *insn_idx_p)	6497	int *insn_idx_p)
6436	{	6498	{
6437	const struct bpf_func_proto *fn = NULL;	6499	const struct bpf_func_proto *fn = NULL;
		6500	enum bpf_return_type ret_type;
		6501	enum bpf_type_flag ret_flag;
6438	struct bpf_reg_state *regs;	6502	struct bpf_reg_state *regs;
6439	struct bpf_call_arg_meta meta;	6503	struct bpf_call_arg_meta meta;
6440	int insn_idx = *insn_idx_p;	6504	int insn_idx = *insn_idx_p;
@@ -6574,13 +6638,14 @@ static int check_helper_call(struct bpf_verifier_env env, struct bpf_insn insn · 133 unmodified lines
6574	regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;	6638	regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;
6575		6639
6576	/* update return register (already marked as written above) */	6640	/* update return register (already marked as written above) */
6577	if (fn->ret_type == RET_INTEGER) {	6641	ret_type = fn->ret_type;
		6642	ret_flag = type_flag(fn->ret_type);
		6643	if (ret_type == RET_INTEGER) {
6578	/* sets type to SCALAR_VALUE */	6644	/* sets type to SCALAR_VALUE */
6579	mark_reg_unknown(env, regs, BPF_REG_0);	6645	mark_reg_unknown(env, regs, BPF_REG_0);
6580	} else if (fn->ret_type == RET_VOID) {	6646	} else if (ret_type == RET_VOID) {
6581	regs[BPF_REG_0].type = NOT_INIT;	6647	regs[BPF_REG_0].type = NOT_INIT;
6582	} else if (fn->ret_type == RET_PTR_TO_MAP_VALUE_OR_NULL \|\|	6648	} else if (base_type(ret_type) == RET_PTR_TO_MAP_VALUE) {
6583	fn->ret_type == RET_PTR_TO_MAP_VALUE) {
6584	/* There is no offset yet applied, variable or fixed */	6649	/* There is no offset yet applied, variable or fixed */
6585	mark_reg_known_zero(env, regs, BPF_REG_0);	6650	mark_reg_known_zero(env, regs, BPF_REG_0);
6586	/* remember map_ptr, so that check_map_access()	6651	/* remember map_ptr, so that check_map_access()
@@ -6594,28 +6659,25 @@ static int check_helper_call(struct bpf_verifier_env env, struct bpf_insn insn · 7 unmodified lines
6594	}	6659	}
6595	regs[BPF_REG_0].map_ptr = meta.map_ptr;	6660	regs[BPF_REG_0].map_ptr = meta.map_ptr;
6596	regs[BPF_REG_0].map_uid = meta.map_uid;	6661	regs[BPF_REG_0].map_uid = meta.map_uid;
6597	if (fn->ret_type == RET_PTR_TO_MAP_VALUE) {	6662	regs[BPF_REG_0].type = PTR_TO_MAP_VALUE \| ret_flag;
6598	regs[BPF_REG_0].type = PTR_TO_MAP_VALUE;	6663	if (!type_may_be_null(ret_type) &&
6599	if (map_value_has_spin_lock(meta.map_ptr))	6664	map_value_has_spin_lock(meta.map_ptr)) {
6600	regs[BPF_REG_0].id = ++env->id_gen;	6665	regs[BPF_REG_0].id = ++env->id_gen;
6601	} else {
6602	regs[BPF_REG_0].type = PTR_TO_MAP_VALUE_OR_NULL;
6603	}	6666	}
6604	} else if (fn->ret_type == RET_PTR_TO_SOCKET_OR_NULL) {	6667	} else if (base_type(ret_type) == RET_PTR_TO_SOCKET) {
6605	mark_reg_known_zero(env, regs, BPF_REG_0);	6668	mark_reg_known_zero(env, regs, BPF_REG_0);
6606	regs[BPF_REG_0].type = PTR_TO_SOCKET_OR_NULL;	6669	regs[BPF_REG_0].type = PTR_TO_SOCKET \| ret_flag;
6607	} else if (fn->ret_type == RET_PTR_TO_SOCK_COMMON_OR_NULL) {	6670	} else if (base_type(ret_type) == RET_PTR_TO_SOCK_COMMON) {
6608	mark_reg_known_zero(env, regs, BPF_REG_0);	6671	mark_reg_known_zero(env, regs, BPF_REG_0);
6609	regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON_OR_NULL;	6672	regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON \| ret_flag;
6610	} else if (fn->ret_type == RET_PTR_TO_TCP_SOCK_OR_NULL) {	6673	} else if (base_type(ret_type) == RET_PTR_TO_TCP_SOCK) {
6611	mark_reg_known_zero(env, regs, BPF_REG_0);	6674	mark_reg_known_zero(env, regs, BPF_REG_0);
6612	regs[BPF_REG_0].type = PTR_TO_TCP_SOCK_OR_NULL;	6675	regs[BPF_REG_0].type = PTR_TO_TCP_SOCK \| ret_flag;
6613	} else if (fn->ret_type == RET_PTR_TO_ALLOC_MEM_OR_NULL) {	6676	} else if (base_type(ret_type) == RET_PTR_TO_ALLOC_MEM) {
6614	mark_reg_known_zero(env, regs, BPF_REG_0);	6677	mark_reg_known_zero(env, regs, BPF_REG_0);
6615	regs[BPF_REG_0].type = PTR_TO_MEM_OR_NULL;	6678	regs[BPF_REG_0].type = PTR_TO_MEM \| ret_flag;
6616	regs[BPF_REG_0].mem_size = meta.mem_size;	6679	regs[BPF_REG_0].mem_size = meta.mem_size;
6617	} else if (fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL \|\|	6680	} else if (base_type(ret_type) == RET_PTR_TO_MEM_OR_BTF_ID) {
6618	fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID) {
6619	const struct btf_type *t;	6681	const struct btf_type *t;
6620		6682
6621	mark_reg_known_zero(env, regs, BPF_REG_0);	6683	mark_reg_known_zero(env, regs, BPF_REG_0);
@@ -6633,29 +6695,30 @@ static int check_helper_call(struct bpf_verifier_env env, struct bpf_insn insn · 11 unmodified lines
6633	tname, PTR_ERR(ret));	6695	tname, PTR_ERR(ret));
6634	return -EINVAL;	6696	return -EINVAL;
6635	}	6697	}
6636	regs[BPF_REG_0].type =	6698	regs[BPF_REG_0].type = PTR_TO_MEM \| ret_flag;
6637	fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID ?
6638	PTR_TO_MEM : PTR_TO_MEM_OR_NULL;
6639	regs[BPF_REG_0].mem_size = tsize;	6699	regs[BPF_REG_0].mem_size = tsize;
6640	} else {	6700	} else {
6641	regs[BPF_REG_0].type =	6701	/* MEM_RDONLY may be carried from ret_flag, but it
6642	fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID ?	6702	* doesn't apply on PTR_TO_BTF_ID. Fold it, otherwise
6643	PTR_TO_BTF_ID : PTR_TO_BTF_ID_OR_NULL;	6703	* it will confuse the check of PTR_TO_BTF_ID in
		6704	* check_mem_access().
		6705	*/
		6706	ret_flag &= ~MEM_RDONLY;
		6707
		6708	regs[BPF_REG_0].type = PTR_TO_BTF_ID \| ret_flag;
6644	regs[BPF_REG_0].btf = meta.ret_btf;	6709	regs[BPF_REG_0].btf = meta.ret_btf;
6645	regs[BPF_REG_0].btf_id = meta.ret_btf_id;	6710	regs[BPF_REG_0].btf_id = meta.ret_btf_id;
6646	}	6711	}
6647	} else if (fn->ret_type == RET_PTR_TO_BTF_ID_OR_NULL \|\|	6712	} else if (base_type(ret_type) == RET_PTR_TO_BTF_ID) {
6648	fn->ret_type == RET_PTR_TO_BTF_ID) {
6649	int ret_btf_id;	6713	int ret_btf_id;
6650		6714
6651	mark_reg_known_zero(env, regs, BPF_REG_0);	6715	mark_reg_known_zero(env, regs, BPF_REG_0);
6652	regs[BPF_REG_0].type = fn->ret_type == RET_PTR_TO_BTF_ID ?	6716	regs[BPF_REG_0].type = PTR_TO_BTF_ID \| ret_flag;
6653	PTR_TO_BTF_ID :
6654	PTR_TO_BTF_ID_OR_NULL;
6655	ret_btf_id = *fn->ret_btf_id;	6717	ret_btf_id = *fn->ret_btf_id;
6656	if (ret_btf_id == 0) {	6718	if (ret_btf_id == 0) {
6657	verbose(env, "invalid return type %d of func %s#%d\n",	6719	verbose(env, "invalid return type %u of func %s#%d\n",
6658	fn->ret_type, func_id_name(func_id), func_id);	6720	base_type(ret_type), func_id_name(func_id),
		6721	func_id);
6659	return -EINVAL;	6722	return -EINVAL;
6660	}	6723	}
6661	/* current BPF helper definitions are only coming from	6724	/* current BPF helper definitions are only coming from
@@ -6664,12 +6727,12 @@ static int check_helper_call(struct bpf_verifier_env env, struct bpf_insn insn · 2 unmodified lines
6664	regs[BPF_REG_0].btf = btf_vmlinux;	6727	regs[BPF_REG_0].btf = btf_vmlinux;
6665	regs[BPF_REG_0].btf_id = ret_btf_id;	6728	regs[BPF_REG_0].btf_id = ret_btf_id;
6666	} else {	6729	} else {
6667	verbose(env, "unknown return type %d of func %s#%d\n",	6730	verbose(env, "unknown return type %u of func %s#%d\n",
6668	fn->ret_type, func_id_name(func_id), func_id);	6731	base_type(ret_type), func_id_name(func_id), func_id);
6669	return -EINVAL;	6732	return -EINVAL;
6670	}	6733	}
6671		6734
6672	if (reg_type_may_be_null(regs[BPF_REG_0].type))	6735	if (type_may_be_null(regs[BPF_REG_0].type))
6673	regs[BPF_REG_0].id = ++env->id_gen;	6736	regs[BPF_REG_0].id = ++env->id_gen;
6674		6737
6675	if (is_ptr_cast_function(func_id)) {	6738	if (is_ptr_cast_function(func_id)) {
@@ -6878,25 +6941,25 @@ static bool check_reg_sane_offset(struct bpf_verifier_env *env, · 202 unmodified lines
6878		6941
6879	if (known && (val >= BPF_MAX_VAR_OFF \|\| val <= -BPF_MAX_VAR_OFF)) {	6942	if (known && (val >= BPF_MAX_VAR_OFF \|\| val <= -BPF_MAX_VAR_OFF)) {
6880	verbose(env, "math between %s pointer and %lld is not allowed\n",	6943	verbose(env, "math between %s pointer and %lld is not allowed\n",
6881	reg_type_str[type], val);	6944	reg_type_str(env, type), val);
6882	return false;	6945	return false;
6883	}	6946	}
6884		6947
6885	if (reg->off >= BPF_MAX_VAR_OFF \|\| reg->off <= -BPF_MAX_VAR_OFF) {	6948	if (reg->off >= BPF_MAX_VAR_OFF \|\| reg->off <= -BPF_MAX_VAR_OFF) {
6886	verbose(env, "%s pointer offset %d is not allowed\n",	6949	verbose(env, "%s pointer offset %d is not allowed\n",
6887	reg_type_str[type], reg->off);	6950	reg_type_str(env, type), reg->off);
6888	return false;	6951	return false;
6889	}	6952	}
6890		6953
6891	if (smin == S64_MIN) {	6954	if (smin == S64_MIN) {
6892	verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n",	6955	verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n",
6893	reg_type_str[type]);	6956	reg_type_str(env, type));
6894	return false;	6957	return false;
6895	}	6958	}
6896		6959
6897	if (smin >= BPF_MAX_VAR_OFF \|\| smin <= -BPF_MAX_VAR_OFF) {	6960	if (smin >= BPF_MAX_VAR_OFF \|\| smin <= -BPF_MAX_VAR_OFF) {
6898	verbose(env, "value %lld makes %s pointer be out of bounds\n",	6961	verbose(env, "value %lld makes %s pointer be out of bounds\n",
6899	smin, reg_type_str[type]);	6962	smin, reg_type_str(env, type));
6900	return false;	6963	return false;
6901	}	6964	}
6902		6965
@@ -7273,11 +7336,13 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, · 370 unmodified lines
7273	return -EACCES;	7336	return -EACCES;
7274	}	7337	}
7275		7338
7276	switch (ptr_reg->type) {	7339	if (ptr_reg->type & PTR_MAYBE_NULL) {
7277	case PTR_TO_MAP_VALUE_OR_NULL:
7278	verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n",	7340	verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n",
7279	dst, reg_type_str[ptr_reg->type]);	7341	dst, reg_type_str(env, ptr_reg->type));
7280	return -EACCES;	7342	return -EACCES;
		7343	}
		7344
		7345	switch (base_type(ptr_reg->type)) {
7281	case CONST_PTR_TO_MAP:	7346	case CONST_PTR_TO_MAP:
7282	/* smin_val represents the known value */	7347	/* smin_val represents the known value */
7283	if (known && smin_val == 0 && opcode == BPF_ADD)	7348	if (known && smin_val == 0 && opcode == BPF_ADD)
@@ -7285,14 +7350,11 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, · 1 unmodified line
7285	fallthrough;	7350	fallthrough;
7286	case PTR_TO_PACKET_END:	7351	case PTR_TO_PACKET_END:
7287	case PTR_TO_SOCKET:	7352	case PTR_TO_SOCKET:
7288	case PTR_TO_SOCKET_OR_NULL:
7289	case PTR_TO_SOCK_COMMON:	7353	case PTR_TO_SOCK_COMMON:
7290	case PTR_TO_SOCK_COMMON_OR_NULL:
7291	case PTR_TO_TCP_SOCK:	7354	case PTR_TO_TCP_SOCK:
7292	case PTR_TO_TCP_SOCK_OR_NULL:
7293	case PTR_TO_XDP_SOCK:	7355	case PTR_TO_XDP_SOCK:
7294	verbose(env, "R%d pointer arithmetic on %s prohibited\n",	7356	verbose(env, "R%d pointer arithmetic on %s prohibited\n",
7295	dst, reg_type_str[ptr_reg->type]);	7357	dst, reg_type_str(env, ptr_reg->type));
7296	return -EACCES;	7358	return -EACCES;
7297	default:	7359	default:
7298	break;	7360	break;
@@ -8265,12 +8327,12 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, · 966 unmodified lines
8265		8327
8266	/* Got here implies adding two SCALAR_VALUEs */	8328	/* Got here implies adding two SCALAR_VALUEs */
8267	if (WARN_ON_ONCE(ptr_reg)) {	8329	if (WARN_ON_ONCE(ptr_reg)) {
8268	print_verifier_state(env, state);	8330	print_verifier_state(env, state, true);
8269	verbose(env, "verifier internal error: unexpected ptr_reg\n");	8331	verbose(env, "verifier internal error: unexpected ptr_reg\n");
8270	return -EINVAL;	8332	return -EINVAL;
8271	}	8333	}
8272	if (WARN_ON(!src_reg)) {	8334	if (WARN_ON(!src_reg)) {
8273	print_verifier_state(env, state);	8335	print_verifier_state(env, state, true);
8274	verbose(env, "verifier internal error: no src_reg\n");	8336	verbose(env, "verifier internal error: no src_reg\n");
8275	return -EINVAL;	8337	return -EINVAL;
8276	}	8338	}
@@ -9015,7 +9077,7 @@ static void mark_ptr_or_null_reg(struct bpf_func_state *state, · 738 unmodified lines
9015	struct bpf_reg_state *reg, u32 id,	9077	struct bpf_reg_state *reg, u32 id,
9016	bool is_null)	9078	bool is_null)
9017	{	9079	{
9018	if (reg_type_may_be_null(reg->type) && reg->id == id &&	9080	if (type_may_be_null(reg->type) && reg->id == id &&
9019	!WARN_ON_ONCE(!reg->id)) {	9081	!WARN_ON_ONCE(!reg->id)) {
9020	/* Old offset (both fixed and variable parts) should	9082	/* Old offset (both fixed and variable parts) should
9021	* have been known-zero, because we don't allow pointer	9083	* have been known-zero, because we don't allow pointer
@@ -9393,7 +9455,7 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, · 371 unmodified lines
9393	*/	9455	*/
9394	if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K &&	9456	if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K &&
9395	insn->imm == 0 && (opcode == BPF_JEQ \|\| opcode == BPF_JNE) &&	9457	insn->imm == 0 && (opcode == BPF_JEQ \|\| opcode == BPF_JNE) &&
9396	reg_type_may_be_null(dst_reg->type)) {	9458	type_may_be_null(dst_reg->type)) {
9397	/* Mark all identical registers in each branch as either	9459	/* Mark all identical registers in each branch as either
9398	* safe or unknown depending R == 0 or R != 0 conditional.	9460	* safe or unknown depending R == 0 or R != 0 conditional.
9399	*/	9461	*/
@@ -9409,7 +9471,7 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, · 9 unmodified lines
9409	return -EACCES;	9471	return -EACCES;
9410	}	9472	}
9411	if (env->log.level & BPF_LOG_LEVEL)	9473	if (env->log.level & BPF_LOG_LEVEL)
9412	print_verifier_state(env, this_branch->frame[this_branch->curframe]);	9474	print_insn_state(env, this_branch->frame[this_branch->curframe]);
9413	return 0;	9475	return 0;
9414	}	9476	}
9415		9477
@@ -9448,7 +9510,7 @@ static int check_ld_imm(struct bpf_verifier_env env, struct bpf_insn insn) · 32 unmodified lines
9448	mark_reg_known_zero(env, regs, insn->dst_reg);	9510	mark_reg_known_zero(env, regs, insn->dst_reg);
9449		9511
9450	dst_reg->type = aux->btf_var.reg_type;	9512	dst_reg->type = aux->btf_var.reg_type;
9451	switch (dst_reg->type) {	9513	switch (base_type(dst_reg->type)) {
9452	case PTR_TO_MEM:	9514	case PTR_TO_MEM:
9453	dst_reg->mem_size = aux->btf_var.mem_size;	9515	dst_reg->mem_size = aux->btf_var.mem_size;
9454	break;	9516	break;
@@ -9647,7 +9709,7 @@ static int check_return_code(struct bpf_verifier_env *env) · 192 unmodified lines
9647	/* enforce return zero from async callbacks like timer */	9709	/* enforce return zero from async callbacks like timer */
9648	if (reg->type != SCALAR_VALUE) {	9710	if (reg->type != SCALAR_VALUE) {
9649	verbose(env, "In async callback the register R0 is not a known value (%s)\n",	9711	verbose(env, "In async callback the register R0 is not a known value (%s)\n",
9650	reg_type_str[reg->type]);	9712	reg_type_str(env, reg->type));
9651	return -EINVAL;	9713	return -EINVAL;
9652	}	9714	}
9653		9715
@@ -9661,7 +9723,7 @@ static int check_return_code(struct bpf_verifier_env *env) · 7 unmodified lines
9661	if (is_subprog) {	9723	if (is_subprog) {
9662	if (reg->type != SCALAR_VALUE) {	9724	if (reg->type != SCALAR_VALUE) {
9663	verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n",	9725	verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n",
9664	reg_type_str[reg->type]);	9726	reg_type_str(env, reg->type));
9665	return -EINVAL;	9727	return -EINVAL;
9666	}	9728	}
9667	return 0;	9729	return 0;
@@ -9725,7 +9787,7 @@ static int check_return_code(struct bpf_verifier_env *env) · 57 unmodified lines
9725		9787
9726	if (reg->type != SCALAR_VALUE) {	9788	if (reg->type != SCALAR_VALUE) {
9727	verbose(env, "At program exit the register R0 is not a known value (%s)\n",	9789	verbose(env, "At program exit the register R0 is not a known value (%s)\n",
9728	reg_type_str[reg->type]);	9790	reg_type_str(env, reg->type));
9729	return -EINVAL;	9791	return -EINVAL;
9730	}	9792	}
9731		9793
@@ -10582,7 +10644,7 @@ static bool regsafe(struct bpf_verifier_env env, struct bpf_reg_state rold, · 850 unmodified lines
10582	return true;	10644	return true;
10583	if (rcur->type == NOT_INIT)	10645	if (rcur->type == NOT_INIT)
10584	return false;	10646	return false;
10585	switch (rold->type) {	10647	switch (base_type(rold->type)) {
10586	case SCALAR_VALUE:	10648	case SCALAR_VALUE:
10587	if (env->explore_alu_limits)	10649	if (env->explore_alu_limits)
10588	return false;	10650	return false;
@@ -10604,6 +10666,22 @@ static bool regsafe(struct bpf_verifier_env env, struct bpf_reg_state rold, · 15 unmodified lines
10604	}	10666	}
10605	case PTR_TO_MAP_KEY:	10667	case PTR_TO_MAP_KEY:
10606	case PTR_TO_MAP_VALUE:	10668	case PTR_TO_MAP_VALUE:
		10669	/* a PTR_TO_MAP_VALUE could be safe to use as a
		10670	* PTR_TO_MAP_VALUE_OR_NULL into the same map.
		10671	* However, if the old PTR_TO_MAP_VALUE_OR_NULL then got NULL-
		10672	* checked, doing so could have affected others with the same
		10673	* id, and we can't check for that because we lost the id when
		10674	* we converted to a PTR_TO_MAP_VALUE.
		10675	*/
		10676	if (type_may_be_null(rold->type)) {
		10677	if (!type_may_be_null(rcur->type))
		10678	return false;
		10679	if (memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)))
		10680	return false;
		10681	/* Check our ids match any regs they're supposed to */
		10682	return check_ids(rold->id, rcur->id, idmap);
		10683	}
		10684
10607	/* If the new min/max/var_off satisfy the old ones and	10685	/* If the new min/max/var_off satisfy the old ones and
10608	* everything else matches, we are OK.	10686	* everything else matches, we are OK.
10609	* 'id' is not compared, since it's only used for maps with	10687	* 'id' is not compared, since it's only used for maps with
@@ -10615,20 +10693,6 @@ static bool regsafe(struct bpf_verifier_env env, struct bpf_reg_state rold, · 5 unmodified lines
10615	return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 &&	10693	return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 &&
10616	range_within(rold, rcur) &&	10694	range_within(rold, rcur) &&
10617	tnum_in(rold->var_off, rcur->var_off);	10695	tnum_in(rold->var_off, rcur->var_off);
10618	case PTR_TO_MAP_VALUE_OR_NULL:
10619	/* a PTR_TO_MAP_VALUE could be safe to use as a
10620	* PTR_TO_MAP_VALUE_OR_NULL into the same map.
10621	* However, if the old PTR_TO_MAP_VALUE_OR_NULL then got NULL-
10622	* checked, doing so could have affected others with the same
10623	* id, and we can't check for that because we lost the id when
10624	* we converted to a PTR_TO_MAP_VALUE.
10625	*/
10626	if (rcur->type != PTR_TO_MAP_VALUE_OR_NULL)
10627	return false;
10628	if (memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)))
10629	return false;
10630	/* Check our ids match any regs they're supposed to */
10631	return check_ids(rold->id, rcur->id, idmap);
10632	case PTR_TO_PACKET_META:	10696	case PTR_TO_PACKET_META:
10633	case PTR_TO_PACKET:	10697	case PTR_TO_PACKET:
10634	if (rcur->type != rold->type)	10698	if (rcur->type != rold->type)
@@ -10657,11 +10721,8 @@ static bool regsafe(struct bpf_verifier_env env, struct bpf_reg_state rold, · 22 unmodified lines
10657	case PTR_TO_PACKET_END:	10721	case PTR_TO_PACKET_END:
10658	case PTR_TO_FLOW_KEYS:	10722	case PTR_TO_FLOW_KEYS:
10659	case PTR_TO_SOCKET:	10723	case PTR_TO_SOCKET:
10660	case PTR_TO_SOCKET_OR_NULL:
10661	case PTR_TO_SOCK_COMMON:	10724	case PTR_TO_SOCK_COMMON:
10662	case PTR_TO_SOCK_COMMON_OR_NULL:
10663	case PTR_TO_TCP_SOCK:	10725	case PTR_TO_TCP_SOCK:
10664	case PTR_TO_TCP_SOCK_OR_NULL:
10665	case PTR_TO_XDP_SOCK:	10726	case PTR_TO_XDP_SOCK:
10666	/* Only valid matches are exact, which memcmp() above	10727	/* Only valid matches are exact, which memcmp() above
10667	* would have accepted	10728	* would have accepted
@@ -11187,17 +11248,13 @@ next: · 519 unmodified lines
11187	/* Return true if it's OK to have the same insn return a different type. */	11248	/* Return true if it's OK to have the same insn return a different type. */
11188	static bool reg_type_mismatch_ok(enum bpf_reg_type type)	11249	static bool reg_type_mismatch_ok(enum bpf_reg_type type)
11189	{	11250	{
11190	switch (type) {	11251	switch (base_type(type)) {
11191	case PTR_TO_CTX:	11252	case PTR_TO_CTX:
11192	case PTR_TO_SOCKET:	11253	case PTR_TO_SOCKET:
11193	case PTR_TO_SOCKET_OR_NULL:
11194	case PTR_TO_SOCK_COMMON:	11254	case PTR_TO_SOCK_COMMON:
11195	case PTR_TO_SOCK_COMMON_OR_NULL:
11196	case PTR_TO_TCP_SOCK:	11255	case PTR_TO_TCP_SOCK:
11197	case PTR_TO_TCP_SOCK_OR_NULL:
11198	case PTR_TO_XDP_SOCK:	11256	case PTR_TO_XDP_SOCK:
11199	case PTR_TO_BTF_ID:	11257	case PTR_TO_BTF_ID:
11200	case PTR_TO_BTF_ID_OR_NULL:
11201	return false;	11258	return false;
11202	default:	11259	default:
11203	return true;	11260	return true;
@@ -11277,16 +11334,12 @@ static int do_check(struct bpf_verifier_env *env) · 73 unmodified lines
11277	if (need_resched())	11334	if (need_resched())
11278	cond_resched();	11335	cond_resched();
11279		11336
11280	if (env->log.level & BPF_LOG_LEVEL2 \|\|	11337	if (env->log.level & BPF_LOG_LEVEL2 && do_print_state) {
11281	(env->log.level & BPF_LOG_LEVEL && do_print_state)) {	11338	verbose(env, "\nfrom %d to %d%s:",
11282	if (env->log.level & BPF_LOG_LEVEL2)	11339	env->prev_insn_idx, env->insn_idx,
11283	verbose(env, "%d:", env->insn_idx);	11340	env->cur_state->speculative ?
11284	else	11341	" (speculative execution)" : "");
11285	verbose(env, "\nfrom %d to %d%s:",	11342	print_verifier_state(env, state->frame[state->curframe], true);
11286	env->prev_insn_idx, env->insn_idx,
11287	env->cur_state->speculative ?
11288	" (speculative execution)" : "");
11289	print_verifier_state(env, state->frame[state->curframe]);
11290	do_print_state = false;	11343	do_print_state = false;
11291	}	11344	}
11292		11345
@@ -11297,9 +11350,15 @@ static int do_check(struct bpf_verifier_env *env) · 4 unmodified lines
11297	.private_data = env,	11350	.private_data = env,
11298	};	11351	};
11299		11352
		11353	if (verifier_state_scratched(env))
		11354	print_insn_state(env, state->frame[state->curframe]);
		11355
11300	verbose_linfo(env, env->insn_idx, "; ");	11356	verbose_linfo(env, env->insn_idx, "; ");
		11357	env->prev_log_len = env->log.len_used;
11301	verbose(env, "%d: ", env->insn_idx);	11358	verbose(env, "%d: ", env->insn_idx);
11302	print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);	11359	print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
		11360	env->prev_insn_print_len = env->log.len_used - env->prev_log_len;
		11361	env->prev_log_len = env->log.len_used;
11303	}	11362	}
11304		11363
11305	if (bpf_prog_is_dev_bound(env->prog->aux)) {	11364	if (bpf_prog_is_dev_bound(env->prog->aux)) {
@@ -11421,7 +11480,7 @@ static int do_check(struct bpf_verifier_env *env) · 115 unmodified lines
11421	if (is_ctx_reg(env, insn->dst_reg)) {	11480	if (is_ctx_reg(env, insn->dst_reg)) {
11422	verbose(env, "BPF_ST stores into R%d %s is not allowed\n",	11481	verbose(env, "BPF_ST stores into R%d %s is not allowed\n",
11423	insn->dst_reg,	11482	insn->dst_reg,
11424	reg_type_str[reg_state(env, insn->dst_reg)->type]);	11483	reg_type_str(env, reg_state(env, insn->dst_reg)->type));
11425	return -EACCES;	11484	return -EACCES;
11426	}	11485	}
11427		11486
@@ -11508,6 +11567,7 @@ static int do_check(struct bpf_verifier_env *env) · 80 unmodified lines
11508	if (err)	11567	if (err)
11509	return err;	11568	return err;
11510	process_bpf_exit:	11569	process_bpf_exit:
		11570	mark_verifier_state_scratched(env);
11511	update_branch_counts(env, env->cur_state);	11571	update_branch_counts(env, env->cur_state);
11512	err = pop_stack(env, &prev_insn_idx,	11572	err = pop_stack(env, &prev_insn_idx,
11513	&env->insn_idx, pop_log);	11573	&env->insn_idx, pop_log);
@@ -11673,7 +11733,7 @@ static int check_pseudo_btf_id(struct bpf_verifier_env *env, · 159 unmodified lines
11673	err = -EINVAL;	11733	err = -EINVAL;
11674	goto err_put;	11734	goto err_put;
11675	}	11735	}
11676	aux->btf_var.reg_type = PTR_TO_MEM;	11736	aux->btf_var.reg_type = PTR_TO_MEM \| MEM_RDONLY;
11677	aux->btf_var.mem_size = tsize;	11737	aux->btf_var.mem_size = tsize;
11678	} else {	11738	} else {
11679	aux->btf_var.reg_type = PTR_TO_BTF_ID;	11739	aux->btf_var.reg_type = PTR_TO_BTF_ID;
@@ -11833,6 +11893,9 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, · 153 unmodified lines
11833	}	11893	}
11834	break;	11894	break;
11835	case BPF_MAP_TYPE_RINGBUF:	11895	case BPF_MAP_TYPE_RINGBUF:
		11896	case BPF_MAP_TYPE_INODE_STORAGE:
		11897	case BPF_MAP_TYPE_SK_STORAGE:
		11898	case BPF_MAP_TYPE_TASK_STORAGE:
11836	break;	11899	break;
11837	default:	11900	default:
11838	verbose(env,	11901	verbose(env,
@@ -13016,6 +13079,7 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, · 1177 unmodified lines
13016	static int do_misc_fixups(struct bpf_verifier_env *env)	13079	static int do_misc_fixups(struct bpf_verifier_env *env)
13017	{	13080	{
13018	struct bpf_prog *prog = env->prog;	13081	struct bpf_prog *prog = env->prog;
		13082	enum bpf_attach_type eatype = prog->expected_attach_type;
13019	bool expect_blinding = bpf_jit_blinding_enabled(prog);	13083	bool expect_blinding = bpf_jit_blinding_enabled(prog);
13020	enum bpf_prog_type prog_type = resolve_prog_type(prog);	13084	enum bpf_prog_type prog_type = resolve_prog_type(prog);
13021	struct bpf_insn *insn = prog->insnsi;	13085	struct bpf_insn *insn = prog->insnsi;
@@ -13386,11 +13450,79 @@ patch_map_ops_generic: · 364 unmodified lines
13386	continue;	13450	continue;
13387	}	13451	}
13388		13452
		13453	/* Implement bpf_get_func_arg inline. */
		13454	if (prog_type == BPF_PROG_TYPE_TRACING &&
		13455	insn->imm == BPF_FUNC_get_func_arg) {
		13456	/* Load nr_args from ctx - 8 */
		13457	insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
		13458	insn_buf[1] = BPF_JMP32_REG(BPF_JGE, BPF_REG_2, BPF_REG_0, 6);
		13459	insn_buf[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 3);
		13460	insn_buf[3] = BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_1);
		13461	insn_buf[4] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0);
		13462	insn_buf[5] = BPF_STX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0);
		13463	insn_buf[6] = BPF_MOV64_IMM(BPF_REG_0, 0);
		13464	insn_buf[7] = BPF_JMP_A(1);
		13465	insn_buf[8] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL);
		13466	cnt = 9;
		13467
		13468	new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
		13469	if (!new_prog)
		13470	return -ENOMEM;
		13471
		13472	delta += cnt - 1;
		13473	env->prog = prog = new_prog;
		13474	insn = new_prog->insnsi + i + delta;
		13475	continue;
		13476	}
		13477
		13478	/* Implement bpf_get_func_ret inline. */
		13479	if (prog_type == BPF_PROG_TYPE_TRACING &&
		13480	insn->imm == BPF_FUNC_get_func_ret) {
		13481	if (eatype == BPF_TRACE_FEXIT \|\|
		13482	eatype == BPF_MODIFY_RETURN) {
		13483	/* Load nr_args from ctx - 8 */
		13484	insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
		13485	insn_buf[1] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_0, 3);
		13486	insn_buf[2] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1);
		13487	insn_buf[3] = BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0);
		13488	insn_buf[4] = BPF_STX_MEM(BPF_DW, BPF_REG_2, BPF_REG_3, 0);
		13489	insn_buf[5] = BPF_MOV64_IMM(BPF_REG_0, 0);
		13490	cnt = 6;
		13491	} else {
		13492	insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, -EOPNOTSUPP);
		13493	cnt = 1;
		13494	}
		13495
		13496	new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
		13497	if (!new_prog)
		13498	return -ENOMEM;
		13499
		13500	delta += cnt - 1;
		13501	env->prog = prog = new_prog;
		13502	insn = new_prog->insnsi + i + delta;
		13503	continue;
		13504	}
		13505
		13506	/* Implement get_func_arg_cnt inline. */
		13507	if (prog_type == BPF_PROG_TYPE_TRACING &&
		13508	insn->imm == BPF_FUNC_get_func_arg_cnt) {
		13509	/* Load nr_args from ctx - 8 */
		13510	insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
		13511
		13512	new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1);
		13513	if (!new_prog)
		13514	return -ENOMEM;
		13515
		13516	env->prog = prog = new_prog;
		13517	insn = new_prog->insnsi + i + delta;
		13518	continue;
		13519	}
		13520
13389	/* Implement bpf_get_func_ip inline. */	13521	/* Implement bpf_get_func_ip inline. */
13390	if (prog_type == BPF_PROG_TYPE_TRACING &&	13522	if (prog_type == BPF_PROG_TYPE_TRACING &&
13391	insn->imm == BPF_FUNC_get_func_ip) {	13523	insn->imm == BPF_FUNC_get_func_ip) {
13392	/* Load IP address from ctx - 8 */	13524	/* Load IP address from ctx - 16 */
13393	insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);	13525	insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -16);
13394		13526
13395	new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1);	13527	new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1);
13396	if (!new_prog)	13528	if (!new_prog)
@@ -13504,7 +13636,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) · 107 unmodified lines
13504	mark_reg_known_zero(env, regs, i);	13636	mark_reg_known_zero(env, regs, i);
13505	else if (regs[i].type == SCALAR_VALUE)	13637	else if (regs[i].type == SCALAR_VALUE)
13506	mark_reg_unknown(env, regs, i);	13638	mark_reg_unknown(env, regs, i);
13507	else if (regs[i].type == PTR_TO_MEM_OR_NULL) {	13639	else if (base_type(regs[i].type) == PTR_TO_MEM) {
13508	const u32 mem_size = regs[i].mem_size;	13640	const u32 mem_size = regs[i].mem_size;
13509		13641
13510	mark_reg_known_zero(env, regs, i);	13642	mark_reg_known_zero(env, regs, i);
@@ -14099,6 +14231,8 @@ int bpf_check(struct bpf_prog *prog, union bpf_attr attr, bpfptr_t uattr) · 588 unmodified lines
14099	}	14231	}
14100	}	14232	}
14101		14233
		14234	mark_verifier_state_clean(env);
		14235
14102	if (IS_ERR(btf_vmlinux)) {	14236	if (IS_ERR(btf_vmlinux)) {
14103	/* Either gcc or pahole or kernel are broken. */	14237	/* Either gcc or pahole or kernel are broken. */
14104	verbose(env, "in-kernel BTF is malformed\n");	14238	verbose(env, "in-kernel BTF is malformed\n");

kernel/cgroup/cgroup.c+1−0

@@ -30,6 +30,7 @@ · 29 unmodified lines
30		30
31	#include "cgroup-internal.h"	31	#include "cgroup-internal.h"
32		32
		33	#include <linux/bpf-cgroup.h>
33	#include <linux/cred.h>	34	#include <linux/cred.h>
34	#include <linux/errno.h>	35	#include <linux/errno.h>
35	#include <linux/init_task.h>	36	#include <linux/init_task.h>

kernel/sysctl.c+1−0

@@ -33,6 +33,7 @@ · 32 unmodified lines
33	#include <linux/security.h>	33	#include <linux/security.h>
34	#include <linux/ctype.h>	34	#include <linux/ctype.h>
35	#include <linux/kmemleak.h>	35	#include <linux/kmemleak.h>
		36	#include <linux/filter.h>
36	#include <linux/fs.h>	37	#include <linux/fs.h>
37	#include <linux/init.h>	38	#include <linux/init.h>
38	#include <linux/kernel.h>	39	#include <linux/kernel.h>

kernel/trace/bpf_trace.c+67−14

 	.gpl_only	= true,
 	.ret_type	= RET_INTEGER,
 	.arg1_type	= ARG_ANYTHING,
-	.arg2_type	= ARG_PTR_TO_MEM,
+	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
 	.arg3_type	= ARG_CONST_SIZE,
 };
 	.func		= bpf_trace_printk,
 	.gpl_only	= true,
 	.ret_type	= RET_INTEGER,
-	.arg1_type	= ARG_PTR_TO_MEM,
+	.arg1_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
 	.arg2_type	= ARG_CONST_SIZE,
 };
 	.func		= bpf_trace_vprintk,
 	.gpl_only	= true,
 	.ret_type	= RET_INTEGER,
-	.arg1_type	= ARG_PTR_TO_MEM,
+	.arg1_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
 	.arg2_type	= ARG_CONST_SIZE,
-	.arg3_type	= ARG_PTR_TO_MEM_OR_NULL,
+	.arg3_type	= ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY,
 	.arg4_type	= ARG_CONST_SIZE_OR_ZERO,
 };
 	.ret_type	= RET_INTEGER,
 	.arg1_type	= ARG_PTR_TO_BTF_ID,
 	.arg1_btf_id	= &btf_seq_file_ids[0],
-	.arg2_type	= ARG_PTR_TO_MEM,
+	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
 	.arg3_type	= ARG_CONST_SIZE,
-	.arg4_type      = ARG_PTR_TO_MEM_OR_NULL,
+	.arg4_type      = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY,
 	.arg5_type      = ARG_CONST_SIZE_OR_ZERO,
 };
 	.ret_type	= RET_INTEGER,
 	.arg1_type	= ARG_PTR_TO_BTF_ID,
 	.arg1_btf_id	= &btf_seq_file_ids[0],
-	.arg2_type	= ARG_PTR_TO_MEM,
+	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
 	.arg3_type	= ARG_CONST_SIZE_OR_ZERO,
 };
 	.ret_type	= RET_INTEGER,
 	.arg1_type	= ARG_PTR_TO_BTF_ID,
 	.arg1_btf_id	= &btf_seq_file_ids[0],
-	.arg2_type	= ARG_PTR_TO_MEM,
+	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
 	.arg3_type	= ARG_CONST_SIZE_OR_ZERO,
 	.arg4_type	= ARG_ANYTHING,
 };
 	.arg1_type	= ARG_PTR_TO_CTX,
 	.arg2_type	= ARG_CONST_MAP_PTR,
 	.arg3_type	= ARG_ANYTHING,
-	.arg4_type	= ARG_PTR_TO_MEM,
+	.arg4_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
 	.arg5_type	= ARG_CONST_SIZE_OR_ZERO,
 };
 	.ret_type	= RET_INTEGER,
 	.arg1_type	= ARG_PTR_TO_MEM,
 	.arg2_type	= ARG_CONST_SIZE,
-	.arg3_type	= ARG_PTR_TO_MEM,
+	.arg3_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
 	.arg4_type	= ARG_CONST_SIZE,
 	.arg5_type	= ARG_ANYTHING,
 };
 BPF_CALL_1(bpf_get_func_ip_tracing, void *, ctx)
 {
 	/* This helper call is inlined by verifier. */
-	return ((u64 *)ctx)[-1];
+	return ((u64 *)ctx)[-2];
 }
 static const struct bpf_func_proto bpf_get_func_ip_proto_tracing = {
 	.arg2_type	= ARG_CONST_SIZE_OR_ZERO,
 };
+BPF_CALL_3(get_func_arg, void *, ctx, u32, n, u64 *, value)
+{
+	/* This helper call is inlined by verifier. */
+	u64 nr_args = ((u64 *)ctx)[-1];
+	if ((u64) n >= nr_args)
+		return -EINVAL;
+	*value = ((u64 *)ctx)[n];
+	return 0;
+}
+static const struct bpf_func_proto bpf_get_func_arg_proto = {
+	.func		= get_func_arg,
+	.ret_type	= RET_INTEGER,
+	.arg1_type	= ARG_PTR_TO_CTX,
+	.arg2_type	= ARG_ANYTHING,
+	.arg3_type	= ARG_PTR_TO_LONG,
+};
+BPF_CALL_2(get_func_ret, void *, ctx, u64 *, value)
+{
+	/* This helper call is inlined by verifier. */
+	u64 nr_args = ((u64 *)ctx)[-1];
+	*value = ((u64 *)ctx)[nr_args];
+	return 0;
+}
+static const struct bpf_func_proto bpf_get_func_ret_proto = {
+	.func		= get_func_ret,
+	.ret_type	= RET_INTEGER,
+	.arg1_type	= ARG_PTR_TO_CTX,
+	.arg2_type	= ARG_PTR_TO_LONG,
+};
+BPF_CALL_1(get_func_arg_cnt, void *, ctx)
+{
+	/* This helper call is inlined by verifier. */
+	return ((u64 *)ctx)[-1];
+}
+static const struct bpf_func_proto bpf_get_func_arg_cnt_proto = {
+	.func		= get_func_arg_cnt,
+	.ret_type	= RET_INTEGER,
+	.arg1_type	= ARG_PTR_TO_CTX,
+};
 static const struct bpf_func_proto *
 bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 {
 	.arg1_type	= ARG_PTR_TO_CTX,
 	.arg2_type	= ARG_CONST_MAP_PTR,
 	.arg3_type	= ARG_ANYTHING,
-	.arg4_type	= ARG_PTR_TO_MEM,
+	.arg4_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
 	.arg5_type	= ARG_CONST_SIZE_OR_ZERO,
 };
 	.arg1_type	= ARG_PTR_TO_CTX,
 	.arg2_type	= ARG_CONST_MAP_PTR,
 	.arg3_type	= ARG_ANYTHING,
-	.arg4_type	= ARG_PTR_TO_MEM,
+	.arg4_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
 	.arg5_type	= ARG_CONST_SIZE_OR_ZERO,
 };
 	.gpl_only	= true,
 	.ret_type	= RET_INTEGER,
 	.arg1_type	= ARG_PTR_TO_CTX,
-	.arg2_type	= ARG_PTR_TO_MEM,
+	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
 	.arg3_type	= ARG_CONST_SIZE_OR_ZERO,
 	.arg4_type	= ARG_ANYTHING,
 };
 		       NULL;
 	case BPF_FUNC_d_path:
 		return &bpf_d_path_proto;
+	case BPF_FUNC_get_func_arg:
+		return bpf_prog_has_trampoline(prog) ? &bpf_get_func_arg_proto : NULL;
+	case BPF_FUNC_get_func_ret:
+		return bpf_prog_has_trampoline(prog) ? &bpf_get_func_ret_proto : NULL;
+	case BPF_FUNC_get_func_arg_cnt:
+		return bpf_prog_has_trampoline(prog) ? &bpf_get_func_arg_cnt_proto : NULL;
 	default:
 		fn = raw_tp_prog_func_proto(func_id, prog);
 		if (!fn && prog->expected_attach_type == BPF_TRACE_ITER)

kernel/trace/trace_kprobe.c+1−0

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	*/	7	*/
8	#define pr_fmt(fmt) "trace_kprobe: " fmt	8	#define pr_fmt(fmt) "trace_kprobe: " fmt
9		9
		10	#include <linux/bpf-cgroup.h>
10	#include <linux/security.h>	11	#include <linux/security.h>
11	#include <linux/module.h>	12	#include <linux/module.h>
12	#include <linux/uaccess.h>	13	#include <linux/uaccess.h>

kernel/trace/trace_uprobe.c+1−0

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	*/	7	*/
8	#define pr_fmt(fmt) "trace_uprobe: " fmt	8	#define pr_fmt(fmt) "trace_uprobe: " fmt
9		9
		10	#include <linux/bpf-cgroup.h>
10	#include <linux/security.h>	11	#include <linux/security.h>
11	#include <linux/ctype.h>	12	#include <linux/ctype.h>
12	#include <linux/module.h>	13	#include <linux/module.h>