Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Daniel Borkmann says: ==================== pull-request: bpf-next 2021-02-16 The following pull-request contains BPF updates for your *net-next* tree. There's a small merge conflict between 7eeba1706eba ("tcp: Add receive timestamp support for receive zerocopy.") from net-next tree and 9cacf81f8161 ("bpf: Remove extra lock_sock for TCP_ZEROCOPY_RECEIVE") from bpf-next tree. Resolve as follows: [...] lock_sock(sk); err = tcp_zerocopy_receive(sk, &zc, &tss); err = BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sk, level, optname, &zc, &len, err); release_sock(sk); [...] We've added 116 non-merge commits during the last 27 day(s) which contain a total of 156 files changed, 5662 insertions(+), 1489 deletions(-). The main changes are: 1) Adds support of pointers to types with known size among global function args to overcome the limit on max # of allowed args, from Dmitrii Banshchikov. 2) Add bpf_iter for task_vma which can be used to generate information similar to /proc/pid/maps, from Song Liu. 3) Enable bpf_{g,s}etsockopt() from all sock_addr related program hooks. Allow rewriting bind user ports from BPF side below the ip_unprivileged_port_start range, both from Stanislav Fomichev. 4) Prevent recursion on fentry/fexit & sleepable programs and allow map-in-map as well as per-cpu maps for the latter, from Alexei Starovoitov. 5) Add selftest script to run BPF CI locally. Also enable BPF ringbuffer for sleepable programs, both from KP Singh. 6) Extend verifier to enable variable offset read/write access to the BPF program stack, from Andrei Matei. 7) Improve tc & XDP MTU handling and add a new bpf_check_mtu() helper to query device MTU from programs, from Jesper Dangaard Brouer. 8) Allow bpf_get_socket_cookie() helper also be called from [sleepable] BPF tracing programs, from Florent Revest. 9) Extend x86 JIT to pad JMPs with NOPs for helping image to converge when otherwise too many passes are required, from Gary Lin. 10) Verifier fixes on atomics with BPF_FETCH as well as function-by-function verification both related to zero-extension handling, from Ilya Leoshkevich. 11) Better kernel build integration of resolve_btfids tool, from Jiri Olsa. 12) Batch of AF_XDP selftest cleanups and small performance improvement for libbpf's xsk map redirect for newer kernels, from Björn Töpel. 13) Follow-up BPF doc and verifier improvements around atomics with BPF_FETCH, from Brendan Jackman. 14) Permit zero-sized data sections e.g. if ELF .rodata section contains read-only data from local variables, from Yonghong Song. 15) veth driver skb bulk-allocation for ndo_xdp_xmit, from Lorenzo Bianconi. ====================

David S. Miller committed 5 years ago

Commit b8af4172 parents 9ec5eea 45159b2156 files · +5662 −1489Browse files

Showing the changes to kernel (15 of 156 files). Show all

Unified · Split

15 files

kernel/bpf/bpf_iter.c +1 −1
kernel/bpf/bpf_lru_list.c +4 −3
kernel/bpf/btf.c +55 −21
kernel/bpf/cgroup.c +103 −17
kernel/bpf/core.c +14 −4
kernel/bpf/cpumap.c +2 −44
kernel/bpf/devmap.c +1 −3
kernel/bpf/disasm.c +1 −1
kernel/bpf/hashtab.c +2 −2
kernel/bpf/helpers.c +4 −8
kernel/bpf/syscall.c +11 −5
kernel/bpf/task_iter.c +266 −1
kernel/bpf/trampoline.c +61 −16
kernel/bpf/verifier.c +662 −215
kernel/trace/bpf_trace.c +6 −0

kernel/bpf/bpf_iter.c+1−1

@@ -287,7 +287,7 @@ int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info) · 286 unmodified lines
287	{	287	{
288	struct bpf_iter_target_info *tinfo;	288	struct bpf_iter_target_info *tinfo;
289		289
290	tinfo = kmalloc(sizeof(*tinfo), GFP_KERNEL);	290	tinfo = kzalloc(sizeof(*tinfo), GFP_KERNEL);
291	if (!tinfo)	291	if (!tinfo)
292	return -ENOMEM;	292	return -ENOMEM;
293		293

kernel/bpf/bpf_lru_list.c+4−3

 static void bpf_common_lru_push_free(struct bpf_lru *lru,
 				     struct bpf_lru_node *node)
 {
+	u8 node_type = READ_ONCE(node->type);
 	unsigned long flags;
-	if (WARN_ON_ONCE(node->type == BPF_LRU_LIST_T_FREE) ||
+	if (WARN_ON_ONCE(node_type == BPF_LRU_LIST_T_FREE) ||
-	    WARN_ON_ONCE(node->type == BPF_LRU_LOCAL_LIST_T_FREE))
+	    WARN_ON_ONCE(node_type == BPF_LRU_LOCAL_LIST_T_FREE))
 		return;
-	if (node->type == BPF_LRU_LOCAL_LIST_T_PENDING) {
+	if (node_type == BPF_LRU_LOCAL_LIST_T_PENDING) {
 		struct bpf_lru_locallist *loc_l;
 		loc_l = per_cpu_ptr(lru->common_lru.local_list, node->cpu);

kernel/bpf/btf.c+55−21

 		return -EINVAL;
 	}
-	if (!btf_type_vlen(t)) {
-		btf_verifier_log_type(env, t, "vlen == 0");
-		return -EINVAL;
-	}
 	if (!t->size) {
 		btf_verifier_log_type(env, t, "size == 0");
 		return -EINVAL;
  * Only PTR_TO_CTX and SCALAR_VALUE states are recognized.
  */
 int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog,
-			     struct bpf_reg_state *reg)
+			     struct bpf_reg_state *regs)
 {
 	struct bpf_verifier_log *log = &env->log;
 	struct bpf_prog *prog = env->prog;
 	struct btf *btf = prog->aux->btf;
 	const struct btf_param *args;
-	const struct btf_type *t;
+	const struct btf_type *t, *ref_t;
-	u32 i, nargs, btf_id;
+	u32 i, nargs, btf_id, type_size;
 	const char *tname;
+	bool is_global;
 	if (!prog->aux->func_info)
 		return -EINVAL;
 		bpf_log(log, "Function %s has %d > 5 args\n", tname, nargs);
 		goto out;
 	}
+	is_global = prog->aux->func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL;
 	/* check that BTF function arguments match actual types that the
 	 * verifier sees.
 	 */
 	for (i = 0; i < nargs; i++) {
+		struct bpf_reg_state *reg = &regs[i + 1];
 		t = btf_type_by_id(btf, args[i].type);
 		while (btf_type_is_modifier(t))
 			t = btf_type_by_id(btf, t->type);
 		if (btf_type_is_int(t) || btf_type_is_enum(t)) {
-			if (reg[i + 1].type == SCALAR_VALUE)
+			if (reg->type == SCALAR_VALUE)
 				continue;
 			bpf_log(log, "R%d is not a scalar\n", i + 1);
 			goto out;
 		}
 		if (btf_type_is_ptr(t)) {
-			if (reg[i + 1].type == SCALAR_VALUE) {
-				bpf_log(log, "R%d is not a pointer\n", i + 1);
-				goto out;
-			}
 			/* If function expects ctx type in BTF check that caller
 			 * is passing PTR_TO_CTX.
 			 */
 			if (btf_get_prog_ctx_type(log, btf, t, prog->type, i)) {
-				if (reg[i + 1].type != PTR_TO_CTX) {
+				if (reg->type != PTR_TO_CTX) {
 					bpf_log(log,
 						"arg#%d expected pointer to ctx, but got %s\n",
 						i, btf_kind_str[BTF_INFO_KIND(t->info)]);
 					goto out;
 				}
-				if (check_ctx_reg(env, &reg[i + 1], i + 1))
+				if (check_ctx_reg(env, reg, i + 1))
 					goto out;
 				continue;
 			}
+			if (!is_global)
+				goto out;
+			t = btf_type_skip_modifiers(btf, t->type, NULL);
+			ref_t = btf_resolve_size(btf, t, &type_size);
+			if (IS_ERR(ref_t)) {
+				bpf_log(log,
+				    "arg#%d reference type('%s %s') size cannot be determined: %ld\n",
+				    i, btf_type_str(t), btf_name_by_offset(btf, t->name_off),
+					PTR_ERR(ref_t));
+				goto out;
+			}
+			if (check_mem_reg(env, reg, i + 1, type_size))
+				goto out;
+			continue;
 		}
 		bpf_log(log, "Unrecognized arg#%d type %s\n",
 			i, btf_kind_str[BTF_INFO_KIND(t->info)]);
  * (either PTR_TO_CTX or SCALAR_VALUE).
  */
 int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog,
-			  struct bpf_reg_state *reg)
+			  struct bpf_reg_state *regs)
 {
 	struct bpf_verifier_log *log = &env->log;
 	struct bpf_prog *prog = env->prog;
 	enum bpf_prog_type prog_type = prog->type;
 	struct btf *btf = prog->aux->btf;
 	const struct btf_param *args;
-	const struct btf_type *t;
+	const struct btf_type *t, *ref_t;
 	u32 i, nargs, btf_id;
 	const char *tname;
 	 * Only PTR_TO_CTX and SCALAR are supported atm.
 	 */
 	for (i = 0; i < nargs; i++) {
+		struct bpf_reg_state *reg = &regs[i + 1];
 		t = btf_type_by_id(btf, args[i].type);
 		while (btf_type_is_modifier(t))
 			t = btf_type_by_id(btf, t->type);
 		if (btf_type_is_int(t) || btf_type_is_enum(t)) {
-			reg[i + 1].type = SCALAR_VALUE;
+			reg->type = SCALAR_VALUE;
 			continue;
 		}
-		if (btf_type_is_ptr(t) &&
+		if (btf_type_is_ptr(t)) {
-		    btf_get_prog_ctx_type(log, btf, t, prog_type, i)) {
+			if (btf_get_prog_ctx_type(log, btf, t, prog_type, i)) {
-			reg[i + 1].type = PTR_TO_CTX;
+				reg->type = PTR_TO_CTX;
+				continue;
+			}
+			t = btf_type_skip_modifiers(btf, t->type, NULL);
+			ref_t = btf_resolve_size(btf, t, &reg->mem_size);
+			if (IS_ERR(ref_t)) {
+				bpf_log(log,
+				    "arg#%d reference type('%s %s') size cannot be determined: %ld\n",
+				    i, btf_type_str(t), btf_name_by_offset(btf, t->name_off),
+					PTR_ERR(ref_t));
+				return -EINVAL;
+			}
+			reg->type = PTR_TO_MEM_OR_NULL;
+			reg->id = ++env->id_gen;
 			continue;
 		}
 		bpf_log(log, "Arg#%d type %s in %s() is not supported yet.\n",

kernel/bpf/cgroup.c+103−17

 #include "../cgroup/cgroup-internal.h"
-DEFINE_STATIC_KEY_FALSE(cgroup_bpf_enabled_key);
+DEFINE_STATIC_KEY_ARRAY_FALSE(cgroup_bpf_enabled_key, MAX_BPF_ATTACH_TYPE);
 EXPORT_SYMBOL(cgroup_bpf_enabled_key);
 void cgroup_bpf_offline(struct cgroup *cgrp)
 			if (pl->link)
 				bpf_cgroup_link_auto_detach(pl->link);
 			kfree(pl);
-			static_branch_dec(&cgroup_bpf_enabled_key);
+			static_branch_dec(&cgroup_bpf_enabled_key[type]);
 		}
 		old_array = rcu_dereference_protected(
 				cgrp->bpf.effective[type],
 	if (old_prog)
 		bpf_prog_put(old_prog);
 	else
-		static_branch_inc(&cgroup_bpf_enabled_key);
+		static_branch_inc(&cgroup_bpf_enabled_key[type]);
 	bpf_cgroup_storages_link(new_storage, cgrp, type);
 	return 0;
 		cgrp->bpf.flags[type] = 0;
 	if (old_prog)
 		bpf_prog_put(old_prog);
-	static_branch_dec(&cgroup_bpf_enabled_key);
+	static_branch_dec(&cgroup_bpf_enabled_key[type]);
 	return 0;
 cleanup:
  * @uaddr: sockaddr struct provided by user
  * @type: The type of program to be exectuted
  * @t_ctx: Pointer to attach type specific context
+ * @flags: Pointer to u32 which contains higher bits of BPF program
+ *         return value (OR'ed together).
  *
  * socket is expected to be of type INET or INET6.
  *
 int __cgroup_bpf_run_filter_sock_addr(struct sock *sk,
 				      struct sockaddr *uaddr,
 				      enum bpf_attach_type type,
-				      void *t_ctx)
+				      void *t_ctx,
+				      u32 *flags)
 {
 	struct bpf_sock_addr_kern ctx = {
 		.sk = sk,
 	}
 	cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
-	ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], &ctx, BPF_PROG_RUN);
+	ret = BPF_PROG_RUN_ARRAY_FLAGS(cgrp->bpf.effective[type], &ctx,
+				       BPF_PROG_RUN, flags);
 	return ret == 1 ? 0 : -EPERM;
 }
 	return empty;
 }
-static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen)
+static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen,
+			     struct bpf_sockopt_buf *buf)
 {
 	if (unlikely(max_optlen < 0))
 		return -EINVAL;
 		max_optlen = PAGE_SIZE;
 	}
+	if (max_optlen <= sizeof(buf->data)) {
+		/* When the optval fits into BPF_SOCKOPT_KERN_BUF_SIZE
+		 * bytes avoid the cost of kzalloc.
+		 */
+		ctx->optval = buf->data;
+		ctx->optval_end = ctx->optval + max_optlen;
+		return max_optlen;
+	}
 	ctx->optval = kzalloc(max_optlen, GFP_USER);
 	if (!ctx->optval)
 		return -ENOMEM;
 	return max_optlen;
 }
-static void sockopt_free_buf(struct bpf_sockopt_kern *ctx)
+static void sockopt_free_buf(struct bpf_sockopt_kern *ctx,
+			     struct bpf_sockopt_buf *buf)
 {
+	if (ctx->optval == buf->data)
+		return;
 	kfree(ctx->optval);
 }
+static bool sockopt_buf_allocated(struct bpf_sockopt_kern *ctx,
+				  struct bpf_sockopt_buf *buf)
+{
+	return ctx->optval != buf->data;
+}
 int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level,
 				       int *optname, char __user *optval,
 				       int *optlen, char **kernel_optval)
 {
 	struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
+	struct bpf_sockopt_buf buf = {};
 	struct bpf_sockopt_kern ctx = {
 		.sk = sk,
 		.level = *level,
 	 * attached to the hook so we don't waste time allocating
 	 * memory and locking the socket.
 	 */
-	if (!cgroup_bpf_enabled ||
+	if (__cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_SETSOCKOPT))
-	    __cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_SETSOCKOPT))
 		return 0;
 	/* Allocate a bit more than the initial user buffer for
 	 */
 	max_optlen = max_t(int, 16, *optlen);
-	max_optlen = sockopt_alloc_buf(&ctx, max_optlen);
+	max_optlen = sockopt_alloc_buf(&ctx, max_optlen, &buf);
 	if (max_optlen < 0)
 		return max_optlen;
 		 */
 		if (ctx.optlen != 0) {
 			*optlen = ctx.optlen;
-			*kernel_optval = ctx.optval;
+			/* We've used bpf_sockopt_kern->buf as an intermediary
+			 * storage, but the BPF program indicates that we need
+			 * to pass this data to the kernel setsockopt handler.
+			 * No way to export on-stack buf, have to allocate a
+			 * new buffer.
+			 */
+			if (!sockopt_buf_allocated(&ctx, &buf)) {
+				void *p = kmalloc(ctx.optlen, GFP_USER);
+				if (!p) {
+					ret = -ENOMEM;
+					goto out;
+				}
+				memcpy(p, ctx.optval, ctx.optlen);
+				*kernel_optval = p;
+			} else {
+				*kernel_optval = ctx.optval;
+			}
 			/* export and don't free sockopt buf */
 			return 0;
 		}
 	}
 out:
-	sockopt_free_buf(&ctx);
+	sockopt_free_buf(&ctx, &buf);
 	return ret;
 }
 				       int retval)
 {
 	struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
+	struct bpf_sockopt_buf buf = {};
 	struct bpf_sockopt_kern ctx = {
 		.sk = sk,
 		.level = level,
 	 * attached to the hook so we don't waste time allocating
 	 * memory and locking the socket.
 	 */
-	if (!cgroup_bpf_enabled ||
+	if (__cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_GETSOCKOPT))
-	    __cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_GETSOCKOPT))
 		return retval;
 	ctx.optlen = max_optlen;
-	max_optlen = sockopt_alloc_buf(&ctx, max_optlen);
+	max_optlen = sockopt_alloc_buf(&ctx, max_optlen, &buf);
 	if (max_optlen < 0)
 		return max_optlen;
 	ret = ctx.retval;
 out:
-	sockopt_free_buf(&ctx);
+	sockopt_free_buf(&ctx, &buf);
 	return ret;
 }
+int __cgroup_bpf_run_filter_getsockopt_kern(struct sock *sk, int level,
+					    int optname, void *optval,
+					    int *optlen, int retval)
+{
+	struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
+	struct bpf_sockopt_kern ctx = {
+		.sk = sk,
+		.level = level,
+		.optname = optname,
+		.retval = retval,
+		.optlen = *optlen,
+		.optval = optval,
+		.optval_end = optval + *optlen,
+	};
+	int ret;
+	/* Note that __cgroup_bpf_run_filter_getsockopt doesn't copy
+	 * user data back into BPF buffer when reval != 0. This is
+	 * done as an optimization to avoid extra copy, assuming
+	 * kernel won't populate the data in case of an error.
+	 * Here we always pass the data and memset() should
+	 * be called if that data shouldn't be "exported".
+	 */
+	ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[BPF_CGROUP_GETSOCKOPT],
+				 &ctx, BPF_PROG_RUN);
+	if (!ret)
+		return -EPERM;
+	if (ctx.optlen > *optlen)
+		return -EFAULT;
+	/* BPF programs only allowed to set retval to 0, not some
+	 * arbitrary value.
+	 */
+	if (ctx.retval != 0 && ctx.retval != retval)
+		return -EFAULT;
+	/* BPF programs can shrink the buffer, export the modifications.
+	 */
+	if (ctx.optlen != 0)
+		*optlen = ctx.optlen;
+	return ctx.retval;
+}
 #endif
 static ssize_t sysctl_cpy_dir(const struct ctl_dir *dir, char **bufp,

kernel/bpf/core.c+14−4

 		vfree(fp);
 		return NULL;
 	}
+	fp->active = alloc_percpu_gfp(int, GFP_KERNEL_ACCOUNT | gfp_extra_flags);
+	if (!fp->active) {
+		vfree(fp);
+		kfree(aux);
+		return NULL;
+	}
 	fp->pages = size / PAGE_SIZE;
 	fp->aux = aux;
 	if (!prog)
 		return NULL;
-	prog->aux->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags);
+	prog->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags);
-	if (!prog->aux->stats) {
+	if (!prog->stats) {
+		free_percpu(prog->active);
 		kfree(prog->aux);
 		vfree(prog);
 		return NULL;
 	for_each_possible_cpu(cpu) {
 		struct bpf_prog_stats *pstats;
-		pstats = per_cpu_ptr(prog->aux->stats, cpu);
+		pstats = per_cpu_ptr(prog->stats, cpu);
 		u64_stats_init(&pstats->syncp);
 	}
 	return prog;
 		 * reallocated structure.
 		 */
 		fp_old->aux = NULL;
+		fp_old->stats = NULL;
+		fp_old->active = NULL;
 		__bpf_prog_free(fp_old);
 	}
 	if (fp->aux) {
 		mutex_destroy(&fp->aux->used_maps_mutex);
 		mutex_destroy(&fp->aux->dst_mutex);
-		free_percpu(fp->aux->stats);
 		kfree(fp->aux->poke_tab);
 		kfree(fp->aux);
 	}
+	free_percpu(fp->stats);
+	free_percpu(fp->active);
 	vfree(fp);
 }

kernel/bpf/cpumap.c+2−44

@@ -141,49 +141,6 @@ static void cpu_map_kthread_stop(struct work_struct *work) · 140 unmodified lines
141	kthread_stop(rcpu->kthread);	141	kthread_stop(rcpu->kthread);
142	}	142	}
143		143
144	static struct sk_buff cpu_map_build_skb(struct xdp_frame xdpf,
145	struct sk_buff *skb)
146	{
147	unsigned int hard_start_headroom;
148	unsigned int frame_size;
149	void *pkt_data_start;
150
151	/* Part of headroom was reserved to xdpf */
152	hard_start_headroom = sizeof(struct xdp_frame) + xdpf->headroom;
153
154	/* Memory size backing xdp_frame data already have reserved
155	* room for build_skb to place skb_shared_info in tailroom.
156	*/
157	frame_size = xdpf->frame_sz;
158
159	pkt_data_start = xdpf->data - hard_start_headroom;
160	skb = build_skb_around(skb, pkt_data_start, frame_size);
161	if (unlikely(!skb))
162	return NULL;
163
164	skb_reserve(skb, hard_start_headroom);
165	__skb_put(skb, xdpf->len);
166	if (xdpf->metasize)
167	skb_metadata_set(skb, xdpf->metasize);
168
169	/* Essential SKB info: protocol and skb->dev */
170	skb->protocol = eth_type_trans(skb, xdpf->dev_rx);
171
172	/* Optional SKB info, currently missing:
173	* - HW checksum info (skb->ip_summed)
174	* - HW RX hash (skb_set_hash)
175	* - RX ring dev queue index (skb_record_rx_queue)
176	*/
177
178	/* Until page_pool get SKB return path, release DMA here */
179	xdp_release_frame(xdpf);
180
181	/* Allow SKB to reuse area used by xdp_frame */
182	xdp_scrub_frame(xdpf);
183
184	return skb;
185	}
186
187	static void __cpu_map_ring_cleanup(struct ptr_ring *ring)	144	static void __cpu_map_ring_cleanup(struct ptr_ring *ring)
188	{	145	{
189	/* The tear-down procedure should have made sure that queue is	146	/* The tear-down procedure should have made sure that queue is
@@ -350,7 +307,8 @@ static int cpu_map_kthread_run(void *data) · 160 unmodified lines
350	struct sk_buff *skb = skbs[i];	307	struct sk_buff *skb = skbs[i];
351	int ret;	308	int ret;
352		309
353	skb = cpu_map_build_skb(xdpf, skb);	310	skb = __xdp_build_skb_from_frame(xdpf, skb,
		311	xdpf->dev_rx);
354	if (!skb) {	312	if (!skb) {
355	xdp_return_frame(xdpf);	313	xdp_return_frame(xdpf);
356	continue;	314	continue;

kernel/bpf/devmap.c+1−3

@@ -802,9 +802,7 @@ static int dev_map_notification(struct notifier_block *notifier, · 801 unmodified lines
802	break;	802	break;
803		803
804	/* will be freed in free_netdev() */	804	/* will be freed in free_netdev() */
805	netdev->xdp_bulkq =	805	netdev->xdp_bulkq = alloc_percpu(struct xdp_dev_bulk_queue);
806	__alloc_percpu_gfp(sizeof(struct xdp_dev_bulk_queue),
807	sizeof(void *), GFP_ATOMIC);
808	if (!netdev->xdp_bulkq)	806	if (!netdev->xdp_bulkq)
809	return NOTIFY_BAD;	807	return NOTIFY_BAD;
810		808

kernel/bpf/disasm.c+1−1

@@ -161,7 +161,7 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, · 160 unmodified lines
161	insn->dst_reg,	161	insn->dst_reg,
162	insn->off, insn->src_reg);	162	insn->off, insn->src_reg);
163	else if (BPF_MODE(insn->code) == BPF_ATOMIC &&	163	else if (BPF_MODE(insn->code) == BPF_ATOMIC &&
164	(insn->imm == BPF_ADD \|\| insn->imm == BPF_ADD \|\|	164	(insn->imm == BPF_ADD \|\| insn->imm == BPF_AND \|\|
165	insn->imm == BPF_OR \|\| insn->imm == BPF_XOR)) {	165	insn->imm == BPF_OR \|\| insn->imm == BPF_XOR)) {
166	verbose(cbs->private_data, "(%02x) lock (%s )(r%d %+d) %s r%d\n",	166	verbose(cbs->private_data, "(%02x) lock (%s )(r%d %+d) %s r%d\n",
167	insn->code,	167	insn->code,

kernel/bpf/hashtab.c+2−2

@@ -1148,7 +1148,7 @@ static int __htab_percpu_map_update_elem(struct bpf_map map, void key, · 1147 unmodified lines
1148	/* unknown flags */	1148	/* unknown flags */
1149	return -EINVAL;	1149	return -EINVAL;
1150		1150
1151	WARN_ON_ONCE(!rcu_read_lock_held());	1151	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
1152		1152
1153	key_size = map->key_size;	1153	key_size = map->key_size;
1154		1154
@@ -1202,7 +1202,7 @@ static int __htab_lru_percpu_map_update_elem(struct bpf_map map, void key, · 47 unmodified lines
1202	/* unknown flags */	1202	/* unknown flags */
1203	return -EINVAL;	1203	return -EINVAL;
1204		1204
1205	WARN_ON_ONCE(!rcu_read_lock_held());	1205	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
1206		1206
1207	key_size = map->key_size;	1207	key_size = map->key_size;
1208		1208

kernel/bpf/helpers.c+4−8

 		return &bpf_spin_lock_proto;
 	case BPF_FUNC_spin_unlock:
 		return &bpf_spin_unlock_proto;
-	case BPF_FUNC_trace_printk:
-		if (!perfmon_capable())
-			return NULL;
-		return bpf_get_trace_printk_proto();
-	case BPF_FUNC_snprintf_btf:
-		if (!perfmon_capable())
-			return NULL;
-		return &bpf_snprintf_btf_proto;
 	case BPF_FUNC_jiffies64:
 		return &bpf_jiffies64_proto;
 	case BPF_FUNC_per_cpu_ptr:
 		return NULL;
 	switch (func_id) {
+	case BPF_FUNC_trace_printk:
+		return bpf_get_trace_printk_proto();
 	case BPF_FUNC_get_current_task:
 		return &bpf_get_current_task_proto;
 	case BPF_FUNC_probe_read_user:
 		return &bpf_probe_read_user_str_proto;
 	case BPF_FUNC_probe_read_kernel_str:
 		return &bpf_probe_read_kernel_str_proto;
+	case BPF_FUNC_snprintf_btf:
+		return &bpf_snprintf_btf_proto;
 	default:
 		return NULL;
 	}

kernel/bpf/syscall.c+11−5

 static void bpf_prog_get_stats(const struct bpf_prog *prog,
 			       struct bpf_prog_stats *stats)
 {
-	u64 nsecs = 0, cnt = 0;
+	u64 nsecs = 0, cnt = 0, misses = 0;
 	int cpu;
 	for_each_possible_cpu(cpu) {
 		const struct bpf_prog_stats *st;
 		unsigned int start;
-		u64 tnsecs, tcnt;
+		u64 tnsecs, tcnt, tmisses;
-		st = per_cpu_ptr(prog->aux->stats, cpu);
+		st = per_cpu_ptr(prog->stats, cpu);
 		do {
 			start = u64_stats_fetch_begin_irq(&st->syncp);
 			tnsecs = st->nsecs;
 			tcnt = st->cnt;
+			tmisses = st->misses;
 		} while (u64_stats_fetch_retry_irq(&st->syncp, start));
 		nsecs += tnsecs;
 		cnt += tcnt;
+		misses += tmisses;
 	}
 	stats->nsecs = nsecs;
 	stats->cnt = cnt;
+	stats->misses = misses;
 }
 #ifdef CONFIG_PROC_FS
 		   "memlock:\t%llu\n"
 		   "prog_id:\t%u\n"
 		   "run_time_ns:\t%llu\n"
-		   "run_cnt:\t%llu\n",
+		   "run_cnt:\t%llu\n"
+		   "recursion_misses:\t%llu\n",
 		   prog->type,
 		   prog->jited,
 		   prog_tag,
 		   prog->pages * 1ULL << PAGE_SHIFT,
 		   prog->aux->id,
 		   stats.nsecs,
-		   stats.cnt);
+		   stats.cnt,
+		   stats.misses);
 }
 #endif
 	bpf_prog_get_stats(prog, &stats);
 	info.run_time_ns = stats.nsecs;
 	info.run_cnt = stats.cnt;
+	info.recursion_misses = stats.misses;
 	if (!bpf_capable()) {
 		info.jited_prog_len = 0;

kernel/bpf/task_iter.c+266−1

@@ -286,9 +286,248 @@ static const struct seq_operations task_file_seq_ops = { · 285 unmodified lines
286	.show = task_file_seq_show,	286	.show = task_file_seq_show,
287	};	287	};
288		288
		289	struct bpf_iter_seq_task_vma_info {
		290	/* The first field must be struct bpf_iter_seq_task_common.
		291	* this is assumed by {init, fini}_seq_pidns() callback functions.
		292	*/
		293	struct bpf_iter_seq_task_common common;
		294	struct task_struct *task;
		295	struct vm_area_struct *vma;
		296	u32 tid;
		297	unsigned long prev_vm_start;
		298	unsigned long prev_vm_end;
		299	};
		300
		301	enum bpf_task_vma_iter_find_op {
		302	task_vma_iter_first_vma, /* use mm->mmap */
		303	task_vma_iter_next_vma, /* use curr_vma->vm_next */
		304	task_vma_iter_find_vma, /* use find_vma() to find next vma */
		305	};
		306
		307	static struct vm_area_struct *
		308	task_vma_seq_get_next(struct bpf_iter_seq_task_vma_info *info)
		309	{
		310	struct pid_namespace *ns = info->common.ns;
		311	enum bpf_task_vma_iter_find_op op;
		312	struct vm_area_struct *curr_vma;
		313	struct task_struct *curr_task;
		314	u32 curr_tid = info->tid;
		315
		316	/* If this function returns a non-NULL vma, it holds a reference to
		317	* the task_struct, and holds read lock on vma->mm->mmap_lock.
		318	* If this function returns NULL, it does not hold any reference or
		319	* lock.
		320	*/
		321	if (info->task) {
		322	curr_task = info->task;
		323	curr_vma = info->vma;
		324	/* In case of lock contention, drop mmap_lock to unblock
		325	* the writer.
		326	*
		327	* After relock, call find(mm, prev_vm_end - 1) to find
		328	* new vma to process.
		329	*
		330	* +------+------+-----------+
		331	* \| VMA1 \| VMA2 \| VMA3 \|
		332	* +------+------+-----------+
		333	* \| \| \| \|
		334	* 4k 8k 16k 400k
		335	*
		336	* For example, curr_vma == VMA2. Before unlock, we set
		337	*
		338	* prev_vm_start = 8k
		339	* prev_vm_end = 16k
		340	*
		341	* There are a few cases:
		342	*
		343	* 1) VMA2 is freed, but VMA3 exists.
		344	*
		345	* find_vma() will return VMA3, just process VMA3.
		346	*
		347	* 2) VMA2 still exists.
		348	*
		349	* find_vma() will return VMA2, process VMA2->next.
		350	*
		351	* 3) no more vma in this mm.
		352	*
		353	* Process the next task.
		354	*
		355	* 4) find_vma() returns a different vma, VMA2'.
		356	*
		357	* 4.1) If VMA2 covers same range as VMA2', skip VMA2',
		358	* because we already covered the range;
		359	* 4.2) VMA2 and VMA2' covers different ranges, process
		360	* VMA2'.
		361	*/
		362	if (mmap_lock_is_contended(curr_task->mm)) {
		363	info->prev_vm_start = curr_vma->vm_start;
		364	info->prev_vm_end = curr_vma->vm_end;
		365	op = task_vma_iter_find_vma;
		366	mmap_read_unlock(curr_task->mm);
		367	if (mmap_read_lock_killable(curr_task->mm))
		368	goto finish;
		369	} else {
		370	op = task_vma_iter_next_vma;
		371	}
		372	} else {
		373	again:
		374	curr_task = task_seq_get_next(ns, &curr_tid, true);
		375	if (!curr_task) {
		376	info->tid = curr_tid + 1;
		377	goto finish;
		378	}
		379
		380	if (curr_tid != info->tid) {
		381	info->tid = curr_tid;
		382	/* new task, process the first vma */
		383	op = task_vma_iter_first_vma;
		384	} else {
		385	/* Found the same tid, which means the user space
		386	* finished data in previous buffer and read more.
		387	* We dropped mmap_lock before returning to user
		388	* space, so it is necessary to use find_vma() to
		389	* find the next vma to process.
		390	*/
		391	op = task_vma_iter_find_vma;
		392	}
		393
		394	if (!curr_task->mm)
		395	goto next_task;
		396
		397	if (mmap_read_lock_killable(curr_task->mm))
		398	goto finish;
		399	}
		400
		401	switch (op) {
		402	case task_vma_iter_first_vma:
		403	curr_vma = curr_task->mm->mmap;
		404	break;
		405	case task_vma_iter_next_vma:
		406	curr_vma = curr_vma->vm_next;
		407	break;
		408	case task_vma_iter_find_vma:
		409	/* We dropped mmap_lock so it is necessary to use find_vma
		410	* to find the next vma. This is similar to the mechanism
		411	* in show_smaps_rollup().
		412	*/
		413	curr_vma = find_vma(curr_task->mm, info->prev_vm_end - 1);
		414	/* case 1) and 4.2) above just use curr_vma */
		415
		416	/* check for case 2) or case 4.1) above */
		417	if (curr_vma &&
		418	curr_vma->vm_start == info->prev_vm_start &&
		419	curr_vma->vm_end == info->prev_vm_end)
		420	curr_vma = curr_vma->vm_next;
		421	break;
		422	}
		423	if (!curr_vma) {
		424	/* case 3) above, or case 2) 4.1) with vma->next == NULL */
		425	mmap_read_unlock(curr_task->mm);
		426	goto next_task;
		427	}
		428	info->task = curr_task;
		429	info->vma = curr_vma;
		430	return curr_vma;
		431
		432	next_task:
		433	put_task_struct(curr_task);
		434	info->task = NULL;
		435	curr_tid++;
		436	goto again;
		437
		438	finish:
		439	if (curr_task)
		440	put_task_struct(curr_task);
		441	info->task = NULL;
		442	info->vma = NULL;
		443	return NULL;
		444	}
		445
		446	static void task_vma_seq_start(struct seq_file seq, loff_t *pos)
		447	{
		448	struct bpf_iter_seq_task_vma_info *info = seq->private;
		449	struct vm_area_struct *vma;
		450
		451	vma = task_vma_seq_get_next(info);
		452	if (vma && *pos == 0)
		453	++*pos;
		454
		455	return vma;
		456	}
		457
		458	static void task_vma_seq_next(struct seq_file seq, void v, loff_t pos)
		459	{
		460	struct bpf_iter_seq_task_vma_info *info = seq->private;
		461
		462	++*pos;
		463	return task_vma_seq_get_next(info);
		464	}
		465
		466	struct bpf_iter__task_vma {
		467	__bpf_md_ptr(struct bpf_iter_meta *, meta);
		468	__bpf_md_ptr(struct task_struct *, task);
		469	__bpf_md_ptr(struct vm_area_struct *, vma);
		470	};
		471
		472	DEFINE_BPF_ITER_FUNC(task_vma, struct bpf_iter_meta *meta,
		473	struct task_struct task, struct vm_area_struct vma)
		474
		475	static int __task_vma_seq_show(struct seq_file *seq, bool in_stop)
		476	{
		477	struct bpf_iter_seq_task_vma_info *info = seq->private;
		478	struct bpf_iter__task_vma ctx;
		479	struct bpf_iter_meta meta;
		480	struct bpf_prog *prog;
		481
		482	meta.seq = seq;
		483	prog = bpf_iter_get_info(&meta, in_stop);
		484	if (!prog)
		485	return 0;
		486
		487	ctx.meta = &meta;
		488	ctx.task = info->task;
		489	ctx.vma = info->vma;
		490	return bpf_iter_run_prog(prog, &ctx);
		491	}
		492
		493	static int task_vma_seq_show(struct seq_file seq, void v)
		494	{
		495	return __task_vma_seq_show(seq, false);
		496	}
		497
		498	static void task_vma_seq_stop(struct seq_file seq, void v)
		499	{
		500	struct bpf_iter_seq_task_vma_info *info = seq->private;
		501
		502	if (!v) {
		503	(void)__task_vma_seq_show(seq, true);
		504	} else {
		505	/* info->vma has not been seen by the BPF program. If the
		506	* user space reads more, task_vma_seq_get_next should
		507	* return this vma again. Set prev_vm_start to ~0UL,
		508	* so that we don't skip the vma returned by the next
		509	* find_vma() (case task_vma_iter_find_vma in
		510	* task_vma_seq_get_next()).
		511	*/
		512	info->prev_vm_start = ~0UL;
		513	info->prev_vm_end = info->vma->vm_end;
		514	mmap_read_unlock(info->task->mm);
		515	put_task_struct(info->task);
		516	info->task = NULL;
		517	}
		518	}
		519
		520	static const struct seq_operations task_vma_seq_ops = {
		521	.start = task_vma_seq_start,
		522	.next = task_vma_seq_next,
		523	.stop = task_vma_seq_stop,
		524	.show = task_vma_seq_show,
		525	};
		526
289	BTF_ID_LIST(btf_task_file_ids)	527	BTF_ID_LIST(btf_task_file_ids)
290	BTF_ID(struct, task_struct)	528	BTF_ID(struct, task_struct)
291	BTF_ID(struct, file)	529	BTF_ID(struct, file)
		530	BTF_ID(struct, vm_area_struct)
292		531
293	static const struct bpf_iter_seq_info task_seq_info = {	532	static const struct bpf_iter_seq_info task_seq_info = {
294	.seq_ops = &task_seq_ops,	533	.seq_ops = &task_seq_ops,
@@ -328,6 +567,26 @@ static struct bpf_iter_reg task_file_reg_info = { · 33 unmodified lines
328	.seq_info = &task_file_seq_info,	567	.seq_info = &task_file_seq_info,
329	};	568	};
330		569
		570	static const struct bpf_iter_seq_info task_vma_seq_info = {
		571	.seq_ops = &task_vma_seq_ops,
		572	.init_seq_private = init_seq_pidns,
		573	.fini_seq_private = fini_seq_pidns,
		574	.seq_priv_size = sizeof(struct bpf_iter_seq_task_vma_info),
		575	};
		576
		577	static struct bpf_iter_reg task_vma_reg_info = {
		578	.target = "task_vma",
		579	.feature = BPF_ITER_RESCHED,
		580	.ctx_arg_info_size = 2,
		581	.ctx_arg_info = {
		582	{ offsetof(struct bpf_iter__task_vma, task),
		583	PTR_TO_BTF_ID_OR_NULL },
		584	{ offsetof(struct bpf_iter__task_vma, vma),
		585	PTR_TO_BTF_ID_OR_NULL },
		586	},
		587	.seq_info = &task_vma_seq_info,
		588	};
		589
331	static int __init task_iter_init(void)	590	static int __init task_iter_init(void)
332	{	591	{
333	int ret;	592	int ret;
@@ -339,6 +598,12 @@ static int __init task_iter_init(void) · 5 unmodified lines
339		598
340	task_file_reg_info.ctx_arg_info[0].btf_id = btf_task_file_ids[0];	599	task_file_reg_info.ctx_arg_info[0].btf_id = btf_task_file_ids[0];
341	task_file_reg_info.ctx_arg_info[1].btf_id = btf_task_file_ids[1];	600	task_file_reg_info.ctx_arg_info[1].btf_id = btf_task_file_ids[1];
342	return bpf_iter_reg_target(&task_file_reg_info);	601	ret = bpf_iter_reg_target(&task_file_reg_info);
		602	if (ret)
		603	return ret;
		604
		605	task_vma_reg_info.ctx_arg_info[0].btf_id = btf_task_file_ids[0];
		606	task_vma_reg_info.ctx_arg_info[1].btf_id = btf_task_file_ids[2];
		607	return bpf_iter_reg_target(&task_vma_reg_info);
343	}	608	}
344	late_initcall(task_iter_init);	609	late_initcall(task_iter_init);

kernel/bpf/trampoline.c+61−16

 	mutex_unlock(&trampoline_mutex);
 }
+#define NO_START_TIME 1
+static u64 notrace bpf_prog_start_time(void)
+{
+	u64 start = NO_START_TIME;
+	if (static_branch_unlikely(&bpf_stats_enabled_key)) {
+		start = sched_clock();
+		if (unlikely(!start))
+			start = NO_START_TIME;
+	}
+	return start;
+}
+static void notrace inc_misses_counter(struct bpf_prog *prog)
+{
+	struct bpf_prog_stats *stats;
+	stats = this_cpu_ptr(prog->stats);
+	u64_stats_update_begin(&stats->syncp);
+	stats->misses++;
+	u64_stats_update_end(&stats->syncp);
+}
 /* The logic is similar to BPF_PROG_RUN, but with an explicit
  * rcu_read_lock() and migrate_disable() which are required
  * for the trampoline. The macro is split into
- * call _bpf_prog_enter
+ * call __bpf_prog_enter
  * call prog->bpf_func
  * call __bpf_prog_exit
+ *
+ * __bpf_prog_enter returns:
+ * 0 - skip execution of the bpf prog
+ * 1 - execute bpf prog
+ * [2..MAX_U64] - excute bpf prog and record execution time.
+ *     This is start time.
  */
-u64 notrace __bpf_prog_enter(void)
+u64 notrace __bpf_prog_enter(struct bpf_prog *prog)
 	__acquires(RCU)
 {
-	u64 start = 0;
 	rcu_read_lock();
 	migrate_disable();
-	if (static_branch_unlikely(&bpf_stats_enabled_key))
+	if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) {
-		start = sched_clock();
+		inc_misses_counter(prog);
-	return start;
+		return 0;
+	}
+	return bpf_prog_start_time();
 }
-void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start)
+static void notrace update_prog_stats(struct bpf_prog *prog,
-	__releases(RCU)
+				      u64 start)
 {
 	struct bpf_prog_stats *stats;
 	if (static_branch_unlikely(&bpf_stats_enabled_key) &&
-	    /* static_key could be enabled in __bpf_prog_enter
+	    /* static_key could be enabled in __bpf_prog_enter*
-	     * and disabled in __bpf_prog_exit.
+	     * and disabled in __bpf_prog_exit*.
 	     * And vice versa.
-	     * Hence check that 'start' is not zero.
+	     * Hence check that 'start' is valid.
 	     */
-	    start) {
+	    start > NO_START_TIME) {
-		stats = this_cpu_ptr(prog->aux->stats);
+		stats = this_cpu_ptr(prog->stats);
 		u64_stats_update_begin(&stats->syncp);
 		stats->cnt++;
 		stats->nsecs += sched_clock() - start;
 		u64_stats_update_end(&stats->syncp);
 	}
+}
+void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start)
+	__releases(RCU)
+{
+	update_prog_stats(prog, start);
+	__this_cpu_dec(*(prog->active));
 	migrate_enable();
 	rcu_read_unlock();
 }
-void notrace __bpf_prog_enter_sleepable(void)
+u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog)
 {
 	rcu_read_lock_trace();
+	migrate_disable();
 	might_fault();
+	if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) {
+		inc_misses_counter(prog);
+		return 0;
+	}
+	return bpf_prog_start_time();
 }
-void notrace __bpf_prog_exit_sleepable(void)
+void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start)
 {
+	update_prog_stats(prog, start);
+	__this_cpu_dec(*(prog->active));
+	migrate_enable();
 	rcu_read_unlock_trace();
 }

kernel/bpf/verifier.c+662−215

@@ -228,6 +228,12 @@ static void bpf_map_key_store(struct bpf_insn_aux_data *aux, u64 state) · 227 unmodified lines
228	(poisoned ? BPF_MAP_KEY_POISON : 0ULL);	228	(poisoned ? BPF_MAP_KEY_POISON : 0ULL);
229	}	229	}
230		230
		231	static bool bpf_pseudo_call(const struct bpf_insn *insn)
		232	{
		233	return insn->code == (BPF_JMP \| BPF_CALL) &&
		234	insn->src_reg == BPF_PSEUDO_CALL;
		235	}
		236
231	struct bpf_call_arg_meta {	237	struct bpf_call_arg_meta {
232	struct bpf_map *map_ptr;	238	struct bpf_map *map_ptr;
233	bool raw_mode;	239	bool raw_mode;
@@ -1073,6 +1079,51 @@ static void mark_reg_known_zero(struct bpf_verifier_env *env, · 839 unmodified lines
1073	__mark_reg_known_zero(regs + regno);	1079	__mark_reg_known_zero(regs + regno);
1074	}	1080	}
1075		1081
		1082	static void mark_ptr_not_null_reg(struct bpf_reg_state *reg)
		1083	{
		1084	switch (reg->type) {
		1085	case PTR_TO_MAP_VALUE_OR_NULL: {
		1086	const struct bpf_map *map = reg->map_ptr;
		1087
		1088	if (map->inner_map_meta) {
		1089	reg->type = CONST_PTR_TO_MAP;
		1090	reg->map_ptr = map->inner_map_meta;
		1091	} else if (map->map_type == BPF_MAP_TYPE_XSKMAP) {
		1092	reg->type = PTR_TO_XDP_SOCK;
		1093	} else if (map->map_type == BPF_MAP_TYPE_SOCKMAP \|\|
		1094	map->map_type == BPF_MAP_TYPE_SOCKHASH) {
		1095	reg->type = PTR_TO_SOCKET;
		1096	} else {
		1097	reg->type = PTR_TO_MAP_VALUE;
		1098	}
		1099	break;
		1100	}
		1101	case PTR_TO_SOCKET_OR_NULL:
		1102	reg->type = PTR_TO_SOCKET;
		1103	break;
		1104	case PTR_TO_SOCK_COMMON_OR_NULL:
		1105	reg->type = PTR_TO_SOCK_COMMON;
		1106	break;
		1107	case PTR_TO_TCP_SOCK_OR_NULL:
		1108	reg->type = PTR_TO_TCP_SOCK;
		1109	break;
		1110	case PTR_TO_BTF_ID_OR_NULL:
		1111	reg->type = PTR_TO_BTF_ID;
		1112	break;
		1113	case PTR_TO_MEM_OR_NULL:
		1114	reg->type = PTR_TO_MEM;
		1115	break;
		1116	case PTR_TO_RDONLY_BUF_OR_NULL:
		1117	reg->type = PTR_TO_RDONLY_BUF;
		1118	break;
		1119	case PTR_TO_RDWR_BUF_OR_NULL:
		1120	reg->type = PTR_TO_RDWR_BUF;
		1121	break;
		1122	default:
		1123	WARN_ON("unknown nullable register type");
		1124	}
		1125	}
		1126
1076	static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg)	1127	static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg)
1077	{	1128	{
1078	return type_is_pkt_pointer(reg->type);	1129	return type_is_pkt_pointer(reg->type);
@@ -1486,9 +1537,7 @@ static int check_subprogs(struct bpf_verifier_env *env) · 407 unmodified lines
1486		1537
1487	/* determine subprog starts. The end is one before the next starts */	1538	/* determine subprog starts. The end is one before the next starts */
1488	for (i = 0; i < insn_cnt; i++) {	1539	for (i = 0; i < insn_cnt; i++) {
1489	if (insn[i].code != (BPF_JMP \| BPF_CALL))	1540	if (!bpf_pseudo_call(insn + i))
1490	continue;
1491	if (insn[i].src_reg != BPF_PSEUDO_CALL)
1492	continue;	1541	continue;
1493	if (!env->bpf_capable) {	1542	if (!env->bpf_capable) {
1494	verbose(env,	1543	verbose(env,
@@ -2271,12 +2320,14 @@ static void save_register_state(struct bpf_func_state *state, · 776 unmodified lines
2271	state->stack[spi].slot_type[i] = STACK_SPILL;	2320	state->stack[spi].slot_type[i] = STACK_SPILL;
2272	}	2321	}
2273		2322
2274	/* check_stack_read/write functions track spill/fill of registers,	2323	/* check_stack_{read,write}_fixed_off functions track spill/fill of registers,
2275	* stack boundary and alignment are checked in check_mem_access()	2324	* stack boundary and alignment are checked in check_mem_access()
2276	*/	2325	*/
2277	static int check_stack_write(struct bpf_verifier_env *env,	2326	static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
2278	struct bpf_func_state state, / func where register points to */	2327	/* stack frame we're writing to */
2279	int off, int size, int value_regno, int insn_idx)	2328	struct bpf_func_state *state,
		2329	int off, int size, int value_regno,
		2330	int insn_idx)
2280	{	2331	{
2281	struct bpf_func_state cur; / state of the current function */	2332	struct bpf_func_state cur; / state of the current function */
2282	int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err;	2333	int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err;
@@ -2402,9 +2453,175 @@ static int check_stack_write(struct bpf_verifier_env *env, · 119 unmodified lines
2402	return 0;	2453	return 0;
2403	}	2454	}
2404		2455
2405	static int check_stack_read(struct bpf_verifier_env *env,	2456	/* Write the stack: 'stack[ptr_regno + off] = value_regno'. 'ptr_regno' is
2406	struct bpf_func_state reg_state / func where register points to */,	2457	* known to contain a variable offset.
2407	int off, int size, int value_regno)	2458	* This function checks whether the write is permitted and conservatively
		2459	* tracks the effects of the write, considering that each stack slot in the
		2460	* dynamic range is potentially written to.
		2461	*
		2462	* 'off' includes 'regno->off'.
		2463	* 'value_regno' can be -1, meaning that an unknown value is being written to
		2464	* the stack.
		2465	*
		2466	* Spilled pointers in range are not marked as written because we don't know
		2467	* what's going to be actually written. This means that read propagation for
		2468	* future reads cannot be terminated by this write.
		2469	*
		2470	* For privileged programs, uninitialized stack slots are considered
		2471	* initialized by this write (even though we don't know exactly what offsets
		2472	* are going to be written to). The idea is that we don't want the verifier to
		2473	* reject future reads that access slots written to through variable offsets.
		2474	*/
		2475	static int check_stack_write_var_off(struct bpf_verifier_env *env,
		2476	/* func where register points to */
		2477	struct bpf_func_state *state,
		2478	int ptr_regno, int off, int size,
		2479	int value_regno, int insn_idx)
		2480	{
		2481	struct bpf_func_state cur; / state of the current function */
		2482	int min_off, max_off;
		2483	int i, err;
		2484	struct bpf_reg_state ptr_reg = NULL, value_reg = NULL;
		2485	bool writing_zero = false;
		2486	/* set if the fact that we're writing a zero is used to let any
		2487	* stack slots remain STACK_ZERO
		2488	*/
		2489	bool zero_used = false;
		2490
		2491	cur = env->cur_state->frame[env->cur_state->curframe];
		2492	ptr_reg = &cur->regs[ptr_regno];
		2493	min_off = ptr_reg->smin_value + off;
		2494	max_off = ptr_reg->smax_value + off + size;
		2495	if (value_regno >= 0)
		2496	value_reg = &cur->regs[value_regno];
		2497	if (value_reg && register_is_null(value_reg))
		2498	writing_zero = true;
		2499
		2500	err = realloc_func_state(state, round_up(-min_off, BPF_REG_SIZE),
		2501	state->acquired_refs, true);
		2502	if (err)
		2503	return err;
		2504
		2505
		2506	/* Variable offset writes destroy any spilled pointers in range. */
		2507	for (i = min_off; i < max_off; i++) {
		2508	u8 new_type, *stype;
		2509	int slot, spi;
		2510
		2511	slot = -i - 1;
		2512	spi = slot / BPF_REG_SIZE;
		2513	stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE];
		2514
		2515	if (!env->allow_ptr_leaks
		2516	&& *stype != NOT_INIT
		2517	&& *stype != SCALAR_VALUE) {
		2518	/* Reject the write if there's are spilled pointers in
		2519	* range. If we didn't reject here, the ptr status
		2520	* would be erased below (even though not all slots are
		2521	* actually overwritten), possibly opening the door to
		2522	* leaks.
		2523	*/
		2524	verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d",
		2525	insn_idx, i);
		2526	return -EINVAL;
		2527	}
		2528
		2529	/* Erase all spilled pointers. */
		2530	state->stack[spi].spilled_ptr.type = NOT_INIT;
		2531
		2532	/* Update the slot type. */
		2533	new_type = STACK_MISC;
		2534	if (writing_zero && *stype == STACK_ZERO) {
		2535	new_type = STACK_ZERO;
		2536	zero_used = true;
		2537	}
		2538	/* If the slot is STACK_INVALID, we check whether it's OK to
		2539	* pretend that it will be initialized by this write. The slot
		2540	* might not actually be written to, and so if we mark it as
		2541	* initialized future reads might leak uninitialized memory.
		2542	* For privileged programs, we will accept such reads to slots
		2543	* that may or may not be written because, if we're reject
		2544	* them, the error would be too confusing.
		2545	*/
		2546	if (*stype == STACK_INVALID && !env->allow_uninit_stack) {
		2547	verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d",
		2548	insn_idx, i);
		2549	return -EINVAL;
		2550	}
		2551	*stype = new_type;
		2552	}
		2553	if (zero_used) {
		2554	/* backtracking doesn't work for STACK_ZERO yet. */
		2555	err = mark_chain_precision(env, value_regno);
		2556	if (err)
		2557	return err;
		2558	}
		2559	return 0;
		2560	}
		2561
		2562	/* When register 'dst_regno' is assigned some values from stack[min_off,
		2563	* max_off), we set the register's type according to the types of the
		2564	* respective stack slots. If all the stack values are known to be zeros, then
		2565	* so is the destination reg. Otherwise, the register is considered to be
		2566	* SCALAR. This function does not deal with register filling; the caller must
		2567	* ensure that all spilled registers in the stack range have been marked as
		2568	* read.
		2569	*/
		2570	static void mark_reg_stack_read(struct bpf_verifier_env *env,
		2571	/* func where src register points to */
		2572	struct bpf_func_state *ptr_state,
		2573	int min_off, int max_off, int dst_regno)
		2574	{
		2575	struct bpf_verifier_state *vstate = env->cur_state;
		2576	struct bpf_func_state *state = vstate->frame[vstate->curframe];
		2577	int i, slot, spi;
		2578	u8 *stype;
		2579	int zeros = 0;
		2580
		2581	for (i = min_off; i < max_off; i++) {
		2582	slot = -i - 1;
		2583	spi = slot / BPF_REG_SIZE;
		2584	stype = ptr_state->stack[spi].slot_type;
		2585	if (stype[slot % BPF_REG_SIZE] != STACK_ZERO)
		2586	break;
		2587	zeros++;
		2588	}
		2589	if (zeros == max_off - min_off) {
		2590	/* any access_size read into register is zero extended,
		2591	* so the whole register == const_zero
		2592	*/
		2593	__mark_reg_const_zero(&state->regs[dst_regno]);
		2594	/* backtracking doesn't support STACK_ZERO yet,
		2595	* so mark it precise here, so that later
		2596	* backtracking can stop here.
		2597	* Backtracking may not need this if this register
		2598	* doesn't participate in pointer adjustment.
		2599	* Forward propagation of precise flag is not
		2600	* necessary either. This mark is only to stop
		2601	* backtracking. Any register that contributed
		2602	* to const 0 was marked precise before spill.
		2603	*/
		2604	state->regs[dst_regno].precise = true;
		2605	} else {
		2606	/* have read misc data from the stack */
		2607	mark_reg_unknown(env, state->regs, dst_regno);
		2608	}
		2609	state->regs[dst_regno].live \|= REG_LIVE_WRITTEN;
		2610	}
		2611
		2612	/* Read the stack at 'off' and put the results into the register indicated by
		2613	* 'dst_regno'. It handles reg filling if the addressed stack slot is a
		2614	* spilled reg.
		2615	*
		2616	* 'dst_regno' can be -1, meaning that the read value is not going to a
		2617	* register.
		2618	*
		2619	* The access is assumed to be within the current stack bounds.
		2620	*/
		2621	static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
		2622	/* func where src register points to */
		2623	struct bpf_func_state *reg_state,
		2624	int off, int size, int dst_regno)
2408	{	2625	{
2409	struct bpf_verifier_state *vstate = env->cur_state;	2626	struct bpf_verifier_state *vstate = env->cur_state;
2410	struct bpf_func_state *state = vstate->frame[vstate->curframe];	2627	struct bpf_func_state *state = vstate->frame[vstate->curframe];
@@ -2412,11 +2629,6 @@ static int check_stack_read(struct bpf_verifier_env *env, · 1 unmodified line
2412	struct bpf_reg_state *reg;	2629	struct bpf_reg_state *reg;
2413	u8 *stype;	2630	u8 *stype;
2414		2631
2415	if (reg_state->allocated_stack <= slot) {
2416	verbose(env, "invalid read from stack off %d+0 size %d\n",
2417	off, size);
2418	return -EACCES;
2419	}
2420	stype = reg_state->stack[spi].slot_type;	2632	stype = reg_state->stack[spi].slot_type;
2421	reg = &reg_state->stack[spi].spilled_ptr;	2633	reg = &reg_state->stack[spi].spilled_ptr;
2422		2634
@@ -2427,9 +2639,9 @@ static int check_stack_read(struct bpf_verifier_env *env, · 4 unmodified lines
2427	verbose(env, "invalid size of register fill\n");	2639	verbose(env, "invalid size of register fill\n");
2428	return -EACCES;	2640	return -EACCES;
2429	}	2641	}
2430	if (value_regno >= 0) {	2642	if (dst_regno >= 0) {
2431	mark_reg_unknown(env, state->regs, value_regno);	2643	mark_reg_unknown(env, state->regs, dst_regno);
2432	state->regs[value_regno].live \|= REG_LIVE_WRITTEN;	2644	state->regs[dst_regno].live \|= REG_LIVE_WRITTEN;
2433	}	2645	}
2434	mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);	2646	mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
2435	return 0;	2647	return 0;
@@ -2441,16 +2653,16 @@ static int check_stack_read(struct bpf_verifier_env *env, · 5 unmodified lines
2441	}	2653	}
2442	}	2654	}
2443		2655
2444	if (value_regno >= 0) {	2656	if (dst_regno >= 0) {
2445	/* restore register state from stack */	2657	/* restore register state from stack */
2446	state->regs[value_regno] = *reg;	2658	state->regs[dst_regno] = *reg;
2447	/* mark reg as written since spilled pointer state likely	2659	/* mark reg as written since spilled pointer state likely
2448	* has its liveness marks cleared by is_state_visited()	2660	* has its liveness marks cleared by is_state_visited()
2449	* which resets stack/reg liveness for state transitions	2661	* which resets stack/reg liveness for state transitions
2450	*/	2662	*/
2451	state->regs[value_regno].live \|= REG_LIVE_WRITTEN;	2663	state->regs[dst_regno].live \|= REG_LIVE_WRITTEN;
2452	} else if (__is_pointer_value(env->allow_ptr_leaks, reg)) {	2664	} else if (__is_pointer_value(env->allow_ptr_leaks, reg)) {
2453	/* If value_regno==-1, the caller is asking us whether	2665	/* If dst_regno==-1, the caller is asking us whether
2454	* it is acceptable to use this value as a SCALAR_VALUE	2666	* it is acceptable to use this value as a SCALAR_VALUE
2455	* (e.g. for XADD).	2667	* (e.g. for XADD).
2456	* We must not allow unprivileged callers to do that	2668	* We must not allow unprivileged callers to do that
@@ -2462,70 +2674,167 @@ static int check_stack_read(struct bpf_verifier_env *env, · 5 unmodified lines
2462	}	2674	}
2463	mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);	2675	mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
2464	} else {	2676	} else {
2465	int zeros = 0;	2677	u8 type;
2466		2678
2467	for (i = 0; i < size; i++) {	2679	for (i = 0; i < size; i++) {
2468	if (stype[(slot - i) % BPF_REG_SIZE] == STACK_MISC)	2680	type = stype[(slot - i) % BPF_REG_SIZE];
		2681	if (type == STACK_MISC)
2469	continue;	2682	continue;
2470	if (stype[(slot - i) % BPF_REG_SIZE] == STACK_ZERO) {	2683	if (type == STACK_ZERO)
2471	zeros++;
2472	continue;	2684	continue;
2473	}
2474	verbose(env, "invalid read from stack off %d+%d size %d\n",	2685	verbose(env, "invalid read from stack off %d+%d size %d\n",
2475	off, i, size);	2686	off, i, size);
2476	return -EACCES;	2687	return -EACCES;
2477	}	2688	}
2478	mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);	2689	mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
2479	if (value_regno >= 0) {	2690	if (dst_regno >= 0)
2480	if (zeros == size) {	2691	mark_reg_stack_read(env, reg_state, off, off + size, dst_regno);
2481	/* any size read into register is zero extended,
2482	* so the whole register == const_zero
2483	*/
2484	__mark_reg_const_zero(&state->regs[value_regno]);
2485	/* backtracking doesn't support STACK_ZERO yet,
2486	* so mark it precise here, so that later
2487	* backtracking can stop here.
2488	* Backtracking may not need this if this register
2489	* doesn't participate in pointer adjustment.
2490	* Forward propagation of precise flag is not
2491	* necessary either. This mark is only to stop
2492	* backtracking. Any register that contributed
2493	* to const 0 was marked precise before spill.
2494	*/
2495	state->regs[value_regno].precise = true;
2496	} else {
2497	/* have read misc data from the stack */
2498	mark_reg_unknown(env, state->regs, value_regno);
2499	}
2500	state->regs[value_regno].live \|= REG_LIVE_WRITTEN;
2501	}
2502	}	2692	}
2503	return 0;	2693	return 0;
2504	}	2694	}
2505		2695
2506	static int check_stack_access(struct bpf_verifier_env *env,	2696	enum stack_access_src {
2507	const struct bpf_reg_state *reg,	2697	ACCESS_DIRECT = 1, /* the access is performed by an instruction */
2508	int off, int size)	2698	ACCESS_HELPER = 2, /* the access is performed by a helper */
		2699	};
		2700
		2701	static int check_stack_range_initialized(struct bpf_verifier_env *env,
		2702	int regno, int off, int access_size,
		2703	bool zero_size_allowed,
		2704	enum stack_access_src type,
		2705	struct bpf_call_arg_meta *meta);
		2706
		2707	static struct bpf_reg_state reg_state(struct bpf_verifier_env env, int regno)
		2708	{
		2709	return cur_regs(env) + regno;
		2710	}
		2711
		2712	/* Read the stack at 'ptr_regno + off' and put the result into the register
		2713	* 'dst_regno'.
		2714	* 'off' includes the pointer register's fixed offset(i.e. 'ptr_regno.off'),
		2715	* but not its variable offset.
		2716	* 'size' is assumed to be <= reg size and the access is assumed to be aligned.
		2717	*
		2718	* As opposed to check_stack_read_fixed_off, this function doesn't deal with
		2719	* filling registers (i.e. reads of spilled register cannot be detected when
		2720	* the offset is not fixed). We conservatively mark 'dst_regno' as containing
		2721	* SCALAR_VALUE. That's why we assert that the 'ptr_regno' has a variable
		2722	* offset; for a fixed offset check_stack_read_fixed_off should be used
		2723	* instead.
		2724	*/
		2725	static int check_stack_read_var_off(struct bpf_verifier_env *env,
		2726	int ptr_regno, int off, int size, int dst_regno)
2509	{	2727	{
2510	/* Stack accesses must be at a fixed offset, so that we	2728	/* The state of the source register. */
2511	* can determine what type of data were returned. See	2729	struct bpf_reg_state *reg = reg_state(env, ptr_regno);
2512	* check_stack_read().	2730	struct bpf_func_state *ptr_state = func(env, reg);
		2731	int err;
		2732	int min_off, max_off;
		2733
		2734	/* Note that we pass a NULL meta, so raw access will not be permitted.
2513	*/	2735	*/
2514	if (!tnum_is_const(reg->var_off)) {	2736	err = check_stack_range_initialized(env, ptr_regno, off, size,
		2737	false, ACCESS_DIRECT, NULL);
		2738	if (err)
		2739	return err;
		2740
		2741	min_off = reg->smin_value + off;
		2742	max_off = reg->smax_value + off;
		2743	mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno);
		2744	return 0;
		2745	}
		2746
		2747	/* check_stack_read dispatches to check_stack_read_fixed_off or
		2748	* check_stack_read_var_off.
		2749	*
		2750	* The caller must ensure that the offset falls within the allocated stack
		2751	* bounds.
		2752	*
		2753	* 'dst_regno' is a register which will receive the value from the stack. It
		2754	* can be -1, meaning that the read value is not going to a register.
		2755	*/
		2756	static int check_stack_read(struct bpf_verifier_env *env,
		2757	int ptr_regno, int off, int size,
		2758	int dst_regno)
		2759	{
		2760	struct bpf_reg_state *reg = reg_state(env, ptr_regno);
		2761	struct bpf_func_state *state = func(env, reg);
		2762	int err;
		2763	/* Some accesses are only permitted with a static offset. */
		2764	bool var_off = !tnum_is_const(reg->var_off);
		2765
		2766	/* The offset is required to be static when reads don't go to a
		2767	* register, in order to not leak pointers (see
		2768	* check_stack_read_fixed_off).
		2769	*/
		2770	if (dst_regno < 0 && var_off) {
2515	char tn_buf[48];	2771	char tn_buf[48];
2516		2772
2517	tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);	2773	tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
2518	verbose(env, "variable stack access var_off=%s off=%d size=%d\n",	2774	verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n",
2519	tn_buf, off, size);	2775	tn_buf, off, size);
2520	return -EACCES;	2776	return -EACCES;
2521	}	2777	}
		2778	/* Variable offset is prohibited for unprivileged mode for simplicity
		2779	* since it requires corresponding support in Spectre masking for stack
		2780	* ALU. See also retrieve_ptr_limit().
		2781	*/
		2782	if (!env->bypass_spec_v1 && var_off) {
		2783	char tn_buf[48];
2522		2784
2523	if (off >= 0 \|\| off < -MAX_BPF_STACK) {	2785	tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
2524	verbose(env, "invalid stack off=%d size=%d\n", off, size);	2786	verbose(env, "R%d variable offset stack access prohibited for !root, var_off=%s\n",
		2787	ptr_regno, tn_buf);
2525	return -EACCES;	2788	return -EACCES;
2526	}	2789	}
2527		2790
2528	return 0;	2791	if (!var_off) {
		2792	off += reg->var_off.value;
		2793	err = check_stack_read_fixed_off(env, state, off, size,
		2794	dst_regno);
		2795	} else {
		2796	/* Variable offset stack reads need more conservative handling
		2797	* than fixed offset ones. Note that dst_regno >= 0 on this
		2798	* branch.
		2799	*/
		2800	err = check_stack_read_var_off(env, ptr_regno, off, size,
		2801	dst_regno);
		2802	}
		2803	return err;
		2804	}
		2805
		2806
		2807	/* check_stack_write dispatches to check_stack_write_fixed_off or
		2808	* check_stack_write_var_off.
		2809	*
		2810	* 'ptr_regno' is the register used as a pointer into the stack.
		2811	* 'off' includes 'ptr_regno->off', but not its variable offset (if any).
		2812	* 'value_regno' is the register whose value we're writing to the stack. It can
		2813	* be -1, meaning that we're not writing from a register.
		2814	*
		2815	* The caller must ensure that the offset falls within the maximum stack size.
		2816	*/
		2817	static int check_stack_write(struct bpf_verifier_env *env,
		2818	int ptr_regno, int off, int size,
		2819	int value_regno, int insn_idx)
		2820	{
		2821	struct bpf_reg_state *reg = reg_state(env, ptr_regno);
		2822	struct bpf_func_state *state = func(env, reg);
		2823	int err;
		2824
		2825	if (tnum_is_const(reg->var_off)) {
		2826	off += reg->var_off.value;
		2827	err = check_stack_write_fixed_off(env, state, off, size,
		2828	value_regno, insn_idx);
		2829	} else {
		2830	/* Variable offset stack reads need more conservative handling
		2831	* than fixed offset ones.
		2832	*/
		2833	err = check_stack_write_var_off(env, state,
		2834	ptr_regno, off, size,
		2835	value_regno, insn_idx);
		2836	}
		2837	return err;
2529	}	2838	}
2530		2839
2531	static int check_map_access_type(struct bpf_verifier_env *env, u32 regno,	2840	static int check_map_access_type(struct bpf_verifier_env *env, u32 regno,
@@ -2858,11 +3167,6 @@ static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, · 326 unmodified lines
2858	return -EACCES;	3167	return -EACCES;
2859	}	3168	}
2860		3169
2861	static struct bpf_reg_state reg_state(struct bpf_verifier_env env, int regno)
2862	{
2863	return cur_regs(env) + regno;
2864	}
2865
2866	static bool is_pointer_value(struct bpf_verifier_env *env, int regno)	3170	static bool is_pointer_value(struct bpf_verifier_env *env, int regno)
2867	{	3171	{
2868	return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno));	3172	return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno));
@@ -2981,8 +3285,8 @@ static int check_ptr_alignment(struct bpf_verifier_env *env, · 112 unmodified lines
2981	break;	3285	break;
2982	case PTR_TO_STACK:	3286	case PTR_TO_STACK:
2983	pointer_desc = "stack ";	3287	pointer_desc = "stack ";
2984	/* The stack spill tracking logic in check_stack_write()	3288	/* The stack spill tracking logic in check_stack_write_fixed_off()
2985	* and check_stack_read() relies on stack accesses being	3289	* and check_stack_read_fixed_off() relies on stack accesses being
2986	* aligned.	3290	* aligned.
2987	*/	3291	*/
2988	strict = true;	3292	strict = true;
@@ -3074,9 +3378,7 @@ process_func: · 85 unmodified lines
3074	continue_func:	3378	continue_func:
3075	subprog_end = subprog[idx + 1].start;	3379	subprog_end = subprog[idx + 1].start;
3076	for (; i < subprog_end; i++) {	3380	for (; i < subprog_end; i++) {
3077	if (insn[i].code != (BPF_JMP \| BPF_CALL))	3381	if (!bpf_pseudo_call(insn + i))
3078	continue;
3079	if (insn[i].src_reg != BPF_PSEUDO_CALL)
3080	continue;	3382	continue;
3081	/* remember insn and function to return to */	3383	/* remember insn and function to return to */
3082	ret_insn[frame] = i + 1;	3384	ret_insn[frame] = i + 1;
@@ -3400,6 +3702,91 @@ static int check_ptr_to_map_access(struct bpf_verifier_env *env, · 317 unmodified lines
3400	return 0;	3702	return 0;
3401	}	3703	}
3402		3704
		3705	/* Check that the stack access at the given offset is within bounds. The
		3706	* maximum valid offset is -1.
		3707	*
		3708	* The minimum valid offset is -MAX_BPF_STACK for writes, and
		3709	* -state->allocated_stack for reads.
		3710	*/
		3711	static int check_stack_slot_within_bounds(int off,
		3712	struct bpf_func_state *state,
		3713	enum bpf_access_type t)
		3714	{
		3715	int min_valid_off;
		3716
		3717	if (t == BPF_WRITE)
		3718	min_valid_off = -MAX_BPF_STACK;
		3719	else
		3720	min_valid_off = -state->allocated_stack;
		3721
		3722	if (off < min_valid_off \|\| off > -1)
		3723	return -EACCES;
		3724	return 0;
		3725	}
		3726
		3727	/* Check that the stack access at 'regno + off' falls within the maximum stack
		3728	* bounds.
		3729	*
		3730	* 'off' includes `regno->offset`, but not its dynamic part (if any).
		3731	*/
		3732	static int check_stack_access_within_bounds(
		3733	struct bpf_verifier_env *env,
		3734	int regno, int off, int access_size,
		3735	enum stack_access_src src, enum bpf_access_type type)
		3736	{
		3737	struct bpf_reg_state *regs = cur_regs(env);
		3738	struct bpf_reg_state *reg = regs + regno;
		3739	struct bpf_func_state *state = func(env, reg);
		3740	int min_off, max_off;
		3741	int err;
		3742	char *err_extra;
		3743
		3744	if (src == ACCESS_HELPER)
		3745	/* We don't know if helpers are reading or writing (or both). */
		3746	err_extra = " indirect access to";
		3747	else if (type == BPF_READ)
		3748	err_extra = " read from";
		3749	else
		3750	err_extra = " write to";
		3751
		3752	if (tnum_is_const(reg->var_off)) {
		3753	min_off = reg->var_off.value + off;
		3754	if (access_size > 0)
		3755	max_off = min_off + access_size - 1;
		3756	else
		3757	max_off = min_off;
		3758	} else {
		3759	if (reg->smax_value >= BPF_MAX_VAR_OFF \|\|
		3760	reg->smin_value <= -BPF_MAX_VAR_OFF) {
		3761	verbose(env, "invalid unbounded variable-offset%s stack R%d\n",
		3762	err_extra, regno);
		3763	return -EACCES;
		3764	}
		3765	min_off = reg->smin_value + off;
		3766	if (access_size > 0)
		3767	max_off = reg->smax_value + off + access_size - 1;
		3768	else
		3769	max_off = min_off;
		3770	}
		3771
		3772	err = check_stack_slot_within_bounds(min_off, state, type);
		3773	if (!err)
		3774	err = check_stack_slot_within_bounds(max_off, state, type);
		3775
		3776	if (err) {
		3777	if (tnum_is_const(reg->var_off)) {
		3778	verbose(env, "invalid%s stack R%d off=%d size=%d\n",
		3779	err_extra, regno, off, access_size);
		3780	} else {
		3781	char tn_buf[48];
		3782
		3783	tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
		3784	verbose(env, "invalid variable-offset%s stack R%d var_off=%s size=%d\n",
		3785	err_extra, regno, tn_buf, access_size);
		3786	}
		3787	}
		3788	return err;
		3789	}
3403		3790
3404	/* check whether memory at (regno + off) is accessible for t = (read \| write)	3791	/* check whether memory at (regno + off) is accessible for t = (read \| write)
3405	* if t==write, value_regno is a register which value is stored into memory	3792	* if t==write, value_regno is a register which value is stored into memory
@@ -3515,8 +3902,8 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 109 unmodified lines
3515	}	3902	}
3516		3903
3517	} else if (reg->type == PTR_TO_STACK) {	3904	} else if (reg->type == PTR_TO_STACK) {
3518	off += reg->var_off.value;	3905	/* Basic bounds checks. */
3519	err = check_stack_access(env, reg, off, size);	3906	err = check_stack_access_within_bounds(env, regno, off, size, ACCESS_DIRECT, t);
3520	if (err)	3907	if (err)
3521	return err;	3908	return err;
3522		3909
@@ -3525,12 +3912,12 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 2 unmodified lines
3525	if (err)	3912	if (err)
3526	return err;	3913	return err;
3527		3914
3528	if (t == BPF_WRITE)	3915	if (t == BPF_READ)
3529	err = check_stack_write(env, state, off, size,	3916	err = check_stack_read(env, regno, off, size,
3530	value_regno, insn_idx);
3531	else
3532	err = check_stack_read(env, state, off, size,
3533	value_regno);	3917	value_regno);
		3918	else
		3919	err = check_stack_write(env, regno, off, size,
		3920	value_regno, insn_idx);
3534	} else if (reg_is_pkt_pointer(reg)) {	3921	} else if (reg_is_pkt_pointer(reg)) {
3535	if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) {	3922	if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) {
3536	verbose(env, "cannot write into packet\n");	3923	verbose(env, "cannot write into packet\n");
@@ -3665,9 +4052,26 @@ static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_i · 128 unmodified lines
3665	return -EACCES;	4052	return -EACCES;
3666	}	4053	}
3667		4054
		4055	if (insn->imm & BPF_FETCH) {
		4056	if (insn->imm == BPF_CMPXCHG)
		4057	load_reg = BPF_REG_0;
		4058	else
		4059	load_reg = insn->src_reg;
		4060
		4061	/* check and record load of old value */
		4062	err = check_reg_arg(env, load_reg, DST_OP);
		4063	if (err)
		4064	return err;
		4065	} else {
		4066	/* This instruction accesses a memory location but doesn't
		4067	* actually load it into a register.
		4068	*/
		4069	load_reg = -1;
		4070	}
		4071
3668	/* check whether we can read the memory */	4072	/* check whether we can read the memory */
3669	err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,	4073	err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
3670	BPF_SIZE(insn->code), BPF_READ, -1, true);	4074	BPF_SIZE(insn->code), BPF_READ, load_reg, true);
3671	if (err)	4075	if (err)
3672	return err;	4076	return err;
3673		4077
@@ -3677,65 +4081,56 @@ static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_i · 3 unmodified lines
3677	if (err)	4081	if (err)
3678	return err;	4082	return err;
3679		4083
3680	if (!(insn->imm & BPF_FETCH))
3681	return 0;
3682
3683	if (insn->imm == BPF_CMPXCHG)
3684	load_reg = BPF_REG_0;
3685	else
3686	load_reg = insn->src_reg;
3687
3688	/* check and record load of old value */
3689	err = check_reg_arg(env, load_reg, DST_OP);
3690	if (err)
3691	return err;
3692
3693	return 0;	4084	return 0;
3694	}	4085	}
3695		4086
3696	static int __check_stack_boundary(struct bpf_verifier_env *env, u32 regno,	4087	/* When register 'regno' is used to read the stack (either directly or through
3697	int off, int access_size,	4088	* a helper function) make sure that it's within stack boundary and, depending
3698	bool zero_size_allowed)	4089	* on the access type, that all elements of the stack are initialized.
		4090	*
		4091	* 'off' includes 'regno->off', but not its dynamic part (if any).
		4092	*
		4093	* All registers that have been spilled on the stack in the slots within the
		4094	* read offsets are marked as read.
		4095	*/
		4096	static int check_stack_range_initialized(
		4097	struct bpf_verifier_env *env, int regno, int off,
		4098	int access_size, bool zero_size_allowed,
		4099	enum stack_access_src type, struct bpf_call_arg_meta *meta)
3699	{	4100	{
3700	struct bpf_reg_state *reg = reg_state(env, regno);	4101	struct bpf_reg_state *reg = reg_state(env, regno);
		4102	struct bpf_func_state *state = func(env, reg);
		4103	int err, min_off, max_off, i, j, slot, spi;
		4104	char *err_extra = type == ACCESS_HELPER ? " indirect" : "";
		4105	enum bpf_access_type bounds_check_type;
		4106	/* Some accesses can write anything into the stack, others are
		4107	* read-only.
		4108	*/
		4109	bool clobber = false;
3701		4110
3702	if (off >= 0 \|\| off < -MAX_BPF_STACK \|\| off + access_size > 0 \|\|	4111	if (access_size == 0 && !zero_size_allowed) {
3703	access_size < 0 \|\| (access_size == 0 && !zero_size_allowed)) {	4112	verbose(env, "invalid zero-sized read\n");
3704	if (tnum_is_const(reg->var_off)) {
3705	verbose(env, "invalid stack type R%d off=%d access_size=%d\n",
3706	regno, off, access_size);
3707	} else {
3708	char tn_buf[48];
3709
3710	tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
3711	verbose(env, "invalid stack type R%d var_off=%s access_size=%d\n",
3712	regno, tn_buf, access_size);
3713	}
3714	return -EACCES;	4113	return -EACCES;
3715	}	4114	}
3716	return 0;
3717	}
3718		4115
3719	/* when register 'regno' is passed into function that will read 'access_size'	4116	if (type == ACCESS_HELPER) {
3720	* bytes from that pointer, make sure that it's within stack boundary	4117	/* The bounds checks for writes are more permissive than for
3721	* and all elements of stack are initialized.	4118	* reads. However, if raw_mode is not set, we'll do extra
3722	* Unlike most pointer bounds-checking functions, this one doesn't take an	4119	* checks below.
3723	* 'off' argument, so it has to add in reg->off itself.	4120	*/
3724	*/	4121	bounds_check_type = BPF_WRITE;
3725	static int check_stack_boundary(struct bpf_verifier_env *env, int regno,	4122	clobber = true;
3726	int access_size, bool zero_size_allowed,	4123	} else {
3727	struct bpf_call_arg_meta *meta)	4124	bounds_check_type = BPF_READ;
3728	{	4125	}
3729	struct bpf_reg_state *reg = reg_state(env, regno);	4126	err = check_stack_access_within_bounds(env, regno, off, access_size,
3730	struct bpf_func_state *state = func(env, reg);	4127	type, bounds_check_type);
3731	int err, min_off, max_off, i, j, slot, spi;	4128	if (err)
		4129	return err;
		4130
3732		4131
3733	if (tnum_is_const(reg->var_off)) {	4132	if (tnum_is_const(reg->var_off)) {
3734	min_off = max_off = reg->var_off.value + reg->off;	4133	min_off = max_off = reg->var_off.value + off;
3735	err = __check_stack_boundary(env, regno, min_off, access_size,
3736	zero_size_allowed);
3737	if (err)
3738	return err;
3739	} else {	4134	} else {
3740	/* Variable offset is prohibited for unprivileged mode for	4135	/* Variable offset is prohibited for unprivileged mode for
3741	* simplicity since it requires corresponding support in	4136	* simplicity since it requires corresponding support in
@@ -3746,8 +4141,8 @@ static int check_stack_boundary(struct bpf_verifier_env *env, int regno, · 4 unmodified lines
3746	char tn_buf[48];	4141	char tn_buf[48];
3747		4142
3748	tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);	4143	tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
3749	verbose(env, "R%d indirect variable offset stack access prohibited for !root, var_off=%s\n",	4144	verbose(env, "R%d%s variable offset stack access prohibited for !root, var_off=%s\n",
3750	regno, tn_buf);	4145	regno, err_extra, tn_buf);
3751	return -EACCES;	4146	return -EACCES;
3752	}	4147	}
3753	/* Only initialized buffer on stack is allowed to be accessed	4148	/* Only initialized buffer on stack is allowed to be accessed
@@ -3759,28 +4154,8 @@ static int check_stack_boundary(struct bpf_verifier_env *env, int regno, · 5 unmodified lines
3759	if (meta && meta->raw_mode)	4154	if (meta && meta->raw_mode)
3760	meta = NULL;	4155	meta = NULL;
3761		4156
3762	if (reg->smax_value >= BPF_MAX_VAR_OFF \|\|	4157	min_off = reg->smin_value + off;
3763	reg->smax_value <= -BPF_MAX_VAR_OFF) {	4158	max_off = reg->smax_value + off;
3764	verbose(env, "R%d unbounded indirect variable offset stack access\n",
3765	regno);
3766	return -EACCES;
3767	}
3768	min_off = reg->smin_value + reg->off;
3769	max_off = reg->smax_value + reg->off;
3770	err = __check_stack_boundary(env, regno, min_off, access_size,
3771	zero_size_allowed);
3772	if (err) {
3773	verbose(env, "R%d min value is outside of stack bound\n",
3774	regno);
3775	return err;
3776	}
3777	err = __check_stack_boundary(env, regno, max_off, access_size,
3778	zero_size_allowed);
3779	if (err) {
3780	verbose(env, "R%d max value is outside of stack bound\n",
3781	regno);
3782	return err;
3783	}
3784	}	4159	}
3785		4160
3786	if (meta && meta->raw_mode) {	4161	if (meta && meta->raw_mode) {
@@ -3800,8 +4175,10 @@ static int check_stack_boundary(struct bpf_verifier_env *env, int regno, · 13 unmodified lines
3800	if (*stype == STACK_MISC)	4175	if (*stype == STACK_MISC)
3801	goto mark;	4176	goto mark;
3802	if (*stype == STACK_ZERO) {	4177	if (*stype == STACK_ZERO) {
3803	/* helper can write anything into the stack */	4178	if (clobber) {
3804	*stype = STACK_MISC;	4179	/* helper can write anything into the stack */
		4180	*stype = STACK_MISC;
		4181	}
3805	goto mark;	4182	goto mark;
3806	}	4183	}
3807		4184
@@ -3812,22 +4189,24 @@ static int check_stack_boundary(struct bpf_verifier_env *env, int regno, · 4 unmodified lines
3812	if (state->stack[spi].slot_type[0] == STACK_SPILL &&	4189	if (state->stack[spi].slot_type[0] == STACK_SPILL &&
3813	(state->stack[spi].spilled_ptr.type == SCALAR_VALUE \|\|	4190	(state->stack[spi].spilled_ptr.type == SCALAR_VALUE \|\|
3814	env->allow_ptr_leaks)) {	4191	env->allow_ptr_leaks)) {
3815	__mark_reg_unknown(env, &state->stack[spi].spilled_ptr);	4192	if (clobber) {
3816	for (j = 0; j < BPF_REG_SIZE; j++)	4193	__mark_reg_unknown(env, &state->stack[spi].spilled_ptr);
3817	state->stack[spi].slot_type[j] = STACK_MISC;	4194	for (j = 0; j < BPF_REG_SIZE; j++)
		4195	state->stack[spi].slot_type[j] = STACK_MISC;
		4196	}
3818	goto mark;	4197	goto mark;
3819	}	4198	}
3820		4199
3821	err:	4200	err:
3822	if (tnum_is_const(reg->var_off)) {	4201	if (tnum_is_const(reg->var_off)) {
3823	verbose(env, "invalid indirect read from stack off %d+%d size %d\n",	4202	verbose(env, "invalid%s read from stack R%d off %d+%d size %d\n",
3824	min_off, i - min_off, access_size);	4203	err_extra, regno, min_off, i - min_off, access_size);
3825	} else {	4204	} else {
3826	char tn_buf[48];	4205	char tn_buf[48];
3827		4206
3828	tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);	4207	tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
3829	verbose(env, "invalid indirect read from stack var_off %s+%d size %d\n",	4208	verbose(env, "invalid%s read from stack R%d var_off %s+%d size %d\n",
3830	tn_buf, i - min_off, access_size);	4209	err_extra, regno, tn_buf, i - min_off, access_size);
3831	}	4210	}
3832	return -EACCES;	4211	return -EACCES;
3833	mark:	4212	mark:
@@ -3876,8 +4255,10 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, · 42 unmodified lines
3876	"rdwr",	4255	"rdwr",
3877	&env->prog->aux->max_rdwr_access);	4256	&env->prog->aux->max_rdwr_access);
3878	case PTR_TO_STACK:	4257	case PTR_TO_STACK:
3879	return check_stack_boundary(env, regno, access_size,	4258	return check_stack_range_initialized(
3880	zero_size_allowed, meta);	4259	env,
		4260	regno, reg->off, access_size,
		4261	zero_size_allowed, ACCESS_HELPER, meta);
3881	default: /* scalar_value or invalid ptr */	4262	default: /* scalar_value or invalid ptr */
3882	/* Allow zero-byte read from NULL, regardless of pointer type */	4263	/* Allow zero-byte read from NULL, regardless of pointer type */
3883	if (zero_size_allowed && access_size == 0 &&	4264	if (zero_size_allowed && access_size == 0 &&
@@ -3891,6 +4272,29 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, · 7 unmodified lines
3891	}	4272	}
3892	}	4273	}
3893		4274
		4275	int check_mem_reg(struct bpf_verifier_env env, struct bpf_reg_state reg,
		4276	u32 regno, u32 mem_size)
		4277	{
		4278	if (register_is_null(reg))
		4279	return 0;
		4280
		4281	if (reg_type_may_be_null(reg->type)) {
		4282	/* Assuming that the register contains a value check if the memory
		4283	* access is safe. Temporarily save and restore the register's state as
		4284	* the conversion shouldn't be visible to a caller.
		4285	*/
		4286	const struct bpf_reg_state saved_reg = *reg;
		4287	int rv;
		4288
		4289	mark_ptr_not_null_reg(reg);
		4290	rv = check_helper_mem_access(env, regno, mem_size, true, NULL);
		4291	*reg = saved_reg;
		4292	return rv;
		4293	}
		4294
		4295	return check_helper_mem_access(env, regno, mem_size, true, NULL);
		4296	}
		4297
3894	/* Implementation details:	4298	/* Implementation details:
3895	* bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL	4299	* bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL
3896	* Two bpf_map_lookups (even with the same key) will have different reg->id.	4300	* Two bpf_map_lookups (even with the same key) will have different reg->id.
@@ -4875,8 +5279,9 @@ static int check_func_call(struct bpf_verifier_env env, struct bpf_insn insn, · 978 unmodified lines
4875	subprog);	5279	subprog);
4876	clear_caller_saved_regs(env, caller->regs);	5280	clear_caller_saved_regs(env, caller->regs);
4877		5281
4878	/* All global functions return SCALAR_VALUE */	5282	/* All global functions return a 64-bit SCALAR_VALUE */
4879	mark_reg_unknown(env, caller->regs, BPF_REG_0);	5283	mark_reg_unknown(env, caller->regs, BPF_REG_0);
		5284	caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;
4880		5285
4881	/* continue with next insn after call */	5286	/* continue with next insn after call */
4882	return 0;	5287	return 0;
@@ -5541,6 +5946,41 @@ do_sim: · 658 unmodified lines
5541	return !ret ? -EFAULT : 0;	5946	return !ret ? -EFAULT : 0;
5542	}	5947	}
5543		5948
		5949	/* check that stack access falls within stack limits and that 'reg' doesn't
		5950	* have a variable offset.
		5951	*
		5952	* Variable offset is prohibited for unprivileged mode for simplicity since it
		5953	* requires corresponding support in Spectre masking for stack ALU. See also
		5954	* retrieve_ptr_limit().
		5955	*
		5956	*
		5957	* 'off' includes 'reg->off'.
		5958	*/
		5959	static int check_stack_access_for_ptr_arithmetic(
		5960	struct bpf_verifier_env *env,
		5961	int regno,
		5962	const struct bpf_reg_state *reg,
		5963	int off)
		5964	{
		5965	if (!tnum_is_const(reg->var_off)) {
		5966	char tn_buf[48];
		5967
		5968	tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
		5969	verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n",
		5970	regno, tn_buf, off);
		5971	return -EACCES;
		5972	}
		5973
		5974	if (off >= 0 \|\| off < -MAX_BPF_STACK) {
		5975	verbose(env, "R%d stack pointer arithmetic goes out of range, "
		5976	"prohibited for !root; off=%d\n", regno, off);
		5977	return -EACCES;
		5978	}
		5979
		5980	return 0;
		5981	}
		5982
		5983
5544	/* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off.	5984	/* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off.
5545	* Caller should also handle BPF_MOV case separately.	5985	* Caller should also handle BPF_MOV case separately.
5546	* If we return -EACCES, caller may want to try again treating pointer as a	5986	* If we return -EACCES, caller may want to try again treating pointer as a
@@ -5784,10 +6224,9 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, · 237 unmodified lines
5784	"prohibited for !root\n", dst);	6224	"prohibited for !root\n", dst);
5785	return -EACCES;	6225	return -EACCES;
5786	} else if (dst_reg->type == PTR_TO_STACK &&	6226	} else if (dst_reg->type == PTR_TO_STACK &&
5787	check_stack_access(env, dst_reg, dst_reg->off +	6227	check_stack_access_for_ptr_arithmetic(
5788	dst_reg->var_off.value, 1)) {	6228	env, dst, dst_reg, dst_reg->off +
5789	verbose(env, "R%d stack pointer arithmetic goes out of range, "	6229	dst_reg->var_off.value)) {
5790	"prohibited for !root\n", dst);
5791	return -EACCES;	6230	return -EACCES;
5792	}	6231	}
5793	}	6232	}
@@ -6266,7 +6705,7 @@ static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg, · 472 unmodified lines
6266	* 3) the signed bounds cross zero, so they tell us nothing	6705	* 3) the signed bounds cross zero, so they tell us nothing
6267	* about the result	6706	* about the result
6268	* If the value in dst_reg is known nonnegative, then again the	6707	* If the value in dst_reg is known nonnegative, then again the
6269	* unsigned bounts capture the signed bounds.	6708	* unsigned bounds capture the signed bounds.
6270	* Thus, in all cases it suffices to blow away our signed bounds	6709	* Thus, in all cases it suffices to blow away our signed bounds
6271	* and rely on inferring new ones from the unsigned bounds and	6710	* and rely on inferring new ones from the unsigned bounds and
6272	* var_off of the result.	6711	* var_off of the result.
@@ -6297,7 +6736,7 @@ static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg, · 24 unmodified lines
6297	* 3) the signed bounds cross zero, so they tell us nothing	6736	* 3) the signed bounds cross zero, so they tell us nothing
6298	* about the result	6737	* about the result
6299	* If the value in dst_reg is known nonnegative, then again the	6738	* If the value in dst_reg is known nonnegative, then again the
6300	* unsigned bounts capture the signed bounds.	6739	* unsigned bounds capture the signed bounds.
6301	* Thus, in all cases it suffices to blow away our signed bounds	6740	* Thus, in all cases it suffices to blow away our signed bounds
6302	* and rely on inferring new ones from the unsigned bounds and	6741	* and rely on inferring new ones from the unsigned bounds and
6303	* var_off of the result.	6742	* var_off of the result.
@@ -7367,43 +7806,19 @@ static void mark_ptr_or_null_reg(struct bpf_func_state *state, · 1063 unmodified lines
7367	}	7806	}
7368	if (is_null) {	7807	if (is_null) {
7369	reg->type = SCALAR_VALUE;	7808	reg->type = SCALAR_VALUE;
7370	} else if (reg->type == PTR_TO_MAP_VALUE_OR_NULL) {
7371	const struct bpf_map *map = reg->map_ptr;
7372
7373	if (map->inner_map_meta) {
7374	reg->type = CONST_PTR_TO_MAP;
7375	reg->map_ptr = map->inner_map_meta;
7376	} else if (map->map_type == BPF_MAP_TYPE_XSKMAP) {
7377	reg->type = PTR_TO_XDP_SOCK;
7378	} else if (map->map_type == BPF_MAP_TYPE_SOCKMAP \|\|
7379	map->map_type == BPF_MAP_TYPE_SOCKHASH) {
7380	reg->type = PTR_TO_SOCKET;
7381	} else {
7382	reg->type = PTR_TO_MAP_VALUE;
7383	}
7384	} else if (reg->type == PTR_TO_SOCKET_OR_NULL) {
7385	reg->type = PTR_TO_SOCKET;
7386	} else if (reg->type == PTR_TO_SOCK_COMMON_OR_NULL) {
7387	reg->type = PTR_TO_SOCK_COMMON;
7388	} else if (reg->type == PTR_TO_TCP_SOCK_OR_NULL) {
7389	reg->type = PTR_TO_TCP_SOCK;
7390	} else if (reg->type == PTR_TO_BTF_ID_OR_NULL) {
7391	reg->type = PTR_TO_BTF_ID;
7392	} else if (reg->type == PTR_TO_MEM_OR_NULL) {
7393	reg->type = PTR_TO_MEM;
7394	} else if (reg->type == PTR_TO_RDONLY_BUF_OR_NULL) {
7395	reg->type = PTR_TO_RDONLY_BUF;
7396	} else if (reg->type == PTR_TO_RDWR_BUF_OR_NULL) {
7397	reg->type = PTR_TO_RDWR_BUF;
7398	}
7399	if (is_null) {
7400	/* We don't need id and ref_obj_id from this point	7809	/* We don't need id and ref_obj_id from this point
7401	* onwards anymore, thus we should better reset it,	7810	* onwards anymore, thus we should better reset it,
7402	* so that state pruning has chances to take effect.	7811	* so that state pruning has chances to take effect.
7403	*/	7812	*/
7404	reg->id = 0;	7813	reg->id = 0;
7405	reg->ref_obj_id = 0;	7814	reg->ref_obj_id = 0;
7406	} else if (!reg_may_point_to_spin_lock(reg)) {	7815
		7816	return;
		7817	}
		7818
		7819	mark_ptr_not_null_reg(reg);
		7820
		7821	if (!reg_may_point_to_spin_lock(reg)) {
7407	/* For not-NULL ptr, reg->ref_obj_id will be reset	7822	/* For not-NULL ptr, reg->ref_obj_id will be reset
7408	* in release_reg_references().	7823	* in release_reg_references().
7409	*	7824	*
@@ -7986,6 +8401,9 @@ static int check_return_code(struct bpf_verifier_env *env) · 576 unmodified lines
7986	env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME \|\|	8401	env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME \|\|
7987	env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME)	8402	env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME)
7988	range = tnum_range(1, 1);	8403	range = tnum_range(1, 1);
		8404	if (env->prog->expected_attach_type == BPF_CGROUP_INET4_BIND \|\|
		8405	env->prog->expected_attach_type == BPF_CGROUP_INET6_BIND)
		8406	range = tnum_range(0, 3);
7989	break;	8407	break;
7990	case BPF_PROG_TYPE_CGROUP_SKB:	8408	case BPF_PROG_TYPE_CGROUP_SKB:
7991	if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) {	8409	if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) {
@@ -10015,15 +10433,22 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, · 2023 unmodified lines
10015	case BPF_MAP_TYPE_HASH:	10433	case BPF_MAP_TYPE_HASH:
10016	case BPF_MAP_TYPE_LRU_HASH:	10434	case BPF_MAP_TYPE_LRU_HASH:
10017	case BPF_MAP_TYPE_ARRAY:	10435	case BPF_MAP_TYPE_ARRAY:
		10436	case BPF_MAP_TYPE_PERCPU_HASH:
		10437	case BPF_MAP_TYPE_PERCPU_ARRAY:
		10438	case BPF_MAP_TYPE_LRU_PERCPU_HASH:
		10439	case BPF_MAP_TYPE_ARRAY_OF_MAPS:
		10440	case BPF_MAP_TYPE_HASH_OF_MAPS:
10018	if (!is_preallocated_map(map)) {	10441	if (!is_preallocated_map(map)) {
10019	verbose(env,	10442	verbose(env,
10020	"Sleepable programs can only use preallocated hash maps\n");	10443	"Sleepable programs can only use preallocated maps\n");
10021	return -EINVAL;	10444	return -EINVAL;
10022	}	10445	}
10023	break;	10446	break;
		10447	case BPF_MAP_TYPE_RINGBUF:
		10448	break;
10024	default:	10449	default:
10025	verbose(env,	10450	verbose(env,
10026	"Sleepable programs can only use array and hash maps\n");	10451	"Sleepable programs can only use array, hash, and ringbuf maps\n");
10027	return -EINVAL;	10452	return -EINVAL;
10028	}	10453	}
10029		10454
@@ -10581,6 +11006,7 @@ static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, · 551 unmodified lines
10581	for (i = 0; i < len; i++) {	11006	for (i = 0; i < len; i++) {
10582	int adj_idx = i + delta;	11007	int adj_idx = i + delta;
10583	struct bpf_insn insn;	11008	struct bpf_insn insn;
		11009	u8 load_reg;
10584		11010
10585	insn = insns[adj_idx];	11011	insn = insns[adj_idx];
10586	if (!aux[adj_idx].zext_dst) {	11012	if (!aux[adj_idx].zext_dst) {
@@ -10623,9 +11049,27 @@ static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, · 36 unmodified lines
10623	if (!bpf_jit_needs_zext())	11049	if (!bpf_jit_needs_zext())
10624	continue;	11050	continue;
10625		11051
		11052	/* zext_dst means that we want to zero-extend whatever register
		11053	* the insn defines, which is dst_reg most of the time, with
		11054	* the notable exception of BPF_STX + BPF_ATOMIC + BPF_FETCH.
		11055	*/
		11056	if (BPF_CLASS(insn.code) == BPF_STX &&
		11057	BPF_MODE(insn.code) == BPF_ATOMIC) {
		11058	/* BPF_STX + BPF_ATOMIC insns without BPF_FETCH do not
		11059	* define any registers, therefore zext_dst cannot be
		11060	* set.
		11061	*/
		11062	if (WARN_ON(!(insn.imm & BPF_FETCH)))
		11063	return -EINVAL;
		11064	load_reg = insn.imm == BPF_CMPXCHG ? BPF_REG_0
		11065	: insn.src_reg;
		11066	} else {
		11067	load_reg = insn.dst_reg;
		11068	}
		11069
10626	zext_patch[0] = insn;	11070	zext_patch[0] = insn;
10627	zext_patch[1].dst_reg = insn.dst_reg;	11071	zext_patch[1].dst_reg = load_reg;
10628	zext_patch[1].src_reg = insn.dst_reg;	11072	zext_patch[1].src_reg = load_reg;
10629	patch = zext_patch;	11073	patch = zext_patch;
10630	patch_len = 2;	11074	patch_len = 2;
10631	apply_patch_buffer:	11075	apply_patch_buffer:
@@ -10841,8 +11285,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 209 unmodified lines
10841	return 0;	11285	return 0;
10842		11286
10843	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {	11287	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
10844	if (insn->code != (BPF_JMP \| BPF_CALL) \|\|	11288	if (!bpf_pseudo_call(insn))
10845	insn->src_reg != BPF_PSEUDO_CALL)
10846	continue;	11289	continue;
10847	/* Upon error here we cannot fall back to interpreter but	11290	/* Upon error here we cannot fall back to interpreter but
10848	* need a hard reject of the program. Thus -EFAULT is	11291	* need a hard reject of the program. Thus -EFAULT is
@@ -10883,7 +11326,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 34 unmodified lines
10883	/* BPF_PROG_RUN doesn't call subprogs directly,	11326	/* BPF_PROG_RUN doesn't call subprogs directly,
10884	* hence main prog stats include the runtime of subprogs.	11327	* hence main prog stats include the runtime of subprogs.
10885	* subprogs don't have IDs and not reachable via prog_get_next_id	11328	* subprogs don't have IDs and not reachable via prog_get_next_id
10886	* func[i]->aux->stats will never be accessed and stays NULL	11329	* func[i]->stats will never be accessed and stays NULL
10887	*/	11330	*/
10888	func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER);	11331	func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER);
10889	if (!func[i])	11332	if (!func[i])
@@ -10971,8 +11414,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 81 unmodified lines
10971	for (i = 0; i < env->subprog_cnt; i++) {	11414	for (i = 0; i < env->subprog_cnt; i++) {
10972	insn = func[i]->insnsi;	11415	insn = func[i]->insnsi;
10973	for (j = 0; j < func[i]->len; j++, insn++) {	11416	for (j = 0; j < func[i]->len; j++, insn++) {
10974	if (insn->code != (BPF_JMP \| BPF_CALL) \|\|	11417	if (!bpf_pseudo_call(insn))
10975	insn->src_reg != BPF_PSEUDO_CALL)
10976	continue;	11418	continue;
10977	subprog = insn->off;	11419	subprog = insn->off;
10978	insn->imm = BPF_CAST_CALL(func[subprog]->bpf_func) -	11420	insn->imm = BPF_CAST_CALL(func[subprog]->bpf_func) -
@@ -11017,8 +11459,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 38 unmodified lines
11017	* later look the same as if they were interpreted only.	11459	* later look the same as if they were interpreted only.
11018	*/	11460	*/
11019	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {	11461	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
11020	if (insn->code != (BPF_JMP \| BPF_CALL) \|\|	11462	if (!bpf_pseudo_call(insn))
11021	insn->src_reg != BPF_PSEUDO_CALL)
11022	continue;	11463	continue;
11023	insn->off = env->insn_aux_data[i].call_imm;	11464	insn->off = env->insn_aux_data[i].call_imm;
11024	subprog = find_subprog(env, i + insn->off + 1);	11465	subprog = find_subprog(env, i + insn->off + 1);
@@ -11047,8 +11488,7 @@ out_undo_insn: · 22 unmodified lines
11047	/* cleanup main prog to be interpreted */	11488	/* cleanup main prog to be interpreted */
11048	prog->jit_requested = 0;	11489	prog->jit_requested = 0;
11049	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {	11490	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
11050	if (insn->code != (BPF_JMP \| BPF_CALL) \|\|	11491	if (!bpf_pseudo_call(insn))
11051	insn->src_reg != BPF_PSEUDO_CALL)
11052	continue;	11492	continue;
11053	insn->off = 0;	11493	insn->off = 0;
11054	insn->imm = env->insn_aux_data[i].call_imm;	11494	insn->imm = env->insn_aux_data[i].call_imm;
@@ -11083,8 +11523,7 @@ static int fixup_call_args(struct bpf_verifier_env *env) · 28 unmodified lines
11083	return -EINVAL;	11523	return -EINVAL;
11084	}	11524	}
11085	for (i = 0; i < prog->len; i++, insn++) {	11525	for (i = 0; i < prog->len; i++, insn++) {
11086	if (insn->code != (BPF_JMP \| BPF_CALL) \|\|	11526	if (!bpf_pseudo_call(insn))
11087	insn->src_reg != BPF_PSEUDO_CALL)
11088	continue;	11527	continue;
11089	depth = get_callee_stack_depth(env, insn, i);	11528	depth = get_callee_stack_depth(env, insn, i);
11090	if (depth < 0)	11529	if (depth < 0)
@@ -11547,6 +11986,13 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) · 456 unmodified lines
11547	mark_reg_known_zero(env, regs, i);	11986	mark_reg_known_zero(env, regs, i);
11548	else if (regs[i].type == SCALAR_VALUE)	11987	else if (regs[i].type == SCALAR_VALUE)
11549	mark_reg_unknown(env, regs, i);	11988	mark_reg_unknown(env, regs, i);
		11989	else if (regs[i].type == PTR_TO_MEM_OR_NULL) {
		11990	const u32 mem_size = regs[i].mem_size;
		11991
		11992	mark_reg_known_zero(env, regs, i);
		11993	regs[i].mem_size = mem_size;
		11994	regs[i].id = ++env->id_gen;
		11995	}
11550	}	11996	}
11551	} else {	11997	} else {
11552	/* 1st arg to a function */	11998	/* 1st arg to a function */
@@ -12125,6 +12571,7 @@ int bpf_check(struct bpf_prog *prog, union bpf_attr attr, · 572 unmodified lines
12125	env->strict_alignment = false;	12571	env->strict_alignment = false;
12126		12572
12127	env->allow_ptr_leaks = bpf_allow_ptr_leaks();	12573	env->allow_ptr_leaks = bpf_allow_ptr_leaks();
		12574	env->allow_uninit_stack = bpf_allow_uninit_stack();
12128	env->allow_ptr_to_map_access = bpf_allow_ptr_to_map_access();	12575	env->allow_ptr_to_map_access = bpf_allow_ptr_to_map_access();
12129	env->bypass_spec_v1 = bpf_bypass_spec_v1();	12576	env->bypass_spec_v1 = bpf_bypass_spec_v1();
12130	env->bypass_spec_v4 = bpf_bypass_spec_v4();	12577	env->bypass_spec_v4 = bpf_bypass_spec_v4();

kernel/trace/bpf_trace.c+6−0

 static bool bpf_d_path_allowed(const struct bpf_prog *prog)
 {
+	if (prog->type == BPF_PROG_TYPE_TRACING &&
+	    prog->expected_attach_type == BPF_TRACE_ITER)
+		return true;
 	if (prog->type == BPF_PROG_TYPE_LSM)
 		return bpf_lsm_is_sleepable_hook(prog->aux->attach_btf_id);
 		return &bpf_sk_storage_delete_tracing_proto;
 	case BPF_FUNC_sock_from_file:
 		return &bpf_sock_from_file_proto;
+	case BPF_FUNC_get_socket_cookie:
+		return &bpf_get_socket_ptr_cookie_proto;
 #endif
 	case BPF_FUNC_seq_printf:
 		return prog->expected_attach_type == BPF_TRACE_ITER ?

Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code