Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Alexei Starovoitov says: ==================== pull-request: bpf-next 2021-04-01 The following pull-request contains BPF updates for your *net-next* tree. We've added 68 non-merge commits during the last 7 day(s) which contain a total of 70 files changed, 2944 insertions(+), 1139 deletions(-). The main changes are: 1) UDP support for sockmap, from Cong. 2) Verifier merge conflict resolution fix, from Daniel. 3) xsk selftests enhancements, from Maciej. 4) Unstable helpers aka kernel func calling, from Martin. 5) Batches ops for LPM map, from Pedro. 6) Fix race in bpf_get_local_storage, from Yonghong. ====================

David S. Miller committed 5 years ago

Commit c2bcb4c2 parents bd78980 89d69c570 files · +2944 −1139Browse files

Showing the changes to kernel (8 of 70 files). Show all

Unified · Split

8 files

kernel/bpf/btf.c +143 −76
kernel/bpf/core.c +24 −23
kernel/bpf/disasm.c +10 −3
kernel/bpf/helpers.c +11 −4
kernel/bpf/local_storage.c +3 −2
kernel/bpf/lpm_trie.c +3 −0
kernel/bpf/syscall.c +4 −1
kernel/bpf/verifier.c +348 −42

kernel/bpf/btf.c+143−76

@@ -283,7 +283,7 @@ static const char * const btf_kind_str[NR_BTF_KINDS] = { · 282 unmodified lines
283	[BTF_KIND_FLOAT] = "FLOAT",	283	[BTF_KIND_FLOAT] = "FLOAT",
284	};	284	};
285		285
286	static const char btf_type_str(const struct btf_type t)	286	const char btf_type_str(const struct btf_type t)
287	{	287	{
288	return btf_kind_str[BTF_INFO_KIND(t->info)];	288	return btf_kind_str[BTF_INFO_KIND(t->info)];
289	}	289	}
@@ -789,7 +789,6 @@ static const struct btf_type btf_type_skip_qualifiers(const struct btf btf, · 499 unmodified lines
789		789
790	while (btf_type_is_modifier(t) &&	790	while (btf_type_is_modifier(t) &&
791	BTF_INFO_KIND(t->info) != BTF_KIND_TYPEDEF) {	791	BTF_INFO_KIND(t->info) != BTF_KIND_TYPEDEF) {
792	id = t->type;
793	t = btf_type_by_id(btf, t->type);	792	t = btf_type_by_id(btf, t->type);
794	}	793	}
795		794
@@ -4377,7 +4376,7 @@ static u8 bpf_ctx_convert_map[] = { · 3581 unmodified lines
4377	#undef BPF_LINK_TYPE	4376	#undef BPF_LINK_TYPE
4378		4377
4379	static const struct btf_member *	4378	static const struct btf_member *
4380	btf_get_prog_ctx_type(struct bpf_verifier_log log, struct btf btf,	4379	btf_get_prog_ctx_type(struct bpf_verifier_log log, const struct btf btf,
4381	const struct btf_type *t, enum bpf_prog_type prog_type,	4380	const struct btf_type *t, enum bpf_prog_type prog_type,
4382	int arg)	4381	int arg)
4383	{	4382	{
@@ -5362,122 +5361,190 @@ int btf_check_type_match(struct bpf_verifier_log log, const struct bpf_prog pr · 978 unmodified lines
5362	return btf_check_func_type_match(log, btf1, t1, btf2, t2);	5361	return btf_check_func_type_match(log, btf1, t1, btf2, t2);
5363	}	5362	}
5364		5363
5365	/* Compare BTF of a function with given bpf_reg_state.	5364	static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = {
5366	* Returns:	5365	#ifdef CONFIG_NET
5367	* EFAULT - there is a verifier bug. Abort verification.	5366	[PTR_TO_SOCKET] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK],
5368	* EINVAL - there is a type mismatch or BTF is not available.	5367	[PTR_TO_SOCK_COMMON] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
5369	* 0 - BTF matches with what bpf_reg_state expects.	5368	[PTR_TO_TCP_SOCK] = &btf_sock_ids[BTF_SOCK_TYPE_TCP],
5370	* Only PTR_TO_CTX and SCALAR_VALUE states are recognized.	5369	#endif
5371	*/	5370	};
5372	int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog,	5371
5373	struct bpf_reg_state *regs)	5372	static int btf_check_func_arg_match(struct bpf_verifier_env *env,
		5373	const struct btf *btf, u32 func_id,
		5374	struct bpf_reg_state *regs,
		5375	bool ptr_to_mem_ok)
5374	{	5376	{
5375	struct bpf_verifier_log *log = &env->log;	5377	struct bpf_verifier_log *log = &env->log;
5376	struct bpf_prog *prog = env->prog;	5378	const char func_name, ref_tname;
5377	struct btf *btf = prog->aux->btf;
5378	const struct btf_param *args;
5379	const struct btf_type t, ref_t;	5379	const struct btf_type t, ref_t;
5380	u32 i, nargs, btf_id, type_size;	5380	const struct btf_param *args;
5381	const char *tname;	5381	u32 i, nargs, ref_id;
5382	bool is_global;
5383
5384	if (!prog->aux->func_info)
5385	return -EINVAL;
5386
5387	btf_id = prog->aux->func_info[subprog].type_id;
5388	if (!btf_id)
5389	return -EFAULT;
5390
5391	if (prog->aux->func_info_aux[subprog].unreliable)
5392	return -EINVAL;
5393		5382
5394	t = btf_type_by_id(btf, btf_id);	5383	t = btf_type_by_id(btf, func_id);
5395	if (!t \|\| !btf_type_is_func(t)) {	5384	if (!t \|\| !btf_type_is_func(t)) {
5396	/* These checks were already done by the verifier while loading	5385	/* These checks were already done by the verifier while loading
5397	* struct bpf_func_info	5386	* struct bpf_func_info or in add_kfunc_call().
5398	*/	5387	*/
5399	bpf_log(log, "BTF of func#%d doesn't point to KIND_FUNC\n",	5388	bpf_log(log, "BTF of func_id %u doesn't point to KIND_FUNC\n",
5400	subprog);	5389	func_id);
5401	return -EFAULT;	5390	return -EFAULT;
5402	}	5391	}
5403	tname = btf_name_by_offset(btf, t->name_off);	5392	func_name = btf_name_by_offset(btf, t->name_off);
5404		5393
5405	t = btf_type_by_id(btf, t->type);	5394	t = btf_type_by_id(btf, t->type);
5406	if (!t \|\| !btf_type_is_func_proto(t)) {	5395	if (!t \|\| !btf_type_is_func_proto(t)) {
5407	bpf_log(log, "Invalid BTF of func %s\n", tname);	5396	bpf_log(log, "Invalid BTF of func %s\n", func_name);
5408	return -EFAULT;	5397	return -EFAULT;
5409	}	5398	}
5410	args = (const struct btf_param *)(t + 1);	5399	args = (const struct btf_param *)(t + 1);
5411	nargs = btf_type_vlen(t);	5400	nargs = btf_type_vlen(t);
5412	if (nargs > MAX_BPF_FUNC_REG_ARGS) {	5401	if (nargs > MAX_BPF_FUNC_REG_ARGS) {
5413	bpf_log(log, "Function %s has %d > %d args\n", tname, nargs,	5402	bpf_log(log, "Function %s has %d > %d args\n", func_name, nargs,
5414	MAX_BPF_FUNC_REG_ARGS);	5403	MAX_BPF_FUNC_REG_ARGS);
5415	goto out;	5404	return -EINVAL;
5416	}	5405	}
5417		5406
5418	is_global = prog->aux->func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL;
5419	/* check that BTF function arguments match actual types that the	5407	/* check that BTF function arguments match actual types that the
5420	* verifier sees.	5408	* verifier sees.
5421	*/	5409	*/
5422	for (i = 0; i < nargs; i++) {	5410	for (i = 0; i < nargs; i++) {
5423	struct bpf_reg_state *reg = &regs[i + 1];	5411	u32 regno = i + 1;
		5412	struct bpf_reg_state *reg = &regs[regno];
5424		5413
5425	t = btf_type_by_id(btf, args[i].type);	5414	t = btf_type_skip_modifiers(btf, args[i].type, NULL);
5426	while (btf_type_is_modifier(t))	5415	if (btf_type_is_scalar(t)) {
5427	t = btf_type_by_id(btf, t->type);
5428	if (btf_type_is_int(t) \|\| btf_type_is_enum(t)) {
5429	if (reg->type == SCALAR_VALUE)	5416	if (reg->type == SCALAR_VALUE)
5430	continue;	5417	continue;
5431	bpf_log(log, "R%d is not a scalar\n", i + 1);	5418	bpf_log(log, "R%d is not a scalar\n", regno);
5432	goto out;	5419	return -EINVAL;
5433	}	5420	}
5434	if (btf_type_is_ptr(t)) {	5421
		5422	if (!btf_type_is_ptr(t)) {
		5423	bpf_log(log, "Unrecognized arg#%d type %s\n",
		5424	i, btf_type_str(t));
		5425	return -EINVAL;
		5426	}
		5427
		5428	ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id);
		5429	ref_tname = btf_name_by_offset(btf, ref_t->name_off);
		5430	if (btf_is_kernel(btf)) {
		5431	const struct btf_type *reg_ref_t;
		5432	const struct btf *reg_btf;
		5433	const char *reg_ref_tname;
		5434	u32 reg_ref_id;
		5435
		5436	if (!btf_type_is_struct(ref_t)) {
		5437	bpf_log(log, "kernel function %s args#%d pointer type %s %s is not supported\n",
		5438	func_name, i, btf_type_str(ref_t),
		5439	ref_tname);
		5440	return -EINVAL;
		5441	}
		5442
		5443	if (reg->type == PTR_TO_BTF_ID) {
		5444	reg_btf = reg->btf;
		5445	reg_ref_id = reg->btf_id;
		5446	} else if (reg2btf_ids[reg->type]) {
		5447	reg_btf = btf_vmlinux;
		5448	reg_ref_id = *reg2btf_ids[reg->type];
		5449	} else {
		5450	bpf_log(log, "kernel function %s args#%d expected pointer to %s %s but R%d is not a pointer to btf_id\n",
		5451	func_name, i,
		5452	btf_type_str(ref_t), ref_tname, regno);
		5453	return -EINVAL;
		5454	}
		5455
		5456	reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id,
		5457	&reg_ref_id);
		5458	reg_ref_tname = btf_name_by_offset(reg_btf,
		5459	reg_ref_t->name_off);
		5460	if (!btf_struct_ids_match(log, reg_btf, reg_ref_id,
		5461	reg->off, btf, ref_id)) {
		5462	bpf_log(log, "kernel function %s args#%d expected pointer to %s %s but R%d has a pointer to %s %s\n",
		5463	func_name, i,
		5464	btf_type_str(ref_t), ref_tname,
		5465	regno, btf_type_str(reg_ref_t),
		5466	reg_ref_tname);
		5467	return -EINVAL;
		5468	}
		5469	} else if (btf_get_prog_ctx_type(log, btf, t,
		5470	env->prog->type, i)) {
5435	/* If function expects ctx type in BTF check that caller	5471	/* If function expects ctx type in BTF check that caller
5436	* is passing PTR_TO_CTX.	5472	* is passing PTR_TO_CTX.
5437	*/	5473	*/
5438	if (btf_get_prog_ctx_type(log, btf, t, prog->type, i)) {	5474	if (reg->type != PTR_TO_CTX) {
5439	if (reg->type != PTR_TO_CTX) {	5475	bpf_log(log,
5440	bpf_log(log,	5476	"arg#%d expected pointer to ctx, but got %s\n",
5441	"arg#%d expected pointer to ctx, but got %s\n",	5477	i, btf_type_str(t));
5442	i, btf_kind_str[BTF_INFO_KIND(t->info)]);	5478	return -EINVAL;
5443	goto out;
5444	}
5445	if (check_ctx_reg(env, reg, i + 1))
5446	goto out;
5447	continue;
5448	}	5479	}
		5480	if (check_ctx_reg(env, reg, regno))
		5481	return -EINVAL;
		5482	} else if (ptr_to_mem_ok) {
		5483	const struct btf_type *resolve_ret;
		5484	u32 type_size;
5449		5485
5450	if (!is_global)	5486	resolve_ret = btf_resolve_size(btf, ref_t, &type_size);
5451	goto out;	5487	if (IS_ERR(resolve_ret)) {
5452
5453	t = btf_type_skip_modifiers(btf, t->type, NULL);
5454
5455	ref_t = btf_resolve_size(btf, t, &type_size);
5456	if (IS_ERR(ref_t)) {
5457	bpf_log(log,	5488	bpf_log(log,
5458	"arg#%d reference type('%s %s') size cannot be determined: %ld\n",	5489	"arg#%d reference type('%s %s') size cannot be determined: %ld\n",
5459	i, btf_type_str(t), btf_name_by_offset(btf, t->name_off),	5490	i, btf_type_str(ref_t), ref_tname,
5460	PTR_ERR(ref_t));	5491	PTR_ERR(resolve_ret));
5461	goto out;	5492	return -EINVAL;
5462	}	5493	}
5463		5494
5464	if (check_mem_reg(env, reg, i + 1, type_size))	5495	if (check_mem_reg(env, reg, regno, type_size))
5465	goto out;	5496	return -EINVAL;
5466		5497	} else {
5467	continue;	5498	return -EINVAL;
5468	}	5499	}
5469	bpf_log(log, "Unrecognized arg#%d type %s\n",
5470	i, btf_kind_str[BTF_INFO_KIND(t->info)]);
5471	goto out;
5472	}	5500	}
		5501
5473	return 0;	5502	return 0;
5474	out:	5503	}
		5504
		5505	/* Compare BTF of a function with given bpf_reg_state.
		5506	* Returns:
		5507	* EFAULT - there is a verifier bug. Abort verification.
		5508	* EINVAL - there is a type mismatch or BTF is not available.
		5509	* 0 - BTF matches with what bpf_reg_state expects.
		5510	* Only PTR_TO_CTX and SCALAR_VALUE states are recognized.
		5511	*/
		5512	int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog,
		5513	struct bpf_reg_state *regs)
		5514	{
		5515	struct bpf_prog *prog = env->prog;
		5516	struct btf *btf = prog->aux->btf;
		5517	bool is_global;
		5518	u32 btf_id;
		5519	int err;
		5520
		5521	if (!prog->aux->func_info)
		5522	return -EINVAL;
		5523
		5524	btf_id = prog->aux->func_info[subprog].type_id;
		5525	if (!btf_id)
		5526	return -EFAULT;
		5527
		5528	if (prog->aux->func_info_aux[subprog].unreliable)
		5529	return -EINVAL;
		5530
		5531	is_global = prog->aux->func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL;
		5532	err = btf_check_func_arg_match(env, btf, btf_id, regs, is_global);
		5533
5475	/* Compiler optimizations can remove arguments from static functions	5534	/* Compiler optimizations can remove arguments from static functions
5476	* or mismatched type can be passed into a global function.	5535	* or mismatched type can be passed into a global function.
5477	* In such cases mark the function as unreliable from BTF point of view.	5536	* In such cases mark the function as unreliable from BTF point of view.
5478	*/	5537	*/
5479	prog->aux->func_info_aux[subprog].unreliable = true;	5538	if (err)
5480	return -EINVAL;	5539	prog->aux->func_info_aux[subprog].unreliable = true;
		5540	return err;
		5541	}
		5542
		5543	int btf_check_kfunc_arg_match(struct bpf_verifier_env *env,
		5544	const struct btf *btf, u32 func_id,
		5545	struct bpf_reg_state *regs)
		5546	{
		5547	return btf_check_func_arg_match(env, btf, func_id, regs, false);
5481	}	5548	}
5482		5549
5483	/* Convert BTF of a function into bpf_reg_state if possible	5550	/* Convert BTF of a function into bpf_reg_state if possible

kernel/bpf/core.c+24−23

 	if (!prog->aux->nr_linfo || !prog->jit_requested)
 		return 0;
-	prog->aux->jited_linfo = kcalloc(prog->aux->nr_linfo,
+	prog->aux->jited_linfo = kvcalloc(prog->aux->nr_linfo,
-					 sizeof(*prog->aux->jited_linfo),
+					  sizeof(*prog->aux->jited_linfo),
-					 GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
+					  GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
 	if (!prog->aux->jited_linfo)
 		return -ENOMEM;
 	return 0;
 }
-void bpf_prog_free_jited_linfo(struct bpf_prog *prog)
+void bpf_prog_jit_attempt_done(struct bpf_prog *prog)
 {
-	kfree(prog->aux->jited_linfo);
+	if (prog->aux->jited_linfo &&
-	prog->aux->jited_linfo = NULL;
+	    (!prog->jited || !prog->aux->jited_linfo[0])) {
-}
+		kvfree(prog->aux->jited_linfo);
+		prog->aux->jited_linfo = NULL;
+	}
-void bpf_prog_free_unused_jited_linfo(struct bpf_prog *prog)
+	kfree(prog->aux->kfunc_tab);
-{
+	prog->aux->kfunc_tab = NULL;
-	if (prog->aux->jited_linfo && !prog->aux->jited_linfo[0])
-		bpf_prog_free_jited_linfo(prog);
 }
 /* The jit engine is responsible to provide an array
 			insn_to_jit_off[linfo[i].insn_off - insn_start - 1];
 }
-void bpf_prog_free_linfo(struct bpf_prog *prog)
-{
-	bpf_prog_free_jited_linfo(prog);
-	kvfree(prog->aux->linfo);
-}
 struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size,
 				  gfp_t gfp_extra_flags)
 {
 	/* In case of BPF to BPF calls, verifier did all the prep
 	 * work with regards to JITing, etc.
 	 */
+	bool jit_needed = false;
 	if (fp->bpf_func)
 		goto finalize;
+	if (IS_ENABLED(CONFIG_BPF_JIT_ALWAYS_ON) ||
+	    bpf_prog_has_kfunc_call(fp))
+		jit_needed = true;
 	bpf_prog_select_func(fp);
 	/* eBPF JITs can rewrite the program in case constant
 			return fp;
 		fp = bpf_int_jit_compile(fp);
-		if (!fp->jited) {
+		bpf_prog_jit_attempt_done(fp);
-			bpf_prog_free_jited_linfo(fp);
+		if (!fp->jited && jit_needed) {
-#ifdef CONFIG_BPF_JIT_ALWAYS_ON
 			*err = -ENOTSUPP;
 			return fp;
-#endif
-		} else {
-			bpf_prog_free_unused_jited_linfo(fp);
 		}
 	} else {
 		*err = bpf_prog_offload_compile(fp);
 	return false;
 }
+bool __weak bpf_jit_supports_kfunc_call(void)
+{
+	return false;
+}
 /* To execute LD_ABS/LD_IND instructions __bpf_prog_run() may call
  * skb_copy_bits(), so provide a weak definition of it for NET-less config.
  */

kernel/bpf/disasm.c+10−3

 {
 	BUILD_BUG_ON(ARRAY_SIZE(func_id_str) != __BPF_FUNC_MAX_ID);
-	if (insn->src_reg != BPF_PSEUDO_CALL &&
+	if (!insn->src_reg &&
 	    insn->imm >= 0 && insn->imm < __BPF_FUNC_MAX_ID &&
 	    func_id_str[insn->imm])
 		return func_id_str[insn->imm];
-	if (cbs && cbs->cb_call)
+	if (cbs && cbs->cb_call) {
-		return cbs->cb_call(cbs->private_data, insn);
+		const char *res;
+		res = cbs->cb_call(cbs->private_data, insn);
+		if (res)
+			return res;
+	}
 	if (insn->src_reg == BPF_PSEUDO_CALL)
 		snprintf(buff, len, "%+d", insn->imm);
+	else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL)
+		snprintf(buff, len, "kernel-function");
 	return buff;
 }

kernel/bpf/helpers.c+11−4

 };
 #ifdef CONFIG_CGROUP_BPF
-DECLARE_PER_CPU(struct bpf_cgroup_storage*,
+DECLARE_PER_CPU(struct bpf_cgroup_storage_info,
-		bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]);
+		bpf_cgroup_storage_info[BPF_CGROUP_STORAGE_NEST_MAX]);
 BPF_CALL_2(bpf_get_local_storage, struct bpf_map *, map, u64, flags)
 {
 	 * verifier checks that its value is correct.
 	 */
 	enum bpf_cgroup_storage_type stype = cgroup_storage_type(map);
-	struct bpf_cgroup_storage *storage;
+	struct bpf_cgroup_storage *storage = NULL;
 	void *ptr;
+	int i;
-	storage = this_cpu_read(bpf_cgroup_storage[stype]);
+	for (i = 0; i < BPF_CGROUP_STORAGE_NEST_MAX; i++) {
+		if (unlikely(this_cpu_read(bpf_cgroup_storage_info[i].task) != current))
+			continue;
+		storage = this_cpu_read(bpf_cgroup_storage_info[i].storage[stype]);
+		break;
+	}
 	if (stype == BPF_CGROUP_STORAGE_SHARED)
 		ptr = &READ_ONCE(storage->buf)->data[0];

kernel/bpf/local_storage.c+3−2

 #include <linux/slab.h>
 #include <uapi/linux/btf.h>
-DEFINE_PER_CPU(struct bpf_cgroup_storage*, bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]);
 #ifdef CONFIG_CGROUP_BPF
+DEFINE_PER_CPU(struct bpf_cgroup_storage_info,
+	       bpf_cgroup_storage_info[BPF_CGROUP_STORAGE_NEST_MAX]);
 #include "../cgroup/cgroup-internal.h"
 #define LOCAL_STORAGE_CREATE_FLAG_MASK					\

kernel/bpf/lpm_trie.c+3−0

 	.map_lookup_elem = trie_lookup_elem,
 	.map_update_elem = trie_update_elem,
 	.map_delete_elem = trie_delete_elem,
+	.map_lookup_batch = generic_map_lookup_batch,
+	.map_update_batch = generic_map_update_batch,
+	.map_delete_batch = generic_map_delete_batch,
 	.map_check_btf = trie_check_btf,
 	.map_btf_name = "lpm_trie",
 	.map_btf_id = &trie_map_btf_id,

kernel/bpf/syscall.c+4−1

 {
 	bpf_prog_kallsyms_del_all(prog);
 	btf_put(prog->aux->btf);
-	bpf_prog_free_linfo(prog);
+	kvfree(prog->aux->jited_linfo);
+	kvfree(prog->aux->linfo);
+	kfree(prog->aux->kfunc_tab);
 	if (prog->aux->attach_btf)
 		btf_put(prog->aux->attach_btf);
 		return BPF_PROG_TYPE_SK_MSG;
 	case BPF_SK_SKB_STREAM_PARSER:
 	case BPF_SK_SKB_STREAM_VERDICT:
+	case BPF_SK_SKB_VERDICT:
 		return BPF_PROG_TYPE_SK_SKB;
 	case BPF_LIRC_MODE2:
 		return BPF_PROG_TYPE_LIRC_MODE2;

kernel/bpf/verifier.c+348−42

@@ -234,6 +234,12 @@ static bool bpf_pseudo_call(const struct bpf_insn *insn) · 233 unmodified lines
234	insn->src_reg == BPF_PSEUDO_CALL;	234	insn->src_reg == BPF_PSEUDO_CALL;
235	}	235	}
236		236
		237	static bool bpf_pseudo_kfunc_call(const struct bpf_insn *insn)
		238	{
		239	return insn->code == (BPF_JMP \| BPF_CALL) &&
		240	insn->src_reg == BPF_PSEUDO_KFUNC_CALL;
		241	}
		242
237	static bool bpf_pseudo_func(const struct bpf_insn *insn)	243	static bool bpf_pseudo_func(const struct bpf_insn *insn)
238	{	244	{
239	return insn->code == (BPF_LD \| BPF_IMM \| BPF_DW) &&	245	return insn->code == (BPF_LD \| BPF_IMM \| BPF_DW) &&
@@ -1554,47 +1560,205 @@ static int add_subprog(struct bpf_verifier_env *env, int off) · 1314 unmodified lines
1554	verbose(env, "too many subprograms\n");	1560	verbose(env, "too many subprograms\n");
1555	return -E2BIG;	1561	return -E2BIG;
1556	}	1562	}
		1563	/* determine subprog starts. The end is one before the next starts */
1557	env->subprog_info[env->subprog_cnt++].start = off;	1564	env->subprog_info[env->subprog_cnt++].start = off;
1558	sort(env->subprog_info, env->subprog_cnt,	1565	sort(env->subprog_info, env->subprog_cnt,
1559	sizeof(env->subprog_info[0]), cmp_subprogs, NULL);	1566	sizeof(env->subprog_info[0]), cmp_subprogs, NULL);
1560	return env->subprog_cnt - 1;	1567	return env->subprog_cnt - 1;
1561	}	1568	}
1562		1569
1563	static int check_subprogs(struct bpf_verifier_env *env)	1570	struct bpf_kfunc_desc {
		1571	struct btf_func_model func_model;
		1572	u32 func_id;
		1573	s32 imm;
		1574	};
		1575
		1576	#define MAX_KFUNC_DESCS 256
		1577	struct bpf_kfunc_desc_tab {
		1578	struct bpf_kfunc_desc descs[MAX_KFUNC_DESCS];
		1579	u32 nr_descs;
		1580	};
		1581
		1582	static int kfunc_desc_cmp_by_id(const void a, const void b)
		1583	{
		1584	const struct bpf_kfunc_desc *d0 = a;
		1585	const struct bpf_kfunc_desc *d1 = b;
		1586
		1587	/* func_id is not greater than BTF_MAX_TYPE */
		1588	return d0->func_id - d1->func_id;
		1589	}
		1590
		1591	static const struct bpf_kfunc_desc *
		1592	find_kfunc_desc(const struct bpf_prog *prog, u32 func_id)
		1593	{
		1594	struct bpf_kfunc_desc desc = {
		1595	.func_id = func_id,
		1596	};
		1597	struct bpf_kfunc_desc_tab *tab;
		1598
		1599	tab = prog->aux->kfunc_tab;
		1600	return bsearch(&desc, tab->descs, tab->nr_descs,
		1601	sizeof(tab->descs[0]), kfunc_desc_cmp_by_id);
		1602	}
		1603
		1604	static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id)
		1605	{
		1606	const struct btf_type func, func_proto;
		1607	struct bpf_kfunc_desc_tab *tab;
		1608	struct bpf_prog_aux *prog_aux;
		1609	struct bpf_kfunc_desc *desc;
		1610	const char *func_name;
		1611	unsigned long addr;
		1612	int err;
		1613
		1614	prog_aux = env->prog->aux;
		1615	tab = prog_aux->kfunc_tab;
		1616	if (!tab) {
		1617	if (!btf_vmlinux) {
		1618	verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n");
		1619	return -ENOTSUPP;
		1620	}
		1621
		1622	if (!env->prog->jit_requested) {
		1623	verbose(env, "JIT is required for calling kernel function\n");
		1624	return -ENOTSUPP;
		1625	}
		1626
		1627	if (!bpf_jit_supports_kfunc_call()) {
		1628	verbose(env, "JIT does not support calling kernel function\n");
		1629	return -ENOTSUPP;
		1630	}
		1631
		1632	if (!env->prog->gpl_compatible) {
		1633	verbose(env, "cannot call kernel function from non-GPL compatible program\n");
		1634	return -EINVAL;
		1635	}
		1636
		1637	tab = kzalloc(sizeof(*tab), GFP_KERNEL);
		1638	if (!tab)
		1639	return -ENOMEM;
		1640	prog_aux->kfunc_tab = tab;
		1641	}
		1642
		1643	if (find_kfunc_desc(env->prog, func_id))
		1644	return 0;
		1645
		1646	if (tab->nr_descs == MAX_KFUNC_DESCS) {
		1647	verbose(env, "too many different kernel function calls\n");
		1648	return -E2BIG;
		1649	}
		1650
		1651	func = btf_type_by_id(btf_vmlinux, func_id);
		1652	if (!func \|\| !btf_type_is_func(func)) {
		1653	verbose(env, "kernel btf_id %u is not a function\n",
		1654	func_id);
		1655	return -EINVAL;
		1656	}
		1657	func_proto = btf_type_by_id(btf_vmlinux, func->type);
		1658	if (!func_proto \|\| !btf_type_is_func_proto(func_proto)) {
		1659	verbose(env, "kernel function btf_id %u does not have a valid func_proto\n",
		1660	func_id);
		1661	return -EINVAL;
		1662	}
		1663
		1664	func_name = btf_name_by_offset(btf_vmlinux, func->name_off);
		1665	addr = kallsyms_lookup_name(func_name);
		1666	if (!addr) {
		1667	verbose(env, "cannot find address for kernel function %s\n",
		1668	func_name);
		1669	return -EINVAL;
		1670	}
		1671
		1672	desc = &tab->descs[tab->nr_descs++];
		1673	desc->func_id = func_id;
		1674	desc->imm = BPF_CAST_CALL(addr) - __bpf_call_base;
		1675	err = btf_distill_func_proto(&env->log, btf_vmlinux,
		1676	func_proto, func_name,
		1677	&desc->func_model);
		1678	if (!err)
		1679	sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
		1680	kfunc_desc_cmp_by_id, NULL);
		1681	return err;
		1682	}
		1683
		1684	static int kfunc_desc_cmp_by_imm(const void a, const void b)
		1685	{
		1686	const struct bpf_kfunc_desc *d0 = a;
		1687	const struct bpf_kfunc_desc *d1 = b;
		1688
		1689	if (d0->imm > d1->imm)
		1690	return 1;
		1691	else if (d0->imm < d1->imm)
		1692	return -1;
		1693	return 0;
		1694	}
		1695
		1696	static void sort_kfunc_descs_by_imm(struct bpf_prog *prog)
		1697	{
		1698	struct bpf_kfunc_desc_tab *tab;
		1699
		1700	tab = prog->aux->kfunc_tab;
		1701	if (!tab)
		1702	return;
		1703
		1704	sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
		1705	kfunc_desc_cmp_by_imm, NULL);
		1706	}
		1707
		1708	bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog)
		1709	{
		1710	return !!prog->aux->kfunc_tab;
		1711	}
		1712
		1713	const struct btf_func_model *
		1714	bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
		1715	const struct bpf_insn *insn)
		1716	{
		1717	const struct bpf_kfunc_desc desc = {
		1718	.imm = insn->imm,
		1719	};
		1720	const struct bpf_kfunc_desc *res;
		1721	struct bpf_kfunc_desc_tab *tab;
		1722
		1723	tab = prog->aux->kfunc_tab;
		1724	res = bsearch(&desc, tab->descs, tab->nr_descs,
		1725	sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm);
		1726
		1727	return res ? &res->func_model : NULL;
		1728	}
		1729
		1730	static int add_subprog_and_kfunc(struct bpf_verifier_env *env)
1564	{	1731	{
1565	int i, ret, subprog_start, subprog_end, off, cur_subprog = 0;
1566	struct bpf_subprog_info *subprog = env->subprog_info;	1732	struct bpf_subprog_info *subprog = env->subprog_info;
1567	struct bpf_insn *insn = env->prog->insnsi;	1733	struct bpf_insn *insn = env->prog->insnsi;
1568	int insn_cnt = env->prog->len;	1734	int i, ret, insn_cnt = env->prog->len;
1569		1735
1570	/* Add entry function. */	1736	/* Add entry function. */
1571	ret = add_subprog(env, 0);	1737	ret = add_subprog(env, 0);
1572	if (ret < 0)	1738	if (ret)
1573	return ret;	1739	return ret;
1574		1740
1575	/* determine subprog starts. The end is one before the next starts */	1741	for (i = 0; i < insn_cnt; i++, insn++) {
1576	for (i = 0; i < insn_cnt; i++) {	1742	if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn) &&
1577	if (bpf_pseudo_func(insn + i)) {	1743	!bpf_pseudo_kfunc_call(insn))
1578	if (!env->bpf_capable) {
1579	verbose(env,
1580	"function pointers are allowed for CAP_BPF and CAP_SYS_ADMIN\n");
1581	return -EPERM;
1582	}
1583	ret = add_subprog(env, i + insn[i].imm + 1);
1584	if (ret < 0)
1585	return ret;
1586	/* remember subprog */
1587	insn[i + 1].imm = ret;
1588	continue;
1589	}
1590	if (!bpf_pseudo_call(insn + i))
1591	continue;	1744	continue;
		1745
1592	if (!env->bpf_capable) {	1746	if (!env->bpf_capable) {
1593	verbose(env,	1747	verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n");
1594	"function calls to other bpf functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n");
1595	return -EPERM;	1748	return -EPERM;
1596	}	1749	}
1597	ret = add_subprog(env, i + insn[i].imm + 1);	1750
		1751	if (bpf_pseudo_func(insn)) {
		1752	ret = add_subprog(env, i + insn->imm + 1);
		1753	if (ret >= 0)
		1754	/* remember subprog */
		1755	insn[1].imm = ret;
		1756	} else if (bpf_pseudo_call(insn)) {
		1757	ret = add_subprog(env, i + insn->imm + 1);
		1758	} else {
		1759	ret = add_kfunc_call(env, insn->imm);
		1760	}
		1761
1598	if (ret < 0)	1762	if (ret < 0)
1599	return ret;	1763	return ret;
1600	}	1764	}
@@ -1608,6 +1772,16 @@ static int check_subprogs(struct bpf_verifier_env *env) · 7 unmodified lines
1608	for (i = 0; i < env->subprog_cnt; i++)	1772	for (i = 0; i < env->subprog_cnt; i++)
1609	verbose(env, "func#%d @%d\n", i, subprog[i].start);	1773	verbose(env, "func#%d @%d\n", i, subprog[i].start);
1610		1774
		1775	return 0;
		1776	}
		1777
		1778	static int check_subprogs(struct bpf_verifier_env *env)
		1779	{
		1780	int i, subprog_start, subprog_end, off, cur_subprog = 0;
		1781	struct bpf_subprog_info *subprog = env->subprog_info;
		1782	struct bpf_insn *insn = env->prog->insnsi;
		1783	int insn_cnt = env->prog->len;
		1784
1611	/* now check that all jumps are within the same subprog */	1785	/* now check that all jumps are within the same subprog */
1612	subprog_start = subprog[cur_subprog].start;	1786	subprog_start = subprog[cur_subprog].start;
1613	subprog_end = subprog[cur_subprog + 1].start;	1787	subprog_end = subprog[cur_subprog + 1].start;
@@ -1916,6 +2090,17 @@ static int get_prev_insn_idx(struct bpf_verifier_state *st, int i, · 302 unmodified lines
1916	return i;	2090	return i;
1917	}	2091	}
1918		2092
		2093	static const char disasm_kfunc_name(void data, const struct bpf_insn *insn)
		2094	{
		2095	const struct btf_type *func;
		2096
		2097	if (insn->src_reg != BPF_PSEUDO_KFUNC_CALL)
		2098	return NULL;
		2099
		2100	func = btf_type_by_id(btf_vmlinux, insn->imm);
		2101	return btf_name_by_offset(btf_vmlinux, func->name_off);
		2102	}
		2103
1919	/* For given verifier state backtrack_insn() is called from the last insn to	2104	/* For given verifier state backtrack_insn() is called from the last insn to
1920	* the first insn. Its purpose is to compute a bitmask of registers and	2105	* the first insn. Its purpose is to compute a bitmask of registers and
1921	* stack slots that needs precision in the parent verifier state.	2106	* stack slots that needs precision in the parent verifier state.
@@ -1924,6 +2109,7 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, · 2 unmodified lines
1924	u32 reg_mask, u64 stack_mask)	2109	u32 reg_mask, u64 stack_mask)
1925	{	2110	{
1926	const struct bpf_insn_cbs cbs = {	2111	const struct bpf_insn_cbs cbs = {
		2112	.cb_call = disasm_kfunc_name,
1927	.cb_print = verbose,	2113	.cb_print = verbose,
1928	.private_data = env,	2114	.private_data = env,
1929	};	2115	};
@@ -5365,7 +5551,7 @@ static int __check_func_call(struct bpf_verifier_env env, struct bpf_insn insn · 3435 unmodified lines
5365	func_info_aux = env->prog->aux->func_info_aux;	5551	func_info_aux = env->prog->aux->func_info_aux;
5366	if (func_info_aux)	5552	if (func_info_aux)
5367	is_global = func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL;	5553	is_global = func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL;
5368	err = btf_check_func_arg_match(env, subprog, caller->regs);	5554	err = btf_check_subprog_arg_match(env, subprog, caller->regs);
5369	if (err == -EFAULT)	5555	if (err == -EFAULT)
5370	return err;	5556	return err;
5371	if (is_global) {	5557	if (is_global) {
@@ -5960,6 +6146,98 @@ static int check_helper_call(struct bpf_verifier_env env, struct bpf_insn insn · 588 unmodified lines
5960	return 0;	6146	return 0;
5961	}	6147	}
5962		6148
		6149	/* mark_btf_func_reg_size() is used when the reg size is determined by
		6150	* the BTF func_proto's return value size and argument.
		6151	*/
		6152	static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno,
		6153	size_t reg_size)
		6154	{
		6155	struct bpf_reg_state *reg = &cur_regs(env)[regno];
		6156
		6157	if (regno == BPF_REG_0) {
		6158	/* Function return value */
		6159	reg->live \|= REG_LIVE_WRITTEN;
		6160	reg->subreg_def = reg_size == sizeof(u64) ?
		6161	DEF_NOT_SUBREG : env->insn_idx + 1;
		6162	} else {
		6163	/* Function argument */
		6164	if (reg_size == sizeof(u64)) {
		6165	mark_insn_zext(env, reg);
		6166	mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
		6167	} else {
		6168	mark_reg_read(env, reg, reg->parent, REG_LIVE_READ32);
		6169	}
		6170	}
		6171	}
		6172
		6173	static int check_kfunc_call(struct bpf_verifier_env env, struct bpf_insn insn)
		6174	{
		6175	const struct btf_type t, func, func_proto, ptr_type;
		6176	struct bpf_reg_state *regs = cur_regs(env);
		6177	const char func_name, ptr_type_name;
		6178	u32 i, nargs, func_id, ptr_type_id;
		6179	const struct btf_param *args;
		6180	int err;
		6181
		6182	func_id = insn->imm;
		6183	func = btf_type_by_id(btf_vmlinux, func_id);
		6184	func_name = btf_name_by_offset(btf_vmlinux, func->name_off);
		6185	func_proto = btf_type_by_id(btf_vmlinux, func->type);
		6186
		6187	if (!env->ops->check_kfunc_call \|\|
		6188	!env->ops->check_kfunc_call(func_id)) {
		6189	verbose(env, "calling kernel function %s is not allowed\n",
		6190	func_name);
		6191	return -EACCES;
		6192	}
		6193
		6194	/* Check the arguments */
		6195	err = btf_check_kfunc_arg_match(env, btf_vmlinux, func_id, regs);
		6196	if (err)
		6197	return err;
		6198
		6199	for (i = 0; i < CALLER_SAVED_REGS; i++)
		6200	mark_reg_not_init(env, regs, caller_saved[i]);
		6201
		6202	/* Check return type */
		6203	t = btf_type_skip_modifiers(btf_vmlinux, func_proto->type, NULL);
		6204	if (btf_type_is_scalar(t)) {
		6205	mark_reg_unknown(env, regs, BPF_REG_0);
		6206	mark_btf_func_reg_size(env, BPF_REG_0, t->size);
		6207	} else if (btf_type_is_ptr(t)) {
		6208	ptr_type = btf_type_skip_modifiers(btf_vmlinux, t->type,
		6209	&ptr_type_id);
		6210	if (!btf_type_is_struct(ptr_type)) {
		6211	ptr_type_name = btf_name_by_offset(btf_vmlinux,
		6212	ptr_type->name_off);
		6213	verbose(env, "kernel function %s returns pointer type %s %s is not supported\n",
		6214	func_name, btf_type_str(ptr_type),
		6215	ptr_type_name);
		6216	return -EINVAL;
		6217	}
		6218	mark_reg_known_zero(env, regs, BPF_REG_0);
		6219	regs[BPF_REG_0].btf = btf_vmlinux;
		6220	regs[BPF_REG_0].type = PTR_TO_BTF_ID;
		6221	regs[BPF_REG_0].btf_id = ptr_type_id;
		6222	mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *));
		6223	} /* else { add_kfunc_call() ensures it is btf_type_is_void(t) } */
		6224
		6225	nargs = btf_type_vlen(func_proto);
		6226	args = (const struct btf_param *)(func_proto + 1);
		6227	for (i = 0; i < nargs; i++) {
		6228	u32 regno = i + 1;
		6229
		6230	t = btf_type_skip_modifiers(btf_vmlinux, args[i].type, NULL);
		6231	if (btf_type_is_ptr(t))
		6232	mark_btf_func_reg_size(env, regno, sizeof(void *));
		6233	else
		6234	/* scalar. ensured by btf_check_kfunc_arg_match() */
		6235	mark_btf_func_reg_size(env, regno, t->size);
		6236	}
		6237
		6238	return 0;
		6239	}
		6240
5963	static bool signed_add_overflows(s64 a, s64 b)	6241	static bool signed_add_overflows(s64 a, s64 b)
5964	{	6242	{
5965	/* Do the add in u64, where overflow is well-defined */	6243	/* Do the add in u64, where overflow is well-defined */
@@ -6062,19 +6340,6 @@ static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg, · 96 unmodified lines
6062	else	6340	else
6063	*ptr_limit = -off - 1;	6341	*ptr_limit = -off - 1;
6064	return *ptr_limit >= max ? -ERANGE : 0;	6342	return *ptr_limit >= max ? -ERANGE : 0;
6065	case PTR_TO_MAP_KEY:
6066	/* Currently, this code is not exercised as the only use
6067	* is bpf_for_each_map_elem() helper which requires
6068	* bpf_capble. The code has been tested manually for
6069	* future use.
6070	*/
6071	if (mask_to_left) {
6072	*ptr_limit = ptr_reg->umax_value + ptr_reg->off;
6073	} else {
6074	off = ptr_reg->smin_value + ptr_reg->off;
6075	*ptr_limit = ptr_reg->map_ptr->key_size - off;
6076	}
6077	return 0;
6078	case PTR_TO_MAP_VALUE:	6343	case PTR_TO_MAP_VALUE:
6079	max = ptr_reg->map_ptr->value_size;	6344	max = ptr_reg->map_ptr->value_size;
6080	if (mask_to_left) {	6345	if (mask_to_left) {
@@ -6281,7 +6546,6 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, · 200 unmodified lines
6281	verbose(env, "R%d pointer arithmetic on %s prohibited\n",	6546	verbose(env, "R%d pointer arithmetic on %s prohibited\n",
6282	dst, reg_type_str[ptr_reg->type]);	6547	dst, reg_type_str[ptr_reg->type]);
6283	return -EACCES;	6548	return -EACCES;
6284	case PTR_TO_MAP_KEY:
6285	case PTR_TO_MAP_VALUE:	6549	case PTR_TO_MAP_VALUE:
6286	if (!env->allow_ptr_leaks && !known && (smin_val < 0) != (smax_val < 0)) {	6550	if (!env->allow_ptr_leaks && !known && (smin_val < 0) != (smax_val < 0)) {
6287	verbose(env, "R%d has unknown scalar with mixed signed bounds, pointer arithmetic with it prohibited for !root\n",	6551	verbose(env, "R%d has unknown scalar with mixed signed bounds, pointer arithmetic with it prohibited for !root\n",
@@ -10176,6 +10440,7 @@ static int do_check(struct bpf_verifier_env *env) · 3888 unmodified lines
10176		10440
10177	if (env->log.level & BPF_LOG_LEVEL) {	10441	if (env->log.level & BPF_LOG_LEVEL) {
10178	const struct bpf_insn_cbs cbs = {	10442	const struct bpf_insn_cbs cbs = {
		10443	.cb_call = disasm_kfunc_name,
10179	.cb_print = verbose,	10444	.cb_print = verbose,
10180	.private_data = env,	10445	.private_data = env,
10181	};	10446	};
@@ -10323,7 +10588,8 @@ static int do_check(struct bpf_verifier_env *env) · 141 unmodified lines
10323	if (BPF_SRC(insn->code) != BPF_K \|\|	10588	if (BPF_SRC(insn->code) != BPF_K \|\|
10324	insn->off != 0 \|\|	10589	insn->off != 0 \|\|
10325	(insn->src_reg != BPF_REG_0 &&	10590	(insn->src_reg != BPF_REG_0 &&
10326	insn->src_reg != BPF_PSEUDO_CALL) \|\|	10591	insn->src_reg != BPF_PSEUDO_CALL &&
		10592	insn->src_reg != BPF_PSEUDO_KFUNC_CALL) \|\|
10327	insn->dst_reg != BPF_REG_0 \|\|	10593	insn->dst_reg != BPF_REG_0 \|\|
10328	class == BPF_JMP32) {	10594	class == BPF_JMP32) {
10329	verbose(env, "BPF_CALL uses reserved fields\n");	10595	verbose(env, "BPF_CALL uses reserved fields\n");
@@ -10338,6 +10604,8 @@ static int do_check(struct bpf_verifier_env *env) · 8 unmodified lines
10338	}	10604	}
10339	if (insn->src_reg == BPF_PSEUDO_CALL)	10605	if (insn->src_reg == BPF_PSEUDO_CALL)
10340	err = check_func_call(env, insn, &env->insn_idx);	10606	err = check_func_call(env, insn, &env->insn_idx);
		10607	else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL)
		10608	err = check_kfunc_call(env, insn);
10341	else	10609	else
10342	err = check_helper_call(env, insn, &env->insn_idx);	10610	err = check_helper_call(env, insn, &env->insn_idx);
10343	if (err)	10611	if (err)
@@ -11648,6 +11916,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 1304 unmodified lines
11648	func[i]->aux->name[0] = 'F';	11916	func[i]->aux->name[0] = 'F';
11649	func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;	11917	func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;
11650	func[i]->jit_requested = 1;	11918	func[i]->jit_requested = 1;
		11919	func[i]->aux->kfunc_tab = prog->aux->kfunc_tab;
11651	func[i]->aux->linfo = prog->aux->linfo;	11920	func[i]->aux->linfo = prog->aux->linfo;
11652	func[i]->aux->nr_linfo = prog->aux->nr_linfo;	11921	func[i]->aux->nr_linfo = prog->aux->nr_linfo;
11653	func[i]->aux->jited_linfo = prog->aux->jited_linfo;	11922	func[i]->aux->jited_linfo = prog->aux->jited_linfo;
@@ -11755,7 +12024,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 101 unmodified lines
11755	prog->bpf_func = func[0]->bpf_func;	12024	prog->bpf_func = func[0]->bpf_func;
11756	prog->aux->func = func;	12025	prog->aux->func = func;
11757	prog->aux->func_cnt = env->subprog_cnt;	12026	prog->aux->func_cnt = env->subprog_cnt;
11758	bpf_prog_free_unused_jited_linfo(prog);	12027	bpf_prog_jit_attempt_done(prog);
11759	return 0;	12028	return 0;
11760	out_free:	12029	out_free:
11761	for (i = 0; i < env->subprog_cnt; i++) {	12030	for (i = 0; i < env->subprog_cnt; i++) {
@@ -11778,7 +12047,7 @@ out_undo_insn: · 16 unmodified lines
11778	insn->off = 0;	12047	insn->off = 0;
11779	insn->imm = env->insn_aux_data[i].call_imm;	12048	insn->imm = env->insn_aux_data[i].call_imm;
11780	}	12049	}
11781	bpf_prog_free_jited_linfo(prog);	12050	bpf_prog_jit_attempt_done(prog);
11782	return err;	12051	return err;
11783	}	12052	}
11784		12053
@@ -11787,6 +12056,7 @@ static int fixup_call_args(struct bpf_verifier_env *env) · 2 unmodified lines
11787	#ifndef CONFIG_BPF_JIT_ALWAYS_ON	12056	#ifndef CONFIG_BPF_JIT_ALWAYS_ON
11788	struct bpf_prog *prog = env->prog;	12057	struct bpf_prog *prog = env->prog;
11789	struct bpf_insn *insn = prog->insnsi;	12058	struct bpf_insn *insn = prog->insnsi;
		12059	bool has_kfunc_call = bpf_prog_has_kfunc_call(prog);
11790	int i, depth;	12060	int i, depth;
11791	#endif	12061	#endif
11792	int err = 0;	12062	int err = 0;
@@ -11800,6 +12070,10 @@ static int fixup_call_args(struct bpf_verifier_env *env) · 7 unmodified lines
11800	return err;	12070	return err;
11801	}	12071	}
11802	#ifndef CONFIG_BPF_JIT_ALWAYS_ON	12072	#ifndef CONFIG_BPF_JIT_ALWAYS_ON
		12073	if (has_kfunc_call) {
		12074	verbose(env, "calling kernel functions are not allowed in non-JITed programs\n");
		12075	return -EINVAL;
		12076	}
11803	if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) {	12077	if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) {
11804	/* When JIT fails the progs with bpf2bpf calls and tail_calls	12078	/* When JIT fails the progs with bpf2bpf calls and tail_calls
11805	* have to be rejected, since interpreter doesn't support them yet.	12079	* have to be rejected, since interpreter doesn't support them yet.
@@ -11828,6 +12102,26 @@ static int fixup_call_args(struct bpf_verifier_env *env) · 22 unmodified lines
11828	return err;	12102	return err;
11829	}	12103	}
11830		12104
		12105	static int fixup_kfunc_call(struct bpf_verifier_env *env,
		12106	struct bpf_insn *insn)
		12107	{
		12108	const struct bpf_kfunc_desc *desc;
		12109
		12110	/* insn->imm has the btf func_id. Replace it with
		12111	* an address (relative to __bpf_base_call).
		12112	*/
		12113	desc = find_kfunc_desc(env->prog, insn->imm);
		12114	if (!desc) {
		12115	verbose(env, "verifier internal error: kernel function descriptor not found for func_id %u\n",
		12116	insn->imm);
		12117	return -EFAULT;
		12118	}
		12119
		12120	insn->imm = desc->imm;
		12121
		12122	return 0;
		12123	}
		12124
11831	/* Do various post-verification rewrites in a single program pass.	12125	/* Do various post-verification rewrites in a single program pass.
11832	* These rewrites simplify JIT and interpreter implementations.	12126	* These rewrites simplify JIT and interpreter implementations.
11833	*/	12127	*/
@@ -11963,6 +12257,12 @@ static int do_misc_fixups(struct bpf_verifier_env *env) · 129 unmodified lines
11963	continue;	12257	continue;
11964	if (insn->src_reg == BPF_PSEUDO_CALL)	12258	if (insn->src_reg == BPF_PSEUDO_CALL)
11965	continue;	12259	continue;
		12260	if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
		12261	ret = fixup_kfunc_call(env, insn);
		12262	if (ret)
		12263	return ret;
		12264	continue;
		12265	}
11966		12266
11967	if (insn->imm == BPF_FUNC_get_route_realm)	12267	if (insn->imm == BPF_FUNC_get_route_realm)
11968	prog->dst_needed = 1;	12268	prog->dst_needed = 1;
@@ -12192,6 +12492,8 @@ patch_call_imm: · 223 unmodified lines
12192	}	12492	}
12193	}	12493	}
12194		12494
		12495	sort_kfunc_descs_by_imm(env->prog);
		12496
12195	return 0;	12497	return 0;
12196	}	12498	}
12197		12499
@@ -12302,7 +12604,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) · 104 unmodified lines
12302	/* 1st arg to a function */	12604	/* 1st arg to a function */
12303	regs[BPF_REG_1].type = PTR_TO_CTX;	12605	regs[BPF_REG_1].type = PTR_TO_CTX;
12304	mark_reg_known_zero(env, regs, BPF_REG_1);	12606	mark_reg_known_zero(env, regs, BPF_REG_1);
12305	ret = btf_check_func_arg_match(env, subprog, regs);	12607	ret = btf_check_subprog_arg_match(env, subprog, regs);
12306	if (ret == -EFAULT)	12608	if (ret == -EFAULT)
12307	/* unlikely verifier bug. abort.	12609	/* unlikely verifier bug. abort.
12308	* ret == 0 and ret < 0 are sadly acceptable for	12610	* ret == 0 and ret < 0 are sadly acceptable for
@@ -12897,6 +13199,10 @@ int bpf_check(struct bpf_prog *prog, union bpf_attr attr, · 588 unmodified lines
12897	if (!env->explored_states)	13199	if (!env->explored_states)
12898	goto skip_full_check;	13200	goto skip_full_check;
12899		13201
		13202	ret = add_subprog_and_kfunc(env);
		13203	if (ret < 0)
		13204	goto skip_full_check;
		13205
12900	ret = check_subprogs(env);	13206	ret = check_subprogs(env);
12901	if (ret < 0)	13207	if (ret < 0)
12902	goto skip_full_check;	13208	goto skip_full_check;

Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code