Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Alexei Starovoitov says: ==================== pull-request: bpf-next 2021-04-01 The following pull-request contains BPF updates for your *net-next* tree. We've added 68 non-merge commits during the last 7 day(s) which contain a total of 70 files changed, 2944 insertions(+), 1139 deletions(-). The main changes are: 1) UDP support for sockmap, from Cong. 2) Verifier merge conflict resolution fix, from Daniel. 3) xsk selftests enhancements, from Maciej. 4) Unstable helpers aka kernel func calling, from Martin. 5) Batches ops for LPM map, from Pedro. 6) Fix race in bpf_get_local_storage, from Yonghong. ====================

David S. Miller committed 5 years ago

Commit c2bcb4c2 parents bd78980 89d69c570 files · +2944 −1139Browse files

Showing the changes to kernel (8 of 70 files). Show all

Unified · Split

8 files

kernel/bpf/btf.c +143 −76
kernel/bpf/core.c +24 −23
kernel/bpf/disasm.c +10 −3
kernel/bpf/helpers.c +11 −4
kernel/bpf/local_storage.c +3 −2
kernel/bpf/lpm_trie.c +3 −0
kernel/bpf/syscall.c +4 −1
kernel/bpf/verifier.c +348 −42

kernel/bpf/btf.c+143−76

@@ -283,7 +283,7 @@ static const char * const btf_kind_str[NR_BTF_KINDS] = { · 282 unmodified lines
283	283	[BTF_KIND_FLOAT] = "FLOAT",
284	284	};
285	285
286		−static const char btf_type_str(const struct btf_type t)
	286	+const char btf_type_str(const struct btf_type t)
287	287	{
288	288	return btf_kind_str[BTF_INFO_KIND(t->info)];
289	289	}
@@ -789,7 +789,6 @@ static const struct btf_type btf_type_skip_qualifiers(const struct btf btf, · 499 unmodified lines
789	789
790	790	while (btf_type_is_modifier(t) &&
791	791	BTF_INFO_KIND(t->info) != BTF_KIND_TYPEDEF) {
792		− id = t->type;
793	792	t = btf_type_by_id(btf, t->type);
794	793	}
795	794
@@ -4377,7 +4376,7 @@ static u8 bpf_ctx_convert_map[] = { · 3581 unmodified lines
4377	4376	#undef BPF_LINK_TYPE
4378	4377
4379	4378	static const struct btf_member *
4380		−btf_get_prog_ctx_type(struct bpf_verifier_log log, struct btf btf,
	4379	+btf_get_prog_ctx_type(struct bpf_verifier_log log, const struct btf btf,
4381	4380	const struct btf_type *t, enum bpf_prog_type prog_type,
4382	4381	int arg)
4383	4382	{
@@ -5362,122 +5361,190 @@ int btf_check_type_match(struct bpf_verifier_log log, const struct bpf_prog pr · 978 unmodified lines
5362	5361	return btf_check_func_type_match(log, btf1, t1, btf2, t2);
5363	5362	}
5364	5363
5365		−/* Compare BTF of a function with given bpf_reg_state.
5366		− * Returns:
5367		− * EFAULT - there is a verifier bug. Abort verification.
5368		− * EINVAL - there is a type mismatch or BTF is not available.
5369		− * 0 - BTF matches with what bpf_reg_state expects.
5370		− * Only PTR_TO_CTX and SCALAR_VALUE states are recognized.
5371		− */
5372		−int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog,
5373		− struct bpf_reg_state *regs)
	5364	+static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = {
	5365	+#ifdef CONFIG_NET
	5366	+ [PTR_TO_SOCKET] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK],
	5367	+ [PTR_TO_SOCK_COMMON] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
	5368	+ [PTR_TO_TCP_SOCK] = &btf_sock_ids[BTF_SOCK_TYPE_TCP],
	5369	+#endif
	5370	+};
	5371	+
	5372	+static int btf_check_func_arg_match(struct bpf_verifier_env *env,
	5373	+ const struct btf *btf, u32 func_id,
	5374	+ struct bpf_reg_state *regs,
	5375	+ bool ptr_to_mem_ok)
5374	5376	{
5375	5377	struct bpf_verifier_log *log = &env->log;
5376		− struct bpf_prog *prog = env->prog;
5377		− struct btf *btf = prog->aux->btf;
5378		− const struct btf_param *args;
	5378	+ const char func_name, ref_tname;
5379	5379	const struct btf_type t, ref_t;
5380		− u32 i, nargs, btf_id, type_size;
5381		− const char *tname;
5382		− bool is_global;
5383		−
5384		− if (!prog->aux->func_info)
5385		− return -EINVAL;
5386		−
5387		− btf_id = prog->aux->func_info[subprog].type_id;
5388		− if (!btf_id)
5389		− return -EFAULT;
5390		−
5391		− if (prog->aux->func_info_aux[subprog].unreliable)
5392		− return -EINVAL;
	5380	+ const struct btf_param *args;
	5381	+ u32 i, nargs, ref_id;
5393	5382
5394		− t = btf_type_by_id(btf, btf_id);
	5383	+ t = btf_type_by_id(btf, func_id);
5395	5384	if (!t \|\| !btf_type_is_func(t)) {
5396	5385	/* These checks were already done by the verifier while loading
5397		− * struct bpf_func_info
	5386	+ * struct bpf_func_info or in add_kfunc_call().
5398	5387	*/
5399		− bpf_log(log, "BTF of func#%d doesn't point to KIND_FUNC\n",
5400		− subprog);
	5388	+ bpf_log(log, "BTF of func_id %u doesn't point to KIND_FUNC\n",
	5389	+ func_id);
5401	5390	return -EFAULT;
5402	5391	}
5403		− tname = btf_name_by_offset(btf, t->name_off);
	5392	+ func_name = btf_name_by_offset(btf, t->name_off);
5404	5393
5405	5394	t = btf_type_by_id(btf, t->type);
5406	5395	if (!t \|\| !btf_type_is_func_proto(t)) {
5407		− bpf_log(log, "Invalid BTF of func %s\n", tname);
	5396	+ bpf_log(log, "Invalid BTF of func %s\n", func_name);
5408	5397	return -EFAULT;
5409	5398	}
5410	5399	args = (const struct btf_param *)(t + 1);
5411	5400	nargs = btf_type_vlen(t);
5412	5401	if (nargs > MAX_BPF_FUNC_REG_ARGS) {
5413		− bpf_log(log, "Function %s has %d > %d args\n", tname, nargs,
	5402	+ bpf_log(log, "Function %s has %d > %d args\n", func_name, nargs,
5414	5403	MAX_BPF_FUNC_REG_ARGS);
5415		− goto out;
	5404	+ return -EINVAL;
5416	5405	}
5417	5406
5418		− is_global = prog->aux->func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL;
5419	5407	/* check that BTF function arguments match actual types that the
5420	5408	* verifier sees.
5421	5409	*/
5422	5410	for (i = 0; i < nargs; i++) {
5423		− struct bpf_reg_state *reg = &regs[i + 1];
	5411	+ u32 regno = i + 1;
	5412	+ struct bpf_reg_state *reg = &regs[regno];
5424	5413
5425		− t = btf_type_by_id(btf, args[i].type);
5426		− while (btf_type_is_modifier(t))
5427		− t = btf_type_by_id(btf, t->type);
5428		− if (btf_type_is_int(t) \|\| btf_type_is_enum(t)) {
	5414	+ t = btf_type_skip_modifiers(btf, args[i].type, NULL);
	5415	+ if (btf_type_is_scalar(t)) {
5429	5416	if (reg->type == SCALAR_VALUE)
5430	5417	continue;
5431		− bpf_log(log, "R%d is not a scalar\n", i + 1);
5432		− goto out;
	5418	+ bpf_log(log, "R%d is not a scalar\n", regno);
	5419	+ return -EINVAL;
5433	5420	}
5434		− if (btf_type_is_ptr(t)) {
	5421	+
	5422	+ if (!btf_type_is_ptr(t)) {
	5423	+ bpf_log(log, "Unrecognized arg#%d type %s\n",
	5424	+ i, btf_type_str(t));
	5425	+ return -EINVAL;
	5426	+ }
	5427	+
	5428	+ ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id);
	5429	+ ref_tname = btf_name_by_offset(btf, ref_t->name_off);
	5430	+ if (btf_is_kernel(btf)) {
	5431	+ const struct btf_type *reg_ref_t;
	5432	+ const struct btf *reg_btf;
	5433	+ const char *reg_ref_tname;
	5434	+ u32 reg_ref_id;
	5435	+
	5436	+ if (!btf_type_is_struct(ref_t)) {
	5437	+ bpf_log(log, "kernel function %s args#%d pointer type %s %s is not supported\n",
	5438	+ func_name, i, btf_type_str(ref_t),
	5439	+ ref_tname);
	5440	+ return -EINVAL;
	5441	+ }
	5442	+
	5443	+ if (reg->type == PTR_TO_BTF_ID) {
	5444	+ reg_btf = reg->btf;
	5445	+ reg_ref_id = reg->btf_id;
	5446	+ } else if (reg2btf_ids[reg->type]) {
	5447	+ reg_btf = btf_vmlinux;
	5448	+ reg_ref_id = *reg2btf_ids[reg->type];
	5449	+ } else {
	5450	+ bpf_log(log, "kernel function %s args#%d expected pointer to %s %s but R%d is not a pointer to btf_id\n",
	5451	+ func_name, i,
	5452	+ btf_type_str(ref_t), ref_tname, regno);
	5453	+ return -EINVAL;
	5454	+ }
	5455	+
	5456	+ reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id,
	5457	+ &reg_ref_id);
	5458	+ reg_ref_tname = btf_name_by_offset(reg_btf,
	5459	+ reg_ref_t->name_off);
	5460	+ if (!btf_struct_ids_match(log, reg_btf, reg_ref_id,
	5461	+ reg->off, btf, ref_id)) {
	5462	+ bpf_log(log, "kernel function %s args#%d expected pointer to %s %s but R%d has a pointer to %s %s\n",
	5463	+ func_name, i,
	5464	+ btf_type_str(ref_t), ref_tname,
	5465	+ regno, btf_type_str(reg_ref_t),
	5466	+ reg_ref_tname);
	5467	+ return -EINVAL;
	5468	+ }
	5469	+ } else if (btf_get_prog_ctx_type(log, btf, t,
	5470	+ env->prog->type, i)) {
5435	5471	/* If function expects ctx type in BTF check that caller
5436	5472	* is passing PTR_TO_CTX.
5437	5473	*/
5438		− if (btf_get_prog_ctx_type(log, btf, t, prog->type, i)) {
5439		− if (reg->type != PTR_TO_CTX) {
5440		− bpf_log(log,
5441		− "arg#%d expected pointer to ctx, but got %s\n",
5442		− i, btf_kind_str[BTF_INFO_KIND(t->info)]);
5443		− goto out;
5444		− }
5445		− if (check_ctx_reg(env, reg, i + 1))
5446		− goto out;
5447		− continue;
	5474	+ if (reg->type != PTR_TO_CTX) {
	5475	+ bpf_log(log,
	5476	+ "arg#%d expected pointer to ctx, but got %s\n",
	5477	+ i, btf_type_str(t));
	5478	+ return -EINVAL;
5448	5479	}
	5480	+ if (check_ctx_reg(env, reg, regno))
	5481	+ return -EINVAL;
	5482	+ } else if (ptr_to_mem_ok) {
	5483	+ const struct btf_type *resolve_ret;
	5484	+ u32 type_size;
5449	5485
5450		− if (!is_global)
5451		− goto out;
5452		−
5453		− t = btf_type_skip_modifiers(btf, t->type, NULL);
5454		−
5455		− ref_t = btf_resolve_size(btf, t, &type_size);
5456		− if (IS_ERR(ref_t)) {
	5486	+ resolve_ret = btf_resolve_size(btf, ref_t, &type_size);
	5487	+ if (IS_ERR(resolve_ret)) {
5457	5488	bpf_log(log,
5458		− "arg#%d reference type('%s %s') size cannot be determined: %ld\n",
5459		− i, btf_type_str(t), btf_name_by_offset(btf, t->name_off),
5460		− PTR_ERR(ref_t));
5461		− goto out;
	5489	+ "arg#%d reference type('%s %s') size cannot be determined: %ld\n",
	5490	+ i, btf_type_str(ref_t), ref_tname,
	5491	+ PTR_ERR(resolve_ret));
	5492	+ return -EINVAL;
5462	5493	}
5463	5494
5464		− if (check_mem_reg(env, reg, i + 1, type_size))
5465		− goto out;
5466		−
5467		− continue;
	5495	+ if (check_mem_reg(env, reg, regno, type_size))
	5496	+ return -EINVAL;
	5497	+ } else {
	5498	+ return -EINVAL;
5468	5499	}
5469		− bpf_log(log, "Unrecognized arg#%d type %s\n",
5470		− i, btf_kind_str[BTF_INFO_KIND(t->info)]);
5471		− goto out;
5472	5500	}
	5501	+
5473	5502	return 0;
5474		−out:
	5503	+}
	5504	+
	5505	+/* Compare BTF of a function with given bpf_reg_state.
	5506	+ * Returns:
	5507	+ * EFAULT - there is a verifier bug. Abort verification.
	5508	+ * EINVAL - there is a type mismatch or BTF is not available.
	5509	+ * 0 - BTF matches with what bpf_reg_state expects.
	5510	+ * Only PTR_TO_CTX and SCALAR_VALUE states are recognized.
	5511	+ */
	5512	+int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog,
	5513	+ struct bpf_reg_state *regs)
	5514	+{
	5515	+ struct bpf_prog *prog = env->prog;
	5516	+ struct btf *btf = prog->aux->btf;
	5517	+ bool is_global;
	5518	+ u32 btf_id;
	5519	+ int err;
	5520	+
	5521	+ if (!prog->aux->func_info)
	5522	+ return -EINVAL;
	5523	+
	5524	+ btf_id = prog->aux->func_info[subprog].type_id;
	5525	+ if (!btf_id)
	5526	+ return -EFAULT;
	5527	+
	5528	+ if (prog->aux->func_info_aux[subprog].unreliable)
	5529	+ return -EINVAL;
	5530	+
	5531	+ is_global = prog->aux->func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL;
	5532	+ err = btf_check_func_arg_match(env, btf, btf_id, regs, is_global);
	5533	+
5475	5534	/* Compiler optimizations can remove arguments from static functions
5476	5535	* or mismatched type can be passed into a global function.
5477	5536	* In such cases mark the function as unreliable from BTF point of view.
5478	5537	*/
5479		− prog->aux->func_info_aux[subprog].unreliable = true;
5480		− return -EINVAL;
	5538	+ if (err)
	5539	+ prog->aux->func_info_aux[subprog].unreliable = true;
	5540	+ return err;
	5541	+}
	5542	+
	5543	+int btf_check_kfunc_arg_match(struct bpf_verifier_env *env,
	5544	+ const struct btf *btf, u32 func_id,
	5545	+ struct bpf_reg_state *regs)
	5546	+{
	5547	+ return btf_check_func_arg_match(env, btf, func_id, regs, false);
5481	5548	}
5482	5549
5483	5550	/* Convert BTF of a function into bpf_reg_state if possible

kernel/bpf/core.c+24−23

@@ -143,25 +143,25 @@ int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog) · 142 unmodified lines
143	143	if (!prog->aux->nr_linfo \|\| !prog->jit_requested)
144	144	return 0;
145	145
146		− prog->aux->jited_linfo = kcalloc(prog->aux->nr_linfo,
147		− sizeof(*prog->aux->jited_linfo),
148		− GFP_KERNEL_ACCOUNT \| __GFP_NOWARN);
	146	+ prog->aux->jited_linfo = kvcalloc(prog->aux->nr_linfo,
	147	+ sizeof(*prog->aux->jited_linfo),
	148	+ GFP_KERNEL_ACCOUNT \| __GFP_NOWARN);
149	149	if (!prog->aux->jited_linfo)
150	150	return -ENOMEM;
151	151
152	152	return 0;
153	153	}
154	154
155		−void bpf_prog_free_jited_linfo(struct bpf_prog *prog)
	155	+void bpf_prog_jit_attempt_done(struct bpf_prog *prog)
156	156	{
157		− kfree(prog->aux->jited_linfo);
158		− prog->aux->jited_linfo = NULL;
159		−}
	157	+ if (prog->aux->jited_linfo &&
	158	+ (!prog->jited \|\| !prog->aux->jited_linfo[0])) {
	159	+ kvfree(prog->aux->jited_linfo);
	160	+ prog->aux->jited_linfo = NULL;
	161	+ }
160	162
161		−void bpf_prog_free_unused_jited_linfo(struct bpf_prog *prog)
162		−{
163		− if (prog->aux->jited_linfo && !prog->aux->jited_linfo[0])
164		− bpf_prog_free_jited_linfo(prog);
	163	+ kfree(prog->aux->kfunc_tab);
	164	+ prog->aux->kfunc_tab = NULL;
165	165	}
166	166
167	167	/* The jit engine is responsible to provide an array
@@ -217,12 +217,6 @@ void bpf_prog_fill_jited_linfo(struct bpf_prog *prog, · 49 unmodified lines
217	217	insn_to_jit_off[linfo[i].insn_off - insn_start - 1];
218	218	}
219	219
220		−void bpf_prog_free_linfo(struct bpf_prog *prog)
221		−{
222		− bpf_prog_free_jited_linfo(prog);
223		− kvfree(prog->aux->linfo);
224		−}
225		−
226	220	struct bpf_prog bpf_prog_realloc(struct bpf_prog fp_old, unsigned int size,
227	221	gfp_t gfp_extra_flags)
228	222	{
@@ -1849,9 +1843,15 @@ struct bpf_prog bpf_prog_select_runtime(struct bpf_prog fp, int *err) · 1620 unmodified lines
1849	1843	/* In case of BPF to BPF calls, verifier did all the prep
1850	1844	* work with regards to JITing, etc.
1851	1845	*/
	1846	+ bool jit_needed = false;
	1847	+
1852	1848	if (fp->bpf_func)
1853	1849	goto finalize;
1854	1850
	1851	+ if (IS_ENABLED(CONFIG_BPF_JIT_ALWAYS_ON) \|\|
	1852	+ bpf_prog_has_kfunc_call(fp))
	1853	+ jit_needed = true;
	1854	+
1855	1855	bpf_prog_select_func(fp);
1856	1856
1857	1857	/* eBPF JITs can rewrite the program in case constant
@@ -1866,14 +1866,10 @@ struct bpf_prog bpf_prog_select_runtime(struct bpf_prog fp, int *err) · 8 unmodified lines
1866	1866	return fp;
1867	1867
1868	1868	fp = bpf_int_jit_compile(fp);
1869		− if (!fp->jited) {
1870		− bpf_prog_free_jited_linfo(fp);
1871		−#ifdef CONFIG_BPF_JIT_ALWAYS_ON
	1869	+ bpf_prog_jit_attempt_done(fp);
	1870	+ if (!fp->jited && jit_needed) {
1872	1871	*err = -ENOTSUPP;
1873	1872	return fp;
1874		−#endif
1875		− } else {
1876		− bpf_prog_free_unused_jited_linfo(fp);
1877	1873	}
1878	1874	} else {
1879	1875	*err = bpf_prog_offload_compile(fp);
@@ -2354,6 +2350,11 @@ bool __weak bpf_jit_needs_zext(void) · 474 unmodified lines
2354	2350	return false;
2355	2351	}
2356	2352
	2353	+bool __weak bpf_jit_supports_kfunc_call(void)
	2354	+{
	2355	+ return false;
	2356	+}
	2357	+
2357	2358	/* To execute LD_ABS/LD_IND instructions __bpf_prog_run() may call
2358	2359	* skb_copy_bits(), so provide a weak definition of it for NET-less config.
2359	2360	*/

kernel/bpf/disasm.c+10−3

@@ -19,16 +19,23 @@ static const char __func_get_name(const struct bpf_insn_cbs cbs, · 18 unmodified lines
19	19	{
20	20	BUILD_BUG_ON(ARRAY_SIZE(func_id_str) != __BPF_FUNC_MAX_ID);
21	21
22		− if (insn->src_reg != BPF_PSEUDO_CALL &&
	22	+ if (!insn->src_reg &&
23	23	insn->imm >= 0 && insn->imm < __BPF_FUNC_MAX_ID &&
24	24	func_id_str[insn->imm])
25	25	return func_id_str[insn->imm];
26	26
27		− if (cbs && cbs->cb_call)
28		− return cbs->cb_call(cbs->private_data, insn);
	27	+ if (cbs && cbs->cb_call) {
	28	+ const char *res;
	29	+
	30	+ res = cbs->cb_call(cbs->private_data, insn);
	31	+ if (res)
	32	+ return res;
	33	+ }
29	34
30	35	if (insn->src_reg == BPF_PSEUDO_CALL)
31	36	snprintf(buff, len, "%+d", insn->imm);
	37	+ else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL)
	38	+ snprintf(buff, len, "kernel-function");
32	39
33	40	return buff;
34	41	}

kernel/bpf/helpers.c+11−4

@@ -382,8 +382,8 @@ const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto = { · 381 unmodified lines
382	382	};
383	383
384	384	#ifdef CONFIG_CGROUP_BPF
385		−DECLARE_PER_CPU(struct bpf_cgroup_storage*,
386		− bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]);
	385	+DECLARE_PER_CPU(struct bpf_cgroup_storage_info,
	386	+ bpf_cgroup_storage_info[BPF_CGROUP_STORAGE_NEST_MAX]);
387	387
388	388	BPF_CALL_2(bpf_get_local_storage, struct bpf_map *, map, u64, flags)
389	389	{
@@ -392,10 +392,17 @@ BPF_CALL_2(bpf_get_local_storage, struct bpf_map *, map, u64, flags) · 2 unmodified lines
392	392	* verifier checks that its value is correct.
393	393	*/
394	394	enum bpf_cgroup_storage_type stype = cgroup_storage_type(map);
395		− struct bpf_cgroup_storage *storage;
	395	+ struct bpf_cgroup_storage *storage = NULL;
396	396	void *ptr;
	397	+ int i;
397	398
398		− storage = this_cpu_read(bpf_cgroup_storage[stype]);
	399	+ for (i = 0; i < BPF_CGROUP_STORAGE_NEST_MAX; i++) {
	400	+ if (unlikely(this_cpu_read(bpf_cgroup_storage_info[i].task) != current))
	401	+ continue;
	402	+
	403	+ storage = this_cpu_read(bpf_cgroup_storage_info[i].storage[stype]);
	404	+ break;
	405	+ }
399	406
400	407	if (stype == BPF_CGROUP_STORAGE_SHARED)
401	408	ptr = &READ_ONCE(storage->buf)->data[0];

kernel/bpf/local_storage.c+3−2

@@ -9,10 +9,11 @@ · 8 unmodified lines
9	9	#include <linux/slab.h>
10	10	#include <uapi/linux/btf.h>
11	11
12		−DEFINE_PER_CPU(struct bpf_cgroup_storage*, bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]);
13		−
14	12	#ifdef CONFIG_CGROUP_BPF
15	13
	14	+DEFINE_PER_CPU(struct bpf_cgroup_storage_info,
	15	+ bpf_cgroup_storage_info[BPF_CGROUP_STORAGE_NEST_MAX]);
	16	+
16	17	#include "../cgroup/cgroup-internal.h"
17	18
18	19	#define LOCAL_STORAGE_CREATE_FLAG_MASK \

kernel/bpf/lpm_trie.c+3−0

@@ -726,6 +726,9 @@ const struct bpf_map_ops trie_map_ops = { · 725 unmodified lines
726	726	.map_lookup_elem = trie_lookup_elem,
727	727	.map_update_elem = trie_update_elem,
728	728	.map_delete_elem = trie_delete_elem,
	729	+ .map_lookup_batch = generic_map_lookup_batch,
	730	+ .map_update_batch = generic_map_update_batch,
	731	+ .map_delete_batch = generic_map_delete_batch,
729	732	.map_check_btf = trie_check_btf,
730	733	.map_btf_name = "lpm_trie",
731	734	.map_btf_id = &trie_map_btf_id,

kernel/bpf/syscall.c+4−1

@@ -1694,7 +1694,9 @@ static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) · 1693 unmodified lines
1694	1694	{
1695	1695	bpf_prog_kallsyms_del_all(prog);
1696	1696	btf_put(prog->aux->btf);
1697		− bpf_prog_free_linfo(prog);
	1697	+ kvfree(prog->aux->jited_linfo);
	1698	+ kvfree(prog->aux->linfo);
	1699	+ kfree(prog->aux->kfunc_tab);
1698	1700	if (prog->aux->attach_btf)
1699	1701	btf_put(prog->aux->attach_btf);
1700	1702
@@ -2946,6 +2948,7 @@ attach_type_to_prog_type(enum bpf_attach_type attach_type) · 1245 unmodified lines
2946	2948	return BPF_PROG_TYPE_SK_MSG;
2947	2949	case BPF_SK_SKB_STREAM_PARSER:
2948	2950	case BPF_SK_SKB_STREAM_VERDICT:
	2951	+ case BPF_SK_SKB_VERDICT:
2949	2952	return BPF_PROG_TYPE_SK_SKB;
2950	2953	case BPF_LIRC_MODE2:
2951	2954	return BPF_PROG_TYPE_LIRC_MODE2;

kernel/bpf/verifier.c+348−42

@@ -234,6 +234,12 @@ static bool bpf_pseudo_call(const struct bpf_insn *insn) · 233 unmodified lines
234	234	insn->src_reg == BPF_PSEUDO_CALL;
235	235	}
236	236
	237	+static bool bpf_pseudo_kfunc_call(const struct bpf_insn *insn)
	238	+{
	239	+ return insn->code == (BPF_JMP \| BPF_CALL) &&
	240	+ insn->src_reg == BPF_PSEUDO_KFUNC_CALL;
	241	+}
	242	+
237	243	static bool bpf_pseudo_func(const struct bpf_insn *insn)
238	244	{
239	245	return insn->code == (BPF_LD \| BPF_IMM \| BPF_DW) &&
@@ -1554,47 +1560,205 @@ static int add_subprog(struct bpf_verifier_env *env, int off) · 1314 unmodified lines
1554	1560	verbose(env, "too many subprograms\n");
1555	1561	return -E2BIG;
1556	1562	}
	1563	+ /* determine subprog starts. The end is one before the next starts */
1557	1564	env->subprog_info[env->subprog_cnt++].start = off;
1558	1565	sort(env->subprog_info, env->subprog_cnt,
1559	1566	sizeof(env->subprog_info[0]), cmp_subprogs, NULL);
1560	1567	return env->subprog_cnt - 1;
1561	1568	}
1562	1569
1563		−static int check_subprogs(struct bpf_verifier_env *env)
	1570	+struct bpf_kfunc_desc {
	1571	+ struct btf_func_model func_model;
	1572	+ u32 func_id;
	1573	+ s32 imm;
	1574	+};
	1575	+
	1576	+#define MAX_KFUNC_DESCS 256
	1577	+struct bpf_kfunc_desc_tab {
	1578	+ struct bpf_kfunc_desc descs[MAX_KFUNC_DESCS];
	1579	+ u32 nr_descs;
	1580	+};
	1581	+
	1582	+static int kfunc_desc_cmp_by_id(const void a, const void b)
	1583	+{
	1584	+ const struct bpf_kfunc_desc *d0 = a;
	1585	+ const struct bpf_kfunc_desc *d1 = b;
	1586	+
	1587	+ /* func_id is not greater than BTF_MAX_TYPE */
	1588	+ return d0->func_id - d1->func_id;
	1589	+}
	1590	+
	1591	+static const struct bpf_kfunc_desc *
	1592	+find_kfunc_desc(const struct bpf_prog *prog, u32 func_id)
	1593	+{
	1594	+ struct bpf_kfunc_desc desc = {
	1595	+ .func_id = func_id,
	1596	+ };
	1597	+ struct bpf_kfunc_desc_tab *tab;
	1598	+
	1599	+ tab = prog->aux->kfunc_tab;
	1600	+ return bsearch(&desc, tab->descs, tab->nr_descs,
	1601	+ sizeof(tab->descs[0]), kfunc_desc_cmp_by_id);
	1602	+}
	1603	+
	1604	+static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id)
	1605	+{
	1606	+ const struct btf_type func, func_proto;
	1607	+ struct bpf_kfunc_desc_tab *tab;
	1608	+ struct bpf_prog_aux *prog_aux;
	1609	+ struct bpf_kfunc_desc *desc;
	1610	+ const char *func_name;
	1611	+ unsigned long addr;
	1612	+ int err;
	1613	+
	1614	+ prog_aux = env->prog->aux;
	1615	+ tab = prog_aux->kfunc_tab;
	1616	+ if (!tab) {
	1617	+ if (!btf_vmlinux) {
	1618	+ verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n");
	1619	+ return -ENOTSUPP;
	1620	+ }
	1621	+
	1622	+ if (!env->prog->jit_requested) {
	1623	+ verbose(env, "JIT is required for calling kernel function\n");
	1624	+ return -ENOTSUPP;
	1625	+ }
	1626	+
	1627	+ if (!bpf_jit_supports_kfunc_call()) {
	1628	+ verbose(env, "JIT does not support calling kernel function\n");
	1629	+ return -ENOTSUPP;
	1630	+ }
	1631	+
	1632	+ if (!env->prog->gpl_compatible) {
	1633	+ verbose(env, "cannot call kernel function from non-GPL compatible program\n");
	1634	+ return -EINVAL;
	1635	+ }
	1636	+
	1637	+ tab = kzalloc(sizeof(*tab), GFP_KERNEL);
	1638	+ if (!tab)
	1639	+ return -ENOMEM;
	1640	+ prog_aux->kfunc_tab = tab;
	1641	+ }
	1642	+
	1643	+ if (find_kfunc_desc(env->prog, func_id))
	1644	+ return 0;
	1645	+
	1646	+ if (tab->nr_descs == MAX_KFUNC_DESCS) {
	1647	+ verbose(env, "too many different kernel function calls\n");
	1648	+ return -E2BIG;
	1649	+ }
	1650	+
	1651	+ func = btf_type_by_id(btf_vmlinux, func_id);
	1652	+ if (!func \|\| !btf_type_is_func(func)) {
	1653	+ verbose(env, "kernel btf_id %u is not a function\n",
	1654	+ func_id);
	1655	+ return -EINVAL;
	1656	+ }
	1657	+ func_proto = btf_type_by_id(btf_vmlinux, func->type);
	1658	+ if (!func_proto \|\| !btf_type_is_func_proto(func_proto)) {
	1659	+ verbose(env, "kernel function btf_id %u does not have a valid func_proto\n",
	1660	+ func_id);
	1661	+ return -EINVAL;
	1662	+ }
	1663	+
	1664	+ func_name = btf_name_by_offset(btf_vmlinux, func->name_off);
	1665	+ addr = kallsyms_lookup_name(func_name);
	1666	+ if (!addr) {
	1667	+ verbose(env, "cannot find address for kernel function %s\n",
	1668	+ func_name);
	1669	+ return -EINVAL;
	1670	+ }
	1671	+
	1672	+ desc = &tab->descs[tab->nr_descs++];
	1673	+ desc->func_id = func_id;
	1674	+ desc->imm = BPF_CAST_CALL(addr) - __bpf_call_base;
	1675	+ err = btf_distill_func_proto(&env->log, btf_vmlinux,
	1676	+ func_proto, func_name,
	1677	+ &desc->func_model);
	1678	+ if (!err)
	1679	+ sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
	1680	+ kfunc_desc_cmp_by_id, NULL);
	1681	+ return err;
	1682	+}
	1683	+
	1684	+static int kfunc_desc_cmp_by_imm(const void a, const void b)
	1685	+{
	1686	+ const struct bpf_kfunc_desc *d0 = a;
	1687	+ const struct bpf_kfunc_desc *d1 = b;
	1688	+
	1689	+ if (d0->imm > d1->imm)
	1690	+ return 1;
	1691	+ else if (d0->imm < d1->imm)
	1692	+ return -1;
	1693	+ return 0;
	1694	+}
	1695	+
	1696	+static void sort_kfunc_descs_by_imm(struct bpf_prog *prog)
	1697	+{
	1698	+ struct bpf_kfunc_desc_tab *tab;
	1699	+
	1700	+ tab = prog->aux->kfunc_tab;
	1701	+ if (!tab)
	1702	+ return;
	1703	+
	1704	+ sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
	1705	+ kfunc_desc_cmp_by_imm, NULL);
	1706	+}
	1707	+
	1708	+bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog)
	1709	+{
	1710	+ return !!prog->aux->kfunc_tab;
	1711	+}
	1712	+
	1713	+const struct btf_func_model *
	1714	+bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
	1715	+ const struct bpf_insn *insn)
	1716	+{
	1717	+ const struct bpf_kfunc_desc desc = {
	1718	+ .imm = insn->imm,
	1719	+ };
	1720	+ const struct bpf_kfunc_desc *res;
	1721	+ struct bpf_kfunc_desc_tab *tab;
	1722	+
	1723	+ tab = prog->aux->kfunc_tab;
	1724	+ res = bsearch(&desc, tab->descs, tab->nr_descs,
	1725	+ sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm);
	1726	+
	1727	+ return res ? &res->func_model : NULL;
	1728	+}
	1729	+
	1730	+static int add_subprog_and_kfunc(struct bpf_verifier_env *env)
1564	1731	{
1565		− int i, ret, subprog_start, subprog_end, off, cur_subprog = 0;
1566	1732	struct bpf_subprog_info *subprog = env->subprog_info;
1567	1733	struct bpf_insn *insn = env->prog->insnsi;
1568		− int insn_cnt = env->prog->len;
	1734	+ int i, ret, insn_cnt = env->prog->len;
1569	1735
1570	1736	/* Add entry function. */
1571	1737	ret = add_subprog(env, 0);
1572		− if (ret < 0)
	1738	+ if (ret)
1573	1739	return ret;
1574	1740
1575		− /* determine subprog starts. The end is one before the next starts */
1576		− for (i = 0; i < insn_cnt; i++) {
1577		− if (bpf_pseudo_func(insn + i)) {
1578		− if (!env->bpf_capable) {
1579		− verbose(env,
1580		− "function pointers are allowed for CAP_BPF and CAP_SYS_ADMIN\n");
1581		− return -EPERM;
1582		− }
1583		− ret = add_subprog(env, i + insn[i].imm + 1);
1584		− if (ret < 0)
1585		− return ret;
1586		− /* remember subprog */
1587		− insn[i + 1].imm = ret;
1588		− continue;
1589		− }
1590		− if (!bpf_pseudo_call(insn + i))
	1741	+ for (i = 0; i < insn_cnt; i++, insn++) {
	1742	+ if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn) &&
	1743	+ !bpf_pseudo_kfunc_call(insn))
1591	1744	continue;
	1745	+
1592	1746	if (!env->bpf_capable) {
1593		− verbose(env,
1594		− "function calls to other bpf functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n");
	1747	+ verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n");
1595	1748	return -EPERM;
1596	1749	}
1597		− ret = add_subprog(env, i + insn[i].imm + 1);
	1750	+
	1751	+ if (bpf_pseudo_func(insn)) {
	1752	+ ret = add_subprog(env, i + insn->imm + 1);
	1753	+ if (ret >= 0)
	1754	+ /* remember subprog */
	1755	+ insn[1].imm = ret;
	1756	+ } else if (bpf_pseudo_call(insn)) {
	1757	+ ret = add_subprog(env, i + insn->imm + 1);
	1758	+ } else {
	1759	+ ret = add_kfunc_call(env, insn->imm);
	1760	+ }
	1761	+
1598	1762	if (ret < 0)
1599	1763	return ret;
1600	1764	}
@@ -1608,6 +1772,16 @@ static int check_subprogs(struct bpf_verifier_env *env) · 7 unmodified lines
1608	1772	for (i = 0; i < env->subprog_cnt; i++)
1609	1773	verbose(env, "func#%d @%d\n", i, subprog[i].start);
1610	1774
	1775	+ return 0;
	1776	+}
	1777	+
	1778	+static int check_subprogs(struct bpf_verifier_env *env)
	1779	+{
	1780	+ int i, subprog_start, subprog_end, off, cur_subprog = 0;
	1781	+ struct bpf_subprog_info *subprog = env->subprog_info;
	1782	+ struct bpf_insn *insn = env->prog->insnsi;
	1783	+ int insn_cnt = env->prog->len;
	1784	+
1611	1785	/* now check that all jumps are within the same subprog */
1612	1786	subprog_start = subprog[cur_subprog].start;
1613	1787	subprog_end = subprog[cur_subprog + 1].start;
@@ -1916,6 +2090,17 @@ static int get_prev_insn_idx(struct bpf_verifier_state *st, int i, · 302 unmodified lines
1916	2090	return i;
1917	2091	}
1918	2092
	2093	+static const char disasm_kfunc_name(void data, const struct bpf_insn *insn)
	2094	+{
	2095	+ const struct btf_type *func;
	2096	+
	2097	+ if (insn->src_reg != BPF_PSEUDO_KFUNC_CALL)
	2098	+ return NULL;
	2099	+
	2100	+ func = btf_type_by_id(btf_vmlinux, insn->imm);
	2101	+ return btf_name_by_offset(btf_vmlinux, func->name_off);
	2102	+}
	2103	+
1919	2104	/* For given verifier state backtrack_insn() is called from the last insn to
1920	2105	* the first insn. Its purpose is to compute a bitmask of registers and
1921	2106	* stack slots that needs precision in the parent verifier state.
@@ -1924,6 +2109,7 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, · 2 unmodified lines
1924	2109	u32 reg_mask, u64 stack_mask)
1925	2110	{
1926	2111	const struct bpf_insn_cbs cbs = {
	2112	+ .cb_call = disasm_kfunc_name,
1927	2113	.cb_print = verbose,
1928	2114	.private_data = env,
1929	2115	};
@@ -5365,7 +5551,7 @@ static int __check_func_call(struct bpf_verifier_env env, struct bpf_insn insn · 3435 unmodified lines
5365	5551	func_info_aux = env->prog->aux->func_info_aux;
5366	5552	if (func_info_aux)
5367	5553	is_global = func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL;
5368		− err = btf_check_func_arg_match(env, subprog, caller->regs);
	5554	+ err = btf_check_subprog_arg_match(env, subprog, caller->regs);
5369	5555	if (err == -EFAULT)
5370	5556	return err;
5371	5557	if (is_global) {
@@ -5960,6 +6146,98 @@ static int check_helper_call(struct bpf_verifier_env env, struct bpf_insn insn · 588 unmodified lines
5960	6146	return 0;
5961	6147	}
5962	6148
	6149	+/* mark_btf_func_reg_size() is used when the reg size is determined by
	6150	+ * the BTF func_proto's return value size and argument.
	6151	+ */
	6152	+static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno,
	6153	+ size_t reg_size)
	6154	+{
	6155	+ struct bpf_reg_state *reg = &cur_regs(env)[regno];
	6156	+
	6157	+ if (regno == BPF_REG_0) {
	6158	+ /* Function return value */
	6159	+ reg->live \|= REG_LIVE_WRITTEN;
	6160	+ reg->subreg_def = reg_size == sizeof(u64) ?
	6161	+ DEF_NOT_SUBREG : env->insn_idx + 1;
	6162	+ } else {
	6163	+ /* Function argument */
	6164	+ if (reg_size == sizeof(u64)) {
	6165	+ mark_insn_zext(env, reg);
	6166	+ mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
	6167	+ } else {
	6168	+ mark_reg_read(env, reg, reg->parent, REG_LIVE_READ32);
	6169	+ }
	6170	+ }
	6171	+}
	6172	+
	6173	+static int check_kfunc_call(struct bpf_verifier_env env, struct bpf_insn insn)
	6174	+{
	6175	+ const struct btf_type t, func, func_proto, ptr_type;
	6176	+ struct bpf_reg_state *regs = cur_regs(env);
	6177	+ const char func_name, ptr_type_name;
	6178	+ u32 i, nargs, func_id, ptr_type_id;
	6179	+ const struct btf_param *args;
	6180	+ int err;
	6181	+
	6182	+ func_id = insn->imm;
	6183	+ func = btf_type_by_id(btf_vmlinux, func_id);
	6184	+ func_name = btf_name_by_offset(btf_vmlinux, func->name_off);
	6185	+ func_proto = btf_type_by_id(btf_vmlinux, func->type);
	6186	+
	6187	+ if (!env->ops->check_kfunc_call \|\|
	6188	+ !env->ops->check_kfunc_call(func_id)) {
	6189	+ verbose(env, "calling kernel function %s is not allowed\n",
	6190	+ func_name);
	6191	+ return -EACCES;
	6192	+ }
	6193	+
	6194	+ /* Check the arguments */
	6195	+ err = btf_check_kfunc_arg_match(env, btf_vmlinux, func_id, regs);
	6196	+ if (err)
	6197	+ return err;
	6198	+
	6199	+ for (i = 0; i < CALLER_SAVED_REGS; i++)
	6200	+ mark_reg_not_init(env, regs, caller_saved[i]);
	6201	+
	6202	+ /* Check return type */
	6203	+ t = btf_type_skip_modifiers(btf_vmlinux, func_proto->type, NULL);
	6204	+ if (btf_type_is_scalar(t)) {
	6205	+ mark_reg_unknown(env, regs, BPF_REG_0);
	6206	+ mark_btf_func_reg_size(env, BPF_REG_0, t->size);
	6207	+ } else if (btf_type_is_ptr(t)) {
	6208	+ ptr_type = btf_type_skip_modifiers(btf_vmlinux, t->type,
	6209	+ &ptr_type_id);
	6210	+ if (!btf_type_is_struct(ptr_type)) {
	6211	+ ptr_type_name = btf_name_by_offset(btf_vmlinux,
	6212	+ ptr_type->name_off);
	6213	+ verbose(env, "kernel function %s returns pointer type %s %s is not supported\n",
	6214	+ func_name, btf_type_str(ptr_type),
	6215	+ ptr_type_name);
	6216	+ return -EINVAL;
	6217	+ }
	6218	+ mark_reg_known_zero(env, regs, BPF_REG_0);
	6219	+ regs[BPF_REG_0].btf = btf_vmlinux;
	6220	+ regs[BPF_REG_0].type = PTR_TO_BTF_ID;
	6221	+ regs[BPF_REG_0].btf_id = ptr_type_id;
	6222	+ mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *));
	6223	+ } /* else { add_kfunc_call() ensures it is btf_type_is_void(t) } */
	6224	+
	6225	+ nargs = btf_type_vlen(func_proto);
	6226	+ args = (const struct btf_param *)(func_proto + 1);
	6227	+ for (i = 0; i < nargs; i++) {
	6228	+ u32 regno = i + 1;
	6229	+
	6230	+ t = btf_type_skip_modifiers(btf_vmlinux, args[i].type, NULL);
	6231	+ if (btf_type_is_ptr(t))
	6232	+ mark_btf_func_reg_size(env, regno, sizeof(void *));
	6233	+ else
	6234	+ /* scalar. ensured by btf_check_kfunc_arg_match() */
	6235	+ mark_btf_func_reg_size(env, regno, t->size);
	6236	+ }
	6237	+
	6238	+ return 0;
	6239	+}
	6240	+
5963	6241	static bool signed_add_overflows(s64 a, s64 b)
5964	6242	{
5965	6243	/* Do the add in u64, where overflow is well-defined */
@@ -6062,19 +6340,6 @@ static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg, · 96 unmodified lines
6062	6340	else
6063	6341	*ptr_limit = -off - 1;
6064	6342	return *ptr_limit >= max ? -ERANGE : 0;
6065		− case PTR_TO_MAP_KEY:
6066		− /* Currently, this code is not exercised as the only use
6067		− * is bpf_for_each_map_elem() helper which requires
6068		− * bpf_capble. The code has been tested manually for
6069		− * future use.
6070		− */
6071		− if (mask_to_left) {
6072		− *ptr_limit = ptr_reg->umax_value + ptr_reg->off;
6073		− } else {
6074		− off = ptr_reg->smin_value + ptr_reg->off;
6075		− *ptr_limit = ptr_reg->map_ptr->key_size - off;
6076		− }
6077		− return 0;
6078	6343	case PTR_TO_MAP_VALUE:
6079	6344	max = ptr_reg->map_ptr->value_size;
6080	6345	if (mask_to_left) {
@@ -6281,7 +6546,6 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, · 200 unmodified lines
6281	6546	verbose(env, "R%d pointer arithmetic on %s prohibited\n",
6282	6547	dst, reg_type_str[ptr_reg->type]);
6283	6548	return -EACCES;
6284		− case PTR_TO_MAP_KEY:
6285	6549	case PTR_TO_MAP_VALUE:
6286	6550	if (!env->allow_ptr_leaks && !known && (smin_val < 0) != (smax_val < 0)) {
6287	6551	verbose(env, "R%d has unknown scalar with mixed signed bounds, pointer arithmetic with it prohibited for !root\n",
@@ -10176,6 +10440,7 @@ static int do_check(struct bpf_verifier_env *env) · 3888 unmodified lines
10176	10440
10177	10441	if (env->log.level & BPF_LOG_LEVEL) {
10178	10442	const struct bpf_insn_cbs cbs = {
	10443	+ .cb_call = disasm_kfunc_name,
10179	10444	.cb_print = verbose,
10180	10445	.private_data = env,
10181	10446	};
@@ -10323,7 +10588,8 @@ static int do_check(struct bpf_verifier_env *env) · 141 unmodified lines
10323	10588	if (BPF_SRC(insn->code) != BPF_K \|\|
10324	10589	insn->off != 0 \|\|
10325	10590	(insn->src_reg != BPF_REG_0 &&
10326		− insn->src_reg != BPF_PSEUDO_CALL) \|\|
	10591	+ insn->src_reg != BPF_PSEUDO_CALL &&
	10592	+ insn->src_reg != BPF_PSEUDO_KFUNC_CALL) \|\|
10327	10593	insn->dst_reg != BPF_REG_0 \|\|
10328	10594	class == BPF_JMP32) {
10329	10595	verbose(env, "BPF_CALL uses reserved fields\n");
@@ -10338,6 +10604,8 @@ static int do_check(struct bpf_verifier_env *env) · 8 unmodified lines
10338	10604	}
10339	10605	if (insn->src_reg == BPF_PSEUDO_CALL)
10340	10606	err = check_func_call(env, insn, &env->insn_idx);
	10607	+ else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL)
	10608	+ err = check_kfunc_call(env, insn);
10341	10609	else
10342	10610	err = check_helper_call(env, insn, &env->insn_idx);
10343	10611	if (err)
@@ -11648,6 +11916,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 1304 unmodified lines
11648	11916	func[i]->aux->name[0] = 'F';
11649	11917	func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;
11650	11918	func[i]->jit_requested = 1;
	11919	+ func[i]->aux->kfunc_tab = prog->aux->kfunc_tab;
11651	11920	func[i]->aux->linfo = prog->aux->linfo;
11652	11921	func[i]->aux->nr_linfo = prog->aux->nr_linfo;
11653	11922	func[i]->aux->jited_linfo = prog->aux->jited_linfo;
@@ -11755,7 +12024,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 101 unmodified lines
11755	12024	prog->bpf_func = func[0]->bpf_func;
11756	12025	prog->aux->func = func;
11757	12026	prog->aux->func_cnt = env->subprog_cnt;
11758		− bpf_prog_free_unused_jited_linfo(prog);
	12027	+ bpf_prog_jit_attempt_done(prog);
11759	12028	return 0;
11760	12029	out_free:
11761	12030	for (i = 0; i < env->subprog_cnt; i++) {
@@ -11778,7 +12047,7 @@ out_undo_insn: · 16 unmodified lines
11778	12047	insn->off = 0;
11779	12048	insn->imm = env->insn_aux_data[i].call_imm;
11780	12049	}
11781		− bpf_prog_free_jited_linfo(prog);
	12050	+ bpf_prog_jit_attempt_done(prog);
11782	12051	return err;
11783	12052	}
11784	12053
@@ -11787,6 +12056,7 @@ static int fixup_call_args(struct bpf_verifier_env *env) · 2 unmodified lines
11787	12056	#ifndef CONFIG_BPF_JIT_ALWAYS_ON
11788	12057	struct bpf_prog *prog = env->prog;
11789	12058	struct bpf_insn *insn = prog->insnsi;
	12059	+ bool has_kfunc_call = bpf_prog_has_kfunc_call(prog);
11790	12060	int i, depth;
11791	12061	#endif
11792	12062	int err = 0;
@@ -11800,6 +12070,10 @@ static int fixup_call_args(struct bpf_verifier_env *env) · 7 unmodified lines
11800	12070	return err;
11801	12071	}
11802	12072	#ifndef CONFIG_BPF_JIT_ALWAYS_ON
	12073	+ if (has_kfunc_call) {
	12074	+ verbose(env, "calling kernel functions are not allowed in non-JITed programs\n");
	12075	+ return -EINVAL;
	12076	+ }
11803	12077	if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) {
11804	12078	/* When JIT fails the progs with bpf2bpf calls and tail_calls
11805	12079	* have to be rejected, since interpreter doesn't support them yet.
@@ -11828,6 +12102,26 @@ static int fixup_call_args(struct bpf_verifier_env *env) · 22 unmodified lines
11828	12102	return err;
11829	12103	}
11830	12104
	12105	+static int fixup_kfunc_call(struct bpf_verifier_env *env,
	12106	+ struct bpf_insn *insn)
	12107	+{
	12108	+ const struct bpf_kfunc_desc *desc;
	12109	+
	12110	+ /* insn->imm has the btf func_id. Replace it with
	12111	+ * an address (relative to __bpf_base_call).
	12112	+ */
	12113	+ desc = find_kfunc_desc(env->prog, insn->imm);
	12114	+ if (!desc) {
	12115	+ verbose(env, "verifier internal error: kernel function descriptor not found for func_id %u\n",
	12116	+ insn->imm);
	12117	+ return -EFAULT;
	12118	+ }
	12119	+
	12120	+ insn->imm = desc->imm;
	12121	+
	12122	+ return 0;
	12123	+}
	12124	+
11831	12125	/* Do various post-verification rewrites in a single program pass.
11832	12126	* These rewrites simplify JIT and interpreter implementations.
11833	12127	*/
@@ -11963,6 +12257,12 @@ static int do_misc_fixups(struct bpf_verifier_env *env) · 129 unmodified lines
11963	12257	continue;
11964	12258	if (insn->src_reg == BPF_PSEUDO_CALL)
11965	12259	continue;
	12260	+ if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
	12261	+ ret = fixup_kfunc_call(env, insn);
	12262	+ if (ret)
	12263	+ return ret;
	12264	+ continue;
	12265	+ }
11966	12266
11967	12267	if (insn->imm == BPF_FUNC_get_route_realm)
11968	12268	prog->dst_needed = 1;
@@ -12192,6 +12492,8 @@ patch_call_imm: · 223 unmodified lines
12192	12492	}
12193	12493	}
12194	12494
	12495	+ sort_kfunc_descs_by_imm(env->prog);
	12496	+
12195	12497	return 0;
12196	12498	}
12197	12499
@@ -12302,7 +12604,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) · 104 unmodified lines
12302	12604	/* 1st arg to a function */
12303	12605	regs[BPF_REG_1].type = PTR_TO_CTX;
12304	12606	mark_reg_known_zero(env, regs, BPF_REG_1);
12305		− ret = btf_check_func_arg_match(env, subprog, regs);
	12607	+ ret = btf_check_subprog_arg_match(env, subprog, regs);
12306	12608	if (ret == -EFAULT)
12307	12609	/* unlikely verifier bug. abort.
12308	12610	* ret == 0 and ret < 0 are sadly acceptable for
@@ -12897,6 +13199,10 @@ int bpf_check(struct bpf_prog *prog, union bpf_attr attr, · 588 unmodified lines
12897	13199	if (!env->explored_states)
12898	13200	goto skip_full_check;
12899	13201
	13202	+ ret = add_subprog_and_kfunc(env);
	13203	+ if (ret < 0)
	13204	+ goto skip_full_check;
	13205	+
12900	13206	ret = check_subprogs(env);
12901	13207	if (ret < 0)
12902	13208	goto skip_full_check;

Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code