bpf: Support bpf program calling kernel function

This patch adds support to BPF verifier to allow bpf program calling kernel function directly. The use case included in this set is to allow bpf-tcp-cc to directly call some tcp-cc helper functions (e.g. "tcp_cong_avoid_ai()"). Those functions have already been used by some kernel tcp-cc implementations. This set will also allow the bpf-tcp-cc program to directly call the kernel tcp-cc implementation, For example, a bpf_dctcp may only want to implement its own dctcp_cwnd_event() and reuse other dctcp_*() directly from the kernel tcp_dctcp.c instead of reimplementing (or copy-and-pasting) them. The tcp-cc kernel functions mentioned above will be white listed for the struct_ops bpf-tcp-cc programs to use in a later patch. The white listed functions are not bounded to a fixed ABI contract. Those functions have already been used by the existing kernel tcp-cc. If any of them has changed, both in-tree and out-of-tree kernel tcp-cc implementations have to be changed. The same goes for the struct_ops bpf-tcp-cc programs which have to be adjusted accordingly. This patch is to make the required changes in the bpf verifier. First change is in btf.c, it adds a case in "btf_check_func_arg_match()". When the passed in "btf->kernel_btf == true", it means matching the verifier regs' states with a kernel function. This will handle the PTR_TO_BTF_ID reg. It also maps PTR_TO_SOCK_COMMON, PTR_TO_SOCKET, and PTR_TO_TCP_SOCK to its kernel's btf_id. In the later libbpf patch, the insn calling a kernel function will look like: insn->code == (BPF_JMP | BPF_CALL) insn->src_reg == BPF_PSEUDO_KFUNC_CALL /* <- new in this patch */ insn->imm == func_btf_id /* btf_id of the running kernel */ [ For the future calling function-in-kernel-module support, an array of module btf_fds can be passed at the load time and insn->off can be used to index into this array. ] At the early stage of verifier, the verifier will collect all kernel function calls into "struct bpf_kfunc_desc". Those descriptors are stored in "prog->aux->kfunc_tab" and will be available to the JIT. Since this "add" operation is similar to the current "add_subprog()" and looking for the same insn->code, they are done together in the new "add_subprog_and_kfunc()". In the "do_check()" stage, the new "check_kfunc_call()" is added to verify the kernel function call instruction: 1. Ensure the kernel function can be used by a particular BPF_PROG_TYPE. A new bpf_verifier_ops "check_kfunc_call" is added to do that. The bpf-tcp-cc struct_ops program will implement this function in a later patch. 2. Call "btf_check_kfunc_args_match()" to ensure the regs can be used as the args of a kernel function. 3. Mark the regs' type, subreg_def, and zext_dst. At the later do_misc_fixups() stage, the new fixup_kfunc_call() will replace the insn->imm with the function address (relative to __bpf_call_base). If needed, the jit can find the btf_func_model by calling the new bpf_jit_find_kfunc_model(prog, insn). With the imm set to the function address, "bpftool prog dump xlated" will be able to display the kernel function calls the same way as it displays other bpf helper calls. gpl_compatible program is required to call kernel function. This feature currently requires JIT. The verifier selftests are adjusted because of the changes in the verbose log in add_subprog_and_kfunc().

Martin KaFai Lau committed 5 years ago

Commit e6ac2451 parent 34747c413 files · +480 −46Browse files

Unified · Split

13 files

arch/x86/net/bpf_jit_comp.c +5 −0
include/linux/bpf.h +24 −0
include/linux/btf.h +1 −0
include/linux/filter.h +1 −0
include/uapi/linux/bpf.h +4 −0
kernel/bpf/btf.c +60 −5
kernel/bpf/core.c +15 −3
kernel/bpf/disasm.c +10 −3
kernel/bpf/syscall.c +1 −0
kernel/bpf/verifier.c +344 −24
tools/include/uapi/linux/bpf.h +4 −0
tools/testing/selftests/bpf/verifier/calls.c +6 −6
tools/testing/selftests/bpf/verifier/dead_code.c +5 −5

arch/x86/net/bpf_jit_comp.c+5−0

@@ -2346,3 +2346,8 @@ out: · 2345 unmodified lines
2346	2346	tmp : orig_prog);
2347	2347	return prog;
2348	2348	}
	2349	+
	2350	+bool bpf_jit_supports_kfunc_call(void)
	2351	+{
	2352	+ return true;
	2353	+}

include/linux/bpf.h+24−0

@@ -427,6 +427,7 @@ enum bpf_reg_type { · 426 unmodified lines
427	427	PTR_TO_PERCPU_BTF_ID, /* reg points to a percpu kernel variable */
428	428	PTR_TO_FUNC, /* reg points to a bpf program function */
429	429	PTR_TO_MAP_KEY, /* reg points to a map element key */
	430	+ __BPF_REG_TYPE_MAX,
430	431	};
431	432
432	433	/* The information passed from prog-specific *_is_valid_access
@@ -480,6 +481,7 @@ struct bpf_verifier_ops { · 47 unmodified lines
480	481	const struct btf_type *t, int off, int size,
481	482	enum bpf_access_type atype,
482	483	u32 *next_btf_id);
	484	+ bool (*check_kfunc_call)(u32 kfunc_btf_id);
483	485	};
484	486
485	487	struct bpf_prog_offload_ops {
@@ -796,6 +798,8 @@ struct btf_mod_pair { · 310 unmodified lines
796	798	struct module *module;
797	799	};
798	800
	801	+struct bpf_kfunc_desc_tab;
	802	+
799	803	struct bpf_prog_aux {
800	804	atomic64_t refcnt;
801	805	u32 used_map_cnt;
@@ -832,6 +836,7 @@ struct bpf_prog_aux { · 30 unmodified lines
832	836	struct bpf_prog **func;
833	837	void jit_data; / JIT specific data. arch dependent */
834	838	struct bpf_jit_poke_descriptor *poke_tab;
	839	+ struct bpf_kfunc_desc_tab *kfunc_tab;
835	840	u32 size_poke_tab;
836	841	struct bpf_ksym ksym;
837	842	const struct bpf_prog_ops *ops;
@@ -1547,6 +1552,9 @@ int btf_distill_func_proto(struct bpf_verifier_log *log, · 709 unmodified lines
1547	1552	struct bpf_reg_state;
1548	1553	int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog,
1549	1554	struct bpf_reg_state *regs);
	1555	+int btf_check_kfunc_arg_match(struct bpf_verifier_env *env,
	1556	+ const struct btf *btf, u32 func_id,
	1557	+ struct bpf_reg_state *regs);
1550	1558	int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog,
1551	1559	struct bpf_reg_state *reg);
1552	1560	int btf_check_type_match(struct bpf_verifier_log log, const struct bpf_prog prog,
@@ -1557,6 +1565,10 @@ struct bpf_link *bpf_link_by_id(u32 id); · 4 unmodified lines
1557	1565
1558	1566	const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id);
1559	1567	void bpf_task_storage_free(struct task_struct *task);
	1568	+bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog);
	1569	+const struct btf_func_model *
	1570	+bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
	1571	+ const struct bpf_insn *insn);
1560	1572	#else /* !CONFIG_BPF_SYSCALL */
1561	1573	static inline struct bpf_prog *bpf_prog_get(u32 ufd)
1562	1574	{
@@ -1737,6 +1749,18 @@ bpf_base_func_proto(enum bpf_func_id func_id) · 174 unmodified lines
1737	1749	static inline void bpf_task_storage_free(struct task_struct *task)
1738	1750	{
1739	1751	}
	1752	+
	1753	+static inline bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog)
	1754	+{
	1755	+ return false;
	1756	+}
	1757	+
	1758	+static inline const struct btf_func_model *
	1759	+bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
	1760	+ const struct bpf_insn *insn)
	1761	+{
	1762	+ return NULL;
	1763	+}
1740	1764	#endif /* CONFIG_BPF_SYSCALL */
1741	1765
1742	1766	void __bpf_free_used_btfs(struct bpf_prog_aux *aux,

include/linux/btf.h+1−0

@@ -110,6 +110,7 @@ const struct btf_type btf_type_resolve_func_ptr(const struct btf btf, · 109 unmodified lines
110	110	const struct btf_type *
111	111	btf_resolve_size(const struct btf btf, const struct btf_type type,
112	112	u32 *type_size);
	113	+const char btf_type_str(const struct btf_type t);
113	114
114	115	#define for_each_member(i, struct_type, member) \
115	116	for (i = 0, member = btf_type_member(struct_type); \

include/linux/filter.h+1−0

@@ -918,6 +918,7 @@ u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); · 917 unmodified lines
918	918	struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog);
919	919	void bpf_jit_compile(struct bpf_prog *prog);
920	920	bool bpf_jit_needs_zext(void);
	921	+bool bpf_jit_supports_kfunc_call(void);
921	922	bool bpf_helper_changes_pkt_data(void *func);
922	923
923	924	static inline bool bpf_dump_raw_ok(const struct cred *cred)

include/uapi/linux/bpf.h+4−0

@@ -1117,6 +1117,10 @@ enum bpf_link_type { · 1116 unmodified lines
1117	1117	* offset to another bpf function
1118	1118	*/
1119	1119	#define BPF_PSEUDO_CALL 1
	1120	+/* when bpf_call->src_reg == BPF_PSEUDO_KFUNC_CALL,
	1121	+ * bpf_call->imm == btf_id of a BTF_KIND_FUNC in the running kernel
	1122	+ */
	1123	+#define BPF_PSEUDO_KFUNC_CALL 2
1120	1124
1121	1125	/* flags for BPF_MAP_UPDATE_ELEM command */
1122	1126	enum {

kernel/bpf/btf.c+60−5

@@ -283,7 +283,7 @@ static const char * const btf_kind_str[NR_BTF_KINDS] = { · 282 unmodified lines
283	283	[BTF_KIND_FLOAT] = "FLOAT",
284	284	};
285	285
286		−static const char btf_type_str(const struct btf_type t)
	286	+const char btf_type_str(const struct btf_type t)
287	287	{
288	288	return btf_kind_str[BTF_INFO_KIND(t->info)];
289	289	}
@@ -5362,6 +5362,14 @@ int btf_check_type_match(struct bpf_verifier_log log, const struct bpf_prog pr · 5072 unmodified lines
5362	5362	return btf_check_func_type_match(log, btf1, t1, btf2, t2);
5363	5363	}
5364	5364
	5365	+static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = {
	5366	+#ifdef CONFIG_NET
	5367	+ [PTR_TO_SOCKET] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK],
	5368	+ [PTR_TO_SOCK_COMMON] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
	5369	+ [PTR_TO_TCP_SOCK] = &btf_sock_ids[BTF_SOCK_TYPE_TCP],
	5370	+#endif
	5371	+};
	5372	+
5365	5373	static int btf_check_func_arg_match(struct bpf_verifier_env *env,
5366	5374	const struct btf *btf, u32 func_id,
5367	5375	struct bpf_reg_state *regs,
@@ -5371,12 +5379,12 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, · 3 unmodified lines
5371	5379	const char func_name, ref_tname;
5372	5380	const struct btf_type t, ref_t;
5373	5381	const struct btf_param *args;
5374		− u32 i, nargs;
	5382	+ u32 i, nargs, ref_id;
5375	5383
5376	5384	t = btf_type_by_id(btf, func_id);
5377	5385	if (!t \|\| !btf_type_is_func(t)) {
5378	5386	/* These checks were already done by the verifier while loading
5379		− * struct bpf_func_info
	5387	+ * struct bpf_func_info or in add_kfunc_call().
5380	5388	*/
5381	5389	bpf_log(log, "BTF of func_id %u doesn't point to KIND_FUNC\n",
5382	5390	func_id);
@@ -5418,9 +5426,49 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, · 35 unmodified lines
5418	5426	return -EINVAL;
5419	5427	}
5420	5428
5421		− ref_t = btf_type_skip_modifiers(btf, t->type, NULL);
	5429	+ ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id);
5422	5430	ref_tname = btf_name_by_offset(btf, ref_t->name_off);
5423		− if (btf_get_prog_ctx_type(log, btf, t, env->prog->type, i)) {
	5431	+ if (btf_is_kernel(btf)) {
	5432	+ const struct btf_type *reg_ref_t;
	5433	+ const struct btf *reg_btf;
	5434	+ const char *reg_ref_tname;
	5435	+ u32 reg_ref_id;
	5436	+
	5437	+ if (!btf_type_is_struct(ref_t)) {
	5438	+ bpf_log(log, "kernel function %s args#%d pointer type %s %s is not supported\n",
	5439	+ func_name, i, btf_type_str(ref_t),
	5440	+ ref_tname);
	5441	+ return -EINVAL;
	5442	+ }
	5443	+
	5444	+ if (reg->type == PTR_TO_BTF_ID) {
	5445	+ reg_btf = reg->btf;
	5446	+ reg_ref_id = reg->btf_id;
	5447	+ } else if (reg2btf_ids[reg->type]) {
	5448	+ reg_btf = btf_vmlinux;
	5449	+ reg_ref_id = *reg2btf_ids[reg->type];
	5450	+ } else {
	5451	+ bpf_log(log, "kernel function %s args#%d expected pointer to %s %s but R%d is not a pointer to btf_id\n",
	5452	+ func_name, i,
	5453	+ btf_type_str(ref_t), ref_tname, regno);
	5454	+ return -EINVAL;
	5455	+ }
	5456	+
	5457	+ reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id,
	5458	+ &reg_ref_id);
	5459	+ reg_ref_tname = btf_name_by_offset(reg_btf,
	5460	+ reg_ref_t->name_off);
	5461	+ if (!btf_struct_ids_match(log, reg_btf, reg_ref_id,
	5462	+ reg->off, btf, ref_id)) {
	5463	+ bpf_log(log, "kernel function %s args#%d expected pointer to %s %s but R%d has a pointer to %s %s\n",
	5464	+ func_name, i,
	5465	+ btf_type_str(ref_t), ref_tname,
	5466	+ regno, btf_type_str(reg_ref_t),
	5467	+ reg_ref_tname);
	5468	+ return -EINVAL;
	5469	+ }
	5470	+ } else if (btf_get_prog_ctx_type(log, btf, t,
	5471	+ env->prog->type, i)) {
5424	5472	/* If function expects ctx type in BTF check that caller
5425	5473	* is passing PTR_TO_CTX.
5426	5474	*/
@@ -5493,6 +5541,13 @@ int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog, · 66 unmodified lines
5493	5541	return err;
5494	5542	}
5495	5543
	5544	+int btf_check_kfunc_arg_match(struct bpf_verifier_env *env,
	5545	+ const struct btf *btf, u32 func_id,
	5546	+ struct bpf_reg_state *regs)
	5547	+{
	5548	+ return btf_check_func_arg_match(env, btf, func_id, regs, false);
	5549	+}
	5550	+
5496	5551	/* Convert BTF of a function into bpf_reg_state if possible
5497	5552	* Returns:
5498	5553	* EFAULT - there is a verifier bug. Abort verification.

kernel/bpf/core.c+15−3

@@ -159,6 +159,9 @@ void bpf_prog_jit_attempt_done(struct bpf_prog *prog) · 158 unmodified lines
159	159	kvfree(prog->aux->jited_linfo);
160	160	prog->aux->jited_linfo = NULL;
161	161	}
	162	+
	163	+ kfree(prog->aux->kfunc_tab);
	164	+ prog->aux->kfunc_tab = NULL;
162	165	}
163	166
164	167	/* The jit engine is responsible to provide an array
@@ -1840,9 +1843,15 @@ struct bpf_prog bpf_prog_select_runtime(struct bpf_prog fp, int *err) · 1675 unmodified lines
1840	1843	/* In case of BPF to BPF calls, verifier did all the prep
1841	1844	* work with regards to JITing, etc.
1842	1845	*/
	1846	+ bool jit_needed = false;
	1847	+
1843	1848	if (fp->bpf_func)
1844	1849	goto finalize;
1845	1850
	1851	+ if (IS_ENABLED(CONFIG_BPF_JIT_ALWAYS_ON) \|\|
	1852	+ bpf_prog_has_kfunc_call(fp))
	1853	+ jit_needed = true;
	1854	+
1846	1855	bpf_prog_select_func(fp);
1847	1856
1848	1857	/* eBPF JITs can rewrite the program in case constant
@@ -1858,12 +1867,10 @@ struct bpf_prog bpf_prog_select_runtime(struct bpf_prog fp, int *err) · 9 unmodified lines
1858	1867
1859	1868	fp = bpf_int_jit_compile(fp);
1860	1869	bpf_prog_jit_attempt_done(fp);
1861		−#ifdef CONFIG_BPF_JIT_ALWAYS_ON
1862		− if (!fp->jited) {
	1870	+ if (!fp->jited && jit_needed) {
1863	1871	*err = -ENOTSUPP;
1864	1872	return fp;
1865	1873	}
1866		−#endif
1867	1874	} else {
1868	1875	*err = bpf_prog_offload_compile(fp);
1869	1876	if (*err)
@@ -2343,6 +2350,11 @@ bool __weak bpf_jit_needs_zext(void) · 473 unmodified lines
2343	2350	return false;
2344	2351	}
2345	2352
	2353	+bool __weak bpf_jit_supports_kfunc_call(void)
	2354	+{
	2355	+ return false;
	2356	+}
	2357	+
2346	2358	/* To execute LD_ABS/LD_IND instructions __bpf_prog_run() may call
2347	2359	* skb_copy_bits(), so provide a weak definition of it for NET-less config.
2348	2360	*/

kernel/bpf/disasm.c+10−3

@@ -19,16 +19,23 @@ static const char __func_get_name(const struct bpf_insn_cbs cbs, · 18 unmodified lines
19	19	{
20	20	BUILD_BUG_ON(ARRAY_SIZE(func_id_str) != __BPF_FUNC_MAX_ID);
21	21
22		− if (insn->src_reg != BPF_PSEUDO_CALL &&
	22	+ if (!insn->src_reg &&
23	23	insn->imm >= 0 && insn->imm < __BPF_FUNC_MAX_ID &&
24	24	func_id_str[insn->imm])
25	25	return func_id_str[insn->imm];
26	26
27		− if (cbs && cbs->cb_call)
28		− return cbs->cb_call(cbs->private_data, insn);
	27	+ if (cbs && cbs->cb_call) {
	28	+ const char *res;
	29	+
	30	+ res = cbs->cb_call(cbs->private_data, insn);
	31	+ if (res)
	32	+ return res;
	33	+ }
29	34
30	35	if (insn->src_reg == BPF_PSEUDO_CALL)
31	36	snprintf(buff, len, "%+d", insn->imm);
	37	+ else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL)
	38	+ snprintf(buff, len, "kernel-function");
32	39
33	40	return buff;
34	41	}

kernel/bpf/syscall.c+1−0

@@ -1696,6 +1696,7 @@ static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) · 1695 unmodified lines
1696	1696	btf_put(prog->aux->btf);
1697	1697	kvfree(prog->aux->jited_linfo);
1698	1698	kvfree(prog->aux->linfo);
	1699	+ kfree(prog->aux->kfunc_tab);
1699	1700	if (prog->aux->attach_btf)
1700	1701	btf_put(prog->aux->attach_btf);
1701	1702

kernel/bpf/verifier.c+344−24

@@ -234,6 +234,12 @@ static bool bpf_pseudo_call(const struct bpf_insn *insn) · 233 unmodified lines
234	234	insn->src_reg == BPF_PSEUDO_CALL;
235	235	}
236	236
	237	+static bool bpf_pseudo_kfunc_call(const struct bpf_insn *insn)
	238	+{
	239	+ return insn->code == (BPF_JMP \| BPF_CALL) &&
	240	+ insn->src_reg == BPF_PSEUDO_KFUNC_CALL;
	241	+}
	242	+
237	243	static bool bpf_pseudo_func(const struct bpf_insn *insn)
238	244	{
239	245	return insn->code == (BPF_LD \| BPF_IMM \| BPF_DW) &&
@@ -1554,47 +1560,205 @@ static int add_subprog(struct bpf_verifier_env *env, int off) · 1314 unmodified lines
1554	1560	verbose(env, "too many subprograms\n");
1555	1561	return -E2BIG;
1556	1562	}
	1563	+ /* determine subprog starts. The end is one before the next starts */
1557	1564	env->subprog_info[env->subprog_cnt++].start = off;
1558	1565	sort(env->subprog_info, env->subprog_cnt,
1559	1566	sizeof(env->subprog_info[0]), cmp_subprogs, NULL);
1560	1567	return env->subprog_cnt - 1;
1561	1568	}
1562	1569
1563		−static int check_subprogs(struct bpf_verifier_env *env)
	1570	+struct bpf_kfunc_desc {
	1571	+ struct btf_func_model func_model;
	1572	+ u32 func_id;
	1573	+ s32 imm;
	1574	+};
	1575	+
	1576	+#define MAX_KFUNC_DESCS 256
	1577	+struct bpf_kfunc_desc_tab {
	1578	+ struct bpf_kfunc_desc descs[MAX_KFUNC_DESCS];
	1579	+ u32 nr_descs;
	1580	+};
	1581	+
	1582	+static int kfunc_desc_cmp_by_id(const void a, const void b)
	1583	+{
	1584	+ const struct bpf_kfunc_desc *d0 = a;
	1585	+ const struct bpf_kfunc_desc *d1 = b;
	1586	+
	1587	+ /* func_id is not greater than BTF_MAX_TYPE */
	1588	+ return d0->func_id - d1->func_id;
	1589	+}
	1590	+
	1591	+static const struct bpf_kfunc_desc *
	1592	+find_kfunc_desc(const struct bpf_prog *prog, u32 func_id)
	1593	+{
	1594	+ struct bpf_kfunc_desc desc = {
	1595	+ .func_id = func_id,
	1596	+ };
	1597	+ struct bpf_kfunc_desc_tab *tab;
	1598	+
	1599	+ tab = prog->aux->kfunc_tab;
	1600	+ return bsearch(&desc, tab->descs, tab->nr_descs,
	1601	+ sizeof(tab->descs[0]), kfunc_desc_cmp_by_id);
	1602	+}
	1603	+
	1604	+static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id)
	1605	+{
	1606	+ const struct btf_type func, func_proto;
	1607	+ struct bpf_kfunc_desc_tab *tab;
	1608	+ struct bpf_prog_aux *prog_aux;
	1609	+ struct bpf_kfunc_desc *desc;
	1610	+ const char *func_name;
	1611	+ unsigned long addr;
	1612	+ int err;
	1613	+
	1614	+ prog_aux = env->prog->aux;
	1615	+ tab = prog_aux->kfunc_tab;
	1616	+ if (!tab) {
	1617	+ if (!btf_vmlinux) {
	1618	+ verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n");
	1619	+ return -ENOTSUPP;
	1620	+ }
	1621	+
	1622	+ if (!env->prog->jit_requested) {
	1623	+ verbose(env, "JIT is required for calling kernel function\n");
	1624	+ return -ENOTSUPP;
	1625	+ }
	1626	+
	1627	+ if (!bpf_jit_supports_kfunc_call()) {
	1628	+ verbose(env, "JIT does not support calling kernel function\n");
	1629	+ return -ENOTSUPP;
	1630	+ }
	1631	+
	1632	+ if (!env->prog->gpl_compatible) {
	1633	+ verbose(env, "cannot call kernel function from non-GPL compatible program\n");
	1634	+ return -EINVAL;
	1635	+ }
	1636	+
	1637	+ tab = kzalloc(sizeof(*tab), GFP_KERNEL);
	1638	+ if (!tab)
	1639	+ return -ENOMEM;
	1640	+ prog_aux->kfunc_tab = tab;
	1641	+ }
	1642	+
	1643	+ if (find_kfunc_desc(env->prog, func_id))
	1644	+ return 0;
	1645	+
	1646	+ if (tab->nr_descs == MAX_KFUNC_DESCS) {
	1647	+ verbose(env, "too many different kernel function calls\n");
	1648	+ return -E2BIG;
	1649	+ }
	1650	+
	1651	+ func = btf_type_by_id(btf_vmlinux, func_id);
	1652	+ if (!func \|\| !btf_type_is_func(func)) {
	1653	+ verbose(env, "kernel btf_id %u is not a function\n",
	1654	+ func_id);
	1655	+ return -EINVAL;
	1656	+ }
	1657	+ func_proto = btf_type_by_id(btf_vmlinux, func->type);
	1658	+ if (!func_proto \|\| !btf_type_is_func_proto(func_proto)) {
	1659	+ verbose(env, "kernel function btf_id %u does not have a valid func_proto\n",
	1660	+ func_id);
	1661	+ return -EINVAL;
	1662	+ }
	1663	+
	1664	+ func_name = btf_name_by_offset(btf_vmlinux, func->name_off);
	1665	+ addr = kallsyms_lookup_name(func_name);
	1666	+ if (!addr) {
	1667	+ verbose(env, "cannot find address for kernel function %s\n",
	1668	+ func_name);
	1669	+ return -EINVAL;
	1670	+ }
	1671	+
	1672	+ desc = &tab->descs[tab->nr_descs++];
	1673	+ desc->func_id = func_id;
	1674	+ desc->imm = BPF_CAST_CALL(addr) - __bpf_call_base;
	1675	+ err = btf_distill_func_proto(&env->log, btf_vmlinux,
	1676	+ func_proto, func_name,
	1677	+ &desc->func_model);
	1678	+ if (!err)
	1679	+ sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
	1680	+ kfunc_desc_cmp_by_id, NULL);
	1681	+ return err;
	1682	+}
	1683	+
	1684	+static int kfunc_desc_cmp_by_imm(const void a, const void b)
	1685	+{
	1686	+ const struct bpf_kfunc_desc *d0 = a;
	1687	+ const struct bpf_kfunc_desc *d1 = b;
	1688	+
	1689	+ if (d0->imm > d1->imm)
	1690	+ return 1;
	1691	+ else if (d0->imm < d1->imm)
	1692	+ return -1;
	1693	+ return 0;
	1694	+}
	1695	+
	1696	+static void sort_kfunc_descs_by_imm(struct bpf_prog *prog)
	1697	+{
	1698	+ struct bpf_kfunc_desc_tab *tab;
	1699	+
	1700	+ tab = prog->aux->kfunc_tab;
	1701	+ if (!tab)
	1702	+ return;
	1703	+
	1704	+ sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
	1705	+ kfunc_desc_cmp_by_imm, NULL);
	1706	+}
	1707	+
	1708	+bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog)
	1709	+{
	1710	+ return !!prog->aux->kfunc_tab;
	1711	+}
	1712	+
	1713	+const struct btf_func_model *
	1714	+bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
	1715	+ const struct bpf_insn *insn)
	1716	+{
	1717	+ const struct bpf_kfunc_desc desc = {
	1718	+ .imm = insn->imm,
	1719	+ };
	1720	+ const struct bpf_kfunc_desc *res;
	1721	+ struct bpf_kfunc_desc_tab *tab;
	1722	+
	1723	+ tab = prog->aux->kfunc_tab;
	1724	+ res = bsearch(&desc, tab->descs, tab->nr_descs,
	1725	+ sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm);
	1726	+
	1727	+ return res ? &res->func_model : NULL;
	1728	+}
	1729	+
	1730	+static int add_subprog_and_kfunc(struct bpf_verifier_env *env)
1564	1731	{
1565		− int i, ret, subprog_start, subprog_end, off, cur_subprog = 0;
1566	1732	struct bpf_subprog_info *subprog = env->subprog_info;
1567	1733	struct bpf_insn *insn = env->prog->insnsi;
1568		− int insn_cnt = env->prog->len;
	1734	+ int i, ret, insn_cnt = env->prog->len;
1569	1735
1570	1736	/* Add entry function. */
1571	1737	ret = add_subprog(env, 0);
1572		− if (ret < 0)
	1738	+ if (ret)
1573	1739	return ret;
1574	1740
1575		− /* determine subprog starts. The end is one before the next starts */
1576		− for (i = 0; i < insn_cnt; i++) {
1577		− if (bpf_pseudo_func(insn + i)) {
1578		− if (!env->bpf_capable) {
1579		− verbose(env,
1580		− "function pointers are allowed for CAP_BPF and CAP_SYS_ADMIN\n");
1581		− return -EPERM;
1582		− }
1583		− ret = add_subprog(env, i + insn[i].imm + 1);
1584		− if (ret < 0)
1585		− return ret;
1586		− /* remember subprog */
1587		− insn[i + 1].imm = ret;
1588		− continue;
1589		− }
1590		− if (!bpf_pseudo_call(insn + i))
	1741	+ for (i = 0; i < insn_cnt; i++, insn++) {
	1742	+ if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn) &&
	1743	+ !bpf_pseudo_kfunc_call(insn))
1591	1744	continue;
	1745	+
1592	1746	if (!env->bpf_capable) {
1593		− verbose(env,
1594		− "function calls to other bpf functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n");
	1747	+ verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n");
1595	1748	return -EPERM;
1596	1749	}
1597		− ret = add_subprog(env, i + insn[i].imm + 1);
	1750	+
	1751	+ if (bpf_pseudo_func(insn)) {
	1752	+ ret = add_subprog(env, i + insn->imm + 1);
	1753	+ if (ret >= 0)
	1754	+ /* remember subprog */
	1755	+ insn[1].imm = ret;
	1756	+ } else if (bpf_pseudo_call(insn)) {
	1757	+ ret = add_subprog(env, i + insn->imm + 1);
	1758	+ } else {
	1759	+ ret = add_kfunc_call(env, insn->imm);
	1760	+ }
	1761	+
1598	1762	if (ret < 0)
1599	1763	return ret;
1600	1764	}
@@ -1608,6 +1772,16 @@ static int check_subprogs(struct bpf_verifier_env *env) · 7 unmodified lines
1608	1772	for (i = 0; i < env->subprog_cnt; i++)
1609	1773	verbose(env, "func#%d @%d\n", i, subprog[i].start);
1610	1774
	1775	+ return 0;
	1776	+}
	1777	+
	1778	+static int check_subprogs(struct bpf_verifier_env *env)
	1779	+{
	1780	+ int i, subprog_start, subprog_end, off, cur_subprog = 0;
	1781	+ struct bpf_subprog_info *subprog = env->subprog_info;
	1782	+ struct bpf_insn *insn = env->prog->insnsi;
	1783	+ int insn_cnt = env->prog->len;
	1784	+
1611	1785	/* now check that all jumps are within the same subprog */
1612	1786	subprog_start = subprog[cur_subprog].start;
1613	1787	subprog_end = subprog[cur_subprog + 1].start;
@@ -1916,6 +2090,17 @@ static int get_prev_insn_idx(struct bpf_verifier_state *st, int i, · 302 unmodified lines
1916	2090	return i;
1917	2091	}
1918	2092
	2093	+static const char disasm_kfunc_name(void data, const struct bpf_insn *insn)
	2094	+{
	2095	+ const struct btf_type *func;
	2096	+
	2097	+ if (insn->src_reg != BPF_PSEUDO_KFUNC_CALL)
	2098	+ return NULL;
	2099	+
	2100	+ func = btf_type_by_id(btf_vmlinux, insn->imm);
	2101	+ return btf_name_by_offset(btf_vmlinux, func->name_off);
	2102	+}
	2103	+
1919	2104	/* For given verifier state backtrack_insn() is called from the last insn to
1920	2105	* the first insn. Its purpose is to compute a bitmask of registers and
1921	2106	* stack slots that needs precision in the parent verifier state.
@@ -1924,6 +2109,7 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, · 2 unmodified lines
1924	2109	u32 reg_mask, u64 stack_mask)
1925	2110	{
1926	2111	const struct bpf_insn_cbs cbs = {
	2112	+ .cb_call = disasm_kfunc_name,
1927	2113	.cb_print = verbose,
1928	2114	.private_data = env,
1929	2115	};
@@ -5960,6 +6146,98 @@ static int check_helper_call(struct bpf_verifier_env env, struct bpf_insn insn · 4030 unmodified lines
5960	6146	return 0;
5961	6147	}
5962	6148
	6149	+/* mark_btf_func_reg_size() is used when the reg size is determined by
	6150	+ * the BTF func_proto's return value size and argument.
	6151	+ */
	6152	+static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno,
	6153	+ size_t reg_size)
	6154	+{
	6155	+ struct bpf_reg_state *reg = &cur_regs(env)[regno];
	6156	+
	6157	+ if (regno == BPF_REG_0) {
	6158	+ /* Function return value */
	6159	+ reg->live \|= REG_LIVE_WRITTEN;
	6160	+ reg->subreg_def = reg_size == sizeof(u64) ?
	6161	+ DEF_NOT_SUBREG : env->insn_idx + 1;
	6162	+ } else {
	6163	+ /* Function argument */
	6164	+ if (reg_size == sizeof(u64)) {
	6165	+ mark_insn_zext(env, reg);
	6166	+ mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
	6167	+ } else {
	6168	+ mark_reg_read(env, reg, reg->parent, REG_LIVE_READ32);
	6169	+ }
	6170	+ }
	6171	+}
	6172	+
	6173	+static int check_kfunc_call(struct bpf_verifier_env env, struct bpf_insn insn)
	6174	+{
	6175	+ const struct btf_type t, func, func_proto, ptr_type;
	6176	+ struct bpf_reg_state *regs = cur_regs(env);
	6177	+ const char func_name, ptr_type_name;
	6178	+ u32 i, nargs, func_id, ptr_type_id;
	6179	+ const struct btf_param *args;
	6180	+ int err;
	6181	+
	6182	+ func_id = insn->imm;
	6183	+ func = btf_type_by_id(btf_vmlinux, func_id);
	6184	+ func_name = btf_name_by_offset(btf_vmlinux, func->name_off);
	6185	+ func_proto = btf_type_by_id(btf_vmlinux, func->type);
	6186	+
	6187	+ if (!env->ops->check_kfunc_call \|\|
	6188	+ !env->ops->check_kfunc_call(func_id)) {
	6189	+ verbose(env, "calling kernel function %s is not allowed\n",
	6190	+ func_name);
	6191	+ return -EACCES;
	6192	+ }
	6193	+
	6194	+ /* Check the arguments */
	6195	+ err = btf_check_kfunc_arg_match(env, btf_vmlinux, func_id, regs);
	6196	+ if (err)
	6197	+ return err;
	6198	+
	6199	+ for (i = 0; i < CALLER_SAVED_REGS; i++)
	6200	+ mark_reg_not_init(env, regs, caller_saved[i]);
	6201	+
	6202	+ /* Check return type */
	6203	+ t = btf_type_skip_modifiers(btf_vmlinux, func_proto->type, NULL);
	6204	+ if (btf_type_is_scalar(t)) {
	6205	+ mark_reg_unknown(env, regs, BPF_REG_0);
	6206	+ mark_btf_func_reg_size(env, BPF_REG_0, t->size);
	6207	+ } else if (btf_type_is_ptr(t)) {
	6208	+ ptr_type = btf_type_skip_modifiers(btf_vmlinux, t->type,
	6209	+ &ptr_type_id);
	6210	+ if (!btf_type_is_struct(ptr_type)) {
	6211	+ ptr_type_name = btf_name_by_offset(btf_vmlinux,
	6212	+ ptr_type->name_off);
	6213	+ verbose(env, "kernel function %s returns pointer type %s %s is not supported\n",
	6214	+ func_name, btf_type_str(ptr_type),
	6215	+ ptr_type_name);
	6216	+ return -EINVAL;
	6217	+ }
	6218	+ mark_reg_known_zero(env, regs, BPF_REG_0);
	6219	+ regs[BPF_REG_0].btf = btf_vmlinux;
	6220	+ regs[BPF_REG_0].type = PTR_TO_BTF_ID;
	6221	+ regs[BPF_REG_0].btf_id = ptr_type_id;
	6222	+ mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *));
	6223	+ } /* else { add_kfunc_call() ensures it is btf_type_is_void(t) } */
	6224	+
	6225	+ nargs = btf_type_vlen(func_proto);
	6226	+ args = (const struct btf_param *)(func_proto + 1);
	6227	+ for (i = 0; i < nargs; i++) {
	6228	+ u32 regno = i + 1;
	6229	+
	6230	+ t = btf_type_skip_modifiers(btf_vmlinux, args[i].type, NULL);
	6231	+ if (btf_type_is_ptr(t))
	6232	+ mark_btf_func_reg_size(env, regno, sizeof(void *));
	6233	+ else
	6234	+ /* scalar. ensured by btf_check_kfunc_arg_match() */
	6235	+ mark_btf_func_reg_size(env, regno, t->size);
	6236	+ }
	6237	+
	6238	+ return 0;
	6239	+}
	6240	+
5963	6241	static bool signed_add_overflows(s64 a, s64 b)
5964	6242	{
5965	6243	/* Do the add in u64, where overflow is well-defined */
@@ -10162,6 +10440,7 @@ static int do_check(struct bpf_verifier_env *env) · 4196 unmodified lines
10162	10440
10163	10441	if (env->log.level & BPF_LOG_LEVEL) {
10164	10442	const struct bpf_insn_cbs cbs = {
	10443	+ .cb_call = disasm_kfunc_name,
10165	10444	.cb_print = verbose,
10166	10445	.private_data = env,
10167	10446	};
@@ -10309,7 +10588,8 @@ static int do_check(struct bpf_verifier_env *env) · 141 unmodified lines
10309	10588	if (BPF_SRC(insn->code) != BPF_K \|\|
10310	10589	insn->off != 0 \|\|
10311	10590	(insn->src_reg != BPF_REG_0 &&
10312		− insn->src_reg != BPF_PSEUDO_CALL) \|\|
	10591	+ insn->src_reg != BPF_PSEUDO_CALL &&
	10592	+ insn->src_reg != BPF_PSEUDO_KFUNC_CALL) \|\|
10313	10593	insn->dst_reg != BPF_REG_0 \|\|
10314	10594	class == BPF_JMP32) {
10315	10595	verbose(env, "BPF_CALL uses reserved fields\n");
@@ -10324,6 +10604,8 @@ static int do_check(struct bpf_verifier_env *env) · 8 unmodified lines
10324	10604	}
10325	10605	if (insn->src_reg == BPF_PSEUDO_CALL)
10326	10606	err = check_func_call(env, insn, &env->insn_idx);
	10607	+ else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL)
	10608	+ err = check_kfunc_call(env, insn);
10327	10609	else
10328	10610	err = check_helper_call(env, insn, &env->insn_idx);
10329	10611	if (err)
@@ -11634,6 +11916,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 1304 unmodified lines
11634	11916	func[i]->aux->name[0] = 'F';
11635	11917	func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;
11636	11918	func[i]->jit_requested = 1;
	11919	+ func[i]->aux->kfunc_tab = prog->aux->kfunc_tab;
11637	11920	func[i]->aux->linfo = prog->aux->linfo;
11638	11921	func[i]->aux->nr_linfo = prog->aux->nr_linfo;
11639	11922	func[i]->aux->jited_linfo = prog->aux->jited_linfo;
@@ -11773,6 +12056,7 @@ static int fixup_call_args(struct bpf_verifier_env *env) · 133 unmodified lines
11773	12056	#ifndef CONFIG_BPF_JIT_ALWAYS_ON
11774	12057	struct bpf_prog *prog = env->prog;
11775	12058	struct bpf_insn *insn = prog->insnsi;
	12059	+ bool has_kfunc_call = bpf_prog_has_kfunc_call(prog);
11776	12060	int i, depth;
11777	12061	#endif
11778	12062	int err = 0;
@@ -11786,6 +12070,10 @@ static int fixup_call_args(struct bpf_verifier_env *env) · 7 unmodified lines
11786	12070	return err;
11787	12071	}
11788	12072	#ifndef CONFIG_BPF_JIT_ALWAYS_ON
	12073	+ if (has_kfunc_call) {
	12074	+ verbose(env, "calling kernel functions are not allowed in non-JITed programs\n");
	12075	+ return -EINVAL;
	12076	+ }
11789	12077	if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) {
11790	12078	/* When JIT fails the progs with bpf2bpf calls and tail_calls
11791	12079	* have to be rejected, since interpreter doesn't support them yet.
@@ -11814,6 +12102,26 @@ static int fixup_call_args(struct bpf_verifier_env *env) · 22 unmodified lines
11814	12102	return err;
11815	12103	}
11816	12104
	12105	+static int fixup_kfunc_call(struct bpf_verifier_env *env,
	12106	+ struct bpf_insn *insn)
	12107	+{
	12108	+ const struct bpf_kfunc_desc *desc;
	12109	+
	12110	+ /* insn->imm has the btf func_id. Replace it with
	12111	+ * an address (relative to __bpf_base_call).
	12112	+ */
	12113	+ desc = find_kfunc_desc(env->prog, insn->imm);
	12114	+ if (!desc) {
	12115	+ verbose(env, "verifier internal error: kernel function descriptor not found for func_id %u\n",
	12116	+ insn->imm);
	12117	+ return -EFAULT;
	12118	+ }
	12119	+
	12120	+ insn->imm = desc->imm;
	12121	+
	12122	+ return 0;
	12123	+}
	12124	+
11817	12125	/* Do various post-verification rewrites in a single program pass.
11818	12126	* These rewrites simplify JIT and interpreter implementations.
11819	12127	*/
@@ -11949,6 +12257,12 @@ static int do_misc_fixups(struct bpf_verifier_env *env) · 129 unmodified lines
11949	12257	continue;
11950	12258	if (insn->src_reg == BPF_PSEUDO_CALL)
11951	12259	continue;
	12260	+ if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
	12261	+ ret = fixup_kfunc_call(env, insn);
	12262	+ if (ret)
	12263	+ return ret;
	12264	+ continue;
	12265	+ }
11952	12266
11953	12267	if (insn->imm == BPF_FUNC_get_route_realm)
11954	12268	prog->dst_needed = 1;
@@ -12178,6 +12492,8 @@ patch_call_imm: · 223 unmodified lines
12178	12492	}
12179	12493	}
12180	12494
	12495	+ sort_kfunc_descs_by_imm(env->prog);
	12496	+
12181	12497	return 0;
12182	12498	}
12183	12499
@@ -12883,6 +13199,10 @@ int bpf_check(struct bpf_prog *prog, union bpf_attr attr, · 699 unmodified lines
12883	13199	if (!env->explored_states)
12884	13200	goto skip_full_check;
12885	13201
	13202	+ ret = add_subprog_and_kfunc(env);
	13203	+ if (ret < 0)
	13204	+ goto skip_full_check;
	13205	+
12886	13206	ret = check_subprogs(env);
12887	13207	if (ret < 0)
12888	13208	goto skip_full_check;

tools/include/uapi/linux/bpf.h+4−0

@@ -1117,6 +1117,10 @@ enum bpf_link_type { · 1116 unmodified lines
1117	1117	* offset to another bpf function
1118	1118	*/
1119	1119	#define BPF_PSEUDO_CALL 1
	1120	+/* when bpf_call->src_reg == BPF_PSEUDO_KFUNC_CALL,
	1121	+ * bpf_call->imm == btf_id of a BTF_KIND_FUNC in the running kernel
	1122	+ */
	1123	+#define BPF_PSEUDO_KFUNC_CALL 2
1120	1124
1121	1125	/* flags for BPF_MAP_UPDATE_ELEM command */
1122	1126	enum {

tools/testing/selftests/bpf/verifier/calls.c+6−6

@@ -19,7 +19,7 @@ · 18 unmodified lines
19	19	BPF_MOV64_IMM(BPF_REG_0, 2),
20	20	BPF_EXIT_INSN(),
21	21	},
22		− .errstr_unpriv = "function calls to other bpf functions are allowed for",
	22	+ .errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
23	23	.result_unpriv = REJECT,
24	24	.result = ACCEPT,
25	25	.retval = 1,
@@ -136,7 +136,7 @@ · 110 unmodified lines
136	136	{
137	137	"calls: wrong src reg",
138	138	.insns = {
139		− BPF_RAW_INSN(BPF_JMP \| BPF_CALL, 0, 2, 0, 0),
	139	+ BPF_RAW_INSN(BPF_JMP \| BPF_CALL, 0, 3, 0, 0),
140	140	BPF_MOV64_IMM(BPF_REG_0, 1),
141	141	BPF_EXIT_INSN(),
142	142	},
@@ -397,7 +397,7 @@ · 254 unmodified lines
397	397	BPF_MOV64_IMM(BPF_REG_0, 1),
398	398	BPF_EXIT_INSN(),
399	399	},
400		− .errstr_unpriv = "function calls to other bpf functions are allowed for",
	400	+ .errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
401	401	.fixup_map_hash_48b = { 3 },
402	402	.result_unpriv = REJECT,
403	403	.result = ACCEPT,
@@ -1977,7 +1977,7 @@ · 1573 unmodified lines
1977	1977	BPF_EXIT_INSN(),
1978	1978	},
1979	1979	.prog_type = BPF_PROG_TYPE_SOCKET_FILTER,
1980		− .errstr_unpriv = "function calls to other bpf functions are allowed for",
	1980	+ .errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
1981	1981	.result_unpriv = REJECT,
1982	1982	.result = ACCEPT,
1983	1983	},
@@ -2003,7 +2003,7 @@ · 19 unmodified lines
2003	2003	BPF_EXIT_INSN(),
2004	2004	},
2005	2005	.prog_type = BPF_PROG_TYPE_SOCKET_FILTER,
2006		− .errstr_unpriv = "function calls to other bpf functions are allowed for",
	2006	+ .errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
2007	2007	.errstr = "!read_ok",
2008	2008	.result = REJECT,
2009	2009	},
@@ -2028,7 +2028,7 @@ · 18 unmodified lines
2028	2028	BPF_EXIT_INSN(),
2029	2029	},
2030	2030	.prog_type = BPF_PROG_TYPE_SOCKET_FILTER,
2031		− .errstr_unpriv = "function calls to other bpf functions are allowed for",
	2031	+ .errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
2032	2032	.errstr = "!read_ok",
2033	2033	.result = REJECT,
2034	2034	},

tools/testing/selftests/bpf/verifier/dead_code.c+5−5

@@ -85,7 +85,7 @@ · 84 unmodified lines
85	85	BPF_MOV64_IMM(BPF_REG_0, 12),
86	86	BPF_EXIT_INSN(),
87	87	},
88		− .errstr_unpriv = "function calls to other bpf functions are allowed for",
	88	+ .errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
89	89	.result_unpriv = REJECT,
90	90	.result = ACCEPT,
91	91	.retval = 7,
@@ -103,7 +103,7 @@ · 11 unmodified lines
103	103	BPF_MOV64_IMM(BPF_REG_0, 12),
104	104	BPF_EXIT_INSN(),
105	105	},
106		− .errstr_unpriv = "function calls to other bpf functions are allowed for",
	106	+ .errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
107	107	.result_unpriv = REJECT,
108	108	.result = ACCEPT,
109	109	.retval = 7,
@@ -121,7 +121,7 @@ · 11 unmodified lines
121	121	BPF_RAW_INSN(BPF_JMP \| BPF_CALL, 0, 1, 0, -5),
122	122	BPF_EXIT_INSN(),
123	123	},
124		− .errstr_unpriv = "function calls to other bpf functions are allowed for",
	124	+ .errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
125	125	.result_unpriv = REJECT,
126	126	.result = ACCEPT,
127	127	.retval = 7,
@@ -137,7 +137,7 @@ · 9 unmodified lines
137	137	BPF_MOV64_REG(BPF_REG_0, BPF_REG_1),
138	138	BPF_EXIT_INSN(),
139	139	},
140		− .errstr_unpriv = "function calls to other bpf functions are allowed for",
	140	+ .errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
141	141	.result_unpriv = REJECT,
142	142	.result = ACCEPT,
143	143	.retval = 2,
@@ -152,7 +152,7 @@ · 8 unmodified lines
152	152	BPF_MOV64_REG(BPF_REG_0, BPF_REG_1),
153	153	BPF_EXIT_INSN(),
154	154	},
155		− .errstr_unpriv = "function calls to other bpf functions are allowed for",
	155	+ .errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
156	156	.result_unpriv = REJECT,
157	157	.result = ACCEPT,
158	158	.retval = 2,