History

bpf: Support bpf program calling kernel function

This patch adds support to BPF verifier to allow bpf program calling kernel function directly. The use case included in this set is to allow bpf-tcp-cc to directly call some tcp-cc helper functions (e.g. "tcp_cong_avoid_ai()"). Those functions have already been used by some kernel tcp-cc implementations. This set will also allow the bpf-tcp-cc program to directly call the kernel tcp-cc implementation, For example, a bpf_dctcp may only want to implement its own dctcp_cwnd_event() and reuse other dctcp_*() directly from the kernel tcp_dctcp.c instead of reimplementing (or copy-and-pasting) them. The tcp-cc kernel functions mentioned above will be white listed for the struct_ops bpf-tcp-cc programs to use in a later patch. The white listed functions are not bounded to a fixed ABI contract. Those functions have already been used by the existing kernel tcp-cc. If any of them has changed, both in-tree and out-of-tree kernel tcp-cc implementations have to be changed. The same goes for the struct_ops bpf-tcp-cc programs which have to be adjusted accordingly. This patch is to make the required changes in the bpf verifier. First change is in btf.c, it adds a case in "btf_check_func_arg_match()". When the passed in "btf->kernel_btf == true", it means matching the verifier regs' states with a kernel function. This will handle the PTR_TO_BTF_ID reg. It also maps PTR_TO_SOCK_COMMON, PTR_TO_SOCKET, and PTR_TO_TCP_SOCK to its kernel's btf_id. In the later libbpf patch, the insn calling a kernel function will look like: insn->code == (BPF_JMP | BPF_CALL) insn->src_reg == BPF_PSEUDO_KFUNC_CALL /* <- new in this patch */ insn->imm == func_btf_id /* btf_id of the running kernel */ [ For the future calling function-in-kernel-module support, an array of module btf_fds can be passed at the load time and insn->off can be used to index into this array. ] At the early stage of verifier, the verifier will collect all kernel function calls into "struct bpf_kfunc_desc". Those descriptors are stored in "prog->aux->kfunc_tab" and will be available to the JIT. Since this "add" operation is similar to the current "add_subprog()" and looking for the same insn->code, they are done together in the new "add_subprog_and_kfunc()". In the "do_check()" stage, the new "check_kfunc_call()" is added to verify the kernel function call instruction: 1. Ensure the kernel function can be used by a particular BPF_PROG_TYPE. A new bpf_verifier_ops "check_kfunc_call" is added to do that. The bpf-tcp-cc struct_ops program will implement this function in a later patch. 2. Call "btf_check_kfunc_args_match()" to ensure the regs can be used as the args of a kernel function. 3. Mark the regs' type, subreg_def, and zext_dst. At the later do_misc_fixups() stage, the new fixup_kfunc_call() will replace the insn->imm with the function address (relative to __bpf_call_base). If needed, the jit can find the btf_func_model by calling the new bpf_jit_find_kfunc_model(prog, insn). With the imm set to the function address, "bpftool prog dump xlated" will be able to display the kernel function calls the same way as it displays other bpf helper calls. gpl_compatible program is required to call kernel function. This feature currently requires JIT. The verifier selftests are adjusted because of the changes in the verbose log in add_subprog_and_kfunc().

Martin KaFai Lau committed 5 years ago

Commit e6ac2451 parent 34747c413 files · +480 −46Browse files

Unified · Split

13 files
  1. arch/x86/net/bpf_jit_comp.c +5 −0
  2. include/linux/bpf.h +24 −0
  3. include/linux/btf.h +1 −0
  4. include/linux/filter.h +1 −0
  5. include/uapi/linux/bpf.h +4 −0
  6. kernel/bpf/btf.c +60 −5
  7. kernel/bpf/core.c +15 −3
  8. kernel/bpf/disasm.c +10 −3
  9. kernel/bpf/syscall.c +1 −0
  10. kernel/bpf/verifier.c +344 −24
  11. tools/include/uapi/linux/bpf.h +4 −0
  12. tools/testing/selftests/bpf/verifier/calls.c +6 −6
  13. tools/testing/selftests/bpf/verifier/dead_code.c +5 −5
arch/x86/net/bpf_jit_comp.c+5−0
@@ -2346,3 +2346,8 @@ out: · 2345 unmodified lines
2346 tmp : orig_prog);2346 tmp : orig_prog);
2347 return prog;2347 return prog;
2348}2348}
2349
2350bool bpf_jit_supports_kfunc_call(void)
2351{
2352 return true;
2353}
include/linux/bpf.h+24−0
@@ -427,6 +427,7 @@ enum bpf_reg_type { · 426 unmodified lines
427 PTR_TO_PERCPU_BTF_ID, /* reg points to a percpu kernel variable */427 PTR_TO_PERCPU_BTF_ID, /* reg points to a percpu kernel variable */
428 PTR_TO_FUNC, /* reg points to a bpf program function */428 PTR_TO_FUNC, /* reg points to a bpf program function */
429 PTR_TO_MAP_KEY, /* reg points to a map element key */429 PTR_TO_MAP_KEY, /* reg points to a map element key */
430 __BPF_REG_TYPE_MAX,
430};431};
431 432
432/* The information passed from prog-specific *_is_valid_access433/* The information passed from prog-specific *_is_valid_access
@@ -480,6 +481,7 @@ struct bpf_verifier_ops { · 47 unmodified lines
480 const struct btf_type *t, int off, int size,481 const struct btf_type *t, int off, int size,
481 enum bpf_access_type atype,482 enum bpf_access_type atype,
482 u32 *next_btf_id);483 u32 *next_btf_id);
484 bool (*check_kfunc_call)(u32 kfunc_btf_id);
483};485};
484 486
485struct bpf_prog_offload_ops {487struct bpf_prog_offload_ops {
@@ -796,6 +798,8 @@ struct btf_mod_pair { · 310 unmodified lines
796 struct module *module;798 struct module *module;
797};799};
798 800
801struct bpf_kfunc_desc_tab;
802
799struct bpf_prog_aux {803struct bpf_prog_aux {
800 atomic64_t refcnt;804 atomic64_t refcnt;
801 u32 used_map_cnt;805 u32 used_map_cnt;
@@ -832,6 +836,7 @@ struct bpf_prog_aux { · 30 unmodified lines
832 struct bpf_prog **func;836 struct bpf_prog **func;
833 void *jit_data; /* JIT specific data. arch dependent */837 void *jit_data; /* JIT specific data. arch dependent */
834 struct bpf_jit_poke_descriptor *poke_tab;838 struct bpf_jit_poke_descriptor *poke_tab;
839 struct bpf_kfunc_desc_tab *kfunc_tab;
835 u32 size_poke_tab;840 u32 size_poke_tab;
836 struct bpf_ksym ksym;841 struct bpf_ksym ksym;
837 const struct bpf_prog_ops *ops;842 const struct bpf_prog_ops *ops;
@@ -1547,6 +1552,9 @@ int btf_distill_func_proto(struct bpf_verifier_log *log, · 709 unmodified lines
1547struct bpf_reg_state;1552struct bpf_reg_state;
1548int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog,1553int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog,
1549 struct bpf_reg_state *regs);1554 struct bpf_reg_state *regs);
1555int btf_check_kfunc_arg_match(struct bpf_verifier_env *env,
1556 const struct btf *btf, u32 func_id,
1557 struct bpf_reg_state *regs);
1550int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog,1558int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog,
1551 struct bpf_reg_state *reg);1559 struct bpf_reg_state *reg);
1552int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog,1560int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog,
@@ -1557,6 +1565,10 @@ struct bpf_link *bpf_link_by_id(u32 id); · 4 unmodified lines
1557 1565
1558const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id);1566const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id);
1559void bpf_task_storage_free(struct task_struct *task);1567void bpf_task_storage_free(struct task_struct *task);
1568bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog);
1569const struct btf_func_model *
1570bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
1571 const struct bpf_insn *insn);
1560#else /* !CONFIG_BPF_SYSCALL */1572#else /* !CONFIG_BPF_SYSCALL */
1561static inline struct bpf_prog *bpf_prog_get(u32 ufd)1573static inline struct bpf_prog *bpf_prog_get(u32 ufd)
1562{1574{
@@ -1737,6 +1749,18 @@ bpf_base_func_proto(enum bpf_func_id func_id) · 174 unmodified lines
1737static inline void bpf_task_storage_free(struct task_struct *task)1749static inline void bpf_task_storage_free(struct task_struct *task)
1738{1750{
1739}1751}
1752
1753static inline bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog)
1754{
1755 return false;
1756}
1757
1758static inline const struct btf_func_model *
1759bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
1760 const struct bpf_insn *insn)
1761{
1762 return NULL;
1763}
1740#endif /* CONFIG_BPF_SYSCALL */1764#endif /* CONFIG_BPF_SYSCALL */
1741 1765
1742void __bpf_free_used_btfs(struct bpf_prog_aux *aux,1766void __bpf_free_used_btfs(struct bpf_prog_aux *aux,
include/linux/btf.h+1−0
@@ -110,6 +110,7 @@ const struct btf_type *btf_type_resolve_func_ptr(const struct btf *btf, · 109 unmodified lines
110const struct btf_type *110const struct btf_type *
111btf_resolve_size(const struct btf *btf, const struct btf_type *type,111btf_resolve_size(const struct btf *btf, const struct btf_type *type,
112 u32 *type_size);112 u32 *type_size);
113const char *btf_type_str(const struct btf_type *t);
113 114
114#define for_each_member(i, struct_type, member) \115#define for_each_member(i, struct_type, member) \
115 for (i = 0, member = btf_type_member(struct_type); \116 for (i = 0, member = btf_type_member(struct_type); \
include/linux/filter.h+1−0
@@ -918,6 +918,7 @@ u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); · 917 unmodified lines
918struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog);918struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog);
919void bpf_jit_compile(struct bpf_prog *prog);919void bpf_jit_compile(struct bpf_prog *prog);
920bool bpf_jit_needs_zext(void);920bool bpf_jit_needs_zext(void);
921bool bpf_jit_supports_kfunc_call(void);
921bool bpf_helper_changes_pkt_data(void *func);922bool bpf_helper_changes_pkt_data(void *func);
922 923
923static inline bool bpf_dump_raw_ok(const struct cred *cred)924static inline bool bpf_dump_raw_ok(const struct cred *cred)
include/uapi/linux/bpf.h+4−0
@@ -1117,6 +1117,10 @@ enum bpf_link_type { · 1116 unmodified lines
1117 * offset to another bpf function1117 * offset to another bpf function
1118 */1118 */
1119#define BPF_PSEUDO_CALL 11119#define BPF_PSEUDO_CALL 1
1120/* when bpf_call->src_reg == BPF_PSEUDO_KFUNC_CALL,
1121 * bpf_call->imm == btf_id of a BTF_KIND_FUNC in the running kernel
1122 */
1123#define BPF_PSEUDO_KFUNC_CALL 2
1120 1124
1121/* flags for BPF_MAP_UPDATE_ELEM command */1125/* flags for BPF_MAP_UPDATE_ELEM command */
1122enum {1126enum {
kernel/bpf/btf.c+60−5
@@ -283,7 +283,7 @@ static const char * const btf_kind_str[NR_BTF_KINDS] = { · 282 unmodified lines
283 [BTF_KIND_FLOAT] = "FLOAT",283 [BTF_KIND_FLOAT] = "FLOAT",
284};284};
285 285
286static const char *btf_type_str(const struct btf_type *t)286const char *btf_type_str(const struct btf_type *t)
287{287{
288 return btf_kind_str[BTF_INFO_KIND(t->info)];288 return btf_kind_str[BTF_INFO_KIND(t->info)];
289}289}
@@ -5362,6 +5362,14 @@ int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *pr · 5072 unmodified lines
5362 return btf_check_func_type_match(log, btf1, t1, btf2, t2);5362 return btf_check_func_type_match(log, btf1, t1, btf2, t2);
5363}5363}
5364 5364
5365static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = {
5366#ifdef CONFIG_NET
5367 [PTR_TO_SOCKET] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK],
5368 [PTR_TO_SOCK_COMMON] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
5369 [PTR_TO_TCP_SOCK] = &btf_sock_ids[BTF_SOCK_TYPE_TCP],
5370#endif
5371};
5372
5365static int btf_check_func_arg_match(struct bpf_verifier_env *env,5373static int btf_check_func_arg_match(struct bpf_verifier_env *env,
5366 const struct btf *btf, u32 func_id,5374 const struct btf *btf, u32 func_id,
5367 struct bpf_reg_state *regs,5375 struct bpf_reg_state *regs,
@@ -5371,12 +5379,12 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, · 3 unmodified lines
5371 const char *func_name, *ref_tname;5379 const char *func_name, *ref_tname;
5372 const struct btf_type *t, *ref_t;5380 const struct btf_type *t, *ref_t;
5373 const struct btf_param *args;5381 const struct btf_param *args;
5374 u32 i, nargs;5382 u32 i, nargs, ref_id;
5375 5383
5376 t = btf_type_by_id(btf, func_id);5384 t = btf_type_by_id(btf, func_id);
5377 if (!t || !btf_type_is_func(t)) {5385 if (!t || !btf_type_is_func(t)) {
5378 /* These checks were already done by the verifier while loading5386 /* These checks were already done by the verifier while loading
5379 * struct bpf_func_info5387 * struct bpf_func_info or in add_kfunc_call().
5380 */5388 */
5381 bpf_log(log, "BTF of func_id %u doesn't point to KIND_FUNC\n",5389 bpf_log(log, "BTF of func_id %u doesn't point to KIND_FUNC\n",
5382 func_id);5390 func_id);
@@ -5418,9 +5426,49 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, · 35 unmodified lines
5418 return -EINVAL;5426 return -EINVAL;
5419 }5427 }
5420 5428
5421 ref_t = btf_type_skip_modifiers(btf, t->type, NULL);5429 ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id);
5422 ref_tname = btf_name_by_offset(btf, ref_t->name_off);5430 ref_tname = btf_name_by_offset(btf, ref_t->name_off);
5423 if (btf_get_prog_ctx_type(log, btf, t, env->prog->type, i)) {5431 if (btf_is_kernel(btf)) {
5432 const struct btf_type *reg_ref_t;
5433 const struct btf *reg_btf;
5434 const char *reg_ref_tname;
5435 u32 reg_ref_id;
5436
5437 if (!btf_type_is_struct(ref_t)) {
5438 bpf_log(log, "kernel function %s args#%d pointer type %s %s is not supported\n",
5439 func_name, i, btf_type_str(ref_t),
5440 ref_tname);
5441 return -EINVAL;
5442 }
5443
5444 if (reg->type == PTR_TO_BTF_ID) {
5445 reg_btf = reg->btf;
5446 reg_ref_id = reg->btf_id;
5447 } else if (reg2btf_ids[reg->type]) {
5448 reg_btf = btf_vmlinux;
5449 reg_ref_id = *reg2btf_ids[reg->type];
5450 } else {
5451 bpf_log(log, "kernel function %s args#%d expected pointer to %s %s but R%d is not a pointer to btf_id\n",
5452 func_name, i,
5453 btf_type_str(ref_t), ref_tname, regno);
5454 return -EINVAL;
5455 }
5456
5457 reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id,
5458 &reg_ref_id);
5459 reg_ref_tname = btf_name_by_offset(reg_btf,
5460 reg_ref_t->name_off);
5461 if (!btf_struct_ids_match(log, reg_btf, reg_ref_id,
5462 reg->off, btf, ref_id)) {
5463 bpf_log(log, "kernel function %s args#%d expected pointer to %s %s but R%d has a pointer to %s %s\n",
5464 func_name, i,
5465 btf_type_str(ref_t), ref_tname,
5466 regno, btf_type_str(reg_ref_t),
5467 reg_ref_tname);
5468 return -EINVAL;
5469 }
5470 } else if (btf_get_prog_ctx_type(log, btf, t,
5471 env->prog->type, i)) {
5424 /* If function expects ctx type in BTF check that caller5472 /* If function expects ctx type in BTF check that caller
5425 * is passing PTR_TO_CTX.5473 * is passing PTR_TO_CTX.
5426 */5474 */
@@ -5493,6 +5541,13 @@ int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog, · 66 unmodified lines
5493 return err;5541 return err;
5494}5542}
5495 5543
5544int btf_check_kfunc_arg_match(struct bpf_verifier_env *env,
5545 const struct btf *btf, u32 func_id,
5546 struct bpf_reg_state *regs)
5547{
5548 return btf_check_func_arg_match(env, btf, func_id, regs, false);
5549}
5550
5496/* Convert BTF of a function into bpf_reg_state if possible5551/* Convert BTF of a function into bpf_reg_state if possible
5497 * Returns:5552 * Returns:
5498 * EFAULT - there is a verifier bug. Abort verification.5553 * EFAULT - there is a verifier bug. Abort verification.
kernel/bpf/core.c+15−3
@@ -159,6 +159,9 @@ void bpf_prog_jit_attempt_done(struct bpf_prog *prog) · 158 unmodified lines
159 kvfree(prog->aux->jited_linfo);159 kvfree(prog->aux->jited_linfo);
160 prog->aux->jited_linfo = NULL;160 prog->aux->jited_linfo = NULL;
161 }161 }
162
163 kfree(prog->aux->kfunc_tab);
164 prog->aux->kfunc_tab = NULL;
162}165}
163 166
164/* The jit engine is responsible to provide an array167/* The jit engine is responsible to provide an array
@@ -1840,9 +1843,15 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err) · 1675 unmodified lines
1840 /* In case of BPF to BPF calls, verifier did all the prep1843 /* In case of BPF to BPF calls, verifier did all the prep
1841 * work with regards to JITing, etc.1844 * work with regards to JITing, etc.
1842 */1845 */
1846 bool jit_needed = false;
1847
1843 if (fp->bpf_func)1848 if (fp->bpf_func)
1844 goto finalize;1849 goto finalize;
1845 1850
1851 if (IS_ENABLED(CONFIG_BPF_JIT_ALWAYS_ON) ||
1852 bpf_prog_has_kfunc_call(fp))
1853 jit_needed = true;
1854
1846 bpf_prog_select_func(fp);1855 bpf_prog_select_func(fp);
1847 1856
1848 /* eBPF JITs can rewrite the program in case constant1857 /* eBPF JITs can rewrite the program in case constant
@@ -1858,12 +1867,10 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err) · 9 unmodified lines
1858 1867
1859 fp = bpf_int_jit_compile(fp);1868 fp = bpf_int_jit_compile(fp);
1860 bpf_prog_jit_attempt_done(fp);1869 bpf_prog_jit_attempt_done(fp);
1861#ifdef CONFIG_BPF_JIT_ALWAYS_ON1870 if (!fp->jited && jit_needed) {
1862 if (!fp->jited) {
1863 *err = -ENOTSUPP;1871 *err = -ENOTSUPP;
1864 return fp;1872 return fp;
1865 }1873 }
1866#endif
1867 } else {1874 } else {
1868 *err = bpf_prog_offload_compile(fp);1875 *err = bpf_prog_offload_compile(fp);
1869 if (*err)1876 if (*err)
@@ -2343,6 +2350,11 @@ bool __weak bpf_jit_needs_zext(void) · 473 unmodified lines
2343 return false;2350 return false;
2344}2351}
2345 2352
2353bool __weak bpf_jit_supports_kfunc_call(void)
2354{
2355 return false;
2356}
2357
2346/* To execute LD_ABS/LD_IND instructions __bpf_prog_run() may call2358/* To execute LD_ABS/LD_IND instructions __bpf_prog_run() may call
2347 * skb_copy_bits(), so provide a weak definition of it for NET-less config.2359 * skb_copy_bits(), so provide a weak definition of it for NET-less config.
2348 */2360 */
kernel/bpf/disasm.c+10−3
@@ -19,16 +19,23 @@ static const char *__func_get_name(const struct bpf_insn_cbs *cbs, · 18 unmodified lines
19{19{
20 BUILD_BUG_ON(ARRAY_SIZE(func_id_str) != __BPF_FUNC_MAX_ID);20 BUILD_BUG_ON(ARRAY_SIZE(func_id_str) != __BPF_FUNC_MAX_ID);
21 21
22 if (insn->src_reg != BPF_PSEUDO_CALL &&22 if (!insn->src_reg &&
23 insn->imm >= 0 && insn->imm < __BPF_FUNC_MAX_ID &&23 insn->imm >= 0 && insn->imm < __BPF_FUNC_MAX_ID &&
24 func_id_str[insn->imm])24 func_id_str[insn->imm])
25 return func_id_str[insn->imm];25 return func_id_str[insn->imm];
26 26
27 if (cbs && cbs->cb_call)27 if (cbs && cbs->cb_call) {
28 return cbs->cb_call(cbs->private_data, insn);28 const char *res;
29
30 res = cbs->cb_call(cbs->private_data, insn);
31 if (res)
32 return res;
33 }
29 34
30 if (insn->src_reg == BPF_PSEUDO_CALL)35 if (insn->src_reg == BPF_PSEUDO_CALL)
31 snprintf(buff, len, "%+d", insn->imm);36 snprintf(buff, len, "%+d", insn->imm);
37 else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL)
38 snprintf(buff, len, "kernel-function");
32 39
33 return buff;40 return buff;
34}41}
kernel/bpf/syscall.c+1−0
@@ -1696,6 +1696,7 @@ static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) · 1695 unmodified lines
1696 btf_put(prog->aux->btf);1696 btf_put(prog->aux->btf);
1697 kvfree(prog->aux->jited_linfo);1697 kvfree(prog->aux->jited_linfo);
1698 kvfree(prog->aux->linfo);1698 kvfree(prog->aux->linfo);
1699 kfree(prog->aux->kfunc_tab);
1699 if (prog->aux->attach_btf)1700 if (prog->aux->attach_btf)
1700 btf_put(prog->aux->attach_btf);1701 btf_put(prog->aux->attach_btf);
1701 1702
kernel/bpf/verifier.c+344−24
@@ -234,6 +234,12 @@ static bool bpf_pseudo_call(const struct bpf_insn *insn) · 233 unmodified lines
234 insn->src_reg == BPF_PSEUDO_CALL;234 insn->src_reg == BPF_PSEUDO_CALL;
235}235}
236 236
237static bool bpf_pseudo_kfunc_call(const struct bpf_insn *insn)
238{
239 return insn->code == (BPF_JMP | BPF_CALL) &&
240 insn->src_reg == BPF_PSEUDO_KFUNC_CALL;
241}
242
237static bool bpf_pseudo_func(const struct bpf_insn *insn)243static bool bpf_pseudo_func(const struct bpf_insn *insn)
238{244{
239 return insn->code == (BPF_LD | BPF_IMM | BPF_DW) &&245 return insn->code == (BPF_LD | BPF_IMM | BPF_DW) &&
@@ -1554,47 +1560,205 @@ static int add_subprog(struct bpf_verifier_env *env, int off) · 1314 unmodified lines
1554 verbose(env, "too many subprograms\n");1560 verbose(env, "too many subprograms\n");
1555 return -E2BIG;1561 return -E2BIG;
1556 }1562 }
1563 /* determine subprog starts. The end is one before the next starts */
1557 env->subprog_info[env->subprog_cnt++].start = off;1564 env->subprog_info[env->subprog_cnt++].start = off;
1558 sort(env->subprog_info, env->subprog_cnt,1565 sort(env->subprog_info, env->subprog_cnt,
1559 sizeof(env->subprog_info[0]), cmp_subprogs, NULL);1566 sizeof(env->subprog_info[0]), cmp_subprogs, NULL);
1560 return env->subprog_cnt - 1;1567 return env->subprog_cnt - 1;
1561}1568}
1562 1569
1563static int check_subprogs(struct bpf_verifier_env *env)1570struct bpf_kfunc_desc {
1571 struct btf_func_model func_model;
1572 u32 func_id;
1573 s32 imm;
1574};
1575
1576#define MAX_KFUNC_DESCS 256
1577struct bpf_kfunc_desc_tab {
1578 struct bpf_kfunc_desc descs[MAX_KFUNC_DESCS];
1579 u32 nr_descs;
1580};
1581
1582static int kfunc_desc_cmp_by_id(const void *a, const void *b)
1583{
1584 const struct bpf_kfunc_desc *d0 = a;
1585 const struct bpf_kfunc_desc *d1 = b;
1586
1587 /* func_id is not greater than BTF_MAX_TYPE */
1588 return d0->func_id - d1->func_id;
1589}
1590
1591static const struct bpf_kfunc_desc *
1592find_kfunc_desc(const struct bpf_prog *prog, u32 func_id)
1593{
1594 struct bpf_kfunc_desc desc = {
1595 .func_id = func_id,
1596 };
1597 struct bpf_kfunc_desc_tab *tab;
1598
1599 tab = prog->aux->kfunc_tab;
1600 return bsearch(&desc, tab->descs, tab->nr_descs,
1601 sizeof(tab->descs[0]), kfunc_desc_cmp_by_id);
1602}
1603
1604static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id)
1605{
1606 const struct btf_type *func, *func_proto;
1607 struct bpf_kfunc_desc_tab *tab;
1608 struct bpf_prog_aux *prog_aux;
1609 struct bpf_kfunc_desc *desc;
1610 const char *func_name;
1611 unsigned long addr;
1612 int err;
1613
1614 prog_aux = env->prog->aux;
1615 tab = prog_aux->kfunc_tab;
1616 if (!tab) {
1617 if (!btf_vmlinux) {
1618 verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n");
1619 return -ENOTSUPP;
1620 }
1621
1622 if (!env->prog->jit_requested) {
1623 verbose(env, "JIT is required for calling kernel function\n");
1624 return -ENOTSUPP;
1625 }
1626
1627 if (!bpf_jit_supports_kfunc_call()) {
1628 verbose(env, "JIT does not support calling kernel function\n");
1629 return -ENOTSUPP;
1630 }
1631
1632 if (!env->prog->gpl_compatible) {
1633 verbose(env, "cannot call kernel function from non-GPL compatible program\n");
1634 return -EINVAL;
1635 }
1636
1637 tab = kzalloc(sizeof(*tab), GFP_KERNEL);
1638 if (!tab)
1639 return -ENOMEM;
1640 prog_aux->kfunc_tab = tab;
1641 }
1642
1643 if (find_kfunc_desc(env->prog, func_id))
1644 return 0;
1645
1646 if (tab->nr_descs == MAX_KFUNC_DESCS) {
1647 verbose(env, "too many different kernel function calls\n");
1648 return -E2BIG;
1649 }
1650
1651 func = btf_type_by_id(btf_vmlinux, func_id);
1652 if (!func || !btf_type_is_func(func)) {
1653 verbose(env, "kernel btf_id %u is not a function\n",
1654 func_id);
1655 return -EINVAL;
1656 }
1657 func_proto = btf_type_by_id(btf_vmlinux, func->type);
1658 if (!func_proto || !btf_type_is_func_proto(func_proto)) {
1659 verbose(env, "kernel function btf_id %u does not have a valid func_proto\n",
1660 func_id);
1661 return -EINVAL;
1662 }
1663
1664 func_name = btf_name_by_offset(btf_vmlinux, func->name_off);
1665 addr = kallsyms_lookup_name(func_name);
1666 if (!addr) {
1667 verbose(env, "cannot find address for kernel function %s\n",
1668 func_name);
1669 return -EINVAL;
1670 }
1671
1672 desc = &tab->descs[tab->nr_descs++];
1673 desc->func_id = func_id;
1674 desc->imm = BPF_CAST_CALL(addr) - __bpf_call_base;
1675 err = btf_distill_func_proto(&env->log, btf_vmlinux,
1676 func_proto, func_name,
1677 &desc->func_model);
1678 if (!err)
1679 sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
1680 kfunc_desc_cmp_by_id, NULL);
1681 return err;
1682}
1683
1684static int kfunc_desc_cmp_by_imm(const void *a, const void *b)
1685{
1686 const struct bpf_kfunc_desc *d0 = a;
1687 const struct bpf_kfunc_desc *d1 = b;
1688
1689 if (d0->imm > d1->imm)
1690 return 1;
1691 else if (d0->imm < d1->imm)
1692 return -1;
1693 return 0;
1694}
1695
1696static void sort_kfunc_descs_by_imm(struct bpf_prog *prog)
1697{
1698 struct bpf_kfunc_desc_tab *tab;
1699
1700 tab = prog->aux->kfunc_tab;
1701 if (!tab)
1702 return;
1703
1704 sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
1705 kfunc_desc_cmp_by_imm, NULL);
1706}
1707
1708bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog)
1709{
1710 return !!prog->aux->kfunc_tab;
1711}
1712
1713const struct btf_func_model *
1714bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
1715 const struct bpf_insn *insn)
1716{
1717 const struct bpf_kfunc_desc desc = {
1718 .imm = insn->imm,
1719 };
1720 const struct bpf_kfunc_desc *res;
1721 struct bpf_kfunc_desc_tab *tab;
1722
1723 tab = prog->aux->kfunc_tab;
1724 res = bsearch(&desc, tab->descs, tab->nr_descs,
1725 sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm);
1726
1727 return res ? &res->func_model : NULL;
1728}
1729
1730static int add_subprog_and_kfunc(struct bpf_verifier_env *env)
1564{1731{
1565 int i, ret, subprog_start, subprog_end, off, cur_subprog = 0;
1566 struct bpf_subprog_info *subprog = env->subprog_info;1732 struct bpf_subprog_info *subprog = env->subprog_info;
1567 struct bpf_insn *insn = env->prog->insnsi;1733 struct bpf_insn *insn = env->prog->insnsi;
1568 int insn_cnt = env->prog->len;1734 int i, ret, insn_cnt = env->prog->len;
1569 1735
1570 /* Add entry function. */1736 /* Add entry function. */
1571 ret = add_subprog(env, 0);1737 ret = add_subprog(env, 0);
1572 if (ret < 0)1738 if (ret)
1573 return ret;1739 return ret;
1574 1740
1575 /* determine subprog starts. The end is one before the next starts */1741 for (i = 0; i < insn_cnt; i++, insn++) {
1576 for (i = 0; i < insn_cnt; i++) {1742 if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn) &&
1577 if (bpf_pseudo_func(insn + i)) {1743 !bpf_pseudo_kfunc_call(insn))
1578 if (!env->bpf_capable) {
1579 verbose(env,
1580 "function pointers are allowed for CAP_BPF and CAP_SYS_ADMIN\n");
1581 return -EPERM;
1582 }
1583 ret = add_subprog(env, i + insn[i].imm + 1);
1584 if (ret < 0)
1585 return ret;
1586 /* remember subprog */
1587 insn[i + 1].imm = ret;
1588 continue;
1589 }
1590 if (!bpf_pseudo_call(insn + i))
1591 continue;1744 continue;
1745
1592 if (!env->bpf_capable) {1746 if (!env->bpf_capable) {
1593 verbose(env,1747 verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n");
1594 "function calls to other bpf functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n");
1595 return -EPERM;1748 return -EPERM;
1596 }1749 }
1597 ret = add_subprog(env, i + insn[i].imm + 1);1750
1751 if (bpf_pseudo_func(insn)) {
1752 ret = add_subprog(env, i + insn->imm + 1);
1753 if (ret >= 0)
1754 /* remember subprog */
1755 insn[1].imm = ret;
1756 } else if (bpf_pseudo_call(insn)) {
1757 ret = add_subprog(env, i + insn->imm + 1);
1758 } else {
1759 ret = add_kfunc_call(env, insn->imm);
1760 }
1761
1598 if (ret < 0)1762 if (ret < 0)
1599 return ret;1763 return ret;
1600 }1764 }
@@ -1608,6 +1772,16 @@ static int check_subprogs(struct bpf_verifier_env *env) · 7 unmodified lines
1608 for (i = 0; i < env->subprog_cnt; i++)1772 for (i = 0; i < env->subprog_cnt; i++)
1609 verbose(env, "func#%d @%d\n", i, subprog[i].start);1773 verbose(env, "func#%d @%d\n", i, subprog[i].start);
1610 1774
1775 return 0;
1776}
1777
1778static int check_subprogs(struct bpf_verifier_env *env)
1779{
1780 int i, subprog_start, subprog_end, off, cur_subprog = 0;
1781 struct bpf_subprog_info *subprog = env->subprog_info;
1782 struct bpf_insn *insn = env->prog->insnsi;
1783 int insn_cnt = env->prog->len;
1784
1611 /* now check that all jumps are within the same subprog */1785 /* now check that all jumps are within the same subprog */
1612 subprog_start = subprog[cur_subprog].start;1786 subprog_start = subprog[cur_subprog].start;
1613 subprog_end = subprog[cur_subprog + 1].start;1787 subprog_end = subprog[cur_subprog + 1].start;
@@ -1916,6 +2090,17 @@ static int get_prev_insn_idx(struct bpf_verifier_state *st, int i, · 302 unmodified lines
1916 return i;2090 return i;
1917}2091}
1918 2092
2093static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn)
2094{
2095 const struct btf_type *func;
2096
2097 if (insn->src_reg != BPF_PSEUDO_KFUNC_CALL)
2098 return NULL;
2099
2100 func = btf_type_by_id(btf_vmlinux, insn->imm);
2101 return btf_name_by_offset(btf_vmlinux, func->name_off);
2102}
2103
1919/* For given verifier state backtrack_insn() is called from the last insn to2104/* For given verifier state backtrack_insn() is called from the last insn to
1920 * the first insn. Its purpose is to compute a bitmask of registers and2105 * the first insn. Its purpose is to compute a bitmask of registers and
1921 * stack slots that needs precision in the parent verifier state.2106 * stack slots that needs precision in the parent verifier state.
@@ -1924,6 +2109,7 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, · 2 unmodified lines
1924 u32 *reg_mask, u64 *stack_mask)2109 u32 *reg_mask, u64 *stack_mask)
1925{2110{
1926 const struct bpf_insn_cbs cbs = {2111 const struct bpf_insn_cbs cbs = {
2112 .cb_call = disasm_kfunc_name,
1927 .cb_print = verbose,2113 .cb_print = verbose,
1928 .private_data = env,2114 .private_data = env,
1929 };2115 };
@@ -5960,6 +6146,98 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn · 4030 unmodified lines
5960 return 0;6146 return 0;
5961}6147}
5962 6148
6149/* mark_btf_func_reg_size() is used when the reg size is determined by
6150 * the BTF func_proto's return value size and argument.
6151 */
6152static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno,
6153 size_t reg_size)
6154{
6155 struct bpf_reg_state *reg = &cur_regs(env)[regno];
6156
6157 if (regno == BPF_REG_0) {
6158 /* Function return value */
6159 reg->live |= REG_LIVE_WRITTEN;
6160 reg->subreg_def = reg_size == sizeof(u64) ?
6161 DEF_NOT_SUBREG : env->insn_idx + 1;
6162 } else {
6163 /* Function argument */
6164 if (reg_size == sizeof(u64)) {
6165 mark_insn_zext(env, reg);
6166 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
6167 } else {
6168 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ32);
6169 }
6170 }
6171}
6172
6173static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn)
6174{
6175 const struct btf_type *t, *func, *func_proto, *ptr_type;
6176 struct bpf_reg_state *regs = cur_regs(env);
6177 const char *func_name, *ptr_type_name;
6178 u32 i, nargs, func_id, ptr_type_id;
6179 const struct btf_param *args;
6180 int err;
6181
6182 func_id = insn->imm;
6183 func = btf_type_by_id(btf_vmlinux, func_id);
6184 func_name = btf_name_by_offset(btf_vmlinux, func->name_off);
6185 func_proto = btf_type_by_id(btf_vmlinux, func->type);
6186
6187 if (!env->ops->check_kfunc_call ||
6188 !env->ops->check_kfunc_call(func_id)) {
6189 verbose(env, "calling kernel function %s is not allowed\n",
6190 func_name);
6191 return -EACCES;
6192 }
6193
6194 /* Check the arguments */
6195 err = btf_check_kfunc_arg_match(env, btf_vmlinux, func_id, regs);
6196 if (err)
6197 return err;
6198
6199 for (i = 0; i < CALLER_SAVED_REGS; i++)
6200 mark_reg_not_init(env, regs, caller_saved[i]);
6201
6202 /* Check return type */
6203 t = btf_type_skip_modifiers(btf_vmlinux, func_proto->type, NULL);
6204 if (btf_type_is_scalar(t)) {
6205 mark_reg_unknown(env, regs, BPF_REG_0);
6206 mark_btf_func_reg_size(env, BPF_REG_0, t->size);
6207 } else if (btf_type_is_ptr(t)) {
6208 ptr_type = btf_type_skip_modifiers(btf_vmlinux, t->type,
6209 &ptr_type_id);
6210 if (!btf_type_is_struct(ptr_type)) {
6211 ptr_type_name = btf_name_by_offset(btf_vmlinux,
6212 ptr_type->name_off);
6213 verbose(env, "kernel function %s returns pointer type %s %s is not supported\n",
6214 func_name, btf_type_str(ptr_type),
6215 ptr_type_name);
6216 return -EINVAL;
6217 }
6218 mark_reg_known_zero(env, regs, BPF_REG_0);
6219 regs[BPF_REG_0].btf = btf_vmlinux;
6220 regs[BPF_REG_0].type = PTR_TO_BTF_ID;
6221 regs[BPF_REG_0].btf_id = ptr_type_id;
6222 mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *));
6223 } /* else { add_kfunc_call() ensures it is btf_type_is_void(t) } */
6224
6225 nargs = btf_type_vlen(func_proto);
6226 args = (const struct btf_param *)(func_proto + 1);
6227 for (i = 0; i < nargs; i++) {
6228 u32 regno = i + 1;
6229
6230 t = btf_type_skip_modifiers(btf_vmlinux, args[i].type, NULL);
6231 if (btf_type_is_ptr(t))
6232 mark_btf_func_reg_size(env, regno, sizeof(void *));
6233 else
6234 /* scalar. ensured by btf_check_kfunc_arg_match() */
6235 mark_btf_func_reg_size(env, regno, t->size);
6236 }
6237
6238 return 0;
6239}
6240
5963static bool signed_add_overflows(s64 a, s64 b)6241static bool signed_add_overflows(s64 a, s64 b)
5964{6242{
5965 /* Do the add in u64, where overflow is well-defined */6243 /* Do the add in u64, where overflow is well-defined */
@@ -10162,6 +10440,7 @@ static int do_check(struct bpf_verifier_env *env) · 4196 unmodified lines
10162 10440
10163 if (env->log.level & BPF_LOG_LEVEL) {10441 if (env->log.level & BPF_LOG_LEVEL) {
10164 const struct bpf_insn_cbs cbs = {10442 const struct bpf_insn_cbs cbs = {
10443 .cb_call = disasm_kfunc_name,
10165 .cb_print = verbose,10444 .cb_print = verbose,
10166 .private_data = env,10445 .private_data = env,
10167 };10446 };
@@ -10309,7 +10588,8 @@ static int do_check(struct bpf_verifier_env *env) · 141 unmodified lines
10309 if (BPF_SRC(insn->code) != BPF_K ||10588 if (BPF_SRC(insn->code) != BPF_K ||
10310 insn->off != 0 ||10589 insn->off != 0 ||
10311 (insn->src_reg != BPF_REG_0 &&10590 (insn->src_reg != BPF_REG_0 &&
10312 insn->src_reg != BPF_PSEUDO_CALL) ||10591 insn->src_reg != BPF_PSEUDO_CALL &&
10592 insn->src_reg != BPF_PSEUDO_KFUNC_CALL) ||
10313 insn->dst_reg != BPF_REG_0 ||10593 insn->dst_reg != BPF_REG_0 ||
10314 class == BPF_JMP32) {10594 class == BPF_JMP32) {
10315 verbose(env, "BPF_CALL uses reserved fields\n");10595 verbose(env, "BPF_CALL uses reserved fields\n");
@@ -10324,6 +10604,8 @@ static int do_check(struct bpf_verifier_env *env) · 8 unmodified lines
10324 }10604 }
10325 if (insn->src_reg == BPF_PSEUDO_CALL)10605 if (insn->src_reg == BPF_PSEUDO_CALL)
10326 err = check_func_call(env, insn, &env->insn_idx);10606 err = check_func_call(env, insn, &env->insn_idx);
10607 else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL)
10608 err = check_kfunc_call(env, insn);
10327 else10609 else
10328 err = check_helper_call(env, insn, &env->insn_idx);10610 err = check_helper_call(env, insn, &env->insn_idx);
10329 if (err)10611 if (err)
@@ -11634,6 +11916,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 1304 unmodified lines
11634 func[i]->aux->name[0] = 'F';11916 func[i]->aux->name[0] = 'F';
11635 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;11917 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;
11636 func[i]->jit_requested = 1;11918 func[i]->jit_requested = 1;
11919 func[i]->aux->kfunc_tab = prog->aux->kfunc_tab;
11637 func[i]->aux->linfo = prog->aux->linfo;11920 func[i]->aux->linfo = prog->aux->linfo;
11638 func[i]->aux->nr_linfo = prog->aux->nr_linfo;11921 func[i]->aux->nr_linfo = prog->aux->nr_linfo;
11639 func[i]->aux->jited_linfo = prog->aux->jited_linfo;11922 func[i]->aux->jited_linfo = prog->aux->jited_linfo;
@@ -11773,6 +12056,7 @@ static int fixup_call_args(struct bpf_verifier_env *env) · 133 unmodified lines
11773#ifndef CONFIG_BPF_JIT_ALWAYS_ON12056#ifndef CONFIG_BPF_JIT_ALWAYS_ON
11774 struct bpf_prog *prog = env->prog;12057 struct bpf_prog *prog = env->prog;
11775 struct bpf_insn *insn = prog->insnsi;12058 struct bpf_insn *insn = prog->insnsi;
12059 bool has_kfunc_call = bpf_prog_has_kfunc_call(prog);
11776 int i, depth;12060 int i, depth;
11777#endif12061#endif
11778 int err = 0;12062 int err = 0;
@@ -11786,6 +12070,10 @@ static int fixup_call_args(struct bpf_verifier_env *env) · 7 unmodified lines
11786 return err;12070 return err;
11787 }12071 }
11788#ifndef CONFIG_BPF_JIT_ALWAYS_ON12072#ifndef CONFIG_BPF_JIT_ALWAYS_ON
12073 if (has_kfunc_call) {
12074 verbose(env, "calling kernel functions are not allowed in non-JITed programs\n");
12075 return -EINVAL;
12076 }
11789 if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) {12077 if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) {
11790 /* When JIT fails the progs with bpf2bpf calls and tail_calls12078 /* When JIT fails the progs with bpf2bpf calls and tail_calls
11791 * have to be rejected, since interpreter doesn't support them yet.12079 * have to be rejected, since interpreter doesn't support them yet.
@@ -11814,6 +12102,26 @@ static int fixup_call_args(struct bpf_verifier_env *env) · 22 unmodified lines
11814 return err;12102 return err;
11815}12103}
11816 12104
12105static int fixup_kfunc_call(struct bpf_verifier_env *env,
12106 struct bpf_insn *insn)
12107{
12108 const struct bpf_kfunc_desc *desc;
12109
12110 /* insn->imm has the btf func_id. Replace it with
12111 * an address (relative to __bpf_base_call).
12112 */
12113 desc = find_kfunc_desc(env->prog, insn->imm);
12114 if (!desc) {
12115 verbose(env, "verifier internal error: kernel function descriptor not found for func_id %u\n",
12116 insn->imm);
12117 return -EFAULT;
12118 }
12119
12120 insn->imm = desc->imm;
12121
12122 return 0;
12123}
12124
11817/* Do various post-verification rewrites in a single program pass.12125/* Do various post-verification rewrites in a single program pass.
11818 * These rewrites simplify JIT and interpreter implementations.12126 * These rewrites simplify JIT and interpreter implementations.
11819 */12127 */
@@ -11949,6 +12257,12 @@ static int do_misc_fixups(struct bpf_verifier_env *env) · 129 unmodified lines
11949 continue;12257 continue;
11950 if (insn->src_reg == BPF_PSEUDO_CALL)12258 if (insn->src_reg == BPF_PSEUDO_CALL)
11951 continue;12259 continue;
12260 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
12261 ret = fixup_kfunc_call(env, insn);
12262 if (ret)
12263 return ret;
12264 continue;
12265 }
11952 12266
11953 if (insn->imm == BPF_FUNC_get_route_realm)12267 if (insn->imm == BPF_FUNC_get_route_realm)
11954 prog->dst_needed = 1;12268 prog->dst_needed = 1;
@@ -12178,6 +12492,8 @@ patch_call_imm: · 223 unmodified lines
12178 }12492 }
12179 }12493 }
12180 12494
12495 sort_kfunc_descs_by_imm(env->prog);
12496
12181 return 0;12497 return 0;
12182}12498}
12183 12499
@@ -12883,6 +13199,10 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, · 699 unmodified lines
12883 if (!env->explored_states)13199 if (!env->explored_states)
12884 goto skip_full_check;13200 goto skip_full_check;
12885 13201
13202 ret = add_subprog_and_kfunc(env);
13203 if (ret < 0)
13204 goto skip_full_check;
13205
12886 ret = check_subprogs(env);13206 ret = check_subprogs(env);
12887 if (ret < 0)13207 if (ret < 0)
12888 goto skip_full_check;13208 goto skip_full_check;
tools/include/uapi/linux/bpf.h+4−0
@@ -1117,6 +1117,10 @@ enum bpf_link_type { · 1116 unmodified lines
1117 * offset to another bpf function1117 * offset to another bpf function
1118 */1118 */
1119#define BPF_PSEUDO_CALL 11119#define BPF_PSEUDO_CALL 1
1120/* when bpf_call->src_reg == BPF_PSEUDO_KFUNC_CALL,
1121 * bpf_call->imm == btf_id of a BTF_KIND_FUNC in the running kernel
1122 */
1123#define BPF_PSEUDO_KFUNC_CALL 2
1120 1124
1121/* flags for BPF_MAP_UPDATE_ELEM command */1125/* flags for BPF_MAP_UPDATE_ELEM command */
1122enum {1126enum {
tools/testing/selftests/bpf/verifier/calls.c+6−6
@@ -19,7 +19,7 @@ · 18 unmodified lines
19 BPF_MOV64_IMM(BPF_REG_0, 2),19 BPF_MOV64_IMM(BPF_REG_0, 2),
20 BPF_EXIT_INSN(),20 BPF_EXIT_INSN(),
21 },21 },
22 .errstr_unpriv = "function calls to other bpf functions are allowed for",22 .errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
23 .result_unpriv = REJECT,23 .result_unpriv = REJECT,
24 .result = ACCEPT,24 .result = ACCEPT,
25 .retval = 1,25 .retval = 1,
@@ -136,7 +136,7 @@ · 110 unmodified lines
136{136{
137 "calls: wrong src reg",137 "calls: wrong src reg",
138 .insns = {138 .insns = {
139 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 2, 0, 0),139 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 3, 0, 0),
140 BPF_MOV64_IMM(BPF_REG_0, 1),140 BPF_MOV64_IMM(BPF_REG_0, 1),
141 BPF_EXIT_INSN(),141 BPF_EXIT_INSN(),
142 },142 },
@@ -397,7 +397,7 @@ · 254 unmodified lines
397 BPF_MOV64_IMM(BPF_REG_0, 1),397 BPF_MOV64_IMM(BPF_REG_0, 1),
398 BPF_EXIT_INSN(),398 BPF_EXIT_INSN(),
399 },399 },
400 .errstr_unpriv = "function calls to other bpf functions are allowed for",400 .errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
401 .fixup_map_hash_48b = { 3 },401 .fixup_map_hash_48b = { 3 },
402 .result_unpriv = REJECT,402 .result_unpriv = REJECT,
403 .result = ACCEPT,403 .result = ACCEPT,
@@ -1977,7 +1977,7 @@ · 1573 unmodified lines
1977 BPF_EXIT_INSN(),1977 BPF_EXIT_INSN(),
1978 },1978 },
1979 .prog_type = BPF_PROG_TYPE_SOCKET_FILTER,1979 .prog_type = BPF_PROG_TYPE_SOCKET_FILTER,
1980 .errstr_unpriv = "function calls to other bpf functions are allowed for",1980 .errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
1981 .result_unpriv = REJECT,1981 .result_unpriv = REJECT,
1982 .result = ACCEPT,1982 .result = ACCEPT,
1983},1983},
@@ -2003,7 +2003,7 @@ · 19 unmodified lines
2003 BPF_EXIT_INSN(),2003 BPF_EXIT_INSN(),
2004 },2004 },
2005 .prog_type = BPF_PROG_TYPE_SOCKET_FILTER,2005 .prog_type = BPF_PROG_TYPE_SOCKET_FILTER,
2006 .errstr_unpriv = "function calls to other bpf functions are allowed for",2006 .errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
2007 .errstr = "!read_ok",2007 .errstr = "!read_ok",
2008 .result = REJECT,2008 .result = REJECT,
2009},2009},
@@ -2028,7 +2028,7 @@ · 18 unmodified lines
2028 BPF_EXIT_INSN(),2028 BPF_EXIT_INSN(),
2029 },2029 },
2030 .prog_type = BPF_PROG_TYPE_SOCKET_FILTER,2030 .prog_type = BPF_PROG_TYPE_SOCKET_FILTER,
2031 .errstr_unpriv = "function calls to other bpf functions are allowed for",2031 .errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
2032 .errstr = "!read_ok",2032 .errstr = "!read_ok",
2033 .result = REJECT,2033 .result = REJECT,
2034},2034},
tools/testing/selftests/bpf/verifier/dead_code.c+5−5
@@ -85,7 +85,7 @@ · 84 unmodified lines
85 BPF_MOV64_IMM(BPF_REG_0, 12),85 BPF_MOV64_IMM(BPF_REG_0, 12),
86 BPF_EXIT_INSN(),86 BPF_EXIT_INSN(),
87 },87 },
88 .errstr_unpriv = "function calls to other bpf functions are allowed for",88 .errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
89 .result_unpriv = REJECT,89 .result_unpriv = REJECT,
90 .result = ACCEPT,90 .result = ACCEPT,
91 .retval = 7,91 .retval = 7,
@@ -103,7 +103,7 @@ · 11 unmodified lines
103 BPF_MOV64_IMM(BPF_REG_0, 12),103 BPF_MOV64_IMM(BPF_REG_0, 12),
104 BPF_EXIT_INSN(),104 BPF_EXIT_INSN(),
105 },105 },
106 .errstr_unpriv = "function calls to other bpf functions are allowed for",106 .errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
107 .result_unpriv = REJECT,107 .result_unpriv = REJECT,
108 .result = ACCEPT,108 .result = ACCEPT,
109 .retval = 7,109 .retval = 7,
@@ -121,7 +121,7 @@ · 11 unmodified lines
121 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, -5),121 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, -5),
122 BPF_EXIT_INSN(),122 BPF_EXIT_INSN(),
123 },123 },
124 .errstr_unpriv = "function calls to other bpf functions are allowed for",124 .errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
125 .result_unpriv = REJECT,125 .result_unpriv = REJECT,
126 .result = ACCEPT,126 .result = ACCEPT,
127 .retval = 7,127 .retval = 7,
@@ -137,7 +137,7 @@ · 9 unmodified lines
137 BPF_MOV64_REG(BPF_REG_0, BPF_REG_1),137 BPF_MOV64_REG(BPF_REG_0, BPF_REG_1),
138 BPF_EXIT_INSN(),138 BPF_EXIT_INSN(),
139 },139 },
140 .errstr_unpriv = "function calls to other bpf functions are allowed for",140 .errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
141 .result_unpriv = REJECT,141 .result_unpriv = REJECT,
142 .result = ACCEPT,142 .result = ACCEPT,
143 .retval = 2,143 .retval = 2,
@@ -152,7 +152,7 @@ · 8 unmodified lines
152 BPF_MOV64_REG(BPF_REG_0, BPF_REG_1),152 BPF_MOV64_REG(BPF_REG_0, BPF_REG_1),
153 BPF_EXIT_INSN(),153 BPF_EXIT_INSN(),
154 },154 },
155 .errstr_unpriv = "function calls to other bpf functions are allowed for",155 .errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
156 .result_unpriv = REJECT,156 .result_unpriv = REJECT,
157 .result = ACCEPT,157 .result = ACCEPT,
158 .retval = 2,158 .retval = 2,