Pseudo_btf_id is a type of ld_imm insn that associates a btf_id to a ksym so that further dereferences on the ksym can use the BTF info to validate accesses. Internally, when seeing a pseudo_btf_id ld insn, the verifier reads the btf_id stored in the insn[0]'s imm field and marks the dst_reg as PTR_TO_BTF_ID. The btf_id points to a VAR_KIND, which is encoded in btf_vminux by pahole. If the VAR is not of a struct type, the dst reg will be marked as PTR_TO_MEM instead of PTR_TO_BTF_ID and the mem_size is resolved to the size of the VAR's type. >From the VAR btf_id, the verifier can also read the address of the ksym's corresponding kernel var from kallsyms and use that to fill dst_reg. Therefore, the proper functionality of pseudo_btf_id depends on (1) kallsyms and (2) the encoding of kernel global VARs in pahole, which should be available since pahole v1.18.
Showing the changes to kernel (2 of 6 files). Show all
Unified · Split
| @@ -440,16 +440,6 @@ static bool btf_type_nosize_or_null(const struct btf_type *t) · 439 unmodified lines | |||
| 440 | return !t || btf_type_nosize(t); | 440 | return !t || btf_type_nosize(t); |
| 441 | } | 441 | } |
| 442 | 442 | ||
| 443 | /* union is only a special case of struct: | ||
| 444 | * all its offsetof(member) == 0 | ||
| 445 | */ | ||
| 446 | static bool btf_type_is_struct(const struct btf_type *t) | ||
| 447 | { | ||
| 448 | u8 kind = BTF_INFO_KIND(t->info); | ||
| 449 | |||
| 450 | return kind == BTF_KIND_STRUCT || kind == BTF_KIND_UNION; | ||
| 451 | } | ||
| 452 | |||
| 453 | static bool __btf_type_is_struct(const struct btf_type *t) | 443 | static bool __btf_type_is_struct(const struct btf_type *t) |
| 454 | { | 444 | { |
| 455 | return BTF_INFO_KIND(t->info) == BTF_KIND_STRUCT; | 445 | return BTF_INFO_KIND(t->info) == BTF_KIND_STRUCT; |
| @@ -460,11 +450,6 @@ static bool btf_type_is_array(const struct btf_type *t) · 4 unmodified lines | |||
| 460 | return BTF_INFO_KIND(t->info) == BTF_KIND_ARRAY; | 450 | return BTF_INFO_KIND(t->info) == BTF_KIND_ARRAY; |
| 461 | } | 451 | } |
| 462 | 452 | ||
| 463 | static bool btf_type_is_var(const struct btf_type *t) | ||
| 464 | { | ||
| 465 | return BTF_INFO_KIND(t->info) == BTF_KIND_VAR; | ||
| 466 | } | ||
| 467 | |||
| 468 | static bool btf_type_is_datasec(const struct btf_type *t) | 453 | static bool btf_type_is_datasec(const struct btf_type *t) |
| 469 | { | 454 | { |
| 470 | return BTF_INFO_KIND(t->info) == BTF_KIND_DATASEC; | 455 | return BTF_INFO_KIND(t->info) == BTF_KIND_DATASEC; |
| @@ -7488,6 +7488,7 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) · 7487 unmodified lines | |||
| 7488 | { | 7488 | { |
| 7489 | struct bpf_insn_aux_data *aux = cur_aux(env); | 7489 | struct bpf_insn_aux_data *aux = cur_aux(env); |
| 7490 | struct bpf_reg_state *regs = cur_regs(env); | 7490 | struct bpf_reg_state *regs = cur_regs(env); |
| 7491 | struct bpf_reg_state *dst_reg; | ||
| 7491 | struct bpf_map *map; | 7492 | struct bpf_map *map; |
| 7492 | int err; | 7493 | int err; |
| 7493 | 7494 | ||
| @@ -7504,25 +7505,44 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) · 10 unmodified lines | |||
| 7504 | if (err) | 7505 | if (err) |
| 7505 | return err; | 7506 | return err; |
| 7506 | 7507 | ||
| 7508 | dst_reg = ®s[insn->dst_reg]; | ||
| 7507 | if (insn->src_reg == 0) { | 7509 | if (insn->src_reg == 0) { |
| 7508 | u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; | 7510 | u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; |
| 7509 | 7511 | ||
| 7510 | regs[insn->dst_reg].type = SCALAR_VALUE; | 7512 | dst_reg->type = SCALAR_VALUE; |
| 7511 | __mark_reg_known(®s[insn->dst_reg], imm); | 7513 | __mark_reg_known(®s[insn->dst_reg], imm); |
| 7512 | return 0; | 7514 | return 0; |
| 7513 | } | 7515 | } |
| 7514 | 7516 | ||
| 7517 | if (insn->src_reg == BPF_PSEUDO_BTF_ID) { | ||
| 7518 | mark_reg_known_zero(env, regs, insn->dst_reg); | ||
| 7519 | |||
| 7520 | dst_reg->type = aux->btf_var.reg_type; | ||
| 7521 | switch (dst_reg->type) { | ||
| 7522 | case PTR_TO_MEM: | ||
| 7523 | dst_reg->mem_size = aux->btf_var.mem_size; | ||
| 7524 | break; | ||
| 7525 | case PTR_TO_BTF_ID: | ||
| 7526 | dst_reg->btf_id = aux->btf_var.btf_id; | ||
| 7527 | break; | ||
| 7528 | default: | ||
| 7529 | verbose(env, "bpf verifier is misconfigured\n"); | ||
| 7530 | return -EFAULT; | ||
| 7531 | } | ||
| 7532 | return 0; | ||
| 7533 | } | ||
| 7534 | |||
| 7515 | map = env->used_maps[aux->map_index]; | 7535 | map = env->used_maps[aux->map_index]; |
| 7516 | mark_reg_known_zero(env, regs, insn->dst_reg); | 7536 | mark_reg_known_zero(env, regs, insn->dst_reg); |
| 7517 | regs[insn->dst_reg].map_ptr = map; | 7537 | dst_reg->map_ptr = map; |
| 7518 | 7538 | ||
| 7519 | if (insn->src_reg == BPF_PSEUDO_MAP_VALUE) { | 7539 | if (insn->src_reg == BPF_PSEUDO_MAP_VALUE) { |
| 7520 | regs[insn->dst_reg].type = PTR_TO_MAP_VALUE; | 7540 | dst_reg->type = PTR_TO_MAP_VALUE; |
| 7521 | regs[insn->dst_reg].off = aux->map_off; | 7541 | dst_reg->off = aux->map_off; |
| 7522 | if (map_value_has_spin_lock(map)) | 7542 | if (map_value_has_spin_lock(map)) |
| 7523 | regs[insn->dst_reg].id = ++env->id_gen; | 7543 | dst_reg->id = ++env->id_gen; |
| 7524 | } else if (insn->src_reg == BPF_PSEUDO_MAP_FD) { | 7544 | } else if (insn->src_reg == BPF_PSEUDO_MAP_FD) { |
| 7525 | regs[insn->dst_reg].type = CONST_PTR_TO_MAP; | 7545 | dst_reg->type = CONST_PTR_TO_MAP; |
| 7526 | } else { | 7546 | } else { |
| 7527 | verbose(env, "bpf verifier is misconfigured\n"); | 7547 | verbose(env, "bpf verifier is misconfigured\n"); |
| 7528 | return -EINVAL; | 7548 | return -EINVAL; |
| @@ -9424,6 +9444,73 @@ process_bpf_exit: · 1895 unmodified lines | |||
| 9424 | return 0; | 9444 | return 0; |
| 9425 | } | 9445 | } |
| 9426 | 9446 | ||
| 9447 | /* replace pseudo btf_id with kernel symbol address */ | ||
| 9448 | static int check_pseudo_btf_id(struct bpf_verifier_env *env, | ||
| 9449 | struct bpf_insn *insn, | ||
| 9450 | struct bpf_insn_aux_data *aux) | ||
| 9451 | { | ||
| 9452 | u32 type, id = insn->imm; | ||
| 9453 | const struct btf_type *t; | ||
| 9454 | const char *sym_name; | ||
| 9455 | u64 addr; | ||
| 9456 | |||
| 9457 | if (!btf_vmlinux) { | ||
| 9458 | verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n"); | ||
| 9459 | return -EINVAL; | ||
| 9460 | } | ||
| 9461 | |||
| 9462 | if (insn[1].imm != 0) { | ||
| 9463 | verbose(env, "reserved field (insn[1].imm) is used in pseudo_btf_id ldimm64 insn.\n"); | ||
| 9464 | return -EINVAL; | ||
| 9465 | } | ||
| 9466 | |||
| 9467 | t = btf_type_by_id(btf_vmlinux, id); | ||
| 9468 | if (!t) { | ||
| 9469 | verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id); | ||
| 9470 | return -ENOENT; | ||
| 9471 | } | ||
| 9472 | |||
| 9473 | if (!btf_type_is_var(t)) { | ||
| 9474 | verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR.\n", | ||
| 9475 | id); | ||
| 9476 | return -EINVAL; | ||
| 9477 | } | ||
| 9478 | |||
| 9479 | sym_name = btf_name_by_offset(btf_vmlinux, t->name_off); | ||
| 9480 | addr = kallsyms_lookup_name(sym_name); | ||
| 9481 | if (!addr) { | ||
| 9482 | verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n", | ||
| 9483 | sym_name); | ||
| 9484 | return -ENOENT; | ||
| 9485 | } | ||
| 9486 | |||
| 9487 | insn[0].imm = (u32)addr; | ||
| 9488 | insn[1].imm = addr >> 32; | ||
| 9489 | |||
| 9490 | type = t->type; | ||
| 9491 | t = btf_type_skip_modifiers(btf_vmlinux, type, NULL); | ||
| 9492 | if (!btf_type_is_struct(t)) { | ||
| 9493 | const struct btf_type *ret; | ||
| 9494 | const char *tname; | ||
| 9495 | u32 tsize; | ||
| 9496 | |||
| 9497 | /* resolve the type size of ksym. */ | ||
| 9498 | ret = btf_resolve_size(btf_vmlinux, t, &tsize); | ||
| 9499 | if (IS_ERR(ret)) { | ||
| 9500 | tname = btf_name_by_offset(btf_vmlinux, t->name_off); | ||
| 9501 | verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n", | ||
| 9502 | tname, PTR_ERR(ret)); | ||
| 9503 | return -EINVAL; | ||
| 9504 | } | ||
| 9505 | aux->btf_var.reg_type = PTR_TO_MEM; | ||
| 9506 | aux->btf_var.mem_size = tsize; | ||
| 9507 | } else { | ||
| 9508 | aux->btf_var.reg_type = PTR_TO_BTF_ID; | ||
| 9509 | aux->btf_var.btf_id = type; | ||
| 9510 | } | ||
| 9511 | return 0; | ||
| 9512 | } | ||
| 9513 | |||
| 9427 | static int check_map_prealloc(struct bpf_map *map) | 9514 | static int check_map_prealloc(struct bpf_map *map) |
| 9428 | { | 9515 | { |
| 9429 | return (map->map_type != BPF_MAP_TYPE_HASH && | 9516 | return (map->map_type != BPF_MAP_TYPE_HASH && |
| @@ -9534,10 +9621,14 @@ static bool bpf_map_is_cgroup_storage(struct bpf_map *map) · 104 unmodified lines | |||
| 9534 | map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE); | 9621 | map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE); |
| 9535 | } | 9622 | } |
| 9536 | 9623 | ||
| 9537 | /* look for pseudo eBPF instructions that access map FDs and | 9624 | /* find and rewrite pseudo imm in ld_imm64 instructions: |
| 9538 | * replace them with actual map pointers | 9625 | * |
| 9626 | * 1. if it accesses map FD, replace it with actual map pointer. | ||
| 9627 | * 2. if it accesses btf_id of a VAR, replace it with pointer to the var. | ||
| 9628 | * | ||
| 9629 | * NOTE: btf_vmlinux is required for converting pseudo btf_id. | ||
| 9539 | */ | 9630 | */ |
| 9540 | static int replace_map_fd_with_map_ptr(struct bpf_verifier_env *env) | 9631 | static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env) |
| 9541 | { | 9632 | { |
| 9542 | struct bpf_insn *insn = env->prog->insnsi; | 9633 | struct bpf_insn *insn = env->prog->insnsi; |
| 9543 | int insn_cnt = env->prog->len; | 9634 | int insn_cnt = env->prog->len; |
| @@ -9578,6 +9669,14 @@ static int replace_map_fd_with_map_ptr(struct bpf_verifier_env *env) · 34 unmodified lines | |||
| 9578 | /* valid generic load 64-bit imm */ | 9669 | /* valid generic load 64-bit imm */ |
| 9579 | goto next_insn; | 9670 | goto next_insn; |
| 9580 | 9671 | ||
| 9672 | if (insn[0].src_reg == BPF_PSEUDO_BTF_ID) { | ||
| 9673 | aux = &env->insn_aux_data[i]; | ||
| 9674 | err = check_pseudo_btf_id(env, insn, aux); | ||
| 9675 | if (err) | ||
| 9676 | return err; | ||
| 9677 | goto next_insn; | ||
| 9678 | } | ||
| 9679 | |||
| 9581 | /* In final convert_pseudo_ld_imm64() step, this is | 9680 | /* In final convert_pseudo_ld_imm64() step, this is |
| 9582 | * converted into regular 64-bit imm load insn. | 9681 | * converted into regular 64-bit imm load insn. |
| 9583 | */ | 9682 | */ |
| @@ -11633,10 +11732,6 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, · 2049 unmodified lines | |||
| 11633 | if (is_priv) | 11732 | if (is_priv) |
| 11634 | env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; | 11733 | env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; |
| 11635 | 11734 | ||
| 11636 | ret = replace_map_fd_with_map_ptr(env); | ||
| 11637 | if (ret < 0) | ||
| 11638 | goto skip_full_check; | ||
| 11639 | |||
| 11640 | if (bpf_prog_is_dev_bound(env->prog->aux)) { | 11735 | if (bpf_prog_is_dev_bound(env->prog->aux)) { |
| 11641 | ret = bpf_prog_offload_verifier_prep(env->prog); | 11736 | ret = bpf_prog_offload_verifier_prep(env->prog); |
| 11642 | if (ret) | 11737 | if (ret) |
| @@ -11662,6 +11757,10 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, · 19 unmodified lines | |||
| 11662 | if (ret) | 11757 | if (ret) |
| 11663 | goto skip_full_check; | 11758 | goto skip_full_check; |
| 11664 | 11759 | ||
| 11760 | ret = resolve_pseudo_ldimm64(env); | ||
| 11761 | if (ret < 0) | ||
| 11762 | goto skip_full_check; | ||
| 11763 | |||
| 11665 | ret = check_cfg(env); | 11764 | ret = check_cfg(env); |
| 11666 | if (ret < 0) | 11765 | if (ret < 0) |
| 11667 | goto skip_full_check; | 11766 | goto skip_full_check; |