Pseudo_btf_id is a type of ld_imm insn that associates a btf_id to a ksym so that further dereferences on the ksym can use the BTF info to validate accesses. Internally, when seeing a pseudo_btf_id ld insn, the verifier reads the btf_id stored in the insn[0]'s imm field and marks the dst_reg as PTR_TO_BTF_ID. The btf_id points to a VAR_KIND, which is encoded in btf_vminux by pahole. If the VAR is not of a struct type, the dst reg will be marked as PTR_TO_MEM instead of PTR_TO_BTF_ID and the mem_size is resolved to the size of the VAR's type. >From the VAR btf_id, the verifier can also read the address of the ksym's corresponding kernel var from kallsyms and use that to fill dst_reg. Therefore, the proper functionality of pseudo_btf_id depends on (1) kallsyms and (2) the encoding of kernel global VARs in pahole, which should be available since pahole v1.18.
Showing the changes to kernel (2 of 6 files). Show all
Unified · Split
| @@ -440,16 +440,6 @@ static bool btf_type_nosize_or_null(const struct btf_type *t) · 439 unmodified lines | ||
| 440 | 440 | return !t || btf_type_nosize(t); |
| 441 | 441 | } |
| 442 | 442 | |
| 443 | −/* union is only a special case of struct: | |
| 444 | − * all its offsetof(member) == 0 | |
| 445 | − */ | |
| 446 | −static bool btf_type_is_struct(const struct btf_type *t) | |
| 447 | −{ | |
| 448 | − u8 kind = BTF_INFO_KIND(t->info); | |
| 449 | − | |
| 450 | − return kind == BTF_KIND_STRUCT || kind == BTF_KIND_UNION; | |
| 451 | −} | |
| 452 | − | |
| 453 | 443 | static bool __btf_type_is_struct(const struct btf_type *t) |
| 454 | 444 | { |
| 455 | 445 | return BTF_INFO_KIND(t->info) == BTF_KIND_STRUCT; |
| @@ -460,11 +450,6 @@ static bool btf_type_is_array(const struct btf_type *t) · 4 unmodified lines | ||
| 460 | 450 | return BTF_INFO_KIND(t->info) == BTF_KIND_ARRAY; |
| 461 | 451 | } |
| 462 | 452 | |
| 463 | −static bool btf_type_is_var(const struct btf_type *t) | |
| 464 | −{ | |
| 465 | − return BTF_INFO_KIND(t->info) == BTF_KIND_VAR; | |
| 466 | −} | |
| 467 | − | |
| 468 | 453 | static bool btf_type_is_datasec(const struct btf_type *t) |
| 469 | 454 | { |
| 470 | 455 | return BTF_INFO_KIND(t->info) == BTF_KIND_DATASEC; |
| @@ -7488,6 +7488,7 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) · 7487 unmodified lines | ||
| 7488 | 7488 | { |
| 7489 | 7489 | struct bpf_insn_aux_data *aux = cur_aux(env); |
| 7490 | 7490 | struct bpf_reg_state *regs = cur_regs(env); |
| 7491 | + struct bpf_reg_state *dst_reg; | |
| 7491 | 7492 | struct bpf_map *map; |
| 7492 | 7493 | int err; |
| 7493 | 7494 | |
| @@ -7504,25 +7505,44 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) · 10 unmodified lines | ||
| 7504 | 7505 | if (err) |
| 7505 | 7506 | return err; |
| 7506 | 7507 | |
| 7508 | + dst_reg = ®s[insn->dst_reg]; | |
| 7507 | 7509 | if (insn->src_reg == 0) { |
| 7508 | 7510 | u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; |
| 7509 | 7511 | |
| 7510 | − regs[insn->dst_reg].type = SCALAR_VALUE; | |
| 7512 | + dst_reg->type = SCALAR_VALUE; | |
| 7511 | 7513 | __mark_reg_known(®s[insn->dst_reg], imm); |
| 7512 | 7514 | return 0; |
| 7513 | 7515 | } |
| 7514 | 7516 | |
| 7517 | + if (insn->src_reg == BPF_PSEUDO_BTF_ID) { | |
| 7518 | + mark_reg_known_zero(env, regs, insn->dst_reg); | |
| 7519 | + | |
| 7520 | + dst_reg->type = aux->btf_var.reg_type; | |
| 7521 | + switch (dst_reg->type) { | |
| 7522 | + case PTR_TO_MEM: | |
| 7523 | + dst_reg->mem_size = aux->btf_var.mem_size; | |
| 7524 | + break; | |
| 7525 | + case PTR_TO_BTF_ID: | |
| 7526 | + dst_reg->btf_id = aux->btf_var.btf_id; | |
| 7527 | + break; | |
| 7528 | + default: | |
| 7529 | + verbose(env, "bpf verifier is misconfigured\n"); | |
| 7530 | + return -EFAULT; | |
| 7531 | + } | |
| 7532 | + return 0; | |
| 7533 | + } | |
| 7534 | + | |
| 7515 | 7535 | map = env->used_maps[aux->map_index]; |
| 7516 | 7536 | mark_reg_known_zero(env, regs, insn->dst_reg); |
| 7517 | − regs[insn->dst_reg].map_ptr = map; | |
| 7537 | + dst_reg->map_ptr = map; | |
| 7518 | 7538 | |
| 7519 | 7539 | if (insn->src_reg == BPF_PSEUDO_MAP_VALUE) { |
| 7520 | − regs[insn->dst_reg].type = PTR_TO_MAP_VALUE; | |
| 7521 | − regs[insn->dst_reg].off = aux->map_off; | |
| 7540 | + dst_reg->type = PTR_TO_MAP_VALUE; | |
| 7541 | + dst_reg->off = aux->map_off; | |
| 7522 | 7542 | if (map_value_has_spin_lock(map)) |
| 7523 | − regs[insn->dst_reg].id = ++env->id_gen; | |
| 7543 | + dst_reg->id = ++env->id_gen; | |
| 7524 | 7544 | } else if (insn->src_reg == BPF_PSEUDO_MAP_FD) { |
| 7525 | − regs[insn->dst_reg].type = CONST_PTR_TO_MAP; | |
| 7545 | + dst_reg->type = CONST_PTR_TO_MAP; | |
| 7526 | 7546 | } else { |
| 7527 | 7547 | verbose(env, "bpf verifier is misconfigured\n"); |
| 7528 | 7548 | return -EINVAL; |
| @@ -9424,6 +9444,73 @@ process_bpf_exit: · 1895 unmodified lines | ||
| 9424 | 9444 | return 0; |
| 9425 | 9445 | } |
| 9426 | 9446 | |
| 9447 | +/* replace pseudo btf_id with kernel symbol address */ | |
| 9448 | +static int check_pseudo_btf_id(struct bpf_verifier_env *env, | |
| 9449 | + struct bpf_insn *insn, | |
| 9450 | + struct bpf_insn_aux_data *aux) | |
| 9451 | +{ | |
| 9452 | + u32 type, id = insn->imm; | |
| 9453 | + const struct btf_type *t; | |
| 9454 | + const char *sym_name; | |
| 9455 | + u64 addr; | |
| 9456 | + | |
| 9457 | + if (!btf_vmlinux) { | |
| 9458 | + verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n"); | |
| 9459 | + return -EINVAL; | |
| 9460 | + } | |
| 9461 | + | |
| 9462 | + if (insn[1].imm != 0) { | |
| 9463 | + verbose(env, "reserved field (insn[1].imm) is used in pseudo_btf_id ldimm64 insn.\n"); | |
| 9464 | + return -EINVAL; | |
| 9465 | + } | |
| 9466 | + | |
| 9467 | + t = btf_type_by_id(btf_vmlinux, id); | |
| 9468 | + if (!t) { | |
| 9469 | + verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id); | |
| 9470 | + return -ENOENT; | |
| 9471 | + } | |
| 9472 | + | |
| 9473 | + if (!btf_type_is_var(t)) { | |
| 9474 | + verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR.\n", | |
| 9475 | + id); | |
| 9476 | + return -EINVAL; | |
| 9477 | + } | |
| 9478 | + | |
| 9479 | + sym_name = btf_name_by_offset(btf_vmlinux, t->name_off); | |
| 9480 | + addr = kallsyms_lookup_name(sym_name); | |
| 9481 | + if (!addr) { | |
| 9482 | + verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n", | |
| 9483 | + sym_name); | |
| 9484 | + return -ENOENT; | |
| 9485 | + } | |
| 9486 | + | |
| 9487 | + insn[0].imm = (u32)addr; | |
| 9488 | + insn[1].imm = addr >> 32; | |
| 9489 | + | |
| 9490 | + type = t->type; | |
| 9491 | + t = btf_type_skip_modifiers(btf_vmlinux, type, NULL); | |
| 9492 | + if (!btf_type_is_struct(t)) { | |
| 9493 | + const struct btf_type *ret; | |
| 9494 | + const char *tname; | |
| 9495 | + u32 tsize; | |
| 9496 | + | |
| 9497 | + /* resolve the type size of ksym. */ | |
| 9498 | + ret = btf_resolve_size(btf_vmlinux, t, &tsize); | |
| 9499 | + if (IS_ERR(ret)) { | |
| 9500 | + tname = btf_name_by_offset(btf_vmlinux, t->name_off); | |
| 9501 | + verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n", | |
| 9502 | + tname, PTR_ERR(ret)); | |
| 9503 | + return -EINVAL; | |
| 9504 | + } | |
| 9505 | + aux->btf_var.reg_type = PTR_TO_MEM; | |
| 9506 | + aux->btf_var.mem_size = tsize; | |
| 9507 | + } else { | |
| 9508 | + aux->btf_var.reg_type = PTR_TO_BTF_ID; | |
| 9509 | + aux->btf_var.btf_id = type; | |
| 9510 | + } | |
| 9511 | + return 0; | |
| 9512 | +} | |
| 9513 | + | |
| 9427 | 9514 | static int check_map_prealloc(struct bpf_map *map) |
| 9428 | 9515 | { |
| 9429 | 9516 | return (map->map_type != BPF_MAP_TYPE_HASH && |
| @@ -9534,10 +9621,14 @@ static bool bpf_map_is_cgroup_storage(struct bpf_map *map) · 104 unmodified lines | ||
| 9534 | 9621 | map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE); |
| 9535 | 9622 | } |
| 9536 | 9623 | |
| 9537 | −/* look for pseudo eBPF instructions that access map FDs and | |
| 9538 | − * replace them with actual map pointers | |
| 9624 | +/* find and rewrite pseudo imm in ld_imm64 instructions: | |
| 9625 | + * | |
| 9626 | + * 1. if it accesses map FD, replace it with actual map pointer. | |
| 9627 | + * 2. if it accesses btf_id of a VAR, replace it with pointer to the var. | |
| 9628 | + * | |
| 9629 | + * NOTE: btf_vmlinux is required for converting pseudo btf_id. | |
| 9539 | 9630 | */ |
| 9540 | −static int replace_map_fd_with_map_ptr(struct bpf_verifier_env *env) | |
| 9631 | +static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env) | |
| 9541 | 9632 | { |
| 9542 | 9633 | struct bpf_insn *insn = env->prog->insnsi; |
| 9543 | 9634 | int insn_cnt = env->prog->len; |
| @@ -9578,6 +9669,14 @@ static int replace_map_fd_with_map_ptr(struct bpf_verifier_env *env) · 34 unmodified lines | ||
| 9578 | 9669 | /* valid generic load 64-bit imm */ |
| 9579 | 9670 | goto next_insn; |
| 9580 | 9671 | |
| 9672 | + if (insn[0].src_reg == BPF_PSEUDO_BTF_ID) { | |
| 9673 | + aux = &env->insn_aux_data[i]; | |
| 9674 | + err = check_pseudo_btf_id(env, insn, aux); | |
| 9675 | + if (err) | |
| 9676 | + return err; | |
| 9677 | + goto next_insn; | |
| 9678 | + } | |
| 9679 | + | |
| 9581 | 9680 | /* In final convert_pseudo_ld_imm64() step, this is |
| 9582 | 9681 | * converted into regular 64-bit imm load insn. |
| 9583 | 9682 | */ |
| @@ -11633,10 +11732,6 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, · 2049 unmodified lines | ||
| 11633 | 11732 | if (is_priv) |
| 11634 | 11733 | env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; |
| 11635 | 11734 | |
| 11636 | − ret = replace_map_fd_with_map_ptr(env); | |
| 11637 | − if (ret < 0) | |
| 11638 | − goto skip_full_check; | |
| 11639 | − | |
| 11640 | 11735 | if (bpf_prog_is_dev_bound(env->prog->aux)) { |
| 11641 | 11736 | ret = bpf_prog_offload_verifier_prep(env->prog); |
| 11642 | 11737 | if (ret) |
| @@ -11662,6 +11757,10 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, · 19 unmodified lines | ||
| 11662 | 11757 | if (ret) |
| 11663 | 11758 | goto skip_full_check; |
| 11664 | 11759 | |
| 11760 | + ret = resolve_pseudo_ldimm64(env); | |
| 11761 | + if (ret < 0) | |
| 11762 | + goto skip_full_check; | |
| 11763 | + | |
| 11665 | 11764 | ret = check_cfg(env); |
| 11666 | 11765 | if (ret < 0) |
| 11667 | 11766 | goto skip_full_check; |