Add bpf_reg_type PTR_TO_BTF_ID_OR_NULL support. For tracing/iter program, the bpf program context definition, e.g., for previous bpf_map target, looks like struct bpf_iter__bpf_map { struct bpf_iter_meta *meta; struct bpf_map *map; }; The kernel guarantees that meta is not NULL, but map pointer maybe NULL. The NULL map indicates that all objects have been traversed, so bpf program can take proper action, e.g., do final aggregation and/or send final report to user space. Add btf_id_or_null_non0_off to prog->aux structure, to indicate that if the context access offset is not 0, set to PTR_TO_BTF_ID_OR_NULL instead of PTR_TO_BTF_ID. This bit is set for tracing/iter program.
Showing the changes to kernel (2 of 3 files). Show all
Unified · Split
| @@ -3790,7 +3790,10 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, · 3789 unmodified lines | |||
| 3790 | return true; | 3790 | return true; |
| 3791 | 3791 | ||
| 3792 | /* this is a pointer to another type */ | 3792 | /* this is a pointer to another type */ |
| 3793 | info->reg_type = PTR_TO_BTF_ID; | 3793 | if (off != 0 && prog->aux->btf_id_or_null_non0_off) |
| 3794 | info->reg_type = PTR_TO_BTF_ID_OR_NULL; | ||
| 3795 | else | ||
| 3796 | info->reg_type = PTR_TO_BTF_ID; | ||
| 3794 | 3797 | ||
| 3795 | if (tgt_prog) { | 3798 | if (tgt_prog) { |
| 3796 | ret = btf_translate_to_vmlinux(log, btf, t, tgt_prog->type, arg); | 3799 | ret = btf_translate_to_vmlinux(log, btf, t, tgt_prog->type, arg); |
| @@ -398,7 +398,8 @@ static bool reg_type_may_be_null(enum bpf_reg_type type) · 397 unmodified lines | |||
| 398 | return type == PTR_TO_MAP_VALUE_OR_NULL || | 398 | return type == PTR_TO_MAP_VALUE_OR_NULL || |
| 399 | type == PTR_TO_SOCKET_OR_NULL || | 399 | type == PTR_TO_SOCKET_OR_NULL || |
| 400 | type == PTR_TO_SOCK_COMMON_OR_NULL || | 400 | type == PTR_TO_SOCK_COMMON_OR_NULL || |
| 401 | type == PTR_TO_TCP_SOCK_OR_NULL; | 401 | type == PTR_TO_TCP_SOCK_OR_NULL || |
| 402 | type == PTR_TO_BTF_ID_OR_NULL; | ||
| 402 | } | 403 | } |
| 403 | 404 | ||
| 404 | static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg) | 405 | static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg) |
| @@ -483,6 +484,7 @@ static const char * const reg_type_str[] = { · 78 unmodified lines | |||
| 483 | [PTR_TO_TP_BUFFER] = "tp_buffer", | 484 | [PTR_TO_TP_BUFFER] = "tp_buffer", |
| 484 | [PTR_TO_XDP_SOCK] = "xdp_sock", | 485 | [PTR_TO_XDP_SOCK] = "xdp_sock", |
| 485 | [PTR_TO_BTF_ID] = "ptr_", | 486 | [PTR_TO_BTF_ID] = "ptr_", |
| 487 | [PTR_TO_BTF_ID_OR_NULL] = "ptr_or_null_", | ||
| 486 | }; | 488 | }; |
| 487 | 489 | ||
| 488 | static char slot_type_char[] = { | 490 | static char slot_type_char[] = { |
| @@ -543,7 +545,7 @@ static void print_verifier_state(struct bpf_verifier_env *env, · 54 unmodified lines | |||
| 543 | /* reg->off should be 0 for SCALAR_VALUE */ | 545 | /* reg->off should be 0 for SCALAR_VALUE */ |
| 544 | verbose(env, "%lld", reg->var_off.value + reg->off); | 546 | verbose(env, "%lld", reg->var_off.value + reg->off); |
| 545 | } else { | 547 | } else { |
| 546 | if (t == PTR_TO_BTF_ID) | 548 | if (t == PTR_TO_BTF_ID || t == PTR_TO_BTF_ID_OR_NULL) |
| 547 | verbose(env, "%s", kernel_type_name(reg->btf_id)); | 549 | verbose(env, "%s", kernel_type_name(reg->btf_id)); |
| 548 | verbose(env, "(id=%d", reg->id); | 550 | verbose(env, "(id=%d", reg->id); |
| 549 | if (reg_type_may_be_refcounted_or_null(t)) | 551 | if (reg_type_may_be_refcounted_or_null(t)) |
| @@ -2139,6 +2141,7 @@ static bool is_spillable_regtype(enum bpf_reg_type type) · 1589 unmodified lines | |||
| 2139 | case PTR_TO_TCP_SOCK_OR_NULL: | 2141 | case PTR_TO_TCP_SOCK_OR_NULL: |
| 2140 | case PTR_TO_XDP_SOCK: | 2142 | case PTR_TO_XDP_SOCK: |
| 2141 | case PTR_TO_BTF_ID: | 2143 | case PTR_TO_BTF_ID: |
| 2144 | case PTR_TO_BTF_ID_OR_NULL: | ||
| 2142 | return true; | 2145 | return true; |
| 2143 | default: | 2146 | default: |
| 2144 | return false; | 2147 | return false; |
| @@ -2659,7 +2662,7 @@ static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, · 514 unmodified lines | |||
| 2659 | */ | 2662 | */ |
| 2660 | *reg_type = info.reg_type; | 2663 | *reg_type = info.reg_type; |
| 2661 | 2664 | ||
| 2662 | if (*reg_type == PTR_TO_BTF_ID) | 2665 | if (*reg_type == PTR_TO_BTF_ID || *reg_type == PTR_TO_BTF_ID_OR_NULL) |
| 2663 | *btf_id = info.btf_id; | 2666 | *btf_id = info.btf_id; |
| 2664 | else | 2667 | else |
| 2665 | env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size; | 2668 | env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size; |
| @@ -3243,7 +3246,8 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 577 unmodified lines | |||
| 3243 | * a sub-register. | 3246 | * a sub-register. |
| 3244 | */ | 3247 | */ |
| 3245 | regs[value_regno].subreg_def = DEF_NOT_SUBREG; | 3248 | regs[value_regno].subreg_def = DEF_NOT_SUBREG; |
| 3246 | if (reg_type == PTR_TO_BTF_ID) | 3249 | if (reg_type == PTR_TO_BTF_ID || |
| 3250 | reg_type == PTR_TO_BTF_ID_OR_NULL) | ||
| 3247 | regs[value_regno].btf_id = btf_id; | 3251 | regs[value_regno].btf_id = btf_id; |
| 3248 | } | 3252 | } |
| 3249 | regs[value_regno].type = reg_type; | 3253 | regs[value_regno].type = reg_type; |
| @@ -6572,6 +6576,8 @@ static void mark_ptr_or_null_reg(struct bpf_func_state *state, · 3322 unmodified lines | |||
| 6572 | reg->type = PTR_TO_SOCK_COMMON; | 6576 | reg->type = PTR_TO_SOCK_COMMON; |
| 6573 | } else if (reg->type == PTR_TO_TCP_SOCK_OR_NULL) { | 6577 | } else if (reg->type == PTR_TO_TCP_SOCK_OR_NULL) { |
| 6574 | reg->type = PTR_TO_TCP_SOCK; | 6578 | reg->type = PTR_TO_TCP_SOCK; |
| 6579 | } else if (reg->type == PTR_TO_BTF_ID_OR_NULL) { | ||
| 6580 | reg->type = PTR_TO_BTF_ID; | ||
| 6575 | } | 6581 | } |
| 6576 | if (is_null) { | 6582 | if (is_null) { |
| 6577 | /* We don't need id and ref_obj_id from this point | 6583 | /* We don't need id and ref_obj_id from this point |
| @@ -8429,6 +8435,7 @@ static bool reg_type_mismatch_ok(enum bpf_reg_type type) · 1851 unmodified lines | |||
| 8429 | case PTR_TO_TCP_SOCK_OR_NULL: | 8435 | case PTR_TO_TCP_SOCK_OR_NULL: |
| 8430 | case PTR_TO_XDP_SOCK: | 8436 | case PTR_TO_XDP_SOCK: |
| 8431 | case PTR_TO_BTF_ID: | 8437 | case PTR_TO_BTF_ID: |
| 8438 | case PTR_TO_BTF_ID_OR_NULL: | ||
| 8432 | return false; | 8439 | return false; |
| 8433 | default: | 8440 | default: |
| 8434 | return true; | 8441 | return true; |
| @@ -10640,6 +10647,7 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) · 2205 unmodified lines | |||
| 10640 | prog->aux->attach_func_proto = t; | 10647 | prog->aux->attach_func_proto = t; |
| 10641 | if (!bpf_iter_prog_supported(prog)) | 10648 | if (!bpf_iter_prog_supported(prog)) |
| 10642 | return -EINVAL; | 10649 | return -EINVAL; |
| 10650 | prog->aux->btf_id_or_null_non0_off = true; | ||
| 10643 | ret = btf_distill_func_proto(&env->log, btf, t, | 10651 | ret = btf_distill_func_proto(&env->log, btf, t, |
| 10644 | tname, &fmodel); | 10652 | tname, &fmodel); |
| 10645 | return ret; | 10653 | return ret; |