Pass the fd of a btf from the userspace to the bpf() syscall, and then convert the fd into a btf. The btf is generated from the module that defines the target BPF struct_ops type. In order to inform the kernel about the module that defines the target struct_ops type, the userspace program needs to provide a btf fd for the respective module's btf. This btf contains essential information on the types defined within the module, including the target struct_ops type. A btf fd must be provided to the kernel for struct_ops maps and for the bpf programs attached to those maps. In the case of the bpf programs, the attach_btf_obj_fd parameter is passed as part of the bpf_attr and is converted into a btf. This btf is then stored in the prog->aux->attach_btf field. Here, it just let the verifier access attach_btf directly. In the case of struct_ops maps, a btf fd is passed as value_type_btf_obj_fd of bpf_attr. The bpf_struct_ops_map_alloc() function converts the fd to a btf and stores it as st_map->btf. A flag BPF_F_VTYPE_BTF_OBJ_FD is added for map_flags to indicate that the value of value_type_btf_obj_fd is set.
Unified · Split
| @@ -1330,6 +1330,9 @@ enum { · 1329 unmodified lines | |||
| 1330 | 1330 | ||
| 1331 | /* Get path from provided FD in BPF_OBJ_PIN/BPF_OBJ_GET commands */ | 1331 | /* Get path from provided FD in BPF_OBJ_PIN/BPF_OBJ_GET commands */ |
| 1332 | BPF_F_PATH_FD = (1U << 14), | 1332 | BPF_F_PATH_FD = (1U << 14), |
| 1333 | |||
| 1334 | /* Flag for value_type_btf_obj_fd, the fd is available */ | ||
| 1335 | BPF_F_VTYPE_BTF_OBJ_FD = (1U << 15), | ||
| 1333 | }; | 1336 | }; |
| 1334 | 1337 | ||
| 1335 | /* Flags for BPF_PROG_QUERY. */ | 1338 | /* Flags for BPF_PROG_QUERY. */ |
| @@ -1403,6 +1406,11 @@ union bpf_attr { · 67 unmodified lines | |||
| 1403 | * to using 5 hash functions). | 1406 | * to using 5 hash functions). |
| 1404 | */ | 1407 | */ |
| 1405 | __u64 map_extra; | 1408 | __u64 map_extra; |
| 1409 | |||
| 1410 | __s32 value_type_btf_obj_fd; /* fd pointing to a BTF | ||
| 1411 | * type data for | ||
| 1412 | * btf_vmlinux_value_type_id. | ||
| 1413 | */ | ||
| 1406 | }; | 1414 | }; |
| 1407 | 1415 | ||
| 1408 | struct { /* anonymous struct used by BPF_MAP_*_ELEM commands */ | 1416 | struct { /* anonymous struct used by BPF_MAP_*_ELEM commands */ |
| @@ -641,6 +641,7 @@ static void __bpf_struct_ops_map_free(struct bpf_map *map) · 640 unmodified lines | |||
| 641 | bpf_jit_uncharge_modmem(PAGE_SIZE); | 641 | bpf_jit_uncharge_modmem(PAGE_SIZE); |
| 642 | } | 642 | } |
| 643 | bpf_map_area_free(st_map->uvalue); | 643 | bpf_map_area_free(st_map->uvalue); |
| 644 | btf_put(st_map->btf); | ||
| 644 | bpf_map_area_free(st_map); | 645 | bpf_map_area_free(st_map); |
| 645 | } | 646 | } |
| 646 | 647 | ||
| @@ -669,7 +670,8 @@ static void bpf_struct_ops_map_free(struct bpf_map *map) · 22 unmodified lines | |||
| 669 | static int bpf_struct_ops_map_alloc_check(union bpf_attr *attr) | 670 | static int bpf_struct_ops_map_alloc_check(union bpf_attr *attr) |
| 670 | { | 671 | { |
| 671 | if (attr->key_size != sizeof(unsigned int) || attr->max_entries != 1 || | 672 | if (attr->key_size != sizeof(unsigned int) || attr->max_entries != 1 || |
| 672 | (attr->map_flags & ~BPF_F_LINK) || !attr->btf_vmlinux_value_type_id) | 673 | (attr->map_flags & ~(BPF_F_LINK | BPF_F_VTYPE_BTF_OBJ_FD)) || |
| 674 | !attr->btf_vmlinux_value_type_id) | ||
| 673 | return -EINVAL; | 675 | return -EINVAL; |
| 674 | return 0; | 676 | return 0; |
| 675 | } | 677 | } |
| @@ -681,15 +683,36 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) · 5 unmodified lines | |||
| 681 | struct bpf_struct_ops_map *st_map; | 683 | struct bpf_struct_ops_map *st_map; |
| 682 | const struct btf_type *t, *vt; | 684 | const struct btf_type *t, *vt; |
| 683 | struct bpf_map *map; | 685 | struct bpf_map *map; |
| 686 | struct btf *btf; | ||
| 684 | int ret; | 687 | int ret; |
| 685 | 688 | ||
| 686 | st_ops_desc = bpf_struct_ops_find_value(btf_vmlinux, attr->btf_vmlinux_value_type_id); | 689 | if (attr->map_flags & BPF_F_VTYPE_BTF_OBJ_FD) { |
| 687 | if (!st_ops_desc) | 690 | /* The map holds btf for its whole life time. */ |
| 688 | return ERR_PTR(-ENOTSUPP); | 691 | btf = btf_get_by_fd(attr->value_type_btf_obj_fd); |
| 692 | if (IS_ERR(btf)) | ||
| 693 | return ERR_CAST(btf); | ||
| 694 | if (!btf_is_module(btf)) { | ||
| 695 | btf_put(btf); | ||
| 696 | return ERR_PTR(-EINVAL); | ||
| 697 | } | ||
| 698 | } else { | ||
| 699 | btf = bpf_get_btf_vmlinux(); | ||
| 700 | if (IS_ERR(btf)) | ||
| 701 | return ERR_CAST(btf); | ||
| 702 | btf_get(btf); | ||
| 703 | } | ||
| 704 | |||
| 705 | st_ops_desc = bpf_struct_ops_find_value(btf, attr->btf_vmlinux_value_type_id); | ||
| 706 | if (!st_ops_desc) { | ||
| 707 | ret = -ENOTSUPP; | ||
| 708 | goto errout; | ||
| 709 | } | ||
| 689 | 710 | ||
| 690 | vt = st_ops_desc->value_type; | 711 | vt = st_ops_desc->value_type; |
| 691 | if (attr->value_size != vt->size) | 712 | if (attr->value_size != vt->size) { |
| 692 | return ERR_PTR(-EINVAL); | 713 | ret = -EINVAL; |
| 714 | goto errout; | ||
| 715 | } | ||
| 693 | 716 | ||
| 694 | t = st_ops_desc->type; | 717 | t = st_ops_desc->type; |
| 695 | 718 | ||
| @@ -700,17 +723,17 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) · 4 unmodified lines | |||
| 700 | (vt->size - sizeof(struct bpf_struct_ops_value)); | 723 | (vt->size - sizeof(struct bpf_struct_ops_value)); |
| 701 | 724 | ||
| 702 | st_map = bpf_map_area_alloc(st_map_size, NUMA_NO_NODE); | 725 | st_map = bpf_map_area_alloc(st_map_size, NUMA_NO_NODE); |
| 703 | if (!st_map) | 726 | if (!st_map) { |
| 704 | return ERR_PTR(-ENOMEM); | 727 | ret = -ENOMEM; |
| 728 | goto errout; | ||
| 729 | } | ||
| 705 | 730 | ||
| 706 | st_map->st_ops_desc = st_ops_desc; | 731 | st_map->st_ops_desc = st_ops_desc; |
| 707 | map = &st_map->map; | 732 | map = &st_map->map; |
| 708 | 733 | ||
| 709 | ret = bpf_jit_charge_modmem(PAGE_SIZE); | 734 | ret = bpf_jit_charge_modmem(PAGE_SIZE); |
| 710 | if (ret) { | 735 | if (ret) |
| 711 | __bpf_struct_ops_map_free(map); | 736 | goto errout_free; |
| 712 | return ERR_PTR(ret); | ||
| 713 | } | ||
| 714 | 737 | ||
| 715 | st_map->image = arch_alloc_bpf_trampoline(PAGE_SIZE); | 738 | st_map->image = arch_alloc_bpf_trampoline(PAGE_SIZE); |
| 716 | if (!st_map->image) { | 739 | if (!st_map->image) { |
| @@ -719,24 +742,30 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) · 2 unmodified lines | |||
| 719 | * here. | 742 | * here. |
| 720 | */ | 743 | */ |
| 721 | bpf_jit_uncharge_modmem(PAGE_SIZE); | 744 | bpf_jit_uncharge_modmem(PAGE_SIZE); |
| 722 | __bpf_struct_ops_map_free(map); | 745 | ret = -ENOMEM; |
| 723 | return ERR_PTR(-ENOMEM); | 746 | goto errout_free; |
| 724 | } | 747 | } |
| 725 | st_map->uvalue = bpf_map_area_alloc(vt->size, NUMA_NO_NODE); | 748 | st_map->uvalue = bpf_map_area_alloc(vt->size, NUMA_NO_NODE); |
| 726 | st_map->links = | 749 | st_map->links = |
| 727 | bpf_map_area_alloc(btf_type_vlen(t) * sizeof(struct bpf_links *), | 750 | bpf_map_area_alloc(btf_type_vlen(t) * sizeof(struct bpf_links *), |
| 728 | NUMA_NO_NODE); | 751 | NUMA_NO_NODE); |
| 729 | if (!st_map->uvalue || !st_map->links) { | 752 | if (!st_map->uvalue || !st_map->links) { |
| 730 | __bpf_struct_ops_map_free(map); | 753 | ret = -ENOMEM; |
| 731 | return ERR_PTR(-ENOMEM); | 754 | goto errout_free; |
| 732 | } | 755 | } |
| 733 | 756 | st_map->btf = btf; | |
| 734 | st_map->btf = btf_vmlinux; | ||
| 735 | 757 | ||
| 736 | mutex_init(&st_map->lock); | 758 | mutex_init(&st_map->lock); |
| 737 | bpf_map_init_from_attr(map, attr); | 759 | bpf_map_init_from_attr(map, attr); |
| 738 | 760 | ||
| 739 | return map; | 761 | return map; |
| 762 | |||
| 763 | errout_free: | ||
| 764 | __bpf_struct_ops_map_free(map); | ||
| 765 | errout: | ||
| 766 | btf_put(btf); | ||
| 767 | |||
| 768 | return ERR_PTR(ret); | ||
| 740 | } | 769 | } |
| 741 | 770 | ||
| 742 | static u64 bpf_struct_ops_map_mem_usage(const struct bpf_map *map) | 771 | static u64 bpf_struct_ops_map_mem_usage(const struct bpf_map *map) |
| @@ -1123,7 +1123,7 @@ free_map_tab: · 1122 unmodified lines | |||
| 1123 | return ret; | 1123 | return ret; |
| 1124 | } | 1124 | } |
| 1125 | 1125 | ||
| 1126 | #define BPF_MAP_CREATE_LAST_FIELD map_extra | 1126 | #define BPF_MAP_CREATE_LAST_FIELD value_type_btf_obj_fd |
| 1127 | /* called via syscall */ | 1127 | /* called via syscall */ |
| 1128 | static int map_create(union bpf_attr *attr) | 1128 | static int map_create(union bpf_attr *attr) |
| 1129 | { | 1129 | { |
| @@ -20290,6 +20290,7 @@ static int check_struct_ops_btf_id(struct bpf_verifier_env *env) · 20289 unmodified lines | |||
| 20290 | const struct btf_member *member; | 20290 | const struct btf_member *member; |
| 20291 | struct bpf_prog *prog = env->prog; | 20291 | struct bpf_prog *prog = env->prog; |
| 20292 | u32 btf_id, member_idx; | 20292 | u32 btf_id, member_idx; |
| 20293 | struct btf *btf; | ||
| 20293 | const char *mname; | 20294 | const char *mname; |
| 20294 | 20295 | ||
| 20295 | if (!prog->gpl_compatible) { | 20296 | if (!prog->gpl_compatible) { |
| @@ -20297,8 +20298,10 @@ static int check_struct_ops_btf_id(struct bpf_verifier_env *env) · 1 unmodified line | |||
| 20297 | return -EINVAL; | 20298 | return -EINVAL; |
| 20298 | } | 20299 | } |
| 20299 | 20300 | ||
| 20301 | btf = prog->aux->attach_btf ?: bpf_get_btf_vmlinux(); | ||
| 20302 | |||
| 20300 | btf_id = prog->aux->attach_btf_id; | 20303 | btf_id = prog->aux->attach_btf_id; |
| 20301 | st_ops_desc = bpf_struct_ops_find(btf_vmlinux, btf_id); | 20304 | st_ops_desc = bpf_struct_ops_find(btf, btf_id); |
| 20302 | if (!st_ops_desc) { | 20305 | if (!st_ops_desc) { |
| 20303 | verbose(env, "attach_btf_id %u is not a supported struct\n", | 20306 | verbose(env, "attach_btf_id %u is not a supported struct\n", |
| 20304 | btf_id); | 20307 | btf_id); |
| @@ -20315,8 +20318,8 @@ static int check_struct_ops_btf_id(struct bpf_verifier_env *env) · 10 unmodified lines | |||
| 20315 | } | 20318 | } |
| 20316 | 20319 | ||
| 20317 | member = &btf_type_member(t)[member_idx]; | 20320 | member = &btf_type_member(t)[member_idx]; |
| 20318 | mname = btf_name_by_offset(btf_vmlinux, member->name_off); | 20321 | mname = btf_name_by_offset(btf, member->name_off); |
| 20319 | func_proto = btf_type_resolve_func_ptr(btf_vmlinux, member->type, | 20322 | func_proto = btf_type_resolve_func_ptr(btf, member->type, |
| 20320 | NULL); | 20323 | NULL); |
| 20321 | if (!func_proto) { | 20324 | if (!func_proto) { |
| 20322 | verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n", | 20325 | verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n", |
| @@ -1330,6 +1330,9 @@ enum { · 1329 unmodified lines | |||
| 1330 | 1330 | ||
| 1331 | /* Get path from provided FD in BPF_OBJ_PIN/BPF_OBJ_GET commands */ | 1331 | /* Get path from provided FD in BPF_OBJ_PIN/BPF_OBJ_GET commands */ |
| 1332 | BPF_F_PATH_FD = (1U << 14), | 1332 | BPF_F_PATH_FD = (1U << 14), |
| 1333 | |||
| 1334 | /* Flag for value_type_btf_obj_fd, the fd is available */ | ||
| 1335 | BPF_F_VTYPE_BTF_OBJ_FD = (1U << 15), | ||
| 1333 | }; | 1336 | }; |
| 1334 | 1337 | ||
| 1335 | /* Flags for BPF_PROG_QUERY. */ | 1338 | /* Flags for BPF_PROG_QUERY. */ |
| @@ -1403,6 +1406,11 @@ union bpf_attr { · 67 unmodified lines | |||
| 1403 | * to using 5 hash functions). | 1406 | * to using 5 hash functions). |
| 1404 | */ | 1407 | */ |
| 1405 | __u64 map_extra; | 1408 | __u64 map_extra; |
| 1409 | |||
| 1410 | __s32 value_type_btf_obj_fd; /* fd pointing to a BTF | ||
| 1411 | * type data for | ||
| 1412 | * btf_vmlinux_value_type_id. | ||
| 1413 | */ | ||
| 1406 | }; | 1414 | }; |
| 1407 | 1415 | ||
| 1408 | struct { /* anonymous struct used by BPF_MAP_*_ELEM commands */ | 1416 | struct { /* anonymous struct used by BPF_MAP_*_ELEM commands */ |