// SPDX-License-Identifier: GPL-2.0-only/* Copyright (c) 2019 Facebook */ #include <linux/bpf.h>#include <linux/bpf_verifier.h>#include <linux/btf.h>#include <linux/filter.h>#include <linux/slab.h>#include <linux/numa.h>#include <linux/seq_file.h>#include <linux/refcount.h>#include <linux/mutex.h>#include <linux/btf_ids.h>#include <linux/rcupdate_wait.h>#include <linux/poll.h> struct bpf_struct_ops_value { struct bpf_struct_ops_common_value common; char data[] ____cacheline_aligned_in_smp;}; #define MAX_TRAMP_IMAGE_PAGES 8 struct bpf_struct_ops_map { struct bpf_map map; const struct bpf_struct_ops_desc *st_ops_desc; /* protect map_update */ struct mutex lock; /* link has all the bpf_links that is populated * to the func ptr of the kernel's struct * (in kvalue.data). */ struct bpf_link **links; /* ksyms for bpf trampolines */ struct bpf_ksym **ksyms; u32 funcs_cnt; u32 image_pages_cnt; /* image_pages is an array of pages that has all the trampolines * that stores the func args before calling the bpf_prog. */ void *image_pages[MAX_TRAMP_IMAGE_PAGES]; /* The owner moduler's btf. */ struct btf *btf; /* uvalue->data stores the kernel struct * (e.g. tcp_congestion_ops) that is more useful * to userspace than the kvalue. For example, * the bpf_prog's id is stored instead of the kernel * address of a func ptr. */ struct bpf_struct_ops_value *uvalue; /* kvalue.data stores the actual kernel's struct * (e.g. tcp_congestion_ops) that will be * registered to the kernel subsystem. */ struct bpf_struct_ops_value kvalue;}; struct bpf_struct_ops_link { struct bpf_link link; struct bpf_map __rcu *map; wait_queue_head_t wait_hup;}; static DEFINE_MUTEX(update_mutex); #define VALUE_PREFIX "bpf_struct_ops_"#define VALUE_PREFIX_LEN (sizeof(VALUE_PREFIX) - 1) const struct bpf_verifier_ops bpf_struct_ops_verifier_ops = {}; const struct bpf_prog_ops bpf_struct_ops_prog_ops = {#ifdef CONFIG_NET .test_run = bpf_struct_ops_test_run,#endif}; BTF_ID_LIST(st_ops_ids)BTF_ID(struct, module)BTF_ID(struct, bpf_struct_ops_common_value) enum { IDX_MODULE_ID, IDX_ST_OPS_COMMON_VALUE_ID,}; extern struct btf *btf_vmlinux; static bool is_valid_value_type(struct btf *btf, s32 value_id, const struct btf_type *type, const char *value_name){ const struct btf_type *common_value_type; const struct btf_member *member; const struct btf_type *vt, *mt; vt = btf_type_by_id(btf, value_id); if (btf_vlen(vt) != 2) { pr_warn("The number of %s's members should be 2, but we get %d\n", value_name, btf_vlen(vt)); return false; } member = btf_type_member(vt); mt = btf_type_by_id(btf, member->type); common_value_type = btf_type_by_id(btf_vmlinux, st_ops_ids[IDX_ST_OPS_COMMON_VALUE_ID]); if (mt != common_value_type) { pr_warn("The first member of %s should be bpf_struct_ops_common_value\n", value_name); return false; } member++; mt = btf_type_by_id(btf, member->type); if (mt != type) { pr_warn("The second member of %s should be %s\n", value_name, btf_name_by_offset(btf, type->name_off)); return false; } return true;} static void *bpf_struct_ops_image_alloc(void){ void *image; int err; err = bpf_jit_charge_modmem(PAGE_SIZE); if (err) return ERR_PTR(err); image = arch_alloc_bpf_trampoline(PAGE_SIZE); if (!image) { bpf_jit_uncharge_modmem(PAGE_SIZE); return ERR_PTR(-ENOMEM); } return image;} void bpf_struct_ops_image_free(void *image){ if (image) { arch_free_bpf_trampoline(image, PAGE_SIZE); bpf_jit_uncharge_modmem(PAGE_SIZE); }} #define MAYBE_NULL_SUFFIX "__nullable"#define MAX_STUB_NAME 128 /* Return the type info of a stub function, if it exists. * * The name of a stub function is made up of the name of the struct_ops and * the name of the function pointer member, separated by "__". For example, * if the struct_ops type is named "foo_ops" and the function pointer * member is named "bar", the stub function name would be "foo_ops__bar". */static const struct btf_type *find_stub_func_proto(const struct btf *btf, const char *st_op_name, const char *member_name){ char stub_func_name[MAX_STUB_NAME]; const struct btf_type *func_type; s32 btf_id; int cp; cp = snprintf(stub_func_name, MAX_STUB_NAME, "%s__%s", st_op_name, member_name); if (cp >= MAX_STUB_NAME) { pr_warn("Stub function name too long\n"); return NULL; } btf_id = btf_find_by_name_kind(btf, stub_func_name, BTF_KIND_FUNC); if (btf_id < 0) return NULL; func_type = btf_type_by_id(btf, btf_id); if (!func_type) return NULL; return btf_type_by_id(btf, func_type->type); /* FUNC_PROTO */} /* Prepare argument info for every nullable argument of a member of a * struct_ops type. * * Initialize a struct bpf_struct_ops_arg_info according to type info of * the arguments of a stub function. (Check kCFI for more information about * stub functions.) * * Each member in the struct_ops type has a struct bpf_struct_ops_arg_info * to provide an array of struct bpf_ctx_arg_aux, which in turn provides * the information that used by the verifier to check the arguments of the * BPF struct_ops program assigned to the member. Here, we only care about * the arguments that are marked as __nullable. * * The array of struct bpf_ctx_arg_aux is eventually assigned to * prog->aux->ctx_arg_info of BPF struct_ops programs and passed to the * verifier. (See check_struct_ops_btf_id()) * * arg_info->info will be the list of struct bpf_ctx_arg_aux if success. If * fails, it will be kept untouched. */static int prepare_arg_info(struct btf *btf, const char *st_ops_name, const char *member_name, const struct btf_type *func_proto, struct bpf_struct_ops_arg_info *arg_info){ const struct btf_type *stub_func_proto, *pointed_type; const struct btf_param *stub_args, *args; struct bpf_ctx_arg_aux *info, *info_buf; u32 nargs, arg_no, info_cnt = 0; u32 arg_btf_id; int offset; stub_func_proto = find_stub_func_proto(btf, st_ops_name, member_name); if (!stub_func_proto) return 0; /* Check if the number of arguments of the stub function is the same * as the number of arguments of the function pointer. */ nargs = btf_type_vlen(func_proto); if (nargs != btf_type_vlen(stub_func_proto)) { pr_warn("the number of arguments of the stub function %s__%s does not match the number of arguments of the member %s of struct %s\n", st_ops_name, member_name, member_name, st_ops_name); return -EINVAL; } if (!nargs) return 0; args = btf_params(func_proto); stub_args = btf_params(stub_func_proto); info_buf = kcalloc(nargs, sizeof(*info_buf), GFP_KERNEL); if (!info_buf) return -ENOMEM; /* Prepare info for every nullable argument */ info = info_buf; for (arg_no = 0; arg_no < nargs; arg_no++) { /* Skip arguments that is not suffixed with * "__nullable". */ if (!btf_param_match_suffix(btf, &stub_args[arg_no], MAYBE_NULL_SUFFIX)) continue; /* Should be a pointer to struct */ pointed_type = btf_type_resolve_ptr(btf, args[arg_no].type, &arg_btf_id); if (!pointed_type || !btf_type_is_struct(pointed_type)) { pr_warn("stub function %s__%s has %s tagging to an unsupported type\n", st_ops_name, member_name, MAYBE_NULL_SUFFIX); goto err_out; } offset = btf_ctx_arg_offset(btf, func_proto, arg_no); if (offset < 0) { pr_warn("stub function %s__%s has an invalid trampoline ctx offset for arg#%u\n", st_ops_name, member_name, arg_no); goto err_out; } if (args[arg_no].type != stub_args[arg_no].type) { pr_warn("arg#%u type in stub function %s__%s does not match with its original func_proto\n", arg_no, st_ops_name, member_name); goto err_out; } /* Fill the information of the new argument */ info->reg_type = PTR_TRUSTED | PTR_TO_BTF_ID | PTR_MAYBE_NULL; info->btf_id = arg_btf_id; info->btf = btf; info->offset = offset; info++; info_cnt++; } if (info_cnt) { arg_info->info = info_buf; arg_info->cnt = info_cnt; } else { kfree(info_buf); } return 0; err_out: kfree(info_buf); return -EINVAL;} /* Clean up the arg_info in a struct bpf_struct_ops_desc. */void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_ops_desc){ struct bpf_struct_ops_arg_info *arg_info; int i; arg_info = st_ops_desc->arg_info; for (i = 0; i < btf_type_vlen(st_ops_desc->type); i++) kfree(arg_info[i].info); kfree(arg_info);} int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, struct btf *btf, struct bpf_verifier_log *log){ struct bpf_struct_ops *st_ops = st_ops_desc->st_ops; struct bpf_struct_ops_arg_info *arg_info; const struct btf_member *member; const struct btf_type *t; s32 type_id, value_id; char value_name[128]; const char *mname; int i, err; if (strlen(st_ops->name) + VALUE_PREFIX_LEN >= sizeof(value_name)) { pr_warn("struct_ops name %s is too long\n", st_ops->name); return -EINVAL; } sprintf(value_name, "%s%s", VALUE_PREFIX, st_ops->name); if (!st_ops->cfi_stubs) { pr_warn("struct_ops for %s has no cfi_stubs\n", st_ops->name); return -EINVAL; } type_id = btf_find_by_name_kind(btf, st_ops->name, BTF_KIND_STRUCT); if (type_id < 0) { pr_warn("Cannot find struct %s in %s\n", st_ops->name, btf_get_name(btf)); return -EINVAL; } t = btf_type_by_id(btf, type_id); if (btf_type_vlen(t) > BPF_STRUCT_OPS_MAX_NR_MEMBERS) { pr_warn("Cannot support #%u members in struct %s\n", btf_type_vlen(t), st_ops->name); return -EINVAL; } value_id = btf_find_by_name_kind(btf, value_name, BTF_KIND_STRUCT); if (value_id < 0) { pr_warn("Cannot find struct %s in %s\n", value_name, btf_get_name(btf)); return -EINVAL; } if (!is_valid_value_type(btf, value_id, t, value_name)) return -EINVAL; arg_info = kcalloc(btf_type_vlen(t), sizeof(*arg_info), GFP_KERNEL); if (!arg_info) return -ENOMEM; st_ops_desc->arg_info = arg_info; st_ops_desc->type = t; st_ops_desc->type_id = type_id; st_ops_desc->value_id = value_id; st_ops_desc->value_type = btf_type_by_id(btf, value_id); for_each_member(i, t, member) { const struct btf_type *func_proto; mname = btf_name_by_offset(btf, member->name_off); if (!*mname) { pr_warn("anon member in struct %s is not supported\n", st_ops->name); err = -EOPNOTSUPP; goto errout; } if (__btf_member_bitfield_size(t, member)) { pr_warn("bit field member %s in struct %s is not supported\n", mname, st_ops->name); err = -EOPNOTSUPP; goto errout; } func_proto = btf_type_resolve_func_ptr(btf, member->type, NULL); if (!func_proto) continue; if (btf_distill_func_proto(log, btf, func_proto, mname, &st_ops->func_models[i])) { pr_warn("Error in parsing func ptr %s in struct %s\n", mname, st_ops->name); err = -EINVAL; goto errout; } err = prepare_arg_info(btf, st_ops->name, mname, func_proto, arg_info + i); if (err) goto errout; } if (st_ops->init(btf)) { pr_warn("Error in init bpf_struct_ops %s\n", st_ops->name); err = -EINVAL; goto errout; } return 0; errout: bpf_struct_ops_desc_release(st_ops_desc); return err;} static int bpf_struct_ops_map_get_next_key(struct bpf_map *map, void *key, void *next_key){ if (key && *(u32 *)key == 0) return -ENOENT; *(u32 *)next_key = 0; return 0;} int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key, void *value){ struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; struct bpf_struct_ops_value *uvalue, *kvalue; enum bpf_struct_ops_state state; s64 refcnt; if (unlikely(*(u32 *)key != 0)) return -ENOENT; kvalue = &st_map->kvalue; /* Pair with smp_store_release() during map_update */ state = smp_load_acquire(&kvalue->common.state); if (state == BPF_STRUCT_OPS_STATE_INIT) { memset(value, 0, map->value_size); return 0; } /* No lock is needed. state and refcnt do not need * to be updated together under atomic context. */ uvalue = value; memcpy(uvalue, st_map->uvalue, map->value_size); uvalue->common.state = state; /* This value offers the user space a general estimate of how * many sockets are still utilizing this struct_ops for TCP * congestion control. The number might not be exact, but it * should sufficiently meet our present goals. */ refcnt = atomic64_read(&map->refcnt) - atomic64_read(&map->usercnt); refcount_set(&uvalue->common.refcnt, max_t(s64, refcnt, 0)); return 0;} static void *bpf_struct_ops_map_lookup_elem(struct bpf_map *map, void *key){ return ERR_PTR(-EINVAL);} static void bpf_struct_ops_map_put_progs(struct bpf_struct_ops_map *st_map){ u32 i; for (i = 0; i < st_map->funcs_cnt; i++) { if (!st_map->links[i]) break; bpf_link_put(st_map->links[i]); st_map->links[i] = NULL; }} static void bpf_struct_ops_map_free_image(struct bpf_struct_ops_map *st_map){ int i; for (i = 0; i < st_map->image_pages_cnt; i++) bpf_struct_ops_image_free(st_map->image_pages[i]); st_map->image_pages_cnt = 0;}Lines 1–500 of 1353. Load more
// SPDX-License-Identifier: GPL-2.0-only /* Copyright (c) 2019 Facebook */ #include <linux/bpf.h> #include <linux/bpf_verifier.h> #include <linux/btf.h> #include <linux/filter.h> #include <linux/slab.h> #include <linux/numa.h> #include <linux/seq_file.h> #include <linux/refcount.h> #include <linux/mutex.h> #include <linux/btf_ids.h> #include <linux/rcupdate_wait.h> #include <linux/poll.h> struct bpf_struct_ops_value { struct bpf_struct_ops_common_value common; char data[] ____cacheline_aligned_in_smp; }; #define MAX_TRAMP_IMAGE_PAGES 8 struct bpf_struct_ops_map { struct bpf_map map; const struct bpf_struct_ops_desc *st_ops_desc; /* protect map_update */ struct mutex lock; /* link has all the bpf_links that is populated * to the func ptr of the kernel's struct * (in kvalue.data). */ struct bpf_link **links; /* ksyms for bpf trampolines */ struct bpf_ksym **ksyms; u32 funcs_cnt; u32 image_pages_cnt; /* image_pages is an array of pages that has all the trampolines * that stores the func args before calling the bpf_prog. */ void *image_pages[MAX_TRAMP_IMAGE_PAGES]; /* The owner moduler's btf. */ struct btf *btf; /* uvalue->data stores the kernel struct * (e.g. tcp_congestion_ops) that is more useful * to userspace than the kvalue. For example, * the bpf_prog's id is stored instead of the kernel * address of a func ptr. */ struct bpf_struct_ops_value *uvalue; /* kvalue.data stores the actual kernel's struct * (e.g. tcp_congestion_ops) that will be * registered to the kernel subsystem. */ struct bpf_struct_ops_value kvalue; }; struct bpf_struct_ops_link { struct bpf_link link; struct bpf_map __rcu *map; wait_queue_head_t wait_hup; }; static DEFINE_MUTEX(update_mutex); #define VALUE_PREFIX "bpf_struct_ops_" #define VALUE_PREFIX_LEN (sizeof(VALUE_PREFIX) - 1) const struct bpf_verifier_ops bpf_struct_ops_verifier_ops = { }; const struct bpf_prog_ops bpf_struct_ops_prog_ops = { #ifdef CONFIG_NET .test_run = bpf_struct_ops_test_run, #endif }; BTF_ID_LIST(st_ops_ids) BTF_ID(struct, module) BTF_ID(struct, bpf_struct_ops_common_value) enum { IDX_MODULE_ID, IDX_ST_OPS_COMMON_VALUE_ID, }; extern struct btf *btf_vmlinux; static bool is_valid_value_type(struct btf *btf, s32 value_id, const struct btf_type *type, const char *value_name) { const struct btf_type *common_value_type; const struct btf_member *member; const struct btf_type *vt, *mt; vt = btf_type_by_id(btf, value_id); if (btf_vlen(vt) != 2) { pr_warn("The number of %s's members should be 2, but we get %d\n", value_name, btf_vlen(vt)); return false; } member = btf_type_member(vt); mt = btf_type_by_id(btf, member->type); common_value_type = btf_type_by_id(btf_vmlinux, st_ops_ids[IDX_ST_OPS_COMMON_VALUE_ID]); if (mt != common_value_type) { pr_warn("The first member of %s should be bpf_struct_ops_common_value\n", value_name); return false; } member++; mt = btf_type_by_id(btf, member->type); if (mt != type) { pr_warn("The second member of %s should be %s\n", value_name, btf_name_by_offset(btf, type->name_off)); return false; } return true; } static void *bpf_struct_ops_image_alloc(void) { void *image; int err; err = bpf_jit_charge_modmem(PAGE_SIZE); if (err) return ERR_PTR(err); image = arch_alloc_bpf_trampoline(PAGE_SIZE); if (!image) { bpf_jit_uncharge_modmem(PAGE_SIZE); return ERR_PTR(-ENOMEM); } return image; } void bpf_struct_ops_image_free(void *image) { if (image) { arch_free_bpf_trampoline(image, PAGE_SIZE); bpf_jit_uncharge_modmem(PAGE_SIZE); } } #define MAYBE_NULL_SUFFIX "__nullable" #define MAX_STUB_NAME 128 /* Return the type info of a stub function, if it exists. * * The name of a stub function is made up of the name of the struct_ops and * the name of the function pointer member, separated by "__". For example, * if the struct_ops type is named "foo_ops" and the function pointer * member is named "bar", the stub function name would be "foo_ops__bar". */ static const struct btf_type * find_stub_func_proto(const struct btf *btf, const char *st_op_name, const char *member_name) { char stub_func_name[MAX_STUB_NAME]; const struct btf_type *func_type; s32 btf_id; int cp; cp = snprintf(stub_func_name, MAX_STUB_NAME, "%s__%s", st_op_name, member_name); if (cp >= MAX_STUB_NAME) { pr_warn("Stub function name too long\n"); return NULL; } btf_id = btf_find_by_name_kind(btf, stub_func_name, BTF_KIND_FUNC); if (btf_id < 0) return NULL; func_type = btf_type_by_id(btf, btf_id); if (!func_type) return NULL; return btf_type_by_id(btf, func_type->type); /* FUNC_PROTO */ } /* Prepare argument info for every nullable argument of a member of a * struct_ops type. * * Initialize a struct bpf_struct_ops_arg_info according to type info of * the arguments of a stub function. (Check kCFI for more information about * stub functions.) * * Each member in the struct_ops type has a struct bpf_struct_ops_arg_info * to provide an array of struct bpf_ctx_arg_aux, which in turn provides * the information that used by the verifier to check the arguments of the * BPF struct_ops program assigned to the member. Here, we only care about * the arguments that are marked as __nullable. * * The array of struct bpf_ctx_arg_aux is eventually assigned to * prog->aux->ctx_arg_info of BPF struct_ops programs and passed to the * verifier. (See check_struct_ops_btf_id()) * * arg_info->info will be the list of struct bpf_ctx_arg_aux if success. If * fails, it will be kept untouched. */ static int prepare_arg_info(struct btf *btf, const char *st_ops_name, const char *member_name, const struct btf_type *func_proto, struct bpf_struct_ops_arg_info *arg_info) { const struct btf_type *stub_func_proto, *pointed_type; const struct btf_param *stub_args, *args; struct bpf_ctx_arg_aux *info, *info_buf; u32 nargs, arg_no, info_cnt = 0; u32 arg_btf_id; int offset; stub_func_proto = find_stub_func_proto(btf, st_ops_name, member_name); if (!stub_func_proto) return 0; /* Check if the number of arguments of the stub function is the same * as the number of arguments of the function pointer. */ nargs = btf_type_vlen(func_proto); if (nargs != btf_type_vlen(stub_func_proto)) { pr_warn("the number of arguments of the stub function %s__%s does not match the number of arguments of the member %s of struct %s\n", st_ops_name, member_name, member_name, st_ops_name); return -EINVAL; } if (!nargs) return 0; args = btf_params(func_proto); stub_args = btf_params(stub_func_proto); info_buf = kcalloc(nargs, sizeof(*info_buf), GFP_KERNEL); if (!info_buf) return -ENOMEM; /* Prepare info for every nullable argument */ info = info_buf; for (arg_no = 0; arg_no < nargs; arg_no++) { /* Skip arguments that is not suffixed with * "__nullable". */ if (!btf_param_match_suffix(btf, &stub_args[arg_no], MAYBE_NULL_SUFFIX)) continue; /* Should be a pointer to struct */ pointed_type = btf_type_resolve_ptr(btf, args[arg_no].type, &arg_btf_id); if (!pointed_type || !btf_type_is_struct(pointed_type)) { pr_warn("stub function %s__%s has %s tagging to an unsupported type\n", st_ops_name, member_name, MAYBE_NULL_SUFFIX); goto err_out; } offset = btf_ctx_arg_offset(btf, func_proto, arg_no); if (offset < 0) { pr_warn("stub function %s__%s has an invalid trampoline ctx offset for arg#%u\n", st_ops_name, member_name, arg_no); goto err_out; } if (args[arg_no].type != stub_args[arg_no].type) { pr_warn("arg#%u type in stub function %s__%s does not match with its original func_proto\n", arg_no, st_ops_name, member_name); goto err_out; } /* Fill the information of the new argument */ info->reg_type = PTR_TRUSTED | PTR_TO_BTF_ID | PTR_MAYBE_NULL; info->btf_id = arg_btf_id; info->btf = btf; info->offset = offset; info++; info_cnt++; } if (info_cnt) { arg_info->info = info_buf; arg_info->cnt = info_cnt; } else { kfree(info_buf); } return 0; err_out: kfree(info_buf); return -EINVAL; } /* Clean up the arg_info in a struct bpf_struct_ops_desc. */ void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_ops_desc) { struct bpf_struct_ops_arg_info *arg_info; int i; arg_info = st_ops_desc->arg_info; for (i = 0; i < btf_type_vlen(st_ops_desc->type); i++) kfree(arg_info[i].info); kfree(arg_info); } int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, struct btf *btf, struct bpf_verifier_log *log) { struct bpf_struct_ops *st_ops = st_ops_desc->st_ops; struct bpf_struct_ops_arg_info *arg_info; const struct btf_member *member; const struct btf_type *t; s32 type_id, value_id; char value_name[128]; const char *mname; int i, err; if (strlen(st_ops->name) + VALUE_PREFIX_LEN >= sizeof(value_name)) { pr_warn("struct_ops name %s is too long\n", st_ops->name); return -EINVAL; } sprintf(value_name, "%s%s", VALUE_PREFIX, st_ops->name); if (!st_ops->cfi_stubs) { pr_warn("struct_ops for %s has no cfi_stubs\n", st_ops->name); return -EINVAL; } type_id = btf_find_by_name_kind(btf, st_ops->name, BTF_KIND_STRUCT); if (type_id < 0) { pr_warn("Cannot find struct %s in %s\n", st_ops->name, btf_get_name(btf)); return -EINVAL; } t = btf_type_by_id(btf, type_id); if (btf_type_vlen(t) > BPF_STRUCT_OPS_MAX_NR_MEMBERS) { pr_warn("Cannot support #%u members in struct %s\n", btf_type_vlen(t), st_ops->name); return -EINVAL; } value_id = btf_find_by_name_kind(btf, value_name, BTF_KIND_STRUCT); if (value_id < 0) { pr_warn("Cannot find struct %s in %s\n", value_name, btf_get_name(btf)); return -EINVAL; } if (!is_valid_value_type(btf, value_id, t, value_name)) return -EINVAL; arg_info = kcalloc(btf_type_vlen(t), sizeof(*arg_info), GFP_KERNEL); if (!arg_info) return -ENOMEM; st_ops_desc->arg_info = arg_info; st_ops_desc->type = t; st_ops_desc->type_id = type_id; st_ops_desc->value_id = value_id; st_ops_desc->value_type = btf_type_by_id(btf, value_id); for_each_member(i, t, member) { const struct btf_type *func_proto; mname = btf_name_by_offset(btf, member->name_off); if (!*mname) { pr_warn("anon member in struct %s is not supported\n", st_ops->name); err = -EOPNOTSUPP; goto errout; } if (__btf_member_bitfield_size(t, member)) { pr_warn("bit field member %s in struct %s is not supported\n", mname, st_ops->name); err = -EOPNOTSUPP; goto errout; } func_proto = btf_type_resolve_func_ptr(btf, member->type, NULL); if (!func_proto) continue; if (btf_distill_func_proto(log, btf, func_proto, mname, &st_ops->func_models[i])) { pr_warn("Error in parsing func ptr %s in struct %s\n", mname, st_ops->name); err = -EINVAL; goto errout; } err = prepare_arg_info(btf, st_ops->name, mname, func_proto, arg_info + i); if (err) goto errout; } if (st_ops->init(btf)) { pr_warn("Error in init bpf_struct_ops %s\n", st_ops->name); err = -EINVAL; goto errout; } return 0; errout: bpf_struct_ops_desc_release(st_ops_desc); return err; } static int bpf_struct_ops_map_get_next_key(struct bpf_map *map, void *key, void *next_key) { if (key && *(u32 *)key == 0) return -ENOENT; *(u32 *)next_key = 0; return 0; } int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key, void *value) { struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; struct bpf_struct_ops_value *uvalue, *kvalue; enum bpf_struct_ops_state state; s64 refcnt; if (unlikely(*(u32 *)key != 0)) return -ENOENT; kvalue = &st_map->kvalue; /* Pair with smp_store_release() during map_update */ state = smp_load_acquire(&kvalue->common.state); if (state == BPF_STRUCT_OPS_STATE_INIT) { memset(value, 0, map->value_size); return 0; } /* No lock is needed. state and refcnt do not need * to be updated together under atomic context. */ uvalue = value; memcpy(uvalue, st_map->uvalue, map->value_size); uvalue->common.state = state; /* This value offers the user space a general estimate of how * many sockets are still utilizing this struct_ops for TCP * congestion control. The number might not be exact, but it * should sufficiently meet our present goals. */ refcnt = atomic64_read(&map->refcnt) - atomic64_read(&map->usercnt); refcount_set(&uvalue->common.refcnt, max_t(s64, refcnt, 0)); return 0; } static void *bpf_struct_ops_map_lookup_elem(struct bpf_map *map, void *key) { return ERR_PTR(-EINVAL); } static void bpf_struct_ops_map_put_progs(struct bpf_struct_ops_map *st_map) { u32 i; for (i = 0; i < st_map->funcs_cnt; i++) { if (!st_map->links[i]) break; bpf_link_put(st_map->links[i]); st_map->links[i] = NULL; } } static void bpf_struct_ops_map_free_image(struct bpf_struct_ops_map *st_map) { int i; for (i = 0; i < st_map->image_pages_cnt; i++) bpf_struct_ops_image_free(st_map->image_pages[i]); st_map->image_pages_cnt = 0; }