Structs bpf_rb_node and bpf_list_node are opaquely defined in uapi/linux/bpf.h, as BPF program writers are not expected to touch their fields - nor does the verifier allow them to do so. Currently these structs are simple wrappers around structs rb_node and list_head and linked_list / rbtree implementation just casts and passes to library functions for those data structures. Later patches in this series, though, will add an "owner" field to bpf_{rb,list}_node, such that they're not just wrapping an underlying node type. Moreover, the bpf linked_list and rbtree implementations will deal with these owner pointers directly in a few different places. To avoid having to do void *owner = (void*)bpf_list_node + sizeof(struct list_head) with opaque UAPI node types, add bpf_{list,rb}_node_kern struct definitions to internal headers and modify linked_list and rbtree to use the internal types where appropriate.
Unified · Split
| @@ -228,6 +228,16 @@ struct btf_record { · 227 unmodified lines | |||
| 228 | struct btf_field fields[]; | 228 | struct btf_field fields[]; |
| 229 | }; | 229 | }; |
| 230 | 230 | ||
| 231 | /* Non-opaque version of bpf_rb_node in uapi/linux/bpf.h */ | ||
| 232 | struct bpf_rb_node_kern { | ||
| 233 | struct rb_node rb_node; | ||
| 234 | } __attribute__((aligned(8))); | ||
| 235 | |||
| 236 | /* Non-opaque version of bpf_list_node in uapi/linux/bpf.h */ | ||
| 237 | struct bpf_list_node_kern { | ||
| 238 | struct list_head list_head; | ||
| 239 | } __attribute__((aligned(8))); | ||
| 240 | |||
| 231 | struct bpf_map { | 241 | struct bpf_map { |
| 232 | /* The first two cachelines with read-mostly members of which some | 242 | /* The first two cachelines with read-mostly members of which some |
| 233 | * are also accessed in fast-path (e.g. ops, max_entries). | 243 | * are also accessed in fast-path (e.g. ops, max_entries). |
| @@ -1942,10 +1942,11 @@ __bpf_kfunc void *bpf_refcount_acquire_impl(void *p__refcounted_kptr, void *meta · 1941 unmodified lines | |||
| 1942 | return (void *)p__refcounted_kptr; | 1942 | return (void *)p__refcounted_kptr; |
| 1943 | } | 1943 | } |
| 1944 | 1944 | ||
| 1945 | static int __bpf_list_add(struct bpf_list_node *node, struct bpf_list_head *head, | 1945 | static int __bpf_list_add(struct bpf_list_node_kern *node, |
| 1946 | struct bpf_list_head *head, | ||
| 1946 | bool tail, struct btf_record *rec, u64 off) | 1947 | bool tail, struct btf_record *rec, u64 off) |
| 1947 | { | 1948 | { |
| 1948 | struct list_head *n = (void *)node, *h = (void *)head; | 1949 | struct list_head *n = &node->list_head, *h = (void *)head; |
| 1949 | 1950 | ||
| 1950 | /* If list_head was 0-initialized by map, bpf_obj_init_field wasn't | 1951 | /* If list_head was 0-initialized by map, bpf_obj_init_field wasn't |
| 1951 | * called on its fields, so init here | 1952 | * called on its fields, so init here |
| @@ -1967,20 +1968,20 @@ __bpf_kfunc int bpf_list_push_front_impl(struct bpf_list_head *head, · 15 unmodified lines | |||
| 1967 | struct bpf_list_node *node, | 1968 | struct bpf_list_node *node, |
| 1968 | void *meta__ign, u64 off) | 1969 | void *meta__ign, u64 off) |
| 1969 | { | 1970 | { |
| 1971 | struct bpf_list_node_kern *n = (void *)node; | ||
| 1970 | struct btf_struct_meta *meta = meta__ign; | 1972 | struct btf_struct_meta *meta = meta__ign; |
| 1971 | 1973 | ||
| 1972 | return __bpf_list_add(node, head, false, | 1974 | return __bpf_list_add(n, head, false, meta ? meta->record : NULL, off); |
| 1973 | meta ? meta->record : NULL, off); | ||
| 1974 | } | 1975 | } |
| 1975 | 1976 | ||
| 1976 | __bpf_kfunc int bpf_list_push_back_impl(struct bpf_list_head *head, | 1977 | __bpf_kfunc int bpf_list_push_back_impl(struct bpf_list_head *head, |
| 1977 | struct bpf_list_node *node, | 1978 | struct bpf_list_node *node, |
| 1978 | void *meta__ign, u64 off) | 1979 | void *meta__ign, u64 off) |
| 1979 | { | 1980 | { |
| 1981 | struct bpf_list_node_kern *n = (void *)node; | ||
| 1980 | struct btf_struct_meta *meta = meta__ign; | 1982 | struct btf_struct_meta *meta = meta__ign; |
| 1981 | 1983 | ||
| 1982 | return __bpf_list_add(node, head, true, | 1984 | return __bpf_list_add(n, head, true, meta ? meta->record : NULL, off); |
| 1983 | meta ? meta->record : NULL, off); | ||
| 1984 | } | 1985 | } |
| 1985 | 1986 | ||
| 1986 | static struct bpf_list_node *__bpf_list_del(struct bpf_list_head *head, bool tail) | 1987 | static struct bpf_list_node *__bpf_list_del(struct bpf_list_head *head, bool tail) |
| @@ -2013,7 +2014,7 @@ __bpf_kfunc struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root, · 26 unmodified lines | |||
| 2013 | struct bpf_rb_node *node) | 2014 | struct bpf_rb_node *node) |
| 2014 | { | 2015 | { |
| 2015 | struct rb_root_cached *r = (struct rb_root_cached *)root; | 2016 | struct rb_root_cached *r = (struct rb_root_cached *)root; |
| 2016 | struct rb_node *n = (struct rb_node *)node; | 2017 | struct rb_node *n = &((struct bpf_rb_node_kern *)node)->rb_node; |
| 2017 | 2018 | ||
| 2018 | if (RB_EMPTY_NODE(n)) | 2019 | if (RB_EMPTY_NODE(n)) |
| 2019 | return NULL; | 2020 | return NULL; |
| @@ -2026,11 +2027,12 @@ __bpf_kfunc struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root, · 6 unmodified lines | |||
| 2026 | /* Need to copy rbtree_add_cached's logic here because our 'less' is a BPF | 2027 | /* Need to copy rbtree_add_cached's logic here because our 'less' is a BPF |
| 2027 | * program | 2028 | * program |
| 2028 | */ | 2029 | */ |
| 2029 | static int __bpf_rbtree_add(struct bpf_rb_root *root, struct bpf_rb_node *node, | 2030 | static int __bpf_rbtree_add(struct bpf_rb_root *root, |
| 2031 | struct bpf_rb_node_kern *node, | ||
| 2030 | void *less, struct btf_record *rec, u64 off) | 2032 | void *less, struct btf_record *rec, u64 off) |
| 2031 | { | 2033 | { |
| 2032 | struct rb_node **link = &((struct rb_root_cached *)root)->rb_root.rb_node; | 2034 | struct rb_node **link = &((struct rb_root_cached *)root)->rb_root.rb_node; |
| 2033 | struct rb_node *parent = NULL, *n = (struct rb_node *)node; | 2035 | struct rb_node *parent = NULL, *n = &node->rb_node; |
| 2034 | bpf_callback_t cb = (bpf_callback_t)less; | 2036 | bpf_callback_t cb = (bpf_callback_t)less; |
| 2035 | bool leftmost = true; | 2037 | bool leftmost = true; |
| 2036 | 2038 | ||
| @@ -2060,8 +2062,9 @@ __bpf_kfunc int bpf_rbtree_add_impl(struct bpf_rb_root *root, struct bpf_rb_node · 23 unmodified lines | |||
| 2060 | void *meta__ign, u64 off) | 2062 | void *meta__ign, u64 off) |
| 2061 | { | 2063 | { |
| 2062 | struct btf_struct_meta *meta = meta__ign; | 2064 | struct btf_struct_meta *meta = meta__ign; |
| 2065 | struct bpf_rb_node_kern *n = (void *)node; | ||
| 2063 | 2066 | ||
| 2064 | return __bpf_rbtree_add(root, node, (void *)less, meta ? meta->record : NULL, off); | 2067 | return __bpf_rbtree_add(root, n, (void *)less, meta ? meta->record : NULL, off); |
| 2065 | } | 2068 | } |
| 2066 | 2069 | ||
| 2067 | __bpf_kfunc struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) | 2070 | __bpf_kfunc struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) |