Structs bpf_rb_node and bpf_list_node are opaquely defined in uapi/linux/bpf.h, as BPF program writers are not expected to touch their fields - nor does the verifier allow them to do so. Currently these structs are simple wrappers around structs rb_node and list_head and linked_list / rbtree implementation just casts and passes to library functions for those data structures. Later patches in this series, though, will add an "owner" field to bpf_{rb,list}_node, such that they're not just wrapping an underlying node type. Moreover, the bpf linked_list and rbtree implementations will deal with these owner pointers directly in a few different places. To avoid having to do void *owner = (void*)bpf_list_node + sizeof(struct list_head) with opaque UAPI node types, add bpf_{list,rb}_node_kern struct definitions to internal headers and modify linked_list and rbtree to use the internal types where appropriate.
Unified · Split
| @@ -228,6 +228,16 @@ struct btf_record { · 227 unmodified lines | ||
| 228 | 228 | struct btf_field fields[]; |
| 229 | 229 | }; |
| 230 | 230 | |
| 231 | +/* Non-opaque version of bpf_rb_node in uapi/linux/bpf.h */ | |
| 232 | +struct bpf_rb_node_kern { | |
| 233 | + struct rb_node rb_node; | |
| 234 | +} __attribute__((aligned(8))); | |
| 235 | + | |
| 236 | +/* Non-opaque version of bpf_list_node in uapi/linux/bpf.h */ | |
| 237 | +struct bpf_list_node_kern { | |
| 238 | + struct list_head list_head; | |
| 239 | +} __attribute__((aligned(8))); | |
| 240 | + | |
| 231 | 241 | struct bpf_map { |
| 232 | 242 | /* The first two cachelines with read-mostly members of which some |
| 233 | 243 | * are also accessed in fast-path (e.g. ops, max_entries). |
| @@ -1942,10 +1942,11 @@ __bpf_kfunc void *bpf_refcount_acquire_impl(void *p__refcounted_kptr, void *meta · 1941 unmodified lines | ||
| 1942 | 1942 | return (void *)p__refcounted_kptr; |
| 1943 | 1943 | } |
| 1944 | 1944 | |
| 1945 | −static int __bpf_list_add(struct bpf_list_node *node, struct bpf_list_head *head, | |
| 1945 | +static int __bpf_list_add(struct bpf_list_node_kern *node, | |
| 1946 | + struct bpf_list_head *head, | |
| 1946 | 1947 | bool tail, struct btf_record *rec, u64 off) |
| 1947 | 1948 | { |
| 1948 | − struct list_head *n = (void *)node, *h = (void *)head; | |
| 1949 | + struct list_head *n = &node->list_head, *h = (void *)head; | |
| 1949 | 1950 | |
| 1950 | 1951 | /* If list_head was 0-initialized by map, bpf_obj_init_field wasn't |
| 1951 | 1952 | * called on its fields, so init here |
| @@ -1967,20 +1968,20 @@ __bpf_kfunc int bpf_list_push_front_impl(struct bpf_list_head *head, · 15 unmodified lines | ||
| 1967 | 1968 | struct bpf_list_node *node, |
| 1968 | 1969 | void *meta__ign, u64 off) |
| 1969 | 1970 | { |
| 1971 | + struct bpf_list_node_kern *n = (void *)node; | |
| 1970 | 1972 | struct btf_struct_meta *meta = meta__ign; |
| 1971 | 1973 | |
| 1972 | − return __bpf_list_add(node, head, false, | |
| 1973 | − meta ? meta->record : NULL, off); | |
| 1974 | + return __bpf_list_add(n, head, false, meta ? meta->record : NULL, off); | |
| 1974 | 1975 | } |
| 1975 | 1976 | |
| 1976 | 1977 | __bpf_kfunc int bpf_list_push_back_impl(struct bpf_list_head *head, |
| 1977 | 1978 | struct bpf_list_node *node, |
| 1978 | 1979 | void *meta__ign, u64 off) |
| 1979 | 1980 | { |
| 1981 | + struct bpf_list_node_kern *n = (void *)node; | |
| 1980 | 1982 | struct btf_struct_meta *meta = meta__ign; |
| 1981 | 1983 | |
| 1982 | − return __bpf_list_add(node, head, true, | |
| 1983 | − meta ? meta->record : NULL, off); | |
| 1984 | + return __bpf_list_add(n, head, true, meta ? meta->record : NULL, off); | |
| 1984 | 1985 | } |
| 1985 | 1986 | |
| 1986 | 1987 | static struct bpf_list_node *__bpf_list_del(struct bpf_list_head *head, bool tail) |
| @@ -2013,7 +2014,7 @@ __bpf_kfunc struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root, · 26 unmodified lines | ||
| 2013 | 2014 | struct bpf_rb_node *node) |
| 2014 | 2015 | { |
| 2015 | 2016 | struct rb_root_cached *r = (struct rb_root_cached *)root; |
| 2016 | − struct rb_node *n = (struct rb_node *)node; | |
| 2017 | + struct rb_node *n = &((struct bpf_rb_node_kern *)node)->rb_node; | |
| 2017 | 2018 | |
| 2018 | 2019 | if (RB_EMPTY_NODE(n)) |
| 2019 | 2020 | return NULL; |
| @@ -2026,11 +2027,12 @@ __bpf_kfunc struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root, · 6 unmodified lines | ||
| 2026 | 2027 | /* Need to copy rbtree_add_cached's logic here because our 'less' is a BPF |
| 2027 | 2028 | * program |
| 2028 | 2029 | */ |
| 2029 | −static int __bpf_rbtree_add(struct bpf_rb_root *root, struct bpf_rb_node *node, | |
| 2030 | +static int __bpf_rbtree_add(struct bpf_rb_root *root, | |
| 2031 | + struct bpf_rb_node_kern *node, | |
| 2030 | 2032 | void *less, struct btf_record *rec, u64 off) |
| 2031 | 2033 | { |
| 2032 | 2034 | struct rb_node **link = &((struct rb_root_cached *)root)->rb_root.rb_node; |
| 2033 | − struct rb_node *parent = NULL, *n = (struct rb_node *)node; | |
| 2035 | + struct rb_node *parent = NULL, *n = &node->rb_node; | |
| 2034 | 2036 | bpf_callback_t cb = (bpf_callback_t)less; |
| 2035 | 2037 | bool leftmost = true; |
| 2036 | 2038 | |
| @@ -2060,8 +2062,9 @@ __bpf_kfunc int bpf_rbtree_add_impl(struct bpf_rb_root *root, struct bpf_rb_node · 23 unmodified lines | ||
| 2060 | 2062 | void *meta__ign, u64 off) |
| 2061 | 2063 | { |
| 2062 | 2064 | struct btf_struct_meta *meta = meta__ign; |
| 2065 | + struct bpf_rb_node_kern *n = (void *)node; | |
| 2063 | 2066 | |
| 2064 | − return __bpf_rbtree_add(root, node, (void *)less, meta ? meta->record : NULL, off); | |
| 2067 | + return __bpf_rbtree_add(root, n, (void *)less, meta ? meta->record : NULL, off); | |
| 2065 | 2068 | } |
| 2066 | 2069 | |
| 2067 | 2070 | __bpf_kfunc struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) |