Implement the first open-coded iterator type over a range of integers. It's public API consists of: - bpf_iter_num_new() constructor, which accepts [start, end) range (that is, start is inclusive, end is exclusive). - bpf_iter_num_next() which will keep returning read-only pointer to int until the range is exhausted, at which point NULL will be returned. If bpf_iter_num_next() is kept calling after this, NULL will be persistently returned. - bpf_iter_num_destroy() destructor, which needs to be called at some point to clean up iterator state. BPF verifier enforces that iterator destructor is called at some point before BPF program exits. Note that `start = end = X` is a valid combination to setup an empty iterator. bpf_iter_num_new() will return 0 (success) for any such combination. If bpf_iter_num_new() detects invalid combination of input arguments, it returns error, resets iterator state to, effectively, empty iterator, so any subsequent call to bpf_iter_num_next() will keep returning NULL. BPF verifier has no knowledge that returned integers are in the [start, end) value range, as both `start` and `end` are not statically known and enforced: they are runtime values. While the implementation is pretty trivial, some care needs to be taken to avoid overflows and underflows. Subsequent selftests will validate correctness of [start, end) semantics, especially around extremes (INT_MIN and INT_MAX). Similarly to bpf_loop(), we enforce that no more than BPF_MAX_LOOPS can be specified. bpf_iter_num_{new,next,destroy}() is a logical evolution from bounded BPF loops and bpf_loop() helper and is the basis for implementing ergonomic BPF loops with no statically known or verified bounds. Subsequent patches implement bpf_for() macro, demonstrating how this can be wrapped into something that works and feels like a normal for() loop in C language.
Unified · Split
| @@ -1617,8 +1617,12 @@ struct bpf_array { · 1616 unmodified lines | |||
| 1617 | #define BPF_COMPLEXITY_LIMIT_INSNS 1000000 /* yes. 1M insns */ | 1617 | #define BPF_COMPLEXITY_LIMIT_INSNS 1000000 /* yes. 1M insns */ |
| 1618 | #define MAX_TAIL_CALL_CNT 33 | 1618 | #define MAX_TAIL_CALL_CNT 33 |
| 1619 | 1619 | ||
| 1620 | /* Maximum number of loops for bpf_loop */ | 1620 | /* Maximum number of loops for bpf_loop and bpf_iter_num. |
| 1621 | #define BPF_MAX_LOOPS BIT(23) | 1621 | * It's enum to expose it (and thus make it discoverable) through BTF. |
| 1622 | */ | ||
| 1623 | enum { | ||
| 1624 | BPF_MAX_LOOPS = 8 * 1024 * 1024, | ||
| 1625 | }; | ||
| 1622 | 1626 | ||
| 1623 | #define BPF_F_ACCESS_MASK (BPF_F_RDONLY | \ | 1627 | #define BPF_F_ACCESS_MASK (BPF_F_RDONLY | \ |
| 1624 | BPF_F_RDONLY_PROG | \ | 1628 | BPF_F_RDONLY_PROG | \ |
| @@ -7112,4 +7112,12 @@ enum { · 7111 unmodified lines | |||
| 7112 | BPF_F_TIMER_ABS = (1ULL << 0), | 7112 | BPF_F_TIMER_ABS = (1ULL << 0), |
| 7113 | }; | 7113 | }; |
| 7114 | 7114 | ||
| 7115 | /* BPF numbers iterator state */ | ||
| 7116 | struct bpf_iter_num { | ||
| 7117 | /* opaque iterator state; having __u64 here allows to preserve correct | ||
| 7118 | * alignment requirements in vmlinux.h, generated from BTF | ||
| 7119 | */ | ||
| 7120 | __u64 __opaque[1]; | ||
| 7121 | } __attribute__((aligned(8))); | ||
| 7122 | |||
| 7115 | #endif /* _UAPI__LINUX_BPF_H__ */ | 7123 | #endif /* _UAPI__LINUX_BPF_H__ */ |
| @@ -776,3 +776,73 @@ const struct bpf_func_proto bpf_loop_proto = { · 775 unmodified lines | |||
| 776 | .arg3_type = ARG_PTR_TO_STACK_OR_NULL, | 776 | .arg3_type = ARG_PTR_TO_STACK_OR_NULL, |
| 777 | .arg4_type = ARG_ANYTHING, | 777 | .arg4_type = ARG_ANYTHING, |
| 778 | }; | 778 | }; |
| 779 | |||
| 780 | struct bpf_iter_num_kern { | ||
| 781 | int cur; /* current value, inclusive */ | ||
| 782 | int end; /* final value, exclusive */ | ||
| 783 | } __aligned(8); | ||
| 784 | |||
| 785 | __diag_push(); | ||
| 786 | __diag_ignore_all("-Wmissing-prototypes", | ||
| 787 | "Global functions as their definitions will be in vmlinux BTF"); | ||
| 788 | |||
| 789 | __bpf_kfunc int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end) | ||
| 790 | { | ||
| 791 | struct bpf_iter_num_kern *s = (void *)it; | ||
| 792 | |||
| 793 | BUILD_BUG_ON(sizeof(struct bpf_iter_num_kern) != sizeof(struct bpf_iter_num)); | ||
| 794 | BUILD_BUG_ON(__alignof__(struct bpf_iter_num_kern) != __alignof__(struct bpf_iter_num)); | ||
| 795 | |||
| 796 | BTF_TYPE_EMIT(struct btf_iter_num); | ||
| 797 | |||
| 798 | /* start == end is legit, it's an empty range and we'll just get NULL | ||
| 799 | * on first (and any subsequent) bpf_iter_num_next() call | ||
| 800 | */ | ||
| 801 | if (start > end) { | ||
| 802 | s->cur = s->end = 0; | ||
| 803 | return -EINVAL; | ||
| 804 | } | ||
| 805 | |||
| 806 | /* avoid overflows, e.g., if start == INT_MIN and end == INT_MAX */ | ||
| 807 | if ((s64)end - (s64)start > BPF_MAX_LOOPS) { | ||
| 808 | s->cur = s->end = 0; | ||
| 809 | return -E2BIG; | ||
| 810 | } | ||
| 811 | |||
| 812 | /* user will call bpf_iter_num_next() first, | ||
| 813 | * which will set s->cur to exactly start value; | ||
| 814 | * underflow shouldn't matter | ||
| 815 | */ | ||
| 816 | s->cur = start - 1; | ||
| 817 | s->end = end; | ||
| 818 | |||
| 819 | return 0; | ||
| 820 | } | ||
| 821 | |||
| 822 | __bpf_kfunc int *bpf_iter_num_next(struct bpf_iter_num* it) | ||
| 823 | { | ||
| 824 | struct bpf_iter_num_kern *s = (void *)it; | ||
| 825 | |||
| 826 | /* check failed initialization or if we are done (same behavior); | ||
| 827 | * need to be careful about overflow, so convert to s64 for checks, | ||
| 828 | * e.g., if s->cur == s->end == INT_MAX, we can't just do | ||
| 829 | * s->cur + 1 >= s->end | ||
| 830 | */ | ||
| 831 | if ((s64)(s->cur + 1) >= s->end) { | ||
| 832 | s->cur = s->end = 0; | ||
| 833 | return NULL; | ||
| 834 | } | ||
| 835 | |||
| 836 | s->cur++; | ||
| 837 | |||
| 838 | return &s->cur; | ||
| 839 | } | ||
| 840 | |||
| 841 | __bpf_kfunc void bpf_iter_num_destroy(struct bpf_iter_num *it) | ||
| 842 | { | ||
| 843 | struct bpf_iter_num_kern *s = (void *)it; | ||
| 844 | |||
| 845 | s->cur = s->end = 0; | ||
| 846 | } | ||
| 847 | |||
| 848 | __diag_pop(); | ||
| @@ -2411,6 +2411,9 @@ BTF_ID_FLAGS(func, bpf_rcu_read_lock) · 2410 unmodified lines | |||
| 2411 | BTF_ID_FLAGS(func, bpf_rcu_read_unlock) | 2411 | BTF_ID_FLAGS(func, bpf_rcu_read_unlock) |
| 2412 | BTF_ID_FLAGS(func, bpf_dynptr_slice, KF_RET_NULL) | 2412 | BTF_ID_FLAGS(func, bpf_dynptr_slice, KF_RET_NULL) |
| 2413 | BTF_ID_FLAGS(func, bpf_dynptr_slice_rdwr, KF_RET_NULL) | 2413 | BTF_ID_FLAGS(func, bpf_dynptr_slice_rdwr, KF_RET_NULL) |
| 2414 | BTF_ID_FLAGS(func, bpf_iter_num_new, KF_ITER_NEW) | ||
| 2415 | BTF_ID_FLAGS(func, bpf_iter_num_next, KF_ITER_NEXT | KF_RET_NULL) | ||
| 2416 | BTF_ID_FLAGS(func, bpf_iter_num_destroy, KF_ITER_DESTROY) | ||
| 2414 | BTF_SET8_END(common_btf_ids) | 2417 | BTF_SET8_END(common_btf_ids) |
| 2415 | 2418 | ||
| 2416 | static const struct btf_kfunc_id_set common_kfunc_set = { | 2419 | static const struct btf_kfunc_id_set common_kfunc_set = { |
| @@ -7112,4 +7112,12 @@ enum { · 7111 unmodified lines | |||
| 7112 | BPF_F_TIMER_ABS = (1ULL << 0), | 7112 | BPF_F_TIMER_ABS = (1ULL << 0), |
| 7113 | }; | 7113 | }; |
| 7114 | 7114 | ||
| 7115 | /* BPF numbers iterator state */ | ||
| 7116 | struct bpf_iter_num { | ||
| 7117 | /* opaque iterator state; having __u64 here allows to preserve correct | ||
| 7118 | * alignment requirements in vmlinux.h, generated from BTF | ||
| 7119 | */ | ||
| 7120 | __u64 __opaque[1]; | ||
| 7121 | } __attribute__((aligned(8))); | ||
| 7122 | |||
| 7115 | #endif /* _UAPI__LINUX_BPF_H__ */ | 7123 | #endif /* _UAPI__LINUX_BPF_H__ */ |