History

Merge tag 'bpf-next-6.16' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Pull bpf updates from Alexei Starovoitov: - Fix and improve BTF deduplication of identical BTF types (Alan Maguire and Andrii Nakryiko) - Support up to 12 arguments in BPF trampoline on arm64 (Xu Kuohai and Alexis Lothoré) - Support load-acquire and store-release instructions in BPF JIT on riscv64 (Andrea Parri) - Fix uninitialized values in BPF_{CORE,PROBE}_READ macros (Anton Protopopov) - Streamline allowed helpers across program types (Feng Yang) - Support atomic update for hashtab of BPF maps (Hou Tao) - Implement json output for BPF helpers (Ihor Solodrai) - Several s390 JIT fixes (Ilya Leoshkevich) - Various sockmap fixes (Jiayuan Chen) - Support mmap of vmlinux BTF data (Lorenz Bauer) - Support BPF rbtree traversal and list peeking (Martin KaFai Lau) - Tests for sockmap/sockhash redirection (Michal Luczaj) - Introduce kfuncs for memory reads into dynptrs (Mykyta Yatsenko) - Add support for dma-buf iterators in BPF (T.J. Mercier) - The verifier support for __bpf_trap() (Yonghong Song) * tag 'bpf-next-6.16' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next: (135 commits) bpf, arm64: Remove unused-but-set function and variable. selftests/bpf: Add tests with stack ptr register in conditional jmp bpf: Do not include stack ptr register in precision backtracking bookkeeping selftests/bpf: enable many-args tests for arm64 bpf, arm64: Support up to 12 function arguments bpf: Check rcu_read_lock_trace_held() in bpf_map_lookup_percpu_elem() bpf: Avoid __bpf_prog_ret0_warn when jit fails bpftool: Add support for custom BTF path in prog load/loadall selftests/bpf: Add unit tests with __bpf_trap() kfunc bpf: Warn with __bpf_trap() kfunc maybe due to uninitialized variable bpf: Remove special_kfunc_set from verifier selftests/bpf: Add test for open coded dmabuf_iter selftests/bpf: Add test for dmabuf_iter bpf: Add open coded dmabuf iterator bpf: Add dmabuf iterator dma-buf: Rename debugfs symbols bpf: Fix error return value in bpf_copy_from_user_dynptr libbpf: Use mmap to parse vmlinux BTF from sysfs selftests: bpf: Add a test for mmapable vmlinux BTF btf: Allow mmap of vmlinux btf ...

Linus Torvalds committed 1 year ago

Commit 90b83ef2 parents 1b98f35 c5cebb2108 files · +5801 −1713Browse files

Unified · Split

108 files
  1. Documentation/bpf/bpf_iterators.rst +112 −5
  2. Documentation/bpf/kfuncs.rst +17 −0
  3. arch/arm64/net/bpf_jit_comp.c +171 −71
  4. arch/riscv/net/bpf_jit.h +15 −0
  5. arch/riscv/net/bpf_jit_comp64.c +227 −105
  6. arch/riscv/net/bpf_jit_core.c +1 −2
  7. arch/s390/include/asm/nospec-branch.h +0 −4
  8. arch/s390/net/bpf_jit_comp.c +66 −72
  9. drivers/dma-buf/dma-buf.c +78 −20
  10. include/asm-generic/vmlinux.lds.h +2 −1
  11. include/linux/bpf-cgroup.h +0 −8
  12. include/linux/bpf.h +20 −0
  13. include/linux/bpf_verifier.h +20 −4
  14. include/linux/dma-buf.h +2 −2
  15. include/uapi/linux/bpf.h +12 −7
  16. kernel/bpf/Makefile +3 −0
  17. kernel/bpf/bpf_struct_ops.c +1 −1
  18. kernel/bpf/btf.c +20 −25
  19. kernel/bpf/cgroup.c +0 −32
  20. kernel/bpf/core.c +17 −12
  21. kernel/bpf/dmabuf_iter.c +150 −0
  22. kernel/bpf/hashtab.c +72 −76
  23. kernel/bpf/helpers.c +118 −15
  24. kernel/bpf/syscall.c +6 −4
  25. kernel/bpf/sysfs_btf.c +32 −0
  26. kernel/bpf/verifier.c +334 −302
  27. kernel/sched/ext.c +1 −14
  28. kernel/trace/bpf_trace.c +211 −110
  29. kernel/trace/trace_uprobe.c +1 −1
  30. net/bpf/test_run.c +7 −1
  31. net/core/filter.c +0 −14
  32. net/core/skmsg.c +35 −21
  33. net/tls/tls_sw.c +13 −2
  34. scripts/Makefile.btf +2 −0
  35. scripts/bpf_doc.py +99 −20
  36. tools/bpf/bpftool/Documentation/bpftool-prog.rst +8 −2
  37. tools/bpf/bpftool/bash-completion/bpftool +2 −2
  38. tools/bpf/bpftool/cgroup.c +7 −7
  39. tools/bpf/bpftool/link.c +3 −0
  40. tools/bpf/bpftool/prog.c +11 −1
  41. tools/include/uapi/linux/bpf.h +12 −7
  42. tools/lib/bpf/bpf_core_read.h +6 −0
  43. tools/lib/bpf/bpf_helpers.h +8 −0
  44. tools/lib/bpf/btf.c +171 −55
  45. tools/lib/bpf/libbpf.c +48 −39
  46. tools/lib/bpf/libbpf.h +6 −0
  47. tools/lib/bpf/libbpf.map +4 −0
  48. tools/lib/bpf/libbpf_internal.h +9 −0
  49. tools/lib/bpf/linker.c +3 −3
  50. tools/lib/bpf/nlattr.c +7 −8
  51. tools/testing/selftests/bpf/DENYLIST +1 −0
  52. tools/testing/selftests/bpf/DENYLIST.aarch64 +0 −2
  53. tools/testing/selftests/bpf/Makefile +11 −5
  54. tools/testing/selftests/bpf/bench.c +4 −0
  55. tools/testing/selftests/bpf/benchs/bench_htab_mem.c +1 −2
  56. tools/testing/selftests/bpf/benchs/bench_sockmap.c +598 −0
  57. tools/testing/selftests/bpf/bpf_experimental.h +5 −0
  58. tools/testing/selftests/bpf/config +3 −0
  59. tools/testing/selftests/bpf/prog_tests/arena_spin_lock.c +8 −6
  60. tools/testing/selftests/bpf/prog_tests/attach_probe.c +84 −0
  61. tools/testing/selftests/bpf/prog_tests/bpf_nf.c +6 −0
  62. tools/testing/selftests/bpf/prog_tests/btf_dedup_split.c +101 −0
  63. tools/testing/selftests/bpf/prog_tests/btf_split.c +52 −6
  64. tools/testing/selftests/bpf/prog_tests/btf_sysfs.c +81 −0
  65. tools/testing/selftests/bpf/prog_tests/dmabuf_iter.c +285 −0
  66. tools/testing/selftests/bpf/prog_tests/dynptr.c +13 −0
  67. tools/testing/selftests/bpf/prog_tests/fd_htab_lookup.c +192 −0
  68. tools/testing/selftests/bpf/prog_tests/fill_link_info.c +16 −2
  69. tools/testing/selftests/bpf/prog_tests/kmem_cache_iter.c +1 −1
  70. tools/testing/selftests/bpf/prog_tests/linked_list.c +6 −0
  71. tools/testing/selftests/bpf/prog_tests/rbtree.c +6 −0
  72. tools/testing/selftests/bpf/prog_tests/sk_assign.c +3 −1
  73. tools/testing/selftests/bpf/prog_tests/socket_helpers.h +79 −5
  74. tools/testing/selftests/bpf/prog_tests/sockmap_helpers.h +11 −14
  75. tools/testing/selftests/bpf/prog_tests/sockmap_ktls.c +239 −58
  76. tools/testing/selftests/bpf/prog_tests/sockmap_listen.c +0 −457
  77. tools/testing/selftests/bpf/prog_tests/sockmap_redir.c +465 −0
  78. tools/testing/selftests/bpf/prog_tests/tc_redirect.c +6 −5
  79. tools/testing/selftests/bpf/prog_tests/test_btf_ext.c +64 −0
  80. tools/testing/selftests/bpf/prog_tests/test_veristat.c +5 −0
  81. tools/testing/selftests/bpf/prog_tests/verifier.c +2 −0
  82. tools/testing/selftests/bpf/prog_tests/xdp_metadata.c +21 −1
  83. tools/testing/selftests/bpf/progs/bench_sockmap_prog.c +65 −0
  84. tools/testing/selftests/bpf/progs/bpf_arena_spin_lock.h +12 −3
  85. tools/testing/selftests/bpf/progs/bpf_misc.h +3 −2
  86. tools/testing/selftests/bpf/progs/dmabuf_iter.c +101 −0
  87. tools/testing/selftests/bpf/progs/dynptr_success.c +230 −0
  88. tools/testing/selftests/bpf/progs/fd_htab_lookup.c +25 −0
  89. tools/testing/selftests/bpf/progs/iters.c +0 −2
  90. tools/testing/selftests/bpf/progs/linked_list_peek.c +113 −0
  91. tools/testing/selftests/bpf/progs/prepare.c +0 −1
  92. tools/testing/selftests/bpf/progs/rbtree_fail.c +15 −14
  93. tools/testing/selftests/bpf/progs/rbtree_search.c +206 −0
  94. tools/testing/selftests/bpf/progs/set_global_vars.c +41 −0
  95. tools/testing/selftests/bpf/progs/test_btf_ext.c +22 −0
  96. tools/testing/selftests/bpf/progs/test_sockmap_ktls.c +36 −0
  97. tools/testing/selftests/bpf/progs/test_sockmap_redir.c +68 −0
  98. tools/testing/selftests/bpf/progs/test_tcp_custom_syncookie.c +3 −1
  99. tools/testing/selftests/bpf/progs/verifier_bpf_trap.c +71 −0
  100. tools/testing/selftests/bpf/progs/verifier_btf_ctx_access.c +12 −0
  101. tools/testing/selftests/bpf/progs/verifier_load_acquire.c +32 −16
  102. tools/testing/selftests/bpf/progs/verifier_precision.c +55 −3
  103. tools/testing/selftests/bpf/progs/verifier_store_release.c +27 −12
  104. tools/testing/selftests/bpf/progs/xdp_metadata.c +13 −0
  105. tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +6 −2
  106. tools/testing/selftests/bpf/test_loader.c +8 −6
  107. tools/testing/selftests/bpf/test_verifier.c +4 −4
  108. tools/testing/selftests/bpf/veristat.c +94 −7
Documentation/bpf/bpf_iterators.rst+112−5
@@ -2,10 +2,117 @@ · 1 unmodified line
22 BPF Iterators
33 =============
44
5+--------
6+Overview
7+--------
8+
9+BPF supports two separate entities collectively known as "BPF iterators": BPF
10+iterator *program type* and *open-coded* BPF iterators. The former is
11+a stand-alone BPF program type which, when attached and activated by user,
12+will be called once for each entity (task_struct, cgroup, etc) that is being
13+iterated. The latter is a set of BPF-side APIs implementing iterator
14+functionality and available across multiple BPF program types. Open-coded
15+iterators provide similar functionality to BPF iterator programs, but gives
16+more flexibility and control to all other BPF program types. BPF iterator
17+programs, on the other hand, can be used to implement anonymous or BPF
18+FS-mounted special files, whose contents are generated by attached BPF iterator
19+program, backed by seq_file functionality. Both are useful depending on
20+specific needs.
21+
22+When adding a new BPF iterator program, it is expected that similar
23+functionality will be added as open-coded iterator for maximum flexibility.
24+It's also expected that iteration logic and code will be maximally shared and
25+reused between two iterator API surfaces.
526
6−----------
7−Motivation
8−----------
27+------------------------
28+Open-coded BPF Iterators
29+------------------------
30+
31+Open-coded BPF iterators are implemented as tightly-coupled trios of kfuncs
32+(constructor, next element fetch, destructor) and iterator-specific type
33+describing on-the-stack iterator state, which is guaranteed by the BPF
34+verifier to not be tampered with outside of the corresponding
35+constructor/destructor/next APIs.
36+
37+Each kind of open-coded BPF iterator has its own associated
38+struct bpf_iter_<type>, where <type> denotes a specific type of iterator.
39+bpf_iter_<type> state needs to live on BPF program stack, so make sure it's
40+small enough to fit on BPF stack. For performance reasons its best to avoid
41+dynamic memory allocation for iterator state and size the state struct big
42+enough to fit everything necessary. But if necessary, dynamic memory
43+allocation is a way to bypass BPF stack limitations. Note, state struct size
44+is part of iterator's user-visible API, so changing it will break backwards
45+compatibility, so be deliberate about designing it.
46+
47+All kfuncs (constructor, next, destructor) have to be named consistently as
48+bpf_iter_<type>_{new,next,destroy}(), respectively. <type> represents iterator
49+type, and iterator state should be represented as a matching
50+`struct bpf_iter_<type>` state type. Also, all iter kfuncs should have
51+a pointer to this `struct bpf_iter_<type>` as the very first argument.
52+
53+Additionally:
54+ - Constructor, i.e., `bpf_iter_<type>_new()`, can have arbitrary extra
55+ number of arguments. Return type is not enforced either.
56+ - Next method, i.e., `bpf_iter_<type>_next()`, has to return a pointer
57+ type and should have exactly one argument: `struct bpf_iter_<type> *`
58+ (const/volatile/restrict and typedefs are ignored).
59+ - Destructor, i.e., `bpf_iter_<type>_destroy()`, should return void and
60+ should have exactly one argument, similar to the next method.
61+ - `struct bpf_iter_<type>` size is enforced to be positive and
62+ a multiple of 8 bytes (to fit stack slots correctly).
63+
64+Such strictness and consistency allows to build generic helpers abstracting
65+important, but boilerplate, details to be able to use open-coded iterators
66+effectively and ergonomically (see libbpf's bpf_for_each() macro). This is
67+enforced at kfunc registration point by the kernel.
68+
69+Constructor/next/destructor implementation contract is as follows:
70+ - constructor, `bpf_iter_<type>_new()`, always initializes iterator state on
71+ the stack. If any of the input arguments are invalid, constructor should
72+ make sure to still initialize it such that subsequent next() calls will
73+ return NULL. I.e., on error, *return error and construct empty iterator*.
74+ Constructor kfunc is marked with KF_ITER_NEW flag.
75+
76+ - next method, `bpf_iter_<type>_next()`, accepts pointer to iterator state
77+ and produces an element. Next method should always return a pointer. The
78+ contract between BPF verifier is that next method *guarantees* that it
79+ will eventually return NULL when elements are exhausted. Once NULL is
80+ returned, subsequent next calls *should keep returning NULL*. Next method
81+ is marked with KF_ITER_NEXT (and should also have KF_RET_NULL as
82+ NULL-returning kfunc, of course).
83+
84+ - destructor, `bpf_iter_<type>_destroy()`, is always called once. Even if
85+ constructor failed or next returned nothing. Destructor frees up any
86+ resources and marks stack space used by `struct bpf_iter_<type>` as usable
87+ for something else. Destructor is marked with KF_ITER_DESTROY flag.
88+
89+Any open-coded BPF iterator implementation has to implement at least these
90+three methods. It is enforced that for any given type of iterator only
91+applicable constructor/destructor/next are callable. I.e., verifier ensures
92+you can't pass number iterator state into, say, cgroup iterator's next method.
93+
94+From a 10,000-feet BPF verification point of view, next methods are the points
95+of forking a verification state, which are conceptually similar to what
96+verifier is doing when validating conditional jumps. Verifier is branching out
97+`call bpf_iter_<type>_next` instruction and simulates two outcomes: NULL
98+(iteration is done) and non-NULL (new element is returned). NULL is simulated
99+first and is supposed to reach exit without looping. After that non-NULL case
100+is validated and it either reaches exit (for trivial examples with no real
101+loop), or reaches another `call bpf_iter_<type>_next` instruction with the
102+state equivalent to already (partially) validated one. State equivalency at
103+that point means we technically are going to be looping forever without
104+"breaking out" out of established "state envelope" (i.e., subsequent
105+iterations don't add any new knowledge or constraints to the verifier state,
106+so running 1, 2, 10, or a million of them doesn't matter). But taking into
107+account the contract stating that iterator next method *has to* return NULL
108+eventually, we can conclude that loop body is safe and will eventually
109+terminate. Given we validated logic outside of the loop (NULL case), and
110+concluded that loop body is safe (though potentially looping many times),
111+verifier can claim safety of the overall program logic.
112+
113+------------------------
114+BPF Iterators Motivation
115+------------------------
9116
10117 There are a few existing ways to dump kernel data into user space. The most
11118 popular one is the ``/proc`` system. For example, ``cat /proc/net/tcp6`` dumps
@@ -323,8 +430,8 @@ Now, in the userspace program, pass the pointer of struct to the · 311 unmodified lines
323430
324431 ::
325432
326− link = bpf_program__attach_iter(prog, &opts); iter_fd =
327− bpf_iter_create(bpf_link__fd(link));
433+ link = bpf_program__attach_iter(prog, &opts);
434+ iter_fd = bpf_iter_create(bpf_link__fd(link));
328435
329436 If both *tid* and *pid* are zero, an iterator created from this struct
330437 ``bpf_iter_attach_opts`` will include every opened file of every task in the
Documentation/bpf/kfuncs.rst+17−0
@@ -160,6 +160,23 @@ Or:: · 159 unmodified lines
160160 ...
161161 }
162162
163+2.2.6 __prog Annotation
164+---------------------------
165+This annotation is used to indicate that the argument needs to be fixed up to
166+the bpf_prog_aux of the caller BPF program. Any value passed into this argument
167+is ignored, and rewritten by the verifier.
168+
169+An example is given below::
170+
171+ __bpf_kfunc int bpf_wq_set_callback_impl(struct bpf_wq *wq,
172+ int (callback_fn)(void *map, int *key, void *value),
173+ unsigned int flags,
174+ void *aux__prog)
175+ {
176+ struct bpf_prog_aux *aux = aux__prog;
177+ ...
178+ }
179+
163180 .. _BPF_kfunc_nodef:
164181
165182 2.3 Using an existing kernel function
arch/arm64/net/bpf_jit_comp.c+171−71
@@ -2113,7 +2113,7 @@ bool bpf_jit_supports_subprog_tailcalls(void) · 2112 unmodified lines
21132113 }
21142114
21152115 static void invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_link *l,
2116− int args_off, int retval_off, int run_ctx_off,
2116+ int bargs_off, int retval_off, int run_ctx_off,
21172117 bool save_ret)
21182118 {
21192119 __le32 *branch;
@@ -2155,7 +2155,7 @@ static void invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_link *l, · 35 unmodified lines
21552155 branch = ctx->image + ctx->idx;
21562156 emit(A64_NOP, ctx);
21572157
2158− emit(A64_ADD_I(1, A64_R(0), A64_SP, args_off), ctx);
2158+ emit(A64_ADD_I(1, A64_R(0), A64_SP, bargs_off), ctx);
21592159 if (!p->jited)
21602160 emit_addr_mov_i64(A64_R(1), (const u64)p->insnsi, ctx);
21612161
@@ -2180,7 +2180,7 @@ static void invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_link *l, · 18 unmodified lines
21802180 }
21812181
21822182 static void invoke_bpf_mod_ret(struct jit_ctx *ctx, struct bpf_tramp_links *tl,
2183− int args_off, int retval_off, int run_ctx_off,
2183+ int bargs_off, int retval_off, int run_ctx_off,
21842184 __le32 **branches)
21852185 {
21862186 int i;
@@ -2190,7 +2190,7 @@ static void invoke_bpf_mod_ret(struct jit_ctx *ctx, struct bpf_tramp_links *tl, · 3 unmodified lines
21902190 */
21912191 emit(A64_STR64I(A64_ZR, A64_SP, retval_off), ctx);
21922192 for (i = 0; i < tl->nr_links; i++) {
2193− invoke_bpf_prog(ctx, tl->links[i], args_off, retval_off,
2193+ invoke_bpf_prog(ctx, tl->links[i], bargs_off, retval_off,
21942194 run_ctx_off, true);
21952195 /* if (*(u64 *)(sp + retval_off) != 0)
21962196 * goto do_fexit;
@@ -2204,23 +2204,125 @@ static void invoke_bpf_mod_ret(struct jit_ctx *ctx, struct bpf_tramp_links *tl, · 7 unmodified lines
22042204 }
22052205 }
22062206
2207−static void save_args(struct jit_ctx *ctx, int args_off, int nregs)
2207+struct arg_aux {
2208+ /* how many args are passed through registers, the rest of the args are
2209+ * passed through stack
2210+ */
2211+ int args_in_regs;
2212+ /* how many registers are used to pass arguments */
2213+ int regs_for_args;
2214+ /* how much stack is used for additional args passed to bpf program
2215+ * that did not fit in original function registers
2216+ */
2217+ int bstack_for_args;
2218+ /* home much stack is used for additional args passed to the
2219+ * original function when called from trampoline (this one needs
2220+ * arguments to be properly aligned)
2221+ */
2222+ int ostack_for_args;
2223+};
2224+
2225+static int calc_arg_aux(const struct btf_func_model *m,
2226+ struct arg_aux *a)
22082227 {
2209− int i;
2228+ int stack_slots, nregs, slots, i;
2229+
2230+ /* verifier ensures m->nr_args <= MAX_BPF_FUNC_ARGS */
2231+ for (i = 0, nregs = 0; i < m->nr_args; i++) {
2232+ slots = (m->arg_size[i] + 7) / 8;
2233+ if (nregs + slots <= 8) /* passed through register ? */
2234+ nregs += slots;
2235+ else
2236+ break;
2237+ }
2238+
2239+ a->args_in_regs = i;
2240+ a->regs_for_args = nregs;
2241+ a->ostack_for_args = 0;
2242+ a->bstack_for_args = 0;
2243+
2244+ /* the rest arguments are passed through stack */
2245+ for (; i < m->nr_args; i++) {
2246+ /* We can not know for sure about exact alignment needs for
2247+ * struct passed on stack, so deny those
2248+ */
2249+ if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG)
2250+ return -ENOTSUPP;
2251+ stack_slots = (m->arg_size[i] + 7) / 8;
2252+ a->bstack_for_args += stack_slots * 8;
2253+ a->ostack_for_args = a->ostack_for_args + stack_slots * 8;
2254+ }
2255+
2256+ return 0;
2257+}
22102258
2211− for (i = 0; i < nregs; i++) {
2212− emit(A64_STR64I(i, A64_SP, args_off), ctx);
2213− args_off += 8;
2259+static void clear_garbage(struct jit_ctx *ctx, int reg, int effective_bytes)
2260+{
2261+ if (effective_bytes) {
2262+ int garbage_bits = 64 - 8 * effective_bytes;
2263+#ifdef CONFIG_CPU_BIG_ENDIAN
2264+ /* garbage bits are at the right end */
2265+ emit(A64_LSR(1, reg, reg, garbage_bits), ctx);
2266+ emit(A64_LSL(1, reg, reg, garbage_bits), ctx);
2267+#else
2268+ /* garbage bits are at the left end */
2269+ emit(A64_LSL(1, reg, reg, garbage_bits), ctx);
2270+ emit(A64_LSR(1, reg, reg, garbage_bits), ctx);
2271+#endif
22142272 }
22152273 }
22162274
2217−static void restore_args(struct jit_ctx *ctx, int args_off, int nregs)
2275+static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
2276+ const struct btf_func_model *m,
2277+ const struct arg_aux *a,
2278+ bool for_call_origin)
22182279 {
22192280 int i;
2281+ int reg;
2282+ int doff;
2283+ int soff;
2284+ int slots;
2285+ u8 tmp = bpf2a64[TMP_REG_1];
2286+
2287+ /* store arguments to the stack for the bpf program, or restore
2288+ * arguments from stack for the original function
2289+ */
2290+ for (reg = 0; reg < a->regs_for_args; reg++) {
2291+ emit(for_call_origin ?
2292+ A64_LDR64I(reg, A64_SP, bargs_off) :
2293+ A64_STR64I(reg, A64_SP, bargs_off),
2294+ ctx);
2295+ bargs_off += 8;
2296+ }
2297+
2298+ soff = 32; /* on stack arguments start from FP + 32 */
2299+ doff = (for_call_origin ? oargs_off : bargs_off);
2300+
2301+ /* save on stack arguments */
2302+ for (i = a->args_in_regs; i < m->nr_args; i++) {
2303+ slots = (m->arg_size[i] + 7) / 8;
2304+ /* verifier ensures arg_size <= 16, so slots equals 1 or 2 */
2305+ while (slots-- > 0) {
2306+ emit(A64_LDR64I(tmp, A64_FP, soff), ctx);
2307+ /* if there is unused space in the last slot, clear
2308+ * the garbage contained in the space.
2309+ */
2310+ if (slots == 0 && !for_call_origin)
2311+ clear_garbage(ctx, tmp, m->arg_size[i] % 8);
2312+ emit(A64_STR64I(tmp, A64_SP, doff), ctx);
2313+ soff += 8;
2314+ doff += 8;
2315+ }
2316+ }
2317+}
22202318
2221− for (i = 0; i < nregs; i++) {
2222− emit(A64_LDR64I(i, A64_SP, args_off), ctx);
2223− args_off += 8;
2319+static void restore_args(struct jit_ctx *ctx, int bargs_off, int nregs)
2320+{
2321+ int reg;
2322+
2323+ for (reg = 0; reg < nregs; reg++) {
2324+ emit(A64_LDR64I(reg, A64_SP, bargs_off), ctx);
2325+ bargs_off += 8;
22242326 }
22252327 }
22262328
@@ -2243,17 +2345,21 @@ static bool is_struct_ops_tramp(const struct bpf_tramp_links *fentry_links) · 16 unmodified lines
22432345 */
22442346 static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
22452347 struct bpf_tramp_links *tlinks, void *func_addr,
2246− int nregs, u32 flags)
2348+ const struct btf_func_model *m,
2349+ const struct arg_aux *a,
2350+ u32 flags)
22472351 {
22482352 int i;
22492353 int stack_size;
22502354 int retaddr_off;
22512355 int regs_off;
22522356 int retval_off;
2253− int args_off;
2254− int nregs_off;
2357+ int bargs_off;
2358+ int nfuncargs_off;
22552359 int ip_off;
22562360 int run_ctx_off;
2361+ int oargs_off;
2362+ int nfuncargs;
22572363 struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];
22582364 struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];
22592365 struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];
@@ -2262,31 +2368,38 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, · 2 unmodified lines
22622368 bool is_struct_ops = is_struct_ops_tramp(fentry);
22632369
22642370 /* trampoline stack layout:
2265− * [ parent ip ]
2266− * [ FP ]
2267− * SP + retaddr_off [ self ip ]
2268− * [ FP ]
2371+ * [ parent ip ]
2372+ * [ FP ]
2373+ * SP + retaddr_off [ self ip ]
2374+ * [ FP ]
22692375 *
2270− * [ padding ] align SP to multiples of 16
2376+ * [ padding ] align SP to multiples of 16
22712377 *
2272− * [ x20 ] callee saved reg x20
2273− * SP + regs_off [ x19 ] callee saved reg x19
2378+ * [ x20 ] callee saved reg x20
2379+ * SP + regs_off [ x19 ] callee saved reg x19
22742380 *
2275− * SP + retval_off [ return value ] BPF_TRAMP_F_CALL_ORIG or
2276− * BPF_TRAMP_F_RET_FENTRY_RET
2381+ * SP + retval_off [ return value ] BPF_TRAMP_F_CALL_ORIG or
2382+ * BPF_TRAMP_F_RET_FENTRY_RET
2383+ * [ arg reg N ]
2384+ * [ ... ]
2385+ * SP + bargs_off [ arg reg 1 ] for bpf
22772386 *
2278− * [ arg reg N ]
2279− * [ ... ]
2280− * SP + args_off [ arg reg 1 ]
2387+ * SP + nfuncargs_off [ arg regs count ]
22812388 *
2282− * SP + nregs_off [ arg regs count ]
2389+ * SP + ip_off [ traced function ] BPF_TRAMP_F_IP_ARG flag
22832390 *
2284− * SP + ip_off [ traced function ] BPF_TRAMP_F_IP_ARG flag
2391+ * SP + run_ctx_off [ bpf_tramp_run_ctx ]
22852392 *
2286− * SP + run_ctx_off [ bpf_tramp_run_ctx ]
2393+ * [ stack arg N ]
2394+ * [ ... ]
2395+ * SP + oargs_off [ stack arg 1 ] for original func
22872396 */
22882397
22892398 stack_size = 0;
2399+ oargs_off = stack_size;
2400+ if (flags & BPF_TRAMP_F_CALL_ORIG)
2401+ stack_size += a->ostack_for_args;
2402+
22902403 run_ctx_off = stack_size;
22912404 /* room for bpf_tramp_run_ctx */
22922405 stack_size += round_up(sizeof(struct bpf_tramp_run_ctx), 8);
@@ -2296,13 +2409,14 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, · 3 unmodified lines
22962409 if (flags & BPF_TRAMP_F_IP_ARG)
22972410 stack_size += 8;
22982411
2299− nregs_off = stack_size;
2412+ nfuncargs_off = stack_size;
23002413 /* room for args count */
23012414 stack_size += 8;
23022415
2303− args_off = stack_size;
2416+ bargs_off = stack_size;
23042417 /* room for args */
2305− stack_size += nregs * 8;
2418+ nfuncargs = a->regs_for_args + a->bstack_for_args / 8;
2419+ stack_size += 8 * nfuncargs;
23062420
23072421 /* room for return value */
23082422 retval_off = stack_size;
@@ -2349,11 +2463,11 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, · 40 unmodified lines
23492463 }
23502464
23512465 /* save arg regs count*/
2352− emit(A64_MOVZ(1, A64_R(10), nregs, 0), ctx);
2353− emit(A64_STR64I(A64_R(10), A64_SP, nregs_off), ctx);
2466+ emit(A64_MOVZ(1, A64_R(10), nfuncargs, 0), ctx);
2467+ emit(A64_STR64I(A64_R(10), A64_SP, nfuncargs_off), ctx);
23542468
2355− /* save arg regs */
2356− save_args(ctx, args_off, nregs);
2469+ /* save args for bpf */
2470+ save_args(ctx, bargs_off, oargs_off, m, a, false);
23572471
23582472 /* save callee saved registers */
23592473 emit(A64_STR64I(A64_R(19), A64_SP, regs_off), ctx);
@@ -2369,7 +2483,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, · 9 unmodified lines
23692483 }
23702484
23712485 for (i = 0; i < fentry->nr_links; i++)
2372− invoke_bpf_prog(ctx, fentry->links[i], args_off,
2486+ invoke_bpf_prog(ctx, fentry->links[i], bargs_off,
23732487 retval_off, run_ctx_off,
23742488 flags & BPF_TRAMP_F_RET_FENTRY_RET);
23752489
@@ -2379,12 +2493,13 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, · 3 unmodified lines
23792493 if (!branches)
23802494 return -ENOMEM;
23812495
2382− invoke_bpf_mod_ret(ctx, fmod_ret, args_off, retval_off,
2496+ invoke_bpf_mod_ret(ctx, fmod_ret, bargs_off, retval_off,
23832497 run_ctx_off, branches);
23842498 }
23852499
23862500 if (flags & BPF_TRAMP_F_CALL_ORIG) {
2387− restore_args(ctx, args_off, nregs);
2501+ /* save args for original func */
2502+ save_args(ctx, bargs_off, oargs_off, m, a, true);
23882503 /* call original func */
23892504 emit(A64_LDR64I(A64_R(10), A64_SP, retaddr_off), ctx);
23902505 emit(A64_ADR(A64_LR, AARCH64_INSN_SIZE * 2), ctx);
@@ -2403,7 +2518,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, · 12 unmodified lines
24032518 }
24042519
24052520 for (i = 0; i < fexit->nr_links; i++)
2406− invoke_bpf_prog(ctx, fexit->links[i], args_off, retval_off,
2521+ invoke_bpf_prog(ctx, fexit->links[i], bargs_off, retval_off,
24072522 run_ctx_off, false);
24082523
24092524 if (flags & BPF_TRAMP_F_CALL_ORIG) {
@@ -2417,7 +2532,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, · 7 unmodified lines
24172532 }
24182533
24192534 if (flags & BPF_TRAMP_F_RESTORE_REGS)
2420− restore_args(ctx, args_off, nregs);
2535+ restore_args(ctx, bargs_off, a->regs_for_args);
24212536
24222537 /* restore callee saved register x19 and x20 */
24232538 emit(A64_LDR64I(A64_R(19), A64_SP, regs_off), ctx);
@@ -2454,21 +2569,6 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, · 30 unmodified lines
24542569 return ctx->idx;
24552570 }
24562571
2457−static int btf_func_model_nregs(const struct btf_func_model *m)
2458−{
2459− int nregs = m->nr_args;
2460− int i;
2461−
2462− /* extra registers needed for struct argument */
2463− for (i = 0; i < MAX_BPF_FUNC_ARGS; i++) {
2464− /* The arg_size is at most 16 bytes, enforced by the verifier. */
2465− if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG)
2466− nregs += (m->arg_size[i] + 7) / 8 - 1;
2467− }
2468−
2469− return nregs;
2470−}
2471−
24722572 int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
24732573 struct bpf_tramp_links *tlinks, void *func_addr)
24742574 {
@@ -2477,14 +2577,14 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, · 2 unmodified lines
24772577 .idx = 0,
24782578 };
24792579 struct bpf_tramp_image im;
2480− int nregs, ret;
2580+ struct arg_aux aaux;
2581+ int ret;
24812582
2482− nregs = btf_func_model_nregs(m);
2483− /* the first 8 registers are used for arguments */
2484− if (nregs > 8)
2485− return -ENOTSUPP;
2583+ ret = calc_arg_aux(m, &aaux);
2584+ if (ret < 0)
2585+ return ret;
24862586
2487− ret = prepare_trampoline(&ctx, &im, tlinks, func_addr, nregs, flags);
2587+ ret = prepare_trampoline(&ctx, &im, tlinks, func_addr, m, &aaux, flags);
24882588 if (ret < 0)
24892589 return ret;
24902590
@@ -2511,9 +2611,10 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *ro_image, · 20 unmodified lines
25112611 u32 flags, struct bpf_tramp_links *tlinks,
25122612 void *func_addr)
25132613 {
2514− int ret, nregs;
2515− void *image, *tmp;
25162614 u32 size = ro_image_end - ro_image;
2615+ struct arg_aux aaux;
2616+ void *image, *tmp;
2617+ int ret;
25172618
25182619 /* image doesn't need to be in module memory range, so we can
25192620 * use kvmalloc.
@@ -2529,13 +2630,12 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *ro_image, · 9 unmodified lines
25292630 .write = true,
25302631 };
25312632
2532− nregs = btf_func_model_nregs(m);
2533− /* the first 8 registers are used for arguments */
2534− if (nregs > 8)
2535− return -ENOTSUPP;
25362633
25372634 jit_fill_hole(image, (unsigned int)(ro_image_end - ro_image));
2538− ret = prepare_trampoline(&ctx, im, tlinks, func_addr, nregs, flags);
2635+ ret = calc_arg_aux(m, &aaux);
2636+ if (ret)
2637+ goto out;
2638+ ret = prepare_trampoline(&ctx, im, tlinks, func_addr, m, &aaux, flags);
25392639
25402640 if (ret > 0 && validate_code(&ctx) < 0) {
25412641 ret = -EINVAL;
arch/riscv/net/bpf_jit.h+15−0
@@ -608,6 +608,21 @@ static inline u32 rv_fence(u8 pred, u8 succ) · 607 unmodified lines
608608 return rv_i_insn(imm11_0, 0, 0, 0, 0xf);
609609 }
610610
611+static inline void emit_fence_r_rw(struct rv_jit_context *ctx)
612+{
613+ emit(rv_fence(0x2, 0x3), ctx);
614+}
615+
616+static inline void emit_fence_rw_w(struct rv_jit_context *ctx)
617+{
618+ emit(rv_fence(0x3, 0x1), ctx);
619+}
620+
621+static inline void emit_fence_rw_rw(struct rv_jit_context *ctx)
622+{
623+ emit(rv_fence(0x3, 0x3), ctx);
624+}
625+
611626 static inline u32 rv_nop(void)
612627 {
613628 return rv_i_insn(0, 0, 0, 0, 0x13);
arch/riscv/net/bpf_jit_comp64.c+227−105
@@ -473,11 +473,212 @@ static inline void emit_kcfi(u32 hash, struct rv_jit_context *ctx) · 472 unmodified lines
473473 emit(hash, ctx);
474474 }
475475
476−static void emit_atomic(u8 rd, u8 rs, s16 off, s32 imm, bool is64,
477− struct rv_jit_context *ctx)
476+static int emit_load_8(bool sign_ext, u8 rd, s32 off, u8 rs, struct rv_jit_context *ctx)
477+{
478+ int insns_start;
479+
480+ if (is_12b_int(off)) {
481+ insns_start = ctx->ninsns;
482+ if (sign_ext)
483+ emit(rv_lb(rd, off, rs), ctx);
484+ else
485+ emit(rv_lbu(rd, off, rs), ctx);
486+ return ctx->ninsns - insns_start;
487+ }
488+
489+ emit_imm(RV_REG_T1, off, ctx);
490+ emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
491+ insns_start = ctx->ninsns;
492+ if (sign_ext)
493+ emit(rv_lb(rd, 0, RV_REG_T1), ctx);
494+ else
495+ emit(rv_lbu(rd, 0, RV_REG_T1), ctx);
496+ return ctx->ninsns - insns_start;
497+}
498+
499+static int emit_load_16(bool sign_ext, u8 rd, s32 off, u8 rs, struct rv_jit_context *ctx)
500+{
501+ int insns_start;
502+
503+ if (is_12b_int(off)) {
504+ insns_start = ctx->ninsns;
505+ if (sign_ext)
506+ emit(rv_lh(rd, off, rs), ctx);
507+ else
508+ emit(rv_lhu(rd, off, rs), ctx);
509+ return ctx->ninsns - insns_start;
510+ }
511+
512+ emit_imm(RV_REG_T1, off, ctx);
513+ emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
514+ insns_start = ctx->ninsns;
515+ if (sign_ext)
516+ emit(rv_lh(rd, 0, RV_REG_T1), ctx);
517+ else
518+ emit(rv_lhu(rd, 0, RV_REG_T1), ctx);
519+ return ctx->ninsns - insns_start;
520+}
521+
522+static int emit_load_32(bool sign_ext, u8 rd, s32 off, u8 rs, struct rv_jit_context *ctx)
523+{
524+ int insns_start;
525+
526+ if (is_12b_int(off)) {
527+ insns_start = ctx->ninsns;
528+ if (sign_ext)
529+ emit(rv_lw(rd, off, rs), ctx);
530+ else
531+ emit(rv_lwu(rd, off, rs), ctx);
532+ return ctx->ninsns - insns_start;
533+ }
534+
535+ emit_imm(RV_REG_T1, off, ctx);
536+ emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
537+ insns_start = ctx->ninsns;
538+ if (sign_ext)
539+ emit(rv_lw(rd, 0, RV_REG_T1), ctx);
540+ else
541+ emit(rv_lwu(rd, 0, RV_REG_T1), ctx);
542+ return ctx->ninsns - insns_start;
543+}
544+
545+static int emit_load_64(bool sign_ext, u8 rd, s32 off, u8 rs, struct rv_jit_context *ctx)
546+{
547+ int insns_start;
548+
549+ if (is_12b_int(off)) {
550+ insns_start = ctx->ninsns;
551+ emit_ld(rd, off, rs, ctx);
552+ return ctx->ninsns - insns_start;
553+ }
554+
555+ emit_imm(RV_REG_T1, off, ctx);
556+ emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
557+ insns_start = ctx->ninsns;
558+ emit_ld(rd, 0, RV_REG_T1, ctx);
559+ return ctx->ninsns - insns_start;
560+}
561+
562+static void emit_store_8(u8 rd, s32 off, u8 rs, struct rv_jit_context *ctx)
563+{
564+ if (is_12b_int(off)) {
565+ emit(rv_sb(rd, off, rs), ctx);
566+ return;
567+ }
568+
569+ emit_imm(RV_REG_T1, off, ctx);
570+ emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
571+ emit(rv_sb(RV_REG_T1, 0, rs), ctx);
572+}
573+
574+static void emit_store_16(u8 rd, s32 off, u8 rs, struct rv_jit_context *ctx)
575+{
576+ if (is_12b_int(off)) {
577+ emit(rv_sh(rd, off, rs), ctx);
578+ return;
579+ }
580+
581+ emit_imm(RV_REG_T1, off, ctx);
582+ emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
583+ emit(rv_sh(RV_REG_T1, 0, rs), ctx);
584+}
585+
586+static void emit_store_32(u8 rd, s32 off, u8 rs, struct rv_jit_context *ctx)
587+{
588+ if (is_12b_int(off)) {
589+ emit_sw(rd, off, rs, ctx);
590+ return;
591+ }
592+
593+ emit_imm(RV_REG_T1, off, ctx);
594+ emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
595+ emit_sw(RV_REG_T1, 0, rs, ctx);
596+}
597+
598+static void emit_store_64(u8 rd, s32 off, u8 rs, struct rv_jit_context *ctx)
599+{
600+ if (is_12b_int(off)) {
601+ emit_sd(rd, off, rs, ctx);
602+ return;
603+ }
604+
605+ emit_imm(RV_REG_T1, off, ctx);
606+ emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
607+ emit_sd(RV_REG_T1, 0, rs, ctx);
608+}
609+
610+static int emit_atomic_ld_st(u8 rd, u8 rs, const struct bpf_insn *insn,
611+ struct rv_jit_context *ctx)
612+{
613+ u8 code = insn->code;
614+ s32 imm = insn->imm;
615+ s16 off = insn->off;
616+
617+ switch (imm) {
618+ /* dst_reg = load_acquire(src_reg + off16) */
619+ case BPF_LOAD_ACQ:
620+ switch (BPF_SIZE(code)) {
621+ case BPF_B:
622+ emit_load_8(false, rd, off, rs, ctx);
623+ break;
624+ case BPF_H:
625+ emit_load_16(false, rd, off, rs, ctx);
626+ break;
627+ case BPF_W:
628+ emit_load_32(false, rd, off, rs, ctx);
629+ break;
630+ case BPF_DW:
631+ emit_load_64(false, rd, off, rs, ctx);
632+ break;
633+ }
634+ emit_fence_r_rw(ctx);
635+
636+ /* If our next insn is a redundant zext, return 1 to tell
637+ * build_body() to skip it.
638+ */
639+ if (BPF_SIZE(code) != BPF_DW && insn_is_zext(&insn[1]))
640+ return 1;
641+ break;
642+ /* store_release(dst_reg + off16, src_reg) */
643+ case BPF_STORE_REL:
644+ emit_fence_rw_w(ctx);
645+ switch (BPF_SIZE(code)) {
646+ case BPF_B:
647+ emit_store_8(rd, off, rs, ctx);
648+ break;
649+ case BPF_H:
650+ emit_store_16(rd, off, rs, ctx);
651+ break;
652+ case BPF_W:
653+ emit_store_32(rd, off, rs, ctx);
654+ break;
655+ case BPF_DW:
656+ emit_store_64(rd, off, rs, ctx);
657+ break;
658+ }
659+ break;
660+ default:
661+ pr_err_once("bpf-jit: invalid atomic load/store opcode %02x\n", imm);
662+ return -EINVAL;
663+ }
664+
665+ return 0;
666+}
667+
668+static int emit_atomic_rmw(u8 rd, u8 rs, const struct bpf_insn *insn,
669+ struct rv_jit_context *ctx)
478670 {
479− u8 r0;
671+ u8 r0, code = insn->code;
672+ s16 off = insn->off;
673+ s32 imm = insn->imm;
480674 int jmp_offset;
675+ bool is64;
676+
677+ if (BPF_SIZE(code) != BPF_W && BPF_SIZE(code) != BPF_DW) {
678+ pr_err_once("bpf-jit: 1- and 2-byte RMW atomics are not supported\n");
679+ return -EINVAL;
680+ }
681+ is64 = BPF_SIZE(code) == BPF_DW;
481682
482683 if (off) {
483684 if (is_12b_int(off)) {
@@ -554,9 +755,14 @@ static void emit_atomic(u8 rd, u8 rs, s16 off, s32 imm, bool is64, · 70 unmodified lines
554755 rv_sc_w(RV_REG_T3, rs, rd, 0, 1), ctx);
555756 jmp_offset = ninsns_rvoff(-6);
556757 emit(rv_bne(RV_REG_T3, 0, jmp_offset >> 1), ctx);
557− emit(rv_fence(0x3, 0x3), ctx);
758+ emit_fence_rw_rw(ctx);
558759 break;
760+ default:
761+ pr_err_once("bpf-jit: invalid atomic RMW opcode %02x\n", imm);
762+ return -EINVAL;
559763 }
764+
765+ return 0;
560766 }
561767
562768 #define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0)
@@ -1650,8 +1856,8 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, · 1087 unmodified lines
16501856 case BPF_LDX | BPF_PROBE_MEM32 | BPF_W:
16511857 case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW:
16521858 {
1653− int insn_len, insns_start;
16541859 bool sign_ext;
1860+ int insn_len;
16551861
16561862 sign_ext = BPF_MODE(insn->code) == BPF_MEMSX ||
16571863 BPF_MODE(insn->code) == BPF_PROBE_MEMSX;
@@ -1663,78 +1869,16 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, · 5 unmodified lines
16631869
16641870 switch (BPF_SIZE(code)) {
16651871 case BPF_B:
1666− if (is_12b_int(off)) {
1667− insns_start = ctx->ninsns;
1668− if (sign_ext)
1669− emit(rv_lb(rd, off, rs), ctx);
1670− else
1671− emit(rv_lbu(rd, off, rs), ctx);
1672− insn_len = ctx->ninsns - insns_start;
1673− break;
1674− }
1675−
1676− emit_imm(RV_REG_T1, off, ctx);
1677− emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
1678− insns_start = ctx->ninsns;
1679− if (sign_ext)
1680− emit(rv_lb(rd, 0, RV_REG_T1), ctx);
1681− else
1682− emit(rv_lbu(rd, 0, RV_REG_T1), ctx);
1683− insn_len = ctx->ninsns - insns_start;
1872+ insn_len = emit_load_8(sign_ext, rd, off, rs, ctx);
16841873 break;
16851874 case BPF_H:
1686− if (is_12b_int(off)) {
1687− insns_start = ctx->ninsns;
1688− if (sign_ext)
1689− emit(rv_lh(rd, off, rs), ctx);
1690− else
1691− emit(rv_lhu(rd, off, rs), ctx);
1692− insn_len = ctx->ninsns - insns_start;
1693− break;
1694− }
1695−
1696− emit_imm(RV_REG_T1, off, ctx);
1697− emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
1698− insns_start = ctx->ninsns;
1699− if (sign_ext)
1700− emit(rv_lh(rd, 0, RV_REG_T1), ctx);
1701− else
1702− emit(rv_lhu(rd, 0, RV_REG_T1), ctx);
1703− insn_len = ctx->ninsns - insns_start;
1875+ insn_len = emit_load_16(sign_ext, rd, off, rs, ctx);
17041876 break;
17051877 case BPF_W:
1706− if (is_12b_int(off)) {
1707− insns_start = ctx->ninsns;
1708− if (sign_ext)
1709− emit(rv_lw(rd, off, rs), ctx);
1710− else
1711− emit(rv_lwu(rd, off, rs), ctx);
1712− insn_len = ctx->ninsns - insns_start;
1713− break;
1714− }
1715−
1716− emit_imm(RV_REG_T1, off, ctx);
1717− emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
1718− insns_start = ctx->ninsns;
1719− if (sign_ext)
1720− emit(rv_lw(rd, 0, RV_REG_T1), ctx);
1721− else
1722− emit(rv_lwu(rd, 0, RV_REG_T1), ctx);
1723− insn_len = ctx->ninsns - insns_start;
1878+ insn_len = emit_load_32(sign_ext, rd, off, rs, ctx);
17241879 break;
17251880 case BPF_DW:
1726− if (is_12b_int(off)) {
1727− insns_start = ctx->ninsns;
1728− emit_ld(rd, off, rs, ctx);
1729− insn_len = ctx->ninsns - insns_start;
1730− break;
1731− }
1732−
1733− emit_imm(RV_REG_T1, off, ctx);
1734− emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
1735− insns_start = ctx->ninsns;
1736− emit_ld(rd, 0, RV_REG_T1, ctx);
1737− insn_len = ctx->ninsns - insns_start;
1881+ insn_len = emit_load_64(sign_ext, rd, off, rs, ctx);
17381882 break;
17391883 }
17401884
@@ -1879,49 +2023,27 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, · 138 unmodified lines
18792023
18802024 /* STX: *(size *)(dst + off) = src */
18812025 case BPF_STX | BPF_MEM | BPF_B:
1882− if (is_12b_int(off)) {
1883− emit(rv_sb(rd, off, rs), ctx);
1884− break;
1885− }
1886−
1887− emit_imm(RV_REG_T1, off, ctx);
1888− emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
1889− emit(rv_sb(RV_REG_T1, 0, rs), ctx);
2026+ emit_store_8(rd, off, rs, ctx);
18902027 break;
18912028 case BPF_STX | BPF_MEM | BPF_H:
1892− if (is_12b_int(off)) {
1893− emit(rv_sh(rd, off, rs), ctx);
1894− break;
1895− }
1896−
1897− emit_imm(RV_REG_T1, off, ctx);
1898− emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
1899− emit(rv_sh(RV_REG_T1, 0, rs), ctx);
2029+ emit_store_16(rd, off, rs, ctx);
19002030 break;
19012031 case BPF_STX | BPF_MEM | BPF_W:
1902− if (is_12b_int(off)) {
1903− emit_sw(rd, off, rs, ctx);
1904− break;
1905− }
1906−
1907− emit_imm(RV_REG_T1, off, ctx);
1908− emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
1909− emit_sw(RV_REG_T1, 0, rs, ctx);
2032+ emit_store_32(rd, off, rs, ctx);
19102033 break;
19112034 case BPF_STX | BPF_MEM | BPF_DW:
1912− if (is_12b_int(off)) {
1913− emit_sd(rd, off, rs, ctx);
1914− break;
1915− }
1916−
1917− emit_imm(RV_REG_T1, off, ctx);
1918− emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
1919− emit_sd(RV_REG_T1, 0, rs, ctx);
2035+ emit_store_64(rd, off, rs, ctx);
19202036 break;
2037+ case BPF_STX | BPF_ATOMIC | BPF_B:
2038+ case BPF_STX | BPF_ATOMIC | BPF_H:
19212039 case BPF_STX | BPF_ATOMIC | BPF_W:
19222040 case BPF_STX | BPF_ATOMIC | BPF_DW:
1923− emit_atomic(rd, rs, off, imm,
1924− BPF_SIZE(code) == BPF_DW, ctx);
2041+ if (bpf_atomic_is_load_store(insn))
2042+ ret = emit_atomic_ld_st(rd, rs, insn, ctx);
2043+ else
2044+ ret = emit_atomic_rmw(rd, rs, insn, ctx);
2045+ if (ret)
2046+ return ret;
19252047 break;
19262048
19272049 case BPF_STX | BPF_PROBE_MEM32 | BPF_B:
arch/riscv/net/bpf_jit_core.c+1−2
@@ -26,9 +26,8 @@ static int build_body(struct rv_jit_context *ctx, bool extra_pass, int *offset) · 25 unmodified lines
2626 int ret;
2727
2828 ret = bpf_jit_emit_insn(insn, ctx, extra_pass);
29− /* BPF_LD | BPF_IMM | BPF_DW: skip the next instruction. */
3029 if (ret > 0)
31− i++;
30+ i++; /* skip the next instruction */
3231 if (offset)
3332 offset[i] = ctx->ninsns;
3433 if (ret < 0)
arch/s390/include/asm/nospec-branch.h+0−4
@@ -26,8 +26,6 @@ static inline bool nospec_uses_trampoline(void) · 25 unmodified lines
2626 return __is_defined(CC_USING_EXPOLINE) && !nospec_disable;
2727 }
2828
29−#ifdef CONFIG_EXPOLINE_EXTERN
30−
3129 void __s390_indirect_jump_r1(void);
3230 void __s390_indirect_jump_r2(void);
3331 void __s390_indirect_jump_r3(void);
@@ -44,8 +42,6 @@ void __s390_indirect_jump_r13(void); · 10 unmodified lines
4442 void __s390_indirect_jump_r14(void);
4543 void __s390_indirect_jump_r15(void);
4644
47−#endif
48−
4945 #endif /* __ASSEMBLY__ */
5046
5147 #endif /* _ASM_S390_EXPOLINE_H */
arch/s390/net/bpf_jit_comp.c+66−72
@@ -48,8 +48,6 @@ struct bpf_jit { · 47 unmodified lines
4848 int lit64; /* Current position in 64-bit literal pool */
4949 int base_ip; /* Base address for literal pool */
5050 int exit_ip; /* Address of exit */
51− int r1_thunk_ip; /* Address of expoline thunk for 'br %r1' */
52− int r14_thunk_ip; /* Address of expoline thunk for 'br %r14' */
5351 int tail_call_start; /* Tail call start offset */
5452 int excnt; /* Number of exception table entries */
5553 int prologue_plt_ret; /* Return address for prologue hotpatch PLT */
@@ -127,6 +125,18 @@ static inline void reg_set_seen(struct bpf_jit *jit, u32 b1) · 71 unmodified lines
127125 jit->seen_regs |= (1 << r1);
128126 }
129127
128+static s32 off_to_pcrel(struct bpf_jit *jit, u32 off)
129+{
130+ return off - jit->prg;
131+}
132+
133+static s64 ptr_to_pcrel(struct bpf_jit *jit, const void *ptr)
134+{
135+ if (jit->prg_buf)
136+ return (const u8 *)ptr - ((const u8 *)jit->prg_buf + jit->prg);
137+ return 0;
138+}
139+
130140 #define REG_SET_SEEN(b1) \
131141 ({ \
132142 reg_set_seen(jit, b1); \
@@ -201,7 +211,7 @@ static inline void reg_set_seen(struct bpf_jit *jit, u32 b1) · 68 unmodified lines
201211
202212 #define EMIT4_PCREL_RIC(op, mask, target) \
203213 ({ \
204− int __rel = ((target) - jit->prg) / 2; \
214+ int __rel = off_to_pcrel(jit, target) / 2; \
205215 _EMIT4((op) | (mask) << 20 | (__rel & 0xffff)); \
206216 })
207217
@@ -239,7 +249,7 @@ static inline void reg_set_seen(struct bpf_jit *jit, u32 b1) · 31 unmodified lines
239249
240250 #define EMIT6_PCREL_RIEB(op1, op2, b1, b2, mask, target) \
241251 ({ \
242− unsigned int rel = (int)((target) - jit->prg) / 2; \
252+ unsigned int rel = off_to_pcrel(jit, target) / 2; \
243253 _EMIT6((op1) | reg(b1, b2) << 16 | (rel & 0xffff), \
244254 (op2) | (mask) << 12); \
245255 REG_SET_SEEN(b1); \
@@ -248,7 +258,7 @@ static inline void reg_set_seen(struct bpf_jit *jit, u32 b1) · 2 unmodified lines
248258
249259 #define EMIT6_PCREL_RIEC(op1, op2, b1, imm, mask, target) \
250260 ({ \
251− unsigned int rel = (int)((target) - jit->prg) / 2; \
261+ unsigned int rel = off_to_pcrel(jit, target) / 2; \
252262 _EMIT6((op1) | (reg_high(b1) | (mask)) << 16 | \
253263 (rel & 0xffff), (op2) | ((imm) & 0xff) << 8); \
254264 REG_SET_SEEN(b1); \
@@ -257,29 +267,41 @@ static inline void reg_set_seen(struct bpf_jit *jit, u32 b1) · 2 unmodified lines
257267
258268 #define EMIT6_PCREL(op1, op2, b1, b2, i, off, mask) \
259269 ({ \
260− int rel = (addrs[(i) + (off) + 1] - jit->prg) / 2; \
270+ int rel = off_to_pcrel(jit, addrs[(i) + (off) + 1]) / 2;\
261271 _EMIT6((op1) | reg(b1, b2) << 16 | (rel & 0xffff), (op2) | (mask));\
262272 REG_SET_SEEN(b1); \
263273 REG_SET_SEEN(b2); \
264274 })
265275
276+static void emit6_pcrel_ril(struct bpf_jit *jit, u32 op, s64 pcrel)
277+{
278+ u32 pc32dbl = (s32)(pcrel / 2);
279+
280+ _EMIT6(op | pc32dbl >> 16, pc32dbl & 0xffff);
281+}
282+
283+static void emit6_pcrel_rilb(struct bpf_jit *jit, u32 op, u8 b, s64 pcrel)
284+{
285+ emit6_pcrel_ril(jit, op | reg_high(b) << 16, pcrel);
286+ REG_SET_SEEN(b);
287+}
288+
266289 #define EMIT6_PCREL_RILB(op, b, target) \
267−({ \
268− unsigned int rel = (int)((target) - jit->prg) / 2; \
269− _EMIT6((op) | reg_high(b) << 16 | rel >> 16, rel & 0xffff);\
270− REG_SET_SEEN(b); \
271−})
290+ emit6_pcrel_rilb(jit, op, b, off_to_pcrel(jit, target))
272291
273−#define EMIT6_PCREL_RIL(op, target) \
274−({ \
275− unsigned int rel = (int)((target) - jit->prg) / 2; \
276− _EMIT6((op) | rel >> 16, rel & 0xffff); \
277−})
292+#define EMIT6_PCREL_RILB_PTR(op, b, target_ptr) \
293+ emit6_pcrel_rilb(jit, op, b, ptr_to_pcrel(jit, target_ptr))
294+
295+static void emit6_pcrel_rilc(struct bpf_jit *jit, u32 op, u8 mask, s64 pcrel)
296+{
297+ emit6_pcrel_ril(jit, op | mask << 20, pcrel);
298+}
278299
279300 #define EMIT6_PCREL_RILC(op, mask, target) \
280−({ \
281− EMIT6_PCREL_RIL((op) | (mask) << 20, (target)); \
282−})
301+ emit6_pcrel_rilc(jit, op, mask, off_to_pcrel(jit, target))
302+
303+#define EMIT6_PCREL_RILC_PTR(op, mask, target_ptr) \
304+ emit6_pcrel_rilc(jit, op, mask, ptr_to_pcrel(jit, target_ptr))
283305
284306 #define _EMIT6_IMM(op, imm) \
285307 ({ \
@@ -503,7 +525,7 @@ static void bpf_skip(struct bpf_jit *jit, int size) · 217 unmodified lines
503525 {
504526 if (size >= 6 && !is_valid_rel(size)) {
505527 /* brcl 0xf,size */
506− EMIT6_PCREL_RIL(0xc0f4000000, size);
528+ EMIT6_PCREL_RILC(0xc0040000, 0xf, size);
507529 size -= 6;
508530 } else if (size >= 4 && is_valid_rel(size)) {
509531 /* brc 0xf,size */
@@ -605,43 +627,30 @@ static void bpf_jit_prologue(struct bpf_jit *jit, struct bpf_prog *fp, · 95 unmodified lines
605627 }
606628 /* Setup stack and backchain */
607629 if (is_first_pass(jit) || (jit->seen & SEEN_STACK)) {
608− if (is_first_pass(jit) || (jit->seen & SEEN_FUNC))
609− /* lgr %w1,%r15 (backchain) */
610− EMIT4(0xb9040000, REG_W1, REG_15);
630+ /* lgr %w1,%r15 (backchain) */
631+ EMIT4(0xb9040000, REG_W1, REG_15);
611632 /* la %bfp,STK_160_UNUSED(%r15) (BPF frame pointer) */
612633 EMIT4_DISP(0x41000000, BPF_REG_FP, REG_15, STK_160_UNUSED);
613634 /* aghi %r15,-STK_OFF */
614635 EMIT4_IMM(0xa70b0000, REG_15, -(STK_OFF + stack_depth));
615− if (is_first_pass(jit) || (jit->seen & SEEN_FUNC))
616− /* stg %w1,152(%r15) (backchain) */
617− EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0,
618− REG_15, 152);
636+ /* stg %w1,152(%r15) (backchain) */
637+ EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0,
638+ REG_15, 152);
619639 }
620640 }
621641
622642 /*
623− * Emit an expoline for a jump that follows
643+ * Jump using a register either directly or via an expoline thunk
624644 */
625−static void emit_expoline(struct bpf_jit *jit)
626−{
627− /* exrl %r0,.+10 */
628− EMIT6_PCREL_RIL(0xc6000000, jit->prg + 10);
629− /* j . */
630− EMIT4_PCREL(0xa7f40000, 0);
631−}
632−
633−/*
634− * Emit __s390_indirect_jump_r1 thunk if necessary
635− */
636−static void emit_r1_thunk(struct bpf_jit *jit)
637−{
638− if (nospec_uses_trampoline()) {
639− jit->r1_thunk_ip = jit->prg;
640− emit_expoline(jit);
641− /* br %r1 */
642− _EMIT2(0x07f1);
643− }
644−}
645+#define EMIT_JUMP_REG(reg) do { \
646+ if (nospec_uses_trampoline()) \
647+ /* brcl 0xf,__s390_indirect_jump_rN */ \
648+ EMIT6_PCREL_RILC_PTR(0xc0040000, 0x0f, \
649+ __s390_indirect_jump_r ## reg); \
650+ else \
651+ /* br %rN */ \
652+ _EMIT2(0x07f0 | reg); \
653+} while (0)
645654
646655 /*
647656 * Call r1 either directly or via __s390_indirect_jump_r1 thunk
@@ -650,7 +659,8 @@ static void call_r1(struct bpf_jit *jit) · 2 unmodified lines
650659 {
651660 if (nospec_uses_trampoline())
652661 /* brasl %r14,__s390_indirect_jump_r1 */
653− EMIT6_PCREL_RILB(0xc0050000, REG_14, jit->r1_thunk_ip);
662+ EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14,
663+ __s390_indirect_jump_r1);
654664 else
655665 /* basr %r14,%r1 */
656666 EMIT2(0x0d00, REG_14, REG_1);
@@ -666,16 +676,7 @@ static void bpf_jit_epilogue(struct bpf_jit *jit, u32 stack_depth) · 9 unmodified lines
666676 EMIT4(0xb9040000, REG_2, BPF_REG_0);
667677 /* Restore registers */
668678 save_restore_regs(jit, REGS_RESTORE, stack_depth, 0);
669− if (nospec_uses_trampoline()) {
670− jit->r14_thunk_ip = jit->prg;
671− /* Generate __s390_indirect_jump_r14 thunk */
672− emit_expoline(jit);
673− }
674− /* br %r14 */
675− _EMIT2(0x07fe);
676−
677− if (is_first_pass(jit) || (jit->seen & SEEN_FUNC))
678− emit_r1_thunk(jit);
679+ EMIT_JUMP_REG(14);
679680
680681 jit->prg = ALIGN(jit->prg, 8);
681682 jit->prologue_plt = jit->prg;
@@ -1877,7 +1878,8 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, · 1195 unmodified lines
18771878 /* aghi %r1,tail_call_start */
18781879 EMIT4_IMM(0xa70b0000, REG_1, jit->tail_call_start);
18791880 /* brcl 0xf,__s390_indirect_jump_r1 */
1880− EMIT6_PCREL_RILC(0xc0040000, 0xf, jit->r1_thunk_ip);
1881+ EMIT6_PCREL_RILC_PTR(0xc0040000, 0xf,
1882+ __s390_indirect_jump_r1);
18811883 } else {
18821884 /* bc 0xf,tail_call_start(%r1) */
18831885 _EMIT4(0x47f01000 + jit->tail_call_start);
@@ -2585,9 +2587,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, · 701 unmodified lines
25852587 if (nr_stack_args > MAX_NR_STACK_ARGS)
25862588 return -ENOTSUPP;
25872589
2588− /* Return to %r14, since func_addr and %r0 are not available. */
2589− if ((!func_addr && !(flags & BPF_TRAMP_F_ORIG_STACK)) ||
2590− (flags & BPF_TRAMP_F_INDIRECT))
2590+ /* Return to %r14 in the struct_ops case. */
2591+ if (flags & BPF_TRAMP_F_INDIRECT)
25912592 flags |= BPF_TRAMP_F_SKIP_FRAME;
25922593
25932594 /*
@@ -2847,17 +2848,10 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, · 253 unmodified lines
28472848 0xf000 | tjit->tccnt_off);
28482849 /* aghi %r15,stack_size */
28492850 EMIT4_IMM(0xa70b0000, REG_15, tjit->stack_size);
2850− /* Emit an expoline for the following indirect jump. */
2851− if (nospec_uses_trampoline())
2852− emit_expoline(jit);
28532851 if (flags & BPF_TRAMP_F_SKIP_FRAME)
2854− /* br %r14 */
2855− _EMIT2(0x07fe);
2852+ EMIT_JUMP_REG(14);
28562853 else
2857− /* br %r1 */
2858− _EMIT2(0x07f1);
2859−
2860− emit_r1_thunk(jit);
2854+ EMIT_JUMP_REG(1);
28612855
28622856 return 0;
28632857 }
drivers/dma-buf/dma-buf.c+78−20
@@ -19,7 +19,9 @@ · 18 unmodified lines
1919 #include <linux/anon_inodes.h>
2020 #include <linux/export.h>
2121 #include <linux/debugfs.h>
22+#include <linux/list.h>
2223 #include <linux/module.h>
24+#include <linux/mutex.h>
2325 #include <linux/seq_file.h>
2426 #include <linux/sync_file.h>
2527 #include <linux/poll.h>
@@ -35,35 +37,91 @@ · 9 unmodified lines
3537
3638 static inline int is_dma_buf_file(struct file *);
3739
38−#if IS_ENABLED(CONFIG_DEBUG_FS)
39−static DEFINE_MUTEX(debugfs_list_mutex);
40−static LIST_HEAD(debugfs_list);
40+static DEFINE_MUTEX(dmabuf_list_mutex);
41+static LIST_HEAD(dmabuf_list);
4142
42−static void __dma_buf_debugfs_list_add(struct dma_buf *dmabuf)
43+static void __dma_buf_list_add(struct dma_buf *dmabuf)
4344 {
44− mutex_lock(&debugfs_list_mutex);
45− list_add(&dmabuf->list_node, &debugfs_list);
46− mutex_unlock(&debugfs_list_mutex);
45+ mutex_lock(&dmabuf_list_mutex);
46+ list_add(&dmabuf->list_node, &dmabuf_list);
47+ mutex_unlock(&dmabuf_list_mutex);
4748 }
4849
49−static void __dma_buf_debugfs_list_del(struct dma_buf *dmabuf)
50+static void __dma_buf_list_del(struct dma_buf *dmabuf)
5051 {
5152 if (!dmabuf)
5253 return;
5354
54− mutex_lock(&debugfs_list_mutex);
55+ mutex_lock(&dmabuf_list_mutex);
5556 list_del(&dmabuf->list_node);
56− mutex_unlock(&debugfs_list_mutex);
57+ mutex_unlock(&dmabuf_list_mutex);
5758 }
58−#else
59−static void __dma_buf_debugfs_list_add(struct dma_buf *dmabuf)
59+
60+/**
61+ * dma_buf_iter_begin - begin iteration through global list of all DMA buffers
62+ *
63+ * Returns the first buffer in the global list of DMA-bufs that's not in the
64+ * process of being destroyed. Increments that buffer's reference count to
65+ * prevent buffer destruction. Callers must release the reference, either by
66+ * continuing iteration with dma_buf_iter_next(), or with dma_buf_put().
67+ *
68+ * Return:
69+ * * First buffer from global list, with refcount elevated
70+ * * NULL if no active buffers are present
71+ */
72+struct dma_buf *dma_buf_iter_begin(void)
6073 {
74+ struct dma_buf *ret = NULL, *dmabuf;
75+
76+ /*
77+ * The list mutex does not protect a dmabuf's refcount, so it can be
78+ * zeroed while we are iterating. We cannot call get_dma_buf() since the
79+ * caller may not already own a reference to the buffer.
80+ */
81+ mutex_lock(&dmabuf_list_mutex);
82+ list_for_each_entry(dmabuf, &dmabuf_list, list_node) {
83+ if (file_ref_get(&dmabuf->file->f_ref)) {
84+ ret = dmabuf;
85+ break;
86+ }
87+ }
88+ mutex_unlock(&dmabuf_list_mutex);
89+ return ret;
6190 }
6291
63−static void __dma_buf_debugfs_list_del(struct dma_buf *dmabuf)
92+/**
93+ * dma_buf_iter_next - continue iteration through global list of all DMA buffers
94+ * @dmabuf: [in] pointer to dma_buf
95+ *
96+ * Decrements the reference count on the provided buffer. Returns the next
97+ * buffer from the remainder of the global list of DMA-bufs with its reference
98+ * count incremented. Callers must release the reference, either by continuing
99+ * iteration with dma_buf_iter_next(), or with dma_buf_put().
100+ *
101+ * Return:
102+ * * Next buffer from global list, with refcount elevated
103+ * * NULL if no additional active buffers are present
104+ */
105+struct dma_buf *dma_buf_iter_next(struct dma_buf *dmabuf)
64106 {
107+ struct dma_buf *ret = NULL;
108+
109+ /*
110+ * The list mutex does not protect a dmabuf's refcount, so it can be
111+ * zeroed while we are iterating. We cannot call get_dma_buf() since the
112+ * caller may not already own a reference to the buffer.
113+ */
114+ mutex_lock(&dmabuf_list_mutex);
115+ dma_buf_put(dmabuf);
116+ list_for_each_entry_continue(dmabuf, &dmabuf_list, list_node) {
117+ if (file_ref_get(&dmabuf->file->f_ref)) {
118+ ret = dmabuf;
119+ break;
120+ }
121+ }
122+ mutex_unlock(&dmabuf_list_mutex);
123+ return ret;
65124 }
66−#endif
67125
68126 static char *dmabuffs_dname(struct dentry *dentry, char *buffer, int buflen)
69127 {
@@ -115,7 +173,7 @@ static int dma_buf_file_release(struct inode *inode, struct file *file) · 45 unmodified lines
115173 if (!is_dma_buf_file(file))
116174 return -EINVAL;
117175
118− __dma_buf_debugfs_list_del(file->private_data);
176+ __dma_buf_list_del(file->private_data);
119177
120178 return 0;
121179 }
@@ -685,7 +743,7 @@ struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info) · 563 unmodified lines
685743 file->f_path.dentry->d_fsdata = dmabuf;
686744 dmabuf->file = file;
687745
688− __dma_buf_debugfs_list_add(dmabuf);
746+ __dma_buf_list_add(dmabuf);
689747
690748 return dmabuf;
691749
@@ -1563,7 +1621,7 @@ static int dma_buf_debug_show(struct seq_file *s, void *unused) · 871 unmodified lines
15631621 size_t size = 0;
15641622 int ret;
15651623
1566− ret = mutex_lock_interruptible(&debugfs_list_mutex);
1624+ ret = mutex_lock_interruptible(&dmabuf_list_mutex);
15671625
15681626 if (ret)
15691627 return ret;
@@ -1572,7 +1630,7 @@ static int dma_buf_debug_show(struct seq_file *s, void *unused) · 2 unmodified lines
15721630 seq_printf(s, "%-8s\t%-8s\t%-8s\t%-8s\texp_name\t%-8s\tname\n",
15731631 "size", "flags", "mode", "count", "ino");
15741632
1575− list_for_each_entry(buf_obj, &debugfs_list, list_node) {
1633+ list_for_each_entry(buf_obj, &dmabuf_list, list_node) {
15761634
15771635 ret = dma_resv_lock_interruptible(buf_obj->resv, NULL);
15781636 if (ret)
@@ -1609,11 +1667,11 @@ static int dma_buf_debug_show(struct seq_file *s, void *unused) · 30 unmodified lines
16091667
16101668 seq_printf(s, "\nTotal %d objects, %zu bytes\n", count, size);
16111669
1612− mutex_unlock(&debugfs_list_mutex);
1670+ mutex_unlock(&dmabuf_list_mutex);
16131671 return 0;
16141672
16151673 error_unlock:
1616− mutex_unlock(&debugfs_list_mutex);
1674+ mutex_unlock(&dmabuf_list_mutex);
16171675 return ret;
16181676 }
16191677
include/asm-generic/vmlinux.lds.h+2−1
@@ -667,10 +667,11 @@ defined(CONFIG_AUTOFDO_CLANG) || defined(CONFIG_PROPELLER_CLANG) · 666 unmodified lines
667667 */
668668 #ifdef CONFIG_DEBUG_INFO_BTF
669669 #define BTF \
670+ . = ALIGN(PAGE_SIZE); \
670671 .BTF : AT(ADDR(.BTF) - LOAD_OFFSET) { \
671672 BOUNDED_SECTION_BY(.BTF, _BTF) \
672673 } \
673− . = ALIGN(4); \
674+ . = ALIGN(PAGE_SIZE); \
674675 .BTF_ids : AT(ADDR(.BTF_ids) - LOAD_OFFSET) { \
675676 *(.BTF_ids) \
676677 }
include/linux/bpf-cgroup.h+0−8
@@ -426,8 +426,6 @@ int cgroup_bpf_prog_query(const union bpf_attr *attr, · 425 unmodified lines
426426
427427 const struct bpf_func_proto *
428428 cgroup_common_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog);
429−const struct bpf_func_proto *
430−cgroup_current_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog);
431429 #else
432430
433431 static inline void cgroup_bpf_lifetime_notifier_init(void)
@@ -466,12 +464,6 @@ cgroup_common_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 32 unmodified lines
466464 return NULL;
467465 }
468466
469−static inline const struct bpf_func_proto *
470−cgroup_current_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
471−{
472− return NULL;
473−}
474−
475467 static inline int bpf_cgroup_storage_assign(struct bpf_prog_aux *aux,
476468 struct bpf_map *map) { return 0; }
477469 static inline struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(
include/linux/bpf.h+20−0
@@ -346,6 +346,12 @@ static inline const char *btf_field_type_name(enum btf_field_type type) · 345 unmodified lines
346346 }
347347 }
348348
349+#if IS_ENABLED(CONFIG_DEBUG_KERNEL)
350+#define BPF_WARN_ONCE(cond, format...) WARN_ONCE(cond, format)
351+#else
352+#define BPF_WARN_ONCE(cond, format...) BUILD_BUG_ON_INVALID(cond)
353+#endif
354+
349355 static inline u32 btf_field_type_size(enum btf_field_type type)
350356 {
351357 switch (type) {
@@ -1349,6 +1355,20 @@ u32 __bpf_dynptr_size(const struct bpf_dynptr_kern *ptr); · 997 unmodified lines
13491355 const void *__bpf_dynptr_data(const struct bpf_dynptr_kern *ptr, u32 len);
13501356 void *__bpf_dynptr_data_rw(const struct bpf_dynptr_kern *ptr, u32 len);
13511357 bool __bpf_dynptr_is_rdonly(const struct bpf_dynptr_kern *ptr);
1358+int __bpf_dynptr_write(const struct bpf_dynptr_kern *dst, u32 offset,
1359+ void *src, u32 len, u64 flags);
1360+void *bpf_dynptr_slice_rdwr(const struct bpf_dynptr *p, u32 offset,
1361+ void *buffer__opt, u32 buffer__szk);
1362+
1363+static inline int bpf_dynptr_check_off_len(const struct bpf_dynptr_kern *ptr, u32 offset, u32 len)
1364+{
1365+ u32 size = __bpf_dynptr_size(ptr);
1366+
1367+ if (len > size || offset > size - len)
1368+ return -E2BIG;
1369+
1370+ return 0;
1371+}
13521372
13531373 #ifdef CONFIG_BPF_JIT
13541374 int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
include/linux/bpf_verifier.h+20−4
@@ -356,7 +356,11 @@ enum { · 355 unmodified lines
356356 INSN_F_SPI_MASK = 0x3f, /* 6 bits */
357357 INSN_F_SPI_SHIFT = 3, /* shifted 3 bits to the left */
358358
359− INSN_F_STACK_ACCESS = BIT(9), /* we need 10 bits total */
359+ INSN_F_STACK_ACCESS = BIT(9),
360+
361+ INSN_F_DST_REG_STACK = BIT(10), /* dst_reg is PTR_TO_STACK */
362+ INSN_F_SRC_REG_STACK = BIT(11), /* src_reg is PTR_TO_STACK */
363+ /* total 12 bits are used now. */
360364 };
361365
362366 static_assert(INSN_F_FRAMENO_MASK + 1 >= MAX_CALL_FRAMES);
@@ -365,9 +369,9 @@ static_assert(INSN_F_SPI_MASK + 1 >= MAX_BPF_STACK / 8); · 2 unmodified lines
365369 struct bpf_insn_hist_entry {
366370 u32 idx;
367371 /* insn idx can't be bigger than 1 million */
368− u32 prev_idx : 22;
369− /* special flags, e.g., whether insn is doing register stack spill/load */
370− u32 flags : 10;
372+ u32 prev_idx : 20;
373+ /* special INSN_F_xxx flags */
374+ u32 flags : 12;
371375 /* additional registers that need precision tracking when this
372376 * jump is backtracked, vector of six 10-bit records
373377 */
@@ -591,6 +595,7 @@ struct bpf_insn_aux_data { · 217 unmodified lines
591595 * bpf_fastcall pattern.
592596 */
593597 u8 fastcall_spills_num:3;
598+ u8 arg_prog:4;
594599
595600 /* below fields are initialized once */
596601 unsigned int orig_idx; /* original instruction index */
@@ -838,6 +843,17 @@ __printf(3, 4) void verbose_linfo(struct bpf_verifier_env *env, · 241 unmodified lines
838843 u32 insn_off,
839844 const char *prefix_fmt, ...);
840845
846+#define verifier_bug_if(cond, env, fmt, args...) \
847+ ({ \
848+ bool __cond = (cond); \
849+ if (unlikely(__cond)) { \
850+ BPF_WARN_ONCE(1, "verifier bug: " fmt "(" #cond ")\n", ##args); \
851+ bpf_log(&env->log, "verifier bug: " fmt "(" #cond ")\n", ##args); \
852+ } \
853+ (__cond); \
854+ })
855+#define verifier_bug(env, fmt, args...) verifier_bug_if(1, env, fmt, ##args)
856+
841857 static inline struct bpf_func_state *cur_func(struct bpf_verifier_env *env)
842858 {
843859 struct bpf_verifier_state *cur = env->cur_state;
include/linux/dma-buf.h+2−2
@@ -361,10 +361,8 @@ struct dma_buf { · 360 unmodified lines
361361 */
362362 struct module *owner;
363363
364−#if IS_ENABLED(CONFIG_DEBUG_FS)
365364 /** @list_node: node for dma_buf accounting and debugging. */
366365 struct list_head list_node;
367−#endif
368366
369367 /** @priv: exporter specific private data for this buffer object. */
370368 void *priv;
@@ -609,4 +607,6 @@ int dma_buf_vmap(struct dma_buf *dmabuf, struct iosys_map *map); · 238 unmodified lines
609607 void dma_buf_vunmap(struct dma_buf *dmabuf, struct iosys_map *map);
610608 int dma_buf_vmap_unlocked(struct dma_buf *dmabuf, struct iosys_map *map);
611609 void dma_buf_vunmap_unlocked(struct dma_buf *dmabuf, struct iosys_map *map);
610+struct dma_buf *dma_buf_iter_begin(void);
611+struct dma_buf *dma_buf_iter_next(struct dma_buf *dmbuf);
612612 #endif /* __DMA_BUF_H__ */
include/uapi/linux/bpf.h+12−7
@@ -1506,7 +1506,7 @@ union bpf_attr { · 1505 unmodified lines
15061506 __s32 map_token_fd;
15071507 };
15081508
1509− struct { /* anonymous struct used by BPF_MAP_*_ELEM commands */
1509+ struct { /* anonymous struct used by BPF_MAP_*_ELEM and BPF_MAP_FREEZE commands */
15101510 __u32 map_fd;
15111511 __aligned_u64 key;
15121512 union {
@@ -1995,11 +1995,15 @@ union bpf_attr { · 482 unmodified lines
19951995 * long bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from, u32 len, u64 flags)
19961996 * Description
19971997 * Store *len* bytes from address *from* into the packet
1998− * associated to *skb*, at *offset*. *flags* are a combination of
1999− * **BPF_F_RECOMPUTE_CSUM** (automatically recompute the
2000− * checksum for the packet after storing the bytes) and
2001− * **BPF_F_INVALIDATE_HASH** (set *skb*\ **->hash**, *skb*\
2002− * **->swhash** and *skb*\ **->l4hash** to 0).
1998+ * associated to *skb*, at *offset*. The *flags* are a combination
1999+ * of the following values:
2000+ *
2001+ * **BPF_F_RECOMPUTE_CSUM**
2002+ * Automatically update *skb*\ **->csum** after storing the
2003+ * bytes.
2004+ * **BPF_F_INVALIDATE_HASH**
2005+ * Set *skb*\ **->hash**, *skb*\ **->swhash** and *skb*\
2006+ * **->l4hash** to 0.
20032007 *
20042008 * A call to this helper is susceptible to change the underlying
20052009 * packet buffer. Therefore, at load time, all checks on pointers
@@ -2051,7 +2055,7 @@ union bpf_attr { · 45 unmodified lines
20512055 * untouched (unless **BPF_F_MARK_ENFORCE** is added as well), and
20522056 * for updates resulting in a null checksum the value is set to
20532057 * **CSUM_MANGLED_0** instead. Flag **BPF_F_PSEUDO_HDR** indicates
2054− * the checksum is to be computed against a pseudo-header.
2058+ * that the modified header field is part of the pseudo-header.
20552059 *
20562060 * This helper works in combination with **bpf_csum_diff**\ (),
20572061 * which does not update the checksum in-place, but offers more
@@ -6723,6 +6727,7 @@ struct bpf_link_info { · 4665 unmodified lines
67236727 __u32 name_len;
67246728 __u32 offset; /* offset from file_name */
67256729 __u64 cookie;
6730+ __u64 ref_ctr_offset;
67266731 } uprobe; /* BPF_PERF_EVENT_UPROBE, BPF_PERF_EVENT_URETPROBE */
67276732 struct {
67286733 __aligned_u64 func_name; /* in/out */
kernel/bpf/Makefile+3−0
@@ -53,6 +53,9 @@ obj-$(CONFIG_BPF_SYSCALL) += relo_core.o · 52 unmodified lines
5353 obj-$(CONFIG_BPF_SYSCALL) += btf_iter.o
5454 obj-$(CONFIG_BPF_SYSCALL) += btf_relocate.o
5555 obj-$(CONFIG_BPF_SYSCALL) += kmem_cache_iter.o
56+ifeq ($(CONFIG_DMA_SHARED_BUFFER),y)
57+obj-$(CONFIG_BPF_SYSCALL) += dmabuf_iter.o
58+endif
5659
5760 CFLAGS_REMOVE_percpu_freelist.o = $(CC_FLAGS_FTRACE)
5861 CFLAGS_REMOVE_bpf_lru_list.o = $(CC_FLAGS_FTRACE)
kernel/bpf/bpf_struct_ops.c+1−1
@@ -601,7 +601,7 @@ int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks, · 600 unmodified lines
601601 if (model->ret_size > 0)
602602 flags |= BPF_TRAMP_F_RET_FENTRY_RET;
603603
604− size = arch_bpf_trampoline_size(model, flags, tlinks, NULL);
604+ size = arch_bpf_trampoline_size(model, flags, tlinks, stub_func);
605605 if (size <= 0)
606606 return size ? : -EFAULT;
607607
kernel/bpf/btf.c+20−25
@@ -26,6 +26,7 @@ · 25 unmodified lines
2626 #include <linux/bsearch.h>
2727 #include <linux/kobject.h>
2828 #include <linux/sysfs.h>
29+#include <linux/overflow.h>
2930
3031 #include <net/netfilter/nf_bpf_link.h>
3132
@@ -3957,7 +3958,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type · 3925 unmodified lines
39573958 /* This needs to be kzalloc to zero out padding and unused fields, see
39583959 * comment in btf_record_equal.
39593960 */
3960− rec = kzalloc(offsetof(struct btf_record, fields[cnt]), GFP_KERNEL | __GFP_NOWARN);
3961+ rec = kzalloc(struct_size(rec, fields, cnt), GFP_KERNEL | __GFP_NOWARN);
39613962 if (!rec)
39623963 return ERR_PTR(-ENOMEM);
39633964
@@ -5583,7 +5584,7 @@ btf_parse_struct_metas(struct bpf_verifier_log *log, struct btf *btf) · 1619 unmodified lines
55835584 if (id < 0)
55845585 continue;
55855586
5586− new_aof = krealloc(aof, offsetof(struct btf_id_set, ids[aof->cnt + 1]),
5587+ new_aof = krealloc(aof, struct_size(new_aof, ids, aof->cnt + 1),
55875588 GFP_KERNEL | __GFP_NOWARN);
55885589 if (!new_aof) {
55895590 ret = -ENOMEM;
@@ -5610,7 +5611,7 @@ btf_parse_struct_metas(struct bpf_verifier_log *log, struct btf *btf) · 20 unmodified lines
56105611 if (ret != BTF_FIELD_FOUND)
56115612 continue;
56125613
5613− new_aof = krealloc(aof, offsetof(struct btf_id_set, ids[aof->cnt + 1]),
5614+ new_aof = krealloc(aof, struct_size(new_aof, ids, aof->cnt + 1),
56145615 GFP_KERNEL | __GFP_NOWARN);
56155616 if (!new_aof) {
56165617 ret = -ENOMEM;
@@ -5647,7 +5648,7 @@ btf_parse_struct_metas(struct bpf_verifier_log *log, struct btf *btf) · 30 unmodified lines
56475648 continue;
56485649 parse:
56495650 tab_cnt = tab ? tab->cnt : 0;
5650− new_tab = krealloc(tab, offsetof(struct btf_struct_metas, types[tab_cnt + 1]),
5651+ new_tab = krealloc(tab, struct_size(new_tab, types, tab_cnt + 1),
56515652 GFP_KERNEL | __GFP_NOWARN);
56525653 if (!new_tab) {
56535654 ret = -ENOMEM;
@@ -6383,12 +6384,11 @@ struct btf *bpf_prog_get_target_btf(const struct bpf_prog *prog) · 729 unmodified lines
63836384 return prog->aux->attach_btf;
63846385 }
63856386
6386−static bool is_int_ptr(struct btf *btf, const struct btf_type *t)
6387+static bool is_void_or_int_ptr(struct btf *btf, const struct btf_type *t)
63876388 {
63886389 /* skip modifiers */
63896390 t = btf_type_skip_modifiers(btf, t->type, NULL);
6390−
6391− return btf_type_is_int(t);
6391+ return btf_type_is_void(t) || btf_type_is_int(t);
63926392 }
63936393
63946394 u32 btf_ctx_arg_idx(struct btf *btf, const struct btf_type *func_proto,
@@ -6777,14 +6777,11 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, · 382 unmodified lines
67776777 }
67786778 }
67796779
6780− if (t->type == 0)
6781− /* This is a pointer to void.
6782− * It is the same as scalar from the verifier safety pov.
6783− * No further pointer walking is allowed.
6784− */
6785− return true;
6786−
6787− if (is_int_ptr(btf, t))
6780+ /*
6781+ * If it's a pointer to void, it's the same as scalar from the verifier
6782+ * safety POV. Either way, no futher pointer walking is allowed.
6783+ */
6784+ if (is_void_or_int_ptr(btf, t))
67886785 return true;
67896786
67906787 /* this is a pointer to another type */
@@ -6830,10 +6827,10 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, · 39 unmodified lines
68306827 /* Is this a func with potential NULL args? */
68316828 if (strcmp(tname, raw_tp_null_args[i].func))
68326829 continue;
6833− if (raw_tp_null_args[i].mask & (0x1 << (arg * 4)))
6830+ if (raw_tp_null_args[i].mask & (0x1ULL << (arg * 4)))
68346831 info->reg_type |= PTR_MAYBE_NULL;
68356832 /* Is the current arg IS_ERR? */
6836− if (raw_tp_null_args[i].mask & (0x2 << (arg * 4)))
6833+ if (raw_tp_null_args[i].mask & (0x2ULL << (arg * 4)))
68376834 ptr_err_raw_tp = true;
68386835 break;
68396836 }
@@ -7663,7 +7660,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) · 823 unmodified lines
76637660 return 0;
76647661
76657662 if (!prog->aux->func_info) {
7666− bpf_log(log, "Verifier bug\n");
7663+ verifier_bug(env, "func_info undefined");
76677664 return -EFAULT;
76687665 }
76697666
@@ -7687,7 +7684,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) · 17 unmodified lines
76877684 tname = btf_name_by_offset(btf, fn_t->name_off);
76887685
76897686 if (prog->aux->func_info_aux[subprog].unreliable) {
7690− bpf_log(log, "Verifier bug in function %s()\n", tname);
7687+ verifier_bug(env, "unreliable BTF for function %s()", tname);
76917688 return -EFAULT;
76927689 }
76937690 if (prog_type == BPF_PROG_TYPE_EXT)
@@ -8564,7 +8561,7 @@ static int btf_populate_kfunc_set(struct btf *btf, enum btf_kfunc_hook hook, · 870 unmodified lines
85648561
85658562 /* Grow set */
85668563 set = krealloc(tab->sets[hook],
8567− offsetof(struct btf_id_set8, pairs[set_cnt + add_set->cnt]),
8564+ struct_size(set, pairs, set_cnt + add_set->cnt),
85688565 GFP_KERNEL | __GFP_NOWARN);
85698566 if (!set) {
85708567 ret = -ENOMEM;
@@ -8850,7 +8847,7 @@ int register_btf_id_dtor_kfuncs(const struct btf_id_dtor_kfunc *dtors, u32 add_c · 279 unmodified lines
88508847 }
88518848
88528849 tab = krealloc(btf->dtor_kfunc_tab,
8853− offsetof(struct btf_id_dtor_kfunc_tab, dtors[tab_cnt + add_cnt]),
8850+ struct_size(tab, dtors, tab_cnt + add_cnt),
88548851 GFP_KERNEL | __GFP_NOWARN);
88558852 if (!tab) {
88568853 ret = -ENOMEM;
@@ -9408,8 +9405,7 @@ btf_add_struct_ops(struct btf *btf, struct bpf_struct_ops *st_ops, · 551 unmodified lines
94089405
94099406 tab = btf->struct_ops_tab;
94109407 if (!tab) {
9411− tab = kzalloc(offsetof(struct btf_struct_ops_tab, ops[4]),
9412− GFP_KERNEL);
9408+ tab = kzalloc(struct_size(tab, ops, 4), GFP_KERNEL);
94139409 if (!tab)
94149410 return -ENOMEM;
94159411 tab->capacity = 4;
@@ -9422,8 +9418,7 @@ btf_add_struct_ops(struct btf *btf, struct bpf_struct_ops *st_ops, · 6 unmodified lines
94229418
94239419 if (tab->cnt == tab->capacity) {
94249420 new_tab = krealloc(tab,
9425− offsetof(struct btf_struct_ops_tab,
9426− ops[tab->capacity * 2]),
9421+ struct_size(tab, ops, tab->capacity * 2),
94279422 GFP_KERNEL);
94289423 if (!new_tab)
94299424 return -ENOMEM;
kernel/bpf/cgroup.c+0−32
@@ -1687,10 +1687,6 @@ cgroup_dev_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 1686 unmodified lines
16871687 if (func_proto)
16881688 return func_proto;
16891689
1690− func_proto = cgroup_current_func_proto(func_id, prog);
1691− if (func_proto)
1692− return func_proto;
1693−
16941690 switch (func_id) {
16951691 case BPF_FUNC_perf_event_output:
16961692 return &bpf_event_output_data_proto;
@@ -2238,10 +2234,6 @@ sysctl_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 541 unmodified lines
22382234 if (func_proto)
22392235 return func_proto;
22402236
2241− func_proto = cgroup_current_func_proto(func_id, prog);
2242− if (func_proto)
2243− return func_proto;
2244−
22452237 switch (func_id) {
22462238 case BPF_FUNC_sysctl_get_name:
22472239 return &bpf_sysctl_get_name_proto;
@@ -2385,10 +2377,6 @@ cg_sockopt_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 137 unmodified lines
23852377 if (func_proto)
23862378 return func_proto;
23872379
2388− func_proto = cgroup_current_func_proto(func_id, prog);
2389− if (func_proto)
2390− return func_proto;
2391−
23922380 switch (func_id) {
23932381 #ifdef CONFIG_NET
23942382 case BPF_FUNC_get_netns_cookie:
@@ -2635,23 +2623,3 @@ cgroup_common_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 240 unmodified lines
26352623 return NULL;
26362624 }
26372625 }
2638−
2639−/* Common helpers for cgroup hooks with valid process context. */
2640−const struct bpf_func_proto *
2641−cgroup_current_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
2642−{
2643− switch (func_id) {
2644− case BPF_FUNC_get_current_uid_gid:
2645− return &bpf_get_current_uid_gid_proto;
2646− case BPF_FUNC_get_current_comm:
2647− return &bpf_get_current_comm_proto;
2648−#ifdef CONFIG_CGROUP_NET_CLASSID
2649− case BPF_FUNC_get_cgroup_classid:
2650− return &bpf_get_cgroup_classid_curr_proto;
2651−#endif
2652− case BPF_FUNC_current_task_under_cgroup:
2653− return &bpf_current_task_under_cgroup_proto;
2654− default:
2655− return NULL;
2656− }
2657−}
kernel/bpf/core.c+17−12
@@ -2358,8 +2358,8 @@ static unsigned int __bpf_prog_ret0_warn(const void *ctx, · 2357 unmodified lines
23582358 return 0;
23592359 }
23602360
2361−bool bpf_prog_map_compatible(struct bpf_map *map,
2362− const struct bpf_prog *fp)
2361+static bool __bpf_prog_map_compatible(struct bpf_map *map,
2362+ const struct bpf_prog *fp)
23632363 {
23642364 enum bpf_prog_type prog_type = resolve_prog_type(fp);
23652365 bool ret;
@@ -2368,14 +2368,6 @@ bool bpf_prog_map_compatible(struct bpf_map *map, · 2 unmodified lines
23682368 if (fp->kprobe_override)
23692369 return false;
23702370
2371− /* XDP programs inserted into maps are not guaranteed to run on
2372− * a particular netdev (and can run outside driver context entirely
2373− * in the case of devmap and cpumap). Until device checks
2374− * are implemented, prohibit adding dev-bound programs to program maps.
2375− */
2376− if (bpf_prog_is_dev_bound(aux))
2377− return false;
2378−
23792371 spin_lock(&map->owner.lock);
23802372 if (!map->owner.type) {
23812373 /* There's no owner yet where we could check for
@@ -2409,6 +2401,19 @@ bool bpf_prog_map_compatible(struct bpf_map *map, · 27 unmodified lines
24092401 return ret;
24102402 }
24112403
2404+bool bpf_prog_map_compatible(struct bpf_map *map, const struct bpf_prog *fp)
2405+{
2406+ /* XDP programs inserted into maps are not guaranteed to run on
2407+ * a particular netdev (and can run outside driver context entirely
2408+ * in the case of devmap and cpumap). Until device checks
2409+ * are implemented, prohibit adding dev-bound programs to program maps.
2410+ */
2411+ if (bpf_prog_is_dev_bound(fp->aux))
2412+ return false;
2413+
2414+ return __bpf_prog_map_compatible(map, fp);
2415+}
2416+
24122417 static int bpf_check_tail_call(const struct bpf_prog *fp)
24132418 {
24142419 struct bpf_prog_aux *aux = fp->aux;
@@ -2421,7 +2426,7 @@ static int bpf_check_tail_call(const struct bpf_prog *fp) · 6 unmodified lines
24212426 if (!map_type_contains_progs(map))
24222427 continue;
24232428
2424− if (!bpf_prog_map_compatible(map, fp)) {
2429+ if (!__bpf_prog_map_compatible(map, fp)) {
24252430 ret = -EINVAL;
24262431 goto out;
24272432 }
@@ -2469,7 +2474,7 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err) · 41 unmodified lines
24692474 /* In case of BPF to BPF calls, verifier did all the prep
24702475 * work with regards to JITing, etc.
24712476 */
2472− bool jit_needed = false;
2477+ bool jit_needed = fp->jit_requested;
24732478
24742479 if (fp->bpf_func)
24752480 goto finalize;
kernel/bpf/dmabuf_iter.c+150−0
@@ -0,0 +1,150 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+/* Copyright (c) 2025 Google LLC */
3+#include <linux/bpf.h>
4+#include <linux/btf_ids.h>
5+#include <linux/dma-buf.h>
6+#include <linux/kernel.h>
7+#include <linux/seq_file.h>
8+
9+static void *dmabuf_iter_seq_start(struct seq_file *seq, loff_t *pos)
10+{
11+ if (*pos)
12+ return NULL;
13+
14+ return dma_buf_iter_begin();
15+}
16+
17+static void *dmabuf_iter_seq_next(struct seq_file *seq, void *v, loff_t *pos)
18+{
19+ struct dma_buf *dmabuf = v;
20+
21+ ++*pos;
22+
23+ return dma_buf_iter_next(dmabuf);
24+}
25+
26+struct bpf_iter__dmabuf {
27+ __bpf_md_ptr(struct bpf_iter_meta *, meta);
28+ __bpf_md_ptr(struct dma_buf *, dmabuf);
29+};
30+
31+static int __dmabuf_seq_show(struct seq_file *seq, void *v, bool in_stop)
32+{
33+ struct bpf_iter_meta meta = {
34+ .seq = seq,
35+ };
36+ struct bpf_iter__dmabuf ctx = {
37+ .meta = &meta,
38+ .dmabuf = v,
39+ };
40+ struct bpf_prog *prog = bpf_iter_get_info(&meta, in_stop);
41+
42+ if (prog)
43+ return bpf_iter_run_prog(prog, &ctx);
44+
45+ return 0;
46+}
47+
48+static int dmabuf_iter_seq_show(struct seq_file *seq, void *v)
49+{
50+ return __dmabuf_seq_show(seq, v, false);
51+}
52+
53+static void dmabuf_iter_seq_stop(struct seq_file *seq, void *v)
54+{
55+ struct dma_buf *dmabuf = v;
56+
57+ if (dmabuf)
58+ dma_buf_put(dmabuf);
59+}
60+
61+static const struct seq_operations dmabuf_iter_seq_ops = {
62+ .start = dmabuf_iter_seq_start,
63+ .next = dmabuf_iter_seq_next,
64+ .stop = dmabuf_iter_seq_stop,
65+ .show = dmabuf_iter_seq_show,
66+};
67+
68+static void bpf_iter_dmabuf_show_fdinfo(const struct bpf_iter_aux_info *aux,
69+ struct seq_file *seq)
70+{
71+ seq_puts(seq, "dmabuf iter\n");
72+}
73+
74+static const struct bpf_iter_seq_info dmabuf_iter_seq_info = {
75+ .seq_ops = &dmabuf_iter_seq_ops,
76+ .init_seq_private = NULL,
77+ .fini_seq_private = NULL,
78+ .seq_priv_size = 0,
79+};
80+
81+static struct bpf_iter_reg bpf_dmabuf_reg_info = {
82+ .target = "dmabuf",
83+ .feature = BPF_ITER_RESCHED,
84+ .show_fdinfo = bpf_iter_dmabuf_show_fdinfo,
85+ .ctx_arg_info_size = 1,
86+ .ctx_arg_info = {
87+ { offsetof(struct bpf_iter__dmabuf, dmabuf),
88+ PTR_TO_BTF_ID_OR_NULL },
89+ },
90+ .seq_info = &dmabuf_iter_seq_info,
91+};
92+
93+DEFINE_BPF_ITER_FUNC(dmabuf, struct bpf_iter_meta *meta, struct dma_buf *dmabuf)
94+BTF_ID_LIST_SINGLE(bpf_dmabuf_btf_id, struct, dma_buf)
95+
96+static int __init dmabuf_iter_init(void)
97+{
98+ bpf_dmabuf_reg_info.ctx_arg_info[0].btf_id = bpf_dmabuf_btf_id[0];
99+ return bpf_iter_reg_target(&bpf_dmabuf_reg_info);
100+}
101+
102+late_initcall(dmabuf_iter_init);
103+
104+struct bpf_iter_dmabuf {
105+ /*
106+ * opaque iterator state; having __u64 here allows to preserve correct
107+ * alignment requirements in vmlinux.h, generated from BTF
108+ */
109+ __u64 __opaque[1];
110+} __aligned(8);
111+
112+/* Non-opaque version of bpf_iter_dmabuf */
113+struct bpf_iter_dmabuf_kern {
114+ struct dma_buf *dmabuf;
115+} __aligned(8);
116+
117+__bpf_kfunc_start_defs();
118+
119+__bpf_kfunc int bpf_iter_dmabuf_new(struct bpf_iter_dmabuf *it)
120+{
121+ struct bpf_iter_dmabuf_kern *kit = (void *)it;
122+
123+ BUILD_BUG_ON(sizeof(*kit) > sizeof(*it));
124+ BUILD_BUG_ON(__alignof__(*kit) != __alignof__(*it));
125+
126+ kit->dmabuf = NULL;
127+ return 0;
128+}
129+
130+__bpf_kfunc struct dma_buf *bpf_iter_dmabuf_next(struct bpf_iter_dmabuf *it)
131+{
132+ struct bpf_iter_dmabuf_kern *kit = (void *)it;
133+
134+ if (kit->dmabuf)
135+ kit->dmabuf = dma_buf_iter_next(kit->dmabuf);
136+ else
137+ kit->dmabuf = dma_buf_iter_begin();
138+
139+ return kit->dmabuf;
140+}
141+
142+__bpf_kfunc void bpf_iter_dmabuf_destroy(struct bpf_iter_dmabuf *it)
143+{
144+ struct bpf_iter_dmabuf_kern *kit = (void *)it;
145+
146+ if (kit->dmabuf)
147+ dma_buf_put(kit->dmabuf);
148+}
149+
150+__bpf_kfunc_end_defs();
kernel/bpf/hashtab.c+72−76
@@ -175,20 +175,30 @@ static bool htab_is_percpu(const struct bpf_htab *htab) · 174 unmodified lines
175175 htab->map.map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH;
176176 }
177177
178+static inline bool is_fd_htab(const struct bpf_htab *htab)
179+{
180+ return htab->map.map_type == BPF_MAP_TYPE_HASH_OF_MAPS;
181+}
182+
183+static inline void *htab_elem_value(struct htab_elem *l, u32 key_size)
184+{
185+ return l->key + round_up(key_size, 8);
186+}
187+
178188 static inline void htab_elem_set_ptr(struct htab_elem *l, u32 key_size,
179189 void __percpu *pptr)
180190 {
181− *(void __percpu **)(l->key + roundup(key_size, 8)) = pptr;
191+ *(void __percpu **)htab_elem_value(l, key_size) = pptr;
182192 }
183193
184194 static inline void __percpu *htab_elem_get_ptr(struct htab_elem *l, u32 key_size)
185195 {
186− return *(void __percpu **)(l->key + roundup(key_size, 8));
196+ return *(void __percpu **)htab_elem_value(l, key_size);
187197 }
188198
189199 static void *fd_htab_map_get_ptr(const struct bpf_map *map, struct htab_elem *l)
190200 {
191− return *(void **)(l->key + roundup(map->key_size, 8));
201+ return *(void **)htab_elem_value(l, map->key_size);
192202 }
193203
194204 static struct htab_elem *get_htab_elem(struct bpf_htab *htab, int i)
@@ -196,9 +206,13 @@ static struct htab_elem *get_htab_elem(struct bpf_htab *htab, int i) · 1 unmodified line
196206 return (struct htab_elem *) (htab->elems + i * (u64)htab->elem_size);
197207 }
198208
209+/* Both percpu and fd htab support in-place update, so no need for
210+ * extra elem. LRU itself can remove the least used element, so
211+ * there is no need for an extra elem during map_update.
212+ */
199213 static bool htab_has_extra_elems(struct bpf_htab *htab)
200214 {
201− return !htab_is_percpu(htab) && !htab_is_lru(htab);
215+ return !htab_is_percpu(htab) && !htab_is_lru(htab) && !is_fd_htab(htab);
202216 }
203217
204218 static void htab_free_prealloced_timers_and_wq(struct bpf_htab *htab)
@@ -215,10 +229,10 @@ static void htab_free_prealloced_timers_and_wq(struct bpf_htab *htab) · 10 unmodified lines
215229 elem = get_htab_elem(htab, i);
216230 if (btf_record_has_field(htab->map.record, BPF_TIMER))
217231 bpf_obj_free_timer(htab->map.record,
218− elem->key + round_up(htab->map.key_size, 8));
232+ htab_elem_value(elem, htab->map.key_size));
219233 if (btf_record_has_field(htab->map.record, BPF_WORKQUEUE))
220234 bpf_obj_free_workqueue(htab->map.record,
221− elem->key + round_up(htab->map.key_size, 8));
235+ htab_elem_value(elem, htab->map.key_size));
222236 cond_resched();
223237 }
224238 }
@@ -245,7 +259,8 @@ static void htab_free_prealloced_fields(struct bpf_htab *htab) · 20 unmodified lines
245259 cond_resched();
246260 }
247261 } else {
248− bpf_obj_free_fields(htab->map.record, elem->key + round_up(htab->map.key_size, 8));
262+ bpf_obj_free_fields(htab->map.record,
263+ htab_elem_value(elem, htab->map.key_size));
249264 cond_resched();
250265 }
251266 cond_resched();
@@ -453,8 +468,6 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr) · 201 unmodified lines
453468 {
454469 bool percpu = (attr->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
455470 attr->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH);
456− bool lru = (attr->map_type == BPF_MAP_TYPE_LRU_HASH ||
457− attr->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH);
458471 /* percpu_lru means each cpu has its own LRU list.
459472 * it is different from BPF_MAP_TYPE_PERCPU_HASH where
460473 * the map's value itself is percpu. percpu_lru has
@@ -549,10 +562,7 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr) · 88 unmodified lines
549562 if (err)
550563 goto free_map_locked;
551564
552− if (!percpu && !lru) {
553− /* lru itself can remove the least used element, so
554− * there is no need for an extra elem during map_update.
555− */
565+ if (htab_has_extra_elems(htab)) {
556566 err = alloc_extra_elems(htab);
557567 if (err)
558568 goto free_prealloc;
@@ -670,7 +680,7 @@ static void *htab_map_lookup_elem(struct bpf_map *map, void *key) · 111 unmodified lines
670680 struct htab_elem *l = __htab_map_lookup_elem(map, key);
671681
672682 if (l)
673− return l->key + round_up(map->key_size, 8);
683+ return htab_elem_value(l, map->key_size);
674684
675685 return NULL;
676686 }
@@ -709,7 +719,7 @@ static __always_inline void *__htab_lru_map_lookup_elem(struct bpf_map *map, · 32 unmodified lines
709719 if (l) {
710720 if (mark)
711721 bpf_lru_node_set_ref(&l->lru_node);
712− return l->key + round_up(map->key_size, 8);
722+ return htab_elem_value(l, map->key_size);
713723 }
714724
715725 return NULL;
@@ -763,7 +773,7 @@ static void check_and_free_fields(struct bpf_htab *htab, · 47 unmodified lines
763773 for_each_possible_cpu(cpu)
764774 bpf_obj_free_fields(htab->map.record, per_cpu_ptr(pptr, cpu));
765775 } else {
766− void *map_value = elem->key + round_up(htab->map.key_size, 8);
776+ void *map_value = htab_elem_value(elem, htab->map.key_size);
767777
768778 bpf_obj_free_fields(htab->map.record, map_value);
769779 }
@@ -968,8 +978,7 @@ static void pcpu_init_value(struct bpf_htab *htab, void __percpu *pptr, · 198 unmodified lines
968978
969979 static bool fd_htab_map_needs_adjust(const struct bpf_htab *htab)
970980 {
971− return htab->map.map_type == BPF_MAP_TYPE_HASH_OF_MAPS &&
972− BITS_PER_LONG == 64;
981+ return is_fd_htab(htab) && BITS_PER_LONG == 64;
973982 }
974983
975984 static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key,
@@ -1039,11 +1048,9 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, · 63 unmodified lines
10391048 htab_elem_set_ptr(l_new, key_size, pptr);
10401049 } else if (fd_htab_map_needs_adjust(htab)) {
10411050 size = round_up(size, 8);
1042− memcpy(l_new->key + round_up(key_size, 8), value, size);
1051+ memcpy(htab_elem_value(l_new, key_size), value, size);
10431052 } else {
1044− copy_map_value(&htab->map,
1045− l_new->key + round_up(key_size, 8),
1046− value);
1053+ copy_map_value(&htab->map, htab_elem_value(l_new, key_size), value);
10471054 }
10481055
10491056 l_new->hash = hash;
@@ -1072,10 +1079,9 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value, · 22 unmodified lines
10721079 u64 map_flags)
10731080 {
10741081 struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
1075− struct htab_elem *l_new = NULL, *l_old;
1082+ struct htab_elem *l_new, *l_old;
10761083 struct hlist_nulls_head *head;
10771084 unsigned long flags;
1078− void *old_map_ptr;
10791085 struct bucket *b;
10801086 u32 key_size, hash;
10811087 int ret;
@@ -1106,7 +1112,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value, · 24 unmodified lines
11061112 if (l_old) {
11071113 /* grab the element lock and update value in place */
11081114 copy_map_value_locked(map,
1109− l_old->key + round_up(key_size, 8),
1115+ htab_elem_value(l_old, key_size),
11101116 value, false);
11111117 return 0;
11121118 }
@@ -1134,7 +1140,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value, · 21 unmodified lines
11341140 * and update element in place
11351141 */
11361142 copy_map_value_locked(map,
1137− l_old->key + round_up(key_size, 8),
1143+ htab_elem_value(l_old, key_size),
11381144 value, false);
11391145 ret = 0;
11401146 goto err;
@@ -1156,24 +1162,14 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value, · 15 unmodified lines
11561162 hlist_nulls_del_rcu(&l_old->hash_node);
11571163
11581164 /* l_old has already been stashed in htab->extra_elems, free
1159− * its special fields before it is available for reuse. Also
1160− * save the old map pointer in htab of maps before unlock
1161− * and release it after unlock.
1165+ * its special fields before it is available for reuse.
11621166 */
1163− old_map_ptr = NULL;
1164− if (htab_is_prealloc(htab)) {
1165− if (map->ops->map_fd_put_ptr)
1166− old_map_ptr = fd_htab_map_get_ptr(map, l_old);
1167+ if (htab_is_prealloc(htab))
11671168 check_and_free_fields(htab, l_old);
1168− }
11691169 }
11701170 htab_unlock_bucket(b, flags);
1171− if (l_old) {
1172− if (old_map_ptr)
1173− map->ops->map_fd_put_ptr(map, old_map_ptr, true);
1174− if (!htab_is_prealloc(htab))
1175− free_htab_elem(htab, l_old);
1176− }
1171+ if (l_old && !htab_is_prealloc(htab))
1172+ free_htab_elem(htab, l_old);
11771173 return 0;
11781174 err:
11791175 htab_unlock_bucket(b, flags);
@@ -1220,8 +1216,7 @@ static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value · 40 unmodified lines
12201216 l_new = prealloc_lru_pop(htab, key, hash);
12211217 if (!l_new)
12221218 return -ENOMEM;
1223− copy_map_value(&htab->map,
1224− l_new->key + round_up(map->key_size, 8), value);
1219+ copy_map_value(&htab->map, htab_elem_value(l_new, map->key_size), value);
12251220
12261221 ret = htab_lock_bucket(b, &flags);
12271222 if (ret)
@@ -1255,13 +1250,14 @@ err_lock_bucket: · 27 unmodified lines
12551250 return ret;
12561251 }
12571252
1258−static long __htab_percpu_map_update_elem(struct bpf_map *map, void *key,
1253+static long htab_map_update_elem_in_place(struct bpf_map *map, void *key,
12591254 void *value, u64 map_flags,
1260− bool onallcpus)
1255+ bool percpu, bool onallcpus)
12611256 {
12621257 struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
1263− struct htab_elem *l_new = NULL, *l_old;
1258+ struct htab_elem *l_new, *l_old;
12641259 struct hlist_nulls_head *head;
1260+ void *old_map_ptr = NULL;
12651261 unsigned long flags;
12661262 struct bucket *b;
12671263 u32 key_size, hash;
@@ -1292,21 +1288,29 @@ static long __htab_percpu_map_update_elem(struct bpf_map *map, void *key, · 24 unmodified lines
12921288 goto err;
12931289
12941290 if (l_old) {
1295− /* per-cpu hash map can update value in-place */
1296− pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size),
1297− value, onallcpus);
1291+ /* Update value in-place */
1292+ if (percpu) {
1293+ pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size),
1294+ value, onallcpus);
1295+ } else {
1296+ void **inner_map_pptr = htab_elem_value(l_old, key_size);
1297+
1298+ old_map_ptr = *inner_map_pptr;
1299+ WRITE_ONCE(*inner_map_pptr, *(void **)value);
1300+ }
12981301 } else {
12991302 l_new = alloc_htab_elem(htab, key, value, key_size,
1300− hash, true, onallcpus, NULL);
1303+ hash, percpu, onallcpus, NULL);
13011304 if (IS_ERR(l_new)) {
13021305 ret = PTR_ERR(l_new);
13031306 goto err;
13041307 }
13051308 hlist_nulls_add_head_rcu(&l_new->hash_node, head);
13061309 }
1307− ret = 0;
13081310 err:
13091311 htab_unlock_bucket(b, flags);
1312+ if (old_map_ptr)
1313+ map->ops->map_fd_put_ptr(map, old_map_ptr, true);
13101314 return ret;
13111315 }
13121316
@@ -1383,7 +1387,7 @@ err_lock_bucket: · 70 unmodified lines
13831387 static long htab_percpu_map_update_elem(struct bpf_map *map, void *key,
13841388 void *value, u64 map_flags)
13851389 {
1386− return __htab_percpu_map_update_elem(map, key, value, map_flags, false);
1390+ return htab_map_update_elem_in_place(map, key, value, map_flags, true, false);
13871391 }
13881392
13891393 static long htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key,
@@ -1500,10 +1504,10 @@ static void htab_free_malloced_timers_and_wq(struct bpf_htab *htab) · 110 unmodified lines
15001504 /* We only free timer on uref dropping to zero */
15011505 if (btf_record_has_field(htab->map.record, BPF_TIMER))
15021506 bpf_obj_free_timer(htab->map.record,
1503− l->key + round_up(htab->map.key_size, 8));
1507+ htab_elem_value(l, htab->map.key_size));
15041508 if (btf_record_has_field(htab->map.record, BPF_WORKQUEUE))
15051509 bpf_obj_free_workqueue(htab->map.record,
1506− l->key + round_up(htab->map.key_size, 8));
1510+ htab_elem_value(l, htab->map.key_size));
15071511 }
15081512 cond_resched_rcu();
15091513 }
@@ -1615,15 +1619,12 @@ static int __htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key, · 105 unmodified lines
16151619 off += roundup_value_size;
16161620 }
16171621 } else {
1618− u32 roundup_key_size = round_up(map->key_size, 8);
1622+ void *src = htab_elem_value(l, map->key_size);
16191623
16201624 if (flags & BPF_F_LOCK)
1621− copy_map_value_locked(map, value, l->key +
1622− roundup_key_size,
1623− true);
1625+ copy_map_value_locked(map, value, src, true);
16241626 else
1625− copy_map_value(map, value, l->key +
1626− roundup_key_size);
1627+ copy_map_value(map, value, src);
16271628 /* Zeroing special fields in the temp buffer */
16281629 check_and_init_map_value(map, value);
16291630 }
@@ -1680,12 +1681,12 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map, · 50 unmodified lines
16801681 bool is_percpu)
16811682 {
16821683 struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
1683− u32 bucket_cnt, total, key_size, value_size, roundup_key_size;
16841684 void *keys = NULL, *values = NULL, *value, *dst_key, *dst_val;
16851685 void __user *uvalues = u64_to_user_ptr(attr->batch.values);
16861686 void __user *ukeys = u64_to_user_ptr(attr->batch.keys);
16871687 void __user *ubatch = u64_to_user_ptr(attr->batch.in_batch);
16881688 u32 batch, max_count, size, bucket_size, map_id;
1689+ u32 bucket_cnt, total, key_size, value_size;
16891690 struct htab_elem *node_to_free = NULL;
16901691 u64 elem_map_flags, map_flags;
16911692 struct hlist_nulls_head *head;
@@ -1720,7 +1721,6 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map, · 28 unmodified lines
17201721 return -ENOENT;
17211722
17221723 key_size = htab->map.key_size;
1723− roundup_key_size = round_up(htab->map.key_size, 8);
17241724 value_size = htab->map.value_size;
17251725 size = round_up(value_size, 8);
17261726 if (is_percpu)
@@ -1812,8 +1812,8 @@ again_nocopy: · 85 unmodified lines
18121812 off += size;
18131813 }
18141814 } else {
1815− value = l->key + roundup_key_size;
1816− if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) {
1815+ value = htab_elem_value(l, key_size);
1816+ if (is_fd_htab(htab)) {
18171817 struct bpf_map **inner_map = value;
18181818
18191819 /* Actual value is the id of the inner map */
@@ -2063,11 +2063,11 @@ static void *bpf_hash_map_seq_next(struct seq_file *seq, void *v, loff_t *pos) · 243 unmodified lines
20632063 static int __bpf_hash_map_seq_show(struct seq_file *seq, struct htab_elem *elem)
20642064 {
20652065 struct bpf_iter_seq_hash_map_info *info = seq->private;
2066− u32 roundup_key_size, roundup_value_size;
20672066 struct bpf_iter__bpf_map_elem ctx = {};
20682067 struct bpf_map *map = info->map;
20692068 struct bpf_iter_meta meta;
20702069 int ret = 0, off = 0, cpu;
2070+ u32 roundup_value_size;
20712071 struct bpf_prog *prog;
20722072 void __percpu *pptr;
20732073
@@ -2077,10 +2077,9 @@ static int __bpf_hash_map_seq_show(struct seq_file *seq, struct htab_elem *elem) · 3 unmodified lines
20772077 ctx.meta = &meta;
20782078 ctx.map = info->map;
20792079 if (elem) {
2080− roundup_key_size = round_up(map->key_size, 8);
20812080 ctx.key = elem->key;
20822081 if (!info->percpu_value_buf) {
2083− ctx.value = elem->key + roundup_key_size;
2082+ ctx.value = htab_elem_value(elem, map->key_size);
20842083 } else {
20852084 roundup_value_size = round_up(map->value_size, 8);
20862085 pptr = htab_elem_get_ptr(elem, map->key_size);
@@ -2165,7 +2164,6 @@ static long bpf_for_each_hash_elem(struct bpf_map *map, bpf_callback_t callback_ · 78 unmodified lines
21652164 struct hlist_nulls_head *head;
21662165 struct hlist_nulls_node *n;
21672166 struct htab_elem *elem;
2168− u32 roundup_key_size;
21692167 int i, num_elems = 0;
21702168 void __percpu *pptr;
21712169 struct bucket *b;
@@ -2180,7 +2178,6 @@ static long bpf_for_each_hash_elem(struct bpf_map *map, bpf_callback_t callback_ · 8 unmodified lines
21802178
21812179 is_percpu = htab_is_percpu(htab);
21822180
2183− roundup_key_size = round_up(map->key_size, 8);
21842181 /* migration has been disabled, so percpu value prepared here will be
21852182 * the same as the one seen by the bpf program with
21862183 * bpf_map_lookup_elem().
@@ -2196,7 +2193,7 @@ static long bpf_for_each_hash_elem(struct bpf_map *map, bpf_callback_t callback_ · 9 unmodified lines
21962193 pptr = htab_elem_get_ptr(elem, map->key_size);
21972194 val = this_cpu_ptr(pptr);
21982195 } else {
2199− val = elem->key + roundup_key_size;
2196+ val = htab_elem_value(elem, map->key_size);
22002197 }
22012198 num_elems++;
22022199 ret = callback_fn((u64)(long)map, (u64)(long)key,
@@ -2411,8 +2408,8 @@ int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value, · 208 unmodified lines
24112408 ret = __htab_lru_percpu_map_update_elem(map, key, value,
24122409 map_flags, true);
24132410 else
2414− ret = __htab_percpu_map_update_elem(map, key, value, map_flags,
2415− true);
2411+ ret = htab_map_update_elem_in_place(map, key, value, map_flags,
2412+ true, true);
24162413 rcu_read_unlock();
24172414
24182415 return ret;
@@ -2536,24 +2533,23 @@ int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value) · 117 unmodified lines
25362533 return ret;
25372534 }
25382535
2539−/* only called from syscall */
2536+/* Only called from syscall */
25402537 int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file,
25412538 void *key, void *value, u64 map_flags)
25422539 {
25432540 void *ptr;
25442541 int ret;
2545− u32 ufd = *(u32 *)value;
25462542
2547− ptr = map->ops->map_fd_get_ptr(map, map_file, ufd);
2543+ ptr = map->ops->map_fd_get_ptr(map, map_file, *(int *)value);
25482544 if (IS_ERR(ptr))
25492545 return PTR_ERR(ptr);
25502546
25512547 /* The htab bucket lock is always held during update operations in fd
25522548 * htab map, and the following rcu_read_lock() is only used to avoid
2553− * the WARN_ON_ONCE in htab_map_update_elem().
2549+ * the WARN_ON_ONCE in htab_map_update_elem_in_place().
25542550 */
25552551 rcu_read_lock();
2556− ret = htab_map_update_elem(map, key, &ptr, map_flags);
2552+ ret = htab_map_update_elem_in_place(map, key, &ptr, map_flags, false, false);
25572553 rcu_read_unlock();
25582554 if (ret)
25592555 map->ops->map_fd_put_ptr(map, ptr, false);
kernel/bpf/helpers.c+118−15
@@ -23,6 +23,7 @@ · 22 unmodified lines
2323 #include <linux/btf_ids.h>
2424 #include <linux/bpf_mem_alloc.h>
2525 #include <linux/kasan.h>
26+#include <linux/bpf_verifier.h>
2627
2728 #include "../../lib/kstrtox.h"
2829
@@ -129,7 +130,8 @@ const struct bpf_func_proto bpf_map_peek_elem_proto = { · 100 unmodified lines
129130
130131 BPF_CALL_3(bpf_map_lookup_percpu_elem, struct bpf_map *, map, void *, key, u32, cpu)
131132 {
132− WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
133+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
134+ !rcu_read_lock_bh_held());
133135 return (unsigned long) map->ops->map_lookup_percpu_elem(map, key, cpu);
134136 }
135137
@@ -1713,16 +1715,6 @@ void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr) · 1577 unmodified lines
17131715 memset(ptr, 0, sizeof(*ptr));
17141716 }
17151717
1716−static int bpf_dynptr_check_off_len(const struct bpf_dynptr_kern *ptr, u32 offset, u32 len)
1717−{
1718− u32 size = __bpf_dynptr_size(ptr);
1719−
1720− if (len > size || offset > size - len)
1721− return -E2BIG;
1722−
1723− return 0;
1724−}
1725−
17261718 BPF_CALL_4(bpf_dynptr_from_mem, void *, data, u32, size, u64, flags, struct bpf_dynptr_kern *, ptr)
17271719 {
17281720 int err;
@@ -1809,8 +1801,8 @@ static const struct bpf_func_proto bpf_dynptr_read_proto = { · 80 unmodified lines
18091801 .arg5_type = ARG_ANYTHING,
18101802 };
18111803
1812−static int __bpf_dynptr_write(const struct bpf_dynptr_kern *dst, u32 offset, void *src,
1813− u32 len, u64 flags)
1804+int __bpf_dynptr_write(const struct bpf_dynptr_kern *dst, u32 offset, void *src,
1805+ u32 len, u64 flags)
18141806 {
18151807 enum bpf_dynptr_type type;
18161808 int err;
@@ -1912,6 +1904,12 @@ const struct bpf_func_proto bpf_probe_read_user_str_proto __weak; · 95 unmodified lines
19121904 const struct bpf_func_proto bpf_probe_read_kernel_proto __weak;
19131905 const struct bpf_func_proto bpf_probe_read_kernel_str_proto __weak;
19141906 const struct bpf_func_proto bpf_task_pt_regs_proto __weak;
1907+const struct bpf_func_proto bpf_perf_event_read_proto __weak;
1908+const struct bpf_func_proto bpf_send_signal_proto __weak;
1909+const struct bpf_func_proto bpf_send_signal_thread_proto __weak;
1910+const struct bpf_func_proto bpf_get_task_stack_sleepable_proto __weak;
1911+const struct bpf_func_proto bpf_get_task_stack_proto __weak;
1912+const struct bpf_func_proto bpf_get_branch_snapshot_proto __weak;
19151913
19161914 const struct bpf_func_proto *
19171915 bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
@@ -1965,6 +1963,8 @@ bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 47 unmodified lines
19651963 return &bpf_get_current_pid_tgid_proto;
19661964 case BPF_FUNC_get_ns_current_pid_tgid:
19671965 return &bpf_get_ns_current_pid_tgid_proto;
1966+ case BPF_FUNC_get_current_uid_gid:
1967+ return &bpf_get_current_uid_gid_proto;
19681968 default:
19691969 break;
19701970 }
@@ -2022,7 +2022,21 @@ bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 51 unmodified lines
20222022 return &bpf_get_current_cgroup_id_proto;
20232023 case BPF_FUNC_get_current_ancestor_cgroup_id:
20242024 return &bpf_get_current_ancestor_cgroup_id_proto;
2025+ case BPF_FUNC_current_task_under_cgroup:
2026+ return &bpf_current_task_under_cgroup_proto;
20252027 #endif
2028+#ifdef CONFIG_CGROUP_NET_CLASSID
2029+ case BPF_FUNC_get_cgroup_classid:
2030+ return &bpf_get_cgroup_classid_curr_proto;
2031+#endif
2032+ case BPF_FUNC_task_storage_get:
2033+ if (bpf_prog_check_recur(prog))
2034+ return &bpf_task_storage_get_recur_proto;
2035+ return &bpf_task_storage_get_proto;
2036+ case BPF_FUNC_task_storage_delete:
2037+ if (bpf_prog_check_recur(prog))
2038+ return &bpf_task_storage_delete_recur_proto;
2039+ return &bpf_task_storage_delete_proto;
20262040 default:
20272041 break;
20282042 }
@@ -2037,6 +2051,8 @@ bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 8 unmodified lines
20372051 return &bpf_get_current_task_proto;
20382052 case BPF_FUNC_get_current_task_btf:
20392053 return &bpf_get_current_task_btf_proto;
2054+ case BPF_FUNC_get_current_comm:
2055+ return &bpf_get_current_comm_proto;
20402056 case BPF_FUNC_probe_read_user:
20412057 return &bpf_probe_read_user_proto;
20422058 case BPF_FUNC_probe_read_kernel:
@@ -2047,6 +2063,10 @@ bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 4 unmodified lines
20472063 case BPF_FUNC_probe_read_kernel_str:
20482064 return security_locked_down(LOCKDOWN_BPF_READ_KERNEL) < 0 ?
20492065 NULL : &bpf_probe_read_kernel_str_proto;
2066+ case BPF_FUNC_copy_from_user:
2067+ return &bpf_copy_from_user_proto;
2068+ case BPF_FUNC_copy_from_user_task:
2069+ return &bpf_copy_from_user_task_proto;
20502070 case BPF_FUNC_snprintf_btf:
20512071 return &bpf_snprintf_btf_proto;
20522072 case BPF_FUNC_snprintf:
@@ -2057,6 +2077,19 @@ bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 4 unmodified lines
20572077 return bpf_get_trace_vprintk_proto();
20582078 case BPF_FUNC_perf_event_read_value:
20592079 return bpf_get_perf_event_read_value_proto();
2080+ case BPF_FUNC_perf_event_read:
2081+ return &bpf_perf_event_read_proto;
2082+ case BPF_FUNC_send_signal:
2083+ return &bpf_send_signal_proto;
2084+ case BPF_FUNC_send_signal_thread:
2085+ return &bpf_send_signal_thread_proto;
2086+ case BPF_FUNC_get_task_stack:
2087+ return prog->sleepable ? &bpf_get_task_stack_sleepable_proto
2088+ : &bpf_get_task_stack_proto;
2089+ case BPF_FUNC_get_branch_snapshot:
2090+ return &bpf_get_branch_snapshot_proto;
2091+ case BPF_FUNC_find_vma:
2092+ return &bpf_find_vma_proto;
20602093 default:
20612094 return NULL;
20622095 }
@@ -2293,6 +2326,26 @@ __bpf_kfunc struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) · 230 unmodified lines
22932326 return __bpf_list_del(head, true);
22942327 }
22952328
2329+__bpf_kfunc struct bpf_list_node *bpf_list_front(struct bpf_list_head *head)
2330+{
2331+ struct list_head *h = (struct list_head *)head;
2332+
2333+ if (list_empty(h) || unlikely(!h->next))
2334+ return NULL;
2335+
2336+ return (struct bpf_list_node *)h->next;
2337+}
2338+
2339+__bpf_kfunc struct bpf_list_node *bpf_list_back(struct bpf_list_head *head)
2340+{
2341+ struct list_head *h = (struct list_head *)head;
2342+
2343+ if (list_empty(h) || unlikely(!h->next))
2344+ return NULL;
2345+
2346+ return (struct bpf_list_node *)h->prev;
2347+}
2348+
22962349 __bpf_kfunc struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root,
22972350 struct bpf_rb_node *node)
22982351 {
@@ -2366,6 +2419,33 @@ __bpf_kfunc struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) · 67 unmodified lines
23662419 return (struct bpf_rb_node *)rb_first_cached(r);
23672420 }
23682421
2422+__bpf_kfunc struct bpf_rb_node *bpf_rbtree_root(struct bpf_rb_root *root)
2423+{
2424+ struct rb_root_cached *r = (struct rb_root_cached *)root;
2425+
2426+ return (struct bpf_rb_node *)r->rb_root.rb_node;
2427+}
2428+
2429+__bpf_kfunc struct bpf_rb_node *bpf_rbtree_left(struct bpf_rb_root *root, struct bpf_rb_node *node)
2430+{
2431+ struct bpf_rb_node_kern *node_internal = (struct bpf_rb_node_kern *)node;
2432+
2433+ if (READ_ONCE(node_internal->owner) != root)
2434+ return NULL;
2435+
2436+ return (struct bpf_rb_node *)node_internal->rb_node.rb_left;
2437+}
2438+
2439+__bpf_kfunc struct bpf_rb_node *bpf_rbtree_right(struct bpf_rb_root *root, struct bpf_rb_node *node)
2440+{
2441+ struct bpf_rb_node_kern *node_internal = (struct bpf_rb_node_kern *)node;
2442+
2443+ if (READ_ONCE(node_internal->owner) != root)
2444+ return NULL;
2445+
2446+ return (struct bpf_rb_node *)node_internal->rb_node.rb_right;
2447+}
2448+
23692449 /**
23702450 * bpf_task_acquire - Acquire a reference to a task. A task acquired by this
23712451 * kfunc which is not stored in a map as a kptr, must be released by calling
@@ -2923,9 +3003,9 @@ __bpf_kfunc int bpf_wq_start(struct bpf_wq *wq, unsigned int flags) · 551 unmodified lines
29233003 __bpf_kfunc int bpf_wq_set_callback_impl(struct bpf_wq *wq,
29243004 int (callback_fn)(void *map, int *key, void *value),
29253005 unsigned int flags,
2926− void *aux__ign)
3006+ void *aux__prog)
29273007 {
2928− struct bpf_prog_aux *aux = (struct bpf_prog_aux *)aux__ign;
3008+ struct bpf_prog_aux *aux = (struct bpf_prog_aux *)aux__prog;
29293009 struct bpf_async_kern *async = (struct bpf_async_kern *)wq;
29303010
29313011 if (flags)
@@ -3194,6 +3274,10 @@ __bpf_kfunc void bpf_local_irq_restore(unsigned long *flags__irq_flag) · 262 unmodified lines
31943274 local_irq_restore(*flags__irq_flag);
31953275 }
31963276
3277+__bpf_kfunc void __bpf_trap(void)
3278+{
3279+}
3280+
31973281 __bpf_kfunc_end_defs();
31983282
31993283 BTF_KFUNCS_START(generic_btf_ids)
@@ -3209,11 +3293,16 @@ BTF_ID_FLAGS(func, bpf_list_push_front_impl) · 9 unmodified lines
32093293 BTF_ID_FLAGS(func, bpf_list_push_back_impl)
32103294 BTF_ID_FLAGS(func, bpf_list_pop_front, KF_ACQUIRE | KF_RET_NULL)
32113295 BTF_ID_FLAGS(func, bpf_list_pop_back, KF_ACQUIRE | KF_RET_NULL)
3296+BTF_ID_FLAGS(func, bpf_list_front, KF_RET_NULL)
3297+BTF_ID_FLAGS(func, bpf_list_back, KF_RET_NULL)
32123298 BTF_ID_FLAGS(func, bpf_task_acquire, KF_ACQUIRE | KF_RCU | KF_RET_NULL)
32133299 BTF_ID_FLAGS(func, bpf_task_release, KF_RELEASE)
32143300 BTF_ID_FLAGS(func, bpf_rbtree_remove, KF_ACQUIRE | KF_RET_NULL)
32153301 BTF_ID_FLAGS(func, bpf_rbtree_add_impl)
32163302 BTF_ID_FLAGS(func, bpf_rbtree_first, KF_RET_NULL)
3303+BTF_ID_FLAGS(func, bpf_rbtree_root, KF_RET_NULL)
3304+BTF_ID_FLAGS(func, bpf_rbtree_left, KF_RET_NULL)
3305+BTF_ID_FLAGS(func, bpf_rbtree_right, KF_RET_NULL)
32173306
32183307 #ifdef CONFIG_CGROUPS
32193308 BTF_ID_FLAGS(func, bpf_cgroup_acquire, KF_ACQUIRE | KF_RCU | KF_RET_NULL)
@@ -3294,6 +3383,20 @@ BTF_ID_FLAGS(func, bpf_iter_kmem_cache_next, KF_ITER_NEXT | KF_RET_NULL | KF_SLE · 74 unmodified lines
32943383 BTF_ID_FLAGS(func, bpf_iter_kmem_cache_destroy, KF_ITER_DESTROY | KF_SLEEPABLE)
32953384 BTF_ID_FLAGS(func, bpf_local_irq_save)
32963385 BTF_ID_FLAGS(func, bpf_local_irq_restore)
3386+BTF_ID_FLAGS(func, bpf_probe_read_user_dynptr)
3387+BTF_ID_FLAGS(func, bpf_probe_read_kernel_dynptr)
3388+BTF_ID_FLAGS(func, bpf_probe_read_user_str_dynptr)
3389+BTF_ID_FLAGS(func, bpf_probe_read_kernel_str_dynptr)
3390+BTF_ID_FLAGS(func, bpf_copy_from_user_dynptr, KF_SLEEPABLE)
3391+BTF_ID_FLAGS(func, bpf_copy_from_user_str_dynptr, KF_SLEEPABLE)
3392+BTF_ID_FLAGS(func, bpf_copy_from_user_task_dynptr, KF_SLEEPABLE | KF_TRUSTED_ARGS)
3393+BTF_ID_FLAGS(func, bpf_copy_from_user_task_str_dynptr, KF_SLEEPABLE | KF_TRUSTED_ARGS)
3394+#ifdef CONFIG_DMA_SHARED_BUFFER
3395+BTF_ID_FLAGS(func, bpf_iter_dmabuf_new, KF_ITER_NEW | KF_SLEEPABLE)
3396+BTF_ID_FLAGS(func, bpf_iter_dmabuf_next, KF_ITER_NEXT | KF_RET_NULL | KF_SLEEPABLE)
3397+BTF_ID_FLAGS(func, bpf_iter_dmabuf_destroy, KF_ITER_DESTROY | KF_SLEEPABLE)
3398+#endif
3399+BTF_ID_FLAGS(func, __bpf_trap)
32973400 BTF_KFUNCS_END(common_btf_ids)
32983401
32993402 static const struct btf_kfunc_id_set common_kfunc_set = {
kernel/bpf/syscall.c+6−4
@@ -36,6 +36,7 @@ · 35 unmodified lines
3636 #include <linux/memcontrol.h>
3737 #include <linux/trace_events.h>
3838 #include <linux/tracepoint.h>
39+#include <linux/overflow.h>
3940
4041 #include <net/netfilter/nf_bpf_link.h>
4142 #include <net/netkit.h>
@@ -693,7 +694,7 @@ struct btf_record *btf_record_dup(const struct btf_record *rec) · 651 unmodified lines
693694
694695 if (IS_ERR_OR_NULL(rec))
695696 return NULL;
696− size = offsetof(struct btf_record, fields[rec->cnt]);
697+ size = struct_size(rec, fields, rec->cnt);
697698 new_rec = kmemdup(rec, size, GFP_KERNEL | __GFP_NOWARN);
698699 if (!new_rec)
699700 return ERR_PTR(-ENOMEM);
@@ -748,7 +749,7 @@ bool btf_record_equal(const struct btf_record *rec_a, const struct btf_record *r · 48 unmodified lines
748749 return false;
749750 if (rec_a->cnt != rec_b->cnt)
750751 return false;
751− size = offsetof(struct btf_record, fields[rec_a->cnt]);
752+ size = struct_size(rec_a, fields, rec_a->cnt);
752753 /* btf_parse_fields uses kzalloc to allocate a btf_record, so unused
753754 * members are zeroed out. So memcmp is safe to do without worrying
754755 * about padding/unused fields.
@@ -3799,14 +3800,14 @@ static int bpf_perf_link_fill_kprobe(const struct perf_event *event, · 3044 unmodified lines
37993800 static int bpf_perf_link_fill_uprobe(const struct perf_event *event,
38003801 struct bpf_link_info *info)
38013802 {
3803+ u64 ref_ctr_offset, offset;
38023804 char __user *uname;
3803− u64 addr, offset;
38043805 u32 ulen, type;
38053806 int err;
38063807
38073808 uname = u64_to_user_ptr(info->perf_event.uprobe.file_name);
38083809 ulen = info->perf_event.uprobe.name_len;
3809− err = bpf_perf_link_fill_common(event, uname, &ulen, &offset, &addr,
3810+ err = bpf_perf_link_fill_common(event, uname, &ulen, &offset, &ref_ctr_offset,
38103811 &type, NULL);
38113812 if (err)
38123813 return err;
@@ -3818,6 +3819,7 @@ static int bpf_perf_link_fill_uprobe(const struct perf_event *event, · 5 unmodified lines
38183819 info->perf_event.uprobe.name_len = ulen;
38193820 info->perf_event.uprobe.offset = offset;
38203821 info->perf_event.uprobe.cookie = event->bpf_cookie;
3822+ info->perf_event.uprobe.ref_ctr_offset = ref_ctr_offset;
38213823 return 0;
38223824 }
38233825 #endif
kernel/bpf/sysfs_btf.c+32−0
@@ -7,14 +7,46 @@ · 6 unmodified lines
77 #include <linux/kobject.h>
88 #include <linux/init.h>
99 #include <linux/sysfs.h>
10+#include <linux/mm.h>
11+#include <linux/io.h>
12+#include <linux/btf.h>
1013
1114 /* See scripts/link-vmlinux.sh, gen_btf() func for details */
1215 extern char __start_BTF[];
1316 extern char __stop_BTF[];
1417
18+static int btf_sysfs_vmlinux_mmap(struct file *filp, struct kobject *kobj,
19+ const struct bin_attribute *attr,
20+ struct vm_area_struct *vma)
21+{
22+ unsigned long pages = PAGE_ALIGN(attr->size) >> PAGE_SHIFT;
23+ size_t vm_size = vma->vm_end - vma->vm_start;
24+ phys_addr_t addr = virt_to_phys(__start_BTF);
25+ unsigned long pfn = addr >> PAGE_SHIFT;
26+
27+ if (attr->private != __start_BTF || !PAGE_ALIGNED(addr))
28+ return -EINVAL;
29+
30+ if (vma->vm_pgoff)
31+ return -EINVAL;
32+
33+ if (vma->vm_flags & (VM_WRITE | VM_EXEC | VM_MAYSHARE))
34+ return -EACCES;
35+
36+ if (pfn + pages < pfn)
37+ return -EINVAL;
38+
39+ if ((vm_size >> PAGE_SHIFT) > pages)
40+ return -EINVAL;
41+
42+ vm_flags_mod(vma, VM_DONTDUMP, VM_MAYEXEC | VM_MAYWRITE);
43+ return remap_pfn_range(vma, vma->vm_start, pfn, vm_size, vma->vm_page_prot);
44+}
45+
1546 static struct bin_attribute bin_attr_btf_vmlinux __ro_after_init = {
1647 .attr = { .name = "vmlinux", .mode = 0444, },
1748 .read_new = sysfs_bin_attr_simple_read,
49+ .mmap = btf_sysfs_vmlinux_mmap,
1850 };
1951
2052 struct kobject *btf_kobj;
kernel/bpf/verifier.c+334−302
@@ -322,6 +322,7 @@ struct bpf_kfunc_call_arg_meta { · 321 unmodified lines
322322 struct btf *arg_btf;
323323 u32 arg_btf_id;
324324 bool arg_owning_ref;
325+ bool arg_prog;
325326
326327 struct {
327328 struct btf_field *field;
@@ -1923,11 +1924,8 @@ static struct bpf_verifier_state *get_loop_entry(struct bpf_verifier_env *env, · 1595 unmodified lines
19231924 u32 steps = 0;
19241925
19251926 while (topmost && topmost->loop_entry) {
1926− if (steps++ > st->dfs_depth) {
1927− WARN_ONCE(true, "verifier bug: infinite loop in get_loop_entry\n");
1928− verbose(env, "verifier bug: infinite loop in get_loop_entry()\n");
1927+ if (verifier_bug_if(steps++ > st->dfs_depth, env, "infinite loop"))
19291928 return ERR_PTR(-EFAULT);
1930− }
19311929 topmost = topmost->loop_entry;
19321930 }
19331931 return topmost;
@@ -3459,12 +3457,11 @@ static int mark_reg_read(struct bpf_verifier_env *env, · 1525 unmodified lines
34593457 /* if read wasn't screened by an earlier write ... */
34603458 if (writes && state->live & REG_LIVE_WRITTEN)
34613459 break;
3462− if (parent->live & REG_LIVE_DONE) {
3463− verbose(env, "verifier BUG type %s var_off %lld off %d\n",
3464− reg_type_str(env, parent->type),
3465− parent->var_off.value, parent->off);
3460+ if (verifier_bug_if(parent->live & REG_LIVE_DONE, env,
3461+ "type %s var_off %lld off %d",
3462+ reg_type_str(env, parent->type),
3463+ parent->var_off.value, parent->off))
34663464 return -EFAULT;
3467− }
34683465 /* The first condition is more likely to be true than the
34693466 * second, checked it first.
34703467 */
@@ -3649,16 +3646,16 @@ static int insn_def_regno(const struct bpf_insn *insn) · 178 unmodified lines
36493646 case BPF_ST:
36503647 return -1;
36513648 case BPF_STX:
3652− if ((BPF_MODE(insn->code) == BPF_ATOMIC ||
3653− BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) &&
3654− (insn->imm & BPF_FETCH)) {
3649+ if (BPF_MODE(insn->code) == BPF_ATOMIC ||
3650+ BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) {
36553651 if (insn->imm == BPF_CMPXCHG)
36563652 return BPF_REG_0;
3657− else
3653+ else if (insn->imm == BPF_LOAD_ACQ)
3654+ return insn->dst_reg;
3655+ else if (insn->imm & BPF_FETCH)
36583656 return insn->src_reg;
3659− } else {
3660− return -1;
36613657 }
3658+ return -1;
36623659 default:
36633660 return insn->dst_reg;
36643661 }
@@ -3857,14 +3854,14 @@ static int push_insn_history(struct bpf_verifier_env *env, struct bpf_verifier_s · 192 unmodified lines
38573854 /* atomic instructions push insn_flags twice, for READ and
38583855 * WRITE sides, but they should agree on stack slot
38593856 */
3860− WARN_ONCE((env->cur_hist_ent->flags & insn_flags) &&
3861− (env->cur_hist_ent->flags & insn_flags) != insn_flags,
3862− "verifier insn history bug: insn_idx %d cur flags %x new flags %x\n",
3863− env->insn_idx, env->cur_hist_ent->flags, insn_flags);
3857+ verifier_bug_if((env->cur_hist_ent->flags & insn_flags) &&
3858+ (env->cur_hist_ent->flags & insn_flags) != insn_flags,
3859+ env, "insn history: insn_idx %d cur flags %x new flags %x",
3860+ env->insn_idx, env->cur_hist_ent->flags, insn_flags);
38643861 env->cur_hist_ent->flags |= insn_flags;
3865− WARN_ONCE(env->cur_hist_ent->linked_regs != 0,
3866− "verifier insn history bug: insn_idx %d linked_regs != 0: %#llx\n",
3867− env->insn_idx, env->cur_hist_ent->linked_regs);
3862+ verifier_bug_if(env->cur_hist_ent->linked_regs != 0, env,
3863+ "insn history: insn_idx %d linked_regs: %#llx",
3864+ env->insn_idx, env->cur_hist_ent->linked_regs);
38683865 env->cur_hist_ent->linked_regs = linked_regs;
38693866 return 0;
38703867 }
@@ -3987,8 +3984,7 @@ static inline u32 bt_empty(struct backtrack_state *bt) · 116 unmodified lines
39873984 static inline int bt_subprog_enter(struct backtrack_state *bt)
39883985 {
39893986 if (bt->frame == MAX_CALL_FRAMES - 1) {
3990− verbose(bt->env, "BUG subprog enter from frame %d\n", bt->frame);
3991− WARN_ONCE(1, "verifier backtracking bug");
3987+ verifier_bug(bt->env, "subprog enter from frame %d", bt->frame);
39923988 return -EFAULT;
39933989 }
39943990 bt->frame++;
@@ -3998,8 +3994,7 @@ static inline int bt_subprog_enter(struct backtrack_state *bt) · 3 unmodified lines
39983994 static inline int bt_subprog_exit(struct backtrack_state *bt)
39993995 {
40003996 if (bt->frame == 0) {
4001− verbose(bt->env, "BUG subprog exit from frame 0\n");
4002− WARN_ONCE(1, "verifier backtracking bug");
3997+ verifier_bug(bt->env, "subprog exit from frame 0");
40033998 return -EFAULT;
40043999 }
40054000 bt->frame--;
@@ -4277,14 +4272,15 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, · 271 unmodified lines
42774272 * should be literally next instruction in
42784273 * caller program
42794274 */
4280− WARN_ONCE(idx + 1 != subseq_idx, "verifier backtracking bug");
4275+ verifier_bug_if(idx + 1 != subseq_idx, env,
4276+ "extra insn from subprog");
42814277 /* r1-r5 are invalidated after subprog call,
42824278 * so for global func call it shouldn't be set
42834279 * anymore
42844280 */
42854281 if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) {
4286− verbose(env, "BUG regs %x\n", bt_reg_mask(bt));
4287− WARN_ONCE(1, "verifier backtracking bug");
4282+ verifier_bug(env, "global subprog unexpected regs %x",
4283+ bt_reg_mask(bt));
42884284 return -EFAULT;
42894285 }
42904286 /* global subprog always sets R0 */
@@ -4298,16 +4294,17 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, · 7 unmodified lines
42984294 * the current frame should be zero by now
42994295 */
43004296 if (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) {
4301− verbose(env, "BUG regs %x\n", bt_reg_mask(bt));
4302− WARN_ONCE(1, "verifier backtracking bug");
4297+ verifier_bug(env, "static subprog unexpected regs %x",
4298+ bt_reg_mask(bt));
43034299 return -EFAULT;
43044300 }
43054301 /* we are now tracking register spills correctly,
43064302 * so any instance of leftover slots is a bug
43074303 */
43084304 if (bt_stack_mask(bt) != 0) {
4309− verbose(env, "BUG stack slots %llx\n", bt_stack_mask(bt));
4310− WARN_ONCE(1, "verifier backtracking bug (subprog leftover stack slots)");
4305+ verifier_bug(env,
4306+ "static subprog leftover stack slots %llx",
4307+ bt_stack_mask(bt));
43114308 return -EFAULT;
43124309 }
43134310 /* propagate r1-r5 to the caller */
@@ -4330,13 +4327,13 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, · 16 unmodified lines
43304327 * not actually arguments passed directly to callback subprogs
43314328 */
43324329 if (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) {
4333− verbose(env, "BUG regs %x\n", bt_reg_mask(bt));
4334− WARN_ONCE(1, "verifier backtracking bug");
4330+ verifier_bug(env, "callback unexpected regs %x",
4331+ bt_reg_mask(bt));
43354332 return -EFAULT;
43364333 }
43374334 if (bt_stack_mask(bt) != 0) {
4338− verbose(env, "BUG stack slots %llx\n", bt_stack_mask(bt));
4339− WARN_ONCE(1, "verifier backtracking bug (callback leftover stack slots)");
4335+ verifier_bug(env, "callback leftover stack slots %llx",
4336+ bt_stack_mask(bt));
43404337 return -EFAULT;
43414338 }
43424339 /* clear r1-r5 in callback subprog's mask */
@@ -4355,11 +4352,11 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, · 12 unmodified lines
43554352 /* regular helper call sets R0 */
43564353 bt_clear_reg(bt, BPF_REG_0);
43574354 if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) {
4358− /* if backtracing was looking for registers R1-R5
4355+ /* if backtracking was looking for registers R1-R5
43594356 * they should have been found already.
43604357 */
4361− verbose(env, "BUG regs %x\n", bt_reg_mask(bt));
4362− WARN_ONCE(1, "verifier backtracking bug");
4358+ verifier_bug(env, "backtracking call unexpected regs %x",
4359+ bt_reg_mask(bt));
43634360 return -EFAULT;
43644361 }
43654362 } else if (opcode == BPF_EXIT) {
@@ -4377,8 +4374,8 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, · 11 unmodified lines
43774374 for (i = BPF_REG_1; i <= BPF_REG_5; i++)
43784375 bt_clear_reg(bt, i);
43794376 if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) {
4380− verbose(env, "BUG regs %x\n", bt_reg_mask(bt));
4381− WARN_ONCE(1, "verifier backtracking bug");
4377+ verifier_bug(env, "backtracking exit unexpected regs %x",
4378+ bt_reg_mask(bt));
43824379 return -EFAULT;
43834380 }
43844381
@@ -4413,8 +4410,10 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, · 28 unmodified lines
44134410 * before it would be equally necessary to
44144411 * propagate it to dreg.
44154412 */
4416− bt_set_reg(bt, dreg);
4417− bt_set_reg(bt, sreg);
4413+ if (!hist || !(hist->flags & INSN_F_SRC_REG_STACK))
4414+ bt_set_reg(bt, sreg);
4415+ if (!hist || !(hist->flags & INSN_F_DST_REG_STACK))
4416+ bt_set_reg(bt, dreg);
44184417 } else if (BPF_SRC(insn->code) == BPF_K) {
44194418 /* dreg <cond> K
44204419 * Only dreg still needs precision before
@@ -4719,9 +4718,8 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int regno) · 298 unmodified lines
47194718 return 0;
47204719 }
47214720
4722− verbose(env, "BUG backtracking func entry subprog %d reg_mask %x stack_mask %llx\n",
4723− st->frame[0]->subprogno, bt_reg_mask(bt), bt_stack_mask(bt));
4724− WARN_ONCE(1, "verifier backtracking bug");
4721+ verifier_bug(env, "backtracking func entry subprog %d reg_mask %x stack_mask %llx",
4722+ st->frame[0]->subprogno, bt_reg_mask(bt), bt_stack_mask(bt));
47254723 return -EFAULT;
47264724 }
47274725
@@ -4757,8 +4755,7 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int regno) · 29 unmodified lines
47574755 * It means the backtracking missed the spot where
47584756 * particular register was initialized with a constant.
47594757 */
4760− verbose(env, "BUG backtracking idx %d\n", i);
4761− WARN_ONCE(1, "verifier backtracking bug");
4758+ verifier_bug(env, "backtracking idx %d", i);
47624759 return -EFAULT;
47634760 }
47644761 }
@@ -4783,12 +4780,10 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int regno) · 18 unmodified lines
47834780
47844781 bitmap_from_u64(mask, bt_frame_stack_mask(bt, fr));
47854782 for_each_set_bit(i, mask, 64) {
4786− if (i >= func->allocated_stack / BPF_REG_SIZE) {
4787− verbose(env, "BUG backtracking (stack slot %d, total slots %d)\n",
4788− i, func->allocated_stack / BPF_REG_SIZE);
4789− WARN_ONCE(1, "verifier backtracking bug (stack slot out of bounds)");
4783+ if (verifier_bug_if(i >= func->allocated_stack / BPF_REG_SIZE,
4784+ env, "stack slot %d, total slots %d",
4785+ i, func->allocated_stack / BPF_REG_SIZE))
47904786 return -EFAULT;
4791− }
47924787
47934788 if (!is_spilled_scalar_reg(&func->stack[i])) {
47944789 bt_clear_frame_slot(bt, fr, i);
@@ -6561,21 +6556,18 @@ continue_func: · 1766 unmodified lines
65616556 /* find the callee */
65626557 next_insn = i + insn[i].imm + 1;
65636558 sidx = find_subprog(env, next_insn);
6564− if (sidx < 0) {
6565− WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
6566− next_insn);
6559+ if (verifier_bug_if(sidx < 0, env, "callee not found at insn %d", next_insn))
65676560 return -EFAULT;
6568− }
65696561 if (subprog[sidx].is_async_cb) {
65706562 if (subprog[sidx].has_tail_call) {
6571− verbose(env, "verifier bug. subprog has tail_call and async cb\n");
6563+ verifier_bug(env, "subprog has tail_call and async cb");
65726564 return -EFAULT;
65736565 }
65746566 /* async callbacks don't increase bpf prog stack size unless called directly */
65756567 if (!bpf_pseudo_call(insn + i))
65766568 continue;
65776569 if (subprog[sidx].is_exception_cb) {
6578− verbose(env, "insn %d cannot call exception cb directly\n", i);
6570+ verbose(env, "insn %d cannot call exception cb directly", i);
65796571 return -EINVAL;
65806572 }
65816573 }
@@ -6675,11 +6667,8 @@ static int get_callee_stack_depth(struct bpf_verifier_env *env, · 93 unmodified lines
66756667 int start = idx + insn->imm + 1, subprog;
66766668
66776669 subprog = find_subprog(env, start);
6678− if (subprog < 0) {
6679− WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
6680− start);
6670+ if (verifier_bug_if(subprog < 0, env, "get stack depth: no program at insn %d", start))
66816671 return -EFAULT;
6682− }
66836672 return env->subprog_info[subprog].stack_depth;
66846673 }
66856674 #endif
@@ -7984,7 +7973,7 @@ static int check_stack_range_initialized( · 1298 unmodified lines
79847973 slot = -i - 1;
79857974 spi = slot / BPF_REG_SIZE;
79867975 if (state->allocated_stack <= slot) {
7987− verbose(env, "verifier bug: allocated_stack too small\n");
7976+ verbose(env, "allocated_stack too small\n");
79887977 return -EFAULT;
79897978 }
79907979
@@ -8413,7 +8402,7 @@ static int process_timer_func(struct bpf_verifier_env *env, int regno, · 422 unmodified lines
84138402 return -EINVAL;
84148403 }
84158404 if (meta->map_ptr) {
8416− verbose(env, "verifier bug. Two map pointers in a timer helper\n");
8405+ verifier_bug(env, "Two map pointers in a timer helper");
84178406 return -EFAULT;
84188407 }
84198408 meta->map_uid = reg->map_uid;
@@ -10285,8 +10274,7 @@ static int setup_func_entry(struct bpf_verifier_env *env, int subprog, int calls · 1865 unmodified lines
1028510274 }
1028610275
1028710276 if (state->frame[state->curframe + 1]) {
10288− verbose(env, "verifier bug. Frame %d already allocated\n",
10289− state->curframe + 1);
10277+ verifier_bug(env, "Frame %d already allocated", state->curframe + 1);
1029010278 return -EFAULT;
1029110279 }
1029210280
@@ -10400,8 +10388,7 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, · 107 unmodified lines
1040010388 if (err)
1040110389 return err;
1040210390 } else {
10403− bpf_log(log, "verifier bug: unrecognized arg#%d type %d\n",
10404− i, arg->arg_type);
10391+ verifier_bug(env, "unrecognized arg#%d type %d", i, arg->arg_type);
1040510392 return -EFAULT;
1040610393 }
1040710394 }
@@ -10464,13 +10451,13 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins · 56 unmodified lines
1046410451 env->subprog_info[subprog].is_cb = true;
1046510452 if (bpf_pseudo_kfunc_call(insn) &&
1046610453 !is_callback_calling_kfunc(insn->imm)) {
10467− verbose(env, "verifier bug: kfunc %s#%d not marked as callback-calling\n",
10468− func_id_name(insn->imm), insn->imm);
10454+ verifier_bug(env, "kfunc %s#%d not marked as callback-calling",
10455+ func_id_name(insn->imm), insn->imm);
1046910456 return -EFAULT;
1047010457 } else if (!bpf_pseudo_kfunc_call(insn) &&
1047110458 !is_callback_calling_function(insn->imm)) { /* helper */
10472− verbose(env, "verifier bug: helper %s#%d not marked as callback-calling\n",
10473− func_id_name(insn->imm), insn->imm);
10459+ verifier_bug(env, "helper %s#%d not marked as callback-calling",
10460+ func_id_name(insn->imm), insn->imm);
1047410461 return -EFAULT;
1047510462 }
1047610463
@@ -10522,10 +10509,9 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · 45 unmodified lines
1052210509
1052310510 target_insn = *insn_idx + insn->imm + 1;
1052410511 subprog = find_subprog(env, target_insn);
10525− if (subprog < 0) {
10526− verbose(env, "verifier bug. No program starts at insn %d\n", target_insn);
10512+ if (verifier_bug_if(subprog < 0, env, "target of func call at insn %d is not a program",
10513+ target_insn))
1052710514 return -EFAULT;
10528− }
1052910515
1053010516 caller = state->frame[state->curframe];
1053110517 err = btf_check_subprog_call(env, subprog, caller->regs);
@@ -11124,7 +11110,7 @@ static int check_bpf_snprintf_call(struct bpf_verifier_env *env, · 592 unmodified lines
1112411110 err = fmt_map->ops->map_direct_value_addr(fmt_map, &fmt_addr,
1112511111 fmt_map_off);
1112611112 if (err) {
11127− verbose(env, "verifier bug\n");
11113+ verbose(env, "failed to retrieve map value address\n");
1112811114 return -EFAULT;
1112911115 }
1113011116 fmt = (char *)(long)fmt_addr + fmt_map_off;
@@ -11897,6 +11883,11 @@ static bool is_kfunc_arg_irq_flag(const struct btf *btf, const struct btf_param · 766 unmodified lines
1189711883 return btf_param_match_suffix(btf, arg, "__irq_flag");
1189811884 }
1189911885
11886+static bool is_kfunc_arg_prog(const struct btf *btf, const struct btf_param *arg)
11887+{
11888+ return btf_param_match_suffix(btf, arg, "__prog");
11889+}
11890+
1190011891 static bool is_kfunc_arg_scalar_with_name(const struct btf *btf,
1190111892 const struct btf_param *arg,
1190211893 const char *name)
@@ -11987,6 +11978,16 @@ static bool is_kfunc_arg_res_spin_lock(const struct btf *btf, const struct btf_p · 84 unmodified lines
1198711978 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RES_SPIN_LOCK_ID);
1198811979 }
1198911980
11981+static bool is_rbtree_node_type(const struct btf_type *t)
11982+{
11983+ return t == btf_type_by_id(btf_vmlinux, kf_arg_btf_ids[KF_ARG_RB_NODE_ID]);
11984+}
11985+
11986+static bool is_list_node_type(const struct btf_type *t)
11987+{
11988+ return t == btf_type_by_id(btf_vmlinux, kf_arg_btf_ids[KF_ARG_LIST_NODE_ID]);
11989+}
11990+
1199011991 static bool is_kfunc_arg_callback(struct bpf_verifier_env *env, const struct btf *btf,
1199111992 const struct btf_param *arg)
1199211993 {
@@ -12069,6 +12070,8 @@ enum special_kfunc_type { · 76 unmodified lines
1206912070 KF_bpf_list_push_back_impl,
1207012071 KF_bpf_list_pop_front,
1207112072 KF_bpf_list_pop_back,
12073+ KF_bpf_list_front,
12074+ KF_bpf_list_back,
1207212075 KF_bpf_cast_to_kern_ctx,
1207312076 KF_bpf_rdonly_cast,
1207412077 KF_bpf_rcu_read_lock,
@@ -12076,6 +12079,9 @@ enum special_kfunc_type { · 1 unmodified line
1207612079 KF_bpf_rbtree_remove,
1207712080 KF_bpf_rbtree_add_impl,
1207812081 KF_bpf_rbtree_first,
12082+ KF_bpf_rbtree_root,
12083+ KF_bpf_rbtree_left,
12084+ KF_bpf_rbtree_right,
1207912085 KF_bpf_dynptr_from_skb,
1208012086 KF_bpf_dynptr_from_xdp,
1208112087 KF_bpf_dynptr_slice,
@@ -12101,41 +12107,9 @@ enum special_kfunc_type { · 19 unmodified lines
1210112107 KF_bpf_res_spin_unlock,
1210212108 KF_bpf_res_spin_lock_irqsave,
1210312109 KF_bpf_res_spin_unlock_irqrestore,
12110+ KF___bpf_trap,
1210412111 };
1210512112
12106−BTF_SET_START(special_kfunc_set)
12107−BTF_ID(func, bpf_obj_new_impl)
12108−BTF_ID(func, bpf_obj_drop_impl)
12109−BTF_ID(func, bpf_refcount_acquire_impl)
12110−BTF_ID(func, bpf_list_push_front_impl)
12111−BTF_ID(func, bpf_list_push_back_impl)
12112−BTF_ID(func, bpf_list_pop_front)
12113−BTF_ID(func, bpf_list_pop_back)
12114−BTF_ID(func, bpf_cast_to_kern_ctx)
12115−BTF_ID(func, bpf_rdonly_cast)
12116−BTF_ID(func, bpf_rbtree_remove)
12117−BTF_ID(func, bpf_rbtree_add_impl)
12118−BTF_ID(func, bpf_rbtree_first)
12119−#ifdef CONFIG_NET
12120−BTF_ID(func, bpf_dynptr_from_skb)
12121−BTF_ID(func, bpf_dynptr_from_xdp)
12122−#endif
12123−BTF_ID(func, bpf_dynptr_slice)
12124−BTF_ID(func, bpf_dynptr_slice_rdwr)
12125−BTF_ID(func, bpf_dynptr_clone)
12126−BTF_ID(func, bpf_percpu_obj_new_impl)
12127−BTF_ID(func, bpf_percpu_obj_drop_impl)
12128−BTF_ID(func, bpf_throw)
12129−BTF_ID(func, bpf_wq_set_callback_impl)
12130−#ifdef CONFIG_CGROUPS
12131−BTF_ID(func, bpf_iter_css_task_new)
12132−#endif
12133−#ifdef CONFIG_BPF_LSM
12134−BTF_ID(func, bpf_set_dentry_xattr)
12135−BTF_ID(func, bpf_remove_dentry_xattr)
12136−#endif
12137−BTF_SET_END(special_kfunc_set)
12138−
1213912113 BTF_ID_LIST(special_kfunc_list)
1214012114 BTF_ID(func, bpf_obj_new_impl)
1214112115 BTF_ID(func, bpf_obj_drop_impl)
@@ -12144,6 +12118,8 @@ BTF_ID(func, bpf_list_push_front_impl) · 2 unmodified lines
1214412118 BTF_ID(func, bpf_list_push_back_impl)
1214512119 BTF_ID(func, bpf_list_pop_front)
1214612120 BTF_ID(func, bpf_list_pop_back)
12121+BTF_ID(func, bpf_list_front)
12122+BTF_ID(func, bpf_list_back)
1214712123 BTF_ID(func, bpf_cast_to_kern_ctx)
1214812124 BTF_ID(func, bpf_rdonly_cast)
1214912125 BTF_ID(func, bpf_rcu_read_lock)
@@ -12151,6 +12127,9 @@ BTF_ID(func, bpf_rcu_read_unlock) · 1 unmodified line
1215112127 BTF_ID(func, bpf_rbtree_remove)
1215212128 BTF_ID(func, bpf_rbtree_add_impl)
1215312129 BTF_ID(func, bpf_rbtree_first)
12130+BTF_ID(func, bpf_rbtree_root)
12131+BTF_ID(func, bpf_rbtree_left)
12132+BTF_ID(func, bpf_rbtree_right)
1215412133 #ifdef CONFIG_NET
1215512134 BTF_ID(func, bpf_dynptr_from_skb)
1215612135 BTF_ID(func, bpf_dynptr_from_xdp)
@@ -12194,6 +12173,7 @@ BTF_ID(func, bpf_res_spin_lock) · 37 unmodified lines
1219412173 BTF_ID(func, bpf_res_spin_unlock)
1219512174 BTF_ID(func, bpf_res_spin_lock_irqsave)
1219612175 BTF_ID(func, bpf_res_spin_unlock_irqrestore)
12176+BTF_ID(func, __bpf_trap)
1219712177
1219812178 static bool is_kfunc_ret_null(struct bpf_kfunc_call_arg_meta *meta)
1219912179 {
@@ -12579,14 +12559,19 @@ static bool is_bpf_list_api_kfunc(u32 btf_id) · 379 unmodified lines
1257912559 return btf_id == special_kfunc_list[KF_bpf_list_push_front_impl] ||
1258012560 btf_id == special_kfunc_list[KF_bpf_list_push_back_impl] ||
1258112561 btf_id == special_kfunc_list[KF_bpf_list_pop_front] ||
12582− btf_id == special_kfunc_list[KF_bpf_list_pop_back];
12562+ btf_id == special_kfunc_list[KF_bpf_list_pop_back] ||
12563+ btf_id == special_kfunc_list[KF_bpf_list_front] ||
12564+ btf_id == special_kfunc_list[KF_bpf_list_back];
1258312565 }
1258412566
1258512567 static bool is_bpf_rbtree_api_kfunc(u32 btf_id)
1258612568 {
1258712569 return btf_id == special_kfunc_list[KF_bpf_rbtree_add_impl] ||
1258812570 btf_id == special_kfunc_list[KF_bpf_rbtree_remove] ||
12589− btf_id == special_kfunc_list[KF_bpf_rbtree_first];
12571+ btf_id == special_kfunc_list[KF_bpf_rbtree_first] ||
12572+ btf_id == special_kfunc_list[KF_bpf_rbtree_root] ||
12573+ btf_id == special_kfunc_list[KF_bpf_rbtree_left] ||
12574+ btf_id == special_kfunc_list[KF_bpf_rbtree_right];
1259012575 }
1259112576
1259212577 static bool is_bpf_iter_num_api_kfunc(u32 btf_id)
@@ -12686,7 +12671,9 @@ static bool check_kfunc_is_graph_node_api(struct bpf_verifier_env *env, · 93 unmodified lines
1268612671 break;
1268712672 case BPF_RB_NODE:
1268812673 ret = (kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_remove] ||
12689− kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_add_impl]);
12674+ kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_add_impl] ||
12675+ kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_left] ||
12676+ kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_right]);
1269012677 break;
1269112678 default:
1269212679 verbose(env, "verifier internal error: unexpected graph node argument type %s\n",
@@ -12906,6 +12893,17 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ · 213 unmodified lines
1290612893 if (is_kfunc_arg_ignore(btf, &args[i]))
1290712894 continue;
1290812895
12896+ if (is_kfunc_arg_prog(btf, &args[i])) {
12897+ /* Used to reject repeated use of __prog. */
12898+ if (meta->arg_prog) {
12899+ verbose(env, "Only 1 prog->aux argument supported per-kfunc\n");
12900+ return -EFAULT;
12901+ }
12902+ meta->arg_prog = true;
12903+ cur_aux(env)->arg_prog = regno;
12904+ continue;
12905+ }
12906+
1290912907 if (btf_type_is_scalar(t)) {
1291012908 if (reg->type != SCALAR_VALUE) {
1291112909 verbose(env, "R%d is not a scalar\n", regno);
@@ -13200,22 +13198,22 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ · 288 unmodified lines
1320013198 return ret;
1320113199 break;
1320213200 case KF_ARG_PTR_TO_RB_NODE:
13203− if (meta->func_id == special_kfunc_list[KF_bpf_rbtree_remove]) {
13204− if (!type_is_non_owning_ref(reg->type) || reg->ref_obj_id) {
13205− verbose(env, "rbtree_remove node input must be non-owning ref\n");
13201+ if (meta->func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) {
13202+ if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
13203+ verbose(env, "arg#%d expected pointer to allocated object\n", i);
1320613204 return -EINVAL;
1320713205 }
13208− if (in_rbtree_lock_required_cb(env)) {
13209− verbose(env, "rbtree_remove not allowed in rbtree cb\n");
13206+ if (!reg->ref_obj_id) {
13207+ verbose(env, "allocated object must be referenced\n");
1321013208 return -EINVAL;
1321113209 }
1321213210 } else {
13213− if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
13214− verbose(env, "arg#%d expected pointer to allocated object\n", i);
13211+ if (!type_is_non_owning_ref(reg->type) && !reg->ref_obj_id) {
13212+ verbose(env, "%s can only take non-owning or refcounted bpf_rb_node pointer\n", func_name);
1321513213 return -EINVAL;
1321613214 }
13217− if (!reg->ref_obj_id) {
13218− verbose(env, "allocated object must be referenced\n");
13215+ if (in_rbtree_lock_required_cb(env)) {
13216+ verbose(env, "%s not allowed in rbtree cb\n", func_name);
1321913217 return -EINVAL;
1322013218 }
1322113219 }
@@ -13420,6 +13418,178 @@ static int fetch_kfunc_meta(struct bpf_verifier_env *env, · 198 unmodified lines
1342013418 return 0;
1342113419 }
1342213420
13421+/* check special kfuncs and return:
13422+ * 1 - not fall-through to 'else' branch, continue verification
13423+ * 0 - fall-through to 'else' branch
13424+ * < 0 - not fall-through to 'else' branch, return error
13425+ */
13426+static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta,
13427+ struct bpf_reg_state *regs, struct bpf_insn_aux_data *insn_aux,
13428+ const struct btf_type *ptr_type, struct btf *desc_btf)
13429+{
13430+ const struct btf_type *ret_t;
13431+ int err = 0;
13432+
13433+ if (meta->btf != btf_vmlinux)
13434+ return 0;
13435+
13436+ if (meta->func_id == special_kfunc_list[KF_bpf_obj_new_impl] ||
13437+ meta->func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) {
13438+ struct btf_struct_meta *struct_meta;
13439+ struct btf *ret_btf;
13440+ u32 ret_btf_id;
13441+
13442+ if (meta->func_id == special_kfunc_list[KF_bpf_obj_new_impl] && !bpf_global_ma_set)
13443+ return -ENOMEM;
13444+
13445+ if (((u64)(u32)meta->arg_constant.value) != meta->arg_constant.value) {
13446+ verbose(env, "local type ID argument must be in range [0, U32_MAX]\n");
13447+ return -EINVAL;
13448+ }
13449+
13450+ ret_btf = env->prog->aux->btf;
13451+ ret_btf_id = meta->arg_constant.value;
13452+
13453+ /* This may be NULL due to user not supplying a BTF */
13454+ if (!ret_btf) {
13455+ verbose(env, "bpf_obj_new/bpf_percpu_obj_new requires prog BTF\n");
13456+ return -EINVAL;
13457+ }
13458+
13459+ ret_t = btf_type_by_id(ret_btf, ret_btf_id);
13460+ if (!ret_t || !__btf_type_is_struct(ret_t)) {
13461+ verbose(env, "bpf_obj_new/bpf_percpu_obj_new type ID argument must be of a struct\n");
13462+ return -EINVAL;
13463+ }
13464+
13465+ if (meta->func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) {
13466+ if (ret_t->size > BPF_GLOBAL_PERCPU_MA_MAX_SIZE) {
13467+ verbose(env, "bpf_percpu_obj_new type size (%d) is greater than %d\n",
13468+ ret_t->size, BPF_GLOBAL_PERCPU_MA_MAX_SIZE);
13469+ return -EINVAL;
13470+ }
13471+
13472+ if (!bpf_global_percpu_ma_set) {
13473+ mutex_lock(&bpf_percpu_ma_lock);
13474+ if (!bpf_global_percpu_ma_set) {
13475+ /* Charge memory allocated with bpf_global_percpu_ma to
13476+ * root memcg. The obj_cgroup for root memcg is NULL.
13477+ */
13478+ err = bpf_mem_alloc_percpu_init(&bpf_global_percpu_ma, NULL);
13479+ if (!err)
13480+ bpf_global_percpu_ma_set = true;
13481+ }
13482+ mutex_unlock(&bpf_percpu_ma_lock);
13483+ if (err)
13484+ return err;
13485+ }
13486+
13487+ mutex_lock(&bpf_percpu_ma_lock);
13488+ err = bpf_mem_alloc_percpu_unit_init(&bpf_global_percpu_ma, ret_t->size);
13489+ mutex_unlock(&bpf_percpu_ma_lock);
13490+ if (err)
13491+ return err;
13492+ }
13493+
13494+ struct_meta = btf_find_struct_meta(ret_btf, ret_btf_id);
13495+ if (meta->func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) {
13496+ if (!__btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) {
13497+ verbose(env, "bpf_percpu_obj_new type ID argument must be of a struct of scalars\n");
13498+ return -EINVAL;
13499+ }
13500+
13501+ if (struct_meta) {
13502+ verbose(env, "bpf_percpu_obj_new type ID argument must not contain special fields\n");
13503+ return -EINVAL;
13504+ }
13505+ }
13506+
13507+ mark_reg_known_zero(env, regs, BPF_REG_0);
13508+ regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC;
13509+ regs[BPF_REG_0].btf = ret_btf;
13510+ regs[BPF_REG_0].btf_id = ret_btf_id;
13511+ if (meta->func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl])
13512+ regs[BPF_REG_0].type |= MEM_PERCPU;
13513+
13514+ insn_aux->obj_new_size = ret_t->size;
13515+ insn_aux->kptr_struct_meta = struct_meta;
13516+ } else if (meta->func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl]) {
13517+ mark_reg_known_zero(env, regs, BPF_REG_0);
13518+ regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC;
13519+ regs[BPF_REG_0].btf = meta->arg_btf;
13520+ regs[BPF_REG_0].btf_id = meta->arg_btf_id;
13521+
13522+ insn_aux->kptr_struct_meta =
13523+ btf_find_struct_meta(meta->arg_btf,
13524+ meta->arg_btf_id);
13525+ } else if (is_list_node_type(ptr_type)) {
13526+ struct btf_field *field = meta->arg_list_head.field;
13527+
13528+ mark_reg_graph_node(regs, BPF_REG_0, &field->graph_root);
13529+ } else if (is_rbtree_node_type(ptr_type)) {
13530+ struct btf_field *field = meta->arg_rbtree_root.field;
13531+
13532+ mark_reg_graph_node(regs, BPF_REG_0, &field->graph_root);
13533+ } else if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) {
13534+ mark_reg_known_zero(env, regs, BPF_REG_0);
13535+ regs[BPF_REG_0].type = PTR_TO_BTF_ID | PTR_TRUSTED;
13536+ regs[BPF_REG_0].btf = desc_btf;
13537+ regs[BPF_REG_0].btf_id = meta->ret_btf_id;
13538+ } else if (meta->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
13539+ ret_t = btf_type_by_id(desc_btf, meta->arg_constant.value);
13540+ if (!ret_t || !btf_type_is_struct(ret_t)) {
13541+ verbose(env,
13542+ "kfunc bpf_rdonly_cast type ID argument must be of a struct\n");
13543+ return -EINVAL;
13544+ }
13545+
13546+ mark_reg_known_zero(env, regs, BPF_REG_0);
13547+ regs[BPF_REG_0].type = PTR_TO_BTF_ID | PTR_UNTRUSTED;
13548+ regs[BPF_REG_0].btf = desc_btf;
13549+ regs[BPF_REG_0].btf_id = meta->arg_constant.value;
13550+ } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_slice] ||
13551+ meta->func_id == special_kfunc_list[KF_bpf_dynptr_slice_rdwr]) {
13552+ enum bpf_type_flag type_flag = get_dynptr_type_flag(meta->initialized_dynptr.type);
13553+
13554+ mark_reg_known_zero(env, regs, BPF_REG_0);
13555+
13556+ if (!meta->arg_constant.found) {
13557+ verbose(env, "verifier internal error: bpf_dynptr_slice(_rdwr) no constant size\n");
13558+ return -EFAULT;
13559+ }
13560+
13561+ regs[BPF_REG_0].mem_size = meta->arg_constant.value;
13562+
13563+ /* PTR_MAYBE_NULL will be added when is_kfunc_ret_null is checked */
13564+ regs[BPF_REG_0].type = PTR_TO_MEM | type_flag;
13565+
13566+ if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_slice]) {
13567+ regs[BPF_REG_0].type |= MEM_RDONLY;
13568+ } else {
13569+ /* this will set env->seen_direct_write to true */
13570+ if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE)) {
13571+ verbose(env, "the prog does not allow writes to packet data\n");
13572+ return -EINVAL;
13573+ }
13574+ }
13575+
13576+ if (!meta->initialized_dynptr.id) {
13577+ verbose(env, "verifier internal error: no dynptr id\n");
13578+ return -EFAULT;
13579+ }
13580+ regs[BPF_REG_0].dynptr_id = meta->initialized_dynptr.id;
13581+
13582+ /* we don't need to set BPF_REG_0's ref obj id
13583+ * because packet slices are not refcounted (see
13584+ * dynptr_type_refcounted)
13585+ */
13586+ } else {
13587+ return 0;
13588+ }
13589+
13590+ return 1;
13591+}
13592+
1342313593 static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name);
1342413594
1342513595 static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
@@ -13434,7 +13604,6 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · 8 unmodified lines
1343413604 struct bpf_insn_aux_data *insn_aux;
1343513605 int err, insn_idx = *insn_idx_p;
1343613606 const struct btf_param *args;
13437− const struct btf_type *ret_t;
1343813607 struct btf *desc_btf;
1343913608
1344013609 /* skip for now, but return error when we find this in fixup_kfunc_call */
@@ -13476,6 +13645,9 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · 35 unmodified lines
1347613645 return err;
1347713646 }
1347813647 __mark_btf_func_reg_size(env, regs, BPF_REG_0, sizeof(u32));
13648+ } else if (!insn->off && insn->imm == special_kfunc_list[KF___bpf_trap]) {
13649+ verbose(env, "unexpected __bpf_trap() due to uninitialized variable?\n");
13650+ return -EFAULT;
1347913651 }
1348013652
1348113653 if (is_kfunc_destructive(&meta) && !capable(CAP_SYS_BOOT)) {
@@ -13654,165 +13826,10 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · 172 unmodified lines
1365413826 mark_btf_func_reg_size(env, BPF_REG_0, t->size);
1365513827 } else if (btf_type_is_ptr(t)) {
1365613828 ptr_type = btf_type_skip_modifiers(desc_btf, t->type, &ptr_type_id);
13657−
13658− if (meta.btf == btf_vmlinux && btf_id_set_contains(&special_kfunc_set, meta.func_id)) {
13659− if (meta.func_id == special_kfunc_list[KF_bpf_obj_new_impl] ||
13660− meta.func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) {
13661− struct btf_struct_meta *struct_meta;
13662− struct btf *ret_btf;
13663− u32 ret_btf_id;
13664−
13665− if (meta.func_id == special_kfunc_list[KF_bpf_obj_new_impl] && !bpf_global_ma_set)
13666− return -ENOMEM;
13667−
13668− if (((u64)(u32)meta.arg_constant.value) != meta.arg_constant.value) {
13669− verbose(env, "local type ID argument must be in range [0, U32_MAX]\n");
13670− return -EINVAL;
13671− }
13672−
13673− ret_btf = env->prog->aux->btf;
13674− ret_btf_id = meta.arg_constant.value;
13675−
13676− /* This may be NULL due to user not supplying a BTF */
13677− if (!ret_btf) {
13678− verbose(env, "bpf_obj_new/bpf_percpu_obj_new requires prog BTF\n");
13679− return -EINVAL;
13680− }
13681−
13682− ret_t = btf_type_by_id(ret_btf, ret_btf_id);
13683− if (!ret_t || !__btf_type_is_struct(ret_t)) {
13684− verbose(env, "bpf_obj_new/bpf_percpu_obj_new type ID argument must be of a struct\n");
13685− return -EINVAL;
13686− }
13687−
13688− if (meta.func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) {
13689− if (ret_t->size > BPF_GLOBAL_PERCPU_MA_MAX_SIZE) {
13690− verbose(env, "bpf_percpu_obj_new type size (%d) is greater than %d\n",
13691− ret_t->size, BPF_GLOBAL_PERCPU_MA_MAX_SIZE);
13692− return -EINVAL;
13693− }
13694−
13695− if (!bpf_global_percpu_ma_set) {
13696− mutex_lock(&bpf_percpu_ma_lock);
13697− if (!bpf_global_percpu_ma_set) {
13698− /* Charge memory allocated with bpf_global_percpu_ma to
13699− * root memcg. The obj_cgroup for root memcg is NULL.
13700− */
13701− err = bpf_mem_alloc_percpu_init(&bpf_global_percpu_ma, NULL);
13702− if (!err)
13703− bpf_global_percpu_ma_set = true;
13704− }
13705− mutex_unlock(&bpf_percpu_ma_lock);
13706− if (err)
13707− return err;
13708− }
13709−
13710− mutex_lock(&bpf_percpu_ma_lock);
13711− err = bpf_mem_alloc_percpu_unit_init(&bpf_global_percpu_ma, ret_t->size);
13712− mutex_unlock(&bpf_percpu_ma_lock);
13713− if (err)
13714− return err;
13715− }
13716−
13717− struct_meta = btf_find_struct_meta(ret_btf, ret_btf_id);
13718− if (meta.func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) {
13719− if (!__btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) {
13720− verbose(env, "bpf_percpu_obj_new type ID argument must be of a struct of scalars\n");
13721− return -EINVAL;
13722− }
13723−
13724− if (struct_meta) {
13725− verbose(env, "bpf_percpu_obj_new type ID argument must not contain special fields\n");
13726− return -EINVAL;
13727− }
13728− }
13729−
13730− mark_reg_known_zero(env, regs, BPF_REG_0);
13731− regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC;
13732− regs[BPF_REG_0].btf = ret_btf;
13733− regs[BPF_REG_0].btf_id = ret_btf_id;
13734− if (meta.func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl])
13735− regs[BPF_REG_0].type |= MEM_PERCPU;
13736−
13737− insn_aux->obj_new_size = ret_t->size;
13738− insn_aux->kptr_struct_meta = struct_meta;
13739− } else if (meta.func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl]) {
13740− mark_reg_known_zero(env, regs, BPF_REG_0);
13741− regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC;
13742− regs[BPF_REG_0].btf = meta.arg_btf;
13743− regs[BPF_REG_0].btf_id = meta.arg_btf_id;
13744−
13745− insn_aux->kptr_struct_meta =
13746− btf_find_struct_meta(meta.arg_btf,
13747− meta.arg_btf_id);
13748− } else if (meta.func_id == special_kfunc_list[KF_bpf_list_pop_front] ||
13749− meta.func_id == special_kfunc_list[KF_bpf_list_pop_back]) {
13750− struct btf_field *field = meta.arg_list_head.field;
13751−
13752− mark_reg_graph_node(regs, BPF_REG_0, &field->graph_root);
13753− } else if (meta.func_id == special_kfunc_list[KF_bpf_rbtree_remove] ||
13754− meta.func_id == special_kfunc_list[KF_bpf_rbtree_first]) {
13755− struct btf_field *field = meta.arg_rbtree_root.field;
13756−
13757− mark_reg_graph_node(regs, BPF_REG_0, &field->graph_root);
13758− } else if (meta.func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) {
13759− mark_reg_known_zero(env, regs, BPF_REG_0);
13760− regs[BPF_REG_0].type = PTR_TO_BTF_ID | PTR_TRUSTED;
13761− regs[BPF_REG_0].btf = desc_btf;
13762− regs[BPF_REG_0].btf_id = meta.ret_btf_id;
13763− } else if (meta.func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
13764− ret_t = btf_type_by_id(desc_btf, meta.arg_constant.value);
13765− if (!ret_t || !btf_type_is_struct(ret_t)) {
13766− verbose(env,
13767− "kfunc bpf_rdonly_cast type ID argument must be of a struct\n");
13768− return -EINVAL;
13769− }
13770−
13771− mark_reg_known_zero(env, regs, BPF_REG_0);
13772− regs[BPF_REG_0].type = PTR_TO_BTF_ID | PTR_UNTRUSTED;
13773− regs[BPF_REG_0].btf = desc_btf;
13774− regs[BPF_REG_0].btf_id = meta.arg_constant.value;
13775− } else if (meta.func_id == special_kfunc_list[KF_bpf_dynptr_slice] ||
13776− meta.func_id == special_kfunc_list[KF_bpf_dynptr_slice_rdwr]) {
13777− enum bpf_type_flag type_flag = get_dynptr_type_flag(meta.initialized_dynptr.type);
13778−
13779− mark_reg_known_zero(env, regs, BPF_REG_0);
13780−
13781− if (!meta.arg_constant.found) {
13782− verbose(env, "verifier internal error: bpf_dynptr_slice(_rdwr) no constant size\n");
13783− return -EFAULT;
13784− }
13785−
13786− regs[BPF_REG_0].mem_size = meta.arg_constant.value;
13787−
13788− /* PTR_MAYBE_NULL will be added when is_kfunc_ret_null is checked */
13789− regs[BPF_REG_0].type = PTR_TO_MEM | type_flag;
13790−
13791− if (meta.func_id == special_kfunc_list[KF_bpf_dynptr_slice]) {
13792− regs[BPF_REG_0].type |= MEM_RDONLY;
13793− } else {
13794− /* this will set env->seen_direct_write to true */
13795− if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE)) {
13796− verbose(env, "the prog does not allow writes to packet data\n");
13797− return -EINVAL;
13798− }
13799− }
13800−
13801− if (!meta.initialized_dynptr.id) {
13802− verbose(env, "verifier internal error: no dynptr id\n");
13803− return -EFAULT;
13804− }
13805− regs[BPF_REG_0].dynptr_id = meta.initialized_dynptr.id;
13806−
13807− /* we don't need to set BPF_REG_0's ref obj id
13808− * because packet slices are not refcounted (see
13809− * dynptr_type_refcounted)
13810− */
13811− } else {
13812− verbose(env, "kernel function %s unhandled dynamic return type\n",
13813− meta.func_name);
13814− return -EFAULT;
13815− }
13829+ err = check_special_kfunc(env, &meta, regs, insn_aux, ptr_type, desc_btf);
13830+ if (err) {
13831+ if (err < 0)
13832+ return err;
1381613833 } else if (btf_type_is_void(ptr_type)) {
1381713834 /* kfunc returning 'void *' is equivalent to returning scalar */
1381813835 mark_reg_unknown(env, regs, BPF_REG_0);
@@ -13881,14 +13898,14 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · 62 unmodified lines
1388113898 if (is_kfunc_ret_null(&meta))
1388213899 regs[BPF_REG_0].id = id;
1388313900 regs[BPF_REG_0].ref_obj_id = id;
13884− } else if (meta.func_id == special_kfunc_list[KF_bpf_rbtree_first]) {
13901+ } else if (is_rbtree_node_type(ptr_type) || is_list_node_type(ptr_type)) {
1388513902 ref_set_non_owning(env, &regs[BPF_REG_0]);
1388613903 }
1388713904
1388813905 if (reg_may_point_to_spin_lock(&regs[BPF_REG_0]) && !regs[BPF_REG_0].id)
1388913906 regs[BPF_REG_0].id = ++env->id_gen;
1389013907 } else if (btf_type_is_void(t)) {
13891− if (meta.btf == btf_vmlinux && btf_id_set_contains(&special_kfunc_set, meta.func_id)) {
13908+ if (meta.btf == btf_vmlinux) {
1389213909 if (meta.func_id == special_kfunc_list[KF_bpf_obj_drop_impl] ||
1389313910 meta.func_id == special_kfunc_list[KF_bpf_percpu_obj_drop_impl]) {
1389413911 insn_aux->kptr_struct_meta =
@@ -16377,6 +16394,7 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, · 2482 unmodified lines
1637716394 struct bpf_reg_state *eq_branch_regs;
1637816395 struct linked_regs linked_regs = {};
1637916396 u8 opcode = BPF_OP(insn->code);
16397+ int insn_flags = 0;
1638016398 bool is_jmp32;
1638116399 int pred = -1;
1638216400 int err;
@@ -16435,6 +16453,11 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, · 52 unmodified lines
1643516453 insn->src_reg);
1643616454 return -EACCES;
1643716455 }
16456+
16457+ if (src_reg->type == PTR_TO_STACK)
16458+ insn_flags |= INSN_F_SRC_REG_STACK;
16459+ if (dst_reg->type == PTR_TO_STACK)
16460+ insn_flags |= INSN_F_DST_REG_STACK;
1643816461 } else {
1643916462 if (insn->src_reg != BPF_REG_0) {
1644016463 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");
@@ -16444,6 +16467,15 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, · 3 unmodified lines
1644416467 memset(src_reg, 0, sizeof(*src_reg));
1644516468 src_reg->type = SCALAR_VALUE;
1644616469 __mark_reg_known(src_reg, insn->imm);
16470+
16471+ if (dst_reg->type == PTR_TO_STACK)
16472+ insn_flags |= INSN_F_DST_REG_STACK;
16473+ }
16474+
16475+ if (insn_flags) {
16476+ err = push_insn_history(env, this_branch, insn_flags, 0);
16477+ if (err)
16478+ return err;
1644716479 }
1644816480
1644916481 is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;
@@ -19659,10 +19691,9 @@ process_bpf_exit: · 3209 unmodified lines
1965919691 return err;
1966019692 break;
1966119693 } else {
19662− if (WARN_ON_ONCE(env->cur_state->loop_entry)) {
19663− verbose(env, "verifier bug: env->cur_state->loop_entry != NULL\n");
19694+ if (verifier_bug_if(env->cur_state->loop_entry, env,
19695+ "broken loop detection"))
1966419696 return -EFAULT;
19665− }
1966619697 do_print_state = true;
1966719698 continue;
1966819699 }
@@ -20720,10 +20751,9 @@ static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, · 1051 unmodified lines
2072020751 if (bpf_pseudo_kfunc_call(&insn))
2072120752 continue;
2072220753
20723− if (WARN_ON(load_reg == -1)) {
20724− verbose(env, "verifier bug. zext_dst is set, but no reg is defined\n");
20754+ if (verifier_bug_if(load_reg == -1, env,
20755+ "zext_dst is set, but no reg is defined"))
2072520756 return -EFAULT;
20726− }
2072720757
2072820758 zext_patch[0] = insn;
2072920759 zext_patch[1].dst_reg = load_reg;
@@ -21040,11 +21070,9 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 310 unmodified lines
2104021070 * propagated in any case.
2104121071 */
2104221072 subprog = find_subprog(env, i + insn->imm + 1);
21043− if (subprog < 0) {
21044− WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
21045− i + insn->imm + 1);
21073+ if (verifier_bug_if(subprog < 0, env, "No program to jit at insn %d",
21074+ i + insn->imm + 1))
2104621075 return -EFAULT;
21047− }
2104821076 /* temporarily remember subprog id inside insn instead of
2104921077 * aux_data, since next loop will split up all insns into funcs
2105021078 */
@@ -21487,13 +21515,17 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · 436 unmodified lines
2148721515 desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
2148821516 insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1);
2148921517 *cnt = 1;
21490− } else if (is_bpf_wq_set_callback_impl_kfunc(desc->func_id)) {
21491− struct bpf_insn ld_addrs[2] = { BPF_LD_IMM64(BPF_REG_4, (long)env->prog->aux) };
21518+ }
2149221519
21493− insn_buf[0] = ld_addrs[0];
21494− insn_buf[1] = ld_addrs[1];
21495− insn_buf[2] = *insn;
21496− *cnt = 3;
21520+ if (env->insn_aux_data[insn_idx].arg_prog) {
21521+ u32 regno = env->insn_aux_data[insn_idx].arg_prog;
21522+ struct bpf_insn ld_addrs[2] = { BPF_LD_IMM64(regno, (long)env->prog->aux) };
21523+ int idx = *cnt;
21524+
21525+ insn_buf[idx++] = ld_addrs[0];
21526+ insn_buf[idx++] = ld_addrs[1];
21527+ insn_buf[idx++] = *insn;
21528+ *cnt = idx;
2149721529 }
2149821530 return 0;
2149921531 }
@@ -22403,7 +22435,7 @@ next_insn: · 903 unmodified lines
2240322435 continue;
2240422436 /* We need two slots in case timed may_goto is supported. */
2240522437 if (stack_slots > slots) {
22406− verbose(env, "verifier bug: stack_slots supports may_goto only\n");
22438+ verifier_bug(env, "stack_slots supports may_goto only");
2240722439 return -EFAULT;
2240822440 }
2240922441
kernel/sched/ext.c+1−14
@@ -5791,21 +5791,8 @@ static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log, · 5790 unmodified lines
57915791 return -EACCES;
57925792 }
57935793
5794−static const struct bpf_func_proto *
5795−bpf_scx_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
5796−{
5797− switch (func_id) {
5798− case BPF_FUNC_task_storage_get:
5799− return &bpf_task_storage_get_proto;
5800− case BPF_FUNC_task_storage_delete:
5801− return &bpf_task_storage_delete_proto;
5802− default:
5803− return bpf_base_func_proto(func_id, prog);
5804− }
5805−}
5806−
58075794 static const struct bpf_verifier_ops bpf_scx_verifier_ops = {
5808− .get_func_proto = bpf_scx_get_func_proto,
5795+ .get_func_proto = bpf_base_func_proto,
58095796 .is_valid_access = bpf_scx_is_valid_access,
58105797 .btf_struct_access = bpf_scx_btf_struct_access,
58115798 };
kernel/trace/bpf_trace.c+211−110
@@ -572,7 +572,7 @@ BPF_CALL_2(bpf_perf_event_read, struct bpf_map *, map, u64, flags) · 571 unmodified lines
572572 return value;
573573 }
574574
575−static const struct bpf_func_proto bpf_perf_event_read_proto = {
575+const struct bpf_func_proto bpf_perf_event_read_proto = {
576576 .func = bpf_perf_event_read,
577577 .gpl_only = true,
578578 .ret_type = RET_INTEGER,
@@ -882,7 +882,7 @@ BPF_CALL_1(bpf_send_signal, u32, sig) · 303 unmodified lines
882882 return bpf_send_signal_common(sig, PIDTYPE_TGID, NULL, 0);
883883 }
884884
885−static const struct bpf_func_proto bpf_send_signal_proto = {
885+const struct bpf_func_proto bpf_send_signal_proto = {
886886 .func = bpf_send_signal,
887887 .gpl_only = false,
888888 .ret_type = RET_INTEGER,
@@ -894,7 +894,7 @@ BPF_CALL_1(bpf_send_signal_thread, u32, sig) · 5 unmodified lines
894894 return bpf_send_signal_common(sig, PIDTYPE_PID, NULL, 0);
895895 }
896896
897−static const struct bpf_func_proto bpf_send_signal_thread_proto = {
897+const struct bpf_func_proto bpf_send_signal_thread_proto = {
898898 .func = bpf_send_signal_thread,
899899 .gpl_only = false,
900900 .ret_type = RET_INTEGER,
@@ -1185,7 +1185,7 @@ BPF_CALL_3(bpf_get_branch_snapshot, void *, buf, u32, size, u64, flags) · 284 unmodified lines
11851185 return entry_cnt * br_entry_size;
11861186 }
11871187
1188−static const struct bpf_func_proto bpf_get_branch_snapshot_proto = {
1188+const struct bpf_func_proto bpf_get_branch_snapshot_proto = {
11891189 .func = bpf_get_branch_snapshot,
11901190 .gpl_only = true,
11911191 .ret_type = RET_INTEGER,
@@ -1430,56 +1430,8 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 238 unmodified lines
14301430 const struct bpf_func_proto *func_proto;
14311431
14321432 switch (func_id) {
1433− case BPF_FUNC_map_lookup_elem:
1434− return &bpf_map_lookup_elem_proto;
1435− case BPF_FUNC_map_update_elem:
1436− return &bpf_map_update_elem_proto;
1437− case BPF_FUNC_map_delete_elem:
1438− return &bpf_map_delete_elem_proto;
1439− case BPF_FUNC_map_push_elem:
1440− return &bpf_map_push_elem_proto;
1441− case BPF_FUNC_map_pop_elem:
1442− return &bpf_map_pop_elem_proto;
1443− case BPF_FUNC_map_peek_elem:
1444− return &bpf_map_peek_elem_proto;
1445− case BPF_FUNC_map_lookup_percpu_elem:
1446− return &bpf_map_lookup_percpu_elem_proto;
1447− case BPF_FUNC_ktime_get_ns:
1448− return &bpf_ktime_get_ns_proto;
1449− case BPF_FUNC_ktime_get_boot_ns:
1450− return &bpf_ktime_get_boot_ns_proto;
1451− case BPF_FUNC_tail_call:
1452− return &bpf_tail_call_proto;
1453− case BPF_FUNC_get_current_task:
1454− return &bpf_get_current_task_proto;
1455− case BPF_FUNC_get_current_task_btf:
1456− return &bpf_get_current_task_btf_proto;
1457− case BPF_FUNC_task_pt_regs:
1458− return &bpf_task_pt_regs_proto;
1459− case BPF_FUNC_get_current_uid_gid:
1460− return &bpf_get_current_uid_gid_proto;
1461− case BPF_FUNC_get_current_comm:
1462− return &bpf_get_current_comm_proto;
1463− case BPF_FUNC_trace_printk:
1464− return bpf_get_trace_printk_proto();
14651433 case BPF_FUNC_get_smp_processor_id:
14661434 return &bpf_get_smp_processor_id_proto;
1467− case BPF_FUNC_get_numa_node_id:
1468− return &bpf_get_numa_node_id_proto;
1469− case BPF_FUNC_perf_event_read:
1470− return &bpf_perf_event_read_proto;
1471− case BPF_FUNC_get_prandom_u32:
1472− return &bpf_get_prandom_u32_proto;
1473− case BPF_FUNC_probe_read_user:
1474− return &bpf_probe_read_user_proto;
1475− case BPF_FUNC_probe_read_kernel:
1476− return security_locked_down(LOCKDOWN_BPF_READ_KERNEL) < 0 ?
1477− NULL : &bpf_probe_read_kernel_proto;
1478− case BPF_FUNC_probe_read_user_str:
1479− return &bpf_probe_read_user_str_proto;
1480− case BPF_FUNC_probe_read_kernel_str:
1481− return security_locked_down(LOCKDOWN_BPF_READ_KERNEL) < 0 ?
1482− NULL : &bpf_probe_read_kernel_str_proto;
14831435 #ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
14841436 case BPF_FUNC_probe_read:
14851437 return security_locked_down(LOCKDOWN_BPF_READ_KERNEL) < 0 ?
@@ -1488,65 +1440,8 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 2 unmodified lines
14881440 return security_locked_down(LOCKDOWN_BPF_READ_KERNEL) < 0 ?
14891441 NULL : &bpf_probe_read_compat_str_proto;
14901442 #endif
1491−#ifdef CONFIG_CGROUPS
1492− case BPF_FUNC_cgrp_storage_get:
1493− return &bpf_cgrp_storage_get_proto;
1494− case BPF_FUNC_cgrp_storage_delete:
1495− return &bpf_cgrp_storage_delete_proto;
1496− case BPF_FUNC_current_task_under_cgroup:
1497− return &bpf_current_task_under_cgroup_proto;
1498−#endif
1499− case BPF_FUNC_send_signal:
1500− return &bpf_send_signal_proto;
1501− case BPF_FUNC_send_signal_thread:
1502− return &bpf_send_signal_thread_proto;
1503− case BPF_FUNC_perf_event_read_value:
1504− return &bpf_perf_event_read_value_proto;
1505− case BPF_FUNC_ringbuf_output:
1506− return &bpf_ringbuf_output_proto;
1507− case BPF_FUNC_ringbuf_reserve:
1508− return &bpf_ringbuf_reserve_proto;
1509− case BPF_FUNC_ringbuf_submit:
1510− return &bpf_ringbuf_submit_proto;
1511− case BPF_FUNC_ringbuf_discard:
1512− return &bpf_ringbuf_discard_proto;
1513− case BPF_FUNC_ringbuf_query:
1514− return &bpf_ringbuf_query_proto;
1515− case BPF_FUNC_jiffies64:
1516− return &bpf_jiffies64_proto;
1517− case BPF_FUNC_get_task_stack:
1518− return prog->sleepable ? &bpf_get_task_stack_sleepable_proto
1519− : &bpf_get_task_stack_proto;
1520− case BPF_FUNC_copy_from_user:
1521− return &bpf_copy_from_user_proto;
1522− case BPF_FUNC_copy_from_user_task:
1523− return &bpf_copy_from_user_task_proto;
1524− case BPF_FUNC_snprintf_btf:
1525− return &bpf_snprintf_btf_proto;
1526− case BPF_FUNC_per_cpu_ptr:
1527− return &bpf_per_cpu_ptr_proto;
1528− case BPF_FUNC_this_cpu_ptr:
1529− return &bpf_this_cpu_ptr_proto;
1530− case BPF_FUNC_task_storage_get:
1531− if (bpf_prog_check_recur(prog))
1532− return &bpf_task_storage_get_recur_proto;
1533− return &bpf_task_storage_get_proto;
1534− case BPF_FUNC_task_storage_delete:
1535− if (bpf_prog_check_recur(prog))
1536− return &bpf_task_storage_delete_recur_proto;
1537− return &bpf_task_storage_delete_proto;
1538− case BPF_FUNC_for_each_map_elem:
1539− return &bpf_for_each_map_elem_proto;
1540− case BPF_FUNC_snprintf:
1541− return &bpf_snprintf_proto;
15421443 case BPF_FUNC_get_func_ip:
15431444 return &bpf_get_func_ip_proto_tracing;
1544− case BPF_FUNC_get_branch_snapshot:
1545− return &bpf_get_branch_snapshot_proto;
1546− case BPF_FUNC_find_vma:
1547− return &bpf_find_vma_proto;
1548− case BPF_FUNC_trace_vprintk:
1549− return bpf_get_trace_vprintk_proto();
15501445 default:
15511446 break;
15521447 }
@@ -1858,7 +1753,7 @@ static struct pt_regs *get_bpf_raw_tp_regs(void) · 305 unmodified lines
18581753 struct bpf_raw_tp_regs *tp_regs = this_cpu_ptr(&bpf_raw_tp_regs);
18591754 int nest_level = this_cpu_inc_return(bpf_raw_tp_nest_level);
18601755
1861− if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(tp_regs->regs))) {
1756+ if (nest_level > ARRAY_SIZE(tp_regs->regs)) {
18621757 this_cpu_dec(bpf_raw_tp_nest_level);
18631758 return ERR_PTR(-EBUSY);
18641759 }
@@ -2987,6 +2882,9 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr · 1122 unmodified lines
29872882 if (sizeof(u64) != sizeof(void *))
29882883 return -EOPNOTSUPP;
29892884
2885+ if (attr->link_create.flags)
2886+ return -EINVAL;
2887+
29902888 if (!is_kprobe_multi(prog))
29912889 return -EINVAL;
29922890
@@ -3376,6 +3274,9 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr · 383 unmodified lines
33763274 if (sizeof(u64) != sizeof(void *))
33773275 return -EOPNOTSUPP;
33783276
3277+ if (attr->link_create.flags)
3278+ return -EINVAL;
3279+
33793280 if (!is_uprobe_multi(prog))
33803281 return -EINVAL;
33813282
@@ -3417,7 +3318,9 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr · 35 unmodified lines
34173318 }
34183319
34193320 if (pid) {
3321+ rcu_read_lock();
34203322 task = get_pid_task(find_vpid(pid), PIDTYPE_TGID);
3323+ rcu_read_unlock();
34213324 if (!task) {
34223325 err = -ESRCH;
34233326 goto error_path_put;
@@ -3565,6 +3468,146 @@ static int __init bpf_kprobe_multi_kfuncs_init(void) · 141 unmodified lines
35653468
35663469 late_initcall(bpf_kprobe_multi_kfuncs_init);
35673470
3471+typedef int (*copy_fn_t)(void *dst, const void *src, u32 size, struct task_struct *tsk);
3472+
3473+/*
3474+ * The __always_inline is to make sure the compiler doesn't
3475+ * generate indirect calls into callbacks, which is expensive,
3476+ * on some kernel configurations. This allows compiler to put
3477+ * direct calls into all the specific callback implementations
3478+ * (copy_user_data_sleepable, copy_user_data_nofault, and so on)
3479+ */
3480+static __always_inline int __bpf_dynptr_copy_str(struct bpf_dynptr *dptr, u32 doff, u32 size,
3481+ const void *unsafe_src,
3482+ copy_fn_t str_copy_fn,
3483+ struct task_struct *tsk)
3484+{
3485+ struct bpf_dynptr_kern *dst;
3486+ u32 chunk_sz, off;
3487+ void *dst_slice;
3488+ int cnt, err;
3489+ char buf[256];
3490+
3491+ dst_slice = bpf_dynptr_slice_rdwr(dptr, doff, NULL, size);
3492+ if (likely(dst_slice))
3493+ return str_copy_fn(dst_slice, unsafe_src, size, tsk);
3494+
3495+ dst = (struct bpf_dynptr_kern *)dptr;
3496+ if (bpf_dynptr_check_off_len(dst, doff, size))
3497+ return -E2BIG;
3498+
3499+ for (off = 0; off < size; off += chunk_sz - 1) {
3500+ chunk_sz = min_t(u32, sizeof(buf), size - off);
3501+ /* Expect str_copy_fn to return count of copied bytes, including
3502+ * zero terminator. Next iteration increment off by chunk_sz - 1 to
3503+ * overwrite NUL.
3504+ */
3505+ cnt = str_copy_fn(buf, unsafe_src + off, chunk_sz, tsk);
3506+ if (cnt < 0)
3507+ return cnt;
3508+ err = __bpf_dynptr_write(dst, doff + off, buf, cnt, 0);
3509+ if (err)
3510+ return err;
3511+ if (cnt < chunk_sz || chunk_sz == 1) /* we are done */
3512+ return off + cnt;
3513+ }
3514+ return off;
3515+}
3516+
3517+static __always_inline int __bpf_dynptr_copy(const struct bpf_dynptr *dptr, u32 doff,
3518+ u32 size, const void *unsafe_src,
3519+ copy_fn_t copy_fn, struct task_struct *tsk)
3520+{
3521+ struct bpf_dynptr_kern *dst;
3522+ void *dst_slice;
3523+ char buf[256];
3524+ u32 off, chunk_sz;
3525+ int err;
3526+
3527+ dst_slice = bpf_dynptr_slice_rdwr(dptr, doff, NULL, size);
3528+ if (likely(dst_slice))
3529+ return copy_fn(dst_slice, unsafe_src, size, tsk);
3530+
3531+ dst = (struct bpf_dynptr_kern *)dptr;
3532+ if (bpf_dynptr_check_off_len(dst, doff, size))
3533+ return -E2BIG;
3534+
3535+ for (off = 0; off < size; off += chunk_sz) {
3536+ chunk_sz = min_t(u32, sizeof(buf), size - off);
3537+ err = copy_fn(buf, unsafe_src + off, chunk_sz, tsk);
3538+ if (err)
3539+ return err;
3540+ err = __bpf_dynptr_write(dst, doff + off, buf, chunk_sz, 0);
3541+ if (err)
3542+ return err;
3543+ }
3544+ return 0;
3545+}
3546+
3547+static __always_inline int copy_user_data_nofault(void *dst, const void *unsafe_src,
3548+ u32 size, struct task_struct *tsk)
3549+{
3550+ return copy_from_user_nofault(dst, (const void __user *)unsafe_src, size);
3551+}
3552+
3553+static __always_inline int copy_user_data_sleepable(void *dst, const void *unsafe_src,
3554+ u32 size, struct task_struct *tsk)
3555+{
3556+ int ret;
3557+
3558+ if (!tsk) { /* Read from the current task */
3559+ ret = copy_from_user(dst, (const void __user *)unsafe_src, size);
3560+ if (ret)
3561+ return -EFAULT;
3562+ return 0;
3563+ }
3564+
3565+ ret = access_process_vm(tsk, (unsigned long)unsafe_src, dst, size, 0);
3566+ if (ret != size)
3567+ return -EFAULT;
3568+ return 0;
3569+}
3570+
3571+static __always_inline int copy_kernel_data_nofault(void *dst, const void *unsafe_src,
3572+ u32 size, struct task_struct *tsk)
3573+{
3574+ return copy_from_kernel_nofault(dst, unsafe_src, size);
3575+}
3576+
3577+static __always_inline int copy_user_str_nofault(void *dst, const void *unsafe_src,
3578+ u32 size, struct task_struct *tsk)
3579+{
3580+ return strncpy_from_user_nofault(dst, (const void __user *)unsafe_src, size);
3581+}
3582+
3583+static __always_inline int copy_user_str_sleepable(void *dst, const void *unsafe_src,
3584+ u32 size, struct task_struct *tsk)
3585+{
3586+ int ret;
3587+
3588+ if (unlikely(size == 0))
3589+ return 0;
3590+
3591+ if (tsk) {
3592+ ret = copy_remote_vm_str(tsk, (unsigned long)unsafe_src, dst, size, 0);
3593+ } else {
3594+ ret = strncpy_from_user(dst, (const void __user *)unsafe_src, size - 1);
3595+ /* strncpy_from_user does not guarantee NUL termination */
3596+ if (ret >= 0)
3597+ ((char *)dst)[ret] = '\0';
3598+ }
3599+
3600+ if (ret < 0)
3601+ return ret;
3602+ return ret + 1;
3603+}
3604+
3605+static __always_inline int copy_kernel_str_nofault(void *dst, const void *unsafe_src,
3606+ u32 size, struct task_struct *tsk)
3607+{
3608+ return strncpy_from_kernel_nofault(dst, unsafe_src, size);
3609+}
3610+
35683611 __bpf_kfunc_start_defs();
35693612
35703613 __bpf_kfunc int bpf_send_signal_task(struct task_struct *task, int sig, enum pid_type type,
@@ -3576,4 +3619,62 @@ __bpf_kfunc int bpf_send_signal_task(struct task_struct *task, int sig, enum pid · 5 unmodified lines
35763619 return bpf_send_signal_common(sig, type, task, value);
35773620 }
35783621
3622+__bpf_kfunc int bpf_probe_read_user_dynptr(struct bpf_dynptr *dptr, u32 off,
3623+ u32 size, const void __user *unsafe_ptr__ign)
3624+{
3625+ return __bpf_dynptr_copy(dptr, off, size, (const void *)unsafe_ptr__ign,
3626+ copy_user_data_nofault, NULL);
3627+}
3628+
3629+__bpf_kfunc int bpf_probe_read_kernel_dynptr(struct bpf_dynptr *dptr, u32 off,
3630+ u32 size, const void *unsafe_ptr__ign)
3631+{
3632+ return __bpf_dynptr_copy(dptr, off, size, unsafe_ptr__ign,
3633+ copy_kernel_data_nofault, NULL);
3634+}
3635+
3636+__bpf_kfunc int bpf_probe_read_user_str_dynptr(struct bpf_dynptr *dptr, u32 off,
3637+ u32 size, const void __user *unsafe_ptr__ign)
3638+{
3639+ return __bpf_dynptr_copy_str(dptr, off, size, (const void *)unsafe_ptr__ign,
3640+ copy_user_str_nofault, NULL);
3641+}
3642+
3643+__bpf_kfunc int bpf_probe_read_kernel_str_dynptr(struct bpf_dynptr *dptr, u32 off,
3644+ u32 size, const void *unsafe_ptr__ign)
3645+{
3646+ return __bpf_dynptr_copy_str(dptr, off, size, unsafe_ptr__ign,
3647+ copy_kernel_str_nofault, NULL);
3648+}
3649+
3650+__bpf_kfunc int bpf_copy_from_user_dynptr(struct bpf_dynptr *dptr, u32 off,
3651+ u32 size, const void __user *unsafe_ptr__ign)
3652+{
3653+ return __bpf_dynptr_copy(dptr, off, size, (const void *)unsafe_ptr__ign,
3654+ copy_user_data_sleepable, NULL);
3655+}
3656+
3657+__bpf_kfunc int bpf_copy_from_user_str_dynptr(struct bpf_dynptr *dptr, u32 off,
3658+ u32 size, const void __user *unsafe_ptr__ign)
3659+{
3660+ return __bpf_dynptr_copy_str(dptr, off, size, (const void *)unsafe_ptr__ign,
3661+ copy_user_str_sleepable, NULL);
3662+}
3663+
3664+__bpf_kfunc int bpf_copy_from_user_task_dynptr(struct bpf_dynptr *dptr, u32 off,
3665+ u32 size, const void __user *unsafe_ptr__ign,
3666+ struct task_struct *tsk)
3667+{
3668+ return __bpf_dynptr_copy(dptr, off, size, (const void *)unsafe_ptr__ign,
3669+ copy_user_data_sleepable, tsk);
3670+}
3671+
3672+__bpf_kfunc int bpf_copy_from_user_task_str_dynptr(struct bpf_dynptr *dptr, u32 off,
3673+ u32 size, const void __user *unsafe_ptr__ign,
3674+ struct task_struct *tsk)
3675+{
3676+ return __bpf_dynptr_copy_str(dptr, off, size, (const void *)unsafe_ptr__ign,
3677+ copy_user_str_sleepable, tsk);
3678+}
3679+
35793680 __bpf_kfunc_end_defs();
kernel/trace/trace_uprobe.c+1−1
@@ -1489,7 +1489,7 @@ int bpf_get_uprobe_info(const struct perf_event *event, u32 *fd_type, · 1488 unmodified lines
14891489 : BPF_FD_TYPE_UPROBE;
14901490 *filename = tu->filename;
14911491 *probe_offset = tu->offset;
1492− *probe_addr = 0;
1492+ *probe_addr = tu->ref_ctr_offset;
14931493 return 0;
14941494 }
14951495 #endif /* CONFIG_PERF_EVENTS */
net/bpf/test_run.c+7−1
@@ -569,6 +569,11 @@ __bpf_kfunc u32 bpf_fentry_test9(u32 *a) · 568 unmodified lines
569569 return *a;
570570 }
571571
572+int noinline bpf_fentry_test10(const void *a)
573+{
574+ return (long)a;
575+}
576+
572577 void noinline bpf_fentry_test_sinfo(struct skb_shared_info *sinfo)
573578 {
574579 }
@@ -699,7 +704,8 @@ int bpf_prog_test_run_tracing(struct bpf_prog *prog, · 124 unmodified lines
699704 bpf_fentry_test6(16, (void *)17, 18, 19, (void *)20, 21) != 111 ||
700705 bpf_fentry_test7((struct bpf_fentry_test_t *)0) != 0 ||
701706 bpf_fentry_test8(&arg) != 0 ||
702− bpf_fentry_test9(&retval) != 0)
707+ bpf_fentry_test9(&retval) != 0 ||
708+ bpf_fentry_test10((void *)0) != 0)
703709 goto out;
704710 break;
705711 case BPF_MODIFY_RETURN:
net/core/filter.c+0−14
@@ -8023,10 +8023,6 @@ sock_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 8022 unmodified lines
80238023 if (func_proto)
80248024 return func_proto;
80258025
8026− func_proto = cgroup_current_func_proto(func_id, prog);
8027− if (func_proto)
8028− return func_proto;
8029−
80308026 switch (func_id) {
80318027 case BPF_FUNC_get_socket_cookie:
80328028 return &bpf_get_socket_cookie_sock_proto;
@@ -8052,10 +8048,6 @@ sock_addr_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 19 unmodified lines
80528048 if (func_proto)
80538049 return func_proto;
80548050
8055− func_proto = cgroup_current_func_proto(func_id, prog);
8056− if (func_proto)
8057− return func_proto;
8058−
80598051 switch (func_id) {
80608052 case BPF_FUNC_bind:
80618053 switch (prog->expected_attach_type) {
@@ -8489,18 +8481,12 @@ sk_msg_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 427 unmodified lines
84898481 return &bpf_msg_pop_data_proto;
84908482 case BPF_FUNC_perf_event_output:
84918483 return &bpf_event_output_data_proto;
8492− case BPF_FUNC_get_current_uid_gid:
8493− return &bpf_get_current_uid_gid_proto;
84948484 case BPF_FUNC_sk_storage_get:
84958485 return &bpf_sk_storage_get_proto;
84968486 case BPF_FUNC_sk_storage_delete:
84978487 return &bpf_sk_storage_delete_proto;
84988488 case BPF_FUNC_get_netns_cookie:
84998489 return &bpf_get_netns_cookie_sk_msg_proto;
8500−#ifdef CONFIG_CGROUP_NET_CLASSID
8501− case BPF_FUNC_get_cgroup_classid:
8502− return &bpf_get_cgroup_classid_curr_proto;
8503−#endif
85048490 default:
85058491 return bpf_sk_base_func_proto(func_id, prog);
85068492 }
net/core/skmsg.c+35−21
@@ -530,16 +530,22 @@ static int sk_psock_skb_ingress_enqueue(struct sk_buff *skb, · 529 unmodified lines
530530 u32 off, u32 len,
531531 struct sk_psock *psock,
532532 struct sock *sk,
533− struct sk_msg *msg)
533+ struct sk_msg *msg,
534+ bool take_ref)
534535 {
535536 int num_sge, copied;
536537
538+ /* skb_to_sgvec will fail when the total number of fragments in
539+ * frag_list and frags exceeds MAX_MSG_FRAGS. For example, the
540+ * caller may aggregate multiple skbs.
541+ */
537542 num_sge = skb_to_sgvec(skb, msg->sg.data, off, len);
538543 if (num_sge < 0) {
539544 /* skb linearize may fail with ENOMEM, but lets simply try again
540545 * later if this happens. Under memory pressure we don't want to
541546 * drop the skb. We need to linearize the skb so that the mapping
542547 * in skb_to_sgvec can not error.
548+ * Note that skb_linearize requires the skb not to be shared.
543549 */
544550 if (skb_linearize(skb))
545551 return -EAGAIN;
@@ -556,7 +562,7 @@ static int sk_psock_skb_ingress_enqueue(struct sk_buff *skb, · 10 unmodified lines
556562 msg->sg.start = 0;
557563 msg->sg.size = copied;
558564 msg->sg.end = num_sge;
559− msg->skb = skb;
565+ msg->skb = take_ref ? skb_get(skb) : skb;
560566
561567 sk_psock_queue_msg(psock, msg);
562568 sk_psock_data_ready(sk, psock);
@@ -564,7 +570,7 @@ static int sk_psock_skb_ingress_enqueue(struct sk_buff *skb, · 1 unmodified line
564570 }
565571
566572 static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb,
567− u32 off, u32 len);
573+ u32 off, u32 len, bool take_ref);
568574
569575 static int sk_psock_skb_ingress(struct sk_psock *psock, struct sk_buff *skb,
570576 u32 off, u32 len)
@@ -578,7 +584,7 @@ static int sk_psock_skb_ingress(struct sk_psock *psock, struct sk_buff *skb, · 7 unmodified lines
578584 * correctly.
579585 */
580586 if (unlikely(skb->sk == sk))
581− return sk_psock_skb_ingress_self(psock, skb, off, len);
587+ return sk_psock_skb_ingress_self(psock, skb, off, len, true);
582588 msg = sk_psock_create_ingress_msg(sk, skb);
583589 if (!msg)
584590 return -EAGAIN;
@@ -590,7 +596,7 @@ static int sk_psock_skb_ingress(struct sk_psock *psock, struct sk_buff *skb, · 5 unmodified lines
590596 * into user buffers.
591597 */
592598 skb_set_owner_r(skb, sk);
593− err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg);
599+ err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg, true);
594600 if (err < 0)
595601 kfree(msg);
596602 return err;
@@ -601,7 +607,7 @@ static int sk_psock_skb_ingress(struct sk_psock *psock, struct sk_buff *skb, · 4 unmodified lines
601607 * because the skb is already accounted for here.
602608 */
603609 static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb,
604− u32 off, u32 len)
610+ u32 off, u32 len, bool take_ref)
605611 {
606612 struct sk_msg *msg = alloc_sk_msg(GFP_ATOMIC);
607613 struct sock *sk = psock->sk;
@@ -610,7 +616,7 @@ static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb · 2 unmodified lines
610616 if (unlikely(!msg))
611617 return -EAGAIN;
612618 skb_set_owner_r(skb, sk);
613− err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg);
619+ err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg, take_ref);
614620 if (err < 0)
615621 kfree(msg);
616622 return err;
@@ -619,18 +625,13 @@ static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb · 2 unmodified lines
619625 static int sk_psock_handle_skb(struct sk_psock *psock, struct sk_buff *skb,
620626 u32 off, u32 len, bool ingress)
621627 {
622− int err = 0;
623−
624628 if (!ingress) {
625629 if (!sock_writeable(psock->sk))
626630 return -EAGAIN;
627631 return skb_send_sock(psock->sk, skb, off, len);
628632 }
629− skb_get(skb);
630− err = sk_psock_skb_ingress(psock, skb, off, len);
631− if (err < 0)
632− kfree_skb(skb);
633− return err;
633+
634+ return sk_psock_skb_ingress(psock, skb, off, len);
634635 }
635636
636637 static void sk_psock_skb_state(struct sk_psock *psock,
@@ -655,12 +656,14 @@ static void sk_psock_backlog(struct work_struct *work) · 18 unmodified lines
655656 bool ingress;
656657 int ret;
657658
659+ /* Increment the psock refcnt to synchronize with close(fd) path in
660+ * sock_map_close(), ensuring we wait for backlog thread completion
661+ * before sk_socket freed. If refcnt increment fails, it indicates
662+ * sock_map_close() completed with sk_socket potentially already freed.
663+ */
664+ if (!sk_psock_get(psock->sk))
665+ return;
658666 mutex_lock(&psock->work_mutex);
659− if (unlikely(state->len)) {
660− len = state->len;
661− off = state->off;
662− }
663−
664667 while ((skb = skb_peek(&psock->ingress_skb))) {
665668 len = skb->len;
666669 off = 0;
@@ -670,6 +673,13 @@ static void sk_psock_backlog(struct work_struct *work) · 3 unmodified lines
670673 off = stm->offset;
671674 len = stm->full_len;
672675 }
676+
677+ /* Resume processing from previous partial state */
678+ if (unlikely(state->len)) {
679+ len = state->len;
680+ off = state->off;
681+ }
682+
673683 ingress = skb_bpf_ingress(skb);
674684 skb_bpf_redirect_clear(skb);
675685 do {
@@ -680,7 +690,8 @@ static void sk_psock_backlog(struct work_struct *work) · 4 unmodified lines
680690 if (ret <= 0) {
681691 if (ret == -EAGAIN) {
682692 sk_psock_skb_state(psock, state, len, off);
683−
693+ /* Restore redir info we cleared before */
694+ skb_bpf_set_redir(skb, psock->sk, ingress);
684695 /* Delay slightly to prioritize any
685696 * other work that might be here.
686697 */
@@ -697,11 +708,14 @@ static void sk_psock_backlog(struct work_struct *work) · 10 unmodified lines
697708 len -= ret;
698709 } while (len);
699710
711+ /* The entire skb sent, clear state */
712+ sk_psock_skb_state(psock, state, 0, 0);
700713 skb = skb_dequeue(&psock->ingress_skb);
701714 kfree_skb(skb);
702715 }
703716 end:
704717 mutex_unlock(&psock->work_mutex);
718+ sk_psock_put(psock->sk, psock);
705719 }
706720
707721 struct sk_psock *sk_psock_init(struct sock *sk, int node)
@@ -1014,7 +1028,7 @@ static int sk_psock_verdict_apply(struct sk_psock *psock, struct sk_buff *skb, · 306 unmodified lines
10141028 off = stm->offset;
10151029 len = stm->full_len;
10161030 }
1017− err = sk_psock_skb_ingress_self(psock, skb, off, len);
1031+ err = sk_psock_skb_ingress_self(psock, skb, off, len, false);
10181032 }
10191033 if (err < 0) {
10201034 spin_lock_bh(&psock->ingress_lock);
net/tls/tls_sw.c+13−2
@@ -908,6 +908,13 @@ more_data: · 907 unmodified lines
908908 &msg_redir, send, flags);
909909 lock_sock(sk);
910910 if (err < 0) {
911+ /* Regardless of whether the data represented by
912+ * msg_redir is sent successfully, we have already
913+ * uncharged it via sk_msg_return_zero(). The
914+ * msg->sg.size represents the remaining unprocessed
915+ * data, which needs to be uncharged here.
916+ */
917+ sk_mem_uncharge(sk, msg->sg.size);
911918 *copied -= sk_msg_free_nocharge(sk, &msg_redir);
912919 msg->sg.size = 0;
913920 }
@@ -1120,9 +1127,13 @@ alloc_encrypted: · 206 unmodified lines
11201127 num_async++;
11211128 else if (ret == -ENOMEM)
11221129 goto wait_for_memory;
1123− else if (ctx->open_rec && ret == -ENOSPC)
1130+ else if (ctx->open_rec && ret == -ENOSPC) {
1131+ if (msg_pl->cork_bytes) {
1132+ ret = 0;
1133+ goto send_end;
1134+ }
11241135 goto rollback_iter;
1125− else if (ret != -EAGAIN)
1136+ } else if (ret != -EAGAIN)
11261137 goto send_end;
11271138 }
11281139 continue;
scripts/Makefile.btf+2−0
@@ -23,6 +23,8 @@ else · 22 unmodified lines
2323 # Switch to using --btf_features for v1.26 and later.
2424 pahole-flags-$(call test-ge, $(pahole-ver), 126) = -j$(JOBS) --btf_features=encode_force,var,float,enum64,decl_tag,type_tag,optimized_func,consistent_func,decl_tag_kfuncs
2525
26+pahole-flags-$(call test-ge, $(pahole-ver), 130) += --btf_features=attributes
27+
2628 ifneq ($(KBUILD_EXTMOD),)
2729 module-pahole-flags-$(call test-ge, $(pahole-ver), 128) += --btf_features=distilled_base
2830 endif
scripts/bpf_doc.py+99−20
@@ -8,6 +8,7 @@ · 7 unmodified lines
88 from __future__ import print_function
99
1010 import argparse
11+import json
1112 import re
1213 import sys, os
1314 import subprocess
@@ -37,11 +38,17 @@ class APIElement(object): · 23 unmodified lines
3738 @desc: textual description of the symbol
3839 @ret: (optional) description of any associated return value
3940 """
40− def __init__(self, proto='', desc='', ret='', attrs=[]):
41+ def __init__(self, proto='', desc='', ret=''):
4142 self.proto = proto
4243 self.desc = desc
4344 self.ret = ret
44− self.attrs = attrs
45+
46+ def to_dict(self):
47+ return {
48+ 'proto': self.proto,
49+ 'desc': self.desc,
50+ 'ret': self.ret
51+ }
4552
4653
4754 class Helper(APIElement):
@@ -51,8 +58,9 @@ class Helper(APIElement): · 3 unmodified lines
5158 @desc: textual description of the helper function
5259 @ret: description of the return value of the helper function
5360 """
54− def __init__(self, *args, **kwargs):
55− super().__init__(*args, **kwargs)
61+ def __init__(self, proto='', desc='', ret='', attrs=[]):
62+ super().__init__(proto, desc, ret)
63+ self.attrs = attrs
5664 self.enum_val = None
5765
5866 def proto_break_down(self):
@@ -81,6 +89,12 @@ class Helper(APIElement): · 22 unmodified lines
8189
8290 return res
8391
92+ def to_dict(self):
93+ d = super().to_dict()
94+ d["attrs"] = self.attrs
95+ d.update(self.proto_break_down())
96+ return d
97+
8498
8599 ATTRS = {
86100 '__bpf_fastcall': 'bpf_fastcall'
@@ -675,7 +689,7 @@ COMMANDS · 588 unmodified lines
675689 self.print_elem(command)
676690
677691
678−class PrinterHelpers(Printer):
692+class PrinterHelpersHeader(Printer):
679693 """
680694 A printer for dumping collected information about helpers as C header to
681695 be included from BPF program.
@@ -896,6 +910,43 @@ class PrinterHelpers(Printer): · 214 unmodified lines
896910 print(') = (void *) %d;' % helper.enum_val)
897911 print('')
898912
913+
914+class PrinterHelpersJSON(Printer):
915+ """
916+ A printer for dumping collected information about helpers as a JSON file.
917+ @parser: A HeaderParser with Helper objects
918+ """
919+
920+ def __init__(self, parser):
921+ self.elements = parser.helpers
922+ self.elem_number_check(
923+ parser.desc_unique_helpers,
924+ parser.define_unique_helpers,
925+ "helper",
926+ "___BPF_FUNC_MAPPER",
927+ )
928+
929+ def print_all(self):
930+ helper_dicts = [helper.to_dict() for helper in self.elements]
931+ out_dict = {'helpers': helper_dicts}
932+ print(json.dumps(out_dict, indent=4))
933+
934+
935+class PrinterSyscallJSON(Printer):
936+ """
937+ A printer for dumping collected syscall information as a JSON file.
938+ @parser: A HeaderParser with APIElement objects
939+ """
940+
941+ def __init__(self, parser):
942+ self.elements = parser.commands
943+ self.elem_number_check(parser.desc_syscalls, parser.enum_syscalls, 'syscall', 'bpf_cmd')
944+
945+ def print_all(self):
946+ syscall_dicts = [syscall.to_dict() for syscall in self.elements]
947+ out_dict = {'syscall': syscall_dicts}
948+ print(json.dumps(out_dict, indent=4))
949+
899950 ###############################################################################
900951
901952 # If script is launched from scripts/ from kernel tree and can access
@@ -905,9 +956,17 @@ script = os.path.abspath(sys.argv[0]) · 3 unmodified lines
905956 linuxRoot = os.path.dirname(os.path.dirname(script))
906957 bpfh = os.path.join(linuxRoot, 'include/uapi/linux/bpf.h')
907958
959+# target -> output format -> printer
908960 printers = {
909− 'helpers': PrinterHelpersRST,
910− 'syscall': PrinterSyscallRST,
961+ 'helpers': {
962+ 'rst': PrinterHelpersRST,
963+ 'json': PrinterHelpersJSON,
964+ 'header': PrinterHelpersHeader,
965+ },
966+ 'syscall': {
967+ 'rst': PrinterSyscallRST,
968+ 'json': PrinterSyscallJSON
969+ },
911970 }
912971
913972 argParser = argparse.ArgumentParser(description="""
@@ -917,6 +976,8 @@ rst2man utility. · 3 unmodified lines
917976 """)
918977 argParser.add_argument('--header', action='store_true',
919978 help='generate C header file')
979+argParser.add_argument('--json', action='store_true',
980+ help='generate a JSON')
920981 if (os.path.isfile(bpfh)):
921982 argParser.add_argument('--filename', help='path to include/uapi/linux/bpf.h',
922983 default=bpfh)
@@ -924,17 +985,35 @@ else: · 1 unmodified line
924985 argParser.add_argument('--filename', help='path to include/uapi/linux/bpf.h')
925986 argParser.add_argument('target', nargs='?', default='helpers',
926987 choices=printers.keys(), help='eBPF API target')
927−args = argParser.parse_args()
928−
929−# Parse file.
930−headerParser = HeaderParser(args.filename)
931−headerParser.run()
932988
933−# Print formatted output to standard output.
934−if args.header:
935− if args.target != 'helpers':
936− raise NotImplementedError('Only helpers header generation is supported')
937− printer = PrinterHelpers(headerParser)
938−else:
939− printer = printers[args.target](headerParser)
940−printer.print_all()
989+def error_die(message: str):
990+ argParser.print_usage(file=sys.stderr)
991+ print('Error: {}'.format(message), file=sys.stderr)
992+ exit(1)
993+
994+def parse_and_dump():
995+ args = argParser.parse_args()
996+
997+ # Parse file.
998+ headerParser = HeaderParser(args.filename)
999+ headerParser.run()
1000+
1001+ if args.header and args.json:
1002+ error_die('Use either --header or --json, not both')
1003+
1004+ output_format = 'rst'
1005+ if args.header:
1006+ output_format = 'header'
1007+ elif args.json:
1008+ output_format = 'json'
1009+
1010+ try:
1011+ printer = printers[args.target][output_format](headerParser)
1012+ # Print formatted output to standard output.
1013+ printer.print_all()
1014+ except KeyError:
1015+ error_die('Unsupported target/format combination: "{}", "{}"'
1016+ .format(args.target, output_format))
1017+
1018+if __name__ == "__main__":
1019+ parse_and_dump()
tools/bpf/bpftool/Documentation/bpftool-prog.rst+8−2
@@ -31,7 +31,7 @@ PROG COMMANDS · 30 unmodified lines
3131 | **bpftool** **prog dump xlated** *PROG* [{ **file** *FILE* | [**opcodes**] [**linum**] [**visual**] }]
3232 | **bpftool** **prog dump jited** *PROG* [{ **file** *FILE* | [**opcodes**] [**linum**] }]
3333 | **bpftool** **prog pin** *PROG* *FILE*
34−| **bpftool** **prog** { **load** | **loadall** } *OBJ* *PATH* [**type** *TYPE*] [**map** { **idx** *IDX* | **name** *NAME* } *MAP*] [{ **offload_dev** | **xdpmeta_dev** } *NAME*] [**pinmaps** *MAP_DIR*] [**autoattach**]
34+| **bpftool** **prog** { **load** | **loadall** } *OBJ* *PATH* [**type** *TYPE*] [**map** { **idx** *IDX* | **name** *NAME* } *MAP*] [{ **offload_dev** | **xdpmeta_dev** } *NAME*] [**pinmaps** *MAP_DIR*] [**autoattach**] [**kernel_btf** *BTF_FILE*]
3535 | **bpftool** **prog attach** *PROG* *ATTACH_TYPE* [*MAP*]
3636 | **bpftool** **prog detach** *PROG* *ATTACH_TYPE* [*MAP*]
3737 | **bpftool** **prog tracelog**
@@ -127,7 +127,7 @@ bpftool prog pin *PROG* *FILE* · 89 unmodified lines
127127 Note: *FILE* must be located in *bpffs* mount. It must not contain a dot
128128 character ('.'), which is reserved for future extensions of *bpffs*.
129129
130−bpftool prog { load | loadall } *OBJ* *PATH* [type *TYPE*] [map { idx *IDX* | name *NAME* } *MAP*] [{ offload_dev | xdpmeta_dev } *NAME*] [pinmaps *MAP_DIR*] [autoattach]
130+bpftool prog { load | loadall } *OBJ* *PATH* [type *TYPE*] [map { idx *IDX* | name *NAME* } *MAP*] [{ offload_dev | xdpmeta_dev } *NAME*] [pinmaps *MAP_DIR*] [autoattach] [kernel_btf *BTF_FILE*]
131131 Load bpf program(s) from binary *OBJ* and pin as *PATH*. **bpftool prog
132132 load** pins only the first program from the *OBJ* as *PATH*. **bpftool prog
133133 loadall** pins all programs from the *OBJ* under *PATH* directory. **type**
@@ -153,6 +153,12 @@ bpftool prog { load | loadall } *OBJ* *PATH* [type *TYPE*] [map { idx *IDX* | na · 19 unmodified lines
153153 program does not support autoattach, bpftool falls back to regular pinning
154154 for that program instead.
155155
156+ The **kernel_btf** option allows specifying an external BTF file to replace
157+ the system's own vmlinux BTF file for CO-RE relocations. Note that any
158+ other feature relying on BTF (such as fentry/fexit programs, struct_ops)
159+ requires the BTF file for the actual kernel running on the host, often
160+ exposed at /sys/kernel/btf/vmlinux.
161+
156162 Note: *PATH* must be located in *bpffs* mount. It must not contain a dot
157163 character ('.'), which is reserved for future extensions of *bpffs*.
158164
tools/bpf/bpftool/bash-completion/bpftool+2−2
@@ -505,13 +505,13 @@ _bpftool() · 504 unmodified lines
505505 _bpftool_get_map_names
506506 return 0
507507 ;;
508− pinned|pinmaps)
508+ pinned|pinmaps|kernel_btf)
509509 _filedir
510510 return 0
511511 ;;
512512 *)
513513 COMPREPLY=( $( compgen -W "map" -- "$cur" ) )
514− _bpftool_once_attr 'type pinmaps autoattach'
514+ _bpftool_once_attr 'type pinmaps autoattach kernel_btf'
515515 _bpftool_one_of_list 'offload_dev xdpmeta_dev'
516516 return 0
517517 ;;
tools/bpf/bpftool/cgroup.c+7−7
@@ -221,7 +221,7 @@ static int cgroup_has_attached_progs(int cgroup_fd) · 220 unmodified lines
221221 for (i = 0; i < ARRAY_SIZE(cgroup_attach_types); i++) {
222222 int count = count_attached_bpf_progs(cgroup_fd, cgroup_attach_types[i]);
223223
224− if (count < 0)
224+ if (count < 0 && errno != EINVAL)
225225 return -1;
226226
227227 if (count > 0) {
@@ -318,11 +318,11 @@ static int show_bpf_progs(int cgroup_fd, enum bpf_attach_type type, · 90 unmodified lines
318318
319319 static int do_show(int argc, char **argv)
320320 {
321− enum bpf_attach_type type;
322321 int has_attached_progs;
323322 const char *path;
324323 int cgroup_fd;
325324 int ret = -1;
325+ unsigned int i;
326326
327327 query_flags = 0;
328328
@@ -370,14 +370,14 @@ static int do_show(int argc, char **argv) · 41 unmodified lines
370370 "AttachFlags", "Name");
371371
372372 btf_vmlinux = libbpf_find_kernel_btf();
373− for (type = 0; type < __MAX_BPF_ATTACH_TYPE; type++) {
373+ for (i = 0; i < ARRAY_SIZE(cgroup_attach_types); i++) {
374374 /*
375375 * Not all attach types may be supported, so it's expected,
376376 * that some requests will fail.
377377 * If we were able to get the show for at least one
378378 * attach type, let's return 0.
379379 */
380− if (show_bpf_progs(cgroup_fd, type, 0) == 0)
380+ if (show_bpf_progs(cgroup_fd, cgroup_attach_types[i], 0) == 0)
381381 ret = 0;
382382 }
383383
@@ -400,9 +400,9 @@ exit: · 16 unmodified lines
400400 static int do_show_tree_fn(const char *fpath, const struct stat *sb,
401401 int typeflag, struct FTW *ftw)
402402 {
403− enum bpf_attach_type type;
404403 int has_attached_progs;
405404 int cgroup_fd;
405+ unsigned int i;
406406
407407 if (typeflag != FTW_D)
408408 return 0;
@@ -434,8 +434,8 @@ static int do_show_tree_fn(const char *fpath, const struct stat *sb, · 25 unmodified lines
434434 }
435435
436436 btf_vmlinux = libbpf_find_kernel_btf();
437− for (type = 0; type < __MAX_BPF_ATTACH_TYPE; type++)
438− show_bpf_progs(cgroup_fd, type, ftw->level);
437+ for (i = 0; i < ARRAY_SIZE(cgroup_attach_types); i++)
438+ show_bpf_progs(cgroup_fd, cgroup_attach_types[i], ftw->level);
439439
440440 if (errno == EINVAL)
441441 /* Last attach type does not support query.
tools/bpf/bpftool/link.c+3−0
@@ -380,6 +380,7 @@ show_perf_event_uprobe_json(struct bpf_link_info *info, json_writer_t *wtr) · 379 unmodified lines
380380 u64_to_ptr(info->perf_event.uprobe.file_name));
381381 jsonw_uint_field(wtr, "offset", info->perf_event.uprobe.offset);
382382 jsonw_uint_field(wtr, "cookie", info->perf_event.uprobe.cookie);
383+ jsonw_uint_field(wtr, "ref_ctr_offset", info->perf_event.uprobe.ref_ctr_offset);
383384 }
384385
385386 static void
@@ -823,6 +824,8 @@ static void show_perf_event_uprobe_plain(struct bpf_link_info *info) · 437 unmodified lines
823824 printf("%s+%#x ", buf, info->perf_event.uprobe.offset);
824825 if (info->perf_event.uprobe.cookie)
825826 printf("cookie %llu ", info->perf_event.uprobe.cookie);
827+ if (info->perf_event.uprobe.ref_ctr_offset)
828+ printf("ref_ctr_offset 0x%llx ", info->perf_event.uprobe.ref_ctr_offset);
826829 }
827830
828831 static void show_perf_event_tracepoint_plain(struct bpf_link_info *info)
tools/bpf/bpftool/prog.c+11−1
@@ -1681,8 +1681,17 @@ offload_dev: · 1680 unmodified lines
16811681 } else if (is_prefix(*argv, "autoattach")) {
16821682 auto_attach = true;
16831683 NEXT_ARG();
1684+ } else if (is_prefix(*argv, "kernel_btf")) {
1685+ NEXT_ARG();
1686+
1687+ if (!REQ_ARGS(1))
1688+ goto err_free_reuse_maps;
1689+
1690+ open_opts.btf_custom_path = GET_ARG();
16841691 } else {
1685− p_err("expected no more arguments, 'type', 'map' or 'dev', got: '%s'?",
1692+ p_err("expected no more arguments, "
1693+ "'type', 'map', 'offload_dev', 'xdpmeta_dev', 'pinmaps', "
1694+ "'autoattach', or 'kernel_btf', got: '%s'?",
16861695 *argv);
16871696 goto err_free_reuse_maps;
16881697 }
@@ -2474,6 +2483,7 @@ static int do_help(int argc, char **argv) · 785 unmodified lines
24742483 " [map { idx IDX | name NAME } MAP]\\\n"
24752484 " [pinmaps MAP_DIR]\n"
24762485 " [autoattach]\n"
2486+ " [kernel_btf BTF_FILE]\n"
24772487 " %1$s %2$s attach PROG ATTACH_TYPE [MAP]\n"
24782488 " %1$s %2$s detach PROG ATTACH_TYPE [MAP]\n"
24792489 " %1$s %2$s run PROG \\\n"
tools/include/uapi/linux/bpf.h+12−7
@@ -1506,7 +1506,7 @@ union bpf_attr { · 1505 unmodified lines
15061506 __s32 map_token_fd;
15071507 };
15081508
1509− struct { /* anonymous struct used by BPF_MAP_*_ELEM commands */
1509+ struct { /* anonymous struct used by BPF_MAP_*_ELEM and BPF_MAP_FREEZE commands */
15101510 __u32 map_fd;
15111511 __aligned_u64 key;
15121512 union {
@@ -1995,11 +1995,15 @@ union bpf_attr { · 482 unmodified lines
19951995 * long bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from, u32 len, u64 flags)
19961996 * Description
19971997 * Store *len* bytes from address *from* into the packet
1998− * associated to *skb*, at *offset*. *flags* are a combination of
1999− * **BPF_F_RECOMPUTE_CSUM** (automatically recompute the
2000− * checksum for the packet after storing the bytes) and
2001− * **BPF_F_INVALIDATE_HASH** (set *skb*\ **->hash**, *skb*\
2002− * **->swhash** and *skb*\ **->l4hash** to 0).
1998+ * associated to *skb*, at *offset*. The *flags* are a combination
1999+ * of the following values:
2000+ *
2001+ * **BPF_F_RECOMPUTE_CSUM**
2002+ * Automatically update *skb*\ **->csum** after storing the
2003+ * bytes.
2004+ * **BPF_F_INVALIDATE_HASH**
2005+ * Set *skb*\ **->hash**, *skb*\ **->swhash** and *skb*\
2006+ * **->l4hash** to 0.
20032007 *
20042008 * A call to this helper is susceptible to change the underlying
20052009 * packet buffer. Therefore, at load time, all checks on pointers
@@ -2051,7 +2055,7 @@ union bpf_attr { · 45 unmodified lines
20512055 * untouched (unless **BPF_F_MARK_ENFORCE** is added as well), and
20522056 * for updates resulting in a null checksum the value is set to
20532057 * **CSUM_MANGLED_0** instead. Flag **BPF_F_PSEUDO_HDR** indicates
2054− * the checksum is to be computed against a pseudo-header.
2058+ * that the modified header field is part of the pseudo-header.
20552059 *
20562060 * This helper works in combination with **bpf_csum_diff**\ (),
20572061 * which does not update the checksum in-place, but offers more
@@ -6723,6 +6727,7 @@ struct bpf_link_info { · 4665 unmodified lines
67236727 __u32 name_len;
67246728 __u32 offset; /* offset from file_name */
67256729 __u64 cookie;
6730+ __u64 ref_ctr_offset;
67266731 } uprobe; /* BPF_PERF_EVENT_UPROBE, BPF_PERF_EVENT_URETPROBE */
67276732 struct {
67286733 __aligned_u64 func_name; /* in/out */
tools/lib/bpf/bpf_core_read.h+6−0
@@ -388,7 +388,13 @@ extern void *bpf_rdonly_cast(const void *obj, __u32 btf_id) __ksym __weak; · 387 unmodified lines
388388 #define ___arrow10(a, b, c, d, e, f, g, h, i, j) a->b->c->d->e->f->g->h->i->j
389389 #define ___arrow(...) ___apply(___arrow, ___narg(__VA_ARGS__))(__VA_ARGS__)
390390
391+#if defined(__clang__) && (__clang_major__ >= 19)
392+#define ___type(...) __typeof_unqual__(___arrow(__VA_ARGS__))
393+#elif defined(__GNUC__) && (__GNUC__ >= 14)
394+#define ___type(...) __typeof_unqual__(___arrow(__VA_ARGS__))
395+#else
391396 #define ___type(...) typeof(___arrow(__VA_ARGS__))
397+#endif
392398
393399 #define ___read(read_fn, dst, src_type, src, accessor) \
394400 read_fn((void *)(dst), sizeof(*(dst)), &((src_type)(src))->accessor)
tools/lib/bpf/bpf_helpers.h+8−0
@@ -15,6 +15,14 @@ · 14 unmodified lines
1515 #define __array(name, val) typeof(val) *name[]
1616 #define __ulong(name, val) enum { ___bpf_concat(__unique_value, __COUNTER__) = val } name
1717
18+#ifndef likely
19+#define likely(x) (__builtin_expect(!!(x), 1))
20+#endif
21+
22+#ifndef unlikely
23+#define unlikely(x) (__builtin_expect(!!(x), 0))
24+#endif
25+
1826 /*
1927 * Helper macro to place programs, maps, license in
2028 * different sections in elf_bpf file. Section names
tools/lib/bpf/btf.c+171−55
@@ -12,6 +12,7 @@ · 11 unmodified lines
1212 #include <sys/utsname.h>
1313 #include <sys/param.h>
1414 #include <sys/stat.h>
15+#include <sys/mman.h>
1516 #include <linux/kernel.h>
1617 #include <linux/err.h>
1718 #include <linux/btf.h>
@@ -120,6 +121,9 @@ struct btf { · 102 unmodified lines
120121 /* whether base_btf should be freed in btf_free for this instance */
121122 bool owns_base;
122123
124+ /* whether raw_data is a (read-only) mmap */
125+ bool raw_data_is_mmap;
126+
123127 /* BTF object FD, if loaded into kernel */
124128 int fd;
125129
@@ -951,6 +955,17 @@ static bool btf_is_modifiable(const struct btf *btf) · 825 unmodified lines
951955 return (void *)btf->hdr != btf->raw_data;
952956 }
953957
958+static void btf_free_raw_data(struct btf *btf)
959+{
960+ if (btf->raw_data_is_mmap) {
961+ munmap(btf->raw_data, btf->raw_size);
962+ btf->raw_data_is_mmap = false;
963+ } else {
964+ free(btf->raw_data);
965+ }
966+ btf->raw_data = NULL;
967+}
968+
954969 void btf__free(struct btf *btf)
955970 {
956971 if (IS_ERR_OR_NULL(btf))
@@ -970,7 +985,7 @@ void btf__free(struct btf *btf) · 13 unmodified lines
970985 free(btf->types_data);
971986 strset__free(btf->strs_set);
972987 }
973− free(btf->raw_data);
988+ btf_free_raw_data(btf);
974989 free(btf->raw_data_swapped);
975990 free(btf->type_offs);
976991 if (btf->owns_base)
@@ -996,7 +1011,7 @@ static struct btf *btf_new_empty(struct btf *base_btf) · 19 unmodified lines
9961011 if (base_btf) {
9971012 btf->base_btf = base_btf;
9981013 btf->start_id = btf__type_cnt(base_btf);
999− btf->start_str_off = base_btf->hdr->str_len;
1014+ btf->start_str_off = base_btf->hdr->str_len + base_btf->start_str_off;
10001015 btf->swapped_endian = base_btf->swapped_endian;
10011016 }
10021017
@@ -1030,7 +1045,7 @@ struct btf *btf__new_empty_split(struct btf *base_btf) · 27 unmodified lines
10301045 return libbpf_ptr(btf_new_empty(base_btf));
10311046 }
10321047
1033−static struct btf *btf_new(const void *data, __u32 size, struct btf *base_btf)
1048+static struct btf *btf_new(const void *data, __u32 size, struct btf *base_btf, bool is_mmap)
10341049 {
10351050 struct btf *btf;
10361051 int err;
@@ -1050,12 +1065,18 @@ static struct btf *btf_new(const void *data, __u32 size, struct btf *base_btf) · 13 unmodified lines
10501065 btf->start_str_off = base_btf->hdr->str_len;
10511066 }
10521067
1053− btf->raw_data = malloc(size);
1054− if (!btf->raw_data) {
1055− err = -ENOMEM;
1056− goto done;
1068+ if (is_mmap) {
1069+ btf->raw_data = (void *)data;
1070+ btf->raw_data_is_mmap = true;
1071+ } else {
1072+ btf->raw_data = malloc(size);
1073+ if (!btf->raw_data) {
1074+ err = -ENOMEM;
1075+ goto done;
1076+ }
1077+ memcpy(btf->raw_data, data, size);
10571078 }
1058− memcpy(btf->raw_data, data, size);
1079+
10591080 btf->raw_size = size;
10601081
10611082 btf->hdr = btf->raw_data;
@@ -1083,12 +1104,12 @@ done: · 21 unmodified lines
10831104
10841105 struct btf *btf__new(const void *data, __u32 size)
10851106 {
1086− return libbpf_ptr(btf_new(data, size, NULL));
1107+ return libbpf_ptr(btf_new(data, size, NULL, false));
10871108 }
10881109
10891110 struct btf *btf__new_split(const void *data, __u32 size, struct btf *base_btf)
10901111 {
1091− return libbpf_ptr(btf_new(data, size, base_btf));
1112+ return libbpf_ptr(btf_new(data, size, base_btf, false));
10921113 }
10931114
10941115 struct btf_elf_secs {
@@ -1148,6 +1169,12 @@ static int btf_find_elf_sections(Elf *elf, const char *path, struct btf_elf_secs · 53 unmodified lines
11481169 else
11491170 continue;
11501171
1172+ if (sh.sh_type != SHT_PROGBITS) {
1173+ pr_warn("unexpected section type (%d) of section(%d, %s) from %s\n",
1174+ sh.sh_type, idx, name, path);
1175+ goto err;
1176+ }
1177+
11511178 data = elf_getdata(scn, 0);
11521179 if (!data) {
11531180 pr_warn("failed to get section(%d, %s) data from %s\n",
@@ -1203,7 +1230,7 @@ static struct btf *btf_parse_elf(const char *path, struct btf *base_btf, · 49 unmodified lines
12031230
12041231 if (secs.btf_base_data) {
12051232 dist_base_btf = btf_new(secs.btf_base_data->d_buf, secs.btf_base_data->d_size,
1206− NULL);
1233+ NULL, false);
12071234 if (IS_ERR(dist_base_btf)) {
12081235 err = PTR_ERR(dist_base_btf);
12091236 dist_base_btf = NULL;
@@ -1212,7 +1239,7 @@ static struct btf *btf_parse_elf(const char *path, struct btf *base_btf, · 2 unmodified lines
12121239 }
12131240
12141241 btf = btf_new(secs.btf_data->d_buf, secs.btf_data->d_size,
1215− dist_base_btf ?: base_btf);
1242+ dist_base_btf ?: base_btf, false);
12161243 if (IS_ERR(btf)) {
12171244 err = PTR_ERR(btf);
12181245 goto done;
@@ -1329,7 +1356,7 @@ static struct btf *btf_parse_raw(const char *path, struct btf *base_btf) · 110 unmodified lines
13291356 }
13301357
13311358 /* finally parse BTF data */
1332− btf = btf_new(data, sz, base_btf);
1359+ btf = btf_new(data, sz, base_btf, false);
13331360
13341361 err_out:
13351362 free(data);
@@ -1348,6 +1375,37 @@ struct btf *btf__parse_raw_split(const char *path, struct btf *base_btf) · 12 unmodified lines
13481375 return libbpf_ptr(btf_parse_raw(path, base_btf));
13491376 }
13501377
1378+static struct btf *btf_parse_raw_mmap(const char *path, struct btf *base_btf)
1379+{
1380+ struct stat st;
1381+ void *data;
1382+ struct btf *btf;
1383+ int fd, err;
1384+
1385+ fd = open(path, O_RDONLY);
1386+ if (fd < 0)
1387+ return libbpf_err_ptr(-errno);
1388+
1389+ if (fstat(fd, &st) < 0) {
1390+ err = -errno;
1391+ close(fd);
1392+ return libbpf_err_ptr(err);
1393+ }
1394+
1395+ data = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
1396+ err = -errno;
1397+ close(fd);
1398+
1399+ if (data == MAP_FAILED)
1400+ return libbpf_err_ptr(err);
1401+
1402+ btf = btf_new(data, st.st_size, base_btf, true);
1403+ if (IS_ERR(btf))
1404+ munmap(data, st.st_size);
1405+
1406+ return btf;
1407+}
1408+
13511409 static struct btf *btf_parse(const char *path, struct btf *base_btf, struct btf_ext **btf_ext)
13521410 {
13531411 struct btf *btf;
@@ -1612,7 +1670,7 @@ struct btf *btf_get_from_fd(int btf_fd, struct btf *base_btf) · 258 unmodified lines
16121670 goto exit_free;
16131671 }
16141672
1615− btf = btf_new(ptr, btf_info.btf_size, base_btf);
1673+ btf = btf_new(ptr, btf_info.btf_size, base_btf, false);
16161674
16171675 exit_free:
16181676 free(ptr);
@@ -1652,10 +1710,8 @@ struct btf *btf__load_from_kernel_by_id(__u32 id) · 33 unmodified lines
16521710
16531711 static void btf_invalidate_raw_data(struct btf *btf)
16541712 {
1655− if (btf->raw_data) {
1656− free(btf->raw_data);
1657− btf->raw_data = NULL;
1658− }
1713+ if (btf->raw_data)
1714+ btf_free_raw_data(btf);
16591715 if (btf->raw_data_swapped) {
16601716 free(btf->raw_data_swapped);
16611717 btf->raw_data_swapped = NULL;
@@ -4350,46 +4406,109 @@ static inline __u16 btf_fwd_kind(struct btf_type *t) · 2688 unmodified lines
43504406 return btf_kflag(t) ? BTF_KIND_UNION : BTF_KIND_STRUCT;
43514407 }
43524408
4353−/* Check if given two types are identical ARRAY definitions */
4354−static bool btf_dedup_identical_arrays(struct btf_dedup *d, __u32 id1, __u32 id2)
4409+static bool btf_dedup_identical_types(struct btf_dedup *d, __u32 id1, __u32 id2, int depth)
43554410 {
43564411 struct btf_type *t1, *t2;
4412+ int k1, k2;
4413+recur:
4414+ if (depth <= 0)
4415+ return false;
43574416
43584417 t1 = btf_type_by_id(d->btf, id1);
43594418 t2 = btf_type_by_id(d->btf, id2);
4360− if (!btf_is_array(t1) || !btf_is_array(t2))
4419+
4420+ k1 = btf_kind(t1);
4421+ k2 = btf_kind(t2);
4422+ if (k1 != k2)
43614423 return false;
43624424
4363− return btf_equal_array(t1, t2);
4364−}
4425+ switch (k1) {
4426+ case BTF_KIND_UNKN: /* VOID */
4427+ return true;
4428+ case BTF_KIND_INT:
4429+ return btf_equal_int_tag(t1, t2);
4430+ case BTF_KIND_ENUM:
4431+ case BTF_KIND_ENUM64:
4432+ return btf_compat_enum(t1, t2);
4433+ case BTF_KIND_FWD:
4434+ case BTF_KIND_FLOAT:
4435+ return btf_equal_common(t1, t2);
4436+ case BTF_KIND_CONST:
4437+ case BTF_KIND_VOLATILE:
4438+ case BTF_KIND_RESTRICT:
4439+ case BTF_KIND_PTR:
4440+ case BTF_KIND_TYPEDEF:
4441+ case BTF_KIND_FUNC:
4442+ case BTF_KIND_TYPE_TAG:
4443+ if (t1->info != t2->info || t1->name_off != t2->name_off)
4444+ return false;
4445+ id1 = t1->type;
4446+ id2 = t2->type;
4447+ goto recur;
4448+ case BTF_KIND_ARRAY: {
4449+ struct btf_array *a1, *a2;
43654450
4366−/* Check if given two types are identical STRUCT/UNION definitions */
4367−static bool btf_dedup_identical_structs(struct btf_dedup *d, __u32 id1, __u32 id2)
4368−{
4369− const struct btf_member *m1, *m2;
4370− struct btf_type *t1, *t2;
4371− int n, i;
4451+ if (!btf_compat_array(t1, t2))
4452+ return false;
43724453
4373− t1 = btf_type_by_id(d->btf, id1);
4374− t2 = btf_type_by_id(d->btf, id2);
4454+ a1 = btf_array(t1);
4455+ a2 = btf_array(t1);
43754456
4376− if (!btf_is_composite(t1) || btf_kind(t1) != btf_kind(t2))
4377− return false;
4457+ if (a1->index_type != a2->index_type &&
4458+ !btf_dedup_identical_types(d, a1->index_type, a2->index_type, depth - 1))
4459+ return false;
43784460
4379− if (!btf_shallow_equal_struct(t1, t2))
4380− return false;
4461+ if (a1->type != a2->type &&
4462+ !btf_dedup_identical_types(d, a1->type, a2->type, depth - 1))
4463+ return false;
43814464
4382− m1 = btf_members(t1);
4383− m2 = btf_members(t2);
4384− for (i = 0, n = btf_vlen(t1); i < n; i++, m1++, m2++) {
4385− if (m1->type != m2->type &&
4386− !btf_dedup_identical_arrays(d, m1->type, m2->type) &&
4387− !btf_dedup_identical_structs(d, m1->type, m2->type))
4465+ return true;
4466+ }
4467+ case BTF_KIND_STRUCT:
4468+ case BTF_KIND_UNION: {
4469+ const struct btf_member *m1, *m2;
4470+ int i, n;
4471+
4472+ if (!btf_shallow_equal_struct(t1, t2))
43884473 return false;
4474+
4475+ m1 = btf_members(t1);
4476+ m2 = btf_members(t2);
4477+ for (i = 0, n = btf_vlen(t1); i < n; i++, m1++, m2++) {
4478+ if (m1->type == m2->type)
4479+ continue;
4480+ if (!btf_dedup_identical_types(d, m1->type, m2->type, depth - 1))
4481+ return false;
4482+ }
4483+ return true;
4484+ }
4485+ case BTF_KIND_FUNC_PROTO: {
4486+ const struct btf_param *p1, *p2;
4487+ int i, n;
4488+
4489+ if (!btf_compat_fnproto(t1, t2))
4490+ return false;
4491+
4492+ if (t1->type != t2->type &&
4493+ !btf_dedup_identical_types(d, t1->type, t2->type, depth - 1))
4494+ return false;
4495+
4496+ p1 = btf_params(t1);
4497+ p2 = btf_params(t2);
4498+ for (i = 0, n = btf_vlen(t1); i < n; i++, p1++, p2++) {
4499+ if (p1->type == p2->type)
4500+ continue;
4501+ if (!btf_dedup_identical_types(d, p1->type, p2->type, depth - 1))
4502+ return false;
4503+ }
4504+ return true;
4505+ }
4506+ default:
4507+ return false;
43894508 }
4390− return true;
43914509 }
43924510
4511+
43934512 /*
43944513 * Check equivalence of BTF type graph formed by candidate struct/union (we'll
43954514 * call it "candidate graph" in this description for brevity) to a type graph
@@ -4508,19 +4627,13 @@ static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id, · 112 unmodified lines
45084627 * different fields within the *same* struct. This breaks type
45094628 * equivalence check, which makes an assumption that candidate
45104629 * types sub-graph has a consistent and deduped-by-compiler
4511− * types within a single CU. So work around that by explicitly
4512− * allowing identical array types here.
4630+ * types within a single CU. And similar situation can happen
4631+ * with struct/union sometimes, and event with pointers.
4632+ * So accommodate cases like this doing a structural
4633+ * comparison recursively, but avoiding being stuck in endless
4634+ * loops by limiting the depth up to which we check.
45134635 */
4514− if (btf_dedup_identical_arrays(d, hypot_type_id, cand_id))
4515− return 1;
4516− /* It turns out that similar situation can happen with
4517− * struct/union sometimes, sigh... Handle the case where
4518− * structs/unions are exactly the same, down to the referenced
4519− * type IDs. Anything more complicated (e.g., if referenced
4520− * types are different, but equivalent) is *way more*
4521− * complicated and requires a many-to-many equivalence mapping.
4522− */
4523− if (btf_dedup_identical_structs(d, hypot_type_id, cand_id))
4636+ if (btf_dedup_identical_types(d, hypot_type_id, cand_id, 16))
45244637 return 1;
45254638 return 0;
45264639 }
@@ -5268,7 +5381,10 @@ struct btf *btf__load_vmlinux_btf(void) · 741 unmodified lines
52685381 pr_warn("kernel BTF is missing at '%s', was CONFIG_DEBUG_INFO_BTF enabled?\n",
52695382 sysfs_btf_path);
52705383 } else {
5271− btf = btf__parse(sysfs_btf_path, NULL);
5384+ btf = btf_parse_raw_mmap(sysfs_btf_path, NULL);
5385+ if (IS_ERR(btf))
5386+ btf = btf__parse(sysfs_btf_path, NULL);
5387+
52725388 if (!btf) {
52735389 err = -errno;
52745390 pr_warn("failed to read kernel BTF from '%s': %s\n",
tools/lib/bpf/libbpf.c+48−39
@@ -60,6 +60,8 @@ · 59 unmodified lines
6060 #define BPF_FS_MAGIC 0xcafe4a11
6161 #endif
6262
63+#define MAX_EVENT_NAME_LEN 64
64+
6365 #define BPF_FS_DEFAULT_PATH "/sys/fs/bpf"
6466
6567 #define BPF_INSN_SZ (sizeof(struct bpf_insn))
@@ -284,7 +286,7 @@ void libbpf_print(enum libbpf_print_level level, const char *format, ...) · 218 unmodified lines
284286 old_errno = errno;
285287
286288 va_start(args, format);
287− __libbpf_pr(level, format, args);
289+ print_fn(level, format, args);
288290 va_end(args);
289291
290292 errno = old_errno;
@@ -896,7 +898,7 @@ bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data, · 605 unmodified lines
896898 return -LIBBPF_ERRNO__FORMAT;
897899 }
898900
899− if (sec_off + prog_sz > sec_sz) {
901+ if (sec_off + prog_sz > sec_sz || sec_off + prog_sz < sec_off) {
900902 pr_warn("sec '%s': program at offset %zu crosses section boundary\n",
901903 sec_name, sec_off);
902904 return -LIBBPF_ERRNO__FORMAT;
@@ -1725,15 +1727,6 @@ static Elf64_Sym *find_elf_var_sym(const struct bpf_object *obj, const char *nam · 822 unmodified lines
17251727 return ERR_PTR(-ENOENT);
17261728 }
17271729
1728−/* Some versions of Android don't provide memfd_create() in their libc
1729− * implementation, so avoid complications and just go straight to Linux
1730− * syscall.
1731− */
1732−static int sys_memfd_create(const char *name, unsigned flags)
1733−{
1734− return syscall(__NR_memfd_create, name, flags);
1735−}
1736−
17371730 #ifndef MFD_CLOEXEC
17381731 #define MFD_CLOEXEC 0x0001U
17391732 #endif
@@ -9455,6 +9448,30 @@ int bpf_program__set_log_buf(struct bpf_program *prog, char *log_buf, size_t log · 7715 unmodified lines
94559448 return 0;
94569449 }
94579450
9451+struct bpf_func_info *bpf_program__func_info(const struct bpf_program *prog)
9452+{
9453+ if (prog->func_info_rec_size != sizeof(struct bpf_func_info))
9454+ return libbpf_err_ptr(-EOPNOTSUPP);
9455+ return prog->func_info;
9456+}
9457+
9458+__u32 bpf_program__func_info_cnt(const struct bpf_program *prog)
9459+{
9460+ return prog->func_info_cnt;
9461+}
9462+
9463+struct bpf_line_info *bpf_program__line_info(const struct bpf_program *prog)
9464+{
9465+ if (prog->line_info_rec_size != sizeof(struct bpf_line_info))
9466+ return libbpf_err_ptr(-EOPNOTSUPP);
9467+ return prog->line_info;
9468+}
9469+
9470+__u32 bpf_program__line_info_cnt(const struct bpf_program *prog)
9471+{
9472+ return prog->line_info_cnt;
9473+}
9474+
94589475 #define SEC_DEF(sec_pfx, ptype, atype, flags, ...) { \
94599476 .sec = (char *)sec_pfx, \
94609477 .prog_type = BPF_PROG_TYPE_##ptype, \
@@ -11121,16 +11138,16 @@ static const char *tracefs_available_filter_functions_addrs(void) · 1660 unmodified lines
1112111138 : TRACEFS"/available_filter_functions_addrs";
1112211139 }
1112311140
11124−static void gen_kprobe_legacy_event_name(char *buf, size_t buf_sz,
11125− const char *kfunc_name, size_t offset)
11141+static void gen_probe_legacy_event_name(char *buf, size_t buf_sz,
11142+ const char *name, size_t offset)
1112611143 {
1112711144 static int index = 0;
1112811145 int i;
1112911146
11130− snprintf(buf, buf_sz, "libbpf_%u_%s_0x%zx_%d", getpid(), kfunc_name, offset,
11131− __sync_fetch_and_add(&index, 1));
11147+ snprintf(buf, buf_sz, "libbpf_%u_%d_%s_0x%zx", getpid(),
11148+ __sync_fetch_and_add(&index, 1), name, offset);
1113211149
11133− /* sanitize binary_path in the probe name */
11150+ /* sanitize name in the probe name */
1113411151 for (i = 0; buf[i]; i++) {
1113511152 if (!isalnum(buf[i]))
1113611153 buf[i] = '_';
@@ -11255,9 +11272,9 @@ int probe_kern_syscall_wrapper(int token_fd) · 118 unmodified lines
1125511272
1125611273 return pfd >= 0 ? 1 : 0;
1125711274 } else { /* legacy mode */
11258− char probe_name[128];
11275+ char probe_name[MAX_EVENT_NAME_LEN];
1125911276
11260− gen_kprobe_legacy_event_name(probe_name, sizeof(probe_name), syscall_name, 0);
11277+ gen_probe_legacy_event_name(probe_name, sizeof(probe_name), syscall_name, 0);
1126111278 if (add_kprobe_event_legacy(probe_name, false, syscall_name, 0) < 0)
1126211279 return 0;
1126311280
@@ -11313,10 +11330,10 @@ bpf_program__attach_kprobe_opts(const struct bpf_program *prog, · 49 unmodified lines
1131311330 func_name, offset,
1131411331 -1 /* pid */, 0 /* ref_ctr_off */);
1131511332 } else {
11316− char probe_name[256];
11333+ char probe_name[MAX_EVENT_NAME_LEN];
1131711334
11318− gen_kprobe_legacy_event_name(probe_name, sizeof(probe_name),
11319− func_name, offset);
11335+ gen_probe_legacy_event_name(probe_name, sizeof(probe_name),
11336+ func_name, offset);
1132011337
1132111338 legacy_probe = strdup(probe_name);
1132211339 if (!legacy_probe)
@@ -11860,20 +11877,6 @@ static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, stru · 537 unmodified lines
1186011877 return ret;
1186111878 }
1186211879
11863−static void gen_uprobe_legacy_event_name(char *buf, size_t buf_sz,
11864− const char *binary_path, uint64_t offset)
11865−{
11866− int i;
11867−
11868− snprintf(buf, buf_sz, "libbpf_%u_%s_0x%zx", getpid(), binary_path, (size_t)offset);
11869−
11870− /* sanitize binary_path in the probe name */
11871− for (i = 0; buf[i]; i++) {
11872− if (!isalnum(buf[i]))
11873− buf[i] = '_';
11874− }
11875−}
11876−
1187711880 static inline int add_uprobe_event_legacy(const char *probe_name, bool retprobe,
1187811881 const char *binary_path, size_t offset)
1187911882 {
@@ -12297,13 +12300,14 @@ bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid, · 417 unmodified lines
1229712300 pfd = perf_event_open_probe(true /* uprobe */, retprobe, binary_path,
1229812301 func_offset, pid, ref_ctr_off);
1229912302 } else {
12300− char probe_name[PATH_MAX + 64];
12303+ char probe_name[MAX_EVENT_NAME_LEN];
1230112304
1230212305 if (ref_ctr_off)
1230312306 return libbpf_err_ptr(-EINVAL);
1230412307
12305− gen_uprobe_legacy_event_name(probe_name, sizeof(probe_name),
12306− binary_path, func_offset);
12308+ gen_probe_legacy_event_name(probe_name, sizeof(probe_name),
12309+ strrchr(binary_path, '/') ? : binary_path,
12310+ func_offset);
1230712311
1230812312 legacy_probe = strdup(probe_name);
1230912313 if (!legacy_probe)
@@ -13371,7 +13375,6 @@ struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt, · 1061 unmodified lines
1337113375 attr.config = PERF_COUNT_SW_BPF_OUTPUT;
1337213376 attr.type = PERF_TYPE_SOFTWARE;
1337313377 attr.sample_type = PERF_SAMPLE_RAW;
13374− attr.sample_period = sample_period;
1337513378 attr.wakeup_events = sample_period;
1337613379
1337713380 p.attr = &attr;
@@ -14099,6 +14102,12 @@ int bpf_object__attach_skeleton(struct bpf_object_skeleton *s) · 721 unmodified lines
1409914102 }
1410014103
1410114104 link = map_skel->link;
14105+ if (!link) {
14106+ pr_warn("map '%s': BPF map skeleton link is uninitialized\n",
14107+ bpf_map__name(map));
14108+ continue;
14109+ }
14110+
1410214111 if (*link)
1410314112 continue;
1410414113
tools/lib/bpf/libbpf.h+6−0
@@ -940,6 +940,12 @@ LIBBPF_API int bpf_program__set_log_level(struct bpf_program *prog, __u32 log_le · 939 unmodified lines
940940 LIBBPF_API const char *bpf_program__log_buf(const struct bpf_program *prog, size_t *log_size);
941941 LIBBPF_API int bpf_program__set_log_buf(struct bpf_program *prog, char *log_buf, size_t log_size);
942942
943+LIBBPF_API struct bpf_func_info *bpf_program__func_info(const struct bpf_program *prog);
944+LIBBPF_API __u32 bpf_program__func_info_cnt(const struct bpf_program *prog);
945+
946+LIBBPF_API struct bpf_line_info *bpf_program__line_info(const struct bpf_program *prog);
947+LIBBPF_API __u32 bpf_program__line_info_cnt(const struct bpf_program *prog);
948+
943949 /**
944950 * @brief **bpf_program__set_attach_target()** sets BTF-based attach target
945951 * for supported BPF program types:
tools/lib/bpf/libbpf.map+4−0
@@ -437,6 +437,10 @@ LIBBPF_1.6.0 { · 436 unmodified lines
437437 bpf_linker__add_fd;
438438 bpf_linker__new_fd;
439439 bpf_object__prepare;
440+ bpf_program__func_info;
441+ bpf_program__func_info_cnt;
442+ bpf_program__line_info;
443+ bpf_program__line_info_cnt;
440444 btf__add_decl_attr;
441445 btf__add_type_attr;
442446 } LIBBPF_1.5.0;
tools/lib/bpf/libbpf_internal.h+9−0
@@ -667,6 +667,15 @@ static inline int sys_dup3(int oldfd, int newfd, int flags) · 666 unmodified lines
667667 return syscall(__NR_dup3, oldfd, newfd, flags);
668668 }
669669
670+/* Some versions of Android don't provide memfd_create() in their libc
671+ * implementation, so avoid complications and just go straight to Linux
672+ * syscall.
673+ */
674+static inline int sys_memfd_create(const char *name, unsigned flags)
675+{
676+ return syscall(__NR_memfd_create, name, flags);
677+}
678+
670679 /* Point *fixed_fd* to the same file that *tmp_fd* points to.
671680 * Regardless of success, *tmp_fd* is closed.
672681 * Whatever *fixed_fd* pointed to is closed silently.
tools/lib/bpf/linker.c+3−3
@@ -573,7 +573,7 @@ int bpf_linker__add_buf(struct bpf_linker *linker, void *buf, size_t buf_sz, · 572 unmodified lines
573573
574574 snprintf(filename, sizeof(filename), "mem:%p+%zu", buf, buf_sz);
575575
576− fd = memfd_create(filename, 0);
576+ fd = sys_memfd_create(filename, 0);
577577 if (fd < 0) {
578578 ret = -errno;
579579 pr_warn("failed to create memfd '%s': %s\n", filename, errstr(ret));
@@ -1376,7 +1376,7 @@ static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj · 796 unmodified lines
13761376 } else {
13771377 if (!secs_match(dst_sec, src_sec)) {
13781378 pr_warn("ELF sections %s are incompatible\n", src_sec->sec_name);
1379− return -1;
1379+ return -EINVAL;
13801380 }
13811381
13821382 /* "license" and "version" sections are deduped */
@@ -2223,7 +2223,7 @@ static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *ob · 840 unmodified lines
22232223 }
22242224 } else if (!secs_match(dst_sec, src_sec)) {
22252225 pr_warn("sections %s are not compatible\n", src_sec->sec_name);
2226− return -1;
2226+ return -EINVAL;
22272227 }
22282228
22292229 /* shdr->sh_link points to SYMTAB */
tools/lib/bpf/nlattr.c+7−8
@@ -63,16 +63,16 @@ static int validate_nla(struct nlattr *nla, int maxtype, · 62 unmodified lines
6363 minlen = nla_attr_minlen[pt->type];
6464
6565 if (libbpf_nla_len(nla) < minlen)
66− return -1;
66+ return -EINVAL;
6767
6868 if (pt->maxlen && libbpf_nla_len(nla) > pt->maxlen)
69− return -1;
69+ return -EINVAL;
7070
7171 if (pt->type == LIBBPF_NLA_STRING) {
7272 char *data = libbpf_nla_data(nla);
7373
7474 if (data[libbpf_nla_len(nla) - 1] != '\0')
75− return -1;
75+ return -EINVAL;
7676 }
7777
7878 return 0;
@@ -118,19 +118,18 @@ int libbpf_nla_parse(struct nlattr *tb[], int maxtype, struct nlattr *head, · 39 unmodified lines
118118 if (policy) {
119119 err = validate_nla(nla, maxtype, policy);
120120 if (err < 0)
121− goto errout;
121+ return err;
122122 }
123123
124− if (tb[type])
124+ if (tb[type]) {
125125 pr_warn("Attribute of type %#x found multiple times in message, "
126126 "previous attribute is being ignored.\n", type);
127+ }
127128
128129 tb[type] = nla;
129130 }
130131
131− err = 0;
132−errout:
133− return err;
132+ return 0;
134133 }
135134
136135 /**
tools/testing/selftests/bpf/DENYLIST+1−0
@@ -1,5 +1,6 @@
11 # TEMPORARY
22 # Alphabetical order
3+dynptr/test_probe_read_user_str_dynptr # disabled until https://patchwork.kernel.org/project/linux-mm/patch/20250422131449.57177-1-mykyta.yatsenko5@gmail.com/ makes it into the bpf-next
34 get_stack_raw_tp # spams with kernel warnings until next bpf -> bpf-next merge
45 stacktrace_build_id
56 stacktrace_build_id_nmi
tools/testing/selftests/bpf/DENYLIST.aarch64+0−2
@@ -1,3 +1 @@
1−fentry_test/fentry_many_args # fentry_many_args:FAIL:fentry_many_args_attach unexpected error: -524
2−fexit_test/fexit_many_args # fexit_many_args:FAIL:fexit_many_args_attach unexpected error: -524
31 tracing_struct/struct_many_args # struct_many_args:FAIL:tracing_struct_many_args__attach unexpected error: -524
tools/testing/selftests/bpf/Makefile+11−5
@@ -34,6 +34,9 @@ OPT_FLAGS ?= $(if $(RELEASE),-O2,-O0) · 33 unmodified lines
3434 LIBELF_CFLAGS := $(shell $(PKG_CONFIG) libelf --cflags 2>/dev/null)
3535 LIBELF_LIBS := $(shell $(PKG_CONFIG) libelf --libs 2>/dev/null || echo -lelf)
3636
37+SKIP_DOCS ?=
38+SKIP_LLVM ?=
39+
3740 ifeq ($(srctree),)
3841 srctree := $(patsubst %/,%,$(dir $(CURDIR)))
3942 srctree := $(patsubst %/,%,$(dir $(srctree)))
@@ -172,6 +175,7 @@ override OUTPUT := $(patsubst %/,%,$(OUTPUT)) · 132 unmodified lines
172175 endif
173176 endif
174177
178+ifneq ($(SKIP_LLVM),1)
175179 ifeq ($(feature-llvm),1)
176180 LLVM_CFLAGS += -DHAVE_LLVM_SUPPORT
177181 LLVM_CONFIG_LIB_COMPONENTS := mcdisassembler all-targets
@@ -180,13 +184,14 @@ ifeq ($(feature-llvm),1) · 2 unmodified lines
180184 # Prefer linking statically if it's available, otherwise fallback to shared
181185 ifeq ($(shell $(LLVM_CONFIG) --link-static --libs >/dev/null 2>&1 && echo static),static)
182186 LLVM_LDLIBS += $(shell $(LLVM_CONFIG) --link-static --libs $(LLVM_CONFIG_LIB_COMPONENTS))
183− LLVM_LDLIBS += $(shell $(LLVM_CONFIG) --link-static --system-libs $(LLVM_CONFIG_LIB_COMPONENTS))
187+ LLVM_LDLIBS += $(filter-out -lxml2,$(shell $(LLVM_CONFIG) --link-static --system-libs $(LLVM_CONFIG_LIB_COMPONENTS)))
184188 LLVM_LDLIBS += -lstdc++
185189 else
186190 LLVM_LDLIBS += $(shell $(LLVM_CONFIG) --link-shared --libs $(LLVM_CONFIG_LIB_COMPONENTS))
187191 endif
188192 LLVM_LDFLAGS += $(shell $(LLVM_CONFIG) --ldflags)
189193 endif
194+endif
190195
191196 SCRATCH_DIR := $(OUTPUT)/tools
192197 BUILD_DIR := $(SCRATCH_DIR)/build
@@ -358,7 +363,9 @@ $(CROSS_BPFTOOL): $(wildcard $(BPFTOOLDIR)/*.[ch] $(BPFTOOLDIR)/Makefile) \ · 165 unmodified lines
358363 prefix= DESTDIR=$(SCRATCH_DIR)/ install-bin
359364 endif
360365
366+ifneq ($(SKIP_DOCS),1)
361367 all: docs
368+endif
362369
363370 docs:
364371 $(Q)RST2MAN_OPTS="--exit-status=1" $(MAKE) $(submake_extras) \
@@ -673,9 +680,6 @@ ifneq ($2:$(OUTPUT),:$(shell pwd)) · 308 unmodified lines
673680 $(Q)rsync -aq $$^ $(TRUNNER_OUTPUT)/
674681 endif
675682
676−$(OUTPUT)/$(TRUNNER_BINARY): LDLIBS += $$(LLVM_LDLIBS)
677−$(OUTPUT)/$(TRUNNER_BINARY): LDFLAGS += $$(LLVM_LDFLAGS)
678−
679683 # some X.test.o files have runtime dependencies on Y.bpf.o files
680684 $(OUTPUT)/$(TRUNNER_BINARY): | $(TRUNNER_BPF_OBJS)
681685
@@ -686,7 +690,7 @@ $(OUTPUT)/$(TRUNNER_BINARY): $(TRUNNER_TEST_OBJS) \ · 4 unmodified lines
686690 $(OUTPUT)/veristat \
687691 | $(TRUNNER_BINARY)-extras
688692 $$(call msg,BINARY,,$$@)
689− $(Q)$$(CC) $$(CFLAGS) $$(filter %.a %.o,$$^) $$(LDLIBS) $$(LDFLAGS) -o $$@
693+ $(Q)$$(CC) $$(CFLAGS) $$(filter %.a %.o,$$^) $$(LDLIBS) $$(LLVM_LDLIBS) $$(LDFLAGS) $$(LLVM_LDFLAGS) -o $$@
690694 $(Q)$(RESOLVE_BTFIDS) --btf $(TRUNNER_OUTPUT)/btf_data.bpf.o $$@
691695 $(Q)ln -sf $(if $2,..,.)/tools/build/bpftool/$(USE_BOOTSTRAP)bpftool \
692696 $(OUTPUT)/$(if $2,$2/)bpftool
@@ -811,6 +815,7 @@ $(OUTPUT)/bench_local_storage_create.o: $(OUTPUT)/bench_local_storage_create.ske · 118 unmodified lines
811815 $(OUTPUT)/bench_bpf_hashmap_lookup.o: $(OUTPUT)/bpf_hashmap_lookup.skel.h
812816 $(OUTPUT)/bench_htab_mem.o: $(OUTPUT)/htab_mem_bench.skel.h
813817 $(OUTPUT)/bench_bpf_crypto.o: $(OUTPUT)/crypto_bench.skel.h
818+$(OUTPUT)/bench_sockmap.o: $(OUTPUT)/bench_sockmap_prog.skel.h
814819 $(OUTPUT)/bench.o: bench.h testing_helpers.h $(BPFOBJ)
815820 $(OUTPUT)/bench: LDLIBS += -lm
816821 $(OUTPUT)/bench: $(OUTPUT)/bench.o \
@@ -831,6 +836,7 @@ $(OUTPUT)/bench: $(OUTPUT)/bench.o \ · 14 unmodified lines
831836 $(OUTPUT)/bench_local_storage_create.o \
832837 $(OUTPUT)/bench_htab_mem.o \
833838 $(OUTPUT)/bench_bpf_crypto.o \
839+ $(OUTPUT)/bench_sockmap.o \
834840 #
835841 $(call msg,BINARY,,$@)
836842 $(Q)$(CC) $(CFLAGS) $(LDFLAGS) $(filter %.a %.o,$^) $(LDLIBS) -o $@
tools/testing/selftests/bpf/bench.c+4−0
@@ -283,6 +283,7 @@ extern struct argp bench_local_storage_create_argp; · 282 unmodified lines
283283 extern struct argp bench_htab_mem_argp;
284284 extern struct argp bench_trigger_batch_argp;
285285 extern struct argp bench_crypto_argp;
286+extern struct argp bench_sockmap_argp;
286287
287288 static const struct argp_child bench_parsers[] = {
288289 { &bench_ringbufs_argp, 0, "Ring buffers benchmark", 0 },
@@ -297,6 +298,7 @@ static const struct argp_child bench_parsers[] = { · 8 unmodified lines
297298 { &bench_htab_mem_argp, 0, "hash map memory benchmark", 0 },
298299 { &bench_trigger_batch_argp, 0, "BPF triggering benchmark", 0 },
299300 { &bench_crypto_argp, 0, "bpf crypto benchmark", 0 },
301+ { &bench_sockmap_argp, 0, "bpf sockmap benchmark", 0 },
300302 {},
301303 };
302304
@@ -555,6 +557,7 @@ extern const struct bench bench_local_storage_create; · 252 unmodified lines
555557 extern const struct bench bench_htab_mem;
556558 extern const struct bench bench_crypto_encrypt;
557559 extern const struct bench bench_crypto_decrypt;
560+extern const struct bench bench_sockmap;
558561
559562 static const struct bench *benchs[] = {
560563 &bench_count_global,
@@ -621,6 +624,7 @@ static const struct bench *benchs[] = { · 60 unmodified lines
621624 &bench_htab_mem,
622625 &bench_crypto_encrypt,
623626 &bench_crypto_decrypt,
627+ &bench_sockmap,
624628 };
625629
626630 static void find_benchmark(void)
tools/testing/selftests/bpf/benchs/bench_htab_mem.c+1−2
@@ -279,6 +279,7 @@ static void htab_mem_read_mem_cgrp_file(const char *name, unsigned long *value) · 278 unmodified lines
279279 }
280280
281281 got = read(fd, buf, sizeof(buf) - 1);
282+ close(fd);
282283 if (got <= 0) {
283284 *value = 0;
284285 return;
@@ -286,8 +287,6 @@ static void htab_mem_read_mem_cgrp_file(const char *name, unsigned long *value) · 1 unmodified line
286287 buf[got] = 0;
287288
288289 *value = strtoull(buf, NULL, 0);
289−
290− close(fd);
291290 }
292291
293292 static void htab_mem_measure(struct bench_res *res)
tools/testing/selftests/bpf/benchs/bench_sockmap.c+598−0
@@ -0,0 +1,598 @@
1+// SPDX-License-Identifier: GPL-2.0
2+
3+#include <error.h>
4+#include <sys/types.h>
5+#include <sys/socket.h>
6+#include <netinet/in.h>
7+#include <sys/sendfile.h>
8+#include <arpa/inet.h>
9+#include <fcntl.h>
10+#include <argp.h>
11+#include "bench.h"
12+#include "bench_sockmap_prog.skel.h"
13+
14+#define FILE_SIZE (128 * 1024)
15+#define DATA_REPEAT_SIZE 10
16+
17+static const char snd_data[DATA_REPEAT_SIZE] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
18+
19+/* c1 <-> [p1, p2] <-> c2
20+ * RX bench(BPF_SK_SKB_STREAM_VERDICT):
21+ * ARG_FW_RX_PASS:
22+ * send(p2) -> recv(c2) -> bpf skb passthrough -> recv(c2)
23+ * ARG_FW_RX_VERDICT_EGRESS:
24+ * send(c1) -> verdict skb to tx queuec of p2 -> recv(c2)
25+ * ARG_FW_RX_VERDICT_INGRESS:
26+ * send(c1) -> verdict skb to rx queuec of c2 -> recv(c2)
27+ *
28+ * TX bench(BPF_SK_MSG_VERDIC):
29+ * ARG_FW_TX_PASS:
30+ * send(p2) -> bpf msg passthrough -> send(p2) -> recv(c2)
31+ * ARG_FW_TX_VERDICT_INGRESS:
32+ * send(p2) -> verdict msg to rx queue of c2 -> recv(c2)
33+ * ARG_FW_TX_VERDICT_EGRESS:
34+ * send(p1) -> verdict msg to tx queue of p2 -> recv(c2)
35+ */
36+enum SOCKMAP_ARG_FLAG {
37+ ARG_FW_RX_NORMAL = 11000,
38+ ARG_FW_RX_PASS,
39+ ARG_FW_RX_VERDICT_EGRESS,
40+ ARG_FW_RX_VERDICT_INGRESS,
41+ ARG_FW_TX_NORMAL,
42+ ARG_FW_TX_PASS,
43+ ARG_FW_TX_VERDICT_INGRESS,
44+ ARG_FW_TX_VERDICT_EGRESS,
45+ ARG_CTL_RX_STRP,
46+ ARG_CONSUMER_DELAY_TIME,
47+ ARG_PRODUCER_DURATION,
48+};
49+
50+#define TXMODE_NORMAL() \
51+ ((ctx.mode) == ARG_FW_TX_NORMAL)
52+
53+#define TXMODE_BPF_INGRESS() \
54+ ((ctx.mode) == ARG_FW_TX_VERDICT_INGRESS)
55+
56+#define TXMODE_BPF_EGRESS() \
57+ ((ctx.mode) == ARG_FW_TX_VERDICT_EGRESS)
58+
59+#define TXMODE_BPF_PASS() \
60+ ((ctx.mode) == ARG_FW_TX_PASS)
61+
62+#define TXMODE_BPF() ( \
63+ TXMODE_BPF_PASS() || \
64+ TXMODE_BPF_INGRESS() || \
65+ TXMODE_BPF_EGRESS())
66+
67+#define TXMODE() ( \
68+ TXMODE_NORMAL() || \
69+ TXMODE_BPF())
70+
71+#define RXMODE_NORMAL() \
72+ ((ctx.mode) == ARG_FW_RX_NORMAL)
73+
74+#define RXMODE_BPF_PASS() \
75+ ((ctx.mode) == ARG_FW_RX_PASS)
76+
77+#define RXMODE_BPF_VERDICT_EGRESS() \
78+ ((ctx.mode) == ARG_FW_RX_VERDICT_EGRESS)
79+
80+#define RXMODE_BPF_VERDICT_INGRESS() \
81+ ((ctx.mode) == ARG_FW_RX_VERDICT_INGRESS)
82+
83+#define RXMODE_BPF_VERDICT() ( \
84+ RXMODE_BPF_VERDICT_INGRESS() || \
85+ RXMODE_BPF_VERDICT_EGRESS())
86+
87+#define RXMODE_BPF() ( \
88+ RXMODE_BPF_PASS() || \
89+ RXMODE_BPF_VERDICT())
90+
91+#define RXMODE() ( \
92+ RXMODE_NORMAL() || \
93+ RXMODE_BPF())
94+
95+static struct socmap_ctx {
96+ struct bench_sockmap_prog *skel;
97+ enum SOCKMAP_ARG_FLAG mode;
98+ #define c1 fds[0]
99+ #define p1 fds[1]
100+ #define c2 fds[2]
101+ #define p2 fds[3]
102+ #define sfd fds[4]
103+ int fds[5];
104+ long send_calls;
105+ long read_calls;
106+ long prod_send;
107+ long user_read;
108+ int file_size;
109+ int delay_consumer;
110+ int prod_run_time;
111+ int strp_size;
112+} ctx = {
113+ .prod_send = 0,
114+ .user_read = 0,
115+ .file_size = FILE_SIZE,
116+ .mode = ARG_FW_RX_VERDICT_EGRESS,
117+ .fds = {0},
118+ .delay_consumer = 0,
119+ .prod_run_time = 0,
120+ .strp_size = 0,
121+};
122+
123+static void bench_sockmap_prog_destroy(void)
124+{
125+ int i;
126+
127+ for (i = 0; i < sizeof(ctx.fds); i++) {
128+ if (ctx.fds[0] > 0)
129+ close(ctx.fds[i]);
130+ }
131+
132+ bench_sockmap_prog__destroy(ctx.skel);
133+}
134+
135+static void init_addr(struct sockaddr_storage *ss,
136+ socklen_t *len)
137+{
138+ struct sockaddr_in *addr4 = memset(ss, 0, sizeof(*ss));
139+
140+ addr4->sin_family = AF_INET;
141+ addr4->sin_port = 0;
142+ addr4->sin_addr.s_addr = htonl(INADDR_LOOPBACK);
143+ *len = sizeof(*addr4);
144+}
145+
146+static bool set_non_block(int fd, bool blocking)
147+{
148+ int flags = fcntl(fd, F_GETFL, 0);
149+
150+ if (flags == -1)
151+ return false;
152+ flags = blocking ? (flags | O_NONBLOCK) : (flags & ~O_NONBLOCK);
153+ return (fcntl(fd, F_SETFL, flags) == 0);
154+}
155+
156+static int create_pair(int *c, int *p, int type)
157+{
158+ struct sockaddr_storage addr;
159+ int err, cfd, pfd;
160+ socklen_t addr_len = sizeof(struct sockaddr_storage);
161+
162+ err = getsockname(ctx.sfd, (struct sockaddr *)&addr, &addr_len);
163+ if (err) {
164+ fprintf(stderr, "getsockname error %d\n", errno);
165+ return err;
166+ }
167+ cfd = socket(AF_INET, type, 0);
168+ if (cfd < 0) {
169+ fprintf(stderr, "socket error %d\n", errno);
170+ return err;
171+ }
172+
173+ err = connect(cfd, (struct sockaddr *)&addr, addr_len);
174+ if (err && errno != EINPROGRESS) {
175+ fprintf(stderr, "connect error %d\n", errno);
176+ return err;
177+ }
178+
179+ pfd = accept(ctx.sfd, NULL, NULL);
180+ if (pfd < 0) {
181+ fprintf(stderr, "accept error %d\n", errno);
182+ return err;
183+ }
184+ *c = cfd;
185+ *p = pfd;
186+ return 0;
187+}
188+
189+static int create_sockets(void)
190+{
191+ struct sockaddr_storage addr;
192+ int err, one = 1;
193+ socklen_t addr_len;
194+
195+ init_addr(&addr, &addr_len);
196+ ctx.sfd = socket(AF_INET, SOCK_STREAM, 0);
197+ if (ctx.sfd < 0) {
198+ fprintf(stderr, "socket error:%d\n", errno);
199+ return ctx.sfd;
200+ }
201+ err = setsockopt(ctx.sfd, SOL_SOCKET, SO_REUSEPORT, &one, sizeof(one));
202+ if (err) {
203+ fprintf(stderr, "setsockopt error:%d\n", errno);
204+ return err;
205+ }
206+
207+ err = bind(ctx.sfd, (struct sockaddr *)&addr, addr_len);
208+ if (err) {
209+ fprintf(stderr, "bind error:%d\n", errno);
210+ return err;
211+ }
212+
213+ err = listen(ctx.sfd, SOMAXCONN);
214+ if (err) {
215+ fprintf(stderr, "listen error:%d\n", errno);
216+ return err;
217+ }
218+
219+ err = create_pair(&ctx.c1, &ctx.p1, SOCK_STREAM);
220+ if (err) {
221+ fprintf(stderr, "create_pair 1 error\n");
222+ return err;
223+ }
224+
225+ err = create_pair(&ctx.c2, &ctx.p2, SOCK_STREAM);
226+ if (err) {
227+ fprintf(stderr, "create_pair 2 error\n");
228+ return err;
229+ }
230+ printf("create socket fd c1:%d p1:%d c2:%d p2:%d\n",
231+ ctx.c1, ctx.p1, ctx.c2, ctx.p2);
232+ return 0;
233+}
234+
235+static void validate(void)
236+{
237+ if (env.consumer_cnt != 2 || env.producer_cnt != 1 ||
238+ !env.affinity)
239+ goto err;
240+ return;
241+err:
242+ fprintf(stderr, "argument '-c 2 -p 1 -a' is necessary");
243+ exit(1);
244+}
245+
246+static int setup_rx_sockmap(void)
247+{
248+ int verdict, pass, parser, map;
249+ int zero = 0, one = 1;
250+ int err;
251+
252+ parser = bpf_program__fd(ctx.skel->progs.prog_skb_parser);
253+ verdict = bpf_program__fd(ctx.skel->progs.prog_skb_verdict);
254+ pass = bpf_program__fd(ctx.skel->progs.prog_skb_pass);
255+ map = bpf_map__fd(ctx.skel->maps.sock_map_rx);
256+
257+ if (ctx.strp_size != 0) {
258+ ctx.skel->bss->pkt_size = ctx.strp_size;
259+ err = bpf_prog_attach(parser, map, BPF_SK_SKB_STREAM_PARSER, 0);
260+ if (err)
261+ return err;
262+ }
263+
264+ if (RXMODE_BPF_VERDICT())
265+ err = bpf_prog_attach(verdict, map, BPF_SK_SKB_STREAM_VERDICT, 0);
266+ else if (RXMODE_BPF_PASS())
267+ err = bpf_prog_attach(pass, map, BPF_SK_SKB_STREAM_VERDICT, 0);
268+ if (err)
269+ return err;
270+
271+ if (RXMODE_BPF_PASS())
272+ return bpf_map_update_elem(map, &zero, &ctx.c2, BPF_NOEXIST);
273+
274+ err = bpf_map_update_elem(map, &zero, &ctx.p1, BPF_NOEXIST);
275+ if (err < 0)
276+ return err;
277+
278+ if (RXMODE_BPF_VERDICT_INGRESS()) {
279+ ctx.skel->bss->verdict_dir = BPF_F_INGRESS;
280+ err = bpf_map_update_elem(map, &one, &ctx.c2, BPF_NOEXIST);
281+ } else {
282+ err = bpf_map_update_elem(map, &one, &ctx.p2, BPF_NOEXIST);
283+ }
284+ if (err < 0)
285+ return err;
286+
287+ return 0;
288+}
289+
290+static int setup_tx_sockmap(void)
291+{
292+ int zero = 0, one = 1;
293+ int prog, map;
294+ int err;
295+
296+ map = bpf_map__fd(ctx.skel->maps.sock_map_tx);
297+ prog = TXMODE_BPF_PASS() ?
298+ bpf_program__fd(ctx.skel->progs.prog_skmsg_pass) :
299+ bpf_program__fd(ctx.skel->progs.prog_skmsg_verdict);
300+
301+ err = bpf_prog_attach(prog, map, BPF_SK_MSG_VERDICT, 0);
302+ if (err)
303+ return err;
304+
305+ if (TXMODE_BPF_EGRESS()) {
306+ err = bpf_map_update_elem(map, &zero, &ctx.p1, BPF_NOEXIST);
307+ err |= bpf_map_update_elem(map, &one, &ctx.p2, BPF_NOEXIST);
308+ } else {
309+ ctx.skel->bss->verdict_dir = BPF_F_INGRESS;
310+ err = bpf_map_update_elem(map, &zero, &ctx.p2, BPF_NOEXIST);
311+ err |= bpf_map_update_elem(map, &one, &ctx.c2, BPF_NOEXIST);
312+ }
313+
314+ if (err < 0)
315+ return err;
316+
317+ return 0;
318+}
319+
320+static void setup(void)
321+{
322+ int err;
323+
324+ ctx.skel = bench_sockmap_prog__open_and_load();
325+ if (!ctx.skel) {
326+ fprintf(stderr, "error loading skel\n");
327+ exit(1);
328+ }
329+
330+ if (create_sockets()) {
331+ fprintf(stderr, "create_net_mode error\n");
332+ goto err;
333+ }
334+
335+ if (RXMODE_BPF()) {
336+ err = setup_rx_sockmap();
337+ if (err) {
338+ fprintf(stderr, "setup_rx_sockmap error:%d\n", err);
339+ goto err;
340+ }
341+ } else if (TXMODE_BPF()) {
342+ err = setup_tx_sockmap();
343+ if (err) {
344+ fprintf(stderr, "setup_tx_sockmap error:%d\n", err);
345+ goto err;
346+ }
347+ } else {
348+ fprintf(stderr, "unknown sockmap bench mode: %d\n", ctx.mode);
349+ goto err;
350+ }
351+
352+ return;
353+
354+err:
355+ bench_sockmap_prog_destroy();
356+ exit(1);
357+}
358+
359+static void measure(struct bench_res *res)
360+{
361+ res->drops = atomic_swap(&ctx.prod_send, 0);
362+ res->hits = atomic_swap(&ctx.skel->bss->process_byte, 0);
363+ res->false_hits = atomic_swap(&ctx.user_read, 0);
364+ res->important_hits = atomic_swap(&ctx.send_calls, 0);
365+ res->important_hits |= atomic_swap(&ctx.read_calls, 0) << 32;
366+}
367+
368+static void verify_data(int *check_pos, char *buf, int rcv)
369+{
370+ for (int i = 0 ; i < rcv; i++) {
371+ if (buf[i] != snd_data[(*check_pos) % DATA_REPEAT_SIZE]) {
372+ fprintf(stderr, "verify data fail");
373+ exit(1);
374+ }
375+ (*check_pos)++;
376+ if (*check_pos >= FILE_SIZE)
377+ *check_pos = 0;
378+ }
379+}
380+
381+static void *consumer(void *input)
382+{
383+ int rcv, sent;
384+ int check_pos = 0;
385+ int tid = (long)input;
386+ int recv_buf_size = FILE_SIZE;
387+ char *buf = malloc(recv_buf_size);
388+ int delay_read = ctx.delay_consumer;
389+
390+ if (!buf) {
391+ fprintf(stderr, "fail to init read buffer");
392+ return NULL;
393+ }
394+
395+ while (true) {
396+ if (tid == 1) {
397+ /* consumer 1 is unused for tx test and stream verdict test */
398+ if (RXMODE_BPF() || TXMODE())
399+ return NULL;
400+ /* it's only for RX_NORMAL which service as reserve-proxy mode */
401+ rcv = read(ctx.p1, buf, recv_buf_size);
402+ if (rcv < 0) {
403+ fprintf(stderr, "fail to read p1");
404+ return NULL;
405+ }
406+
407+ sent = send(ctx.p2, buf, recv_buf_size, 0);
408+ if (sent < 0) {
409+ fprintf(stderr, "fail to send p2");
410+ return NULL;
411+ }
412+ } else {
413+ if (delay_read != 0) {
414+ if (delay_read < 0)
415+ return NULL;
416+ sleep(delay_read);
417+ delay_read = 0;
418+ }
419+ /* read real endpoint by consumer 0 */
420+ atomic_inc(&ctx.read_calls);
421+ rcv = read(ctx.c2, buf, recv_buf_size);
422+ if (rcv < 0 && errno != EAGAIN) {
423+ fprintf(stderr, "%s fail to read c2 %d\n", __func__, errno);
424+ return NULL;
425+ }
426+ verify_data(&check_pos, buf, rcv);
427+ atomic_add(&ctx.user_read, rcv);
428+ }
429+ }
430+
431+ return NULL;
432+}
433+
434+static void *producer(void *input)
435+{
436+ int off = 0, fp, need_sent, sent;
437+ int file_size = ctx.file_size;
438+ struct timespec ts1, ts2;
439+ int target;
440+ FILE *file;
441+
442+ file = tmpfile();
443+ if (!file) {
444+ fprintf(stderr, "create file for sendfile");
445+ return NULL;
446+ }
447+
448+ /* we need simple verify */
449+ for (int i = 0; i < file_size; i++) {
450+ if (fwrite(&snd_data[off], sizeof(char), 1, file) != 1) {
451+ fprintf(stderr, "init tmpfile error");
452+ return NULL;
453+ }
454+ if (++off >= sizeof(snd_data))
455+ off = 0;
456+ }
457+ fflush(file);
458+ fseek(file, 0, SEEK_SET);
459+
460+ fp = fileno(file);
461+ need_sent = file_size;
462+ clock_gettime(CLOCK_MONOTONIC, &ts1);
463+
464+ if (RXMODE_BPF_VERDICT())
465+ target = ctx.c1;
466+ else if (TXMODE_BPF_EGRESS())
467+ target = ctx.p1;
468+ else
469+ target = ctx.p2;
470+ set_non_block(target, true);
471+ while (true) {
472+ if (ctx.prod_run_time) {
473+ clock_gettime(CLOCK_MONOTONIC, &ts2);
474+ if (ts2.tv_sec - ts1.tv_sec > ctx.prod_run_time)
475+ return NULL;
476+ }
477+
478+ errno = 0;
479+ atomic_inc(&ctx.send_calls);
480+ sent = sendfile(target, fp, NULL, need_sent);
481+ if (sent < 0) {
482+ if (errno != EAGAIN && errno != ENOMEM && errno != ENOBUFS) {
483+ fprintf(stderr, "sendfile return %d, errorno %d:%s\n",
484+ sent, errno, strerror(errno));
485+ return NULL;
486+ }
487+ continue;
488+ } else if (sent < need_sent) {
489+ need_sent -= sent;
490+ atomic_add(&ctx.prod_send, sent);
491+ continue;
492+ }
493+ atomic_add(&ctx.prod_send, need_sent);
494+ need_sent = file_size;
495+ lseek(fp, 0, SEEK_SET);
496+ }
497+
498+ return NULL;
499+}
500+
501+static void report_progress(int iter, struct bench_res *res, long delta_ns)
502+{
503+ double speed_mbs, prod_mbs, bpf_mbs, send_hz, read_hz;
504+
505+ prod_mbs = res->drops / 1000000.0 / (delta_ns / 1000000000.0);
506+ speed_mbs = res->false_hits / 1000000.0 / (delta_ns / 1000000000.0);
507+ bpf_mbs = res->hits / 1000000.0 / (delta_ns / 1000000000.0);
508+ send_hz = (res->important_hits & 0xFFFFFFFF) / (delta_ns / 1000000000.0);
509+ read_hz = (res->important_hits >> 32) / (delta_ns / 1000000000.0);
510+
511+ printf("Iter %3d (%7.3lfus): ",
512+ iter, (delta_ns - 1000000000) / 1000.0);
513+ printf("Send Speed %8.3lf MB/s (%8.3lf calls/s), BPF Speed %8.3lf MB/s, "
514+ "Rcv Speed %8.3lf MB/s (%8.3lf calls/s)\n",
515+ prod_mbs, send_hz, bpf_mbs, speed_mbs, read_hz);
516+}
517+
518+static void report_final(struct bench_res res[], int res_cnt)
519+{
520+ double verdict_mbs_mean = 0.0;
521+ long verdict_total = 0;
522+ int i;
523+
524+ for (i = 0; i < res_cnt; i++) {
525+ verdict_mbs_mean += res[i].hits / 1000000.0 / (0.0 + res_cnt);
526+ verdict_total += res[i].hits / 1000000.0;
527+ }
528+
529+ printf("Summary: total trans %8.3lu MB \u00B1 %5.3lf MB/s\n",
530+ verdict_total, verdict_mbs_mean);
531+}
532+
533+static const struct argp_option opts[] = {
534+ { "rx-normal", ARG_FW_RX_NORMAL, NULL, 0,
535+ "simple reserve-proxy mode, no bfp enabled"},
536+ { "rx-pass", ARG_FW_RX_PASS, NULL, 0,
537+ "run bpf prog but no redir applied"},
538+ { "rx-strp", ARG_CTL_RX_STRP, "Byte", 0,
539+ "enable strparser and set the encapsulation size"},
540+ { "rx-verdict-egress", ARG_FW_RX_VERDICT_EGRESS, NULL, 0,
541+ "forward data with bpf(stream verdict)"},
542+ { "rx-verdict-ingress", ARG_FW_RX_VERDICT_INGRESS, NULL, 0,
543+ "forward data with bpf(stream verdict)"},
544+ { "tx-normal", ARG_FW_TX_NORMAL, NULL, 0,
545+ "simple c-s mode, no bfp enabled"},
546+ { "tx-pass", ARG_FW_TX_PASS, NULL, 0,
547+ "run bpf prog but no redir applied"},
548+ { "tx-verdict-ingress", ARG_FW_TX_VERDICT_INGRESS, NULL, 0,
549+ "forward msg to ingress queue of another socket"},
550+ { "tx-verdict-egress", ARG_FW_TX_VERDICT_EGRESS, NULL, 0,
551+ "forward msg to egress queue of another socket"},
552+ { "delay-consumer", ARG_CONSUMER_DELAY_TIME, "SEC", 0,
553+ "delay consumer start"},
554+ { "producer-duration", ARG_PRODUCER_DURATION, "SEC", 0,
555+ "producer duration"},
556+ {},
557+};
558+
559+static error_t parse_arg(int key, char *arg, struct argp_state *state)
560+{
561+ switch (key) {
562+ case ARG_FW_RX_NORMAL...ARG_FW_TX_VERDICT_EGRESS:
563+ ctx.mode = key;
564+ break;
565+ case ARG_CONSUMER_DELAY_TIME:
566+ ctx.delay_consumer = strtol(arg, NULL, 10);
567+ break;
568+ case ARG_PRODUCER_DURATION:
569+ ctx.prod_run_time = strtol(arg, NULL, 10);
570+ break;
571+ case ARG_CTL_RX_STRP:
572+ ctx.strp_size = strtol(arg, NULL, 10);
573+ break;
574+ default:
575+ return ARGP_ERR_UNKNOWN;
576+ }
577+
578+ return 0;
579+}
580+
581+/* exported into benchmark runner */
582+const struct argp bench_sockmap_argp = {
583+ .options = opts,
584+ .parser = parse_arg,
585+};
586+
587+/* Benchmark performance of creating bpf local storage */
588+const struct bench bench_sockmap = {
589+ .name = "sockmap",
590+ .argp = &bench_sockmap_argp,
591+ .validate = validate,
592+ .setup = setup,
593+ .producer_thread = producer,
594+ .consumer_thread = consumer,
595+ .measure = measure,
596+ .report_progress = report_progress,
597+ .report_final = report_final,
598+};
tools/testing/selftests/bpf/bpf_experimental.h+5−0
@@ -591,4 +591,9 @@ extern int bpf_iter_kmem_cache_new(struct bpf_iter_kmem_cache *it) __weak __ksym · 590 unmodified lines
591591 extern struct kmem_cache *bpf_iter_kmem_cache_next(struct bpf_iter_kmem_cache *it) __weak __ksym;
592592 extern void bpf_iter_kmem_cache_destroy(struct bpf_iter_kmem_cache *it) __weak __ksym;
593593
594+struct bpf_iter_dmabuf;
595+extern int bpf_iter_dmabuf_new(struct bpf_iter_dmabuf *it) __weak __ksym;
596+extern struct dma_buf *bpf_iter_dmabuf_next(struct bpf_iter_dmabuf *it) __weak __ksym;
597+extern void bpf_iter_dmabuf_destroy(struct bpf_iter_dmabuf *it) __weak __ksym;
598+
594599 #endif
tools/testing/selftests/bpf/config+3−0
@@ -22,6 +22,8 @@ CONFIG_CRYPTO_AES=y · 21 unmodified lines
2222 CONFIG_DEBUG_INFO=y
2323 CONFIG_DEBUG_INFO_BTF=y
2424 CONFIG_DEBUG_INFO_DWARF4=y
25+CONFIG_DMABUF_HEAPS=y
26+CONFIG_DMABUF_HEAPS_SYSTEM=y
2527 CONFIG_DUMMY=y
2628 CONFIG_DYNAMIC_FTRACE=y
2729 CONFIG_FPROBE=y
@@ -108,6 +110,7 @@ CONFIG_SECURITY=y · 80 unmodified lines
108110 CONFIG_SECURITYFS=y
109111 CONFIG_SYN_COOKIES=y
110112 CONFIG_TEST_BPF=m
113+CONFIG_UDMABUF=y
111114 CONFIG_USERFAULTFD=y
112115 CONFIG_VSOCKETS=y
113116 CONFIG_VXLAN=y
tools/testing/selftests/bpf/prog_tests/arena_spin_lock.c+8−6
@@ -51,9 +51,11 @@ static void test_arena_spin_lock_size(int size) · 50 unmodified lines
5151 struct arena_spin_lock *skel;
5252 pthread_t thread_id[16];
5353 int prog_fd, i, err;
54+ int nthreads;
5455 void *ret;
5556
56− if (get_nprocs() < 2) {
57+ nthreads = MIN(get_nprocs(), ARRAY_SIZE(thread_id));
58+ if (nthreads < 2) {
5759 test__skip();
5860 return;
5961 }
@@ -66,25 +68,25 @@ static void test_arena_spin_lock_size(int size) · 6 unmodified lines
6668 goto end;
6769 }
6870 skel->bss->cs_count = size;
69− skel->bss->limit = repeat * 16;
71+ skel->bss->limit = repeat * nthreads;
7072
71− ASSERT_OK(pthread_barrier_init(&barrier, NULL, 16), "barrier init");
73+ ASSERT_OK(pthread_barrier_init(&barrier, NULL, nthreads), "barrier init");
7274
7375 prog_fd = bpf_program__fd(skel->progs.prog);
74− for (i = 0; i < 16; i++) {
76+ for (i = 0; i < nthreads; i++) {
7577 err = pthread_create(&thread_id[i], NULL, &spin_lock_thread, &prog_fd);
7678 if (!ASSERT_OK(err, "pthread_create"))
7779 goto end_barrier;
7880 }
7981
80− for (i = 0; i < 16; i++) {
82+ for (i = 0; i < nthreads; i++) {
8183 if (!ASSERT_OK(pthread_join(thread_id[i], &ret), "pthread_join"))
8284 goto end_barrier;
8385 if (!ASSERT_EQ(ret, &prog_fd, "ret == prog_fd"))
8486 goto end_barrier;
8587 }
8688
87− ASSERT_EQ(skel->bss->counter, repeat * 16, "check counter value");
89+ ASSERT_EQ(skel->bss->counter, repeat * nthreads, "check counter value");
8890
8991 end_barrier:
9092 pthread_barrier_destroy(&barrier);
tools/testing/selftests/bpf/prog_tests/attach_probe.c+84−0
@@ -122,6 +122,85 @@ cleanup: · 121 unmodified lines
122122 test_attach_probe_manual__destroy(skel);
123123 }
124124
125+/* attach uprobe/uretprobe long event name testings */
126+static void test_attach_uprobe_long_event_name(void)
127+{
128+ DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, uprobe_opts);
129+ struct bpf_link *uprobe_link, *uretprobe_link;
130+ struct test_attach_probe_manual *skel;
131+ ssize_t uprobe_offset;
132+ char path[PATH_MAX] = {0};
133+
134+ skel = test_attach_probe_manual__open_and_load();
135+ if (!ASSERT_OK_PTR(skel, "skel_kprobe_manual_open_and_load"))
136+ return;
137+
138+ uprobe_offset = get_uprobe_offset(&trigger_func);
139+ if (!ASSERT_GE(uprobe_offset, 0, "uprobe_offset"))
140+ goto cleanup;
141+
142+ if (!ASSERT_GT(readlink("/proc/self/exe", path, PATH_MAX - 1), 0, "readlink"))
143+ goto cleanup;
144+
145+ /* manual-attach uprobe/uretprobe */
146+ uprobe_opts.attach_mode = PROBE_ATTACH_MODE_LEGACY;
147+ uprobe_opts.ref_ctr_offset = 0;
148+ uprobe_opts.retprobe = false;
149+ uprobe_link = bpf_program__attach_uprobe_opts(skel->progs.handle_uprobe,
150+ 0 /* self pid */,
151+ path,
152+ uprobe_offset,
153+ &uprobe_opts);
154+ if (!ASSERT_OK_PTR(uprobe_link, "attach_uprobe_long_event_name"))
155+ goto cleanup;
156+ skel->links.handle_uprobe = uprobe_link;
157+
158+ uprobe_opts.retprobe = true;
159+ uretprobe_link = bpf_program__attach_uprobe_opts(skel->progs.handle_uretprobe,
160+ -1 /* any pid */,
161+ path,
162+ uprobe_offset, &uprobe_opts);
163+ if (!ASSERT_OK_PTR(uretprobe_link, "attach_uretprobe_long_event_name"))
164+ goto cleanup;
165+ skel->links.handle_uretprobe = uretprobe_link;
166+
167+cleanup:
168+ test_attach_probe_manual__destroy(skel);
169+}
170+
171+/* attach kprobe/kretprobe long event name testings */
172+static void test_attach_kprobe_long_event_name(void)
173+{
174+ DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, kprobe_opts);
175+ struct bpf_link *kprobe_link, *kretprobe_link;
176+ struct test_attach_probe_manual *skel;
177+
178+ skel = test_attach_probe_manual__open_and_load();
179+ if (!ASSERT_OK_PTR(skel, "skel_kprobe_manual_open_and_load"))
180+ return;
181+
182+ /* manual-attach kprobe/kretprobe */
183+ kprobe_opts.attach_mode = PROBE_ATTACH_MODE_LEGACY;
184+ kprobe_opts.retprobe = false;
185+ kprobe_link = bpf_program__attach_kprobe_opts(skel->progs.handle_kprobe,
186+ "bpf_testmod_looooooooooooooooooooooooooooooong_name",
187+ &kprobe_opts);
188+ if (!ASSERT_OK_PTR(kprobe_link, "attach_kprobe_long_event_name"))
189+ goto cleanup;
190+ skel->links.handle_kprobe = kprobe_link;
191+
192+ kprobe_opts.retprobe = true;
193+ kretprobe_link = bpf_program__attach_kprobe_opts(skel->progs.handle_kretprobe,
194+ "bpf_testmod_looooooooooooooooooooooooooooooong_name",
195+ &kprobe_opts);
196+ if (!ASSERT_OK_PTR(kretprobe_link, "attach_kretprobe_long_event_name"))
197+ goto cleanup;
198+ skel->links.handle_kretprobe = kretprobe_link;
199+
200+cleanup:
201+ test_attach_probe_manual__destroy(skel);
202+}
203+
125204 static void test_attach_probe_auto(struct test_attach_probe *skel)
126205 {
127206 struct bpf_link *uprobe_err_link;
@@ -323,6 +402,11 @@ void test_attach_probe(void) · 195 unmodified lines
323402 if (test__start_subtest("uprobe-ref_ctr"))
324403 test_uprobe_ref_ctr(skel);
325404
405+ if (test__start_subtest("uprobe-long_name"))
406+ test_attach_uprobe_long_event_name();
407+ if (test__start_subtest("kprobe-long_name"))
408+ test_attach_kprobe_long_event_name();
409+
326410 cleanup:
327411 test_attach_probe__destroy(skel);
328412 ASSERT_EQ(uprobe_ref_ctr, 0, "uprobe_ref_ctr_cleanup");
tools/testing/selftests/bpf/prog_tests/bpf_nf.c+6−0
@@ -63,6 +63,12 @@ static void test_bpf_nf_ct(int mode) · 62 unmodified lines
6363 .repeat = 1,
6464 );
6565
66+ if (SYS_NOFAIL("iptables-legacy --version")) {
67+ fprintf(stdout, "Missing required iptables-legacy tool\n");
68+ test__skip();
69+ return;
70+ }
71+
6672 skel = test_bpf_nf__open_and_load();
6773 if (!ASSERT_OK_PTR(skel, "test_bpf_nf__open_and_load"))
6874 return;
tools/testing/selftests/bpf/prog_tests/btf_dedup_split.c+101−0
@@ -440,6 +440,105 @@ cleanup: · 439 unmodified lines
440440 btf__free(btf1);
441441 }
442442
443+/* Ensure module split BTF dedup worked correctly; when dedup fails badly
444+ * core kernel types are in split BTF also, so ensure that references to
445+ * such types point at base - not split - BTF.
446+ *
447+ * bpf_testmod_test_write() has multiple core kernel type parameters;
448+ *
449+ * ssize_t
450+ * bpf_testmod_test_write(struct file *file, struct kobject *kobj,
451+ * struct bin_attribute *bin_attr,
452+ * char *buf, loff_t off, size_t len);
453+ *
454+ * Ensure each of the FUNC_PROTO params is a core kernel type.
455+ *
456+ * Do the same for
457+ *
458+ * __bpf_kfunc struct sock *bpf_kfunc_call_test3(struct sock *sk);
459+ *
460+ * ...and
461+ *
462+ * __bpf_kfunc void bpf_kfunc_call_test_pass_ctx(struct __sk_buff *skb);
463+ *
464+ */
465+const char *mod_funcs[] = {
466+ "bpf_testmod_test_write",
467+ "bpf_kfunc_call_test3",
468+ "bpf_kfunc_call_test_pass_ctx"
469+};
470+
471+static void test_split_module(void)
472+{
473+ struct btf *vmlinux_btf, *btf1 = NULL;
474+ int i, nr_base_types;
475+
476+ vmlinux_btf = btf__load_vmlinux_btf();
477+ if (!ASSERT_OK_PTR(vmlinux_btf, "vmlinux_btf"))
478+ return;
479+ nr_base_types = btf__type_cnt(vmlinux_btf);
480+ if (!ASSERT_GT(nr_base_types, 0, "nr_base_types"))
481+ goto cleanup;
482+
483+ btf1 = btf__parse_split("/sys/kernel/btf/bpf_testmod", vmlinux_btf);
484+ if (!ASSERT_OK_PTR(btf1, "split_btf"))
485+ return;
486+
487+ for (i = 0; i < ARRAY_SIZE(mod_funcs); i++) {
488+ const struct btf_param *p;
489+ const struct btf_type *t;
490+ __u16 vlen;
491+ __u32 id;
492+ int j;
493+
494+ id = btf__find_by_name_kind(btf1, mod_funcs[i], BTF_KIND_FUNC);
495+ if (!ASSERT_GE(id, nr_base_types, "func_id"))
496+ goto cleanup;
497+ t = btf__type_by_id(btf1, id);
498+ if (!ASSERT_OK_PTR(t, "func_id_type"))
499+ goto cleanup;
500+ t = btf__type_by_id(btf1, t->type);
501+ if (!ASSERT_OK_PTR(t, "func_proto_id_type"))
502+ goto cleanup;
503+ if (!ASSERT_EQ(btf_is_func_proto(t), true, "is_func_proto"))
504+ goto cleanup;
505+ vlen = btf_vlen(t);
506+
507+ for (j = 0, p = btf_params(t); j < vlen; j++, p++) {
508+ /* bpf_testmod uses resilient split BTF, so any
509+ * reference types will be added to split BTF and their
510+ * associated targets will be base BTF types; for example
511+ * for a "struct sock *" the PTR will be in split BTF
512+ * while the "struct sock" will be in base.
513+ *
514+ * In some cases like loff_t we have to resolve
515+ * multiple typedefs hence the while() loop below.
516+ *
517+ * Note that resilient split BTF generation depends
518+ * on pahole version, so we do not assert that
519+ * reference types are in split BTF, as if pahole
520+ * does not support resilient split BTF they will
521+ * also be base BTF types.
522+ */
523+ id = p->type;
524+ do {
525+ t = btf__type_by_id(btf1, id);
526+ if (!ASSERT_OK_PTR(t, "param_ref_type"))
527+ goto cleanup;
528+ if (!btf_is_mod(t) && !btf_is_ptr(t) && !btf_is_typedef(t))
529+ break;
530+ id = t->type;
531+ } while (true);
532+
533+ if (!ASSERT_LT(id, nr_base_types, "verify_base_type"))
534+ goto cleanup;
535+ }
536+ }
537+cleanup:
538+ btf__free(btf1);
539+ btf__free(vmlinux_btf);
540+}
541+
443542 void test_btf_dedup_split()
444543 {
445544 if (test__start_subtest("split_simple"))
@@ -450,4 +549,6 @@ void test_btf_dedup_split() · 4 unmodified lines
450549 test_split_fwd_resolve();
451550 if (test__start_subtest("split_dup_struct_in_cu"))
452551 test_split_dup_struct_in_cu();
552+ if (test__start_subtest("split_module"))
553+ test_split_module();
453554 }
tools/testing/selftests/bpf/prog_tests/btf_split.c+52−6
@@ -12,10 +12,11 @@ static void btf_dump_printf(void *ctx, const char *fmt, va_list args) · 11 unmodified lines
1212 vfprintf(ctx, fmt, args);
1313 }
1414
15−void test_btf_split() {
15+static void __test_btf_split(bool multi)
16+{
1617 struct btf_dump *d = NULL;
1718 const struct btf_type *t;
18− struct btf *btf1, *btf2;
19+ struct btf *btf1, *btf2, *btf3 = NULL;
1920 int str_off, i, err;
2021
2122 btf1 = btf__new_empty();
@@ -63,14 +64,46 @@ void test_btf_split() { · 41 unmodified lines
6364 ASSERT_EQ(btf_vlen(t), 3, "split_struct_vlen");
6465 ASSERT_STREQ(btf__str_by_offset(btf2, t->name_off), "s2", "split_struct_name");
6566
67+ if (multi) {
68+ btf3 = btf__new_empty_split(btf2);
69+ if (!ASSERT_OK_PTR(btf3, "multi_split_btf"))
70+ goto cleanup;
71+ } else {
72+ btf3 = btf2;
73+ }
74+
75+ btf__add_union(btf3, "u1", 16); /* [5] union u1 { */
76+ btf__add_field(btf3, "f1", 4, 0, 0); /* struct s2 f1; */
77+ btf__add_field(btf3, "uf2", 1, 0, 0); /* int f2; */
78+ /* } */
79+
80+ if (multi) {
81+ t = btf__type_by_id(btf2, 5);
82+ ASSERT_NULL(t, "multisplit_type_in_first_split");
83+ }
84+
85+ t = btf__type_by_id(btf3, 5);
86+ if (!ASSERT_OK_PTR(t, "split_union_type"))
87+ goto cleanup;
88+ ASSERT_EQ(btf_is_union(t), true, "split_union_kind");
89+ ASSERT_EQ(btf_vlen(t), 2, "split_union_vlen");
90+ ASSERT_STREQ(btf__str_by_offset(btf3, t->name_off), "u1", "split_union_name");
91+ ASSERT_EQ(btf__type_cnt(btf3), 6, "split_type_cnt");
92+
93+ t = btf__type_by_id(btf3, 1);
94+ if (!ASSERT_OK_PTR(t, "split_base_type"))
95+ goto cleanup;
96+ ASSERT_EQ(btf_is_int(t), true, "split_base_int");
97+ ASSERT_STREQ(btf__str_by_offset(btf3, t->name_off), "int", "split_base_type_name");
98+
6699 /* BTF-to-C dump of split BTF */
67100 dump_buf_file = open_memstream(&dump_buf, &dump_buf_sz);
68101 if (!ASSERT_OK_PTR(dump_buf_file, "dump_memstream"))
69102 return;
70− d = btf_dump__new(btf2, btf_dump_printf, dump_buf_file, NULL);
103+ d = btf_dump__new(btf3, btf_dump_printf, dump_buf_file, NULL);
71104 if (!ASSERT_OK_PTR(d, "btf_dump__new"))
72105 goto cleanup;
73− for (i = 1; i < btf__type_cnt(btf2); i++) {
106+ for (i = 1; i < btf__type_cnt(btf3); i++) {
74107 err = btf_dump__dump_type(d, i);
75108 ASSERT_OK(err, "dump_type_ok");
76109 }
@@ -79,12 +112,15 @@ void test_btf_split() { · 2 unmodified lines
79112 ASSERT_STREQ(dump_buf,
80113 "struct s1 {\n"
81114 " int f1;\n"
82−"};\n"
83−"\n"
115+"};\n\n"
84116 "struct s2 {\n"
85117 " struct s1 f1;\n"
86118 " int f2;\n"
87119 " int *f3;\n"
120+"};\n\n"
121+"union u1 {\n"
122+" struct s2 f1;\n"
123+" int uf2;\n"
88124 "};\n\n", "c_dump");
89125
90126 cleanup:
@@ -94,4 +130,14 @@ cleanup: · 3 unmodified lines
94130 btf_dump__free(d);
95131 btf__free(btf1);
96132 btf__free(btf2);
133+ if (btf2 != btf3)
134+ btf__free(btf3);
135+}
136+
137+void test_btf_split(void)
138+{
139+ if (test__start_subtest("single_split"))
140+ __test_btf_split(false);
141+ if (test__start_subtest("multi_split"))
142+ __test_btf_split(true);
97143 }
tools/testing/selftests/bpf/prog_tests/btf_sysfs.c+81−0
@@ -0,0 +1,81 @@
1+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2+/* Copyright (c) 2025 Isovalent */
3+
4+#include <test_progs.h>
5+#include <bpf/btf.h>
6+#include <sys/stat.h>
7+#include <sys/mman.h>
8+#include <fcntl.h>
9+#include <unistd.h>
10+
11+static void test_btf_mmap_sysfs(const char *path, struct btf *base)
12+{
13+ struct stat st;
14+ __u64 btf_size, end;
15+ void *raw_data = NULL;
16+ int fd = -1;
17+ long page_size;
18+ struct btf *btf = NULL;
19+
20+ page_size = sysconf(_SC_PAGESIZE);
21+ if (!ASSERT_GE(page_size, 0, "get_page_size"))
22+ goto cleanup;
23+
24+ if (!ASSERT_OK(stat(path, &st), "stat_btf"))
25+ goto cleanup;
26+
27+ btf_size = st.st_size;
28+ end = (btf_size + page_size - 1) / page_size * page_size;
29+
30+ fd = open(path, O_RDONLY);
31+ if (!ASSERT_GE(fd, 0, "open_btf"))
32+ goto cleanup;
33+
34+ raw_data = mmap(NULL, btf_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
35+ if (!ASSERT_EQ(raw_data, MAP_FAILED, "mmap_btf_writable"))
36+ goto cleanup;
37+
38+ raw_data = mmap(NULL, btf_size, PROT_READ, MAP_SHARED, fd, 0);
39+ if (!ASSERT_EQ(raw_data, MAP_FAILED, "mmap_btf_shared"))
40+ goto cleanup;
41+
42+ raw_data = mmap(NULL, end + 1, PROT_READ, MAP_PRIVATE, fd, 0);
43+ if (!ASSERT_EQ(raw_data, MAP_FAILED, "mmap_btf_invalid_size"))
44+ goto cleanup;
45+
46+ raw_data = mmap(NULL, end, PROT_READ, MAP_PRIVATE, fd, 0);
47+ if (!ASSERT_OK_PTR(raw_data, "mmap_btf"))
48+ goto cleanup;
49+
50+ if (!ASSERT_EQ(mprotect(raw_data, btf_size, PROT_READ | PROT_WRITE), -1,
51+ "mprotect_writable"))
52+ goto cleanup;
53+
54+ if (!ASSERT_EQ(mprotect(raw_data, btf_size, PROT_READ | PROT_EXEC), -1,
55+ "mprotect_executable"))
56+ goto cleanup;
57+
58+ /* Check padding is zeroed */
59+ for (int i = btf_size; i < end; i++) {
60+ if (((__u8 *)raw_data)[i] != 0) {
61+ PRINT_FAIL("tail of BTF is not zero at page offset %d\n", i);
62+ goto cleanup;
63+ }
64+ }
65+
66+ btf = btf__new_split(raw_data, btf_size, base);
67+ if (!ASSERT_OK_PTR(btf, "parse_btf"))
68+ goto cleanup;
69+
70+cleanup:
71+ btf__free(btf);
72+ if (raw_data && raw_data != MAP_FAILED)
73+ munmap(raw_data, btf_size);
74+ if (fd >= 0)
75+ close(fd);
76+}
77+
78+void test_btf_sysfs(void)
79+{
80+ test_btf_mmap_sysfs("/sys/kernel/btf/vmlinux", NULL);
81+}
tools/testing/selftests/bpf/prog_tests/dmabuf_iter.c+285−0
@@ -0,0 +1,285 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/* Copyright (c) 2025 Google */
3+
4+#include <test_progs.h>
5+#include <bpf/libbpf.h>
6+#include <bpf/btf.h>
7+#include "dmabuf_iter.skel.h"
8+
9+#include <fcntl.h>
10+#include <stdbool.h>
11+#include <stdio.h>
12+#include <stdlib.h>
13+#include <string.h>
14+#include <sys/ioctl.h>
15+#include <sys/mman.h>
16+#include <unistd.h>
17+
18+#include <linux/dma-buf.h>
19+#include <linux/dma-heap.h>
20+#include <linux/udmabuf.h>
21+
22+static int udmabuf = -1;
23+static const char udmabuf_test_buffer_name[DMA_BUF_NAME_LEN] = "udmabuf_test_buffer_for_iter";
24+static size_t udmabuf_test_buffer_size;
25+static int sysheap_dmabuf = -1;
26+static const char sysheap_test_buffer_name[DMA_BUF_NAME_LEN] = "sysheap_test_buffer_for_iter";
27+static size_t sysheap_test_buffer_size;
28+
29+static int create_udmabuf(void)
30+{
31+ struct udmabuf_create create;
32+ int dev_udmabuf, memfd, local_udmabuf;
33+
34+ udmabuf_test_buffer_size = 10 * getpagesize();
35+
36+ if (!ASSERT_LE(sizeof(udmabuf_test_buffer_name), DMA_BUF_NAME_LEN, "NAMETOOLONG"))
37+ return -1;
38+
39+ memfd = memfd_create("memfd_test", MFD_ALLOW_SEALING);
40+ if (!ASSERT_OK_FD(memfd, "memfd_create"))
41+ return -1;
42+
43+ if (!ASSERT_OK(ftruncate(memfd, udmabuf_test_buffer_size), "ftruncate"))
44+ goto close_memfd;
45+
46+ if (!ASSERT_OK(fcntl(memfd, F_ADD_SEALS, F_SEAL_SHRINK), "seal"))
47+ goto close_memfd;
48+
49+ dev_udmabuf = open("/dev/udmabuf", O_RDONLY);
50+ if (!ASSERT_OK_FD(dev_udmabuf, "open udmabuf"))
51+ goto close_memfd;
52+
53+ memset(&create, 0, sizeof(create));
54+ create.memfd = memfd;
55+ create.flags = UDMABUF_FLAGS_CLOEXEC;
56+ create.offset = 0;
57+ create.size = udmabuf_test_buffer_size;
58+
59+ local_udmabuf = ioctl(dev_udmabuf, UDMABUF_CREATE, &create);
60+ close(dev_udmabuf);
61+ if (!ASSERT_OK_FD(local_udmabuf, "udmabuf_create"))
62+ goto close_memfd;
63+
64+ if (!ASSERT_OK(ioctl(local_udmabuf, DMA_BUF_SET_NAME_B, udmabuf_test_buffer_name), "name"))
65+ goto close_udmabuf;
66+
67+ return local_udmabuf;
68+
69+close_udmabuf:
70+ close(local_udmabuf);
71+close_memfd:
72+ close(memfd);
73+ return -1;
74+}
75+
76+static int create_sys_heap_dmabuf(void)
77+{
78+ sysheap_test_buffer_size = 20 * getpagesize();
79+
80+ struct dma_heap_allocation_data data = {
81+ .len = sysheap_test_buffer_size,
82+ .fd = 0,
83+ .fd_flags = O_RDWR | O_CLOEXEC,
84+ .heap_flags = 0,
85+ };
86+ int heap_fd, ret;
87+
88+ if (!ASSERT_LE(sizeof(sysheap_test_buffer_name), DMA_BUF_NAME_LEN, "NAMETOOLONG"))
89+ return -1;
90+
91+ heap_fd = open("/dev/dma_heap/system", O_RDONLY);
92+ if (!ASSERT_OK_FD(heap_fd, "open dma heap"))
93+ return -1;
94+
95+ ret = ioctl(heap_fd, DMA_HEAP_IOCTL_ALLOC, &data);
96+ close(heap_fd);
97+ if (!ASSERT_OK(ret, "syheap alloc"))
98+ return -1;
99+
100+ if (!ASSERT_OK(ioctl(data.fd, DMA_BUF_SET_NAME_B, sysheap_test_buffer_name), "name"))
101+ goto close_sysheap_dmabuf;
102+
103+ return data.fd;
104+
105+close_sysheap_dmabuf:
106+ close(data.fd);
107+ return -1;
108+}
109+
110+static int create_test_buffers(void)
111+{
112+ udmabuf = create_udmabuf();
113+ sysheap_dmabuf = create_sys_heap_dmabuf();
114+
115+ if (udmabuf < 0 || sysheap_dmabuf < 0)
116+ return -1;
117+
118+ return 0;
119+}
120+
121+static void destroy_test_buffers(void)
122+{
123+ close(udmabuf);
124+ udmabuf = -1;
125+
126+ close(sysheap_dmabuf);
127+ sysheap_dmabuf = -1;
128+}
129+
130+enum Fields { INODE, SIZE, NAME, EXPORTER, FIELD_COUNT };
131+struct DmabufInfo {
132+ unsigned long inode;
133+ unsigned long size;
134+ char name[DMA_BUF_NAME_LEN];
135+ char exporter[32];
136+};
137+
138+static bool check_dmabuf_info(const struct DmabufInfo *bufinfo,
139+ unsigned long size,
140+ const char *name, const char *exporter)
141+{
142+ return size == bufinfo->size &&
143+ !strcmp(name, bufinfo->name) &&
144+ !strcmp(exporter, bufinfo->exporter);
145+}
146+
147+static void subtest_dmabuf_iter_check_no_infinite_reads(struct dmabuf_iter *skel)
148+{
149+ int iter_fd;
150+ char buf[256];
151+
152+ iter_fd = bpf_iter_create(bpf_link__fd(skel->links.dmabuf_collector));
153+ if (!ASSERT_OK_FD(iter_fd, "iter_create"))
154+ return;
155+
156+ while (read(iter_fd, buf, sizeof(buf)) > 0)
157+ ; /* Read out all contents */
158+
159+ /* Next reads should return 0 */
160+ ASSERT_EQ(read(iter_fd, buf, sizeof(buf)), 0, "read");
161+
162+ close(iter_fd);
163+}
164+
165+static void subtest_dmabuf_iter_check_default_iter(struct dmabuf_iter *skel)
166+{
167+ bool found_test_sysheap_dmabuf = false;
168+ bool found_test_udmabuf = false;
169+ struct DmabufInfo bufinfo;
170+ size_t linesize = 0;
171+ char *line = NULL;
172+ FILE *iter_file;
173+ int iter_fd, f = INODE;
174+
175+ iter_fd = bpf_iter_create(bpf_link__fd(skel->links.dmabuf_collector));
176+ if (!ASSERT_OK_FD(iter_fd, "iter_create"))
177+ return;
178+
179+ iter_file = fdopen(iter_fd, "r");
180+ if (!ASSERT_OK_PTR(iter_file, "fdopen"))
181+ goto close_iter_fd;
182+
183+ while (getline(&line, &linesize, iter_file) != -1) {
184+ if (f % FIELD_COUNT == INODE) {
185+ ASSERT_EQ(sscanf(line, "%ld", &bufinfo.inode), 1,
186+ "read inode");
187+ } else if (f % FIELD_COUNT == SIZE) {
188+ ASSERT_EQ(sscanf(line, "%ld", &bufinfo.size), 1,
189+ "read size");
190+ } else if (f % FIELD_COUNT == NAME) {
191+ ASSERT_EQ(sscanf(line, "%s", bufinfo.name), 1,
192+ "read name");
193+ } else if (f % FIELD_COUNT == EXPORTER) {
194+ ASSERT_EQ(sscanf(line, "%31s", bufinfo.exporter), 1,
195+ "read exporter");
196+
197+ if (check_dmabuf_info(&bufinfo,
198+ sysheap_test_buffer_size,
199+ sysheap_test_buffer_name,
200+ "system"))
201+ found_test_sysheap_dmabuf = true;
202+ else if (check_dmabuf_info(&bufinfo,
203+ udmabuf_test_buffer_size,
204+ udmabuf_test_buffer_name,
205+ "udmabuf"))
206+ found_test_udmabuf = true;
207+ }
208+ ++f;
209+ }
210+
211+ ASSERT_EQ(f % FIELD_COUNT, INODE, "number of fields");
212+
213+ ASSERT_TRUE(found_test_sysheap_dmabuf, "found_test_sysheap_dmabuf");
214+ ASSERT_TRUE(found_test_udmabuf, "found_test_udmabuf");
215+
216+ free(line);
217+ fclose(iter_file);
218+close_iter_fd:
219+ close(iter_fd);
220+}
221+
222+static void subtest_dmabuf_iter_check_open_coded(struct dmabuf_iter *skel, int map_fd)
223+{
224+ LIBBPF_OPTS(bpf_test_run_opts, topts);
225+ char key[DMA_BUF_NAME_LEN];
226+ int err, fd;
227+ bool found;
228+
229+ /* No need to attach it, just run it directly */
230+ fd = bpf_program__fd(skel->progs.iter_dmabuf_for_each);
231+
232+ err = bpf_prog_test_run_opts(fd, &topts);
233+ if (!ASSERT_OK(err, "test_run_opts err"))
234+ return;
235+ if (!ASSERT_OK(topts.retval, "test_run_opts retval"))
236+ return;
237+
238+ if (!ASSERT_OK(bpf_map_get_next_key(map_fd, NULL, key), "get next key"))
239+ return;
240+
241+ do {
242+ ASSERT_OK(bpf_map_lookup_elem(map_fd, key, &found), "lookup");
243+ ASSERT_TRUE(found, "found test buffer");
244+ } while (bpf_map_get_next_key(map_fd, key, key));
245+}
246+
247+void test_dmabuf_iter(void)
248+{
249+ struct dmabuf_iter *skel = NULL;
250+ int map_fd;
251+ const bool f = false;
252+
253+ skel = dmabuf_iter__open_and_load();
254+ if (!ASSERT_OK_PTR(skel, "dmabuf_iter__open_and_load"))
255+ return;
256+
257+ map_fd = bpf_map__fd(skel->maps.testbuf_hash);
258+ if (!ASSERT_OK_FD(map_fd, "map_fd"))
259+ goto destroy_skel;
260+
261+ if (!ASSERT_OK(bpf_map_update_elem(map_fd, udmabuf_test_buffer_name, &f, BPF_ANY),
262+ "insert udmabuf"))
263+ goto destroy_skel;
264+ if (!ASSERT_OK(bpf_map_update_elem(map_fd, sysheap_test_buffer_name, &f, BPF_ANY),
265+ "insert sysheap buffer"))
266+ goto destroy_skel;
267+
268+ if (!ASSERT_OK(create_test_buffers(), "create_test_buffers"))
269+ goto destroy;
270+
271+ if (!ASSERT_OK(dmabuf_iter__attach(skel), "skel_attach"))
272+ goto destroy;
273+
274+ if (test__start_subtest("no_infinite_reads"))
275+ subtest_dmabuf_iter_check_no_infinite_reads(skel);
276+ if (test__start_subtest("default_iter"))
277+ subtest_dmabuf_iter_check_default_iter(skel);
278+ if (test__start_subtest("open_coded"))
279+ subtest_dmabuf_iter_check_open_coded(skel, map_fd);
280+
281+destroy:
282+ destroy_test_buffers();
283+destroy_skel:
284+ dmabuf_iter__destroy(skel);
285+}
tools/testing/selftests/bpf/prog_tests/dynptr.c+13−0
@@ -33,10 +33,19 @@ static struct { · 32 unmodified lines
3333 {"test_dynptr_skb_no_buff", SETUP_SKB_PROG},
3434 {"test_dynptr_skb_strcmp", SETUP_SKB_PROG},
3535 {"test_dynptr_skb_tp_btf", SETUP_SKB_PROG_TP},
36+ {"test_probe_read_user_dynptr", SETUP_XDP_PROG},
37+ {"test_probe_read_kernel_dynptr", SETUP_XDP_PROG},
38+ {"test_probe_read_user_str_dynptr", SETUP_XDP_PROG},
39+ {"test_probe_read_kernel_str_dynptr", SETUP_XDP_PROG},
40+ {"test_copy_from_user_dynptr", SETUP_SYSCALL_SLEEP},
41+ {"test_copy_from_user_str_dynptr", SETUP_SYSCALL_SLEEP},
42+ {"test_copy_from_user_task_dynptr", SETUP_SYSCALL_SLEEP},
43+ {"test_copy_from_user_task_str_dynptr", SETUP_SYSCALL_SLEEP},
3644 };
3745
3846 static void verify_success(const char *prog_name, enum test_setup_type setup_type)
3947 {
48+ char user_data[384] = {[0 ... 382] = 'a', '\0'};
4049 struct dynptr_success *skel;
4150 struct bpf_program *prog;
4251 struct bpf_link *link;
@@ -58,6 +67,10 @@ static void verify_success(const char *prog_name, enum test_setup_type setup_typ · 15 unmodified lines
5867 if (!ASSERT_OK(err, "dynptr_success__load"))
5968 goto cleanup;
6069
70+ skel->bss->user_ptr = user_data;
71+ skel->data->test_len[0] = sizeof(user_data);
72+ memcpy(skel->bss->expected_str, user_data, sizeof(user_data));
73+
6174 switch (setup_type) {
6275 case SETUP_SYSCALL_SLEEP:
6376 link = bpf_program__attach(prog);
tools/testing/selftests/bpf/prog_tests/fd_htab_lookup.c+192−0
@@ -0,0 +1,192 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/* Copyright (C) 2025. Huawei Technologies Co., Ltd */
3+#define _GNU_SOURCE
4+#include <stdbool.h>
5+#include <test_progs.h>
6+#include "fd_htab_lookup.skel.h"
7+
8+struct htab_op_ctx {
9+ int fd;
10+ int loop;
11+ unsigned int entries;
12+ bool stop;
13+};
14+
15+#define ERR_TO_RETVAL(where, err) ((void *)(long)(((where) << 12) | (-err)))
16+
17+static void *htab_lookup_fn(void *arg)
18+{
19+ struct htab_op_ctx *ctx = arg;
20+ int i = 0;
21+
22+ while (i++ < ctx->loop && !ctx->stop) {
23+ unsigned int j;
24+
25+ for (j = 0; j < ctx->entries; j++) {
26+ unsigned int key = j, zero = 0, value;
27+ int inner_fd, err;
28+
29+ err = bpf_map_lookup_elem(ctx->fd, &key, &value);
30+ if (err) {
31+ ctx->stop = true;
32+ return ERR_TO_RETVAL(1, err);
33+ }
34+
35+ inner_fd = bpf_map_get_fd_by_id(value);
36+ if (inner_fd < 0) {
37+ /* The old map has been freed */
38+ if (inner_fd == -ENOENT)
39+ continue;
40+ ctx->stop = true;
41+ return ERR_TO_RETVAL(2, inner_fd);
42+ }
43+
44+ err = bpf_map_lookup_elem(inner_fd, &zero, &value);
45+ if (err) {
46+ close(inner_fd);
47+ ctx->stop = true;
48+ return ERR_TO_RETVAL(3, err);
49+ }
50+ close(inner_fd);
51+
52+ if (value != key) {
53+ ctx->stop = true;
54+ return ERR_TO_RETVAL(4, -EINVAL);
55+ }
56+ }
57+ }
58+
59+ return NULL;
60+}
61+
62+static void *htab_update_fn(void *arg)
63+{
64+ struct htab_op_ctx *ctx = arg;
65+ int i = 0;
66+
67+ while (i++ < ctx->loop && !ctx->stop) {
68+ unsigned int j;
69+
70+ for (j = 0; j < ctx->entries; j++) {
71+ unsigned int key = j, zero = 0;
72+ int inner_fd, err;
73+
74+ inner_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, 4, 4, 1, NULL);
75+ if (inner_fd < 0) {
76+ ctx->stop = true;
77+ return ERR_TO_RETVAL(1, inner_fd);
78+ }
79+
80+ err = bpf_map_update_elem(inner_fd, &zero, &key, 0);
81+ if (err) {
82+ close(inner_fd);
83+ ctx->stop = true;
84+ return ERR_TO_RETVAL(2, err);
85+ }
86+
87+ err = bpf_map_update_elem(ctx->fd, &key, &inner_fd, BPF_EXIST);
88+ if (err) {
89+ close(inner_fd);
90+ ctx->stop = true;
91+ return ERR_TO_RETVAL(3, err);
92+ }
93+ close(inner_fd);
94+ }
95+ }
96+
97+ return NULL;
98+}
99+
100+static int setup_htab(int fd, unsigned int entries)
101+{
102+ unsigned int i;
103+
104+ for (i = 0; i < entries; i++) {
105+ unsigned int key = i, zero = 0;
106+ int inner_fd, err;
107+
108+ inner_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, 4, 4, 1, NULL);
109+ if (!ASSERT_OK_FD(inner_fd, "new array"))
110+ return -1;
111+
112+ err = bpf_map_update_elem(inner_fd, &zero, &key, 0);
113+ if (!ASSERT_OK(err, "init array")) {
114+ close(inner_fd);
115+ return -1;
116+ }
117+
118+ err = bpf_map_update_elem(fd, &key, &inner_fd, 0);
119+ if (!ASSERT_OK(err, "init outer")) {
120+ close(inner_fd);
121+ return -1;
122+ }
123+ close(inner_fd);
124+ }
125+
126+ return 0;
127+}
128+
129+static int get_int_from_env(const char *name, int dft)
130+{
131+ const char *value;
132+
133+ value = getenv(name);
134+ if (!value)
135+ return dft;
136+
137+ return atoi(value);
138+}
139+
140+void test_fd_htab_lookup(void)
141+{
142+ unsigned int i, wr_nr = 8, rd_nr = 16;
143+ pthread_t tids[wr_nr + rd_nr];
144+ struct fd_htab_lookup *skel;
145+ struct htab_op_ctx ctx;
146+ int err;
147+
148+ skel = fd_htab_lookup__open_and_load();
149+ if (!ASSERT_OK_PTR(skel, "fd_htab_lookup__open_and_load"))
150+ return;
151+
152+ ctx.fd = bpf_map__fd(skel->maps.outer_map);
153+ ctx.loop = get_int_from_env("FD_HTAB_LOOP_NR", 5);
154+ ctx.stop = false;
155+ ctx.entries = 8;
156+
157+ err = setup_htab(ctx.fd, ctx.entries);
158+ if (err)
159+ goto destroy;
160+
161+ memset(tids, 0, sizeof(tids));
162+ for (i = 0; i < wr_nr; i++) {
163+ err = pthread_create(&tids[i], NULL, htab_update_fn, &ctx);
164+ if (!ASSERT_OK(err, "pthread_create")) {
165+ ctx.stop = true;
166+ goto reap;
167+ }
168+ }
169+ for (i = 0; i < rd_nr; i++) {
170+ err = pthread_create(&tids[i + wr_nr], NULL, htab_lookup_fn, &ctx);
171+ if (!ASSERT_OK(err, "pthread_create")) {
172+ ctx.stop = true;
173+ goto reap;
174+ }
175+ }
176+
177+reap:
178+ for (i = 0; i < wr_nr + rd_nr; i++) {
179+ void *ret = NULL;
180+ char desc[32];
181+
182+ if (!tids[i])
183+ continue;
184+
185+ snprintf(desc, sizeof(desc), "thread %u", i + 1);
186+ err = pthread_join(tids[i], &ret);
187+ ASSERT_OK(err, desc);
188+ ASSERT_EQ(ret, NULL, desc);
189+ }
190+destroy:
191+ fd_htab_lookup__destroy(skel);
192+}
tools/testing/selftests/bpf/prog_tests/fill_link_info.c+16−2
@@ -37,6 +37,7 @@ static noinline void uprobe_func(void) · 36 unmodified lines
3737 static int verify_perf_link_info(int fd, enum bpf_perf_event_type type, long addr,
3838 ssize_t offset, ssize_t entry_offset)
3939 {
40+ ssize_t ref_ctr_offset = entry_offset /* ref_ctr_offset for uprobes */;
4041 struct bpf_link_info info;
4142 __u32 len = sizeof(info);
4243 char buf[PATH_MAX];
@@ -97,6 +98,7 @@ again: · 54 unmodified lines
9798 case BPF_PERF_EVENT_UPROBE:
9899 case BPF_PERF_EVENT_URETPROBE:
99100 ASSERT_EQ(info.perf_event.uprobe.offset, offset, "uprobe_offset");
101+ ASSERT_EQ(info.perf_event.uprobe.ref_ctr_offset, ref_ctr_offset, "uprobe_ref_ctr_offset");
100102
101103 ASSERT_EQ(info.perf_event.uprobe.name_len, strlen(UPROBE_FILE) + 1,
102104 "name_len");
@@ -241,20 +243,32 @@ static void test_uprobe_fill_link_info(struct test_fill_link_info *skel, · 138 unmodified lines
241243 .retprobe = type == BPF_PERF_EVENT_URETPROBE,
242244 .bpf_cookie = PERF_EVENT_COOKIE,
243245 );
246+ const char *sema[1] = {
247+ "uprobe_link_info_sema_1",
248+ };
249+ __u64 *ref_ctr_offset;
244250 struct bpf_link *link;
245251 int link_fd, err;
246252
253+ err = elf_resolve_syms_offsets("/proc/self/exe", 1, sema,
254+ (unsigned long **) &ref_ctr_offset, STT_OBJECT);
255+ if (!ASSERT_OK(err, "elf_resolve_syms_offsets_object"))
256+ return;
257+
258+ opts.ref_ctr_offset = *ref_ctr_offset;
247259 link = bpf_program__attach_uprobe_opts(skel->progs.uprobe_run,
248260 0, /* self pid */
249261 UPROBE_FILE, uprobe_offset,
250262 &opts);
251263 if (!ASSERT_OK_PTR(link, "attach_uprobe"))
252− return;
264+ goto out;
253265
254266 link_fd = bpf_link__fd(link);
255− err = verify_perf_link_info(link_fd, type, 0, uprobe_offset, 0);
267+ err = verify_perf_link_info(link_fd, type, 0, uprobe_offset, *ref_ctr_offset);
256268 ASSERT_OK(err, "verify_perf_link_info");
257269 bpf_link__destroy(link);
270+out:
271+ free(ref_ctr_offset);
258272 }
259273
260274 static int verify_kmulti_link_info(int fd, bool retprobe, bool has_cookies)
tools/testing/selftests/bpf/prog_tests/kmem_cache_iter.c+1−1
@@ -104,7 +104,7 @@ void test_kmem_cache_iter(void) · 103 unmodified lines
104104 goto destroy;
105105
106106 memset(buf, 0, sizeof(buf));
107− while (read(iter_fd, buf, sizeof(buf) > 0)) {
107+ while (read(iter_fd, buf, sizeof(buf)) > 0) {
108108 /* Read out all contents */
109109 printf("%s", buf);
110110 }
tools/testing/selftests/bpf/prog_tests/linked_list.c+6−0
@@ -7,6 +7,7 @@ · 6 unmodified lines
77
88 #include "linked_list.skel.h"
99 #include "linked_list_fail.skel.h"
10+#include "linked_list_peek.skel.h"
1011
1112 static char log_buf[1024 * 1024];
1213
@@ -805,3 +806,8 @@ void test_linked_list(void) · 792 unmodified lines
805806 test_linked_list_success(LIST_IN_LIST, true);
806807 test_linked_list_success(TEST_ALL, false);
807808 }
809+
810+void test_linked_list_peek(void)
811+{
812+ RUN_TESTS(linked_list_peek);
813+}
tools/testing/selftests/bpf/prog_tests/rbtree.c+6−0
@@ -8,6 +8,7 @@ · 7 unmodified lines
88 #include "rbtree_fail.skel.h"
99 #include "rbtree_btf_fail__wrong_node_type.skel.h"
1010 #include "rbtree_btf_fail__add_wrong_type.skel.h"
11+#include "rbtree_search.skel.h"
1112
1213 static void test_rbtree_add_nodes(void)
1314 {
@@ -187,3 +188,8 @@ void test_rbtree_fail(void) · 173 unmodified lines
187188 {
188189 RUN_TESTS(rbtree_fail);
189190 }
191+
192+void test_rbtree_search(void)
193+{
194+ RUN_TESTS(rbtree_search);
195+}
tools/testing/selftests/bpf/prog_tests/sk_assign.c+3−1
@@ -37,8 +37,10 @@ configure_stack(void) · 36 unmodified lines
3737 tc = popen("tc -V", "r");
3838 if (CHECK_FAIL(!tc))
3939 return false;
40− if (CHECK_FAIL(!fgets(tc_version, sizeof(tc_version), tc)))
40+ if (CHECK_FAIL(!fgets(tc_version, sizeof(tc_version), tc))) {
41+ pclose(tc);
4142 return false;
43+ }
4244 if (strstr(tc_version, ", libbpf "))
4345 prog = "test_sk_assign_libbpf.bpf.o";
4446 else
tools/testing/selftests/bpf/prog_tests/socket_helpers.h+79−5
@@ -3,6 +3,7 @@ · 2 unmodified lines
33 #ifndef __SOCKET_HELPERS__
44 #define __SOCKET_HELPERS__
55
6+#include <sys/un.h>
67 #include <linux/vm_sockets.h>
78
89 /* include/linux/net.h */
@@ -169,6 +170,15 @@ static inline void init_addr_loopback6(struct sockaddr_storage *ss, · 160 unmodified lines
169170 *len = sizeof(*addr6);
170171 }
171172
173+static inline void init_addr_loopback_unix(struct sockaddr_storage *ss,
174+ socklen_t *len)
175+{
176+ struct sockaddr_un *addr = memset(ss, 0, sizeof(*ss));
177+
178+ addr->sun_family = AF_UNIX;
179+ *len = sizeof(sa_family_t);
180+}
181+
172182 static inline void init_addr_loopback_vsock(struct sockaddr_storage *ss,
173183 socklen_t *len)
174184 {
@@ -190,6 +200,9 @@ static inline void init_addr_loopback(int family, struct sockaddr_storage *ss, · 15 unmodified lines
190200 case AF_INET6:
191201 init_addr_loopback6(ss, len);
192202 return;
203+ case AF_UNIX:
204+ init_addr_loopback_unix(ss, len);
205+ return;
193206 case AF_VSOCK:
194207 init_addr_loopback_vsock(ss, len);
195208 return;
@@ -315,21 +328,27 @@ static inline int create_pair(int family, int sotype, int *p0, int *p1) · 119 unmodified lines
315328 {
316329 __close_fd int s, c = -1, p = -1;
317330 struct sockaddr_storage addr;
318− socklen_t len = sizeof(addr);
331+ socklen_t len;
319332 int err;
320333
321334 s = socket_loopback(family, sotype);
322335 if (s < 0)
323336 return s;
324337
325− err = xgetsockname(s, sockaddr(&addr), &len);
326− if (err)
327− return err;
328−
329338 c = xsocket(family, sotype, 0);
330339 if (c < 0)
331340 return c;
332341
342+ init_addr_loopback(family, &addr, &len);
343+ err = xbind(c, sockaddr(&addr), len);
344+ if (err)
345+ return err;
346+
347+ len = sizeof(addr);
348+ err = xgetsockname(s, sockaddr(&addr), &len);
349+ if (err)
350+ return err;
351+
333352 err = connect(c, sockaddr(&addr), len);
334353 if (err) {
335354 if (errno != EINPROGRESS) {
@@ -391,4 +410,59 @@ static inline int create_socket_pairs(int family, int sotype, int *c0, int *c1, · 55 unmodified lines
391410 return err;
392411 }
393412
413+static inline const char *socket_kind_to_str(int sock_fd)
414+{
415+ socklen_t opt_len;
416+ int domain, type;
417+
418+ opt_len = sizeof(domain);
419+ if (getsockopt(sock_fd, SOL_SOCKET, SO_DOMAIN, &domain, &opt_len))
420+ FAIL_ERRNO("getsockopt(SO_DOMAIN)");
421+
422+ opt_len = sizeof(type);
423+ if (getsockopt(sock_fd, SOL_SOCKET, SO_TYPE, &type, &opt_len))
424+ FAIL_ERRNO("getsockopt(SO_TYPE)");
425+
426+ switch (domain) {
427+ case AF_INET:
428+ switch (type) {
429+ case SOCK_STREAM:
430+ return "tcp4";
431+ case SOCK_DGRAM:
432+ return "udp4";
433+ }
434+ break;
435+ case AF_INET6:
436+ switch (type) {
437+ case SOCK_STREAM:
438+ return "tcp6";
439+ case SOCK_DGRAM:
440+ return "udp6";
441+ }
442+ break;
443+ case AF_UNIX:
444+ switch (type) {
445+ case SOCK_STREAM:
446+ return "u_str";
447+ case SOCK_DGRAM:
448+ return "u_dgr";
449+ case SOCK_SEQPACKET:
450+ return "u_seq";
451+ }
452+ break;
453+ case AF_VSOCK:
454+ switch (type) {
455+ case SOCK_STREAM:
456+ return "v_str";
457+ case SOCK_DGRAM:
458+ return "v_dgr";
459+ case SOCK_SEQPACKET:
460+ return "v_seq";
461+ }
462+ break;
463+ }
464+
465+ return "???";
466+}
467+
394468 #endif // __SOCKET_HELPERS__
tools/testing/selftests/bpf/prog_tests/sockmap_helpers.h+11−14
@@ -5,12 +5,15 @@ · 4 unmodified lines
55
66 #define MAX_TEST_NAME 80
77
8+#define u32(v) ((u32){(v)})
9+#define u64(v) ((u64){(v)})
10+
811 #define __always_unused __attribute__((__unused__))
912
1013 #define xbpf_map_delete_elem(fd, key) \
1114 ({ \
1215 int __ret = bpf_map_delete_elem((fd), (key)); \
13− if (__ret < 0) \
16+ if (__ret < 0) \
1417 FAIL_ERRNO("map_delete"); \
1518 __ret; \
1619 })
@@ -18,7 +21,7 @@ · 1 unmodified line
1821 #define xbpf_map_lookup_elem(fd, key, val) \
1922 ({ \
2023 int __ret = bpf_map_lookup_elem((fd), (key), (val)); \
21− if (__ret < 0) \
24+ if (__ret < 0) \
2225 FAIL_ERRNO("map_lookup"); \
2326 __ret; \
2427 })
@@ -26,7 +29,7 @@ · 1 unmodified line
2629 #define xbpf_map_update_elem(fd, key, val, flags) \
2730 ({ \
2831 int __ret = bpf_map_update_elem((fd), (key), (val), (flags)); \
29− if (__ret < 0) \
32+ if (__ret < 0) \
3033 FAIL_ERRNO("map_update"); \
3134 __ret; \
3235 })
@@ -35,7 +38,7 @@ · 2 unmodified lines
3538 ({ \
3639 int __ret = \
3740 bpf_prog_attach((prog), (target), (type), (flags)); \
38− if (__ret < 0) \
41+ if (__ret < 0) \
3942 FAIL_ERRNO("prog_attach(" #type ")"); \
4043 __ret; \
4144 })
@@ -43,7 +46,7 @@ · 1 unmodified line
4346 #define xbpf_prog_detach2(prog, target, type) \
4447 ({ \
4548 int __ret = bpf_prog_detach2((prog), (target), (type)); \
46− if (__ret < 0) \
49+ if (__ret < 0) \
4750 FAIL_ERRNO("prog_detach2(" #type ")"); \
4851 __ret; \
4952 })
@@ -66,21 +69,15 @@ · 16 unmodified lines
6669 __ret; \
6770 })
6871
69−static inline int add_to_sockmap(int sock_mapfd, int fd1, int fd2)
72+static inline int add_to_sockmap(int mapfd, int fd1, int fd2)
7073 {
71− u64 value;
72− u32 key;
7374 int err;
7475
75− key = 0;
76− value = fd1;
77− err = xbpf_map_update_elem(sock_mapfd, &key, &value, BPF_NOEXIST);
76+ err = xbpf_map_update_elem(mapfd, &u32(0), &u64(fd1), BPF_NOEXIST);
7877 if (err)
7978 return err;
8079
81− key = 1;
82− value = fd2;
83− return xbpf_map_update_elem(sock_mapfd, &key, &value, BPF_NOEXIST);
80+ return xbpf_map_update_elem(mapfd, &u32(1), &u64(fd2), BPF_NOEXIST);
8481 }
8582
8683 #endif // __SOCKMAP_HELPERS__
tools/testing/selftests/bpf/prog_tests/sockmap_ktls.c+239−58
@@ -3,76 +3,62 @@ · 2 unmodified lines
33 /*
44 * Tests for sockmap/sockhash holding kTLS sockets.
55 */
6−
6+#include <error.h>
77 #include <netinet/tcp.h>
8+#include <linux/tls.h>
89 #include "test_progs.h"
10+#include "sockmap_helpers.h"
11+#include "test_skmsg_load_helpers.skel.h"
12+#include "test_sockmap_ktls.skel.h"
913
1014 #define MAX_TEST_NAME 80
1115 #define TCP_ULP 31
1216
13−static int tcp_server(int family)
17+static int init_ktls_pairs(int c, int p)
1418 {
15− int err, s;
16−
17− s = socket(family, SOCK_STREAM, 0);
18− if (!ASSERT_GE(s, 0, "socket"))
19− return -1;
20−
21− err = listen(s, SOMAXCONN);
22− if (!ASSERT_OK(err, "listen"))
23− return -1;
24−
25− return s;
26−}
19+ int err;
20+ struct tls12_crypto_info_aes_gcm_128 crypto_rx;
21+ struct tls12_crypto_info_aes_gcm_128 crypto_tx;
2722
28−static int disconnect(int fd)
29−{
30− struct sockaddr unspec = { AF_UNSPEC };
23+ err = setsockopt(c, IPPROTO_TCP, TCP_ULP, "tls", strlen("tls"));
24+ if (!ASSERT_OK(err, "setsockopt(TCP_ULP)"))
25+ goto out;
3126
32− return connect(fd, &unspec, sizeof(unspec));
27+ err = setsockopt(p, IPPROTO_TCP, TCP_ULP, "tls", strlen("tls"));
28+ if (!ASSERT_OK(err, "setsockopt(TCP_ULP)"))
29+ goto out;
30+
31+ memset(&crypto_rx, 0, sizeof(crypto_rx));
32+ memset(&crypto_tx, 0, sizeof(crypto_tx));
33+ crypto_rx.info.version = TLS_1_2_VERSION;
34+ crypto_tx.info.version = TLS_1_2_VERSION;
35+ crypto_rx.info.cipher_type = TLS_CIPHER_AES_GCM_128;
36+ crypto_tx.info.cipher_type = TLS_CIPHER_AES_GCM_128;
37+
38+ err = setsockopt(c, SOL_TLS, TLS_TX, &crypto_tx, sizeof(crypto_tx));
39+ if (!ASSERT_OK(err, "setsockopt(TLS_TX)"))
40+ goto out;
41+
42+ err = setsockopt(p, SOL_TLS, TLS_RX, &crypto_rx, sizeof(crypto_rx));
43+ if (!ASSERT_OK(err, "setsockopt(TLS_RX)"))
44+ goto out;
45+ return 0;
46+out:
47+ return -1;
3348 }
3449
35−/* Disconnect (unhash) a kTLS socket after removing it from sockmap. */
36−static void test_sockmap_ktls_disconnect_after_delete(int family, int map)
50+static int create_ktls_pairs(int family, int sotype, int *c, int *p)
3751 {
38− struct sockaddr_storage addr = {0};
39− socklen_t len = sizeof(addr);
40− int err, cli, srv, zero = 0;
41−
42− srv = tcp_server(family);
43− if (srv == -1)
44− return;
45−
46− err = getsockname(srv, (struct sockaddr *)&addr, &len);
47− if (!ASSERT_OK(err, "getsockopt"))
48− goto close_srv;
52+ int err;
4953
50− cli = socket(family, SOCK_STREAM, 0);
51− if (!ASSERT_GE(cli, 0, "socket"))
52− goto close_srv;
53−
54− err = connect(cli, (struct sockaddr *)&addr, len);
55− if (!ASSERT_OK(err, "connect"))
56− goto close_cli;
57−
58− err = bpf_map_update_elem(map, &zero, &cli, 0);
59− if (!ASSERT_OK(err, "bpf_map_update_elem"))
60− goto close_cli;
61−
62− err = setsockopt(cli, IPPROTO_TCP, TCP_ULP, "tls", strlen("tls"));
63− if (!ASSERT_OK(err, "setsockopt(TCP_ULP)"))
64− goto close_cli;
65−
66− err = bpf_map_delete_elem(map, &zero);
67− if (!ASSERT_OK(err, "bpf_map_delete_elem"))
68− goto close_cli;
69−
70− err = disconnect(cli);
54+ err = create_pair(family, sotype, c, p);
55+ if (!ASSERT_OK(err, "create_pair()"))
56+ return -1;
7157
72−close_cli:
73− close(cli);
74−close_srv:
75− close(srv);
58+ err = init_ktls_pairs(*c, *p);
59+ if (!ASSERT_OK(err, "init_ktls_pairs(c, p)"))
60+ return -1;
61+ return 0;
7662 }
7763
7864 static void test_sockmap_ktls_update_fails_when_sock_has_ulp(int family, int map)
@@ -145,6 +131,189 @@ static const char *fmt_test_name(const char *subtest_name, int family, · 66 unmodified lines
145131 return test_name;
146132 }
147133
134+static void test_sockmap_ktls_offload(int family, int sotype)
135+{
136+ int err;
137+ int c = 0, p = 0, sent, recvd;
138+ char msg[12] = "hello world\0";
139+ char rcv[13];
140+
141+ err = create_ktls_pairs(family, sotype, &c, &p);
142+ if (!ASSERT_OK(err, "create_ktls_pairs()"))
143+ goto out;
144+
145+ sent = send(c, msg, sizeof(msg), 0);
146+ if (!ASSERT_OK(err, "send(msg)"))
147+ goto out;
148+
149+ recvd = recv(p, rcv, sizeof(rcv), 0);
150+ if (!ASSERT_OK(err, "recv(msg)") ||
151+ !ASSERT_EQ(recvd, sent, "length mismatch"))
152+ goto out;
153+
154+ ASSERT_OK(memcmp(msg, rcv, sizeof(msg)), "data mismatch");
155+
156+out:
157+ if (c)
158+ close(c);
159+ if (p)
160+ close(p);
161+}
162+
163+static void test_sockmap_ktls_tx_cork(int family, int sotype, bool push)
164+{
165+ int err, off;
166+ int i, j;
167+ int start_push = 0, push_len = 0;
168+ int c = 0, p = 0, one = 1, sent, recvd;
169+ int prog_fd, map_fd;
170+ char msg[12] = "hello world\0";
171+ char rcv[20] = {0};
172+ struct test_sockmap_ktls *skel;
173+
174+ skel = test_sockmap_ktls__open_and_load();
175+ if (!ASSERT_TRUE(skel, "open ktls skel"))
176+ return;
177+
178+ err = create_pair(family, sotype, &c, &p);
179+ if (!ASSERT_OK(err, "create_pair()"))
180+ goto out;
181+
182+ prog_fd = bpf_program__fd(skel->progs.prog_sk_policy);
183+ map_fd = bpf_map__fd(skel->maps.sock_map);
184+
185+ err = bpf_prog_attach(prog_fd, map_fd, BPF_SK_MSG_VERDICT, 0);
186+ if (!ASSERT_OK(err, "bpf_prog_attach sk msg"))
187+ goto out;
188+
189+ err = bpf_map_update_elem(map_fd, &one, &c, BPF_NOEXIST);
190+ if (!ASSERT_OK(err, "bpf_map_update_elem(c)"))
191+ goto out;
192+
193+ err = init_ktls_pairs(c, p);
194+ if (!ASSERT_OK(err, "init_ktls_pairs(c, p)"))
195+ goto out;
196+
197+ skel->bss->cork_byte = sizeof(msg);
198+ if (push) {
199+ start_push = 1;
200+ push_len = 2;
201+ }
202+ skel->bss->push_start = start_push;
203+ skel->bss->push_end = push_len;
204+
205+ off = sizeof(msg) / 2;
206+ sent = send(c, msg, off, 0);
207+ if (!ASSERT_EQ(sent, off, "send(msg)"))
208+ goto out;
209+
210+ recvd = recv_timeout(p, rcv, sizeof(rcv), MSG_DONTWAIT, 1);
211+ if (!ASSERT_EQ(-1, recvd, "expected no data"))
212+ goto out;
213+
214+ /* send remaining msg */
215+ sent = send(c, msg + off, sizeof(msg) - off, 0);
216+ if (!ASSERT_EQ(sent, sizeof(msg) - off, "send remaining data"))
217+ goto out;
218+
219+ recvd = recv_timeout(p, rcv, sizeof(rcv), MSG_DONTWAIT, 1);
220+ if (!ASSERT_OK(err, "recv(msg)") ||
221+ !ASSERT_EQ(recvd, sizeof(msg) + push_len, "check length mismatch"))
222+ goto out;
223+
224+ for (i = 0, j = 0; i < recvd;) {
225+ /* skip checking the data that has been pushed in */
226+ if (i >= start_push && i <= start_push + push_len - 1) {
227+ i++;
228+ continue;
229+ }
230+ if (!ASSERT_EQ(rcv[i], msg[j], "data mismatch"))
231+ goto out;
232+ i++;
233+ j++;
234+ }
235+out:
236+ if (c)
237+ close(c);
238+ if (p)
239+ close(p);
240+ test_sockmap_ktls__destroy(skel);
241+}
242+
243+static void test_sockmap_ktls_tx_no_buf(int family, int sotype, bool push)
244+{
245+ int c = -1, p = -1, one = 1, two = 2;
246+ struct test_sockmap_ktls *skel;
247+ unsigned char *data = NULL;
248+ struct msghdr msg = {0};
249+ struct iovec iov[2];
250+ int prog_fd, map_fd;
251+ int txrx_buf = 1024;
252+ int iov_length = 8192;
253+ int err;
254+
255+ skel = test_sockmap_ktls__open_and_load();
256+ if (!ASSERT_TRUE(skel, "open ktls skel"))
257+ return;
258+
259+ err = create_pair(family, sotype, &c, &p);
260+ if (!ASSERT_OK(err, "create_pair()"))
261+ goto out;
262+
263+ err = setsockopt(c, SOL_SOCKET, SO_RCVBUFFORCE, &txrx_buf, sizeof(int));
264+ err |= setsockopt(p, SOL_SOCKET, SO_SNDBUFFORCE, &txrx_buf, sizeof(int));
265+ if (!ASSERT_OK(err, "set buf limit"))
266+ goto out;
267+
268+ prog_fd = bpf_program__fd(skel->progs.prog_sk_policy_redir);
269+ map_fd = bpf_map__fd(skel->maps.sock_map);
270+
271+ err = bpf_prog_attach(prog_fd, map_fd, BPF_SK_MSG_VERDICT, 0);
272+ if (!ASSERT_OK(err, "bpf_prog_attach sk msg"))
273+ goto out;
274+
275+ err = bpf_map_update_elem(map_fd, &one, &c, BPF_NOEXIST);
276+ if (!ASSERT_OK(err, "bpf_map_update_elem(c)"))
277+ goto out;
278+
279+ err = bpf_map_update_elem(map_fd, &two, &p, BPF_NOEXIST);
280+ if (!ASSERT_OK(err, "bpf_map_update_elem(p)"))
281+ goto out;
282+
283+ skel->bss->apply_bytes = 1024;
284+
285+ err = init_ktls_pairs(c, p);
286+ if (!ASSERT_OK(err, "init_ktls_pairs(c, p)"))
287+ goto out;
288+
289+ data = calloc(iov_length, sizeof(char));
290+ if (!data)
291+ goto out;
292+
293+ iov[0].iov_base = data;
294+ iov[0].iov_len = iov_length;
295+ iov[1].iov_base = data;
296+ iov[1].iov_len = iov_length;
297+ msg.msg_iov = iov;
298+ msg.msg_iovlen = 2;
299+
300+ for (;;) {
301+ err = sendmsg(c, &msg, MSG_DONTWAIT);
302+ if (err <= 0)
303+ break;
304+ }
305+
306+out:
307+ if (data)
308+ free(data);
309+ if (c != -1)
310+ close(c);
311+ if (p != -1)
312+ close(p);
313+
314+ test_sockmap_ktls__destroy(skel);
315+}
316+
148317 static void run_tests(int family, enum bpf_map_type map_type)
149318 {
150319 int map;
@@ -153,18 +322,30 @@ static void run_tests(int family, enum bpf_map_type map_type) · 2 unmodified lines
153322 if (!ASSERT_GE(map, 0, "bpf_map_create"))
154323 return;
155324
156− if (test__start_subtest(fmt_test_name("disconnect_after_delete", family, map_type)))
157− test_sockmap_ktls_disconnect_after_delete(family, map);
158325 if (test__start_subtest(fmt_test_name("update_fails_when_sock_has_ulp", family, map_type)))
159326 test_sockmap_ktls_update_fails_when_sock_has_ulp(family, map);
160327
161328 close(map);
162329 }
163330
331+static void run_ktls_test(int family, int sotype)
332+{
333+ if (test__start_subtest("tls simple offload"))
334+ test_sockmap_ktls_offload(family, sotype);
335+ if (test__start_subtest("tls tx cork"))
336+ test_sockmap_ktls_tx_cork(family, sotype, false);
337+ if (test__start_subtest("tls tx cork with push"))
338+ test_sockmap_ktls_tx_cork(family, sotype, true);
339+ if (test__start_subtest("tls tx egress with no buf"))
340+ test_sockmap_ktls_tx_no_buf(family, sotype, true);
341+}
342+
164343 void test_sockmap_ktls(void)
165344 {
166345 run_tests(AF_INET, BPF_MAP_TYPE_SOCKMAP);
167346 run_tests(AF_INET, BPF_MAP_TYPE_SOCKHASH);
168347 run_tests(AF_INET6, BPF_MAP_TYPE_SOCKMAP);
169348 run_tests(AF_INET6, BPF_MAP_TYPE_SOCKHASH);
349+ run_ktls_test(AF_INET, SOCK_STREAM);
350+ run_ktls_test(AF_INET6, SOCK_STREAM);
170351 }
tools/testing/selftests/bpf/prog_tests/sockmap_listen.c+0−457
@@ -1366,237 +1366,6 @@ static void test_redir(struct test_sockmap_listen *skel, struct bpf_map *map, · 1365 unmodified lines
13661366 }
13671367 }
13681368
1369−static void pairs_redir_to_connected(int cli0, int peer0, int cli1, int peer1,
1370− int sock_mapfd, int nop_mapfd,
1371− int verd_mapfd, enum redir_mode mode,
1372− int send_flags)
1373−{
1374− const char *log_prefix = redir_mode_str(mode);
1375− unsigned int pass;
1376− int err, n;
1377− u32 key;
1378− char b;
1379−
1380− zero_verdict_count(verd_mapfd);
1381−
1382− err = add_to_sockmap(sock_mapfd, peer0, peer1);
1383− if (err)
1384− return;
1385−
1386− if (nop_mapfd >= 0) {
1387− err = add_to_sockmap(nop_mapfd, cli0, cli1);
1388− if (err)
1389− return;
1390− }
1391−
1392− /* Last byte is OOB data when send_flags has MSG_OOB bit set */
1393− n = xsend(cli1, "ab", 2, send_flags);
1394− if (n >= 0 && n < 2)
1395− FAIL("%s: incomplete send", log_prefix);
1396− if (n < 2)
1397− return;
1398−
1399− key = SK_PASS;
1400− err = xbpf_map_lookup_elem(verd_mapfd, &key, &pass);
1401− if (err)
1402− return;
1403− if (pass != 1)
1404− FAIL("%s: want pass count 1, have %d", log_prefix, pass);
1405−
1406− n = recv_timeout(mode == REDIR_INGRESS ? peer0 : cli0, &b, 1, 0, IO_TIMEOUT_SEC);
1407− if (n < 0)
1408− FAIL_ERRNO("%s: recv_timeout", log_prefix);
1409− if (n == 0)
1410− FAIL("%s: incomplete recv", log_prefix);
1411−
1412− if (send_flags & MSG_OOB) {
1413− /* Check that we can't read OOB while in sockmap */
1414− errno = 0;
1415− n = recv(peer1, &b, 1, MSG_OOB | MSG_DONTWAIT);
1416− if (n != -1 || errno != EOPNOTSUPP)
1417− FAIL("%s: recv(MSG_OOB): expected EOPNOTSUPP: retval=%d errno=%d",
1418− log_prefix, n, errno);
1419−
1420− /* Remove peer1 from sockmap */
1421− xbpf_map_delete_elem(sock_mapfd, &(int){ 1 });
1422−
1423− /* Check that OOB was dropped on redirect */
1424− errno = 0;
1425− n = recv(peer1, &b, 1, MSG_OOB | MSG_DONTWAIT);
1426− if (n != -1 || errno != EINVAL)
1427− FAIL("%s: recv(MSG_OOB): expected EINVAL: retval=%d errno=%d",
1428− log_prefix, n, errno);
1429− }
1430−}
1431−
1432−static void unix_redir_to_connected(int sotype, int sock_mapfd,
1433− int verd_mapfd, enum redir_mode mode)
1434−{
1435− int c0, c1, p0, p1;
1436− int sfd[2];
1437−
1438− if (socketpair(AF_UNIX, sotype | SOCK_NONBLOCK, 0, sfd))
1439− return;
1440− c0 = sfd[0], p0 = sfd[1];
1441−
1442− if (socketpair(AF_UNIX, sotype | SOCK_NONBLOCK, 0, sfd))
1443− goto close0;
1444− c1 = sfd[0], p1 = sfd[1];
1445−
1446− pairs_redir_to_connected(c0, p0, c1, p1, sock_mapfd, -1, verd_mapfd,
1447− mode, NO_FLAGS);
1448−
1449− xclose(c1);
1450− xclose(p1);
1451−close0:
1452− xclose(c0);
1453− xclose(p0);
1454−}
1455−
1456−static void unix_skb_redir_to_connected(struct test_sockmap_listen *skel,
1457− struct bpf_map *inner_map, int sotype)
1458−{
1459− int verdict = bpf_program__fd(skel->progs.prog_skb_verdict);
1460− int verdict_map = bpf_map__fd(skel->maps.verdict_map);
1461− int sock_map = bpf_map__fd(inner_map);
1462− int err;
1463−
1464− err = xbpf_prog_attach(verdict, sock_map, BPF_SK_SKB_VERDICT, 0);
1465− if (err)
1466− return;
1467−
1468− skel->bss->test_ingress = false;
1469− unix_redir_to_connected(sotype, sock_map, verdict_map, REDIR_EGRESS);
1470− skel->bss->test_ingress = true;
1471− unix_redir_to_connected(sotype, sock_map, verdict_map, REDIR_INGRESS);
1472−
1473− xbpf_prog_detach2(verdict, sock_map, BPF_SK_SKB_VERDICT);
1474−}
1475−
1476−static void test_unix_redir(struct test_sockmap_listen *skel, struct bpf_map *map,
1477− int sotype)
1478−{
1479− const char *family_name, *map_name;
1480− char s[MAX_TEST_NAME];
1481−
1482− family_name = family_str(AF_UNIX);
1483− map_name = map_type_str(map);
1484− snprintf(s, sizeof(s), "%s %s %s", map_name, family_name, __func__);
1485− if (!test__start_subtest(s))
1486− return;
1487− unix_skb_redir_to_connected(skel, map, sotype);
1488−}
1489−
1490−/* Returns two connected loopback vsock sockets */
1491−static int vsock_socketpair_connectible(int sotype, int *v0, int *v1)
1492−{
1493− return create_pair(AF_VSOCK, sotype | SOCK_NONBLOCK, v0, v1);
1494−}
1495−
1496−static void vsock_unix_redir_connectible(int sock_mapfd, int verd_mapfd,
1497− enum redir_mode mode, int sotype)
1498−{
1499− const char *log_prefix = redir_mode_str(mode);
1500− char a = 'a', b = 'b';
1501− int u0, u1, v0, v1;
1502− int sfd[2];
1503− unsigned int pass;
1504− int err, n;
1505− u32 key;
1506−
1507− zero_verdict_count(verd_mapfd);
1508−
1509− if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, 0, sfd))
1510− return;
1511−
1512− u0 = sfd[0];
1513− u1 = sfd[1];
1514−
1515− err = vsock_socketpair_connectible(sotype, &v0, &v1);
1516− if (err) {
1517− FAIL("vsock_socketpair_connectible() failed");
1518− goto close_uds;
1519− }
1520−
1521− err = add_to_sockmap(sock_mapfd, u0, v0);
1522− if (err) {
1523− FAIL("add_to_sockmap failed");
1524− goto close_vsock;
1525− }
1526−
1527− n = write(v1, &a, sizeof(a));
1528− if (n < 0)
1529− FAIL_ERRNO("%s: write", log_prefix);
1530− if (n == 0)
1531− FAIL("%s: incomplete write", log_prefix);
1532− if (n < 1)
1533− goto out;
1534−
1535− n = xrecv_nonblock(mode == REDIR_INGRESS ? u0 : u1, &b, sizeof(b), 0);
1536− if (n < 0)
1537− FAIL("%s: recv() err, errno=%d", log_prefix, errno);
1538− if (n == 0)
1539− FAIL("%s: incomplete recv", log_prefix);
1540− if (b != a)
1541− FAIL("%s: vsock socket map failed, %c != %c", log_prefix, a, b);
1542−
1543− key = SK_PASS;
1544− err = xbpf_map_lookup_elem(verd_mapfd, &key, &pass);
1545− if (err)
1546− goto out;
1547− if (pass != 1)
1548− FAIL("%s: want pass count 1, have %d", log_prefix, pass);
1549−out:
1550− key = 0;
1551− bpf_map_delete_elem(sock_mapfd, &key);
1552− key = 1;
1553− bpf_map_delete_elem(sock_mapfd, &key);
1554−
1555−close_vsock:
1556− close(v0);
1557− close(v1);
1558−
1559−close_uds:
1560− close(u0);
1561− close(u1);
1562−}
1563−
1564−static void vsock_unix_skb_redir_connectible(struct test_sockmap_listen *skel,
1565− struct bpf_map *inner_map,
1566− int sotype)
1567−{
1568− int verdict = bpf_program__fd(skel->progs.prog_skb_verdict);
1569− int verdict_map = bpf_map__fd(skel->maps.verdict_map);
1570− int sock_map = bpf_map__fd(inner_map);
1571− int err;
1572−
1573− err = xbpf_prog_attach(verdict, sock_map, BPF_SK_SKB_VERDICT, 0);
1574− if (err)
1575− return;
1576−
1577− skel->bss->test_ingress = false;
1578− vsock_unix_redir_connectible(sock_map, verdict_map, REDIR_EGRESS, sotype);
1579− skel->bss->test_ingress = true;
1580− vsock_unix_redir_connectible(sock_map, verdict_map, REDIR_INGRESS, sotype);
1581−
1582− xbpf_prog_detach2(verdict, sock_map, BPF_SK_SKB_VERDICT);
1583−}
1584−
1585−static void test_vsock_redir(struct test_sockmap_listen *skel, struct bpf_map *map)
1586−{
1587− const char *family_name, *map_name;
1588− char s[MAX_TEST_NAME];
1589−
1590− family_name = family_str(AF_VSOCK);
1591− map_name = map_type_str(map);
1592− snprintf(s, sizeof(s), "%s %s %s", map_name, family_name, __func__);
1593− if (!test__start_subtest(s))
1594− return;
1595−
1596− vsock_unix_skb_redir_connectible(skel, map, SOCK_STREAM);
1597− vsock_unix_skb_redir_connectible(skel, map, SOCK_SEQPACKET);
1598−}
1599−
16001369 static void test_reuseport(struct test_sockmap_listen *skel,
16011370 struct bpf_map *map, int family, int sotype)
16021371 {
@@ -1637,224 +1406,6 @@ static void test_reuseport(struct test_sockmap_listen *skel, · 34 unmodified lines
16371406 }
16381407 }
16391408
1640−static int inet_socketpair(int family, int type, int *s, int *c)
1641−{
1642− return create_pair(family, type | SOCK_NONBLOCK, s, c);
1643−}
1644−
1645−static void udp_redir_to_connected(int family, int sock_mapfd, int verd_mapfd,
1646− enum redir_mode mode)
1647−{
1648− int c0, c1, p0, p1;
1649− int err;
1650−
1651− err = inet_socketpair(family, SOCK_DGRAM, &p0, &c0);
1652− if (err)
1653− return;
1654− err = inet_socketpair(family, SOCK_DGRAM, &p1, &c1);
1655− if (err)
1656− goto close_cli0;
1657−
1658− pairs_redir_to_connected(c0, p0, c1, p1, sock_mapfd, -1, verd_mapfd,
1659− mode, NO_FLAGS);
1660−
1661− xclose(c1);
1662− xclose(p1);
1663−close_cli0:
1664− xclose(c0);
1665− xclose(p0);
1666−}
1667−
1668−static void udp_skb_redir_to_connected(struct test_sockmap_listen *skel,
1669− struct bpf_map *inner_map, int family)
1670−{
1671− int verdict = bpf_program__fd(skel->progs.prog_skb_verdict);
1672− int verdict_map = bpf_map__fd(skel->maps.verdict_map);
1673− int sock_map = bpf_map__fd(inner_map);
1674− int err;
1675−
1676− err = xbpf_prog_attach(verdict, sock_map, BPF_SK_SKB_VERDICT, 0);
1677− if (err)
1678− return;
1679−
1680− skel->bss->test_ingress = false;
1681− udp_redir_to_connected(family, sock_map, verdict_map, REDIR_EGRESS);
1682− skel->bss->test_ingress = true;
1683− udp_redir_to_connected(family, sock_map, verdict_map, REDIR_INGRESS);
1684−
1685− xbpf_prog_detach2(verdict, sock_map, BPF_SK_SKB_VERDICT);
1686−}
1687−
1688−static void test_udp_redir(struct test_sockmap_listen *skel, struct bpf_map *map,
1689− int family)
1690−{
1691− const char *family_name, *map_name;
1692− char s[MAX_TEST_NAME];
1693−
1694− family_name = family_str(family);
1695− map_name = map_type_str(map);
1696− snprintf(s, sizeof(s), "%s %s %s", map_name, family_name, __func__);
1697− if (!test__start_subtest(s))
1698− return;
1699− udp_skb_redir_to_connected(skel, map, family);
1700−}
1701−
1702−static void inet_unix_redir_to_connected(int family, int type, int sock_mapfd,
1703− int verd_mapfd, enum redir_mode mode)
1704−{
1705− int c0, c1, p0, p1;
1706− int sfd[2];
1707− int err;
1708−
1709− if (socketpair(AF_UNIX, type | SOCK_NONBLOCK, 0, sfd))
1710− return;
1711− c0 = sfd[0], p0 = sfd[1];
1712−
1713− err = inet_socketpair(family, type, &p1, &c1);
1714− if (err)
1715− goto close;
1716−
1717− pairs_redir_to_connected(c0, p0, c1, p1, sock_mapfd, -1, verd_mapfd,
1718− mode, NO_FLAGS);
1719−
1720− xclose(c1);
1721− xclose(p1);
1722−close:
1723− xclose(c0);
1724− xclose(p0);
1725−}
1726−
1727−static void inet_unix_skb_redir_to_connected(struct test_sockmap_listen *skel,
1728− struct bpf_map *inner_map, int family)
1729−{
1730− int verdict = bpf_program__fd(skel->progs.prog_skb_verdict);
1731− int verdict_map = bpf_map__fd(skel->maps.verdict_map);
1732− int sock_map = bpf_map__fd(inner_map);
1733− int err;
1734−
1735− err = xbpf_prog_attach(verdict, sock_map, BPF_SK_SKB_VERDICT, 0);
1736− if (err)
1737− return;
1738−
1739− skel->bss->test_ingress = false;
1740− inet_unix_redir_to_connected(family, SOCK_DGRAM, sock_map, verdict_map,
1741− REDIR_EGRESS);
1742− inet_unix_redir_to_connected(family, SOCK_STREAM, sock_map, verdict_map,
1743− REDIR_EGRESS);
1744− skel->bss->test_ingress = true;
1745− inet_unix_redir_to_connected(family, SOCK_DGRAM, sock_map, verdict_map,
1746− REDIR_INGRESS);
1747− inet_unix_redir_to_connected(family, SOCK_STREAM, sock_map, verdict_map,
1748− REDIR_INGRESS);
1749−
1750− xbpf_prog_detach2(verdict, sock_map, BPF_SK_SKB_VERDICT);
1751−}
1752−
1753−static void unix_inet_redir_to_connected(int family, int type, int sock_mapfd,
1754− int nop_mapfd, int verd_mapfd,
1755− enum redir_mode mode, int send_flags)
1756−{
1757− int c0, c1, p0, p1;
1758− int sfd[2];
1759− int err;
1760−
1761− err = inet_socketpair(family, type, &p0, &c0);
1762− if (err)
1763− return;
1764−
1765− if (socketpair(AF_UNIX, type | SOCK_NONBLOCK, 0, sfd))
1766− goto close_cli0;
1767− c1 = sfd[0], p1 = sfd[1];
1768−
1769− pairs_redir_to_connected(c0, p0, c1, p1, sock_mapfd, nop_mapfd,
1770− verd_mapfd, mode, send_flags);
1771−
1772− xclose(c1);
1773− xclose(p1);
1774−close_cli0:
1775− xclose(c0);
1776− xclose(p0);
1777−}
1778−
1779−static void unix_inet_skb_redir_to_connected(struct test_sockmap_listen *skel,
1780− struct bpf_map *inner_map, int family)
1781−{
1782− int verdict = bpf_program__fd(skel->progs.prog_skb_verdict);
1783− int nop_map = bpf_map__fd(skel->maps.nop_map);
1784− int verdict_map = bpf_map__fd(skel->maps.verdict_map);
1785− int sock_map = bpf_map__fd(inner_map);
1786− int err;
1787−
1788− err = xbpf_prog_attach(verdict, sock_map, BPF_SK_SKB_VERDICT, 0);
1789− if (err)
1790− return;
1791−
1792− skel->bss->test_ingress = false;
1793− unix_inet_redir_to_connected(family, SOCK_DGRAM,
1794− sock_map, -1, verdict_map,
1795− REDIR_EGRESS, NO_FLAGS);
1796− unix_inet_redir_to_connected(family, SOCK_STREAM,
1797− sock_map, -1, verdict_map,
1798− REDIR_EGRESS, NO_FLAGS);
1799−
1800− unix_inet_redir_to_connected(family, SOCK_DGRAM,
1801− sock_map, nop_map, verdict_map,
1802− REDIR_EGRESS, NO_FLAGS);
1803− unix_inet_redir_to_connected(family, SOCK_STREAM,
1804− sock_map, nop_map, verdict_map,
1805− REDIR_EGRESS, NO_FLAGS);
1806−
1807− /* MSG_OOB not supported by AF_UNIX SOCK_DGRAM */
1808− unix_inet_redir_to_connected(family, SOCK_STREAM,
1809− sock_map, nop_map, verdict_map,
1810− REDIR_EGRESS, MSG_OOB);
1811−
1812− skel->bss->test_ingress = true;
1813− unix_inet_redir_to_connected(family, SOCK_DGRAM,
1814− sock_map, -1, verdict_map,
1815− REDIR_INGRESS, NO_FLAGS);
1816− unix_inet_redir_to_connected(family, SOCK_STREAM,
1817− sock_map, -1, verdict_map,
1818− REDIR_INGRESS, NO_FLAGS);
1819−
1820− unix_inet_redir_to_connected(family, SOCK_DGRAM,
1821− sock_map, nop_map, verdict_map,
1822− REDIR_INGRESS, NO_FLAGS);
1823− unix_inet_redir_to_connected(family, SOCK_STREAM,
1824− sock_map, nop_map, verdict_map,
1825− REDIR_INGRESS, NO_FLAGS);
1826−
1827− /* MSG_OOB not supported by AF_UNIX SOCK_DGRAM */
1828− unix_inet_redir_to_connected(family, SOCK_STREAM,
1829− sock_map, nop_map, verdict_map,
1830− REDIR_INGRESS, MSG_OOB);
1831−
1832− xbpf_prog_detach2(verdict, sock_map, BPF_SK_SKB_VERDICT);
1833−}
1834−
1835−static void test_udp_unix_redir(struct test_sockmap_listen *skel, struct bpf_map *map,
1836− int family)
1837−{
1838− const char *family_name, *map_name;
1839− struct netns_obj *netns;
1840− char s[MAX_TEST_NAME];
1841−
1842− family_name = family_str(family);
1843− map_name = map_type_str(map);
1844− snprintf(s, sizeof(s), "%s %s %s", map_name, family_name, __func__);
1845− if (!test__start_subtest(s))
1846− return;
1847−
1848− netns = netns_new("sockmap_listen", true);
1849− if (!ASSERT_OK_PTR(netns, "netns_new"))
1850− return;
1851−
1852− inet_unix_skb_redir_to_connected(skel, map, family);
1853− unix_inet_skb_redir_to_connected(skel, map, family);
1854−
1855− netns_free(netns);
1856−}
1857−
18581409 static void run_tests(struct test_sockmap_listen *skel, struct bpf_map *map,
18591410 int family)
18601411 {
@@ -1863,8 +1414,6 @@ static void run_tests(struct test_sockmap_listen *skel, struct bpf_map *map, · 2 unmodified lines
18631414 test_redir(skel, map, family, SOCK_STREAM);
18641415 test_reuseport(skel, map, family, SOCK_STREAM);
18651416 test_reuseport(skel, map, family, SOCK_DGRAM);
1866− test_udp_redir(skel, map, family);
1867− test_udp_unix_redir(skel, map, family);
18681417 }
18691418
18701419 void serial_test_sockmap_listen(void)
@@ -1880,16 +1429,10 @@ void serial_test_sockmap_listen(void) · 9 unmodified lines
18801429 skel->bss->test_sockmap = true;
18811430 run_tests(skel, skel->maps.sock_map, AF_INET);
18821431 run_tests(skel, skel->maps.sock_map, AF_INET6);
1883− test_unix_redir(skel, skel->maps.sock_map, SOCK_DGRAM);
1884− test_unix_redir(skel, skel->maps.sock_map, SOCK_STREAM);
1885− test_vsock_redir(skel, skel->maps.sock_map);
18861432
18871433 skel->bss->test_sockmap = false;
18881434 run_tests(skel, skel->maps.sock_hash, AF_INET);
18891435 run_tests(skel, skel->maps.sock_hash, AF_INET6);
1890− test_unix_redir(skel, skel->maps.sock_hash, SOCK_DGRAM);
1891− test_unix_redir(skel, skel->maps.sock_hash, SOCK_STREAM);
1892− test_vsock_redir(skel, skel->maps.sock_hash);
18931436
18941437 test_sockmap_listen__destroy(skel);
18951438 }
tools/testing/selftests/bpf/prog_tests/sockmap_redir.c+465−0
@@ -0,0 +1,465 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/*
3+ * Test for sockmap/sockhash redirection.
4+ *
5+ * BPF_MAP_TYPE_SOCKMAP
6+ * BPF_MAP_TYPE_SOCKHASH
7+ * x
8+ * sk_msg-to-egress
9+ * sk_msg-to-ingress
10+ * sk_skb-to-egress
11+ * sk_skb-to-ingress
12+ * x
13+ * AF_INET, SOCK_STREAM
14+ * AF_INET6, SOCK_STREAM
15+ * AF_INET, SOCK_DGRAM
16+ * AF_INET6, SOCK_DGRAM
17+ * AF_UNIX, SOCK_STREAM
18+ * AF_UNIX, SOCK_DGRAM
19+ * AF_VSOCK, SOCK_STREAM
20+ * AF_VSOCK, SOCK_SEQPACKET
21+ */
22+
23+#include <errno.h>
24+#include <error.h>
25+#include <sched.h>
26+#include <stdio.h>
27+#include <unistd.h>
28+
29+#include <netinet/in.h>
30+#include <sys/socket.h>
31+#include <sys/types.h>
32+#include <sys/un.h>
33+#include <linux/string.h>
34+#include <linux/vm_sockets.h>
35+
36+#include <bpf/bpf.h>
37+#include <bpf/libbpf.h>
38+
39+#include "linux/const.h"
40+#include "test_progs.h"
41+#include "sockmap_helpers.h"
42+#include "test_sockmap_redir.skel.h"
43+
44+/* The meaning of SUPPORTED is "will redirect packet as expected".
45+ */
46+#define SUPPORTED _BITUL(0)
47+
48+/* Note on sk_skb-to-ingress ->af_vsock:
49+ *
50+ * Peer socket may receive the packet some time after the return from sendmsg().
51+ * In a typical usage scenario, recvmsg() will block until the redirected packet
52+ * appears in the destination queue, or timeout if the packet was dropped. By
53+ * that point, the verdict map has already been updated to reflect what has
54+ * happened.
55+ *
56+ * But sk_skb-to-ingress/af_vsock is an unsupported combination, so no recvmsg()
57+ * takes place. Which means we may race the execution of the verdict logic and
58+ * read map_verd before it has been updated, i.e. we might observe
59+ * map_verd[SK_DROP]=0 instead of map_verd[SK_DROP]=1.
60+ *
61+ * This confuses the selftest logic: if there was no packet dropped, where's the
62+ * packet? So here's a heuristic: on map_verd[SK_DROP]=map_verd[SK_PASS]=0
63+ * (which implies the verdict program has not been ran) just re-read the verdict
64+ * map again.
65+ */
66+#define UNSUPPORTED_RACY_VERD _BITUL(1)
67+
68+enum prog_type {
69+ SK_MSG_EGRESS,
70+ SK_MSG_INGRESS,
71+ SK_SKB_EGRESS,
72+ SK_SKB_INGRESS,
73+};
74+
75+enum {
76+ SEND_INNER = 0,
77+ SEND_OUTER,
78+};
79+
80+enum {
81+ RECV_INNER = 0,
82+ RECV_OUTER,
83+};
84+
85+struct maps {
86+ int in;
87+ int out;
88+ int verd;
89+};
90+
91+struct combo_spec {
92+ enum prog_type prog_type;
93+ const char *in, *out;
94+};
95+
96+struct redir_spec {
97+ const char *name;
98+ int idx_send;
99+ int idx_recv;
100+ enum prog_type prog_type;
101+};
102+
103+struct socket_spec {
104+ int family;
105+ int sotype;
106+ int send_flags;
107+ int in[2];
108+ int out[2];
109+};
110+
111+static int socket_spec_pairs(struct socket_spec *s)
112+{
113+ return create_socket_pairs(s->family, s->sotype,
114+ &s->in[0], &s->out[0],
115+ &s->in[1], &s->out[1]);
116+}
117+
118+static void socket_spec_close(struct socket_spec *s)
119+{
120+ xclose(s->in[0]);
121+ xclose(s->in[1]);
122+ xclose(s->out[0]);
123+ xclose(s->out[1]);
124+}
125+
126+static void get_redir_params(struct redir_spec *redir,
127+ struct test_sockmap_redir *skel, int *prog_fd,
128+ enum bpf_attach_type *attach_type,
129+ int *redirect_flags)
130+{
131+ enum prog_type type = redir->prog_type;
132+ struct bpf_program *prog;
133+ bool sk_msg;
134+
135+ sk_msg = type == SK_MSG_INGRESS || type == SK_MSG_EGRESS;
136+ prog = sk_msg ? skel->progs.prog_msg_verdict : skel->progs.prog_skb_verdict;
137+
138+ *prog_fd = bpf_program__fd(prog);
139+ *attach_type = sk_msg ? BPF_SK_MSG_VERDICT : BPF_SK_SKB_VERDICT;
140+
141+ if (type == SK_MSG_INGRESS || type == SK_SKB_INGRESS)
142+ *redirect_flags = BPF_F_INGRESS;
143+ else
144+ *redirect_flags = 0;
145+}
146+
147+static void try_recv(const char *prefix, int fd, int flags, bool expect_success)
148+{
149+ ssize_t n;
150+ char buf;
151+
152+ errno = 0;
153+ n = recv(fd, &buf, 1, flags);
154+ if (n < 0 && expect_success)
155+ FAIL_ERRNO("%s: unexpected failure: retval=%zd", prefix, n);
156+ if (!n && !expect_success)
157+ FAIL("%s: expected failure: retval=%zd", prefix, n);
158+}
159+
160+static void handle_unsupported(int sd_send, int sd_peer, int sd_in, int sd_out,
161+ int sd_recv, int map_verd, int status)
162+{
163+ unsigned int drop, pass;
164+ char recv_buf;
165+ ssize_t n;
166+
167+get_verdict:
168+ if (xbpf_map_lookup_elem(map_verd, &u32(SK_DROP), &drop) ||
169+ xbpf_map_lookup_elem(map_verd, &u32(SK_PASS), &pass))
170+ return;
171+
172+ if (pass == 0 && drop == 0 && (status & UNSUPPORTED_RACY_VERD)) {
173+ sched_yield();
174+ goto get_verdict;
175+ }
176+
177+ if (pass != 0) {
178+ FAIL("unsupported: wanted verdict pass 0, have %u", pass);
179+ return;
180+ }
181+
182+ /* If nothing was dropped, packet should have reached the peer */
183+ if (drop == 0) {
184+ errno = 0;
185+ n = recv_timeout(sd_peer, &recv_buf, 1, 0, IO_TIMEOUT_SEC);
186+ if (n != 1)
187+ FAIL_ERRNO("unsupported: packet missing, retval=%zd", n);
188+ }
189+
190+ /* Ensure queues are empty */
191+ try_recv("bpf.recv(sd_send)", sd_send, MSG_DONTWAIT, false);
192+ if (sd_in != sd_send)
193+ try_recv("bpf.recv(sd_in)", sd_in, MSG_DONTWAIT, false);
194+
195+ try_recv("bpf.recv(sd_out)", sd_out, MSG_DONTWAIT, false);
196+ if (sd_recv != sd_out)
197+ try_recv("bpf.recv(sd_recv)", sd_recv, MSG_DONTWAIT, false);
198+}
199+
200+static void test_send_redir_recv(int sd_send, int send_flags, int sd_peer,
201+ int sd_in, int sd_out, int sd_recv,
202+ struct maps *maps, int status)
203+{
204+ unsigned int drop, pass;
205+ char *send_buf = "ab";
206+ char recv_buf = '\0';
207+ ssize_t n, len = 1;
208+
209+ /* Zero out the verdict map */
210+ if (xbpf_map_update_elem(maps->verd, &u32(SK_DROP), &u32(0), BPF_ANY) ||
211+ xbpf_map_update_elem(maps->verd, &u32(SK_PASS), &u32(0), BPF_ANY))
212+ return;
213+
214+ if (xbpf_map_update_elem(maps->in, &u32(0), &u64(sd_in), BPF_NOEXIST))
215+ return;
216+
217+ if (xbpf_map_update_elem(maps->out, &u32(0), &u64(sd_out), BPF_NOEXIST))
218+ goto del_in;
219+
220+ /* Last byte is OOB data when send_flags has MSG_OOB bit set */
221+ if (send_flags & MSG_OOB)
222+ len++;
223+ n = send(sd_send, send_buf, len, send_flags);
224+ if (n >= 0 && n < len)
225+ FAIL("incomplete send");
226+ if (n < 0) {
227+ /* sk_msg redirect combo not supported? */
228+ if (status & SUPPORTED || errno != EACCES)
229+ FAIL_ERRNO("send");
230+ goto out;
231+ }
232+
233+ if (!(status & SUPPORTED)) {
234+ handle_unsupported(sd_send, sd_peer, sd_in, sd_out, sd_recv,
235+ maps->verd, status);
236+ goto out;
237+ }
238+
239+ errno = 0;
240+ n = recv_timeout(sd_recv, &recv_buf, 1, 0, IO_TIMEOUT_SEC);
241+ if (n != 1) {
242+ FAIL_ERRNO("recv_timeout()");
243+ goto out;
244+ }
245+
246+ /* Check verdict _after_ recv(); af_vsock may need time to catch up */
247+ if (xbpf_map_lookup_elem(maps->verd, &u32(SK_DROP), &drop) ||
248+ xbpf_map_lookup_elem(maps->verd, &u32(SK_PASS), &pass))
249+ goto out;
250+
251+ if (drop != 0 || pass != 1)
252+ FAIL("unexpected verdict drop/pass: wanted 0/1, have %u/%u",
253+ drop, pass);
254+
255+ if (recv_buf != send_buf[0])
256+ FAIL("recv(): payload check, %02x != %02x", recv_buf, send_buf[0]);
257+
258+ if (send_flags & MSG_OOB) {
259+ /* Fail reading OOB while in sockmap */
260+ try_recv("bpf.recv(sd_out, MSG_OOB)", sd_out,
261+ MSG_OOB | MSG_DONTWAIT, false);
262+
263+ /* Remove sd_out from sockmap */
264+ xbpf_map_delete_elem(maps->out, &u32(0));
265+
266+ /* Check that OOB was dropped on redirect */
267+ try_recv("recv(sd_out, MSG_OOB)", sd_out,
268+ MSG_OOB | MSG_DONTWAIT, false);
269+
270+ goto del_in;
271+ }
272+out:
273+ xbpf_map_delete_elem(maps->out, &u32(0));
274+del_in:
275+ xbpf_map_delete_elem(maps->in, &u32(0));
276+}
277+
278+static int is_redir_supported(enum prog_type type, const char *in,
279+ const char *out)
280+{
281+ /* Matching based on strings returned by socket_kind_to_str():
282+ * tcp4, udp4, tcp6, udp6, u_str, u_dgr, v_str, v_seq
283+ * Plus a wildcard: any
284+ * Not in use: u_seq, v_dgr
285+ */
286+ struct combo_spec *c, combos[] = {
287+ /* Send to local: TCP -> any, but vsock */
288+ { SK_MSG_INGRESS, "tcp", "tcp" },
289+ { SK_MSG_INGRESS, "tcp", "udp" },
290+ { SK_MSG_INGRESS, "tcp", "u_str" },
291+ { SK_MSG_INGRESS, "tcp", "u_dgr" },
292+
293+ /* Send to egress: TCP -> TCP */
294+ { SK_MSG_EGRESS, "tcp", "tcp" },
295+
296+ /* Ingress to egress: any -> any */
297+ { SK_SKB_EGRESS, "any", "any" },
298+
299+ /* Ingress to local: any -> any, but vsock */
300+ { SK_SKB_INGRESS, "any", "tcp" },
301+ { SK_SKB_INGRESS, "any", "udp" },
302+ { SK_SKB_INGRESS, "any", "u_str" },
303+ { SK_SKB_INGRESS, "any", "u_dgr" },
304+ };
305+
306+ for (c = combos; c < combos + ARRAY_SIZE(combos); c++) {
307+ if (c->prog_type == type &&
308+ (!strcmp(c->in, "any") || strstarts(in, c->in)) &&
309+ (!strcmp(c->out, "any") || strstarts(out, c->out)))
310+ return SUPPORTED;
311+ }
312+
313+ return 0;
314+}
315+
316+static int get_support_status(enum prog_type type, const char *in,
317+ const char *out)
318+{
319+ int status = is_redir_supported(type, in, out);
320+
321+ if (type == SK_SKB_INGRESS && strstarts(out, "v_"))
322+ status |= UNSUPPORTED_RACY_VERD;
323+
324+ return status;
325+}
326+
327+static void test_socket(enum bpf_map_type type, struct redir_spec *redir,
328+ struct maps *maps, struct socket_spec *s_in,
329+ struct socket_spec *s_out)
330+{
331+ int fd_in, fd_out, fd_send, fd_peer, fd_recv, flags, status;
332+ const char *in_str, *out_str;
333+ char s[MAX_TEST_NAME];
334+
335+ fd_in = s_in->in[0];
336+ fd_out = s_out->out[0];
337+ fd_send = s_in->in[redir->idx_send];
338+ fd_peer = s_in->in[redir->idx_send ^ 1];
339+ fd_recv = s_out->out[redir->idx_recv];
340+ flags = s_in->send_flags;
341+
342+ in_str = socket_kind_to_str(fd_in);
343+ out_str = socket_kind_to_str(fd_out);
344+ status = get_support_status(redir->prog_type, in_str, out_str);
345+
346+ snprintf(s, sizeof(s),
347+ "%-4s %-17s %-5s %s %-5s%6s",
348+ /* hash sk_skb-to-ingress u_str → v_str (OOB) */
349+ type == BPF_MAP_TYPE_SOCKMAP ? "map" : "hash",
350+ redir->name,
351+ in_str,
352+ status & SUPPORTED ? "→" : " ",
353+ out_str,
354+ (flags & MSG_OOB) ? "(OOB)" : "");
355+
356+ if (!test__start_subtest(s))
357+ return;
358+
359+ test_send_redir_recv(fd_send, flags, fd_peer, fd_in, fd_out, fd_recv,
360+ maps, status);
361+}
362+
363+static void test_redir(enum bpf_map_type type, struct redir_spec *redir,
364+ struct maps *maps)
365+{
366+ struct socket_spec *s, sockets[] = {
367+ { AF_INET, SOCK_STREAM },
368+ // { AF_INET, SOCK_STREAM, MSG_OOB }, /* Known to be broken */
369+ { AF_INET6, SOCK_STREAM },
370+ { AF_INET, SOCK_DGRAM },
371+ { AF_INET6, SOCK_DGRAM },
372+ { AF_UNIX, SOCK_STREAM },
373+ { AF_UNIX, SOCK_STREAM, MSG_OOB },
374+ { AF_UNIX, SOCK_DGRAM },
375+ // { AF_UNIX, SOCK_SEQPACKET}, /* Unsupported BPF_MAP_UPDATE_ELEM */
376+ { AF_VSOCK, SOCK_STREAM },
377+ // { AF_VSOCK, SOCK_DGRAM }, /* Unsupported socket() */
378+ { AF_VSOCK, SOCK_SEQPACKET },
379+ };
380+
381+ for (s = sockets; s < sockets + ARRAY_SIZE(sockets); s++)
382+ if (socket_spec_pairs(s))
383+ goto out;
384+
385+ /* Intra-proto */
386+ for (s = sockets; s < sockets + ARRAY_SIZE(sockets); s++)
387+ test_socket(type, redir, maps, s, s);
388+
389+ /* Cross-proto */
390+ for (int i = 0; i < ARRAY_SIZE(sockets); i++) {
391+ for (int j = 0; j < ARRAY_SIZE(sockets); j++) {
392+ struct socket_spec *out = &sockets[j];
393+ struct socket_spec *in = &sockets[i];
394+
395+ /* Skip intra-proto and between variants */
396+ if (out->send_flags ||
397+ (in->family == out->family &&
398+ in->sotype == out->sotype))
399+ continue;
400+
401+ test_socket(type, redir, maps, in, out);
402+ }
403+ }
404+out:
405+ while (--s >= sockets)
406+ socket_spec_close(s);
407+}
408+
409+static void test_map(enum bpf_map_type type)
410+{
411+ struct redir_spec *r, redirs[] = {
412+ { "sk_msg-to-ingress", SEND_INNER, RECV_INNER, SK_MSG_INGRESS },
413+ { "sk_msg-to-egress", SEND_INNER, RECV_OUTER, SK_MSG_EGRESS },
414+ { "sk_skb-to-egress", SEND_OUTER, RECV_OUTER, SK_SKB_EGRESS },
415+ { "sk_skb-to-ingress", SEND_OUTER, RECV_INNER, SK_SKB_INGRESS },
416+ };
417+
418+ for (r = redirs; r < redirs + ARRAY_SIZE(redirs); r++) {
419+ enum bpf_attach_type attach_type;
420+ struct test_sockmap_redir *skel;
421+ struct maps maps;
422+ int prog_fd;
423+
424+ skel = test_sockmap_redir__open_and_load();
425+ if (!skel) {
426+ FAIL("open_and_load");
427+ return;
428+ }
429+
430+ switch (type) {
431+ case BPF_MAP_TYPE_SOCKMAP:
432+ maps.in = bpf_map__fd(skel->maps.nop_map);
433+ maps.out = bpf_map__fd(skel->maps.sock_map);
434+ break;
435+ case BPF_MAP_TYPE_SOCKHASH:
436+ maps.in = bpf_map__fd(skel->maps.nop_hash);
437+ maps.out = bpf_map__fd(skel->maps.sock_hash);
438+ break;
439+ default:
440+ FAIL("Unsupported bpf_map_type");
441+ return;
442+ }
443+
444+ skel->bss->redirect_type = type;
445+ maps.verd = bpf_map__fd(skel->maps.verdict_map);
446+ get_redir_params(r, skel, &prog_fd, &attach_type,
447+ &skel->bss->redirect_flags);
448+
449+ if (xbpf_prog_attach(prog_fd, maps.in, attach_type, 0))
450+ return;
451+
452+ test_redir(type, r, &maps);
453+
454+ if (xbpf_prog_detach2(prog_fd, maps.in, attach_type))
455+ return;
456+
457+ test_sockmap_redir__destroy(skel);
458+ }
459+}
460+
461+void serial_test_sockmap_redir(void)
462+{
463+ test_map(BPF_MAP_TYPE_SOCKMAP);
464+ test_map(BPF_MAP_TYPE_SOCKHASH);
465+}
tools/testing/selftests/bpf/prog_tests/tc_redirect.c+6−5
@@ -56,6 +56,8 @@ · 55 unmodified lines
5656
5757 #define MAC_DST_FWD "00:11:22:33:44:55"
5858 #define MAC_DST "00:22:33:44:55:66"
59+#define MAC_SRC_FWD "00:33:44:55:66:77"
60+#define MAC_SRC "00:44:55:66:77:88"
5961
6062 #define IFADDR_STR_LEN 18
6163 #define PING_ARGS "-i 0.2 -c 3 -w 10 -q"
@@ -207,11 +209,10 @@ static int netns_setup_links_and_routes(struct netns_setup_result *result) · 145 unmodified lines
207209 int err;
208210
209211 if (result->dev_mode == MODE_VETH) {
210− SYS(fail, "ip link add src type veth peer name src_fwd");
211− SYS(fail, "ip link add dst type veth peer name dst_fwd");
212−
213− SYS(fail, "ip link set dst_fwd address " MAC_DST_FWD);
214− SYS(fail, "ip link set dst address " MAC_DST);
212+ SYS(fail, "ip link add src address " MAC_SRC " type veth "
213+ "peer name src_fwd address " MAC_SRC_FWD);
214+ SYS(fail, "ip link add dst address " MAC_DST " type veth "
215+ "peer name dst_fwd address " MAC_DST_FWD);
215216 } else if (result->dev_mode == MODE_NETKIT) {
216217 err = create_netkit(NETKIT_L3, "src", "src_fwd");
217218 if (!ASSERT_OK(err, "create_ifindex_src"))
tools/testing/selftests/bpf/prog_tests/test_btf_ext.c+64−0
@@ -0,0 +1,64 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/* Copyright (c) 2025 Meta Platforms Inc. */
3+#include <test_progs.h>
4+#include "test_btf_ext.skel.h"
5+#include "btf_helpers.h"
6+
7+static void subtest_line_func_info(void)
8+{
9+ struct test_btf_ext *skel;
10+ struct bpf_prog_info info;
11+ struct bpf_line_info line_info[128], *libbpf_line_info;
12+ struct bpf_func_info func_info[128], *libbpf_func_info;
13+ __u32 info_len = sizeof(info), libbbpf_line_info_cnt, libbbpf_func_info_cnt;
14+ int err, fd;
15+
16+ skel = test_btf_ext__open_and_load();
17+ if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))
18+ return;
19+
20+ fd = bpf_program__fd(skel->progs.global_func);
21+
22+ memset(&info, 0, sizeof(info));
23+ info.line_info = ptr_to_u64(&line_info);
24+ info.nr_line_info = sizeof(line_info);
25+ info.line_info_rec_size = sizeof(*line_info);
26+ err = bpf_prog_get_info_by_fd(fd, &info, &info_len);
27+ if (!ASSERT_OK(err, "prog_line_info"))
28+ goto out;
29+
30+ libbpf_line_info = bpf_program__line_info(skel->progs.global_func);
31+ libbbpf_line_info_cnt = bpf_program__line_info_cnt(skel->progs.global_func);
32+
33+ memset(&info, 0, sizeof(info));
34+ info.func_info = ptr_to_u64(&func_info);
35+ info.nr_func_info = sizeof(func_info);
36+ info.func_info_rec_size = sizeof(*func_info);
37+ err = bpf_prog_get_info_by_fd(fd, &info, &info_len);
38+ if (!ASSERT_OK(err, "prog_func_info"))
39+ goto out;
40+
41+ libbpf_func_info = bpf_program__func_info(skel->progs.global_func);
42+ libbbpf_func_info_cnt = bpf_program__func_info_cnt(skel->progs.global_func);
43+
44+ if (!ASSERT_OK_PTR(libbpf_line_info, "bpf_program__line_info"))
45+ goto out;
46+ if (!ASSERT_EQ(libbbpf_line_info_cnt, info.nr_line_info, "line_info_cnt"))
47+ goto out;
48+ if (!ASSERT_OK_PTR(libbpf_func_info, "bpf_program__func_info"))
49+ goto out;
50+ if (!ASSERT_EQ(libbbpf_func_info_cnt, info.nr_func_info, "func_info_cnt"))
51+ goto out;
52+ ASSERT_MEMEQ(libbpf_line_info, line_info, libbbpf_line_info_cnt * sizeof(*line_info),
53+ "line_info");
54+ ASSERT_MEMEQ(libbpf_func_info, func_info, libbbpf_func_info_cnt * sizeof(*func_info),
55+ "func_info");
56+out:
57+ test_btf_ext__destroy(skel);
58+}
59+
60+void test_btf_ext(void)
61+{
62+ if (test__start_subtest("line_func_info"))
63+ subtest_line_func_info();
64+}
tools/testing/selftests/bpf/prog_tests/test_veristat.c+5−0
@@ -63,6 +63,9 @@ static void test_set_global_vars_succeeds(void) · 62 unmodified lines
6363 " -G \"var_eb = EB2\" "\
6464 " -G \"var_ec = EC2\" "\
6565 " -G \"var_b = 1\" "\
66+ " -G \"struct1.struct2.u.var_u8 = 170\" "\
67+ " -G \"union1.struct3.var_u8_l = 0xaa\" "\
68+ " -G \"union1.struct3.var_u8_h = 0xaa\" "\
6669 "-vl2 > %s", fix->veristat, fix->tmpfile);
6770
6871 read(fix->fd, fix->output, fix->sz);
@@ -78,6 +81,8 @@ static void test_set_global_vars_succeeds(void) · 9 unmodified lines
7881 __CHECK_STR("_w=12 ", "var_eb = EB2");
7982 __CHECK_STR("_w=13 ", "var_ec = EC2");
8083 __CHECK_STR("_w=1 ", "var_b = 1");
84+ __CHECK_STR("_w=170 ", "struct1.struct2.u.var_u8 = 170");
85+ __CHECK_STR("_w=0xaaaa ", "union1.var_u16 = 0xaaaa");
8186
8287 out:
8388 teardown_fixture(fix);
tools/testing/selftests/bpf/prog_tests/verifier.c+2−0
@@ -14,6 +14,7 @@ · 13 unmodified lines
1414 #include "verifier_bounds_deduction_non_const.skel.h"
1515 #include "verifier_bounds_mix_sign_unsign.skel.h"
1616 #include "verifier_bpf_get_stack.skel.h"
17+#include "verifier_bpf_trap.skel.h"
1718 #include "verifier_bswap.skel.h"
1819 #include "verifier_btf_ctx_access.skel.h"
1920 #include "verifier_btf_unreliable_prog.skel.h"
@@ -148,6 +149,7 @@ void test_verifier_bounds_deduction(void) { RUN(verifier_bounds_deduction); · 128 unmodified lines
148149 void test_verifier_bounds_deduction_non_const(void) { RUN(verifier_bounds_deduction_non_const); }
149150 void test_verifier_bounds_mix_sign_unsign(void) { RUN(verifier_bounds_mix_sign_unsign); }
150151 void test_verifier_bpf_get_stack(void) { RUN(verifier_bpf_get_stack); }
152+void test_verifier_bpf_trap(void) { RUN(verifier_bpf_trap); }
151153 void test_verifier_bswap(void) { RUN(verifier_bswap); }
152154 void test_verifier_btf_ctx_access(void) { RUN(verifier_btf_ctx_access); }
153155 void test_verifier_btf_unreliable_prog(void) { RUN(verifier_btf_unreliable_prog); }
tools/testing/selftests/bpf/prog_tests/xdp_metadata.c+21−1
@@ -351,9 +351,10 @@ void test_xdp_metadata(void) · 350 unmodified lines
351351 struct xdp_metadata2 *bpf_obj2 = NULL;
352352 struct xdp_metadata *bpf_obj = NULL;
353353 struct bpf_program *new_prog, *prog;
354+ struct bpf_devmap_val devmap_e = {};
355+ struct bpf_map *prog_arr, *devmap;
354356 struct nstoken *tok = NULL;
355357 __u32 queue_id = QUEUE_ID;
356− struct bpf_map *prog_arr;
357358 struct xsk tx_xsk = {};
358359 struct xsk rx_xsk = {};
359360 __u32 val, key = 0;
@@ -409,6 +410,13 @@ void test_xdp_metadata(void) · 49 unmodified lines
409410 bpf_program__set_ifindex(prog, rx_ifindex);
410411 bpf_program__set_flags(prog, BPF_F_XDP_DEV_BOUND_ONLY);
411412
413+ /* Make sure we can load a dev-bound program that performs
414+ * XDP_REDIRECT into a devmap.
415+ */
416+ new_prog = bpf_object__find_program_by_name(bpf_obj->obj, "redirect");
417+ bpf_program__set_ifindex(new_prog, rx_ifindex);
418+ bpf_program__set_flags(new_prog, BPF_F_XDP_DEV_BOUND_ONLY);
419+
412420 if (!ASSERT_OK(xdp_metadata__load(bpf_obj), "load skeleton"))
413421 goto out;
414422
@@ -423,6 +431,18 @@ void test_xdp_metadata(void) · 8 unmodified lines
423431 "update prog_arr"))
424432 goto out;
425433
434+ /* Make sure we can't add dev-bound programs to devmaps. */
435+ devmap = bpf_object__find_map_by_name(bpf_obj->obj, "dev_map");
436+ if (!ASSERT_OK_PTR(devmap, "no dev_map found"))
437+ goto out;
438+
439+ devmap_e.bpf_prog.fd = val;
440+ if (!ASSERT_ERR(bpf_map__update_elem(devmap, &key, sizeof(key),
441+ &devmap_e, sizeof(devmap_e),
442+ BPF_ANY),
443+ "update dev_map"))
444+ goto out;
445+
426446 /* Attach BPF program to RX interface. */
427447
428448 ret = bpf_xdp_attach(rx_ifindex,
tools/testing/selftests/bpf/progs/bench_sockmap_prog.c+65−0
@@ -0,0 +1,65 @@
1+// SPDX-License-Identifier: GPL-2.0
2+#include <linux/bpf.h>
3+#include <bpf/bpf_helpers.h>
4+#include <bpf/bpf_endian.h>
5+
6+long process_byte = 0;
7+int verdict_dir = 0;
8+int dropped = 0;
9+int pkt_size = 0;
10+struct {
11+ __uint(type, BPF_MAP_TYPE_SOCKMAP);
12+ __uint(max_entries, 20);
13+ __type(key, int);
14+ __type(value, int);
15+} sock_map_rx SEC(".maps");
16+
17+struct {
18+ __uint(type, BPF_MAP_TYPE_SOCKMAP);
19+ __uint(max_entries, 20);
20+ __type(key, int);
21+ __type(value, int);
22+} sock_map_tx SEC(".maps");
23+
24+SEC("sk_skb/stream_parser")
25+int prog_skb_parser(struct __sk_buff *skb)
26+{
27+ return pkt_size;
28+}
29+
30+SEC("sk_skb/stream_verdict")
31+int prog_skb_verdict(struct __sk_buff *skb)
32+{
33+ int one = 1;
34+ int ret = bpf_sk_redirect_map(skb, &sock_map_rx, one, verdict_dir);
35+
36+ if (ret == SK_DROP)
37+ dropped++;
38+ __sync_fetch_and_add(&process_byte, skb->len);
39+ return ret;
40+}
41+
42+SEC("sk_skb/stream_verdict")
43+int prog_skb_pass(struct __sk_buff *skb)
44+{
45+ __sync_fetch_and_add(&process_byte, skb->len);
46+ return SK_PASS;
47+}
48+
49+SEC("sk_msg")
50+int prog_skmsg_verdict(struct sk_msg_md *msg)
51+{
52+ int one = 1;
53+
54+ __sync_fetch_and_add(&process_byte, msg->size);
55+ return bpf_msg_redirect_map(msg, &sock_map_tx, one, verdict_dir);
56+}
57+
58+SEC("sk_msg")
59+int prog_skmsg_pass(struct sk_msg_md *msg)
60+{
61+ __sync_fetch_and_add(&process_byte, msg->size);
62+ return SK_PASS;
63+}
64+
65+char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/bpf_arena_spin_lock.hrenamed from tools/testing/selftests/bpf/bpf_arena_spin_lock.h+12−3
@@ -32,6 +32,7 @@ extern unsigned long CONFIG_NR_CPUS __kconfig; · 31 unmodified lines
3232 struct __qspinlock {
3333 union {
3434 atomic_t val;
35+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
3536 struct {
3637 u8 locked;
3738 u8 pending;
@@ -40,6 +41,17 @@ struct __qspinlock { · 2 unmodified lines
4041 u16 locked_pending;
4142 u16 tail;
4243 };
44+#else
45+ struct {
46+ u16 tail;
47+ u16 locked_pending;
48+ };
49+ struct {
50+ u8 reserved[2];
51+ u8 pending;
52+ u8 locked;
53+ };
54+#endif
4355 };
4456 };
4557
@@ -95,9 +107,6 @@ struct arena_qnode { · 49 unmodified lines
95107 #define _Q_LOCKED_VAL (1U << _Q_LOCKED_OFFSET)
96108 #define _Q_PENDING_VAL (1U << _Q_PENDING_OFFSET)
97109
98−#define likely(x) __builtin_expect(!!(x), 1)
99−#define unlikely(x) __builtin_expect(!!(x), 0)
100−
101110 struct arena_qnode __arena qnodes[_Q_MAX_CPUS][_Q_MAX_NODES];
102111
103112 static inline u32 encode_tail(int cpu, int idx)
tools/testing/selftests/bpf/progs/bpf_misc.h+3−2
@@ -225,8 +225,9 @@ · 224 unmodified lines
225225 #define CAN_USE_BPF_ST
226226 #endif
227227
228−#if __clang_major__ >= 18 && defined(ENABLE_ATOMICS_TESTS) && \
229− (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86))
228+#if __clang_major__ >= 18 && defined(ENABLE_ATOMICS_TESTS) && \
229+ (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \
230+ (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64))
230231 #define CAN_USE_LOAD_ACQ_STORE_REL
231232 #endif
232233
tools/testing/selftests/bpf/progs/dmabuf_iter.c+101−0
@@ -0,0 +1,101 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/* Copyright (c) 2025 Google LLC */
3+#include <vmlinux.h>
4+#include <bpf/bpf_core_read.h>
5+#include <bpf/bpf_helpers.h>
6+
7+/* From uapi/linux/dma-buf.h */
8+#define DMA_BUF_NAME_LEN 32
9+
10+char _license[] SEC("license") = "GPL";
11+
12+struct {
13+ __uint(type, BPF_MAP_TYPE_HASH);
14+ __uint(key_size, DMA_BUF_NAME_LEN);
15+ __type(value, bool);
16+ __uint(max_entries, 5);
17+} testbuf_hash SEC(".maps");
18+
19+/*
20+ * Fields output by this iterator are delimited by newlines. Convert any
21+ * newlines in user-provided printed strings to spaces.
22+ */
23+static void sanitize_string(char *src, size_t size)
24+{
25+ for (char *c = src; (size_t)(c - src) < size && *c; ++c)
26+ if (*c == '\n')
27+ *c = ' ';
28+}
29+
30+SEC("iter/dmabuf")
31+int dmabuf_collector(struct bpf_iter__dmabuf *ctx)
32+{
33+ const struct dma_buf *dmabuf = ctx->dmabuf;
34+ struct seq_file *seq = ctx->meta->seq;
35+ unsigned long inode = 0;
36+ size_t size;
37+ const char *pname, *exporter;
38+ char name[DMA_BUF_NAME_LEN] = {'\0'};
39+
40+ if (!dmabuf)
41+ return 0;
42+
43+ if (BPF_CORE_READ_INTO(&inode, dmabuf, file, f_inode, i_ino) ||
44+ bpf_core_read(&size, sizeof(size), &dmabuf->size) ||
45+ bpf_core_read(&pname, sizeof(pname), &dmabuf->name) ||
46+ bpf_core_read(&exporter, sizeof(exporter), &dmabuf->exp_name))
47+ return 1;
48+
49+ /* Buffers are not required to be named */
50+ if (pname) {
51+ if (bpf_probe_read_kernel(name, sizeof(name), pname))
52+ return 1;
53+
54+ /* Name strings can be provided by userspace */
55+ sanitize_string(name, sizeof(name));
56+ }
57+
58+ BPF_SEQ_PRINTF(seq, "%lu\n%llu\n%s\n%s\n", inode, size, name, exporter);
59+ return 0;
60+}
61+
62+SEC("syscall")
63+int iter_dmabuf_for_each(const void *ctx)
64+{
65+ struct dma_buf *d;
66+
67+ bpf_for_each(dmabuf, d) {
68+ char name[DMA_BUF_NAME_LEN];
69+ const char *pname;
70+ bool *found;
71+ long len;
72+ int i;
73+
74+ if (bpf_core_read(&pname, sizeof(pname), &d->name))
75+ return 1;
76+
77+ /* Buffers are not required to be named */
78+ if (!pname)
79+ continue;
80+
81+ len = bpf_probe_read_kernel_str(name, sizeof(name), pname);
82+ if (len < 0)
83+ return 1;
84+
85+ /*
86+ * The entire name buffer is used as a map key.
87+ * Zeroize any uninitialized trailing bytes after the NUL.
88+ */
89+ bpf_for(i, len, DMA_BUF_NAME_LEN)
90+ name[i] = 0;
91+
92+ found = bpf_map_lookup_elem(&testbuf_hash, name);
93+ if (found) {
94+ bool t = true;
95+
96+ bpf_map_update_elem(&testbuf_hash, name, &t, BPF_EXIST);
97+ }
98+ }
99+
100+ return 0;
101+}
tools/testing/selftests/bpf/progs/dynptr_success.c+230−0
@@ -680,3 +680,233 @@ out: · 679 unmodified lines
680680 bpf_ringbuf_discard_dynptr(&ptr_buf, 0);
681681 return XDP_DROP;
682682 }
683+
684+void *user_ptr;
685+/* Contains the copy of the data pointed by user_ptr.
686+ * Size 384 to make it not fit into a single kernel chunk when copying
687+ * but less than the maximum bpf stack size (512).
688+ */
689+char expected_str[384];
690+__u32 test_len[7] = {0/* placeholder */, 0, 1, 2, 255, 256, 257};
691+
692+typedef int (*bpf_read_dynptr_fn_t)(struct bpf_dynptr *dptr, u32 off,
693+ u32 size, const void *unsafe_ptr);
694+
695+/* Returns the offset just before the end of the maximum sized xdp fragment.
696+ * Any write larger than 32 bytes will be split between 2 fragments.
697+ */
698+__u32 xdp_near_frag_end_offset(void)
699+{
700+ const __u32 headroom = 256;
701+ const __u32 max_frag_size = __PAGE_SIZE - headroom - sizeof(struct skb_shared_info);
702+
703+ /* 32 bytes before the approximate end of the fragment */
704+ return max_frag_size - 32;
705+}
706+
707+/* Use __always_inline on test_dynptr_probe[_str][_xdp]() and callbacks
708+ * of type bpf_read_dynptr_fn_t to prevent compiler from generating
709+ * indirect calls that make program fail to load with "unknown opcode" error.
710+ */
711+static __always_inline void test_dynptr_probe(void *ptr, bpf_read_dynptr_fn_t bpf_read_dynptr_fn)
712+{
713+ char buf[sizeof(expected_str)];
714+ struct bpf_dynptr ptr_buf;
715+ int i;
716+
717+ if (bpf_get_current_pid_tgid() >> 32 != pid)
718+ return;
719+
720+ err = bpf_ringbuf_reserve_dynptr(&ringbuf, sizeof(buf), 0, &ptr_buf);
721+
722+ bpf_for(i, 0, ARRAY_SIZE(test_len)) {
723+ __u32 len = test_len[i];
724+
725+ err = err ?: bpf_read_dynptr_fn(&ptr_buf, 0, test_len[i], ptr);
726+ if (len > sizeof(buf))
727+ break;
728+ err = err ?: bpf_dynptr_read(&buf, len, &ptr_buf, 0, 0);
729+
730+ if (err || bpf_memcmp(expected_str, buf, len))
731+ err = 1;
732+
733+ /* Reset buffer and dynptr */
734+ __builtin_memset(buf, 0, sizeof(buf));
735+ err = err ?: bpf_dynptr_write(&ptr_buf, 0, buf, len, 0);
736+ }
737+ bpf_ringbuf_discard_dynptr(&ptr_buf, 0);
738+}
739+
740+static __always_inline void test_dynptr_probe_str(void *ptr,
741+ bpf_read_dynptr_fn_t bpf_read_dynptr_fn)
742+{
743+ char buf[sizeof(expected_str)];
744+ struct bpf_dynptr ptr_buf;
745+ __u32 cnt, i;
746+
747+ if (bpf_get_current_pid_tgid() >> 32 != pid)
748+ return;
749+
750+ bpf_ringbuf_reserve_dynptr(&ringbuf, sizeof(buf), 0, &ptr_buf);
751+
752+ bpf_for(i, 0, ARRAY_SIZE(test_len)) {
753+ __u32 len = test_len[i];
754+
755+ cnt = bpf_read_dynptr_fn(&ptr_buf, 0, len, ptr);
756+ if (cnt != len)
757+ err = 1;
758+
759+ if (len > sizeof(buf))
760+ continue;
761+ err = err ?: bpf_dynptr_read(&buf, len, &ptr_buf, 0, 0);
762+ if (!len)
763+ continue;
764+ if (err || bpf_memcmp(expected_str, buf, len - 1) || buf[len - 1] != '\0')
765+ err = 1;
766+ }
767+ bpf_ringbuf_discard_dynptr(&ptr_buf, 0);
768+}
769+
770+static __always_inline void test_dynptr_probe_xdp(struct xdp_md *xdp, void *ptr,
771+ bpf_read_dynptr_fn_t bpf_read_dynptr_fn)
772+{
773+ struct bpf_dynptr ptr_xdp;
774+ char buf[sizeof(expected_str)];
775+ __u32 off, i;
776+
777+ if (bpf_get_current_pid_tgid() >> 32 != pid)
778+ return;
779+
780+ off = xdp_near_frag_end_offset();
781+ err = bpf_dynptr_from_xdp(xdp, 0, &ptr_xdp);
782+
783+ bpf_for(i, 0, ARRAY_SIZE(test_len)) {
784+ __u32 len = test_len[i];
785+
786+ err = err ?: bpf_read_dynptr_fn(&ptr_xdp, off, len, ptr);
787+ if (len > sizeof(buf))
788+ continue;
789+ err = err ?: bpf_dynptr_read(&buf, len, &ptr_xdp, off, 0);
790+ if (err || bpf_memcmp(expected_str, buf, len))
791+ err = 1;
792+ /* Reset buffer and dynptr */
793+ __builtin_memset(buf, 0, sizeof(buf));
794+ err = err ?: bpf_dynptr_write(&ptr_xdp, off, buf, len, 0);
795+ }
796+}
797+
798+static __always_inline void test_dynptr_probe_str_xdp(struct xdp_md *xdp, void *ptr,
799+ bpf_read_dynptr_fn_t bpf_read_dynptr_fn)
800+{
801+ struct bpf_dynptr ptr_xdp;
802+ char buf[sizeof(expected_str)];
803+ __u32 cnt, off, i;
804+
805+ if (bpf_get_current_pid_tgid() >> 32 != pid)
806+ return;
807+
808+ off = xdp_near_frag_end_offset();
809+ err = bpf_dynptr_from_xdp(xdp, 0, &ptr_xdp);
810+ if (err)
811+ return;
812+
813+ bpf_for(i, 0, ARRAY_SIZE(test_len)) {
814+ __u32 len = test_len[i];
815+
816+ cnt = bpf_read_dynptr_fn(&ptr_xdp, off, len, ptr);
817+ if (cnt != len)
818+ err = 1;
819+
820+ if (len > sizeof(buf))
821+ continue;
822+ err = err ?: bpf_dynptr_read(&buf, len, &ptr_xdp, off, 0);
823+
824+ if (!len)
825+ continue;
826+ if (err || bpf_memcmp(expected_str, buf, len - 1) || buf[len - 1] != '\0')
827+ err = 1;
828+
829+ __builtin_memset(buf, 0, sizeof(buf));
830+ err = err ?: bpf_dynptr_write(&ptr_xdp, off, buf, len, 0);
831+ }
832+}
833+
834+SEC("xdp")
835+int test_probe_read_user_dynptr(struct xdp_md *xdp)
836+{
837+ test_dynptr_probe(user_ptr, bpf_probe_read_user_dynptr);
838+ if (!err)
839+ test_dynptr_probe_xdp(xdp, user_ptr, bpf_probe_read_user_dynptr);
840+ return XDP_PASS;
841+}
842+
843+SEC("xdp")
844+int test_probe_read_kernel_dynptr(struct xdp_md *xdp)
845+{
846+ test_dynptr_probe(expected_str, bpf_probe_read_kernel_dynptr);
847+ if (!err)
848+ test_dynptr_probe_xdp(xdp, expected_str, bpf_probe_read_kernel_dynptr);
849+ return XDP_PASS;
850+}
851+
852+SEC("xdp")
853+int test_probe_read_user_str_dynptr(struct xdp_md *xdp)
854+{
855+ test_dynptr_probe_str(user_ptr, bpf_probe_read_user_str_dynptr);
856+ if (!err)
857+ test_dynptr_probe_str_xdp(xdp, user_ptr, bpf_probe_read_user_str_dynptr);
858+ return XDP_PASS;
859+}
860+
861+SEC("xdp")
862+int test_probe_read_kernel_str_dynptr(struct xdp_md *xdp)
863+{
864+ test_dynptr_probe_str(expected_str, bpf_probe_read_kernel_str_dynptr);
865+ if (!err)
866+ test_dynptr_probe_str_xdp(xdp, expected_str, bpf_probe_read_kernel_str_dynptr);
867+ return XDP_PASS;
868+}
869+
870+SEC("fentry.s/" SYS_PREFIX "sys_nanosleep")
871+int test_copy_from_user_dynptr(void *ctx)
872+{
873+ test_dynptr_probe(user_ptr, bpf_copy_from_user_dynptr);
874+ return 0;
875+}
876+
877+SEC("fentry.s/" SYS_PREFIX "sys_nanosleep")
878+int test_copy_from_user_str_dynptr(void *ctx)
879+{
880+ test_dynptr_probe_str(user_ptr, bpf_copy_from_user_str_dynptr);
881+ return 0;
882+}
883+
884+static int bpf_copy_data_from_user_task(struct bpf_dynptr *dptr, u32 off,
885+ u32 size, const void *unsafe_ptr)
886+{
887+ struct task_struct *task = bpf_get_current_task_btf();
888+
889+ return bpf_copy_from_user_task_dynptr(dptr, off, size, unsafe_ptr, task);
890+}
891+
892+static int bpf_copy_data_from_user_task_str(struct bpf_dynptr *dptr, u32 off,
893+ u32 size, const void *unsafe_ptr)
894+{
895+ struct task_struct *task = bpf_get_current_task_btf();
896+
897+ return bpf_copy_from_user_task_str_dynptr(dptr, off, size, unsafe_ptr, task);
898+}
899+
900+SEC("fentry.s/" SYS_PREFIX "sys_nanosleep")
901+int test_copy_from_user_task_dynptr(void *ctx)
902+{
903+ test_dynptr_probe(user_ptr, bpf_copy_data_from_user_task);
904+ return 0;
905+}
906+
907+SEC("fentry.s/" SYS_PREFIX "sys_nanosleep")
908+int test_copy_from_user_task_str_dynptr(void *ctx)
909+{
910+ test_dynptr_probe_str(user_ptr, bpf_copy_data_from_user_task_str);
911+ return 0;
912+}
tools/testing/selftests/bpf/progs/fd_htab_lookup.c+25−0
@@ -0,0 +1,25 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/* Copyright (C) 2025. Huawei Technologies Co., Ltd */
3+#include <linux/bpf.h>
4+#include <bpf/bpf_helpers.h>
5+
6+char _license[] SEC("license") = "GPL";
7+
8+struct inner_map_type {
9+ __uint(type, BPF_MAP_TYPE_ARRAY);
10+ __uint(key_size, 4);
11+ __uint(value_size, 4);
12+ __uint(max_entries, 1);
13+} inner_map SEC(".maps");
14+
15+struct {
16+ __uint(type, BPF_MAP_TYPE_HASH_OF_MAPS);
17+ __uint(max_entries, 64);
18+ __type(key, int);
19+ __type(value, int);
20+ __array(values, struct inner_map_type);
21+} outer_map SEC(".maps") = {
22+ .values = {
23+ [0] = &inner_map,
24+ },
25+};
tools/testing/selftests/bpf/progs/iters.c+0−2
@@ -7,8 +7,6 @@ · 6 unmodified lines
77 #include "bpf_misc.h"
88 #include "bpf_compiler.h"
99
10−#define unlikely(x) __builtin_expect(!!(x), 0)
11−
1210 static volatile int zero = 0;
1311
1412 int my_pid;
tools/testing/selftests/bpf/progs/linked_list_peek.c+113−0
@@ -0,0 +1,113 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
3+
4+#include <vmlinux.h>
5+#include <bpf/bpf_helpers.h>
6+#include "bpf_misc.h"
7+#include "bpf_experimental.h"
8+
9+struct node_data {
10+ struct bpf_list_node l;
11+ int key;
12+};
13+
14+#define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8)))
15+private(A) struct bpf_spin_lock glock;
16+private(A) struct bpf_list_head ghead __contains(node_data, l);
17+
18+#define list_entry(ptr, type, member) container_of(ptr, type, member)
19+#define NR_NODES 16
20+
21+int zero = 0;
22+
23+SEC("syscall")
24+__retval(0)
25+long list_peek(void *ctx)
26+{
27+ struct bpf_list_node *l_n;
28+ struct node_data *n;
29+ int i, err = 0;
30+
31+ bpf_spin_lock(&glock);
32+ l_n = bpf_list_front(&ghead);
33+ bpf_spin_unlock(&glock);
34+ if (l_n)
35+ return __LINE__;
36+
37+ bpf_spin_lock(&glock);
38+ l_n = bpf_list_back(&ghead);
39+ bpf_spin_unlock(&glock);
40+ if (l_n)
41+ return __LINE__;
42+
43+ for (i = zero; i < NR_NODES && can_loop; i++) {
44+ n = bpf_obj_new(typeof(*n));
45+ if (!n)
46+ return __LINE__;
47+ n->key = i;
48+ bpf_spin_lock(&glock);
49+ bpf_list_push_back(&ghead, &n->l);
50+ bpf_spin_unlock(&glock);
51+ }
52+
53+ bpf_spin_lock(&glock);
54+
55+ l_n = bpf_list_front(&ghead);
56+ if (!l_n) {
57+ err = __LINE__;
58+ goto done;
59+ }
60+
61+ n = list_entry(l_n, struct node_data, l);
62+ if (n->key != 0) {
63+ err = __LINE__;
64+ goto done;
65+ }
66+
67+ l_n = bpf_list_back(&ghead);
68+ if (!l_n) {
69+ err = __LINE__;
70+ goto done;
71+ }
72+
73+ n = list_entry(l_n, struct node_data, l);
74+ if (n->key != NR_NODES - 1) {
75+ err = __LINE__;
76+ goto done;
77+ }
78+
79+done:
80+ bpf_spin_unlock(&glock);
81+ return err;
82+}
83+
84+#define TEST_FB(op, dolock) \
85+SEC("syscall") \
86+__failure __msg(MSG) \
87+long test_##op##_spinlock_##dolock(void *ctx) \
88+{ \
89+ struct bpf_list_node *l_n; \
90+ __u64 jiffies = 0; \
91+ \
92+ if (dolock) \
93+ bpf_spin_lock(&glock); \
94+ l_n = bpf_list_##op(&ghead); \
95+ if (l_n) \
96+ jiffies = bpf_jiffies64(); \
97+ if (dolock) \
98+ bpf_spin_unlock(&glock); \
99+ \
100+ return !!jiffies; \
101+}
102+
103+#define MSG "call bpf_list_{{(front|back).+}}; R0{{(_w)?}}=ptr_or_null_node_data(id={{[0-9]+}},non_own_ref"
104+TEST_FB(front, true)
105+TEST_FB(back, true)
106+#undef MSG
107+
108+#define MSG "bpf_spin_lock at off=0 must be held for bpf_list_head"
109+TEST_FB(front, false)
110+TEST_FB(back, false)
111+#undef MSG
112+
113+char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/prepare.c+0−1
@@ -2,7 +2,6 @@ · 1 unmodified line
22 /* Copyright (c) 2025 Meta */
33 #include <vmlinux.h>
44 #include <bpf/bpf_helpers.h>
5−//#include <bpf/bpf_tracing.h>
65
76 char _license[] SEC("license") = "GPL";
87
tools/testing/selftests/bpf/progs/rbtree_fail.c+15−14
@@ -69,11 +69,11 @@ long rbtree_api_nolock_first(void *ctx) · 68 unmodified lines
6969 }
7070
7171 SEC("?tc")
72−__failure __msg("rbtree_remove node input must be non-owning ref")
72+__retval(0)
7373 long rbtree_api_remove_unadded_node(void *ctx)
7474 {
7575 struct node_data *n, *m;
76− struct bpf_rb_node *res;
76+ struct bpf_rb_node *res_n, *res_m;
7777
7878 n = bpf_obj_new(typeof(*n));
7979 if (!n)
@@ -88,19 +88,20 @@ long rbtree_api_remove_unadded_node(void *ctx) · 8 unmodified lines
8888 bpf_spin_lock(&glock);
8989 bpf_rbtree_add(&groot, &n->node, less);
9090
91− /* This remove should pass verifier */
92− res = bpf_rbtree_remove(&groot, &n->node);
93− n = container_of(res, struct node_data, node);
91+ res_n = bpf_rbtree_remove(&groot, &n->node);
9492
95− /* This remove shouldn't, m isn't in an rbtree */
96− res = bpf_rbtree_remove(&groot, &m->node);
97− m = container_of(res, struct node_data, node);
93+ res_m = bpf_rbtree_remove(&groot, &m->node);
9894 bpf_spin_unlock(&glock);
9995
100− if (n)
101− bpf_obj_drop(n);
102− if (m)
103− bpf_obj_drop(m);
96+ bpf_obj_drop(m);
97+ if (res_n)
98+ bpf_obj_drop(container_of(res_n, struct node_data, node));
99+ if (res_m) {
100+ bpf_obj_drop(container_of(res_m, struct node_data, node));
101+ /* m was not added to the rbtree */
102+ return 2;
103+ }
104+
104105 return 0;
105106 }
106107
@@ -178,7 +179,7 @@ err_out: · 71 unmodified lines
178179 }
179180
180181 SEC("?tc")
181−__failure __msg("rbtree_remove node input must be non-owning ref")
182+__failure __msg("bpf_rbtree_remove can only take non-owning or refcounted bpf_rb_node pointer")
182183 long rbtree_api_add_release_unlock_escape(void *ctx)
183184 {
184185 struct node_data *n;
@@ -202,7 +203,7 @@ long rbtree_api_add_release_unlock_escape(void *ctx) · 17 unmodified lines
202203 }
203204
204205 SEC("?tc")
205−__failure __msg("rbtree_remove node input must be non-owning ref")
206+__failure __msg("bpf_rbtree_remove can only take non-owning or refcounted bpf_rb_node pointer")
206207 long rbtree_api_first_release_unlock_escape(void *ctx)
207208 {
208209 struct bpf_rb_node *res;
tools/testing/selftests/bpf/progs/rbtree_search.c+206−0
@@ -0,0 +1,206 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
3+
4+#include <vmlinux.h>
5+#include <bpf/bpf_helpers.h>
6+#include "bpf_misc.h"
7+#include "bpf_experimental.h"
8+
9+struct node_data {
10+ struct bpf_refcount ref;
11+ struct bpf_rb_node r0;
12+ struct bpf_rb_node r1;
13+ int key0;
14+ int key1;
15+};
16+
17+#define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8)))
18+private(A) struct bpf_spin_lock glock0;
19+private(A) struct bpf_rb_root groot0 __contains(node_data, r0);
20+
21+private(B) struct bpf_spin_lock glock1;
22+private(B) struct bpf_rb_root groot1 __contains(node_data, r1);
23+
24+#define rb_entry(ptr, type, member) container_of(ptr, type, member)
25+#define NR_NODES 16
26+
27+int zero = 0;
28+
29+static bool less0(struct bpf_rb_node *a, const struct bpf_rb_node *b)
30+{
31+ struct node_data *node_a;
32+ struct node_data *node_b;
33+
34+ node_a = rb_entry(a, struct node_data, r0);
35+ node_b = rb_entry(b, struct node_data, r0);
36+
37+ return node_a->key0 < node_b->key0;
38+}
39+
40+static bool less1(struct bpf_rb_node *a, const struct bpf_rb_node *b)
41+{
42+ struct node_data *node_a;
43+ struct node_data *node_b;
44+
45+ node_a = rb_entry(a, struct node_data, r1);
46+ node_b = rb_entry(b, struct node_data, r1);
47+
48+ return node_a->key1 < node_b->key1;
49+}
50+
51+SEC("syscall")
52+__retval(0)
53+long rbtree_search(void *ctx)
54+{
55+ struct bpf_rb_node *rb_n, *rb_m, *gc_ns[NR_NODES];
56+ long lookup_key = NR_NODES / 2;
57+ struct node_data *n, *m;
58+ int i, nr_gc = 0;
59+
60+ for (i = zero; i < NR_NODES && can_loop; i++) {
61+ n = bpf_obj_new(typeof(*n));
62+ if (!n)
63+ return __LINE__;
64+
65+ m = bpf_refcount_acquire(n);
66+
67+ n->key0 = i;
68+ m->key1 = i;
69+
70+ bpf_spin_lock(&glock0);
71+ bpf_rbtree_add(&groot0, &n->r0, less0);
72+ bpf_spin_unlock(&glock0);
73+
74+ bpf_spin_lock(&glock1);
75+ bpf_rbtree_add(&groot1, &m->r1, less1);
76+ bpf_spin_unlock(&glock1);
77+ }
78+
79+ n = NULL;
80+ bpf_spin_lock(&glock0);
81+ rb_n = bpf_rbtree_root(&groot0);
82+ while (can_loop) {
83+ if (!rb_n) {
84+ bpf_spin_unlock(&glock0);
85+ return __LINE__;
86+ }
87+
88+ n = rb_entry(rb_n, struct node_data, r0);
89+ if (lookup_key == n->key0)
90+ break;
91+ if (nr_gc < NR_NODES)
92+ gc_ns[nr_gc++] = rb_n;
93+ if (lookup_key < n->key0)
94+ rb_n = bpf_rbtree_left(&groot0, rb_n);
95+ else
96+ rb_n = bpf_rbtree_right(&groot0, rb_n);
97+ }
98+
99+ if (!n || lookup_key != n->key0) {
100+ bpf_spin_unlock(&glock0);
101+ return __LINE__;
102+ }
103+
104+ for (i = 0; i < nr_gc; i++) {
105+ rb_n = gc_ns[i];
106+ gc_ns[i] = bpf_rbtree_remove(&groot0, rb_n);
107+ }
108+
109+ m = bpf_refcount_acquire(n);
110+ bpf_spin_unlock(&glock0);
111+
112+ for (i = 0; i < nr_gc; i++) {
113+ rb_n = gc_ns[i];
114+ if (rb_n) {
115+ n = rb_entry(rb_n, struct node_data, r0);
116+ bpf_obj_drop(n);
117+ }
118+ }
119+
120+ if (!m)
121+ return __LINE__;
122+
123+ bpf_spin_lock(&glock1);
124+ rb_m = bpf_rbtree_remove(&groot1, &m->r1);
125+ bpf_spin_unlock(&glock1);
126+ bpf_obj_drop(m);
127+ if (!rb_m)
128+ return __LINE__;
129+ bpf_obj_drop(rb_entry(rb_m, struct node_data, r1));
130+
131+ return 0;
132+}
133+
134+#define TEST_ROOT(dolock) \
135+SEC("syscall") \
136+__failure __msg(MSG) \
137+long test_root_spinlock_##dolock(void *ctx) \
138+{ \
139+ struct bpf_rb_node *rb_n; \
140+ __u64 jiffies = 0; \
141+ \
142+ if (dolock) \
143+ bpf_spin_lock(&glock0); \
144+ rb_n = bpf_rbtree_root(&groot0); \
145+ if (rb_n) \
146+ jiffies = bpf_jiffies64(); \
147+ if (dolock) \
148+ bpf_spin_unlock(&glock0); \
149+ \
150+ return !!jiffies; \
151+}
152+
153+#define TEST_LR(op, dolock) \
154+SEC("syscall") \
155+__failure __msg(MSG) \
156+long test_##op##_spinlock_##dolock(void *ctx) \
157+{ \
158+ struct bpf_rb_node *rb_n; \
159+ struct node_data *n; \
160+ __u64 jiffies = 0; \
161+ \
162+ bpf_spin_lock(&glock0); \
163+ rb_n = bpf_rbtree_root(&groot0); \
164+ if (!rb_n) { \
165+ bpf_spin_unlock(&glock0); \
166+ return 1; \
167+ } \
168+ n = rb_entry(rb_n, struct node_data, r0); \
169+ n = bpf_refcount_acquire(n); \
170+ bpf_spin_unlock(&glock0); \
171+ if (!n) \
172+ return 1; \
173+ \
174+ if (dolock) \
175+ bpf_spin_lock(&glock0); \
176+ rb_n = bpf_rbtree_##op(&groot0, &n->r0); \
177+ if (rb_n) \
178+ jiffies = bpf_jiffies64(); \
179+ if (dolock) \
180+ bpf_spin_unlock(&glock0); \
181+ \
182+ return !!jiffies; \
183+}
184+
185+/*
186+ * Use a spearate MSG macro instead of passing to TEST_XXX(..., MSG)
187+ * to ensure the message itself is not in the bpf prog lineinfo
188+ * which the verifier includes in its log.
189+ * Otherwise, the test_loader will incorrectly match the prog lineinfo
190+ * instead of the log generated by the verifier.
191+ */
192+#define MSG "call bpf_rbtree_root{{.+}}; R0{{(_w)?}}=rcu_ptr_or_null_node_data(id={{[0-9]+}},non_own_ref"
193+TEST_ROOT(true)
194+#undef MSG
195+#define MSG "call bpf_rbtree_{{(left|right).+}}; R0{{(_w)?}}=rcu_ptr_or_null_node_data(id={{[0-9]+}},non_own_ref"
196+TEST_LR(left, true)
197+TEST_LR(right, true)
198+#undef MSG
199+
200+#define MSG "bpf_spin_lock at off=0 must be held for bpf_rb_root"
201+TEST_ROOT(false)
202+TEST_LR(left, false)
203+TEST_LR(right, false)
204+#undef MSG
205+
206+char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/set_global_vars.c+41−0
@@ -24,6 +24,44 @@ const volatile enum Enumu64 var_eb = EB1; · 23 unmodified lines
2424 const volatile enum Enums64 var_ec = EC1;
2525 const volatile bool var_b = false;
2626
27+struct Struct {
28+ int:16;
29+ __u16 filler;
30+ struct {
31+ const __u16 filler2;
32+ };
33+ struct Struct2 {
34+ __u16 filler;
35+ volatile struct {
36+ const int:1;
37+ union {
38+ const volatile __u8 var_u8;
39+ const volatile __s16 filler3;
40+ const int:1;
41+ } u;
42+ };
43+ } struct2;
44+};
45+
46+const volatile __u32 stru = 0; /* same prefix as below */
47+const volatile struct Struct struct1 = {.struct2 = {.u = {.var_u8 = 1}}};
48+
49+union Union {
50+ __u16 var_u16;
51+ struct Struct3 {
52+ struct {
53+ __u8 var_u8_l;
54+ };
55+ struct {
56+ struct {
57+ __u8 var_u8_h;
58+ };
59+ };
60+ } struct3;
61+};
62+
63+const volatile union Union union1 = {.var_u16 = -1};
64+
2765 char arr[4] = {0};
2866
2967 SEC("socket")
@@ -43,5 +81,8 @@ int test_set_globals(void *ctx) · 13 unmodified lines
4381 a = var_eb;
4482 a = var_ec;
4583 a = var_b;
84+ a = struct1.struct2.u.var_u8;
85+ a = union1.var_u16;
86+
4687 return a;
4788 }
tools/testing/selftests/bpf/progs/test_btf_ext.c+22−0
@@ -0,0 +1,22 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+/* Copyright (c) 2025 Meta Platforms Inc. */
3+
4+#include <linux/bpf.h>
5+#include <bpf/bpf_helpers.h>
6+#include "bpf_misc.h"
7+
8+char _license[] SEC("license") = "GPL";
9+
10+__noinline static void f0(void)
11+{
12+ __u64 a = 1;
13+
14+ __sink(a);
15+}
16+
17+SEC("xdp")
18+__u64 global_func(struct xdp_md *xdp)
19+{
20+ f0();
21+ return XDP_DROP;
22+}
tools/testing/selftests/bpf/progs/test_sockmap_ktls.c+36−0
@@ -0,0 +1,36 @@
1+// SPDX-License-Identifier: GPL-2.0
2+#include <linux/bpf.h>
3+#include <bpf/bpf_helpers.h>
4+#include <bpf/bpf_endian.h>
5+
6+int cork_byte;
7+int push_start;
8+int push_end;
9+int apply_bytes;
10+
11+struct {
12+ __uint(type, BPF_MAP_TYPE_SOCKMAP);
13+ __uint(max_entries, 20);
14+ __type(key, int);
15+ __type(value, int);
16+} sock_map SEC(".maps");
17+
18+SEC("sk_msg")
19+int prog_sk_policy(struct sk_msg_md *msg)
20+{
21+ if (cork_byte > 0)
22+ bpf_msg_cork_bytes(msg, cork_byte);
23+ if (push_start > 0 && push_end > 0)
24+ bpf_msg_push_data(msg, push_start, push_end, 0);
25+
26+ return SK_PASS;
27+}
28+
29+SEC("sk_msg")
30+int prog_sk_policy_redir(struct sk_msg_md *msg)
31+{
32+ int two = 2;
33+
34+ bpf_msg_apply_bytes(msg, apply_bytes);
35+ return bpf_msg_redirect_map(msg, &sock_map, two, 0);
36+}
tools/testing/selftests/bpf/progs/test_sockmap_redir.c+68−0
@@ -0,0 +1,68 @@
1+// SPDX-License-Identifier: GPL-2.0
2+
3+#include <linux/bpf.h>
4+#include <bpf/bpf_helpers.h>
5+#include "bpf_misc.h"
6+
7+SEC(".maps") struct {
8+ __uint(type, BPF_MAP_TYPE_SOCKMAP);
9+ __uint(max_entries, 1);
10+ __type(key, __u32);
11+ __type(value, __u64);
12+} nop_map, sock_map;
13+
14+SEC(".maps") struct {
15+ __uint(type, BPF_MAP_TYPE_SOCKHASH);
16+ __uint(max_entries, 1);
17+ __type(key, __u32);
18+ __type(value, __u64);
19+} nop_hash, sock_hash;
20+
21+SEC(".maps") struct {
22+ __uint(type, BPF_MAP_TYPE_ARRAY);
23+ __uint(max_entries, 2);
24+ __type(key, int);
25+ __type(value, unsigned int);
26+} verdict_map;
27+
28+/* Set by user space */
29+int redirect_type;
30+int redirect_flags;
31+
32+#define redirect_map(__data) \
33+ _Generic((__data), \
34+ struct __sk_buff * : bpf_sk_redirect_map, \
35+ struct sk_msg_md * : bpf_msg_redirect_map \
36+ )((__data), &sock_map, (__u32){0}, redirect_flags)
37+
38+#define redirect_hash(__data) \
39+ _Generic((__data), \
40+ struct __sk_buff * : bpf_sk_redirect_hash, \
41+ struct sk_msg_md * : bpf_msg_redirect_hash \
42+ )((__data), &sock_hash, &(__u32){0}, redirect_flags)
43+
44+#define DEFINE_PROG(__type, __param) \
45+SEC("sk_" XSTR(__type)) \
46+int prog_ ## __type ## _verdict(__param data) \
47+{ \
48+ unsigned int *count; \
49+ int verdict; \
50+ \
51+ if (redirect_type == BPF_MAP_TYPE_SOCKMAP) \
52+ verdict = redirect_map(data); \
53+ else if (redirect_type == BPF_MAP_TYPE_SOCKHASH) \
54+ verdict = redirect_hash(data); \
55+ else \
56+ verdict = redirect_type - __MAX_BPF_MAP_TYPE; \
57+ \
58+ count = bpf_map_lookup_elem(&verdict_map, &verdict); \
59+ if (count) \
60+ (*count)++; \
61+ \
62+ return verdict; \
63+}
64+
65+DEFINE_PROG(skb, struct __sk_buff *);
66+DEFINE_PROG(msg, struct sk_msg_md *);
67+
68+char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/test_tcp_custom_syncookie.c+3−1
@@ -294,7 +294,9 @@ static int tcp_validate_sysctl(struct tcp_syncookie *ctx) · 293 unmodified lines
294294 (ctx->ipv6 && ctx->attrs.mss != MSS_LOCAL_IPV6))
295295 goto err;
296296
297− if (!ctx->attrs.wscale_ok || ctx->attrs.snd_wscale != 7)
297+ if (!ctx->attrs.wscale_ok ||
298+ !ctx->attrs.snd_wscale ||
299+ ctx->attrs.snd_wscale >= BPF_SYNCOOKIE_WSCALE_MASK)
298300 goto err;
299301
300302 if (!ctx->attrs.tstamp_ok)
tools/testing/selftests/bpf/progs/verifier_bpf_trap.c+71−0
@@ -0,0 +1,71 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
3+#include <vmlinux.h>
4+#include <bpf/bpf_helpers.h>
5+#include "bpf_misc.h"
6+
7+#if __clang_major__ >= 21 && 0
8+SEC("socket")
9+__description("__builtin_trap with simple c code")
10+__failure __msg("unexpected __bpf_trap() due to uninitialized variable?")
11+void bpf_builtin_trap_with_simple_c(void)
12+{
13+ __builtin_trap();
14+}
15+#endif
16+
17+SEC("socket")
18+__description("__bpf_trap with simple c code")
19+__failure __msg("unexpected __bpf_trap() due to uninitialized variable?")
20+void bpf_trap_with_simple_c(void)
21+{
22+ __bpf_trap();
23+}
24+
25+SEC("socket")
26+__description("__bpf_trap as the second-from-last insn")
27+__failure __msg("unexpected __bpf_trap() due to uninitialized variable?")
28+__naked void bpf_trap_at_func_end(void)
29+{
30+ asm volatile (
31+ "r0 = 0;"
32+ "call %[__bpf_trap];"
33+ "exit;"
34+ :
35+ : __imm(__bpf_trap)
36+ : __clobber_all);
37+}
38+
39+SEC("socket")
40+__description("dead code __bpf_trap in the middle of code")
41+__success
42+__naked void dead_bpf_trap_in_middle(void)
43+{
44+ asm volatile (
45+ "r0 = 0;"
46+ "if r0 == 0 goto +1;"
47+ "call %[__bpf_trap];"
48+ "r0 = 2;"
49+ "exit;"
50+ :
51+ : __imm(__bpf_trap)
52+ : __clobber_all);
53+}
54+
55+SEC("socket")
56+__description("reachable __bpf_trap in the middle of code")
57+__failure __msg("unexpected __bpf_trap() due to uninitialized variable?")
58+__naked void live_bpf_trap_in_middle(void)
59+{
60+ asm volatile (
61+ "r0 = 0;"
62+ "if r0 == 1 goto +1;"
63+ "call %[__bpf_trap];"
64+ "r0 = 2;"
65+ "exit;"
66+ :
67+ : __imm(__bpf_trap)
68+ : __clobber_all);
69+}
70+
71+char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/verifier_btf_ctx_access.c+12−0
@@ -65,4 +65,16 @@ __naked void ctx_access_u32_pointer_reject_8(void) · 64 unmodified lines
6565 " ::: __clobber_all);
6666 }
6767
68+SEC("fentry/bpf_fentry_test10")
69+__description("btf_ctx_access const void pointer accept")
70+__success __retval(0)
71+__naked void ctx_access_const_void_pointer_accept(void)
72+{
73+ asm volatile (" \
74+ r2 = *(u64 *)(r1 + 0); /* load 1st argument value (const void pointer) */\
75+ r0 = 0; \
76+ exit; \
77+" ::: __clobber_all);
78+}
79+
6880 char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/verifier_load_acquire.c+32−16
@@ -10,65 +10,81 @@ · 9 unmodified lines
1010
1111 SEC("socket")
1212 __description("load-acquire, 8-bit")
13−__success __success_unpriv __retval(0x12)
13+__success __success_unpriv __retval(0)
1414 __naked void load_acquire_8(void)
1515 {
1616 asm volatile (
17− "w1 = 0x12;"
17+ "r0 = 0;"
18+ "w1 = 0xfe;"
1819 "*(u8 *)(r10 - 1) = w1;"
19− ".8byte %[load_acquire_insn];" // w0 = load_acquire((u8 *)(r10 - 1));
20+ ".8byte %[load_acquire_insn];" // w2 = load_acquire((u8 *)(r10 - 1));
21+ "if r2 == r1 goto 1f;"
22+ "r0 = 1;"
23+"1:"
2024 "exit;"
2125 :
2226 : __imm_insn(load_acquire_insn,
23− BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -1))
27+ BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -1))
2428 : __clobber_all);
2529 }
2630
2731 SEC("socket")
2832 __description("load-acquire, 16-bit")
29−__success __success_unpriv __retval(0x1234)
33+__success __success_unpriv __retval(0)
3034 __naked void load_acquire_16(void)
3135 {
3236 asm volatile (
33− "w1 = 0x1234;"
37+ "r0 = 0;"
38+ "w1 = 0xfedc;"
3439 "*(u16 *)(r10 - 2) = w1;"
35− ".8byte %[load_acquire_insn];" // w0 = load_acquire((u16 *)(r10 - 2));
40+ ".8byte %[load_acquire_insn];" // w2 = load_acquire((u16 *)(r10 - 2));
41+ "if r2 == r1 goto 1f;"
42+ "r0 = 1;"
43+"1:"
3644 "exit;"
3745 :
3846 : __imm_insn(load_acquire_insn,
39− BPF_ATOMIC_OP(BPF_H, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -2))
47+ BPF_ATOMIC_OP(BPF_H, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -2))
4048 : __clobber_all);
4149 }
4250
4351 SEC("socket")
4452 __description("load-acquire, 32-bit")
45−__success __success_unpriv __retval(0x12345678)
53+__success __success_unpriv __retval(0)
4654 __naked void load_acquire_32(void)
4755 {
4856 asm volatile (
49− "w1 = 0x12345678;"
57+ "r0 = 0;"
58+ "w1 = 0xfedcba09;"
5059 "*(u32 *)(r10 - 4) = w1;"
51− ".8byte %[load_acquire_insn];" // w0 = load_acquire((u32 *)(r10 - 4));
60+ ".8byte %[load_acquire_insn];" // w2 = load_acquire((u32 *)(r10 - 4));
61+ "if r2 == r1 goto 1f;"
62+ "r0 = 1;"
63+"1:"
5264 "exit;"
5365 :
5466 : __imm_insn(load_acquire_insn,
55− BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -4))
67+ BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -4))
5668 : __clobber_all);
5769 }
5870
5971 SEC("socket")
6072 __description("load-acquire, 64-bit")
61−__success __success_unpriv __retval(0x1234567890abcdef)
73+__success __success_unpriv __retval(0)
6274 __naked void load_acquire_64(void)
6375 {
6476 asm volatile (
65− "r1 = 0x1234567890abcdef ll;"
77+ "r0 = 0;"
78+ "r1 = 0xfedcba0987654321 ll;"
6679 "*(u64 *)(r10 - 8) = r1;"
67− ".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r10 - 8));
80+ ".8byte %[load_acquire_insn];" // r2 = load_acquire((u64 *)(r10 - 8));
81+ "if r2 == r1 goto 1f;"
82+ "r0 = 1;"
83+"1:"
6884 "exit;"
6985 :
7086 : __imm_insn(load_acquire_insn,
71− BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -8))
87+ BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -8))
7288 : __clobber_all);
7389 }
7490
tools/testing/selftests/bpf/progs/verifier_precision.c+55−3
@@ -91,8 +91,7 @@ __naked int bpf_end_bswap(void) · 90 unmodified lines
9191 ::: __clobber_all);
9292 }
9393
94−#if defined(ENABLE_ATOMICS_TESTS) && \
95− (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86))
94+#ifdef CAN_USE_LOAD_ACQ_STORE_REL
9695
9796 SEC("?raw_tp")
9897 __success __log_level(2)
@@ -138,7 +137,7 @@ __naked int bpf_store_release(void) · 39 unmodified lines
138137 : __clobber_all);
139138 }
140139
141−#endif /* load-acquire, store-release */
140+#endif /* CAN_USE_LOAD_ACQ_STORE_REL */
142141 #endif /* v4 instruction */
143142
144143 SEC("?raw_tp")
@@ -179,4 +178,57 @@ __naked int state_loop_first_last_equal(void) · 34 unmodified lines
179178 );
180179 }
181180
181+__used __naked static void __bpf_cond_op_r10(void)
182+{
183+ asm volatile (
184+ "r2 = 2314885393468386424 ll;"
185+ "goto +0;"
186+ "if r2 <= r10 goto +3;"
187+ "if r1 >= -1835016 goto +0;"
188+ "if r2 <= 8 goto +0;"
189+ "if r3 <= 0 goto +0;"
190+ "exit;"
191+ ::: __clobber_all);
192+}
193+
194+SEC("?raw_tp")
195+__success __log_level(2)
196+__msg("8: (bd) if r2 <= r10 goto pc+3")
197+__msg("9: (35) if r1 >= 0xffe3fff8 goto pc+0")
198+__msg("10: (b5) if r2 <= 0x8 goto pc+0")
199+__msg("mark_precise: frame1: last_idx 10 first_idx 0 subseq_idx -1")
200+__msg("mark_precise: frame1: regs=r2 stack= before 9: (35) if r1 >= 0xffe3fff8 goto pc+0")
201+__msg("mark_precise: frame1: regs=r2 stack= before 8: (bd) if r2 <= r10 goto pc+3")
202+__msg("mark_precise: frame1: regs=r2 stack= before 7: (05) goto pc+0")
203+__naked void bpf_cond_op_r10(void)
204+{
205+ asm volatile (
206+ "r3 = 0 ll;"
207+ "call __bpf_cond_op_r10;"
208+ "r0 = 0;"
209+ "exit;"
210+ ::: __clobber_all);
211+}
212+
213+SEC("?raw_tp")
214+__success __log_level(2)
215+__msg("3: (bf) r3 = r10")
216+__msg("4: (bd) if r3 <= r2 goto pc+1")
217+__msg("5: (b5) if r2 <= 0x8 goto pc+2")
218+__msg("mark_precise: frame0: last_idx 5 first_idx 0 subseq_idx -1")
219+__msg("mark_precise: frame0: regs=r2 stack= before 4: (bd) if r3 <= r2 goto pc+1")
220+__msg("mark_precise: frame0: regs=r2 stack= before 3: (bf) r3 = r10")
221+__naked void bpf_cond_op_not_r10(void)
222+{
223+ asm volatile (
224+ "r0 = 0;"
225+ "r2 = 2314885393468386424 ll;"
226+ "r3 = r10;"
227+ "if r3 <= r2 goto +1;"
228+ "if r2 <= 8 goto +2;"
229+ "r0 = 2 ll;"
230+ "exit;"
231+ ::: __clobber_all);
232+}
233+
182234 char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/verifier_store_release.c+27−12
@@ -6,18 +6,21 @@ · 5 unmodified lines
66 #include "../../../include/linux/filter.h"
77 #include "bpf_misc.h"
88
9−#if __clang_major__ >= 18 && defined(ENABLE_ATOMICS_TESTS) && \
10− (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86))
9+#ifdef CAN_USE_LOAD_ACQ_STORE_REL
1110
1211 SEC("socket")
1312 __description("store-release, 8-bit")
14−__success __success_unpriv __retval(0x12)
13+__success __success_unpriv __retval(0)
1514 __naked void store_release_8(void)
1615 {
1716 asm volatile (
17+ "r0 = 0;"
1818 "w1 = 0x12;"
1919 ".8byte %[store_release_insn];" // store_release((u8 *)(r10 - 1), w1);
20− "w0 = *(u8 *)(r10 - 1);"
20+ "w2 = *(u8 *)(r10 - 1);"
21+ "if r2 == r1 goto 1f;"
22+ "r0 = 1;"
23+"1:"
2124 "exit;"
2225 :
2326 : __imm_insn(store_release_insn,
@@ -27,13 +30,17 @@ __naked void store_release_8(void) · 3 unmodified lines
2730
2831 SEC("socket")
2932 __description("store-release, 16-bit")
30−__success __success_unpriv __retval(0x1234)
33+__success __success_unpriv __retval(0)
3134 __naked void store_release_16(void)
3235 {
3336 asm volatile (
37+ "r0 = 0;"
3438 "w1 = 0x1234;"
3539 ".8byte %[store_release_insn];" // store_release((u16 *)(r10 - 2), w1);
36− "w0 = *(u16 *)(r10 - 2);"
40+ "w2 = *(u16 *)(r10 - 2);"
41+ "if r2 == r1 goto 1f;"
42+ "r0 = 1;"
43+"1:"
3744 "exit;"
3845 :
3946 : __imm_insn(store_release_insn,
@@ -43,13 +50,17 @@ __naked void store_release_16(void) · 3 unmodified lines
4350
4451 SEC("socket")
4552 __description("store-release, 32-bit")
46−__success __success_unpriv __retval(0x12345678)
53+__success __success_unpriv __retval(0)
4754 __naked void store_release_32(void)
4855 {
4956 asm volatile (
57+ "r0 = 0;"
5058 "w1 = 0x12345678;"
5159 ".8byte %[store_release_insn];" // store_release((u32 *)(r10 - 4), w1);
52− "w0 = *(u32 *)(r10 - 4);"
60+ "w2 = *(u32 *)(r10 - 4);"
61+ "if r2 == r1 goto 1f;"
62+ "r0 = 1;"
63+"1:"
5364 "exit;"
5465 :
5566 : __imm_insn(store_release_insn,
@@ -59,13 +70,17 @@ __naked void store_release_32(void) · 3 unmodified lines
5970
6071 SEC("socket")
6172 __description("store-release, 64-bit")
62−__success __success_unpriv __retval(0x1234567890abcdef)
73+__success __success_unpriv __retval(0)
6374 __naked void store_release_64(void)
6475 {
6576 asm volatile (
77+ "r0 = 0;"
6678 "r1 = 0x1234567890abcdef ll;"
6779 ".8byte %[store_release_insn];" // store_release((u64 *)(r10 - 8), r1);
68− "r0 = *(u64 *)(r10 - 8);"
80+ "r2 = *(u64 *)(r10 - 8);"
81+ "if r2 == r1 goto 1f;"
82+ "r0 = 1;"
83+"1:"
6984 "exit;"
7085 :
7186 : __imm_insn(store_release_insn,
@@ -271,7 +286,7 @@ __naked void store_release_with_invalid_reg(void) · 199 unmodified lines
271286 : __clobber_all);
272287 }
273288
274−#else
289+#else /* CAN_USE_LOAD_ACQ_STORE_REL */
275290
276291 SEC("socket")
277292 __description("Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support store-release, use a dummy test")
@@ -281,6 +296,6 @@ int dummy_test(void) · 3 unmodified lines
281296 return 0;
282297 }
283298
284−#endif
299+#endif /* CAN_USE_LOAD_ACQ_STORE_REL */
285300
286301 char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/xdp_metadata.c+13−0
@@ -19,6 +19,13 @@ struct { · 18 unmodified lines
1919 __type(value, __u32);
2020 } prog_arr SEC(".maps");
2121
22+struct {
23+ __uint(type, BPF_MAP_TYPE_DEVMAP);
24+ __uint(key_size, sizeof(__u32));
25+ __uint(value_size, sizeof(struct bpf_devmap_val));
26+ __uint(max_entries, 1);
27+} dev_map SEC(".maps");
28+
2229 extern int bpf_xdp_metadata_rx_timestamp(const struct xdp_md *ctx,
2330 __u64 *timestamp) __ksym;
2431 extern int bpf_xdp_metadata_rx_hash(const struct xdp_md *ctx, __u32 *hash,
@@ -95,4 +102,10 @@ int rx(struct xdp_md *ctx) · 70 unmodified lines
95102 return bpf_redirect_map(&xsk, ctx->rx_queue_index, XDP_PASS);
96103 }
97104
105+SEC("xdp")
106+int redirect(struct xdp_md *ctx)
107+{
108+ return bpf_redirect_map(&dev_map, ctx->rx_queue_index, XDP_PASS);
109+}
110+
98111 char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/test_kmods/bpf_testmod.c+6−2
@@ -134,6 +134,10 @@ bpf_testmod_test_arg_ptr_to_struct(struct bpf_testmod_struct_arg_1 *a) { · 133 unmodified lines
134134 return bpf_testmod_test_struct_arg_result;
135135 }
136136
137+__weak noinline void bpf_testmod_looooooooooooooooooooooooooooooong_name(void)
138+{
139+}
140+
137141 __bpf_kfunc void
138142 bpf_testmod_test_mod_kfunc(int i)
139143 {
@@ -1340,7 +1344,7 @@ static int st_ops_gen_prologue_with_kfunc(struct bpf_insn *insn_buf, bool direct · 1200 unmodified lines
13401344 *insn++ = BPF_STX_MEM(BPF_DW, BPF_REG_6, BPF_REG_7, offsetof(struct st_ops_args, a));
13411345 *insn++ = BPF_JMP_IMM(BPF_JA, 0, 0, 2);
13421346 *insn++ = BPF_MOV64_REG(BPF_REG_1, BPF_REG_0);
1343− *insn++ = BPF_CALL_KFUNC(0, bpf_cgroup_release_id),
1347+ *insn++ = BPF_CALL_KFUNC(0, bpf_cgroup_release_id);
13441348 *insn++ = BPF_MOV64_REG(BPF_REG_1, BPF_REG_8);
13451349 *insn++ = prog->insnsi[0];
13461350
@@ -1379,7 +1383,7 @@ static int st_ops_gen_epilogue_with_kfunc(struct bpf_insn *insn_buf, const struc · 32 unmodified lines
13791383 *insn++ = BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_6, offsetof(struct st_ops_args, a));
13801384 *insn++ = BPF_JMP_IMM(BPF_JA, 0, 0, 2);
13811385 *insn++ = BPF_MOV64_REG(BPF_REG_1, BPF_REG_0);
1382− *insn++ = BPF_CALL_KFUNC(0, bpf_cgroup_release_id),
1386+ *insn++ = BPF_CALL_KFUNC(0, bpf_cgroup_release_id);
13831387 *insn++ = BPF_MOV64_REG(BPF_REG_0, BPF_REG_6);
13841388 *insn++ = BPF_ALU64_IMM(BPF_MUL, BPF_REG_0, 2);
13851389 *insn++ = BPF_EXIT_INSN();
tools/testing/selftests/bpf/test_loader.c+8−6
@@ -1042,6 +1042,14 @@ void run_subtest(struct test_loader *tester, · 1041 unmodified lines
10421042 emit_verifier_log(tester->log_buf, false /*force*/);
10431043 validate_msgs(tester->log_buf, &subspec->expect_msgs, emit_verifier_log);
10441044
1045+ /* Restore capabilities because the kernel will silently ignore requests
1046+ * for program info (such as xlated program text) if we are not
1047+ * bpf-capable. Also, for some reason test_verifier executes programs
1048+ * with all capabilities restored. Do the same here.
1049+ */
1050+ if (restore_capabilities(&caps))
1051+ goto tobj_cleanup;
1052+
10451053 if (subspec->expect_xlated.cnt) {
10461054 err = get_xlated_program_text(bpf_program__fd(tprog),
10471055 tester->log_buf, tester->log_buf_sz);
@@ -1067,12 +1075,6 @@ void run_subtest(struct test_loader *tester, · 19 unmodified lines
10671075 }
10681076
10691077 if (should_do_test_run(spec, subspec)) {
1070− /* For some reason test_verifier executes programs
1071− * with all capabilities restored. Do the same here.
1072− */
1073− if (restore_capabilities(&caps))
1074− goto tobj_cleanup;
1075−
10761078 /* Do bpf_map__attach_struct_ops() for each struct_ops map.
10771079 * This should trigger bpf_struct_ops->reg callback on kernel side.
10781080 */
tools/testing/selftests/bpf/test_verifier.c+4−4
@@ -734,7 +734,7 @@ static __u32 btf_raw_types[] = { · 733 unmodified lines
734734 BTF_MEMBER_ENC(71, 13, 128), /* struct prog_test_member __kptr *ptr; */
735735 };
736736
737−static char bpf_vlog[UINT_MAX >> 8];
737+static char bpf_vlog[UINT_MAX >> 5];
738738
739739 static int load_btf_spec(__u32 *types, int types_len,
740740 const char *strings, int strings_len)
@@ -1559,10 +1559,10 @@ static void do_test_single(struct bpf_test *test, bool unpriv, · 818 unmodified lines
15591559 test->errstr_unpriv : test->errstr;
15601560
15611561 opts.expected_attach_type = test->expected_attach_type;
1562− if (verbose)
1563− opts.log_level = verif_log_level | 4; /* force stats */
1564− else if (expected_ret == VERBOSE_ACCEPT)
1562+ if (expected_ret == VERBOSE_ACCEPT)
15651563 opts.log_level = 2;
1564+ else if (verbose)
1565+ opts.log_level = verif_log_level | 4; /* force stats */
15661566 else
15671567 opts.log_level = DEFAULT_LIBBPF_LOG_LEVEL;
15681568 opts.prog_flags = pflags;
tools/testing/selftests/bpf/veristat.c+94−7
@@ -1486,7 +1486,84 @@ static bool is_preset_supported(const struct btf_type *t) · 1485 unmodified lines
14861486 return btf_is_int(t) || btf_is_enum(t) || btf_is_enum64(t);
14871487 }
14881488
1489−static int set_global_var(struct bpf_object *obj, struct btf *btf, const struct btf_type *t,
1489+const int btf_find_member(const struct btf *btf,
1490+ const struct btf_type *parent_type,
1491+ __u32 parent_offset,
1492+ const char *member_name,
1493+ int *member_tid,
1494+ __u32 *member_offset)
1495+{
1496+ int i;
1497+
1498+ if (!btf_is_composite(parent_type))
1499+ return -EINVAL;
1500+
1501+ for (i = 0; i < btf_vlen(parent_type); ++i) {
1502+ const struct btf_member *member;
1503+ const struct btf_type *member_type;
1504+ int tid;
1505+
1506+ member = btf_members(parent_type) + i;
1507+ tid = btf__resolve_type(btf, member->type);
1508+ if (tid < 0)
1509+ return -EINVAL;
1510+
1511+ member_type = btf__type_by_id(btf, tid);
1512+ if (member->name_off) {
1513+ const char *name = btf__name_by_offset(btf, member->name_off);
1514+
1515+ if (strcmp(member_name, name) == 0) {
1516+ if (btf_member_bitfield_size(parent_type, i) != 0) {
1517+ fprintf(stderr, "Bitfield presets are not supported %s\n",
1518+ name);
1519+ return -EINVAL;
1520+ }
1521+ *member_offset = parent_offset + member->offset;
1522+ *member_tid = tid;
1523+ return 0;
1524+ }
1525+ } else if (btf_is_composite(member_type)) {
1526+ int err;
1527+
1528+ err = btf_find_member(btf, member_type, parent_offset + member->offset,
1529+ member_name, member_tid, member_offset);
1530+ if (!err)
1531+ return 0;
1532+ }
1533+ }
1534+
1535+ return -EINVAL;
1536+}
1537+
1538+static int adjust_var_secinfo(struct btf *btf, const struct btf_type *t,
1539+ struct btf_var_secinfo *sinfo, const char *var)
1540+{
1541+ char expr[256], *saveptr;
1542+ const struct btf_type *base_type, *member_type;
1543+ int err, member_tid;
1544+ char *name;
1545+ __u32 member_offset = 0;
1546+
1547+ base_type = btf__type_by_id(btf, btf__resolve_type(btf, t->type));
1548+ snprintf(expr, sizeof(expr), "%s", var);
1549+ strtok_r(expr, ".", &saveptr);
1550+
1551+ while ((name = strtok_r(NULL, ".", &saveptr))) {
1552+ err = btf_find_member(btf, base_type, 0, name, &member_tid, &member_offset);
1553+ if (err) {
1554+ fprintf(stderr, "Could not find member %s for variable %s\n", name, var);
1555+ return err;
1556+ }
1557+ member_type = btf__type_by_id(btf, member_tid);
1558+ sinfo->offset += member_offset / 8;
1559+ sinfo->size = member_type->size;
1560+ sinfo->type = member_tid;
1561+ base_type = member_type;
1562+ }
1563+ return 0;
1564+}
1565+
1566+static int set_global_var(struct bpf_object *obj, struct btf *btf,
14901567 struct bpf_map *map, struct btf_var_secinfo *sinfo,
14911568 struct var_preset *preset)
14921569 {
@@ -1495,9 +1572,9 @@ static int set_global_var(struct bpf_object *obj, struct btf *btf, const struct · 2 unmodified lines
14951572 long long value = preset->ivalue;
14961573 size_t size;
14971574
1498− base_type = btf__type_by_id(btf, btf__resolve_type(btf, t->type));
1575+ base_type = btf__type_by_id(btf, btf__resolve_type(btf, sinfo->type));
14991576 if (!base_type) {
1500− fprintf(stderr, "Failed to resolve type %d\n", t->type);
1577+ fprintf(stderr, "Failed to resolve type %d\n", sinfo->type);
15011578 return -EINVAL;
15021579 }
15031580 if (!is_preset_supported(base_type)) {
@@ -1530,7 +1607,7 @@ static int set_global_var(struct bpf_object *obj, struct btf *btf, const struct · 26 unmodified lines
15301607 if (value >= max_val || value < -max_val) {
15311608 fprintf(stderr,
15321609 "Variable %s value %lld is out of range [%lld; %lld]\n",
1533− btf__name_by_offset(btf, t->name_off), value,
1610+ btf__name_by_offset(btf, base_type->name_off), value,
15341611 is_signed ? -max_val : 0, max_val - 1);
15351612 return -EINVAL;
15361613 }
@@ -1583,14 +1660,20 @@ static int set_global_vars(struct bpf_object *obj, struct var_preset *presets, i · 46 unmodified lines
15831660 for (j = 0; j < n; ++j, ++sinfo) {
15841661 const struct btf_type *var_type = btf__type_by_id(btf, sinfo->type);
15851662 const char *var_name;
1663+ int var_len;
15861664
15871665 if (!btf_is_var(var_type))
15881666 continue;
15891667
15901668 var_name = btf__name_by_offset(btf, var_type->name_off);
1669+ var_len = strlen(var_name);
15911670
15921671 for (k = 0; k < npresets; ++k) {
1593− if (strcmp(var_name, presets[k].name) != 0)
1672+ struct btf_var_secinfo tmp_sinfo;
1673+
1674+ if (strncmp(var_name, presets[k].name, var_len) != 0 ||
1675+ (presets[k].name[var_len] != '\0' &&
1676+ presets[k].name[var_len] != '.'))
15941677 continue;
15951678
15961679 if (presets[k].applied) {
@@ -1598,13 +1681,17 @@ static int set_global_vars(struct bpf_object *obj, struct var_preset *presets, i · 1 unmodified line
15981681 var_name);
15991682 return -EINVAL;
16001683 }
1684+ tmp_sinfo = *sinfo;
1685+ err = adjust_var_secinfo(btf, var_type,
1686+ &tmp_sinfo, presets[k].name);
1687+ if (err)
1688+ return err;
16011689
1602− err = set_global_var(obj, btf, var_type, map, sinfo, presets + k);
1690+ err = set_global_var(obj, btf, map, &tmp_sinfo, presets + k);
16031691 if (err)
16041692 return err;
16051693
16061694 presets[k].applied = true;
1607− break;
16081695 }
16091696 }
16101697 }