Merge tag 'bpf-next-6.15' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Pull bpf updates from Alexei Starovoitov: "For this merge window we're splitting BPF pull request into three for higher visibility: main changes, res_spin_lock, try_alloc_pages. These are the main BPF changes: - Add DFA-based live registers analysis to improve verification of programs with loops (Eduard Zingerman) - Introduce load_acquire and store_release BPF instructions and add x86, arm64 JIT support (Peilin Ye) - Fix loop detection logic in the verifier (Eduard Zingerman) - Drop unnecesary lock in bpf_map_inc_not_zero() (Eric Dumazet) - Add kfunc for populating cpumask bits (Emil Tsalapatis) - Convert various shell based tests to selftests/bpf/test_progs format (Bastien Curutchet) - Allow passing referenced kptrs into struct_ops callbacks (Amery Hung) - Add a flag to LSM bpf hook to facilitate bpf program signing (Blaise Boscaccy) - Track arena arguments in kfuncs (Ihor Solodrai) - Add copy_remote_vm_str() helper for reading strings from remote VM and bpf_copy_from_user_task_str() kfunc (Jordan Rome) - Add support for timed may_goto instruction (Kumar Kartikeya Dwivedi) - Allow bpf_get_netns_cookie() int cgroup_skb programs (Mahe Tardy) - Reduce bpf_cgrp_storage_busy false positives when accessing cgroup local storage (Martin KaFai Lau) - Introduce bpf_dynptr_copy() kfunc (Mykyta Yatsenko) - Allow retrieving BTF data with BTF token (Mykyta Yatsenko) - Add BPF kfuncs to set and get xattrs with 'security.bpf.' prefix (Song Liu) - Reject attaching programs to noreturn functions (Yafang Shao) - Introduce pre-order traversal of cgroup bpf programs (Yonghong Song)" * tag 'bpf-next-6.15' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next: (186 commits) selftests/bpf: Add selftests for load-acquire/store-release when register number is invalid bpf: Fix out-of-bounds read in check_atomic_load/store() libbpf: Add namespace for errstr making it libbpf_errstr bpf: Add struct_ops context information to struct bpf_prog_aux selftests/bpf: Sanitize pointer prior fclose() selftests/bpf: Migrate test_xdp_vlan.sh into test_progs selftests/bpf: test_xdp_vlan: Rename BPF sections bpf: clarify a misleading verifier error message selftests/bpf: Add selftest for attaching fexit to __noreturn functions bpf: Reject attaching fexit/fmod_ret to __noreturn functions bpf: Only fails the busy counter check in bpf_cgrp_storage_get if it creates storage bpf: Make perf_event_read_output accessible in all program types. bpftool: Using the right format specifiers bpftool: Add -Wformat-signedness flag to detect format errors selftests/bpf: Test freplace from user namespace libbpf: Pass BPF token from find_prog_btf_id to BPF_BTF_GET_FD_BY_ID bpf: Return prog btf_id without capable check bpf: BPF token support for BPF_BTF_GET_FD_BY_ID bpf, x86: Fix objtool warning for timed may_goto bpf: Check map->record at the beginning of check_and_free_fields() ...

Linus Torvalds committed 1 year ago

Commit fa593d02 parents 7f2ff7b 9aa8fe2213 files · +10643 −3497Browse files

Unified · Split

213 files

Documentation/bpf/bpf_iterators.rst +1 −1
Documentation/bpf/btf.rst +21 −4
Documentation/bpf/standardization/instruction-set.rst +14 −6
arch/arm64/include/asm/insn.h +10 −2
arch/arm64/lib/insn.c +29 −0
arch/arm64/net/bpf_jit.h +20 −0
arch/arm64/net/bpf_jit_comp.c +87 −5
arch/s390/net/bpf_jit_comp.c +10 −4
arch/x86/net/Makefile +1 −1
arch/x86/net/bpf_jit_comp.c +87 −13
arch/x86/net/bpf_timed_may_goto.S +55 −0
fs/bpf_fs_kfuncs.c +214 −11
include/linux/bpf-cgroup.h +1 −0
include/linux/bpf.h +29 −2
include/linux/bpf_lsm.h +18 −0
include/linux/bpf_verifier.h +23 −9
include/linux/btf.h +3 −0
include/linux/filter.h +20 −0
include/linux/lsm_hook_defs.h +3 −3
include/linux/mm.h +5 −0
include/linux/security.h +6 −6
include/uapi/linux/bpf.h +9 −1
include/uapi/linux/btf.h +2 −1
include/uapi/linux/xattr.h +4 −0
kernel/bpf/arena.c +2 −2
kernel/bpf/bpf_cgrp_storage.c +6 −5
kernel/bpf/bpf_iter.c +6 −7
kernel/bpf/bpf_lsm.c +2 −0
kernel/bpf/bpf_struct_ops.c +75 −60
kernel/bpf/btf.c +113 −14
kernel/bpf/cgroup.c +25 −8
kernel/bpf/core.c +107 −10
kernel/bpf/cpumask.c +53 −0
kernel/bpf/disasm.c +14 −2
kernel/bpf/hashtab.c +6 −3
kernel/bpf/helpers.c +119 −4
kernel/bpf/preload/bpf_preload_kern.c +1 −0
kernel/bpf/syscall.c +33 −15
kernel/bpf/verifier.c +1043 −333
kernel/trace/bpf_trace.c +10 −4
mm/memory.c +118 −0
mm/nommu.c +79 −0
net/core/filter.c +4 −2
samples/bpf/Makefile +1 −1
scripts/Makefile.btf +1 −1
security/security.c +9 −6
security/selinux/hooks.c +3 −3
tools/bpf/bpftool/Makefile +6 −1
tools/bpf/bpftool/btf.c +7 −7
tools/bpf/bpftool/btf_dumper.c +1 −1
tools/bpf/bpftool/cgroup.c +1 −1
tools/bpf/bpftool/common.c +4 −3
tools/bpf/bpftool/gen.c +6 −6
tools/bpf/bpftool/jit_disasm.c +2 −1
tools/bpf/bpftool/link.c +7 −7
tools/bpf/bpftool/main.c +4 −4
tools/bpf/bpftool/map.c +9 −5
tools/bpf/bpftool/map_perf_ring.c +3 −3
tools/bpf/bpftool/net.c +2 −2
tools/bpf/bpftool/netlink_dumper.c +3 −3
tools/bpf/bpftool/prog.c +7 −6
tools/bpf/bpftool/tracelog.c +1 −1
tools/bpf/bpftool/xlated_dumper.c +3 −3
tools/bpf/runqslower/Makefile +2 −1
tools/include/uapi/linux/bpf.h +9 −1
tools/include/uapi/linux/btf.h +2 −1
tools/lib/bpf/bpf.c +2 −1
tools/lib/bpf/bpf.h +2 −1
tools/lib/bpf/btf.c +78 −27
tools/lib/bpf/btf.h +3 −0
tools/lib/bpf/btf_dump.c +4 −1
tools/lib/bpf/libbpf.c +151 −86
tools/lib/bpf/libbpf.h +13 −0
tools/lib/bpf/libbpf.map +3 −0
tools/lib/bpf/libbpf_internal.h +1 −0
tools/lib/bpf/linker.c +1 −1
tools/lib/bpf/relo_core.c +20 −4
tools/lib/bpf/str_error.c +1 −1
tools/lib/bpf/str_error.h +5 −2
tools/lib/bpf/usdt.bpf.h +32 −0
tools/testing/selftests/bpf/DENYLIST.aarch64 +0 −9
tools/testing/selftests/bpf/Makefile +14 −14
tools/testing/selftests/bpf/bpf_arena_spin_lock.h +533 −0
tools/testing/selftests/bpf/bpf_atomic.h +140 −0
tools/testing/selftests/bpf/bpf_experimental.h +9 −6
tools/testing/selftests/bpf/bpf_kfuncs.h +5 −0
tools/testing/selftests/bpf/cap_helpers.c +4 −4
tools/testing/selftests/bpf/cap_helpers.h +1 −0
tools/testing/selftests/bpf/network_helpers.c +76 −35
tools/testing/selftests/bpf/network_helpers.h +21 −0
tools/testing/selftests/bpf/prog_tests/align.c +10 −1
tools/testing/selftests/bpf/prog_tests/arena_atomics.c +65 −1
tools/testing/selftests/bpf/prog_tests/arena_spin_lock.c +108 −0
tools/testing/selftests/bpf/prog_tests/bloom_filter_map.c +5 −0
tools/testing/selftests/bpf/prog_tests/bpf_iter.c +68 −0
tools/testing/selftests/bpf/prog_tests/bpf_nf.c +6 −3
tools/testing/selftests/bpf/prog_tests/btf.c +19 −4
tools/testing/selftests/bpf/prog_tests/btf_dump.c +110 −37
tools/testing/selftests/bpf/prog_tests/cgroup_preorder.c +128 −0
tools/testing/selftests/bpf/prog_tests/cgroup_v1v2.c +9 −4
tools/testing/selftests/bpf/prog_tests/changes_pkt_data.c +0 −107
tools/testing/selftests/bpf/prog_tests/compute_live_registers.c +9 −0
tools/testing/selftests/bpf/prog_tests/core_reloc.c +4 −2
tools/testing/selftests/bpf/prog_tests/cpumask.c +4 −1
tools/testing/selftests/bpf/prog_tests/dynptr.c +21 −0
tools/testing/selftests/bpf/prog_tests/fd_array.c +2 −2
tools/testing/selftests/bpf/prog_tests/fexit_noreturns.c +9 −0
tools/testing/selftests/bpf/prog_tests/fs_kfuncs.c +152 −10
tools/testing/selftests/bpf/prog_tests/kernel_flag.c +43 −0
tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c +540 −0
tools/testing/selftests/bpf/prog_tests/lwt_seg6local.c +176 −0
tools/testing/selftests/bpf/prog_tests/netns_cookie.c +14 −7
tools/testing/selftests/bpf/prog_tests/ns_current_pid_tgid.c +17 −30
tools/testing/selftests/bpf/prog_tests/prepare.c +99 −0
tools/testing/selftests/bpf/prog_tests/pro_epilogue.c +2 −0
tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c +3 −0
tools/testing/selftests/bpf/prog_tests/read_vsyscall.c +1 −0
tools/testing/selftests/bpf/prog_tests/setget_sockopt.c +1 −1
tools/testing/selftests/bpf/prog_tests/spin_lock.c +3 −0
tools/testing/selftests/bpf/prog_tests/summarization.c +144 −0
tools/testing/selftests/bpf/prog_tests/tailcalls.c +1 −0
tools/testing/selftests/bpf/prog_tests/tc_links.c +14 −14
tools/testing/selftests/bpf/prog_tests/tc_opts.c +20 −20
tools/testing/selftests/bpf/prog_tests/test_struct_ops_kptr_return.c +16 −0
tools/testing/selftests/bpf/prog_tests/test_struct_ops_refcounted.c +14 −0
tools/testing/selftests/bpf/prog_tests/test_tunnel.c +530 −103
tools/testing/selftests/bpf/prog_tests/test_veristat.c +139 −0
tools/testing/selftests/bpf/prog_tests/test_xdp_veth.c +512 −126
tools/testing/selftests/bpf/prog_tests/token.c +96 −1
tools/testing/selftests/bpf/prog_tests/usdt.c +10 −1
tools/testing/selftests/bpf/prog_tests/verifier.c +6 −2
tools/testing/selftests/bpf/prog_tests/xdp_vlan.c +175 −0
tools/testing/selftests/bpf/progs/arena_atomics.c +119 −2
tools/testing/selftests/bpf/progs/arena_spin_lock.c +51 −0
tools/testing/selftests/bpf/progs/bpf_iter_tasks.c +110 −0
tools/testing/selftests/bpf/progs/bpf_misc.h +22 −0
tools/testing/selftests/bpf/progs/bpf_tracing_net.h +4 −0
tools/testing/selftests/bpf/progs/btf__core_reloc_arrays___err_bad_signed_arr_elem_sz.c +3 −0
tools/testing/selftests/bpf/progs/cgroup_preorder.c +41 −0
tools/testing/selftests/bpf/progs/changes_pkt_data.c +0 −39
tools/testing/selftests/bpf/progs/compute_live_registers.c +424 −0
tools/testing/selftests/bpf/progs/connect4_dropper.c +3 −1
tools/testing/selftests/bpf/progs/core_reloc_types.h +10 −0
tools/testing/selftests/bpf/progs/cpumask_common.h +1 −0
tools/testing/selftests/bpf/progs/cpumask_failure.c +38 −0
tools/testing/selftests/bpf/progs/cpumask_success.c +119 −1
tools/testing/selftests/bpf/progs/dynptr_success.c +118 −5
tools/testing/selftests/bpf/progs/fexit_noreturns.c +15 −0
tools/testing/selftests/bpf/progs/irq.c +70 −1
tools/testing/selftests/bpf/progs/iters.c +139 −0
tools/testing/selftests/bpf/progs/netns_cookie_prog.c +9 −0
tools/testing/selftests/bpf/progs/preempt_lock.c +67 −1
tools/testing/selftests/bpf/progs/prepare.c +28 −0
tools/testing/selftests/bpf/progs/priv_freplace_prog.c +13 −0
tools/testing/selftests/bpf/progs/priv_prog.c +3 −3
tools/testing/selftests/bpf/progs/pro_epilogue_with_kfunc.c +88 −0
tools/testing/selftests/bpf/progs/rcu_read_lock.c +60 −1
tools/testing/selftests/bpf/progs/read_vsyscall.c +9 −2
tools/testing/selftests/bpf/progs/set_global_vars.c +47 −0
tools/testing/selftests/bpf/progs/strncmp_bench.c +4 −1
tools/testing/selftests/bpf/progs/struct_ops_kptr_return.c +30 −0
tools/testing/selftests/bpf/progs/struct_ops_kptr_return_fail__invalid_scalar.c +26 −0
tools/testing/selftests/bpf/progs/struct_ops_kptr_return_fail__local_kptr.c +34 −0
tools/testing/selftests/bpf/progs/struct_ops_kptr_return_fail__nonzero_offset.c +25 −0
tools/testing/selftests/bpf/progs/struct_ops_kptr_return_fail__wrong_type.c +30 −0
tools/testing/selftests/bpf/progs/struct_ops_refcounted.c +31 −0
tools/testing/selftests/bpf/progs/struct_ops_refcounted_fail__global_subprog.c +39 −0
tools/testing/selftests/bpf/progs/struct_ops_refcounted_fail__ref_leak.c +22 −0
tools/testing/selftests/bpf/progs/struct_ops_refcounted_fail__tail_call.c +36 −0
tools/testing/selftests/bpf/progs/summarization.c +78 −0
tools/testing/selftests/bpf/progs/summarization_freplace.c +16 −1
tools/testing/selftests/bpf/progs/test_cgroup1_hierarchy.c +2 −2
tools/testing/selftests/bpf/progs/test_core_reloc_arrays.c +5 −0
tools/testing/selftests/bpf/progs/test_get_xattr.c +24 −4
tools/testing/selftests/bpf/progs/test_kernel_flag.c +28 −0
tools/testing/selftests/bpf/progs/test_kfunc_dynptr_param.c +3 −3
tools/testing/selftests/bpf/progs/test_lookup_key.c +1 −1
tools/testing/selftests/bpf/progs/test_ptr_untrusted.c +1 −1
tools/testing/selftests/bpf/progs/test_select_reuseport_kern.c +0 −1
tools/testing/selftests/bpf/progs/test_set_remove_xattr.c +133 −0
tools/testing/selftests/bpf/progs/test_spin_lock_fail.c +69 −0
tools/testing/selftests/bpf/progs/test_task_under_cgroup.c +1 −1
tools/testing/selftests/bpf/progs/test_usdt.c +14 −0
tools/testing/selftests/bpf/progs/test_verify_pkcs7_sig.c +1 −1
tools/testing/selftests/bpf/progs/test_xdp_vlan.c +10 −10
tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c +48 −10
tools/testing/selftests/bpf/progs/verifier_gotol.c +1 −5
tools/testing/selftests/bpf/progs/verifier_iterating_callbacks.c +1 −5
tools/testing/selftests/bpf/progs/verifier_load_acquire.c +218 −0
tools/testing/selftests/bpf/progs/verifier_may_goto_1.c +32 −2
tools/testing/selftests/bpf/progs/verifier_precision.c +49 −0
tools/testing/selftests/bpf/progs/verifier_stack_ptr.c +52 −0
tools/testing/selftests/bpf/progs/verifier_store_release.c +286 −0
tools/testing/selftests/bpf/progs/xdp_redirect_map.c +88 −0
tools/testing/selftests/bpf/progs/xdp_redirect_multi_kern.c +29 −12
tools/testing/selftests/bpf/test_btf.h +6 −0
tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +108 −0
tools/testing/selftests/bpf/test_kmods/bpf_testmod.h +6 −0
tools/testing/selftests/bpf/test_loader.c +29 −3
tools/testing/selftests/bpf/test_lwt_ip_encap.sh +0 −476
tools/testing/selftests/bpf/test_lwt_seg6local.sh +0 −156
tools/testing/selftests/bpf/test_maps.c +5 −4
tools/testing/selftests/bpf/test_progs.c +43 −29
tools/testing/selftests/bpf/test_progs.h +8 −0
tools/testing/selftests/bpf/test_tunnel.sh +0 −645
tools/testing/selftests/bpf/test_xdp_redirect_multi.sh +0 −214
tools/testing/selftests/bpf/test_xdp_vlan.sh +0 −233
tools/testing/selftests/bpf/test_xdp_vlan_mode_generic.sh +0 −9
tools/testing/selftests/bpf/test_xdp_vlan_mode_native.sh +0 −9
tools/testing/selftests/bpf/veristat.c +355 −12
tools/testing/selftests/bpf/with_addr.sh +0 −54
tools/testing/selftests/bpf/with_tunnels.sh +0 −36
tools/testing/selftests/bpf/xdp_redirect_multi.c +0 −226

Documentation/bpf/bpf_iterators.rst+1−1

@@ -86,7 +86,7 @@ following steps: · 85 unmodified lines
86	86	The following are a few examples of selftest BPF iterator programs:
87	87
88	88	* `bpf_iter_tcp4.c <https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/tree/tools/testing/selftests/bpf/progs/bpf_iter_tcp4.c>`_
89		−* `bpf_iter_task_vma.c <https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/tree/tools/testing/selftests/bpf/progs/bpf_iter_task_vma.c>`_
	89	+* `bpf_iter_task_vmas.c <https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/tree/tools/testing/selftests/bpf/progs/bpf_iter_task_vmas.c>`_
90	90	* `bpf_iter_task_file.c <https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/tree/tools/testing/selftests/bpf/progs/bpf_iter_task_file.c>`_
91	91
92	92	Let us look at ``bpf_iter_task_file.c``, which runs in kernel space:

Documentation/bpf/btf.rst+21−4

@@ -102,7 +102,8 @@ Each type contains the following common data:: · 101 unmodified lines
102	102	* bits 24-28: kind (e.g. int, ptr, array...etc)
103	103	* bits 29-30: unused
104	104	* bit 31: kind_flag, currently used by
105		− * struct, union, fwd, enum and enum64.
	105	+ * struct, union, enum, fwd, enum64,
	106	+ * decl_tag and type_tag
106	107	*/
107	108	__u32 info;
108	109	/* "size" is used by INT, ENUM, STRUCT, UNION and ENUM64.
@@ -478,7 +479,7 @@ No additional type data follow ``btf_type``. · 369 unmodified lines
478	479
479	480	``struct btf_type`` encoding requirement:
480	481	* ``name_off``: offset to a non-empty string
481		− * ``info.kind_flag``: 0
	482	+ * ``info.kind_flag``: 0 or 1
482	483	* ``info.kind``: BTF_KIND_DECL_TAG
483	484	* ``info.vlen``: 0
484	485	* ``type``: ``struct``, ``union``, ``func``, ``var`` or ``typedef``
@@ -489,7 +490,6 @@ No additional type data follow ``btf_type``. · 4 unmodified lines
489	490	__u32 component_idx;
490	491	};
491	492
492		−The ``name_off`` encodes btf_decl_tag attribute string.
493	493	The ``type`` should be ``struct``, ``union``, ``func``, ``var`` or ``typedef``.
494	494	For ``var`` or ``typedef`` type, ``btf_decl_tag.component_idx`` must be ``-1``.
495	495	For the other three types, if the btf_decl_tag attribute is
@@ -499,12 +499,21 @@ the attribute is applied to a ``struct``/``union`` member or · 3 unmodified lines
499	499	a ``func`` argument, and ``btf_decl_tag.component_idx`` should be a
500	500	valid index (starting from 0) pointing to a member or an argument.
501	501
	502	+If ``info.kind_flag`` is 0, then this is a normal decl tag, and the
	503	+``name_off`` encodes btf_decl_tag attribute string.
	504	+
	505	+If ``info.kind_flag`` is 1, then the decl tag represents an arbitrary
	506	+__attribute__. In this case, ``name_off`` encodes a string
	507	+representing the attribute-list of the attribute specifier. For
	508	+example, for an ``__attribute__((aligned(4)))`` the string's contents
	509	+is ``aligned(4)``.
	510	+
502	511	2.2.18 BTF_KIND_TYPE_TAG
503	512	~~~~~~~~~~~~~~~~~~~~~~~~
504	513
505	514	``struct btf_type`` encoding requirement:
506	515	* ``name_off``: offset to a non-empty string
507		− * ``info.kind_flag``: 0
	516	+ * ``info.kind_flag``: 0 or 1
508	517	* ``info.kind``: BTF_KIND_TYPE_TAG
509	518	* ``info.vlen``: 0
510	519	* ``type``: the type with ``btf_type_tag`` attribute
@@ -522,6 +531,14 @@ type_tag, then zero or more const/volatile/restrict/typedef · 11 unmodified lines
522	531	and finally the base type. The base type is one of
523	532	int, ptr, array, struct, union, enum, func_proto and float types.
524	533
	534	+Similarly to decl tags, if the ``info.kind_flag`` is 0, then this is a
	535	+normal type tag, and the ``name_off`` encodes btf_type_tag attribute
	536	+string.
	537	+
	538	+If ``info.kind_flag`` is 1, then the type tag represents an arbitrary
	539	+__attribute__, and the ``name_off`` encodes a string representing the
	540	+attribute-list of the attribute specifier.
	541	+
525	542	2.2.19 BTF_KIND_ENUM64
526	543	~~~~~~~~~~~~~~~~~~~~~~
527	544

Documentation/bpf/standardization/instruction-set.rst+14−6

@@ -324,34 +324,42 @@ register. · 323 unmodified lines
324	324
325	325	.. table:: Arithmetic instructions
326	326
327		− ===== ===== ======= ==========================================================
	327	+ ===== ===== ======= ===================================================================================
328	328	name code offset description
329		− ===== ===== ======= ==========================================================
	329	+ ===== ===== ======= ===================================================================================
330	330	ADD 0x0 0 dst += src
331	331	SUB 0x1 0 dst -= src
332	332	MUL 0x2 0 dst \*= src
333	333	DIV 0x3 0 dst = (src != 0) ? (dst / src) : 0
334		− SDIV 0x3 1 dst = (src != 0) ? (dst s/ src) : 0
	334	+ SDIV 0x3 1 dst = (src == 0) ? 0 : ((src == -1 && dst == LLONG_MIN) ? LLONG_MIN : (dst s/ src))
335	335	OR 0x4 0 dst \\|= src
336	336	AND 0x5 0 dst &= src
337	337	LSH 0x6 0 dst <<= (src & mask)
338	338	RSH 0x7 0 dst >>= (src & mask)
339	339	NEG 0x8 0 dst = -dst
340	340	MOD 0x9 0 dst = (src != 0) ? (dst % src) : dst
341		− SMOD 0x9 1 dst = (src != 0) ? (dst s% src) : dst
	341	+ SMOD 0x9 1 dst = (src == 0) ? dst : ((src == -1 && dst == LLONG_MIN) ? 0: (dst s% src))
342	342	XOR 0xa 0 dst ^= src
343	343	MOV 0xb 0 dst = src
344	344	MOVSX 0xb 8/16/32 dst = (s8,s16,s32)src
345	345	ARSH 0xc 0 :term:`sign extending<Sign Extend>` dst >>= (src & mask)
346	346	END 0xd 0 byte swap operations (see `Byte swap instructions`_ below)
347		− ===== ===== ======= ==========================================================
	347	+ ===== ===== ======= ===================================================================================
348	348
349	349	Underflow and overflow are allowed during arithmetic operations, meaning
350	350	the 64-bit or 32-bit value will wrap. If BPF program execution would
351	351	result in division by zero, the destination register is instead set to zero.
	352	+Otherwise, for ``ALU64``, if execution would result in ``LLONG_MIN``
	353	+dividing -1, the desination register is instead set to ``LLONG_MIN``. For
	354	+``ALU``, if execution would result in ``INT_MIN`` dividing -1, the
	355	+desination register is instead set to ``INT_MIN``.
	356	+
352	357	If execution would result in modulo by zero, for ``ALU64`` the value of
353	358	the destination register is unchanged whereas for ``ALU`` the upper
354		−32 bits of the destination register are zeroed.
	359	+32 bits of the destination register are zeroed. Otherwise, for ``ALU64``,
	360	+if execution would resuslt in ``LLONG_MIN`` modulo -1, the destination
	361	+register is instead set to 0. For ``ALU``, if execution would result in
	362	+``INT_MIN`` modulo -1, the destination register is instead set to 0.
355	363
356	364	``{ADD, X, ALU}``, where 'code' = ``ADD``, 'source' = ``X``, and 'class' = ``ALU``, means::
357	365

arch/arm64/include/asm/insn.h+10−2

@@ -188,8 +188,10 @@ enum aarch64_insn_ldst_type { · 187 unmodified lines
188	188	AARCH64_INSN_LDST_STORE_PAIR_PRE_INDEX,
189	189	AARCH64_INSN_LDST_LOAD_PAIR_POST_INDEX,
190	190	AARCH64_INSN_LDST_STORE_PAIR_POST_INDEX,
	191	+ AARCH64_INSN_LDST_LOAD_ACQ,
191	192	AARCH64_INSN_LDST_LOAD_EX,
192	193	AARCH64_INSN_LDST_LOAD_ACQ_EX,
	194	+ AARCH64_INSN_LDST_STORE_REL,
193	195	AARCH64_INSN_LDST_STORE_EX,
194	196	AARCH64_INSN_LDST_STORE_REL_EX,
195	197	AARCH64_INSN_LDST_SIGNED_LOAD_IMM_OFFSET,
@@ -351,8 +353,10 @@ __AARCH64_INSN_FUNCS(ldr_imm, 0x3FC00000, 0x39400000) · 155 unmodified lines
351	353	__AARCH64_INSN_FUNCS(ldr_lit, 0xBF000000, 0x18000000)
352	354	__AARCH64_INSN_FUNCS(ldrsw_lit, 0xFF000000, 0x98000000)
353	355	__AARCH64_INSN_FUNCS(exclusive, 0x3F800000, 0x08000000)
354		−__AARCH64_INSN_FUNCS(load_ex, 0x3F400000, 0x08400000)
355		−__AARCH64_INSN_FUNCS(store_ex, 0x3F400000, 0x08000000)
	356	+__AARCH64_INSN_FUNCS(load_acq, 0x3FDFFC00, 0x08DFFC00)
	357	+__AARCH64_INSN_FUNCS(store_rel, 0x3FDFFC00, 0x089FFC00)
	358	+__AARCH64_INSN_FUNCS(load_ex, 0x3FC00000, 0x08400000)
	359	+__AARCH64_INSN_FUNCS(store_ex, 0x3FC00000, 0x08000000)
356	360	__AARCH64_INSN_FUNCS(mops, 0x3B200C00, 0x19000400)
357	361	__AARCH64_INSN_FUNCS(stp, 0x7FC00000, 0x29000000)
358	362	__AARCH64_INSN_FUNCS(ldp, 0x7FC00000, 0x29400000)
@@ -602,6 +606,10 @@ u32 aarch64_insn_gen_load_store_pair(enum aarch64_insn_register reg1, · 243 unmodified lines
602	606	int offset,
603	607	enum aarch64_insn_variant variant,
604	608	enum aarch64_insn_ldst_type type);
	609	+u32 aarch64_insn_gen_load_acq_store_rel(enum aarch64_insn_register reg,
	610	+ enum aarch64_insn_register base,
	611	+ enum aarch64_insn_size_type size,
	612	+ enum aarch64_insn_ldst_type type);
605	613	u32 aarch64_insn_gen_load_store_ex(enum aarch64_insn_register reg,
606	614	enum aarch64_insn_register base,
607	615	enum aarch64_insn_register state,

arch/arm64/lib/insn.c+29−0

@@ -540,6 +540,35 @@ u32 aarch64_insn_gen_load_store_pair(enum aarch64_insn_register reg1, · 539 unmodified lines
540	540	offset >> shift);
541	541	}
542	542
	543	+u32 aarch64_insn_gen_load_acq_store_rel(enum aarch64_insn_register reg,
	544	+ enum aarch64_insn_register base,
	545	+ enum aarch64_insn_size_type size,
	546	+ enum aarch64_insn_ldst_type type)
	547	+{
	548	+ u32 insn;
	549	+
	550	+ switch (type) {
	551	+ case AARCH64_INSN_LDST_LOAD_ACQ:
	552	+ insn = aarch64_insn_get_load_acq_value();
	553	+ break;
	554	+ case AARCH64_INSN_LDST_STORE_REL:
	555	+ insn = aarch64_insn_get_store_rel_value();
	556	+ break;
	557	+ default:
	558	+ pr_err("%s: unknown load-acquire/store-release encoding %d\n",
	559	+ __func__, type);
	560	+ return AARCH64_BREAK_FAULT;
	561	+ }
	562	+
	563	+ insn = aarch64_insn_encode_ldst_size(size, insn);
	564	+
	565	+ insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn,
	566	+ reg);
	567	+
	568	+ return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
	569	+ base);
	570	+}
	571	+
543	572	u32 aarch64_insn_gen_load_store_ex(enum aarch64_insn_register reg,
544	573	enum aarch64_insn_register base,
545	574	enum aarch64_insn_register state,

arch/arm64/net/bpf_jit.h+20−0

@@ -119,6 +119,26 @@ · 118 unmodified lines
119	119	aarch64_insn_gen_load_store_ex(Rt, Rn, Rs, A64_SIZE(sf), \
120	120	AARCH64_INSN_LDST_STORE_REL_EX)
121	121
	122	+/* Load-acquire & store-release */
	123	+#define A64_LDAR(Rt, Rn, size) \
	124	+ aarch64_insn_gen_load_acq_store_rel(Rt, Rn, AARCH64_INSN_SIZE_##size, \
	125	+ AARCH64_INSN_LDST_LOAD_ACQ)
	126	+#define A64_STLR(Rt, Rn, size) \
	127	+ aarch64_insn_gen_load_acq_store_rel(Rt, Rn, AARCH64_INSN_SIZE_##size, \
	128	+ AARCH64_INSN_LDST_STORE_REL)
	129	+
	130	+/* Rt = [Rn] (load acquire) */
	131	+#define A64_LDARB(Wt, Xn) A64_LDAR(Wt, Xn, 8)
	132	+#define A64_LDARH(Wt, Xn) A64_LDAR(Wt, Xn, 16)
	133	+#define A64_LDAR32(Wt, Xn) A64_LDAR(Wt, Xn, 32)
	134	+#define A64_LDAR64(Xt, Xn) A64_LDAR(Xt, Xn, 64)
	135	+
	136	+/* [Rn] = Rt (store release) */
	137	+#define A64_STLRB(Wt, Xn) A64_STLR(Wt, Xn, 8)
	138	+#define A64_STLRH(Wt, Xn) A64_STLR(Wt, Xn, 16)
	139	+#define A64_STLR32(Wt, Xn) A64_STLR(Wt, Xn, 32)
	140	+#define A64_STLR64(Xt, Xn) A64_STLR(Xt, Xn, 64)
	141	+
122	142	/*
123	143	* LSE atomics
124	144	*

arch/arm64/net/bpf_jit_comp.c+87−5

@@ -272,7 +272,7 @@ static inline void emit_a64_add_i(const bool is64, const int dst, const int src, · 271 unmodified lines
272	272	{
273	273	if (is_addsub_imm(imm)) {
274	274	emit(A64_ADD_I(is64, dst, src, imm), ctx);
275		− } else if (is_addsub_imm(-imm)) {
	275	+ } else if (is_addsub_imm(-(u32)imm)) {
276	276	emit(A64_SUB_I(is64, dst, src, -imm), ctx);
277	277	} else {
278	278	emit_a64_mov_i(is64, tmp, imm, ctx);
@@ -647,6 +647,81 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) · 368 unmodified lines
647	647	return 0;
648	648	}
649	649
	650	+static int emit_atomic_ld_st(const struct bpf_insn insn, struct jit_ctx ctx)
	651	+{
	652	+ const s32 imm = insn->imm;
	653	+ const s16 off = insn->off;
	654	+ const u8 code = insn->code;
	655	+ const bool arena = BPF_MODE(code) == BPF_PROBE_ATOMIC;
	656	+ const u8 arena_vm_base = bpf2a64[ARENA_VM_START];
	657	+ const u8 dst = bpf2a64[insn->dst_reg];
	658	+ const u8 src = bpf2a64[insn->src_reg];
	659	+ const u8 tmp = bpf2a64[TMP_REG_1];
	660	+ u8 reg;
	661	+
	662	+ switch (imm) {
	663	+ case BPF_LOAD_ACQ:
	664	+ reg = src;
	665	+ break;
	666	+ case BPF_STORE_REL:
	667	+ reg = dst;
	668	+ break;
	669	+ default:
	670	+ pr_err_once("unknown atomic load/store op code %02x\n", imm);
	671	+ return -EINVAL;
	672	+ }
	673	+
	674	+ if (off) {
	675	+ emit_a64_add_i(1, tmp, reg, tmp, off, ctx);
	676	+ reg = tmp;
	677	+ }
	678	+ if (arena) {
	679	+ emit(A64_ADD(1, tmp, reg, arena_vm_base), ctx);
	680	+ reg = tmp;
	681	+ }
	682	+
	683	+ switch (imm) {
	684	+ case BPF_LOAD_ACQ:
	685	+ switch (BPF_SIZE(code)) {
	686	+ case BPF_B:
	687	+ emit(A64_LDARB(dst, reg), ctx);
	688	+ break;
	689	+ case BPF_H:
	690	+ emit(A64_LDARH(dst, reg), ctx);
	691	+ break;
	692	+ case BPF_W:
	693	+ emit(A64_LDAR32(dst, reg), ctx);
	694	+ break;
	695	+ case BPF_DW:
	696	+ emit(A64_LDAR64(dst, reg), ctx);
	697	+ break;
	698	+ }
	699	+ break;
	700	+ case BPF_STORE_REL:
	701	+ switch (BPF_SIZE(code)) {
	702	+ case BPF_B:
	703	+ emit(A64_STLRB(src, reg), ctx);
	704	+ break;
	705	+ case BPF_H:
	706	+ emit(A64_STLRH(src, reg), ctx);
	707	+ break;
	708	+ case BPF_W:
	709	+ emit(A64_STLR32(src, reg), ctx);
	710	+ break;
	711	+ case BPF_DW:
	712	+ emit(A64_STLR64(src, reg), ctx);
	713	+ break;
	714	+ }
	715	+ break;
	716	+ default:
	717	+ pr_err_once("unexpected atomic load/store op code %02x\n",
	718	+ imm);
	719	+ return -EINVAL;
	720	+ }
	721	+
	722	+ return 0;
	723	+}
	724	+
650	725	#ifdef CONFIG_ARM64_LSE_ATOMICS
651	726	static int emit_lse_atomic(const struct bpf_insn insn, struct jit_ctx ctx)
652	727	{
@@ -1159,7 +1234,7 @@ emit_bswap_uxt: · 506 unmodified lines
1159	1234	case BPF_ALU64 \| BPF_SUB \| BPF_K:
1160	1235	if (is_addsub_imm(imm)) {
1161	1236	emit(A64_SUB_I(is64, dst, dst, imm), ctx);
1162		− } else if (is_addsub_imm(-imm)) {
	1237	+ } else if (is_addsub_imm(-(u32)imm)) {
1163	1238	emit(A64_ADD_I(is64, dst, dst, -imm), ctx);
1164	1239	} else {
1165	1240	emit_a64_mov_i(is64, tmp, imm, ctx);
@@ -1330,7 +1405,7 @@ emit_cond_jmp: · 164 unmodified lines
1330	1405	case BPF_JMP32 \| BPF_JSLE \| BPF_K:
1331	1406	if (is_addsub_imm(imm)) {
1332	1407	emit(A64_CMP_I(is64, dst, imm), ctx);
1333		− } else if (is_addsub_imm(-imm)) {
	1408	+ } else if (is_addsub_imm(-(u32)imm)) {
1334	1409	emit(A64_CMN_I(is64, dst, -imm), ctx);
1335	1410	} else {
1336	1411	emit_a64_mov_i(is64, tmp, imm, ctx);
@@ -1641,11 +1716,17 @@ emit_cond_jmp: · 304 unmodified lines
1641	1716	return ret;
1642	1717	break;
1643	1718
	1719	+ case BPF_STX \| BPF_ATOMIC \| BPF_B:
	1720	+ case BPF_STX \| BPF_ATOMIC \| BPF_H:
1644	1721	case BPF_STX \| BPF_ATOMIC \| BPF_W:
1645	1722	case BPF_STX \| BPF_ATOMIC \| BPF_DW:
	1723	+ case BPF_STX \| BPF_PROBE_ATOMIC \| BPF_B:
	1724	+ case BPF_STX \| BPF_PROBE_ATOMIC \| BPF_H:
1646	1725	case BPF_STX \| BPF_PROBE_ATOMIC \| BPF_W:
1647	1726	case BPF_STX \| BPF_PROBE_ATOMIC \| BPF_DW:
1648		− if (cpus_have_cap(ARM64_HAS_LSE_ATOMICS))
	1727	+ if (bpf_atomic_is_load_store(insn))
	1728	+ ret = emit_atomic_ld_st(insn, ctx);
	1729	+ else if (cpus_have_cap(ARM64_HAS_LSE_ATOMICS))
1649	1730	ret = emit_lse_atomic(insn, ctx);
1650	1731	else
1651	1732	ret = emit_ll_sc_atomic(insn, ctx);
@@ -2669,7 +2750,8 @@ bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) · 1017 unmodified lines
2669	2750	switch (insn->code) {
2670	2751	case BPF_STX \| BPF_ATOMIC \| BPF_W:
2671	2752	case BPF_STX \| BPF_ATOMIC \| BPF_DW:
2672		− if (!cpus_have_cap(ARM64_HAS_LSE_ATOMICS))
	2753	+ if (!bpf_atomic_is_load_store(insn) &&
	2754	+ !cpus_have_cap(ARM64_HAS_LSE_ATOMICS))
2673	2755	return false;
2674	2756	}
2675	2757	return true;

arch/s390/net/bpf_jit_comp.c+10−4

@@ -2919,10 +2919,16 @@ bool bpf_jit_supports_arena(void) · 2918 unmodified lines
2919	2919
2920	2920	bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)
2921	2921	{
2922		− /*
2923		− * Currently the verifier uses this function only to check which
2924		− * atomic stores to arena are supported, and they all are.
2925		− */
	2922	+ if (!in_arena)
	2923	+ return true;
	2924	+ switch (insn->code) {
	2925	+ case BPF_STX \| BPF_ATOMIC \| BPF_B:
	2926	+ case BPF_STX \| BPF_ATOMIC \| BPF_H:
	2927	+ case BPF_STX \| BPF_ATOMIC \| BPF_W:
	2928	+ case BPF_STX \| BPF_ATOMIC \| BPF_DW:
	2929	+ if (bpf_atomic_is_load_store(insn))
	2930	+ return false;
	2931	+ }
2926	2932	return true;
2927	2933	}
2928	2934

arch/x86/net/Makefile+1−1

@@ -6,5 +6,5 @@ · 5 unmodified lines
6	6	ifeq ($(CONFIG_X86_32),y)
7	7	obj-$(CONFIG_BPF_JIT) += bpf_jit_comp32.o
8	8	else
9		− obj-$(CONFIG_BPF_JIT) += bpf_jit_comp.o
	9	+ obj-$(CONFIG_BPF_JIT) += bpf_jit_comp.o bpf_timed_may_goto.o
10	10	endif

arch/x86/net/bpf_jit_comp.c+87−13

@@ -1250,8 +1250,8 @@ static void emit_st_r12(u8 **pprog, u32 size, u32 dst_reg, int off, int imm) · 1249 unmodified lines
1250	1250	emit_st_index(pprog, size, dst_reg, X86_REG_R12, off, imm);
1251	1251	}
1252	1252
1253		−static int emit_atomic(u8 **pprog, u8 atomic_op,
1254		− u32 dst_reg, u32 src_reg, s16 off, u8 bpf_size)
	1253	+static int emit_atomic_rmw(u8 **pprog, u32 atomic_op,
	1254	+ u32 dst_reg, u32 src_reg, s16 off, u8 bpf_size)
1255	1255	{
1256	1256	u8 prog = pprog;
1257	1257
@@ -1291,8 +1291,9 @@ static int emit_atomic(u8 **pprog, u8 atomic_op, · 33 unmodified lines
1291	1291	return 0;
1292	1292	}
1293	1293
1294		−static int emit_atomic_index(u8 **pprog, u8 atomic_op, u32 size,
1295		− u32 dst_reg, u32 src_reg, u32 index_reg, int off)
	1294	+static int emit_atomic_rmw_index(u8 **pprog, u32 atomic_op, u32 size,
	1295	+ u32 dst_reg, u32 src_reg, u32 index_reg,
	1296	+ int off)
1296	1297	{
1297	1298	u8 prog = pprog;
1298	1299
@@ -1305,7 +1306,7 @@ static int emit_atomic_index(u8 **pprog, u8 atomic_op, u32 size, · 6 unmodified lines
1305	1306	EMIT1(add_3mod(0x48, dst_reg, src_reg, index_reg));
1306	1307	break;
1307	1308	default:
1308		− pr_err("bpf_jit: 1 and 2 byte atomics are not supported\n");
	1309	+ pr_err("bpf_jit: 1- and 2-byte RMW atomics are not supported\n");
1309	1310	return -EFAULT;
1310	1311	}
1311	1312
@@ -1339,6 +1340,49 @@ static int emit_atomic_index(u8 **pprog, u8 atomic_op, u32 size, · 27 unmodified lines
1339	1340	return 0;
1340	1341	}
1341	1342
	1343	+static int emit_atomic_ld_st(u8 **pprog, u32 atomic_op, u32 dst_reg,
	1344	+ u32 src_reg, s16 off, u8 bpf_size)
	1345	+{
	1346	+ switch (atomic_op) {
	1347	+ case BPF_LOAD_ACQ:
	1348	+ /* dst_reg = smp_load_acquire(src_reg + off16) */
	1349	+ emit_ldx(pprog, bpf_size, dst_reg, src_reg, off);
	1350	+ break;
	1351	+ case BPF_STORE_REL:
	1352	+ /* smp_store_release(dst_reg + off16, src_reg) */
	1353	+ emit_stx(pprog, bpf_size, dst_reg, src_reg, off);
	1354	+ break;
	1355	+ default:
	1356	+ pr_err("bpf_jit: unknown atomic load/store opcode %02x\n",
	1357	+ atomic_op);
	1358	+ return -EFAULT;
	1359	+ }
	1360	+
	1361	+ return 0;
	1362	+}
	1363	+
	1364	+static int emit_atomic_ld_st_index(u8 **pprog, u32 atomic_op, u32 size,
	1365	+ u32 dst_reg, u32 src_reg, u32 index_reg,
	1366	+ int off)
	1367	+{
	1368	+ switch (atomic_op) {
	1369	+ case BPF_LOAD_ACQ:
	1370	+ /* dst_reg = smp_load_acquire(src_reg + idx_reg + off16) */
	1371	+ emit_ldx_index(pprog, size, dst_reg, src_reg, index_reg, off);
	1372	+ break;
	1373	+ case BPF_STORE_REL:
	1374	+ /* smp_store_release(dst_reg + idx_reg + off16, src_reg) */
	1375	+ emit_stx_index(pprog, size, dst_reg, src_reg, index_reg, off);
	1376	+ break;
	1377	+ default:
	1378	+ pr_err("bpf_jit: unknown atomic load/store opcode %02x\n",
	1379	+ atomic_op);
	1380	+ return -EFAULT;
	1381	+ }
	1382	+
	1383	+ return 0;
	1384	+}
	1385	+
1342	1386	#define DONT_CLEAR 1
1343	1387
1344	1388	bool ex_handler_bpf(const struct exception_table_entry x, struct pt_regs regs)
@@ -2121,6 +2165,13 @@ populate_extable: · 776 unmodified lines
2121	2165	}
2122	2166	break;
2123	2167
	2168	+ case BPF_STX \| BPF_ATOMIC \| BPF_B:
	2169	+ case BPF_STX \| BPF_ATOMIC \| BPF_H:
	2170	+ if (!bpf_atomic_is_load_store(insn)) {
	2171	+ pr_err("bpf_jit: 1- and 2-byte RMW atomics are not supported\n");
	2172	+ return -EFAULT;
	2173	+ }
	2174	+ fallthrough;
2124	2175	case BPF_STX \| BPF_ATOMIC \| BPF_W:
2125	2176	case BPF_STX \| BPF_ATOMIC \| BPF_DW:
2126	2177	if (insn->imm == (BPF_AND \| BPF_FETCH) \|\|
@@ -2156,10 +2207,10 @@ populate_extable: · 29 unmodified lines
2156	2207	EMIT2(simple_alu_opcodes[BPF_OP(insn->imm)],
2157	2208	add_2reg(0xC0, AUX_REG, real_src_reg));
2158	2209	/* Attempt to swap in new value */
2159		− err = emit_atomic(&prog, BPF_CMPXCHG,
2160		− real_dst_reg, AUX_REG,
2161		− insn->off,
2162		− BPF_SIZE(insn->code));
	2210	+ err = emit_atomic_rmw(&prog, BPF_CMPXCHG,
	2211	+ real_dst_reg, AUX_REG,
	2212	+ insn->off,
	2213	+ BPF_SIZE(insn->code));
2163	2214	if (WARN_ON(err))
2164	2215	return err;
2165	2216	/*
@@ -2174,17 +2225,35 @@ populate_extable: · 8 unmodified lines
2174	2225	break;
2175	2226	}
2176	2227
2177		− err = emit_atomic(&prog, insn->imm, dst_reg, src_reg,
2178		− insn->off, BPF_SIZE(insn->code));
	2228	+ if (bpf_atomic_is_load_store(insn))
	2229	+ err = emit_atomic_ld_st(&prog, insn->imm, dst_reg, src_reg,
	2230	+ insn->off, BPF_SIZE(insn->code));
	2231	+ else
	2232	+ err = emit_atomic_rmw(&prog, insn->imm, dst_reg, src_reg,
	2233	+ insn->off, BPF_SIZE(insn->code));
2179	2234	if (err)
2180	2235	return err;
2181	2236	break;
2182	2237
	2238	+ case BPF_STX \| BPF_PROBE_ATOMIC \| BPF_B:
	2239	+ case BPF_STX \| BPF_PROBE_ATOMIC \| BPF_H:
	2240	+ if (!bpf_atomic_is_load_store(insn)) {
	2241	+ pr_err("bpf_jit: 1- and 2-byte RMW atomics are not supported\n");
	2242	+ return -EFAULT;
	2243	+ }
	2244	+ fallthrough;
2183	2245	case BPF_STX \| BPF_PROBE_ATOMIC \| BPF_W:
2184	2246	case BPF_STX \| BPF_PROBE_ATOMIC \| BPF_DW:
2185	2247	start_of_ldx = prog;
2186		− err = emit_atomic_index(&prog, insn->imm, BPF_SIZE(insn->code),
2187		− dst_reg, src_reg, X86_REG_R12, insn->off);
	2248	+
	2249	+ if (bpf_atomic_is_load_store(insn))
	2250	+ err = emit_atomic_ld_st_index(&prog, insn->imm,
	2251	+ BPF_SIZE(insn->code), dst_reg,
	2252	+ src_reg, X86_REG_R12, insn->off);
	2253	+ else
	2254	+ err = emit_atomic_rmw_index(&prog, insn->imm, BPF_SIZE(insn->code),
	2255	+ dst_reg, src_reg, X86_REG_R12,
	2256	+ insn->off);
2188	2257	if (err)
2189	2258	return err;
2190	2259	goto populate_extable;
@@ -3801,3 +3870,8 @@ u64 bpf_arch_uaddress_limit(void) · 1610 unmodified lines
3801	3870	{
3802	3871	return 0;
3803	3872	}
	3873	+
	3874	+bool bpf_jit_supports_timed_may_goto(void)
	3875	+{
	3876	+ return true;
	3877	+}

arch/x86/net/bpf_timed_may_goto.S+55−0

@@ -0,0 +1,55 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
	3	+
	4	+#include <linux/export.h>
	5	+#include <linux/linkage.h>
	6	+#include <asm/nospec-branch.h>
	7	+
	8	+ .code64
	9	+ .section .text, "ax"
	10	+
	11	+SYM_FUNC_START(arch_bpf_timed_may_goto)
	12	+ ANNOTATE_NOENDBR
	13	+
	14	+ /*
	15	+ * r10 passes us stack depth, load the pointer to count and timestamp
	16	+ * into r10 by adding it to BPF frame pointer.
	17	+ */
	18	+ leaq (%rbp, %r10, 1), %r10
	19	+
	20	+ /* Setup frame. */
	21	+ pushq %rbp
	22	+ movq %rsp, %rbp
	23	+
	24	+ /* Save r0-r5. */
	25	+ pushq %rax
	26	+ pushq %rdi
	27	+ pushq %rsi
	28	+ pushq %rdx
	29	+ pushq %rcx
	30	+ pushq %r8
	31	+
	32	+ /*
	33	+ * r10 has the pointer to count and timestamp, pass it as first
	34	+ * argument.
	35	+ */
	36	+ movq %r10, %rdi
	37	+
	38	+ /* Emit call depth accounting for call below. */
	39	+ CALL_DEPTH_ACCOUNT
	40	+ call bpf_check_timed_may_goto
	41	+
	42	+ /* BPF_REG_AX=r10 will be stored into count, so move return value to it. */
	43	+ movq %rax, %r10
	44	+
	45	+ /* Restore r5-r0. */
	46	+ popq %r8
	47	+ popq %rcx
	48	+ popq %rdx
	49	+ popq %rsi
	50	+ popq %rdi
	51	+ popq %rax
	52	+
	53	+ leave
	54	+ RET
	55	+SYM_FUNC_END(arch_bpf_timed_may_goto)

fs/bpf_fs_kfuncs.c+214−11

@@ -2,10 +2,12 @@ · 1 unmodified line
2	2	/* Copyright (c) 2024 Google LLC. */
3	3
4	4	#include <linux/bpf.h>
	5	+#include <linux/bpf_lsm.h>
5	6	#include <linux/btf.h>
6	7	#include <linux/btf_ids.h>
7	8	#include <linux/dcache.h>
8	9	#include <linux/fs.h>
	10	+#include <linux/fsnotify.h>
9	11	#include <linux/file.h>
10	12	#include <linux/mm.h>
11	13	#include <linux/xattr.h>
@@ -93,6 +95,24 @@ __bpf_kfunc int bpf_path_d_path(struct path path, char buf, size_t buf__sz) · 81 unmodified lines
93	95	return len;
94	96	}
95	97
	98	+static bool match_security_bpf_prefix(const char *name__str)
	99	+{
	100	+ return !strncmp(name__str, XATTR_NAME_BPF_LSM, XATTR_NAME_BPF_LSM_LEN);
	101	+}
	102	+
	103	+static int bpf_xattr_read_permission(const char name, struct inode inode)
	104	+{
	105	+ if (WARN_ON(!inode))
	106	+ return -EINVAL;
	107	+
	108	+ /* Allow reading xattr with user. and security.bpf. prefix */
	109	+ if (strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN) &&
	110	+ !match_security_bpf_prefix(name))
	111	+ return -EPERM;
	112	+
	113	+ return inode_permission(&nop_mnt_idmap, inode, MAY_READ);
	114	+}
	115	+
96	116	/**
97	117	* bpf_get_dentry_xattr - get xattr of a dentry
98	118	* @dentry: dentry to get xattr from
@@ -101,9 +121,10 @@ __bpf_kfunc int bpf_path_d_path(struct path path, char buf, size_t buf__sz) · 2 unmodified lines
101	121	*
102	122	* Get xattr name__str of dentry and store the output in value_ptr.
103	123	*
104		− * For security reasons, only name__str with prefix "user." is allowed.
	124	+ * For security reasons, only name__str with prefixes "user." or
	125	+ * "security.bpf." are allowed.
105	126	*
106		− * Return: 0 on success, a negative value on error.
	127	+ * Return: length of the xattr value on success, a negative value on error.
107	128	*/
108	129	__bpf_kfunc int bpf_get_dentry_xattr(struct dentry dentry, const char name__str,
109	130	struct bpf_dynptr *value_p)
@@ -114,18 +135,12 @@ __bpf_kfunc int bpf_get_dentry_xattr(struct dentry dentry, const char name__st · 4 unmodified lines
114	135	void *value;
115	136	int ret;
116	137
117		− if (WARN_ON(!inode))
118		− return -EINVAL;
119		−
120		− if (strncmp(name__str, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN))
121		− return -EPERM;
122		−
123	138	value_len = __bpf_dynptr_size(value_ptr);
124	139	value = __bpf_dynptr_data_rw(value_ptr, value_len);
125	140	if (!value)
126	141	return -EINVAL;
127	142
128		− ret = inode_permission(&nop_mnt_idmap, inode, MAY_READ);
	143	+ ret = bpf_xattr_read_permission(name__str, inode);
129	144	if (ret)
130	145	return ret;
131	146	return __vfs_getxattr(dentry, inode, name__str, value, value_len);
@@ -139,9 +154,10 @@ __bpf_kfunc int bpf_get_dentry_xattr(struct dentry dentry, const char name__st · 7 unmodified lines
139	154	*
140	155	* Get xattr name__str of file and store the output in value_ptr.
141	156	*
142		− * For security reasons, only name__str with prefix "user." is allowed.
	157	+ * For security reasons, only name__str with prefixes "user." or
	158	+ * "security.bpf." are allowed.
143	159	*
144		− * Return: 0 on success, a negative value on error.
	160	+ * Return: length of the xattr value on success, a negative value on error.
145	161	*/
146	162	__bpf_kfunc int bpf_get_file_xattr(struct file file, const char name__str,
147	163	struct bpf_dynptr *value_p)
@@ -154,6 +170,160 @@ __bpf_kfunc int bpf_get_file_xattr(struct file file, const char name__str, · 6 unmodified lines
154	170
155	171	__bpf_kfunc_end_defs();
156	172
	173	+static int bpf_xattr_write_permission(const char name, struct inode inode)
	174	+{
	175	+ if (WARN_ON(!inode))
	176	+ return -EINVAL;
	177	+
	178	+ /* Only allow setting and removing security.bpf. xattrs */
	179	+ if (!match_security_bpf_prefix(name))
	180	+ return -EPERM;
	181	+
	182	+ return inode_permission(&nop_mnt_idmap, inode, MAY_WRITE);
	183	+}
	184	+
	185	+/**
	186	+ * bpf_set_dentry_xattr_locked - set a xattr of a dentry
	187	+ * @dentry: dentry to get xattr from
	188	+ * @name__str: name of the xattr
	189	+ * @value_p: xattr value
	190	+ * @flags: flags to pass into filesystem operations
	191	+ *
	192	+ * Set xattr name__str of dentry to the value in value_ptr.
	193	+ *
	194	+ * For security reasons, only name__str with prefix "security.bpf."
	195	+ * is allowed.
	196	+ *
	197	+ * The caller already locked dentry->d_inode.
	198	+ *
	199	+ * Return: 0 on success, a negative value on error.
	200	+ */
	201	+int bpf_set_dentry_xattr_locked(struct dentry dentry, const char name__str,
	202	+ const struct bpf_dynptr *value_p, int flags)
	203	+{
	204	+
	205	+ struct bpf_dynptr_kern value_ptr = (struct bpf_dynptr_kern )value_p;
	206	+ struct inode *inode = d_inode(dentry);
	207	+ const void *value;
	208	+ u32 value_len;
	209	+ int ret;
	210	+
	211	+ value_len = __bpf_dynptr_size(value_ptr);
	212	+ value = __bpf_dynptr_data(value_ptr, value_len);
	213	+ if (!value)
	214	+ return -EINVAL;
	215	+
	216	+ ret = bpf_xattr_write_permission(name__str, inode);
	217	+ if (ret)
	218	+ return ret;
	219	+
	220	+ ret = __vfs_setxattr(&nop_mnt_idmap, dentry, inode, name__str,
	221	+ value, value_len, flags);
	222	+ if (!ret) {
	223	+ fsnotify_xattr(dentry);
	224	+
	225	+ /* This xattr is set by BPF LSM, so we do not call
	226	+ * security_inode_post_setxattr. Otherwise, we would
	227	+ * risk deadlocks by calling back to the same kfunc.
	228	+ *
	229	+ * This is the same as security_inode_setsecurity().
	230	+ */
	231	+ }
	232	+ return ret;
	233	+}
	234	+
	235	+/**
	236	+ * bpf_remove_dentry_xattr_locked - remove a xattr of a dentry
	237	+ * @dentry: dentry to get xattr from
	238	+ * @name__str: name of the xattr
	239	+ *
	240	+ * Rmove xattr name__str of dentry.
	241	+ *
	242	+ * For security reasons, only name__str with prefix "security.bpf."
	243	+ * is allowed.
	244	+ *
	245	+ * The caller already locked dentry->d_inode.
	246	+ *
	247	+ * Return: 0 on success, a negative value on error.
	248	+ */
	249	+int bpf_remove_dentry_xattr_locked(struct dentry dentry, const char name__str)
	250	+{
	251	+ struct inode *inode = d_inode(dentry);
	252	+ int ret;
	253	+
	254	+ ret = bpf_xattr_write_permission(name__str, inode);
	255	+ if (ret)
	256	+ return ret;
	257	+
	258	+ ret = __vfs_removexattr(&nop_mnt_idmap, dentry, name__str);
	259	+ if (!ret) {
	260	+ fsnotify_xattr(dentry);
	261	+
	262	+ /* This xattr is removed by BPF LSM, so we do not call
	263	+ * security_inode_post_removexattr. Otherwise, we would
	264	+ * risk deadlocks by calling back to the same kfunc.
	265	+ */
	266	+ }
	267	+ return ret;
	268	+}
	269	+
	270	+__bpf_kfunc_start_defs();
	271	+
	272	+/**
	273	+ * bpf_set_dentry_xattr - set a xattr of a dentry
	274	+ * @dentry: dentry to get xattr from
	275	+ * @name__str: name of the xattr
	276	+ * @value_p: xattr value
	277	+ * @flags: flags to pass into filesystem operations
	278	+ *
	279	+ * Set xattr name__str of dentry to the value in value_ptr.
	280	+ *
	281	+ * For security reasons, only name__str with prefix "security.bpf."
	282	+ * is allowed.
	283	+ *
	284	+ * The caller has not locked dentry->d_inode.
	285	+ *
	286	+ * Return: 0 on success, a negative value on error.
	287	+ */
	288	+__bpf_kfunc int bpf_set_dentry_xattr(struct dentry dentry, const char name__str,
	289	+ const struct bpf_dynptr *value_p, int flags)
	290	+{
	291	+ struct inode *inode = d_inode(dentry);
	292	+ int ret;
	293	+
	294	+ inode_lock(inode);
	295	+ ret = bpf_set_dentry_xattr_locked(dentry, name__str, value_p, flags);
	296	+ inode_unlock(inode);
	297	+ return ret;
	298	+}
	299	+
	300	+/**
	301	+ * bpf_remove_dentry_xattr - remove a xattr of a dentry
	302	+ * @dentry: dentry to get xattr from
	303	+ * @name__str: name of the xattr
	304	+ *
	305	+ * Rmove xattr name__str of dentry.
	306	+ *
	307	+ * For security reasons, only name__str with prefix "security.bpf."
	308	+ * is allowed.
	309	+ *
	310	+ * The caller has not locked dentry->d_inode.
	311	+ *
	312	+ * Return: 0 on success, a negative value on error.
	313	+ */
	314	+__bpf_kfunc int bpf_remove_dentry_xattr(struct dentry dentry, const char name__str)
	315	+{
	316	+ struct inode *inode = d_inode(dentry);
	317	+ int ret;
	318	+
	319	+ inode_lock(inode);
	320	+ ret = bpf_remove_dentry_xattr_locked(dentry, name__str);
	321	+ inode_unlock(inode);
	322	+ return ret;
	323	+}
	324	+
	325	+__bpf_kfunc_end_defs();
	326	+
157	327	BTF_KFUNCS_START(bpf_fs_kfunc_set_ids)
158	328	BTF_ID_FLAGS(func, bpf_get_task_exe_file,
159	329	KF_ACQUIRE \| KF_TRUSTED_ARGS \| KF_RET_NULL)
@@ -161,6 +331,8 @@ BTF_ID_FLAGS(func, bpf_put_file, KF_RELEASE) · 1 unmodified line
161	331	BTF_ID_FLAGS(func, bpf_path_d_path, KF_TRUSTED_ARGS)
162	332	BTF_ID_FLAGS(func, bpf_get_dentry_xattr, KF_SLEEPABLE \| KF_TRUSTED_ARGS)
163	333	BTF_ID_FLAGS(func, bpf_get_file_xattr, KF_SLEEPABLE \| KF_TRUSTED_ARGS)
	334	+BTF_ID_FLAGS(func, bpf_set_dentry_xattr, KF_SLEEPABLE \| KF_TRUSTED_ARGS)
	335	+BTF_ID_FLAGS(func, bpf_remove_dentry_xattr, KF_SLEEPABLE \| KF_TRUSTED_ARGS)
164	336	BTF_KFUNCS_END(bpf_fs_kfunc_set_ids)
165	337
166	338	static int bpf_fs_kfuncs_filter(const struct bpf_prog *prog, u32 kfunc_id)
@@ -171,6 +343,37 @@ static int bpf_fs_kfuncs_filter(const struct bpf_prog *prog, u32 kfunc_id) · 4 unmodified lines
171	343	return -EACCES;
172	344	}
173	345
	346	+/* bpf_[set\|remove]_dentry_xattr.* hooks have KF_TRUSTED_ARGS and
	347	+ * KF_SLEEPABLE, so they are only available to sleepable hooks with
	348	+ * dentry arguments.
	349	+ *
	350	+ * Setting and removing xattr requires exclusive lock on dentry->d_inode.
	351	+ * Some hooks already locked d_inode, while some hooks have not locked
	352	+ * d_inode. Therefore, we need different kfuncs for different hooks.
	353	+ * Specifically, hooks in the following list (d_inode_locked_hooks)
	354	+ * should call bpf_[set\|remove]_dentry_xattr_locked; while other hooks
	355	+ * should call bpf_[set\|remove]_dentry_xattr.
	356	+ */
	357	+BTF_SET_START(d_inode_locked_hooks)
	358	+BTF_ID(func, bpf_lsm_inode_post_removexattr)
	359	+BTF_ID(func, bpf_lsm_inode_post_setattr)
	360	+BTF_ID(func, bpf_lsm_inode_post_setxattr)
	361	+BTF_ID(func, bpf_lsm_inode_removexattr)
	362	+BTF_ID(func, bpf_lsm_inode_rmdir)
	363	+BTF_ID(func, bpf_lsm_inode_setattr)
	364	+BTF_ID(func, bpf_lsm_inode_setxattr)
	365	+BTF_ID(func, bpf_lsm_inode_unlink)
	366	+#ifdef CONFIG_SECURITY_PATH
	367	+BTF_ID(func, bpf_lsm_path_unlink)
	368	+BTF_ID(func, bpf_lsm_path_rmdir)
	369	+#endif /* CONFIG_SECURITY_PATH */
	370	+BTF_SET_END(d_inode_locked_hooks)
	371	+
	372	+bool bpf_lsm_has_d_inode_locked(const struct bpf_prog *prog)
	373	+{
	374	+ return btf_id_set_contains(&d_inode_locked_hooks, prog->aux->attach_btf_id);
	375	+}
	376	+
174	377	static const struct btf_kfunc_id_set bpf_fs_kfunc_set = {
175	378	.owner = THIS_MODULE,
176	379	.set = &bpf_fs_kfunc_set_ids,

include/linux/bpf-cgroup.h+1−0

@@ -111,6 +111,7 @@ struct bpf_prog_list { · 110 unmodified lines
111	111	struct bpf_prog *prog;
112	112	struct bpf_cgroup_link *link;
113	113	struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE];
	114	+ u32 flags;
114	115	};
115	116
116	117	int cgroup_bpf_inherit(struct cgroup *cgrp);

include/linux/bpf.h+29−2

@@ -968,6 +968,7 @@ struct bpf_insn_access_aux { · 967 unmodified lines
968	968	struct {
969	969	struct btf *btf;
970	970	u32 btf_id;
	971	+ u32 ref_obj_id;
971	972	};
972	973	};
973	974	struct bpf_verifier_log log; / for verbose logs */
@@ -990,6 +991,21 @@ static inline bool bpf_pseudo_func(const struct bpf_insn *insn) · 16 unmodified lines
990	991	return bpf_is_ldimm64(insn) && insn->src_reg == BPF_PSEUDO_FUNC;
991	992	}
992	993
	994	+/* Given a BPF_ATOMIC instruction @atomic_insn, return true if it is an
	995	+ * atomic load or store, and false if it is a read-modify-write instruction.
	996	+ */
	997	+static inline bool
	998	+bpf_atomic_is_load_store(const struct bpf_insn *atomic_insn)
	999	+{
	1000	+ switch (atomic_insn->imm) {
	1001	+ case BPF_LOAD_ACQ:
	1002	+ case BPF_STORE_REL:
	1003	+ return true;
	1004	+ default:
	1005	+ return false;
	1006	+ }
	1007	+}
	1008	+
993	1009	struct bpf_prog_ops {
994	1010	int (test_run)(struct bpf_prog prog, const union bpf_attr *kattr,
995	1011	union bpf_attr __user *uattr);
@@ -1481,6 +1497,8 @@ struct bpf_ctx_arg_aux { · 485 unmodified lines
1481	1497	enum bpf_reg_type reg_type;
1482	1498	struct btf *btf;
1483	1499	u32 btf_id;
	1500	+ u32 ref_obj_id;
	1501	+ bool refcounted;
1484	1502	};
1485	1503
1486	1504	struct btf_mod_pair {
@@ -1503,11 +1521,12 @@ struct bpf_prog_aux { · 16 unmodified lines
1503	1521	u32 real_func_cnt; /* includes hidden progs, only used for JIT and freeing progs */
1504	1522	u32 func_idx; /* 0 for non-func prog, the index in func array for func prog */
1505	1523	u32 attach_btf_id; /* in-kernel BTF type id to attach to */
	1524	+ u32 attach_st_ops_member_off;
1506	1525	u32 ctx_arg_info_size;
1507	1526	u32 max_rdonly_access;
1508	1527	u32 max_rdwr_access;
1509	1528	struct btf *attach_btf;
1510		− const struct bpf_ctx_arg_aux *ctx_arg_info;
	1529	+ struct bpf_ctx_arg_aux *ctx_arg_info;
1511	1530	void __percpu *priv_stack_ptr;
1512	1531	struct mutex dst_mutex; /* protects dst_* pointers below, after prog becomes visible */
1513	1532	struct bpf_prog *dst_prog;
@@ -1528,6 +1547,7 @@ struct bpf_prog_aux { · 14 unmodified lines
1528	1547	bool jits_use_priv_stack;
1529	1548	bool priv_stack_requested;
1530	1549	bool changes_pkt_data;
	1550	+ bool might_sleep;
1531	1551	u64 prog_array_member_cnt; /* counts how many times as member of prog_array */
1532	1552	struct mutex ext_mutex; /* mutex for is_extended and prog_array_member_cnt */
1533	1553	struct bpf_arena *arena;
@@ -1547,6 +1567,7 @@ struct bpf_prog_aux { · 13 unmodified lines
1547	1567	#endif
1548	1568	struct bpf_ksym ksym;
1549	1569	const struct bpf_prog_ops *ops;
	1570	+ const struct bpf_struct_ops *st_ops;
1550	1571	struct bpf_map **used_maps;
1551	1572	struct mutex used_maps_mutex; /* mutex for used_maps and used_map_cnt */
1552	1573	struct btf_mod_pair *used_btfs;
@@ -1945,6 +1966,9 @@ static inline void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_op · 392 unmodified lines
1945	1966
1946	1967	#endif
1947	1968
	1969	+int bpf_prog_ctx_arg_info_init(struct bpf_prog *prog,
	1970	+ const struct bpf_ctx_arg_aux *info, u32 cnt);
	1971	+
1948	1972	#if defined(CONFIG_CGROUP_BPF) && defined(CONFIG_BPF_LSM)
1949	1973	int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
1950	1974	int cgroup_atype);
@@ -1980,6 +2004,7 @@ struct bpf_array { · 29 unmodified lines
1980	2004	*/
1981	2005	enum {
1982	2006	BPF_MAX_LOOPS = 8 * 1024 * 1024,
	2007	+ BPF_MAX_TIMED_LOOPS = 0xffff,
1983	2008	};
1984	2009
1985	2010	#define BPF_F_ACCESS_MASK (BPF_F_RDONLY \| \
@@ -2036,6 +2061,8 @@ int bpf_prog_calc_tag(struct bpf_prog *fp); · 50 unmodified lines
2036	2061	const struct bpf_func_proto *bpf_get_trace_printk_proto(void);
2037	2062	const struct bpf_func_proto *bpf_get_trace_vprintk_proto(void);
2038	2063
	2064	+const struct bpf_func_proto *bpf_get_perf_event_read_value_proto(void);
	2065	+
2039	2066	typedef unsigned long (bpf_ctx_copy_t)(void dst, const void *src,
2040	2067	unsigned long off, unsigned long len);
2041	2068	typedef u32 (*bpf_convert_ctx_access_t)(enum bpf_access_type type,
@@ -2546,7 +2573,7 @@ struct bpf_iter__bpf_map_elem { · 504 unmodified lines
2546	2573
2547	2574	int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info);
2548	2575	void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info);
2549		−bool bpf_iter_prog_supported(struct bpf_prog *prog);
	2576	+int bpf_iter_prog_supported(struct bpf_prog *prog);
2550	2577	const struct bpf_func_proto *
2551	2578	bpf_iter_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog);
2552	2579	int bpf_iter_link_attach(const union bpf_attr attr, bpfptr_t uattr, struct bpf_prog prog);

include/linux/bpf_lsm.h+18−0

@@ -48,6 +48,11 @@ void bpf_lsm_find_cgroup_shim(const struct bpf_prog prog, bpf_func_t bpf_func) · 47 unmodified lines
48	48
49	49	int bpf_lsm_get_retval_range(const struct bpf_prog *prog,
50	50	struct bpf_retval_range *range);
	51	+int bpf_set_dentry_xattr_locked(struct dentry dentry, const char name__str,
	52	+ const struct bpf_dynptr *value_p, int flags);
	53	+int bpf_remove_dentry_xattr_locked(struct dentry dentry, const char name__str);
	54	+bool bpf_lsm_has_d_inode_locked(const struct bpf_prog *prog);
	55	+
51	56	#else /* !CONFIG_BPF_LSM */
52	57
53	58	static inline bool bpf_lsm_is_sleepable_hook(u32 btf_id)
@@ -86,6 +91,19 @@ static inline int bpf_lsm_get_retval_range(const struct bpf_prog *prog, · 32 unmodified lines
86	91	{
87	92	return -EOPNOTSUPP;
88	93	}
	94	+static inline int bpf_set_dentry_xattr_locked(struct dentry dentry, const char name__str,
	95	+ const struct bpf_dynptr *value_p, int flags)
	96	+{
	97	+ return -EOPNOTSUPP;
	98	+}
	99	+static inline int bpf_remove_dentry_xattr_locked(struct dentry dentry, const char name__str)
	100	+{
	101	+ return -EOPNOTSUPP;
	102	+}
	103	+static inline bool bpf_lsm_has_d_inode_locked(const struct bpf_prog *prog)
	104	+{
	105	+ return false;
	106	+}
89	107	#endif /* CONFIG_BPF_LSM */
90	108
91	109	#endif /* _LINUX_BPF_LSM_H */

include/linux/bpf_verifier.h+23−9

@@ -427,11 +427,6 @@ struct bpf_verifier_state { · 426 unmodified lines
427	427	bool active_rcu_lock;
428	428
429	429	bool speculative;
430		− /* If this state was ever pointed-to by other state's loop_entry field
431		− * this flag would be set to true. Used to avoid freeing such states
432		− * while they are still in use.
433		− */
434		− bool used_as_loop_entry;
435	430	bool in_sleepable;
436	431
437	432	/* first and last insn idx of this verifier state */
@@ -458,6 +453,11 @@ struct bpf_verifier_state { · 20 unmodified lines
458	453	u32 dfs_depth;
459	454	u32 callback_unroll_depth;
460	455	u32 may_goto_depth;
	456	+ /* If this state was ever pointed-to by other state's loop_entry field
	457	+ * this flag would be set to true. Used to avoid freeing such states
	458	+ * while they are still in use.
	459	+ */
	460	+ u32 used_as_loop_entry;
461	461	};
462	462
463	463	#define bpf_get_spilled_reg(slot, frame, mask) \
@@ -498,8 +498,10 @@ struct bpf_verifier_state { · 34 unmodified lines
498	498	/* linked list of verifier states used to prune search */
499	499	struct bpf_verifier_state_list {
500	500	struct bpf_verifier_state state;
501		− struct bpf_verifier_state_list *next;
502		− int miss_cnt, hit_cnt;
	501	+ struct list_head node;
	502	+ u32 miss_cnt;
	503	+ u32 hit_cnt:31;
	504	+ u32 in_free_list:1;
503	505	};
504	506
505	507	struct bpf_loop_inline_state {
@@ -589,6 +591,8 @@ struct bpf_insn_aux_data { · 83 unmodified lines
589	591	* accepts callback function as a parameter.
590	592	*/
591	593	bool calls_callback;
	594	+ /* registers alive before this instruction. */
	595	+ u16 live_regs_before;
592	596	};
593	597
594	598	#define MAX_USED_MAPS 64 /* max number of maps accessed by one eBPF program */
@@ -665,6 +669,7 @@ struct bpf_subprog_info { · 70 unmodified lines
665	669	/* true if bpf_fastcall stack region is used by functions that can't be inlined */
666	670	bool keep_fastcall_stack: 1;
667	671	bool changes_pkt_data: 1;
	672	+ bool might_sleep: 1;
668	673
669	674	enum priv_stack_mode priv_stack_mode;
670	675	u8 arg_cnt;
@@ -710,8 +715,11 @@ struct bpf_verifier_env { · 39 unmodified lines
710	715	bool test_state_freq; /* test verifier with different pruning frequency */
711	716	bool test_reg_invariants; /* fail verification on register invariants violations */
712	717	struct bpf_verifier_state cur_state; / current verifier state */
713		− struct bpf_verifier_state_list *explored_states; / search pruning optimization */
714		− struct bpf_verifier_state_list *free_list;
	718	+ /* Search pruning optimization, array of list_heads for
	719	+ * lists of struct bpf_verifier_state_list.
	720	+ */
	721	+ struct list_head *explored_states;
	722	+ struct list_head free_list; /* list of struct bpf_verifier_state_list */
715	723	struct bpf_map used_maps[MAX_USED_MAPS]; / array of map's used by eBPF program */
716	724	struct btf_mod_pair used_btfs[MAX_USED_BTFS]; /* array of BTF's used by BPF program */
717	725	u32 used_map_cnt; /* number of used maps */
@@ -742,7 +750,11 @@ struct bpf_verifier_env { · 24 unmodified lines
742	750	struct {
743	751	int *insn_state;
744	752	int *insn_stack;
	753	+ /* vector of instruction indexes sorted in post-order */
	754	+ int *insn_postorder;
745	755	int cur_stack;
	756	+ /* current position in the insn_postorder vector */
	757	+ int cur_postorder;
746	758	} cfg;
747	759	struct backtrack_state bt;
748	760	struct bpf_insn_hist_entry *insn_hist;
@@ -767,6 +779,8 @@ struct bpf_verifier_env { · 18 unmodified lines
767	779	u32 peak_states;
768	780	/* longest register parentage chain walked for liveness marking */
769	781	u32 longest_mark_read_walk;
	782	+ u32 free_list_size;
	783	+ u32 explored_states_size;
770	784	bpfptr_t fd_array;
771	785
772	786	/* bit mask to keep track of whether a register has been accessed

include/linux/btf.h+3−0

@@ -76,6 +76,9 @@ · 75 unmodified lines
76	76	#define KF_ITER_DESTROY (1 << 10) /* kfunc implements BPF iter destructor */
77	77	#define KF_RCU_PROTECTED (1 << 11) /* kfunc should be protected by rcu cs when they are invoked */
78	78	#define KF_FASTCALL (1 << 12) /* kfunc supports bpf_fastcall protocol */
	79	+#define KF_ARENA_RET (1 << 13) /* kfunc returns an arena pointer */
	80	+#define KF_ARENA_ARG1 (1 << 14) /* kfunc takes an arena pointer as its first argument */
	81	+#define KF_ARENA_ARG2 (1 << 15) /* kfunc takes an arena pointer as its second argument */
79	82
80	83	/*
81	84	* Tag marking a kernel function as a kfunc. This is meant to minimize the

include/linux/filter.h+20−0

@@ -364,6 +364,8 @@ static inline bool insn_is_cast_user(const struct bpf_insn *insn) · 363 unmodified lines
364	364	* BPF_XOR \| BPF_FETCH src_reg = atomic_fetch_xor(dst_reg + off16, src_reg);
365	365	* BPF_XCHG src_reg = atomic_xchg(dst_reg + off16, src_reg)
366	366	* BPF_CMPXCHG r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg)
	367	+ * BPF_LOAD_ACQ dst_reg = smp_load_acquire(src_reg + off16)
	368	+ * BPF_STORE_REL smp_store_release(dst_reg + off16, src_reg)
367	369	*/
368	370
369	371	#define BPF_ATOMIC_OP(SIZE, OP, DST, SRC, OFF) \
@@ -469,6 +471,16 @@ static inline bool insn_is_cast_user(const struct bpf_insn *insn) · 99 unmodified lines
469	471	.off = 0, \
470	472	.imm = BPF_CALL_IMM(FUNC) })
471	473
	474	+/* Kfunc call */
	475	+
	476	+#define BPF_CALL_KFUNC(OFF, IMM) \
	477	+ ((struct bpf_insn) { \
	478	+ .code = BPF_JMP \| BPF_CALL, \
	479	+ .dst_reg = 0, \
	480	+ .src_reg = BPF_PSEUDO_KFUNC_CALL, \
	481	+ .off = OFF, \
	482	+ .imm = IMM })
	483	+
472	484	/* Raw code statement block */
473	485
474	486	#define BPF_RAW_INSN(CODE, DST, SRC, OFF, IMM) \
@@ -659,6 +671,11 @@ struct bpf_prog_stats { · 184 unmodified lines
659	671	struct u64_stats_sync syncp;
660	672	} __aligned(2 * sizeof(u64));
661	673
	674	+struct bpf_timed_may_goto {
	675	+ u64 count;
	676	+ u64 timestamp;
	677	+};
	678	+
662	679	struct sk_filter {
663	680	refcount_t refcnt;
664	681	struct rcu_head rcu;
@@ -1120,8 +1137,11 @@ bool bpf_jit_supports_ptr_xchg(void); · 455 unmodified lines
1120	1137	bool bpf_jit_supports_arena(void);
1121	1138	bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena);
1122	1139	bool bpf_jit_supports_private_stack(void);
	1140	+bool bpf_jit_supports_timed_may_goto(void);
1123	1141	u64 bpf_arch_uaddress_limit(void);
1124	1142	void arch_bpf_stack_walk(bool (consume_fn)(void cookie, u64 ip, u64 sp, u64 bp), void *cookie);
	1143	+u64 arch_bpf_timed_may_goto(void);
	1144	+u64 bpf_check_timed_may_goto(struct bpf_timed_may_goto *);
1125	1145	bool bpf_helper_changes_pkt_data(enum bpf_func_id func_id);
1126	1146
1127	1147	static inline bool bpf_dump_raw_ok(const struct cred *cred)

include/linux/lsm_hook_defs.h+3−3

@@ -426,14 +426,14 @@ LSM_HOOK(void, LSM_RET_VOID, audit_rule_free, void *lsmrule) · 425 unmodified lines
426	426	#endif /* CONFIG_AUDIT */
427	427
428	428	#ifdef CONFIG_BPF_SYSCALL
429		−LSM_HOOK(int, 0, bpf, int cmd, union bpf_attr *attr, unsigned int size)
	429	+LSM_HOOK(int, 0, bpf, int cmd, union bpf_attr *attr, unsigned int size, bool kernel)
430	430	LSM_HOOK(int, 0, bpf_map, struct bpf_map *map, fmode_t fmode)
431	431	LSM_HOOK(int, 0, bpf_prog, struct bpf_prog *prog)
432	432	LSM_HOOK(int, 0, bpf_map_create, struct bpf_map map, union bpf_attr attr,
433		− struct bpf_token *token)
	433	+ struct bpf_token *token, bool kernel)
434	434	LSM_HOOK(void, LSM_RET_VOID, bpf_map_free, struct bpf_map *map)
435	435	LSM_HOOK(int, 0, bpf_prog_load, struct bpf_prog prog, union bpf_attr attr,
436		− struct bpf_token *token)
	436	+ struct bpf_token *token, bool kernel)
437	437	LSM_HOOK(void, LSM_RET_VOID, bpf_prog_free, struct bpf_prog *prog)
438	438	LSM_HOOK(int, 0, bpf_token_create, struct bpf_token token, union bpf_attr attr,
439	439	const struct path *path)

include/linux/mm.h+5−0

@@ -2477,6 +2477,11 @@ extern int access_process_vm(struct task_struct *tsk, unsigned long addr, · 2476 unmodified lines
2477	2477	extern int access_remote_vm(struct mm_struct *mm, unsigned long addr,
2478	2478	void *buf, int len, unsigned int gup_flags);
2479	2479
	2480	+#ifdef CONFIG_BPF_SYSCALL
	2481	+extern int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr,
	2482	+ void *buf, int len, unsigned int gup_flags);
	2483	+#endif
	2484	+
2480	2485	long get_user_pages_remote(struct mm_struct *mm,
2481	2486	unsigned long start, unsigned long nr_pages,
2482	2487	unsigned int gup_flags, struct page **pages,

include/linux/security.h+6−6

@@ -2249,14 +2249,14 @@ struct bpf_map; · 2248 unmodified lines
2249	2249	struct bpf_prog;
2250	2250	struct bpf_token;
2251	2251	#ifdef CONFIG_SECURITY
2252		−extern int security_bpf(int cmd, union bpf_attr *attr, unsigned int size);
	2252	+extern int security_bpf(int cmd, union bpf_attr *attr, unsigned int size, bool kernel);
2253	2253	extern int security_bpf_map(struct bpf_map *map, fmode_t fmode);
2254	2254	extern int security_bpf_prog(struct bpf_prog *prog);
2255	2255	extern int security_bpf_map_create(struct bpf_map map, union bpf_attr attr,
2256		− struct bpf_token *token);
	2256	+ struct bpf_token *token, bool kernel);
2257	2257	extern void security_bpf_map_free(struct bpf_map *map);
2258	2258	extern int security_bpf_prog_load(struct bpf_prog prog, union bpf_attr attr,
2259		− struct bpf_token *token);
	2259	+ struct bpf_token *token, bool kernel);
2260	2260	extern void security_bpf_prog_free(struct bpf_prog *prog);
2261	2261	extern int security_bpf_token_create(struct bpf_token token, union bpf_attr attr,
2262	2262	const struct path *path);
@@ -2265,7 +2265,7 @@ extern int security_bpf_token_cmd(const struct bpf_token *token, enum bpf_cmd cm · 2 unmodified lines
2265	2265	extern int security_bpf_token_capable(const struct bpf_token *token, int cap);
2266	2266	#else
2267	2267	static inline int security_bpf(int cmd, union bpf_attr *attr,
2268		− unsigned int size)
	2268	+ unsigned int size, bool kernel)
2269	2269	{
2270	2270	return 0;
2271	2271	}
@@ -2281,7 +2281,7 @@ static inline int security_bpf_prog(struct bpf_prog *prog) · 9 unmodified lines
2281	2281	}
2282	2282
2283	2283	static inline int security_bpf_map_create(struct bpf_map map, union bpf_attr attr,
2284		− struct bpf_token *token)
	2284	+ struct bpf_token *token, bool kernel)
2285	2285	{
2286	2286	return 0;
2287	2287	}
@@ -2290,7 +2290,7 @@ static inline void security_bpf_map_free(struct bpf_map *map) · 2 unmodified lines
2290	2290	{ }
2291	2291
2292	2292	static inline int security_bpf_prog_load(struct bpf_prog prog, union bpf_attr attr,
2293		− struct bpf_token *token)
	2293	+ struct bpf_token *token, bool kernel)
2294	2294	{
2295	2295	return 0;
2296	2296	}

include/uapi/linux/bpf.h+9−1

@@ -51,6 +51,9 @@ · 50 unmodified lines
51	51	#define BPF_XCHG (0xe0 \| BPF_FETCH) /* atomic exchange */
52	52	#define BPF_CMPXCHG (0xf0 \| BPF_FETCH) /* atomic compare-and-write */
53	53
	54	+#define BPF_LOAD_ACQ 0x100 /* load-acquire */
	55	+#define BPF_STORE_REL 0x110 /* store-release */
	56	+
54	57	enum bpf_cond_pseudo_jmp {
55	58	BPF_MAY_GOTO = 0,
56	59	};
@@ -1207,6 +1210,7 @@ enum bpf_perf_event_type { · 1150 unmodified lines
1207	1210	#define BPF_F_BEFORE (1U << 3)
1208	1211	#define BPF_F_AFTER (1U << 4)
1209	1212	#define BPF_F_ID (1U << 5)
	1213	+#define BPF_F_PREORDER (1U << 6)
1210	1214	#define BPF_F_LINK BPF_F_LINK /* 1 << 13 */
1211	1215
1212	1216	/* If BPF_F_STRICT_ALIGNMENT is used in BPF_PROG_LOAD command, the
@@ -1648,6 +1652,7 @@ union bpf_attr { · 435 unmodified lines
1648	1652	};
1649	1653	__u32 next_id;
1650	1654	__u32 open_flags;
	1655	+ __s32 fd_by_id_token_fd;
1651	1656	};
1652	1657
1653	1658	struct { /* anonymous struct used by BPF_OBJ_GET_INFO_BY_FD */
@@ -6019,7 +6024,10 @@ union bpf_attr { · 4365 unmodified lines
6019	6024	FN(user_ringbuf_drain, 209, ##ctx) \
6020	6025	FN(cgrp_storage_get, 210, ##ctx) \
6021	6026	FN(cgrp_storage_delete, 211, ##ctx) \
6022		− /* */
	6027	+ /* This helper list is effectively frozen. If you are trying to \
	6028	+ * add a new helper, you should add a kfunc instead which has \
	6029	+ * less stability guarantees. See Documentation/bpf/kfuncs.rst \
	6030	+ */
6023	6031
6024	6032	/* backwards-compatibility macros for users of __BPF_FUNC_MAPPER that don't
6025	6033	* know or care about integer value that is now passed as second argument

include/uapi/linux/btf.h+2−1

@@ -36,7 +36,8 @@ struct btf_type { · 35 unmodified lines
36	36	* bits 24-28: kind (e.g. int, ptr, array...etc)
37	37	* bits 29-30: unused
38	38	* bit 31: kind_flag, currently used by
39		− * struct, union, enum, fwd and enum64
	39	+ * struct, union, enum, fwd, enum64,
	40	+ * decl_tag and type_tag
40	41	*/
41	42	__u32 info;
42	43	/* "size" is used by INT, ENUM, STRUCT, UNION, DATASEC and ENUM64.

include/uapi/linux/xattr.h+4−0

@@ -83,6 +83,10 @@ struct xattr_args { · 82 unmodified lines
83	83	#define XATTR_CAPS_SUFFIX "capability"
84	84	#define XATTR_NAME_CAPS XATTR_SECURITY_PREFIX XATTR_CAPS_SUFFIX
85	85
	86	+#define XATTR_BPF_LSM_SUFFIX "bpf."
	87	+#define XATTR_NAME_BPF_LSM (XATTR_SECURITY_PREFIX XATTR_BPF_LSM_SUFFIX)
	88	+#define XATTR_NAME_BPF_LSM_LEN (sizeof(XATTR_NAME_BPF_LSM) - 1)
	89	+
86	90	#define XATTR_POSIX_ACL_ACCESS "posix_acl_access"
87	91	#define XATTR_NAME_POSIX_ACL_ACCESS XATTR_SYSTEM_PREFIX XATTR_POSIX_ACL_ACCESS
88	92	#define XATTR_POSIX_ACL_DEFAULT "posix_acl_default"

kernel/bpf/arena.c+2−2

@@ -577,8 +577,8 @@ __bpf_kfunc void bpf_arena_free_pages(void p__map, void ptr__ign, u32 page_cnt · 576 unmodified lines
577	577	__bpf_kfunc_end_defs();
578	578
579	579	BTF_KFUNCS_START(arena_kfuncs)
580		−BTF_ID_FLAGS(func, bpf_arena_alloc_pages, KF_TRUSTED_ARGS \| KF_SLEEPABLE)
581		−BTF_ID_FLAGS(func, bpf_arena_free_pages, KF_TRUSTED_ARGS \| KF_SLEEPABLE)
	580	+BTF_ID_FLAGS(func, bpf_arena_alloc_pages, KF_TRUSTED_ARGS \| KF_SLEEPABLE \| KF_ARENA_RET \| KF_ARENA_ARG2)
	581	+BTF_ID_FLAGS(func, bpf_arena_free_pages, KF_TRUSTED_ARGS \| KF_SLEEPABLE \| KF_ARENA_ARG2)
582	582	BTF_KFUNCS_END(arena_kfuncs)
583	583
584	584	static const struct btf_kfunc_id_set common_kfunc_set = {

kernel/bpf/bpf_cgrp_storage.c+6−5

@@ -161,6 +161,7 @@ BPF_CALL_5(bpf_cgrp_storage_get, struct bpf_map , map, struct cgroup , cgroup, · 160 unmodified lines
161	161	void *, value, u64, flags, gfp_t, gfp_flags)
162	162	{
163	163	struct bpf_local_storage_data *sdata;
	164	+ bool nobusy;
164	165
165	166	WARN_ON_ONCE(!bpf_rcu_lock_held());
166	167	if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
@@ -169,21 +170,21 @@ BPF_CALL_5(bpf_cgrp_storage_get, struct bpf_map , map, struct cgroup , cgroup, · 2 unmodified lines
169	170	if (!cgroup)
170	171	return (unsigned long)NULL;
171	172
172		− if (!bpf_cgrp_storage_trylock())
173		− return (unsigned long)NULL;
	173	+ nobusy = bpf_cgrp_storage_trylock();
174	174
175		− sdata = cgroup_storage_lookup(cgroup, map, true);
	175	+ sdata = cgroup_storage_lookup(cgroup, map, nobusy);
176	176	if (sdata)
177	177	goto unlock;
178	178
179	179	/* only allocate new storage, when the cgroup is refcounted */
180	180	if (!percpu_ref_is_dying(&cgroup->self.refcnt) &&
181		− (flags & BPF_LOCAL_STORAGE_GET_F_CREATE))
	181	+ (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) && nobusy)
182	182	sdata = bpf_local_storage_update(cgroup, (struct bpf_local_storage_map *)map,
183	183	value, BPF_NOEXIST, false, gfp_flags);
184	184
185	185	unlock:
186		− bpf_cgrp_storage_unlock();
	186	+ if (nobusy)
	187	+ bpf_cgrp_storage_unlock();
187	188	return IS_ERR_OR_NULL(sdata) ? (unsigned long)NULL : (unsigned long)sdata->data;
188	189	}
189	190

kernel/bpf/bpf_iter.c+6−7

@@ -335,7 +335,7 @@ static void cache_btf_id(struct bpf_iter_target_info *tinfo, · 334 unmodified lines
335	335	tinfo->btf_id = prog->aux->attach_btf_id;
336	336	}
337	337
338		−bool bpf_iter_prog_supported(struct bpf_prog *prog)
	338	+int bpf_iter_prog_supported(struct bpf_prog *prog)
339	339	{
340	340	const char *attach_fname = prog->aux->attach_func_name;
341	341	struct bpf_iter_target_info tinfo = NULL, iter;
@@ -344,7 +344,7 @@ bool bpf_iter_prog_supported(struct bpf_prog *prog) · 2 unmodified lines
344	344	int prefix_len = strlen(prefix);
345	345
346	346	if (strncmp(attach_fname, prefix, prefix_len))
347		− return false;
	347	+ return -EINVAL;
348	348
349	349	mutex_lock(&targets_mutex);
350	350	list_for_each_entry(iter, &targets, list) {
@@ -360,12 +360,11 @@ bool bpf_iter_prog_supported(struct bpf_prog *prog) · 9 unmodified lines
360	360	}
361	361	mutex_unlock(&targets_mutex);
362	362
363		− if (tinfo) {
364		− prog->aux->ctx_arg_info_size = tinfo->reg_info->ctx_arg_info_size;
365		− prog->aux->ctx_arg_info = tinfo->reg_info->ctx_arg_info;
366		− }
	363	+ if (!tinfo)
	364	+ return -EINVAL;
367	365
368		− return tinfo != NULL;
	366	+ return bpf_prog_ctx_arg_info_init(prog, tinfo->reg_info->ctx_arg_info,
	367	+ tinfo->reg_info->ctx_arg_info_size);
369	368	}
370	369
371	370	const struct bpf_func_proto *

kernel/bpf/bpf_lsm.c+2−0

@@ -316,7 +316,9 @@ BTF_ID(func, bpf_lsm_inode_getxattr) · 315 unmodified lines
316	316	BTF_ID(func, bpf_lsm_inode_mknod)
317	317	BTF_ID(func, bpf_lsm_inode_need_killpriv)
318	318	BTF_ID(func, bpf_lsm_inode_post_setxattr)
	319	+BTF_ID(func, bpf_lsm_inode_post_removexattr)
319	320	BTF_ID(func, bpf_lsm_inode_readlink)
	321	+BTF_ID(func, bpf_lsm_inode_removexattr)
320	322	BTF_ID(func, bpf_lsm_inode_rename)
321	323	BTF_ID(func, bpf_lsm_inode_rmdir)
322	324	BTF_ID(func, bpf_lsm_inode_setattr)

kernel/bpf/bpf_struct_ops.c+75−60

@@ -146,39 +146,7 @@ void bpf_struct_ops_image_free(void *image) · 145 unmodified lines
146	146	}
147	147
148	148	#define MAYBE_NULL_SUFFIX "__nullable"
149		−#define MAX_STUB_NAME 128
150		−
151		−/* Return the type info of a stub function, if it exists.
152		− *
153		− * The name of a stub function is made up of the name of the struct_ops and
154		− * the name of the function pointer member, separated by "__". For example,
155		− * if the struct_ops type is named "foo_ops" and the function pointer
156		− * member is named "bar", the stub function name would be "foo_ops__bar".
157		− */
158		−static const struct btf_type *
159		−find_stub_func_proto(const struct btf btf, const char st_op_name,
160		− const char *member_name)
161		−{
162		− char stub_func_name[MAX_STUB_NAME];
163		− const struct btf_type *func_type;
164		− s32 btf_id;
165		− int cp;
166		−
167		− cp = snprintf(stub_func_name, MAX_STUB_NAME, "%s__%s",
168		− st_op_name, member_name);
169		− if (cp >= MAX_STUB_NAME) {
170		− pr_warn("Stub function name too long\n");
171		− return NULL;
172		− }
173		− btf_id = btf_find_by_name_kind(btf, stub_func_name, BTF_KIND_FUNC);
174		− if (btf_id < 0)
175		− return NULL;
176		− func_type = btf_type_by_id(btf, btf_id);
177		− if (!func_type)
178		− return NULL;
179		−
180		− return btf_type_by_id(btf, func_type->type); /* FUNC_PROTO */
181		−}
	149	+#define REFCOUNTED_SUFFIX "__ref"
182	150
183	151	/* Prepare argument info for every nullable argument of a member of a
184	152	* struct_ops type.
@@ -203,27 +171,44 @@ find_stub_func_proto(const struct btf btf, const char st_op_name, · 18 unmodified lines
203	171	static int prepare_arg_info(struct btf *btf,
204	172	const char *st_ops_name,
205	173	const char *member_name,
206		− const struct btf_type *func_proto,
	174	+ const struct btf_type func_proto, void stub_func_addr,
207	175	struct bpf_struct_ops_arg_info *arg_info)
208	176	{
209	177	const struct btf_type stub_func_proto, pointed_type;
	178	+ bool is_nullable = false, is_refcounted = false;
210	179	const struct btf_param stub_args, args;
211	180	struct bpf_ctx_arg_aux info, info_buf;
212	181	u32 nargs, arg_no, info_cnt = 0;
	182	+ char ksym[KSYM_SYMBOL_LEN];
	183	+ const char *stub_fname;
	184	+ const char *suffix;
	185	+ s32 stub_func_id;
213	186	u32 arg_btf_id;
214	187	int offset;
215	188
216		− stub_func_proto = find_stub_func_proto(btf, st_ops_name, member_name);
217		− if (!stub_func_proto)
218		− return 0;
	189	+ stub_fname = kallsyms_lookup((unsigned long)stub_func_addr, NULL, NULL, NULL, ksym);
	190	+ if (!stub_fname) {
	191	+ pr_warn("Cannot find the stub function name for the %s in struct %s\n",
	192	+ member_name, st_ops_name);
	193	+ return -ENOENT;
	194	+ }
	195	+
	196	+ stub_func_id = btf_find_by_name_kind(btf, stub_fname, BTF_KIND_FUNC);
	197	+ if (stub_func_id < 0) {
	198	+ pr_warn("Cannot find the stub function %s in btf\n", stub_fname);
	199	+ return -ENOENT;
	200	+ }
	201	+
	202	+ stub_func_proto = btf_type_by_id(btf, stub_func_id);
	203	+ stub_func_proto = btf_type_by_id(btf, stub_func_proto->type);
219	204
220	205	/* Check if the number of arguments of the stub function is the same
221	206	* as the number of arguments of the function pointer.
222	207	*/
223	208	nargs = btf_type_vlen(func_proto);
224	209	if (nargs != btf_type_vlen(stub_func_proto)) {
225		− pr_warn("the number of arguments of the stub function %s__%s does not match the number of arguments of the member %s of struct %s\n",
226		− st_ops_name, member_name, member_name, st_ops_name);
	210	+ pr_warn("the number of arguments of the stub function %s does not match the number of arguments of the member %s of struct %s\n",
	211	+ stub_fname, member_name, st_ops_name);
227	212	return -EINVAL;
228	213	}
229	214
@@ -241,10 +226,18 @@ static int prepare_arg_info(struct btf *btf, · 11 unmodified lines
241	226	info = info_buf;
242	227	for (arg_no = 0; arg_no < nargs; arg_no++) {
243	228	/* Skip arguments that is not suffixed with
244		− * "__nullable".
	229	+ * "__nullable or __ref".
245	230	*/
246		− if (!btf_param_match_suffix(btf, &stub_args[arg_no],
247		− MAYBE_NULL_SUFFIX))
	231	+ is_nullable = btf_param_match_suffix(btf, &stub_args[arg_no],
	232	+ MAYBE_NULL_SUFFIX);
	233	+ is_refcounted = btf_param_match_suffix(btf, &stub_args[arg_no],
	234	+ REFCOUNTED_SUFFIX);
	235	+
	236	+ if (is_nullable)
	237	+ suffix = MAYBE_NULL_SUFFIX;
	238	+ else if (is_refcounted)
	239	+ suffix = REFCOUNTED_SUFFIX;
	240	+ else
248	241	continue;
249	242
250	243	/* Should be a pointer to struct */
@@ -253,30 +246,34 @@ static int prepare_arg_info(struct btf *btf, · 2 unmodified lines
253	246	&arg_btf_id);
254	247	if (!pointed_type \|\|
255	248	!btf_type_is_struct(pointed_type)) {
256		− pr_warn("stub function %s__%s has %s tagging to an unsupported type\n",
257		− st_ops_name, member_name, MAYBE_NULL_SUFFIX);
	249	+ pr_warn("stub function %s has %s tagging to an unsupported type\n",
	250	+ stub_fname, suffix);
258	251	goto err_out;
259	252	}
260	253
261	254	offset = btf_ctx_arg_offset(btf, func_proto, arg_no);
262	255	if (offset < 0) {
263		− pr_warn("stub function %s__%s has an invalid trampoline ctx offset for arg#%u\n",
264		− st_ops_name, member_name, arg_no);
	256	+ pr_warn("stub function %s has an invalid trampoline ctx offset for arg#%u\n",
	257	+ stub_fname, arg_no);
265	258	goto err_out;
266	259	}
267	260
268	261	if (args[arg_no].type != stub_args[arg_no].type) {
269		− pr_warn("arg#%u type in stub function %s__%s does not match with its original func_proto\n",
270		− arg_no, st_ops_name, member_name);
	262	+ pr_warn("arg#%u type in stub function %s does not match with its original func_proto\n",
	263	+ arg_no, stub_fname);
271	264	goto err_out;
272	265	}
273	266
274	267	/* Fill the information of the new argument */
275		− info->reg_type =
276		− PTR_TRUSTED \| PTR_TO_BTF_ID \| PTR_MAYBE_NULL;
277	268	info->btf_id = arg_btf_id;
278	269	info->btf = btf;
279	270	info->offset = offset;
	271	+ if (is_nullable) {
	272	+ info->reg_type = PTR_TRUSTED \| PTR_TO_BTF_ID \| PTR_MAYBE_NULL;
	273	+ } else if (is_refcounted) {
	274	+ info->reg_type = PTR_TRUSTED \| PTR_TO_BTF_ID;
	275	+ info->refcounted = true;
	276	+ }
280	277
281	278	info++;
282	279	info_cnt++;
@@ -324,6 +321,13 @@ static bool is_module_member(const struct btf *btf, u32 id) · 41 unmodified lines
324	321	return !strcmp(btf_name_by_offset(btf, t->name_off), "module");
325	322	}
326	323
	324	+int bpf_struct_ops_supported(const struct bpf_struct_ops *st_ops, u32 moff)
	325	+{
	326	+ void func_ptr = (void **)(st_ops->cfi_stubs + moff);
	327	+
	328	+ return func_ptr ? 0 : -ENOTSUPP;
	329	+}
	330	+
327	331	int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc,
328	332	struct btf *btf,
329	333	struct bpf_verifier_log *log)
@@ -386,8 +390,11 @@ int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, · 56 unmodified lines
386	390	st_ops_desc->value_type = btf_type_by_id(btf, value_id);
387	391
388	392	for_each_member(i, t, member) {
389		− const struct btf_type *func_proto;
	393	+ const struct btf_type func_proto, ret_type;
	394	+ void **stub_func_addr;
	395	+ u32 moff;
390	396
	397	+ moff = __btf_member_bit_offset(t, member) / 8;
391	398	mname = btf_name_by_offset(btf, member->name_off);
392	399	if (!*mname) {
393	400	pr_warn("anon member in struct %s is not supported\n",
@@ -413,9 +420,23 @@ int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, · 19 unmodified lines
413	420	func_proto = btf_type_resolve_func_ptr(btf,
414	421	member->type,
415	422	NULL);
416		− if (!func_proto)
	423	+
	424	+ /* The member is not a function pointer or
	425	+ * the function pointer is not supported.
	426	+ */
	427	+ if (!func_proto \|\| bpf_struct_ops_supported(st_ops, moff))
417	428	continue;
418	429
	430	+ if (func_proto->type) {
	431	+ ret_type = btf_type_resolve_ptr(btf, func_proto->type, NULL);
	432	+ if (ret_type && !__btf_type_is_struct(ret_type)) {
	433	+ pr_warn("func ptr %s in struct %s returns non-struct pointer, which is not supported\n",
	434	+ mname, st_ops->name);
	435	+ err = -EOPNOTSUPP;
	436	+ goto errout;
	437	+ }
	438	+ }
	439	+
419	440	if (btf_distill_func_proto(log, btf,
420	441	func_proto, mname,
421	442	&st_ops->func_models[i])) {
@@ -425,8 +446,9 @@ int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, · 3 unmodified lines
425	446	goto errout;
426	447	}
427	448
	449	+ stub_func_addr = (void *)(st_ops->cfi_stubs + moff);
428	450	err = prepare_arg_info(btf, st_ops->name, mname,
429		− func_proto,
	451	+ func_proto, stub_func_addr,
430	452	arg_info + i);
431	453	if (err)
432	454	goto errout;
@@ -1152,13 +1174,6 @@ void bpf_struct_ops_put(const void *kdata) · 719 unmodified lines
1152	1174	bpf_map_put(&st_map->map);
1153	1175	}
1154	1176
1155		−int bpf_struct_ops_supported(const struct bpf_struct_ops *st_ops, u32 moff)
1156		−{
1157		− void func_ptr = (void **)(st_ops->cfi_stubs + moff);
1158		−
1159		− return func_ptr ? 0 : -ENOTSUPP;
1160		−}
1161		−
1162	1177	static bool bpf_struct_ops_valid_to_reg(struct bpf_map *map)
1163	1178	{
1164	1179	struct bpf_struct_ops_map st_map = (struct bpf_struct_ops_map )map;

kernel/bpf/btf.c+113−14

@@ -606,6 +606,7 @@ s32 bpf_find_btf_id(const char name, u32 kind, struct btf *btf_p) · 605 unmodified lines
606	606	spin_unlock_bh(&btf_idr_lock);
607	607	return ret;
608	608	}
	609	+EXPORT_SYMBOL_GPL(bpf_find_btf_id);
609	610
610	611	const struct btf_type btf_type_skip_modifiers(const struct btf btf,
611	612	u32 id, u32 *res_id)
@@ -2575,7 +2576,7 @@ static int btf_ref_type_check_meta(struct btf_verifier_env *env, · 1963 unmodified lines
2575	2576	return -EINVAL;
2576	2577	}
2577	2578
2578		− if (btf_type_kflag(t)) {
	2579	+ if (btf_type_kflag(t) && !btf_type_is_type_tag(t)) {
2579	2580	btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
2580	2581	return -EINVAL;
2581	2582	}
@@ -3332,6 +3333,8 @@ static int btf_find_kptr(const struct btf btf, const struct btf_type t, · 750 unmodified lines
3332	3333	u32 off, int sz, struct btf_field_info *info, u32 field_mask)
3333	3334	{
3334	3335	enum btf_field_type type;
	3336	+ const char *tag_value;
	3337	+ bool is_type_tag;
3335	3338	u32 res_id;
3336	3339
3337	3340	/* Permit modifiers on the pointer itself */
@@ -3341,19 +3344,20 @@ static int btf_find_kptr(const struct btf btf, const struct btf_type t, · 3 unmodified lines
3341	3344	if (!btf_type_is_ptr(t))
3342	3345	return BTF_FIELD_IGNORE;
3343	3346	t = btf_type_by_id(btf, t->type);
3344		−
3345		− if (!btf_type_is_type_tag(t))
	3347	+ is_type_tag = btf_type_is_type_tag(t) && !btf_type_kflag(t);
	3348	+ if (!is_type_tag)
3346	3349	return BTF_FIELD_IGNORE;
3347	3350	/* Reject extra tags */
3348	3351	if (btf_type_is_type_tag(btf_type_by_id(btf, t->type)))
3349	3352	return -EINVAL;
3350		− if (!strcmp("kptr_untrusted", __btf_name_by_offset(btf, t->name_off)))
	3353	+ tag_value = __btf_name_by_offset(btf, t->name_off);
	3354	+ if (!strcmp("kptr_untrusted", tag_value))
3351	3355	type = BPF_KPTR_UNREF;
3352		− else if (!strcmp("kptr", __btf_name_by_offset(btf, t->name_off)))
	3356	+ else if (!strcmp("kptr", tag_value))
3353	3357	type = BPF_KPTR_REF;
3354		− else if (!strcmp("percpu_kptr", __btf_name_by_offset(btf, t->name_off)))
	3358	+ else if (!strcmp("percpu_kptr", tag_value))
3355	3359	type = BPF_KPTR_PERCPU;
3356		− else if (!strcmp("uptr", __btf_name_by_offset(btf, t->name_off)))
	3360	+ else if (!strcmp("uptr", tag_value))
3357	3361	type = BPF_UPTR;
3358	3362	else
3359	3363	return -EINVAL;
@@ -4944,11 +4948,6 @@ static s32 btf_decl_tag_check_meta(struct btf_verifier_env *env, · 1584 unmodified lines
4944	4948	return -EINVAL;
4945	4949	}
4946	4950
4947		− if (btf_type_kflag(t)) {
4948		− btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
4949		− return -EINVAL;
4950		− }
4951		−
4952	4951	component_idx = btf_type_decl_tag(t)->component_idx;
4953	4952	if (component_idx < -1) {
4954	4953	btf_verifier_log_type(env, t, "Invalid component_idx");
@@ -6507,6 +6506,8 @@ static const struct bpf_raw_tp_null_args raw_tp_null_args[] = { · 1552 unmodified lines
6507	6506	/* rxrpc */
6508	6507	{ "rxrpc_recvdata", 0x1 },
6509	6508	{ "rxrpc_resend", 0x10 },
	6509	+ { "rxrpc_tq", 0x10 },
	6510	+ { "rxrpc_client", 0x1 },
6510	6511	/* skb */
6511	6512	{"kfree_skb", 0x1000},
6512	6513	/* sunrpc */
@@ -6529,6 +6530,103 @@ static const struct bpf_raw_tp_null_args raw_tp_null_args[] = { · 16 unmodified lines
6529	6530	{ "mr_integ_alloc", 0x2000 },
6530	6531	/* bpf_testmod */
6531	6532	{ "bpf_testmod_test_read", 0x0 },
	6533	+ /* amdgpu */
	6534	+ { "amdgpu_vm_bo_map", 0x1 },
	6535	+ { "amdgpu_vm_bo_unmap", 0x1 },
	6536	+ /* netfs */
	6537	+ { "netfs_folioq", 0x1 },
	6538	+ /* xfs from xfs_defer_pending_class */
	6539	+ { "xfs_defer_create_intent", 0x1 },
	6540	+ { "xfs_defer_cancel_list", 0x1 },
	6541	+ { "xfs_defer_pending_finish", 0x1 },
	6542	+ { "xfs_defer_pending_abort", 0x1 },
	6543	+ { "xfs_defer_relog_intent", 0x1 },
	6544	+ { "xfs_defer_isolate_paused", 0x1 },
	6545	+ { "xfs_defer_item_pause", 0x1 },
	6546	+ { "xfs_defer_item_unpause", 0x1 },
	6547	+ /* xfs from xfs_defer_pending_item_class */
	6548	+ { "xfs_defer_add_item", 0x1 },
	6549	+ { "xfs_defer_cancel_item", 0x1 },
	6550	+ { "xfs_defer_finish_item", 0x1 },
	6551	+ /* xfs from xfs_icwalk_class */
	6552	+ { "xfs_ioc_free_eofblocks", 0x10 },
	6553	+ { "xfs_blockgc_free_space", 0x10 },
	6554	+ /* xfs from xfs_btree_cur_class */
	6555	+ { "xfs_btree_updkeys", 0x100 },
	6556	+ { "xfs_btree_overlapped_query_range", 0x100 },
	6557	+ /* xfs from xfs_imap_class*/
	6558	+ { "xfs_map_blocks_found", 0x10000 },
	6559	+ { "xfs_map_blocks_alloc", 0x10000 },
	6560	+ { "xfs_iomap_alloc", 0x1000 },
	6561	+ { "xfs_iomap_found", 0x1000 },
	6562	+ /* xfs from xfs_fs_class */
	6563	+ { "xfs_inodegc_flush", 0x1 },
	6564	+ { "xfs_inodegc_push", 0x1 },
	6565	+ { "xfs_inodegc_start", 0x1 },
	6566	+ { "xfs_inodegc_stop", 0x1 },
	6567	+ { "xfs_inodegc_queue", 0x1 },
	6568	+ { "xfs_inodegc_throttle", 0x1 },
	6569	+ { "xfs_fs_sync_fs", 0x1 },
	6570	+ { "xfs_blockgc_start", 0x1 },
	6571	+ { "xfs_blockgc_stop", 0x1 },
	6572	+ { "xfs_blockgc_worker", 0x1 },
	6573	+ { "xfs_blockgc_flush_all", 0x1 },
	6574	+ /* xfs_scrub */
	6575	+ { "xchk_nlinks_live_update", 0x10 },
	6576	+ /* xfs_scrub from xchk_metapath_class */
	6577	+ { "xchk_metapath_lookup", 0x100 },
	6578	+ /* nfsd */
	6579	+ { "nfsd_dirent", 0x1 },
	6580	+ { "nfsd_file_acquire", 0x1001 },
	6581	+ { "nfsd_file_insert_err", 0x1 },
	6582	+ { "nfsd_file_cons_err", 0x1 },
	6583	+ /* nfs4 */
	6584	+ { "nfs4_setup_sequence", 0x1 },
	6585	+ { "pnfs_update_layout", 0x10000 },
	6586	+ { "nfs4_inode_callback_event", 0x200 },
	6587	+ { "nfs4_inode_stateid_callback_event", 0x200 },
	6588	+ /* nfs from pnfs_layout_event */
	6589	+ { "pnfs_mds_fallback_pg_init_read", 0x10000 },
	6590	+ { "pnfs_mds_fallback_pg_init_write", 0x10000 },
	6591	+ { "pnfs_mds_fallback_pg_get_mirror_count", 0x10000 },
	6592	+ { "pnfs_mds_fallback_read_done", 0x10000 },
	6593	+ { "pnfs_mds_fallback_write_done", 0x10000 },
	6594	+ { "pnfs_mds_fallback_read_pagelist", 0x10000 },
	6595	+ { "pnfs_mds_fallback_write_pagelist", 0x10000 },
	6596	+ /* coda */
	6597	+ { "coda_dec_pic_run", 0x10 },
	6598	+ { "coda_dec_pic_done", 0x10 },
	6599	+ /* cfg80211 */
	6600	+ { "cfg80211_scan_done", 0x11 },
	6601	+ { "rdev_set_coalesce", 0x10 },
	6602	+ { "cfg80211_report_wowlan_wakeup", 0x100 },
	6603	+ { "cfg80211_inform_bss_frame", 0x100 },
	6604	+ { "cfg80211_michael_mic_failure", 0x10000 },
	6605	+ /* cfg80211 from wiphy_work_event */
	6606	+ { "wiphy_work_queue", 0x10 },
	6607	+ { "wiphy_work_run", 0x10 },
	6608	+ { "wiphy_work_cancel", 0x10 },
	6609	+ { "wiphy_work_flush", 0x10 },
	6610	+ /* hugetlbfs */
	6611	+ { "hugetlbfs_alloc_inode", 0x10 },
	6612	+ /* spufs */
	6613	+ { "spufs_context", 0x10 },
	6614	+ /* kvm_hv */
	6615	+ { "kvm_page_fault_enter", 0x100 },
	6616	+ /* dpu */
	6617	+ { "dpu_crtc_setup_mixer", 0x100 },
	6618	+ /* binder */
	6619	+ { "binder_transaction", 0x100 },
	6620	+ /* bcachefs */
	6621	+ { "btree_path_free", 0x100 },
	6622	+ /* hfi1_tx */
	6623	+ { "hfi1_sdma_progress", 0x1000 },
	6624	+ /* iptfs */
	6625	+ { "iptfs_ingress_postq_event", 0x1000 },
	6626	+ /* neigh */
	6627	+ { "neigh_update", 0x10 },
	6628	+ /* snd_firewire_lib */
	6629	+ { "amdtp_packet", 0x100 },
6532	6630	};
6533	6631
6534	6632	bool btf_ctx_access(int off, int size, enum bpf_access_type type,
@@ -6679,6 +6777,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, · 144 unmodified lines
6679	6777	info->reg_type = ctx_arg_info->reg_type;
6680	6778	info->btf = ctx_arg_info->btf ? : btf_vmlinux;
6681	6779	info->btf_id = ctx_arg_info->btf_id;
	6780	+ info->ref_obj_id = ctx_arg_info->ref_obj_id;
6682	6781	return true;
6683	6782	}
6684	6783	}
@@ -6745,7 +6844,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, · 60 unmodified lines
6745	6844	info->btf_id = t->type;
6746	6845	t = btf_type_by_id(btf, t->type);
6747	6846
6748		− if (btf_type_is_type_tag(t)) {
	6847	+ if (btf_type_is_type_tag(t) && !btf_type_kflag(t)) {
6749	6848	tag_value = __btf_name_by_offset(btf, t->name_off);
6750	6849	if (strcmp(tag_value, "user") == 0)
6751	6850	info->reg_type \|= MEM_USER;
@@ -7004,7 +7103,7 @@ error: · 252 unmodified lines
7004	7103
7005	7104	/* check type tag */
7006	7105	t = btf_type_by_id(btf, mtype->type);
7007		− if (btf_type_is_type_tag(t)) {
	7106	+ if (btf_type_is_type_tag(t) && !btf_type_kflag(t)) {
7008	7107	tag_value = __btf_name_by_offset(btf, t->name_off);
7009	7108	/* check __user tag */
7010	7109	if (strcmp(tag_value, "user") == 0)

kernel/bpf/cgroup.c+25−8

@@ -369,7 +369,7 @@ static struct bpf_prog prog_list_prog(struct bpf_prog_list pl) · 368 unmodified lines
369	369	/* count number of elements in the list.
370	370	* it's slow but the list cannot be long
371	371	*/
372		−static u32 prog_list_length(struct hlist_head *head)
	372	+static u32 prog_list_length(struct hlist_head head, int preorder_cnt)
373	373	{
374	374	struct bpf_prog_list *pl;
375	375	u32 cnt = 0;
@@ -377,6 +377,8 @@ static u32 prog_list_length(struct hlist_head *head) · 1 unmodified line
377	377	hlist_for_each_entry(pl, head, node) {
378	378	if (!prog_list_prog(pl))
379	379	continue;
	380	+ if (preorder_cnt && (pl->flags & BPF_F_PREORDER))
	381	+ (*preorder_cnt)++;
380	382	cnt++;
381	383	}
382	384	return cnt;
@@ -400,7 +402,7 @@ static bool hierarchy_allows_attach(struct cgroup *cgrp, · 17 unmodified lines
400	402
401	403	if (flags & BPF_F_ALLOW_MULTI)
402	404	return true;
403		− cnt = prog_list_length(&p->bpf.progs[atype]);
	405	+ cnt = prog_list_length(&p->bpf.progs[atype], NULL);
404	406	WARN_ON_ONCE(cnt > 1);
405	407	if (cnt == 1)
406	408	return !!(flags & BPF_F_ALLOW_OVERRIDE);
@@ -423,12 +425,12 @@ static int compute_effective_progs(struct cgroup *cgrp, · 16 unmodified lines
423	425	struct bpf_prog_array *progs;
424	426	struct bpf_prog_list *pl;
425	427	struct cgroup *p = cgrp;
426		− int cnt = 0;
	428	+ int i, j, cnt = 0, preorder_cnt = 0, fstart, bstart, init_bstart;
427	429
428	430	/* count number of effective programs by walking parents */
429	431	do {
430	432	if (cnt == 0 \|\| (p->bpf.flags[atype] & BPF_F_ALLOW_MULTI))
431		− cnt += prog_list_length(&p->bpf.progs[atype]);
	433	+ cnt += prog_list_length(&p->bpf.progs[atype], &preorder_cnt);
432	434	p = cgroup_parent(p);
433	435	} while (p);
434	436
@@ -439,20 +441,34 @@ static int compute_effective_progs(struct cgroup *cgrp, · 4 unmodified lines
439	441	/* populate the array with effective progs */
440	442	cnt = 0;
441	443	p = cgrp;
	444	+ fstart = preorder_cnt;
	445	+ bstart = preorder_cnt - 1;
442	446	do {
443	447	if (cnt > 0 && !(p->bpf.flags[atype] & BPF_F_ALLOW_MULTI))
444	448	continue;
445	449
	450	+ init_bstart = bstart;
446	451	hlist_for_each_entry(pl, &p->bpf.progs[atype], node) {
447	452	if (!prog_list_prog(pl))
448	453	continue;
449	454
450		− item = &progs->items[cnt];
	455	+ if (pl->flags & BPF_F_PREORDER) {
	456	+ item = &progs->items[bstart];
	457	+ bstart--;
	458	+ } else {
	459	+ item = &progs->items[fstart];
	460	+ fstart++;
	461	+ }
451	462	item->prog = prog_list_prog(pl);
452	463	bpf_cgroup_storages_assign(item->cgroup_storage,
453	464	pl->storage);
454	465	cnt++;
455	466	}
	467	+
	468	+ /* reverse pre-ordering progs at this cgroup level */
	469	+ for (i = bstart + 1, j = init_bstart; i < j; i++, j--)
	470	+ swap(progs->items[i], progs->items[j]);
	471	+
456	472	} while ((p = cgroup_parent(p)));
457	473
458	474	*array = progs;
@@ -663,7 +679,7 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, · 204 unmodified lines
663	679	*/
664	680	return -EPERM;
665	681
666		− if (prog_list_length(progs) >= BPF_CGROUP_MAX_PROGS)
	682	+ if (prog_list_length(progs, NULL) >= BPF_CGROUP_MAX_PROGS)
667	683	return -E2BIG;
668	684
669	685	pl = find_attach_entry(progs, prog, link, replace_prog,
@@ -698,6 +714,7 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, · 28 unmodified lines
698	714
699	715	pl->prog = prog;
700	716	pl->link = link;
	717	+ pl->flags = flags;
701	718	bpf_cgroup_storages_assign(pl->storage, storage);
702	719	cgrp->bpf.flags[atype] = saved_flags;
703	720
@@ -1073,7 +1090,7 @@ static int __cgroup_bpf_query(struct cgroup cgrp, const union bpf_attr attr, · 369 unmodified lines
1073	1090	lockdep_is_held(&cgroup_mutex));
1074	1091	total_cnt += bpf_prog_array_length(effective);
1075	1092	} else {
1076		− total_cnt += prog_list_length(&cgrp->bpf.progs[atype]);
	1093	+ total_cnt += prog_list_length(&cgrp->bpf.progs[atype], NULL);
1077	1094	}
1078	1095	}
1079	1096
@@ -1105,7 +1122,7 @@ static int __cgroup_bpf_query(struct cgroup cgrp, const union bpf_attr attr, · 25 unmodified lines
1105	1122	u32 id;
1106	1123
1107	1124	progs = &cgrp->bpf.progs[atype];
1108		− cnt = min_t(int, prog_list_length(progs), total_cnt);
	1125	+ cnt = min_t(int, prog_list_length(progs, NULL), total_cnt);
1109	1126	i = 0;
1110	1127	hlist_for_each_entry(pl, progs, node) {
1111	1128	prog = prog_list_prog(pl);

kernel/bpf/core.c+107−10

@@ -1663,14 +1663,17 @@ EXPORT_SYMBOL_GPL(__bpf_call_base); · 1662 unmodified lines
1663	1663	INSN_3(JMP, JSET, K), \
1664	1664	INSN_2(JMP, JA), \
1665	1665	INSN_2(JMP32, JA), \
	1666	+ /* Atomic operations. */ \
	1667	+ INSN_3(STX, ATOMIC, B), \
	1668	+ INSN_3(STX, ATOMIC, H), \
	1669	+ INSN_3(STX, ATOMIC, W), \
	1670	+ INSN_3(STX, ATOMIC, DW), \
1666	1671	/* Store instructions. */ \
1667	1672	/* Register based. */ \
1668	1673	INSN_3(STX, MEM, B), \
1669	1674	INSN_3(STX, MEM, H), \
1670	1675	INSN_3(STX, MEM, W), \
1671	1676	INSN_3(STX, MEM, DW), \
1672		− INSN_3(STX, ATOMIC, W), \
1673		− INSN_3(STX, ATOMIC, DW), \
1674	1677	/* Immediate based. */ \
1675	1678	INSN_3(ST, MEM, B), \
1676	1679	INSN_3(ST, MEM, H), \
@@ -2152,24 +2155,33 @@ out: · 475 unmodified lines
2152	2155	if (BPF_SIZE(insn->code) == BPF_W) \
2153	2156	atomic_##KOP((u32) SRC, (atomic_t *)(unsigned long) \
2154	2157	(DST + insn->off)); \
2155		− else \
	2158	+ else if (BPF_SIZE(insn->code) == BPF_DW) \
2156	2159	atomic64_##KOP((u64) SRC, (atomic64_t *)(unsigned long) \
2157	2160	(DST + insn->off)); \
	2161	+ else \
	2162	+ goto default_label; \
2158	2163	break; \
2159	2164	case BOP \| BPF_FETCH: \
2160	2165	if (BPF_SIZE(insn->code) == BPF_W) \
2161	2166	SRC = (u32) atomic_fetch_##KOP( \
2162	2167	(u32) SRC, \
2163	2168	(atomic_t *)(unsigned long) (DST + insn->off)); \
2164		− else \
	2169	+ else if (BPF_SIZE(insn->code) == BPF_DW) \
2165	2170	SRC = (u64) atomic64_fetch_##KOP( \
2166	2171	(u64) SRC, \
2167	2172	(atomic64_t *)(unsigned long) (DST + insn->off)); \
	2173	+ else \
	2174	+ goto default_label; \
2168	2175	break;
2169	2176
2170	2177	STX_ATOMIC_DW:
2171	2178	STX_ATOMIC_W:
	2179	+ STX_ATOMIC_H:
	2180	+ STX_ATOMIC_B:
2172	2181	switch (IMM) {
	2182	+ /* Atomic read-modify-write instructions support only W and DW
	2183	+ * size modifiers.
	2184	+ */
2173	2185	ATOMIC_ALU_OP(BPF_ADD, add)
2174	2186	ATOMIC_ALU_OP(BPF_AND, and)
2175	2187	ATOMIC_ALU_OP(BPF_OR, or)
@@ -2181,20 +2193,63 @@ out: · 5 unmodified lines
2181	2193	SRC = (u32) atomic_xchg(
2182	2194	(atomic_t *)(unsigned long) (DST + insn->off),
2183	2195	(u32) SRC);
2184		− else
	2196	+ else if (BPF_SIZE(insn->code) == BPF_DW)
2185	2197	SRC = (u64) atomic64_xchg(
2186	2198	(atomic64_t *)(unsigned long) (DST + insn->off),
2187	2199	(u64) SRC);
	2200	+ else
	2201	+ goto default_label;
2188	2202	break;
2189	2203	case BPF_CMPXCHG:
2190	2204	if (BPF_SIZE(insn->code) == BPF_W)
2191	2205	BPF_R0 = (u32) atomic_cmpxchg(
2192	2206	(atomic_t *)(unsigned long) (DST + insn->off),
2193	2207	(u32) BPF_R0, (u32) SRC);
2194		− else
	2208	+ else if (BPF_SIZE(insn->code) == BPF_DW)
2195	2209	BPF_R0 = (u64) atomic64_cmpxchg(
2196	2210	(atomic64_t *)(unsigned long) (DST + insn->off),
2197	2211	(u64) BPF_R0, (u64) SRC);
	2212	+ else
	2213	+ goto default_label;
	2214	+ break;
	2215	+ /* Atomic load and store instructions support all size
	2216	+ * modifiers.
	2217	+ */
	2218	+ case BPF_LOAD_ACQ:
	2219	+ switch (BPF_SIZE(insn->code)) {
	2220	+#define LOAD_ACQUIRE(SIZEOP, SIZE) \
	2221	+ case BPF_##SIZEOP: \
	2222	+ DST = (SIZE)smp_load_acquire( \
	2223	+ (SIZE *)(unsigned long)(SRC + insn->off)); \
	2224	+ break;
	2225	+ LOAD_ACQUIRE(B, u8)
	2226	+ LOAD_ACQUIRE(H, u16)
	2227	+ LOAD_ACQUIRE(W, u32)
	2228	+#ifdef CONFIG_64BIT
	2229	+ LOAD_ACQUIRE(DW, u64)
	2230	+#endif
	2231	+#undef LOAD_ACQUIRE
	2232	+ default:
	2233	+ goto default_label;
	2234	+ }
	2235	+ break;
	2236	+ case BPF_STORE_REL:
	2237	+ switch (BPF_SIZE(insn->code)) {
	2238	+#define STORE_RELEASE(SIZEOP, SIZE) \
	2239	+ case BPF_##SIZEOP: \
	2240	+ smp_store_release( \
	2241	+ (SIZE *)(unsigned long)(DST + insn->off), (SIZE)SRC); \
	2242	+ break;
	2243	+ STORE_RELEASE(B, u8)
	2244	+ STORE_RELEASE(H, u16)
	2245	+ STORE_RELEASE(W, u32)
	2246	+#ifdef CONFIG_64BIT
	2247	+ STORE_RELEASE(DW, u64)
	2248	+#endif
	2249	+#undef STORE_RELEASE
	2250	+ default:
	2251	+ goto default_label;
	2252	+ }
2198	2253	break;
2199	2254
2200	2255	default:
@@ -2290,17 +2345,18 @@ void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth) · 89 unmodified lines
2290	2345	insn->code = BPF_JMP \| BPF_CALL_ARGS;
2291	2346	}
2292	2347	#endif
2293		−#else
	2348	+#endif
	2349	+
2294	2350	static unsigned int __bpf_prog_ret0_warn(const void *ctx,
2295	2351	const struct bpf_insn *insn)
2296	2352	{
2297	2353	/* If this handler ever gets executed, then BPF_JIT_ALWAYS_ON
2298		− * is not working properly, so warn about it!
	2354	+ * is not working properly, or interpreter is being used when
	2355	+ * prog->jit_requested is not 0, so warn about it!
2299	2356	*/
2300	2357	WARN_ON_ONCE(1);
2301	2358	return 0;
2302	2359	}
2303		−#endif
2304	2360
2305	2361	bool bpf_prog_map_compatible(struct bpf_map *map,
2306	2362	const struct bpf_prog *fp)
@@ -2380,8 +2436,18 @@ static void bpf_prog_select_func(struct bpf_prog *fp) · 73 unmodified lines
2380	2436	{
2381	2437	#ifndef CONFIG_BPF_JIT_ALWAYS_ON
2382	2438	u32 stack_depth = max_t(u32, fp->aux->stack_depth, 1);
	2439	+ u32 idx = (round_up(stack_depth, 32) / 32) - 1;
2383	2440
2384		− fp->bpf_func = interpreters[(round_up(stack_depth, 32) / 32) - 1];
	2441	+ /* may_goto may cause stack size > 512, leading to idx out-of-bounds.
	2442	+ * But for non-JITed programs, we don't need bpf_func, so no bounds
	2443	+ * check needed.
	2444	+ */
	2445	+ if (!fp->jit_requested &&
	2446	+ !WARN_ON_ONCE(idx >= ARRAY_SIZE(interpreters))) {
	2447	+ fp->bpf_func = interpreters[idx];
	2448	+ } else {
	2449	+ fp->bpf_func = __bpf_prog_ret0_warn;
	2450	+ }
2385	2451	#else
2386	2452	fp->bpf_func = __bpf_prog_ret0_warn;
2387	2453	#endif
@@ -2906,6 +2972,11 @@ const struct bpf_func_proto * __weak bpf_get_trace_vprintk_proto(void) · 518 unmodified lines
2906	2972	return NULL;
2907	2973	}
2908	2974
	2975	+const struct bpf_func_proto * __weak bpf_get_perf_event_read_value_proto(void)
	2976	+{
	2977	+ return NULL;
	2978	+}
	2979	+
2909	2980	u64 __weak
2910	2981	bpf_event_output(struct bpf_map map, u64 flags, void meta, u64 meta_size,
2911	2982	void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy)
@@ -3058,6 +3129,32 @@ void __weak arch_bpf_stack_walk(bool (consume_fn)(void cookie, u64 ip, u64 sp, · 146 unmodified lines
3058	3129	{
3059	3130	}
3060	3131
	3132	+bool __weak bpf_jit_supports_timed_may_goto(void)
	3133	+{
	3134	+ return false;
	3135	+}
	3136	+
	3137	+u64 __weak arch_bpf_timed_may_goto(void)
	3138	+{
	3139	+ return 0;
	3140	+}
	3141	+
	3142	+u64 bpf_check_timed_may_goto(struct bpf_timed_may_goto *p)
	3143	+{
	3144	+ u64 time = ktime_get_mono_fast_ns();
	3145	+
	3146	+ /* Populate the timestamp for this stack frame, and refresh count. */
	3147	+ if (!p->timestamp) {
	3148	+ p->timestamp = time;
	3149	+ return BPF_MAX_TIMED_LOOPS;
	3150	+ }
	3151	+ /* Check if we've exhausted our time slice, and zero count. */
	3152	+ if (time - p->timestamp >= (NSEC_PER_SEC / 4))
	3153	+ return 0;
	3154	+ /* Refresh the count for the stack frame. */
	3155	+ return BPF_MAX_TIMED_LOOPS;
	3156	+}
	3157	+
3061	3158	/* for configs without MMU or 32-bit */
3062	3159	__weak const struct bpf_map_ops arena_map_ops;
3063	3160	__weak u64 bpf_arena_get_user_vm_start(struct bpf_arena *arena)

kernel/bpf/cpumask.c+53−0

@@ -45,6 +45,10 @@ __bpf_kfunc_start_defs(); · 44 unmodified lines
45	45	*
46	46	* bpf_cpumask_create() allocates memory using the BPF memory allocator, and
47	47	* will not block. It may return NULL if no memory is available.
	48	+ *
	49	+ * Return:
	50	+ * * A pointer to a new struct bpf_cpumask instance on success.
	51	+ * * NULL if the BPF memory allocator is out of memory.
48	52	*/
49	53	__bpf_kfunc struct bpf_cpumask *bpf_cpumask_create(void)
50	54	{
@@ -71,6 +75,10 @@ __bpf_kfunc struct bpf_cpumask *bpf_cpumask_create(void) · 20 unmodified lines
71	75	* Acquires a reference to a BPF cpumask. The cpumask returned by this function
72	76	* must either be embedded in a map as a kptr, or freed with
73	77	* bpf_cpumask_release().
	78	+ *
	79	+ * Return:
	80	+ * * The struct bpf_cpumask pointer passed to the function.
	81	+ *
74	82	*/
75	83	__bpf_kfunc struct bpf_cpumask bpf_cpumask_acquire(struct bpf_cpumask cpumask)
76	84	{
@@ -106,6 +114,9 @@ CFI_NOSEAL(bpf_cpumask_release_dtor); · 29 unmodified lines
106	114	*
107	115	* Find the index of the first nonzero bit of the cpumask. A struct bpf_cpumask
108	116	* pointer may be safely passed to this function.
	117	+ *
	118	+ * Return:
	119	+ * * The index of the first nonzero bit in the struct cpumask.
109	120	*/
110	121	__bpf_kfunc u32 bpf_cpumask_first(const struct cpumask *cpumask)
111	122	{
@@ -119,6 +130,9 @@ __bpf_kfunc u32 bpf_cpumask_first(const struct cpumask *cpumask) · 7 unmodified lines
119	130	*
120	131	* Find the index of the first unset bit of the cpumask. A struct bpf_cpumask
121	132	* pointer may be safely passed to this function.
	133	+ *
	134	+ * Return:
	135	+ * * The index of the first zero bit in the struct cpumask.
122	136	*/
123	137	__bpf_kfunc u32 bpf_cpumask_first_zero(const struct cpumask *cpumask)
124	138	{
@@ -133,6 +147,9 @@ __bpf_kfunc u32 bpf_cpumask_first_zero(const struct cpumask *cpumask) · 8 unmodified lines
133	147	*
134	148	* Find the index of the first nonzero bit of the AND of two cpumasks.
135	149	* struct bpf_cpumask pointers may be safely passed to @src1 and @src2.
	150	+ *
	151	+ * Return:
	152	+ * * The index of the first bit that is nonzero in both cpumask instances.
136	153	*/
137	154	__bpf_kfunc u32 bpf_cpumask_first_and(const struct cpumask *src1,
138	155	const struct cpumask *src2)
@@ -414,12 +431,47 @@ __bpf_kfunc u32 bpf_cpumask_any_and_distribute(const struct cpumask *src1, · 275 unmodified lines
414	431	* @cpumask: The cpumask being queried.
415	432	*
416	433	* Count the number of set bits in the given cpumask.
	434	+ *
	435	+ * Return:
	436	+ * * The number of bits set in the mask.
417	437	*/
418	438	__bpf_kfunc u32 bpf_cpumask_weight(const struct cpumask *cpumask)
419	439	{
420	440	return cpumask_weight(cpumask);
421	441	}
422	442
	443	+/**
	444	+ * bpf_cpumask_populate() - Populate the CPU mask from the contents of
	445	+ * a BPF memory region.
	446	+ *
	447	+ * @cpumask: The cpumask being populated.
	448	+ * @src: The BPF memory holding the bit pattern.
	449	+ * @src__sz: Length of the BPF memory region in bytes.
	450	+ *
	451	+ * Return:
	452	+ * * 0 if the struct cpumask * instance was populated successfully.
	453	+ * * -EACCES if the memory region is too small to populate the cpumask.
	454	+ * * -EINVAL if the memory region is not aligned to the size of a long
	455	+ * and the architecture does not support efficient unaligned accesses.
	456	+ */
	457	+__bpf_kfunc int bpf_cpumask_populate(struct cpumask cpumask, void src, size_t src__sz)
	458	+{
	459	+ unsigned long source = (unsigned long)src;
	460	+
	461	+ /* The memory region must be large enough to populate the entire CPU mask. */
	462	+ if (src__sz < bitmap_size(nr_cpu_ids))
	463	+ return -EACCES;
	464	+
	465	+ /* If avoiding unaligned accesses, the input region must be aligned to the nearest long. */
	466	+ if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) &&
	467	+ !IS_ALIGNED(source, sizeof(long)))
	468	+ return -EINVAL;
	469	+
	470	+ bitmap_copy(cpumask_bits(cpumask), src, nr_cpu_ids);
	471	+
	472	+ return 0;
	473	+}
	474	+
423	475	__bpf_kfunc_end_defs();
424	476
425	477	BTF_KFUNCS_START(cpumask_kfunc_btf_ids)
@@ -448,6 +500,7 @@ BTF_ID_FLAGS(func, bpf_cpumask_copy, KF_RCU) · 22 unmodified lines
448	500	BTF_ID_FLAGS(func, bpf_cpumask_any_distribute, KF_RCU)
449	501	BTF_ID_FLAGS(func, bpf_cpumask_any_and_distribute, KF_RCU)
450	502	BTF_ID_FLAGS(func, bpf_cpumask_weight, KF_RCU)
	503	+BTF_ID_FLAGS(func, bpf_cpumask_populate, KF_RCU)
451	504	BTF_KFUNCS_END(cpumask_kfunc_btf_ids)
452	505
453	506	static const struct btf_kfunc_id_set cpumask_kfunc_set = {

kernel/bpf/disasm.c+14−2

@@ -202,7 +202,7 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, · 201 unmodified lines
202	202	insn->dst_reg, class == BPF_ALU ? 'w' : 'r',
203	203	insn->dst_reg);
204	204	} else if (is_addr_space_cast(insn)) {
205		− verbose(cbs->private_data, "(%02x) r%d = addr_space_cast(r%d, %d, %d)\n",
	205	+ verbose(cbs->private_data, "(%02x) r%d = addr_space_cast(r%d, %u, %u)\n",
206	206	insn->code, insn->dst_reg,
207	207	insn->src_reg, ((u32)insn->imm) >> 16, (u16)insn->imm);
208	208	} else if (is_mov_percpu_addr(insn)) {
@@ -267,6 +267,18 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, · 58 unmodified lines
267	267	BPF_SIZE(insn->code) == BPF_DW ? "64" : "",
268	268	bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
269	269	insn->dst_reg, insn->off, insn->src_reg);
	270	+ } else if (BPF_MODE(insn->code) == BPF_ATOMIC &&
	271	+ insn->imm == BPF_LOAD_ACQ) {
	272	+ verbose(cbs->private_data, "(%02x) r%d = load_acquire((%s *)(r%d %+d))\n",
	273	+ insn->code, insn->dst_reg,
	274	+ bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
	275	+ insn->src_reg, insn->off);
	276	+ } else if (BPF_MODE(insn->code) == BPF_ATOMIC &&
	277	+ insn->imm == BPF_STORE_REL) {
	278	+ verbose(cbs->private_data, "(%02x) store_release((%s *)(r%d %+d), r%d)\n",
	279	+ insn->code,
	280	+ bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
	281	+ insn->dst_reg, insn->off, insn->src_reg);
270	282	} else {
271	283	verbose(cbs->private_data, "BUG_%02x\n", insn->code);
272	284	}
@@ -369,7 +381,7 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, · 96 unmodified lines
369	381	insn->code, class == BPF_JMP32 ? 'w' : 'r',
370	382	insn->dst_reg,
371	383	bpf_jmp_string[BPF_OP(insn->code) >> 4],
372		− insn->imm, insn->off);
	384	+ (u32)insn->imm, insn->off);
373	385	}
374	386	} else {
375	387	verbose(cbs->private_data, "(%02x) %s\n",

kernel/bpf/hashtab.c+6−3

@@ -198,12 +198,12 @@ static bool htab_is_percpu(const struct bpf_htab *htab) · 197 unmodified lines
198	198	static inline void htab_elem_set_ptr(struct htab_elem *l, u32 key_size,
199	199	void __percpu *pptr)
200	200	{
201		− (void __percpu *)(l->key + key_size) = pptr;
	201	+ (void __percpu *)(l->key + roundup(key_size, 8)) = pptr;
202	202	}
203	203
204	204	static inline void __percpu htab_elem_get_ptr(struct htab_elem l, u32 key_size)
205	205	{
206		− return (void __percpu *)(l->key + key_size);
	206	+ return (void __percpu *)(l->key + roundup(key_size, 8));
207	207	}
208	208
209	209	static void fd_htab_map_get_ptr(const struct bpf_map map, struct htab_elem *l)
@@ -787,6 +787,9 @@ static int htab_lru_map_gen_lookup(struct bpf_map *map, · 577 unmodified lines
787	787	static void check_and_free_fields(struct bpf_htab *htab,
788	788	struct htab_elem *elem)
789	789	{
	790	+ if (IS_ERR_OR_NULL(htab->map.record))
	791	+ return;
	792	+
790	793	if (htab_is_percpu(htab)) {
791	794	void __percpu *pptr = htab_elem_get_ptr(elem, htab->map.key_size);
792	795	int cpu;
@@ -2354,7 +2357,7 @@ static int htab_percpu_map_gen_lookup(struct bpf_map map, struct bpf_insn insn · 1561 unmodified lines
2354	2357	*insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem);
2355	2358	*insn++ = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3);
2356	2359	*insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_0,
2357		− offsetof(struct htab_elem, key) + map->key_size);
	2360	+ offsetof(struct htab_elem, key) + roundup(map->key_size, 8));
2358	2361	*insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0);
2359	2362	*insn++ = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0);
2360	2363

kernel/bpf/helpers.c+119−4

@@ -1758,8 +1758,8 @@ static const struct bpf_func_proto bpf_dynptr_from_mem_proto = { · 1757 unmodified lines
1758	1758	.arg4_type = ARG_PTR_TO_DYNPTR \| DYNPTR_TYPE_LOCAL \| MEM_UNINIT \| MEM_WRITE,
1759	1759	};
1760	1760
1761		−BPF_CALL_5(bpf_dynptr_read, void , dst, u32, len, const struct bpf_dynptr_kern , src,
1762		− u32, offset, u64, flags)
	1761	+static int __bpf_dynptr_read(void dst, u32 len, const struct bpf_dynptr_kern src,
	1762	+ u32 offset, u64 flags)
1763	1763	{
1764	1764	enum bpf_dynptr_type type;
1765	1765	int err;
@@ -1792,6 +1792,12 @@ BPF_CALL_5(bpf_dynptr_read, void *, dst, u32, len, const struct bpf_dynptr_kern · 26 unmodified lines
1792	1792	}
1793	1793	}
1794	1794
	1795	+BPF_CALL_5(bpf_dynptr_read, void , dst, u32, len, const struct bpf_dynptr_kern , src,
	1796	+ u32, offset, u64, flags)
	1797	+{
	1798	+ return __bpf_dynptr_read(dst, len, src, offset, flags);
	1799	+}
	1800	+
1795	1801	static const struct bpf_func_proto bpf_dynptr_read_proto = {
1796	1802	.func = bpf_dynptr_read,
1797	1803	.gpl_only = false,
@@ -1803,8 +1809,8 @@ static const struct bpf_func_proto bpf_dynptr_read_proto = { · 5 unmodified lines
1803	1809	.arg5_type = ARG_ANYTHING,
1804	1810	};
1805	1811
1806		−BPF_CALL_5(bpf_dynptr_write, const struct bpf_dynptr_kern , dst, u32, offset, void , src,
1807		− u32, len, u64, flags)
	1812	+static int __bpf_dynptr_write(const struct bpf_dynptr_kern dst, u32 offset, void src,
	1813	+ u32 len, u64 flags)
1808	1814	{
1809	1815	enum bpf_dynptr_type type;
1810	1816	int err;
@@ -1842,6 +1848,12 @@ BPF_CALL_5(bpf_dynptr_write, const struct bpf_dynptr_kern *, dst, u32, offset, v · 31 unmodified lines
1842	1848	}
1843	1849	}
1844	1850
	1851	+BPF_CALL_5(bpf_dynptr_write, const struct bpf_dynptr_kern , dst, u32, offset, void , src,
	1852	+ u32, len, u64, flags)
	1853	+{
	1854	+ return __bpf_dynptr_write(dst, offset, src, len, flags);
	1855	+}
	1856	+
1845	1857	static const struct bpf_func_proto bpf_dynptr_write_proto = {
1846	1858	.func = bpf_dynptr_write,
1847	1859	.gpl_only = false,
@@ -2043,6 +2055,8 @@ bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 195 unmodified lines
2043	2055	return &bpf_task_pt_regs_proto;
2044	2056	case BPF_FUNC_trace_vprintk:
2045	2057	return bpf_get_trace_vprintk_proto();
	2058	+ case BPF_FUNC_perf_event_read_value:
	2059	+ return bpf_get_perf_event_read_value_proto();
2046	2060	default:
2047	2061	return NULL;
2048	2062	}
@@ -2757,6 +2771,61 @@ __bpf_kfunc int bpf_dynptr_clone(const struct bpf_dynptr *p, · 708 unmodified lines
2757	2771	return 0;
2758	2772	}
2759	2773
	2774	+/**
	2775	+ * bpf_dynptr_copy() - Copy data from one dynptr to another.
	2776	+ * @dst_ptr: Destination dynptr - where data should be copied to
	2777	+ * @dst_off: Offset into the destination dynptr
	2778	+ * @src_ptr: Source dynptr - where data should be copied from
	2779	+ * @src_off: Offset into the source dynptr
	2780	+ * @size: Length of the data to copy from source to destination
	2781	+ *
	2782	+ * Copies data from source dynptr to destination dynptr.
	2783	+ * Returns 0 on success; negative error, otherwise.
	2784	+ */
	2785	+__bpf_kfunc int bpf_dynptr_copy(struct bpf_dynptr *dst_ptr, u32 dst_off,
	2786	+ struct bpf_dynptr *src_ptr, u32 src_off, u32 size)
	2787	+{
	2788	+ struct bpf_dynptr_kern dst = (struct bpf_dynptr_kern )dst_ptr;
	2789	+ struct bpf_dynptr_kern src = (struct bpf_dynptr_kern )src_ptr;
	2790	+ void src_slice, dst_slice;
	2791	+ char buf[256];
	2792	+ u32 off;
	2793	+
	2794	+ src_slice = bpf_dynptr_slice(src_ptr, src_off, NULL, size);
	2795	+ dst_slice = bpf_dynptr_slice_rdwr(dst_ptr, dst_off, NULL, size);
	2796	+
	2797	+ if (src_slice && dst_slice) {
	2798	+ memmove(dst_slice, src_slice, size);
	2799	+ return 0;
	2800	+ }
	2801	+
	2802	+ if (src_slice)
	2803	+ return __bpf_dynptr_write(dst, dst_off, src_slice, size, 0);
	2804	+
	2805	+ if (dst_slice)
	2806	+ return __bpf_dynptr_read(dst_slice, size, src, src_off, 0);
	2807	+
	2808	+ if (bpf_dynptr_check_off_len(dst, dst_off, size) \|\|
	2809	+ bpf_dynptr_check_off_len(src, src_off, size))
	2810	+ return -E2BIG;
	2811	+
	2812	+ off = 0;
	2813	+ while (off < size) {
	2814	+ u32 chunk_sz = min_t(u32, sizeof(buf), size - off);
	2815	+ int err;
	2816	+
	2817	+ err = __bpf_dynptr_read(buf, chunk_sz, src, src_off + off, 0);
	2818	+ if (err)
	2819	+ return err;
	2820	+ err = __bpf_dynptr_write(dst, dst_off + off, buf, chunk_sz, 0);
	2821	+ if (err)
	2822	+ return err;
	2823	+
	2824	+ off += chunk_sz;
	2825	+ }
	2826	+ return 0;
	2827	+}
	2828	+
2760	2829	__bpf_kfunc void bpf_cast_to_kern_ctx(void obj)
2761	2830	{
2762	2831	return obj;
@@ -3066,6 +3135,50 @@ __bpf_kfunc int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void __user · 303 unmodified lines
3066	3135	return ret + 1;
3067	3136	}
3068	3137
	3138	+/**
	3139	+ * bpf_copy_from_user_task_str() - Copy a string from an task's address space
	3140	+ * @dst: Destination address, in kernel space. This buffer must be
	3141	+ * at least @dst__sz bytes long.
	3142	+ * @dst__sz: Maximum number of bytes to copy, includes the trailing NUL.
	3143	+ * @unsafe_ptr__ign: Source address in the task's address space.
	3144	+ * @tsk: The task whose address space will be used
	3145	+ * @flags: The only supported flag is BPF_F_PAD_ZEROS
	3146	+ *
	3147	+ * Copies a NUL terminated string from a task's address space to @dst__sz
	3148	+ * buffer. If user string is too long this will still ensure zero termination
	3149	+ * in the @dst__sz buffer unless buffer size is 0.
	3150	+ *
	3151	+ * If BPF_F_PAD_ZEROS flag is set, memset the tail of @dst__sz to 0 on success
	3152	+ * and memset all of @dst__sz on failure.
	3153	+ *
	3154	+ * Return: The number of copied bytes on success including the NUL terminator.
	3155	+ * A negative error code on failure.
	3156	+ */
	3157	+__bpf_kfunc int bpf_copy_from_user_task_str(void *dst, u32 dst__sz,
	3158	+ const void __user *unsafe_ptr__ign,
	3159	+ struct task_struct *tsk, u64 flags)
	3160	+{
	3161	+ int ret;
	3162	+
	3163	+ if (unlikely(flags & ~BPF_F_PAD_ZEROS))
	3164	+ return -EINVAL;
	3165	+
	3166	+ if (unlikely(dst__sz == 0))
	3167	+ return 0;
	3168	+
	3169	+ ret = copy_remote_vm_str(tsk, (unsigned long)unsafe_ptr__ign, dst, dst__sz, 0);
	3170	+ if (ret < 0) {
	3171	+ if (flags & BPF_F_PAD_ZEROS)
	3172	+ memset(dst, 0, dst__sz);
	3173	+ return ret;
	3174	+ }
	3175	+
	3176	+ if (flags & BPF_F_PAD_ZEROS)
	3177	+ memset(dst + ret, 0, dst__sz - ret);
	3178	+
	3179	+ return ret + 1;
	3180	+}
	3181	+
3069	3182	/* Keep unsinged long in prototype so that kfunc is usable when emitted to
3070	3183	* vmlinux.h in BPF programs directly, but note that while in BPF prog, the
3071	3184	* unsigned long always points to 8-byte region on stack, the kernel may only
@@ -3161,6 +3274,7 @@ BTF_ID_FLAGS(func, bpf_dynptr_is_null) · 89 unmodified lines
3161	3274	BTF_ID_FLAGS(func, bpf_dynptr_is_rdonly)
3162	3275	BTF_ID_FLAGS(func, bpf_dynptr_size)
3163	3276	BTF_ID_FLAGS(func, bpf_dynptr_clone)
	3277	+BTF_ID_FLAGS(func, bpf_dynptr_copy)
3164	3278	#ifdef CONFIG_NET
3165	3279	BTF_ID_FLAGS(func, bpf_modify_return_test_tp)
3166	3280	#endif
@@ -3173,6 +3287,7 @@ BTF_ID_FLAGS(func, bpf_iter_bits_new, KF_ITER_NEW) · 6 unmodified lines
3173	3287	BTF_ID_FLAGS(func, bpf_iter_bits_next, KF_ITER_NEXT \| KF_RET_NULL)
3174	3288	BTF_ID_FLAGS(func, bpf_iter_bits_destroy, KF_ITER_DESTROY)
3175	3289	BTF_ID_FLAGS(func, bpf_copy_from_user_str, KF_SLEEPABLE)
	3290	+BTF_ID_FLAGS(func, bpf_copy_from_user_task_str, KF_SLEEPABLE)
3176	3291	BTF_ID_FLAGS(func, bpf_get_kmem_cache)
3177	3292	BTF_ID_FLAGS(func, bpf_iter_kmem_cache_new, KF_ITER_NEW \| KF_SLEEPABLE)
3178	3293	BTF_ID_FLAGS(func, bpf_iter_kmem_cache_next, KF_ITER_NEXT \| KF_RET_NULL \| KF_SLEEPABLE)

kernel/bpf/preload/bpf_preload_kern.c+1−0

@@ -90,3 +90,4 @@ static void __exit fini(void) · 89 unmodified lines
90	90	late_initcall(load);
91	91	module_exit(fini);
92	92	MODULE_LICENSE("GPL");
	93	+MODULE_DESCRIPTION("Embedded BPF programs for introspection in bpffs");

kernel/bpf/syscall.c+33−15

@@ -1315,7 +1315,7 @@ static bool bpf_net_capable(void) · 1314 unmodified lines
1315	1315
1316	1316	#define BPF_MAP_CREATE_LAST_FIELD map_token_fd
1317	1317	/* called via syscall */
1318		−static int map_create(union bpf_attr *attr)
	1318	+static int map_create(union bpf_attr *attr, bool kernel)
1319	1319	{
1320	1320	const struct bpf_map_ops *ops;
1321	1321	struct bpf_token *token = NULL;
@@ -1505,7 +1505,7 @@ static int map_create(union bpf_attr *attr) · 183 unmodified lines
1505	1505	attr->btf_vmlinux_value_type_id;
1506	1506	}
1507	1507
1508		− err = security_bpf_map_create(map, attr, token);
	1508	+ err = security_bpf_map_create(map, attr, token, kernel);
1509	1509	if (err)
1510	1510	goto free_map_sec;
1511	1511
@@ -1593,11 +1593,8 @@ struct bpf_map __bpf_map_inc_not_zero(struct bpf_map map, bool uref) · 81 unmodified lines
1593	1593
1594	1594	struct bpf_map bpf_map_inc_not_zero(struct bpf_map map)
1595	1595	{
1596		− spin_lock_bh(&map_idr_lock);
1597		− map = __bpf_map_inc_not_zero(map, false);
1598		− spin_unlock_bh(&map_idr_lock);
1599		−
1600		− return map;
	1596	+ lockdep_assert(rcu_read_lock_held());
	1597	+ return __bpf_map_inc_not_zero(map, false);
1601	1598	}
1602	1599	EXPORT_SYMBOL_GPL(bpf_map_inc_not_zero);
1603	1600
@@ -2314,6 +2311,7 @@ static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) · 710 unmodified lines
2314	2311	kvfree(prog->aux->jited_linfo);
2315	2312	kvfree(prog->aux->linfo);
2316	2313	kfree(prog->aux->kfunc_tab);
	2314	+ kfree(prog->aux->ctx_arg_info);
2317	2315	if (prog->aux->attach_btf)
2318	2316	btf_put(prog->aux->attach_btf);
2319	2317
@@ -2944,7 +2942,7 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) · 624 unmodified lines
2944	2942	if (err < 0)
2945	2943	goto free_prog;
2946	2944
2947		− err = security_bpf_prog_load(prog, attr, token);
	2945	+ err = security_bpf_prog_load(prog, attr, token, uattr.is_kernel);
2948	2946	if (err)
2949	2947	goto free_prog_sec;
2950	2948
@@ -4169,7 +4167,8 @@ static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog, · 1218 unmodified lines
4169	4167	#define BPF_F_ATTACH_MASK_BASE \
4170	4168	(BPF_F_ALLOW_OVERRIDE \| \
4171	4169	BPF_F_ALLOW_MULTI \| \
4172		− BPF_F_REPLACE)
	4170	+ BPF_F_REPLACE \| \
	4171	+ BPF_F_PREORDER)
4173	4172
4174	4173	#define BPF_F_ATTACH_MASK_MPROG \
4175	4174	(BPF_F_REPLACE \| \
@@ -4733,6 +4732,8 @@ static int bpf_prog_get_info_by_fd(struct file *file, · 557 unmodified lines
4733	4732	info.recursion_misses = stats.misses;
4734	4733
4735	4734	info.verified_insns = prog->aux->verified_insns;
	4735	+ if (prog->aux->btf)
	4736	+ info.btf_id = btf_obj_id(prog->aux->btf);
4736	4737
4737	4738	if (!bpf_capable()) {
4738	4739	info.jited_prog_len = 0;
@@ -4879,8 +4880,6 @@ static int bpf_prog_get_info_by_fd(struct file *file, · 140 unmodified lines
4879	4880	}
4880	4881	}
4881	4882
4882		− if (prog->aux->btf)
4883		− info.btf_id = btf_obj_id(prog->aux->btf);
4884	4883	info.attach_btf_id = prog->aux->attach_btf_id;
4885	4884	if (attach_btf)
4886	4885	info.attach_btf_obj_id = btf_obj_id(attach_btf);
@@ -5121,15 +5120,34 @@ static int bpf_btf_load(const union bpf_attr *attr, bpfptr_t uattr, __u32 uattr_ · 234 unmodified lines
5121	5120	return btf_new_fd(attr, uattr, uattr_size);
5122	5121	}
5123	5122
5124		−#define BPF_BTF_GET_FD_BY_ID_LAST_FIELD btf_id
	5123	+#define BPF_BTF_GET_FD_BY_ID_LAST_FIELD fd_by_id_token_fd
5125	5124
5126	5125	static int bpf_btf_get_fd_by_id(const union bpf_attr *attr)
5127	5126	{
	5127	+ struct bpf_token *token = NULL;
	5128	+
5128	5129	if (CHECK_ATTR(BPF_BTF_GET_FD_BY_ID))
5129	5130	return -EINVAL;
5130	5131
5131		− if (!capable(CAP_SYS_ADMIN))
	5132	+ if (attr->open_flags & ~BPF_F_TOKEN_FD)
	5133	+ return -EINVAL;
	5134	+
	5135	+ if (attr->open_flags & BPF_F_TOKEN_FD) {
	5136	+ token = bpf_token_get_from_fd(attr->fd_by_id_token_fd);
	5137	+ if (IS_ERR(token))
	5138	+ return PTR_ERR(token);
	5139	+ if (!bpf_token_allow_cmd(token, BPF_BTF_GET_FD_BY_ID)) {
	5140	+ bpf_token_put(token);
	5141	+ token = NULL;
	5142	+ }
	5143	+ }
	5144	+
	5145	+ if (!bpf_token_capable(token, CAP_SYS_ADMIN)) {
	5146	+ bpf_token_put(token);
5132	5147	return -EPERM;
	5148	+ }
	5149	+
	5150	+ bpf_token_put(token);
5133	5151
5134	5152	return btf_get_fd_by_id(attr->btf_id);
5135	5153	}
@@ -5768,13 +5786,13 @@ static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size) · 632 unmodified lines
5768	5786	if (copy_from_bpfptr(&attr, uattr, size) != 0)
5769	5787	return -EFAULT;
5770	5788
5771		− err = security_bpf(cmd, &attr, size);
	5789	+ err = security_bpf(cmd, &attr, size, uattr.is_kernel);
5772	5790	if (err < 0)
5773	5791	return err;
5774	5792
5775	5793	switch (cmd) {
5776	5794	case BPF_MAP_CREATE:
5777		− err = map_create(&attr);
	5795	+ err = map_create(&attr, uattr.is_kernel);
5778	5796	break;
5779	5797	case BPF_MAP_LOOKUP_ELEM:
5780	5798	err = map_lookup_elem(&attr);

kernel/bpf/verifier.c+1043−333

@@ -579,6 +579,13 @@ static bool is_cmpxchg_insn(const struct bpf_insn *insn) · 578 unmodified lines
579	579	insn->imm == BPF_CMPXCHG;
580	580	}
581	581
	582	+static bool is_atomic_load_insn(const struct bpf_insn *insn)
	583	+{
	584	+ return BPF_CLASS(insn->code) == BPF_STX &&
	585	+ BPF_MODE(insn->code) == BPF_ATOMIC &&
	586	+ insn->imm == BPF_LOAD_ACQ;
	587	+}
	588	+
582	589	static int __get_spi(s32 off)
583	590	{
584	591	return (-off - 1) / BPF_REG_SIZE;
@@ -1545,6 +1552,17 @@ static void release_reference_state(struct bpf_verifier_state *state, int idx) · 960 unmodified lines
1545	1552	return;
1546	1553	}
1547	1554
	1555	+static bool find_reference_state(struct bpf_verifier_state *state, int ptr_id)
	1556	+{
	1557	+ int i;
	1558	+
	1559	+ for (i = 0; i < state->acquired_refs; i++)
	1560	+ if (state->refs[i].id == ptr_id)
	1561	+ return true;
	1562	+
	1563	+ return false;
	1564	+}
	1565	+
1548	1566	static int release_lock_state(struct bpf_verifier_state state, int type, int id, void ptr)
1549	1567	{
1550	1568	int i;
@@ -1600,6 +1618,14 @@ static struct bpf_reference_state find_lock_state(struct bpf_verifier_state st · 49 unmodified lines
1600	1618	return NULL;
1601	1619	}
1602	1620
	1621	+static void update_peak_states(struct bpf_verifier_env *env)
	1622	+{
	1623	+ u32 cur_states;
	1624	+
	1625	+ cur_states = env->explored_states_size + env->free_list_size;
	1626	+ env->peak_states = max(env->peak_states, cur_states);
	1627	+}
	1628	+
1603	1629	static void free_func_state(struct bpf_func_state *state)
1604	1630	{
1605	1631	if (!state)
@@ -1622,6 +1648,50 @@ static void free_verifier_state(struct bpf_verifier_state *state, · 16 unmodified lines
1622	1648	kfree(state);
1623	1649	}
1624	1650
	1651	+/* struct bpf_verifier_state->{parent,loop_entry} refer to states
	1652	+ * that are in either of env->{expored_states,free_list}.
	1653	+ * In both cases the state is contained in struct bpf_verifier_state_list.
	1654	+ */
	1655	+static struct bpf_verifier_state_list state_parent_as_list(struct bpf_verifier_state st)
	1656	+{
	1657	+ if (st->parent)
	1658	+ return container_of(st->parent, struct bpf_verifier_state_list, state);
	1659	+ return NULL;
	1660	+}
	1661	+
	1662	+static struct bpf_verifier_state_list state_loop_entry_as_list(struct bpf_verifier_state st)
	1663	+{
	1664	+ if (st->loop_entry)
	1665	+ return container_of(st->loop_entry, struct bpf_verifier_state_list, state);
	1666	+ return NULL;
	1667	+}
	1668	+
	1669	+/* A state can be freed if it is no longer referenced:
	1670	+ * - is in the env->free_list;
	1671	+ * - has no children states;
	1672	+ * - is not used as loop_entry.
	1673	+ *
	1674	+ * Freeing a state can make it's loop_entry free-able.
	1675	+ */
	1676	+static void maybe_free_verifier_state(struct bpf_verifier_env *env,
	1677	+ struct bpf_verifier_state_list *sl)
	1678	+{
	1679	+ struct bpf_verifier_state_list *loop_entry_sl;
	1680	+
	1681	+ while (sl && sl->in_free_list &&
	1682	+ sl->state.branches == 0 &&
	1683	+ sl->state.used_as_loop_entry == 0) {
	1684	+ loop_entry_sl = state_loop_entry_as_list(&sl->state);
	1685	+ if (loop_entry_sl)
	1686	+ loop_entry_sl->state.used_as_loop_entry--;
	1687	+ list_del(&sl->node);
	1688	+ free_verifier_state(&sl->state, false);
	1689	+ kfree(sl);
	1690	+ env->free_list_size--;
	1691	+ sl = loop_entry_sl;
	1692	+ }
	1693	+}
	1694	+
1625	1695	/* copy verifier state from src to dst growing dst stack space
1626	1696	* when necessary to accommodate larger src stack
1627	1697	*/
@@ -1661,6 +1731,7 @@ static int copy_verifier_state(struct bpf_verifier_state *dst_state, · 33 unmodified lines
1661	1731	dst_state->callback_unroll_depth = src->callback_unroll_depth;
1662	1732	dst_state->used_as_loop_entry = src->used_as_loop_entry;
1663	1733	dst_state->may_goto_depth = src->may_goto_depth;
	1734	+ dst_state->loop_entry = src->loop_entry;
1664	1735	for (i = 0; i <= src->curframe; i++) {
1665	1736	dst = dst_state->frame[i];
1666	1737	if (!dst) {
@@ -1681,7 +1752,7 @@ static u32 state_htab_size(struct bpf_verifier_env *env) · 14 unmodified lines
1681	1752	return env->prog->len;
1682	1753	}
1683	1754
1684		−static struct bpf_verifier_state_list *explored_state(struct bpf_verifier_env env, int idx)
	1755	+static struct list_head explored_state(struct bpf_verifier_env env, int idx)
1685	1756	{
1686	1757	struct bpf_verifier_state *cur = env->cur_state;
1687	1758	struct bpf_func_state *state = cur->frame[cur->curframe];
@@ -1789,16 +1860,13 @@ static bool same_callsites(struct bpf_verifier_state *a, struct bpf_verifier_sta · 101 unmodified lines
1789	1860	* # Find outermost loop entry known for n
1790	1861	* def get_loop_entry(n):
1791	1862	* h = entries.get(n, None)
1792		− * while h in entries and entries[h] != h:
	1863	+ * while h in entries:
1793	1864	* h = entries[h]
1794	1865	* return h
1795	1866	*
1796		− * # Update n's loop entry if h's outermost entry comes
1797		− * # before n's outermost entry in current DFS path.
	1867	+ * # Update n's loop entry if h comes before n in current DFS path.
1798	1868	* def update_loop_entry(n, h):
1799		− * n1 = get_loop_entry(n) or n
1800		− * h1 = get_loop_entry(h) or h
1801		− * if h1 in path and depths[h1] <= depths[n1]:
	1869	+ * if h in path and depths[entries.get(n, n)] < depths[n]:
1802	1870	* entries[n] = h1
1803	1871	*
1804	1872	* def dfs(n, depth):
@@ -1810,7 +1878,7 @@ static bool same_callsites(struct bpf_verifier_state *a, struct bpf_verifier_sta · 5 unmodified lines
1810	1878	* # Case A: explore succ and update cur's loop entry
1811	1879	* # only if succ's entry is in current DFS path.
1812	1880	* dfs(succ, depth + 1)
1813		− * h = get_loop_entry(succ)
	1881	+ * h = entries.get(succ, None)
1814	1882	* update_loop_entry(n, h)
1815	1883	* else:
1816	1884	* # Case B or C depending on `h1 in path` check in update_loop_entry().
@@ -1822,46 +1890,49 @@ static bool same_callsites(struct bpf_verifier_state *a, struct bpf_verifier_sta · 5 unmodified lines
1822	1890	* and cur's loop entry has to be updated (case A), handle this in
1823	1891	* update_branch_counts();
1824	1892	* - use st->branch > 0 as a signal that st is in the current DFS path;
1825		− * - handle cases B and C in is_state_visited();
1826		− * - update topmost loop entry for intermediate states in get_loop_entry().
	1893	+ * - handle cases B and C in is_state_visited().
1827	1894	*/
1828		−static struct bpf_verifier_state get_loop_entry(struct bpf_verifier_state st)
	1895	+static struct bpf_verifier_state get_loop_entry(struct bpf_verifier_env env,
	1896	+ struct bpf_verifier_state *st)
1829	1897	{
1830		− struct bpf_verifier_state topmost = st->loop_entry, old;
	1898	+ struct bpf_verifier_state *topmost = st->loop_entry;
	1899	+ u32 steps = 0;
1831	1900
1832		− while (topmost && topmost->loop_entry && topmost != topmost->loop_entry)
	1901	+ while (topmost && topmost->loop_entry) {
	1902	+ if (steps++ > st->dfs_depth) {
	1903	+ WARN_ONCE(true, "verifier bug: infinite loop in get_loop_entry\n");
	1904	+ verbose(env, "verifier bug: infinite loop in get_loop_entry()\n");
	1905	+ return ERR_PTR(-EFAULT);
	1906	+ }
1833	1907	topmost = topmost->loop_entry;
1834		− /* Update loop entries for intermediate states to avoid this
1835		− * traversal in future get_loop_entry() calls.
1836		− */
1837		− while (st && st->loop_entry != topmost) {
1838		− old = st->loop_entry;
1839		− st->loop_entry = topmost;
1840		− st = old;
1841	1908	}
1842	1909	return topmost;
1843	1910	}
1844	1911
1845		−static void update_loop_entry(struct bpf_verifier_state cur, struct bpf_verifier_state hdr)
	1912	+static void update_loop_entry(struct bpf_verifier_env *env,
	1913	+ struct bpf_verifier_state cur, struct bpf_verifier_state hdr)
1846	1914	{
1847		− struct bpf_verifier_state cur1, hdr1;
1848		−
1849		− cur1 = get_loop_entry(cur) ?: cur;
1850		− hdr1 = get_loop_entry(hdr) ?: hdr;
1851		− /* The head1->branches check decides between cases B and C in
1852		− * comment for get_loop_entry(). If hdr1->branches == 0 then
	1915	+ /* The hdr->branches check decides between cases B and C in
	1916	+ * comment for get_loop_entry(). If hdr->branches == 0 then
1853	1917	* head's topmost loop entry is not in current DFS path,
1854	1918	* hence 'cur' and 'hdr' are not in the same loop and there is
1855	1919	* no need to update cur->loop_entry.
1856	1920	*/
1857		− if (hdr1->branches && hdr1->dfs_depth <= cur1->dfs_depth) {
	1921	+ if (hdr->branches && hdr->dfs_depth < (cur->loop_entry ?: cur)->dfs_depth) {
	1922	+ if (cur->loop_entry) {
	1923	+ cur->loop_entry->used_as_loop_entry--;
	1924	+ maybe_free_verifier_state(env, state_loop_entry_as_list(cur));
	1925	+ }
1858	1926	cur->loop_entry = hdr;
1859		− hdr->used_as_loop_entry = true;
	1927	+ hdr->used_as_loop_entry++;
1860	1928	}
1861	1929	}
1862	1930
1863	1931	static void update_branch_counts(struct bpf_verifier_env env, struct bpf_verifier_state st)
1864	1932	{
	1933	+ struct bpf_verifier_state_list sl = NULL, parent_sl;
	1934	+ struct bpf_verifier_state *parent;
	1935	+
1865	1936	while (st) {
1866	1937	u32 br = --st->branches;
1867	1938
@@ -1871,7 +1942,7 @@ static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifi · 3 unmodified lines
1871	1942	* This is a part of 'case A' in get_loop_entry() comment.
1872	1943	*/
1873	1944	if (br == 0 && st->parent && st->loop_entry)
1874		− update_loop_entry(st->parent, st->loop_entry);
	1945	+ update_loop_entry(env, st->parent, st->loop_entry);
1875	1946
1876	1947	/* WARN_ON(br > 1) technically makes sense here,
1877	1948	* but see comment in push_stack(), hence:
@@ -1881,7 +1952,12 @@ static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifi · 3 unmodified lines
1881	1952	br);
1882	1953	if (br)
1883	1954	break;
1884		− st = st->parent;
	1955	+ parent = st->parent;
	1956	+ parent_sl = state_parent_as_list(st);
	1957	+ if (sl)
	1958	+ maybe_free_verifier_state(env, sl);
	1959	+ st = parent;
	1960	+ sl = parent_sl;
1885	1961	}
1886	1962	}
1887	1963
@@ -3206,6 +3282,21 @@ bpf_jit_find_kfunc_model(const struct bpf_prog *prog, · 1318 unmodified lines
3206	3282	return res ? &res->func_model : NULL;
3207	3283	}
3208	3284
	3285	+static int add_kfunc_in_insns(struct bpf_verifier_env *env,
	3286	+ struct bpf_insn *insn, int cnt)
	3287	+{
	3288	+ int i, ret;
	3289	+
	3290	+ for (i = 0; i < cnt; i++, insn++) {
	3291	+ if (bpf_pseudo_kfunc_call(insn)) {
	3292	+ ret = add_kfunc_call(env, insn->imm, insn->off);
	3293	+ if (ret < 0)
	3294	+ return ret;
	3295	+ }
	3296	+ }
	3297	+ return 0;
	3298	+}
	3299	+
3209	3300	static int add_subprog_and_kfunc(struct bpf_verifier_env *env)
3210	3301	{
3211	3302	struct bpf_subprog_info *subprog = env->subprog_info;
@@ -3269,6 +3360,15 @@ static int add_subprog_and_kfunc(struct bpf_verifier_env *env) · 57 unmodified lines
3269	3360	return 0;
3270	3361	}
3271	3362
	3363	+static int jmp_offset(struct bpf_insn *insn)
	3364	+{
	3365	+ u8 code = insn->code;
	3366	+
	3367	+ if (code == (BPF_JMP32 \| BPF_JA))
	3368	+ return insn->imm;
	3369	+ return insn->off;
	3370	+}
	3371	+
3272	3372	static int check_subprogs(struct bpf_verifier_env *env)
3273	3373	{
3274	3374	int i, subprog_start, subprog_end, off, cur_subprog = 0;
@@ -3295,10 +3395,7 @@ static int check_subprogs(struct bpf_verifier_env *env) · 20 unmodified lines
3295	3395	goto next;
3296	3396	if (BPF_OP(code) == BPF_EXIT \|\| BPF_OP(code) == BPF_CALL)
3297	3397	goto next;
3298		− if (code == (BPF_JMP32 \| BPF_JA))
3299		− off = i + insn[i].imm + 1;
3300		− else
3301		− off = i + insn[i].off + 1;
	3398	+ off = i + jmp_offset(&insn[i]) + 1;
3302	3399	if (off < subprog_start \|\| off >= subprog_end) {
3303	3400	verbose(env, "jump out of range from insn %d to %d\n", i, off);
3304	3401	return -EINVAL;
@@ -3483,7 +3580,7 @@ static bool is_reg64(struct bpf_verifier_env env, struct bpf_insn insn, · 178 unmodified lines
3483	3580	}
3484	3581
3485	3582	if (class == BPF_STX) {
3486		− /* BPF_STX (including atomic variants) has multiple source
	3583	+ /* BPF_STX (including atomic variants) has one or more source
3487	3584	* operands, one of which is a ptr. Check whether the caller is
3488	3585	* asking about it.
3489	3586	*/
@@ -3828,6 +3925,17 @@ static const char disasm_kfunc_name(void data, const struct bpf_insn *insn) · 338 unmodified lines
3828	3925	return btf_name_by_offset(desc_btf, func->name_off);
3829	3926	}
3830	3927
	3928	+static void verbose_insn(struct bpf_verifier_env env, struct bpf_insn insn)
	3929	+{
	3930	+ const struct bpf_insn_cbs cbs = {
	3931	+ .cb_call = disasm_kfunc_name,
	3932	+ .cb_print = verbose,
	3933	+ .private_data = env,
	3934	+ };
	3935	+
	3936	+ print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
	3937	+}
	3938	+
3831	3939	static inline void bt_init(struct backtrack_state *bt, u32 frame)
3832	3940	{
3833	3941	bt->frame = frame;
@@ -4028,11 +4136,6 @@ static bool calls_callback(struct bpf_verifier_env *env, int insn_idx); · 194 unmodified lines
4028	4136	static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
4029	4137	struct bpf_insn_hist_entry hist, struct backtrack_state bt)
4030	4138	{
4031		− const struct bpf_insn_cbs cbs = {
4032		− .cb_call = disasm_kfunc_name,
4033		− .cb_print = verbose,
4034		− .private_data = env,
4035		− };
4036	4139	struct bpf_insn *insn = env->prog->insnsi + idx;
4037	4140	u8 class = BPF_CLASS(insn->code);
4038	4141	u8 opcode = BPF_OP(insn->code);
@@ -4050,7 +4153,7 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, · 11 unmodified lines
4050	4153	fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_stack_mask(bt));
4051	4154	verbose(env, "stack=%s before ", env->tmp_str_buf);
4052	4155	verbose(env, "%d: ", idx);
4053		− print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
	4156	+ verbose_insn(env, insn);
4054	4157	}
4055	4158
4056	4159	/* If there is a history record that some registers gained range at this insn,
@@ -4097,7 +4200,7 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, · 40 unmodified lines
4097	4200	* dreg still needs precision before this insn
4098	4201	*/
4099	4202	}
4100		− } else if (class == BPF_LDX) {
	4203	+ } else if (class == BPF_LDX \|\| is_atomic_load_insn(insn)) {
4101	4204	if (!bt_is_reg_set(bt, dreg))
4102	4205	return 0;
4103	4206	bt_clear_reg(bt, dreg);
@@ -5982,18 +6085,10 @@ static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off, · 1878 unmodified lines
5982	6085
5983	6086	/* check access to 'struct bpf_context' fields. Supports fixed offsets only */
5984	6087	static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size,
5985		− enum bpf_access_type t, enum bpf_reg_type *reg_type,
5986		− struct btf *btf, u32 btf_id, bool *is_retval, bool is_ldsx)
	6088	+ enum bpf_access_type t, struct bpf_insn_access_aux *info)
5987	6089	{
5988		− struct bpf_insn_access_aux info = {
5989		− .reg_type = *reg_type,
5990		− .log = &env->log,
5991		− .is_retval = false,
5992		− .is_ldsx = is_ldsx,
5993		− };
5994		−
5995	6090	if (env->ops->is_valid_access &&
5996		− env->ops->is_valid_access(off, size, t, env->prog, &info)) {
	6091	+ env->ops->is_valid_access(off, size, t, env->prog, info)) {
5997	6092	/* A non zero info.ctx_field_size indicates that this field is a
5998	6093	* candidate for later verifier transformation to load the whole
5999	6094	* field and then apply a mask when accessed with a narrower
@@ -6001,14 +6096,15 @@ static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, · 1 unmodified line
6001	6096	* will only allow for whole field access and rejects any other
6002	6097	* type of narrower access.
6003	6098	*/
6004		− *reg_type = info.reg_type;
6005		− *is_retval = info.is_retval;
6006		−
6007		− if (base_type(*reg_type) == PTR_TO_BTF_ID) {
6008		− *btf = info.btf;
6009		− *btf_id = info.btf_id;
	6099	+ if (base_type(info->reg_type) == PTR_TO_BTF_ID) {
	6100	+ if (info->ref_obj_id &&
	6101	+ !find_reference_state(env->cur_state, info->ref_obj_id)) {
	6102	+ verbose(env, "invalid bpf_context access off=%d. Reference may already be released\n",
	6103	+ off);
	6104	+ return -EACCES;
	6105	+ }
6010	6106	} else {
6011		− env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size;
	6107	+ env->insn_aux_data[insn_idx].ctx_field_size = info->ctx_field_size;
6012	6108	}
6013	6109	/* remember the offset of last byte accessed in ctx */
6014	6110	if (env->prog->aux->max_ctx_offset < off + size)
@@ -6118,6 +6214,26 @@ static bool is_arena_reg(struct bpf_verifier_env *env, int regno) · 103 unmodified lines
6118	6214	return reg->type == PTR_TO_ARENA;
6119	6215	}
6120	6216
	6217	+/* Return false if @regno contains a pointer whose type isn't supported for
	6218	+ * atomic instruction @insn.
	6219	+ */
	6220	+static bool atomic_ptr_type_ok(struct bpf_verifier_env *env, int regno,
	6221	+ struct bpf_insn *insn)
	6222	+{
	6223	+ if (is_ctx_reg(env, regno))
	6224	+ return false;
	6225	+ if (is_pkt_reg(env, regno))
	6226	+ return false;
	6227	+ if (is_flow_key_reg(env, regno))
	6228	+ return false;
	6229	+ if (is_sk_reg(env, regno))
	6230	+ return false;
	6231	+ if (is_arena_reg(env, regno))
	6232	+ return bpf_jit_supports_insn(insn, true);
	6233	+
	6234	+ return true;
	6235	+}
	6236	+
6121	6237	static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = {
6122	6238	#ifdef CONFIG_NET
6123	6239	[PTR_TO_SOCKET] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK],
@@ -7365,11 +7481,12 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 1241 unmodified lines
7365	7481	if (!err && value_regno >= 0 && (t == BPF_READ \|\| rdonly_mem))
7366	7482	mark_reg_unknown(env, regs, value_regno);
7367	7483	} else if (reg->type == PTR_TO_CTX) {
7368		− bool is_retval = false;
7369	7484	struct bpf_retval_range range;
7370		− enum bpf_reg_type reg_type = SCALAR_VALUE;
7371		− struct btf *btf = NULL;
7372		− u32 btf_id = 0;
	7485	+ struct bpf_insn_access_aux info = {
	7486	+ .reg_type = SCALAR_VALUE,
	7487	+ .is_ldsx = is_ldsx,
	7488	+ .log = &env->log,
	7489	+ };
7373	7490
7374	7491	if (t == BPF_WRITE && value_regno >= 0 &&
7375	7492	is_pointer_value(env, value_regno)) {
@@ -7381,8 +7498,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 5 unmodified lines
7381	7498	if (err < 0)
7382	7499	return err;
7383	7500
7384		− err = check_ctx_access(env, insn_idx, off, size, t, &reg_type, &btf,
7385		− &btf_id, &is_retval, is_ldsx);
	7501	+ err = check_ctx_access(env, insn_idx, off, size, t, &info);
7386	7502	if (err)
7387	7503	verbose_linfo(env, insn_idx, "; ");
7388	7504	if (!err && t == BPF_READ && value_regno >= 0) {
@@ -7390,8 +7506,8 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 1 unmodified line
7390	7506	* PTR_TO_PACKET[_META,_END]. In the latter
7391	7507	* case, we know the offset is zero.
7392	7508	*/
7393		− if (reg_type == SCALAR_VALUE) {
7394		− if (is_retval && get_func_retval_range(env->prog, &range)) {
	7509	+ if (info.reg_type == SCALAR_VALUE) {
	7510	+ if (info.is_retval && get_func_retval_range(env->prog, &range)) {
7395	7511	err = __mark_reg_s32_range(env, regs, value_regno,
7396	7512	range.minval, range.maxval);
7397	7513	if (err)
@@ -7402,7 +7518,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 4 unmodified lines
7402	7518	} else {
7403	7519	mark_reg_known_zero(env, regs,
7404	7520	value_regno);
7405		− if (type_may_be_null(reg_type))
	7521	+ if (type_may_be_null(info.reg_type))
7406	7522	regs[value_regno].id = ++env->id_gen;
7407	7523	/* A load of ctx field could have different
7408	7524	* actual load size with the one encoded in the
@@ -7410,12 +7526,13 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 1 unmodified line
7410	7526	* a sub-register.
7411	7527	*/
7412	7528	regs[value_regno].subreg_def = DEF_NOT_SUBREG;
7413		− if (base_type(reg_type) == PTR_TO_BTF_ID) {
7414		− regs[value_regno].btf = btf;
7415		− regs[value_regno].btf_id = btf_id;
	7529	+ if (base_type(info.reg_type) == PTR_TO_BTF_ID) {
	7530	+ regs[value_regno].btf = info.btf;
	7531	+ regs[value_regno].btf_id = info.btf_id;
	7532	+ regs[value_regno].ref_obj_id = info.ref_obj_id;
7416	7533	}
7417	7534	}
7418		− regs[value_regno].type = reg_type;
	7535	+ regs[value_regno].type = info.reg_type;
7419	7536	}
7420	7537
7421	7538	} else if (reg->type == PTR_TO_STACK) {
@@ -7518,27 +7635,72 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 96 unmodified lines
7518	7635	static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type,
7519	7636	bool allow_trust_mismatch);
7520	7637
7521		−static int check_atomic(struct bpf_verifier_env env, int insn_idx, struct bpf_insn insn)
	7638	+static int check_load_mem(struct bpf_verifier_env env, struct bpf_insn insn,
	7639	+ bool strict_alignment_once, bool is_ldsx,
	7640	+ bool allow_trust_mismatch, const char *ctx)
7522	7641	{
7523		− int load_reg;
	7642	+ struct bpf_reg_state *regs = cur_regs(env);
	7643	+ enum bpf_reg_type src_reg_type;
7524	7644	int err;
7525	7645
7526		− switch (insn->imm) {
7527		− case BPF_ADD:
7528		− case BPF_ADD \| BPF_FETCH:
7529		− case BPF_AND:
7530		− case BPF_AND \| BPF_FETCH:
7531		− case BPF_OR:
7532		− case BPF_OR \| BPF_FETCH:
7533		− case BPF_XOR:
7534		− case BPF_XOR \| BPF_FETCH:
7535		− case BPF_XCHG:
7536		− case BPF_CMPXCHG:
7537		− break;
7538		− default:
7539		− verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n", insn->imm);
7540		− return -EINVAL;
7541		− }
	7646	+ /* check src operand */
	7647	+ err = check_reg_arg(env, insn->src_reg, SRC_OP);
	7648	+ if (err)
	7649	+ return err;
	7650	+
	7651	+ /* check dst operand */
	7652	+ err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
	7653	+ if (err)
	7654	+ return err;
	7655	+
	7656	+ src_reg_type = regs[insn->src_reg].type;
	7657	+
	7658	+ /* Check if (src_reg + off) is readable. The state of dst_reg will be
	7659	+ * updated by this call.
	7660	+ */
	7661	+ err = check_mem_access(env, env->insn_idx, insn->src_reg, insn->off,
	7662	+ BPF_SIZE(insn->code), BPF_READ, insn->dst_reg,
	7663	+ strict_alignment_once, is_ldsx);
	7664	+ err = err ?: save_aux_ptr_type(env, src_reg_type,
	7665	+ allow_trust_mismatch);
	7666	+ err = err ?: reg_bounds_sanity_check(env, &regs[insn->dst_reg], ctx);
	7667	+
	7668	+ return err;
	7669	+}
	7670	+
	7671	+static int check_store_reg(struct bpf_verifier_env env, struct bpf_insn insn,
	7672	+ bool strict_alignment_once)
	7673	+{
	7674	+ struct bpf_reg_state *regs = cur_regs(env);
	7675	+ enum bpf_reg_type dst_reg_type;
	7676	+ int err;
	7677	+
	7678	+ /* check src1 operand */
	7679	+ err = check_reg_arg(env, insn->src_reg, SRC_OP);
	7680	+ if (err)
	7681	+ return err;
	7682	+
	7683	+ /* check src2 operand */
	7684	+ err = check_reg_arg(env, insn->dst_reg, SRC_OP);
	7685	+ if (err)
	7686	+ return err;
	7687	+
	7688	+ dst_reg_type = regs[insn->dst_reg].type;
	7689	+
	7690	+ /* Check if (dst_reg + off) is writeable. */
	7691	+ err = check_mem_access(env, env->insn_idx, insn->dst_reg, insn->off,
	7692	+ BPF_SIZE(insn->code), BPF_WRITE, insn->src_reg,
	7693	+ strict_alignment_once, false);
	7694	+ err = err ?: save_aux_ptr_type(env, dst_reg_type, false);
	7695	+
	7696	+ return err;
	7697	+}
	7698	+
	7699	+static int check_atomic_rmw(struct bpf_verifier_env *env,
	7700	+ struct bpf_insn *insn)
	7701	+{
	7702	+ int load_reg;
	7703	+ int err;
7542	7704
7543	7705	if (BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) {
7544	7706	verbose(env, "invalid atomic operand size\n");
@@ -7574,11 +7736,7 @@ static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_i · 29 unmodified lines
7574	7736	return -EACCES;
7575	7737	}
7576	7738
7577		− if (is_ctx_reg(env, insn->dst_reg) \|\|
7578		− is_pkt_reg(env, insn->dst_reg) \|\|
7579		− is_flow_key_reg(env, insn->dst_reg) \|\|
7580		− is_sk_reg(env, insn->dst_reg) \|\|
7581		− (is_arena_reg(env, insn->dst_reg) && !bpf_jit_supports_insn(insn, true))) {
	7739	+ if (!atomic_ptr_type_ok(env, insn->dst_reg, insn)) {
7582	7740	verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n",
7583	7741	insn->dst_reg,
7584	7742	reg_type_str(env, reg_state(env, insn->dst_reg)->type));
@@ -7605,12 +7763,12 @@ static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_i · 20 unmodified lines
7605	7763	/* Check whether we can read the memory, with second call for fetch
7606	7764	* case to simulate the register fill.
7607	7765	*/
7608		− err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
	7766	+ err = check_mem_access(env, env->insn_idx, insn->dst_reg, insn->off,
7609	7767	BPF_SIZE(insn->code), BPF_READ, -1, true, false);
7610	7768	if (!err && load_reg >= 0)
7611		− err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
7612		− BPF_SIZE(insn->code), BPF_READ, load_reg,
7613		− true, false);
	7769	+ err = check_mem_access(env, env->insn_idx, insn->dst_reg,
	7770	+ insn->off, BPF_SIZE(insn->code),
	7771	+ BPF_READ, load_reg, true, false);
7614	7772	if (err)
7615	7773	return err;
7616	7774
@@ -7620,13 +7778,86 @@ static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_i · 3 unmodified lines
7620	7778	return err;
7621	7779	}
7622	7780	/* Check whether we can write into the same memory. */
7623		− err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
	7781	+ err = check_mem_access(env, env->insn_idx, insn->dst_reg, insn->off,
7624	7782	BPF_SIZE(insn->code), BPF_WRITE, -1, true, false);
7625	7783	if (err)
7626	7784	return err;
7627	7785	return 0;
7628	7786	}
7629	7787
	7788	+static int check_atomic_load(struct bpf_verifier_env *env,
	7789	+ struct bpf_insn *insn)
	7790	+{
	7791	+ int err;
	7792	+
	7793	+ err = check_load_mem(env, insn, true, false, false, "atomic_load");
	7794	+ if (err)
	7795	+ return err;
	7796	+
	7797	+ if (!atomic_ptr_type_ok(env, insn->src_reg, insn)) {
	7798	+ verbose(env, "BPF_ATOMIC loads from R%d %s is not allowed\n",
	7799	+ insn->src_reg,
	7800	+ reg_type_str(env, reg_state(env, insn->src_reg)->type));
	7801	+ return -EACCES;
	7802	+ }
	7803	+
	7804	+ return 0;
	7805	+}
	7806	+
	7807	+static int check_atomic_store(struct bpf_verifier_env *env,
	7808	+ struct bpf_insn *insn)
	7809	+{
	7810	+ int err;
	7811	+
	7812	+ err = check_store_reg(env, insn, true);
	7813	+ if (err)
	7814	+ return err;
	7815	+
	7816	+ if (!atomic_ptr_type_ok(env, insn->dst_reg, insn)) {
	7817	+ verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n",
	7818	+ insn->dst_reg,
	7819	+ reg_type_str(env, reg_state(env, insn->dst_reg)->type));
	7820	+ return -EACCES;
	7821	+ }
	7822	+
	7823	+ return 0;
	7824	+}
	7825	+
	7826	+static int check_atomic(struct bpf_verifier_env env, struct bpf_insn insn)
	7827	+{
	7828	+ switch (insn->imm) {
	7829	+ case BPF_ADD:
	7830	+ case BPF_ADD \| BPF_FETCH:
	7831	+ case BPF_AND:
	7832	+ case BPF_AND \| BPF_FETCH:
	7833	+ case BPF_OR:
	7834	+ case BPF_OR \| BPF_FETCH:
	7835	+ case BPF_XOR:
	7836	+ case BPF_XOR \| BPF_FETCH:
	7837	+ case BPF_XCHG:
	7838	+ case BPF_CMPXCHG:
	7839	+ return check_atomic_rmw(env, insn);
	7840	+ case BPF_LOAD_ACQ:
	7841	+ if (BPF_SIZE(insn->code) == BPF_DW && BITS_PER_LONG != 64) {
	7842	+ verbose(env,
	7843	+ "64-bit load-acquires are only supported on 64-bit arches\n");
	7844	+ return -EOPNOTSUPP;
	7845	+ }
	7846	+ return check_atomic_load(env, insn);
	7847	+ case BPF_STORE_REL:
	7848	+ if (BPF_SIZE(insn->code) == BPF_DW && BITS_PER_LONG != 64) {
	7849	+ verbose(env,
	7850	+ "64-bit store-releases are only supported on 64-bit arches\n");
	7851	+ return -EOPNOTSUPP;
	7852	+ }
	7853	+ return check_atomic_store(env, insn);
	7854	+ default:
	7855	+ verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n",
	7856	+ insn->imm);
	7857	+ return -EINVAL;
	7858	+ }
	7859	+}
	7860	+
7630	7861	/* When register 'regno' is used to read the stack (either directly or through
7631	7862	* a helper function) make sure that it's within stack boundary and, depending
7632	7863	* on the access type and privileges, that all elements of the stack are
@@ -8431,10 +8662,12 @@ static struct bpf_verifier_state find_prev_entry(struct bpf_verifier_env env, · 798 unmodified lines
8431	8662	{
8432	8663	struct bpf_verifier_state_list *sl;
8433	8664	struct bpf_verifier_state *st;
	8665	+ struct list_head pos, head;
8434	8666
8435	8667	/* Explored states are pushed in stack order, most recent states come first */
8436		− sl = *explored_state(env, insn_idx);
8437		− for (; sl; sl = sl->next) {
	8668	+ head = explored_state(env, insn_idx);
	8669	+ list_for_each(pos, head) {
	8670	+ sl = container_of(pos, struct bpf_verifier_state_list, node);
8438	8671	/* If st->branches != 0 state is a part of current DFS verification path,
8439	8672	* hence cur & st for a loop.
8440	8673	*/
@@ -9666,7 +9899,7 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env, · 1225 unmodified lines
9666	9899	if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY)
9667	9900	goto error;
9668	9901	if (env->subprog_cnt > 1 && !allow_tail_call_in_subprogs(env)) {
9669		− verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n");
	9902	+ verbose(env, "mixing of tail_calls and bpf-to-bpf calls is not supported\n");
9670	9903	return -EINVAL;
9671	9904	}
9672	9905	break;
@@ -10237,23 +10470,18 @@ static int check_func_call(struct bpf_verifier_env env, struct bpf_insn insn, · 564 unmodified lines
10237	10470	if (subprog_is_global(env, subprog)) {
10238	10471	const char *sub_name = subprog_name(env, subprog);
10239	10472
10240		− /* Only global subprogs cannot be called with a lock held. */
10241	10473	if (env->cur_state->active_locks) {
10242	10474	verbose(env, "global function calls are not allowed while holding a lock,\n"
10243	10475	"use static function instead\n");
10244	10476	return -EINVAL;
10245	10477	}
10246	10478
10247		− /* Only global subprogs cannot be called with preemption disabled. */
10248		− if (env->cur_state->active_preempt_locks) {
10249		− verbose(env, "global function calls are not allowed with preemption disabled,\n"
10250		− "use static function instead\n");
10251		− return -EINVAL;
10252		− }
10253		−
10254		− if (env->cur_state->active_irq_id) {
10255		− verbose(env, "global function calls are not allowed with IRQs disabled,\n"
10256		− "use static function instead\n");
	10479	+ if (env->subprog_info[subprog].might_sleep &&
	10480	+ (env->cur_state->active_rcu_lock \|\| env->cur_state->active_preempt_locks \|\|
	10481	+ env->cur_state->active_irq_id \|\| !in_sleepable(env))) {
	10482	+ verbose(env, "global functions that may sleep are not allowed in non-sleepable context,\n"
	10483	+ "i.e., in a RCU/IRQ/preempt-disabled section, or in\n"
	10484	+ "a non-sleepable BPF program context\n");
10257	10485	return -EINVAL;
10258	10486	}
10259	10487
@@ -10752,6 +10980,8 @@ record_func_key(struct bpf_verifier_env env, struct bpf_call_arg_meta meta, · 492 unmodified lines
10752	10980	static int check_reference_leak(struct bpf_verifier_env *env, bool exception_exit)
10753	10981	{
10754	10982	struct bpf_verifier_state *state = env->cur_state;
	10983	+ enum bpf_prog_type type = resolve_prog_type(env->prog);
	10984	+ struct bpf_reg_state *reg = reg_state(env, BPF_REG_0);
10755	10985	bool refs_lingering = false;
10756	10986	int i;
10757	10987
@@ -10761,6 +10991,12 @@ static int check_reference_leak(struct bpf_verifier_env *env, bool exception_exi · 3 unmodified lines
10761	10991	for (i = 0; i < state->acquired_refs; i++) {
10762	10992	if (state->refs[i].type != REF_TYPE_PTR)
10763	10993	continue;
	10994	+ /* Allow struct_ops programs to return a referenced kptr back to
	10995	+ * kernel. Type checks are performed later in check_return_code.
	10996	+ */
	10997	+ if (type == BPF_PROG_TYPE_STRUCT_OPS && !exception_exit &&
	10998	+ reg->ref_obj_id == state->refs[i].id)
	10999	+ continue;
10764	11000	verbose(env, "Unreleased reference id=%d alloc_insn=%d\n",
10765	11001	state->refs[i].id, state->refs[i].insn_idx);
10766	11002	refs_lingering = true;
@@ -11781,6 +12017,8 @@ enum special_kfunc_type { · 1014 unmodified lines
11781	12017	KF_bpf_iter_num_new,
11782	12018	KF_bpf_iter_num_next,
11783	12019	KF_bpf_iter_num_destroy,
	12020	+ KF_bpf_set_dentry_xattr,
	12021	+ KF_bpf_remove_dentry_xattr,
11784	12022	};
11785	12023
11786	12024	BTF_SET_START(special_kfunc_set)
@@ -11810,6 +12048,10 @@ BTF_ID(func, bpf_wq_set_callback_impl) · 23 unmodified lines
11810	12048	#ifdef CONFIG_CGROUPS
11811	12049	BTF_ID(func, bpf_iter_css_task_new)
11812	12050	#endif
	12051	+#ifdef CONFIG_BPF_LSM
	12052	+BTF_ID(func, bpf_set_dentry_xattr)
	12053	+BTF_ID(func, bpf_remove_dentry_xattr)
	12054	+#endif
11813	12055	BTF_SET_END(special_kfunc_set)
11814	12056
11815	12057	BTF_ID_LIST(special_kfunc_list)
@@ -11859,6 +12101,13 @@ BTF_ID(func, bpf_local_irq_restore) · 43 unmodified lines
11859	12101	BTF_ID(func, bpf_iter_num_new)
11860	12102	BTF_ID(func, bpf_iter_num_next)
11861	12103	BTF_ID(func, bpf_iter_num_destroy)
	12104	+#ifdef CONFIG_BPF_LSM
	12105	+BTF_ID(func, bpf_set_dentry_xattr)
	12106	+BTF_ID(func, bpf_remove_dentry_xattr)
	12107	+#else
	12108	+BTF_ID_UNUSED
	12109	+BTF_ID_UNUSED
	12110	+#endif
11862	12111
11863	12112	static bool is_kfunc_ret_null(struct bpf_kfunc_call_arg_meta *meta)
11864	12113	{
@@ -16399,13 +16648,14 @@ static int check_return_code(struct bpf_verifier_env *env, int regno, const char · 4534 unmodified lines
16399	16648	const char *exit_ctx = "At program exit";
16400	16649	struct tnum enforce_attach_type_range = tnum_unknown;
16401	16650	const struct bpf_prog *prog = env->prog;
16402		− struct bpf_reg_state *reg;
	16651	+ struct bpf_reg_state *reg = reg_state(env, regno);
16403	16652	struct bpf_retval_range range = retval_range(0, 1);
16404	16653	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
16405	16654	int err;
16406	16655	struct bpf_func_state *frame = env->cur_state->frame[0];
16407	16656	const bool is_subprog = frame->subprogno;
16408	16657	bool return_32bit = false;
	16658	+ const struct btf_type reg_type, ret_type = NULL;
16409	16659
16410	16660	/* LSM and struct_ops func-ptr's return type could be "void" */
16411	16661	if (!is_subprog \|\| frame->in_exception_callback_fn) {
@@ -16414,10 +16664,26 @@ static int check_return_code(struct bpf_verifier_env *env, int regno, const char · 2 unmodified lines
16414	16664	if (prog->expected_attach_type == BPF_LSM_CGROUP)
16415	16665	/* See below, can be 0 or 0-1 depending on hook. */
16416	16666	break;
16417		− fallthrough;
	16667	+ if (!prog->aux->attach_func_proto->type)
	16668	+ return 0;
	16669	+ break;
16418	16670	case BPF_PROG_TYPE_STRUCT_OPS:
16419	16671	if (!prog->aux->attach_func_proto->type)
16420	16672	return 0;
	16673	+
	16674	+ if (frame->in_exception_callback_fn)
	16675	+ break;
	16676	+
	16677	+ /* Allow a struct_ops program to return a referenced kptr if it
	16678	+ * matches the operator's return type and is in its unmodified
	16679	+ * form. A scalar zero (i.e., a null pointer) is also allowed.
	16680	+ */
	16681	+ reg_type = reg->btf ? btf_type_by_id(reg->btf, reg->btf_id) : NULL;
	16682	+ ret_type = btf_type_resolve_ptr(prog->aux->attach_btf,
	16683	+ prog->aux->attach_func_proto->type,
	16684	+ NULL);
	16685	+ if (ret_type && ret_type == reg_type && reg->ref_obj_id)
	16686	+ return __check_ptr_off_reg(env, reg, regno, false);
16421	16687	break;
16422	16688	default:
16423	16689	break;
@@ -16439,8 +16705,6 @@ static int check_return_code(struct bpf_verifier_env *env, int regno, const char · 15 unmodified lines
16439	16705	return -EACCES;
16440	16706	}
16441	16707
16442		− reg = cur_regs(env) + regno;
16443		−
16444	16708	if (frame->in_async_callback_fn) {
16445	16709	/* enforce return zero from async callbacks like timer */
16446	16710	exit_ctx = "At async callback return";
@@ -16539,6 +16803,11 @@ static int check_return_code(struct bpf_verifier_env *env, int regno, const char · 92 unmodified lines
16539	16803	case BPF_PROG_TYPE_NETFILTER:
16540	16804	range = retval_range(NF_DROP, NF_ACCEPT);
16541	16805	break;
	16806	+ case BPF_PROG_TYPE_STRUCT_OPS:
	16807	+ if (!ret_type)
	16808	+ return 0;
	16809	+ range = retval_range(0, 0);
	16810	+ break;
16542	16811	case BPF_PROG_TYPE_EXT:
16543	16812	/* freplace program can return anything as its return value
16544	16813	* depends on the to-be-replaced kernel func or bpf program.
@@ -16582,6 +16851,14 @@ static void mark_subprog_changes_pkt_data(struct bpf_verifier_env *env, int off) · 37 unmodified lines
16582	16851	subprog->changes_pkt_data = true;
16583	16852	}
16584	16853
	16854	+static void mark_subprog_might_sleep(struct bpf_verifier_env *env, int off)
	16855	+{
	16856	+ struct bpf_subprog_info *subprog;
	16857	+
	16858	+ subprog = find_containing_subprog(env, off);
	16859	+ subprog->might_sleep = true;
	16860	+}
	16861	+
16585	16862	/* 't' is an index of a call-site.
16586	16863	* 'w' is a callee entry point.
16587	16864	* Eventually this function would be called when env->cfg.insn_state[w] == EXPLORED.
@@ -16595,6 +16872,7 @@ static void merge_callee_effects(struct bpf_verifier_env *env, int t, int w) · 7 unmodified lines
16595	16872	caller = find_containing_subprog(env, t);
16596	16873	callee = find_containing_subprog(env, w);
16597	16874	caller->changes_pkt_data \|= callee->changes_pkt_data;
	16875	+ caller->might_sleep \|= callee->might_sleep;
16598	16876	}
16599	16877
16600	16878	/* non-recursive DFS pseudo code
@@ -16753,27 +17031,6 @@ static int visit_func_call_insn(int t, struct bpf_insn *insns, · 152 unmodified lines
16753	17031	/* Bitmask with 1s for all caller saved registers */
16754	17032	#define ALL_CALLER_SAVED_REGS ((1u << CALLER_SAVED_REGS) - 1)
16755	17033
16756		−/* Return a bitmask specifying which caller saved registers are
16757		− * clobbered by a call to a helper as if this helper follows
16758		− * bpf_fastcall contract:
16759		− * - includes R0 if function is non-void;
16760		− * - includes R1-R5 if corresponding parameter has is described
16761		− * in the function prototype.
16762		− */
16763		−static u32 helper_fastcall_clobber_mask(const struct bpf_func_proto *fn)
16764		−{
16765		− u32 mask;
16766		− int i;
16767		−
16768		− mask = 0;
16769		− if (fn->ret_type != RET_VOID)
16770		− mask \|= BIT(BPF_REG_0);
16771		− for (i = 0; i < ARRAY_SIZE(fn->arg_type); ++i)
16772		− if (fn->arg_type[i] != ARG_DONTCARE)
16773		− mask \|= BIT(BPF_REG_1 + i);
16774		− return mask;
16775		−}
16776		−
16777	17034	/* True if do_misc_fixups() replaces calls to helper number 'imm',
16778	17035	* replacement patch is presumed to follow bpf_fastcall contract
16779	17036	* (see mark_fastcall_pattern_for_call() below).
@@ -16790,34 +17047,64 @@ static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm) · 10 unmodified lines
16790	17047	}
16791	17048	}
16792	17049
16793		−/* Same as helper_fastcall_clobber_mask() but for kfuncs, see comment above */
16794		−static u32 kfunc_fastcall_clobber_mask(struct bpf_kfunc_call_arg_meta *meta)
	17050	+struct call_summary {
	17051	+ u8 num_params;
	17052	+ bool is_void;
	17053	+ bool fastcall;
	17054	+};
	17055	+
	17056	+/* If @call is a kfunc or helper call, fills @cs and returns true,
	17057	+ * otherwise returns false.
	17058	+ */
	17059	+static bool get_call_summary(struct bpf_verifier_env env, struct bpf_insn call,
	17060	+ struct call_summary *cs)
16795	17061	{
16796		− u32 vlen, i, mask;
	17062	+ struct bpf_kfunc_call_arg_meta meta;
	17063	+ const struct bpf_func_proto *fn;
	17064	+ int i;
16797	17065
16798		− vlen = btf_type_vlen(meta->func_proto);
16799		− mask = 0;
16800		− if (!btf_type_is_void(btf_type_by_id(meta->btf, meta->func_proto->type)))
16801		− mask \|= BIT(BPF_REG_0);
16802		− for (i = 0; i < vlen; ++i)
16803		− mask \|= BIT(BPF_REG_1 + i);
16804		− return mask;
16805		−}
	17066	+ if (bpf_helper_call(call)) {
16806	17067
16807		−/* Same as verifier_inlines_helper_call() but for kfuncs, see comment above */
16808		−static bool is_fastcall_kfunc_call(struct bpf_kfunc_call_arg_meta *meta)
16809		−{
16810		− return meta->kfunc_flags & KF_FASTCALL;
16811		−}
	17068	+ if (get_helper_proto(env, call->imm, &fn) < 0)
	17069	+ /* error would be reported later */
	17070	+ return false;
	17071	+ cs->fastcall = fn->allow_fastcall &&
	17072	+ (verifier_inlines_helper_call(env, call->imm) \|\|
	17073	+ bpf_jit_inlines_helper_call(call->imm));
	17074	+ cs->is_void = fn->ret_type == RET_VOID;
	17075	+ cs->num_params = 0;
	17076	+ for (i = 0; i < ARRAY_SIZE(fn->arg_type); ++i) {
	17077	+ if (fn->arg_type[i] == ARG_DONTCARE)
	17078	+ break;
	17079	+ cs->num_params++;
	17080	+ }
	17081	+ return true;
	17082	+ }
16812	17083
16813		−/* LLVM define a bpf_fastcall function attribute.
16814		− * This attribute means that function scratches only some of
16815		− * the caller saved registers defined by ABI.
16816		− * For BPF the set of such registers could be defined as follows:
16817		− * - R0 is scratched only if function is non-void;
16818		− * - R1-R5 are scratched only if corresponding parameter type is defined
16819		− * in the function prototype.
16820		− *
	17084	+ if (bpf_pseudo_kfunc_call(call)) {
	17085	+ int err;
	17086	+
	17087	+ err = fetch_kfunc_meta(env, call, &meta, NULL);
	17088	+ if (err < 0)
	17089	+ /* error would be reported later */
	17090	+ return false;
	17091	+ cs->num_params = btf_type_vlen(meta.func_proto);
	17092	+ cs->fastcall = meta.kfunc_flags & KF_FASTCALL;
	17093	+ cs->is_void = btf_type_is_void(btf_type_by_id(meta.btf, meta.func_proto->type));
	17094	+ return true;
	17095	+ }
	17096	+
	17097	+ return false;
	17098	+}
	17099	+
	17100	+/* LLVM define a bpf_fastcall function attribute.
	17101	+ * This attribute means that function scratches only some of
	17102	+ * the caller saved registers defined by ABI.
	17103	+ * For BPF the set of such registers could be defined as follows:
	17104	+ * - R0 is scratched only if function is non-void;
	17105	+ * - R1-R5 are scratched only if corresponding parameter type is defined
	17106	+ * in the function prototype.
	17107	+ *
16821	17108	* The contract between kernel and clang allows to simultaneously use
16822	17109	* such functions and maintain backwards compatibility with old
16823	17110	* kernels that don't understand bpf_fastcall calls:
@@ -16890,39 +17177,23 @@ static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env, · 66 unmodified lines
16890	17177	{
16891	17178	struct bpf_insn insns = env->prog->insnsi, stx, *ldx;
16892	17179	struct bpf_insn *call = &env->prog->insnsi[insn_idx];
16893		− const struct bpf_func_proto *fn;
16894		− u32 clobbered_regs_mask = ALL_CALLER_SAVED_REGS;
	17180	+ u32 clobbered_regs_mask;
	17181	+ struct call_summary cs;
16895	17182	u32 expected_regs_mask;
16896		− bool can_be_inlined = false;
16897	17183	s16 off;
16898	17184	int i;
16899	17185
16900		− if (bpf_helper_call(call)) {
16901		− if (get_helper_proto(env, call->imm, &fn) < 0)
16902		− /* error would be reported later */
16903		− return;
16904		− clobbered_regs_mask = helper_fastcall_clobber_mask(fn);
16905		− can_be_inlined = fn->allow_fastcall &&
16906		− (verifier_inlines_helper_call(env, call->imm) \|\|
16907		− bpf_jit_inlines_helper_call(call->imm));
16908		− }
16909		−
16910		− if (bpf_pseudo_kfunc_call(call)) {
16911		− struct bpf_kfunc_call_arg_meta meta;
16912		− int err;
16913		−
16914		− err = fetch_kfunc_meta(env, call, &meta, NULL);
16915		− if (err < 0)
16916		− /* error would be reported later */
16917		− return;
16918		−
16919		− clobbered_regs_mask = kfunc_fastcall_clobber_mask(&meta);
16920		− can_be_inlined = is_fastcall_kfunc_call(&meta);
16921		− }
16922		−
16923		− if (clobbered_regs_mask == ALL_CALLER_SAVED_REGS)
	17186	+ if (!get_call_summary(env, call, &cs))
16924	17187	return;
16925	17188
	17189	+ /* A bitmask specifying which caller saved registers are clobbered
	17190	+ * by a call to a helper/kfunc as if this helper/kfunc follows
	17191	+ * bpf_fastcall contract:
	17192	+ * - includes R0 if function is non-void;
	17193	+ * - includes R1-R5 if corresponding parameter has is described
	17194	+ * in the function prototype.
	17195	+ */
	17196	+ clobbered_regs_mask = GENMASK(cs.num_params, cs.is_void ? 1 : 0);
16926	17197	/* e.g. if helper call clobbers r{0,1}, expect r{2,3,4,5} in the pattern */
16927	17198	expected_regs_mask = ~clobbered_regs_mask & ALL_CALLER_SAVED_REGS;
16928	17199
@@ -16980,7 +17251,7 @@ static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env, · 51 unmodified lines
16980	17251	* don't set 'fastcall_spills_num' for call B so that remove_fastcall_spills_fills()
16981	17252	* does not remove spill/fill pair {4,6}.
16982	17253	*/
16983		− if (can_be_inlined)
	17254	+ if (cs.fastcall)
16984	17255	env->insn_aux_data[insn_idx].fastcall_spills_num = i - 1;
16985	17256	else
16986	17257	subprog->keep_fastcall_stack = 1;
@@ -17062,9 +17333,20 @@ static int visit_insn(int t, struct bpf_verifier_env *env) · 75 unmodified lines
17062	17333	mark_prune_point(env, t);
17063	17334	mark_jmp_point(env, t);
17064	17335	}
17065		− if (bpf_helper_call(insn) && bpf_helper_changes_pkt_data(insn->imm))
17066		− mark_subprog_changes_pkt_data(env, t);
17067		− if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
	17336	+ if (bpf_helper_call(insn)) {
	17337	+ const struct bpf_func_proto *fp;
	17338	+
	17339	+ ret = get_helper_proto(env, insn->imm, &fp);
	17340	+ /* If called in a non-sleepable context program will be
	17341	+ * rejected anyway, so we should end up with precise
	17342	+ * sleepable marks on subprogs, except for dead code
	17343	+ * elimination.
	17344	+ */
	17345	+ if (ret == 0 && fp->might_sleep)
	17346	+ mark_subprog_might_sleep(env, t);
	17347	+ if (bpf_helper_changes_pkt_data(insn->imm))
	17348	+ mark_subprog_changes_pkt_data(env, t);
	17349	+ } else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
17068	17350	struct bpf_kfunc_call_arg_meta meta;
17069	17351
17070	17352	ret = fetch_kfunc_meta(env, insn, &meta, NULL);
@@ -17083,6 +17365,13 @@ static int visit_insn(int t, struct bpf_verifier_env *env) · 12 unmodified lines
17083	17365	*/
17084	17366	mark_force_checkpoint(env, t);
17085	17367	}
	17368	+ /* Same as helpers, if called in a non-sleepable context
	17369	+ * program will be rejected anyway, so we should end up
	17370	+ * with precise sleepable marks on subprogs, except for
	17371	+ * dead code elimination.
	17372	+ */
	17373	+ if (ret == 0 && is_kfunc_sleepable(&meta))
	17374	+ mark_subprog_might_sleep(env, t);
17086	17375	}
17087	17376	return visit_func_call_insn(t, insns, env, insn->src_reg == BPF_PSEUDO_CALL);
17088	17377
@@ -17125,9 +17414,8 @@ static int visit_insn(int t, struct bpf_verifier_env *env) · 36 unmodified lines
17125	17414	static int check_cfg(struct bpf_verifier_env *env)
17126	17415	{
17127	17416	int insn_cnt = env->prog->len;
17128		− int insn_stack, insn_state;
	17417	+ int insn_stack, insn_state, *insn_postorder;
17129	17418	int ex_insn_beg, i, ret = 0;
17130		− bool ex_done = false;
17131	17419
17132	17420	insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
17133	17421	if (!insn_state)
@@ -17139,6 +17427,17 @@ static int check_cfg(struct bpf_verifier_env *env) · 5 unmodified lines
17139	17427	return -ENOMEM;
17140	17428	}
17141	17429
	17430	+ insn_postorder = env->cfg.insn_postorder = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
	17431	+ if (!insn_postorder) {
	17432	+ kvfree(insn_state);
	17433	+ kvfree(insn_stack);
	17434	+ return -ENOMEM;
	17435	+ }
	17436	+
	17437	+ ex_insn_beg = env->exception_callback_subprog
	17438	+ ? env->subprog_info[env->exception_callback_subprog].start
	17439	+ : 0;
	17440	+
17142	17441	insn_state[0] = DISCOVERED; /* mark 1st insn as discovered */
17143	17442	insn_stack[0] = 0; /* 0 is the first instruction */
17144	17443	env->cfg.cur_stack = 1;
@@ -17152,6 +17451,7 @@ walk_cfg: · 7 unmodified lines
17152	17451	case DONE_EXPLORING:
17153	17452	insn_state[t] = EXPLORED;
17154	17453	env->cfg.cur_stack--;
	17454	+ insn_postorder[env->cfg.cur_postorder++] = t;
17155	17455	break;
17156	17456	case KEEP_EXPLORING:
17157	17457	break;
@@ -17170,13 +17470,10 @@ walk_cfg: · 12 unmodified lines
17170	17470	goto err_free;
17171	17471	}
17172	17472
17173		− if (env->exception_callback_subprog && !ex_done) {
17174		− ex_insn_beg = env->subprog_info[env->exception_callback_subprog].start;
17175		−
	17473	+ if (ex_insn_beg && insn_state[ex_insn_beg] != EXPLORED) {
17176	17474	insn_state[ex_insn_beg] = DISCOVERED;
17177	17475	insn_stack[0] = ex_insn_beg;
17178	17476	env->cfg.cur_stack = 1;
17179		− ex_done = true;
17180	17477	goto walk_cfg;
17181	17478	}
17182	17479
@@ -17199,6 +17496,7 @@ walk_cfg: · 16 unmodified lines
17199	17496	}
17200	17497	ret = 0; /* cfg looks good */
17201	17498	env->prog->aux->changes_pkt_data = env->subprog_info[0].changes_pkt_data;
	17499	+ env->prog->aux->might_sleep = env->subprog_info[0].might_sleep;
17202	17500
17203	17501	err_free:
17204	17502	kvfree(insn_state);
@@ -17815,18 +18113,22 @@ static void clean_verifier_state(struct bpf_verifier_env *env, · 610 unmodified lines
17815	18113	static void clean_live_states(struct bpf_verifier_env *env, int insn,
17816	18114	struct bpf_verifier_state *cur)
17817	18115	{
	18116	+ struct bpf_verifier_state *loop_entry;
17818	18117	struct bpf_verifier_state_list *sl;
	18118	+ struct list_head pos, head;
17819	18119
17820		− sl = *explored_state(env, insn);
17821		− while (sl) {
	18120	+ head = explored_state(env, insn);
	18121	+ list_for_each(pos, head) {
	18122	+ sl = container_of(pos, struct bpf_verifier_state_list, node);
17822	18123	if (sl->state.branches)
17823		− goto next;
	18124	+ continue;
	18125	+ loop_entry = get_loop_entry(env, &sl->state);
	18126	+ if (!IS_ERR_OR_NULL(loop_entry) && loop_entry->branches)
	18127	+ continue;
17824	18128	if (sl->state.insn_idx != insn \|\|
17825	18129	!same_callsites(&sl->state, cur))
17826		− goto next;
	18130	+ continue;
17827	18131	clean_verifier_state(env, &sl->state);
17828		−next:
17829		− sl = sl->next;
17830	18132	}
17831	18133	}
17832	18134
@@ -18210,15 +18512,17 @@ static bool refsafe(struct bpf_verifier_state old, struct bpf_verifier_state c · 377 unmodified lines
18210	18512	* the current state will reach 'bpf_exit' instruction safely
18211	18513	*/
18212	18514	static bool func_states_equal(struct bpf_verifier_env env, struct bpf_func_state old,
18213		− struct bpf_func_state *cur, enum exact_level exact)
	18515	+ struct bpf_func_state *cur, u32 insn_idx, enum exact_level exact)
18214	18516	{
18215		− int i;
	18517	+ u16 live_regs = env->insn_aux_data[insn_idx].live_regs_before;
	18518	+ u16 i;
18216	18519
18217	18520	if (old->callback_depth > cur->callback_depth)
18218	18521	return false;
18219	18522
18220	18523	for (i = 0; i < MAX_BPF_REG; i++)
18221		− if (!regsafe(env, &old->regs[i], &cur->regs[i],
	18524	+ if (((1 << i) & live_regs) &&
	18525	+ !regsafe(env, &old->regs[i], &cur->regs[i],
18222	18526	&env->idmap_scratch, exact))
18223	18527	return false;
18224	18528
@@ -18239,6 +18543,7 @@ static bool states_equal(struct bpf_verifier_env *env, · 14 unmodified lines
18239	18543	struct bpf_verifier_state *cur,
18240	18544	enum exact_level exact)
18241	18545	{
	18546	+ u32 insn_idx;
18242	18547	int i;
18243	18548
18244	18549	if (old->curframe != cur->curframe)
@@ -18262,9 +18567,12 @@ static bool states_equal(struct bpf_verifier_env *env, · 17 unmodified lines
18262	18567	* and all frame states need to be equivalent
18263	18568	*/
18264	18569	for (i = 0; i <= old->curframe; i++) {
	18570	+ insn_idx = i == old->curframe
	18571	+ ? env->insn_idx
	18572	+ : old->frame[i + 1]->callsite;
18265	18573	if (old->frame[i]->callsite != cur->frame[i]->callsite)
18266	18574	return false;
18267		− if (!func_states_equal(env, old->frame[i], cur->frame[i], exact))
	18575	+ if (!func_states_equal(env, old->frame[i], cur->frame[i], insn_idx, exact))
18268	18576	return false;
18269	18577	}
18270	18578	return true;
@@ -18517,10 +18825,11 @@ static bool iter_active_depths_differ(struct bpf_verifier_state *old, struct bpf · 246 unmodified lines
18517	18825	static int is_state_visited(struct bpf_verifier_env *env, int insn_idx)
18518	18826	{
18519	18827	struct bpf_verifier_state_list *new_sl;
18520		− struct bpf_verifier_state_list sl, *pprev;
	18828	+ struct bpf_verifier_state_list *sl;
18521	18829	struct bpf_verifier_state cur = env->cur_state, new, *loop_entry;
18522	18830	int i, j, n, err, states_cnt = 0;
18523	18831	bool force_new_state, add_new_state, force_exact;
	18832	+ struct list_head pos, tmp, *head;
18524	18833
18525	18834	force_new_state = env->test_state_freq \|\| is_force_checkpoint(env, insn_idx) \|\|
18526	18835	/* Avoid accumulating infinitely long jmp history */
@@ -18539,15 +18848,14 @@ static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) · 12 unmodified lines
18539	18848	env->insn_processed - env->prev_insn_processed >= 8)
18540	18849	add_new_state = true;
18541	18850
18542		− pprev = explored_state(env, insn_idx);
18543		− sl = *pprev;
18544		−
18545	18851	clean_live_states(env, insn_idx, cur);
18546	18852
18547		− while (sl) {
	18853	+ head = explored_state(env, insn_idx);
	18854	+ list_for_each_safe(pos, tmp, head) {
	18855	+ sl = container_of(pos, struct bpf_verifier_state_list, node);
18548	18856	states_cnt++;
18549	18857	if (sl->state.insn_idx != insn_idx)
18550		− goto next;
	18858	+ continue;
18551	18859
18552	18860	if (sl->state.branches) {
18553	18861	struct bpf_func_state *frame = sl->state.frame[sl->state.curframe];
@@ -18621,7 +18929,7 @@ static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) · 67 unmodified lines
18621	18929	spi = __get_spi(iter_reg->off + iter_reg->var_off.value);
18622	18930	iter_state = &func(env, iter_reg)->stack[spi].spilled_ptr;
18623	18931	if (iter_state->iter.state == BPF_ITER_STATE_ACTIVE) {
18624		− update_loop_entry(cur, &sl->state);
	18932	+ update_loop_entry(env, cur, &sl->state);
18625	18933	goto hit;
18626	18934	}
18627	18935	}
@@ -18630,7 +18938,7 @@ static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) · 2 unmodified lines
18630	18938	if (is_may_goto_insn_at(env, insn_idx)) {
18631	18939	if (sl->state.may_goto_depth != cur->may_goto_depth &&
18632	18940	states_equal(env, &sl->state, cur, RANGE_WITHIN)) {
18633		− update_loop_entry(cur, &sl->state);
	18941	+ update_loop_entry(env, cur, &sl->state);
18634	18942	goto hit;
18635	18943	}
18636	18944	}
@@ -18697,11 +19005,13 @@ skip_inf_loop_check: · 60 unmodified lines
18697	19005	*
18698	19006	* Additional details are in the comment before get_loop_entry().
18699	19007	*/
18700		− loop_entry = get_loop_entry(&sl->state);
	19008	+ loop_entry = get_loop_entry(env, &sl->state);
	19009	+ if (IS_ERR(loop_entry))
	19010	+ return PTR_ERR(loop_entry);
18701	19011	force_exact = loop_entry && loop_entry->branches > 0;
18702	19012	if (states_equal(env, &sl->state, cur, force_exact ? RANGE_WITHIN : NOT_EXACT)) {
18703	19013	if (force_exact)
18704		− update_loop_entry(cur, loop_entry);
	19014	+ update_loop_entry(env, cur, loop_entry);
18705	19015	hit:
18706	19016	sl->hit_cnt++;
18707	19017	/* reached equivalent register/stack state,
@@ -18750,31 +19060,13 @@ miss: · 42 unmodified lines
18750	19060	/* the state is unlikely to be useful. Remove it to
18751	19061	* speed up verification
18752	19062	*/
18753		− *pprev = sl->next;
18754		− if (sl->state.frame[0]->regs[0].live & REG_LIVE_DONE &&
18755		− !sl->state.used_as_loop_entry) {
18756		− u32 br = sl->state.branches;
18757		−
18758		− WARN_ONCE(br,
18759		− "BUG live_done but branches_to_explore %d\n",
18760		− br);
18761		− free_verifier_state(&sl->state, false);
18762		− kfree(sl);
18763		− env->peak_states--;
18764		− } else {
18765		− /* cannot free this state, since parentage chain may
18766		− * walk it later. Add it for free_list instead to
18767		− * be freed at the end of verification
18768		− */
18769		− sl->next = env->free_list;
18770		− env->free_list = sl;
18771		− }
18772		− sl = *pprev;
18773		− continue;
	19063	+ sl->in_free_list = true;
	19064	+ list_del(&sl->node);
	19065	+ list_add(&sl->node, &env->free_list);
	19066	+ env->free_list_size++;
	19067	+ env->explored_states_size--;
	19068	+ maybe_free_verifier_state(env, sl);
18774	19069	}
18775		−next:
18776		− pprev = &sl->next;
18777		− sl = *pprev;
18778	19070	}
18779	19071
18780	19072	if (env->max_states_per_insn < states_cnt)
@@ -18799,7 +19091,8 @@ next: · 18 unmodified lines
18799	19091	if (!new_sl)
18800	19092	return -ENOMEM;
18801	19093	env->total_states++;
18802		− env->peak_states++;
	19094	+ env->explored_states_size++;
	19095	+ update_peak_states(env);
18803	19096	env->prev_jmps_processed = env->jmps_processed;
18804	19097	env->prev_insn_processed = env->insn_processed;
18805	19098
@@ -18823,8 +19116,8 @@ next: · 17 unmodified lines
18823	19116	cur->first_insn_idx = insn_idx;
18824	19117	cur->insn_hist_start = cur->insn_hist_end;
18825	19118	cur->dfs_depth = new->dfs_depth + 1;
18826		− new_sl->next = *explored_state(env, insn_idx);
18827		− *explored_state(env, insn_idx) = new_sl;
	19119	+ list_add(&new_sl->node, head);
	19120	+
18828	19121	/* connect new state to parentage chain. Current frame needs all
18829	19122	* registers connected. Only r6 - r9 of the callers are alive (pushed
18830	19123	* to the stack implicitly by JITs) so in callers' frames connect just
@@ -19011,19 +19304,13 @@ static int do_check(struct bpf_verifier_env *env) · 180 unmodified lines
19011	19304	}
19012	19305
19013	19306	if (env->log.level & BPF_LOG_LEVEL) {
19014		− const struct bpf_insn_cbs cbs = {
19015		− .cb_call = disasm_kfunc_name,
19016		− .cb_print = verbose,
19017		− .private_data = env,
19018		− };
19019		−
19020	19307	if (verifier_state_scratched(env))
19021	19308	print_insn_state(env, state, state->curframe);
19022	19309
19023	19310	verbose_linfo(env, env->insn_idx, "; ");
19024	19311	env->prev_log_pos = env->log.end_pos;
19025	19312	verbose(env, "%d: ", env->insn_idx);
19026		− print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
	19313	+ verbose_insn(env, insn);
19027	19314	env->prev_insn_print_pos = env->log.end_pos - env->prev_log_pos;
19028	19315	env->prev_log_pos = env->log.end_pos;
19029	19316	}
@@ -19045,37 +19332,18 @@ static int do_check(struct bpf_verifier_env *env) · 15 unmodified lines
19045	19332	return err;
19046	19333
19047	19334	} else if (class == BPF_LDX) {
19048		− enum bpf_reg_type src_reg_type;
19049		−
19050		− /* check for reserved fields is already done */
19051		−
19052		− /* check src operand */
19053		− err = check_reg_arg(env, insn->src_reg, SRC_OP);
19054		− if (err)
19055		− return err;
19056		−
19057		− err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
19058		− if (err)
19059		− return err;
19060		−
19061		− src_reg_type = regs[insn->src_reg].type;
	19335	+ bool is_ldsx = BPF_MODE(insn->code) == BPF_MEMSX;
19062	19336
19063		− /* check that memory (src_reg + off) is readable,
19064		− * the state of dst_reg will be updated by this func
	19337	+ /* Check for reserved fields is already done in
	19338	+ * resolve_pseudo_ldimm64().
19065	19339	*/
19066		− err = check_mem_access(env, env->insn_idx, insn->src_reg,
19067		− insn->off, BPF_SIZE(insn->code),
19068		− BPF_READ, insn->dst_reg, false,
19069		− BPF_MODE(insn->code) == BPF_MEMSX);
19070		− err = err ?: save_aux_ptr_type(env, src_reg_type, true);
19071		− err = err ?: reg_bounds_sanity_check(env, &regs[insn->dst_reg], "ldx");
	19340	+ err = check_load_mem(env, insn, false, is_ldsx, true,
	19341	+ "ldx");
19072	19342	if (err)
19073	19343	return err;
19074	19344	} else if (class == BPF_STX) {
19075		− enum bpf_reg_type dst_reg_type;
19076		−
19077	19345	if (BPF_MODE(insn->code) == BPF_ATOMIC) {
19078		− err = check_atomic(env, env->insn_idx, insn);
	19346	+ err = check_atomic(env, insn);
19079	19347	if (err)
19080	19348	return err;
19081	19349	env->insn_idx++;
@@ -19087,25 +19355,7 @@ static int do_check(struct bpf_verifier_env *env) · 5 unmodified lines
19087	19355	return -EINVAL;
19088	19356	}
19089	19357
19090		− /* check src1 operand */
19091		− err = check_reg_arg(env, insn->src_reg, SRC_OP);
19092		− if (err)
19093		− return err;
19094		− /* check src2 operand */
19095		− err = check_reg_arg(env, insn->dst_reg, SRC_OP);
19096		− if (err)
19097		− return err;
19098		−
19099		− dst_reg_type = regs[insn->dst_reg].type;
19100		−
19101		− /* check that memory (dst_reg + off) is writeable */
19102		− err = check_mem_access(env, env->insn_idx, insn->dst_reg,
19103		− insn->off, BPF_SIZE(insn->code),
19104		− BPF_WRITE, insn->src_reg, false, false);
19105		− if (err)
19106		− return err;
19107		−
19108		− err = save_aux_ptr_type(env, dst_reg_type, false);
	19358	+ err = check_store_reg(env, insn, false);
19109	19359	if (err)
19110	19360	return err;
19111	19361	} else if (class == BPF_ST) {
@@ -19245,6 +19495,10 @@ process_bpf_exit: · 133 unmodified lines
19245	19495	return err;
19246	19496	break;
19247	19497	} else {
	19498	+ if (WARN_ON_ONCE(env->cur_state->loop_entry)) {
	19499	+ verbose(env, "verifier bug: env->cur_state->loop_entry != NULL\n");
	19500	+ return -EFAULT;
	19501	+ }
19248	19502	do_print_state = true;
19249	19503	continue;
19250	19504	}
@@ -20334,7 +20588,7 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env) · 1083 unmodified lines
20334	20588	{
20335	20589	struct bpf_subprog_info *subprogs = env->subprog_info;
20336	20590	const struct bpf_verifier_ops *ops = env->ops;
20337		− int i, cnt, size, ctx_field_size, delta = 0, epilogue_cnt = 0;
	20591	+ int i, cnt, size, ctx_field_size, ret, delta = 0, epilogue_cnt = 0;
20338	20592	const int insn_cnt = env->prog->len;
20339	20593	struct bpf_insn *epilogue_buf = env->epilogue_buf;
20340	20594	struct bpf_insn *insn_buf = env->insn_buf;
@@ -20363,6 +20617,10 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env) · 22 unmodified lines
20363	20617	return -ENOMEM;
20364	20618	env->prog = new_prog;
20365	20619	delta += cnt - 1;
	20620	+
	20621	+ ret = add_kfunc_in_insns(env, epilogue_buf, epilogue_cnt - 1);
	20622	+ if (ret < 0)
	20623	+ return ret;
20366	20624	}
20367	20625	}
20368	20626
@@ -20383,6 +20641,10 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env) · 14 unmodified lines
20383	20641
20384	20642	env->prog = new_prog;
20385	20643	delta += cnt - 1;
	20644	+
	20645	+ ret = add_kfunc_in_insns(env, insn_buf, cnt - 1);
	20646	+ if (ret < 0)
	20647	+ return ret;
20386	20648	}
20387	20649	}
20388	20650
@@ -20415,7 +20677,9 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env) · 26 unmodified lines
20415	20677	insn->code == (BPF_ST \| BPF_MEM \| BPF_W) \|\|
20416	20678	insn->code == (BPF_ST \| BPF_MEM \| BPF_DW)) {
20417	20679	type = BPF_WRITE;
20418		− } else if ((insn->code == (BPF_STX \| BPF_ATOMIC \| BPF_W) \|\|
	20680	+ } else if ((insn->code == (BPF_STX \| BPF_ATOMIC \| BPF_B) \|\|
	20681	+ insn->code == (BPF_STX \| BPF_ATOMIC \| BPF_H) \|\|
	20682	+ insn->code == (BPF_STX \| BPF_ATOMIC \| BPF_W) \|\|
20419	20683	insn->code == (BPF_STX \| BPF_ATOMIC \| BPF_DW)) &&
20420	20684	env->insn_aux_data[i + delta].ptr_type == PTR_TO_ARENA) {
20421	20685	insn->code = BPF_STX \| BPF_PROBE_ATOMIC \| BPF_SIZE(insn->code);
@@ -20723,6 +20987,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 301 unmodified lines
20723	20987	func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable;
20724	20988	func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb;
20725	20989	func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data;
	20990	+ func[i]->aux->might_sleep = env->subprog_info[i].might_sleep;
20726	20991	if (!i)
20727	20992	func[i]->aux->exception_boundary = env->seen_exception;
20728	20993	func[i] = bpf_int_jit_compile(func[i]);
@@ -20939,6 +21204,14 @@ static void specialize_kfunc(struct bpf_verifier_env *env, · 210 unmodified lines
20939	21204	*/
20940	21205	env->seen_direct_write = seen_direct_write;
20941	21206	}
	21207	+
	21208	+ if (func_id == special_kfunc_list[KF_bpf_set_dentry_xattr] &&
	21209	+ bpf_lsm_has_d_inode_locked(prog))
	21210	+ *addr = (unsigned long)bpf_set_dentry_xattr_locked;
	21211	+
	21212	+ if (func_id == special_kfunc_list[KF_bpf_remove_dentry_xattr] &&
	21213	+ bpf_lsm_has_d_inode_locked(prog))
	21214	+ *addr = (unsigned long)bpf_remove_dentry_xattr_locked;
20942	21215	}
20943	21216
20944	21217	static void __fixup_collection_insert_kfunc(struct bpf_insn_aux_data *insn_aux,
@@ -21373,7 +21646,50 @@ static int do_misc_fixups(struct bpf_verifier_env *env) · 428 unmodified lines
21373	21646	goto next_insn;
21374	21647	}
21375	21648
21376		− if (is_may_goto_insn(insn)) {
	21649	+ if (is_may_goto_insn(insn) && bpf_jit_supports_timed_may_goto()) {
	21650	+ int stack_off_cnt = -stack_depth - 16;
	21651	+
	21652	+ /*
	21653	+ * Two 8 byte slots, depth-16 stores the count, and
	21654	+ * depth-8 stores the start timestamp of the loop.
	21655	+ *
	21656	+ * The starting value of count is BPF_MAX_TIMED_LOOPS
	21657	+ * (0xffff). Every iteration loads it and subs it by 1,
	21658	+ * until the value becomes 0 in AX (thus, 1 in stack),
	21659	+ * after which we call arch_bpf_timed_may_goto, which
	21660	+ * either sets AX to 0xffff to keep looping, or to 0
	21661	+ * upon timeout. AX is then stored into the stack. In
	21662	+ * the next iteration, we either see 0 and break out, or
	21663	+ * continue iterating until the next time value is 0
	21664	+ * after subtraction, rinse and repeat.
	21665	+ */
	21666	+ stack_depth_extra = 16;
	21667	+ insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off_cnt);
	21668	+ if (insn->off >= 0)
	21669	+ insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 5);
	21670	+ else
	21671	+ insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off - 1);
	21672	+ insn_buf[2] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1);
	21673	+ insn_buf[3] = BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 2);
	21674	+ /*
	21675	+ * AX is used as an argument to pass in stack_off_cnt
	21676	+ * (to add to r10/fp), and also as the return value of
	21677	+ * the call to arch_bpf_timed_may_goto.
	21678	+ */
	21679	+ insn_buf[4] = BPF_MOV64_IMM(BPF_REG_AX, stack_off_cnt);
	21680	+ insn_buf[5] = BPF_EMIT_CALL(arch_bpf_timed_may_goto);
	21681	+ insn_buf[6] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off_cnt);
	21682	+ cnt = 7;
	21683	+
	21684	+ new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
	21685	+ if (!new_prog)
	21686	+ return -ENOMEM;
	21687	+
	21688	+ delta += cnt - 1;
	21689	+ env->prog = prog = new_prog;
	21690	+ insn = new_prog->insnsi + i + delta;
	21691	+ goto next_insn;
	21692	+ } else if (is_may_goto_insn(insn)) {
21377	21693	int stack_off = -stack_depth - 8;
21378	21694
21379	21695	stack_depth_extra = 8;
@@ -21897,6 +22213,13 @@ next_insn: · 517 unmodified lines
21897	22213	if (subprogs[cur_subprog + 1].start == i + delta + 1) {
21898	22214	subprogs[cur_subprog].stack_depth += stack_depth_extra;
21899	22215	subprogs[cur_subprog].stack_extra = stack_depth_extra;
	22216	+
	22217	+ stack_depth = subprogs[cur_subprog].stack_depth;
	22218	+ if (stack_depth > MAX_BPF_STACK && !prog->jit_requested) {
	22219	+ verbose(env, "stack size %d(extra %d) is too large\n",
	22220	+ stack_depth, stack_depth_extra);
	22221	+ return -EINVAL;
	22222	+ }
21900	22223	cur_subprog++;
21901	22224	stack_depth = subprogs[cur_subprog].stack_depth;
21902	22225	stack_depth_extra = 0;
@@ -21907,23 +22230,33 @@ next_insn: · 4 unmodified lines
21907	22230
21908	22231	env->prog->aux->stack_depth = subprogs[0].stack_depth;
21909	22232	for (i = 0; i < env->subprog_cnt; i++) {
	22233	+ int delta = bpf_jit_supports_timed_may_goto() ? 2 : 1;
21910	22234	int subprog_start = subprogs[i].start;
21911	22235	int stack_slots = subprogs[i].stack_extra / 8;
	22236	+ int slots = delta, cnt = 0;
21912	22237
21913	22238	if (!stack_slots)
21914	22239	continue;
21915		− if (stack_slots > 1) {
	22240	+ /* We need two slots in case timed may_goto is supported. */
	22241	+ if (stack_slots > slots) {
21916	22242	verbose(env, "verifier bug: stack_slots supports may_goto only\n");
21917	22243	return -EFAULT;
21918	22244	}
21919	22245
21920		− /* Add ST insn to subprog prologue to init extra stack */
21921		− insn_buf[0] = BPF_ST_MEM(BPF_DW, BPF_REG_FP,
21922		− -subprogs[i].stack_depth, BPF_MAX_LOOPS);
	22246	+ stack_depth = subprogs[i].stack_depth;
	22247	+ if (bpf_jit_supports_timed_may_goto()) {
	22248	+ insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth,
	22249	+ BPF_MAX_TIMED_LOOPS);
	22250	+ insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth + 8, 0);
	22251	+ } else {
	22252	+ /* Add ST insn to subprog prologue to init extra stack */
	22253	+ insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth,
	22254	+ BPF_MAX_LOOPS);
	22255	+ }
21923	22256	/* Copy first actual insn to preserve it */
21924		− insn_buf[1] = env->prog->insnsi[subprog_start];
	22257	+ insn_buf[cnt++] = env->prog->insnsi[subprog_start];
21925	22258
21926		− new_prog = bpf_patch_insn_data(env, subprog_start, insn_buf, 2);
	22259	+ new_prog = bpf_patch_insn_data(env, subprog_start, insn_buf, cnt);
21927	22260	if (!new_prog)
21928	22261	return -ENOMEM;
21929	22262	env->prog = prog = new_prog;
@@ -21933,7 +22266,7 @@ next_insn: · 3 unmodified lines
21933	22266	* to insn after BPF_ST that inits may_goto count.
21934	22267	* Adjustment will succeed because bpf_patch_insn_data() didn't fail.
21935	22268	*/
21936		− WARN_ON(adjust_jmp_off(env->prog, subprog_start, 1));
	22269	+ WARN_ON(adjust_jmp_off(env->prog, subprog_start, delta));
21937	22270	}
21938	22271
21939	22272	/* Since poke tab is now finalized, publish aux to tracker. */
@@ -22131,31 +22464,29 @@ static int remove_fastcall_spills_fills(struct bpf_verifier_env *env) · 191 unmodified lines
22131	22464
22132	22465	static void free_states(struct bpf_verifier_env *env)
22133	22466	{
22134		− struct bpf_verifier_state_list sl, sln;
	22467	+ struct bpf_verifier_state_list *sl;
	22468	+ struct list_head head, pos, *tmp;
22135	22469	int i;
22136	22470
22137		− sl = env->free_list;
22138		− while (sl) {
22139		− sln = sl->next;
	22471	+ list_for_each_safe(pos, tmp, &env->free_list) {
	22472	+ sl = container_of(pos, struct bpf_verifier_state_list, node);
22140	22473	free_verifier_state(&sl->state, false);
22141	22474	kfree(sl);
22142		− sl = sln;
22143	22475	}
22144		− env->free_list = NULL;
	22476	+ INIT_LIST_HEAD(&env->free_list);
22145	22477
22146	22478	if (!env->explored_states)
22147	22479	return;
22148	22480
22149	22481	for (i = 0; i < state_htab_size(env); i++) {
22150		− sl = env->explored_states[i];
	22482	+ head = &env->explored_states[i];
22151	22483
22152		− while (sl) {
22153		− sln = sl->next;
	22484	+ list_for_each_safe(pos, tmp, head) {
	22485	+ sl = container_of(pos, struct bpf_verifier_state_list, node);
22154	22486	free_verifier_state(&sl->state, false);
22155	22487	kfree(sl);
22156		− sl = sln;
22157	22488	}
22158		− env->explored_states[i] = NULL;
	22489	+ INIT_LIST_HEAD(&env->explored_states[i]);
22159	22490	}
22160	22491	}
22161	22492
@@ -22163,6 +22494,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) · 1 unmodified line
22163	22494	{
22164	22495	bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);
22165	22496	struct bpf_subprog_info *sub = subprog_info(env, subprog);
	22497	+ struct bpf_prog_aux *aux = env->prog->aux;
22166	22498	struct bpf_verifier_state *state;
22167	22499	struct bpf_reg_state *regs;
22168	22500	int ret, i;
@@ -22270,6 +22602,13 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) · 101 unmodified lines
22270	22602	mark_reg_known_zero(env, regs, BPF_REG_1);
22271	22603	}
22272	22604
	22605	+ /* Acquire references for struct_ops program arguments tagged with "__ref" */
	22606	+ if (!subprog && env->prog->type == BPF_PROG_TYPE_STRUCT_OPS) {
	22607	+ for (i = 0; i < aux->ctx_arg_info_size; i++)
	22608	+ aux->ctx_arg_info[i].ref_obj_id = aux->ctx_arg_info[i].refcounted ?
	22609	+ acquire_reference(env, 0) : 0;
	22610	+ }
	22611	+
22273	22612	ret = do_check(env);
22274	22613	out:
22275	22614	/* check for NULL is necessary, since cur_state can be freed inside
@@ -22392,6 +22731,15 @@ static void print_verification_stats(struct bpf_verifier_env *env) · 116 unmodified lines
22392	22731	env->peak_states, env->longest_mark_read_walk);
22393	22732	}
22394	22733
	22734	+int bpf_prog_ctx_arg_info_init(struct bpf_prog *prog,
	22735	+ const struct bpf_ctx_arg_aux *info, u32 cnt)
	22736	+{
	22737	+ prog->aux->ctx_arg_info = kmemdup_array(info, cnt, sizeof(*info), GFP_KERNEL);
	22738	+ prog->aux->ctx_arg_info_size = cnt;
	22739	+
	22740	+ return prog->aux->ctx_arg_info ? 0 : -ENOMEM;
	22741	+}
	22742	+
22395	22743	static int check_struct_ops_btf_id(struct bpf_verifier_env *env)
22396	22744	{
22397	22745	const struct btf_type t, func_proto;
@@ -22399,10 +22747,11 @@ static int check_struct_ops_btf_id(struct bpf_verifier_env *env) · 1 unmodified line
22399	22747	const struct bpf_struct_ops *st_ops;
22400	22748	const struct btf_member *member;
22401	22749	struct bpf_prog *prog = env->prog;
22402		− u32 btf_id, member_idx;
	22750	+ bool has_refcounted_arg = false;
	22751	+ u32 btf_id, member_idx, member_off;
22403	22752	struct btf *btf;
22404	22753	const char *mname;
22405		− int err;
	22754	+ int i, err;
22406	22755
22407	22756	if (!prog->gpl_compatible) {
22408	22757	verbose(env, "struct ops programs must have a GPL compatible license\n");
@@ -22450,7 +22799,8 @@ static int check_struct_ops_btf_id(struct bpf_verifier_env *env) · 41 unmodified lines
22450	22799	return -EINVAL;
22451	22800	}
22452	22801
22453		− err = bpf_struct_ops_supported(st_ops, __btf_member_bit_offset(t, member) / 8);
	22802	+ member_off = __btf_member_bit_offset(t, member) / 8;
	22803	+ err = bpf_struct_ops_supported(st_ops, member_off);
22454	22804	if (err) {
22455	22805	verbose(env, "attach to unsupported member %s of struct %s\n",
22456	22806	mname, st_ops->name);
@@ -22472,17 +22822,32 @@ static int check_struct_ops_btf_id(struct bpf_verifier_env *env) · 15 unmodified lines
22472	22822	return -EACCES;
22473	22823	}
22474	22824
22475		− /* btf_ctx_access() used this to provide argument type info */
22476		− prog->aux->ctx_arg_info =
22477		− st_ops_desc->arg_info[member_idx].info;
22478		− prog->aux->ctx_arg_info_size =
22479		− st_ops_desc->arg_info[member_idx].cnt;
	22825	+ for (i = 0; i < st_ops_desc->arg_info[member_idx].cnt; i++) {
	22826	+ if (st_ops_desc->arg_info[member_idx].info->refcounted) {
	22827	+ has_refcounted_arg = true;
	22828	+ break;
	22829	+ }
	22830	+ }
	22831	+
	22832	+ /* Tail call is not allowed for programs with refcounted arguments since we
	22833	+ * cannot guarantee that valid refcounted kptrs will be passed to the callee.
	22834	+ */
	22835	+ for (i = 0; i < env->subprog_cnt; i++) {
	22836	+ if (has_refcounted_arg && env->subprog_info[i].has_tail_call) {
	22837	+ verbose(env, "program with __ref argument cannot tail call\n");
	22838	+ return -EINVAL;
	22839	+ }
	22840	+ }
	22841	+
	22842	+ prog->aux->st_ops = st_ops;
	22843	+ prog->aux->attach_st_ops_member_off = member_off;
22480	22844
22481	22845	prog->aux->attach_func_proto = func_proto;
22482	22846	prog->aux->attach_func_name = mname;
22483	22847	env->ops = st_ops->verifier_ops;
22484	22848
22485		− return 0;
	22849	+ return bpf_prog_ctx_arg_info_init(prog, st_ops_desc->arg_info[member_idx].info,
	22850	+ st_ops_desc->arg_info[member_idx].cnt);
22486	22851	}
22487	22852	#define SECURITY_PREFIX "security_"
22488	22853
@@ -22558,6 +22923,7 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, · 69 unmodified lines
22558	22923	if (tgt_prog) {
22559	22924	struct bpf_prog_aux *aux = tgt_prog->aux;
22560	22925	bool tgt_changes_pkt_data;
	22926	+ bool tgt_might_sleep;
22561	22927
22562	22928	if (bpf_prog_is_dev_bound(prog->aux) &&
22563	22929	!bpf_prog_dev_bound_match(prog, tgt_prog)) {
@@ -22600,6 +22966,15 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, · 36 unmodified lines
22600	22966	"Extension program changes packet data, while original does not\n");
22601	22967	return -EINVAL;
22602	22968	}
	22969	+
	22970	+ tgt_might_sleep = aux->func
	22971	+ ? aux->func[subprog]->aux->might_sleep
	22972	+ : aux->might_sleep;
	22973	+ if (prog->aux->might_sleep && !tgt_might_sleep) {
	22974	+ bpf_log(log,
	22975	+ "Extension program may sleep, while original does not\n");
	22976	+ return -EINVAL;
	22977	+ }
22603	22978	}
22604	22979	if (!tgt_prog->jited) {
22605	22980	bpf_log(log, "Can attach to only JITed progs\n");
@@ -22856,6 +23231,33 @@ BTF_ID(func, __rcu_read_unlock) · 250 unmodified lines
22856	23231	#endif
22857	23232	BTF_SET_END(btf_id_deny)
22858	23233
	23234	+/* fexit and fmod_ret can't be used to attach to __noreturn functions.
	23235	+ * Currently, we must manually list all __noreturn functions here. Once a more
	23236	+ * robust solution is implemented, this workaround can be removed.
	23237	+ */
	23238	+BTF_SET_START(noreturn_deny)
	23239	+#ifdef CONFIG_IA32_EMULATION
	23240	+BTF_ID(func, __ia32_sys_exit)
	23241	+BTF_ID(func, __ia32_sys_exit_group)
	23242	+#endif
	23243	+#ifdef CONFIG_KUNIT
	23244	+BTF_ID(func, __kunit_abort)
	23245	+BTF_ID(func, kunit_try_catch_throw)
	23246	+#endif
	23247	+#ifdef CONFIG_MODULES
	23248	+BTF_ID(func, __module_put_and_kthread_exit)
	23249	+#endif
	23250	+#ifdef CONFIG_X86_64
	23251	+BTF_ID(func, __x64_sys_exit)
	23252	+BTF_ID(func, __x64_sys_exit_group)
	23253	+#endif
	23254	+BTF_ID(func, do_exit)
	23255	+BTF_ID(func, do_group_exit)
	23256	+BTF_ID(func, kthread_complete_and_exit)
	23257	+BTF_ID(func, kthread_exit)
	23258	+BTF_ID(func, make_task_dead)
	23259	+BTF_SET_END(noreturn_deny)
	23260	+
22859	23261	static bool can_be_sleepable(struct bpf_prog *prog)
22860	23262	{
22861	23263	if (prog->type == BPF_PROG_TYPE_TRACING) {
@@ -22932,9 +23334,7 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) · 70 unmodified lines
22932	23334	prog->aux->attach_btf_trace = true;
22933	23335	return 0;
22934	23336	} else if (prog->expected_attach_type == BPF_TRACE_ITER) {
22935		− if (!bpf_iter_prog_supported(prog))
22936		− return -EINVAL;
22937		− return 0;
	23337	+ return bpf_iter_prog_supported(prog);
22938	23338	}
22939	23339
22940	23340	if (prog->type == BPF_PROG_TYPE_LSM) {
@@ -22944,6 +23344,11 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) · 3 unmodified lines
22944	23344	} else if (prog->type == BPF_PROG_TYPE_TRACING &&
22945	23345	btf_id_set_contains(&btf_id_deny, btf_id)) {
22946	23346	return -EINVAL;
	23347	+ } else if ((prog->expected_attach_type == BPF_TRACE_FEXIT \|\|
	23348	+ prog->expected_attach_type == BPF_MODIFY_RETURN) &&
	23349	+ btf_id_set_contains(&noreturn_deny, btf_id)) {
	23350	+ verbose(env, "Attaching fexit/fmod_ret to __noreturn functions is rejected.\n");
	23351	+ return -EINVAL;
22947	23352	}
22948	23353
22949	23354	key = bpf_trampoline_compute_key(tgt_prog, prog->aux->attach_btf, btf_id);
@@ -23036,6 +23441,302 @@ static int process_fd_array(struct bpf_verifier_env env, union bpf_attr attr, · 86 unmodified lines
23036	23441	return 0;
23037	23442	}
23038	23443
	23444	+static bool can_fallthrough(struct bpf_insn *insn)
	23445	+{
	23446	+ u8 class = BPF_CLASS(insn->code);
	23447	+ u8 opcode = BPF_OP(insn->code);
	23448	+
	23449	+ if (class != BPF_JMP && class != BPF_JMP32)
	23450	+ return true;
	23451	+
	23452	+ if (opcode == BPF_EXIT \|\| opcode == BPF_JA)
	23453	+ return false;
	23454	+
	23455	+ return true;
	23456	+}
	23457	+
	23458	+static bool can_jump(struct bpf_insn *insn)
	23459	+{
	23460	+ u8 class = BPF_CLASS(insn->code);
	23461	+ u8 opcode = BPF_OP(insn->code);
	23462	+
	23463	+ if (class != BPF_JMP && class != BPF_JMP32)
	23464	+ return false;
	23465	+
	23466	+ switch (opcode) {
	23467	+ case BPF_JA:
	23468	+ case BPF_JEQ:
	23469	+ case BPF_JNE:
	23470	+ case BPF_JLT:
	23471	+ case BPF_JLE:
	23472	+ case BPF_JGT:
	23473	+ case BPF_JGE:
	23474	+ case BPF_JSGT:
	23475	+ case BPF_JSGE:
	23476	+ case BPF_JSLT:
	23477	+ case BPF_JSLE:
	23478	+ case BPF_JCOND:
	23479	+ return true;
	23480	+ }
	23481	+
	23482	+ return false;
	23483	+}
	23484	+
	23485	+static int insn_successors(struct bpf_prog *prog, u32 idx, u32 succ[2])
	23486	+{
	23487	+ struct bpf_insn *insn = &prog->insnsi[idx];
	23488	+ int i = 0, insn_sz;
	23489	+ u32 dst;
	23490	+
	23491	+ insn_sz = bpf_is_ldimm64(insn) ? 2 : 1;
	23492	+ if (can_fallthrough(insn) && idx + 1 < prog->len)
	23493	+ succ[i++] = idx + insn_sz;
	23494	+
	23495	+ if (can_jump(insn)) {
	23496	+ dst = idx + jmp_offset(insn) + 1;
	23497	+ if (i == 0 \|\| succ[0] != dst)
	23498	+ succ[i++] = dst;
	23499	+ }
	23500	+
	23501	+ return i;
	23502	+}
	23503	+
	23504	+/* Each field is a register bitmask */
	23505	+struct insn_live_regs {
	23506	+ u16 use; /* registers read by instruction */
	23507	+ u16 def; /* registers written by instruction */
	23508	+ u16 in; /* registers that may be alive before instruction */
	23509	+ u16 out; /* registers that may be alive after instruction */
	23510	+};
	23511	+
	23512	+/* Bitmask with 1s for all caller saved registers */
	23513	+#define ALL_CALLER_SAVED_REGS ((1u << CALLER_SAVED_REGS) - 1)
	23514	+
	23515	+/* Compute info->{use,def} fields for the instruction */
	23516	+static void compute_insn_live_regs(struct bpf_verifier_env *env,
	23517	+ struct bpf_insn *insn,
	23518	+ struct insn_live_regs *info)
	23519	+{
	23520	+ struct call_summary cs;
	23521	+ u8 class = BPF_CLASS(insn->code);
	23522	+ u8 code = BPF_OP(insn->code);
	23523	+ u8 mode = BPF_MODE(insn->code);
	23524	+ u16 src = BIT(insn->src_reg);
	23525	+ u16 dst = BIT(insn->dst_reg);
	23526	+ u16 r0 = BIT(0);
	23527	+ u16 def = 0;
	23528	+ u16 use = 0xffff;
	23529	+
	23530	+ switch (class) {
	23531	+ case BPF_LD:
	23532	+ switch (mode) {
	23533	+ case BPF_IMM:
	23534	+ if (BPF_SIZE(insn->code) == BPF_DW) {
	23535	+ def = dst;
	23536	+ use = 0;
	23537	+ }
	23538	+ break;
	23539	+ case BPF_LD \| BPF_ABS:
	23540	+ case BPF_LD \| BPF_IND:
	23541	+ /* stick with defaults */
	23542	+ break;
	23543	+ }
	23544	+ break;
	23545	+ case BPF_LDX:
	23546	+ switch (mode) {
	23547	+ case BPF_MEM:
	23548	+ case BPF_MEMSX:
	23549	+ def = dst;
	23550	+ use = src;
	23551	+ break;
	23552	+ }
	23553	+ break;
	23554	+ case BPF_ST:
	23555	+ switch (mode) {
	23556	+ case BPF_MEM:
	23557	+ def = 0;
	23558	+ use = dst;
	23559	+ break;
	23560	+ }
	23561	+ break;
	23562	+ case BPF_STX:
	23563	+ switch (mode) {
	23564	+ case BPF_MEM:
	23565	+ def = 0;
	23566	+ use = dst \| src;
	23567	+ break;
	23568	+ case BPF_ATOMIC:
	23569	+ switch (insn->imm) {
	23570	+ case BPF_CMPXCHG:
	23571	+ use = r0 \| dst \| src;
	23572	+ def = r0;
	23573	+ break;
	23574	+ case BPF_LOAD_ACQ:
	23575	+ def = dst;
	23576	+ use = src;
	23577	+ break;
	23578	+ case BPF_STORE_REL:
	23579	+ def = 0;
	23580	+ use = dst \| src;
	23581	+ break;
	23582	+ default:
	23583	+ use = dst \| src;
	23584	+ if (insn->imm & BPF_FETCH)
	23585	+ def = src;
	23586	+ else
	23587	+ def = 0;
	23588	+ }
	23589	+ break;
	23590	+ }
	23591	+ break;
	23592	+ case BPF_ALU:
	23593	+ case BPF_ALU64:
	23594	+ switch (code) {
	23595	+ case BPF_END:
	23596	+ use = dst;
	23597	+ def = dst;
	23598	+ break;
	23599	+ case BPF_MOV:
	23600	+ def = dst;
	23601	+ if (BPF_SRC(insn->code) == BPF_K)
	23602	+ use = 0;
	23603	+ else
	23604	+ use = src;
	23605	+ break;
	23606	+ default:
	23607	+ def = dst;
	23608	+ if (BPF_SRC(insn->code) == BPF_K)
	23609	+ use = dst;
	23610	+ else
	23611	+ use = dst \| src;
	23612	+ }
	23613	+ break;
	23614	+ case BPF_JMP:
	23615	+ case BPF_JMP32:
	23616	+ switch (code) {
	23617	+ case BPF_JA:
	23618	+ case BPF_JCOND:
	23619	+ def = 0;
	23620	+ use = 0;
	23621	+ break;
	23622	+ case BPF_EXIT:
	23623	+ def = 0;
	23624	+ use = r0;
	23625	+ break;
	23626	+ case BPF_CALL:
	23627	+ def = ALL_CALLER_SAVED_REGS;
	23628	+ use = def & ~BIT(BPF_REG_0);
	23629	+ if (get_call_summary(env, insn, &cs))
	23630	+ use = GENMASK(cs.num_params, 1);
	23631	+ break;
	23632	+ default:
	23633	+ def = 0;
	23634	+ if (BPF_SRC(insn->code) == BPF_K)
	23635	+ use = dst;
	23636	+ else
	23637	+ use = dst \| src;
	23638	+ }
	23639	+ break;
	23640	+ }
	23641	+
	23642	+ info->def = def;
	23643	+ info->use = use;
	23644	+}
	23645	+
	23646	+/* Compute may-live registers after each instruction in the program.
	23647	+ * The register is live after the instruction I if it is read by some
	23648	+ * instruction S following I during program execution and is not
	23649	+ * overwritten between I and S.
	23650	+ *
	23651	+ * Store result in env->insn_aux_data[i].live_regs.
	23652	+ */
	23653	+static int compute_live_registers(struct bpf_verifier_env *env)
	23654	+{
	23655	+ struct bpf_insn_aux_data *insn_aux = env->insn_aux_data;
	23656	+ struct bpf_insn *insns = env->prog->insnsi;
	23657	+ struct insn_live_regs *state;
	23658	+ int insn_cnt = env->prog->len;
	23659	+ int err = 0, i, j;
	23660	+ bool changed;
	23661	+
	23662	+ /* Use the following algorithm:
	23663	+ * - define the following:
	23664	+ * - I.use : a set of all registers read by instruction I;
	23665	+ * - I.def : a set of all registers written by instruction I;
	23666	+ * - I.in : a set of all registers that may be alive before I execution;
	23667	+ * - I.out : a set of all registers that may be alive after I execution;
	23668	+ * - insn_successors(I): a set of instructions S that might immediately
	23669	+ * follow I for some program execution;
	23670	+ * - associate separate empty sets 'I.in' and 'I.out' with each instruction;
	23671	+ * - visit each instruction in a postorder and update
	23672	+ * state[i].in, state[i].out as follows:
	23673	+ *
	23674	+ * state[i].out = U [state[s].in for S in insn_successors(i)]
	23675	+ * state[i].in = (state[i].out / state[i].def) U state[i].use
	23676	+ *
	23677	+ * (where U stands for set union, / stands for set difference)
	23678	+ * - repeat the computation while {in,out} fields changes for
	23679	+ * any instruction.
	23680	+ */
	23681	+ state = kvcalloc(insn_cnt, sizeof(*state), GFP_KERNEL);
	23682	+ if (!state) {
	23683	+ err = -ENOMEM;
	23684	+ goto out;
	23685	+ }
	23686	+
	23687	+ for (i = 0; i < insn_cnt; ++i)
	23688	+ compute_insn_live_regs(env, &insns[i], &state[i]);
	23689	+
	23690	+ changed = true;
	23691	+ while (changed) {
	23692	+ changed = false;
	23693	+ for (i = 0; i < env->cfg.cur_postorder; ++i) {
	23694	+ int insn_idx = env->cfg.insn_postorder[i];
	23695	+ struct insn_live_regs *live = &state[insn_idx];
	23696	+ int succ_num;
	23697	+ u32 succ[2];
	23698	+ u16 new_out = 0;
	23699	+ u16 new_in = 0;
	23700	+
	23701	+ succ_num = insn_successors(env->prog, insn_idx, succ);
	23702	+ for (int s = 0; s < succ_num; ++s)
	23703	+ new_out \|= state[succ[s]].in;
	23704	+ new_in = (new_out & ~live->def) \| live->use;
	23705	+ if (new_out != live->out \|\| new_in != live->in) {
	23706	+ live->in = new_in;
	23707	+ live->out = new_out;
	23708	+ changed = true;
	23709	+ }
	23710	+ }
	23711	+ }
	23712	+
	23713	+ for (i = 0; i < insn_cnt; ++i)
	23714	+ insn_aux[i].live_regs_before = state[i].in;
	23715	+
	23716	+ if (env->log.level & BPF_LOG_LEVEL2) {
	23717	+ verbose(env, "Live regs before insn:\n");
	23718	+ for (i = 0; i < insn_cnt; ++i) {
	23719	+ verbose(env, "%3d: ", i);
	23720	+ for (j = BPF_REG_0; j < BPF_REG_10; ++j)
	23721	+ if (insn_aux[i].live_regs_before & BIT(j))
	23722	+ verbose(env, "%d", j);
	23723	+ else
	23724	+ verbose(env, ".");
	23725	+ verbose(env, " ");
	23726	+ verbose_insn(env, &insns[i]);
	23727	+ if (bpf_is_ldimm64(&insns[i]))
	23728	+ i++;
	23729	+ }
	23730	+ }
	23731	+
	23732	+out:
	23733	+ kvfree(state);
	23734	+ kvfree(env->cfg.insn_postorder);
	23735	+ env->cfg.insn_postorder = NULL;
	23736	+ env->cfg.cur_postorder = 0;
	23737	+ return err;
	23738	+}
	23739	+
23039	23740	int bpf_check(struct bpf_prog *prog, union bpf_attr attr, bpfptr_t uattr, __u32 uattr_size)
23040	23741	{
23041	23742	u64 start_time = ktime_get_ns();
@@ -23113,12 +23814,16 @@ int bpf_check(struct bpf_prog *prog, union bpf_attr attr, bpfptr_t uattr, __u3 · 71 unmodified lines
23113	23814	env->test_reg_invariants = attr->prog_flags & BPF_F_TEST_REG_INVARIANTS;
23114	23815
23115	23816	env->explored_states = kvcalloc(state_htab_size(env),
23116		− sizeof(struct bpf_verifier_state_list *),
	23817	+ sizeof(struct list_head),
23117	23818	GFP_USER);
23118	23819	ret = -ENOMEM;
23119	23820	if (!env->explored_states)
23120	23821	goto skip_full_check;
23121	23822
	23823	+ for (i = 0; i < state_htab_size(env); i++)
	23824	+ INIT_LIST_HEAD(&env->explored_states[i]);
	23825	+ INIT_LIST_HEAD(&env->free_list);
	23826	+
23122	23827	ret = check_btf_info_early(env, attr, uattr);
23123	23828	if (ret < 0)
23124	23829	goto skip_full_check;
@@ -23153,6 +23858,10 @@ int bpf_check(struct bpf_prog *prog, union bpf_attr attr, bpfptr_t uattr, __u3 · 28 unmodified lines
23153	23858	if (ret)
23154	23859	goto skip_full_check;
23155	23860
	23861	+ ret = compute_live_registers(env);
	23862	+ if (ret < 0)
	23863	+ goto skip_full_check;
	23864	+
23156	23865	ret = mark_fastcall_patterns(env);
23157	23866	if (ret < 0)
23158	23867	goto skip_full_check;
@@ -23291,6 +24000,7 @@ err_unlock: · 132 unmodified lines
23291	24000	vfree(env->insn_aux_data);
23292	24001	kvfree(env->insn_hist);
23293	24002	err_free_env:
	24003	+ kvfree(env->cfg.insn_postorder);
23294	24004	kvfree(env);
23295	24005	return ret;
23296	24006	}

kernel/trace/bpf_trace.c+10−4

@@ -392,7 +392,7 @@ static const struct bpf_func_proto bpf_trace_printk_proto = { · 391 unmodified lines
392	392	.arg2_type = ARG_CONST_SIZE,
393	393	};
394	394
395		−static void __set_printk_clr_event(void)
	395	+static void __set_printk_clr_event(struct work_struct *work)
396	396	{
397	397	/*
398	398	* This program might be calling bpf_trace_printk,
@@ -405,10 +405,11 @@ static void __set_printk_clr_event(void) · 6 unmodified lines
405	405	if (trace_set_clr_event("bpf_trace", "bpf_trace_printk", 1))
406	406	pr_warn_ratelimited("could not enable bpf_trace_printk events");
407	407	}
	408	+static DECLARE_WORK(set_printk_work, __set_printk_clr_event);
408	409
409	410	const struct bpf_func_proto *bpf_get_trace_printk_proto(void)
410	411	{
411		− __set_printk_clr_event();
	412	+ schedule_work(&set_printk_work);
412	413	return &bpf_trace_printk_proto;
413	414	}
414	415
@@ -451,7 +452,7 @@ static const struct bpf_func_proto bpf_trace_vprintk_proto = { · 36 unmodified lines
451	452
452	453	const struct bpf_func_proto *bpf_get_trace_vprintk_proto(void)
453	454	{
454		− __set_printk_clr_event();
	455	+ schedule_work(&set_printk_work);
455	456	return &bpf_trace_vprintk_proto;
456	457	}
457	458
@@ -606,6 +607,11 @@ static const struct bpf_func_proto bpf_perf_event_read_value_proto = { · 148 unmodified lines
606	607	.arg4_type = ARG_CONST_SIZE,
607	608	};
608	609
	610	+const struct bpf_func_proto *bpf_get_perf_event_read_value_proto(void)
	611	+{
	612	+ return &bpf_perf_event_read_value_proto;
	613	+}
	614	+
609	615	static __always_inline u64
610	616	__bpf_perf_event_output(struct pt_regs regs, struct bpf_map map,
611	617	u64 flags, struct perf_raw_record *raw,
@@ -843,7 +849,7 @@ static int bpf_send_signal_common(u32 sig, enum pid_type type, struct task_struc · 231 unmodified lines
843	849	if (unlikely(is_global_init(task)))
844	850	return -EPERM;
845	851
846		− if (!preemptible()) {
	852	+ if (preempt_count() != 0 \|\| irqs_disabled()) {
847	853	/* Do an early check on signal validity. Otherwise,
848	854	* the error is lost in deferred irq_work.
849	855	*/

mm/memory.c+118−0

@@ -6807,6 +6807,124 @@ int access_process_vm(struct task_struct *tsk, unsigned long addr, · 6806 unmodified lines
6807	6807	}
6808	6808	EXPORT_SYMBOL_GPL(access_process_vm);
6809	6809
	6810	+#ifdef CONFIG_BPF_SYSCALL
	6811	+/*
	6812	+ * Copy a string from another process's address space as given in mm.
	6813	+ * If there is any error return -EFAULT.
	6814	+ */
	6815	+static int __copy_remote_vm_str(struct mm_struct *mm, unsigned long addr,
	6816	+ void *buf, int len, unsigned int gup_flags)
	6817	+{
	6818	+ void *old_buf = buf;
	6819	+ int err = 0;
	6820	+
	6821	+ (char )buf = '\0';
	6822	+
	6823	+ if (mmap_read_lock_killable(mm))
	6824	+ return -EFAULT;
	6825	+
	6826	+ addr = untagged_addr_remote(mm, addr);
	6827	+
	6828	+ /* Avoid triggering the temporary warning in __get_user_pages */
	6829	+ if (!vma_lookup(mm, addr)) {
	6830	+ err = -EFAULT;
	6831	+ goto out;
	6832	+ }
	6833	+
	6834	+ while (len) {
	6835	+ int bytes, offset, retval;
	6836	+ void *maddr;
	6837	+ struct page *page;
	6838	+ struct vm_area_struct *vma = NULL;
	6839	+
	6840	+ page = get_user_page_vma_remote(mm, addr, gup_flags, &vma);
	6841	+ if (IS_ERR(page)) {
	6842	+ /*
	6843	+ * Treat as a total failure for now until we decide how
	6844	+ * to handle the CONFIG_HAVE_IOREMAP_PROT case and
	6845	+ * stack expansion.
	6846	+ */
	6847	+ (char )buf = '\0';
	6848	+ err = -EFAULT;
	6849	+ goto out;
	6850	+ }
	6851	+
	6852	+ bytes = len;
	6853	+ offset = addr & (PAGE_SIZE - 1);
	6854	+ if (bytes > PAGE_SIZE - offset)
	6855	+ bytes = PAGE_SIZE - offset;
	6856	+
	6857	+ maddr = kmap_local_page(page);
	6858	+ retval = strscpy(buf, maddr + offset, bytes);
	6859	+ if (retval >= 0) {
	6860	+ /* Found the end of the string */
	6861	+ buf += retval;
	6862	+ unmap_and_put_page(page, maddr);
	6863	+ break;
	6864	+ }
	6865	+
	6866	+ buf += bytes - 1;
	6867	+ /*
	6868	+ * Because strscpy always NUL terminates we need to
	6869	+ * copy the last byte in the page if we are going to
	6870	+ * load more pages
	6871	+ */
	6872	+ if (bytes != len) {
	6873	+ addr += bytes - 1;
	6874	+ copy_from_user_page(vma, page, addr, buf, maddr + (PAGE_SIZE - 1), 1);
	6875	+ buf += 1;
	6876	+ addr += 1;
	6877	+ }
	6878	+ len -= bytes;
	6879	+
	6880	+ unmap_and_put_page(page, maddr);
	6881	+ }
	6882	+
	6883	+out:
	6884	+ mmap_read_unlock(mm);
	6885	+ if (err)
	6886	+ return err;
	6887	+ return buf - old_buf;
	6888	+}
	6889	+
	6890	+/**
	6891	+ * copy_remote_vm_str - copy a string from another process's address space.
	6892	+ * @tsk: the task of the target address space
	6893	+ * @addr: start address to read from
	6894	+ * @buf: destination buffer
	6895	+ * @len: number of bytes to copy
	6896	+ * @gup_flags: flags modifying lookup behaviour
	6897	+ *
	6898	+ * The caller must hold a reference on @mm.
	6899	+ *
	6900	+ * Return: number of bytes copied from @addr (source) to @buf (destination);
	6901	+ * not including the trailing NUL. Always guaranteed to leave NUL-terminated
	6902	+ * buffer. On any error, return -EFAULT.
	6903	+ */
	6904	+int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr,
	6905	+ void *buf, int len, unsigned int gup_flags)
	6906	+{
	6907	+ struct mm_struct *mm;
	6908	+ int ret;
	6909	+
	6910	+ if (unlikely(len == 0))
	6911	+ return 0;
	6912	+
	6913	+ mm = get_task_mm(tsk);
	6914	+ if (!mm) {
	6915	+ (char )buf = '\0';
	6916	+ return -EFAULT;
	6917	+ }
	6918	+
	6919	+ ret = __copy_remote_vm_str(mm, addr, buf, len, gup_flags);
	6920	+
	6921	+ mmput(mm);
	6922	+
	6923	+ return ret;
	6924	+}
	6925	+EXPORT_SYMBOL_GPL(copy_remote_vm_str);
	6926	+#endif /* CONFIG_BPF_SYSCALL */
	6927	+
6810	6928	/*
6811	6929	* Print the name of a VMA.
6812	6930	*/

mm/nommu.c+79−0

@@ -1714,6 +1714,85 @@ int access_process_vm(struct task_struct tsk, unsigned long addr, void buf, in · 1713 unmodified lines
1714	1714	}
1715	1715	EXPORT_SYMBOL_GPL(access_process_vm);
1716	1716
	1717	+#ifdef CONFIG_BPF_SYSCALL
	1718	+/*
	1719	+ * Copy a string from another process's address space as given in mm.
	1720	+ * If there is any error return -EFAULT.
	1721	+ */
	1722	+static int __copy_remote_vm_str(struct mm_struct *mm, unsigned long addr,
	1723	+ void *buf, int len)
	1724	+{
	1725	+ unsigned long addr_end;
	1726	+ struct vm_area_struct *vma;
	1727	+ int ret = -EFAULT;
	1728	+
	1729	+ (char )buf = '\0';
	1730	+
	1731	+ if (mmap_read_lock_killable(mm))
	1732	+ return ret;
	1733	+
	1734	+ /* the access must start within one of the target process's mappings */
	1735	+ vma = find_vma(mm, addr);
	1736	+ if (!vma)
	1737	+ goto out;
	1738	+
	1739	+ if (check_add_overflow(addr, len, &addr_end))
	1740	+ goto out;
	1741	+
	1742	+ /* don't overrun this mapping */
	1743	+ if (addr_end > vma->vm_end)
	1744	+ len = vma->vm_end - addr;
	1745	+
	1746	+ /* only read mappings where it is permitted */
	1747	+ if (vma->vm_flags & VM_MAYREAD) {
	1748	+ ret = strscpy(buf, (char *)addr, len);
	1749	+ if (ret < 0)
	1750	+ ret = len - 1;
	1751	+ }
	1752	+
	1753	+out:
	1754	+ mmap_read_unlock(mm);
	1755	+ return ret;
	1756	+}
	1757	+
	1758	+/**
	1759	+ * copy_remote_vm_str - copy a string from another process's address space.
	1760	+ * @tsk: the task of the target address space
	1761	+ * @addr: start address to read from
	1762	+ * @buf: destination buffer
	1763	+ * @len: number of bytes to copy
	1764	+ * @gup_flags: flags modifying lookup behaviour (unused)
	1765	+ *
	1766	+ * The caller must hold a reference on @mm.
	1767	+ *
	1768	+ * Return: number of bytes copied from @addr (source) to @buf (destination);
	1769	+ * not including the trailing NUL. Always guaranteed to leave NUL-terminated
	1770	+ * buffer. On any error, return -EFAULT.
	1771	+ */
	1772	+int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr,
	1773	+ void *buf, int len, unsigned int gup_flags)
	1774	+{
	1775	+ struct mm_struct *mm;
	1776	+ int ret;
	1777	+
	1778	+ if (unlikely(len == 0))
	1779	+ return 0;
	1780	+
	1781	+ mm = get_task_mm(tsk);
	1782	+ if (!mm) {
	1783	+ (char )buf = '\0';
	1784	+ return -EFAULT;
	1785	+ }
	1786	+
	1787	+ ret = __copy_remote_vm_str(mm, addr, buf, len);
	1788	+
	1789	+ mmput(mm);
	1790	+
	1791	+ return ret;
	1792	+}
	1793	+EXPORT_SYMBOL_GPL(copy_remote_vm_str);
	1794	+#endif /* CONFIG_BPF_SYSCALL */
	1795	+
1717	1796	/**
1718	1797	* nommu_shrink_inode_mappings - Shrink the shared mappings on an inode
1719	1798	* @inode: The inode to check

net/core/filter.c+4−2

@@ -8137,6 +8137,8 @@ sk_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 8136 unmodified lines
8137	8137	return &bpf_skb_load_bytes_relative_proto;
8138	8138	case BPF_FUNC_get_socket_cookie:
8139	8139	return &bpf_get_socket_cookie_proto;
	8140	+ case BPF_FUNC_get_netns_cookie:
	8141	+ return &bpf_get_netns_cookie_proto;
8140	8142	case BPF_FUNC_get_socket_uid:
8141	8143	return &bpf_get_socket_uid_proto;
8142	8144	case BPF_FUNC_perf_event_output:
@@ -9697,7 +9699,7 @@ static u32 bpf_convert_ctx_access(enum bpf_access_type type, · 1554 unmodified lines
9697	9699
9698	9700	case offsetof(struct __sk_buff, queue_mapping):
9699	9701	if (type == BPF_WRITE) {
9700		− u32 off = bpf_target_off(struct sk_buff, queue_mapping, 2, target_size);
	9702	+ u32 offset = bpf_target_off(struct sk_buff, queue_mapping, 2, target_size);
9701	9703
9702	9704	if (BPF_CLASS(si->code) == BPF_ST && si->imm >= NO_QUEUE_MAPPING) {
9703	9705	insn++ = BPF_JMP_A(0); / noop */
@@ -9706,7 +9708,7 @@ static u32 bpf_convert_ctx_access(enum bpf_access_type type, · 2 unmodified lines
9706	9708
9707	9709	if (BPF_CLASS(si->code) == BPF_STX)
9708	9710	*insn++ = BPF_JMP_IMM(BPF_JGE, si->src_reg, NO_QUEUE_MAPPING, 1);
9709		− *insn++ = BPF_EMIT_STORE(BPF_H, si, off);
	9711	+ *insn++ = BPF_EMIT_STORE(BPF_H, si, offset);
9710	9712	} else {
9711	9713	*insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
9712	9714	bpf_target_off(struct sk_buff,

samples/bpf/Makefile+1−1

@@ -307,7 +307,7 @@ $(obj)/$(TRACE_HELPERS): TPROGS_CFLAGS := $(TPROGS_CFLAGS) -D__must_check= · 306 unmodified lines
307	307
308	308	VMLINUX_BTF_PATHS ?= $(abspath $(if $(O),$(O)/vmlinux)) \
309	309	$(abspath $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux)) \
310		− $(abspath ./vmlinux)
	310	+ $(abspath $(objtree)/vmlinux)
311	311	VMLINUX_BTF ?= $(abspath $(firstword $(wildcard $(VMLINUX_BTF_PATHS))))
312	312
313	313	$(obj)/vmlinux.h: $(VMLINUX_BTF) $(BPFTOOL)

scripts/Makefile.btf+1−1

@@ -24,7 +24,7 @@ else · 23 unmodified lines
24	24	pahole-flags-$(call test-ge, $(pahole-ver), 126) = -j$(JOBS) --btf_features=encode_force,var,float,enum64,decl_tag,type_tag,optimized_func,consistent_func,decl_tag_kfuncs
25	25
26	26	ifneq ($(KBUILD_EXTMOD),)
27		−module-pahole-flags-$(call test-ge, $(pahole-ver), 126) += --btf_features=distilled_base
	27	+module-pahole-flags-$(call test-ge, $(pahole-ver), 128) += --btf_features=distilled_base
28	28	endif
29	29
30	30	endif

security/security.c+9−6

@@ -5627,6 +5627,7 @@ int security_audit_rule_match(struct lsm_prop *prop, u32 field, u32 op, · 5626 unmodified lines
5627	5627	* @cmd: command
5628	5628	* @attr: bpf attribute
5629	5629	* @size: size
	5630	+ * @kernel: whether or not call originated from kernel
5630	5631	*
5631	5632	* Do a initial check for all bpf syscalls after the attribute is copied into
5632	5633	* the kernel. The actual security module can implement their own rules to
@@ -5634,9 +5635,9 @@ int security_audit_rule_match(struct lsm_prop *prop, u32 field, u32 op, · 1 unmodified line
5634	5635	*
5635	5636	* Return: Returns 0 if permission is granted.
5636	5637	*/
5637		−int security_bpf(int cmd, union bpf_attr *attr, unsigned int size)
	5638	+int security_bpf(int cmd, union bpf_attr *attr, unsigned int size, bool kernel)
5638	5639	{
5639		− return call_int_hook(bpf, cmd, attr, size);
	5640	+ return call_int_hook(bpf, cmd, attr, size, kernel);
5640	5641	}
5641	5642
5642	5643	/**
@@ -5673,6 +5674,7 @@ int security_bpf_prog(struct bpf_prog *prog) · 30 unmodified lines
5673	5674	* @map: BPF map object
5674	5675	* @attr: BPF syscall attributes used to create BPF map
5675	5676	* @token: BPF token used to grant user access
	5677	+ * @kernel: whether or not call originated from kernel
5676	5678	*
5677	5679	* Do a check when the kernel creates a new BPF map. This is also the
5678	5680	* point where LSM blob is allocated for LSMs that need them.
@@ -5680,9 +5682,9 @@ int security_bpf_prog(struct bpf_prog *prog) · 1 unmodified line
5680	5682	* Return: Returns 0 on success, error on failure.
5681	5683	*/
5682	5684	int security_bpf_map_create(struct bpf_map map, union bpf_attr attr,
5683		− struct bpf_token *token)
	5685	+ struct bpf_token *token, bool kernel)
5684	5686	{
5685		− return call_int_hook(bpf_map_create, map, attr, token);
	5687	+ return call_int_hook(bpf_map_create, map, attr, token, kernel);
5686	5688	}
5687	5689
5688	5690	/**
@@ -5690,6 +5692,7 @@ int security_bpf_map_create(struct bpf_map map, union bpf_attr attr, · 1 unmodified line
5690	5692	* @prog: BPF program object
5691	5693	* @attr: BPF syscall attributes used to create BPF program
5692	5694	* @token: BPF token used to grant user access to BPF subsystem
	5695	+ * @kernel: whether or not call originated from kernel
5693	5696	*
5694	5697	* Perform an access control check when the kernel loads a BPF program and
5695	5698	* allocates associated BPF program object. This hook is also responsible for
@@ -5698,9 +5701,9 @@ int security_bpf_map_create(struct bpf_map map, union bpf_attr attr, · 2 unmodified lines
5698	5701	* Return: Returns 0 on success, error on failure.
5699	5702	*/
5700	5703	int security_bpf_prog_load(struct bpf_prog prog, union bpf_attr attr,
5701		− struct bpf_token *token)
	5704	+ struct bpf_token *token, bool kernel)
5702	5705	{
5703		− return call_int_hook(bpf_prog_load, prog, attr, token);
	5706	+ return call_int_hook(bpf_prog_load, prog, attr, token, kernel);
5704	5707	}
5705	5708
5706	5709	/**

security/selinux/hooks.c+3−3

@@ -6907,7 +6907,7 @@ static int selinux_ib_alloc_security(void *ib_sec) · 6906 unmodified lines
6907	6907
6908	6908	#ifdef CONFIG_BPF_SYSCALL
6909	6909	static int selinux_bpf(int cmd, union bpf_attr *attr,
6910		− unsigned int size)
	6910	+ unsigned int size, bool kernel)
6911	6911	{
6912	6912	u32 sid = current_sid();
6913	6913	int ret;
@@ -6994,7 +6994,7 @@ static int selinux_bpf_prog(struct bpf_prog *prog) · 80 unmodified lines
6994	6994	}
6995	6995
6996	6996	static int selinux_bpf_map_create(struct bpf_map map, union bpf_attr attr,
6997		− struct bpf_token *token)
	6997	+ struct bpf_token *token, bool kernel)
6998	6998	{
6999	6999	struct bpf_security_struct *bpfsec;
7000	7000
@@ -7017,7 +7017,7 @@ static void selinux_bpf_map_free(struct bpf_map *map) · 16 unmodified lines
7017	7017	}
7018	7018
7019	7019	static int selinux_bpf_prog_load(struct bpf_prog prog, union bpf_attr attr,
7020		− struct bpf_token *token)
	7020	+ struct bpf_token *token, bool kernel)
7021	7021	{
7022	7022	struct bpf_security_struct *bpfsec;
7023	7023

tools/bpf/bpftool/Makefile+6−1

@@ -65,7 +65,12 @@ prefix ?= /usr/local · 64 unmodified lines
65	65	bash_compdir ?= /usr/share/bash-completion/completions
66	66
67	67	CFLAGS += -O2
68		−CFLAGS += -W -Wall -Wextra -Wno-unused-parameter -Wno-missing-field-initializers
	68	+CFLAGS += -W
	69	+CFLAGS += -Wall
	70	+CFLAGS += -Wextra
	71	+CFLAGS += -Wformat-signedness
	72	+CFLAGS += -Wno-unused-parameter
	73	+CFLAGS += -Wno-missing-field-initializers
69	74	CFLAGS += $(filter-out -Wswitch-enum -Wnested-externs,$(EXTRA_WARNINGS))
70	75	CFLAGS += -DPACKAGE='"bpftool"' -D__EXPORTED_HEADERS__ \
71	76	-I$(or $(OUTPUT),.) \

tools/bpf/bpftool/btf.c+7−7

@@ -253,7 +253,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id, · 252 unmodified lines
253	253	if (btf_kflag(t))
254	254	printf("\n\t'%s' val=%d", name, v->val);
255	255	else
256		− printf("\n\t'%s' val=%u", name, v->val);
	256	+ printf("\n\t'%s' val=%u", name, (__u32)v->val);
257	257	}
258	258	}
259	259	if (json_output)
@@ -1022,7 +1022,7 @@ static int do_dump(int argc, char **argv) · 762 unmodified lines
1022	1022	for (i = 0; i < root_type_cnt; i++) {
1023	1023	if (root_type_ids[i] == root_id) {
1024	1024	err = -EINVAL;
1025		− p_err("duplicate root_id %d supplied", root_id);
	1025	+ p_err("duplicate root_id %u supplied", root_id);
1026	1026	goto done;
1027	1027	}
1028	1028	}
@@ -1132,7 +1132,7 @@ build_btf_type_table(struct hashmap *tab, enum bpf_obj_type type, · 103 unmodified lines
1132	1132	break;
1133	1133	default:
1134	1134	err = -1;
1135		− p_err("unexpected object type: %d", type);
	1135	+ p_err("unexpected object type: %u", type);
1136	1136	goto err_free;
1137	1137	}
1138	1138	if (err) {
@@ -1155,7 +1155,7 @@ build_btf_type_table(struct hashmap *tab, enum bpf_obj_type type, · 16 unmodified lines
1155	1155	break;
1156	1156	default:
1157	1157	err = -1;
1158		− p_err("unexpected object type: %d", type);
	1158	+ p_err("unexpected object type: %u", type);
1159	1159	goto err_free;
1160	1160	}
1161	1161	if (fd < 0) {
@@ -1188,7 +1188,7 @@ build_btf_type_table(struct hashmap *tab, enum bpf_obj_type type, · 26 unmodified lines
1188	1188	break;
1189	1189	default:
1190	1190	err = -1;
1191		− p_err("unexpected object type: %d", type);
	1191	+ p_err("unexpected object type: %u", type);
1192	1192	goto err_free;
1193	1193	}
1194	1194	if (!btf_id)
@@ -1254,12 +1254,12 @@ show_btf_plain(struct bpf_btf_info *info, int fd, · 59 unmodified lines
1254	1254
1255	1255	n = 0;
1256	1256	hashmap__for_each_key_entry(btf_prog_table, entry, info->id) {
1257		− printf("%s%lu", n++ == 0 ? " prog_ids " : ",", entry->value);
	1257	+ printf("%s%lu", n++ == 0 ? " prog_ids " : ",", (unsigned long)entry->value);
1258	1258	}
1259	1259
1260	1260	n = 0;
1261	1261	hashmap__for_each_key_entry(btf_map_table, entry, info->id) {
1262		− printf("%s%lu", n++ == 0 ? " map_ids " : ",", entry->value);
	1262	+ printf("%s%lu", n++ == 0 ? " map_ids " : ",", (unsigned long)entry->value);
1263	1263	}
1264	1264
1265	1265	emit_obj_refs_plain(refs_table, info->id, "\n\tpids ");

tools/bpf/bpftool/btf_dumper.c+1−1

@@ -653,7 +653,7 @@ static int __btf_dumper_type_only(const struct btf *btf, __u32 type_id, · 652 unmodified lines
653	653	case BTF_KIND_ARRAY:
654	654	array = (struct btf_array *)(t + 1);
655	655	BTF_PRINT_TYPE(array->type);
656		− BTF_PRINT_ARG("[%d]", array->nelems);
	656	+ BTF_PRINT_ARG("[%u]", array->nelems);
657	657	break;
658	658	case BTF_KIND_PTR:
659	659	BTF_PRINT_TYPE(t->type);

tools/bpf/bpftool/cgroup.c+1−1

@@ -191,7 +191,7 @@ static int show_bpf_prog(int id, enum bpf_attach_type attach_type, · 190 unmodified lines
191	191	if (attach_btf_name)
192	192	printf(" %-15s", attach_btf_name);
193	193	else if (info.attach_btf_id)
194		− printf(" attach_btf_obj_id=%d attach_btf_id=%d",
	194	+ printf(" attach_btf_obj_id=%u attach_btf_id=%u",
195	195	info.attach_btf_obj_id, info.attach_btf_id);
196	196	printf("\n");
197	197	}

tools/bpf/bpftool/common.c+4−3

@@ -461,10 +461,11 @@ int get_fd_type(int fd) · 460 unmodified lines
461	461	p_err("can't read link type: %s", strerror(errno));
462	462	return -1;
463	463	}
464		− if (n == sizeof(path)) {
	464	+ if (n == sizeof(buf)) {
465	465	p_err("can't read link type: path too long!");
466	466	return -1;
467	467	}
	468	+ buf[n] = '\0';
468	469
469	470	if (strstr(buf, "bpf-map"))
470	471	return BPF_OBJ_MAP;
@@ -713,7 +714,7 @@ ifindex_to_arch(__u32 ifindex, __u64 ns_dev, __u64 ns_ino, const char **opt) · 242 unmodified lines
713	714	int vendor_id;
714	715
715	716	if (!ifindex_to_name_ns(ifindex, ns_dev, ns_ino, devname)) {
716		− p_err("Can't get net device name for ifindex %d: %s", ifindex,
	717	+ p_err("Can't get net device name for ifindex %u: %s", ifindex,
717	718	strerror(errno));
718	719	return NULL;
719	720	}
@@ -738,7 +739,7 @@ ifindex_to_arch(__u32 ifindex, __u64 ns_dev, __u64 ns_ino, const char **opt) · 18 unmodified lines
738	739	/* No NFP support in LLVM, we have no valid triple to return. */
739	740	default:
740	741	p_err("Can't get arch name for device vendor id 0x%04x",
741		− vendor_id);
	742	+ (unsigned int)vendor_id);
742	743	return NULL;
743	744	}
744	745	}

tools/bpf/bpftool/gen.c+6−6

@@ -670,7 +670,7 @@ static void codegen_destroy(struct bpf_object obj, const char obj_name) · 669 unmodified lines
670	670	continue;
671	671	if (bpf_map__is_internal(map) &&
672	672	(bpf_map__map_flags(map) & BPF_F_MMAPABLE))
673		− printf("\tskel_free_map_data(skel->%1$s, skel->maps.%1$s.initial_value, %2$zd);\n",
	673	+ printf("\tskel_free_map_data(skel->%1$s, skel->maps.%1$s.initial_value, %2$zu);\n",
674	674	ident, bpf_map_mmap_sz(map));
675	675	codegen("\
676	676	\n\
@@ -984,7 +984,7 @@ static int walk_st_ops_shadow_vars(struct btf btf, const char ident, · 307 unmodified lines
984	984
985	985	offset = m->offset / 8;
986	986	if (next_offset < offset)
987		− printf("\t\t\tchar __padding_%d[%d];\n", i, offset - next_offset);
	987	+ printf("\t\t\tchar __padding_%d[%u];\n", i, offset - next_offset);
988	988
989	989	switch (btf_kind(member_type)) {
990	990	case BTF_KIND_INT:
@@ -1052,7 +1052,7 @@ static int walk_st_ops_shadow_vars(struct btf btf, const char ident, · 61 unmodified lines
1052	1052	/* Cannot fail since it must be a struct type */
1053	1053	size = btf__resolve_size(btf, map_type_id);
1054	1054	if (next_offset < (__u32)size)
1055		− printf("\t\t\tchar __padding_end[%d];\n", size - next_offset);
	1055	+ printf("\t\t\tchar __padding_end[%u];\n", size - next_offset);
1056	1056
1057	1057	out:
1058	1058	btf_dump__free(d);
@@ -2095,7 +2095,7 @@ btfgen_mark_type(struct btfgen_info *info, unsigned int type_id, bool follow_poi · 1036 unmodified lines
2095	2095	break;
2096	2096	/* tells if some other type needs to be handled */
2097	2097	default:
2098		− p_err("unsupported kind: %s (%d)", btf_kind_str(btf_type), type_id);
	2098	+ p_err("unsupported kind: %s (%u)", btf_kind_str(btf_type), type_id);
2099	2099	return -EINVAL;
2100	2100	}
2101	2101
@@ -2147,7 +2147,7 @@ static int btfgen_record_field_relo(struct btfgen_info *info, struct bpf_core_sp · 45 unmodified lines
2147	2147	btf_type = btf__type_by_id(btf, type_id);
2148	2148	break;
2149	2149	default:
2150		− p_err("unsupported kind: %s (%d)",
	2150	+ p_err("unsupported kind: %s (%u)",
2151	2151	btf_kind_str(btf_type), btf_type->type);
2152	2152	return -EINVAL;
2153	2153	}
@@ -2246,7 +2246,7 @@ static int btfgen_mark_type_match(struct btfgen_info *info, __u32 type_id, bool · 92 unmodified lines
2246	2246	}
2247	2247	/* tells if some other type needs to be handled */
2248	2248	default:
2249		− p_err("unsupported kind: %s (%d)", btf_kind_str(btf_type), type_id);
	2249	+ p_err("unsupported kind: %s (%u)", btf_kind_str(btf_type), type_id);
2250	2250	return -EINVAL;
2251	2251	}
2252	2252

tools/bpf/bpftool/jit_disasm.c+2−1

@@ -343,7 +343,8 @@ int disasm_print_insn(unsigned char *image, ssize_t len, int opcodes, · 342 unmodified lines
343	343	{
344	344	const struct bpf_line_info *linfo = NULL;
345	345	unsigned int nr_skip = 0;
346		− int count, i, pc = 0;
	346	+ int count, i;
	347	+ unsigned int pc = 0;
347	348	disasm_ctx_t ctx;
348	349
349	350	if (!len)

tools/bpf/bpftool/link.c+7−7

@@ -107,7 +107,7 @@ static int link_parse_fd(int argc, char **argv) · 106 unmodified lines
107	107
108	108	fd = bpf_link_get_fd_by_id(id);
109	109	if (fd < 0)
110		− p_err("failed to get link with ID %d: %s", id, strerror(errno));
	110	+ p_err("failed to get link with ID %u: %s", id, strerror(errno));
111	111	return fd;
112	112	} else if (is_prefix(**argv, "pinned")) {
113	113	char *path;
@@ -404,7 +404,7 @@ static char *perf_config_hw_cache_str(__u64 config) · 290 unmodified lines
404	404	if (hw_cache)
405	405	snprintf(str, PERF_HW_CACHE_LEN, "%s-", hw_cache);
406	406	else
407		− snprintf(str, PERF_HW_CACHE_LEN, "%lld-", config & 0xff);
	407	+ snprintf(str, PERF_HW_CACHE_LEN, "%llu-", config & 0xff);
408	408
409	409	op = perf_event_name(evsel__hw_cache_op, (config >> 8) & 0xff);
410	410	if (op)
@@ -412,7 +412,7 @@ static char *perf_config_hw_cache_str(__u64 config) · 1 unmodified line
412	412	"%s-", op);
413	413	else
414	414	snprintf(str + strlen(str), PERF_HW_CACHE_LEN - strlen(str),
415		− "%lld-", (config >> 8) & 0xff);
	415	+ "%llu-", (config >> 8) & 0xff);
416	416
417	417	result = perf_event_name(evsel__hw_cache_result, config >> 16);
418	418	if (result)
@@ -420,7 +420,7 @@ static char *perf_config_hw_cache_str(__u64 config) · 1 unmodified line
420	420	"%s", result);
421	421	else
422	422	snprintf(str + strlen(str), PERF_HW_CACHE_LEN - strlen(str),
423		− "%lld", config >> 16);
	423	+ "%llu", config >> 16);
424	424	return str;
425	425	}
426	426
@@ -623,7 +623,7 @@ static void show_link_ifindex_plain(__u32 ifindex) · 196 unmodified lines
623	623	else
624	624	snprintf(devname, sizeof(devname), "(detached)");
625	625	if (ret)
626		− snprintf(devname, sizeof(devname), "%s(%d)",
	626	+ snprintf(devname, sizeof(devname), "%s(%u)",
627	627	tmpname, ifindex);
628	628	printf("ifindex %s ", devname);
629	629	}
@@ -699,7 +699,7 @@ void netfilter_dump_plain(const struct bpf_link_info *info) · 69 unmodified lines
699	699	if (pfname)
700	700	printf("\n\t%s", pfname);
701	701	else
702		− printf("\n\tpf: %d", pf);
	702	+ printf("\n\tpf: %u", pf);
703	703
704	704	if (hookname)
705	705	printf(" %s", hookname);
@@ -773,7 +773,7 @@ static void show_uprobe_multi_plain(struct bpf_link_info *info) · 67 unmodified lines
773	773	printf("func_cnt %u ", info->uprobe_multi.count);
774	774
775	775	if (info->uprobe_multi.pid)
776		− printf("pid %d ", info->uprobe_multi.pid);
	776	+ printf("pid %u ", info->uprobe_multi.pid);
777	777
778	778	printf("\n\t%-16s %-16s %-16s", "offset", "ref_ctr_offset", "cookies");
779	779	for (i = 0; i < info->uprobe_multi.count; i++) {

tools/bpf/bpftool/main.c+4−4

@@ -152,7 +152,7 @@ static int do_version(int argc, char **argv) · 151 unmodified lines
152	152	BPFTOOL_MINOR_VERSION, BPFTOOL_PATCH_VERSION);
153	153	#endif
154	154	jsonw_name(json_wtr, "libbpf_version");
155		− jsonw_printf(json_wtr, "\"%d.%d\"",
	155	+ jsonw_printf(json_wtr, "\"%u.%u\"",
156	156	libbpf_major_version(), libbpf_minor_version());
157	157
158	158	jsonw_name(json_wtr, "features");
@@ -370,7 +370,7 @@ static int do_batch(int argc, char **argv) · 211 unmodified lines
370	370	while ((cp = strstr(buf, "\\\n")) != NULL) {
371	371	if (!fgets(contline, sizeof(contline), fp) \|\|
372	372	strlen(contline) == 0) {
373		− p_err("missing continuation line on command %d",
	373	+ p_err("missing continuation line on command %u",
374	374	lines);
375	375	err = -1;
376	376	goto err_close;
@@ -381,7 +381,7 @@ static int do_batch(int argc, char **argv) · 4 unmodified lines
381	381	*cp = '\0';
382	382
383	383	if (strlen(buf) + strlen(contline) + 1 > sizeof(buf)) {
384		− p_err("command %d is too long", lines);
	384	+ p_err("command %u is too long", lines);
385	385	err = -1;
386	386	goto err_close;
387	387	}
@@ -423,7 +423,7 @@ static int do_batch(int argc, char **argv) · 35 unmodified lines
423	423	err = -1;
424	424	} else {
425	425	if (!json_output)
426		− printf("processed %d commands\n", lines);
	426	+ printf("processed %u commands\n", lines);
427	427	}
428	428	err_close:
429	429	if (fp != stdin)

tools/bpf/bpftool/map.c+9−5

@@ -285,7 +285,7 @@ static void print_entry_plain(struct bpf_map_info info, unsigned char key, · 284 unmodified lines
285	285	}
286	286	if (info->value_size) {
287	287	for (i = 0; i < n; i++) {
288		− printf("value (CPU %02d):%c",
	288	+ printf("value (CPU %02u):%c",
289	289	i, info->value_size > 16 ? '\n' : ' ');
290	290	fprint_hex(stdout, value + i * step,
291	291	info->value_size, " ");
@@ -316,7 +316,7 @@ static char parse_bytes(char argv, const char name, unsigned char val, · 24 unmodified lines
316	316	}
317	317
318	318	if (i != n) {
319		− p_err("%s expected %d bytes got %d", name, n, i);
	319	+ p_err("%s expected %u bytes got %u", name, n, i);
320	320	return NULL;
321	321	}
322	322
@@ -462,7 +462,7 @@ static void show_map_header_json(struct bpf_map_info info, json_writer_t wtr) · 139 unmodified lines
462	462	jsonw_string_field(wtr, "name", info->name);
463	463
464	464	jsonw_name(wtr, "flags");
465		− jsonw_printf(wtr, "%d", info->map_flags);
	465	+ jsonw_printf(wtr, "%u", info->map_flags);
466	466	}
467	467
468	468	static int show_map_close_json(int fd, struct bpf_map_info *info)
@@ -588,7 +588,7 @@ static int show_map_close_plain(int fd, struct bpf_map_info *info) · 119 unmodified lines
588	588	if (prog_type_str)
589	589	printf("owner_prog_type %s ", prog_type_str);
590	590	else
591		− printf("owner_prog_type %d ", prog_type);
	591	+ printf("owner_prog_type %u ", prog_type);
592	592	}
593	593	if (owner_jited)
594	594	printf("owner%s jited",
@@ -615,7 +615,7 @@ static int show_map_close_plain(int fd, struct bpf_map_info *info) · 20 unmodified lines
615	615	printf("\n\t");
616	616
617	617	if (info->btf_id)
618		− printf("btf_id %d", info->btf_id);
	618	+ printf("btf_id %u", info->btf_id);
619	619
620	620	if (frozen)
621	621	printf("%sfrozen", info->btf_id ? " " : "");
@@ -1270,6 +1270,10 @@ static int do_create(int argc, char **argv) · 648 unmodified lines
1270	1270	} else if (is_prefix(*argv, "name")) {
1271	1271	NEXT_ARG();
1272	1272	map_name = GET_ARG();
	1273	+ if (strlen(map_name) > BPF_OBJ_NAME_LEN - 1) {
	1274	+ p_info("Warning: map name is longer than %u characters, it will be truncated.",
	1275	+ BPF_OBJ_NAME_LEN - 1);
	1276	+ }
1273	1277	} else if (is_prefix(*argv, "key")) {
1274	1278	if (parse_u32_arg(&argc, &argv, &key_size,
1275	1279	"key size"))

tools/bpf/bpftool/map_perf_ring.c+3−3

@@ -91,15 +91,15 @@ print_bpf_output(void private_data, int cpu, struct perf_event_header event) · 90 unmodified lines
91	91	jsonw_end_object(json_wtr);
92	92	} else {
93	93	if (e->header.type == PERF_RECORD_SAMPLE) {
94		− printf("== @%lld.%09lld CPU: %d index: %d =====\n",
	94	+ printf("== @%llu.%09llu CPU: %d index: %d =====\n",
95	95	e->time / 1000000000ULL, e->time % 1000000000ULL,
96	96	cpu, idx);
97	97	fprint_hex(stdout, e->data, e->size, " ");
98	98	printf("\n");
99	99	} else if (e->header.type == PERF_RECORD_LOST) {
100		− printf("lost %lld events\n", lost->lost);
	100	+ printf("lost %llu events\n", lost->lost);
101	101	} else {
102		− printf("unknown event type=%d size=%d\n",
	102	+ printf("unknown event type=%u size=%u\n",
103	103	e->header.type, e->header.size);
104	104	}
105	105	}

tools/bpf/bpftool/net.c+2−2

@@ -476,7 +476,7 @@ static void __show_dev_tc_bpf(const struct ip_devname_ifindex *dev, · 475 unmodified lines
476	476	for (i = 0; i < optq.count; i++) {
477	477	NET_START_OBJECT;
478	478	NET_DUMP_STR("devname", "%s", dev->devname);
479		− NET_DUMP_UINT("ifindex", "(%u)", dev->ifindex);
	479	+ NET_DUMP_UINT("ifindex", "(%u)", (unsigned int)dev->ifindex);
480	480	NET_DUMP_STR("kind", " %s", attach_loc_strings[loc]);
481	481	ret = __show_dev_tc_bpf_name(prog_ids[i], prog_name,
482	482	sizeof(prog_name));
@@ -831,7 +831,7 @@ static void show_link_netfilter(void) · 348 unmodified lines
831	831	if (err) {
832	832	if (errno == ENOENT)
833	833	break;
834		− p_err("can't get next link: %s (id %d)", strerror(errno), id);
	834	+ p_err("can't get next link: %s (id %u)", strerror(errno), id);
835	835	break;
836	836	}
837	837

tools/bpf/bpftool/netlink_dumper.c+3−3

@@ -45,7 +45,7 @@ static int do_xdp_dump_one(struct nlattr *attr, unsigned int ifindex, · 44 unmodified lines
45	45	NET_START_OBJECT;
46	46	if (name)
47	47	NET_DUMP_STR("devname", "%s", name);
48		− NET_DUMP_UINT("ifindex", "(%d)", ifindex);
	48	+ NET_DUMP_UINT("ifindex", "(%u)", ifindex);
49	49
50	50	if (mode == XDP_ATTACHED_MULTI) {
51	51	if (json_output) {
@@ -74,7 +74,7 @@ int do_xdp_dump(struct ifinfomsg ifinfo, struct nlattr *tb) · 22 unmodified lines
74	74	if (!tb[IFLA_XDP])
75	75	return 0;
76	76
77		− return do_xdp_dump_one(tb[IFLA_XDP], ifinfo->ifi_index,
	77	+ return do_xdp_dump_one(tb[IFLA_XDP], (unsigned int)ifinfo->ifi_index,
78	78	libbpf_nla_getattr_str(tb[IFLA_IFNAME]));
79	79	}
80	80
@@ -168,7 +168,7 @@ int do_filter_dump(struct tcmsg info, struct nlattr tb, const char kind, · 87 unmodified lines
168	168	NET_START_OBJECT;
169	169	if (devname[0] != '\0')
170	170	NET_DUMP_STR("devname", "%s", devname);
171		− NET_DUMP_UINT("ifindex", "(%u)", ifindex);
	171	+ NET_DUMP_UINT("ifindex", "(%u)", (unsigned int)ifindex);
172	172	NET_DUMP_STR("kind", " %s", kind);
173	173	ret = do_bpf_filter_dump(tb[TCA_OPTIONS]);
174	174	NET_END_OBJECT_FINAL;

tools/bpf/bpftool/prog.c+7−6

@@ -521,10 +521,10 @@ static void print_prog_header_plain(struct bpf_prog_info *info, int fd) · 520 unmodified lines
521	521	print_dev_plain(info->ifindex, info->netns_dev, info->netns_ino);
522	522	printf("%s", info->gpl_compatible ? " gpl" : "");
523	523	if (info->run_time_ns)
524		− printf(" run_time_ns %lld run_cnt %lld",
	524	+ printf(" run_time_ns %llu run_cnt %llu",
525	525	info->run_time_ns, info->run_cnt);
526	526	if (info->recursion_misses)
527		− printf(" recursion_misses %lld", info->recursion_misses);
	527	+ printf(" recursion_misses %llu", info->recursion_misses);
528	528	printf("\n");
529	529	}
530	530
@@ -569,7 +569,7 @@ static void print_prog_plain(struct bpf_prog_info *info, int fd, bool orphaned) · 38 unmodified lines
569	569	}
570	570
571	571	if (info->btf_id)
572		− printf("\n\tbtf_id %d", info->btf_id);
	572	+ printf("\n\tbtf_id %u", info->btf_id);
573	573
574	574	emit_obj_refs_plain(refs_table, info->id, "\n\tpids ");
575	575
@@ -1164,7 +1164,7 @@ static int get_run_data(const char fname, void data_ptr, unsigned int size) · 588 unmodified lines
1164	1164	}
1165	1165	if (nb_read > buf_size - block_size) {
1166	1166	if (buf_size == UINT32_MAX) {
1167		− p_err("data_in/ctx_in is too long (max: %d)",
	1167	+ p_err("data_in/ctx_in is too long (max: %u)",
1168	1168	UINT32_MAX);
1169	1169	goto err_free;
1170	1170	}
@@ -1928,6 +1928,7 @@ static int do_loader(int argc, char **argv) · 757 unmodified lines
1928	1928
1929	1929	obj = bpf_object__open_file(file, &open_opts);
1930	1930	if (!obj) {
	1931	+ err = -1;
1931	1932	p_err("failed to open object file");
1932	1933	goto err_close_obj;
1933	1934	}
@@ -2251,7 +2252,7 @@ static char *profile_target_name(int tgt_fd) · 317 unmodified lines
2251	2252
2252	2253	t = btf__type_by_id(btf, func_info.type_id);
2253	2254	if (!t) {
2254		− p_err("btf %d doesn't have type %d",
	2255	+ p_err("btf %u doesn't have type %u",
2255	2256	info.btf_id, func_info.type_id);
2256	2257	goto out;
2257	2258	}
@@ -2329,7 +2330,7 @@ static int profile_open_perf_events(struct profiler_bpf *obj) · 71 unmodified lines
2329	2330	continue;
2330	2331	for (cpu = 0; cpu < obj->rodata->num_cpu; cpu++) {
2331	2332	if (profile_open_perf_event(m, cpu, map_fd)) {
2332		− p_err("failed to create event %s on cpu %d",
	2333	+ p_err("failed to create event %s on cpu %u",
2333	2334	metrics[m].name, cpu);
2334	2335	return -1;
2335	2336	}

tools/bpf/bpftool/tracelog.c+1−1

@@ -78,7 +78,7 @@ static bool get_tracefs_pipe(char *mnt) · 77 unmodified lines
78	78	return false;
79	79
80	80	/* Allow room for NULL terminating byte and pipe file name */
81		− snprintf(format, sizeof(format), "%%s %%%zds %%99s %%s %%d %%d\\n",
	81	+ snprintf(format, sizeof(format), "%%s %%%zus %%99s %%s %%d %%d\\n",
82	82	PATH_MAX - strlen(pipe_name) - 1);
83	83	while (fscanf(fp, format, mnt, type) == 2)
84	84	if (strcmp(type, fstype) == 0) {

tools/bpf/bpftool/xlated_dumper.c+3−3

@@ -199,13 +199,13 @@ static const char print_imm(void private_data, · 198 unmodified lines
199	199
200	200	if (insn->src_reg == BPF_PSEUDO_MAP_FD)
201	201	snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
202		− "map[id:%u]", insn->imm);
	202	+ "map[id:%d]", insn->imm);
203	203	else if (insn->src_reg == BPF_PSEUDO_MAP_VALUE)
204	204	snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
205		− "map[id:%u][0]+%u", insn->imm, (insn + 1)->imm);
	205	+ "map[id:%d][0]+%d", insn->imm, (insn + 1)->imm);
206	206	else if (insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE)
207	207	snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
208		− "map[idx:%u]+%u", insn->imm, (insn + 1)->imm);
	208	+ "map[idx:%d]+%d", insn->imm, (insn + 1)->imm);
209	209	else if (insn->src_reg == BPF_PSEUDO_FUNC)
210	210	snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
211	211	"subprog[%+d]", insn->imm);

tools/bpf/runqslower/Makefile+2−1

@@ -6,6 +6,7 @@ OUTPUT ?= $(abspath .output)/ · 5 unmodified lines
6	6	BPFTOOL_OUTPUT := $(OUTPUT)bpftool/
7	7	DEFAULT_BPFTOOL := $(BPFTOOL_OUTPUT)bootstrap/bpftool
8	8	BPFTOOL ?= $(DEFAULT_BPFTOOL)
	9	+BPF_TARGET_ENDIAN ?= --target=bpf
9	10	LIBBPF_SRC := $(abspath ../../lib/bpf)
10	11	BPFOBJ_OUTPUT := $(OUTPUT)libbpf/
11	12	BPFOBJ := $(BPFOBJ_OUTPUT)libbpf.a
@@ -60,7 +61,7 @@ $(OUTPUT)/%.skel.h: $(OUTPUT)/%.bpf.o \| $(BPFTOOL) · 48 unmodified lines
60	61	$(QUIET_GEN)$(BPFTOOL) gen skeleton $< > $@
61	62
62	63	$(OUTPUT)/%.bpf.o: %.bpf.c $(BPFOBJ) \| $(OUTPUT)
63		− $(QUIET_GEN)$(CLANG) -g -O2 --target=bpf $(INCLUDES) \
	64	+ $(QUIET_GEN)$(CLANG) -g -O2 $(BPF_TARGET_ENDIAN) $(INCLUDES) \
64	65	-c $(filter %.c,$^) -o $@ && \
65	66	$(LLVM_STRIP) -g $@
66	67

tools/include/uapi/linux/bpf.h+9−1

@@ -51,6 +51,9 @@ · 50 unmodified lines
51	51	#define BPF_XCHG (0xe0 \| BPF_FETCH) /* atomic exchange */
52	52	#define BPF_CMPXCHG (0xf0 \| BPF_FETCH) /* atomic compare-and-write */
53	53
	54	+#define BPF_LOAD_ACQ 0x100 /* load-acquire */
	55	+#define BPF_STORE_REL 0x110 /* store-release */
	56	+
54	57	enum bpf_cond_pseudo_jmp {
55	58	BPF_MAY_GOTO = 0,
56	59	};
@@ -1207,6 +1210,7 @@ enum bpf_perf_event_type { · 1150 unmodified lines
1207	1210	#define BPF_F_BEFORE (1U << 3)
1208	1211	#define BPF_F_AFTER (1U << 4)
1209	1212	#define BPF_F_ID (1U << 5)
	1213	+#define BPF_F_PREORDER (1U << 6)
1210	1214	#define BPF_F_LINK BPF_F_LINK /* 1 << 13 */
1211	1215
1212	1216	/* If BPF_F_STRICT_ALIGNMENT is used in BPF_PROG_LOAD command, the
@@ -1648,6 +1652,7 @@ union bpf_attr { · 435 unmodified lines
1648	1652	};
1649	1653	__u32 next_id;
1650	1654	__u32 open_flags;
	1655	+ __s32 fd_by_id_token_fd;
1651	1656	};
1652	1657
1653	1658	struct { /* anonymous struct used by BPF_OBJ_GET_INFO_BY_FD */
@@ -6019,7 +6024,10 @@ union bpf_attr { · 4365 unmodified lines
6019	6024	FN(user_ringbuf_drain, 209, ##ctx) \
6020	6025	FN(cgrp_storage_get, 210, ##ctx) \
6021	6026	FN(cgrp_storage_delete, 211, ##ctx) \
6022		− /* */
	6027	+ /* This helper list is effectively frozen. If you are trying to \
	6028	+ * add a new helper, you should add a kfunc instead which has \
	6029	+ * less stability guarantees. See Documentation/bpf/kfuncs.rst \
	6030	+ */
6023	6031
6024	6032	/* backwards-compatibility macros for users of __BPF_FUNC_MAPPER that don't
6025	6033	* know or care about integer value that is now passed as second argument

tools/include/uapi/linux/btf.h+2−1

@@ -36,7 +36,8 @@ struct btf_type { · 35 unmodified lines
36	36	* bits 24-28: kind (e.g. int, ptr, array...etc)
37	37	* bits 29-30: unused
38	38	* bit 31: kind_flag, currently used by
39		− * struct, union, enum, fwd and enum64
	39	+ * struct, union, enum, fwd, enum64,
	40	+ * decl_tag and type_tag
40	41	*/
41	42	__u32 info;
42	43	/* "size" is used by INT, ENUM, STRUCT, UNION, DATASEC and ENUM64.

tools/lib/bpf/bpf.c+2−1

@@ -1097,7 +1097,7 @@ int bpf_map_get_fd_by_id(__u32 id) · 1096 unmodified lines
1097	1097	int bpf_btf_get_fd_by_id_opts(__u32 id,
1098	1098	const struct bpf_get_fd_by_id_opts *opts)
1099	1099	{
1100		− const size_t attr_sz = offsetofend(union bpf_attr, open_flags);
	1100	+ const size_t attr_sz = offsetofend(union bpf_attr, fd_by_id_token_fd);
1101	1101	union bpf_attr attr;
1102	1102	int fd;
1103	1103
@@ -1107,6 +1107,7 @@ int bpf_btf_get_fd_by_id_opts(__u32 id, · 3 unmodified lines
1107	1107	memset(&attr, 0, attr_sz);
1108	1108	attr.btf_id = id;
1109	1109	attr.open_flags = OPTS_GET(opts, open_flags, 0);
	1110	+ attr.fd_by_id_token_fd = OPTS_GET(opts, token_fd, 0);
1110	1111
1111	1112	fd = sys_bpf_fd(BPF_BTF_GET_FD_BY_ID, &attr, attr_sz);
1112	1113	return libbpf_err_errno(fd);

tools/lib/bpf/bpf.h+2−1

@@ -487,9 +487,10 @@ LIBBPF_API int bpf_link_get_next_id(__u32 start_id, __u32 *next_id); · 486 unmodified lines
487	487	struct bpf_get_fd_by_id_opts {
488	488	size_t sz; /* size of this struct for forward/backward compatibility */
489	489	__u32 open_flags; /* permissions requested for the operation on fd */
	490	+ __u32 token_fd;
490	491	size_t :0;
491	492	};
492		−#define bpf_get_fd_by_id_opts__last_field open_flags
	493	+#define bpf_get_fd_by_id_opts__last_field token_fd
493	494
494	495	LIBBPF_API int bpf_prog_get_fd_by_id(__u32 id);
495	496	LIBBPF_API int bpf_prog_get_fd_by_id_opts(__u32 id,

tools/lib/bpf/btf.c+78−27

@@ -1619,12 +1619,18 @@ exit_free: · 1618 unmodified lines
1619	1619	return btf;
1620	1620	}
1621	1621
1622		−struct btf btf__load_from_kernel_by_id_split(__u32 id, struct btf base_btf)
	1622	+struct btf btf_load_from_kernel(__u32 id, struct btf base_btf, int token_fd)
1623	1623	{
1624	1624	struct btf *btf;
1625	1625	int btf_fd;
	1626	+ LIBBPF_OPTS(bpf_get_fd_by_id_opts, opts);
	1627	+
	1628	+ if (token_fd) {
	1629	+ opts.open_flags \|= BPF_F_TOKEN_FD;
	1630	+ opts.token_fd = token_fd;
	1631	+ }
1626	1632
1627		− btf_fd = bpf_btf_get_fd_by_id(id);
	1633	+ btf_fd = bpf_btf_get_fd_by_id_opts(id, &opts);
1628	1634	if (btf_fd < 0)
1629	1635	return libbpf_err_ptr(-errno);
1630	1636
@@ -1634,6 +1640,11 @@ struct btf btf__load_from_kernel_by_id_split(__u32 id, struct btf base_btf) · 3 unmodified lines
1634	1640	return libbpf_ptr(btf);
1635	1641	}
1636	1642
	1643	+struct btf btf__load_from_kernel_by_id_split(__u32 id, struct btf base_btf)
	1644	+{
	1645	+ return btf_load_from_kernel(id, base_btf, 0);
	1646	+}
	1647	+
1637	1648	struct btf *btf__load_from_kernel_by_id(__u32 id)
1638	1649	{
1639	1650	return btf__load_from_kernel_by_id_split(id, NULL);
@@ -2090,7 +2101,7 @@ static int validate_type_id(int id) · 450 unmodified lines
2090	2101	}
2091	2102
2092	2103	/* generic append function for PTR, TYPEDEF, CONST/VOLATILE/RESTRICT */
2093		−static int btf_add_ref_kind(struct btf btf, int kind, const char name, int ref_type_id)
	2104	+static int btf_add_ref_kind(struct btf btf, int kind, const char name, int ref_type_id, int kflag)
2094	2105	{
2095	2106	struct btf_type *t;
2096	2107	int sz, name_off = 0;
@@ -2113,7 +2124,7 @@ static int btf_add_ref_kind(struct btf btf, int kind, const char name, int ref · 16 unmodified lines
2113	2124	}
2114	2125
2115	2126	t->name_off = name_off;
2116		− t->info = btf_type_info(kind, 0, 0);
	2127	+ t->info = btf_type_info(kind, 0, kflag);
2117	2128	t->type = ref_type_id;
2118	2129
2119	2130	return btf_commit_type(btf, sz);
@@ -2128,7 +2139,7 @@ static int btf_add_ref_kind(struct btf btf, int kind, const char name, int ref · 8 unmodified lines
2128	2139	*/
2129	2140	int btf__add_ptr(struct btf *btf, int ref_type_id)
2130	2141	{
2131		− return btf_add_ref_kind(btf, BTF_KIND_PTR, NULL, ref_type_id);
	2142	+ return btf_add_ref_kind(btf, BTF_KIND_PTR, NULL, ref_type_id, 0);
2132	2143	}
2133	2144
2134	2145	/*
@@ -2506,7 +2517,7 @@ int btf__add_fwd(struct btf btf, const char name, enum btf_fwd_kind fwd_kind) · 371 unmodified lines
2506	2517	struct btf_type *t;
2507	2518	int id;
2508	2519
2509		− id = btf_add_ref_kind(btf, BTF_KIND_FWD, name, 0);
	2520	+ id = btf_add_ref_kind(btf, BTF_KIND_FWD, name, 0, 0);
2510	2521	if (id <= 0)
2511	2522	return id;
2512	2523	t = btf_type_by_id(btf, id);
@@ -2536,7 +2547,7 @@ int btf__add_typedef(struct btf btf, const char name, int ref_type_id) · 23 unmodified lines
2536	2547	if (!name \|\| !name[0])
2537	2548	return libbpf_err(-EINVAL);
2538	2549
2539		− return btf_add_ref_kind(btf, BTF_KIND_TYPEDEF, name, ref_type_id);
	2550	+ return btf_add_ref_kind(btf, BTF_KIND_TYPEDEF, name, ref_type_id, 0);
2540	2551	}
2541	2552
2542	2553	/*
@@ -2548,7 +2559,7 @@ int btf__add_typedef(struct btf btf, const char name, int ref_type_id) · 5 unmodified lines
2548	2559	*/
2549	2560	int btf__add_volatile(struct btf *btf, int ref_type_id)
2550	2561	{
2551		− return btf_add_ref_kind(btf, BTF_KIND_VOLATILE, NULL, ref_type_id);
	2562	+ return btf_add_ref_kind(btf, BTF_KIND_VOLATILE, NULL, ref_type_id, 0);
2552	2563	}
2553	2564
2554	2565	/*
@@ -2560,7 +2571,7 @@ int btf__add_volatile(struct btf *btf, int ref_type_id) · 5 unmodified lines
2560	2571	*/
2561	2572	int btf__add_const(struct btf *btf, int ref_type_id)
2562	2573	{
2563		− return btf_add_ref_kind(btf, BTF_KIND_CONST, NULL, ref_type_id);
	2574	+ return btf_add_ref_kind(btf, BTF_KIND_CONST, NULL, ref_type_id, 0);
2564	2575	}
2565	2576
2566	2577	/*
@@ -2572,7 +2583,7 @@ int btf__add_const(struct btf *btf, int ref_type_id) · 5 unmodified lines
2572	2583	*/
2573	2584	int btf__add_restrict(struct btf *btf, int ref_type_id)
2574	2585	{
2575		− return btf_add_ref_kind(btf, BTF_KIND_RESTRICT, NULL, ref_type_id);
	2586	+ return btf_add_ref_kind(btf, BTF_KIND_RESTRICT, NULL, ref_type_id, 0);
2576	2587	}
2577	2588
2578	2589	/*
@@ -2588,7 +2599,24 @@ int btf__add_type_tag(struct btf btf, const char value, int ref_type_id) · 9 unmodified lines
2588	2599	if (!value \|\| !value[0])
2589	2600	return libbpf_err(-EINVAL);
2590	2601
2591		− return btf_add_ref_kind(btf, BTF_KIND_TYPE_TAG, value, ref_type_id);
	2602	+ return btf_add_ref_kind(btf, BTF_KIND_TYPE_TAG, value, ref_type_id, 0);
	2603	+}
	2604	+
	2605	+/*
	2606	+ * Append new BTF_KIND_TYPE_TAG type with:
	2607	+ * - value, non-empty/non-NULL tag value;
	2608	+ * - ref_type_id - referenced type ID, it might not exist yet;
	2609	+ * Set info->kflag to 1, indicating this tag is an __attribute__
	2610	+ * Returns:
	2611	+ * - >0, type ID of newly added BTF type;
	2612	+ * - <0, on error.
	2613	+ */
	2614	+int btf__add_type_attr(struct btf btf, const char value, int ref_type_id)
	2615	+{
	2616	+ if (!value \|\| !value[0])
	2617	+ return libbpf_err(-EINVAL);
	2618	+
	2619	+ return btf_add_ref_kind(btf, BTF_KIND_TYPE_TAG, value, ref_type_id, 1);
2592	2620	}
2593	2621
2594	2622	/*
@@ -2610,7 +2638,7 @@ int btf__add_func(struct btf btf, const char name, · 15 unmodified lines
2610	2638	linkage != BTF_FUNC_EXTERN)
2611	2639	return libbpf_err(-EINVAL);
2612	2640
2613		− id = btf_add_ref_kind(btf, BTF_KIND_FUNC, name, proto_type_id);
	2641	+ id = btf_add_ref_kind(btf, BTF_KIND_FUNC, name, proto_type_id, 0);
2614	2642	if (id > 0) {
2615	2643	struct btf_type *t = btf_type_by_id(btf, id);
2616	2644
@@ -2845,18 +2873,8 @@ int btf__add_datasec_var_info(struct btf *btf, int var_type_id, __u32 offset, __ · 228 unmodified lines
2845	2873	return 0;
2846	2874	}
2847	2875
2848		−/*
2849		− * Append new BTF_KIND_DECL_TAG type with:
2850		− * - value - non-empty/non-NULL string;
2851		− * - ref_type_id - referenced type ID, it might not exist yet;
2852		− * - component_idx - -1 for tagging reference type, otherwise struct/union
2853		− * member or function argument index;
2854		− * Returns:
2855		− * - >0, type ID of newly added BTF type;
2856		− * - <0, on error.
2857		− */
2858		−int btf__add_decl_tag(struct btf btf, const char value, int ref_type_id,
2859		− int component_idx)
	2876	+static int btf_add_decl_tag(struct btf btf, const char value, int ref_type_id,
	2877	+ int component_idx, int kflag)
2860	2878	{
2861	2879	struct btf_type *t;
2862	2880	int sz, value_off;
@@ -2880,13 +2898,46 @@ int btf__add_decl_tag(struct btf btf, const char value, int ref_type_id, · 17 unmodified lines
2880	2898	return value_off;
2881	2899
2882	2900	t->name_off = value_off;
2883		− t->info = btf_type_info(BTF_KIND_DECL_TAG, 0, false);
	2901	+ t->info = btf_type_info(BTF_KIND_DECL_TAG, 0, kflag);
2884	2902	t->type = ref_type_id;
2885	2903	btf_decl_tag(t)->component_idx = component_idx;
2886	2904
2887	2905	return btf_commit_type(btf, sz);
2888	2906	}
2889	2907
	2908	+/*
	2909	+ * Append new BTF_KIND_DECL_TAG type with:
	2910	+ * - value - non-empty/non-NULL string;
	2911	+ * - ref_type_id - referenced type ID, it might not exist yet;
	2912	+ * - component_idx - -1 for tagging reference type, otherwise struct/union
	2913	+ * member or function argument index;
	2914	+ * Returns:
	2915	+ * - >0, type ID of newly added BTF type;
	2916	+ * - <0, on error.
	2917	+ */
	2918	+int btf__add_decl_tag(struct btf btf, const char value, int ref_type_id,
	2919	+ int component_idx)
	2920	+{
	2921	+ return btf_add_decl_tag(btf, value, ref_type_id, component_idx, 0);
	2922	+}
	2923	+
	2924	+/*
	2925	+ * Append new BTF_KIND_DECL_TAG type with:
	2926	+ * - value - non-empty/non-NULL string;
	2927	+ * - ref_type_id - referenced type ID, it might not exist yet;
	2928	+ * - component_idx - -1 for tagging reference type, otherwise struct/union
	2929	+ * member or function argument index;
	2930	+ * Set info->kflag to 1, indicating this tag is an __attribute__
	2931	+ * Returns:
	2932	+ * - >0, type ID of newly added BTF type;
	2933	+ * - <0, on error.
	2934	+ */
	2935	+int btf__add_decl_attr(struct btf btf, const char value, int ref_type_id,
	2936	+ int component_idx)
	2937	+{
	2938	+ return btf_add_decl_tag(btf, value, ref_type_id, component_idx, 1);
	2939	+}
	2940	+
2890	2941	struct btf_ext_sec_info_param {
2891	2942	__u32 off;
2892	2943	__u32 len;
@@ -3015,8 +3066,6 @@ static int btf_ext_parse_info(struct btf_ext *btf_ext, bool is_native) · 122 unmodified lines
3015	3066	.desc = "line_info",
3016	3067	};
3017	3068	struct btf_ext_sec_info_param core_relo = {
3018		− .off = btf_ext->hdr->core_relo_off,
3019		− .len = btf_ext->hdr->core_relo_len,
3020	3069	.min_rec_size = sizeof(struct bpf_core_relo),
3021	3070	.ext_info = &btf_ext->core_relo_info,
3022	3071	.desc = "core_relo",
@@ -3034,6 +3083,8 @@ static int btf_ext_parse_info(struct btf_ext *btf_ext, bool is_native) · 11 unmodified lines
3034	3083	if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, core_relo_len))
3035	3084	return 0; /* skip core relos parsing */
3036	3085
	3086	+ core_relo.off = btf_ext->hdr->core_relo_off;
	3087	+ core_relo.len = btf_ext->hdr->core_relo_len;
3037	3088	err = btf_ext_parse_sec_info(btf_ext, &core_relo, is_native);
3038	3089	if (err)
3039	3090	return err;

tools/lib/bpf/btf.h+3−0

@@ -227,6 +227,7 @@ LIBBPF_API int btf__add_volatile(struct btf *btf, int ref_type_id); · 226 unmodified lines
227	227	LIBBPF_API int btf__add_const(struct btf *btf, int ref_type_id);
228	228	LIBBPF_API int btf__add_restrict(struct btf *btf, int ref_type_id);
229	229	LIBBPF_API int btf__add_type_tag(struct btf btf, const char value, int ref_type_id);
	230	+LIBBPF_API int btf__add_type_attr(struct btf btf, const char value, int ref_type_id);
230	231
231	232	/* func and func_proto construction APIs */
232	233	LIBBPF_API int btf__add_func(struct btf btf, const char name,
@@ -243,6 +244,8 @@ LIBBPF_API int btf__add_datasec_var_info(struct btf *btf, int var_type_id, · 10 unmodified lines
243	244	/* tag construction API */
244	245	LIBBPF_API int btf__add_decl_tag(struct btf btf, const char value, int ref_type_id,
245	246	int component_idx);
	247	+LIBBPF_API int btf__add_decl_attr(struct btf btf, const char value, int ref_type_id,
	248	+ int component_idx);
246	249
247	250	struct btf_dedup_opts {
248	251	size_t sz;

tools/lib/bpf/btf_dump.c+4−1

@@ -1494,7 +1494,10 @@ static void btf_dump_emit_type_chain(struct btf_dump *d, · 1493 unmodified lines
1494	1494	case BTF_KIND_TYPE_TAG:
1495	1495	btf_dump_emit_mods(d, decls);
1496	1496	name = btf_name_of(d, t->name_off);
1497		− btf_dump_printf(d, " __attribute__((btf_type_tag(\"%s\")))", name);
	1497	+ if (btf_kflag(t))
	1498	+ btf_dump_printf(d, " __attribute__((%s))", name);
	1499	+ else
	1500	+ btf_dump_printf(d, " __attribute__((btf_type_tag(\"%s\")))", name);
1498	1501	break;
1499	1502	case BTF_KIND_ARRAY: {
1500	1503	const struct btf_array *a = btf_array(t);

tools/lib/bpf/libbpf.c+151−86

@@ -670,11 +670,18 @@ struct elf_state { · 669 unmodified lines
670	670
671	671	struct usdt_manager;
672	672
	673	+enum bpf_object_state {
	674	+ OBJ_OPEN,
	675	+ OBJ_PREPARED,
	676	+ OBJ_LOADED,
	677	+};
	678	+
673	679	struct bpf_object {
674	680	char name[BPF_OBJ_NAME_LEN];
675	681	char license[64];
676	682	__u32 kern_version;
677	683
	684	+ enum bpf_object_state state;
678	685	struct bpf_program *programs;
679	686	size_t nr_programs;
680	687	struct bpf_map *maps;
@@ -686,7 +693,6 @@ struct bpf_object { · 5 unmodified lines
686	693	int nr_extern;
687	694	int kconfig_map_idx;
688	695
689		− bool loaded;
690	696	bool has_subcalls;
691	697	bool has_rodata;
692	698
@@ -1511,7 +1517,7 @@ static struct bpf_object bpf_object__new(const char path, · 818 unmodified lines
1511	1517	obj->kconfig_map_idx = -1;
1512	1518
1513	1519	obj->kern_version = get_kernel_version();
1514		− obj->loaded = false;
	1520	+ obj->state = OBJ_OPEN;
1515	1521
1516	1522	return obj;
1517	1523	}
@@ -2106,7 +2112,7 @@ static int set_kcfg_value_str(struct extern_desc ext, char ext_val, · 588 unmodified lines
2106	2112	}
2107	2113
2108	2114	len = strlen(value);
2109		− if (value[len - 1] != '"') {
	2115	+ if (len < 2 \|\| value[len - 1] != '"') {
2110	2116	pr_warn("extern (kcfg) '%s': invalid string config '%s'\n",
2111	2117	ext->name, value);
2112	2118	return -EINVAL;
@@ -4845,6 +4851,11 @@ static int bpf_get_map_info_from_fdinfo(int fd, struct bpf_map_info *info) · 2732 unmodified lines
4845	4851	return 0;
4846	4852	}
4847	4853
	4854	+static bool map_is_created(const struct bpf_map *map)
	4855	+{
	4856	+ return map->obj->state >= OBJ_PREPARED \|\| map->reused;
	4857	+}
	4858	+
4848	4859	bool bpf_map__autocreate(const struct bpf_map *map)
4849	4860	{
4850	4861	return map->autocreate;
@@ -4852,7 +4863,7 @@ bool bpf_map__autocreate(const struct bpf_map *map) · 1 unmodified line
4852	4863
4853	4864	int bpf_map__set_autocreate(struct bpf_map *map, bool autocreate)
4854	4865	{
4855		− if (map->obj->loaded)
	4866	+ if (map_is_created(map))
4856	4867	return libbpf_err(-EBUSY);
4857	4868
4858	4869	map->autocreate = autocreate;
@@ -4946,7 +4957,7 @@ struct bpf_map bpf_map__inner_map(struct bpf_map map) · 87 unmodified lines
4946	4957
4947	4958	int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries)
4948	4959	{
4949		− if (map->obj->loaded)
	4960	+ if (map_is_created(map))
4950	4961	return libbpf_err(-EBUSY);
4951	4962
4952	4963	map->def.max_entries = max_entries;
@@ -5191,11 +5202,6 @@ bpf_object__populate_internal_map(struct bpf_object obj, struct bpf_map map) · 238 unmodified lines
5191	5202
5192	5203	static void bpf_map__destroy(struct bpf_map *map);
5193	5204
5194		−static bool map_is_created(const struct bpf_map *map)
5195		−{
5196		− return map->obj->loaded \|\| map->reused;
5197		−}
5198		−
5199	5205	static int bpf_object__create_map(struct bpf_object obj, struct bpf_map map, bool is_inner)
5200	5206	{
5201	5207	LIBBPF_OPTS(bpf_map_create_opts, create_attr);
@@ -7895,13 +7901,6 @@ bpf_object__load_progs(struct bpf_object *obj, int log_level) · 2693 unmodified lines
7895	7901	size_t i;
7896	7902	int err;
7897	7903
7898		− for (i = 0; i < obj->nr_programs; i++) {
7899		− prog = &obj->programs[i];
7900		− err = bpf_object__sanitize_prog(obj, prog);
7901		− if (err)
7902		− return err;
7903		− }
7904		−
7905	7904	for (i = 0; i < obj->nr_programs; i++) {
7906	7905	prog = &obj->programs[i];
7907	7906	if (prog_is_subprog(obj, prog))
@@ -7927,6 +7926,21 @@ bpf_object__load_progs(struct bpf_object *obj, int log_level) · 19 unmodified lines
7927	7926	return 0;
7928	7927	}
7929	7928
	7929	+static int bpf_object_prepare_progs(struct bpf_object *obj)
	7930	+{
	7931	+ struct bpf_program *prog;
	7932	+ size_t i;
	7933	+ int err;
	7934	+
	7935	+ for (i = 0; i < obj->nr_programs; i++) {
	7936	+ prog = &obj->programs[i];
	7937	+ err = bpf_object__sanitize_prog(obj, prog);
	7938	+ if (err)
	7939	+ return err;
	7940	+ }
	7941	+ return 0;
	7942	+}
	7943	+
7930	7944	static const struct bpf_sec_def find_sec_def(const char sec_name);
7931	7945
7932	7946	static int bpf_object_init_progs(struct bpf_object obj, const struct bpf_object_open_opts opts)
@@ -8543,14 +8557,77 @@ static int bpf_object_prepare_struct_ops(struct bpf_object *obj) · 610 unmodified lines
8543	8557	return 0;
8544	8558	}
8545	8559
	8560	+static void bpf_object_unpin(struct bpf_object *obj)
	8561	+{
	8562	+ int i;
	8563	+
	8564	+ /* unpin any maps that were auto-pinned during load */
	8565	+ for (i = 0; i < obj->nr_maps; i++)
	8566	+ if (obj->maps[i].pinned && !obj->maps[i].reused)
	8567	+ bpf_map__unpin(&obj->maps[i], NULL);
	8568	+}
	8569	+
	8570	+static void bpf_object_post_load_cleanup(struct bpf_object *obj)
	8571	+{
	8572	+ int i;
	8573	+
	8574	+ /* clean up fd_array */
	8575	+ zfree(&obj->fd_array);
	8576	+
	8577	+ /* clean up module BTFs */
	8578	+ for (i = 0; i < obj->btf_module_cnt; i++) {
	8579	+ close(obj->btf_modules[i].fd);
	8580	+ btf__free(obj->btf_modules[i].btf);
	8581	+ free(obj->btf_modules[i].name);
	8582	+ }
	8583	+ obj->btf_module_cnt = 0;
	8584	+ zfree(&obj->btf_modules);
	8585	+
	8586	+ /* clean up vmlinux BTF */
	8587	+ btf__free(obj->btf_vmlinux);
	8588	+ obj->btf_vmlinux = NULL;
	8589	+}
	8590	+
	8591	+static int bpf_object_prepare(struct bpf_object obj, const char target_btf_path)
	8592	+{
	8593	+ int err;
	8594	+
	8595	+ if (obj->state >= OBJ_PREPARED) {
	8596	+ pr_warn("object '%s': prepare loading can't be attempted twice\n", obj->name);
	8597	+ return -EINVAL;
	8598	+ }
	8599	+
	8600	+ err = bpf_object_prepare_token(obj);
	8601	+ err = err ? : bpf_object__probe_loading(obj);
	8602	+ err = err ? : bpf_object__load_vmlinux_btf(obj, false);
	8603	+ err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
	8604	+ err = err ? : bpf_object__sanitize_maps(obj);
	8605	+ err = err ? : bpf_object__init_kern_struct_ops_maps(obj);
	8606	+ err = err ? : bpf_object_adjust_struct_ops_autoload(obj);
	8607	+ err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : target_btf_path);
	8608	+ err = err ? : bpf_object__sanitize_and_load_btf(obj);
	8609	+ err = err ? : bpf_object__create_maps(obj);
	8610	+ err = err ? : bpf_object_prepare_progs(obj);
	8611	+
	8612	+ if (err) {
	8613	+ bpf_object_unpin(obj);
	8614	+ bpf_object_unload(obj);
	8615	+ obj->state = OBJ_LOADED;
	8616	+ return err;
	8617	+ }
	8618	+
	8619	+ obj->state = OBJ_PREPARED;
	8620	+ return 0;
	8621	+}
	8622	+
8546	8623	static int bpf_object_load(struct bpf_object obj, int extra_log_level, const char target_btf_path)
8547	8624	{
8548		− int err, i;
	8625	+ int err;
8549	8626
8550	8627	if (!obj)
8551	8628	return libbpf_err(-EINVAL);
8552	8629
8553		− if (obj->loaded) {
	8630	+ if (obj->state >= OBJ_LOADED) {
8554	8631	pr_warn("object '%s': load can't be attempted twice\n", obj->name);
8555	8632	return libbpf_err(-EINVAL);
8556	8633	}
@@ -8565,17 +8642,12 @@ static int bpf_object_load(struct bpf_object *obj, int extra_log_level, const ch · 8 unmodified lines
8565	8642	return libbpf_err(-LIBBPF_ERRNO__ENDIAN);
8566	8643	}
8567	8644
8568		− err = bpf_object_prepare_token(obj);
8569		− err = err ? : bpf_object__probe_loading(obj);
8570		− err = err ? : bpf_object__load_vmlinux_btf(obj, false);
8571		− err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
8572		− err = err ? : bpf_object__sanitize_maps(obj);
8573		− err = err ? : bpf_object__init_kern_struct_ops_maps(obj);
8574		− err = err ? : bpf_object_adjust_struct_ops_autoload(obj);
8575		− err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : target_btf_path);
8576		− err = err ? : bpf_object__sanitize_and_load_btf(obj);
8577		− err = err ? : bpf_object__create_maps(obj);
8578		− err = err ? : bpf_object__load_progs(obj, extra_log_level);
	8645	+ if (obj->state < OBJ_PREPARED) {
	8646	+ err = bpf_object_prepare(obj, target_btf_path);
	8647	+ if (err)
	8648	+ return libbpf_err(err);
	8649	+ }
	8650	+ err = bpf_object__load_progs(obj, extra_log_level);
8579	8651	err = err ? : bpf_object_init_prog_arrays(obj);
8580	8652	err = err ? : bpf_object_prepare_struct_ops(obj);
8581	8653
@@ -8587,36 +8659,22 @@ static int bpf_object_load(struct bpf_object *obj, int extra_log_level, const ch · 5 unmodified lines
8587	8659	err = bpf_gen__finish(obj->gen_loader, obj->nr_programs, obj->nr_maps);
8588	8660	}
8589	8661
8590		− /* clean up fd_array */
8591		− zfree(&obj->fd_array);
	8662	+ bpf_object_post_load_cleanup(obj);
	8663	+ obj->state = OBJ_LOADED; /* doesn't matter if successfully or not */
8592	8664
8593		− /* clean up module BTFs */
8594		− for (i = 0; i < obj->btf_module_cnt; i++) {
8595		− close(obj->btf_modules[i].fd);
8596		− btf__free(obj->btf_modules[i].btf);
8597		− free(obj->btf_modules[i].name);
	8665	+ if (err) {
	8666	+ bpf_object_unpin(obj);
	8667	+ bpf_object_unload(obj);
	8668	+ pr_warn("failed to load object '%s'\n", obj->path);
	8669	+ return libbpf_err(err);
8598	8670	}
8599		− free(obj->btf_modules);
8600		−
8601		− /* clean up vmlinux BTF */
8602		− btf__free(obj->btf_vmlinux);
8603		− obj->btf_vmlinux = NULL;
8604		−
8605		− obj->loaded = true; /* doesn't matter if successfully or not */
8606		−
8607		− if (err)
8608		− goto out;
8609	8671
8610	8672	return 0;
8611		−out:
8612		− /* unpin any maps that were auto-pinned during load */
8613		− for (i = 0; i < obj->nr_maps; i++)
8614		− if (obj->maps[i].pinned && !obj->maps[i].reused)
8615		− bpf_map__unpin(&obj->maps[i], NULL);
	8673	+}
8616	8674
8617		− bpf_object_unload(obj);
8618		− pr_warn("failed to load object '%s'\n", obj->path);
8619		− return libbpf_err(err);
	8675	+int bpf_object__prepare(struct bpf_object *obj)
	8676	+{
	8677	+ return libbpf_err(bpf_object_prepare(obj, NULL));
8620	8678	}
8621	8679
8622	8680	int bpf_object__load(struct bpf_object *obj)
@@ -8866,7 +8924,7 @@ int bpf_object__pin_maps(struct bpf_object obj, const char path) · 243 unmodified lines
8866	8924	if (!obj)
8867	8925	return libbpf_err(-ENOENT);
8868	8926
8869		− if (!obj->loaded) {
	8927	+ if (obj->state < OBJ_PREPARED) {
8870	8928	pr_warn("object not yet loaded; load it first\n");
8871	8929	return libbpf_err(-ENOENT);
8872	8930	}
@@ -8945,7 +9003,7 @@ int bpf_object__pin_programs(struct bpf_object obj, const char path) · 72 unmodified lines
8945	9003	if (!obj)
8946	9004	return libbpf_err(-ENOENT);
8947	9005
8948		− if (!obj->loaded) {
	9006	+ if (obj->state < OBJ_LOADED) {
8949	9007	pr_warn("object not yet loaded; load it first\n");
8950	9008	return libbpf_err(-ENOENT);
8951	9009	}
@@ -9064,6 +9122,13 @@ void bpf_object__close(struct bpf_object *obj) · 112 unmodified lines
9064	9122	if (IS_ERR_OR_NULL(obj))
9065	9123	return;
9066	9124
	9125	+ /*
	9126	+ * if user called bpf_object__prepare() without ever getting to
	9127	+ * bpf_object__load(), we need to clean up stuff that is normally
	9128	+ * cleaned up at the end of loading step
	9129	+ */
	9130	+ bpf_object_post_load_cleanup(obj);
	9131	+
9067	9132	usdt_manager_free(obj->usdt_man);
9068	9133	obj->usdt_man = NULL;
9069	9134
@@ -9132,7 +9197,7 @@ int bpf_object__btf_fd(const struct bpf_object *obj) · 62 unmodified lines
9132	9197
9133	9198	int bpf_object__set_kversion(struct bpf_object *obj, __u32 kern_version)
9134	9199	{
9135		− if (obj->loaded)
	9200	+ if (obj->state >= OBJ_LOADED)
9136	9201	return libbpf_err(-EINVAL);
9137	9202
9138	9203	obj->kern_version = kern_version;
@@ -9145,12 +9210,12 @@ int bpf_object__gen_loader(struct bpf_object obj, struct gen_loader_opts opts) · 6 unmodified lines
9145	9210	struct bpf_gen *gen;
9146	9211
9147	9212	if (!opts)
9148		− return -EFAULT;
	9213	+ return libbpf_err(-EFAULT);
9149	9214	if (!OPTS_VALID(opts, gen_loader_opts))
9150		− return -EINVAL;
	9215	+ return libbpf_err(-EINVAL);
9151	9216	gen = calloc(sizeof(*gen), 1);
9152	9217	if (!gen)
9153		− return -ENOMEM;
	9218	+ return libbpf_err(-ENOMEM);
9154	9219	gen->opts = opts;
9155	9220	gen->swapped_endian = !is_native_endianness(obj);
9156	9221	obj->gen_loader = gen;
@@ -9229,7 +9294,7 @@ bool bpf_program__autoload(const struct bpf_program *prog) · 72 unmodified lines
9229	9294
9230	9295	int bpf_program__set_autoload(struct bpf_program *prog, bool autoload)
9231	9296	{
9232		− if (prog->obj->loaded)
	9297	+ if (prog->obj->state >= OBJ_LOADED)
9233	9298	return libbpf_err(-EINVAL);
9234	9299
9235	9300	prog->autoload = autoload;
@@ -9261,14 +9326,14 @@ int bpf_program__set_insns(struct bpf_program *prog, · 25 unmodified lines
9261	9326	{
9262	9327	struct bpf_insn *insns;
9263	9328
9264		− if (prog->obj->loaded)
9265		− return -EBUSY;
	9329	+ if (prog->obj->state >= OBJ_LOADED)
	9330	+ return libbpf_err(-EBUSY);
9266	9331
9267	9332	insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns));
9268	9333	/* NULL is a valid return from reallocarray if the new count is zero */
9269	9334	if (!insns && new_insn_cnt) {
9270	9335	pr_warn("prog '%s': failed to realloc prog code\n", prog->name);
9271		− return -ENOMEM;
	9336	+ return libbpf_err(-ENOMEM);
9272	9337	}
9273	9338	memcpy(insns, new_insns, new_insn_cnt * sizeof(*insns));
9274	9339
@@ -9304,7 +9369,7 @@ static int last_custom_sec_def_handler_id; · 29 unmodified lines
9304	9369
9305	9370	int bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
9306	9371	{
9307		− if (prog->obj->loaded)
	9372	+ if (prog->obj->state >= OBJ_LOADED)
9308	9373	return libbpf_err(-EBUSY);
9309	9374
9310	9375	/* if type is not changed, do nothing */
@@ -9335,7 +9400,7 @@ enum bpf_attach_type bpf_program__expected_attach_type(const struct bpf_program · 24 unmodified lines
9335	9400	int bpf_program__set_expected_attach_type(struct bpf_program *prog,
9336	9401	enum bpf_attach_type type)
9337	9402	{
9338		− if (prog->obj->loaded)
	9403	+ if (prog->obj->state >= OBJ_LOADED)
9339	9404	return libbpf_err(-EBUSY);
9340	9405
9341	9406	prog->expected_attach_type = type;
@@ -9349,7 +9414,7 @@ __u32 bpf_program__flags(const struct bpf_program *prog) · 7 unmodified lines
9349	9414
9350	9415	int bpf_program__set_flags(struct bpf_program *prog, __u32 flags)
9351	9416	{
9352		− if (prog->obj->loaded)
	9417	+ if (prog->obj->state >= OBJ_LOADED)
9353	9418	return libbpf_err(-EBUSY);
9354	9419
9355	9420	prog->prog_flags = flags;
@@ -9363,7 +9428,7 @@ __u32 bpf_program__log_level(const struct bpf_program *prog) · 7 unmodified lines
9363	9428
9364	9429	int bpf_program__set_log_level(struct bpf_program *prog, __u32 log_level)
9365	9430	{
9366		− if (prog->obj->loaded)
	9431	+ if (prog->obj->state >= OBJ_LOADED)
9367	9432	return libbpf_err(-EBUSY);
9368	9433
9369	9434	prog->log_level = log_level;
@@ -9379,11 +9444,11 @@ const char bpf_program__log_buf(const struct bpf_program prog, size_t *log_siz · 9 unmodified lines
9379	9444	int bpf_program__set_log_buf(struct bpf_program prog, char log_buf, size_t log_size)
9380	9445	{
9381	9446	if (log_size && !log_buf)
9382		− return -EINVAL;
	9447	+ return libbpf_err(-EINVAL);
9383	9448	if (prog->log_size > UINT_MAX)
9384		− return -EINVAL;
9385		− if (prog->obj->loaded)
9386		− return -EBUSY;
	9449	+ return libbpf_err(-EINVAL);
	9450	+ if (prog->obj->state >= OBJ_LOADED)
	9451	+ return libbpf_err(-EBUSY);
9387	9452
9388	9453	prog->log_buf = log_buf;
9389	9454	prog->log_size = log_size;
@@ -9959,7 +10024,7 @@ int libbpf_find_vmlinux_btf_id(const char *name, · 569 unmodified lines
9959	10024	return libbpf_err(err);
9960	10025	}
9961	10026
9962		−static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd)
	10027	+static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd, int token_fd)
9963	10028	{
9964	10029	struct bpf_prog_info info;
9965	10030	__u32 info_len = sizeof(info);
@@ -9979,7 +10044,7 @@ static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd) · 13 unmodified lines
9979	10044	pr_warn("The target program doesn't have BTF\n");
9980	10045	goto out;
9981	10046	}
9982		− btf = btf__load_from_kernel_by_id(info.btf_id);
	10047	+ btf = btf_load_from_kernel(info.btf_id, NULL, token_fd);
9983	10048	err = libbpf_get_error(btf);
9984	10049	if (err) {
9985	10050	pr_warn("Failed to get BTF %d of the program: %s\n", info.btf_id, errstr(err));
@@ -10062,7 +10127,7 @@ static int libbpf_find_attach_btf_id(struct bpf_program prog, const char attac · 76 unmodified lines
10062	10127	pr_warn("prog '%s': attach program FD is not set\n", prog->name);
10063	10128	return -EINVAL;
10064	10129	}
10065		− err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd);
	10130	+ err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd, prog->obj->token_fd);
10066	10131	if (err < 0) {
10067	10132	pr_warn("prog '%s': failed to find BPF program (FD %d) BTF ID for '%s': %s\n",
10068	10133	prog->name, attach_prog_fd, attach_name, errstr(err));
@@ -10299,7 +10364,7 @@ static int map_btf_datasec_resize(struct bpf_map *map, __u32 size) · 230 unmodified lines
10299	10364
10300	10365	int bpf_map__set_value_size(struct bpf_map *map, __u32 size)
10301	10366	{
10302		− if (map->obj->loaded \|\| map->reused)
	10367	+ if (map_is_created(map))
10303	10368	return libbpf_err(-EBUSY);
10304	10369
10305	10370	if (map->mmaped) {
@@ -10307,7 +10372,7 @@ int bpf_map__set_value_size(struct bpf_map *map, __u32 size) · 1 unmodified line
10307	10372	int err;
10308	10373
10309	10374	if (map->def.type != BPF_MAP_TYPE_ARRAY)
10310		− return -EOPNOTSUPP;
	10375	+ return libbpf_err(-EOPNOTSUPP);
10311	10376
10312	10377	mmap_old_sz = bpf_map_mmap_sz(map);
10313	10378	mmap_new_sz = array_map_mmap_sz(size, map->def.max_entries);
@@ -10315,7 +10380,7 @@ int bpf_map__set_value_size(struct bpf_map *map, __u32 size) · 1 unmodified line
10315	10380	if (err) {
10316	10381	pr_warn("map '%s': failed to resize memory-mapped region: %s\n",
10317	10382	bpf_map__name(map), errstr(err));
10318		− return err;
	10383	+ return libbpf_err(err);
10319	10384	}
10320	10385	err = map_btf_datasec_resize(map, size);
10321	10386	if (err && err != -ENOENT) {
@@ -10345,7 +10410,7 @@ int bpf_map__set_initial_value(struct bpf_map *map, · 23 unmodified lines
10345	10410	{
10346	10411	size_t actual_sz;
10347	10412
10348		− if (map->obj->loaded \|\| map->reused)
	10413	+ if (map_is_created(map))
10349	10414	return libbpf_err(-EBUSY);
10350	10415
10351	10416	if (!map->mmaped \|\| map->libbpf_type == LIBBPF_MAP_KCONFIG)
@@ -12858,7 +12923,7 @@ struct bpf_link bpf_program__attach_freplace(const struct bpf_program prog, · 2506 unmodified lines
12858	12923	if (target_fd) {
12859	12924	LIBBPF_OPTS(bpf_link_create_opts, target_opts);
12860	12925
12861		− btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd);
	12926	+ btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd, prog->obj->token_fd);
12862	12927	if (btf_id < 0)
12863	12928	return libbpf_err_ptr(btf_id);
12864	12929
@@ -13070,17 +13135,17 @@ int bpf_link__update_map(struct bpf_link link, const struct bpf_map map) · 205 unmodified lines
13070	13135	int err;
13071	13136
13072	13137	if (!bpf_map__is_struct_ops(map))
13073		− return -EINVAL;
	13138	+ return libbpf_err(-EINVAL);
13074	13139
13075	13140	if (map->fd < 0) {
13076	13141	pr_warn("map '%s': can't use BPF map without FD (was it created?)\n", map->name);
13077		− return -EINVAL;
	13142	+ return libbpf_err(-EINVAL);
13078	13143	}
13079	13144
13080	13145	st_ops_link = container_of(link, struct bpf_link_struct_ops, link);
13081	13146	/* Ensure the type of a link is correct */
13082	13147	if (st_ops_link->map_fd < 0)
13083		− return -EINVAL;
	13148	+ return libbpf_err(-EINVAL);
13084	13149
13085	13150	err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0);
13086	13151	/* It can be EBUSY if the map has been used to create or
@@ -13666,7 +13731,7 @@ int bpf_program__set_attach_target(struct bpf_program *prog, · 579 unmodified lines
13666	13731	if (!prog \|\| attach_prog_fd < 0)
13667	13732	return libbpf_err(-EINVAL);
13668	13733
13669		− if (prog->obj->loaded)
	13734	+ if (prog->obj->state >= OBJ_LOADED)
13670	13735	return libbpf_err(-EINVAL);
13671	13736
13672	13737	if (attach_prog_fd && !attach_func_name) {
@@ -13679,7 +13744,7 @@ int bpf_program__set_attach_target(struct bpf_program *prog, · 6 unmodified lines
13679	13744
13680	13745	if (attach_prog_fd) {
13681	13746	btf_id = libbpf_find_prog_btf_id(attach_func_name,
13682		− attach_prog_fd);
	13747	+ attach_prog_fd, prog->obj->token_fd);
13683	13748	if (btf_id < 0)
13684	13749	return libbpf_err(btf_id);
13685	13750	} else {

tools/lib/bpf/libbpf.h+13−0

@@ -241,6 +241,19 @@ LIBBPF_API struct bpf_object * · 240 unmodified lines
241	241	bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz,
242	242	const struct bpf_object_open_opts *opts);
243	243
	244	+/**
	245	+ * @brief bpf_object__prepare() prepares BPF object for loading:
	246	+ * performs ELF processing, relocations, prepares final state of BPF program
	247	+ * instructions (accessible with bpf_program__insns()), creates and
	248	+ * (potentially) pins maps. Leaves BPF object in the state ready for program
	249	+ * loading.
	250	+ * @param obj Pointer to a valid BPF object instance returned by
	251	+ * *bpf_object__open()** API
	252	+ * @return 0, on success; negative error code, otherwise, error code is
	253	+ * stored in errno
	254	+ */
	255	+int bpf_object__prepare(struct bpf_object *obj);
	256	+
244	257	/**
245	258	* @brief bpf_object__load() loads BPF object into kernel.
246	259	* @param obj Pointer to a valid BPF object instance returned by

tools/lib/bpf/libbpf.map+3−0

@@ -436,4 +436,7 @@ LIBBPF_1.6.0 { · 435 unmodified lines
436	436	bpf_linker__add_buf;
437	437	bpf_linker__add_fd;
438	438	bpf_linker__new_fd;
	439	+ bpf_object__prepare;
	440	+ btf__add_decl_attr;
	441	+ btf__add_type_attr;
439	442	} LIBBPF_1.5.0;

tools/lib/bpf/libbpf_internal.h+1−0

@@ -409,6 +409,7 @@ int libbpf__load_raw_btf(const char *raw_types, size_t types_len, · 408 unmodified lines
409	409	int btf_load_into_kernel(struct btf *btf,
410	410	char *log_buf, size_t log_sz, __u32 log_level,
411	411	int token_fd);
	412	+struct btf btf_load_from_kernel(__u32 id, struct btf base_btf, int token_fd);
412	413
413	414	struct btf btf_get_from_fd(int btf_fd, struct btf base_btf);
414	415	void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type,

tools/lib/bpf/linker.c+1−1

@@ -2163,7 +2163,7 @@ add_sym: · 2162 unmodified lines
2163	2163
2164	2164	obj->sym_map[src_sym_idx] = dst_sym_idx;
2165	2165
2166		− if (sym_type == STT_SECTION && dst_sym) {
	2166	+ if (sym_type == STT_SECTION && dst_sec) {
2167	2167	dst_sec->sec_sym_idx = dst_sym_idx;
2168	2168	dst_sym->st_value = 0;
2169	2169	}

tools/lib/bpf/relo_core.c+20−4

@@ -683,7 +683,7 @@ static int bpf_core_calc_field_relo(const char *prog_name, · 682 unmodified lines
683	683	{
684	684	const struct bpf_core_accessor *acc;
685	685	const struct btf_type *t;
686		− __u32 byte_off, byte_sz, bit_off, bit_sz, field_type_id;
	686	+ __u32 byte_off, byte_sz, bit_off, bit_sz, field_type_id, elem_id;
687	687	const struct btf_member *m;
688	688	const struct btf_type *mt;
689	689	bool bitfield;
@@ -706,8 +706,14 @@ static int bpf_core_calc_field_relo(const char *prog_name, · 16 unmodified lines
706	706	if (!acc->name) {
707	707	if (relo->kind == BPF_CORE_FIELD_BYTE_OFFSET) {
708	708	*val = spec->bit_offset / 8;
709		− /* remember field size for load/store mem size */
710		− sz = btf__resolve_size(spec->btf, acc->type_id);
	709	+ /* remember field size for load/store mem size;
	710	+ * note, for arrays we care about individual element
	711	+ * sizes, not the overall array size
	712	+ */
	713	+ t = skip_mods_and_typedefs(spec->btf, acc->type_id, &elem_id);
	714	+ while (btf_is_array(t))
	715	+ t = skip_mods_and_typedefs(spec->btf, btf_array(t)->type, &elem_id);
	716	+ sz = btf__resolve_size(spec->btf, elem_id);
711	717	if (sz < 0)
712	718	return -EINVAL;
713	719	*field_sz = sz;
@@ -767,7 +773,17 @@ static int bpf_core_calc_field_relo(const char *prog_name, · 53 unmodified lines
767	773	case BPF_CORE_FIELD_BYTE_OFFSET:
768	774	*val = byte_off;
769	775	if (!bitfield) {
770		− *field_sz = byte_sz;
	776	+ /* remember field size for load/store mem size;
	777	+ * note, for arrays we care about individual element
	778	+ * sizes, not the overall array size
	779	+ */
	780	+ t = skip_mods_and_typedefs(spec->btf, field_type_id, &elem_id);
	781	+ while (btf_is_array(t))
	782	+ t = skip_mods_and_typedefs(spec->btf, btf_array(t)->type, &elem_id);
	783	+ sz = btf__resolve_size(spec->btf, elem_id);
	784	+ if (sz < 0)
	785	+ return -EINVAL;
	786	+ *field_sz = sz;
771	787	*type_id = field_type_id;
772	788	}
773	789	break;

tools/lib/bpf/str_error.c+1−1

@@ -36,7 +36,7 @@ char libbpf_strerror_r(int err, char dst, int len) · 35 unmodified lines
36	36	return dst;
37	37	}
38	38
39		−const char *errstr(int err)
	39	+const char *libbpf_errstr(int err)
40	40	{
41	41	static __thread char buf[12];
42	42

tools/lib/bpf/str_error.h+5−2

@@ -7,10 +7,13 @@ · 6 unmodified lines
7	7	char libbpf_strerror_r(int err, char dst, int len);
8	8
9	9	/**
10		− * @brief errstr() returns string corresponding to numeric errno
	10	+ * @brief libbpf_errstr() returns string corresponding to numeric errno
11	11	* @param err negative numeric errno
12	12	* @return pointer to string representation of the errno, that is invalidated
13	13	* upon the next call.
14	14	*/
15		−const char *errstr(int err);
	15	+const char *libbpf_errstr(int err);
	16	+
	17	+#define errstr(err) libbpf_errstr(err)
	18	+
16	19	#endif /* __LIBBPF_STR_ERROR_H */

tools/lib/bpf/usdt.bpf.h+32−0

@@ -108,6 +108,38 @@ int bpf_usdt_arg_cnt(struct pt_regs *ctx) · 107 unmodified lines
108	108	return spec->arg_cnt;
109	109	}
110	110
	111	+/* Returns the size in bytes of the #arg_num (zero-indexed) USDT argument.
	112	+ * Returns negative error if argument is not found or arg_num is invalid.
	113	+ */
	114	+static __always_inline
	115	+int bpf_usdt_arg_size(struct pt_regs *ctx, __u64 arg_num)
	116	+{
	117	+ struct __bpf_usdt_arg_spec *arg_spec;
	118	+ struct __bpf_usdt_spec *spec;
	119	+ int spec_id;
	120	+
	121	+ spec_id = __bpf_usdt_spec_id(ctx);
	122	+ if (spec_id < 0)
	123	+ return -ESRCH;
	124	+
	125	+ spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id);
	126	+ if (!spec)
	127	+ return -ESRCH;
	128	+
	129	+ if (arg_num >= BPF_USDT_MAX_ARG_CNT)
	130	+ return -ENOENT;
	131	+ barrier_var(arg_num);
	132	+ if (arg_num >= spec->arg_cnt)
	133	+ return -ENOENT;
	134	+
	135	+ arg_spec = &spec->args[arg_num];
	136	+
	137	+ /* arg_spec->arg_bitshift = 64 - arg_sz * 8
	138	+ * so: arg_sz = (64 - arg_spec->arg_bitshift) / 8
	139	+ */
	140	+ return (unsigned int)(64 - arg_spec->arg_bitshift) / 8;
	141	+}
	142	+
111	143	/* Fetch USDT argument #arg_num (zero-indexed) and put its value into *res.
112	144	* Returns 0 on success; negative error, otherwise.
113	145	* On error *res is guaranteed to be set to zero.

tools/testing/selftests/bpf/DENYLIST.aarch64+0−9

@@ -1,12 +1,3 @@
1		−bpf_cookie/multi_kprobe_attach_api # kprobe_multi_link_api_subtest:FAIL:fentry_raw_skel_load unexpected error: -3
2		−bpf_cookie/multi_kprobe_link_api # kprobe_multi_link_api_subtest:FAIL:fentry_raw_skel_load unexpected error: -3
3		−kprobe_multi_bench_attach # needs CONFIG_FPROBE
4		−kprobe_multi_test # needs CONFIG_FPROBE
5		−module_attach # prog 'kprobe_multi': failed to auto-attach: -95
6	1	fentry_test/fentry_many_args # fentry_many_args:FAIL:fentry_many_args_attach unexpected error: -524
7	2	fexit_test/fexit_many_args # fexit_many_args:FAIL:fexit_many_args_attach unexpected error: -524
8	3	tracing_struct/struct_many_args # struct_many_args:FAIL:tracing_struct_many_args__attach unexpected error: -524
9		−fill_link_info/kprobe_multi_link_info # bpf_program__attach_kprobe_multi_opts unexpected error: -95
10		−fill_link_info/kretprobe_multi_link_info # bpf_program__attach_kprobe_multi_opts unexpected error: -95
11		−fill_link_info/kprobe_multi_invalid_ubuff # bpf_program__attach_kprobe_multi_opts unexpected error: -95
12		−missed/kprobe_recursion # missed_kprobe_recursion__attach unexpected error: -95 (errno 95)

tools/testing/selftests/bpf/Makefile+14−14

@@ -95,18 +95,12 @@ TEST_GEN_PROGS += test_progs-cpuv4 · 94 unmodified lines
95	95	TEST_INST_SUBDIRS += cpuv4
96	96	endif
97	97
98		−TEST_GEN_FILES = test_lwt_ip_encap.bpf.o test_tc_edt.bpf.o
	98	+TEST_GEN_FILES = test_tc_edt.bpf.o
99	99	TEST_FILES = xsk_prereqs.sh $(wildcard progs/btf_dump_test_case_*.c)
100	100
101	101	# Order correspond to 'make run_tests' order
102	102	TEST_PROGS := test_kmod.sh \
103		− test_xdp_redirect_multi.sh \
104		− test_tunnel.sh \
105		− test_lwt_seg6local.sh \
106	103	test_lirc_mode2.sh \
107		− test_xdp_vlan_mode_generic.sh \
108		− test_xdp_vlan_mode_native.sh \
109		− test_lwt_ip_encap.sh \
110	104	test_tc_tunnel.sh \
111	105	test_tc_edt.sh \
112	106	test_xdping.sh \
@@ -117,9 +111,9 @@ TEST_PROGS := test_kmod.sh \ · 4 unmodified lines
117	111	test_xsk.sh \
118	112	test_xdp_features.sh
119	113
120		−TEST_PROGS_EXTENDED := with_addr.sh \
121		− with_tunnels.sh ima_setup.sh verify_sig_setup.sh \
122		− test_xdp_vlan.sh test_bpftool.py
	114	+TEST_PROGS_EXTENDED := \
	115	+ ima_setup.sh verify_sig_setup.sh \
	116	+ test_bpftool.py
123	117
124	118	TEST_KMODS := bpf_testmod.ko bpf_test_no_cfi.ko bpf_test_modorder_x.ko \
125	119	bpf_test_modorder_y.ko
@@ -135,7 +129,6 @@ TEST_GEN_PROGS_EXTENDED = \ · 9 unmodified lines
135	129	veristat \
136	130	xdp_features \
137	131	xdp_hw_metadata \
138		− xdp_redirect_multi \
139	132	xdp_synproxy \
140	133	xdping \
141	134	xskxceiver
@@ -184,9 +177,14 @@ ifeq ($(feature-llvm),1) · 42 unmodified lines
184	177	LLVM_CONFIG_LIB_COMPONENTS := mcdisassembler all-targets
185	178	# both llvm-config and lib.mk add -D_GNU_SOURCE, which ends up as conflict
186	179	LLVM_CFLAGS += $(filter-out -D_GNU_SOURCE,$(shell $(LLVM_CONFIG) --cflags))
187		− LLVM_LDLIBS += $(shell $(LLVM_CONFIG) --link-static --libs $(LLVM_CONFIG_LIB_COMPONENTS))
188		− LLVM_LDLIBS += $(shell $(LLVM_CONFIG) --link-static --system-libs $(LLVM_CONFIG_LIB_COMPONENTS))
189		− LLVM_LDLIBS += -lstdc++
	180	+ # Prefer linking statically if it's available, otherwise fallback to shared
	181	+ ifeq ($(shell $(LLVM_CONFIG) --link-static --libs >/dev/null 2>&1 && echo static),static)
	182	+ LLVM_LDLIBS += $(shell $(LLVM_CONFIG) --link-static --libs $(LLVM_CONFIG_LIB_COMPONENTS))
	183	+ LLVM_LDLIBS += $(shell $(LLVM_CONFIG) --link-static --system-libs $(LLVM_CONFIG_LIB_COMPONENTS))
	184	+ LLVM_LDLIBS += -lstdc++
	185	+ else
	186	+ LLVM_LDLIBS += $(shell $(LLVM_CONFIG) --link-shared --libs $(LLVM_CONFIG_LIB_COMPONENTS))
	187	+ endif
190	188	LLVM_LDFLAGS += $(shell $(LLVM_CONFIG) --ldflags)
191	189	endif
192	190
@@ -306,6 +304,7 @@ $(OUTPUT)/runqslower: $(BPFOBJ) \| $(DEFAULT_BPFTOOL) $(RUNQSLOWER_OUTPUT) · 113 unmodified lines
306	304	BPFTOOL_OUTPUT=$(HOST_BUILD_DIR)/bpftool/ \
307	305	BPFOBJ_OUTPUT=$(BUILD_DIR)/libbpf/ \
308	306	BPFOBJ=$(BPFOBJ) BPF_INCLUDE=$(INCLUDE_DIR) \
	307	+ BPF_TARGET_ENDIAN=$(BPF_TARGET_ENDIAN) \
309	308	EXTRA_CFLAGS='-g $(OPT_FLAGS) $(SAN_CFLAGS) $(EXTRA_CFLAGS)' \
310	309	EXTRA_LDFLAGS='$(SAN_LDFLAGS) $(EXTRA_LDFLAGS)' && \
311	310	cp $(RUNQSLOWER_OUTPUT)runqslower $@
@@ -684,6 +683,7 @@ $(OUTPUT)/$(TRUNNER_BINARY): $(TRUNNER_TEST_OBJS) \ · 372 unmodified lines
684	683	$(TRUNNER_EXTRA_OBJS) $$(BPFOBJ) \
685	684	$(RESOLVE_BTFIDS) \
686	685	$(TRUNNER_BPFTOOL) \
	686	+ $(OUTPUT)/veristat \
687	687	\| $(TRUNNER_BINARY)-extras
688	688	$$(call msg,BINARY,,$$@)
689	689	$(Q)$$(CC) $$(CFLAGS) $$(filter %.a %.o,$$^) $$(LDLIBS) $$(LDFLAGS) -o $$@

tools/testing/selftests/bpf/bpf_arena_spin_lock.h+533−0

@@ -0,0 +1,533 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
	3	+#ifndef BPF_ARENA_SPIN_LOCK_H
	4	+#define BPF_ARENA_SPIN_LOCK_H
	5	+
	6	+#include <vmlinux.h>
	7	+#include <bpf/bpf_helpers.h>
	8	+#include "bpf_atomic.h"
	9	+
	10	+#define arch_mcs_spin_lock_contended_label(l, label) smp_cond_load_acquire_label(l, VAL, label)
	11	+#define arch_mcs_spin_unlock_contended(l) smp_store_release((l), 1)
	12	+
	13	+#if defined(ENABLE_ATOMICS_TESTS) && defined(__BPF_FEATURE_ADDR_SPACE_CAST)
	14	+
	15	+#define EBUSY 16
	16	+#define EOPNOTSUPP 95
	17	+#define ETIMEDOUT 110
	18	+
	19	+#ifndef __arena
	20	+#define __arena __attribute__((address_space(1)))
	21	+#endif
	22	+
	23	+extern unsigned long CONFIG_NR_CPUS __kconfig;
	24	+
	25	+/*
	26	+ * Typically, we'd just rely on the definition in vmlinux.h for qspinlock, but
	27	+ * PowerPC overrides the definition to define lock->val as u32 instead of
	28	+ * atomic_t, leading to compilation errors. Import a local definition below so
	29	+ * that we don't depend on the vmlinux.h version.
	30	+ */
	31	+
	32	+struct __qspinlock {
	33	+ union {
	34	+ atomic_t val;
	35	+ struct {
	36	+ u8 locked;
	37	+ u8 pending;
	38	+ };
	39	+ struct {
	40	+ u16 locked_pending;
	41	+ u16 tail;
	42	+ };
	43	+ };
	44	+};
	45	+
	46	+#define arena_spinlock_t struct __qspinlock
	47	+/* FIXME: Using typedef causes CO-RE relocation error */
	48	+/* typedef struct qspinlock arena_spinlock_t; */
	49	+
	50	+struct arena_mcs_spinlock {
	51	+ struct arena_mcs_spinlock __arena *next;
	52	+ int locked;
	53	+ int count;
	54	+};
	55	+
	56	+struct arena_qnode {
	57	+ struct arena_mcs_spinlock mcs;
	58	+};
	59	+
	60	+#define _Q_MAX_NODES 4
	61	+#define _Q_PENDING_LOOPS 1
	62	+
	63	+/*
	64	+ * Bitfields in the atomic value:
	65	+ *
	66	+ * 0- 7: locked byte
	67	+ * 8: pending
	68	+ * 9-15: not used
	69	+ * 16-17: tail index
	70	+ * 18-31: tail cpu (+1)
	71	+ */
	72	+#define _Q_MAX_CPUS 1024
	73	+
	74	+#define _Q_SET_MASK(type) (((1U << _Q_ ## type ## _BITS) - 1)\
	75	+ << _Q_ ## type ## _OFFSET)
	76	+#define _Q_LOCKED_OFFSET 0
	77	+#define _Q_LOCKED_BITS 8
	78	+#define _Q_LOCKED_MASK _Q_SET_MASK(LOCKED)
	79	+
	80	+#define _Q_PENDING_OFFSET (_Q_LOCKED_OFFSET + _Q_LOCKED_BITS)
	81	+#define _Q_PENDING_BITS 8
	82	+#define _Q_PENDING_MASK _Q_SET_MASK(PENDING)
	83	+
	84	+#define _Q_TAIL_IDX_OFFSET (_Q_PENDING_OFFSET + _Q_PENDING_BITS)
	85	+#define _Q_TAIL_IDX_BITS 2
	86	+#define _Q_TAIL_IDX_MASK _Q_SET_MASK(TAIL_IDX)
	87	+
	88	+#define _Q_TAIL_CPU_OFFSET (_Q_TAIL_IDX_OFFSET + _Q_TAIL_IDX_BITS)
	89	+#define _Q_TAIL_CPU_BITS (32 - _Q_TAIL_CPU_OFFSET)
	90	+#define _Q_TAIL_CPU_MASK _Q_SET_MASK(TAIL_CPU)
	91	+
	92	+#define _Q_TAIL_OFFSET _Q_TAIL_IDX_OFFSET
	93	+#define _Q_TAIL_MASK (_Q_TAIL_IDX_MASK \| _Q_TAIL_CPU_MASK)
	94	+
	95	+#define _Q_LOCKED_VAL (1U << _Q_LOCKED_OFFSET)
	96	+#define _Q_PENDING_VAL (1U << _Q_PENDING_OFFSET)
	97	+
	98	+#define likely(x) __builtin_expect(!!(x), 1)
	99	+#define unlikely(x) __builtin_expect(!!(x), 0)
	100	+
	101	+struct arena_qnode __arena qnodes[_Q_MAX_CPUS][_Q_MAX_NODES];
	102	+
	103	+static inline u32 encode_tail(int cpu, int idx)
	104	+{
	105	+ u32 tail;
	106	+
	107	+ tail = (cpu + 1) << _Q_TAIL_CPU_OFFSET;
	108	+ tail \|= idx << _Q_TAIL_IDX_OFFSET; /* assume < 4 */
	109	+
	110	+ return tail;
	111	+}
	112	+
	113	+static inline struct arena_mcs_spinlock __arena *decode_tail(u32 tail)
	114	+{
	115	+ u32 cpu = (tail >> _Q_TAIL_CPU_OFFSET) - 1;
	116	+ u32 idx = (tail & _Q_TAIL_IDX_MASK) >> _Q_TAIL_IDX_OFFSET;
	117	+
	118	+ return &qnodes[cpu][idx].mcs;
	119	+}
	120	+
	121	+static inline
	122	+struct arena_mcs_spinlock __arena grab_mcs_node(struct arena_mcs_spinlock __arena base, int idx)
	123	+{
	124	+ return &((struct arena_qnode __arena *)base + idx)->mcs;
	125	+}
	126	+
	127	+#define _Q_LOCKED_PENDING_MASK (_Q_LOCKED_MASK \| _Q_PENDING_MASK)
	128	+
	129	+/**
	130	+ * xchg_tail - Put in the new queue tail code word & retrieve previous one
	131	+ * @lock : Pointer to queued spinlock structure
	132	+ * @tail : The new queue tail code word
	133	+ * Return: The previous queue tail code word
	134	+ *
	135	+ * xchg(lock, tail)
	136	+ *
	137	+ * p,, -> n,, ; prev = xchg(lock, node)
	138	+ */
	139	+static __always_inline u32 xchg_tail(arena_spinlock_t __arena *lock, u32 tail)
	140	+{
	141	+ u32 old, new;
	142	+
	143	+ old = atomic_read(&lock->val);
	144	+ do {
	145	+ new = (old & _Q_LOCKED_PENDING_MASK) \| tail;
	146	+ /*
	147	+ * We can use relaxed semantics since the caller ensures that
	148	+ * the MCS node is properly initialized before updating the
	149	+ * tail.
	150	+ */
	151	+ /* These loops are not expected to stall, but we still need to
	152	+ * prove to the verifier they will terminate eventually.
	153	+ */
	154	+ cond_break_label(out);
	155	+ } while (!atomic_try_cmpxchg_relaxed(&lock->val, &old, new));
	156	+
	157	+ return old;
	158	+out:
	159	+ bpf_printk("RUNTIME ERROR: %s unexpected cond_break exit!!!", __func__);
	160	+ return old;
	161	+}
	162	+
	163	+/**
	164	+ * clear_pending - clear the pending bit.
	165	+ * @lock: Pointer to queued spinlock structure
	166	+ *
	167	+ * ,1, -> ,0,
	168	+ */
	169	+static __always_inline void clear_pending(arena_spinlock_t __arena *lock)
	170	+{
	171	+ WRITE_ONCE(lock->pending, 0);
	172	+}
	173	+
	174	+/**
	175	+ * clear_pending_set_locked - take ownership and clear the pending bit.
	176	+ * @lock: Pointer to queued spinlock structure
	177	+ *
	178	+ * ,1,0 -> ,0,1
	179	+ *
	180	+ * Lock stealing is not allowed if this function is used.
	181	+ */
	182	+static __always_inline void clear_pending_set_locked(arena_spinlock_t __arena *lock)
	183	+{
	184	+ WRITE_ONCE(lock->locked_pending, _Q_LOCKED_VAL);
	185	+}
	186	+
	187	+/**
	188	+ * set_locked - Set the lock bit and own the lock
	189	+ * @lock: Pointer to queued spinlock structure
	190	+ *
	191	+ * ,,0 -> *,0,1
	192	+ */
	193	+static __always_inline void set_locked(arena_spinlock_t __arena *lock)
	194	+{
	195	+ WRITE_ONCE(lock->locked, _Q_LOCKED_VAL);
	196	+}
	197	+
	198	+static __always_inline
	199	+u32 arena_fetch_set_pending_acquire(arena_spinlock_t __arena *lock)
	200	+{
	201	+ u32 old, new;
	202	+
	203	+ old = atomic_read(&lock->val);
	204	+ do {
	205	+ new = old \| _Q_PENDING_VAL;
	206	+ /*
	207	+ * These loops are not expected to stall, but we still need to
	208	+ * prove to the verifier they will terminate eventually.
	209	+ */
	210	+ cond_break_label(out);
	211	+ } while (!atomic_try_cmpxchg_acquire(&lock->val, &old, new));
	212	+
	213	+ return old;
	214	+out:
	215	+ bpf_printk("RUNTIME ERROR: %s unexpected cond_break exit!!!", __func__);
	216	+ return old;
	217	+}
	218	+
	219	+/**
	220	+ * arena_spin_trylock - try to acquire the queued spinlock
	221	+ * @lock : Pointer to queued spinlock structure
	222	+ * Return: 1 if lock acquired, 0 if failed
	223	+ */
	224	+static __always_inline int arena_spin_trylock(arena_spinlock_t __arena *lock)
	225	+{
	226	+ int val = atomic_read(&lock->val);
	227	+
	228	+ if (unlikely(val))
	229	+ return 0;
	230	+
	231	+ return likely(atomic_try_cmpxchg_acquire(&lock->val, &val, _Q_LOCKED_VAL));
	232	+}
	233	+
	234	+__noinline
	235	+int arena_spin_lock_slowpath(arena_spinlock_t __arena __arg_arena *lock, u32 val)
	236	+{
	237	+ struct arena_mcs_spinlock __arena prev, next, node0, node;
	238	+ int ret = -ETIMEDOUT;
	239	+ u32 old, tail;
	240	+ int idx;
	241	+
	242	+ /*
	243	+ * Wait for in-progress pending->locked hand-overs with a bounded
	244	+ * number of spins so that we guarantee forward progress.
	245	+ *
	246	+ * 0,1,0 -> 0,0,1
	247	+ */
	248	+ if (val == _Q_PENDING_VAL) {
	249	+ int cnt = _Q_PENDING_LOOPS;
	250	+ val = atomic_cond_read_relaxed_label(&lock->val,
	251	+ (VAL != _Q_PENDING_VAL) \|\| !cnt--,
	252	+ release_err);
	253	+ }
	254	+
	255	+ /*
	256	+ * If we observe any contention; queue.
	257	+ */
	258	+ if (val & ~_Q_LOCKED_MASK)
	259	+ goto queue;
	260	+
	261	+ /*
	262	+ * trylock \|\| pending
	263	+ *
	264	+ * 0,0,* -> 0,1,* -> 0,0,1 pending, trylock
	265	+ */
	266	+ val = arena_fetch_set_pending_acquire(lock);
	267	+
	268	+ /*
	269	+ * If we observe contention, there is a concurrent locker.
	270	+ *
	271	+ * Undo and queue; our setting of PENDING might have made the
	272	+ * n,0,0 -> 0,0,0 transition fail and it will now be waiting
	273	+ * on @next to become !NULL.
	274	+ */
	275	+ if (unlikely(val & ~_Q_LOCKED_MASK)) {
	276	+
	277	+ /* Undo PENDING if we set it. */
	278	+ if (!(val & _Q_PENDING_MASK))
	279	+ clear_pending(lock);
	280	+
	281	+ goto queue;
	282	+ }
	283	+
	284	+ /*
	285	+ * We're pending, wait for the owner to go away.
	286	+ *
	287	+ * 0,1,1 -> *,1,0
	288	+ *
	289	+ * this wait loop must be a load-acquire such that we match the
	290	+ * store-release that clears the locked bit and create lock
	291	+ * sequentiality; this is because not all
	292	+ * clear_pending_set_locked() implementations imply full
	293	+ * barriers.
	294	+ */
	295	+ if (val & _Q_LOCKED_MASK)
	296	+ smp_cond_load_acquire_label(&lock->locked, !VAL, release_err);
	297	+
	298	+ /*
	299	+ * take ownership and clear the pending bit.
	300	+ *
	301	+ * 0,1,0 -> 0,0,1
	302	+ */
	303	+ clear_pending_set_locked(lock);
	304	+ return 0;
	305	+
	306	+ /*
	307	+ * End of pending bit optimistic spinning and beginning of MCS
	308	+ * queuing.
	309	+ */
	310	+queue:
	311	+ node0 = &(qnodes[bpf_get_smp_processor_id()])[0].mcs;
	312	+ idx = node0->count++;
	313	+ tail = encode_tail(bpf_get_smp_processor_id(), idx);
	314	+
	315	+ /*
	316	+ * 4 nodes are allocated based on the assumption that there will not be
	317	+ * nested NMIs taking spinlocks. That may not be true in some
	318	+ * architectures even though the chance of needing more than 4 nodes
	319	+ * will still be extremely unlikely. When that happens, we simply return
	320	+ * an error. Original qspinlock has a trylock fallback in this case.
	321	+ */
	322	+ if (unlikely(idx >= _Q_MAX_NODES)) {
	323	+ ret = -EBUSY;
	324	+ goto release_node_err;
	325	+ }
	326	+
	327	+ node = grab_mcs_node(node0, idx);
	328	+
	329	+ /*
	330	+ * Ensure that we increment the head node->count before initialising
	331	+ * the actual node. If the compiler is kind enough to reorder these
	332	+ * stores, then an IRQ could overwrite our assignments.
	333	+ */
	334	+ barrier();
	335	+
	336	+ node->locked = 0;
	337	+ node->next = NULL;
	338	+
	339	+ /*
	340	+ * We touched a (possibly) cold cacheline in the per-cpu queue node;
	341	+ * attempt the trylock once more in the hope someone let go while we
	342	+ * weren't watching.
	343	+ */
	344	+ if (arena_spin_trylock(lock))
	345	+ goto release;
	346	+
	347	+ /*
	348	+ * Ensure that the initialisation of @node is complete before we
	349	+ * publish the updated tail via xchg_tail() and potentially link
	350	+ * @node into the waitqueue via WRITE_ONCE(prev->next, node) below.
	351	+ */
	352	+ smp_wmb();
	353	+
	354	+ /*
	355	+ * Publish the updated tail.
	356	+ * We have already touched the queueing cacheline; don't bother with
	357	+ * pending stuff.
	358	+ *
	359	+ * p,, -> n,,
	360	+ */
	361	+ old = xchg_tail(lock, tail);
	362	+ next = NULL;
	363	+
	364	+ /*
	365	+ * if there was a previous node; link it and wait until reaching the
	366	+ * head of the waitqueue.
	367	+ */
	368	+ if (old & _Q_TAIL_MASK) {
	369	+ prev = decode_tail(old);
	370	+
	371	+ /* Link @node into the waitqueue. */
	372	+ WRITE_ONCE(prev->next, node);
	373	+
	374	+ arch_mcs_spin_lock_contended_label(&node->locked, release_node_err);
	375	+
	376	+ /*
	377	+ * While waiting for the MCS lock, the next pointer may have
	378	+ * been set by another lock waiter. We cannot prefetch here
	379	+ * due to lack of equivalent instruction in BPF ISA.
	380	+ */
	381	+ next = READ_ONCE(node->next);
	382	+ }
	383	+
	384	+ /*
	385	+ * we're at the head of the waitqueue, wait for the owner & pending to
	386	+ * go away.
	387	+ *
	388	+ * ,x,y -> ,0,0
	389	+ *
	390	+ * this wait loop must use a load-acquire such that we match the
	391	+ * store-release that clears the locked bit and create lock
	392	+ * sequentiality; this is because the set_locked() function below
	393	+ * does not imply a full barrier.
	394	+ */
	395	+ val = atomic_cond_read_acquire_label(&lock->val, !(VAL & _Q_LOCKED_PENDING_MASK),
	396	+ release_node_err);
	397	+
	398	+ /*
	399	+ * claim the lock:
	400	+ *
	401	+ * n,0,0 -> 0,0,1 : lock, uncontended
	402	+ * ,,0 -> ,,1 : lock, contended
	403	+ *
	404	+ * If the queue head is the only one in the queue (lock value == tail)
	405	+ * and nobody is pending, clear the tail code and grab the lock.
	406	+ * Otherwise, we only need to grab the lock.
	407	+ */
	408	+
	409	+ /*
	410	+ * In the PV case we might already have _Q_LOCKED_VAL set, because
	411	+ * of lock stealing; therefore we must also allow:
	412	+ *
	413	+ * n,0,1 -> 0,0,1
	414	+ *
	415	+ * Note: at this point: (val & _Q_PENDING_MASK) == 0, because of the
	416	+ * above wait condition, therefore any concurrent setting of
	417	+ * PENDING will make the uncontended transition fail.
	418	+ */
	419	+ if ((val & _Q_TAIL_MASK) == tail) {
	420	+ if (atomic_try_cmpxchg_relaxed(&lock->val, &val, _Q_LOCKED_VAL))
	421	+ goto release; /* No contention */
	422	+ }
	423	+
	424	+ /*
	425	+ * Either somebody is queued behind us or _Q_PENDING_VAL got set
	426	+ * which will then detect the remaining tail and queue behind us
	427	+ * ensuring we'll see a @next.
	428	+ */
	429	+ set_locked(lock);
	430	+
	431	+ /*
	432	+ * contended path; wait for next if not observed yet, release.
	433	+ */
	434	+ if (!next)
	435	+ next = smp_cond_load_relaxed_label(&node->next, (VAL), release_node_err);
	436	+
	437	+ arch_mcs_spin_unlock_contended(&next->locked);
	438	+
	439	+release:;
	440	+ /*
	441	+ * release the node
	442	+ *
	443	+ * Doing a normal dec vs this_cpu_dec is fine. An upper context always
	444	+ * decrements count it incremented before returning, thus we're fine.
	445	+ * For contexts interrupting us, they either observe our dec or not.
	446	+ * Just ensure the compiler doesn't reorder this statement, as a
	447	+ * this_cpu_dec implicitly implied that.
	448	+ */
	449	+ barrier();
	450	+ node0->count--;
	451	+ return 0;
	452	+release_node_err:
	453	+ barrier();
	454	+ node0->count--;
	455	+ goto release_err;
	456	+release_err:
	457	+ return ret;
	458	+}
	459	+
	460	+/**
	461	+ * arena_spin_lock - acquire a queued spinlock
	462	+ * @lock: Pointer to queued spinlock structure
	463	+ *
	464	+ * On error, returned value will be negative.
	465	+ * On success, zero is returned.
	466	+ *
	467	+ * The return value _must_ be tested against zero for success,
	468	+ * instead of checking it against negative, for passing the
	469	+ * BPF verifier.
	470	+ *
	471	+ * The user should do:
	472	+ * if (arena_spin_lock(...) != 0) // failure
	473	+ * or
	474	+ * if (arena_spin_lock(...) == 0) // success
	475	+ * or
	476	+ * if (arena_spin_lock(...)) // failure
	477	+ * or
	478	+ * if (!arena_spin_lock(...)) // success
	479	+ * instead of:
	480	+ * if (arena_spin_lock(...) < 0) // failure
	481	+ *
	482	+ * The return value can still be inspected later.
	483	+ */
	484	+static __always_inline int arena_spin_lock(arena_spinlock_t __arena *lock)
	485	+{
	486	+ int val = 0;
	487	+
	488	+ if (CONFIG_NR_CPUS > 1024)
	489	+ return -EOPNOTSUPP;
	490	+
	491	+ bpf_preempt_disable();
	492	+ if (likely(atomic_try_cmpxchg_acquire(&lock->val, &val, _Q_LOCKED_VAL)))
	493	+ return 0;
	494	+
	495	+ val = arena_spin_lock_slowpath(lock, val);
	496	+ /* FIXME: bpf_assert_range(-MAX_ERRNO, 0) once we have it working for all cases. */
	497	+ if (val)
	498	+ bpf_preempt_enable();
	499	+ return val;
	500	+}
	501	+
	502	+/**
	503	+ * arena_spin_unlock - release a queued spinlock
	504	+ * @lock : Pointer to queued spinlock structure
	505	+ */
	506	+static __always_inline void arena_spin_unlock(arena_spinlock_t __arena *lock)
	507	+{
	508	+ /*
	509	+ * unlock() needs release semantics:
	510	+ */
	511	+ smp_store_release(&lock->locked, 0);
	512	+ bpf_preempt_enable();
	513	+}
	514	+
	515	+#define arena_spin_lock_irqsave(lock, flags) \
	516	+ ({ \
	517	+ int __ret; \
	518	+ bpf_local_irq_save(&(flags)); \
	519	+ __ret = arena_spin_lock((lock)); \
	520	+ if (__ret) \
	521	+ bpf_local_irq_restore(&(flags)); \
	522	+ (__ret); \
	523	+ })
	524	+
	525	+#define arena_spin_unlock_irqrestore(lock, flags) \
	526	+ ({ \
	527	+ arena_spin_unlock((lock)); \
	528	+ bpf_local_irq_restore(&(flags)); \
	529	+ })
	530	+
	531	+#endif
	532	+
	533	+#endif /* BPF_ARENA_SPIN_LOCK_H */

tools/testing/selftests/bpf/bpf_atomic.h+140−0

@@ -0,0 +1,140 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
	3	+#ifndef BPF_ATOMIC_H
	4	+#define BPF_ATOMIC_H
	5	+
	6	+#include <vmlinux.h>
	7	+#include <bpf/bpf_helpers.h>
	8	+#include "bpf_experimental.h"
	9	+
	10	+extern bool CONFIG_X86_64 __kconfig __weak;
	11	+
	12	+/*
	13	+ * __unqual_typeof(x) - Declare an unqualified scalar type, leaving
	14	+ * non-scalar types unchanged,
	15	+ *
	16	+ * Prefer C11 _Generic for better compile-times and simpler code. Note: 'char'
	17	+ * is not type-compatible with 'signed char', and we define a separate case.
	18	+ *
	19	+ * This is copied verbatim from kernel's include/linux/compiler_types.h, but
	20	+ * with default expression (for pointers) changed from (x) to (typeof(x)0).
	21	+ *
	22	+ * This is because LLVM has a bug where for lvalue (x), it does not get rid of
	23	+ * an extra address_space qualifier, but does in case of rvalue (typeof(x)0).
	24	+ * Hence, for pointers, we need to create an rvalue expression to get the
	25	+ * desired type. See https://github.com/llvm/llvm-project/issues/53400.
	26	+ */
	27	+#define __scalar_type_to_expr_cases(type) \
	28	+ unsigned type : (unsigned type)0, signed type : (signed type)0
	29	+
	30	+#define __unqual_typeof(x) \
	31	+ typeof(_Generic((x), \
	32	+ char: (char)0, \
	33	+ __scalar_type_to_expr_cases(char), \
	34	+ __scalar_type_to_expr_cases(short), \
	35	+ __scalar_type_to_expr_cases(int), \
	36	+ __scalar_type_to_expr_cases(long), \
	37	+ __scalar_type_to_expr_cases(long long), \
	38	+ default: (typeof(x))0))
	39	+
	40	+/* No-op for BPF */
	41	+#define cpu_relax() ({})
	42	+
	43	+#define READ_ONCE(x) ((volatile typeof(x) )&(x))
	44	+
	45	+#define WRITE_ONCE(x, val) (((volatile typeof(x) )&(x)) = (val))
	46	+
	47	+#define cmpxchg(p, old, new) __sync_val_compare_and_swap((p), old, new)
	48	+
	49	+#define try_cmpxchg(p, pold, new) \
	50	+ ({ \
	51	+ __unqual_typeof((pold)) __o = (pold); \
	52	+ __unqual_typeof(*(p)) __r = cmpxchg(p, __o, new); \
	53	+ if (__r != __o) \
	54	+ *(pold) = __r; \
	55	+ __r == __o; \
	56	+ })
	57	+
	58	+#define try_cmpxchg_relaxed(p, pold, new) try_cmpxchg(p, pold, new)
	59	+
	60	+#define try_cmpxchg_acquire(p, pold, new) try_cmpxchg(p, pold, new)
	61	+
	62	+#define smp_mb() \
	63	+ ({ \
	64	+ unsigned long __val; \
	65	+ __sync_fetch_and_add(&__val, 0); \
	66	+ })
	67	+
	68	+#define smp_rmb() \
	69	+ ({ \
	70	+ if (!CONFIG_X86_64) \
	71	+ smp_mb(); \
	72	+ else \
	73	+ barrier(); \
	74	+ })
	75	+
	76	+#define smp_wmb() \
	77	+ ({ \
	78	+ if (!CONFIG_X86_64) \
	79	+ smp_mb(); \
	80	+ else \
	81	+ barrier(); \
	82	+ })
	83	+
	84	+/* Control dependency provides LOAD->STORE, provide LOAD->LOAD */
	85	+#define smp_acquire__after_ctrl_dep() ({ smp_rmb(); })
	86	+
	87	+#define smp_load_acquire(p) \
	88	+ ({ \
	89	+ __unqual_typeof((p)) __v = READ_ONCE((p)); \
	90	+ if (!CONFIG_X86_64) \
	91	+ smp_mb(); \
	92	+ barrier(); \
	93	+ __v; \
	94	+ })
	95	+
	96	+#define smp_store_release(p, val) \
	97	+ ({ \
	98	+ if (!CONFIG_X86_64) \
	99	+ smp_mb(); \
	100	+ barrier(); \
	101	+ WRITE_ONCE(*(p), val); \
	102	+ })
	103	+
	104	+#define smp_cond_load_relaxed_label(p, cond_expr, label) \
	105	+ ({ \
	106	+ typeof(p) __ptr = (p); \
	107	+ __unqual_typeof(*(p)) VAL; \
	108	+ for (;;) { \
	109	+ VAL = (__unqual_typeof((p)))READ_ONCE(__ptr); \
	110	+ if (cond_expr) \
	111	+ break; \
	112	+ cond_break_label(label); \
	113	+ cpu_relax(); \
	114	+ } \
	115	+ (typeof(*(p)))VAL; \
	116	+ })
	117	+
	118	+#define smp_cond_load_acquire_label(p, cond_expr, label) \
	119	+ ({ \
	120	+ __unqual_typeof(*p) __val = \
	121	+ smp_cond_load_relaxed_label(p, cond_expr, label); \
	122	+ smp_acquire__after_ctrl_dep(); \
	123	+ (typeof(*(p)))__val; \
	124	+ })
	125	+
	126	+#define atomic_read(p) READ_ONCE((p)->counter)
	127	+
	128	+#define atomic_cond_read_relaxed_label(p, cond_expr, label) \
	129	+ smp_cond_load_relaxed_label(&(p)->counter, cond_expr, label)
	130	+
	131	+#define atomic_cond_read_acquire_label(p, cond_expr, label) \
	132	+ smp_cond_load_acquire_label(&(p)->counter, cond_expr, label)
	133	+
	134	+#define atomic_try_cmpxchg_relaxed(p, pold, new) \
	135	+ try_cmpxchg_relaxed(&(p)->counter, pold, new)
	136	+
	137	+#define atomic_try_cmpxchg_acquire(p, pold, new) \
	138	+ try_cmpxchg_acquire(&(p)->counter, pold, new)
	139	+
	140	+#endif /* BPF_ATOMIC_H */

tools/testing/selftests/bpf/bpf_experimental.h+9−6

@@ -368,12 +368,12 @@ l_true: \ · 367 unmodified lines
368	368	ret; \
369	369	})
370	370
371		−#define cond_break \
	371	+#define __cond_break(expr) \
372	372	({ __label__ l_break, l_continue; \
373	373	asm volatile goto("may_goto %l[l_break]" \
374	374	:::: l_break); \
375	375	goto l_continue; \
376		− l_break: break; \
	376	+ l_break: expr; \
377	377	l_continue:; \
378	378	})
379	379	#else
@@ -392,7 +392,7 @@ l_true: \ · 12 unmodified lines
392	392	ret; \
393	393	})
394	394
395		−#define cond_break \
	395	+#define __cond_break(expr) \
396	396	({ __label__ l_break, l_continue; \
397	397	asm volatile goto("1:.byte 0xe5; \
398	398	.byte 0; \
@@ -400,7 +400,7 @@ l_true: \ · 1 unmodified line
400	400	.short 0" \
401	401	:::: l_break); \
402	402	goto l_continue; \
403		− l_break: break; \
	403	+ l_break: expr; \
404	404	l_continue:; \
405	405	})
406	406	#else
@@ -418,7 +418,7 @@ l_true: \ · 11 unmodified lines
418	418	ret; \
419	419	})
420	420
421		−#define cond_break \
	421	+#define __cond_break(expr) \
422	422	({ __label__ l_break, l_continue; \
423	423	asm volatile goto("1:.byte 0xe5; \
424	424	.byte 0; \
@@ -426,12 +426,15 @@ l_true: \ · 1 unmodified line
426	426	.short 0" \
427	427	:::: l_break); \
428	428	goto l_continue; \
429		− l_break: break; \
	429	+ l_break: expr; \
430	430	l_continue:; \
431	431	})
432	432	#endif
433	433	#endif
434	434
	435	+#define cond_break __cond_break(break)
	436	+#define cond_break_label(label) __cond_break(goto label)
	437	+
435	438	#ifndef bpf_nop_mov
436	439	#define bpf_nop_mov(var) \
437	440	asm volatile("%[reg]=%[reg]"::[reg]"r"((short)var))

tools/testing/selftests/bpf/bpf_kfuncs.h+5−0

@@ -87,4 +87,9 @@ struct dentry; · 86 unmodified lines
87	87	*/
88	88	extern int bpf_get_dentry_xattr(struct dentry dentry, const char name,
89	89	struct bpf_dynptr *value_ptr) __ksym __weak;
	90	+
	91	+extern int bpf_set_dentry_xattr(struct dentry dentry, const char name__str,
	92	+ const struct bpf_dynptr *value_p, int flags) __ksym __weak;
	93	+extern int bpf_remove_dentry_xattr(struct dentry dentry, const char name__str) __ksym __weak;
	94	+
90	95	#endif

tools/testing/selftests/bpf/cap_helpers.c+4−4

@@ -19,7 +19,7 @@ int cap_enable_effective(__u64 caps, __u64 *old_caps) · 18 unmodified lines
19	19
20	20	err = capget(&hdr, data);
21	21	if (err)
22		− return err;
	22	+ return -errno;
23	23
24	24	if (old_caps)
25	25	*old_caps = (__u64)(data[1].effective) << 32 \| data[0].effective;
@@ -32,7 +32,7 @@ int cap_enable_effective(__u64 caps, __u64 *old_caps) · 6 unmodified lines
32	32	data[1].effective \|= cap1;
33	33	err = capset(&hdr, data);
34	34	if (err)
35		− return err;
	35	+ return -errno;
36	36
37	37	return 0;
38	38	}
@@ -49,7 +49,7 @@ int cap_disable_effective(__u64 caps, __u64 *old_caps) · 10 unmodified lines
49	49
50	50	err = capget(&hdr, data);
51	51	if (err)
52		− return err;
	52	+ return -errno;
53	53
54	54	if (old_caps)
55	55	*old_caps = (__u64)(data[1].effective) << 32 \| data[0].effective;
@@ -61,7 +61,7 @@ int cap_disable_effective(__u64 caps, __u64 *old_caps) · 5 unmodified lines
61	61	data[1].effective &= ~cap1;
62	62	err = capset(&hdr, data);
63	63	if (err)
64		− return err;
	64	+ return -errno;
65	65
66	66	return 0;
67	67	}

tools/testing/selftests/bpf/cap_helpers.h+1−0

@@ -4,6 +4,7 @@ · 3 unmodified lines
4	4
5	5	#include <linux/types.h>
6	6	#include <linux/capability.h>
	7	+#include <errno.h>
7	8
8	9	#ifndef CAP_PERFMON
9	10	#define CAP_PERFMON 38

tools/testing/selftests/bpf/network_helpers.c+76−35

@@ -446,6 +446,23 @@ char *ping_command(int family) · 445 unmodified lines
446	446	return "ping";
447	447	}
448	448
	449	+int append_tid(char *str, size_t sz)
	450	+{
	451	+ size_t end;
	452	+
	453	+ if (!str)
	454	+ return -1;
	455	+
	456	+ end = strlen(str);
	457	+ if (end + 8 > sz)
	458	+ return -1;
	459	+
	460	+ sprintf(&str[end], "%07d", gettid());
	461	+ str[end + 7] = '\0';
	462	+
	463	+ return 0;
	464	+}
	465	+
449	466	int remove_netns(const char *name)
450	467	{
451	468	char *cmd;
@@ -761,6 +778,36 @@ struct tmonitor_ctx { · 309 unmodified lines
761	778	int pcap_fd;
762	779	};
763	780
	781	+static int __base_pr(const char *format, va_list args)
	782	+{
	783	+ return vfprintf(stdout, format, args);
	784	+}
	785	+
	786	+static tm_print_fn_t __tm_pr = __base_pr;
	787	+
	788	+tm_print_fn_t traffic_monitor_set_print(tm_print_fn_t fn)
	789	+{
	790	+ tm_print_fn_t old_print_fn;
	791	+
	792	+ old_print_fn = __atomic_exchange_n(&__tm_pr, fn, __ATOMIC_RELAXED);
	793	+
	794	+ return old_print_fn;
	795	+}
	796	+
	797	+void tm_print(const char *format, ...)
	798	+{
	799	+ tm_print_fn_t print_fn;
	800	+ va_list args;
	801	+
	802	+ print_fn = __atomic_load_n(&__tm_pr, __ATOMIC_RELAXED);
	803	+ if (!print_fn)
	804	+ return;
	805	+
	806	+ va_start(args, format);
	807	+ print_fn(format, args);
	808	+ va_end(args);
	809	+}
	810	+
764	811	/* Is this packet captured with a Ethernet protocol type? */
765	812	static bool is_ethernet(const u_char *packet)
766	813	{
@@ -778,7 +825,7 @@ static bool is_ethernet(const u_char *packet) · 11 unmodified lines
778	825	case 770: /* ARPHRD_FRAD */
779	826	case 778: /* ARPHDR_IPGRE */
780	827	case 803: /* ARPHRD_IEEE80211_RADIOTAP */
781		− printf("Packet captured: arphdr_type=%d\n", arphdr_type);
	828	+ tm_print("Packet captured: arphdr_type=%d\n", arphdr_type);
782	829	return false;
783	830	}
784	831	return true;
@@ -799,12 +846,13 @@ static const char *pkt_type_str(u16 pkt_type) · 14 unmodified lines
799	846	return "Unknown";
800	847	}
801	848
	849	+#define MAX_FLAGS_STRLEN 21
802	850	/* Show the information of the transport layer in the packet */
803	851	static void show_transport(const u_char *packet, u16 len, u32 ifindex,
804	852	const char src_addr, const char dst_addr,
805	853	u16 proto, bool ipv6, u8 pkt_type)
806	854	{
807		− char *ifname, _ifname[IF_NAMESIZE];
	855	+ char *ifname, _ifname[IF_NAMESIZE], flags[MAX_FLAGS_STRLEN] = "";
808	856	const char *transport_str;
809	857	u16 src_port, dst_port;
810	858	struct udphdr *udp;
@@ -827,47 +875,39 @@ static void show_transport(const u_char *packet, u16 len, u32 ifindex, · 16 unmodified lines
827	875	dst_port = ntohs(tcp->dest);
828	876	transport_str = "TCP";
829	877	} else if (proto == IPPROTO_ICMP) {
830		− printf("%-7s %-3s IPv4 %s > %s: ICMP, length %d, type %d, code %d\n",
831		− ifname, pkt_type_str(pkt_type), src_addr, dst_addr, len,
832		− packet[0], packet[1]);
	878	+ tm_print("%-7s %-3s IPv4 %s > %s: ICMP, length %d, type %d, code %d\n",
	879	+ ifname, pkt_type_str(pkt_type), src_addr, dst_addr, len,
	880	+ packet[0], packet[1]);
833	881	return;
834	882	} else if (proto == IPPROTO_ICMPV6) {
835		− printf("%-7s %-3s IPv6 %s > %s: ICMPv6, length %d, type %d, code %d\n",
836		− ifname, pkt_type_str(pkt_type), src_addr, dst_addr, len,
837		− packet[0], packet[1]);
	883	+ tm_print("%-7s %-3s IPv6 %s > %s: ICMPv6, length %d, type %d, code %d\n",
	884	+ ifname, pkt_type_str(pkt_type), src_addr, dst_addr, len,
	885	+ packet[0], packet[1]);
838	886	return;
839	887	} else {
840		− printf("%-7s %-3s %s %s > %s: protocol %d\n",
841		− ifname, pkt_type_str(pkt_type), ipv6 ? "IPv6" : "IPv4",
842		− src_addr, dst_addr, proto);
	888	+ tm_print("%-7s %-3s %s %s > %s: protocol %d\n",
	889	+ ifname, pkt_type_str(pkt_type), ipv6 ? "IPv6" : "IPv4",
	890	+ src_addr, dst_addr, proto);
843	891	return;
844	892	}
845	893
846	894	/* TCP or UDP*/
847	895
848		− flockfile(stdout);
	896	+ if (proto == IPPROTO_TCP)
	897	+ snprintf(flags, MAX_FLAGS_STRLEN, "%s%s%s%s",
	898	+ tcp->fin ? ", FIN" : "",
	899	+ tcp->syn ? ", SYN" : "",
	900	+ tcp->rst ? ", RST" : "",
	901	+ tcp->ack ? ", ACK" : "");
	902	+
849	903	if (ipv6)
850		− printf("%-7s %-3s IPv6 %s.%d > %s.%d: %s, length %d",
851		− ifname, pkt_type_str(pkt_type), src_addr, src_port,
852		− dst_addr, dst_port, transport_str, len);
	904	+ tm_print("%-7s %-3s IPv6 %s.%d > %s.%d: %s, length %d%s\n",
	905	+ ifname, pkt_type_str(pkt_type), src_addr, src_port,
	906	+ dst_addr, dst_port, transport_str, len, flags);
853	907	else
854		− printf("%-7s %-3s IPv4 %s:%d > %s:%d: %s, length %d",
855		− ifname, pkt_type_str(pkt_type), src_addr, src_port,
856		− dst_addr, dst_port, transport_str, len);
857		−
858		− if (proto == IPPROTO_TCP) {
859		− if (tcp->fin)
860		− printf(", FIN");
861		− if (tcp->syn)
862		− printf(", SYN");
863		− if (tcp->rst)
864		− printf(", RST");
865		− if (tcp->ack)
866		− printf(", ACK");
867		− }
868		−
869		− printf("\n");
870		− funlockfile(stdout);
	908	+ tm_print("%-7s %-3s IPv4 %s:%d > %s:%d: %s, length %d%s\n",
	909	+ ifname, pkt_type_str(pkt_type), src_addr, src_port,
	910	+ dst_addr, dst_port, transport_str, len, flags);
871	911	}
872	912
873	913	static void show_ipv6_packet(const u_char *packet, u32 ifindex, u8 pkt_type)
@@ -982,8 +1022,8 @@ static void traffic_monitor_thread(void arg) · 108 unmodified lines
982	1022	ifname = _ifname;
983	1023	}
984	1024
985		− printf("%-7s %-3s Unknown network protocol type 0x%x\n",
986		− ifname, pkt_type_str(ptype), proto);
	1025	+ tm_print("%-7s %-3s Unknown network protocol type 0x%x\n",
	1026	+ ifname, pkt_type_str(ptype), proto);
987	1027	}
988	1028	}
989	1029
@@ -1183,8 +1223,9 @@ void traffic_monitor_stop(struct tmonitor_ctx *ctx) · 193 unmodified lines
1183	1223	write(ctx->wake_fd, &w, sizeof(w));
1184	1224	pthread_join(ctx->thread, NULL);
1185	1225
1186		− printf("Packet file: %s\n", strrchr(ctx->pkt_fname, '/') + 1);
	1226	+ tm_print("Packet file: %s\n", strrchr(ctx->pkt_fname, '/') + 1);
1187	1227
1188	1228	traffic_monitor_release(ctx);
1189	1229	}
	1230	+
1190	1231	#endif /* TRAFFIC_MONITOR */

tools/testing/selftests/bpf/network_helpers.h+21−0

@@ -18,6 +18,7 @@ typedef __u16 __sum16; · 17 unmodified lines
18	18	#include <netinet/udp.h>
19	19	#include <bpf/bpf_endian.h>
20	20	#include <net/if.h>
	21	+#include <stdio.h>
21	22
22	23	#define MAGIC_VAL 0x1234
23	24	#define NUM_ITER 100000
@@ -101,6 +102,18 @@ int send_recv_data(int lfd, int fd, uint32_t total_bytes); · 77 unmodified lines
101	102	int make_netns(const char *name);
102	103	int remove_netns(const char *name);
103	104
	105	+/**
	106	+ * append_tid() - Append thread ID to the given string.
	107	+ *
	108	+ * @str: string to extend
	109	+ * @sz: string's size
	110	+ *
	111	+ * 8 characters are used to append the thread ID (7 digits + '\0')
	112	+ *
	113	+ * Returns -1 on errors, 0 otherwise
	114	+ */
	115	+int append_tid(char *str, size_t sz);
	116	+
104	117	static __u16 csum_fold(__u32 csum)
105	118	{
106	119	csum = (csum & 0xffff) + (csum >> 16);
@@ -240,10 +253,13 @@ static inline __sum16 build_udp_v6_csum(const struct ipv6hdr *ip6h, · 133 unmodified lines
240	253
241	254	struct tmonitor_ctx;
242	255
	256	+typedef int (tm_print_fn_t)(const char format, va_list args);
	257	+
243	258	#ifdef TRAFFIC_MONITOR
244	259	struct tmonitor_ctx traffic_monitor_start(const char netns, const char *test_name,
245	260	const char *subtest_name);
246	261	void traffic_monitor_stop(struct tmonitor_ctx *ctx);
	262	+tm_print_fn_t traffic_monitor_set_print(tm_print_fn_t fn);
247	263	#else
248	264	static inline struct tmonitor_ctx traffic_monitor_start(const char netns, const char *test_name,
249	265	const char *subtest_name)
@@ -254,6 +270,11 @@ static inline struct tmonitor_ctx traffic_monitor_start(const char netns, cons · 4 unmodified lines
254	270	static inline void traffic_monitor_stop(struct tmonitor_ctx *ctx)
255	271	{
256	272	}
	273	+
	274	+static inline tm_print_fn_t traffic_monitor_set_print(tm_print_fn_t fn)
	275	+{
	276	+ return NULL;
	277	+}
257	278	#endif
258	279
259	280	#endif

tools/testing/selftests/bpf/prog_tests/align.c+10−1

@@ -610,9 +610,11 @@ static int do_test_single(struct bpf_align_test *test) · 609 unmodified lines
610	610	.log_size = sizeof(bpf_vlog),
611	611	.log_level = 2,
612	612	);
	613	+ const char *main_pass_start = "0: R1=ctx() R10=fp0";
613	614	const char *line_ptr;
614	615	int cur_line = -1;
615	616	int prog_len, i;
	617	+ char *start;
616	618	int fd_prog;
617	619	int ret;
618	620
@@ -632,7 +634,13 @@ static int do_test_single(struct bpf_align_test *test) · 13 unmodified lines
632	634	ret = 0;
633	635	/* We make a local copy so that we can strtok() it */
634	636	strncpy(bpf_vlog_copy, bpf_vlog, sizeof(bpf_vlog_copy));
635		− line_ptr = strtok(bpf_vlog_copy, "\n");
	637	+ start = strstr(bpf_vlog_copy, main_pass_start);
	638	+ if (!start) {
	639	+ ret = 1;
	640	+ printf("Can't find initial line '%s'\n", main_pass_start);
	641	+ goto out;
	642	+ }
	643	+ line_ptr = strtok(start, "\n");
636	644	for (i = 0; i < MAX_MATCHES; i++) {
637	645	struct bpf_reg_match m = test->matches[i];
638	646	const char *p;
@@ -682,6 +690,7 @@ static int do_test_single(struct bpf_align_test *test) · 43 unmodified lines
682	690	break;
683	691	}
684	692	}
	693	+out:
685	694	if (fd_prog >= 0)
686	695	close(fd_prog);
687	696	}

tools/testing/selftests/bpf/prog_tests/arena_atomics.c+65−1

@@ -162,6 +162,66 @@ static void test_uaf(struct arena_atomics *skel) · 161 unmodified lines
162	162	ASSERT_EQ(skel->arena->uaf_recovery_fails, 0, "uaf_recovery_fails");
163	163	}
164	164
	165	+static void test_load_acquire(struct arena_atomics *skel)
	166	+{
	167	+ LIBBPF_OPTS(bpf_test_run_opts, topts);
	168	+ int err, prog_fd;
	169	+
	170	+ if (skel->data->skip_lacq_srel_tests) {
	171	+ printf("%s:SKIP: ENABLE_ATOMICS_TESTS not defined, Clang doesn't support addr_space_cast, and/or JIT doesn't support load-acquire\n",
	172	+ __func__);
	173	+ test__skip();
	174	+ return;
	175	+ }
	176	+
	177	+ /* No need to attach it, just run it directly */
	178	+ prog_fd = bpf_program__fd(skel->progs.load_acquire);
	179	+ err = bpf_prog_test_run_opts(prog_fd, &topts);
	180	+ if (!ASSERT_OK(err, "test_run_opts err"))
	181	+ return;
	182	+ if (!ASSERT_OK(topts.retval, "test_run_opts retval"))
	183	+ return;
	184	+
	185	+ ASSERT_EQ(skel->arena->load_acquire8_result, 0x12,
	186	+ "load_acquire8_result");
	187	+ ASSERT_EQ(skel->arena->load_acquire16_result, 0x1234,
	188	+ "load_acquire16_result");
	189	+ ASSERT_EQ(skel->arena->load_acquire32_result, 0x12345678,
	190	+ "load_acquire32_result");
	191	+ ASSERT_EQ(skel->arena->load_acquire64_result, 0x1234567890abcdef,
	192	+ "load_acquire64_result");
	193	+}
	194	+
	195	+static void test_store_release(struct arena_atomics *skel)
	196	+{
	197	+ LIBBPF_OPTS(bpf_test_run_opts, topts);
	198	+ int err, prog_fd;
	199	+
	200	+ if (skel->data->skip_lacq_srel_tests) {
	201	+ printf("%s:SKIP: ENABLE_ATOMICS_TESTS not defined, Clang doesn't support addr_space_cast, and/or JIT doesn't support store-release\n",
	202	+ __func__);
	203	+ test__skip();
	204	+ return;
	205	+ }
	206	+
	207	+ /* No need to attach it, just run it directly */
	208	+ prog_fd = bpf_program__fd(skel->progs.store_release);
	209	+ err = bpf_prog_test_run_opts(prog_fd, &topts);
	210	+ if (!ASSERT_OK(err, "test_run_opts err"))
	211	+ return;
	212	+ if (!ASSERT_OK(topts.retval, "test_run_opts retval"))
	213	+ return;
	214	+
	215	+ ASSERT_EQ(skel->arena->store_release8_result, 0x12,
	216	+ "store_release8_result");
	217	+ ASSERT_EQ(skel->arena->store_release16_result, 0x1234,
	218	+ "store_release16_result");
	219	+ ASSERT_EQ(skel->arena->store_release32_result, 0x12345678,
	220	+ "store_release32_result");
	221	+ ASSERT_EQ(skel->arena->store_release64_result, 0x1234567890abcdef,
	222	+ "store_release64_result");
	223	+}
	224	+
165	225	void test_arena_atomics(void)
166	226	{
167	227	struct arena_atomics *skel;
@@ -171,7 +231,7 @@ void test_arena_atomics(void) · 3 unmodified lines
171	231	if (!ASSERT_OK_PTR(skel, "arena atomics skeleton open"))
172	232	return;
173	233
174		− if (skel->data->skip_tests) {
	234	+ if (skel->data->skip_all_tests) {
175	235	printf("%s:SKIP:no ENABLE_ATOMICS_TESTS or no addr_space_cast support in clang",
176	236	__func__);
177	237	test__skip();
@@ -198,6 +258,10 @@ void test_arena_atomics(void) · 20 unmodified lines
198	258	test_xchg(skel);
199	259	if (test__start_subtest("uaf"))
200	260	test_uaf(skel);
	261	+ if (test__start_subtest("load_acquire"))
	262	+ test_load_acquire(skel);
	263	+ if (test__start_subtest("store_release"))
	264	+ test_store_release(skel);
201	265
202	266	cleanup:
203	267	arena_atomics__destroy(skel);

tools/testing/selftests/bpf/prog_tests/arena_spin_lock.c+108−0

@@ -0,0 +1,108 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
	3	+#include <test_progs.h>
	4	+#include <network_helpers.h>
	5	+#include <sys/sysinfo.h>
	6	+
	7	+struct __qspinlock { int val; };
	8	+typedef struct __qspinlock arena_spinlock_t;
	9	+
	10	+struct arena_qnode {
	11	+ unsigned long next;
	12	+ int count;
	13	+ int locked;
	14	+};
	15	+
	16	+#include "arena_spin_lock.skel.h"
	17	+
	18	+static long cpu;
	19	+static int repeat;
	20	+
	21	+pthread_barrier_t barrier;
	22	+
	23	+static void spin_lock_thread(void arg)
	24	+{
	25	+ int err, prog_fd = (u32 )arg;
	26	+ LIBBPF_OPTS(bpf_test_run_opts, topts,
	27	+ .data_in = &pkt_v4,
	28	+ .data_size_in = sizeof(pkt_v4),
	29	+ .repeat = repeat,
	30	+ );
	31	+ cpu_set_t cpuset;
	32	+
	33	+ CPU_ZERO(&cpuset);
	34	+ CPU_SET(__sync_fetch_and_add(&cpu, 1), &cpuset);
	35	+ ASSERT_OK(pthread_setaffinity_np(pthread_self(), sizeof(cpuset), &cpuset), "cpu affinity");
	36	+
	37	+ err = pthread_barrier_wait(&barrier);
	38	+ if (err != PTHREAD_BARRIER_SERIAL_THREAD && err != 0)
	39	+ ASSERT_FALSE(true, "pthread_barrier");
	40	+
	41	+ err = bpf_prog_test_run_opts(prog_fd, &topts);
	42	+ ASSERT_OK(err, "test_run err");
	43	+ ASSERT_EQ((int)topts.retval, 0, "test_run retval");
	44	+
	45	+ pthread_exit(arg);
	46	+}
	47	+
	48	+static void test_arena_spin_lock_size(int size)
	49	+{
	50	+ LIBBPF_OPTS(bpf_test_run_opts, topts);
	51	+ struct arena_spin_lock *skel;
	52	+ pthread_t thread_id[16];
	53	+ int prog_fd, i, err;
	54	+ void *ret;
	55	+
	56	+ if (get_nprocs() < 2) {
	57	+ test__skip();
	58	+ return;
	59	+ }
	60	+
	61	+ skel = arena_spin_lock__open_and_load();
	62	+ if (!ASSERT_OK_PTR(skel, "arena_spin_lock__open_and_load"))
	63	+ return;
	64	+ if (skel->data->test_skip == 2) {
	65	+ test__skip();
	66	+ goto end;
	67	+ }
	68	+ skel->bss->cs_count = size;
	69	+ skel->bss->limit = repeat * 16;
	70	+
	71	+ ASSERT_OK(pthread_barrier_init(&barrier, NULL, 16), "barrier init");
	72	+
	73	+ prog_fd = bpf_program__fd(skel->progs.prog);
	74	+ for (i = 0; i < 16; i++) {
	75	+ err = pthread_create(&thread_id[i], NULL, &spin_lock_thread, &prog_fd);
	76	+ if (!ASSERT_OK(err, "pthread_create"))
	77	+ goto end_barrier;
	78	+ }
	79	+
	80	+ for (i = 0; i < 16; i++) {
	81	+ if (!ASSERT_OK(pthread_join(thread_id[i], &ret), "pthread_join"))
	82	+ goto end_barrier;
	83	+ if (!ASSERT_EQ(ret, &prog_fd, "ret == prog_fd"))
	84	+ goto end_barrier;
	85	+ }
	86	+
	87	+ ASSERT_EQ(skel->bss->counter, repeat * 16, "check counter value");
	88	+
	89	+end_barrier:
	90	+ pthread_barrier_destroy(&barrier);
	91	+end:
	92	+ arena_spin_lock__destroy(skel);
	93	+ return;
	94	+}
	95	+
	96	+void test_arena_spin_lock(void)
	97	+{
	98	+ repeat = 1000;
	99	+ if (test__start_subtest("arena_spin_lock_1"))
	100	+ test_arena_spin_lock_size(1);
	101	+ cpu = 0;
	102	+ if (test__start_subtest("arena_spin_lock_1000"))
	103	+ test_arena_spin_lock_size(1000);
	104	+ cpu = 0;
	105	+ repeat = 100;
	106	+ if (test__start_subtest("arena_spin_lock_50000"))
	107	+ test_arena_spin_lock_size(50000);
	108	+}

tools/testing/selftests/bpf/prog_tests/bloom_filter_map.c+5−0

@@ -6,6 +6,10 @@ · 5 unmodified lines
6	6	#include <test_progs.h>
7	7	#include "bloom_filter_map.skel.h"
8	8
	9	+#ifndef NUMA_NO_NODE
	10	+#define NUMA_NO_NODE (-1)
	11	+#endif
	12	+
9	13	static void test_fail_cases(void)
10	14	{
11	15	LIBBPF_OPTS(bpf_map_create_opts, opts);
@@ -69,6 +73,7 @@ static void test_success_cases(void) · 57 unmodified lines
69	73
70	74	/* Create a map */
71	75	opts.map_flags = BPF_F_ZERO_SEED \| BPF_F_NUMA_NODE;
	76	+ opts.numa_node = NUMA_NO_NODE;
72	77	fd = bpf_map_create(BPF_MAP_TYPE_BLOOM_FILTER, NULL, 0, sizeof(value), 100, &opts);
73	78	if (!ASSERT_GE(fd, 0, "bpf_map_create bloom filter success case"))
74	79	return;

tools/testing/selftests/bpf/prog_tests/bpf_iter.c+68−0

@@ -323,19 +323,87 @@ static void test_task_pidfd(void) · 322 unmodified lines
323	323	static void test_task_sleepable(void)
324	324	{
325	325	struct bpf_iter_tasks *skel;
	326	+ int pid, status, err, data_pipe[2], finish_pipe[2], c;
	327	+ char *test_data = NULL;
	328	+ char *test_data_long = NULL;
	329	+ char *data[2];
	330	+
	331	+ if (!ASSERT_OK(pipe(data_pipe), "data_pipe") \|\|
	332	+ !ASSERT_OK(pipe(finish_pipe), "finish_pipe"))
	333	+ return;
326	334
327	335	skel = bpf_iter_tasks__open_and_load();
328	336	if (!ASSERT_OK_PTR(skel, "bpf_iter_tasks__open_and_load"))
329	337	return;
330	338
	339	+ pid = fork();
	340	+ if (!ASSERT_GE(pid, 0, "fork"))
	341	+ return;
	342	+
	343	+ if (pid == 0) {
	344	+ /* child */
	345	+ close(data_pipe[0]);
	346	+ close(finish_pipe[1]);
	347	+
	348	+ test_data = malloc(sizeof(char) * 10);
	349	+ strncpy(test_data, "test_data", 10);
	350	+ test_data[9] = '\0';
	351	+
	352	+ test_data_long = malloc(sizeof(char) * 5000);
	353	+ for (int i = 0; i < 5000; ++i) {
	354	+ if (i % 2 == 0)
	355	+ test_data_long[i] = 'b';
	356	+ else
	357	+ test_data_long[i] = 'a';
	358	+ }
	359	+ test_data_long[4999] = '\0';
	360	+
	361	+ data[0] = test_data;
	362	+ data[1] = test_data_long;
	363	+
	364	+ write(data_pipe[1], &data, sizeof(data));
	365	+
	366	+ /* keep child alive until after the test */
	367	+ err = read(finish_pipe[0], &c, 1);
	368	+ if (err != 1)
	369	+ exit(-1);
	370	+
	371	+ close(data_pipe[1]);
	372	+ close(finish_pipe[0]);
	373	+ _exit(0);
	374	+ }
	375	+
	376	+ /* parent */
	377	+ close(data_pipe[1]);
	378	+ close(finish_pipe[0]);
	379	+
	380	+ err = read(data_pipe[0], &data, sizeof(data));
	381	+ ASSERT_EQ(err, sizeof(data), "read_check");
	382	+
	383	+ skel->bss->user_ptr = data[0];
	384	+ skel->bss->user_ptr_long = data[1];
	385	+ skel->bss->pid = pid;
	386	+
331	387	do_dummy_read(skel->progs.dump_task_sleepable);
332	388
333	389	ASSERT_GT(skel->bss->num_expected_failure_copy_from_user_task, 0,
334	390	"num_expected_failure_copy_from_user_task");
335	391	ASSERT_GT(skel->bss->num_success_copy_from_user_task, 0,
336	392	"num_success_copy_from_user_task");
	393	+ ASSERT_GT(skel->bss->num_expected_failure_copy_from_user_task_str, 0,
	394	+ "num_expected_failure_copy_from_user_task_str");
	395	+ ASSERT_GT(skel->bss->num_success_copy_from_user_task_str, 0,
	396	+ "num_success_copy_from_user_task_str");
337	397
338	398	bpf_iter_tasks__destroy(skel);
	399	+
	400	+ write(finish_pipe[1], &c, 1);
	401	+ err = waitpid(pid, &status, 0);
	402	+ ASSERT_EQ(err, pid, "waitpid");
	403	+ ASSERT_EQ(status, 0, "zero_child_exit");
	404	+
	405	+ close(data_pipe[0]);
	406	+ close(finish_pipe[1]);
339	407	}
340	408
341	409	static void test_task_stack(void)

tools/testing/selftests/bpf/prog_tests/bpf_nf.c+6−3

@@ -72,11 +72,14 @@ static void test_bpf_nf_ct(int mode) · 71 unmodified lines
72	72	if (!ASSERT_OK(system(cmd), cmd))
73	73	goto end;
74	74
75		− srv_port = (mode == TEST_XDP) ? 5005 : 5006;
76		− srv_fd = start_server(AF_INET, SOCK_STREAM, "127.0.0.1", srv_port, TIMEOUT_MS);
	75	+ srv_fd = start_server(AF_INET, SOCK_STREAM, "127.0.0.1", 0, TIMEOUT_MS);
77	76	if (!ASSERT_GE(srv_fd, 0, "start_server"))
78	77	goto end;
79	78
	79	+ srv_port = get_socket_local_port(srv_fd);
	80	+ if (!ASSERT_GE(srv_port, 0, "get_sock_local_port"))
	81	+ goto end;
	82	+
80	83	client_fd = connect_to_server(srv_fd);
81	84	if (!ASSERT_GE(client_fd, 0, "connect_to_server"))
82	85	goto end;
@@ -91,7 +94,7 @@ static void test_bpf_nf_ct(int mode) · 8 unmodified lines
91	94	skel->bss->saddr = peer_addr.sin_addr.s_addr;
92	95	skel->bss->sport = peer_addr.sin_port;
93	96	skel->bss->daddr = peer_addr.sin_addr.s_addr;
94		− skel->bss->dport = htons(srv_port);
	97	+ skel->bss->dport = srv_port;
95	98
96	99	if (mode == TEST_XDP)
97	100	prog_fd = bpf_program__fd(skel->progs.nf_xdp_ct_test);

tools/testing/selftests/bpf/prog_tests/btf.c+19−4

@@ -3866,11 +3866,11 @@ static struct btf_raw_test raw_tests[] = { · 3865 unmodified lines
3866	3866	.err_str = "vlen != 0",
3867	3867	},
3868	3868	{
3869		− .descr = "decl_tag test #8, invalid kflag",
	3869	+ .descr = "decl_tag test #8, tag with kflag",
3870	3870	.raw_types = {
3871	3871	BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
3872	3872	BTF_VAR_ENC(NAME_TBD, 1, 0), /* [2] */
3873		− BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_DECL_TAG, 1, 0), 2), (-1),
	3873	+ BTF_DECL_ATTR_ENC(NAME_TBD, 2, -1),
3874	3874	BTF_END_RAW,
3875	3875	},
3876	3876	BTF_STR_SEC("\0local\0tag1"),
@@ -3881,8 +3881,6 @@ static struct btf_raw_test raw_tests[] = { · 4 unmodified lines
3881	3881	.key_type_id = 1,
3882	3882	.value_type_id = 1,
3883	3883	.max_entries = 1,
3884		− .btf_load_err = true,
3885		− .err_str = "Invalid btf_info kind_flag",
3886	3884	},
3887	3885	{
3888	3886	.descr = "decl_tag test #9, var, invalid component_idx",
@@ -4206,6 +4204,23 @@ static struct btf_raw_test raw_tests[] = { · 317 unmodified lines
4206	4204	.btf_load_err = true,
4207	4205	.err_str = "Type tags don't precede modifiers",
4208	4206	},
	4207	+{
	4208	+ .descr = "type_tag test #7, tag with kflag",
	4209	+ .raw_types = {
	4210	+ BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
	4211	+ BTF_TYPE_ATTR_ENC(NAME_TBD, 1), /* [2] */
	4212	+ BTF_PTR_ENC(2), /* [3] */
	4213	+ BTF_END_RAW,
	4214	+ },
	4215	+ BTF_STR_SEC("\0tag"),
	4216	+ .map_type = BPF_MAP_TYPE_ARRAY,
	4217	+ .map_name = "tag_type_check_btf",
	4218	+ .key_size = sizeof(int),
	4219	+ .value_size = 4,
	4220	+ .key_type_id = 1,
	4221	+ .value_type_id = 1,
	4222	+ .max_entries = 1,
	4223	+},
4209	4224	{
4210	4225	.descr = "enum64 test #1, unsigned, size 8",
4211	4226	.raw_types = {

tools/testing/selftests/bpf/prog_tests/btf_dump.c+110−37

@@ -126,26 +126,69 @@ done: · 125 unmodified lines
126	126	return err;
127	127	}
128	128
129		−static char *dump_buf;
130		−static size_t dump_buf_sz;
131		−static FILE *dump_buf_file;
	129	+struct test_ctx {
	130	+ struct btf *btf;
	131	+ struct btf_dump *d;
	132	+ char *dump_buf;
	133	+ size_t dump_buf_sz;
	134	+ FILE *dump_buf_file;
	135	+};
132	136
133		−static void test_btf_dump_incremental(void)
	137	+static void test_ctx__free(struct test_ctx *t)
134	138	{
135		− struct btf *btf = NULL;
136		− struct btf_dump *d = NULL;
137		− int id, err, i;
	139	+ fclose(t->dump_buf_file);
	140	+ free(t->dump_buf);
	141	+ btf_dump__free(t->d);
	142	+ btf__free(t->btf);
	143	+}
138	144
139		− dump_buf_file = open_memstream(&dump_buf, &dump_buf_sz);
140		− if (!ASSERT_OK_PTR(dump_buf_file, "dump_memstream"))
141		− return;
142		− btf = btf__new_empty();
143		− if (!ASSERT_OK_PTR(btf, "new_empty"))
	145	+static int test_ctx__init(struct test_ctx *t)
	146	+{
	147	+ t->dump_buf_file = open_memstream(&t->dump_buf, &t->dump_buf_sz);
	148	+ if (!ASSERT_OK_PTR(t->dump_buf_file, "dump_memstream"))
	149	+ return -1;
	150	+ t->btf = btf__new_empty();
	151	+ if (!ASSERT_OK_PTR(t->btf, "new_empty"))
144	152	goto err_out;
145		− d = btf_dump__new(btf, btf_dump_printf, dump_buf_file, NULL);
146		− if (!ASSERT_OK(libbpf_get_error(d), "btf_dump__new"))
	153	+ t->d = btf_dump__new(t->btf, btf_dump_printf, t->dump_buf_file, NULL);
	154	+ if (!ASSERT_OK(libbpf_get_error(t->d), "btf_dump__new"))
147	155	goto err_out;
148	156
	157	+ return 0;
	158	+
	159	+err_out:
	160	+ test_ctx__free(t);
	161	+ return -1;
	162	+}
	163	+
	164	+static void test_ctx__dump_and_compare(struct test_ctx *t,
	165	+ const char *expected_output,
	166	+ const char *message)
	167	+{
	168	+ int i, err;
	169	+
	170	+ for (i = 1; i < btf__type_cnt(t->btf); i++) {
	171	+ err = btf_dump__dump_type(t->d, i);
	172	+ ASSERT_OK(err, "dump_type_ok");
	173	+ }
	174	+
	175	+ fflush(t->dump_buf_file);
	176	+ t->dump_buf[t->dump_buf_sz] = 0; /* some libc implementations don't do this */
	177	+
	178	+ ASSERT_STREQ(t->dump_buf, expected_output, message);
	179	+}
	180	+
	181	+static void test_btf_dump_incremental(void)
	182	+{
	183	+ struct test_ctx t = {};
	184	+ struct btf *btf;
	185	+ int id, err;
	186	+
	187	+ if (test_ctx__init(&t))
	188	+ return;
	189	+
	190	+ btf = t.btf;
	191	+
149	192	/* First, generate BTF corresponding to the following C code:
150	193	*
151	194	* enum x;
@@ -182,15 +225,7 @@ static void test_btf_dump_incremental(void) · 30 unmodified lines
182	225	err = btf__add_field(btf, "x", 4, 0, 0);
183	226	ASSERT_OK(err, "field_ok");
184	227
185		− for (i = 1; i < btf__type_cnt(btf); i++) {
186		− err = btf_dump__dump_type(d, i);
187		− ASSERT_OK(err, "dump_type_ok");
188		− }
189		−
190		− fflush(dump_buf_file);
191		− dump_buf[dump_buf_sz] = 0; /* some libc implementations don't do this */
192		−
193		− ASSERT_STREQ(dump_buf,
	228	+ test_ctx__dump_and_compare(&t,
194	229	"enum x;\n"
195	230	"\n"
196	231	"enum x {\n"
@@ -221,7 +256,7 @@ static void test_btf_dump_incremental(void) · 24 unmodified lines
221	256	* enum values don't conflict;
222	257	*
223	258	*/
224		− fseek(dump_buf_file, 0, SEEK_SET);
	259	+ fseek(t.dump_buf_file, 0, SEEK_SET);
225	260
226	261	id = btf__add_struct(btf, "s", 4);
227	262	ASSERT_EQ(id, 7, "struct_id");
@@ -232,14 +267,7 @@ static void test_btf_dump_incremental(void) · 4 unmodified lines
232	267	err = btf__add_field(btf, "s", 6, 64, 0);
233	268	ASSERT_OK(err, "field_ok");
234	269
235		− for (i = 1; i < btf__type_cnt(btf); i++) {
236		− err = btf_dump__dump_type(d, i);
237		− ASSERT_OK(err, "dump_type_ok");
238		− }
239		−
240		− fflush(dump_buf_file);
241		− dump_buf[dump_buf_sz] = 0; /* some libc implementations don't do this */
242		− ASSERT_STREQ(dump_buf,
	270	+ test_ctx__dump_and_compare(&t,
243	271	"struct s___2 {\n"
244	272	" enum x x;\n"
245	273	" enum {\n"
@@ -248,11 +276,53 @@ static void test_btf_dump_incremental(void) · 2 unmodified lines
248	276	" struct s s;\n"
249	277	"};\n\n" , "c_dump1");
250	278
251		−err_out:
252		− fclose(dump_buf_file);
253		− free(dump_buf);
254		− btf_dump__free(d);
255		− btf__free(btf);
	279	+ test_ctx__free(&t);
	280	+}
	281	+
	282	+static void test_btf_dump_type_tags(void)
	283	+{
	284	+ struct test_ctx t = {};
	285	+ struct btf *btf;
	286	+ int id, err;
	287	+
	288	+ if (test_ctx__init(&t))
	289	+ return;
	290	+
	291	+ btf = t.btf;
	292	+
	293	+ /* Generate BTF corresponding to the following C code:
	294	+ *
	295	+ * struct s {
	296	+ * void __attribute__((btf_type_tag(\"void_tag\"))) *p1;
	297	+ * void __attribute__((void_attr)) *p2;
	298	+ * };
	299	+ *
	300	+ */
	301	+
	302	+ id = btf__add_type_tag(btf, "void_tag", 0);
	303	+ ASSERT_EQ(id, 1, "type_tag_id");
	304	+ id = btf__add_ptr(btf, id);
	305	+ ASSERT_EQ(id, 2, "void_ptr_id1");
	306	+
	307	+ id = btf__add_type_attr(btf, "void_attr", 0);
	308	+ ASSERT_EQ(id, 3, "type_attr_id");
	309	+ id = btf__add_ptr(btf, id);
	310	+ ASSERT_EQ(id, 4, "void_ptr_id2");
	311	+
	312	+ id = btf__add_struct(btf, "s", 8);
	313	+ ASSERT_EQ(id, 5, "struct_id");
	314	+ err = btf__add_field(btf, "p1", 2, 0, 0);
	315	+ ASSERT_OK(err, "field_ok1");
	316	+ err = btf__add_field(btf, "p2", 4, 0, 0);
	317	+ ASSERT_OK(err, "field_ok2");
	318	+
	319	+ test_ctx__dump_and_compare(&t,
	320	+"struct s {\n"
	321	+" void __attribute__((btf_type_tag(\"void_tag\"))) *p1;\n"
	322	+" void __attribute__((void_attr)) *p2;\n"
	323	+"};\n\n", "dump_and_compare");
	324	+
	325	+ test_ctx__free(&t);
256	326	}
257	327
258	328	#define STRSIZE 4096
@@ -874,6 +944,9 @@ void test_btf_dump() { · 615 unmodified lines
874	944	if (test__start_subtest("btf_dump: incremental"))
875	945	test_btf_dump_incremental();
876	946
	947	+ if (test__start_subtest("btf_dump: type_tags"))
	948	+ test_btf_dump_type_tags();
	949	+
877	950	btf = libbpf_find_kernel_btf();
878	951	if (!ASSERT_OK_PTR(btf, "no kernel BTF found"))
879	952	return;

tools/testing/selftests/bpf/prog_tests/cgroup_preorder.c+128−0

@@ -0,0 +1,128 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
	3	+#include <test_progs.h>
	4	+#include "cgroup_helpers.h"
	5	+#include "cgroup_preorder.skel.h"
	6	+
	7	+static int run_getsockopt_test(int cg_parent, int cg_child, int sock_fd, bool all_preorder)
	8	+{
	9	+ LIBBPF_OPTS(bpf_prog_attach_opts, opts);
	10	+ enum bpf_attach_type prog_c_atype, prog_c2_atype, prog_p_atype, prog_p2_atype;
	11	+ int prog_c_fd, prog_c2_fd, prog_p_fd, prog_p2_fd;
	12	+ struct cgroup_preorder *skel = NULL;
	13	+ struct bpf_program *prog;
	14	+ __u8 *result, buf;
	15	+ socklen_t optlen;
	16	+ int err = 0;
	17	+
	18	+ skel = cgroup_preorder__open_and_load();
	19	+ if (!ASSERT_OK_PTR(skel, "cgroup_preorder__open_and_load"))
	20	+ return 0;
	21	+
	22	+ buf = 0x00;
	23	+ err = setsockopt(sock_fd, SOL_IP, IP_TOS, &buf, 1);
	24	+ if (!ASSERT_OK(err, "setsockopt"))
	25	+ goto close_skel;
	26	+
	27	+ opts.flags = BPF_F_ALLOW_MULTI;
	28	+ if (all_preorder)
	29	+ opts.flags \|= BPF_F_PREORDER;
	30	+ prog = skel->progs.child;
	31	+ prog_c_fd = bpf_program__fd(prog);
	32	+ prog_c_atype = bpf_program__expected_attach_type(prog);
	33	+ err = bpf_prog_attach_opts(prog_c_fd, cg_child, prog_c_atype, &opts);
	34	+ if (!ASSERT_OK(err, "bpf_prog_attach_opts-child"))
	35	+ goto close_skel;
	36	+
	37	+ opts.flags = BPF_F_ALLOW_MULTI \| BPF_F_PREORDER;
	38	+ prog = skel->progs.child_2;
	39	+ prog_c2_fd = bpf_program__fd(prog);
	40	+ prog_c2_atype = bpf_program__expected_attach_type(prog);
	41	+ err = bpf_prog_attach_opts(prog_c2_fd, cg_child, prog_c2_atype, &opts);
	42	+ if (!ASSERT_OK(err, "bpf_prog_attach_opts-child_2"))
	43	+ goto detach_child;
	44	+
	45	+ optlen = 1;
	46	+ err = getsockopt(sock_fd, SOL_IP, IP_TOS, &buf, &optlen);
	47	+ if (!ASSERT_OK(err, "getsockopt"))
	48	+ goto detach_child_2;
	49	+
	50	+ result = skel->bss->result;
	51	+ if (all_preorder)
	52	+ ASSERT_TRUE(result[0] == 1 && result[1] == 2, "child only");
	53	+ else
	54	+ ASSERT_TRUE(result[0] == 2 && result[1] == 1, "child only");
	55	+
	56	+ skel->bss->idx = 0;
	57	+ memset(result, 0, 4);
	58	+
	59	+ opts.flags = BPF_F_ALLOW_MULTI;
	60	+ if (all_preorder)
	61	+ opts.flags \|= BPF_F_PREORDER;
	62	+ prog = skel->progs.parent;
	63	+ prog_p_fd = bpf_program__fd(prog);
	64	+ prog_p_atype = bpf_program__expected_attach_type(prog);
	65	+ err = bpf_prog_attach_opts(prog_p_fd, cg_parent, prog_p_atype, &opts);
	66	+ if (!ASSERT_OK(err, "bpf_prog_attach_opts-parent"))
	67	+ goto detach_child_2;
	68	+
	69	+ opts.flags = BPF_F_ALLOW_MULTI \| BPF_F_PREORDER;
	70	+ prog = skel->progs.parent_2;
	71	+ prog_p2_fd = bpf_program__fd(prog);
	72	+ prog_p2_atype = bpf_program__expected_attach_type(prog);
	73	+ err = bpf_prog_attach_opts(prog_p2_fd, cg_parent, prog_p2_atype, &opts);
	74	+ if (!ASSERT_OK(err, "bpf_prog_attach_opts-parent_2"))
	75	+ goto detach_parent;
	76	+
	77	+ err = getsockopt(sock_fd, SOL_IP, IP_TOS, &buf, &optlen);
	78	+ if (!ASSERT_OK(err, "getsockopt"))
	79	+ goto detach_parent_2;
	80	+
	81	+ if (all_preorder)
	82	+ ASSERT_TRUE(result[0] == 3 && result[1] == 4 && result[2] == 1 && result[3] == 2,
	83	+ "parent and child");
	84	+ else
	85	+ ASSERT_TRUE(result[0] == 4 && result[1] == 2 && result[2] == 1 && result[3] == 3,
	86	+ "parent and child");
	87	+
	88	+detach_parent_2:
	89	+ ASSERT_OK(bpf_prog_detach2(prog_p2_fd, cg_parent, prog_p2_atype),
	90	+ "bpf_prog_detach2-parent_2");
	91	+detach_parent:
	92	+ ASSERT_OK(bpf_prog_detach2(prog_p_fd, cg_parent, prog_p_atype),
	93	+ "bpf_prog_detach2-parent");
	94	+detach_child_2:
	95	+ ASSERT_OK(bpf_prog_detach2(prog_c2_fd, cg_child, prog_c2_atype),
	96	+ "bpf_prog_detach2-child_2");
	97	+detach_child:
	98	+ ASSERT_OK(bpf_prog_detach2(prog_c_fd, cg_child, prog_c_atype),
	99	+ "bpf_prog_detach2-child");
	100	+close_skel:
	101	+ cgroup_preorder__destroy(skel);
	102	+ return err;
	103	+}
	104	+
	105	+void test_cgroup_preorder(void)
	106	+{
	107	+ int cg_parent = -1, cg_child = -1, sock_fd = -1;
	108	+
	109	+ cg_parent = test__join_cgroup("/parent");
	110	+ if (!ASSERT_GE(cg_parent, 0, "join_cgroup /parent"))
	111	+ goto out;
	112	+
	113	+ cg_child = test__join_cgroup("/parent/child");
	114	+ if (!ASSERT_GE(cg_child, 0, "join_cgroup /parent/child"))
	115	+ goto out;
	116	+
	117	+ sock_fd = socket(AF_INET, SOCK_STREAM, 0);
	118	+ if (!ASSERT_GE(sock_fd, 0, "socket"))
	119	+ goto out;
	120	+
	121	+ ASSERT_OK(run_getsockopt_test(cg_parent, cg_child, sock_fd, false), "getsockopt_test_1");
	122	+ ASSERT_OK(run_getsockopt_test(cg_parent, cg_child, sock_fd, true), "getsockopt_test_2");
	123	+
	124	+out:
	125	+ close(sock_fd);
	126	+ close(cg_child);
	127	+ close(cg_parent);
	128	+}

tools/testing/selftests/bpf/prog_tests/cgroup_v1v2.c+9−4

@@ -10,12 +10,18 @@ · 9 unmodified lines
10	10	static int run_test(int cgroup_fd, int server_fd, bool classid)
11	11	{
12	12	struct connect4_dropper *skel;
13		− int fd, err = 0;
	13	+ int fd, err = 0, port;
14	14
15	15	skel = connect4_dropper__open_and_load();
16	16	if (!ASSERT_OK_PTR(skel, "skel_open"))
17	17	return -1;
18	18
	19	+ port = get_socket_local_port(server_fd);
	20	+ if (!ASSERT_GE(port, 0, "get_socket_local_port"))
	21	+ return -1;
	22	+
	23	+ skel->bss->port = ntohs(port);
	24	+
19	25	skel->links.connect_v4_dropper =
20	26	bpf_program__attach_cgroup(skel->progs.connect_v4_dropper,
21	27	cgroup_fd);
@@ -48,10 +54,9 @@ void test_cgroup_v1v2(void) · 26 unmodified lines
48	54	{
49	55	struct network_helper_opts opts = {};
50	56	int server_fd, client_fd, cgroup_fd;
51		− static const int port = 60120;
52	57
53	58	/* Step 1: Check base connectivity works without any BPF. */
54		− server_fd = start_server(AF_INET, SOCK_STREAM, NULL, port, 0);
	59	+ server_fd = start_server(AF_INET, SOCK_STREAM, NULL, 0, 0);
55	60	if (!ASSERT_GE(server_fd, 0, "server_fd"))
56	61	return;
57	62	client_fd = connect_to_fd_opts(server_fd, &opts);
@@ -66,7 +71,7 @@ void test_cgroup_v1v2(void) · 8 unmodified lines
66	71	cgroup_fd = test__join_cgroup("/connect_dropper");
67	72	if (!ASSERT_GE(cgroup_fd, 0, "cgroup_fd"))
68	73	return;
69		− server_fd = start_server(AF_INET, SOCK_STREAM, NULL, port, 0);
	74	+ server_fd = start_server(AF_INET, SOCK_STREAM, NULL, 0, 0);
70	75	if (!ASSERT_GE(server_fd, 0, "server_fd")) {
71	76	close(cgroup_fd);
72	77	return;

tools/testing/selftests/bpf/prog_tests/changes_pkt_data.c+0−107

@@ -1,107 +0,0 @@
1		−// SPDX-License-Identifier: GPL-2.0
2		−#include "bpf/libbpf.h"
3		−#include "changes_pkt_data_freplace.skel.h"
4		−#include "changes_pkt_data.skel.h"
5		−#include <test_progs.h>
6		−
7		−static void print_verifier_log(const char *log)
8		−{
9		− if (env.verbosity >= VERBOSE_VERY)
10		− fprintf(stdout, "VERIFIER LOG:\n=============\n%s=============\n", log);
11		−}
12		−
13		−static void test_aux(const char *main_prog_name,
14		− const char *to_be_replaced,
15		− const char *replacement,
16		− bool expect_load)
17		−{
18		− struct changes_pkt_data_freplace *freplace = NULL;
19		− struct bpf_program *freplace_prog = NULL;
20		− struct bpf_program *main_prog = NULL;
21		− LIBBPF_OPTS(bpf_object_open_opts, opts);
22		− struct changes_pkt_data *main = NULL;
23		− char log[16*1024];
24		− int err;
25		−
26		− opts.kernel_log_buf = log;
27		− opts.kernel_log_size = sizeof(log);
28		− if (env.verbosity >= VERBOSE_SUPER)
29		− opts.kernel_log_level = 1 \| 2 \| 4;
30		− main = changes_pkt_data__open_opts(&opts);
31		− if (!ASSERT_OK_PTR(main, "changes_pkt_data__open"))
32		− goto out;
33		− main_prog = bpf_object__find_program_by_name(main->obj, main_prog_name);
34		− if (!ASSERT_OK_PTR(main_prog, "main_prog"))
35		− goto out;
36		− bpf_program__set_autoload(main_prog, true);
37		− err = changes_pkt_data__load(main);
38		− print_verifier_log(log);
39		− if (!ASSERT_OK(err, "changes_pkt_data__load"))
40		− goto out;
41		− freplace = changes_pkt_data_freplace__open_opts(&opts);
42		− if (!ASSERT_OK_PTR(freplace, "changes_pkt_data_freplace__open"))
43		− goto out;
44		− freplace_prog = bpf_object__find_program_by_name(freplace->obj, replacement);
45		− if (!ASSERT_OK_PTR(freplace_prog, "freplace_prog"))
46		− goto out;
47		− bpf_program__set_autoload(freplace_prog, true);
48		− bpf_program__set_autoattach(freplace_prog, true);
49		− bpf_program__set_attach_target(freplace_prog,
50		− bpf_program__fd(main_prog),
51		− to_be_replaced);
52		− err = changes_pkt_data_freplace__load(freplace);
53		− print_verifier_log(log);
54		− if (expect_load) {
55		− ASSERT_OK(err, "changes_pkt_data_freplace__load");
56		− } else {
57		− ASSERT_ERR(err, "changes_pkt_data_freplace__load");
58		− ASSERT_HAS_SUBSTR(log, "Extension program changes packet data", "error log");
59		− }
60		−
61		−out:
62		− changes_pkt_data_freplace__destroy(freplace);
63		− changes_pkt_data__destroy(main);
64		−}
65		−
66		−/* There are two global subprograms in both changes_pkt_data.skel.h:
67		− * - one changes packet data;
68		− * - another does not.
69		− * It is ok to freplace subprograms that change packet data with those
70		− * that either do or do not. It is only ok to freplace subprograms
71		− * that do not change packet data with those that do not as well.
72		− * The below tests check outcomes for each combination of such freplace.
73		− * Also test a case when main subprogram itself is replaced and is a single
74		− * subprogram in a program.
75		− */
76		−void test_changes_pkt_data_freplace(void)
77		−{
78		− struct {
79		− const char *main;
80		− const char *to_be_replaced;
81		− bool changes;
82		− } mains[] = {
83		− { "main_with_subprogs", "changes_pkt_data", true },
84		− { "main_with_subprogs", "does_not_change_pkt_data", false },
85		− { "main_changes", "main_changes", true },
86		− { "main_does_not_change", "main_does_not_change", false },
87		− };
88		− struct {
89		− const char *func;
90		− bool changes;
91		− } replacements[] = {
92		− { "changes_pkt_data", true },
93		− { "does_not_change_pkt_data", false }
94		− };
95		− char buf[64];
96		−
97		− for (int i = 0; i < ARRAY_SIZE(mains); ++i) {
98		− for (int j = 0; j < ARRAY_SIZE(replacements); ++j) {
99		− snprintf(buf, sizeof(buf), "%s_with_%s",
100		− mains[i].to_be_replaced, replacements[j].func);
101		− if (!test__start_subtest(buf))
102		− continue;
103		− test_aux(mains[i].main, mains[i].to_be_replaced, replacements[j].func,
104		− mains[i].changes \|\| !replacements[j].changes);
105		− }
106		− }
107		−}

tools/testing/selftests/bpf/prog_tests/compute_live_registers.c+9−0

@@ -0,0 +1,9 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+
	3	+#include "compute_live_registers.skel.h"
	4	+#include "test_progs.h"
	5	+
	6	+void test_compute_live_registers(void)
	7	+{
	8	+ RUN_TESTS(compute_live_registers);
	9	+}

tools/testing/selftests/bpf/prog_tests/core_reloc.c+4−2

@@ -85,11 +85,11 @@ static int duration = 0; · 84 unmodified lines
85	85	#define NESTING_ERR_CASE(name) { \
86	86	NESTING_CASE_COMMON(name), \
87	87	.fails = true, \
88		− .run_btfgen_fails = true, \
	88	+ .run_btfgen_fails = true, \
89	89	}
90	90
91	91	#define ARRAYS_DATA(struct_name) STRUCT_TO_CHAR_PTR(struct_name) { \
92		− .a = { [2] = 1 }, \
	92	+ .a = { [2] = 1, [3] = 11 }, \
93	93	.b = { [1] = { [2] = { [3] = 2 } } }, \
94	94	.c = { [1] = { .c = 3 } }, \
95	95	.d = { [0] = { [0] = { .d = 4 } } }, \
@@ -108,6 +108,7 @@ static int duration = 0; · 12 unmodified lines
108	108	.input_len = sizeof(struct core_reloc_##name), \
109	109	.output = STRUCT_TO_CHAR_PTR(core_reloc_arrays_output) { \
110	110	.a2 = 1, \
	111	+ .a3 = 12, \
111	112	.b123 = 2, \
112	113	.c1c = 3, \
113	114	.d00d = 4, \
@@ -602,6 +603,7 @@ static const struct core_reloc_test_case test_cases[] = { · 488 unmodified lines
602	603	ARRAYS_ERR_CASE(arrays___err_non_array),
603	604	ARRAYS_ERR_CASE(arrays___err_wrong_val_type),
604	605	ARRAYS_ERR_CASE(arrays___err_bad_zero_sz_arr),
	606	+ ARRAYS_ERR_CASE(arrays___err_bad_signed_arr_elem_sz),
605	607
606	608	/* enum/ptr/int handling scenarios */
607	609	PRIMITIVES_CASE(primitives),

tools/testing/selftests/bpf/prog_tests/cpumask.c+4−1

@@ -25,6 +25,10 @@ static const char * const cpumask_success_testcases[] = { · 24 unmodified lines
25	25	"test_global_mask_nested_deep_rcu",
26	26	"test_global_mask_nested_deep_array_rcu",
27	27	"test_cpumask_weight",
	28	+ "test_refcount_null_tracking",
	29	+ "test_populate_reject_small_mask",
	30	+ "test_populate_reject_unaligned",
	31	+ "test_populate",
28	32	};
29	33
30	34	static void verify_success(const char *prog_name)
@@ -78,6 +82,5 @@ void test_cpumask(void) · 47 unmodified lines
78	82	verify_success(cpumask_success_testcases[i]);
79	83	}
80	84
81		− RUN_TESTS(cpumask_success);
82	85	RUN_TESTS(cpumask_failure);
83	86	}

tools/testing/selftests/bpf/prog_tests/dynptr.c+21−0

@@ -10,6 +10,7 @@ enum test_setup_type { · 9 unmodified lines
10	10	SETUP_SYSCALL_SLEEP,
11	11	SETUP_SKB_PROG,
12	12	SETUP_SKB_PROG_TP,
	13	+ SETUP_XDP_PROG,
13	14	};
14	15
15	16	static struct {
@@ -18,6 +19,8 @@ static struct { · 2 unmodified lines
18	19	} success_tests[] = {
19	20	{"test_read_write", SETUP_SYSCALL_SLEEP},
20	21	{"test_dynptr_data", SETUP_SYSCALL_SLEEP},
	22	+ {"test_dynptr_copy", SETUP_SYSCALL_SLEEP},
	23	+ {"test_dynptr_copy_xdp", SETUP_XDP_PROG},
21	24	{"test_ringbuf", SETUP_SYSCALL_SLEEP},
22	25	{"test_skb_readonly", SETUP_SKB_PROG},
23	26	{"test_dynptr_skb_data", SETUP_SKB_PROG},
@@ -120,6 +123,24 @@ static void verify_success(const char *prog_name, enum test_setup_type setup_typ · 96 unmodified lines
120	123
121	124	break;
122	125	}
	126	+ case SETUP_XDP_PROG:
	127	+ {
	128	+ char data[5000];
	129	+ int err, prog_fd;
	130	+ LIBBPF_OPTS(bpf_test_run_opts, opts,
	131	+ .data_in = &data,
	132	+ .data_size_in = sizeof(data),
	133	+ .repeat = 1,
	134	+ );
	135	+
	136	+ prog_fd = bpf_program__fd(prog);
	137	+ err = bpf_prog_test_run_opts(prog_fd, &opts);
	138	+
	139	+ if (!ASSERT_OK(err, "test_run"))
	140	+ goto cleanup;
	141	+
	142	+ break;
	143	+ }
123	144	}
124	145
125	146	ASSERT_EQ(skel->bss->err, 0, "err");

tools/testing/selftests/bpf/prog_tests/fd_array.c+2−2

@@ -83,8 +83,8 @@ static inline int bpf_prog_get_map_ids(int prog_fd, __u32 nr_map_ids, __u32 ma · 82 unmodified lines
83	83	int err;
84	84
85	85	memset(&info, 0, len);
86		− info.nr_map_ids = *nr_map_ids,
87		− info.map_ids = ptr_to_u64(map_ids),
	86	+ info.nr_map_ids = *nr_map_ids;
	87	+ info.map_ids = ptr_to_u64(map_ids);
88	88
89	89	err = bpf_prog_get_info_by_fd(prog_fd, &info, &len);
90	90	if (!ASSERT_OK(err, "bpf_prog_get_info_by_fd"))

tools/testing/selftests/bpf/prog_tests/fexit_noreturns.c+9−0

@@ -0,0 +1,9 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+
	3	+#include <test_progs.h>
	4	+#include "fexit_noreturns.skel.h"
	5	+
	6	+void test_fexit_noreturns(void)
	7	+{
	8	+ RUN_TESTS(fexit_noreturns);
	9	+}

tools/testing/selftests/bpf/prog_tests/fs_kfuncs.c+152−10

@@ -8,11 +8,12 @@ · 7 unmodified lines
8	8	#include <unistd.h>
9	9	#include <test_progs.h>
10	10	#include "test_get_xattr.skel.h"
	11	+#include "test_set_remove_xattr.skel.h"
11	12	#include "test_fsverity.skel.h"
12	13
13	14	static const char testfile[] = "/tmp/test_progs_fs_kfuncs";
14	15
15		−static void test_xattr(void)
	16	+static void test_get_xattr(const char name, const char value, bool allow_access)
16	17	{
17	18	struct test_get_xattr *skel = NULL;
18	19	int fd = -1, err;
@@ -25,7 +26,7 @@ static void test_xattr(void) · 6 unmodified lines
25	26	close(fd);
26	27	fd = -1;
27	28
28		− err = setxattr(testfile, "user.kfuncs", "hello", sizeof("hello"), 0);
	29	+ err = setxattr(testfile, name, value, strlen(value) + 1, 0);
29	30	if (err && errno == EOPNOTSUPP) {
30	31	printf("%s:SKIP:local fs doesn't support xattr (%d)\n"
31	32	"To run this test, make sure /tmp filesystem supports xattr.\n",
@@ -48,16 +49,23 @@ static void test_xattr(void) · 16 unmodified lines
48	49	goto out;
49	50
50	51	fd = open(testfile, O_RDONLY, 0644);
	52	+
51	53	if (!ASSERT_GE(fd, 0, "open_file"))
52	54	goto out;
53	55
54		− ASSERT_EQ(skel->bss->found_xattr_from_file, 1, "found_xattr_from_file");
55		−
56	56	/* Trigger security_inode_getxattr */
57		− err = getxattr(testfile, "user.kfuncs", v, sizeof(v));
58		− ASSERT_EQ(err, -1, "getxattr_return");
59		− ASSERT_EQ(errno, EINVAL, "getxattr_errno");
60		− ASSERT_EQ(skel->bss->found_xattr_from_dentry, 1, "found_xattr_from_dentry");
	57	+ err = getxattr(testfile, name, v, sizeof(v));
	58	+
	59	+ if (allow_access) {
	60	+ ASSERT_EQ(err, -1, "getxattr_return");
	61	+ ASSERT_EQ(errno, EINVAL, "getxattr_errno");
	62	+ ASSERT_EQ(skel->bss->found_xattr_from_file, 1, "found_xattr_from_file");
	63	+ ASSERT_EQ(skel->bss->found_xattr_from_dentry, 1, "found_xattr_from_dentry");
	64	+ } else {
	65	+ ASSERT_EQ(err, strlen(value) + 1, "getxattr_return");
	66	+ ASSERT_EQ(skel->bss->found_xattr_from_file, 0, "found_xattr_from_file");
	67	+ ASSERT_EQ(skel->bss->found_xattr_from_dentry, 0, "found_xattr_from_dentry");
	68	+ }
61	69
62	70	out:
63	71	close(fd);
@@ -65,6 +73,127 @@ out: · 1 unmodified line
65	73	remove(testfile);
66	74	}
67	75
	76	+/* xattr value we will set to security.bpf.foo */
	77	+static const char value_foo[] = "hello";
	78	+
	79	+static void read_and_validate_foo(struct test_set_remove_xattr *skel)
	80	+{
	81	+ char value_out[32];
	82	+ int err;
	83	+
	84	+ err = getxattr(testfile, skel->rodata->xattr_foo, value_out, sizeof(value_out));
	85	+ ASSERT_EQ(err, sizeof(value_foo), "getxattr size foo");
	86	+ ASSERT_EQ(strncmp(value_out, value_foo, sizeof(value_foo)), 0, "strncmp value_foo");
	87	+}
	88	+
	89	+static void set_foo(struct test_set_remove_xattr *skel)
	90	+{
	91	+ ASSERT_OK(setxattr(testfile, skel->rodata->xattr_foo, value_foo, strlen(value_foo) + 1, 0),
	92	+ "setxattr foo");
	93	+}
	94	+
	95	+static void validate_bar_match(struct test_set_remove_xattr *skel)
	96	+{
	97	+ char value_out[32];
	98	+ int err;
	99	+
	100	+ err = getxattr(testfile, skel->rodata->xattr_bar, value_out, sizeof(value_out));
	101	+ ASSERT_EQ(err, sizeof(skel->data->value_bar), "getxattr size bar");
	102	+ ASSERT_EQ(strncmp(value_out, skel->data->value_bar, sizeof(skel->data->value_bar)), 0,
	103	+ "strncmp value_bar");
	104	+}
	105	+
	106	+static void validate_bar_removed(struct test_set_remove_xattr *skel)
	107	+{
	108	+ char value_out[32];
	109	+ int err;
	110	+
	111	+ err = getxattr(testfile, skel->rodata->xattr_bar, value_out, sizeof(value_out));
	112	+ ASSERT_LT(err, 0, "getxattr size bar should fail");
	113	+}
	114	+
	115	+static void test_set_remove_xattr(void)
	116	+{
	117	+ struct test_set_remove_xattr *skel = NULL;
	118	+ int fd = -1, err;
	119	+
	120	+ fd = open(testfile, O_CREAT \| O_RDONLY, 0644);
	121	+ if (!ASSERT_GE(fd, 0, "create_file"))
	122	+ return;
	123	+
	124	+ close(fd);
	125	+ fd = -1;
	126	+
	127	+ skel = test_set_remove_xattr__open_and_load();
	128	+ if (!ASSERT_OK_PTR(skel, "test_set_remove_xattr__open_and_load"))
	129	+ return;
	130	+
	131	+ /* Set security.bpf.foo to "hello" */
	132	+ err = setxattr(testfile, skel->rodata->xattr_foo, value_foo, strlen(value_foo) + 1, 0);
	133	+ if (err && errno == EOPNOTSUPP) {
	134	+ printf("%s:SKIP:local fs doesn't support xattr (%d)\n"
	135	+ "To run this test, make sure /tmp filesystem supports xattr.\n",
	136	+ __func__, errno);
	137	+ test__skip();
	138	+ goto out;
	139	+ }
	140	+
	141	+ if (!ASSERT_OK(err, "setxattr"))
	142	+ goto out;
	143	+
	144	+ skel->bss->monitored_pid = getpid();
	145	+ err = test_set_remove_xattr__attach(skel);
	146	+ if (!ASSERT_OK(err, "test_set_remove_xattr__attach"))
	147	+ goto out;
	148	+
	149	+ /* First, test not _locked version of the kfuncs with getxattr. */
	150	+
	151	+ /* Read security.bpf.foo and trigger test_inode_getxattr. This
	152	+ * bpf program will set security.bpf.bar to "world".
	153	+ */
	154	+ read_and_validate_foo(skel);
	155	+ validate_bar_match(skel);
	156	+
	157	+ /* Read security.bpf.foo and trigger test_inode_getxattr again.
	158	+ * This will remove xattr security.bpf.bar.
	159	+ */
	160	+ read_and_validate_foo(skel);
	161	+ validate_bar_removed(skel);
	162	+
	163	+ ASSERT_TRUE(skel->bss->set_security_bpf_bar_success, "set_security_bpf_bar_success");
	164	+ ASSERT_TRUE(skel->bss->remove_security_bpf_bar_success, "remove_security_bpf_bar_success");
	165	+ ASSERT_TRUE(skel->bss->set_security_selinux_fail, "set_security_selinux_fail");
	166	+ ASSERT_TRUE(skel->bss->remove_security_selinux_fail, "remove_security_selinux_fail");
	167	+
	168	+ /* Second, test _locked version of the kfuncs, with setxattr */
	169	+
	170	+ /* Set security.bpf.foo and trigger test_inode_setxattr. This
	171	+ * bpf program will set security.bpf.bar to "world".
	172	+ */
	173	+ set_foo(skel);
	174	+ validate_bar_match(skel);
	175	+
	176	+ /* Set security.bpf.foo and trigger test_inode_setxattr again.
	177	+ * This will remove xattr security.bpf.bar.
	178	+ */
	179	+ set_foo(skel);
	180	+ validate_bar_removed(skel);
	181	+
	182	+ ASSERT_TRUE(skel->bss->locked_set_security_bpf_bar_success,
	183	+ "locked_set_security_bpf_bar_success");
	184	+ ASSERT_TRUE(skel->bss->locked_remove_security_bpf_bar_success,
	185	+ "locked_remove_security_bpf_bar_success");
	186	+ ASSERT_TRUE(skel->bss->locked_set_security_selinux_fail,
	187	+ "locked_set_security_selinux_fail");
	188	+ ASSERT_TRUE(skel->bss->locked_remove_security_selinux_fail,
	189	+ "locked_remove_security_selinux_fail");
	190	+
	191	+out:
	192	+ close(fd);
	193	+ test_set_remove_xattr__destroy(skel);
	194	+ remove(testfile);
	195	+}
	196	+
68	197	#ifndef SHA256_DIGEST_SIZE
69	198	#define SHA256_DIGEST_SIZE 32
70	199	#endif
@@ -141,8 +270,21 @@ out: · 70 unmodified lines
141	270
142	271	void test_fs_kfuncs(void)
143	272	{
144		− if (test__start_subtest("xattr"))
145		− test_xattr();
	273	+ /* Matches xattr_names in progs/test_get_xattr.c */
	274	+ if (test__start_subtest("user_xattr"))
	275	+ test_get_xattr("user.kfuncs", "hello", true);
	276	+
	277	+ if (test__start_subtest("security_bpf_xattr"))
	278	+ test_get_xattr("security.bpf.xxx", "hello", true);
	279	+
	280	+ if (test__start_subtest("security_bpf_xattr_error"))
	281	+ test_get_xattr("security.bpf", "hello", false);
	282	+
	283	+ if (test__start_subtest("security_selinux_xattr_error"))
	284	+ test_get_xattr("security.selinux", "hello", false);
	285	+
	286	+ if (test__start_subtest("set_remove_xattr"))
	287	+ test_set_remove_xattr();
146	288
147	289	if (test__start_subtest("fsverity"))
148	290	test_fsverity();

tools/testing/selftests/bpf/prog_tests/kernel_flag.c+43−0

@@ -0,0 +1,43 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2025 Microsoft */
	3	+#include <test_progs.h>
	4	+#include "kfunc_call_test.skel.h"
	5	+#include "kfunc_call_test.lskel.h"
	6	+#include "test_kernel_flag.skel.h"
	7	+
	8	+void test_kernel_flag(void)
	9	+{
	10	+ struct test_kernel_flag *lsm_skel;
	11	+ struct kfunc_call_test *skel = NULL;
	12	+ struct kfunc_call_test_lskel *lskel = NULL;
	13	+ int ret;
	14	+
	15	+ lsm_skel = test_kernel_flag__open_and_load();
	16	+ if (!ASSERT_OK_PTR(lsm_skel, "lsm_skel"))
	17	+ return;
	18	+
	19	+ lsm_skel->bss->monitored_tid = gettid();
	20	+
	21	+ ret = test_kernel_flag__attach(lsm_skel);
	22	+ if (!ASSERT_OK(ret, "test_kernel_flag__attach"))
	23	+ goto close_prog;
	24	+
	25	+ /* Test with skel. This should pass the gatekeeper */
	26	+ skel = kfunc_call_test__open_and_load();
	27	+ if (!ASSERT_OK_PTR(skel, "skel"))
	28	+ goto close_prog;
	29	+
	30	+ /* Test with lskel. This should fail due to blocking kernel-based bpf() invocations */
	31	+ lskel = kfunc_call_test_lskel__open_and_load();
	32	+ if (!ASSERT_ERR_PTR(lskel, "lskel"))
	33	+ goto close_prog;
	34	+
	35	+close_prog:
	36	+ if (skel)
	37	+ kfunc_call_test__destroy(skel);
	38	+ if (lskel)
	39	+ kfunc_call_test_lskel__destroy(lskel);
	40	+
	41	+ lsm_skel->bss->monitored_tid = 0;
	42	+ test_kernel_flag__destroy(lsm_skel);
	43	+}

tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c+540−0

@@ -0,0 +1,540 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
	2	+#include <netinet/in.h>
	3	+
	4	+#include "network_helpers.h"
	5	+#include "test_progs.h"
	6	+
	7	+#define BPF_FILE "test_lwt_ip_encap.bpf.o"
	8	+
	9	+#define NETNS_NAME_SIZE 32
	10	+#define NETNS_BASE "ns-lwt-ip-encap"
	11	+
	12	+#define IP4_ADDR_1 "172.16.1.100"
	13	+#define IP4_ADDR_2 "172.16.2.100"
	14	+#define IP4_ADDR_3 "172.16.3.100"
	15	+#define IP4_ADDR_4 "172.16.4.100"
	16	+#define IP4_ADDR_5 "172.16.5.100"
	17	+#define IP4_ADDR_6 "172.16.6.100"
	18	+#define IP4_ADDR_7 "172.16.7.100"
	19	+#define IP4_ADDR_8 "172.16.8.100"
	20	+#define IP4_ADDR_GRE "172.16.16.100"
	21	+
	22	+#define IP4_ADDR_SRC IP4_ADDR_1
	23	+#define IP4_ADDR_DST IP4_ADDR_4
	24	+
	25	+#define IP6_ADDR_1 "fb01::1"
	26	+#define IP6_ADDR_2 "fb02::1"
	27	+#define IP6_ADDR_3 "fb03::1"
	28	+#define IP6_ADDR_4 "fb04::1"
	29	+#define IP6_ADDR_5 "fb05::1"
	30	+#define IP6_ADDR_6 "fb06::1"
	31	+#define IP6_ADDR_7 "fb07::1"
	32	+#define IP6_ADDR_8 "fb08::1"
	33	+#define IP6_ADDR_GRE "fb10::1"
	34	+
	35	+#define IP6_ADDR_SRC IP6_ADDR_1
	36	+#define IP6_ADDR_DST IP6_ADDR_4
	37	+
	38	+/* Setup/topology:
	39	+ *
	40	+ * NS1 NS2 NS3
	41	+ * veth1 <---> veth2 veth3 <---> veth4 (the top route)
	42	+ * veth5 <---> veth6 veth7 <---> veth8 (the bottom route)
	43	+ *
	44	+ * Each vethN gets IP[4\|6]_ADDR_N address.
	45	+ *
	46	+ * IP_ADDR_SRC = IP_ADDR_1
	47	+ * IP_ADDR_DST = IP_ADDR_4
	48	+ *
	49	+ * All tests test pings from IP_ADDR__SRC to IP_ADDR_DST.
	50	+ *
	51	+ * By default, routes are configured to allow packets to go
	52	+ * IP_ADDR_1 <=> IP_ADDR_2 <=> IP_ADDR_3 <=> IP_ADDR_4 (the top route).
	53	+ *
	54	+ * A GRE device is installed in NS3 with IP*_ADDR_GRE, and
	55	+ * NS1/NS2 are configured to route packets to IP_ADDR_GRE via IP_ADDR_8
	56	+ * (the bottom route).
	57	+ *
	58	+ * Tests:
	59	+ *
	60	+ * 1. Routes NS2->IP*_ADDR_DST are brought down, so the only way a ping
	61	+ * from IP_ADDR_SRC to IP_ADDR_DST can work is via IP*_ADDR_GRE.
	62	+ *
	63	+ * 2a. In an egress test, a bpf LWT_XMIT program is installed on veth1
	64	+ * that encaps the packets with an IP/GRE header to route to IP*_ADDR_GRE.
	65	+ *
	66	+ * ping: SRC->[encap at veth1:egress]->GRE:decap->DST
	67	+ * ping replies go DST->SRC directly
	68	+ *
	69	+ * 2b. In an ingress test, a bpf LWT_IN program is installed on veth2
	70	+ * that encaps the packets with an IP/GRE header to route to IP*_ADDR_GRE.
	71	+ *
	72	+ * ping: SRC->[encap at veth2:ingress]->GRE:decap->DST
	73	+ * ping replies go DST->SRC directly
	74	+ */
	75	+
	76	+static int create_ns(char *name, size_t name_sz)
	77	+{
	78	+ if (!name)
	79	+ goto fail;
	80	+
	81	+ if (!ASSERT_OK(append_tid(name, name_sz), "append TID"))
	82	+ goto fail;
	83	+
	84	+ SYS(fail, "ip netns add %s", name);
	85	+
	86	+ /* rp_filter gets confused by what these tests are doing, so disable it */
	87	+ SYS(fail, "ip netns exec %s sysctl -wq net.ipv4.conf.all.rp_filter=0", name);
	88	+ SYS(fail, "ip netns exec %s sysctl -wq net.ipv4.conf.default.rp_filter=0", name);
	89	+ /* Disable IPv6 DAD because it sometimes takes too long and fails tests */
	90	+ SYS(fail, "ip netns exec %s sysctl -wq net.ipv6.conf.all.accept_dad=0", name);
	91	+ SYS(fail, "ip netns exec %s sysctl -wq net.ipv6.conf.default.accept_dad=0", name);
	92	+
	93	+ return 0;
	94	+fail:
	95	+ return -1;
	96	+}
	97	+
	98	+static int set_top_addr(const char ns1, const char ns2, const char *ns3)
	99	+{
	100	+ SYS(fail, "ip -n %s a add %s/24 dev veth1", ns1, IP4_ADDR_1);
	101	+ SYS(fail, "ip -n %s a add %s/24 dev veth2", ns2, IP4_ADDR_2);
	102	+ SYS(fail, "ip -n %s a add %s/24 dev veth3", ns2, IP4_ADDR_3);
	103	+ SYS(fail, "ip -n %s a add %s/24 dev veth4", ns3, IP4_ADDR_4);
	104	+ SYS(fail, "ip -n %s -6 a add %s/128 dev veth1", ns1, IP6_ADDR_1);
	105	+ SYS(fail, "ip -n %s -6 a add %s/128 dev veth2", ns2, IP6_ADDR_2);
	106	+ SYS(fail, "ip -n %s -6 a add %s/128 dev veth3", ns2, IP6_ADDR_3);
	107	+ SYS(fail, "ip -n %s -6 a add %s/128 dev veth4", ns3, IP6_ADDR_4);
	108	+
	109	+ SYS(fail, "ip -n %s link set dev veth1 up", ns1);
	110	+ SYS(fail, "ip -n %s link set dev veth2 up", ns2);
	111	+ SYS(fail, "ip -n %s link set dev veth3 up", ns2);
	112	+ SYS(fail, "ip -n %s link set dev veth4 up", ns3);
	113	+
	114	+ return 0;
	115	+fail:
	116	+ return 1;
	117	+}
	118	+
	119	+static int set_bottom_addr(const char ns1, const char ns2, const char *ns3)
	120	+{
	121	+ SYS(fail, "ip -n %s a add %s/24 dev veth5", ns1, IP4_ADDR_5);
	122	+ SYS(fail, "ip -n %s a add %s/24 dev veth6", ns2, IP4_ADDR_6);
	123	+ SYS(fail, "ip -n %s a add %s/24 dev veth7", ns2, IP4_ADDR_7);
	124	+ SYS(fail, "ip -n %s a add %s/24 dev veth8", ns3, IP4_ADDR_8);
	125	+ SYS(fail, "ip -n %s -6 a add %s/128 dev veth5", ns1, IP6_ADDR_5);
	126	+ SYS(fail, "ip -n %s -6 a add %s/128 dev veth6", ns2, IP6_ADDR_6);
	127	+ SYS(fail, "ip -n %s -6 a add %s/128 dev veth7", ns2, IP6_ADDR_7);
	128	+ SYS(fail, "ip -n %s -6 a add %s/128 dev veth8", ns3, IP6_ADDR_8);
	129	+
	130	+ SYS(fail, "ip -n %s link set dev veth5 up", ns1);
	131	+ SYS(fail, "ip -n %s link set dev veth6 up", ns2);
	132	+ SYS(fail, "ip -n %s link set dev veth7 up", ns2);
	133	+ SYS(fail, "ip -n %s link set dev veth8 up", ns3);
	134	+
	135	+ return 0;
	136	+fail:
	137	+ return 1;
	138	+}
	139	+
	140	+static int configure_vrf(const char ns1, const char ns2)
	141	+{
	142	+ if (!ns1 \|\| !ns2)
	143	+ goto fail;
	144	+
	145	+ SYS(fail, "ip -n %s link add red type vrf table 1001", ns1);
	146	+ SYS(fail, "ip -n %s link set red up", ns1);
	147	+ SYS(fail, "ip -n %s route add table 1001 unreachable default metric 8192", ns1);
	148	+ SYS(fail, "ip -n %s -6 route add table 1001 unreachable default metric 8192", ns1);
	149	+ SYS(fail, "ip -n %s link set veth1 vrf red", ns1);
	150	+ SYS(fail, "ip -n %s link set veth5 vrf red", ns1);
	151	+
	152	+ SYS(fail, "ip -n %s link add red type vrf table 1001", ns2);
	153	+ SYS(fail, "ip -n %s link set red up", ns2);
	154	+ SYS(fail, "ip -n %s route add table 1001 unreachable default metric 8192", ns2);
	155	+ SYS(fail, "ip -n %s -6 route add table 1001 unreachable default metric 8192", ns2);
	156	+ SYS(fail, "ip -n %s link set veth2 vrf red", ns2);
	157	+ SYS(fail, "ip -n %s link set veth3 vrf red", ns2);
	158	+ SYS(fail, "ip -n %s link set veth6 vrf red", ns2);
	159	+ SYS(fail, "ip -n %s link set veth7 vrf red", ns2);
	160	+
	161	+ return 0;
	162	+fail:
	163	+ return -1;
	164	+}
	165	+
	166	+static int configure_ns1(const char ns1, const char vrf)
	167	+{
	168	+ struct nstoken *nstoken = NULL;
	169	+
	170	+ if (!ns1 \|\| !vrf)
	171	+ goto fail;
	172	+
	173	+ nstoken = open_netns(ns1);
	174	+ if (!ASSERT_OK_PTR(nstoken, "open ns1"))
	175	+ goto fail;
	176	+
	177	+ /* Top route */
	178	+ SYS(fail, "ip route add %s/32 dev veth1 %s", IP4_ADDR_2, vrf);
	179	+ SYS(fail, "ip route add default dev veth1 via %s %s", IP4_ADDR_2, vrf);
	180	+ SYS(fail, "ip -6 route add %s/128 dev veth1 %s", IP6_ADDR_2, vrf);
	181	+ SYS(fail, "ip -6 route add default dev veth1 via %s %s", IP6_ADDR_2, vrf);
	182	+ /* Bottom route */
	183	+ SYS(fail, "ip route add %s/32 dev veth5 %s", IP4_ADDR_6, vrf);
	184	+ SYS(fail, "ip route add %s/32 dev veth5 via %s %s", IP4_ADDR_7, IP4_ADDR_6, vrf);
	185	+ SYS(fail, "ip route add %s/32 dev veth5 via %s %s", IP4_ADDR_8, IP4_ADDR_6, vrf);
	186	+ SYS(fail, "ip -6 route add %s/128 dev veth5 %s", IP6_ADDR_6, vrf);
	187	+ SYS(fail, "ip -6 route add %s/128 dev veth5 via %s %s", IP6_ADDR_7, IP6_ADDR_6, vrf);
	188	+ SYS(fail, "ip -6 route add %s/128 dev veth5 via %s %s", IP6_ADDR_8, IP6_ADDR_6, vrf);
	189	+
	190	+ close_netns(nstoken);
	191	+ return 0;
	192	+fail:
	193	+ close_netns(nstoken);
	194	+ return -1;
	195	+}
	196	+
	197	+static int configure_ns2(const char ns2, const char vrf)
	198	+{
	199	+ struct nstoken *nstoken = NULL;
	200	+
	201	+ if (!ns2 \|\| !vrf)
	202	+ goto fail;
	203	+
	204	+ nstoken = open_netns(ns2);
	205	+ if (!ASSERT_OK_PTR(nstoken, "open ns2"))
	206	+ goto fail;
	207	+
	208	+ SYS(fail, "ip netns exec %s sysctl -wq net.ipv4.ip_forward=1", ns2);
	209	+ SYS(fail, "ip netns exec %s sysctl -wq net.ipv6.conf.all.forwarding=1", ns2);
	210	+
	211	+ /* Top route */
	212	+ SYS(fail, "ip route add %s/32 dev veth2 %s", IP4_ADDR_1, vrf);
	213	+ SYS(fail, "ip route add %s/32 dev veth3 %s", IP4_ADDR_4, vrf);
	214	+ SYS(fail, "ip -6 route add %s/128 dev veth2 %s", IP6_ADDR_1, vrf);
	215	+ SYS(fail, "ip -6 route add %s/128 dev veth3 %s", IP6_ADDR_4, vrf);
	216	+ /* Bottom route */
	217	+ SYS(fail, "ip route add %s/32 dev veth6 %s", IP4_ADDR_5, vrf);
	218	+ SYS(fail, "ip route add %s/32 dev veth7 %s", IP4_ADDR_8, vrf);
	219	+ SYS(fail, "ip -6 route add %s/128 dev veth6 %s", IP6_ADDR_5, vrf);
	220	+ SYS(fail, "ip -6 route add %s/128 dev veth7 %s", IP6_ADDR_8, vrf);
	221	+
	222	+ close_netns(nstoken);
	223	+ return 0;
	224	+fail:
	225	+ close_netns(nstoken);
	226	+ return -1;
	227	+}
	228	+
	229	+static int configure_ns3(const char *ns3)
	230	+{
	231	+ struct nstoken *nstoken = NULL;
	232	+
	233	+ if (!ns3)
	234	+ goto fail;
	235	+
	236	+ nstoken = open_netns(ns3);
	237	+ if (!ASSERT_OK_PTR(nstoken, "open ns3"))
	238	+ goto fail;
	239	+
	240	+ /* Top route */
	241	+ SYS(fail, "ip route add %s/32 dev veth4", IP4_ADDR_3);
	242	+ SYS(fail, "ip route add %s/32 dev veth4 via %s", IP4_ADDR_1, IP4_ADDR_3);
	243	+ SYS(fail, "ip route add %s/32 dev veth4 via %s", IP4_ADDR_2, IP4_ADDR_3);
	244	+ SYS(fail, "ip -6 route add %s/128 dev veth4", IP6_ADDR_3);
	245	+ SYS(fail, "ip -6 route add %s/128 dev veth4 via %s", IP6_ADDR_1, IP6_ADDR_3);
	246	+ SYS(fail, "ip -6 route add %s/128 dev veth4 via %s", IP6_ADDR_2, IP6_ADDR_3);
	247	+ /* Bottom route */
	248	+ SYS(fail, "ip route add %s/32 dev veth8", IP4_ADDR_7);
	249	+ SYS(fail, "ip route add %s/32 dev veth8 via %s", IP4_ADDR_5, IP4_ADDR_7);
	250	+ SYS(fail, "ip route add %s/32 dev veth8 via %s", IP4_ADDR_6, IP4_ADDR_7);
	251	+ SYS(fail, "ip -6 route add %s/128 dev veth8", IP6_ADDR_7);
	252	+ SYS(fail, "ip -6 route add %s/128 dev veth8 via %s", IP6_ADDR_5, IP6_ADDR_7);
	253	+ SYS(fail, "ip -6 route add %s/128 dev veth8 via %s", IP6_ADDR_6, IP6_ADDR_7);
	254	+
	255	+ /* Configure IPv4 GRE device */
	256	+ SYS(fail, "ip tunnel add gre_dev mode gre remote %s local %s ttl 255",
	257	+ IP4_ADDR_1, IP4_ADDR_GRE);
	258	+ SYS(fail, "ip link set gre_dev up");
	259	+ SYS(fail, "ip a add %s dev gre_dev", IP4_ADDR_GRE);
	260	+
	261	+ /* Configure IPv6 GRE device */
	262	+ SYS(fail, "ip tunnel add gre6_dev mode ip6gre remote %s local %s ttl 255",
	263	+ IP6_ADDR_1, IP6_ADDR_GRE);
	264	+ SYS(fail, "ip link set gre6_dev up");
	265	+ SYS(fail, "ip a add %s dev gre6_dev", IP6_ADDR_GRE);
	266	+
	267	+ close_netns(nstoken);
	268	+ return 0;
	269	+fail:
	270	+ close_netns(nstoken);
	271	+ return -1;
	272	+}
	273	+
	274	+static int setup_network(char ns1, char ns2, char ns3, const char vrf)
	275	+{
	276	+ if (!ns1 \|\| !ns2 \|\| !ns3 \|\| !vrf)
	277	+ goto fail;
	278	+
	279	+ SYS(fail, "ip -n %s link add veth1 type veth peer name veth2 netns %s", ns1, ns2);
	280	+ SYS(fail, "ip -n %s link add veth3 type veth peer name veth4 netns %s", ns2, ns3);
	281	+ SYS(fail, "ip -n %s link add veth5 type veth peer name veth6 netns %s", ns1, ns2);
	282	+ SYS(fail, "ip -n %s link add veth7 type veth peer name veth8 netns %s", ns2, ns3);
	283	+
	284	+ if (vrf[0]) {
	285	+ if (!ASSERT_OK(configure_vrf(ns1, ns2), "configure vrf"))
	286	+ goto fail;
	287	+ }
	288	+ if (!ASSERT_OK(set_top_addr(ns1, ns2, ns3), "set top addresses"))
	289	+ goto fail;
	290	+
	291	+ if (!ASSERT_OK(set_bottom_addr(ns1, ns2, ns3), "set bottom addresses"))
	292	+ goto fail;
	293	+
	294	+ if (!ASSERT_OK(configure_ns1(ns1, vrf), "configure ns1 routes"))
	295	+ goto fail;
	296	+
	297	+ if (!ASSERT_OK(configure_ns2(ns2, vrf), "configure ns2 routes"))
	298	+ goto fail;
	299	+
	300	+ if (!ASSERT_OK(configure_ns3(ns3), "configure ns3 routes"))
	301	+ goto fail;
	302	+
	303	+ /* Link bottom route to the GRE tunnels */
	304	+ SYS(fail, "ip -n %s route add %s/32 dev veth5 via %s %s",
	305	+ ns1, IP4_ADDR_GRE, IP4_ADDR_6, vrf);
	306	+ SYS(fail, "ip -n %s route add %s/32 dev veth7 via %s %s",
	307	+ ns2, IP4_ADDR_GRE, IP4_ADDR_8, vrf);
	308	+ SYS(fail, "ip -n %s -6 route add %s/128 dev veth5 via %s %s",
	309	+ ns1, IP6_ADDR_GRE, IP6_ADDR_6, vrf);
	310	+ SYS(fail, "ip -n %s -6 route add %s/128 dev veth7 via %s %s",
	311	+ ns2, IP6_ADDR_GRE, IP6_ADDR_8, vrf);
	312	+
	313	+ return 0;
	314	+fail:
	315	+ return -1;
	316	+}
	317	+
	318	+static int remove_routes_to_gredev(const char ns1, const char ns2, const char *vrf)
	319	+{
	320	+ SYS(fail, "ip -n %s route del %s dev veth5 %s", ns1, IP4_ADDR_GRE, vrf);
	321	+ SYS(fail, "ip -n %s route del %s dev veth7 %s", ns2, IP4_ADDR_GRE, vrf);
	322	+ SYS(fail, "ip -n %s -6 route del %s/128 dev veth5 %s", ns1, IP6_ADDR_GRE, vrf);
	323	+ SYS(fail, "ip -n %s -6 route del %s/128 dev veth7 %s", ns2, IP6_ADDR_GRE, vrf);
	324	+
	325	+ return 0;
	326	+fail:
	327	+ return -1;
	328	+}
	329	+
	330	+static int add_unreachable_routes_to_gredev(const char ns1, const char ns2, const char *vrf)
	331	+{
	332	+ SYS(fail, "ip -n %s route add unreachable %s/32 %s", ns1, IP4_ADDR_GRE, vrf);
	333	+ SYS(fail, "ip -n %s route add unreachable %s/32 %s", ns2, IP4_ADDR_GRE, vrf);
	334	+ SYS(fail, "ip -n %s -6 route add unreachable %s/128 %s", ns1, IP6_ADDR_GRE, vrf);
	335	+ SYS(fail, "ip -n %s -6 route add unreachable %s/128 %s", ns2, IP6_ADDR_GRE, vrf);
	336	+
	337	+ return 0;
	338	+fail:
	339	+ return -1;
	340	+}
	341	+
	342	+#define GSO_SIZE 5000
	343	+#define GSO_TCP_PORT 9000
	344	+/* This tests the fix from commit ea0371f78799 ("net: fix GSO in bpf_lwt_push_ip_encap") */
	345	+static int test_gso_fix(const char ns1, const char ns3, int family)
	346	+{
	347	+ const char *ip_addr = family == AF_INET ? IP4_ADDR_DST : IP6_ADDR_DST;
	348	+ char gso_packet[GSO_SIZE] = {};
	349	+ struct nstoken *nstoken = NULL;
	350	+ int sfd, cfd, afd;
	351	+ ssize_t bytes;
	352	+ int ret = -1;
	353	+
	354	+ if (!ns1 \|\| !ns3)
	355	+ return ret;
	356	+
	357	+ nstoken = open_netns(ns3);
	358	+ if (!ASSERT_OK_PTR(nstoken, "open ns3"))
	359	+ return ret;
	360	+
	361	+ sfd = start_server_str(family, SOCK_STREAM, ip_addr, GSO_TCP_PORT, NULL);
	362	+ if (!ASSERT_OK_FD(sfd, "start server"))
	363	+ goto close_netns;
	364	+
	365	+ close_netns(nstoken);
	366	+
	367	+ nstoken = open_netns(ns1);
	368	+ if (!ASSERT_OK_PTR(nstoken, "open ns1"))
	369	+ goto close_server;
	370	+
	371	+ cfd = connect_to_addr_str(family, SOCK_STREAM, ip_addr, GSO_TCP_PORT, NULL);
	372	+ if (!ASSERT_OK_FD(cfd, "connect to server"))
	373	+ goto close_server;
	374	+
	375	+ close_netns(nstoken);
	376	+ nstoken = NULL;
	377	+
	378	+ afd = accept(sfd, NULL, NULL);
	379	+ if (!ASSERT_OK_FD(afd, "accept"))
	380	+ goto close_client;
	381	+
	382	+ /* Send a packet larger than MTU */
	383	+ bytes = send(cfd, gso_packet, GSO_SIZE, 0);
	384	+ if (!ASSERT_EQ(bytes, GSO_SIZE, "send packet"))
	385	+ goto close_accept;
	386	+
	387	+ /* Verify we received all expected bytes */
	388	+ bytes = read(afd, gso_packet, GSO_SIZE);
	389	+ if (!ASSERT_EQ(bytes, GSO_SIZE, "receive packet"))
	390	+ goto close_accept;
	391	+
	392	+ ret = 0;
	393	+
	394	+close_accept:
	395	+ close(afd);
	396	+close_client:
	397	+ close(cfd);
	398	+close_server:
	399	+ close(sfd);
	400	+close_netns:
	401	+ close_netns(nstoken);
	402	+
	403	+ return ret;
	404	+}
	405	+
	406	+static int check_ping_ok(const char *ns1)
	407	+{
	408	+ SYS(fail, "ip netns exec %s ping -c 1 -W1 -I veth1 %s > /dev/null", ns1, IP4_ADDR_DST);
	409	+ SYS(fail, "ip netns exec %s ping6 -c 1 -W1 -I veth1 %s > /dev/null", ns1, IP6_ADDR_DST);
	410	+ return 0;
	411	+fail:
	412	+ return -1;
	413	+}
	414	+
	415	+static int check_ping_fails(const char *ns1)
	416	+{
	417	+ int ret;
	418	+
	419	+ ret = SYS_NOFAIL("ip netns exec %s ping -c 1 -W1 -I veth1 %s", ns1, IP4_ADDR_DST);
	420	+ if (!ret)
	421	+ return -1;
	422	+
	423	+ ret = SYS_NOFAIL("ip netns exec %s ping6 -c 1 -W1 -I veth1 %s", ns1, IP6_ADDR_DST);
	424	+ if (!ret)
	425	+ return -1;
	426	+
	427	+ return 0;
	428	+}
	429	+
	430	+#define EGRESS true
	431	+#define INGRESS false
	432	+#define IPV4_ENCAP true
	433	+#define IPV6_ENCAP false
	434	+static void lwt_ip_encap(bool ipv4_encap, bool egress, const char *vrf)
	435	+{
	436	+ char ns1[NETNS_NAME_SIZE] = NETNS_BASE "-1-";
	437	+ char ns2[NETNS_NAME_SIZE] = NETNS_BASE "-2-";
	438	+ char ns3[NETNS_NAME_SIZE] = NETNS_BASE "-3-";
	439	+ char *sec = ipv4_encap ? "encap_gre" : "encap_gre6";
	440	+
	441	+ if (!vrf)
	442	+ return;
	443	+
	444	+ if (!ASSERT_OK(create_ns(ns1, NETNS_NAME_SIZE), "create ns1"))
	445	+ goto out;
	446	+ if (!ASSERT_OK(create_ns(ns2, NETNS_NAME_SIZE), "create ns2"))
	447	+ goto out;
	448	+ if (!ASSERT_OK(create_ns(ns3, NETNS_NAME_SIZE), "create ns3"))
	449	+ goto out;
	450	+
	451	+ if (!ASSERT_OK(setup_network(ns1, ns2, ns3, vrf), "setup network"))
	452	+ goto out;
	453	+
	454	+ /* By default, pings work */
	455	+ if (!ASSERT_OK(check_ping_ok(ns1), "ping OK"))
	456	+ goto out;
	457	+
	458	+ /* Remove NS2->DST routes, ping fails */
	459	+ SYS(out, "ip -n %s route del %s/32 dev veth3 %s", ns2, IP4_ADDR_DST, vrf);
	460	+ SYS(out, "ip -n %s -6 route del %s/128 dev veth3 %s", ns2, IP6_ADDR_DST, vrf);
	461	+ if (!ASSERT_OK(check_ping_fails(ns1), "ping expected fail"))
	462	+ goto out;
	463	+
	464	+ /* Install replacement routes (LWT/eBPF), pings succeed */
	465	+ if (egress) {
	466	+ SYS(out, "ip -n %s route add %s encap bpf xmit obj %s sec %s dev veth1 %s",
	467	+ ns1, IP4_ADDR_DST, BPF_FILE, sec, vrf);
	468	+ SYS(out, "ip -n %s -6 route add %s encap bpf xmit obj %s sec %s dev veth1 %s",
	469	+ ns1, IP6_ADDR_DST, BPF_FILE, sec, vrf);
	470	+ } else {
	471	+ SYS(out, "ip -n %s route add %s encap bpf in obj %s sec %s dev veth2 %s",
	472	+ ns2, IP4_ADDR_DST, BPF_FILE, sec, vrf);
	473	+ SYS(out, "ip -n %s -6 route add %s encap bpf in obj %s sec %s dev veth2 %s",
	474	+ ns2, IP6_ADDR_DST, BPF_FILE, sec, vrf);
	475	+ }
	476	+
	477	+ if (!ASSERT_OK(check_ping_ok(ns1), "ping OK"))
	478	+ goto out;
	479	+
	480	+ /* Skip GSO tests with VRF: VRF routing needs properly assigned
	481	+ * source IP/device, which is easy to do with ping but hard with TCP.
	482	+ */
	483	+ if (egress && !vrf[0]) {
	484	+ if (!ASSERT_OK(test_gso_fix(ns1, ns3, AF_INET), "test GSO"))
	485	+ goto out;
	486	+ }
	487	+
	488	+ /* Negative test: remove routes to GRE devices: ping fails */
	489	+ if (!ASSERT_OK(remove_routes_to_gredev(ns1, ns2, vrf), "remove routes to gredev"))
	490	+ goto out;
	491	+ if (!ASSERT_OK(check_ping_fails(ns1), "ping expected fail"))
	492	+ goto out;
	493	+
	494	+ /* Another negative test */
	495	+ if (!ASSERT_OK(add_unreachable_routes_to_gredev(ns1, ns2, vrf),
	496	+ "add unreachable routes"))
	497	+ goto out;
	498	+ ASSERT_OK(check_ping_fails(ns1), "ping expected fail");
	499	+
	500	+out:
	501	+ SYS_NOFAIL("ip netns del %s", ns1);
	502	+ SYS_NOFAIL("ip netns del %s", ns2);
	503	+ SYS_NOFAIL("ip netns del %s", ns3);
	504	+}
	505	+
	506	+void test_lwt_ip_encap_vrf_ipv6(void)
	507	+{
	508	+ if (test__start_subtest("egress"))
	509	+ lwt_ip_encap(IPV6_ENCAP, EGRESS, "vrf red");
	510	+
	511	+ if (test__start_subtest("ingress"))
	512	+ lwt_ip_encap(IPV6_ENCAP, INGRESS, "vrf red");
	513	+}
	514	+
	515	+void test_lwt_ip_encap_vrf_ipv4(void)
	516	+{
	517	+ if (test__start_subtest("egress"))
	518	+ lwt_ip_encap(IPV4_ENCAP, EGRESS, "vrf red");
	519	+
	520	+ if (test__start_subtest("ingress"))
	521	+ lwt_ip_encap(IPV4_ENCAP, INGRESS, "vrf red");
	522	+}
	523	+
	524	+void test_lwt_ip_encap_ipv6(void)
	525	+{
	526	+ if (test__start_subtest("egress"))
	527	+ lwt_ip_encap(IPV6_ENCAP, EGRESS, "");
	528	+
	529	+ if (test__start_subtest("ingress"))
	530	+ lwt_ip_encap(IPV6_ENCAP, INGRESS, "");
	531	+}
	532	+
	533	+void test_lwt_ip_encap_ipv4(void)
	534	+{
	535	+ if (test__start_subtest("egress"))
	536	+ lwt_ip_encap(IPV4_ENCAP, EGRESS, "");
	537	+
	538	+ if (test__start_subtest("ingress"))
	539	+ lwt_ip_encap(IPV4_ENCAP, INGRESS, "");
	540	+}

tools/testing/selftests/bpf/prog_tests/lwt_seg6local.c+176−0

@@ -0,0 +1,176 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
	2	+
	3	+/* Connects 6 network namespaces through veths.
	4	+ * Each NS may have different IPv6 global scope addresses :
	5	+ *
	6	+ * NS1 NS2 NS3 NS4 NS5 NS6
	7	+ * lo veth1 <-> veth2 veth3 <-> veth4 veth5 <-> veth6 lo veth7 <-> veth8 veth9 <-> veth10 lo
	8	+ * fb00 ::1 ::12 ::21 ::34 ::43 ::56 ::65 ::78 ::87 ::910 ::109 ::6
	9	+ * fd00 ::4
	10	+ * fc42 ::1
	11	+ *
	12	+ * All IPv6 packets going to fb00::/16 through NS2 will be encapsulated in a
	13	+ * IPv6 header with a Segment Routing Header, with segments :
	14	+ * fd00::1 -> fd00::2 -> fd00::3 -> fd00::4
	15	+ *
	16	+ * 3 fd00::/16 IPv6 addresses are binded to seg6local End.BPF actions :
	17	+ * - fd00::1 : add a TLV, change the flags and apply a End.X action to fc42::1
	18	+ * - fd00::2 : remove the TLV, change the flags, add a tag
	19	+ * - fd00::3 : apply an End.T action to fd00::4, through routing table 117
	20	+ *
	21	+ * fd00::4 is a simple Segment Routing node decapsulating the inner IPv6 packet.
	22	+ * Each End.BPF action will validate the operations applied on the SRH by the
	23	+ * previous BPF program in the chain, otherwise the packet is dropped.
	24	+ *
	25	+ * An UDP datagram is sent from fb00::1 to fb00::6. The test succeeds if this
	26	+ * datagram can be read on NS6 when binding to fb00::6.
	27	+ */
	28	+
	29	+#include "network_helpers.h"
	30	+#include "test_progs.h"
	31	+
	32	+#define NETNS_BASE "lwt-seg6local-"
	33	+#define BPF_FILE "test_lwt_seg6local.bpf.o"
	34	+
	35	+static void cleanup(void)
	36	+{
	37	+ int ns;
	38	+
	39	+ for (ns = 1; ns < 7; ns++)
	40	+ SYS_NOFAIL("ip netns del %s%d", NETNS_BASE, ns);
	41	+}
	42	+
	43	+static int setup(void)
	44	+{
	45	+ int ns;
	46	+
	47	+ for (ns = 1; ns < 7; ns++)
	48	+ SYS(fail, "ip netns add %s%d", NETNS_BASE, ns);
	49	+
	50	+ SYS(fail, "ip -n %s6 link set dev lo up", NETNS_BASE);
	51	+
	52	+ for (ns = 1; ns < 6; ns++) {
	53	+ int local_id = ns * 2 - 1;
	54	+ int peer_id = ns * 2;
	55	+ int next_ns = ns + 1;
	56	+
	57	+ SYS(fail, "ip -n %s%d link add veth%d type veth peer name veth%d netns %s%d",
	58	+ NETNS_BASE, ns, local_id, peer_id, NETNS_BASE, next_ns);
	59	+
	60	+ SYS(fail, "ip -n %s%d link set dev veth%d up", NETNS_BASE, ns, local_id);
	61	+ SYS(fail, "ip -n %s%d link set dev veth%d up", NETNS_BASE, next_ns, peer_id);
	62	+
	63	+ /* All link scope addresses to veths */
	64	+ SYS(fail, "ip -n %s%d -6 addr add fb00::%d%d/16 dev veth%d scope link",
	65	+ NETNS_BASE, ns, local_id, peer_id, local_id);
	66	+ SYS(fail, "ip -n %s%d -6 addr add fb00::%d%d/16 dev veth%d scope link",
	67	+ NETNS_BASE, next_ns, peer_id, local_id, peer_id);
	68	+ }
	69	+
	70	+
	71	+ SYS(fail, "ip -n %s5 -6 route add fb00::109 table 117 dev veth9 scope link", NETNS_BASE);
	72	+
	73	+ SYS(fail, "ip -n %s1 -6 addr add fb00::1/16 dev lo", NETNS_BASE);
	74	+ SYS(fail, "ip -n %s1 -6 route add fb00::6 dev veth1 via fb00::21", NETNS_BASE);
	75	+
	76	+ SYS(fail, "ip -n %s2 -6 route add fb00::6 encap bpf in obj %s sec encap_srh dev veth2",
	77	+ NETNS_BASE, BPF_FILE);
	78	+ SYS(fail, "ip -n %s2 -6 route add fd00::1 dev veth3 via fb00::43 scope link", NETNS_BASE);
	79	+
	80	+ SYS(fail, "ip -n %s3 -6 route add fc42::1 dev veth5 via fb00::65", NETNS_BASE);
	81	+ SYS(fail,
	82	+ "ip -n %s3 -6 route add fd00::1 encap seg6local action End.BPF endpoint obj %s sec add_egr_x dev veth4",
	83	+ NETNS_BASE, BPF_FILE);
	84	+
	85	+ SYS(fail,
	86	+ "ip -n %s4 -6 route add fd00::2 encap seg6local action End.BPF endpoint obj %s sec pop_egr dev veth6",
	87	+ NETNS_BASE, BPF_FILE);
	88	+ SYS(fail, "ip -n %s4 -6 addr add fc42::1 dev lo", NETNS_BASE);
	89	+ SYS(fail, "ip -n %s4 -6 route add fd00::3 dev veth7 via fb00::87", NETNS_BASE);
	90	+
	91	+ SYS(fail, "ip -n %s5 -6 route add fd00::4 table 117 dev veth9 via fb00::109", NETNS_BASE);
	92	+ SYS(fail,
	93	+ "ip -n %s5 -6 route add fd00::3 encap seg6local action End.BPF endpoint obj %s sec inspect_t dev veth8",
	94	+ NETNS_BASE, BPF_FILE);
	95	+
	96	+ SYS(fail, "ip -n %s6 -6 addr add fb00::6/16 dev lo", NETNS_BASE);
	97	+ SYS(fail, "ip -n %s6 -6 addr add fd00::4/16 dev lo", NETNS_BASE);
	98	+
	99	+ for (ns = 1; ns < 6; ns++)
	100	+ SYS(fail, "ip netns exec %s%d sysctl -wq net.ipv6.conf.all.forwarding=1",
	101	+ NETNS_BASE, ns);
	102	+
	103	+ SYS(fail, "ip netns exec %s6 sysctl -wq net.ipv6.conf.all.seg6_enabled=1", NETNS_BASE);
	104	+ SYS(fail, "ip netns exec %s6 sysctl -wq net.ipv6.conf.lo.seg6_enabled=1", NETNS_BASE);
	105	+ SYS(fail, "ip netns exec %s6 sysctl -wq net.ipv6.conf.veth10.seg6_enabled=1", NETNS_BASE);
	106	+
	107	+ return 0;
	108	+fail:
	109	+ return -1;
	110	+}
	111	+
	112	+#define SERVER_PORT 7330
	113	+#define CLIENT_PORT 2121
	114	+void test_lwt_seg6local(void)
	115	+{
	116	+ struct sockaddr_in6 server_addr = {};
	117	+ const char *ns1 = NETNS_BASE "1";
	118	+ const char *ns6 = NETNS_BASE "6";
	119	+ struct nstoken *nstoken = NULL;
	120	+ const char *foobar = "foobar";
	121	+ ssize_t bytes;
	122	+ int sfd, cfd;
	123	+ char buf[7];
	124	+
	125	+ if (!ASSERT_OK(setup(), "setup"))
	126	+ goto out;
	127	+
	128	+ nstoken = open_netns(ns6);
	129	+ if (!ASSERT_OK_PTR(nstoken, "open ns6"))
	130	+ goto out;
	131	+
	132	+ sfd = start_server_str(AF_INET6, SOCK_DGRAM, "fb00::6", SERVER_PORT, NULL);
	133	+ if (!ASSERT_OK_FD(sfd, "start server"))
	134	+ goto close_netns;
	135	+
	136	+ close_netns(nstoken);
	137	+
	138	+ nstoken = open_netns(ns1);
	139	+ if (!ASSERT_OK_PTR(nstoken, "open ns1"))
	140	+ goto close_server;
	141	+
	142	+ cfd = start_server_str(AF_INET6, SOCK_DGRAM, "fb00::1", CLIENT_PORT, NULL);
	143	+ if (!ASSERT_OK_FD(cfd, "start client"))
	144	+ goto close_server;
	145	+
	146	+ close_netns(nstoken);
	147	+ nstoken = NULL;
	148	+
	149	+ /* Send a packet larger than MTU */
	150	+ server_addr.sin6_family = AF_INET6;
	151	+ server_addr.sin6_port = htons(SERVER_PORT);
	152	+ if (!ASSERT_EQ(inet_pton(AF_INET6, "fb00::6", &server_addr.sin6_addr), 1,
	153	+ "build target addr"))
	154	+ goto close_client;
	155	+
	156	+ bytes = sendto(cfd, foobar, sizeof(foobar), 0,
	157	+ (struct sockaddr *)&server_addr, sizeof(server_addr));
	158	+ if (!ASSERT_EQ(bytes, sizeof(foobar), "send packet"))
	159	+ goto close_client;
	160	+
	161	+ /* Verify we received all expected bytes */
	162	+ bytes = read(sfd, buf, sizeof(buf));
	163	+ if (!ASSERT_EQ(bytes, sizeof(buf), "receive packet"))
	164	+ goto close_client;
	165	+ ASSERT_STREQ(buf, foobar, "check udp packet");
	166	+
	167	+close_client:
	168	+ close(cfd);
	169	+close_server:
	170	+ close(sfd);
	171	+close_netns:
	172	+ close_netns(nstoken);
	173	+
	174	+out:
	175	+ cleanup();
	176	+}

tools/testing/selftests/bpf/prog_tests/netns_cookie.c+14−7

@@ -33,20 +33,25 @@ void test_netns_cookie(void) · 32 unmodified lines
33	33
34	34	skel->links.get_netns_cookie_sockops = bpf_program__attach_cgroup(
35	35	skel->progs.get_netns_cookie_sockops, cgroup_fd);
36		− if (!ASSERT_OK_PTR(skel->links.get_netns_cookie_sockops, "prog_attach"))
	36	+ if (!ASSERT_OK_PTR(skel->links.get_netns_cookie_sockops, "prog_attach_sockops"))
37	37	goto done;
38	38
39	39	verdict = bpf_program__fd(skel->progs.get_netns_cookie_sk_msg);
40	40	map = bpf_map__fd(skel->maps.sock_map);
41	41	err = bpf_prog_attach(verdict, map, BPF_SK_MSG_VERDICT, 0);
42		− if (!ASSERT_OK(err, "prog_attach"))
	42	+ if (!ASSERT_OK(err, "prog_attach_sk_msg"))
43	43	goto done;
44	44
45	45	tc_fd = bpf_program__fd(skel->progs.get_netns_cookie_tcx);
46	46	err = bpf_prog_attach_opts(tc_fd, loopback, BPF_TCX_INGRESS, &opta);
47		− if (!ASSERT_OK(err, "prog_attach"))
	47	+ if (!ASSERT_OK(err, "prog_attach_tcx"))
48	48	goto done;
49	49
	50	+ skel->links.get_netns_cookie_cgroup_skb = bpf_program__attach_cgroup(
	51	+ skel->progs.get_netns_cookie_cgroup_skb, cgroup_fd);
	52	+ if (!ASSERT_OK_PTR(skel->links.get_netns_cookie_cgroup_skb, "prog_attach_cgroup_skb"))
	53	+ goto cleanup_tc;
	54	+
50	55	server_fd = start_server(AF_INET6, SOCK_STREAM, "::1", 0, 0);
51	56	if (CHECK(server_fd < 0, "start_server", "errno %d\n", errno))
52	57	goto cleanup_tc;
@@ -69,16 +74,18 @@ void test_netns_cookie(void) · 16 unmodified lines
69	74	if (!ASSERT_OK(err, "getsockopt"))
70	75	goto cleanup_tc;
71	76
72		− ASSERT_EQ(val, cookie_expected_value, "cookie_value");
	77	+ ASSERT_EQ(val, cookie_expected_value, "cookie_value_sockops");
73	78
74	79	err = bpf_map_lookup_elem(bpf_map__fd(skel->maps.sk_msg_netns_cookies),
75	80	&client_fd, &val);
76	81	if (!ASSERT_OK(err, "map_lookup(sk_msg_netns_cookies)"))
77	82	goto cleanup_tc;
78	83
79		− ASSERT_EQ(val, cookie_expected_value, "cookie_value");
80		− ASSERT_EQ(skel->bss->tcx_init_netns_cookie, cookie_expected_value, "cookie_value");
81		− ASSERT_EQ(skel->bss->tcx_netns_cookie, cookie_expected_value, "cookie_value");
	84	+ ASSERT_EQ(val, cookie_expected_value, "cookie_value_sk_msg");
	85	+ ASSERT_EQ(skel->bss->tcx_init_netns_cookie, cookie_expected_value, "cookie_value_init_tcx");
	86	+ ASSERT_EQ(skel->bss->tcx_netns_cookie, cookie_expected_value, "cookie_value_tcx");
	87	+ ASSERT_EQ(skel->bss->cgroup_skb_init_netns_cookie, cookie_expected_value, "cookie_value_init_cgroup_skb");
	88	+ ASSERT_EQ(skel->bss->cgroup_skb_netns_cookie, cookie_expected_value, "cookie_value_cgroup_skb");
82	89
83	90	cleanup_tc:
84	91	err = bpf_prog_detach_opts(tc_fd, loopback, BPF_TCX_INGRESS, &optd);

tools/testing/selftests/bpf/prog_tests/ns_current_pid_tgid.c+17−30

@@ -200,41 +200,28 @@ static void test_ns_current_pid_tgid_new_ns(int (fn)(void ), void *arg) · 199 unmodified lines
200	200	return;
201	201	}
202	202
203		−static void test_in_netns(int (fn)(void ), void *arg)
204		−{
205		− struct nstoken *nstoken = NULL;
206		−
207		− SYS(cleanup, "ip netns add ns_current_pid_tgid");
208		− SYS(cleanup, "ip -net ns_current_pid_tgid link set dev lo up");
209		−
210		− nstoken = open_netns("ns_current_pid_tgid");
211		− if (!ASSERT_OK_PTR(nstoken, "open_netns"))
212		− goto cleanup;
213		−
214		− test_ns_current_pid_tgid_new_ns(fn, arg);
215		−
216		−cleanup:
217		− if (nstoken)
218		− close_netns(nstoken);
219		− SYS_NOFAIL("ip netns del ns_current_pid_tgid");
220		−}
221		−
222	203	/* TODO: use a different tracepoint */
223		−void serial_test_ns_current_pid_tgid(void)
	204	+void serial_test_current_pid_tgid(void)
224	205	{
225	206	if (test__start_subtest("root_ns_tp"))
226	207	test_current_pid_tgid_tp(NULL);
227	208	if (test__start_subtest("new_ns_tp"))
228	209	test_ns_current_pid_tgid_new_ns(test_current_pid_tgid_tp, NULL);
229		− if (test__start_subtest("new_ns_cgrp")) {
230		− int cgroup_fd = -1;
231		−
232		− cgroup_fd = test__join_cgroup("/sock_addr");
233		− if (ASSERT_GE(cgroup_fd, 0, "join_cgroup")) {
234		− test_in_netns(test_current_pid_tgid_cgrp, &cgroup_fd);
235		− close(cgroup_fd);
236		− }
	210	+}
	211	+
	212	+void test_ns_current_pid_tgid_cgrp(void)
	213	+{
	214	+ int cgroup_fd = test__join_cgroup("/sock_addr");
	215	+
	216	+ if (ASSERT_OK_FD(cgroup_fd, "join_cgroup")) {
	217	+ test_ns_current_pid_tgid_new_ns(test_current_pid_tgid_cgrp, &cgroup_fd);
	218	+ close(cgroup_fd);
237	219	}
238		− if (test__start_subtest("new_ns_sk_msg"))
239		− test_in_netns(test_current_pid_tgid_sk_msg, NULL);
240	220	}
	221	+
	222	+void test_ns_current_pid_tgid_sk_msg(void)
	223	+{
	224	+ test_ns_current_pid_tgid_new_ns(test_current_pid_tgid_sk_msg, NULL);
	225	+}
	226	+
	227	+

tools/testing/selftests/bpf/prog_tests/prepare.c+99−0

@@ -0,0 +1,99 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2025 Meta */
	3	+
	4	+#include <test_progs.h>
	5	+#include <network_helpers.h>
	6	+#include "prepare.skel.h"
	7	+
	8	+static bool check_prepared(struct bpf_object *obj)
	9	+{
	10	+ bool is_prepared = true;
	11	+ const struct bpf_map *map;
	12	+
	13	+ bpf_object__for_each_map(map, obj) {
	14	+ if (bpf_map__fd(map) < 0)
	15	+ is_prepared = false;
	16	+ }
	17	+
	18	+ return is_prepared;
	19	+}
	20	+
	21	+static void test_prepare_no_load(void)
	22	+{
	23	+ struct prepare *skel;
	24	+ int err;
	25	+ LIBBPF_OPTS(bpf_test_run_opts, topts,
	26	+ .data_in = &pkt_v4,
	27	+ .data_size_in = sizeof(pkt_v4),
	28	+ );
	29	+
	30	+ skel = prepare__open();
	31	+ if (!ASSERT_OK_PTR(skel, "prepare__open"))
	32	+ return;
	33	+
	34	+ if (!ASSERT_FALSE(check_prepared(skel->obj), "not check_prepared"))
	35	+ goto cleanup;
	36	+
	37	+ err = bpf_object__prepare(skel->obj);
	38	+
	39	+ if (!ASSERT_TRUE(check_prepared(skel->obj), "check_prepared"))
	40	+ goto cleanup;
	41	+
	42	+ if (!ASSERT_OK(err, "bpf_object__prepare"))
	43	+ goto cleanup;
	44	+
	45	+cleanup:
	46	+ prepare__destroy(skel);
	47	+}
	48	+
	49	+static void test_prepare_load(void)
	50	+{
	51	+ struct prepare *skel;
	52	+ int err, prog_fd;
	53	+ LIBBPF_OPTS(bpf_test_run_opts, topts,
	54	+ .data_in = &pkt_v4,
	55	+ .data_size_in = sizeof(pkt_v4),
	56	+ );
	57	+
	58	+ skel = prepare__open();
	59	+ if (!ASSERT_OK_PTR(skel, "prepare__open"))
	60	+ return;
	61	+
	62	+ if (!ASSERT_FALSE(check_prepared(skel->obj), "not check_prepared"))
	63	+ goto cleanup;
	64	+
	65	+ err = bpf_object__prepare(skel->obj);
	66	+ if (!ASSERT_OK(err, "bpf_object__prepare"))
	67	+ goto cleanup;
	68	+
	69	+ err = prepare__load(skel);
	70	+ if (!ASSERT_OK(err, "prepare__load"))
	71	+ goto cleanup;
	72	+
	73	+ if (!ASSERT_TRUE(check_prepared(skel->obj), "check_prepared"))
	74	+ goto cleanup;
	75	+
	76	+ prog_fd = bpf_program__fd(skel->progs.program);
	77	+ if (!ASSERT_GE(prog_fd, 0, "prog_fd"))
	78	+ goto cleanup;
	79	+
	80	+ err = bpf_prog_test_run_opts(prog_fd, &topts);
	81	+ if (!ASSERT_OK(err, "test_run_opts err"))
	82	+ goto cleanup;
	83	+
	84	+ if (!ASSERT_OK(topts.retval, "test_run_opts retval"))
	85	+ goto cleanup;
	86	+
	87	+ ASSERT_EQ(skel->bss->err, 0, "err");
	88	+
	89	+cleanup:
	90	+ prepare__destroy(skel);
	91	+}
	92	+
	93	+void test_prepare(void)
	94	+{
	95	+ if (test__start_subtest("prepare_load"))
	96	+ test_prepare_load();
	97	+ if (test__start_subtest("prepare_no_load"))
	98	+ test_prepare_no_load();
	99	+}

tools/testing/selftests/bpf/prog_tests/pro_epilogue.c+2−0

@@ -6,6 +6,7 @@ · 5 unmodified lines
6	6	#include "epilogue_tailcall.skel.h"
7	7	#include "pro_epilogue_goto_start.skel.h"
8	8	#include "epilogue_exit.skel.h"
	9	+#include "pro_epilogue_with_kfunc.skel.h"
9	10
10	11	struct st_ops_args {
11	12	__u64 a;
@@ -55,6 +56,7 @@ void test_pro_epilogue(void) · 43 unmodified lines
55	56	RUN_TESTS(pro_epilogue);
56	57	RUN_TESTS(pro_epilogue_goto_start);
57	58	RUN_TESTS(epilogue_exit);
	59	+ RUN_TESTS(pro_epilogue_with_kfunc);
58	60	if (test__start_subtest("tailcall"))
59	61	test_tailcall();
60	62	}

tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c+3−0

@@ -81,6 +81,9 @@ static const char * const inproper_region_tests[] = { · 80 unmodified lines
81	81	"nested_rcu_region",
82	82	"rcu_read_lock_global_subprog_lock",
83	83	"rcu_read_lock_global_subprog_unlock",
	84	+ "rcu_read_lock_sleepable_helper_global_subprog",
	85	+ "rcu_read_lock_sleepable_kfunc_global_subprog",
	86	+ "rcu_read_lock_sleepable_global_subprog_indirect",
84	87	};
85	88
86	89	static void test_inproper_region(void)

tools/testing/selftests/bpf/prog_tests/read_vsyscall.c+1−0

@@ -24,6 +24,7 @@ struct read_ret_desc { · 23 unmodified lines
24	24	{ .name = "copy_from_user", .ret = -EFAULT },
25	25	{ .name = "copy_from_user_task", .ret = -EFAULT },
26	26	{ .name = "copy_from_user_str", .ret = -EFAULT },
	27	+ { .name = "copy_from_user_task_str", .ret = -EFAULT },
27	28	};
28	29
29	30	void test_read_vsyscall(void)

tools/testing/selftests/bpf/prog_tests/setget_sockopt.c+1−1

@@ -202,7 +202,7 @@ err_out: · 201 unmodified lines
202	202	void test_setget_sockopt(void)
203	203	{
204	204	cg_fd = test__join_cgroup(CG_NAME);
205		− if (cg_fd < 0)
	205	+ if (!ASSERT_OK_FD(cg_fd, "join cgroup"))
206	206	return;
207	207
208	208	if (create_netns())

tools/testing/selftests/bpf/prog_tests/spin_lock.c+3−0

@@ -50,6 +50,9 @@ static struct { · 49 unmodified lines
50	50	{ "lock_id_mismatch_innermapval_mapval", "bpf_spin_unlock of different lock" },
51	51	{ "lock_global_subprog_call1", "global function calls are not allowed while holding a lock" },
52	52	{ "lock_global_subprog_call2", "global function calls are not allowed while holding a lock" },
	53	+ { "lock_global_sleepable_helper_subprog", "global function calls are not allowed while holding a lock" },
	54	+ { "lock_global_sleepable_kfunc_subprog", "global function calls are not allowed while holding a lock" },
	55	+ { "lock_global_sleepable_subprog_indirect", "global function calls are not allowed while holding a lock" },
53	56	};
54	57
55	58	static int match_regex(const char pattern, const char string)

tools/testing/selftests/bpf/prog_tests/summarization.c+144−0

@@ -0,0 +1,144 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+#include "bpf/libbpf.h"
	3	+#include "summarization_freplace.skel.h"
	4	+#include "summarization.skel.h"
	5	+#include <test_progs.h>
	6	+
	7	+static void print_verifier_log(const char *log)
	8	+{
	9	+ if (env.verbosity >= VERBOSE_VERY)
	10	+ fprintf(stdout, "VERIFIER LOG:\n=============\n%s=============\n", log);
	11	+}
	12	+
	13	+static void test_aux(const char *main_prog_name,
	14	+ const char *to_be_replaced,
	15	+ const char *replacement,
	16	+ bool expect_load,
	17	+ const char *err_msg)
	18	+{
	19	+ struct summarization_freplace *freplace = NULL;
	20	+ struct bpf_program *freplace_prog = NULL;
	21	+ struct bpf_program *main_prog = NULL;
	22	+ LIBBPF_OPTS(bpf_object_open_opts, opts);
	23	+ struct summarization *main = NULL;
	24	+ char log[16*1024];
	25	+ int err;
	26	+
	27	+ opts.kernel_log_buf = log;
	28	+ opts.kernel_log_size = sizeof(log);
	29	+ if (env.verbosity >= VERBOSE_SUPER)
	30	+ opts.kernel_log_level = 1 \| 2 \| 4;
	31	+ main = summarization__open_opts(&opts);
	32	+ if (!ASSERT_OK_PTR(main, "summarization__open"))
	33	+ goto out;
	34	+ main_prog = bpf_object__find_program_by_name(main->obj, main_prog_name);
	35	+ if (!ASSERT_OK_PTR(main_prog, "main_prog"))
	36	+ goto out;
	37	+ bpf_program__set_autoload(main_prog, true);
	38	+ err = summarization__load(main);
	39	+ print_verifier_log(log);
	40	+ if (!ASSERT_OK(err, "summarization__load"))
	41	+ goto out;
	42	+ freplace = summarization_freplace__open_opts(&opts);
	43	+ if (!ASSERT_OK_PTR(freplace, "summarization_freplace__open"))
	44	+ goto out;
	45	+ freplace_prog = bpf_object__find_program_by_name(freplace->obj, replacement);
	46	+ if (!ASSERT_OK_PTR(freplace_prog, "freplace_prog"))
	47	+ goto out;
	48	+ bpf_program__set_autoload(freplace_prog, true);
	49	+ bpf_program__set_autoattach(freplace_prog, true);
	50	+ bpf_program__set_attach_target(freplace_prog,
	51	+ bpf_program__fd(main_prog),
	52	+ to_be_replaced);
	53	+ err = summarization_freplace__load(freplace);
	54	+ print_verifier_log(log);
	55	+
	56	+ /* The might_sleep extension doesn't work yet as sleepable calls are not
	57	+ * allowed, but preserve the check in case it's supported later and then
	58	+ * this particular combination can be enabled.
	59	+ */
	60	+ if (!strcmp("might_sleep", replacement) && err) {
	61	+ ASSERT_HAS_SUBSTR(log, "helper call might sleep in a non-sleepable prog", "error log");
	62	+ ASSERT_EQ(err, -EINVAL, "err");
	63	+ test__skip();
	64	+ goto out;
	65	+ }
	66	+
	67	+ if (expect_load) {
	68	+ ASSERT_OK(err, "summarization_freplace__load");
	69	+ } else {
	70	+ ASSERT_ERR(err, "summarization_freplace__load");
	71	+ ASSERT_HAS_SUBSTR(log, err_msg, "error log");
	72	+ }
	73	+
	74	+out:
	75	+ summarization_freplace__destroy(freplace);
	76	+ summarization__destroy(main);
	77	+}
	78	+
	79	+/* There are two global subprograms in both summarization.skel.h:
	80	+ * - one changes packet data;
	81	+ * - another does not.
	82	+ * It is ok to freplace subprograms that change packet data with those
	83	+ * that either do or do not. It is only ok to freplace subprograms
	84	+ * that do not change packet data with those that do not as well.
	85	+ * The below tests check outcomes for each combination of such freplace.
	86	+ * Also test a case when main subprogram itself is replaced and is a single
	87	+ * subprogram in a program.
	88	+ *
	89	+ * This holds for might_sleep programs. It is ok to replace might_sleep with
	90	+ * might_sleep and with does_not_sleep, but does_not_sleep cannot be replaced
	91	+ * with might_sleep.
	92	+ */
	93	+void test_summarization_freplace(void)
	94	+{
	95	+ struct {
	96	+ const char *main;
	97	+ const char *to_be_replaced;
	98	+ bool has_side_effect;
	99	+ } mains[2][4] = {
	100	+ {
	101	+ { "main_changes_with_subprogs", "changes_pkt_data", true },
	102	+ { "main_changes_with_subprogs", "does_not_change_pkt_data", false },
	103	+ { "main_changes", "main_changes", true },
	104	+ { "main_does_not_change", "main_does_not_change", false },
	105	+ },
	106	+ {
	107	+ { "main_might_sleep_with_subprogs", "might_sleep", true },
	108	+ { "main_might_sleep_with_subprogs", "does_not_sleep", false },
	109	+ { "main_might_sleep", "main_might_sleep", true },
	110	+ { "main_does_not_sleep", "main_does_not_sleep", false },
	111	+ },
	112	+ };
	113	+ const char *pkt_err = "Extension program changes packet data";
	114	+ const char *slp_err = "Extension program may sleep";
	115	+ struct {
	116	+ const char *func;
	117	+ bool has_side_effect;
	118	+ const char *err_msg;
	119	+ } replacements[2][2] = {
	120	+ {
	121	+ { "changes_pkt_data", true, pkt_err },
	122	+ { "does_not_change_pkt_data", false, pkt_err },
	123	+ },
	124	+ {
	125	+ { "might_sleep", true, slp_err },
	126	+ { "does_not_sleep", false, slp_err },
	127	+ },
	128	+ };
	129	+ char buf[64];
	130	+
	131	+ for (int t = 0; t < 2; t++) {
	132	+ for (int i = 0; i < ARRAY_SIZE(mains); ++i) {
	133	+ for (int j = 0; j < ARRAY_SIZE(replacements); ++j) {
	134	+ snprintf(buf, sizeof(buf), "%s_with_%s",
	135	+ mains[t][i].to_be_replaced, replacements[t][j].func);
	136	+ if (!test__start_subtest(buf))
	137	+ continue;
	138	+ test_aux(mains[t][i].main, mains[t][i].to_be_replaced, replacements[t][j].func,
	139	+ mains[t][i].has_side_effect \|\| !replacements[t][j].has_side_effect,
	140	+ replacements[t][j].err_msg);
	141	+ }
	142	+ }
	143	+ }
	144	+}

tools/testing/selftests/bpf/prog_tests/tailcalls.c+1−0

@@ -1600,6 +1600,7 @@ static void test_tailcall_bpf2bpf_freplace(void) · 1599 unmodified lines
1600	1600	goto out;
1601	1601
1602	1602	err = bpf_link__destroy(freplace_link);
	1603	+ freplace_link = NULL;
1603	1604	if (!ASSERT_OK(err, "destroy link"))
1604	1605	goto out;
1605	1606

tools/testing/selftests/bpf/prog_tests/tc_links.c+14−14

@@ -13,7 +13,7 @@ · 12 unmodified lines
13	13	#include "netlink_helpers.h"
14	14	#include "tc_helpers.h"
15	15
16		−void serial_test_tc_links_basic(void)
	16	+void test_ns_tc_links_basic(void)
17	17	{
18	18	LIBBPF_OPTS(bpf_prog_query_opts, optq);
19	19	LIBBPF_OPTS(bpf_tcx_opts, optl);
@@ -260,7 +260,7 @@ cleanup: · 240 unmodified lines
260	260	assert_mprog_count(target, 0);
261	261	}
262	262
263		−void serial_test_tc_links_before(void)
	263	+void test_ns_tc_links_before(void)
264	264	{
265	265	test_tc_links_before_target(BPF_TCX_INGRESS);
266	266	test_tc_links_before_target(BPF_TCX_EGRESS);
@@ -414,7 +414,7 @@ cleanup: · 147 unmodified lines
414	414	assert_mprog_count(target, 0);
415	415	}
416	416
417		−void serial_test_tc_links_after(void)
	417	+void test_ns_tc_links_after(void)
418	418	{
419	419	test_tc_links_after_target(BPF_TCX_INGRESS);
420	420	test_tc_links_after_target(BPF_TCX_EGRESS);
@@ -514,7 +514,7 @@ cleanup: · 93 unmodified lines
514	514	assert_mprog_count(target, 0);
515	515	}
516	516
517		−void serial_test_tc_links_revision(void)
	517	+void test_ns_tc_links_revision(void)
518	518	{
519	519	test_tc_links_revision_target(BPF_TCX_INGRESS);
520	520	test_tc_links_revision_target(BPF_TCX_EGRESS);
@@ -618,7 +618,7 @@ cleanup: · 97 unmodified lines
618	618	assert_mprog_count(target, 0);
619	619	}
620	620
621		−void serial_test_tc_links_chain_classic(void)
	621	+void test_ns_tc_links_chain_classic(void)
622	622	{
623	623	test_tc_chain_classic(BPF_TCX_INGRESS, false);
624	624	test_tc_chain_classic(BPF_TCX_EGRESS, false);
@@ -846,7 +846,7 @@ cleanup: · 221 unmodified lines
846	846	assert_mprog_count(target, 0);
847	847	}
848	848
849		−void serial_test_tc_links_replace(void)
	849	+void test_ns_tc_links_replace(void)
850	850	{
851	851	test_tc_links_replace_target(BPF_TCX_INGRESS);
852	852	test_tc_links_replace_target(BPF_TCX_EGRESS);
@@ -1158,7 +1158,7 @@ cleanup: · 305 unmodified lines
1158	1158	assert_mprog_count(target, 0);
1159	1159	}
1160	1160
1161		−void serial_test_tc_links_invalid(void)
	1161	+void test_ns_tc_links_invalid(void)
1162	1162	{
1163	1163	test_tc_links_invalid_target(BPF_TCX_INGRESS);
1164	1164	test_tc_links_invalid_target(BPF_TCX_EGRESS);
@@ -1314,7 +1314,7 @@ cleanup: · 149 unmodified lines
1314	1314	assert_mprog_count(target, 0);
1315	1315	}
1316	1316
1317		−void serial_test_tc_links_prepend(void)
	1317	+void test_ns_tc_links_prepend(void)
1318	1318	{
1319	1319	test_tc_links_prepend_target(BPF_TCX_INGRESS);
1320	1320	test_tc_links_prepend_target(BPF_TCX_EGRESS);
@@ -1470,7 +1470,7 @@ cleanup: · 149 unmodified lines
1470	1470	assert_mprog_count(target, 0);
1471	1471	}
1472	1472
1473		−void serial_test_tc_links_append(void)
	1473	+void test_ns_tc_links_append(void)
1474	1474	{
1475	1475	test_tc_links_append_target(BPF_TCX_INGRESS);
1476	1476	test_tc_links_append_target(BPF_TCX_EGRESS);
@@ -1568,7 +1568,7 @@ cleanup: · 91 unmodified lines
1568	1568	ASSERT_EQ(if_nametoindex("tcx_opts2"), 0, "dev2_removed");
1569	1569	}
1570	1570
1571		−void serial_test_tc_links_dev_cleanup(void)
	1571	+void test_ns_tc_links_dev_cleanup(void)
1572	1572	{
1573	1573	test_tc_links_dev_cleanup_target(BPF_TCX_INGRESS);
1574	1574	test_tc_links_dev_cleanup_target(BPF_TCX_EGRESS);
@@ -1672,7 +1672,7 @@ cleanup: · 97 unmodified lines
1672	1672	test_tc_link__destroy(skel);
1673	1673	}
1674	1674
1675		−void serial_test_tc_links_chain_mixed(void)
	1675	+void test_ns_tc_links_chain_mixed(void)
1676	1676	{
1677	1677	test_tc_chain_mixed(BPF_TCX_INGRESS);
1678	1678	test_tc_chain_mixed(BPF_TCX_EGRESS);
@@ -1782,7 +1782,7 @@ cleanup: · 103 unmodified lines
1782	1782	assert_mprog_count(target, 0);
1783	1783	}
1784	1784
1785		−void serial_test_tc_links_ingress(void)
	1785	+void test_ns_tc_links_ingress(void)
1786	1786	{
1787	1787	test_tc_links_ingress(BPF_TCX_INGRESS, true, true);
1788	1788	test_tc_links_ingress(BPF_TCX_INGRESS, true, false);
@@ -1823,7 +1823,7 @@ static int qdisc_replace(int ifindex, const char *kind, bool block) · 34 unmodified lines
1823	1823	return err;
1824	1824	}
1825	1825
1826		−void serial_test_tc_links_dev_chain0(void)
	1826	+void test_ns_tc_links_dev_chain0(void)
1827	1827	{
1828	1828	int err, ifindex;
1829	1829
@@ -1955,7 +1955,7 @@ cleanup: · 125 unmodified lines
1955	1955	ASSERT_EQ(if_nametoindex("tcx_opts2"), 0, "dev2_removed");
1956	1956	}
1957	1957
1958		−void serial_test_tc_links_dev_mixed(void)
	1958	+void test_ns_tc_links_dev_mixed(void)
1959	1959	{
1960	1960	test_tc_links_dev_mixed(BPF_TCX_INGRESS);
1961	1961	test_tc_links_dev_mixed(BPF_TCX_EGRESS);

tools/testing/selftests/bpf/prog_tests/tc_opts.c+20−20

@@ -10,7 +10,7 @@ · 9 unmodified lines
10	10	#include "test_tc_link.skel.h"
11	11	#include "tc_helpers.h"
12	12
13		−void serial_test_tc_opts_basic(void)
	13	+void test_ns_tc_opts_basic(void)
14	14	{
15	15	LIBBPF_OPTS(bpf_prog_attach_opts, opta);
16	16	LIBBPF_OPTS(bpf_prog_detach_opts, optd);
@@ -254,7 +254,7 @@ cleanup: · 237 unmodified lines
254	254	test_tc_link__destroy(skel);
255	255	}
256	256
257		−void serial_test_tc_opts_before(void)
	257	+void test_ns_tc_opts_before(void)
258	258	{
259	259	test_tc_opts_before_target(BPF_TCX_INGRESS);
260	260	test_tc_opts_before_target(BPF_TCX_EGRESS);
@@ -445,7 +445,7 @@ cleanup: · 184 unmodified lines
445	445	test_tc_link__destroy(skel);
446	446	}
447	447
448		−void serial_test_tc_opts_after(void)
	448	+void test_ns_tc_opts_after(void)
449	449	{
450	450	test_tc_opts_after_target(BPF_TCX_INGRESS);
451	451	test_tc_opts_after_target(BPF_TCX_EGRESS);
@@ -554,7 +554,7 @@ cleanup: · 102 unmodified lines
554	554	test_tc_link__destroy(skel);
555	555	}
556	556
557		−void serial_test_tc_opts_revision(void)
	557	+void test_ns_tc_opts_revision(void)
558	558	{
559	559	test_tc_opts_revision_target(BPF_TCX_INGRESS);
560	560	test_tc_opts_revision_target(BPF_TCX_EGRESS);
@@ -655,7 +655,7 @@ cleanup: · 94 unmodified lines
655	655	assert_mprog_count(target, 0);
656	656	}
657	657
658		−void serial_test_tc_opts_chain_classic(void)
	658	+void test_ns_tc_opts_chain_classic(void)
659	659	{
660	660	test_tc_chain_classic(BPF_TCX_INGRESS, false);
661	661	test_tc_chain_classic(BPF_TCX_EGRESS, false);
@@ -864,7 +864,7 @@ cleanup: · 202 unmodified lines
864	864	test_tc_link__destroy(skel);
865	865	}
866	866
867		−void serial_test_tc_opts_replace(void)
	867	+void test_ns_tc_opts_replace(void)
868	868	{
869	869	test_tc_opts_replace_target(BPF_TCX_INGRESS);
870	870	test_tc_opts_replace_target(BPF_TCX_EGRESS);
@@ -1017,7 +1017,7 @@ cleanup: · 146 unmodified lines
1017	1017	test_tc_link__destroy(skel);
1018	1018	}
1019	1019
1020		−void serial_test_tc_opts_invalid(void)
	1020	+void test_ns_tc_opts_invalid(void)
1021	1021	{
1022	1022	test_tc_opts_invalid_target(BPF_TCX_INGRESS);
1023	1023	test_tc_opts_invalid_target(BPF_TCX_EGRESS);
@@ -1157,7 +1157,7 @@ cleanup: · 133 unmodified lines
1157	1157	test_tc_link__destroy(skel);
1158	1158	}
1159	1159
1160		−void serial_test_tc_opts_prepend(void)
	1160	+void test_ns_tc_opts_prepend(void)
1161	1161	{
1162	1162	test_tc_opts_prepend_target(BPF_TCX_INGRESS);
1163	1163	test_tc_opts_prepend_target(BPF_TCX_EGRESS);
@@ -1297,7 +1297,7 @@ cleanup: · 133 unmodified lines
1297	1297	test_tc_link__destroy(skel);
1298	1298	}
1299	1299
1300		−void serial_test_tc_opts_append(void)
	1300	+void test_ns_tc_opts_append(void)
1301	1301	{
1302	1302	test_tc_opts_append_target(BPF_TCX_INGRESS);
1303	1303	test_tc_opts_append_target(BPF_TCX_EGRESS);
@@ -1387,7 +1387,7 @@ cleanup: · 83 unmodified lines
1387	1387	ASSERT_EQ(if_nametoindex("tcx_opts2"), 0, "dev2_removed");
1388	1388	}
1389	1389
1390		−void serial_test_tc_opts_dev_cleanup(void)
	1390	+void test_ns_tc_opts_dev_cleanup(void)
1391	1391	{
1392	1392	test_tc_opts_dev_cleanup_target(BPF_TCX_INGRESS);
1393	1393	test_tc_opts_dev_cleanup_target(BPF_TCX_EGRESS);
@@ -1563,7 +1563,7 @@ cleanup: · 169 unmodified lines
1563	1563	assert_mprog_count(target, 0);
1564	1564	}
1565	1565
1566		−void serial_test_tc_opts_mixed(void)
	1566	+void test_ns_tc_opts_mixed(void)
1567	1567	{
1568	1568	test_tc_opts_mixed_target(BPF_TCX_INGRESS);
1569	1569	test_tc_opts_mixed_target(BPF_TCX_EGRESS);
@@ -1642,7 +1642,7 @@ cleanup: · 72 unmodified lines
1642	1642	assert_mprog_count(target, 0);
1643	1643	}
1644	1644
1645		−void serial_test_tc_opts_demixed(void)
	1645	+void test_ns_tc_opts_demixed(void)
1646	1646	{
1647	1647	test_tc_opts_demixed_target(BPF_TCX_INGRESS);
1648	1648	test_tc_opts_demixed_target(BPF_TCX_EGRESS);
@@ -1813,7 +1813,7 @@ cleanup: · 164 unmodified lines
1813	1813	test_tc_link__destroy(skel);
1814	1814	}
1815	1815
1816		−void serial_test_tc_opts_detach(void)
	1816	+void test_ns_tc_opts_detach(void)
1817	1817	{
1818	1818	test_tc_opts_detach_target(BPF_TCX_INGRESS);
1819	1819	test_tc_opts_detach_target(BPF_TCX_EGRESS);
@@ -2020,7 +2020,7 @@ cleanup: · 200 unmodified lines
2020	2020	test_tc_link__destroy(skel);
2021	2021	}
2022	2022
2023		−void serial_test_tc_opts_detach_before(void)
	2023	+void test_ns_tc_opts_detach_before(void)
2024	2024	{
2025	2025	test_tc_opts_detach_before_target(BPF_TCX_INGRESS);
2026	2026	test_tc_opts_detach_before_target(BPF_TCX_EGRESS);
@@ -2236,7 +2236,7 @@ cleanup: · 209 unmodified lines
2236	2236	test_tc_link__destroy(skel);
2237	2237	}
2238	2238
2239		−void serial_test_tc_opts_detach_after(void)
	2239	+void test_ns_tc_opts_detach_after(void)
2240	2240	{
2241	2241	test_tc_opts_detach_after_target(BPF_TCX_INGRESS);
2242	2242	test_tc_opts_detach_after_target(BPF_TCX_EGRESS);
@@ -2265,7 +2265,7 @@ static void test_tc_opts_delete_empty(int target, bool chain_tc_old) · 22 unmodified lines
2265	2265	assert_mprog_count(target, 0);
2266	2266	}
2267	2267
2268		−void serial_test_tc_opts_delete_empty(void)
	2268	+void test_ns_tc_opts_delete_empty(void)
2269	2269	{
2270	2270	test_tc_opts_delete_empty(BPF_TCX_INGRESS, false);
2271	2271	test_tc_opts_delete_empty(BPF_TCX_EGRESS, false);
@@ -2372,7 +2372,7 @@ cleanup: · 100 unmodified lines
2372	2372	test_tc_link__destroy(skel);
2373	2373	}
2374	2374
2375		−void serial_test_tc_opts_chain_mixed(void)
	2375	+void test_ns_tc_opts_chain_mixed(void)
2376	2376	{
2377	2377	test_tc_chain_mixed(BPF_TCX_INGRESS);
2378	2378	test_tc_chain_mixed(BPF_TCX_EGRESS);
@@ -2446,7 +2446,7 @@ cleanup: · 67 unmodified lines
2446	2446	ASSERT_EQ(if_nametoindex("tcx_opts2"), 0, "dev2_removed");
2447	2447	}
2448	2448
2449		−void serial_test_tc_opts_max(void)
	2449	+void test_ns_tc_opts_max(void)
2450	2450	{
2451	2451	test_tc_opts_max_target(BPF_TCX_INGRESS, 0, false);
2452	2452	test_tc_opts_max_target(BPF_TCX_EGRESS, 0, false);
@@ -2748,7 +2748,7 @@ cleanup: · 295 unmodified lines
2748	2748	test_tc_link__destroy(skel);
2749	2749	}
2750	2750
2751		−void serial_test_tc_opts_query(void)
	2751	+void test_ns_tc_opts_query(void)
2752	2752	{
2753	2753	test_tc_opts_query_target(BPF_TCX_INGRESS);
2754	2754	test_tc_opts_query_target(BPF_TCX_EGRESS);
@@ -2807,7 +2807,7 @@ cleanup: · 52 unmodified lines
2807	2807	test_tc_link__destroy(skel);
2808	2808	}
2809	2809
2810		−void serial_test_tc_opts_query_attach(void)
	2810	+void test_ns_tc_opts_query_attach(void)
2811	2811	{
2812	2812	test_tc_opts_query_attach_target(BPF_TCX_INGRESS);
2813	2813	test_tc_opts_query_attach_target(BPF_TCX_EGRESS);

tools/testing/selftests/bpf/prog_tests/test_struct_ops_kptr_return.c+16−0

@@ -0,0 +1,16 @@
	1	+#include <test_progs.h>
	2	+
	3	+#include "struct_ops_kptr_return.skel.h"
	4	+#include "struct_ops_kptr_return_fail__wrong_type.skel.h"
	5	+#include "struct_ops_kptr_return_fail__invalid_scalar.skel.h"
	6	+#include "struct_ops_kptr_return_fail__nonzero_offset.skel.h"
	7	+#include "struct_ops_kptr_return_fail__local_kptr.skel.h"
	8	+
	9	+void test_struct_ops_kptr_return(void)
	10	+{
	11	+ RUN_TESTS(struct_ops_kptr_return);
	12	+ RUN_TESTS(struct_ops_kptr_return_fail__wrong_type);
	13	+ RUN_TESTS(struct_ops_kptr_return_fail__invalid_scalar);
	14	+ RUN_TESTS(struct_ops_kptr_return_fail__nonzero_offset);
	15	+ RUN_TESTS(struct_ops_kptr_return_fail__local_kptr);
	16	+}

tools/testing/selftests/bpf/prog_tests/test_struct_ops_refcounted.c+14−0

@@ -0,0 +1,14 @@
	1	+#include <test_progs.h>
	2	+
	3	+#include "struct_ops_refcounted.skel.h"
	4	+#include "struct_ops_refcounted_fail__ref_leak.skel.h"
	5	+#include "struct_ops_refcounted_fail__global_subprog.skel.h"
	6	+#include "struct_ops_refcounted_fail__tail_call.skel.h"
	7	+
	8	+void test_struct_ops_refcounted(void)
	9	+{
	10	+ RUN_TESTS(struct_ops_refcounted);
	11	+ RUN_TESTS(struct_ops_refcounted_fail__ref_leak);
	12	+ RUN_TESTS(struct_ops_refcounted_fail__global_subprog);
	13	+ RUN_TESTS(struct_ops_refcounted_fail__tail_call);
	14	+}

tools/testing/selftests/bpf/prog_tests/test_tunnel.c+530−103

@@ -71,6 +71,8 @@ · 70 unmodified lines
71	71	#define IP4_ADDR2_VETH1 "172.16.1.20"
72	72	#define IP4_ADDR_TUNL_DEV0 "10.1.1.100"
73	73	#define IP4_ADDR_TUNL_DEV1 "10.1.1.200"
	74	+#define IP6_ADDR_TUNL_DEV0 "fc80::100"
	75	+#define IP6_ADDR_TUNL_DEV1 "fc80::200"
74	76
75	77	#define IP6_ADDR_VETH0 "::11"
76	78	#define IP6_ADDR1_VETH1 "::22"
@@ -98,6 +100,27 @@ · 21 unmodified lines
98	100	#define XFRM_SPI_IN_TO_OUT 0x1
99	101	#define XFRM_SPI_OUT_TO_IN 0x2
100	102
	103	+#define GRE_TUNL_DEV0 "gre00"
	104	+#define GRE_TUNL_DEV1 "gre11"
	105	+
	106	+#define IP6GRE_TUNL_DEV0 "ip6gre00"
	107	+#define IP6GRE_TUNL_DEV1 "ip6gre11"
	108	+
	109	+#define ERSPAN_TUNL_DEV0 "erspan00"
	110	+#define ERSPAN_TUNL_DEV1 "erspan11"
	111	+
	112	+#define IP6ERSPAN_TUNL_DEV0 "ip6erspan00"
	113	+#define IP6ERSPAN_TUNL_DEV1 "ip6erspan11"
	114	+
	115	+#define GENEVE_TUNL_DEV0 "geneve00"
	116	+#define GENEVE_TUNL_DEV1 "geneve11"
	117	+
	118	+#define IP6GENEVE_TUNL_DEV0 "ip6geneve00"
	119	+#define IP6GENEVE_TUNL_DEV1 "ip6geneve11"
	120	+
	121	+#define IP6TNL_TUNL_DEV0 "ip6tnl00"
	122	+#define IP6TNL_TUNL_DEV1 "ip6tnl11"
	123	+
101	124	#define PING_ARGS "-i 0.01 -c 3 -w 10 -q"
102	125
103	126	static int config_device(void)
@@ -216,6 +239,18 @@ fail: · 112 unmodified lines
216	239	return -1;
217	240	}
218	241
	242	+static int set_ipv4_addr(const char dev0, const char dev1)
	243	+{
	244	+ SYS(fail, "ip -n at_ns0 link set dev %s up", dev0);
	245	+ SYS(fail, "ip -n at_ns0 addr add dev %s %s/24", dev0, IP4_ADDR_TUNL_DEV0);
	246	+ SYS(fail, "ip link set dev %s up", dev1);
	247	+ SYS(fail, "ip addr add dev %s %s/24", dev1, IP4_ADDR_TUNL_DEV1);
	248	+
	249	+ return 0;
	250	+fail:
	251	+ return 1;
	252	+}
	253	+
219	254	static int add_ipip_tunnel(enum ipip_encap encap)
220	255	{
221	256	int err;
@@ -356,6 +391,99 @@ static void delete_xfrm_tunnel(void) · 134 unmodified lines
356	391	IP4_ADDR1_VETH1, IP4_ADDR_VETH0, XFRM_SPI_OUT_TO_IN);
357	392	}
358	393
	394	+static int add_ipv4_tunnel(const char dev0, const char dev1,
	395	+ const char type, const char opt)
	396	+{
	397	+ if (!type \|\| !opt \|\| !dev0 \|\| !dev1)
	398	+ return -1;
	399	+
	400	+ SYS(fail, "ip -n at_ns0 link add dev %s type %s %s local %s remote %s",
	401	+ dev0, type, opt, IP4_ADDR_VETH0, IP4_ADDR1_VETH1);
	402	+
	403	+ SYS(fail, "ip link add dev %s type %s external", dev1, type);
	404	+
	405	+ return set_ipv4_addr(dev0, dev1);
	406	+fail:
	407	+ return -1;
	408	+}
	409	+
	410	+static void delete_tunnel(const char dev0, const char dev1)
	411	+{
	412	+ if (!dev0 \|\| !dev1)
	413	+ return;
	414	+
	415	+ SYS_NOFAIL("ip netns exec at_ns0 ip link delete dev %s", dev0);
	416	+ SYS_NOFAIL("ip link delete dev %s", dev1);
	417	+}
	418	+
	419	+static int set_ipv6_addr(const char dev0, const char dev1)
	420	+{
	421	+ /* disable IPv6 DAD because it might take too long and fail tests */
	422	+ SYS(fail, "ip -n at_ns0 addr add %s/96 dev veth0 nodad", IP6_ADDR_VETH0);
	423	+ SYS(fail, "ip -n at_ns0 link set dev veth0 up");
	424	+ SYS(fail, "ip addr add %s/96 dev veth1 nodad", IP6_ADDR1_VETH1);
	425	+ SYS(fail, "ip link set dev veth1 up");
	426	+
	427	+ SYS(fail, "ip -n at_ns0 addr add dev %s %s/24", dev0, IP4_ADDR_TUNL_DEV0);
	428	+ SYS(fail, "ip -n at_ns0 addr add dev %s %s/96 nodad", dev0, IP6_ADDR_TUNL_DEV0);
	429	+ SYS(fail, "ip -n at_ns0 link set dev %s up", dev0);
	430	+
	431	+ SYS(fail, "ip addr add dev %s %s/24", dev1, IP4_ADDR_TUNL_DEV1);
	432	+ SYS(fail, "ip addr add dev %s %s/96 nodad", dev1, IP6_ADDR_TUNL_DEV1);
	433	+ SYS(fail, "ip link set dev %s up", dev1);
	434	+ return 0;
	435	+fail:
	436	+ return 1;
	437	+}
	438	+
	439	+static int add_ipv6_tunnel(const char dev0, const char dev1,
	440	+ const char type, const char opt)
	441	+{
	442	+ if (!type \|\| !opt \|\| !dev0 \|\| !dev1)
	443	+ return -1;
	444	+
	445	+ SYS(fail, "ip -n at_ns0 link add dev %s type %s %s local %s remote %s",
	446	+ dev0, type, opt, IP6_ADDR_VETH0, IP6_ADDR1_VETH1);
	447	+
	448	+ SYS(fail, "ip link add dev %s type %s external", dev1, type);
	449	+
	450	+ return set_ipv6_addr(dev0, dev1);
	451	+fail:
	452	+ return -1;
	453	+}
	454	+
	455	+static int add_geneve_tunnel(const char dev0, const char dev1,
	456	+ const char type, const char opt)
	457	+{
	458	+ if (!type \|\| !opt \|\| !dev0 \|\| !dev1)
	459	+ return -1;
	460	+
	461	+ SYS(fail, "ip -n at_ns0 link add dev %s type %s id 2 %s remote %s",
	462	+ dev0, type, opt, IP4_ADDR1_VETH1);
	463	+
	464	+ SYS(fail, "ip link add dev %s type %s %s external", dev1, type, opt);
	465	+
	466	+ return set_ipv4_addr(dev0, dev1);
	467	+fail:
	468	+ return -1;
	469	+}
	470	+
	471	+static int add_ip6geneve_tunnel(const char dev0, const char dev1,
	472	+ const char type, const char opt)
	473	+{
	474	+ if (!type \|\| !opt \|\| !dev0 \|\| !dev1)
	475	+ return -1;
	476	+
	477	+ SYS(fail, "ip -n at_ns0 link add dev %s type %s id 22 %s remote %s",
	478	+ dev0, type, opt, IP6_ADDR1_VETH1);
	479	+
	480	+ SYS(fail, "ip link add dev %s type %s %s external", dev1, type, opt);
	481	+
	482	+ return set_ipv6_addr(dev0, dev1);
	483	+fail:
	484	+ return -1;
	485	+}
	486	+
359	487	static int test_ping(int family, const char *addr)
360	488	{
361	489	SYS(fail, "%s %s %s > /dev/null", ping_command(family), PING_ARGS, addr);
@@ -364,32 +492,76 @@ fail: · 2 unmodified lines
364	492	return -1;
365	493	}
366	494
367		−static int attach_tc_prog(struct bpf_tc_hook *hook, int igr_fd, int egr_fd)
	495	+static void ping_dev0(void)
368	496	{
	497	+ /* ping from root namespace test */
	498	+ test_ping(AF_INET, IP4_ADDR_TUNL_DEV0);
	499	+}
	500	+
	501	+static void ping_dev1(void)
	502	+{
	503	+ struct nstoken *nstoken;
	504	+
	505	+ /* ping from at_ns0 namespace test */
	506	+ nstoken = open_netns("at_ns0");
	507	+ if (!ASSERT_OK_PTR(nstoken, "setns"))
	508	+ return;
	509	+
	510	+ test_ping(AF_INET, IP4_ADDR_TUNL_DEV1);
	511	+ close_netns(nstoken);
	512	+}
	513	+
	514	+static void ping6_veth0(void)
	515	+{
	516	+ test_ping(AF_INET6, IP6_ADDR_VETH0);
	517	+}
	518	+
	519	+static void ping6_dev0(void)
	520	+{
	521	+ test_ping(AF_INET6, IP6_ADDR_TUNL_DEV0);
	522	+}
	523	+
	524	+static void ping6_dev1(void)
	525	+{
	526	+ struct nstoken *nstoken;
	527	+
	528	+ /* ping from at_ns0 namespace test */
	529	+ nstoken = open_netns("at_ns0");
	530	+ if (!ASSERT_OK_PTR(nstoken, "setns"))
	531	+ return;
	532	+
	533	+ test_ping(AF_INET, IP6_ADDR_TUNL_DEV1);
	534	+ close_netns(nstoken);
	535	+}
	536	+
	537	+static int attach_tc_prog(int ifindex, int igr_fd, int egr_fd)
	538	+{
	539	+ DECLARE_LIBBPF_OPTS(bpf_tc_hook, hook, .ifindex = ifindex,
	540	+ .attach_point = BPF_TC_INGRESS \| BPF_TC_EGRESS);
369	541	DECLARE_LIBBPF_OPTS(bpf_tc_opts, opts1, .handle = 1,
370	542	.priority = 1, .prog_fd = igr_fd);
371	543	DECLARE_LIBBPF_OPTS(bpf_tc_opts, opts2, .handle = 1,
372	544	.priority = 1, .prog_fd = egr_fd);
373	545	int ret;
374	546
375		− ret = bpf_tc_hook_create(hook);
	547	+ ret = bpf_tc_hook_create(&hook);
376	548	if (!ASSERT_OK(ret, "create tc hook"))
377	549	return ret;
378	550
379	551	if (igr_fd >= 0) {
380		− hook->attach_point = BPF_TC_INGRESS;
381		− ret = bpf_tc_attach(hook, &opts1);
	552	+ hook.attach_point = BPF_TC_INGRESS;
	553	+ ret = bpf_tc_attach(&hook, &opts1);
382	554	if (!ASSERT_OK(ret, "bpf_tc_attach")) {
383		− bpf_tc_hook_destroy(hook);
	555	+ bpf_tc_hook_destroy(&hook);
384	556	return ret;
385	557	}
386	558	}
387	559
388	560	if (egr_fd >= 0) {
389		− hook->attach_point = BPF_TC_EGRESS;
390		− ret = bpf_tc_attach(hook, &opts2);
	561	+ hook.attach_point = BPF_TC_EGRESS;
	562	+ ret = bpf_tc_attach(&hook, &opts2);
391	563	if (!ASSERT_OK(ret, "bpf_tc_attach")) {
392		− bpf_tc_hook_destroy(hook);
	564	+ bpf_tc_hook_destroy(&hook);
393	565	return ret;
394	566	}
395	567	}
@@ -397,6 +569,50 @@ static int attach_tc_prog(struct bpf_tc_hook *hook, int igr_fd, int egr_fd) · 1 unmodified line
397	569	return 0;
398	570	}
399	571
	572	+static int generic_attach(const char *dev, int igr_fd, int egr_fd)
	573	+{
	574	+ int ifindex;
	575	+
	576	+ if (!ASSERT_OK_FD(igr_fd, "check ingress fd"))
	577	+ return -1;
	578	+ if (!ASSERT_OK_FD(egr_fd, "check egress fd"))
	579	+ return -1;
	580	+
	581	+ ifindex = if_nametoindex(dev);
	582	+ if (!ASSERT_NEQ(ifindex, 0, "get ifindex"))
	583	+ return -1;
	584	+
	585	+ return attach_tc_prog(ifindex, igr_fd, egr_fd);
	586	+}
	587	+
	588	+static int generic_attach_igr(const char *dev, int igr_fd)
	589	+{
	590	+ int ifindex;
	591	+
	592	+ if (!ASSERT_OK_FD(igr_fd, "check ingress fd"))
	593	+ return -1;
	594	+
	595	+ ifindex = if_nametoindex(dev);
	596	+ if (!ASSERT_NEQ(ifindex, 0, "get ifindex"))
	597	+ return -1;
	598	+
	599	+ return attach_tc_prog(ifindex, igr_fd, -1);
	600	+}
	601	+
	602	+static int generic_attach_egr(const char *dev, int egr_fd)
	603	+{
	604	+ int ifindex;
	605	+
	606	+ if (!ASSERT_OK_FD(egr_fd, "check egress fd"))
	607	+ return -1;
	608	+
	609	+ ifindex = if_nametoindex(dev);
	610	+ if (!ASSERT_NEQ(ifindex, 0, "get ifindex"))
	611	+ return -1;
	612	+
	613	+ return attach_tc_prog(ifindex, -1, egr_fd);
	614	+}
	615	+
400	616	static void test_vxlan_tunnel(void)
401	617	{
402	618	struct test_tunnel_kern *skel = NULL;
@@ -404,11 +620,9 @@ static void test_vxlan_tunnel(void) · 1 unmodified line
404	620	int local_ip_map_fd = -1;
405	621	int set_src_prog_fd, get_src_prog_fd;
406	622	int set_dst_prog_fd;
407		− int key = 0, ifindex = -1;
	623	+ int key = 0;
408	624	uint local_ip;
409	625	int err;
410		− DECLARE_LIBBPF_OPTS(bpf_tc_hook, tc_hook,
411		− .attach_point = BPF_TC_INGRESS);
412	626
413	627	/* add vxlan tunnel */
414	628	err = add_vxlan_tunnel();
@@ -419,42 +633,22 @@ static void test_vxlan_tunnel(void) · 4 unmodified lines
419	633	skel = test_tunnel_kern__open_and_load();
420	634	if (!ASSERT_OK_PTR(skel, "test_tunnel_kern__open_and_load"))
421	635	goto done;
422		− ifindex = if_nametoindex(VXLAN_TUNL_DEV1);
423		− if (!ASSERT_NEQ(ifindex, 0, "vxlan11 ifindex"))
424		− goto done;
425		− tc_hook.ifindex = ifindex;
426	636	get_src_prog_fd = bpf_program__fd(skel->progs.vxlan_get_tunnel_src);
427	637	set_src_prog_fd = bpf_program__fd(skel->progs.vxlan_set_tunnel_src);
428		− if (!ASSERT_GE(get_src_prog_fd, 0, "bpf_program__fd"))
429		− goto done;
430		− if (!ASSERT_GE(set_src_prog_fd, 0, "bpf_program__fd"))
431		− goto done;
432		− if (attach_tc_prog(&tc_hook, get_src_prog_fd, set_src_prog_fd))
	638	+ if (generic_attach(VXLAN_TUNL_DEV1, get_src_prog_fd, set_src_prog_fd))
433	639	goto done;
434	640
435	641	/* load and attach bpf prog to veth dev tc hook point */
436		− ifindex = if_nametoindex("veth1");
437		− if (!ASSERT_NEQ(ifindex, 0, "veth1 ifindex"))
438		− goto done;
439		− tc_hook.ifindex = ifindex;
440	642	set_dst_prog_fd = bpf_program__fd(skel->progs.veth_set_outer_dst);
441		− if (!ASSERT_GE(set_dst_prog_fd, 0, "bpf_program__fd"))
442		− goto done;
443		− if (attach_tc_prog(&tc_hook, set_dst_prog_fd, -1))
	643	+ if (generic_attach_igr("veth1", set_dst_prog_fd))
444	644	goto done;
445	645
446	646	/* load and attach prog set_md to tunnel dev tc hook point at_ns0 */
447	647	nstoken = open_netns("at_ns0");
448	648	if (!ASSERT_OK_PTR(nstoken, "setns src"))
449	649	goto done;
450		− ifindex = if_nametoindex(VXLAN_TUNL_DEV0);
451		− if (!ASSERT_NEQ(ifindex, 0, "vxlan00 ifindex"))
452		− goto done;
453		− tc_hook.ifindex = ifindex;
454	650	set_dst_prog_fd = bpf_program__fd(skel->progs.vxlan_set_tunnel_dst);
455		− if (!ASSERT_GE(set_dst_prog_fd, 0, "bpf_program__fd"))
456		− goto done;
457		− if (attach_tc_prog(&tc_hook, -1, set_dst_prog_fd))
	651	+ if (generic_attach_egr(VXLAN_TUNL_DEV0, set_dst_prog_fd))
458	652	goto done;
459	653	close_netns(nstoken);
460	654
@@ -468,9 +662,7 @@ static void test_vxlan_tunnel(void) · 7 unmodified lines
468	662	goto done;
469	663
470	664	/* ping test */
471		− err = test_ping(AF_INET, IP4_ADDR_TUNL_DEV0);
472		− if (!ASSERT_OK(err, "test_ping"))
473		− goto done;
	665	+ ping_dev0();
474	666
475	667	done:
476	668	/* delete vxlan tunnel */
@@ -488,11 +680,9 @@ static void test_ip6vxlan_tunnel(void) · 11 unmodified lines
488	680	int local_ip_map_fd = -1;
489	681	int set_src_prog_fd, get_src_prog_fd;
490	682	int set_dst_prog_fd;
491		− int key = 0, ifindex = -1;
	683	+ int key = 0;
492	684	uint local_ip;
493	685	int err;
494		− DECLARE_LIBBPF_OPTS(bpf_tc_hook, tc_hook,
495		− .attach_point = BPF_TC_INGRESS);
496	686
497	687	/* add vxlan tunnel */
498	688	err = add_ip6vxlan_tunnel();
@@ -503,31 +693,17 @@ static void test_ip6vxlan_tunnel(void) · 4 unmodified lines
503	693	skel = test_tunnel_kern__open_and_load();
504	694	if (!ASSERT_OK_PTR(skel, "test_tunnel_kern__open_and_load"))
505	695	goto done;
506		− ifindex = if_nametoindex(IP6VXLAN_TUNL_DEV1);
507		− if (!ASSERT_NEQ(ifindex, 0, "ip6vxlan11 ifindex"))
508		− goto done;
509		− tc_hook.ifindex = ifindex;
510	696	get_src_prog_fd = bpf_program__fd(skel->progs.ip6vxlan_get_tunnel_src);
511	697	set_src_prog_fd = bpf_program__fd(skel->progs.ip6vxlan_set_tunnel_src);
512		− if (!ASSERT_GE(set_src_prog_fd, 0, "bpf_program__fd"))
513		− goto done;
514		− if (!ASSERT_GE(get_src_prog_fd, 0, "bpf_program__fd"))
515		− goto done;
516		− if (attach_tc_prog(&tc_hook, get_src_prog_fd, set_src_prog_fd))
	698	+ if (generic_attach(IP6VXLAN_TUNL_DEV1, get_src_prog_fd, set_src_prog_fd))
517	699	goto done;
518	700
519	701	/* load and attach prog set_md to tunnel dev tc hook point at_ns0 */
520	702	nstoken = open_netns("at_ns0");
521	703	if (!ASSERT_OK_PTR(nstoken, "setns src"))
522	704	goto done;
523		− ifindex = if_nametoindex(IP6VXLAN_TUNL_DEV0);
524		− if (!ASSERT_NEQ(ifindex, 0, "ip6vxlan00 ifindex"))
525		− goto done;
526		− tc_hook.ifindex = ifindex;
527	705	set_dst_prog_fd = bpf_program__fd(skel->progs.ip6vxlan_set_tunnel_dst);
528		− if (!ASSERT_GE(set_dst_prog_fd, 0, "bpf_program__fd"))
529		− goto done;
530		− if (attach_tc_prog(&tc_hook, -1, set_dst_prog_fd))
	706	+ if (generic_attach_egr(IP6VXLAN_TUNL_DEV0, set_dst_prog_fd))
531	707	goto done;
532	708	close_netns(nstoken);
533	709
@@ -541,9 +717,7 @@ static void test_ip6vxlan_tunnel(void) · 7 unmodified lines
541	717	goto done;
542	718
543	719	/* ping test */
544		− err = test_ping(AF_INET, IP4_ADDR_TUNL_DEV0);
545		− if (!ASSERT_OK(err, "test_ping"))
546		− goto done;
	720	+ ping_dev0();
547	721
548	722	done:
549	723	/* delete ipv6 vxlan tunnel */
@@ -557,12 +731,8 @@ done: · 7 unmodified lines
557	731	static void test_ipip_tunnel(enum ipip_encap encap)
558	732	{
559	733	struct test_tunnel_kern *skel = NULL;
560		− struct nstoken *nstoken;
561	734	int set_src_prog_fd, get_src_prog_fd;
562		− int ifindex = -1;
563	735	int err;
564		− DECLARE_LIBBPF_OPTS(bpf_tc_hook, tc_hook,
565		− .attach_point = BPF_TC_INGRESS);
566	736
567	737	/* add ipip tunnel */
568	738	err = add_ipip_tunnel(encap);
@@ -573,10 +743,6 @@ static void test_ipip_tunnel(enum ipip_encap encap) · 4 unmodified lines
573	743	skel = test_tunnel_kern__open_and_load();
574	744	if (!ASSERT_OK_PTR(skel, "test_tunnel_kern__open_and_load"))
575	745	goto done;
576		− ifindex = if_nametoindex(IPIP_TUNL_DEV1);
577		− if (!ASSERT_NEQ(ifindex, 0, "ipip11 ifindex"))
578		− goto done;
579		− tc_hook.ifindex = ifindex;
580	746
581	747	switch (encap) {
582	748	case FOU:
@@ -598,26 +764,11 @@ static void test_ipip_tunnel(enum ipip_encap encap) · 15 unmodified lines
598	764	skel->progs.ipip_set_tunnel);
599	765	}
600	766
601		− if (!ASSERT_GE(set_src_prog_fd, 0, "bpf_program__fd"))
602		− goto done;
603		− if (!ASSERT_GE(get_src_prog_fd, 0, "bpf_program__fd"))
604		− goto done;
605		− if (attach_tc_prog(&tc_hook, get_src_prog_fd, set_src_prog_fd))
	767	+ if (generic_attach(IPIP_TUNL_DEV1, get_src_prog_fd, set_src_prog_fd))
606	768	goto done;
607	769
608		− /* ping from root namespace test */
609		− err = test_ping(AF_INET, IP4_ADDR_TUNL_DEV0);
610		− if (!ASSERT_OK(err, "test_ping"))
611		− goto done;
612		−
613		− /* ping from at_ns0 namespace test */
614		− nstoken = open_netns("at_ns0");
615		− if (!ASSERT_OK_PTR(nstoken, "setns"))
616		− goto done;
617		− err = test_ping(AF_INET, IP4_ADDR_TUNL_DEV1);
618		− if (!ASSERT_OK(err, "test_ping"))
619		− goto done;
620		− close_netns(nstoken);
	770	+ ping_dev0();
	771	+ ping_dev1();
621	772
622	773	done:
623	774	/* delete ipip tunnel */
@@ -628,11 +779,8 @@ done: · 4 unmodified lines
628	779
629	780	static void test_xfrm_tunnel(void)
630	781	{
631		− DECLARE_LIBBPF_OPTS(bpf_tc_hook, tc_hook,
632		− .attach_point = BPF_TC_INGRESS);
633	782	LIBBPF_OPTS(bpf_xdp_attach_opts, opts);
634	783	struct test_tunnel_kern *skel = NULL;
635		− struct nstoken *nstoken;
636	784	int xdp_prog_fd;
637	785	int tc_prog_fd;
638	786	int ifindex;
@@ -646,19 +794,16 @@ static void test_xfrm_tunnel(void) · 7 unmodified lines
646	794	if (!ASSERT_OK_PTR(skel, "test_tunnel_kern__open_and_load"))
647	795	goto done;
648	796
649		− ifindex = if_nametoindex("veth1");
650		− if (!ASSERT_NEQ(ifindex, 0, "veth1 ifindex"))
651		− goto done;
652	797
653	798	/* attach tc prog to tunnel dev */
654		− tc_hook.ifindex = ifindex;
655	799	tc_prog_fd = bpf_program__fd(skel->progs.xfrm_get_state);
656		− if (!ASSERT_GE(tc_prog_fd, 0, "bpf_program__fd"))
657		− goto done;
658		− if (attach_tc_prog(&tc_hook, tc_prog_fd, -1))
	800	+ if (generic_attach_igr("veth1", tc_prog_fd))
659	801	goto done;
660	802
661	803	/* attach xdp prog to tunnel dev */
	804	+ ifindex = if_nametoindex("veth1");
	805	+ if (!ASSERT_NEQ(ifindex, 0, "veth1 ifindex"))
	806	+ goto done;
662	807	xdp_prog_fd = bpf_program__fd(skel->progs.xfrm_get_state_xdp);
663	808	if (!ASSERT_GE(xdp_prog_fd, 0, "bpf_program__fd"))
664	809	goto done;
@@ -666,14 +811,7 @@ static void test_xfrm_tunnel(void) · 1 unmodified line
666	811	if (!ASSERT_OK(err, "bpf_xdp_attach"))
667	812	goto done;
668	813
669		− /* ping from at_ns0 namespace test */
670		− nstoken = open_netns("at_ns0");
671		− if (!ASSERT_OK_PTR(nstoken, "setns"))
672		− goto done;
673		− err = test_ping(AF_INET, IP4_ADDR_TUNL_DEV1);
674		− close_netns(nstoken);
675		− if (!ASSERT_OK(err, "test_ping"))
676		− goto done;
	814	+ ping_dev1();
677	815
678	816	if (!ASSERT_EQ(skel->bss->xfrm_reqid, 1, "req_id"))
679	817	goto done;
@@ -690,6 +828,281 @@ done: · 10 unmodified lines
690	828	test_tunnel_kern__destroy(skel);
691	829	}
692	830
	831	+enum gre_test {
	832	+ GRE,
	833	+ GRE_NOKEY,
	834	+ GRETAP,
	835	+ GRETAP_NOKEY,
	836	+};
	837	+
	838	+static void test_gre_tunnel(enum gre_test test)
	839	+{
	840	+ struct test_tunnel_kern *skel;
	841	+ int set_fd, get_fd;
	842	+ int err;
	843	+
	844	+ skel = test_tunnel_kern__open_and_load();
	845	+ if (!ASSERT_OK_PTR(skel, "test_tunnel_kern__open_and_load"))
	846	+ return;
	847	+
	848	+ switch (test) {
	849	+ case GRE:
	850	+ err = add_ipv4_tunnel(GRE_TUNL_DEV0, GRE_TUNL_DEV1, "gre", "seq");
	851	+ set_fd = bpf_program__fd(skel->progs.gre_set_tunnel_no_key);
	852	+ get_fd = bpf_program__fd(skel->progs.gre_get_tunnel);
	853	+ break;
	854	+ case GRE_NOKEY:
	855	+ err = add_ipv4_tunnel(GRE_TUNL_DEV0, GRE_TUNL_DEV1, "gre", "seq key 2");
	856	+ set_fd = bpf_program__fd(skel->progs.gre_set_tunnel);
	857	+ get_fd = bpf_program__fd(skel->progs.gre_get_tunnel);
	858	+ break;
	859	+ case GRETAP:
	860	+ err = add_ipv4_tunnel(GRE_TUNL_DEV0, GRE_TUNL_DEV1, "gretap", "seq");
	861	+ set_fd = bpf_program__fd(skel->progs.gre_set_tunnel_no_key);
	862	+ get_fd = bpf_program__fd(skel->progs.gre_get_tunnel);
	863	+ break;
	864	+ case GRETAP_NOKEY:
	865	+ err = add_ipv4_tunnel(GRE_TUNL_DEV0, GRE_TUNL_DEV1, "gretap", "seq key 2");
	866	+ set_fd = bpf_program__fd(skel->progs.gre_set_tunnel);
	867	+ get_fd = bpf_program__fd(skel->progs.gre_get_tunnel);
	868	+ break;
	869	+ }
	870	+ if (!ASSERT_OK(err, "add tunnel"))
	871	+ goto done;
	872	+
	873	+ if (generic_attach(GRE_TUNL_DEV1, get_fd, set_fd))
	874	+ goto done;
	875	+
	876	+ ping_dev0();
	877	+ ping_dev1();
	878	+
	879	+done:
	880	+ delete_tunnel(GRE_TUNL_DEV0, GRE_TUNL_DEV1);
	881	+ test_tunnel_kern__destroy(skel);
	882	+}
	883	+
	884	+enum ip6gre_test {
	885	+ IP6GRE,
	886	+ IP6GRETAP
	887	+};
	888	+
	889	+static void test_ip6gre_tunnel(enum ip6gre_test test)
	890	+{
	891	+ struct test_tunnel_kern *skel;
	892	+ int set_fd, get_fd;
	893	+ int err;
	894	+
	895	+ skel = test_tunnel_kern__open_and_load();
	896	+ if (!ASSERT_OK_PTR(skel, "test_tunnel_kern__open_and_load"))
	897	+ return;
	898	+
	899	+ switch (test) {
	900	+ case IP6GRE:
	901	+ err = add_ipv6_tunnel(IP6GRE_TUNL_DEV0, IP6GRE_TUNL_DEV1,
	902	+ "ip6gre", "flowlabel 0xbcdef key 2");
	903	+ break;
	904	+ case IP6GRETAP:
	905	+ err = add_ipv6_tunnel(IP6GRE_TUNL_DEV0, IP6GRE_TUNL_DEV1,
	906	+ "ip6gretap", "flowlabel 0xbcdef key 2");
	907	+ break;
	908	+ }
	909	+ if (!ASSERT_OK(err, "add tunnel"))
	910	+ goto done;
	911	+
	912	+ set_fd = bpf_program__fd(skel->progs.ip6gretap_set_tunnel);
	913	+ get_fd = bpf_program__fd(skel->progs.ip6gretap_get_tunnel);
	914	+ if (generic_attach(IP6GRE_TUNL_DEV1, get_fd, set_fd))
	915	+ goto done;
	916	+
	917	+ ping6_veth0();
	918	+ ping6_dev1();
	919	+ ping_dev0();
	920	+ ping_dev1();
	921	+done:
	922	+ delete_tunnel(IP6GRE_TUNL_DEV0, IP6GRE_TUNL_DEV1);
	923	+ test_tunnel_kern__destroy(skel);
	924	+}
	925	+
	926	+enum erspan_test {
	927	+ V1,
	928	+ V2
	929	+};
	930	+
	931	+static void test_erspan_tunnel(enum erspan_test test)
	932	+{
	933	+ struct test_tunnel_kern *skel;
	934	+ int set_fd, get_fd;
	935	+ int err;
	936	+
	937	+ skel = test_tunnel_kern__open_and_load();
	938	+ if (!ASSERT_OK_PTR(skel, "test_tunnel_kern__open_and_load"))
	939	+ return;
	940	+
	941	+ switch (test) {
	942	+ case V1:
	943	+ err = add_ipv4_tunnel(ERSPAN_TUNL_DEV0, ERSPAN_TUNL_DEV1,
	944	+ "erspan", "seq key 2 erspan_ver 1 erspan 123");
	945	+ break;
	946	+ case V2:
	947	+ err = add_ipv4_tunnel(ERSPAN_TUNL_DEV0, ERSPAN_TUNL_DEV1,
	948	+ "erspan",
	949	+ "seq key 2 erspan_ver 2 erspan_dir egress erspan_hwid 3");
	950	+ break;
	951	+ }
	952	+ if (!ASSERT_OK(err, "add tunnel"))
	953	+ goto done;
	954	+
	955	+ set_fd = bpf_program__fd(skel->progs.erspan_set_tunnel);
	956	+ get_fd = bpf_program__fd(skel->progs.erspan_get_tunnel);
	957	+ if (generic_attach(ERSPAN_TUNL_DEV1, get_fd, set_fd))
	958	+ goto done;
	959	+
	960	+ ping_dev0();
	961	+ ping_dev1();
	962	+done:
	963	+ delete_tunnel(ERSPAN_TUNL_DEV0, ERSPAN_TUNL_DEV1);
	964	+ test_tunnel_kern__destroy(skel);
	965	+}
	966	+
	967	+static void test_ip6erspan_tunnel(enum erspan_test test)
	968	+{
	969	+ struct test_tunnel_kern *skel;
	970	+ int set_fd, get_fd;
	971	+ int err;
	972	+
	973	+ skel = test_tunnel_kern__open_and_load();
	974	+ if (!ASSERT_OK_PTR(skel, "test_tunnel_kern__open_and_load"))
	975	+ return;
	976	+
	977	+ switch (test) {
	978	+ case V1:
	979	+ err = add_ipv6_tunnel(IP6ERSPAN_TUNL_DEV0, IP6ERSPAN_TUNL_DEV1,
	980	+ "ip6erspan", "seq key 2 erspan_ver 1 erspan 123");
	981	+ break;
	982	+ case V2:
	983	+ err = add_ipv6_tunnel(IP6ERSPAN_TUNL_DEV0, IP6ERSPAN_TUNL_DEV1,
	984	+ "ip6erspan",
	985	+ "seq key 2 erspan_ver 2 erspan_dir egress erspan_hwid 7");
	986	+ break;
	987	+ }
	988	+ if (!ASSERT_OK(err, "add tunnel"))
	989	+ goto done;
	990	+
	991	+ set_fd = bpf_program__fd(skel->progs.ip4ip6erspan_set_tunnel);
	992	+ get_fd = bpf_program__fd(skel->progs.ip4ip6erspan_get_tunnel);
	993	+ if (generic_attach(IP6ERSPAN_TUNL_DEV1, get_fd, set_fd))
	994	+ goto done;
	995	+
	996	+ ping6_veth0();
	997	+ ping_dev1();
	998	+done:
	999	+ delete_tunnel(IP6ERSPAN_TUNL_DEV0, IP6ERSPAN_TUNL_DEV1);
	1000	+ test_tunnel_kern__destroy(skel);
	1001	+}
	1002	+
	1003	+static void test_geneve_tunnel(void)
	1004	+{
	1005	+ struct test_tunnel_kern *skel;
	1006	+ int set_fd, get_fd;
	1007	+ int err;
	1008	+
	1009	+ skel = test_tunnel_kern__open_and_load();
	1010	+ if (!ASSERT_OK_PTR(skel, "test_tunnel_kern__open_and_load"))
	1011	+ return;
	1012	+
	1013	+ err = add_geneve_tunnel(GENEVE_TUNL_DEV0, GENEVE_TUNL_DEV1,
	1014	+ "geneve", "dstport 6081");
	1015	+ if (!ASSERT_OK(err, "add tunnel"))
	1016	+ goto done;
	1017	+
	1018	+ set_fd = bpf_program__fd(skel->progs.geneve_set_tunnel);
	1019	+ get_fd = bpf_program__fd(skel->progs.geneve_get_tunnel);
	1020	+ if (generic_attach(GENEVE_TUNL_DEV1, get_fd, set_fd))
	1021	+ goto done;
	1022	+
	1023	+ ping_dev0();
	1024	+ ping_dev1();
	1025	+done:
	1026	+ delete_tunnel(GENEVE_TUNL_DEV0, GENEVE_TUNL_DEV1);
	1027	+ test_tunnel_kern__destroy(skel);
	1028	+}
	1029	+
	1030	+static void test_ip6geneve_tunnel(void)
	1031	+{
	1032	+ struct test_tunnel_kern *skel;
	1033	+ int set_fd, get_fd;
	1034	+ int err;
	1035	+
	1036	+ skel = test_tunnel_kern__open_and_load();
	1037	+ if (!ASSERT_OK_PTR(skel, "test_tunnel_kern__open_and_load"))
	1038	+ return;
	1039	+
	1040	+ err = add_ip6geneve_tunnel(IP6GENEVE_TUNL_DEV0, IP6GENEVE_TUNL_DEV1,
	1041	+ "geneve", "");
	1042	+ if (!ASSERT_OK(err, "add tunnel"))
	1043	+ goto done;
	1044	+
	1045	+ set_fd = bpf_program__fd(skel->progs.ip6geneve_set_tunnel);
	1046	+ get_fd = bpf_program__fd(skel->progs.ip6geneve_get_tunnel);
	1047	+ if (generic_attach(IP6GENEVE_TUNL_DEV1, get_fd, set_fd))
	1048	+ goto done;
	1049	+
	1050	+ ping_dev0();
	1051	+ ping_dev1();
	1052	+done:
	1053	+ delete_tunnel(IP6GENEVE_TUNL_DEV0, IP6GENEVE_TUNL_DEV1);
	1054	+ test_tunnel_kern__destroy(skel);
	1055	+}
	1056	+
	1057	+enum ip6tnl_test {
	1058	+ IPIP6,
	1059	+ IP6IP6
	1060	+};
	1061	+
	1062	+static void test_ip6tnl_tunnel(enum ip6tnl_test test)
	1063	+{
	1064	+ struct test_tunnel_kern *skel;
	1065	+ int set_fd, get_fd;
	1066	+ int err;
	1067	+
	1068	+ skel = test_tunnel_kern__open_and_load();
	1069	+ if (!ASSERT_OK_PTR(skel, "test_tunnel_kern__open_and_load"))
	1070	+ return;
	1071	+
	1072	+ err = add_ipv6_tunnel(IP6TNL_TUNL_DEV0, IP6TNL_TUNL_DEV1, "ip6tnl", "");
	1073	+ if (!ASSERT_OK(err, "add tunnel"))
	1074	+ goto done;
	1075	+
	1076	+ switch (test) {
	1077	+ case IPIP6:
	1078	+ set_fd = bpf_program__fd(skel->progs.ipip6_set_tunnel);
	1079	+ get_fd = bpf_program__fd(skel->progs.ipip6_get_tunnel);
	1080	+ break;
	1081	+ case IP6IP6:
	1082	+ set_fd = bpf_program__fd(skel->progs.ip6ip6_set_tunnel);
	1083	+ get_fd = bpf_program__fd(skel->progs.ip6ip6_get_tunnel);
	1084	+ break;
	1085	+ }
	1086	+ if (generic_attach(IP6TNL_TUNL_DEV1, get_fd, set_fd))
	1087	+ goto done;
	1088	+
	1089	+ ping6_veth0();
	1090	+ switch (test) {
	1091	+ case IPIP6:
	1092	+ ping_dev0();
	1093	+ ping_dev1();
	1094	+ break;
	1095	+ case IP6IP6:
	1096	+ ping6_dev0();
	1097	+ ping6_dev1();
	1098	+ break;
	1099	+ }
	1100	+
	1101	+done:
	1102	+ delete_tunnel(IP6TNL_TUNL_DEV0, IP6TNL_TUNL_DEV1);
	1103	+ test_tunnel_kern__destroy(skel);
	1104	+}
	1105	+
693	1106	#define RUN_TEST(name, ...) \
694	1107	({ \
695	1108	if (test__start_subtest(#name)) { \
@@ -707,6 +1120,20 @@ static void test_tunnel_run_tests(void arg) · 11 unmodified lines
707	1120	RUN_TEST(ipip_tunnel, FOU);
708	1121	RUN_TEST(ipip_tunnel, GUE);
709	1122	RUN_TEST(xfrm_tunnel);
	1123	+ RUN_TEST(gre_tunnel, GRE);
	1124	+ RUN_TEST(gre_tunnel, GRE_NOKEY);
	1125	+ RUN_TEST(gre_tunnel, GRETAP);
	1126	+ RUN_TEST(gre_tunnel, GRETAP_NOKEY);
	1127	+ RUN_TEST(ip6gre_tunnel, IP6GRE);
	1128	+ RUN_TEST(ip6gre_tunnel, IP6GRETAP);
	1129	+ RUN_TEST(erspan_tunnel, V1);
	1130	+ RUN_TEST(erspan_tunnel, V2);
	1131	+ RUN_TEST(ip6erspan_tunnel, V1);
	1132	+ RUN_TEST(ip6erspan_tunnel, V2);
	1133	+ RUN_TEST(geneve_tunnel);
	1134	+ RUN_TEST(ip6geneve_tunnel);
	1135	+ RUN_TEST(ip6tnl_tunnel, IPIP6);
	1136	+ RUN_TEST(ip6tnl_tunnel, IP6IP6);
710	1137
711	1138	return NULL;
712	1139	}

tools/testing/selftests/bpf/prog_tests/test_veristat.c+139−0

@@ -0,0 +1,139 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
	3	+#include <test_progs.h>
	4	+#include <string.h>
	5	+#include <stdio.h>
	6	+
	7	+#define __CHECK_STR(str, name) \
	8	+ do { \
	9	+ if (!ASSERT_HAS_SUBSTR(fix->output, (str), (name))) \
	10	+ goto out; \
	11	+ } while (0)
	12	+
	13	+struct fixture {
	14	+ char tmpfile[80];
	15	+ int fd;
	16	+ char *output;
	17	+ size_t sz;
	18	+ char veristat[80];
	19	+};
	20	+
	21	+static struct fixture *init_fixture(void)
	22	+{
	23	+ struct fixture *fix = malloc(sizeof(struct fixture));
	24	+
	25	+ /* for no_alu32 and cpuv4 veristat is in parent folder */
	26	+ if (access("./veristat", F_OK) == 0)
	27	+ strcpy(fix->veristat, "./veristat");
	28	+ else if (access("../veristat", F_OK) == 0)
	29	+ strcpy(fix->veristat, "../veristat");
	30	+ else
	31	+ PRINT_FAIL("Can't find veristat binary");
	32	+
	33	+ snprintf(fix->tmpfile, sizeof(fix->tmpfile), "/tmp/test_veristat.XXXXXX");
	34	+ fix->fd = mkstemp(fix->tmpfile);
	35	+ fix->sz = 1000000;
	36	+ fix->output = malloc(fix->sz);
	37	+ return fix;
	38	+}
	39	+
	40	+static void teardown_fixture(struct fixture *fix)
	41	+{
	42	+ free(fix->output);
	43	+ close(fix->fd);
	44	+ remove(fix->tmpfile);
	45	+ free(fix);
	46	+}
	47	+
	48	+static void test_set_global_vars_succeeds(void)
	49	+{
	50	+ struct fixture *fix = init_fixture();
	51	+
	52	+ SYS(out,
	53	+ "%s set_global_vars.bpf.o"\
	54	+ " -G \"var_s64 = 0xf000000000000001\" "\
	55	+ " -G \"var_u64 = 0xfedcba9876543210\" "\
	56	+ " -G \"var_s32 = -0x80000000\" "\
	57	+ " -G \"var_u32 = 0x76543210\" "\
	58	+ " -G \"var_s16 = -32768\" "\
	59	+ " -G \"var_u16 = 60652\" "\
	60	+ " -G \"var_s8 = -128\" "\
	61	+ " -G \"var_u8 = 255\" "\
	62	+ " -G \"var_ea = EA2\" "\
	63	+ " -G \"var_eb = EB2\" "\
	64	+ " -G \"var_ec = EC2\" "\
	65	+ " -G \"var_b = 1\" "\
	66	+ "-vl2 > %s", fix->veristat, fix->tmpfile);
	67	+
	68	+ read(fix->fd, fix->output, fix->sz);
	69	+ __CHECK_STR("_w=0xf000000000000001 ", "var_s64 = 0xf000000000000001");
	70	+ __CHECK_STR("_w=0xfedcba9876543210 ", "var_u64 = 0xfedcba9876543210");
	71	+ __CHECK_STR("_w=0x80000000 ", "var_s32 = -0x80000000");
	72	+ __CHECK_STR("_w=0x76543210 ", "var_u32 = 0x76543210");
	73	+ __CHECK_STR("_w=0x8000 ", "var_s16 = -32768");
	74	+ __CHECK_STR("_w=0xecec ", "var_u16 = 60652");
	75	+ __CHECK_STR("_w=128 ", "var_s8 = -128");
	76	+ __CHECK_STR("_w=255 ", "var_u8 = 255");
	77	+ __CHECK_STR("_w=11 ", "var_ea = EA2");
	78	+ __CHECK_STR("_w=12 ", "var_eb = EB2");
	79	+ __CHECK_STR("_w=13 ", "var_ec = EC2");
	80	+ __CHECK_STR("_w=1 ", "var_b = 1");
	81	+
	82	+out:
	83	+ teardown_fixture(fix);
	84	+}
	85	+
	86	+static void test_set_global_vars_from_file_succeeds(void)
	87	+{
	88	+ struct fixture *fix = init_fixture();
	89	+ char input_file[80];
	90	+ const char *vars = "var_s16 = -32768\nvar_u16 = 60652";
	91	+ int fd;
	92	+
	93	+ snprintf(input_file, sizeof(input_file), "/tmp/veristat_input.XXXXXX");
	94	+ fd = mkstemp(input_file);
	95	+ if (!ASSERT_GE(fd, 0, "valid fd"))
	96	+ goto out;
	97	+
	98	+ write(fd, vars, strlen(vars));
	99	+ syncfs(fd);
	100	+ SYS(out, "%s set_global_vars.bpf.o -G \"@%s\" -vl2 > %s",
	101	+ fix->veristat, input_file, fix->tmpfile);
	102	+ read(fix->fd, fix->output, fix->sz);
	103	+ __CHECK_STR("_w=0x8000 ", "var_s16 = -32768");
	104	+ __CHECK_STR("_w=0xecec ", "var_u16 = 60652");
	105	+
	106	+out:
	107	+ close(fd);
	108	+ remove(input_file);
	109	+ teardown_fixture(fix);
	110	+}
	111	+
	112	+static void test_set_global_vars_out_of_range(void)
	113	+{
	114	+ struct fixture *fix = init_fixture();
	115	+
	116	+ SYS_FAIL(out,
	117	+ "%s set_global_vars.bpf.o -G \"var_s32 = 2147483648\" -vl2 2> %s",
	118	+ fix->veristat, fix->tmpfile);
	119	+
	120	+ read(fix->fd, fix->output, fix->sz);
	121	+ __CHECK_STR("is out of range [-2147483648; 2147483647]", "out of range");
	122	+
	123	+out:
	124	+ teardown_fixture(fix);
	125	+}
	126	+
	127	+void test_veristat(void)
	128	+{
	129	+ if (test__start_subtest("set_global_vars_succeeds"))
	130	+ test_set_global_vars_succeeds();
	131	+
	132	+ if (test__start_subtest("set_global_vars_out_of_range"))
	133	+ test_set_global_vars_out_of_range();
	134	+
	135	+ if (test__start_subtest("set_global_vars_from_file_succeeds"))
	136	+ test_set_global_vars_from_file_succeeds();
	137	+}
	138	+
	139	+#undef __CHECK_STR

tools/testing/selftests/bpf/prog_tests/test_xdp_veth.c+512−126

@@ -3,17 +3,50 @@ · 2 unmodified lines
3	3	/* Create 3 namespaces with 3 veth peers, and forward packets in-between using
4	4	* native XDP
5	5	*
6		− * XDP_TX
7		− * NS1(veth11) NS2(veth22) NS3(veth33)
8		− * \| \| \|
9		− * \| \| \|
10		− * (veth1, (veth2, (veth3,
11		− * id:111) id:122) id:133)
12		− * ^ \| ^ \| ^ \|
13		− * \| \| XDP_REDIRECT \| \| XDP_REDIRECT \| \|
14		− * \| ------------------ ------------------ \|
15		− * -----------------------------------------
16		− * XDP_REDIRECT
	6	+ * Network topology:
	7	+ * ---------- ---------- ----------
	8	+ * \| NS1 \| \| NS2 \| \| NS3 \|
	9	+ * \| veth11 \| \| veth22 \| \| veth33 \|
	10	+ * ----\|----- -----\|---- -----\|----
	11	+ * \| \| \|
	12	+ * ----\|------------------\|----------------\|----
	13	+ * \| veth1 veth2 veth3 \|
	14	+ * \| \|
	15	+ * \| NSO \|
	16	+ * ---------------------------------------------
	17	+ *
	18	+ * Test cases:
	19	+ * - [test_xdp_veth_redirect] : ping veth33 from veth11
	20	+ *
	21	+ * veth11 veth22 veth33
	22	+ * (XDP_PASS) (XDP_TX) (XDP_PASS)
	23	+ * \| \| \|
	24	+ * \| \| \|
	25	+ * veth1 veth2 veth3
	26	+ * (XDP_REDIRECT) (XDP_REDIRECT) (XDP_REDIRECT)
	27	+ * ^ \| ^ \| ^ \|
	28	+ * \| \| \| \| \| \|
	29	+ * \| ------------------ ------------------ \|
	30	+ * -----------------------------------------
	31	+ *
	32	+ * - [test_xdp_veth_broadcast_redirect]: broadcast from veth11
	33	+ * - IPv4 ping : BPF_F_BROADCAST \| BPF_F_EXCLUDE_INGRESS
	34	+ * -> echo request received by all except veth11
	35	+ * - IPv4 ping : BPF_F_BROADCAST
	36	+ * -> echo request received by all veth
	37	+ * - [test_xdp_veth_egress]:
	38	+ * - all src mac should be the magic mac
	39	+ *
	40	+ * veth11 veth22 veth33
	41	+ * (XDP_PASS) (XDP_PASS) (XDP_PASS)
	42	+ * \| \| \|
	43	+ * \| \| \|
	44	+ * veth1 veth2 veth3
	45	+ * (XDP_REDIRECT) (XDP_REDIRECT) (XDP_REDIRECT)
	46	+ * \| ^ ^
	47	+ * \| \| \|
	48	+ * ----------------------------------------
	49	+ *
17	50	*/
18	51
19	52	#define _GNU_SOURCE
@@ -22,192 +55,545 @@ · 2 unmodified lines
22	55	#include "network_helpers.h"
23	56	#include "xdp_dummy.skel.h"
24	57	#include "xdp_redirect_map.skel.h"
	58	+#include "xdp_redirect_multi_kern.skel.h"
25	59	#include "xdp_tx.skel.h"
	60	+#include <uapi/linux/if_link.h>
26	61
27	62	#define VETH_PAIRS_COUNT 3
28		−#define NS_SUFFIX_LEN 6
29		−#define VETH_NAME_MAX_LEN 16
	63	+#define VETH_NAME_MAX_LEN 32
	64	+#define IP_MAX_LEN 16
30	65	#define IP_SRC "10.1.1.11"
31	66	#define IP_DST "10.1.1.33"
32		−#define IP_CMD_MAX_LEN 128
33		−
34		−struct skeletons {
35		− struct xdp_dummy *xdp_dummy;
36		− struct xdp_tx *xdp_tx;
37		− struct xdp_redirect_map *xdp_redirect_maps;
38		−};
	67	+#define IP_NEIGH "10.1.1.253"
	68	+#define PROG_NAME_MAX_LEN 128
	69	+#define NS_NAME_MAX_LEN 32
39	70
40	71	struct veth_configuration {
41	72	char local_veth[VETH_NAME_MAX_LEN]; /* Interface in main namespace */
42	73	char remote_veth[VETH_NAME_MAX_LEN]; /* Peer interface in dedicated namespace*/
43		− const char namespace; / Namespace for the remote veth */
44		− char next_veth[VETH_NAME_MAX_LEN]; /* Local interface to redirect traffic to */
45		− char remote_addr; / IP address of the remote veth */
	74	+ char namespace[NS_NAME_MAX_LEN]; /* Namespace for the remote veth */
	75	+ int next_veth; /* Local interface to redirect traffic to */
	76	+ char remote_addr[IP_MAX_LEN]; /* IP address of the remote veth */
46	77	};
47	78
48		−static struct veth_configuration config[VETH_PAIRS_COUNT] = {
49		− {
50		− .local_veth = "veth1",
51		− .remote_veth = "veth11",
52		− .next_veth = "veth2",
53		− .remote_addr = IP_SRC,
54		− .namespace = "ns-veth11"
55		− },
56		− {
57		− .local_veth = "veth2",
58		− .remote_veth = "veth22",
59		− .next_veth = "veth3",
60		− .remote_addr = NULL,
61		− .namespace = "ns-veth22"
62		− },
	79	+struct net_configuration {
	80	+ char ns0_name[NS_NAME_MAX_LEN];
	81	+ struct veth_configuration veth_cfg[VETH_PAIRS_COUNT];
	82	+};
	83	+
	84	+static const struct net_configuration default_config = {
	85	+ .ns0_name = "ns0-",
63	86	{
64		− .local_veth = "veth3",
65		− .remote_veth = "veth33",
66		− .next_veth = "veth1",
67		− .remote_addr = IP_DST,
68		− .namespace = "ns-veth33"
	87	+ {
	88	+ .local_veth = "veth1-",
	89	+ .remote_veth = "veth11",
	90	+ .next_veth = 1,
	91	+ .remote_addr = IP_SRC,
	92	+ .namespace = "ns-veth11-"
	93	+ },
	94	+ {
	95	+ .local_veth = "veth2-",
	96	+ .remote_veth = "veth22",
	97	+ .next_veth = 2,
	98	+ .remote_addr = "",
	99	+ .namespace = "ns-veth22-"
	100	+ },
	101	+ {
	102	+ .local_veth = "veth3-",
	103	+ .remote_veth = "veth33",
	104	+ .next_veth = 0,
	105	+ .remote_addr = IP_DST,
	106	+ .namespace = "ns-veth33-"
	107	+ }
69	108	}
70	109	};
71	110
72		−static int attach_programs_to_veth_pair(struct skeletons *skeletons, int index)
	111	+struct prog_configuration {
	112	+ char local_name[PROG_NAME_MAX_LEN]; /* BPF prog to attach to local_veth */
	113	+ char remote_name[PROG_NAME_MAX_LEN]; /* BPF prog to attach to remote_veth */
	114	+ u32 local_flags; /* XDP flags to use on local_veth */
	115	+ u32 remote_flags; /* XDP flags to use on remote_veth */
	116	+};
	117	+
	118	+static int attach_programs_to_veth_pair(struct bpf_object **objs, size_t nb_obj,
	119	+ struct net_configuration *net_config,
	120	+ struct prog_configuration *prog, int index)
73	121	{
74	122	struct bpf_program local_prog, remote_prog;
75		− struct bpf_link local_link, remote_link;
76	123	struct nstoken *nstoken;
77		− struct bpf_link *link;
78		− int interface;
79		−
80		− switch (index) {
81		− case 0:
82		− local_prog = skeletons->xdp_redirect_maps->progs.xdp_redirect_map_0;
83		− local_link = &skeletons->xdp_redirect_maps->links.xdp_redirect_map_0;
84		− remote_prog = skeletons->xdp_dummy->progs.xdp_dummy_prog;
85		− remote_link = &skeletons->xdp_dummy->links.xdp_dummy_prog;
86		− break;
87		− case 1:
88		− local_prog = skeletons->xdp_redirect_maps->progs.xdp_redirect_map_1;
89		− local_link = &skeletons->xdp_redirect_maps->links.xdp_redirect_map_1;
90		− remote_prog = skeletons->xdp_tx->progs.xdp_tx;
91		− remote_link = &skeletons->xdp_tx->links.xdp_tx;
92		− break;
93		− case 2:
94		− local_prog = skeletons->xdp_redirect_maps->progs.xdp_redirect_map_2;
95		− local_link = &skeletons->xdp_redirect_maps->links.xdp_redirect_map_2;
96		− remote_prog = skeletons->xdp_dummy->progs.xdp_dummy_prog;
97		− remote_link = &skeletons->xdp_dummy->links.xdp_dummy_prog;
98		− break;
	124	+ int interface, ret, i;
	125	+
	126	+ for (i = 0; i < nb_obj; i++) {
	127	+ local_prog = bpf_object__find_program_by_name(objs[i], prog[index].local_name);
	128	+ if (local_prog)
	129	+ break;
99	130	}
100		− interface = if_nametoindex(config[index].local_veth);
	131	+ if (!ASSERT_OK_PTR(local_prog, "find local program"))
	132	+ return -1;
	133	+
	134	+ for (i = 0; i < nb_obj; i++) {
	135	+ remote_prog = bpf_object__find_program_by_name(objs[i], prog[index].remote_name);
	136	+ if (remote_prog)
	137	+ break;
	138	+ }
	139	+ if (!ASSERT_OK_PTR(remote_prog, "find remote program"))
	140	+ return -1;
	141	+
	142	+ interface = if_nametoindex(net_config->veth_cfg[index].local_veth);
101	143	if (!ASSERT_NEQ(interface, 0, "non zero interface index"))
102	144	return -1;
103		− link = bpf_program__attach_xdp(local_prog, interface);
104		− if (!ASSERT_OK_PTR(link, "attach xdp program to local veth"))
	145	+
	146	+ ret = bpf_xdp_attach(interface, bpf_program__fd(local_prog),
	147	+ prog[index].local_flags, NULL);
	148	+ if (!ASSERT_OK(ret, "attach xdp program to local veth"))
105	149	return -1;
106		− *local_link = link;
107		− nstoken = open_netns(config[index].namespace);
	150	+
	151	+ nstoken = open_netns(net_config->veth_cfg[index].namespace);
108	152	if (!ASSERT_OK_PTR(nstoken, "switch to remote veth namespace"))
109	153	return -1;
110		− interface = if_nametoindex(config[index].remote_veth);
	154	+
	155	+ interface = if_nametoindex(net_config->veth_cfg[index].remote_veth);
111	156	if (!ASSERT_NEQ(interface, 0, "non zero interface index")) {
112	157	close_netns(nstoken);
113	158	return -1;
114	159	}
115		− link = bpf_program__attach_xdp(remote_prog, interface);
116		− *remote_link = link;
117		− close_netns(nstoken);
118		− if (!ASSERT_OK_PTR(link, "attach xdp program to remote veth"))
	160	+
	161	+ ret = bpf_xdp_attach(interface, bpf_program__fd(remote_prog),
	162	+ prog[index].remote_flags, NULL);
	163	+ if (!ASSERT_OK(ret, "attach xdp program to remote veth")) {
	164	+ close_netns(nstoken);
119	165	return -1;
	166	+ }
120	167
	168	+ close_netns(nstoken);
121	169	return 0;
122	170	}
123	171
124		−static int configure_network(struct skeletons *skeletons)
	172	+static int create_network(struct net_configuration *net_config)
125	173	{
126		− int interface_id;
127		− int map_fd;
128		− int err;
129		− int i = 0;
	174	+ struct nstoken *nstoken = NULL;
	175	+ int i, err;
	176	+
	177	+ memcpy(net_config, &default_config, sizeof(struct net_configuration));
	178	+
	179	+ /* Create unique namespaces */
	180	+ err = append_tid(net_config->ns0_name, NS_NAME_MAX_LEN);
	181	+ if (!ASSERT_OK(err, "append TID to ns0 name"))
	182	+ goto fail;
	183	+ SYS(fail, "ip netns add %s", net_config->ns0_name);
130	184
131		− /* First create and configure all interfaces */
132	185	for (i = 0; i < VETH_PAIRS_COUNT; i++) {
133		− SYS(fail, "ip netns add %s", config[i].namespace);
134		− SYS(fail, "ip link add %s type veth peer name %s netns %s",
135		− config[i].local_veth, config[i].remote_veth, config[i].namespace);
136		− SYS(fail, "ip link set dev %s up", config[i].local_veth);
137		− if (config[i].remote_addr)
138		− SYS(fail, "ip -n %s addr add %s/24 dev %s", config[i].namespace,
139		− config[i].remote_addr, config[i].remote_veth);
140		− SYS(fail, "ip -n %s link set dev %s up", config[i].namespace,
141		− config[i].remote_veth);
	186	+ err = append_tid(net_config->veth_cfg[i].namespace, NS_NAME_MAX_LEN);
	187	+ if (!ASSERT_OK(err, "append TID to ns name"))
	188	+ goto fail;
	189	+ SYS(fail, "ip netns add %s", net_config->veth_cfg[i].namespace);
142	190	}
143	191
144		− /* Then configure the redirect map and attach programs to interfaces */
145		− map_fd = bpf_map__fd(skeletons->xdp_redirect_maps->maps.tx_port);
146		− if (!ASSERT_GE(map_fd, 0, "open redirect map"))
	192	+ /* Create interfaces */
	193	+ nstoken = open_netns(net_config->ns0_name);
	194	+ if (!nstoken)
147	195	goto fail;
	196	+
148	197	for (i = 0; i < VETH_PAIRS_COUNT; i++) {
149		− interface_id = if_nametoindex(config[i].next_veth);
150		− if (!ASSERT_NEQ(interface_id, 0, "non zero interface index"))
151		− goto fail;
152		− err = bpf_map_update_elem(map_fd, &i, &interface_id, BPF_ANY);
153		− if (!ASSERT_OK(err, "configure interface redirection through map"))
154		− goto fail;
155		− if (attach_programs_to_veth_pair(skeletons, i))
156		− goto fail;
	198	+ SYS(fail, "ip link add %s type veth peer name %s netns %s",
	199	+ net_config->veth_cfg[i].local_veth, net_config->veth_cfg[i].remote_veth,
	200	+ net_config->veth_cfg[i].namespace);
	201	+ SYS(fail, "ip link set dev %s up", net_config->veth_cfg[i].local_veth);
	202	+ if (net_config->veth_cfg[i].remote_addr[0])
	203	+ SYS(fail, "ip -n %s addr add %s/24 dev %s",
	204	+ net_config->veth_cfg[i].namespace,
	205	+ net_config->veth_cfg[i].remote_addr,
	206	+ net_config->veth_cfg[i].remote_veth);
	207	+ SYS(fail, "ip -n %s link set dev %s up", net_config->veth_cfg[i].namespace,
	208	+ net_config->veth_cfg[i].remote_veth);
157	209	}
158	210
	211	+ close_netns(nstoken);
159	212	return 0;
160	213
161	214	fail:
	215	+ close_netns(nstoken);
162	216	return -1;
163	217	}
164	218
165		−static void cleanup_network(void)
	219	+static void cleanup_network(struct net_configuration *net_config)
166	220	{
167	221	int i;
168	222
169		− /* Deleting namespaces is enough to automatically remove veth pairs as well
170		− */
	223	+ SYS_NOFAIL("ip netns del %s", net_config->ns0_name);
171	224	for (i = 0; i < VETH_PAIRS_COUNT; i++)
172		− SYS_NOFAIL("ip netns del %s", config[i].namespace);
	225	+ SYS_NOFAIL("ip netns del %s", net_config->veth_cfg[i].namespace);
173	226	}
174	227
175		−static int check_ping(struct skeletons *skeletons)
	228	+#define VETH_REDIRECT_SKEL_NB 3
	229	+static void xdp_veth_redirect(u32 flags)
176	230	{
	231	+ struct prog_configuration ping_config[VETH_PAIRS_COUNT] = {
	232	+ {
	233	+ .local_name = "xdp_redirect_map_0",
	234	+ .remote_name = "xdp_dummy_prog",
	235	+ .local_flags = flags,
	236	+ .remote_flags = flags,
	237	+ },
	238	+ {
	239	+ .local_name = "xdp_redirect_map_1",
	240	+ .remote_name = "xdp_tx",
	241	+ .local_flags = flags,
	242	+ .remote_flags = flags,
	243	+ },
	244	+ {
	245	+ .local_name = "xdp_redirect_map_2",
	246	+ .remote_name = "xdp_dummy_prog",
	247	+ .local_flags = flags,
	248	+ .remote_flags = flags,
	249	+ }
	250	+ };
	251	+ struct bpf_object *bpf_objs[VETH_REDIRECT_SKEL_NB];
	252	+ struct xdp_redirect_map *xdp_redirect_map;
	253	+ struct net_configuration net_config;
	254	+ struct nstoken *nstoken = NULL;
	255	+ struct xdp_dummy *xdp_dummy;
	256	+ struct xdp_tx *xdp_tx;
	257	+ int map_fd;
	258	+ int i;
	259	+
	260	+ xdp_dummy = xdp_dummy__open_and_load();
	261	+ if (!ASSERT_OK_PTR(xdp_dummy, "xdp_dummy__open_and_load"))
	262	+ return;
	263	+
	264	+ xdp_tx = xdp_tx__open_and_load();
	265	+ if (!ASSERT_OK_PTR(xdp_tx, "xdp_tx__open_and_load"))
	266	+ goto destroy_xdp_dummy;
	267	+
	268	+ xdp_redirect_map = xdp_redirect_map__open_and_load();
	269	+ if (!ASSERT_OK_PTR(xdp_redirect_map, "xdp_redirect_map__open_and_load"))
	270	+ goto destroy_xdp_tx;
	271	+
	272	+ if (!ASSERT_OK(create_network(&net_config), "create network"))
	273	+ goto destroy_xdp_redirect_map;
	274	+
	275	+ /* Then configure the redirect map and attach programs to interfaces */
	276	+ map_fd = bpf_map__fd(xdp_redirect_map->maps.tx_port);
	277	+ if (!ASSERT_OK_FD(map_fd, "open redirect map"))
	278	+ goto destroy_xdp_redirect_map;
	279	+
	280	+ bpf_objs[0] = xdp_dummy->obj;
	281	+ bpf_objs[1] = xdp_tx->obj;
	282	+ bpf_objs[2] = xdp_redirect_map->obj;
	283	+
	284	+ nstoken = open_netns(net_config.ns0_name);
	285	+ if (!ASSERT_OK_PTR(nstoken, "open NS0"))
	286	+ goto destroy_xdp_redirect_map;
	287	+
	288	+ for (i = 0; i < VETH_PAIRS_COUNT; i++) {
	289	+ int next_veth = net_config.veth_cfg[i].next_veth;
	290	+ int interface_id;
	291	+ int err;
	292	+
	293	+ interface_id = if_nametoindex(net_config.veth_cfg[next_veth].local_veth);
	294	+ if (!ASSERT_NEQ(interface_id, 0, "non zero interface index"))
	295	+ goto destroy_xdp_redirect_map;
	296	+ err = bpf_map_update_elem(map_fd, &i, &interface_id, BPF_ANY);
	297	+ if (!ASSERT_OK(err, "configure interface redirection through map"))
	298	+ goto destroy_xdp_redirect_map;
	299	+ if (attach_programs_to_veth_pair(bpf_objs, VETH_REDIRECT_SKEL_NB,
	300	+ &net_config, ping_config, i))
	301	+ goto destroy_xdp_redirect_map;
	302	+ }
	303	+
177	304	/* Test: if all interfaces are properly configured, we must be able to ping
178	305	* veth33 from veth11
179	306	*/
180		− return SYS_NOFAIL("ip netns exec %s ping -c 1 -W 1 %s > /dev/null",
181		− config[0].namespace, IP_DST);
	307	+ ASSERT_OK(SYS_NOFAIL("ip netns exec %s ping -c 1 -W 1 %s > /dev/null",
	308	+ net_config.veth_cfg[0].namespace, IP_DST), "ping");
	309	+
	310	+destroy_xdp_redirect_map:
	311	+ close_netns(nstoken);
	312	+ xdp_redirect_map__destroy(xdp_redirect_map);
	313	+destroy_xdp_tx:
	314	+ xdp_tx__destroy(xdp_tx);
	315	+destroy_xdp_dummy:
	316	+ xdp_dummy__destroy(xdp_dummy);
	317	+
	318	+ cleanup_network(&net_config);
182	319	}
183	320
184		−void test_xdp_veth_redirect(void)
	321	+#define BROADCAST_REDIRECT_SKEL_NB 2
	322	+static void xdp_veth_broadcast_redirect(u32 attach_flags, u64 redirect_flags)
185	323	{
186		− struct skeletons skeletons = {};
	324	+ struct prog_configuration prog_cfg[VETH_PAIRS_COUNT] = {
	325	+ {
	326	+ .local_name = "xdp_redirect_map_multi_prog",
	327	+ .remote_name = "xdp_count_0",
	328	+ .local_flags = attach_flags,
	329	+ .remote_flags = attach_flags,
	330	+ },
	331	+ {
	332	+ .local_name = "xdp_redirect_map_multi_prog",
	333	+ .remote_name = "xdp_count_1",
	334	+ .local_flags = attach_flags,
	335	+ .remote_flags = attach_flags,
	336	+ },
	337	+ {
	338	+ .local_name = "xdp_redirect_map_multi_prog",
	339	+ .remote_name = "xdp_count_2",
	340	+ .local_flags = attach_flags,
	341	+ .remote_flags = attach_flags,
	342	+ }
	343	+ };
	344	+ struct bpf_object *bpf_objs[BROADCAST_REDIRECT_SKEL_NB];
	345	+ struct xdp_redirect_multi_kern *xdp_redirect_multi_kern;
	346	+ struct xdp_redirect_map *xdp_redirect_map;
	347	+ struct bpf_devmap_val devmap_val = {};
	348	+ struct net_configuration net_config;
	349	+ struct nstoken *nstoken = NULL;
	350	+ u16 protocol = ETH_P_IP;
	351	+ int group_map;
	352	+ int flags_map;
	353	+ int cnt_map;
	354	+ u64 cnt = 0;
	355	+ int i, err;
187	356
188		− skeletons.xdp_dummy = xdp_dummy__open_and_load();
189		− if (!ASSERT_OK_PTR(skeletons.xdp_dummy, "xdp_dummy__open_and_load"))
	357	+ xdp_redirect_multi_kern = xdp_redirect_multi_kern__open_and_load();
	358	+ if (!ASSERT_OK_PTR(xdp_redirect_multi_kern, "xdp_redirect_multi_kern__open_and_load"))
190	359	return;
191	360
192		− skeletons.xdp_tx = xdp_tx__open_and_load();
193		− if (!ASSERT_OK_PTR(skeletons.xdp_tx, "xdp_tx__open_and_load"))
	361	+ xdp_redirect_map = xdp_redirect_map__open_and_load();
	362	+ if (!ASSERT_OK_PTR(xdp_redirect_map, "xdp_redirect_map__open_and_load"))
	363	+ goto destroy_xdp_redirect_multi_kern;
	364	+
	365	+ if (!ASSERT_OK(create_network(&net_config), "create network"))
	366	+ goto destroy_xdp_redirect_map;
	367	+
	368	+ group_map = bpf_map__fd(xdp_redirect_multi_kern->maps.map_all);
	369	+ if (!ASSERT_OK_FD(group_map, "open map_all"))
	370	+ goto destroy_xdp_redirect_map;
	371	+
	372	+ flags_map = bpf_map__fd(xdp_redirect_multi_kern->maps.redirect_flags);
	373	+ if (!ASSERT_OK_FD(group_map, "open map_all"))
	374	+ goto destroy_xdp_redirect_map;
	375	+
	376	+ err = bpf_map_update_elem(flags_map, &protocol, &redirect_flags, BPF_NOEXIST);
	377	+ if (!ASSERT_OK(err, "init IP count"))
	378	+ goto destroy_xdp_redirect_map;
	379	+
	380	+ cnt_map = bpf_map__fd(xdp_redirect_map->maps.rxcnt);
	381	+ if (!ASSERT_OK_FD(cnt_map, "open rxcnt map"))
	382	+ goto destroy_xdp_redirect_map;
	383	+
	384	+ bpf_objs[0] = xdp_redirect_multi_kern->obj;
	385	+ bpf_objs[1] = xdp_redirect_map->obj;
	386	+
	387	+ nstoken = open_netns(net_config.ns0_name);
	388	+ if (!ASSERT_OK_PTR(nstoken, "open NS0"))
	389	+ goto destroy_xdp_redirect_map;
	390	+
	391	+ for (i = 0; i < VETH_PAIRS_COUNT; i++) {
	392	+ int ifindex = if_nametoindex(net_config.veth_cfg[i].local_veth);
	393	+
	394	+ if (attach_programs_to_veth_pair(bpf_objs, BROADCAST_REDIRECT_SKEL_NB,
	395	+ &net_config, prog_cfg, i))
	396	+ goto destroy_xdp_redirect_map;
	397	+
	398	+ SYS(destroy_xdp_redirect_map,
	399	+ "ip -n %s neigh add %s lladdr 00:00:00:00:00:01 dev %s",
	400	+ net_config.veth_cfg[i].namespace, IP_NEIGH, net_config.veth_cfg[i].remote_veth);
	401	+
	402	+ devmap_val.ifindex = ifindex;
	403	+ err = bpf_map_update_elem(group_map, &ifindex, &devmap_val, 0);
	404	+ if (!ASSERT_OK(err, "bpf_map_update_elem"))
	405	+ goto destroy_xdp_redirect_map;
	406	+
	407	+ }
	408	+
	409	+ SYS_NOFAIL("ip netns exec %s ping %s -i 0.1 -c 4 -W1 > /dev/null ",
	410	+ net_config.veth_cfg[0].namespace, IP_NEIGH);
	411	+
	412	+ for (i = 0; i < VETH_PAIRS_COUNT; i++) {
	413	+ err = bpf_map_lookup_elem(cnt_map, &i, &cnt);
	414	+ if (!ASSERT_OK(err, "get IP cnt"))
	415	+ goto destroy_xdp_redirect_map;
	416	+
	417	+ if (redirect_flags & BPF_F_EXCLUDE_INGRESS)
	418	+ /* veth11 shouldn't receive the ICMP requests;
	419	+ * others should
	420	+ */
	421	+ ASSERT_EQ(cnt, i ? 4 : 0, "compare IP cnt");
	422	+ else
	423	+ /* All remote veth should receive the ICMP requests */
	424	+ ASSERT_EQ(cnt, 4, "compare IP cnt");
	425	+ }
	426	+
	427	+destroy_xdp_redirect_map:
	428	+ close_netns(nstoken);
	429	+ xdp_redirect_map__destroy(xdp_redirect_map);
	430	+destroy_xdp_redirect_multi_kern:
	431	+ xdp_redirect_multi_kern__destroy(xdp_redirect_multi_kern);
	432	+
	433	+ cleanup_network(&net_config);
	434	+}
	435	+
	436	+#define VETH_EGRESS_SKEL_NB 3
	437	+static void xdp_veth_egress(u32 flags)
	438	+{
	439	+ struct prog_configuration prog_cfg[VETH_PAIRS_COUNT] = {
	440	+ {
	441	+ .local_name = "xdp_redirect_map_all_prog",
	442	+ .remote_name = "xdp_dummy_prog",
	443	+ .local_flags = flags,
	444	+ .remote_flags = flags,
	445	+ },
	446	+ {
	447	+ .local_name = "xdp_redirect_map_all_prog",
	448	+ .remote_name = "store_mac_1",
	449	+ .local_flags = flags,
	450	+ .remote_flags = flags,
	451	+ },
	452	+ {
	453	+ .local_name = "xdp_redirect_map_all_prog",
	454	+ .remote_name = "store_mac_2",
	455	+ .local_flags = flags,
	456	+ .remote_flags = flags,
	457	+ }
	458	+ };
	459	+ const char magic_mac[6] = { 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF};
	460	+ struct xdp_redirect_multi_kern *xdp_redirect_multi_kern;
	461	+ struct bpf_object *bpf_objs[VETH_EGRESS_SKEL_NB];
	462	+ struct xdp_redirect_map *xdp_redirect_map;
	463	+ struct bpf_devmap_val devmap_val = {};
	464	+ struct net_configuration net_config;
	465	+ int mac_map, egress_map, res_map;
	466	+ struct nstoken *nstoken = NULL;
	467	+ struct xdp_dummy *xdp_dummy;
	468	+ int err;
	469	+ int i;
	470	+
	471	+ xdp_dummy = xdp_dummy__open_and_load();
	472	+ if (!ASSERT_OK_PTR(xdp_dummy, "xdp_dummy__open_and_load"))
	473	+ return;
	474	+
	475	+ xdp_redirect_multi_kern = xdp_redirect_multi_kern__open_and_load();
	476	+ if (!ASSERT_OK_PTR(xdp_redirect_multi_kern, "xdp_redirect_multi_kern__open_and_load"))
194	477	goto destroy_xdp_dummy;
195	478
196		− skeletons.xdp_redirect_maps = xdp_redirect_map__open_and_load();
197		− if (!ASSERT_OK_PTR(skeletons.xdp_redirect_maps, "xdp_redirect_map__open_and_load"))
198		− goto destroy_xdp_tx;
	479	+ xdp_redirect_map = xdp_redirect_map__open_and_load();
	480	+ if (!ASSERT_OK_PTR(xdp_redirect_map, "xdp_redirect_map__open_and_load"))
	481	+ goto destroy_xdp_redirect_multi_kern;
199	482
200		− if (configure_network(&skeletons))
	483	+ if (!ASSERT_OK(create_network(&net_config), "create network"))
201	484	goto destroy_xdp_redirect_map;
202	485
203		− ASSERT_OK(check_ping(&skeletons), "ping");
	486	+ mac_map = bpf_map__fd(xdp_redirect_multi_kern->maps.mac_map);
	487	+ if (!ASSERT_OK_FD(mac_map, "open mac_map"))
	488	+ goto destroy_xdp_redirect_map;
	489	+
	490	+ egress_map = bpf_map__fd(xdp_redirect_multi_kern->maps.map_egress);
	491	+ if (!ASSERT_OK_FD(egress_map, "open map_egress"))
	492	+ goto destroy_xdp_redirect_map;
	493	+
	494	+ devmap_val.bpf_prog.fd = bpf_program__fd(xdp_redirect_multi_kern->progs.xdp_devmap_prog);
	495	+
	496	+ bpf_objs[0] = xdp_dummy->obj;
	497	+ bpf_objs[1] = xdp_redirect_multi_kern->obj;
	498	+ bpf_objs[2] = xdp_redirect_map->obj;
	499	+
	500	+ nstoken = open_netns(net_config.ns0_name);
	501	+ if (!ASSERT_OK_PTR(nstoken, "open NS0"))
	502	+ goto destroy_xdp_redirect_map;
	503	+
	504	+ for (i = 0; i < VETH_PAIRS_COUNT; i++) {
	505	+ int ifindex = if_nametoindex(net_config.veth_cfg[i].local_veth);
	506	+
	507	+ SYS(destroy_xdp_redirect_map,
	508	+ "ip -n %s neigh add %s lladdr 00:00:00:00:00:01 dev %s",
	509	+ net_config.veth_cfg[i].namespace, IP_NEIGH, net_config.veth_cfg[i].remote_veth);
	510	+
	511	+ if (attach_programs_to_veth_pair(bpf_objs, VETH_REDIRECT_SKEL_NB,
	512	+ &net_config, prog_cfg, i))
	513	+ goto destroy_xdp_redirect_map;
	514	+
	515	+ err = bpf_map_update_elem(mac_map, &ifindex, magic_mac, 0);
	516	+ if (!ASSERT_OK(err, "bpf_map_update_elem"))
	517	+ goto destroy_xdp_redirect_map;
	518	+
	519	+ devmap_val.ifindex = ifindex;
	520	+ err = bpf_map_update_elem(egress_map, &ifindex, &devmap_val, 0);
	521	+ if (!ASSERT_OK(err, "bpf_map_update_elem"))
	522	+ goto destroy_xdp_redirect_map;
	523	+ }
	524	+
	525	+ SYS_NOFAIL("ip netns exec %s ping %s -i 0.1 -c 4 -W1 > /dev/null ",
	526	+ net_config.veth_cfg[0].namespace, IP_NEIGH);
	527	+
	528	+ res_map = bpf_map__fd(xdp_redirect_map->maps.rx_mac);
	529	+ if (!ASSERT_OK_FD(res_map, "open rx_map"))
	530	+ goto destroy_xdp_redirect_map;
	531	+
	532	+ for (i = 0; i < 2; i++) {
	533	+ u32 key = i;
	534	+ u64 res;
	535	+
	536	+ err = bpf_map_lookup_elem(res_map, &key, &res);
	537	+ if (!ASSERT_OK(err, "get MAC res"))
	538	+ goto destroy_xdp_redirect_map;
	539	+
	540	+ ASSERT_STRNEQ((const char *)&res, magic_mac, ETH_ALEN, "compare mac");
	541	+ }
204	542
205	543	destroy_xdp_redirect_map:
206		− xdp_redirect_map__destroy(skeletons.xdp_redirect_maps);
207		−destroy_xdp_tx:
208		− xdp_tx__destroy(skeletons.xdp_tx);
	544	+ close_netns(nstoken);
	545	+ xdp_redirect_map__destroy(xdp_redirect_map);
	546	+destroy_xdp_redirect_multi_kern:
	547	+ xdp_redirect_multi_kern__destroy(xdp_redirect_multi_kern);
209	548	destroy_xdp_dummy:
210		− xdp_dummy__destroy(skeletons.xdp_dummy);
	549	+ xdp_dummy__destroy(xdp_dummy);
	550	+
	551	+ cleanup_network(&net_config);
	552	+}
	553	+
	554	+void test_xdp_veth_redirect(void)
	555	+{
	556	+ if (test__start_subtest("0"))
	557	+ xdp_veth_redirect(0);
	558	+
	559	+ if (test__start_subtest("DRV_MODE"))
	560	+ xdp_veth_redirect(XDP_FLAGS_DRV_MODE);
	561	+
	562	+ if (test__start_subtest("SKB_MODE"))
	563	+ xdp_veth_redirect(XDP_FLAGS_SKB_MODE);
	564	+}
	565	+
	566	+void test_xdp_veth_broadcast_redirect(void)
	567	+{
	568	+ if (test__start_subtest("0/BROADCAST"))
	569	+ xdp_veth_broadcast_redirect(0, BPF_F_BROADCAST);
	570	+
	571	+ if (test__start_subtest("0/(BROADCAST \| EXCLUDE_INGRESS)"))
	572	+ xdp_veth_broadcast_redirect(0, BPF_F_BROADCAST \| BPF_F_EXCLUDE_INGRESS);
	573	+
	574	+ if (test__start_subtest("DRV_MODE/BROADCAST"))
	575	+ xdp_veth_broadcast_redirect(XDP_FLAGS_DRV_MODE, BPF_F_BROADCAST);
	576	+
	577	+ if (test__start_subtest("DRV_MODE/(BROADCAST \| EXCLUDE_INGRESS)"))
	578	+ xdp_veth_broadcast_redirect(XDP_FLAGS_DRV_MODE,
	579	+ BPF_F_BROADCAST \| BPF_F_EXCLUDE_INGRESS);
	580	+
	581	+ if (test__start_subtest("SKB_MODE/BROADCAST"))
	582	+ xdp_veth_broadcast_redirect(XDP_FLAGS_SKB_MODE, BPF_F_BROADCAST);
	583	+
	584	+ if (test__start_subtest("SKB_MODE/(BROADCAST \| EXCLUDE_INGRESS)"))
	585	+ xdp_veth_broadcast_redirect(XDP_FLAGS_SKB_MODE,
	586	+ BPF_F_BROADCAST \| BPF_F_EXCLUDE_INGRESS);
	587	+}
	588	+
	589	+void test_xdp_veth_egress(void)
	590	+{
	591	+ if (test__start_subtest("0/egress"))
	592	+ xdp_veth_egress(0);
	593	+
	594	+ if (test__start_subtest("DRV_MODE/egress"))
	595	+ xdp_veth_egress(XDP_FLAGS_DRV_MODE);
211	596
212		− cleanup_network();
	597	+ if (test__start_subtest("SKB_MODE/egress"))
	598	+ xdp_veth_egress(XDP_FLAGS_SKB_MODE);
213	599	}

tools/testing/selftests/bpf/prog_tests/token.c+96−1

@@ -19,6 +19,7 @@ · 18 unmodified lines
19	19	#include "priv_prog.skel.h"
20	20	#include "dummy_st_ops_success.skel.h"
21	21	#include "token_lsm.skel.h"
	22	+#include "priv_freplace_prog.skel.h"
22	23
23	24	static inline int sys_mount(const char dev_name, const char dir_name,
24	25	const char *type, unsigned long flags,
@@ -788,6 +789,84 @@ static int userns_obj_priv_prog(int mnt_fd, struct token_lsm *lsm_skel) · 763 unmodified lines
788	789	return 0;
789	790	}
790	791
	792	+static int userns_obj_priv_freplace_setup(int mnt_fd, struct priv_freplace_prog **fr_skel,
	793	+ struct priv_prog *skel, int tgt_fd)
	794	+{
	795	+ LIBBPF_OPTS(bpf_object_open_opts, opts);
	796	+ int err;
	797	+ char buf[256];
	798	+
	799	+ /* use bpf_token_path to provide BPF FS path */
	800	+ snprintf(buf, sizeof(buf), "/proc/self/fd/%d", mnt_fd);
	801	+ opts.bpf_token_path = buf;
	802	+ *skel = priv_prog__open_opts(&opts);
	803	+ if (!ASSERT_OK_PTR(*skel, "priv_prog__open_opts"))
	804	+ return -EINVAL;
	805	+ err = priv_prog__load(*skel);
	806	+ if (!ASSERT_OK(err, "priv_prog__load"))
	807	+ return -EINVAL;
	808	+
	809	+ *fr_skel = priv_freplace_prog__open_opts(&opts);
	810	+ if (!ASSERT_OK_PTR(*skel, "priv_freplace_prog__open_opts"))
	811	+ return -EINVAL;
	812	+
	813	+ tgt_fd = bpf_program__fd((skel)->progs.xdp_prog1);
	814	+ return 0;
	815	+}
	816	+
	817	+/* Verify that freplace works from user namespace, because bpf token is loaded
	818	+ * in bpf_object__prepare
	819	+ */
	820	+static int userns_obj_priv_freplace_prog(int mnt_fd, struct token_lsm *lsm_skel)
	821	+{
	822	+ struct priv_freplace_prog *fr_skel = NULL;
	823	+ struct priv_prog *skel = NULL;
	824	+ int err, tgt_fd;
	825	+
	826	+ err = userns_obj_priv_freplace_setup(mnt_fd, &fr_skel, &skel, &tgt_fd);
	827	+ if (!ASSERT_OK(err, "setup"))
	828	+ goto out;
	829	+
	830	+ err = bpf_object__prepare(fr_skel->obj);
	831	+ if (!ASSERT_OK(err, "freplace__prepare"))
	832	+ goto out;
	833	+
	834	+ err = bpf_program__set_attach_target(fr_skel->progs.new_xdp_prog2, tgt_fd, "xdp_prog1");
	835	+ if (!ASSERT_OK(err, "set_attach_target"))
	836	+ goto out;
	837	+
	838	+ err = priv_freplace_prog__load(fr_skel);
	839	+ ASSERT_OK(err, "priv_freplace_prog__load");
	840	+
	841	+out:
	842	+ priv_freplace_prog__destroy(fr_skel);
	843	+ priv_prog__destroy(skel);
	844	+ return err;
	845	+}
	846	+
	847	+/* Verify that replace fails to set attach target from user namespace without bpf token */
	848	+static int userns_obj_priv_freplace_prog_fail(int mnt_fd, struct token_lsm *lsm_skel)
	849	+{
	850	+ struct priv_freplace_prog *fr_skel = NULL;
	851	+ struct priv_prog *skel = NULL;
	852	+ int err, tgt_fd;
	853	+
	854	+ err = userns_obj_priv_freplace_setup(mnt_fd, &fr_skel, &skel, &tgt_fd);
	855	+ if (!ASSERT_OK(err, "setup"))
	856	+ goto out;
	857	+
	858	+ err = bpf_program__set_attach_target(fr_skel->progs.new_xdp_prog2, tgt_fd, "xdp_prog1");
	859	+ if (ASSERT_ERR(err, "attach fails"))
	860	+ err = 0;
	861	+ else
	862	+ err = -EINVAL;
	863	+
	864	+out:
	865	+ priv_freplace_prog__destroy(fr_skel);
	866	+ priv_prog__destroy(skel);
	867	+ return err;
	868	+}
	869	+
791	870	/* this test is called with BPF FS that doesn't delegate BPF_BTF_LOAD command,
792	871	* which should cause struct_ops application to fail, as BTF won't be uploaded
793	872	* into the kernel, even if STRUCT_OPS programs themselves are allowed
@@ -1004,12 +1083,28 @@ void test_token(void) · 210 unmodified lines
1004	1083	if (test__start_subtest("obj_priv_prog")) {
1005	1084	struct bpffs_opts opts = {
1006	1085	.cmds = bit(BPF_PROG_LOAD),
1007		− .progs = bit(BPF_PROG_TYPE_KPROBE),
	1086	+ .progs = bit(BPF_PROG_TYPE_XDP),
1008	1087	.attachs = ~0ULL,
1009	1088	};
1010	1089
1011	1090	subtest_userns(&opts, userns_obj_priv_prog);
1012	1091	}
	1092	+ if (test__start_subtest("obj_priv_freplace_prog")) {
	1093	+ struct bpffs_opts opts = {
	1094	+ .cmds = bit(BPF_BTF_LOAD) \| bit(BPF_PROG_LOAD) \| bit(BPF_BTF_GET_FD_BY_ID),
	1095	+ .progs = bit(BPF_PROG_TYPE_EXT) \| bit(BPF_PROG_TYPE_XDP),
	1096	+ .attachs = ~0ULL,
	1097	+ };
	1098	+ subtest_userns(&opts, userns_obj_priv_freplace_prog);
	1099	+ }
	1100	+ if (test__start_subtest("obj_priv_freplace_prog_fail")) {
	1101	+ struct bpffs_opts opts = {
	1102	+ .cmds = bit(BPF_BTF_LOAD) \| bit(BPF_PROG_LOAD) \| bit(BPF_BTF_GET_FD_BY_ID),
	1103	+ .progs = bit(BPF_PROG_TYPE_EXT) \| bit(BPF_PROG_TYPE_XDP),
	1104	+ .attachs = ~0ULL,
	1105	+ };
	1106	+ subtest_userns(&opts, userns_obj_priv_freplace_prog_fail);
	1107	+ }
1013	1108	if (test__start_subtest("obj_priv_btf_fail")) {
1014	1109	struct bpffs_opts opts = {
1015	1110	/* disallow BTF loading */

tools/testing/selftests/bpf/prog_tests/usdt.c+10−1

@@ -45,7 +45,7 @@ static void subtest_basic_usdt(void) · 44 unmodified lines
45	45	LIBBPF_OPTS(bpf_usdt_opts, opts);
46	46	struct test_usdt *skel;
47	47	struct test_usdt__bss *bss;
48		− int err;
	48	+ int err, i;
49	49
50	50	skel = test_usdt__open_and_load();
51	51	if (!ASSERT_OK_PTR(skel, "skel_open"))
@@ -75,6 +75,7 @@ static void subtest_basic_usdt(void) · 23 unmodified lines
75	75	ASSERT_EQ(bss->usdt0_cookie, 0xcafedeadbeeffeed, "usdt0_cookie");
76	76	ASSERT_EQ(bss->usdt0_arg_cnt, 0, "usdt0_arg_cnt");
77	77	ASSERT_EQ(bss->usdt0_arg_ret, -ENOENT, "usdt0_arg_ret");
	78	+ ASSERT_EQ(bss->usdt0_arg_size, -ENOENT, "usdt0_arg_size");
78	79
79	80	/* auto-attached usdt3 gets default zero cookie value */
80	81	ASSERT_EQ(bss->usdt3_cookie, 0, "usdt3_cookie");
@@ -86,6 +87,9 @@ static void subtest_basic_usdt(void) · 5 unmodified lines
86	87	ASSERT_EQ(bss->usdt3_args[0], 1, "usdt3_arg1");
87	88	ASSERT_EQ(bss->usdt3_args[1], 42, "usdt3_arg2");
88	89	ASSERT_EQ(bss->usdt3_args[2], (uintptr_t)&bla, "usdt3_arg3");
	90	+ ASSERT_EQ(bss->usdt3_arg_sizes[0], 4, "usdt3_arg1_size");
	91	+ ASSERT_EQ(bss->usdt3_arg_sizes[1], 8, "usdt3_arg2_size");
	92	+ ASSERT_EQ(bss->usdt3_arg_sizes[2], 8, "usdt3_arg3_size");
89	93
90	94	/* auto-attached usdt12 gets default zero cookie value */
91	95	ASSERT_EQ(bss->usdt12_cookie, 0, "usdt12_cookie");
@@ -104,6 +108,11 @@ static void subtest_basic_usdt(void) · 12 unmodified lines
104	108	ASSERT_EQ(bss->usdt12_args[10], nums[idx], "usdt12_arg11");
105	109	ASSERT_EQ(bss->usdt12_args[11], t1.y, "usdt12_arg12");
106	110
	111	+ int usdt12_expected_arg_sizes[12] = { 4, 4, 8, 8, 4, 8, 8, 8, 4, 2, 2, 1 };
	112	+
	113	+ for (i = 0; i < 12; i++)
	114	+ ASSERT_EQ(bss->usdt12_arg_sizes[i], usdt12_expected_arg_sizes[i], "usdt12_arg_size");
	115	+
107	116	/* trigger_func() is marked __always_inline, so USDT invocations will be
108	117	* inlined in two different places, meaning that each USDT will have
109	118	* at least 2 different places to be attached to. This verifies that

tools/testing/selftests/bpf/prog_tests/verifier.c+6−2

@@ -45,6 +45,7 @@ · 44 unmodified lines
45	45	#include "verifier_ldsx.skel.h"
46	46	#include "verifier_leak_ptr.skel.h"
47	47	#include "verifier_linked_scalars.skel.h"
	48	+#include "verifier_load_acquire.skel.h"
48	49	#include "verifier_loops1.skel.h"
49	50	#include "verifier_lwt.skel.h"
50	51	#include "verifier_map_in_map.skel.h"
@@ -80,6 +81,7 @@ · 29 unmodified lines
80	81	#include "verifier_spill_fill.skel.h"
81	82	#include "verifier_spin_lock.skel.h"
82	83	#include "verifier_stack_ptr.skel.h"
	84	+#include "verifier_store_release.skel.h"
83	85	#include "verifier_subprog_precision.skel.h"
84	86	#include "verifier_subreg.skel.h"
85	87	#include "verifier_tailcall_jit.skel.h"
@@ -121,7 +123,7 @@ static void run_tests_aux(const char *skel_name, · 35 unmodified lines
121	123	/* test_verifier tests are executed w/o CAP_SYS_ADMIN, do the same here */
122	124	err = cap_disable_effective(1ULL << CAP_SYS_ADMIN, &old_caps);
123	125	if (err) {
124		− PRINT_FAIL("failed to drop CAP_SYS_ADMIN: %i, %s\n", err, strerror(err));
	126	+ PRINT_FAIL("failed to drop CAP_SYS_ADMIN: %i, %s\n", err, strerror(-err));
125	127	return;
126	128	}
127	129
@@ -131,7 +133,7 @@ static void run_tests_aux(const char *skel_name, · 3 unmodified lines
131	133
132	134	err = cap_enable_effective(old_caps, NULL);
133	135	if (err)
134		− PRINT_FAIL("failed to restore CAP_SYS_ADMIN: %i, %s\n", err, strerror(err));
	136	+ PRINT_FAIL("failed to restore CAP_SYS_ADMIN: %i, %s\n", err, strerror(-err));
135	137	}
136	138
137	139	#define RUN(skel) run_tests_aux(#skel, skel##__elf_bytes, NULL)
@@ -173,6 +175,7 @@ void test_verifier_int_ptr(void) { RUN(verifier_int_ptr); } · 35 unmodified lines
173	175	void test_verifier_iterating_callbacks(void) { RUN(verifier_iterating_callbacks); }
174	176	void test_verifier_jeq_infer_not_null(void) { RUN(verifier_jeq_infer_not_null); }
175	177	void test_verifier_jit_convergence(void) { RUN(verifier_jit_convergence); }
	178	+void test_verifier_load_acquire(void) { RUN(verifier_load_acquire); }
176	179	void test_verifier_ld_ind(void) { RUN(verifier_ld_ind); }
177	180	void test_verifier_ldsx(void) { RUN(verifier_ldsx); }
178	181	void test_verifier_leak_ptr(void) { RUN(verifier_leak_ptr); }
@@ -211,6 +214,7 @@ void test_verifier_sockmap_mutate(void) { RUN(verifier_sockmap_mutate); } · 32 unmodified lines
211	214	void test_verifier_spill_fill(void) { RUN(verifier_spill_fill); }
212	215	void test_verifier_spin_lock(void) { RUN(verifier_spin_lock); }
213	216	void test_verifier_stack_ptr(void) { RUN(verifier_stack_ptr); }
	217	+void test_verifier_store_release(void) { RUN(verifier_store_release); }
214	218	void test_verifier_subprog_precision(void) { RUN(verifier_subprog_precision); }
215	219	void test_verifier_subreg(void) { RUN(verifier_subreg); }
216	220	void test_verifier_tailcall_jit(void) { RUN(verifier_tailcall_jit); }

tools/testing/selftests/bpf/prog_tests/xdp_vlan.c+175−0

@@ -0,0 +1,175 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+
	3	+/*
	4	+ * Network topology:
	5	+ * ----------- -----------
	6	+ * \| NS1 \| \| NS2 \|
	7	+ * \| veth0 -\|--------\|- veth0 \|
	8	+ * ----------- -----------
	9	+ *
	10	+ */
	11	+
	12	+#define _GNU_SOURCE
	13	+#include <net/if.h>
	14	+#include <uapi/linux/if_link.h>
	15	+
	16	+#include "network_helpers.h"
	17	+#include "test_progs.h"
	18	+#include "test_xdp_vlan.skel.h"
	19	+
	20	+
	21	+#define VETH_NAME "veth0"
	22	+#define NS_MAX_SIZE 32
	23	+#define NS1_NAME "ns-xdp-vlan-1-"
	24	+#define NS2_NAME "ns-xdp-vlan-2-"
	25	+#define NS1_IP_ADDR "100.64.10.1"
	26	+#define NS2_IP_ADDR "100.64.10.2"
	27	+#define VLAN_ID 4011
	28	+
	29	+static int setup_network(char ns1, char ns2)
	30	+{
	31	+ if (!ASSERT_OK(append_tid(ns1, NS_MAX_SIZE), "create ns1 name"))
	32	+ goto fail;
	33	+ if (!ASSERT_OK(append_tid(ns2, NS_MAX_SIZE), "create ns2 name"))
	34	+ goto fail;
	35	+
	36	+ SYS(fail, "ip netns add %s", ns1);
	37	+ SYS(fail, "ip netns add %s", ns2);
	38	+ SYS(fail, "ip -n %s link add %s type veth peer name %s netns %s",
	39	+ ns1, VETH_NAME, VETH_NAME, ns2);
	40	+
	41	+ /* NOTICE: XDP require VLAN header inside packet payload
	42	+ * - Thus, disable VLAN offloading driver features
	43	+ */
	44	+ SYS(fail, "ip netns exec %s ethtool -K %s rxvlan off txvlan off", ns1, VETH_NAME);
	45	+ SYS(fail, "ip netns exec %s ethtool -K %s rxvlan off txvlan off", ns2, VETH_NAME);
	46	+
	47	+ /* NS1 configuration */
	48	+ SYS(fail, "ip -n %s addr add %s/24 dev %s", ns1, NS1_IP_ADDR, VETH_NAME);
	49	+ SYS(fail, "ip -n %s link set %s up", ns1, VETH_NAME);
	50	+
	51	+ /* NS2 configuration */
	52	+ SYS(fail, "ip -n %s link add link %s name %s.%d type vlan id %d",
	53	+ ns2, VETH_NAME, VETH_NAME, VLAN_ID, VLAN_ID);
	54	+ SYS(fail, "ip -n %s addr add %s/24 dev %s.%d", ns2, NS2_IP_ADDR, VETH_NAME, VLAN_ID);
	55	+ SYS(fail, "ip -n %s link set %s up", ns2, VETH_NAME);
	56	+ SYS(fail, "ip -n %s link set %s.%d up", ns2, VETH_NAME, VLAN_ID);
	57	+
	58	+ /* At this point ping should fail because VLAN tags are only used by NS2 */
	59	+ return !SYS_NOFAIL("ip netns exec %s ping -W 1 -c1 %s", ns2, NS1_IP_ADDR);
	60	+
	61	+fail:
	62	+ return -1;
	63	+}
	64	+
	65	+static void cleanup_network(const char ns1, const char ns2)
	66	+{
	67	+ SYS_NOFAIL("ip netns del %s", ns1);
	68	+ SYS_NOFAIL("ip netns del %s", ns2);
	69	+}
	70	+
	71	+static void xdp_vlan(struct bpf_program xdp, struct bpf_program tc, u32 flags)
	72	+{
	73	+ LIBBPF_OPTS(bpf_tc_hook, tc_hook, .attach_point = BPF_TC_EGRESS);
	74	+ LIBBPF_OPTS(bpf_tc_opts, tc_opts, .handle = 1, .priority = 1);
	75	+ char ns1[NS_MAX_SIZE] = NS1_NAME;
	76	+ char ns2[NS_MAX_SIZE] = NS2_NAME;
	77	+ struct nstoken *nstoken = NULL;
	78	+ int interface;
	79	+ int ret;
	80	+
	81	+ if (!ASSERT_OK(setup_network(ns1, ns2), "setup network"))
	82	+ goto cleanup;
	83	+
	84	+ nstoken = open_netns(ns1);
	85	+ if (!ASSERT_OK_PTR(nstoken, "open NS1"))
	86	+ goto cleanup;
	87	+
	88	+ interface = if_nametoindex(VETH_NAME);
	89	+ if (!ASSERT_NEQ(interface, 0, "get interface index"))
	90	+ goto cleanup;
	91	+
	92	+ ret = bpf_xdp_attach(interface, bpf_program__fd(xdp), flags, NULL);
	93	+ if (!ASSERT_OK(ret, "attach xdp_vlan_change"))
	94	+ goto cleanup;
	95	+
	96	+ tc_hook.ifindex = interface;
	97	+ ret = bpf_tc_hook_create(&tc_hook);
	98	+ if (!ASSERT_OK(ret, "bpf_tc_hook_create"))
	99	+ goto detach_xdp;
	100	+
	101	+ /* Now we'll use BPF programs to pop/push the VLAN tags */
	102	+ tc_opts.prog_fd = bpf_program__fd(tc);
	103	+ ret = bpf_tc_attach(&tc_hook, &tc_opts);
	104	+ if (!ASSERT_OK(ret, "bpf_tc_attach"))
	105	+ goto detach_xdp;
	106	+
	107	+ close_netns(nstoken);
	108	+ nstoken = NULL;
	109	+
	110	+ /* Now the namespaces can reach each-other, test with pings */
	111	+ SYS(detach_tc, "ip netns exec %s ping -i 0.2 -W 2 -c 2 %s > /dev/null", ns1, NS2_IP_ADDR);
	112	+ SYS(detach_tc, "ip netns exec %s ping -i 0.2 -W 2 -c 2 %s > /dev/null", ns2, NS1_IP_ADDR);
	113	+
	114	+
	115	+detach_tc:
	116	+ bpf_tc_detach(&tc_hook, &tc_opts);
	117	+detach_xdp:
	118	+ bpf_xdp_detach(interface, flags, NULL);
	119	+cleanup:
	120	+ close_netns(nstoken);
	121	+ cleanup_network(ns1, ns2);
	122	+}
	123	+
	124	+/* First test: Remove VLAN by setting VLAN ID 0, using "xdp_vlan_change"
	125	+ * egress use TC to add back VLAN tag 4011
	126	+ */
	127	+void test_xdp_vlan_change(void)
	128	+{
	129	+ struct test_xdp_vlan *skel;
	130	+
	131	+ skel = test_xdp_vlan__open_and_load();
	132	+ if (!ASSERT_OK_PTR(skel, "xdp_vlan__open_and_load"))
	133	+ return;
	134	+
	135	+ if (test__start_subtest("0"))
	136	+ xdp_vlan(skel->progs.xdp_vlan_change, skel->progs.tc_vlan_push, 0);
	137	+
	138	+ if (test__start_subtest("DRV_MODE"))
	139	+ xdp_vlan(skel->progs.xdp_vlan_change, skel->progs.tc_vlan_push,
	140	+ XDP_FLAGS_DRV_MODE);
	141	+
	142	+ if (test__start_subtest("SKB_MODE"))
	143	+ xdp_vlan(skel->progs.xdp_vlan_change, skel->progs.tc_vlan_push,
	144	+ XDP_FLAGS_SKB_MODE);
	145	+
	146	+ test_xdp_vlan__destroy(skel);
	147	+}
	148	+
	149	+/* Second test: XDP prog fully remove vlan header
	150	+ *
	151	+ * Catch kernel bug for generic-XDP, that doesn't allow us to
	152	+ * remove a VLAN header, because skb->protocol still contain VLAN
	153	+ * ETH_P_8021Q indication, and this cause overwriting of our changes.
	154	+ */
	155	+void test_xdp_vlan_remove(void)
	156	+{
	157	+ struct test_xdp_vlan *skel;
	158	+
	159	+ skel = test_xdp_vlan__open_and_load();
	160	+ if (!ASSERT_OK_PTR(skel, "xdp_vlan__open_and_load"))
	161	+ return;
	162	+
	163	+ if (test__start_subtest("0"))
	164	+ xdp_vlan(skel->progs.xdp_vlan_remove_outer2, skel->progs.tc_vlan_push, 0);
	165	+
	166	+ if (test__start_subtest("DRV_MODE"))
	167	+ xdp_vlan(skel->progs.xdp_vlan_remove_outer2, skel->progs.tc_vlan_push,
	168	+ XDP_FLAGS_DRV_MODE);
	169	+
	170	+ if (test__start_subtest("SKB_MODE"))
	171	+ xdp_vlan(skel->progs.xdp_vlan_remove_outer2, skel->progs.tc_vlan_push,
	172	+ XDP_FLAGS_SKB_MODE);
	173	+
	174	+ test_xdp_vlan__destroy(skel);
	175	+}

tools/testing/selftests/bpf/progs/arena_atomics.c+119−2

@@ -6,6 +6,8 @@ · 5 unmodified lines
6	6	#include <stdbool.h>
7	7	#include <stdatomic.h>
8	8	#include "bpf_arena_common.h"
	9	+#include "../../../include/linux/filter.h"
	10	+#include "bpf_misc.h"
9	11
10	12	struct {
11	13	__uint(type, BPF_MAP_TYPE_ARENA);
@@ -19,9 +21,17 @@ struct { · 7 unmodified lines
19	21	} arena SEC(".maps");
20	22
21	23	#if defined(ENABLE_ATOMICS_TESTS) && defined(__BPF_FEATURE_ADDR_SPACE_CAST)
22		−bool skip_tests __attribute((__section__(".data"))) = false;
	24	+bool skip_all_tests __attribute((__section__(".data"))) = false;
23	25	#else
24		−bool skip_tests = true;
	26	+bool skip_all_tests = true;
	27	+#endif
	28	+
	29	+#if defined(ENABLE_ATOMICS_TESTS) && \
	30	+ defined(__BPF_FEATURE_ADDR_SPACE_CAST) && \
	31	+ (defined(__TARGET_ARCH_arm64) \|\| defined(__TARGET_ARCH_x86))
	32	+bool skip_lacq_srel_tests __attribute((__section__(".data"))) = false;
	33	+#else
	34	+bool skip_lacq_srel_tests = true;
25	35	#endif
26	36
27	37	__u32 pid = 0;
@@ -274,4 +284,111 @@ int uaf(const void *ctx) · 246 unmodified lines
274	284	return 0;
275	285	}
276	286
	287	+#if __clang_major__ >= 18
	288	+__u8 __arena_global load_acquire8_value = 0x12;
	289	+__u16 __arena_global load_acquire16_value = 0x1234;
	290	+__u32 __arena_global load_acquire32_value = 0x12345678;
	291	+__u64 __arena_global load_acquire64_value = 0x1234567890abcdef;
	292	+
	293	+__u8 __arena_global load_acquire8_result = 0;
	294	+__u16 __arena_global load_acquire16_result = 0;
	295	+__u32 __arena_global load_acquire32_result = 0;
	296	+__u64 __arena_global load_acquire64_result = 0;
	297	+#else
	298	+/* clang-17 crashes if the .addr_space.1 ELF section has holes. Work around
	299	+ * this issue by defining the below variables as 64-bit.
	300	+ */
	301	+__u64 __arena_global load_acquire8_value;
	302	+__u64 __arena_global load_acquire16_value;
	303	+__u64 __arena_global load_acquire32_value;
	304	+__u64 __arena_global load_acquire64_value;
	305	+
	306	+__u64 __arena_global load_acquire8_result;
	307	+__u64 __arena_global load_acquire16_result;
	308	+__u64 __arena_global load_acquire32_result;
	309	+__u64 __arena_global load_acquire64_result;
	310	+#endif
	311	+
	312	+SEC("raw_tp/sys_enter")
	313	+int load_acquire(const void *ctx)
	314	+{
	315	+#if defined(ENABLE_ATOMICS_TESTS) && \
	316	+ defined(__BPF_FEATURE_ADDR_SPACE_CAST) && \
	317	+ (defined(__TARGET_ARCH_arm64) \|\| defined(__TARGET_ARCH_x86))
	318	+
	319	+#define LOAD_ACQUIRE_ARENA(SIZEOP, SIZE, SRC, DST) \
	320	+ { asm volatile ( \
	321	+ "r1 = %[" #SRC "] ll;" \
	322	+ "r1 = addr_space_cast(r1, 0x0, 0x1);" \
	323	+ ".8byte %[load_acquire_insn];" \
	324	+ "r3 = %[" #DST "] ll;" \
	325	+ "r3 = addr_space_cast(r3, 0x0, 0x1);" \
	326	+ "(" #SIZE " )(r3 + 0) = r2;" \
	327	+ : \
	328	+ : __imm_addr(SRC), \
	329	+ __imm_insn(load_acquire_insn, \
	330	+ BPF_ATOMIC_OP(BPF_##SIZEOP, BPF_LOAD_ACQ, \
	331	+ BPF_REG_2, BPF_REG_1, 0)), \
	332	+ __imm_addr(DST) \
	333	+ : __clobber_all); } \
	334	+
	335	+ LOAD_ACQUIRE_ARENA(B, u8, load_acquire8_value, load_acquire8_result)
	336	+ LOAD_ACQUIRE_ARENA(H, u16, load_acquire16_value,
	337	+ load_acquire16_result)
	338	+ LOAD_ACQUIRE_ARENA(W, u32, load_acquire32_value,
	339	+ load_acquire32_result)
	340	+ LOAD_ACQUIRE_ARENA(DW, u64, load_acquire64_value,
	341	+ load_acquire64_result)
	342	+#undef LOAD_ACQUIRE_ARENA
	343	+
	344	+#endif
	345	+ return 0;
	346	+}
	347	+
	348	+#if __clang_major__ >= 18
	349	+__u8 __arena_global store_release8_result = 0;
	350	+__u16 __arena_global store_release16_result = 0;
	351	+__u32 __arena_global store_release32_result = 0;
	352	+__u64 __arena_global store_release64_result = 0;
	353	+#else
	354	+/* clang-17 crashes if the .addr_space.1 ELF section has holes. Work around
	355	+ * this issue by defining the below variables as 64-bit.
	356	+ */
	357	+__u64 __arena_global store_release8_result;
	358	+__u64 __arena_global store_release16_result;
	359	+__u64 __arena_global store_release32_result;
	360	+__u64 __arena_global store_release64_result;
	361	+#endif
	362	+
	363	+SEC("raw_tp/sys_enter")
	364	+int store_release(const void *ctx)
	365	+{
	366	+#if defined(ENABLE_ATOMICS_TESTS) && \
	367	+ defined(__BPF_FEATURE_ADDR_SPACE_CAST) && \
	368	+ (defined(__TARGET_ARCH_arm64) \|\| defined(__TARGET_ARCH_x86))
	369	+
	370	+#define STORE_RELEASE_ARENA(SIZEOP, DST, VAL) \
	371	+ { asm volatile ( \
	372	+ "r1 = " VAL ";" \
	373	+ "r2 = %[" #DST "] ll;" \
	374	+ "r2 = addr_space_cast(r2, 0x0, 0x1);" \
	375	+ ".8byte %[store_release_insn];" \
	376	+ : \
	377	+ : __imm_addr(DST), \
	378	+ __imm_insn(store_release_insn, \
	379	+ BPF_ATOMIC_OP(BPF_##SIZEOP, BPF_STORE_REL, \
	380	+ BPF_REG_2, BPF_REG_1, 0)) \
	381	+ : __clobber_all); } \
	382	+
	383	+ STORE_RELEASE_ARENA(B, store_release8_result, "0x12")
	384	+ STORE_RELEASE_ARENA(H, store_release16_result, "0x1234")
	385	+ STORE_RELEASE_ARENA(W, store_release32_result, "0x12345678")
	386	+ STORE_RELEASE_ARENA(DW, store_release64_result,
	387	+ "0x1234567890abcdef ll")
	388	+#undef STORE_RELEASE_ARENA
	389	+
	390	+#endif
	391	+ return 0;
	392	+}
	393	+
277	394	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/arena_spin_lock.c+51−0

@@ -0,0 +1,51 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
	3	+#include <vmlinux.h>
	4	+#include <bpf/bpf_tracing.h>
	5	+#include <bpf/bpf_helpers.h>
	6	+#include "bpf_misc.h"
	7	+#include "bpf_arena_spin_lock.h"
	8	+
	9	+struct {
	10	+ __uint(type, BPF_MAP_TYPE_ARENA);
	11	+ __uint(map_flags, BPF_F_MMAPABLE);
	12	+ __uint(max_entries, 100); /* number of pages */
	13	+#ifdef __TARGET_ARCH_arm64
	14	+ __ulong(map_extra, 0x1ull << 32); /* start of mmap() region */
	15	+#else
	16	+ __ulong(map_extra, 0x1ull << 44); /* start of mmap() region */
	17	+#endif
	18	+} arena SEC(".maps");
	19	+
	20	+int cs_count;
	21	+
	22	+#if defined(ENABLE_ATOMICS_TESTS) && defined(__BPF_FEATURE_ADDR_SPACE_CAST)
	23	+arena_spinlock_t __arena lock;
	24	+int test_skip = 1;
	25	+#else
	26	+int test_skip = 2;
	27	+#endif
	28	+
	29	+int counter;
	30	+int limit;
	31	+
	32	+SEC("tc")
	33	+int prog(void *ctx)
	34	+{
	35	+ int ret = -2;
	36	+
	37	+#if defined(ENABLE_ATOMICS_TESTS) && defined(__BPF_FEATURE_ADDR_SPACE_CAST)
	38	+ unsigned long flags;
	39	+
	40	+ if ((ret = arena_spin_lock_irqsave(&lock, flags)))
	41	+ return ret;
	42	+ if (counter != limit)
	43	+ counter++;
	44	+ bpf_repeat(cs_count);
	45	+ ret = 0;
	46	+ arena_spin_unlock_irqrestore(&lock, flags);
	47	+#endif
	48	+ return ret;
	49	+}
	50	+
	51	+char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/bpf_iter_tasks.c+110−0

@@ -9,6 +9,13 @@ char _license[] SEC("license") = "GPL"; · 8 unmodified lines
9	9	uint32_t tid = 0;
10	10	int num_unknown_tid = 0;
11	11	int num_known_tid = 0;
	12	+void *user_ptr = 0;
	13	+void *user_ptr_long = 0;
	14	+uint32_t pid = 0;
	15	+
	16	+static char big_str1[5000];
	17	+static char big_str2[5005];
	18	+static char big_str3[4996];
12	19
13	20	SEC("iter/task")
14	21	int dump_task(struct bpf_iter__task *ctx)
@@ -35,7 +42,9 @@ int dump_task(struct bpf_iter__task *ctx) · 20 unmodified lines
35	42	}
36	43
37	44	int num_expected_failure_copy_from_user_task = 0;
	45	+int num_expected_failure_copy_from_user_task_str = 0;
38	46	int num_success_copy_from_user_task = 0;
	47	+int num_success_copy_from_user_task_str = 0;
39	48
40	49	SEC("iter.s/task")
41	50	int dump_task_sleepable(struct bpf_iter__task *ctx)
@@ -44,6 +53,9 @@ int dump_task_sleepable(struct bpf_iter__task *ctx) · 2 unmodified lines
44	53	struct task_struct *task = ctx->task;
45	54	static const char info[] = " === END ===";
46	55	struct pt_regs *regs;
	56	+ char task_str1[10] = "aaaaaaaaaa";
	57	+ char task_str2[10], task_str3[10];
	58	+ char task_str4[20] = "aaaaaaaaaaaaaaaaaaaa";
47	59	void *ptr;
48	60	uint32_t user_data = 0;
49	61	int ret;
@@ -78,8 +90,106 @@ int dump_task_sleepable(struct bpf_iter__task *ctx) · 28 unmodified lines
78	90	BPF_SEQ_PRINTF(seq, "%s\n", info);
79	91	return 0;
80	92	}
	93	+
81	94	++num_success_copy_from_user_task;
82	95
	96	+ /* Read an invalid pointer and ensure we get an error */
	97	+ ptr = NULL;
	98	+ ret = bpf_copy_from_user_task_str((char *)task_str1, sizeof(task_str1), ptr, task, 0);
	99	+ if (ret >= 0 \|\| task_str1[9] != 'a' \|\| task_str1[0] != '\0') {
	100	+ BPF_SEQ_PRINTF(seq, "%s\n", info);
	101	+ return 0;
	102	+ }
	103	+
	104	+ /* Read an invalid pointer and ensure we get error with pad zeros flag */
	105	+ ptr = NULL;
	106	+ ret = bpf_copy_from_user_task_str((char *)task_str1, sizeof(task_str1),
	107	+ ptr, task, BPF_F_PAD_ZEROS);
	108	+ if (ret >= 0 \|\| task_str1[9] != '\0' \|\| task_str1[0] != '\0') {
	109	+ BPF_SEQ_PRINTF(seq, "%s\n", info);
	110	+ return 0;
	111	+ }
	112	+
	113	+ ++num_expected_failure_copy_from_user_task_str;
	114	+
	115	+ /* Same length as the string */
	116	+ ret = bpf_copy_from_user_task_str((char *)task_str2, 10, user_ptr, task, 0);
	117	+ /* only need to do the task pid check once */
	118	+ if (bpf_strncmp(task_str2, 10, "test_data\0") != 0 \|\| ret != 10 \|\| task->tgid != pid) {
	119	+ BPF_SEQ_PRINTF(seq, "%s\n", info);
	120	+ return 0;
	121	+ }
	122	+
	123	+ /* Shorter length than the string */
	124	+ ret = bpf_copy_from_user_task_str((char *)task_str3, 2, user_ptr, task, 0);
	125	+ if (bpf_strncmp(task_str3, 2, "t\0") != 0 \|\| ret != 2) {
	126	+ BPF_SEQ_PRINTF(seq, "%s\n", info);
	127	+ return 0;
	128	+ }
	129	+
	130	+ /* Longer length than the string */
	131	+ ret = bpf_copy_from_user_task_str((char *)task_str4, 20, user_ptr, task, 0);
	132	+ if (bpf_strncmp(task_str4, 10, "test_data\0") != 0 \|\| ret != 10
	133	+ \|\| task_str4[sizeof(task_str4) - 1] != 'a') {
	134	+ BPF_SEQ_PRINTF(seq, "%s\n", info);
	135	+ return 0;
	136	+ }
	137	+
	138	+ /* Longer length than the string with pad zeros flag */
	139	+ ret = bpf_copy_from_user_task_str((char *)task_str4, 20, user_ptr, task, BPF_F_PAD_ZEROS);
	140	+ if (bpf_strncmp(task_str4, 10, "test_data\0") != 0 \|\| ret != 10
	141	+ \|\| task_str4[sizeof(task_str4) - 1] != '\0') {
	142	+ BPF_SEQ_PRINTF(seq, "%s\n", info);
	143	+ return 0;
	144	+ }
	145	+
	146	+ /* Longer length than the string past a page boundary */
	147	+ ret = bpf_copy_from_user_task_str(big_str1, 5000, user_ptr, task, 0);
	148	+ if (bpf_strncmp(big_str1, 10, "test_data\0") != 0 \|\| ret != 10) {
	149	+ BPF_SEQ_PRINTF(seq, "%s\n", info);
	150	+ return 0;
	151	+ }
	152	+
	153	+ /* String that crosses a page boundary */
	154	+ ret = bpf_copy_from_user_task_str(big_str1, 5000, user_ptr_long, task, BPF_F_PAD_ZEROS);
	155	+ if (bpf_strncmp(big_str1, 4, "baba") != 0 \|\| ret != 5000
	156	+ \|\| bpf_strncmp(big_str1 + 4996, 4, "bab\0") != 0) {
	157	+ BPF_SEQ_PRINTF(seq, "%s\n", info);
	158	+ return 0;
	159	+ }
	160	+
	161	+ for (int i = 0; i < 4999; ++i) {
	162	+ if (i % 2 == 0) {
	163	+ if (big_str1[i] != 'b') {
	164	+ BPF_SEQ_PRINTF(seq, "%s\n", info);
	165	+ return 0;
	166	+ }
	167	+ } else {
	168	+ if (big_str1[i] != 'a') {
	169	+ BPF_SEQ_PRINTF(seq, "%s\n", info);
	170	+ return 0;
	171	+ }
	172	+ }
	173	+ }
	174	+
	175	+ /* Longer length than the string that crosses a page boundary */
	176	+ ret = bpf_copy_from_user_task_str(big_str2, 5005, user_ptr_long, task, BPF_F_PAD_ZEROS);
	177	+ if (bpf_strncmp(big_str2, 4, "baba") != 0 \|\| ret != 5000
	178	+ \|\| bpf_strncmp(big_str2 + 4996, 5, "bab\0\0") != 0) {
	179	+ BPF_SEQ_PRINTF(seq, "%s\n", info);
	180	+ return 0;
	181	+ }
	182	+
	183	+ /* Shorter length than the string that crosses a page boundary */
	184	+ ret = bpf_copy_from_user_task_str(big_str3, 4996, user_ptr_long, task, 0);
	185	+ if (bpf_strncmp(big_str3, 4, "baba") != 0 \|\| ret != 4996
	186	+ \|\| bpf_strncmp(big_str3 + 4992, 4, "bab\0") != 0) {
	187	+ BPF_SEQ_PRINTF(seq, "%s\n", info);
	188	+ return 0;
	189	+ }
	190	+
	191	+ ++num_success_copy_from_user_task_str;
	192	+
83	193	if (ctx->meta->seq_num == 0)
84	194	BPF_SEQ_PRINTF(seq, " tgid gid data\n");
85	195

tools/testing/selftests/bpf/progs/bpf_misc.h+22−0

@@ -135,6 +135,8 @@ · 134 unmodified lines
135	135	#define __arch_arm64 __arch("ARM64")
136	136	#define __arch_riscv64 __arch("RISCV64")
137	137	#define __caps_unpriv(caps) __attribute__((btf_decl_tag("comment:test_caps_unpriv=" EXPAND_QUOTE(caps))))
	138	+#define __load_if_JITed() __attribute__((btf_decl_tag("comment:load_mode=jited")))
	139	+#define __load_if_no_JITed() __attribute__((btf_decl_tag("comment:load_mode=no_jited")))
138	140
139	141	/* Define common capabilities tested using __caps_unpriv */
140	142	#define CAP_NET_ADMIN 12
@@ -172,6 +174,9 @@ · 31 unmodified lines
172	174	#elif defined(__TARGET_ARCH_riscv)
173	175	#define SYSCALL_WRAPPER 1
174	176	#define SYS_PREFIX "__riscv_"
	177	+#elif defined(__TARGET_ARCH_powerpc)
	178	+#define SYSCALL_WRAPPER 1
	179	+#define SYS_PREFIX ""
175	180	#else
176	181	#define SYSCALL_WRAPPER 0
177	182	#define SYS_PREFIX "__se_"
@@ -208,4 +213,21 @@ · 30 unmodified lines
208	213	#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
209	214	#endif
210	215
	216	+#if (defined(__TARGET_ARCH_arm64) \|\| defined(__TARGET_ARCH_x86) \|\| \
	217	+ (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) \|\| \
	218	+ defined(__TARGET_ARCH_arm) \|\| defined(__TARGET_ARCH_s390) \|\| \
	219	+ defined(__TARGET_ARCH_loongarch)) && \
	220	+ __clang_major__ >= 18
	221	+#define CAN_USE_GOTOL
	222	+#endif
	223	+
	224	+#if _clang_major__ >= 18
	225	+#define CAN_USE_BPF_ST
	226	+#endif
	227	+
	228	+#if __clang_major__ >= 18 && defined(ENABLE_ATOMICS_TESTS) && \
	229	+ (defined(__TARGET_ARCH_arm64) \|\| defined(__TARGET_ARCH_x86))
	230	+#define CAN_USE_LOAD_ACQ_STORE_REL
	231	+#endif
	232	+
211	233	#endif

tools/testing/selftests/bpf/progs/bpf_tracing_net.h+4−0

@@ -15,7 +15,11 @@ · 14 unmodified lines
15	15	#define SO_KEEPALIVE 9
16	16	#define SO_PRIORITY 12
17	17	#define SO_REUSEPORT 15
	18	+#if defined(__TARGET_ARCH_powerpc)
	19	+#define SO_RCVLOWAT 16
	20	+#else
18	21	#define SO_RCVLOWAT 18
	22	+#endif
19	23	#define SO_BINDTODEVICE 25
20	24	#define SO_MARK 36
21	25	#define SO_MAX_PACING_RATE 47

tools/testing/selftests/bpf/progs/btf__core_reloc_arrays___err_bad_signed_arr_elem_sz.c+3−0

@@ -0,0 +1,3 @@
	1	+#include "core_reloc_types.h"
	2	+
	3	+void f(struct core_reloc_arrays___err_bad_signed_arr_elem_sz x) {}

tools/testing/selftests/bpf/progs/cgroup_preorder.c+41−0

@@ -0,0 +1,41 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
	3	+#include <vmlinux.h>
	4	+#include <bpf/bpf_helpers.h>
	5	+
	6	+char _license[] SEC("license") = "GPL";
	7	+
	8	+unsigned int idx;
	9	+__u8 result[4];
	10	+
	11	+SEC("cgroup/getsockopt")
	12	+int child(struct bpf_sockopt *ctx)
	13	+{
	14	+ if (idx < 4)
	15	+ result[idx++] = 1;
	16	+ return 1;
	17	+}
	18	+
	19	+SEC("cgroup/getsockopt")
	20	+int child_2(struct bpf_sockopt *ctx)
	21	+{
	22	+ if (idx < 4)
	23	+ result[idx++] = 2;
	24	+ return 1;
	25	+}
	26	+
	27	+SEC("cgroup/getsockopt")
	28	+int parent(struct bpf_sockopt *ctx)
	29	+{
	30	+ if (idx < 4)
	31	+ result[idx++] = 3;
	32	+ return 1;
	33	+}
	34	+
	35	+SEC("cgroup/getsockopt")
	36	+int parent_2(struct bpf_sockopt *ctx)
	37	+{
	38	+ if (idx < 4)
	39	+ result[idx++] = 4;
	40	+ return 1;
	41	+}

tools/testing/selftests/bpf/progs/changes_pkt_data.c+0−39

@@ -1,39 +0,0 @@
1		−// SPDX-License-Identifier: GPL-2.0
2		−
3		−#include <linux/bpf.h>
4		−#include <bpf/bpf_helpers.h>
5		−
6		−__noinline
7		−long changes_pkt_data(struct __sk_buff *sk)
8		−{
9		− return bpf_skb_pull_data(sk, 0);
10		−}
11		−
12		−__noinline __weak
13		−long does_not_change_pkt_data(struct __sk_buff *sk)
14		−{
15		− return 0;
16		−}
17		−
18		−SEC("?tc")
19		−int main_with_subprogs(struct __sk_buff *sk)
20		−{
21		− changes_pkt_data(sk);
22		− does_not_change_pkt_data(sk);
23		− return 0;
24		−}
25		−
26		−SEC("?tc")
27		−int main_changes(struct __sk_buff *sk)
28		−{
29		− bpf_skb_pull_data(sk, 0);
30		− return 0;
31		−}
32		−
33		−SEC("?tc")
34		−int main_does_not_change(struct __sk_buff *sk)
35		−{
36		− return 0;
37		−}
38		−
39		−char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/compute_live_registers.c+424−0

@@ -0,0 +1,424 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+
	3	+#include <linux/bpf.h>
	4	+#include <bpf/bpf_helpers.h>
	5	+#include "../../../include/linux/filter.h"
	6	+#include "bpf_arena_common.h"
	7	+#include "bpf_misc.h"
	8	+
	9	+struct {
	10	+ __uint(type, BPF_MAP_TYPE_ARRAY);
	11	+ __uint(max_entries, 1);
	12	+ __type(key, __u32);
	13	+ __type(value, __u64);
	14	+} test_map SEC(".maps");
	15	+
	16	+struct {
	17	+ __uint(type, BPF_MAP_TYPE_ARENA);
	18	+ __uint(map_flags, BPF_F_MMAPABLE);
	19	+ __uint(max_entries, 1);
	20	+} arena SEC(".maps");
	21	+
	22	+SEC("socket")
	23	+__log_level(2)
	24	+__msg(" 0: .......... (b7) r0 = 42")
	25	+__msg(" 1: 0......... (bf) r1 = r0")
	26	+__msg(" 2: .1........ (bf) r2 = r1")
	27	+__msg(" 3: ..2....... (bf) r3 = r2")
	28	+__msg(" 4: ...3...... (bf) r4 = r3")
	29	+__msg(" 5: ....4..... (bf) r5 = r4")
	30	+__msg(" 6: .....5.... (bf) r6 = r5")
	31	+__msg(" 7: ......6... (bf) r7 = r6")
	32	+__msg(" 8: .......7.. (bf) r8 = r7")
	33	+__msg(" 9: ........8. (bf) r9 = r8")
	34	+__msg("10: .........9 (bf) r0 = r9")
	35	+__msg("11: 0......... (95) exit")
	36	+__naked void assign_chain(void)
	37	+{
	38	+ asm volatile (
	39	+ "r0 = 42;"
	40	+ "r1 = r0;"
	41	+ "r2 = r1;"
	42	+ "r3 = r2;"
	43	+ "r4 = r3;"
	44	+ "r5 = r4;"
	45	+ "r6 = r5;"
	46	+ "r7 = r6;"
	47	+ "r8 = r7;"
	48	+ "r9 = r8;"
	49	+ "r0 = r9;"
	50	+ "exit;"
	51	+ ::: __clobber_all);
	52	+}
	53	+
	54	+SEC("socket")
	55	+__log_level(2)
	56	+__msg("0: .......... (b7) r1 = 7")
	57	+__msg("1: .1........ (07) r1 += 7")
	58	+__msg("2: .......... (b7) r2 = 7")
	59	+__msg("3: ..2....... (b7) r3 = 42")
	60	+__msg("4: ..23...... (0f) r2 += r3")
	61	+__msg("5: .......... (b7) r0 = 0")
	62	+__msg("6: 0......... (95) exit")
	63	+__naked void arithmetics(void)
	64	+{
	65	+ asm volatile (
	66	+ "r1 = 7;"
	67	+ "r1 += 7;"
	68	+ "r2 = 7;"
	69	+ "r3 = 42;"
	70	+ "r2 += r3;"
	71	+ "r0 = 0;"
	72	+ "exit;"
	73	+ ::: __clobber_all);
	74	+}
	75	+
	76	+#ifdef CAN_USE_BPF_ST
	77	+SEC("socket")
	78	+__log_level(2)
	79	+__msg(" 1: .1........ (07) r1 += -8")
	80	+__msg(" 2: .1........ (7a) (u64 )(r1 +0) = 7")
	81	+__msg(" 3: .1........ (b7) r2 = 42")
	82	+__msg(" 4: .12....... (7b) (u64 )(r1 +0) = r2")
	83	+__msg(" 5: .12....... (7b) (u64 )(r1 +0) = r2")
	84	+__msg(" 6: .......... (b7) r0 = 0")
	85	+__naked void store(void)
	86	+{
	87	+ asm volatile (
	88	+ "r1 = r10;"
	89	+ "r1 += -8;"
	90	+ "(u64 )(r1 +0) = 7;"
	91	+ "r2 = 42;"
	92	+ "(u64 )(r1 +0) = r2;"
	93	+ "(u64 )(r1 +0) = r2;"
	94	+ "r0 = 0;"
	95	+ "exit;"
	96	+ ::: __clobber_all);
	97	+}
	98	+#endif
	99	+
	100	+SEC("socket")
	101	+__log_level(2)
	102	+__msg("1: ....4..... (07) r4 += -8")
	103	+__msg("2: ....4..... (79) r5 = (u64 )(r4 +0)")
	104	+__msg("3: ....45.... (07) r4 += -8")
	105	+__naked void load(void)
	106	+{
	107	+ asm volatile (
	108	+ "r4 = r10;"
	109	+ "r4 += -8;"
	110	+ "r5 = (u64 )(r4 +0);"
	111	+ "r4 += -8;"
	112	+ "r0 = r5;"
	113	+ "exit;"
	114	+ ::: __clobber_all);
	115	+}
	116	+
	117	+SEC("socket")
	118	+__log_level(2)
	119	+__msg("0: .1........ (61) r2 = (u32 )(r1 +0)")
	120	+__msg("1: ..2....... (d4) r2 = le64 r2")
	121	+__msg("2: ..2....... (bf) r0 = r2")
	122	+__naked void endian(void)
	123	+{
	124	+ asm volatile (
	125	+ "r2 = (u32 )(r1 +0);"
	126	+ "r2 = le64 r2;"
	127	+ "r0 = r2;"
	128	+ "exit;"
	129	+ ::: __clobber_all);
	130	+}
	131	+
	132	+SEC("socket")
	133	+__log_level(2)
	134	+__msg(" 8: 0......... (b7) r1 = 1")
	135	+__msg(" 9: 01........ (db) r1 = atomic64_fetch_add((u64 *)(r0 +0), r1)")
	136	+__msg("10: 01........ (c3) lock (u32 )(r0 +0) += r1")
	137	+__msg("11: 01........ (db) r1 = atomic64_xchg((u64 *)(r0 +0), r1)")
	138	+__msg("12: 01........ (bf) r2 = r0")
	139	+__msg("13: .12....... (bf) r0 = r1")
	140	+__msg("14: 012....... (db) r0 = atomic64_cmpxchg((u64 *)(r2 +0), r0, r1)")
	141	+__naked void atomic(void)
	142	+{
	143	+ asm volatile (
	144	+ "r2 = r10;"
	145	+ "r2 += -8;"
	146	+ "r1 = 0;"
	147	+ "(u64 )(r2 +0) = r1;"
	148	+ "r1 = %[test_map] ll;"
	149	+ "call %[bpf_map_lookup_elem];"
	150	+ "if r0 == 0 goto 1f;"
	151	+ "r1 = 1;"
	152	+ "r1 = atomic_fetch_add((u64 *)(r0 +0), r1);"
	153	+ ".8byte %[add_nofetch];" /* same as "lock (u32 )(r0 +0) += r1;" */
	154	+ "r1 = xchg_64(r0 + 0, r1);"
	155	+ "r2 = r0;"
	156	+ "r0 = r1;"
	157	+ "r0 = cmpxchg_64(r2 + 0, r0, r1);"
	158	+ "1: exit;"
	159	+ :
	160	+ : __imm(bpf_map_lookup_elem),
	161	+ __imm_addr(test_map),
	162	+ __imm_insn(add_nofetch, BPF_ATOMIC_OP(BPF_W, BPF_ADD, BPF_REG_0, BPF_REG_1, 0))
	163	+ : __clobber_all);
	164	+}
	165	+
	166	+#ifdef CAN_USE_LOAD_ACQ_STORE_REL
	167	+
	168	+SEC("socket")
	169	+__log_level(2)
	170	+__msg("2: .12....... (db) store_release((u64 *)(r2 -8), r1)")
	171	+__msg("3: .......... (bf) r3 = r10")
	172	+__msg("4: ...3...... (db) r4 = load_acquire((u64 *)(r3 -8))")
	173	+__naked void atomic_load_acq_store_rel(void)
	174	+{
	175	+ asm volatile (
	176	+ "r1 = 42;"
	177	+ "r2 = r10;"
	178	+ ".8byte %[store_release_insn];" /* store_release((u64 )(r2 - 8), r1); /
	179	+ "r3 = r10;"
	180	+ ".8byte %[load_acquire_insn];" /* r4 = load_acquire((u64 )(r3 + 0)); /
	181	+ "r0 = r4;"
	182	+ "exit;"
	183	+ :
	184	+ : __imm_insn(store_release_insn,
	185	+ BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_2, BPF_REG_1, -8)),
	186	+ __imm_insn(load_acquire_insn,
	187	+ BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_4, BPF_REG_3, -8))
	188	+ : __clobber_all);
	189	+}
	190	+
	191	+#endif /* CAN_USE_LOAD_ACQ_STORE_REL */
	192	+
	193	+SEC("socket")
	194	+__log_level(2)
	195	+__msg("4: .12....7.. (85) call bpf_trace_printk#6")
	196	+__msg("5: 0......7.. (0f) r0 += r7")
	197	+__naked void regular_call(void)
	198	+{
	199	+ asm volatile (
	200	+ "r7 = 1;"
	201	+ "r1 = r10;"
	202	+ "r1 += -8;"
	203	+ "r2 = 1;"
	204	+ "call %[bpf_trace_printk];"
	205	+ "r0 += r7;"
	206	+ "exit;"
	207	+ :
	208	+ : __imm(bpf_trace_printk)
	209	+ : __clobber_all);
	210	+}
	211	+
	212	+SEC("socket")
	213	+__log_level(2)
	214	+__msg("2: 012....... (25) if r1 > 0x7 goto pc+1")
	215	+__msg("3: ..2....... (bf) r0 = r2")
	216	+__naked void if1(void)
	217	+{
	218	+ asm volatile (
	219	+ "r0 = 1;"
	220	+ "r2 = 2;"
	221	+ "if r1 > 0x7 goto +1;"
	222	+ "r0 = r2;"
	223	+ "exit;"
	224	+ ::: __clobber_all);
	225	+}
	226	+
	227	+SEC("socket")
	228	+__log_level(2)
	229	+__msg("3: 0123...... (2d) if r1 > r3 goto pc+1")
	230	+__msg("4: ..2....... (bf) r0 = r2")
	231	+__naked void if2(void)
	232	+{
	233	+ asm volatile (
	234	+ "r0 = 1;"
	235	+ "r2 = 2;"
	236	+ "r3 = 7;"
	237	+ "if r1 > r3 goto +1;"
	238	+ "r0 = r2;"
	239	+ "exit;"
	240	+ ::: __clobber_all);
	241	+}
	242	+
	243	+SEC("socket")
	244	+__log_level(2)
	245	+__msg("0: .......... (b7) r1 = 0")
	246	+__msg("1: .1........ (b7) r2 = 7")
	247	+__msg("2: .12....... (25) if r1 > 0x7 goto pc+4")
	248	+__msg("3: .12....... (07) r1 += 1")
	249	+__msg("4: .12....... (27) r2 *= 2")
	250	+__msg("5: .12....... (05) goto pc+0")
	251	+__msg("6: .12....... (05) goto pc-5")
	252	+__msg("7: .......... (b7) r0 = 0")
	253	+__msg("8: 0......... (95) exit")
	254	+__naked void loop(void)
	255	+{
	256	+ asm volatile (
	257	+ "r1 = 0;"
	258	+ "r2 = 7;"
	259	+ "if r1 > 0x7 goto +4;"
	260	+ "r1 += 1;"
	261	+ "r2 *= 2;"
	262	+ "goto +0;"
	263	+ "goto -5;"
	264	+ "r0 = 0;"
	265	+ "exit;"
	266	+ :
	267	+ : __imm(bpf_trace_printk)
	268	+ : __clobber_all);
	269	+}
	270	+
	271	+#ifdef CAN_USE_GOTOL
	272	+SEC("socket")
	273	+__log_level(2)
	274	+__msg("2: .123...... (25) if r1 > 0x7 goto pc+2")
	275	+__msg("3: ..2....... (bf) r0 = r2")
	276	+__msg("4: 0......... (06) gotol pc+1")
	277	+__msg("5: ...3...... (bf) r0 = r3")
	278	+__msg("6: 0......... (95) exit")
	279	+__naked void gotol(void)
	280	+{
	281	+ asm volatile (
	282	+ "r2 = 42;"
	283	+ "r3 = 24;"
	284	+ "if r1 > 0x7 goto +2;"
	285	+ "r0 = r2;"
	286	+ "gotol +1;"
	287	+ "r0 = r3;"
	288	+ "exit;"
	289	+ :
	290	+ : __imm(bpf_trace_printk)
	291	+ : __clobber_all);
	292	+}
	293	+#endif
	294	+
	295	+SEC("socket")
	296	+__log_level(2)
	297	+__msg("0: .......... (b7) r1 = 1")
	298	+__msg("1: .1........ (e5) may_goto pc+1")
	299	+__msg("2: .......... (05) goto pc-3")
	300	+__msg("3: .1........ (bf) r0 = r1")
	301	+__msg("4: 0......... (95) exit")
	302	+__naked void may_goto(void)
	303	+{
	304	+ asm volatile (
	305	+ "1: r1 = 1;"
	306	+ ".8byte %[may_goto];"
	307	+ "goto 1b;"
	308	+ "r0 = r1;"
	309	+ "exit;"
	310	+ :
	311	+ : __imm(bpf_get_smp_processor_id),
	312	+ __imm_insn(may_goto, BPF_RAW_INSN(BPF_JMP \| BPF_JCOND, 0, 0, +1 /* offset */, 0))
	313	+ : __clobber_all);
	314	+}
	315	+
	316	+SEC("socket")
	317	+__log_level(2)
	318	+__msg("1: 0......... (18) r2 = 0x7")
	319	+__msg("3: 0.2....... (0f) r0 += r2")
	320	+__naked void ldimm64(void)
	321	+{
	322	+ asm volatile (
	323	+ "r0 = 0;"
	324	+ "r2 = 0x7 ll;"
	325	+ "r0 += r2;"
	326	+ "exit;"
	327	+ :
	328	+ :: __clobber_all);
	329	+}
	330	+
	331	+/* No rules specific for LD_ABS/LD_IND, default behaviour kicks in */
	332	+SEC("socket")
	333	+__log_level(2)
	334	+__msg("2: 0123456789 (30) r0 = (u8 )skb[42]")
	335	+__msg("3: 012.456789 (0f) r7 += r0")
	336	+__msg("4: 012.456789 (b7) r3 = 42")
	337	+__msg("5: 0123456789 (50) r0 = (u8 )skb[r3 + 0]")
	338	+__msg("6: 0......7.. (0f) r7 += r0")
	339	+__naked void ldabs(void)
	340	+{
	341	+ asm volatile (
	342	+ "r6 = r1;"
	343	+ "r7 = 0;"
	344	+ "r0 = (u8 )skb[42];"
	345	+ "r7 += r0;"
	346	+ "r3 = 42;"
	347	+ ".8byte %[ld_ind];" /* same as "r0 = (u8 )skb[r3];" */
	348	+ "r7 += r0;"
	349	+ "r0 = r7;"
	350	+ "exit;"
	351	+ :
	352	+ : __imm_insn(ld_ind, BPF_LD_IND(BPF_B, BPF_REG_3, 0))
	353	+ : __clobber_all);
	354	+}
	355	+
	356	+
	357	+#ifdef __BPF_FEATURE_ADDR_SPACE_CAST
	358	+SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
	359	+__log_level(2)
	360	+__msg(" 6: .12345.... (85) call bpf_arena_alloc_pages")
	361	+__msg(" 7: 0......... (bf) r1 = addr_space_cast(r0, 0, 1)")
	362	+__msg(" 8: .1........ (b7) r2 = 42")
	363	+__naked void addr_space_cast(void)
	364	+{
	365	+ asm volatile (
	366	+ "r1 = %[arena] ll;"
	367	+ "r2 = 0;"
	368	+ "r3 = 1;"
	369	+ "r4 = 0;"
	370	+ "r5 = 0;"
	371	+ "call %[bpf_arena_alloc_pages];"
	372	+ "r1 = addr_space_cast(r0, 0, 1);"
	373	+ "r2 = 42;"
	374	+ "(u64 )(r1 +0) = r2;"
	375	+ "r0 = 0;"
	376	+ "exit;"
	377	+ :
	378	+ : __imm(bpf_arena_alloc_pages),
	379	+ __imm_addr(arena)
	380	+ : __clobber_all);
	381	+}
	382	+#endif
	383	+
	384	+static __used __naked int aux1(void)
	385	+{
	386	+ asm volatile (
	387	+ "r0 = r1;"
	388	+ "r0 += r2;"
	389	+ "exit;"
	390	+ ::: __clobber_all);
	391	+}
	392	+
	393	+SEC("socket")
	394	+__log_level(2)
	395	+__msg("0: ....45.... (b7) r1 = 1")
	396	+__msg("1: .1..45.... (b7) r2 = 2")
	397	+__msg("2: .12.45.... (b7) r3 = 3")
	398	+/* Conservative liveness for subprog parameters. */
	399	+__msg("3: .12345.... (85) call pc+2")
	400	+__msg("4: .......... (b7) r0 = 0")
	401	+__msg("5: 0......... (95) exit")
	402	+__msg("6: .12....... (bf) r0 = r1")
	403	+__msg("7: 0.2....... (0f) r0 += r2")
	404	+/* Conservative liveness for subprog return value. */
	405	+__msg("8: 0......... (95) exit")
	406	+__naked void subprog1(void)
	407	+{
	408	+ asm volatile (
	409	+ "r1 = 1;"
	410	+ "r2 = 2;"
	411	+ "r3 = 3;"
	412	+ "call aux1;"
	413	+ "r0 = 0;"
	414	+ "exit;"
	415	+ ::: __clobber_all);
	416	+}
	417	+
	418	+/* to retain debug info for BTF generation */
	419	+void kfunc_root(void)
	420	+{
	421	+ bpf_arena_alloc_pages(0, 0, 0, 0, 0);
	422	+}
	423	+
	424	+char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/connect4_dropper.c+3−1

@@ -13,12 +13,14 @@ · 12 unmodified lines
13	13	#define VERDICT_REJECT 0
14	14	#define VERDICT_PROCEED 1
15	15
	16	+int port;
	17	+
16	18	SEC("cgroup/connect4")
17	19	int connect_v4_dropper(struct bpf_sock_addr *ctx)
18	20	{
19	21	if (ctx->type != SOCK_STREAM)
20	22	return VERDICT_PROCEED;
21		− if (ctx->user_port == bpf_htons(60120))
	23	+ if (ctx->user_port == bpf_htons(port))
22	24	return VERDICT_REJECT;
23	25	return VERDICT_PROCEED;
24	26	}

tools/testing/selftests/bpf/progs/core_reloc_types.h+10−0

@@ -347,6 +347,7 @@ struct core_reloc_nesting___err_too_deep { · 346 unmodified lines
347	347	*/
348	348	struct core_reloc_arrays_output {
349	349	int a2;
	350	+ int a3;
350	351	char b123;
351	352	int c1c;
352	353	int d00d;
@@ -455,6 +456,15 @@ struct core_reloc_arrays___err_bad_zero_sz_arr { · 102 unmodified lines
455	456	struct core_reloc_arrays_substruct d[1][2];
456	457	};
457	458
	459	+struct core_reloc_arrays___err_bad_signed_arr_elem_sz {
	460	+ /* int -> short (signed!): not supported case */
	461	+ short a[5];
	462	+ char b[2][3][4];
	463	+ struct core_reloc_arrays_substruct c[3];
	464	+ struct core_reloc_arrays_substruct d[1][2];
	465	+ struct core_reloc_arrays_substruct f[][2];
	466	+};
	467	+
458	468	/*
459	469	* PRIMITIVES
460	470	*/

tools/testing/selftests/bpf/progs/cpumask_common.h+1−0

@@ -61,6 +61,7 @@ u32 bpf_cpumask_any_distribute(const struct cpumask *src) __ksym __weak; · 60 unmodified lines
61	61	u32 bpf_cpumask_any_and_distribute(const struct cpumask *src1,
62	62	const struct cpumask *src2) __ksym __weak;
63	63	u32 bpf_cpumask_weight(const struct cpumask *cpumask) __ksym __weak;
	64	+int bpf_cpumask_populate(struct cpumask cpumask, void src, size_t src__sz) __ksym __weak;
64	65
65	66	void bpf_rcu_read_lock(void) __ksym __weak;
66	67	void bpf_rcu_read_unlock(void) __ksym __weak;

tools/testing/selftests/bpf/progs/cpumask_failure.c+38−0

@@ -222,3 +222,41 @@ int BPF_PROG(test_invalid_nested_array, struct task_struct *task, u64 clone_flag · 221 unmodified lines
222	222
223	223	return 0;
224	224	}
	225	+
	226	+SEC("tp_btf/task_newtask")
	227	+__failure __msg("type=scalar expected=fp")
	228	+int BPF_PROG(test_populate_invalid_destination, struct task_struct *task, u64 clone_flags)
	229	+{
	230	+ struct bpf_cpumask invalid = (struct bpf_cpumask )0x123456;
	231	+ u64 bits;
	232	+ int ret;
	233	+
	234	+ ret = bpf_cpumask_populate((struct cpumask *)invalid, &bits, sizeof(bits));
	235	+ if (!ret)
	236	+ err = 2;
	237	+
	238	+ return 0;
	239	+}
	240	+
	241	+SEC("tp_btf/task_newtask")
	242	+__failure __msg("leads to invalid memory access")
	243	+int BPF_PROG(test_populate_invalid_source, struct task_struct *task, u64 clone_flags)
	244	+{
	245	+ void garbage = (void )0x123456;
	246	+ struct bpf_cpumask *local;
	247	+ int ret;
	248	+
	249	+ local = create_cpumask();
	250	+ if (!local) {
	251	+ err = 1;
	252	+ return 0;
	253	+ }
	254	+
	255	+ ret = bpf_cpumask_populate((struct cpumask *)local, garbage, 8);
	256	+ if (!ret)
	257	+ err = 2;
	258	+
	259	+ bpf_cpumask_release(local);
	260	+
	261	+ return 0;
	262	+}

tools/testing/selftests/bpf/progs/cpumask_success.c+119−1

@@ -749,7 +749,6 @@ out: · 748 unmodified lines
749	749	}
750	750
751	751	SEC("tp_btf/task_newtask")
752		−__success
753	752	int BPF_PROG(test_refcount_null_tracking, struct task_struct *task, u64 clone_flags)
754	753	{
755	754	struct bpf_cpumask mask1, mask2;
@@ -770,3 +769,122 @@ free_masks_return: · 14 unmodified lines
770	769	bpf_cpumask_release(mask2);
771	770	return 0;
772	771	}
	772	+
	773	+SEC("tp_btf/task_newtask")
	774	+int BPF_PROG(test_populate_reject_small_mask, struct task_struct *task, u64 clone_flags)
	775	+{
	776	+ struct bpf_cpumask *local;
	777	+ u8 toofewbits;
	778	+ int ret;
	779	+
	780	+ if (!is_test_task())
	781	+ return 0;
	782	+
	783	+ local = create_cpumask();
	784	+ if (!local)
	785	+ return 0;
	786	+
	787	+ /* The kfunc should prevent this operation */
	788	+ ret = bpf_cpumask_populate((struct cpumask *)local, &toofewbits, sizeof(toofewbits));
	789	+ if (ret != -EACCES)
	790	+ err = 2;
	791	+
	792	+ bpf_cpumask_release(local);
	793	+
	794	+ return 0;
	795	+}
	796	+
	797	+/* Mask is guaranteed to be large enough for bpf_cpumask_t. */
	798	+#define CPUMASK_TEST_MASKLEN (sizeof(cpumask_t))
	799	+
	800	+/* Add an extra word for the test_populate_reject_unaligned test. */
	801	+u64 bits[CPUMASK_TEST_MASKLEN / 8 + 1];
	802	+extern bool CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS __kconfig __weak;
	803	+
	804	+SEC("tp_btf/task_newtask")
	805	+int BPF_PROG(test_populate_reject_unaligned, struct task_struct *task, u64 clone_flags)
	806	+{
	807	+ struct bpf_cpumask *mask;
	808	+ char *src;
	809	+ int ret;
	810	+
	811	+ if (!is_test_task())
	812	+ return 0;
	813	+
	814	+ /* Skip if unaligned accesses are fine for this arch. */
	815	+ if (CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)
	816	+ return 0;
	817	+
	818	+ mask = bpf_cpumask_create();
	819	+ if (!mask) {
	820	+ err = 1;
	821	+ return 0;
	822	+ }
	823	+
	824	+ /* Misalign the source array by a byte. */
	825	+ src = &((char *)bits)[1];
	826	+
	827	+ ret = bpf_cpumask_populate((struct cpumask *)mask, src, CPUMASK_TEST_MASKLEN);
	828	+ if (ret != -EINVAL)
	829	+ err = 2;
	830	+
	831	+ bpf_cpumask_release(mask);
	832	+
	833	+ return 0;
	834	+}
	835	+
	836	+
	837	+SEC("tp_btf/task_newtask")
	838	+int BPF_PROG(test_populate, struct task_struct *task, u64 clone_flags)
	839	+{
	840	+ struct bpf_cpumask *mask;
	841	+ bool bit;
	842	+ int ret;
	843	+ int i;
	844	+
	845	+ if (!is_test_task())
	846	+ return 0;
	847	+
	848	+ /* Set only odd bits. */
	849	+ __builtin_memset(bits, 0xaa, CPUMASK_TEST_MASKLEN);
	850	+
	851	+ mask = bpf_cpumask_create();
	852	+ if (!mask) {
	853	+ err = 1;
	854	+ return 0;
	855	+ }
	856	+
	857	+ /* Pass the entire bits array, the kfunc will only copy the valid bits. */
	858	+ ret = bpf_cpumask_populate((struct cpumask *)mask, bits, CPUMASK_TEST_MASKLEN);
	859	+ if (ret) {
	860	+ err = 2;
	861	+ goto out;
	862	+ }
	863	+
	864	+ /*
	865	+ * Test is there to appease the verifier. We cannot directly
	866	+ * access NR_CPUS, the upper bound for nr_cpus, so we infer
	867	+ * it from the size of cpumask_t.
	868	+ */
	869	+ if (nr_cpus < 0 \|\| nr_cpus >= CPUMASK_TEST_MASKLEN * 8) {
	870	+ err = 3;
	871	+ goto out;
	872	+ }
	873	+
	874	+ bpf_for(i, 0, nr_cpus) {
	875	+ /* Odd-numbered bits should be set, even ones unset. */
	876	+ bit = bpf_cpumask_test_cpu(i, (const struct cpumask *)mask);
	877	+ if (bit == (i % 2 != 0))
	878	+ continue;
	879	+
	880	+ err = 4;
	881	+ break;
	882	+ }
	883	+
	884	+out:
	885	+ bpf_cpumask_release(mask);
	886	+
	887	+ return 0;
	888	+}
	889	+
	890	+#undef CPUMASK_TEST_MASKLEN

tools/testing/selftests/bpf/progs/dynptr_success.c+118−5

@@ -1,20 +1,19 @@
1	1	// SPDX-License-Identifier: GPL-2.0
2	2	/* Copyright (c) 2022 Facebook */
3	3
	4	+#include <vmlinux.h>
4	5	#include <string.h>
5	6	#include <stdbool.h>
6		−#include <linux/bpf.h>
7	7	#include <bpf/bpf_helpers.h>
8	8	#include <bpf/bpf_tracing.h>
9	9	#include "bpf_misc.h"
10		−#include "bpf_kfuncs.h"
11	10	#include "errno.h"
12	11
13	12	char _license[] SEC("license") = "GPL";
14	13
15	14	int pid, err, val;
16	15
17		−struct sample {
	16	+struct ringbuf_sample {
18	17	int pid;
19	18	int seq;
20	19	long value;
@@ -121,7 +120,7 @@ int test_dynptr_data(void *ctx) · 100 unmodified lines
121	120
122	121	static int ringbuf_callback(__u32 index, void *data)
123	122	{
124		− struct sample *sample;
	123	+ struct ringbuf_sample *sample;
125	124
126	125	struct bpf_dynptr ptr = (struct bpf_dynptr )data;
127	126
@@ -138,7 +137,7 @@ SEC("?tp/syscalls/sys_enter_nanosleep") · 10 unmodified lines
138	137	int test_ringbuf(void *ctx)
139	138	{
140	139	struct bpf_dynptr ptr;
141		− struct sample *sample;
	140	+ struct ringbuf_sample *sample;
142	141
143	142	if (bpf_get_current_pid_tgid() >> 32 != pid)
144	143	return 0;
@@ -567,3 +566,117 @@ int BPF_PROG(test_dynptr_skb_tp_btf, void skb, void location) · 422 unmodified lines
567	566
568	567	return 1;
569	568	}
	569	+
	570	+static inline int bpf_memcmp(const char a, const char b, u32 size)
	571	+{
	572	+ int i;
	573	+
	574	+ bpf_for(i, 0, size) {
	575	+ if (a[i] != b[i])
	576	+ return a[i] < b[i] ? -1 : 1;
	577	+ }
	578	+ return 0;
	579	+}
	580	+
	581	+SEC("?tp/syscalls/sys_enter_nanosleep")
	582	+int test_dynptr_copy(void *ctx)
	583	+{
	584	+ char data[] = "hello there, world!!";
	585	+ char buf[32] = {'\0'};
	586	+ __u32 sz = sizeof(data);
	587	+ struct bpf_dynptr src, dst;
	588	+
	589	+ bpf_ringbuf_reserve_dynptr(&ringbuf, sz, 0, &src);
	590	+ bpf_ringbuf_reserve_dynptr(&ringbuf, sz, 0, &dst);
	591	+
	592	+ /* Test basic case of copying contiguous memory backed dynptrs */
	593	+ err = bpf_dynptr_write(&src, 0, data, sz, 0);
	594	+ err = err ?: bpf_dynptr_copy(&dst, 0, &src, 0, sz);
	595	+ err = err ?: bpf_dynptr_read(buf, sz, &dst, 0, 0);
	596	+ err = err ?: bpf_memcmp(data, buf, sz);
	597	+
	598	+ /* Test that offsets are handled correctly */
	599	+ err = err ?: bpf_dynptr_copy(&dst, 3, &src, 5, sz - 5);
	600	+ err = err ?: bpf_dynptr_read(buf, sz - 5, &dst, 3, 0);
	601	+ err = err ?: bpf_memcmp(data + 5, buf, sz - 5);
	602	+
	603	+ bpf_ringbuf_discard_dynptr(&src, 0);
	604	+ bpf_ringbuf_discard_dynptr(&dst, 0);
	605	+ return 0;
	606	+}
	607	+
	608	+SEC("xdp")
	609	+int test_dynptr_copy_xdp(struct xdp_md *xdp)
	610	+{
	611	+ struct bpf_dynptr ptr_buf, ptr_xdp;
	612	+ char data[] = "qwertyuiopasdfghjkl";
	613	+ char buf[32] = {'\0'};
	614	+ __u32 len = sizeof(data);
	615	+ int i, chunks = 200;
	616	+
	617	+ /* ptr_xdp is backed by non-contiguous memory */
	618	+ bpf_dynptr_from_xdp(xdp, 0, &ptr_xdp);
	619	+ bpf_ringbuf_reserve_dynptr(&ringbuf, len * chunks, 0, &ptr_buf);
	620	+
	621	+ /* Destination dynptr is backed by non-contiguous memory */
	622	+ bpf_for(i, 0, chunks) {
	623	+ err = bpf_dynptr_write(&ptr_buf, i * len, data, len, 0);
	624	+ if (err)
	625	+ goto out;
	626	+ }
	627	+
	628	+ err = bpf_dynptr_copy(&ptr_xdp, 0, &ptr_buf, 0, len * chunks);
	629	+ if (err)
	630	+ goto out;
	631	+
	632	+ bpf_for(i, 0, chunks) {
	633	+ __builtin_memset(buf, 0, sizeof(buf));
	634	+ err = bpf_dynptr_read(&buf, len, &ptr_xdp, i * len, 0);
	635	+ if (err)
	636	+ goto out;
	637	+ if (bpf_memcmp(data, buf, len) != 0)
	638	+ goto out;
	639	+ }
	640	+
	641	+ /* Source dynptr is backed by non-contiguous memory */
	642	+ __builtin_memset(buf, 0, sizeof(buf));
	643	+ bpf_for(i, 0, chunks) {
	644	+ err = bpf_dynptr_write(&ptr_buf, i * len, buf, len, 0);
	645	+ if (err)
	646	+ goto out;
	647	+ }
	648	+
	649	+ err = bpf_dynptr_copy(&ptr_buf, 0, &ptr_xdp, 0, len * chunks);
	650	+ if (err)
	651	+ goto out;
	652	+
	653	+ bpf_for(i, 0, chunks) {
	654	+ __builtin_memset(buf, 0, sizeof(buf));
	655	+ err = bpf_dynptr_read(&buf, len, &ptr_buf, i * len, 0);
	656	+ if (err)
	657	+ goto out;
	658	+ if (bpf_memcmp(data, buf, len) != 0)
	659	+ goto out;
	660	+ }
	661	+
	662	+ /* Both source and destination dynptrs are backed by non-contiguous memory */
	663	+ err = bpf_dynptr_copy(&ptr_xdp, 2, &ptr_xdp, len, len * (chunks - 1));
	664	+ if (err)
	665	+ goto out;
	666	+
	667	+ bpf_for(i, 0, chunks - 1) {
	668	+ __builtin_memset(buf, 0, sizeof(buf));
	669	+ err = bpf_dynptr_read(&buf, len, &ptr_xdp, 2 + i * len, 0);
	670	+ if (err)
	671	+ goto out;
	672	+ if (bpf_memcmp(data, buf, len) != 0)
	673	+ goto out;
	674	+ }
	675	+
	676	+ if (bpf_dynptr_copy(&ptr_xdp, 2000, &ptr_xdp, 0, len * chunks) != -E2BIG)
	677	+ err = 1;
	678	+
	679	+out:
	680	+ bpf_ringbuf_discard_dynptr(&ptr_buf, 0);
	681	+ return XDP_DROP;
	682	+}

tools/testing/selftests/bpf/progs/fexit_noreturns.c+15−0

@@ -0,0 +1,15 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+
	3	+#include <linux/bpf.h>
	4	+#include <bpf/bpf_helpers.h>
	5	+#include <bpf/bpf_tracing.h>
	6	+#include "bpf_misc.h"
	7	+
	8	+char _license[] SEC("license") = "GPL";
	9	+
	10	+SEC("fexit/do_exit")
	11	+__failure __msg("Attaching fexit/fmod_ret to __noreturn functions is rejected.")
	12	+int BPF_PROG(noreturns)
	13	+{
	14	+ return 0;
	15	+}

tools/testing/selftests/bpf/progs/irq.c+70−1

@@ -222,7 +222,7 @@ int __noinline global_local_irq_balance(void) · 221 unmodified lines
222	222	}
223	223
224	224	SEC("?tc")
225		−__failure __msg("global function calls are not allowed with IRQs disabled")
	225	+__success
226	226	int irq_global_subprog(struct __sk_buff *ctx)
227	227	{
228	228	unsigned long flags;
@@ -441,4 +441,73 @@ int irq_ooo_refs_array(struct __sk_buff *ctx) · 212 unmodified lines
441	441	return 0;
442	442	}
443	443
	444	+int __noinline
	445	+global_subprog(int i)
	446	+{
	447	+ if (i)
	448	+ bpf_printk("%p", &i);
	449	+ return i;
	450	+}
	451	+
	452	+int __noinline
	453	+global_sleepable_helper_subprog(int i)
	454	+{
	455	+ if (i)
	456	+ bpf_copy_from_user(&i, sizeof(i), NULL);
	457	+ return i;
	458	+}
	459	+
	460	+int __noinline
	461	+global_sleepable_kfunc_subprog(int i)
	462	+{
	463	+ if (i)
	464	+ bpf_copy_from_user_str(&i, sizeof(i), NULL, 0);
	465	+ global_subprog(i);
	466	+ return i;
	467	+}
	468	+
	469	+int __noinline
	470	+global_subprog_calling_sleepable_global(int i)
	471	+{
	472	+ if (!i)
	473	+ global_sleepable_kfunc_subprog(i);
	474	+ return i;
	475	+}
	476	+
	477	+SEC("?syscall")
	478	+__success
	479	+int irq_non_sleepable_global_subprog(void *ctx)
	480	+{
	481	+ unsigned long flags;
	482	+
	483	+ bpf_local_irq_save(&flags);
	484	+ global_subprog(0);
	485	+ bpf_local_irq_restore(&flags);
	486	+ return 0;
	487	+}
	488	+
	489	+SEC("?syscall")
	490	+__failure __msg("global functions that may sleep are not allowed in non-sleepable context")
	491	+int irq_sleepable_helper_global_subprog(void *ctx)
	492	+{
	493	+ unsigned long flags;
	494	+
	495	+ bpf_local_irq_save(&flags);
	496	+ global_sleepable_helper_subprog(0);
	497	+ bpf_local_irq_restore(&flags);
	498	+ return 0;
	499	+}
	500	+
	501	+SEC("?syscall")
	502	+__failure __msg("global functions that may sleep are not allowed in non-sleepable context")
	503	+int irq_sleepable_global_subprog_indirect(void *ctx)
	504	+{
	505	+ unsigned long flags;
	506	+
	507	+ bpf_local_irq_save(&flags);
	508	+ global_subprog_calling_sleepable_global(0);
	509	+ bpf_local_irq_restore(&flags);
	510	+ return 0;
	511	+}
	512	+
444	513	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/iters.c+139−0

@@ -7,6 +7,8 @@ · 6 unmodified lines
7	7	#include "bpf_misc.h"
8	8	#include "bpf_compiler.h"
9	9
	10	+#define unlikely(x) __builtin_expect(!!(x), 0)
	11	+
10	12	static volatile int zero = 0;
11	13
12	14	int my_pid;
@@ -1174,6 +1176,122 @@ __naked int loop_state_deps2(void) · 1161 unmodified lines
1174	1176	);
1175	1177	}
1176	1178
	1179	+SEC("?raw_tp")
	1180	+__failure
	1181	+__msg("math between fp pointer and register with unbounded")
	1182	+__flag(BPF_F_TEST_STATE_FREQ)
	1183	+__naked int loop_state_deps3(void)
	1184	+{
	1185	+ /* This is equivalent to a C program below.
	1186	+ *
	1187	+ * if (random() != 24) { // assume false branch is placed first
	1188	+ * i = iter_new(); // fp[-8]
	1189	+ * while (iter_next(i));
	1190	+ * iter_destroy(i);
	1191	+ * return;
	1192	+ * }
	1193	+ *
	1194	+ * for (i = 10; i > 0; i--); // increase dfs_depth for child states
	1195	+ *
	1196	+ * i = iter_new(); // fp[-8]
	1197	+ * b = -24; // r8
	1198	+ * for (;;) { // checkpoint (L)
	1199	+ * if (iter_next(i)) // checkpoint (N)
	1200	+ * break;
	1201	+ * if (random() == 77) { // assume false branch is placed first
	1202	+ * (u64 )(r10 + b) = 7; // this is not safe when b == -25
	1203	+ * iter_destroy(i);
	1204	+ * return;
	1205	+ * }
	1206	+ * if (random() == 42) { // assume false branch is placed first
	1207	+ * b = -25;
	1208	+ * }
	1209	+ * }
	1210	+ * iter_destroy(i);
	1211	+ *
	1212	+ * In case of a buggy verifier first loop might poison
	1213	+ * env->cur_state->loop_entry with a state having 0 branches
	1214	+ * and small dfs_depth. This would trigger NOT_EXACT states
	1215	+ * comparison for some states within second loop.
	1216	+ * Specifically, checkpoint (L) might be problematic if:
	1217	+ * - branch with '(u64 )(r10 + b) = 7' is not explored yet;
	1218	+ * - checkpoint (L) is first reached in state {b=-24};
	1219	+ * - traversal is pruned at checkpoint (N) setting checkpoint's (L)
	1220	+ * branch count to 0, thus making it eligible for use in pruning;
	1221	+ * - checkpoint (L) is next reached in state {b=-25},
	1222	+ * this would cause NOT_EXACT comparison with a state {b=-24}
	1223	+ * while 'b' is not marked precise yet.
	1224	+ */
	1225	+ asm volatile (
	1226	+ "call %[bpf_get_prandom_u32];"
	1227	+ "if r0 == 24 goto 2f;"
	1228	+ "r1 = r10;"
	1229	+ "r1 += -8;"
	1230	+ "r2 = 0;"
	1231	+ "r3 = 5;"
	1232	+ "call %[bpf_iter_num_new];"
	1233	+ "1:"
	1234	+ "r1 = r10;"
	1235	+ "r1 += -8;"
	1236	+ "call %[bpf_iter_num_next];"
	1237	+ "if r0 != 0 goto 1b;"
	1238	+ "r1 = r10;"
	1239	+ "r1 += -8;"
	1240	+ "call %[bpf_iter_num_destroy];"
	1241	+ "r0 = 0;"
	1242	+ "exit;"
	1243	+ "2:"
	1244	+ /* loop to increase dfs_depth */
	1245	+ "r0 = 10;"
	1246	+ "3:"
	1247	+ "r0 -= 1;"
	1248	+ "if r0 != 0 goto 3b;"
	1249	+ /* end of loop */
	1250	+ "r1 = r10;"
	1251	+ "r1 += -8;"
	1252	+ "r2 = 0;"
	1253	+ "r3 = 10;"
	1254	+ "call %[bpf_iter_num_new];"
	1255	+ "r8 = -24;"
	1256	+ "main_loop_%=:"
	1257	+ "r1 = r10;"
	1258	+ "r1 += -8;"
	1259	+ "call %[bpf_iter_num_next];"
	1260	+ "if r0 == 0 goto main_loop_end_%=;"
	1261	+ /* first if */
	1262	+ "call %[bpf_get_prandom_u32];"
	1263	+ "if r0 == 77 goto unsafe_write_%=;"
	1264	+ /* second if */
	1265	+ "call %[bpf_get_prandom_u32];"
	1266	+ "if r0 == 42 goto poison_r8_%=;"
	1267	+ /* iterate */
	1268	+ "goto main_loop_%=;"
	1269	+ "main_loop_end_%=:"
	1270	+ "r1 = r10;"
	1271	+ "r1 += -8;"
	1272	+ "call %[bpf_iter_num_destroy];"
	1273	+ "r0 = 0;"
	1274	+ "exit;"
	1275	+
	1276	+ "unsafe_write_%=:"
	1277	+ "r0 = r10;"
	1278	+ "r0 += r8;"
	1279	+ "r1 = 7;"
	1280	+ "(u64 )(r0 + 0) = r1;"
	1281	+ "goto main_loop_end_%=;"
	1282	+
	1283	+ "poison_r8_%=:"
	1284	+ "r8 = -25;"
	1285	+ "goto main_loop_%=;"
	1286	+ :
	1287	+ : __imm(bpf_get_prandom_u32),
	1288	+ __imm(bpf_iter_num_new),
	1289	+ __imm(bpf_iter_num_next),
	1290	+ __imm(bpf_iter_num_destroy)
	1291	+ : __clobber_all
	1292	+ );
	1293	+}
	1294	+
1177	1295	SEC("?raw_tp")
1178	1296	__success
1179	1297	__naked int triple_continue(void)
@@ -1512,4 +1630,25 @@ int iter_destroy_bad_arg(const void *ctx) · 332 unmodified lines
1512	1630	return 0;
1513	1631	}
1514	1632
	1633	+SEC("raw_tp")
	1634	+__success
	1635	+int clean_live_states(const void *ctx)
	1636	+{
	1637	+ char buf[1];
	1638	+ int i, j, k, l, m, n, o;
	1639	+
	1640	+ bpf_for(i, 0, 10)
	1641	+ bpf_for(j, 0, 10)
	1642	+ bpf_for(k, 0, 10)
	1643	+ bpf_for(l, 0, 10)
	1644	+ bpf_for(m, 0, 10)
	1645	+ bpf_for(n, 0, 10)
	1646	+ bpf_for(o, 0, 10) {
	1647	+ if (unlikely(bpf_get_prandom_u32()))
	1648	+ buf[0] = 42;
	1649	+ bpf_printk("%s", buf);
	1650	+ }
	1651	+ return 0;
	1652	+}
	1653	+
1515	1654	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/netns_cookie_prog.c+9−0

@@ -28,6 +28,7 @@ struct { · 27 unmodified lines
28	28	} sock_map SEC(".maps");
29	29
30	30	int tcx_init_netns_cookie, tcx_netns_cookie;
	31	+int cgroup_skb_init_netns_cookie, cgroup_skb_netns_cookie;
31	32
32	33	SEC("sockops")
33	34	int get_netns_cookie_sockops(struct bpf_sock_ops *ctx)
@@ -91,4 +92,12 @@ int get_netns_cookie_tcx(struct __sk_buff *skb) · 57 unmodified lines
91	92	return TCX_PASS;
92	93	}
93	94
	95	+SEC("cgroup_skb/ingress")
	96	+int get_netns_cookie_cgroup_skb(struct __sk_buff *skb)
	97	+{
	98	+ cgroup_skb_init_netns_cookie = bpf_get_netns_cookie(NULL);
	99	+ cgroup_skb_netns_cookie = bpf_get_netns_cookie(skb);
	100	+ return SK_PASS;
	101	+}
	102	+
94	103	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/preempt_lock.c+67−1

@@ -134,7 +134,7 @@ int __noinline preempt_global_subprog(void) · 133 unmodified lines
134	134	}
135	135
136	136	SEC("?tc")
137		−__failure __msg("global function calls are not allowed with preemption disabled")
	137	+__success
138	138	int preempt_global_subprog_test(struct __sk_buff *ctx)
139	139	{
140	140	preempt_disable();
@@ -143,4 +143,70 @@ int preempt_global_subprog_test(struct __sk_buff *ctx) · 2 unmodified lines
143	143	return 0;
144	144	}
145	145
	146	+int __noinline
	147	+global_subprog(int i)
	148	+{
	149	+ if (i)
	150	+ bpf_printk("%p", &i);
	151	+ return i;
	152	+}
	153	+
	154	+int __noinline
	155	+global_sleepable_helper_subprog(int i)
	156	+{
	157	+ if (i)
	158	+ bpf_copy_from_user(&i, sizeof(i), NULL);
	159	+ return i;
	160	+}
	161	+
	162	+int __noinline
	163	+global_sleepable_kfunc_subprog(int i)
	164	+{
	165	+ if (i)
	166	+ bpf_copy_from_user_str(&i, sizeof(i), NULL, 0);
	167	+ global_subprog(i);
	168	+ return i;
	169	+}
	170	+
	171	+int __noinline
	172	+global_subprog_calling_sleepable_global(int i)
	173	+{
	174	+ if (!i)
	175	+ global_sleepable_kfunc_subprog(i);
	176	+ return i;
	177	+}
	178	+
	179	+SEC("?syscall")
	180	+__failure __msg("global functions that may sleep are not allowed in non-sleepable context")
	181	+int preempt_global_sleepable_helper_subprog(struct __sk_buff *ctx)
	182	+{
	183	+ preempt_disable();
	184	+ if (ctx->mark)
	185	+ global_sleepable_helper_subprog(ctx->mark);
	186	+ preempt_enable();
	187	+ return 0;
	188	+}
	189	+
	190	+SEC("?syscall")
	191	+__failure __msg("global functions that may sleep are not allowed in non-sleepable context")
	192	+int preempt_global_sleepable_kfunc_subprog(struct __sk_buff *ctx)
	193	+{
	194	+ preempt_disable();
	195	+ if (ctx->mark)
	196	+ global_sleepable_kfunc_subprog(ctx->mark);
	197	+ preempt_enable();
	198	+ return 0;
	199	+}
	200	+
	201	+SEC("?syscall")
	202	+__failure __msg("global functions that may sleep are not allowed in non-sleepable context")
	203	+int preempt_global_sleepable_subprog_indirect(struct __sk_buff *ctx)
	204	+{
	205	+ preempt_disable();
	206	+ if (ctx->mark)
	207	+ global_subprog_calling_sleepable_global(ctx->mark);
	208	+ preempt_enable();
	209	+ return 0;
	210	+}
	211	+
146	212	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/prepare.c+28−0

@@ -0,0 +1,28 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2025 Meta */
	3	+#include <vmlinux.h>
	4	+#include <bpf/bpf_helpers.h>
	5	+//#include <bpf/bpf_tracing.h>
	6	+
	7	+char _license[] SEC("license") = "GPL";
	8	+
	9	+int err;
	10	+
	11	+struct {
	12	+ __uint(type, BPF_MAP_TYPE_RINGBUF);
	13	+ __uint(max_entries, 4096);
	14	+} ringbuf SEC(".maps");
	15	+
	16	+struct {
	17	+ __uint(type, BPF_MAP_TYPE_ARRAY);
	18	+ __uint(max_entries, 1);
	19	+ __type(key, __u32);
	20	+ __type(value, __u32);
	21	+} array_map SEC(".maps");
	22	+
	23	+SEC("cgroup_skb/egress")
	24	+int program(struct __sk_buff *skb)
	25	+{
	26	+ err = 0;
	27	+ return 0;
	28	+}

tools/testing/selftests/bpf/progs/priv_freplace_prog.c+13−0

@@ -0,0 +1,13 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
	3	+
	4	+#include "vmlinux.h"
	5	+#include <bpf/bpf_helpers.h>
	6	+
	7	+char _license[] SEC("license") = "GPL";
	8	+
	9	+SEC("freplace/xdp_prog1")
	10	+int new_xdp_prog2(struct xdp_md *xd)
	11	+{
	12	+ return XDP_DROP;
	13	+}

tools/testing/selftests/bpf/progs/priv_prog.c+3−3

@@ -6,8 +6,8 @@ · 5 unmodified lines
6	6
7	7	char _license[] SEC("license") = "GPL";
8	8
9		−SEC("kprobe")
10		−int kprobe_prog(void *ctx)
	9	+SEC("xdp")
	10	+int xdp_prog1(struct xdp_md *xdp)
11	11	{
12		− return 1;
	12	+ return XDP_DROP;
13	13	}

tools/testing/selftests/bpf/progs/pro_epilogue_with_kfunc.c+88−0

@@ -0,0 +1,88 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
	3	+
	4	+#include <vmlinux.h>
	5	+#include <bpf/bpf_tracing.h>
	6	+#include "bpf_misc.h"
	7	+#include "../test_kmods/bpf_testmod.h"
	8	+#include "../test_kmods/bpf_testmod_kfunc.h"
	9	+
	10	+char _license[] SEC("license") = "GPL";
	11	+
	12	+void __kfunc_btf_root(void)
	13	+{
	14	+ bpf_kfunc_st_ops_inc10(NULL);
	15	+}
	16	+
	17	+static __noinline __used int subprog(struct st_ops_args *args)
	18	+{
	19	+ args->a += 1;
	20	+ return args->a;
	21	+}
	22	+
	23	+__success
	24	+/* prologue */
	25	+__xlated("0: r8 = r1")
	26	+__xlated("1: r1 = 0")
	27	+__xlated("2: call kernel-function")
	28	+__xlated("3: if r0 != 0x0 goto pc+5")
	29	+__xlated("4: r6 = (u64 )(r8 +0)")
	30	+__xlated("5: r7 = (u64 )(r6 +0)")
	31	+__xlated("6: r7 += 1000")
	32	+__xlated("7: (u64 )(r6 +0) = r7")
	33	+__xlated("8: goto pc+2")
	34	+__xlated("9: r1 = r0")
	35	+__xlated("10: call kernel-function")
	36	+__xlated("11: r1 = r8")
	37	+/* save __u64 ctx to stack /
	38	+__xlated("12: (u64 )(r10 -8) = r1")
	39	+/* main prog */
	40	+__xlated("13: r1 = (u64 )(r1 +0)")
	41	+__xlated("14: r6 = r1")
	42	+__xlated("15: call kernel-function")
	43	+__xlated("16: r1 = r6")
	44	+__xlated("17: call pc+")
	45	+/* epilogue */
	46	+__xlated("18: r1 = 0")
	47	+__xlated("19: r6 = 0")
	48	+__xlated("20: call kernel-function")
	49	+__xlated("21: if r0 != 0x0 goto pc+6")
	50	+__xlated("22: r1 = (u64 )(r10 -8)")
	51	+__xlated("23: r1 = (u64 )(r1 +0)")
	52	+__xlated("24: r6 = (u64 )(r1 +0)")
	53	+__xlated("25: r6 += 10000")
	54	+__xlated("26: (u64 )(r1 +0) = r6")
	55	+__xlated("27: goto pc+2")
	56	+__xlated("28: r1 = r0")
	57	+__xlated("29: call kernel-function")
	58	+__xlated("30: r0 = r6")
	59	+__xlated("31: r0 *= 2")
	60	+__xlated("32: exit")
	61	+SEC("struct_ops/test_pro_epilogue")
	62	+__naked int test_kfunc_pro_epilogue(void)
	63	+{
	64	+ asm volatile (
	65	+ "r1 = (u64 )(r1 +0);"
	66	+ "r6 = r1;"
	67	+ "call %[bpf_kfunc_st_ops_inc10];"
	68	+ "r1 = r6;"
	69	+ "call subprog;"
	70	+ "exit;"
	71	+ :
	72	+ : __imm(bpf_kfunc_st_ops_inc10)
	73	+ : __clobber_all);
	74	+}
	75	+
	76	+SEC("syscall")
	77	+__retval(22022) /* (PROLOGUE_A [1000] + KFUNC_INC10 + SUBPROG_A [1] + EPILOGUE_A [10000]) * 2 */
	78	+int syscall_pro_epilogue(void *ctx)
	79	+{
	80	+ struct st_ops_args args = {};
	81	+
	82	+ return bpf_kfunc_st_ops_test_pro_epilogue(&args);
	83	+}
	84	+
	85	+SEC(".struct_ops.link")
	86	+struct bpf_testmod_st_ops pro_epilogue_with_kfunc = {
	87	+ .test_pro_epilogue = (void *)test_kfunc_pro_epilogue,
	88	+};

tools/testing/selftests/bpf/progs/rcu_read_lock.c+60−1

@@ -242,7 +242,8 @@ out: · 241 unmodified lines
242	242	}
243	243
244	244	SEC("?lsm.s/bpf")
245		−int BPF_PROG(inproper_sleepable_kfunc, int cmd, union bpf_attr *attr, unsigned int size)
	245	+int BPF_PROG(inproper_sleepable_kfunc, int cmd, union bpf_attr *attr, unsigned int size,
	246	+ bool kernel)
246	247	{
247	248	struct bpf_key *bkey;
248	249
@@ -439,3 +440,61 @@ int rcu_read_lock_global_subprog_unlock(void *ctx) · 190 unmodified lines
439	440	ret += global_subprog_unlock(ret);
440	441	return 0;
441	442	}
	443	+
	444	+int __noinline
	445	+global_sleepable_helper_subprog(int i)
	446	+{
	447	+ if (i)
	448	+ bpf_copy_from_user(&i, sizeof(i), NULL);
	449	+ return i;
	450	+}
	451	+
	452	+int __noinline
	453	+global_sleepable_kfunc_subprog(int i)
	454	+{
	455	+ if (i)
	456	+ bpf_copy_from_user_str(&i, sizeof(i), NULL, 0);
	457	+ global_subprog(i);
	458	+ return i;
	459	+}
	460	+
	461	+int __noinline
	462	+global_subprog_calling_sleepable_global(int i)
	463	+{
	464	+ if (!i)
	465	+ global_sleepable_kfunc_subprog(i);
	466	+ return i;
	467	+}
	468	+
	469	+SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
	470	+int rcu_read_lock_sleepable_helper_global_subprog(void *ctx)
	471	+{
	472	+ volatile int ret = 0;
	473	+
	474	+ bpf_rcu_read_lock();
	475	+ ret += global_sleepable_helper_subprog(ret);
	476	+ bpf_rcu_read_unlock();
	477	+ return 0;
	478	+}
	479	+
	480	+SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
	481	+int rcu_read_lock_sleepable_kfunc_global_subprog(void *ctx)
	482	+{
	483	+ volatile int ret = 0;
	484	+
	485	+ bpf_rcu_read_lock();
	486	+ ret += global_sleepable_kfunc_subprog(ret);
	487	+ bpf_rcu_read_unlock();
	488	+ return 0;
	489	+}
	490	+
	491	+SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
	492	+int rcu_read_lock_sleepable_global_subprog_indirect(void *ctx)
	493	+{
	494	+ volatile int ret = 0;
	495	+
	496	+ bpf_rcu_read_lock();
	497	+ ret += global_subprog_calling_sleepable_global(ret);
	498	+ bpf_rcu_read_unlock();
	499	+ return 0;
	500	+}

tools/testing/selftests/bpf/progs/read_vsyscall.c+9−2

@@ -8,14 +8,16 @@ · 7 unmodified lines
8	8
9	9	int target_pid = 0;
10	10	void *user_ptr = 0;
11		−int read_ret[9];
	11	+int read_ret[10];
12	12
13	13	char _license[] SEC("license") = "GPL";
14	14
15	15	/*
16		− * This is the only kfunc, the others are helpers
	16	+ * These are the kfuncs, the others are helpers
17	17	*/
18	18	int bpf_copy_from_user_str(void dst, u32, const void , u64) __weak __ksym;
	19	+int bpf_copy_from_user_task_str(void dst, u32, const void ,
	20	+ struct task_struct *, u64) __weak __ksym;
19	21
20	22	SEC("fentry/" SYS_PREFIX "sys_nanosleep")
21	23	int do_probe_read(void *ctx)
@@ -47,6 +49,11 @@ int do_copy_from_user(void *ctx) · 25 unmodified lines
47	49	read_ret[7] = bpf_copy_from_user_task(buf, sizeof(buf), user_ptr,
48	50	bpf_get_current_task_btf(), 0);
49	51	read_ret[8] = bpf_copy_from_user_str((char *)buf, sizeof(buf), user_ptr, 0);
	52	+ read_ret[9] = bpf_copy_from_user_task_str((char *)buf,
	53	+ sizeof(buf),
	54	+ user_ptr,
	55	+ bpf_get_current_task_btf(),
	56	+ 0);
50	57
51	58	return 0;
52	59	}

tools/testing/selftests/bpf/progs/set_global_vars.c+47−0

@@ -0,0 +1,47 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
	3	+#include "bpf_experimental.h"
	4	+#include <bpf/bpf_helpers.h>
	5	+#include "bpf_misc.h"
	6	+#include <stdbool.h>
	7	+
	8	+char _license[] SEC("license") = "GPL";
	9	+
	10	+enum Enum { EA1 = 0, EA2 = 11 };
	11	+enum Enumu64 {EB1 = 0llu, EB2 = 12llu };
	12	+enum Enums64 { EC1 = 0ll, EC2 = 13ll };
	13	+
	14	+const volatile __s64 var_s64 = -1;
	15	+const volatile __u64 var_u64 = 0;
	16	+const volatile __s32 var_s32 = -1;
	17	+const volatile __u32 var_u32 = 0;
	18	+const volatile __s16 var_s16 = -1;
	19	+const volatile __u16 var_u16 = 0;
	20	+const volatile __s8 var_s8 = -1;
	21	+const volatile __u8 var_u8 = 0;
	22	+const volatile enum Enum var_ea = EA1;
	23	+const volatile enum Enumu64 var_eb = EB1;
	24	+const volatile enum Enums64 var_ec = EC1;
	25	+const volatile bool var_b = false;
	26	+
	27	+char arr[4] = {0};
	28	+
	29	+SEC("socket")
	30	+int test_set_globals(void *ctx)
	31	+{
	32	+ volatile __s8 a;
	33	+
	34	+ a = var_s64;
	35	+ a = var_u64;
	36	+ a = var_s32;
	37	+ a = var_u32;
	38	+ a = var_s16;
	39	+ a = var_u16;
	40	+ a = var_s8;
	41	+ a = var_u8;
	42	+ a = var_ea;
	43	+ a = var_eb;
	44	+ a = var_ec;
	45	+ a = var_b;
	46	+ return a;
	47	+}

tools/testing/selftests/bpf/progs/strncmp_bench.c+4−1

@@ -35,7 +35,10 @@ static __always_inline int local_strncmp(const char *s1, unsigned int sz, · 34 unmodified lines
35	35	SEC("tp/syscalls/sys_enter_getpgid")
36	36	int strncmp_no_helper(void *ctx)
37	37	{
38		− if (local_strncmp(str, cmp_str_len + 1, target) < 0)
	38	+ const char *target_str = target;
	39	+
	40	+ barrier_var(target_str);
	41	+ if (local_strncmp(str, cmp_str_len + 1, target_str) < 0)
39	42	__sync_add_and_fetch(&hits, 1);
40	43	return 0;
41	44	}

tools/testing/selftests/bpf/progs/struct_ops_kptr_return.c+30−0

@@ -0,0 +1,30 @@
	1	+#include <vmlinux.h>
	2	+#include <bpf/bpf_tracing.h>
	3	+#include "../test_kmods/bpf_testmod.h"
	4	+#include "bpf_misc.h"
	5	+
	6	+char _license[] SEC("license") = "GPL";
	7	+
	8	+void bpf_task_release(struct task_struct *p) __ksym;
	9	+
	10	+/* This test struct_ops BPF programs returning referenced kptr. The verifier should
	11	+ * allow a referenced kptr or a NULL pointer to be returned. A referenced kptr to task
	12	+ * here is acquried automatically as the task argument is tagged with "__ref".
	13	+ */
	14	+SEC("struct_ops/test_return_ref_kptr")
	15	+struct task_struct *BPF_PROG(kptr_return, int dummy,
	16	+ struct task_struct task, struct cgroup cgrp)
	17	+{
	18	+ if (dummy % 2) {
	19	+ bpf_task_release(task);
	20	+ return NULL;
	21	+ }
	22	+ return task;
	23	+}
	24	+
	25	+SEC(".struct_ops.link")
	26	+struct bpf_testmod_ops testmod_kptr_return = {
	27	+ .test_return_ref_kptr = (void *)kptr_return,
	28	+};
	29	+
	30	+

tools/testing/selftests/bpf/progs/struct_ops_kptr_return_fail__invalid_scalar.c+26−0

@@ -0,0 +1,26 @@
	1	+#include <vmlinux.h>
	2	+#include <bpf/bpf_tracing.h>
	3	+#include "../test_kmods/bpf_testmod.h"
	4	+#include "bpf_misc.h"
	5	+
	6	+char _license[] SEC("license") = "GPL";
	7	+
	8	+struct cgroup bpf_cgroup_acquire(struct cgroup p) __ksym;
	9	+void bpf_task_release(struct task_struct *p) __ksym;
	10	+
	11	+/* This test struct_ops BPF programs returning referenced kptr. The verifier should
	12	+ * reject programs returning a non-zero scalar value.
	13	+ */
	14	+SEC("struct_ops/test_return_ref_kptr")
	15	+__failure __msg("At program exit the register R0 has smin=1 smax=1 should have been in [0, 0]")
	16	+struct task_struct *BPF_PROG(kptr_return_fail__invalid_scalar, int dummy,
	17	+ struct task_struct task, struct cgroup cgrp)
	18	+{
	19	+ bpf_task_release(task);
	20	+ return (struct task_struct *)1;
	21	+}
	22	+
	23	+SEC(".struct_ops.link")
	24	+struct bpf_testmod_ops testmod_kptr_return = {
	25	+ .test_return_ref_kptr = (void *)kptr_return_fail__invalid_scalar,
	26	+};

tools/testing/selftests/bpf/progs/struct_ops_kptr_return_fail__local_kptr.c+34−0

@@ -0,0 +1,34 @@
	1	+#include <vmlinux.h>
	2	+#include <bpf/bpf_tracing.h>
	3	+#include "../test_kmods/bpf_testmod.h"
	4	+#include "bpf_experimental.h"
	5	+#include "bpf_misc.h"
	6	+
	7	+char _license[] SEC("license") = "GPL";
	8	+
	9	+struct cgroup bpf_cgroup_acquire(struct cgroup p) __ksym;
	10	+void bpf_task_release(struct task_struct *p) __ksym;
	11	+
	12	+/* This test struct_ops BPF programs returning referenced kptr. The verifier should
	13	+ * reject programs returning a local kptr.
	14	+ */
	15	+SEC("struct_ops/test_return_ref_kptr")
	16	+__failure __msg("At program exit the register R0 is not a known value (ptr_or_null_)")
	17	+struct task_struct *BPF_PROG(kptr_return_fail__local_kptr, int dummy,
	18	+ struct task_struct task, struct cgroup cgrp)
	19	+{
	20	+ struct task_struct *t;
	21	+
	22	+ bpf_task_release(task);
	23	+
	24	+ t = bpf_obj_new(typeof(*task));
	25	+ if (!t)
	26	+ return NULL;
	27	+
	28	+ return t;
	29	+}
	30	+
	31	+SEC(".struct_ops.link")
	32	+struct bpf_testmod_ops testmod_kptr_return = {
	33	+ .test_return_ref_kptr = (void *)kptr_return_fail__local_kptr,
	34	+};

tools/testing/selftests/bpf/progs/struct_ops_kptr_return_fail__nonzero_offset.c+25−0

@@ -0,0 +1,25 @@
	1	+#include <vmlinux.h>
	2	+#include <bpf/bpf_tracing.h>
	3	+#include "../test_kmods/bpf_testmod.h"
	4	+#include "bpf_misc.h"
	5	+
	6	+char _license[] SEC("license") = "GPL";
	7	+
	8	+struct cgroup bpf_cgroup_acquire(struct cgroup p) __ksym;
	9	+void bpf_task_release(struct task_struct *p) __ksym;
	10	+
	11	+/* This test struct_ops BPF programs returning referenced kptr. The verifier should
	12	+ * reject programs returning a modified referenced kptr.
	13	+ */
	14	+SEC("struct_ops/test_return_ref_kptr")
	15	+__failure __msg("dereference of modified trusted_ptr_ ptr R0 off={{[0-9]+}} disallowed")
	16	+struct task_struct *BPF_PROG(kptr_return_fail__nonzero_offset, int dummy,
	17	+ struct task_struct task, struct cgroup cgrp)
	18	+{
	19	+ return (struct task_struct *)&task->jobctl;
	20	+}
	21	+
	22	+SEC(".struct_ops.link")
	23	+struct bpf_testmod_ops testmod_kptr_return = {
	24	+ .test_return_ref_kptr = (void *)kptr_return_fail__nonzero_offset,
	25	+};

tools/testing/selftests/bpf/progs/struct_ops_kptr_return_fail__wrong_type.c+30−0

@@ -0,0 +1,30 @@
	1	+#include <vmlinux.h>
	2	+#include <bpf/bpf_tracing.h>
	3	+#include "../test_kmods/bpf_testmod.h"
	4	+#include "bpf_misc.h"
	5	+
	6	+char _license[] SEC("license") = "GPL";
	7	+
	8	+struct cgroup bpf_cgroup_acquire(struct cgroup p) __ksym;
	9	+void bpf_task_release(struct task_struct *p) __ksym;
	10	+
	11	+/* This test struct_ops BPF programs returning referenced kptr. The verifier should
	12	+ * reject programs returning a referenced kptr of the wrong type.
	13	+ */
	14	+SEC("struct_ops/test_return_ref_kptr")
	15	+__failure __msg("At program exit the register R0 is not a known value (ptr_or_null_)")
	16	+struct task_struct *BPF_PROG(kptr_return_fail__wrong_type, int dummy,
	17	+ struct task_struct task, struct cgroup cgrp)
	18	+{
	19	+ struct task_struct *ret;
	20	+
	21	+ ret = (struct task_struct *)bpf_cgroup_acquire(cgrp);
	22	+ bpf_task_release(task);
	23	+
	24	+ return ret;
	25	+}
	26	+
	27	+SEC(".struct_ops.link")
	28	+struct bpf_testmod_ops testmod_kptr_return = {
	29	+ .test_return_ref_kptr = (void *)kptr_return_fail__wrong_type,
	30	+};

tools/testing/selftests/bpf/progs/struct_ops_refcounted.c+31−0

@@ -0,0 +1,31 @@
	1	+#include <vmlinux.h>
	2	+#include <bpf/bpf_tracing.h>
	3	+#include "../test_kmods/bpf_testmod.h"
	4	+#include "bpf_misc.h"
	5	+
	6	+char _license[] SEC("license") = "GPL";
	7	+
	8	+__attribute__((nomerge)) extern void bpf_task_release(struct task_struct *p) __ksym;
	9	+
	10	+/* This is a test BPF program that uses struct_ops to access a referenced
	11	+ * kptr argument. This is a test for the verifier to ensure that it
	12	+ * 1) recongnizes the task as a referenced object (i.e., ref_obj_id > 0), and
	13	+ * 2) the same reference can be acquired from multiple paths as long as it
	14	+ * has not been released.
	15	+ */
	16	+SEC("struct_ops/test_refcounted")
	17	+int BPF_PROG(refcounted, int dummy, struct task_struct *task)
	18	+{
	19	+ if (dummy == 1)
	20	+ bpf_task_release(task);
	21	+ else
	22	+ bpf_task_release(task);
	23	+ return 0;
	24	+}
	25	+
	26	+SEC(".struct_ops.link")
	27	+struct bpf_testmod_ops testmod_refcounted = {
	28	+ .test_refcounted = (void *)refcounted,
	29	+};
	30	+
	31	+

tools/testing/selftests/bpf/progs/struct_ops_refcounted_fail__global_subprog.c+39−0

@@ -0,0 +1,39 @@
	1	+#include <vmlinux.h>
	2	+#include <bpf/bpf_tracing.h>
	3	+#include "../test_kmods/bpf_testmod.h"
	4	+#include "bpf_misc.h"
	5	+
	6	+char _license[] SEC("license") = "GPL";
	7	+
	8	+extern void bpf_task_release(struct task_struct *p) __ksym;
	9	+
	10	+__noinline int subprog_release(__u64 *ctx __arg_ctx)
	11	+{
	12	+ struct task_struct task = (struct task_struct )ctx[1];
	13	+ int dummy = (int)ctx[0];
	14	+
	15	+ bpf_task_release(task);
	16	+
	17	+ return dummy + 1;
	18	+}
	19	+
	20	+/* Test that the verifier rejects a program that contains a global
	21	+ * subprogram with referenced kptr arguments
	22	+ */
	23	+SEC("struct_ops/test_refcounted")
	24	+__failure __log_level(2)
	25	+__msg("Validating subprog_release() func#1...")
	26	+__msg("invalid bpf_context access off=8. Reference may already be released")
	27	+int refcounted_fail__global_subprog(unsigned long long *ctx)
	28	+{
	29	+ struct task_struct task = (struct task_struct )ctx[1];
	30	+
	31	+ bpf_task_release(task);
	32	+
	33	+ return subprog_release(ctx);
	34	+}
	35	+
	36	+SEC(".struct_ops.link")
	37	+struct bpf_testmod_ops testmod_ref_acquire = {
	38	+ .test_refcounted = (void *)refcounted_fail__global_subprog,
	39	+};

tools/testing/selftests/bpf/progs/struct_ops_refcounted_fail__ref_leak.c+22−0

@@ -0,0 +1,22 @@
	1	+#include <vmlinux.h>
	2	+#include <bpf/bpf_tracing.h>
	3	+#include "../test_kmods/bpf_testmod.h"
	4	+#include "bpf_misc.h"
	5	+
	6	+char _license[] SEC("license") = "GPL";
	7	+
	8	+/* Test that the verifier rejects a program that acquires a referenced
	9	+ * kptr through context without releasing the reference
	10	+ */
	11	+SEC("struct_ops/test_refcounted")
	12	+__failure __msg("Unreleased reference id=1 alloc_insn=0")
	13	+int BPF_PROG(refcounted_fail__ref_leak, int dummy,
	14	+ struct task_struct *task)
	15	+{
	16	+ return 0;
	17	+}
	18	+
	19	+SEC(".struct_ops.link")
	20	+struct bpf_testmod_ops testmod_ref_acquire = {
	21	+ .test_refcounted = (void *)refcounted_fail__ref_leak,
	22	+};

tools/testing/selftests/bpf/progs/struct_ops_refcounted_fail__tail_call.c+36−0

@@ -0,0 +1,36 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
	3	+#include <vmlinux.h>
	4	+#include <bpf/bpf_tracing.h>
	5	+#include "../test_kmods/bpf_testmod.h"
	6	+#include "bpf_misc.h"
	7	+
	8	+char _license[] SEC("license") = "GPL";
	9	+
	10	+struct {
	11	+ __uint(type, BPF_MAP_TYPE_PROG_ARRAY);
	12	+ __uint(max_entries, 1);
	13	+ __uint(key_size, sizeof(__u32));
	14	+ __uint(value_size, sizeof(__u32));
	15	+} prog_array SEC(".maps");
	16	+
	17	+/* Test that the verifier rejects a program with referenced kptr arguments
	18	+ * that tail call
	19	+ */
	20	+SEC("struct_ops/test_refcounted")
	21	+__failure __msg("program with __ref argument cannot tail call")
	22	+int refcounted_fail__tail_call(unsigned long long *ctx)
	23	+{
	24	+ struct task_struct task = (struct task_struct )ctx[1];
	25	+
	26	+ bpf_task_release(task);
	27	+ bpf_tail_call(ctx, &prog_array, 0);
	28	+
	29	+ return 0;
	30	+}
	31	+
	32	+SEC(".struct_ops.link")
	33	+struct bpf_testmod_ops testmod_ref_acquire = {
	34	+ .test_refcounted = (void *)refcounted_fail__tail_call,
	35	+};
	36	+

tools/testing/selftests/bpf/progs/summarization.c+78−0

@@ -0,0 +1,78 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+
	3	+#include <vmlinux.h>
	4	+#include <bpf/bpf_helpers.h>
	5	+#include "bpf_misc.h"
	6	+
	7	+__noinline
	8	+long changes_pkt_data(struct __sk_buff *sk)
	9	+{
	10	+ return bpf_skb_pull_data(sk, 0);
	11	+}
	12	+
	13	+__noinline __weak
	14	+long does_not_change_pkt_data(struct __sk_buff *sk)
	15	+{
	16	+ return 0;
	17	+}
	18	+
	19	+SEC("?tc")
	20	+int main_changes_with_subprogs(struct __sk_buff *sk)
	21	+{
	22	+ changes_pkt_data(sk);
	23	+ does_not_change_pkt_data(sk);
	24	+ return 0;
	25	+}
	26	+
	27	+SEC("?tc")
	28	+int main_changes(struct __sk_buff *sk)
	29	+{
	30	+ bpf_skb_pull_data(sk, 0);
	31	+ return 0;
	32	+}
	33	+
	34	+SEC("?tc")
	35	+int main_does_not_change(struct __sk_buff *sk)
	36	+{
	37	+ return 0;
	38	+}
	39	+
	40	+__noinline
	41	+long might_sleep(struct pt_regs *ctx __arg_ctx)
	42	+{
	43	+ int i;
	44	+
	45	+ bpf_copy_from_user(&i, sizeof(i), NULL);
	46	+ return i;
	47	+}
	48	+
	49	+__noinline __weak
	50	+long does_not_sleep(struct pt_regs *ctx __arg_ctx)
	51	+{
	52	+ return 0;
	53	+}
	54	+
	55	+SEC("?uprobe.s")
	56	+int main_might_sleep_with_subprogs(struct pt_regs *ctx)
	57	+{
	58	+ might_sleep(ctx);
	59	+ does_not_sleep(ctx);
	60	+ return 0;
	61	+}
	62	+
	63	+SEC("?uprobe.s")
	64	+int main_might_sleep(struct pt_regs *ctx)
	65	+{
	66	+ int i;
	67	+
	68	+ bpf_copy_from_user(&i, sizeof(i), NULL);
	69	+ return i;
	70	+}
	71	+
	72	+SEC("?uprobe.s")
	73	+int main_does_not_sleep(struct pt_regs *ctx)
	74	+{
	75	+ return 0;
	76	+}
	77	+
	78	+char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/summarization_freplace.crenamed from tools/testing/selftests/bpf/progs/changes_pkt_data_freplace.c+16−1

@@ -1,6 +1,6 @@
1	1	// SPDX-License-Identifier: GPL-2.0
2	2
3		−#include <linux/bpf.h>
	3	+#include <vmlinux.h>
4	4	#include <bpf/bpf_helpers.h>
5	5
6	6	SEC("?freplace")
@@ -15,4 +15,19 @@ long does_not_change_pkt_data(struct __sk_buff *sk) · 8 unmodified lines
15	15	return 0;
16	16	}
17	17
	18	+SEC("?freplace")
	19	+long might_sleep(struct pt_regs *ctx)
	20	+{
	21	+ int i;
	22	+
	23	+ bpf_copy_from_user(&i, sizeof(i), NULL);
	24	+ return i;
	25	+}
	26	+
	27	+SEC("?freplace")
	28	+long does_not_sleep(struct pt_regs *ctx)
	29	+{
	30	+ return 0;
	31	+}
	32	+
18	33	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/test_cgroup1_hierarchy.c+2−2

@@ -51,13 +51,13 @@ out: · 50 unmodified lines
51	51	}
52	52
53	53	SEC("lsm/bpf")
54		−int BPF_PROG(lsm_run, int cmd, union bpf_attr *attr, unsigned int size)
	54	+int BPF_PROG(lsm_run, int cmd, union bpf_attr *attr, unsigned int size, bool kernel)
55	55	{
56	56	return bpf_link_create_verify(cmd);
57	57	}
58	58
59	59	SEC("lsm.s/bpf")
60		−int BPF_PROG(lsm_s_run, int cmd, union bpf_attr *attr, unsigned int size)
	60	+int BPF_PROG(lsm_s_run, int cmd, union bpf_attr *attr, unsigned int size, bool kernel)
61	61	{
62	62	return bpf_link_create_verify(cmd);
63	63	}

tools/testing/selftests/bpf/progs/test_core_reloc_arrays.c+5−0

@@ -15,6 +15,7 @@ struct { · 14 unmodified lines
15	15
16	16	struct core_reloc_arrays_output {
17	17	int a2;
	18	+ int a3;
18	19	char b123;
19	20	int c1c;
20	21	int d00d;
@@ -41,6 +42,7 @@ int test_core_arrays(void *ctx) · 20 unmodified lines
41	42	{
42	43	struct core_reloc_arrays in = (void )&data.in;
43	44	struct core_reloc_arrays_output out = (void )&data.out;
	45	+ int *a;
44	46
45	47	if (CORE_READ(&out->a2, &in->a[2]))
46	48	return 1;
@@ -53,6 +55,9 @@ int test_core_arrays(void *ctx) · 6 unmodified lines
53	55	if (CORE_READ(&out->f01c, &in->f[0][1].c))
54	56	return 1;
55	57
	58	+ a = __builtin_preserve_access_index(({ in->a; }));
	59	+ out->a3 = a[0] + a[1] + a[2] + a[3];
	60	+
56	61	return 0;
57	62	}
58	63

tools/testing/selftests/bpf/progs/test_get_xattr.c+24−4

@@ -6,6 +6,7 @@ · 5 unmodified lines
6	6	#include <bpf/bpf_helpers.h>
7	7	#include <bpf/bpf_tracing.h>
8	8	#include "bpf_kfuncs.h"
	9	+#include "bpf_misc.h"
9	10
10	11	char _license[] SEC("license") = "GPL";
11	12
@@ -17,12 +18,23 @@ static const char expected_value[] = "hello"; · 5 unmodified lines
17	18	char value1[32];
18	19	char value2[32];
19	20
	21	+/* Matches caller of test_get_xattr() in prog_tests/fs_kfuncs.c */
	22	+static const char xattr_names[][64] = {
	23	+ /* The following work. */
	24	+ "user.kfuncs",
	25	+ "security.bpf.xxx",
	26	+
	27	+ /* The following do not work. */
	28	+ "security.bpf",
	29	+ "security.selinux"
	30	+};
	31	+
20	32	SEC("lsm.s/file_open")
21	33	int BPF_PROG(test_file_open, struct file *f)
22	34	{
23	35	struct bpf_dynptr value_ptr;
24	36	__u32 pid;
25		− int ret;
	37	+ int ret, i;
26	38
27	39	pid = bpf_get_current_pid_tgid() >> 32;
28	40	if (pid != monitored_pid)
@@ -30,7 +42,11 @@ int BPF_PROG(test_file_open, struct file *f) · 1 unmodified line
30	42
31	43	bpf_dynptr_from_mem(value1, sizeof(value1), 0, &value_ptr);
32	44
33		− ret = bpf_get_file_xattr(f, "user.kfuncs", &value_ptr);
	45	+ for (i = 0; i < ARRAY_SIZE(xattr_names); i++) {
	46	+ ret = bpf_get_file_xattr(f, xattr_names[i], &value_ptr);
	47	+ if (ret == sizeof(expected_value))
	48	+ break;
	49	+ }
34	50	if (ret != sizeof(expected_value))
35	51	return 0;
36	52	if (bpf_strncmp(value1, ret, expected_value))
@@ -44,7 +60,7 @@ int BPF_PROG(test_inode_getxattr, struct dentry dentry, char name) · 7 unmodified lines
44	60	{
45	61	struct bpf_dynptr value_ptr;
46	62	__u32 pid;
47		− int ret;
	63	+ int ret, i;
48	64
49	65	pid = bpf_get_current_pid_tgid() >> 32;
50	66	if (pid != monitored_pid)
@@ -52,7 +68,11 @@ int BPF_PROG(test_inode_getxattr, struct dentry dentry, char name) · 1 unmodified line
52	68
53	69	bpf_dynptr_from_mem(value2, sizeof(value2), 0, &value_ptr);
54	70
55		− ret = bpf_get_dentry_xattr(dentry, "user.kfuncs", &value_ptr);
	71	+ for (i = 0; i < ARRAY_SIZE(xattr_names); i++) {
	72	+ ret = bpf_get_dentry_xattr(dentry, xattr_names[i], &value_ptr);
	73	+ if (ret == sizeof(expected_value))
	74	+ break;
	75	+ }
56	76	if (ret != sizeof(expected_value))
57	77	return 0;
58	78	if (bpf_strncmp(value2, ret, expected_value))

tools/testing/selftests/bpf/progs/test_kernel_flag.c+28−0

@@ -0,0 +1,28 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+
	3	+/*
	4	+ * Copyright (C) 2025 Microsoft Corporation
	5	+ *
	6	+ * Author: Blaise Boscaccy <bboscaccy@linux.microsoft.com>
	7	+ */
	8	+
	9	+#include "vmlinux.h"
	10	+#include <errno.h>
	11	+#include <bpf/bpf_helpers.h>
	12	+#include <bpf/bpf_tracing.h>
	13	+
	14	+char _license[] SEC("license") = "GPL";
	15	+
	16	+__u32 monitored_tid;
	17	+
	18	+SEC("lsm.s/bpf")
	19	+int BPF_PROG(bpf, int cmd, union bpf_attr *attr, unsigned int size, bool kernel)
	20	+{
	21	+ __u32 tid;
	22	+
	23	+ tid = bpf_get_current_pid_tgid() & 0xFFFFFFFF;
	24	+ if (!kernel \|\| tid != monitored_tid)
	25	+ return 0;
	26	+ else
	27	+ return -EINVAL;
	28	+}

tools/testing/selftests/bpf/progs/test_kfunc_dynptr_param.c+3−3

@@ -36,7 +36,7 @@ char _license[] SEC("license") = "GPL"; · 35 unmodified lines
36	36
37	37	SEC("?lsm.s/bpf")
38	38	__failure __msg("cannot pass in dynptr at an offset=-8")
39		−int BPF_PROG(not_valid_dynptr, int cmd, union bpf_attr *attr, unsigned int size)
	39	+int BPF_PROG(not_valid_dynptr, int cmd, union bpf_attr *attr, unsigned int size, bool kernel)
40	40	{
41	41	unsigned long val;
42	42
@@ -46,7 +46,7 @@ int BPF_PROG(not_valid_dynptr, int cmd, union bpf_attr *attr, unsigned int size) · 3 unmodified lines
46	46
47	47	SEC("?lsm.s/bpf")
48	48	__failure __msg("arg#0 expected pointer to stack or const struct bpf_dynptr")
49		−int BPF_PROG(not_ptr_to_stack, int cmd, union bpf_attr *attr, unsigned int size)
	49	+int BPF_PROG(not_ptr_to_stack, int cmd, union bpf_attr *attr, unsigned int size, bool kernel)
50	50	{
51	51	unsigned long val = 0;
52	52
@@ -55,7 +55,7 @@ int BPF_PROG(not_ptr_to_stack, int cmd, union bpf_attr *attr, unsigned int size) · 2 unmodified lines
55	55	}
56	56
57	57	SEC("lsm.s/bpf")
58		−int BPF_PROG(dynptr_data_null, int cmd, union bpf_attr *attr, unsigned int size)
	58	+int BPF_PROG(dynptr_data_null, int cmd, union bpf_attr *attr, unsigned int size, bool kernel)
59	59	{
60	60	struct bpf_key *trusted_keyring;
61	61	struct bpf_dynptr ptr;

tools/testing/selftests/bpf/progs/test_lookup_key.c+1−1

@@ -23,7 +23,7 @@ extern struct bpf_key *bpf_lookup_system_key(__u64 id) __ksym; · 22 unmodified lines
23	23	extern void bpf_key_put(struct bpf_key *key) __ksym;
24	24
25	25	SEC("lsm.s/bpf")
26		−int BPF_PROG(bpf, int cmd, union bpf_attr *attr, unsigned int size)
	26	+int BPF_PROG(bpf, int cmd, union bpf_attr *attr, unsigned int size, bool kernel)
27	27	{
28	28	struct bpf_key *bkey;
29	29	__u32 pid;

tools/testing/selftests/bpf/progs/test_ptr_untrusted.c+1−1

@@ -7,7 +7,7 @@ · 6 unmodified lines
7	7	char tp_name[128];
8	8
9	9	SEC("lsm.s/bpf")
10		−int BPF_PROG(lsm_run, int cmd, union bpf_attr *attr, unsigned int size)
	10	+int BPF_PROG(lsm_run, int cmd, union bpf_attr *attr, unsigned int size, bool kernel)
11	11	{
12	12	switch (cmd) {
13	13	case BPF_RAW_TRACEPOINT_OPEN:

tools/testing/selftests/bpf/progs/test_select_reuseport_kern.c+0−1

@@ -1,7 +1,6 @@
1	1	// SPDX-License-Identifier: GPL-2.0
2	2	/* Copyright (c) 2018 Facebook */
3	3
4		−#include <stdlib.h>
5	4	#include <linux/in.h>
6	5	#include <linux/ip.h>
7	6	#include <linux/ipv6.h>

tools/testing/selftests/bpf/progs/test_set_remove_xattr.c+133−0

@@ -0,0 +1,133 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
	3	+
	4	+#include "vmlinux.h"
	5	+#include <errno.h>
	6	+#include <bpf/bpf_tracing.h>
	7	+#include "bpf_kfuncs.h"
	8	+#include "bpf_misc.h"
	9	+
	10	+char _license[] SEC("license") = "GPL";
	11	+
	12	+__u32 monitored_pid;
	13	+
	14	+const char xattr_foo[] = "security.bpf.foo";
	15	+const char xattr_bar[] = "security.bpf.bar";
	16	+static const char xattr_selinux[] = "security.selinux";
	17	+char value_bar[] = "world";
	18	+char read_value[32];
	19	+
	20	+bool set_security_bpf_bar_success;
	21	+bool remove_security_bpf_bar_success;
	22	+bool set_security_selinux_fail;
	23	+bool remove_security_selinux_fail;
	24	+
	25	+char name_buf[32];
	26	+
	27	+static inline bool name_match_foo(const char *name)
	28	+{
	29	+ bpf_probe_read_kernel(name_buf, sizeof(name_buf), name);
	30	+
	31	+ return !bpf_strncmp(name_buf, sizeof(xattr_foo), xattr_foo);
	32	+}
	33	+
	34	+/* Test bpf_set_dentry_xattr and bpf_remove_dentry_xattr */
	35	+SEC("lsm.s/inode_getxattr")
	36	+int BPF_PROG(test_inode_getxattr, struct dentry dentry, char name)
	37	+{
	38	+ struct bpf_dynptr value_ptr;
	39	+ __u32 pid;
	40	+ int ret;
	41	+
	42	+ pid = bpf_get_current_pid_tgid() >> 32;
	43	+ if (pid != monitored_pid)
	44	+ return 0;
	45	+
	46	+ /* Only do the following for security.bpf.foo */
	47	+ if (!name_match_foo(name))
	48	+ return 0;
	49	+
	50	+ bpf_dynptr_from_mem(read_value, sizeof(read_value), 0, &value_ptr);
	51	+
	52	+ /* read security.bpf.bar */
	53	+ ret = bpf_get_dentry_xattr(dentry, xattr_bar, &value_ptr);
	54	+
	55	+ if (ret < 0) {
	56	+ /* If security.bpf.bar doesn't exist, set it */
	57	+ bpf_dynptr_from_mem(value_bar, sizeof(value_bar), 0, &value_ptr);
	58	+
	59	+ ret = bpf_set_dentry_xattr(dentry, xattr_bar, &value_ptr, 0);
	60	+ if (!ret)
	61	+ set_security_bpf_bar_success = true;
	62	+ ret = bpf_set_dentry_xattr(dentry, xattr_selinux, &value_ptr, 0);
	63	+ if (ret)
	64	+ set_security_selinux_fail = true;
	65	+ } else {
	66	+ /* If security.bpf.bar exists, remove it */
	67	+ ret = bpf_remove_dentry_xattr(dentry, xattr_bar);
	68	+ if (!ret)
	69	+ remove_security_bpf_bar_success = true;
	70	+
	71	+ ret = bpf_remove_dentry_xattr(dentry, xattr_selinux);
	72	+ if (ret)
	73	+ remove_security_selinux_fail = true;
	74	+ }
	75	+
	76	+ return 0;
	77	+}
	78	+
	79	+bool locked_set_security_bpf_bar_success;
	80	+bool locked_remove_security_bpf_bar_success;
	81	+bool locked_set_security_selinux_fail;
	82	+bool locked_remove_security_selinux_fail;
	83	+
	84	+/* Test bpf_set_dentry_xattr_locked and bpf_remove_dentry_xattr_locked.
	85	+ * It not necessary to differentiate the _locked version and the
	86	+ * not-_locked version in the BPF program. The verifier will fix them up
	87	+ * properly.
	88	+ */
	89	+SEC("lsm.s/inode_setxattr")
	90	+int BPF_PROG(test_inode_setxattr, struct mnt_idmap *idmap,
	91	+ struct dentry dentry, const char name,
	92	+ const void *value, size_t size, int flags)
	93	+{
	94	+ struct bpf_dynptr value_ptr;
	95	+ __u32 pid;
	96	+ int ret;
	97	+
	98	+ pid = bpf_get_current_pid_tgid() >> 32;
	99	+ if (pid != monitored_pid)
	100	+ return 0;
	101	+
	102	+ /* Only do the following for security.bpf.foo */
	103	+ if (!name_match_foo(name))
	104	+ return 0;
	105	+
	106	+ bpf_dynptr_from_mem(read_value, sizeof(read_value), 0, &value_ptr);
	107	+
	108	+ /* read security.bpf.bar */
	109	+ ret = bpf_get_dentry_xattr(dentry, xattr_bar, &value_ptr);
	110	+
	111	+ if (ret < 0) {
	112	+ /* If security.bpf.bar doesn't exist, set it */
	113	+ bpf_dynptr_from_mem(value_bar, sizeof(value_bar), 0, &value_ptr);
	114	+
	115	+ ret = bpf_set_dentry_xattr(dentry, xattr_bar, &value_ptr, 0);
	116	+ if (!ret)
	117	+ locked_set_security_bpf_bar_success = true;
	118	+ ret = bpf_set_dentry_xattr(dentry, xattr_selinux, &value_ptr, 0);
	119	+ if (ret)
	120	+ locked_set_security_selinux_fail = true;
	121	+ } else {
	122	+ /* If security.bpf.bar exists, remove it */
	123	+ ret = bpf_remove_dentry_xattr(dentry, xattr_bar);
	124	+ if (!ret)
	125	+ locked_remove_security_bpf_bar_success = true;
	126	+
	127	+ ret = bpf_remove_dentry_xattr(dentry, xattr_selinux);
	128	+ if (ret)
	129	+ locked_remove_security_selinux_fail = true;
	130	+ }
	131	+
	132	+ return 0;
	133	+}

tools/testing/selftests/bpf/progs/test_spin_lock_fail.c+69−0

@@ -245,4 +245,73 @@ int lock_global_subprog_call2(struct __sk_buff *ctx) · 244 unmodified lines
245	245	return ret;
246	246	}
247	247
	248	+int __noinline
	249	+global_subprog_int(int i)
	250	+{
	251	+ if (i)
	252	+ bpf_printk("%p", &i);
	253	+ return i;
	254	+}
	255	+
	256	+int __noinline
	257	+global_sleepable_helper_subprog(int i)
	258	+{
	259	+ if (i)
	260	+ bpf_copy_from_user(&i, sizeof(i), NULL);
	261	+ return i;
	262	+}
	263	+
	264	+int __noinline
	265	+global_sleepable_kfunc_subprog(int i)
	266	+{
	267	+ if (i)
	268	+ bpf_copy_from_user_str(&i, sizeof(i), NULL, 0);
	269	+ global_subprog_int(i);
	270	+ return i;
	271	+}
	272	+
	273	+int __noinline
	274	+global_subprog_calling_sleepable_global(int i)
	275	+{
	276	+ if (!i)
	277	+ global_sleepable_kfunc_subprog(i);
	278	+ return i;
	279	+}
	280	+
	281	+SEC("?syscall")
	282	+int lock_global_sleepable_helper_subprog(struct __sk_buff *ctx)
	283	+{
	284	+ int ret = 0;
	285	+
	286	+ bpf_spin_lock(&lockA);
	287	+ if (ctx->mark == 42)
	288	+ ret = global_sleepable_helper_subprog(ctx->mark);
	289	+ bpf_spin_unlock(&lockA);
	290	+ return ret;
	291	+}
	292	+
	293	+SEC("?syscall")
	294	+int lock_global_sleepable_kfunc_subprog(struct __sk_buff *ctx)
	295	+{
	296	+ int ret = 0;
	297	+
	298	+ bpf_spin_lock(&lockA);
	299	+ if (ctx->mark == 42)
	300	+ ret = global_sleepable_kfunc_subprog(ctx->mark);
	301	+ bpf_spin_unlock(&lockA);
	302	+ return ret;
	303	+}
	304	+
	305	+SEC("?syscall")
	306	+int lock_global_sleepable_subprog_indirect(struct __sk_buff *ctx)
	307	+{
	308	+ int ret = 0;
	309	+
	310	+ bpf_spin_lock(&lockA);
	311	+ if (ctx->mark == 42)
	312	+ ret = global_subprog_calling_sleepable_global(ctx->mark);
	313	+ bpf_spin_unlock(&lockA);
	314	+ return ret;
	315	+}
	316	+
248	317	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/test_task_under_cgroup.c+1−1

@@ -49,7 +49,7 @@ out: · 48 unmodified lines
49	49	}
50	50
51	51	SEC("lsm.s/bpf")
52		−int BPF_PROG(lsm_run, int cmd, union bpf_attr *attr, unsigned int size)
	52	+int BPF_PROG(lsm_run, int cmd, union bpf_attr *attr, unsigned int size, bool kernel)
53	53	{
54	54	struct cgroup *cgrp = NULL;
55	55	struct task_struct *task;

tools/testing/selftests/bpf/progs/test_usdt.c+14−0

@@ -11,6 +11,7 @@ int usdt0_called; · 10 unmodified lines
11	11	u64 usdt0_cookie;
12	12	int usdt0_arg_cnt;
13	13	int usdt0_arg_ret;
	14	+int usdt0_arg_size;
14	15
15	16	SEC("usdt")
16	17	int usdt0(struct pt_regs *ctx)
@@ -26,6 +27,7 @@ int usdt0(struct pt_regs *ctx) · 9 unmodified lines
26	27	usdt0_arg_cnt = bpf_usdt_arg_cnt(ctx);
27	28	/* should return -ENOENT for any arg_num */
28	29	usdt0_arg_ret = bpf_usdt_arg(ctx, bpf_get_prandom_u32(), &tmp);
	30	+ usdt0_arg_size = bpf_usdt_arg_size(ctx, bpf_get_prandom_u32());
29	31	return 0;
30	32	}
31	33
@@ -34,6 +36,7 @@ u64 usdt3_cookie; · 2 unmodified lines
34	36	int usdt3_arg_cnt;
35	37	int usdt3_arg_rets[3];
36	38	u64 usdt3_args[3];
	39	+int usdt3_arg_sizes[3];
37	40
38	41	SEC("usdt//proc/self/exe:test:usdt3")
39	42	int usdt3(struct pt_regs *ctx)
@@ -50,12 +53,15 @@ int usdt3(struct pt_regs *ctx) · 10 unmodified lines
50	53
51	54	usdt3_arg_rets[0] = bpf_usdt_arg(ctx, 0, &tmp);
52	55	usdt3_args[0] = (int)tmp;
	56	+ usdt3_arg_sizes[0] = bpf_usdt_arg_size(ctx, 0);
53	57
54	58	usdt3_arg_rets[1] = bpf_usdt_arg(ctx, 1, &tmp);
55	59	usdt3_args[1] = (long)tmp;
	60	+ usdt3_arg_sizes[1] = bpf_usdt_arg_size(ctx, 1);
56	61
57	62	usdt3_arg_rets[2] = bpf_usdt_arg(ctx, 2, &tmp);
58	63	usdt3_args[2] = (uintptr_t)tmp;
	64	+ usdt3_arg_sizes[2] = bpf_usdt_arg_size(ctx, 2);
59	65
60	66	return 0;
61	67	}
@@ -64,12 +70,15 @@ int usdt12_called; · 2 unmodified lines
64	70	u64 usdt12_cookie;
65	71	int usdt12_arg_cnt;
66	72	u64 usdt12_args[12];
	73	+int usdt12_arg_sizes[12];
67	74
68	75	SEC("usdt//proc/self/exe:test:usdt12")
69	76	int BPF_USDT(usdt12, int a1, int a2, long a3, long a4, unsigned a5,
70	77	long a6, __u64 a7, uintptr_t a8, int a9, short a10,
71	78	short a11, signed char a12)
72	79	{
	80	+ int i;
	81	+
73	82	if (my_pid != (bpf_get_current_pid_tgid() >> 32))
74	83	return 0;
75	84
@@ -90,6 +99,11 @@ int BPF_USDT(usdt12, int a1, int a2, long a3, long a4, unsigned a5, · 14 unmodified lines
90	99	usdt12_args[9] = a10;
91	100	usdt12_args[10] = a11;
92	101	usdt12_args[11] = a12;
	102	+
	103	+ bpf_for(i, 0, 12) {
	104	+ usdt12_arg_sizes[i] = bpf_usdt_arg_size(ctx, i);
	105	+ }
	106	+
93	107	return 0;
94	108	}
95	109

tools/testing/selftests/bpf/progs/test_verify_pkcs7_sig.c+1−1

@@ -37,7 +37,7 @@ struct { · 36 unmodified lines
37	37	char _license[] SEC("license") = "GPL";
38	38
39	39	SEC("lsm.s/bpf")
40		−int BPF_PROG(bpf, int cmd, union bpf_attr *attr, unsigned int size)
	40	+int BPF_PROG(bpf, int cmd, union bpf_attr *attr, unsigned int size, bool kernel)
41	41	{
42	42	struct bpf_dynptr data_ptr, sig_ptr;
43	43	struct data *data_val;

tools/testing/selftests/bpf/progs/test_xdp_vlan.c+10−10

@@ -102,8 +102,8 @@ bool parse_eth_frame(struct ethhdr eth, void data_end, struct parse_pkt *pkt) · 101 unmodified lines
102	102	#define TESTVLAN 4011 /* 0xFAB */
103	103	// #define TO_VLAN 4000 /* 0xFA0 (hint 0xOA0 = 160) */
104	104
105		−SEC("xdp_drop_vlan_4011")
106		−int xdp_prognum0(struct xdp_md *ctx)
	105	+SEC("xdp")
	106	+int xdp_drop_vlan_4011(struct xdp_md *ctx)
107	107	{
108	108	void data_end = (void )(long)ctx->data_end;
109	109	void data = (void )(long)ctx->data;
@@ -144,8 +144,8 @@ Load prog with ip tool: · 34 unmodified lines
144	144	/* Changing VLAN to zero, have same practical effect as removing the VLAN. */
145	145	#define TO_VLAN 0
146	146
147		−SEC("xdp_vlan_change")
148		−int xdp_prognum1(struct xdp_md *ctx)
	147	+SEC("xdp")
	148	+int xdp_vlan_change(struct xdp_md *ctx)
149	149	{
150	150	void data_end = (void )(long)ctx->data_end;
151	151	void data = (void )(long)ctx->data;
@@ -178,8 +178,8 @@ int xdp_prognum1(struct xdp_md *ctx) · 26 unmodified lines
178	178	#endif
179	179	#define VLAN_HDR_SZ 4 /* bytes */
180	180
181		−SEC("xdp_vlan_remove_outer")
182		−int xdp_prognum2(struct xdp_md *ctx)
	181	+SEC("xdp")
	182	+int xdp_vlan_remove_outer(struct xdp_md *ctx)
183	183	{
184	184	void data_end = (void )(long)ctx->data_end;
185	185	void data = (void )(long)ctx->data;
@@ -224,8 +224,8 @@ void shift_mac_4bytes_32bit(void *data) · 38 unmodified lines
224	224	p[1] = p[0];
225	225	}
226	226
227		−SEC("xdp_vlan_remove_outer2")
228		−int xdp_prognum3(struct xdp_md *ctx)
	227	+SEC("xdp")
	228	+int xdp_vlan_remove_outer2(struct xdp_md *ctx)
229	229	{
230	230	void data_end = (void )(long)ctx->data_end;
231	231	void data = (void )(long)ctx->data;
@@ -254,8 +254,8 @@ int xdp_prognum3(struct xdp_md *ctx) · 22 unmodified lines
254	254	* The TC-clsact eBPF programs (currently) need to be attach via TC commands
255	255	*/
256	256
257		−SEC("tc_vlan_push")
258		−int _tc_progA(struct __sk_buff *ctx)
	257	+SEC("tc")
	258	+int tc_vlan_push(struct __sk_buff *ctx)
259	259	{
260	260	bpf_skb_vlan_push(ctx, bpf_htons(ETH_P_8021Q), TESTVLAN);
261	261

tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c+48−10

@@ -620,23 +620,61 @@ __naked void helper_call_does_not_prevent_bpf_fastcall(void) · 619 unmodified lines
620	620
621	621	SEC("raw_tp")
622	622	__arch_x86_64
	623	+__log_level(4) __msg("stack depth 24")
	624	+/* may_goto counter at -24 */
	625	+__xlated("0: (u64 )(r10 -24) =")
	626	+/* may_goto timestamp at -16 */
	627	+__xlated("1: (u64 )(r10 -16) =")
	628	+__xlated("2: r1 = 1")
	629	+__xlated("...")
	630	+__xlated("4: r0 = &(void __percpu *)(r0)")
	631	+__xlated("...")
	632	+/* may_goto expansion starts */
	633	+__xlated("6: r11 = (u64 )(r10 -24)")
	634	+__xlated("7: if r11 == 0x0 goto pc+6")
	635	+__xlated("8: r11 -= 1")
	636	+__xlated("9: if r11 != 0x0 goto pc+2")
	637	+__xlated("10: r11 = -24")
	638	+__xlated("11: call unknown")
	639	+__xlated("12: (u64 )(r10 -24) = r11")
	640	+/* may_goto expansion ends */
	641	+__xlated("13: (u64 )(r10 -8) = r1")
	642	+__xlated("14: exit")
	643	+__success
	644	+__naked void may_goto_interaction_x86_64(void)
	645	+{
	646	+ asm volatile (
	647	+ "r1 = 1;"
	648	+ "(u64 )(r10 - 16) = r1;"
	649	+ "call %[bpf_get_smp_processor_id];"
	650	+ "r1 = (u64 )(r10 - 16);"
	651	+ ".8byte %[may_goto];"
	652	+ /* just touch some stack at -8 */
	653	+ "(u64 )(r10 - 8) = r1;"
	654	+ "exit;"
	655	+ :
	656	+ : __imm(bpf_get_smp_processor_id),
	657	+ __imm_insn(may_goto, BPF_RAW_INSN(BPF_JMP \| BPF_JCOND, 0, 0, +1 /* offset */, 0))
	658	+ : __clobber_all);
	659	+}
	660	+
	661	+SEC("raw_tp")
	662	+__arch_arm64
623	663	__log_level(4) __msg("stack depth 16")
624	664	/* may_goto counter at -16 */
625	665	__xlated("0: (u64 )(r10 -16) =")
626	666	__xlated("1: r1 = 1")
627		−__xlated("...")
628		−__xlated("3: r0 = &(void __percpu *)(r0)")
629		−__xlated("...")
	667	+__xlated("2: call bpf_get_smp_processor_id")
630	668	/* may_goto expansion starts */
631		−__xlated("5: r11 = (u64 )(r10 -16)")
632		−__xlated("6: if r11 == 0x0 goto pc+3")
633		−__xlated("7: r11 -= 1")
634		−__xlated("8: (u64 )(r10 -16) = r11")
	669	+__xlated("3: r11 = (u64 )(r10 -16)")
	670	+__xlated("4: if r11 == 0x0 goto pc+3")
	671	+__xlated("5: r11 -= 1")
	672	+__xlated("6: (u64 )(r10 -16) = r11")
635	673	/* may_goto expansion ends */
636		−__xlated("9: (u64 )(r10 -8) = r1")
637		−__xlated("10: exit")
	674	+__xlated("7: (u64 )(r10 -8) = r1")
	675	+__xlated("8: exit")
638	676	__success
639		−__naked void may_goto_interaction(void)
	677	+__naked void may_goto_interaction_arm64(void)
640	678	{
641	679	asm volatile (
642	680	"r1 = 1;"

tools/testing/selftests/bpf/progs/verifier_gotol.c+1−5

@@ -4,11 +4,7 @@ · 3 unmodified lines
4	4	#include <bpf/bpf_helpers.h>
5	5	#include "bpf_misc.h"
6	6
7		−#if (defined(__TARGET_ARCH_arm64) \|\| defined(__TARGET_ARCH_x86) \|\| \
8		− (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) \|\| \
9		− defined(__TARGET_ARCH_arm) \|\| defined(__TARGET_ARCH_s390) \|\| \
10		− defined(__TARGET_ARCH_loongarch)) && \
11		− __clang_major__ >= 18
	7	+#ifdef CAN_USE_GOTOL
12	8
13	9	SEC("socket")
14	10	__description("gotol, small_imm")

tools/testing/selftests/bpf/progs/verifier_iterating_callbacks.c+1−5

@@ -407,11 +407,7 @@ l0_%=: call %[bpf_jiffies64]; \ · 406 unmodified lines
407	407	: __clobber_all);
408	408	}
409	409
410		−#if (defined(__TARGET_ARCH_arm64) \|\| defined(__TARGET_ARCH_x86) \|\| \
411		− (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) \|\| \
412		− defined(__TARGET_ARCH_arm) \|\| defined(__TARGET_ARCH_s390) \|\| \
413		− defined(__TARGET_ARCH_loongarch)) && \
414		− __clang_major__ >= 18
	410	+#ifdef CAN_USE_GOTOL
415	411	SEC("socket")
416	412	__success __retval(0)
417	413	__naked void gotol_and_may_goto(void)

tools/testing/selftests/bpf/progs/verifier_load_acquire.c+218−0

@@ -0,0 +1,218 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2025 Google LLC. */
	3	+
	4	+#include <linux/bpf.h>
	5	+#include <bpf/bpf_helpers.h>
	6	+#include "../../../include/linux/filter.h"
	7	+#include "bpf_misc.h"
	8	+
	9	+#ifdef CAN_USE_LOAD_ACQ_STORE_REL
	10	+
	11	+SEC("socket")
	12	+__description("load-acquire, 8-bit")
	13	+__success __success_unpriv __retval(0x12)
	14	+__naked void load_acquire_8(void)
	15	+{
	16	+ asm volatile (
	17	+ "w1 = 0x12;"
	18	+ "(u8 )(r10 - 1) = w1;"
	19	+ ".8byte %[load_acquire_insn];" // w0 = load_acquire((u8 *)(r10 - 1));
	20	+ "exit;"
	21	+ :
	22	+ : __imm_insn(load_acquire_insn,
	23	+ BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -1))
	24	+ : __clobber_all);
	25	+}
	26	+
	27	+SEC("socket")
	28	+__description("load-acquire, 16-bit")
	29	+__success __success_unpriv __retval(0x1234)
	30	+__naked void load_acquire_16(void)
	31	+{
	32	+ asm volatile (
	33	+ "w1 = 0x1234;"
	34	+ "(u16 )(r10 - 2) = w1;"
	35	+ ".8byte %[load_acquire_insn];" // w0 = load_acquire((u16 *)(r10 - 2));
	36	+ "exit;"
	37	+ :
	38	+ : __imm_insn(load_acquire_insn,
	39	+ BPF_ATOMIC_OP(BPF_H, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -2))
	40	+ : __clobber_all);
	41	+}
	42	+
	43	+SEC("socket")
	44	+__description("load-acquire, 32-bit")
	45	+__success __success_unpriv __retval(0x12345678)
	46	+__naked void load_acquire_32(void)
	47	+{
	48	+ asm volatile (
	49	+ "w1 = 0x12345678;"
	50	+ "(u32 )(r10 - 4) = w1;"
	51	+ ".8byte %[load_acquire_insn];" // w0 = load_acquire((u32 *)(r10 - 4));
	52	+ "exit;"
	53	+ :
	54	+ : __imm_insn(load_acquire_insn,
	55	+ BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -4))
	56	+ : __clobber_all);
	57	+}
	58	+
	59	+SEC("socket")
	60	+__description("load-acquire, 64-bit")
	61	+__success __success_unpriv __retval(0x1234567890abcdef)
	62	+__naked void load_acquire_64(void)
	63	+{
	64	+ asm volatile (
	65	+ "r1 = 0x1234567890abcdef ll;"
	66	+ "(u64 )(r10 - 8) = r1;"
	67	+ ".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r10 - 8));
	68	+ "exit;"
	69	+ :
	70	+ : __imm_insn(load_acquire_insn,
	71	+ BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -8))
	72	+ : __clobber_all);
	73	+}
	74	+
	75	+SEC("socket")
	76	+__description("load-acquire with uninitialized src_reg")
	77	+__failure __failure_unpriv __msg("R2 !read_ok")
	78	+__naked void load_acquire_with_uninitialized_src_reg(void)
	79	+{
	80	+ asm volatile (
	81	+ ".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r2 + 0));
	82	+ "exit;"
	83	+ :
	84	+ : __imm_insn(load_acquire_insn,
	85	+ BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_2, 0))
	86	+ : __clobber_all);
	87	+}
	88	+
	89	+SEC("socket")
	90	+__description("load-acquire with non-pointer src_reg")
	91	+__failure __failure_unpriv __msg("R1 invalid mem access 'scalar'")
	92	+__naked void load_acquire_with_non_pointer_src_reg(void)
	93	+{
	94	+ asm volatile (
	95	+ "r1 = 0;"
	96	+ ".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r1 + 0));
	97	+ "exit;"
	98	+ :
	99	+ : __imm_insn(load_acquire_insn,
	100	+ BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_1, 0))
	101	+ : __clobber_all);
	102	+}
	103	+
	104	+SEC("socket")
	105	+__description("misaligned load-acquire")
	106	+__failure __failure_unpriv __msg("misaligned stack access off")
	107	+__flag(BPF_F_ANY_ALIGNMENT)
	108	+__naked void load_acquire_misaligned(void)
	109	+{
	110	+ asm volatile (
	111	+ "r1 = 0;"
	112	+ "(u64 )(r10 - 8) = r1;"
	113	+ ".8byte %[load_acquire_insn];" // w0 = load_acquire((u32 *)(r10 - 5));
	114	+ "exit;"
	115	+ :
	116	+ : __imm_insn(load_acquire_insn,
	117	+ BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -5))
	118	+ : __clobber_all);
	119	+}
	120	+
	121	+SEC("socket")
	122	+__description("load-acquire from ctx pointer")
	123	+__failure __failure_unpriv __msg("BPF_ATOMIC loads from R1 ctx is not allowed")
	124	+__naked void load_acquire_from_ctx_pointer(void)
	125	+{
	126	+ asm volatile (
	127	+ ".8byte %[load_acquire_insn];" // w0 = load_acquire((u8 *)(r1 + 0));
	128	+ "exit;"
	129	+ :
	130	+ : __imm_insn(load_acquire_insn,
	131	+ BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_1, 0))
	132	+ : __clobber_all);
	133	+}
	134	+
	135	+SEC("xdp")
	136	+__description("load-acquire from pkt pointer")
	137	+__failure __msg("BPF_ATOMIC loads from R2 pkt is not allowed")
	138	+__naked void load_acquire_from_pkt_pointer(void)
	139	+{
	140	+ asm volatile (
	141	+ "r2 = (u32 )(r1 + %[xdp_md_data]);"
	142	+ "r3 = (u32 )(r1 + %[xdp_md_data_end]);"
	143	+ "r1 = r2;"
	144	+ "r1 += 8;"
	145	+ "if r1 >= r3 goto l0_%=;"
	146	+ ".8byte %[load_acquire_insn];" // w0 = load_acquire((u8 *)(r2 + 0));
	147	+"l0_%=: r0 = 0;"
	148	+ "exit;"
	149	+ :
	150	+ : __imm_const(xdp_md_data, offsetof(struct xdp_md, data)),
	151	+ __imm_const(xdp_md_data_end, offsetof(struct xdp_md, data_end)),
	152	+ __imm_insn(load_acquire_insn,
	153	+ BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_2, 0))
	154	+ : __clobber_all);
	155	+}
	156	+
	157	+SEC("flow_dissector")
	158	+__description("load-acquire from flow_keys pointer")
	159	+__failure __msg("BPF_ATOMIC loads from R2 flow_keys is not allowed")
	160	+__naked void load_acquire_from_flow_keys_pointer(void)
	161	+{
	162	+ asm volatile (
	163	+ "r2 = (u64 )(r1 + %[__sk_buff_flow_keys]);"
	164	+ ".8byte %[load_acquire_insn];" // w0 = load_acquire((u8 *)(r2 + 0));
	165	+ "exit;"
	166	+ :
	167	+ : __imm_const(__sk_buff_flow_keys,
	168	+ offsetof(struct __sk_buff, flow_keys)),
	169	+ __imm_insn(load_acquire_insn,
	170	+ BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_2, 0))
	171	+ : __clobber_all);
	172	+}
	173	+
	174	+SEC("sk_reuseport")
	175	+__description("load-acquire from sock pointer")
	176	+__failure __msg("BPF_ATOMIC loads from R2 sock is not allowed")
	177	+__naked void load_acquire_from_sock_pointer(void)
	178	+{
	179	+ asm volatile (
	180	+ "r2 = (u64 )(r1 + %[sk_reuseport_md_sk]);"
	181	+ // w0 = load_acquire((u8 *)(r2 + offsetof(struct bpf_sock, family)));
	182	+ ".8byte %[load_acquire_insn];"
	183	+ "exit;"
	184	+ :
	185	+ : __imm_const(sk_reuseport_md_sk, offsetof(struct sk_reuseport_md, sk)),
	186	+ __imm_insn(load_acquire_insn,
	187	+ BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_2,
	188	+ offsetof(struct bpf_sock, family)))
	189	+ : __clobber_all);
	190	+}
	191	+
	192	+SEC("socket")
	193	+__description("load-acquire with invalid register R15")
	194	+__failure __failure_unpriv __msg("R15 is invalid")
	195	+__naked void load_acquire_with_invalid_reg(void)
	196	+{
	197	+ asm volatile (
	198	+ ".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r15 + 0));
	199	+ "exit;"
	200	+ :
	201	+ : __imm_insn(load_acquire_insn,
	202	+ BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, 15 /* invalid reg */, 0))
	203	+ : __clobber_all);
	204	+}
	205	+
	206	+#else /* CAN_USE_LOAD_ACQ_STORE_REL */
	207	+
	208	+SEC("socket")
	209	+__description("Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support load-acquire, use a dummy test")
	210	+__success
	211	+int dummy_test(void)
	212	+{
	213	+ return 0;
	214	+}
	215	+
	216	+#endif /* CAN_USE_LOAD_ACQ_STORE_REL */
	217	+
	218	+char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/verifier_may_goto_1.c+32−2

@@ -69,8 +69,38 @@ __naked void may_goto_batch_1(void) · 68 unmodified lines
69	69	}
70	70
71	71	SEC("raw_tp")
72		−__description("may_goto batch with offsets 2/0")
	72	+__description("may_goto batch with offsets 2/0 - x86_64")
73	73	__arch_x86_64
	74	+__xlated("0: (u64 )(r10 -16) = 65535")
	75	+__xlated("1: (u64 )(r10 -8) = 0")
	76	+__xlated("2: r11 = (u64 )(r10 -16)")
	77	+__xlated("3: if r11 == 0x0 goto pc+6")
	78	+__xlated("4: r11 -= 1")
	79	+__xlated("5: if r11 != 0x0 goto pc+2")
	80	+__xlated("6: r11 = -16")
	81	+__xlated("7: call unknown")
	82	+__xlated("8: (u64 )(r10 -16) = r11")
	83	+__xlated("9: r0 = 1")
	84	+__xlated("10: r0 = 2")
	85	+__xlated("11: exit")
	86	+__success
	87	+__naked void may_goto_batch_2_x86_64(void)
	88	+{
	89	+ asm volatile (
	90	+ ".8byte %[may_goto1];"
	91	+ ".8byte %[may_goto3];"
	92	+ "r0 = 1;"
	93	+ "r0 = 2;"
	94	+ "exit;"
	95	+ :
	96	+ : __imm_insn(may_goto1, BPF_RAW_INSN(BPF_JMP \| BPF_JCOND, 0, 0, 2 /* offset */, 0)),
	97	+ __imm_insn(may_goto3, BPF_RAW_INSN(BPF_JMP \| BPF_JCOND, 0, 0, 0 /* offset */, 0))
	98	+ : __clobber_all);
	99	+}
	100	+
	101	+SEC("raw_tp")
	102	+__description("may_goto batch with offsets 2/0 - arm64")
	103	+__arch_arm64
74	104	__xlated("0: (u64 )(r10 -8) = 8388608")
75	105	__xlated("1: r11 = (u64 )(r10 -8)")
76	106	__xlated("2: if r11 == 0x0 goto pc+3")
@@ -80,7 +110,7 @@ __xlated("5: r0 = 1") · 3 unmodified lines
80	110	__xlated("6: r0 = 2")
81	111	__xlated("7: exit")
82	112	__success
83		−__naked void may_goto_batch_2(void)
	113	+__naked void may_goto_batch_2_arm64(void)
84	114	{
85	115	asm volatile (
86	116	".8byte %[may_goto1];"

tools/testing/selftests/bpf/progs/verifier_precision.c+49−0

@@ -2,6 +2,7 @@ · 1 unmodified line
2	2	/* Copyright (C) 2023 SUSE LLC */
3	3	#include <linux/bpf.h>
4	4	#include <bpf/bpf_helpers.h>
	5	+#include "../../../include/linux/filter.h"
5	6	#include "bpf_misc.h"
6	7
7	8	SEC("?raw_tp")
@@ -90,6 +91,54 @@ __naked int bpf_end_bswap(void) · 82 unmodified lines
90	91	::: __clobber_all);
91	92	}
92	93
	94	+#if defined(ENABLE_ATOMICS_TESTS) && \
	95	+ (defined(__TARGET_ARCH_arm64) \|\| defined(__TARGET_ARCH_x86))
	96	+
	97	+SEC("?raw_tp")
	98	+__success __log_level(2)
	99	+__msg("mark_precise: frame0: regs=r2 stack= before 3: (bf) r3 = r10")
	100	+__msg("mark_precise: frame0: regs=r2 stack= before 2: (db) r2 = load_acquire((u64 *)(r10 -8))")
	101	+__msg("mark_precise: frame0: regs= stack=-8 before 1: (7b) (u64 )(r10 -8) = r1")
	102	+__msg("mark_precise: frame0: regs=r1 stack= before 0: (b7) r1 = 8")
	103	+__naked int bpf_load_acquire(void)
	104	+{
	105	+ asm volatile (
	106	+ "r1 = 8;"
	107	+ "(u64 )(r10 - 8) = r1;"
	108	+ ".8byte %[load_acquire_insn];" /* r2 = load_acquire((u64 )(r10 - 8)); /
	109	+ "r3 = r10;"
	110	+ "r3 += r2;" /* mark_precise */
	111	+ "r0 = 0;"
	112	+ "exit;"
	113	+ :
	114	+ : __imm_insn(load_acquire_insn,
	115	+ BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -8))
	116	+ : __clobber_all);
	117	+}
	118	+
	119	+SEC("?raw_tp")
	120	+__success __log_level(2)
	121	+__msg("mark_precise: frame0: regs=r1 stack= before 3: (bf) r2 = r10")
	122	+__msg("mark_precise: frame0: regs=r1 stack= before 2: (79) r1 = (u64 )(r10 -8)")
	123	+__msg("mark_precise: frame0: regs= stack=-8 before 1: (db) store_release((u64 *)(r10 -8), r1)")
	124	+__msg("mark_precise: frame0: regs=r1 stack= before 0: (b7) r1 = 8")
	125	+__naked int bpf_store_release(void)
	126	+{
	127	+ asm volatile (
	128	+ "r1 = 8;"
	129	+ ".8byte %[store_release_insn];" /* store_release((u64 )(r10 - 8), r1); /
	130	+ "r1 = (u64 )(r10 - 8);"
	131	+ "r2 = r10;"
	132	+ "r2 += r1;" /* mark_precise */
	133	+ "r0 = 0;"
	134	+ "exit;"
	135	+ :
	136	+ : __imm_insn(store_release_insn,
	137	+ BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_10, BPF_REG_1, -8))
	138	+ : __clobber_all);
	139	+}
	140	+
	141	+#endif /* load-acquire, store-release */
93	142	#endif /* v4 instruction */
94	143
95	144	SEC("?raw_tp")

tools/testing/selftests/bpf/progs/verifier_stack_ptr.c+52−0

@@ -481,4 +481,56 @@ l1_%=: r0 = 42; \ · 480 unmodified lines
481	481	: __clobber_all);
482	482	}
483	483
	484	+SEC("socket")
	485	+__description("PTR_TO_STACK stack size > 512")
	486	+__failure __msg("invalid write to stack R1 off=-520 size=8")
	487	+__naked void stack_check_size_gt_512(void)
	488	+{
	489	+ asm volatile (" \
	490	+ r1 = r10; \
	491	+ r1 += -520; \
	492	+ r0 = 42; \
	493	+ (u64)(r1 + 0) = r0; \
	494	+ exit; \
	495	+" ::: __clobber_all);
	496	+}
	497	+
	498	+#ifdef __BPF_FEATURE_MAY_GOTO
	499	+SEC("socket")
	500	+__description("PTR_TO_STACK stack size 512 with may_goto with jit")
	501	+__load_if_JITed()
	502	+__success __retval(42)
	503	+__naked void stack_check_size_512_with_may_goto_jit(void)
	504	+{
	505	+ asm volatile (" \
	506	+ r1 = r10; \
	507	+ r1 += -512; \
	508	+ r0 = 42; \
	509	+ (u32)(r1 + 0) = r0; \
	510	+ may_goto l0_%=; \
	511	+ r2 = 100; \
	512	+ l0_%=: \
	513	+ exit; \
	514	+" ::: __clobber_all);
	515	+}
	516	+
	517	+SEC("socket")
	518	+__description("PTR_TO_STACK stack size 512 with may_goto without jit")
	519	+__load_if_no_JITed()
	520	+__failure __msg("stack size 520(extra 8) is too large")
	521	+__naked void stack_check_size_512_with_may_goto(void)
	522	+{
	523	+ asm volatile (" \
	524	+ r1 = r10; \
	525	+ r1 += -512; \
	526	+ r0 = 42; \
	527	+ (u32)(r1 + 0) = r0; \
	528	+ may_goto l0_%=; \
	529	+ r2 = 100; \
	530	+ l0_%=: \
	531	+ exit; \
	532	+" ::: __clobber_all);
	533	+}
	534	+#endif
	535	+
484	536	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/verifier_store_release.c+286−0

@@ -0,0 +1,286 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2025 Google LLC. */
	3	+
	4	+#include <linux/bpf.h>
	5	+#include <bpf/bpf_helpers.h>
	6	+#include "../../../include/linux/filter.h"
	7	+#include "bpf_misc.h"
	8	+
	9	+#if __clang_major__ >= 18 && defined(ENABLE_ATOMICS_TESTS) && \
	10	+ (defined(__TARGET_ARCH_arm64) \|\| defined(__TARGET_ARCH_x86))
	11	+
	12	+SEC("socket")
	13	+__description("store-release, 8-bit")
	14	+__success __success_unpriv __retval(0x12)
	15	+__naked void store_release_8(void)
	16	+{
	17	+ asm volatile (
	18	+ "w1 = 0x12;"
	19	+ ".8byte %[store_release_insn];" // store_release((u8 *)(r10 - 1), w1);
	20	+ "w0 = (u8 )(r10 - 1);"
	21	+ "exit;"
	22	+ :
	23	+ : __imm_insn(store_release_insn,
	24	+ BPF_ATOMIC_OP(BPF_B, BPF_STORE_REL, BPF_REG_10, BPF_REG_1, -1))
	25	+ : __clobber_all);
	26	+}
	27	+
	28	+SEC("socket")
	29	+__description("store-release, 16-bit")
	30	+__success __success_unpriv __retval(0x1234)
	31	+__naked void store_release_16(void)
	32	+{
	33	+ asm volatile (
	34	+ "w1 = 0x1234;"
	35	+ ".8byte %[store_release_insn];" // store_release((u16 *)(r10 - 2), w1);
	36	+ "w0 = (u16 )(r10 - 2);"
	37	+ "exit;"
	38	+ :
	39	+ : __imm_insn(store_release_insn,
	40	+ BPF_ATOMIC_OP(BPF_H, BPF_STORE_REL, BPF_REG_10, BPF_REG_1, -2))
	41	+ : __clobber_all);
	42	+}
	43	+
	44	+SEC("socket")
	45	+__description("store-release, 32-bit")
	46	+__success __success_unpriv __retval(0x12345678)
	47	+__naked void store_release_32(void)
	48	+{
	49	+ asm volatile (
	50	+ "w1 = 0x12345678;"
	51	+ ".8byte %[store_release_insn];" // store_release((u32 *)(r10 - 4), w1);
	52	+ "w0 = (u32 )(r10 - 4);"
	53	+ "exit;"
	54	+ :
	55	+ : __imm_insn(store_release_insn,
	56	+ BPF_ATOMIC_OP(BPF_W, BPF_STORE_REL, BPF_REG_10, BPF_REG_1, -4))
	57	+ : __clobber_all);
	58	+}
	59	+
	60	+SEC("socket")
	61	+__description("store-release, 64-bit")
	62	+__success __success_unpriv __retval(0x1234567890abcdef)
	63	+__naked void store_release_64(void)
	64	+{
	65	+ asm volatile (
	66	+ "r1 = 0x1234567890abcdef ll;"
	67	+ ".8byte %[store_release_insn];" // store_release((u64 *)(r10 - 8), r1);
	68	+ "r0 = (u64 )(r10 - 8);"
	69	+ "exit;"
	70	+ :
	71	+ : __imm_insn(store_release_insn,
	72	+ BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_10, BPF_REG_1, -8))
	73	+ : __clobber_all);
	74	+}
	75	+
	76	+SEC("socket")
	77	+__description("store-release with uninitialized src_reg")
	78	+__failure __failure_unpriv __msg("R2 !read_ok")
	79	+__naked void store_release_with_uninitialized_src_reg(void)
	80	+{
	81	+ asm volatile (
	82	+ ".8byte %[store_release_insn];" // store_release((u64 *)(r10 - 8), r2);
	83	+ "exit;"
	84	+ :
	85	+ : __imm_insn(store_release_insn,
	86	+ BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_10, BPF_REG_2, -8))
	87	+ : __clobber_all);
	88	+}
	89	+
	90	+SEC("socket")
	91	+__description("store-release with uninitialized dst_reg")
	92	+__failure __failure_unpriv __msg("R2 !read_ok")
	93	+__naked void store_release_with_uninitialized_dst_reg(void)
	94	+{
	95	+ asm volatile (
	96	+ "r1 = 0;"
	97	+ ".8byte %[store_release_insn];" // store_release((u64 *)(r2 - 8), r1);
	98	+ "exit;"
	99	+ :
	100	+ : __imm_insn(store_release_insn,
	101	+ BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_2, BPF_REG_1, -8))
	102	+ : __clobber_all);
	103	+}
	104	+
	105	+SEC("socket")
	106	+__description("store-release with non-pointer dst_reg")
	107	+__failure __failure_unpriv __msg("R1 invalid mem access 'scalar'")
	108	+__naked void store_release_with_non_pointer_dst_reg(void)
	109	+{
	110	+ asm volatile (
	111	+ "r1 = 0;"
	112	+ ".8byte %[store_release_insn];" // store_release((u64 *)(r1 + 0), r1);
	113	+ "exit;"
	114	+ :
	115	+ : __imm_insn(store_release_insn,
	116	+ BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_1, BPF_REG_1, 0))
	117	+ : __clobber_all);
	118	+}
	119	+
	120	+SEC("socket")
	121	+__description("misaligned store-release")
	122	+__failure __failure_unpriv __msg("misaligned stack access off")
	123	+__flag(BPF_F_ANY_ALIGNMENT)
	124	+__naked void store_release_misaligned(void)
	125	+{
	126	+ asm volatile (
	127	+ "w0 = 0;"
	128	+ ".8byte %[store_release_insn];" // store_release((u32 *)(r10 - 5), w0);
	129	+ "exit;"
	130	+ :
	131	+ : __imm_insn(store_release_insn,
	132	+ BPF_ATOMIC_OP(BPF_W, BPF_STORE_REL, BPF_REG_10, BPF_REG_0, -5))
	133	+ : __clobber_all);
	134	+}
	135	+
	136	+SEC("socket")
	137	+__description("store-release to ctx pointer")
	138	+__failure __failure_unpriv __msg("BPF_ATOMIC stores into R1 ctx is not allowed")
	139	+__naked void store_release_to_ctx_pointer(void)
	140	+{
	141	+ asm volatile (
	142	+ "w0 = 0;"
	143	+ // store_release((u8 *)(r1 + offsetof(struct __sk_buff, cb[0])), w0);
	144	+ ".8byte %[store_release_insn];"
	145	+ "exit;"
	146	+ :
	147	+ : __imm_insn(store_release_insn,
	148	+ BPF_ATOMIC_OP(BPF_B, BPF_STORE_REL, BPF_REG_1, BPF_REG_0,
	149	+ offsetof(struct __sk_buff, cb[0])))
	150	+ : __clobber_all);
	151	+}
	152	+
	153	+SEC("xdp")
	154	+__description("store-release to pkt pointer")
	155	+__failure __msg("BPF_ATOMIC stores into R2 pkt is not allowed")
	156	+__naked void store_release_to_pkt_pointer(void)
	157	+{
	158	+ asm volatile (
	159	+ "w0 = 0;"
	160	+ "r2 = (u32 )(r1 + %[xdp_md_data]);"
	161	+ "r3 = (u32 )(r1 + %[xdp_md_data_end]);"
	162	+ "r1 = r2;"
	163	+ "r1 += 8;"
	164	+ "if r1 >= r3 goto l0_%=;"
	165	+ ".8byte %[store_release_insn];" // store_release((u8 *)(r2 + 0), w0);
	166	+"l0_%=: r0 = 0;"
	167	+ "exit;"
	168	+ :
	169	+ : __imm_const(xdp_md_data, offsetof(struct xdp_md, data)),
	170	+ __imm_const(xdp_md_data_end, offsetof(struct xdp_md, data_end)),
	171	+ __imm_insn(store_release_insn,
	172	+ BPF_ATOMIC_OP(BPF_B, BPF_STORE_REL, BPF_REG_2, BPF_REG_0, 0))
	173	+ : __clobber_all);
	174	+}
	175	+
	176	+SEC("flow_dissector")
	177	+__description("store-release to flow_keys pointer")
	178	+__failure __msg("BPF_ATOMIC stores into R2 flow_keys is not allowed")
	179	+__naked void store_release_to_flow_keys_pointer(void)
	180	+{
	181	+ asm volatile (
	182	+ "w0 = 0;"
	183	+ "r2 = (u64 )(r1 + %[__sk_buff_flow_keys]);"
	184	+ ".8byte %[store_release_insn];" // store_release((u8 *)(r2 + 0), w0);
	185	+ "exit;"
	186	+ :
	187	+ : __imm_const(__sk_buff_flow_keys,
	188	+ offsetof(struct __sk_buff, flow_keys)),
	189	+ __imm_insn(store_release_insn,
	190	+ BPF_ATOMIC_OP(BPF_B, BPF_STORE_REL, BPF_REG_2, BPF_REG_0, 0))
	191	+ : __clobber_all);
	192	+}
	193	+
	194	+SEC("sk_reuseport")
	195	+__description("store-release to sock pointer")
	196	+__failure __msg("R2 cannot write into sock")
	197	+__naked void store_release_to_sock_pointer(void)
	198	+{
	199	+ asm volatile (
	200	+ "w0 = 0;"
	201	+ "r2 = (u64 )(r1 + %[sk_reuseport_md_sk]);"
	202	+ ".8byte %[store_release_insn];" // store_release((u8 *)(r2 + 0), w0);
	203	+ "exit;"
	204	+ :
	205	+ : __imm_const(sk_reuseport_md_sk, offsetof(struct sk_reuseport_md, sk)),
	206	+ __imm_insn(store_release_insn,
	207	+ BPF_ATOMIC_OP(BPF_B, BPF_STORE_REL, BPF_REG_2, BPF_REG_0, 0))
	208	+ : __clobber_all);
	209	+}
	210	+
	211	+SEC("socket")
	212	+__description("store-release, leak pointer to stack")
	213	+__success __success_unpriv __retval(0)
	214	+__naked void store_release_leak_pointer_to_stack(void)
	215	+{
	216	+ asm volatile (
	217	+ ".8byte %[store_release_insn];" // store_release((u64 *)(r10 - 8), r1);
	218	+ "r0 = 0;"
	219	+ "exit;"
	220	+ :
	221	+ : __imm_insn(store_release_insn,
	222	+ BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_10, BPF_REG_1, -8))
	223	+ : __clobber_all);
	224	+}
	225	+
	226	+struct {
	227	+ __uint(type, BPF_MAP_TYPE_HASH);
	228	+ __uint(max_entries, 1);
	229	+ __type(key, long long);
	230	+ __type(value, long long);
	231	+} map_hash_8b SEC(".maps");
	232	+
	233	+SEC("socket")
	234	+__description("store-release, leak pointer to map")
	235	+__success __retval(0)
	236	+__failure_unpriv __msg_unpriv("R6 leaks addr into map")
	237	+__naked void store_release_leak_pointer_to_map(void)
	238	+{
	239	+ asm volatile (
	240	+ "r6 = r1;"
	241	+ "r1 = %[map_hash_8b] ll;"
	242	+ "r2 = 0;"
	243	+ "(u64 )(r10 - 8) = r2;"
	244	+ "r2 = r10;"
	245	+ "r2 += -8;"
	246	+ "call %[bpf_map_lookup_elem];"
	247	+ "if r0 == 0 goto l0_%=;"
	248	+ ".8byte %[store_release_insn];" // store_release((u64 *)(r0 + 0), r6);
	249	+"l0_%=:"
	250	+ "r0 = 0;"
	251	+ "exit;"
	252	+ :
	253	+ : __imm_addr(map_hash_8b),
	254	+ __imm(bpf_map_lookup_elem),
	255	+ __imm_insn(store_release_insn,
	256	+ BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_0, BPF_REG_6, 0))
	257	+ : __clobber_all);
	258	+}
	259	+
	260	+SEC("socket")
	261	+__description("store-release with invalid register R15")
	262	+__failure __failure_unpriv __msg("R15 is invalid")
	263	+__naked void store_release_with_invalid_reg(void)
	264	+{
	265	+ asm volatile (
	266	+ ".8byte %[store_release_insn];" // store_release((u64 *)(r15 + 0), r1);
	267	+ "exit;"
	268	+ :
	269	+ : __imm_insn(store_release_insn,
	270	+ BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, 15 /* invalid reg */, BPF_REG_1, 0))
	271	+ : __clobber_all);
	272	+}
	273	+
	274	+#else
	275	+
	276	+SEC("socket")
	277	+__description("Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support store-release, use a dummy test")
	278	+__success
	279	+int dummy_test(void)
	280	+{
	281	+ return 0;
	282	+}
	283	+
	284	+#endif
	285	+
	286	+char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/xdp_redirect_map.c+88−0

@@ -1,7 +1,10 @@
1	1	// SPDX-License-Identifier: GPL-2.0
2	2
	3	+#include <linux/if_ether.h>
	4	+
3	5	#include <linux/bpf.h>
4	6	#include <bpf/bpf_helpers.h>
	7	+#include <bpf/bpf_endian.h>
5	8
6	9	struct {
7	10	__uint(type, BPF_MAP_TYPE_DEVMAP);
@@ -28,4 +31,89 @@ int xdp_redirect_map_2(struct xdp_md *xdp) · 20 unmodified lines
28	31	return bpf_redirect_map(&tx_port, 2, 0);
29	32	}
30	33
	34	+struct {
	35	+ __uint(type, BPF_MAP_TYPE_ARRAY);
	36	+ __uint(max_entries, 3);
	37	+ __type(key, __u32);
	38	+ __type(value, __u64);
	39	+} rxcnt SEC(".maps");
	40	+
	41	+static int xdp_count(struct xdp_md *xdp, __u32 key)
	42	+{
	43	+ void data_end = (void )(long)xdp->data_end;
	44	+ void data = (void )(long)xdp->data;
	45	+ struct ethhdr *eth = data;
	46	+ __u64 *count;
	47	+
	48	+ if (data + sizeof(*eth) > data_end)
	49	+ return XDP_DROP;
	50	+
	51	+ if (bpf_htons(eth->h_proto) == ETH_P_IP) {
	52	+ /* We only count IPv4 packets */
	53	+ count = bpf_map_lookup_elem(&rxcnt, &key);
	54	+ if (count)
	55	+ *count += 1;
	56	+ }
	57	+
	58	+ return XDP_PASS;
	59	+}
	60	+
	61	+SEC("xdp")
	62	+int xdp_count_0(struct xdp_md *xdp)
	63	+{
	64	+ return xdp_count(xdp, 0);
	65	+}
	66	+
	67	+SEC("xdp")
	68	+int xdp_count_1(struct xdp_md *xdp)
	69	+{
	70	+ return xdp_count(xdp, 1);
	71	+}
	72	+
	73	+SEC("xdp")
	74	+int xdp_count_2(struct xdp_md *xdp)
	75	+{
	76	+ return xdp_count(xdp, 2);
	77	+}
	78	+
	79	+struct {
	80	+ __uint(type, BPF_MAP_TYPE_ARRAY);
	81	+ __uint(max_entries, 2);
	82	+ __type(key, __u32);
	83	+ __type(value, __be64);
	84	+} rx_mac SEC(".maps");
	85	+
	86	+static int store_mac(struct xdp_md *xdp, __u32 id)
	87	+{
	88	+ void data_end = (void )(long)xdp->data_end;
	89	+ void data = (void )(long)xdp->data;
	90	+ struct ethhdr *eth = data;
	91	+ __u32 key = id;
	92	+ __be64 mac = 0;
	93	+
	94	+ if (data + sizeof(*eth) > data_end)
	95	+ return XDP_DROP;
	96	+
	97	+ /* Only store IPv4 MAC to avoid being polluted by IPv6 packets */
	98	+ if (eth->h_proto == bpf_htons(ETH_P_IP)) {
	99	+ __builtin_memcpy(&mac, eth->h_source, ETH_ALEN);
	100	+ bpf_map_update_elem(&rx_mac, &key, &mac, 0);
	101	+ bpf_printk("%s - %x", __func__, mac);
	102	+ }
	103	+
	104	+ return XDP_PASS;
	105	+}
	106	+
	107	+SEC("xdp")
	108	+int store_mac_1(struct xdp_md *xdp)
	109	+{
	110	+ return store_mac(xdp, 0);
	111	+}
	112	+
	113	+SEC("xdp")
	114	+int store_mac_2(struct xdp_md *xdp)
	115	+{
	116	+ return store_mac(xdp, 1);
	117	+}
	118	+
31	119	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/xdp_redirect_multi_kern.c+29−12

@@ -34,6 +34,14 @@ struct { · 33 unmodified lines
34	34	__uint(max_entries, 128);
35	35	} mac_map SEC(".maps");
36	36
	37	+/* map to store redirect flags for each protocol*/
	38	+struct {
	39	+ __uint(type, BPF_MAP_TYPE_HASH);
	40	+ __type(key, __u16);
	41	+ __type(value, __u64);
	42	+ __uint(max_entries, 16);
	43	+} redirect_flags SEC(".maps");
	44	+
37	45	SEC("xdp")
38	46	int xdp_redirect_map_multi_prog(struct xdp_md *ctx)
39	47	{
@@ -41,25 +49,34 @@ int xdp_redirect_map_multi_prog(struct xdp_md *ctx) · 1 unmodified line
41	49	void data = (void )(long)ctx->data;
42	50	int if_index = ctx->ingress_ifindex;
43	51	struct ethhdr *eth = data;
	52	+ __u64 *flags_from_map;
44	53	__u16 h_proto;
45	54	__u64 nh_off;
	55	+ __u64 flags;
46	56
47	57	nh_off = sizeof(*eth);
48	58	if (data + nh_off > data_end)
49	59	return XDP_DROP;
50	60
51		− h_proto = eth->h_proto;
52		−
53		− /* Using IPv4 for (BPF_F_BROADCAST \| BPF_F_EXCLUDE_INGRESS) testing */
54		− if (h_proto == bpf_htons(ETH_P_IP))
55		− return bpf_redirect_map(&map_all, 0,
56		− BPF_F_BROADCAST \| BPF_F_EXCLUDE_INGRESS);
57		− /* Using IPv6 for none flag testing */
58		− else if (h_proto == bpf_htons(ETH_P_IPV6))
59		− return bpf_redirect_map(&map_all, if_index, 0);
60		− /* All others for BPF_F_BROADCAST testing */
61		− else
62		− return bpf_redirect_map(&map_all, 0, BPF_F_BROADCAST);
	61	+ h_proto = bpf_htons(eth->h_proto);
	62	+
	63	+ flags_from_map = bpf_map_lookup_elem(&redirect_flags, &h_proto);
	64	+
	65	+ /* Default flags for IPv4 : (BPF_F_BROADCAST \| BPF_F_EXCLUDE_INGRESS) */
	66	+ if (h_proto == ETH_P_IP) {
	67	+ flags = flags_from_map ? *flags_from_map : BPF_F_BROADCAST \| BPF_F_EXCLUDE_INGRESS;
	68	+ return bpf_redirect_map(&map_all, 0, flags);
	69	+ }
	70	+ /* Default flags for IPv6 : 0 */
	71	+ if (h_proto == ETH_P_IPV6) {
	72	+ flags = flags_from_map ? *flags_from_map : 0;
	73	+ return bpf_redirect_map(&map_all, if_index, flags);
	74	+ }
	75	+ /* Default flags for others BPF_F_BROADCAST : 0 */
	76	+ else {
	77	+ flags = flags_from_map ? *flags_from_map : BPF_F_BROADCAST;
	78	+ return bpf_redirect_map(&map_all, 0, flags);
	79	+ }
63	80	}
64	81
65	82	/* The following 2 progs are for 2nd devmap prog testing */

tools/testing/selftests/bpf/test_btf.h+6−0

@@ -72,9 +72,15 @@ · 71 unmodified lines
72	72	#define BTF_TYPE_FLOAT_ENC(name, sz) \
73	73	BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_FLOAT, 0, 0), sz)
74	74
	75	+#define BTF_DECL_ATTR_ENC(value, type, component_idx) \
	76	+ BTF_TYPE_ENC(value, BTF_INFO_ENC(BTF_KIND_DECL_TAG, 1, 0), type), (component_idx)
	77	+
75	78	#define BTF_DECL_TAG_ENC(value, type, component_idx) \
76	79	BTF_TYPE_ENC(value, BTF_INFO_ENC(BTF_KIND_DECL_TAG, 0, 0), type), (component_idx)
77	80
	81	+#define BTF_TYPE_ATTR_ENC(value, type) \
	82	+ BTF_TYPE_ENC(value, BTF_INFO_ENC(BTF_KIND_TYPE_TAG, 1, 0), type)
	83	+
78	84	#define BTF_TYPE_TAG_ENC(value, type) \
79	85	BTF_TYPE_ENC(value, BTF_INFO_ENC(BTF_KIND_TYPE_TAG, 0, 0), type)
80	86

tools/testing/selftests/bpf/test_kmods/bpf_testmod.c+108−0

@@ -1130,6 +1130,7 @@ static const struct btf_kfunc_id_set bpf_testmod_kfunc_set = { · 1129 unmodified lines
1130	1130	};
1131	1131
1132	1132	static const struct bpf_verifier_ops bpf_testmod_verifier_ops = {
	1133	+ .get_func_proto = bpf_base_func_proto,
1133	1134	.is_valid_access = bpf_testmod_ops_is_valid_access,
1134	1135	};
1135	1136
@@ -1176,10 +1177,25 @@ static int bpf_testmod_ops__test_maybe_null(int dummy, · 40 unmodified lines
1176	1177	return 0;
1177	1178	}
1178	1179
	1180	+static int bpf_testmod_ops__test_refcounted(int dummy,
	1181	+ struct task_struct *task__ref)
	1182	+{
	1183	+ return 0;
	1184	+}
	1185	+
	1186	+static struct task_struct *
	1187	+bpf_testmod_ops__test_return_ref_kptr(int dummy, struct task_struct *task__ref,
	1188	+ struct cgroup *cgrp)
	1189	+{
	1190	+ return NULL;
	1191	+}
	1192	+
1179	1193	static struct bpf_testmod_ops __bpf_testmod_ops = {
1180	1194	.test_1 = bpf_testmod_test_1,
1181	1195	.test_2 = bpf_testmod_test_2,
1182	1196	.test_maybe_null = bpf_testmod_ops__test_maybe_null,
	1197	+ .test_refcounted = bpf_testmod_ops__test_refcounted,
	1198	+ .test_return_ref_kptr = bpf_testmod_ops__test_return_ref_kptr,
1183	1199	};
1184	1200
1185	1201	struct bpf_struct_ops bpf_bpf_testmod_ops = {
@@ -1293,6 +1309,85 @@ static int bpf_test_mod_st_ops__test_pro_epilogue(struct st_ops_args *args) · 107 unmodified lines
1293	1309	return 0;
1294	1310	}
1295	1311
	1312	+static int bpf_cgroup_from_id_id;
	1313	+static int bpf_cgroup_release_id;
	1314	+
	1315	+static int st_ops_gen_prologue_with_kfunc(struct bpf_insn *insn_buf, bool direct_write,
	1316	+ const struct bpf_prog *prog)
	1317	+{
	1318	+ struct bpf_insn *insn = insn_buf;
	1319	+
	1320	+ /* r8 = r1; // r8 will be "u64 *ctx".
	1321	+ * r1 = 0;
	1322	+ * r0 = bpf_cgroup_from_id(r1);
	1323	+ * if r0 != 0 goto pc+5;
	1324	+ * r6 = r8[0]; // r6 will be "struct st_ops *args".
	1325	+ * r7 = r6->a;
	1326	+ * r7 += 1000;
	1327	+ * r6->a = r7;
	1328	+ * goto pc+2;
	1329	+ * r1 = r0;
	1330	+ * bpf_cgroup_release(r1);
	1331	+ * r1 = r8;
	1332	+ */
	1333	+ *insn++ = BPF_MOV64_REG(BPF_REG_8, BPF_REG_1);
	1334	+ *insn++ = BPF_MOV64_IMM(BPF_REG_1, 0);
	1335	+ *insn++ = BPF_CALL_KFUNC(0, bpf_cgroup_from_id_id);
	1336	+ *insn++ = BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 5);
	1337	+ *insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_8, 0);
	1338	+ *insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_6, offsetof(struct st_ops_args, a));
	1339	+ *insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_7, 1000);
	1340	+ *insn++ = BPF_STX_MEM(BPF_DW, BPF_REG_6, BPF_REG_7, offsetof(struct st_ops_args, a));
	1341	+ *insn++ = BPF_JMP_IMM(BPF_JA, 0, 0, 2);
	1342	+ *insn++ = BPF_MOV64_REG(BPF_REG_1, BPF_REG_0);
	1343	+ *insn++ = BPF_CALL_KFUNC(0, bpf_cgroup_release_id),
	1344	+ *insn++ = BPF_MOV64_REG(BPF_REG_1, BPF_REG_8);
	1345	+ *insn++ = prog->insnsi[0];
	1346	+
	1347	+ return insn - insn_buf;
	1348	+}
	1349	+
	1350	+static int st_ops_gen_epilogue_with_kfunc(struct bpf_insn insn_buf, const struct bpf_prog prog,
	1351	+ s16 ctx_stack_off)
	1352	+{
	1353	+ struct bpf_insn *insn = insn_buf;
	1354	+
	1355	+ /* r1 = 0;
	1356	+ * r6 = 0;
	1357	+ * r0 = bpf_cgroup_from_id(r1);
	1358	+ * if r0 != 0 goto pc+6;
	1359	+ * r1 = stack[ctx_stack_off]; // r1 will be "u64 *ctx"
	1360	+ * r1 = r1[0]; // r1 will be "struct st_ops *args"
	1361	+ * r6 = r1->a;
	1362	+ * r6 += 10000;
	1363	+ * r1->a = r6;
	1364	+ * goto pc+2
	1365	+ * r1 = r0;
	1366	+ * bpf_cgroup_release(r1);
	1367	+ * r0 = r6;
	1368	+ * r0 *= 2;
	1369	+ * BPF_EXIT;
	1370	+ */
	1371	+ *insn++ = BPF_MOV64_IMM(BPF_REG_1, 0);
	1372	+ *insn++ = BPF_MOV64_IMM(BPF_REG_6, 0);
	1373	+ *insn++ = BPF_CALL_KFUNC(0, bpf_cgroup_from_id_id);
	1374	+ *insn++ = BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 6);
	1375	+ *insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_FP, ctx_stack_off);
	1376	+ *insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0);
	1377	+ *insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_1, offsetof(struct st_ops_args, a));
	1378	+ *insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_6, 10000);
	1379	+ *insn++ = BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_6, offsetof(struct st_ops_args, a));
	1380	+ *insn++ = BPF_JMP_IMM(BPF_JA, 0, 0, 2);
	1381	+ *insn++ = BPF_MOV64_REG(BPF_REG_1, BPF_REG_0);
	1382	+ *insn++ = BPF_CALL_KFUNC(0, bpf_cgroup_release_id),
	1383	+ *insn++ = BPF_MOV64_REG(BPF_REG_0, BPF_REG_6);
	1384	+ *insn++ = BPF_ALU64_IMM(BPF_MUL, BPF_REG_0, 2);
	1385	+ *insn++ = BPF_EXIT_INSN();
	1386	+
	1387	+ return insn - insn_buf;
	1388	+}
	1389	+
	1390	+#define KFUNC_PRO_EPI_PREFIX "test_kfunc_"
1296	1391	static int st_ops_gen_prologue(struct bpf_insn *insn_buf, bool direct_write,
1297	1392	const struct bpf_prog *prog)
1298	1393	{
@@ -1302,6 +1397,9 @@ static int st_ops_gen_prologue(struct bpf_insn *insn_buf, bool direct_write, · 3 unmodified lines
1302	1397	strcmp(prog->aux->attach_func_name, "test_pro_epilogue"))
1303	1398	return 0;
1304	1399
	1400	+ if (!strncmp(prog->aux->name, KFUNC_PRO_EPI_PREFIX, strlen(KFUNC_PRO_EPI_PREFIX)))
	1401	+ return st_ops_gen_prologue_with_kfunc(insn_buf, direct_write, prog);
	1402	+
1305	1403	/* r6 = r1[0]; // r6 will be "struct st_ops args". r1 is "u64 ctx".
1306	1404	* r7 = r6->a;
1307	1405	* r7 += 1000;
@@ -1325,6 +1423,9 @@ static int st_ops_gen_epilogue(struct bpf_insn *insn_buf, const struct bpf_prog · 17 unmodified lines
1325	1423	strcmp(prog->aux->attach_func_name, "test_pro_epilogue"))
1326	1424	return 0;
1327	1425
	1426	+ if (!strncmp(prog->aux->name, KFUNC_PRO_EPI_PREFIX, strlen(KFUNC_PRO_EPI_PREFIX)))
	1427	+ return st_ops_gen_epilogue_with_kfunc(insn_buf, prog, ctx_stack_off);
	1428	+
1328	1429	/* r1 = stack[ctx_stack_off]; // r1 will be "u64 *ctx"
1329	1430	* r1 = r1[0]; // r1 will be "struct st_ops *args"
1330	1431	* r6 = r1->a;
@@ -1395,6 +1496,13 @@ static void st_ops_unreg(void kdata, struct bpf_link link) · 64 unmodified lines
1395	1496
1396	1497	static int st_ops_init(struct btf *btf)
1397	1498	{
	1499	+ struct btf *kfunc_btf;
	1500	+
	1501	+ bpf_cgroup_from_id_id = bpf_find_btf_id("bpf_cgroup_from_id", BTF_KIND_FUNC, &kfunc_btf);
	1502	+ bpf_cgroup_release_id = bpf_find_btf_id("bpf_cgroup_release", BTF_KIND_FUNC, &kfunc_btf);
	1503	+ if (bpf_cgroup_from_id_id < 0 \|\| bpf_cgroup_release_id < 0)
	1504	+ return -EINVAL;
	1505	+
1398	1506	return 0;
1399	1507	}
1400	1508

tools/testing/selftests/bpf/test_kmods/bpf_testmod.h+6−0

@@ -6,6 +6,7 @@ · 5 unmodified lines
6	6	#include <linux/types.h>
7	7
8	8	struct task_struct;
	9	+struct cgroup;
9	10
10	11	struct bpf_testmod_test_read_ctx {
11	12	char *buf;
@@ -36,6 +37,11 @@ struct bpf_testmod_ops { · 24 unmodified lines
36	37	/* Used to test nullable arguments. */
37	38	int (test_maybe_null)(int dummy, struct task_struct task);
38	39	int (*unsupported_ops)(void);
	40	+ /* Used to test ref_acquired arguments. */
	41	+ int (test_refcounted)(int dummy, struct task_struct task);
	42	+ /* Used to test returning referenced kptr. */
	43	+ struct task_struct (test_return_ref_kptr)(int dummy, struct task_struct *task,
	44	+ struct cgroup *cgrp);
39	45
40	46	/* The following fields are used to test shadow copies. */
41	47	char onebyte;

tools/testing/selftests/bpf/test_loader.c+29−3

@@ -37,6 +37,7 @@ · 36 unmodified lines
37	37	#define TEST_TAG_JITED_PFX "comment:test_jited="
38	38	#define TEST_TAG_JITED_PFX_UNPRIV "comment:test_jited_unpriv="
39	39	#define TEST_TAG_CAPS_UNPRIV "comment:test_caps_unpriv="
	40	+#define TEST_TAG_LOAD_MODE_PFX "comment:load_mode="
40	41
41	42	/* Warning: duplicated in bpf_misc.h */
42	43	#define POINTER_VALUE 0xcafe4all
@@ -55,6 +56,11 @@ enum mode { · 12 unmodified lines
55	56	UNPRIV = 2
56	57	};
57	58
	59	+enum load_mode {
	60	+ JITED = 1 << 0,
	61	+ NO_JITED = 1 << 1,
	62	+};
	63	+
58	64	struct expect_msg {
59	65	const char substr; / substring match */
60	66	regex_t regex;
@@ -87,6 +93,7 @@ struct test_spec { · 26 unmodified lines
87	93	int prog_flags;
88	94	int mode_mask;
89	95	int arch_mask;
	96	+ int load_mask;
90	97	bool auxiliary;
91	98	bool valid;
92	99	};
@@ -406,6 +413,7 @@ static int parse_test_spec(struct test_loader *tester, · 313 unmodified lines
406	413	bool collect_jit = false;
407	414	int func_id, i, err = 0;
408	415	u32 arch_mask = 0;
	416	+ u32 load_mask = 0;
409	417	struct btf *btf;
410	418	enum arch arch;
411	419
@@ -580,10 +588,22 @@ static int parse_test_spec(struct test_loader *tester, · 168 unmodified lines
580	588	if (err)
581	589	goto cleanup;
582	590	spec->mode_mask \|= UNPRIV;
	591	+ } else if (str_has_pfx(s, TEST_TAG_LOAD_MODE_PFX)) {
	592	+ val = s + sizeof(TEST_TAG_LOAD_MODE_PFX) - 1;
	593	+ if (strcmp(val, "jited") == 0) {
	594	+ load_mask = JITED;
	595	+ } else if (strcmp(val, "no_jited") == 0) {
	596	+ load_mask = NO_JITED;
	597	+ } else {
	598	+ PRINT_FAIL("bad load spec: '%s'", val);
	599	+ err = -EINVAL;
	600	+ goto cleanup;
	601	+ }
583	602	}
584	603	}
585	604
586	605	spec->arch_mask = arch_mask ?: -1;
	606	+ spec->load_mask = load_mask ?: (JITED \| NO_JITED);
587	607
588	608	if (spec->mode_mask == 0)
589	609	spec->mode_mask = PRIV;
@@ -773,7 +793,7 @@ static int drop_capabilities(struct cap_state *caps) · 183 unmodified lines
773	793
774	794	err = cap_disable_effective(caps_to_drop, &caps->old_caps);
775	795	if (err) {
776		− PRINT_FAIL("failed to drop capabilities: %i, %s\n", err, strerror(err));
	796	+ PRINT_FAIL("failed to drop capabilities: %i, %s\n", err, strerror(-err));
777	797	return err;
778	798	}
779	799
@@ -790,7 +810,7 @@ static int restore_capabilities(struct cap_state *caps) · 10 unmodified lines
790	810
791	811	err = cap_enable_effective(caps->old_caps, NULL);
792	812	if (err)
793		− PRINT_FAIL("failed to restore capabilities: %i, %s\n", err, strerror(err));
	813	+ PRINT_FAIL("failed to restore capabilities: %i, %s\n", err, strerror(-err));
794	814	caps->initialized = false;
795	815	return err;
796	816	}
@@ -928,6 +948,7 @@ void run_subtest(struct test_loader *tester, · 131 unmodified lines
928	948	bool unpriv)
929	949	{
930	950	struct test_subspec *subspec = unpriv ? &spec->unpriv : &spec->priv;
	951	+ int current_runtime = is_jit_enabled() ? JITED : NO_JITED;
931	952	struct bpf_program tprog = NULL, tprog_iter;
932	953	struct bpf_link link, links[32] = {};
933	954	struct test_spec *spec_iter;
@@ -946,6 +967,11 @@ void run_subtest(struct test_loader *tester, · 12 unmodified lines
946	967	return;
947	968	}
948	969
	970	+ if ((current_runtime & spec->load_mask) == 0) {
	971	+ test__skip();
	972	+ return;
	973	+ }
	974	+
949	975	if (unpriv) {
950	976	if (!can_execute_unpriv(tester, spec)) {
951	977	test__skip();
@@ -959,7 +985,7 @@ void run_subtest(struct test_loader *tester, · 7 unmodified lines
959	985	if (subspec->caps) {
960	986	err = cap_enable_effective(subspec->caps, NULL);
961	987	if (err) {
962		− PRINT_FAIL("failed to set capabilities: %i, %s\n", err, strerror(err));
	988	+ PRINT_FAIL("failed to set capabilities: %i, %s\n", err, strerror(-err));
963	989	goto subtest_cleanup;
964	990	}
965	991	}

tools/testing/selftests/bpf/test_lwt_ip_encap.sh+0−476

@@ -1,476 +0,0 @@
1		−#!/bin/bash
2		−# SPDX-License-Identifier: GPL-2.0
3		−#
4		−# Setup/topology:
5		−#
6		−# NS1 NS2 NS3
7		−# veth1 <---> veth2 veth3 <---> veth4 (the top route)
8		−# veth5 <---> veth6 veth7 <---> veth8 (the bottom route)
9		−#
10		−# each vethN gets IPv[4\|6]_N address
11		−#
12		−# IPv_SRC = IPv_1
13		−# IPv_DST = IPv_4
14		−#
15		−# all tests test pings from IPv_SRC to IPv_DST
16		−#
17		−# by default, routes are configured to allow packets to go
18		−# IP_1 <=> IP_2 <=> IP_3 <=> IP_4 (the top route)
19		−#
20		−# a GRE device is installed in NS3 with IPv*_GRE, and
21		−# NS1/NS2 are configured to route packets to IPv_GRE via IP_8
22		−# (the bottom route)
23		−#
24		−# Tests:
25		−#
26		−# 1. routes NS2->IPv*_DST are brought down, so the only way a ping
27		−# from IP_SRC to IP_DST can work is via IPv*_GRE
28		−#
29		−# 2a. in an egress test, a bpf LWT_XMIT program is installed on veth1
30		−# that encaps the packets with an IP/GRE header to route to IPv*_GRE
31		−#
32		−# ping: SRC->[encap at veth1:egress]->GRE:decap->DST
33		−# ping replies go DST->SRC directly
34		−#
35		−# 2b. in an ingress test, a bpf LWT_IN program is installed on veth2
36		−# that encaps the packets with an IP/GRE header to route to IPv*_GRE
37		−#
38		−# ping: SRC->[encap at veth2:ingress]->GRE:decap->DST
39		−# ping replies go DST->SRC directly
40		−
41		−BPF_FILE="test_lwt_ip_encap.bpf.o"
42		−if [[ $EUID -ne 0 ]]; then
43		− echo "This script must be run as root"
44		− echo "FAIL"
45		− exit 1
46		−fi
47		−
48		−readonly NS1="ns1-$(mktemp -u XXXXXX)"
49		−readonly NS2="ns2-$(mktemp -u XXXXXX)"
50		−readonly NS3="ns3-$(mktemp -u XXXXXX)"
51		−
52		−readonly IPv4_1="172.16.1.100"
53		−readonly IPv4_2="172.16.2.100"
54		−readonly IPv4_3="172.16.3.100"
55		−readonly IPv4_4="172.16.4.100"
56		−readonly IPv4_5="172.16.5.100"
57		−readonly IPv4_6="172.16.6.100"
58		−readonly IPv4_7="172.16.7.100"
59		−readonly IPv4_8="172.16.8.100"
60		−readonly IPv4_GRE="172.16.16.100"
61		−
62		−readonly IPv4_SRC=$IPv4_1
63		−readonly IPv4_DST=$IPv4_4
64		−
65		−readonly IPv6_1="fb01::1"
66		−readonly IPv6_2="fb02::1"
67		−readonly IPv6_3="fb03::1"
68		−readonly IPv6_4="fb04::1"
69		−readonly IPv6_5="fb05::1"
70		−readonly IPv6_6="fb06::1"
71		−readonly IPv6_7="fb07::1"
72		−readonly IPv6_8="fb08::1"
73		−readonly IPv6_GRE="fb10::1"
74		−
75		−readonly IPv6_SRC=$IPv6_1
76		−readonly IPv6_DST=$IPv6_4
77		−
78		−TEST_STATUS=0
79		−TESTS_SUCCEEDED=0
80		−TESTS_FAILED=0
81		−
82		−TMPFILE=""
83		−
84		−process_test_results()
85		−{
86		− if [[ "${TEST_STATUS}" -eq 0 ]] ; then
87		− echo "PASS"
88		− TESTS_SUCCEEDED=$((TESTS_SUCCEEDED+1))
89		− else
90		− echo "FAIL"
91		− TESTS_FAILED=$((TESTS_FAILED+1))
92		− fi
93		−}
94		−
95		−print_test_summary_and_exit()
96		−{
97		− echo "passed tests: ${TESTS_SUCCEEDED}"
98		− echo "failed tests: ${TESTS_FAILED}"
99		− if [ "${TESTS_FAILED}" -eq "0" ] ; then
100		− exit 0
101		− else
102		− exit 1
103		− fi
104		−}
105		−
106		−setup()
107		−{
108		− set -e # exit on error
109		− TEST_STATUS=0
110		−
111		− # create devices and namespaces
112		− ip netns add "${NS1}"
113		− ip netns add "${NS2}"
114		− ip netns add "${NS3}"
115		−
116		− # rp_filter gets confused by what these tests are doing, so disable it
117		− ip netns exec ${NS1} sysctl -wq net.ipv4.conf.all.rp_filter=0
118		− ip netns exec ${NS2} sysctl -wq net.ipv4.conf.all.rp_filter=0
119		− ip netns exec ${NS3} sysctl -wq net.ipv4.conf.all.rp_filter=0
120		− ip netns exec ${NS1} sysctl -wq net.ipv4.conf.default.rp_filter=0
121		− ip netns exec ${NS2} sysctl -wq net.ipv4.conf.default.rp_filter=0
122		− ip netns exec ${NS3} sysctl -wq net.ipv4.conf.default.rp_filter=0
123		−
124		− # disable IPv6 DAD because it sometimes takes too long and fails tests
125		− ip netns exec ${NS1} sysctl -wq net.ipv6.conf.all.accept_dad=0
126		− ip netns exec ${NS2} sysctl -wq net.ipv6.conf.all.accept_dad=0
127		− ip netns exec ${NS3} sysctl -wq net.ipv6.conf.all.accept_dad=0
128		− ip netns exec ${NS1} sysctl -wq net.ipv6.conf.default.accept_dad=0
129		− ip netns exec ${NS2} sysctl -wq net.ipv6.conf.default.accept_dad=0
130		− ip netns exec ${NS3} sysctl -wq net.ipv6.conf.default.accept_dad=0
131		−
132		− ip link add veth1 type veth peer name veth2
133		− ip link add veth3 type veth peer name veth4
134		− ip link add veth5 type veth peer name veth6
135		− ip link add veth7 type veth peer name veth8
136		−
137		− ip netns exec ${NS2} sysctl -wq net.ipv4.ip_forward=1
138		− ip netns exec ${NS2} sysctl -wq net.ipv6.conf.all.forwarding=1
139		−
140		− ip link set veth1 netns ${NS1}
141		− ip link set veth2 netns ${NS2}
142		− ip link set veth3 netns ${NS2}
143		− ip link set veth4 netns ${NS3}
144		− ip link set veth5 netns ${NS1}
145		− ip link set veth6 netns ${NS2}
146		− ip link set veth7 netns ${NS2}
147		− ip link set veth8 netns ${NS3}
148		−
149		− if [ ! -z "${VRF}" ] ; then
150		− ip -netns ${NS1} link add red type vrf table 1001
151		− ip -netns ${NS1} link set red up
152		− ip -netns ${NS1} route add table 1001 unreachable default metric 8192
153		− ip -netns ${NS1} -6 route add table 1001 unreachable default metric 8192
154		− ip -netns ${NS1} link set veth1 vrf red
155		− ip -netns ${NS1} link set veth5 vrf red
156		−
157		− ip -netns ${NS2} link add red type vrf table 1001
158		− ip -netns ${NS2} link set red up
159		− ip -netns ${NS2} route add table 1001 unreachable default metric 8192
160		− ip -netns ${NS2} -6 route add table 1001 unreachable default metric 8192
161		− ip -netns ${NS2} link set veth2 vrf red
162		− ip -netns ${NS2} link set veth3 vrf red
163		− ip -netns ${NS2} link set veth6 vrf red
164		− ip -netns ${NS2} link set veth7 vrf red
165		− fi
166		−
167		− # configure addesses: the top route (1-2-3-4)
168		− ip -netns ${NS1} addr add ${IPv4_1}/24 dev veth1
169		− ip -netns ${NS2} addr add ${IPv4_2}/24 dev veth2
170		− ip -netns ${NS2} addr add ${IPv4_3}/24 dev veth3
171		− ip -netns ${NS3} addr add ${IPv4_4}/24 dev veth4
172		− ip -netns ${NS1} -6 addr add ${IPv6_1}/128 nodad dev veth1
173		− ip -netns ${NS2} -6 addr add ${IPv6_2}/128 nodad dev veth2
174		− ip -netns ${NS2} -6 addr add ${IPv6_3}/128 nodad dev veth3
175		− ip -netns ${NS3} -6 addr add ${IPv6_4}/128 nodad dev veth4
176		−
177		− # configure addresses: the bottom route (5-6-7-8)
178		− ip -netns ${NS1} addr add ${IPv4_5}/24 dev veth5
179		− ip -netns ${NS2} addr add ${IPv4_6}/24 dev veth6
180		− ip -netns ${NS2} addr add ${IPv4_7}/24 dev veth7
181		− ip -netns ${NS3} addr add ${IPv4_8}/24 dev veth8
182		− ip -netns ${NS1} -6 addr add ${IPv6_5}/128 nodad dev veth5
183		− ip -netns ${NS2} -6 addr add ${IPv6_6}/128 nodad dev veth6
184		− ip -netns ${NS2} -6 addr add ${IPv6_7}/128 nodad dev veth7
185		− ip -netns ${NS3} -6 addr add ${IPv6_8}/128 nodad dev veth8
186		−
187		− ip -netns ${NS1} link set dev veth1 up
188		− ip -netns ${NS2} link set dev veth2 up
189		− ip -netns ${NS2} link set dev veth3 up
190		− ip -netns ${NS3} link set dev veth4 up
191		− ip -netns ${NS1} link set dev veth5 up
192		− ip -netns ${NS2} link set dev veth6 up
193		− ip -netns ${NS2} link set dev veth7 up
194		− ip -netns ${NS3} link set dev veth8 up
195		−
196		− # configure routes: IP_SRC -> veth1/IP_2 (= top route) default;
197		− # the bottom route to specific bottom addresses
198		−
199		− # NS1
200		− # top route
201		− ip -netns ${NS1} route add ${IPv4_2}/32 dev veth1 ${VRF}
202		− ip -netns ${NS1} route add default dev veth1 via ${IPv4_2} ${VRF} # go top by default
203		− ip -netns ${NS1} -6 route add ${IPv6_2}/128 dev veth1 ${VRF}
204		− ip -netns ${NS1} -6 route add default dev veth1 via ${IPv6_2} ${VRF} # go top by default
205		− # bottom route
206		− ip -netns ${NS1} route add ${IPv4_6}/32 dev veth5 ${VRF}
207		− ip -netns ${NS1} route add ${IPv4_7}/32 dev veth5 via ${IPv4_6} ${VRF}
208		− ip -netns ${NS1} route add ${IPv4_8}/32 dev veth5 via ${IPv4_6} ${VRF}
209		− ip -netns ${NS1} -6 route add ${IPv6_6}/128 dev veth5 ${VRF}
210		− ip -netns ${NS1} -6 route add ${IPv6_7}/128 dev veth5 via ${IPv6_6} ${VRF}
211		− ip -netns ${NS1} -6 route add ${IPv6_8}/128 dev veth5 via ${IPv6_6} ${VRF}
212		−
213		− # NS2
214		− # top route
215		− ip -netns ${NS2} route add ${IPv4_1}/32 dev veth2 ${VRF}
216		− ip -netns ${NS2} route add ${IPv4_4}/32 dev veth3 ${VRF}
217		− ip -netns ${NS2} -6 route add ${IPv6_1}/128 dev veth2 ${VRF}
218		− ip -netns ${NS2} -6 route add ${IPv6_4}/128 dev veth3 ${VRF}
219		− # bottom route
220		− ip -netns ${NS2} route add ${IPv4_5}/32 dev veth6 ${VRF}
221		− ip -netns ${NS2} route add ${IPv4_8}/32 dev veth7 ${VRF}
222		− ip -netns ${NS2} -6 route add ${IPv6_5}/128 dev veth6 ${VRF}
223		− ip -netns ${NS2} -6 route add ${IPv6_8}/128 dev veth7 ${VRF}
224		−
225		− # NS3
226		− # top route
227		− ip -netns ${NS3} route add ${IPv4_3}/32 dev veth4
228		− ip -netns ${NS3} route add ${IPv4_1}/32 dev veth4 via ${IPv4_3}
229		− ip -netns ${NS3} route add ${IPv4_2}/32 dev veth4 via ${IPv4_3}
230		− ip -netns ${NS3} -6 route add ${IPv6_3}/128 dev veth4
231		− ip -netns ${NS3} -6 route add ${IPv6_1}/128 dev veth4 via ${IPv6_3}
232		− ip -netns ${NS3} -6 route add ${IPv6_2}/128 dev veth4 via ${IPv6_3}
233		− # bottom route
234		− ip -netns ${NS3} route add ${IPv4_7}/32 dev veth8
235		− ip -netns ${NS3} route add ${IPv4_5}/32 dev veth8 via ${IPv4_7}
236		− ip -netns ${NS3} route add ${IPv4_6}/32 dev veth8 via ${IPv4_7}
237		− ip -netns ${NS3} -6 route add ${IPv6_7}/128 dev veth8
238		− ip -netns ${NS3} -6 route add ${IPv6_5}/128 dev veth8 via ${IPv6_7}
239		− ip -netns ${NS3} -6 route add ${IPv6_6}/128 dev veth8 via ${IPv6_7}
240		−
241		− # configure IPv4 GRE device in NS3, and a route to it via the "bottom" route
242		− ip -netns ${NS3} tunnel add gre_dev mode gre remote ${IPv4_1} local ${IPv4_GRE} ttl 255
243		− ip -netns ${NS3} link set gre_dev up
244		− ip -netns ${NS3} addr add ${IPv4_GRE} dev gre_dev
245		− ip -netns ${NS1} route add ${IPv4_GRE}/32 dev veth5 via ${IPv4_6} ${VRF}
246		− ip -netns ${NS2} route add ${IPv4_GRE}/32 dev veth7 via ${IPv4_8} ${VRF}
247		−
248		−
249		− # configure IPv6 GRE device in NS3, and a route to it via the "bottom" route
250		− ip -netns ${NS3} -6 tunnel add name gre6_dev mode ip6gre remote ${IPv6_1} local ${IPv6_GRE} ttl 255
251		− ip -netns ${NS3} link set gre6_dev up
252		− ip -netns ${NS3} -6 addr add ${IPv6_GRE} nodad dev gre6_dev
253		− ip -netns ${NS1} -6 route add ${IPv6_GRE}/128 dev veth5 via ${IPv6_6} ${VRF}
254		− ip -netns ${NS2} -6 route add ${IPv6_GRE}/128 dev veth7 via ${IPv6_8} ${VRF}
255		−
256		− TMPFILE=$(mktemp /tmp/test_lwt_ip_encap.XXXXXX)
257		−
258		− sleep 1 # reduce flakiness
259		− set +e
260		−}
261		−
262		−cleanup()
263		−{
264		− if [ -f ${TMPFILE} ] ; then
265		− rm ${TMPFILE}
266		− fi
267		−
268		− ip netns del ${NS1} 2> /dev/null
269		− ip netns del ${NS2} 2> /dev/null
270		− ip netns del ${NS3} 2> /dev/null
271		−}
272		−
273		−trap cleanup EXIT
274		−
275		−remove_routes_to_gredev()
276		−{
277		− ip -netns ${NS1} route del ${IPv4_GRE} dev veth5 ${VRF}
278		− ip -netns ${NS2} route del ${IPv4_GRE} dev veth7 ${VRF}
279		− ip -netns ${NS1} -6 route del ${IPv6_GRE}/128 dev veth5 ${VRF}
280		− ip -netns ${NS2} -6 route del ${IPv6_GRE}/128 dev veth7 ${VRF}
281		−}
282		−
283		−add_unreachable_routes_to_gredev()
284		−{
285		− ip -netns ${NS1} route add unreachable ${IPv4_GRE}/32 ${VRF}
286		− ip -netns ${NS2} route add unreachable ${IPv4_GRE}/32 ${VRF}
287		− ip -netns ${NS1} -6 route add unreachable ${IPv6_GRE}/128 ${VRF}
288		− ip -netns ${NS2} -6 route add unreachable ${IPv6_GRE}/128 ${VRF}
289		−}
290		−
291		−test_ping()
292		−{
293		− local readonly PROTO=$1
294		− local readonly EXPECTED=$2
295		− local RET=0
296		−
297		− if [ "${PROTO}" == "IPv4" ] ; then
298		− ip netns exec ${NS1} ping -c 1 -W 1 -I veth1 ${IPv4_DST} 2>&1 > /dev/null
299		− RET=$?
300		− elif [ "${PROTO}" == "IPv6" ] ; then
301		− ip netns exec ${NS1} ping6 -c 1 -W 1 -I veth1 ${IPv6_DST} 2>&1 > /dev/null
302		− RET=$?
303		− else
304		− echo " test_ping: unknown PROTO: ${PROTO}"
305		− TEST_STATUS=1
306		− fi
307		−
308		− if [ "0" != "${RET}" ]; then
309		− RET=1
310		− fi
311		−
312		− if [ "${EXPECTED}" != "${RET}" ] ; then
313		− echo " test_ping failed: expected: ${EXPECTED}; got ${RET}"
314		− TEST_STATUS=1
315		− fi
316		−}
317		−
318		−test_gso()
319		−{
320		− local readonly PROTO=$1
321		− local readonly PKT_SZ=5000
322		− local IP_DST=""
323		− : > ${TMPFILE} # trim the capture file
324		−
325		− # check that nc is present
326		− command -v nc >/dev/null 2>&1 \|\| \
327		− { echo >&2 "nc is not available: skipping TSO tests"; return; }
328		−
329		− # listen on port 9000, capture TCP into $TMPFILE
330		− if [ "${PROTO}" == "IPv4" ] ; then
331		− IP_DST=${IPv4_DST}
332		− ip netns exec ${NS3} bash -c \
333		− "nc -4 -l -p 9000 > ${TMPFILE} &"
334		− elif [ "${PROTO}" == "IPv6" ] ; then
335		− IP_DST=${IPv6_DST}
336		− ip netns exec ${NS3} bash -c \
337		− "nc -6 -l -p 9000 > ${TMPFILE} &"
338		− RET=$?
339		− else
340		− echo " test_gso: unknown PROTO: ${PROTO}"
341		− TEST_STATUS=1
342		− fi
343		− sleep 1 # let nc start listening
344		−
345		− # send a packet larger than MTU
346		− ip netns exec ${NS1} bash -c \
347		− "dd if=/dev/zero bs=$PKT_SZ count=1 > /dev/tcp/${IP_DST}/9000 2>/dev/null"
348		− sleep 2 # let the packet get delivered
349		−
350		− # verify we received all expected bytes
351		− SZ=$(stat -c %s ${TMPFILE})
352		− if [ "$SZ" != "$PKT_SZ" ] ; then
353		− echo " test_gso failed: ${PROTO}"
354		− TEST_STATUS=1
355		− fi
356		−}
357		−
358		−test_egress()
359		−{
360		− local readonly ENCAP=$1
361		− echo "starting egress ${ENCAP} encap test ${VRF}"
362		− setup
363		−
364		− # by default, pings work
365		− test_ping IPv4 0
366		− test_ping IPv6 0
367		−
368		− # remove NS2->DST routes, ping fails
369		− ip -netns ${NS2} route del ${IPv4_DST}/32 dev veth3 ${VRF}
370		− ip -netns ${NS2} -6 route del ${IPv6_DST}/128 dev veth3 ${VRF}
371		− test_ping IPv4 1
372		− test_ping IPv6 1
373		−
374		− # install replacement routes (LWT/eBPF), pings succeed
375		− if [ "${ENCAP}" == "IPv4" ] ; then
376		− ip -netns ${NS1} route add ${IPv4_DST} encap bpf xmit obj \
377		− ${BPF_FILE} sec encap_gre dev veth1 ${VRF}
378		− ip -netns ${NS1} -6 route add ${IPv6_DST} encap bpf xmit obj \
379		− ${BPF_FILE} sec encap_gre dev veth1 ${VRF}
380		− elif [ "${ENCAP}" == "IPv6" ] ; then
381		− ip -netns ${NS1} route add ${IPv4_DST} encap bpf xmit obj \
382		− ${BPF_FILE} sec encap_gre6 dev veth1 ${VRF}
383		− ip -netns ${NS1} -6 route add ${IPv6_DST} encap bpf xmit obj \
384		− ${BPF_FILE} sec encap_gre6 dev veth1 ${VRF}
385		− else
386		− echo " unknown encap ${ENCAP}"
387		− TEST_STATUS=1
388		− fi
389		− test_ping IPv4 0
390		− test_ping IPv6 0
391		−
392		− # skip GSO tests with VRF: VRF routing needs properly assigned
393		− # source IP/device, which is easy to do with ping and hard with dd/nc.
394		− if [ -z "${VRF}" ] ; then
395		− test_gso IPv4
396		− test_gso IPv6
397		− fi
398		−
399		− # a negative test: remove routes to GRE devices: ping fails
400		− remove_routes_to_gredev
401		− test_ping IPv4 1
402		− test_ping IPv6 1
403		−
404		− # another negative test
405		− add_unreachable_routes_to_gredev
406		− test_ping IPv4 1
407		− test_ping IPv6 1
408		−
409		− cleanup
410		− process_test_results
411		−}
412		−
413		−test_ingress()
414		−{
415		− local readonly ENCAP=$1
416		− echo "starting ingress ${ENCAP} encap test ${VRF}"
417		− setup
418		−
419		− # need to wait a bit for IPv6 to autoconf, otherwise
420		− # ping6 sometimes fails with "unable to bind to address"
421		−
422		− # by default, pings work
423		− test_ping IPv4 0
424		− test_ping IPv6 0
425		−
426		− # remove NS2->DST routes, pings fail
427		− ip -netns ${NS2} route del ${IPv4_DST}/32 dev veth3 ${VRF}
428		− ip -netns ${NS2} -6 route del ${IPv6_DST}/128 dev veth3 ${VRF}
429		− test_ping IPv4 1
430		− test_ping IPv6 1
431		−
432		− # install replacement routes (LWT/eBPF), pings succeed
433		− if [ "${ENCAP}" == "IPv4" ] ; then
434		− ip -netns ${NS2} route add ${IPv4_DST} encap bpf in obj \
435		− ${BPF_FILE} sec encap_gre dev veth2 ${VRF}
436		− ip -netns ${NS2} -6 route add ${IPv6_DST} encap bpf in obj \
437		− ${BPF_FILE} sec encap_gre dev veth2 ${VRF}
438		− elif [ "${ENCAP}" == "IPv6" ] ; then
439		− ip -netns ${NS2} route add ${IPv4_DST} encap bpf in obj \
440		− ${BPF_FILE} sec encap_gre6 dev veth2 ${VRF}
441		− ip -netns ${NS2} -6 route add ${IPv6_DST} encap bpf in obj \
442		− ${BPF_FILE} sec encap_gre6 dev veth2 ${VRF}
443		− else
444		− echo "FAIL: unknown encap ${ENCAP}"
445		− TEST_STATUS=1
446		− fi
447		− test_ping IPv4 0
448		− test_ping IPv6 0
449		−
450		− # a negative test: remove routes to GRE devices: ping fails
451		− remove_routes_to_gredev
452		− test_ping IPv4 1
453		− test_ping IPv6 1
454		−
455		− # another negative test
456		− add_unreachable_routes_to_gredev
457		− test_ping IPv4 1
458		− test_ping IPv6 1
459		−
460		− cleanup
461		− process_test_results
462		−}
463		−
464		−VRF=""
465		−test_egress IPv4
466		−test_egress IPv6
467		−test_ingress IPv4
468		−test_ingress IPv6
469		−
470		−VRF="vrf red"
471		−test_egress IPv4
472		−test_egress IPv6
473		−test_ingress IPv4
474		−test_ingress IPv6
475		−
476		−print_test_summary_and_exit

tools/testing/selftests/bpf/test_lwt_seg6local.sh+0−156

@@ -1,156 +0,0 @@
1		−#!/bin/bash
2		−# Connects 6 network namespaces through veths.
3		−# Each NS may have different IPv6 global scope addresses :
4		−# NS1 ---- NS2 ---- NS3 ---- NS4 ---- NS5 ---- NS6
5		−# fb00::1 fd00::1 fd00::2 fd00::3 fb00::6
6		−# fc42::1 fd00::4
7		−#
8		−# All IPv6 packets going to fb00::/16 through NS2 will be encapsulated in a
9		−# IPv6 header with a Segment Routing Header, with segments :
10		−# fd00::1 -> fd00::2 -> fd00::3 -> fd00::4
11		−#
12		−# 3 fd00::/16 IPv6 addresses are binded to seg6local End.BPF actions :
13		−# - fd00::1 : add a TLV, change the flags and apply a End.X action to fc42::1
14		−# - fd00::2 : remove the TLV, change the flags, add a tag
15		−# - fd00::3 : apply an End.T action to fd00::4, through routing table 117
16		−#
17		−# fd00::4 is a simple Segment Routing node decapsulating the inner IPv6 packet.
18		−# Each End.BPF action will validate the operations applied on the SRH by the
19		−# previous BPF program in the chain, otherwise the packet is dropped.
20		−#
21		−# An UDP datagram is sent from fb00::1 to fb00::6. The test succeeds if this
22		−# datagram can be read on NS6 when binding to fb00::6.
23		−
24		−# Kselftest framework requirement - SKIP code is 4.
25		−ksft_skip=4
26		−BPF_FILE="test_lwt_seg6local.bpf.o"
27		−readonly NS1="ns1-$(mktemp -u XXXXXX)"
28		−readonly NS2="ns2-$(mktemp -u XXXXXX)"
29		−readonly NS3="ns3-$(mktemp -u XXXXXX)"
30		−readonly NS4="ns4-$(mktemp -u XXXXXX)"
31		−readonly NS5="ns5-$(mktemp -u XXXXXX)"
32		−readonly NS6="ns6-$(mktemp -u XXXXXX)"
33		−
34		−msg="skip all tests:"
35		−if [ $UID != 0 ]; then
36		− echo $msg please run this as root >&2
37		− exit $ksft_skip
38		−fi
39		−
40		−TMP_FILE="/tmp/selftest_lwt_seg6local.txt"
41		−
42		−cleanup()
43		−{
44		− if [ "$?" = "0" ]; then
45		− echo "selftests: test_lwt_seg6local [PASS]";
46		− else
47		− echo "selftests: test_lwt_seg6local [FAILED]";
48		− fi
49		−
50		− set +e
51		− ip netns del ${NS1} 2> /dev/null
52		− ip netns del ${NS2} 2> /dev/null
53		− ip netns del ${NS3} 2> /dev/null
54		− ip netns del ${NS4} 2> /dev/null
55		− ip netns del ${NS5} 2> /dev/null
56		− ip netns del ${NS6} 2> /dev/null
57		− rm -f $TMP_FILE
58		−}
59		−
60		−set -e
61		−
62		−ip netns add ${NS1}
63		−ip netns add ${NS2}
64		−ip netns add ${NS3}
65		−ip netns add ${NS4}
66		−ip netns add ${NS5}
67		−ip netns add ${NS6}
68		−
69		−trap cleanup 0 2 3 6 9
70		−
71		−ip link add veth1 type veth peer name veth2
72		−ip link add veth3 type veth peer name veth4
73		−ip link add veth5 type veth peer name veth6
74		−ip link add veth7 type veth peer name veth8
75		−ip link add veth9 type veth peer name veth10
76		−
77		−ip link set veth1 netns ${NS1}
78		−ip link set veth2 netns ${NS2}
79		−ip link set veth3 netns ${NS2}
80		−ip link set veth4 netns ${NS3}
81		−ip link set veth5 netns ${NS3}
82		−ip link set veth6 netns ${NS4}
83		−ip link set veth7 netns ${NS4}
84		−ip link set veth8 netns ${NS5}
85		−ip link set veth9 netns ${NS5}
86		−ip link set veth10 netns ${NS6}
87		−
88		−ip netns exec ${NS1} ip link set dev veth1 up
89		−ip netns exec ${NS2} ip link set dev veth2 up
90		−ip netns exec ${NS2} ip link set dev veth3 up
91		−ip netns exec ${NS3} ip link set dev veth4 up
92		−ip netns exec ${NS3} ip link set dev veth5 up
93		−ip netns exec ${NS4} ip link set dev veth6 up
94		−ip netns exec ${NS4} ip link set dev veth7 up
95		−ip netns exec ${NS5} ip link set dev veth8 up
96		−ip netns exec ${NS5} ip link set dev veth9 up
97		−ip netns exec ${NS6} ip link set dev veth10 up
98		−ip netns exec ${NS6} ip link set dev lo up
99		−
100		−# All link scope addresses and routes required between veths
101		−ip netns exec ${NS1} ip -6 addr add fb00::12/16 dev veth1 scope link
102		−ip netns exec ${NS1} ip -6 route add fb00::21 dev veth1 scope link
103		−ip netns exec ${NS2} ip -6 addr add fb00::21/16 dev veth2 scope link
104		−ip netns exec ${NS2} ip -6 addr add fb00::34/16 dev veth3 scope link
105		−ip netns exec ${NS2} ip -6 route add fb00::43 dev veth3 scope link
106		−ip netns exec ${NS3} ip -6 route add fb00::65 dev veth5 scope link
107		−ip netns exec ${NS3} ip -6 addr add fb00::43/16 dev veth4 scope link
108		−ip netns exec ${NS3} ip -6 addr add fb00::56/16 dev veth5 scope link
109		−ip netns exec ${NS4} ip -6 addr add fb00::65/16 dev veth6 scope link
110		−ip netns exec ${NS4} ip -6 addr add fb00::78/16 dev veth7 scope link
111		−ip netns exec ${NS4} ip -6 route add fb00::87 dev veth7 scope link
112		−ip netns exec ${NS5} ip -6 addr add fb00::87/16 dev veth8 scope link
113		−ip netns exec ${NS5} ip -6 addr add fb00::910/16 dev veth9 scope link
114		−ip netns exec ${NS5} ip -6 route add fb00::109 dev veth9 scope link
115		−ip netns exec ${NS5} ip -6 route add fb00::109 table 117 dev veth9 scope link
116		−ip netns exec ${NS6} ip -6 addr add fb00::109/16 dev veth10 scope link
117		−
118		−ip netns exec ${NS1} ip -6 addr add fb00::1/16 dev lo
119		−ip netns exec ${NS1} ip -6 route add fb00::6 dev veth1 via fb00::21
120		−
121		−ip netns exec ${NS2} ip -6 route add fb00::6 encap bpf in obj ${BPF_FILE} sec encap_srh dev veth2
122		−ip netns exec ${NS2} ip -6 route add fd00::1 dev veth3 via fb00::43 scope link
123		−
124		−ip netns exec ${NS3} ip -6 route add fc42::1 dev veth5 via fb00::65
125		−ip netns exec ${NS3} ip -6 route add fd00::1 encap seg6local action End.BPF endpoint obj ${BPF_FILE} sec add_egr_x dev veth4
126		−
127		−ip netns exec ${NS4} ip -6 route add fd00::2 encap seg6local action End.BPF endpoint obj ${BPF_FILE} sec pop_egr dev veth6
128		−ip netns exec ${NS4} ip -6 addr add fc42::1 dev lo
129		−ip netns exec ${NS4} ip -6 route add fd00::3 dev veth7 via fb00::87
130		−
131		−ip netns exec ${NS5} ip -6 route add fd00::4 table 117 dev veth9 via fb00::109
132		−ip netns exec ${NS5} ip -6 route add fd00::3 encap seg6local action End.BPF endpoint obj ${BPF_FILE} sec inspect_t dev veth8
133		−
134		−ip netns exec ${NS6} ip -6 addr add fb00::6/16 dev lo
135		−ip netns exec ${NS6} ip -6 addr add fd00::4/16 dev lo
136		−
137		−ip netns exec ${NS1} sysctl net.ipv6.conf.all.forwarding=1 > /dev/null
138		−ip netns exec ${NS2} sysctl net.ipv6.conf.all.forwarding=1 > /dev/null
139		−ip netns exec ${NS3} sysctl net.ipv6.conf.all.forwarding=1 > /dev/null
140		−ip netns exec ${NS4} sysctl net.ipv6.conf.all.forwarding=1 > /dev/null
141		−ip netns exec ${NS5} sysctl net.ipv6.conf.all.forwarding=1 > /dev/null
142		−
143		−ip netns exec ${NS6} sysctl net.ipv6.conf.all.seg6_enabled=1 > /dev/null
144		−ip netns exec ${NS6} sysctl net.ipv6.conf.lo.seg6_enabled=1 > /dev/null
145		−ip netns exec ${NS6} sysctl net.ipv6.conf.veth10.seg6_enabled=1 > /dev/null
146		−
147		−ip netns exec ${NS6} nc -l -6 -u -d 7330 > $TMP_FILE &
148		−ip netns exec ${NS1} bash -c "echo 'foobar' \| nc -w0 -6 -u -p 2121 -s fb00::1 fb00::6 7330"
149		−sleep 5 # wait enough time to ensure the UDP datagram arrived to the last segment
150		−kill -TERM $!
151		−
152		−if [[ $(< $TMP_FILE) != "foobar" ]]; then
153		− exit 1
154		−fi
155		−
156		−exit 0

tools/testing/selftests/bpf/test_maps.c+5−4

@@ -1396,9 +1396,10 @@ static void test_map_stress(void) · 1395 unmodified lines
1396	1396	#define MAX_DELAY_US 50000
1397	1397	#define MIN_DELAY_RANGE_US 5000
1398	1398
1399		−static bool retry_for_again_or_busy(int err)
	1399	+static bool can_retry(int err)
1400	1400	{
1401		− return (err == EAGAIN \|\| err == EBUSY);
	1401	+ return (err == EAGAIN \|\| err == EBUSY \|\|
	1402	+ (err == ENOMEM && map_opts.map_flags == BPF_F_NO_PREALLOC));
1402	1403	}
1403	1404
1404	1405	int map_update_retriable(int map_fd, const void key, const void value, int flags, int attempts,
@@ -1451,12 +1452,12 @@ static void test_update_delete(unsigned int fn, void *data) · 46 unmodified lines
1451	1452
1452	1453	if (do_update) {
1453	1454	err = map_update_retriable(fd, &key, &value, BPF_NOEXIST, MAP_RETRIES,
1454		− retry_for_again_or_busy);
	1455	+ can_retry);
1455	1456	if (err)
1456	1457	printf("error %d %d\n", err, errno);
1457	1458	assert(err == 0);
1458	1459	err = map_update_retriable(fd, &key, &value, BPF_EXIST, MAP_RETRIES,
1459		− retry_for_again_or_busy);
	1460	+ can_retry);
1460	1461	if (err)
1461	1462	printf("error %d %d\n", err, errno);
1462	1463	assert(err == 0);

tools/testing/selftests/bpf/test_progs.c+43−29

@@ -88,7 +88,9 @@ static void stdio_hijack(char *log_buf, size_t log_cnt) · 87 unmodified lines
88	88	#endif
89	89	}
90	90
91		−static void stdio_restore_cleanup(void)
	91	+static pthread_mutex_t stdout_lock = PTHREAD_MUTEX_INITIALIZER;
	92	+
	93	+static void stdio_restore(void)
92	94	{
93	95	#ifdef __GLIBC__
94	96	if (verbose() && env.worker_id == -1) {
@@ -98,34 +100,33 @@ static void stdio_restore_cleanup(void) · 3 unmodified lines
98	100
99	101	fflush(stdout);
100	102
	103	+ pthread_mutex_lock(&stdout_lock);
	104	+
101	105	if (env.subtest_state) {
102		− fclose(env.subtest_state->stdout_saved);
	106	+ if (env.subtest_state->stdout_saved)
	107	+ fclose(env.subtest_state->stdout_saved);
103	108	env.subtest_state->stdout_saved = NULL;
104	109	stdout = env.test_state->stdout_saved;
105	110	stderr = env.test_state->stdout_saved;
106	111	} else {
107		− fclose(env.test_state->stdout_saved);
	112	+ if (env.test_state->stdout_saved)
	113	+ fclose(env.test_state->stdout_saved);
108	114	env.test_state->stdout_saved = NULL;
	115	+ stdout = env.stdout_saved;
	116	+ stderr = env.stderr_saved;
109	117	}
	118	+
	119	+ pthread_mutex_unlock(&stdout_lock);
110	120	#endif
111	121	}
112	122
113		−static void stdio_restore(void)
	123	+static int traffic_monitor_print_fn(const char *format, va_list args)
114	124	{
115		−#ifdef __GLIBC__
116		− if (verbose() && env.worker_id == -1) {
117		− /* nothing to do, output to stdout by default */
118		− return;
119		− }
120		−
121		− if (stdout == env.stdout_saved)
122		− return;
123		−
124		− stdio_restore_cleanup();
	125	+ pthread_mutex_lock(&stdout_lock);
	126	+ vfprintf(stdout, format, args);
	127	+ pthread_mutex_unlock(&stdout_lock);
125	128
126		− stdout = env.stdout_saved;
127		− stderr = env.stderr_saved;
128		−#endif
	129	+ return 0;
129	130	}
130	131
131	132	/* Adapted from perf/util/string.c */
@@ -474,8 +475,6 @@ static void dump_test_log(const struct prog_test_def *test, · 342 unmodified lines
474	475	print_test_result(test, test_state);
475	476	}
476	477
477		−static void stdio_restore(void);
478		−
479	478	/* A bunch of tests set custom affinity per-thread and/or per-process. Reset
480	479	* it after each test/sub-test.
481	480	*/
@@ -490,13 +489,11 @@ static void reset_affinity(void) · 8 unmodified lines
490	489
491	490	err = sched_setaffinity(0, sizeof(cpuset), &cpuset);
492	491	if (err < 0) {
493		− stdio_restore();
494	492	fprintf(stderr, "Failed to reset process affinity: %d!\n", err);
495	493	exit(EXIT_ERR_SETUP_INFRA);
496	494	}
497	495	err = pthread_setaffinity_np(pthread_self(), sizeof(cpuset), &cpuset);
498	496	if (err < 0) {
499		− stdio_restore();
500	497	fprintf(stderr, "Failed to reset thread affinity: %d!\n", err);
501	498	exit(EXIT_ERR_SETUP_INFRA);
502	499	}
@@ -514,7 +511,6 @@ static void save_netns(void) · 11 unmodified lines
514	511	static void restore_netns(void)
515	512	{
516	513	if (setns(env.saved_netns_fd, CLONE_NEWNET) == -1) {
517		− stdio_restore();
518	514	perror("setns(CLONE_NEWNS)");
519	515	exit(EXIT_ERR_SETUP_INFRA);
520	516	}
@@ -541,7 +537,8 @@ void test__end_subtest(void) · 20 unmodified lines
541	537	test_result(subtest_state->error_cnt,
542	538	subtest_state->skipped));
543	539
544		− stdio_restore_cleanup();
	540	+ stdio_restore();
	541	+
545	542	env.subtest_state = NULL;
546	543	}
547	544
@@ -1270,8 +1267,10 @@ void crash_handler(int signum) · 722 unmodified lines
1270	1267
1271	1268	sz = backtrace(bt, ARRAY_SIZE(bt));
1272	1269
1273		− if (env.stdout_saved)
1274		− stdio_restore();
	1270	+ fflush(stdout);
	1271	+ stdout = env.stdout_saved;
	1272	+ stderr = env.stderr_saved;
	1273	+
1275	1274	if (env.test) {
1276	1275	env.test_state->error_cnt++;
1277	1276	dump_test_log(env.test, env.test_state, true, false, NULL);
@@ -1365,10 +1364,19 @@ static int recv_message(int sock, struct msg *msg) · 87 unmodified lines
1365	1364	return ret;
1366	1365	}
1367	1366
	1367	+static bool ns_is_needed(const char *test_name)
	1368	+{
	1369	+ if (strlen(test_name) < 3)
	1370	+ return false;
	1371	+
	1372	+ return !strncmp(test_name, "ns_", 3);
	1373	+}
	1374	+
1368	1375	static void run_one_test(int test_num)
1369	1376	{
1370	1377	struct prog_test_def *test = &prog_test_defs[test_num];
1371	1378	struct test_state *state = &test_states[test_num];
	1379	+ struct netns_obj *ns = NULL;
1372	1380
1373	1381	env.test = test;
1374	1382	env.test_state = state;
@@ -1376,10 +1384,13 @@ static void run_one_test(int test_num) · 1 unmodified line
1376	1384	stdio_hijack(&state->log_buf, &state->log_cnt);
1377	1385
1378	1386	watchdog_start();
	1387	+ if (ns_is_needed(test->test_name))
	1388	+ ns = netns_new(test->test_name, true);
1379	1389	if (test->run_test)
1380	1390	test->run_test();
1381	1391	else if (test->run_serial_test)
1382	1392	test->run_serial_test();
	1393	+ netns_free(ns);
1383	1394	watchdog_stop();
1384	1395
1385	1396	/* ensure last sub-test is finalized properly */
@@ -1388,6 +1399,8 @@ static void run_one_test(int test_num) · 2 unmodified lines
1388	1399
1389	1400	state->tested = true;
1390	1401
	1402	+ stdio_restore();
	1403	+
1391	1404	if (verbose() && env.worker_id == -1)
1392	1405	print_test_result(test, state);
1393	1406
@@ -1396,7 +1409,6 @@ static void run_one_test(int test_num) · 2 unmodified lines
1396	1409	if (test->need_cgroup_cleanup)
1397	1410	cleanup_cgroup_environment();
1398	1411
1399		− stdio_restore();
1400	1412	free(stop_libbpf_log_capture());
1401	1413
1402	1414	dump_test_log(test, state, false, false, NULL);
@@ -1931,6 +1943,9 @@ int main(int argc, char **argv) · 528 unmodified lines
1931	1943
1932	1944	sigaction(SIGSEGV, &sigact, NULL);
1933	1945
	1946	+ env.stdout_saved = stdout;
	1947	+ env.stderr_saved = stderr;
	1948	+
1934	1949	env.secs_till_notify = 10;
1935	1950	env.secs_till_kill = 120;
1936	1951	err = argp_parse(&argp, argc, argv, 0, NULL, &env);
@@ -1947,6 +1962,8 @@ int main(int argc, char **argv) · 10 unmodified lines
1947	1962	libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
1948	1963	libbpf_set_print(libbpf_print_fn);
1949	1964
	1965	+ traffic_monitor_set_print(traffic_monitor_print_fn);
	1966	+
1950	1967	srand(time(NULL));
1951	1968
1952	1969	env.jit_enabled = is_jit_enabled();
@@ -1957,9 +1974,6 @@ int main(int argc, char **argv) · 4 unmodified lines
1957	1974	return -1;
1958	1975	}
1959	1976
1960		− env.stdout_saved = stdout;
1961		− env.stderr_saved = stderr;
1962		−
1963	1977	env.has_testmod = true;
1964	1978	if (!env.list_test_names) {
1965	1979	/* ensure previous instance of the module is unloaded */

tools/testing/selftests/bpf/test_progs.h+8−0

@@ -427,6 +427,14 @@ void hexdump(const char prefix, const void buf, size_t len); · 426 unmodified lines
427	427	goto goto_label; \
428	428	})
429	429
	430	+#define SYS_FAIL(goto_label, fmt, ...) \
	431	+ ({ \
	432	+ char cmd[1024]; \
	433	+ snprintf(cmd, sizeof(cmd), fmt, ##__VA_ARGS__); \
	434	+ if (!ASSERT_NEQ(0, system(cmd), cmd)) \
	435	+ goto goto_label; \
	436	+ })
	437	+
430	438	#define ALL_TO_DEV_NULL " >/dev/null 2>&1"
431	439
432	440	#define SYS_NOFAIL(fmt, ...) \

tools/testing/selftests/bpf/test_tunnel.sh+0−645

@@ -1,645 +0,0 @@
1		−#!/bin/bash
2		−# SPDX-License-Identifier: GPL-2.0
3		−
4		−# End-to-end eBPF tunnel test suite
5		−# The script tests BPF network tunnel implementation.
6		−#
7		−# Topology:
8		−# ---------
9		−# root namespace \| at_ns0 namespace
10		−# \|
11		−# ----------- \| -----------
12		−# \| tnl dev \| \| \| tnl dev \| (overlay network)
13		−# ----------- \| -----------
14		−# metadata-mode \| native-mode
15		−# with bpf \|
16		−# \|
17		−# ---------- \| ----------
18		−# \| veth1 \| --------- \| veth0 \| (underlay network)
19		−# ---------- peer ----------
20		−#
21		−#
22		−# Device Configuration
23		−# --------------------
24		−# Root namespace with metadata-mode tunnel + BPF
25		−# Device names and addresses:
26		−# veth1 IP: 172.16.1.200, IPv6: 00::22 (underlay)
27		−# tunnel dev <type>11, ex: gre11, IPv4: 10.1.1.200, IPv6: 1::22 (overlay)
28		−#
29		−# Namespace at_ns0 with native tunnel
30		−# Device names and addresses:
31		−# veth0 IPv4: 172.16.1.100, IPv6: 00::11 (underlay)
32		−# tunnel dev <type>00, ex: gre00, IPv4: 10.1.1.100, IPv6: 1::11 (overlay)
33		−#
34		−#
35		−# End-to-end ping packet flow
36		−# ---------------------------
37		−# Most of the tests start by namespace creation, device configuration,
38		−# then ping the underlay and overlay network. When doing 'ping 10.1.1.100'
39		−# from root namespace, the following operations happen:
40		−# 1) Route lookup shows 10.1.1.100/24 belongs to tnl dev, fwd to tnl dev.
41		−# 2) Tnl device's egress BPF program is triggered and set the tunnel metadata,
42		−# with remote_ip=172.16.1.100 and others.
43		−# 3) Outer tunnel header is prepended and route the packet to veth1's egress
44		−# 4) veth0's ingress queue receive the tunneled packet at namespace at_ns0
45		−# 5) Tunnel protocol handler, ex: vxlan_rcv, decap the packet
46		−# 6) Forward the packet to the overlay tnl dev
47		−
48		−BPF_FILE="test_tunnel_kern.bpf.o"
49		−BPF_PIN_TUNNEL_DIR="/sys/fs/bpf/tc/tunnel"
50		−PING_ARG="-c 3 -w 10 -q"
51		−ret=0
52		−GREEN='\033[0;92m'
53		−RED='\033[0;31m'
54		−NC='\033[0m' # No Color
55		−
56		−config_device()
57		−{
58		− ip netns add at_ns0
59		− ip link add veth0 type veth peer name veth1
60		− ip link set veth0 netns at_ns0
61		− ip netns exec at_ns0 ip addr add 172.16.1.100/24 dev veth0
62		− ip netns exec at_ns0 ip link set dev veth0 up
63		− ip link set dev veth1 up mtu 1500
64		− ip addr add dev veth1 172.16.1.200/24
65		−}
66		−
67		−add_gre_tunnel()
68		−{
69		− tun_key=
70		− if [ -n "$1" ]; then
71		− tun_key="key $1"
72		− fi
73		−
74		− # at_ns0 namespace
75		− ip netns exec at_ns0 \
76		− ip link add dev $DEV_NS type $TYPE seq $tun_key \
77		− local 172.16.1.100 remote 172.16.1.200
78		− ip netns exec at_ns0 ip link set dev $DEV_NS up
79		− ip netns exec at_ns0 ip addr add dev $DEV_NS 10.1.1.100/24
80		−
81		− # root namespace
82		− ip link add dev $DEV type $TYPE $tun_key external
83		− ip link set dev $DEV up
84		− ip addr add dev $DEV 10.1.1.200/24
85		−}
86		−
87		−add_ip6gretap_tunnel()
88		−{
89		−
90		− # assign ipv6 address
91		− ip netns exec at_ns0 ip addr add ::11/96 dev veth0
92		− ip netns exec at_ns0 ip link set dev veth0 up
93		− ip addr add dev veth1 ::22/96
94		− ip link set dev veth1 up
95		−
96		− # at_ns0 namespace
97		− ip netns exec at_ns0 \
98		− ip link add dev $DEV_NS type $TYPE seq flowlabel 0xbcdef key 2 \
99		− local ::11 remote ::22
100		−
101		− ip netns exec at_ns0 ip addr add dev $DEV_NS 10.1.1.100/24
102		− ip netns exec at_ns0 ip addr add dev $DEV_NS fc80::100/96
103		− ip netns exec at_ns0 ip link set dev $DEV_NS up
104		−
105		− # root namespace
106		− ip link add dev $DEV type $TYPE external
107		− ip addr add dev $DEV 10.1.1.200/24
108		− ip addr add dev $DEV fc80::200/24
109		− ip link set dev $DEV up
110		−}
111		−
112		−add_erspan_tunnel()
113		−{
114		− # at_ns0 namespace
115		− if [ "$1" == "v1" ]; then
116		− ip netns exec at_ns0 \
117		− ip link add dev $DEV_NS type $TYPE seq key 2 \
118		− local 172.16.1.100 remote 172.16.1.200 \
119		− erspan_ver 1 erspan 123
120		− else
121		− ip netns exec at_ns0 \
122		− ip link add dev $DEV_NS type $TYPE seq key 2 \
123		− local 172.16.1.100 remote 172.16.1.200 \
124		− erspan_ver 2 erspan_dir egress erspan_hwid 3
125		− fi
126		− ip netns exec at_ns0 ip link set dev $DEV_NS up
127		− ip netns exec at_ns0 ip addr add dev $DEV_NS 10.1.1.100/24
128		−
129		− # root namespace
130		− ip link add dev $DEV type $TYPE external
131		− ip link set dev $DEV up
132		− ip addr add dev $DEV 10.1.1.200/24
133		−}
134		−
135		−add_ip6erspan_tunnel()
136		−{
137		−
138		− # assign ipv6 address
139		− ip netns exec at_ns0 ip addr add ::11/96 dev veth0
140		− ip netns exec at_ns0 ip link set dev veth0 up
141		− ip addr add dev veth1 ::22/96
142		− ip link set dev veth1 up
143		−
144		− # at_ns0 namespace
145		− if [ "$1" == "v1" ]; then
146		− ip netns exec at_ns0 \
147		− ip link add dev $DEV_NS type $TYPE seq key 2 \
148		− local ::11 remote ::22 \
149		− erspan_ver 1 erspan 123
150		− else
151		− ip netns exec at_ns0 \
152		− ip link add dev $DEV_NS type $TYPE seq key 2 \
153		− local ::11 remote ::22 \
154		− erspan_ver 2 erspan_dir egress erspan_hwid 7
155		− fi
156		− ip netns exec at_ns0 ip addr add dev $DEV_NS 10.1.1.100/24
157		− ip netns exec at_ns0 ip link set dev $DEV_NS up
158		−
159		− # root namespace
160		− ip link add dev $DEV type $TYPE external
161		− ip addr add dev $DEV 10.1.1.200/24
162		− ip link set dev $DEV up
163		−}
164		−
165		−add_geneve_tunnel()
166		−{
167		− # at_ns0 namespace
168		− ip netns exec at_ns0 \
169		− ip link add dev $DEV_NS type $TYPE \
170		− id 2 dstport 6081 remote 172.16.1.200
171		− ip netns exec at_ns0 ip link set dev $DEV_NS up
172		− ip netns exec at_ns0 ip addr add dev $DEV_NS 10.1.1.100/24
173		−
174		− # root namespace
175		− ip link add dev $DEV type $TYPE dstport 6081 external
176		− ip link set dev $DEV up
177		− ip addr add dev $DEV 10.1.1.200/24
178		−}
179		−
180		−add_ip6geneve_tunnel()
181		−{
182		− ip netns exec at_ns0 ip addr add ::11/96 dev veth0
183		− ip netns exec at_ns0 ip link set dev veth0 up
184		− ip addr add dev veth1 ::22/96
185		− ip link set dev veth1 up
186		−
187		− # at_ns0 namespace
188		− ip netns exec at_ns0 \
189		− ip link add dev $DEV_NS type $TYPE id 22 \
190		− remote ::22 # geneve has no local option
191		− ip netns exec at_ns0 ip addr add dev $DEV_NS 10.1.1.100/24
192		− ip netns exec at_ns0 ip link set dev $DEV_NS up
193		−
194		− # root namespace
195		− ip link add dev $DEV type $TYPE external
196		− ip addr add dev $DEV 10.1.1.200/24
197		− ip link set dev $DEV up
198		−}
199		−
200		−add_ipip_tunnel()
201		−{
202		− # at_ns0 namespace
203		− ip netns exec at_ns0 \
204		− ip link add dev $DEV_NS type $TYPE \
205		− local 172.16.1.100 remote 172.16.1.200
206		− ip netns exec at_ns0 ip link set dev $DEV_NS up
207		− ip netns exec at_ns0 ip addr add dev $DEV_NS 10.1.1.100/24
208		−
209		− # root namespace
210		− ip link add dev $DEV type $TYPE external
211		− ip link set dev $DEV up
212		− ip addr add dev $DEV 10.1.1.200/24
213		−}
214		−
215		−add_ip6tnl_tunnel()
216		−{
217		− ip netns exec at_ns0 ip addr add ::11/96 dev veth0
218		− ip netns exec at_ns0 ip link set dev veth0 up
219		− ip addr add dev veth1 ::22/96
220		− ip link set dev veth1 up
221		−
222		− # at_ns0 namespace
223		− ip netns exec at_ns0 \
224		− ip link add dev $DEV_NS type $TYPE \
225		− local ::11 remote ::22
226		− ip netns exec at_ns0 ip addr add dev $DEV_NS 10.1.1.100/24
227		− ip netns exec at_ns0 ip addr add dev $DEV_NS 1::11/96
228		− ip netns exec at_ns0 ip link set dev $DEV_NS up
229		−
230		− # root namespace
231		− ip link add dev $DEV type $TYPE external
232		− ip addr add dev $DEV 10.1.1.200/24
233		− ip addr add dev $DEV 1::22/96
234		− ip link set dev $DEV up
235		−}
236		−
237		−test_gre()
238		−{
239		− TYPE=gretap
240		− DEV_NS=gretap00
241		− DEV=gretap11
242		− ret=0
243		−
244		− check $TYPE
245		− config_device
246		− add_gre_tunnel 2
247		− attach_bpf $DEV gre_set_tunnel gre_get_tunnel
248		− ping $PING_ARG 10.1.1.100
249		− check_err $?
250		− ip netns exec at_ns0 ping $PING_ARG 10.1.1.200
251		− check_err $?
252		− cleanup
253		−
254		− if [ $ret -ne 0 ]; then
255		− echo -e ${RED}"FAIL: $TYPE"${NC}
256		− return 1
257		− fi
258		− echo -e ${GREEN}"PASS: $TYPE"${NC}
259		−}
260		−
261		−test_gre_no_tunnel_key()
262		−{
263		− TYPE=gre
264		− DEV_NS=gre00
265		− DEV=gre11
266		− ret=0
267		−
268		− check $TYPE
269		− config_device
270		− add_gre_tunnel
271		− attach_bpf $DEV gre_set_tunnel_no_key gre_get_tunnel
272		− ping $PING_ARG 10.1.1.100
273		− check_err $?
274		− ip netns exec at_ns0 ping $PING_ARG 10.1.1.200
275		− check_err $?
276		− cleanup
277		−
278		− if [ $ret -ne 0 ]; then
279		− echo -e ${RED}"FAIL: $TYPE"${NC}
280		− return 1
281		− fi
282		− echo -e ${GREEN}"PASS: $TYPE"${NC}
283		−}
284		−
285		−test_ip6gre()
286		−{
287		− TYPE=ip6gre
288		− DEV_NS=ip6gre00
289		− DEV=ip6gre11
290		− ret=0
291		−
292		− check $TYPE
293		− config_device
294		− # reuse the ip6gretap function
295		− add_ip6gretap_tunnel
296		− attach_bpf $DEV ip6gretap_set_tunnel ip6gretap_get_tunnel
297		− # underlay
298		− ping6 $PING_ARG ::11
299		− # overlay: ipv4 over ipv6
300		− ip netns exec at_ns0 ping $PING_ARG 10.1.1.200
301		− ping $PING_ARG 10.1.1.100
302		− check_err $?
303		− # overlay: ipv6 over ipv6
304		− ip netns exec at_ns0 ping6 $PING_ARG fc80::200
305		− check_err $?
306		− cleanup
307		−
308		− if [ $ret -ne 0 ]; then
309		− echo -e ${RED}"FAIL: $TYPE"${NC}
310		− return 1
311		− fi
312		− echo -e ${GREEN}"PASS: $TYPE"${NC}
313		−}
314		−
315		−test_ip6gretap()
316		−{
317		− TYPE=ip6gretap
318		− DEV_NS=ip6gretap00
319		− DEV=ip6gretap11
320		− ret=0
321		−
322		− check $TYPE
323		− config_device
324		− add_ip6gretap_tunnel
325		− attach_bpf $DEV ip6gretap_set_tunnel ip6gretap_get_tunnel
326		− # underlay
327		− ping6 $PING_ARG ::11
328		− # overlay: ipv4 over ipv6
329		− ip netns exec at_ns0 ping $PING_ARG 10.1.1.200
330		− ping $PING_ARG 10.1.1.100
331		− check_err $?
332		− # overlay: ipv6 over ipv6
333		− ip netns exec at_ns0 ping6 $PING_ARG fc80::200
334		− check_err $?
335		− cleanup
336		−
337		− if [ $ret -ne 0 ]; then
338		− echo -e ${RED}"FAIL: $TYPE"${NC}
339		− return 1
340		− fi
341		− echo -e ${GREEN}"PASS: $TYPE"${NC}
342		−}
343		−
344		−test_erspan()
345		−{
346		− TYPE=erspan
347		− DEV_NS=erspan00
348		− DEV=erspan11
349		− ret=0
350		−
351		− check $TYPE
352		− config_device
353		− add_erspan_tunnel $1
354		− attach_bpf $DEV erspan_set_tunnel erspan_get_tunnel
355		− ping $PING_ARG 10.1.1.100
356		− check_err $?
357		− ip netns exec at_ns0 ping $PING_ARG 10.1.1.200
358		− check_err $?
359		− cleanup
360		−
361		− if [ $ret -ne 0 ]; then
362		− echo -e ${RED}"FAIL: $TYPE"${NC}
363		− return 1
364		− fi
365		− echo -e ${GREEN}"PASS: $TYPE"${NC}
366		−}
367		−
368		−test_ip6erspan()
369		−{
370		− TYPE=ip6erspan
371		− DEV_NS=ip6erspan00
372		− DEV=ip6erspan11
373		− ret=0
374		−
375		− check $TYPE
376		− config_device
377		− add_ip6erspan_tunnel $1
378		− attach_bpf $DEV ip4ip6erspan_set_tunnel ip4ip6erspan_get_tunnel
379		− ping6 $PING_ARG ::11
380		− ip netns exec at_ns0 ping $PING_ARG 10.1.1.200
381		− check_err $?
382		− cleanup
383		−
384		− if [ $ret -ne 0 ]; then
385		− echo -e ${RED}"FAIL: $TYPE"${NC}
386		− return 1
387		− fi
388		− echo -e ${GREEN}"PASS: $TYPE"${NC}
389		−}
390		−
391		−test_geneve()
392		−{
393		− TYPE=geneve
394		− DEV_NS=geneve00
395		− DEV=geneve11
396		− ret=0
397		−
398		− check $TYPE
399		− config_device
400		− add_geneve_tunnel
401		− attach_bpf $DEV geneve_set_tunnel geneve_get_tunnel
402		− ping $PING_ARG 10.1.1.100
403		− check_err $?
404		− ip netns exec at_ns0 ping $PING_ARG 10.1.1.200
405		− check_err $?
406		− cleanup
407		−
408		− if [ $ret -ne 0 ]; then
409		− echo -e ${RED}"FAIL: $TYPE"${NC}
410		− return 1
411		− fi
412		− echo -e ${GREEN}"PASS: $TYPE"${NC}
413		−}
414		−
415		−test_ip6geneve()
416		−{
417		− TYPE=geneve
418		− DEV_NS=ip6geneve00
419		− DEV=ip6geneve11
420		− ret=0
421		−
422		− check $TYPE
423		− config_device
424		− add_ip6geneve_tunnel
425		− attach_bpf $DEV ip6geneve_set_tunnel ip6geneve_get_tunnel
426		− ping $PING_ARG 10.1.1.100
427		− check_err $?
428		− ip netns exec at_ns0 ping $PING_ARG 10.1.1.200
429		− check_err $?
430		− cleanup
431		−
432		− if [ $ret -ne 0 ]; then
433		− echo -e ${RED}"FAIL: ip6$TYPE"${NC}
434		− return 1
435		− fi
436		− echo -e ${GREEN}"PASS: ip6$TYPE"${NC}
437		−}
438		−
439		−test_ipip()
440		−{
441		− TYPE=ipip
442		− DEV_NS=ipip00
443		− DEV=ipip11
444		− ret=0
445		−
446		− check $TYPE
447		− config_device
448		− add_ipip_tunnel
449		− ip link set dev veth1 mtu 1500
450		− attach_bpf $DEV ipip_set_tunnel ipip_get_tunnel
451		− ping $PING_ARG 10.1.1.100
452		− check_err $?
453		− ip netns exec at_ns0 ping $PING_ARG 10.1.1.200
454		− check_err $?
455		− cleanup
456		−
457		− if [ $ret -ne 0 ]; then
458		− echo -e ${RED}"FAIL: $TYPE"${NC}
459		− return 1
460		− fi
461		− echo -e ${GREEN}"PASS: $TYPE"${NC}
462		−}
463		−
464		−test_ipip6()
465		−{
466		− TYPE=ip6tnl
467		− DEV_NS=ipip6tnl00
468		− DEV=ipip6tnl11
469		− ret=0
470		−
471		− check $TYPE
472		− config_device
473		− add_ip6tnl_tunnel
474		− ip link set dev veth1 mtu 1500
475		− attach_bpf $DEV ipip6_set_tunnel ipip6_get_tunnel
476		− # underlay
477		− ping6 $PING_ARG ::11
478		− # ip4 over ip6
479		− ping $PING_ARG 10.1.1.100
480		− check_err $?
481		− ip netns exec at_ns0 ping $PING_ARG 10.1.1.200
482		− check_err $?
483		− cleanup
484		−
485		− if [ $ret -ne 0 ]; then
486		− echo -e ${RED}"FAIL: $TYPE"${NC}
487		− return 1
488		− fi
489		− echo -e ${GREEN}"PASS: $TYPE"${NC}
490		−}
491		−
492		−test_ip6ip6()
493		−{
494		− TYPE=ip6tnl
495		− DEV_NS=ip6ip6tnl00
496		− DEV=ip6ip6tnl11
497		− ret=0
498		−
499		− check $TYPE
500		− config_device
501		− add_ip6tnl_tunnel
502		− ip link set dev veth1 mtu 1500
503		− attach_bpf $DEV ip6ip6_set_tunnel ip6ip6_get_tunnel
504		− # underlay
505		− ping6 $PING_ARG ::11
506		− # ip6 over ip6
507		− ping6 $PING_ARG 1::11
508		− check_err $?
509		− ip netns exec at_ns0 ping6 $PING_ARG 1::22
510		− check_err $?
511		− cleanup
512		−
513		− if [ $ret -ne 0 ]; then
514		− echo -e ${RED}"FAIL: ip6$TYPE"${NC}
515		− return 1
516		− fi
517		− echo -e ${GREEN}"PASS: ip6$TYPE"${NC}
518		−}
519		−
520		−attach_bpf()
521		−{
522		− DEV=$1
523		− SET=$2
524		− GET=$3
525		− mkdir -p ${BPF_PIN_TUNNEL_DIR}
526		− bpftool prog loadall ${BPF_FILE} ${BPF_PIN_TUNNEL_DIR}/
527		− tc qdisc add dev $DEV clsact
528		− tc filter add dev $DEV egress bpf da object-pinned ${BPF_PIN_TUNNEL_DIR}/$SET
529		− tc filter add dev $DEV ingress bpf da object-pinned ${BPF_PIN_TUNNEL_DIR}/$GET
530		−}
531		−
532		−cleanup()
533		−{
534		− rm -rf ${BPF_PIN_TUNNEL_DIR}
535		−
536		− ip netns delete at_ns0 2> /dev/null
537		− ip link del veth1 2> /dev/null
538		− ip link del ipip11 2> /dev/null
539		− ip link del ipip6tnl11 2> /dev/null
540		− ip link del ip6ip6tnl11 2> /dev/null
541		− ip link del gretap11 2> /dev/null
542		− ip link del gre11 2> /dev/null
543		− ip link del ip6gre11 2> /dev/null
544		− ip link del ip6gretap11 2> /dev/null
545		− ip link del geneve11 2> /dev/null
546		− ip link del ip6geneve11 2> /dev/null
547		− ip link del erspan11 2> /dev/null
548		− ip link del ip6erspan11 2> /dev/null
549		−}
550		−
551		−cleanup_exit()
552		−{
553		− echo "CATCH SIGKILL or SIGINT, cleanup and exit"
554		− cleanup
555		− exit 0
556		−}
557		−
558		−check()
559		−{
560		− ip link help 2>&1 \| grep -q "\s$1\s"
561		− if [ $? -ne 0 ];then
562		− echo "SKIP $1: iproute2 not support"
563		− cleanup
564		− return 1
565		− fi
566		−}
567		−
568		−enable_debug()
569		−{
570		− echo 'file ip_gre.c +p' > /sys/kernel/debug/dynamic_debug/control
571		− echo 'file ip6_gre.c +p' > /sys/kernel/debug/dynamic_debug/control
572		− echo 'file geneve.c +p' > /sys/kernel/debug/dynamic_debug/control
573		− echo 'file ipip.c +p' > /sys/kernel/debug/dynamic_debug/control
574		−}
575		−
576		−check_err()
577		−{
578		− if [ $ret -eq 0 ]; then
579		− ret=$1
580		− fi
581		−}
582		−
583		−bpf_tunnel_test()
584		−{
585		− local errors=0
586		−
587		− echo "Testing GRE tunnel..."
588		− test_gre
589		− errors=$(( $errors + $? ))
590		−
591		− echo "Testing GRE tunnel (without tunnel keys)..."
592		− test_gre_no_tunnel_key
593		− errors=$(( $errors + $? ))
594		−
595		− echo "Testing IP6GRE tunnel..."
596		− test_ip6gre
597		− errors=$(( $errors + $? ))
598		−
599		− echo "Testing IP6GRETAP tunnel..."
600		− test_ip6gretap
601		− errors=$(( $errors + $? ))
602		−
603		− echo "Testing ERSPAN tunnel..."
604		− test_erspan v2
605		− errors=$(( $errors + $? ))
606		−
607		− echo "Testing IP6ERSPAN tunnel..."
608		− test_ip6erspan v2
609		− errors=$(( $errors + $? ))
610		−
611		− echo "Testing GENEVE tunnel..."
612		− test_geneve
613		− errors=$(( $errors + $? ))
614		−
615		− echo "Testing IP6GENEVE tunnel..."
616		− test_ip6geneve
617		− errors=$(( $errors + $? ))
618		−
619		− echo "Testing IPIP tunnel..."
620		− test_ipip
621		− errors=$(( $errors + $? ))
622		−
623		− echo "Testing IPIP6 tunnel..."
624		− test_ipip6
625		− errors=$(( $errors + $? ))
626		−
627		− echo "Testing IP6IP6 tunnel..."
628		− test_ip6ip6
629		− errors=$(( $errors + $? ))
630		−
631		− return $errors
632		−}
633		−
634		−trap cleanup 0 3 6
635		−trap cleanup_exit 2 9
636		−
637		−cleanup
638		−bpf_tunnel_test
639		−
640		−if [ $? -ne 0 ]; then
641		− echo -e "$(basename $0): ${RED}FAIL${NC}"
642		− exit 1
643		−fi
644		−echo -e "$(basename $0): ${GREEN}PASS${NC}"
645		−exit 0

tools/testing/selftests/bpf/test_xdp_redirect_multi.sh+0−214

@@ -1,214 +0,0 @@
1		−#!/bin/bash
2		−# SPDX-License-Identifier: GPL-2.0
3		−#
4		−# Test topology:
5		−# - - - - - - - - - - - - - - - - - - -
6		−# \| veth1 veth2 veth3 \| ns0
7		−# - -\| - - - - - - \| - - - - - - \| - -
8		−# --------- --------- ---------
9		−# \| veth0 \| \| veth0 \| \| veth0 \|
10		−# --------- --------- ---------
11		−# ns1 ns2 ns3
12		−#
13		−# Test modules:
14		−# XDP modes: generic, native, native + egress_prog
15		−#
16		−# Test cases:
17		−# ARP: Testing BPF_F_BROADCAST, the ingress interface also should receive
18		−# the redirects.
19		−# ns1 -> gw: ns1, ns2, ns3, should receive the arp request
20		−# IPv4: Testing BPF_F_BROADCAST \| BPF_F_EXCLUDE_INGRESS, the ingress
21		−# interface should not receive the redirects.
22		−# ns1 -> gw: ns1 should not receive, ns2, ns3 should receive redirects.
23		−# IPv6: Testing none flag, all the pkts should be redirected back
24		−# ping test: ns1 -> ns2 (block), echo requests will be redirect back
25		−# egress_prog:
26		−# all src mac should be egress interface's mac
27		−
28		−# netns numbers
29		−NUM=3
30		−IFACES=""
31		−DRV_MODE="xdpgeneric xdpdrv xdpegress"
32		−PASS=0
33		−FAIL=0
34		−LOG_DIR=$(mktemp -d)
35		−declare -a NS
36		−NS[0]="ns0-$(mktemp -u XXXXXX)"
37		−NS[1]="ns1-$(mktemp -u XXXXXX)"
38		−NS[2]="ns2-$(mktemp -u XXXXXX)"
39		−NS[3]="ns3-$(mktemp -u XXXXXX)"
40		−
41		−test_pass()
42		−{
43		− echo "Pass: $@"
44		− PASS=$((PASS + 1))
45		−}
46		−
47		−test_fail()
48		−{
49		− echo "fail: $@"
50		− FAIL=$((FAIL + 1))
51		−}
52		−
53		−clean_up()
54		−{
55		− for i in $(seq 0 $NUM); do
56		− ip netns del ${NS[$i]} 2> /dev/null
57		− done
58		−}
59		−
60		−# Kselftest framework requirement - SKIP code is 4.
61		−check_env()
62		−{
63		− ip link set dev lo xdpgeneric off &>/dev/null
64		− if [ $? -ne 0 ];then
65		− echo "selftests: [SKIP] Could not run test without the ip xdpgeneric support"
66		− exit 4
67		− fi
68		−
69		− which tcpdump &>/dev/null
70		− if [ $? -ne 0 ];then
71		− echo "selftests: [SKIP] Could not run test without tcpdump"
72		− exit 4
73		− fi
74		−}
75		−
76		−setup_ns()
77		−{
78		− local mode=$1
79		− IFACES=""
80		−
81		− if [ "$mode" = "xdpegress" ]; then
82		− mode="xdpdrv"
83		− fi
84		−
85		− ip netns add ${NS[0]}
86		− for i in $(seq $NUM); do
87		− ip netns add ${NS[$i]}
88		− ip -n ${NS[$i]} link add veth0 type veth peer name veth$i netns ${NS[0]}
89		− ip -n ${NS[$i]} link set veth0 up
90		− ip -n ${NS[0]} link set veth$i up
91		−
92		− ip -n ${NS[$i]} addr add 192.0.2.$i/24 dev veth0
93		− ip -n ${NS[$i]} addr add 2001:db8::$i/64 dev veth0
94		− # Add a neigh entry for IPv4 ping test
95		− ip -n ${NS[$i]} neigh add 192.0.2.253 lladdr 00:00:00:00:00:01 dev veth0
96		− ip -n ${NS[$i]} link set veth0 $mode obj \
97		− xdp_dummy.bpf.o sec xdp &> /dev/null \|\| \
98		− { test_fail "Unable to load dummy xdp" && exit 1; }
99		− IFACES="$IFACES veth$i"
100		− veth_mac[$i]=$(ip -n ${NS[0]} link show veth$i \| awk '/link\/ether/ {print $2}')
101		− done
102		−}
103		−
104		−do_egress_tests()
105		−{
106		− local mode=$1
107		−
108		− # mac test
109		− ip netns exec ${NS[2]} tcpdump -e -i veth0 -nn -l -e &> ${LOG_DIR}/mac_ns1-2_${mode}.log &
110		− ip netns exec ${NS[3]} tcpdump -e -i veth0 -nn -l -e &> ${LOG_DIR}/mac_ns1-3_${mode}.log &
111		− sleep 0.5
112		− ip netns exec ${NS[1]} ping 192.0.2.254 -i 0.1 -c 4 &> /dev/null
113		− sleep 0.5
114		− pkill tcpdump
115		−
116		− # mac check
117		− grep -q "${veth_mac[2]} > ff:ff:ff:ff:ff:ff" ${LOG_DIR}/mac_ns1-2_${mode}.log && \
118		− test_pass "$mode mac ns1-2" \|\| test_fail "$mode mac ns1-2"
119		− grep -q "${veth_mac[3]} > ff:ff:ff:ff:ff:ff" ${LOG_DIR}/mac_ns1-3_${mode}.log && \
120		− test_pass "$mode mac ns1-3" \|\| test_fail "$mode mac ns1-3"
121		−}
122		−
123		−do_ping_tests()
124		−{
125		− local mode=$1
126		−
127		− # ping6 test: echo request should be redirect back to itself, not others
128		− ip netns exec ${NS[1]} ip neigh add 2001:db8::2 dev veth0 lladdr 00:00:00:00:00:02
129		−
130		− ip netns exec ${NS[1]} tcpdump -i veth0 -nn -l -e &> ${LOG_DIR}/ns1-1_${mode}.log &
131		− ip netns exec ${NS[2]} tcpdump -i veth0 -nn -l -e &> ${LOG_DIR}/ns1-2_${mode}.log &
132		− ip netns exec ${NS[3]} tcpdump -i veth0 -nn -l -e &> ${LOG_DIR}/ns1-3_${mode}.log &
133		− sleep 0.5
134		− # ARP test
135		− ip netns exec ${NS[1]} arping -q -c 2 -I veth0 192.0.2.254
136		− # IPv4 test
137		− ip netns exec ${NS[1]} ping 192.0.2.253 -i 0.1 -c 4 &> /dev/null
138		− # IPv6 test
139		− ip netns exec ${NS[1]} ping6 2001:db8::2 -i 0.1 -c 2 &> /dev/null
140		− sleep 0.5
141		− pkill tcpdump
142		−
143		− # All netns should receive the redirect arp requests
144		− [ $(grep -cF "who-has 192.0.2.254" ${LOG_DIR}/ns1-1_${mode}.log) -eq 4 ] && \
145		− test_pass "$mode arp(F_BROADCAST) ns1-1" \|\| \
146		− test_fail "$mode arp(F_BROADCAST) ns1-1"
147		− [ $(grep -cF "who-has 192.0.2.254" ${LOG_DIR}/ns1-2_${mode}.log) -eq 2 ] && \
148		− test_pass "$mode arp(F_BROADCAST) ns1-2" \|\| \
149		− test_fail "$mode arp(F_BROADCAST) ns1-2"
150		− [ $(grep -cF "who-has 192.0.2.254" ${LOG_DIR}/ns1-3_${mode}.log) -eq 2 ] && \
151		− test_pass "$mode arp(F_BROADCAST) ns1-3" \|\| \
152		− test_fail "$mode arp(F_BROADCAST) ns1-3"
153		−
154		− # ns1 should not receive the redirect echo request, others should
155		− [ $(grep -c "ICMP echo request" ${LOG_DIR}/ns1-1_${mode}.log) -eq 4 ] && \
156		− test_pass "$mode IPv4 (F_BROADCAST\|F_EXCLUDE_INGRESS) ns1-1" \|\| \
157		− test_fail "$mode IPv4 (F_BROADCAST\|F_EXCLUDE_INGRESS) ns1-1"
158		− [ $(grep -c "ICMP echo request" ${LOG_DIR}/ns1-2_${mode}.log) -eq 4 ] && \
159		− test_pass "$mode IPv4 (F_BROADCAST\|F_EXCLUDE_INGRESS) ns1-2" \|\| \
160		− test_fail "$mode IPv4 (F_BROADCAST\|F_EXCLUDE_INGRESS) ns1-2"
161		− [ $(grep -c "ICMP echo request" ${LOG_DIR}/ns1-3_${mode}.log) -eq 4 ] && \
162		− test_pass "$mode IPv4 (F_BROADCAST\|F_EXCLUDE_INGRESS) ns1-3" \|\| \
163		− test_fail "$mode IPv4 (F_BROADCAST\|F_EXCLUDE_INGRESS) ns1-3"
164		−
165		− # ns1 should receive the echo request, ns2 should not
166		− [ $(grep -c "ICMP6, echo request" ${LOG_DIR}/ns1-1_${mode}.log) -eq 4 ] && \
167		− test_pass "$mode IPv6 (no flags) ns1-1" \|\| \
168		− test_fail "$mode IPv6 (no flags) ns1-1"
169		− [ $(grep -c "ICMP6, echo request" ${LOG_DIR}/ns1-2_${mode}.log) -eq 0 ] && \
170		− test_pass "$mode IPv6 (no flags) ns1-2" \|\| \
171		− test_fail "$mode IPv6 (no flags) ns1-2"
172		−}
173		−
174		−do_tests()
175		−{
176		− local mode=$1
177		− local drv_p
178		−
179		− case ${mode} in
180		− xdpdrv) drv_p="-N";;
181		− xdpegress) drv_p="-X";;
182		− xdpgeneric) drv_p="-S";;
183		− esac
184		−
185		− ip netns exec ${NS[0]} ./xdp_redirect_multi $drv_p $IFACES &> ${LOG_DIR}/xdp_redirect_${mode}.log &
186		− xdp_pid=$!
187		− sleep 1
188		− if ! ps -p $xdp_pid > /dev/null; then
189		− test_fail "$mode xdp_redirect_multi start failed"
190		− return 1
191		− fi
192		−
193		− if [ "$mode" = "xdpegress" ]; then
194		− do_egress_tests $mode
195		− else
196		− do_ping_tests $mode
197		− fi
198		−
199		− kill $xdp_pid
200		−}
201		−
202		−check_env
203		−
204		−trap clean_up EXIT
205		−
206		−for mode in ${DRV_MODE}; do
207		− setup_ns $mode
208		− do_tests $mode
209		− clean_up
210		−done
211		−rm -rf ${LOG_DIR}
212		−
213		−echo "Summary: PASS $PASS, FAIL $FAIL"
214		−[ $FAIL -eq 0 ] && exit 0 \|\| exit 1

tools/testing/selftests/bpf/test_xdp_vlan.sh+0−233

@@ -1,233 +0,0 @@
1		−#!/bin/bash
2		−# SPDX-License-Identifier: GPL-2.0
3		−# Author: Jesper Dangaard Brouer <hawk@kernel.org>
4		−
5		−# Kselftest framework requirement - SKIP code is 4.
6		−readonly KSFT_SKIP=4
7		−readonly NS1="ns1-$(mktemp -u XXXXXX)"
8		−readonly NS2="ns2-$(mktemp -u XXXXXX)"
9		−
10		−# Allow wrapper scripts to name test
11		−if [ -z "$TESTNAME" ]; then
12		− TESTNAME=xdp_vlan
13		−fi
14		−
15		−# Default XDP mode
16		−XDP_MODE=xdpgeneric
17		−
18		−usage() {
19		− echo "Testing XDP + TC eBPF VLAN manipulations: $TESTNAME"
20		− echo ""
21		− echo "Usage: $0 [-vfh]"
22		− echo " -v \| --verbose : Verbose"
23		− echo " --flush : Flush before starting (e.g. after --interactive)"
24		− echo " --interactive : Keep netns setup running after test-run"
25		− echo " --mode=XXX : Choose XDP mode (xdp \| xdpgeneric \| xdpdrv)"
26		− echo ""
27		−}
28		−
29		−valid_xdp_mode()
30		−{
31		− local mode=$1
32		−
33		− case "$mode" in
34		− xdpgeneric \| xdpdrv \| xdp)
35		− return 0
36		− ;;
37		− *)
38		− return 1
39		− esac
40		−}
41		−
42		−cleanup()
43		−{
44		− local status=$?
45		−
46		− if [ "$status" = "0" ]; then
47		− echo "selftests: $TESTNAME [PASS]";
48		− else
49		− echo "selftests: $TESTNAME [FAILED]";
50		− fi
51		−
52		− if [ -n "$INTERACTIVE" ]; then
53		− echo "Namespace setup still active explore with:"
54		− echo " ip netns exec ${NS1} bash"
55		− echo " ip netns exec ${NS2} bash"
56		− exit $status
57		− fi
58		−
59		− set +e
60		− ip link del veth1 2> /dev/null
61		− ip netns del ${NS1} 2> /dev/null
62		− ip netns del ${NS2} 2> /dev/null
63		−}
64		−
65		−# Using external program "getopt" to get --long-options
66		−OPTIONS=$(getopt -o hvfi: \
67		− --long verbose,flush,help,interactive,debug,mode: -- "$@")
68		−if (( $? != 0 )); then
69		− usage
70		− echo "selftests: $TESTNAME [FAILED] Error calling getopt, unknown option?"
71		− exit 2
72		−fi
73		−eval set -- "$OPTIONS"
74		−
75		−## --- Parse command line arguments / parameters ---
76		−while true; do
77		− case "$1" in
78		− -v \| --verbose)
79		− export VERBOSE=yes
80		− shift
81		− ;;
82		− -i \| --interactive \| --debug )
83		− INTERACTIVE=yes
84		− shift
85		− ;;
86		− -f \| --flush )
87		− cleanup
88		− shift
89		− ;;
90		− --mode )
91		− shift
92		− XDP_MODE=$1
93		− shift
94		− ;;
95		− -- )
96		− shift
97		− break
98		− ;;
99		− -h \| --help )
100		− usage;
101		− echo "selftests: $TESTNAME [SKIP] usage help info requested"
102		− exit $KSFT_SKIP
103		− ;;
104		− * )
105		− shift
106		− break
107		− ;;
108		− esac
109		−done
110		−
111		−if [ "$EUID" -ne 0 ]; then
112		− echo "selftests: $TESTNAME [FAILED] need root privileges"
113		− exit 1
114		−fi
115		−
116		−valid_xdp_mode $XDP_MODE
117		−if [ $? -ne 0 ]; then
118		− echo "selftests: $TESTNAME [FAILED] unknown XDP mode ($XDP_MODE)"
119		− exit 1
120		−fi
121		−
122		−ip link set dev lo xdpgeneric off 2>/dev/null > /dev/null
123		−if [ $? -ne 0 ]; then
124		− echo "selftests: $TESTNAME [SKIP] need ip xdp support"
125		− exit $KSFT_SKIP
126		−fi
127		−
128		−# Interactive mode likely require us to cleanup netns
129		−if [ -n "$INTERACTIVE" ]; then
130		− ip link del veth1 2> /dev/null
131		− ip netns del ${NS1} 2> /dev/null
132		− ip netns del ${NS2} 2> /dev/null
133		−fi
134		−
135		−# Exit on failure
136		−set -e
137		−
138		−# Some shell-tools dependencies
139		−which ip > /dev/null
140		−which tc > /dev/null
141		−which ethtool > /dev/null
142		−
143		−# Make rest of shell verbose, showing comments as doc/info
144		−if [ -n "$VERBOSE" ]; then
145		− set -v
146		−fi
147		−
148		−# Create two namespaces
149		−ip netns add ${NS1}
150		−ip netns add ${NS2}
151		−
152		−# Run cleanup if failing or on kill
153		−trap cleanup 0 2 3 6 9
154		−
155		−# Create veth pair
156		−ip link add veth1 type veth peer name veth2
157		−
158		−# Move veth1 and veth2 into the respective namespaces
159		−ip link set veth1 netns ${NS1}
160		−ip link set veth2 netns ${NS2}
161		−
162		−# NOTICE: XDP require VLAN header inside packet payload
163		−# - Thus, disable VLAN offloading driver features
164		−# - For veth REMEMBER TX side VLAN-offload
165		−#
166		−# Disable rx-vlan-offload (mostly needed on ns1)
167		−ip netns exec ${NS1} ethtool -K veth1 rxvlan off
168		−ip netns exec ${NS2} ethtool -K veth2 rxvlan off
169		−#
170		−# Disable tx-vlan-offload (mostly needed on ns2)
171		−ip netns exec ${NS2} ethtool -K veth2 txvlan off
172		−ip netns exec ${NS1} ethtool -K veth1 txvlan off
173		−
174		−export IPADDR1=100.64.41.1
175		−export IPADDR2=100.64.41.2
176		−
177		−# In ns1/veth1 add IP-addr on plain net_device
178		−ip netns exec ${NS1} ip addr add ${IPADDR1}/24 dev veth1
179		−ip netns exec ${NS1} ip link set veth1 up
180		−
181		−# In ns2/veth2 create VLAN device
182		−export VLAN=4011
183		−export DEVNS2=veth2
184		−ip netns exec ${NS2} ip link add link $DEVNS2 name $DEVNS2.$VLAN type vlan id $VLAN
185		−ip netns exec ${NS2} ip addr add ${IPADDR2}/24 dev $DEVNS2.$VLAN
186		−ip netns exec ${NS2} ip link set $DEVNS2 up
187		−ip netns exec ${NS2} ip link set $DEVNS2.$VLAN up
188		−
189		−# Bringup lo in netns (to avoids confusing people using --interactive)
190		−ip netns exec ${NS1} ip link set lo up
191		−ip netns exec ${NS2} ip link set lo up
192		−
193		−# At this point, the hosts cannot reach each-other,
194		−# because ns2 are using VLAN tags on the packets.
195		−
196		−ip netns exec ${NS2} sh -c 'ping -W 1 -c 1 100.64.41.1 \|\| echo "Success: First ping must fail"'
197		−
198		−
199		−# Now we can use the test_xdp_vlan.c program to pop/push these VLAN tags
200		−# ----------------------------------------------------------------------
201		−# In ns1: ingress use XDP to remove VLAN tags
202		−export DEVNS1=veth1
203		−export BPF_FILE=test_xdp_vlan.bpf.o
204		−
205		−# First test: Remove VLAN by setting VLAN ID 0, using "xdp_vlan_change"
206		−export XDP_PROG=xdp_vlan_change
207		−ip netns exec ${NS1} ip link set $DEVNS1 $XDP_MODE object $BPF_FILE section $XDP_PROG
208		−
209		−# In ns1: egress use TC to add back VLAN tag 4011
210		−# (del cmd)
211		−# tc qdisc del dev $DEVNS1 clsact 2> /dev/null
212		−#
213		−ip netns exec ${NS1} tc qdisc add dev $DEVNS1 clsact
214		−ip netns exec ${NS1} tc filter add dev $DEVNS1 egress \
215		− prio 1 handle 1 bpf da obj $BPF_FILE sec tc_vlan_push
216		−
217		−# Now the namespaces can reach each-other, test with ping:
218		−ip netns exec ${NS2} ping -i 0.2 -W 2 -c 2 $IPADDR1
219		−ip netns exec ${NS1} ping -i 0.2 -W 2 -c 2 $IPADDR2
220		−
221		−# Second test: Replace xdp prog, that fully remove vlan header
222		−#
223		−# Catch kernel bug for generic-XDP, that does didn't allow us to
224		−# remove a VLAN header, because skb->protocol still contain VLAN
225		−# ETH_P_8021Q indication, and this cause overwriting of our changes.
226		−#
227		−export XDP_PROG=xdp_vlan_remove_outer2
228		−ip netns exec ${NS1} ip link set $DEVNS1 $XDP_MODE off
229		−ip netns exec ${NS1} ip link set $DEVNS1 $XDP_MODE object $BPF_FILE section $XDP_PROG
230		−
231		−# Now the namespaces should still be able reach each-other, test with ping:
232		−ip netns exec ${NS2} ping -i 0.2 -W 2 -c 2 $IPADDR1
233		−ip netns exec ${NS1} ping -i 0.2 -W 2 -c 2 $IPADDR2

tools/testing/selftests/bpf/test_xdp_vlan_mode_generic.sh+0−9

@@ -1,9 +0,0 @@
1		−#!/bin/bash
2		−# SPDX-License-Identifier: GPL-2.0
3		−
4		−# Exit on failure
5		−set -e
6		−
7		−# Wrapper script to test generic-XDP
8		−export TESTNAME=xdp_vlan_mode_generic
9		−./test_xdp_vlan.sh --mode=xdpgeneric

tools/testing/selftests/bpf/test_xdp_vlan_mode_native.sh+0−9

@@ -1,9 +0,0 @@
1		−#!/bin/bash
2		−# SPDX-License-Identifier: GPL-2.0
3		−
4		−# Exit on failure
5		−set -e
6		−
7		−# Wrapper script to test native-XDP
8		−export TESTNAME=xdp_vlan_mode_native
9		−./test_xdp_vlan.sh --mode=xdpdrv

tools/testing/selftests/bpf/veristat.c+355−12

@@ -3,6 +3,7 @@ · 2 unmodified lines
3	3	#define _GNU_SOURCE
4	4	#include <argp.h>
5	5	#include <libgen.h>
	6	+#include <ctype.h>
6	7	#include <string.h>
7	8	#include <stdlib.h>
8	9	#include <sched.h>
@@ -154,6 +155,16 @@ struct filter { · 145 unmodified lines
154	155	bool abs;
155	156	};
156	157
	158	+struct var_preset {
	159	+ char *name;
	160	+ enum { INTEGRAL, ENUMERATOR } type;
	161	+ union {
	162	+ long long ivalue;
	163	+ char *svalue;
	164	+ };
	165	+ bool applied;
	166	+};
	167	+
157	168	static struct env {
158	169	char **filenames;
159	170	int filename_cnt;
@@ -195,6 +206,8 @@ static struct env { · 35 unmodified lines
195	206	int progs_processed;
196	207	int progs_skipped;
197	208	int top_src_lines;
	209	+ struct var_preset *presets;
	210	+ int npresets;
198	211	} env;
199	212
200	213	static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args)
@@ -246,16 +259,20 @@ static const struct argp_option opts[] = { · 45 unmodified lines
246	259	{ "test-reg-invariants", 'r', NULL, 0,
247	260	"Force BPF verifier failure on register invariant violation (BPF_F_TEST_REG_INVARIANTS program flag)" },
248	261	{ "top-src-lines", 'S', "N", 0, "Emit N most frequent source code lines" },
	262	+ { "set-global-vars", 'G', "GLOBAL", 0, "Set global variables provided in the expression, for example \"var1 = 1\"" },
249	263	{},
250	264	};
251	265
252	266	static int parse_stats(const char stats_str, struct stat_specs specs);
253	267	static int append_filter(struct filter *filters, int cnt, const char *str);
254	268	static int append_filter_file(const char *path);
	269	+static int append_var_preset(struct var_preset *presets, int cnt, const char *expr);
	270	+static int append_var_preset_file(const char *filename);
	271	+static int append_file(const char *path);
	272	+static int append_file_from_file(const char *path);
255	273
256	274	static error_t parse_arg(int key, char arg, struct argp_state state)
257	275	{
258		− void *tmp;
259	276	int err;
260	277
261	278	switch (key) {
@@ -353,15 +370,26 @@ static error_t parse_arg(int key, char arg, struct argp_state state) · 91 unmodified lines
353	370	argp_usage(state);
354	371	}
355	372	break;
	373	+ case 'G': {
	374	+ if (arg[0] == '@')
	375	+ err = append_var_preset_file(arg + 1);
	376	+ else
	377	+ err = append_var_preset(&env.presets, &env.npresets, arg);
	378	+ if (err) {
	379	+ fprintf(stderr, "Failed to parse global variable presets: %s\n", arg);
	380	+ return err;
	381	+ }
	382	+ break;
	383	+ }
356	384	case ARGP_KEY_ARG:
357		− tmp = realloc(env.filenames, (env.filename_cnt + 1) * sizeof(*env.filenames));
358		− if (!tmp)
359		− return -ENOMEM;
360		− env.filenames = tmp;
361		− env.filenames[env.filename_cnt] = strdup(arg);
362		− if (!env.filenames[env.filename_cnt])
363		− return -ENOMEM;
364		− env.filename_cnt++;
	385	+ if (arg[0] == '@')
	386	+ err = append_file_from_file(arg + 1);
	387	+ else
	388	+ err = append_file(arg);
	389	+ if (err) {
	390	+ fprintf(stderr, "Failed to collect BPF object files: %d\n", err);
	391	+ return err;
	392	+ }
365	393	break;
366	394	default:
367	395	return ARGP_ERR_UNKNOWN;
@@ -632,7 +660,7 @@ static int append_filter_file(const char *path) · 264 unmodified lines
632	660	f = fopen(path, "r");
633	661	if (!f) {
634	662	err = -errno;
635		− fprintf(stderr, "Failed to open filters in '%s': %d\n", path, err);
	663	+ fprintf(stderr, "Failed to open filters in '%s': %s\n", path, strerror(-err));
636	664	return err;
637	665	}
638	666
@@ -662,6 +690,49 @@ static const struct stat_specs default_output_spec = { · 23 unmodified lines
662	690	},
663	691	};
664	692
	693	+static int append_file(const char *path)
	694	+{
	695	+ void *tmp;
	696	+
	697	+ tmp = realloc(env.filenames, (env.filename_cnt + 1) * sizeof(*env.filenames));
	698	+ if (!tmp)
	699	+ return -ENOMEM;
	700	+ env.filenames = tmp;
	701	+ env.filenames[env.filename_cnt] = strdup(path);
	702	+ if (!env.filenames[env.filename_cnt])
	703	+ return -ENOMEM;
	704	+ env.filename_cnt++;
	705	+ return 0;
	706	+}
	707	+
	708	+static int append_file_from_file(const char *path)
	709	+{
	710	+ char buf[1024];
	711	+ int err = 0;
	712	+ FILE *f;
	713	+
	714	+ f = fopen(path, "r");
	715	+ if (!f) {
	716	+ err = -errno;
	717	+ fprintf(stderr, "Failed to open object files list in '%s': %s\n",
	718	+ path, strerror(errno));
	719	+ return err;
	720	+ }
	721	+
	722	+ while (fscanf(f, " %1023[^\n]\n", buf) == 1) {
	723	+ /* lines starting with # are comments, skip them */
	724	+ if (buf[0] == '\0' \|\| buf[0] == '#')
	725	+ continue;
	726	+ err = append_file(buf);
	727	+ if (err)
	728	+ goto cleanup;
	729	+ }
	730	+
	731	+cleanup:
	732	+ fclose(f);
	733	+ return err;
	734	+}
	735	+
665	736	static const struct stat_specs default_csv_output_spec = {
666	737	.spec_cnt = 14,
667	738	.ids = {
@@ -1163,13 +1234,13 @@ static void fixup_obj(struct bpf_object obj, struct bpf_program prog, const ch · 495 unmodified lines
1163	1234	bpf_program__set_expected_attach_type(prog, attach_type);
1164	1235
1165	1236	if (!env.quiet) {
1166		− printf("Using guessed program type '%s' for %s/%s...\n",
	1237	+ fprintf(stderr, "Using guessed program type '%s' for %s/%s...\n",
1167	1238	libbpf_bpf_prog_type_str(prog_type),
1168	1239	filename, prog_name);
1169	1240	}
1170	1241	} else {
1171	1242	if (!env.quiet) {
1172		− printf("Failed to guess program type for freplace program with context type name '%s' for %s/%s. Consider using canonical type names to help veristat...\n",
	1243	+ fprintf(stderr, "Failed to guess program type for freplace program with context type name '%s' for %s/%s. Consider using canonical type names to help veristat...\n",
1173	1244	ctx_name, filename, prog_name);
1174	1245	}
1175	1246	}
@@ -1292,6 +1363,261 @@ static int process_prog(const char filename, struct bpf_object obj, struct bpf · 116 unmodified lines
1292	1363	return 0;
1293	1364	};
1294	1365
	1366	+static int append_var_preset(struct var_preset *presets, int cnt, const char *expr)
	1367	+{
	1368	+ void *tmp;
	1369	+ struct var_preset *cur;
	1370	+ char var[256], val[256], *val_end;
	1371	+ long long value;
	1372	+ int n;
	1373	+
	1374	+ tmp = realloc(presets, (cnt + 1) * sizeof(**presets));
	1375	+ if (!tmp)
	1376	+ return -ENOMEM;
	1377	+ *presets = tmp;
	1378	+ cur = &(presets)[cnt];
	1379	+ memset(cur, 0, sizeof(*cur));
	1380	+ (*cnt)++;
	1381	+
	1382	+ if (sscanf(expr, "%s = %s %n", var, val, &n) != 2 \|\| n != strlen(expr)) {
	1383	+ fprintf(stderr, "Failed to parse expression '%s'\n", expr);
	1384	+ return -EINVAL;
	1385	+ }
	1386	+
	1387	+ if (val[0] == '-' \|\| isdigit(val[0])) {
	1388	+ /* must be a number */
	1389	+ errno = 0;
	1390	+ value = strtoll(val, &val_end, 0);
	1391	+ if (errno == ERANGE) {
	1392	+ errno = 0;
	1393	+ value = strtoull(val, &val_end, 0);
	1394	+ }
	1395	+ if (errno \|\| *val_end != '\0') {
	1396	+ fprintf(stderr, "Failed to parse value '%s'\n", val);
	1397	+ return -EINVAL;
	1398	+ }
	1399	+ cur->ivalue = value;
	1400	+ cur->type = INTEGRAL;
	1401	+ } else {
	1402	+ /* if not a number, consider it enum value */
	1403	+ cur->svalue = strdup(val);
	1404	+ if (!cur->svalue)
	1405	+ return -ENOMEM;
	1406	+ cur->type = ENUMERATOR;
	1407	+ }
	1408	+
	1409	+ cur->name = strdup(var);
	1410	+ if (!cur->name)
	1411	+ return -ENOMEM;
	1412	+
	1413	+ return 0;
	1414	+}
	1415	+
	1416	+static int append_var_preset_file(const char *filename)
	1417	+{
	1418	+ char buf[1024];
	1419	+ FILE *f;
	1420	+ int err = 0;
	1421	+
	1422	+ f = fopen(filename, "rt");
	1423	+ if (!f) {
	1424	+ err = -errno;
	1425	+ fprintf(stderr, "Failed to open presets in '%s': %s\n", filename, strerror(-err));
	1426	+ return -EINVAL;
	1427	+ }
	1428	+
	1429	+ while (fscanf(f, " %1023[^\n]\n", buf) == 1) {
	1430	+ if (buf[0] == '\0' \|\| buf[0] == '#')
	1431	+ continue;
	1432	+
	1433	+ err = append_var_preset(&env.presets, &env.npresets, buf);
	1434	+ if (err)
	1435	+ goto cleanup;
	1436	+ }
	1437	+
	1438	+cleanup:
	1439	+ fclose(f);
	1440	+ return err;
	1441	+}
	1442	+
	1443	+static bool is_signed_type(const struct btf_type *t)
	1444	+{
	1445	+ if (btf_is_int(t))
	1446	+ return btf_int_encoding(t) & BTF_INT_SIGNED;
	1447	+ if (btf_is_any_enum(t))
	1448	+ return btf_kflag(t);
	1449	+ return true;
	1450	+}
	1451	+
	1452	+static int enum_value_from_name(const struct btf btf, const struct btf_type t,
	1453	+ const char evalue, long long retval)
	1454	+{
	1455	+ if (btf_is_enum(t)) {
	1456	+ struct btf_enum *e = btf_enum(t);
	1457	+ int i, n = btf_vlen(t);
	1458	+
	1459	+ for (i = 0; i < n; ++i, ++e) {
	1460	+ const char *cur_name = btf__name_by_offset(btf, e->name_off);
	1461	+
	1462	+ if (strcmp(cur_name, evalue) == 0) {
	1463	+ *retval = e->val;
	1464	+ return 0;
	1465	+ }
	1466	+ }
	1467	+ } else if (btf_is_enum64(t)) {
	1468	+ struct btf_enum64 *e = btf_enum64(t);
	1469	+ int i, n = btf_vlen(t);
	1470	+
	1471	+ for (i = 0; i < n; ++i, ++e) {
	1472	+ const char *cur_name = btf__name_by_offset(btf, e->name_off);
	1473	+ __u64 value = btf_enum64_value(e);
	1474	+
	1475	+ if (strcmp(cur_name, evalue) == 0) {
	1476	+ *retval = value;
	1477	+ return 0;
	1478	+ }
	1479	+ }
	1480	+ }
	1481	+ return -EINVAL;
	1482	+}
	1483	+
	1484	+static bool is_preset_supported(const struct btf_type *t)
	1485	+{
	1486	+ return btf_is_int(t) \|\| btf_is_enum(t) \|\| btf_is_enum64(t);
	1487	+}
	1488	+
	1489	+static int set_global_var(struct bpf_object obj, struct btf btf, const struct btf_type *t,
	1490	+ struct bpf_map map, struct btf_var_secinfo sinfo,
	1491	+ struct var_preset *preset)
	1492	+{
	1493	+ const struct btf_type *base_type;
	1494	+ void *ptr;
	1495	+ long long value = preset->ivalue;
	1496	+ size_t size;
	1497	+
	1498	+ base_type = btf__type_by_id(btf, btf__resolve_type(btf, t->type));
	1499	+ if (!base_type) {
	1500	+ fprintf(stderr, "Failed to resolve type %d\n", t->type);
	1501	+ return -EINVAL;
	1502	+ }
	1503	+ if (!is_preset_supported(base_type)) {
	1504	+ fprintf(stderr, "Setting value for type %s is not supported\n",
	1505	+ btf__name_by_offset(btf, base_type->name_off));
	1506	+ return -EINVAL;
	1507	+ }
	1508	+
	1509	+ if (preset->type == ENUMERATOR) {
	1510	+ if (btf_is_any_enum(base_type)) {
	1511	+ if (enum_value_from_name(btf, base_type, preset->svalue, &value)) {
	1512	+ fprintf(stderr,
	1513	+ "Failed to find integer value for enum element %s\n",
	1514	+ preset->svalue);
	1515	+ return -EINVAL;
	1516	+ }
	1517	+ } else {
	1518	+ fprintf(stderr, "Value %s is not supported for type %s\n",
	1519	+ preset->svalue, btf__name_by_offset(btf, base_type->name_off));
	1520	+ return -EINVAL;
	1521	+ }
	1522	+ }
	1523	+
	1524	+ /* Check if value fits into the target variable size */
	1525	+ if (sinfo->size < sizeof(value)) {
	1526	+ bool is_signed = is_signed_type(base_type);
	1527	+ __u32 unsigned_bits = sinfo->size * 8 - (is_signed ? 1 : 0);
	1528	+ long long max_val = 1ll << unsigned_bits;
	1529	+
	1530	+ if (value >= max_val \|\| value < -max_val) {
	1531	+ fprintf(stderr,
	1532	+ "Variable %s value %lld is out of range [%lld; %lld]\n",
	1533	+ btf__name_by_offset(btf, t->name_off), value,
	1534	+ is_signed ? -max_val : 0, max_val - 1);
	1535	+ return -EINVAL;
	1536	+ }
	1537	+ }
	1538	+
	1539	+ ptr = bpf_map__initial_value(map, &size);
	1540	+ if (!ptr \|\| sinfo->offset + sinfo->size > size)
	1541	+ return -EINVAL;
	1542	+
	1543	+ if (__BYTE_ORDER == __LITTLE_ENDIAN) {
	1544	+ memcpy(ptr + sinfo->offset, &value, sinfo->size);
	1545	+ } else { /* __BYTE_ORDER == __BIG_ENDIAN */
	1546	+ __u8 src_offset = sizeof(value) - sinfo->size;
	1547	+
	1548	+ memcpy(ptr + sinfo->offset, (void *)&value + src_offset, sinfo->size);
	1549	+ }
	1550	+ return 0;
	1551	+}
	1552	+
	1553	+static int set_global_vars(struct bpf_object obj, struct var_preset presets, int npresets)
	1554	+{
	1555	+ struct btf_var_secinfo *sinfo;
	1556	+ const char *sec_name;
	1557	+ const struct btf_type *t;
	1558	+ struct bpf_map *map;
	1559	+ struct btf *btf;
	1560	+ int i, j, k, n, cnt, err = 0;
	1561	+
	1562	+ if (npresets == 0)
	1563	+ return 0;
	1564	+
	1565	+ btf = bpf_object__btf(obj);
	1566	+ if (!btf)
	1567	+ return -EINVAL;
	1568	+
	1569	+ cnt = btf__type_cnt(btf);
	1570	+ for (i = 1; i != cnt; ++i) {
	1571	+ t = btf__type_by_id(btf, i);
	1572	+
	1573	+ if (!btf_is_datasec(t))
	1574	+ continue;
	1575	+
	1576	+ sinfo = btf_var_secinfos(t);
	1577	+ sec_name = btf__name_by_offset(btf, t->name_off);
	1578	+ map = bpf_object__find_map_by_name(obj, sec_name);
	1579	+ if (!map)
	1580	+ continue;
	1581	+
	1582	+ n = btf_vlen(t);
	1583	+ for (j = 0; j < n; ++j, ++sinfo) {
	1584	+ const struct btf_type *var_type = btf__type_by_id(btf, sinfo->type);
	1585	+ const char *var_name;
	1586	+
	1587	+ if (!btf_is_var(var_type))
	1588	+ continue;
	1589	+
	1590	+ var_name = btf__name_by_offset(btf, var_type->name_off);
	1591	+
	1592	+ for (k = 0; k < npresets; ++k) {
	1593	+ if (strcmp(var_name, presets[k].name) != 0)
	1594	+ continue;
	1595	+
	1596	+ if (presets[k].applied) {
	1597	+ fprintf(stderr, "Variable %s is set more than once",
	1598	+ var_name);
	1599	+ return -EINVAL;
	1600	+ }
	1601	+
	1602	+ err = set_global_var(obj, btf, var_type, map, sinfo, presets + k);
	1603	+ if (err)
	1604	+ return err;
	1605	+
	1606	+ presets[k].applied = true;
	1607	+ break;
	1608	+ }
	1609	+ }
	1610	+ }
	1611	+ for (i = 0; i < npresets; ++i) {
	1612	+ if (!presets[i].applied) {
	1613	+ fprintf(stderr, "Global variable preset %s has not been applied\n",
	1614	+ presets[i].name);
	1615	+ }
	1616	+ presets[i].applied = false;
	1617	+ }
	1618	+ return err;
	1619	+}
	1620	+
1295	1621	static int process_obj(const char *filename)
1296	1622	{
1297	1623	const char *base_filename = basename(strdupa(filename));
@@ -1341,6 +1667,11 @@ static int process_obj(const char *filename) · 43 unmodified lines
1341	1667	if (prog_cnt == 1) {
1342	1668	prog = bpf_object__next_program(obj, NULL);
1343	1669	bpf_program__set_autoload(prog, true);
	1670	+ err = set_global_vars(obj, env.presets, env.npresets);
	1671	+ if (err) {
	1672	+ fprintf(stderr, "Failed to set global variables %d\n", err);
	1673	+ goto cleanup;
	1674	+ }
1344	1675	process_prog(filename, obj, prog);
1345	1676	goto cleanup;
1346	1677	}
@@ -1355,6 +1686,12 @@ static int process_obj(const char *filename) · 8 unmodified lines
1355	1686	goto cleanup;
1356	1687	}
1357	1688
	1689	+ err = set_global_vars(tobj, env.presets, env.npresets);
	1690	+ if (err) {
	1691	+ fprintf(stderr, "Failed to set global variables %d\n", err);
	1692	+ goto cleanup;
	1693	+ }
	1694	+
1358	1695	lprog = NULL;
1359	1696	bpf_object__for_each_program(tprog, tobj) {
1360	1697	const char *tprog_name = bpf_program__name(tprog);
@@ -2460,5 +2797,11 @@ int main(int argc, char **argv) · 1099 unmodified lines
2460	2797	free(env.deny_filters[i].prog_glob);
2461	2798	}
2462	2799	free(env.deny_filters);
	2800	+ for (i = 0; i < env.npresets; ++i) {
	2801	+ free(env.presets[i].name);
	2802	+ if (env.presets[i].type == ENUMERATOR)
	2803	+ free(env.presets[i].svalue);
	2804	+ }
	2805	+ free(env.presets);
2463	2806	return -err;
2464	2807	}

tools/testing/selftests/bpf/with_addr.sh+0−54

@@ -1,54 +0,0 @@
1		−#!/bin/bash
2		−# SPDX-License-Identifier: GPL-2.0
3		−#
4		−# add private ipv4 and ipv6 addresses to loopback
5		−
6		−readonly V6_INNER='100::a/128'
7		−readonly V4_INNER='192.168.0.1/32'
8		−
9		−if getopts ":s" opt; then
10		− readonly SIT_DEV_NAME='sixtofourtest0'
11		− readonly V6_SIT='2::/64'
12		− readonly V4_SIT='172.17.0.1/32'
13		− shift
14		−fi
15		−
16		−fail() {
17		− echo "error: $*" 1>&2
18		− exit 1
19		−}
20		−
21		−setup() {
22		− ip -6 addr add "${V6_INNER}" dev lo \|\| fail 'failed to setup v6 address'
23		− ip -4 addr add "${V4_INNER}" dev lo \|\| fail 'failed to setup v4 address'
24		−
25		− if [[ -n "${V6_SIT}" ]]; then
26		− ip link add "${SIT_DEV_NAME}" type sit remote any local any \
27		− \|\| fail 'failed to add sit'
28		− ip link set dev "${SIT_DEV_NAME}" up \
29		− \|\| fail 'failed to bring sit device up'
30		− ip -6 addr add "${V6_SIT}" dev "${SIT_DEV_NAME}" \
31		− \|\| fail 'failed to setup v6 SIT address'
32		− ip -4 addr add "${V4_SIT}" dev "${SIT_DEV_NAME}" \
33		− \|\| fail 'failed to setup v4 SIT address'
34		− fi
35		−
36		− sleep 2 # avoid race causing bind to fail
37		−}
38		−
39		−cleanup() {
40		− if [[ -n "${V6_SIT}" ]]; then
41		− ip -4 addr del "${V4_SIT}" dev "${SIT_DEV_NAME}"
42		− ip -6 addr del "${V6_SIT}" dev "${SIT_DEV_NAME}"
43		− ip link del "${SIT_DEV_NAME}"
44		− fi
45		−
46		− ip -4 addr del "${V4_INNER}" dev lo
47		− ip -6 addr del "${V6_INNER}" dev lo
48		−}
49		−
50		−trap cleanup EXIT
51		−
52		−setup
53		−"$@"
54		−exit "$?"

tools/testing/selftests/bpf/with_tunnels.sh+0−36

@@ -1,36 +0,0 @@
1		−#!/bin/bash
2		−# SPDX-License-Identifier: GPL-2.0
3		−#
4		−# setup tunnels for flow dissection test
5		−
6		−readonly SUFFIX="test_$(mktemp -u XXXX)"
7		−CONFIG="remote 127.0.0.2 local 127.0.0.1 dev lo"
8		−
9		−setup() {
10		− ip link add "ipip_${SUFFIX}" type ipip ${CONFIG}
11		− ip link add "gre_${SUFFIX}" type gre ${CONFIG}
12		− ip link add "sit_${SUFFIX}" type sit ${CONFIG}
13		−
14		− echo "tunnels before test:"
15		− ip tunnel show
16		−
17		− ip link set "ipip_${SUFFIX}" up
18		− ip link set "gre_${SUFFIX}" up
19		− ip link set "sit_${SUFFIX}" up
20		−}
21		−
22		−
23		−cleanup() {
24		− ip tunnel del "ipip_${SUFFIX}"
25		− ip tunnel del "gre_${SUFFIX}"
26		− ip tunnel del "sit_${SUFFIX}"
27		−
28		− echo "tunnels after test:"
29		− ip tunnel show
30		−}
31		−
32		−trap cleanup EXIT
33		−
34		−setup
35		−"$@"
36		−exit "$?"

tools/testing/selftests/bpf/xdp_redirect_multi.c+0−226

@@ -1,226 +0,0 @@
1		−// SPDX-License-Identifier: GPL-2.0
2		−#include <linux/bpf.h>
3		−#include <linux/if_link.h>
4		−#include <assert.h>
5		−#include <errno.h>
6		−#include <signal.h>
7		−#include <stdio.h>
8		−#include <stdlib.h>
9		−#include <string.h>
10		−#include <net/if.h>
11		−#include <unistd.h>
12		−#include <libgen.h>
13		−#include <sys/ioctl.h>
14		−#include <sys/types.h>
15		−#include <sys/socket.h>
16		−#include <netinet/in.h>
17		−
18		−#include "bpf_util.h"
19		−#include <bpf/bpf.h>
20		−#include <bpf/libbpf.h>
21		−
22		−#define MAX_IFACE_NUM 32
23		−#define MAX_INDEX_NUM 1024
24		−
25		−static __u32 xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST;
26		−static int ifaces[MAX_IFACE_NUM] = {};
27		−
28		−static void int_exit(int sig)
29		−{
30		− __u32 prog_id = 0;
31		− int i;
32		−
33		− for (i = 0; ifaces[i] > 0; i++) {
34		− if (bpf_xdp_query_id(ifaces[i], xdp_flags, &prog_id)) {
35		− printf("bpf_xdp_query_id failed\n");
36		− exit(1);
37		− }
38		− if (prog_id)
39		− bpf_xdp_detach(ifaces[i], xdp_flags, NULL);
40		− }
41		−
42		− exit(0);
43		−}
44		−
45		−static int get_mac_addr(unsigned int ifindex, void *mac_addr)
46		−{
47		− char ifname[IF_NAMESIZE];
48		− struct ifreq ifr;
49		− int fd, ret = -1;
50		−
51		− fd = socket(AF_INET, SOCK_DGRAM, 0);
52		− if (fd < 0)
53		− return ret;
54		−
55		− if (!if_indextoname(ifindex, ifname))
56		− goto err_out;
57		−
58		− strcpy(ifr.ifr_name, ifname);
59		−
60		− if (ioctl(fd, SIOCGIFHWADDR, &ifr) != 0)
61		− goto err_out;
62		−
63		− memcpy(mac_addr, ifr.ifr_hwaddr.sa_data, 6 * sizeof(char));
64		− ret = 0;
65		−
66		−err_out:
67		− close(fd);
68		− return ret;
69		−}
70		−
71		−static void usage(const char *prog)
72		−{
73		− fprintf(stderr,
74		− "usage: %s [OPTS] <IFNAME\|IFINDEX> <IFNAME\|IFINDEX> ...\n"
75		− "OPTS:\n"
76		− " -S use skb-mode\n"
77		− " -N enforce native mode\n"
78		− " -F force loading prog\n"
79		− " -X load xdp program on egress\n",
80		− prog);
81		−}
82		−
83		−int main(int argc, char **argv)
84		−{
85		− int prog_fd, group_all, mac_map;
86		− struct bpf_program ingress_prog, egress_prog;
87		− int i, err, ret, opt, egress_prog_fd = 0;
88		− struct bpf_devmap_val devmap_val;
89		− bool attach_egress_prog = false;
90		− unsigned char mac_addr[6];
91		− char ifname[IF_NAMESIZE];
92		− struct bpf_object *obj;
93		− unsigned int ifindex;
94		− char filename[256];
95		−
96		− while ((opt = getopt(argc, argv, "SNFX")) != -1) {
97		− switch (opt) {
98		− case 'S':
99		− xdp_flags \|= XDP_FLAGS_SKB_MODE;
100		− break;
101		− case 'N':
102		− /* default, set below */
103		− break;
104		− case 'F':
105		− xdp_flags &= ~XDP_FLAGS_UPDATE_IF_NOEXIST;
106		− break;
107		− case 'X':
108		− attach_egress_prog = true;
109		− break;
110		− default:
111		− usage(basename(argv[0]));
112		− return 1;
113		− }
114		− }
115		−
116		− if (!(xdp_flags & XDP_FLAGS_SKB_MODE)) {
117		− xdp_flags \|= XDP_FLAGS_DRV_MODE;
118		− } else if (attach_egress_prog) {
119		− printf("Load xdp program on egress with SKB mode not supported yet\n");
120		− goto err_out;
121		− }
122		−
123		− if (optind == argc) {
124		− printf("usage: %s <IFNAME\|IFINDEX> <IFNAME\|IFINDEX> ...\n", argv[0]);
125		− goto err_out;
126		− }
127		−
128		− printf("Get interfaces:");
129		− for (i = 0; i < MAX_IFACE_NUM && argv[optind + i]; i++) {
130		− ifaces[i] = if_nametoindex(argv[optind + i]);
131		− if (!ifaces[i])
132		− ifaces[i] = strtoul(argv[optind + i], NULL, 0);
133		− if (!if_indextoname(ifaces[i], ifname)) {
134		− perror("Invalid interface name or i");
135		− goto err_out;
136		− }
137		− if (ifaces[i] > MAX_INDEX_NUM) {
138		− printf(" interface index too large\n");
139		− goto err_out;
140		− }
141		− printf(" %d", ifaces[i]);
142		− }
143		− printf("\n");
144		−
145		− snprintf(filename, sizeof(filename), "%s_kern.bpf.o", argv[0]);
146		− obj = bpf_object__open_file(filename, NULL);
147		− err = libbpf_get_error(obj);
148		− if (err)
149		− goto err_out;
150		− err = bpf_object__load(obj);
151		− if (err)
152		− goto err_out;
153		− prog_fd = bpf_program__fd(bpf_object__next_program(obj, NULL));
154		−
155		− if (attach_egress_prog)
156		− group_all = bpf_object__find_map_fd_by_name(obj, "map_egress");
157		− else
158		− group_all = bpf_object__find_map_fd_by_name(obj, "map_all");
159		− mac_map = bpf_object__find_map_fd_by_name(obj, "mac_map");
160		−
161		− if (group_all < 0 \|\| mac_map < 0) {
162		− printf("bpf_object__find_map_fd_by_name failed\n");
163		− goto err_out;
164		− }
165		−
166		− if (attach_egress_prog) {
167		− /* Find ingress/egress prog for 2nd xdp prog */
168		− ingress_prog = bpf_object__find_program_by_name(obj, "xdp_redirect_map_all_prog");
169		− egress_prog = bpf_object__find_program_by_name(obj, "xdp_devmap_prog");
170		− if (!ingress_prog \|\| !egress_prog) {
171		− printf("finding ingress/egress_prog in obj file failed\n");
172		− goto err_out;
173		− }
174		− prog_fd = bpf_program__fd(ingress_prog);
175		− egress_prog_fd = bpf_program__fd(egress_prog);
176		− if (prog_fd < 0 \|\| egress_prog_fd < 0) {
177		− printf("find egress_prog fd failed\n");
178		− goto err_out;
179		− }
180		− }
181		−
182		− signal(SIGINT, int_exit);
183		− signal(SIGTERM, int_exit);
184		−
185		− /* Init forward multicast groups and exclude group */
186		− for (i = 0; ifaces[i] > 0; i++) {
187		− ifindex = ifaces[i];
188		−
189		− if (attach_egress_prog) {
190		− ret = get_mac_addr(ifindex, mac_addr);
191		− if (ret < 0) {
192		− printf("get interface %d mac failed\n", ifindex);
193		− goto err_out;
194		− }
195		− ret = bpf_map_update_elem(mac_map, &ifindex, mac_addr, 0);
196		− if (ret) {
197		− perror("bpf_update_elem mac_map failed\n");
198		− goto err_out;
199		− }
200		− }
201		−
202		− /* Add all the interfaces to group all */
203		− devmap_val.ifindex = ifindex;
204		− devmap_val.bpf_prog.fd = egress_prog_fd;
205		− ret = bpf_map_update_elem(group_all, &ifindex, &devmap_val, 0);
206		− if (ret) {
207		− perror("bpf_map_update_elem");
208		− goto err_out;
209		− }
210		−
211		− /* bind prog_fd to each interface */
212		− ret = bpf_xdp_attach(ifindex, prog_fd, xdp_flags, NULL);
213		− if (ret) {
214		− printf("Set xdp fd failed on %d\n", ifindex);
215		− goto err_out;
216		− }
217		− }
218		−
219		− /* sleep some time for testing */
220		− sleep(999);
221		−
222		− return 0;
223		−
224		−err_out:
225		− return 1;
226		−}