Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Alexei Starovoitov says: ==================== pull-request: bpf-next 2021-12-30 The following pull-request contains BPF updates for your *net-next* tree. We've added 72 non-merge commits during the last 20 day(s) which contain a total of 223 files changed, 3510 insertions(+), 1591 deletions(-). The main changes are: 1) Automatic setrlimit in libbpf when bpf is memcg's in the kernel, from Andrii. 2) Beautify and de-verbose verifier logs, from Christy. 3) Composable verifier types, from Hao. 4) bpf_strncmp helper, from Hou. 5) bpf.h header dependency cleanup, from Jakub. 6) get_func_[arg|ret|arg_cnt] helpers, from Jiri. 7) Sleepable local storage, from KP. 8) Extend kfunc with PTR_TO_CTX, PTR_TO_MEM argument support, from Kumar. ====================

David S. Miller committed 4 years ago

Commit e63a0232 parents ce2b6eb 9e6b19a221 files · +3506 −1591Browse files

Unified · Split

221 files

Documentation/bpf/classic_vs_extended.rst +376 −0
Documentation/bpf/index.rst +1 −0
Documentation/bpf/instruction-set.rst +150 −364
arch/s390/mm/hugetlbpage.c +1 −0
arch/x86/net/bpf_jit_comp.c +41 −14
drivers/bluetooth/btqca.c +1 −0
drivers/infiniband/core/cache.c +1 −0
drivers/infiniband/hw/irdma/ctrl.c +2 −0
drivers/infiniband/hw/irdma/uda.c +2 −0
drivers/infiniband/hw/mlx5/doorbell.c +1 −0
drivers/infiniband/hw/mlx5/qp.c +1 −0
drivers/net/amt.c +1 −0
drivers/net/appletalk/ipddp.c +1 −0
drivers/net/bonding/bond_main.c +1 −0
drivers/net/can/usb/peak_usb/pcan_usb.c +1 −0
drivers/net/dsa/microchip/ksz8795.c +1 −0
drivers/net/dsa/xrs700x/xrs700x.c +1 −0
drivers/net/ethernet/amazon/ena/ena_netdev.c +1 −1
drivers/net/ethernet/amazon/ena/ena_netdev.h +1 −0
drivers/net/ethernet/broadcom/bnxt/bnxt_devlink.c +1 −0
drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c +1 −1
drivers/net/ethernet/cavium/thunder/nicvf_main.c +1 −1
drivers/net/ethernet/cavium/thunder/nicvf_queues.c +1 −0
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +1 −1
drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c +1 −1
drivers/net/ethernet/freescale/enetc/enetc.c +1 −1
drivers/net/ethernet/huawei/hinic/hinic_tx.c +1 −0
drivers/net/ethernet/intel/i40e/i40e_txrx.c +1 −1
drivers/net/ethernet/intel/i40e/i40e_xsk.c +1 −1
drivers/net/ethernet/intel/i40e/i40e_xsk.h +0 −1
drivers/net/ethernet/intel/ice/ice_devlink.c +2 −0
drivers/net/ethernet/intel/ice/ice_txrx.c +1 −1
drivers/net/ethernet/intel/ice/ice_txrx_lib.c +2 −0
drivers/net/ethernet/intel/ice/ice_xsk.c +1 −1
drivers/net/ethernet/intel/igb/igb_main.c +1 −1
drivers/net/ethernet/intel/igc/igc_main.c +1 −1
drivers/net/ethernet/intel/igc/igc_xdp.c +1 −0
drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +1 −1
drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h +0 −2
drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c +1 −1
drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +1 −1
drivers/net/ethernet/marvell/mvneta.c +1 −1
drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c +1 −1
drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c +1 −1
drivers/net/ethernet/mellanox/mlx4/en_netdev.c +1 −0
drivers/net/ethernet/mellanox/mlx4/en_rx.c +1 −1
drivers/net/ethernet/mellanox/mlx5/core/en/qos.c +1 −0
drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c +1 −1
drivers/net/ethernet/microsoft/mana/mana_bpf.c +1 −1
drivers/net/ethernet/microsoft/mana/mana_en.c +2 −0
drivers/net/ethernet/netronome/nfp/nfp_net_common.c +1 −1
drivers/net/ethernet/qlogic/qede/qede_fp.c +1 −1
drivers/net/ethernet/sfc/efx.c +1 −0
drivers/net/ethernet/sfc/efx_channels.c +1 −0
drivers/net/ethernet/sfc/efx_common.c +1 −0
drivers/net/ethernet/sfc/rx.c +1 −1
drivers/net/ethernet/socionext/netsec.c +1 −1
drivers/net/ethernet/stmicro/stmmac/stmmac.h +1 −0
drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +1 −1
drivers/net/ethernet/ti/cpsw_priv.c +1 −1
drivers/net/ethernet/ti/cpsw_priv.h +2 −0
drivers/net/hamradio/hdlcdrv.c +1 −0
drivers/net/hamradio/scc.c +1 −0
drivers/net/hyperv/netvsc_bpf.c +1 −1
drivers/net/loopback.c +1 −0
drivers/net/tun.c +1 −1
drivers/net/veth.c +2 −2
drivers/net/virtio_net.c +2 −2
drivers/net/vrf.c +1 −0
drivers/net/xen-netfront.c +1 −1
fs/nfs/dir.c +1 −0
fs/nfs/fs_context.c +1 −0
fs/select.c +1 −0
include/linux/bpf-cgroup-defs.h +70 −0
include/linux/bpf-cgroup.h +1 −56
include/linux/bpf-netns.h +1 −7
include/linux/bpf.h +83 −24
include/linux/bpf_local_storage.h +6 −0
include/linux/bpf_verifier.h +27 −0
include/linux/cgroup-defs.h +1 −1
include/linux/dsa/loop.h +1 −0
include/linux/filter.h +1 −1
include/linux/perf_event.h +1 −0
include/net/ip6_fib.h +1 −0
include/net/ipv6.h +2 −0
include/net/netns/bpf.h +8 −1
include/net/route.h +1 −0
include/net/sock.h +1 −1
include/net/xdp_priv.h +0 −1
include/net/xdp_sock.h +1 −0
include/uapi/linux/bpf.h +39 −0
kernel/bpf/bloom_filter.c +6 −0
kernel/bpf/bpf_inode_storage.c +5 −1
kernel/bpf/bpf_local_storage.c +37 −13
kernel/bpf/bpf_task_storage.c +5 −1
kernel/bpf/btf.c +88 −36
kernel/bpf/cgroup.c +1 −1
kernel/bpf/cpumap.c +2 −2
kernel/bpf/devmap.c +2 −2
kernel/bpf/helpers.c +23 −6
kernel/bpf/local_storage.c +1 −2
kernel/bpf/lpm_trie.c +1 −1
kernel/bpf/map_iter.c +2 −2
kernel/bpf/net_namespace.c +1 −0
kernel/bpf/reuseport_array.c +1 −5
kernel/bpf/ringbuf.c +1 −1
kernel/bpf/syscall.c +2 −1
kernel/bpf/trampoline.c +8 −0
kernel/bpf/verifier.c +415 −281
kernel/cgroup/cgroup.c +1 −0
kernel/sysctl.c +1 −0
kernel/trace/bpf_trace.c +67 −14
kernel/trace/trace_kprobe.c +1 −0
kernel/trace/trace_uprobe.c +1 −0
net/bluetooth/bnep/sock.c +1 −0
net/bluetooth/eir.h +2 −0
net/bluetooth/hidp/sock.c +1 −0
net/bluetooth/l2cap_sock.c +1 −0
net/bridge/br_ioctl.c +1 −0
net/caif/caif_socket.c +1 −0
net/core/bpf_sk_storage.c +8 −2
net/core/dev.c +1 −1
net/core/devlink.c +1 −0
net/core/filter.c +36 −36
net/core/flow_dissector.c +1 −0
net/core/lwt_bpf.c +1 −0
net/core/sock_diag.c +1 −0
net/core/sock_map.c +1 −1
net/core/sysctl_net_core.c +1 −0
net/decnet/dn_nsp_in.c +1 −0
net/dsa/dsa_priv.h +1 −0
net/ethtool/ioctl.c +1 −0
net/ipv4/nexthop.c +1 −0
net/ipv4/udp.c +1 −0
net/ipv6/ip6_fib.c +1 −0
net/ipv6/seg6_local.c +1 −0
net/ipv6/udp.c +1 −0
net/iucv/af_iucv.c +1 −0
net/kcm/kcmsock.c +1 −0
net/netfilter/nfnetlink_hook.c +1 −0
net/netfilter/nft_reject_netdev.c +1 −0
net/netlink/af_netlink.c +2 −0
net/packet/af_packet.c +1 −0
net/rose/rose_in.c +1 −0
net/sched/sch_frag.c +1 −0
net/smc/smc_ib.c +2 −0
net/smc/smc_ism.c +1 −0
net/socket.c +1 −0
net/unix/af_unix.c +1 −0
net/vmw_vsock/af_vsock.c +1 −0
net/xdp/xskmap.c +1 −0
net/xfrm/xfrm_state.c +1 −0
net/xfrm/xfrm_user.c +1 −0
samples/bpf/hbm.c +10 −1
samples/bpf/xdp_fwd_user.c +10 −2
security/device_cgroup.c +1 −0
tools/bpf/bpftool/Makefile +7 −6
tools/bpf/bpftool/feature.c +23 −3
tools/bpf/bpftool/main.c +2 −0
tools/bpf/resolve_btfids/Makefile +1 −0
tools/bpf/runqslower/Makefile +2 −2
tools/include/uapi/linux/bpf.h +39 −0
tools/lib/bpf/Makefile +2 −1
tools/lib/bpf/bpf.c +83 −2
tools/lib/bpf/bpf.h +2 −0
tools/lib/bpf/bpf_tracing.h +174 −257
tools/lib/bpf/btf.h +6 −0
tools/lib/bpf/btf_dump.c +2 −2
tools/lib/bpf/gen_loader.c +5 −6
tools/lib/bpf/libbpf.c +41 −44
tools/lib/bpf/libbpf.h +73 −4
tools/lib/bpf/libbpf.map +4 −0
tools/lib/bpf/libbpf_internal.h +60 −0
tools/lib/bpf/libbpf_legacy.h +11 −1
tools/lib/bpf/libbpf_probes.c +185 −66
tools/lib/bpf/relo_core.c +15 −5
tools/lib/bpf/xsk.c +3 −6
tools/perf/builtin-trace.c +12 −1
tools/scripts/Makefile.include +12 −1
tools/testing/selftests/bpf/Makefile +7 −5
tools/testing/selftests/bpf/bench.c +7 −17
tools/testing/selftests/bpf/bench.h +5 −4
tools/testing/selftests/bpf/benchs/bench_count.c +1 −1
tools/testing/selftests/bpf/benchs/bench_rename.c +8 −8
tools/testing/selftests/bpf/benchs/bench_ringbufs.c +7 −7
tools/testing/selftests/bpf/benchs/bench_strncmp.c +161 −0
tools/testing/selftests/bpf/benchs/bench_trigger.c +12 −12
tools/testing/selftests/bpf/benchs/run_bench_strncmp.sh +12 −0
tools/testing/selftests/bpf/config +2 −0
tools/testing/selftests/bpf/prog_tests/align.c +103 −88
tools/testing/selftests/bpf/prog_tests/bpf_obj_id.c +2 −2
tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c +3 −4
tools/testing/selftests/bpf/prog_tests/btf.c +0 −1
tools/testing/selftests/bpf/prog_tests/connect_force_port.c +9 −9
tools/testing/selftests/bpf/prog_tests/core_reloc.c +53 −26
tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c +13 −4
tools/testing/selftests/bpf/prog_tests/get_func_args_test.c +44 −0
tools/testing/selftests/bpf/prog_tests/get_stack_raw_tp.c +2 −2
tools/testing/selftests/bpf/prog_tests/ksyms_btf.c +14 −0
tools/testing/selftests/bpf/prog_tests/libbpf_probes.c +124 −0
tools/testing/selftests/bpf/prog_tests/select_reuseport.c +0 −1
tools/testing/selftests/bpf/prog_tests/sk_lookup.c +0 −1
tools/testing/selftests/bpf/prog_tests/sock_fields.c +0 −1
tools/testing/selftests/bpf/prog_tests/sockopt_inherit.c +8 −7
tools/testing/selftests/bpf/prog_tests/stacktrace_map.c +2 −2
tools/testing/selftests/bpf/prog_tests/stacktrace_map_raw_tp.c +2 −2
tools/testing/selftests/bpf/prog_tests/test_local_storage.c +6 −14
tools/testing/selftests/bpf/prog_tests/test_overhead.c +10 −10
tools/testing/selftests/bpf/prog_tests/test_strncmp.c +167 −0
tools/testing/selftests/bpf/prog_tests/trampoline_count.c +3 −3
tools/testing/selftests/bpf/progs/get_func_args_test.c +123 −0
tools/testing/selftests/bpf/progs/local_storage.c +5 −19
tools/testing/selftests/bpf/progs/strncmp_bench.c +50 −0
tools/testing/selftests/bpf/progs/strncmp_test.c +54 −0
tools/testing/selftests/bpf/progs/test_ksyms_btf_write_check.c +29 −0
tools/testing/selftests/bpf/test_cpp.cpp +8 −1
tools/testing/selftests/bpf/test_maps.c +0 −1
tools/testing/selftests/bpf/test_progs.c +0 −2
tools/testing/selftests/bpf/test_verifier.c +8 −10
tools/testing/selftests/bpf/verifier/btf_ctx_access.c +12 −0
tools/testing/selftests/bpf/vmtest.sh +1 −1

Documentation/bpf/classic_vs_extended.rst+376−0

@@ -0,0 +1,376 @@
	1	+
	2	+===================
	3	+Classic BPF vs eBPF
	4	+===================
	5	+
	6	+eBPF is designed to be JITed with one to one mapping, which can also open up
	7	+the possibility for GCC/LLVM compilers to generate optimized eBPF code through
	8	+an eBPF backend that performs almost as fast as natively compiled code.
	9	+
	10	+Some core changes of the eBPF format from classic BPF:
	11	+
	12	+- Number of registers increase from 2 to 10:
	13	+
	14	+ The old format had two registers A and X, and a hidden frame pointer. The
	15	+ new layout extends this to be 10 internal registers and a read-only frame
	16	+ pointer. Since 64-bit CPUs are passing arguments to functions via registers
	17	+ the number of args from eBPF program to in-kernel function is restricted
	18	+ to 5 and one register is used to accept return value from an in-kernel
	19	+ function. Natively, x86_64 passes first 6 arguments in registers, aarch64/
	20	+ sparcv9/mips64 have 7 - 8 registers for arguments; x86_64 has 6 callee saved
	21	+ registers, and aarch64/sparcv9/mips64 have 11 or more callee saved registers.
	22	+
	23	+ Thus, all eBPF registers map one to one to HW registers on x86_64, aarch64,
	24	+ etc, and eBPF calling convention maps directly to ABIs used by the kernel on
	25	+ 64-bit architectures.
	26	+
	27	+ On 32-bit architectures JIT may map programs that use only 32-bit arithmetic
	28	+ and may let more complex programs to be interpreted.
	29	+
	30	+ R0 - R5 are scratch registers and eBPF program needs spill/fill them if
	31	+ necessary across calls. Note that there is only one eBPF program (== one
	32	+ eBPF main routine) and it cannot call other eBPF functions, it can only
	33	+ call predefined in-kernel functions, though.
	34	+
	35	+- Register width increases from 32-bit to 64-bit:
	36	+
	37	+ Still, the semantics of the original 32-bit ALU operations are preserved
	38	+ via 32-bit subregisters. All eBPF registers are 64-bit with 32-bit lower
	39	+ subregisters that zero-extend into 64-bit if they are being written to.
	40	+ That behavior maps directly to x86_64 and arm64 subregister definition, but
	41	+ makes other JITs more difficult.
	42	+
	43	+ 32-bit architectures run 64-bit eBPF programs via interpreter.
	44	+ Their JITs may convert BPF programs that only use 32-bit subregisters into
	45	+ native instruction set and let the rest being interpreted.
	46	+
	47	+ Operation is 64-bit, because on 64-bit architectures, pointers are also
	48	+ 64-bit wide, and we want to pass 64-bit values in/out of kernel functions,
	49	+ so 32-bit eBPF registers would otherwise require to define register-pair
	50	+ ABI, thus, there won't be able to use a direct eBPF register to HW register
	51	+ mapping and JIT would need to do combine/split/move operations for every
	52	+ register in and out of the function, which is complex, bug prone and slow.
	53	+ Another reason is the use of atomic 64-bit counters.
	54	+
	55	+- Conditional jt/jf targets replaced with jt/fall-through:
	56	+
	57	+ While the original design has constructs such as ``if (cond) jump_true;
	58	+ else jump_false;``, they are being replaced into alternative constructs like
	59	+ ``if (cond) jump_true; /* else fall-through */``.
	60	+
	61	+- Introduces bpf_call insn and register passing convention for zero overhead
	62	+ calls from/to other kernel functions:
	63	+
	64	+ Before an in-kernel function call, the eBPF program needs to
	65	+ place function arguments into R1 to R5 registers to satisfy calling
	66	+ convention, then the interpreter will take them from registers and pass
	67	+ to in-kernel function. If R1 - R5 registers are mapped to CPU registers
	68	+ that are used for argument passing on given architecture, the JIT compiler
	69	+ doesn't need to emit extra moves. Function arguments will be in the correct
	70	+ registers and BPF_CALL instruction will be JITed as single 'call' HW
	71	+ instruction. This calling convention was picked to cover common call
	72	+ situations without performance penalty.
	73	+
	74	+ After an in-kernel function call, R1 - R5 are reset to unreadable and R0 has
	75	+ a return value of the function. Since R6 - R9 are callee saved, their state
	76	+ is preserved across the call.
	77	+
	78	+ For example, consider three C functions::
	79	+
	80	+ u64 f1() { return (*_f2)(1); }
	81	+ u64 f2(u64 a) { return f3(a + 1, a); }
	82	+ u64 f3(u64 a, u64 b) { return a - b; }
	83	+
	84	+ GCC can compile f1, f3 into x86_64::
	85	+
	86	+ f1:
	87	+ movl $1, %edi
	88	+ movq _f2(%rip), %rax
	89	+ jmp *%rax
	90	+ f3:
	91	+ movq %rdi, %rax
	92	+ subq %rsi, %rax
	93	+ ret
	94	+
	95	+ Function f2 in eBPF may look like::
	96	+
	97	+ f2:
	98	+ bpf_mov R2, R1
	99	+ bpf_add R1, 1
	100	+ bpf_call f3
	101	+ bpf_exit
	102	+
	103	+ If f2 is JITed and the pointer stored to ``_f2``. The calls f1 -> f2 -> f3 and
	104	+ returns will be seamless. Without JIT, __bpf_prog_run() interpreter needs to
	105	+ be used to call into f2.
	106	+
	107	+ For practical reasons all eBPF programs have only one argument 'ctx' which is
	108	+ already placed into R1 (e.g. on __bpf_prog_run() startup) and the programs
	109	+ can call kernel functions with up to 5 arguments. Calls with 6 or more arguments
	110	+ are currently not supported, but these restrictions can be lifted if necessary
	111	+ in the future.
	112	+
	113	+ On 64-bit architectures all register map to HW registers one to one. For
	114	+ example, x86_64 JIT compiler can map them as ...
	115	+
	116	+ ::
	117	+
	118	+ R0 - rax
	119	+ R1 - rdi
	120	+ R2 - rsi
	121	+ R3 - rdx
	122	+ R4 - rcx
	123	+ R5 - r8
	124	+ R6 - rbx
	125	+ R7 - r13
	126	+ R8 - r14
	127	+ R9 - r15
	128	+ R10 - rbp
	129	+
	130	+ ... since x86_64 ABI mandates rdi, rsi, rdx, rcx, r8, r9 for argument passing
	131	+ and rbx, r12 - r15 are callee saved.
	132	+
	133	+ Then the following eBPF pseudo-program::
	134	+
	135	+ bpf_mov R6, R1 /* save ctx */
	136	+ bpf_mov R2, 2
	137	+ bpf_mov R3, 3
	138	+ bpf_mov R4, 4
	139	+ bpf_mov R5, 5
	140	+ bpf_call foo
	141	+ bpf_mov R7, R0 /* save foo() return value */
	142	+ bpf_mov R1, R6 /* restore ctx for next call */
	143	+ bpf_mov R2, 6
	144	+ bpf_mov R3, 7
	145	+ bpf_mov R4, 8
	146	+ bpf_mov R5, 9
	147	+ bpf_call bar
	148	+ bpf_add R0, R7
	149	+ bpf_exit
	150	+
	151	+ After JIT to x86_64 may look like::
	152	+
	153	+ push %rbp
	154	+ mov %rsp,%rbp
	155	+ sub $0x228,%rsp
	156	+ mov %rbx,-0x228(%rbp)
	157	+ mov %r13,-0x220(%rbp)
	158	+ mov %rdi,%rbx
	159	+ mov $0x2,%esi
	160	+ mov $0x3,%edx
	161	+ mov $0x4,%ecx
	162	+ mov $0x5,%r8d
	163	+ callq foo
	164	+ mov %rax,%r13
	165	+ mov %rbx,%rdi
	166	+ mov $0x6,%esi
	167	+ mov $0x7,%edx
	168	+ mov $0x8,%ecx
	169	+ mov $0x9,%r8d
	170	+ callq bar
	171	+ add %r13,%rax
	172	+ mov -0x228(%rbp),%rbx
	173	+ mov -0x220(%rbp),%r13
	174	+ leaveq
	175	+ retq
	176	+
	177	+ Which is in this example equivalent in C to::
	178	+
	179	+ u64 bpf_filter(u64 ctx)
	180	+ {
	181	+ return foo(ctx, 2, 3, 4, 5) + bar(ctx, 6, 7, 8, 9);
	182	+ }
	183	+
	184	+ In-kernel functions foo() and bar() with prototype: u64 (*)(u64 arg1, u64
	185	+ arg2, u64 arg3, u64 arg4, u64 arg5); will receive arguments in proper
	186	+ registers and place their return value into ``%rax`` which is R0 in eBPF.
	187	+ Prologue and epilogue are emitted by JIT and are implicit in the
	188	+ interpreter. R0-R5 are scratch registers, so eBPF program needs to preserve
	189	+ them across the calls as defined by calling convention.
	190	+
	191	+ For example the following program is invalid::
	192	+
	193	+ bpf_mov R1, 1
	194	+ bpf_call foo
	195	+ bpf_mov R0, R1
	196	+ bpf_exit
	197	+
	198	+ After the call the registers R1-R5 contain junk values and cannot be read.
	199	+ An in-kernel verifier.rst is used to validate eBPF programs.
	200	+
	201	+Also in the new design, eBPF is limited to 4096 insns, which means that any
	202	+program will terminate quickly and will only call a fixed number of kernel
	203	+functions. Original BPF and eBPF are two operand instructions,
	204	+which helps to do one-to-one mapping between eBPF insn and x86 insn during JIT.
	205	+
	206	+The input context pointer for invoking the interpreter function is generic,
	207	+its content is defined by a specific use case. For seccomp register R1 points
	208	+to seccomp_data, for converted BPF filters R1 points to a skb.
	209	+
	210	+A program, that is translated internally consists of the following elements::
	211	+
	212	+ op:16, jt:8, jf:8, k:32 ==> op:8, dst_reg:4, src_reg:4, off:16, imm:32
	213	+
	214	+So far 87 eBPF instructions were implemented. 8-bit 'op' opcode field
	215	+has room for new instructions. Some of them may use 16/24/32 byte encoding. New
	216	+instructions must be multiple of 8 bytes to preserve backward compatibility.
	217	+
	218	+eBPF is a general purpose RISC instruction set. Not every register and
	219	+every instruction are used during translation from original BPF to eBPF.
	220	+For example, socket filters are not using ``exclusive add`` instruction, but
	221	+tracing filters may do to maintain counters of events, for example. Register R9
	222	+is not used by socket filters either, but more complex filters may be running
	223	+out of registers and would have to resort to spill/fill to stack.
	224	+
	225	+eBPF can be used as a generic assembler for last step performance
	226	+optimizations, socket filters and seccomp are using it as assembler. Tracing
	227	+filters may use it as assembler to generate code from kernel. In kernel usage
	228	+may not be bounded by security considerations, since generated eBPF code
	229	+may be optimizing internal code path and not being exposed to the user space.
	230	+Safety of eBPF can come from the verifier.rst. In such use cases as
	231	+described, it may be used as safe instruction set.
	232	+
	233	+Just like the original BPF, eBPF runs within a controlled environment,
	234	+is deterministic and the kernel can easily prove that. The safety of the program
	235	+can be determined in two steps: first step does depth-first-search to disallow
	236	+loops and other CFG validation; second step starts from the first insn and
	237	+descends all possible paths. It simulates execution of every insn and observes
	238	+the state change of registers and stack.
	239	+
	240	+opcode encoding
	241	+===============
	242	+
	243	+eBPF is reusing most of the opcode encoding from classic to simplify conversion
	244	+of classic BPF to eBPF.
	245	+
	246	+For arithmetic and jump instructions the 8-bit 'code' field is divided into three
	247	+parts::
	248	+
	249	+ +----------------+--------+--------------------+
	250	+ \| 4 bits \| 1 bit \| 3 bits \|
	251	+ \| operation code \| source \| instruction class \|
	252	+ +----------------+--------+--------------------+
	253	+ (MSB) (LSB)
	254	+
	255	+Three LSB bits store instruction class which is one of:
	256	+
	257	+ =================== ===============
	258	+ Classic BPF classes eBPF classes
	259	+ =================== ===============
	260	+ BPF_LD 0x00 BPF_LD 0x00
	261	+ BPF_LDX 0x01 BPF_LDX 0x01
	262	+ BPF_ST 0x02 BPF_ST 0x02
	263	+ BPF_STX 0x03 BPF_STX 0x03
	264	+ BPF_ALU 0x04 BPF_ALU 0x04
	265	+ BPF_JMP 0x05 BPF_JMP 0x05
	266	+ BPF_RET 0x06 BPF_JMP32 0x06
	267	+ BPF_MISC 0x07 BPF_ALU64 0x07
	268	+ =================== ===============
	269	+
	270	+The 4th bit encodes the source operand ...
	271	+
	272	+ ::
	273	+
	274	+ BPF_K 0x00
	275	+ BPF_X 0x08
	276	+
	277	+ * in classic BPF, this means::
	278	+
	279	+ BPF_SRC(code) == BPF_X - use register X as source operand
	280	+ BPF_SRC(code) == BPF_K - use 32-bit immediate as source operand
	281	+
	282	+ * in eBPF, this means::
	283	+
	284	+ BPF_SRC(code) == BPF_X - use 'src_reg' register as source operand
	285	+ BPF_SRC(code) == BPF_K - use 32-bit immediate as source operand
	286	+
	287	+... and four MSB bits store operation code.
	288	+
	289	+If BPF_CLASS(code) == BPF_ALU or BPF_ALU64 [ in eBPF ], BPF_OP(code) is one of::
	290	+
	291	+ BPF_ADD 0x00
	292	+ BPF_SUB 0x10
	293	+ BPF_MUL 0x20
	294	+ BPF_DIV 0x30
	295	+ BPF_OR 0x40
	296	+ BPF_AND 0x50
	297	+ BPF_LSH 0x60
	298	+ BPF_RSH 0x70
	299	+ BPF_NEG 0x80
	300	+ BPF_MOD 0x90
	301	+ BPF_XOR 0xa0
	302	+ BPF_MOV 0xb0 /* eBPF only: mov reg to reg */
	303	+ BPF_ARSH 0xc0 /* eBPF only: sign extending shift right */
	304	+ BPF_END 0xd0 /* eBPF only: endianness conversion */
	305	+
	306	+If BPF_CLASS(code) == BPF_JMP or BPF_JMP32 [ in eBPF ], BPF_OP(code) is one of::
	307	+
	308	+ BPF_JA 0x00 /* BPF_JMP only */
	309	+ BPF_JEQ 0x10
	310	+ BPF_JGT 0x20
	311	+ BPF_JGE 0x30
	312	+ BPF_JSET 0x40
	313	+ BPF_JNE 0x50 /* eBPF only: jump != */
	314	+ BPF_JSGT 0x60 /* eBPF only: signed '>' */
	315	+ BPF_JSGE 0x70 /* eBPF only: signed '>=' */
	316	+ BPF_CALL 0x80 /* eBPF BPF_JMP only: function call */
	317	+ BPF_EXIT 0x90 /* eBPF BPF_JMP only: function return */
	318	+ BPF_JLT 0xa0 /* eBPF only: unsigned '<' */
	319	+ BPF_JLE 0xb0 /* eBPF only: unsigned '<=' */
	320	+ BPF_JSLT 0xc0 /* eBPF only: signed '<' */
	321	+ BPF_JSLE 0xd0 /* eBPF only: signed '<=' */
	322	+
	323	+So BPF_ADD \| BPF_X \| BPF_ALU means 32-bit addition in both classic BPF
	324	+and eBPF. There are only two registers in classic BPF, so it means A += X.
	325	+In eBPF it means dst_reg = (u32) dst_reg + (u32) src_reg; similarly,
	326	+BPF_XOR \| BPF_K \| BPF_ALU means A ^= imm32 in classic BPF and analogous
	327	+src_reg = (u32) src_reg ^ (u32) imm32 in eBPF.
	328	+
	329	+Classic BPF is using BPF_MISC class to represent A = X and X = A moves.
	330	+eBPF is using BPF_MOV \| BPF_X \| BPF_ALU code instead. Since there are no
	331	+BPF_MISC operations in eBPF, the class 7 is used as BPF_ALU64 to mean
	332	+exactly the same operations as BPF_ALU, but with 64-bit wide operands
	333	+instead. So BPF_ADD \| BPF_X \| BPF_ALU64 means 64-bit addition, i.e.:
	334	+dst_reg = dst_reg + src_reg
	335	+
	336	+Classic BPF wastes the whole BPF_RET class to represent a single ``ret``
	337	+operation. Classic BPF_RET \| BPF_K means copy imm32 into return register
	338	+and perform function exit. eBPF is modeled to match CPU, so BPF_JMP \| BPF_EXIT
	339	+in eBPF means function exit only. The eBPF program needs to store return
	340	+value into register R0 before doing a BPF_EXIT. Class 6 in eBPF is used as
	341	+BPF_JMP32 to mean exactly the same operations as BPF_JMP, but with 32-bit wide
	342	+operands for the comparisons instead.
	343	+
	344	+For load and store instructions the 8-bit 'code' field is divided as::
	345	+
	346	+ +--------+--------+-------------------+
	347	+ \| 3 bits \| 2 bits \| 3 bits \|
	348	+ \| mode \| size \| instruction class \|
	349	+ +--------+--------+-------------------+
	350	+ (MSB) (LSB)
	351	+
	352	+Size modifier is one of ...
	353	+
	354	+::
	355	+
	356	+ BPF_W 0x00 /* word */
	357	+ BPF_H 0x08 /* half word */
	358	+ BPF_B 0x10 /* byte */
	359	+ BPF_DW 0x18 /* eBPF only, double word */
	360	+
	361	+... which encodes size of load/store operation::
	362	+
	363	+ B - 1 byte
	364	+ H - 2 byte
	365	+ W - 4 byte
	366	+ DW - 8 byte (eBPF only)
	367	+
	368	+Mode modifier is one of::
	369	+
	370	+ BPF_IMM 0x00 /* used for 32-bit mov in classic BPF and 64-bit in eBPF */
	371	+ BPF_ABS 0x20
	372	+ BPF_IND 0x40
	373	+ BPF_MEM 0x60
	374	+ BPF_LEN 0x80 /* classic BPF only, reserved in eBPF */
	375	+ BPF_MSH 0xa0 /* classic BPF only, reserved in eBPF */
	376	+ BPF_ATOMIC 0xc0 /* eBPF only, atomic operations */

Documentation/bpf/index.rst+1−0

@@ -21,6 +21,7 @@ that goes into great technical depth about the BPF Architecture. · 20 unmodified lines
21	21	helpers
22	22	programs
23	23	maps
	24	+ classic_vs_extended.rst
24	25	bpf_licensing
25	26	test_debug
26	27	other

Documentation/bpf/instruction-set.rst+150−364

@@ -3,296 +3,68 @@ · 2 unmodified lines
3	3	eBPF Instruction Set
4	4	====================
5	5
6		−eBPF is designed to be JITed with one to one mapping, which can also open up
7		−the possibility for GCC/LLVM compilers to generate optimized eBPF code through
8		−an eBPF backend that performs almost as fast as natively compiled code.
9		−
10		−Some core changes of the eBPF format from classic BPF:
11		−
12		−- Number of registers increase from 2 to 10:
13		−
14		− The old format had two registers A and X, and a hidden frame pointer. The
15		− new layout extends this to be 10 internal registers and a read-only frame
16		− pointer. Since 64-bit CPUs are passing arguments to functions via registers
17		− the number of args from eBPF program to in-kernel function is restricted
18		− to 5 and one register is used to accept return value from an in-kernel
19		− function. Natively, x86_64 passes first 6 arguments in registers, aarch64/
20		− sparcv9/mips64 have 7 - 8 registers for arguments; x86_64 has 6 callee saved
21		− registers, and aarch64/sparcv9/mips64 have 11 or more callee saved registers.
22		−
23		− Therefore, eBPF calling convention is defined as:
24		−
25		− * R0 - return value from in-kernel function, and exit value for eBPF program
26		− * R1 - R5 - arguments from eBPF program to in-kernel function
27		− * R6 - R9 - callee saved registers that in-kernel function will preserve
28		− * R10 - read-only frame pointer to access stack
29		−
30		− Thus, all eBPF registers map one to one to HW registers on x86_64, aarch64,
31		− etc, and eBPF calling convention maps directly to ABIs used by the kernel on
32		− 64-bit architectures.
33		−
34		− On 32-bit architectures JIT may map programs that use only 32-bit arithmetic
35		− and may let more complex programs to be interpreted.
36		−
37		− R0 - R5 are scratch registers and eBPF program needs spill/fill them if
38		− necessary across calls. Note that there is only one eBPF program (== one
39		− eBPF main routine) and it cannot call other eBPF functions, it can only
40		− call predefined in-kernel functions, though.
41		−
42		−- Register width increases from 32-bit to 64-bit:
43		−
44		− Still, the semantics of the original 32-bit ALU operations are preserved
45		− via 32-bit subregisters. All eBPF registers are 64-bit with 32-bit lower
46		− subregisters that zero-extend into 64-bit if they are being written to.
47		− That behavior maps directly to x86_64 and arm64 subregister definition, but
48		− makes other JITs more difficult.
49		−
50		− 32-bit architectures run 64-bit eBPF programs via interpreter.
51		− Their JITs may convert BPF programs that only use 32-bit subregisters into
52		− native instruction set and let the rest being interpreted.
53		−
54		− Operation is 64-bit, because on 64-bit architectures, pointers are also
55		− 64-bit wide, and we want to pass 64-bit values in/out of kernel functions,
56		− so 32-bit eBPF registers would otherwise require to define register-pair
57		− ABI, thus, there won't be able to use a direct eBPF register to HW register
58		− mapping and JIT would need to do combine/split/move operations for every
59		− register in and out of the function, which is complex, bug prone and slow.
60		− Another reason is the use of atomic 64-bit counters.
61		−
62		−- Conditional jt/jf targets replaced with jt/fall-through:
63		−
64		− While the original design has constructs such as ``if (cond) jump_true;
65		− else jump_false;``, they are being replaced into alternative constructs like
66		− ``if (cond) jump_true; /* else fall-through */``.
67		−
68		−- Introduces bpf_call insn and register passing convention for zero overhead
69		− calls from/to other kernel functions:
70		−
71		− Before an in-kernel function call, the eBPF program needs to
72		− place function arguments into R1 to R5 registers to satisfy calling
73		− convention, then the interpreter will take them from registers and pass
74		− to in-kernel function. If R1 - R5 registers are mapped to CPU registers
75		− that are used for argument passing on given architecture, the JIT compiler
76		− doesn't need to emit extra moves. Function arguments will be in the correct
77		− registers and BPF_CALL instruction will be JITed as single 'call' HW
78		− instruction. This calling convention was picked to cover common call
79		− situations without performance penalty.
80		−
81		− After an in-kernel function call, R1 - R5 are reset to unreadable and R0 has
82		− a return value of the function. Since R6 - R9 are callee saved, their state
83		− is preserved across the call.
84		−
85		− For example, consider three C functions::
86		−
87		− u64 f1() { return (*_f2)(1); }
88		− u64 f2(u64 a) { return f3(a + 1, a); }
89		− u64 f3(u64 a, u64 b) { return a - b; }
90		−
91		− GCC can compile f1, f3 into x86_64::
92		−
93		− f1:
94		− movl $1, %edi
95		− movq _f2(%rip), %rax
96		− jmp *%rax
97		− f3:
98		− movq %rdi, %rax
99		− subq %rsi, %rax
100		− ret
101		−
102		− Function f2 in eBPF may look like::
103		−
104		− f2:
105		− bpf_mov R2, R1
106		− bpf_add R1, 1
107		− bpf_call f3
108		− bpf_exit
109		−
110		− If f2 is JITed and the pointer stored to ``_f2``. The calls f1 -> f2 -> f3 and
111		− returns will be seamless. Without JIT, __bpf_prog_run() interpreter needs to
112		− be used to call into f2.
113		−
114		− For practical reasons all eBPF programs have only one argument 'ctx' which is
115		− already placed into R1 (e.g. on __bpf_prog_run() startup) and the programs
116		− can call kernel functions with up to 5 arguments. Calls with 6 or more arguments
117		− are currently not supported, but these restrictions can be lifted if necessary
118		− in the future.
119		−
120		− On 64-bit architectures all register map to HW registers one to one. For
121		− example, x86_64 JIT compiler can map them as ...
122		−
123		− ::
124		−
125		− R0 - rax
126		− R1 - rdi
127		− R2 - rsi
128		− R3 - rdx
129		− R4 - rcx
130		− R5 - r8
131		− R6 - rbx
132		− R7 - r13
133		− R8 - r14
134		− R9 - r15
135		− R10 - rbp
136		−
137		− ... since x86_64 ABI mandates rdi, rsi, rdx, rcx, r8, r9 for argument passing
138		− and rbx, r12 - r15 are callee saved.
139		−
140		− Then the following eBPF pseudo-program::
141		−
142		− bpf_mov R6, R1 /* save ctx */
143		− bpf_mov R2, 2
144		− bpf_mov R3, 3
145		− bpf_mov R4, 4
146		− bpf_mov R5, 5
147		− bpf_call foo
148		− bpf_mov R7, R0 /* save foo() return value */
149		− bpf_mov R1, R6 /* restore ctx for next call */
150		− bpf_mov R2, 6
151		− bpf_mov R3, 7
152		− bpf_mov R4, 8
153		− bpf_mov R5, 9
154		− bpf_call bar
155		− bpf_add R0, R7
156		− bpf_exit
157		−
158		− After JIT to x86_64 may look like::
159		−
160		− push %rbp
161		− mov %rsp,%rbp
162		− sub $0x228,%rsp
163		− mov %rbx,-0x228(%rbp)
164		− mov %r13,-0x220(%rbp)
165		− mov %rdi,%rbx
166		− mov $0x2,%esi
167		− mov $0x3,%edx
168		− mov $0x4,%ecx
169		− mov $0x5,%r8d
170		− callq foo
171		− mov %rax,%r13
172		− mov %rbx,%rdi
173		− mov $0x6,%esi
174		− mov $0x7,%edx
175		− mov $0x8,%ecx
176		− mov $0x9,%r8d
177		− callq bar
178		− add %r13,%rax
179		− mov -0x228(%rbp),%rbx
180		− mov -0x220(%rbp),%r13
181		− leaveq
182		− retq
183		−
184		− Which is in this example equivalent in C to::
185		−
186		− u64 bpf_filter(u64 ctx)
187		− {
188		− return foo(ctx, 2, 3, 4, 5) + bar(ctx, 6, 7, 8, 9);
189		− }
190		−
191		− In-kernel functions foo() and bar() with prototype: u64 (*)(u64 arg1, u64
192		− arg2, u64 arg3, u64 arg4, u64 arg5); will receive arguments in proper
193		− registers and place their return value into ``%rax`` which is R0 in eBPF.
194		− Prologue and epilogue are emitted by JIT and are implicit in the
195		− interpreter. R0-R5 are scratch registers, so eBPF program needs to preserve
196		− them across the calls as defined by calling convention.
197		−
198		− For example the following program is invalid::
199		−
200		− bpf_mov R1, 1
201		− bpf_call foo
202		− bpf_mov R0, R1
203		− bpf_exit
204		−
205		− After the call the registers R1-R5 contain junk values and cannot be read.
206		− An in-kernel `eBPF verifier`_ is used to validate eBPF programs.
207		−
208		−Also in the new design, eBPF is limited to 4096 insns, which means that any
209		−program will terminate quickly and will only call a fixed number of kernel
210		−functions. Original BPF and eBPF are two operand instructions,
211		−which helps to do one-to-one mapping between eBPF insn and x86 insn during JIT.
212		−
213		−The input context pointer for invoking the interpreter function is generic,
214		−its content is defined by a specific use case. For seccomp register R1 points
215		−to seccomp_data, for converted BPF filters R1 points to a skb.
216		−
217		−A program, that is translated internally consists of the following elements::
218		−
219		− op:16, jt:8, jf:8, k:32 ==> op:8, dst_reg:4, src_reg:4, off:16, imm:32
220		−
221		−So far 87 eBPF instructions were implemented. 8-bit 'op' opcode field
222		−has room for new instructions. Some of them may use 16/24/32 byte encoding. New
223		−instructions must be multiple of 8 bytes to preserve backward compatibility.
224		−
225		−eBPF is a general purpose RISC instruction set. Not every register and
226		−every instruction are used during translation from original BPF to eBPF.
227		−For example, socket filters are not using ``exclusive add`` instruction, but
228		−tracing filters may do to maintain counters of events, for example. Register R9
229		−is not used by socket filters either, but more complex filters may be running
230		−out of registers and would have to resort to spill/fill to stack.
231		−
232		−eBPF can be used as a generic assembler for last step performance
233		−optimizations, socket filters and seccomp are using it as assembler. Tracing
234		−filters may use it as assembler to generate code from kernel. In kernel usage
235		−may not be bounded by security considerations, since generated eBPF code
236		−may be optimizing internal code path and not being exposed to the user space.
237		−Safety of eBPF can come from the `eBPF verifier`_. In such use cases as
238		−described, it may be used as safe instruction set.
239		−
240		−Just like the original BPF, eBPF runs within a controlled environment,
241		−is deterministic and the kernel can easily prove that. The safety of the program
242		−can be determined in two steps: first step does depth-first-search to disallow
243		−loops and other CFG validation; second step starts from the first insn and
244		−descends all possible paths. It simulates execution of every insn and observes
245		−the state change of registers and stack.
246		−
247		−eBPF opcode encoding
248		−====================
	6	+Registers and calling convention
	7	+================================
	8	+
	9	+eBPF has 10 general purpose registers and a read-only frame pointer register,
	10	+all of which are 64-bits wide.
249	11
250		−eBPF is reusing most of the opcode encoding from classic to simplify conversion
251		−of classic BPF to eBPF. For arithmetic and jump instructions the 8-bit 'code'
252		−field is divided into three parts::
	12	+The eBPF calling convention is defined as:
253	13
254		− +----------------+--------+--------------------+
255		− \| 4 bits \| 1 bit \| 3 bits \|
256		− \| operation code \| source \| instruction class \|
257		− +----------------+--------+--------------------+
258		− (MSB) (LSB)
	14	+ * R0: return value from function calls, and exit value for eBPF programs
	15	+ * R1 - R5: arguments for function calls
	16	+ * R6 - R9: callee saved registers that function calls will preserve
	17	+ * R10: read-only frame pointer to access stack
259	18
260		−Three LSB bits store instruction class which is one of:
	19	+R0 - R5 are scratch registers and eBPF programs needs to spill/fill them if
	20	+necessary across calls.
261	21
262		− =================== ===============
263		− Classic BPF classes eBPF classes
264		− =================== ===============
265		− BPF_LD 0x00 BPF_LD 0x00
266		− BPF_LDX 0x01 BPF_LDX 0x01
267		− BPF_ST 0x02 BPF_ST 0x02
268		− BPF_STX 0x03 BPF_STX 0x03
269		− BPF_ALU 0x04 BPF_ALU 0x04
270		− BPF_JMP 0x05 BPF_JMP 0x05
271		− BPF_RET 0x06 BPF_JMP32 0x06
272		− BPF_MISC 0x07 BPF_ALU64 0x07
273		− =================== ===============
	22	+Instruction classes
	23	+===================
274	24
275		−When BPF_CLASS(code) == BPF_ALU or BPF_JMP, 4th bit encodes source operand ...
	25	+The three LSB bits of the 'opcode' field store the instruction class:
276	26
277		− ::
	27	+ ========= =====
	28	+ class value
	29	+ ========= =====
	30	+ BPF_LD 0x00
	31	+ BPF_LDX 0x01
	32	+ BPF_ST 0x02
	33	+ BPF_STX 0x03
	34	+ BPF_ALU 0x04
	35	+ BPF_JMP 0x05
	36	+ BPF_JMP32 0x06
	37	+ BPF_ALU64 0x07
	38	+ ========= =====
278	39
279		− BPF_K 0x00
280		− BPF_X 0x08
	40	+Arithmetic and jump instructions
	41	+================================
281	42
282		− * in classic BPF, this means::
	43	+For arithmetic and jump instructions (BPF_ALU, BPF_ALU64, BPF_JMP and
	44	+BPF_JMP32), the 8-bit 'opcode' field is divided into three parts:
283	45
284		− BPF_SRC(code) == BPF_X - use register X as source operand
285		− BPF_SRC(code) == BPF_K - use 32-bit immediate as source operand
	46	+ ============== ====== =================
	47	+ 4 bits (MSB) 1 bit 3 bits (LSB)
	48	+ ============== ====== =================
	49	+ operation code source instruction class
	50	+ ============== ====== =================
286	51
287		− * in eBPF, this means::
	52	+The 4th bit encodes the source operand:
288	53
289		− BPF_SRC(code) == BPF_X - use 'src_reg' register as source operand
290		− BPF_SRC(code) == BPF_K - use 32-bit immediate as source operand
	54	+ ====== ===== ========================================
	55	+ source value description
	56	+ ====== ===== ========================================
	57	+ BPF_K 0x00 use 32-bit immediate as source operand
	58	+ BPF_X 0x08 use 'src_reg' register as source operand
	59	+ ====== ===== ========================================
291	60
292		−... and four MSB bits store operation code.
	61	+The four MSB bits store the operation code.
293	62
294		−If BPF_CLASS(code) == BPF_ALU or BPF_ALU64 [ in eBPF ], BPF_OP(code) is one of::
	63	+For class BPF_ALU or BPF_ALU64:
295	64
	65	+ ======== ===== =========================
	66	+ code value description
	67	+ ======== ===== =========================
296	68	BPF_ADD 0x00
297	69	BPF_SUB 0x10
298	70	BPF_MUL 0x20
@@ -304,116 +76,105 @@ If BPF_CLASS(code) == BPF_ALU or BPF_ALU64 [ in eBPF ], BPF_OP(code) is one of:: · 5 unmodified lines
304	76	BPF_NEG 0x80
305	77	BPF_MOD 0x90
306	78	BPF_XOR 0xa0
307		− BPF_MOV 0xb0 /* eBPF only: mov reg to reg */
308		− BPF_ARSH 0xc0 /* eBPF only: sign extending shift right */
309		− BPF_END 0xd0 /* eBPF only: endianness conversion */
	79	+ BPF_MOV 0xb0 mov reg to reg
	80	+ BPF_ARSH 0xc0 sign extending shift right
	81	+ BPF_END 0xd0 endianness conversion
	82	+ ======== ===== =========================
310	83
311		−If BPF_CLASS(code) == BPF_JMP or BPF_JMP32 [ in eBPF ], BPF_OP(code) is one of::
	84	+For class BPF_JMP or BPF_JMP32:
312	85
313		− BPF_JA 0x00 /* BPF_JMP only */
	86	+ ======== ===== =========================
	87	+ code value description
	88	+ ======== ===== =========================
	89	+ BPF_JA 0x00 BPF_JMP only
314	90	BPF_JEQ 0x10
315	91	BPF_JGT 0x20
316	92	BPF_JGE 0x30
317	93	BPF_JSET 0x40
318		− BPF_JNE 0x50 /* eBPF only: jump != */
319		− BPF_JSGT 0x60 /* eBPF only: signed '>' */
320		− BPF_JSGE 0x70 /* eBPF only: signed '>=' */
321		− BPF_CALL 0x80 /* eBPF BPF_JMP only: function call */
322		− BPF_EXIT 0x90 /* eBPF BPF_JMP only: function return */
323		− BPF_JLT 0xa0 /* eBPF only: unsigned '<' */
324		− BPF_JLE 0xb0 /* eBPF only: unsigned '<=' */
325		− BPF_JSLT 0xc0 /* eBPF only: signed '<' */
326		− BPF_JSLE 0xd0 /* eBPF only: signed '<=' */
327		−
328		−So BPF_ADD \| BPF_X \| BPF_ALU means 32-bit addition in both classic BPF
329		−and eBPF. There are only two registers in classic BPF, so it means A += X.
330		−In eBPF it means dst_reg = (u32) dst_reg + (u32) src_reg; similarly,
331		−BPF_XOR \| BPF_K \| BPF_ALU means A ^= imm32 in classic BPF and analogous
332		−src_reg = (u32) src_reg ^ (u32) imm32 in eBPF.
333		−
334		−Classic BPF is using BPF_MISC class to represent A = X and X = A moves.
335		−eBPF is using BPF_MOV \| BPF_X \| BPF_ALU code instead. Since there are no
336		−BPF_MISC operations in eBPF, the class 7 is used as BPF_ALU64 to mean
337		−exactly the same operations as BPF_ALU, but with 64-bit wide operands
338		−instead. So BPF_ADD \| BPF_X \| BPF_ALU64 means 64-bit addition, i.e.:
339		−dst_reg = dst_reg + src_reg
340		−
341		−Classic BPF wastes the whole BPF_RET class to represent a single ``ret``
342		−operation. Classic BPF_RET \| BPF_K means copy imm32 into return register
343		−and perform function exit. eBPF is modeled to match CPU, so BPF_JMP \| BPF_EXIT
344		−in eBPF means function exit only. The eBPF program needs to store return
345		−value into register R0 before doing a BPF_EXIT. Class 6 in eBPF is used as
346		−BPF_JMP32 to mean exactly the same operations as BPF_JMP, but with 32-bit wide
347		−operands for the comparisons instead.
	94	+ BPF_JNE 0x50 jump '!='
	95	+ BPF_JSGT 0x60 signed '>'
	96	+ BPF_JSGE 0x70 signed '>='
	97	+ BPF_CALL 0x80 function call
	98	+ BPF_EXIT 0x90 function return
	99	+ BPF_JLT 0xa0 unsigned '<'
	100	+ BPF_JLE 0xb0 unsigned '<='
	101	+ BPF_JSLT 0xc0 signed '<'
	102	+ BPF_JSLE 0xd0 signed '<='
	103	+ ======== ===== =========================
348	104
349		−For load and store instructions the 8-bit 'code' field is divided as::
	105	+So BPF_ADD \| BPF_X \| BPF_ALU means::
350	106
351		− +--------+--------+-------------------+
352		− \| 3 bits \| 2 bits \| 3 bits \|
353		− \| mode \| size \| instruction class \|
354		− +--------+--------+-------------------+
355		− (MSB) (LSB)
	107	+ dst_reg = (u32) dst_reg + (u32) src_reg;
356	108
357		−Size modifier is one of ...
	109	+Similarly, BPF_XOR \| BPF_K \| BPF_ALU means::
358	110
359		−::
	111	+ src_reg = (u32) src_reg ^ (u32) imm32
360	112
361		− BPF_W 0x00 /* word */
362		− BPF_H 0x08 /* half word */
363		− BPF_B 0x10 /* byte */
364		− BPF_DW 0x18 /* eBPF only, double word */
	113	+eBPF is using BPF_MOV \| BPF_X \| BPF_ALU to represent A = B moves. BPF_ALU64
	114	+is used to mean exactly the same operations as BPF_ALU, but with 64-bit wide
	115	+operands instead. So BPF_ADD \| BPF_X \| BPF_ALU64 means 64-bit addition, i.e.::
365	116
366		−... which encodes size of load/store operation::
	117	+ dst_reg = dst_reg + src_reg
367	118
368		− B - 1 byte
369		− H - 2 byte
370		− W - 4 byte
371		− DW - 8 byte (eBPF only)
	119	+BPF_JMP \| BPF_EXIT means function exit only. The eBPF program needs to store
	120	+the return value into register R0 before doing a BPF_EXIT. Class 6 is used as
	121	+BPF_JMP32 to mean exactly the same operations as BPF_JMP, but with 32-bit wide
	122	+operands for the comparisons instead.
372	123
373		−Mode modifier is one of::
374	124
375		− BPF_IMM 0x00 /* used for 32-bit mov in classic BPF and 64-bit in eBPF */
376		− BPF_ABS 0x20
377		− BPF_IND 0x40
378		− BPF_MEM 0x60
379		− BPF_LEN 0x80 /* classic BPF only, reserved in eBPF */
380		− BPF_MSH 0xa0 /* classic BPF only, reserved in eBPF */
381		− BPF_ATOMIC 0xc0 /* eBPF only, atomic operations */
	125	+Load and store instructions
	126	+===========================
382	127
383		−eBPF has two non-generic instructions: (BPF_ABS \| <size> \| BPF_LD) and
384		−(BPF_IND \| <size> \| BPF_LD) which are used to access packet data.
	128	+For load and store instructions (BPF_LD, BPF_LDX, BPF_ST and BPF_STX), the
	129	+8-bit 'opcode' field is divided as:
385	130
386		−They had to be carried over from classic to have strong performance of
387		−socket filters running in eBPF interpreter. These instructions can only
388		−be used when interpreter context is a pointer to ``struct sk_buff`` and
389		−have seven implicit operands. Register R6 is an implicit input that must
390		−contain pointer to sk_buff. Register R0 is an implicit output which contains
391		−the data fetched from the packet. Registers R1-R5 are scratch registers
392		−and must not be used to store the data across BPF_ABS \| BPF_LD or
393		−BPF_IND \| BPF_LD instructions.
	131	+ ============ ====== =================
	132	+ 3 bits (MSB) 2 bits 3 bits (LSB)
	133	+ ============ ====== =================
	134	+ mode size instruction class
	135	+ ============ ====== =================
394	136
395		−These instructions have implicit program exit condition as well. When
396		−eBPF program is trying to access the data beyond the packet boundary,
397		−the interpreter will abort the execution of the program. JIT compilers
398		−therefore must preserve this property. src_reg and imm32 fields are
399		−explicit inputs to these instructions.
	137	+The size modifier is one of:
	138	+
	139	+ ============= ===== =====================
	140	+ size modifier value description
	141	+ ============= ===== =====================
	142	+ BPF_W 0x00 word (4 bytes)
	143	+ BPF_H 0x08 half word (2 bytes)
	144	+ BPF_B 0x10 byte
	145	+ BPF_DW 0x18 double word (8 bytes)
	146	+ ============= ===== =====================
	147	+
	148	+The mode modifier is one of:
	149	+
	150	+ ============= ===== =====================
	151	+ mode modifier value description
	152	+ ============= ===== =====================
	153	+ BPF_IMM 0x00 used for 64-bit mov
	154	+ BPF_ABS 0x20
	155	+ BPF_IND 0x40
	156	+ BPF_MEM 0x60
	157	+ BPF_ATOMIC 0xc0 atomic operations
	158	+ ============= ===== =====================
400	159
401		−For example::
	160	+BPF_MEM \| <size> \| BPF_STX means::
402	161
403		− BPF_IND \| BPF_W \| BPF_LD means:
	162	+ (size ) (dst_reg + off) = src_reg
404	163
405		− R0 = ntohl((u32 ) (((struct sk_buff *) R6)->data + src_reg + imm32))
406		− and R1 - R5 were scratched.
	164	+BPF_MEM \| <size> \| BPF_ST means::
407	165
408		−Unlike classic BPF instruction set, eBPF has generic load/store operations::
	166	+ (size ) (dst_reg + off) = imm32
409	167
410		− BPF_MEM \| <size> \| BPF_STX: (size ) (dst_reg + off) = src_reg
411		− BPF_MEM \| <size> \| BPF_ST: (size ) (dst_reg + off) = imm32
412		− BPF_MEM \| <size> \| BPF_LDX: dst_reg = (size ) (src_reg + off)
	168	+BPF_MEM \| <size> \| BPF_LDX means::
	169	+
	170	+ dst_reg = (size ) (src_reg + off)
413	171
414	172	Where size is one of: BPF_B or BPF_H or BPF_W or BPF_DW.
415	173
416		−It also includes atomic operations, which use the immediate field for extra
	174	+Atomic operations
	175	+-----------------
	176	+
	177	+eBPF includes atomic operations, which use the immediate field for extra
417	178	encoding::
418	179
419	180	.imm = BPF_ADD, .code = BPF_ATOMIC \| BPF_W \| BPF_STX: lock xadd (u32 )(dst_reg + off16) += src_reg
@@ -457,11 +218,36 @@ You may encounter ``BPF_XADD`` - this is a legacy name for ``BPF_ATOMIC``, · 37 unmodified lines
457	218	referring to the exclusive-add operation encoded when the immediate field is
458	219	zero.
459	220
	221	+16-byte instructions
	222	+--------------------
	223	+
460	224	eBPF has one 16-byte instruction: ``BPF_LD \| BPF_DW \| BPF_IMM`` which consists
461	225	of two consecutive ``struct bpf_insn`` 8-byte blocks and interpreted as single
462	226	instruction that loads 64-bit immediate value into a dst_reg.
463		−Classic BPF has similar instruction: ``BPF_LD \| BPF_W \| BPF_IMM`` which loads
464		−32-bit immediate value into a register.
465	227
466		−.. Links:
467		−.. _eBPF verifier: verifiers.rst
	228	+Packet access instructions
	229	+--------------------------
	230	+
	231	+eBPF has two non-generic instructions: (BPF_ABS \| <size> \| BPF_LD) and
	232	+(BPF_IND \| <size> \| BPF_LD) which are used to access packet data.
	233	+
	234	+They had to be carried over from classic BPF to have strong performance of
	235	+socket filters running in eBPF interpreter. These instructions can only
	236	+be used when interpreter context is a pointer to ``struct sk_buff`` and
	237	+have seven implicit operands. Register R6 is an implicit input that must
	238	+contain pointer to sk_buff. Register R0 is an implicit output which contains
	239	+the data fetched from the packet. Registers R1-R5 are scratch registers
	240	+and must not be used to store the data across BPF_ABS \| BPF_LD or
	241	+BPF_IND \| BPF_LD instructions.
	242	+
	243	+These instructions have implicit program exit condition as well. When
	244	+eBPF program is trying to access the data beyond the packet boundary,
	245	+the interpreter will abort the execution of the program. JIT compilers
	246	+therefore must preserve this property. src_reg and imm32 fields are
	247	+explicit inputs to these instructions.
	248	+
	249	+For example, BPF_IND \| BPF_W \| BPF_LD means::
	250	+
	251	+ R0 = ntohl((u32 ) (((struct sk_buff *) R6)->data + src_reg + imm32))
	252	+
	253	+and R1 - R5 are clobbered.

arch/s390/mm/hugetlbpage.c+1−0

@@ -9,6 +9,7 @@ · 8 unmodified lines
9	9	#define KMSG_COMPONENT "hugetlb"
10	10	#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
11	11
	12	+#include <asm/pgalloc.h>
12	13	#include <linux/mm.h>
13	14	#include <linux/hugetlb.h>
14	15	#include <linux/mman.h>

arch/x86/net/bpf_jit_comp.c+41−14

@@ -1976,7 +1976,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image im, void image, void *i · 1975 unmodified lines
1976	1976	void *orig_call)
1977	1977	{
1978	1978	int ret, i, nr_args = m->nr_args;
1979		− int stack_size = nr_args * 8;
	1979	+ int regs_off, ip_off, args_off, stack_size = nr_args * 8;
1980	1980	struct bpf_tramp_progs *fentry = &tprogs[BPF_TRAMP_FENTRY];
1981	1981	struct bpf_tramp_progs *fexit = &tprogs[BPF_TRAMP_FEXIT];
1982	1982	struct bpf_tramp_progs *fmod_ret = &tprogs[BPF_TRAMP_MODIFY_RETURN];
@@ -1991,14 +1991,39 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image im, void image, void *i · 8 unmodified lines
1991	1991	if (!is_valid_bpf_tramp_flags(flags))
1992	1992	return -EINVAL;
1993	1993
	1994	+ /* Generated trampoline stack layout:
	1995	+ *
	1996	+ * RBP + 8 [ return address ]
	1997	+ * RBP + 0 [ RBP ]
	1998	+ *
	1999	+ * RBP - 8 [ return value ] BPF_TRAMP_F_CALL_ORIG or
	2000	+ * BPF_TRAMP_F_RET_FENTRY_RET flags
	2001	+ *
	2002	+ * [ reg_argN ] always
	2003	+ * [ ... ]
	2004	+ * RBP - regs_off [ reg_arg1 ] program's ctx pointer
	2005	+ *
	2006	+ * RBP - args_off [ args count ] always
	2007	+ *
	2008	+ * RBP - ip_off [ traced function ] BPF_TRAMP_F_IP_ARG flag
	2009	+ */
	2010	+
1994	2011	/* room for return value of orig_call or fentry prog */
1995	2012	save_ret = flags & (BPF_TRAMP_F_CALL_ORIG \| BPF_TRAMP_F_RET_FENTRY_RET);
1996	2013	if (save_ret)
1997	2014	stack_size += 8;
1998	2015
	2016	+ regs_off = stack_size;
	2017	+
	2018	+ /* args count */
	2019	+ stack_size += 8;
	2020	+ args_off = stack_size;
	2021	+
1999	2022	if (flags & BPF_TRAMP_F_IP_ARG)
2000	2023	stack_size += 8; /* room for IP address argument */
2001	2024
	2025	+ ip_off = stack_size;
	2026	+
2002	2027	if (flags & BPF_TRAMP_F_SKIP_FRAME)
2003	2028	/* skip patched call instruction and point orig_call to actual
2004	2029	* body of the kernel function.
@@ -2012,23 +2037,25 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image im, void image, void *i · 7 unmodified lines
2012	2037	EMIT4(0x48, 0x83, 0xEC, stack_size); /* sub rsp, stack_size */
2013	2038	EMIT1(0x53); /* push rbx */
2014	2039
	2040	+ /* Store number of arguments of the traced function:
	2041	+ * mov rax, nr_args
	2042	+ * mov QWORD PTR [rbp - args_off], rax
	2043	+ */
	2044	+ emit_mov_imm64(&prog, BPF_REG_0, 0, (u32) nr_args);
	2045	+ emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -args_off);
	2046	+
2015	2047	if (flags & BPF_TRAMP_F_IP_ARG) {
2016	2048	/* Store IP address of the traced function:
2017	2049	* mov rax, QWORD PTR [rbp + 8]
2018	2050	* sub rax, X86_PATCH_SIZE
2019		− * mov QWORD PTR [rbp - stack_size], rax
	2051	+ * mov QWORD PTR [rbp - ip_off], rax
2020	2052	*/
2021	2053	emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, 8);
2022	2054	EMIT4(0x48, 0x83, 0xe8, X86_PATCH_SIZE);
2023		− emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -stack_size);
2024		−
2025		− /* Continue with stack_size for regs storage, stack will
2026		− * be correctly restored with 'leave' instruction.
2027		− */
2028		− stack_size -= 8;
	2055	+ emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -ip_off);
2029	2056	}
2030	2057
2031		− save_regs(m, &prog, nr_args, stack_size);
	2058	+ save_regs(m, &prog, nr_args, regs_off);
2032	2059
2033	2060	if (flags & BPF_TRAMP_F_CALL_ORIG) {
2034	2061	/* arg1: mov rdi, im */
@@ -2040,7 +2067,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image im, void image, void *i · 5 unmodified lines
2040	2067	}
2041	2068
2042	2069	if (fentry->nr_progs)
2043		− if (invoke_bpf(m, &prog, fentry, stack_size,
	2070	+ if (invoke_bpf(m, &prog, fentry, regs_off,
2044	2071	flags & BPF_TRAMP_F_RET_FENTRY_RET))
2045	2072	return -EINVAL;
2046	2073
@@ -2050,7 +2077,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image im, void image, void *i · 3 unmodified lines
2050	2077	if (!branches)
2051	2078	return -ENOMEM;
2052	2079
2053		− if (invoke_bpf_mod_ret(m, &prog, fmod_ret, stack_size,
	2080	+ if (invoke_bpf_mod_ret(m, &prog, fmod_ret, regs_off,
2054	2081	branches)) {
2055	2082	ret = -EINVAL;
2056	2083	goto cleanup;
@@ -2058,7 +2085,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image im, void image, void *i · 1 unmodified line
2058	2085	}
2059	2086
2060	2087	if (flags & BPF_TRAMP_F_CALL_ORIG) {
2061		− restore_regs(m, &prog, nr_args, stack_size);
	2088	+ restore_regs(m, &prog, nr_args, regs_off);
2062	2089
2063	2090	/* call original function */
2064	2091	if (emit_call(&prog, orig_call, prog)) {
@@ -2088,13 +2115,13 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image im, void image, void *i · 23 unmodified lines
2088	2115	}
2089	2116
2090	2117	if (fexit->nr_progs)
2091		− if (invoke_bpf(m, &prog, fexit, stack_size, false)) {
	2118	+ if (invoke_bpf(m, &prog, fexit, regs_off, false)) {
2092	2119	ret = -EINVAL;
2093	2120	goto cleanup;
2094	2121	}
2095	2122
2096	2123	if (flags & BPF_TRAMP_F_RESTORE_REGS)
2097		− restore_regs(m, &prog, nr_args, stack_size);
	2124	+ restore_regs(m, &prog, nr_args, regs_off);
2098	2125
2099	2126	/* This needs to be done regardless. If there were fmod_ret programs,
2100	2127	* the return value is only updated on the stack and still needs to be

drivers/bluetooth/btqca.c+1−0

@@ -6,6 +6,7 @@ · 5 unmodified lines
6	6	*/
7	7	#include <linux/module.h>
8	8	#include <linux/firmware.h>
	9	+#include <linux/vmalloc.h>
9	10
10	11	#include <net/bluetooth/bluetooth.h>
11	12	#include <net/bluetooth/hci_core.h>

drivers/infiniband/core/cache.c+1−0

@@ -33,6 +33,7 @@ · 32 unmodified lines
33	33	* SOFTWARE.
34	34	*/
35	35
	36	+#include <linux/if_vlan.h>
36	37	#include <linux/module.h>
37	38	#include <linux/errno.h>
38	39	#include <linux/slab.h>

drivers/infiniband/hw/irdma/ctrl.c+2−0

@@ -1,5 +1,7 @@
1	1	// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
2	2	/* Copyright (c) 2015 - 2021 Intel Corporation */
	3	+#include <linux/etherdevice.h>
	4	+
3	5	#include "osdep.h"
4	6	#include "status.h"
5	7	#include "hmc.h"

drivers/infiniband/hw/irdma/uda.c+2−0

@@ -1,5 +1,7 @@
1	1	// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
2	2	/* Copyright (c) 2016 - 2021 Intel Corporation */
	3	+#include <linux/etherdevice.h>
	4	+
3	5	#include "osdep.h"
4	6	#include "status.h"
5	7	#include "hmc.h"

drivers/infiniband/hw/mlx5/doorbell.c+1−0

@@ -32,6 +32,7 @@ · 31 unmodified lines
32	32
33	33	#include <linux/kref.h>
34	34	#include <linux/slab.h>
	35	+#include <linux/sched/mm.h>
35	36	#include <rdma/ib_umem.h>
36	37
37	38	#include "mlx5_ib.h"

drivers/infiniband/hw/mlx5/qp.c+1−0

@@ -30,6 +30,7 @@ · 29 unmodified lines
30	30	* SOFTWARE.
31	31	*/
32	32
	33	+#include <linux/etherdevice.h>
33	34	#include <linux/module.h>
34	35	#include <rdma/ib_umem.h>
35	36	#include <rdma/ib_cache.h>

drivers/net/amt.c+1−0

@@ -11,6 +11,7 @@ · 10 unmodified lines
11	11	#include <linux/net.h>
12	12	#include <linux/igmp.h>
13	13	#include <linux/workqueue.h>
	14	+#include <net/sch_generic.h>
14	15	#include <net/net_namespace.h>
15	16	#include <net/ip.h>
16	17	#include <net/udp.h>

drivers/net/appletalk/ipddp.c+1−0

@@ -23,6 +23,7 @@ · 22 unmodified lines
23	23	* of the GNU General Public License, incorporated herein by reference.
24	24	*/
25	25
	26	+#include <linux/compat.h>
26	27	#include <linux/module.h>
27	28	#include <linux/kernel.h>
28	29	#include <linux/init.h>

drivers/net/bonding/bond_main.c+1−0

@@ -35,6 +35,7 @@ · 34 unmodified lines
35	35	#include <linux/module.h>
36	36	#include <linux/types.h>
37	37	#include <linux/fcntl.h>
	38	+#include <linux/filter.h>
38	39	#include <linux/interrupt.h>
39	40	#include <linux/ptrace.h>
40	41	#include <linux/ioport.h>

drivers/net/can/usb/peak_usb/pcan_usb.c+1−0

@@ -8,6 +8,7 @@ · 7 unmodified lines
8	8	*
9	9	* Many thanks to Klaus Hitschler <klaus.hitschler@gmx.de>
10	10	*/
	11	+#include <asm/unaligned.h>
11	12	#include <linux/netdevice.h>
12	13	#include <linux/usb.h>
13	14	#include <linux/module.h>

drivers/net/dsa/microchip/ksz8795.c+1−0

@@ -10,6 +10,7 @@ · 9 unmodified lines
10	10	#include <linux/delay.h>
11	11	#include <linux/export.h>
12	12	#include <linux/gpio.h>
	13	+#include <linux/if_vlan.h>
13	14	#include <linux/kernel.h>
14	15	#include <linux/module.h>
15	16	#include <linux/platform_data/microchip-ksz.h>

drivers/net/dsa/xrs700x/xrs700x.c+1−0

@@ -5,6 +5,7 @@ · 4 unmodified lines
5	5	*/
6	6
7	7	#include <net/dsa.h>
	8	+#include <linux/etherdevice.h>
8	9	#include <linux/if_bridge.h>
9	10	#include <linux/of_device.h>
10	11	#include <linux/netdev_features.h>

drivers/net/ethernet/amazon/ena/ena_netdev.c+1−1

@@ -434,7 +434,7 @@ static int ena_xdp_execute(struct ena_ring rx_ring, struct xdp_buff xdp) · 433 unmodified lines
434	434	xdp_stat = &rx_ring->rx_stats.xdp_pass;
435	435	break;
436	436	default:
437		− bpf_warn_invalid_xdp_action(verdict);
	437	+ bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, verdict);
438	438	xdp_stat = &rx_ring->rx_stats.xdp_invalid;
439	439	}
440	440

drivers/net/ethernet/amazon/ena/ena_netdev.h+1−0

@@ -14,6 +14,7 @@ · 13 unmodified lines
14	14	#include <linux/interrupt.h>
15	15	#include <linux/netdevice.h>
16	16	#include <linux/skbuff.h>
	17	+#include <uapi/linux/bpf.h>
17	18
18	19	#include "ena_com.h"
19	20	#include "ena_eth_com.h"

drivers/net/ethernet/broadcom/bnxt/bnxt_devlink.c+1−0

@@ -9,6 +9,7 @@ · 8 unmodified lines
9	9
10	10	#include <linux/pci.h>
11	11	#include <linux/netdevice.h>
	12	+#include <linux/vmalloc.h>
12	13	#include <net/devlink.h>
13	14	#include "bnxt_hsi.h"
14	15	#include "bnxt.h"

drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c+1−1

@@ -195,7 +195,7 @@ bool bnxt_rx_xdp(struct bnxt bp, struct bnxt_rx_ring_info rxr, u16 cons, · 194 unmodified lines
195	195	*event \|= BNXT_REDIRECT_EVENT;
196	196	break;
197	197	default:
198		− bpf_warn_invalid_xdp_action(act);
	198	+ bpf_warn_invalid_xdp_action(bp->dev, xdp_prog, act);
199	199	fallthrough;
200	200	case XDP_ABORTED:
201	201	trace_xdp_exception(bp->dev, xdp_prog, act);

drivers/net/ethernet/cavium/thunder/nicvf_main.c+1−1

@@ -590,7 +590,7 @@ static inline bool nicvf_xdp_rx(struct nicvf nic, struct bpf_prog prog, · 589 unmodified lines
590	590	nicvf_xdp_sq_append_pkt(nic, sq, (u64)xdp.data, dma_addr, len);
591	591	return true;
592	592	default:
593		− bpf_warn_invalid_xdp_action(action);
	593	+ bpf_warn_invalid_xdp_action(nic->netdev, prog, action);
594	594	fallthrough;
595	595	case XDP_ABORTED:
596	596	trace_xdp_exception(nic->netdev, prog, action);

drivers/net/ethernet/cavium/thunder/nicvf_queues.c+1−0

@@ -10,6 +10,7 @@ · 9 unmodified lines
10	10	#include <linux/iommu.h>
11	11	#include <net/ip.h>
12	12	#include <net/tso.h>
	13	+#include <uapi/linux/bpf.h>
13	14
14	15	#include "nic_reg.h"
15	16	#include "nic.h"

drivers/net/ethernet/freescale/dpaa/dpaa_eth.c+1−1

@@ -2623,7 +2623,7 @@ static u32 dpaa_run_xdp(struct dpaa_priv priv, struct qm_fd fd, void *vaddr, · 2622 unmodified lines
2623	2623	}
2624	2624	break;
2625	2625	default:
2626		− bpf_warn_invalid_xdp_action(xdp_act);
	2626	+ bpf_warn_invalid_xdp_action(priv->net_dev, xdp_prog, xdp_act);
2627	2627	fallthrough;
2628	2628	case XDP_ABORTED:
2629	2629	trace_xdp_exception(priv->net_dev, xdp_prog, xdp_act);

drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c+1−1

@@ -374,7 +374,7 @@ static u32 dpaa2_eth_run_xdp(struct dpaa2_eth_priv *priv, · 373 unmodified lines
374	374	dpaa2_eth_xdp_enqueue(priv, ch, fd, vaddr, rx_fq->flowid);
375	375	break;
376	376	default:
377		− bpf_warn_invalid_xdp_action(xdp_act);
	377	+ bpf_warn_invalid_xdp_action(priv->net_dev, xdp_prog, xdp_act);
378	378	fallthrough;
379	379	case XDP_ABORTED:
380	380	trace_xdp_exception(priv->net_dev, xdp_prog, xdp_act);

drivers/net/ethernet/freescale/enetc/enetc.c+1−1

@@ -1547,7 +1547,7 @@ static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring, · 1546 unmodified lines
1547	1547
1548	1548	switch (xdp_act) {
1549	1549	default:
1550		− bpf_warn_invalid_xdp_action(xdp_act);
	1550	+ bpf_warn_invalid_xdp_action(rx_ring->ndev, prog, xdp_act);
1551	1551	fallthrough;
1552	1552	case XDP_ABORTED:
1553	1553	trace_xdp_exception(rx_ring->ndev, prog, xdp_act);

drivers/net/ethernet/huawei/hinic/hinic_tx.c+1−0

@@ -4,6 +4,7 @@ · 3 unmodified lines
4	4	* Copyright(c) 2017 Huawei Technologies Co., Ltd
5	5	*/
6	6
	7	+#include <linux/if_vlan.h>
7	8	#include <linux/kernel.h>
8	9	#include <linux/netdevice.h>
9	10	#include <linux/u64_stats_sync.h>

drivers/net/ethernet/intel/i40e/i40e_txrx.c+1−1

@@ -2322,7 +2322,7 @@ static int i40e_run_xdp(struct i40e_ring rx_ring, struct xdp_buff xdp) · 2321 unmodified lines
2322	2322	result = I40E_XDP_REDIR;
2323	2323	break;
2324	2324	default:
2325		− bpf_warn_invalid_xdp_action(act);
	2325	+ bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act);
2326	2326	fallthrough;
2327	2327	case XDP_ABORTED:
2328	2328	out_failure:

drivers/net/ethernet/intel/i40e/i40e_xsk.c+1−1

@@ -176,7 +176,7 @@ static int i40e_run_xdp_zc(struct i40e_ring rx_ring, struct xdp_buff xdp) · 175 unmodified lines
176	176	goto out_failure;
177	177	break;
178	178	default:
179		− bpf_warn_invalid_xdp_action(act);
	179	+ bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act);
180	180	fallthrough;
181	181	case XDP_ABORTED:
182	182	out_failure:

drivers/net/ethernet/intel/i40e/i40e_xsk.h+0−1

@@ -22,7 +22,6 @@ · 21 unmodified lines
22	22
23	23	struct i40e_vsi;
24	24	struct xsk_buff_pool;
25		−struct zero_copy_allocator;
26	25
27	26	int i40e_queue_pair_disable(struct i40e_vsi *vsi, int queue_pair);
28	27	int i40e_queue_pair_enable(struct i40e_vsi *vsi, int queue_pair);

drivers/net/ethernet/intel/ice/ice_devlink.c+2−0

@@ -1,6 +1,8 @@
1	1	// SPDX-License-Identifier: GPL-2.0
2	2	/* Copyright (c) 2020, Intel Corporation. */
3	3
	4	+#include <linux/vmalloc.h>
	5	+
4	6	#include "ice.h"
5	7	#include "ice_lib.h"
6	8	#include "ice_devlink.h"

drivers/net/ethernet/intel/ice/ice_txrx.c+1−1

@@ -576,7 +576,7 @@ ice_run_xdp(struct ice_rx_ring rx_ring, struct xdp_buff xdp, · 575 unmodified lines
576	576	goto out_failure;
577	577	return ICE_XDP_REDIR;
578	578	default:
579		− bpf_warn_invalid_xdp_action(act);
	579	+ bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act);
580	580	fallthrough;
581	581	case XDP_ABORTED:
582	582	out_failure:

drivers/net/ethernet/intel/ice/ice_txrx_lib.c+2−0

@@ -1,6 +1,8 @@
1	1	// SPDX-License-Identifier: GPL-2.0
2	2	/* Copyright (c) 2019, Intel Corporation. */
3	3
	4	+#include <linux/filter.h>
	5	+
4	6	#include "ice_txrx_lib.h"
5	7	#include "ice_eswitch.h"
6	8	#include "ice_lib.h"

drivers/net/ethernet/intel/ice/ice_xsk.c+1−1

@@ -481,7 +481,7 @@ ice_run_xdp_zc(struct ice_rx_ring rx_ring, struct xdp_buff xdp, · 480 unmodified lines
481	481	goto out_failure;
482	482	break;
483	483	default:
484		− bpf_warn_invalid_xdp_action(act);
	484	+ bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act);
485	485	fallthrough;
486	486	case XDP_ABORTED:
487	487	out_failure:

drivers/net/ethernet/intel/igb/igb_main.c+1−1

@@ -8500,7 +8500,7 @@ static struct sk_buff igb_run_xdp(struct igb_adapter adapter, · 8499 unmodified lines
8500	8500	result = IGB_XDP_REDIR;
8501	8501	break;
8502	8502	default:
8503		− bpf_warn_invalid_xdp_action(act);
	8503	+ bpf_warn_invalid_xdp_action(adapter->netdev, xdp_prog, act);
8504	8504	fallthrough;
8505	8505	case XDP_ABORTED:
8506	8506	out_failure:

drivers/net/ethernet/intel/igc/igc_main.c+1−1

@@ -2241,7 +2241,7 @@ static int __igc_xdp_run_prog(struct igc_adapter *adapter, · 2240 unmodified lines
2241	2241	return IGC_XDP_REDIRECT;
2242	2242	break;
2243	2243	default:
2244		− bpf_warn_invalid_xdp_action(act);
	2244	+ bpf_warn_invalid_xdp_action(adapter->netdev, prog, act);
2245	2245	fallthrough;
2246	2246	case XDP_ABORTED:
2247	2247	out_failure:

drivers/net/ethernet/intel/igc/igc_xdp.c+1−0

@@ -1,6 +1,7 @@
1	1	// SPDX-License-Identifier: GPL-2.0
2	2	/* Copyright (c) 2020, Intel Corporation. */
3	3
	4	+#include <linux/if_vlan.h>
4	5	#include <net/xdp_sock_drv.h>
5	6
6	7	#include "igc.h"

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c+1−1

@@ -2235,7 +2235,7 @@ static struct sk_buff ixgbe_run_xdp(struct ixgbe_adapter adapter, · 2234 unmodified lines
2235	2235	result = IXGBE_XDP_REDIR;
2236	2236	break;
2237	2237	default:
2238		− bpf_warn_invalid_xdp_action(act);
	2238	+ bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act);
2239	2239	fallthrough;
2240	2240	case XDP_ABORTED:
2241	2241	out_failure:

drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h+0−2

@@ -35,8 +35,6 @@ int ixgbe_xsk_pool_setup(struct ixgbe_adapter *adapter, · 34 unmodified lines
35	35	struct xsk_buff_pool *pool,
36	36	u16 qid);
37	37
38		−void ixgbe_zca_free(struct zero_copy_allocator *alloc, unsigned long handle);
39		−
40	38	bool ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 cleaned_count);
41	39	int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector,
42	40	struct ixgbe_ring *rx_ring,

drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c+1−1

@@ -131,7 +131,7 @@ static int ixgbe_run_xdp_zc(struct ixgbe_adapter *adapter, · 130 unmodified lines
131	131	goto out_failure;
132	132	break;
133	133	default:
134		− bpf_warn_invalid_xdp_action(act);
	134	+ bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act);
135	135	fallthrough;
136	136	case XDP_ABORTED:
137	137	out_failure:

drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c+1−1

@@ -1070,7 +1070,7 @@ static struct sk_buff ixgbevf_run_xdp(struct ixgbevf_adapter adapter, · 1069 unmodified lines
1070	1070	goto out_failure;
1071	1071	break;
1072	1072	default:
1073		− bpf_warn_invalid_xdp_action(act);
	1073	+ bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act);
1074	1074	fallthrough;
1075	1075	case XDP_ABORTED:
1076	1076	out_failure:

drivers/net/ethernet/marvell/mvneta.c+1−1

@@ -2240,7 +2240,7 @@ mvneta_run_xdp(struct mvneta_port pp, struct mvneta_rx_queue rxq, · 2239 unmodified lines
2240	2240	mvneta_xdp_put_buff(pp, rxq, xdp, sinfo, sync);
2241	2241	break;
2242	2242	default:
2243		− bpf_warn_invalid_xdp_action(act);
	2243	+ bpf_warn_invalid_xdp_action(pp->dev, prog, act);
2244	2244	fallthrough;
2245	2245	case XDP_ABORTED:
2246	2246	trace_xdp_exception(pp->dev, prog, act);

drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c+1−1

@@ -3823,7 +3823,7 @@ mvpp2_run_xdp(struct mvpp2_port port, struct bpf_prog prog, · 3822 unmodified lines
3823	3823	}
3824	3824	break;
3825	3825	default:
3826		− bpf_warn_invalid_xdp_action(act);
	3826	+ bpf_warn_invalid_xdp_action(port->dev, prog, act);
3827	3827	fallthrough;
3828	3828	case XDP_ABORTED:
3829	3829	trace_xdp_exception(port->dev, prog, act);

drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c+1−1

@@ -1198,7 +1198,7 @@ static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf, · 1197 unmodified lines
1198	1198	put_page(page);
1199	1199	break;
1200	1200	default:
1201		− bpf_warn_invalid_xdp_action(act);
	1201	+ bpf_warn_invalid_xdp_action(pfvf->netdev, prog, act);
1202	1202	break;
1203	1203	case XDP_ABORTED:
1204	1204	trace_xdp_exception(pfvf->netdev, prog, act);

drivers/net/ethernet/mellanox/mlx4/en_netdev.c+1−0

@@ -33,6 +33,7 @@ · 32 unmodified lines
33	33
34	34	#include <linux/bpf.h>
35	35	#include <linux/etherdevice.h>
	36	+#include <linux/filter.h>
36	37	#include <linux/tcp.h>
37	38	#include <linux/if_vlan.h>
38	39	#include <linux/delay.h>

drivers/net/ethernet/mellanox/mlx4/en_rx.c+1−1

@@ -812,7 +812,7 @@ int mlx4_en_process_rx_cq(struct net_device dev, struct mlx4_en_cq cq, int bud · 811 unmodified lines
812	812	trace_xdp_exception(dev, xdp_prog, act);
813	813	goto xdp_drop_no_cnt; /* Drop on xmit failure */
814	814	default:
815		− bpf_warn_invalid_xdp_action(act);
	815	+ bpf_warn_invalid_xdp_action(dev, xdp_prog, act);
816	816	fallthrough;
817	817	case XDP_ABORTED:
818	818	trace_xdp_exception(dev, xdp_prog, act);

drivers/net/ethernet/mellanox/mlx5/core/en/qos.c+1−0

@@ -1,5 +1,6 @@
1	1	// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2	2	/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */
	3	+#include <net/sch_generic.h>
3	4
4	5	#include "en.h"
5	6	#include "params.h"

drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c+1−1

@@ -151,7 +151,7 @@ bool mlx5e_xdp_handle(struct mlx5e_rq rq, struct mlx5e_dma_info di, · 150 unmodified lines
151	151	rq->stats->xdp_redirect++;
152	152	return true;
153	153	default:
154		− bpf_warn_invalid_xdp_action(act);
	154	+ bpf_warn_invalid_xdp_action(rq->netdev, prog, act);
155	155	fallthrough;
156	156	case XDP_ABORTED:
157	157	xdp_abort:

drivers/net/ethernet/microsoft/mana/mana_bpf.c+1−1

@@ -60,7 +60,7 @@ u32 mana_run_xdp(struct net_device ndev, struct mana_rxq rxq, · 59 unmodified lines
60	60	break;
61	61
62	62	default:
63		− bpf_warn_invalid_xdp_action(act);
	63	+ bpf_warn_invalid_xdp_action(ndev, prog, act);
64	64	}
65	65
66	66	out:

drivers/net/ethernet/microsoft/mana/mana_en.c+2−0

@@ -1,6 +1,8 @@
1	1	// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2	2	/* Copyright (c) 2021, Microsoft Corporation. */
3	3
	4	+#include <uapi/linux/bpf.h>
	5	+
4	6	#include <linux/inetdevice.h>
5	7	#include <linux/etherdevice.h>
6	8	#include <linux/ethtool.h>

drivers/net/ethernet/netronome/nfp/nfp_net_common.c+1−1

@@ -1944,7 +1944,7 @@ static int nfp_net_rx(struct nfp_net_rx_ring *rx_ring, int budget) · 1943 unmodified lines
1944	1944	xdp_prog, act);
1945	1945	continue;
1946	1946	default:
1947		− bpf_warn_invalid_xdp_action(act);
	1947	+ bpf_warn_invalid_xdp_action(dp->netdev, xdp_prog, act);
1948	1948	fallthrough;
1949	1949	case XDP_ABORTED:
1950	1950	trace_xdp_exception(dp->netdev, xdp_prog, act);

drivers/net/ethernet/qlogic/qede/qede_fp.c+1−1

@@ -1153,7 +1153,7 @@ static bool qede_rx_xdp(struct qede_dev *edev, · 1152 unmodified lines
1153	1153	qede_rx_bd_ring_consume(rxq);
1154	1154	break;
1155	1155	default:
1156		− bpf_warn_invalid_xdp_action(act);
	1156	+ bpf_warn_invalid_xdp_action(edev->ndev, prog, act);
1157	1157	fallthrough;
1158	1158	case XDP_ABORTED:
1159	1159	trace_xdp_exception(edev->ndev, prog, act);

drivers/net/ethernet/sfc/efx.c+1−0

@@ -5,6 +5,7 @@ · 4 unmodified lines
5	5	* Copyright 2005-2013 Solarflare Communications Inc.
6	6	*/
7	7
	8	+#include <linux/filter.h>
8	9	#include <linux/module.h>
9	10	#include <linux/pci.h>
10	11	#include <linux/netdevice.h>

drivers/net/ethernet/sfc/efx_channels.c+1−0

@@ -10,6 +10,7 @@ · 9 unmodified lines
10	10
11	11	#include "net_driver.h"
12	12	#include <linux/module.h>
	13	+#include <linux/filter.h>
13	14	#include "efx_channels.h"
14	15	#include "efx.h"
15	16	#include "efx_common.h"

drivers/net/ethernet/sfc/efx_common.c+1−0

@@ -9,6 +9,7 @@ · 8 unmodified lines
9	9	*/
10	10
11	11	#include "net_driver.h"
	12	+#include <linux/filter.h>
12	13	#include <linux/module.h>
13	14	#include <linux/netdevice.h>
14	15	#include <net/gre.h>

drivers/net/ethernet/sfc/rx.c+1−1

@@ -338,7 +338,7 @@ static bool efx_do_xdp(struct efx_nic efx, struct efx_channel channel, · 337 unmodified lines
338	338	break;
339	339
340	340	default:
341		− bpf_warn_invalid_xdp_action(xdp_act);
	341	+ bpf_warn_invalid_xdp_action(efx->net_dev, xdp_prog, xdp_act);
342	342	efx_free_rx_buffers(rx_queue, rx_buf, 1);
343	343	channel->n_rx_xdp_bad_drops++;
344	344	trace_xdp_exception(efx->net_dev, xdp_prog, xdp_act);

drivers/net/ethernet/socionext/netsec.c+1−1

@@ -933,7 +933,7 @@ static u32 netsec_run_xdp(struct netsec_priv priv, struct bpf_prog prog, · 932 unmodified lines
933	933	}
934	934	break;
935	935	default:
936		− bpf_warn_invalid_xdp_action(act);
	936	+ bpf_warn_invalid_xdp_action(priv->ndev, prog, act);
937	937	fallthrough;
938	938	case XDP_ABORTED:
939	939	trace_xdp_exception(priv->ndev, prog, act);

drivers/net/ethernet/stmicro/stmmac/stmmac.h+1−0

@@ -22,6 +22,7 @@ · 21 unmodified lines
22	22	#include <linux/net_tstamp.h>
23	23	#include <linux/reset.h>
24	24	#include <net/page_pool.h>
	25	+#include <uapi/linux/bpf.h>
25	26
26	27	struct stmmac_resources {
27	28	void __iomem *addr;

drivers/net/ethernet/stmicro/stmmac/stmmac_main.c+1−1

@@ -4723,7 +4723,7 @@ static int __stmmac_xdp_run_prog(struct stmmac_priv *priv, · 4722 unmodified lines
4723	4723	res = STMMAC_XDP_REDIRECT;
4724	4724	break;
4725	4725	default:
4726		− bpf_warn_invalid_xdp_action(act);
	4726	+ bpf_warn_invalid_xdp_action(priv->dev, prog, act);
4727	4727	fallthrough;
4728	4728	case XDP_ABORTED:
4729	4729	trace_xdp_exception(priv->dev, prog, act);

drivers/net/ethernet/ti/cpsw_priv.c+1−1

@@ -1362,7 +1362,7 @@ int cpsw_run_xdp(struct cpsw_priv priv, int ch, struct xdp_buff xdp, · 1361 unmodified lines
1362	1362	xdp_do_flush_map();
1363	1363	break;
1364	1364	default:
1365		− bpf_warn_invalid_xdp_action(act);
	1365	+ bpf_warn_invalid_xdp_action(ndev, prog, act);
1366	1366	fallthrough;
1367	1367	case XDP_ABORTED:
1368	1368	trace_xdp_exception(ndev, prog, act);

drivers/net/ethernet/ti/cpsw_priv.h+2−0

@@ -6,6 +6,8 @@ · 5 unmodified lines
6	6	#ifndef DRIVERS_NET_ETHERNET_TI_CPSW_PRIV_H_
7	7	#define DRIVERS_NET_ETHERNET_TI_CPSW_PRIV_H_
8	8
	9	+#include <uapi/linux/bpf.h>
	10	+
9	11	#include "davinci_cpdma.h"
10	12
11	13	#define CPSW_DEBUG (NETIF_MSG_HW \| NETIF_MSG_WOL \| \

drivers/net/hamradio/hdlcdrv.c+1−0

@@ -30,6 +30,7 @@ · 29 unmodified lines
30	30	/*****************************************************************************/
31	31
32	32	#include <linux/capability.h>
	33	+#include <linux/compat.h>
33	34	#include <linux/module.h>
34	35	#include <linux/types.h>
35	36	#include <linux/net.h>

drivers/net/hamradio/scc.c+1−0

@@ -148,6 +148,7 @@ · 147 unmodified lines
148	148
149	149	/* ----------------------------------------------------------------------- */
150	150
	151	+#include <linux/compat.h>
151	152	#include <linux/module.h>
152	153	#include <linux/errno.h>
153	154	#include <linux/signal.h>

drivers/net/hyperv/netvsc_bpf.c+1−1

@@ -68,7 +68,7 @@ u32 netvsc_run_xdp(struct net_device ndev, struct netvsc_channel nvchan, · 67 unmodified lines
68	68	break;
69	69
70	70	default:
71		− bpf_warn_invalid_xdp_action(act);
	71	+ bpf_warn_invalid_xdp_action(ndev, prog, act);
72	72	}
73	73
74	74	out:

drivers/net/loopback.c+1−0

@@ -44,6 +44,7 @@ · 43 unmodified lines
44	44	#include <linux/etherdevice.h>
45	45	#include <linux/skbuff.h>
46	46	#include <linux/ethtool.h>
	47	+#include <net/sch_generic.h>
47	48	#include <net/sock.h>
48	49	#include <net/checksum.h>
49	50	#include <linux/if_ether.h> /* For the statistics structure. */

drivers/net/tun.c+1−1

@@ -1602,7 +1602,7 @@ static int tun_xdp_act(struct tun_struct tun, struct bpf_prog xdp_prog, · 1601 unmodified lines
1602	1602	case XDP_PASS:
1603	1603	break;
1604	1604	default:
1605		− bpf_warn_invalid_xdp_action(act);
	1605	+ bpf_warn_invalid_xdp_action(tun->dev, xdp_prog, act);
1606	1606	fallthrough;
1607	1607	case XDP_ABORTED:
1608	1608	trace_xdp_exception(tun->dev, xdp_prog, act);

drivers/net/veth.c+2−2

@@ -644,7 +644,7 @@ static struct xdp_frame veth_xdp_rcv_one(struct veth_rq rq, · 643 unmodified lines
644	644	rcu_read_unlock();
645	645	goto xdp_xmit;
646	646	default:
647		− bpf_warn_invalid_xdp_action(act);
	647	+ bpf_warn_invalid_xdp_action(rq->dev, xdp_prog, act);
648	648	fallthrough;
649	649	case XDP_ABORTED:
650	650	trace_xdp_exception(rq->dev, xdp_prog, act);
@@ -794,7 +794,7 @@ static struct sk_buff veth_xdp_rcv_skb(struct veth_rq rq, · 143 unmodified lines
794	794	rcu_read_unlock();
795	795	goto xdp_xmit;
796	796	default:
797		− bpf_warn_invalid_xdp_action(act);
	797	+ bpf_warn_invalid_xdp_action(rq->dev, xdp_prog, act);
798	798	fallthrough;
799	799	case XDP_ABORTED:
800	800	trace_xdp_exception(rq->dev, xdp_prog, act);

drivers/net/virtio_net.c+2−2

@@ -812,7 +812,7 @@ static struct sk_buff receive_small(struct net_device dev, · 811 unmodified lines
812	812	rcu_read_unlock();
813	813	goto xdp_xmit;
814	814	default:
815		− bpf_warn_invalid_xdp_action(act);
	815	+ bpf_warn_invalid_xdp_action(vi->dev, xdp_prog, act);
816	816	fallthrough;
817	817	case XDP_ABORTED:
818	818	trace_xdp_exception(vi->dev, xdp_prog, act);
@@ -1022,7 +1022,7 @@ static struct sk_buff receive_mergeable(struct net_device dev, · 203 unmodified lines
1022	1022	rcu_read_unlock();
1023	1023	goto xdp_xmit;
1024	1024	default:
1025		− bpf_warn_invalid_xdp_action(act);
	1025	+ bpf_warn_invalid_xdp_action(vi->dev, xdp_prog, act);
1026	1026	fallthrough;
1027	1027	case XDP_ABORTED:
1028	1028	trace_xdp_exception(vi->dev, xdp_prog, act);

drivers/net/vrf.c+1−0

@@ -34,6 +34,7 @@ · 33 unmodified lines
34	34	#include <net/addrconf.h>
35	35	#include <net/l3mdev.h>
36	36	#include <net/fib_rules.h>
	37	+#include <net/sch_generic.h>
37	38	#include <net/netns/generic.h>
38	39	#include <net/netfilter/nf_conntrack.h>
39	40

drivers/net/xen-netfront.c+1−1

@@ -948,7 +948,7 @@ static u32 xennet_run_xdp(struct netfront_queue queue, struct page pdata, · 947 unmodified lines
948	948	break;
949	949
950	950	default:
951		− bpf_warn_invalid_xdp_action(act);
	951	+ bpf_warn_invalid_xdp_action(queue->info->netdev, prog, act);
952	952	}
953	953
954	954	return act;

fs/nfs/dir.c+1−0

@@ -18,6 +18,7 @@ · 17 unmodified lines
18	18	* 6 Jun 1999 Cache readdir lookups in the page cache. -DaveM
19	19	*/
20	20
	21	+#include <linux/compat.h>
21	22	#include <linux/module.h>
22	23	#include <linux/time.h>
23	24	#include <linux/errno.h>

fs/nfs/fs_context.c+1−0

@@ -10,6 +10,7 @@ · 9 unmodified lines
10	10	* Split from fs/nfs/super.c by David Howells <dhowells@redhat.com>
11	11	*/
12	12
	13	+#include <linux/compat.h>
13	14	#include <linux/module.h>
14	15	#include <linux/fs.h>
15	16	#include <linux/fs_context.h>

fs/select.c+1−0

@@ -15,6 +15,7 @@ · 14 unmodified lines
15	15	* of fds to overcome nfds < 16390 descriptors limit (Tigran Aivazian).
16	16	*/
17	17
	18	+#include <linux/compat.h>
18	19	#include <linux/kernel.h>
19	20	#include <linux/sched/signal.h>
20	21	#include <linux/sched/rt.h>

include/linux/bpf-cgroup-defs.h+70−0

@@ -0,0 +1,70 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+#ifndef _BPF_CGROUP_DEFS_H
	3	+#define _BPF_CGROUP_DEFS_H
	4	+
	5	+#ifdef CONFIG_CGROUP_BPF
	6	+
	7	+#include <linux/list.h>
	8	+#include <linux/percpu-refcount.h>
	9	+#include <linux/workqueue.h>
	10	+
	11	+struct bpf_prog_array;
	12	+
	13	+enum cgroup_bpf_attach_type {
	14	+ CGROUP_BPF_ATTACH_TYPE_INVALID = -1,
	15	+ CGROUP_INET_INGRESS = 0,
	16	+ CGROUP_INET_EGRESS,
	17	+ CGROUP_INET_SOCK_CREATE,
	18	+ CGROUP_SOCK_OPS,
	19	+ CGROUP_DEVICE,
	20	+ CGROUP_INET4_BIND,
	21	+ CGROUP_INET6_BIND,
	22	+ CGROUP_INET4_CONNECT,
	23	+ CGROUP_INET6_CONNECT,
	24	+ CGROUP_INET4_POST_BIND,
	25	+ CGROUP_INET6_POST_BIND,
	26	+ CGROUP_UDP4_SENDMSG,
	27	+ CGROUP_UDP6_SENDMSG,
	28	+ CGROUP_SYSCTL,
	29	+ CGROUP_UDP4_RECVMSG,
	30	+ CGROUP_UDP6_RECVMSG,
	31	+ CGROUP_GETSOCKOPT,
	32	+ CGROUP_SETSOCKOPT,
	33	+ CGROUP_INET4_GETPEERNAME,
	34	+ CGROUP_INET6_GETPEERNAME,
	35	+ CGROUP_INET4_GETSOCKNAME,
	36	+ CGROUP_INET6_GETSOCKNAME,
	37	+ CGROUP_INET_SOCK_RELEASE,
	38	+ MAX_CGROUP_BPF_ATTACH_TYPE
	39	+};
	40	+
	41	+struct cgroup_bpf {
	42	+ /* array of effective progs in this cgroup */
	43	+ struct bpf_prog_array __rcu *effective[MAX_CGROUP_BPF_ATTACH_TYPE];
	44	+
	45	+ /* attached progs to this cgroup and attach flags
	46	+ * when flags == 0 or BPF_F_ALLOW_OVERRIDE the progs list will
	47	+ * have either zero or one element
	48	+ * when BPF_F_ALLOW_MULTI the list can have up to BPF_CGROUP_MAX_PROGS
	49	+ */
	50	+ struct list_head progs[MAX_CGROUP_BPF_ATTACH_TYPE];
	51	+ u32 flags[MAX_CGROUP_BPF_ATTACH_TYPE];
	52	+
	53	+ /* list of cgroup shared storages */
	54	+ struct list_head storages;
	55	+
	56	+ /* temp storage for effective prog array used by prog_attach/detach */
	57	+ struct bpf_prog_array *inactive;
	58	+
	59	+ /* reference counter used to detach bpf programs after cgroup removal */
	60	+ struct percpu_ref refcnt;
	61	+
	62	+ /* cgroup_bpf is released using a work queue */
	63	+ struct work_struct release_work;
	64	+};
	65	+
	66	+#else /* CONFIG_CGROUP_BPF */
	67	+struct cgroup_bpf {};
	68	+#endif /* CONFIG_CGROUP_BPF */
	69	+
	70	+#endif

include/linux/bpf-cgroup.h+1−56

@@ -3,10 +3,10 @@ · 2 unmodified lines
3	3	#define _BPF_CGROUP_H
4	4
5	5	#include <linux/bpf.h>
	6	+#include <linux/bpf-cgroup-defs.h>
6	7	#include <linux/errno.h>
7	8	#include <linux/jump_label.h>
8	9	#include <linux/percpu.h>
9		−#include <linux/percpu-refcount.h>
10	10	#include <linux/rbtree.h>
11	11	#include <uapi/linux/bpf.h>
12	12
@@ -23,33 +23,6 @@ struct ctl_table_header; · 10 unmodified lines
23	23	struct task_struct;
24	24
25	25	#ifdef CONFIG_CGROUP_BPF
26		−enum cgroup_bpf_attach_type {
27		− CGROUP_BPF_ATTACH_TYPE_INVALID = -1,
28		− CGROUP_INET_INGRESS = 0,
29		− CGROUP_INET_EGRESS,
30		− CGROUP_INET_SOCK_CREATE,
31		− CGROUP_SOCK_OPS,
32		− CGROUP_DEVICE,
33		− CGROUP_INET4_BIND,
34		− CGROUP_INET6_BIND,
35		− CGROUP_INET4_CONNECT,
36		− CGROUP_INET6_CONNECT,
37		− CGROUP_INET4_POST_BIND,
38		− CGROUP_INET6_POST_BIND,
39		− CGROUP_UDP4_SENDMSG,
40		− CGROUP_UDP6_SENDMSG,
41		− CGROUP_SYSCTL,
42		− CGROUP_UDP4_RECVMSG,
43		− CGROUP_UDP6_RECVMSG,
44		− CGROUP_GETSOCKOPT,
45		− CGROUP_SETSOCKOPT,
46		− CGROUP_INET4_GETPEERNAME,
47		− CGROUP_INET6_GETPEERNAME,
48		− CGROUP_INET4_GETSOCKNAME,
49		− CGROUP_INET6_GETSOCKNAME,
50		− CGROUP_INET_SOCK_RELEASE,
51		− MAX_CGROUP_BPF_ATTACH_TYPE
52		−};
53	26
54	27	#define CGROUP_ATYPE(type) \
55	28	case BPF_##type: return type
@@ -127,33 +100,6 @@ struct bpf_prog_list { · 71 unmodified lines
127	100	struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE];
128	101	};
129	102
130		−struct bpf_prog_array;
131		−
132		−struct cgroup_bpf {
133		− /* array of effective progs in this cgroup */
134		− struct bpf_prog_array __rcu *effective[MAX_CGROUP_BPF_ATTACH_TYPE];
135		−
136		− /* attached progs to this cgroup and attach flags
137		− * when flags == 0 or BPF_F_ALLOW_OVERRIDE the progs list will
138		− * have either zero or one element
139		− * when BPF_F_ALLOW_MULTI the list can have up to BPF_CGROUP_MAX_PROGS
140		− */
141		− struct list_head progs[MAX_CGROUP_BPF_ATTACH_TYPE];
142		− u32 flags[MAX_CGROUP_BPF_ATTACH_TYPE];
143		−
144		− /* list of cgroup shared storages */
145		− struct list_head storages;
146		−
147		− /* temp storage for effective prog array used by prog_attach/detach */
148		− struct bpf_prog_array *inactive;
149		−
150		− /* reference counter used to detach bpf programs after cgroup removal */
151		− struct percpu_ref refcnt;
152		−
153		− /* cgroup_bpf is released using a work queue */
154		− struct work_struct release_work;
155		−};
156		−
157	103	int cgroup_bpf_inherit(struct cgroup *cgrp);
158	104	void cgroup_bpf_offline(struct cgroup *cgrp);
159	105
@@ -451,7 +397,6 @@ int cgroup_bpf_prog_query(const union bpf_attr *attr, · 291 unmodified lines
451	397	union bpf_attr __user *uattr);
452	398	#else
453	399
454		−struct cgroup_bpf {};
455	400	static inline int cgroup_bpf_inherit(struct cgroup *cgrp) { return 0; }
456	401	static inline void cgroup_bpf_offline(struct cgroup *cgrp) {}
457	402

include/linux/bpf-netns.h+1−7

@@ -3,15 +3,9 @@ · 2 unmodified lines
3	3	#define _BPF_NETNS_H
4	4
5	5	#include <linux/mutex.h>
	6	+#include <net/netns/bpf.h>
6	7	#include <uapi/linux/bpf.h>
7	8
8		−enum netns_bpf_attach_type {
9		− NETNS_BPF_INVALID = -1,
10		− NETNS_BPF_FLOW_DISSECTOR = 0,
11		− NETNS_BPF_SK_LOOKUP,
12		− MAX_NETNS_BPF_ATTACH_TYPE
13		−};
14		−
15	9	static inline enum netns_bpf_attach_type
16	10	to_netns_bpf_attach_type(enum bpf_attach_type attach_type)
17	11	{

include/linux/bpf.h+83−24

@@ -297,6 +297,34 @@ bool bpf_map_meta_equal(const struct bpf_map *meta0, · 296 unmodified lines
297	297
298	298	extern const struct bpf_map_ops bpf_map_offload_ops;
299	299
	300	+/* bpf_type_flag contains a set of flags that are applicable to the values of
	301	+ * arg_type, ret_type and reg_type. For example, a pointer value may be null,
	302	+ * or a memory is read-only. We classify types into two categories: base types
	303	+ * and extended types. Extended types are base types combined with a type flag.
	304	+ *
	305	+ * Currently there are no more than 32 base types in arg_type, ret_type and
	306	+ * reg_types.
	307	+ */
	308	+#define BPF_BASE_TYPE_BITS 8
	309	+
	310	+enum bpf_type_flag {
	311	+ /* PTR may be NULL. */
	312	+ PTR_MAYBE_NULL = BIT(0 + BPF_BASE_TYPE_BITS),
	313	+
	314	+ /* MEM is read-only. When applied on bpf_arg, it indicates the arg is
	315	+ * compatible with both mutable and immutable memory.
	316	+ */
	317	+ MEM_RDONLY = BIT(1 + BPF_BASE_TYPE_BITS),
	318	+
	319	+ __BPF_TYPE_LAST_FLAG = MEM_RDONLY,
	320	+};
	321	+
	322	+/* Max number of base types. */
	323	+#define BPF_BASE_TYPE_LIMIT (1UL << BPF_BASE_TYPE_BITS)
	324	+
	325	+/* Max number of all types. */
	326	+#define BPF_TYPE_LIMIT (__BPF_TYPE_LAST_FLAG \| (__BPF_TYPE_LAST_FLAG - 1))
	327	+
300	328	/* function argument constraints */
301	329	enum bpf_arg_type {
302	330	ARG_DONTCARE = 0, /* unused argument in helper function */
@@ -308,13 +336,11 @@ enum bpf_arg_type { · 5 unmodified lines
308	336	ARG_PTR_TO_MAP_KEY, /* pointer to stack used as map key */
309	337	ARG_PTR_TO_MAP_VALUE, /* pointer to stack used as map value */
310	338	ARG_PTR_TO_UNINIT_MAP_VALUE, /* pointer to valid memory used to store a map value */
311		− ARG_PTR_TO_MAP_VALUE_OR_NULL, /* pointer to stack used as map value or NULL */
312	339
313	340	/* the following constraints used to prototype bpf_memcmp() and other
314	341	* functions that access data on eBPF program stack
315	342	*/
316	343	ARG_PTR_TO_MEM, /* pointer to valid memory (stack, packet, map value) */
317		− ARG_PTR_TO_MEM_OR_NULL, /* pointer to valid memory or NULL */
318	344	ARG_PTR_TO_UNINIT_MEM, /* pointer to memory does not need to be initialized,
319	345	* helper function must fill all bytes or clear
320	346	* them in error case.
@@ -324,42 +350,65 @@ enum bpf_arg_type { · 3 unmodified lines
324	350	ARG_CONST_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */
325	351
326	352	ARG_PTR_TO_CTX, /* pointer to context */
327		− ARG_PTR_TO_CTX_OR_NULL, /* pointer to context or NULL */
328	353	ARG_ANYTHING, /* any (initialized) argument is ok */
329	354	ARG_PTR_TO_SPIN_LOCK, /* pointer to bpf_spin_lock */
330	355	ARG_PTR_TO_SOCK_COMMON, /* pointer to sock_common */
331	356	ARG_PTR_TO_INT, /* pointer to int */
332	357	ARG_PTR_TO_LONG, /* pointer to long */
333	358	ARG_PTR_TO_SOCKET, /* pointer to bpf_sock (fullsock) */
334		− ARG_PTR_TO_SOCKET_OR_NULL, /* pointer to bpf_sock (fullsock) or NULL */
335	359	ARG_PTR_TO_BTF_ID, /* pointer to in-kernel struct */
336	360	ARG_PTR_TO_ALLOC_MEM, /* pointer to dynamically allocated memory */
337		− ARG_PTR_TO_ALLOC_MEM_OR_NULL, /* pointer to dynamically allocated memory or NULL */
338	361	ARG_CONST_ALLOC_SIZE_OR_ZERO, /* number of allocated bytes requested */
339	362	ARG_PTR_TO_BTF_ID_SOCK_COMMON, /* pointer to in-kernel sock_common or bpf-mirrored bpf_sock */
340	363	ARG_PTR_TO_PERCPU_BTF_ID, /* pointer to in-kernel percpu type */
341	364	ARG_PTR_TO_FUNC, /* pointer to a bpf program function */
342		− ARG_PTR_TO_STACK_OR_NULL, /* pointer to stack or NULL */
	365	+ ARG_PTR_TO_STACK, /* pointer to stack */
343	366	ARG_PTR_TO_CONST_STR, /* pointer to a null terminated read-only string */
344	367	ARG_PTR_TO_TIMER, /* pointer to bpf_timer */
345	368	__BPF_ARG_TYPE_MAX,
	369	+
	370	+ /* Extended arg_types. */
	371	+ ARG_PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL \| ARG_PTR_TO_MAP_VALUE,
	372	+ ARG_PTR_TO_MEM_OR_NULL = PTR_MAYBE_NULL \| ARG_PTR_TO_MEM,
	373	+ ARG_PTR_TO_CTX_OR_NULL = PTR_MAYBE_NULL \| ARG_PTR_TO_CTX,
	374	+ ARG_PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL \| ARG_PTR_TO_SOCKET,
	375	+ ARG_PTR_TO_ALLOC_MEM_OR_NULL = PTR_MAYBE_NULL \| ARG_PTR_TO_ALLOC_MEM,
	376	+ ARG_PTR_TO_STACK_OR_NULL = PTR_MAYBE_NULL \| ARG_PTR_TO_STACK,
	377	+
	378	+ /* This must be the last entry. Its purpose is to ensure the enum is
	379	+ * wide enough to hold the higher bits reserved for bpf_type_flag.
	380	+ */
	381	+ __BPF_ARG_TYPE_LIMIT = BPF_TYPE_LIMIT,
346	382	};
	383	+static_assert(__BPF_ARG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);
347	384
348	385	/* type of values returned from helper functions */
349	386	enum bpf_return_type {
350	387	RET_INTEGER, /* function returns integer */
351	388	RET_VOID, /* function doesn't return anything */
352	389	RET_PTR_TO_MAP_VALUE, /* returns a pointer to map elem value */
353		− RET_PTR_TO_MAP_VALUE_OR_NULL, /* returns a pointer to map elem value or NULL */
354		− RET_PTR_TO_SOCKET_OR_NULL, /* returns a pointer to a socket or NULL */
355		− RET_PTR_TO_TCP_SOCK_OR_NULL, /* returns a pointer to a tcp_sock or NULL */
356		− RET_PTR_TO_SOCK_COMMON_OR_NULL, /* returns a pointer to a sock_common or NULL */
357		− RET_PTR_TO_ALLOC_MEM_OR_NULL, /* returns a pointer to dynamically allocated memory or NULL */
358		− RET_PTR_TO_BTF_ID_OR_NULL, /* returns a pointer to a btf_id or NULL */
359		− RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL, /* returns a pointer to a valid memory or a btf_id or NULL */
	390	+ RET_PTR_TO_SOCKET, /* returns a pointer to a socket */
	391	+ RET_PTR_TO_TCP_SOCK, /* returns a pointer to a tcp_sock */
	392	+ RET_PTR_TO_SOCK_COMMON, /* returns a pointer to a sock_common */
	393	+ RET_PTR_TO_ALLOC_MEM, /* returns a pointer to dynamically allocated memory */
360	394	RET_PTR_TO_MEM_OR_BTF_ID, /* returns a pointer to a valid memory or a btf_id */
361	395	RET_PTR_TO_BTF_ID, /* returns a pointer to a btf_id */
	396	+ __BPF_RET_TYPE_MAX,
	397	+
	398	+ /* Extended ret_types. */
	399	+ RET_PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL \| RET_PTR_TO_MAP_VALUE,
	400	+ RET_PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL \| RET_PTR_TO_SOCKET,
	401	+ RET_PTR_TO_TCP_SOCK_OR_NULL = PTR_MAYBE_NULL \| RET_PTR_TO_TCP_SOCK,
	402	+ RET_PTR_TO_SOCK_COMMON_OR_NULL = PTR_MAYBE_NULL \| RET_PTR_TO_SOCK_COMMON,
	403	+ RET_PTR_TO_ALLOC_MEM_OR_NULL = PTR_MAYBE_NULL \| RET_PTR_TO_ALLOC_MEM,
	404	+ RET_PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL \| RET_PTR_TO_BTF_ID,
	405	+
	406	+ /* This must be the last entry. Its purpose is to ensure the enum is
	407	+ * wide enough to hold the higher bits reserved for bpf_type_flag.
	408	+ */
	409	+ __BPF_RET_TYPE_LIMIT = BPF_TYPE_LIMIT,
362	410	};
	411	+static_assert(__BPF_RET_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);
363	412
364	413	/* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs
365	414	* to in-kernel helper functions and for adjusting imm32 field in BPF_CALL
@@ -421,18 +470,15 @@ enum bpf_reg_type { · 55 unmodified lines
421	470	PTR_TO_CTX, /* reg points to bpf_context */
422	471	CONST_PTR_TO_MAP, /* reg points to struct bpf_map */
423	472	PTR_TO_MAP_VALUE, /* reg points to map element value */
424		− PTR_TO_MAP_VALUE_OR_NULL,/* points to map elem value or NULL */
	473	+ PTR_TO_MAP_KEY, /* reg points to a map element key */
425	474	PTR_TO_STACK, /* reg == frame_pointer + offset */
426	475	PTR_TO_PACKET_META, /* skb->data - meta_len */
427	476	PTR_TO_PACKET, /* reg points to skb->data */
428	477	PTR_TO_PACKET_END, /* skb->data + headlen */
429	478	PTR_TO_FLOW_KEYS, /* reg points to bpf_flow_keys */
430	479	PTR_TO_SOCKET, /* reg points to struct bpf_sock */
431		− PTR_TO_SOCKET_OR_NULL, /* reg points to struct bpf_sock or NULL */
432	480	PTR_TO_SOCK_COMMON, /* reg points to sock_common */
433		− PTR_TO_SOCK_COMMON_OR_NULL, /* reg points to sock_common or NULL */
434	481	PTR_TO_TCP_SOCK, /* reg points to struct tcp_sock */
435		− PTR_TO_TCP_SOCK_OR_NULL, /* reg points to struct tcp_sock or NULL */
436	482	PTR_TO_TP_BUFFER, /* reg points to a writable raw tp's buffer */
437	483	PTR_TO_XDP_SOCK, /* reg points to struct xdp_sock */
438	484	/* PTR_TO_BTF_ID points to a kernel struct that does not need
@@ -450,18 +496,25 @@ enum bpf_reg_type { · 11 unmodified lines
450	496	* been checked for null. Used primarily to inform the verifier
451	497	* an explicit null check is required for this struct.
452	498	*/
453		− PTR_TO_BTF_ID_OR_NULL,
454	499	PTR_TO_MEM, /* reg points to valid memory region */
455		− PTR_TO_MEM_OR_NULL, /* reg points to valid memory region or NULL */
456		− PTR_TO_RDONLY_BUF, /* reg points to a readonly buffer */
457		− PTR_TO_RDONLY_BUF_OR_NULL, /* reg points to a readonly buffer or NULL */
458		− PTR_TO_RDWR_BUF, /* reg points to a read/write buffer */
459		− PTR_TO_RDWR_BUF_OR_NULL, /* reg points to a read/write buffer or NULL */
	500	+ PTR_TO_BUF, /* reg points to a read/write buffer */
460	501	PTR_TO_PERCPU_BTF_ID, /* reg points to a percpu kernel variable */
461	502	PTR_TO_FUNC, /* reg points to a bpf program function */
462		− PTR_TO_MAP_KEY, /* reg points to a map element key */
463	503	__BPF_REG_TYPE_MAX,
	504	+
	505	+ /* Extended reg_types. */
	506	+ PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL \| PTR_TO_MAP_VALUE,
	507	+ PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL \| PTR_TO_SOCKET,
	508	+ PTR_TO_SOCK_COMMON_OR_NULL = PTR_MAYBE_NULL \| PTR_TO_SOCK_COMMON,
	509	+ PTR_TO_TCP_SOCK_OR_NULL = PTR_MAYBE_NULL \| PTR_TO_TCP_SOCK,
	510	+ PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL \| PTR_TO_BTF_ID,
	511	+
	512	+ /* This must be the last entry. Its purpose is to ensure the enum is
	513	+ * wide enough to hold the higher bits reserved for bpf_type_flag.
	514	+ */
	515	+ __BPF_REG_TYPE_LIMIT = BPF_TYPE_LIMIT,
464	516	};
	517	+static_assert(__BPF_REG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);
465	518
466	519	/* The information passed from prog-specific *_is_valid_access
467	520	* back to the verifier.
@@ -777,6 +830,7 @@ void bpf_ksym_add(struct bpf_ksym *ksym); · 309 unmodified lines
777	830	void bpf_ksym_del(struct bpf_ksym *ksym);
778	831	int bpf_jit_charge_modmem(u32 pages);
779	832	void bpf_jit_uncharge_modmem(u32 pages);
	833	+bool bpf_prog_has_trampoline(const struct bpf_prog *prog);
780	834	#else
781	835	static inline int bpf_trampoline_link_prog(struct bpf_prog *prog,
782	836	struct bpf_trampoline *tr)
@@ -805,6 +859,10 @@ static inline bool is_bpf_image_address(unsigned long address) · 22 unmodified lines
805	859	{
806	860	return false;
807	861	}
	862	+static inline bool bpf_prog_has_trampoline(const struct bpf_prog *prog)
	863	+{
	864	+ return false;
	865	+}
808	866	#endif
809	867
810	868	struct bpf_func_info_aux {
@@ -2163,6 +2221,7 @@ extern const struct bpf_func_proto bpf_sk_getsockopt_proto; · 1352 unmodified lines
2163	2221	extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto;
2164	2222	extern const struct bpf_func_proto bpf_find_vma_proto;
2165	2223	extern const struct bpf_func_proto bpf_loop_proto;
	2224	+extern const struct bpf_func_proto bpf_strncmp_proto;
2166	2225
2167	2226	const struct bpf_func_proto *tracing_prog_func_proto(
2168	2227	enum bpf_func_id func_id, const struct bpf_prog *prog);

include/linux/bpf_local_storage.h+6−0

@@ -8,6 +8,7 @@ · 7 unmodified lines
8	8	#define _BPF_LOCAL_STORAGE_H
9	9
10	10	#include <linux/bpf.h>
	11	+#include <linux/filter.h>
11	12	#include <linux/rculist.h>
12	13	#include <linux/list.h>
13	14	#include <linux/hash.h>
@@ -16,6 +17,9 @@ · 2 unmodified lines
16	17
17	18	#define BPF_LOCAL_STORAGE_CACHE_SIZE 16
18	19
	20	+#define bpf_rcu_lock_held() \
	21	+ (rcu_read_lock_held() \|\| rcu_read_lock_trace_held() \|\| \
	22	+ rcu_read_lock_bh_held())
19	23	struct bpf_local_storage_map_bucket {
20	24	struct hlist_head list;
21	25	raw_spinlock_t lock;
@@ -161,4 +165,6 @@ struct bpf_local_storage_data * · 139 unmodified lines
161	165	bpf_local_storage_update(void owner, struct bpf_local_storage_map smap,
162	166	void *value, u64 map_flags);
163	167
	168	+void bpf_local_storage_free_rcu(struct rcu_head *rcu);
	169	+
164	170	#endif /* _BPF_LOCAL_STORAGE_H */

include/linux/bpf_verifier.h+27−0

@@ -18,6 +18,8 @@ · 17 unmodified lines
18	18	* that converting umax_value to int cannot overflow.
19	19	*/
20	20	#define BPF_MAX_VAR_SIZ (1 << 29)
	21	+/* size of type_str_buf in bpf_verifier. */
	22	+#define TYPE_STR_BUF_LEN 64
21	23
22	24	/* Liveness marks, used for registers and spilled-regs (in stack slots).
23	25	* Read marks propagate upwards until they find a write mark; they record that
@@ -388,6 +390,8 @@ static inline bool bpf_verifier_log_full(const struct bpf_verifier_log *log) · 364 unmodified lines
388	390	#define BPF_LOG_LEVEL (BPF_LOG_LEVEL1 \| BPF_LOG_LEVEL2)
389	391	#define BPF_LOG_MASK (BPF_LOG_LEVEL \| BPF_LOG_STATS)
390	392	#define BPF_LOG_KERNEL (BPF_LOG_MASK + 1) /* kernel internal flag */
	393	+#define BPF_LOG_MIN_ALIGNMENT 8U
	394	+#define BPF_LOG_ALIGNMENT 40U
391	395
392	396	static inline bool bpf_verifier_log_needed(const struct bpf_verifier_log *log)
393	397	{
@@ -474,6 +478,16 @@ struct bpf_verifier_env { · 80 unmodified lines
474	478	/* longest register parentage chain walked for liveness marking */
475	479	u32 longest_mark_read_walk;
476	480	bpfptr_t fd_array;
	481	+
	482	+ /* bit mask to keep track of whether a register has been accessed
	483	+ * since the last time the function state was printed
	484	+ */
	485	+ u32 scratched_regs;
	486	+ /* Same as scratched_regs but for stack slots */
	487	+ u64 scratched_stack_slots;
	488	+ u32 prev_log_len, prev_insn_print_len;
	489	+ /* buffer used in reg_type_str() to generate reg_type string */
	490	+ char type_str_buf[TYPE_STR_BUF_LEN];
477	491	};
478	492
479	493	__printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log,
@@ -536,5 +550,18 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, · 56 unmodified lines
536	550	struct bpf_attach_target_info *tgt_info);
537	551	void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab);
538	552
	553	+#define BPF_BASE_TYPE_MASK GENMASK(BPF_BASE_TYPE_BITS - 1, 0)
	554	+
	555	+/* extract base type from bpf_{arg, return, reg}_type. */
	556	+static inline u32 base_type(u32 type)
	557	+{
	558	+ return type & BPF_BASE_TYPE_MASK;
	559	+}
	560	+
	561	+/* extract flags from an extended type. See bpf_type_flag in bpf.h. */
	562	+static inline u32 type_flag(u32 type)
	563	+{
	564	+ return type & ~BPF_BASE_TYPE_MASK;
	565	+}
539	566
540	567	#endif /* _LINUX_BPF_VERIFIER_H */

include/linux/cgroup-defs.h+1−1

@@ -19,7 +19,7 @@ · 18 unmodified lines
19	19	#include <linux/percpu-rwsem.h>
20	20	#include <linux/u64_stats_sync.h>
21	21	#include <linux/workqueue.h>
22		−#include <linux/bpf-cgroup.h>
	22	+#include <linux/bpf-cgroup-defs.h>
23	23	#include <linux/psi_types.h>
24	24
25	25	#ifdef CONFIG_CGROUPS

include/linux/dsa/loop.h+1−0

@@ -2,6 +2,7 @@ · 1 unmodified line
2	2	#ifndef DSA_LOOP_H
3	3	#define DSA_LOOP_H
4	4
	5	+#include <linux/if_vlan.h>
5	6	#include <linux/types.h>
6	7	#include <linux/ethtool.h>
7	8	#include <net/dsa.h>

include/linux/filter.h+1−1

@@ -1027,7 +1027,7 @@ void xdp_do_flush(void); · 1026 unmodified lines
1027	1027	*/
1028	1028	#define xdp_do_flush_map xdp_do_flush
1029	1029
1030		−void bpf_warn_invalid_xdp_action(u32 act);
	1030	+void bpf_warn_invalid_xdp_action(struct net_device dev, struct bpf_prog prog, u32 act);
1031	1031
1032	1032	#ifdef CONFIG_INET
1033	1033	struct sock bpf_run_sk_reuseport(struct sock_reuseport reuse, struct sock *sk,

include/linux/perf_event.h+1−0

@@ -611,6 +611,7 @@ struct swevent_hlist { · 610 unmodified lines
611	611	#define PERF_ATTACH_SCHED_CB 0x20
612	612	#define PERF_ATTACH_CHILD 0x40
613	613
	614	+struct bpf_prog;
614	615	struct perf_cgroup;
615	616	struct perf_buffer;
616	617

include/net/ip6_fib.h+1−0

@@ -20,6 +20,7 @@ · 19 unmodified lines
20	20	#include <net/inetpeer.h>
21	21	#include <net/fib_notifier.h>
22	22	#include <linux/indirect_call_wrapper.h>
	23	+#include <uapi/linux/bpf.h>
23	24
24	25	#ifdef CONFIG_IPV6_MULTIPLE_TABLES
25	26	#define FIB6_TABLE_HASHSZ 256

include/net/ipv6.h+2−0

@@ -21,6 +21,8 @@ · 20 unmodified lines
21	21	#include <net/snmp.h>
22	22	#include <net/netns/hash.h>
23	23
	24	+struct ip_tunnel_info;
	25	+
24	26	#define SIN6_LEN_RFC2133 24
25	27
26	28	#define IPV6_MAXPLEN 65535

include/net/netns/bpf.h+8−1

@@ -6,11 +6,18 @@ · 5 unmodified lines
6	6	#ifndef __NETNS_BPF_H__
7	7	#define __NETNS_BPF_H__
8	8
9		−#include <linux/bpf-netns.h>
	9	+#include <linux/list.h>
10	10
11	11	struct bpf_prog;
12	12	struct bpf_prog_array;
13	13
	14	+enum netns_bpf_attach_type {
	15	+ NETNS_BPF_INVALID = -1,
	16	+ NETNS_BPF_FLOW_DISSECTOR = 0,
	17	+ NETNS_BPF_SK_LOOKUP,
	18	+ MAX_NETNS_BPF_ATTACH_TYPE
	19	+};
	20	+
14	21	struct netns_bpf {
15	22	/* Array of programs to run compiled from progs or links */
16	23	struct bpf_prog_array __rcu *run_array[MAX_NETNS_BPF_ATTACH_TYPE];

include/net/route.h+1−0

@@ -43,6 +43,7 @@ · 42 unmodified lines
43	43	#define RT_CONN_FLAGS(sk) (RT_TOS(inet_sk(sk)->tos) \| sock_flag(sk, SOCK_LOCALROUTE))
44	44	#define RT_CONN_FLAGS_TOS(sk,tos) (RT_TOS(tos) \| sock_flag(sk, SOCK_LOCALROUTE))
45	45
	46	+struct ip_tunnel_info;
46	47	struct fib_nh;
47	48	struct fib_info;
48	49	struct uncached_list;

include/net/sock.h+1−1

@@ -56,7 +56,6 @@ · 55 unmodified lines
56	56	#include <linux/wait.h>
57	57	#include <linux/cgroup-defs.h>
58	58	#include <linux/rbtree.h>
59		−#include <linux/filter.h>
60	59	#include <linux/rculist_nulls.h>
61	60	#include <linux/poll.h>
62	61	#include <linux/sockptr.h>
@@ -249,6 +248,7 @@ struct sock_common { · 186 unmodified lines
249	248	};
250	249
251	250	struct bpf_local_storage;
	251	+struct sk_filter;
252	252
253	253	/**
254	254	* struct sock - network layer representation of sockets

include/net/xdp_priv.h+0−1

@@ -10,7 +10,6 @@ struct xdp_mem_allocator { · 9 unmodified lines
10	10	union {
11	11	void *allocator;
12	12	struct page_pool *page_pool;
13		− struct zero_copy_allocator *zc_alloc;
14	13	};
15	14	struct rhash_head node;
16	15	struct rcu_head rcu;

include/net/xdp_sock.h+1−0

@@ -6,6 +6,7 @@ · 5 unmodified lines
6	6	#ifndef _LINUX_XDP_SOCK_H
7	7	#define _LINUX_XDP_SOCK_H
8	8
	9	+#include <linux/bpf.h>
9	10	#include <linux/workqueue.h>
10	11	#include <linux/if_xdp.h>
11	12	#include <linux/mutex.h>

include/uapi/linux/bpf.h+39−0

@@ -4983,6 +4983,41 @@ union bpf_attr { · 4982 unmodified lines
4983	4983	* Return
4984	4984	* The number of loops performed, -EINVAL for invalid flags,
4985	4985	* -E2BIG if nr_loops exceeds the maximum number of loops.
	4986	+ *
	4987	+ * long bpf_strncmp(const char s1, u32 s1_sz, const char s2)
	4988	+ * Description
	4989	+ * Do strncmp() between s1 and s2. s1 doesn't need
	4990	+ * to be null-terminated and s1_sz is the maximum storage
	4991	+ * size of s1. s2 must be a read-only string.
	4992	+ * Return
	4993	+ * An integer less than, equal to, or greater than zero
	4994	+ * if the first s1_sz bytes of s1 is found to be
	4995	+ * less than, to match, or be greater than s2.
	4996	+ *
	4997	+ * long bpf_get_func_arg(void ctx, u32 n, u64 value)
	4998	+ * Description
	4999	+ * Get n-th argument (zero based) of the traced function (for tracing programs)
	5000	+ * returned in value.
	5001	+ *
	5002	+ * Return
	5003	+ * 0 on success.
	5004	+ * -EINVAL if n >= arguments count of traced function.
	5005	+ *
	5006	+ * long bpf_get_func_ret(void ctx, u64 value)
	5007	+ * Description
	5008	+ * Get return value of the traced function (for tracing programs)
	5009	+ * in value.
	5010	+ *
	5011	+ * Return
	5012	+ * 0 on success.
	5013	+ * -EOPNOTSUPP for tracing programs other than BPF_TRACE_FEXIT or BPF_MODIFY_RETURN.
	5014	+ *
	5015	+ * long bpf_get_func_arg_cnt(void *ctx)
	5016	+ * Description
	5017	+ * Get number of arguments of the traced function (for tracing programs).
	5018	+ *
	5019	+ * Return
	5020	+ * The number of arguments of the traced function.
4986	5021	*/
4987	5022	#define __BPF_FUNC_MAPPER(FN) \
4988	5023	FN(unspec), \
@@ -5167,6 +5202,10 @@ union bpf_attr { · 178 unmodified lines
5167	5202	FN(kallsyms_lookup_name), \
5168	5203	FN(find_vma), \
5169	5204	FN(loop), \
	5205	+ FN(strncmp), \
	5206	+ FN(get_func_arg), \
	5207	+ FN(get_func_ret), \
	5208	+ FN(get_func_arg_cnt), \
5170	5209	/* */
5171	5210
5172	5211	/* integer value in 'imm' field of BPF_CALL instruction selects which helper

kernel/bpf/bloom_filter.c+6−0

@@ -82,6 +82,11 @@ static int bloom_map_delete_elem(struct bpf_map map, void value) · 81 unmodified lines
82	82	return -EOPNOTSUPP;
83	83	}
84	84
	85	+static int bloom_map_get_next_key(struct bpf_map map, void key, void *next_key)
	86	+{
	87	+ return -EOPNOTSUPP;
	88	+}
	89	+
85	90	static struct bpf_map bloom_map_alloc(union bpf_attr attr)
86	91	{
87	92	u32 bitset_bytes, bitset_mask, nr_hash_funcs, nr_bits;
@@ -192,6 +197,7 @@ const struct bpf_map_ops bloom_filter_map_ops = { · 104 unmodified lines
192	197	.map_meta_equal = bpf_map_meta_equal,
193	198	.map_alloc = bloom_map_alloc,
194	199	.map_free = bloom_map_free,
	200	+ .map_get_next_key = bloom_map_get_next_key,
195	201	.map_push_elem = bloom_map_push_elem,
196	202	.map_peek_elem = bloom_map_peek_elem,
197	203	.map_pop_elem = bloom_map_pop_elem,

kernel/bpf/bpf_inode_storage.c+5−1

@@ -17,6 +17,7 @@ · 16 unmodified lines
17	17	#include <linux/bpf_lsm.h>
18	18	#include <linux/btf_ids.h>
19	19	#include <linux/fdtable.h>
	20	+#include <linux/rcupdate_trace.h>
20	21
21	22	DEFINE_BPF_STORAGE_CACHE(inode_cache);
22	23
@@ -44,7 +45,8 @@ static struct bpf_local_storage_data inode_storage_lookup(struct inode inode, · 21 unmodified lines
44	45	if (!bsb)
45	46	return NULL;
46	47
47		− inode_storage = rcu_dereference(bsb->storage);
	48	+ inode_storage =
	49	+ rcu_dereference_check(bsb->storage, bpf_rcu_lock_held());
48	50	if (!inode_storage)
49	51	return NULL;
50	52
@@ -172,6 +174,7 @@ BPF_CALL_4(bpf_inode_storage_get, struct bpf_map , map, struct inode , inode, · 121 unmodified lines
172	174	{
173	175	struct bpf_local_storage_data *sdata;
174	176
	177	+ WARN_ON_ONCE(!bpf_rcu_lock_held());
175	178	if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
176	179	return (unsigned long)NULL;
177	180
@@ -204,6 +207,7 @@ BPF_CALL_4(bpf_inode_storage_get, struct bpf_map , map, struct inode , inode, · 26 unmodified lines
204	207	BPF_CALL_2(bpf_inode_storage_delete,
205	208	struct bpf_map , map, struct inode , inode)
206	209	{
	210	+ WARN_ON_ONCE(!bpf_rcu_lock_held());
207	211	if (!inode)
208	212	return -EINVAL;
209	213

kernel/bpf/bpf_local_storage.c+37−13

@@ -11,6 +11,9 @@ · 10 unmodified lines
11	11	#include <net/sock.h>
12	12	#include <uapi/linux/sock_diag.h>
13	13	#include <uapi/linux/btf.h>
	14	+#include <linux/rcupdate.h>
	15	+#include <linux/rcupdate_trace.h>
	16	+#include <linux/rcupdate_wait.h>
14	17
15	18	#define BPF_LOCAL_STORAGE_CREATE_FLAG_MASK (BPF_F_NO_PREALLOC \| BPF_F_CLONE)
16	19
@@ -81,6 +84,22 @@ bpf_selem_alloc(struct bpf_local_storage_map smap, void owner, · 64 unmodified lines
81	84	return NULL;
82	85	}
83	86
	87	+void bpf_local_storage_free_rcu(struct rcu_head *rcu)
	88	+{
	89	+ struct bpf_local_storage *local_storage;
	90	+
	91	+ local_storage = container_of(rcu, struct bpf_local_storage, rcu);
	92	+ kfree_rcu(local_storage, rcu);
	93	+}
	94	+
	95	+static void bpf_selem_free_rcu(struct rcu_head *rcu)
	96	+{
	97	+ struct bpf_local_storage_elem *selem;
	98	+
	99	+ selem = container_of(rcu, struct bpf_local_storage_elem, rcu);
	100	+ kfree_rcu(selem, rcu);
	101	+}
	102	+
84	103	/* local_storage->lock must be held and selem->local_storage == local_storage.
85	104	* The caller must ensure selem->smap is still valid to be
86	105	* dereferenced for its smap->elem_size and smap->cache_idx.
@@ -93,7 +112,7 @@ bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage, · 6 unmodified lines
93	112	bool free_local_storage;
94	113	void *owner;
95	114
96		− smap = rcu_dereference(SDATA(selem)->smap);
	115	+ smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held());
97	116	owner = local_storage->owner;
98	117
99	118	/* All uncharging on the owner must be done first.
@@ -118,12 +137,12 @@ bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage, · 18 unmodified lines
118	137	*
119	138	* Although the unlock will be done under
120	139	* rcu_read_lock(), it is more intutivie to
121		− * read if kfree_rcu(local_storage, rcu) is done
	140	+ * read if the freeing of the storage is done
122	141	* after the raw_spin_unlock_bh(&local_storage->lock).
123	142	*
124	143	* Hence, a "bool free_local_storage" is returned
125		− * to the caller which then calls the kfree_rcu()
126		− * after unlock.
	144	+ * to the caller which then calls then frees the storage after
	145	+ * all the RCU grace periods have expired.
127	146	*/
128	147	}
129	148	hlist_del_init_rcu(&selem->snode);
@@ -131,8 +150,7 @@ bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage, · 1 unmodified line
131	150	SDATA(selem))
132	151	RCU_INIT_POINTER(local_storage->cache[smap->cache_idx], NULL);
133	152
134		− kfree_rcu(selem, rcu);
135		−
	153	+ call_rcu_tasks_trace(&selem->rcu, bpf_selem_free_rcu);
136	154	return free_local_storage;
137	155	}
138	156
@@ -146,7 +164,8 @@ static void __bpf_selem_unlink_storage(struct bpf_local_storage_elem *selem) · 7 unmodified lines
146	164	/* selem has already been unlinked from sk */
147	165	return;
148	166
149		− local_storage = rcu_dereference(selem->local_storage);
	167	+ local_storage = rcu_dereference_check(selem->local_storage,
	168	+ bpf_rcu_lock_held());
150	169	raw_spin_lock_irqsave(&local_storage->lock, flags);
151	170	if (likely(selem_linked_to_storage(selem)))
152	171	free_local_storage = bpf_selem_unlink_storage_nolock(
@@ -154,7 +173,8 @@ static void __bpf_selem_unlink_storage(struct bpf_local_storage_elem *selem) · 1 unmodified line
154	173	raw_spin_unlock_irqrestore(&local_storage->lock, flags);
155	174
156	175	if (free_local_storage)
157		− kfree_rcu(local_storage, rcu);
	176	+ call_rcu_tasks_trace(&local_storage->rcu,
	177	+ bpf_local_storage_free_rcu);
158	178	}
159	179
160	180	void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,
@@ -174,7 +194,7 @@ void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem) · 13 unmodified lines
174	194	/* selem has already be unlinked from smap */
175	195	return;
176	196
177		− smap = rcu_dereference(SDATA(selem)->smap);
	197	+ smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held());
178	198	b = select_bucket(smap, selem);
179	199	raw_spin_lock_irqsave(&b->lock, flags);
180	200	if (likely(selem_linked_to_map(selem)))
@@ -213,12 +233,14 @@ bpf_local_storage_lookup(struct bpf_local_storage *local_storage, · 32 unmodified lines
213	233	struct bpf_local_storage_elem *selem;
214	234
215	235	/* Fast path (cache hit) */
216		− sdata = rcu_dereference(local_storage->cache[smap->cache_idx]);
	236	+ sdata = rcu_dereference_check(local_storage->cache[smap->cache_idx],
	237	+ bpf_rcu_lock_held());
217	238	if (sdata && rcu_access_pointer(sdata->smap) == smap)
218	239	return sdata;
219	240
220	241	/* Slow path (cache miss) */
221		− hlist_for_each_entry_rcu(selem, &local_storage->list, snode)
	242	+ hlist_for_each_entry_rcu(selem, &local_storage->list, snode,
	243	+ rcu_read_lock_trace_held())
222	244	if (rcu_access_pointer(SDATA(selem)->smap) == smap)
223	245	break;
224	246
@@ -306,7 +328,8 @@ int bpf_local_storage_alloc(void *owner, · 81 unmodified lines
306	328	* bucket->list, first_selem can be freed immediately
307	329	* (instead of kfree_rcu) because
308	330	* bpf_local_storage_map_free() does a
309		− * synchronize_rcu() before walking the bucket->list.
	331	+ * synchronize_rcu_mult (waiting for both sleepable and
	332	+ * normal programs) before walking the bucket->list.
310	333	* Hence, no one is accessing selem from the
311	334	* bucket->list under rcu_read_lock().
312	335	*/
@@ -342,7 +365,8 @@ bpf_local_storage_update(void owner, struct bpf_local_storage_map smap, · 29 unmodified lines
342	365	!map_value_has_spin_lock(&smap->map)))
343	366	return ERR_PTR(-EINVAL);
344	367
345		− local_storage = rcu_dereference(*owner_storage(smap, owner));
	368	+ local_storage = rcu_dereference_check(*owner_storage(smap, owner),
	369	+ bpf_rcu_lock_held());
346	370	if (!local_storage \|\| hlist_empty(&local_storage->list)) {
347	371	/* Very first elem for the owner */
348	372	err = check_flags(NULL, map_flags);

kernel/bpf/bpf_task_storage.c+5−1

@@ -17,6 +17,7 @@ · 16 unmodified lines
17	17	#include <uapi/linux/btf.h>
18	18	#include <linux/btf_ids.h>
19	19	#include <linux/fdtable.h>
	20	+#include <linux/rcupdate_trace.h>
20	21
21	22	DEFINE_BPF_STORAGE_CACHE(task_cache);
22	23
@@ -59,7 +60,8 @@ task_storage_lookup(struct task_struct task, struct bpf_map map, · 36 unmodified lines
59	60	struct bpf_local_storage *task_storage;
60	61	struct bpf_local_storage_map *smap;
61	62
62		− task_storage = rcu_dereference(task->bpf_storage);
	63	+ task_storage =
	64	+ rcu_dereference_check(task->bpf_storage, bpf_rcu_lock_held());
63	65	if (!task_storage)
64	66	return NULL;
65	67
@@ -229,6 +231,7 @@ BPF_CALL_4(bpf_task_storage_get, struct bpf_map , map, struct task_struct , · 163 unmodified lines
229	231	{
230	232	struct bpf_local_storage_data *sdata;
231	233
	234	+ WARN_ON_ONCE(!bpf_rcu_lock_held());
232	235	if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
233	236	return (unsigned long)NULL;
234	237
@@ -260,6 +263,7 @@ BPF_CALL_2(bpf_task_storage_delete, struct bpf_map , map, struct task_struct , · 25 unmodified lines
260	263	{
261	264	int ret;
262	265
	266	+ WARN_ON_ONCE(!bpf_rcu_lock_held());
263	267	if (!task)
264	268	return -EINVAL;
265	269

kernel/bpf/btf.c+88−36

@@ -4826,7 +4826,7 @@ struct btf bpf_prog_get_target_btf(const struct bpf_prog prog) · 4825 unmodified lines
4826	4826	return prog->aux->attach_btf;
4827	4827	}
4828	4828
4829		−static bool is_string_ptr(struct btf btf, const struct btf_type t)
	4829	+static bool is_int_ptr(struct btf btf, const struct btf_type t)
4830	4830	{
4831	4831	/* t comes in already as a pointer */
4832	4832	t = btf_type_by_id(btf, t->type);
@@ -4835,8 +4835,7 @@ static bool is_string_ptr(struct btf btf, const struct btf_type t) · 2 unmodified lines
4835	4835	if (BTF_INFO_KIND(t->info) == BTF_KIND_CONST)
4836	4836	t = btf_type_by_id(btf, t->type);
4837	4837
4838		− /* char, signed char, unsigned char */
4839		− return btf_type_is_int(t) && t->size == 1;
	4838	+ return btf_type_is_int(t);
4840	4839	}
4841	4840
4842	4841	bool btf_ctx_access(int off, int size, enum bpf_access_type type,
@@ -4941,10 +4940,12 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, · 98 unmodified lines
4941	4940	/* check for PTR_TO_RDONLY_BUF_OR_NULL or PTR_TO_RDWR_BUF_OR_NULL */
4942	4941	for (i = 0; i < prog->aux->ctx_arg_info_size; i++) {
4943	4942	const struct bpf_ctx_arg_aux *ctx_arg_info = &prog->aux->ctx_arg_info[i];
	4943	+ u32 type, flag;
4944	4944
4945		− if (ctx_arg_info->offset == off &&
4946		− (ctx_arg_info->reg_type == PTR_TO_RDONLY_BUF_OR_NULL \|\|
4947		− ctx_arg_info->reg_type == PTR_TO_RDWR_BUF_OR_NULL)) {
	4945	+ type = base_type(ctx_arg_info->reg_type);
	4946	+ flag = type_flag(ctx_arg_info->reg_type);
	4947	+ if (ctx_arg_info->offset == off && type == PTR_TO_BUF &&
	4948	+ (flag & PTR_MAYBE_NULL)) {
4948	4949	info->reg_type = ctx_arg_info->reg_type;
4949	4950	return true;
4950	4951	}
@@ -4957,7 +4958,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, · 6 unmodified lines
4957	4958	*/
4958	4959	return true;
4959	4960
4960		− if (is_string_ptr(btf, t))
	4961	+ if (is_int_ptr(btf, t))
4961	4962	return true;
4962	4963
4963	4964	/* this is a pointer to another type */
@@ -5575,12 +5576,53 @@ static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = { · 611 unmodified lines
5575	5576	#endif
5576	5577	};
5577	5578
	5579	+/* Returns true if struct is composed of scalars, 4 levels of nesting allowed */
	5580	+static bool __btf_type_is_scalar_struct(struct bpf_verifier_log *log,
	5581	+ const struct btf *btf,
	5582	+ const struct btf_type *t, int rec)
	5583	+{
	5584	+ const struct btf_type *member_type;
	5585	+ const struct btf_member *member;
	5586	+ u32 i;
	5587	+
	5588	+ if (!btf_type_is_struct(t))
	5589	+ return false;
	5590	+
	5591	+ for_each_member(i, t, member) {
	5592	+ const struct btf_array *array;
	5593	+
	5594	+ member_type = btf_type_skip_modifiers(btf, member->type, NULL);
	5595	+ if (btf_type_is_struct(member_type)) {
	5596	+ if (rec >= 3) {
	5597	+ bpf_log(log, "max struct nesting depth exceeded\n");
	5598	+ return false;
	5599	+ }
	5600	+ if (!__btf_type_is_scalar_struct(log, btf, member_type, rec + 1))
	5601	+ return false;
	5602	+ continue;
	5603	+ }
	5604	+ if (btf_type_is_array(member_type)) {
	5605	+ array = btf_type_array(member_type);
	5606	+ if (!array->nelems)
	5607	+ return false;
	5608	+ member_type = btf_type_skip_modifiers(btf, array->type, NULL);
	5609	+ if (!btf_type_is_scalar(member_type))
	5610	+ return false;
	5611	+ continue;
	5612	+ }
	5613	+ if (!btf_type_is_scalar(member_type))
	5614	+ return false;
	5615	+ }
	5616	+ return true;
	5617	+}
	5618	+
5578	5619	static int btf_check_func_arg_match(struct bpf_verifier_env *env,
5579	5620	const struct btf *btf, u32 func_id,
5580	5621	struct bpf_reg_state *regs,
5581	5622	bool ptr_to_mem_ok)
5582	5623	{
5583	5624	struct bpf_verifier_log *log = &env->log;
	5625	+ bool is_kfunc = btf_is_kernel(btf);
5584	5626	const char func_name, ref_tname;
5585	5627	const struct btf_type t, ref_t;
5586	5628	const struct btf_param *args;
@@ -5633,7 +5675,20 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, · 46 unmodified lines
5633	5675
5634	5676	ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id);
5635	5677	ref_tname = btf_name_by_offset(btf, ref_t->name_off);
5636		− if (btf_is_kernel(btf)) {
	5678	+ if (btf_get_prog_ctx_type(log, btf, t,
	5679	+ env->prog->type, i)) {
	5680	+ /* If function expects ctx type in BTF check that caller
	5681	+ * is passing PTR_TO_CTX.
	5682	+ */
	5683	+ if (reg->type != PTR_TO_CTX) {
	5684	+ bpf_log(log,
	5685	+ "arg#%d expected pointer to ctx, but got %s\n",
	5686	+ i, btf_type_str(t));
	5687	+ return -EINVAL;
	5688	+ }
	5689	+ if (check_ctx_reg(env, reg, regno))
	5690	+ return -EINVAL;
	5691	+ } else if (is_kfunc && (reg->type == PTR_TO_BTF_ID \|\| reg2btf_ids[reg->type])) {
5637	5692	const struct btf_type *reg_ref_t;
5638	5693	const struct btf *reg_btf;
5639	5694	const char *reg_ref_tname;
@@ -5649,14 +5704,9 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, · 9 unmodified lines
5649	5704	if (reg->type == PTR_TO_BTF_ID) {
5650	5705	reg_btf = reg->btf;
5651	5706	reg_ref_id = reg->btf_id;
5652		− } else if (reg2btf_ids[reg->type]) {
	5707	+ } else {
5653	5708	reg_btf = btf_vmlinux;
5654	5709	reg_ref_id = *reg2btf_ids[reg->type];
5655		− } else {
5656		− bpf_log(log, "kernel function %s args#%d expected pointer to %s %s but R%d is not a pointer to btf_id\n",
5657		− func_name, i,
5658		− btf_type_str(ref_t), ref_tname, regno);
5659		− return -EINVAL;
5660	5710	}
5661	5711
5662	5712	reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id,
@@ -5672,23 +5722,24 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, · 9 unmodified lines
5672	5722	reg_ref_tname);
5673	5723	return -EINVAL;
5674	5724	}
5675		− } else if (btf_get_prog_ctx_type(log, btf, t,
5676		− env->prog->type, i)) {
5677		− /* If function expects ctx type in BTF check that caller
5678		− * is passing PTR_TO_CTX.
5679		− */
5680		− if (reg->type != PTR_TO_CTX) {
5681		− bpf_log(log,
5682		− "arg#%d expected pointer to ctx, but got %s\n",
5683		− i, btf_type_str(t));
5684		− return -EINVAL;
5685		− }
5686		− if (check_ctx_reg(env, reg, regno))
5687		− return -EINVAL;
5688	5725	} else if (ptr_to_mem_ok) {
5689	5726	const struct btf_type *resolve_ret;
5690	5727	u32 type_size;
5691	5728
	5729	+ if (is_kfunc) {
	5730	+ /* Permit pointer to mem, but only when argument
	5731	+ * type is pointer to scalar, or struct composed
	5732	+ * (recursively) of scalars.
	5733	+ */
	5734	+ if (!btf_type_is_scalar(ref_t) &&
	5735	+ !__btf_type_is_scalar_struct(log, btf, ref_t, 0)) {
	5736	+ bpf_log(log,
	5737	+ "arg#%d pointer type %s %s must point to scalar or struct with scalar\n",
	5738	+ i, btf_type_str(ref_t), ref_tname);
	5739	+ return -EINVAL;
	5740	+ }
	5741	+ }
	5742	+
5692	5743	resolve_ret = btf_resolve_size(btf, ref_t, &type_size);
5693	5744	if (IS_ERR(resolve_ret)) {
5694	5745	bpf_log(log,
@@ -5701,6 +5752,8 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, · 6 unmodified lines
5701	5752	if (check_mem_reg(env, reg, regno, type_size))
5702	5753	return -EINVAL;
5703	5754	} else {
	5755	+ bpf_log(log, "reg type unsupported for arg#%d %sfunction %s#%d\n", i,
	5756	+ is_kfunc ? "kernel " : "", func_name, func_id);
5704	5757	return -EINVAL;
5705	5758	}
5706	5759	}
@@ -5750,7 +5803,7 @@ int btf_check_kfunc_arg_match(struct bpf_verifier_env *env, · 43 unmodified lines
5750	5803	const struct btf *btf, u32 func_id,
5751	5804	struct bpf_reg_state *regs)
5752	5805	{
5753		− return btf_check_func_arg_match(env, btf, func_id, regs, false);
	5806	+ return btf_check_func_arg_match(env, btf, func_id, regs, true);
5754	5807	}
5755	5808
5756	5809	/* Convert BTF of a function into bpf_reg_state if possible
@@ -5858,7 +5911,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, · 101 unmodified lines
5858	5911	return -EINVAL;
5859	5912	}
5860	5913
5861		− reg->type = PTR_TO_MEM_OR_NULL;
	5914	+ reg->type = PTR_TO_MEM \| PTR_MAYBE_NULL;
5862	5915	reg->id = ++env->id_gen;
5863	5916
5864	5917	continue;
@@ -6352,7 +6405,7 @@ const struct bpf_func_proto bpf_btf_find_by_name_kind_proto = { · 487 unmodified lines
6352	6405	.func = bpf_btf_find_by_name_kind,
6353	6406	.gpl_only = false,
6354	6407	.ret_type = RET_INTEGER,
6355		− .arg1_type = ARG_PTR_TO_MEM,
	6408	+ .arg1_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
6356	6409	.arg2_type = ARG_CONST_SIZE,
6357	6410	.arg3_type = ARG_ANYTHING,
6358	6411	.arg4_type = ARG_ANYTHING,
@@ -6534,12 +6587,11 @@ static struct bpf_cand_cache populate_cand_cache(struct bpf_cand_cache cands, · 175 unmodified lines
6534	6587	bpf_free_cands_from_cache(*cc);
6535	6588	*cc = NULL;
6536	6589	}
6537		− new_cands = kmalloc(sizeof_cands(cands->cnt), GFP_KERNEL);
	6590	+ new_cands = kmemdup(cands, sizeof_cands(cands->cnt), GFP_KERNEL);
6538	6591	if (!new_cands) {
6539	6592	bpf_free_cands(cands);
6540	6593	return ERR_PTR(-ENOMEM);
6541	6594	}
6542		− memcpy(new_cands, cands, sizeof_cands(cands->cnt));
6543	6595	/* strdup the name, since it will stay in cache.
6544	6596	* the cands->name points to strings in prog's BTF and the prog can be unloaded.
6545	6597	*/
@@ -6657,7 +6709,7 @@ bpf_core_find_cands(struct bpf_core_ctx *ctx, u32 local_type_id) · 111 unmodified lines
6657	6709
6658	6710	main_btf = bpf_get_btf_vmlinux();
6659	6711	if (IS_ERR(main_btf))
6660		− return (void *)main_btf;
	6712	+ return ERR_CAST(main_btf);
6661	6713
6662	6714	local_type = btf_type_by_id(local_btf, local_type_id);
6663	6715	if (!local_type)
@@ -6684,14 +6736,14 @@ bpf_core_find_cands(struct bpf_core_ctx *ctx, u32 local_type_id) · 20 unmodified lines
6684	6736	/* Attempt to find target candidates in vmlinux BTF first */
6685	6737	cands = bpf_core_add_cands(cands, main_btf, 1);
6686	6738	if (IS_ERR(cands))
6687		− return cands;
	6739	+ return ERR_CAST(cands);
6688	6740
6689	6741	/* cands is a pointer to kmalloced memory here if cands->cnt > 0 */
6690	6742
6691	6743	/* populate cache even when cands->cnt == 0 */
6692	6744	cc = populate_cand_cache(cands, vmlinux_cand_cache, VMLINUX_CAND_CACHE_SIZE);
6693	6745	if (IS_ERR(cc))
6694		− return cc;
	6746	+ return ERR_CAST(cc);
6695	6747
6696	6748	/* if vmlinux BTF has any candidate, don't go for module BTFs */
6697	6749	if (cc->cnt)
@@ -6717,7 +6769,7 @@ check_modules: · 19 unmodified lines
6717	6769	cands = bpf_core_add_cands(cands, mod_btf, btf_nr_types(main_btf));
6718	6770	if (IS_ERR(cands)) {
6719	6771	btf_put(mod_btf);
6720		− return cands;
	6772	+ return ERR_CAST(cands);
6721	6773	}
6722	6774	spin_lock_bh(&btf_idr_lock);
6723	6775	btf_put(mod_btf);

kernel/bpf/cgroup.c+1−1

@@ -1789,7 +1789,7 @@ static const struct bpf_func_proto bpf_sysctl_set_new_value_proto = { · 1788 unmodified lines
1789	1789	.gpl_only = false,
1790	1790	.ret_type = RET_INTEGER,
1791	1791	.arg1_type = ARG_PTR_TO_CTX,
1792		− .arg2_type = ARG_PTR_TO_MEM,
	1792	+ .arg2_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
1793	1793	.arg3_type = ARG_CONST_SIZE,
1794	1794	};
1795	1795

kernel/bpf/cpumap.c+2−2

@@ -195,7 +195,7 @@ static void cpu_map_bpf_prog_run_skb(struct bpf_cpu_map_entry *rcpu, · 194 unmodified lines
195	195	}
196	196	return;
197	197	default:
198		− bpf_warn_invalid_xdp_action(act);
	198	+ bpf_warn_invalid_xdp_action(NULL, rcpu->prog, act);
199	199	fallthrough;
200	200	case XDP_ABORTED:
201	201	trace_xdp_exception(skb->dev, rcpu->prog, act);
@@ -254,7 +254,7 @@ static int cpu_map_bpf_prog_run_xdp(struct bpf_cpu_map_entry *rcpu, · 52 unmodified lines
254	254	}
255	255	break;
256	256	default:
257		− bpf_warn_invalid_xdp_action(act);
	257	+ bpf_warn_invalid_xdp_action(NULL, rcpu->prog, act);
258	258	fallthrough;
259	259	case XDP_DROP:
260	260	xdp_return_frame(xdpf);

kernel/bpf/devmap.c+2−2

@@ -348,7 +348,7 @@ static int dev_map_bpf_prog_run(struct bpf_prog *xdp_prog, · 347 unmodified lines
348	348	frames[nframes++] = xdpf;
349	349	break;
350	350	default:
351		− bpf_warn_invalid_xdp_action(act);
	351	+ bpf_warn_invalid_xdp_action(NULL, xdp_prog, act);
352	352	fallthrough;
353	353	case XDP_ABORTED:
354	354	trace_xdp_exception(dev, xdp_prog, act);
@@ -507,7 +507,7 @@ static u32 dev_map_bpf_prog_run_skb(struct sk_buff *skb, struct bpf_dtab_netdev · 152 unmodified lines
507	507	__skb_push(skb, skb->mac_len);
508	508	break;
509	509	default:
510		− bpf_warn_invalid_xdp_action(act);
	510	+ bpf_warn_invalid_xdp_action(NULL, dst->xdp_prog, act);
511	511	fallthrough;
512	512	case XDP_ABORTED:
513	513	trace_xdp_exception(dst->dev, dst->xdp_prog, act);

kernel/bpf/helpers.c+23−6

@@ -2,6 +2,7 @@ · 1 unmodified line
2	2	/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
3	3	*/
4	4	#include <linux/bpf.h>
	5	+#include <linux/bpf-cgroup.h>
5	6	#include <linux/rcupdate.h>
6	7	#include <linux/random.h>
7	8	#include <linux/smp.h>
@@ -530,7 +531,7 @@ const struct bpf_func_proto bpf_strtol_proto = { · 522 unmodified lines
530	531	.func = bpf_strtol,
531	532	.gpl_only = false,
532	533	.ret_type = RET_INTEGER,
533		− .arg1_type = ARG_PTR_TO_MEM,
	534	+ .arg1_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
534	535	.arg2_type = ARG_CONST_SIZE,
535	536	.arg3_type = ARG_ANYTHING,
536	537	.arg4_type = ARG_PTR_TO_LONG,
@@ -558,13 +559,27 @@ const struct bpf_func_proto bpf_strtoul_proto = { · 21 unmodified lines
558	559	.func = bpf_strtoul,
559	560	.gpl_only = false,
560	561	.ret_type = RET_INTEGER,
561		− .arg1_type = ARG_PTR_TO_MEM,
	562	+ .arg1_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
562	563	.arg2_type = ARG_CONST_SIZE,
563	564	.arg3_type = ARG_ANYTHING,
564	565	.arg4_type = ARG_PTR_TO_LONG,
565	566	};
566	567	#endif
567	568
	569	+BPF_CALL_3(bpf_strncmp, const char , s1, u32, s1_sz, const char , s2)
	570	+{
	571	+ return strncmp(s1, s2, s1_sz);
	572	+}
	573	+
	574	+const struct bpf_func_proto bpf_strncmp_proto = {
	575	+ .func = bpf_strncmp,
	576	+ .gpl_only = false,
	577	+ .ret_type = RET_INTEGER,
	578	+ .arg1_type = ARG_PTR_TO_MEM,
	579	+ .arg2_type = ARG_CONST_SIZE,
	580	+ .arg3_type = ARG_PTR_TO_CONST_STR,
	581	+};
	582	+
568	583	BPF_CALL_4(bpf_get_ns_current_pid_tgid, u64, dev, u64, ino,
569	584	struct bpf_pidns_info *, nsdata, u32, size)
570	585	{
@@ -630,7 +645,7 @@ const struct bpf_func_proto bpf_event_output_data_proto = { · 59 unmodified lines
630	645	.arg1_type = ARG_PTR_TO_CTX,
631	646	.arg2_type = ARG_CONST_MAP_PTR,
632	647	.arg3_type = ARG_ANYTHING,
633		− .arg4_type = ARG_PTR_TO_MEM,
	648	+ .arg4_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
634	649	.arg5_type = ARG_CONST_SIZE_OR_ZERO,
635	650	};
636	651
@@ -667,7 +682,7 @@ BPF_CALL_2(bpf_per_cpu_ptr, const void *, ptr, u32, cpu) · 30 unmodified lines
667	682	const struct bpf_func_proto bpf_per_cpu_ptr_proto = {
668	683	.func = bpf_per_cpu_ptr,
669	684	.gpl_only = false,
670		− .ret_type = RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL,
	685	+ .ret_type = RET_PTR_TO_MEM_OR_BTF_ID \| PTR_MAYBE_NULL \| MEM_RDONLY,
671	686	.arg1_type = ARG_PTR_TO_PERCPU_BTF_ID,
672	687	.arg2_type = ARG_ANYTHING,
673	688	};
@@ -680,7 +695,7 @@ BPF_CALL_1(bpf_this_cpu_ptr, const void *, percpu_ptr) · 6 unmodified lines
680	695	const struct bpf_func_proto bpf_this_cpu_ptr_proto = {
681	696	.func = bpf_this_cpu_ptr,
682	697	.gpl_only = false,
683		− .ret_type = RET_PTR_TO_MEM_OR_BTF_ID,
	698	+ .ret_type = RET_PTR_TO_MEM_OR_BTF_ID \| MEM_RDONLY,
684	699	.arg1_type = ARG_PTR_TO_PERCPU_BTF_ID,
685	700	};
686	701
@@ -1011,7 +1026,7 @@ const struct bpf_func_proto bpf_snprintf_proto = { · 324 unmodified lines
1011	1026	.arg1_type = ARG_PTR_TO_MEM_OR_NULL,
1012	1027	.arg2_type = ARG_CONST_SIZE_OR_ZERO,
1013	1028	.arg3_type = ARG_PTR_TO_CONST_STR,
1014		− .arg4_type = ARG_PTR_TO_MEM_OR_NULL,
	1029	+ .arg4_type = ARG_PTR_TO_MEM \| PTR_MAYBE_NULL \| MEM_RDONLY,
1015	1030	.arg5_type = ARG_CONST_SIZE_OR_ZERO,
1016	1031	};
1017	1032
@@ -1378,6 +1393,8 @@ bpf_base_func_proto(enum bpf_func_id func_id) · 360 unmodified lines
1378	1393	return &bpf_for_each_map_elem_proto;
1379	1394	case BPF_FUNC_loop:
1380	1395	return &bpf_loop_proto;
	1396	+ case BPF_FUNC_strncmp:
	1397	+ return &bpf_strncmp_proto;
1381	1398	default:
1382	1399	break;
1383	1400	}

kernel/bpf/local_storage.c+1−2

@@ -163,8 +163,7 @@ static int cgroup_storage_update_elem(struct bpf_map map, void key, · 162 unmodified lines
163	163	return 0;
164	164	}
165	165
166		− new = bpf_map_kmalloc_node(map, sizeof(struct bpf_storage_buffer) +
167		− map->value_size,
	166	+ new = bpf_map_kmalloc_node(map, struct_size(new, data, map->value_size),
168	167	__GFP_ZERO \| GFP_ATOMIC \| __GFP_NOWARN,
169	168	map->numa_node);
170	169	if (!new)

kernel/bpf/lpm_trie.c+1−1

@@ -412,7 +412,7 @@ static int trie_update_elem(struct bpf_map *map, · 411 unmodified lines
412	412	rcu_assign_pointer(im_node->child[1], node);
413	413	}
414	414
415		− /* Finally, assign the intermediate node to the determined spot */
	415	+ /* Finally, assign the intermediate node to the determined slot */
416	416	rcu_assign_pointer(*slot, im_node);
417	417
418	418	out:

kernel/bpf/map_iter.c+2−2

@@ -174,9 +174,9 @@ static const struct bpf_iter_reg bpf_map_elem_reg_info = { · 173 unmodified lines
174	174	.ctx_arg_info_size = 2,
175	175	.ctx_arg_info = {
176	176	{ offsetof(struct bpf_iter__bpf_map_elem, key),
177		− PTR_TO_RDONLY_BUF_OR_NULL },
	177	+ PTR_TO_BUF \| PTR_MAYBE_NULL \| MEM_RDONLY },
178	178	{ offsetof(struct bpf_iter__bpf_map_elem, value),
179		− PTR_TO_RDWR_BUF_OR_NULL },
	179	+ PTR_TO_BUF \| PTR_MAYBE_NULL },
180	180	},
181	181	};
182	182

kernel/bpf/net_namespace.c+1−0

@@ -1,6 +1,7 @@
1	1	// SPDX-License-Identifier: GPL-2.0
2	2
3	3	#include <linux/bpf.h>
	4	+#include <linux/bpf-netns.h>
4	5	#include <linux/filter.h>
5	6	#include <net/net_namespace.h>
6	7

kernel/bpf/reuseport_array.c+1−5

@@ -152,16 +152,12 @@ static struct bpf_map reuseport_array_alloc(union bpf_attr attr) · 151 unmodified lines
152	152	{
153	153	int numa_node = bpf_map_attr_numa_node(attr);
154	154	struct reuseport_array *array;
155		− u64 array_size;
156	155
157	156	if (!bpf_capable())
158	157	return ERR_PTR(-EPERM);
159	158
160		− array_size = sizeof(*array);
161		− array_size += (u64)attr->max_entries * sizeof(struct sock *);
162		−
163	159	/* allocate all map elements and zero-initialize them */
164		− array = bpf_map_area_alloc(array_size, numa_node);
	160	+ array = bpf_map_area_alloc(struct_size(array, ptrs, attr->max_entries), numa_node);
165	161	if (!array)
166	162	return ERR_PTR(-ENOMEM);
167	163

kernel/bpf/ringbuf.c+1−1

@@ -444,7 +444,7 @@ const struct bpf_func_proto bpf_ringbuf_output_proto = { · 443 unmodified lines
444	444	.func = bpf_ringbuf_output,
445	445	.ret_type = RET_INTEGER,
446	446	.arg1_type = ARG_CONST_MAP_PTR,
447		− .arg2_type = ARG_PTR_TO_MEM,
	447	+ .arg2_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
448	448	.arg3_type = ARG_CONST_SIZE_OR_ZERO,
449	449	.arg4_type = ARG_ANYTHING,
450	450	};

kernel/bpf/syscall.c+2−1

@@ -2,6 +2,7 @@ · 1 unmodified line
2	2	/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
3	3	*/
4	4	#include <linux/bpf.h>
	5	+#include <linux/bpf-cgroup.h>
5	6	#include <linux/bpf_trace.h>
6	7	#include <linux/bpf_lirc.h>
7	8	#include <linux/bpf_verifier.h>
@@ -4772,7 +4773,7 @@ static const struct bpf_func_proto bpf_sys_bpf_proto = { · 4764 unmodified lines
4772	4773	.gpl_only = false,
4773	4774	.ret_type = RET_INTEGER,
4774	4775	.arg1_type = ARG_ANYTHING,
4775		− .arg2_type = ARG_PTR_TO_MEM,
	4776	+ .arg2_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
4776	4777	.arg3_type = ARG_CONST_SIZE,
4777	4778	};
4778	4779

kernel/bpf/trampoline.c+8−0

@@ -27,6 +27,14 @@ static struct hlist_head trampoline_table[TRAMPOLINE_TABLE_SIZE]; · 26 unmodified lines
27	27	/* serializes access to trampoline_table */
28	28	static DEFINE_MUTEX(trampoline_mutex);
29	29
	30	+bool bpf_prog_has_trampoline(const struct bpf_prog *prog)
	31	+{
	32	+ enum bpf_attach_type eatype = prog->expected_attach_type;
	33	+
	34	+ return eatype == BPF_TRACE_FENTRY \|\| eatype == BPF_TRACE_FEXIT \|\|
	35	+ eatype == BPF_MODIFY_RETURN;
	36	+}
	37	+
30	38	void *bpf_jit_alloc_exec_page(void)
31	39	{
32	40	void *image;

kernel/bpf/verifier.c+415−281

@@ -4,6 +4,7 @@ · 3 unmodified lines
4	4	* Copyright (c) 2018 Covalent IO, Inc. http://covalent.io
5	5	*/
6	6	#include <uapi/linux/btf.h>
	7	+#include <linux/bpf-cgroup.h>
7	8	#include <linux/kernel.h>
8	9	#include <linux/types.h>
9	10	#include <linux/slab.h>
@@ -441,18 +442,6 @@ static bool reg_type_not_null(enum bpf_reg_type type) · 431 unmodified lines
441	442	type == PTR_TO_SOCK_COMMON;
442	443	}
443	444
444		−static bool reg_type_may_be_null(enum bpf_reg_type type)
445		−{
446		− return type == PTR_TO_MAP_VALUE_OR_NULL \|\|
447		− type == PTR_TO_SOCKET_OR_NULL \|\|
448		− type == PTR_TO_SOCK_COMMON_OR_NULL \|\|
449		− type == PTR_TO_TCP_SOCK_OR_NULL \|\|
450		− type == PTR_TO_BTF_ID_OR_NULL \|\|
451		− type == PTR_TO_MEM_OR_NULL \|\|
452		− type == PTR_TO_RDONLY_BUF_OR_NULL \|\|
453		− type == PTR_TO_RDWR_BUF_OR_NULL;
454		−}
455		−
456	445	static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg)
457	446	{
458	447	return reg->type == PTR_TO_MAP_VALUE &&
@@ -461,12 +450,14 @@ static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg) · 2 unmodified lines
461	450
462	451	static bool reg_type_may_be_refcounted_or_null(enum bpf_reg_type type)
463	452	{
464		− return type == PTR_TO_SOCKET \|\|
465		− type == PTR_TO_SOCKET_OR_NULL \|\|
466		− type == PTR_TO_TCP_SOCK \|\|
467		− type == PTR_TO_TCP_SOCK_OR_NULL \|\|
468		− type == PTR_TO_MEM \|\|
469		− type == PTR_TO_MEM_OR_NULL;
	453	+ return base_type(type) == PTR_TO_SOCKET \|\|
	454	+ base_type(type) == PTR_TO_TCP_SOCK \|\|
	455	+ base_type(type) == PTR_TO_MEM;
	456	+}
	457	+
	458	+static bool type_is_rdonly_mem(u32 type)
	459	+{
	460	+ return type & MEM_RDONLY;
470	461	}
471	462
472	463	static bool arg_type_may_be_refcounted(enum bpf_arg_type type)
@@ -474,14 +465,9 @@ static bool arg_type_may_be_refcounted(enum bpf_arg_type type) · 1 unmodified line
474	465	return type == ARG_PTR_TO_SOCK_COMMON;
475	466	}
476	467
477		−static bool arg_type_may_be_null(enum bpf_arg_type type)
	468	+static bool type_may_be_null(u32 type)
478	469	{
479		− return type == ARG_PTR_TO_MAP_VALUE_OR_NULL \|\|
480		− type == ARG_PTR_TO_MEM_OR_NULL \|\|
481		− type == ARG_PTR_TO_CTX_OR_NULL \|\|
482		− type == ARG_PTR_TO_SOCKET_OR_NULL \|\|
483		− type == ARG_PTR_TO_ALLOC_MEM_OR_NULL \|\|
484		− type == ARG_PTR_TO_STACK_OR_NULL;
	470	+ return type & PTR_MAYBE_NULL;
485	471	}
486	472
487	473	/* Determine whether the function releases some resources allocated by another
@@ -541,39 +527,54 @@ static bool is_cmpxchg_insn(const struct bpf_insn *insn) · 53 unmodified lines
541	527	insn->imm == BPF_CMPXCHG;
542	528	}
543	529
544		−/* string representation of 'enum bpf_reg_type' */
545		−static const char * const reg_type_str[] = {
546		− [NOT_INIT] = "?",
547		− [SCALAR_VALUE] = "inv",
548		− [PTR_TO_CTX] = "ctx",
549		− [CONST_PTR_TO_MAP] = "map_ptr",
550		− [PTR_TO_MAP_VALUE] = "map_value",
551		− [PTR_TO_MAP_VALUE_OR_NULL] = "map_value_or_null",
552		− [PTR_TO_STACK] = "fp",
553		− [PTR_TO_PACKET] = "pkt",
554		− [PTR_TO_PACKET_META] = "pkt_meta",
555		− [PTR_TO_PACKET_END] = "pkt_end",
556		− [PTR_TO_FLOW_KEYS] = "flow_keys",
557		− [PTR_TO_SOCKET] = "sock",
558		− [PTR_TO_SOCKET_OR_NULL] = "sock_or_null",
559		− [PTR_TO_SOCK_COMMON] = "sock_common",
560		− [PTR_TO_SOCK_COMMON_OR_NULL] = "sock_common_or_null",
561		− [PTR_TO_TCP_SOCK] = "tcp_sock",
562		− [PTR_TO_TCP_SOCK_OR_NULL] = "tcp_sock_or_null",
563		− [PTR_TO_TP_BUFFER] = "tp_buffer",
564		− [PTR_TO_XDP_SOCK] = "xdp_sock",
565		− [PTR_TO_BTF_ID] = "ptr_",
566		− [PTR_TO_BTF_ID_OR_NULL] = "ptr_or_null_",
567		− [PTR_TO_PERCPU_BTF_ID] = "percpu_ptr_",
568		− [PTR_TO_MEM] = "mem",
569		− [PTR_TO_MEM_OR_NULL] = "mem_or_null",
570		− [PTR_TO_RDONLY_BUF] = "rdonly_buf",
571		− [PTR_TO_RDONLY_BUF_OR_NULL] = "rdonly_buf_or_null",
572		− [PTR_TO_RDWR_BUF] = "rdwr_buf",
573		− [PTR_TO_RDWR_BUF_OR_NULL] = "rdwr_buf_or_null",
574		− [PTR_TO_FUNC] = "func",
575		− [PTR_TO_MAP_KEY] = "map_key",
576		−};
	530	+/* string representation of 'enum bpf_reg_type'
	531	+ *
	532	+ * Note that reg_type_str() can not appear more than once in a single verbose()
	533	+ * statement.
	534	+ */
	535	+static const char reg_type_str(struct bpf_verifier_env env,
	536	+ enum bpf_reg_type type)
	537	+{
	538	+ char postfix[16] = {0}, prefix[16] = {0};
	539	+ static const char * const str[] = {
	540	+ [NOT_INIT] = "?",
	541	+ [SCALAR_VALUE] = "inv",
	542	+ [PTR_TO_CTX] = "ctx",
	543	+ [CONST_PTR_TO_MAP] = "map_ptr",
	544	+ [PTR_TO_MAP_VALUE] = "map_value",
	545	+ [PTR_TO_STACK] = "fp",
	546	+ [PTR_TO_PACKET] = "pkt",
	547	+ [PTR_TO_PACKET_META] = "pkt_meta",
	548	+ [PTR_TO_PACKET_END] = "pkt_end",
	549	+ [PTR_TO_FLOW_KEYS] = "flow_keys",
	550	+ [PTR_TO_SOCKET] = "sock",
	551	+ [PTR_TO_SOCK_COMMON] = "sock_common",
	552	+ [PTR_TO_TCP_SOCK] = "tcp_sock",
	553	+ [PTR_TO_TP_BUFFER] = "tp_buffer",
	554	+ [PTR_TO_XDP_SOCK] = "xdp_sock",
	555	+ [PTR_TO_BTF_ID] = "ptr_",
	556	+ [PTR_TO_PERCPU_BTF_ID] = "percpu_ptr_",
	557	+ [PTR_TO_MEM] = "mem",
	558	+ [PTR_TO_BUF] = "buf",
	559	+ [PTR_TO_FUNC] = "func",
	560	+ [PTR_TO_MAP_KEY] = "map_key",
	561	+ };
	562	+
	563	+ if (type & PTR_MAYBE_NULL) {
	564	+ if (base_type(type) == PTR_TO_BTF_ID \|\|
	565	+ base_type(type) == PTR_TO_PERCPU_BTF_ID)
	566	+ strncpy(postfix, "or_null_", 16);
	567	+ else
	568	+ strncpy(postfix, "_or_null", 16);
	569	+ }
	570	+
	571	+ if (type & MEM_RDONLY)
	572	+ strncpy(prefix, "rdonly_", 16);
	573	+
	574	+ snprintf(env->type_str_buf, TYPE_STR_BUF_LEN, "%s%s%s",
	575	+ prefix, str[base_type(type)], postfix);
	576	+ return env->type_str_buf;
	577	+}
577	578
578	579	static char slot_type_char[] = {
579	580	[STACK_INVALID] = '?',
@@ -608,6 +609,44 @@ static const char kernel_type_name(const struct btf btf, u32 id) · 28 unmodified lines
608	609	return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off);
609	610	}
610	611
	612	+static void mark_reg_scratched(struct bpf_verifier_env *env, u32 regno)
	613	+{
	614	+ env->scratched_regs \|= 1U << regno;
	615	+}
	616	+
	617	+static void mark_stack_slot_scratched(struct bpf_verifier_env *env, u32 spi)
	618	+{
	619	+ env->scratched_stack_slots \|= 1UL << spi;
	620	+}
	621	+
	622	+static bool reg_scratched(const struct bpf_verifier_env *env, u32 regno)
	623	+{
	624	+ return (env->scratched_regs >> regno) & 1;
	625	+}
	626	+
	627	+static bool stack_slot_scratched(const struct bpf_verifier_env *env, u64 regno)
	628	+{
	629	+ return (env->scratched_stack_slots >> regno) & 1;
	630	+}
	631	+
	632	+static bool verifier_state_scratched(const struct bpf_verifier_env *env)
	633	+{
	634	+ return env->scratched_regs \|\| env->scratched_stack_slots;
	635	+}
	636	+
	637	+static void mark_verifier_state_clean(struct bpf_verifier_env *env)
	638	+{
	639	+ env->scratched_regs = 0U;
	640	+ env->scratched_stack_slots = 0UL;
	641	+}
	642	+
	643	+/* Used for printing the entire verifier state. */
	644	+static void mark_verifier_state_scratched(struct bpf_verifier_env *env)
	645	+{
	646	+ env->scratched_regs = ~0U;
	647	+ env->scratched_stack_slots = ~0UL;
	648	+}
	649	+
611	650	/* The reg state of a pointer or a bounded scalar was saved when
612	651	* it was spilled to the stack.
613	652	*/
@@ -623,7 +662,8 @@ static void scrub_spilled_slot(u8 *stype) · 9 unmodified lines
623	662	}
624	663
625	664	static void print_verifier_state(struct bpf_verifier_env *env,
626		− const struct bpf_func_state *state)
	665	+ const struct bpf_func_state *state,
	666	+ bool print_all)
627	667	{
628	668	const struct bpf_reg_state *reg;
629	669	enum bpf_reg_type t;
@@ -636,9 +676,11 @@ static void print_verifier_state(struct bpf_verifier_env *env, · 6 unmodified lines
636	676	t = reg->type;
637	677	if (t == NOT_INIT)
638	678	continue;
	679	+ if (!print_all && !reg_scratched(env, i))
	680	+ continue;
639	681	verbose(env, " R%d", i);
640	682	print_liveness(env, reg->live);
641		− verbose(env, "=%s", reg_type_str[t]);
	683	+ verbose(env, "=%s", reg_type_str(env, t));
642	684	if (t == SCALAR_VALUE && reg->precise)
643	685	verbose(env, "P");
644	686	if ((t == SCALAR_VALUE \|\| t == PTR_TO_STACK) &&
@@ -646,9 +688,8 @@ static void print_verifier_state(struct bpf_verifier_env *env, · 1 unmodified line
646	688	/* reg->off should be 0 for SCALAR_VALUE */
647	689	verbose(env, "%lld", reg->var_off.value + reg->off);
648	690	} else {
649		− if (t == PTR_TO_BTF_ID \|\|
650		− t == PTR_TO_BTF_ID_OR_NULL \|\|
651		− t == PTR_TO_PERCPU_BTF_ID)
	691	+ if (base_type(t) == PTR_TO_BTF_ID \|\|
	692	+ base_type(t) == PTR_TO_PERCPU_BTF_ID)
652	693	verbose(env, "%s", kernel_type_name(reg->btf, reg->btf_id));
653	694	verbose(env, "(id=%d", reg->id);
654	695	if (reg_type_may_be_refcounted_or_null(t))
@@ -657,10 +698,9 @@ static void print_verifier_state(struct bpf_verifier_env *env, · 2 unmodified lines
657	698	verbose(env, ",off=%d", reg->off);
658	699	if (type_is_pkt_pointer(t))
659	700	verbose(env, ",r=%d", reg->range);
660		− else if (t == CONST_PTR_TO_MAP \|\|
661		− t == PTR_TO_MAP_KEY \|\|
662		− t == PTR_TO_MAP_VALUE \|\|
663		− t == PTR_TO_MAP_VALUE_OR_NULL)
	701	+ else if (base_type(t) == CONST_PTR_TO_MAP \|\|
	702	+ base_type(t) == PTR_TO_MAP_KEY \|\|
	703	+ base_type(t) == PTR_TO_MAP_VALUE)
664	704	verbose(env, ",ks=%d,vs=%d",
665	705	reg->map_ptr->key_size,
666	706	reg->map_ptr->value_size);
@@ -725,12 +765,14 @@ static void print_verifier_state(struct bpf_verifier_env *env, · 58 unmodified lines
725	765	types_buf[BPF_REG_SIZE] = 0;
726	766	if (!valid)
727	767	continue;
	768	+ if (!print_all && !stack_slot_scratched(env, i))
	769	+ continue;
728	770	verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE);
729	771	print_liveness(env, state->stack[i].spilled_ptr.live);
730	772	if (is_spilled_reg(&state->stack[i])) {
731	773	reg = &state->stack[i].spilled_ptr;
732	774	t = reg->type;
733		− verbose(env, "=%s", reg_type_str[t]);
	775	+ verbose(env, "=%s", reg_type_str(env, t));
734	776	if (t == SCALAR_VALUE && reg->precise)
735	777	verbose(env, "P");
736	778	if (t == SCALAR_VALUE && tnum_is_const(reg->var_off))
@@ -750,6 +792,26 @@ static void print_verifier_state(struct bpf_verifier_env *env, · 13 unmodified lines
750	792	if (state->in_async_callback_fn)
751	793	verbose(env, " async_cb");
752	794	verbose(env, "\n");
	795	+ mark_verifier_state_clean(env);
	796	+}
	797	+
	798	+static inline u32 vlog_alignment(u32 pos)
	799	+{
	800	+ return round_up(max(pos + BPF_LOG_MIN_ALIGNMENT / 2, BPF_LOG_ALIGNMENT),
	801	+ BPF_LOG_MIN_ALIGNMENT) - pos - 1;
	802	+}
	803	+
	804	+static void print_insn_state(struct bpf_verifier_env *env,
	805	+ const struct bpf_func_state *state)
	806	+{
	807	+ if (env->prev_log_len && env->prev_log_len == env->log.len_used) {
	808	+ /* remove new line character */
	809	+ bpf_vlog_reset(&env->log, env->prev_log_len - 1);
	810	+ verbose(env, "%*c;", vlog_alignment(env->prev_insn_print_len), ' ');
	811	+ } else {
	812	+ verbose(env, "%d:", env->insn_idx);
	813	+ }
	814	+ print_verifier_state(env, state, false);
753	815	}
754	816
755	817	/* copy array src of length n * size bytes to dst. dst is reallocated if it's too
@@ -1143,8 +1205,7 @@ static void mark_reg_known_zero(struct bpf_verifier_env *env, · 387 unmodified lines
1143	1205
1144	1206	static void mark_ptr_not_null_reg(struct bpf_reg_state *reg)
1145	1207	{
1146		− switch (reg->type) {
1147		− case PTR_TO_MAP_VALUE_OR_NULL: {
	1208	+ if (base_type(reg->type) == PTR_TO_MAP_VALUE) {
1148	1209	const struct bpf_map *map = reg->map_ptr;
1149	1210
1150	1211	if (map->inner_map_meta) {
@@ -1163,32 +1224,10 @@ static void mark_ptr_not_null_reg(struct bpf_reg_state *reg) · 12 unmodified lines
1163	1224	} else {
1164	1225	reg->type = PTR_TO_MAP_VALUE;
1165	1226	}
1166		− break;
1167		− }
1168		− case PTR_TO_SOCKET_OR_NULL:
1169		− reg->type = PTR_TO_SOCKET;
1170		− break;
1171		− case PTR_TO_SOCK_COMMON_OR_NULL:
1172		− reg->type = PTR_TO_SOCK_COMMON;
1173		− break;
1174		− case PTR_TO_TCP_SOCK_OR_NULL:
1175		− reg->type = PTR_TO_TCP_SOCK;
1176		− break;
1177		− case PTR_TO_BTF_ID_OR_NULL:
1178		− reg->type = PTR_TO_BTF_ID;
1179		− break;
1180		− case PTR_TO_MEM_OR_NULL:
1181		− reg->type = PTR_TO_MEM;
1182		− break;
1183		− case PTR_TO_RDONLY_BUF_OR_NULL:
1184		− reg->type = PTR_TO_RDONLY_BUF;
1185		− break;
1186		− case PTR_TO_RDWR_BUF_OR_NULL:
1187		− reg->type = PTR_TO_RDWR_BUF;
1188		− break;
1189		− default:
1190		− WARN_ONCE(1, "unknown nullable register type");
	1227	+ return;
1191	1228	}
	1229	+
	1230	+ reg->type &= ~PTR_MAYBE_NULL;
1192	1231	}
1193	1232
1194	1233	static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg)
@@ -1546,6 +1585,7 @@ static void init_func_state(struct bpf_verifier_env *env, · 351 unmodified lines
1546	1585	state->frameno = frameno;
1547	1586	state->subprogno = subprogno;
1548	1587	init_reg_state(env, state);
	1588	+ mark_verifier_state_scratched(env);
1549	1589	}
1550	1590
1551	1591	/* Similar to push_stack(), but for async callbacks */
@@ -2049,7 +2089,7 @@ static int mark_reg_read(struct bpf_verifier_env *env, · 497 unmodified lines
2049	2089	break;
2050	2090	if (parent->live & REG_LIVE_DONE) {
2051	2091	verbose(env, "verifier BUG type %s var_off %lld off %d\n",
2052		− reg_type_str[parent->type],
	2092	+ reg_type_str(env, parent->type),
2053	2093	parent->var_off.value, parent->off);
2054	2094	return -EFAULT;
2055	2095	}
@@ -2233,6 +2273,8 @@ static int check_reg_arg(struct bpf_verifier_env *env, u32 regno, · 177 unmodified lines
2233	2273	return -EINVAL;
2234	2274	}
2235	2275
	2276	+ mark_reg_scratched(env, regno);
	2277	+
2236	2278	reg = &regs[regno];
2237	2279	rw64 = is_reg64(env, insn, regno, reg, t);
2238	2280	if (t == SRC_OP) {
@@ -2337,7 +2379,7 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, · 98 unmodified lines
2337	2379
2338	2380	if (insn->code == 0)
2339	2381	return 0;
2340		− if (env->log.level & BPF_LOG_LEVEL) {
	2382	+ if (env->log.level & BPF_LOG_LEVEL2) {
2341	2383	verbose(env, "regs=%x stack=%llx before ", reg_mask, stack_mask);
2342	2384	verbose(env, "%d: ", idx);
2343	2385	print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
@@ -2591,7 +2633,7 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int regno, · 247 unmodified lines
2591	2633	DECLARE_BITMAP(mask, 64);
2592	2634	u32 history = st->jmp_history_cnt;
2593	2635
2594		− if (env->log.level & BPF_LOG_LEVEL)
	2636	+ if (env->log.level & BPF_LOG_LEVEL2)
2595	2637	verbose(env, "last_idx %d first_idx %d\n", last_idx, first_idx);
2596	2638	for (i = last_idx;;) {
2597	2639	if (skip_first) {
@@ -2678,11 +2720,11 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int regno, · 80 unmodified lines
2678	2720	new_marks = true;
2679	2721	reg->precise = true;
2680	2722	}
2681		− if (env->log.level & BPF_LOG_LEVEL) {
2682		− print_verifier_state(env, func);
2683		− verbose(env, "parent %s regs=%x stack=%llx marks\n",
	2723	+ if (env->log.level & BPF_LOG_LEVEL2) {
	2724	+ verbose(env, "parent %s regs=%x stack=%llx marks:",
2684	2725	new_marks ? "didn't have" : "already had",
2685	2726	reg_mask, stack_mask);
	2727	+ print_verifier_state(env, func, true);
2686	2728	}
2687	2729
2688	2730	if (!reg_mask && !stack_mask)
@@ -2708,9 +2750,8 @@ static int mark_chain_precision_stack(struct bpf_verifier_env *env, int spi) · 19 unmodified lines
2708	2750
2709	2751	static bool is_spillable_regtype(enum bpf_reg_type type)
2710	2752	{
2711		− switch (type) {
	2753	+ switch (base_type(type)) {
2712	2754	case PTR_TO_MAP_VALUE:
2713		− case PTR_TO_MAP_VALUE_OR_NULL:
2714	2755	case PTR_TO_STACK:
2715	2756	case PTR_TO_CTX:
2716	2757	case PTR_TO_PACKET:
@@ -2719,21 +2760,13 @@ static bool is_spillable_regtype(enum bpf_reg_type type) · 2 unmodified lines
2719	2760	case PTR_TO_FLOW_KEYS:
2720	2761	case CONST_PTR_TO_MAP:
2721	2762	case PTR_TO_SOCKET:
2722		− case PTR_TO_SOCKET_OR_NULL:
2723	2763	case PTR_TO_SOCK_COMMON:
2724		− case PTR_TO_SOCK_COMMON_OR_NULL:
2725	2764	case PTR_TO_TCP_SOCK:
2726		− case PTR_TO_TCP_SOCK_OR_NULL:
2727	2765	case PTR_TO_XDP_SOCK:
2728	2766	case PTR_TO_BTF_ID:
2729		− case PTR_TO_BTF_ID_OR_NULL:
2730		− case PTR_TO_RDONLY_BUF:
2731		− case PTR_TO_RDONLY_BUF_OR_NULL:
2732		− case PTR_TO_RDWR_BUF:
2733		− case PTR_TO_RDWR_BUF_OR_NULL:
	2767	+ case PTR_TO_BUF:
2734	2768	case PTR_TO_PERCPU_BTF_ID:
2735	2769	case PTR_TO_MEM:
2736		− case PTR_TO_MEM_OR_NULL:
2737	2770	case PTR_TO_FUNC:
2738	2771	case PTR_TO_MAP_KEY:
2739	2772	return true;
@@ -2838,6 +2871,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, · 98 unmodified lines
2838	2871	env->insn_aux_data[insn_idx].sanitize_stack_spill = true;
2839	2872	}
2840	2873
	2874	+ mark_stack_slot_scratched(env, spi);
2841	2875	if (reg && !(off % BPF_REG_SIZE) && register_is_bounded(reg) &&
2842	2876	!register_is_null(reg) && env->bpf_capable) {
2843	2877	if (dst_reg != BPF_REG_FP) {
@@ -2959,6 +2993,7 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env, · 115 unmodified lines
2959	2993	slot = -i - 1;
2960	2994	spi = slot / BPF_REG_SIZE;
2961	2995	stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE];
	2996	+ mark_stack_slot_scratched(env, spi);
2962	2997
2963	2998	if (!env->allow_ptr_leaks
2964	2999	&& *stype != NOT_INIT
@@ -3375,11 +3410,8 @@ static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno, · 410 unmodified lines
3375	3410	/* We may have adjusted the register pointing to memory region, so we
3376	3411	* need to try adding each of min_value and max_value to off
3377	3412	* to make sure our theoretical access will be safe.
3378		− */
3379		− if (env->log.level & BPF_LOG_LEVEL)
3380		− print_verifier_state(env, state);
3381		−
3382		− /* The minimum value is only important with signed
	3413	+ *
	3414	+ * The minimum value is only important with signed
3383	3415	* comparisons where we can't assume the floor of a
3384	3416	* value is 0. If we are using signed variables for our
3385	3417	* index'es we need to make sure that whatever we use
@@ -3574,7 +3606,7 @@ static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, · 188 unmodified lines
3574	3606	*/
3575	3607	*reg_type = info.reg_type;
3576	3608
3577		− if (reg_type == PTR_TO_BTF_ID \|\| reg_type == PTR_TO_BTF_ID_OR_NULL) {
	3609	+ if (base_type(*reg_type) == PTR_TO_BTF_ID) {
3578	3610	*btf = info.btf;
3579	3611	*btf_id = info.btf_id;
3580	3612	} else {
@@ -3642,7 +3674,7 @@ static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, · 61 unmodified lines
3642	3674	}
3643	3675
3644	3676	verbose(env, "R%d invalid %s access off=%d size=%d\n",
3645		− regno, reg_type_str[reg->type], off, size);
	3677	+ regno, reg_type_str(env, reg->type), off, size);
3646	3678
3647	3679	return -EACCES;
3648	3680	}
@@ -4369,15 +4401,30 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 720 unmodified lines
4369	4401	mark_reg_unknown(env, regs, value_regno);
4370	4402	}
4371	4403	}
4372		− } else if (reg->type == PTR_TO_MEM) {
	4404	+ } else if (base_type(reg->type) == PTR_TO_MEM) {
	4405	+ bool rdonly_mem = type_is_rdonly_mem(reg->type);
	4406	+
	4407	+ if (type_may_be_null(reg->type)) {
	4408	+ verbose(env, "R%d invalid mem access '%s'\n", regno,
	4409	+ reg_type_str(env, reg->type));
	4410	+ return -EACCES;
	4411	+ }
	4412	+
	4413	+ if (t == BPF_WRITE && rdonly_mem) {
	4414	+ verbose(env, "R%d cannot write into %s\n",
	4415	+ regno, reg_type_str(env, reg->type));
	4416	+ return -EACCES;
	4417	+ }
	4418	+
4373	4419	if (t == BPF_WRITE && value_regno >= 0 &&
4374	4420	is_pointer_value(env, value_regno)) {
4375	4421	verbose(env, "R%d leaks addr into mem\n", value_regno);
4376	4422	return -EACCES;
4377	4423	}
	4424	+
4378	4425	err = check_mem_region_access(env, regno, off, size,
4379	4426	reg->mem_size, false);
4380		− if (!err && t == BPF_READ && value_regno >= 0)
	4427	+ if (!err && value_regno >= 0 && (t == BPF_READ \|\| rdonly_mem))
4381	4428	mark_reg_unknown(env, regs, value_regno);
4382	4429	} else if (reg->type == PTR_TO_CTX) {
4383	4430	enum bpf_reg_type reg_type = SCALAR_VALUE;
@@ -4407,7 +4454,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 23 unmodified lines
4407	4454	} else {
4408	4455	mark_reg_known_zero(env, regs,
4409	4456	value_regno);
4410		− if (reg_type_may_be_null(reg_type))
	4457	+ if (type_may_be_null(reg_type))
4411	4458	regs[value_regno].id = ++env->id_gen;
4412	4459	/* A load of ctx field could have different
4413	4460	* actual load size with the one encoded in the
@@ -4415,8 +4462,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 1 unmodified line
4415	4462	* a sub-register.
4416	4463	*/
4417	4464	regs[value_regno].subreg_def = DEF_NOT_SUBREG;
4418		− if (reg_type == PTR_TO_BTF_ID \|\|
4419		− reg_type == PTR_TO_BTF_ID_OR_NULL) {
	4465	+ if (base_type(reg_type) == PTR_TO_BTF_ID) {
4420	4466	regs[value_regno].btf = btf;
4421	4467	regs[value_regno].btf_id = btf_id;
4422	4468	}
@@ -4469,7 +4515,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 46 unmodified lines
4469	4515	} else if (type_is_sk_pointer(reg->type)) {
4470	4516	if (t == BPF_WRITE) {
4471	4517	verbose(env, "R%d cannot write into %s\n",
4472		− regno, reg_type_str[reg->type]);
	4518	+ regno, reg_type_str(env, reg->type));
4473	4519	return -EACCES;
4474	4520	}
4475	4521	err = check_sock_access(env, insn_idx, regno, off, size, t);
@@ -4485,26 +4531,32 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 9 unmodified lines
4485	4531	} else if (reg->type == CONST_PTR_TO_MAP) {
4486	4532	err = check_ptr_to_map_access(env, regs, regno, off, size, t,
4487	4533	value_regno);
4488		− } else if (reg->type == PTR_TO_RDONLY_BUF) {
4489		− if (t == BPF_WRITE) {
4490		− verbose(env, "R%d cannot write into %s\n",
4491		− regno, reg_type_str[reg->type]);
4492		− return -EACCES;
	4534	+ } else if (base_type(reg->type) == PTR_TO_BUF) {
	4535	+ bool rdonly_mem = type_is_rdonly_mem(reg->type);
	4536	+ const char *buf_info;
	4537	+ u32 *max_access;
	4538	+
	4539	+ if (rdonly_mem) {
	4540	+ if (t == BPF_WRITE) {
	4541	+ verbose(env, "R%d cannot write into %s\n",
	4542	+ regno, reg_type_str(env, reg->type));
	4543	+ return -EACCES;
	4544	+ }
	4545	+ buf_info = "rdonly";
	4546	+ max_access = &env->prog->aux->max_rdonly_access;
	4547	+ } else {
	4548	+ buf_info = "rdwr";
	4549	+ max_access = &env->prog->aux->max_rdwr_access;
4493	4550	}
	4551	+
4494	4552	err = check_buffer_access(env, reg, regno, off, size, false,
4495		− "rdonly",
4496		− &env->prog->aux->max_rdonly_access);
4497		− if (!err && value_regno >= 0)
4498		− mark_reg_unknown(env, regs, value_regno);
4499		− } else if (reg->type == PTR_TO_RDWR_BUF) {
4500		− err = check_buffer_access(env, reg, regno, off, size, false,
4501		− "rdwr",
4502		− &env->prog->aux->max_rdwr_access);
4503		− if (!err && t == BPF_READ && value_regno >= 0)
	4553	+ buf_info, max_access);
	4554	+
	4555	+ if (!err && value_regno >= 0 && (rdonly_mem \|\| t == BPF_READ))
4504	4556	mark_reg_unknown(env, regs, value_regno);
4505	4557	} else {
4506	4558	verbose(env, "R%d invalid mem access '%s'\n", regno,
4507		− reg_type_str[reg->type]);
	4559	+ reg_type_str(env, reg->type));
4508	4560	return -EACCES;
4509	4561	}
4510	4562
@@ -4578,7 +4630,7 @@ static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_i · 67 unmodified lines
4578	4630	is_sk_reg(env, insn->dst_reg)) {
4579	4631	verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n",
4580	4632	insn->dst_reg,
4581		− reg_type_str[reg_state(env, insn->dst_reg)->type]);
	4633	+ reg_type_str(env, reg_state(env, insn->dst_reg)->type));
4582	4634	return -EACCES;
4583	4635	}
4584	4636
@@ -4761,8 +4813,10 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, · 176 unmodified lines
4761	4813	struct bpf_call_arg_meta *meta)
4762	4814	{
4763	4815	struct bpf_reg_state regs = cur_regs(env), reg = &regs[regno];
	4816	+ const char *buf_info;
	4817	+ u32 *max_access;
4764	4818
4765		− switch (reg->type) {
	4819	+ switch (base_type(reg->type)) {
4766	4820	case PTR_TO_PACKET:
4767	4821	case PTR_TO_PACKET_META:
4768	4822	return check_packet_access(env, regno, reg->off, access_size,
@@ -4781,18 +4835,20 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, · 12 unmodified lines
4781	4835	return check_mem_region_access(env, regno, reg->off,
4782	4836	access_size, reg->mem_size,
4783	4837	zero_size_allowed);
4784		− case PTR_TO_RDONLY_BUF:
4785		− if (meta && meta->raw_mode)
4786		− return -EACCES;
4787		− return check_buffer_access(env, reg, regno, reg->off,
4788		− access_size, zero_size_allowed,
4789		− "rdonly",
4790		− &env->prog->aux->max_rdonly_access);
4791		− case PTR_TO_RDWR_BUF:
	4838	+ case PTR_TO_BUF:
	4839	+ if (type_is_rdonly_mem(reg->type)) {
	4840	+ if (meta && meta->raw_mode)
	4841	+ return -EACCES;
	4842	+
	4843	+ buf_info = "rdonly";
	4844	+ max_access = &env->prog->aux->max_rdonly_access;
	4845	+ } else {
	4846	+ buf_info = "rdwr";
	4847	+ max_access = &env->prog->aux->max_rdwr_access;
	4848	+ }
4792	4849	return check_buffer_access(env, reg, regno, reg->off,
4793	4850	access_size, zero_size_allowed,
4794		− "rdwr",
4795		− &env->prog->aux->max_rdwr_access);
	4851	+ buf_info, max_access);
4796	4852	case PTR_TO_STACK:
4797	4853	return check_stack_range_initialized(
4798	4854	env,
@@ -4804,9 +4860,9 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, · 5 unmodified lines
4804	4860	register_is_null(reg))
4805	4861	return 0;
4806	4862
4807		− verbose(env, "R%d type=%s expected=%s\n", regno,
4808		− reg_type_str[reg->type],
4809		− reg_type_str[PTR_TO_STACK]);
	4863	+ verbose(env, "R%d type=%s ", regno,
	4864	+ reg_type_str(env, reg->type));
	4865	+ verbose(env, "expected=%s\n", reg_type_str(env, PTR_TO_STACK));
4810	4866	return -EACCES;
4811	4867	}
4812	4868	}
@@ -4817,7 +4873,7 @@ int check_mem_reg(struct bpf_verifier_env env, struct bpf_reg_state reg, · 4 unmodified lines
4817	4873	if (register_is_null(reg))
4818	4874	return 0;
4819	4875
4820		− if (reg_type_may_be_null(reg->type)) {
	4876	+ if (type_may_be_null(reg->type)) {
4821	4877	/* Assuming that the register contains a value check if the memory
4822	4878	* access is safe. Temporarily save and restore the register's state as
4823	4879	* the conversion shouldn't be visible to a caller.
@@ -4965,9 +5021,8 @@ static int process_timer_func(struct bpf_verifier_env *env, int regno, · 141 unmodified lines
4965	5021
4966	5022	static bool arg_type_is_mem_ptr(enum bpf_arg_type type)
4967	5023	{
4968		− return type == ARG_PTR_TO_MEM \|\|
4969		− type == ARG_PTR_TO_MEM_OR_NULL \|\|
4970		− type == ARG_PTR_TO_UNINIT_MEM;
	5024	+ return base_type(type) == ARG_PTR_TO_MEM \|\|
	5025	+ base_type(type) == ARG_PTR_TO_UNINIT_MEM;
4971	5026	}
4972	5027
4973	5028	static bool arg_type_is_mem_size(enum bpf_arg_type type)
@@ -5072,8 +5127,7 @@ static const struct bpf_reg_types mem_types = { · 98 unmodified lines
5072	5127	PTR_TO_MAP_KEY,
5073	5128	PTR_TO_MAP_VALUE,
5074	5129	PTR_TO_MEM,
5075		− PTR_TO_RDONLY_BUF,
5076		− PTR_TO_RDWR_BUF,
	5130	+ PTR_TO_BUF,
5077	5131	},
5078	5132	};
5079	5133
@@ -5104,31 +5158,26 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { · 24 unmodified lines
5104	5158	[ARG_PTR_TO_MAP_KEY] = &map_key_value_types,
5105	5159	[ARG_PTR_TO_MAP_VALUE] = &map_key_value_types,
5106	5160	[ARG_PTR_TO_UNINIT_MAP_VALUE] = &map_key_value_types,
5107		− [ARG_PTR_TO_MAP_VALUE_OR_NULL] = &map_key_value_types,
5108	5161	[ARG_CONST_SIZE] = &scalar_types,
5109	5162	[ARG_CONST_SIZE_OR_ZERO] = &scalar_types,
5110	5163	[ARG_CONST_ALLOC_SIZE_OR_ZERO] = &scalar_types,
5111	5164	[ARG_CONST_MAP_PTR] = &const_map_ptr_types,
5112	5165	[ARG_PTR_TO_CTX] = &context_types,
5113		− [ARG_PTR_TO_CTX_OR_NULL] = &context_types,
5114	5166	[ARG_PTR_TO_SOCK_COMMON] = &sock_types,
5115	5167	#ifdef CONFIG_NET
5116	5168	[ARG_PTR_TO_BTF_ID_SOCK_COMMON] = &btf_id_sock_common_types,
5117	5169	#endif
5118	5170	[ARG_PTR_TO_SOCKET] = &fullsock_types,
5119		− [ARG_PTR_TO_SOCKET_OR_NULL] = &fullsock_types,
5120	5171	[ARG_PTR_TO_BTF_ID] = &btf_ptr_types,
5121	5172	[ARG_PTR_TO_SPIN_LOCK] = &spin_lock_types,
5122	5173	[ARG_PTR_TO_MEM] = &mem_types,
5123		− [ARG_PTR_TO_MEM_OR_NULL] = &mem_types,
5124	5174	[ARG_PTR_TO_UNINIT_MEM] = &mem_types,
5125	5175	[ARG_PTR_TO_ALLOC_MEM] = &alloc_mem_types,
5126		− [ARG_PTR_TO_ALLOC_MEM_OR_NULL] = &alloc_mem_types,
5127	5176	[ARG_PTR_TO_INT] = &int_ptr_types,
5128	5177	[ARG_PTR_TO_LONG] = &int_ptr_types,
5129	5178	[ARG_PTR_TO_PERCPU_BTF_ID] = &percpu_btf_ptr_types,
5130	5179	[ARG_PTR_TO_FUNC] = &func_ptr_types,
5131		− [ARG_PTR_TO_STACK_OR_NULL] = &stack_ptr_types,
	5180	+ [ARG_PTR_TO_STACK] = &stack_ptr_types,
5132	5181	[ARG_PTR_TO_CONST_STR] = &const_str_ptr_types,
5133	5182	[ARG_PTR_TO_TIMER] = &timer_types,
5134	5183	};
@@ -5142,12 +5191,27 @@ static int check_reg_type(struct bpf_verifier_env *env, u32 regno, · 7 unmodified lines
5142	5191	const struct bpf_reg_types *compatible;
5143	5192	int i, j;
5144	5193
5145		− compatible = compatible_reg_types[arg_type];
	5194	+ compatible = compatible_reg_types[base_type(arg_type)];
5146	5195	if (!compatible) {
5147	5196	verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type);
5148	5197	return -EFAULT;
5149	5198	}
5150	5199
	5200	+ /* ARG_PTR_TO_MEM + RDONLY is compatible with PTR_TO_MEM and PTR_TO_MEM + RDONLY,
	5201	+ * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM and NOT with PTR_TO_MEM + RDONLY
	5202	+ *
	5203	+ * Same for MAYBE_NULL:
	5204	+ *
	5205	+ * ARG_PTR_TO_MEM + MAYBE_NULL is compatible with PTR_TO_MEM and PTR_TO_MEM + MAYBE_NULL,
	5206	+ * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM but NOT with PTR_TO_MEM + MAYBE_NULL
	5207	+ *
	5208	+ * Therefore we fold these flags depending on the arg_type before comparison.
	5209	+ */
	5210	+ if (arg_type & MEM_RDONLY)
	5211	+ type &= ~MEM_RDONLY;
	5212	+ if (arg_type & PTR_MAYBE_NULL)
	5213	+ type &= ~PTR_MAYBE_NULL;
	5214	+
5151	5215	for (i = 0; i < ARRAY_SIZE(compatible->types); i++) {
5152	5216	expected = compatible->types[i];
5153	5217	if (expected == NOT_INIT)
@@ -5157,14 +5221,14 @@ static int check_reg_type(struct bpf_verifier_env *env, u32 regno, · 3 unmodified lines
5157	5221	goto found;
5158	5222	}
5159	5223
5160		− verbose(env, "R%d type=%s expected=", regno, reg_type_str[type]);
	5224	+ verbose(env, "R%d type=%s expected=", regno, reg_type_str(env, reg->type));
5161	5225	for (j = 0; j + 1 < i; j++)
5162		− verbose(env, "%s, ", reg_type_str[compatible->types[j]]);
5163		− verbose(env, "%s\n", reg_type_str[compatible->types[j]]);
	5226	+ verbose(env, "%s, ", reg_type_str(env, compatible->types[j]));
	5227	+ verbose(env, "%s\n", reg_type_str(env, compatible->types[j]));
5164	5228	return -EACCES;
5165	5229
5166	5230	found:
5167		− if (type == PTR_TO_BTF_ID) {
	5231	+ if (reg->type == PTR_TO_BTF_ID) {
5168	5232	if (!arg_btf_id) {
5169	5233	if (!compatible->btf_id) {
5170	5234	verbose(env, "verifier internal error: missing arg compatible BTF ID\n");
@@ -5223,15 +5287,14 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, · 52 unmodified lines
5223	5287	return -EACCES;
5224	5288	}
5225	5289
5226		− if (arg_type == ARG_PTR_TO_MAP_VALUE \|\|
5227		− arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE \|\|
5228		− arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL) {
	5290	+ if (base_type(arg_type) == ARG_PTR_TO_MAP_VALUE \|\|
	5291	+ base_type(arg_type) == ARG_PTR_TO_UNINIT_MAP_VALUE) {
5229	5292	err = resolve_map_arg_type(env, meta, &arg_type);
5230	5293	if (err)
5231	5294	return err;
5232	5295	}
5233	5296
5234		− if (register_is_null(reg) && arg_type_may_be_null(arg_type))
	5297	+ if (register_is_null(reg) && type_may_be_null(arg_type))
5235	5298	/* A NULL register has a SCALAR_VALUE type, so skip
5236	5299	* type checking.
5237	5300	*/
@@ -5300,10 +5363,11 @@ skip_type_check: · 62 unmodified lines
5300	5363	err = check_helper_mem_access(env, regno,
5301	5364	meta->map_ptr->key_size, false,
5302	5365	NULL);
5303		− } else if (arg_type == ARG_PTR_TO_MAP_VALUE \|\|
5304		− (arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL &&
5305		− !register_is_null(reg)) \|\|
5306		− arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE) {
	5366	+ } else if (base_type(arg_type) == ARG_PTR_TO_MAP_VALUE \|\|
	5367	+ base_type(arg_type) == ARG_PTR_TO_UNINIT_MAP_VALUE) {
	5368	+ if (type_may_be_null(arg_type) && register_is_null(reg))
	5369	+ return 0;
	5370	+
5307	5371	/* bpf_map_xxx(..., map_ptr, ..., value) call:
5308	5372	* check [value, value + map->value_size) validity
5309	5373	*/
@@ -6025,9 +6089,9 @@ static int __check_func_call(struct bpf_verifier_env env, struct bpf_insn insn · 715 unmodified lines
6025	6089
6026	6090	if (env->log.level & BPF_LOG_LEVEL) {
6027	6091	verbose(env, "caller:\n");
6028		− print_verifier_state(env, caller);
	6092	+ print_verifier_state(env, caller, true);
6029	6093	verbose(env, "callee:\n");
6030		− print_verifier_state(env, callee);
	6094	+ print_verifier_state(env, callee, true);
6031	6095	}
6032	6096	return 0;
6033	6097	}
@@ -6242,9 +6306,9 @@ static int prepare_func_exit(struct bpf_verifier_env env, int insn_idx) · 208 unmodified lines
6242	6306	*insn_idx = callee->callsite + 1;
6243	6307	if (env->log.level & BPF_LOG_LEVEL) {
6244	6308	verbose(env, "returning from callee:\n");
6245		− print_verifier_state(env, callee);
	6309	+ print_verifier_state(env, callee, true);
6246	6310	verbose(env, "to caller at %d:\n", *insn_idx);
6247		− print_verifier_state(env, caller);
	6311	+ print_verifier_state(env, caller, true);
6248	6312	}
6249	6313	/* clear everything in the callee */
6250	6314	free_func_state(callee);
@@ -6410,13 +6474,11 @@ static int check_bpf_snprintf_call(struct bpf_verifier_env *env, · 159 unmodified lines
6410	6474
6411	6475	static int check_get_func_ip(struct bpf_verifier_env *env)
6412	6476	{
6413		− enum bpf_attach_type eatype = env->prog->expected_attach_type;
6414	6477	enum bpf_prog_type type = resolve_prog_type(env->prog);
6415	6478	int func_id = BPF_FUNC_get_func_ip;
6416	6479
6417	6480	if (type == BPF_PROG_TYPE_TRACING) {
6418		− if (eatype != BPF_TRACE_FENTRY && eatype != BPF_TRACE_FEXIT &&
6419		− eatype != BPF_MODIFY_RETURN) {
	6481	+ if (!bpf_prog_has_trampoline(env->prog)) {
6420	6482	verbose(env, "func %s#%d supported only for fentry/fexit/fmod_ret programs\n",
6421	6483	func_id_name(func_id), func_id);
6422	6484	return -ENOTSUPP;
@@ -6435,6 +6497,8 @@ static int check_helper_call(struct bpf_verifier_env env, struct bpf_insn insn · 12 unmodified lines
6435	6497	int *insn_idx_p)
6436	6498	{
6437	6499	const struct bpf_func_proto *fn = NULL;
	6500	+ enum bpf_return_type ret_type;
	6501	+ enum bpf_type_flag ret_flag;
6438	6502	struct bpf_reg_state *regs;
6439	6503	struct bpf_call_arg_meta meta;
6440	6504	int insn_idx = *insn_idx_p;
@@ -6574,13 +6638,14 @@ static int check_helper_call(struct bpf_verifier_env env, struct bpf_insn insn · 133 unmodified lines
6574	6638	regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;
6575	6639
6576	6640	/* update return register (already marked as written above) */
6577		− if (fn->ret_type == RET_INTEGER) {
	6641	+ ret_type = fn->ret_type;
	6642	+ ret_flag = type_flag(fn->ret_type);
	6643	+ if (ret_type == RET_INTEGER) {
6578	6644	/* sets type to SCALAR_VALUE */
6579	6645	mark_reg_unknown(env, regs, BPF_REG_0);
6580		− } else if (fn->ret_type == RET_VOID) {
	6646	+ } else if (ret_type == RET_VOID) {
6581	6647	regs[BPF_REG_0].type = NOT_INIT;
6582		− } else if (fn->ret_type == RET_PTR_TO_MAP_VALUE_OR_NULL \|\|
6583		− fn->ret_type == RET_PTR_TO_MAP_VALUE) {
	6648	+ } else if (base_type(ret_type) == RET_PTR_TO_MAP_VALUE) {
6584	6649	/* There is no offset yet applied, variable or fixed */
6585	6650	mark_reg_known_zero(env, regs, BPF_REG_0);
6586	6651	/* remember map_ptr, so that check_map_access()
@@ -6594,28 +6659,25 @@ static int check_helper_call(struct bpf_verifier_env env, struct bpf_insn insn · 7 unmodified lines
6594	6659	}
6595	6660	regs[BPF_REG_0].map_ptr = meta.map_ptr;
6596	6661	regs[BPF_REG_0].map_uid = meta.map_uid;
6597		− if (fn->ret_type == RET_PTR_TO_MAP_VALUE) {
6598		− regs[BPF_REG_0].type = PTR_TO_MAP_VALUE;
6599		− if (map_value_has_spin_lock(meta.map_ptr))
6600		− regs[BPF_REG_0].id = ++env->id_gen;
6601		− } else {
6602		− regs[BPF_REG_0].type = PTR_TO_MAP_VALUE_OR_NULL;
	6662	+ regs[BPF_REG_0].type = PTR_TO_MAP_VALUE \| ret_flag;
	6663	+ if (!type_may_be_null(ret_type) &&
	6664	+ map_value_has_spin_lock(meta.map_ptr)) {
	6665	+ regs[BPF_REG_0].id = ++env->id_gen;
6603	6666	}
6604		− } else if (fn->ret_type == RET_PTR_TO_SOCKET_OR_NULL) {
	6667	+ } else if (base_type(ret_type) == RET_PTR_TO_SOCKET) {
6605	6668	mark_reg_known_zero(env, regs, BPF_REG_0);
6606		− regs[BPF_REG_0].type = PTR_TO_SOCKET_OR_NULL;
6607		− } else if (fn->ret_type == RET_PTR_TO_SOCK_COMMON_OR_NULL) {
	6669	+ regs[BPF_REG_0].type = PTR_TO_SOCKET \| ret_flag;
	6670	+ } else if (base_type(ret_type) == RET_PTR_TO_SOCK_COMMON) {
6608	6671	mark_reg_known_zero(env, regs, BPF_REG_0);
6609		− regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON_OR_NULL;
6610		− } else if (fn->ret_type == RET_PTR_TO_TCP_SOCK_OR_NULL) {
	6672	+ regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON \| ret_flag;
	6673	+ } else if (base_type(ret_type) == RET_PTR_TO_TCP_SOCK) {
6611	6674	mark_reg_known_zero(env, regs, BPF_REG_0);
6612		− regs[BPF_REG_0].type = PTR_TO_TCP_SOCK_OR_NULL;
6613		− } else if (fn->ret_type == RET_PTR_TO_ALLOC_MEM_OR_NULL) {
	6675	+ regs[BPF_REG_0].type = PTR_TO_TCP_SOCK \| ret_flag;
	6676	+ } else if (base_type(ret_type) == RET_PTR_TO_ALLOC_MEM) {
6614	6677	mark_reg_known_zero(env, regs, BPF_REG_0);
6615		− regs[BPF_REG_0].type = PTR_TO_MEM_OR_NULL;
	6678	+ regs[BPF_REG_0].type = PTR_TO_MEM \| ret_flag;
6616	6679	regs[BPF_REG_0].mem_size = meta.mem_size;
6617		− } else if (fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL \|\|
6618		− fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID) {
	6680	+ } else if (base_type(ret_type) == RET_PTR_TO_MEM_OR_BTF_ID) {
6619	6681	const struct btf_type *t;
6620	6682
6621	6683	mark_reg_known_zero(env, regs, BPF_REG_0);
@@ -6633,29 +6695,30 @@ static int check_helper_call(struct bpf_verifier_env env, struct bpf_insn insn · 11 unmodified lines
6633	6695	tname, PTR_ERR(ret));
6634	6696	return -EINVAL;
6635	6697	}
6636		− regs[BPF_REG_0].type =
6637		− fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID ?
6638		− PTR_TO_MEM : PTR_TO_MEM_OR_NULL;
	6698	+ regs[BPF_REG_0].type = PTR_TO_MEM \| ret_flag;
6639	6699	regs[BPF_REG_0].mem_size = tsize;
6640	6700	} else {
6641		− regs[BPF_REG_0].type =
6642		− fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID ?
6643		− PTR_TO_BTF_ID : PTR_TO_BTF_ID_OR_NULL;
	6701	+ /* MEM_RDONLY may be carried from ret_flag, but it
	6702	+ * doesn't apply on PTR_TO_BTF_ID. Fold it, otherwise
	6703	+ * it will confuse the check of PTR_TO_BTF_ID in
	6704	+ * check_mem_access().
	6705	+ */
	6706	+ ret_flag &= ~MEM_RDONLY;
	6707	+
	6708	+ regs[BPF_REG_0].type = PTR_TO_BTF_ID \| ret_flag;
6644	6709	regs[BPF_REG_0].btf = meta.ret_btf;
6645	6710	regs[BPF_REG_0].btf_id = meta.ret_btf_id;
6646	6711	}
6647		− } else if (fn->ret_type == RET_PTR_TO_BTF_ID_OR_NULL \|\|
6648		− fn->ret_type == RET_PTR_TO_BTF_ID) {
	6712	+ } else if (base_type(ret_type) == RET_PTR_TO_BTF_ID) {
6649	6713	int ret_btf_id;
6650	6714
6651	6715	mark_reg_known_zero(env, regs, BPF_REG_0);
6652		− regs[BPF_REG_0].type = fn->ret_type == RET_PTR_TO_BTF_ID ?
6653		− PTR_TO_BTF_ID :
6654		− PTR_TO_BTF_ID_OR_NULL;
	6716	+ regs[BPF_REG_0].type = PTR_TO_BTF_ID \| ret_flag;
6655	6717	ret_btf_id = *fn->ret_btf_id;
6656	6718	if (ret_btf_id == 0) {
6657		− verbose(env, "invalid return type %d of func %s#%d\n",
6658		− fn->ret_type, func_id_name(func_id), func_id);
	6719	+ verbose(env, "invalid return type %u of func %s#%d\n",
	6720	+ base_type(ret_type), func_id_name(func_id),
	6721	+ func_id);
6659	6722	return -EINVAL;
6660	6723	}
6661	6724	/* current BPF helper definitions are only coming from
@@ -6664,12 +6727,12 @@ static int check_helper_call(struct bpf_verifier_env env, struct bpf_insn insn · 2 unmodified lines
6664	6727	regs[BPF_REG_0].btf = btf_vmlinux;
6665	6728	regs[BPF_REG_0].btf_id = ret_btf_id;
6666	6729	} else {
6667		− verbose(env, "unknown return type %d of func %s#%d\n",
6668		− fn->ret_type, func_id_name(func_id), func_id);
	6730	+ verbose(env, "unknown return type %u of func %s#%d\n",
	6731	+ base_type(ret_type), func_id_name(func_id), func_id);
6669	6732	return -EINVAL;
6670	6733	}
6671	6734
6672		− if (reg_type_may_be_null(regs[BPF_REG_0].type))
	6735	+ if (type_may_be_null(regs[BPF_REG_0].type))
6673	6736	regs[BPF_REG_0].id = ++env->id_gen;
6674	6737
6675	6738	if (is_ptr_cast_function(func_id)) {
@@ -6878,25 +6941,25 @@ static bool check_reg_sane_offset(struct bpf_verifier_env *env, · 202 unmodified lines
6878	6941
6879	6942	if (known && (val >= BPF_MAX_VAR_OFF \|\| val <= -BPF_MAX_VAR_OFF)) {
6880	6943	verbose(env, "math between %s pointer and %lld is not allowed\n",
6881		− reg_type_str[type], val);
	6944	+ reg_type_str(env, type), val);
6882	6945	return false;
6883	6946	}
6884	6947
6885	6948	if (reg->off >= BPF_MAX_VAR_OFF \|\| reg->off <= -BPF_MAX_VAR_OFF) {
6886	6949	verbose(env, "%s pointer offset %d is not allowed\n",
6887		− reg_type_str[type], reg->off);
	6950	+ reg_type_str(env, type), reg->off);
6888	6951	return false;
6889	6952	}
6890	6953
6891	6954	if (smin == S64_MIN) {
6892	6955	verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n",
6893		− reg_type_str[type]);
	6956	+ reg_type_str(env, type));
6894	6957	return false;
6895	6958	}
6896	6959
6897	6960	if (smin >= BPF_MAX_VAR_OFF \|\| smin <= -BPF_MAX_VAR_OFF) {
6898	6961	verbose(env, "value %lld makes %s pointer be out of bounds\n",
6899		− smin, reg_type_str[type]);
	6962	+ smin, reg_type_str(env, type));
6900	6963	return false;
6901	6964	}
6902	6965
@@ -7273,11 +7336,13 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, · 370 unmodified lines
7273	7336	return -EACCES;
7274	7337	}
7275	7338
7276		− switch (ptr_reg->type) {
7277		− case PTR_TO_MAP_VALUE_OR_NULL:
	7339	+ if (ptr_reg->type & PTR_MAYBE_NULL) {
7278	7340	verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n",
7279		− dst, reg_type_str[ptr_reg->type]);
	7341	+ dst, reg_type_str(env, ptr_reg->type));
7280	7342	return -EACCES;
	7343	+ }
	7344	+
	7345	+ switch (base_type(ptr_reg->type)) {
7281	7346	case CONST_PTR_TO_MAP:
7282	7347	/* smin_val represents the known value */
7283	7348	if (known && smin_val == 0 && opcode == BPF_ADD)
@@ -7285,14 +7350,11 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, · 1 unmodified line
7285	7350	fallthrough;
7286	7351	case PTR_TO_PACKET_END:
7287	7352	case PTR_TO_SOCKET:
7288		− case PTR_TO_SOCKET_OR_NULL:
7289	7353	case PTR_TO_SOCK_COMMON:
7290		− case PTR_TO_SOCK_COMMON_OR_NULL:
7291	7354	case PTR_TO_TCP_SOCK:
7292		− case PTR_TO_TCP_SOCK_OR_NULL:
7293	7355	case PTR_TO_XDP_SOCK:
7294	7356	verbose(env, "R%d pointer arithmetic on %s prohibited\n",
7295		− dst, reg_type_str[ptr_reg->type]);
	7357	+ dst, reg_type_str(env, ptr_reg->type));
7296	7358	return -EACCES;
7297	7359	default:
7298	7360	break;
@@ -8265,12 +8327,12 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, · 966 unmodified lines
8265	8327
8266	8328	/* Got here implies adding two SCALAR_VALUEs */
8267	8329	if (WARN_ON_ONCE(ptr_reg)) {
8268		− print_verifier_state(env, state);
	8330	+ print_verifier_state(env, state, true);
8269	8331	verbose(env, "verifier internal error: unexpected ptr_reg\n");
8270	8332	return -EINVAL;
8271	8333	}
8272	8334	if (WARN_ON(!src_reg)) {
8273		− print_verifier_state(env, state);
	8335	+ print_verifier_state(env, state, true);
8274	8336	verbose(env, "verifier internal error: no src_reg\n");
8275	8337	return -EINVAL;
8276	8338	}
@@ -9015,7 +9077,7 @@ static void mark_ptr_or_null_reg(struct bpf_func_state *state, · 738 unmodified lines
9015	9077	struct bpf_reg_state *reg, u32 id,
9016	9078	bool is_null)
9017	9079	{
9018		− if (reg_type_may_be_null(reg->type) && reg->id == id &&
	9080	+ if (type_may_be_null(reg->type) && reg->id == id &&
9019	9081	!WARN_ON_ONCE(!reg->id)) {
9020	9082	/* Old offset (both fixed and variable parts) should
9021	9083	* have been known-zero, because we don't allow pointer
@@ -9393,7 +9455,7 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, · 371 unmodified lines
9393	9455	*/
9394	9456	if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K &&
9395	9457	insn->imm == 0 && (opcode == BPF_JEQ \|\| opcode == BPF_JNE) &&
9396		− reg_type_may_be_null(dst_reg->type)) {
	9458	+ type_may_be_null(dst_reg->type)) {
9397	9459	/* Mark all identical registers in each branch as either
9398	9460	* safe or unknown depending R == 0 or R != 0 conditional.
9399	9461	*/
@@ -9409,7 +9471,7 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, · 9 unmodified lines
9409	9471	return -EACCES;
9410	9472	}
9411	9473	if (env->log.level & BPF_LOG_LEVEL)
9412		− print_verifier_state(env, this_branch->frame[this_branch->curframe]);
	9474	+ print_insn_state(env, this_branch->frame[this_branch->curframe]);
9413	9475	return 0;
9414	9476	}
9415	9477
@@ -9448,7 +9510,7 @@ static int check_ld_imm(struct bpf_verifier_env env, struct bpf_insn insn) · 32 unmodified lines
9448	9510	mark_reg_known_zero(env, regs, insn->dst_reg);
9449	9511
9450	9512	dst_reg->type = aux->btf_var.reg_type;
9451		− switch (dst_reg->type) {
	9513	+ switch (base_type(dst_reg->type)) {
9452	9514	case PTR_TO_MEM:
9453	9515	dst_reg->mem_size = aux->btf_var.mem_size;
9454	9516	break;
@@ -9647,7 +9709,7 @@ static int check_return_code(struct bpf_verifier_env *env) · 192 unmodified lines
9647	9709	/* enforce return zero from async callbacks like timer */
9648	9710	if (reg->type != SCALAR_VALUE) {
9649	9711	verbose(env, "In async callback the register R0 is not a known value (%s)\n",
9650		− reg_type_str[reg->type]);
	9712	+ reg_type_str(env, reg->type));
9651	9713	return -EINVAL;
9652	9714	}
9653	9715
@@ -9661,7 +9723,7 @@ static int check_return_code(struct bpf_verifier_env *env) · 7 unmodified lines
9661	9723	if (is_subprog) {
9662	9724	if (reg->type != SCALAR_VALUE) {
9663	9725	verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n",
9664		− reg_type_str[reg->type]);
	9726	+ reg_type_str(env, reg->type));
9665	9727	return -EINVAL;
9666	9728	}
9667	9729	return 0;
@@ -9725,7 +9787,7 @@ static int check_return_code(struct bpf_verifier_env *env) · 57 unmodified lines
9725	9787
9726	9788	if (reg->type != SCALAR_VALUE) {
9727	9789	verbose(env, "At program exit the register R0 is not a known value (%s)\n",
9728		− reg_type_str[reg->type]);
	9790	+ reg_type_str(env, reg->type));
9729	9791	return -EINVAL;
9730	9792	}
9731	9793
@@ -10582,7 +10644,7 @@ static bool regsafe(struct bpf_verifier_env env, struct bpf_reg_state rold, · 850 unmodified lines
10582	10644	return true;
10583	10645	if (rcur->type == NOT_INIT)
10584	10646	return false;
10585		− switch (rold->type) {
	10647	+ switch (base_type(rold->type)) {
10586	10648	case SCALAR_VALUE:
10587	10649	if (env->explore_alu_limits)
10588	10650	return false;
@@ -10604,6 +10666,22 @@ static bool regsafe(struct bpf_verifier_env env, struct bpf_reg_state rold, · 15 unmodified lines
10604	10666	}
10605	10667	case PTR_TO_MAP_KEY:
10606	10668	case PTR_TO_MAP_VALUE:
	10669	+ /* a PTR_TO_MAP_VALUE could be safe to use as a
	10670	+ * PTR_TO_MAP_VALUE_OR_NULL into the same map.
	10671	+ * However, if the old PTR_TO_MAP_VALUE_OR_NULL then got NULL-
	10672	+ * checked, doing so could have affected others with the same
	10673	+ * id, and we can't check for that because we lost the id when
	10674	+ * we converted to a PTR_TO_MAP_VALUE.
	10675	+ */
	10676	+ if (type_may_be_null(rold->type)) {
	10677	+ if (!type_may_be_null(rcur->type))
	10678	+ return false;
	10679	+ if (memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)))
	10680	+ return false;
	10681	+ /* Check our ids match any regs they're supposed to */
	10682	+ return check_ids(rold->id, rcur->id, idmap);
	10683	+ }
	10684	+
10607	10685	/* If the new min/max/var_off satisfy the old ones and
10608	10686	* everything else matches, we are OK.
10609	10687	* 'id' is not compared, since it's only used for maps with
@@ -10615,20 +10693,6 @@ static bool regsafe(struct bpf_verifier_env env, struct bpf_reg_state rold, · 5 unmodified lines
10615	10693	return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 &&
10616	10694	range_within(rold, rcur) &&
10617	10695	tnum_in(rold->var_off, rcur->var_off);
10618		− case PTR_TO_MAP_VALUE_OR_NULL:
10619		− /* a PTR_TO_MAP_VALUE could be safe to use as a
10620		− * PTR_TO_MAP_VALUE_OR_NULL into the same map.
10621		− * However, if the old PTR_TO_MAP_VALUE_OR_NULL then got NULL-
10622		− * checked, doing so could have affected others with the same
10623		− * id, and we can't check for that because we lost the id when
10624		− * we converted to a PTR_TO_MAP_VALUE.
10625		− */
10626		− if (rcur->type != PTR_TO_MAP_VALUE_OR_NULL)
10627		− return false;
10628		− if (memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)))
10629		− return false;
10630		− /* Check our ids match any regs they're supposed to */
10631		− return check_ids(rold->id, rcur->id, idmap);
10632	10696	case PTR_TO_PACKET_META:
10633	10697	case PTR_TO_PACKET:
10634	10698	if (rcur->type != rold->type)
@@ -10657,11 +10721,8 @@ static bool regsafe(struct bpf_verifier_env env, struct bpf_reg_state rold, · 22 unmodified lines
10657	10721	case PTR_TO_PACKET_END:
10658	10722	case PTR_TO_FLOW_KEYS:
10659	10723	case PTR_TO_SOCKET:
10660		− case PTR_TO_SOCKET_OR_NULL:
10661	10724	case PTR_TO_SOCK_COMMON:
10662		− case PTR_TO_SOCK_COMMON_OR_NULL:
10663	10725	case PTR_TO_TCP_SOCK:
10664		− case PTR_TO_TCP_SOCK_OR_NULL:
10665	10726	case PTR_TO_XDP_SOCK:
10666	10727	/* Only valid matches are exact, which memcmp() above
10667	10728	* would have accepted
@@ -11187,17 +11248,13 @@ next: · 519 unmodified lines
11187	11248	/* Return true if it's OK to have the same insn return a different type. */
11188	11249	static bool reg_type_mismatch_ok(enum bpf_reg_type type)
11189	11250	{
11190		− switch (type) {
	11251	+ switch (base_type(type)) {
11191	11252	case PTR_TO_CTX:
11192	11253	case PTR_TO_SOCKET:
11193		− case PTR_TO_SOCKET_OR_NULL:
11194	11254	case PTR_TO_SOCK_COMMON:
11195		− case PTR_TO_SOCK_COMMON_OR_NULL:
11196	11255	case PTR_TO_TCP_SOCK:
11197		− case PTR_TO_TCP_SOCK_OR_NULL:
11198	11256	case PTR_TO_XDP_SOCK:
11199	11257	case PTR_TO_BTF_ID:
11200		− case PTR_TO_BTF_ID_OR_NULL:
11201	11258	return false;
11202	11259	default:
11203	11260	return true;
@@ -11277,16 +11334,12 @@ static int do_check(struct bpf_verifier_env *env) · 73 unmodified lines
11277	11334	if (need_resched())
11278	11335	cond_resched();
11279	11336
11280		− if (env->log.level & BPF_LOG_LEVEL2 \|\|
11281		− (env->log.level & BPF_LOG_LEVEL && do_print_state)) {
11282		− if (env->log.level & BPF_LOG_LEVEL2)
11283		− verbose(env, "%d:", env->insn_idx);
11284		− else
11285		− verbose(env, "\nfrom %d to %d%s:",
11286		− env->prev_insn_idx, env->insn_idx,
11287		− env->cur_state->speculative ?
11288		− " (speculative execution)" : "");
11289		− print_verifier_state(env, state->frame[state->curframe]);
	11337	+ if (env->log.level & BPF_LOG_LEVEL2 && do_print_state) {
	11338	+ verbose(env, "\nfrom %d to %d%s:",
	11339	+ env->prev_insn_idx, env->insn_idx,
	11340	+ env->cur_state->speculative ?
	11341	+ " (speculative execution)" : "");
	11342	+ print_verifier_state(env, state->frame[state->curframe], true);
11290	11343	do_print_state = false;
11291	11344	}
11292	11345
@@ -11297,9 +11350,15 @@ static int do_check(struct bpf_verifier_env *env) · 4 unmodified lines
11297	11350	.private_data = env,
11298	11351	};
11299	11352
	11353	+ if (verifier_state_scratched(env))
	11354	+ print_insn_state(env, state->frame[state->curframe]);
	11355	+
11300	11356	verbose_linfo(env, env->insn_idx, "; ");
	11357	+ env->prev_log_len = env->log.len_used;
11301	11358	verbose(env, "%d: ", env->insn_idx);
11302	11359	print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
	11360	+ env->prev_insn_print_len = env->log.len_used - env->prev_log_len;
	11361	+ env->prev_log_len = env->log.len_used;
11303	11362	}
11304	11363
11305	11364	if (bpf_prog_is_dev_bound(env->prog->aux)) {
@@ -11421,7 +11480,7 @@ static int do_check(struct bpf_verifier_env *env) · 115 unmodified lines
11421	11480	if (is_ctx_reg(env, insn->dst_reg)) {
11422	11481	verbose(env, "BPF_ST stores into R%d %s is not allowed\n",
11423	11482	insn->dst_reg,
11424		− reg_type_str[reg_state(env, insn->dst_reg)->type]);
	11483	+ reg_type_str(env, reg_state(env, insn->dst_reg)->type));
11425	11484	return -EACCES;
11426	11485	}
11427	11486
@@ -11508,6 +11567,7 @@ static int do_check(struct bpf_verifier_env *env) · 80 unmodified lines
11508	11567	if (err)
11509	11568	return err;
11510	11569	process_bpf_exit:
	11570	+ mark_verifier_state_scratched(env);
11511	11571	update_branch_counts(env, env->cur_state);
11512	11572	err = pop_stack(env, &prev_insn_idx,
11513	11573	&env->insn_idx, pop_log);
@@ -11673,7 +11733,7 @@ static int check_pseudo_btf_id(struct bpf_verifier_env *env, · 159 unmodified lines
11673	11733	err = -EINVAL;
11674	11734	goto err_put;
11675	11735	}
11676		− aux->btf_var.reg_type = PTR_TO_MEM;
	11736	+ aux->btf_var.reg_type = PTR_TO_MEM \| MEM_RDONLY;
11677	11737	aux->btf_var.mem_size = tsize;
11678	11738	} else {
11679	11739	aux->btf_var.reg_type = PTR_TO_BTF_ID;
@@ -11833,6 +11893,9 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, · 153 unmodified lines
11833	11893	}
11834	11894	break;
11835	11895	case BPF_MAP_TYPE_RINGBUF:
	11896	+ case BPF_MAP_TYPE_INODE_STORAGE:
	11897	+ case BPF_MAP_TYPE_SK_STORAGE:
	11898	+ case BPF_MAP_TYPE_TASK_STORAGE:
11836	11899	break;
11837	11900	default:
11838	11901	verbose(env,
@@ -13016,6 +13079,7 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, · 1177 unmodified lines
13016	13079	static int do_misc_fixups(struct bpf_verifier_env *env)
13017	13080	{
13018	13081	struct bpf_prog *prog = env->prog;
	13082	+ enum bpf_attach_type eatype = prog->expected_attach_type;
13019	13083	bool expect_blinding = bpf_jit_blinding_enabled(prog);
13020	13084	enum bpf_prog_type prog_type = resolve_prog_type(prog);
13021	13085	struct bpf_insn *insn = prog->insnsi;
@@ -13386,11 +13450,79 @@ patch_map_ops_generic: · 364 unmodified lines
13386	13450	continue;
13387	13451	}
13388	13452
	13453	+ /* Implement bpf_get_func_arg inline. */
	13454	+ if (prog_type == BPF_PROG_TYPE_TRACING &&
	13455	+ insn->imm == BPF_FUNC_get_func_arg) {
	13456	+ /* Load nr_args from ctx - 8 */
	13457	+ insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
	13458	+ insn_buf[1] = BPF_JMP32_REG(BPF_JGE, BPF_REG_2, BPF_REG_0, 6);
	13459	+ insn_buf[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 3);
	13460	+ insn_buf[3] = BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_1);
	13461	+ insn_buf[4] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0);
	13462	+ insn_buf[5] = BPF_STX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0);
	13463	+ insn_buf[6] = BPF_MOV64_IMM(BPF_REG_0, 0);
	13464	+ insn_buf[7] = BPF_JMP_A(1);
	13465	+ insn_buf[8] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL);
	13466	+ cnt = 9;
	13467	+
	13468	+ new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
	13469	+ if (!new_prog)
	13470	+ return -ENOMEM;
	13471	+
	13472	+ delta += cnt - 1;
	13473	+ env->prog = prog = new_prog;
	13474	+ insn = new_prog->insnsi + i + delta;
	13475	+ continue;
	13476	+ }
	13477	+
	13478	+ /* Implement bpf_get_func_ret inline. */
	13479	+ if (prog_type == BPF_PROG_TYPE_TRACING &&
	13480	+ insn->imm == BPF_FUNC_get_func_ret) {
	13481	+ if (eatype == BPF_TRACE_FEXIT \|\|
	13482	+ eatype == BPF_MODIFY_RETURN) {
	13483	+ /* Load nr_args from ctx - 8 */
	13484	+ insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
	13485	+ insn_buf[1] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_0, 3);
	13486	+ insn_buf[2] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1);
	13487	+ insn_buf[3] = BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0);
	13488	+ insn_buf[4] = BPF_STX_MEM(BPF_DW, BPF_REG_2, BPF_REG_3, 0);
	13489	+ insn_buf[5] = BPF_MOV64_IMM(BPF_REG_0, 0);
	13490	+ cnt = 6;
	13491	+ } else {
	13492	+ insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, -EOPNOTSUPP);
	13493	+ cnt = 1;
	13494	+ }
	13495	+
	13496	+ new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
	13497	+ if (!new_prog)
	13498	+ return -ENOMEM;
	13499	+
	13500	+ delta += cnt - 1;
	13501	+ env->prog = prog = new_prog;
	13502	+ insn = new_prog->insnsi + i + delta;
	13503	+ continue;
	13504	+ }
	13505	+
	13506	+ /* Implement get_func_arg_cnt inline. */
	13507	+ if (prog_type == BPF_PROG_TYPE_TRACING &&
	13508	+ insn->imm == BPF_FUNC_get_func_arg_cnt) {
	13509	+ /* Load nr_args from ctx - 8 */
	13510	+ insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
	13511	+
	13512	+ new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1);
	13513	+ if (!new_prog)
	13514	+ return -ENOMEM;
	13515	+
	13516	+ env->prog = prog = new_prog;
	13517	+ insn = new_prog->insnsi + i + delta;
	13518	+ continue;
	13519	+ }
	13520	+
13389	13521	/* Implement bpf_get_func_ip inline. */
13390	13522	if (prog_type == BPF_PROG_TYPE_TRACING &&
13391	13523	insn->imm == BPF_FUNC_get_func_ip) {
13392		− /* Load IP address from ctx - 8 */
13393		− insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
	13524	+ /* Load IP address from ctx - 16 */
	13525	+ insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -16);
13394	13526
13395	13527	new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1);
13396	13528	if (!new_prog)
@@ -13504,7 +13636,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) · 107 unmodified lines
13504	13636	mark_reg_known_zero(env, regs, i);
13505	13637	else if (regs[i].type == SCALAR_VALUE)
13506	13638	mark_reg_unknown(env, regs, i);
13507		− else if (regs[i].type == PTR_TO_MEM_OR_NULL) {
	13639	+ else if (base_type(regs[i].type) == PTR_TO_MEM) {
13508	13640	const u32 mem_size = regs[i].mem_size;
13509	13641
13510	13642	mark_reg_known_zero(env, regs, i);
@@ -14099,6 +14231,8 @@ int bpf_check(struct bpf_prog *prog, union bpf_attr attr, bpfptr_t uattr) · 588 unmodified lines
14099	14231	}
14100	14232	}
14101	14233
	14234	+ mark_verifier_state_clean(env);
	14235	+
14102	14236	if (IS_ERR(btf_vmlinux)) {
14103	14237	/* Either gcc or pahole or kernel are broken. */
14104	14238	verbose(env, "in-kernel BTF is malformed\n");

kernel/cgroup/cgroup.c+1−0

@@ -30,6 +30,7 @@ · 29 unmodified lines
30	30
31	31	#include "cgroup-internal.h"
32	32
	33	+#include <linux/bpf-cgroup.h>
33	34	#include <linux/cred.h>
34	35	#include <linux/errno.h>
35	36	#include <linux/init_task.h>

kernel/sysctl.c+1−0

@@ -33,6 +33,7 @@ · 32 unmodified lines
33	33	#include <linux/security.h>
34	34	#include <linux/ctype.h>
35	35	#include <linux/kmemleak.h>
	36	+#include <linux/filter.h>
36	37	#include <linux/fs.h>
37	38	#include <linux/init.h>
38	39	#include <linux/kernel.h>

kernel/trace/bpf_trace.c+67−14

@@ -345,7 +345,7 @@ static const struct bpf_func_proto bpf_probe_write_user_proto = { · 344 unmodified lines
345	345	.gpl_only = true,
346	346	.ret_type = RET_INTEGER,
347	347	.arg1_type = ARG_ANYTHING,
348		− .arg2_type = ARG_PTR_TO_MEM,
	348	+ .arg2_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
349	349	.arg3_type = ARG_CONST_SIZE,
350	350	};
351	351
@@ -394,7 +394,7 @@ static const struct bpf_func_proto bpf_trace_printk_proto = { · 42 unmodified lines
394	394	.func = bpf_trace_printk,
395	395	.gpl_only = true,
396	396	.ret_type = RET_INTEGER,
397		− .arg1_type = ARG_PTR_TO_MEM,
	397	+ .arg1_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
398	398	.arg2_type = ARG_CONST_SIZE,
399	399	};
400	400
@@ -450,9 +450,9 @@ static const struct bpf_func_proto bpf_trace_vprintk_proto = { · 49 unmodified lines
450	450	.func = bpf_trace_vprintk,
451	451	.gpl_only = true,
452	452	.ret_type = RET_INTEGER,
453		− .arg1_type = ARG_PTR_TO_MEM,
	453	+ .arg1_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
454	454	.arg2_type = ARG_CONST_SIZE,
455		− .arg3_type = ARG_PTR_TO_MEM_OR_NULL,
	455	+ .arg3_type = ARG_PTR_TO_MEM \| PTR_MAYBE_NULL \| MEM_RDONLY,
456	456	.arg4_type = ARG_CONST_SIZE_OR_ZERO,
457	457	};
458	458
@@ -492,9 +492,9 @@ static const struct bpf_func_proto bpf_seq_printf_proto = { · 33 unmodified lines
492	492	.ret_type = RET_INTEGER,
493	493	.arg1_type = ARG_PTR_TO_BTF_ID,
494	494	.arg1_btf_id = &btf_seq_file_ids[0],
495		− .arg2_type = ARG_PTR_TO_MEM,
	495	+ .arg2_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
496	496	.arg3_type = ARG_CONST_SIZE,
497		− .arg4_type = ARG_PTR_TO_MEM_OR_NULL,
	497	+ .arg4_type = ARG_PTR_TO_MEM \| PTR_MAYBE_NULL \| MEM_RDONLY,
498	498	.arg5_type = ARG_CONST_SIZE_OR_ZERO,
499	499	};
500	500
@@ -509,7 +509,7 @@ static const struct bpf_func_proto bpf_seq_write_proto = { · 8 unmodified lines
509	509	.ret_type = RET_INTEGER,
510	510	.arg1_type = ARG_PTR_TO_BTF_ID,
511	511	.arg1_btf_id = &btf_seq_file_ids[0],
512		− .arg2_type = ARG_PTR_TO_MEM,
	512	+ .arg2_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
513	513	.arg3_type = ARG_CONST_SIZE_OR_ZERO,
514	514	};
515	515
@@ -533,7 +533,7 @@ static const struct bpf_func_proto bpf_seq_printf_btf_proto = { · 17 unmodified lines
533	533	.ret_type = RET_INTEGER,
534	534	.arg1_type = ARG_PTR_TO_BTF_ID,
535	535	.arg1_btf_id = &btf_seq_file_ids[0],
536		− .arg2_type = ARG_PTR_TO_MEM,
	536	+ .arg2_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
537	537	.arg3_type = ARG_CONST_SIZE_OR_ZERO,
538	538	.arg4_type = ARG_ANYTHING,
539	539	};
@@ -694,7 +694,7 @@ static const struct bpf_func_proto bpf_perf_event_output_proto = { · 154 unmodified lines
694	694	.arg1_type = ARG_PTR_TO_CTX,
695	695	.arg2_type = ARG_CONST_MAP_PTR,
696	696	.arg3_type = ARG_ANYTHING,
697		− .arg4_type = ARG_PTR_TO_MEM,
	697	+ .arg4_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
698	698	.arg5_type = ARG_CONST_SIZE_OR_ZERO,
699	699	};
700	700
@@ -1004,7 +1004,7 @@ const struct bpf_func_proto bpf_snprintf_btf_proto = { · 303 unmodified lines
1004	1004	.ret_type = RET_INTEGER,
1005	1005	.arg1_type = ARG_PTR_TO_MEM,
1006	1006	.arg2_type = ARG_CONST_SIZE,
1007		− .arg3_type = ARG_PTR_TO_MEM,
	1007	+ .arg3_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
1008	1008	.arg4_type = ARG_CONST_SIZE,
1009	1009	.arg5_type = ARG_ANYTHING,
1010	1010	};
@@ -1012,7 +1012,7 @@ const struct bpf_func_proto bpf_snprintf_btf_proto = { · 1 unmodified line
1012	1012	BPF_CALL_1(bpf_get_func_ip_tracing, void *, ctx)
1013	1013	{
1014	1014	/* This helper call is inlined by verifier. */
1015		− return ((u64 *)ctx)[-1];
	1015	+ return ((u64 *)ctx)[-2];
1016	1016	}
1017	1017
1018	1018	static const struct bpf_func_proto bpf_get_func_ip_proto_tracing = {
@@ -1091,6 +1091,53 @@ static const struct bpf_func_proto bpf_get_branch_snapshot_proto = { · 72 unmodified lines
1091	1091	.arg2_type = ARG_CONST_SIZE_OR_ZERO,
1092	1092	};
1093	1093
	1094	+BPF_CALL_3(get_func_arg, void , ctx, u32, n, u64 , value)
	1095	+{
	1096	+ /* This helper call is inlined by verifier. */
	1097	+ u64 nr_args = ((u64 *)ctx)[-1];
	1098	+
	1099	+ if ((u64) n >= nr_args)
	1100	+ return -EINVAL;
	1101	+ value = ((u64 )ctx)[n];
	1102	+ return 0;
	1103	+}
	1104	+
	1105	+static const struct bpf_func_proto bpf_get_func_arg_proto = {
	1106	+ .func = get_func_arg,
	1107	+ .ret_type = RET_INTEGER,
	1108	+ .arg1_type = ARG_PTR_TO_CTX,
	1109	+ .arg2_type = ARG_ANYTHING,
	1110	+ .arg3_type = ARG_PTR_TO_LONG,
	1111	+};
	1112	+
	1113	+BPF_CALL_2(get_func_ret, void , ctx, u64 , value)
	1114	+{
	1115	+ /* This helper call is inlined by verifier. */
	1116	+ u64 nr_args = ((u64 *)ctx)[-1];
	1117	+
	1118	+ value = ((u64 )ctx)[nr_args];
	1119	+ return 0;
	1120	+}
	1121	+
	1122	+static const struct bpf_func_proto bpf_get_func_ret_proto = {
	1123	+ .func = get_func_ret,
	1124	+ .ret_type = RET_INTEGER,
	1125	+ .arg1_type = ARG_PTR_TO_CTX,
	1126	+ .arg2_type = ARG_PTR_TO_LONG,
	1127	+};
	1128	+
	1129	+BPF_CALL_1(get_func_arg_cnt, void *, ctx)
	1130	+{
	1131	+ /* This helper call is inlined by verifier. */
	1132	+ return ((u64 *)ctx)[-1];
	1133	+}
	1134	+
	1135	+static const struct bpf_func_proto bpf_get_func_arg_cnt_proto = {
	1136	+ .func = get_func_arg_cnt,
	1137	+ .ret_type = RET_INTEGER,
	1138	+ .arg1_type = ARG_PTR_TO_CTX,
	1139	+};
	1140	+
1094	1141	static const struct bpf_func_proto *
1095	1142	bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
1096	1143	{
@@ -1287,7 +1334,7 @@ static const struct bpf_func_proto bpf_perf_event_output_proto_tp = { · 190 unmodified lines
1287	1334	.arg1_type = ARG_PTR_TO_CTX,
1288	1335	.arg2_type = ARG_CONST_MAP_PTR,
1289	1336	.arg3_type = ARG_ANYTHING,
1290		− .arg4_type = ARG_PTR_TO_MEM,
	1337	+ .arg4_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
1291	1338	.arg5_type = ARG_CONST_SIZE_OR_ZERO,
1292	1339	};
1293	1340
@@ -1509,7 +1556,7 @@ static const struct bpf_func_proto bpf_perf_event_output_proto_raw_tp = { · 215 unmodified lines
1509	1556	.arg1_type = ARG_PTR_TO_CTX,
1510	1557	.arg2_type = ARG_CONST_MAP_PTR,
1511	1558	.arg3_type = ARG_ANYTHING,
1512		− .arg4_type = ARG_PTR_TO_MEM,
	1559	+ .arg4_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
1513	1560	.arg5_type = ARG_CONST_SIZE_OR_ZERO,
1514	1561	};
1515	1562
@@ -1563,7 +1610,7 @@ static const struct bpf_func_proto bpf_get_stack_proto_raw_tp = { · 47 unmodified lines
1563	1610	.gpl_only = true,
1564	1611	.ret_type = RET_INTEGER,
1565	1612	.arg1_type = ARG_PTR_TO_CTX,
1566		− .arg2_type = ARG_PTR_TO_MEM,
	1613	+ .arg2_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
1567	1614	.arg3_type = ARG_CONST_SIZE_OR_ZERO,
1568	1615	.arg4_type = ARG_ANYTHING,
1569	1616	};
@@ -1629,6 +1676,12 @@ tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 59 unmodified lines
1629	1676	NULL;
1630	1677	case BPF_FUNC_d_path:
1631	1678	return &bpf_d_path_proto;
	1679	+ case BPF_FUNC_get_func_arg:
	1680	+ return bpf_prog_has_trampoline(prog) ? &bpf_get_func_arg_proto : NULL;
	1681	+ case BPF_FUNC_get_func_ret:
	1682	+ return bpf_prog_has_trampoline(prog) ? &bpf_get_func_ret_proto : NULL;
	1683	+ case BPF_FUNC_get_func_arg_cnt:
	1684	+ return bpf_prog_has_trampoline(prog) ? &bpf_get_func_arg_cnt_proto : NULL;
1632	1685	default:
1633	1686	fn = raw_tp_prog_func_proto(func_id, prog);
1634	1687	if (!fn && prog->expected_attach_type == BPF_TRACE_ITER)

kernel/trace/trace_kprobe.c+1−0

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	7	*/
8	8	#define pr_fmt(fmt) "trace_kprobe: " fmt
9	9
	10	+#include <linux/bpf-cgroup.h>
10	11	#include <linux/security.h>
11	12	#include <linux/module.h>
12	13	#include <linux/uaccess.h>

kernel/trace/trace_uprobe.c+1−0

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	7	*/
8	8	#define pr_fmt(fmt) "trace_uprobe: " fmt
9	9
	10	+#include <linux/bpf-cgroup.h>
10	11	#include <linux/security.h>
11	12	#include <linux/ctype.h>
12	13	#include <linux/module.h>

net/bluetooth/bnep/sock.c+1−0

@@ -24,6 +24,7 @@ · 23 unmodified lines
24	24	SOFTWARE IS DISCLAIMED.
25	25	*/
26	26
	27	+#include <linux/compat.h>
27	28	#include <linux/export.h>
28	29	#include <linux/file.h>
29	30

net/bluetooth/eir.h+2−0

@@ -5,6 +5,8 @@ · 4 unmodified lines
5	5	* Copyright (C) 2021 Intel Corporation
6	6	*/
7	7
	8	+#include <asm/unaligned.h>
	9	+
8	10	void eir_create(struct hci_dev hdev, u8 data);
9	11
10	12	u8 eir_create_adv_data(struct hci_dev hdev, u8 instance, u8 ptr);

net/bluetooth/hidp/sock.c+1−0

@@ -20,6 +20,7 @@ · 19 unmodified lines
20	20	SOFTWARE IS DISCLAIMED.
21	21	*/
22	22
	23	+#include <linux/compat.h>
23	24	#include <linux/export.h>
24	25	#include <linux/file.h>
25	26

net/bluetooth/l2cap_sock.c+1−0

@@ -29,6 +29,7 @@ · 28 unmodified lines
29	29
30	30	#include <linux/module.h>
31	31	#include <linux/export.h>
	32	+#include <linux/filter.h>
32	33	#include <linux/sched/signal.h>
33	34
34	35	#include <net/bluetooth/bluetooth.h>

net/bridge/br_ioctl.c+1−0

@@ -8,6 +8,7 @@ · 7 unmodified lines
8	8	*/
9	9
10	10	#include <linux/capability.h>
	11	+#include <linux/compat.h>
11	12	#include <linux/kernel.h>
12	13	#include <linux/if_bridge.h>
13	14	#include <linux/netdevice.h>

net/caif/caif_socket.c+1−0

@@ -6,6 +6,7 @@ · 5 unmodified lines
6	6
7	7	#define pr_fmt(fmt) KBUILD_MODNAME ":%s(): " fmt, __func__
8	8
	9	+#include <linux/filter.h>
9	10	#include <linux/fs.h>
10	11	#include <linux/init.h>
11	12	#include <linux/module.h>

net/core/bpf_sk_storage.c+8−2

@@ -13,6 +13,7 @@ · 12 unmodified lines
13	13	#include <net/sock.h>
14	14	#include <uapi/linux/sock_diag.h>
15	15	#include <uapi/linux/btf.h>
	16	+#include <linux/rcupdate_trace.h>
16	17
17	18	DEFINE_BPF_STORAGE_CACHE(sk_cache);
18	19
@@ -22,7 +23,8 @@ bpf_sk_storage_lookup(struct sock sk, struct bpf_map map, bool cacheit_lockit) · 3 unmodified lines
22	23	struct bpf_local_storage *sk_storage;
23	24	struct bpf_local_storage_map *smap;
24	25
25		− sk_storage = rcu_dereference(sk->sk_bpf_storage);
	26	+ sk_storage =
	27	+ rcu_dereference_check(sk->sk_bpf_storage, bpf_rcu_lock_held());
26	28	if (!sk_storage)
27	29	return NULL;
28	30
@@ -258,6 +260,7 @@ BPF_CALL_4(bpf_sk_storage_get, struct bpf_map , map, struct sock , sk, · 229 unmodified lines
258	260	{
259	261	struct bpf_local_storage_data *sdata;
260	262
	263	+ WARN_ON_ONCE(!bpf_rcu_lock_held());
261	264	if (!sk \|\| !sk_fullsock(sk) \|\| flags > BPF_SK_STORAGE_GET_F_CREATE)
262	265	return (unsigned long)NULL;
263	266
@@ -288,6 +291,7 @@ BPF_CALL_4(bpf_sk_storage_get, struct bpf_map , map, struct sock , sk, · 24 unmodified lines
288	291
289	292	BPF_CALL_2(bpf_sk_storage_delete, struct bpf_map , map, struct sock , sk)
290	293	{
	294	+ WARN_ON_ONCE(!bpf_rcu_lock_held());
291	295	if (!sk \|\| !sk_fullsock(sk))
292	296	return -EINVAL;
293	297
@@ -416,6 +420,7 @@ static bool bpf_sk_storage_tracing_allowed(const struct bpf_prog *prog) · 122 unmodified lines
416	420	BPF_CALL_4(bpf_sk_storage_get_tracing, struct bpf_map , map, struct sock , sk,
417	421	void *, value, u64, flags)
418	422	{
	423	+ WARN_ON_ONCE(!bpf_rcu_lock_held());
419	424	if (in_hardirq() \|\| in_nmi())
420	425	return (unsigned long)NULL;
421	426
@@ -425,6 +430,7 @@ BPF_CALL_4(bpf_sk_storage_get_tracing, struct bpf_map , map, struct sock , sk, · 3 unmodified lines
425	430	BPF_CALL_2(bpf_sk_storage_delete_tracing, struct bpf_map *, map,
426	431	struct sock *, sk)
427	432	{
	433	+ WARN_ON_ONCE(!bpf_rcu_lock_held());
428	434	if (in_hardirq() \|\| in_nmi())
429	435	return -EPERM;
430	436
@@ -929,7 +935,7 @@ static struct bpf_iter_reg bpf_sk_storage_map_reg_info = { · 498 unmodified lines
929	935	{ offsetof(struct bpf_iter__bpf_sk_storage_map, sk),
930	936	PTR_TO_BTF_ID_OR_NULL },
931	937	{ offsetof(struct bpf_iter__bpf_sk_storage_map, value),
932		− PTR_TO_RDWR_BUF_OR_NULL },
	938	+ PTR_TO_BUF \| PTR_MAYBE_NULL },
933	939	},
934	940	.seq_info = &iter_seq_info,
935	941	};

net/core/dev.c+1−1

@@ -4712,7 +4712,7 @@ static u32 netif_receive_generic_xdp(struct sk_buff *skb, · 4711 unmodified lines
4712	4712	case XDP_PASS:
4713	4713	break;
4714	4714	default:
4715		− bpf_warn_invalid_xdp_action(act);
	4715	+ bpf_warn_invalid_xdp_action(skb->dev, xdp_prog, act);
4716	4716	fallthrough;
4717	4717	case XDP_ABORTED:
4718	4718	trace_xdp_exception(skb->dev, xdp_prog, act);

net/core/devlink.c+1−0

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	7	* Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
8	8	*/
9	9
	10	+#include <linux/etherdevice.h>
10	11	#include <linux/kernel.h>
11	12	#include <linux/module.h>
12	13	#include <linux/types.h>

net/core/filter.c+36−36

@@ -1712,7 +1712,7 @@ static const struct bpf_func_proto bpf_skb_store_bytes_proto = { · 1711 unmodified lines
1712	1712	.ret_type = RET_INTEGER,
1713	1713	.arg1_type = ARG_PTR_TO_CTX,
1714	1714	.arg2_type = ARG_ANYTHING,
1715		− .arg3_type = ARG_PTR_TO_MEM,
	1715	+ .arg3_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
1716	1716	.arg4_type = ARG_CONST_SIZE,
1717	1717	.arg5_type = ARG_ANYTHING,
1718	1718	};
@@ -2017,9 +2017,9 @@ static const struct bpf_func_proto bpf_csum_diff_proto = { · 298 unmodified lines
2017	2017	.gpl_only = false,
2018	2018	.pkt_access = true,
2019	2019	.ret_type = RET_INTEGER,
2020		− .arg1_type = ARG_PTR_TO_MEM_OR_NULL,
	2020	+ .arg1_type = ARG_PTR_TO_MEM \| PTR_MAYBE_NULL \| MEM_RDONLY,
2021	2021	.arg2_type = ARG_CONST_SIZE_OR_ZERO,
2022		− .arg3_type = ARG_PTR_TO_MEM_OR_NULL,
	2022	+ .arg3_type = ARG_PTR_TO_MEM \| PTR_MAYBE_NULL \| MEM_RDONLY,
2023	2023	.arg4_type = ARG_CONST_SIZE_OR_ZERO,
2024	2024	.arg5_type = ARG_ANYTHING,
2025	2025	};
@@ -2540,7 +2540,7 @@ static const struct bpf_func_proto bpf_redirect_neigh_proto = { · 514 unmodified lines
2540	2540	.gpl_only = false,
2541	2541	.ret_type = RET_INTEGER,
2542	2542	.arg1_type = ARG_ANYTHING,
2543		− .arg2_type = ARG_PTR_TO_MEM_OR_NULL,
	2543	+ .arg2_type = ARG_PTR_TO_MEM \| PTR_MAYBE_NULL \| MEM_RDONLY,
2544	2544	.arg3_type = ARG_CONST_SIZE_OR_ZERO,
2545	2545	.arg4_type = ARG_ANYTHING,
2546	2546	};
@@ -4173,7 +4173,7 @@ static const struct bpf_func_proto bpf_skb_event_output_proto = { · 1626 unmodified lines
4173	4173	.arg1_type = ARG_PTR_TO_CTX,
4174	4174	.arg2_type = ARG_CONST_MAP_PTR,
4175	4175	.arg3_type = ARG_ANYTHING,
4176		− .arg4_type = ARG_PTR_TO_MEM,
	4176	+ .arg4_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
4177	4177	.arg5_type = ARG_CONST_SIZE_OR_ZERO,
4178	4178	};
4179	4179
@@ -4187,7 +4187,7 @@ const struct bpf_func_proto bpf_skb_output_proto = { · 7 unmodified lines
4187	4187	.arg1_btf_id = &bpf_skb_output_btf_ids[0],
4188	4188	.arg2_type = ARG_CONST_MAP_PTR,
4189	4189	.arg3_type = ARG_ANYTHING,
4190		− .arg4_type = ARG_PTR_TO_MEM,
	4190	+ .arg4_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
4191	4191	.arg5_type = ARG_CONST_SIZE_OR_ZERO,
4192	4192	};
4193	4193
@@ -4370,7 +4370,7 @@ static const struct bpf_func_proto bpf_skb_set_tunnel_key_proto = { · 176 unmodified lines
4370	4370	.gpl_only = false,
4371	4371	.ret_type = RET_INTEGER,
4372	4372	.arg1_type = ARG_PTR_TO_CTX,
4373		− .arg2_type = ARG_PTR_TO_MEM,
	4373	+ .arg2_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
4374	4374	.arg3_type = ARG_CONST_SIZE,
4375	4375	.arg4_type = ARG_ANYTHING,
4376	4376	};
@@ -4396,7 +4396,7 @@ static const struct bpf_func_proto bpf_skb_set_tunnel_opt_proto = { · 19 unmodified lines
4396	4396	.gpl_only = false,
4397	4397	.ret_type = RET_INTEGER,
4398	4398	.arg1_type = ARG_PTR_TO_CTX,
4399		− .arg2_type = ARG_PTR_TO_MEM,
	4399	+ .arg2_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
4400	4400	.arg3_type = ARG_CONST_SIZE,
4401	4401	};
4402	4402
@@ -4566,7 +4566,7 @@ static const struct bpf_func_proto bpf_xdp_event_output_proto = { · 163 unmodified lines
4566	4566	.arg1_type = ARG_PTR_TO_CTX,
4567	4567	.arg2_type = ARG_CONST_MAP_PTR,
4568	4568	.arg3_type = ARG_ANYTHING,
4569		− .arg4_type = ARG_PTR_TO_MEM,
	4569	+ .arg4_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
4570	4570	.arg5_type = ARG_CONST_SIZE_OR_ZERO,
4571	4571	};
4572	4572
@@ -4580,7 +4580,7 @@ const struct bpf_func_proto bpf_xdp_output_proto = { · 7 unmodified lines
4580	4580	.arg1_btf_id = &bpf_xdp_output_btf_ids[0],
4581	4581	.arg2_type = ARG_CONST_MAP_PTR,
4582	4582	.arg3_type = ARG_ANYTHING,
4583		− .arg4_type = ARG_PTR_TO_MEM,
	4583	+ .arg4_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
4584	4584	.arg5_type = ARG_CONST_SIZE_OR_ZERO,
4585	4585	};
4586	4586
@@ -5066,7 +5066,7 @@ const struct bpf_func_proto bpf_sk_setsockopt_proto = { · 479 unmodified lines
5066	5066	.arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
5067	5067	.arg2_type = ARG_ANYTHING,
5068	5068	.arg3_type = ARG_ANYTHING,
5069		− .arg4_type = ARG_PTR_TO_MEM,
	5069	+ .arg4_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
5070	5070	.arg5_type = ARG_CONST_SIZE,
5071	5071	};
5072	5072
@@ -5100,7 +5100,7 @@ static const struct bpf_func_proto bpf_sock_addr_setsockopt_proto = { · 27 unmodified lines
5100	5100	.arg1_type = ARG_PTR_TO_CTX,
5101	5101	.arg2_type = ARG_ANYTHING,
5102	5102	.arg3_type = ARG_ANYTHING,
5103		− .arg4_type = ARG_PTR_TO_MEM,
	5103	+ .arg4_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
5104	5104	.arg5_type = ARG_CONST_SIZE,
5105	5105	};
5106	5106
@@ -5134,7 +5134,7 @@ static const struct bpf_func_proto bpf_sock_ops_setsockopt_proto = { · 27 unmodified lines
5134	5134	.arg1_type = ARG_PTR_TO_CTX,
5135	5135	.arg2_type = ARG_ANYTHING,
5136	5136	.arg3_type = ARG_ANYTHING,
5137		− .arg4_type = ARG_PTR_TO_MEM,
	5137	+ .arg4_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
5138	5138	.arg5_type = ARG_CONST_SIZE,
5139	5139	};
5140	5140
@@ -5309,7 +5309,7 @@ static const struct bpf_func_proto bpf_bind_proto = { · 168 unmodified lines
5309	5309	.gpl_only = false,
5310	5310	.ret_type = RET_INTEGER,
5311	5311	.arg1_type = ARG_PTR_TO_CTX,
5312		− .arg2_type = ARG_PTR_TO_MEM,
	5312	+ .arg2_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
5313	5313	.arg3_type = ARG_CONST_SIZE,
5314	5314	};
5315	5315
@@ -5897,7 +5897,7 @@ static const struct bpf_func_proto bpf_lwt_in_push_encap_proto = { · 581 unmodified lines
5897	5897	.ret_type = RET_INTEGER,
5898	5898	.arg1_type = ARG_PTR_TO_CTX,
5899	5899	.arg2_type = ARG_ANYTHING,
5900		− .arg3_type = ARG_PTR_TO_MEM,
	5900	+ .arg3_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
5901	5901	.arg4_type = ARG_CONST_SIZE
5902	5902	};
5903	5903
@@ -5907,7 +5907,7 @@ static const struct bpf_func_proto bpf_lwt_xmit_push_encap_proto = { · 3 unmodified lines
5907	5907	.ret_type = RET_INTEGER,
5908	5908	.arg1_type = ARG_PTR_TO_CTX,
5909	5909	.arg2_type = ARG_ANYTHING,
5910		− .arg3_type = ARG_PTR_TO_MEM,
	5910	+ .arg3_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
5911	5911	.arg4_type = ARG_CONST_SIZE
5912	5912	};
5913	5913
@@ -5950,7 +5950,7 @@ static const struct bpf_func_proto bpf_lwt_seg6_store_bytes_proto = { · 36 unmodified lines
5950	5950	.ret_type = RET_INTEGER,
5951	5951	.arg1_type = ARG_PTR_TO_CTX,
5952	5952	.arg2_type = ARG_ANYTHING,
5953		− .arg3_type = ARG_PTR_TO_MEM,
	5953	+ .arg3_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
5954	5954	.arg4_type = ARG_CONST_SIZE
5955	5955	};
5956	5956
@@ -6038,7 +6038,7 @@ static const struct bpf_func_proto bpf_lwt_seg6_action_proto = { · 81 unmodified lines
6038	6038	.ret_type = RET_INTEGER,
6039	6039	.arg1_type = ARG_PTR_TO_CTX,
6040	6040	.arg2_type = ARG_ANYTHING,
6041		− .arg3_type = ARG_PTR_TO_MEM,
	6041	+ .arg3_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
6042	6042	.arg4_type = ARG_CONST_SIZE
6043	6043	};
6044	6044
@@ -6263,7 +6263,7 @@ static const struct bpf_func_proto bpf_skc_lookup_tcp_proto = { · 218 unmodified lines
6263	6263	.pkt_access = true,
6264	6264	.ret_type = RET_PTR_TO_SOCK_COMMON_OR_NULL,
6265	6265	.arg1_type = ARG_PTR_TO_CTX,
6266		− .arg2_type = ARG_PTR_TO_MEM,
	6266	+ .arg2_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
6267	6267	.arg3_type = ARG_CONST_SIZE,
6268	6268	.arg4_type = ARG_ANYTHING,
6269	6269	.arg5_type = ARG_ANYTHING,
@@ -6282,7 +6282,7 @@ static const struct bpf_func_proto bpf_sk_lookup_tcp_proto = { · 12 unmodified lines
6282	6282	.pkt_access = true,
6283	6283	.ret_type = RET_PTR_TO_SOCKET_OR_NULL,
6284	6284	.arg1_type = ARG_PTR_TO_CTX,
6285		− .arg2_type = ARG_PTR_TO_MEM,
	6285	+ .arg2_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
6286	6286	.arg3_type = ARG_CONST_SIZE,
6287	6287	.arg4_type = ARG_ANYTHING,
6288	6288	.arg5_type = ARG_ANYTHING,
@@ -6301,7 +6301,7 @@ static const struct bpf_func_proto bpf_sk_lookup_udp_proto = { · 12 unmodified lines
6301	6301	.pkt_access = true,
6302	6302	.ret_type = RET_PTR_TO_SOCKET_OR_NULL,
6303	6303	.arg1_type = ARG_PTR_TO_CTX,
6304		− .arg2_type = ARG_PTR_TO_MEM,
	6304	+ .arg2_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
6305	6305	.arg3_type = ARG_CONST_SIZE,
6306	6306	.arg4_type = ARG_ANYTHING,
6307	6307	.arg5_type = ARG_ANYTHING,
@@ -6338,7 +6338,7 @@ static const struct bpf_func_proto bpf_xdp_sk_lookup_udp_proto = { · 30 unmodified lines
6338	6338	.pkt_access = true,
6339	6339	.ret_type = RET_PTR_TO_SOCKET_OR_NULL,
6340	6340	.arg1_type = ARG_PTR_TO_CTX,
6341		− .arg2_type = ARG_PTR_TO_MEM,
	6341	+ .arg2_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
6342	6342	.arg3_type = ARG_CONST_SIZE,
6343	6343	.arg4_type = ARG_ANYTHING,
6344	6344	.arg5_type = ARG_ANYTHING,
@@ -6361,7 +6361,7 @@ static const struct bpf_func_proto bpf_xdp_skc_lookup_tcp_proto = { · 16 unmodified lines
6361	6361	.pkt_access = true,
6362	6362	.ret_type = RET_PTR_TO_SOCK_COMMON_OR_NULL,
6363	6363	.arg1_type = ARG_PTR_TO_CTX,
6364		− .arg2_type = ARG_PTR_TO_MEM,
	6364	+ .arg2_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
6365	6365	.arg3_type = ARG_CONST_SIZE,
6366	6366	.arg4_type = ARG_ANYTHING,
6367	6367	.arg5_type = ARG_ANYTHING,
@@ -6384,7 +6384,7 @@ static const struct bpf_func_proto bpf_xdp_sk_lookup_tcp_proto = { · 16 unmodified lines
6384	6384	.pkt_access = true,
6385	6385	.ret_type = RET_PTR_TO_SOCKET_OR_NULL,
6386	6386	.arg1_type = ARG_PTR_TO_CTX,
6387		− .arg2_type = ARG_PTR_TO_MEM,
	6387	+ .arg2_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
6388	6388	.arg3_type = ARG_CONST_SIZE,
6389	6389	.arg4_type = ARG_ANYTHING,
6390	6390	.arg5_type = ARG_ANYTHING,
@@ -6403,7 +6403,7 @@ static const struct bpf_func_proto bpf_sock_addr_skc_lookup_tcp_proto = { · 12 unmodified lines
6403	6403	.gpl_only = false,
6404	6404	.ret_type = RET_PTR_TO_SOCK_COMMON_OR_NULL,
6405	6405	.arg1_type = ARG_PTR_TO_CTX,
6406		− .arg2_type = ARG_PTR_TO_MEM,
	6406	+ .arg2_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
6407	6407	.arg3_type = ARG_CONST_SIZE,
6408	6408	.arg4_type = ARG_ANYTHING,
6409	6409	.arg5_type = ARG_ANYTHING,
@@ -6422,7 +6422,7 @@ static const struct bpf_func_proto bpf_sock_addr_sk_lookup_tcp_proto = { · 12 unmodified lines
6422	6422	.gpl_only = false,
6423	6423	.ret_type = RET_PTR_TO_SOCKET_OR_NULL,
6424	6424	.arg1_type = ARG_PTR_TO_CTX,
6425		− .arg2_type = ARG_PTR_TO_MEM,
	6425	+ .arg2_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
6426	6426	.arg3_type = ARG_CONST_SIZE,
6427	6427	.arg4_type = ARG_ANYTHING,
6428	6428	.arg5_type = ARG_ANYTHING,
@@ -6441,7 +6441,7 @@ static const struct bpf_func_proto bpf_sock_addr_sk_lookup_udp_proto = { · 12 unmodified lines
6441	6441	.gpl_only = false,
6442	6442	.ret_type = RET_PTR_TO_SOCKET_OR_NULL,
6443	6443	.arg1_type = ARG_PTR_TO_CTX,
6444		− .arg2_type = ARG_PTR_TO_MEM,
	6444	+ .arg2_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
6445	6445	.arg3_type = ARG_CONST_SIZE,
6446	6446	.arg4_type = ARG_ANYTHING,
6447	6447	.arg5_type = ARG_ANYTHING,
@@ -6754,9 +6754,9 @@ static const struct bpf_func_proto bpf_tcp_check_syncookie_proto = { · 306 unmodified lines
6754	6754	.pkt_access = true,
6755	6755	.ret_type = RET_INTEGER,
6756	6756	.arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
6757		− .arg2_type = ARG_PTR_TO_MEM,
	6757	+ .arg2_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
6758	6758	.arg3_type = ARG_CONST_SIZE,
6759		− .arg4_type = ARG_PTR_TO_MEM,
	6759	+ .arg4_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
6760	6760	.arg5_type = ARG_CONST_SIZE,
6761	6761	};
6762	6762
@@ -6823,9 +6823,9 @@ static const struct bpf_func_proto bpf_tcp_gen_syncookie_proto = { · 60 unmodified lines
6823	6823	.pkt_access = true,
6824	6824	.ret_type = RET_INTEGER,
6825	6825	.arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
6826		− .arg2_type = ARG_PTR_TO_MEM,
	6826	+ .arg2_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
6827	6827	.arg3_type = ARG_CONST_SIZE,
6828		− .arg4_type = ARG_PTR_TO_MEM,
	6828	+ .arg4_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
6829	6829	.arg5_type = ARG_CONST_SIZE,
6830	6830	};
6831	6831
@@ -7054,7 +7054,7 @@ static const struct bpf_func_proto bpf_sock_ops_store_hdr_opt_proto = { · 222 unmodified lines
7054	7054	.gpl_only = false,
7055	7055	.ret_type = RET_INTEGER,
7056	7056	.arg1_type = ARG_PTR_TO_CTX,
7057		− .arg2_type = ARG_PTR_TO_MEM,
	7057	+ .arg2_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
7058	7058	.arg3_type = ARG_CONST_SIZE,
7059	7059	.arg4_type = ARG_ANYTHING,
7060	7060	};
@@ -8180,13 +8180,13 @@ static bool xdp_is_valid_access(int off, int size, · 1119 unmodified lines
8180	8180	return __is_valid_xdp_access(off, size);
8181	8181	}
8182	8182
8183		−void bpf_warn_invalid_xdp_action(u32 act)
	8183	+void bpf_warn_invalid_xdp_action(struct net_device dev, struct bpf_prog prog, u32 act)
8184	8184	{
8185	8185	const u32 act_max = XDP_REDIRECT;
8186	8186
8187		− WARN_ONCE(1, "%s XDP return value %u, expect packet loss!\n",
8188		− act > act_max ? "Illegal" : "Driver unsupported",
8189		− act);
	8187	+ pr_warn_once("%s XDP return value %u on prog %s (id %d) dev %s, expect packet loss!\n",
	8188	+ act > act_max ? "Illegal" : "Driver unsupported",
	8189	+ act, prog->aux->name, prog->aux->id, dev ? dev->name : "N/A");
8190	8190	}
8191	8191	EXPORT_SYMBOL_GPL(bpf_warn_invalid_xdp_action);
8192	8192

net/core/flow_dissector.c+1−0

@@ -5,6 +5,7 @@ · 4 unmodified lines
5	5	#include <linux/ip.h>
6	6	#include <linux/ipv6.h>
7	7	#include <linux/if_vlan.h>
	8	+#include <linux/filter.h>
8	9	#include <net/dsa.h>
9	10	#include <net/dst_metadata.h>
10	11	#include <net/ip.h>

net/core/lwt_bpf.c+1−0

@@ -2,6 +2,7 @@ · 1 unmodified line
2	2	/* Copyright (c) 2016 Thomas Graf <tgraf@tgraf.ch>
3	3	*/
4	4
	5	+#include <linux/filter.h>
5	6	#include <linux/kernel.h>
6	7	#include <linux/module.h>
7	8	#include <linux/skbuff.h>

net/core/sock_diag.c+1−0

@@ -1,5 +1,6 @@
1	1	/* License: GPL */
2	2
	3	+#include <linux/filter.h>
3	4	#include <linux/mutex.h>
4	5	#include <linux/socket.h>
5	6	#include <linux/skbuff.h>

net/core/sock_map.c+1−1

@@ -1564,7 +1564,7 @@ static struct bpf_iter_reg sock_map_iter_reg = { · 1563 unmodified lines
1564	1564	.ctx_arg_info_size = 2,
1565	1565	.ctx_arg_info = {
1566	1566	{ offsetof(struct bpf_iter__sockmap, key),
1567		− PTR_TO_RDONLY_BUF_OR_NULL },
	1567	+ PTR_TO_BUF \| PTR_MAYBE_NULL \| MEM_RDONLY },
1568	1568	{ offsetof(struct bpf_iter__sockmap, sk),
1569	1569	PTR_TO_BTF_ID_OR_NULL },
1570	1570	},

net/core/sysctl_net_core.c+1−0

@@ -6,6 +6,7 @@ · 5 unmodified lines
6	6	* Added /proc/sys/net/core directory entry (empty =) ). [MS]
7	7	*/
8	8
	9	+#include <linux/filter.h>
9	10	#include <linux/mm.h>
10	11	#include <linux/sysctl.h>
11	12	#include <linux/module.h>

net/decnet/dn_nsp_in.c+1−0

@@ -38,6 +38,7 @@ · 37 unmodified lines
38	38	*******************************************************************************/
39	39
40	40	#include <linux/errno.h>
	41	+#include <linux/filter.h>
41	42	#include <linux/types.h>
42	43	#include <linux/socket.h>
43	44	#include <linux/in.h>

net/dsa/dsa_priv.h+1−0

@@ -8,6 +8,7 @@ · 7 unmodified lines
8	8	#define __DSA_PRIV_H
9	9
10	10	#include <linux/if_bridge.h>
	11	+#include <linux/if_vlan.h>
11	12	#include <linux/phy.h>
12	13	#include <linux/netdevice.h>
13	14	#include <linux/netpoll.h>

net/ethtool/ioctl.c+1−0

@@ -8,6 +8,7 @@ · 7 unmodified lines
8	8	*/
9	9
10	10	#include <linux/compat.h>
	11	+#include <linux/etherdevice.h>
11	12	#include <linux/module.h>
12	13	#include <linux/types.h>
13	14	#include <linux/capability.h>

net/ipv4/nexthop.c+1−0

@@ -8,6 +8,7 @@ · 7 unmodified lines
8	8	#include <linux/nexthop.h>
9	9	#include <linux/rtnetlink.h>
10	10	#include <linux/slab.h>
	11	+#include <linux/vmalloc.h>
11	12	#include <net/arp.h>
12	13	#include <net/ipv6_stubs.h>
13	14	#include <net/lwtunnel.h>

net/ipv4/udp.c+1−0

@@ -74,6 +74,7 @@ · 73 unmodified lines
74	74
75	75	#define pr_fmt(fmt) "UDP: " fmt
76	76
	77	+#include <linux/bpf-cgroup.h>
77	78	#include <linux/uaccess.h>
78	79	#include <asm/ioctls.h>
79	80	#include <linux/memblock.h>

net/ipv6/ip6_fib.c+1−0

@@ -15,6 +15,7 @@ · 14 unmodified lines
15	15
16	16	#define pr_fmt(fmt) "IPv6: " fmt
17	17
	18	+#include <linux/bpf.h>
18	19	#include <linux/errno.h>
19	20	#include <linux/types.h>
20	21	#include <linux/net.h>

net/ipv6/seg6_local.c+1−0

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	7	* eBPF support: Mathieu Xhonneux <m.xhonneux@gmail.com>
8	8	*/
9	9
	10	+#include <linux/filter.h>
10	11	#include <linux/types.h>
11	12	#include <linux/skbuff.h>
12	13	#include <linux/net.h>

net/ipv6/udp.c+1−0

@@ -17,6 +17,7 @@ · 16 unmodified lines
17	17	* YOSHIFUJI Hideaki @USAGI: convert /proc/net/udp6 to seq_file.
18	18	*/
19	19
	20	+#include <linux/bpf-cgroup.h>
20	21	#include <linux/errno.h>
21	22	#include <linux/types.h>
22	23	#include <linux/socket.h>

net/iucv/af_iucv.c+1−0

@@ -13,6 +13,7 @@ · 12 unmodified lines
13	13	#define KMSG_COMPONENT "af_iucv"
14	14	#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
15	15
	16	+#include <linux/filter.h>
16	17	#include <linux/module.h>
17	18	#include <linux/netdevice.h>
18	19	#include <linux/types.h>

net/kcm/kcmsock.c+1−0

@@ -9,6 +9,7 @@ · 8 unmodified lines
9	9	#include <linux/errno.h>
10	10	#include <linux/errqueue.h>
11	11	#include <linux/file.h>
	12	+#include <linux/filter.h>
12	13	#include <linux/in.h>
13	14	#include <linux/kernel.h>
14	15	#include <linux/module.h>

net/netfilter/nfnetlink_hook.c+1−0

@@ -6,6 +6,7 @@ · 5 unmodified lines
6	6	*/
7	7
8	8	#include <linux/module.h>
	9	+#include <linux/kallsyms.h>
9	10	#include <linux/kernel.h>
10	11	#include <linux/types.h>
11	12	#include <linux/skbuff.h>

net/netfilter/nft_reject_netdev.c+1−0

@@ -4,6 +4,7 @@ · 3 unmodified lines
4	4	* Copyright (c) 2020 Jose M. Guisado <guigom@riseup.net>
5	5	*/
6	6
	7	+#include <linux/etherdevice.h>
7	8	#include <linux/kernel.h>
8	9	#include <linux/init.h>
9	10	#include <linux/module.h>

net/netlink/af_netlink.c+2−0

@@ -20,8 +20,10 @@ · 19 unmodified lines
20	20
21	21	#include <linux/module.h>
22	22
	23	+#include <linux/bpf.h>
23	24	#include <linux/capability.h>
24	25	#include <linux/kernel.h>
	26	+#include <linux/filter.h>
25	27	#include <linux/init.h>
26	28	#include <linux/signal.h>
27	29	#include <linux/sched.h>

net/packet/af_packet.c+1−0

@@ -49,6 +49,7 @@ · 48 unmodified lines
49	49	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
50	50
51	51	#include <linux/ethtool.h>
	52	+#include <linux/filter.h>
52	53	#include <linux/types.h>
53	54	#include <linux/mm.h>
54	55	#include <linux/capability.h>

net/rose/rose_in.c+1−0

@@ -9,6 +9,7 @@ · 8 unmodified lines
9	9	* diagrams as the code is not obvious and probably very easy to break.
10	10	*/
11	11	#include <linux/errno.h>
	12	+#include <linux/filter.h>
12	13	#include <linux/types.h>
13	14	#include <linux/socket.h>
14	15	#include <linux/in.h>

net/sched/sch_frag.c+1−0

@@ -1,4 +1,5 @@
1	1	// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
	2	+#include <linux/if_vlan.h>
2	3	#include <net/netlink.h>
3	4	#include <net/sch_generic.h>
4	5	#include <net/pkt_sched.h>

net/smc/smc_ib.c+2−0

@@ -12,6 +12,8 @@ · 11 unmodified lines
12	12	* Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com>
13	13	*/
14	14
	15	+#include <linux/etherdevice.h>
	16	+#include <linux/if_vlan.h>
15	17	#include <linux/random.h>
16	18	#include <linux/workqueue.h>
17	19	#include <linux/scatterlist.h>

net/smc/smc_ism.c+1−0

@@ -6,6 +6,7 @@ · 5 unmodified lines
6	6	* Copyright IBM Corp. 2018
7	7	*/
8	8
	9	+#include <linux/if_vlan.h>
9	10	#include <linux/spinlock.h>
10	11	#include <linux/mutex.h>
11	12	#include <linux/slab.h>

net/socket.c+1−0

@@ -52,6 +52,7 @@ · 51 unmodified lines
52	52	* Based upon Swansea University Computer Society NET3.039
53	53	*/
54	54
	55	+#include <linux/bpf-cgroup.h>
55	56	#include <linux/ethtool.h>
56	57	#include <linux/mm.h>
57	58	#include <linux/socket.h>

net/unix/af_unix.c+1−0

@@ -89,6 +89,7 @@ · 88 unmodified lines
89	89	#include <linux/socket.h>
90	90	#include <linux/un.h>
91	91	#include <linux/fcntl.h>
	92	+#include <linux/filter.h>
92	93	#include <linux/termios.h>
93	94	#include <linux/sockios.h>
94	95	#include <linux/net.h>

net/vmw_vsock/af_vsock.c+1−0

@@ -85,6 +85,7 @@ · 84 unmodified lines
85	85	* TCP_LISTEN - listening
86	86	*/
87	87
	88	+#include <linux/compat.h>
88	89	#include <linux/types.h>
89	90	#include <linux/bitops.h>
90	91	#include <linux/cred.h>

net/xdp/xskmap.c+1−0

@@ -4,6 +4,7 @@ · 3 unmodified lines
4	4	*/
5	5
6	6	#include <linux/bpf.h>
	7	+#include <linux/filter.h>
7	8	#include <linux/capability.h>
8	9	#include <net/xdp_sock.h>
9	10	#include <linux/slab.h>

net/xfrm/xfrm_state.c+1−0

@@ -14,6 +14,7 @@ · 13 unmodified lines
14	14	*
15	15	*/
16	16
	17	+#include <linux/compat.h>
17	18	#include <linux/workqueue.h>
18	19	#include <net/xfrm.h>
19	20	#include <linux/pfkeyv2.h>

net/xfrm/xfrm_user.c+1−0

@@ -11,6 +11,7 @@ · 10 unmodified lines
11	11	*
12	12	*/
13	13
	14	+#include <linux/compat.h>
14	15	#include <linux/crypto.h>
15	16	#include <linux/module.h>
16	17	#include <linux/kernel.h>

samples/bpf/hbm.c+10−1

@@ -120,6 +120,9 @@ static void do_error(char *msg, bool errno_flag) · 119 unmodified lines
120	120
121	121	static int prog_load(char *prog)
122	122	{
	123	+ struct bpf_program *pos;
	124	+ const char *sec_name;
	125	+
123	126	obj = bpf_object__open_file(prog, NULL);
124	127	if (libbpf_get_error(obj)) {
125	128	printf("ERROR: opening BPF object file failed\n");
@@ -132,7 +135,13 @@ static int prog_load(char *prog) · 6 unmodified lines
132	135	goto err;
133	136	}
134	137
135		− bpf_prog = bpf_object__find_program_by_title(obj, "cgroup_skb/egress");
	138	+ bpf_object__for_each_program(pos, obj) {
	139	+ sec_name = bpf_program__section_name(pos);
	140	+ if (sec_name && !strcmp(sec_name, "cgroup_skb/egress")) {
	141	+ bpf_prog = pos;
	142	+ break;
	143	+ }
	144	+ }
136	145	if (!bpf_prog) {
137	146	printf("ERROR: finding a prog in obj file failed\n");
138	147	goto err;

samples/bpf/xdp_fwd_user.c+10−2

@@ -79,7 +79,9 @@ int main(int argc, char **argv) · 78 unmodified lines
79	79	.prog_type = BPF_PROG_TYPE_XDP,
80	80	};
81	81	const char *prog_name = "xdp_fwd";
82		− struct bpf_program *prog;
	82	+ struct bpf_program *prog = NULL;
	83	+ struct bpf_program *pos;
	84	+ const char *sec_name;
83	85	int prog_fd, map_fd = -1;
84	86	char filename[PATH_MAX];
85	87	struct bpf_object *obj;
@@ -134,7 +136,13 @@ int main(int argc, char **argv) · 48 unmodified lines
134	136	return 1;
135	137	}
136	138
137		− prog = bpf_object__find_program_by_title(obj, prog_name);
	139	+ bpf_object__for_each_program(pos, obj) {
	140	+ sec_name = bpf_program__section_name(pos);
	141	+ if (sec_name && !strcmp(sec_name, prog_name)) {
	142	+ prog = pos;
	143	+ break;
	144	+ }
	145	+ }
138	146	prog_fd = bpf_program__fd(prog);
139	147	if (prog_fd < 0) {
140	148	printf("program not found: %s\n", strerror(prog_fd));

security/device_cgroup.c+1−0

@@ -5,6 +5,7 @@ · 4 unmodified lines
5	5	* Copyright 2007 IBM Corp
6	6	*/
7	7
	8	+#include <linux/bpf-cgroup.h>
8	9	#include <linux/device_cgroup.h>
9	10	#include <linux/cgroup.h>
10	11	#include <linux/ctype.h>

tools/bpf/bpftool/Makefile+7−6

@@ -57,7 +57,7 @@ $(LIBBPF_INTERNAL_HDRS): $(LIBBPF_HDRS_DIR)/%.h: $(BPF_DIR)/%.h \| $(LIBBPF_HDRS_ · 56 unmodified lines
57	57	$(LIBBPF_BOOTSTRAP): $(wildcard $(BPF_DIR)/*.[ch] $(BPF_DIR)/Makefile) \| $(LIBBPF_BOOTSTRAP_OUTPUT)
58	58	$(Q)$(MAKE) -C $(BPF_DIR) OUTPUT=$(LIBBPF_BOOTSTRAP_OUTPUT) \
59	59	DESTDIR=$(LIBBPF_BOOTSTRAP_DESTDIR) prefix= \
60		− ARCH= CC=$(HOSTCC) LD=$(HOSTLD) $@ install_headers
	60	+ ARCH= CROSS_COMPILE= CC=$(HOSTCC) LD=$(HOSTLD) $@ install_headers
61	61
62	62	$(LIBBPF_BOOTSTRAP_INTERNAL_HDRS): $(LIBBPF_BOOTSTRAP_HDRS_DIR)/%.h: $(BPF_DIR)/%.h \| $(LIBBPF_BOOTSTRAP_HDRS_DIR)
63	63	$(call QUIET_INSTALL, $@)
@@ -152,6 +152,9 @@ CFLAGS += -DHAVE_LIBBFD_SUPPORT · 88 unmodified lines
152	152	SRCS += $(BFD_SRCS)
153	153	endif
154	154
	155	+HOST_CFLAGS = $(subst -I$(LIBBPF_INCLUDE),-I$(LIBBPF_BOOTSTRAP_INCLUDE),\
	156	+ $(subst $(CLANG_CROSS_FLAGS),,$(CFLAGS)))
	157	+
155	158	BPFTOOL_BOOTSTRAP := $(BOOTSTRAP_OUTPUT)bpftool
156	159
157	160	BOOTSTRAP_OBJS = $(addprefix $(BOOTSTRAP_OUTPUT),main.o common.o json_writer.o gen.o btf.o xlated_dumper.o btf_dumper.o disasm.o)
@@ -202,7 +205,7 @@ endif · 44 unmodified lines
202	205	CFLAGS += $(if $(BUILD_BPF_SKELS),,-DBPFTOOL_WITHOUT_SKELETONS)
203	206
204	207	$(BOOTSTRAP_OUTPUT)disasm.o: $(srctree)/kernel/bpf/disasm.c
205		− $(QUIET_CC)$(HOSTCC) $(CFLAGS) -c -MMD $< -o $@
	208	+ $(QUIET_CC)$(HOSTCC) $(HOST_CFLAGS) -c -MMD $< -o $@
206	209
207	210	$(OUTPUT)disasm.o: $(srctree)/kernel/bpf/disasm.c
208	211	$(QUIET_CC)$(CC) $(CFLAGS) -c -MMD $< -o $@
@@ -213,15 +216,13 @@ ifneq ($(feature-zlib), 1) · 4 unmodified lines
213	216	endif
214	217
215	218	$(BPFTOOL_BOOTSTRAP): $(BOOTSTRAP_OBJS) $(LIBBPF_BOOTSTRAP)
216		− $(QUIET_LINK)$(HOSTCC) $(CFLAGS) $(LDFLAGS) $(BOOTSTRAP_OBJS) $(LIBS_BOOTSTRAP) -o $@
	219	+ $(QUIET_LINK)$(HOSTCC) $(HOST_CFLAGS) $(LDFLAGS) $(BOOTSTRAP_OBJS) $(LIBS_BOOTSTRAP) -o $@
217	220
218	221	$(OUTPUT)bpftool: $(OBJS) $(LIBBPF)
219	222	$(QUIET_LINK)$(CC) $(CFLAGS) $(LDFLAGS) $(OBJS) $(LIBS) -o $@
220	223
221	224	$(BOOTSTRAP_OUTPUT)%.o: %.c $(LIBBPF_BOOTSTRAP_INTERNAL_HDRS) \| $(BOOTSTRAP_OUTPUT)
222		− $(QUIET_CC)$(HOSTCC) \
223		− $(subst -I$(LIBBPF_INCLUDE),-I$(LIBBPF_BOOTSTRAP_INCLUDE),$(CFLAGS)) \
224		− -c -MMD $< -o $@
	225	+ $(QUIET_CC)$(HOSTCC) $(HOST_CFLAGS) -c -MMD $< -o $@
225	226
226	227	$(OUTPUT)%.o: %.c
227	228	$(QUIET_CC)$(CC) $(CFLAGS) -c -MMD $< -o $@

tools/bpf/bpftool/feature.c+23−3

@@ -642,12 +642,32 @@ probe_helpers_for_progtype(enum bpf_prog_type prog_type, bool supported_type, · 641 unmodified lines
642	642	printf("\n");
643	643	}
644	644
645		−static void
646		−probe_large_insn_limit(const char *define_prefix, __u32 ifindex)
	645	+/*
	646	+ * Probe for availability of kernel commit (5.3):
	647	+ *
	648	+ * c04c0d2b968a ("bpf: increase complexity limit and maximum program size")
	649	+ */
	650	+static void probe_large_insn_limit(const char *define_prefix, __u32 ifindex)
647	651	{
	652	+ LIBBPF_OPTS(bpf_prog_load_opts, opts,
	653	+ .prog_ifindex = ifindex,
	654	+ );
	655	+ struct bpf_insn insns[BPF_MAXINSNS + 1];
648	656	bool res;
	657	+ int i, fd;
	658	+
	659	+ for (i = 0; i < BPF_MAXINSNS; i++)
	660	+ insns[i] = BPF_MOV64_IMM(BPF_REG_0, 1);
	661	+ insns[BPF_MAXINSNS] = BPF_EXIT_INSN();
	662	+
	663	+ errno = 0;
	664	+ fd = bpf_prog_load(BPF_PROG_TYPE_SCHED_CLS, NULL, "GPL",
	665	+ insns, ARRAY_SIZE(insns), &opts);
	666	+ res = fd >= 0 \|\| (errno != E2BIG && errno != EINVAL);
	667	+
	668	+ if (fd >= 0)
	669	+ close(fd);
649	670
650		− res = bpf_probe_large_insn_limit(ifindex);
651	671	print_bool_feature("have_large_insn_limit",
652	672	"Large program size limit",
653	673	"LARGE_INSN_LIMIT",

tools/bpf/bpftool/main.c+2−0

@@ -408,6 +408,8 @@ int main(int argc, char **argv) · 407 unmodified lines
408	408	bool version_requested = false;
409	409	int opt, ret;
410	410
	411	+ setlinebuf(stdout);
	412	+
411	413	last_do_help = do_help;
412	414	pretty_output = false;
413	415	json_output = false;

tools/bpf/resolve_btfids/Makefile+1−0

@@ -19,6 +19,7 @@ CC = $(HOSTCC) · 18 unmodified lines
19	19	LD = $(HOSTLD)
20	20	ARCH = $(HOSTARCH)
21	21	RM ?= rm
	22	+CROSS_COMPILE =
22	23
23	24	OUTPUT ?= $(srctree)/tools/bpf/resolve_btfids/
24	25

tools/bpf/runqslower/Makefile+2−2

@@ -12,7 +12,7 @@ BPFOBJ := $(BPFOBJ_OUTPUT)libbpf.a · 11 unmodified lines
12	12	BPF_DESTDIR := $(BPFOBJ_OUTPUT)
13	13	BPF_INCLUDE := $(BPF_DESTDIR)/include
14	14	INCLUDES := -I$(OUTPUT) -I$(BPF_INCLUDE) -I$(abspath ../../include/uapi)
15		−CFLAGS := -g -Wall
	15	+CFLAGS := -g -Wall $(CLANG_CROSS_FLAGS)
16	16
17	17	# Try to detect best kernel BTF source
18	18	KERNEL_REL := $(shell uname -r)
@@ -88,4 +88,4 @@ $(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) \| $(BPFOBJ_OU · 69 unmodified lines
88	88
89	89	$(DEFAULT_BPFTOOL): $(BPFOBJ) \| $(BPFTOOL_OUTPUT)
90	90	$(Q)$(MAKE) $(submake_extras) -C ../bpftool OUTPUT=$(BPFTOOL_OUTPUT) \
91		− CC=$(HOSTCC) LD=$(HOSTLD)
	91	+ ARCH= CROSS_COMPILE= CC=$(HOSTCC) LD=$(HOSTLD)

tools/include/uapi/linux/bpf.h+39−0

@@ -4983,6 +4983,41 @@ union bpf_attr { · 4982 unmodified lines
4983	4983	* Return
4984	4984	* The number of loops performed, -EINVAL for invalid flags,
4985	4985	* -E2BIG if nr_loops exceeds the maximum number of loops.
	4986	+ *
	4987	+ * long bpf_strncmp(const char s1, u32 s1_sz, const char s2)
	4988	+ * Description
	4989	+ * Do strncmp() between s1 and s2. s1 doesn't need
	4990	+ * to be null-terminated and s1_sz is the maximum storage
	4991	+ * size of s1. s2 must be a read-only string.
	4992	+ * Return
	4993	+ * An integer less than, equal to, or greater than zero
	4994	+ * if the first s1_sz bytes of s1 is found to be
	4995	+ * less than, to match, or be greater than s2.
	4996	+ *
	4997	+ * long bpf_get_func_arg(void ctx, u32 n, u64 value)
	4998	+ * Description
	4999	+ * Get n-th argument (zero based) of the traced function (for tracing programs)
	5000	+ * returned in value.
	5001	+ *
	5002	+ * Return
	5003	+ * 0 on success.
	5004	+ * -EINVAL if n >= arguments count of traced function.
	5005	+ *
	5006	+ * long bpf_get_func_ret(void ctx, u64 value)
	5007	+ * Description
	5008	+ * Get return value of the traced function (for tracing programs)
	5009	+ * in value.
	5010	+ *
	5011	+ * Return
	5012	+ * 0 on success.
	5013	+ * -EOPNOTSUPP for tracing programs other than BPF_TRACE_FEXIT or BPF_MODIFY_RETURN.
	5014	+ *
	5015	+ * long bpf_get_func_arg_cnt(void *ctx)
	5016	+ * Description
	5017	+ * Get number of arguments of the traced function (for tracing programs).
	5018	+ *
	5019	+ * Return
	5020	+ * The number of arguments of the traced function.
4986	5021	*/
4987	5022	#define __BPF_FUNC_MAPPER(FN) \
4988	5023	FN(unspec), \
@@ -5167,6 +5202,10 @@ union bpf_attr { · 178 unmodified lines
5167	5202	FN(kallsyms_lookup_name), \
5168	5203	FN(find_vma), \
5169	5204	FN(loop), \
	5205	+ FN(strncmp), \
	5206	+ FN(get_func_arg), \
	5207	+ FN(get_func_ret), \
	5208	+ FN(get_func_arg_cnt), \
5170	5209	/* */
5171	5210
5172	5211	/* integer value in 'imm' field of BPF_CALL instruction selects which helper

tools/lib/bpf/Makefile+2−1

@@ -90,6 +90,7 @@ override CFLAGS += -Werror -Wall · 89 unmodified lines
90	90	override CFLAGS += $(INCLUDES)
91	91	override CFLAGS += -fvisibility=hidden
92	92	override CFLAGS += -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64
	93	+override CFLAGS += $(CLANG_CROSS_FLAGS)
93	94
94	95	# flags specific for shared library
95	96	SHLIB_FLAGS := -DSHARED -fPIC
@@ -162,7 +163,7 @@ $(BPF_HELPER_DEFS): $(srctree)/tools/include/uapi/linux/bpf.h · 66 unmodified lines
162	163	$(OUTPUT)libbpf.so: $(OUTPUT)libbpf.so.$(LIBBPF_VERSION)
163	164
164	165	$(OUTPUT)libbpf.so.$(LIBBPF_VERSION): $(BPF_IN_SHARED) $(VERSION_SCRIPT)
165		− $(QUIET_LINK)$(CC) $(LDFLAGS) \
	166	+ $(QUIET_LINK)$(CC) $(CFLAGS) $(LDFLAGS) \
166	167	--shared -Wl,-soname,libbpf.so.$(LIBBPF_MAJOR_VERSION) \
167	168	-Wl,--version-script=$(VERSION_SCRIPT) $< -lelf -lz -o $@
168	169	@ln -sf $(@F) $(OUTPUT)libbpf.so

tools/lib/bpf/bpf.c+83−2

@@ -28,7 +28,9 @@ · 27 unmodified lines
28	28	#include <asm/unistd.h>
29	29	#include <errno.h>
30	30	#include <linux/bpf.h>
	31	+#include <linux/filter.h>
31	32	#include <limits.h>
	33	+#include <sys/resource.h>
32	34	#include "bpf.h"
33	35	#include "libbpf.h"
34	36	#include "libbpf_internal.h"
@@ -94,6 +96,77 @@ static inline int sys_bpf_prog_load(union bpf_attr *attr, unsigned int size, int · 59 unmodified lines
94	96	return fd;
95	97	}
96	98
	99	+/* Probe whether kernel switched from memlock-based (RLIMIT_MEMLOCK) to
	100	+ * memcg-based memory accounting for BPF maps and progs. This was done in [0].
	101	+ * We use the support for bpf_ktime_get_coarse_ns() helper, which was added in
	102	+ * the same 5.11 Linux release ([1]), to detect memcg-based accounting for BPF.
	103	+ *
	104	+ * [0] https://lore.kernel.org/bpf/20201201215900.3569844-1-guro@fb.com/
	105	+ * [1] d05512618056 ("bpf: Add bpf_ktime_get_coarse_ns helper")
	106	+ */
	107	+int probe_memcg_account(void)
	108	+{
	109	+ const size_t prog_load_attr_sz = offsetofend(union bpf_attr, attach_btf_obj_fd);
	110	+ struct bpf_insn insns[] = {
	111	+ BPF_EMIT_CALL(BPF_FUNC_ktime_get_coarse_ns),
	112	+ BPF_EXIT_INSN(),
	113	+ };
	114	+ size_t insn_cnt = sizeof(insns) / sizeof(insns[0]);
	115	+ union bpf_attr attr;
	116	+ int prog_fd;
	117	+
	118	+ /* attempt loading freplace trying to use custom BTF */
	119	+ memset(&attr, 0, prog_load_attr_sz);
	120	+ attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
	121	+ attr.insns = ptr_to_u64(insns);
	122	+ attr.insn_cnt = insn_cnt;
	123	+ attr.license = ptr_to_u64("GPL");
	124	+
	125	+ prog_fd = sys_bpf_fd(BPF_PROG_LOAD, &attr, prog_load_attr_sz);
	126	+ if (prog_fd >= 0) {
	127	+ close(prog_fd);
	128	+ return 1;
	129	+ }
	130	+ return 0;
	131	+}
	132	+
	133	+static bool memlock_bumped;
	134	+static rlim_t memlock_rlim = RLIM_INFINITY;
	135	+
	136	+int libbpf_set_memlock_rlim(size_t memlock_bytes)
	137	+{
	138	+ if (memlock_bumped)
	139	+ return libbpf_err(-EBUSY);
	140	+
	141	+ memlock_rlim = memlock_bytes;
	142	+ return 0;
	143	+}
	144	+
	145	+int bump_rlimit_memlock(void)
	146	+{
	147	+ struct rlimit rlim;
	148	+
	149	+ /* this the default in libbpf 1.0, but for now user has to opt-in explicitly */
	150	+ if (!(libbpf_mode & LIBBPF_STRICT_AUTO_RLIMIT_MEMLOCK))
	151	+ return 0;
	152	+
	153	+ /* if kernel supports memcg-based accounting, skip bumping RLIMIT_MEMLOCK */
	154	+ if (memlock_bumped \|\| kernel_supports(NULL, FEAT_MEMCG_ACCOUNT))
	155	+ return 0;
	156	+
	157	+ memlock_bumped = true;
	158	+
	159	+ /* zero memlock_rlim_max disables auto-bumping RLIMIT_MEMLOCK */
	160	+ if (memlock_rlim == 0)
	161	+ return 0;
	162	+
	163	+ rlim.rlim_cur = rlim.rlim_max = memlock_rlim;
	164	+ if (setrlimit(RLIMIT_MEMLOCK, &rlim))
	165	+ return -errno;
	166	+
	167	+ return 0;
	168	+}
	169	+
97	170	int bpf_map_create(enum bpf_map_type map_type,
98	171	const char *map_name,
99	172	__u32 key_size,
@@ -105,6 +178,8 @@ int bpf_map_create(enum bpf_map_type map_type, · 5 unmodified lines
105	178	union bpf_attr attr;
106	179	int fd;
107	180
	181	+ bump_rlimit_memlock();
	182	+
108	183	memset(&attr, 0, attr_sz);
109	184
110	185	if (!OPTS_VALID(opts, bpf_map_create_opts))
@@ -112,7 +187,7 @@ int bpf_map_create(enum bpf_map_type map_type, · 1 unmodified line
112	187
113	188	attr.map_type = map_type;
114	189	if (map_name)
115		− strncat(attr.map_name, map_name, sizeof(attr.map_name) - 1);
	190	+ libbpf_strlcpy(attr.map_name, map_name, sizeof(attr.map_name));
116	191	attr.key_size = key_size;
117	192	attr.value_size = value_size;
118	193	attr.max_entries = max_entries;
@@ -251,6 +326,8 @@ int bpf_prog_load_v0_6_0(enum bpf_prog_type prog_type, · 132 unmodified lines
251	326	union bpf_attr attr;
252	327	char *log_buf;
253	328
	329	+ bump_rlimit_memlock();
	330	+
254	331	if (!OPTS_VALID(opts, bpf_prog_load_opts))
255	332	return libbpf_err(-EINVAL);
256	333
@@ -271,7 +348,7 @@ int bpf_prog_load_v0_6_0(enum bpf_prog_type prog_type, · 14 unmodified lines
271	348	attr.kern_version = OPTS_GET(opts, kern_version, 0);
272	349
273	350	if (prog_name)
274		− strncat(attr.prog_name, prog_name, sizeof(attr.prog_name) - 1);
	351	+ libbpf_strlcpy(attr.prog_name, prog_name, sizeof(attr.prog_name));
275	352	attr.license = ptr_to_u64(license);
276	353
277	354	if (insn_cnt > UINT_MAX)
@@ -456,6 +533,8 @@ int bpf_verify_program(enum bpf_prog_type type, const struct bpf_insn *insns, · 178 unmodified lines
456	533	union bpf_attr attr;
457	534	int fd;
458	535
	536	+ bump_rlimit_memlock();
	537	+
459	538	memset(&attr, 0, sizeof(attr));
460	539	attr.prog_type = type;
461	540	attr.insn_cnt = (__u32)insns_cnt;
@@ -1056,6 +1135,8 @@ int bpf_btf_load(const void *btf_data, size_t btf_size, const struct bpf_btf_loa · 594 unmodified lines
1056	1135	__u32 log_level;
1057	1136	int fd;
1058	1137
	1138	+ bump_rlimit_memlock();
	1139	+
1059	1140	memset(&attr, 0, attr_sz);
1060	1141
1061	1142	if (!OPTS_VALID(opts, bpf_btf_load_opts))

tools/lib/bpf/bpf.h+2−0

@@ -35,6 +35,8 @@ · 34 unmodified lines
35	35	extern "C" {
36	36	#endif
37	37
	38	+int libbpf_set_memlock_rlim(size_t memlock_bytes);
	39	+
38	40	struct bpf_map_create_opts {
39	41	size_t sz; /* size of this struct for forward/backward compatibility */
40	42

tools/lib/bpf/bpf_tracing.h+174−257

@@ -66,277 +66,204 @@ · 65 unmodified lines
66	66
67	67	#if defined(__KERNEL__) \|\| defined(__VMLINUX_H__)
68	68
69		−#define PT_REGS_PARM1(x) ((x)->di)
70		−#define PT_REGS_PARM2(x) ((x)->si)
71		−#define PT_REGS_PARM3(x) ((x)->dx)
72		−#define PT_REGS_PARM4(x) ((x)->cx)
73		−#define PT_REGS_PARM5(x) ((x)->r8)
74		−#define PT_REGS_RET(x) ((x)->sp)
75		−#define PT_REGS_FP(x) ((x)->bp)
76		−#define PT_REGS_RC(x) ((x)->ax)
77		−#define PT_REGS_SP(x) ((x)->sp)
78		−#define PT_REGS_IP(x) ((x)->ip)
79		−
80		−#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ((x), di)
81		−#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ((x), si)
82		−#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ((x), dx)
83		−#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ((x), cx)
84		−#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ((x), r8)
85		−#define PT_REGS_RET_CORE(x) BPF_CORE_READ((x), sp)
86		−#define PT_REGS_FP_CORE(x) BPF_CORE_READ((x), bp)
87		−#define PT_REGS_RC_CORE(x) BPF_CORE_READ((x), ax)
88		−#define PT_REGS_SP_CORE(x) BPF_CORE_READ((x), sp)
89		−#define PT_REGS_IP_CORE(x) BPF_CORE_READ((x), ip)
	69	+#define __PT_PARM1_REG di
	70	+#define __PT_PARM2_REG si
	71	+#define __PT_PARM3_REG dx
	72	+#define __PT_PARM4_REG cx
	73	+#define __PT_PARM5_REG r8
	74	+#define __PT_RET_REG sp
	75	+#define __PT_FP_REG bp
	76	+#define __PT_RC_REG ax
	77	+#define __PT_SP_REG sp
	78	+#define __PT_IP_REG ip
90	79
91	80	#else
92	81
93	82	#ifdef __i386__
94		−/* i386 kernel is built with -mregparm=3 */
95		−#define PT_REGS_PARM1(x) ((x)->eax)
96		−#define PT_REGS_PARM2(x) ((x)->edx)
97		−#define PT_REGS_PARM3(x) ((x)->ecx)
98		−#define PT_REGS_PARM4(x) 0
99		−#define PT_REGS_PARM5(x) 0
100		−#define PT_REGS_RET(x) ((x)->esp)
101		−#define PT_REGS_FP(x) ((x)->ebp)
102		−#define PT_REGS_RC(x) ((x)->eax)
103		−#define PT_REGS_SP(x) ((x)->esp)
104		−#define PT_REGS_IP(x) ((x)->eip)
105		−
106		−#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ((x), eax)
107		−#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ((x), edx)
108		−#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ((x), ecx)
109		−#define PT_REGS_PARM4_CORE(x) 0
110		−#define PT_REGS_PARM5_CORE(x) 0
111		−#define PT_REGS_RET_CORE(x) BPF_CORE_READ((x), esp)
112		−#define PT_REGS_FP_CORE(x) BPF_CORE_READ((x), ebp)
113		−#define PT_REGS_RC_CORE(x) BPF_CORE_READ((x), eax)
114		−#define PT_REGS_SP_CORE(x) BPF_CORE_READ((x), esp)
115		−#define PT_REGS_IP_CORE(x) BPF_CORE_READ((x), eip)
116		−
117		−#else
118	83
119		−#define PT_REGS_PARM1(x) ((x)->rdi)
120		−#define PT_REGS_PARM2(x) ((x)->rsi)
121		−#define PT_REGS_PARM3(x) ((x)->rdx)
122		−#define PT_REGS_PARM4(x) ((x)->rcx)
123		−#define PT_REGS_PARM5(x) ((x)->r8)
124		−#define PT_REGS_RET(x) ((x)->rsp)
125		−#define PT_REGS_FP(x) ((x)->rbp)
126		−#define PT_REGS_RC(x) ((x)->rax)
127		−#define PT_REGS_SP(x) ((x)->rsp)
128		−#define PT_REGS_IP(x) ((x)->rip)
129		−
130		−#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ((x), rdi)
131		−#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ((x), rsi)
132		−#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ((x), rdx)
133		−#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ((x), rcx)
134		−#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ((x), r8)
135		−#define PT_REGS_RET_CORE(x) BPF_CORE_READ((x), rsp)
136		−#define PT_REGS_FP_CORE(x) BPF_CORE_READ((x), rbp)
137		−#define PT_REGS_RC_CORE(x) BPF_CORE_READ((x), rax)
138		−#define PT_REGS_SP_CORE(x) BPF_CORE_READ((x), rsp)
139		−#define PT_REGS_IP_CORE(x) BPF_CORE_READ((x), rip)
140		−
141		−#endif
142		−#endif
	84	+#define __PT_PARM1_REG eax
	85	+#define __PT_PARM2_REG edx
	86	+#define __PT_PARM3_REG ecx
	87	+/* i386 kernel is built with -mregparm=3 */
	88	+#define __PT_PARM4_REG __unsupported__
	89	+#define __PT_PARM5_REG __unsupported__
	90	+#define __PT_RET_REG esp
	91	+#define __PT_FP_REG ebp
	92	+#define __PT_RC_REG eax
	93	+#define __PT_SP_REG esp
	94	+#define __PT_IP_REG eip
	95	+
	96	+#else /* __i386__ */
	97	+
	98	+#define __PT_PARM1_REG rdi
	99	+#define __PT_PARM2_REG rsi
	100	+#define __PT_PARM3_REG rdx
	101	+#define __PT_PARM4_REG rcx
	102	+#define __PT_PARM5_REG r8
	103	+#define __PT_RET_REG rsp
	104	+#define __PT_FP_REG rbp
	105	+#define __PT_RC_REG rax
	106	+#define __PT_SP_REG rsp
	107	+#define __PT_IP_REG rip
	108	+
	109	+#endif /* __i386__ */
	110	+
	111	+#endif /* __KERNEL__ \|\| __VMLINUX_H__ */
143	112
144	113	#elif defined(bpf_target_s390)
145	114
146	115	/* s390 provides user_pt_regs instead of struct pt_regs to userspace */
147		−struct pt_regs;
148		−#define PT_REGS_S390 const volatile user_pt_regs
149		−#define PT_REGS_PARM1(x) (((PT_REGS_S390 *)(x))->gprs[2])
150		−#define PT_REGS_PARM2(x) (((PT_REGS_S390 *)(x))->gprs[3])
151		−#define PT_REGS_PARM3(x) (((PT_REGS_S390 *)(x))->gprs[4])
152		−#define PT_REGS_PARM4(x) (((PT_REGS_S390 *)(x))->gprs[5])
153		−#define PT_REGS_PARM5(x) (((PT_REGS_S390 *)(x))->gprs[6])
154		−#define PT_REGS_RET(x) (((PT_REGS_S390 *)(x))->gprs[14])
155		−/* Works only with CONFIG_FRAME_POINTER */
156		−#define PT_REGS_FP(x) (((PT_REGS_S390 *)(x))->gprs[11])
157		−#define PT_REGS_RC(x) (((PT_REGS_S390 *)(x))->gprs[2])
158		−#define PT_REGS_SP(x) (((PT_REGS_S390 *)(x))->gprs[15])
159		−#define PT_REGS_IP(x) (((PT_REGS_S390 *)(x))->psw.addr)
160		−
161		−#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ((PT_REGS_S390 *)(x), gprs[2])
162		−#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ((PT_REGS_S390 *)(x), gprs[3])
163		−#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ((PT_REGS_S390 *)(x), gprs[4])
164		−#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ((PT_REGS_S390 *)(x), gprs[5])
165		−#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ((PT_REGS_S390 *)(x), gprs[6])
166		−#define PT_REGS_RET_CORE(x) BPF_CORE_READ((PT_REGS_S390 *)(x), gprs[14])
167		−#define PT_REGS_FP_CORE(x) BPF_CORE_READ((PT_REGS_S390 *)(x), gprs[11])
168		−#define PT_REGS_RC_CORE(x) BPF_CORE_READ((PT_REGS_S390 *)(x), gprs[2])
169		−#define PT_REGS_SP_CORE(x) BPF_CORE_READ((PT_REGS_S390 *)(x), gprs[15])
170		−#define PT_REGS_IP_CORE(x) BPF_CORE_READ((PT_REGS_S390 *)(x), psw.addr)
	116	+#define __PT_REGS_CAST(x) ((const user_pt_regs *)(x))
	117	+#define __PT_PARM1_REG gprs[2]
	118	+#define __PT_PARM2_REG gprs[3]
	119	+#define __PT_PARM3_REG gprs[4]
	120	+#define __PT_PARM4_REG gprs[5]
	121	+#define __PT_PARM5_REG gprs[6]
	122	+#define __PT_RET_REG grps[14]
	123	+#define __PT_FP_REG gprs[11] /* Works only with CONFIG_FRAME_POINTER */
	124	+#define __PT_RC_REG gprs[2]
	125	+#define __PT_SP_REG gprs[15]
	126	+#define __PT_IP_REG psw.addr
171	127
172	128	#elif defined(bpf_target_arm)
173	129
174		−#define PT_REGS_PARM1(x) ((x)->uregs[0])
175		−#define PT_REGS_PARM2(x) ((x)->uregs[1])
176		−#define PT_REGS_PARM3(x) ((x)->uregs[2])
177		−#define PT_REGS_PARM4(x) ((x)->uregs[3])
178		−#define PT_REGS_PARM5(x) ((x)->uregs[4])
179		−#define PT_REGS_RET(x) ((x)->uregs[14])
180		−#define PT_REGS_FP(x) ((x)->uregs[11]) /* Works only with CONFIG_FRAME_POINTER */
181		−#define PT_REGS_RC(x) ((x)->uregs[0])
182		−#define PT_REGS_SP(x) ((x)->uregs[13])
183		−#define PT_REGS_IP(x) ((x)->uregs[12])
184		−
185		−#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ((x), uregs[0])
186		−#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ((x), uregs[1])
187		−#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ((x), uregs[2])
188		−#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ((x), uregs[3])
189		−#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ((x), uregs[4])
190		−#define PT_REGS_RET_CORE(x) BPF_CORE_READ((x), uregs[14])
191		−#define PT_REGS_FP_CORE(x) BPF_CORE_READ((x), uregs[11])
192		−#define PT_REGS_RC_CORE(x) BPF_CORE_READ((x), uregs[0])
193		−#define PT_REGS_SP_CORE(x) BPF_CORE_READ((x), uregs[13])
194		−#define PT_REGS_IP_CORE(x) BPF_CORE_READ((x), uregs[12])
	130	+#define __PT_PARM1_REG uregs[0]
	131	+#define __PT_PARM2_REG uregs[1]
	132	+#define __PT_PARM3_REG uregs[2]
	133	+#define __PT_PARM4_REG uregs[3]
	134	+#define __PT_PARM5_REG uregs[4]
	135	+#define __PT_RET_REG uregs[14]
	136	+#define __PT_FP_REG uregs[11] /* Works only with CONFIG_FRAME_POINTER */
	137	+#define __PT_RC_REG uregs[0]
	138	+#define __PT_SP_REG uregs[13]
	139	+#define __PT_IP_REG uregs[12]
195	140
196	141	#elif defined(bpf_target_arm64)
197	142
198	143	/* arm64 provides struct user_pt_regs instead of struct pt_regs to userspace */
199		−struct pt_regs;
200		−#define PT_REGS_ARM64 const volatile struct user_pt_regs
201		−#define PT_REGS_PARM1(x) (((PT_REGS_ARM64 *)(x))->regs[0])
202		−#define PT_REGS_PARM2(x) (((PT_REGS_ARM64 *)(x))->regs[1])
203		−#define PT_REGS_PARM3(x) (((PT_REGS_ARM64 *)(x))->regs[2])
204		−#define PT_REGS_PARM4(x) (((PT_REGS_ARM64 *)(x))->regs[3])
205		−#define PT_REGS_PARM5(x) (((PT_REGS_ARM64 *)(x))->regs[4])
206		−#define PT_REGS_RET(x) (((PT_REGS_ARM64 *)(x))->regs[30])
207		−/* Works only with CONFIG_FRAME_POINTER */
208		−#define PT_REGS_FP(x) (((PT_REGS_ARM64 *)(x))->regs[29])
209		−#define PT_REGS_RC(x) (((PT_REGS_ARM64 *)(x))->regs[0])
210		−#define PT_REGS_SP(x) (((PT_REGS_ARM64 *)(x))->sp)
211		−#define PT_REGS_IP(x) (((PT_REGS_ARM64 *)(x))->pc)
212		−
213		−#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ((PT_REGS_ARM64 *)(x), regs[0])
214		−#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ((PT_REGS_ARM64 *)(x), regs[1])
215		−#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ((PT_REGS_ARM64 *)(x), regs[2])
216		−#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ((PT_REGS_ARM64 *)(x), regs[3])
217		−#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ((PT_REGS_ARM64 *)(x), regs[4])
218		−#define PT_REGS_RET_CORE(x) BPF_CORE_READ((PT_REGS_ARM64 *)(x), regs[30])
219		−#define PT_REGS_FP_CORE(x) BPF_CORE_READ((PT_REGS_ARM64 *)(x), regs[29])
220		−#define PT_REGS_RC_CORE(x) BPF_CORE_READ((PT_REGS_ARM64 *)(x), regs[0])
221		−#define PT_REGS_SP_CORE(x) BPF_CORE_READ((PT_REGS_ARM64 *)(x), sp)
222		−#define PT_REGS_IP_CORE(x) BPF_CORE_READ((PT_REGS_ARM64 *)(x), pc)
	144	+#define __PT_REGS_CAST(x) ((const struct user_pt_regs *)(x))
	145	+#define __PT_PARM1_REG regs[0]
	146	+#define __PT_PARM2_REG regs[1]
	147	+#define __PT_PARM3_REG regs[2]
	148	+#define __PT_PARM4_REG regs[3]
	149	+#define __PT_PARM5_REG regs[4]
	150	+#define __PT_RET_REG regs[30]
	151	+#define __PT_FP_REG regs[29] /* Works only with CONFIG_FRAME_POINTER */
	152	+#define __PT_RC_REG regs[0]
	153	+#define __PT_SP_REG sp
	154	+#define __PT_IP_REG pc
223	155
224	156	#elif defined(bpf_target_mips)
225	157
226		−#define PT_REGS_PARM1(x) ((x)->regs[4])
227		−#define PT_REGS_PARM2(x) ((x)->regs[5])
228		−#define PT_REGS_PARM3(x) ((x)->regs[6])
229		−#define PT_REGS_PARM4(x) ((x)->regs[7])
230		−#define PT_REGS_PARM5(x) ((x)->regs[8])
231		−#define PT_REGS_RET(x) ((x)->regs[31])
232		−#define PT_REGS_FP(x) ((x)->regs[30]) /* Works only with CONFIG_FRAME_POINTER */
233		−#define PT_REGS_RC(x) ((x)->regs[2])
234		−#define PT_REGS_SP(x) ((x)->regs[29])
235		−#define PT_REGS_IP(x) ((x)->cp0_epc)
236		−
237		−#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ((x), regs[4])
238		−#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ((x), regs[5])
239		−#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ((x), regs[6])
240		−#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ((x), regs[7])
241		−#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ((x), regs[8])
242		−#define PT_REGS_RET_CORE(x) BPF_CORE_READ((x), regs[31])
243		−#define PT_REGS_FP_CORE(x) BPF_CORE_READ((x), regs[30])
244		−#define PT_REGS_RC_CORE(x) BPF_CORE_READ((x), regs[2])
245		−#define PT_REGS_SP_CORE(x) BPF_CORE_READ((x), regs[29])
246		−#define PT_REGS_IP_CORE(x) BPF_CORE_READ((x), cp0_epc)
	158	+#define __PT_PARM1_REG regs[4]
	159	+#define __PT_PARM2_REG regs[5]
	160	+#define __PT_PARM3_REG regs[6]
	161	+#define __PT_PARM4_REG regs[7]
	162	+#define __PT_PARM5_REG regs[8]
	163	+#define __PT_RET_REG regs[31]
	164	+#define __PT_FP_REG regs[30] /* Works only with CONFIG_FRAME_POINTER */
	165	+#define __PT_RC_REG regs[2]
	166	+#define __PT_SP_REG regs[29]
	167	+#define __PT_IP_REG cp0_epc
247	168
248	169	#elif defined(bpf_target_powerpc)
249	170
250		−#define PT_REGS_PARM1(x) ((x)->gpr[3])
251		−#define PT_REGS_PARM2(x) ((x)->gpr[4])
252		−#define PT_REGS_PARM3(x) ((x)->gpr[5])
253		−#define PT_REGS_PARM4(x) ((x)->gpr[6])
254		−#define PT_REGS_PARM5(x) ((x)->gpr[7])
255		−#define PT_REGS_RC(x) ((x)->gpr[3])
256		−#define PT_REGS_SP(x) ((x)->sp)
257		−#define PT_REGS_IP(x) ((x)->nip)
258		−
259		−#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ((x), gpr[3])
260		−#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ((x), gpr[4])
261		−#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ((x), gpr[5])
262		−#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ((x), gpr[6])
263		−#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ((x), gpr[7])
264		−#define PT_REGS_RC_CORE(x) BPF_CORE_READ((x), gpr[3])
265		−#define PT_REGS_SP_CORE(x) BPF_CORE_READ((x), sp)
266		−#define PT_REGS_IP_CORE(x) BPF_CORE_READ((x), nip)
	171	+#define __PT_PARM1_REG gpr[3]
	172	+#define __PT_PARM2_REG gpr[4]
	173	+#define __PT_PARM3_REG gpr[5]
	174	+#define __PT_PARM4_REG gpr[6]
	175	+#define __PT_PARM5_REG gpr[7]
	176	+#define __PT_RET_REG regs[31]
	177	+#define __PT_FP_REG __unsupported__
	178	+#define __PT_RC_REG gpr[3]
	179	+#define __PT_SP_REG sp
	180	+#define __PT_IP_REG nip
267	181
268	182	#elif defined(bpf_target_sparc)
269	183
270		−#define PT_REGS_PARM1(x) ((x)->u_regs[UREG_I0])
271		−#define PT_REGS_PARM2(x) ((x)->u_regs[UREG_I1])
272		−#define PT_REGS_PARM3(x) ((x)->u_regs[UREG_I2])
273		−#define PT_REGS_PARM4(x) ((x)->u_regs[UREG_I3])
274		−#define PT_REGS_PARM5(x) ((x)->u_regs[UREG_I4])
275		−#define PT_REGS_RET(x) ((x)->u_regs[UREG_I7])
276		−#define PT_REGS_RC(x) ((x)->u_regs[UREG_I0])
277		−#define PT_REGS_SP(x) ((x)->u_regs[UREG_FP])
278		−
279		−#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ((x), u_regs[UREG_I0])
280		−#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ((x), u_regs[UREG_I1])
281		−#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ((x), u_regs[UREG_I2])
282		−#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ((x), u_regs[UREG_I3])
283		−#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ((x), u_regs[UREG_I4])
284		−#define PT_REGS_RET_CORE(x) BPF_CORE_READ((x), u_regs[UREG_I7])
285		−#define PT_REGS_RC_CORE(x) BPF_CORE_READ((x), u_regs[UREG_I0])
286		−#define PT_REGS_SP_CORE(x) BPF_CORE_READ((x), u_regs[UREG_FP])
287		−
	184	+#define __PT_PARM1_REG u_regs[UREG_I0]
	185	+#define __PT_PARM2_REG u_regs[UREG_I1]
	186	+#define __PT_PARM3_REG u_regs[UREG_I2]
	187	+#define __PT_PARM4_REG u_regs[UREG_I3]
	188	+#define __PT_PARM5_REG u_regs[UREG_I4]
	189	+#define __PT_RET_REG u_regs[UREG_I7]
	190	+#define __PT_FP_REG __unsupported__
	191	+#define __PT_RC_REG u_regs[UREG_I0]
	192	+#define __PT_SP_REG u_regs[UREG_FP]
288	193	/* Should this also be a bpf_target check for the sparc case? */
289	194	#if defined(__arch64__)
290		−#define PT_REGS_IP(x) ((x)->tpc)
291		−#define PT_REGS_IP_CORE(x) BPF_CORE_READ((x), tpc)
	195	+#define __PT_IP_REG tpc
292	196	#else
293		−#define PT_REGS_IP(x) ((x)->pc)
294		−#define PT_REGS_IP_CORE(x) BPF_CORE_READ((x), pc)
	197	+#define __PT_IP_REG pc
295	198	#endif
296	199
297	200	#elif defined(bpf_target_riscv)
298	201
	202	+#define __PT_REGS_CAST(x) ((const struct user_regs_struct *)(x))
	203	+#define __PT_PARM1_REG a0
	204	+#define __PT_PARM2_REG a1
	205	+#define __PT_PARM3_REG a2
	206	+#define __PT_PARM4_REG a3
	207	+#define __PT_PARM5_REG a4
	208	+#define __PT_RET_REG ra
	209	+#define __PT_FP_REG fp
	210	+#define __PT_RC_REG a5
	211	+#define __PT_SP_REG sp
	212	+#define __PT_IP_REG epc
	213	+
	214	+#endif
	215	+
	216	+#if defined(bpf_target_defined)
	217	+
299	218	struct pt_regs;
300		−#define PT_REGS_RV const volatile struct user_regs_struct
301		−#define PT_REGS_PARM1(x) (((PT_REGS_RV *)(x))->a0)
302		−#define PT_REGS_PARM2(x) (((PT_REGS_RV *)(x))->a1)
303		−#define PT_REGS_PARM3(x) (((PT_REGS_RV *)(x))->a2)
304		−#define PT_REGS_PARM4(x) (((PT_REGS_RV *)(x))->a3)
305		−#define PT_REGS_PARM5(x) (((PT_REGS_RV *)(x))->a4)
306		−#define PT_REGS_RET(x) (((PT_REGS_RV *)(x))->ra)
307		−#define PT_REGS_FP(x) (((PT_REGS_RV *)(x))->s5)
308		−#define PT_REGS_RC(x) (((PT_REGS_RV *)(x))->a5)
309		−#define PT_REGS_SP(x) (((PT_REGS_RV *)(x))->sp)
310		−#define PT_REGS_IP(x) (((PT_REGS_RV *)(x))->epc)
311		−
312		−#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), a0)
313		−#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), a1)
314		−#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), a2)
315		−#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), a3)
316		−#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), a4)
317		−#define PT_REGS_RET_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), ra)
318		−#define PT_REGS_FP_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), fp)
319		−#define PT_REGS_RC_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), a5)
320		−#define PT_REGS_SP_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), sp)
321		−#define PT_REGS_IP_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), epc)
322	219
	220	+/* allow some architecutres to override `struct pt_regs` */
	221	+#ifndef __PT_REGS_CAST
	222	+#define __PT_REGS_CAST(x) (x)
323	223	#endif
324	224
	225	+#define PT_REGS_PARM1(x) (__PT_REGS_CAST(x)->__PT_PARM1_REG)
	226	+#define PT_REGS_PARM2(x) (__PT_REGS_CAST(x)->__PT_PARM2_REG)
	227	+#define PT_REGS_PARM3(x) (__PT_REGS_CAST(x)->__PT_PARM3_REG)
	228	+#define PT_REGS_PARM4(x) (__PT_REGS_CAST(x)->__PT_PARM4_REG)
	229	+#define PT_REGS_PARM5(x) (__PT_REGS_CAST(x)->__PT_PARM5_REG)
	230	+#define PT_REGS_RET(x) (__PT_REGS_CAST(x)->__PT_RET_REG)
	231	+#define PT_REGS_FP(x) (__PT_REGS_CAST(x)->__PT_FP_REG)
	232	+#define PT_REGS_RC(x) (__PT_REGS_CAST(x)->__PT_RC_REG)
	233	+#define PT_REGS_SP(x) (__PT_REGS_CAST(x)->__PT_SP_REG)
	234	+#define PT_REGS_IP(x) (__PT_REGS_CAST(x)->__PT_IP_REG)
	235	+
	236	+#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM1_REG)
	237	+#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM2_REG)
	238	+#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM3_REG)
	239	+#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM4_REG)
	240	+#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM5_REG)
	241	+#define PT_REGS_RET_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_RET_REG)
	242	+#define PT_REGS_FP_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_FP_REG)
	243	+#define PT_REGS_RC_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_RC_REG)
	244	+#define PT_REGS_SP_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_SP_REG)
	245	+#define PT_REGS_IP_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_IP_REG)
	246	+
325	247	#if defined(bpf_target_powerpc)
	248	+
326	249	#define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ (ip) = (ctx)->link; })
327	250	#define BPF_KRETPROBE_READ_RET_IP BPF_KPROBE_READ_RET_IP
	251	+
328	252	#elif defined(bpf_target_sparc)
	253	+
329	254	#define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ (ip) = PT_REGS_RET(ctx); })
330	255	#define BPF_KRETPROBE_READ_RET_IP BPF_KPROBE_READ_RET_IP
331		−#elif defined(bpf_target_defined)
	256	+
	257	+#else
	258	+
332	259	#define BPF_KPROBE_READ_RET_IP(ip, ctx) \
333	260	({ bpf_probe_read_kernel(&(ip), sizeof(ip), (void *)PT_REGS_RET(ctx)); })
334	261	#define BPF_KRETPROBE_READ_RET_IP(ip, ctx) \
335		− ({ bpf_probe_read_kernel(&(ip), sizeof(ip), \
336		− (void *)(PT_REGS_FP(ctx) + sizeof(ip))); })
	262	+ ({ bpf_probe_read_kernel(&(ip), sizeof(ip), (void *)(PT_REGS_FP(ctx) + sizeof(ip))); })
	263	+
337	264	#endif
338	265
339		−#if !defined(bpf_target_defined)
	266	+#else /* defined(bpf_target_defined) */
340	267
341	268	#define PT_REGS_PARM1(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
342	269	#define PT_REGS_PARM2(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
@@ -363,7 +290,7 @@ struct pt_regs; · 20 unmodified lines
363	290	#define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
364	291	#define BPF_KRETPROBE_READ_RET_IP(ip, ctx) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
365	292
366		−#endif /* !defined(bpf_target_defined) */
	293	+#endif /* defined(bpf_target_defined) */
367	294
368	295	#ifndef ___bpf_concat
369	296	#define ___bpf_concat(a, b) a ## b
@@ -375,25 +302,23 @@ struct pt_regs; · 5 unmodified lines
375	302	#define ___bpf_nth(_, _1, _2, _3, _4, _5, _6, _7, _8, _9, _a, _b, _c, N, ...) N
376	303	#endif
377	304	#ifndef ___bpf_narg
378		−#define ___bpf_narg(...) \
379		− ___bpf_nth(_, ##__VA_ARGS__, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)
	305	+#define ___bpf_narg(...) ___bpf_nth(_, ##__VA_ARGS__, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)
380	306	#endif
381	307
382		−#define ___bpf_ctx_cast0() ctx
383		−#define ___bpf_ctx_cast1(x) ___bpf_ctx_cast0(), (void *)ctx[0]
384		−#define ___bpf_ctx_cast2(x, args...) ___bpf_ctx_cast1(args), (void *)ctx[1]
385		−#define ___bpf_ctx_cast3(x, args...) ___bpf_ctx_cast2(args), (void *)ctx[2]
386		−#define ___bpf_ctx_cast4(x, args...) ___bpf_ctx_cast3(args), (void *)ctx[3]
387		−#define ___bpf_ctx_cast5(x, args...) ___bpf_ctx_cast4(args), (void *)ctx[4]
388		−#define ___bpf_ctx_cast6(x, args...) ___bpf_ctx_cast5(args), (void *)ctx[5]
389		−#define ___bpf_ctx_cast7(x, args...) ___bpf_ctx_cast6(args), (void *)ctx[6]
390		−#define ___bpf_ctx_cast8(x, args...) ___bpf_ctx_cast7(args), (void *)ctx[7]
391		−#define ___bpf_ctx_cast9(x, args...) ___bpf_ctx_cast8(args), (void *)ctx[8]
	308	+#define ___bpf_ctx_cast0() ctx
	309	+#define ___bpf_ctx_cast1(x) ___bpf_ctx_cast0(), (void *)ctx[0]
	310	+#define ___bpf_ctx_cast2(x, args...) ___bpf_ctx_cast1(args), (void *)ctx[1]
	311	+#define ___bpf_ctx_cast3(x, args...) ___bpf_ctx_cast2(args), (void *)ctx[2]
	312	+#define ___bpf_ctx_cast4(x, args...) ___bpf_ctx_cast3(args), (void *)ctx[3]
	313	+#define ___bpf_ctx_cast5(x, args...) ___bpf_ctx_cast4(args), (void *)ctx[4]
	314	+#define ___bpf_ctx_cast6(x, args...) ___bpf_ctx_cast5(args), (void *)ctx[5]
	315	+#define ___bpf_ctx_cast7(x, args...) ___bpf_ctx_cast6(args), (void *)ctx[6]
	316	+#define ___bpf_ctx_cast8(x, args...) ___bpf_ctx_cast7(args), (void *)ctx[7]
	317	+#define ___bpf_ctx_cast9(x, args...) ___bpf_ctx_cast8(args), (void *)ctx[8]
392	318	#define ___bpf_ctx_cast10(x, args...) ___bpf_ctx_cast9(args), (void *)ctx[9]
393	319	#define ___bpf_ctx_cast11(x, args...) ___bpf_ctx_cast10(args), (void *)ctx[10]
394	320	#define ___bpf_ctx_cast12(x, args...) ___bpf_ctx_cast11(args), (void *)ctx[11]
395		−#define ___bpf_ctx_cast(args...) \
396		− ___bpf_apply(___bpf_ctx_cast, ___bpf_narg(args))(args)
	321	+#define ___bpf_ctx_cast(args...) ___bpf_apply(___bpf_ctx_cast, ___bpf_narg(args))(args)
397	322
398	323	/*
399	324	* BPF_PROG is a convenience wrapper for generic tp_btf/fentry/fexit and
@@ -426,19 +351,13 @@ ____##name(unsigned long long *ctx, ##args) · 26 unmodified lines
426	351
427	352	struct pt_regs;
428	353
429		−#define ___bpf_kprobe_args0() ctx
430		−#define ___bpf_kprobe_args1(x) \
431		− ___bpf_kprobe_args0(), (void *)PT_REGS_PARM1(ctx)
432		−#define ___bpf_kprobe_args2(x, args...) \
433		− ___bpf_kprobe_args1(args), (void *)PT_REGS_PARM2(ctx)
434		−#define ___bpf_kprobe_args3(x, args...) \
435		− ___bpf_kprobe_args2(args), (void *)PT_REGS_PARM3(ctx)
436		−#define ___bpf_kprobe_args4(x, args...) \
437		− ___bpf_kprobe_args3(args), (void *)PT_REGS_PARM4(ctx)
438		−#define ___bpf_kprobe_args5(x, args...) \
439		− ___bpf_kprobe_args4(args), (void *)PT_REGS_PARM5(ctx)
440		−#define ___bpf_kprobe_args(args...) \
441		− ___bpf_apply(___bpf_kprobe_args, ___bpf_narg(args))(args)
	354	+#define ___bpf_kprobe_args0() ctx
	355	+#define ___bpf_kprobe_args1(x) ___bpf_kprobe_args0(), (void *)PT_REGS_PARM1(ctx)
	356	+#define ___bpf_kprobe_args2(x, args...) ___bpf_kprobe_args1(args), (void *)PT_REGS_PARM2(ctx)
	357	+#define ___bpf_kprobe_args3(x, args...) ___bpf_kprobe_args2(args), (void *)PT_REGS_PARM3(ctx)
	358	+#define ___bpf_kprobe_args4(x, args...) ___bpf_kprobe_args3(args), (void *)PT_REGS_PARM4(ctx)
	359	+#define ___bpf_kprobe_args5(x, args...) ___bpf_kprobe_args4(args), (void *)PT_REGS_PARM5(ctx)
	360	+#define ___bpf_kprobe_args(args...) ___bpf_apply(___bpf_kprobe_args, ___bpf_narg(args))(args)
442	361
443	362	/*
444	363	* BPF_KPROBE serves the same purpose for kprobes as BPF_PROG for
@@ -464,11 +383,9 @@ typeof(name(0)) name(struct pt_regs *ctx) \ · 19 unmodified lines
464	383	static __attribute__((always_inline)) typeof(name(0)) \
465	384	____##name(struct pt_regs *ctx, ##args)
466	385
467		−#define ___bpf_kretprobe_args0() ctx
468		−#define ___bpf_kretprobe_args1(x) \
469		− ___bpf_kretprobe_args0(), (void *)PT_REGS_RC(ctx)
470		−#define ___bpf_kretprobe_args(args...) \
471		− ___bpf_apply(___bpf_kretprobe_args, ___bpf_narg(args))(args)
	386	+#define ___bpf_kretprobe_args0() ctx
	387	+#define ___bpf_kretprobe_args1(x) ___bpf_kretprobe_args0(), (void *)PT_REGS_RC(ctx)
	388	+#define ___bpf_kretprobe_args(args...) ___bpf_apply(___bpf_kretprobe_args, ___bpf_narg(args))(args)
472	389
473	390	/*
474	391	* BPF_KRETPROBE is similar to BPF_KPROBE, except, it only provides optional

tools/lib/bpf/btf.h+6−0

@@ -313,12 +313,18 @@ LIBBPF_API struct btf_dump btf_dump__new_deprecated(const struct btf btf, · 312 unmodified lines
313	313	*
314	314	* The rest works just like in case of ___libbpf_override() usage with symbol
315	315	* versioning.
	316	+ *
	317	+ * C++ compilers don't support __builtin_types_compatible_p(), so at least
	318	+ * don't screw up compilation for them and let C++ users pick btf_dump__new
	319	+ * vs btf_dump__new_deprecated explicitly.
316	320	*/
	321	+#ifndef __cplusplus
317	322	#define btf_dump__new(a1, a2, a3, a4) __builtin_choose_expr( \
318	323	__builtin_types_compatible_p(typeof(a4), btf_dump_printf_fn_t) \|\| \
319	324	__builtin_types_compatible_p(typeof(a4), void(void , const char , va_list)), \
320	325	btf_dump__new_deprecated((void )a1, (void )a2, (void )a3, (void )a4), \
321	326	btf_dump__new((void )a1, (void )a2, (void )a3, (void )a4))
	327	+#endif
322	328
323	329	LIBBPF_API void btf_dump__free(struct btf_dump *d);
324	330

tools/lib/bpf/btf_dump.c+2−2

@@ -2321,8 +2321,8 @@ int btf_dump__dump_type_data(struct btf_dump *d, __u32 id, · 2320 unmodified lines
2321	2321	if (!opts->indent_str)
2322	2322	d->typed_dump->indent_str[0] = '\t';
2323	2323	else
2324		− strncat(d->typed_dump->indent_str, opts->indent_str,
2325		− sizeof(d->typed_dump->indent_str) - 1);
	2324	+ libbpf_strlcpy(d->typed_dump->indent_str, opts->indent_str,
	2325	+ sizeof(d->typed_dump->indent_str));
2326	2326
2327	2327	d->typed_dump->compact = OPTS_GET(opts, compact, false);
2328	2328	d->typed_dump->skip_names = OPTS_GET(opts, skip_names, false);

tools/lib/bpf/gen_loader.c+5−6

@@ -371,8 +371,9 @@ int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps) · 370 unmodified lines
371	371	{
372	372	int i;
373	373
374		− if (nr_progs != gen->nr_progs \|\| nr_maps != gen->nr_maps) {
375		− pr_warn("progs/maps mismatch\n");
	374	+ if (nr_progs < gen->nr_progs \|\| nr_maps != gen->nr_maps) {
	375	+ pr_warn("nr_progs %d/%d nr_maps %d/%d mismatch\n",
	376	+ nr_progs, gen->nr_progs, nr_maps, gen->nr_maps);
376	377	gen->error = -EFAULT;
377	378	return gen->error;
378	379	}
@@ -462,8 +463,7 @@ void bpf_gen__map_create(struct bpf_gen *gen, · 83 unmodified lines
462	463	attr.map_flags = map_attr->map_flags;
463	464	attr.map_extra = map_attr->map_extra;
464	465	if (map_name)
465		− memcpy(attr.map_name, map_name,
466		− min((unsigned)strlen(map_name), BPF_OBJ_NAME_LEN - 1));
	466	+ libbpf_strlcpy(attr.map_name, map_name, sizeof(attr.map_name));
467	467	attr.numa_node = map_attr->numa_node;
468	468	attr.map_ifindex = map_attr->map_ifindex;
469	469	attr.max_entries = max_entries;
@@ -969,8 +969,7 @@ void bpf_gen__prog_load(struct bpf_gen *gen, · 499 unmodified lines
969	969	core_relos = add_data(gen, gen->core_relos,
970	970	attr.core_relo_cnt * attr.core_relo_rec_size);
971	971
972		− memcpy(attr.prog_name, prog_name,
973		− min((unsigned)strlen(prog_name), BPF_OBJ_NAME_LEN - 1));
	972	+ libbpf_strlcpy(attr.prog_name, prog_name, sizeof(attr.prog_name));
974	973	prog_load_attr = add_data(gen, &attr, attr_size);
975	974
976	975	/* populate union bpf_attr with a pointer to license */

tools/lib/bpf/libbpf.c+41−44

@@ -187,42 +187,6 @@ const char *libbpf_version_string(void) · 186 unmodified lines
187	187	#undef __S
188	188	}
189	189
190		−enum kern_feature_id {
191		− /* v4.14: kernel support for program & map names. */
192		− FEAT_PROG_NAME,
193		− /* v5.2: kernel support for global data sections. */
194		− FEAT_GLOBAL_DATA,
195		− /* BTF support */
196		− FEAT_BTF,
197		− /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */
198		− FEAT_BTF_FUNC,
199		− /* BTF_KIND_VAR and BTF_KIND_DATASEC support */
200		− FEAT_BTF_DATASEC,
201		− /* BTF_FUNC_GLOBAL is supported */
202		− FEAT_BTF_GLOBAL_FUNC,
203		− /* BPF_F_MMAPABLE is supported for arrays */
204		− FEAT_ARRAY_MMAP,
205		− /* kernel support for expected_attach_type in BPF_PROG_LOAD */
206		− FEAT_EXP_ATTACH_TYPE,
207		− /* bpf_probe_read_{kernel,user}[_str] helpers */
208		− FEAT_PROBE_READ_KERN,
209		− /* BPF_PROG_BIND_MAP is supported */
210		− FEAT_PROG_BIND_MAP,
211		− /* Kernel support for module BTFs */
212		− FEAT_MODULE_BTF,
213		− /* BTF_KIND_FLOAT support */
214		− FEAT_BTF_FLOAT,
215		− /* BPF perf link support */
216		− FEAT_PERF_LINK,
217		− /* BTF_KIND_DECL_TAG support */
218		− FEAT_BTF_DECL_TAG,
219		− /* BTF_KIND_TYPE_TAG support */
220		− FEAT_BTF_TYPE_TAG,
221		− __FEAT_CNT,
222		−};
223		−
224		−static bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id);
225		−
226	190	enum reloc_type {
227	191	RELO_LD64,
228	192	RELO_CALL,
@@ -831,11 +795,36 @@ bpf_object__add_programs(struct bpf_object obj, Elf_Data sec_data, · 602 unmodified lines
831	795	return 0;
832	796	}
833	797
834		−static __u32 get_kernel_version(void)
	798	+__u32 get_kernel_version(void)
835	799	{
	800	+ /* On Ubuntu LINUX_VERSION_CODE doesn't correspond to info.release,
	801	+ * but Ubuntu provides /proc/version_signature file, as described at
	802	+ * https://ubuntu.com/kernel, with an example contents below, which we
	803	+ * can use to get a proper LINUX_VERSION_CODE.
	804	+ *
	805	+ * Ubuntu 5.4.0-12.15-generic 5.4.8
	806	+ *
	807	+ * In the above, 5.4.8 is what kernel is actually expecting, while
	808	+ * uname() call will return 5.4.0 in info.release.
	809	+ */
	810	+ const char *ubuntu_kver_file = "/proc/version_signature";
836	811	__u32 major, minor, patch;
837	812	struct utsname info;
838	813
	814	+ if (access(ubuntu_kver_file, R_OK) == 0) {
	815	+ FILE *f;
	816	+
	817	+ f = fopen(ubuntu_kver_file, "r");
	818	+ if (f) {
	819	+ if (fscanf(f, "%s %s %d.%d.%d\n", &major, &minor, &patch) == 3) {
	820	+ fclose(f);
	821	+ return KERNEL_VERSION(major, minor, patch);
	822	+ }
	823	+ fclose(f);
	824	+ }
	825	+ /* something went wrong, fall back to uname() approach */
	826	+ }
	827	+
839	828	uname(&info);
840	829	if (sscanf(info.release, "%u.%u.%u", &major, &minor, &patch) != 3)
841	830	return 0;
@@ -1201,12 +1190,10 @@ static struct bpf_object bpf_object__new(const char path, · 359 unmodified lines
1201	1190
1202	1191	strcpy(obj->path, path);
1203	1192	if (obj_name) {
1204		− strncpy(obj->name, obj_name, sizeof(obj->name) - 1);
1205		− obj->name[sizeof(obj->name) - 1] = 0;
	1193	+ libbpf_strlcpy(obj->name, obj_name, sizeof(obj->name));
1206	1194	} else {
1207	1195	/* Using basename() GNU version which doesn't modify arg. */
1208		− strncpy(obj->name, basename((void *)path),
1209		− sizeof(obj->name) - 1);
	1196	+ libbpf_strlcpy(obj->name, basename((void *)path), sizeof(obj->name));
1210	1197	end = strchr(obj->name, '.');
1211	1198	if (end)
1212	1199	*end = 0;
@@ -1358,7 +1345,10 @@ static int bpf_object__check_endianness(struct bpf_object *obj) · 145 unmodified lines
1358	1345	static int
1359	1346	bpf_object__init_license(struct bpf_object obj, void data, size_t size)
1360	1347	{
1361		− memcpy(obj->license, data, min(size, sizeof(obj->license) - 1));
	1348	+ /* libbpf_strlcpy() only copies first N - 1 bytes, so size + 1 won't
	1349	+ * go over allowed ELF data section buffer
	1350	+ */
	1351	+ libbpf_strlcpy(obj->license, data, min(size + 1, sizeof(obj->license)));
1362	1352	pr_debug("license of %s is %s\n", obj->path, obj->license);
1363	1353	return 0;
1364	1354	}
@@ -4354,6 +4344,10 @@ bpf_object__probe_loading(struct bpf_object *obj) · 2989 unmodified lines
4354	4344	if (obj->gen_loader)
4355	4345	return 0;
4356	4346
	4347	+ ret = bump_rlimit_memlock();
	4348	+ if (ret)
	4349	+ pr_warn("Failed to bump RLIMIT_MEMLOCK (err = %d), you might need to do it explicitly!\n", ret);
	4350	+
4357	4351	/* make sure basic loading works */
4358	4352	ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL);
4359	4353	if (ret < 0)
@@ -4720,14 +4714,17 @@ static struct kern_feature_desc { · 360 unmodified lines
4720	4714	[FEAT_BTF_TYPE_TAG] = {
4721	4715	"BTF_KIND_TYPE_TAG support", probe_kern_btf_type_tag,
4722	4716	},
	4717	+ [FEAT_MEMCG_ACCOUNT] = {
	4718	+ "memcg-based memory accounting", probe_memcg_account,
	4719	+ },
4723	4720	};
4724	4721
4725		−static bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id)
	4722	+bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id)
4726	4723	{
4727	4724	struct kern_feature_desc *feat = &feature_probes[feat_id];
4728	4725	int ret;
4729	4726
4730		− if (obj->gen_loader)
	4727	+ if (obj && obj->gen_loader)
4731	4728	/* To generate loader program assume the latest kernel
4732	4729	* to avoid doing extra prog_load, map_create syscalls.
4733	4730	*/

tools/lib/bpf/libbpf.h+73−4

@@ -227,6 +227,7 @@ struct btf; · 226 unmodified lines
227	227	LIBBPF_API struct btf bpf_object__btf(const struct bpf_object obj);
228	228	LIBBPF_API int bpf_object__btf_fd(const struct bpf_object *obj);
229	229
	230	+LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_object__find_program_by_name() instead")
230	231	LIBBPF_API struct bpf_program *
231	232	bpf_object__find_program_by_title(const struct bpf_object *obj,
232	233	const char *title);
@@ -339,7 +340,31 @@ LIBBPF_DEPRECATED_SINCE(0, 7, "multi-instance bpf_program support is deprecated" · 106 unmodified lines
339	340	LIBBPF_API int bpf_program__unpin_instance(struct bpf_program *prog,
340	341	const char *path,
341	342	int instance);
	343	+
	344	+/**
	345	+ * @brief bpf_program__pin() pins the BPF program to a file
	346	+ * in the BPF FS specified by a path. This increments the programs
	347	+ * reference count, allowing it to stay loaded after the process
	348	+ * which loaded it has exited.
	349	+ *
	350	+ * @param prog BPF program to pin, must already be loaded
	351	+ * @param path file path in a BPF file system
	352	+ * @return 0, on success; negative error code, otherwise
	353	+ */
342	354	LIBBPF_API int bpf_program__pin(struct bpf_program prog, const char path);
	355	+
	356	+/**
	357	+ * @brief bpf_program__unpin() unpins the BPF program from a file
	358	+ * in the BPFFS specified by a path. This decrements the programs
	359	+ * reference count.
	360	+ *
	361	+ * The file pinning the BPF program can also be unlinked by a different
	362	+ * process in which case this function will return an error.
	363	+ *
	364	+ * @param prog BPF program to unpin
	365	+ * @param path file path to the pin in a BPF file system
	366	+ * @return 0, on success; negative error code, otherwise
	367	+ */
343	368	LIBBPF_API int bpf_program__unpin(struct bpf_program prog, const char path);
344	369	LIBBPF_API void bpf_program__unload(struct bpf_program *prog);
345	370
@@ -1027,13 +1052,57 @@ bpf_prog_linfo__lfind(const struct bpf_prog_linfo *prog_linfo, · 681 unmodified lines
1027	1052	* user, causing subsequent probes to fail. In this case, the caller may want
1028	1053	* to adjust that limit with setrlimit().
1029	1054	*/
1030		−LIBBPF_API bool bpf_probe_prog_type(enum bpf_prog_type prog_type,
1031		− __u32 ifindex);
	1055	+LIBBPF_DEPRECATED_SINCE(0, 8, "use libbpf_probe_bpf_prog_type() instead")
	1056	+LIBBPF_API bool bpf_probe_prog_type(enum bpf_prog_type prog_type, __u32 ifindex);
	1057	+LIBBPF_DEPRECATED_SINCE(0, 8, "use libbpf_probe_bpf_map_type() instead")
1032	1058	LIBBPF_API bool bpf_probe_map_type(enum bpf_map_type map_type, __u32 ifindex);
1033		−LIBBPF_API bool bpf_probe_helper(enum bpf_func_id id,
1034		− enum bpf_prog_type prog_type, __u32 ifindex);
	1059	+LIBBPF_DEPRECATED_SINCE(0, 8, "use libbpf_probe_bpf_helper() instead")
	1060	+LIBBPF_API bool bpf_probe_helper(enum bpf_func_id id, enum bpf_prog_type prog_type, __u32 ifindex);
	1061	+LIBBPF_DEPRECATED_SINCE(0, 8, "implement your own or use bpftool for feature detection")
1035	1062	LIBBPF_API bool bpf_probe_large_insn_limit(__u32 ifindex);
1036	1063
	1064	+/**
	1065	+ * @brief libbpf_probe_bpf_prog_type() detects if host kernel supports
	1066	+ * BPF programs of a given type.
	1067	+ * @param prog_type BPF program type to detect kernel support for
	1068	+ * @param opts reserved for future extensibility, should be NULL
	1069	+ * @return 1, if given program type is supported; 0, if given program type is
	1070	+ * not supported; negative error code if feature detection failed or can't be
	1071	+ * performed
	1072	+ *
	1073	+ * Make sure the process has required set of CAP_* permissions (or runs as
	1074	+ * root) when performing feature checking.
	1075	+ */
	1076	+LIBBPF_API int libbpf_probe_bpf_prog_type(enum bpf_prog_type prog_type, const void *opts);
	1077	+/**
	1078	+ * @brief libbpf_probe_bpf_map_type() detects if host kernel supports
	1079	+ * BPF maps of a given type.
	1080	+ * @param map_type BPF map type to detect kernel support for
	1081	+ * @param opts reserved for future extensibility, should be NULL
	1082	+ * @return 1, if given map type is supported; 0, if given map type is
	1083	+ * not supported; negative error code if feature detection failed or can't be
	1084	+ * performed
	1085	+ *
	1086	+ * Make sure the process has required set of CAP_* permissions (or runs as
	1087	+ * root) when performing feature checking.
	1088	+ */
	1089	+LIBBPF_API int libbpf_probe_bpf_map_type(enum bpf_map_type map_type, const void *opts);
	1090	+/**
	1091	+ * @brief libbpf_probe_bpf_helper() detects if host kernel supports the
	1092	+ * use of a given BPF helper from specified BPF program type.
	1093	+ * @param prog_type BPF program type used to check the support of BPF helper
	1094	+ * @param helper_id BPF helper ID (enum bpf_func_id) to check support for
	1095	+ * @param opts reserved for future extensibility, should be NULL
	1096	+ * @return 1, if given combination of program type and helper is supported; 0,
	1097	+ * if the combination is not supported; negative error code if feature
	1098	+ * detection for provided input arguments failed or can't be performed
	1099	+ *
	1100	+ * Make sure the process has required set of CAP_* permissions (or runs as
	1101	+ * root) when performing feature checking.
	1102	+ */
	1103	+LIBBPF_API int libbpf_probe_bpf_helper(enum bpf_prog_type prog_type,
	1104	+ enum bpf_func_id helper_id, const void *opts);
	1105	+
1037	1106	/*
1038	1107	* Get bpf_prog_info in continuous memory
1039	1108	*

tools/lib/bpf/libbpf.map+4−0

@@ -427,4 +427,8 @@ LIBBPF_0.7.0 { · 426 unmodified lines
427	427	bpf_program__log_level;
428	428	bpf_program__set_log_buf;
429	429	bpf_program__set_log_level;
	430	+ libbpf_probe_bpf_helper;
	431	+ libbpf_probe_bpf_map_type;
	432	+ libbpf_probe_bpf_prog_type;
	433	+ libbpf_set_memlock_rlim_max;
430	434	};

tools/lib/bpf/libbpf_internal.h+60−0

@@ -169,6 +169,27 @@ static inline void libbpf_reallocarray(void ptr, size_t nmemb, size_t size) · 168 unmodified lines
169	169	return realloc(ptr, total);
170	170	}
171	171
	172	+/* Copy up to sz - 1 bytes from zero-terminated src string and ensure that dst
	173	+ * is zero-terminated string no matter what (unless sz == 0, in which case
	174	+ * it's a no-op). It's conceptually close to FreeBSD's strlcpy(), but differs
	175	+ * in what is returned. Given this is internal helper, it's trivial to extend
	176	+ * this, when necessary. Use this instead of strncpy inside libbpf source code.
	177	+ */
	178	+static inline void libbpf_strlcpy(char dst, const char src, size_t sz)
	179	+{
	180	+ size_t i;
	181	+
	182	+ if (sz == 0)
	183	+ return;
	184	+
	185	+ sz--;
	186	+ for (i = 0; i < sz && src[i]; i++)
	187	+ dst[i] = src[i];
	188	+ dst[i] = '\0';
	189	+}
	190	+
	191	+__u32 get_kernel_version(void);
	192	+
172	193	struct btf;
173	194	struct btf_type;
174	195
@@ -272,6 +293,45 @@ static inline bool libbpf_validate_opts(const char *opts, · 97 unmodified lines
272	293	(opts)->sz - __off); \
273	294	})
274	295
	296	+enum kern_feature_id {
	297	+ /* v4.14: kernel support for program & map names. */
	298	+ FEAT_PROG_NAME,
	299	+ /* v5.2: kernel support for global data sections. */
	300	+ FEAT_GLOBAL_DATA,
	301	+ /* BTF support */
	302	+ FEAT_BTF,
	303	+ /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */
	304	+ FEAT_BTF_FUNC,
	305	+ /* BTF_KIND_VAR and BTF_KIND_DATASEC support */
	306	+ FEAT_BTF_DATASEC,
	307	+ /* BTF_FUNC_GLOBAL is supported */
	308	+ FEAT_BTF_GLOBAL_FUNC,
	309	+ /* BPF_F_MMAPABLE is supported for arrays */
	310	+ FEAT_ARRAY_MMAP,
	311	+ /* kernel support for expected_attach_type in BPF_PROG_LOAD */
	312	+ FEAT_EXP_ATTACH_TYPE,
	313	+ /* bpf_probe_read_{kernel,user}[_str] helpers */
	314	+ FEAT_PROBE_READ_KERN,
	315	+ /* BPF_PROG_BIND_MAP is supported */
	316	+ FEAT_PROG_BIND_MAP,
	317	+ /* Kernel support for module BTFs */
	318	+ FEAT_MODULE_BTF,
	319	+ /* BTF_KIND_FLOAT support */
	320	+ FEAT_BTF_FLOAT,
	321	+ /* BPF perf link support */
	322	+ FEAT_PERF_LINK,
	323	+ /* BTF_KIND_DECL_TAG support */
	324	+ FEAT_BTF_DECL_TAG,
	325	+ /* BTF_KIND_TYPE_TAG support */
	326	+ FEAT_BTF_TYPE_TAG,
	327	+ /* memcg-based accounting for BPF maps and progs */
	328	+ FEAT_MEMCG_ACCOUNT,
	329	+ __FEAT_CNT,
	330	+};
	331	+
	332	+int probe_memcg_account(void);
	333	+bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id);
	334	+int bump_rlimit_memlock(void);
275	335
276	336	int parse_cpu_mask_str(const char s, bool mask, int mask_sz);
277	337	int parse_cpu_mask_file(const char fcpu, bool mask, int mask_sz);

tools/lib/bpf/libbpf_legacy.h+11−1

@@ -45,7 +45,6 @@ enum libbpf_strict_mode { · 44 unmodified lines
45	45	* (positive) error code.
46	46	*/
47	47	LIBBPF_STRICT_DIRECT_ERRS = 0x02,
48		−
49	48	/*
50	49	* Enforce strict BPF program section (SEC()) names.
51	50	* E.g., while prefiously SEC("xdp_whatever") or SEC("perf_event_blah") were
@@ -63,6 +62,17 @@ enum libbpf_strict_mode { · 11 unmodified lines
63	62	* Clients can maintain it on their own if it is valuable for them.
64	63	*/
65	64	LIBBPF_STRICT_NO_OBJECT_LIST = 0x08,
	65	+ /*
	66	+ * Automatically bump RLIMIT_MEMLOCK using setrlimit() before the
	67	+ * first BPF program or map creation operation. This is done only if
	68	+ * kernel is too old to support memcg-based memory accounting for BPF
	69	+ * subsystem. By default, RLIMIT_MEMLOCK limit is set to RLIM_INFINITY,
	70	+ * but it can be overriden with libbpf_set_memlock_rlim_max() API.
	71	+ * Note that libbpf_set_memlock_rlim_max() needs to be called before
	72	+ * the very first bpf_prog_load(), bpf_map_create() or bpf_object__load()
	73	+ * operation.
	74	+ */
	75	+ LIBBPF_STRICT_AUTO_RLIMIT_MEMLOCK = 0x10,
66	76
67	77	__LIBBPF_STRICT_LAST,
68	78	};

tools/lib/bpf/libbpf_probes.c+185−66

@@ -48,28 +48,20 @@ static int get_vendor_id(int ifindex) · 47 unmodified lines
48	48	return strtol(buf, NULL, 0);
49	49	}
50	50
51		−static int get_kernel_version(void)
	51	+static int probe_prog_load(enum bpf_prog_type prog_type,
	52	+ const struct bpf_insn *insns, size_t insns_cnt,
	53	+ char *log_buf, size_t log_buf_sz,
	54	+ __u32 ifindex)
52	55	{
53		− int version, subversion, patchlevel;
54		− struct utsname utsn;
55		−
56		− /* Return 0 on failure, and attempt to probe with empty kversion */
57		− if (uname(&utsn))
58		− return 0;
59		−
60		− if (sscanf(utsn.release, "%d.%d.%d",
61		− &version, &subversion, &patchlevel) != 3)
62		− return 0;
63		−
64		− return (version << 16) + (subversion << 8) + patchlevel;
65		−}
66		−
67		−static void
68		−probe_load(enum bpf_prog_type prog_type, const struct bpf_insn *insns,
69		− size_t insns_cnt, char *buf, size_t buf_len, __u32 ifindex)
70		−{
71		− LIBBPF_OPTS(bpf_prog_load_opts, opts);
72		− int fd;
	56	+ LIBBPF_OPTS(bpf_prog_load_opts, opts,
	57	+ .log_buf = log_buf,
	58	+ .log_size = log_buf_sz,
	59	+ .log_level = log_buf ? 1 : 0,
	60	+ .prog_ifindex = ifindex,
	61	+ );
	62	+ int fd, err, exp_err = 0;
	63	+ const char *exp_msg = NULL;
	64	+ char buf[4096];
73	65
74	66	switch (prog_type) {
75	67	case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
@@ -84,6 +76,38 @@ probe_load(enum bpf_prog_type prog_type, const struct bpf_insn *insns, · 8 unmodified lines
84	76	case BPF_PROG_TYPE_KPROBE:
85	77	opts.kern_version = get_kernel_version();
86	78	break;
	79	+ case BPF_PROG_TYPE_LIRC_MODE2:
	80	+ opts.expected_attach_type = BPF_LIRC_MODE2;
	81	+ break;
	82	+ case BPF_PROG_TYPE_TRACING:
	83	+ case BPF_PROG_TYPE_LSM:
	84	+ opts.log_buf = buf;
	85	+ opts.log_size = sizeof(buf);
	86	+ opts.log_level = 1;
	87	+ if (prog_type == BPF_PROG_TYPE_TRACING)
	88	+ opts.expected_attach_type = BPF_TRACE_FENTRY;
	89	+ else
	90	+ opts.expected_attach_type = BPF_MODIFY_RETURN;
	91	+ opts.attach_btf_id = 1;
	92	+
	93	+ exp_err = -EINVAL;
	94	+ exp_msg = "attach_btf_id 1 is not a function";
	95	+ break;
	96	+ case BPF_PROG_TYPE_EXT:
	97	+ opts.log_buf = buf;
	98	+ opts.log_size = sizeof(buf);
	99	+ opts.log_level = 1;
	100	+ opts.attach_btf_id = 1;
	101	+
	102	+ exp_err = -EINVAL;
	103	+ exp_msg = "Cannot replace kernel functions";
	104	+ break;
	105	+ case BPF_PROG_TYPE_SYSCALL:
	106	+ opts.prog_flags = BPF_F_SLEEPABLE;
	107	+ break;
	108	+ case BPF_PROG_TYPE_STRUCT_OPS:
	109	+ exp_err = -524; /* -ENOTSUPP */
	110	+ break;
87	111	case BPF_PROG_TYPE_UNSPEC:
88	112	case BPF_PROG_TYPE_SOCKET_FILTER:
89	113	case BPF_PROG_TYPE_SCHED_CLS:
@@ -103,25 +127,42 @@ probe_load(enum bpf_prog_type prog_type, const struct bpf_insn *insns, · 13 unmodified lines
103	127	case BPF_PROG_TYPE_RAW_TRACEPOINT:
104	128	case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
105	129	case BPF_PROG_TYPE_LWT_SEG6LOCAL:
106		− case BPF_PROG_TYPE_LIRC_MODE2:
107	130	case BPF_PROG_TYPE_SK_REUSEPORT:
108	131	case BPF_PROG_TYPE_FLOW_DISSECTOR:
109	132	case BPF_PROG_TYPE_CGROUP_SYSCTL:
110		− case BPF_PROG_TYPE_TRACING:
111		− case BPF_PROG_TYPE_STRUCT_OPS:
112		− case BPF_PROG_TYPE_EXT:
113		− case BPF_PROG_TYPE_LSM:
114		− default:
115	133	break;
	134	+ default:
	135	+ return -EOPNOTSUPP;
116	136	}
117	137
118		− opts.prog_ifindex = ifindex;
119		− opts.log_buf = buf;
120		− opts.log_size = buf_len;
121		−
122		− fd = bpf_prog_load(prog_type, NULL, "GPL", insns, insns_cnt, NULL);
	138	+ fd = bpf_prog_load(prog_type, NULL, "GPL", insns, insns_cnt, &opts);
	139	+ err = -errno;
123	140	if (fd >= 0)
124	141	close(fd);
	142	+ if (exp_err) {
	143	+ if (fd >= 0 \|\| err != exp_err)
	144	+ return 0;
	145	+ if (exp_msg && !strstr(buf, exp_msg))
	146	+ return 0;
	147	+ return 1;
	148	+ }
	149	+ return fd >= 0 ? 1 : 0;
	150	+}
	151	+
	152	+int libbpf_probe_bpf_prog_type(enum bpf_prog_type prog_type, const void *opts)
	153	+{
	154	+ struct bpf_insn insns[] = {
	155	+ BPF_MOV64_IMM(BPF_REG_0, 0),
	156	+ BPF_EXIT_INSN()
	157	+ };
	158	+ const size_t insn_cnt = ARRAY_SIZE(insns);
	159	+ int ret;
	160	+
	161	+ if (opts)
	162	+ return libbpf_err(-EINVAL);
	163	+
	164	+ ret = probe_prog_load(prog_type, insns, insn_cnt, NULL, 0, 0);
	165	+ return libbpf_err(ret);
125	166	}
126	167
127	168	bool bpf_probe_prog_type(enum bpf_prog_type prog_type, __u32 ifindex)
@@ -131,12 +172,16 @@ bool bpf_probe_prog_type(enum bpf_prog_type prog_type, __u32 ifindex) · 3 unmodified lines
131	172	BPF_EXIT_INSN()
132	173	};
133	174
	175	+ /* prefer libbpf_probe_bpf_prog_type() unless offload is requested */
	176	+ if (ifindex == 0)
	177	+ return libbpf_probe_bpf_prog_type(prog_type, NULL) == 1;
	178	+
134	179	if (ifindex && prog_type == BPF_PROG_TYPE_SCHED_CLS)
135	180	/* nfp returns -EINVAL on exit(0) with TC offload */
136	181	insns[0].imm = 2;
137	182
138	183	errno = 0;
139		− probe_load(prog_type, insns, ARRAY_SIZE(insns), NULL, 0, ifindex);
	184	+ probe_prog_load(prog_type, insns, ARRAY_SIZE(insns), NULL, 0, ifindex);
140	185
141	186	return errno != EINVAL && errno != EOPNOTSUPP;
142	187	}
@@ -197,16 +242,18 @@ static int load_local_storage_btf(void) · 54 unmodified lines
197	242	strs, sizeof(strs));
198	243	}
199	244
200		−bool bpf_probe_map_type(enum bpf_map_type map_type, __u32 ifindex)
	245	+static int probe_map_create(enum bpf_map_type map_type, __u32 ifindex)
201	246	{
202		− int key_size, value_size, max_entries, map_flags;
	247	+ LIBBPF_OPTS(bpf_map_create_opts, opts);
	248	+ int key_size, value_size, max_entries;
203	249	__u32 btf_key_type_id = 0, btf_value_type_id = 0;
204		− int fd = -1, btf_fd = -1, fd_inner;
	250	+ int fd = -1, btf_fd = -1, fd_inner = -1, exp_err = 0, err;
	251	+
	252	+ opts.map_ifindex = ifindex;
205	253
206	254	key_size = sizeof(__u32);
207	255	value_size = sizeof(__u32);
208	256	max_entries = 1;
209		− map_flags = 0;
210	257
211	258	switch (map_type) {
212	259	case BPF_MAP_TYPE_STACK_TRACE:
@@ -215,7 +262,7 @@ bool bpf_probe_map_type(enum bpf_map_type map_type, __u32 ifindex) · 2 unmodified lines
215	262	case BPF_MAP_TYPE_LPM_TRIE:
216	263	key_size = sizeof(__u64);
217	264	value_size = sizeof(__u64);
218		− map_flags = BPF_F_NO_PREALLOC;
	265	+ opts.map_flags = BPF_F_NO_PREALLOC;
219	266	break;
220	267	case BPF_MAP_TYPE_CGROUP_STORAGE:
221	268	case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE:
@@ -234,17 +281,25 @@ bool bpf_probe_map_type(enum bpf_map_type map_type, __u32 ifindex) · 12 unmodified lines
234	281	btf_value_type_id = 3;
235	282	value_size = 8;
236	283	max_entries = 0;
237		− map_flags = BPF_F_NO_PREALLOC;
	284	+ opts.map_flags = BPF_F_NO_PREALLOC;
238	285	btf_fd = load_local_storage_btf();
239	286	if (btf_fd < 0)
240		− return false;
	287	+ return btf_fd;
241	288	break;
242	289	case BPF_MAP_TYPE_RINGBUF:
243	290	key_size = 0;
244	291	value_size = 0;
245	292	max_entries = 4096;
246	293	break;
247		− case BPF_MAP_TYPE_UNSPEC:
	294	+ case BPF_MAP_TYPE_STRUCT_OPS:
	295	+ /* we'll get -ENOTSUPP for invalid BTF type ID for struct_ops */
	296	+ opts.btf_vmlinux_value_type_id = 1;
	297	+ exp_err = -524; /* -ENOTSUPP */
	298	+ break;
	299	+ case BPF_MAP_TYPE_BLOOM_FILTER:
	300	+ key_size = 0;
	301	+ max_entries = 1;
	302	+ break;
248	303	case BPF_MAP_TYPE_HASH:
249	304	case BPF_MAP_TYPE_ARRAY:
250	305	case BPF_MAP_TYPE_PROG_ARRAY:
@@ -263,49 +318,114 @@ bool bpf_probe_map_type(enum bpf_map_type map_type, __u32 ifindex) · 12 unmodified lines
263	318	case BPF_MAP_TYPE_XSKMAP:
264	319	case BPF_MAP_TYPE_SOCKHASH:
265	320	case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY:
266		− case BPF_MAP_TYPE_STRUCT_OPS:
267		− default:
268	321	break;
	322	+ case BPF_MAP_TYPE_UNSPEC:
	323	+ default:
	324	+ return -EOPNOTSUPP;
269	325	}
270	326
271	327	if (map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS \|\|
272	328	map_type == BPF_MAP_TYPE_HASH_OF_MAPS) {
273		− LIBBPF_OPTS(bpf_map_create_opts, opts);
274		−
275	329	/* TODO: probe for device, once libbpf has a function to create
276	330	* map-in-map for offload
277	331	*/
278	332	if (ifindex)
279		− return false;
	333	+ goto cleanup;
280	334
281	335	fd_inner = bpf_map_create(BPF_MAP_TYPE_HASH, NULL,
282	336	sizeof(__u32), sizeof(__u32), 1, NULL);
283	337	if (fd_inner < 0)
284		− return false;
	338	+ goto cleanup;
285	339
286	340	opts.inner_map_fd = fd_inner;
287		− fd = bpf_map_create(map_type, NULL, sizeof(__u32), sizeof(__u32), 1, &opts);
288		− close(fd_inner);
289		− } else {
290		− LIBBPF_OPTS(bpf_map_create_opts, opts);
291		−
292		− /* Note: No other restriction on map type probes for offload */
293		− opts.map_flags = map_flags;
294		− opts.map_ifindex = ifindex;
295		− if (btf_fd >= 0) {
296		− opts.btf_fd = btf_fd;
297		− opts.btf_key_type_id = btf_key_type_id;
298		− opts.btf_value_type_id = btf_value_type_id;
299		− }
	341	+ }
300	342
301		− fd = bpf_map_create(map_type, NULL, key_size, value_size, max_entries, &opts);
	343	+ if (btf_fd >= 0) {
	344	+ opts.btf_fd = btf_fd;
	345	+ opts.btf_key_type_id = btf_key_type_id;
	346	+ opts.btf_value_type_id = btf_value_type_id;
302	347	}
	348	+
	349	+ fd = bpf_map_create(map_type, NULL, key_size, value_size, max_entries, &opts);
	350	+ err = -errno;
	351	+
	352	+cleanup:
303	353	if (fd >= 0)
304	354	close(fd);
	355	+ if (fd_inner >= 0)
	356	+ close(fd_inner);
305	357	if (btf_fd >= 0)
306	358	close(btf_fd);
307	359
308		− return fd >= 0;
	360	+ if (exp_err)
	361	+ return fd < 0 && err == exp_err ? 1 : 0;
	362	+ else
	363	+ return fd >= 0 ? 1 : 0;
	364	+}
	365	+
	366	+int libbpf_probe_bpf_map_type(enum bpf_map_type map_type, const void *opts)
	367	+{
	368	+ int ret;
	369	+
	370	+ if (opts)
	371	+ return libbpf_err(-EINVAL);
	372	+
	373	+ ret = probe_map_create(map_type, 0);
	374	+ return libbpf_err(ret);
	375	+}
	376	+
	377	+bool bpf_probe_map_type(enum bpf_map_type map_type, __u32 ifindex)
	378	+{
	379	+ return probe_map_create(map_type, ifindex) == 1;
	380	+}
	381	+
	382	+int libbpf_probe_bpf_helper(enum bpf_prog_type prog_type, enum bpf_func_id helper_id,
	383	+ const void *opts)
	384	+{
	385	+ struct bpf_insn insns[] = {
	386	+ BPF_EMIT_CALL((__u32)helper_id),
	387	+ BPF_EXIT_INSN(),
	388	+ };
	389	+ const size_t insn_cnt = ARRAY_SIZE(insns);
	390	+ char buf[4096];
	391	+ int ret;
	392	+
	393	+ if (opts)
	394	+ return libbpf_err(-EINVAL);
	395	+
	396	+ /* we can't successfully load all prog types to check for BPF helper
	397	+ * support, so bail out with -EOPNOTSUPP error
	398	+ */
	399	+ switch (prog_type) {
	400	+ case BPF_PROG_TYPE_TRACING:
	401	+ case BPF_PROG_TYPE_EXT:
	402	+ case BPF_PROG_TYPE_LSM:
	403	+ case BPF_PROG_TYPE_STRUCT_OPS:
	404	+ return -EOPNOTSUPP;
	405	+ default:
	406	+ break;
	407	+ }
	408	+
	409	+ buf[0] = '\0';
	410	+ ret = probe_prog_load(prog_type, insns, insn_cnt, buf, sizeof(buf), 0);
	411	+ if (ret < 0)
	412	+ return libbpf_err(ret);
	413	+
	414	+ /* If BPF verifier doesn't recognize BPF helper ID (enum bpf_func_id)
	415	+ * at all, it will emit something like "invalid func unknown#181".
	416	+ * If BPF verifier recognizes BPF helper but it's not supported for
	417	+ * given BPF program type, it will emit "unknown func bpf_sys_bpf#166".
	418	+ * In both cases, provided combination of BPF program type and BPF
	419	+ * helper is not supported by the kernel.
	420	+ * In all other cases, probe_prog_load() above will either succeed (e.g.,
	421	+ * because BPF helper happens to accept no input arguments or it
	422	+ * accepts one input argument and initial PTR_TO_CTX is fine for
	423	+ * that), or we'll get some more specific BPF verifier error about
	424	+ * some unsatisfied conditions.
	425	+ */
	426	+ if (ret == 0 && (strstr(buf, "invalid func ") \|\| strstr(buf, "unknown func ")))
	427	+ return 0;
	428	+ return 1; /* assume supported */
309	429	}
310	430
311	431	bool bpf_probe_helper(enum bpf_func_id id, enum bpf_prog_type prog_type,
@@ -318,8 +438,7 @@ bool bpf_probe_helper(enum bpf_func_id id, enum bpf_prog_type prog_type, · 6 unmodified lines
318	438	char buf[4096] = {};
319	439	bool res;
320	440
321		− probe_load(prog_type, insns, ARRAY_SIZE(insns), buf, sizeof(buf),
322		− ifindex);
	441	+ probe_prog_load(prog_type, insns, ARRAY_SIZE(insns), buf, sizeof(buf), ifindex);
323	442	res = !grep(buf, "invalid func ") && !grep(buf, "unknown func ");
324	443
325	444	if (ifindex) {
@@ -351,8 +470,8 @@ bool bpf_probe_large_insn_limit(__u32 ifindex) · 25 unmodified lines
351	470	insns[BPF_MAXINSNS] = BPF_EXIT_INSN();
352	471
353	472	errno = 0;
354		− probe_load(BPF_PROG_TYPE_SCHED_CLS, insns, ARRAY_SIZE(insns), NULL, 0,
355		− ifindex);
	473	+ probe_prog_load(BPF_PROG_TYPE_SCHED_CLS, insns, ARRAY_SIZE(insns), NULL, 0,
	474	+ ifindex);
356	475
357	476	return errno != E2BIG && errno != EINVAL;
358	477	}

tools/lib/bpf/relo_core.c+15−5

@@ -709,10 +709,14 @@ static int bpf_core_calc_field_relo(const char *prog_name, · 708 unmodified lines
709	709
710	710	static int bpf_core_calc_type_relo(const struct bpf_core_relo *relo,
711	711	const struct bpf_core_spec *spec,
712		− __u32 *val)
	712	+ __u32 val, bool validate)
713	713	{
714	714	__s64 sz;
715	715
	716	+ /* by default, always check expected value in bpf_insn */
	717	+ if (validate)
	718	+ *validate = true;
	719	+
716	720	/* type-based relos return zero when target type is not found */
717	721	if (!spec) {
718	722	*val = 0;
@@ -722,6 +726,11 @@ static int bpf_core_calc_type_relo(const struct bpf_core_relo *relo, · 3 unmodified lines
722	726	switch (relo->kind) {
723	727	case BPF_CORE_TYPE_ID_TARGET:
724	728	*val = spec->root_type_id;
	729	+ /* type ID, embedded in bpf_insn, might change during linking,
	730	+ * so enforcing it is pointless
	731	+ */
	732	+ if (validate)
	733	+ *validate = false;
725	734	break;
726	735	case BPF_CORE_TYPE_EXISTS:
727	736	*val = 1;
@@ -861,8 +870,8 @@ static int bpf_core_calc_relo(const char *prog_name, · 133 unmodified lines
861	870	res->fail_memsz_adjust = true;
862	871	}
863	872	} else if (core_relo_is_type_based(relo->kind)) {
864		− err = bpf_core_calc_type_relo(relo, local_spec, &res->orig_val);
865		− err = err ?: bpf_core_calc_type_relo(relo, targ_spec, &res->new_val);
	873	+ err = bpf_core_calc_type_relo(relo, local_spec, &res->orig_val, &res->validate);
	874	+ err = err ?: bpf_core_calc_type_relo(relo, targ_spec, &res->new_val, NULL);
866	875	} else if (core_relo_is_enumval_based(relo->kind)) {
867	876	err = bpf_core_calc_enumval_relo(relo, local_spec, &res->orig_val);
868	877	err = err ?: bpf_core_calc_enumval_relo(relo, targ_spec, &res->new_val);
@@ -1213,7 +1222,9 @@ int bpf_core_apply_relo_insn(const char prog_name, struct bpf_insn insn, · 344 unmodified lines
1213	1222
1214	1223	/* TYPE_ID_LOCAL relo is special and doesn't need candidate search */
1215	1224	if (relo->kind == BPF_CORE_TYPE_ID_LOCAL) {
1216		− targ_res.validate = true;
	1225	+ /* bpf_insn's imm value could get out of sync during linking */
	1226	+ memset(&targ_res, 0, sizeof(targ_res));
	1227	+ targ_res.validate = false;
1217	1228	targ_res.poison = false;
1218	1229	targ_res.orig_val = local_spec->root_type_id;
1219	1230	targ_res.new_val = local_spec->root_type_id;
@@ -1227,7 +1238,6 @@ int bpf_core_apply_relo_insn(const char prog_name, struct bpf_insn insn, · 7 unmodified lines
1227	1238	return -EOPNOTSUPP;
1228	1239	}
1229	1240
1230		−
1231	1241	for (i = 0, j = 0; i < cands->len; i++) {
1232	1242	err = bpf_core_spec_match(local_spec, cands->cands[i].btf,
1233	1243	cands->cands[i].id, cand_spec);

tools/lib/bpf/xsk.c+3−6

@@ -548,8 +548,7 @@ static int xsk_get_max_queues(struct xsk_socket *xsk) · 547 unmodified lines
548	548	return -errno;
549	549
550	550	ifr.ifr_data = (void *)&channels;
551		− memcpy(ifr.ifr_name, ctx->ifname, IFNAMSIZ - 1);
552		− ifr.ifr_name[IFNAMSIZ - 1] = '\0';
	551	+ libbpf_strlcpy(ifr.ifr_name, ctx->ifname, IFNAMSIZ);
553	552	err = ioctl(fd, SIOCETHTOOL, &ifr);
554	553	if (err && errno != EOPNOTSUPP) {
555	554	ret = -errno;
@@ -768,8 +767,7 @@ static int xsk_create_xsk_struct(int ifindex, struct xsk_socket *xsk) · 212 unmodified lines
768	767	}
769	768
770	769	ctx->ifindex = ifindex;
771		− memcpy(ctx->ifname, ifname, IFNAMSIZ -1);
772		− ctx->ifname[IFNAMSIZ - 1] = 0;
	770	+ libbpf_strlcpy(ctx->ifname, ifname, IFNAMSIZ);
773	771
774	772	xsk->ctx = ctx;
775	773	xsk->ctx->has_bpf_link = xsk_probe_bpf_link();
@@ -951,8 +949,7 @@ static struct xsk_ctx xsk_create_ctx(struct xsk_socket xsk, · 175 unmodified lines
951	949	ctx->refcount = 1;
952	950	ctx->umem = umem;
953	951	ctx->queue_id = queue_id;
954		− memcpy(ctx->ifname, ifname, IFNAMSIZ - 1);
955		− ctx->ifname[IFNAMSIZ - 1] = '\0';
	952	+ libbpf_strlcpy(ctx->ifname, ifname, IFNAMSIZ);
956	953
957	954	ctx->fill = fill;
958	955	ctx->comp = comp;

tools/perf/builtin-trace.c+12−1

@@ -3257,10 +3257,21 @@ static void trace__set_bpf_map_syscalls(struct trace *trace) · 3256 unmodified lines
3257	3257
3258	3258	static struct bpf_program trace__find_bpf_program_by_title(struct trace trace, const char *name)
3259	3259	{
	3260	+ struct bpf_program pos, prog = NULL;
	3261	+ const char *sec_name;
	3262	+
3260	3263	if (trace->bpf_obj == NULL)
3261	3264	return NULL;
3262	3265
3263		− return bpf_object__find_program_by_title(trace->bpf_obj, name);
	3266	+ bpf_object__for_each_program(pos, trace->bpf_obj) {
	3267	+ sec_name = bpf_program__section_name(pos);
	3268	+ if (sec_name && !strcmp(sec_name, name)) {
	3269	+ prog = pos;
	3270	+ break;
	3271	+ }
	3272	+ }
	3273	+
	3274	+ return prog;
3264	3275	}
3265	3276
3266	3277	static struct bpf_program trace__find_syscall_bpf_prog(struct trace trace, struct syscall *sc,

tools/scripts/Makefile.include+12−1

@@ -87,7 +87,18 @@ LLVM_STRIP ?= llvm-strip · 86 unmodified lines
87	87
88	88	ifeq ($(CC_NO_CLANG), 1)
89	89	EXTRA_WARNINGS += -Wstrict-aliasing=3
90		−endif
	90	+
	91	+else ifneq ($(CROSS_COMPILE),)
	92	+CLANG_CROSS_FLAGS := --target=$(notdir $(CROSS_COMPILE:%-=%))
	93	+GCC_TOOLCHAIN_DIR := $(dir $(shell which $(CROSS_COMPILE)gcc))
	94	+ifneq ($(GCC_TOOLCHAIN_DIR),)
	95	+CLANG_CROSS_FLAGS += --prefix=$(GCC_TOOLCHAIN_DIR)$(notdir $(CROSS_COMPILE))
	96	+CLANG_CROSS_FLAGS += --sysroot=$(shell $(CROSS_COMPILE)gcc -print-sysroot)
	97	+CLANG_CROSS_FLAGS += --gcc-toolchain=$(realpath $(GCC_TOOLCHAIN_DIR)/..)
	98	+endif # GCC_TOOLCHAIN_DIR
	99	+CFLAGS += $(CLANG_CROSS_FLAGS)
	100	+AFLAGS += $(CLANG_CROSS_FLAGS)
	101	+endif # CROSS_COMPILE
91	102
92	103	# Hack to avoid type-punned warnings on old systems such as RHEL5:
93	104	# We should be changing CFLAGS and checking gcc version, but this

tools/testing/selftests/bpf/Makefile+7−5

@@ -170,7 +170,7 @@ $(OUTPUT)/%:%.c · 169 unmodified lines
170	170
171	171	$(OUTPUT)/urandom_read: urandom_read.c
172	172	$(call msg,BINARY,,$@)
173		− $(Q)$(CC) $(LDFLAGS) $< $(LDLIBS) -Wl,--build-id=sha1 -o $@
	173	+ $(Q)$(CC) $(CFLAGS) $(LDFLAGS) $< $(LDLIBS) -Wl,--build-id=sha1 -o $@
174	174
175	175	$(OUTPUT)/bpf_testmod.ko: $(VMLINUX_BTF) $(wildcard bpf_testmod/Makefile bpf_testmod/*.[ch])
176	176	$(call msg,MOD,,$@)
@@ -217,7 +217,7 @@ BPFTOOL ?= $(DEFAULT_BPFTOOL) · 40 unmodified lines
217	217	$(DEFAULT_BPFTOOL): $(wildcard $(BPFTOOLDIR)/*.[ch] $(BPFTOOLDIR)/Makefile) \
218	218	$(HOST_BPFOBJ) \| $(HOST_BUILD_DIR)/bpftool
219	219	$(Q)$(MAKE) $(submake_extras) -C $(BPFTOOLDIR) \
220		− CC=$(HOSTCC) LD=$(HOSTLD) \
	220	+ ARCH= CROSS_COMPILE= CC=$(HOSTCC) LD=$(HOSTLD) \
221	221	EXTRA_CFLAGS='-g -O0' \
222	222	OUTPUT=$(HOST_BUILD_DIR)/bpftool/ \
223	223	LIBBPF_OUTPUT=$(HOST_BUILD_DIR)/libbpf/ \
@@ -248,7 +248,7 @@ $(HOST_BPFOBJ): $(wildcard $(BPFDIR)/*.[ch] $(BPFDIR)/Makefile) \ · 24 unmodified lines
248	248	$(APIDIR)/linux/bpf.h \
249	249	\| $(HOST_BUILD_DIR)/libbpf
250	250	$(Q)$(MAKE) $(submake_extras) -C $(BPFDIR) \
251		− EXTRA_CFLAGS='-g -O0' \
	251	+ EXTRA_CFLAGS='-g -O0' ARCH= CROSS_COMPILE= \
252	252	OUTPUT=$(HOST_BUILD_DIR)/libbpf/ CC=$(HOSTCC) LD=$(HOSTLD) \
253	253	DESTDIR=$(HOST_SCRATCH_DIR)/ prefix= all install_headers
254	254	endif
@@ -537,6 +537,7 @@ $(OUTPUT)/bench_ringbufs.o: $(OUTPUT)/ringbuf_bench.skel.h \ · 282 unmodified lines
537	537	$(OUTPUT)/perfbuf_bench.skel.h
538	538	$(OUTPUT)/bench_bloom_filter_map.o: $(OUTPUT)/bloom_filter_bench.skel.h
539	539	$(OUTPUT)/bench_bpf_loop.o: $(OUTPUT)/bpf_loop_bench.skel.h
	540	+$(OUTPUT)/bench_strncmp.o: $(OUTPUT)/strncmp_bench.skel.h
540	541	$(OUTPUT)/bench.o: bench.h testing_helpers.h $(BPFOBJ)
541	542	$(OUTPUT)/bench: LDLIBS += -lm
542	543	$(OUTPUT)/bench: $(OUTPUT)/bench.o \
@@ -547,9 +548,10 @@ $(OUTPUT)/bench: $(OUTPUT)/bench.o \ · 4 unmodified lines
547	548	$(OUTPUT)/bench_trigger.o \
548	549	$(OUTPUT)/bench_ringbufs.o \
549	550	$(OUTPUT)/bench_bloom_filter_map.o \
550		− $(OUTPUT)/bench_bpf_loop.o
	551	+ $(OUTPUT)/bench_bpf_loop.o \
	552	+ $(OUTPUT)/bench_strncmp.o
551	553	$(call msg,BINARY,,$@)
552		− $(Q)$(CC) $(LDFLAGS) $(filter %.a %.o,$^) $(LDLIBS) -o $@
	554	+ $(Q)$(CC) $(CFLAGS) $(LDFLAGS) $(filter %.a %.o,$^) $(LDLIBS) -o $@
553	555
554	556	EXTRA_CLEAN := $(TEST_CUSTOM_PROGS) $(SCRATCH_DIR) $(HOST_SCRATCH_DIR) \
555	557	prog_tests/tests.h map_tests/tests.h verifier/tests.h \

tools/testing/selftests/bpf/bench.c+7−17

@@ -29,26 +29,10 @@ static int libbpf_print_fn(enum libbpf_print_level level, · 28 unmodified lines
29	29	return vfprintf(stderr, format, args);
30	30	}
31	31
32		−static int bump_memlock_rlimit(void)
	32	+void setup_libbpf(void)
33	33	{
34		− struct rlimit rlim_new = {
35		− .rlim_cur = RLIM_INFINITY,
36		− .rlim_max = RLIM_INFINITY,
37		− };
38		−
39		− return setrlimit(RLIMIT_MEMLOCK, &rlim_new);
40		−}
41		−
42		−void setup_libbpf()
43		−{
44		− int err;
45		−
46	34	libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
47	35	libbpf_set_print(libbpf_print_fn);
48		−
49		− err = bump_memlock_rlimit();
50		− if (err)
51		− fprintf(stderr, "failed to increase RLIMIT_MEMLOCK: %d", err);
52	36	}
53	37
54	38	void false_hits_report_progress(int iter, struct bench_res *res, long delta_ns)
@@ -205,11 +189,13 @@ static const struct argp_option opts[] = { · 150 unmodified lines
205	189	extern struct argp bench_ringbufs_argp;
206	190	extern struct argp bench_bloom_map_argp;
207	191	extern struct argp bench_bpf_loop_argp;
	192	+extern struct argp bench_strncmp_argp;
208	193
209	194	static const struct argp_child bench_parsers[] = {
210	195	{ &bench_ringbufs_argp, 0, "Ring buffers benchmark", 0 },
211	196	{ &bench_bloom_map_argp, 0, "Bloom filter map benchmark", 0 },
212	197	{ &bench_bpf_loop_argp, 0, "bpf_loop helper benchmark", 0 },
	198	+ { &bench_strncmp_argp, 0, "bpf_strncmp helper benchmark", 0 },
213	199	{},
214	200	};
215	201
@@ -409,6 +395,8 @@ extern const struct bench bench_bloom_false_positive; · 193 unmodified lines
409	395	extern const struct bench bench_hashmap_without_bloom;
410	396	extern const struct bench bench_hashmap_with_bloom;
411	397	extern const struct bench bench_bpf_loop;
	398	+extern const struct bench bench_strncmp_no_helper;
	399	+extern const struct bench bench_strncmp_helper;
412	400
413	401	static const struct bench *benchs[] = {
414	402	&bench_count_global,
@@ -441,6 +429,8 @@ static const struct bench *benchs[] = { · 26 unmodified lines
441	429	&bench_hashmap_without_bloom,
442	430	&bench_hashmap_with_bloom,
443	431	&bench_bpf_loop,
	432	+ &bench_strncmp_no_helper,
	433	+ &bench_strncmp_helper,
444	434	};
445	435
446	436	static void setup_benchmark()

tools/testing/selftests/bpf/bench.h+5−4

@@ -38,8 +38,8 @@ struct bench_res { · 37 unmodified lines
38	38
39	39	struct bench {
40	40	const char *name;
41		− void (*validate)();
42		− void (*setup)();
	41	+ void (*validate)(void);
	42	+ void (*setup)(void);
43	43	void (producer_thread)(void *ctx);
44	44	void (consumer_thread)(void *ctx);
45	45	void (measure)(struct bench_res res);
@@ -54,7 +54,7 @@ struct counter { · 8 unmodified lines
54	54	extern struct env env;
55	55	extern const struct bench *bench;
56	56
57		−void setup_libbpf();
	57	+void setup_libbpf(void);
58	58	void hits_drops_report_progress(int iter, struct bench_res *res, long delta_ns);
59	59	void hits_drops_report_final(struct bench_res res[], int res_cnt);
60	60	void false_hits_report_progress(int iter, struct bench_res *res, long delta_ns);
@@ -62,7 +62,8 @@ void false_hits_report_final(struct bench_res res[], int res_cnt); · 1 unmodified line
62	62	void ops_report_progress(int iter, struct bench_res *res, long delta_ns);
63	63	void ops_report_final(struct bench_res res[], int res_cnt);
64	64
65		−static inline __u64 get_time_ns() {
	65	+static inline __u64 get_time_ns(void)
	66	+{
66	67	struct timespec t;
67	68
68	69	clock_gettime(CLOCK_MONOTONIC, &t);

tools/testing/selftests/bpf/benchs/bench_count.c+1−1

@@ -36,7 +36,7 @@ static struct count_local_ctx { · 35 unmodified lines
36	36	struct counter *hits;
37	37	} count_local_ctx;
38	38
39		−static void count_local_setup()
	39	+static void count_local_setup(void)
40	40	{
41	41	struct count_local_ctx *ctx = &count_local_ctx;
42	42

tools/testing/selftests/bpf/benchs/bench_rename.c+8−8

@@ -11,7 +11,7 @@ static struct ctx { · 10 unmodified lines
11	11	int fd;
12	12	} ctx;
13	13
14		−static void validate()
	14	+static void validate(void)
15	15	{
16	16	if (env.producer_cnt != 1) {
17	17	fprintf(stderr, "benchmark doesn't support multi-producer!\n");
@@ -43,7 +43,7 @@ static void measure(struct bench_res *res) · 25 unmodified lines
43	43	res->hits = atomic_swap(&ctx.hits.value, 0);
44	44	}
45	45
46		−static void setup_ctx()
	46	+static void setup_ctx(void)
47	47	{
48	48	setup_libbpf();
49	49
@@ -71,36 +71,36 @@ static void attach_bpf(struct bpf_program *prog) · 21 unmodified lines
71	71	}
72	72	}
73	73
74		−static void setup_base()
	74	+static void setup_base(void)
75	75	{
76	76	setup_ctx();
77	77	}
78	78
79		−static void setup_kprobe()
	79	+static void setup_kprobe(void)
80	80	{
81	81	setup_ctx();
82	82	attach_bpf(ctx.skel->progs.prog1);
83	83	}
84	84
85		−static void setup_kretprobe()
	85	+static void setup_kretprobe(void)
86	86	{
87	87	setup_ctx();
88	88	attach_bpf(ctx.skel->progs.prog2);
89	89	}
90	90
91		−static void setup_rawtp()
	91	+static void setup_rawtp(void)
92	92	{
93	93	setup_ctx();
94	94	attach_bpf(ctx.skel->progs.prog3);
95	95	}
96	96
97		−static void setup_fentry()
	97	+static void setup_fentry(void)
98	98	{
99	99	setup_ctx();
100	100	attach_bpf(ctx.skel->progs.prog4);
101	101	}
102	102
103		−static void setup_fexit()
	103	+static void setup_fexit(void)
104	104	{
105	105	setup_ctx();
106	106	attach_bpf(ctx.skel->progs.prog5);

tools/testing/selftests/bpf/benchs/bench_ringbufs.c+7−7

@@ -88,12 +88,12 @@ const struct argp bench_ringbufs_argp = { · 87 unmodified lines
88	88
89	89	static struct counter buf_hits;
90	90
91		−static inline void bufs_trigger_batch()
	91	+static inline void bufs_trigger_batch(void)
92	92	{
93	93	(void)syscall(__NR_getpgid);
94	94	}
95	95
96		−static void bufs_validate()
	96	+static void bufs_validate(void)
97	97	{
98	98	if (env.consumer_cnt != 1) {
99	99	fprintf(stderr, "rb-libbpf benchmark doesn't support multi-consumer!\n");
@@ -132,7 +132,7 @@ static void ringbuf_libbpf_measure(struct bench_res *res) · 32 unmodified lines
132	132	res->drops = atomic_swap(&ctx->skel->bss->dropped, 0);
133	133	}
134	134
135		−static struct ringbuf_bench *ringbuf_setup_skeleton()
	135	+static struct ringbuf_bench *ringbuf_setup_skeleton(void)
136	136	{
137	137	struct ringbuf_bench *skel;
138	138
@@ -167,7 +167,7 @@ static int buf_process_sample(void ctx, void data, size_t len) · 28 unmodified lines
167	167	return 0;
168	168	}
169	169
170		−static void ringbuf_libbpf_setup()
	170	+static void ringbuf_libbpf_setup(void)
171	171	{
172	172	struct ringbuf_libbpf_ctx *ctx = &ringbuf_libbpf_ctx;
173	173	struct bpf_link *link;
@@ -223,7 +223,7 @@ static void ringbuf_custom_measure(struct bench_res *res) · 49 unmodified lines
223	223	res->drops = atomic_swap(&ctx->skel->bss->dropped, 0);
224	224	}
225	225
226		−static void ringbuf_custom_setup()
	226	+static void ringbuf_custom_setup(void)
227	227	{
228	228	struct ringbuf_custom_ctx *ctx = &ringbuf_custom_ctx;
229	229	const size_t page_size = getpagesize();
@@ -352,7 +352,7 @@ static void perfbuf_measure(struct bench_res *res) · 122 unmodified lines
352	352	res->drops = atomic_swap(&ctx->skel->bss->dropped, 0);
353	353	}
354	354
355		−static struct perfbuf_bench *perfbuf_setup_skeleton()
	355	+static struct perfbuf_bench *perfbuf_setup_skeleton(void)
356	356	{
357	357	struct perfbuf_bench *skel;
358	358
@@ -390,7 +390,7 @@ perfbuf_process_sample_raw(void *input_ctx, int cpu, · 31 unmodified lines
390	390	return LIBBPF_PERF_EVENT_CONT;
391	391	}
392	392
393		−static void perfbuf_libbpf_setup()
	393	+static void perfbuf_libbpf_setup(void)
394	394	{
395	395	struct perfbuf_libbpf_ctx *ctx = &perfbuf_libbpf_ctx;
396	396	struct perf_event_attr attr;

tools/testing/selftests/bpf/benchs/bench_strncmp.c+161−0

@@ -0,0 +1,161 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (C) 2021. Huawei Technologies Co., Ltd */
	3	+#include <argp.h>
	4	+#include "bench.h"
	5	+#include "strncmp_bench.skel.h"
	6	+
	7	+static struct strncmp_ctx {
	8	+ struct strncmp_bench *skel;
	9	+} ctx;
	10	+
	11	+static struct strncmp_args {
	12	+ u32 cmp_str_len;
	13	+} args = {
	14	+ .cmp_str_len = 32,
	15	+};
	16	+
	17	+enum {
	18	+ ARG_CMP_STR_LEN = 5000,
	19	+};
	20	+
	21	+static const struct argp_option opts[] = {
	22	+ { "cmp-str-len", ARG_CMP_STR_LEN, "CMP_STR_LEN", 0,
	23	+ "Set the length of compared string" },
	24	+ {},
	25	+};
	26	+
	27	+static error_t strncmp_parse_arg(int key, char arg, struct argp_state state)
	28	+{
	29	+ switch (key) {
	30	+ case ARG_CMP_STR_LEN:
	31	+ args.cmp_str_len = strtoul(arg, NULL, 10);
	32	+ if (!args.cmp_str_len \|\|
	33	+ args.cmp_str_len >= sizeof(ctx.skel->bss->str)) {
	34	+ fprintf(stderr, "Invalid cmp str len (limit %zu)\n",
	35	+ sizeof(ctx.skel->bss->str));
	36	+ argp_usage(state);
	37	+ }
	38	+ break;
	39	+ default:
	40	+ return ARGP_ERR_UNKNOWN;
	41	+ }
	42	+
	43	+ return 0;
	44	+}
	45	+
	46	+const struct argp bench_strncmp_argp = {
	47	+ .options = opts,
	48	+ .parser = strncmp_parse_arg,
	49	+};
	50	+
	51	+static void strncmp_validate(void)
	52	+{
	53	+ if (env.consumer_cnt != 1) {
	54	+ fprintf(stderr, "strncmp benchmark doesn't support multi-consumer!\n");
	55	+ exit(1);
	56	+ }
	57	+}
	58	+
	59	+static void strncmp_setup(void)
	60	+{
	61	+ int err;
	62	+ char *target;
	63	+ size_t i, sz;
	64	+
	65	+ sz = sizeof(ctx.skel->rodata->target);
	66	+ if (!sz \|\| sz < sizeof(ctx.skel->bss->str)) {
	67	+ fprintf(stderr, "invalid string size (target %zu, src %zu)\n",
	68	+ sz, sizeof(ctx.skel->bss->str));
	69	+ exit(1);
	70	+ }
	71	+
	72	+ setup_libbpf();
	73	+
	74	+ ctx.skel = strncmp_bench__open();
	75	+ if (!ctx.skel) {
	76	+ fprintf(stderr, "failed to open skeleton\n");
	77	+ exit(1);
	78	+ }
	79	+
	80	+ srandom(time(NULL));
	81	+ target = ctx.skel->rodata->target;
	82	+ for (i = 0; i < sz - 1; i++)
	83	+ target[i] = '1' + random() % 9;
	84	+ target[sz - 1] = '\0';
	85	+
	86	+ ctx.skel->rodata->cmp_str_len = args.cmp_str_len;
	87	+
	88	+ memcpy(ctx.skel->bss->str, target, args.cmp_str_len);
	89	+ ctx.skel->bss->str[args.cmp_str_len] = '\0';
	90	+ /* Make bss->str < rodata->target */
	91	+ ctx.skel->bss->str[args.cmp_str_len - 1] -= 1;
	92	+
	93	+ err = strncmp_bench__load(ctx.skel);
	94	+ if (err) {
	95	+ fprintf(stderr, "failed to load skeleton\n");
	96	+ strncmp_bench__destroy(ctx.skel);
	97	+ exit(1);
	98	+ }
	99	+}
	100	+
	101	+static void strncmp_attach_prog(struct bpf_program *prog)
	102	+{
	103	+ struct bpf_link *link;
	104	+
	105	+ link = bpf_program__attach(prog);
	106	+ if (!link) {
	107	+ fprintf(stderr, "failed to attach program!\n");
	108	+ exit(1);
	109	+ }
	110	+}
	111	+
	112	+static void strncmp_no_helper_setup(void)
	113	+{
	114	+ strncmp_setup();
	115	+ strncmp_attach_prog(ctx.skel->progs.strncmp_no_helper);
	116	+}
	117	+
	118	+static void strncmp_helper_setup(void)
	119	+{
	120	+ strncmp_setup();
	121	+ strncmp_attach_prog(ctx.skel->progs.strncmp_helper);
	122	+}
	123	+
	124	+static void strncmp_producer(void ctx)
	125	+{
	126	+ while (true)
	127	+ (void)syscall(__NR_getpgid);
	128	+ return NULL;
	129	+}
	130	+
	131	+static void strncmp_consumer(void ctx)
	132	+{
	133	+ return NULL;
	134	+}
	135	+
	136	+static void strncmp_measure(struct bench_res *res)
	137	+{
	138	+ res->hits = atomic_swap(&ctx.skel->bss->hits, 0);
	139	+}
	140	+
	141	+const struct bench bench_strncmp_no_helper = {
	142	+ .name = "strncmp-no-helper",
	143	+ .validate = strncmp_validate,
	144	+ .setup = strncmp_no_helper_setup,
	145	+ .producer_thread = strncmp_producer,
	146	+ .consumer_thread = strncmp_consumer,
	147	+ .measure = strncmp_measure,
	148	+ .report_progress = hits_drops_report_progress,
	149	+ .report_final = hits_drops_report_final,
	150	+};
	151	+
	152	+const struct bench bench_strncmp_helper = {
	153	+ .name = "strncmp-helper",
	154	+ .validate = strncmp_validate,
	155	+ .setup = strncmp_helper_setup,
	156	+ .producer_thread = strncmp_producer,
	157	+ .consumer_thread = strncmp_consumer,
	158	+ .measure = strncmp_measure,
	159	+ .report_progress = hits_drops_report_progress,
	160	+ .report_final = hits_drops_report_final,
	161	+};

tools/testing/selftests/bpf/benchs/bench_trigger.c+12−12

@@ -11,7 +11,7 @@ static struct trigger_ctx { · 10 unmodified lines
11	11
12	12	static struct counter base_hits;
13	13
14		−static void trigger_validate()
	14	+static void trigger_validate(void)
15	15	{
16	16	if (env.consumer_cnt != 1) {
17	17	fprintf(stderr, "benchmark doesn't support multi-consumer!\n");
@@ -45,7 +45,7 @@ static void trigger_measure(struct bench_res *res) · 27 unmodified lines
45	45	res->hits = atomic_swap(&ctx.skel->bss->hits, 0);
46	46	}
47	47
48		−static void setup_ctx()
	48	+static void setup_ctx(void)
49	49	{
50	50	setup_libbpf();
51	51
@@ -67,37 +67,37 @@ static void attach_bpf(struct bpf_program *prog) · 15 unmodified lines
67	67	}
68	68	}
69	69
70		−static void trigger_tp_setup()
	70	+static void trigger_tp_setup(void)
71	71	{
72	72	setup_ctx();
73	73	attach_bpf(ctx.skel->progs.bench_trigger_tp);
74	74	}
75	75
76		−static void trigger_rawtp_setup()
	76	+static void trigger_rawtp_setup(void)
77	77	{
78	78	setup_ctx();
79	79	attach_bpf(ctx.skel->progs.bench_trigger_raw_tp);
80	80	}
81	81
82		−static void trigger_kprobe_setup()
	82	+static void trigger_kprobe_setup(void)
83	83	{
84	84	setup_ctx();
85	85	attach_bpf(ctx.skel->progs.bench_trigger_kprobe);
86	86	}
87	87
88		−static void trigger_fentry_setup()
	88	+static void trigger_fentry_setup(void)
89	89	{
90	90	setup_ctx();
91	91	attach_bpf(ctx.skel->progs.bench_trigger_fentry);
92	92	}
93	93
94		−static void trigger_fentry_sleep_setup()
	94	+static void trigger_fentry_sleep_setup(void)
95	95	{
96	96	setup_ctx();
97	97	attach_bpf(ctx.skel->progs.bench_trigger_fentry_sleep);
98	98	}
99	99
100		−static void trigger_fmodret_setup()
	100	+static void trigger_fmodret_setup(void)
101	101	{
102	102	setup_ctx();
103	103	attach_bpf(ctx.skel->progs.bench_trigger_fmodret);
@@ -183,22 +183,22 @@ static void usetup(bool use_retprobe, bool use_nop) · 79 unmodified lines
183	183	ctx.skel->links.bench_trigger_uprobe = link;
184	184	}
185	185
186		−static void uprobe_setup_with_nop()
	186	+static void uprobe_setup_with_nop(void)
187	187	{
188	188	usetup(false, true);
189	189	}
190	190
191		−static void uretprobe_setup_with_nop()
	191	+static void uretprobe_setup_with_nop(void)
192	192	{
193	193	usetup(true, true);
194	194	}
195	195
196		−static void uprobe_setup_without_nop()
	196	+static void uprobe_setup_without_nop(void)
197	197	{
198	198	usetup(false, false);
199	199	}
200	200
201		−static void uretprobe_setup_without_nop()
	201	+static void uretprobe_setup_without_nop(void)
202	202	{
203	203	usetup(true, false);
204	204	}

tools/testing/selftests/bpf/benchs/run_bench_strncmp.sh+12−0

@@ -0,0 +1,12 @@
	1	+#!/bin/bash
	2	+# SPDX-License-Identifier: GPL-2.0
	3	+
	4	+source ./benchs/run_common.sh
	5	+
	6	+set -eufo pipefail
	7	+
	8	+for s in 1 8 64 512 2048 4095; do
	9	+ for b in no-helper helper; do
	10	+ summarize ${b}-${s} "$($RUN_BENCH --cmp-str-len=$s strncmp-${b})"
	11	+ done
	12	+done

tools/testing/selftests/bpf/config+2−0

@@ -38,7 +38,9 @@ CONFIG_IPV6_SIT=m · 37 unmodified lines
38	38	CONFIG_BPF_JIT=y
39	39	CONFIG_BPF_LSM=y
40	40	CONFIG_SECURITY=y
	41	+CONFIG_RC_CORE=y
41	42	CONFIG_LIRC=y
	43	+CONFIG_BPF_LIRC_MODE2=y
42	44	CONFIG_IMA=y
43	45	CONFIG_SECURITYFS=y
44	46	CONFIG_IMA_WRITE_POLICY=y

tools/testing/selftests/bpf/prog_tests/align.c+103−88

@@ -39,13 +39,13 @@ static struct bpf_align_test tests[] = { · 38 unmodified lines
39	39	},
40	40	.prog_type = BPF_PROG_TYPE_SCHED_CLS,
41	41	.matches = {
42		− {1, "R1=ctx(id=0,off=0,imm=0)"},
43		− {1, "R10=fp0"},
44		− {1, "R3_w=inv2"},
45		− {2, "R3_w=inv4"},
46		− {3, "R3_w=inv8"},
47		− {4, "R3_w=inv16"},
48		− {5, "R3_w=inv32"},
	42	+ {0, "R1=ctx(id=0,off=0,imm=0)"},
	43	+ {0, "R10=fp0"},
	44	+ {0, "R3_w=inv2"},
	45	+ {1, "R3_w=inv4"},
	46	+ {2, "R3_w=inv8"},
	47	+ {3, "R3_w=inv16"},
	48	+ {4, "R3_w=inv32"},
49	49	},
50	50	},
51	51	{
@@ -67,19 +67,19 @@ static struct bpf_align_test tests[] = { · 15 unmodified lines
67	67	},
68	68	.prog_type = BPF_PROG_TYPE_SCHED_CLS,
69	69	.matches = {
70		− {1, "R1=ctx(id=0,off=0,imm=0)"},
71		− {1, "R10=fp0"},
72		− {1, "R3_w=inv1"},
73		− {2, "R3_w=inv2"},
74		− {3, "R3_w=inv4"},
75		− {4, "R3_w=inv8"},
76		− {5, "R3_w=inv16"},
77		− {6, "R3_w=inv1"},
78		− {7, "R4_w=inv32"},
79		− {8, "R4_w=inv16"},
80		− {9, "R4_w=inv8"},
81		− {10, "R4_w=inv4"},
82		− {11, "R4_w=inv2"},
	70	+ {0, "R1=ctx(id=0,off=0,imm=0)"},
	71	+ {0, "R10=fp0"},
	72	+ {0, "R3_w=inv1"},
	73	+ {1, "R3_w=inv2"},
	74	+ {2, "R3_w=inv4"},
	75	+ {3, "R3_w=inv8"},
	76	+ {4, "R3_w=inv16"},
	77	+ {5, "R3_w=inv1"},
	78	+ {6, "R4_w=inv32"},
	79	+ {7, "R4_w=inv16"},
	80	+ {8, "R4_w=inv8"},
	81	+ {9, "R4_w=inv4"},
	82	+ {10, "R4_w=inv2"},
83	83	},
84	84	},
85	85	{
@@ -96,14 +96,14 @@ static struct bpf_align_test tests[] = { · 10 unmodified lines
96	96	},
97	97	.prog_type = BPF_PROG_TYPE_SCHED_CLS,
98	98	.matches = {
99		− {1, "R1=ctx(id=0,off=0,imm=0)"},
100		− {1, "R10=fp0"},
101		− {1, "R3_w=inv4"},
102		− {2, "R3_w=inv8"},
103		− {3, "R3_w=inv10"},
104		− {4, "R4_w=inv8"},
105		− {5, "R4_w=inv12"},
106		− {6, "R4_w=inv14"},
	99	+ {0, "R1=ctx(id=0,off=0,imm=0)"},
	100	+ {0, "R10=fp0"},
	101	+ {0, "R3_w=inv4"},
	102	+ {1, "R3_w=inv8"},
	103	+ {2, "R3_w=inv10"},
	104	+ {3, "R4_w=inv8"},
	105	+ {4, "R4_w=inv12"},
	106	+ {5, "R4_w=inv14"},
107	107	},
108	108	},
109	109	{
@@ -118,12 +118,12 @@ static struct bpf_align_test tests[] = { · 8 unmodified lines
118	118	},
119	119	.prog_type = BPF_PROG_TYPE_SCHED_CLS,
120	120	.matches = {
121		− {1, "R1=ctx(id=0,off=0,imm=0)"},
122		− {1, "R10=fp0"},
	121	+ {0, "R1=ctx(id=0,off=0,imm=0)"},
	122	+ {0, "R10=fp0"},
	123	+ {0, "R3_w=inv7"},
123	124	{1, "R3_w=inv7"},
124		− {2, "R3_w=inv7"},
125		− {3, "R3_w=inv14"},
126		− {4, "R3_w=inv56"},
	125	+ {2, "R3_w=inv14"},
	126	+ {3, "R3_w=inv56"},
127	127	},
128	128	},
129	129
@@ -161,19 +161,19 @@ static struct bpf_align_test tests[] = { · 31 unmodified lines
161	161	},
162	162	.prog_type = BPF_PROG_TYPE_SCHED_CLS,
163	163	.matches = {
164		− {7, "R0_w=pkt(id=0,off=8,r=8,imm=0)"},
165		− {7, "R3_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
166		− {8, "R3_w=inv(id=0,umax_value=510,var_off=(0x0; 0x1fe))"},
167		− {9, "R3_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
168		− {10, "R3_w=inv(id=0,umax_value=2040,var_off=(0x0; 0x7f8))"},
169		− {11, "R3_w=inv(id=0,umax_value=4080,var_off=(0x0; 0xff0))"},
170		− {18, "R3=pkt_end(id=0,off=0,imm=0)"},
171		− {18, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
172		− {19, "R4_w=inv(id=0,umax_value=8160,var_off=(0x0; 0x1fe0))"},
173		− {20, "R4_w=inv(id=0,umax_value=4080,var_off=(0x0; 0xff0))"},
174		− {21, "R4_w=inv(id=0,umax_value=2040,var_off=(0x0; 0x7f8))"},
175		− {22, "R4_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
176		− {23, "R4_w=inv(id=0,umax_value=510,var_off=(0x0; 0x1fe))"},
	164	+ {6, "R0_w=pkt(id=0,off=8,r=8,imm=0)"},
	165	+ {6, "R3_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
	166	+ {7, "R3_w=inv(id=0,umax_value=510,var_off=(0x0; 0x1fe))"},
	167	+ {8, "R3_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
	168	+ {9, "R3_w=inv(id=0,umax_value=2040,var_off=(0x0; 0x7f8))"},
	169	+ {10, "R3_w=inv(id=0,umax_value=4080,var_off=(0x0; 0xff0))"},
	170	+ {12, "R3_w=pkt_end(id=0,off=0,imm=0)"},
	171	+ {17, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
	172	+ {18, "R4_w=inv(id=0,umax_value=8160,var_off=(0x0; 0x1fe0))"},
	173	+ {19, "R4_w=inv(id=0,umax_value=4080,var_off=(0x0; 0xff0))"},
	174	+ {20, "R4_w=inv(id=0,umax_value=2040,var_off=(0x0; 0x7f8))"},
	175	+ {21, "R4_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
	176	+ {22, "R4_w=inv(id=0,umax_value=510,var_off=(0x0; 0x1fe))"},
177	177	},
178	178	},
179	179	{
@@ -194,16 +194,16 @@ static struct bpf_align_test tests[] = { · 14 unmodified lines
194	194	},
195	195	.prog_type = BPF_PROG_TYPE_SCHED_CLS,
196	196	.matches = {
197		− {7, "R3_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
198		− {8, "R4_w=inv(id=1,umax_value=255,var_off=(0x0; 0xff))"},
199		− {9, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
200		− {10, "R4_w=inv(id=1,umax_value=255,var_off=(0x0; 0xff))"},
201		− {11, "R4_w=inv(id=0,umax_value=510,var_off=(0x0; 0x1fe))"},
202		− {12, "R4_w=inv(id=1,umax_value=255,var_off=(0x0; 0xff))"},
203		− {13, "R4_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
204		− {14, "R4_w=inv(id=1,umax_value=255,var_off=(0x0; 0xff))"},
205		− {15, "R4_w=inv(id=0,umax_value=2040,var_off=(0x0; 0x7f8))"},
206		− {16, "R4_w=inv(id=0,umax_value=4080,var_off=(0x0; 0xff0))"},
	197	+ {6, "R3_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
	198	+ {7, "R4_w=inv(id=1,umax_value=255,var_off=(0x0; 0xff))"},
	199	+ {8, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
	200	+ {9, "R4_w=inv(id=1,umax_value=255,var_off=(0x0; 0xff))"},
	201	+ {10, "R4_w=inv(id=0,umax_value=510,var_off=(0x0; 0x1fe))"},
	202	+ {11, "R4_w=inv(id=1,umax_value=255,var_off=(0x0; 0xff))"},
	203	+ {12, "R4_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
	204	+ {13, "R4_w=inv(id=1,umax_value=255,var_off=(0x0; 0xff))"},
	205	+ {14, "R4_w=inv(id=0,umax_value=2040,var_off=(0x0; 0x7f8))"},
	206	+ {15, "R4_w=inv(id=0,umax_value=4080,var_off=(0x0; 0xff0))"},
207	207	},
208	208	},
209	209	{
@@ -234,14 +234,14 @@ static struct bpf_align_test tests[] = { · 24 unmodified lines
234	234	},
235	235	.prog_type = BPF_PROG_TYPE_SCHED_CLS,
236	236	.matches = {
237		− {4, "R5_w=pkt(id=0,off=0,r=0,imm=0)"},
238		− {5, "R5_w=pkt(id=0,off=14,r=0,imm=0)"},
239		− {6, "R4_w=pkt(id=0,off=14,r=0,imm=0)"},
240		− {10, "R2=pkt(id=0,off=0,r=18,imm=0)"},
	237	+ {2, "R5_w=pkt(id=0,off=0,r=0,imm=0)"},
	238	+ {4, "R5_w=pkt(id=0,off=14,r=0,imm=0)"},
	239	+ {5, "R4_w=pkt(id=0,off=14,r=0,imm=0)"},
	240	+ {9, "R2=pkt(id=0,off=0,r=18,imm=0)"},
241	241	{10, "R5=pkt(id=0,off=14,r=18,imm=0)"},
242	242	{10, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
	243	+ {13, "R4_w=inv(id=0,umax_value=65535,var_off=(0x0; 0xffff))"},
243	244	{14, "R4_w=inv(id=0,umax_value=65535,var_off=(0x0; 0xffff))"},
244		− {15, "R4_w=inv(id=0,umax_value=65535,var_off=(0x0; 0xffff))"},
245	245	},
246	246	},
247	247	{
@@ -296,8 +296,8 @@ static struct bpf_align_test tests[] = { · 48 unmodified lines
296	296	/* Calculated offset in R6 has unknown value, but known
297	297	* alignment of 4.
298	298	*/
299		− {8, "R2_w=pkt(id=0,off=0,r=8,imm=0)"},
300		− {8, "R6_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
	299	+ {6, "R2_w=pkt(id=0,off=0,r=8,imm=0)"},
	300	+ {7, "R6_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
301	301	/* Offset is added to packet pointer R5, resulting in
302	302	* known fixed offset, and variable offset from R6.
303	303	*/
@@ -313,11 +313,11 @@ static struct bpf_align_test tests[] = { · 9 unmodified lines
313	313	/* Variable offset is added to R5 packet pointer,
314	314	* resulting in auxiliary alignment of 4.
315	315	*/
316		− {18, "R5_w=pkt(id=2,off=0,r=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
	316	+ {17, "R5_w=pkt(id=2,off=0,r=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
317	317	/* Constant offset is added to R5, resulting in
318	318	* reg->off of 14.
319	319	*/
320		− {19, "R5_w=pkt(id=2,off=14,r=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
	320	+ {18, "R5_w=pkt(id=2,off=14,r=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
321	321	/* At the time the word size load is performed from R5,
322	322	* its total fixed offset is NET_IP_ALIGN + reg->off
323	323	* (14) which is 16. Then the variable offset is 4-byte
@@ -329,18 +329,18 @@ static struct bpf_align_test tests[] = { · 5 unmodified lines
329	329	/* Constant offset is added to R5 packet pointer,
330	330	* resulting in reg->off value of 14.
331	331	*/
332		− {26, "R5_w=pkt(id=0,off=14,r=8"},
	332	+ {25, "R5_w=pkt(id=0,off=14,r=8"},
333	333	/* Variable offset is added to R5, resulting in a
334	334	* variable offset of (4n).
335	335	*/
336		− {27, "R5_w=pkt(id=3,off=14,r=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
	336	+ {26, "R5_w=pkt(id=3,off=14,r=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
337	337	/* Constant is added to R5 again, setting reg->off to 18. */
338		− {28, "R5_w=pkt(id=3,off=18,r=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
	338	+ {27, "R5_w=pkt(id=3,off=18,r=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
339	339	/* And once more we add a variable; resulting var_off
340	340	* is still (4n), fixed offset is not changed.
341	341	* Also, we create a new reg->id.
342	342	*/
343		− {29, "R5_w=pkt(id=4,off=18,r=0,umax_value=2040,var_off=(0x0; 0x7fc)"},
	343	+ {28, "R5_w=pkt(id=4,off=18,r=0,umax_value=2040,var_off=(0x0; 0x7fc)"},
344	344	/* At the time the word size load is performed from R5,
345	345	* its total fixed offset is NET_IP_ALIGN + reg->off (18)
346	346	* which is 20. Then the variable offset is (4n), so
@@ -386,13 +386,13 @@ static struct bpf_align_test tests[] = { · 39 unmodified lines
386	386	/* Calculated offset in R6 has unknown value, but known
387	387	* alignment of 4.
388	388	*/
389		− {8, "R2_w=pkt(id=0,off=0,r=8,imm=0)"},
390		− {8, "R6_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
	389	+ {6, "R2_w=pkt(id=0,off=0,r=8,imm=0)"},
	390	+ {7, "R6_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
391	391	/* Adding 14 makes R6 be (4n+2) */
392		− {9, "R6_w=inv(id=0,umin_value=14,umax_value=1034,var_off=(0x2; 0x7fc))"},
	392	+ {8, "R6_w=inv(id=0,umin_value=14,umax_value=1034,var_off=(0x2; 0x7fc))"},
393	393	/* Packet pointer has (4n+2) offset */
394	394	{11, "R5_w=pkt(id=1,off=0,r=0,umin_value=14,umax_value=1034,var_off=(0x2; 0x7fc)"},
395		− {13, "R4=pkt(id=1,off=4,r=0,umin_value=14,umax_value=1034,var_off=(0x2; 0x7fc)"},
	395	+ {12, "R4=pkt(id=1,off=4,r=0,umin_value=14,umax_value=1034,var_off=(0x2; 0x7fc)"},
396	396	/* At the time the word size load is performed from R5,
397	397	* its total fixed offset is NET_IP_ALIGN + reg->off (0)
398	398	* which is 2. Then the variable offset is (4n+2), so
@@ -403,12 +403,12 @@ static struct bpf_align_test tests[] = { · 4 unmodified lines
403	403	/* Newly read value in R6 was shifted left by 2, so has
404	404	* known alignment of 4.
405	405	*/
406		− {18, "R6_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
	406	+ {17, "R6_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
407	407	/* Added (4n) to packet pointer's (4n+2) var_off, giving
408	408	* another (4n+2).
409	409	*/
410	410	{19, "R5_w=pkt(id=2,off=0,r=0,umin_value=14,umax_value=2054,var_off=(0x2; 0xffc)"},
411		− {21, "R4=pkt(id=2,off=4,r=0,umin_value=14,umax_value=2054,var_off=(0x2; 0xffc)"},
	411	+ {20, "R4=pkt(id=2,off=4,r=0,umin_value=14,umax_value=2054,var_off=(0x2; 0xffc)"},
412	412	/* At the time the word size load is performed from R5,
413	413	* its total fixed offset is NET_IP_ALIGN + reg->off (0)
414	414	* which is 2. Then the variable offset is (4n+2), so
@@ -448,18 +448,18 @@ static struct bpf_align_test tests[] = { · 33 unmodified lines
448	448	.prog_type = BPF_PROG_TYPE_SCHED_CLS,
449	449	.result = REJECT,
450	450	.matches = {
451		− {4, "R5_w=pkt_end(id=0,off=0,imm=0)"},
	451	+ {3, "R5_w=pkt_end(id=0,off=0,imm=0)"},
452	452	/* (ptr - ptr) << 2 == unknown, (4n) */
453		− {6, "R5_w=inv(id=0,smax_value=9223372036854775804,umax_value=18446744073709551612,var_off=(0x0; 0xfffffffffffffffc)"},
	453	+ {5, "R5_w=inv(id=0,smax_value=9223372036854775804,umax_value=18446744073709551612,var_off=(0x0; 0xfffffffffffffffc)"},
454	454	/* (4n) + 14 == (4n+2). We blow our bounds, because
455	455	* the add could overflow.
456	456	*/
457		− {7, "R5_w=inv(id=0,smin_value=-9223372036854775806,smax_value=9223372036854775806,umin_value=2,umax_value=18446744073709551614,var_off=(0x2; 0xfffffffffffffffc)"},
	457	+ {6, "R5_w=inv(id=0,smin_value=-9223372036854775806,smax_value=9223372036854775806,umin_value=2,umax_value=18446744073709551614,var_off=(0x2; 0xfffffffffffffffc)"},
458	458	/* Checked s>=0 */
459	459	{9, "R5=inv(id=0,umin_value=2,umax_value=9223372036854775806,var_off=(0x2; 0x7ffffffffffffffc)"},
460	460	/* packet pointer + nonnegative (4n+2) */
461	461	{11, "R6_w=pkt(id=1,off=0,r=0,umin_value=2,umax_value=9223372036854775806,var_off=(0x2; 0x7ffffffffffffffc)"},
462		− {13, "R4_w=pkt(id=1,off=4,r=0,umin_value=2,umax_value=9223372036854775806,var_off=(0x2; 0x7ffffffffffffffc)"},
	462	+ {12, "R4_w=pkt(id=1,off=4,r=0,umin_value=2,umax_value=9223372036854775806,var_off=(0x2; 0x7ffffffffffffffc)"},
463	463	/* NET_IP_ALIGN + (4n+2) == (4n), alignment is fine.
464	464	* We checked the bounds, but it might have been able
465	465	* to overflow if the packet pointer started in the
@@ -502,14 +502,14 @@ static struct bpf_align_test tests[] = { · 36 unmodified lines
502	502	/* Calculated offset in R6 has unknown value, but known
503	503	* alignment of 4.
504	504	*/
505		− {7, "R2_w=pkt(id=0,off=0,r=8,imm=0)"},
506		− {9, "R6_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
	505	+ {6, "R2_w=pkt(id=0,off=0,r=8,imm=0)"},
	506	+ {8, "R6_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
507	507	/* Adding 14 makes R6 be (4n+2) */
508		− {10, "R6_w=inv(id=0,umin_value=14,umax_value=1034,var_off=(0x2; 0x7fc))"},
	508	+ {9, "R6_w=inv(id=0,umin_value=14,umax_value=1034,var_off=(0x2; 0x7fc))"},
509	509	/* New unknown value in R7 is (4n) */
510		− {11, "R7_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
	510	+ {10, "R7_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
511	511	/* Subtracting it from R6 blows our unsigned bounds */
512		− {12, "R6=inv(id=0,smin_value=-1006,smax_value=1034,umin_value=2,umax_value=18446744073709551614,var_off=(0x2; 0xfffffffffffffffc)"},
	512	+ {11, "R6=inv(id=0,smin_value=-1006,smax_value=1034,umin_value=2,umax_value=18446744073709551614,var_off=(0x2; 0xfffffffffffffffc)"},
513	513	/* Checked s>= 0 */
514	514	{14, "R6=inv(id=0,umin_value=2,umax_value=1034,var_off=(0x2; 0x7fc))"},
515	515	/* At the time the word size load is performed from R5,
@@ -556,14 +556,14 @@ static struct bpf_align_test tests[] = { · 40 unmodified lines
556	556	/* Calculated offset in R6 has unknown value, but known
557	557	* alignment of 4.
558	558	*/
559		− {7, "R2_w=pkt(id=0,off=0,r=8,imm=0)"},
560		− {10, "R6_w=inv(id=0,umax_value=60,var_off=(0x0; 0x3c))"},
	559	+ {6, "R2_w=pkt(id=0,off=0,r=8,imm=0)"},
	560	+ {9, "R6_w=inv(id=0,umax_value=60,var_off=(0x0; 0x3c))"},
561	561	/* Adding 14 makes R6 be (4n+2) */
562		− {11, "R6_w=inv(id=0,umin_value=14,umax_value=74,var_off=(0x2; 0x7c))"},
	562	+ {10, "R6_w=inv(id=0,umin_value=14,umax_value=74,var_off=(0x2; 0x7c))"},
563	563	/* Subtracting from packet pointer overflows ubounds */
564	564	{13, "R5_w=pkt(id=2,off=0,r=8,umin_value=18446744073709551542,umax_value=18446744073709551602,var_off=(0xffffffffffffff82; 0x7c)"},
565	565	/* New unknown value in R7 is (4n), >= 76 */
566		− {15, "R7_w=inv(id=0,umin_value=76,umax_value=1096,var_off=(0x0; 0x7fc))"},
	566	+ {14, "R7_w=inv(id=0,umin_value=76,umax_value=1096,var_off=(0x0; 0x7fc))"},
567	567	/* Adding it to packet pointer gives nice bounds again */
568	568	{16, "R5_w=pkt(id=3,off=0,r=0,umin_value=2,umax_value=1082,var_off=(0x2; 0xfffffffc)"},
569	569	/* At the time the word size load is performed from R5,
@@ -625,12 +625,15 @@ static int do_test_single(struct bpf_align_test *test) · 55 unmodified lines
625	625	line_ptr = strtok(bpf_vlog_copy, "\n");
626	626	for (i = 0; i < MAX_MATCHES; i++) {
627	627	struct bpf_reg_match m = test->matches[i];
	628	+ int tmp;
628	629
629	630	if (!m.match)
630	631	break;
631	632	while (line_ptr) {
632	633	cur_line = -1;
633	634	sscanf(line_ptr, "%u: ", &cur_line);
	635	+ if (cur_line == -1)
	636	+ sscanf(line_ptr, "from %u to %u: ", &tmp, &cur_line);
634	637	if (cur_line == m.line)
635	638	break;
636	639	line_ptr = strtok(NULL, "\n");
@@ -642,7 +645,19 @@ static int do_test_single(struct bpf_align_test *test) · 5 unmodified lines
642	645	printf("%s", bpf_vlog);
643	646	break;
644	647	}
	648	+ /* Check the next line as well in case the previous line
	649	+ * did not have a corresponding bpf insn. Example:
	650	+ * func#0 @0
	651	+ * 0: R1=ctx(id=0,off=0,imm=0) R10=fp0
	652	+ * 0: (b7) r3 = 2 ; R3_w=inv2
	653	+ */
645	654	if (!strstr(line_ptr, m.match)) {
	655	+ cur_line = -1;
	656	+ line_ptr = strtok(NULL, "\n");
	657	+ sscanf(line_ptr, "%u: ", &cur_line);
	658	+ }
	659	+ if (cur_line != m.line \|\| !line_ptr \|\|
	660	+ !strstr(line_ptr, m.match)) {
646	661	printf("Failed to find match %u: %s\n",
647	662	m.line, m.match);
648	663	ret = 1;

tools/testing/selftests/bpf/prog_tests/bpf_obj_id.c+2−2

@@ -65,8 +65,8 @@ void serial_test_bpf_obj_id(void) · 64 unmodified lines
65	65	if (CHECK_FAIL(err))
66	66	goto done;
67	67
68		− prog = bpf_object__find_program_by_title(objs[i],
69		− "raw_tp/sys_enter");
	68	+ prog = bpf_object__find_program_by_name(objs[i],
	69	+ "test_obj_id");
70	70	if (CHECK_FAIL(!prog))
71	71	goto done;
72	72	links[i] = bpf_program__attach(prog);

tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c+3−4

@@ -217,7 +217,7 @@ static bool found; · 216 unmodified lines
217	217	static int libbpf_debug_print(enum libbpf_print_level level,
218	218	const char *format, va_list args)
219	219	{
220		− const char *log_buf;
	220	+ const char prog_name, log_buf;
221	221
222	222	if (level != LIBBPF_WARN \|\|
223	223	!strstr(format, "-- BEGIN PROG LOAD LOG --")) {
@@ -225,15 +225,14 @@ static int libbpf_debug_print(enum libbpf_print_level level, · 1 unmodified line
225	225	return 0;
226	226	}
227	227
228		− /* skip prog_name */
229		− va_arg(args, char *);
	228	+ prog_name = va_arg(args, char *);
230	229	log_buf = va_arg(args, char *);
231	230	if (!log_buf)
232	231	goto out;
233	232	if (err_str && strstr(log_buf, err_str) != NULL)
234	233	found = true;
235	234	out:
236		− printf(format, log_buf);
	235	+ printf(format, prog_name, log_buf);
237	236	return 0;
238	237	}
239	238

tools/testing/selftests/bpf/prog_tests/btf.c+0−1

@@ -22,7 +22,6 @@ · 21 unmodified lines
22	22	#include <bpf/libbpf.h>
23	23	#include <bpf/btf.h>
24	24
25		−#include "bpf_rlimit.h"
26	25	#include "bpf_util.h"
27	26	#include "../test_btf.h"
28	27	#include "test_progs.h"

tools/testing/selftests/bpf/prog_tests/connect_force_port.c+9−9

@@ -67,9 +67,9 @@ static int run_test(int cgroup_fd, int server_fd, int family, int type) · 66 unmodified lines
67	67	goto close_bpf_object;
68	68	}
69	69
70		− prog = bpf_object__find_program_by_title(obj, v4 ?
71		− "cgroup/connect4" :
72		− "cgroup/connect6");
	70	+ prog = bpf_object__find_program_by_name(obj, v4 ?
	71	+ "connect4" :
	72	+ "connect6");
73	73	if (CHECK(!prog, "find_prog", "connect prog not found\n")) {
74	74	err = -EIO;
75	75	goto close_bpf_object;
@@ -83,9 +83,9 @@ static int run_test(int cgroup_fd, int server_fd, int family, int type) · 7 unmodified lines
83	83	goto close_bpf_object;
84	84	}
85	85
86		− prog = bpf_object__find_program_by_title(obj, v4 ?
87		− "cgroup/getpeername4" :
88		− "cgroup/getpeername6");
	86	+ prog = bpf_object__find_program_by_name(obj, v4 ?
	87	+ "getpeername4" :
	88	+ "getpeername6");
89	89	if (CHECK(!prog, "find_prog", "getpeername prog not found\n")) {
90	90	err = -EIO;
91	91	goto close_bpf_object;
@@ -99,9 +99,9 @@ static int run_test(int cgroup_fd, int server_fd, int family, int type) · 7 unmodified lines
99	99	goto close_bpf_object;
100	100	}
101	101
102		− prog = bpf_object__find_program_by_title(obj, v4 ?
103		− "cgroup/getsockname4" :
104		− "cgroup/getsockname6");
	102	+ prog = bpf_object__find_program_by_name(obj, v4 ?
	103	+ "getsockname4" :
	104	+ "getsockname6");
105	105	if (CHECK(!prog, "find_prog", "getsockname prog not found\n")) {
106	106	err = -EIO;
107	107	goto close_bpf_object;

tools/testing/selftests/bpf/prog_tests/core_reloc.c+53−26

@@ -10,7 +10,7 @@ static int duration = 0; · 9 unmodified lines
10	10
11	11	#define STRUCT_TO_CHAR_PTR(struct_name) (const char *)&(struct struct_name)
12	12
13		−#define MODULES_CASE(name, sec_name, tp_name) { \
	13	+#define MODULES_CASE(name, pg_name, tp_name) { \
14	14	.case_name = name, \
15	15	.bpf_obj_file = "test_core_reloc_module.o", \
16	16	.btf_src_file = NULL, /* find in kernel module BTFs */ \
@@ -28,7 +28,7 @@ static int duration = 0; · 11 unmodified lines
28	28	.comm_len = sizeof("test_progs"), \
29	29	}, \
30	30	.output_len = sizeof(struct core_reloc_module_output), \
31		− .prog_sec_name = sec_name, \
	31	+ .prog_name = pg_name, \
32	32	.raw_tp_name = tp_name, \
33	33	.trigger = __trigger_module_test_read, \
34	34	.needs_testmod = true, \
@@ -43,7 +43,9 @@ static int duration = 0; · 8 unmodified lines
43	43	#define FLAVORS_CASE_COMMON(name) \
44	44	.case_name = #name, \
45	45	.bpf_obj_file = "test_core_reloc_flavors.o", \
46		− .btf_src_file = "btf__core_reloc_" #name ".o" \
	46	+ .btf_src_file = "btf__core_reloc_" #name ".o", \
	47	+ .raw_tp_name = "sys_enter", \
	48	+ .prog_name = "test_core_flavors" \
47	49
48	50	#define FLAVORS_CASE(name) { \
49	51	FLAVORS_CASE_COMMON(name), \
@@ -66,7 +68,9 @@ static int duration = 0; · 16 unmodified lines
66	68	#define NESTING_CASE_COMMON(name) \
67	69	.case_name = #name, \
68	70	.bpf_obj_file = "test_core_reloc_nesting.o", \
69		− .btf_src_file = "btf__core_reloc_" #name ".o"
	71	+ .btf_src_file = "btf__core_reloc_" #name ".o", \
	72	+ .raw_tp_name = "sys_enter", \
	73	+ .prog_name = "test_core_nesting" \
70	74
71	75	#define NESTING_CASE(name) { \
72	76	NESTING_CASE_COMMON(name), \
@@ -91,7 +95,9 @@ static int duration = 0; · 18 unmodified lines
91	95	#define ARRAYS_CASE_COMMON(name) \
92	96	.case_name = #name, \
93	97	.bpf_obj_file = "test_core_reloc_arrays.o", \
94		− .btf_src_file = "btf__core_reloc_" #name ".o"
	98	+ .btf_src_file = "btf__core_reloc_" #name ".o", \
	99	+ .raw_tp_name = "sys_enter", \
	100	+ .prog_name = "test_core_arrays" \
95	101
96	102	#define ARRAYS_CASE(name) { \
97	103	ARRAYS_CASE_COMMON(name), \
@@ -123,7 +129,9 @@ static int duration = 0; · 25 unmodified lines
123	129	#define PRIMITIVES_CASE_COMMON(name) \
124	130	.case_name = #name, \
125	131	.bpf_obj_file = "test_core_reloc_primitives.o", \
126		− .btf_src_file = "btf__core_reloc_" #name ".o"
	132	+ .btf_src_file = "btf__core_reloc_" #name ".o", \
	133	+ .raw_tp_name = "sys_enter", \
	134	+ .prog_name = "test_core_primitives" \
127	135
128	136	#define PRIMITIVES_CASE(name) { \
129	137	PRIMITIVES_CASE_COMMON(name), \
@@ -158,6 +166,8 @@ static int duration = 0; · 28 unmodified lines
158	166	.e = 5, .f = 6, .g = 7, .h = 8, \
159	167	}, \
160	168	.output_len = sizeof(struct core_reloc_mods_output), \
	169	+ .raw_tp_name = "sys_enter", \
	170	+ .prog_name = "test_core_mods", \
161	171	}
162	172
163	173	#define PTR_AS_ARR_CASE(name) { \
@@ -174,6 +184,8 @@ static int duration = 0; · 10 unmodified lines
174	184	.a = 3, \
175	185	}, \
176	186	.output_len = sizeof(struct core_reloc_ptr_as_arr), \
	187	+ .raw_tp_name = "sys_enter", \
	188	+ .prog_name = "test_core_ptr_as_arr", \
177	189	}
178	190
179	191	#define INTS_DATA(struct_name) STRUCT_TO_CHAR_PTR(struct_name) { \
@@ -190,7 +202,9 @@ static int duration = 0; · 10 unmodified lines
190	202	#define INTS_CASE_COMMON(name) \
191	203	.case_name = #name, \
192	204	.bpf_obj_file = "test_core_reloc_ints.o", \
193		− .btf_src_file = "btf__core_reloc_" #name ".o"
	205	+ .btf_src_file = "btf__core_reloc_" #name ".o", \
	206	+ .raw_tp_name = "sys_enter", \
	207	+ .prog_name = "test_core_ints"
194	208
195	209	#define INTS_CASE(name) { \
196	210	INTS_CASE_COMMON(name), \
@@ -208,7 +222,9 @@ static int duration = 0; · 11 unmodified lines
208	222	#define FIELD_EXISTS_CASE_COMMON(name) \
209	223	.case_name = #name, \
210	224	.bpf_obj_file = "test_core_reloc_existence.o", \
211		− .btf_src_file = "btf__core_reloc_" #name ".o" \
	225	+ .btf_src_file = "btf__core_reloc_" #name ".o", \
	226	+ .raw_tp_name = "sys_enter", \
	227	+ .prog_name = "test_core_existence"
212	228
213	229	#define BITFIELDS_CASE_COMMON(objfile, test_name_prefix, name) \
214	230	.case_name = test_name_prefix#name, \
@@ -223,6 +239,8 @@ static int duration = 0; · 8 unmodified lines
223	239	.output = STRUCT_TO_CHAR_PTR(core_reloc_bitfields_output) \
224	240	__VA_ARGS__, \
225	241	.output_len = sizeof(struct core_reloc_bitfields_output), \
	242	+ .raw_tp_name = "sys_enter", \
	243	+ .prog_name = "test_core_bitfields", \
226	244	}, { \
227	245	BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_direct.o", \
228	246	"direct:", name), \
@@ -231,7 +249,7 @@ static int duration = 0; · 2 unmodified lines
231	249	.output = STRUCT_TO_CHAR_PTR(core_reloc_bitfields_output) \
232	250	__VA_ARGS__, \
233	251	.output_len = sizeof(struct core_reloc_bitfields_output), \
234		− .prog_sec_name = "tp_btf/sys_enter", \
	252	+ .prog_name = "test_core_bitfields_direct", \
235	253	}
236	254
237	255
@@ -239,17 +257,21 @@ static int duration = 0; · 1 unmodified line
239	257	BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_probed.o", \
240	258	"probed:", name), \
241	259	.fails = true, \
	260	+ .raw_tp_name = "sys_enter", \
	261	+ .prog_name = "test_core_bitfields", \
242	262	}, { \
243	263	BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_direct.o", \
244	264	"direct:", name), \
245		− .prog_sec_name = "tp_btf/sys_enter", \
246	265	.fails = true, \
	266	+ .prog_name = "test_core_bitfields_direct", \
247	267	}
248	268
249	269	#define SIZE_CASE_COMMON(name) \
250	270	.case_name = #name, \
251	271	.bpf_obj_file = "test_core_reloc_size.o", \
252		− .btf_src_file = "btf__core_reloc_" #name ".o"
	272	+ .btf_src_file = "btf__core_reloc_" #name ".o", \
	273	+ .raw_tp_name = "sys_enter", \
	274	+ .prog_name = "test_core_size"
253	275
254	276	#define SIZE_OUTPUT_DATA(type) \
255	277	STRUCT_TO_CHAR_PTR(core_reloc_size_output) { \
@@ -277,8 +299,10 @@ static int duration = 0; · 21 unmodified lines
277	299
278	300	#define TYPE_BASED_CASE_COMMON(name) \
279	301	.case_name = #name, \
280		− .bpf_obj_file = "test_core_reloc_type_based.o", \
281		− .btf_src_file = "btf__core_reloc_" #name ".o" \
	302	+ .bpf_obj_file = "test_core_reloc_type_based.o", \
	303	+ .btf_src_file = "btf__core_reloc_" #name ".o", \
	304	+ .raw_tp_name = "sys_enter", \
	305	+ .prog_name = "test_core_type_based"
282	306
283	307	#define TYPE_BASED_CASE(name, ...) { \
284	308	TYPE_BASED_CASE_COMMON(name), \
@@ -295,7 +319,9 @@ static int duration = 0; · 10 unmodified lines
295	319	#define TYPE_ID_CASE_COMMON(name) \
296	320	.case_name = #name, \
297	321	.bpf_obj_file = "test_core_reloc_type_id.o", \
298		− .btf_src_file = "btf__core_reloc_" #name ".o" \
	322	+ .btf_src_file = "btf__core_reloc_" #name ".o", \
	323	+ .raw_tp_name = "sys_enter", \
	324	+ .prog_name = "test_core_type_id"
299	325
300	326	#define TYPE_ID_CASE(name, setup_fn) { \
301	327	TYPE_ID_CASE_COMMON(name), \
@@ -312,7 +338,9 @@ static int duration = 0; · 10 unmodified lines
312	338	#define ENUMVAL_CASE_COMMON(name) \
313	339	.case_name = #name, \
314	340	.bpf_obj_file = "test_core_reloc_enumval.o", \
315		− .btf_src_file = "btf__core_reloc_" #name ".o" \
	341	+ .btf_src_file = "btf__core_reloc_" #name ".o", \
	342	+ .raw_tp_name = "sys_enter", \
	343	+ .prog_name = "test_core_enumval"
316	344
317	345	#define ENUMVAL_CASE(name, ...) { \
318	346	ENUMVAL_CASE_COMMON(name), \
@@ -342,7 +370,7 @@ struct core_reloc_test_case { · 23 unmodified lines
342	370	bool fails;
343	371	bool needs_testmod;
344	372	bool relaxed_core_relocs;
345		− const char *prog_sec_name;
	373	+ const char *prog_name;
346	374	const char *raw_tp_name;
347	375	setup_test_fn setup;
348	376	trigger_test_fn trigger;
@@ -497,11 +525,13 @@ static struct core_reloc_test_case test_cases[] = { · 148 unmodified lines
497	525	.comm_len = sizeof("test_progs"),
498	526	},
499	527	.output_len = sizeof(struct core_reloc_kernel_output),
	528	+ .raw_tp_name = "sys_enter",
	529	+ .prog_name = "test_core_kernel",
500	530	},
501	531
502	532	/* validate we can find kernel module BTF types for relocs/attach */
503		− MODULES_CASE("module_probed", "raw_tp/bpf_testmod_test_read", "bpf_testmod_test_read"),
504		− MODULES_CASE("module_direct", "tp_btf/bpf_testmod_test_read", NULL),
	533	+ MODULES_CASE("module_probed", "test_core_module_probed", "bpf_testmod_test_read"),
	534	+ MODULES_CASE("module_direct", "test_core_module_direct", NULL),
505	535
506	536	/* validate BPF program can use multiple flavors to match against
507	537	* single target BTF type
@@ -580,6 +610,8 @@ static struct core_reloc_test_case test_cases[] = { · 72 unmodified lines
580	610	.c = 0, /* BUG in clang, should be 3 */
581	611	},
582	612	.output_len = sizeof(struct core_reloc_misc_output),
	613	+ .raw_tp_name = "sys_enter",
	614	+ .prog_name = "test_core_misc",
583	615	},
584	616
585	617	/* validate field existence checks */
@@ -848,14 +880,9 @@ void test_core_reloc(void) · 262 unmodified lines
848	880	if (!ASSERT_OK_PTR(obj, "obj_open"))
849	881	goto cleanup;
850	882
851		− probe_name = "raw_tracepoint/sys_enter";
852		− tp_name = "sys_enter";
853		− if (test_case->prog_sec_name) {
854		− probe_name = test_case->prog_sec_name;
855		− tp_name = test_case->raw_tp_name; /* NULL for tp_btf */
856		− }
857		−
858		− prog = bpf_object__find_program_by_title(obj, probe_name);
	883	+ probe_name = test_case->prog_name;
	884	+ tp_name = test_case->raw_tp_name; /* NULL for tp_btf */
	885	+ prog = bpf_object__find_program_by_name(obj, probe_name);
859	886	if (CHECK(!prog, "find_probe",
860	887	"prog '%s' not found\n", probe_name))
861	888	goto cleanup;

tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c+13−4

@@ -101,6 +101,8 @@ static void test_fexit_bpf2bpf_common(const char *obj_file, · 100 unmodified lines
101	101
102	102	for (i = 0; i < prog_cnt; i++) {
103	103	struct bpf_link_info link_info;
	104	+ struct bpf_program *pos;
	105	+ const char *pos_sec_name;
104	106	char *tgt_name;
105	107	__s32 btf_id;
106	108
@@ -109,7 +111,14 @@ static void test_fexit_bpf2bpf_common(const char *obj_file, · 2 unmodified lines
109	111	goto close_prog;
110	112	btf_id = btf__find_by_name_kind(btf, tgt_name + 1, BTF_KIND_FUNC);
111	113
112		− prog[i] = bpf_object__find_program_by_title(obj, prog_name[i]);
	114	+ prog[i] = NULL;
	115	+ bpf_object__for_each_program(pos, obj) {
	116	+ pos_sec_name = bpf_program__section_name(pos);
	117	+ if (pos_sec_name && !strcmp(pos_sec_name, prog_name[i])) {
	118	+ prog[i] = pos;
	119	+ break;
	120	+ }
	121	+ }
113	122	if (!ASSERT_OK_PTR(prog[i], prog_name[i]))
114	123	goto close_prog;
115	124
@@ -211,8 +220,8 @@ static void test_func_replace_verify(void) · 95 unmodified lines
211	220
212	221	static int test_second_attach(struct bpf_object *obj)
213	222	{
214		− const char *prog_name = "freplace/get_constant";
215		− const char tgt_name = prog_name + 9; / cut off freplace/ */
	223	+ const char *prog_name = "security_new_get_constant";
	224	+ const char *tgt_name = "get_constant";
216	225	const char *tgt_obj_file = "./test_pkt_access.o";
217	226	struct bpf_program *prog = NULL;
218	227	struct bpf_object *tgt_obj;
@@ -220,7 +229,7 @@ static int test_second_attach(struct bpf_object *obj) · 1 unmodified line
220	229	struct bpf_link *link;
221	230	int err = 0, tgt_fd;
222	231
223		− prog = bpf_object__find_program_by_title(obj, prog_name);
	232	+ prog = bpf_object__find_program_by_name(obj, prog_name);
224	233	if (CHECK(!prog, "find_prog", "prog %s not found\n", prog_name))
225	234	return -ENOENT;
226	235

tools/testing/selftests/bpf/prog_tests/get_func_args_test.c+44−0

@@ -0,0 +1,44 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+#include <test_progs.h>
	3	+#include "get_func_args_test.skel.h"
	4	+
	5	+void test_get_func_args_test(void)
	6	+{
	7	+ struct get_func_args_test *skel = NULL;
	8	+ __u32 duration = 0, retval;
	9	+ int err, prog_fd;
	10	+
	11	+ skel = get_func_args_test__open_and_load();
	12	+ if (!ASSERT_OK_PTR(skel, "get_func_args_test__open_and_load"))
	13	+ return;
	14	+
	15	+ err = get_func_args_test__attach(skel);
	16	+ if (!ASSERT_OK(err, "get_func_args_test__attach"))
	17	+ goto cleanup;
	18	+
	19	+ /* This runs bpf_fentry_test* functions and triggers
	20	+ * fentry/fexit programs.
	21	+ */
	22	+ prog_fd = bpf_program__fd(skel->progs.test1);
	23	+ err = bpf_prog_test_run(prog_fd, 1, NULL, 0,
	24	+ NULL, NULL, &retval, &duration);
	25	+ ASSERT_OK(err, "test_run");
	26	+ ASSERT_EQ(retval, 0, "test_run");
	27	+
	28	+ /* This runs bpf_modify_return_test function and triggers
	29	+ * fmod_ret_test and fexit_test programs.
	30	+ */
	31	+ prog_fd = bpf_program__fd(skel->progs.fmod_ret_test);
	32	+ err = bpf_prog_test_run(prog_fd, 1, NULL, 0,
	33	+ NULL, NULL, &retval, &duration);
	34	+ ASSERT_OK(err, "test_run");
	35	+ ASSERT_EQ(retval, 1234, "test_run");
	36	+
	37	+ ASSERT_EQ(skel->bss->test1_result, 1, "test1_result");
	38	+ ASSERT_EQ(skel->bss->test2_result, 1, "test2_result");
	39	+ ASSERT_EQ(skel->bss->test3_result, 1, "test3_result");
	40	+ ASSERT_EQ(skel->bss->test4_result, 1, "test4_result");
	41	+
	42	+cleanup:
	43	+ get_func_args_test__destroy(skel);
	44	+}

tools/testing/selftests/bpf/prog_tests/get_stack_raw_tp.c+2−2

@@ -89,7 +89,7 @@ void test_get_stack_raw_tp(void) · 88 unmodified lines
89	89	{
90	90	const char *file = "./test_get_stack_rawtp.o";
91	91	const char *file_err = "./test_get_stack_rawtp_err.o";
92		− const char *prog_name = "raw_tracepoint/sys_enter";
	92	+ const char *prog_name = "bpf_prog1";
93	93	int i, err, prog_fd, exp_cnt = MAX_CNT_RAWTP;
94	94	struct perf_buffer *pb = NULL;
95	95	struct bpf_link *link = NULL;
@@ -107,7 +107,7 @@ void test_get_stack_raw_tp(void) · 11 unmodified lines
107	107	if (CHECK(err, "prog_load raw tp", "err %d errno %d\n", err, errno))
108	108	return;
109	109
110		− prog = bpf_object__find_program_by_title(obj, prog_name);
	110	+ prog = bpf_object__find_program_by_name(obj, prog_name);
111	111	if (CHECK(!prog, "find_probe", "prog '%s' not found\n", prog_name))
112	112	goto close_prog;
113	113

tools/testing/selftests/bpf/prog_tests/ksyms_btf.c+14−0

@@ -8,6 +8,7 @@ · 7 unmodified lines
8	8	#include "test_ksyms_btf_null_check.skel.h"
9	9	#include "test_ksyms_weak.skel.h"
10	10	#include "test_ksyms_weak.lskel.h"
	11	+#include "test_ksyms_btf_write_check.skel.h"
11	12
12	13	static int duration;
13	14
@@ -137,6 +138,16 @@ cleanup: · 123 unmodified lines
137	138	test_ksyms_weak_lskel__destroy(skel);
138	139	}
139	140
	141	+static void test_write_check(void)
	142	+{
	143	+ struct test_ksyms_btf_write_check *skel;
	144	+
	145	+ skel = test_ksyms_btf_write_check__open_and_load();
	146	+ ASSERT_ERR_PTR(skel, "unexpected load of a prog writing to ksym memory\n");
	147	+
	148	+ test_ksyms_btf_write_check__destroy(skel);
	149	+}
	150	+
140	151	void test_ksyms_btf(void)
141	152	{
142	153	int percpu_datasec;
@@ -167,4 +178,7 @@ void test_ksyms_btf(void) · 24 unmodified lines
167	178
168	179	if (test__start_subtest("weak_ksyms_lskel"))
169	180	test_weak_syms_lskel();
	181	+
	182	+ if (test__start_subtest("write_check"))
	183	+ test_write_check();
170	184	}

tools/testing/selftests/bpf/prog_tests/libbpf_probes.c+124−0

@@ -0,0 +1,124 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+/* Copyright (c) 2021 Facebook */
	3	+
	4	+#include <test_progs.h>
	5	+#include <bpf/btf.h>
	6	+
	7	+void test_libbpf_probe_prog_types(void)
	8	+{
	9	+ struct btf *btf;
	10	+ const struct btf_type *t;
	11	+ const struct btf_enum *e;
	12	+ int i, n, id;
	13	+
	14	+ btf = btf__parse("/sys/kernel/btf/vmlinux", NULL);
	15	+ if (!ASSERT_OK_PTR(btf, "btf_parse"))
	16	+ return;
	17	+
	18	+ /* find enum bpf_prog_type and enumerate each value */
	19	+ id = btf__find_by_name_kind(btf, "bpf_prog_type", BTF_KIND_ENUM);
	20	+ if (!ASSERT_GT(id, 0, "bpf_prog_type_id"))
	21	+ goto cleanup;
	22	+ t = btf__type_by_id(btf, id);
	23	+ if (!ASSERT_OK_PTR(t, "bpf_prog_type_enum"))
	24	+ goto cleanup;
	25	+
	26	+ for (e = btf_enum(t), i = 0, n = btf_vlen(t); i < n; e++, i++) {
	27	+ const char *prog_type_name = btf__str_by_offset(btf, e->name_off);
	28	+ enum bpf_prog_type prog_type = (enum bpf_prog_type)e->val;
	29	+ int res;
	30	+
	31	+ if (prog_type == BPF_PROG_TYPE_UNSPEC)
	32	+ continue;
	33	+
	34	+ if (!test__start_subtest(prog_type_name))
	35	+ continue;
	36	+
	37	+ res = libbpf_probe_bpf_prog_type(prog_type, NULL);
	38	+ ASSERT_EQ(res, 1, prog_type_name);
	39	+ }
	40	+
	41	+cleanup:
	42	+ btf__free(btf);
	43	+}
	44	+
	45	+void test_libbpf_probe_map_types(void)
	46	+{
	47	+ struct btf *btf;
	48	+ const struct btf_type *t;
	49	+ const struct btf_enum *e;
	50	+ int i, n, id;
	51	+
	52	+ btf = btf__parse("/sys/kernel/btf/vmlinux", NULL);
	53	+ if (!ASSERT_OK_PTR(btf, "btf_parse"))
	54	+ return;
	55	+
	56	+ /* find enum bpf_map_type and enumerate each value */
	57	+ id = btf__find_by_name_kind(btf, "bpf_map_type", BTF_KIND_ENUM);
	58	+ if (!ASSERT_GT(id, 0, "bpf_map_type_id"))
	59	+ goto cleanup;
	60	+ t = btf__type_by_id(btf, id);
	61	+ if (!ASSERT_OK_PTR(t, "bpf_map_type_enum"))
	62	+ goto cleanup;
	63	+
	64	+ for (e = btf_enum(t), i = 0, n = btf_vlen(t); i < n; e++, i++) {
	65	+ const char *map_type_name = btf__str_by_offset(btf, e->name_off);
	66	+ enum bpf_map_type map_type = (enum bpf_map_type)e->val;
	67	+ int res;
	68	+
	69	+ if (map_type == BPF_MAP_TYPE_UNSPEC)
	70	+ continue;
	71	+
	72	+ if (!test__start_subtest(map_type_name))
	73	+ continue;
	74	+
	75	+ res = libbpf_probe_bpf_map_type(map_type, NULL);
	76	+ ASSERT_EQ(res, 1, map_type_name);
	77	+ }
	78	+
	79	+cleanup:
	80	+ btf__free(btf);
	81	+}
	82	+
	83	+void test_libbpf_probe_helpers(void)
	84	+{
	85	+#define CASE(prog, helper, supp) { \
	86	+ .prog_type_name = "BPF_PROG_TYPE_" # prog, \
	87	+ .helper_name = "bpf_" # helper, \
	88	+ .prog_type = BPF_PROG_TYPE_ ## prog, \
	89	+ .helper_id = BPF_FUNC_ ## helper, \
	90	+ .supported = supp, \
	91	+}
	92	+ const struct case_def {
	93	+ const char *prog_type_name;
	94	+ const char *helper_name;
	95	+ enum bpf_prog_type prog_type;
	96	+ enum bpf_func_id helper_id;
	97	+ bool supported;
	98	+ } cases[] = {
	99	+ CASE(KPROBE, unspec, false),
	100	+ CASE(KPROBE, map_lookup_elem, true),
	101	+ CASE(KPROBE, loop, true),
	102	+
	103	+ CASE(KPROBE, ktime_get_coarse_ns, false),
	104	+ CASE(SOCKET_FILTER, ktime_get_coarse_ns, true),
	105	+
	106	+ CASE(KPROBE, sys_bpf, false),
	107	+ CASE(SYSCALL, sys_bpf, true),
	108	+ };
	109	+ size_t case_cnt = ARRAY_SIZE(cases), i;
	110	+ char buf[128];
	111	+
	112	+ for (i = 0; i < case_cnt; i++) {
	113	+ const struct case_def *d = &cases[i];
	114	+ int res;
	115	+
	116	+ snprintf(buf, sizeof(buf), "%s+%s", d->prog_type_name, d->helper_name);
	117	+
	118	+ if (!test__start_subtest(buf))
	119	+ continue;
	120	+
	121	+ res = libbpf_probe_bpf_helper(d->prog_type, d->helper_id, NULL);
	122	+ ASSERT_EQ(res, d->supported, buf);
	123	+ }
	124	+}

tools/testing/selftests/bpf/prog_tests/select_reuseport.c+0−1

@@ -18,7 +18,6 @@ · 17 unmodified lines
18	18	#include <netinet/in.h>
19	19	#include <bpf/bpf.h>
20	20	#include <bpf/libbpf.h>
21		−#include "bpf_rlimit.h"
22	21	#include "bpf_util.h"
23	22
24	23	#include "test_progs.h"

tools/testing/selftests/bpf/prog_tests/sk_lookup.c+0−1

@@ -30,7 +30,6 @@ · 29 unmodified lines
30	30	#include <bpf/bpf.h>
31	31
32	32	#include "test_progs.h"
33		−#include "bpf_rlimit.h"
34	33	#include "bpf_util.h"
35	34	#include "cgroup_helpers.h"
36	35	#include "network_helpers.h"

tools/testing/selftests/bpf/prog_tests/sock_fields.c+0−1

@@ -15,7 +15,6 @@ · 14 unmodified lines
15	15	#include "network_helpers.h"
16	16	#include "cgroup_helpers.h"
17	17	#include "test_progs.h"
18		−#include "bpf_rlimit.h"
19	18	#include "test_sock_fields.skel.h"
20	19
21	20	enum bpf_linum_array_idx {

tools/testing/selftests/bpf/prog_tests/sockopt_inherit.c+8−7

@@ -136,7 +136,8 @@ static int start_server(void) · 135 unmodified lines
136	136	return fd;
137	137	}
138	138
139		−static int prog_attach(struct bpf_object obj, int cgroup_fd, const char title)
	139	+static int prog_attach(struct bpf_object obj, int cgroup_fd, const char title,
	140	+ const char *prog_name)
140	141	{
141	142	enum bpf_attach_type attach_type;
142	143	enum bpf_prog_type prog_type;
@@ -145,20 +146,20 @@ static int prog_attach(struct bpf_object obj, int cgroup_fd, const char title) · 2 unmodified lines
145	146
146	147	err = libbpf_prog_type_by_name(title, &prog_type, &attach_type);
147	148	if (err) {
148		− log_err("Failed to deduct types for %s BPF program", title);
	149	+ log_err("Failed to deduct types for %s BPF program", prog_name);
149	150	return -1;
150	151	}
151	152
152		− prog = bpf_object__find_program_by_title(obj, title);
	153	+ prog = bpf_object__find_program_by_name(obj, prog_name);
153	154	if (!prog) {
154		− log_err("Failed to find %s BPF program", title);
	155	+ log_err("Failed to find %s BPF program", prog_name);
155	156	return -1;
156	157	}
157	158
158	159	err = bpf_prog_attach(bpf_program__fd(prog), cgroup_fd,
159	160	attach_type, 0);
160	161	if (err) {
161		− log_err("Failed to attach %s BPF program", title);
	162	+ log_err("Failed to attach %s BPF program", prog_name);
162	163	return -1;
163	164	}
164	165
@@ -181,11 +182,11 @@ static void run_test(int cgroup_fd) · 16 unmodified lines
181	182	if (!ASSERT_OK(err, "obj_load"))
182	183	goto close_bpf_object;
183	184
184		− err = prog_attach(obj, cgroup_fd, "cgroup/getsockopt");
	185	+ err = prog_attach(obj, cgroup_fd, "cgroup/getsockopt", "_getsockopt");
185	186	if (CHECK_FAIL(err))
186	187	goto close_bpf_object;
187	188
188		− err = prog_attach(obj, cgroup_fd, "cgroup/setsockopt");
	189	+ err = prog_attach(obj, cgroup_fd, "cgroup/setsockopt", "_setsockopt");
189	190	if (CHECK_FAIL(err))
190	191	goto close_bpf_object;
191	192

tools/testing/selftests/bpf/prog_tests/stacktrace_map.c+2−2

@@ -4,7 +4,7 @@ · 3 unmodified lines
4	4	void test_stacktrace_map(void)
5	5	{
6	6	int control_map_fd, stackid_hmap_fd, stackmap_fd, stack_amap_fd;
7		− const char *prog_name = "tracepoint/sched/sched_switch";
	7	+ const char *prog_name = "oncpu";
8	8	int err, prog_fd, stack_trace_len;
9	9	const char *file = "./test_stacktrace_map.o";
10	10	__u32 key, val, duration = 0;
@@ -16,7 +16,7 @@ void test_stacktrace_map(void) · 5 unmodified lines
16	16	if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno))
17	17	return;
18	18
19		− prog = bpf_object__find_program_by_title(obj, prog_name);
	19	+ prog = bpf_object__find_program_by_name(obj, prog_name);
20	20	if (CHECK(!prog, "find_prog", "prog '%s' not found\n", prog_name))
21	21	goto close_prog;
22	22

tools/testing/selftests/bpf/prog_tests/stacktrace_map_raw_tp.c+2−2

@@ -3,7 +3,7 @@ · 2 unmodified lines
3	3
4	4	void test_stacktrace_map_raw_tp(void)
5	5	{
6		− const char *prog_name = "tracepoint/sched/sched_switch";
	6	+ const char *prog_name = "oncpu";
7	7	int control_map_fd, stackid_hmap_fd, stackmap_fd;
8	8	const char *file = "./test_stacktrace_map.o";
9	9	__u32 key, val, duration = 0;
@@ -16,7 +16,7 @@ void test_stacktrace_map_raw_tp(void) · 6 unmodified lines
16	16	if (CHECK(err, "prog_load raw tp", "err %d errno %d\n", err, errno))
17	17	return;
18	18
19		− prog = bpf_object__find_program_by_title(obj, prog_name);
	19	+ prog = bpf_object__find_program_by_name(obj, prog_name);
20	20	if (CHECK(!prog, "find_prog", "prog '%s' not found\n", prog_name))
21	21	goto close_prog;
22	22

tools/testing/selftests/bpf/prog_tests/test_local_storage.c+6−14

@@ -28,10 +28,6 @@ static unsigned int duration; · 27 unmodified lines
28	28	struct storage {
29	29	void *inode;
30	30	unsigned int value;
31		− /* Lock ensures that spin locked versions of local stoage operations
32		− * also work, most operations in this tests are still single threaded
33		− */
34		− struct bpf_spin_lock lock;
35	31	};
36	32
37	33	/* Fork and exec the provided rm binary and return the exit code of the
@@ -66,27 +62,24 @@ static int run_self_unlink(int monitored_pid, const char rm_path) · 28 unmodified lines
66	62
67	63	static bool check_syscall_operations(int map_fd, int obj_fd)
68	64	{
69		− struct storage val = { .value = TEST_STORAGE_VALUE, .lock = { 0 } },
70		− lookup_val = { .value = 0, .lock = { 0 } };
	65	+ struct storage val = { .value = TEST_STORAGE_VALUE },
	66	+ lookup_val = { .value = 0 };
71	67	int err;
72	68
73	69	/* Looking up an existing element should fail initially */
74		− err = bpf_map_lookup_elem_flags(map_fd, &obj_fd, &lookup_val,
75		− BPF_F_LOCK);
	70	+ err = bpf_map_lookup_elem_flags(map_fd, &obj_fd, &lookup_val, 0);
76	71	if (CHECK(!err \|\| errno != ENOENT, "bpf_map_lookup_elem",
77	72	"err:%d errno:%d\n", err, errno))
78	73	return false;
79	74
80	75	/* Create a new element */
81		− err = bpf_map_update_elem(map_fd, &obj_fd, &val,
82		− BPF_NOEXIST \| BPF_F_LOCK);
	76	+ err = bpf_map_update_elem(map_fd, &obj_fd, &val, BPF_NOEXIST);
83	77	if (CHECK(err < 0, "bpf_map_update_elem", "err:%d errno:%d\n", err,
84	78	errno))
85	79	return false;
86	80
87	81	/* Lookup the newly created element */
88		− err = bpf_map_lookup_elem_flags(map_fd, &obj_fd, &lookup_val,
89		− BPF_F_LOCK);
	82	+ err = bpf_map_lookup_elem_flags(map_fd, &obj_fd, &lookup_val, 0);
90	83	if (CHECK(err < 0, "bpf_map_lookup_elem", "err:%d errno:%d", err,
91	84	errno))
92	85	return false;
@@ -102,8 +95,7 @@ static bool check_syscall_operations(int map_fd, int obj_fd) · 9 unmodified lines
102	95	return false;
103	96
104	97	/* The lookup should fail, now that the element has been deleted */
105		− err = bpf_map_lookup_elem_flags(map_fd, &obj_fd, &lookup_val,
106		− BPF_F_LOCK);
	98	+ err = bpf_map_lookup_elem_flags(map_fd, &obj_fd, &lookup_val, 0);
107	99	if (CHECK(!err \|\| errno != ENOENT, "bpf_map_lookup_elem",
108	100	"err:%d errno:%d\n", err, errno))
109	101	return false;

tools/testing/selftests/bpf/prog_tests/test_overhead.c+10−10

@@ -56,11 +56,11 @@ static void setaffinity(void) · 55 unmodified lines
56	56
57	57	void test_test_overhead(void)
58	58	{
59		− const char *kprobe_name = "kprobe/__set_task_comm";
60		− const char *kretprobe_name = "kretprobe/__set_task_comm";
61		− const char *raw_tp_name = "raw_tp/task_rename";
62		− const char *fentry_name = "fentry/__set_task_comm";
63		− const char *fexit_name = "fexit/__set_task_comm";
	59	+ const char *kprobe_name = "prog1";
	60	+ const char *kretprobe_name = "prog2";
	61	+ const char *raw_tp_name = "prog3";
	62	+ const char *fentry_name = "prog4";
	63	+ const char *fexit_name = "prog5";
64	64	const char *kprobe_func = "__set_task_comm";
65	65	struct bpf_program kprobe_prog, kretprobe_prog, *raw_tp_prog;
66	66	struct bpf_program fentry_prog, fexit_prog;
@@ -76,23 +76,23 @@ void test_test_overhead(void) · 9 unmodified lines
76	76	if (!ASSERT_OK_PTR(obj, "obj_open_file"))
77	77	return;
78	78
79		− kprobe_prog = bpf_object__find_program_by_title(obj, kprobe_name);
	79	+ kprobe_prog = bpf_object__find_program_by_name(obj, kprobe_name);
80	80	if (CHECK(!kprobe_prog, "find_probe",
81	81	"prog '%s' not found\n", kprobe_name))
82	82	goto cleanup;
83		− kretprobe_prog = bpf_object__find_program_by_title(obj, kretprobe_name);
	83	+ kretprobe_prog = bpf_object__find_program_by_name(obj, kretprobe_name);
84	84	if (CHECK(!kretprobe_prog, "find_probe",
85	85	"prog '%s' not found\n", kretprobe_name))
86	86	goto cleanup;
87		− raw_tp_prog = bpf_object__find_program_by_title(obj, raw_tp_name);
	87	+ raw_tp_prog = bpf_object__find_program_by_name(obj, raw_tp_name);
88	88	if (CHECK(!raw_tp_prog, "find_probe",
89	89	"prog '%s' not found\n", raw_tp_name))
90	90	goto cleanup;
91		− fentry_prog = bpf_object__find_program_by_title(obj, fentry_name);
	91	+ fentry_prog = bpf_object__find_program_by_name(obj, fentry_name);
92	92	if (CHECK(!fentry_prog, "find_probe",
93	93	"prog '%s' not found\n", fentry_name))
94	94	goto cleanup;
95		− fexit_prog = bpf_object__find_program_by_title(obj, fexit_name);
	95	+ fexit_prog = bpf_object__find_program_by_name(obj, fexit_name);
96	96	if (CHECK(!fexit_prog, "find_probe",
97	97	"prog '%s' not found\n", fexit_name))
98	98	goto cleanup;

tools/testing/selftests/bpf/prog_tests/test_strncmp.c+167−0

@@ -0,0 +1,167 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (C) 2021. Huawei Technologies Co., Ltd */
	3	+#include <test_progs.h>
	4	+#include "strncmp_test.skel.h"
	5	+
	6	+static int trigger_strncmp(const struct strncmp_test *skel)
	7	+{
	8	+ int cmp;
	9	+
	10	+ usleep(1);
	11	+
	12	+ cmp = skel->bss->cmp_ret;
	13	+ if (cmp > 0)
	14	+ return 1;
	15	+ if (cmp < 0)
	16	+ return -1;
	17	+ return 0;
	18	+}
	19	+
	20	+/*
	21	+ * Compare str and target after making str[i] != target[i].
	22	+ * When exp is -1, make str[i] < target[i] and delta = -1.
	23	+ */
	24	+static void strncmp_full_str_cmp(struct strncmp_test skel, const char name,
	25	+ int exp)
	26	+{
	27	+ size_t nr = sizeof(skel->bss->str);
	28	+ char *str = skel->bss->str;
	29	+ int delta = exp;
	30	+ int got;
	31	+ size_t i;
	32	+
	33	+ memcpy(str, skel->rodata->target, nr);
	34	+ for (i = 0; i < nr - 1; i++) {
	35	+ str[i] += delta;
	36	+
	37	+ got = trigger_strncmp(skel);
	38	+ ASSERT_EQ(got, exp, name);
	39	+
	40	+ str[i] -= delta;
	41	+ }
	42	+}
	43	+
	44	+static void test_strncmp_ret(void)
	45	+{
	46	+ struct strncmp_test *skel;
	47	+ struct bpf_program *prog;
	48	+ int err, got;
	49	+
	50	+ skel = strncmp_test__open();
	51	+ if (!ASSERT_OK_PTR(skel, "strncmp_test open"))
	52	+ return;
	53	+
	54	+ bpf_object__for_each_program(prog, skel->obj)
	55	+ bpf_program__set_autoload(prog, false);
	56	+
	57	+ bpf_program__set_autoload(skel->progs.do_strncmp, true);
	58	+
	59	+ err = strncmp_test__load(skel);
	60	+ if (!ASSERT_EQ(err, 0, "strncmp_test load"))
	61	+ goto out;
	62	+
	63	+ err = strncmp_test__attach(skel);
	64	+ if (!ASSERT_EQ(err, 0, "strncmp_test attach"))
	65	+ goto out;
	66	+
	67	+ skel->bss->target_pid = getpid();
	68	+
	69	+ /* Empty str */
	70	+ skel->bss->str[0] = '\0';
	71	+ got = trigger_strncmp(skel);
	72	+ ASSERT_EQ(got, -1, "strncmp: empty str");
	73	+
	74	+ /* Same string */
	75	+ memcpy(skel->bss->str, skel->rodata->target, sizeof(skel->bss->str));
	76	+ got = trigger_strncmp(skel);
	77	+ ASSERT_EQ(got, 0, "strncmp: same str");
	78	+
	79	+ /* Not-null-termainted string */
	80	+ memcpy(skel->bss->str, skel->rodata->target, sizeof(skel->bss->str));
	81	+ skel->bss->str[sizeof(skel->bss->str) - 1] = 'A';
	82	+ got = trigger_strncmp(skel);
	83	+ ASSERT_EQ(got, 1, "strncmp: not-null-term str");
	84	+
	85	+ strncmp_full_str_cmp(skel, "strncmp: less than", -1);
	86	+ strncmp_full_str_cmp(skel, "strncmp: greater than", 1);
	87	+out:
	88	+ strncmp_test__destroy(skel);
	89	+}
	90	+
	91	+static void test_strncmp_bad_not_const_str_size(void)
	92	+{
	93	+ struct strncmp_test *skel;
	94	+ struct bpf_program *prog;
	95	+ int err;
	96	+
	97	+ skel = strncmp_test__open();
	98	+ if (!ASSERT_OK_PTR(skel, "strncmp_test open"))
	99	+ return;
	100	+
	101	+ bpf_object__for_each_program(prog, skel->obj)
	102	+ bpf_program__set_autoload(prog, false);
	103	+
	104	+ bpf_program__set_autoload(skel->progs.strncmp_bad_not_const_str_size,
	105	+ true);
	106	+
	107	+ err = strncmp_test__load(skel);
	108	+ ASSERT_ERR(err, "strncmp_test load bad_not_const_str_size");
	109	+
	110	+ strncmp_test__destroy(skel);
	111	+}
	112	+
	113	+static void test_strncmp_bad_writable_target(void)
	114	+{
	115	+ struct strncmp_test *skel;
	116	+ struct bpf_program *prog;
	117	+ int err;
	118	+
	119	+ skel = strncmp_test__open();
	120	+ if (!ASSERT_OK_PTR(skel, "strncmp_test open"))
	121	+ return;
	122	+
	123	+ bpf_object__for_each_program(prog, skel->obj)
	124	+ bpf_program__set_autoload(prog, false);
	125	+
	126	+ bpf_program__set_autoload(skel->progs.strncmp_bad_writable_target,
	127	+ true);
	128	+
	129	+ err = strncmp_test__load(skel);
	130	+ ASSERT_ERR(err, "strncmp_test load bad_writable_target");
	131	+
	132	+ strncmp_test__destroy(skel);
	133	+}
	134	+
	135	+static void test_strncmp_bad_not_null_term_target(void)
	136	+{
	137	+ struct strncmp_test *skel;
	138	+ struct bpf_program *prog;
	139	+ int err;
	140	+
	141	+ skel = strncmp_test__open();
	142	+ if (!ASSERT_OK_PTR(skel, "strncmp_test open"))
	143	+ return;
	144	+
	145	+ bpf_object__for_each_program(prog, skel->obj)
	146	+ bpf_program__set_autoload(prog, false);
	147	+
	148	+ bpf_program__set_autoload(skel->progs.strncmp_bad_not_null_term_target,
	149	+ true);
	150	+
	151	+ err = strncmp_test__load(skel);
	152	+ ASSERT_ERR(err, "strncmp_test load bad_not_null_term_target");
	153	+
	154	+ strncmp_test__destroy(skel);
	155	+}
	156	+
	157	+void test_test_strncmp(void)
	158	+{
	159	+ if (test__start_subtest("strncmp_ret"))
	160	+ test_strncmp_ret();
	161	+ if (test__start_subtest("strncmp_bad_not_const_str_size"))
	162	+ test_strncmp_bad_not_const_str_size();
	163	+ if (test__start_subtest("strncmp_bad_writable_target"))
	164	+ test_strncmp_bad_writable_target();
	165	+ if (test__start_subtest("strncmp_bad_not_null_term_target"))
	166	+ test_strncmp_bad_not_null_term_target();
	167	+}

tools/testing/selftests/bpf/prog_tests/trampoline_count.c+3−3

@@ -35,7 +35,7 @@ static struct bpf_link load(struct bpf_object obj, const char *name) · 34 unmodified lines
35	35	struct bpf_program *prog;
36	36	int duration = 0;
37	37
38		− prog = bpf_object__find_program_by_title(obj, name);
	38	+ prog = bpf_object__find_program_by_name(obj, name);
39	39	if (CHECK(!prog, "find_probe", "prog '%s' not found\n", name))
40	40	return ERR_PTR(-EINVAL);
41	41	return bpf_program__attach_trace(prog);
@@ -44,8 +44,8 @@ static struct bpf_link load(struct bpf_object obj, const char *name) · 2 unmodified lines
44	44	/* TODO: use different target function to run in concurrent mode */
45	45	void serial_test_trampoline_count(void)
46	46	{
47		− const char *fentry_name = "fentry/__set_task_comm";
48		− const char *fexit_name = "fexit/__set_task_comm";
	47	+ const char *fentry_name = "prog1";
	48	+ const char *fexit_name = "prog2";
49	49	const char *object = "test_trampoline_count.o";
50	50	struct inst inst[MAX_TRAMP_PROGS] = {};
51	51	int err, i = 0, duration = 0;

tools/testing/selftests/bpf/progs/get_func_args_test.c+123−0

@@ -0,0 +1,123 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+#include <linux/bpf.h>
	3	+#include <bpf/bpf_helpers.h>
	4	+#include <bpf/bpf_tracing.h>
	5	+#include <errno.h>
	6	+
	7	+char _license[] SEC("license") = "GPL";
	8	+
	9	+__u64 test1_result = 0;
	10	+SEC("fentry/bpf_fentry_test1")
	11	+int BPF_PROG(test1)
	12	+{
	13	+ __u64 cnt = bpf_get_func_arg_cnt(ctx);
	14	+ __u64 a = 0, z = 0, ret = 0;
	15	+ __s64 err;
	16	+
	17	+ test1_result = cnt == 1;
	18	+
	19	+ /* valid arguments */
	20	+ err = bpf_get_func_arg(ctx, 0, &a);
	21	+
	22	+ /* We need to cast access to traced function argument values with
	23	+ * proper type cast, because trampoline uses type specific instruction
	24	+ * to save it, like for 'int a' with 32-bit mov like:
	25	+ *
	26	+ * mov %edi,-0x8(%rbp)
	27	+ *
	28	+ * so the upper 4 bytes are not zeroed.
	29	+ */
	30	+ test1_result &= err == 0 && ((int) a == 1);
	31	+
	32	+ /* not valid argument */
	33	+ err = bpf_get_func_arg(ctx, 1, &z);
	34	+ test1_result &= err == -EINVAL;
	35	+
	36	+ /* return value fails in fentry */
	37	+ err = bpf_get_func_ret(ctx, &ret);
	38	+ test1_result &= err == -EOPNOTSUPP;
	39	+ return 0;
	40	+}
	41	+
	42	+__u64 test2_result = 0;
	43	+SEC("fexit/bpf_fentry_test2")
	44	+int BPF_PROG(test2)
	45	+{
	46	+ __u64 cnt = bpf_get_func_arg_cnt(ctx);
	47	+ __u64 a = 0, b = 0, z = 0, ret = 0;
	48	+ __s64 err;
	49	+
	50	+ test2_result = cnt == 2;
	51	+
	52	+ /* valid arguments */
	53	+ err = bpf_get_func_arg(ctx, 0, &a);
	54	+ test2_result &= err == 0 && (int) a == 2;
	55	+
	56	+ err = bpf_get_func_arg(ctx, 1, &b);
	57	+ test2_result &= err == 0 && b == 3;
	58	+
	59	+ /* not valid argument */
	60	+ err = bpf_get_func_arg(ctx, 2, &z);
	61	+ test2_result &= err == -EINVAL;
	62	+
	63	+ /* return value */
	64	+ err = bpf_get_func_ret(ctx, &ret);
	65	+ test2_result &= err == 0 && ret == 5;
	66	+ return 0;
	67	+}
	68	+
	69	+__u64 test3_result = 0;
	70	+SEC("fmod_ret/bpf_modify_return_test")
	71	+int BPF_PROG(fmod_ret_test, int _a, int *_b, int _ret)
	72	+{
	73	+ __u64 cnt = bpf_get_func_arg_cnt(ctx);
	74	+ __u64 a = 0, b = 0, z = 0, ret = 0;
	75	+ __s64 err;
	76	+
	77	+ test3_result = cnt == 2;
	78	+
	79	+ /* valid arguments */
	80	+ err = bpf_get_func_arg(ctx, 0, &a);
	81	+ test3_result &= err == 0 && ((int) a == 1);
	82	+
	83	+ err = bpf_get_func_arg(ctx, 1, &b);
	84	+ test3_result &= err == 0 && ((int *) b == _b);
	85	+
	86	+ /* not valid argument */
	87	+ err = bpf_get_func_arg(ctx, 2, &z);
	88	+ test3_result &= err == -EINVAL;
	89	+
	90	+ /* return value */
	91	+ err = bpf_get_func_ret(ctx, &ret);
	92	+ test3_result &= err == 0 && ret == 0;
	93	+
	94	+ /* change return value, it's checked in fexit_test program */
	95	+ return 1234;
	96	+}
	97	+
	98	+__u64 test4_result = 0;
	99	+SEC("fexit/bpf_modify_return_test")
	100	+int BPF_PROG(fexit_test, int _a, int *_b, int _ret)
	101	+{
	102	+ __u64 cnt = bpf_get_func_arg_cnt(ctx);
	103	+ __u64 a = 0, b = 0, z = 0, ret = 0;
	104	+ __s64 err;
	105	+
	106	+ test4_result = cnt == 2;
	107	+
	108	+ /* valid arguments */
	109	+ err = bpf_get_func_arg(ctx, 0, &a);
	110	+ test4_result &= err == 0 && ((int) a == 1);
	111	+
	112	+ err = bpf_get_func_arg(ctx, 1, &b);
	113	+ test4_result &= err == 0 && ((int *) b == _b);
	114	+
	115	+ /* not valid argument */
	116	+ err = bpf_get_func_arg(ctx, 2, &z);
	117	+ test4_result &= err == -EINVAL;
	118	+
	119	+ /* return value */
	120	+ err = bpf_get_func_ret(ctx, &ret);
	121	+ test4_result &= err == 0 && ret == 1234;
	122	+ return 0;
	123	+}

tools/testing/selftests/bpf/progs/local_storage.c+5−19

@@ -20,7 +20,6 @@ int sk_storage_result = -1; · 19 unmodified lines
20	20	struct local_storage {
21	21	struct inode *exec_inode;
22	22	__u32 value;
23		− struct bpf_spin_lock lock;
24	23	};
25	24
26	25	struct {
@@ -58,9 +57,7 @@ int BPF_PROG(unlink_hook, struct inode dir, struct dentry victim) · 31 unmodified lines
58	57	bpf_get_current_task_btf(), 0, 0);
59	58	if (storage) {
60	59	/* Don't let an executable delete itself */
61		− bpf_spin_lock(&storage->lock);
62	60	is_self_unlink = storage->exec_inode == victim->d_inode;
63		− bpf_spin_unlock(&storage->lock);
64	61	if (is_self_unlink)
65	62	return -EPERM;
66	63	}
@@ -68,7 +65,7 @@ int BPF_PROG(unlink_hook, struct inode dir, struct dentry victim) · 1 unmodified line
68	65	return 0;
69	66	}
70	67
71		−SEC("lsm/inode_rename")
	68	+SEC("lsm.s/inode_rename")
72	69	int BPF_PROG(inode_rename, struct inode old_dir, struct dentry old_dentry,
73	70	struct inode new_dir, struct dentry new_dentry,
74	71	unsigned int flags)
@@ -89,10 +86,8 @@ int BPF_PROG(inode_rename, struct inode old_dir, struct dentry old_dentry, · 14 unmodified lines
89	86	if (!storage)
90	87	return 0;
91	88
92		− bpf_spin_lock(&storage->lock);
93	89	if (storage->value != DUMMY_STORAGE_VALUE)
94	90	inode_storage_result = -1;
95		− bpf_spin_unlock(&storage->lock);
96	91
97	92	err = bpf_inode_storage_delete(&inode_storage_map, old_dentry->d_inode);
98	93	if (!err)
@@ -101,7 +96,7 @@ int BPF_PROG(inode_rename, struct inode old_dir, struct dentry old_dentry, · 2 unmodified lines
101	96	return 0;
102	97	}
103	98
104		−SEC("lsm/socket_bind")
	99	+SEC("lsm.s/socket_bind")
105	100	int BPF_PROG(socket_bind, struct socket sock, struct sockaddr address,
106	101	int addrlen)
107	102	{
@@ -117,10 +112,8 @@ int BPF_PROG(socket_bind, struct socket sock, struct sockaddr address, · 9 unmodified lines
117	112	if (!storage)
118	113	return 0;
119	114
120		− bpf_spin_lock(&storage->lock);
121	115	if (storage->value != DUMMY_STORAGE_VALUE)
122	116	sk_storage_result = -1;
123		− bpf_spin_unlock(&storage->lock);
124	117
125	118	err = bpf_sk_storage_delete(&sk_storage_map, sock->sk);
126	119	if (!err)
@@ -129,7 +122,7 @@ int BPF_PROG(socket_bind, struct socket sock, struct sockaddr address, · 2 unmodified lines
129	122	return 0;
130	123	}
131	124
132		−SEC("lsm/socket_post_create")
	125	+SEC("lsm.s/socket_post_create")
133	126	int BPF_PROG(socket_post_create, struct socket *sock, int family, int type,
134	127	int protocol, int kern)
135	128	{
@@ -144,9 +137,7 @@ int BPF_PROG(socket_post_create, struct socket *sock, int family, int type, · 8 unmodified lines
144	137	if (!storage)
145	138	return 0;
146	139
147		− bpf_spin_lock(&storage->lock);
148	140	storage->value = DUMMY_STORAGE_VALUE;
149		− bpf_spin_unlock(&storage->lock);
150	141
151	142	return 0;
152	143	}
@@ -154,7 +145,7 @@ int BPF_PROG(socket_post_create, struct socket *sock, int family, int type, · 1 unmodified line
154	145	/* This uses the local storage to remember the inode of the binary that a
155	146	* process was originally executing.
156	147	*/
157		−SEC("lsm/bprm_committed_creds")
	148	+SEC("lsm.s/bprm_committed_creds")
158	149	void BPF_PROG(exec, struct linux_binprm *bprm)
159	150	{
160	151	__u32 pid = bpf_get_current_pid_tgid() >> 32;
@@ -166,18 +157,13 @@ void BPF_PROG(exec, struct linux_binprm *bprm) · 5 unmodified lines
166	157	storage = bpf_task_storage_get(&task_storage_map,
167	158	bpf_get_current_task_btf(), 0,
168	159	BPF_LOCAL_STORAGE_GET_F_CREATE);
169		− if (storage) {
170		− bpf_spin_lock(&storage->lock);
	160	+ if (storage)
171	161	storage->exec_inode = bprm->file->f_inode;
172		− bpf_spin_unlock(&storage->lock);
173		− }
174	162
175	163	storage = bpf_inode_storage_get(&inode_storage_map, bprm->file->f_inode,
176	164	0, BPF_LOCAL_STORAGE_GET_F_CREATE);
177	165	if (!storage)
178	166	return;
179	167
180		− bpf_spin_lock(&storage->lock);
181	168	storage->value = DUMMY_STORAGE_VALUE;
182		− bpf_spin_unlock(&storage->lock);
183	169	}

tools/testing/selftests/bpf/progs/strncmp_bench.c+50−0

@@ -0,0 +1,50 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (C) 2021. Huawei Technologies Co., Ltd */
	3	+#include <linux/types.h>
	4	+#include <linux/bpf.h>
	5	+#include <bpf/bpf_helpers.h>
	6	+#include <bpf/bpf_tracing.h>
	7	+
	8	+#define STRNCMP_STR_SZ 4096
	9	+
	10	+/* Will be updated by benchmark before program loading */
	11	+const volatile unsigned int cmp_str_len = 1;
	12	+const char target[STRNCMP_STR_SZ];
	13	+
	14	+long hits = 0;
	15	+char str[STRNCMP_STR_SZ];
	16	+
	17	+char _license[] SEC("license") = "GPL";
	18	+
	19	+static __always_inline int local_strncmp(const char *s1, unsigned int sz,
	20	+ const char *s2)
	21	+{
	22	+ int ret = 0;
	23	+ unsigned int i;
	24	+
	25	+ for (i = 0; i < sz; i++) {
	26	+ /* E.g. 0xff > 0x31 */
	27	+ ret = (unsigned char)s1[i] - (unsigned char)s2[i];
	28	+ if (ret \|\| !s1[i])
	29	+ break;
	30	+ }
	31	+
	32	+ return ret;
	33	+}
	34	+
	35	+SEC("tp/syscalls/sys_enter_getpgid")
	36	+int strncmp_no_helper(void *ctx)
	37	+{
	38	+ if (local_strncmp(str, cmp_str_len + 1, target) < 0)
	39	+ __sync_add_and_fetch(&hits, 1);
	40	+ return 0;
	41	+}
	42	+
	43	+SEC("tp/syscalls/sys_enter_getpgid")
	44	+int strncmp_helper(void *ctx)
	45	+{
	46	+ if (bpf_strncmp(str, cmp_str_len + 1, target) < 0)
	47	+ __sync_add_and_fetch(&hits, 1);
	48	+ return 0;
	49	+}
	50	+

tools/testing/selftests/bpf/progs/strncmp_test.c+54−0

@@ -0,0 +1,54 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (C) 2021. Huawei Technologies Co., Ltd */
	3	+#include <stdbool.h>
	4	+#include <linux/types.h>
	5	+#include <linux/bpf.h>
	6	+#include <bpf/bpf_helpers.h>
	7	+#include <bpf/bpf_tracing.h>
	8	+
	9	+#define STRNCMP_STR_SZ 8
	10	+
	11	+const char target[STRNCMP_STR_SZ] = "EEEEEEE";
	12	+char str[STRNCMP_STR_SZ];
	13	+int cmp_ret = 0;
	14	+int target_pid = 0;
	15	+
	16	+const char no_str_target[STRNCMP_STR_SZ] = "12345678";
	17	+char writable_target[STRNCMP_STR_SZ];
	18	+unsigned int no_const_str_size = STRNCMP_STR_SZ;
	19	+
	20	+char _license[] SEC("license") = "GPL";
	21	+
	22	+SEC("tp/syscalls/sys_enter_nanosleep")
	23	+int do_strncmp(void *ctx)
	24	+{
	25	+ if ((bpf_get_current_pid_tgid() >> 32) != target_pid)
	26	+ return 0;
	27	+
	28	+ cmp_ret = bpf_strncmp(str, STRNCMP_STR_SZ, target);
	29	+ return 0;
	30	+}
	31	+
	32	+SEC("tp/syscalls/sys_enter_nanosleep")
	33	+int strncmp_bad_not_const_str_size(void *ctx)
	34	+{
	35	+ /* The value of string size is not const, so will fail */
	36	+ cmp_ret = bpf_strncmp(str, no_const_str_size, target);
	37	+ return 0;
	38	+}
	39	+
	40	+SEC("tp/syscalls/sys_enter_nanosleep")
	41	+int strncmp_bad_writable_target(void *ctx)
	42	+{
	43	+ /* Compared target is not read-only, so will fail */
	44	+ cmp_ret = bpf_strncmp(str, STRNCMP_STR_SZ, writable_target);
	45	+ return 0;
	46	+}
	47	+
	48	+SEC("tp/syscalls/sys_enter_nanosleep")
	49	+int strncmp_bad_not_null_term_target(void *ctx)
	50	+{
	51	+ /* Compared target is not null-terminated, so will fail */
	52	+ cmp_ret = bpf_strncmp(str, STRNCMP_STR_SZ, no_str_target);
	53	+ return 0;
	54	+}

tools/testing/selftests/bpf/progs/test_ksyms_btf_write_check.c+29−0

@@ -0,0 +1,29 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2021 Google */
	3	+
	4	+#include "vmlinux.h"
	5	+
	6	+#include <bpf/bpf_helpers.h>
	7	+
	8	+extern const int bpf_prog_active __ksym; /* int type global var. */
	9	+
	10	+SEC("raw_tp/sys_enter")
	11	+int handler(const void *ctx)
	12	+{
	13	+ int *active;
	14	+ __u32 cpu;
	15	+
	16	+ cpu = bpf_get_smp_processor_id();
	17	+ active = (int *)bpf_per_cpu_ptr(&bpf_prog_active, cpu);
	18	+ if (active) {
	19	+ /* Kernel memory obtained from bpf_{per,this}_cpu_ptr
	20	+ * is read-only, should _not_ pass verification.
	21	+ */
	22	+ /* WRITE_ONCE */
	23	+ (volatile int )active = -1;
	24	+ }
	25	+
	26	+ return 0;
	27	+}
	28	+
	29	+char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/test_cpp.cpp+8−1

@@ -7,9 +7,15 @@ · 6 unmodified lines
7	7
8	8	/* do nothing, just make sure we can link successfully */
9	9
	10	+static void dump_printf(void ctx, const char fmt, va_list args)
	11	+{
	12	+}
	13	+
10	14	int main(int argc, char *argv[])
11	15	{
	16	+ struct btf_dump_opts opts = { };
12	17	struct test_core_extern *skel;
	18	+ struct btf *btf;
13	19
14	20	/* libbpf.h */
15	21	libbpf_set_print(NULL);
@@ -18,7 +24,8 @@ int main(int argc, char *argv[]) · 2 unmodified lines
18	24	bpf_prog_get_fd_by_id(0);
19	25
20	26	/* btf.h */
21		− btf__new(NULL, 0);
	27	+ btf = btf__new(NULL, 0);
	28	+ btf_dump__new(btf, dump_printf, nullptr, &opts);
22	29
23	30	/* BPF skeleton */
24	31	skel = test_core_extern__open_and_load();

tools/testing/selftests/bpf/test_maps.c+0−1

@@ -23,7 +23,6 @@ · 22 unmodified lines
23	23	#include <bpf/libbpf.h>
24	24
25	25	#include "bpf_util.h"
26		−#include "bpf_rlimit.h"
27	26	#include "test_maps.h"
28	27	#include "testing_helpers.h"
29	28

tools/testing/selftests/bpf/test_progs.c+0−2

@@ -4,7 +4,6 @@ · 3 unmodified lines
4	4	#define _GNU_SOURCE
5	5	#include "test_progs.h"
6	6	#include "cgroup_helpers.h"
7		−#include "bpf_rlimit.h"
8	7	#include <argp.h>
9	8	#include <pthread.h>
10	9	#include <sched.h>
@@ -1342,7 +1341,6 @@ int main(int argc, char **argv) · 1331 unmodified lines
1342	1341
1343	1342	/* Use libbpf 1.0 API mode */
1344	1343	libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
1345		−
1346	1344	libbpf_set_print(libbpf_print_fn);
1347	1345
1348	1346	srand(time(NULL));

tools/testing/selftests/bpf/test_verifier.c+8−10

@@ -41,7 +41,6 @@ · 40 unmodified lines
41	41	# define CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 1
42	42	# endif
43	43	#endif
44		−#include "bpf_rlimit.h"
45	44	#include "bpf_rand.h"
46	45	#include "bpf_util.h"
47	46	#include "test_btf.h"
@@ -701,22 +700,18 @@ static int create_sk_storage_map(void) · 653 unmodified lines
701	700
702	701	static int create_map_timer(void)
703	702	{
704		− struct bpf_create_map_attr attr = {
705		− .name = "test_map",
706		− .map_type = BPF_MAP_TYPE_ARRAY,
707		− .key_size = 4,
708		− .value_size = 16,
709		− .max_entries = 1,
	703	+ LIBBPF_OPTS(bpf_map_create_opts, opts,
710	704	.btf_key_type_id = 1,
711	705	.btf_value_type_id = 5,
712		− };
	706	+ );
713	707	int fd, btf_fd;
714	708
715	709	btf_fd = load_btf();
716	710	if (btf_fd < 0)
717	711	return -1;
718		− attr.btf_fd = btf_fd;
719		− fd = bpf_create_map_xattr(&attr);
	712	+
	713	+ opts.btf_fd = btf_fd;
	714	+ fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "test_map", 4, 16, 1, &opts);
720	715	if (fd < 0)
721	716	printf("Failed to create map with timer\n");
722	717	return fd;
@@ -1399,6 +1394,9 @@ int main(int argc, char **argv) · 676 unmodified lines
1399	1394	return EXIT_FAILURE;
1400	1395	}
1401	1396
	1397	+ /* Use libbpf 1.0 API mode */
	1398	+ libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
	1399	+
1402	1400	bpf_semi_rand_init();
1403	1401	return do_test(unpriv, from, to);
1404	1402	}

tools/testing/selftests/bpf/verifier/btf_ctx_access.c+12−0

@@ -0,0 +1,12 @@
	1	+{
	2	+ "btf_ctx_access accept",
	3	+ .insns = {
	4	+ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 8), /* load 2nd argument value (int pointer) */
	5	+ BPF_MOV64_IMM(BPF_REG_0, 0),
	6	+ BPF_EXIT_INSN(),
	7	+ },
	8	+ .result = ACCEPT,
	9	+ .prog_type = BPF_PROG_TYPE_TRACING,
	10	+ .expected_attach_type = BPF_TRACE_FENTRY,
	11	+ .kfunc = "bpf_modify_return_test",
	12	+},

tools/testing/selftests/bpf/vmtest.sh+1−1

@@ -32,7 +32,7 @@ ROOTFS_IMAGE="root.img" · 31 unmodified lines
32	32	OUTPUT_DIR="$HOME/.bpf_selftests"
33	33	KCONFIG_URL="https://raw.githubusercontent.com/libbpf/libbpf/master/travis-ci/vmtest/configs/config-latest.${ARCH}"
34	34	KCONFIG_API_URL="https://api.github.com/repos/libbpf/libbpf/contents/travis-ci/vmtest/configs/config-latest.${ARCH}"
35		−INDEX_URL="https://raw.githubusercontent.com/libbpf/libbpf/master/travis-ci/vmtest/configs/INDEX"
	35	+INDEX_URL="https://raw.githubusercontent.com/libbpf/ci/master/INDEX"
36	36	NUM_COMPILE_JOBS="$(nproc)"
37	37	LOG_FILE_BASE="$(date +"bpf_selftests.%Y-%m-%d_%H-%M-%S")"
38	38	LOG_FILE="${LOG_FILE_BASE}.log"