History

Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Daniel Borkmann says: ==================== pull-request: bpf-next 2021-02-16 The following pull-request contains BPF updates for your *net-next* tree. There's a small merge conflict between 7eeba1706eba ("tcp: Add receive timestamp support for receive zerocopy.") from net-next tree and 9cacf81f8161 ("bpf: Remove extra lock_sock for TCP_ZEROCOPY_RECEIVE") from bpf-next tree. Resolve as follows: [...] lock_sock(sk); err = tcp_zerocopy_receive(sk, &zc, &tss); err = BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sk, level, optname, &zc, &len, err); release_sock(sk); [...] We've added 116 non-merge commits during the last 27 day(s) which contain a total of 156 files changed, 5662 insertions(+), 1489 deletions(-). The main changes are: 1) Adds support of pointers to types with known size among global function args to overcome the limit on max # of allowed args, from Dmitrii Banshchikov. 2) Add bpf_iter for task_vma which can be used to generate information similar to /proc/pid/maps, from Song Liu. 3) Enable bpf_{g,s}etsockopt() from all sock_addr related program hooks. Allow rewriting bind user ports from BPF side below the ip_unprivileged_port_start range, both from Stanislav Fomichev. 4) Prevent recursion on fentry/fexit & sleepable programs and allow map-in-map as well as per-cpu maps for the latter, from Alexei Starovoitov. 5) Add selftest script to run BPF CI locally. Also enable BPF ringbuffer for sleepable programs, both from KP Singh. 6) Extend verifier to enable variable offset read/write access to the BPF program stack, from Andrei Matei. 7) Improve tc & XDP MTU handling and add a new bpf_check_mtu() helper to query device MTU from programs, from Jesper Dangaard Brouer. 8) Allow bpf_get_socket_cookie() helper also be called from [sleepable] BPF tracing programs, from Florent Revest. 9) Extend x86 JIT to pad JMPs with NOPs for helping image to converge when otherwise too many passes are required, from Gary Lin. 10) Verifier fixes on atomics with BPF_FETCH as well as function-by-function verification both related to zero-extension handling, from Ilya Leoshkevich. 11) Better kernel build integration of resolve_btfids tool, from Jiri Olsa. 12) Batch of AF_XDP selftest cleanups and small performance improvement for libbpf's xsk map redirect for newer kernels, from Björn Töpel. 13) Follow-up BPF doc and verifier improvements around atomics with BPF_FETCH, from Brendan Jackman. 14) Permit zero-sized data sections e.g. if ELF .rodata section contains read-only data from local variables, from Yonghong Song. 15) veth driver skb bulk-allocation for ndo_xdp_xmit, from Lorenzo Bianconi. ====================

David S. Miller committed 5 years ago

Commit b8af4172 parents 9ec5eea 45159b2156 files · +5662 −1489Browse files

Unified · Split

156 files
  1. Documentation/bpf/bpf_design_QA.rst +6 −0
  2. Documentation/bpf/bpf_devel_QA.rst +7 −4
  3. Documentation/networking/filter.rst +15 −13
  4. Makefile +12 −1
  5. arch/x86/net/bpf_jit_comp.c +158 −47
  6. drivers/net/veth.c +60 −34
  7. include/linux/bpf-cgroup.h +70 −31
  8. include/linux/bpf.h +43 −31
  9. include/linux/bpf_verifier.h +4 −1
  10. include/linux/filter.h +18 −3
  11. include/linux/indirect_call_wrapper.h +6 −0
  12. include/linux/netdevice.h +30 −2
  13. include/linux/skmsg.h +0 −1
  14. include/net/inet_common.h +2 −0
  15. include/net/sock.h +2 −0
  16. include/net/tcp.h +1 −0
  17. include/net/xdp.h +6 −0
  18. include/trace/bpf_probe.h +10 −2
  19. include/uapi/linux/bpf.h +97 −6
  20. kernel/bpf/bpf_iter.c +1 −1
  21. kernel/bpf/bpf_lru_list.c +4 −3
  22. kernel/bpf/btf.c +55 −21
  23. kernel/bpf/cgroup.c +103 −17
  24. kernel/bpf/core.c +14 −4
  25. kernel/bpf/cpumap.c +2 −44
  26. kernel/bpf/devmap.c +1 −3
  27. kernel/bpf/disasm.c +1 −1
  28. kernel/bpf/hashtab.c +2 −2
  29. kernel/bpf/helpers.c +4 −8
  30. kernel/bpf/syscall.c +11 −5
  31. kernel/bpf/task_iter.c +266 −1
  32. kernel/bpf/trampoline.c +61 −16
  33. kernel/bpf/verifier.c +662 −215
  34. kernel/trace/bpf_trace.c +6 −0
  35. lib/test_bpf.c +1 −6
  36. net/core/dev.c +14 −18
  37. net/core/filter.c +174 −21
  38. net/core/skmsg.c +1 −2
  39. net/core/xdp.c +70 −0
  40. net/ipv4/af_inet.c +11 −7
  41. net/ipv4/tcp.c +14 −0
  42. net/ipv4/tcp_ipv4.c +1 −0
  43. net/ipv4/udp.c +3 −4
  44. net/ipv6/af_inet6.c +11 −7
  45. net/ipv6/tcp_ipv6.c +1 −0
  46. net/ipv6/udp.c +3 −4
  47. net/socket.c +3 −0
  48. net/xdp/xsk.c +31 −16
  49. net/xdp/xsk_buff_pool.c +3 −9
  50. samples/bpf/Makefile +9 −1
  51. samples/bpf/README.rst +14 −8
  52. samples/bpf/bpf_insn.h +20 −4
  53. samples/bpf/cookie_uid_helper_example.c +1 −1
  54. samples/bpf/xdp_redirect_map_kern.c +55 −5
  55. samples/bpf/xdp_redirect_map_user.c +92 −20
  56. tools/bpf/bpf_dbg.c +1 −1
  57. tools/bpf/bpftool/Makefile +0 −2
  58. tools/bpf/bpftool/prog.c +4 −0
  59. tools/bpf/resolve_btfids/.gitignore +1 −2
  60. tools/bpf/resolve_btfids/Makefile +22 −22
  61. tools/bpf/runqslower/Makefile +0 −3
  62. tools/build/feature/Makefile +2 −2
  63. tools/include/linux/types.h +3 −0
  64. tools/include/uapi/linux/bpf.h +97 −6
  65. tools/include/uapi/linux/bpf_perf_event.h +1 −0
  66. tools/include/uapi/linux/tcp.h +357 −0
  67. tools/lib/bpf/.gitignore +0 −1
  68. tools/lib/bpf/Makefile +4 −43
  69. tools/lib/bpf/btf.c +10 −2
  70. tools/lib/bpf/libbpf.c +11 −11
  71. tools/lib/bpf/xsk.c +79 −4
  72. tools/perf/Makefile.perf +0 −1
  73. tools/scripts/Makefile.include +7 −0
  74. tools/testing/selftests/bpf/.gitignore +0 −2
  75. tools/testing/selftests/bpf/Makefile +2 −6
  76. tools/testing/selftests/bpf/README.rst +24 −0
  77. tools/testing/selftests/bpf/benchs/bench_ringbufs.c +1 −1
  78. tools/testing/selftests/bpf/bpf_sockopt_helpers.h +21 −0
  79. tools/testing/selftests/bpf/bpf_tcp_helpers.h +1 −0
  80. tools/testing/selftests/bpf/bpf_testmod/bpf_testmod-events.h +6 −0
  81. tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c +20 −1
  82. tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.h +6 −0
  83. tools/testing/selftests/bpf/prog_tests/atomic_bounds.c +17 −0
  84. tools/testing/selftests/bpf/prog_tests/bind_perm.c +109 −0
  85. tools/testing/selftests/bpf/prog_tests/bpf_iter.c +108 −10
  86. tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c +1 −0
  87. tools/testing/selftests/bpf/prog_tests/btf.c +23 −2
  88. tools/testing/selftests/bpf/prog_tests/check_mtu.c +216 −0
  89. tools/testing/selftests/bpf/prog_tests/cls_redirect.c +1 −0
  90. tools/testing/selftests/bpf/prog_tests/fexit_stress.c +2 −2
  91. tools/testing/selftests/bpf/prog_tests/global_func_args.c +60 −0
  92. tools/testing/selftests/bpf/prog_tests/module_attach.c +27 −0
  93. tools/testing/selftests/bpf/prog_tests/ns_current_pid_tgid.c +60 −58
  94. tools/testing/selftests/bpf/prog_tests/recursion.c +41 −0
  95. tools/testing/selftests/bpf/prog_tests/socket_cookie.c +76 −0
  96. tools/testing/selftests/bpf/prog_tests/sockmap_basic.c +1 −0
  97. tools/testing/selftests/bpf/prog_tests/sockopt_sk.c +28 −0
  98. tools/testing/selftests/bpf/prog_tests/stack_var_off.c +35 −0
  99. tools/testing/selftests/bpf/prog_tests/test_global_funcs.c +8 −0
  100. tools/testing/selftests/bpf/prog_tests/test_ima.c +18 −5
  101. tools/testing/selftests/bpf/prog_tests/test_local_storage.c +1 −1
  102. tools/testing/selftests/bpf/prog_tests/trampoline_count.c +2 −2
  103. tools/testing/selftests/bpf/progs/atomic_bounds.c +24 −0
  104. tools/testing/selftests/bpf/progs/bind_perm.c +45 −0
  105. tools/testing/selftests/bpf/progs/bpf_iter.h +8 −0
  106. tools/testing/selftests/bpf/progs/bpf_iter_task_vma.c +58 −0
  107. tools/testing/selftests/bpf/progs/connect_force_port4.c +8 −0
  108. tools/testing/selftests/bpf/progs/connect_force_port6.c +8 −0
  109. tools/testing/selftests/bpf/progs/ima.c +25 −8
  110. tools/testing/selftests/bpf/progs/lsm.c +69 −0
  111. tools/testing/selftests/bpf/progs/recursion.c +46 −0
  112. tools/testing/selftests/bpf/progs/recvmsg4_prog.c +42 −0
  113. tools/testing/selftests/bpf/progs/recvmsg6_prog.c +48 −0
  114. tools/testing/selftests/bpf/progs/sendmsg4_prog.c +7 −0
  115. tools/testing/selftests/bpf/progs/sendmsg6_prog.c +5 −0
  116. tools/testing/selftests/bpf/progs/socket_cookie_prog.c +38 −9
  117. tools/testing/selftests/bpf/progs/sockopt_sk.c +21 −2
  118. tools/testing/selftests/bpf/progs/test_check_mtu.c +198 −0
  119. tools/testing/selftests/bpf/progs/test_cls_redirect.c +7 −0
  120. tools/testing/selftests/bpf/progs/test_global_func10.c +29 −0
  121. tools/testing/selftests/bpf/progs/test_global_func11.c +19 −0
  122. tools/testing/selftests/bpf/progs/test_global_func12.c +21 −0
  123. tools/testing/selftests/bpf/progs/test_global_func13.c +24 −0
  124. tools/testing/selftests/bpf/progs/test_global_func14.c +21 −0
  125. tools/testing/selftests/bpf/progs/test_global_func15.c +22 −0
  126. tools/testing/selftests/bpf/progs/test_global_func16.c +22 −0
  127. tools/testing/selftests/bpf/progs/test_global_func9.c +132 −0
  128. tools/testing/selftests/bpf/progs/test_global_func_args.c +91 −0
  129. tools/testing/selftests/bpf/progs/test_module_attach.c +10 −0
  130. tools/testing/selftests/bpf/progs/test_ns_current_pid_tgid.c +9 −19
  131. tools/testing/selftests/bpf/progs/test_stack_var_off.c +51 −0
  132. tools/testing/selftests/bpf/test_current_pid_tgid_new_ns.c +0 −160
  133. tools/testing/selftests/bpf/test_flow_dissector.c +1 −1
  134. tools/testing/selftests/bpf/test_progs.c +1 −1
  135. tools/testing/selftests/bpf/test_progs.h +0 −1
  136. tools/testing/selftests/bpf/test_sock_addr.c +12 −74
  137. tools/testing/selftests/bpf/test_socket_cookie.c +0 −208
  138. tools/testing/selftests/bpf/test_verifier.c +85 −3
  139. tools/testing/selftests/bpf/verifier/atomic_and.c +1 −1
  140. tools/testing/selftests/bpf/verifier/atomic_bounds.c +27 −0
  141. tools/testing/selftests/bpf/verifier/atomic_or.c +1 −1
  142. tools/testing/selftests/bpf/verifier/atomic_xor.c +1 −1
  143. tools/testing/selftests/bpf/verifier/basic_stack.c +1 −1
  144. tools/testing/selftests/bpf/verifier/calls.c +2 −2
  145. tools/testing/selftests/bpf/verifier/const_or.c +2 −2
  146. tools/testing/selftests/bpf/verifier/helper_access_var_len.c +6 −6
  147. tools/testing/selftests/bpf/verifier/int_ptr.c +3 −3
  148. tools/testing/selftests/bpf/verifier/jit.c +24 −0
  149. tools/testing/selftests/bpf/verifier/raw_stack.c +5 −5
  150. tools/testing/selftests/bpf/verifier/stack_ptr.c +13 −9
  151. tools/testing/selftests/bpf/verifier/unpriv.c +1 −1
  152. tools/testing/selftests/bpf/verifier/var_off.c +105 −10
  153. tools/testing/selftests/bpf/vmtest.sh +368 −0
  154. tools/testing/selftests/bpf/xdpxceiver.c +108 −117
  155. tools/testing/selftests/bpf/xdpxceiver.h +0 −2
  156. tools/testing/selftests/tc-testing/Makefile +1 −2
Documentation/bpf/bpf_design_QA.rst+6−0
@@ -208,6 +208,12 @@ data structures and compile with kernel internal headers. Both of these · 207 unmodified lines
208208 kernel internals are subject to change and can break with newer kernels
209209 such that the program needs to be adapted accordingly.
210210
211+Q: Are tracepoints part of the stable ABI?
212+------------------------------------------
213+A: NO. Tracepoints are tied to internal implementation details hence they are
214+subject to change and can break with newer kernels. BPF programs need to change
215+accordingly when this happens.
216+
211217 Q: How much stack space a BPF program uses?
212218 -------------------------------------------
213219 A: Currently all program types are limited to 512 bytes of stack
Documentation/bpf/bpf_devel_QA.rst+7−4
@@ -501,16 +501,19 @@ All LLVM releases can be found at: http://releases.llvm.org/ · 500 unmodified lines
501501
502502 Q: Got it, so how do I build LLVM manually anyway?
503503 --------------------------------------------------
504−A: You need cmake and gcc-c++ as build requisites for LLVM. Once you have
505−that set up, proceed with building the latest LLVM and clang version
504+A: We recommend that developers who want the fastest incremental builds
505+use the Ninja build system, you can find it in your system's package
506+manager, usually the package is ninja or ninja-build.
507+
508+You need ninja, cmake and gcc-c++ as build requisites for LLVM. Once you
509+have that set up, proceed with building the latest LLVM and clang version
506510 from the git repositories::
507511
508512 $ git clone https://github.com/llvm/llvm-project.git
509− $ mkdir -p llvm-project/llvm/build/install
513+ $ mkdir -p llvm-project/llvm/build
510514 $ cd llvm-project/llvm/build
511515 $ cmake .. -G "Ninja" -DLLVM_TARGETS_TO_BUILD="BPF;X86" \
512516 -DLLVM_ENABLE_PROJECTS="clang" \
513− -DBUILD_SHARED_LIBS=OFF \
514517 -DCMAKE_BUILD_TYPE=Release \
515518 -DLLVM_BUILD_RUNTIME=OFF
516519 $ ninja
Documentation/networking/filter.rst+15−13
@@ -1048,12 +1048,12 @@ Unlike classic BPF instruction set, eBPF has generic load/store operations:: · 1047 unmodified lines
10481048 Where size is one of: BPF_B or BPF_H or BPF_W or BPF_DW.
10491049
10501050 It also includes atomic operations, which use the immediate field for extra
1051−encoding.
1051+encoding::
10521052
10531053 .imm = BPF_ADD, .code = BPF_ATOMIC | BPF_W | BPF_STX: lock xadd *(u32 *)(dst_reg + off16) += src_reg
10541054 .imm = BPF_ADD, .code = BPF_ATOMIC | BPF_DW | BPF_STX: lock xadd *(u64 *)(dst_reg + off16) += src_reg
10551055
1056−The basic atomic operations supported are:
1056+The basic atomic operations supported are::
10571057
10581058 BPF_ADD
10591059 BPF_AND
@@ -1066,33 +1066,35 @@ memory location addresed by ``dst_reg + off`` is atomically modified, with · 6 unmodified lines
10661066 immediate, then these operations also overwrite ``src_reg`` with the
10671067 value that was in memory before it was modified.
10681068
1069−The more special operations are:
1069+The more special operations are::
10701070
10711071 BPF_XCHG
10721072
10731073 This atomically exchanges ``src_reg`` with the value addressed by ``dst_reg +
1074−off``.
1074+off``. ::
10751075
10761076 BPF_CMPXCHG
10771077
10781078 This atomically compares the value addressed by ``dst_reg + off`` with
1079−``R0``. If they match it is replaced with ``src_reg``, The value that was there
1080−before is loaded back to ``R0``.
1079+``R0``. If they match it is replaced with ``src_reg``. In either case, the
1080+value that was there before is zero-extended and loaded back to ``R0``.
10811081
10821082 Note that 1 and 2 byte atomic operations are not supported.
10831083
1084−Except ``BPF_ADD`` _without_ ``BPF_FETCH`` (for legacy reasons), all 4 byte
1085−atomic operations require alu32 mode. Clang enables this mode by default in
1086−architecture v3 (``-mcpu=v3``). For older versions it can be enabled with
1084+Clang can generate atomic instructions by default when ``-mcpu=v3`` is
1085+enabled. If a lower version for ``-mcpu`` is set, the only atomic instruction
1086+Clang can generate is ``BPF_ADD`` *without* ``BPF_FETCH``. If you need to enable
1087+the atomics features, while keeping a lower ``-mcpu`` version, you can use
10871088 ``-Xclang -target-feature -Xclang +alu32``.
10881089
1089−You may encounter BPF_XADD - this is a legacy name for BPF_ATOMIC, referring to
1090−the exclusive-add operation encoded when the immediate field is zero.
1090+You may encounter ``BPF_XADD`` - this is a legacy name for ``BPF_ATOMIC``,
1091+referring to the exclusive-add operation encoded when the immediate field is
1092+zero.
10911093
1092−eBPF has one 16-byte instruction: BPF_LD | BPF_DW | BPF_IMM which consists
1094+eBPF has one 16-byte instruction: ``BPF_LD | BPF_DW | BPF_IMM`` which consists
10931095 of two consecutive ``struct bpf_insn`` 8-byte blocks and interpreted as single
10941096 instruction that loads 64-bit immediate value into a dst_reg.
1095−Classic BPF has similar instruction: BPF_LD | BPF_W | BPF_IMM which loads
1097+Classic BPF has similar instruction: ``BPF_LD | BPF_W | BPF_IMM`` which loads
10961098 32-bit immediate value into a register.
10971099
10981100 eBPF verifier
Makefile+12−1
@@ -1082,6 +1082,17 @@ ifdef CONFIG_STACK_VALIDATION · 1081 unmodified lines
10821082 endif
10831083 endif
10841084
1085+PHONY += resolve_btfids_clean
1086+
1087+resolve_btfids_O = $(abspath $(objtree))/tools/bpf/resolve_btfids
1088+
1089+# tools/bpf/resolve_btfids directory might not exist
1090+# in output directory, skip its clean in that case
1091+resolve_btfids_clean:
1092+ifneq ($(wildcard $(resolve_btfids_O)),)
1093+ $(Q)$(MAKE) -sC $(srctree)/tools/bpf/resolve_btfids O=$(resolve_btfids_O) clean
1094+endif
1095+
10851096 ifdef CONFIG_BPF
10861097 ifdef CONFIG_DEBUG_INFO_BTF
10871098 ifeq ($(has_libelf),1)
@@ -1491,7 +1502,7 @@ vmlinuxclean: · 403 unmodified lines
14911502 $(Q)$(CONFIG_SHELL) $(srctree)/scripts/link-vmlinux.sh clean
14921503 $(Q)$(if $(ARCH_POSTLINK), $(MAKE) -f $(ARCH_POSTLINK) clean)
14931504
1494−clean: archclean vmlinuxclean
1505+clean: archclean vmlinuxclean resolve_btfids_clean
14951506
14961507 # mrproper - Delete all generated files, including .config
14971508 #
arch/x86/net/bpf_jit_comp.c+158−47
@@ -869,8 +869,31 @@ static void detect_reg_usage(struct bpf_insn *insn, int insn_cnt, · 868 unmodified lines
869869 }
870870 }
871871
872+static int emit_nops(u8 **pprog, int len)
873+{
874+ u8 *prog = *pprog;
875+ int i, noplen, cnt = 0;
876+
877+ while (len > 0) {
878+ noplen = len;
879+
880+ if (noplen > ASM_NOP_MAX)
881+ noplen = ASM_NOP_MAX;
882+
883+ for (i = 0; i < noplen; i++)
884+ EMIT1(ideal_nops[noplen][i]);
885+ len -= noplen;
886+ }
887+
888+ *pprog = prog;
889+
890+ return cnt;
891+}
892+
893+#define INSN_SZ_DIFF (((addrs[i] - addrs[i - 1]) - (prog - temp)))
894+
872895 static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
873− int oldproglen, struct jit_context *ctx)
896+ int oldproglen, struct jit_context *ctx, bool jmp_padding)
874897 {
875898 bool tail_call_reachable = bpf_prog->aux->tail_call_reachable;
876899 struct bpf_insn *insn = bpf_prog->insnsi;
@@ -880,7 +903,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, · 3 unmodified lines
880903 bool seen_exit = false;
881904 u8 temp[BPF_MAX_INSN_SIZE + BPF_INSN_SAFETY];
882905 int i, cnt = 0, excnt = 0;
883− int proglen = 0;
906+ int ilen, proglen = 0;
884907 u8 *prog = temp;
885908 int err;
886909
@@ -894,17 +917,24 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, · 7 unmodified lines
894917 bpf_prog_was_classic(bpf_prog), tail_call_reachable,
895918 bpf_prog->aux->func_idx != 0);
896919 push_callee_regs(&prog, callee_regs_used);
897− addrs[0] = prog - temp;
920+
921+ ilen = prog - temp;
922+ if (image)
923+ memcpy(image + proglen, temp, ilen);
924+ proglen += ilen;
925+ addrs[0] = proglen;
926+ prog = temp;
898927
899928 for (i = 1; i <= insn_cnt; i++, insn++) {
900929 const s32 imm32 = insn->imm;
901930 u32 dst_reg = insn->dst_reg;
902931 u32 src_reg = insn->src_reg;
903932 u8 b2 = 0, b3 = 0;
933+ u8 *start_of_ldx;
904934 s64 jmp_offset;
905935 u8 jmp_cond;
906− int ilen;
907936 u8 *func;
937+ int nops;
908938
909939 switch (insn->code) {
910940 /* ALU */
@@ -1249,12 +1279,30 @@ st: if (is_imm8(insn->off)) · 338 unmodified lines
12491279 case BPF_LDX | BPF_PROBE_MEM | BPF_W:
12501280 case BPF_LDX | BPF_MEM | BPF_DW:
12511281 case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
1282+ if (BPF_MODE(insn->code) == BPF_PROBE_MEM) {
1283+ /* test src_reg, src_reg */
1284+ maybe_emit_mod(&prog, src_reg, src_reg, true); /* always 1 byte */
1285+ EMIT2(0x85, add_2reg(0xC0, src_reg, src_reg));
1286+ /* jne start_of_ldx */
1287+ EMIT2(X86_JNE, 0);
1288+ /* xor dst_reg, dst_reg */
1289+ emit_mov_imm32(&prog, false, dst_reg, 0);
1290+ /* jmp byte_after_ldx */
1291+ EMIT2(0xEB, 0);
1292+
1293+ /* populate jmp_offset for JNE above */
1294+ temp[4] = prog - temp - 5 /* sizeof(test + jne) */;
1295+ start_of_ldx = prog;
1296+ }
12521297 emit_ldx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off);
12531298 if (BPF_MODE(insn->code) == BPF_PROBE_MEM) {
12541299 struct exception_table_entry *ex;
12551300 u8 *_insn = image + proglen;
12561301 s64 delta;
12571302
1303+ /* populate jmp_offset for JMP above */
1304+ start_of_ldx[-1] = prog - start_of_ldx;
1305+
12581306 if (!bpf_prog->aux->extable)
12591307 break;
12601308
@@ -1502,6 +1550,30 @@ emit_cond_jmp: /* Convert BPF opcode to x86 */ · 241 unmodified lines
15021550 }
15031551 jmp_offset = addrs[i + insn->off] - addrs[i];
15041552 if (is_imm8(jmp_offset)) {
1553+ if (jmp_padding) {
1554+ /* To keep the jmp_offset valid, the extra bytes are
1555+ * padded before the jump insn, so we substract the
1556+ * 2 bytes of jmp_cond insn from INSN_SZ_DIFF.
1557+ *
1558+ * If the previous pass already emits an imm8
1559+ * jmp_cond, then this BPF insn won't shrink, so
1560+ * "nops" is 0.
1561+ *
1562+ * On the other hand, if the previous pass emits an
1563+ * imm32 jmp_cond, the extra 4 bytes(*) is padded to
1564+ * keep the image from shrinking further.
1565+ *
1566+ * (*) imm32 jmp_cond is 6 bytes, and imm8 jmp_cond
1567+ * is 2 bytes, so the size difference is 4 bytes.
1568+ */
1569+ nops = INSN_SZ_DIFF - 2;
1570+ if (nops != 0 && nops != 4) {
1571+ pr_err("unexpected jmp_cond padding: %d bytes\n",
1572+ nops);
1573+ return -EFAULT;
1574+ }
1575+ cnt += emit_nops(&prog, nops);
1576+ }
15051577 EMIT2(jmp_cond, jmp_offset);
15061578 } else if (is_simm32(jmp_offset)) {
15071579 EMIT2_off32(0x0F, jmp_cond + 0x10, jmp_offset);
@@ -1524,11 +1596,55 @@ emit_cond_jmp: /* Convert BPF opcode to x86 */ · 16 unmodified lines
15241596 else
15251597 jmp_offset = addrs[i + insn->off] - addrs[i];
15261598
1527− if (!jmp_offset)
1528− /* Optimize out nop jumps */
1599+ if (!jmp_offset) {
1600+ /*
1601+ * If jmp_padding is enabled, the extra nops will
1602+ * be inserted. Otherwise, optimize out nop jumps.
1603+ */
1604+ if (jmp_padding) {
1605+ /* There are 3 possible conditions.
1606+ * (1) This BPF_JA is already optimized out in
1607+ * the previous run, so there is no need
1608+ * to pad any extra byte (0 byte).
1609+ * (2) The previous pass emits an imm8 jmp,
1610+ * so we pad 2 bytes to match the previous
1611+ * insn size.
1612+ * (3) Similarly, the previous pass emits an
1613+ * imm32 jmp, and 5 bytes is padded.
1614+ */
1615+ nops = INSN_SZ_DIFF;
1616+ if (nops != 0 && nops != 2 && nops != 5) {
1617+ pr_err("unexpected nop jump padding: %d bytes\n",
1618+ nops);
1619+ return -EFAULT;
1620+ }
1621+ cnt += emit_nops(&prog, nops);
1622+ }
15291623 break;
1624+ }
15301625 emit_jmp:
15311626 if (is_imm8(jmp_offset)) {
1627+ if (jmp_padding) {
1628+ /* To avoid breaking jmp_offset, the extra bytes
1629+ * are padded before the actual jmp insn, so
1630+ * 2 bytes is substracted from INSN_SZ_DIFF.
1631+ *
1632+ * If the previous pass already emits an imm8
1633+ * jmp, there is nothing to pad (0 byte).
1634+ *
1635+ * If it emits an imm32 jmp (5 bytes) previously
1636+ * and now an imm8 jmp (2 bytes), then we pad
1637+ * (5 - 2 = 3) bytes to stop the image from
1638+ * shrinking further.
1639+ */
1640+ nops = INSN_SZ_DIFF - 2;
1641+ if (nops != 0 && nops != 3) {
1642+ pr_err("unexpected jump padding: %d bytes\n",
1643+ nops);
1644+ return -EFAULT;
1645+ }
1646+ cnt += emit_nops(&prog, INSN_SZ_DIFF - 2);
1647+ }
15321648 EMIT2(0xEB, jmp_offset);
15331649 } else if (is_simm32(jmp_offset)) {
15341650 EMIT1_off32(0xE9, jmp_offset);
@@ -1624,17 +1740,25 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, · 89 unmodified lines
16241740 struct bpf_prog *p, int stack_size, bool mod_ret)
16251741 {
16261742 u8 *prog = *pprog;
1743+ u8 *jmp_insn;
16271744 int cnt = 0;
16281745
1629− if (p->aux->sleepable) {
1630− if (emit_call(&prog, __bpf_prog_enter_sleepable, prog))
1631− return -EINVAL;
1632− } else {
1633− if (emit_call(&prog, __bpf_prog_enter, prog))
1746+ /* arg1: mov rdi, progs[i] */
1747+ emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p);
1748+ if (emit_call(&prog,
1749+ p->aux->sleepable ? __bpf_prog_enter_sleepable :
1750+ __bpf_prog_enter, prog))
16341751 return -EINVAL;
1635− /* remember prog start time returned by __bpf_prog_enter */
1636− emit_mov_reg(&prog, true, BPF_REG_6, BPF_REG_0);
1637− }
1752+ /* remember prog start time returned by __bpf_prog_enter */
1753+ emit_mov_reg(&prog, true, BPF_REG_6, BPF_REG_0);
1754+
1755+ /* if (__bpf_prog_enter*(prog) == 0)
1756+ * goto skip_exec_of_prog;
1757+ */
1758+ EMIT3(0x48, 0x85, 0xC0); /* test rax,rax */
1759+ /* emit 2 nops that will be replaced with JE insn */
1760+ jmp_insn = prog;
1761+ emit_nops(&prog, 2);
16381762
16391763 /* arg1: lea rdi, [rbp - stack_size] */
16401764 EMIT4(0x48, 0x8D, 0x7D, -stack_size);
@@ -1654,43 +1778,23 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, · 13 unmodified lines
16541778 if (mod_ret)
16551779 emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);
16561780
1657− if (p->aux->sleepable) {
1658− if (emit_call(&prog, __bpf_prog_exit_sleepable, prog))
1659− return -EINVAL;
1660− } else {
1661− /* arg1: mov rdi, progs[i] */
1662− emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32,
1663− (u32) (long) p);
1664− /* arg2: mov rsi, rbx <- start time in nsec */
1665− emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_6);
1666− if (emit_call(&prog, __bpf_prog_exit, prog))
1781+ /* replace 2 nops with JE insn, since jmp target is known */
1782+ jmp_insn[0] = X86_JE;
1783+ jmp_insn[1] = prog - jmp_insn - 2;
1784+
1785+ /* arg1: mov rdi, progs[i] */
1786+ emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p);
1787+ /* arg2: mov rsi, rbx <- start time in nsec */
1788+ emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_6);
1789+ if (emit_call(&prog,
1790+ p->aux->sleepable ? __bpf_prog_exit_sleepable :
1791+ __bpf_prog_exit, prog))
16671792 return -EINVAL;
1668− }
16691793
16701794 *pprog = prog;
16711795 return 0;
16721796 }
16731797
1674−static void emit_nops(u8 **pprog, unsigned int len)
1675−{
1676− unsigned int i, noplen;
1677− u8 *prog = *pprog;
1678− int cnt = 0;
1679−
1680− while (len > 0) {
1681− noplen = len;
1682−
1683− if (noplen > ASM_NOP_MAX)
1684− noplen = ASM_NOP_MAX;
1685−
1686− for (i = 0; i < noplen; i++)
1687− EMIT1(ideal_nops[noplen][i]);
1688− len -= noplen;
1689− }
1690−
1691− *pprog = prog;
1692−}
1693−
16941798 static void emit_align(u8 **pprog, u32 align)
16951799 {
16961800 u8 *target, *prog = *pprog;
@@ -2065,6 +2169,9 @@ struct x64_jit_data { · 368 unmodified lines
20652169 struct jit_context ctx;
20662170 };
20672171
2172+#define MAX_PASSES 20
2173+#define PADDING_PASSES (MAX_PASSES - 5)
2174+
20682175 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
20692176 {
20702177 struct bpf_binary_header *header = NULL;
@@ -2074,6 +2181,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 3 unmodified lines
20742181 struct jit_context ctx = {};
20752182 bool tmp_blinded = false;
20762183 bool extra_pass = false;
2184+ bool padding = false;
20772185 u8 *image = NULL;
20782186 int *addrs;
20792187 int pass;
@@ -2110,6 +2218,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 30 unmodified lines
21102218 image = jit_data->image;
21112219 header = jit_data->header;
21122220 extra_pass = true;
2221+ padding = true;
21132222 goto skip_init_addrs;
21142223 }
21152224 addrs = kmalloc_array(prog->len + 1, sizeof(*addrs), GFP_KERNEL);
@@ -2135,8 +2244,10 @@ skip_init_addrs: · 19 unmodified lines
21352244 * may converge on the last pass. In such case do one more
21362245 * pass to emit the final image.
21372246 */
2138− for (pass = 0; pass < 20 || image; pass++) {
2139− proglen = do_jit(prog, addrs, image, oldproglen, &ctx);
2247+ for (pass = 0; pass < MAX_PASSES || image; pass++) {
2248+ if (!padding && pass >= PADDING_PASSES)
2249+ padding = true;
2250+ proglen = do_jit(prog, addrs, image, oldproglen, &ctx, padding);
21402251 if (proglen <= 0) {
21412252 out_image:
21422253 image = NULL;
drivers/net/veth.c+60−34
@@ -35,6 +35,7 @@ · 34 unmodified lines
3535 #define VETH_XDP_HEADROOM (XDP_PACKET_HEADROOM + NET_IP_ALIGN)
3636
3737 #define VETH_XDP_TX_BULK_SIZE 16
38+#define VETH_XDP_BATCH 16
3839
3940 struct veth_stats {
4041 u64 rx_drops;
@@ -562,20 +563,13 @@ static int veth_xdp_tx(struct veth_rq *rq, struct xdp_buff *xdp, · 521 unmodified lines
562563 return 0;
563564 }
564565
565−static struct sk_buff *veth_xdp_rcv_one(struct veth_rq *rq,
566− struct xdp_frame *frame,
567− struct veth_xdp_tx_bq *bq,
568− struct veth_stats *stats)
566+static struct xdp_frame *veth_xdp_rcv_one(struct veth_rq *rq,
567+ struct xdp_frame *frame,
568+ struct veth_xdp_tx_bq *bq,
569+ struct veth_stats *stats)
569570 {
570− void *hard_start = frame->data - frame->headroom;
571− int len = frame->len, delta = 0;
572571 struct xdp_frame orig_frame;
573572 struct bpf_prog *xdp_prog;
574− unsigned int headroom;
575− struct sk_buff *skb;
576−
577− /* bpf_xdp_adjust_head() assures BPF cannot access xdp_frame area */
578− hard_start -= sizeof(struct xdp_frame);
579573
580574 rcu_read_lock();
581575 xdp_prog = rcu_dereference(rq->xdp_prog);
@@ -590,8 +584,8 @@ static struct sk_buff *veth_xdp_rcv_one(struct veth_rq *rq, · 8 unmodified lines
590584
591585 switch (act) {
592586 case XDP_PASS:
593− delta = frame->data - xdp.data;
594− len = xdp.data_end - xdp.data;
587+ if (xdp_update_frame_from_buff(&xdp, frame))
588+ goto err_xdp;
595589 break;
596590 case XDP_TX:
597591 orig_frame = *frame;
@@ -629,19 +623,7 @@ static struct sk_buff *veth_xdp_rcv_one(struct veth_rq *rq, · 31 unmodified lines
629623 }
630624 rcu_read_unlock();
631625
632− headroom = sizeof(struct xdp_frame) + frame->headroom - delta;
633− skb = veth_build_skb(hard_start, headroom, len, frame->frame_sz);
634− if (!skb) {
635− xdp_return_frame(frame);
636− stats->rx_drops++;
637− goto err;
638− }
639−
640− xdp_release_frame(frame);
641− xdp_scrub_frame(frame);
642− skb->protocol = eth_type_trans(skb, rq->dev);
643−err:
644− return skb;
626+ return frame;
645627 err_xdp:
646628 rcu_read_unlock();
647629 xdp_return_frame(frame);
@@ -649,6 +631,37 @@ xdp_xmit: · 1 unmodified line
649631 return NULL;
650632 }
651633
634+/* frames array contains VETH_XDP_BATCH at most */
635+static void veth_xdp_rcv_bulk_skb(struct veth_rq *rq, void **frames,
636+ int n_xdpf, struct veth_xdp_tx_bq *bq,
637+ struct veth_stats *stats)
638+{
639+ void *skbs[VETH_XDP_BATCH];
640+ int i;
641+
642+ if (xdp_alloc_skb_bulk(skbs, n_xdpf,
643+ GFP_ATOMIC | __GFP_ZERO) < 0) {
644+ for (i = 0; i < n_xdpf; i++)
645+ xdp_return_frame(frames[i]);
646+ stats->rx_drops += n_xdpf;
647+
648+ return;
649+ }
650+
651+ for (i = 0; i < n_xdpf; i++) {
652+ struct sk_buff *skb = skbs[i];
653+
654+ skb = __xdp_build_skb_from_frame(frames[i], skb,
655+ rq->dev);
656+ if (!skb) {
657+ xdp_return_frame(frames[i]);
658+ stats->rx_drops++;
659+ continue;
660+ }
661+ napi_gro_receive(&rq->xdp_napi, skb);
662+ }
663+}
664+
652665 static struct sk_buff *veth_xdp_rcv_skb(struct veth_rq *rq,
653666 struct sk_buff *skb,
654667 struct veth_xdp_tx_bq *bq,
@@ -796,32 +809,45 @@ static int veth_xdp_rcv(struct veth_rq *rq, int budget, · 141 unmodified lines
796809 struct veth_xdp_tx_bq *bq,
797810 struct veth_stats *stats)
798811 {
799− int i, done = 0;
812+ int i, done = 0, n_xdpf = 0;
813+ void *xdpf[VETH_XDP_BATCH];
800814
801815 for (i = 0; i < budget; i++) {
802816 void *ptr = __ptr_ring_consume(&rq->xdp_ring);
803− struct sk_buff *skb;
804817
805818 if (!ptr)
806819 break;
807820
808821 if (veth_is_xdp_frame(ptr)) {
822+ /* ndo_xdp_xmit */
809823 struct xdp_frame *frame = veth_ptr_to_xdp(ptr);
810824
811825 stats->xdp_bytes += frame->len;
812− skb = veth_xdp_rcv_one(rq, frame, bq, stats);
826+ frame = veth_xdp_rcv_one(rq, frame, bq, stats);
827+ if (frame) {
828+ /* XDP_PASS */
829+ xdpf[n_xdpf++] = frame;
830+ if (n_xdpf == VETH_XDP_BATCH) {
831+ veth_xdp_rcv_bulk_skb(rq, xdpf, n_xdpf,
832+ bq, stats);
833+ n_xdpf = 0;
834+ }
835+ }
813836 } else {
814− skb = ptr;
837+ /* ndo_start_xmit */
838+ struct sk_buff *skb = ptr;
839+
815840 stats->xdp_bytes += skb->len;
816841 skb = veth_xdp_rcv_skb(rq, skb, bq, stats);
842+ if (skb)
843+ napi_gro_receive(&rq->xdp_napi, skb);
817844 }
818−
819− if (skb)
820− napi_gro_receive(&rq->xdp_napi, skb);
821−
822845 done++;
823846 }
824847
848+ if (n_xdpf)
849+ veth_xdp_rcv_bulk_skb(rq, xdpf, n_xdpf, bq, stats);
850+
825851 u64_stats_update_begin(&rq->stats.syncp);
826852 rq->stats.vs.xdp_redirect += stats->xdp_redirect;
827853 rq->stats.vs.xdp_bytes += stats->xdp_bytes;
include/linux/bpf-cgroup.h+70−31
@@ -23,8 +23,8 @@ struct ctl_table_header; · 22 unmodified lines
2323
2424 #ifdef CONFIG_CGROUP_BPF
2525
26−extern struct static_key_false cgroup_bpf_enabled_key;
27−#define cgroup_bpf_enabled static_branch_unlikely(&cgroup_bpf_enabled_key)
26+extern struct static_key_false cgroup_bpf_enabled_key[MAX_BPF_ATTACH_TYPE];
27+#define cgroup_bpf_enabled(type) static_branch_unlikely(&cgroup_bpf_enabled_key[type])
2828
2929 DECLARE_PER_CPU(struct bpf_cgroup_storage*,
3030 bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]);
@@ -125,7 +125,8 @@ int __cgroup_bpf_run_filter_sk(struct sock *sk, · 94 unmodified lines
125125 int __cgroup_bpf_run_filter_sock_addr(struct sock *sk,
126126 struct sockaddr *uaddr,
127127 enum bpf_attach_type type,
128− void *t_ctx);
128+ void *t_ctx,
129+ u32 *flags);
129130
130131 int __cgroup_bpf_run_filter_sock_ops(struct sock *sk,
131132 struct bpf_sock_ops_kern *sock_ops,
@@ -147,6 +148,10 @@ int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level, · 15 unmodified lines
147148 int __user *optlen, int max_optlen,
148149 int retval);
149150
151+int __cgroup_bpf_run_filter_getsockopt_kern(struct sock *sk, int level,
152+ int optname, void *optval,
153+ int *optlen, int retval);
154+
150155 static inline enum bpf_cgroup_storage_type cgroup_storage_type(
151156 struct bpf_map *map)
152157 {
@@ -185,7 +190,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, · 32 unmodified lines
185190 #define BPF_CGROUP_RUN_PROG_INET_INGRESS(sk, skb) \
186191 ({ \
187192 int __ret = 0; \
188− if (cgroup_bpf_enabled) \
193+ if (cgroup_bpf_enabled(BPF_CGROUP_INET_INGRESS)) \
189194 __ret = __cgroup_bpf_run_filter_skb(sk, skb, \
190195 BPF_CGROUP_INET_INGRESS); \
191196 \
@@ -195,7 +200,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, · 3 unmodified lines
195200 #define BPF_CGROUP_RUN_PROG_INET_EGRESS(sk, skb) \
196201 ({ \
197202 int __ret = 0; \
198− if (cgroup_bpf_enabled && sk && sk == skb->sk) { \
203+ if (cgroup_bpf_enabled(BPF_CGROUP_INET_EGRESS) && sk && sk == skb->sk) { \
199204 typeof(sk) __sk = sk_to_full_sk(sk); \
200205 if (sk_fullsock(__sk)) \
201206 __ret = __cgroup_bpf_run_filter_skb(__sk, skb, \
@@ -207,7 +212,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, · 5 unmodified lines
207212 #define BPF_CGROUP_RUN_SK_PROG(sk, type) \
208213 ({ \
209214 int __ret = 0; \
210− if (cgroup_bpf_enabled) { \
215+ if (cgroup_bpf_enabled(type)) { \
211216 __ret = __cgroup_bpf_run_filter_sk(sk, type); \
212217 } \
213218 __ret; \
@@ -227,33 +232,53 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, · 13 unmodified lines
227232
228233 #define BPF_CGROUP_RUN_SA_PROG(sk, uaddr, type) \
229234 ({ \
235+ u32 __unused_flags; \
230236 int __ret = 0; \
231− if (cgroup_bpf_enabled) \
237+ if (cgroup_bpf_enabled(type)) \
232238 __ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type, \
233− NULL); \
239+ NULL, \
240+ &__unused_flags); \
234241 __ret; \
235242 })
236243
237244 #define BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, type, t_ctx) \
238245 ({ \
246+ u32 __unused_flags; \
239247 int __ret = 0; \
240− if (cgroup_bpf_enabled) { \
248+ if (cgroup_bpf_enabled(type)) { \
241249 lock_sock(sk); \
242250 __ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type, \
243− t_ctx); \
251+ t_ctx, \
252+ &__unused_flags); \
244253 release_sock(sk); \
245254 } \
246255 __ret; \
247256 })
248257
249−#define BPF_CGROUP_RUN_PROG_INET4_BIND_LOCK(sk, uaddr) \
250− BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_INET4_BIND, NULL)
251−
252−#define BPF_CGROUP_RUN_PROG_INET6_BIND_LOCK(sk, uaddr) \
253− BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_INET6_BIND, NULL)
258+/* BPF_CGROUP_INET4_BIND and BPF_CGROUP_INET6_BIND can return extra flags
259+ * via upper bits of return code. The only flag that is supported
260+ * (at bit position 0) is to indicate CAP_NET_BIND_SERVICE capability check
261+ * should be bypassed (BPF_RET_BIND_NO_CAP_NET_BIND_SERVICE).
262+ */
263+#define BPF_CGROUP_RUN_PROG_INET_BIND_LOCK(sk, uaddr, type, bind_flags) \
264+({ \
265+ u32 __flags = 0; \
266+ int __ret = 0; \
267+ if (cgroup_bpf_enabled(type)) { \
268+ lock_sock(sk); \
269+ __ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type, \
270+ NULL, &__flags); \
271+ release_sock(sk); \
272+ if (__flags & BPF_RET_BIND_NO_CAP_NET_BIND_SERVICE) \
273+ *bind_flags |= BIND_NO_CAP_NET_BIND_SERVICE; \
274+ } \
275+ __ret; \
276+})
254277
255−#define BPF_CGROUP_PRE_CONNECT_ENABLED(sk) (cgroup_bpf_enabled && \
256− sk->sk_prot->pre_connect)
278+#define BPF_CGROUP_PRE_CONNECT_ENABLED(sk) \
279+ ((cgroup_bpf_enabled(BPF_CGROUP_INET4_CONNECT) || \
280+ cgroup_bpf_enabled(BPF_CGROUP_INET6_CONNECT)) && \
281+ (sk)->sk_prot->pre_connect)
257282
258283 #define BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr) \
259284 BPF_CGROUP_RUN_SA_PROG(sk, uaddr, BPF_CGROUP_INET4_CONNECT)
@@ -297,7 +322,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, · 37 unmodified lines
297322 #define BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(sock_ops, sk) \
298323 ({ \
299324 int __ret = 0; \
300− if (cgroup_bpf_enabled) \
325+ if (cgroup_bpf_enabled(BPF_CGROUP_SOCK_OPS)) \
301326 __ret = __cgroup_bpf_run_filter_sock_ops(sk, \
302327 sock_ops, \
303328 BPF_CGROUP_SOCK_OPS); \
@@ -307,7 +332,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, · 3 unmodified lines
307332 #define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) \
308333 ({ \
309334 int __ret = 0; \
310− if (cgroup_bpf_enabled && (sock_ops)->sk) { \
335+ if (cgroup_bpf_enabled(BPF_CGROUP_SOCK_OPS) && (sock_ops)->sk) { \
311336 typeof(sk) __sk = sk_to_full_sk((sock_ops)->sk); \
312337 if (__sk && sk_fullsock(__sk)) \
313338 __ret = __cgroup_bpf_run_filter_sock_ops(__sk, \
@@ -320,7 +345,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, · 6 unmodified lines
320345 #define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(type, major, minor, access) \
321346 ({ \
322347 int __ret = 0; \
323− if (cgroup_bpf_enabled) \
348+ if (cgroup_bpf_enabled(BPF_CGROUP_DEVICE)) \
324349 __ret = __cgroup_bpf_check_dev_permission(type, major, minor, \
325350 access, \
326351 BPF_CGROUP_DEVICE); \
@@ -332,7 +357,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, · 5 unmodified lines
332357 #define BPF_CGROUP_RUN_PROG_SYSCTL(head, table, write, buf, count, pos) \
333358 ({ \
334359 int __ret = 0; \
335− if (cgroup_bpf_enabled) \
360+ if (cgroup_bpf_enabled(BPF_CGROUP_SYSCTL)) \
336361 __ret = __cgroup_bpf_run_filter_sysctl(head, table, write, \
337362 buf, count, pos, \
338363 BPF_CGROUP_SYSCTL); \
@@ -343,7 +368,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, · 4 unmodified lines
343368 kernel_optval) \
344369 ({ \
345370 int __ret = 0; \
346− if (cgroup_bpf_enabled) \
371+ if (cgroup_bpf_enabled(BPF_CGROUP_SETSOCKOPT)) \
347372 __ret = __cgroup_bpf_run_filter_setsockopt(sock, level, \
348373 optname, optval, \
349374 optlen, \
@@ -354,7 +379,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, · 4 unmodified lines
354379 #define BPF_CGROUP_GETSOCKOPT_MAX_OPTLEN(optlen) \
355380 ({ \
356381 int __ret = 0; \
357− if (cgroup_bpf_enabled) \
382+ if (cgroup_bpf_enabled(BPF_CGROUP_GETSOCKOPT)) \
358383 get_user(__ret, optlen); \
359384 __ret; \
360385 })
@@ -363,11 +388,24 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, · 2 unmodified lines
363388 max_optlen, retval) \
364389 ({ \
365390 int __ret = retval; \
366− if (cgroup_bpf_enabled) \
367− __ret = __cgroup_bpf_run_filter_getsockopt(sock, level, \
368− optname, optval, \
369− optlen, max_optlen, \
370− retval); \
391+ if (cgroup_bpf_enabled(BPF_CGROUP_GETSOCKOPT)) \
392+ if (!(sock)->sk_prot->bpf_bypass_getsockopt || \
393+ !INDIRECT_CALL_INET_1((sock)->sk_prot->bpf_bypass_getsockopt, \
394+ tcp_bpf_bypass_getsockopt, \
395+ level, optname)) \
396+ __ret = __cgroup_bpf_run_filter_getsockopt( \
397+ sock, level, optname, optval, optlen, \
398+ max_optlen, retval); \
399+ __ret; \
400+})
401+
402+#define BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sock, level, optname, optval, \
403+ optlen, retval) \
404+({ \
405+ int __ret = retval; \
406+ if (cgroup_bpf_enabled(BPF_CGROUP_GETSOCKOPT)) \
407+ __ret = __cgroup_bpf_run_filter_getsockopt_kern( \
408+ sock, level, optname, optval, optlen, retval); \
371409 __ret; \
372410 })
373411
@@ -427,15 +465,14 @@ static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map, · 53 unmodified lines
427465 return 0;
428466 }
429467
430−#define cgroup_bpf_enabled (0)
468+#define cgroup_bpf_enabled(type) (0)
431469 #define BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, type, t_ctx) ({ 0; })
432470 #define BPF_CGROUP_PRE_CONNECT_ENABLED(sk) (0)
433471 #define BPF_CGROUP_RUN_PROG_INET_INGRESS(sk,skb) ({ 0; })
434472 #define BPF_CGROUP_RUN_PROG_INET_EGRESS(sk,skb) ({ 0; })
435473 #define BPF_CGROUP_RUN_PROG_INET_SOCK(sk) ({ 0; })
436474 #define BPF_CGROUP_RUN_PROG_INET_SOCK_RELEASE(sk) ({ 0; })
437−#define BPF_CGROUP_RUN_PROG_INET4_BIND_LOCK(sk, uaddr) ({ 0; })
438−#define BPF_CGROUP_RUN_PROG_INET6_BIND_LOCK(sk, uaddr) ({ 0; })
475+#define BPF_CGROUP_RUN_PROG_INET_BIND_LOCK(sk, uaddr, type, flags) ({ 0; })
439476 #define BPF_CGROUP_RUN_PROG_INET4_POST_BIND(sk) ({ 0; })
440477 #define BPF_CGROUP_RUN_PROG_INET6_POST_BIND(sk) ({ 0; })
441478 #define BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr) ({ 0; })
@@ -452,6 +489,8 @@ static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map, · 10 unmodified lines
452489 #define BPF_CGROUP_GETSOCKOPT_MAX_OPTLEN(optlen) ({ 0; })
453490 #define BPF_CGROUP_RUN_PROG_GETSOCKOPT(sock, level, optname, optval, \
454491 optlen, max_optlen, retval) ({ retval; })
492+#define BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sock, level, optname, optval, \
493+ optlen, retval) ({ retval; })
455494 #define BPF_CGROUP_RUN_PROG_SETSOCKOPT(sock, level, optname, optval, optlen, \
456495 kernel_optval) ({ 0; })
457496
include/linux/bpf.h+43−31
@@ -14,7 +14,6 @@ · 13 unmodified lines
1414 #include <linux/numa.h>
1515 #include <linux/mm_types.h>
1616 #include <linux/wait.h>
17−#include <linux/u64_stats_sync.h>
1817 #include <linux/refcount.h>
1918 #include <linux/mutex.h>
2019 #include <linux/module.h>
@@ -507,12 +506,6 @@ enum bpf_cgroup_storage_type { · 486 unmodified lines
507506 */
508507 #define MAX_BPF_FUNC_ARGS 12
509508
510−struct bpf_prog_stats {
511− u64 cnt;
512− u64 nsecs;
513− struct u64_stats_sync syncp;
514−} __aligned(2 * sizeof(u64));
515−
516509 struct btf_func_model {
517510 u8 ret_size;
518511 u8 nr_args;
@@ -536,7 +529,7 @@ struct btf_func_model { · 17 unmodified lines
536529 /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50
537530 * bytes on x86. Pick a number to fit into BPF_IMAGE_SIZE / 2
538531 */
539−#define BPF_MAX_TRAMP_PROGS 40
532+#define BPF_MAX_TRAMP_PROGS 38
540533
541534 struct bpf_tramp_progs {
542535 struct bpf_prog *progs[BPF_MAX_TRAMP_PROGS];
@@ -568,10 +561,10 @@ int arch_prepare_bpf_trampoline(void *image, void *image_end, · 25 unmodified lines
568561 struct bpf_tramp_progs *tprogs,
569562 void *orig_call);
570563 /* these two functions are called from generated trampoline */
571−u64 notrace __bpf_prog_enter(void);
564+u64 notrace __bpf_prog_enter(struct bpf_prog *prog);
572565 void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start);
573−void notrace __bpf_prog_enter_sleepable(void);
574−void notrace __bpf_prog_exit_sleepable(void);
566+u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog);
567+void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start);
575568
576569 struct bpf_ksym {
577570 unsigned long start;
@@ -845,7 +838,6 @@ struct bpf_prog_aux { · 267 unmodified lines
845838 u32 linfo_idx;
846839 u32 num_exentries;
847840 struct exception_table_entry *extable;
848− struct bpf_prog_stats __percpu *stats;
849841 union {
850842 struct work_struct work;
851843 struct rcu_head rcu;
@@ -1073,6 +1065,34 @@ int bpf_prog_array_copy(struct bpf_prog_array *old_array, · 221 unmodified lines
10731065 struct bpf_prog *include_prog,
10741066 struct bpf_prog_array **new_array);
10751067
1068+/* BPF program asks to bypass CAP_NET_BIND_SERVICE in bind. */
1069+#define BPF_RET_BIND_NO_CAP_NET_BIND_SERVICE (1 << 0)
1070+/* BPF program asks to set CN on the packet. */
1071+#define BPF_RET_SET_CN (1 << 0)
1072+
1073+#define BPF_PROG_RUN_ARRAY_FLAGS(array, ctx, func, ret_flags) \
1074+ ({ \
1075+ struct bpf_prog_array_item *_item; \
1076+ struct bpf_prog *_prog; \
1077+ struct bpf_prog_array *_array; \
1078+ u32 _ret = 1; \
1079+ u32 func_ret; \
1080+ migrate_disable(); \
1081+ rcu_read_lock(); \
1082+ _array = rcu_dereference(array); \
1083+ _item = &_array->items[0]; \
1084+ while ((_prog = READ_ONCE(_item->prog))) { \
1085+ bpf_cgroup_storage_set(_item->cgroup_storage); \
1086+ func_ret = func(_prog, ctx); \
1087+ _ret &= (func_ret & 1); \
1088+ *(ret_flags) |= (func_ret >> 1); \
1089+ _item++; \
1090+ } \
1091+ rcu_read_unlock(); \
1092+ migrate_enable(); \
1093+ _ret; \
1094+ })
1095+
10761096 #define __BPF_PROG_RUN_ARRAY(array, ctx, func, check_non_null) \
10771097 ({ \
10781098 struct bpf_prog_array_item *_item; \
@@ -1120,25 +1140,11 @@ _out: \ · 41 unmodified lines
11201140 */
11211141 #define BPF_PROG_CGROUP_INET_EGRESS_RUN_ARRAY(array, ctx, func) \
11221142 ({ \
1123− struct bpf_prog_array_item *_item; \
1124− struct bpf_prog *_prog; \
1125− struct bpf_prog_array *_array; \
1126− u32 ret; \
1127− u32 _ret = 1; \
1128− u32 _cn = 0; \
1129− migrate_disable(); \
1130− rcu_read_lock(); \
1131− _array = rcu_dereference(array); \
1132− _item = &_array->items[0]; \
1133− while ((_prog = READ_ONCE(_item->prog))) { \
1134− bpf_cgroup_storage_set(_item->cgroup_storage); \
1135− ret = func(_prog, ctx); \
1136− _ret &= (ret & 1); \
1137− _cn |= (ret & 2); \
1138− _item++; \
1139− } \
1140− rcu_read_unlock(); \
1141− migrate_enable(); \
1143+ u32 _flags = 0; \
1144+ bool _cn; \
1145+ u32 _ret; \
1146+ _ret = BPF_PROG_RUN_ARRAY_FLAGS(array, ctx, func, &_flags); \
1147+ _cn = _flags & BPF_RET_SET_CN; \
11421148 if (_ret) \
11431149 _ret = (_cn ? NET_XMIT_CN : NET_XMIT_SUCCESS); \
11441150 else \
@@ -1276,6 +1282,11 @@ static inline bool bpf_allow_ptr_leaks(void) · 131 unmodified lines
12761282 return perfmon_capable();
12771283 }
12781284
1285+static inline bool bpf_allow_uninit_stack(void)
1286+{
1287+ return perfmon_capable();
1288+}
1289+
12791290 static inline bool bpf_allow_ptr_to_map_access(void)
12801291 {
12811292 return perfmon_capable();
@@ -1874,6 +1885,7 @@ extern const struct bpf_func_proto bpf_per_cpu_ptr_proto; · 592 unmodified lines
18741885 extern const struct bpf_func_proto bpf_this_cpu_ptr_proto;
18751886 extern const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto;
18761887 extern const struct bpf_func_proto bpf_sock_from_file_proto;
1888+extern const struct bpf_func_proto bpf_get_socket_ptr_cookie_proto;
18771889
18781890 const struct bpf_func_proto *bpf_tracing_func_proto(
18791891 enum bpf_func_id func_id, const struct bpf_prog *prog);
include/linux/bpf_verifier.h+4−1
@@ -195,7 +195,7 @@ struct bpf_func_state { · 194 unmodified lines
195195 * 0 = main function, 1 = first callee.
196196 */
197197 u32 frameno;
198− /* subprog number == index within subprog_stack_depth
198+ /* subprog number == index within subprog_info
199199 * zero == main subprog
200200 */
201201 u32 subprogno;
@@ -404,6 +404,7 @@ struct bpf_verifier_env { · 202 unmodified lines
404404 u32 used_btf_cnt; /* number of used BTF objects */
405405 u32 id_gen; /* used to generate unique reg IDs */
406406 bool allow_ptr_leaks;
407+ bool allow_uninit_stack;
407408 bool allow_ptr_to_map_access;
408409 bool bpf_capable;
409410 bool bypass_spec_v1;
@@ -470,6 +471,8 @@ bpf_prog_offload_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt); · 60 unmodified lines
470471
471472 int check_ctx_reg(struct bpf_verifier_env *env,
472473 const struct bpf_reg_state *reg, int regno);
474+int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
475+ u32 regno, u32 mem_size);
473476
474477 /* this lives here instead of in bpf.h because it needs to dereference tgt_prog */
475478 static inline u64 bpf_trampoline_compute_key(const struct bpf_prog *tgt_prog,
include/linux/filter.h+18−3
@@ -22,6 +22,7 @@ · 21 unmodified lines
2222 #include <linux/vmalloc.h>
2323 #include <linux/sockptr.h>
2424 #include <crypto/sha1.h>
25+#include <linux/u64_stats_sync.h>
2526
2627 #include <net/sch_generic.h>
2728
@@ -539,6 +540,13 @@ struct bpf_binary_header { · 511 unmodified lines
539540 u8 image[] __aligned(BPF_IMAGE_ALIGNMENT);
540541 };
541542
543+struct bpf_prog_stats {
544+ u64 cnt;
545+ u64 nsecs;
546+ u64 misses;
547+ struct u64_stats_sync syncp;
548+} __aligned(2 * sizeof(u64));
549+
542550 struct bpf_prog {
543551 u16 pages; /* Number of allocated pages */
544552 u16 jited:1, /* Is our filter JIT'ed? */
@@ -557,10 +565,12 @@ struct bpf_prog { · 12 unmodified lines
557565 u32 len; /* Number of filter blocks */
558566 u32 jited_len; /* Size of jited insns in bytes */
559567 u8 tag[BPF_TAG_SIZE];
560− struct bpf_prog_aux *aux; /* Auxiliary fields */
561− struct sock_fprog_kern *orig_prog; /* Original BPF program */
568+ struct bpf_prog_stats __percpu *stats;
569+ int __percpu *active;
562570 unsigned int (*bpf_func)(const void *ctx,
563571 const struct bpf_insn *insn);
572+ struct bpf_prog_aux *aux; /* Auxiliary fields */
573+ struct sock_fprog_kern *orig_prog; /* Original BPF program */
564574 /* Instructions for interpreter */
565575 struct sock_filter insns[0];
566576 struct bpf_insn insnsi[];
@@ -581,7 +591,7 @@ DECLARE_STATIC_KEY_FALSE(bpf_stats_enabled_key); · 14 unmodified lines
581591 struct bpf_prog_stats *__stats; \
582592 u64 __start = sched_clock(); \
583593 __ret = dfunc(ctx, (prog)->insnsi, (prog)->bpf_func); \
584− __stats = this_cpu_ptr(prog->aux->stats); \
594+ __stats = this_cpu_ptr(prog->stats); \
585595 u64_stats_update_begin(&__stats->syncp); \
586596 __stats->cnt++; \
587597 __stats->nsecs += sched_clock() - __start; \
@@ -1298,6 +1308,11 @@ struct bpf_sysctl_kern { · 710 unmodified lines
12981308 u64 tmp_reg;
12991309 };
13001310
1311+#define BPF_SOCKOPT_KERN_BUF_SIZE 32
1312+struct bpf_sockopt_buf {
1313+ u8 data[BPF_SOCKOPT_KERN_BUF_SIZE];
1314+};
1315+
13011316 struct bpf_sockopt_kern {
13021317 struct sock *sk;
13031318 u8 *optval;
include/linux/indirect_call_wrapper.h+6−0
@@ -62,4 +62,10 @@ · 61 unmodified lines
6262 #define INDIRECT_CALL_INET(f, f2, f1, ...) f(__VA_ARGS__)
6363 #endif
6464
65+#if IS_ENABLED(CONFIG_INET)
66+#define INDIRECT_CALL_INET_1(f, f1, ...) INDIRECT_CALL_1(f, f1, __VA_ARGS__)
67+#else
68+#define INDIRECT_CALL_INET_1(f, f1, ...) f(__VA_ARGS__)
69+#endif
70+
6571 #endif
include/linux/netdevice.h+30−2
@@ -3931,14 +3931,42 @@ int xdp_umem_query(struct net_device *dev, u16 queue_id); · 3930 unmodified lines
39313931
39323932 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
39333933 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
3934+int dev_forward_skb_nomtu(struct net_device *dev, struct sk_buff *skb);
39343935 bool is_skb_forwardable(const struct net_device *dev,
39353936 const struct sk_buff *skb);
39363937
3938+static __always_inline bool __is_skb_forwardable(const struct net_device *dev,
3939+ const struct sk_buff *skb,
3940+ const bool check_mtu)
3941+{
3942+ const u32 vlan_hdr_len = 4; /* VLAN_HLEN */
3943+ unsigned int len;
3944+
3945+ if (!(dev->flags & IFF_UP))
3946+ return false;
3947+
3948+ if (!check_mtu)
3949+ return true;
3950+
3951+ len = dev->mtu + dev->hard_header_len + vlan_hdr_len;
3952+ if (skb->len <= len)
3953+ return true;
3954+
3955+ /* if TSO is enabled, we don't care about the length as the packet
3956+ * could be forwarded without being segmented before
3957+ */
3958+ if (skb_is_gso(skb))
3959+ return true;
3960+
3961+ return false;
3962+}
3963+
39373964 static __always_inline int ____dev_forward_skb(struct net_device *dev,
3938− struct sk_buff *skb)
3965+ struct sk_buff *skb,
3966+ const bool check_mtu)
39393967 {
39403968 if (skb_orphan_frags(skb, GFP_ATOMIC) ||
3941− unlikely(!is_skb_forwardable(dev, skb))) {
3969+ unlikely(!__is_skb_forwardable(dev, skb, check_mtu))) {
39423970 atomic_long_inc(&dev->rx_dropped);
39433971 kfree_skb(skb);
39443972 return NET_RX_DROP;
include/linux/skmsg.h+0−1
@@ -390,7 +390,6 @@ static inline struct sk_psock *sk_psock_get(struct sock *sk) · 389 unmodified lines
390390 }
391391
392392 void sk_psock_stop(struct sock *sk, struct sk_psock *psock);
393−void sk_psock_destroy(struct rcu_head *rcu);
394393 void sk_psock_drop(struct sock *sk, struct sk_psock *psock);
395394
396395 static inline void sk_psock_put(struct sock *sk, struct sk_psock *psock)
include/net/inet_common.h+2−0
@@ -41,6 +41,8 @@ int inet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len); · 40 unmodified lines
4141 #define BIND_WITH_LOCK (1 << 1)
4242 /* Called from BPF program. */
4343 #define BIND_FROM_BPF (1 << 2)
44+/* Skip CAP_NET_BIND_SERVICE check. */
45+#define BIND_NO_CAP_NET_BIND_SERVICE (1 << 3)
4446 int __inet_bind(struct sock *sk, struct sockaddr *uaddr, int addr_len,
4547 u32 flags);
4648 int inet_getname(struct socket *sock, struct sockaddr *uaddr,
include/net/sock.h+2−0
@@ -1174,6 +1174,8 @@ struct proto { · 1173 unmodified lines
11741174
11751175 int (*backlog_rcv) (struct sock *sk,
11761176 struct sk_buff *skb);
1177+ bool (*bpf_bypass_getsockopt)(int level,
1178+ int optname);
11771179
11781180 void (*release_cb)(struct sock *sk);
11791181
include/net/tcp.h+1−0
@@ -403,6 +403,7 @@ __poll_t tcp_poll(struct file *file, struct socket *sock, · 402 unmodified lines
403403 struct poll_table_struct *wait);
404404 int tcp_getsockopt(struct sock *sk, int level, int optname,
405405 char __user *optval, int __user *optlen);
406+bool tcp_bpf_bypass_getsockopt(int level, int optname);
406407 int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
407408 unsigned int optlen);
408409 void tcp_set_keepalive(struct sock *sk, int val);
include/net/xdp.h+6−0
@@ -164,6 +164,12 @@ void xdp_warn(const char *msg, const char *func, const int line); · 163 unmodified lines
164164 #define XDP_WARN(msg) xdp_warn(msg, __func__, __LINE__)
165165
166166 struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp);
167+struct sk_buff *__xdp_build_skb_from_frame(struct xdp_frame *xdpf,
168+ struct sk_buff *skb,
169+ struct net_device *dev);
170+struct sk_buff *xdp_build_skb_from_frame(struct xdp_frame *xdpf,
171+ struct net_device *dev);
172+int xdp_alloc_skb_bulk(void **skbs, int n_skb, gfp_t gfp);
167173
168174 static inline
169175 void xdp_convert_frame_to_buff(struct xdp_frame *frame, struct xdp_buff *xdp)
include/trace/bpf_probe.h+10−2
@@ -55,8 +55,7 @@ · 54 unmodified lines
5555 /* tracepoints with more than 12 arguments will hit build error */
5656 #define CAST_TO_U64(...) CONCATENATE(__CAST, COUNT_ARGS(__VA_ARGS__))(__VA_ARGS__)
5757
58−#undef DECLARE_EVENT_CLASS
59−#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \
58+#define __BPF_DECLARE_TRACE(call, proto, args) \
6059 static notrace void \
6160 __bpf_trace_##call(void *__data, proto) \
6261 { \
@@ -64,6 +63,10 @@ __bpf_trace_##call(void *__data, proto) \ · 1 unmodified line
6463 CONCATENATE(bpf_trace_run, COUNT_ARGS(args))(prog, CAST_TO_U64(args)); \
6564 }
6665
66+#undef DECLARE_EVENT_CLASS
67+#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \
68+ __BPF_DECLARE_TRACE(call, PARAMS(proto), PARAMS(args))
69+
6770 /*
6871 * This part is compiled out, it is only here as a build time check
6972 * to make sure that if the tracepoint handling changes, the
@@ -111,6 +114,11 @@ __DEFINE_EVENT(template, call, PARAMS(proto), PARAMS(args), size) · 41 unmodified lines
111114 #define DEFINE_EVENT_PRINT(template, name, proto, args, print) \
112115 DEFINE_EVENT(template, name, PARAMS(proto), PARAMS(args))
113116
117+#undef DECLARE_TRACE
118+#define DECLARE_TRACE(call, proto, args) \
119+ __BPF_DECLARE_TRACE(call, PARAMS(proto), PARAMS(args)) \
120+ __DEFINE_EVENT(call, call, PARAMS(proto), PARAMS(args), 0)
121+
114122 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
115123
116124 #undef DEFINE_EVENT_WRITABLE
include/uapi/linux/bpf.h+97−6
@@ -1656,22 +1656,30 @@ union bpf_attr { · 1655 unmodified lines
16561656 * networking traffic statistics as it provides a global socket
16571657 * identifier that can be assumed unique.
16581658 * Return
1659− * A 8-byte long non-decreasing number on success, or 0 if the
1660− * socket field is missing inside *skb*.
1659+ * A 8-byte long unique number on success, or 0 if the socket
1660+ * field is missing inside *skb*.
16611661 *
16621662 * u64 bpf_get_socket_cookie(struct bpf_sock_addr *ctx)
16631663 * Description
16641664 * Equivalent to bpf_get_socket_cookie() helper that accepts
16651665 * *skb*, but gets socket from **struct bpf_sock_addr** context.
16661666 * Return
1667− * A 8-byte long non-decreasing number.
1667+ * A 8-byte long unique number.
16681668 *
16691669 * u64 bpf_get_socket_cookie(struct bpf_sock_ops *ctx)
16701670 * Description
16711671 * Equivalent to **bpf_get_socket_cookie**\ () helper that accepts
16721672 * *skb*, but gets socket from **struct bpf_sock_ops** context.
16731673 * Return
1674− * A 8-byte long non-decreasing number.
1674+ * A 8-byte long unique number.
1675+ *
1676+ * u64 bpf_get_socket_cookie(struct sock *sk)
1677+ * Description
1678+ * Equivalent to **bpf_get_socket_cookie**\ () helper that accepts
1679+ * *sk*, but gets socket from a BTF **struct sock**. This helper
1680+ * also works for sleepable programs.
1681+ * Return
1682+ * A 8-byte long unique number or 0 if *sk* is NULL.
16751683 *
16761684 * u32 bpf_get_socket_uid(struct sk_buff *skb)
16771685 * Return
@@ -2231,6 +2239,9 @@ union bpf_attr { · 553 unmodified lines
22312239 * * > 0 one of **BPF_FIB_LKUP_RET_** codes explaining why the
22322240 * packet is not forwarded or needs assist from full stack
22332241 *
2242+ * If lookup fails with BPF_FIB_LKUP_RET_FRAG_NEEDED, then the MTU
2243+ * was exceeded and output params->mtu_result contains the MTU.
2244+ *
22342245 * long bpf_sock_hash_update(struct bpf_sock_ops *skops, struct bpf_map *map, void *key, u64 flags)
22352246 * Description
22362247 * Add an entry to, or update a sockhash *map* referencing sockets.
@@ -3836,6 +3847,69 @@ union bpf_attr { · 1599 unmodified lines
38363847 * Return
38373848 * A pointer to a struct socket on success or NULL if the file is
38383849 * not a socket.
3850+ *
3851+ * long bpf_check_mtu(void *ctx, u32 ifindex, u32 *mtu_len, s32 len_diff, u64 flags)
3852+ * Description
3853+
3854+ * Check ctx packet size against exceeding MTU of net device (based
3855+ * on *ifindex*). This helper will likely be used in combination
3856+ * with helpers that adjust/change the packet size.
3857+ *
3858+ * The argument *len_diff* can be used for querying with a planned
3859+ * size change. This allows to check MTU prior to changing packet
3860+ * ctx. Providing an *len_diff* adjustment that is larger than the
3861+ * actual packet size (resulting in negative packet size) will in
3862+ * principle not exceed the MTU, why it is not considered a
3863+ * failure. Other BPF-helpers are needed for performing the
3864+ * planned size change, why the responsability for catch a negative
3865+ * packet size belong in those helpers.
3866+ *
3867+ * Specifying *ifindex* zero means the MTU check is performed
3868+ * against the current net device. This is practical if this isn't
3869+ * used prior to redirect.
3870+ *
3871+ * The Linux kernel route table can configure MTUs on a more
3872+ * specific per route level, which is not provided by this helper.
3873+ * For route level MTU checks use the **bpf_fib_lookup**\ ()
3874+ * helper.
3875+ *
3876+ * *ctx* is either **struct xdp_md** for XDP programs or
3877+ * **struct sk_buff** for tc cls_act programs.
3878+ *
3879+ * The *flags* argument can be a combination of one or more of the
3880+ * following values:
3881+ *
3882+ * **BPF_MTU_CHK_SEGS**
3883+ * This flag will only works for *ctx* **struct sk_buff**.
3884+ * If packet context contains extra packet segment buffers
3885+ * (often knows as GSO skb), then MTU check is harder to
3886+ * check at this point, because in transmit path it is
3887+ * possible for the skb packet to get re-segmented
3888+ * (depending on net device features). This could still be
3889+ * a MTU violation, so this flag enables performing MTU
3890+ * check against segments, with a different violation
3891+ * return code to tell it apart. Check cannot use len_diff.
3892+ *
3893+ * On return *mtu_len* pointer contains the MTU value of the net
3894+ * device. Remember the net device configured MTU is the L3 size,
3895+ * which is returned here and XDP and TX length operate at L2.
3896+ * Helper take this into account for you, but remember when using
3897+ * MTU value in your BPF-code. On input *mtu_len* must be a valid
3898+ * pointer and be initialized (to zero), else verifier will reject
3899+ * BPF program.
3900+ *
3901+ * Return
3902+ * * 0 on success, and populate MTU value in *mtu_len* pointer.
3903+ *
3904+ * * < 0 if any input argument is invalid (*mtu_len* not updated)
3905+ *
3906+ * MTU violations return positive values, but also populate MTU
3907+ * value in *mtu_len* pointer, as this can be needed for
3908+ * implementing PMTU handing:
3909+ *
3910+ * * **BPF_MTU_CHK_RET_FRAG_NEEDED**
3911+ * * **BPF_MTU_CHK_RET_SEGS_TOOBIG**
3912+ *
38393913 */
38403914 #define __BPF_FUNC_MAPPER(FN) \
38413915 FN(unspec), \
@@ -4001,6 +4075,7 @@ union bpf_attr { · 159 unmodified lines
40014075 FN(ktime_get_coarse_ns), \
40024076 FN(ima_inode_hash), \
40034077 FN(sock_from_file), \
4078+ FN(check_mtu), \
40044079 /* */
40054080
40064081 /* integer value in 'imm' field of BPF_CALL instruction selects which helper
@@ -4501,6 +4576,7 @@ struct bpf_prog_info { · 494 unmodified lines
45014576 __aligned_u64 prog_tags;
45024577 __u64 run_time_ns;
45034578 __u64 run_cnt;
4579+ __u64 recursion_misses;
45044580 } __attribute__((aligned(8)));
45054581
45064582 struct bpf_map_info {
@@ -4981,9 +5057,13 @@ struct bpf_fib_lookup { · 474 unmodified lines
49815057 __be16 sport;
49825058 __be16 dport;
49835059
4984− /* total length of packet from network header - used for MTU check */
4985− __u16 tot_len;
5060+ union { /* used for MTU check */
5061+ /* input to lookup */
5062+ __u16 tot_len; /* L3 length from network hdr (iph->tot_len) */
49865063
5064+ /* output: MTU value */
5065+ __u16 mtu_result;
5066+ };
49875067 /* input: L3 device index for lookup
49885068 * output: device index from FIB lookup
49895069 */
@@ -5029,6 +5109,17 @@ struct bpf_redir_neigh { · 39 unmodified lines
50295109 };
50305110 };
50315111
5112+/* bpf_check_mtu flags*/
5113+enum bpf_check_mtu_flags {
5114+ BPF_MTU_CHK_SEGS = (1U << 0),
5115+};
5116+
5117+enum bpf_check_mtu_ret {
5118+ BPF_MTU_CHK_RET_SUCCESS, /* check and lookup successful */
5119+ BPF_MTU_CHK_RET_FRAG_NEEDED, /* fragmentation required to fwd */
5120+ BPF_MTU_CHK_RET_SEGS_TOOBIG, /* GSO re-segmentation needed to fwd */
5121+};
5122+
50325123 enum bpf_task_fd_type {
50335124 BPF_FD_TYPE_RAW_TRACEPOINT, /* tp name */
50345125 BPF_FD_TYPE_TRACEPOINT, /* tp name */
kernel/bpf/bpf_iter.c+1−1
@@ -287,7 +287,7 @@ int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info) · 286 unmodified lines
287287 {
288288 struct bpf_iter_target_info *tinfo;
289289
290− tinfo = kmalloc(sizeof(*tinfo), GFP_KERNEL);
290+ tinfo = kzalloc(sizeof(*tinfo), GFP_KERNEL);
291291 if (!tinfo)
292292 return -ENOMEM;
293293
kernel/bpf/bpf_lru_list.c+4−3
@@ -502,13 +502,14 @@ struct bpf_lru_node *bpf_lru_pop_free(struct bpf_lru *lru, u32 hash) · 501 unmodified lines
502502 static void bpf_common_lru_push_free(struct bpf_lru *lru,
503503 struct bpf_lru_node *node)
504504 {
505+ u8 node_type = READ_ONCE(node->type);
505506 unsigned long flags;
506507
507− if (WARN_ON_ONCE(node->type == BPF_LRU_LIST_T_FREE) ||
508− WARN_ON_ONCE(node->type == BPF_LRU_LOCAL_LIST_T_FREE))
508+ if (WARN_ON_ONCE(node_type == BPF_LRU_LIST_T_FREE) ||
509+ WARN_ON_ONCE(node_type == BPF_LRU_LOCAL_LIST_T_FREE))
509510 return;
510511
511− if (node->type == BPF_LRU_LOCAL_LIST_T_PENDING) {
512+ if (node_type == BPF_LRU_LOCAL_LIST_T_PENDING) {
512513 struct bpf_lru_locallist *loc_l;
513514
514515 loc_l = per_cpu_ptr(lru->common_lru.local_list, node->cpu);
kernel/bpf/btf.c+55−21
@@ -3540,11 +3540,6 @@ static s32 btf_datasec_check_meta(struct btf_verifier_env *env, · 3539 unmodified lines
35403540 return -EINVAL;
35413541 }
35423542
3543− if (!btf_type_vlen(t)) {
3544− btf_verifier_log_type(env, t, "vlen == 0");
3545− return -EINVAL;
3546− }
3547−
35483543 if (!t->size) {
35493544 btf_verifier_log_type(env, t, "size == 0");
35503545 return -EINVAL;
@@ -5296,15 +5291,16 @@ int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *pr · 1745 unmodified lines
52965291 * Only PTR_TO_CTX and SCALAR_VALUE states are recognized.
52975292 */
52985293 int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog,
5299− struct bpf_reg_state *reg)
5294+ struct bpf_reg_state *regs)
53005295 {
53015296 struct bpf_verifier_log *log = &env->log;
53025297 struct bpf_prog *prog = env->prog;
53035298 struct btf *btf = prog->aux->btf;
53045299 const struct btf_param *args;
5305− const struct btf_type *t;
5306− u32 i, nargs, btf_id;
5300+ const struct btf_type *t, *ref_t;
5301+ u32 i, nargs, btf_id, type_size;
53075302 const char *tname;
5303+ bool is_global;
53085304
53095305 if (!prog->aux->func_info)
53105306 return -EINVAL;
@@ -5338,38 +5334,57 @@ int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, · 27 unmodified lines
53385334 bpf_log(log, "Function %s has %d > 5 args\n", tname, nargs);
53395335 goto out;
53405336 }
5337+
5338+ is_global = prog->aux->func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL;
53415339 /* check that BTF function arguments match actual types that the
53425340 * verifier sees.
53435341 */
53445342 for (i = 0; i < nargs; i++) {
5343+ struct bpf_reg_state *reg = &regs[i + 1];
5344+
53455345 t = btf_type_by_id(btf, args[i].type);
53465346 while (btf_type_is_modifier(t))
53475347 t = btf_type_by_id(btf, t->type);
53485348 if (btf_type_is_int(t) || btf_type_is_enum(t)) {
5349− if (reg[i + 1].type == SCALAR_VALUE)
5349+ if (reg->type == SCALAR_VALUE)
53505350 continue;
53515351 bpf_log(log, "R%d is not a scalar\n", i + 1);
53525352 goto out;
53535353 }
53545354 if (btf_type_is_ptr(t)) {
5355− if (reg[i + 1].type == SCALAR_VALUE) {
5356− bpf_log(log, "R%d is not a pointer\n", i + 1);
5357− goto out;
5358− }
53595355 /* If function expects ctx type in BTF check that caller
53605356 * is passing PTR_TO_CTX.
53615357 */
53625358 if (btf_get_prog_ctx_type(log, btf, t, prog->type, i)) {
5363− if (reg[i + 1].type != PTR_TO_CTX) {
5359+ if (reg->type != PTR_TO_CTX) {
53645360 bpf_log(log,
53655361 "arg#%d expected pointer to ctx, but got %s\n",
53665362 i, btf_kind_str[BTF_INFO_KIND(t->info)]);
53675363 goto out;
53685364 }
5369− if (check_ctx_reg(env, &reg[i + 1], i + 1))
5365+ if (check_ctx_reg(env, reg, i + 1))
53705366 goto out;
53715367 continue;
53725368 }
5369+
5370+ if (!is_global)
5371+ goto out;
5372+
5373+ t = btf_type_skip_modifiers(btf, t->type, NULL);
5374+
5375+ ref_t = btf_resolve_size(btf, t, &type_size);
5376+ if (IS_ERR(ref_t)) {
5377+ bpf_log(log,
5378+ "arg#%d reference type('%s %s') size cannot be determined: %ld\n",
5379+ i, btf_type_str(t), btf_name_by_offset(btf, t->name_off),
5380+ PTR_ERR(ref_t));
5381+ goto out;
5382+ }
5383+
5384+ if (check_mem_reg(env, reg, i + 1, type_size))
5385+ goto out;
5386+
5387+ continue;
53735388 }
53745389 bpf_log(log, "Unrecognized arg#%d type %s\n",
53755390 i, btf_kind_str[BTF_INFO_KIND(t->info)]);
@@ -5393,14 +5408,14 @@ out: · 17 unmodified lines
53935408 * (either PTR_TO_CTX or SCALAR_VALUE).
53945409 */
53955410 int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog,
5396− struct bpf_reg_state *reg)
5411+ struct bpf_reg_state *regs)
53975412 {
53985413 struct bpf_verifier_log *log = &env->log;
53995414 struct bpf_prog *prog = env->prog;
54005415 enum bpf_prog_type prog_type = prog->type;
54015416 struct btf *btf = prog->aux->btf;
54025417 const struct btf_param *args;
5403− const struct btf_type *t;
5418+ const struct btf_type *t, *ref_t;
54045419 u32 i, nargs, btf_id;
54055420 const char *tname;
54065421
@@ -5464,16 +5479,35 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, · 57 unmodified lines
54645479 * Only PTR_TO_CTX and SCALAR are supported atm.
54655480 */
54665481 for (i = 0; i < nargs; i++) {
5482+ struct bpf_reg_state *reg = &regs[i + 1];
5483+
54675484 t = btf_type_by_id(btf, args[i].type);
54685485 while (btf_type_is_modifier(t))
54695486 t = btf_type_by_id(btf, t->type);
54705487 if (btf_type_is_int(t) || btf_type_is_enum(t)) {
5471− reg[i + 1].type = SCALAR_VALUE;
5488+ reg->type = SCALAR_VALUE;
54725489 continue;
54735490 }
5474− if (btf_type_is_ptr(t) &&
5475− btf_get_prog_ctx_type(log, btf, t, prog_type, i)) {
5476− reg[i + 1].type = PTR_TO_CTX;
5491+ if (btf_type_is_ptr(t)) {
5492+ if (btf_get_prog_ctx_type(log, btf, t, prog_type, i)) {
5493+ reg->type = PTR_TO_CTX;
5494+ continue;
5495+ }
5496+
5497+ t = btf_type_skip_modifiers(btf, t->type, NULL);
5498+
5499+ ref_t = btf_resolve_size(btf, t, &reg->mem_size);
5500+ if (IS_ERR(ref_t)) {
5501+ bpf_log(log,
5502+ "arg#%d reference type('%s %s') size cannot be determined: %ld\n",
5503+ i, btf_type_str(t), btf_name_by_offset(btf, t->name_off),
5504+ PTR_ERR(ref_t));
5505+ return -EINVAL;
5506+ }
5507+
5508+ reg->type = PTR_TO_MEM_OR_NULL;
5509+ reg->id = ++env->id_gen;
5510+
54775511 continue;
54785512 }
54795513 bpf_log(log, "Arg#%d type %s in %s() is not supported yet.\n",
kernel/bpf/cgroup.c+103−17
@@ -19,7 +19,7 @@ · 18 unmodified lines
1919
2020 #include "../cgroup/cgroup-internal.h"
2121
22−DEFINE_STATIC_KEY_FALSE(cgroup_bpf_enabled_key);
22+DEFINE_STATIC_KEY_ARRAY_FALSE(cgroup_bpf_enabled_key, MAX_BPF_ATTACH_TYPE);
2323 EXPORT_SYMBOL(cgroup_bpf_enabled_key);
2424
2525 void cgroup_bpf_offline(struct cgroup *cgrp)
@@ -128,7 +128,7 @@ static void cgroup_bpf_release(struct work_struct *work) · 102 unmodified lines
128128 if (pl->link)
129129 bpf_cgroup_link_auto_detach(pl->link);
130130 kfree(pl);
131− static_branch_dec(&cgroup_bpf_enabled_key);
131+ static_branch_dec(&cgroup_bpf_enabled_key[type]);
132132 }
133133 old_array = rcu_dereference_protected(
134134 cgrp->bpf.effective[type],
@@ -499,7 +499,7 @@ int __cgroup_bpf_attach(struct cgroup *cgrp, · 364 unmodified lines
499499 if (old_prog)
500500 bpf_prog_put(old_prog);
501501 else
502− static_branch_inc(&cgroup_bpf_enabled_key);
502+ static_branch_inc(&cgroup_bpf_enabled_key[type]);
503503 bpf_cgroup_storages_link(new_storage, cgrp, type);
504504 return 0;
505505
@@ -698,7 +698,7 @@ int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, · 192 unmodified lines
698698 cgrp->bpf.flags[type] = 0;
699699 if (old_prog)
700700 bpf_prog_put(old_prog);
701− static_branch_dec(&cgroup_bpf_enabled_key);
701+ static_branch_dec(&cgroup_bpf_enabled_key[type]);
702702 return 0;
703703
704704 cleanup:
@@ -1055,6 +1055,8 @@ EXPORT_SYMBOL(__cgroup_bpf_run_filter_sk); · 350 unmodified lines
10551055 * @uaddr: sockaddr struct provided by user
10561056 * @type: The type of program to be exectuted
10571057 * @t_ctx: Pointer to attach type specific context
1058+ * @flags: Pointer to u32 which contains higher bits of BPF program
1059+ * return value (OR'ed together).
10581060 *
10591061 * socket is expected to be of type INET or INET6.
10601062 *
@@ -1064,7 +1066,8 @@ EXPORT_SYMBOL(__cgroup_bpf_run_filter_sk); · 3 unmodified lines
10641066 int __cgroup_bpf_run_filter_sock_addr(struct sock *sk,
10651067 struct sockaddr *uaddr,
10661068 enum bpf_attach_type type,
1067− void *t_ctx)
1069+ void *t_ctx,
1070+ u32 *flags)
10681071 {
10691072 struct bpf_sock_addr_kern ctx = {
10701073 .sk = sk,
@@ -1087,7 +1090,8 @@ int __cgroup_bpf_run_filter_sock_addr(struct sock *sk, · 16 unmodified lines
10871090 }
10881091
10891092 cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
1090− ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], &ctx, BPF_PROG_RUN);
1093+ ret = BPF_PROG_RUN_ARRAY_FLAGS(cgrp->bpf.effective[type], &ctx,
1094+ BPF_PROG_RUN, flags);
10911095
10921096 return ret == 1 ? 0 : -EPERM;
10931097 }
@@ -1298,7 +1302,8 @@ static bool __cgroup_bpf_prog_array_is_empty(struct cgroup *cgrp, · 204 unmodified lines
12981302 return empty;
12991303 }
13001304
1301−static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen)
1305+static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen,
1306+ struct bpf_sockopt_buf *buf)
13021307 {
13031308 if (unlikely(max_optlen < 0))
13041309 return -EINVAL;
@@ -1310,6 +1315,15 @@ static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen) · 5 unmodified lines
13101315 max_optlen = PAGE_SIZE;
13111316 }
13121317
1318+ if (max_optlen <= sizeof(buf->data)) {
1319+ /* When the optval fits into BPF_SOCKOPT_KERN_BUF_SIZE
1320+ * bytes avoid the cost of kzalloc.
1321+ */
1322+ ctx->optval = buf->data;
1323+ ctx->optval_end = ctx->optval + max_optlen;
1324+ return max_optlen;
1325+ }
1326+
13131327 ctx->optval = kzalloc(max_optlen, GFP_USER);
13141328 if (!ctx->optval)
13151329 return -ENOMEM;
@@ -1319,16 +1333,26 @@ static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen) · 3 unmodified lines
13191333 return max_optlen;
13201334 }
13211335
1322−static void sockopt_free_buf(struct bpf_sockopt_kern *ctx)
1336+static void sockopt_free_buf(struct bpf_sockopt_kern *ctx,
1337+ struct bpf_sockopt_buf *buf)
13231338 {
1339+ if (ctx->optval == buf->data)
1340+ return;
13241341 kfree(ctx->optval);
13251342 }
13261343
1344+static bool sockopt_buf_allocated(struct bpf_sockopt_kern *ctx,
1345+ struct bpf_sockopt_buf *buf)
1346+{
1347+ return ctx->optval != buf->data;
1348+}
1349+
13271350 int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level,
13281351 int *optname, char __user *optval,
13291352 int *optlen, char **kernel_optval)
13301353 {
13311354 struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
1355+ struct bpf_sockopt_buf buf = {};
13321356 struct bpf_sockopt_kern ctx = {
13331357 .sk = sk,
13341358 .level = *level,
@@ -1340,8 +1364,7 @@ int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level, · 5 unmodified lines
13401364 * attached to the hook so we don't waste time allocating
13411365 * memory and locking the socket.
13421366 */
1343− if (!cgroup_bpf_enabled ||
1344− __cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_SETSOCKOPT))
1367+ if (__cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_SETSOCKOPT))
13451368 return 0;
13461369
13471370 /* Allocate a bit more than the initial user buffer for
@@ -1350,7 +1373,7 @@ int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level, · 2 unmodified lines
13501373 */
13511374 max_optlen = max_t(int, 16, *optlen);
13521375
1353− max_optlen = sockopt_alloc_buf(&ctx, max_optlen);
1376+ max_optlen = sockopt_alloc_buf(&ctx, max_optlen, &buf);
13541377 if (max_optlen < 0)
13551378 return max_optlen;
13561379
@@ -1390,14 +1413,31 @@ int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level, · 33 unmodified lines
13901413 */
13911414 if (ctx.optlen != 0) {
13921415 *optlen = ctx.optlen;
1393− *kernel_optval = ctx.optval;
1416+ /* We've used bpf_sockopt_kern->buf as an intermediary
1417+ * storage, but the BPF program indicates that we need
1418+ * to pass this data to the kernel setsockopt handler.
1419+ * No way to export on-stack buf, have to allocate a
1420+ * new buffer.
1421+ */
1422+ if (!sockopt_buf_allocated(&ctx, &buf)) {
1423+ void *p = kmalloc(ctx.optlen, GFP_USER);
1424+
1425+ if (!p) {
1426+ ret = -ENOMEM;
1427+ goto out;
1428+ }
1429+ memcpy(p, ctx.optval, ctx.optlen);
1430+ *kernel_optval = p;
1431+ } else {
1432+ *kernel_optval = ctx.optval;
1433+ }
13941434 /* export and don't free sockopt buf */
13951435 return 0;
13961436 }
13971437 }
13981438
13991439 out:
1400− sockopt_free_buf(&ctx);
1440+ sockopt_free_buf(&ctx, &buf);
14011441 return ret;
14021442 }
14031443
@@ -1407,6 +1447,7 @@ int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level, · 3 unmodified lines
14071447 int retval)
14081448 {
14091449 struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
1450+ struct bpf_sockopt_buf buf = {};
14101451 struct bpf_sockopt_kern ctx = {
14111452 .sk = sk,
14121453 .level = level,
@@ -1419,13 +1460,12 @@ int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level, · 6 unmodified lines
14191460 * attached to the hook so we don't waste time allocating
14201461 * memory and locking the socket.
14211462 */
1422− if (!cgroup_bpf_enabled ||
1423− __cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_GETSOCKOPT))
1463+ if (__cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_GETSOCKOPT))
14241464 return retval;
14251465
14261466 ctx.optlen = max_optlen;
14271467
1428− max_optlen = sockopt_alloc_buf(&ctx, max_optlen);
1468+ max_optlen = sockopt_alloc_buf(&ctx, max_optlen, &buf);
14291469 if (max_optlen < 0)
14301470 return max_optlen;
14311471
@@ -1488,9 +1528,55 @@ int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level, · 56 unmodified lines
14881528 ret = ctx.retval;
14891529
14901530 out:
1491− sockopt_free_buf(&ctx);
1531+ sockopt_free_buf(&ctx, &buf);
14921532 return ret;
14931533 }
1534+
1535+int __cgroup_bpf_run_filter_getsockopt_kern(struct sock *sk, int level,
1536+ int optname, void *optval,
1537+ int *optlen, int retval)
1538+{
1539+ struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
1540+ struct bpf_sockopt_kern ctx = {
1541+ .sk = sk,
1542+ .level = level,
1543+ .optname = optname,
1544+ .retval = retval,
1545+ .optlen = *optlen,
1546+ .optval = optval,
1547+ .optval_end = optval + *optlen,
1548+ };
1549+ int ret;
1550+
1551+ /* Note that __cgroup_bpf_run_filter_getsockopt doesn't copy
1552+ * user data back into BPF buffer when reval != 0. This is
1553+ * done as an optimization to avoid extra copy, assuming
1554+ * kernel won't populate the data in case of an error.
1555+ * Here we always pass the data and memset() should
1556+ * be called if that data shouldn't be "exported".
1557+ */
1558+
1559+ ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[BPF_CGROUP_GETSOCKOPT],
1560+ &ctx, BPF_PROG_RUN);
1561+ if (!ret)
1562+ return -EPERM;
1563+
1564+ if (ctx.optlen > *optlen)
1565+ return -EFAULT;
1566+
1567+ /* BPF programs only allowed to set retval to 0, not some
1568+ * arbitrary value.
1569+ */
1570+ if (ctx.retval != 0 && ctx.retval != retval)
1571+ return -EFAULT;
1572+
1573+ /* BPF programs can shrink the buffer, export the modifications.
1574+ */
1575+ if (ctx.optlen != 0)
1576+ *optlen = ctx.optlen;
1577+
1578+ return ctx.retval;
1579+}
14941580 #endif
14951581
14961582 static ssize_t sysctl_cpy_dir(const struct ctl_dir *dir, char **bufp,
kernel/bpf/core.c+14−4
@@ -91,6 +91,12 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag · 90 unmodified lines
9191 vfree(fp);
9292 return NULL;
9393 }
94+ fp->active = alloc_percpu_gfp(int, GFP_KERNEL_ACCOUNT | gfp_extra_flags);
95+ if (!fp->active) {
96+ vfree(fp);
97+ kfree(aux);
98+ return NULL;
99+ }
94100
95101 fp->pages = size / PAGE_SIZE;
96102 fp->aux = aux;
@@ -114,8 +120,9 @@ struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags) · 17 unmodified lines
114120 if (!prog)
115121 return NULL;
116122
117− prog->aux->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags);
118− if (!prog->aux->stats) {
123+ prog->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags);
124+ if (!prog->stats) {
125+ free_percpu(prog->active);
119126 kfree(prog->aux);
120127 vfree(prog);
121128 return NULL;
@@ -124,7 +131,7 @@ struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags) · 2 unmodified lines
124131 for_each_possible_cpu(cpu) {
125132 struct bpf_prog_stats *pstats;
126133
127− pstats = per_cpu_ptr(prog->aux->stats, cpu);
134+ pstats = per_cpu_ptr(prog->stats, cpu);
128135 u64_stats_init(&pstats->syncp);
129136 }
130137 return prog;
@@ -238,6 +245,8 @@ struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size, · 107 unmodified lines
238245 * reallocated structure.
239246 */
240247 fp_old->aux = NULL;
248+ fp_old->stats = NULL;
249+ fp_old->active = NULL;
241250 __bpf_prog_free(fp_old);
242251 }
243252
@@ -249,10 +258,11 @@ void __bpf_prog_free(struct bpf_prog *fp) · 5 unmodified lines
249258 if (fp->aux) {
250259 mutex_destroy(&fp->aux->used_maps_mutex);
251260 mutex_destroy(&fp->aux->dst_mutex);
252− free_percpu(fp->aux->stats);
253261 kfree(fp->aux->poke_tab);
254262 kfree(fp->aux);
255263 }
264+ free_percpu(fp->stats);
265+ free_percpu(fp->active);
256266 vfree(fp);
257267 }
258268
kernel/bpf/cpumap.c+2−44
@@ -141,49 +141,6 @@ static void cpu_map_kthread_stop(struct work_struct *work) · 140 unmodified lines
141141 kthread_stop(rcpu->kthread);
142142 }
143143
144−static struct sk_buff *cpu_map_build_skb(struct xdp_frame *xdpf,
145− struct sk_buff *skb)
146−{
147− unsigned int hard_start_headroom;
148− unsigned int frame_size;
149− void *pkt_data_start;
150−
151− /* Part of headroom was reserved to xdpf */
152− hard_start_headroom = sizeof(struct xdp_frame) + xdpf->headroom;
153−
154− /* Memory size backing xdp_frame data already have reserved
155− * room for build_skb to place skb_shared_info in tailroom.
156− */
157− frame_size = xdpf->frame_sz;
158−
159− pkt_data_start = xdpf->data - hard_start_headroom;
160− skb = build_skb_around(skb, pkt_data_start, frame_size);
161− if (unlikely(!skb))
162− return NULL;
163−
164− skb_reserve(skb, hard_start_headroom);
165− __skb_put(skb, xdpf->len);
166− if (xdpf->metasize)
167− skb_metadata_set(skb, xdpf->metasize);
168−
169− /* Essential SKB info: protocol and skb->dev */
170− skb->protocol = eth_type_trans(skb, xdpf->dev_rx);
171−
172− /* Optional SKB info, currently missing:
173− * - HW checksum info (skb->ip_summed)
174− * - HW RX hash (skb_set_hash)
175− * - RX ring dev queue index (skb_record_rx_queue)
176− */
177−
178− /* Until page_pool get SKB return path, release DMA here */
179− xdp_release_frame(xdpf);
180−
181− /* Allow SKB to reuse area used by xdp_frame */
182− xdp_scrub_frame(xdpf);
183−
184− return skb;
185−}
186−
187144 static void __cpu_map_ring_cleanup(struct ptr_ring *ring)
188145 {
189146 /* The tear-down procedure should have made sure that queue is
@@ -350,7 +307,8 @@ static int cpu_map_kthread_run(void *data) · 160 unmodified lines
350307 struct sk_buff *skb = skbs[i];
351308 int ret;
352309
353− skb = cpu_map_build_skb(xdpf, skb);
310+ skb = __xdp_build_skb_from_frame(xdpf, skb,
311+ xdpf->dev_rx);
354312 if (!skb) {
355313 xdp_return_frame(xdpf);
356314 continue;
kernel/bpf/devmap.c+1−3
@@ -802,9 +802,7 @@ static int dev_map_notification(struct notifier_block *notifier, · 801 unmodified lines
802802 break;
803803
804804 /* will be freed in free_netdev() */
805− netdev->xdp_bulkq =
806− __alloc_percpu_gfp(sizeof(struct xdp_dev_bulk_queue),
807− sizeof(void *), GFP_ATOMIC);
805+ netdev->xdp_bulkq = alloc_percpu(struct xdp_dev_bulk_queue);
808806 if (!netdev->xdp_bulkq)
809807 return NOTIFY_BAD;
810808
kernel/bpf/disasm.c+1−1
@@ -161,7 +161,7 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, · 160 unmodified lines
161161 insn->dst_reg,
162162 insn->off, insn->src_reg);
163163 else if (BPF_MODE(insn->code) == BPF_ATOMIC &&
164− (insn->imm == BPF_ADD || insn->imm == BPF_ADD ||
164+ (insn->imm == BPF_ADD || insn->imm == BPF_AND ||
165165 insn->imm == BPF_OR || insn->imm == BPF_XOR)) {
166166 verbose(cbs->private_data, "(%02x) lock *(%s *)(r%d %+d) %s r%d\n",
167167 insn->code,
kernel/bpf/hashtab.c+2−2
@@ -1148,7 +1148,7 @@ static int __htab_percpu_map_update_elem(struct bpf_map *map, void *key, · 1147 unmodified lines
11481148 /* unknown flags */
11491149 return -EINVAL;
11501150
1151− WARN_ON_ONCE(!rcu_read_lock_held());
1151+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
11521152
11531153 key_size = map->key_size;
11541154
@@ -1202,7 +1202,7 @@ static int __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, · 47 unmodified lines
12021202 /* unknown flags */
12031203 return -EINVAL;
12041204
1205− WARN_ON_ONCE(!rcu_read_lock_held());
1205+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
12061206
12071207 key_size = map->key_size;
12081208
kernel/bpf/helpers.c+4−8
@@ -720,14 +720,6 @@ bpf_base_func_proto(enum bpf_func_id func_id) · 719 unmodified lines
720720 return &bpf_spin_lock_proto;
721721 case BPF_FUNC_spin_unlock:
722722 return &bpf_spin_unlock_proto;
723− case BPF_FUNC_trace_printk:
724− if (!perfmon_capable())
725− return NULL;
726− return bpf_get_trace_printk_proto();
727− case BPF_FUNC_snprintf_btf:
728− if (!perfmon_capable())
729− return NULL;
730− return &bpf_snprintf_btf_proto;
731723 case BPF_FUNC_jiffies64:
732724 return &bpf_jiffies64_proto;
733725 case BPF_FUNC_per_cpu_ptr:
@@ -742,6 +734,8 @@ bpf_base_func_proto(enum bpf_func_id func_id) · 8 unmodified lines
742734 return NULL;
743735
744736 switch (func_id) {
737+ case BPF_FUNC_trace_printk:
738+ return bpf_get_trace_printk_proto();
745739 case BPF_FUNC_get_current_task:
746740 return &bpf_get_current_task_proto;
747741 case BPF_FUNC_probe_read_user:
@@ -752,6 +746,8 @@ bpf_base_func_proto(enum bpf_func_id func_id) · 4 unmodified lines
752746 return &bpf_probe_read_user_str_proto;
753747 case BPF_FUNC_probe_read_kernel_str:
754748 return &bpf_probe_read_kernel_str_proto;
749+ case BPF_FUNC_snprintf_btf:
750+ return &bpf_snprintf_btf_proto;
755751 default:
756752 return NULL;
757753 }
kernel/bpf/syscall.c+11−5
@@ -1731,25 +1731,28 @@ static int bpf_prog_release(struct inode *inode, struct file *filp) · 1730 unmodified lines
17311731 static void bpf_prog_get_stats(const struct bpf_prog *prog,
17321732 struct bpf_prog_stats *stats)
17331733 {
1734− u64 nsecs = 0, cnt = 0;
1734+ u64 nsecs = 0, cnt = 0, misses = 0;
17351735 int cpu;
17361736
17371737 for_each_possible_cpu(cpu) {
17381738 const struct bpf_prog_stats *st;
17391739 unsigned int start;
1740− u64 tnsecs, tcnt;
1740+ u64 tnsecs, tcnt, tmisses;
17411741
1742− st = per_cpu_ptr(prog->aux->stats, cpu);
1742+ st = per_cpu_ptr(prog->stats, cpu);
17431743 do {
17441744 start = u64_stats_fetch_begin_irq(&st->syncp);
17451745 tnsecs = st->nsecs;
17461746 tcnt = st->cnt;
1747+ tmisses = st->misses;
17471748 } while (u64_stats_fetch_retry_irq(&st->syncp, start));
17481749 nsecs += tnsecs;
17491750 cnt += tcnt;
1751+ misses += tmisses;
17501752 }
17511753 stats->nsecs = nsecs;
17521754 stats->cnt = cnt;
1755+ stats->misses = misses;
17531756 }
17541757
17551758 #ifdef CONFIG_PROC_FS
@@ -1768,14 +1771,16 @@ static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp) · 12 unmodified lines
17681771 "memlock:\t%llu\n"
17691772 "prog_id:\t%u\n"
17701773 "run_time_ns:\t%llu\n"
1771− "run_cnt:\t%llu\n",
1774+ "run_cnt:\t%llu\n"
1775+ "recursion_misses:\t%llu\n",
17721776 prog->type,
17731777 prog->jited,
17741778 prog_tag,
17751779 prog->pages * 1ULL << PAGE_SHIFT,
17761780 prog->aux->id,
17771781 stats.nsecs,
1778− stats.cnt);
1782+ stats.cnt,
1783+ stats.misses);
17791784 }
17801785 #endif
17811786
@@ -3438,6 +3443,7 @@ static int bpf_prog_get_info_by_fd(struct file *file, · 1656 unmodified lines
34383443 bpf_prog_get_stats(prog, &stats);
34393444 info.run_time_ns = stats.nsecs;
34403445 info.run_cnt = stats.cnt;
3446+ info.recursion_misses = stats.misses;
34413447
34423448 if (!bpf_capable()) {
34433449 info.jited_prog_len = 0;
kernel/bpf/task_iter.c+266−1
@@ -286,9 +286,248 @@ static const struct seq_operations task_file_seq_ops = { · 285 unmodified lines
286286 .show = task_file_seq_show,
287287 };
288288
289+struct bpf_iter_seq_task_vma_info {
290+ /* The first field must be struct bpf_iter_seq_task_common.
291+ * this is assumed by {init, fini}_seq_pidns() callback functions.
292+ */
293+ struct bpf_iter_seq_task_common common;
294+ struct task_struct *task;
295+ struct vm_area_struct *vma;
296+ u32 tid;
297+ unsigned long prev_vm_start;
298+ unsigned long prev_vm_end;
299+};
300+
301+enum bpf_task_vma_iter_find_op {
302+ task_vma_iter_first_vma, /* use mm->mmap */
303+ task_vma_iter_next_vma, /* use curr_vma->vm_next */
304+ task_vma_iter_find_vma, /* use find_vma() to find next vma */
305+};
306+
307+static struct vm_area_struct *
308+task_vma_seq_get_next(struct bpf_iter_seq_task_vma_info *info)
309+{
310+ struct pid_namespace *ns = info->common.ns;
311+ enum bpf_task_vma_iter_find_op op;
312+ struct vm_area_struct *curr_vma;
313+ struct task_struct *curr_task;
314+ u32 curr_tid = info->tid;
315+
316+ /* If this function returns a non-NULL vma, it holds a reference to
317+ * the task_struct, and holds read lock on vma->mm->mmap_lock.
318+ * If this function returns NULL, it does not hold any reference or
319+ * lock.
320+ */
321+ if (info->task) {
322+ curr_task = info->task;
323+ curr_vma = info->vma;
324+ /* In case of lock contention, drop mmap_lock to unblock
325+ * the writer.
326+ *
327+ * After relock, call find(mm, prev_vm_end - 1) to find
328+ * new vma to process.
329+ *
330+ * +------+------+-----------+
331+ * | VMA1 | VMA2 | VMA3 |
332+ * +------+------+-----------+
333+ * | | | |
334+ * 4k 8k 16k 400k
335+ *
336+ * For example, curr_vma == VMA2. Before unlock, we set
337+ *
338+ * prev_vm_start = 8k
339+ * prev_vm_end = 16k
340+ *
341+ * There are a few cases:
342+ *
343+ * 1) VMA2 is freed, but VMA3 exists.
344+ *
345+ * find_vma() will return VMA3, just process VMA3.
346+ *
347+ * 2) VMA2 still exists.
348+ *
349+ * find_vma() will return VMA2, process VMA2->next.
350+ *
351+ * 3) no more vma in this mm.
352+ *
353+ * Process the next task.
354+ *
355+ * 4) find_vma() returns a different vma, VMA2'.
356+ *
357+ * 4.1) If VMA2 covers same range as VMA2', skip VMA2',
358+ * because we already covered the range;
359+ * 4.2) VMA2 and VMA2' covers different ranges, process
360+ * VMA2'.
361+ */
362+ if (mmap_lock_is_contended(curr_task->mm)) {
363+ info->prev_vm_start = curr_vma->vm_start;
364+ info->prev_vm_end = curr_vma->vm_end;
365+ op = task_vma_iter_find_vma;
366+ mmap_read_unlock(curr_task->mm);
367+ if (mmap_read_lock_killable(curr_task->mm))
368+ goto finish;
369+ } else {
370+ op = task_vma_iter_next_vma;
371+ }
372+ } else {
373+again:
374+ curr_task = task_seq_get_next(ns, &curr_tid, true);
375+ if (!curr_task) {
376+ info->tid = curr_tid + 1;
377+ goto finish;
378+ }
379+
380+ if (curr_tid != info->tid) {
381+ info->tid = curr_tid;
382+ /* new task, process the first vma */
383+ op = task_vma_iter_first_vma;
384+ } else {
385+ /* Found the same tid, which means the user space
386+ * finished data in previous buffer and read more.
387+ * We dropped mmap_lock before returning to user
388+ * space, so it is necessary to use find_vma() to
389+ * find the next vma to process.
390+ */
391+ op = task_vma_iter_find_vma;
392+ }
393+
394+ if (!curr_task->mm)
395+ goto next_task;
396+
397+ if (mmap_read_lock_killable(curr_task->mm))
398+ goto finish;
399+ }
400+
401+ switch (op) {
402+ case task_vma_iter_first_vma:
403+ curr_vma = curr_task->mm->mmap;
404+ break;
405+ case task_vma_iter_next_vma:
406+ curr_vma = curr_vma->vm_next;
407+ break;
408+ case task_vma_iter_find_vma:
409+ /* We dropped mmap_lock so it is necessary to use find_vma
410+ * to find the next vma. This is similar to the mechanism
411+ * in show_smaps_rollup().
412+ */
413+ curr_vma = find_vma(curr_task->mm, info->prev_vm_end - 1);
414+ /* case 1) and 4.2) above just use curr_vma */
415+
416+ /* check for case 2) or case 4.1) above */
417+ if (curr_vma &&
418+ curr_vma->vm_start == info->prev_vm_start &&
419+ curr_vma->vm_end == info->prev_vm_end)
420+ curr_vma = curr_vma->vm_next;
421+ break;
422+ }
423+ if (!curr_vma) {
424+ /* case 3) above, or case 2) 4.1) with vma->next == NULL */
425+ mmap_read_unlock(curr_task->mm);
426+ goto next_task;
427+ }
428+ info->task = curr_task;
429+ info->vma = curr_vma;
430+ return curr_vma;
431+
432+next_task:
433+ put_task_struct(curr_task);
434+ info->task = NULL;
435+ curr_tid++;
436+ goto again;
437+
438+finish:
439+ if (curr_task)
440+ put_task_struct(curr_task);
441+ info->task = NULL;
442+ info->vma = NULL;
443+ return NULL;
444+}
445+
446+static void *task_vma_seq_start(struct seq_file *seq, loff_t *pos)
447+{
448+ struct bpf_iter_seq_task_vma_info *info = seq->private;
449+ struct vm_area_struct *vma;
450+
451+ vma = task_vma_seq_get_next(info);
452+ if (vma && *pos == 0)
453+ ++*pos;
454+
455+ return vma;
456+}
457+
458+static void *task_vma_seq_next(struct seq_file *seq, void *v, loff_t *pos)
459+{
460+ struct bpf_iter_seq_task_vma_info *info = seq->private;
461+
462+ ++*pos;
463+ return task_vma_seq_get_next(info);
464+}
465+
466+struct bpf_iter__task_vma {
467+ __bpf_md_ptr(struct bpf_iter_meta *, meta);
468+ __bpf_md_ptr(struct task_struct *, task);
469+ __bpf_md_ptr(struct vm_area_struct *, vma);
470+};
471+
472+DEFINE_BPF_ITER_FUNC(task_vma, struct bpf_iter_meta *meta,
473+ struct task_struct *task, struct vm_area_struct *vma)
474+
475+static int __task_vma_seq_show(struct seq_file *seq, bool in_stop)
476+{
477+ struct bpf_iter_seq_task_vma_info *info = seq->private;
478+ struct bpf_iter__task_vma ctx;
479+ struct bpf_iter_meta meta;
480+ struct bpf_prog *prog;
481+
482+ meta.seq = seq;
483+ prog = bpf_iter_get_info(&meta, in_stop);
484+ if (!prog)
485+ return 0;
486+
487+ ctx.meta = &meta;
488+ ctx.task = info->task;
489+ ctx.vma = info->vma;
490+ return bpf_iter_run_prog(prog, &ctx);
491+}
492+
493+static int task_vma_seq_show(struct seq_file *seq, void *v)
494+{
495+ return __task_vma_seq_show(seq, false);
496+}
497+
498+static void task_vma_seq_stop(struct seq_file *seq, void *v)
499+{
500+ struct bpf_iter_seq_task_vma_info *info = seq->private;
501+
502+ if (!v) {
503+ (void)__task_vma_seq_show(seq, true);
504+ } else {
505+ /* info->vma has not been seen by the BPF program. If the
506+ * user space reads more, task_vma_seq_get_next should
507+ * return this vma again. Set prev_vm_start to ~0UL,
508+ * so that we don't skip the vma returned by the next
509+ * find_vma() (case task_vma_iter_find_vma in
510+ * task_vma_seq_get_next()).
511+ */
512+ info->prev_vm_start = ~0UL;
513+ info->prev_vm_end = info->vma->vm_end;
514+ mmap_read_unlock(info->task->mm);
515+ put_task_struct(info->task);
516+ info->task = NULL;
517+ }
518+}
519+
520+static const struct seq_operations task_vma_seq_ops = {
521+ .start = task_vma_seq_start,
522+ .next = task_vma_seq_next,
523+ .stop = task_vma_seq_stop,
524+ .show = task_vma_seq_show,
525+};
526+
289527 BTF_ID_LIST(btf_task_file_ids)
290528 BTF_ID(struct, task_struct)
291529 BTF_ID(struct, file)
530+BTF_ID(struct, vm_area_struct)
292531
293532 static const struct bpf_iter_seq_info task_seq_info = {
294533 .seq_ops = &task_seq_ops,
@@ -328,6 +567,26 @@ static struct bpf_iter_reg task_file_reg_info = { · 33 unmodified lines
328567 .seq_info = &task_file_seq_info,
329568 };
330569
570+static const struct bpf_iter_seq_info task_vma_seq_info = {
571+ .seq_ops = &task_vma_seq_ops,
572+ .init_seq_private = init_seq_pidns,
573+ .fini_seq_private = fini_seq_pidns,
574+ .seq_priv_size = sizeof(struct bpf_iter_seq_task_vma_info),
575+};
576+
577+static struct bpf_iter_reg task_vma_reg_info = {
578+ .target = "task_vma",
579+ .feature = BPF_ITER_RESCHED,
580+ .ctx_arg_info_size = 2,
581+ .ctx_arg_info = {
582+ { offsetof(struct bpf_iter__task_vma, task),
583+ PTR_TO_BTF_ID_OR_NULL },
584+ { offsetof(struct bpf_iter__task_vma, vma),
585+ PTR_TO_BTF_ID_OR_NULL },
586+ },
587+ .seq_info = &task_vma_seq_info,
588+};
589+
331590 static int __init task_iter_init(void)
332591 {
333592 int ret;
@@ -339,6 +598,12 @@ static int __init task_iter_init(void) · 5 unmodified lines
339598
340599 task_file_reg_info.ctx_arg_info[0].btf_id = btf_task_file_ids[0];
341600 task_file_reg_info.ctx_arg_info[1].btf_id = btf_task_file_ids[1];
342− return bpf_iter_reg_target(&task_file_reg_info);
601+ ret = bpf_iter_reg_target(&task_file_reg_info);
602+ if (ret)
603+ return ret;
604+
605+ task_vma_reg_info.ctx_arg_info[0].btf_id = btf_task_file_ids[0];
606+ task_vma_reg_info.ctx_arg_info[1].btf_id = btf_task_file_ids[2];
607+ return bpf_iter_reg_target(&task_vma_reg_info);
343608 }
344609 late_initcall(task_iter_init);
kernel/bpf/trampoline.c+61−16
@@ -381,55 +381,100 @@ out: · 380 unmodified lines
381381 mutex_unlock(&trampoline_mutex);
382382 }
383383
384+#define NO_START_TIME 1
385+static u64 notrace bpf_prog_start_time(void)
386+{
387+ u64 start = NO_START_TIME;
388+
389+ if (static_branch_unlikely(&bpf_stats_enabled_key)) {
390+ start = sched_clock();
391+ if (unlikely(!start))
392+ start = NO_START_TIME;
393+ }
394+ return start;
395+}
396+
397+static void notrace inc_misses_counter(struct bpf_prog *prog)
398+{
399+ struct bpf_prog_stats *stats;
400+
401+ stats = this_cpu_ptr(prog->stats);
402+ u64_stats_update_begin(&stats->syncp);
403+ stats->misses++;
404+ u64_stats_update_end(&stats->syncp);
405+}
406+
384407 /* The logic is similar to BPF_PROG_RUN, but with an explicit
385408 * rcu_read_lock() and migrate_disable() which are required
386409 * for the trampoline. The macro is split into
387− * call _bpf_prog_enter
410+ * call __bpf_prog_enter
388411 * call prog->bpf_func
389412 * call __bpf_prog_exit
413+ *
414+ * __bpf_prog_enter returns:
415+ * 0 - skip execution of the bpf prog
416+ * 1 - execute bpf prog
417+ * [2..MAX_U64] - excute bpf prog and record execution time.
418+ * This is start time.
390419 */
391−u64 notrace __bpf_prog_enter(void)
420+u64 notrace __bpf_prog_enter(struct bpf_prog *prog)
392421 __acquires(RCU)
393422 {
394− u64 start = 0;
395−
396423 rcu_read_lock();
397424 migrate_disable();
398− if (static_branch_unlikely(&bpf_stats_enabled_key))
399− start = sched_clock();
400− return start;
425+ if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) {
426+ inc_misses_counter(prog);
427+ return 0;
428+ }
429+ return bpf_prog_start_time();
401430 }
402431
403−void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start)
404− __releases(RCU)
432+static void notrace update_prog_stats(struct bpf_prog *prog,
433+ u64 start)
405434 {
406435 struct bpf_prog_stats *stats;
407436
408437 if (static_branch_unlikely(&bpf_stats_enabled_key) &&
409− /* static_key could be enabled in __bpf_prog_enter
410− * and disabled in __bpf_prog_exit.
438+ /* static_key could be enabled in __bpf_prog_enter*
439+ * and disabled in __bpf_prog_exit*.
411440 * And vice versa.
412− * Hence check that 'start' is not zero.
441+ * Hence check that 'start' is valid.
413442 */
414− start) {
415− stats = this_cpu_ptr(prog->aux->stats);
443+ start > NO_START_TIME) {
444+ stats = this_cpu_ptr(prog->stats);
416445 u64_stats_update_begin(&stats->syncp);
417446 stats->cnt++;
418447 stats->nsecs += sched_clock() - start;
419448 u64_stats_update_end(&stats->syncp);
420449 }
450+}
451+
452+void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start)
453+ __releases(RCU)
454+{
455+ update_prog_stats(prog, start);
456+ __this_cpu_dec(*(prog->active));
421457 migrate_enable();
422458 rcu_read_unlock();
423459 }
424460
425−void notrace __bpf_prog_enter_sleepable(void)
461+u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog)
426462 {
427463 rcu_read_lock_trace();
464+ migrate_disable();
428465 might_fault();
466+ if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) {
467+ inc_misses_counter(prog);
468+ return 0;
469+ }
470+ return bpf_prog_start_time();
429471 }
430472
431−void notrace __bpf_prog_exit_sleepable(void)
473+void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start)
432474 {
475+ update_prog_stats(prog, start);
476+ __this_cpu_dec(*(prog->active));
477+ migrate_enable();
433478 rcu_read_unlock_trace();
434479 }
435480
kernel/bpf/verifier.c+662−215
@@ -228,6 +228,12 @@ static void bpf_map_key_store(struct bpf_insn_aux_data *aux, u64 state) · 227 unmodified lines
228228 (poisoned ? BPF_MAP_KEY_POISON : 0ULL);
229229 }
230230
231+static bool bpf_pseudo_call(const struct bpf_insn *insn)
232+{
233+ return insn->code == (BPF_JMP | BPF_CALL) &&
234+ insn->src_reg == BPF_PSEUDO_CALL;
235+}
236+
231237 struct bpf_call_arg_meta {
232238 struct bpf_map *map_ptr;
233239 bool raw_mode;
@@ -1073,6 +1079,51 @@ static void mark_reg_known_zero(struct bpf_verifier_env *env, · 839 unmodified lines
10731079 __mark_reg_known_zero(regs + regno);
10741080 }
10751081
1082+static void mark_ptr_not_null_reg(struct bpf_reg_state *reg)
1083+{
1084+ switch (reg->type) {
1085+ case PTR_TO_MAP_VALUE_OR_NULL: {
1086+ const struct bpf_map *map = reg->map_ptr;
1087+
1088+ if (map->inner_map_meta) {
1089+ reg->type = CONST_PTR_TO_MAP;
1090+ reg->map_ptr = map->inner_map_meta;
1091+ } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) {
1092+ reg->type = PTR_TO_XDP_SOCK;
1093+ } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP ||
1094+ map->map_type == BPF_MAP_TYPE_SOCKHASH) {
1095+ reg->type = PTR_TO_SOCKET;
1096+ } else {
1097+ reg->type = PTR_TO_MAP_VALUE;
1098+ }
1099+ break;
1100+ }
1101+ case PTR_TO_SOCKET_OR_NULL:
1102+ reg->type = PTR_TO_SOCKET;
1103+ break;
1104+ case PTR_TO_SOCK_COMMON_OR_NULL:
1105+ reg->type = PTR_TO_SOCK_COMMON;
1106+ break;
1107+ case PTR_TO_TCP_SOCK_OR_NULL:
1108+ reg->type = PTR_TO_TCP_SOCK;
1109+ break;
1110+ case PTR_TO_BTF_ID_OR_NULL:
1111+ reg->type = PTR_TO_BTF_ID;
1112+ break;
1113+ case PTR_TO_MEM_OR_NULL:
1114+ reg->type = PTR_TO_MEM;
1115+ break;
1116+ case PTR_TO_RDONLY_BUF_OR_NULL:
1117+ reg->type = PTR_TO_RDONLY_BUF;
1118+ break;
1119+ case PTR_TO_RDWR_BUF_OR_NULL:
1120+ reg->type = PTR_TO_RDWR_BUF;
1121+ break;
1122+ default:
1123+ WARN_ON("unknown nullable register type");
1124+ }
1125+}
1126+
10761127 static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg)
10771128 {
10781129 return type_is_pkt_pointer(reg->type);
@@ -1486,9 +1537,7 @@ static int check_subprogs(struct bpf_verifier_env *env) · 407 unmodified lines
14861537
14871538 /* determine subprog starts. The end is one before the next starts */
14881539 for (i = 0; i < insn_cnt; i++) {
1489− if (insn[i].code != (BPF_JMP | BPF_CALL))
1490− continue;
1491− if (insn[i].src_reg != BPF_PSEUDO_CALL)
1540+ if (!bpf_pseudo_call(insn + i))
14921541 continue;
14931542 if (!env->bpf_capable) {
14941543 verbose(env,
@@ -2271,12 +2320,14 @@ static void save_register_state(struct bpf_func_state *state, · 776 unmodified lines
22712320 state->stack[spi].slot_type[i] = STACK_SPILL;
22722321 }
22732322
2274−/* check_stack_read/write functions track spill/fill of registers,
2323+/* check_stack_{read,write}_fixed_off functions track spill/fill of registers,
22752324 * stack boundary and alignment are checked in check_mem_access()
22762325 */
2277−static int check_stack_write(struct bpf_verifier_env *env,
2278− struct bpf_func_state *state, /* func where register points to */
2279− int off, int size, int value_regno, int insn_idx)
2326+static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
2327+ /* stack frame we're writing to */
2328+ struct bpf_func_state *state,
2329+ int off, int size, int value_regno,
2330+ int insn_idx)
22802331 {
22812332 struct bpf_func_state *cur; /* state of the current function */
22822333 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err;
@@ -2402,9 +2453,175 @@ static int check_stack_write(struct bpf_verifier_env *env, · 119 unmodified lines
24022453 return 0;
24032454 }
24042455
2405−static int check_stack_read(struct bpf_verifier_env *env,
2406− struct bpf_func_state *reg_state /* func where register points to */,
2407− int off, int size, int value_regno)
2456+/* Write the stack: 'stack[ptr_regno + off] = value_regno'. 'ptr_regno' is
2457+ * known to contain a variable offset.
2458+ * This function checks whether the write is permitted and conservatively
2459+ * tracks the effects of the write, considering that each stack slot in the
2460+ * dynamic range is potentially written to.
2461+ *
2462+ * 'off' includes 'regno->off'.
2463+ * 'value_regno' can be -1, meaning that an unknown value is being written to
2464+ * the stack.
2465+ *
2466+ * Spilled pointers in range are not marked as written because we don't know
2467+ * what's going to be actually written. This means that read propagation for
2468+ * future reads cannot be terminated by this write.
2469+ *
2470+ * For privileged programs, uninitialized stack slots are considered
2471+ * initialized by this write (even though we don't know exactly what offsets
2472+ * are going to be written to). The idea is that we don't want the verifier to
2473+ * reject future reads that access slots written to through variable offsets.
2474+ */
2475+static int check_stack_write_var_off(struct bpf_verifier_env *env,
2476+ /* func where register points to */
2477+ struct bpf_func_state *state,
2478+ int ptr_regno, int off, int size,
2479+ int value_regno, int insn_idx)
2480+{
2481+ struct bpf_func_state *cur; /* state of the current function */
2482+ int min_off, max_off;
2483+ int i, err;
2484+ struct bpf_reg_state *ptr_reg = NULL, *value_reg = NULL;
2485+ bool writing_zero = false;
2486+ /* set if the fact that we're writing a zero is used to let any
2487+ * stack slots remain STACK_ZERO
2488+ */
2489+ bool zero_used = false;
2490+
2491+ cur = env->cur_state->frame[env->cur_state->curframe];
2492+ ptr_reg = &cur->regs[ptr_regno];
2493+ min_off = ptr_reg->smin_value + off;
2494+ max_off = ptr_reg->smax_value + off + size;
2495+ if (value_regno >= 0)
2496+ value_reg = &cur->regs[value_regno];
2497+ if (value_reg && register_is_null(value_reg))
2498+ writing_zero = true;
2499+
2500+ err = realloc_func_state(state, round_up(-min_off, BPF_REG_SIZE),
2501+ state->acquired_refs, true);
2502+ if (err)
2503+ return err;
2504+
2505+
2506+ /* Variable offset writes destroy any spilled pointers in range. */
2507+ for (i = min_off; i < max_off; i++) {
2508+ u8 new_type, *stype;
2509+ int slot, spi;
2510+
2511+ slot = -i - 1;
2512+ spi = slot / BPF_REG_SIZE;
2513+ stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE];
2514+
2515+ if (!env->allow_ptr_leaks
2516+ && *stype != NOT_INIT
2517+ && *stype != SCALAR_VALUE) {
2518+ /* Reject the write if there's are spilled pointers in
2519+ * range. If we didn't reject here, the ptr status
2520+ * would be erased below (even though not all slots are
2521+ * actually overwritten), possibly opening the door to
2522+ * leaks.
2523+ */
2524+ verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d",
2525+ insn_idx, i);
2526+ return -EINVAL;
2527+ }
2528+
2529+ /* Erase all spilled pointers. */
2530+ state->stack[spi].spilled_ptr.type = NOT_INIT;
2531+
2532+ /* Update the slot type. */
2533+ new_type = STACK_MISC;
2534+ if (writing_zero && *stype == STACK_ZERO) {
2535+ new_type = STACK_ZERO;
2536+ zero_used = true;
2537+ }
2538+ /* If the slot is STACK_INVALID, we check whether it's OK to
2539+ * pretend that it will be initialized by this write. The slot
2540+ * might not actually be written to, and so if we mark it as
2541+ * initialized future reads might leak uninitialized memory.
2542+ * For privileged programs, we will accept such reads to slots
2543+ * that may or may not be written because, if we're reject
2544+ * them, the error would be too confusing.
2545+ */
2546+ if (*stype == STACK_INVALID && !env->allow_uninit_stack) {
2547+ verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d",
2548+ insn_idx, i);
2549+ return -EINVAL;
2550+ }
2551+ *stype = new_type;
2552+ }
2553+ if (zero_used) {
2554+ /* backtracking doesn't work for STACK_ZERO yet. */
2555+ err = mark_chain_precision(env, value_regno);
2556+ if (err)
2557+ return err;
2558+ }
2559+ return 0;
2560+}
2561+
2562+/* When register 'dst_regno' is assigned some values from stack[min_off,
2563+ * max_off), we set the register's type according to the types of the
2564+ * respective stack slots. If all the stack values are known to be zeros, then
2565+ * so is the destination reg. Otherwise, the register is considered to be
2566+ * SCALAR. This function does not deal with register filling; the caller must
2567+ * ensure that all spilled registers in the stack range have been marked as
2568+ * read.
2569+ */
2570+static void mark_reg_stack_read(struct bpf_verifier_env *env,
2571+ /* func where src register points to */
2572+ struct bpf_func_state *ptr_state,
2573+ int min_off, int max_off, int dst_regno)
2574+{
2575+ struct bpf_verifier_state *vstate = env->cur_state;
2576+ struct bpf_func_state *state = vstate->frame[vstate->curframe];
2577+ int i, slot, spi;
2578+ u8 *stype;
2579+ int zeros = 0;
2580+
2581+ for (i = min_off; i < max_off; i++) {
2582+ slot = -i - 1;
2583+ spi = slot / BPF_REG_SIZE;
2584+ stype = ptr_state->stack[spi].slot_type;
2585+ if (stype[slot % BPF_REG_SIZE] != STACK_ZERO)
2586+ break;
2587+ zeros++;
2588+ }
2589+ if (zeros == max_off - min_off) {
2590+ /* any access_size read into register is zero extended,
2591+ * so the whole register == const_zero
2592+ */
2593+ __mark_reg_const_zero(&state->regs[dst_regno]);
2594+ /* backtracking doesn't support STACK_ZERO yet,
2595+ * so mark it precise here, so that later
2596+ * backtracking can stop here.
2597+ * Backtracking may not need this if this register
2598+ * doesn't participate in pointer adjustment.
2599+ * Forward propagation of precise flag is not
2600+ * necessary either. This mark is only to stop
2601+ * backtracking. Any register that contributed
2602+ * to const 0 was marked precise before spill.
2603+ */
2604+ state->regs[dst_regno].precise = true;
2605+ } else {
2606+ /* have read misc data from the stack */
2607+ mark_reg_unknown(env, state->regs, dst_regno);
2608+ }
2609+ state->regs[dst_regno].live |= REG_LIVE_WRITTEN;
2610+}
2611+
2612+/* Read the stack at 'off' and put the results into the register indicated by
2613+ * 'dst_regno'. It handles reg filling if the addressed stack slot is a
2614+ * spilled reg.
2615+ *
2616+ * 'dst_regno' can be -1, meaning that the read value is not going to a
2617+ * register.
2618+ *
2619+ * The access is assumed to be within the current stack bounds.
2620+ */
2621+static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
2622+ /* func where src register points to */
2623+ struct bpf_func_state *reg_state,
2624+ int off, int size, int dst_regno)
24082625 {
24092626 struct bpf_verifier_state *vstate = env->cur_state;
24102627 struct bpf_func_state *state = vstate->frame[vstate->curframe];
@@ -2412,11 +2629,6 @@ static int check_stack_read(struct bpf_verifier_env *env, · 1 unmodified line
24122629 struct bpf_reg_state *reg;
24132630 u8 *stype;
24142631
2415− if (reg_state->allocated_stack <= slot) {
2416− verbose(env, "invalid read from stack off %d+0 size %d\n",
2417− off, size);
2418− return -EACCES;
2419− }
24202632 stype = reg_state->stack[spi].slot_type;
24212633 reg = &reg_state->stack[spi].spilled_ptr;
24222634
@@ -2427,9 +2639,9 @@ static int check_stack_read(struct bpf_verifier_env *env, · 4 unmodified lines
24272639 verbose(env, "invalid size of register fill\n");
24282640 return -EACCES;
24292641 }
2430− if (value_regno >= 0) {
2431− mark_reg_unknown(env, state->regs, value_regno);
2432− state->regs[value_regno].live |= REG_LIVE_WRITTEN;
2642+ if (dst_regno >= 0) {
2643+ mark_reg_unknown(env, state->regs, dst_regno);
2644+ state->regs[dst_regno].live |= REG_LIVE_WRITTEN;
24332645 }
24342646 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
24352647 return 0;
@@ -2441,16 +2653,16 @@ static int check_stack_read(struct bpf_verifier_env *env, · 5 unmodified lines
24412653 }
24422654 }
24432655
2444− if (value_regno >= 0) {
2656+ if (dst_regno >= 0) {
24452657 /* restore register state from stack */
2446− state->regs[value_regno] = *reg;
2658+ state->regs[dst_regno] = *reg;
24472659 /* mark reg as written since spilled pointer state likely
24482660 * has its liveness marks cleared by is_state_visited()
24492661 * which resets stack/reg liveness for state transitions
24502662 */
2451− state->regs[value_regno].live |= REG_LIVE_WRITTEN;
2663+ state->regs[dst_regno].live |= REG_LIVE_WRITTEN;
24522664 } else if (__is_pointer_value(env->allow_ptr_leaks, reg)) {
2453− /* If value_regno==-1, the caller is asking us whether
2665+ /* If dst_regno==-1, the caller is asking us whether
24542666 * it is acceptable to use this value as a SCALAR_VALUE
24552667 * (e.g. for XADD).
24562668 * We must not allow unprivileged callers to do that
@@ -2462,70 +2674,167 @@ static int check_stack_read(struct bpf_verifier_env *env, · 5 unmodified lines
24622674 }
24632675 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
24642676 } else {
2465− int zeros = 0;
2677+ u8 type;
24662678
24672679 for (i = 0; i < size; i++) {
2468− if (stype[(slot - i) % BPF_REG_SIZE] == STACK_MISC)
2680+ type = stype[(slot - i) % BPF_REG_SIZE];
2681+ if (type == STACK_MISC)
24692682 continue;
2470− if (stype[(slot - i) % BPF_REG_SIZE] == STACK_ZERO) {
2471− zeros++;
2683+ if (type == STACK_ZERO)
24722684 continue;
2473− }
24742685 verbose(env, "invalid read from stack off %d+%d size %d\n",
24752686 off, i, size);
24762687 return -EACCES;
24772688 }
24782689 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
2479− if (value_regno >= 0) {
2480− if (zeros == size) {
2481− /* any size read into register is zero extended,
2482− * so the whole register == const_zero
2483− */
2484− __mark_reg_const_zero(&state->regs[value_regno]);
2485− /* backtracking doesn't support STACK_ZERO yet,
2486− * so mark it precise here, so that later
2487− * backtracking can stop here.
2488− * Backtracking may not need this if this register
2489− * doesn't participate in pointer adjustment.
2490− * Forward propagation of precise flag is not
2491− * necessary either. This mark is only to stop
2492− * backtracking. Any register that contributed
2493− * to const 0 was marked precise before spill.
2494− */
2495− state->regs[value_regno].precise = true;
2496− } else {
2497− /* have read misc data from the stack */
2498− mark_reg_unknown(env, state->regs, value_regno);
2499− }
2500− state->regs[value_regno].live |= REG_LIVE_WRITTEN;
2501− }
2690+ if (dst_regno >= 0)
2691+ mark_reg_stack_read(env, reg_state, off, off + size, dst_regno);
25022692 }
25032693 return 0;
25042694 }
25052695
2506−static int check_stack_access(struct bpf_verifier_env *env,
2507− const struct bpf_reg_state *reg,
2508− int off, int size)
2696+enum stack_access_src {
2697+ ACCESS_DIRECT = 1, /* the access is performed by an instruction */
2698+ ACCESS_HELPER = 2, /* the access is performed by a helper */
2699+};
2700+
2701+static int check_stack_range_initialized(struct bpf_verifier_env *env,
2702+ int regno, int off, int access_size,
2703+ bool zero_size_allowed,
2704+ enum stack_access_src type,
2705+ struct bpf_call_arg_meta *meta);
2706+
2707+static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno)
2708+{
2709+ return cur_regs(env) + regno;
2710+}
2711+
2712+/* Read the stack at 'ptr_regno + off' and put the result into the register
2713+ * 'dst_regno'.
2714+ * 'off' includes the pointer register's fixed offset(i.e. 'ptr_regno.off'),
2715+ * but not its variable offset.
2716+ * 'size' is assumed to be <= reg size and the access is assumed to be aligned.
2717+ *
2718+ * As opposed to check_stack_read_fixed_off, this function doesn't deal with
2719+ * filling registers (i.e. reads of spilled register cannot be detected when
2720+ * the offset is not fixed). We conservatively mark 'dst_regno' as containing
2721+ * SCALAR_VALUE. That's why we assert that the 'ptr_regno' has a variable
2722+ * offset; for a fixed offset check_stack_read_fixed_off should be used
2723+ * instead.
2724+ */
2725+static int check_stack_read_var_off(struct bpf_verifier_env *env,
2726+ int ptr_regno, int off, int size, int dst_regno)
25092727 {
2510− /* Stack accesses must be at a fixed offset, so that we
2511− * can determine what type of data were returned. See
2512− * check_stack_read().
2728+ /* The state of the source register. */
2729+ struct bpf_reg_state *reg = reg_state(env, ptr_regno);
2730+ struct bpf_func_state *ptr_state = func(env, reg);
2731+ int err;
2732+ int min_off, max_off;
2733+
2734+ /* Note that we pass a NULL meta, so raw access will not be permitted.
25132735 */
2514− if (!tnum_is_const(reg->var_off)) {
2736+ err = check_stack_range_initialized(env, ptr_regno, off, size,
2737+ false, ACCESS_DIRECT, NULL);
2738+ if (err)
2739+ return err;
2740+
2741+ min_off = reg->smin_value + off;
2742+ max_off = reg->smax_value + off;
2743+ mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno);
2744+ return 0;
2745+}
2746+
2747+/* check_stack_read dispatches to check_stack_read_fixed_off or
2748+ * check_stack_read_var_off.
2749+ *
2750+ * The caller must ensure that the offset falls within the allocated stack
2751+ * bounds.
2752+ *
2753+ * 'dst_regno' is a register which will receive the value from the stack. It
2754+ * can be -1, meaning that the read value is not going to a register.
2755+ */
2756+static int check_stack_read(struct bpf_verifier_env *env,
2757+ int ptr_regno, int off, int size,
2758+ int dst_regno)
2759+{
2760+ struct bpf_reg_state *reg = reg_state(env, ptr_regno);
2761+ struct bpf_func_state *state = func(env, reg);
2762+ int err;
2763+ /* Some accesses are only permitted with a static offset. */
2764+ bool var_off = !tnum_is_const(reg->var_off);
2765+
2766+ /* The offset is required to be static when reads don't go to a
2767+ * register, in order to not leak pointers (see
2768+ * check_stack_read_fixed_off).
2769+ */
2770+ if (dst_regno < 0 && var_off) {
25152771 char tn_buf[48];
25162772
25172773 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
2518− verbose(env, "variable stack access var_off=%s off=%d size=%d\n",
2774+ verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n",
25192775 tn_buf, off, size);
25202776 return -EACCES;
25212777 }
2778+ /* Variable offset is prohibited for unprivileged mode for simplicity
2779+ * since it requires corresponding support in Spectre masking for stack
2780+ * ALU. See also retrieve_ptr_limit().
2781+ */
2782+ if (!env->bypass_spec_v1 && var_off) {
2783+ char tn_buf[48];
25222784
2523− if (off >= 0 || off < -MAX_BPF_STACK) {
2524− verbose(env, "invalid stack off=%d size=%d\n", off, size);
2785+ tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
2786+ verbose(env, "R%d variable offset stack access prohibited for !root, var_off=%s\n",
2787+ ptr_regno, tn_buf);
25252788 return -EACCES;
25262789 }
25272790
2528− return 0;
2791+ if (!var_off) {
2792+ off += reg->var_off.value;
2793+ err = check_stack_read_fixed_off(env, state, off, size,
2794+ dst_regno);
2795+ } else {
2796+ /* Variable offset stack reads need more conservative handling
2797+ * than fixed offset ones. Note that dst_regno >= 0 on this
2798+ * branch.
2799+ */
2800+ err = check_stack_read_var_off(env, ptr_regno, off, size,
2801+ dst_regno);
2802+ }
2803+ return err;
2804+}
2805+
2806+
2807+/* check_stack_write dispatches to check_stack_write_fixed_off or
2808+ * check_stack_write_var_off.
2809+ *
2810+ * 'ptr_regno' is the register used as a pointer into the stack.
2811+ * 'off' includes 'ptr_regno->off', but not its variable offset (if any).
2812+ * 'value_regno' is the register whose value we're writing to the stack. It can
2813+ * be -1, meaning that we're not writing from a register.
2814+ *
2815+ * The caller must ensure that the offset falls within the maximum stack size.
2816+ */
2817+static int check_stack_write(struct bpf_verifier_env *env,
2818+ int ptr_regno, int off, int size,
2819+ int value_regno, int insn_idx)
2820+{
2821+ struct bpf_reg_state *reg = reg_state(env, ptr_regno);
2822+ struct bpf_func_state *state = func(env, reg);
2823+ int err;
2824+
2825+ if (tnum_is_const(reg->var_off)) {
2826+ off += reg->var_off.value;
2827+ err = check_stack_write_fixed_off(env, state, off, size,
2828+ value_regno, insn_idx);
2829+ } else {
2830+ /* Variable offset stack reads need more conservative handling
2831+ * than fixed offset ones.
2832+ */
2833+ err = check_stack_write_var_off(env, state,
2834+ ptr_regno, off, size,
2835+ value_regno, insn_idx);
2836+ }
2837+ return err;
25292838 }
25302839
25312840 static int check_map_access_type(struct bpf_verifier_env *env, u32 regno,
@@ -2858,11 +3167,6 @@ static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, · 326 unmodified lines
28583167 return -EACCES;
28593168 }
28603169
2861−static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno)
2862−{
2863− return cur_regs(env) + regno;
2864−}
2865−
28663170 static bool is_pointer_value(struct bpf_verifier_env *env, int regno)
28673171 {
28683172 return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno));
@@ -2981,8 +3285,8 @@ static int check_ptr_alignment(struct bpf_verifier_env *env, · 112 unmodified lines
29813285 break;
29823286 case PTR_TO_STACK:
29833287 pointer_desc = "stack ";
2984− /* The stack spill tracking logic in check_stack_write()
2985− * and check_stack_read() relies on stack accesses being
3288+ /* The stack spill tracking logic in check_stack_write_fixed_off()
3289+ * and check_stack_read_fixed_off() relies on stack accesses being
29863290 * aligned.
29873291 */
29883292 strict = true;
@@ -3074,9 +3378,7 @@ process_func: · 85 unmodified lines
30743378 continue_func:
30753379 subprog_end = subprog[idx + 1].start;
30763380 for (; i < subprog_end; i++) {
3077− if (insn[i].code != (BPF_JMP | BPF_CALL))
3078− continue;
3079− if (insn[i].src_reg != BPF_PSEUDO_CALL)
3381+ if (!bpf_pseudo_call(insn + i))
30803382 continue;
30813383 /* remember insn and function to return to */
30823384 ret_insn[frame] = i + 1;
@@ -3400,6 +3702,91 @@ static int check_ptr_to_map_access(struct bpf_verifier_env *env, · 317 unmodified lines
34003702 return 0;
34013703 }
34023704
3705+/* Check that the stack access at the given offset is within bounds. The
3706+ * maximum valid offset is -1.
3707+ *
3708+ * The minimum valid offset is -MAX_BPF_STACK for writes, and
3709+ * -state->allocated_stack for reads.
3710+ */
3711+static int check_stack_slot_within_bounds(int off,
3712+ struct bpf_func_state *state,
3713+ enum bpf_access_type t)
3714+{
3715+ int min_valid_off;
3716+
3717+ if (t == BPF_WRITE)
3718+ min_valid_off = -MAX_BPF_STACK;
3719+ else
3720+ min_valid_off = -state->allocated_stack;
3721+
3722+ if (off < min_valid_off || off > -1)
3723+ return -EACCES;
3724+ return 0;
3725+}
3726+
3727+/* Check that the stack access at 'regno + off' falls within the maximum stack
3728+ * bounds.
3729+ *
3730+ * 'off' includes `regno->offset`, but not its dynamic part (if any).
3731+ */
3732+static int check_stack_access_within_bounds(
3733+ struct bpf_verifier_env *env,
3734+ int regno, int off, int access_size,
3735+ enum stack_access_src src, enum bpf_access_type type)
3736+{
3737+ struct bpf_reg_state *regs = cur_regs(env);
3738+ struct bpf_reg_state *reg = regs + regno;
3739+ struct bpf_func_state *state = func(env, reg);
3740+ int min_off, max_off;
3741+ int err;
3742+ char *err_extra;
3743+
3744+ if (src == ACCESS_HELPER)
3745+ /* We don't know if helpers are reading or writing (or both). */
3746+ err_extra = " indirect access to";
3747+ else if (type == BPF_READ)
3748+ err_extra = " read from";
3749+ else
3750+ err_extra = " write to";
3751+
3752+ if (tnum_is_const(reg->var_off)) {
3753+ min_off = reg->var_off.value + off;
3754+ if (access_size > 0)
3755+ max_off = min_off + access_size - 1;
3756+ else
3757+ max_off = min_off;
3758+ } else {
3759+ if (reg->smax_value >= BPF_MAX_VAR_OFF ||
3760+ reg->smin_value <= -BPF_MAX_VAR_OFF) {
3761+ verbose(env, "invalid unbounded variable-offset%s stack R%d\n",
3762+ err_extra, regno);
3763+ return -EACCES;
3764+ }
3765+ min_off = reg->smin_value + off;
3766+ if (access_size > 0)
3767+ max_off = reg->smax_value + off + access_size - 1;
3768+ else
3769+ max_off = min_off;
3770+ }
3771+
3772+ err = check_stack_slot_within_bounds(min_off, state, type);
3773+ if (!err)
3774+ err = check_stack_slot_within_bounds(max_off, state, type);
3775+
3776+ if (err) {
3777+ if (tnum_is_const(reg->var_off)) {
3778+ verbose(env, "invalid%s stack R%d off=%d size=%d\n",
3779+ err_extra, regno, off, access_size);
3780+ } else {
3781+ char tn_buf[48];
3782+
3783+ tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
3784+ verbose(env, "invalid variable-offset%s stack R%d var_off=%s size=%d\n",
3785+ err_extra, regno, tn_buf, access_size);
3786+ }
3787+ }
3788+ return err;
3789+}
34033790
34043791 /* check whether memory at (regno + off) is accessible for t = (read | write)
34053792 * if t==write, value_regno is a register which value is stored into memory
@@ -3515,8 +3902,8 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 109 unmodified lines
35153902 }
35163903
35173904 } else if (reg->type == PTR_TO_STACK) {
3518− off += reg->var_off.value;
3519− err = check_stack_access(env, reg, off, size);
3905+ /* Basic bounds checks. */
3906+ err = check_stack_access_within_bounds(env, regno, off, size, ACCESS_DIRECT, t);
35203907 if (err)
35213908 return err;
35223909
@@ -3525,12 +3912,12 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 2 unmodified lines
35253912 if (err)
35263913 return err;
35273914
3528− if (t == BPF_WRITE)
3529− err = check_stack_write(env, state, off, size,
3530− value_regno, insn_idx);
3531− else
3532− err = check_stack_read(env, state, off, size,
3915+ if (t == BPF_READ)
3916+ err = check_stack_read(env, regno, off, size,
35333917 value_regno);
3918+ else
3919+ err = check_stack_write(env, regno, off, size,
3920+ value_regno, insn_idx);
35343921 } else if (reg_is_pkt_pointer(reg)) {
35353922 if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) {
35363923 verbose(env, "cannot write into packet\n");
@@ -3665,9 +4052,26 @@ static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_i · 128 unmodified lines
36654052 return -EACCES;
36664053 }
36674054
4055+ if (insn->imm & BPF_FETCH) {
4056+ if (insn->imm == BPF_CMPXCHG)
4057+ load_reg = BPF_REG_0;
4058+ else
4059+ load_reg = insn->src_reg;
4060+
4061+ /* check and record load of old value */
4062+ err = check_reg_arg(env, load_reg, DST_OP);
4063+ if (err)
4064+ return err;
4065+ } else {
4066+ /* This instruction accesses a memory location but doesn't
4067+ * actually load it into a register.
4068+ */
4069+ load_reg = -1;
4070+ }
4071+
36684072 /* check whether we can read the memory */
36694073 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
3670− BPF_SIZE(insn->code), BPF_READ, -1, true);
4074+ BPF_SIZE(insn->code), BPF_READ, load_reg, true);
36714075 if (err)
36724076 return err;
36734077
@@ -3677,65 +4081,56 @@ static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_i · 3 unmodified lines
36774081 if (err)
36784082 return err;
36794083
3680− if (!(insn->imm & BPF_FETCH))
3681− return 0;
3682−
3683− if (insn->imm == BPF_CMPXCHG)
3684− load_reg = BPF_REG_0;
3685− else
3686− load_reg = insn->src_reg;
3687−
3688− /* check and record load of old value */
3689− err = check_reg_arg(env, load_reg, DST_OP);
3690− if (err)
3691− return err;
3692−
36934084 return 0;
36944085 }
36954086
3696−static int __check_stack_boundary(struct bpf_verifier_env *env, u32 regno,
3697− int off, int access_size,
3698− bool zero_size_allowed)
4087+/* When register 'regno' is used to read the stack (either directly or through
4088+ * a helper function) make sure that it's within stack boundary and, depending
4089+ * on the access type, that all elements of the stack are initialized.
4090+ *
4091+ * 'off' includes 'regno->off', but not its dynamic part (if any).
4092+ *
4093+ * All registers that have been spilled on the stack in the slots within the
4094+ * read offsets are marked as read.
4095+ */
4096+static int check_stack_range_initialized(
4097+ struct bpf_verifier_env *env, int regno, int off,
4098+ int access_size, bool zero_size_allowed,
4099+ enum stack_access_src type, struct bpf_call_arg_meta *meta)
36994100 {
37004101 struct bpf_reg_state *reg = reg_state(env, regno);
4102+ struct bpf_func_state *state = func(env, reg);
4103+ int err, min_off, max_off, i, j, slot, spi;
4104+ char *err_extra = type == ACCESS_HELPER ? " indirect" : "";
4105+ enum bpf_access_type bounds_check_type;
4106+ /* Some accesses can write anything into the stack, others are
4107+ * read-only.
4108+ */
4109+ bool clobber = false;
37014110
3702− if (off >= 0 || off < -MAX_BPF_STACK || off + access_size > 0 ||
3703− access_size < 0 || (access_size == 0 && !zero_size_allowed)) {
3704− if (tnum_is_const(reg->var_off)) {
3705− verbose(env, "invalid stack type R%d off=%d access_size=%d\n",
3706− regno, off, access_size);
3707− } else {
3708− char tn_buf[48];
3709−
3710− tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
3711− verbose(env, "invalid stack type R%d var_off=%s access_size=%d\n",
3712− regno, tn_buf, access_size);
3713− }
4111+ if (access_size == 0 && !zero_size_allowed) {
4112+ verbose(env, "invalid zero-sized read\n");
37144113 return -EACCES;
37154114 }
3716− return 0;
3717−}
37184115
3719−/* when register 'regno' is passed into function that will read 'access_size'
3720− * bytes from that pointer, make sure that it's within stack boundary
3721− * and all elements of stack are initialized.
3722− * Unlike most pointer bounds-checking functions, this one doesn't take an
3723− * 'off' argument, so it has to add in reg->off itself.
3724− */
3725−static int check_stack_boundary(struct bpf_verifier_env *env, int regno,
3726− int access_size, bool zero_size_allowed,
3727− struct bpf_call_arg_meta *meta)
3728−{
3729− struct bpf_reg_state *reg = reg_state(env, regno);
3730− struct bpf_func_state *state = func(env, reg);
3731− int err, min_off, max_off, i, j, slot, spi;
4116+ if (type == ACCESS_HELPER) {
4117+ /* The bounds checks for writes are more permissive than for
4118+ * reads. However, if raw_mode is not set, we'll do extra
4119+ * checks below.
4120+ */
4121+ bounds_check_type = BPF_WRITE;
4122+ clobber = true;
4123+ } else {
4124+ bounds_check_type = BPF_READ;
4125+ }
4126+ err = check_stack_access_within_bounds(env, regno, off, access_size,
4127+ type, bounds_check_type);
4128+ if (err)
4129+ return err;
4130+
37324131
37334132 if (tnum_is_const(reg->var_off)) {
3734− min_off = max_off = reg->var_off.value + reg->off;
3735− err = __check_stack_boundary(env, regno, min_off, access_size,
3736− zero_size_allowed);
3737− if (err)
3738− return err;
4133+ min_off = max_off = reg->var_off.value + off;
37394134 } else {
37404135 /* Variable offset is prohibited for unprivileged mode for
37414136 * simplicity since it requires corresponding support in
@@ -3746,8 +4141,8 @@ static int check_stack_boundary(struct bpf_verifier_env *env, int regno, · 4 unmodified lines
37464141 char tn_buf[48];
37474142
37484143 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
3749− verbose(env, "R%d indirect variable offset stack access prohibited for !root, var_off=%s\n",
3750− regno, tn_buf);
4144+ verbose(env, "R%d%s variable offset stack access prohibited for !root, var_off=%s\n",
4145+ regno, err_extra, tn_buf);
37514146 return -EACCES;
37524147 }
37534148 /* Only initialized buffer on stack is allowed to be accessed
@@ -3759,28 +4154,8 @@ static int check_stack_boundary(struct bpf_verifier_env *env, int regno, · 5 unmodified lines
37594154 if (meta && meta->raw_mode)
37604155 meta = NULL;
37614156
3762− if (reg->smax_value >= BPF_MAX_VAR_OFF ||
3763− reg->smax_value <= -BPF_MAX_VAR_OFF) {
3764− verbose(env, "R%d unbounded indirect variable offset stack access\n",
3765− regno);
3766− return -EACCES;
3767− }
3768− min_off = reg->smin_value + reg->off;
3769− max_off = reg->smax_value + reg->off;
3770− err = __check_stack_boundary(env, regno, min_off, access_size,
3771− zero_size_allowed);
3772− if (err) {
3773− verbose(env, "R%d min value is outside of stack bound\n",
3774− regno);
3775− return err;
3776− }
3777− err = __check_stack_boundary(env, regno, max_off, access_size,
3778− zero_size_allowed);
3779− if (err) {
3780− verbose(env, "R%d max value is outside of stack bound\n",
3781− regno);
3782− return err;
3783− }
4157+ min_off = reg->smin_value + off;
4158+ max_off = reg->smax_value + off;
37844159 }
37854160
37864161 if (meta && meta->raw_mode) {
@@ -3800,8 +4175,10 @@ static int check_stack_boundary(struct bpf_verifier_env *env, int regno, · 13 unmodified lines
38004175 if (*stype == STACK_MISC)
38014176 goto mark;
38024177 if (*stype == STACK_ZERO) {
3803− /* helper can write anything into the stack */
3804− *stype = STACK_MISC;
4178+ if (clobber) {
4179+ /* helper can write anything into the stack */
4180+ *stype = STACK_MISC;
4181+ }
38054182 goto mark;
38064183 }
38074184
@@ -3812,22 +4189,24 @@ static int check_stack_boundary(struct bpf_verifier_env *env, int regno, · 4 unmodified lines
38124189 if (state->stack[spi].slot_type[0] == STACK_SPILL &&
38134190 (state->stack[spi].spilled_ptr.type == SCALAR_VALUE ||
38144191 env->allow_ptr_leaks)) {
3815− __mark_reg_unknown(env, &state->stack[spi].spilled_ptr);
3816− for (j = 0; j < BPF_REG_SIZE; j++)
3817− state->stack[spi].slot_type[j] = STACK_MISC;
4192+ if (clobber) {
4193+ __mark_reg_unknown(env, &state->stack[spi].spilled_ptr);
4194+ for (j = 0; j < BPF_REG_SIZE; j++)
4195+ state->stack[spi].slot_type[j] = STACK_MISC;
4196+ }
38184197 goto mark;
38194198 }
38204199
38214200 err:
38224201 if (tnum_is_const(reg->var_off)) {
3823− verbose(env, "invalid indirect read from stack off %d+%d size %d\n",
3824− min_off, i - min_off, access_size);
4202+ verbose(env, "invalid%s read from stack R%d off %d+%d size %d\n",
4203+ err_extra, regno, min_off, i - min_off, access_size);
38254204 } else {
38264205 char tn_buf[48];
38274206
38284207 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
3829− verbose(env, "invalid indirect read from stack var_off %s+%d size %d\n",
3830− tn_buf, i - min_off, access_size);
4208+ verbose(env, "invalid%s read from stack R%d var_off %s+%d size %d\n",
4209+ err_extra, regno, tn_buf, i - min_off, access_size);
38314210 }
38324211 return -EACCES;
38334212 mark:
@@ -3876,8 +4255,10 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, · 42 unmodified lines
38764255 "rdwr",
38774256 &env->prog->aux->max_rdwr_access);
38784257 case PTR_TO_STACK:
3879− return check_stack_boundary(env, regno, access_size,
3880− zero_size_allowed, meta);
4258+ return check_stack_range_initialized(
4259+ env,
4260+ regno, reg->off, access_size,
4261+ zero_size_allowed, ACCESS_HELPER, meta);
38814262 default: /* scalar_value or invalid ptr */
38824263 /* Allow zero-byte read from NULL, regardless of pointer type */
38834264 if (zero_size_allowed && access_size == 0 &&
@@ -3891,6 +4272,29 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, · 7 unmodified lines
38914272 }
38924273 }
38934274
4275+int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
4276+ u32 regno, u32 mem_size)
4277+{
4278+ if (register_is_null(reg))
4279+ return 0;
4280+
4281+ if (reg_type_may_be_null(reg->type)) {
4282+ /* Assuming that the register contains a value check if the memory
4283+ * access is safe. Temporarily save and restore the register's state as
4284+ * the conversion shouldn't be visible to a caller.
4285+ */
4286+ const struct bpf_reg_state saved_reg = *reg;
4287+ int rv;
4288+
4289+ mark_ptr_not_null_reg(reg);
4290+ rv = check_helper_mem_access(env, regno, mem_size, true, NULL);
4291+ *reg = saved_reg;
4292+ return rv;
4293+ }
4294+
4295+ return check_helper_mem_access(env, regno, mem_size, true, NULL);
4296+}
4297+
38944298 /* Implementation details:
38954299 * bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL
38964300 * Two bpf_map_lookups (even with the same key) will have different reg->id.
@@ -4875,8 +5279,9 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · 978 unmodified lines
48755279 subprog);
48765280 clear_caller_saved_regs(env, caller->regs);
48775281
4878− /* All global functions return SCALAR_VALUE */
5282+ /* All global functions return a 64-bit SCALAR_VALUE */
48795283 mark_reg_unknown(env, caller->regs, BPF_REG_0);
5284+ caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;
48805285
48815286 /* continue with next insn after call */
48825287 return 0;
@@ -5541,6 +5946,41 @@ do_sim: · 658 unmodified lines
55415946 return !ret ? -EFAULT : 0;
55425947 }
55435948
5949+/* check that stack access falls within stack limits and that 'reg' doesn't
5950+ * have a variable offset.
5951+ *
5952+ * Variable offset is prohibited for unprivileged mode for simplicity since it
5953+ * requires corresponding support in Spectre masking for stack ALU. See also
5954+ * retrieve_ptr_limit().
5955+ *
5956+ *
5957+ * 'off' includes 'reg->off'.
5958+ */
5959+static int check_stack_access_for_ptr_arithmetic(
5960+ struct bpf_verifier_env *env,
5961+ int regno,
5962+ const struct bpf_reg_state *reg,
5963+ int off)
5964+{
5965+ if (!tnum_is_const(reg->var_off)) {
5966+ char tn_buf[48];
5967+
5968+ tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
5969+ verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n",
5970+ regno, tn_buf, off);
5971+ return -EACCES;
5972+ }
5973+
5974+ if (off >= 0 || off < -MAX_BPF_STACK) {
5975+ verbose(env, "R%d stack pointer arithmetic goes out of range, "
5976+ "prohibited for !root; off=%d\n", regno, off);
5977+ return -EACCES;
5978+ }
5979+
5980+ return 0;
5981+}
5982+
5983+
55445984 /* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off.
55455985 * Caller should also handle BPF_MOV case separately.
55465986 * If we return -EACCES, caller may want to try again treating pointer as a
@@ -5784,10 +6224,9 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, · 237 unmodified lines
57846224 "prohibited for !root\n", dst);
57856225 return -EACCES;
57866226 } else if (dst_reg->type == PTR_TO_STACK &&
5787− check_stack_access(env, dst_reg, dst_reg->off +
5788− dst_reg->var_off.value, 1)) {
5789− verbose(env, "R%d stack pointer arithmetic goes out of range, "
5790− "prohibited for !root\n", dst);
6227+ check_stack_access_for_ptr_arithmetic(
6228+ env, dst, dst_reg, dst_reg->off +
6229+ dst_reg->var_off.value)) {
57916230 return -EACCES;
57926231 }
57936232 }
@@ -6266,7 +6705,7 @@ static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg, · 472 unmodified lines
62666705 * 3) the signed bounds cross zero, so they tell us nothing
62676706 * about the result
62686707 * If the value in dst_reg is known nonnegative, then again the
6269− * unsigned bounts capture the signed bounds.
6708+ * unsigned bounds capture the signed bounds.
62706709 * Thus, in all cases it suffices to blow away our signed bounds
62716710 * and rely on inferring new ones from the unsigned bounds and
62726711 * var_off of the result.
@@ -6297,7 +6736,7 @@ static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg, · 24 unmodified lines
62976736 * 3) the signed bounds cross zero, so they tell us nothing
62986737 * about the result
62996738 * If the value in dst_reg is known nonnegative, then again the
6300− * unsigned bounts capture the signed bounds.
6739+ * unsigned bounds capture the signed bounds.
63016740 * Thus, in all cases it suffices to blow away our signed bounds
63026741 * and rely on inferring new ones from the unsigned bounds and
63036742 * var_off of the result.
@@ -7367,43 +7806,19 @@ static void mark_ptr_or_null_reg(struct bpf_func_state *state, · 1063 unmodified lines
73677806 }
73687807 if (is_null) {
73697808 reg->type = SCALAR_VALUE;
7370− } else if (reg->type == PTR_TO_MAP_VALUE_OR_NULL) {
7371− const struct bpf_map *map = reg->map_ptr;
7372−
7373− if (map->inner_map_meta) {
7374− reg->type = CONST_PTR_TO_MAP;
7375− reg->map_ptr = map->inner_map_meta;
7376− } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) {
7377− reg->type = PTR_TO_XDP_SOCK;
7378− } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP ||
7379− map->map_type == BPF_MAP_TYPE_SOCKHASH) {
7380− reg->type = PTR_TO_SOCKET;
7381− } else {
7382− reg->type = PTR_TO_MAP_VALUE;
7383− }
7384− } else if (reg->type == PTR_TO_SOCKET_OR_NULL) {
7385− reg->type = PTR_TO_SOCKET;
7386− } else if (reg->type == PTR_TO_SOCK_COMMON_OR_NULL) {
7387− reg->type = PTR_TO_SOCK_COMMON;
7388− } else if (reg->type == PTR_TO_TCP_SOCK_OR_NULL) {
7389− reg->type = PTR_TO_TCP_SOCK;
7390− } else if (reg->type == PTR_TO_BTF_ID_OR_NULL) {
7391− reg->type = PTR_TO_BTF_ID;
7392− } else if (reg->type == PTR_TO_MEM_OR_NULL) {
7393− reg->type = PTR_TO_MEM;
7394− } else if (reg->type == PTR_TO_RDONLY_BUF_OR_NULL) {
7395− reg->type = PTR_TO_RDONLY_BUF;
7396− } else if (reg->type == PTR_TO_RDWR_BUF_OR_NULL) {
7397− reg->type = PTR_TO_RDWR_BUF;
7398− }
7399− if (is_null) {
74007809 /* We don't need id and ref_obj_id from this point
74017810 * onwards anymore, thus we should better reset it,
74027811 * so that state pruning has chances to take effect.
74037812 */
74047813 reg->id = 0;
74057814 reg->ref_obj_id = 0;
7406− } else if (!reg_may_point_to_spin_lock(reg)) {
7815+
7816+ return;
7817+ }
7818+
7819+ mark_ptr_not_null_reg(reg);
7820+
7821+ if (!reg_may_point_to_spin_lock(reg)) {
74077822 /* For not-NULL ptr, reg->ref_obj_id will be reset
74087823 * in release_reg_references().
74097824 *
@@ -7986,6 +8401,9 @@ static int check_return_code(struct bpf_verifier_env *env) · 576 unmodified lines
79868401 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME ||
79878402 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME)
79888403 range = tnum_range(1, 1);
8404+ if (env->prog->expected_attach_type == BPF_CGROUP_INET4_BIND ||
8405+ env->prog->expected_attach_type == BPF_CGROUP_INET6_BIND)
8406+ range = tnum_range(0, 3);
79898407 break;
79908408 case BPF_PROG_TYPE_CGROUP_SKB:
79918409 if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) {
@@ -10015,15 +10433,22 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, · 2023 unmodified lines
1001510433 case BPF_MAP_TYPE_HASH:
1001610434 case BPF_MAP_TYPE_LRU_HASH:
1001710435 case BPF_MAP_TYPE_ARRAY:
10436+ case BPF_MAP_TYPE_PERCPU_HASH:
10437+ case BPF_MAP_TYPE_PERCPU_ARRAY:
10438+ case BPF_MAP_TYPE_LRU_PERCPU_HASH:
10439+ case BPF_MAP_TYPE_ARRAY_OF_MAPS:
10440+ case BPF_MAP_TYPE_HASH_OF_MAPS:
1001810441 if (!is_preallocated_map(map)) {
1001910442 verbose(env,
10020− "Sleepable programs can only use preallocated hash maps\n");
10443+ "Sleepable programs can only use preallocated maps\n");
1002110444 return -EINVAL;
1002210445 }
1002310446 break;
10447+ case BPF_MAP_TYPE_RINGBUF:
10448+ break;
1002410449 default:
1002510450 verbose(env,
10026− "Sleepable programs can only use array and hash maps\n");
10451+ "Sleepable programs can only use array, hash, and ringbuf maps\n");
1002710452 return -EINVAL;
1002810453 }
1002910454
@@ -10581,6 +11006,7 @@ static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, · 551 unmodified lines
1058111006 for (i = 0; i < len; i++) {
1058211007 int adj_idx = i + delta;
1058311008 struct bpf_insn insn;
11009+ u8 load_reg;
1058411010
1058511011 insn = insns[adj_idx];
1058611012 if (!aux[adj_idx].zext_dst) {
@@ -10623,9 +11049,27 @@ static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, · 36 unmodified lines
1062311049 if (!bpf_jit_needs_zext())
1062411050 continue;
1062511051
11052+ /* zext_dst means that we want to zero-extend whatever register
11053+ * the insn defines, which is dst_reg most of the time, with
11054+ * the notable exception of BPF_STX + BPF_ATOMIC + BPF_FETCH.
11055+ */
11056+ if (BPF_CLASS(insn.code) == BPF_STX &&
11057+ BPF_MODE(insn.code) == BPF_ATOMIC) {
11058+ /* BPF_STX + BPF_ATOMIC insns without BPF_FETCH do not
11059+ * define any registers, therefore zext_dst cannot be
11060+ * set.
11061+ */
11062+ if (WARN_ON(!(insn.imm & BPF_FETCH)))
11063+ return -EINVAL;
11064+ load_reg = insn.imm == BPF_CMPXCHG ? BPF_REG_0
11065+ : insn.src_reg;
11066+ } else {
11067+ load_reg = insn.dst_reg;
11068+ }
11069+
1062611070 zext_patch[0] = insn;
10627− zext_patch[1].dst_reg = insn.dst_reg;
10628− zext_patch[1].src_reg = insn.dst_reg;
11071+ zext_patch[1].dst_reg = load_reg;
11072+ zext_patch[1].src_reg = load_reg;
1062911073 patch = zext_patch;
1063011074 patch_len = 2;
1063111075 apply_patch_buffer:
@@ -10841,8 +11285,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 209 unmodified lines
1084111285 return 0;
1084211286
1084311287 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
10844− if (insn->code != (BPF_JMP | BPF_CALL) ||
10845− insn->src_reg != BPF_PSEUDO_CALL)
11288+ if (!bpf_pseudo_call(insn))
1084611289 continue;
1084711290 /* Upon error here we cannot fall back to interpreter but
1084811291 * need a hard reject of the program. Thus -EFAULT is
@@ -10883,7 +11326,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 34 unmodified lines
1088311326 /* BPF_PROG_RUN doesn't call subprogs directly,
1088411327 * hence main prog stats include the runtime of subprogs.
1088511328 * subprogs don't have IDs and not reachable via prog_get_next_id
10886− * func[i]->aux->stats will never be accessed and stays NULL
11329+ * func[i]->stats will never be accessed and stays NULL
1088711330 */
1088811331 func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER);
1088911332 if (!func[i])
@@ -10971,8 +11414,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 81 unmodified lines
1097111414 for (i = 0; i < env->subprog_cnt; i++) {
1097211415 insn = func[i]->insnsi;
1097311416 for (j = 0; j < func[i]->len; j++, insn++) {
10974− if (insn->code != (BPF_JMP | BPF_CALL) ||
10975− insn->src_reg != BPF_PSEUDO_CALL)
11417+ if (!bpf_pseudo_call(insn))
1097611418 continue;
1097711419 subprog = insn->off;
1097811420 insn->imm = BPF_CAST_CALL(func[subprog]->bpf_func) -
@@ -11017,8 +11459,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 38 unmodified lines
1101711459 * later look the same as if they were interpreted only.
1101811460 */
1101911461 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
11020− if (insn->code != (BPF_JMP | BPF_CALL) ||
11021− insn->src_reg != BPF_PSEUDO_CALL)
11462+ if (!bpf_pseudo_call(insn))
1102211463 continue;
1102311464 insn->off = env->insn_aux_data[i].call_imm;
1102411465 subprog = find_subprog(env, i + insn->off + 1);
@@ -11047,8 +11488,7 @@ out_undo_insn: · 22 unmodified lines
1104711488 /* cleanup main prog to be interpreted */
1104811489 prog->jit_requested = 0;
1104911490 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
11050− if (insn->code != (BPF_JMP | BPF_CALL) ||
11051− insn->src_reg != BPF_PSEUDO_CALL)
11491+ if (!bpf_pseudo_call(insn))
1105211492 continue;
1105311493 insn->off = 0;
1105411494 insn->imm = env->insn_aux_data[i].call_imm;
@@ -11083,8 +11523,7 @@ static int fixup_call_args(struct bpf_verifier_env *env) · 28 unmodified lines
1108311523 return -EINVAL;
1108411524 }
1108511525 for (i = 0; i < prog->len; i++, insn++) {
11086− if (insn->code != (BPF_JMP | BPF_CALL) ||
11087− insn->src_reg != BPF_PSEUDO_CALL)
11526+ if (!bpf_pseudo_call(insn))
1108811527 continue;
1108911528 depth = get_callee_stack_depth(env, insn, i);
1109011529 if (depth < 0)
@@ -11547,6 +11986,13 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) · 456 unmodified lines
1154711986 mark_reg_known_zero(env, regs, i);
1154811987 else if (regs[i].type == SCALAR_VALUE)
1154911988 mark_reg_unknown(env, regs, i);
11989+ else if (regs[i].type == PTR_TO_MEM_OR_NULL) {
11990+ const u32 mem_size = regs[i].mem_size;
11991+
11992+ mark_reg_known_zero(env, regs, i);
11993+ regs[i].mem_size = mem_size;
11994+ regs[i].id = ++env->id_gen;
11995+ }
1155011996 }
1155111997 } else {
1155211998 /* 1st arg to a function */
@@ -12125,6 +12571,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, · 572 unmodified lines
1212512571 env->strict_alignment = false;
1212612572
1212712573 env->allow_ptr_leaks = bpf_allow_ptr_leaks();
12574+ env->allow_uninit_stack = bpf_allow_uninit_stack();
1212812575 env->allow_ptr_to_map_access = bpf_allow_ptr_to_map_access();
1212912576 env->bypass_spec_v1 = bpf_bypass_spec_v1();
1213012577 env->bypass_spec_v4 = bpf_bypass_spec_v4();
kernel/trace/bpf_trace.c+6−0
@@ -1188,6 +1188,10 @@ BTF_SET_END(btf_allowlist_d_path) · 1187 unmodified lines
11881188
11891189 static bool bpf_d_path_allowed(const struct bpf_prog *prog)
11901190 {
1191+ if (prog->type == BPF_PROG_TYPE_TRACING &&
1192+ prog->expected_attach_type == BPF_TRACE_ITER)
1193+ return true;
1194+
11911195 if (prog->type == BPF_PROG_TYPE_LSM)
11921196 return bpf_lsm_is_sleepable_hook(prog->aux->attach_btf_id);
11931197
@@ -1757,6 +1761,8 @@ tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 563 unmodified lines
17571761 return &bpf_sk_storage_delete_tracing_proto;
17581762 case BPF_FUNC_sock_from_file:
17591763 return &bpf_sock_from_file_proto;
1764+ case BPF_FUNC_get_socket_cookie:
1765+ return &bpf_get_socket_ptr_cookie_proto;
17601766 #endif
17611767 case BPF_FUNC_seq_printf:
17621768 return prog->expected_attach_type == BPF_TRACE_ITER ?
lib/test_bpf.c+1−6
@@ -345,7 +345,7 @@ static int __bpf_fill_ja(struct bpf_test *self, unsigned int len, · 344 unmodified lines
345345
346346 static int bpf_fill_maxinsns11(struct bpf_test *self)
347347 {
348− /* Hits 70 passes on x86_64, so cannot get JITed there. */
348+ /* Hits 70 passes on x86_64 and triggers NOPs padding. */
349349 return __bpf_fill_ja(self, BPF_MAXINSNS, 68);
350350 }
351351
@@ -5318,15 +5318,10 @@ static struct bpf_test tests[] = { · 4966 unmodified lines
53185318 {
53195319 "BPF_MAXINSNS: Jump, gap, jump, ...",
53205320 { },
5321−#if defined(CONFIG_BPF_JIT_ALWAYS_ON) && defined(CONFIG_X86)
5322− CLASSIC | FLAG_NO_DATA | FLAG_EXPECTED_FAIL,
5323−#else
53245321 CLASSIC | FLAG_NO_DATA,
5325−#endif
53265322 { },
53275323 { { 0, 0xababcbac } },
53285324 .fill_helper = bpf_fill_maxinsns11,
5329− .expected_errcode = -ENOTSUPP,
53305325 },
53315326 {
53325327 "BPF_MAXINSNS: jump over MSH",
net/core/dev.c+14−18
@@ -2217,28 +2217,14 @@ static inline void net_timestamp_set(struct sk_buff *skb) · 2216 unmodified lines
22172217
22182218 bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)
22192219 {
2220− unsigned int len;
2221−
2222− if (!(dev->flags & IFF_UP))
2223− return false;
2224−
2225− len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
2226− if (skb->len <= len)
2227− return true;
2228−
2229− /* if TSO is enabled, we don't care about the length as the packet
2230− * could be forwarded without being segmented before
2231− */
2232− if (skb_is_gso(skb))
2233− return true;
2234−
2235− return false;
2220+ return __is_skb_forwardable(dev, skb, true);
22362221 }
22372222 EXPORT_SYMBOL_GPL(is_skb_forwardable);
22382223
2239−int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
2224+static int __dev_forward_skb2(struct net_device *dev, struct sk_buff *skb,
2225+ bool check_mtu)
22402226 {
2241− int ret = ____dev_forward_skb(dev, skb);
2227+ int ret = ____dev_forward_skb(dev, skb, check_mtu);
22422228
22432229 if (likely(!ret)) {
22442230 skb->protocol = eth_type_trans(skb, dev);
@@ -2247,6 +2233,11 @@ int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb) · 2 unmodified lines
22472233
22482234 return ret;
22492235 }
2236+
2237+int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
2238+{
2239+ return __dev_forward_skb2(dev, skb, true);
2240+}
22502241 EXPORT_SYMBOL_GPL(__dev_forward_skb);
22512242
22522243 /**
@@ -2273,6 +2264,11 @@ int dev_forward_skb(struct net_device *dev, struct sk_buff *skb) · 20 unmodified lines
22732264 }
22742265 EXPORT_SYMBOL_GPL(dev_forward_skb);
22752266
2267+int dev_forward_skb_nomtu(struct net_device *dev, struct sk_buff *skb)
2268+{
2269+ return __dev_forward_skb2(dev, skb, false) ?: netif_rx_internal(skb);
2270+}
2271+
22762272 static inline int deliver_skb(struct sk_buff *skb,
22772273 struct packet_type *pt_prev,
22782274 struct net_device *orig_dev)
net/core/filter.c+174−21
@@ -2083,13 +2083,13 @@ static const struct bpf_func_proto bpf_csum_level_proto = { · 2082 unmodified lines
20832083
20842084 static inline int __bpf_rx_skb(struct net_device *dev, struct sk_buff *skb)
20852085 {
2086− return dev_forward_skb(dev, skb);
2086+ return dev_forward_skb_nomtu(dev, skb);
20872087 }
20882088
20892089 static inline int __bpf_rx_skb_no_mac(struct net_device *dev,
20902090 struct sk_buff *skb)
20912091 {
2092− int ret = ____dev_forward_skb(dev, skb);
2092+ int ret = ____dev_forward_skb(dev, skb, false);
20932093
20942094 if (likely(!ret)) {
20952095 skb->dev = dev;
@@ -2480,7 +2480,7 @@ int skb_do_redirect(struct sk_buff *skb) · 384 unmodified lines
24802480 goto out_drop;
24812481 dev = ops->ndo_get_peer_dev(dev);
24822482 if (unlikely(!dev ||
2483− !is_skb_forwardable(dev, skb) ||
2483+ !(dev->flags & IFF_UP) ||
24842484 net_eq(net, dev_net(dev))))
24852485 goto out_drop;
24862486 skb->dev = dev;
@@ -3552,11 +3552,7 @@ static int bpf_skb_net_shrink(struct sk_buff *skb, u32 off, u32 len_diff, · 1065 unmodified lines
35523552 return 0;
35533553 }
35543554
3555−static u32 __bpf_skb_max_len(const struct sk_buff *skb)
3556−{
3557− return skb->dev ? skb->dev->mtu + skb->dev->hard_header_len :
3558− SKB_MAX_ALLOC;
3559−}
3555+#define BPF_SKB_MAX_LEN SKB_MAX_ALLOC
35603556
35613557 BPF_CALL_4(sk_skb_adjust_room, struct sk_buff *, skb, s32, len_diff,
35623558 u32, mode, u64, flags)
@@ -3605,7 +3601,7 @@ BPF_CALL_4(bpf_skb_adjust_room, struct sk_buff *, skb, s32, len_diff, · 42 unmodified lines
36053601 {
36063602 u32 len_cur, len_diff_abs = abs(len_diff);
36073603 u32 len_min = bpf_skb_net_base_len(skb);
3608− u32 len_max = __bpf_skb_max_len(skb);
3604+ u32 len_max = BPF_SKB_MAX_LEN;
36093605 __be16 proto = skb->protocol;
36103606 bool shrink = len_diff < 0;
36113607 u32 off;
@@ -3688,7 +3684,7 @@ static int bpf_skb_trim_rcsum(struct sk_buff *skb, unsigned int new_len) · 76 unmodified lines
36883684 static inline int __bpf_skb_change_tail(struct sk_buff *skb, u32 new_len,
36893685 u64 flags)
36903686 {
3691− u32 max_len = __bpf_skb_max_len(skb);
3687+ u32 max_len = BPF_SKB_MAX_LEN;
36923688 u32 min_len = __bpf_skb_min_len(skb);
36933689 int ret;
36943690
@@ -3764,7 +3760,7 @@ static const struct bpf_func_proto sk_skb_change_tail_proto = { · 69 unmodified lines
37643760 static inline int __bpf_skb_change_head(struct sk_buff *skb, u32 head_room,
37653761 u64 flags)
37663762 {
3767− u32 max_len = __bpf_skb_max_len(skb);
3763+ u32 max_len = BPF_SKB_MAX_LEN;
37683764 u32 new_len = skb->len + head_room;
37693765 int ret;
37703766
@@ -4631,6 +4627,18 @@ static const struct bpf_func_proto bpf_get_socket_cookie_sock_proto = { · 860 unmodified lines
46314627 .arg1_type = ARG_PTR_TO_CTX,
46324628 };
46334629
4630+BPF_CALL_1(bpf_get_socket_ptr_cookie, struct sock *, sk)
4631+{
4632+ return sk ? sock_gen_cookie(sk) : 0;
4633+}
4634+
4635+const struct bpf_func_proto bpf_get_socket_ptr_cookie_proto = {
4636+ .func = bpf_get_socket_ptr_cookie,
4637+ .gpl_only = false,
4638+ .ret_type = RET_INTEGER,
4639+ .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
4640+};
4641+
46344642 BPF_CALL_1(bpf_get_socket_cookie_sock_ops, struct bpf_sock_ops_kern *, ctx)
46354643 {
46364644 return __sock_gen_cookie(ctx->sk);
@@ -5291,12 +5299,14 @@ static const struct bpf_func_proto bpf_skb_get_xfrm_state_proto = { · 654 unmodified lines
52915299 #if IS_ENABLED(CONFIG_INET) || IS_ENABLED(CONFIG_IPV6)
52925300 static int bpf_fib_set_fwd_params(struct bpf_fib_lookup *params,
52935301 const struct neighbour *neigh,
5294− const struct net_device *dev)
5302+ const struct net_device *dev, u32 mtu)
52955303 {
52965304 memcpy(params->dmac, neigh->ha, ETH_ALEN);
52975305 memcpy(params->smac, dev->dev_addr, ETH_ALEN);
52985306 params->h_vlan_TCI = 0;
52995307 params->h_vlan_proto = 0;
5308+ if (mtu)
5309+ params->mtu_result = mtu; /* union with tot_len */
53005310
53015311 return 0;
53025312 }
@@ -5312,8 +5322,8 @@ static int bpf_ipv4_fib_lookup(struct net *net, struct bpf_fib_lookup *params, · 9 unmodified lines
53125322 struct net_device *dev;
53135323 struct fib_result res;
53145324 struct flowi4 fl4;
5325+ u32 mtu = 0;
53155326 int err;
5316− u32 mtu;
53175327
53185328 dev = dev_get_by_index_rcu(net, params->ifindex);
53195329 if (unlikely(!dev))
@@ -5380,8 +5390,10 @@ static int bpf_ipv4_fib_lookup(struct net *net, struct bpf_fib_lookup *params, · 60 unmodified lines
53805390
53815391 if (check_mtu) {
53825392 mtu = ip_mtu_from_fib_result(&res, params->ipv4_dst);
5383− if (params->tot_len > mtu)
5393+ if (params->tot_len > mtu) {
5394+ params->mtu_result = mtu; /* union with tot_len */
53845395 return BPF_FIB_LKUP_RET_FRAG_NEEDED;
5396+ }
53855397 }
53865398
53875399 nhc = res.nhc;
@@ -5415,7 +5427,7 @@ static int bpf_ipv4_fib_lookup(struct net *net, struct bpf_fib_lookup *params, · 27 unmodified lines
54155427 if (!neigh)
54165428 return BPF_FIB_LKUP_RET_NO_NEIGH;
54175429
5418− return bpf_fib_set_fwd_params(params, neigh, dev);
5430+ return bpf_fib_set_fwd_params(params, neigh, dev, mtu);
54195431 }
54205432 #endif
54215433
@@ -5432,7 +5444,7 @@ static int bpf_ipv6_fib_lookup(struct net *net, struct bpf_fib_lookup *params, · 10 unmodified lines
54325444 struct flowi6 fl6;
54335445 int strict = 0;
54345446 int oif, err;
5435− u32 mtu;
5447+ u32 mtu = 0;
54365448
54375449 /* link local addresses are never forwarded */
54385450 if (rt6_need_strict(dst) || rt6_need_strict(src))
@@ -5507,8 +5519,10 @@ static int bpf_ipv6_fib_lookup(struct net *net, struct bpf_fib_lookup *params, · 68 unmodified lines
55075519
55085520 if (check_mtu) {
55095521 mtu = ipv6_stub->ip6_mtu_from_fib6(&res, dst, src);
5510− if (params->tot_len > mtu)
5522+ if (params->tot_len > mtu) {
5523+ params->mtu_result = mtu; /* union with tot_len */
55115524 return BPF_FIB_LKUP_RET_FRAG_NEEDED;
5525+ }
55125526 }
55135527
55145528 if (res.nh->fib_nh_lws)
@@ -5528,7 +5542,7 @@ static int bpf_ipv6_fib_lookup(struct net *net, struct bpf_fib_lookup *params, · 13 unmodified lines
55285542 if (!neigh)
55295543 return BPF_FIB_LKUP_RET_NO_NEIGH;
55305544
5531− return bpf_fib_set_fwd_params(params, neigh, dev);
5545+ return bpf_fib_set_fwd_params(params, neigh, dev, mtu);
55325546 }
55335547 #endif
55345548
@@ -5571,6 +5585,7 @@ BPF_CALL_4(bpf_skb_fib_lookup, struct sk_buff *, skb, · 36 unmodified lines
55715585 {
55725586 struct net *net = dev_net(skb->dev);
55735587 int rc = -EAFNOSUPPORT;
5588+ bool check_mtu = false;
55745589
55755590 if (plen < sizeof(*params))
55765591 return -EINVAL;
@@ -5578,25 +5593,33 @@ BPF_CALL_4(bpf_skb_fib_lookup, struct sk_buff *, skb, · 1 unmodified line
55785593 if (flags & ~(BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_OUTPUT))
55795594 return -EINVAL;
55805595
5596+ if (params->tot_len)
5597+ check_mtu = true;
5598+
55815599 switch (params->family) {
55825600 #if IS_ENABLED(CONFIG_INET)
55835601 case AF_INET:
5584− rc = bpf_ipv4_fib_lookup(net, params, flags, false);
5602+ rc = bpf_ipv4_fib_lookup(net, params, flags, check_mtu);
55855603 break;
55865604 #endif
55875605 #if IS_ENABLED(CONFIG_IPV6)
55885606 case AF_INET6:
5589− rc = bpf_ipv6_fib_lookup(net, params, flags, false);
5607+ rc = bpf_ipv6_fib_lookup(net, params, flags, check_mtu);
55905608 break;
55915609 #endif
55925610 }
55935611
5594− if (!rc) {
5612+ if (rc == BPF_FIB_LKUP_RET_SUCCESS && !check_mtu) {
55955613 struct net_device *dev;
55965614
5615+ /* When tot_len isn't provided by user, check skb
5616+ * against MTU of FIB lookup resulting net_device
5617+ */
55975618 dev = dev_get_by_index_rcu(net, params->ifindex);
55985619 if (!is_skb_forwardable(dev, skb))
55995620 rc = BPF_FIB_LKUP_RET_FRAG_NEEDED;
5621+
5622+ params->mtu_result = dev->mtu; /* union with tot_len */
56005623 }
56015624
56025625 return rc;
@@ -5612,6 +5635,116 @@ static const struct bpf_func_proto bpf_skb_fib_lookup_proto = { · 9 unmodified lines
56125635 .arg4_type = ARG_ANYTHING,
56135636 };
56145637
5638+static struct net_device *__dev_via_ifindex(struct net_device *dev_curr,
5639+ u32 ifindex)
5640+{
5641+ struct net *netns = dev_net(dev_curr);
5642+
5643+ /* Non-redirect use-cases can use ifindex=0 and save ifindex lookup */
5644+ if (ifindex == 0)
5645+ return dev_curr;
5646+
5647+ return dev_get_by_index_rcu(netns, ifindex);
5648+}
5649+
5650+BPF_CALL_5(bpf_skb_check_mtu, struct sk_buff *, skb,
5651+ u32, ifindex, u32 *, mtu_len, s32, len_diff, u64, flags)
5652+{
5653+ int ret = BPF_MTU_CHK_RET_FRAG_NEEDED;
5654+ struct net_device *dev = skb->dev;
5655+ int skb_len, dev_len;
5656+ int mtu;
5657+
5658+ if (unlikely(flags & ~(BPF_MTU_CHK_SEGS)))
5659+ return -EINVAL;
5660+
5661+ if (unlikely(flags & BPF_MTU_CHK_SEGS && len_diff))
5662+ return -EINVAL;
5663+
5664+ dev = __dev_via_ifindex(dev, ifindex);
5665+ if (unlikely(!dev))
5666+ return -ENODEV;
5667+
5668+ mtu = READ_ONCE(dev->mtu);
5669+
5670+ dev_len = mtu + dev->hard_header_len;
5671+ skb_len = skb->len + len_diff; /* minus result pass check */
5672+ if (skb_len <= dev_len) {
5673+ ret = BPF_MTU_CHK_RET_SUCCESS;
5674+ goto out;
5675+ }
5676+ /* At this point, skb->len exceed MTU, but as it include length of all
5677+ * segments, it can still be below MTU. The SKB can possibly get
5678+ * re-segmented in transmit path (see validate_xmit_skb). Thus, user
5679+ * must choose if segs are to be MTU checked.
5680+ */
5681+ if (skb_is_gso(skb)) {
5682+ ret = BPF_MTU_CHK_RET_SUCCESS;
5683+
5684+ if (flags & BPF_MTU_CHK_SEGS &&
5685+ !skb_gso_validate_network_len(skb, mtu))
5686+ ret = BPF_MTU_CHK_RET_SEGS_TOOBIG;
5687+ }
5688+out:
5689+ /* BPF verifier guarantees valid pointer */
5690+ *mtu_len = mtu;
5691+
5692+ return ret;
5693+}
5694+
5695+BPF_CALL_5(bpf_xdp_check_mtu, struct xdp_buff *, xdp,
5696+ u32, ifindex, u32 *, mtu_len, s32, len_diff, u64, flags)
5697+{
5698+ struct net_device *dev = xdp->rxq->dev;
5699+ int xdp_len = xdp->data_end - xdp->data;
5700+ int ret = BPF_MTU_CHK_RET_SUCCESS;
5701+ int mtu, dev_len;
5702+
5703+ /* XDP variant doesn't support multi-buffer segment check (yet) */
5704+ if (unlikely(flags))
5705+ return -EINVAL;
5706+
5707+ dev = __dev_via_ifindex(dev, ifindex);
5708+ if (unlikely(!dev))
5709+ return -ENODEV;
5710+
5711+ mtu = READ_ONCE(dev->mtu);
5712+
5713+ /* Add L2-header as dev MTU is L3 size */
5714+ dev_len = mtu + dev->hard_header_len;
5715+
5716+ xdp_len += len_diff; /* minus result pass check */
5717+ if (xdp_len > dev_len)
5718+ ret = BPF_MTU_CHK_RET_FRAG_NEEDED;
5719+
5720+ /* BPF verifier guarantees valid pointer */
5721+ *mtu_len = mtu;
5722+
5723+ return ret;
5724+}
5725+
5726+static const struct bpf_func_proto bpf_skb_check_mtu_proto = {
5727+ .func = bpf_skb_check_mtu,
5728+ .gpl_only = true,
5729+ .ret_type = RET_INTEGER,
5730+ .arg1_type = ARG_PTR_TO_CTX,
5731+ .arg2_type = ARG_ANYTHING,
5732+ .arg3_type = ARG_PTR_TO_INT,
5733+ .arg4_type = ARG_ANYTHING,
5734+ .arg5_type = ARG_ANYTHING,
5735+};
5736+
5737+static const struct bpf_func_proto bpf_xdp_check_mtu_proto = {
5738+ .func = bpf_xdp_check_mtu,
5739+ .gpl_only = true,
5740+ .ret_type = RET_INTEGER,
5741+ .arg1_type = ARG_PTR_TO_CTX,
5742+ .arg2_type = ARG_ANYTHING,
5743+ .arg3_type = ARG_PTR_TO_INT,
5744+ .arg4_type = ARG_ANYTHING,
5745+ .arg5_type = ARG_ANYTHING,
5746+};
5747+
56155748 #if IS_ENABLED(CONFIG_IPV6_SEG6_BPF)
56165749 static int bpf_push_seg6_encap(struct sk_buff *skb, u32 type, void *hdr, u32 len)
56175750 {
@@ -7021,6 +7154,14 @@ sock_addr_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 1403 unmodified lines
70217154 case BPF_CGROUP_INET6_BIND:
70227155 case BPF_CGROUP_INET4_CONNECT:
70237156 case BPF_CGROUP_INET6_CONNECT:
7157+ case BPF_CGROUP_UDP4_RECVMSG:
7158+ case BPF_CGROUP_UDP6_RECVMSG:
7159+ case BPF_CGROUP_UDP4_SENDMSG:
7160+ case BPF_CGROUP_UDP6_SENDMSG:
7161+ case BPF_CGROUP_INET4_GETPEERNAME:
7162+ case BPF_CGROUP_INET6_GETPEERNAME:
7163+ case BPF_CGROUP_INET4_GETSOCKNAME:
7164+ case BPF_CGROUP_INET6_GETSOCKNAME:
70247165 return &bpf_sock_addr_setsockopt_proto;
70257166 default:
70267167 return NULL;
@@ -7031,6 +7172,14 @@ sock_addr_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 4 unmodified lines
70317172 case BPF_CGROUP_INET6_BIND:
70327173 case BPF_CGROUP_INET4_CONNECT:
70337174 case BPF_CGROUP_INET6_CONNECT:
7175+ case BPF_CGROUP_UDP4_RECVMSG:
7176+ case BPF_CGROUP_UDP6_RECVMSG:
7177+ case BPF_CGROUP_UDP4_SENDMSG:
7178+ case BPF_CGROUP_UDP6_SENDMSG:
7179+ case BPF_CGROUP_INET4_GETPEERNAME:
7180+ case BPF_CGROUP_INET6_GETPEERNAME:
7181+ case BPF_CGROUP_INET4_GETSOCKNAME:
7182+ case BPF_CGROUP_INET6_GETSOCKNAME:
70347183 return &bpf_sock_addr_getsockopt_proto;
70357184 default:
70367185 return NULL;
@@ -7181,6 +7330,8 @@ tc_cls_act_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 144 unmodified lines
71817330 return &bpf_get_socket_uid_proto;
71827331 case BPF_FUNC_fib_lookup:
71837332 return &bpf_skb_fib_lookup_proto;
7333+ case BPF_FUNC_check_mtu:
7334+ return &bpf_skb_check_mtu_proto;
71847335 case BPF_FUNC_sk_fullsock:
71857336 return &bpf_sk_fullsock_proto;
71867337 case BPF_FUNC_sk_storage_get:
@@ -7250,6 +7401,8 @@ xdp_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 63 unmodified lines
72507401 return &bpf_xdp_adjust_tail_proto;
72517402 case BPF_FUNC_fib_lookup:
72527403 return &bpf_xdp_fib_lookup_proto;
7404+ case BPF_FUNC_check_mtu:
7405+ return &bpf_xdp_check_mtu_proto;
72537406 #ifdef CONFIG_INET
72547407 case BPF_FUNC_sk_lookup_udp:
72557408 return &bpf_xdp_sk_lookup_udp_proto;
net/core/skmsg.c+1−2
@@ -669,14 +669,13 @@ static void sk_psock_destroy_deferred(struct work_struct *gc) · 668 unmodified lines
669669 kfree(psock);
670670 }
671671
672−void sk_psock_destroy(struct rcu_head *rcu)
672+static void sk_psock_destroy(struct rcu_head *rcu)
673673 {
674674 struct sk_psock *psock = container_of(rcu, struct sk_psock, rcu);
675675
676676 INIT_WORK(&psock->gc, sk_psock_destroy_deferred);
677677 schedule_work(&psock->gc);
678678 }
679−EXPORT_SYMBOL_GPL(sk_psock_destroy);
680679
681680 void sk_psock_drop(struct sock *sk, struct sk_psock *psock)
682681 {
net/core/xdp.c+70−0
@@ -513,3 +513,73 @@ void xdp_warn(const char *msg, const char *func, const int line) · 512 unmodified lines
513513 WARN(1, "XDP_WARN: %s(line:%d): %s\n", func, line, msg);
514514 };
515515 EXPORT_SYMBOL_GPL(xdp_warn);
516+
517+int xdp_alloc_skb_bulk(void **skbs, int n_skb, gfp_t gfp)
518+{
519+ n_skb = kmem_cache_alloc_bulk(skbuff_head_cache, gfp,
520+ n_skb, skbs);
521+ if (unlikely(!n_skb))
522+ return -ENOMEM;
523+
524+ return 0;
525+}
526+EXPORT_SYMBOL_GPL(xdp_alloc_skb_bulk);
527+
528+struct sk_buff *__xdp_build_skb_from_frame(struct xdp_frame *xdpf,
529+ struct sk_buff *skb,
530+ struct net_device *dev)
531+{
532+ unsigned int headroom, frame_size;
533+ void *hard_start;
534+
535+ /* Part of headroom was reserved to xdpf */
536+ headroom = sizeof(*xdpf) + xdpf->headroom;
537+
538+ /* Memory size backing xdp_frame data already have reserved
539+ * room for build_skb to place skb_shared_info in tailroom.
540+ */
541+ frame_size = xdpf->frame_sz;
542+
543+ hard_start = xdpf->data - headroom;
544+ skb = build_skb_around(skb, hard_start, frame_size);
545+ if (unlikely(!skb))
546+ return NULL;
547+
548+ skb_reserve(skb, headroom);
549+ __skb_put(skb, xdpf->len);
550+ if (xdpf->metasize)
551+ skb_metadata_set(skb, xdpf->metasize);
552+
553+ /* Essential SKB info: protocol and skb->dev */
554+ skb->protocol = eth_type_trans(skb, dev);
555+
556+ /* Optional SKB info, currently missing:
557+ * - HW checksum info (skb->ip_summed)
558+ * - HW RX hash (skb_set_hash)
559+ * - RX ring dev queue index (skb_record_rx_queue)
560+ */
561+
562+ /* Until page_pool get SKB return path, release DMA here */
563+ xdp_release_frame(xdpf);
564+
565+ /* Allow SKB to reuse area used by xdp_frame */
566+ xdp_scrub_frame(xdpf);
567+
568+ return skb;
569+}
570+EXPORT_SYMBOL_GPL(__xdp_build_skb_from_frame);
571+
572+struct sk_buff *xdp_build_skb_from_frame(struct xdp_frame *xdpf,
573+ struct net_device *dev)
574+{
575+ struct sk_buff *skb;
576+
577+ skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC);
578+ if (unlikely(!skb))
579+ return NULL;
580+
581+ memset(skb, 0, offsetof(struct sk_buff, tail));
582+
583+ return __xdp_build_skb_from_frame(xdpf, skb, dev);
584+}
585+EXPORT_SYMBOL_GPL(xdp_build_skb_from_frame);
net/ipv4/af_inet.c+11−7
@@ -438,6 +438,7 @@ EXPORT_SYMBOL(inet_release); · 437 unmodified lines
438438 int inet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
439439 {
440440 struct sock *sk = sock->sk;
441+ u32 flags = BIND_WITH_LOCK;
441442 int err;
442443
443444 /* If the socket has its own bind function then use it. (RAW) */
@@ -450,11 +451,12 @@ int inet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) · 6 unmodified lines
450451 /* BPF prog is run before any checks are done so that if the prog
451452 * changes context in a wrong way it will be caught.
452453 */
453− err = BPF_CGROUP_RUN_PROG_INET4_BIND_LOCK(sk, uaddr);
454+ err = BPF_CGROUP_RUN_PROG_INET_BIND_LOCK(sk, uaddr,
455+ BPF_CGROUP_INET4_BIND, &flags);
454456 if (err)
455457 return err;
456458
457− return __inet_bind(sk, uaddr, addr_len, BIND_WITH_LOCK);
459+ return __inet_bind(sk, uaddr, addr_len, flags);
458460 }
459461 EXPORT_SYMBOL(inet_bind);
460462
@@ -499,7 +501,8 @@ int __inet_bind(struct sock *sk, struct sockaddr *uaddr, int addr_len, · 38 unmodified lines
499501
500502 snum = ntohs(addr->sin_port);
501503 err = -EACCES;
502− if (snum && inet_port_requires_bind_service(net, snum) &&
504+ if (!(flags & BIND_NO_CAP_NET_BIND_SERVICE) &&
505+ snum && inet_port_requires_bind_service(net, snum) &&
503506 !ns_capable(net->user_ns, CAP_NET_BIND_SERVICE))
504507 goto out;
505508
@@ -777,18 +780,19 @@ int inet_getname(struct socket *sock, struct sockaddr *uaddr, · 271 unmodified lines
777780 return -ENOTCONN;
778781 sin->sin_port = inet->inet_dport;
779782 sin->sin_addr.s_addr = inet->inet_daddr;
783+ BPF_CGROUP_RUN_SA_PROG_LOCK(sk, (struct sockaddr *)sin,
784+ BPF_CGROUP_INET4_GETPEERNAME,
785+ NULL);
780786 } else {
781787 __be32 addr = inet->inet_rcv_saddr;
782788 if (!addr)
783789 addr = inet->inet_saddr;
784790 sin->sin_port = inet->inet_sport;
785791 sin->sin_addr.s_addr = addr;
786− }
787− if (cgroup_bpf_enabled)
788792 BPF_CGROUP_RUN_SA_PROG_LOCK(sk, (struct sockaddr *)sin,
789− peer ? BPF_CGROUP_INET4_GETPEERNAME :
790− BPF_CGROUP_INET4_GETSOCKNAME,
793+ BPF_CGROUP_INET4_GETSOCKNAME,
791794 NULL);
795+ }
792796 memset(sin->sin_zero, 0, sizeof(sin->sin_zero));
793797 return sizeof(*sin);
794798 }
net/ipv4/tcp.c+14−0
@@ -4162,6 +4162,8 @@ static int do_tcp_getsockopt(struct sock *sk, int level, · 4161 unmodified lines
41624162 return -EINVAL;
41634163 lock_sock(sk);
41644164 err = tcp_zerocopy_receive(sk, &zc, &tss);
4165+ err = BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sk, level, optname,
4166+ &zc, &len, err);
41654167 release_sock(sk);
41664168 if (len >= offsetofend(struct tcp_zerocopy_receive, msg_flags))
41674169 goto zerocopy_rcv_cmsg;
@@ -4208,6 +4210,18 @@ zerocopy_rcv_out: · 40 unmodified lines
42084210 return 0;
42094211 }
42104212
4213+bool tcp_bpf_bypass_getsockopt(int level, int optname)
4214+{
4215+ /* TCP do_tcp_getsockopt has optimized getsockopt implementation
4216+ * to avoid extra socket lock for TCP_ZEROCOPY_RECEIVE.
4217+ */
4218+ if (level == SOL_TCP && optname == TCP_ZEROCOPY_RECEIVE)
4219+ return true;
4220+
4221+ return false;
4222+}
4223+EXPORT_SYMBOL(tcp_bpf_bypass_getsockopt);
4224+
42114225 int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval,
42124226 int __user *optlen)
42134227 {
net/ipv4/tcp_ipv4.c+1−0
@@ -2796,6 +2796,7 @@ struct proto tcp_prot = { · 2795 unmodified lines
27962796 .shutdown = tcp_shutdown,
27972797 .setsockopt = tcp_setsockopt,
27982798 .getsockopt = tcp_getsockopt,
2799+ .bpf_bypass_getsockopt = tcp_bpf_bypass_getsockopt,
27992800 .keepalive = tcp_set_keepalive,
28002801 .recvmsg = tcp_recvmsg,
28012802 .sendmsg = tcp_sendmsg,
net/ipv4/udp.c+3−4
@@ -1130,7 +1130,7 @@ int udp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) · 1129 unmodified lines
11301130 rcu_read_unlock();
11311131 }
11321132
1133− if (cgroup_bpf_enabled && !connected) {
1133+ if (cgroup_bpf_enabled(BPF_CGROUP_UDP4_SENDMSG) && !connected) {
11341134 err = BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk,
11351135 (struct sockaddr *)usin, &ipc.addr);
11361136 if (err)
@@ -1864,9 +1864,8 @@ try_again: · 727 unmodified lines
18641864 memset(sin->sin_zero, 0, sizeof(sin->sin_zero));
18651865 *addr_len = sizeof(*sin);
18661866
1867− if (cgroup_bpf_enabled)
1868− BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk,
1869− (struct sockaddr *)sin);
1867+ BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk,
1868+ (struct sockaddr *)sin);
18701869 }
18711870
18721871 if (udp_sk(sk)->gro_enabled)
net/ipv6/af_inet6.c+11−7
@@ -295,7 +295,8 @@ static int __inet6_bind(struct sock *sk, struct sockaddr *uaddr, int addr_len, · 294 unmodified lines
295295 return -EINVAL;
296296
297297 snum = ntohs(addr->sin6_port);
298− if (snum && inet_port_requires_bind_service(net, snum) &&
298+ if (!(flags & BIND_NO_CAP_NET_BIND_SERVICE) &&
299+ snum && inet_port_requires_bind_service(net, snum) &&
299300 !ns_capable(net->user_ns, CAP_NET_BIND_SERVICE))
300301 return -EACCES;
301302
@@ -439,6 +440,7 @@ out_unlock: · 137 unmodified lines
439440 int inet6_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
440441 {
441442 struct sock *sk = sock->sk;
443+ u32 flags = BIND_WITH_LOCK;
442444 int err = 0;
443445
444446 /* If the socket has its own bind function then use it. */
@@ -451,11 +453,12 @@ int inet6_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) · 6 unmodified lines
451453 /* BPF prog is run before any checks are done so that if the prog
452454 * changes context in a wrong way it will be caught.
453455 */
454− err = BPF_CGROUP_RUN_PROG_INET6_BIND_LOCK(sk, uaddr);
456+ err = BPF_CGROUP_RUN_PROG_INET_BIND_LOCK(sk, uaddr,
457+ BPF_CGROUP_INET6_BIND, &flags);
455458 if (err)
456459 return err;
457460
458− return __inet6_bind(sk, uaddr, addr_len, BIND_WITH_LOCK);
461+ return __inet6_bind(sk, uaddr, addr_len, flags);
459462 }
460463 EXPORT_SYMBOL(inet6_bind);
461464
@@ -527,18 +530,19 @@ int inet6_getname(struct socket *sock, struct sockaddr *uaddr, · 65 unmodified lines
527530 sin->sin6_addr = sk->sk_v6_daddr;
528531 if (np->sndflow)
529532 sin->sin6_flowinfo = np->flow_label;
533+ BPF_CGROUP_RUN_SA_PROG_LOCK(sk, (struct sockaddr *)sin,
534+ BPF_CGROUP_INET6_GETPEERNAME,
535+ NULL);
530536 } else {
531537 if (ipv6_addr_any(&sk->sk_v6_rcv_saddr))
532538 sin->sin6_addr = np->saddr;
533539 else
534540 sin->sin6_addr = sk->sk_v6_rcv_saddr;
535541 sin->sin6_port = inet->inet_sport;
536− }
537− if (cgroup_bpf_enabled)
538542 BPF_CGROUP_RUN_SA_PROG_LOCK(sk, (struct sockaddr *)sin,
539− peer ? BPF_CGROUP_INET6_GETPEERNAME :
540− BPF_CGROUP_INET6_GETSOCKNAME,
543+ BPF_CGROUP_INET6_GETSOCKNAME,
541544 NULL);
545+ }
542546 sin->sin6_scope_id = ipv6_iface_scope_id(&sin->sin6_addr,
543547 sk->sk_bound_dev_if);
544548 return sizeof(*sin);
net/ipv6/tcp_ipv6.c+1−0
@@ -2124,6 +2124,7 @@ struct proto tcpv6_prot = { · 2123 unmodified lines
21242124 .shutdown = tcp_shutdown,
21252125 .setsockopt = tcp_setsockopt,
21262126 .getsockopt = tcp_getsockopt,
2127+ .bpf_bypass_getsockopt = tcp_bpf_bypass_getsockopt,
21272128 .keepalive = tcp_set_keepalive,
21282129 .recvmsg = tcp_recvmsg,
21292130 .sendmsg = tcp_sendmsg,
net/ipv6/udp.c+3−4
@@ -409,9 +409,8 @@ try_again: · 408 unmodified lines
409409 }
410410 *addr_len = sizeof(*sin6);
411411
412− if (cgroup_bpf_enabled)
413− BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK(sk,
414− (struct sockaddr *)sin6);
412+ BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK(sk,
413+ (struct sockaddr *)sin6);
415414 }
416415
417416 if (udp_sk(sk)->gro_enabled)
@@ -1462,7 +1461,7 @@ do_udp_sendmsg: · 1044 unmodified lines
14621461 fl6.saddr = np->saddr;
14631462 fl6.fl6_sport = inet->inet_sport;
14641463
1465− if (cgroup_bpf_enabled && !connected) {
1464+ if (cgroup_bpf_enabled(BPF_CGROUP_UDP6_SENDMSG) && !connected) {
14661465 err = BPF_CGROUP_RUN_PROG_UDP6_SENDMSG_LOCK(sk,
14671466 (struct sockaddr *)sin6, &fl6.saddr);
14681467 if (err)
net/socket.c+3−0
@@ -2126,6 +2126,9 @@ SYSCALL_DEFINE5(setsockopt, int, fd, int, level, int, optname, · 2125 unmodified lines
21262126 return __sys_setsockopt(fd, level, optname, optval, optlen);
21272127 }
21282128
2129+INDIRECT_CALLABLE_DECLARE(bool tcp_bpf_bypass_getsockopt(int level,
2130+ int optname));
2131+
21292132 /*
21302133 * Get a socket option. Because we don't know the option lengths we have
21312134 * to pass a user mode parameter for the protocols to sort out.
net/xdp/xsk.c+31−16
@@ -184,12 +184,13 @@ static void xsk_copy_xdp(struct xdp_buff *to, struct xdp_buff *from, u32 len) · 183 unmodified lines
184184 memcpy(to_buf, from_buf, len + metalen);
185185 }
186186
187−static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len,
188− bool explicit_free)
187+static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
189188 {
190189 struct xdp_buff *xsk_xdp;
191190 int err;
191+ u32 len;
192192
193+ len = xdp->data_end - xdp->data;
193194 if (len > xsk_pool_get_rx_frame_size(xs->pool)) {
194195 xs->rx_dropped++;
195196 return -ENOSPC;
@@ -207,8 +208,6 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len, · 11 unmodified lines
207208 xsk_buff_free(xsk_xdp);
208209 return err;
209210 }
210− if (explicit_free)
211− xdp_return_buff(xdp);
212211 return 0;
213212 }
214213
@@ -230,11 +229,8 @@ static bool xsk_is_bound(struct xdp_sock *xs) · 15 unmodified lines
230229 return false;
231230 }
232231
233−static int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp,
234− bool explicit_free)
232+static int xsk_rcv_check(struct xdp_sock *xs, struct xdp_buff *xdp)
235233 {
236− u32 len;
237−
238234 if (!xsk_is_bound(xs))
239235 return -EINVAL;
240236
@@ -242,11 +238,7 @@ static int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, · 1 unmodified line
242238 return -EINVAL;
243239
244240 sk_mark_napi_id_once_xdp(&xs->sk, xdp);
245− len = xdp->data_end - xdp->data;
246−
247− return xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL ?
248− __xsk_rcv_zc(xs, xdp, len) :
249− __xsk_rcv(xs, xdp, len, explicit_free);
241+ return 0;
250242 }
251243
252244 static void xsk_flush(struct xdp_sock *xs)
@@ -261,18 +253,41 @@ int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) · 8 unmodified lines
261253 int err;
262254
263255 spin_lock_bh(&xs->rx_lock);
264− err = xsk_rcv(xs, xdp, false);
265− xsk_flush(xs);
256+ err = xsk_rcv_check(xs, xdp);
257+ if (!err) {
258+ err = __xsk_rcv(xs, xdp);
259+ xsk_flush(xs);
260+ }
266261 spin_unlock_bh(&xs->rx_lock);
267262 return err;
268263 }
269264
265+static int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
266+{
267+ int err;
268+ u32 len;
269+
270+ err = xsk_rcv_check(xs, xdp);
271+ if (err)
272+ return err;
273+
274+ if (xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL) {
275+ len = xdp->data_end - xdp->data;
276+ return __xsk_rcv_zc(xs, xdp, len);
277+ }
278+
279+ err = __xsk_rcv(xs, xdp);
280+ if (!err)
281+ xdp_return_buff(xdp);
282+ return err;
283+}
284+
270285 int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp)
271286 {
272287 struct list_head *flush_list = this_cpu_ptr(&xskmap_flush_list);
273288 int err;
274289
275− err = xsk_rcv(xs, xdp, true);
290+ err = xsk_rcv(xs, xdp);
276291 if (err)
277292 return err;
278293
net/xdp/xsk_buff_pool.c+3−9
@@ -119,8 +119,8 @@ static void xp_disable_drv_zc(struct xsk_buff_pool *pool) · 118 unmodified lines
119119 }
120120 }
121121
122−static int __xp_assign_dev(struct xsk_buff_pool *pool,
123− struct net_device *netdev, u16 queue_id, u16 flags)
122+int xp_assign_dev(struct xsk_buff_pool *pool,
123+ struct net_device *netdev, u16 queue_id, u16 flags)
124124 {
125125 bool force_zc, force_copy;
126126 struct netdev_bpf bpf;
@@ -191,12 +191,6 @@ err_unreg_pool: · 64 unmodified lines
191191 return err;
192192 }
193193
194−int xp_assign_dev(struct xsk_buff_pool *pool, struct net_device *dev,
195− u16 queue_id, u16 flags)
196−{
197− return __xp_assign_dev(pool, dev, queue_id, flags);
198−}
199−
200194 int xp_assign_dev_shared(struct xsk_buff_pool *pool, struct xdp_umem *umem,
201195 struct net_device *dev, u16 queue_id)
202196 {
@@ -210,7 +204,7 @@ int xp_assign_dev_shared(struct xsk_buff_pool *pool, struct xdp_umem *umem, · 7 unmodified lines
210204 if (pool->uses_need_wakeup)
211205 flags |= XDP_USE_NEED_WAKEUP;
212206
213− return __xp_assign_dev(pool, dev, queue_id, flags);
207+ return xp_assign_dev(pool, dev, queue_id, flags);
214208 }
215209
216210 void xp_clear_dev(struct xsk_buff_pool *pool)
samples/bpf/Makefile+9−1
@@ -183,6 +183,14 @@ BPF_EXTRA_CFLAGS := $(ARM_ARCH_SELECTOR) · 182 unmodified lines
183183 TPROGS_CFLAGS += $(ARM_ARCH_SELECTOR)
184184 endif
185185
186+ifeq ($(ARCH), mips)
187+TPROGS_CFLAGS += -D__SANE_USERSPACE_TYPES__
188+ifdef CONFIG_MACH_LOONGSON64
189+BPF_EXTRA_CFLAGS += -I$(srctree)/arch/mips/include/asm/mach-loongson64
190+BPF_EXTRA_CFLAGS += -I$(srctree)/arch/mips/include/asm/mach-generic
191+endif
192+endif
193+
186194 TPROGS_CFLAGS += -Wall -O2
187195 TPROGS_CFLAGS += -Wmissing-prototypes
188196 TPROGS_CFLAGS += -Wstrict-prototypes
@@ -208,7 +216,7 @@ TPROGLDLIBS_xdpsock += -pthread -lcap · 19 unmodified lines
208216 TPROGLDLIBS_xsk_fwd += -pthread
209217
210218 # Allows pointing LLC/CLANG to a LLVM backend with bpf support, redefine on cmdline:
211−# make M=samples/bpf/ LLC=~/git/llvm/build/bin/llc CLANG=~/git/llvm/build/bin/clang
219+# make M=samples/bpf LLC=~/git/llvm-project/llvm/build/bin/llc CLANG=~/git/llvm-project/llvm/build/bin/clang
212220 LLC ?= llc
213221 CLANG ?= clang
214222 OPT ?= opt
samples/bpf/README.rst+14−8
@@ -62,20 +62,26 @@ To generate a smaller llc binary one can use:: · 61 unmodified lines
6262
6363 -DLLVM_TARGETS_TO_BUILD="BPF"
6464
65+We recommend that developers who want the fastest incremental builds
66+use the Ninja build system, you can find it in your system's package
67+manager, usually the package is ninja or ninja-build.
68+
6569 Quick sniplet for manually compiling LLVM and clang
66−(build dependencies are cmake and gcc-c++)::
70+(build dependencies are ninja, cmake and gcc-c++)::
6771
68− $ git clone http://llvm.org/git/llvm.git
69− $ cd llvm/tools
70− $ git clone --depth 1 http://llvm.org/git/clang.git
71− $ cd ..; mkdir build; cd build
72− $ cmake .. -DLLVM_TARGETS_TO_BUILD="BPF;X86"
73− $ make -j $(getconf _NPROCESSORS_ONLN)
72+ $ git clone https://github.com/llvm/llvm-project.git
73+ $ mkdir -p llvm-project/llvm/build
74+ $ cd llvm-project/llvm/build
75+ $ cmake .. -G "Ninja" -DLLVM_TARGETS_TO_BUILD="BPF;X86" \
76+ -DLLVM_ENABLE_PROJECTS="clang" \
77+ -DCMAKE_BUILD_TYPE=Release \
78+ -DLLVM_BUILD_RUNTIME=OFF
79+ $ ninja
7480
7581 It is also possible to point make to the newly compiled 'llc' or
7682 'clang' command via redefining LLC or CLANG on the make command line::
7783
78− make M=samples/bpf LLC=~/git/llvm/build/bin/llc CLANG=~/git/llvm/build/bin/clang
84+ make M=samples/bpf LLC=~/git/llvm-project/llvm/build/bin/llc CLANG=~/git/llvm-project/llvm/build/bin/clang
7985
8086 Cross compiling samples
8187 -----------------------
samples/bpf/bpf_insn.h+20−4
@@ -134,15 +134,31 @@ struct bpf_insn; · 133 unmodified lines
134134 .off = OFF, \
135135 .imm = 0 })
136136
137−/* Atomic memory add, *(uint *)(dst_reg + off16) += src_reg */
138−
139−#define BPF_STX_XADD(SIZE, DST, SRC, OFF) \
137+/*
138+ * Atomic operations:
139+ *
140+ * BPF_ADD *(uint *) (dst_reg + off16) += src_reg
141+ * BPF_AND *(uint *) (dst_reg + off16) &= src_reg
142+ * BPF_OR *(uint *) (dst_reg + off16) |= src_reg
143+ * BPF_XOR *(uint *) (dst_reg + off16) ^= src_reg
144+ * BPF_ADD | BPF_FETCH src_reg = atomic_fetch_add(dst_reg + off16, src_reg);
145+ * BPF_AND | BPF_FETCH src_reg = atomic_fetch_and(dst_reg + off16, src_reg);
146+ * BPF_OR | BPF_FETCH src_reg = atomic_fetch_or(dst_reg + off16, src_reg);
147+ * BPF_XOR | BPF_FETCH src_reg = atomic_fetch_xor(dst_reg + off16, src_reg);
148+ * BPF_XCHG src_reg = atomic_xchg(dst_reg + off16, src_reg)
149+ * BPF_CMPXCHG r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg)
150+ */
151+
152+#define BPF_ATOMIC_OP(SIZE, OP, DST, SRC, OFF) \
140153 ((struct bpf_insn) { \
141154 .code = BPF_STX | BPF_SIZE(SIZE) | BPF_ATOMIC, \
142155 .dst_reg = DST, \
143156 .src_reg = SRC, \
144157 .off = OFF, \
145− .imm = BPF_ADD })
158+ .imm = OP })
159+
160+/* Legacy alias */
161+#define BPF_STX_XADD(SIZE, DST, SRC, OFF) BPF_ATOMIC_OP(SIZE, BPF_ADD, DST, SRC, OFF)
146162
147163 /* Memory store, *(uint *) (dst_reg + off16) = imm32 */
148164
samples/bpf/cookie_uid_helper_example.c+1−1
@@ -313,7 +313,7 @@ int main(int argc, char *argv[]) · 312 unmodified lines
313313 print_table();
314314 printf("\n");
315315 sleep(1);
316− };
316+ }
317317 } else if (cfg_test_cookie) {
318318 udp_client();
319319 }
samples/bpf/xdp_redirect_map_kern.c+55−5
@@ -19,12 +19,22 @@ · 18 unmodified lines
1919 #include <linux/ipv6.h>
2020 #include <bpf/bpf_helpers.h>
2121
22+/* The 2nd xdp prog on egress does not support skb mode, so we define two
23+ * maps, tx_port_general and tx_port_native.
24+ */
2225 struct {
2326 __uint(type, BPF_MAP_TYPE_DEVMAP);
2427 __uint(key_size, sizeof(int));
2528 __uint(value_size, sizeof(int));
2629 __uint(max_entries, 100);
27−} tx_port SEC(".maps");
30+} tx_port_general SEC(".maps");
31+
32+struct {
33+ __uint(type, BPF_MAP_TYPE_DEVMAP);
34+ __uint(key_size, sizeof(int));
35+ __uint(value_size, sizeof(struct bpf_devmap_val));
36+ __uint(max_entries, 100);
37+} tx_port_native SEC(".maps");
2838
2939 /* Count RX packets, as XDP bpf_prog doesn't get direct TX-success
3040 * feedback. Redirect TX errors can be caught via a tracepoint.
@@ -36,6 +46,14 @@ struct { · 5 unmodified lines
3646 __uint(max_entries, 1);
3747 } rxcnt SEC(".maps");
3848
49+/* map to store egress interface mac address */
50+struct {
51+ __uint(type, BPF_MAP_TYPE_ARRAY);
52+ __type(key, u32);
53+ __type(value, __be64);
54+ __uint(max_entries, 1);
55+} tx_mac SEC(".maps");
56+
3957 static void swap_src_dst_mac(void *data)
4058 {
4159 unsigned short *p = data;
@@ -52,17 +70,16 @@ static void swap_src_dst_mac(void *data) · 10 unmodified lines
5270 p[5] = dst[2];
5371 }
5472
55−SEC("xdp_redirect_map")
56−int xdp_redirect_map_prog(struct xdp_md *ctx)
73+static __always_inline int xdp_redirect_map(struct xdp_md *ctx, void *redirect_map)
5774 {
5875 void *data_end = (void *)(long)ctx->data_end;
5976 void *data = (void *)(long)ctx->data;
6077 struct ethhdr *eth = data;
6178 int rc = XDP_DROP;
62− int vport, port = 0, m = 0;
6379 long *value;
6480 u32 key = 0;
6581 u64 nh_off;
82+ int vport;
6683
6784 nh_off = sizeof(*eth);
6885 if (data + nh_off > data_end)
@@ -79,7 +96,40 @@ int xdp_redirect_map_prog(struct xdp_md *ctx) · 10 unmodified lines
7996 swap_src_dst_mac(data);
8097
8198 /* send packet out physical port */
82− return bpf_redirect_map(&tx_port, vport, 0);
99+ return bpf_redirect_map(redirect_map, vport, 0);
100+}
101+
102+SEC("xdp_redirect_general")
103+int xdp_redirect_map_general(struct xdp_md *ctx)
104+{
105+ return xdp_redirect_map(ctx, &tx_port_general);
106+}
107+
108+SEC("xdp_redirect_native")
109+int xdp_redirect_map_native(struct xdp_md *ctx)
110+{
111+ return xdp_redirect_map(ctx, &tx_port_native);
112+}
113+
114+SEC("xdp_devmap/map_prog")
115+int xdp_redirect_map_egress(struct xdp_md *ctx)
116+{
117+ void *data_end = (void *)(long)ctx->data_end;
118+ void *data = (void *)(long)ctx->data;
119+ struct ethhdr *eth = data;
120+ __be64 *mac;
121+ u32 key = 0;
122+ u64 nh_off;
123+
124+ nh_off = sizeof(*eth);
125+ if (data + nh_off > data_end)
126+ return XDP_DROP;
127+
128+ mac = bpf_map_lookup_elem(&tx_mac, &key);
129+ if (mac)
130+ __builtin_memcpy(eth->h_source, mac, ETH_ALEN);
131+
132+ return XDP_PASS;
83133 }
84134
85135 /* Redirect require an XDP bpf_prog loaded on the TX device */
samples/bpf/xdp_redirect_map_user.c+92−20
@@ -14,6 +14,10 @@ · 13 unmodified lines
1414 #include <unistd.h>
1515 #include <libgen.h>
1616 #include <sys/resource.h>
17+#include <sys/ioctl.h>
18+#include <sys/types.h>
19+#include <sys/socket.h>
20+#include <netinet/in.h>
1721
1822 #include "bpf_util.h"
1923 #include <bpf/bpf.h>
@@ -22,6 +26,7 @@ · 2 unmodified lines
2226 static int ifindex_in;
2327 static int ifindex_out;
2428 static bool ifindex_out_xdp_dummy_attached = true;
29+static bool xdp_devmap_attached;
2530 static __u32 prog_id;
2631 static __u32 dummy_prog_id;
2732
@@ -83,6 +88,32 @@ static void poll_stats(int interval, int ifindex) · 55 unmodified lines
8388 }
8489 }
8590
91+static int get_mac_addr(unsigned int ifindex_out, void *mac_addr)
92+{
93+ char ifname[IF_NAMESIZE];
94+ struct ifreq ifr;
95+ int fd, ret = -1;
96+
97+ fd = socket(AF_INET, SOCK_DGRAM, 0);
98+ if (fd < 0)
99+ return ret;
100+
101+ if (!if_indextoname(ifindex_out, ifname))
102+ goto err_out;
103+
104+ strcpy(ifr.ifr_name, ifname);
105+
106+ if (ioctl(fd, SIOCGIFHWADDR, &ifr) != 0)
107+ goto err_out;
108+
109+ memcpy(mac_addr, ifr.ifr_hwaddr.sa_data, 6 * sizeof(char));
110+ ret = 0;
111+
112+err_out:
113+ close(fd);
114+ return ret;
115+}
116+
86117 static void usage(const char *prog)
87118 {
88119 fprintf(stderr,
@@ -90,24 +121,26 @@ static void usage(const char *prog) · 1 unmodified line
90121 "OPTS:\n"
91122 " -S use skb-mode\n"
92123 " -N enforce native mode\n"
93− " -F force loading prog\n",
124+ " -F force loading prog\n"
125+ " -X load xdp program on egress\n",
94126 prog);
95127 }
96128
97129 int main(int argc, char **argv)
98130 {
99131 struct bpf_prog_load_attr prog_load_attr = {
100− .prog_type = BPF_PROG_TYPE_XDP,
132+ .prog_type = BPF_PROG_TYPE_UNSPEC,
101133 };
102− struct bpf_program *prog, *dummy_prog;
134+ struct bpf_program *prog, *dummy_prog, *devmap_prog;
135+ int prog_fd, dummy_prog_fd, devmap_prog_fd = 0;
136+ int tx_port_map_fd, tx_mac_map_fd;
137+ struct bpf_devmap_val devmap_val;
103138 struct bpf_prog_info info = {};
104139 __u32 info_len = sizeof(info);
105− int prog_fd, dummy_prog_fd;
106− const char *optstr = "FSN";
140+ const char *optstr = "FSNX";
107141 struct bpf_object *obj;
108142 int ret, opt, key = 0;
109143 char filename[256];
110− int tx_port_map_fd;
111144
112145 while ((opt = getopt(argc, argv, optstr)) != -1) {
113146 switch (opt) {
@@ -120,14 +153,21 @@ int main(int argc, char **argv) · 6 unmodified lines
120153 case 'F':
121154 xdp_flags &= ~XDP_FLAGS_UPDATE_IF_NOEXIST;
122155 break;
156+ case 'X':
157+ xdp_devmap_attached = true;
158+ break;
123159 default:
124160 usage(basename(argv[0]));
125161 return 1;
126162 }
127163 }
128164
129− if (!(xdp_flags & XDP_FLAGS_SKB_MODE))
165+ if (!(xdp_flags & XDP_FLAGS_SKB_MODE)) {
130166 xdp_flags |= XDP_FLAGS_DRV_MODE;
167+ } else if (xdp_devmap_attached) {
168+ printf("Load xdp program on egress with SKB mode not supported yet\n");
169+ return 1;
170+ }
131171
132172 if (optind == argc) {
133173 printf("usage: %s <IFNAME|IFINDEX>_IN <IFNAME|IFINDEX>_OUT\n", argv[0]);
@@ -150,24 +190,28 @@ int main(int argc, char **argv) · 16 unmodified lines
150190 if (bpf_prog_load_xattr(&prog_load_attr, &obj, &prog_fd))
151191 return 1;
152192
153− prog = bpf_program__next(NULL, obj);
154− dummy_prog = bpf_program__next(prog, obj);
155− if (!prog || !dummy_prog) {
156− printf("finding a prog in obj file failed\n");
157− return 1;
193+ if (xdp_flags & XDP_FLAGS_SKB_MODE) {
194+ prog = bpf_object__find_program_by_name(obj, "xdp_redirect_map_general");
195+ tx_port_map_fd = bpf_object__find_map_fd_by_name(obj, "tx_port_general");
196+ } else {
197+ prog = bpf_object__find_program_by_name(obj, "xdp_redirect_map_native");
198+ tx_port_map_fd = bpf_object__find_map_fd_by_name(obj, "tx_port_native");
199+ }
200+ dummy_prog = bpf_object__find_program_by_name(obj, "xdp_redirect_dummy_prog");
201+ if (!prog || dummy_prog < 0 || tx_port_map_fd < 0) {
202+ printf("finding prog/dummy_prog/tx_port_map in obj file failed\n");
203+ goto out;
158204 }
159− /* bpf_prog_load_xattr gives us the pointer to first prog's fd,
160− * so we're missing only the fd for dummy prog
161− */
205+ prog_fd = bpf_program__fd(prog);
162206 dummy_prog_fd = bpf_program__fd(dummy_prog);
163− if (prog_fd < 0 || dummy_prog_fd < 0) {
207+ if (prog_fd < 0 || dummy_prog_fd < 0 || tx_port_map_fd < 0) {
164208 printf("bpf_prog_load_xattr: %s\n", strerror(errno));
165209 return 1;
166210 }
167211
168− tx_port_map_fd = bpf_object__find_map_fd_by_name(obj, "tx_port");
212+ tx_mac_map_fd = bpf_object__find_map_fd_by_name(obj, "tx_mac");
169213 rxcnt_map_fd = bpf_object__find_map_fd_by_name(obj, "rxcnt");
170− if (tx_port_map_fd < 0 || rxcnt_map_fd < 0) {
214+ if (tx_mac_map_fd < 0 || rxcnt_map_fd < 0) {
171215 printf("bpf_object__find_map_fd_by_name failed\n");
172216 return 1;
173217 }
@@ -199,11 +243,39 @@ int main(int argc, char **argv) · 25 unmodified lines
199243 }
200244 dummy_prog_id = info.id;
201245
246+ /* Load 2nd xdp prog on egress. */
247+ if (xdp_devmap_attached) {
248+ unsigned char mac_addr[6];
249+
250+ devmap_prog = bpf_object__find_program_by_name(obj, "xdp_redirect_map_egress");
251+ if (!devmap_prog) {
252+ printf("finding devmap_prog in obj file failed\n");
253+ goto out;
254+ }
255+ devmap_prog_fd = bpf_program__fd(devmap_prog);
256+ if (devmap_prog_fd < 0) {
257+ printf("finding devmap_prog fd failed\n");
258+ goto out;
259+ }
260+
261+ if (get_mac_addr(ifindex_out, mac_addr) < 0) {
262+ printf("get interface %d mac failed\n", ifindex_out);
263+ goto out;
264+ }
265+
266+ ret = bpf_map_update_elem(tx_mac_map_fd, &key, mac_addr, 0);
267+ if (ret) {
268+ perror("bpf_update_elem tx_mac_map_fd");
269+ goto out;
270+ }
271+ }
272+
202273 signal(SIGINT, int_exit);
203274 signal(SIGTERM, int_exit);
204275
205− /* populate virtual to physical port map */
206− ret = bpf_map_update_elem(tx_port_map_fd, &key, &ifindex_out, 0);
276+ devmap_val.ifindex = ifindex_out;
277+ devmap_val.bpf_prog.fd = devmap_prog_fd;
278+ ret = bpf_map_update_elem(tx_port_map_fd, &key, &devmap_val, 0);
207279 if (ret) {
208280 perror("bpf_update_elem");
209281 goto out;
tools/bpf/bpf_dbg.c+1−1
@@ -890,7 +890,7 @@ static int bpf_run_stepping(struct sock_filter *f, uint16_t bpf_len, · 889 unmodified lines
890890 bool stop = false;
891891 int i = 1;
892892
893− while (bpf_curr.Rs == false && stop == false) {
893+ while (!bpf_curr.Rs && !stop) {
894894 bpf_safe_regs();
895895
896896 if (i++ == next)
tools/bpf/bpftool/Makefile+0−2
@@ -75,8 +75,6 @@ endif · 74 unmodified lines
7575
7676 INSTALL ?= install
7777 RM ?= rm -f
78−CLANG ?= clang
79−LLVM_STRIP ?= llvm-strip
8078
8179 FEATURE_USER = .bpftool
8280 FEATURE_TESTS = libbfd disassembler-four-args reallocarray zlib libcap \
tools/bpf/bpftool/prog.c+4−0
@@ -368,6 +368,8 @@ static void print_prog_header_json(struct bpf_prog_info *info) · 367 unmodified lines
368368 jsonw_uint_field(json_wtr, "run_time_ns", info->run_time_ns);
369369 jsonw_uint_field(json_wtr, "run_cnt", info->run_cnt);
370370 }
371+ if (info->recursion_misses)
372+ jsonw_uint_field(json_wtr, "recursion_misses", info->recursion_misses);
371373 }
372374
373375 static void print_prog_json(struct bpf_prog_info *info, int fd)
@@ -446,6 +448,8 @@ static void print_prog_header_plain(struct bpf_prog_info *info) · 72 unmodified lines
446448 if (info->run_time_ns)
447449 printf(" run_time_ns %lld run_cnt %lld",
448450 info->run_time_ns, info->run_cnt);
451+ if (info->recursion_misses)
452+ printf(" recursion_misses %lld", info->recursion_misses);
449453 printf("\n");
450454 }
451455
tools/bpf/resolve_btfids/.gitignore+1−2
@@ -1,4 +1,3 @@
1−/FEATURE-DUMP.libbpf
2−/bpf_helper_defs.h
31 /fixdep
42 /resolve_btfids
3+/libbpf/
tools/bpf/resolve_btfids/Makefile+22−22
@@ -2,11 +2,7 @@ · 1 unmodified line
22 include ../../scripts/Makefile.include
33 include ../../scripts/Makefile.arch
44
5−ifeq ($(srctree),)
6−srctree := $(patsubst %/,%,$(dir $(CURDIR)))
7−srctree := $(patsubst %/,%,$(dir $(srctree)))
8−srctree := $(patsubst %/,%,$(dir $(srctree)))
9−endif
5+srctree := $(abspath $(CURDIR)/../../../)
106
117 ifeq ($(V),1)
128 Q =
@@ -22,28 +18,29 @@ AR = $(HOSTAR) · 9 unmodified lines
2218 CC = $(HOSTCC)
2319 LD = $(HOSTLD)
2420 ARCH = $(HOSTARCH)
21+RM ?= rm
2522
2623 OUTPUT ?= $(srctree)/tools/bpf/resolve_btfids/
2724
2825 LIBBPF_SRC := $(srctree)/tools/lib/bpf/
2926 SUBCMD_SRC := $(srctree)/tools/lib/subcmd/
3027
31−BPFOBJ := $(OUTPUT)/libbpf.a
32−SUBCMDOBJ := $(OUTPUT)/libsubcmd.a
28+BPFOBJ := $(OUTPUT)/libbpf/libbpf.a
29+SUBCMDOBJ := $(OUTPUT)/libsubcmd/libsubcmd.a
3330
3431 BINARY := $(OUTPUT)/resolve_btfids
3532 BINARY_IN := $(BINARY)-in.o
3633
3734 all: $(BINARY)
3835
39−$(OUTPUT):
36+$(OUTPUT) $(OUTPUT)/libbpf $(OUTPUT)/libsubcmd:
4037 $(call msg,MKDIR,,$@)
41− $(Q)mkdir -p $(OUTPUT)
38+ $(Q)mkdir -p $(@)
4239
43−$(SUBCMDOBJ): fixdep FORCE
44− $(Q)$(MAKE) -C $(SUBCMD_SRC) OUTPUT=$(OUTPUT)
40+$(SUBCMDOBJ): fixdep FORCE | $(OUTPUT)/libsubcmd
41+ $(Q)$(MAKE) -C $(SUBCMD_SRC) OUTPUT=$(abspath $(dir $@))/ $(abspath $@)
4542
46−$(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(OUTPUT)
43+$(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(OUTPUT)/libbpf
4744 $(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) OUTPUT=$(abspath $(dir $@))/ $(abspath $@)
4845
4946 CFLAGS := -g \
@@ -57,24 +54,27 @@ LIBS = -lelf -lz · 7 unmodified lines
5754 export srctree OUTPUT CFLAGS Q
5855 include $(srctree)/tools/build/Makefile.include
5956
60−$(BINARY_IN): fixdep FORCE
57+$(BINARY_IN): fixdep FORCE | $(OUTPUT)
6158 $(Q)$(MAKE) $(build)=resolve_btfids
6259
6360 $(BINARY): $(BPFOBJ) $(SUBCMDOBJ) $(BINARY_IN)
6461 $(call msg,LINK,$@)
6562 $(Q)$(CC) $(BINARY_IN) $(LDFLAGS) -o $@ $(BPFOBJ) $(SUBCMDOBJ) $(LIBS)
6663
67−libsubcmd-clean:
68− $(Q)$(MAKE) -C $(SUBCMD_SRC) OUTPUT=$(OUTPUT) clean
69−
70−libbpf-clean:
71− $(Q)$(MAKE) -C $(LIBBPF_SRC) OUTPUT=$(OUTPUT) clean
64+clean_objects := $(wildcard $(OUTPUT)/*.o \
65+ $(OUTPUT)/.*.o.cmd \
66+ $(OUTPUT)/.*.o.d \
67+ $(OUTPUT)/libbpf \
68+ $(OUTPUT)/libsubcmd \
69+ $(OUTPUT)/resolve_btfids)
7270
73−clean: libsubcmd-clean libbpf-clean fixdep-clean
71+ifneq ($(clean_objects),)
72+clean: fixdep-clean
7473 $(call msg,CLEAN,$(BINARY))
75− $(Q)$(RM) -f $(BINARY); \
76− $(RM) -rf $(if $(OUTPUT),$(OUTPUT),.)/feature; \
77− find $(if $(OUTPUT),$(OUTPUT),.) -name \*.o -or -name \*.o.cmd -or -name \*.o.d | xargs $(RM)
74+ $(Q)$(RM) -rf $(clean_objects)
75+else
76+clean:
77+endif
7878
7979 tags:
8080 $(call msg,GEN,,tags)
tools/bpf/runqslower/Makefile+0−3
@@ -3,9 +3,6 @@ include ../../scripts/Makefile.include · 2 unmodified lines
33
44 OUTPUT ?= $(abspath .output)/
55
6−CLANG ?= clang
7−LLC ?= llc
8−LLVM_STRIP ?= llvm-strip
96 BPFTOOL_OUTPUT := $(OUTPUT)bpftool/
107 DEFAULT_BPFTOOL := $(BPFTOOL_OUTPUT)bpftool
118 BPFTOOL ?= $(DEFAULT_BPFTOOL)
tools/build/feature/Makefile+2−2
@@ -1,4 +1,6 @@
11 # SPDX-License-Identifier: GPL-2.0
2+include ../../scripts/Makefile.include
3+
24 FILES= \
35 test-all.bin \
46 test-backtrace.bin \
@@ -76,8 +78,6 @@ FILES= \ · 71 unmodified lines
7678 FILES := $(addprefix $(OUTPUT),$(FILES))
7779
7880 PKG_CONFIG ?= $(CROSS_COMPILE)pkg-config
79−LLVM_CONFIG ?= llvm-config
80−CLANG ?= clang
8181
8282 all: $(FILES)
8383
tools/include/linux/types.h+3−0
@@ -6,7 +6,10 @@ · 5 unmodified lines
66 #include <stddef.h>
77 #include <stdint.h>
88
9+#ifndef __SANE_USERSPACE_TYPES__
910 #define __SANE_USERSPACE_TYPES__ /* For PPC64, to get LL64 types */
11+#endif
12+
1013 #include <asm/types.h>
1114 #include <asm/posix_types.h>
1215
tools/include/uapi/linux/bpf.h+97−6
@@ -1656,22 +1656,30 @@ union bpf_attr { · 1655 unmodified lines
16561656 * networking traffic statistics as it provides a global socket
16571657 * identifier that can be assumed unique.
16581658 * Return
1659− * A 8-byte long non-decreasing number on success, or 0 if the
1660− * socket field is missing inside *skb*.
1659+ * A 8-byte long unique number on success, or 0 if the socket
1660+ * field is missing inside *skb*.
16611661 *
16621662 * u64 bpf_get_socket_cookie(struct bpf_sock_addr *ctx)
16631663 * Description
16641664 * Equivalent to bpf_get_socket_cookie() helper that accepts
16651665 * *skb*, but gets socket from **struct bpf_sock_addr** context.
16661666 * Return
1667− * A 8-byte long non-decreasing number.
1667+ * A 8-byte long unique number.
16681668 *
16691669 * u64 bpf_get_socket_cookie(struct bpf_sock_ops *ctx)
16701670 * Description
16711671 * Equivalent to **bpf_get_socket_cookie**\ () helper that accepts
16721672 * *skb*, but gets socket from **struct bpf_sock_ops** context.
16731673 * Return
1674− * A 8-byte long non-decreasing number.
1674+ * A 8-byte long unique number.
1675+ *
1676+ * u64 bpf_get_socket_cookie(struct sock *sk)
1677+ * Description
1678+ * Equivalent to **bpf_get_socket_cookie**\ () helper that accepts
1679+ * *sk*, but gets socket from a BTF **struct sock**. This helper
1680+ * also works for sleepable programs.
1681+ * Return
1682+ * A 8-byte long unique number or 0 if *sk* is NULL.
16751683 *
16761684 * u32 bpf_get_socket_uid(struct sk_buff *skb)
16771685 * Return
@@ -2231,6 +2239,9 @@ union bpf_attr { · 553 unmodified lines
22312239 * * > 0 one of **BPF_FIB_LKUP_RET_** codes explaining why the
22322240 * packet is not forwarded or needs assist from full stack
22332241 *
2242+ * If lookup fails with BPF_FIB_LKUP_RET_FRAG_NEEDED, then the MTU
2243+ * was exceeded and output params->mtu_result contains the MTU.
2244+ *
22342245 * long bpf_sock_hash_update(struct bpf_sock_ops *skops, struct bpf_map *map, void *key, u64 flags)
22352246 * Description
22362247 * Add an entry to, or update a sockhash *map* referencing sockets.
@@ -3836,6 +3847,69 @@ union bpf_attr { · 1599 unmodified lines
38363847 * Return
38373848 * A pointer to a struct socket on success or NULL if the file is
38383849 * not a socket.
3850+ *
3851+ * long bpf_check_mtu(void *ctx, u32 ifindex, u32 *mtu_len, s32 len_diff, u64 flags)
3852+ * Description
3853+
3854+ * Check ctx packet size against exceeding MTU of net device (based
3855+ * on *ifindex*). This helper will likely be used in combination
3856+ * with helpers that adjust/change the packet size.
3857+ *
3858+ * The argument *len_diff* can be used for querying with a planned
3859+ * size change. This allows to check MTU prior to changing packet
3860+ * ctx. Providing an *len_diff* adjustment that is larger than the
3861+ * actual packet size (resulting in negative packet size) will in
3862+ * principle not exceed the MTU, why it is not considered a
3863+ * failure. Other BPF-helpers are needed for performing the
3864+ * planned size change, why the responsability for catch a negative
3865+ * packet size belong in those helpers.
3866+ *
3867+ * Specifying *ifindex* zero means the MTU check is performed
3868+ * against the current net device. This is practical if this isn't
3869+ * used prior to redirect.
3870+ *
3871+ * The Linux kernel route table can configure MTUs on a more
3872+ * specific per route level, which is not provided by this helper.
3873+ * For route level MTU checks use the **bpf_fib_lookup**\ ()
3874+ * helper.
3875+ *
3876+ * *ctx* is either **struct xdp_md** for XDP programs or
3877+ * **struct sk_buff** for tc cls_act programs.
3878+ *
3879+ * The *flags* argument can be a combination of one or more of the
3880+ * following values:
3881+ *
3882+ * **BPF_MTU_CHK_SEGS**
3883+ * This flag will only works for *ctx* **struct sk_buff**.
3884+ * If packet context contains extra packet segment buffers
3885+ * (often knows as GSO skb), then MTU check is harder to
3886+ * check at this point, because in transmit path it is
3887+ * possible for the skb packet to get re-segmented
3888+ * (depending on net device features). This could still be
3889+ * a MTU violation, so this flag enables performing MTU
3890+ * check against segments, with a different violation
3891+ * return code to tell it apart. Check cannot use len_diff.
3892+ *
3893+ * On return *mtu_len* pointer contains the MTU value of the net
3894+ * device. Remember the net device configured MTU is the L3 size,
3895+ * which is returned here and XDP and TX length operate at L2.
3896+ * Helper take this into account for you, but remember when using
3897+ * MTU value in your BPF-code. On input *mtu_len* must be a valid
3898+ * pointer and be initialized (to zero), else verifier will reject
3899+ * BPF program.
3900+ *
3901+ * Return
3902+ * * 0 on success, and populate MTU value in *mtu_len* pointer.
3903+ *
3904+ * * < 0 if any input argument is invalid (*mtu_len* not updated)
3905+ *
3906+ * MTU violations return positive values, but also populate MTU
3907+ * value in *mtu_len* pointer, as this can be needed for
3908+ * implementing PMTU handing:
3909+ *
3910+ * * **BPF_MTU_CHK_RET_FRAG_NEEDED**
3911+ * * **BPF_MTU_CHK_RET_SEGS_TOOBIG**
3912+ *
38393913 */
38403914 #define __BPF_FUNC_MAPPER(FN) \
38413915 FN(unspec), \
@@ -4001,6 +4075,7 @@ union bpf_attr { · 159 unmodified lines
40014075 FN(ktime_get_coarse_ns), \
40024076 FN(ima_inode_hash), \
40034077 FN(sock_from_file), \
4078+ FN(check_mtu), \
40044079 /* */
40054080
40064081 /* integer value in 'imm' field of BPF_CALL instruction selects which helper
@@ -4501,6 +4576,7 @@ struct bpf_prog_info { · 494 unmodified lines
45014576 __aligned_u64 prog_tags;
45024577 __u64 run_time_ns;
45034578 __u64 run_cnt;
4579+ __u64 recursion_misses;
45044580 } __attribute__((aligned(8)));
45054581
45064582 struct bpf_map_info {
@@ -4981,9 +5057,13 @@ struct bpf_fib_lookup { · 474 unmodified lines
49815057 __be16 sport;
49825058 __be16 dport;
49835059
4984− /* total length of packet from network header - used for MTU check */
4985− __u16 tot_len;
5060+ union { /* used for MTU check */
5061+ /* input to lookup */
5062+ __u16 tot_len; /* L3 length from network hdr (iph->tot_len) */
49865063
5064+ /* output: MTU value */
5065+ __u16 mtu_result;
5066+ };
49875067 /* input: L3 device index for lookup
49885068 * output: device index from FIB lookup
49895069 */
@@ -5029,6 +5109,17 @@ struct bpf_redir_neigh { · 39 unmodified lines
50295109 };
50305110 };
50315111
5112+/* bpf_check_mtu flags*/
5113+enum bpf_check_mtu_flags {
5114+ BPF_MTU_CHK_SEGS = (1U << 0),
5115+};
5116+
5117+enum bpf_check_mtu_ret {
5118+ BPF_MTU_CHK_RET_SUCCESS, /* check and lookup successful */
5119+ BPF_MTU_CHK_RET_FRAG_NEEDED, /* fragmentation required to fwd */
5120+ BPF_MTU_CHK_RET_SEGS_TOOBIG, /* GSO re-segmentation needed to fwd */
5121+};
5122+
50325123 enum bpf_task_fd_type {
50335124 BPF_FD_TYPE_RAW_TRACEPOINT, /* tp name */
50345125 BPF_FD_TYPE_TRACEPOINT, /* tp name */
tools/include/uapi/linux/bpf_perf_event.h+1−0
@@ -13,6 +13,7 @@ · 12 unmodified lines
1313 struct bpf_perf_event_data {
1414 bpf_user_pt_regs_t regs;
1515 __u64 sample_period;
16+ __u64 addr;
1617 };
1718
1819 #endif /* _UAPI__LINUX_BPF_PERF_EVENT_H__ */
tools/include/uapi/linux/tcp.h+357−0
@@ -0,0 +1,357 @@
1+/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */
2+/*
3+ * INET An implementation of the TCP/IP protocol suite for the LINUX
4+ * operating system. INET is implemented using the BSD Socket
5+ * interface as the means of communication with the user level.
6+ *
7+ * Definitions for the TCP protocol.
8+ *
9+ * Version: @(#)tcp.h 1.0.2 04/28/93
10+ *
11+ * Author: Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12+ *
13+ * This program is free software; you can redistribute it and/or
14+ * modify it under the terms of the GNU General Public License
15+ * as published by the Free Software Foundation; either version
16+ * 2 of the License, or (at your option) any later version.
17+ */
18+#ifndef _UAPI_LINUX_TCP_H
19+#define _UAPI_LINUX_TCP_H
20+
21+#include <linux/types.h>
22+#include <asm/byteorder.h>
23+#include <linux/socket.h>
24+
25+struct tcphdr {
26+ __be16 source;
27+ __be16 dest;
28+ __be32 seq;
29+ __be32 ack_seq;
30+#if defined(__LITTLE_ENDIAN_BITFIELD)
31+ __u16 res1:4,
32+ doff:4,
33+ fin:1,
34+ syn:1,
35+ rst:1,
36+ psh:1,
37+ ack:1,
38+ urg:1,
39+ ece:1,
40+ cwr:1;
41+#elif defined(__BIG_ENDIAN_BITFIELD)
42+ __u16 doff:4,
43+ res1:4,
44+ cwr:1,
45+ ece:1,
46+ urg:1,
47+ ack:1,
48+ psh:1,
49+ rst:1,
50+ syn:1,
51+ fin:1;
52+#else
53+#error "Adjust your <asm/byteorder.h> defines"
54+#endif
55+ __be16 window;
56+ __sum16 check;
57+ __be16 urg_ptr;
58+};
59+
60+/*
61+ * The union cast uses a gcc extension to avoid aliasing problems
62+ * (union is compatible to any of its members)
63+ * This means this part of the code is -fstrict-aliasing safe now.
64+ */
65+union tcp_word_hdr {
66+ struct tcphdr hdr;
67+ __be32 words[5];
68+};
69+
70+#define tcp_flag_word(tp) ( ((union tcp_word_hdr *)(tp))->words [3])
71+
72+enum {
73+ TCP_FLAG_CWR = __constant_cpu_to_be32(0x00800000),
74+ TCP_FLAG_ECE = __constant_cpu_to_be32(0x00400000),
75+ TCP_FLAG_URG = __constant_cpu_to_be32(0x00200000),
76+ TCP_FLAG_ACK = __constant_cpu_to_be32(0x00100000),
77+ TCP_FLAG_PSH = __constant_cpu_to_be32(0x00080000),
78+ TCP_FLAG_RST = __constant_cpu_to_be32(0x00040000),
79+ TCP_FLAG_SYN = __constant_cpu_to_be32(0x00020000),
80+ TCP_FLAG_FIN = __constant_cpu_to_be32(0x00010000),
81+ TCP_RESERVED_BITS = __constant_cpu_to_be32(0x0F000000),
82+ TCP_DATA_OFFSET = __constant_cpu_to_be32(0xF0000000)
83+};
84+
85+/*
86+ * TCP general constants
87+ */
88+#define TCP_MSS_DEFAULT 536U /* IPv4 (RFC1122, RFC2581) */
89+#define TCP_MSS_DESIRED 1220U /* IPv6 (tunneled), EDNS0 (RFC3226) */
90+
91+/* TCP socket options */
92+#define TCP_NODELAY 1 /* Turn off Nagle's algorithm. */
93+#define TCP_MAXSEG 2 /* Limit MSS */
94+#define TCP_CORK 3 /* Never send partially complete segments */
95+#define TCP_KEEPIDLE 4 /* Start keeplives after this period */
96+#define TCP_KEEPINTVL 5 /* Interval between keepalives */
97+#define TCP_KEEPCNT 6 /* Number of keepalives before death */
98+#define TCP_SYNCNT 7 /* Number of SYN retransmits */
99+#define TCP_LINGER2 8 /* Life time of orphaned FIN-WAIT-2 state */
100+#define TCP_DEFER_ACCEPT 9 /* Wake up listener only when data arrive */
101+#define TCP_WINDOW_CLAMP 10 /* Bound advertised window */
102+#define TCP_INFO 11 /* Information about this connection. */
103+#define TCP_QUICKACK 12 /* Block/reenable quick acks */
104+#define TCP_CONGESTION 13 /* Congestion control algorithm */
105+#define TCP_MD5SIG 14 /* TCP MD5 Signature (RFC2385) */
106+#define TCP_THIN_LINEAR_TIMEOUTS 16 /* Use linear timeouts for thin streams*/
107+#define TCP_THIN_DUPACK 17 /* Fast retrans. after 1 dupack */
108+#define TCP_USER_TIMEOUT 18 /* How long for loss retry before timeout */
109+#define TCP_REPAIR 19 /* TCP sock is under repair right now */
110+#define TCP_REPAIR_QUEUE 20
111+#define TCP_QUEUE_SEQ 21
112+#define TCP_REPAIR_OPTIONS 22
113+#define TCP_FASTOPEN 23 /* Enable FastOpen on listeners */
114+#define TCP_TIMESTAMP 24
115+#define TCP_NOTSENT_LOWAT 25 /* limit number of unsent bytes in write queue */
116+#define TCP_CC_INFO 26 /* Get Congestion Control (optional) info */
117+#define TCP_SAVE_SYN 27 /* Record SYN headers for new connections */
118+#define TCP_SAVED_SYN 28 /* Get SYN headers recorded for connection */
119+#define TCP_REPAIR_WINDOW 29 /* Get/set window parameters */
120+#define TCP_FASTOPEN_CONNECT 30 /* Attempt FastOpen with connect */
121+#define TCP_ULP 31 /* Attach a ULP to a TCP connection */
122+#define TCP_MD5SIG_EXT 32 /* TCP MD5 Signature with extensions */
123+#define TCP_FASTOPEN_KEY 33 /* Set the key for Fast Open (cookie) */
124+#define TCP_FASTOPEN_NO_COOKIE 34 /* Enable TFO without a TFO cookie */
125+#define TCP_ZEROCOPY_RECEIVE 35
126+#define TCP_INQ 36 /* Notify bytes available to read as a cmsg on read */
127+
128+#define TCP_CM_INQ TCP_INQ
129+
130+#define TCP_TX_DELAY 37 /* delay outgoing packets by XX usec */
131+
132+
133+#define TCP_REPAIR_ON 1
134+#define TCP_REPAIR_OFF 0
135+#define TCP_REPAIR_OFF_NO_WP -1 /* Turn off without window probes */
136+
137+struct tcp_repair_opt {
138+ __u32 opt_code;
139+ __u32 opt_val;
140+};
141+
142+struct tcp_repair_window {
143+ __u32 snd_wl1;
144+ __u32 snd_wnd;
145+ __u32 max_window;
146+
147+ __u32 rcv_wnd;
148+ __u32 rcv_wup;
149+};
150+
151+enum {
152+ TCP_NO_QUEUE,
153+ TCP_RECV_QUEUE,
154+ TCP_SEND_QUEUE,
155+ TCP_QUEUES_NR,
156+};
157+
158+/* why fastopen failed from client perspective */
159+enum tcp_fastopen_client_fail {
160+ TFO_STATUS_UNSPEC, /* catch-all */
161+ TFO_COOKIE_UNAVAILABLE, /* if not in TFO_CLIENT_NO_COOKIE mode */
162+ TFO_DATA_NOT_ACKED, /* SYN-ACK did not ack SYN data */
163+ TFO_SYN_RETRANSMITTED, /* SYN-ACK did not ack SYN data after timeout */
164+};
165+
166+/* for TCP_INFO socket option */
167+#define TCPI_OPT_TIMESTAMPS 1
168+#define TCPI_OPT_SACK 2
169+#define TCPI_OPT_WSCALE 4
170+#define TCPI_OPT_ECN 8 /* ECN was negociated at TCP session init */
171+#define TCPI_OPT_ECN_SEEN 16 /* we received at least one packet with ECT */
172+#define TCPI_OPT_SYN_DATA 32 /* SYN-ACK acked data in SYN sent or rcvd */
173+
174+/*
175+ * Sender's congestion state indicating normal or abnormal situations
176+ * in the last round of packets sent. The state is driven by the ACK
177+ * information and timer events.
178+ */
179+enum tcp_ca_state {
180+ /*
181+ * Nothing bad has been observed recently.
182+ * No apparent reordering, packet loss, or ECN marks.
183+ */
184+ TCP_CA_Open = 0,
185+#define TCPF_CA_Open (1<<TCP_CA_Open)
186+ /*
187+ * The sender enters disordered state when it has received DUPACKs or
188+ * SACKs in the last round of packets sent. This could be due to packet
189+ * loss or reordering but needs further information to confirm packets
190+ * have been lost.
191+ */
192+ TCP_CA_Disorder = 1,
193+#define TCPF_CA_Disorder (1<<TCP_CA_Disorder)
194+ /*
195+ * The sender enters Congestion Window Reduction (CWR) state when it
196+ * has received ACKs with ECN-ECE marks, or has experienced congestion
197+ * or packet discard on the sender host (e.g. qdisc).
198+ */
199+ TCP_CA_CWR = 2,
200+#define TCPF_CA_CWR (1<<TCP_CA_CWR)
201+ /*
202+ * The sender is in fast recovery and retransmitting lost packets,
203+ * typically triggered by ACK events.
204+ */
205+ TCP_CA_Recovery = 3,
206+#define TCPF_CA_Recovery (1<<TCP_CA_Recovery)
207+ /*
208+ * The sender is in loss recovery triggered by retransmission timeout.
209+ */
210+ TCP_CA_Loss = 4
211+#define TCPF_CA_Loss (1<<TCP_CA_Loss)
212+};
213+
214+struct tcp_info {
215+ __u8 tcpi_state;
216+ __u8 tcpi_ca_state;
217+ __u8 tcpi_retransmits;
218+ __u8 tcpi_probes;
219+ __u8 tcpi_backoff;
220+ __u8 tcpi_options;
221+ __u8 tcpi_snd_wscale : 4, tcpi_rcv_wscale : 4;
222+ __u8 tcpi_delivery_rate_app_limited:1, tcpi_fastopen_client_fail:2;
223+
224+ __u32 tcpi_rto;
225+ __u32 tcpi_ato;
226+ __u32 tcpi_snd_mss;
227+ __u32 tcpi_rcv_mss;
228+
229+ __u32 tcpi_unacked;
230+ __u32 tcpi_sacked;
231+ __u32 tcpi_lost;
232+ __u32 tcpi_retrans;
233+ __u32 tcpi_fackets;
234+
235+ /* Times. */
236+ __u32 tcpi_last_data_sent;
237+ __u32 tcpi_last_ack_sent; /* Not remembered, sorry. */
238+ __u32 tcpi_last_data_recv;
239+ __u32 tcpi_last_ack_recv;
240+
241+ /* Metrics. */
242+ __u32 tcpi_pmtu;
243+ __u32 tcpi_rcv_ssthresh;
244+ __u32 tcpi_rtt;
245+ __u32 tcpi_rttvar;
246+ __u32 tcpi_snd_ssthresh;
247+ __u32 tcpi_snd_cwnd;
248+ __u32 tcpi_advmss;
249+ __u32 tcpi_reordering;
250+
251+ __u32 tcpi_rcv_rtt;
252+ __u32 tcpi_rcv_space;
253+
254+ __u32 tcpi_total_retrans;
255+
256+ __u64 tcpi_pacing_rate;
257+ __u64 tcpi_max_pacing_rate;
258+ __u64 tcpi_bytes_acked; /* RFC4898 tcpEStatsAppHCThruOctetsAcked */
259+ __u64 tcpi_bytes_received; /* RFC4898 tcpEStatsAppHCThruOctetsReceived */
260+ __u32 tcpi_segs_out; /* RFC4898 tcpEStatsPerfSegsOut */
261+ __u32 tcpi_segs_in; /* RFC4898 tcpEStatsPerfSegsIn */
262+
263+ __u32 tcpi_notsent_bytes;
264+ __u32 tcpi_min_rtt;
265+ __u32 tcpi_data_segs_in; /* RFC4898 tcpEStatsDataSegsIn */
266+ __u32 tcpi_data_segs_out; /* RFC4898 tcpEStatsDataSegsOut */
267+
268+ __u64 tcpi_delivery_rate;
269+
270+ __u64 tcpi_busy_time; /* Time (usec) busy sending data */
271+ __u64 tcpi_rwnd_limited; /* Time (usec) limited by receive window */
272+ __u64 tcpi_sndbuf_limited; /* Time (usec) limited by send buffer */
273+
274+ __u32 tcpi_delivered;
275+ __u32 tcpi_delivered_ce;
276+
277+ __u64 tcpi_bytes_sent; /* RFC4898 tcpEStatsPerfHCDataOctetsOut */
278+ __u64 tcpi_bytes_retrans; /* RFC4898 tcpEStatsPerfOctetsRetrans */
279+ __u32 tcpi_dsack_dups; /* RFC4898 tcpEStatsStackDSACKDups */
280+ __u32 tcpi_reord_seen; /* reordering events seen */
281+
282+ __u32 tcpi_rcv_ooopack; /* Out-of-order packets received */
283+
284+ __u32 tcpi_snd_wnd; /* peer's advertised receive window after
285+ * scaling (bytes)
286+ */
287+};
288+
289+/* netlink attributes types for SCM_TIMESTAMPING_OPT_STATS */
290+enum {
291+ TCP_NLA_PAD,
292+ TCP_NLA_BUSY, /* Time (usec) busy sending data */
293+ TCP_NLA_RWND_LIMITED, /* Time (usec) limited by receive window */
294+ TCP_NLA_SNDBUF_LIMITED, /* Time (usec) limited by send buffer */
295+ TCP_NLA_DATA_SEGS_OUT, /* Data pkts sent including retransmission */
296+ TCP_NLA_TOTAL_RETRANS, /* Data pkts retransmitted */
297+ TCP_NLA_PACING_RATE, /* Pacing rate in bytes per second */
298+ TCP_NLA_DELIVERY_RATE, /* Delivery rate in bytes per second */
299+ TCP_NLA_SND_CWND, /* Sending congestion window */
300+ TCP_NLA_REORDERING, /* Reordering metric */
301+ TCP_NLA_MIN_RTT, /* minimum RTT */
302+ TCP_NLA_RECUR_RETRANS, /* Recurring retransmits for the current pkt */
303+ TCP_NLA_DELIVERY_RATE_APP_LMT, /* delivery rate application limited ? */
304+ TCP_NLA_SNDQ_SIZE, /* Data (bytes) pending in send queue */
305+ TCP_NLA_CA_STATE, /* ca_state of socket */
306+ TCP_NLA_SND_SSTHRESH, /* Slow start size threshold */
307+ TCP_NLA_DELIVERED, /* Data pkts delivered incl. out-of-order */
308+ TCP_NLA_DELIVERED_CE, /* Like above but only ones w/ CE marks */
309+ TCP_NLA_BYTES_SENT, /* Data bytes sent including retransmission */
310+ TCP_NLA_BYTES_RETRANS, /* Data bytes retransmitted */
311+ TCP_NLA_DSACK_DUPS, /* DSACK blocks received */
312+ TCP_NLA_REORD_SEEN, /* reordering events seen */
313+ TCP_NLA_SRTT, /* smoothed RTT in usecs */
314+ TCP_NLA_TIMEOUT_REHASH, /* Timeout-triggered rehash attempts */
315+ TCP_NLA_BYTES_NOTSENT, /* Bytes in write queue not yet sent */
316+ TCP_NLA_EDT, /* Earliest departure time (CLOCK_MONOTONIC) */
317+};
318+
319+/* for TCP_MD5SIG socket option */
320+#define TCP_MD5SIG_MAXKEYLEN 80
321+
322+/* tcp_md5sig extension flags for TCP_MD5SIG_EXT */
323+#define TCP_MD5SIG_FLAG_PREFIX 0x1 /* address prefix length */
324+#define TCP_MD5SIG_FLAG_IFINDEX 0x2 /* ifindex set */
325+
326+struct tcp_md5sig {
327+ struct __kernel_sockaddr_storage tcpm_addr; /* address associated */
328+ __u8 tcpm_flags; /* extension flags */
329+ __u8 tcpm_prefixlen; /* address prefix */
330+ __u16 tcpm_keylen; /* key length */
331+ int tcpm_ifindex; /* device index for scope */
332+ __u8 tcpm_key[TCP_MD5SIG_MAXKEYLEN]; /* key (binary) */
333+};
334+
335+/* INET_DIAG_MD5SIG */
336+struct tcp_diag_md5sig {
337+ __u8 tcpm_family;
338+ __u8 tcpm_prefixlen;
339+ __u16 tcpm_keylen;
340+ __be32 tcpm_addr[4];
341+ __u8 tcpm_key[TCP_MD5SIG_MAXKEYLEN];
342+};
343+
344+/* setsockopt(fd, IPPROTO_TCP, TCP_ZEROCOPY_RECEIVE, ...) */
345+
346+#define TCP_RECEIVE_ZEROCOPY_FLAG_TLB_CLEAN_HINT 0x1
347+struct tcp_zerocopy_receive {
348+ __u64 address; /* in: address of mapping */
349+ __u32 length; /* in/out: number of bytes to map/mapped */
350+ __u32 recv_skip_hint; /* out: amount of bytes to skip */
351+ __u32 inq; /* out: amount of bytes in read queue */
352+ __s32 err; /* out: socket error */
353+ __u64 copybuf_address; /* in: copybuf address (small reads) */
354+ __s32 copybuf_len; /* in/out: copybuf bytes avail/used or error */
355+ __u32 flags; /* in: flags */
356+};
357+#endif /* _UAPI_LINUX_TCP_H */
tools/lib/bpf/.gitignore+0−1
@@ -1,7 +1,6 @@
11 # SPDX-License-Identifier: GPL-2.0-only
22 libbpf_version.h
33 libbpf.pc
4−FEATURE-DUMP.libbpf
54 libbpf.so.*
65 TAGS
76 tags
tools/lib/bpf/Makefile+4−43
@@ -58,28 +58,7 @@ ifndef VERBOSE · 57 unmodified lines
5858 VERBOSE = 0
5959 endif
6060
61−FEATURE_USER = .libbpf
62−FEATURE_TESTS = libelf zlib bpf
63−FEATURE_DISPLAY = libelf zlib bpf
64−
6561 INCLUDES = -I. -I$(srctree)/tools/include -I$(srctree)/tools/include/uapi
66−FEATURE_CHECK_CFLAGS-bpf = $(INCLUDES)
67−
68−check_feat := 1
69−NON_CHECK_FEAT_TARGETS := clean TAGS tags cscope help
70−ifdef MAKECMDGOALS
71−ifeq ($(filter-out $(NON_CHECK_FEAT_TARGETS),$(MAKECMDGOALS)),)
72− check_feat := 0
73−endif
74−endif
75−
76−ifeq ($(check_feat),1)
77−ifeq ($(FEATURES_DUMP),)
78−include $(srctree)/tools/build/Makefile.feature
79−else
80−include $(FEATURES_DUMP)
81−endif
82−endif
8362
8463 export prefix libdir src obj
8564
@@ -157,7 +136,7 @@ all: fixdep · 71 unmodified lines
157136
158137 all_cmd: $(CMD_TARGETS) check
159138
160−$(BPF_IN_SHARED): force elfdep zdep bpfdep $(BPF_HELPER_DEFS)
139+$(BPF_IN_SHARED): force $(BPF_HELPER_DEFS)
161140 @(test -f ../../include/uapi/linux/bpf.h -a -f ../../../include/uapi/linux/bpf.h && ( \
162141 (diff -B ../../include/uapi/linux/bpf.h ../../../include/uapi/linux/bpf.h >/dev/null) || \
163142 echo "Warning: Kernel ABI header at 'tools/include/uapi/linux/bpf.h' differs from latest version at 'include/uapi/linux/bpf.h'" >&2 )) || true
@@ -175,7 +154,7 @@ $(BPF_IN_SHARED): force elfdep zdep bpfdep $(BPF_HELPER_DEFS) · 11 unmodified lines
175154 echo "Warning: Kernel ABI header at 'tools/include/uapi/linux/if_xdp.h' differs from latest version at 'include/uapi/linux/if_xdp.h'" >&2 )) || true
176155 $(Q)$(MAKE) $(build)=libbpf OUTPUT=$(SHARED_OBJDIR) CFLAGS="$(CFLAGS) $(SHLIB_FLAGS)"
177156
178−$(BPF_IN_STATIC): force elfdep zdep bpfdep $(BPF_HELPER_DEFS)
157+$(BPF_IN_STATIC): force $(BPF_HELPER_DEFS)
179158 $(Q)$(MAKE) $(build)=libbpf OUTPUT=$(STATIC_OBJDIR)
180159
181160 $(BPF_HELPER_DEFS): $(srctree)/tools/include/uapi/linux/bpf.h
@@ -264,34 +243,16 @@ install_pkgconfig: $(PC_FILE) · 82 unmodified lines
264243
265244 install: install_lib install_pkgconfig install_headers
266245
267−### Cleaning rules
268−
269−config-clean:
270− $(call QUIET_CLEAN, feature-detect)
271− $(Q)$(MAKE) -C $(srctree)/tools/build/feature/ clean >/dev/null
272−
273−clean: config-clean
246+clean:
274247 $(call QUIET_CLEAN, libbpf) $(RM) -rf $(CMD_TARGETS) \
275248 *~ .*.d .*.cmd LIBBPF-CFLAGS $(BPF_HELPER_DEFS) \
276249 $(SHARED_OBJDIR) $(STATIC_OBJDIR) \
277250 $(addprefix $(OUTPUT), \
278251 *.o *.a *.so *.so.$(LIBBPF_MAJOR_VERSION) *.pc)
279− $(call QUIET_CLEAN, core-gen) $(RM) $(OUTPUT)FEATURE-DUMP.libbpf
280−
281−
282252
283−PHONY += force elfdep zdep bpfdep cscope tags
253+PHONY += force cscope tags
284254 force:
285255
286−elfdep:
287− @if [ "$(feature-libelf)" != "1" ]; then echo "No libelf found"; exit 1 ; fi
288−
289−zdep:
290− @if [ "$(feature-zlib)" != "1" ]; then echo "No zlib found"; exit 1 ; fi
291−
292−bpfdep:
293− @if [ "$(feature-bpf)" != "1" ]; then echo "BPF API too old"; exit 1 ; fi
294−
295256 cscope:
296257 ls *.c *.h > cscope.files
297258 cscope -b -q -I $(srctree)/include -f cscope.out
tools/lib/bpf/btf.c+10−2
@@ -858,6 +858,7 @@ static struct btf *btf_parse_elf(const char *path, struct btf *base_btf, · 857 unmodified lines
858858 Elf_Scn *scn = NULL;
859859 Elf *elf = NULL;
860860 GElf_Ehdr ehdr;
861+ size_t shstrndx;
861862
862863 if (elf_version(EV_CURRENT) == EV_NONE) {
863864 pr_warn("failed to init libelf for %s\n", path);
@@ -882,7 +883,14 @@ static struct btf *btf_parse_elf(const char *path, struct btf *base_btf, · 18 unmodified lines
882883 pr_warn("failed to get EHDR from %s\n", path);
883884 goto done;
884885 }
885− if (!elf_rawdata(elf_getscn(elf, ehdr.e_shstrndx), NULL)) {
886+
887+ if (elf_getshdrstrndx(elf, &shstrndx)) {
888+ pr_warn("failed to get section names section index for %s\n",
889+ path);
890+ goto done;
891+ }
892+
893+ if (!elf_rawdata(elf_getscn(elf, shstrndx), NULL)) {
886894 pr_warn("failed to get e_shstrndx from %s\n", path);
887895 goto done;
888896 }
@@ -897,7 +905,7 @@ static struct btf *btf_parse_elf(const char *path, struct btf *base_btf, · 8 unmodified lines
897905 idx, path);
898906 goto done;
899907 }
900− name = elf_strptr(elf, ehdr.e_shstrndx, sh.sh_name);
908+ name = elf_strptr(elf, shstrndx, sh.sh_name);
901909 if (!name) {
902910 pr_warn("failed to get section(%d) name from %s\n",
903911 idx, path);
tools/lib/bpf/libbpf.c+11−11
@@ -884,24 +884,24 @@ static int bpf_map__init_kern_struct_ops(struct bpf_map *map, · 883 unmodified lines
884884 if (btf_is_ptr(mtype)) {
885885 struct bpf_program *prog;
886886
887− mtype = skip_mods_and_typedefs(btf, mtype->type, &mtype_id);
887+ prog = st_ops->progs[i];
888+ if (!prog)
889+ continue;
890+
888891 kern_mtype = skip_mods_and_typedefs(kern_btf,
889892 kern_mtype->type,
890893 &kern_mtype_id);
891− if (!btf_is_func_proto(mtype) ||
892− !btf_is_func_proto(kern_mtype)) {
893− pr_warn("struct_ops init_kern %s: non func ptr %s is not supported\n",
894+
895+ /* mtype->type must be a func_proto which was
896+ * guaranteed in bpf_object__collect_st_ops_relos(),
897+ * so only check kern_mtype for func_proto here.
898+ */
899+ if (!btf_is_func_proto(kern_mtype)) {
900+ pr_warn("struct_ops init_kern %s: kernel member %s is not a func ptr\n",
894901 map->name, mname);
895902 return -ENOTSUP;
896903 }
897904
898− prog = st_ops->progs[i];
899− if (!prog) {
900− pr_debug("struct_ops init_kern %s: func ptr %s is not set\n",
901− map->name, mname);
902− continue;
903− }
904−
905905 prog->attach_btf_id = kern_type_id;
906906 prog->expected_attach_type = kern_member_idx;
907907
tools/lib/bpf/xsk.c+79−4
@@ -46,6 +46,11 @@ · 45 unmodified lines
4646 #define PF_XDP AF_XDP
4747 #endif
4848
49+enum xsk_prog {
50+ XSK_PROG_FALLBACK,
51+ XSK_PROG_REDIRECT_FLAGS,
52+};
53+
4954 struct xsk_umem {
5055 struct xsk_ring_prod *fill_save;
5156 struct xsk_ring_cons *comp_save;
@@ -351,6 +356,54 @@ int xsk_umem__create_v0_0_2(struct xsk_umem **umem_ptr, void *umem_area, · 299 unmodified lines
351356 COMPAT_VERSION(xsk_umem__create_v0_0_2, xsk_umem__create, LIBBPF_0.0.2)
352357 DEFAULT_VERSION(xsk_umem__create_v0_0_4, xsk_umem__create, LIBBPF_0.0.4)
353358
359+static enum xsk_prog get_xsk_prog(void)
360+{
361+ enum xsk_prog detected = XSK_PROG_FALLBACK;
362+ struct bpf_load_program_attr prog_attr;
363+ struct bpf_create_map_attr map_attr;
364+ __u32 size_out, retval, duration;
365+ char data_in = 0, data_out;
366+ struct bpf_insn insns[] = {
367+ BPF_LD_MAP_FD(BPF_REG_1, 0),
368+ BPF_MOV64_IMM(BPF_REG_2, 0),
369+ BPF_MOV64_IMM(BPF_REG_3, XDP_PASS),
370+ BPF_EMIT_CALL(BPF_FUNC_redirect_map),
371+ BPF_EXIT_INSN(),
372+ };
373+ int prog_fd, map_fd, ret;
374+
375+ memset(&map_attr, 0, sizeof(map_attr));
376+ map_attr.map_type = BPF_MAP_TYPE_XSKMAP;
377+ map_attr.key_size = sizeof(int);
378+ map_attr.value_size = sizeof(int);
379+ map_attr.max_entries = 1;
380+
381+ map_fd = bpf_create_map_xattr(&map_attr);
382+ if (map_fd < 0)
383+ return detected;
384+
385+ insns[0].imm = map_fd;
386+
387+ memset(&prog_attr, 0, sizeof(prog_attr));
388+ prog_attr.prog_type = BPF_PROG_TYPE_XDP;
389+ prog_attr.insns = insns;
390+ prog_attr.insns_cnt = ARRAY_SIZE(insns);
391+ prog_attr.license = "GPL";
392+
393+ prog_fd = bpf_load_program_xattr(&prog_attr, NULL, 0);
394+ if (prog_fd < 0) {
395+ close(map_fd);
396+ return detected;
397+ }
398+
399+ ret = bpf_prog_test_run(prog_fd, 0, &data_in, 1, &data_out, &size_out, &retval, &duration);
400+ if (!ret && retval == XDP_PASS)
401+ detected = XSK_PROG_REDIRECT_FLAGS;
402+ close(prog_fd);
403+ close(map_fd);
404+ return detected;
405+}
406+
354407 static int xsk_load_xdp_prog(struct xsk_socket *xsk)
355408 {
356409 static const int log_buf_size = 16 * 1024;
@@ -358,7 +411,7 @@ static int xsk_load_xdp_prog(struct xsk_socket *xsk) · 1 unmodified line
358411 char log_buf[log_buf_size];
359412 int err, prog_fd;
360413
361− /* This is the C-program:
414+ /* This is the fallback C-program:
362415 * SEC("xdp_sock") int xdp_sock_prog(struct xdp_md *ctx)
363416 * {
364417 * int ret, index = ctx->rx_queue_index;
@@ -414,9 +467,31 @@ static int xsk_load_xdp_prog(struct xsk_socket *xsk) · 49 unmodified lines
414467 /* The jumps are to this instruction */
415468 BPF_EXIT_INSN(),
416469 };
417− size_t insns_cnt = sizeof(prog) / sizeof(struct bpf_insn);
418470
419− prog_fd = bpf_load_program(BPF_PROG_TYPE_XDP, prog, insns_cnt,
471+ /* This is the post-5.3 kernel C-program:
472+ * SEC("xdp_sock") int xdp_sock_prog(struct xdp_md *ctx)
473+ * {
474+ * return bpf_redirect_map(&xsks_map, ctx->rx_queue_index, XDP_PASS);
475+ * }
476+ */
477+ struct bpf_insn prog_redirect_flags[] = {
478+ /* r2 = *(u32 *)(r1 + 16) */
479+ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 16),
480+ /* r1 = xskmap[] */
481+ BPF_LD_MAP_FD(BPF_REG_1, ctx->xsks_map_fd),
482+ /* r3 = XDP_PASS */
483+ BPF_MOV64_IMM(BPF_REG_3, 2),
484+ /* call bpf_redirect_map */
485+ BPF_EMIT_CALL(BPF_FUNC_redirect_map),
486+ BPF_EXIT_INSN(),
487+ };
488+ size_t insns_cnt[] = {sizeof(prog) / sizeof(struct bpf_insn),
489+ sizeof(prog_redirect_flags) / sizeof(struct bpf_insn),
490+ };
491+ struct bpf_insn *progs[] = {prog, prog_redirect_flags};
492+ enum xsk_prog option = get_xsk_prog();
493+
494+ prog_fd = bpf_load_program(BPF_PROG_TYPE_XDP, progs[option], insns_cnt[option],
420495 "LGPL-2.1 or BSD-2-Clause", 0, log_buf,
421496 log_buf_size);
422497 if (prog_fd < 0) {
@@ -442,7 +517,7 @@ static int xsk_get_max_queues(struct xsk_socket *xsk) · 19 unmodified lines
442517 struct ifreq ifr = {};
443518 int fd, err, ret;
444519
445− fd = socket(AF_INET, SOCK_DGRAM, 0);
520+ fd = socket(AF_LOCAL, SOCK_DGRAM, 0);
446521 if (fd < 0)
447522 return -errno;
448523
tools/perf/Makefile.perf+0−1
@@ -176,7 +176,6 @@ endef · 175 unmodified lines
176176 LD += $(EXTRA_LDFLAGS)
177177
178178 PKG_CONFIG = $(CROSS_COMPILE)pkg-config
179−LLVM_CONFIG ?= llvm-config
180179
181180 RM = rm -f
182181 LN = ln -f
tools/scripts/Makefile.include+7−0
@@ -69,6 +69,13 @@ HOSTCC ?= gcc · 68 unmodified lines
6969 HOSTLD ?= ld
7070 endif
7171
72+# Some tools require Clang, LLC and/or LLVM utils
73+CLANG ?= clang
74+LLC ?= llc
75+LLVM_CONFIG ?= llvm-config
76+LLVM_OBJCOPY ?= llvm-objcopy
77+LLVM_STRIP ?= llvm-strip
78+
7279 ifeq ($(CC_NO_CLANG), 1)
7380 EXTRA_WARNINGS += -Wstrict-aliasing=3
7481 endif
tools/testing/selftests/bpf/.gitignore+0−2
@@ -17,7 +17,6 @@ test_sockmap · 16 unmodified lines
1717 test_lirc_mode2_user
1818 get_cgroup_id_user
1919 test_skb_cgroup_id_user
20−test_socket_cookie
2120 test_cgroup_storage
2221 test_flow_dissector
2322 flow_dissector_load
@@ -26,7 +25,6 @@ test_tcpnotify_user · 2 unmodified lines
2625 test_libbpf
2726 test_tcp_check_syncookie_user
2827 test_sysctl
29−test_current_pid_tgid_new_ns
3028 xdping
3129 test_cpp
3230 *.skel.h
tools/testing/selftests/bpf/Makefile+2−6
@@ -19,8 +19,6 @@ ifneq ($(wildcard $(GENHDR)),) · 18 unmodified lines
1919 GENFLAGS := -DHAVE_GENHDR
2020 endif
2121
22−CLANG ?= clang
23−LLVM_OBJCOPY ?= llvm-objcopy
2422 BPF_GCC ?= $(shell command -v bpf-gcc;)
2523 SAN_CFLAGS ?=
2624 CFLAGS += -g -rdynamic -Wall -O2 $(GENFLAGS) $(SAN_CFLAGS) \
@@ -33,11 +31,10 @@ LDLIBS += -lcap -lelf -lz -lrt -lpthread · 6 unmodified lines
3331 # Order correspond to 'make run_tests' order
3432 TEST_GEN_PROGS = test_verifier test_tag test_maps test_lru_map test_lpm_map test_progs \
3533 test_verifier_log test_dev_cgroup \
36− test_sock test_sockmap get_cgroup_id_user test_socket_cookie \
34+ test_sock test_sockmap get_cgroup_id_user \
3735 test_cgroup_storage \
3836 test_netcnt test_tcpnotify_user test_sysctl \
39− test_progs-no_alu32 \
40− test_current_pid_tgid_new_ns
37+ test_progs-no_alu32
4138
4239 # Also test bpf-gcc, if present
4340 ifneq ($(BPF_GCC),)
@@ -188,7 +185,6 @@ $(OUTPUT)/test_dev_cgroup: cgroup_helpers.c · 144 unmodified lines
188185 $(OUTPUT)/test_skb_cgroup_id_user: cgroup_helpers.c
189186 $(OUTPUT)/test_sock: cgroup_helpers.c
190187 $(OUTPUT)/test_sock_addr: cgroup_helpers.c
191−$(OUTPUT)/test_socket_cookie: cgroup_helpers.c
192188 $(OUTPUT)/test_sockmap: cgroup_helpers.c
193189 $(OUTPUT)/test_tcpnotify_user: cgroup_helpers.c trace_helpers.c
194190 $(OUTPUT)/get_cgroup_id_user: cgroup_helpers.c
tools/testing/selftests/bpf/README.rst+24−0
@@ -6,6 +6,30 @@ General instructions on running selftests can be found in · 5 unmodified lines
66
77 __ /Documentation/bpf/bpf_devel_QA.rst#q-how-to-run-bpf-selftests
88
9+=========================
10+Running Selftests in a VM
11+=========================
12+
13+It's now possible to run the selftests using ``tools/testing/selftests/bpf/vmtest.sh``.
14+The script tries to ensure that the tests are run with the same environment as they
15+would be run post-submit in the CI used by the Maintainers.
16+
17+This script downloads a suitable Kconfig and VM userspace image from the system used by
18+the CI. It builds the kernel (without overwriting your existing Kconfig), recompiles the
19+bpf selftests, runs them (by default ``tools/testing/selftests/bpf/test_progs``) and
20+saves the resulting output (by default in ``~/.bpf_selftests``).
21+
22+For more information on about using the script, run:
23+
24+.. code-block:: console
25+
26+ $ tools/testing/selftests/bpf/vmtest.sh -h
27+
28+.. note:: The script uses pahole and clang based on host environment setting.
29+ If you want to change pahole and llvm, you can change `PATH` environment
30+ variable in the beginning of script.
31+
32+.. note:: The script currently only supports x86_64.
933
1034 Additional information about selftest failures are
1135 documented here.
tools/testing/selftests/bpf/benchs/bench_ringbufs.c+1−1
@@ -319,7 +319,7 @@ static void ringbuf_custom_process_ring(struct ringbuf_custom *r) · 318 unmodified lines
319319 smp_store_release(r->consumer_pos, cons_pos);
320320 else
321321 break;
322− };
322+ }
323323 }
324324
325325 static void *ringbuf_custom_consumer(void *input)
tools/testing/selftests/bpf/bpf_sockopt_helpers.h+21−0
@@ -0,0 +1,21 @@
1+/* SPDX-License-Identifier: GPL-2.0 */
2+
3+#include <sys/socket.h>
4+#include <bpf/bpf_helpers.h>
5+
6+int get_set_sk_priority(void *ctx)
7+{
8+ int prio;
9+
10+ /* Verify that context allows calling bpf_getsockopt and
11+ * bpf_setsockopt by reading and writing back socket
12+ * priority.
13+ */
14+
15+ if (bpf_getsockopt(ctx, SOL_SOCKET, SO_PRIORITY, &prio, sizeof(prio)))
16+ return 0;
17+ if (bpf_setsockopt(ctx, SOL_SOCKET, SO_PRIORITY, &prio, sizeof(prio)))
18+ return 0;
19+
20+ return 1;
21+}
tools/testing/selftests/bpf/bpf_tcp_helpers.h+1−0
@@ -177,6 +177,7 @@ struct tcp_congestion_ops { · 176 unmodified lines
177177 * after all the ca_state processing. (optional)
178178 */
179179 void (*cong_control)(struct sock *sk, const struct rate_sample *rs);
180+ void *owner;
180181 };
181182
182183 #define min(a, b) ((a) < (b) ? (a) : (b))
tools/testing/selftests/bpf/bpf_testmod/bpf_testmod-events.h+6−0
@@ -28,6 +28,12 @@ TRACE_EVENT(bpf_testmod_test_read, · 27 unmodified lines
2828 __entry->pid, __entry->comm, __entry->off, __entry->len)
2929 );
3030
31+/* A bare tracepoint with no event associated with it */
32+DECLARE_TRACE(bpf_testmod_test_write_bare,
33+ TP_PROTO(struct task_struct *task, struct bpf_testmod_test_write_ctx *ctx),
34+ TP_ARGS(task, ctx)
35+);
36+
3137 #endif /* _BPF_TESTMOD_EVENTS_H */
3238
3339 #undef TRACE_INCLUDE_PATH
tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c+20−1
@@ -31,9 +31,28 @@ bpf_testmod_test_read(struct file *file, struct kobject *kobj, · 30 unmodified lines
3131 EXPORT_SYMBOL(bpf_testmod_test_read);
3232 ALLOW_ERROR_INJECTION(bpf_testmod_test_read, ERRNO);
3333
34+noinline ssize_t
35+bpf_testmod_test_write(struct file *file, struct kobject *kobj,
36+ struct bin_attribute *bin_attr,
37+ char *buf, loff_t off, size_t len)
38+{
39+ struct bpf_testmod_test_write_ctx ctx = {
40+ .buf = buf,
41+ .off = off,
42+ .len = len,
43+ };
44+
45+ trace_bpf_testmod_test_write_bare(current, &ctx);
46+
47+ return -EIO; /* always fail */
48+}
49+EXPORT_SYMBOL(bpf_testmod_test_write);
50+ALLOW_ERROR_INJECTION(bpf_testmod_test_write, ERRNO);
51+
3452 static struct bin_attribute bin_attr_bpf_testmod_file __ro_after_init = {
35− .attr = { .name = "bpf_testmod", .mode = 0444, },
53+ .attr = { .name = "bpf_testmod", .mode = 0666, },
3654 .read = bpf_testmod_test_read,
55+ .write = bpf_testmod_test_write,
3756 };
3857
3958 static int bpf_testmod_init(void)
tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.h+6−0
@@ -11,4 +11,10 @@ struct bpf_testmod_test_read_ctx { · 10 unmodified lines
1111 size_t len;
1212 };
1313
14+struct bpf_testmod_test_write_ctx {
15+ char *buf;
16+ loff_t off;
17+ size_t len;
18+};
19+
1420 #endif /* _BPF_TESTMOD_H */
tools/testing/selftests/bpf/prog_tests/atomic_bounds.c+17−0
@@ -0,0 +1,17 @@
1+// SPDX-License-Identifier: GPL-2.0
2+
3+#include <test_progs.h>
4+
5+#include "atomic_bounds.skel.h"
6+
7+void test_atomic_bounds(void)
8+{
9+ struct atomic_bounds *skel;
10+ __u32 duration = 0;
11+
12+ skel = atomic_bounds__open_and_load();
13+ if (CHECK(!skel, "skel_load", "couldn't load program\n"))
14+ return;
15+
16+ atomic_bounds__destroy(skel);
17+}
tools/testing/selftests/bpf/prog_tests/bind_perm.c+109−0
@@ -0,0 +1,109 @@
1+// SPDX-License-Identifier: GPL-2.0
2+#include <test_progs.h>
3+#include "bind_perm.skel.h"
4+
5+#include <sys/types.h>
6+#include <sys/socket.h>
7+#include <sys/capability.h>
8+
9+static int duration;
10+
11+void try_bind(int family, int port, int expected_errno)
12+{
13+ struct sockaddr_storage addr = {};
14+ struct sockaddr_in6 *sin6;
15+ struct sockaddr_in *sin;
16+ int fd = -1;
17+
18+ fd = socket(family, SOCK_STREAM, 0);
19+ if (CHECK(fd < 0, "fd", "errno %d", errno))
20+ goto close_socket;
21+
22+ if (family == AF_INET) {
23+ sin = (struct sockaddr_in *)&addr;
24+ sin->sin_family = family;
25+ sin->sin_port = htons(port);
26+ } else {
27+ sin6 = (struct sockaddr_in6 *)&addr;
28+ sin6->sin6_family = family;
29+ sin6->sin6_port = htons(port);
30+ }
31+
32+ errno = 0;
33+ bind(fd, (struct sockaddr *)&addr, sizeof(addr));
34+ ASSERT_EQ(errno, expected_errno, "bind");
35+
36+close_socket:
37+ if (fd >= 0)
38+ close(fd);
39+}
40+
41+bool cap_net_bind_service(cap_flag_value_t flag)
42+{
43+ const cap_value_t cap_net_bind_service = CAP_NET_BIND_SERVICE;
44+ cap_flag_value_t original_value;
45+ bool was_effective = false;
46+ cap_t caps;
47+
48+ caps = cap_get_proc();
49+ if (CHECK(!caps, "cap_get_proc", "errno %d", errno))
50+ goto free_caps;
51+
52+ if (CHECK(cap_get_flag(caps, CAP_NET_BIND_SERVICE, CAP_EFFECTIVE,
53+ &original_value),
54+ "cap_get_flag", "errno %d", errno))
55+ goto free_caps;
56+
57+ was_effective = (original_value == CAP_SET);
58+
59+ if (CHECK(cap_set_flag(caps, CAP_EFFECTIVE, 1, &cap_net_bind_service,
60+ flag),
61+ "cap_set_flag", "errno %d", errno))
62+ goto free_caps;
63+
64+ if (CHECK(cap_set_proc(caps), "cap_set_proc", "errno %d", errno))
65+ goto free_caps;
66+
67+free_caps:
68+ CHECK(cap_free(caps), "cap_free", "errno %d", errno);
69+ return was_effective;
70+}
71+
72+void test_bind_perm(void)
73+{
74+ bool cap_was_effective;
75+ struct bind_perm *skel;
76+ int cgroup_fd;
77+
78+ cgroup_fd = test__join_cgroup("/bind_perm");
79+ if (CHECK(cgroup_fd < 0, "cg-join", "errno %d", errno))
80+ return;
81+
82+ skel = bind_perm__open_and_load();
83+ if (!ASSERT_OK_PTR(skel, "skel"))
84+ goto close_cgroup_fd;
85+
86+ skel->links.bind_v4_prog = bpf_program__attach_cgroup(skel->progs.bind_v4_prog, cgroup_fd);
87+ if (!ASSERT_OK_PTR(skel, "bind_v4_prog"))
88+ goto close_skeleton;
89+
90+ skel->links.bind_v6_prog = bpf_program__attach_cgroup(skel->progs.bind_v6_prog, cgroup_fd);
91+ if (!ASSERT_OK_PTR(skel, "bind_v6_prog"))
92+ goto close_skeleton;
93+
94+ cap_was_effective = cap_net_bind_service(CAP_CLEAR);
95+
96+ try_bind(AF_INET, 110, EACCES);
97+ try_bind(AF_INET6, 110, EACCES);
98+
99+ try_bind(AF_INET, 111, 0);
100+ try_bind(AF_INET6, 111, 0);
101+
102+ if (cap_was_effective)
103+ cap_net_bind_service(CAP_SET);
104+
105+close_skeleton:
106+ bind_perm__destroy(skel);
107+close_cgroup_fd:
108+ close(cgroup_fd);
109+}
tools/testing/selftests/bpf/prog_tests/bpf_iter.c+108−10
@@ -7,6 +7,7 @@ · 6 unmodified lines
77 #include "bpf_iter_task.skel.h"
88 #include "bpf_iter_task_stack.skel.h"
99 #include "bpf_iter_task_file.skel.h"
10+#include "bpf_iter_task_vma.skel.h"
1011 #include "bpf_iter_task_btf.skel.h"
1112 #include "bpf_iter_tcp4.skel.h"
1213 #include "bpf_iter_tcp6.skel.h"
@@ -64,6 +65,22 @@ free_link: · 51 unmodified lines
6465 bpf_link__destroy(link);
6566 }
6667
68+static int read_fd_into_buffer(int fd, char *buf, int size)
69+{
70+ int bufleft = size;
71+ int len;
72+
73+ do {
74+ len = read(fd, buf, bufleft);
75+ if (len > 0) {
76+ buf += len;
77+ bufleft -= len;
78+ }
79+ } while (len > 0);
80+
81+ return len < 0 ? len : size - bufleft;
82+}
83+
6784 static void test_ipv6_route(void)
6885 {
6986 struct bpf_iter_ipv6_route *skel;
@@ -177,7 +194,7 @@ static int do_btf_read(struct bpf_iter_task_btf *skel) · 107 unmodified lines
177194 {
178195 struct bpf_program *prog = skel->progs.dump_task_struct;
179196 struct bpf_iter_task_btf__bss *bss = skel->bss;
180− int iter_fd = -1, len = 0, bufleft = TASKBUFSZ;
197+ int iter_fd = -1, err;
181198 struct bpf_link *link;
182199 char *buf = taskbuf;
183200 int ret = 0;
@@ -190,14 +207,7 @@ static int do_btf_read(struct bpf_iter_task_btf *skel) · 6 unmodified lines
190207 if (CHECK(iter_fd < 0, "create_iter", "create_iter failed\n"))
191208 goto free_link;
192209
193− do {
194− len = read(iter_fd, buf, bufleft);
195− if (len > 0) {
196− buf += len;
197− bufleft -= len;
198− }
199− } while (len > 0);
200−
210+ err = read_fd_into_buffer(iter_fd, buf, TASKBUFSZ);
201211 if (bss->skip) {
202212 printf("%s:SKIP:no __builtin_btf_type_id\n", __func__);
203213 ret = 1;
@@ -205,7 +215,7 @@ static int do_btf_read(struct bpf_iter_task_btf *skel) · 1 unmodified line
205215 goto free_link;
206216 }
207217
208− if (CHECK(len < 0, "read", "read failed: %s\n", strerror(errno)))
218+ if (CHECK(err < 0, "read", "read failed: %s\n", strerror(errno)))
209219 goto free_link;
210220
211221 CHECK(strstr(taskbuf, "(struct task_struct)") == NULL,
@@ -1133,6 +1143,92 @@ static void test_buf_neg_offset(void) · 921 unmodified lines
11331143 bpf_iter_test_kern6__destroy(skel);
11341144 }
11351145
1146+#define CMP_BUFFER_SIZE 1024
1147+static char task_vma_output[CMP_BUFFER_SIZE];
1148+static char proc_maps_output[CMP_BUFFER_SIZE];
1149+
1150+/* remove \0 and \t from str, and only keep the first line */
1151+static void str_strip_first_line(char *str)
1152+{
1153+ char *dst = str, *src = str;
1154+
1155+ do {
1156+ if (*src == ' ' || *src == '\t')
1157+ src++;
1158+ else
1159+ *(dst++) = *(src++);
1160+
1161+ } while (*src != '\0' && *src != '\n');
1162+
1163+ *dst = '\0';
1164+}
1165+
1166+#define min(a, b) ((a) < (b) ? (a) : (b))
1167+
1168+static void test_task_vma(void)
1169+{
1170+ int err, iter_fd = -1, proc_maps_fd = -1;
1171+ struct bpf_iter_task_vma *skel;
1172+ int len, read_size = 4;
1173+ char maps_path[64];
1174+
1175+ skel = bpf_iter_task_vma__open();
1176+ if (CHECK(!skel, "bpf_iter_task_vma__open", "skeleton open failed\n"))
1177+ return;
1178+
1179+ skel->bss->pid = getpid();
1180+
1181+ err = bpf_iter_task_vma__load(skel);
1182+ if (CHECK(err, "bpf_iter_task_vma__load", "skeleton load failed\n"))
1183+ goto out;
1184+
1185+ skel->links.proc_maps = bpf_program__attach_iter(
1186+ skel->progs.proc_maps, NULL);
1187+
1188+ if (CHECK(IS_ERR(skel->links.proc_maps), "bpf_program__attach_iter",
1189+ "attach iterator failed\n")) {
1190+ skel->links.proc_maps = NULL;
1191+ goto out;
1192+ }
1193+
1194+ iter_fd = bpf_iter_create(bpf_link__fd(skel->links.proc_maps));
1195+ if (CHECK(iter_fd < 0, "create_iter", "create_iter failed\n"))
1196+ goto out;
1197+
1198+ /* Read CMP_BUFFER_SIZE (1kB) from bpf_iter. Read in small chunks
1199+ * to trigger seq_file corner cases. The expected output is much
1200+ * longer than 1kB, so the while loop will terminate.
1201+ */
1202+ len = 0;
1203+ while (len < CMP_BUFFER_SIZE) {
1204+ err = read_fd_into_buffer(iter_fd, task_vma_output + len,
1205+ min(read_size, CMP_BUFFER_SIZE - len));
1206+ if (CHECK(err < 0, "read_iter_fd", "read_iter_fd failed\n"))
1207+ goto out;
1208+ len += err;
1209+ }
1210+
1211+ /* read CMP_BUFFER_SIZE (1kB) from /proc/pid/maps */
1212+ snprintf(maps_path, 64, "/proc/%u/maps", skel->bss->pid);
1213+ proc_maps_fd = open(maps_path, O_RDONLY);
1214+ if (CHECK(proc_maps_fd < 0, "open_proc_maps", "open_proc_maps failed\n"))
1215+ goto out;
1216+ err = read_fd_into_buffer(proc_maps_fd, proc_maps_output, CMP_BUFFER_SIZE);
1217+ if (CHECK(err < 0, "read_prog_maps_fd", "read_prog_maps_fd failed\n"))
1218+ goto out;
1219+
1220+ /* strip and compare the first line of the two files */
1221+ str_strip_first_line(task_vma_output);
1222+ str_strip_first_line(proc_maps_output);
1223+
1224+ CHECK(strcmp(task_vma_output, proc_maps_output), "compare_output",
1225+ "found mismatch\n");
1226+out:
1227+ close(proc_maps_fd);
1228+ close(iter_fd);
1229+ bpf_iter_task_vma__destroy(skel);
1230+}
1231+
11361232 void test_bpf_iter(void)
11371233 {
11381234 if (test__start_subtest("btf_id_or_null"))
@@ -1149,6 +1245,8 @@ void test_bpf_iter(void) · 10 unmodified lines
11491245 test_task_stack();
11501246 if (test__start_subtest("task_file"))
11511247 test_task_file();
1248+ if (test__start_subtest("task_vma"))
1249+ test_task_vma();
11521250 if (test__start_subtest("task_btf"))
11531251 test_task_btf();
11541252 if (test__start_subtest("tcp4"))
tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c+1−0
@@ -2,6 +2,7 @@ · 1 unmodified line
22 /* Copyright (c) 2019 Facebook */
33
44 #include <linux/err.h>
5+#include <netinet/tcp.h>
56 #include <test_progs.h>
67 #include "bpf_dctcp.skel.h"
78 #include "bpf_cubic.skel.h"
tools/testing/selftests/bpf/prog_tests/btf.c+23−2
@@ -914,7 +914,7 @@ static struct btf_raw_test raw_tests[] = { · 913 unmodified lines
914914 .err_str = "Member exceeds struct_size",
915915 },
916916
917−/* Test member exeeds the size of struct
917+/* Test member exceeds the size of struct
918918 *
919919 * struct A {
920920 * int m;
@@ -948,7 +948,7 @@ static struct btf_raw_test raw_tests[] = { · 27 unmodified lines
948948 .err_str = "Member exceeds struct_size",
949949 },
950950
951−/* Test member exeeds the size of struct
951+/* Test member exceeds the size of struct
952952 *
953953 * struct A {
954954 * int m;
@@ -3509,6 +3509,27 @@ static struct btf_raw_test raw_tests[] = { · 2554 unmodified lines
35093509 .value_type_id = 3 /* arr_t */,
35103510 .max_entries = 4,
35113511 },
3512+/*
3513+ * elf .rodata section size 4 and btf .rodata section vlen 0.
3514+ */
3515+{
3516+ .descr = "datasec: vlen == 0",
3517+ .raw_types = {
3518+ /* int */
3519+ BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
3520+ /* .rodata section */
3521+ BTF_TYPE_ENC(NAME_NTH(1), BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 0), 4),
3522+ /* [2] */
3523+ BTF_END_RAW,
3524+ },
3525+ BTF_STR_SEC("\0.rodata"),
3526+ .map_type = BPF_MAP_TYPE_ARRAY,
3527+ .key_size = sizeof(int),
3528+ .value_size = sizeof(int),
3529+ .key_type_id = 1,
3530+ .value_type_id = 1,
3531+ .max_entries = 1,
3532+},
35123533
35133534 }; /* struct btf_raw_test raw_tests[] */
35143535
tools/testing/selftests/bpf/prog_tests/check_mtu.c+216−0
@@ -0,0 +1,216 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/* Copyright (c) 2020 Jesper Dangaard Brouer */
3+
4+#include <linux/if_link.h> /* before test_progs.h, avoid bpf_util.h redefines */
5+#include <test_progs.h>
6+#include "test_check_mtu.skel.h"
7+#include "network_helpers.h"
8+
9+#include <stdlib.h>
10+#include <inttypes.h>
11+
12+#define IFINDEX_LO 1
13+
14+static __u32 duration; /* Hint: needed for CHECK macro */
15+
16+static int read_mtu_device_lo(void)
17+{
18+ const char *filename = "/sys/class/net/lo/mtu";
19+ char buf[11] = {};
20+ int value, n, fd;
21+
22+ fd = open(filename, 0, O_RDONLY);
23+ if (fd == -1)
24+ return -1;
25+
26+ n = read(fd, buf, sizeof(buf));
27+ close(fd);
28+
29+ if (n == -1)
30+ return -2;
31+
32+ value = strtoimax(buf, NULL, 10);
33+ if (errno == ERANGE)
34+ return -3;
35+
36+ return value;
37+}
38+
39+static void test_check_mtu_xdp_attach(void)
40+{
41+ struct bpf_link_info link_info;
42+ __u32 link_info_len = sizeof(link_info);
43+ struct test_check_mtu *skel;
44+ struct bpf_program *prog;
45+ struct bpf_link *link;
46+ int err = 0;
47+ int fd;
48+
49+ skel = test_check_mtu__open_and_load();
50+ if (CHECK(!skel, "open and load skel", "failed"))
51+ return; /* Exit if e.g. helper unknown to kernel */
52+
53+ prog = skel->progs.xdp_use_helper_basic;
54+
55+ link = bpf_program__attach_xdp(prog, IFINDEX_LO);
56+ if (CHECK(IS_ERR(link), "link_attach", "failed: %ld\n", PTR_ERR(link)))
57+ goto out;
58+ skel->links.xdp_use_helper_basic = link;
59+
60+ memset(&link_info, 0, sizeof(link_info));
61+ fd = bpf_link__fd(link);
62+ err = bpf_obj_get_info_by_fd(fd, &link_info, &link_info_len);
63+ if (CHECK(err, "link_info", "failed: %d\n", err))
64+ goto out;
65+
66+ CHECK(link_info.type != BPF_LINK_TYPE_XDP, "link_type",
67+ "got %u != exp %u\n", link_info.type, BPF_LINK_TYPE_XDP);
68+ CHECK(link_info.xdp.ifindex != IFINDEX_LO, "link_ifindex",
69+ "got %u != exp %u\n", link_info.xdp.ifindex, IFINDEX_LO);
70+
71+ err = bpf_link__detach(link);
72+ CHECK(err, "link_detach", "failed %d\n", err);
73+
74+out:
75+ test_check_mtu__destroy(skel);
76+}
77+
78+static void test_check_mtu_run_xdp(struct test_check_mtu *skel,
79+ struct bpf_program *prog,
80+ __u32 mtu_expect)
81+{
82+ const char *prog_name = bpf_program__name(prog);
83+ int retval_expect = XDP_PASS;
84+ __u32 mtu_result = 0;
85+ char buf[256] = {};
86+ int err;
87+ struct bpf_prog_test_run_attr tattr = {
88+ .repeat = 1,
89+ .data_in = &pkt_v4,
90+ .data_size_in = sizeof(pkt_v4),
91+ .data_out = buf,
92+ .data_size_out = sizeof(buf),
93+ .prog_fd = bpf_program__fd(prog),
94+ };
95+
96+ err = bpf_prog_test_run_xattr(&tattr);
97+ CHECK_ATTR(err != 0, "bpf_prog_test_run",
98+ "prog_name:%s (err %d errno %d retval %d)\n",
99+ prog_name, err, errno, tattr.retval);
100+
101+ CHECK(tattr.retval != retval_expect, "retval",
102+ "progname:%s unexpected retval=%d expected=%d\n",
103+ prog_name, tattr.retval, retval_expect);
104+
105+ /* Extract MTU that BPF-prog got */
106+ mtu_result = skel->bss->global_bpf_mtu_xdp;
107+ ASSERT_EQ(mtu_result, mtu_expect, "MTU-compare-user");
108+}
109+
110+
111+static void test_check_mtu_xdp(__u32 mtu, __u32 ifindex)
112+{
113+ struct test_check_mtu *skel;
114+ int err;
115+
116+ skel = test_check_mtu__open();
117+ if (CHECK(!skel, "skel_open", "failed"))
118+ return;
119+
120+ /* Update "constants" in BPF-prog *BEFORE* libbpf load */
121+ skel->rodata->GLOBAL_USER_MTU = mtu;
122+ skel->rodata->GLOBAL_USER_IFINDEX = ifindex;
123+
124+ err = test_check_mtu__load(skel);
125+ if (CHECK(err, "skel_load", "failed: %d\n", err))
126+ goto cleanup;
127+
128+ test_check_mtu_run_xdp(skel, skel->progs.xdp_use_helper, mtu);
129+ test_check_mtu_run_xdp(skel, skel->progs.xdp_exceed_mtu, mtu);
130+ test_check_mtu_run_xdp(skel, skel->progs.xdp_minus_delta, mtu);
131+
132+cleanup:
133+ test_check_mtu__destroy(skel);
134+}
135+
136+static void test_check_mtu_run_tc(struct test_check_mtu *skel,
137+ struct bpf_program *prog,
138+ __u32 mtu_expect)
139+{
140+ const char *prog_name = bpf_program__name(prog);
141+ int retval_expect = BPF_OK;
142+ __u32 mtu_result = 0;
143+ char buf[256] = {};
144+ int err;
145+ struct bpf_prog_test_run_attr tattr = {
146+ .repeat = 1,
147+ .data_in = &pkt_v4,
148+ .data_size_in = sizeof(pkt_v4),
149+ .data_out = buf,
150+ .data_size_out = sizeof(buf),
151+ .prog_fd = bpf_program__fd(prog),
152+ };
153+
154+ err = bpf_prog_test_run_xattr(&tattr);
155+ CHECK_ATTR(err != 0, "bpf_prog_test_run",
156+ "prog_name:%s (err %d errno %d retval %d)\n",
157+ prog_name, err, errno, tattr.retval);
158+
159+ CHECK(tattr.retval != retval_expect, "retval",
160+ "progname:%s unexpected retval=%d expected=%d\n",
161+ prog_name, tattr.retval, retval_expect);
162+
163+ /* Extract MTU that BPF-prog got */
164+ mtu_result = skel->bss->global_bpf_mtu_tc;
165+ ASSERT_EQ(mtu_result, mtu_expect, "MTU-compare-user");
166+}
167+
168+
169+static void test_check_mtu_tc(__u32 mtu, __u32 ifindex)
170+{
171+ struct test_check_mtu *skel;
172+ int err;
173+
174+ skel = test_check_mtu__open();
175+ if (CHECK(!skel, "skel_open", "failed"))
176+ return;
177+
178+ /* Update "constants" in BPF-prog *BEFORE* libbpf load */
179+ skel->rodata->GLOBAL_USER_MTU = mtu;
180+ skel->rodata->GLOBAL_USER_IFINDEX = ifindex;
181+
182+ err = test_check_mtu__load(skel);
183+ if (CHECK(err, "skel_load", "failed: %d\n", err))
184+ goto cleanup;
185+
186+ test_check_mtu_run_tc(skel, skel->progs.tc_use_helper, mtu);
187+ test_check_mtu_run_tc(skel, skel->progs.tc_exceed_mtu, mtu);
188+ test_check_mtu_run_tc(skel, skel->progs.tc_exceed_mtu_da, mtu);
189+ test_check_mtu_run_tc(skel, skel->progs.tc_minus_delta, mtu);
190+cleanup:
191+ test_check_mtu__destroy(skel);
192+}
193+
194+void test_check_mtu(void)
195+{
196+ __u32 mtu_lo;
197+
198+ if (test__start_subtest("bpf_check_mtu XDP-attach"))
199+ test_check_mtu_xdp_attach();
200+
201+ mtu_lo = read_mtu_device_lo();
202+ if (CHECK(mtu_lo < 0, "reading MTU value", "failed (err:%d)", mtu_lo))
203+ return;
204+
205+ if (test__start_subtest("bpf_check_mtu XDP-run"))
206+ test_check_mtu_xdp(mtu_lo, 0);
207+
208+ if (test__start_subtest("bpf_check_mtu XDP-run ifindex-lookup"))
209+ test_check_mtu_xdp(mtu_lo, IFINDEX_LO);
210+
211+ if (test__start_subtest("bpf_check_mtu TC-run"))
212+ test_check_mtu_tc(mtu_lo, 0);
213+
214+ if (test__start_subtest("bpf_check_mtu TC-run ifindex-lookup"))
215+ test_check_mtu_tc(mtu_lo, IFINDEX_LO);
216+}
tools/testing/selftests/bpf/prog_tests/cls_redirect.c+1−0
@@ -7,6 +7,7 @@ · 6 unmodified lines
77 #include <string.h>
88
99 #include <linux/pkt_cls.h>
10+#include <netinet/tcp.h>
1011
1112 #include <test_progs.h>
1213
tools/testing/selftests/bpf/prog_tests/fexit_stress.c+2−2
@@ -2,8 +2,8 @@ · 1 unmodified line
22 /* Copyright (c) 2019 Facebook */
33 #include <test_progs.h>
44
5−/* x86-64 fits 55 JITed and 43 interpreted progs into half page */
6−#define CNT 40
5+/* that's kernel internal BPF_MAX_TRAMP_PROGS define */
6+#define CNT 38
77
88 void test_fexit_stress(void)
99 {
tools/testing/selftests/bpf/prog_tests/global_func_args.c+60−0
@@ -0,0 +1,60 @@
1+// SPDX-License-Identifier: GPL-2.0
2+#include "test_progs.h"
3+#include "network_helpers.h"
4+
5+static __u32 duration;
6+
7+static void test_global_func_args0(struct bpf_object *obj)
8+{
9+ int err, i, map_fd, actual_value;
10+ const char *map_name = "values";
11+
12+ map_fd = bpf_find_map(__func__, obj, map_name);
13+ if (CHECK(map_fd < 0, "bpf_find_map", "cannot find BPF map %s: %s\n",
14+ map_name, strerror(errno)))
15+ return;
16+
17+ struct {
18+ const char *descr;
19+ int expected_value;
20+ } tests[] = {
21+ {"passing NULL pointer", 0},
22+ {"returning value", 1},
23+ {"reading local variable", 100 },
24+ {"writing local variable", 101 },
25+ {"reading global variable", 42 },
26+ {"writing global variable", 43 },
27+ {"writing to pointer-to-pointer", 1 },
28+ };
29+
30+ for (i = 0; i < ARRAY_SIZE(tests); ++i) {
31+ const int expected_value = tests[i].expected_value;
32+
33+ err = bpf_map_lookup_elem(map_fd, &i, &actual_value);
34+
35+ CHECK(err || actual_value != expected_value, tests[i].descr,
36+ "err %d result %d expected %d\n", err, actual_value, expected_value);
37+ }
38+}
39+
40+void test_global_func_args(void)
41+{
42+ const char *file = "./test_global_func_args.o";
43+ __u32 retval;
44+ struct bpf_object *obj;
45+ int err, prog_fd;
46+
47+ err = bpf_prog_load(file, BPF_PROG_TYPE_CGROUP_SKB, &obj, &prog_fd);
48+ if (CHECK(err, "load program", "error %d loading %s\n", err, file))
49+ return;
50+
51+ err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4),
52+ NULL, NULL, &retval, &duration);
53+ CHECK(err || retval, "pass global func args run",
54+ "err %d errno %d retval %d duration %d\n",
55+ err, errno, retval, duration);
56+
57+ test_global_func_args0(obj);
58+
59+ bpf_object__close(obj);
60+}
tools/testing/selftests/bpf/prog_tests/module_attach.c+27−0
@@ -21,9 +21,34 @@ static int trigger_module_test_read(int read_sz) · 20 unmodified lines
2121 return 0;
2222 }
2323
24+static int trigger_module_test_write(int write_sz)
25+{
26+ int fd, err;
27+ char *buf = malloc(write_sz);
28+
29+ if (!buf)
30+ return -ENOMEM;
31+
32+ memset(buf, 'a', write_sz);
33+ buf[write_sz-1] = '\0';
34+
35+ fd = open("/sys/kernel/bpf_testmod", O_WRONLY);
36+ err = -errno;
37+ if (CHECK(fd < 0, "testmod_file_open", "failed: %d\n", err)) {
38+ free(buf);
39+ return err;
40+ }
41+
42+ write(fd, buf, write_sz);
43+ close(fd);
44+ free(buf);
45+ return 0;
46+}
47+
2448 void test_module_attach(void)
2549 {
2650 const int READ_SZ = 456;
51+ const int WRITE_SZ = 457;
2752 struct test_module_attach* skel;
2853 struct test_module_attach__bss *bss;
2954 int err;
@@ -48,8 +73,10 @@ void test_module_attach(void) · 18 unmodified lines
4873
4974 /* trigger tracepoint */
5075 ASSERT_OK(trigger_module_test_read(READ_SZ), "trigger_read");
76+ ASSERT_OK(trigger_module_test_write(WRITE_SZ), "trigger_write");
5177
5278 ASSERT_EQ(bss->raw_tp_read_sz, READ_SZ, "raw_tp");
79+ ASSERT_EQ(bss->raw_tp_bare_write_sz, WRITE_SZ, "raw_tp_bare");
5380 ASSERT_EQ(bss->tp_btf_read_sz, READ_SZ, "tp_btf");
5481 ASSERT_EQ(bss->fentry_read_sz, READ_SZ, "fentry");
5582 ASSERT_EQ(bss->fentry_manual_read_sz, READ_SZ, "fentry_manual");
tools/testing/selftests/bpf/prog_tests/ns_current_pid_tgid.c+60−58
@@ -1,85 +1,87 @@
11 // SPDX-License-Identifier: GPL-2.0
22 /* Copyright (c) 2020 Carlos Neira cneirabustos@gmail.com */
3+
4+#define _GNU_SOURCE
35 #include <test_progs.h>
6+#include "test_ns_current_pid_tgid.skel.h"
47 #include <sys/stat.h>
58 #include <sys/types.h>
69 #include <unistd.h>
710 #include <sys/syscall.h>
11+#include <sched.h>
12+#include <sys/wait.h>
13+#include <sys/mount.h>
14+#include <sys/fcntl.h>
815
9−struct bss {
10− __u64 dev;
11− __u64 ino;
12− __u64 pid_tgid;
13− __u64 user_pid_tgid;
14−};
16+#define STACK_SIZE (1024 * 1024)
17+static char child_stack[STACK_SIZE];
1518
16−void test_ns_current_pid_tgid(void)
19+static int test_current_pid_tgid(void *args)
1720 {
18− const char *probe_name = "raw_tracepoint/sys_enter";
19− const char *file = "test_ns_current_pid_tgid.o";
20− int err, key = 0, duration = 0;
21− struct bpf_link *link = NULL;
22− struct bpf_program *prog;
23− struct bpf_map *bss_map;
24− struct bpf_object *obj;
25− struct bss bss;
21+ struct test_ns_current_pid_tgid__bss *bss;
22+ struct test_ns_current_pid_tgid *skel;
23+ int err = -1, duration = 0;
24+ pid_t tgid, pid;
2625 struct stat st;
27− __u64 id;
28−
29− obj = bpf_object__open_file(file, NULL);
30− if (CHECK(IS_ERR(obj), "obj_open", "err %ld\n", PTR_ERR(obj)))
31− return;
3226
33− err = bpf_object__load(obj);
34− if (CHECK(err, "obj_load", "err %d errno %d\n", err, errno))
27+ skel = test_ns_current_pid_tgid__open_and_load();
28+ if (CHECK(!skel, "skel_open_load", "failed to load skeleton\n"))
3529 goto cleanup;
3630
37− bss_map = bpf_object__find_map_by_name(obj, "test_ns_.bss");
38− if (CHECK(!bss_map, "find_bss_map", "failed\n"))
31+ pid = syscall(SYS_gettid);
32+ tgid = getpid();
33+
34+ err = stat("/proc/self/ns/pid", &st);
35+ if (CHECK(err, "stat", "failed /proc/self/ns/pid: %d\n", err))
3936 goto cleanup;
4037
41− prog = bpf_object__find_program_by_title(obj, probe_name);
42− if (CHECK(!prog, "find_prog", "prog '%s' not found\n",
43− probe_name))
38+ bss = skel->bss;
39+ bss->dev = st.st_dev;
40+ bss->ino = st.st_ino;
41+ bss->user_pid = 0;
42+ bss->user_tgid = 0;
43+
44+ err = test_ns_current_pid_tgid__attach(skel);
45+ if (CHECK(err, "skel_attach", "skeleton attach failed: %d\n", err))
4446 goto cleanup;
4547
46− memset(&bss, 0, sizeof(bss));
47− pid_t tid = syscall(SYS_gettid);
48− pid_t pid = getpid();
48+ /* trigger tracepoint */
49+ usleep(1);
50+ ASSERT_EQ(bss->user_pid, pid, "pid");
51+ ASSERT_EQ(bss->user_tgid, tgid, "tgid");
52+ err = 0;
4953
50− id = (__u64) tid << 32 | pid;
51− bss.user_pid_tgid = id;
54+cleanup:
55+ test_ns_current_pid_tgid__destroy(skel);
5256
53− if (CHECK_FAIL(stat("/proc/self/ns/pid", &st))) {
54− perror("Failed to stat /proc/self/ns/pid");
55− goto cleanup;
56− }
57+ return err;
58+}
5759
58− bss.dev = st.st_dev;
59− bss.ino = st.st_ino;
60+static void test_ns_current_pid_tgid_new_ns(void)
61+{
62+ int wstatus, duration = 0;
63+ pid_t cpid;
6064
61− err = bpf_map_update_elem(bpf_map__fd(bss_map), &key, &bss, 0);
62− if (CHECK(err, "setting_bss", "failed to set bss : %d\n", err))
63− goto cleanup;
65+ /* Create a process in a new namespace, this process
66+ * will be the init process of this new namespace hence will be pid 1.
67+ */
68+ cpid = clone(test_current_pid_tgid, child_stack + STACK_SIZE,
69+ CLONE_NEWPID | SIGCHLD, NULL);
6470
65− link = bpf_program__attach_raw_tracepoint(prog, "sys_enter");
66− if (CHECK(IS_ERR(link), "attach_raw_tp", "err %ld\n",
67− PTR_ERR(link))) {
68− link = NULL;
69− goto cleanup;
70− }
71+ if (CHECK(cpid == -1, "clone", strerror(errno)))
72+ return;
7173
72− /* trigger some syscalls */
73− usleep(1);
74+ if (CHECK(waitpid(cpid, &wstatus, 0) == -1, "waitpid", strerror(errno)))
75+ return;
7476
75− err = bpf_map_lookup_elem(bpf_map__fd(bss_map), &key, &bss);
76− if (CHECK(err, "set_bss", "failed to get bss : %d\n", err))
77− goto cleanup;
77+ if (CHECK(WEXITSTATUS(wstatus) != 0, "newns_pidtgid", "failed"))
78+ return;
79+}
7880
79− if (CHECK(id != bss.pid_tgid, "Compare user pid/tgid vs. bpf pid/tgid",
80− "User pid/tgid %llu BPF pid/tgid %llu\n", id, bss.pid_tgid))
81− goto cleanup;
82−cleanup:
83− bpf_link__destroy(link);
84− bpf_object__close(obj);
81+void test_ns_current_pid_tgid(void)
82+{
83+ if (test__start_subtest("ns_current_pid_tgid_root_ns"))
84+ test_current_pid_tgid(NULL);
85+ if (test__start_subtest("ns_current_pid_tgid_new_ns"))
86+ test_ns_current_pid_tgid_new_ns();
8587 }
tools/testing/selftests/bpf/prog_tests/recursion.c+41−0
@@ -0,0 +1,41 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/* Copyright (c) 2021 Facebook */
3+#include <test_progs.h>
4+#include "recursion.skel.h"
5+
6+void test_recursion(void)
7+{
8+ struct bpf_prog_info prog_info = {};
9+ __u32 prog_info_len = sizeof(prog_info);
10+ struct recursion *skel;
11+ int key = 0;
12+ int err;
13+
14+ skel = recursion__open_and_load();
15+ if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))
16+ return;
17+
18+ err = recursion__attach(skel);
19+ if (!ASSERT_OK(err, "skel_attach"))
20+ goto out;
21+
22+ ASSERT_EQ(skel->bss->pass1, 0, "pass1 == 0");
23+ bpf_map_lookup_elem(bpf_map__fd(skel->maps.hash1), &key, 0);
24+ ASSERT_EQ(skel->bss->pass1, 1, "pass1 == 1");
25+ bpf_map_lookup_elem(bpf_map__fd(skel->maps.hash1), &key, 0);
26+ ASSERT_EQ(skel->bss->pass1, 2, "pass1 == 2");
27+
28+ ASSERT_EQ(skel->bss->pass2, 0, "pass2 == 0");
29+ bpf_map_lookup_elem(bpf_map__fd(skel->maps.hash2), &key, 0);
30+ ASSERT_EQ(skel->bss->pass2, 1, "pass2 == 1");
31+ bpf_map_lookup_elem(bpf_map__fd(skel->maps.hash2), &key, 0);
32+ ASSERT_EQ(skel->bss->pass2, 2, "pass2 == 2");
33+
34+ err = bpf_obj_get_info_by_fd(bpf_program__fd(skel->progs.on_lookup),
35+ &prog_info, &prog_info_len);
36+ if (!ASSERT_OK(err, "get_prog_info"))
37+ goto out;
38+ ASSERT_EQ(prog_info.recursion_misses, 2, "recursion_misses");
39+out:
40+ recursion__destroy(skel);
41+}
tools/testing/selftests/bpf/prog_tests/socket_cookie.c+76−0
@@ -0,0 +1,76 @@
1+// SPDX-License-Identifier: GPL-2.0
2+// Copyright (c) 2020 Google LLC.
3+// Copyright (c) 2018 Facebook
4+
5+#include <test_progs.h>
6+#include "socket_cookie_prog.skel.h"
7+#include "network_helpers.h"
8+
9+static int duration;
10+
11+struct socket_cookie {
12+ __u64 cookie_key;
13+ __u32 cookie_value;
14+};
15+
16+void test_socket_cookie(void)
17+{
18+ int server_fd = 0, client_fd = 0, cgroup_fd = 0, err = 0;
19+ socklen_t addr_len = sizeof(struct sockaddr_in6);
20+ struct socket_cookie_prog *skel;
21+ __u32 cookie_expected_value;
22+ struct sockaddr_in6 addr;
23+ struct socket_cookie val;
24+
25+ skel = socket_cookie_prog__open_and_load();
26+ if (!ASSERT_OK_PTR(skel, "skel_open"))
27+ return;
28+
29+ cgroup_fd = test__join_cgroup("/socket_cookie");
30+ if (CHECK(cgroup_fd < 0, "join_cgroup", "cgroup creation failed\n"))
31+ goto out;
32+
33+ skel->links.set_cookie = bpf_program__attach_cgroup(
34+ skel->progs.set_cookie, cgroup_fd);
35+ if (!ASSERT_OK_PTR(skel->links.set_cookie, "prog_attach"))
36+ goto close_cgroup_fd;
37+
38+ skel->links.update_cookie_sockops = bpf_program__attach_cgroup(
39+ skel->progs.update_cookie_sockops, cgroup_fd);
40+ if (!ASSERT_OK_PTR(skel->links.update_cookie_sockops, "prog_attach"))
41+ goto close_cgroup_fd;
42+
43+ skel->links.update_cookie_tracing = bpf_program__attach(
44+ skel->progs.update_cookie_tracing);
45+ if (!ASSERT_OK_PTR(skel->links.update_cookie_tracing, "prog_attach"))
46+ goto close_cgroup_fd;
47+
48+ server_fd = start_server(AF_INET6, SOCK_STREAM, "::1", 0, 0);
49+ if (CHECK(server_fd < 0, "start_server", "errno %d\n", errno))
50+ goto close_cgroup_fd;
51+
52+ client_fd = connect_to_fd(server_fd, 0);
53+ if (CHECK(client_fd < 0, "connect_to_fd", "errno %d\n", errno))
54+ goto close_server_fd;
55+
56+ err = bpf_map_lookup_elem(bpf_map__fd(skel->maps.socket_cookies),
57+ &client_fd, &val);
58+ if (!ASSERT_OK(err, "map_lookup(socket_cookies)"))
59+ goto close_client_fd;
60+
61+ err = getsockname(client_fd, (struct sockaddr *)&addr, &addr_len);
62+ if (!ASSERT_OK(err, "getsockname"))
63+ goto close_client_fd;
64+
65+ cookie_expected_value = (ntohs(addr.sin6_port) << 8) | 0xFF;
66+ ASSERT_EQ(val.cookie_value, cookie_expected_value, "cookie_value");
67+
68+close_client_fd:
69+ close(client_fd);
70+close_server_fd:
71+ close(server_fd);
72+close_cgroup_fd:
73+ close(cgroup_fd);
74+out:
75+ socket_cookie_prog__destroy(skel);
76+}
tools/testing/selftests/bpf/prog_tests/sockmap_basic.c+1−0
@@ -1,6 +1,7 @@
11 // SPDX-License-Identifier: GPL-2.0
22 // Copyright (c) 2020 Cloudflare
33 #include <error.h>
4+#include <netinet/tcp.h>
45
56 #include "test_progs.h"
67 #include "test_skmsg_load_helpers.skel.h"
tools/testing/selftests/bpf/prog_tests/sockopt_sk.c+28−0
@@ -2,6 +2,12 @@ · 1 unmodified line
22 #include <test_progs.h>
33 #include "cgroup_helpers.h"
44
5+#include <linux/tcp.h>
6+
7+#ifndef SOL_TCP
8+#define SOL_TCP IPPROTO_TCP
9+#endif
10+
511 #define SOL_CUSTOM 0xdeadbeef
612
713 static int getsetsockopt(void)
@@ -11,6 +17,7 @@ static int getsetsockopt(void) · 3 unmodified lines
1117 char u8[4];
1218 __u32 u32;
1319 char cc[16]; /* TCP_CA_NAME_MAX */
20+ struct tcp_zerocopy_receive zc;
1421 } buf = {};
1522 socklen_t optlen;
1623 char *big_buf = NULL;
@@ -154,6 +161,27 @@ static int getsetsockopt(void) · 137 unmodified lines
154161 goto err;
155162 }
156163
164+ /* TCP_ZEROCOPY_RECEIVE triggers */
165+ memset(&buf, 0, sizeof(buf));
166+ optlen = sizeof(buf.zc);
167+ err = getsockopt(fd, SOL_TCP, TCP_ZEROCOPY_RECEIVE, &buf, &optlen);
168+ if (err) {
169+ log_err("Unexpected getsockopt(TCP_ZEROCOPY_RECEIVE) err=%d errno=%d",
170+ err, errno);
171+ goto err;
172+ }
173+
174+ memset(&buf, 0, sizeof(buf));
175+ buf.zc.address = 12345; /* rejected by BPF */
176+ optlen = sizeof(buf.zc);
177+ errno = 0;
178+ err = getsockopt(fd, SOL_TCP, TCP_ZEROCOPY_RECEIVE, &buf, &optlen);
179+ if (errno != EPERM) {
180+ log_err("Unexpected getsockopt(TCP_ZEROCOPY_RECEIVE) err=%d errno=%d",
181+ err, errno);
182+ goto err;
183+ }
184+
157185 free(big_buf);
158186 close(fd);
159187 return 0;
tools/testing/selftests/bpf/prog_tests/stack_var_off.c+35−0
@@ -0,0 +1,35 @@
1+// SPDX-License-Identifier: GPL-2.0
2+#include <test_progs.h>
3+#include "test_stack_var_off.skel.h"
4+
5+/* Test read and writes to the stack performed with offsets that are not
6+ * statically known.
7+ */
8+void test_stack_var_off(void)
9+{
10+ int duration = 0;
11+ struct test_stack_var_off *skel;
12+
13+ skel = test_stack_var_off__open_and_load();
14+ if (CHECK(!skel, "skel_open", "failed to open skeleton\n"))
15+ return;
16+
17+ /* Give pid to bpf prog so it doesn't trigger for anyone else. */
18+ skel->bss->test_pid = getpid();
19+ /* Initialize the probe's input. */
20+ skel->bss->input[0] = 2;
21+ skel->bss->input[1] = 42; /* This will be returned in probe_res. */
22+
23+ if (!ASSERT_OK(test_stack_var_off__attach(skel), "skel_attach"))
24+ goto cleanup;
25+
26+ /* Trigger probe. */
27+ usleep(1);
28+
29+ if (CHECK(skel->bss->probe_res != 42, "check_probe_res",
30+ "wrong probe res: %d\n", skel->bss->probe_res))
31+ goto cleanup;
32+
33+cleanup:
34+ test_stack_var_off__destroy(skel);
35+}
tools/testing/selftests/bpf/prog_tests/test_global_funcs.c+8−0
@@ -61,6 +61,14 @@ void test_test_global_funcs(void) · 60 unmodified lines
6161 { "test_global_func6.o" , "modified ctx ptr R2" },
6262 { "test_global_func7.o" , "foo() doesn't return scalar" },
6363 { "test_global_func8.o" },
64+ { "test_global_func9.o" },
65+ { "test_global_func10.o", "invalid indirect read from stack" },
66+ { "test_global_func11.o", "Caller passes invalid args into func#1" },
67+ { "test_global_func12.o", "invalid mem access 'mem_or_null'" },
68+ { "test_global_func13.o", "Caller passes invalid args into func#1" },
69+ { "test_global_func14.o", "reference type('FWD S') size cannot be determined" },
70+ { "test_global_func15.o", "At program exit the register R0 has value" },
71+ { "test_global_func16.o", "invalid indirect read from stack" },
6472 };
6573 libbpf_print_fn_t old_print_fn = NULL;
6674 int err, i, duration = 0;
tools/testing/selftests/bpf/prog_tests/test_ima.c+18−5
@@ -9,6 +9,7 @@ · 8 unmodified lines
99 #include <unistd.h>
1010 #include <sys/wait.h>
1111 #include <test_progs.h>
12+#include <linux/ring_buffer.h>
1213
1314 #include "ima.skel.h"
1415
@@ -31,9 +32,18 @@ static int run_measured_process(const char *measured_dir, u32 *monitored_pid) · 16 unmodified lines
3132 return -EINVAL;
3233 }
3334
35+static u64 ima_hash_from_bpf;
36+
37+static int process_sample(void *ctx, void *data, size_t len)
38+{
39+ ima_hash_from_bpf = *((u64 *)data);
40+ return 0;
41+}
42+
3443 void test_test_ima(void)
3544 {
3645 char measured_dir_template[] = "/tmp/ima_measuredXXXXXX";
46+ struct ring_buffer *ringbuf;
3747 const char *measured_dir;
3848 char cmd[256];
3949
@@ -44,6 +54,11 @@ void test_test_ima(void) · 4 unmodified lines
4454 if (CHECK(!skel, "skel_load", "skeleton failed\n"))
4555 goto close_prog;
4656
57+ ringbuf = ring_buffer__new(bpf_map__fd(skel->maps.ringbuf),
58+ process_sample, NULL, NULL);
59+ if (!ASSERT_OK_PTR(ringbuf, "ringbuf"))
60+ goto close_prog;
61+
4762 err = ima__attach(skel);
4863 if (CHECK(err, "attach", "attach failed: %d\n", err))
4964 goto close_prog;
@@ -60,11 +75,9 @@ void test_test_ima(void) · 10 unmodified lines
6075 if (CHECK(err, "run_measured_process", "err = %d\n", err))
6176 goto close_clean;
6277
63− CHECK(skel->data->ima_hash_ret < 0, "ima_hash_ret",
64− "ima_hash_ret = %ld\n", skel->data->ima_hash_ret);
65−
66− CHECK(skel->bss->ima_hash == 0, "ima_hash",
67− "ima_hash = %lu\n", skel->bss->ima_hash);
78+ err = ring_buffer__consume(ringbuf);
79+ ASSERT_EQ(err, 1, "num_samples_or_err");
80+ ASSERT_NEQ(ima_hash_from_bpf, 0, "ima_hash");
6881
6982 close_clean:
7083 snprintf(cmd, sizeof(cmd), "./ima_setup.sh cleanup %s", measured_dir);
tools/testing/selftests/bpf/prog_tests/test_local_storage.c+1−1
@@ -113,7 +113,7 @@ static bool check_syscall_operations(int map_fd, int obj_fd) · 112 unmodified lines
113113
114114 void test_test_local_storage(void)
115115 {
116− char tmp_dir_path[64] = "/tmp/local_storageXXXXXX";
116+ char tmp_dir_path[] = "/tmp/local_storageXXXXXX";
117117 int err, serv_sk = -1, task_fd = -1, rm_fd = -1;
118118 struct local_storage *skel = NULL;
119119 char tmp_exec_path[64];
tools/testing/selftests/bpf/prog_tests/trampoline_count.c+2−2
@@ -4,7 +4,7 @@ · 3 unmodified lines
44 #include <sys/prctl.h>
55 #include <test_progs.h>
66
7−#define MAX_TRAMP_PROGS 40
7+#define MAX_TRAMP_PROGS 38
88
99 struct inst {
1010 struct bpf_object *obj;
@@ -52,7 +52,7 @@ void test_trampoline_count(void) · 41 unmodified lines
5252 struct bpf_link *link;
5353 char comm[16] = {};
5454
55− /* attach 'allowed' 40 trampoline programs */
55+ /* attach 'allowed' trampoline programs */
5656 for (i = 0; i < MAX_TRAMP_PROGS; i++) {
5757 obj = bpf_object__open_file(object, NULL);
5858 if (CHECK(IS_ERR(obj), "obj_open_file", "err %ld\n", PTR_ERR(obj))) {
tools/testing/selftests/bpf/progs/atomic_bounds.c+24−0
@@ -0,0 +1,24 @@
1+// SPDX-License-Identifier: GPL-2.0
2+#include <linux/bpf.h>
3+#include <bpf/bpf_helpers.h>
4+#include <bpf/bpf_tracing.h>
5+#include <stdbool.h>
6+
7+#ifdef ENABLE_ATOMICS_TESTS
8+bool skip_tests __attribute((__section__(".data"))) = false;
9+#else
10+bool skip_tests = true;
11+#endif
12+
13+SEC("fentry/bpf_fentry_test1")
14+int BPF_PROG(sub, int x)
15+{
16+#ifdef ENABLE_ATOMICS_TESTS
17+ int a = 0;
18+ int b = __sync_fetch_and_add(&a, 1);
19+ /* b is certainly 0 here. Can the verifier tell? */
20+ while (b)
21+ continue;
22+#endif
23+ return 0;
24+}
tools/testing/selftests/bpf/progs/bind_perm.c+45−0
@@ -0,0 +1,45 @@
1+// SPDX-License-Identifier: GPL-2.0
2+
3+#include <linux/stddef.h>
4+#include <linux/bpf.h>
5+#include <sys/types.h>
6+#include <sys/socket.h>
7+#include <bpf/bpf_helpers.h>
8+#include <bpf/bpf_endian.h>
9+
10+static __always_inline int bind_prog(struct bpf_sock_addr *ctx, int family)
11+{
12+ struct bpf_sock *sk;
13+
14+ sk = ctx->sk;
15+ if (!sk)
16+ return 0;
17+
18+ if (sk->family != family)
19+ return 0;
20+
21+ if (ctx->type != SOCK_STREAM)
22+ return 0;
23+
24+ /* Return 1 OR'ed with the first bit set to indicate
25+ * that CAP_NET_BIND_SERVICE should be bypassed.
26+ */
27+ if (ctx->user_port == bpf_htons(111))
28+ return (1 | 2);
29+
30+ return 1;
31+}
32+
33+SEC("cgroup/bind4")
34+int bind_v4_prog(struct bpf_sock_addr *ctx)
35+{
36+ return bind_prog(ctx, AF_INET);
37+}
38+
39+SEC("cgroup/bind6")
40+int bind_v6_prog(struct bpf_sock_addr *ctx)
41+{
42+ return bind_prog(ctx, AF_INET6);
43+}
44+
45+char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/bpf_iter.h+8−0
@@ -7,6 +7,7 @@ · 6 unmodified lines
77 #define bpf_iter__netlink bpf_iter__netlink___not_used
88 #define bpf_iter__task bpf_iter__task___not_used
99 #define bpf_iter__task_file bpf_iter__task_file___not_used
10+#define bpf_iter__task_vma bpf_iter__task_vma___not_used
1011 #define bpf_iter__tcp bpf_iter__tcp___not_used
1112 #define tcp6_sock tcp6_sock___not_used
1213 #define bpf_iter__udp bpf_iter__udp___not_used
@@ -26,6 +27,7 @@ · 13 unmodified lines
2627 #undef bpf_iter__netlink
2728 #undef bpf_iter__task
2829 #undef bpf_iter__task_file
30+#undef bpf_iter__task_vma
2931 #undef bpf_iter__tcp
3032 #undef tcp6_sock
3133 #undef bpf_iter__udp
@@ -67,6 +69,12 @@ struct bpf_iter__task_file { · 35 unmodified lines
6769 struct file *file;
6870 } __attribute__((preserve_access_index));
6971
72+struct bpf_iter__task_vma {
73+ struct bpf_iter_meta *meta;
74+ struct task_struct *task;
75+ struct vm_area_struct *vma;
76+} __attribute__((preserve_access_index));
77+
7078 struct bpf_iter__bpf_map {
7179 struct bpf_iter_meta *meta;
7280 struct bpf_map *map;
tools/testing/selftests/bpf/progs/bpf_iter_task_vma.c+58−0
@@ -0,0 +1,58 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/* Copyright (c) 2020 Facebook */
3+#include "bpf_iter.h"
4+#include <bpf/bpf_helpers.h>
5+#include <bpf/bpf_tracing.h>
6+
7+char _license[] SEC("license") = "GPL";
8+
9+/* Copied from mm.h */
10+#define VM_READ 0x00000001
11+#define VM_WRITE 0x00000002
12+#define VM_EXEC 0x00000004
13+#define VM_MAYSHARE 0x00000080
14+
15+/* Copied from kdev_t.h */
16+#define MINORBITS 20
17+#define MINORMASK ((1U << MINORBITS) - 1)
18+#define MAJOR(dev) ((unsigned int) ((dev) >> MINORBITS))
19+#define MINOR(dev) ((unsigned int) ((dev) & MINORMASK))
20+
21+#define D_PATH_BUF_SIZE 1024
22+char d_path_buf[D_PATH_BUF_SIZE] = {};
23+__u32 pid = 0;
24+
25+SEC("iter/task_vma") int proc_maps(struct bpf_iter__task_vma *ctx)
26+{
27+ struct vm_area_struct *vma = ctx->vma;
28+ struct seq_file *seq = ctx->meta->seq;
29+ struct task_struct *task = ctx->task;
30+ struct file *file;
31+ char perm_str[] = "----";
32+
33+ if (task == (void *)0 || vma == (void *)0)
34+ return 0;
35+
36+ file = vma->vm_file;
37+ if (task->tgid != pid)
38+ return 0;
39+ perm_str[0] = (vma->vm_flags & VM_READ) ? 'r' : '-';
40+ perm_str[1] = (vma->vm_flags & VM_WRITE) ? 'w' : '-';
41+ perm_str[2] = (vma->vm_flags & VM_EXEC) ? 'x' : '-';
42+ perm_str[3] = (vma->vm_flags & VM_MAYSHARE) ? 's' : 'p';
43+ BPF_SEQ_PRINTF(seq, "%08llx-%08llx %s ", vma->vm_start, vma->vm_end, perm_str);
44+
45+ if (file) {
46+ __u32 dev = file->f_inode->i_sb->s_dev;
47+
48+ bpf_d_path(&file->f_path, d_path_buf, D_PATH_BUF_SIZE);
49+
50+ BPF_SEQ_PRINTF(seq, "%08llx ", vma->vm_pgoff << 12);
51+ BPF_SEQ_PRINTF(seq, "%02x:%02x %u", MAJOR(dev), MINOR(dev),
52+ file->f_inode->i_ino);
53+ BPF_SEQ_PRINTF(seq, "\t%s\n", d_path_buf);
54+ } else {
55+ BPF_SEQ_PRINTF(seq, "%08llx 00:00 0\n", 0ULL);
56+ }
57+ return 0;
58+}
tools/testing/selftests/bpf/progs/connect_force_port4.c+8−0
@@ -10,6 +10,8 @@ · 9 unmodified lines
1010 #include <bpf/bpf_helpers.h>
1111 #include <bpf/bpf_endian.h>
1212
13+#include <bpf_sockopt_helpers.h>
14+
1315 char _license[] SEC("license") = "GPL";
1416 int _version SEC("version") = 1;
1517
@@ -58,6 +60,9 @@ int connect4(struct bpf_sock_addr *ctx) · 42 unmodified lines
5860 SEC("cgroup/getsockname4")
5961 int getsockname4(struct bpf_sock_addr *ctx)
6062 {
63+ if (!get_set_sk_priority(ctx))
64+ return 1;
65+
6166 /* Expose local server as 1.2.3.4:60000 to client. */
6267 if (ctx->user_port == bpf_htons(60123)) {
6368 ctx->user_ip4 = bpf_htonl(0x01020304);
@@ -71,6 +76,9 @@ int getpeername4(struct bpf_sock_addr *ctx) · 7 unmodified lines
7176 {
7277 struct svc_addr *orig;
7378
79+ if (!get_set_sk_priority(ctx))
80+ return 1;
81+
7482 /* Expose service 1.2.3.4:60000 as peer instead of backend. */
7583 if (ctx->user_port == bpf_htons(60123)) {
7684 orig = bpf_sk_storage_get(&service_mapping, ctx->sk, 0, 0);
tools/testing/selftests/bpf/progs/connect_force_port6.c+8−0
@@ -9,6 +9,8 @@ · 8 unmodified lines
99 #include <bpf/bpf_helpers.h>
1010 #include <bpf/bpf_endian.h>
1111
12+#include <bpf_sockopt_helpers.h>
13+
1214 char _license[] SEC("license") = "GPL";
1315 int _version SEC("version") = 1;
1416
@@ -63,6 +65,9 @@ int connect6(struct bpf_sock_addr *ctx) · 48 unmodified lines
6365 SEC("cgroup/getsockname6")
6466 int getsockname6(struct bpf_sock_addr *ctx)
6567 {
68+ if (!get_set_sk_priority(ctx))
69+ return 1;
70+
6671 /* Expose local server as [fc00::1]:60000 to client. */
6772 if (ctx->user_port == bpf_htons(60124)) {
6873 ctx->user_ip6[0] = bpf_htonl(0xfc000000);
@@ -79,6 +84,9 @@ int getpeername6(struct bpf_sock_addr *ctx) · 10 unmodified lines
7984 {
8085 struct svc_addr *orig;
8186
87+ if (!get_set_sk_priority(ctx))
88+ return 1;
89+
8290 /* Expose service [fc00::1]:60000 as peer instead of backend. */
8391 if (ctx->user_port == bpf_htons(60124)) {
8492 orig = bpf_sk_storage_get(&service_mapping, ctx->sk, 0, 0);
tools/testing/selftests/bpf/progs/ima.c+25−8
@@ -9,20 +9,37 @@ · 8 unmodified lines
99 #include <bpf/bpf_helpers.h>
1010 #include <bpf/bpf_tracing.h>
1111
12−long ima_hash_ret = -1;
13−u64 ima_hash = 0;
1412 u32 monitored_pid = 0;
1513
14+struct {
15+ __uint(type, BPF_MAP_TYPE_RINGBUF);
16+ __uint(max_entries, 1 << 12);
17+} ringbuf SEC(".maps");
18+
1619 char _license[] SEC("license") = "GPL";
1720
1821 SEC("lsm.s/bprm_committed_creds")
19−int BPF_PROG(ima, struct linux_binprm *bprm)
22+void BPF_PROG(ima, struct linux_binprm *bprm)
2023 {
21− u32 pid = bpf_get_current_pid_tgid() >> 32;
24+ u64 ima_hash = 0;
25+ u64 *sample;
26+ int ret;
27+ u32 pid;
28+
29+ pid = bpf_get_current_pid_tgid() >> 32;
30+ if (pid == monitored_pid) {
31+ ret = bpf_ima_inode_hash(bprm->file->f_inode, &ima_hash,
32+ sizeof(ima_hash));
33+ if (ret < 0 || ima_hash == 0)
34+ return;
35+
36+ sample = bpf_ringbuf_reserve(&ringbuf, sizeof(u64), 0);
37+ if (!sample)
38+ return;
2239
23− if (pid == monitored_pid)
24− ima_hash_ret = bpf_ima_inode_hash(bprm->file->f_inode,
25− &ima_hash, sizeof(ima_hash));
40+ *sample = ima_hash;
41+ bpf_ringbuf_submit(sample, 0);
42+ }
2643
27− return 0;
44+ return;
2845 }
tools/testing/selftests/bpf/progs/lsm.c+69−0
@@ -30,6 +30,53 @@ struct { · 29 unmodified lines
3030 __type(value, __u64);
3131 } lru_hash SEC(".maps");
3232
33+struct {
34+ __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
35+ __uint(max_entries, 1);
36+ __type(key, __u32);
37+ __type(value, __u64);
38+} percpu_array SEC(".maps");
39+
40+struct {
41+ __uint(type, BPF_MAP_TYPE_PERCPU_HASH);
42+ __uint(max_entries, 1);
43+ __type(key, __u32);
44+ __type(value, __u64);
45+} percpu_hash SEC(".maps");
46+
47+struct {
48+ __uint(type, BPF_MAP_TYPE_LRU_PERCPU_HASH);
49+ __uint(max_entries, 1);
50+ __type(key, __u32);
51+ __type(value, __u64);
52+} lru_percpu_hash SEC(".maps");
53+
54+struct inner_map {
55+ __uint(type, BPF_MAP_TYPE_ARRAY);
56+ __uint(max_entries, 1);
57+ __type(key, int);
58+ __type(value, __u64);
59+} inner_map SEC(".maps");
60+
61+struct outer_arr {
62+ __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS);
63+ __uint(max_entries, 1);
64+ __uint(key_size, sizeof(int));
65+ __uint(value_size, sizeof(int));
66+ __array(values, struct inner_map);
67+} outer_arr SEC(".maps") = {
68+ .values = { [0] = &inner_map },
69+};
70+
71+struct outer_hash {
72+ __uint(type, BPF_MAP_TYPE_HASH_OF_MAPS);
73+ __uint(max_entries, 1);
74+ __uint(key_size, sizeof(int));
75+ __array(values, struct inner_map);
76+} outer_hash SEC(".maps") = {
77+ .values = { [0] = &inner_map },
78+};
79+
3380 char _license[] SEC("license") = "GPL";
3481
3582 int monitored_pid = 0;
@@ -61,6 +108,7 @@ SEC("lsm.s/bprm_committed_creds") · 25 unmodified lines
61108 int BPF_PROG(test_void_hook, struct linux_binprm *bprm)
62109 {
63110 __u32 pid = bpf_get_current_pid_tgid() >> 32;
111+ struct inner_map *inner_map;
64112 char args[64];
65113 __u32 key = 0;
66114 __u64 *value;
@@ -80,6 +128,27 @@ int BPF_PROG(test_void_hook, struct linux_binprm *bprm) · 13 unmodified lines
80128 value = bpf_map_lookup_elem(&lru_hash, &key);
81129 if (value)
82130 *value = 0;
131+ value = bpf_map_lookup_elem(&percpu_array, &key);
132+ if (value)
133+ *value = 0;
134+ value = bpf_map_lookup_elem(&percpu_hash, &key);
135+ if (value)
136+ *value = 0;
137+ value = bpf_map_lookup_elem(&lru_percpu_hash, &key);
138+ if (value)
139+ *value = 0;
140+ inner_map = bpf_map_lookup_elem(&outer_arr, &key);
141+ if (inner_map) {
142+ value = bpf_map_lookup_elem(inner_map, &key);
143+ if (value)
144+ *value = 0;
145+ }
146+ inner_map = bpf_map_lookup_elem(&outer_hash, &key);
147+ if (inner_map) {
148+ value = bpf_map_lookup_elem(inner_map, &key);
149+ if (value)
150+ *value = 0;
151+ }
83152
84153 return 0;
85154 }
tools/testing/selftests/bpf/progs/recursion.c+46−0
@@ -0,0 +1,46 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/* Copyright (c) 2021 Facebook */
3+
4+#include "vmlinux.h"
5+#include <bpf/bpf_helpers.h>
6+#include <bpf/bpf_tracing.h>
7+
8+char _license[] SEC("license") = "GPL";
9+
10+struct {
11+ __uint(type, BPF_MAP_TYPE_HASH);
12+ __uint(max_entries, 1);
13+ __type(key, int);
14+ __type(value, long);
15+} hash1 SEC(".maps");
16+
17+struct {
18+ __uint(type, BPF_MAP_TYPE_HASH);
19+ __uint(max_entries, 1);
20+ __type(key, int);
21+ __type(value, long);
22+} hash2 SEC(".maps");
23+
24+int pass1 = 0;
25+int pass2 = 0;
26+
27+SEC("fentry/__htab_map_lookup_elem")
28+int BPF_PROG(on_lookup, struct bpf_map *map)
29+{
30+ int key = 0;
31+
32+ if (map == (void *)&hash1) {
33+ pass1++;
34+ return 0;
35+ }
36+ if (map == (void *)&hash2) {
37+ pass2++;
38+ /* htab_map_gen_lookup() will inline below call
39+ * into direct call to __htab_map_lookup_elem()
40+ */
41+ bpf_map_lookup_elem(&hash2, &key);
42+ return 0;
43+ }
44+
45+ return 0;
46+}
tools/testing/selftests/bpf/progs/recvmsg4_prog.c+42−0
@@ -0,0 +1,42 @@
1+// SPDX-License-Identifier: GPL-2.0
2+
3+#include <linux/stddef.h>
4+#include <linux/bpf.h>
5+#include <linux/in.h>
6+#include <sys/socket.h>
7+
8+#include <bpf/bpf_helpers.h>
9+#include <bpf/bpf_endian.h>
10+
11+#include <bpf_sockopt_helpers.h>
12+
13+#define SERV4_IP 0xc0a801feU /* 192.168.1.254 */
14+#define SERV4_PORT 4040
15+
16+SEC("cgroup/recvmsg4")
17+int recvmsg4_prog(struct bpf_sock_addr *ctx)
18+{
19+ struct bpf_sock *sk;
20+ __u32 user_ip4;
21+ __u16 user_port;
22+
23+ sk = ctx->sk;
24+ if (!sk)
25+ return 1;
26+
27+ if (sk->family != AF_INET)
28+ return 1;
29+
30+ if (ctx->type != SOCK_STREAM && ctx->type != SOCK_DGRAM)
31+ return 1;
32+
33+ if (!get_set_sk_priority(ctx))
34+ return 1;
35+
36+ ctx->user_ip4 = bpf_htonl(SERV4_IP);
37+ ctx->user_port = bpf_htons(SERV4_PORT);
38+
39+ return 1;
40+}
41+
42+char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/recvmsg6_prog.c+48−0
@@ -0,0 +1,48 @@
1+// SPDX-License-Identifier: GPL-2.0
2+
3+#include <linux/stddef.h>
4+#include <linux/bpf.h>
5+#include <linux/in6.h>
6+#include <sys/socket.h>
7+
8+#include <bpf/bpf_helpers.h>
9+#include <bpf/bpf_endian.h>
10+
11+#include <bpf_sockopt_helpers.h>
12+
13+#define SERV6_IP_0 0xfaceb00c /* face:b00c:1234:5678::abcd */
14+#define SERV6_IP_1 0x12345678
15+#define SERV6_IP_2 0x00000000
16+#define SERV6_IP_3 0x0000abcd
17+#define SERV6_PORT 6060
18+
19+SEC("cgroup/recvmsg6")
20+int recvmsg6_prog(struct bpf_sock_addr *ctx)
21+{
22+ struct bpf_sock *sk;
23+ __u32 user_ip4;
24+ __u16 user_port;
25+
26+ sk = ctx->sk;
27+ if (!sk)
28+ return 1;
29+
30+ if (sk->family != AF_INET6)
31+ return 1;
32+
33+ if (ctx->type != SOCK_STREAM && ctx->type != SOCK_DGRAM)
34+ return 1;
35+
36+ if (!get_set_sk_priority(ctx))
37+ return 1;
38+
39+ ctx->user_ip6[0] = bpf_htonl(SERV6_IP_0);
40+ ctx->user_ip6[1] = bpf_htonl(SERV6_IP_1);
41+ ctx->user_ip6[2] = bpf_htonl(SERV6_IP_2);
42+ ctx->user_ip6[3] = bpf_htonl(SERV6_IP_3);
43+ ctx->user_port = bpf_htons(SERV6_PORT);
44+
45+ return 1;
46+}
47+
48+char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/sendmsg4_prog.c+7−0
@@ -8,6 +8,8 @@ · 7 unmodified lines
88 #include <bpf/bpf_helpers.h>
99 #include <bpf/bpf_endian.h>
1010
11+#include <bpf_sockopt_helpers.h>
12+
1113 #define SRC1_IP4 0xAC100001U /* 172.16.0.1 */
1214 #define SRC2_IP4 0x00000000U
1315 #define SRC_REWRITE_IP4 0x7f000004U
@@ -21,9 +23,14 @@ int _version SEC("version") = 1; · 7 unmodified lines
2123 SEC("cgroup/sendmsg4")
2224 int sendmsg_v4_prog(struct bpf_sock_addr *ctx)
2325 {
26+ int prio;
27+
2428 if (ctx->type != SOCK_DGRAM)
2529 return 0;
2630
31+ if (!get_set_sk_priority(ctx))
32+ return 0;
33+
2734 /* Rewrite source. */
2835 if (ctx->msg_src_ip4 == bpf_htonl(SRC1_IP4) ||
2936 ctx->msg_src_ip4 == bpf_htonl(SRC2_IP4)) {
tools/testing/selftests/bpf/progs/sendmsg6_prog.c+5−0
@@ -8,6 +8,8 @@ · 7 unmodified lines
88 #include <bpf/bpf_helpers.h>
99 #include <bpf/bpf_endian.h>
1010
11+#include <bpf_sockopt_helpers.h>
12+
1113 #define SRC_REWRITE_IP6_0 0
1214 #define SRC_REWRITE_IP6_1 0
1315 #define SRC_REWRITE_IP6_2 0
@@ -28,6 +30,9 @@ int sendmsg_v6_prog(struct bpf_sock_addr *ctx) · 14 unmodified lines
2830 if (ctx->type != SOCK_DGRAM)
2931 return 0;
3032
33+ if (!get_set_sk_priority(ctx))
34+ return 0;
35+
3136 /* Rewrite source. */
3237 if (ctx->msg_src_ip6[3] == bpf_htonl(1) ||
3338 ctx->msg_src_ip6[3] == bpf_htonl(0)) {
tools/testing/selftests/bpf/progs/socket_cookie_prog.c+38−9
@@ -1,11 +1,13 @@
11 // SPDX-License-Identifier: GPL-2.0
22 // Copyright (c) 2018 Facebook
33
4−#include <linux/bpf.h>
5−#include <sys/socket.h>
4+#include "vmlinux.h"
65
76 #include <bpf/bpf_helpers.h>
87 #include <bpf/bpf_endian.h>
8+#include <bpf/bpf_tracing.h>
9+
10+#define AF_INET6 10
911
1012 struct socket_cookie {
1113 __u64 cookie_key;
@@ -19,6 +21,14 @@ struct { · 7 unmodified lines
1921 __type(value, struct socket_cookie);
2022 } socket_cookies SEC(".maps");
2123
24+/*
25+ * These three programs get executed in a row on connect() syscalls. The
26+ * userspace side of the test creates a client socket, issues a connect() on it
27+ * and then checks that the local storage associated with this socket has:
28+ * cookie_value == local_port << 8 | 0xFF
29+ * The different parts of this cookie_value are appended by those hooks if they
30+ * all agree on the output of bpf_get_socket_cookie().
31+ */
2232 SEC("cgroup/connect6")
2333 int set_cookie(struct bpf_sock_addr *ctx)
2434 {
@@ -32,16 +42,16 @@ int set_cookie(struct bpf_sock_addr *ctx) · 7 unmodified lines
3242 if (!p)
3343 return 1;
3444
35− p->cookie_value = 0xFF;
45+ p->cookie_value = 0xF;
3646 p->cookie_key = bpf_get_socket_cookie(ctx);
3747
3848 return 1;
3949 }
4050
4151 SEC("sockops")
42−int update_cookie(struct bpf_sock_ops *ctx)
52+int update_cookie_sockops(struct bpf_sock_ops *ctx)
4353 {
44− struct bpf_sock *sk;
54+ struct bpf_sock *sk = ctx->sk;
4555 struct socket_cookie *p;
4656
4757 if (ctx->family != AF_INET6)
@@ -50,21 +60,40 @@ int update_cookie(struct bpf_sock_ops *ctx) · 2 unmodified lines
5060 if (ctx->op != BPF_SOCK_OPS_TCP_CONNECT_CB)
5161 return 1;
5262
53− if (!ctx->sk)
63+ if (!sk)
5464 return 1;
5565
56− p = bpf_sk_storage_get(&socket_cookies, ctx->sk, 0, 0);
66+ p = bpf_sk_storage_get(&socket_cookies, sk, 0, 0);
5767 if (!p)
5868 return 1;
5969
6070 if (p->cookie_key != bpf_get_socket_cookie(ctx))
6171 return 1;
6272
63− p->cookie_value = (ctx->local_port << 8) | p->cookie_value;
73+ p->cookie_value |= (ctx->local_port << 8);
6474
6575 return 1;
6676 }
6777
68−int _version SEC("version") = 1;
78+SEC("fexit/inet_stream_connect")
79+int BPF_PROG(update_cookie_tracing, struct socket *sock,
80+ struct sockaddr *uaddr, int addr_len, int flags)
81+{
82+ struct socket_cookie *p;
83+
84+ if (uaddr->sa_family != AF_INET6)
85+ return 0;
86+
87+ p = bpf_sk_storage_get(&socket_cookies, sock->sk, 0, 0);
88+ if (!p)
89+ return 0;
90+
91+ if (p->cookie_key != bpf_get_socket_cookie(sock->sk))
92+ return 0;
93+
94+ p->cookie_value |= 0xF0;
95+
96+ return 0;
97+}
6998
7099 char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/sockopt_sk.c+21−2
@@ -1,8 +1,8 @@
11 // SPDX-License-Identifier: GPL-2.0
22 #include <string.h>
3−#include <netinet/in.h>
4−#include <netinet/tcp.h>
3+#include <linux/tcp.h>
54 #include <linux/bpf.h>
5+#include <netinet/in.h>
66 #include <bpf/bpf_helpers.h>
77
88 char _license[] SEC("license") = "GPL";
@@ -12,6 +12,10 @@ __u32 _version SEC("version") = 1; · 3 unmodified lines
1212 #define PAGE_SIZE 4096
1313 #endif
1414
15+#ifndef SOL_TCP
16+#define SOL_TCP IPPROTO_TCP
17+#endif
18+
1519 #define SOL_CUSTOM 0xdeadbeef
1620
1721 struct sockopt_sk {
@@ -57,6 +61,21 @@ int _getsockopt(struct bpf_sockopt *ctx) · 39 unmodified lines
5761 return 1;
5862 }
5963
64+ if (ctx->level == SOL_TCP && ctx->optname == TCP_ZEROCOPY_RECEIVE) {
65+ /* Verify that TCP_ZEROCOPY_RECEIVE triggers.
66+ * It has a custom implementation for performance
67+ * reasons.
68+ */
69+
70+ if (optval + sizeof(struct tcp_zerocopy_receive) > optval_end)
71+ return 0; /* EPERM, bounds check */
72+
73+ if (((struct tcp_zerocopy_receive *)optval)->address != 0)
74+ return 0; /* EPERM, unexpected data */
75+
76+ return 1;
77+ }
78+
6079 if (ctx->level == SOL_IP && ctx->optname == IP_FREEBIND) {
6180 if (optval + 1 > optval_end)
6281 return 0; /* EPERM, bounds check */
tools/testing/selftests/bpf/progs/test_check_mtu.c+198−0
@@ -0,0 +1,198 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/* Copyright (c) 2020 Jesper Dangaard Brouer */
3+
4+#include <linux/bpf.h>
5+#include <bpf/bpf_helpers.h>
6+#include <linux/if_ether.h>
7+
8+#include <stddef.h>
9+#include <stdint.h>
10+
11+char _license[] SEC("license") = "GPL";
12+
13+/* Userspace will update with MTU it can see on device */
14+static volatile const int GLOBAL_USER_MTU;
15+static volatile const __u32 GLOBAL_USER_IFINDEX;
16+
17+/* BPF-prog will update these with MTU values it can see */
18+__u32 global_bpf_mtu_xdp = 0;
19+__u32 global_bpf_mtu_tc = 0;
20+
21+SEC("xdp")
22+int xdp_use_helper_basic(struct xdp_md *ctx)
23+{
24+ __u32 mtu_len = 0;
25+
26+ if (bpf_check_mtu(ctx, 0, &mtu_len, 0, 0))
27+ return XDP_ABORTED;
28+
29+ return XDP_PASS;
30+}
31+
32+SEC("xdp")
33+int xdp_use_helper(struct xdp_md *ctx)
34+{
35+ int retval = XDP_PASS; /* Expected retval on successful test */
36+ __u32 mtu_len = 0;
37+ __u32 ifindex = 0;
38+ int delta = 0;
39+
40+ /* When ifindex is zero, save net_device lookup and use ctx netdev */
41+ if (GLOBAL_USER_IFINDEX > 0)
42+ ifindex = GLOBAL_USER_IFINDEX;
43+
44+ if (bpf_check_mtu(ctx, ifindex, &mtu_len, delta, 0)) {
45+ /* mtu_len is also valid when check fail */
46+ retval = XDP_ABORTED;
47+ goto out;
48+ }
49+
50+ if (mtu_len != GLOBAL_USER_MTU)
51+ retval = XDP_DROP;
52+
53+out:
54+ global_bpf_mtu_xdp = mtu_len;
55+ return retval;
56+}
57+
58+SEC("xdp")
59+int xdp_exceed_mtu(struct xdp_md *ctx)
60+{
61+ void *data_end = (void *)(long)ctx->data_end;
62+ void *data = (void *)(long)ctx->data;
63+ __u32 ifindex = GLOBAL_USER_IFINDEX;
64+ __u32 data_len = data_end - data;
65+ int retval = XDP_ABORTED; /* Fail */
66+ __u32 mtu_len = 0;
67+ int delta;
68+ int err;
69+
70+ /* Exceed MTU with 1 via delta adjust */
71+ delta = GLOBAL_USER_MTU - (data_len - ETH_HLEN) + 1;
72+
73+ err = bpf_check_mtu(ctx, ifindex, &mtu_len, delta, 0);
74+ if (err) {
75+ retval = XDP_PASS; /* Success in exceeding MTU check */
76+ if (err != BPF_MTU_CHK_RET_FRAG_NEEDED)
77+ retval = XDP_DROP;
78+ }
79+
80+ global_bpf_mtu_xdp = mtu_len;
81+ return retval;
82+}
83+
84+SEC("xdp")
85+int xdp_minus_delta(struct xdp_md *ctx)
86+{
87+ int retval = XDP_PASS; /* Expected retval on successful test */
88+ void *data_end = (void *)(long)ctx->data_end;
89+ void *data = (void *)(long)ctx->data;
90+ __u32 ifindex = GLOBAL_USER_IFINDEX;
91+ __u32 data_len = data_end - data;
92+ __u32 mtu_len = 0;
93+ int delta;
94+
95+ /* Borderline test case: Minus delta exceeding packet length allowed */
96+ delta = -((data_len - ETH_HLEN) + 1);
97+
98+ /* Minus length (adjusted via delta) still pass MTU check, other helpers
99+ * are responsible for catching this, when doing actual size adjust
100+ */
101+ if (bpf_check_mtu(ctx, ifindex, &mtu_len, delta, 0))
102+ retval = XDP_ABORTED;
103+
104+ global_bpf_mtu_xdp = mtu_len;
105+ return retval;
106+}
107+
108+SEC("classifier")
109+int tc_use_helper(struct __sk_buff *ctx)
110+{
111+ int retval = BPF_OK; /* Expected retval on successful test */
112+ __u32 mtu_len = 0;
113+ int delta = 0;
114+
115+ if (bpf_check_mtu(ctx, 0, &mtu_len, delta, 0)) {
116+ retval = BPF_DROP;
117+ goto out;
118+ }
119+
120+ if (mtu_len != GLOBAL_USER_MTU)
121+ retval = BPF_REDIRECT;
122+out:
123+ global_bpf_mtu_tc = mtu_len;
124+ return retval;
125+}
126+
127+SEC("classifier")
128+int tc_exceed_mtu(struct __sk_buff *ctx)
129+{
130+ __u32 ifindex = GLOBAL_USER_IFINDEX;
131+ int retval = BPF_DROP; /* Fail */
132+ __u32 skb_len = ctx->len;
133+ __u32 mtu_len = 0;
134+ int delta;
135+ int err;
136+
137+ /* Exceed MTU with 1 via delta adjust */
138+ delta = GLOBAL_USER_MTU - (skb_len - ETH_HLEN) + 1;
139+
140+ err = bpf_check_mtu(ctx, ifindex, &mtu_len, delta, 0);
141+ if (err) {
142+ retval = BPF_OK; /* Success in exceeding MTU check */
143+ if (err != BPF_MTU_CHK_RET_FRAG_NEEDED)
144+ retval = BPF_DROP;
145+ }
146+
147+ global_bpf_mtu_tc = mtu_len;
148+ return retval;
149+}
150+
151+SEC("classifier")
152+int tc_exceed_mtu_da(struct __sk_buff *ctx)
153+{
154+ /* SKB Direct-Access variant */
155+ void *data_end = (void *)(long)ctx->data_end;
156+ void *data = (void *)(long)ctx->data;
157+ __u32 ifindex = GLOBAL_USER_IFINDEX;
158+ __u32 data_len = data_end - data;
159+ int retval = BPF_DROP; /* Fail */
160+ __u32 mtu_len = 0;
161+ int delta;
162+ int err;
163+
164+ /* Exceed MTU with 1 via delta adjust */
165+ delta = GLOBAL_USER_MTU - (data_len - ETH_HLEN) + 1;
166+
167+ err = bpf_check_mtu(ctx, ifindex, &mtu_len, delta, 0);
168+ if (err) {
169+ retval = BPF_OK; /* Success in exceeding MTU check */
170+ if (err != BPF_MTU_CHK_RET_FRAG_NEEDED)
171+ retval = BPF_DROP;
172+ }
173+
174+ global_bpf_mtu_tc = mtu_len;
175+ return retval;
176+}
177+
178+SEC("classifier")
179+int tc_minus_delta(struct __sk_buff *ctx)
180+{
181+ int retval = BPF_OK; /* Expected retval on successful test */
182+ __u32 ifindex = GLOBAL_USER_IFINDEX;
183+ __u32 skb_len = ctx->len;
184+ __u32 mtu_len = 0;
185+ int delta;
186+
187+ /* Borderline test case: Minus delta exceeding packet length allowed */
188+ delta = -((skb_len - ETH_HLEN) + 1);
189+
190+ /* Minus length (adjusted via delta) still pass MTU check, other helpers
191+ * are responsible for catching this, when doing actual size adjust
192+ */
193+ if (bpf_check_mtu(ctx, ifindex, &mtu_len, delta, 0))
194+ retval = BPF_DROP;
195+
196+ global_bpf_mtu_xdp = mtu_len;
197+ return retval;
198+}
tools/testing/selftests/bpf/progs/test_cls_redirect.c+7−0
@@ -70,6 +70,7 @@ typedef struct { · 69 unmodified lines
7070 uint64_t errors_total_encap_adjust_failed;
7171 uint64_t errors_total_encap_buffer_too_small;
7272 uint64_t errors_total_redirect_loop;
73+ uint64_t errors_total_encap_mtu_violate;
7374 } metrics_t;
7475
7576 typedef enum {
@@ -407,6 +408,7 @@ static INLINING ret_t forward_with_gre(struct __sk_buff *skb, encap_headers_t *e · 331 unmodified lines
407408 payload_off - sizeof(struct ethhdr) - sizeof(struct iphdr);
408409 int32_t delta = sizeof(struct gre_base_hdr) - encap_overhead;
409410 uint16_t proto = ETH_P_IP;
411+ uint32_t mtu_len = 0;
410412
411413 /* Loop protection: the inner packet's TTL is decremented as a safeguard
412414 * against any forwarding loop. As the only interesting field is the TTL
@@ -479,6 +481,11 @@ static INLINING ret_t forward_with_gre(struct __sk_buff *skb, encap_headers_t *e · 66 unmodified lines
479481 }
480482 }
481483
484+ if (bpf_check_mtu(skb, skb->ifindex, &mtu_len, delta, 0)) {
485+ metrics->errors_total_encap_mtu_violate++;
486+ return TC_ACT_SHOT;
487+ }
488+
482489 if (bpf_skb_adjust_room(skb, delta, BPF_ADJ_ROOM_NET,
483490 BPF_F_ADJ_ROOM_FIXED_GSO |
484491 BPF_F_ADJ_ROOM_NO_CSUM_RESET) ||
tools/testing/selftests/bpf/progs/test_global_func10.c+29−0
@@ -0,0 +1,29 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+#include <stddef.h>
3+#include <linux/bpf.h>
4+#include <bpf/bpf_helpers.h>
5+
6+struct Small {
7+ int x;
8+};
9+
10+struct Big {
11+ int x;
12+ int y;
13+};
14+
15+__noinline int foo(const struct Big *big)
16+{
17+ if (big == 0)
18+ return 0;
19+
20+ return bpf_get_prandom_u32() < big->y;
21+}
22+
23+SEC("cgroup_skb/ingress")
24+int test_cls(struct __sk_buff *skb)
25+{
26+ const struct Small small = {.x = skb->len };
27+
28+ return foo((struct Big *)&small) ? 1 : 0;
29+}
tools/testing/selftests/bpf/progs/test_global_func11.c+19−0
@@ -0,0 +1,19 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+#include <stddef.h>
3+#include <linux/bpf.h>
4+#include <bpf/bpf_helpers.h>
5+
6+struct S {
7+ int x;
8+};
9+
10+__noinline int foo(const struct S *s)
11+{
12+ return s ? bpf_get_prandom_u32() < s->x : 0;
13+}
14+
15+SEC("cgroup_skb/ingress")
16+int test_cls(struct __sk_buff *skb)
17+{
18+ return foo(skb);
19+}
tools/testing/selftests/bpf/progs/test_global_func12.c+21−0
@@ -0,0 +1,21 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+#include <stddef.h>
3+#include <linux/bpf.h>
4+#include <bpf/bpf_helpers.h>
5+
6+struct S {
7+ int x;
8+};
9+
10+__noinline int foo(const struct S *s)
11+{
12+ return bpf_get_prandom_u32() < s->x;
13+}
14+
15+SEC("cgroup_skb/ingress")
16+int test_cls(struct __sk_buff *skb)
17+{
18+ const struct S s = {.x = skb->len };
19+
20+ return foo(&s);
21+}
tools/testing/selftests/bpf/progs/test_global_func13.c+24−0
@@ -0,0 +1,24 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+#include <stddef.h>
3+#include <linux/bpf.h>
4+#include <bpf/bpf_helpers.h>
5+
6+struct S {
7+ int x;
8+};
9+
10+__noinline int foo(const struct S *s)
11+{
12+ if (s)
13+ return bpf_get_prandom_u32() < s->x;
14+
15+ return 0;
16+}
17+
18+SEC("cgroup_skb/ingress")
19+int test_cls(struct __sk_buff *skb)
20+{
21+ const struct S *s = (const struct S *)(0xbedabeda);
22+
23+ return foo(s);
24+}
tools/testing/selftests/bpf/progs/test_global_func14.c+21−0
@@ -0,0 +1,21 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+#include <stddef.h>
3+#include <linux/bpf.h>
4+#include <bpf/bpf_helpers.h>
5+
6+struct S;
7+
8+__noinline int foo(const struct S *s)
9+{
10+ if (s)
11+ return bpf_get_prandom_u32() < *(const int *) s;
12+
13+ return 0;
14+}
15+
16+SEC("cgroup_skb/ingress")
17+int test_cls(struct __sk_buff *skb)
18+{
19+
20+ return foo(NULL);
21+}
tools/testing/selftests/bpf/progs/test_global_func15.c+22−0
@@ -0,0 +1,22 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+#include <stddef.h>
3+#include <linux/bpf.h>
4+#include <bpf/bpf_helpers.h>
5+
6+__noinline int foo(unsigned int *v)
7+{
8+ if (v)
9+ *v = bpf_get_prandom_u32();
10+
11+ return 0;
12+}
13+
14+SEC("cgroup_skb/ingress")
15+int test_cls(struct __sk_buff *skb)
16+{
17+ unsigned int v = 1;
18+
19+ foo(&v);
20+
21+ return v;
22+}
tools/testing/selftests/bpf/progs/test_global_func16.c+22−0
@@ -0,0 +1,22 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+#include <stddef.h>
3+#include <linux/bpf.h>
4+#include <bpf/bpf_helpers.h>
5+
6+__noinline int foo(int (*arr)[10])
7+{
8+ if (arr)
9+ return (*arr)[9];
10+
11+ return 0;
12+}
13+
14+SEC("cgroup_skb/ingress")
15+int test_cls(struct __sk_buff *skb)
16+{
17+ int array[10];
18+
19+ const int rv = foo(&array);
20+
21+ return rv ? 1 : 0;
22+}
tools/testing/selftests/bpf/progs/test_global_func9.c+132−0
@@ -0,0 +1,132 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+#include <stddef.h>
3+#include <linux/bpf.h>
4+#include <bpf/bpf_helpers.h>
5+
6+struct S {
7+ int x;
8+};
9+
10+struct C {
11+ int x;
12+ int y;
13+};
14+
15+struct {
16+ __uint(type, BPF_MAP_TYPE_ARRAY);
17+ __uint(max_entries, 1);
18+ __type(key, __u32);
19+ __type(value, struct S);
20+} map SEC(".maps");
21+
22+enum E {
23+ E_ITEM
24+};
25+
26+static int global_data_x = 100;
27+static int volatile global_data_y = 500;
28+
29+__noinline int foo(const struct S *s)
30+{
31+ if (s)
32+ return bpf_get_prandom_u32() < s->x;
33+
34+ return 0;
35+}
36+
37+__noinline int bar(int *x)
38+{
39+ if (x)
40+ *x &= bpf_get_prandom_u32();
41+
42+ return 0;
43+}
44+__noinline int baz(volatile int *x)
45+{
46+ if (x)
47+ *x &= bpf_get_prandom_u32();
48+
49+ return 0;
50+}
51+
52+__noinline int qux(enum E *e)
53+{
54+ if (e)
55+ return *e;
56+
57+ return 0;
58+}
59+
60+__noinline int quux(int (*arr)[10])
61+{
62+ if (arr)
63+ return (*arr)[9];
64+
65+ return 0;
66+}
67+
68+__noinline int quuz(int **p)
69+{
70+ if (p)
71+ *p = NULL;
72+
73+ return 0;
74+}
75+
76+SEC("cgroup_skb/ingress")
77+int test_cls(struct __sk_buff *skb)
78+{
79+ int result = 0;
80+
81+ {
82+ const struct S s = {.x = skb->len };
83+
84+ result |= foo(&s);
85+ }
86+
87+ {
88+ const __u32 key = 1;
89+ const struct S *s = bpf_map_lookup_elem(&map, &key);
90+
91+ result |= foo(s);
92+ }
93+
94+ {
95+ const struct C c = {.x = skb->len, .y = skb->family };
96+
97+ result |= foo((const struct S *)&c);
98+ }
99+
100+ {
101+ result |= foo(NULL);
102+ }
103+
104+ {
105+ bar(&result);
106+ bar(&global_data_x);
107+ }
108+
109+ {
110+ result |= baz(&global_data_y);
111+ }
112+
113+ {
114+ enum E e = E_ITEM;
115+
116+ result |= qux(&e);
117+ }
118+
119+ {
120+ int array[10] = {0};
121+
122+ result |= quux(&array);
123+ }
124+
125+ {
126+ int *p;
127+
128+ result |= quuz(&p);
129+ }
130+
131+ return result ? 1 : 0;
132+}
tools/testing/selftests/bpf/progs/test_global_func_args.c+91−0
@@ -0,0 +1,91 @@
1+// SPDX-License-Identifier: GPL-2.0
2+
3+#include <linux/bpf.h>
4+
5+#include <bpf/bpf_helpers.h>
6+
7+struct S {
8+ int v;
9+};
10+
11+static volatile struct S global_variable;
12+
13+struct {
14+ __uint(type, BPF_MAP_TYPE_ARRAY);
15+ __uint(max_entries, 7);
16+ __type(key, __u32);
17+ __type(value, int);
18+} values SEC(".maps");
19+
20+static void save_value(__u32 index, int value)
21+{
22+ bpf_map_update_elem(&values, &index, &value, 0);
23+}
24+
25+__noinline int foo(__u32 index, struct S *s)
26+{
27+ if (s) {
28+ save_value(index, s->v);
29+ return ++s->v;
30+ }
31+
32+ save_value(index, 0);
33+
34+ return 1;
35+}
36+
37+__noinline int bar(__u32 index, volatile struct S *s)
38+{
39+ if (s) {
40+ save_value(index, s->v);
41+ return ++s->v;
42+ }
43+
44+ save_value(index, 0);
45+
46+ return 1;
47+}
48+
49+__noinline int baz(struct S **s)
50+{
51+ if (s)
52+ *s = 0;
53+
54+ return 0;
55+}
56+
57+SEC("cgroup_skb/ingress")
58+int test_cls(struct __sk_buff *skb)
59+{
60+ __u32 index = 0;
61+
62+ {
63+ const int v = foo(index++, 0);
64+
65+ save_value(index++, v);
66+ }
67+
68+ {
69+ struct S s = { .v = 100 };
70+
71+ foo(index++, &s);
72+ save_value(index++, s.v);
73+ }
74+
75+ {
76+ global_variable.v = 42;
77+ bar(index++, &global_variable);
78+ save_value(index++, global_variable.v);
79+ }
80+
81+ {
82+ struct S v, *p = &v;
83+
84+ baz(&p);
85+ save_value(index++, !p);
86+ }
87+
88+ return 0;
89+}
90+
91+char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/test_module_attach.c+10−0
@@ -17,6 +17,16 @@ int BPF_PROG(handle_raw_tp, · 16 unmodified lines
1717 return 0;
1818 }
1919
20+__u32 raw_tp_bare_write_sz = 0;
21+
22+SEC("raw_tp/bpf_testmod_test_write_bare")
23+int BPF_PROG(handle_raw_tp_bare,
24+ struct task_struct *task, struct bpf_testmod_test_write_ctx *write_ctx)
25+{
26+ raw_tp_bare_write_sz = BPF_CORE_READ(write_ctx, len);
27+ return 0;
28+}
29+
2030 __u32 tp_btf_read_sz = 0;
2131
2232 SEC("tp_btf/bpf_testmod_test_read")
tools/testing/selftests/bpf/progs/test_ns_current_pid_tgid.c+9−19
@@ -5,31 +5,21 @@ · 4 unmodified lines
55 #include <stdint.h>
66 #include <bpf/bpf_helpers.h>
77
8−static volatile struct {
9− __u64 dev;
10− __u64 ino;
11− __u64 pid_tgid;
12− __u64 user_pid_tgid;
13−} res;
8+__u64 user_pid = 0;
9+__u64 user_tgid = 0;
10+__u64 dev = 0;
11+__u64 ino = 0;
1412
15−SEC("raw_tracepoint/sys_enter")
16−int trace(void *ctx)
13+SEC("tracepoint/syscalls/sys_enter_nanosleep")
14+int handler(const void *ctx)
1715 {
18− __u64 ns_pid_tgid, expected_pid;
1916 struct bpf_pidns_info nsdata;
20− __u32 key = 0;
2117
22− if (bpf_get_ns_current_pid_tgid(res.dev, res.ino, &nsdata,
23− sizeof(struct bpf_pidns_info)))
18+ if (bpf_get_ns_current_pid_tgid(dev, ino, &nsdata, sizeof(struct bpf_pidns_info)))
2419 return 0;
2520
26− ns_pid_tgid = (__u64)nsdata.tgid << 32 | nsdata.pid;
27− expected_pid = res.user_pid_tgid;
28−
29− if (expected_pid != ns_pid_tgid)
30− return 0;
31−
32− res.pid_tgid = ns_pid_tgid;
21+ user_pid = nsdata.pid;
22+ user_tgid = nsdata.tgid;
3323
3424 return 0;
3525 }
tools/testing/selftests/bpf/progs/test_stack_var_off.c+51−0
@@ -0,0 +1,51 @@
1+// SPDX-License-Identifier: GPL-2.0
2+
3+#include <linux/bpf.h>
4+#include <bpf/bpf_helpers.h>
5+
6+int probe_res;
7+
8+char input[4] = {};
9+int test_pid;
10+
11+SEC("tracepoint/syscalls/sys_enter_nanosleep")
12+int probe(void *ctx)
13+{
14+ /* This BPF program performs variable-offset reads and writes on a
15+ * stack-allocated buffer.
16+ */
17+ char stack_buf[16];
18+ unsigned long len;
19+ unsigned long last;
20+
21+ if ((bpf_get_current_pid_tgid() >> 32) != test_pid)
22+ return 0;
23+
24+ /* Copy the input to the stack. */
25+ __builtin_memcpy(stack_buf, input, 4);
26+
27+ /* The first byte in the buffer indicates the length. */
28+ len = stack_buf[0] & 0xf;
29+ last = (len - 1) & 0xf;
30+
31+ /* Append something to the buffer. The offset where we write is not
32+ * statically known; this is a variable-offset stack write.
33+ */
34+ stack_buf[len] = 42;
35+
36+ /* Index into the buffer at an unknown offset. This is a
37+ * variable-offset stack read.
38+ *
39+ * Note that if it wasn't for the preceding variable-offset write, this
40+ * read would be rejected because the stack slot cannot be verified as
41+ * being initialized. With the preceding variable-offset write, the
42+ * stack slot still cannot be verified, but the write inhibits the
43+ * respective check on the reasoning that, if there was a
44+ * variable-offset to a higher-or-equal spot, we're probably reading
45+ * what we just wrote.
46+ */
47+ probe_res = stack_buf[last];
48+ return 0;
49+}
50+
51+char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/test_current_pid_tgid_new_ns.c+0−160
@@ -1,160 +0,0 @@
1−// SPDX-License-Identifier: GPL-2.0
2−/* Copyright (c) 2020 Carlos Neira cneirabustos@gmail.com */
3−#define _GNU_SOURCE
4−#include <sys/stat.h>
5−#include <sys/types.h>
6−#include <unistd.h>
7−#include <sys/syscall.h>
8−#include <sched.h>
9−#include <sys/wait.h>
10−#include <sys/mount.h>
11−#include "test_progs.h"
12−
13−#define CHECK_NEWNS(condition, tag, format...) ({ \
14− int __ret = !!(condition); \
15− if (__ret) { \
16− printf("%s:FAIL:%s ", __func__, tag); \
17− printf(format); \
18− } else { \
19− printf("%s:PASS:%s\n", __func__, tag); \
20− } \
21− __ret; \
22−})
23−
24−struct bss {
25− __u64 dev;
26− __u64 ino;
27− __u64 pid_tgid;
28− __u64 user_pid_tgid;
29−};
30−
31−int main(int argc, char **argv)
32−{
33− pid_t pid;
34− int exit_code = 1;
35− struct stat st;
36−
37− printf("Testing bpf_get_ns_current_pid_tgid helper in new ns\n");
38−
39− if (stat("/proc/self/ns/pid", &st)) {
40− perror("stat failed on /proc/self/ns/pid ns\n");
41− printf("%s:FAILED\n", argv[0]);
42− return exit_code;
43− }
44−
45− if (CHECK_NEWNS(unshare(CLONE_NEWPID | CLONE_NEWNS),
46− "unshare CLONE_NEWPID | CLONE_NEWNS", "error errno=%d\n", errno))
47− return exit_code;
48−
49− pid = fork();
50− if (pid == -1) {
51− perror("Fork() failed\n");
52− printf("%s:FAILED\n", argv[0]);
53− return exit_code;
54− }
55−
56− if (pid > 0) {
57− int status;
58−
59− usleep(5);
60− waitpid(pid, &status, 0);
61− return 0;
62− } else {
63−
64− pid = fork();
65− if (pid == -1) {
66− perror("Fork() failed\n");
67− printf("%s:FAILED\n", argv[0]);
68− return exit_code;
69− }
70−
71− if (pid > 0) {
72− int status;
73− waitpid(pid, &status, 0);
74− return 0;
75− } else {
76− if (CHECK_NEWNS(mount("none", "/proc", NULL, MS_PRIVATE|MS_REC, NULL),
77− "Unmounting proc", "Cannot umount proc! errno=%d\n", errno))
78− return exit_code;
79−
80− if (CHECK_NEWNS(mount("proc", "/proc", "proc", MS_NOSUID|MS_NOEXEC|MS_NODEV, NULL),
81− "Mounting proc", "Cannot mount proc! errno=%d\n", errno))
82− return exit_code;
83−
84− const char *probe_name = "raw_tracepoint/sys_enter";
85− const char *file = "test_ns_current_pid_tgid.o";
86− struct bpf_link *link = NULL;
87− struct bpf_program *prog;
88− struct bpf_map *bss_map;
89− struct bpf_object *obj;
90− int exit_code = 1;
91− int err, key = 0;
92− struct bss bss;
93− struct stat st;
94− __u64 id;
95−
96− obj = bpf_object__open_file(file, NULL);
97− if (CHECK_NEWNS(IS_ERR(obj), "obj_open", "err %ld\n", PTR_ERR(obj)))
98− return exit_code;
99−
100− err = bpf_object__load(obj);
101− if (CHECK_NEWNS(err, "obj_load", "err %d errno %d\n", err, errno))
102− goto cleanup;
103−
104− bss_map = bpf_object__find_map_by_name(obj, "test_ns_.bss");
105− if (CHECK_NEWNS(!bss_map, "find_bss_map", "failed\n"))
106− goto cleanup;
107−
108− prog = bpf_object__find_program_by_title(obj, probe_name);
109− if (CHECK_NEWNS(!prog, "find_prog", "prog '%s' not found\n",
110− probe_name))
111− goto cleanup;
112−
113− memset(&bss, 0, sizeof(bss));
114− pid_t tid = syscall(SYS_gettid);
115− pid_t pid = getpid();
116−
117− id = (__u64) tid << 32 | pid;
118− bss.user_pid_tgid = id;
119−
120− if (CHECK_NEWNS(stat("/proc/self/ns/pid", &st),
121− "stat new ns", "Failed to stat /proc/self/ns/pid errno=%d\n", errno))
122− goto cleanup;
123−
124− bss.dev = st.st_dev;
125− bss.ino = st.st_ino;
126−
127− err = bpf_map_update_elem(bpf_map__fd(bss_map), &key, &bss, 0);
128− if (CHECK_NEWNS(err, "setting_bss", "failed to set bss : %d\n", err))
129− goto cleanup;
130−
131− link = bpf_program__attach_raw_tracepoint(prog, "sys_enter");
132− if (CHECK_NEWNS(IS_ERR(link), "attach_raw_tp", "err %ld\n",
133− PTR_ERR(link))) {
134− link = NULL;
135− goto cleanup;
136− }
137−
138− /* trigger some syscalls */
139− usleep(1);
140−
141− err = bpf_map_lookup_elem(bpf_map__fd(bss_map), &key, &bss);
142− if (CHECK_NEWNS(err, "set_bss", "failed to get bss : %d\n", err))
143− goto cleanup;
144−
145− if (CHECK_NEWNS(id != bss.pid_tgid, "Compare user pid/tgid vs. bpf pid/tgid",
146− "User pid/tgid %llu BPF pid/tgid %llu\n", id, bss.pid_tgid))
147− goto cleanup;
148−
149− exit_code = 0;
150− printf("%s:PASS\n", argv[0]);
151−cleanup:
152− if (!link) {
153− bpf_link__destroy(link);
154− link = NULL;
155− }
156− bpf_object__close(obj);
157− }
158− }
159− return 0;
160−}
tools/testing/selftests/bpf/test_flow_dissector.c+1−1
@@ -503,7 +503,7 @@ static int do_rx(int fd) · 502 unmodified lines
503503 if (rbuf != cfg_payload_char)
504504 error(1, 0, "recv: payload mismatch");
505505 num++;
506− };
506+ }
507507
508508 return num;
509509 }
tools/testing/selftests/bpf/test_progs.c+1−1
@@ -390,7 +390,7 @@ static void unload_bpf_testmod(void) · 389 unmodified lines
390390 return;
391391 }
392392 fprintf(env.stderr, "Failed to unload bpf_testmod.ko from kernel: %d\n", -errno);
393− exit(1);
393+ return;
394394 }
395395 if (env.verbosity > VERBOSE_NONE)
396396 fprintf(stdout, "Successfully unloaded bpf_testmod.ko.\n");
tools/testing/selftests/bpf/test_progs.h+0−1
@@ -16,7 +16,6 @@ typedef __u16 __sum16; · 15 unmodified lines
1616 #include <linux/if_packet.h>
1717 #include <linux/ip.h>
1818 #include <linux/ipv6.h>
19−#include <netinet/tcp.h>
2019 #include <linux/filter.h>
2120 #include <linux/perf_event.h>
2221 #include <linux/socket.h>
tools/testing/selftests/bpf/test_sock_addr.c+12−74
@@ -31,6 +31,8 @@ · 30 unmodified lines
3131 #define CONNECT6_PROG_PATH "./connect6_prog.o"
3232 #define SENDMSG4_PROG_PATH "./sendmsg4_prog.o"
3333 #define SENDMSG6_PROG_PATH "./sendmsg6_prog.o"
34+#define RECVMSG4_PROG_PATH "./recvmsg4_prog.o"
35+#define RECVMSG6_PROG_PATH "./recvmsg6_prog.o"
3436 #define BIND4_PROG_PATH "./bind4_prog.o"
3537 #define BIND6_PROG_PATH "./bind6_prog.o"
3638
@@ -94,10 +96,10 @@ static int sendmsg_deny_prog_load(const struct sock_addr_test *test); · 57 unmodified lines
9496 static int recvmsg_allow_prog_load(const struct sock_addr_test *test);
9597 static int recvmsg_deny_prog_load(const struct sock_addr_test *test);
9698 static int sendmsg4_rw_asm_prog_load(const struct sock_addr_test *test);
97−static int recvmsg4_rw_asm_prog_load(const struct sock_addr_test *test);
99+static int recvmsg4_rw_c_prog_load(const struct sock_addr_test *test);
98100 static int sendmsg4_rw_c_prog_load(const struct sock_addr_test *test);
99101 static int sendmsg6_rw_asm_prog_load(const struct sock_addr_test *test);
100−static int recvmsg6_rw_asm_prog_load(const struct sock_addr_test *test);
102+static int recvmsg6_rw_c_prog_load(const struct sock_addr_test *test);
101103 static int sendmsg6_rw_c_prog_load(const struct sock_addr_test *test);
102104 static int sendmsg6_rw_v4mapped_prog_load(const struct sock_addr_test *test);
103105 static int sendmsg6_rw_wildcard_prog_load(const struct sock_addr_test *test);
@@ -573,8 +575,8 @@ static struct sock_addr_test tests[] = { · 469 unmodified lines
573575 LOAD_REJECT,
574576 },
575577 {
576− "recvmsg4: rewrite IP & port (asm)",
577− recvmsg4_rw_asm_prog_load,
578+ "recvmsg4: rewrite IP & port (C)",
579+ recvmsg4_rw_c_prog_load,
578580 BPF_CGROUP_UDP4_RECVMSG,
579581 BPF_CGROUP_UDP4_RECVMSG,
580582 AF_INET,
@@ -587,8 +589,8 @@ static struct sock_addr_test tests[] = { · 6 unmodified lines
587589 SUCCESS,
588590 },
589591 {
590− "recvmsg6: rewrite IP & port (asm)",
591− recvmsg6_rw_asm_prog_load,
592+ "recvmsg6: rewrite IP & port (C)",
593+ recvmsg6_rw_c_prog_load,
592594 BPF_CGROUP_UDP6_RECVMSG,
593595 BPF_CGROUP_UDP6_RECVMSG,
594596 AF_INET6,
@@ -786,45 +788,9 @@ static int sendmsg4_rw_asm_prog_load(const struct sock_addr_test *test) · 191 unmodified lines
786788 return load_insns(test, insns, sizeof(insns) / sizeof(struct bpf_insn));
787789 }
788790
789−static int recvmsg4_rw_asm_prog_load(const struct sock_addr_test *test)
791+static int recvmsg4_rw_c_prog_load(const struct sock_addr_test *test)
790792 {
791− struct sockaddr_in src4_rw_addr;
792−
793− if (mk_sockaddr(AF_INET, SERV4_IP, SERV4_PORT,
794− (struct sockaddr *)&src4_rw_addr,
795− sizeof(src4_rw_addr)) == -1)
796− return -1;
797−
798− struct bpf_insn insns[] = {
799− BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
800−
801− /* if (sk.family == AF_INET && */
802− BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_6,
803− offsetof(struct bpf_sock_addr, family)),
804− BPF_JMP_IMM(BPF_JNE, BPF_REG_7, AF_INET, 6),
805−
806− /* sk.type == SOCK_DGRAM) { */
807− BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_6,
808− offsetof(struct bpf_sock_addr, type)),
809− BPF_JMP_IMM(BPF_JNE, BPF_REG_7, SOCK_DGRAM, 4),
810−
811− /* user_ip4 = src4_rw_addr.sin_addr */
812− BPF_MOV32_IMM(BPF_REG_7, src4_rw_addr.sin_addr.s_addr),
813− BPF_STX_MEM(BPF_W, BPF_REG_6, BPF_REG_7,
814− offsetof(struct bpf_sock_addr, user_ip4)),
815−
816− /* user_port = src4_rw_addr.sin_port */
817− BPF_MOV32_IMM(BPF_REG_7, src4_rw_addr.sin_port),
818− BPF_STX_MEM(BPF_W, BPF_REG_6, BPF_REG_7,
819− offsetof(struct bpf_sock_addr, user_port)),
820− /* } */
821−
822− /* return 1 */
823− BPF_MOV64_IMM(BPF_REG_0, 1),
824− BPF_EXIT_INSN(),
825− };
826−
827− return load_insns(test, insns, sizeof(insns) / sizeof(struct bpf_insn));
793+ return load_path(test, RECVMSG4_PROG_PATH);
828794 }
829795
830796 static int sendmsg4_rw_c_prog_load(const struct sock_addr_test *test)
@@ -890,37 +856,9 @@ static int sendmsg6_rw_asm_prog_load(const struct sock_addr_test *test) · 59 unmodified lines
890856 return sendmsg6_rw_dst_asm_prog_load(test, SERV6_REWRITE_IP);
891857 }
892858
893−static int recvmsg6_rw_asm_prog_load(const struct sock_addr_test *test)
859+static int recvmsg6_rw_c_prog_load(const struct sock_addr_test *test)
894860 {
895− struct sockaddr_in6 src6_rw_addr;
896−
897− if (mk_sockaddr(AF_INET6, SERV6_IP, SERV6_PORT,
898− (struct sockaddr *)&src6_rw_addr,
899− sizeof(src6_rw_addr)) == -1)
900− return -1;
901−
902− struct bpf_insn insns[] = {
903− BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
904−
905− /* if (sk.family == AF_INET6) { */
906− BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_6,
907− offsetof(struct bpf_sock_addr, family)),
908− BPF_JMP_IMM(BPF_JNE, BPF_REG_7, AF_INET6, 10),
909−
910− STORE_IPV6(user_ip6, src6_rw_addr.sin6_addr.s6_addr32),
911−
912− /* user_port = dst6_rw_addr.sin6_port */
913− BPF_MOV32_IMM(BPF_REG_7, src6_rw_addr.sin6_port),
914− BPF_STX_MEM(BPF_W, BPF_REG_6, BPF_REG_7,
915− offsetof(struct bpf_sock_addr, user_port)),
916− /* } */
917−
918− /* return 1 */
919− BPF_MOV64_IMM(BPF_REG_0, 1),
920− BPF_EXIT_INSN(),
921− };
922−
923− return load_insns(test, insns, sizeof(insns) / sizeof(struct bpf_insn));
861+ return load_path(test, RECVMSG6_PROG_PATH);
924862 }
925863
926864 static int sendmsg6_rw_v4mapped_prog_load(const struct sock_addr_test *test)
tools/testing/selftests/bpf/test_socket_cookie.c+0−208
@@ -1,208 +0,0 @@
1−// SPDX-License-Identifier: GPL-2.0
2−// Copyright (c) 2018 Facebook
3−
4−#include <string.h>
5−#include <unistd.h>
6−
7−#include <arpa/inet.h>
8−#include <netinet/in.h>
9−#include <sys/types.h>
10−#include <sys/socket.h>
11−
12−#include <bpf/bpf.h>
13−#include <bpf/libbpf.h>
14−
15−#include "bpf_rlimit.h"
16−#include "cgroup_helpers.h"
17−
18−#define CG_PATH "/foo"
19−#define SOCKET_COOKIE_PROG "./socket_cookie_prog.o"
20−
21−struct socket_cookie {
22− __u64 cookie_key;
23− __u32 cookie_value;
24−};
25−
26−static int start_server(void)
27−{
28− struct sockaddr_in6 addr;
29− int fd;
30−
31− fd = socket(AF_INET6, SOCK_STREAM, 0);
32− if (fd == -1) {
33− log_err("Failed to create server socket");
34− goto out;
35− }
36−
37− memset(&addr, 0, sizeof(addr));
38− addr.sin6_family = AF_INET6;
39− addr.sin6_addr = in6addr_loopback;
40− addr.sin6_port = 0;
41−
42− if (bind(fd, (const struct sockaddr *)&addr, sizeof(addr)) == -1) {
43− log_err("Failed to bind server socket");
44− goto close_out;
45− }
46−
47− if (listen(fd, 128) == -1) {
48− log_err("Failed to listen on server socket");
49− goto close_out;
50− }
51−
52− goto out;
53−
54−close_out:
55− close(fd);
56− fd = -1;
57−out:
58− return fd;
59−}
60−
61−static int connect_to_server(int server_fd)
62−{
63− struct sockaddr_storage addr;
64− socklen_t len = sizeof(addr);
65− int fd;
66−
67− fd = socket(AF_INET6, SOCK_STREAM, 0);
68− if (fd == -1) {
69− log_err("Failed to create client socket");
70− goto out;
71− }
72−
73− if (getsockname(server_fd, (struct sockaddr *)&addr, &len)) {
74− log_err("Failed to get server addr");
75− goto close_out;
76− }
77−
78− if (connect(fd, (const struct sockaddr *)&addr, len) == -1) {
79− log_err("Fail to connect to server");
80− goto close_out;
81− }
82−
83− goto out;
84−
85−close_out:
86− close(fd);
87− fd = -1;
88−out:
89− return fd;
90−}
91−
92−static int validate_map(struct bpf_map *map, int client_fd)
93−{
94− __u32 cookie_expected_value;
95− struct sockaddr_in6 addr;
96− socklen_t len = sizeof(addr);
97− struct socket_cookie val;
98− int err = 0;
99− int map_fd;
100−
101− if (!map) {
102− log_err("Map not found in BPF object");
103− goto err;
104− }
105−
106− map_fd = bpf_map__fd(map);
107−
108− err = bpf_map_lookup_elem(map_fd, &client_fd, &val);
109−
110− err = getsockname(client_fd, (struct sockaddr *)&addr, &len);
111− if (err) {
112− log_err("Can't get client local addr");
113− goto out;
114− }
115−
116− cookie_expected_value = (ntohs(addr.sin6_port) << 8) | 0xFF;
117− if (val.cookie_value != cookie_expected_value) {
118− log_err("Unexpected value in map: %x != %x", val.cookie_value,
119− cookie_expected_value);
120− goto err;
121− }
122−
123− goto out;
124−err:
125− err = -1;
126−out:
127− return err;
128−}
129−
130−static int run_test(int cgfd)
131−{
132− enum bpf_attach_type attach_type;
133− struct bpf_prog_load_attr attr;
134− struct bpf_program *prog;
135− struct bpf_object *pobj;
136− const char *prog_name;
137− int server_fd = -1;
138− int client_fd = -1;
139− int prog_fd = -1;
140− int err = 0;
141−
142− memset(&attr, 0, sizeof(attr));
143− attr.file = SOCKET_COOKIE_PROG;
144− attr.prog_type = BPF_PROG_TYPE_UNSPEC;
145− attr.prog_flags = BPF_F_TEST_RND_HI32;
146−
147− err = bpf_prog_load_xattr(&attr, &pobj, &prog_fd);
148− if (err) {
149− log_err("Failed to load %s", attr.file);
150− goto out;
151− }
152−
153− bpf_object__for_each_program(prog, pobj) {
154− prog_name = bpf_program__section_name(prog);
155−
156− if (libbpf_attach_type_by_name(prog_name, &attach_type))
157− goto err;
158−
159− err = bpf_prog_attach(bpf_program__fd(prog), cgfd, attach_type,
160− BPF_F_ALLOW_OVERRIDE);
161− if (err) {
162− log_err("Failed to attach prog %s", prog_name);
163− goto out;
164− }
165− }
166−
167− server_fd = start_server();
168− if (server_fd == -1)
169− goto err;
170−
171− client_fd = connect_to_server(server_fd);
172− if (client_fd == -1)
173− goto err;
174−
175− if (validate_map(bpf_map__next(NULL, pobj), client_fd))
176− goto err;
177−
178− goto out;
179−err:
180− err = -1;
181−out:
182− close(client_fd);
183− close(server_fd);
184− bpf_object__close(pobj);
185− printf("%s\n", err ? "FAILED" : "PASSED");
186− return err;
187−}
188−
189−int main(int argc, char **argv)
190−{
191− int cgfd = -1;
192− int err = 0;
193−
194− cgfd = cgroup_setup_and_join(CG_PATH);
195− if (cgfd < 0)
196− goto err;
197−
198− if (run_test(cgfd))
199− goto err;
200−
201− goto out;
202−err:
203− err = -1;
204−out:
205− close(cgfd);
206− cleanup_cgroup_environment();
207− return err;
208−}
tools/testing/selftests/bpf/test_verifier.c+85−3
@@ -88,6 +88,10 @@ struct bpf_test { · 87 unmodified lines
8888 int fixup_map_event_output[MAX_FIXUPS];
8989 int fixup_map_reuseport_array[MAX_FIXUPS];
9090 int fixup_map_ringbuf[MAX_FIXUPS];
91+ /* Expected verifier log output for result REJECT or VERBOSE_ACCEPT.
92+ * Can be a tab-separated sequence of expected strings. An empty string
93+ * means no log verification.
94+ */
9195 const char *errstr;
9296 const char *errstr_unpriv;
9397 uint32_t insn_processed;
@@ -297,6 +301,78 @@ static void bpf_fill_scale(struct bpf_test *self) · 203 unmodified lines
297301 }
298302 }
299303
304+static int bpf_fill_torturous_jumps_insn_1(struct bpf_insn *insn)
305+{
306+ unsigned int len = 259, hlen = 128;
307+ int i;
308+
309+ insn[0] = BPF_EMIT_CALL(BPF_FUNC_get_prandom_u32);
310+ for (i = 1; i <= hlen; i++) {
311+ insn[i] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, i, hlen);
312+ insn[i + hlen] = BPF_JMP_A(hlen - i);
313+ }
314+ insn[len - 2] = BPF_MOV64_IMM(BPF_REG_0, 1);
315+ insn[len - 1] = BPF_EXIT_INSN();
316+
317+ return len;
318+}
319+
320+static int bpf_fill_torturous_jumps_insn_2(struct bpf_insn *insn)
321+{
322+ unsigned int len = 4100, jmp_off = 2048;
323+ int i, j;
324+
325+ insn[0] = BPF_EMIT_CALL(BPF_FUNC_get_prandom_u32);
326+ for (i = 1; i <= jmp_off; i++) {
327+ insn[i] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, i, jmp_off);
328+ }
329+ insn[i++] = BPF_JMP_A(jmp_off);
330+ for (; i <= jmp_off * 2 + 1; i+=16) {
331+ for (j = 0; j < 16; j++) {
332+ insn[i + j] = BPF_JMP_A(16 - j - 1);
333+ }
334+ }
335+
336+ insn[len - 2] = BPF_MOV64_IMM(BPF_REG_0, 2);
337+ insn[len - 1] = BPF_EXIT_INSN();
338+
339+ return len;
340+}
341+
342+static void bpf_fill_torturous_jumps(struct bpf_test *self)
343+{
344+ struct bpf_insn *insn = self->fill_insns;
345+ int i = 0;
346+
347+ switch (self->retval) {
348+ case 1:
349+ self->prog_len = bpf_fill_torturous_jumps_insn_1(insn);
350+ return;
351+ case 2:
352+ self->prog_len = bpf_fill_torturous_jumps_insn_2(insn);
353+ return;
354+ case 3:
355+ /* main */
356+ insn[i++] = BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 4);
357+ insn[i++] = BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 262);
358+ insn[i++] = BPF_ST_MEM(BPF_B, BPF_REG_10, -32, 0);
359+ insn[i++] = BPF_MOV64_IMM(BPF_REG_0, 3);
360+ insn[i++] = BPF_EXIT_INSN();
361+
362+ /* subprog 1 */
363+ i += bpf_fill_torturous_jumps_insn_1(insn + i);
364+
365+ /* subprog 2 */
366+ i += bpf_fill_torturous_jumps_insn_2(insn + i);
367+
368+ self->prog_len = i;
369+ return;
370+ default:
371+ self->prog_len = 0;
372+ break;
373+ }
374+}
375+
300376 /* BPF_SK_LOOKUP contains 13 instructions, if you need to fix up maps */
301377 #define BPF_SK_LOOKUP(func) \
302378 /* struct bpf_sock_tuple tuple = {} */ \
@@ -923,13 +999,19 @@ static int do_prog_test_run(int fd_prog, bool unpriv, uint32_t expected_val, · 620 unmodified lines
923999 return 0;
9241000 }
9251001
1002+/* Returns true if every part of exp (tab-separated) appears in log, in order.
1003+ *
1004+ * If exp is an empty string, returns true.
1005+ */
9261006 static bool cmp_str_seq(const char *log, const char *exp)
9271007 {
928− char needle[80];
1008+ char needle[200];
9291009 const char *p, *q;
9301010 int len;
9311011
9321012 do {
1013+ if (!strlen(exp))
1014+ break;
9331015 p = strchr(exp, '\t');
9341016 if (!p)
9351017 p = exp + strlen(exp);
@@ -943,7 +1025,7 @@ static bool cmp_str_seq(const char *log, const char *exp) · 7 unmodified lines
9431025 needle[len] = 0;
9441026 q = strstr(log, needle);
9451027 if (!q) {
946− printf("FAIL\nUnexpected verifier log in successful load!\n"
1028+ printf("FAIL\nUnexpected verifier log!\n"
9471029 "EXP: %s\nRES:\n", needle);
9481030 return false;
9491031 }
@@ -1058,7 +1140,7 @@ static void do_test_single(struct bpf_test *test, bool unpriv, · 108 unmodified lines
10581140 printf("FAIL\nUnexpected success to load!\n");
10591141 goto fail_log;
10601142 }
1061− if (!expected_err || !strstr(bpf_vlog, expected_err)) {
1143+ if (!expected_err || !cmp_str_seq(bpf_vlog, expected_err)) {
10621144 printf("FAIL\nUnexpected error message!\n\tEXP: %s\n\tRES: %s\n",
10631145 expected_err, bpf_vlog);
10641146 goto fail_log;
tools/testing/selftests/bpf/verifier/atomic_and.c+1−1
@@ -7,7 +7,7 @@ · 6 unmodified lines
77 BPF_MOV64_IMM(BPF_REG_1, 0x011),
88 BPF_ATOMIC_OP(BPF_DW, BPF_AND, BPF_REG_10, BPF_REG_1, -8),
99 /* if (val != 0x010) exit(2); */
10− BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_10, -8),
10+ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_10, -8),
1111 BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0x010, 2),
1212 BPF_MOV64_IMM(BPF_REG_0, 2),
1313 BPF_EXIT_INSN(),
tools/testing/selftests/bpf/verifier/atomic_bounds.c+27−0
@@ -0,0 +1,27 @@
1+{
2+ "BPF_ATOMIC bounds propagation, mem->reg",
3+ .insns = {
4+ /* a = 0; */
5+ /*
6+ * Note this is implemented with two separate instructions,
7+ * where you might think one would suffice:
8+ *
9+ * BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
10+ *
11+ * This is because BPF_ST_MEM doesn't seem to set the stack slot
12+ * type to 0 when storing an immediate.
13+ */
14+ BPF_MOV64_IMM(BPF_REG_0, 0),
15+ BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, -8),
16+ /* b = atomic_fetch_add(&a, 1); */
17+ BPF_MOV64_IMM(BPF_REG_1, 1),
18+ BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, BPF_REG_10, BPF_REG_1, -8),
19+ /* Verifier should be able to tell that this infinite loop isn't reachable. */
20+ /* if (b) while (true) continue; */
21+ BPF_JMP_IMM(BPF_JNE, BPF_REG_1, 0, -1),
22+ BPF_EXIT_INSN(),
23+ },
24+ .result = ACCEPT,
25+ .result_unpriv = REJECT,
26+ .errstr_unpriv = "back-edge",
27+},
tools/testing/selftests/bpf/verifier/atomic_or.c+1−1
@@ -7,7 +7,7 @@ · 6 unmodified lines
77 BPF_MOV64_IMM(BPF_REG_1, 0x011),
88 BPF_ATOMIC_OP(BPF_DW, BPF_OR, BPF_REG_10, BPF_REG_1, -8),
99 /* if (val != 0x111) exit(2); */
10− BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_10, -8),
10+ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_10, -8),
1111 BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0x111, 2),
1212 BPF_MOV64_IMM(BPF_REG_0, 2),
1313 BPF_EXIT_INSN(),
tools/testing/selftests/bpf/verifier/atomic_xor.c+1−1
@@ -7,7 +7,7 @@ · 6 unmodified lines
77 BPF_MOV64_IMM(BPF_REG_1, 0x011),
88 BPF_ATOMIC_OP(BPF_DW, BPF_XOR, BPF_REG_10, BPF_REG_1, -8),
99 /* if (val != 0x101) exit(2); */
10− BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_10, -8),
10+ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_10, -8),
1111 BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0x101, 2),
1212 BPF_MOV64_IMM(BPF_REG_0, 2),
1313 BPF_EXIT_INSN(),
tools/testing/selftests/bpf/verifier/basic_stack.c+1−1
@@ -4,7 +4,7 @@ · 3 unmodified lines
44 BPF_ST_MEM(BPF_DW, BPF_REG_10, 8, 0),
55 BPF_EXIT_INSN(),
66 },
7− .errstr = "invalid stack",
7+ .errstr = "invalid write to stack",
88 .result = REJECT,
99 },
1010 {
tools/testing/selftests/bpf/verifier/calls.c+2−2
@@ -1228,7 +1228,7 @@ · 1227 unmodified lines
12281228 .prog_type = BPF_PROG_TYPE_XDP,
12291229 .fixup_map_hash_8b = { 23 },
12301230 .result = REJECT,
1231− .errstr = "invalid read from stack off -16+0 size 8",
1231+ .errstr = "invalid read from stack R7 off=-16 size=8",
12321232 },
12331233 {
12341234 "calls: two calls that receive map_value via arg=ptr_stack_of_caller. test1",
@@ -1958,7 +1958,7 @@ · 723 unmodified lines
19581958 BPF_EXIT_INSN(),
19591959 },
19601960 .fixup_map_hash_48b = { 6 },
1961− .errstr = "invalid indirect read from stack off -8+0 size 8",
1961+ .errstr = "invalid indirect read from stack R2 off -8+0 size 8",
19621962 .result = REJECT,
19631963 .prog_type = BPF_PROG_TYPE_XDP,
19641964 },
tools/testing/selftests/bpf/verifier/const_or.c+2−2
@@ -23,7 +23,7 @@ · 22 unmodified lines
2323 BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel),
2424 BPF_EXIT_INSN(),
2525 },
26− .errstr = "invalid stack type R1 off=-48 access_size=58",
26+ .errstr = "invalid indirect access to stack R1 off=-48 size=58",
2727 .result = REJECT,
2828 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
2929 },
@@ -54,7 +54,7 @@ · 24 unmodified lines
5454 BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel),
5555 BPF_EXIT_INSN(),
5656 },
57− .errstr = "invalid stack type R1 off=-48 access_size=58",
57+ .errstr = "invalid indirect access to stack R1 off=-48 size=58",
5858 .result = REJECT,
5959 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
6060 },
tools/testing/selftests/bpf/verifier/helper_access_var_len.c+6−6
@@ -39,7 +39,7 @@ · 38 unmodified lines
3939 BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel),
4040 BPF_EXIT_INSN(),
4141 },
42− .errstr = "invalid indirect read from stack off -64+0 size 64",
42+ .errstr = "invalid indirect read from stack R1 off -64+0 size 64",
4343 .result = REJECT,
4444 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
4545 },
@@ -59,7 +59,7 @@ · 13 unmodified lines
5959 BPF_MOV64_IMM(BPF_REG_0, 0),
6060 BPF_EXIT_INSN(),
6161 },
62− .errstr = "invalid stack type R1 off=-64 access_size=65",
62+ .errstr = "invalid indirect access to stack R1 off=-64 size=65",
6363 .result = REJECT,
6464 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
6565 },
@@ -136,7 +136,7 @@ · 70 unmodified lines
136136 BPF_MOV64_IMM(BPF_REG_0, 0),
137137 BPF_EXIT_INSN(),
138138 },
139− .errstr = "invalid stack type R1 off=-64 access_size=65",
139+ .errstr = "invalid indirect access to stack R1 off=-64 size=65",
140140 .result = REJECT,
141141 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
142142 },
@@ -156,7 +156,7 @@ · 13 unmodified lines
156156 BPF_MOV64_IMM(BPF_REG_0, 0),
157157 BPF_EXIT_INSN(),
158158 },
159− .errstr = "invalid stack type R1 off=-64 access_size=65",
159+ .errstr = "invalid indirect access to stack R1 off=-64 size=65",
160160 .result = REJECT,
161161 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
162162 },
@@ -194,7 +194,7 @@ · 31 unmodified lines
194194 BPF_MOV64_IMM(BPF_REG_0, 0),
195195 BPF_EXIT_INSN(),
196196 },
197− .errstr = "invalid indirect read from stack off -64+0 size 64",
197+ .errstr = "invalid indirect read from stack R1 off -64+0 size 64",
198198 .result = REJECT,
199199 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
200200 },
@@ -584,7 +584,7 @@ · 383 unmodified lines
584584 BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, -16),
585585 BPF_EXIT_INSN(),
586586 },
587− .errstr = "invalid indirect read from stack off -64+32 size 64",
587+ .errstr = "invalid indirect read from stack R1 off -64+32 size 64",
588588 .result = REJECT,
589589 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
590590 },
tools/testing/selftests/bpf/verifier/int_ptr.c+3−3
@@ -27,7 +27,7 @@ · 26 unmodified lines
2727 },
2828 .result = REJECT,
2929 .prog_type = BPF_PROG_TYPE_CGROUP_SYSCTL,
30− .errstr = "invalid indirect read from stack off -16+0 size 8",
30+ .errstr = "invalid indirect read from stack R4 off -16+0 size 8",
3131 },
3232 {
3333 "ARG_PTR_TO_LONG half-uninitialized",
@@ -59,7 +59,7 @@ · 25 unmodified lines
5959 },
6060 .result = REJECT,
6161 .prog_type = BPF_PROG_TYPE_CGROUP_SYSCTL,
62− .errstr = "invalid indirect read from stack off -16+4 size 8",
62+ .errstr = "invalid indirect read from stack R4 off -16+4 size 8",
6363 },
6464 {
6565 "ARG_PTR_TO_LONG misaligned",
@@ -125,7 +125,7 @@ · 59 unmodified lines
125125 },
126126 .result = REJECT,
127127 .prog_type = BPF_PROG_TYPE_CGROUP_SYSCTL,
128− .errstr = "invalid stack type R4 off=-4 access_size=8",
128+ .errstr = "invalid indirect access to stack R4 off=-4 size=8",
129129 },
130130 {
131131 "ARG_PTR_TO_LONG initialized",
tools/testing/selftests/bpf/verifier/jit.c+24−0
@@ -105,3 +105,27 @@ · 104 unmodified lines
105105 .result = ACCEPT,
106106 .retval = 2,
107107 },
108+{
109+ "jit: torturous jumps, imm8 nop jmp and pure jump padding",
110+ .insns = { },
111+ .fill_helper = bpf_fill_torturous_jumps,
112+ .prog_type = BPF_PROG_TYPE_SCHED_CLS,
113+ .result = ACCEPT,
114+ .retval = 1,
115+},
116+{
117+ "jit: torturous jumps, imm32 nop jmp and jmp_cond padding",
118+ .insns = { },
119+ .fill_helper = bpf_fill_torturous_jumps,
120+ .prog_type = BPF_PROG_TYPE_SCHED_CLS,
121+ .result = ACCEPT,
122+ .retval = 2,
123+},
124+{
125+ "jit: torturous jumps in subprog",
126+ .insns = { },
127+ .fill_helper = bpf_fill_torturous_jumps,
128+ .prog_type = BPF_PROG_TYPE_SCHED_CLS,
129+ .result = ACCEPT,
130+ .retval = 3,
131+},
tools/testing/selftests/bpf/verifier/raw_stack.c+5−5
@@ -11,7 +11,7 @@ · 10 unmodified lines
1111 BPF_EXIT_INSN(),
1212 },
1313 .result = REJECT,
14− .errstr = "invalid read from stack off -8+0 size 8",
14+ .errstr = "invalid read from stack R6 off=-8 size=8",
1515 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
1616 },
1717 {
@@ -59,7 +59,7 @@ · 41 unmodified lines
5959 BPF_EXIT_INSN(),
6060 },
6161 .result = REJECT,
62− .errstr = "invalid stack type R3",
62+ .errstr = "invalid zero-sized read",
6363 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
6464 },
6565 {
@@ -205,7 +205,7 @@ · 139 unmodified lines
205205 BPF_EXIT_INSN(),
206206 },
207207 .result = REJECT,
208− .errstr = "invalid stack type R3 off=-513 access_size=8",
208+ .errstr = "invalid indirect access to stack R3 off=-513 size=8",
209209 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
210210 },
211211 {
@@ -221,7 +221,7 @@ · 9 unmodified lines
221221 BPF_EXIT_INSN(),
222222 },
223223 .result = REJECT,
224− .errstr = "invalid stack type R3 off=-1 access_size=8",
224+ .errstr = "invalid indirect access to stack R3 off=-1 size=8",
225225 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
226226 },
227227 {
@@ -285,7 +285,7 @@ · 57 unmodified lines
285285 BPF_EXIT_INSN(),
286286 },
287287 .result = REJECT,
288− .errstr = "invalid stack type R3 off=-512 access_size=0",
288+ .errstr = "invalid zero-sized read",
289289 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
290290 },
291291 {
tools/testing/selftests/bpf/verifier/stack_ptr.c+13−9
@@ -44,7 +44,7 @@ · 43 unmodified lines
4444 BPF_EXIT_INSN(),
4545 },
4646 .result = REJECT,
47− .errstr = "invalid stack off=-79992 size=8",
47+ .errstr = "invalid write to stack R1 off=-79992 size=8",
4848 .errstr_unpriv = "R1 stack pointer arithmetic goes out of range",
4949 },
5050 {
@@ -57,7 +57,7 @@ · 6 unmodified lines
5757 BPF_EXIT_INSN(),
5858 },
5959 .result = REJECT,
60− .errstr = "invalid stack off=0 size=8",
60+ .errstr = "invalid write to stack R1 off=0 size=8",
6161 },
6262 {
6363 "PTR_TO_STACK check high 1",
@@ -106,7 +106,7 @@ · 42 unmodified lines
106106 BPF_EXIT_INSN(),
107107 },
108108 .errstr_unpriv = "R1 stack pointer arithmetic goes out of range",
109− .errstr = "invalid stack off=0 size=1",
109+ .errstr = "invalid write to stack R1 off=0 size=1",
110110 .result = REJECT,
111111 },
112112 {
@@ -119,7 +119,8 @@ · 6 unmodified lines
119119 BPF_EXIT_INSN(),
120120 },
121121 .result = REJECT,
122− .errstr = "invalid stack off",
122+ .errstr_unpriv = "R1 stack pointer arithmetic goes out of range",
123+ .errstr = "invalid write to stack R1",
123124 },
124125 {
125126 "PTR_TO_STACK check high 6",
@@ -131,7 +132,8 @@ · 5 unmodified lines
131132 BPF_EXIT_INSN(),
132133 },
133134 .result = REJECT,
134− .errstr = "invalid stack off",
135+ .errstr_unpriv = "R1 stack pointer arithmetic goes out of range",
136+ .errstr = "invalid write to stack",
135137 },
136138 {
137139 "PTR_TO_STACK check high 7",
@@ -183,7 +185,7 @@ · 45 unmodified lines
183185 BPF_EXIT_INSN(),
184186 },
185187 .errstr_unpriv = "R1 stack pointer arithmetic goes out of range",
186− .errstr = "invalid stack off=-513 size=1",
188+ .errstr = "invalid write to stack R1 off=-513 size=1",
187189 .result = REJECT,
188190 },
189191 {
@@ -208,7 +210,8 @@ · 18 unmodified lines
208210 BPF_EXIT_INSN(),
209211 },
210212 .result = REJECT,
211− .errstr = "invalid stack off",
213+ .errstr_unpriv = "R1 stack pointer arithmetic goes out of range",
214+ .errstr = "invalid write to stack",
212215 },
213216 {
214217 "PTR_TO_STACK check low 6",
@@ -220,7 +223,8 @@ · 5 unmodified lines
220223 BPF_EXIT_INSN(),
221224 },
222225 .result = REJECT,
223− .errstr = "invalid stack off",
226+ .errstr = "invalid write to stack",
227+ .errstr_unpriv = "R1 stack pointer arithmetic goes out of range",
224228 },
225229 {
226230 "PTR_TO_STACK check low 7",
@@ -292,7 +296,7 @@ · 65 unmodified lines
292296 BPF_EXIT_INSN(),
293297 },
294298 .result_unpriv = REJECT,
295− .errstr_unpriv = "invalid stack off=0 size=1",
299+ .errstr_unpriv = "invalid write to stack R1 off=0 size=1",
296300 .result = ACCEPT,
297301 .retval = 42,
298302 },
tools/testing/selftests/bpf/verifier/unpriv.c+1−1
@@ -108,7 +108,7 @@ · 107 unmodified lines
108108 BPF_EXIT_INSN(),
109109 },
110110 .fixup_map_hash_8b = { 3 },
111− .errstr_unpriv = "invalid indirect read from stack off -8+0 size 8",
111+ .errstr_unpriv = "invalid indirect read from stack R2 off -8+0 size 8",
112112 .result_unpriv = REJECT,
113113 .result = ACCEPT,
114114 },
tools/testing/selftests/bpf/verifier/var_off.c+105−10
@@ -18,7 +18,7 @@ · 17 unmodified lines
1818 .prog_type = BPF_PROG_TYPE_LWT_IN,
1919 },
2020 {
21− "variable-offset stack access",
21+ "variable-offset stack read, priv vs unpriv",
2222 .insns = {
2323 /* Fill the top 8 bytes of the stack */
2424 BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
@@ -31,14 +31,109 @@ · 6 unmodified lines
3131 * we don't know which
3232 */
3333 BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_10),
34− /* dereference it */
34+ /* dereference it for a stack read */
35+ BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_2, 0),
36+ BPF_MOV64_IMM(BPF_REG_0, 0),
37+ BPF_EXIT_INSN(),
38+ },
39+ .result = ACCEPT,
40+ .result_unpriv = REJECT,
41+ .errstr_unpriv = "R2 variable stack access prohibited for !root",
42+ .prog_type = BPF_PROG_TYPE_CGROUP_SKB,
43+},
44+{
45+ "variable-offset stack read, uninitialized",
46+ .insns = {
47+ /* Get an unknown value */
48+ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 0),
49+ /* Make it small and 4-byte aligned */
50+ BPF_ALU64_IMM(BPF_AND, BPF_REG_2, 4),
51+ BPF_ALU64_IMM(BPF_SUB, BPF_REG_2, 8),
52+ /* add it to fp. We now have either fp-4 or fp-8, but
53+ * we don't know which
54+ */
55+ BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_10),
56+ /* dereference it for a stack read */
3557 BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_2, 0),
58+ BPF_MOV64_IMM(BPF_REG_0, 0),
3659 BPF_EXIT_INSN(),
3760 },
38− .errstr = "variable stack access var_off=(0xfffffffffffffff8; 0x4)",
3961 .result = REJECT,
62+ .errstr = "invalid variable-offset read from stack R2",
4063 .prog_type = BPF_PROG_TYPE_LWT_IN,
4164 },
65+{
66+ "variable-offset stack write, priv vs unpriv",
67+ .insns = {
68+ /* Get an unknown value */
69+ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 0),
70+ /* Make it small and 8-byte aligned */
71+ BPF_ALU64_IMM(BPF_AND, BPF_REG_2, 8),
72+ BPF_ALU64_IMM(BPF_SUB, BPF_REG_2, 16),
73+ /* Add it to fp. We now have either fp-8 or fp-16, but
74+ * we don't know which
75+ */
76+ BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_10),
77+ /* Dereference it for a stack write */
78+ BPF_ST_MEM(BPF_DW, BPF_REG_2, 0, 0),
79+ /* Now read from the address we just wrote. This shows
80+ * that, after a variable-offset write, a priviledged
81+ * program can read the slots that were in the range of
82+ * that write (even if the verifier doesn't actually know
83+ * if the slot being read was really written to or not.
84+ */
85+ BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_2, 0),
86+ BPF_MOV64_IMM(BPF_REG_0, 0),
87+ BPF_EXIT_INSN(),
88+ },
89+ /* Variable stack access is rejected for unprivileged.
90+ */
91+ .errstr_unpriv = "R2 variable stack access prohibited for !root",
92+ .result_unpriv = REJECT,
93+ .result = ACCEPT,
94+},
95+{
96+ "variable-offset stack write clobbers spilled regs",
97+ .insns = {
98+ /* Dummy instruction; needed because we need to patch the next one
99+ * and we can't patch the first instruction.
100+ */
101+ BPF_MOV64_IMM(BPF_REG_6, 0),
102+ /* Make R0 a map ptr */
103+ BPF_LD_MAP_FD(BPF_REG_0, 0),
104+ /* Get an unknown value */
105+ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 0),
106+ /* Make it small and 8-byte aligned */
107+ BPF_ALU64_IMM(BPF_AND, BPF_REG_2, 8),
108+ BPF_ALU64_IMM(BPF_SUB, BPF_REG_2, 16),
109+ /* Add it to fp. We now have either fp-8 or fp-16, but
110+ * we don't know which.
111+ */
112+ BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_10),
113+ /* Spill R0(map ptr) into stack */
114+ BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, -8),
115+ /* Dereference the unknown value for a stack write */
116+ BPF_ST_MEM(BPF_DW, BPF_REG_2, 0, 0),
117+ /* Fill the register back into R2 */
118+ BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_10, -8),
119+ /* Try to dereference R2 for a memory load */
120+ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 8),
121+ BPF_EXIT_INSN(),
122+ },
123+ .fixup_map_hash_8b = { 1 },
124+ /* The unpriviledged case is not too interesting; variable
125+ * stack access is rejected.
126+ */
127+ .errstr_unpriv = "R2 variable stack access prohibited for !root",
128+ .result_unpriv = REJECT,
129+ /* In the priviledged case, dereferencing a spilled-and-then-filled
130+ * register is rejected because the previous variable offset stack
131+ * write might have overwritten the spilled pointer (i.e. we lose track
132+ * of the spilled register when we analyze the write).
133+ */
134+ .errstr = "R2 invalid mem access 'inv'",
135+ .result = REJECT,
136+},
42137 {
43138 "indirect variable-offset stack access, unbounded",
44139 .insns = {
@@ -63,7 +158,7 @@ · 18 unmodified lines
63158 BPF_MOV64_IMM(BPF_REG_0, 0),
64159 BPF_EXIT_INSN(),
65160 },
66− .errstr = "R4 unbounded indirect variable offset stack access",
161+ .errstr = "invalid unbounded variable-offset indirect access to stack R4",
67162 .result = REJECT,
68163 .prog_type = BPF_PROG_TYPE_SOCK_OPS,
69164 },
@@ -88,7 +183,7 @@ · 18 unmodified lines
88183 BPF_EXIT_INSN(),
89184 },
90185 .fixup_map_hash_8b = { 5 },
91− .errstr = "R2 max value is outside of stack bound",
186+ .errstr = "invalid variable-offset indirect access to stack R2",
92187 .result = REJECT,
93188 .prog_type = BPF_PROG_TYPE_LWT_IN,
94189 },
@@ -113,7 +208,7 @@ · 18 unmodified lines
113208 BPF_EXIT_INSN(),
114209 },
115210 .fixup_map_hash_8b = { 5 },
116− .errstr = "R2 min value is outside of stack bound",
211+ .errstr = "invalid variable-offset indirect access to stack R2",
117212 .result = REJECT,
118213 .prog_type = BPF_PROG_TYPE_LWT_IN,
119214 },
@@ -138,7 +233,7 @@ · 18 unmodified lines
138233 BPF_EXIT_INSN(),
139234 },
140235 .fixup_map_hash_8b = { 5 },
141− .errstr = "invalid indirect read from stack var_off",
236+ .errstr = "invalid indirect read from stack R2 var_off",
142237 .result = REJECT,
143238 .prog_type = BPF_PROG_TYPE_LWT_IN,
144239 },
@@ -163,7 +258,7 @@ · 18 unmodified lines
163258 BPF_EXIT_INSN(),
164259 },
165260 .fixup_map_hash_8b = { 5 },
166− .errstr = "invalid indirect read from stack var_off",
261+ .errstr = "invalid indirect read from stack R2 var_off",
167262 .result = REJECT,
168263 .prog_type = BPF_PROG_TYPE_LWT_IN,
169264 },
@@ -189,7 +284,7 @@ · 19 unmodified lines
189284 BPF_EXIT_INSN(),
190285 },
191286 .fixup_map_hash_8b = { 6 },
192− .errstr_unpriv = "R2 stack pointer arithmetic goes out of range, prohibited for !root",
287+ .errstr_unpriv = "R2 variable stack access prohibited for !root",
193288 .result_unpriv = REJECT,
194289 .result = ACCEPT,
195290 .prog_type = BPF_PROG_TYPE_CGROUP_SKB,
@@ -217,7 +312,7 @@ · 21 unmodified lines
217312 BPF_MOV64_IMM(BPF_REG_0, 0),
218313 BPF_EXIT_INSN(),
219314 },
220− .errstr = "invalid indirect read from stack var_off",
315+ .errstr = "invalid indirect read from stack R4 var_off",
221316 .result = REJECT,
222317 .prog_type = BPF_PROG_TYPE_SOCK_OPS,
223318 },
tools/testing/selftests/bpf/vmtest.sh+368−0
@@ -0,0 +1,368 @@
1+#!/bin/bash
2+# SPDX-License-Identifier: GPL-2.0
3+
4+set -u
5+set -e
6+
7+# This script currently only works for x86_64, as
8+# it is based on the VM image used by the BPF CI which is
9+# x86_64.
10+QEMU_BINARY="${QEMU_BINARY:="qemu-system-x86_64"}"
11+X86_BZIMAGE="arch/x86/boot/bzImage"
12+DEFAULT_COMMAND="./test_progs"
13+MOUNT_DIR="mnt"
14+ROOTFS_IMAGE="root.img"
15+OUTPUT_DIR="$HOME/.bpf_selftests"
16+KCONFIG_URL="https://raw.githubusercontent.com/libbpf/libbpf/master/travis-ci/vmtest/configs/latest.config"
17+KCONFIG_API_URL="https://api.github.com/repos/libbpf/libbpf/contents/travis-ci/vmtest/configs/latest.config"
18+INDEX_URL="https://raw.githubusercontent.com/libbpf/libbpf/master/travis-ci/vmtest/configs/INDEX"
19+NUM_COMPILE_JOBS="$(nproc)"
20+
21+usage()
22+{
23+ cat <<EOF
24+Usage: $0 [-i] [-d <output_dir>] -- [<command>]
25+
26+<command> is the command you would normally run when you are in
27+tools/testing/selftests/bpf. e.g:
28+
29+ $0 -- ./test_progs -t test_lsm
30+
31+If no command is specified, "${DEFAULT_COMMAND}" will be run by
32+default.
33+
34+If you build your kernel using KBUILD_OUTPUT= or O= options, these
35+can be passed as environment variables to the script:
36+
37+ O=<kernel_build_path> $0 -- ./test_progs -t test_lsm
38+
39+or
40+
41+ KBUILD_OUTPUT=<kernel_build_path> $0 -- ./test_progs -t test_lsm
42+
43+Options:
44+
45+ -i) Update the rootfs image with a newer version.
46+ -d) Update the output directory (default: ${OUTPUT_DIR})
47+ -j) Number of jobs for compilation, similar to -j in make
48+ (default: ${NUM_COMPILE_JOBS})
49+EOF
50+}
51+
52+unset URLS
53+populate_url_map()
54+{
55+ if ! declare -p URLS &> /dev/null; then
56+ # URLS contain the mapping from file names to URLs where
57+ # those files can be downloaded from.
58+ declare -gA URLS
59+ while IFS=$'\t' read -r name url; do
60+ URLS["$name"]="$url"
61+ done < <(curl -Lsf ${INDEX_URL})
62+ fi
63+}
64+
65+download()
66+{
67+ local file="$1"
68+
69+ if [[ ! -v URLS[$file] ]]; then
70+ echo "$file not found" >&2
71+ return 1
72+ fi
73+
74+ echo "Downloading $file..." >&2
75+ curl -Lsf "${URLS[$file]}" "${@:2}"
76+}
77+
78+newest_rootfs_version()
79+{
80+ {
81+ for file in "${!URLS[@]}"; do
82+ if [[ $file =~ ^libbpf-vmtest-rootfs-(.*)\.tar\.zst$ ]]; then
83+ echo "${BASH_REMATCH[1]}"
84+ fi
85+ done
86+ } | sort -rV | head -1
87+}
88+
89+download_rootfs()
90+{
91+ local rootfsversion="$1"
92+ local dir="$2"
93+
94+ if ! which zstd &> /dev/null; then
95+ echo 'Could not find "zstd" on the system, please install zstd'
96+ exit 1
97+ fi
98+
99+ download "libbpf-vmtest-rootfs-$rootfsversion.tar.zst" |
100+ zstd -d | sudo tar -C "$dir" -x
101+}
102+
103+recompile_kernel()
104+{
105+ local kernel_checkout="$1"
106+ local make_command="$2"
107+
108+ cd "${kernel_checkout}"
109+
110+ ${make_command} olddefconfig
111+ ${make_command}
112+}
113+
114+mount_image()
115+{
116+ local rootfs_img="${OUTPUT_DIR}/${ROOTFS_IMAGE}"
117+ local mount_dir="${OUTPUT_DIR}/${MOUNT_DIR}"
118+
119+ sudo mount -o loop "${rootfs_img}" "${mount_dir}"
120+}
121+
122+unmount_image()
123+{
124+ local mount_dir="${OUTPUT_DIR}/${MOUNT_DIR}"
125+
126+ sudo umount "${mount_dir}" &> /dev/null
127+}
128+
129+update_selftests()
130+{
131+ local kernel_checkout="$1"
132+ local selftests_dir="${kernel_checkout}/tools/testing/selftests/bpf"
133+
134+ cd "${selftests_dir}"
135+ ${make_command}
136+
137+ # Mount the image and copy the selftests to the image.
138+ mount_image
139+ sudo rm -rf "${mount_dir}/root/bpf"
140+ sudo cp -r "${selftests_dir}" "${mount_dir}/root"
141+ unmount_image
142+}
143+
144+update_init_script()
145+{
146+ local init_script_dir="${OUTPUT_DIR}/${MOUNT_DIR}/etc/rcS.d"
147+ local init_script="${init_script_dir}/S50-startup"
148+ local command="$1"
149+ local log_file="$2"
150+
151+ mount_image
152+
153+ if [[ ! -d "${init_script_dir}" ]]; then
154+ cat <<EOF
155+Could not find ${init_script_dir} in the mounted image.
156+This likely indicates a bad rootfs image, Please download
157+a new image by passing "-i" to the script
158+EOF
159+ exit 1
160+
161+ fi
162+
163+ sudo bash -c "cat >${init_script}" <<EOF
164+#!/bin/bash
165+
166+{
167+ cd /root/bpf
168+ echo ${command}
169+ stdbuf -oL -eL ${command}
170+} 2>&1 | tee /root/${log_file}
171+poweroff -f
172+EOF
173+
174+ sudo chmod a+x "${init_script}"
175+ unmount_image
176+}
177+
178+create_vm_image()
179+{
180+ local rootfs_img="${OUTPUT_DIR}/${ROOTFS_IMAGE}"
181+ local mount_dir="${OUTPUT_DIR}/${MOUNT_DIR}"
182+
183+ rm -rf "${rootfs_img}"
184+ touch "${rootfs_img}"
185+ chattr +C "${rootfs_img}" >/dev/null 2>&1 || true
186+
187+ truncate -s 2G "${rootfs_img}"
188+ mkfs.ext4 -q "${rootfs_img}"
189+
190+ mount_image
191+ download_rootfs "$(newest_rootfs_version)" "${mount_dir}"
192+ unmount_image
193+}
194+
195+run_vm()
196+{
197+ local kernel_bzimage="$1"
198+ local rootfs_img="${OUTPUT_DIR}/${ROOTFS_IMAGE}"
199+
200+ if ! which "${QEMU_BINARY}" &> /dev/null; then
201+ cat <<EOF
202+Could not find ${QEMU_BINARY}
203+Please install qemu or set the QEMU_BINARY environment variable.
204+EOF
205+ exit 1
206+ fi
207+
208+ ${QEMU_BINARY} \
209+ -nodefaults \
210+ -display none \
211+ -serial mon:stdio \
212+ -cpu kvm64 \
213+ -enable-kvm \
214+ -smp 4 \
215+ -m 2G \
216+ -drive file="${rootfs_img}",format=raw,index=1,media=disk,if=virtio,cache=none \
217+ -kernel "${kernel_bzimage}" \
218+ -append "root=/dev/vda rw console=ttyS0,115200"
219+}
220+
221+copy_logs()
222+{
223+ local mount_dir="${OUTPUT_DIR}/${MOUNT_DIR}"
224+ local log_file="${mount_dir}/root/$1"
225+
226+ mount_image
227+ sudo cp ${log_file} "${OUTPUT_DIR}"
228+ sudo rm -f ${log_file}
229+ unmount_image
230+}
231+
232+is_rel_path()
233+{
234+ local path="$1"
235+
236+ [[ ${path:0:1} != "/" ]]
237+}
238+
239+update_kconfig()
240+{
241+ local kconfig_file="$1"
242+ local update_command="curl -sLf ${KCONFIG_URL} -o ${kconfig_file}"
243+ # Github does not return the "last-modified" header when retrieving the
244+ # raw contents of the file. Use the API call to get the last-modified
245+ # time of the kernel config and only update the config if it has been
246+ # updated after the previously cached config was created. This avoids
247+ # unnecessarily compiling the kernel and selftests.
248+ if [[ -f "${kconfig_file}" ]]; then
249+ local last_modified_date="$(curl -sL -D - "${KCONFIG_API_URL}" -o /dev/null | \
250+ grep "last-modified" | awk -F ': ' '{print $2}')"
251+ local remote_modified_timestamp="$(date -d "${last_modified_date}" +"%s")"
252+ local local_creation_timestamp="$(stat -c %Y "${kconfig_file}")"
253+
254+ if [[ "${remote_modified_timestamp}" -gt "${local_creation_timestamp}" ]]; then
255+ ${update_command}
256+ fi
257+ else
258+ ${update_command}
259+ fi
260+}
261+
262+main()
263+{
264+ local script_dir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
265+ local kernel_checkout=$(realpath "${script_dir}"/../../../../)
266+ local log_file="$(date +"bpf_selftests.%Y-%m-%d_%H-%M-%S.log")"
267+ # By default the script searches for the kernel in the checkout directory but
268+ # it also obeys environment variables O= and KBUILD_OUTPUT=
269+ local kernel_bzimage="${kernel_checkout}/${X86_BZIMAGE}"
270+ local command="${DEFAULT_COMMAND}"
271+ local update_image="no"
272+
273+ while getopts 'hkid:j:' opt; do
274+ case ${opt} in
275+ i)
276+ update_image="yes"
277+ ;;
278+ d)
279+ OUTPUT_DIR="$OPTARG"
280+ ;;
281+ j)
282+ NUM_COMPILE_JOBS="$OPTARG"
283+ ;;
284+ h)
285+ usage
286+ exit 0
287+ ;;
288+ \? )
289+ echo "Invalid Option: -$OPTARG"
290+ usage
291+ exit 1
292+ ;;
293+ : )
294+ echo "Invalid Option: -$OPTARG requires an argument"
295+ usage
296+ exit 1
297+ ;;
298+ esac
299+ done
300+ shift $((OPTIND -1))
301+
302+ if [[ $# -eq 0 ]]; then
303+ echo "No command specified, will run ${DEFAULT_COMMAND} in the vm"
304+ else
305+ command="$@"
306+ fi
307+
308+ local kconfig_file="${OUTPUT_DIR}/latest.config"
309+ local make_command="make -j ${NUM_COMPILE_JOBS} KCONFIG_CONFIG=${kconfig_file}"
310+
311+ # Figure out where the kernel is being built.
312+ # O takes precedence over KBUILD_OUTPUT.
313+ if [[ "${O:=""}" != "" ]]; then
314+ if is_rel_path "${O}"; then
315+ O="$(realpath "${PWD}/${O}")"
316+ fi
317+ kernel_bzimage="${O}/${X86_BZIMAGE}"
318+ make_command="${make_command} O=${O}"
319+ elif [[ "${KBUILD_OUTPUT:=""}" != "" ]]; then
320+ if is_rel_path "${KBUILD_OUTPUT}"; then
321+ KBUILD_OUTPUT="$(realpath "${PWD}/${KBUILD_OUTPUT}")"
322+ fi
323+ kernel_bzimage="${KBUILD_OUTPUT}/${X86_BZIMAGE}"
324+ make_command="${make_command} KBUILD_OUTPUT=${KBUILD_OUTPUT}"
325+ fi
326+
327+ populate_url_map
328+
329+ local rootfs_img="${OUTPUT_DIR}/${ROOTFS_IMAGE}"
330+ local mount_dir="${OUTPUT_DIR}/${MOUNT_DIR}"
331+
332+ echo "Output directory: ${OUTPUT_DIR}"
333+
334+ mkdir -p "${OUTPUT_DIR}"
335+ mkdir -p "${mount_dir}"
336+ update_kconfig "${kconfig_file}"
337+
338+ recompile_kernel "${kernel_checkout}" "${make_command}"
339+
340+ if [[ "${update_image}" == "no" && ! -f "${rootfs_img}" ]]; then
341+ echo "rootfs image not found in ${rootfs_img}"
342+ update_image="yes"
343+ fi
344+
345+ if [[ "${update_image}" == "yes" ]]; then
346+ create_vm_image
347+ fi
348+
349+ update_selftests "${kernel_checkout}" "${make_command}"
350+ update_init_script "${command}" "${log_file}"
351+ run_vm "${kernel_bzimage}"
352+ copy_logs "${log_file}"
353+ echo "Logs saved in ${OUTPUT_DIR}/${log_file}"
354+}
355+
356+catch()
357+{
358+ local exit_code=$1
359+ # This is just a cleanup and the directory may
360+ # have already been unmounted. So, don't let this
361+ # clobber the error code we intend to return.
362+ unmount_image || true
363+ exit ${exit_code}
364+}
365+
366+trap 'catch "$?"' EXIT
367+
368+main "$@"
tools/testing/selftests/bpf/xdpxceiver.c+108−117
@@ -224,14 +224,14 @@ static inline u16 udp_csum(u32 saddr, u32 daddr, u32 len, u8 proto, u16 *udp_pkt · 223 unmodified lines
224224 return csum_tcpudp_magic(saddr, daddr, len, proto, csum);
225225 }
226226
227−static void gen_eth_hdr(void *data, struct ethhdr *eth_hdr)
227+static void gen_eth_hdr(struct ifobject *ifobject, struct ethhdr *eth_hdr)
228228 {
229− memcpy(eth_hdr->h_dest, ((struct ifobject *)data)->dst_mac, ETH_ALEN);
230− memcpy(eth_hdr->h_source, ((struct ifobject *)data)->src_mac, ETH_ALEN);
229+ memcpy(eth_hdr->h_dest, ifobject->dst_mac, ETH_ALEN);
230+ memcpy(eth_hdr->h_source, ifobject->src_mac, ETH_ALEN);
231231 eth_hdr->h_proto = htons(ETH_P_IP);
232232 }
233233
234−static void gen_ip_hdr(void *data, struct iphdr *ip_hdr)
234+static void gen_ip_hdr(struct ifobject *ifobject, struct iphdr *ip_hdr)
235235 {
236236 ip_hdr->version = IP_PKT_VER;
237237 ip_hdr->ihl = 0x5;
@@ -241,18 +241,18 @@ static void gen_ip_hdr(void *data, struct iphdr *ip_hdr) · 3 unmodified lines
241241 ip_hdr->frag_off = 0;
242242 ip_hdr->ttl = IPDEFTTL;
243243 ip_hdr->protocol = IPPROTO_UDP;
244− ip_hdr->saddr = ((struct ifobject *)data)->src_ip;
245− ip_hdr->daddr = ((struct ifobject *)data)->dst_ip;
244+ ip_hdr->saddr = ifobject->src_ip;
245+ ip_hdr->daddr = ifobject->dst_ip;
246246 ip_hdr->check = 0;
247247 }
248248
249−static void gen_udp_hdr(void *data, void *arg, struct udphdr *udp_hdr)
249+static void gen_udp_hdr(struct generic_data *data, struct ifobject *ifobject,
250+ struct udphdr *udp_hdr)
250251 {
251− udp_hdr->source = htons(((struct ifobject *)arg)->src_port);
252− udp_hdr->dest = htons(((struct ifobject *)arg)->dst_port);
252+ udp_hdr->source = htons(ifobject->src_port);
253+ udp_hdr->dest = htons(ifobject->dst_port);
253254 udp_hdr->len = htons(UDP_PKT_SIZE);
254− memset32_htonl(pkt_data + PKT_HDR_SIZE,
255− htonl(((struct generic_data *)data)->seqnum), UDP_PKT_DATA_SIZE);
255+ memset32_htonl(pkt_data + PKT_HDR_SIZE, htonl(data->seqnum), UDP_PKT_DATA_SIZE);
256256 }
257257
258258 static void gen_udp_csum(struct udphdr *udp_hdr, struct iphdr *ip_hdr)
@@ -382,21 +382,19 @@ static bool switch_namespace(int idx) · 123 unmodified lines
382382
383383 static void *nsswitchthread(void *args)
384384 {
385− if (switch_namespace(((struct targs *)args)->idx)) {
386− ifdict[((struct targs *)args)->idx]->ifindex =
387− if_nametoindex(ifdict[((struct targs *)args)->idx]->ifname);
388− if (!ifdict[((struct targs *)args)->idx]->ifindex) {
389− ksft_test_result_fail
390− ("ERROR: [%s] interface \"%s\" does not exist\n",
391− __func__, ifdict[((struct targs *)args)->idx]->ifname);
392− ((struct targs *)args)->retptr = false;
385+ struct targs *targs = args;
386+
387+ targs->retptr = false;
388+
389+ if (switch_namespace(targs->idx)) {
390+ ifdict[targs->idx]->ifindex = if_nametoindex(ifdict[targs->idx]->ifname);
391+ if (!ifdict[targs->idx]->ifindex) {
392+ ksft_test_result_fail("ERROR: [%s] interface \"%s\" does not exist\n",
393+ __func__, ifdict[targs->idx]->ifname);
393394 } else {
394− ksft_print_msg("Interface found: %s\n",
395− ifdict[((struct targs *)args)->idx]->ifname);
396− ((struct targs *)args)->retptr = true;
395+ ksft_print_msg("Interface found: %s\n", ifdict[targs->idx]->ifname);
396+ targs->retptr = true;
397397 }
398− } else {
399− ((struct targs *)args)->retptr = false;
400398 }
401399 pthread_exit(NULL);
402400 }
@@ -413,12 +411,12 @@ static int validate_interfaces(void) · 10 unmodified lines
413411 if (strcmp(ifdict[i]->nsname, "")) {
414412 struct targs *targs;
415413
416− targs = (struct targs *)malloc(sizeof(struct targs));
414+ targs = malloc(sizeof(*targs));
417415 if (!targs)
418416 exit_with_error(errno);
419417
420418 targs->idx = i;
421− if (pthread_create(&ns_thread, NULL, nsswitchthread, (void *)targs))
419+ if (pthread_create(&ns_thread, NULL, nsswitchthread, targs))
422420 exit_with_error(errno);
423421
424422 pthread_join(ns_thread, NULL);
@@ -569,16 +567,18 @@ static void rx_pkt(struct xsk_socket_info *xsk, struct pollfd *fds) · 144 unmodified lines
569567 }
570568
571569 for (i = 0; i < rcvd; i++) {
572− u64 addr = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx)->addr;
573− (void)xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++)->len;
574− u64 orig = xsk_umem__extract_addr(addr);
570+ u64 addr, orig;
571+
572+ addr = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx)->addr;
573+ xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++);
574+ orig = xsk_umem__extract_addr(addr);
575575
576576 addr = xsk_umem__add_offset_to_addr(addr);
577577 pkt_node_rx = malloc(sizeof(struct pkt) + PKT_SIZE);
578578 if (!pkt_node_rx)
579579 exit_with_error(errno);
580580
581− pkt_node_rx->pkt_frame = (char *)malloc(PKT_SIZE);
581+ pkt_node_rx->pkt_frame = malloc(PKT_SIZE);
582582 if (!pkt_node_rx->pkt_frame)
583583 exit_with_error(errno);
584584
@@ -628,28 +628,27 @@ static inline int get_batch_size(int pkt_cnt) · 43 unmodified lines
628628 return opt_pkt_count - pkt_cnt;
629629 }
630630
631−static void complete_tx_only_all(void *arg)
631+static void complete_tx_only_all(struct ifobject *ifobject)
632632 {
633633 bool pending;
634634
635635 do {
636636 pending = false;
637− if (((struct ifobject *)arg)->xsk->outstanding_tx) {
638− complete_tx_only(((struct ifobject *)
639− arg)->xsk, BATCH_SIZE);
640− pending = !!((struct ifobject *)arg)->xsk->outstanding_tx;
637+ if (ifobject->xsk->outstanding_tx) {
638+ complete_tx_only(ifobject->xsk, BATCH_SIZE);
639+ pending = !!ifobject->xsk->outstanding_tx;
641640 }
642641 } while (pending);
643642 }
644643
645−static void tx_only_all(void *arg)
644+static void tx_only_all(struct ifobject *ifobject)
646645 {
647646 struct pollfd fds[MAX_SOCKS] = { };
648647 u32 frame_nb = 0;
649648 int pkt_cnt = 0;
650649 int ret;
651650
652− fds[0].fd = xsk_socket__fd(((struct ifobject *)arg)->xsk->xsk);
651+ fds[0].fd = xsk_socket__fd(ifobject->xsk->xsk);
653652 fds[0].events = POLLOUT;
654653
655654 while ((opt_pkt_count && pkt_cnt < opt_pkt_count) || !opt_pkt_count) {
@@ -664,12 +663,12 @@ static void tx_only_all(void *arg) · 8 unmodified lines
664663 continue;
665664 }
666665
667− tx_only(((struct ifobject *)arg)->xsk, &frame_nb, batch_size);
666+ tx_only(ifobject->xsk, &frame_nb, batch_size);
668667 pkt_cnt += batch_size;
669668 }
670669
671670 if (opt_pkt_count)
672− complete_tx_only_all(arg);
671+ complete_tx_only_all(ifobject);
673672 }
674673
675674 static void worker_pkt_dump(void)
@@ -727,21 +726,21 @@ static void worker_pkt_dump(void) · 51 unmodified lines
727726 static void worker_pkt_validate(void)
728727 {
729728 u32 payloadseqnum = -2;
729+ struct iphdr *iphdr;
730730
731731 while (1) {
732− pkt_node_rx_q = malloc(sizeof(struct pkt));
733732 pkt_node_rx_q = TAILQ_LAST(&head, head_s);
734733 if (!pkt_node_rx_q)
735734 break;
735+
736+ iphdr = (struct iphdr *)(pkt_node_rx_q->pkt_frame + sizeof(struct ethhdr));
737+
736738 /*do not increment pktcounter if !(tos=0x9 and ipv4) */
737− if ((((struct iphdr *)(pkt_node_rx_q->pkt_frame +
738− sizeof(struct ethhdr)))->version == IP_PKT_VER)
739− && (((struct iphdr *)(pkt_node_rx_q->pkt_frame + sizeof(struct ethhdr)))->tos ==
740− IP_PKT_TOS)) {
741− payloadseqnum = *((uint32_t *) (pkt_node_rx_q->pkt_frame + PKT_HDR_SIZE));
739+ if (iphdr->version == IP_PKT_VER && iphdr->tos == IP_PKT_TOS) {
740+ payloadseqnum = *((uint32_t *)(pkt_node_rx_q->pkt_frame + PKT_HDR_SIZE));
742741 if (debug_pkt_dump && payloadseqnum != EOT) {
743− pkt_obj = (struct pkt_frame *)malloc(sizeof(struct pkt_frame));
744− pkt_obj->payload = (char *)malloc(PKT_SIZE);
742+ pkt_obj = malloc(sizeof(*pkt_obj));
743+ pkt_obj->payload = malloc(PKT_SIZE);
745744 memcpy(pkt_obj->payload, pkt_node_rx_q->pkt_frame, PKT_SIZE);
746745 pkt_buf[payloadseqnum] = pkt_obj;
747746 }
@@ -759,35 +758,29 @@ static void worker_pkt_validate(void) · 11 unmodified lines
759758 ksft_exit_xfail();
760759 }
761760
762− TAILQ_REMOVE(&head, pkt_node_rx_q, pkt_nodes);
763− free(pkt_node_rx_q->pkt_frame);
764− free(pkt_node_rx_q);
765− pkt_node_rx_q = NULL;
766761 prev_pkt = payloadseqnum;
767762 pkt_counter++;
768763 } else {
769764 ksft_print_msg("Invalid frame received: ");
770− ksft_print_msg("[IP_PKT_VER: %02X], [IP_PKT_TOS: %02X]\n",
771− ((struct iphdr *)(pkt_node_rx_q->pkt_frame +
772− sizeof(struct ethhdr)))->version,
773− ((struct iphdr *)(pkt_node_rx_q->pkt_frame +
774− sizeof(struct ethhdr)))->tos);
775− TAILQ_REMOVE(&head, pkt_node_rx_q, pkt_nodes);
776− free(pkt_node_rx_q->pkt_frame);
777− free(pkt_node_rx_q);
778− pkt_node_rx_q = NULL;
765+ ksft_print_msg("[IP_PKT_VER: %02X], [IP_PKT_TOS: %02X]\n", iphdr->version,
766+ iphdr->tos);
779767 }
768+
769+ TAILQ_REMOVE(&head, pkt_node_rx_q, pkt_nodes);
770+ free(pkt_node_rx_q->pkt_frame);
771+ free(pkt_node_rx_q);
772+ pkt_node_rx_q = NULL;
780773 }
781774 }
782775
783−static void thread_common_ops(void *arg, void *bufs, pthread_mutex_t *mutexptr,
776+static void thread_common_ops(struct ifobject *ifobject, void *bufs, pthread_mutex_t *mutexptr,
784777 atomic_int *spinningptr)
785778 {
786779 int ctr = 0;
787780 int ret;
788781
789− xsk_configure_umem((struct ifobject *)arg, bufs, num_frames * XSK_UMEM__DEFAULT_FRAME_SIZE);
790− ret = xsk_configure_socket((struct ifobject *)arg);
782+ xsk_configure_umem(ifobject, bufs, num_frames * XSK_UMEM__DEFAULT_FRAME_SIZE);
783+ ret = xsk_configure_socket(ifobject);
791784
792785 /* Retry Create Socket if it fails as xsk_socket__create()
793786 * is asynchronous
@@ -798,9 +791,8 @@ static void thread_common_ops(void *arg, void *bufs, pthread_mutex_t *mutexptr, · 4 unmodified lines
798791 pthread_mutex_lock(mutexptr);
799792 while (ret && ctr < SOCK_RECONF_CTR) {
800793 atomic_store(spinningptr, 1);
801− xsk_configure_umem((struct ifobject *)arg,
802− bufs, num_frames * XSK_UMEM__DEFAULT_FRAME_SIZE);
803− ret = xsk_configure_socket((struct ifobject *)arg);
794+ xsk_configure_umem(ifobject, bufs, num_frames * XSK_UMEM__DEFAULT_FRAME_SIZE);
795+ ret = xsk_configure_socket(ifobject);
804796 usleep(USLEEP_MAX);
805797 ctr++;
806798 }
@@ -815,9 +807,10 @@ static void *worker_testapp_validate(void *arg) · 8 unmodified lines
815807 {
816808 struct udphdr *udp_hdr =
817809 (struct udphdr *)(pkt_data + sizeof(struct ethhdr) + sizeof(struct iphdr));
818− struct generic_data *data = (struct generic_data *)malloc(sizeof(struct generic_data));
819810 struct iphdr *ip_hdr = (struct iphdr *)(pkt_data + sizeof(struct ethhdr));
820811 struct ethhdr *eth_hdr = (struct ethhdr *)pkt_data;
812+ struct ifobject *ifobject = (struct ifobject *)arg;
813+ struct generic_data data;
821814 void *bufs = NULL;
822815
823816 pthread_attr_setstacksize(&attr, THREAD_STACK);
@@ -828,58 +821,56 @@ static void *worker_testapp_validate(void *arg) · 4 unmodified lines
828821 if (bufs == MAP_FAILED)
829822 exit_with_error(errno);
830823
831− if (strcmp(((struct ifobject *)arg)->nsname, ""))
832− switch_namespace(((struct ifobject *)arg)->ifdict_index);
824+ if (strcmp(ifobject->nsname, ""))
825+ switch_namespace(ifobject->ifdict_index);
833826 }
834827
835− if (((struct ifobject *)arg)->fv.vector == tx) {
828+ if (ifobject->fv.vector == tx) {
836829 int spinningrxctr = 0;
837830
838831 if (!bidi_pass)
839− thread_common_ops(arg, bufs, &sync_mutex_tx, &spinning_tx);
832+ thread_common_ops(ifobject, bufs, &sync_mutex_tx, &spinning_tx);
840833
841834 while (atomic_load(&spinning_rx) && spinningrxctr < SOCK_RECONF_CTR) {
842835 spinningrxctr++;
843836 usleep(USLEEP_MAX);
844837 }
845838
846− ksft_print_msg("Interface [%s] vector [Tx]\n", ((struct ifobject *)arg)->ifname);
839+ ksft_print_msg("Interface [%s] vector [Tx]\n", ifobject->ifname);
847840 for (int i = 0; i < num_frames; i++) {
848841 /*send EOT frame */
849842 if (i == (num_frames - 1))
850− data->seqnum = -1;
843+ data.seqnum = -1;
851844 else
852− data->seqnum = i;
853− gen_udp_hdr((void *)data, (void *)arg, udp_hdr);
854− gen_ip_hdr((void *)arg, ip_hdr);
845+ data.seqnum = i;
846+ gen_udp_hdr(&data, ifobject, udp_hdr);
847+ gen_ip_hdr(ifobject, ip_hdr);
855848 gen_udp_csum(udp_hdr, ip_hdr);
856− gen_eth_hdr((void *)arg, eth_hdr);
857− gen_eth_frame(((struct ifobject *)arg)->umem,
858− i * XSK_UMEM__DEFAULT_FRAME_SIZE);
849+ gen_eth_hdr(ifobject, eth_hdr);
850+ gen_eth_frame(ifobject->umem, i * XSK_UMEM__DEFAULT_FRAME_SIZE);
859851 }
860852
861− free(data);
862853 ksft_print_msg("Sending %d packets on interface %s\n",
863− (opt_pkt_count - 1), ((struct ifobject *)arg)->ifname);
864− tx_only_all(arg);
865− } else if (((struct ifobject *)arg)->fv.vector == rx) {
854+ (opt_pkt_count - 1), ifobject->ifname);
855+ tx_only_all(ifobject);
856+ } else if (ifobject->fv.vector == rx) {
866857 struct pollfd fds[MAX_SOCKS] = { };
867858 int ret;
868859
869860 if (!bidi_pass)
870− thread_common_ops(arg, bufs, &sync_mutex_tx, &spinning_rx);
861+ thread_common_ops(ifobject, bufs, &sync_mutex_tx, &spinning_rx);
871862
872− ksft_print_msg("Interface [%s] vector [Rx]\n", ((struct ifobject *)arg)->ifname);
873− xsk_populate_fill_ring(((struct ifobject *)arg)->umem);
863+ ksft_print_msg("Interface [%s] vector [Rx]\n", ifobject->ifname);
864+ xsk_populate_fill_ring(ifobject->umem);
874865
875866 TAILQ_INIT(&head);
876867 if (debug_pkt_dump) {
877− pkt_buf = malloc(sizeof(struct pkt_frame **) * num_frames);
868+ pkt_buf = calloc(num_frames, sizeof(*pkt_buf));
878869 if (!pkt_buf)
879870 exit_with_error(errno);
880871 }
881872
882− fds[0].fd = xsk_socket__fd(((struct ifobject *)arg)->xsk->xsk);
873+ fds[0].fd = xsk_socket__fd(ifobject->xsk->xsk);
883874 fds[0].events = POLLIN;
884875
885876 pthread_mutex_lock(&sync_mutex);
@@ -892,7 +883,7 @@ static void *worker_testapp_validate(void *arg) · 6 unmodified lines
892883 if (ret <= 0)
893884 continue;
894885 }
895− rx_pkt(((struct ifobject *)arg)->xsk, fds);
886+ rx_pkt(ifobject->xsk, fds);
896887 worker_pkt_validate();
897888
898889 if (sigvar)
@@ -900,21 +891,23 @@ static void *worker_testapp_validate(void *arg) · 1 unmodified line
900891 }
901892
902893 ksft_print_msg("Received %d packets on interface %s\n",
903− pkt_counter, ((struct ifobject *)arg)->ifname);
894+ pkt_counter, ifobject->ifname);
904895
905896 if (opt_teardown)
906897 ksft_print_msg("Destroying socket\n");
907898 }
908899
909− if (!opt_bidi || (opt_bidi && bidi_pass)) {
910− xsk_socket__delete(((struct ifobject *)arg)->xsk->xsk);
911− (void)xsk_umem__delete(((struct ifobject *)arg)->umem->umem);
900+ if (!opt_bidi || bidi_pass) {
901+ xsk_socket__delete(ifobject->xsk->xsk);
902+ (void)xsk_umem__delete(ifobject->umem->umem);
912903 }
913904 pthread_exit(NULL);
914905 }
915906
916907 static void testapp_validate(void)
917908 {
909+ struct timespec max_wait = { 0, 0 };
910+
918911 pthread_attr_init(&attr);
919912 pthread_attr_setstacksize(&attr, THREAD_STACK);
920913
@@ -929,18 +922,16 @@ static void testapp_validate(void) · 8 unmodified lines
929922 pthread_mutex_lock(&sync_mutex);
930923
931924 /*Spawn RX thread */
932− if (!opt_bidi || (opt_bidi && !bidi_pass)) {
933− if (pthread_create(&t0, &attr, worker_testapp_validate, (void *)ifdict[1]))
925+ if (!opt_bidi || !bidi_pass) {
926+ if (pthread_create(&t0, &attr, worker_testapp_validate, ifdict[1]))
934927 exit_with_error(errno);
935928 } else if (opt_bidi && bidi_pass) {
936929 /*switch Tx/Rx vectors */
937930 ifdict[0]->fv.vector = rx;
938− if (pthread_create(&t0, &attr, worker_testapp_validate, (void *)ifdict[0]))
931+ if (pthread_create(&t0, &attr, worker_testapp_validate, ifdict[0]))
939932 exit_with_error(errno);
940933 }
941934
942− struct timespec max_wait = { 0, 0 };
943−
944935 if (clock_gettime(CLOCK_REALTIME, &max_wait))
945936 exit_with_error(errno);
946937 max_wait.tv_sec += TMOUT_SEC;
@@ -951,13 +942,13 @@ static void testapp_validate(void) · 4 unmodified lines
951942 pthread_mutex_unlock(&sync_mutex);
952943
953944 /*Spawn TX thread */
954− if (!opt_bidi || (opt_bidi && !bidi_pass)) {
955− if (pthread_create(&t1, &attr, worker_testapp_validate, (void *)ifdict[0]))
945+ if (!opt_bidi || !bidi_pass) {
946+ if (pthread_create(&t1, &attr, worker_testapp_validate, ifdict[0]))
956947 exit_with_error(errno);
957948 } else if (opt_bidi && bidi_pass) {
958949 /*switch Tx/Rx vectors */
959950 ifdict[1]->fv.vector = tx;
960− if (pthread_create(&t1, &attr, worker_testapp_validate, (void *)ifdict[1]))
951+ if (pthread_create(&t1, &attr, worker_testapp_validate, ifdict[1]))
961952 exit_with_error(errno);
962953 }
963954
@@ -991,25 +982,25 @@ static void testapp_sockets(void) · 27 unmodified lines
991982 print_ksft_result();
992983 }
993984
994−static void init_iface_config(void *ifaceconfig)
985+static void init_iface_config(struct ifaceconfigobj *ifaceconfig)
995986 {
996987 /*Init interface0 */
997988 ifdict[0]->fv.vector = tx;
998− memcpy(ifdict[0]->dst_mac, ((struct ifaceconfigobj *)ifaceconfig)->dst_mac, ETH_ALEN);
999− memcpy(ifdict[0]->src_mac, ((struct ifaceconfigobj *)ifaceconfig)->src_mac, ETH_ALEN);
1000− ifdict[0]->dst_ip = ((struct ifaceconfigobj *)ifaceconfig)->dst_ip.s_addr;
1001− ifdict[0]->src_ip = ((struct ifaceconfigobj *)ifaceconfig)->src_ip.s_addr;
1002− ifdict[0]->dst_port = ((struct ifaceconfigobj *)ifaceconfig)->dst_port;
1003− ifdict[0]->src_port = ((struct ifaceconfigobj *)ifaceconfig)->src_port;
989+ memcpy(ifdict[0]->dst_mac, ifaceconfig->dst_mac, ETH_ALEN);
990+ memcpy(ifdict[0]->src_mac, ifaceconfig->src_mac, ETH_ALEN);
991+ ifdict[0]->dst_ip = ifaceconfig->dst_ip.s_addr;
992+ ifdict[0]->src_ip = ifaceconfig->src_ip.s_addr;
993+ ifdict[0]->dst_port = ifaceconfig->dst_port;
994+ ifdict[0]->src_port = ifaceconfig->src_port;
1004995
1005996 /*Init interface1 */
1006997 ifdict[1]->fv.vector = rx;
1007− memcpy(ifdict[1]->dst_mac, ((struct ifaceconfigobj *)ifaceconfig)->src_mac, ETH_ALEN);
1008− memcpy(ifdict[1]->src_mac, ((struct ifaceconfigobj *)ifaceconfig)->dst_mac, ETH_ALEN);
1009− ifdict[1]->dst_ip = ((struct ifaceconfigobj *)ifaceconfig)->src_ip.s_addr;
1010− ifdict[1]->src_ip = ((struct ifaceconfigobj *)ifaceconfig)->dst_ip.s_addr;
1011− ifdict[1]->dst_port = ((struct ifaceconfigobj *)ifaceconfig)->src_port;
1012− ifdict[1]->src_port = ((struct ifaceconfigobj *)ifaceconfig)->dst_port;
998+ memcpy(ifdict[1]->dst_mac, ifaceconfig->src_mac, ETH_ALEN);
999+ memcpy(ifdict[1]->src_mac, ifaceconfig->dst_mac, ETH_ALEN);
1000+ ifdict[1]->dst_ip = ifaceconfig->src_ip.s_addr;
1001+ ifdict[1]->src_ip = ifaceconfig->dst_ip.s_addr;
1002+ ifdict[1]->dst_port = ifaceconfig->src_port;
1003+ ifdict[1]->src_port = ifaceconfig->dst_port;
10131004 }
10141005
10151006 int main(int argc, char **argv)
@@ -1026,7 +1017,7 @@ int main(int argc, char **argv) · 10 unmodified lines
10261017 u16 UDP_DST_PORT = 2020;
10271018 u16 UDP_SRC_PORT = 2121;
10281019
1029− ifaceconfig = (struct ifaceconfigobj *)malloc(sizeof(struct ifaceconfigobj));
1020+ ifaceconfig = malloc(sizeof(struct ifaceconfigobj));
10301021 memcpy(ifaceconfig->dst_mac, MAC1, ETH_ALEN);
10311022 memcpy(ifaceconfig->src_mac, MAC2, ETH_ALEN);
10321023 inet_aton(IP1, &ifaceconfig->dst_ip);
@@ -1035,7 +1026,7 @@ int main(int argc, char **argv) · 2 unmodified lines
10351026 ifaceconfig->src_port = UDP_SRC_PORT;
10361027
10371028 for (int i = 0; i < MAX_INTERFACES; i++) {
1038− ifdict[i] = (struct ifobject *)malloc(sizeof(struct ifobject));
1029+ ifdict[i] = malloc(sizeof(struct ifobject));
10391030 if (!ifdict[i])
10401031 exit_with_error(errno);
10411032
@@ -1048,7 +1039,7 @@ int main(int argc, char **argv) · 6 unmodified lines
10481039
10491040 num_frames = ++opt_pkt_count;
10501041
1051− init_iface_config((void *)ifaceconfig);
1042+ init_iface_config(ifaceconfig);
10521043
10531044 pthread_init_mutex();
10541045
tools/testing/selftests/bpf/xdpxceiver.h+0−2
@@ -92,8 +92,6 @@ struct flow_vector { · 91 unmodified lines
9292 enum fvector {
9393 tx,
9494 rx,
95− bidi,
96− undef,
9795 } vector;
9896 };
9997
tools/testing/selftests/tc-testing/Makefile+1−2
@@ -1,4 +1,5 @@
11 # SPDX-License-Identifier: GPL-2.0
2+include ../../../scripts/Makefile.include
23
34 top_srcdir = $(abspath ../../../..)
45 APIDIR := $(top_scrdir)/include/uapi
@@ -7,8 +8,6 @@ TEST_GEN_FILES = action.o · 2 unmodified lines
78 KSFT_KHDR_INSTALL := 1
89 include ../lib.mk
910
10−CLANG ?= clang
11−LLC ?= llc
1211 PROBE := $(shell $(LLC) -march=bpf -mcpu=probe -filetype=null /dev/null 2>&1)
1312
1413 ifeq ($(PROBE),)