History

Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Daniel Borkmann says: ==================== pull-request: bpf-next 2021-02-16 The following pull-request contains BPF updates for your *net-next* tree. There's a small merge conflict between 7eeba1706eba ("tcp: Add receive timestamp support for receive zerocopy.") from net-next tree and 9cacf81f8161 ("bpf: Remove extra lock_sock for TCP_ZEROCOPY_RECEIVE") from bpf-next tree. Resolve as follows: [...] lock_sock(sk); err = tcp_zerocopy_receive(sk, &zc, &tss); err = BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sk, level, optname, &zc, &len, err); release_sock(sk); [...] We've added 116 non-merge commits during the last 27 day(s) which contain a total of 156 files changed, 5662 insertions(+), 1489 deletions(-). The main changes are: 1) Adds support of pointers to types with known size among global function args to overcome the limit on max # of allowed args, from Dmitrii Banshchikov. 2) Add bpf_iter for task_vma which can be used to generate information similar to /proc/pid/maps, from Song Liu. 3) Enable bpf_{g,s}etsockopt() from all sock_addr related program hooks. Allow rewriting bind user ports from BPF side below the ip_unprivileged_port_start range, both from Stanislav Fomichev. 4) Prevent recursion on fentry/fexit & sleepable programs and allow map-in-map as well as per-cpu maps for the latter, from Alexei Starovoitov. 5) Add selftest script to run BPF CI locally. Also enable BPF ringbuffer for sleepable programs, both from KP Singh. 6) Extend verifier to enable variable offset read/write access to the BPF program stack, from Andrei Matei. 7) Improve tc & XDP MTU handling and add a new bpf_check_mtu() helper to query device MTU from programs, from Jesper Dangaard Brouer. 8) Allow bpf_get_socket_cookie() helper also be called from [sleepable] BPF tracing programs, from Florent Revest. 9) Extend x86 JIT to pad JMPs with NOPs for helping image to converge when otherwise too many passes are required, from Gary Lin. 10) Verifier fixes on atomics with BPF_FETCH as well as function-by-function verification both related to zero-extension handling, from Ilya Leoshkevich. 11) Better kernel build integration of resolve_btfids tool, from Jiri Olsa. 12) Batch of AF_XDP selftest cleanups and small performance improvement for libbpf's xsk map redirect for newer kernels, from Björn Töpel. 13) Follow-up BPF doc and verifier improvements around atomics with BPF_FETCH, from Brendan Jackman. 14) Permit zero-sized data sections e.g. if ELF .rodata section contains read-only data from local variables, from Yonghong Song. 15) veth driver skb bulk-allocation for ndo_xdp_xmit, from Lorenzo Bianconi. ====================

David S. Miller committed 5 years ago

Commit b8af4172 parents 9ec5eea 45159b2156 files · +5662 −1489Browse files

Unified · Split

156 files
  1. Documentation/bpf/bpf_design_QA.rst +6 −0
  2. Documentation/bpf/bpf_devel_QA.rst +7 −4
  3. Documentation/networking/filter.rst +15 −13
  4. Makefile +12 −1
  5. arch/x86/net/bpf_jit_comp.c +158 −47
  6. drivers/net/veth.c +60 −34
  7. include/linux/bpf-cgroup.h +70 −31
  8. include/linux/bpf.h +43 −31
  9. include/linux/bpf_verifier.h +4 −1
  10. include/linux/filter.h +18 −3
  11. include/linux/indirect_call_wrapper.h +6 −0
  12. include/linux/netdevice.h +30 −2
  13. include/linux/skmsg.h +0 −1
  14. include/net/inet_common.h +2 −0
  15. include/net/sock.h +2 −0
  16. include/net/tcp.h +1 −0
  17. include/net/xdp.h +6 −0
  18. include/trace/bpf_probe.h +10 −2
  19. include/uapi/linux/bpf.h +97 −6
  20. kernel/bpf/bpf_iter.c +1 −1
  21. kernel/bpf/bpf_lru_list.c +4 −3
  22. kernel/bpf/btf.c +55 −21
  23. kernel/bpf/cgroup.c +103 −17
  24. kernel/bpf/core.c +14 −4
  25. kernel/bpf/cpumap.c +2 −44
  26. kernel/bpf/devmap.c +1 −3
  27. kernel/bpf/disasm.c +1 −1
  28. kernel/bpf/hashtab.c +2 −2
  29. kernel/bpf/helpers.c +4 −8
  30. kernel/bpf/syscall.c +11 −5
  31. kernel/bpf/task_iter.c +266 −1
  32. kernel/bpf/trampoline.c +61 −16
  33. kernel/bpf/verifier.c +662 −215
  34. kernel/trace/bpf_trace.c +6 −0
  35. lib/test_bpf.c +1 −6
  36. net/core/dev.c +14 −18
  37. net/core/filter.c +174 −21
  38. net/core/skmsg.c +1 −2
  39. net/core/xdp.c +70 −0
  40. net/ipv4/af_inet.c +11 −7
  41. net/ipv4/tcp.c +14 −0
  42. net/ipv4/tcp_ipv4.c +1 −0
  43. net/ipv4/udp.c +3 −4
  44. net/ipv6/af_inet6.c +11 −7
  45. net/ipv6/tcp_ipv6.c +1 −0
  46. net/ipv6/udp.c +3 −4
  47. net/socket.c +3 −0
  48. net/xdp/xsk.c +31 −16
  49. net/xdp/xsk_buff_pool.c +3 −9
  50. samples/bpf/Makefile +9 −1
  51. samples/bpf/README.rst +14 −8
  52. samples/bpf/bpf_insn.h +20 −4
  53. samples/bpf/cookie_uid_helper_example.c +1 −1
  54. samples/bpf/xdp_redirect_map_kern.c +55 −5
  55. samples/bpf/xdp_redirect_map_user.c +92 −20
  56. tools/bpf/bpf_dbg.c +1 −1
  57. tools/bpf/bpftool/Makefile +0 −2
  58. tools/bpf/bpftool/prog.c +4 −0
  59. tools/bpf/resolve_btfids/.gitignore +1 −2
  60. tools/bpf/resolve_btfids/Makefile +22 −22
  61. tools/bpf/runqslower/Makefile +0 −3
  62. tools/build/feature/Makefile +2 −2
  63. tools/include/linux/types.h +3 −0
  64. tools/include/uapi/linux/bpf.h +97 −6
  65. tools/include/uapi/linux/bpf_perf_event.h +1 −0
  66. tools/include/uapi/linux/tcp.h +357 −0
  67. tools/lib/bpf/.gitignore +0 −1
  68. tools/lib/bpf/Makefile +4 −43
  69. tools/lib/bpf/btf.c +10 −2
  70. tools/lib/bpf/libbpf.c +11 −11
  71. tools/lib/bpf/xsk.c +79 −4
  72. tools/perf/Makefile.perf +0 −1
  73. tools/scripts/Makefile.include +7 −0
  74. tools/testing/selftests/bpf/.gitignore +0 −2
  75. tools/testing/selftests/bpf/Makefile +2 −6
  76. tools/testing/selftests/bpf/README.rst +24 −0
  77. tools/testing/selftests/bpf/benchs/bench_ringbufs.c +1 −1
  78. tools/testing/selftests/bpf/bpf_sockopt_helpers.h +21 −0
  79. tools/testing/selftests/bpf/bpf_tcp_helpers.h +1 −0
  80. tools/testing/selftests/bpf/bpf_testmod/bpf_testmod-events.h +6 −0
  81. tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c +20 −1
  82. tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.h +6 −0
  83. tools/testing/selftests/bpf/prog_tests/atomic_bounds.c +17 −0
  84. tools/testing/selftests/bpf/prog_tests/bind_perm.c +109 −0
  85. tools/testing/selftests/bpf/prog_tests/bpf_iter.c +108 −10
  86. tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c +1 −0
  87. tools/testing/selftests/bpf/prog_tests/btf.c +23 −2
  88. tools/testing/selftests/bpf/prog_tests/check_mtu.c +216 −0
  89. tools/testing/selftests/bpf/prog_tests/cls_redirect.c +1 −0
  90. tools/testing/selftests/bpf/prog_tests/fexit_stress.c +2 −2
  91. tools/testing/selftests/bpf/prog_tests/global_func_args.c +60 −0
  92. tools/testing/selftests/bpf/prog_tests/module_attach.c +27 −0
  93. tools/testing/selftests/bpf/prog_tests/ns_current_pid_tgid.c +60 −58
  94. tools/testing/selftests/bpf/prog_tests/recursion.c +41 −0
  95. tools/testing/selftests/bpf/prog_tests/socket_cookie.c +76 −0
  96. tools/testing/selftests/bpf/prog_tests/sockmap_basic.c +1 −0
  97. tools/testing/selftests/bpf/prog_tests/sockopt_sk.c +28 −0
  98. tools/testing/selftests/bpf/prog_tests/stack_var_off.c +35 −0
  99. tools/testing/selftests/bpf/prog_tests/test_global_funcs.c +8 −0
  100. tools/testing/selftests/bpf/prog_tests/test_ima.c +18 −5
  101. tools/testing/selftests/bpf/prog_tests/test_local_storage.c +1 −1
  102. tools/testing/selftests/bpf/prog_tests/trampoline_count.c +2 −2
  103. tools/testing/selftests/bpf/progs/atomic_bounds.c +24 −0
  104. tools/testing/selftests/bpf/progs/bind_perm.c +45 −0
  105. tools/testing/selftests/bpf/progs/bpf_iter.h +8 −0
  106. tools/testing/selftests/bpf/progs/bpf_iter_task_vma.c +58 −0
  107. tools/testing/selftests/bpf/progs/connect_force_port4.c +8 −0
  108. tools/testing/selftests/bpf/progs/connect_force_port6.c +8 −0
  109. tools/testing/selftests/bpf/progs/ima.c +25 −8
  110. tools/testing/selftests/bpf/progs/lsm.c +69 −0
  111. tools/testing/selftests/bpf/progs/recursion.c +46 −0
  112. tools/testing/selftests/bpf/progs/recvmsg4_prog.c +42 −0
  113. tools/testing/selftests/bpf/progs/recvmsg6_prog.c +48 −0
  114. tools/testing/selftests/bpf/progs/sendmsg4_prog.c +7 −0
  115. tools/testing/selftests/bpf/progs/sendmsg6_prog.c +5 −0
  116. tools/testing/selftests/bpf/progs/socket_cookie_prog.c +38 −9
  117. tools/testing/selftests/bpf/progs/sockopt_sk.c +21 −2
  118. tools/testing/selftests/bpf/progs/test_check_mtu.c +198 −0
  119. tools/testing/selftests/bpf/progs/test_cls_redirect.c +7 −0
  120. tools/testing/selftests/bpf/progs/test_global_func10.c +29 −0
  121. tools/testing/selftests/bpf/progs/test_global_func11.c +19 −0
  122. tools/testing/selftests/bpf/progs/test_global_func12.c +21 −0
  123. tools/testing/selftests/bpf/progs/test_global_func13.c +24 −0
  124. tools/testing/selftests/bpf/progs/test_global_func14.c +21 −0
  125. tools/testing/selftests/bpf/progs/test_global_func15.c +22 −0
  126. tools/testing/selftests/bpf/progs/test_global_func16.c +22 −0
  127. tools/testing/selftests/bpf/progs/test_global_func9.c +132 −0
  128. tools/testing/selftests/bpf/progs/test_global_func_args.c +91 −0
  129. tools/testing/selftests/bpf/progs/test_module_attach.c +10 −0
  130. tools/testing/selftests/bpf/progs/test_ns_current_pid_tgid.c +9 −19
  131. tools/testing/selftests/bpf/progs/test_stack_var_off.c +51 −0
  132. tools/testing/selftests/bpf/test_current_pid_tgid_new_ns.c +0 −160
  133. tools/testing/selftests/bpf/test_flow_dissector.c +1 −1
  134. tools/testing/selftests/bpf/test_progs.c +1 −1
  135. tools/testing/selftests/bpf/test_progs.h +0 −1
  136. tools/testing/selftests/bpf/test_sock_addr.c +12 −74
  137. tools/testing/selftests/bpf/test_socket_cookie.c +0 −208
  138. tools/testing/selftests/bpf/test_verifier.c +85 −3
  139. tools/testing/selftests/bpf/verifier/atomic_and.c +1 −1
  140. tools/testing/selftests/bpf/verifier/atomic_bounds.c +27 −0
  141. tools/testing/selftests/bpf/verifier/atomic_or.c +1 −1
  142. tools/testing/selftests/bpf/verifier/atomic_xor.c +1 −1
  143. tools/testing/selftests/bpf/verifier/basic_stack.c +1 −1
  144. tools/testing/selftests/bpf/verifier/calls.c +2 −2
  145. tools/testing/selftests/bpf/verifier/const_or.c +2 −2
  146. tools/testing/selftests/bpf/verifier/helper_access_var_len.c +6 −6
  147. tools/testing/selftests/bpf/verifier/int_ptr.c +3 −3
  148. tools/testing/selftests/bpf/verifier/jit.c +24 −0
  149. tools/testing/selftests/bpf/verifier/raw_stack.c +5 −5
  150. tools/testing/selftests/bpf/verifier/stack_ptr.c +13 −9
  151. tools/testing/selftests/bpf/verifier/unpriv.c +1 −1
  152. tools/testing/selftests/bpf/verifier/var_off.c +105 −10
  153. tools/testing/selftests/bpf/vmtest.sh +368 −0
  154. tools/testing/selftests/bpf/xdpxceiver.c +108 −117
  155. tools/testing/selftests/bpf/xdpxceiver.h +0 −2
  156. tools/testing/selftests/tc-testing/Makefile +1 −2
Documentation/bpf/bpf_design_QA.rst+6−0
@@ -208,6 +208,12 @@ data structures and compile with kernel internal headers. Both of these · 207 unmodified lines
208kernel internals are subject to change and can break with newer kernels208kernel internals are subject to change and can break with newer kernels
209such that the program needs to be adapted accordingly.209such that the program needs to be adapted accordingly.
210 210
211Q: Are tracepoints part of the stable ABI?
212------------------------------------------
213A: NO. Tracepoints are tied to internal implementation details hence they are
214subject to change and can break with newer kernels. BPF programs need to change
215accordingly when this happens.
216
211Q: How much stack space a BPF program uses?217Q: How much stack space a BPF program uses?
212-------------------------------------------218-------------------------------------------
213A: Currently all program types are limited to 512 bytes of stack219A: Currently all program types are limited to 512 bytes of stack
Documentation/bpf/bpf_devel_QA.rst+7−4
@@ -501,16 +501,19 @@ All LLVM releases can be found at: http://releases.llvm.org/ · 500 unmodified lines
501 501
502Q: Got it, so how do I build LLVM manually anyway?502Q: Got it, so how do I build LLVM manually anyway?
503--------------------------------------------------503--------------------------------------------------
504A: You need cmake and gcc-c++ as build requisites for LLVM. Once you have504A: We recommend that developers who want the fastest incremental builds
505that set up, proceed with building the latest LLVM and clang version505use the Ninja build system, you can find it in your system's package
506manager, usually the package is ninja or ninja-build.
507
508You need ninja, cmake and gcc-c++ as build requisites for LLVM. Once you
509have that set up, proceed with building the latest LLVM and clang version
506from the git repositories::510from the git repositories::
507 511
508 $ git clone https://github.com/llvm/llvm-project.git512 $ git clone https://github.com/llvm/llvm-project.git
509 $ mkdir -p llvm-project/llvm/build/install513 $ mkdir -p llvm-project/llvm/build
510 $ cd llvm-project/llvm/build514 $ cd llvm-project/llvm/build
511 $ cmake .. -G "Ninja" -DLLVM_TARGETS_TO_BUILD="BPF;X86" \515 $ cmake .. -G "Ninja" -DLLVM_TARGETS_TO_BUILD="BPF;X86" \
512 -DLLVM_ENABLE_PROJECTS="clang" \516 -DLLVM_ENABLE_PROJECTS="clang" \
513 -DBUILD_SHARED_LIBS=OFF \
514 -DCMAKE_BUILD_TYPE=Release \517 -DCMAKE_BUILD_TYPE=Release \
515 -DLLVM_BUILD_RUNTIME=OFF518 -DLLVM_BUILD_RUNTIME=OFF
516 $ ninja519 $ ninja
Documentation/networking/filter.rst+15−13
@@ -1048,12 +1048,12 @@ Unlike classic BPF instruction set, eBPF has generic load/store operations:: · 1047 unmodified lines
1048Where size is one of: BPF_B or BPF_H or BPF_W or BPF_DW.1048Where size is one of: BPF_B or BPF_H or BPF_W or BPF_DW.
1049 1049
1050It also includes atomic operations, which use the immediate field for extra1050It also includes atomic operations, which use the immediate field for extra
1051encoding.1051encoding::
1052 1052
1053 .imm = BPF_ADD, .code = BPF_ATOMIC | BPF_W | BPF_STX: lock xadd *(u32 *)(dst_reg + off16) += src_reg1053 .imm = BPF_ADD, .code = BPF_ATOMIC | BPF_W | BPF_STX: lock xadd *(u32 *)(dst_reg + off16) += src_reg
1054 .imm = BPF_ADD, .code = BPF_ATOMIC | BPF_DW | BPF_STX: lock xadd *(u64 *)(dst_reg + off16) += src_reg1054 .imm = BPF_ADD, .code = BPF_ATOMIC | BPF_DW | BPF_STX: lock xadd *(u64 *)(dst_reg + off16) += src_reg
1055 1055
1056The basic atomic operations supported are:1056The basic atomic operations supported are::
1057 1057
1058 BPF_ADD1058 BPF_ADD
1059 BPF_AND1059 BPF_AND
@@ -1066,33 +1066,35 @@ memory location addresed by ``dst_reg + off`` is atomically modified, with · 6 unmodified lines
1066immediate, then these operations also overwrite ``src_reg`` with the1066immediate, then these operations also overwrite ``src_reg`` with the
1067value that was in memory before it was modified.1067value that was in memory before it was modified.
1068 1068
1069The more special operations are:1069The more special operations are::
1070 1070
1071 BPF_XCHG1071 BPF_XCHG
1072 1072
1073This atomically exchanges ``src_reg`` with the value addressed by ``dst_reg +1073This atomically exchanges ``src_reg`` with the value addressed by ``dst_reg +
1074off``.1074off``. ::
1075 1075
1076 BPF_CMPXCHG1076 BPF_CMPXCHG
1077 1077
1078This atomically compares the value addressed by ``dst_reg + off`` with1078This atomically compares the value addressed by ``dst_reg + off`` with
1079``R0``. If they match it is replaced with ``src_reg``, The value that was there1079``R0``. If they match it is replaced with ``src_reg``. In either case, the
1080before is loaded back to ``R0``.1080value that was there before is zero-extended and loaded back to ``R0``.
1081 1081
1082Note that 1 and 2 byte atomic operations are not supported.1082Note that 1 and 2 byte atomic operations are not supported.
1083 1083
1084Except ``BPF_ADD`` _without_ ``BPF_FETCH`` (for legacy reasons), all 4 byte1084Clang can generate atomic instructions by default when ``-mcpu=v3`` is
1085atomic operations require alu32 mode. Clang enables this mode by default in1085enabled. If a lower version for ``-mcpu`` is set, the only atomic instruction
1086architecture v3 (``-mcpu=v3``). For older versions it can be enabled with1086Clang can generate is ``BPF_ADD`` *without* ``BPF_FETCH``. If you need to enable
1087the atomics features, while keeping a lower ``-mcpu`` version, you can use
1087``-Xclang -target-feature -Xclang +alu32``.1088``-Xclang -target-feature -Xclang +alu32``.
1088 1089
1089You may encounter BPF_XADD - this is a legacy name for BPF_ATOMIC, referring to1090You may encounter ``BPF_XADD`` - this is a legacy name for ``BPF_ATOMIC``,
1090the exclusive-add operation encoded when the immediate field is zero.1091referring to the exclusive-add operation encoded when the immediate field is
1092zero.
1091 1093
1092eBPF has one 16-byte instruction: BPF_LD | BPF_DW | BPF_IMM which consists1094eBPF has one 16-byte instruction: ``BPF_LD | BPF_DW | BPF_IMM`` which consists
1093of two consecutive ``struct bpf_insn`` 8-byte blocks and interpreted as single1095of two consecutive ``struct bpf_insn`` 8-byte blocks and interpreted as single
1094instruction that loads 64-bit immediate value into a dst_reg.1096instruction that loads 64-bit immediate value into a dst_reg.
1095Classic BPF has similar instruction: BPF_LD | BPF_W | BPF_IMM which loads1097Classic BPF has similar instruction: ``BPF_LD | BPF_W | BPF_IMM`` which loads
109632-bit immediate value into a register.109832-bit immediate value into a register.
1097 1099
1098eBPF verifier1100eBPF verifier
Makefile+12−1
@@ -1082,6 +1082,17 @@ ifdef CONFIG_STACK_VALIDATION · 1081 unmodified lines
1082 endif1082 endif
1083endif1083endif
1084 1084
1085PHONY += resolve_btfids_clean
1086
1087resolve_btfids_O = $(abspath $(objtree))/tools/bpf/resolve_btfids
1088
1089# tools/bpf/resolve_btfids directory might not exist
1090# in output directory, skip its clean in that case
1091resolve_btfids_clean:
1092ifneq ($(wildcard $(resolve_btfids_O)),)
1093 $(Q)$(MAKE) -sC $(srctree)/tools/bpf/resolve_btfids O=$(resolve_btfids_O) clean
1094endif
1095
1085ifdef CONFIG_BPF1096ifdef CONFIG_BPF
1086ifdef CONFIG_DEBUG_INFO_BTF1097ifdef CONFIG_DEBUG_INFO_BTF
1087 ifeq ($(has_libelf),1)1098 ifeq ($(has_libelf),1)
@@ -1491,7 +1502,7 @@ vmlinuxclean: · 403 unmodified lines
1491 $(Q)$(CONFIG_SHELL) $(srctree)/scripts/link-vmlinux.sh clean1502 $(Q)$(CONFIG_SHELL) $(srctree)/scripts/link-vmlinux.sh clean
1492 $(Q)$(if $(ARCH_POSTLINK), $(MAKE) -f $(ARCH_POSTLINK) clean)1503 $(Q)$(if $(ARCH_POSTLINK), $(MAKE) -f $(ARCH_POSTLINK) clean)
1493 1504
1494clean: archclean vmlinuxclean1505clean: archclean vmlinuxclean resolve_btfids_clean
1495 1506
1496# mrproper - Delete all generated files, including .config1507# mrproper - Delete all generated files, including .config
1497#1508#
arch/x86/net/bpf_jit_comp.c+158−47
@@ -869,8 +869,31 @@ static void detect_reg_usage(struct bpf_insn *insn, int insn_cnt, · 868 unmodified lines
869 }869 }
870}870}
871 871
872static int emit_nops(u8 **pprog, int len)
873{
874 u8 *prog = *pprog;
875 int i, noplen, cnt = 0;
876
877 while (len > 0) {
878 noplen = len;
879
880 if (noplen > ASM_NOP_MAX)
881 noplen = ASM_NOP_MAX;
882
883 for (i = 0; i < noplen; i++)
884 EMIT1(ideal_nops[noplen][i]);
885 len -= noplen;
886 }
887
888 *pprog = prog;
889
890 return cnt;
891}
892
893#define INSN_SZ_DIFF (((addrs[i] - addrs[i - 1]) - (prog - temp)))
894
872static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,895static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
873 int oldproglen, struct jit_context *ctx)896 int oldproglen, struct jit_context *ctx, bool jmp_padding)
874{897{
875 bool tail_call_reachable = bpf_prog->aux->tail_call_reachable;898 bool tail_call_reachable = bpf_prog->aux->tail_call_reachable;
876 struct bpf_insn *insn = bpf_prog->insnsi;899 struct bpf_insn *insn = bpf_prog->insnsi;
@@ -880,7 +903,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, · 3 unmodified lines
880 bool seen_exit = false;903 bool seen_exit = false;
881 u8 temp[BPF_MAX_INSN_SIZE + BPF_INSN_SAFETY];904 u8 temp[BPF_MAX_INSN_SIZE + BPF_INSN_SAFETY];
882 int i, cnt = 0, excnt = 0;905 int i, cnt = 0, excnt = 0;
883 int proglen = 0;906 int ilen, proglen = 0;
884 u8 *prog = temp;907 u8 *prog = temp;
885 int err;908 int err;
886 909
@@ -894,17 +917,24 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, · 7 unmodified lines
894 bpf_prog_was_classic(bpf_prog), tail_call_reachable,917 bpf_prog_was_classic(bpf_prog), tail_call_reachable,
895 bpf_prog->aux->func_idx != 0);918 bpf_prog->aux->func_idx != 0);
896 push_callee_regs(&prog, callee_regs_used);919 push_callee_regs(&prog, callee_regs_used);
897 addrs[0] = prog - temp;920
921 ilen = prog - temp;
922 if (image)
923 memcpy(image + proglen, temp, ilen);
924 proglen += ilen;
925 addrs[0] = proglen;
926 prog = temp;
898 927
899 for (i = 1; i <= insn_cnt; i++, insn++) {928 for (i = 1; i <= insn_cnt; i++, insn++) {
900 const s32 imm32 = insn->imm;929 const s32 imm32 = insn->imm;
901 u32 dst_reg = insn->dst_reg;930 u32 dst_reg = insn->dst_reg;
902 u32 src_reg = insn->src_reg;931 u32 src_reg = insn->src_reg;
903 u8 b2 = 0, b3 = 0;932 u8 b2 = 0, b3 = 0;
933 u8 *start_of_ldx;
904 s64 jmp_offset;934 s64 jmp_offset;
905 u8 jmp_cond;935 u8 jmp_cond;
906 int ilen;
907 u8 *func;936 u8 *func;
937 int nops;
908 938
909 switch (insn->code) {939 switch (insn->code) {
910 /* ALU */940 /* ALU */
@@ -1249,12 +1279,30 @@ st: if (is_imm8(insn->off)) · 338 unmodified lines
1249 case BPF_LDX | BPF_PROBE_MEM | BPF_W:1279 case BPF_LDX | BPF_PROBE_MEM | BPF_W:
1250 case BPF_LDX | BPF_MEM | BPF_DW:1280 case BPF_LDX | BPF_MEM | BPF_DW:
1251 case BPF_LDX | BPF_PROBE_MEM | BPF_DW:1281 case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
1282 if (BPF_MODE(insn->code) == BPF_PROBE_MEM) {
1283 /* test src_reg, src_reg */
1284 maybe_emit_mod(&prog, src_reg, src_reg, true); /* always 1 byte */
1285 EMIT2(0x85, add_2reg(0xC0, src_reg, src_reg));
1286 /* jne start_of_ldx */
1287 EMIT2(X86_JNE, 0);
1288 /* xor dst_reg, dst_reg */
1289 emit_mov_imm32(&prog, false, dst_reg, 0);
1290 /* jmp byte_after_ldx */
1291 EMIT2(0xEB, 0);
1292
1293 /* populate jmp_offset for JNE above */
1294 temp[4] = prog - temp - 5 /* sizeof(test + jne) */;
1295 start_of_ldx = prog;
1296 }
1252 emit_ldx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off);1297 emit_ldx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off);
1253 if (BPF_MODE(insn->code) == BPF_PROBE_MEM) {1298 if (BPF_MODE(insn->code) == BPF_PROBE_MEM) {
1254 struct exception_table_entry *ex;1299 struct exception_table_entry *ex;
1255 u8 *_insn = image + proglen;1300 u8 *_insn = image + proglen;
1256 s64 delta;1301 s64 delta;
1257 1302
1303 /* populate jmp_offset for JMP above */
1304 start_of_ldx[-1] = prog - start_of_ldx;
1305
1258 if (!bpf_prog->aux->extable)1306 if (!bpf_prog->aux->extable)
1259 break;1307 break;
1260 1308
@@ -1502,6 +1550,30 @@ emit_cond_jmp: /* Convert BPF opcode to x86 */ · 241 unmodified lines
1502 }1550 }
1503 jmp_offset = addrs[i + insn->off] - addrs[i];1551 jmp_offset = addrs[i + insn->off] - addrs[i];
1504 if (is_imm8(jmp_offset)) {1552 if (is_imm8(jmp_offset)) {
1553 if (jmp_padding) {
1554 /* To keep the jmp_offset valid, the extra bytes are
1555 * padded before the jump insn, so we substract the
1556 * 2 bytes of jmp_cond insn from INSN_SZ_DIFF.
1557 *
1558 * If the previous pass already emits an imm8
1559 * jmp_cond, then this BPF insn won't shrink, so
1560 * "nops" is 0.
1561 *
1562 * On the other hand, if the previous pass emits an
1563 * imm32 jmp_cond, the extra 4 bytes(*) is padded to
1564 * keep the image from shrinking further.
1565 *
1566 * (*) imm32 jmp_cond is 6 bytes, and imm8 jmp_cond
1567 * is 2 bytes, so the size difference is 4 bytes.
1568 */
1569 nops = INSN_SZ_DIFF - 2;
1570 if (nops != 0 && nops != 4) {
1571 pr_err("unexpected jmp_cond padding: %d bytes\n",
1572 nops);
1573 return -EFAULT;
1574 }
1575 cnt += emit_nops(&prog, nops);
1576 }
1505 EMIT2(jmp_cond, jmp_offset);1577 EMIT2(jmp_cond, jmp_offset);
1506 } else if (is_simm32(jmp_offset)) {1578 } else if (is_simm32(jmp_offset)) {
1507 EMIT2_off32(0x0F, jmp_cond + 0x10, jmp_offset);1579 EMIT2_off32(0x0F, jmp_cond + 0x10, jmp_offset);
@@ -1524,11 +1596,55 @@ emit_cond_jmp: /* Convert BPF opcode to x86 */ · 16 unmodified lines
1524 else1596 else
1525 jmp_offset = addrs[i + insn->off] - addrs[i];1597 jmp_offset = addrs[i + insn->off] - addrs[i];
1526 1598
1527 if (!jmp_offset)1599 if (!jmp_offset) {
1528 /* Optimize out nop jumps */1600 /*
1601 * If jmp_padding is enabled, the extra nops will
1602 * be inserted. Otherwise, optimize out nop jumps.
1603 */
1604 if (jmp_padding) {
1605 /* There are 3 possible conditions.
1606 * (1) This BPF_JA is already optimized out in
1607 * the previous run, so there is no need
1608 * to pad any extra byte (0 byte).
1609 * (2) The previous pass emits an imm8 jmp,
1610 * so we pad 2 bytes to match the previous
1611 * insn size.
1612 * (3) Similarly, the previous pass emits an
1613 * imm32 jmp, and 5 bytes is padded.
1614 */
1615 nops = INSN_SZ_DIFF;
1616 if (nops != 0 && nops != 2 && nops != 5) {
1617 pr_err("unexpected nop jump padding: %d bytes\n",
1618 nops);
1619 return -EFAULT;
1620 }
1621 cnt += emit_nops(&prog, nops);
1622 }
1529 break;1623 break;
1624 }
1530emit_jmp:1625emit_jmp:
1531 if (is_imm8(jmp_offset)) {1626 if (is_imm8(jmp_offset)) {
1627 if (jmp_padding) {
1628 /* To avoid breaking jmp_offset, the extra bytes
1629 * are padded before the actual jmp insn, so
1630 * 2 bytes is substracted from INSN_SZ_DIFF.
1631 *
1632 * If the previous pass already emits an imm8
1633 * jmp, there is nothing to pad (0 byte).
1634 *
1635 * If it emits an imm32 jmp (5 bytes) previously
1636 * and now an imm8 jmp (2 bytes), then we pad
1637 * (5 - 2 = 3) bytes to stop the image from
1638 * shrinking further.
1639 */
1640 nops = INSN_SZ_DIFF - 2;
1641 if (nops != 0 && nops != 3) {
1642 pr_err("unexpected jump padding: %d bytes\n",
1643 nops);
1644 return -EFAULT;
1645 }
1646 cnt += emit_nops(&prog, INSN_SZ_DIFF - 2);
1647 }
1532 EMIT2(0xEB, jmp_offset);1648 EMIT2(0xEB, jmp_offset);
1533 } else if (is_simm32(jmp_offset)) {1649 } else if (is_simm32(jmp_offset)) {
1534 EMIT1_off32(0xE9, jmp_offset);1650 EMIT1_off32(0xE9, jmp_offset);
@@ -1624,17 +1740,25 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, · 89 unmodified lines
1624 struct bpf_prog *p, int stack_size, bool mod_ret)1740 struct bpf_prog *p, int stack_size, bool mod_ret)
1625{1741{
1626 u8 *prog = *pprog;1742 u8 *prog = *pprog;
1743 u8 *jmp_insn;
1627 int cnt = 0;1744 int cnt = 0;
1628 1745
1629 if (p->aux->sleepable) {1746 /* arg1: mov rdi, progs[i] */
1630 if (emit_call(&prog, __bpf_prog_enter_sleepable, prog))1747 emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p);
1631 return -EINVAL;1748 if (emit_call(&prog,
1632 } else {1749 p->aux->sleepable ? __bpf_prog_enter_sleepable :
1633 if (emit_call(&prog, __bpf_prog_enter, prog))1750 __bpf_prog_enter, prog))
1634 return -EINVAL;1751 return -EINVAL;
1635 /* remember prog start time returned by __bpf_prog_enter */1752 /* remember prog start time returned by __bpf_prog_enter */
1636 emit_mov_reg(&prog, true, BPF_REG_6, BPF_REG_0);1753 emit_mov_reg(&prog, true, BPF_REG_6, BPF_REG_0);
1637 }1754
1755 /* if (__bpf_prog_enter*(prog) == 0)
1756 * goto skip_exec_of_prog;
1757 */
1758 EMIT3(0x48, 0x85, 0xC0); /* test rax,rax */
1759 /* emit 2 nops that will be replaced with JE insn */
1760 jmp_insn = prog;
1761 emit_nops(&prog, 2);
1638 1762
1639 /* arg1: lea rdi, [rbp - stack_size] */1763 /* arg1: lea rdi, [rbp - stack_size] */
1640 EMIT4(0x48, 0x8D, 0x7D, -stack_size);1764 EMIT4(0x48, 0x8D, 0x7D, -stack_size);
@@ -1654,43 +1778,23 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, · 13 unmodified lines
1654 if (mod_ret)1778 if (mod_ret)
1655 emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);1779 emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);
1656 1780
1657 if (p->aux->sleepable) {1781 /* replace 2 nops with JE insn, since jmp target is known */
1658 if (emit_call(&prog, __bpf_prog_exit_sleepable, prog))1782 jmp_insn[0] = X86_JE;
1659 return -EINVAL;1783 jmp_insn[1] = prog - jmp_insn - 2;
1660 } else {1784
1661 /* arg1: mov rdi, progs[i] */1785 /* arg1: mov rdi, progs[i] */
1662 emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32,1786 emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p);
1663 (u32) (long) p);1787 /* arg2: mov rsi, rbx <- start time in nsec */
1664 /* arg2: mov rsi, rbx <- start time in nsec */1788 emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_6);
1665 emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_6);1789 if (emit_call(&prog,
1666 if (emit_call(&prog, __bpf_prog_exit, prog))1790 p->aux->sleepable ? __bpf_prog_exit_sleepable :
1791 __bpf_prog_exit, prog))
1667 return -EINVAL;1792 return -EINVAL;
1668 }
1669 1793
1670 *pprog = prog;1794 *pprog = prog;
1671 return 0;1795 return 0;
1672}1796}
1673 1797
1674static void emit_nops(u8 **pprog, unsigned int len)
1675{
1676 unsigned int i, noplen;
1677 u8 *prog = *pprog;
1678 int cnt = 0;
1679
1680 while (len > 0) {
1681 noplen = len;
1682
1683 if (noplen > ASM_NOP_MAX)
1684 noplen = ASM_NOP_MAX;
1685
1686 for (i = 0; i < noplen; i++)
1687 EMIT1(ideal_nops[noplen][i]);
1688 len -= noplen;
1689 }
1690
1691 *pprog = prog;
1692}
1693
1694static void emit_align(u8 **pprog, u32 align)1798static void emit_align(u8 **pprog, u32 align)
1695{1799{
1696 u8 *target, *prog = *pprog;1800 u8 *target, *prog = *pprog;
@@ -2065,6 +2169,9 @@ struct x64_jit_data { · 368 unmodified lines
2065 struct jit_context ctx;2169 struct jit_context ctx;
2066};2170};
2067 2171
2172#define MAX_PASSES 20
2173#define PADDING_PASSES (MAX_PASSES - 5)
2174
2068struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)2175struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
2069{2176{
2070 struct bpf_binary_header *header = NULL;2177 struct bpf_binary_header *header = NULL;
@@ -2074,6 +2181,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 3 unmodified lines
2074 struct jit_context ctx = {};2181 struct jit_context ctx = {};
2075 bool tmp_blinded = false;2182 bool tmp_blinded = false;
2076 bool extra_pass = false;2183 bool extra_pass = false;
2184 bool padding = false;
2077 u8 *image = NULL;2185 u8 *image = NULL;
2078 int *addrs;2186 int *addrs;
2079 int pass;2187 int pass;
@@ -2110,6 +2218,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 30 unmodified lines
2110 image = jit_data->image;2218 image = jit_data->image;
2111 header = jit_data->header;2219 header = jit_data->header;
2112 extra_pass = true;2220 extra_pass = true;
2221 padding = true;
2113 goto skip_init_addrs;2222 goto skip_init_addrs;
2114 }2223 }
2115 addrs = kmalloc_array(prog->len + 1, sizeof(*addrs), GFP_KERNEL);2224 addrs = kmalloc_array(prog->len + 1, sizeof(*addrs), GFP_KERNEL);
@@ -2135,8 +2244,10 @@ skip_init_addrs: · 19 unmodified lines
2135 * may converge on the last pass. In such case do one more2244 * may converge on the last pass. In such case do one more
2136 * pass to emit the final image.2245 * pass to emit the final image.
2137 */2246 */
2138 for (pass = 0; pass < 20 || image; pass++) {2247 for (pass = 0; pass < MAX_PASSES || image; pass++) {
2139 proglen = do_jit(prog, addrs, image, oldproglen, &ctx);2248 if (!padding && pass >= PADDING_PASSES)
2249 padding = true;
2250 proglen = do_jit(prog, addrs, image, oldproglen, &ctx, padding);
2140 if (proglen <= 0) {2251 if (proglen <= 0) {
2141out_image:2252out_image:
2142 image = NULL;2253 image = NULL;
drivers/net/veth.c+60−34
@@ -35,6 +35,7 @@ · 34 unmodified lines
35#define VETH_XDP_HEADROOM (XDP_PACKET_HEADROOM + NET_IP_ALIGN)35#define VETH_XDP_HEADROOM (XDP_PACKET_HEADROOM + NET_IP_ALIGN)
36 36
37#define VETH_XDP_TX_BULK_SIZE 1637#define VETH_XDP_TX_BULK_SIZE 16
38#define VETH_XDP_BATCH 16
38 39
39struct veth_stats {40struct veth_stats {
40 u64 rx_drops;41 u64 rx_drops;
@@ -562,20 +563,13 @@ static int veth_xdp_tx(struct veth_rq *rq, struct xdp_buff *xdp, · 521 unmodified lines
562 return 0;563 return 0;
563}564}
564 565
565static struct sk_buff *veth_xdp_rcv_one(struct veth_rq *rq,566static struct xdp_frame *veth_xdp_rcv_one(struct veth_rq *rq,
566 struct xdp_frame *frame,567 struct xdp_frame *frame,
567 struct veth_xdp_tx_bq *bq,568 struct veth_xdp_tx_bq *bq,
568 struct veth_stats *stats)569 struct veth_stats *stats)
569{570{
570 void *hard_start = frame->data - frame->headroom;
571 int len = frame->len, delta = 0;
572 struct xdp_frame orig_frame;571 struct xdp_frame orig_frame;
573 struct bpf_prog *xdp_prog;572 struct bpf_prog *xdp_prog;
574 unsigned int headroom;
575 struct sk_buff *skb;
576
577 /* bpf_xdp_adjust_head() assures BPF cannot access xdp_frame area */
578 hard_start -= sizeof(struct xdp_frame);
579 573
580 rcu_read_lock();574 rcu_read_lock();
581 xdp_prog = rcu_dereference(rq->xdp_prog);575 xdp_prog = rcu_dereference(rq->xdp_prog);
@@ -590,8 +584,8 @@ static struct sk_buff *veth_xdp_rcv_one(struct veth_rq *rq, · 8 unmodified lines
590 584
591 switch (act) {585 switch (act) {
592 case XDP_PASS:586 case XDP_PASS:
593 delta = frame->data - xdp.data;587 if (xdp_update_frame_from_buff(&xdp, frame))
594 len = xdp.data_end - xdp.data;588 goto err_xdp;
595 break;589 break;
596 case XDP_TX:590 case XDP_TX:
597 orig_frame = *frame;591 orig_frame = *frame;
@@ -629,19 +623,7 @@ static struct sk_buff *veth_xdp_rcv_one(struct veth_rq *rq, · 31 unmodified lines
629 }623 }
630 rcu_read_unlock();624 rcu_read_unlock();
631 625
632 headroom = sizeof(struct xdp_frame) + frame->headroom - delta;626 return frame;
633 skb = veth_build_skb(hard_start, headroom, len, frame->frame_sz);
634 if (!skb) {
635 xdp_return_frame(frame);
636 stats->rx_drops++;
637 goto err;
638 }
639
640 xdp_release_frame(frame);
641 xdp_scrub_frame(frame);
642 skb->protocol = eth_type_trans(skb, rq->dev);
643err:
644 return skb;
645err_xdp:627err_xdp:
646 rcu_read_unlock();628 rcu_read_unlock();
647 xdp_return_frame(frame);629 xdp_return_frame(frame);
@@ -649,6 +631,37 @@ xdp_xmit: · 1 unmodified line
649 return NULL;631 return NULL;
650}632}
651 633
634/* frames array contains VETH_XDP_BATCH at most */
635static void veth_xdp_rcv_bulk_skb(struct veth_rq *rq, void **frames,
636 int n_xdpf, struct veth_xdp_tx_bq *bq,
637 struct veth_stats *stats)
638{
639 void *skbs[VETH_XDP_BATCH];
640 int i;
641
642 if (xdp_alloc_skb_bulk(skbs, n_xdpf,
643 GFP_ATOMIC | __GFP_ZERO) < 0) {
644 for (i = 0; i < n_xdpf; i++)
645 xdp_return_frame(frames[i]);
646 stats->rx_drops += n_xdpf;
647
648 return;
649 }
650
651 for (i = 0; i < n_xdpf; i++) {
652 struct sk_buff *skb = skbs[i];
653
654 skb = __xdp_build_skb_from_frame(frames[i], skb,
655 rq->dev);
656 if (!skb) {
657 xdp_return_frame(frames[i]);
658 stats->rx_drops++;
659 continue;
660 }
661 napi_gro_receive(&rq->xdp_napi, skb);
662 }
663}
664
652static struct sk_buff *veth_xdp_rcv_skb(struct veth_rq *rq,665static struct sk_buff *veth_xdp_rcv_skb(struct veth_rq *rq,
653 struct sk_buff *skb,666 struct sk_buff *skb,
654 struct veth_xdp_tx_bq *bq,667 struct veth_xdp_tx_bq *bq,
@@ -796,32 +809,45 @@ static int veth_xdp_rcv(struct veth_rq *rq, int budget, · 141 unmodified lines
796 struct veth_xdp_tx_bq *bq,809 struct veth_xdp_tx_bq *bq,
797 struct veth_stats *stats)810 struct veth_stats *stats)
798{811{
799 int i, done = 0;812 int i, done = 0, n_xdpf = 0;
813 void *xdpf[VETH_XDP_BATCH];
800 814
801 for (i = 0; i < budget; i++) {815 for (i = 0; i < budget; i++) {
802 void *ptr = __ptr_ring_consume(&rq->xdp_ring);816 void *ptr = __ptr_ring_consume(&rq->xdp_ring);
803 struct sk_buff *skb;
804 817
805 if (!ptr)818 if (!ptr)
806 break;819 break;
807 820
808 if (veth_is_xdp_frame(ptr)) {821 if (veth_is_xdp_frame(ptr)) {
822 /* ndo_xdp_xmit */
809 struct xdp_frame *frame = veth_ptr_to_xdp(ptr);823 struct xdp_frame *frame = veth_ptr_to_xdp(ptr);
810 824
811 stats->xdp_bytes += frame->len;825 stats->xdp_bytes += frame->len;
812 skb = veth_xdp_rcv_one(rq, frame, bq, stats);826 frame = veth_xdp_rcv_one(rq, frame, bq, stats);
827 if (frame) {
828 /* XDP_PASS */
829 xdpf[n_xdpf++] = frame;
830 if (n_xdpf == VETH_XDP_BATCH) {
831 veth_xdp_rcv_bulk_skb(rq, xdpf, n_xdpf,
832 bq, stats);
833 n_xdpf = 0;
834 }
835 }
813 } else {836 } else {
814 skb = ptr;837 /* ndo_start_xmit */
838 struct sk_buff *skb = ptr;
839
815 stats->xdp_bytes += skb->len;840 stats->xdp_bytes += skb->len;
816 skb = veth_xdp_rcv_skb(rq, skb, bq, stats);841 skb = veth_xdp_rcv_skb(rq, skb, bq, stats);
842 if (skb)
843 napi_gro_receive(&rq->xdp_napi, skb);
817 }844 }
818
819 if (skb)
820 napi_gro_receive(&rq->xdp_napi, skb);
821
822 done++;845 done++;
823 }846 }
824 847
848 if (n_xdpf)
849 veth_xdp_rcv_bulk_skb(rq, xdpf, n_xdpf, bq, stats);
850
825 u64_stats_update_begin(&rq->stats.syncp);851 u64_stats_update_begin(&rq->stats.syncp);
826 rq->stats.vs.xdp_redirect += stats->xdp_redirect;852 rq->stats.vs.xdp_redirect += stats->xdp_redirect;
827 rq->stats.vs.xdp_bytes += stats->xdp_bytes;853 rq->stats.vs.xdp_bytes += stats->xdp_bytes;
include/linux/bpf-cgroup.h+70−31
@@ -23,8 +23,8 @@ struct ctl_table_header; · 22 unmodified lines
23 23
24#ifdef CONFIG_CGROUP_BPF24#ifdef CONFIG_CGROUP_BPF
25 25
26extern struct static_key_false cgroup_bpf_enabled_key;26extern struct static_key_false cgroup_bpf_enabled_key[MAX_BPF_ATTACH_TYPE];
27#define cgroup_bpf_enabled static_branch_unlikely(&cgroup_bpf_enabled_key)27#define cgroup_bpf_enabled(type) static_branch_unlikely(&cgroup_bpf_enabled_key[type])
28 28
29DECLARE_PER_CPU(struct bpf_cgroup_storage*,29DECLARE_PER_CPU(struct bpf_cgroup_storage*,
30 bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]);30 bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]);
@@ -125,7 +125,8 @@ int __cgroup_bpf_run_filter_sk(struct sock *sk, · 94 unmodified lines
125int __cgroup_bpf_run_filter_sock_addr(struct sock *sk,125int __cgroup_bpf_run_filter_sock_addr(struct sock *sk,
126 struct sockaddr *uaddr,126 struct sockaddr *uaddr,
127 enum bpf_attach_type type,127 enum bpf_attach_type type,
128 void *t_ctx);128 void *t_ctx,
129 u32 *flags);
129 130
130int __cgroup_bpf_run_filter_sock_ops(struct sock *sk,131int __cgroup_bpf_run_filter_sock_ops(struct sock *sk,
131 struct bpf_sock_ops_kern *sock_ops,132 struct bpf_sock_ops_kern *sock_ops,
@@ -147,6 +148,10 @@ int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level, · 15 unmodified lines
147 int __user *optlen, int max_optlen,148 int __user *optlen, int max_optlen,
148 int retval);149 int retval);
149 150
151int __cgroup_bpf_run_filter_getsockopt_kern(struct sock *sk, int level,
152 int optname, void *optval,
153 int *optlen, int retval);
154
150static inline enum bpf_cgroup_storage_type cgroup_storage_type(155static inline enum bpf_cgroup_storage_type cgroup_storage_type(
151 struct bpf_map *map)156 struct bpf_map *map)
152{157{
@@ -185,7 +190,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, · 32 unmodified lines
185#define BPF_CGROUP_RUN_PROG_INET_INGRESS(sk, skb) \190#define BPF_CGROUP_RUN_PROG_INET_INGRESS(sk, skb) \
186({ \191({ \
187 int __ret = 0; \192 int __ret = 0; \
188 if (cgroup_bpf_enabled) \193 if (cgroup_bpf_enabled(BPF_CGROUP_INET_INGRESS)) \
189 __ret = __cgroup_bpf_run_filter_skb(sk, skb, \194 __ret = __cgroup_bpf_run_filter_skb(sk, skb, \
190 BPF_CGROUP_INET_INGRESS); \195 BPF_CGROUP_INET_INGRESS); \
191 \196 \
@@ -195,7 +200,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, · 3 unmodified lines
195#define BPF_CGROUP_RUN_PROG_INET_EGRESS(sk, skb) \200#define BPF_CGROUP_RUN_PROG_INET_EGRESS(sk, skb) \
196({ \201({ \
197 int __ret = 0; \202 int __ret = 0; \
198 if (cgroup_bpf_enabled && sk && sk == skb->sk) { \203 if (cgroup_bpf_enabled(BPF_CGROUP_INET_EGRESS) && sk && sk == skb->sk) { \
199 typeof(sk) __sk = sk_to_full_sk(sk); \204 typeof(sk) __sk = sk_to_full_sk(sk); \
200 if (sk_fullsock(__sk)) \205 if (sk_fullsock(__sk)) \
201 __ret = __cgroup_bpf_run_filter_skb(__sk, skb, \206 __ret = __cgroup_bpf_run_filter_skb(__sk, skb, \
@@ -207,7 +212,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, · 5 unmodified lines
207#define BPF_CGROUP_RUN_SK_PROG(sk, type) \212#define BPF_CGROUP_RUN_SK_PROG(sk, type) \
208({ \213({ \
209 int __ret = 0; \214 int __ret = 0; \
210 if (cgroup_bpf_enabled) { \215 if (cgroup_bpf_enabled(type)) { \
211 __ret = __cgroup_bpf_run_filter_sk(sk, type); \216 __ret = __cgroup_bpf_run_filter_sk(sk, type); \
212 } \217 } \
213 __ret; \218 __ret; \
@@ -227,33 +232,53 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, · 13 unmodified lines
227 232
228#define BPF_CGROUP_RUN_SA_PROG(sk, uaddr, type) \233#define BPF_CGROUP_RUN_SA_PROG(sk, uaddr, type) \
229({ \234({ \
235 u32 __unused_flags; \
230 int __ret = 0; \236 int __ret = 0; \
231 if (cgroup_bpf_enabled) \237 if (cgroup_bpf_enabled(type)) \
232 __ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type, \238 __ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type, \
233 NULL); \239 NULL, \
240 &__unused_flags); \
234 __ret; \241 __ret; \
235})242})
236 243
237#define BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, type, t_ctx) \244#define BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, type, t_ctx) \
238({ \245({ \
246 u32 __unused_flags; \
239 int __ret = 0; \247 int __ret = 0; \
240 if (cgroup_bpf_enabled) { \248 if (cgroup_bpf_enabled(type)) { \
241 lock_sock(sk); \249 lock_sock(sk); \
242 __ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type, \250 __ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type, \
243 t_ctx); \251 t_ctx, \
252 &__unused_flags); \
244 release_sock(sk); \253 release_sock(sk); \
245 } \254 } \
246 __ret; \255 __ret; \
247})256})
248 257
249#define BPF_CGROUP_RUN_PROG_INET4_BIND_LOCK(sk, uaddr) \258/* BPF_CGROUP_INET4_BIND and BPF_CGROUP_INET6_BIND can return extra flags
250 BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_INET4_BIND, NULL)259 * via upper bits of return code. The only flag that is supported
251 260 * (at bit position 0) is to indicate CAP_NET_BIND_SERVICE capability check
252#define BPF_CGROUP_RUN_PROG_INET6_BIND_LOCK(sk, uaddr) \261 * should be bypassed (BPF_RET_BIND_NO_CAP_NET_BIND_SERVICE).
253 BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_INET6_BIND, NULL)262 */
263#define BPF_CGROUP_RUN_PROG_INET_BIND_LOCK(sk, uaddr, type, bind_flags) \
264({ \
265 u32 __flags = 0; \
266 int __ret = 0; \
267 if (cgroup_bpf_enabled(type)) { \
268 lock_sock(sk); \
269 __ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type, \
270 NULL, &__flags); \
271 release_sock(sk); \
272 if (__flags & BPF_RET_BIND_NO_CAP_NET_BIND_SERVICE) \
273 *bind_flags |= BIND_NO_CAP_NET_BIND_SERVICE; \
274 } \
275 __ret; \
276})
254 277
255#define BPF_CGROUP_PRE_CONNECT_ENABLED(sk) (cgroup_bpf_enabled && \278#define BPF_CGROUP_PRE_CONNECT_ENABLED(sk) \
256 sk->sk_prot->pre_connect)279 ((cgroup_bpf_enabled(BPF_CGROUP_INET4_CONNECT) || \
280 cgroup_bpf_enabled(BPF_CGROUP_INET6_CONNECT)) && \
281 (sk)->sk_prot->pre_connect)
257 282
258#define BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr) \283#define BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr) \
259 BPF_CGROUP_RUN_SA_PROG(sk, uaddr, BPF_CGROUP_INET4_CONNECT)284 BPF_CGROUP_RUN_SA_PROG(sk, uaddr, BPF_CGROUP_INET4_CONNECT)
@@ -297,7 +322,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, · 37 unmodified lines
297#define BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(sock_ops, sk) \322#define BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(sock_ops, sk) \
298({ \323({ \
299 int __ret = 0; \324 int __ret = 0; \
300 if (cgroup_bpf_enabled) \325 if (cgroup_bpf_enabled(BPF_CGROUP_SOCK_OPS)) \
301 __ret = __cgroup_bpf_run_filter_sock_ops(sk, \326 __ret = __cgroup_bpf_run_filter_sock_ops(sk, \
302 sock_ops, \327 sock_ops, \
303 BPF_CGROUP_SOCK_OPS); \328 BPF_CGROUP_SOCK_OPS); \
@@ -307,7 +332,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, · 3 unmodified lines
307#define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) \332#define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) \
308({ \333({ \
309 int __ret = 0; \334 int __ret = 0; \
310 if (cgroup_bpf_enabled && (sock_ops)->sk) { \335 if (cgroup_bpf_enabled(BPF_CGROUP_SOCK_OPS) && (sock_ops)->sk) { \
311 typeof(sk) __sk = sk_to_full_sk((sock_ops)->sk); \336 typeof(sk) __sk = sk_to_full_sk((sock_ops)->sk); \
312 if (__sk && sk_fullsock(__sk)) \337 if (__sk && sk_fullsock(__sk)) \
313 __ret = __cgroup_bpf_run_filter_sock_ops(__sk, \338 __ret = __cgroup_bpf_run_filter_sock_ops(__sk, \
@@ -320,7 +345,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, · 6 unmodified lines
320#define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(type, major, minor, access) \345#define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(type, major, minor, access) \
321({ \346({ \
322 int __ret = 0; \347 int __ret = 0; \
323 if (cgroup_bpf_enabled) \348 if (cgroup_bpf_enabled(BPF_CGROUP_DEVICE)) \
324 __ret = __cgroup_bpf_check_dev_permission(type, major, minor, \349 __ret = __cgroup_bpf_check_dev_permission(type, major, minor, \
325 access, \350 access, \
326 BPF_CGROUP_DEVICE); \351 BPF_CGROUP_DEVICE); \
@@ -332,7 +357,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, · 5 unmodified lines
332#define BPF_CGROUP_RUN_PROG_SYSCTL(head, table, write, buf, count, pos) \357#define BPF_CGROUP_RUN_PROG_SYSCTL(head, table, write, buf, count, pos) \
333({ \358({ \
334 int __ret = 0; \359 int __ret = 0; \
335 if (cgroup_bpf_enabled) \360 if (cgroup_bpf_enabled(BPF_CGROUP_SYSCTL)) \
336 __ret = __cgroup_bpf_run_filter_sysctl(head, table, write, \361 __ret = __cgroup_bpf_run_filter_sysctl(head, table, write, \
337 buf, count, pos, \362 buf, count, pos, \
338 BPF_CGROUP_SYSCTL); \363 BPF_CGROUP_SYSCTL); \
@@ -343,7 +368,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, · 4 unmodified lines
343 kernel_optval) \368 kernel_optval) \
344({ \369({ \
345 int __ret = 0; \370 int __ret = 0; \
346 if (cgroup_bpf_enabled) \371 if (cgroup_bpf_enabled(BPF_CGROUP_SETSOCKOPT)) \
347 __ret = __cgroup_bpf_run_filter_setsockopt(sock, level, \372 __ret = __cgroup_bpf_run_filter_setsockopt(sock, level, \
348 optname, optval, \373 optname, optval, \
349 optlen, \374 optlen, \
@@ -354,7 +379,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, · 4 unmodified lines
354#define BPF_CGROUP_GETSOCKOPT_MAX_OPTLEN(optlen) \379#define BPF_CGROUP_GETSOCKOPT_MAX_OPTLEN(optlen) \
355({ \380({ \
356 int __ret = 0; \381 int __ret = 0; \
357 if (cgroup_bpf_enabled) \382 if (cgroup_bpf_enabled(BPF_CGROUP_GETSOCKOPT)) \
358 get_user(__ret, optlen); \383 get_user(__ret, optlen); \
359 __ret; \384 __ret; \
360})385})
@@ -363,11 +388,24 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, · 2 unmodified lines
363 max_optlen, retval) \388 max_optlen, retval) \
364({ \389({ \
365 int __ret = retval; \390 int __ret = retval; \
366 if (cgroup_bpf_enabled) \391 if (cgroup_bpf_enabled(BPF_CGROUP_GETSOCKOPT)) \
367 __ret = __cgroup_bpf_run_filter_getsockopt(sock, level, \392 if (!(sock)->sk_prot->bpf_bypass_getsockopt || \
368 optname, optval, \393 !INDIRECT_CALL_INET_1((sock)->sk_prot->bpf_bypass_getsockopt, \
369 optlen, max_optlen, \394 tcp_bpf_bypass_getsockopt, \
370 retval); \395 level, optname)) \
396 __ret = __cgroup_bpf_run_filter_getsockopt( \
397 sock, level, optname, optval, optlen, \
398 max_optlen, retval); \
399 __ret; \
400})
401
402#define BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sock, level, optname, optval, \
403 optlen, retval) \
404({ \
405 int __ret = retval; \
406 if (cgroup_bpf_enabled(BPF_CGROUP_GETSOCKOPT)) \
407 __ret = __cgroup_bpf_run_filter_getsockopt_kern( \
408 sock, level, optname, optval, optlen, retval); \
371 __ret; \409 __ret; \
372})410})
373 411
@@ -427,15 +465,14 @@ static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map, · 53 unmodified lines
427 return 0;465 return 0;
428}466}
429 467
430#define cgroup_bpf_enabled (0)468#define cgroup_bpf_enabled(type) (0)
431#define BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, type, t_ctx) ({ 0; })469#define BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, type, t_ctx) ({ 0; })
432#define BPF_CGROUP_PRE_CONNECT_ENABLED(sk) (0)470#define BPF_CGROUP_PRE_CONNECT_ENABLED(sk) (0)
433#define BPF_CGROUP_RUN_PROG_INET_INGRESS(sk,skb) ({ 0; })471#define BPF_CGROUP_RUN_PROG_INET_INGRESS(sk,skb) ({ 0; })
434#define BPF_CGROUP_RUN_PROG_INET_EGRESS(sk,skb) ({ 0; })472#define BPF_CGROUP_RUN_PROG_INET_EGRESS(sk,skb) ({ 0; })
435#define BPF_CGROUP_RUN_PROG_INET_SOCK(sk) ({ 0; })473#define BPF_CGROUP_RUN_PROG_INET_SOCK(sk) ({ 0; })
436#define BPF_CGROUP_RUN_PROG_INET_SOCK_RELEASE(sk) ({ 0; })474#define BPF_CGROUP_RUN_PROG_INET_SOCK_RELEASE(sk) ({ 0; })
437#define BPF_CGROUP_RUN_PROG_INET4_BIND_LOCK(sk, uaddr) ({ 0; })475#define BPF_CGROUP_RUN_PROG_INET_BIND_LOCK(sk, uaddr, type, flags) ({ 0; })
438#define BPF_CGROUP_RUN_PROG_INET6_BIND_LOCK(sk, uaddr) ({ 0; })
439#define BPF_CGROUP_RUN_PROG_INET4_POST_BIND(sk) ({ 0; })476#define BPF_CGROUP_RUN_PROG_INET4_POST_BIND(sk) ({ 0; })
440#define BPF_CGROUP_RUN_PROG_INET6_POST_BIND(sk) ({ 0; })477#define BPF_CGROUP_RUN_PROG_INET6_POST_BIND(sk) ({ 0; })
441#define BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr) ({ 0; })478#define BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr) ({ 0; })
@@ -452,6 +489,8 @@ static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map, · 10 unmodified lines
452#define BPF_CGROUP_GETSOCKOPT_MAX_OPTLEN(optlen) ({ 0; })489#define BPF_CGROUP_GETSOCKOPT_MAX_OPTLEN(optlen) ({ 0; })
453#define BPF_CGROUP_RUN_PROG_GETSOCKOPT(sock, level, optname, optval, \490#define BPF_CGROUP_RUN_PROG_GETSOCKOPT(sock, level, optname, optval, \
454 optlen, max_optlen, retval) ({ retval; })491 optlen, max_optlen, retval) ({ retval; })
492#define BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sock, level, optname, optval, \
493 optlen, retval) ({ retval; })
455#define BPF_CGROUP_RUN_PROG_SETSOCKOPT(sock, level, optname, optval, optlen, \494#define BPF_CGROUP_RUN_PROG_SETSOCKOPT(sock, level, optname, optval, optlen, \
456 kernel_optval) ({ 0; })495 kernel_optval) ({ 0; })
457 496
include/linux/bpf.h+43−31
@@ -14,7 +14,6 @@ · 13 unmodified lines
14#include <linux/numa.h>14#include <linux/numa.h>
15#include <linux/mm_types.h>15#include <linux/mm_types.h>
16#include <linux/wait.h>16#include <linux/wait.h>
17#include <linux/u64_stats_sync.h>
18#include <linux/refcount.h>17#include <linux/refcount.h>
19#include <linux/mutex.h>18#include <linux/mutex.h>
20#include <linux/module.h>19#include <linux/module.h>
@@ -507,12 +506,6 @@ enum bpf_cgroup_storage_type { · 486 unmodified lines
507 */506 */
508#define MAX_BPF_FUNC_ARGS 12507#define MAX_BPF_FUNC_ARGS 12
509 508
510struct bpf_prog_stats {
511 u64 cnt;
512 u64 nsecs;
513 struct u64_stats_sync syncp;
514} __aligned(2 * sizeof(u64));
515
516struct btf_func_model {509struct btf_func_model {
517 u8 ret_size;510 u8 ret_size;
518 u8 nr_args;511 u8 nr_args;
@@ -536,7 +529,7 @@ struct btf_func_model { · 17 unmodified lines
536/* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50529/* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50
537 * bytes on x86. Pick a number to fit into BPF_IMAGE_SIZE / 2530 * bytes on x86. Pick a number to fit into BPF_IMAGE_SIZE / 2
538 */531 */
539#define BPF_MAX_TRAMP_PROGS 40532#define BPF_MAX_TRAMP_PROGS 38
540 533
541struct bpf_tramp_progs {534struct bpf_tramp_progs {
542 struct bpf_prog *progs[BPF_MAX_TRAMP_PROGS];535 struct bpf_prog *progs[BPF_MAX_TRAMP_PROGS];
@@ -568,10 +561,10 @@ int arch_prepare_bpf_trampoline(void *image, void *image_end, · 25 unmodified lines
568 struct bpf_tramp_progs *tprogs,561 struct bpf_tramp_progs *tprogs,
569 void *orig_call);562 void *orig_call);
570/* these two functions are called from generated trampoline */563/* these two functions are called from generated trampoline */
571u64 notrace __bpf_prog_enter(void);564u64 notrace __bpf_prog_enter(struct bpf_prog *prog);
572void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start);565void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start);
573void notrace __bpf_prog_enter_sleepable(void);566u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog);
574void notrace __bpf_prog_exit_sleepable(void);567void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start);
575 568
576struct bpf_ksym {569struct bpf_ksym {
577 unsigned long start;570 unsigned long start;
@@ -845,7 +838,6 @@ struct bpf_prog_aux { · 267 unmodified lines
845 u32 linfo_idx;838 u32 linfo_idx;
846 u32 num_exentries;839 u32 num_exentries;
847 struct exception_table_entry *extable;840 struct exception_table_entry *extable;
848 struct bpf_prog_stats __percpu *stats;
849 union {841 union {
850 struct work_struct work;842 struct work_struct work;
851 struct rcu_head rcu;843 struct rcu_head rcu;
@@ -1073,6 +1065,34 @@ int bpf_prog_array_copy(struct bpf_prog_array *old_array, · 221 unmodified lines
1073 struct bpf_prog *include_prog,1065 struct bpf_prog *include_prog,
1074 struct bpf_prog_array **new_array);1066 struct bpf_prog_array **new_array);
1075 1067
1068/* BPF program asks to bypass CAP_NET_BIND_SERVICE in bind. */
1069#define BPF_RET_BIND_NO_CAP_NET_BIND_SERVICE (1 << 0)
1070/* BPF program asks to set CN on the packet. */
1071#define BPF_RET_SET_CN (1 << 0)
1072
1073#define BPF_PROG_RUN_ARRAY_FLAGS(array, ctx, func, ret_flags) \
1074 ({ \
1075 struct bpf_prog_array_item *_item; \
1076 struct bpf_prog *_prog; \
1077 struct bpf_prog_array *_array; \
1078 u32 _ret = 1; \
1079 u32 func_ret; \
1080 migrate_disable(); \
1081 rcu_read_lock(); \
1082 _array = rcu_dereference(array); \
1083 _item = &_array->items[0]; \
1084 while ((_prog = READ_ONCE(_item->prog))) { \
1085 bpf_cgroup_storage_set(_item->cgroup_storage); \
1086 func_ret = func(_prog, ctx); \
1087 _ret &= (func_ret & 1); \
1088 *(ret_flags) |= (func_ret >> 1); \
1089 _item++; \
1090 } \
1091 rcu_read_unlock(); \
1092 migrate_enable(); \
1093 _ret; \
1094 })
1095
1076#define __BPF_PROG_RUN_ARRAY(array, ctx, func, check_non_null) \1096#define __BPF_PROG_RUN_ARRAY(array, ctx, func, check_non_null) \
1077 ({ \1097 ({ \
1078 struct bpf_prog_array_item *_item; \1098 struct bpf_prog_array_item *_item; \
@@ -1120,25 +1140,11 @@ _out: \ · 41 unmodified lines
1120 */1140 */
1121#define BPF_PROG_CGROUP_INET_EGRESS_RUN_ARRAY(array, ctx, func) \1141#define BPF_PROG_CGROUP_INET_EGRESS_RUN_ARRAY(array, ctx, func) \
1122 ({ \1142 ({ \
1123 struct bpf_prog_array_item *_item; \1143 u32 _flags = 0; \
1124 struct bpf_prog *_prog; \1144 bool _cn; \
1125 struct bpf_prog_array *_array; \1145 u32 _ret; \
1126 u32 ret; \1146 _ret = BPF_PROG_RUN_ARRAY_FLAGS(array, ctx, func, &_flags); \
1127 u32 _ret = 1; \1147 _cn = _flags & BPF_RET_SET_CN; \
1128 u32 _cn = 0; \
1129 migrate_disable(); \
1130 rcu_read_lock(); \
1131 _array = rcu_dereference(array); \
1132 _item = &_array->items[0]; \
1133 while ((_prog = READ_ONCE(_item->prog))) { \
1134 bpf_cgroup_storage_set(_item->cgroup_storage); \
1135 ret = func(_prog, ctx); \
1136 _ret &= (ret & 1); \
1137 _cn |= (ret & 2); \
1138 _item++; \
1139 } \
1140 rcu_read_unlock(); \
1141 migrate_enable(); \
1142 if (_ret) \1148 if (_ret) \
1143 _ret = (_cn ? NET_XMIT_CN : NET_XMIT_SUCCESS); \1149 _ret = (_cn ? NET_XMIT_CN : NET_XMIT_SUCCESS); \
1144 else \1150 else \
@@ -1276,6 +1282,11 @@ static inline bool bpf_allow_ptr_leaks(void) · 131 unmodified lines
1276 return perfmon_capable();1282 return perfmon_capable();
1277}1283}
1278 1284
1285static inline bool bpf_allow_uninit_stack(void)
1286{
1287 return perfmon_capable();
1288}
1289
1279static inline bool bpf_allow_ptr_to_map_access(void)1290static inline bool bpf_allow_ptr_to_map_access(void)
1280{1291{
1281 return perfmon_capable();1292 return perfmon_capable();
@@ -1874,6 +1885,7 @@ extern const struct bpf_func_proto bpf_per_cpu_ptr_proto; · 592 unmodified lines
1874extern const struct bpf_func_proto bpf_this_cpu_ptr_proto;1885extern const struct bpf_func_proto bpf_this_cpu_ptr_proto;
1875extern const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto;1886extern const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto;
1876extern const struct bpf_func_proto bpf_sock_from_file_proto;1887extern const struct bpf_func_proto bpf_sock_from_file_proto;
1888extern const struct bpf_func_proto bpf_get_socket_ptr_cookie_proto;
1877 1889
1878const struct bpf_func_proto *bpf_tracing_func_proto(1890const struct bpf_func_proto *bpf_tracing_func_proto(
1879 enum bpf_func_id func_id, const struct bpf_prog *prog);1891 enum bpf_func_id func_id, const struct bpf_prog *prog);
include/linux/bpf_verifier.h+4−1
@@ -195,7 +195,7 @@ struct bpf_func_state { · 194 unmodified lines
195 * 0 = main function, 1 = first callee.195 * 0 = main function, 1 = first callee.
196 */196 */
197 u32 frameno;197 u32 frameno;
198 /* subprog number == index within subprog_stack_depth198 /* subprog number == index within subprog_info
199 * zero == main subprog199 * zero == main subprog
200 */200 */
201 u32 subprogno;201 u32 subprogno;
@@ -404,6 +404,7 @@ struct bpf_verifier_env { · 202 unmodified lines
404 u32 used_btf_cnt; /* number of used BTF objects */404 u32 used_btf_cnt; /* number of used BTF objects */
405 u32 id_gen; /* used to generate unique reg IDs */405 u32 id_gen; /* used to generate unique reg IDs */
406 bool allow_ptr_leaks;406 bool allow_ptr_leaks;
407 bool allow_uninit_stack;
407 bool allow_ptr_to_map_access;408 bool allow_ptr_to_map_access;
408 bool bpf_capable;409 bool bpf_capable;
409 bool bypass_spec_v1;410 bool bypass_spec_v1;
@@ -470,6 +471,8 @@ bpf_prog_offload_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt); · 60 unmodified lines
470 471
471int check_ctx_reg(struct bpf_verifier_env *env,472int check_ctx_reg(struct bpf_verifier_env *env,
472 const struct bpf_reg_state *reg, int regno);473 const struct bpf_reg_state *reg, int regno);
474int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
475 u32 regno, u32 mem_size);
473 476
474/* this lives here instead of in bpf.h because it needs to dereference tgt_prog */477/* this lives here instead of in bpf.h because it needs to dereference tgt_prog */
475static inline u64 bpf_trampoline_compute_key(const struct bpf_prog *tgt_prog,478static inline u64 bpf_trampoline_compute_key(const struct bpf_prog *tgt_prog,
include/linux/filter.h+18−3
@@ -22,6 +22,7 @@ · 21 unmodified lines
22#include <linux/vmalloc.h>22#include <linux/vmalloc.h>
23#include <linux/sockptr.h>23#include <linux/sockptr.h>
24#include <crypto/sha1.h>24#include <crypto/sha1.h>
25#include <linux/u64_stats_sync.h>
25 26
26#include <net/sch_generic.h>27#include <net/sch_generic.h>
27 28
@@ -539,6 +540,13 @@ struct bpf_binary_header { · 511 unmodified lines
539 u8 image[] __aligned(BPF_IMAGE_ALIGNMENT);540 u8 image[] __aligned(BPF_IMAGE_ALIGNMENT);
540};541};
541 542
543struct bpf_prog_stats {
544 u64 cnt;
545 u64 nsecs;
546 u64 misses;
547 struct u64_stats_sync syncp;
548} __aligned(2 * sizeof(u64));
549
542struct bpf_prog {550struct bpf_prog {
543 u16 pages; /* Number of allocated pages */551 u16 pages; /* Number of allocated pages */
544 u16 jited:1, /* Is our filter JIT'ed? */552 u16 jited:1, /* Is our filter JIT'ed? */
@@ -557,10 +565,12 @@ struct bpf_prog { · 12 unmodified lines
557 u32 len; /* Number of filter blocks */565 u32 len; /* Number of filter blocks */
558 u32 jited_len; /* Size of jited insns in bytes */566 u32 jited_len; /* Size of jited insns in bytes */
559 u8 tag[BPF_TAG_SIZE];567 u8 tag[BPF_TAG_SIZE];
560 struct bpf_prog_aux *aux; /* Auxiliary fields */568 struct bpf_prog_stats __percpu *stats;
561 struct sock_fprog_kern *orig_prog; /* Original BPF program */569 int __percpu *active;
562 unsigned int (*bpf_func)(const void *ctx,570 unsigned int (*bpf_func)(const void *ctx,
563 const struct bpf_insn *insn);571 const struct bpf_insn *insn);
572 struct bpf_prog_aux *aux; /* Auxiliary fields */
573 struct sock_fprog_kern *orig_prog; /* Original BPF program */
564 /* Instructions for interpreter */574 /* Instructions for interpreter */
565 struct sock_filter insns[0];575 struct sock_filter insns[0];
566 struct bpf_insn insnsi[];576 struct bpf_insn insnsi[];
@@ -581,7 +591,7 @@ DECLARE_STATIC_KEY_FALSE(bpf_stats_enabled_key); · 14 unmodified lines
581 struct bpf_prog_stats *__stats; \591 struct bpf_prog_stats *__stats; \
582 u64 __start = sched_clock(); \592 u64 __start = sched_clock(); \
583 __ret = dfunc(ctx, (prog)->insnsi, (prog)->bpf_func); \593 __ret = dfunc(ctx, (prog)->insnsi, (prog)->bpf_func); \
584 __stats = this_cpu_ptr(prog->aux->stats); \594 __stats = this_cpu_ptr(prog->stats); \
585 u64_stats_update_begin(&__stats->syncp); \595 u64_stats_update_begin(&__stats->syncp); \
586 __stats->cnt++; \596 __stats->cnt++; \
587 __stats->nsecs += sched_clock() - __start; \597 __stats->nsecs += sched_clock() - __start; \
@@ -1298,6 +1308,11 @@ struct bpf_sysctl_kern { · 710 unmodified lines
1298 u64 tmp_reg;1308 u64 tmp_reg;
1299};1309};
1300 1310
1311#define BPF_SOCKOPT_KERN_BUF_SIZE 32
1312struct bpf_sockopt_buf {
1313 u8 data[BPF_SOCKOPT_KERN_BUF_SIZE];
1314};
1315
1301struct bpf_sockopt_kern {1316struct bpf_sockopt_kern {
1302 struct sock *sk;1317 struct sock *sk;
1303 u8 *optval;1318 u8 *optval;
include/linux/indirect_call_wrapper.h+6−0
@@ -62,4 +62,10 @@ · 61 unmodified lines
62#define INDIRECT_CALL_INET(f, f2, f1, ...) f(__VA_ARGS__)62#define INDIRECT_CALL_INET(f, f2, f1, ...) f(__VA_ARGS__)
63#endif63#endif
64 64
65#if IS_ENABLED(CONFIG_INET)
66#define INDIRECT_CALL_INET_1(f, f1, ...) INDIRECT_CALL_1(f, f1, __VA_ARGS__)
67#else
68#define INDIRECT_CALL_INET_1(f, f1, ...) f(__VA_ARGS__)
69#endif
70
65#endif71#endif
include/linux/netdevice.h+30−2
@@ -3931,14 +3931,42 @@ int xdp_umem_query(struct net_device *dev, u16 queue_id); · 3930 unmodified lines
3931 3931
3932int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb);3932int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
3933int dev_forward_skb(struct net_device *dev, struct sk_buff *skb);3933int dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
3934int dev_forward_skb_nomtu(struct net_device *dev, struct sk_buff *skb);
3934bool is_skb_forwardable(const struct net_device *dev,3935bool is_skb_forwardable(const struct net_device *dev,
3935 const struct sk_buff *skb);3936 const struct sk_buff *skb);
3936 3937
3938static __always_inline bool __is_skb_forwardable(const struct net_device *dev,
3939 const struct sk_buff *skb,
3940 const bool check_mtu)
3941{
3942 const u32 vlan_hdr_len = 4; /* VLAN_HLEN */
3943 unsigned int len;
3944
3945 if (!(dev->flags & IFF_UP))
3946 return false;
3947
3948 if (!check_mtu)
3949 return true;
3950
3951 len = dev->mtu + dev->hard_header_len + vlan_hdr_len;
3952 if (skb->len <= len)
3953 return true;
3954
3955 /* if TSO is enabled, we don't care about the length as the packet
3956 * could be forwarded without being segmented before
3957 */
3958 if (skb_is_gso(skb))
3959 return true;
3960
3961 return false;
3962}
3963
3937static __always_inline int ____dev_forward_skb(struct net_device *dev,3964static __always_inline int ____dev_forward_skb(struct net_device *dev,
3938 struct sk_buff *skb)3965 struct sk_buff *skb,
3966 const bool check_mtu)
3939{3967{
3940 if (skb_orphan_frags(skb, GFP_ATOMIC) ||3968 if (skb_orphan_frags(skb, GFP_ATOMIC) ||
3941 unlikely(!is_skb_forwardable(dev, skb))) {3969 unlikely(!__is_skb_forwardable(dev, skb, check_mtu))) {
3942 atomic_long_inc(&dev->rx_dropped);3970 atomic_long_inc(&dev->rx_dropped);
3943 kfree_skb(skb);3971 kfree_skb(skb);
3944 return NET_RX_DROP;3972 return NET_RX_DROP;
include/linux/skmsg.h+0−1
@@ -390,7 +390,6 @@ static inline struct sk_psock *sk_psock_get(struct sock *sk) · 389 unmodified lines
390}390}
391 391
392void sk_psock_stop(struct sock *sk, struct sk_psock *psock);392void sk_psock_stop(struct sock *sk, struct sk_psock *psock);
393void sk_psock_destroy(struct rcu_head *rcu);
394void sk_psock_drop(struct sock *sk, struct sk_psock *psock);393void sk_psock_drop(struct sock *sk, struct sk_psock *psock);
395 394
396static inline void sk_psock_put(struct sock *sk, struct sk_psock *psock)395static inline void sk_psock_put(struct sock *sk, struct sk_psock *psock)
include/net/inet_common.h+2−0
@@ -41,6 +41,8 @@ int inet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len); · 40 unmodified lines
41#define BIND_WITH_LOCK (1 << 1)41#define BIND_WITH_LOCK (1 << 1)
42/* Called from BPF program. */42/* Called from BPF program. */
43#define BIND_FROM_BPF (1 << 2)43#define BIND_FROM_BPF (1 << 2)
44/* Skip CAP_NET_BIND_SERVICE check. */
45#define BIND_NO_CAP_NET_BIND_SERVICE (1 << 3)
44int __inet_bind(struct sock *sk, struct sockaddr *uaddr, int addr_len,46int __inet_bind(struct sock *sk, struct sockaddr *uaddr, int addr_len,
45 u32 flags);47 u32 flags);
46int inet_getname(struct socket *sock, struct sockaddr *uaddr,48int inet_getname(struct socket *sock, struct sockaddr *uaddr,
include/net/sock.h+2−0
@@ -1174,6 +1174,8 @@ struct proto { · 1173 unmodified lines
1174 1174
1175 int (*backlog_rcv) (struct sock *sk,1175 int (*backlog_rcv) (struct sock *sk,
1176 struct sk_buff *skb);1176 struct sk_buff *skb);
1177 bool (*bpf_bypass_getsockopt)(int level,
1178 int optname);
1177 1179
1178 void (*release_cb)(struct sock *sk);1180 void (*release_cb)(struct sock *sk);
1179 1181
include/net/tcp.h+1−0
@@ -403,6 +403,7 @@ __poll_t tcp_poll(struct file *file, struct socket *sock, · 402 unmodified lines
403 struct poll_table_struct *wait);403 struct poll_table_struct *wait);
404int tcp_getsockopt(struct sock *sk, int level, int optname,404int tcp_getsockopt(struct sock *sk, int level, int optname,
405 char __user *optval, int __user *optlen);405 char __user *optval, int __user *optlen);
406bool tcp_bpf_bypass_getsockopt(int level, int optname);
406int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,407int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
407 unsigned int optlen);408 unsigned int optlen);
408void tcp_set_keepalive(struct sock *sk, int val);409void tcp_set_keepalive(struct sock *sk, int val);
include/net/xdp.h+6−0
@@ -164,6 +164,12 @@ void xdp_warn(const char *msg, const char *func, const int line); · 163 unmodified lines
164#define XDP_WARN(msg) xdp_warn(msg, __func__, __LINE__)164#define XDP_WARN(msg) xdp_warn(msg, __func__, __LINE__)
165 165
166struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp);166struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp);
167struct sk_buff *__xdp_build_skb_from_frame(struct xdp_frame *xdpf,
168 struct sk_buff *skb,
169 struct net_device *dev);
170struct sk_buff *xdp_build_skb_from_frame(struct xdp_frame *xdpf,
171 struct net_device *dev);
172int xdp_alloc_skb_bulk(void **skbs, int n_skb, gfp_t gfp);
167 173
168static inline174static inline
169void xdp_convert_frame_to_buff(struct xdp_frame *frame, struct xdp_buff *xdp)175void xdp_convert_frame_to_buff(struct xdp_frame *frame, struct xdp_buff *xdp)
include/trace/bpf_probe.h+10−2
@@ -55,8 +55,7 @@ · 54 unmodified lines
55/* tracepoints with more than 12 arguments will hit build error */55/* tracepoints with more than 12 arguments will hit build error */
56#define CAST_TO_U64(...) CONCATENATE(__CAST, COUNT_ARGS(__VA_ARGS__))(__VA_ARGS__)56#define CAST_TO_U64(...) CONCATENATE(__CAST, COUNT_ARGS(__VA_ARGS__))(__VA_ARGS__)
57 57
58#undef DECLARE_EVENT_CLASS58#define __BPF_DECLARE_TRACE(call, proto, args) \
59#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \
60static notrace void \59static notrace void \
61__bpf_trace_##call(void *__data, proto) \60__bpf_trace_##call(void *__data, proto) \
62{ \61{ \
@@ -64,6 +63,10 @@ __bpf_trace_##call(void *__data, proto) \ · 1 unmodified line
64 CONCATENATE(bpf_trace_run, COUNT_ARGS(args))(prog, CAST_TO_U64(args)); \63 CONCATENATE(bpf_trace_run, COUNT_ARGS(args))(prog, CAST_TO_U64(args)); \
65}64}
66 65
66#undef DECLARE_EVENT_CLASS
67#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \
68 __BPF_DECLARE_TRACE(call, PARAMS(proto), PARAMS(args))
69
67/*70/*
68 * This part is compiled out, it is only here as a build time check71 * This part is compiled out, it is only here as a build time check
69 * to make sure that if the tracepoint handling changes, the72 * to make sure that if the tracepoint handling changes, the
@@ -111,6 +114,11 @@ __DEFINE_EVENT(template, call, PARAMS(proto), PARAMS(args), size) · 41 unmodified lines
111#define DEFINE_EVENT_PRINT(template, name, proto, args, print) \114#define DEFINE_EVENT_PRINT(template, name, proto, args, print) \
112 DEFINE_EVENT(template, name, PARAMS(proto), PARAMS(args))115 DEFINE_EVENT(template, name, PARAMS(proto), PARAMS(args))
113 116
117#undef DECLARE_TRACE
118#define DECLARE_TRACE(call, proto, args) \
119 __BPF_DECLARE_TRACE(call, PARAMS(proto), PARAMS(args)) \
120 __DEFINE_EVENT(call, call, PARAMS(proto), PARAMS(args), 0)
121
114#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)122#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
115 123
116#undef DEFINE_EVENT_WRITABLE124#undef DEFINE_EVENT_WRITABLE
include/uapi/linux/bpf.h+97−6
@@ -1656,22 +1656,30 @@ union bpf_attr { · 1655 unmodified lines
1656 * networking traffic statistics as it provides a global socket1656 * networking traffic statistics as it provides a global socket
1657 * identifier that can be assumed unique.1657 * identifier that can be assumed unique.
1658 * Return1658 * Return
1659 * A 8-byte long non-decreasing number on success, or 0 if the1659 * A 8-byte long unique number on success, or 0 if the socket
1660 * socket field is missing inside *skb*.1660 * field is missing inside *skb*.
1661 *1661 *
1662 * u64 bpf_get_socket_cookie(struct bpf_sock_addr *ctx)1662 * u64 bpf_get_socket_cookie(struct bpf_sock_addr *ctx)
1663 * Description1663 * Description
1664 * Equivalent to bpf_get_socket_cookie() helper that accepts1664 * Equivalent to bpf_get_socket_cookie() helper that accepts
1665 * *skb*, but gets socket from **struct bpf_sock_addr** context.1665 * *skb*, but gets socket from **struct bpf_sock_addr** context.
1666 * Return1666 * Return
1667 * A 8-byte long non-decreasing number.1667 * A 8-byte long unique number.
1668 *1668 *
1669 * u64 bpf_get_socket_cookie(struct bpf_sock_ops *ctx)1669 * u64 bpf_get_socket_cookie(struct bpf_sock_ops *ctx)
1670 * Description1670 * Description
1671 * Equivalent to **bpf_get_socket_cookie**\ () helper that accepts1671 * Equivalent to **bpf_get_socket_cookie**\ () helper that accepts
1672 * *skb*, but gets socket from **struct bpf_sock_ops** context.1672 * *skb*, but gets socket from **struct bpf_sock_ops** context.
1673 * Return1673 * Return
1674 * A 8-byte long non-decreasing number.1674 * A 8-byte long unique number.
1675 *
1676 * u64 bpf_get_socket_cookie(struct sock *sk)
1677 * Description
1678 * Equivalent to **bpf_get_socket_cookie**\ () helper that accepts
1679 * *sk*, but gets socket from a BTF **struct sock**. This helper
1680 * also works for sleepable programs.
1681 * Return
1682 * A 8-byte long unique number or 0 if *sk* is NULL.
1675 *1683 *
1676 * u32 bpf_get_socket_uid(struct sk_buff *skb)1684 * u32 bpf_get_socket_uid(struct sk_buff *skb)
1677 * Return1685 * Return
@@ -2231,6 +2239,9 @@ union bpf_attr { · 553 unmodified lines
2231 * * > 0 one of **BPF_FIB_LKUP_RET_** codes explaining why the2239 * * > 0 one of **BPF_FIB_LKUP_RET_** codes explaining why the
2232 * packet is not forwarded or needs assist from full stack2240 * packet is not forwarded or needs assist from full stack
2233 *2241 *
2242 * If lookup fails with BPF_FIB_LKUP_RET_FRAG_NEEDED, then the MTU
2243 * was exceeded and output params->mtu_result contains the MTU.
2244 *
2234 * long bpf_sock_hash_update(struct bpf_sock_ops *skops, struct bpf_map *map, void *key, u64 flags)2245 * long bpf_sock_hash_update(struct bpf_sock_ops *skops, struct bpf_map *map, void *key, u64 flags)
2235 * Description2246 * Description
2236 * Add an entry to, or update a sockhash *map* referencing sockets.2247 * Add an entry to, or update a sockhash *map* referencing sockets.
@@ -3836,6 +3847,69 @@ union bpf_attr { · 1599 unmodified lines
3836 * Return3847 * Return
3837 * A pointer to a struct socket on success or NULL if the file is3848 * A pointer to a struct socket on success or NULL if the file is
3838 * not a socket.3849 * not a socket.
3850 *
3851 * long bpf_check_mtu(void *ctx, u32 ifindex, u32 *mtu_len, s32 len_diff, u64 flags)
3852 * Description
3853
3854 * Check ctx packet size against exceeding MTU of net device (based
3855 * on *ifindex*). This helper will likely be used in combination
3856 * with helpers that adjust/change the packet size.
3857 *
3858 * The argument *len_diff* can be used for querying with a planned
3859 * size change. This allows to check MTU prior to changing packet
3860 * ctx. Providing an *len_diff* adjustment that is larger than the
3861 * actual packet size (resulting in negative packet size) will in
3862 * principle not exceed the MTU, why it is not considered a
3863 * failure. Other BPF-helpers are needed for performing the
3864 * planned size change, why the responsability for catch a negative
3865 * packet size belong in those helpers.
3866 *
3867 * Specifying *ifindex* zero means the MTU check is performed
3868 * against the current net device. This is practical if this isn't
3869 * used prior to redirect.
3870 *
3871 * The Linux kernel route table can configure MTUs on a more
3872 * specific per route level, which is not provided by this helper.
3873 * For route level MTU checks use the **bpf_fib_lookup**\ ()
3874 * helper.
3875 *
3876 * *ctx* is either **struct xdp_md** for XDP programs or
3877 * **struct sk_buff** for tc cls_act programs.
3878 *
3879 * The *flags* argument can be a combination of one or more of the
3880 * following values:
3881 *
3882 * **BPF_MTU_CHK_SEGS**
3883 * This flag will only works for *ctx* **struct sk_buff**.
3884 * If packet context contains extra packet segment buffers
3885 * (often knows as GSO skb), then MTU check is harder to
3886 * check at this point, because in transmit path it is
3887 * possible for the skb packet to get re-segmented
3888 * (depending on net device features). This could still be
3889 * a MTU violation, so this flag enables performing MTU
3890 * check against segments, with a different violation
3891 * return code to tell it apart. Check cannot use len_diff.
3892 *
3893 * On return *mtu_len* pointer contains the MTU value of the net
3894 * device. Remember the net device configured MTU is the L3 size,
3895 * which is returned here and XDP and TX length operate at L2.
3896 * Helper take this into account for you, but remember when using
3897 * MTU value in your BPF-code. On input *mtu_len* must be a valid
3898 * pointer and be initialized (to zero), else verifier will reject
3899 * BPF program.
3900 *
3901 * Return
3902 * * 0 on success, and populate MTU value in *mtu_len* pointer.
3903 *
3904 * * < 0 if any input argument is invalid (*mtu_len* not updated)
3905 *
3906 * MTU violations return positive values, but also populate MTU
3907 * value in *mtu_len* pointer, as this can be needed for
3908 * implementing PMTU handing:
3909 *
3910 * * **BPF_MTU_CHK_RET_FRAG_NEEDED**
3911 * * **BPF_MTU_CHK_RET_SEGS_TOOBIG**
3912 *
3839 */3913 */
3840#define __BPF_FUNC_MAPPER(FN) \3914#define __BPF_FUNC_MAPPER(FN) \
3841 FN(unspec), \3915 FN(unspec), \
@@ -4001,6 +4075,7 @@ union bpf_attr { · 159 unmodified lines
4001 FN(ktime_get_coarse_ns), \4075 FN(ktime_get_coarse_ns), \
4002 FN(ima_inode_hash), \4076 FN(ima_inode_hash), \
4003 FN(sock_from_file), \4077 FN(sock_from_file), \
4078 FN(check_mtu), \
4004 /* */4079 /* */
4005 4080
4006/* integer value in 'imm' field of BPF_CALL instruction selects which helper4081/* integer value in 'imm' field of BPF_CALL instruction selects which helper
@@ -4501,6 +4576,7 @@ struct bpf_prog_info { · 494 unmodified lines
4501 __aligned_u64 prog_tags;4576 __aligned_u64 prog_tags;
4502 __u64 run_time_ns;4577 __u64 run_time_ns;
4503 __u64 run_cnt;4578 __u64 run_cnt;
4579 __u64 recursion_misses;
4504} __attribute__((aligned(8)));4580} __attribute__((aligned(8)));
4505 4581
4506struct bpf_map_info {4582struct bpf_map_info {
@@ -4981,9 +5057,13 @@ struct bpf_fib_lookup { · 474 unmodified lines
4981 __be16 sport;5057 __be16 sport;
4982 __be16 dport;5058 __be16 dport;
4983 5059
4984 /* total length of packet from network header - used for MTU check */5060 union { /* used for MTU check */
4985 __u16 tot_len;5061 /* input to lookup */
5062 __u16 tot_len; /* L3 length from network hdr (iph->tot_len) */
4986 5063
5064 /* output: MTU value */
5065 __u16 mtu_result;
5066 };
4987 /* input: L3 device index for lookup5067 /* input: L3 device index for lookup
4988 * output: device index from FIB lookup5068 * output: device index from FIB lookup
4989 */5069 */
@@ -5029,6 +5109,17 @@ struct bpf_redir_neigh { · 39 unmodified lines
5029 };5109 };
5030};5110};
5031 5111
5112/* bpf_check_mtu flags*/
5113enum bpf_check_mtu_flags {
5114 BPF_MTU_CHK_SEGS = (1U << 0),
5115};
5116
5117enum bpf_check_mtu_ret {
5118 BPF_MTU_CHK_RET_SUCCESS, /* check and lookup successful */
5119 BPF_MTU_CHK_RET_FRAG_NEEDED, /* fragmentation required to fwd */
5120 BPF_MTU_CHK_RET_SEGS_TOOBIG, /* GSO re-segmentation needed to fwd */
5121};
5122
5032enum bpf_task_fd_type {5123enum bpf_task_fd_type {
5033 BPF_FD_TYPE_RAW_TRACEPOINT, /* tp name */5124 BPF_FD_TYPE_RAW_TRACEPOINT, /* tp name */
5034 BPF_FD_TYPE_TRACEPOINT, /* tp name */5125 BPF_FD_TYPE_TRACEPOINT, /* tp name */
kernel/bpf/bpf_iter.c+1−1
@@ -287,7 +287,7 @@ int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info) · 286 unmodified lines
287{287{
288 struct bpf_iter_target_info *tinfo;288 struct bpf_iter_target_info *tinfo;
289 289
290 tinfo = kmalloc(sizeof(*tinfo), GFP_KERNEL);290 tinfo = kzalloc(sizeof(*tinfo), GFP_KERNEL);
291 if (!tinfo)291 if (!tinfo)
292 return -ENOMEM;292 return -ENOMEM;
293 293
kernel/bpf/bpf_lru_list.c+4−3
@@ -502,13 +502,14 @@ struct bpf_lru_node *bpf_lru_pop_free(struct bpf_lru *lru, u32 hash) · 501 unmodified lines
502static void bpf_common_lru_push_free(struct bpf_lru *lru,502static void bpf_common_lru_push_free(struct bpf_lru *lru,
503 struct bpf_lru_node *node)503 struct bpf_lru_node *node)
504{504{
505 u8 node_type = READ_ONCE(node->type);
505 unsigned long flags;506 unsigned long flags;
506 507
507 if (WARN_ON_ONCE(node->type == BPF_LRU_LIST_T_FREE) ||508 if (WARN_ON_ONCE(node_type == BPF_LRU_LIST_T_FREE) ||
508 WARN_ON_ONCE(node->type == BPF_LRU_LOCAL_LIST_T_FREE))509 WARN_ON_ONCE(node_type == BPF_LRU_LOCAL_LIST_T_FREE))
509 return;510 return;
510 511
511 if (node->type == BPF_LRU_LOCAL_LIST_T_PENDING) {512 if (node_type == BPF_LRU_LOCAL_LIST_T_PENDING) {
512 struct bpf_lru_locallist *loc_l;513 struct bpf_lru_locallist *loc_l;
513 514
514 loc_l = per_cpu_ptr(lru->common_lru.local_list, node->cpu);515 loc_l = per_cpu_ptr(lru->common_lru.local_list, node->cpu);
kernel/bpf/btf.c+55−21
@@ -3540,11 +3540,6 @@ static s32 btf_datasec_check_meta(struct btf_verifier_env *env, · 3539 unmodified lines
3540 return -EINVAL;3540 return -EINVAL;
3541 }3541 }
3542 3542
3543 if (!btf_type_vlen(t)) {
3544 btf_verifier_log_type(env, t, "vlen == 0");
3545 return -EINVAL;
3546 }
3547
3548 if (!t->size) {3543 if (!t->size) {
3549 btf_verifier_log_type(env, t, "size == 0");3544 btf_verifier_log_type(env, t, "size == 0");
3550 return -EINVAL;3545 return -EINVAL;
@@ -5296,15 +5291,16 @@ int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *pr · 1745 unmodified lines
5296 * Only PTR_TO_CTX and SCALAR_VALUE states are recognized.5291 * Only PTR_TO_CTX and SCALAR_VALUE states are recognized.
5297 */5292 */
5298int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog,5293int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog,
5299 struct bpf_reg_state *reg)5294 struct bpf_reg_state *regs)
5300{5295{
5301 struct bpf_verifier_log *log = &env->log;5296 struct bpf_verifier_log *log = &env->log;
5302 struct bpf_prog *prog = env->prog;5297 struct bpf_prog *prog = env->prog;
5303 struct btf *btf = prog->aux->btf;5298 struct btf *btf = prog->aux->btf;
5304 const struct btf_param *args;5299 const struct btf_param *args;
5305 const struct btf_type *t;5300 const struct btf_type *t, *ref_t;
5306 u32 i, nargs, btf_id;5301 u32 i, nargs, btf_id, type_size;
5307 const char *tname;5302 const char *tname;
5303 bool is_global;
5308 5304
5309 if (!prog->aux->func_info)5305 if (!prog->aux->func_info)
5310 return -EINVAL;5306 return -EINVAL;
@@ -5338,38 +5334,57 @@ int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, · 27 unmodified lines
5338 bpf_log(log, "Function %s has %d > 5 args\n", tname, nargs);5334 bpf_log(log, "Function %s has %d > 5 args\n", tname, nargs);
5339 goto out;5335 goto out;
5340 }5336 }
5337
5338 is_global = prog->aux->func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL;
5341 /* check that BTF function arguments match actual types that the5339 /* check that BTF function arguments match actual types that the
5342 * verifier sees.5340 * verifier sees.
5343 */5341 */
5344 for (i = 0; i < nargs; i++) {5342 for (i = 0; i < nargs; i++) {
5343 struct bpf_reg_state *reg = &regs[i + 1];
5344
5345 t = btf_type_by_id(btf, args[i].type);5345 t = btf_type_by_id(btf, args[i].type);
5346 while (btf_type_is_modifier(t))5346 while (btf_type_is_modifier(t))
5347 t = btf_type_by_id(btf, t->type);5347 t = btf_type_by_id(btf, t->type);
5348 if (btf_type_is_int(t) || btf_type_is_enum(t)) {5348 if (btf_type_is_int(t) || btf_type_is_enum(t)) {
5349 if (reg[i + 1].type == SCALAR_VALUE)5349 if (reg->type == SCALAR_VALUE)
5350 continue;5350 continue;
5351 bpf_log(log, "R%d is not a scalar\n", i + 1);5351 bpf_log(log, "R%d is not a scalar\n", i + 1);
5352 goto out;5352 goto out;
5353 }5353 }
5354 if (btf_type_is_ptr(t)) {5354 if (btf_type_is_ptr(t)) {
5355 if (reg[i + 1].type == SCALAR_VALUE) {
5356 bpf_log(log, "R%d is not a pointer\n", i + 1);
5357 goto out;
5358 }
5359 /* If function expects ctx type in BTF check that caller5355 /* If function expects ctx type in BTF check that caller
5360 * is passing PTR_TO_CTX.5356 * is passing PTR_TO_CTX.
5361 */5357 */
5362 if (btf_get_prog_ctx_type(log, btf, t, prog->type, i)) {5358 if (btf_get_prog_ctx_type(log, btf, t, prog->type, i)) {
5363 if (reg[i + 1].type != PTR_TO_CTX) {5359 if (reg->type != PTR_TO_CTX) {
5364 bpf_log(log,5360 bpf_log(log,
5365 "arg#%d expected pointer to ctx, but got %s\n",5361 "arg#%d expected pointer to ctx, but got %s\n",
5366 i, btf_kind_str[BTF_INFO_KIND(t->info)]);5362 i, btf_kind_str[BTF_INFO_KIND(t->info)]);
5367 goto out;5363 goto out;
5368 }5364 }
5369 if (check_ctx_reg(env, &reg[i + 1], i + 1))5365 if (check_ctx_reg(env, reg, i + 1))
5370 goto out;5366 goto out;
5371 continue;5367 continue;
5372 }5368 }
5369
5370 if (!is_global)
5371 goto out;
5372
5373 t = btf_type_skip_modifiers(btf, t->type, NULL);
5374
5375 ref_t = btf_resolve_size(btf, t, &type_size);
5376 if (IS_ERR(ref_t)) {
5377 bpf_log(log,
5378 "arg#%d reference type('%s %s') size cannot be determined: %ld\n",
5379 i, btf_type_str(t), btf_name_by_offset(btf, t->name_off),
5380 PTR_ERR(ref_t));
5381 goto out;
5382 }
5383
5384 if (check_mem_reg(env, reg, i + 1, type_size))
5385 goto out;
5386
5387 continue;
5373 }5388 }
5374 bpf_log(log, "Unrecognized arg#%d type %s\n",5389 bpf_log(log, "Unrecognized arg#%d type %s\n",
5375 i, btf_kind_str[BTF_INFO_KIND(t->info)]);5390 i, btf_kind_str[BTF_INFO_KIND(t->info)]);
@@ -5393,14 +5408,14 @@ out: · 17 unmodified lines
5393 * (either PTR_TO_CTX or SCALAR_VALUE).5408 * (either PTR_TO_CTX or SCALAR_VALUE).
5394 */5409 */
5395int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog,5410int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog,
5396 struct bpf_reg_state *reg)5411 struct bpf_reg_state *regs)
5397{5412{
5398 struct bpf_verifier_log *log = &env->log;5413 struct bpf_verifier_log *log = &env->log;
5399 struct bpf_prog *prog = env->prog;5414 struct bpf_prog *prog = env->prog;
5400 enum bpf_prog_type prog_type = prog->type;5415 enum bpf_prog_type prog_type = prog->type;
5401 struct btf *btf = prog->aux->btf;5416 struct btf *btf = prog->aux->btf;
5402 const struct btf_param *args;5417 const struct btf_param *args;
5403 const struct btf_type *t;5418 const struct btf_type *t, *ref_t;
5404 u32 i, nargs, btf_id;5419 u32 i, nargs, btf_id;
5405 const char *tname;5420 const char *tname;
5406 5421
@@ -5464,16 +5479,35 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, · 57 unmodified lines
5464 * Only PTR_TO_CTX and SCALAR are supported atm.5479 * Only PTR_TO_CTX and SCALAR are supported atm.
5465 */5480 */
5466 for (i = 0; i < nargs; i++) {5481 for (i = 0; i < nargs; i++) {
5482 struct bpf_reg_state *reg = &regs[i + 1];
5483
5467 t = btf_type_by_id(btf, args[i].type);5484 t = btf_type_by_id(btf, args[i].type);
5468 while (btf_type_is_modifier(t))5485 while (btf_type_is_modifier(t))
5469 t = btf_type_by_id(btf, t->type);5486 t = btf_type_by_id(btf, t->type);
5470 if (btf_type_is_int(t) || btf_type_is_enum(t)) {5487 if (btf_type_is_int(t) || btf_type_is_enum(t)) {
5471 reg[i + 1].type = SCALAR_VALUE;5488 reg->type = SCALAR_VALUE;
5472 continue;5489 continue;
5473 }5490 }
5474 if (btf_type_is_ptr(t) &&5491 if (btf_type_is_ptr(t)) {
5475 btf_get_prog_ctx_type(log, btf, t, prog_type, i)) {5492 if (btf_get_prog_ctx_type(log, btf, t, prog_type, i)) {
5476 reg[i + 1].type = PTR_TO_CTX;5493 reg->type = PTR_TO_CTX;
5494 continue;
5495 }
5496
5497 t = btf_type_skip_modifiers(btf, t->type, NULL);
5498
5499 ref_t = btf_resolve_size(btf, t, &reg->mem_size);
5500 if (IS_ERR(ref_t)) {
5501 bpf_log(log,
5502 "arg#%d reference type('%s %s') size cannot be determined: %ld\n",
5503 i, btf_type_str(t), btf_name_by_offset(btf, t->name_off),
5504 PTR_ERR(ref_t));
5505 return -EINVAL;
5506 }
5507
5508 reg->type = PTR_TO_MEM_OR_NULL;
5509 reg->id = ++env->id_gen;
5510
5477 continue;5511 continue;
5478 }5512 }
5479 bpf_log(log, "Arg#%d type %s in %s() is not supported yet.\n",5513 bpf_log(log, "Arg#%d type %s in %s() is not supported yet.\n",
kernel/bpf/cgroup.c+103−17
@@ -19,7 +19,7 @@ · 18 unmodified lines
19 19
20#include "../cgroup/cgroup-internal.h"20#include "../cgroup/cgroup-internal.h"
21 21
22DEFINE_STATIC_KEY_FALSE(cgroup_bpf_enabled_key);22DEFINE_STATIC_KEY_ARRAY_FALSE(cgroup_bpf_enabled_key, MAX_BPF_ATTACH_TYPE);
23EXPORT_SYMBOL(cgroup_bpf_enabled_key);23EXPORT_SYMBOL(cgroup_bpf_enabled_key);
24 24
25void cgroup_bpf_offline(struct cgroup *cgrp)25void cgroup_bpf_offline(struct cgroup *cgrp)
@@ -128,7 +128,7 @@ static void cgroup_bpf_release(struct work_struct *work) · 102 unmodified lines
128 if (pl->link)128 if (pl->link)
129 bpf_cgroup_link_auto_detach(pl->link);129 bpf_cgroup_link_auto_detach(pl->link);
130 kfree(pl);130 kfree(pl);
131 static_branch_dec(&cgroup_bpf_enabled_key);131 static_branch_dec(&cgroup_bpf_enabled_key[type]);
132 }132 }
133 old_array = rcu_dereference_protected(133 old_array = rcu_dereference_protected(
134 cgrp->bpf.effective[type],134 cgrp->bpf.effective[type],
@@ -499,7 +499,7 @@ int __cgroup_bpf_attach(struct cgroup *cgrp, · 364 unmodified lines
499 if (old_prog)499 if (old_prog)
500 bpf_prog_put(old_prog);500 bpf_prog_put(old_prog);
501 else501 else
502 static_branch_inc(&cgroup_bpf_enabled_key);502 static_branch_inc(&cgroup_bpf_enabled_key[type]);
503 bpf_cgroup_storages_link(new_storage, cgrp, type);503 bpf_cgroup_storages_link(new_storage, cgrp, type);
504 return 0;504 return 0;
505 505
@@ -698,7 +698,7 @@ int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, · 192 unmodified lines
698 cgrp->bpf.flags[type] = 0;698 cgrp->bpf.flags[type] = 0;
699 if (old_prog)699 if (old_prog)
700 bpf_prog_put(old_prog);700 bpf_prog_put(old_prog);
701 static_branch_dec(&cgroup_bpf_enabled_key);701 static_branch_dec(&cgroup_bpf_enabled_key[type]);
702 return 0;702 return 0;
703 703
704cleanup:704cleanup:
@@ -1055,6 +1055,8 @@ EXPORT_SYMBOL(__cgroup_bpf_run_filter_sk); · 350 unmodified lines
1055 * @uaddr: sockaddr struct provided by user1055 * @uaddr: sockaddr struct provided by user
1056 * @type: The type of program to be exectuted1056 * @type: The type of program to be exectuted
1057 * @t_ctx: Pointer to attach type specific context1057 * @t_ctx: Pointer to attach type specific context
1058 * @flags: Pointer to u32 which contains higher bits of BPF program
1059 * return value (OR'ed together).
1058 *1060 *
1059 * socket is expected to be of type INET or INET6.1061 * socket is expected to be of type INET or INET6.
1060 *1062 *
@@ -1064,7 +1066,8 @@ EXPORT_SYMBOL(__cgroup_bpf_run_filter_sk); · 3 unmodified lines
1064int __cgroup_bpf_run_filter_sock_addr(struct sock *sk,1066int __cgroup_bpf_run_filter_sock_addr(struct sock *sk,
1065 struct sockaddr *uaddr,1067 struct sockaddr *uaddr,
1066 enum bpf_attach_type type,1068 enum bpf_attach_type type,
1067 void *t_ctx)1069 void *t_ctx,
1070 u32 *flags)
1068{1071{
1069 struct bpf_sock_addr_kern ctx = {1072 struct bpf_sock_addr_kern ctx = {
1070 .sk = sk,1073 .sk = sk,
@@ -1087,7 +1090,8 @@ int __cgroup_bpf_run_filter_sock_addr(struct sock *sk, · 16 unmodified lines
1087 }1090 }
1088 1091
1089 cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);1092 cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
1090 ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], &ctx, BPF_PROG_RUN);1093 ret = BPF_PROG_RUN_ARRAY_FLAGS(cgrp->bpf.effective[type], &ctx,
1094 BPF_PROG_RUN, flags);
1091 1095
1092 return ret == 1 ? 0 : -EPERM;1096 return ret == 1 ? 0 : -EPERM;
1093}1097}
@@ -1298,7 +1302,8 @@ static bool __cgroup_bpf_prog_array_is_empty(struct cgroup *cgrp, · 204 unmodified lines
1298 return empty;1302 return empty;
1299}1303}
1300 1304
1301static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen)1305static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen,
1306 struct bpf_sockopt_buf *buf)
1302{1307{
1303 if (unlikely(max_optlen < 0))1308 if (unlikely(max_optlen < 0))
1304 return -EINVAL;1309 return -EINVAL;
@@ -1310,6 +1315,15 @@ static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen) · 5 unmodified lines
1310 max_optlen = PAGE_SIZE;1315 max_optlen = PAGE_SIZE;
1311 }1316 }
1312 1317
1318 if (max_optlen <= sizeof(buf->data)) {
1319 /* When the optval fits into BPF_SOCKOPT_KERN_BUF_SIZE
1320 * bytes avoid the cost of kzalloc.
1321 */
1322 ctx->optval = buf->data;
1323 ctx->optval_end = ctx->optval + max_optlen;
1324 return max_optlen;
1325 }
1326
1313 ctx->optval = kzalloc(max_optlen, GFP_USER);1327 ctx->optval = kzalloc(max_optlen, GFP_USER);
1314 if (!ctx->optval)1328 if (!ctx->optval)
1315 return -ENOMEM;1329 return -ENOMEM;
@@ -1319,16 +1333,26 @@ static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen) · 3 unmodified lines
1319 return max_optlen;1333 return max_optlen;
1320}1334}
1321 1335
1322static void sockopt_free_buf(struct bpf_sockopt_kern *ctx)1336static void sockopt_free_buf(struct bpf_sockopt_kern *ctx,
1337 struct bpf_sockopt_buf *buf)
1323{1338{
1339 if (ctx->optval == buf->data)
1340 return;
1324 kfree(ctx->optval);1341 kfree(ctx->optval);
1325}1342}
1326 1343
1344static bool sockopt_buf_allocated(struct bpf_sockopt_kern *ctx,
1345 struct bpf_sockopt_buf *buf)
1346{
1347 return ctx->optval != buf->data;
1348}
1349
1327int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level,1350int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level,
1328 int *optname, char __user *optval,1351 int *optname, char __user *optval,
1329 int *optlen, char **kernel_optval)1352 int *optlen, char **kernel_optval)
1330{1353{
1331 struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);1354 struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
1355 struct bpf_sockopt_buf buf = {};
1332 struct bpf_sockopt_kern ctx = {1356 struct bpf_sockopt_kern ctx = {
1333 .sk = sk,1357 .sk = sk,
1334 .level = *level,1358 .level = *level,
@@ -1340,8 +1364,7 @@ int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level, · 5 unmodified lines
1340 * attached to the hook so we don't waste time allocating1364 * attached to the hook so we don't waste time allocating
1341 * memory and locking the socket.1365 * memory and locking the socket.
1342 */1366 */
1343 if (!cgroup_bpf_enabled ||1367 if (__cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_SETSOCKOPT))
1344 __cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_SETSOCKOPT))
1345 return 0;1368 return 0;
1346 1369
1347 /* Allocate a bit more than the initial user buffer for1370 /* Allocate a bit more than the initial user buffer for
@@ -1350,7 +1373,7 @@ int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level, · 2 unmodified lines
1350 */1373 */
1351 max_optlen = max_t(int, 16, *optlen);1374 max_optlen = max_t(int, 16, *optlen);
1352 1375
1353 max_optlen = sockopt_alloc_buf(&ctx, max_optlen);1376 max_optlen = sockopt_alloc_buf(&ctx, max_optlen, &buf);
1354 if (max_optlen < 0)1377 if (max_optlen < 0)
1355 return max_optlen;1378 return max_optlen;
1356 1379
@@ -1390,14 +1413,31 @@ int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level, · 33 unmodified lines
1390 */1413 */
1391 if (ctx.optlen != 0) {1414 if (ctx.optlen != 0) {
1392 *optlen = ctx.optlen;1415 *optlen = ctx.optlen;
1393 *kernel_optval = ctx.optval;1416 /* We've used bpf_sockopt_kern->buf as an intermediary
1417 * storage, but the BPF program indicates that we need
1418 * to pass this data to the kernel setsockopt handler.
1419 * No way to export on-stack buf, have to allocate a
1420 * new buffer.
1421 */
1422 if (!sockopt_buf_allocated(&ctx, &buf)) {
1423 void *p = kmalloc(ctx.optlen, GFP_USER);
1424
1425 if (!p) {
1426 ret = -ENOMEM;
1427 goto out;
1428 }
1429 memcpy(p, ctx.optval, ctx.optlen);
1430 *kernel_optval = p;
1431 } else {
1432 *kernel_optval = ctx.optval;
1433 }
1394 /* export and don't free sockopt buf */1434 /* export and don't free sockopt buf */
1395 return 0;1435 return 0;
1396 }1436 }
1397 }1437 }
1398 1438
1399out:1439out:
1400 sockopt_free_buf(&ctx);1440 sockopt_free_buf(&ctx, &buf);
1401 return ret;1441 return ret;
1402}1442}
1403 1443
@@ -1407,6 +1447,7 @@ int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level, · 3 unmodified lines
1407 int retval)1447 int retval)
1408{1448{
1409 struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);1449 struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
1450 struct bpf_sockopt_buf buf = {};
1410 struct bpf_sockopt_kern ctx = {1451 struct bpf_sockopt_kern ctx = {
1411 .sk = sk,1452 .sk = sk,
1412 .level = level,1453 .level = level,
@@ -1419,13 +1460,12 @@ int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level, · 6 unmodified lines
1419 * attached to the hook so we don't waste time allocating1460 * attached to the hook so we don't waste time allocating
1420 * memory and locking the socket.1461 * memory and locking the socket.
1421 */1462 */
1422 if (!cgroup_bpf_enabled ||1463 if (__cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_GETSOCKOPT))
1423 __cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_GETSOCKOPT))
1424 return retval;1464 return retval;
1425 1465
1426 ctx.optlen = max_optlen;1466 ctx.optlen = max_optlen;
1427 1467
1428 max_optlen = sockopt_alloc_buf(&ctx, max_optlen);1468 max_optlen = sockopt_alloc_buf(&ctx, max_optlen, &buf);
1429 if (max_optlen < 0)1469 if (max_optlen < 0)
1430 return max_optlen;1470 return max_optlen;
1431 1471
@@ -1488,9 +1528,55 @@ int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level, · 56 unmodified lines
1488 ret = ctx.retval;1528 ret = ctx.retval;
1489 1529
1490out:1530out:
1491 sockopt_free_buf(&ctx);1531 sockopt_free_buf(&ctx, &buf);
1492 return ret;1532 return ret;
1493}1533}
1534
1535int __cgroup_bpf_run_filter_getsockopt_kern(struct sock *sk, int level,
1536 int optname, void *optval,
1537 int *optlen, int retval)
1538{
1539 struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
1540 struct bpf_sockopt_kern ctx = {
1541 .sk = sk,
1542 .level = level,
1543 .optname = optname,
1544 .retval = retval,
1545 .optlen = *optlen,
1546 .optval = optval,
1547 .optval_end = optval + *optlen,
1548 };
1549 int ret;
1550
1551 /* Note that __cgroup_bpf_run_filter_getsockopt doesn't copy
1552 * user data back into BPF buffer when reval != 0. This is
1553 * done as an optimization to avoid extra copy, assuming
1554 * kernel won't populate the data in case of an error.
1555 * Here we always pass the data and memset() should
1556 * be called if that data shouldn't be "exported".
1557 */
1558
1559 ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[BPF_CGROUP_GETSOCKOPT],
1560 &ctx, BPF_PROG_RUN);
1561 if (!ret)
1562 return -EPERM;
1563
1564 if (ctx.optlen > *optlen)
1565 return -EFAULT;
1566
1567 /* BPF programs only allowed to set retval to 0, not some
1568 * arbitrary value.
1569 */
1570 if (ctx.retval != 0 && ctx.retval != retval)
1571 return -EFAULT;
1572
1573 /* BPF programs can shrink the buffer, export the modifications.
1574 */
1575 if (ctx.optlen != 0)
1576 *optlen = ctx.optlen;
1577
1578 return ctx.retval;
1579}
1494#endif1580#endif
1495 1581
1496static ssize_t sysctl_cpy_dir(const struct ctl_dir *dir, char **bufp,1582static ssize_t sysctl_cpy_dir(const struct ctl_dir *dir, char **bufp,
kernel/bpf/core.c+14−4
@@ -91,6 +91,12 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag · 90 unmodified lines
91 vfree(fp);91 vfree(fp);
92 return NULL;92 return NULL;
93 }93 }
94 fp->active = alloc_percpu_gfp(int, GFP_KERNEL_ACCOUNT | gfp_extra_flags);
95 if (!fp->active) {
96 vfree(fp);
97 kfree(aux);
98 return NULL;
99 }
94 100
95 fp->pages = size / PAGE_SIZE;101 fp->pages = size / PAGE_SIZE;
96 fp->aux = aux;102 fp->aux = aux;
@@ -114,8 +120,9 @@ struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags) · 17 unmodified lines
114 if (!prog)120 if (!prog)
115 return NULL;121 return NULL;
116 122
117 prog->aux->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags);123 prog->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags);
118 if (!prog->aux->stats) {124 if (!prog->stats) {
125 free_percpu(prog->active);
119 kfree(prog->aux);126 kfree(prog->aux);
120 vfree(prog);127 vfree(prog);
121 return NULL;128 return NULL;
@@ -124,7 +131,7 @@ struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags) · 2 unmodified lines
124 for_each_possible_cpu(cpu) {131 for_each_possible_cpu(cpu) {
125 struct bpf_prog_stats *pstats;132 struct bpf_prog_stats *pstats;
126 133
127 pstats = per_cpu_ptr(prog->aux->stats, cpu);134 pstats = per_cpu_ptr(prog->stats, cpu);
128 u64_stats_init(&pstats->syncp);135 u64_stats_init(&pstats->syncp);
129 }136 }
130 return prog;137 return prog;
@@ -238,6 +245,8 @@ struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size, · 107 unmodified lines
238 * reallocated structure.245 * reallocated structure.
239 */246 */
240 fp_old->aux = NULL;247 fp_old->aux = NULL;
248 fp_old->stats = NULL;
249 fp_old->active = NULL;
241 __bpf_prog_free(fp_old);250 __bpf_prog_free(fp_old);
242 }251 }
243 252
@@ -249,10 +258,11 @@ void __bpf_prog_free(struct bpf_prog *fp) · 5 unmodified lines
249 if (fp->aux) {258 if (fp->aux) {
250 mutex_destroy(&fp->aux->used_maps_mutex);259 mutex_destroy(&fp->aux->used_maps_mutex);
251 mutex_destroy(&fp->aux->dst_mutex);260 mutex_destroy(&fp->aux->dst_mutex);
252 free_percpu(fp->aux->stats);
253 kfree(fp->aux->poke_tab);261 kfree(fp->aux->poke_tab);
254 kfree(fp->aux);262 kfree(fp->aux);
255 }263 }
264 free_percpu(fp->stats);
265 free_percpu(fp->active);
256 vfree(fp);266 vfree(fp);
257}267}
258 268
kernel/bpf/cpumap.c+2−44
@@ -141,49 +141,6 @@ static void cpu_map_kthread_stop(struct work_struct *work) · 140 unmodified lines
141 kthread_stop(rcpu->kthread);141 kthread_stop(rcpu->kthread);
142}142}
143 143
144static struct sk_buff *cpu_map_build_skb(struct xdp_frame *xdpf,
145 struct sk_buff *skb)
146{
147 unsigned int hard_start_headroom;
148 unsigned int frame_size;
149 void *pkt_data_start;
150
151 /* Part of headroom was reserved to xdpf */
152 hard_start_headroom = sizeof(struct xdp_frame) + xdpf->headroom;
153
154 /* Memory size backing xdp_frame data already have reserved
155 * room for build_skb to place skb_shared_info in tailroom.
156 */
157 frame_size = xdpf->frame_sz;
158
159 pkt_data_start = xdpf->data - hard_start_headroom;
160 skb = build_skb_around(skb, pkt_data_start, frame_size);
161 if (unlikely(!skb))
162 return NULL;
163
164 skb_reserve(skb, hard_start_headroom);
165 __skb_put(skb, xdpf->len);
166 if (xdpf->metasize)
167 skb_metadata_set(skb, xdpf->metasize);
168
169 /* Essential SKB info: protocol and skb->dev */
170 skb->protocol = eth_type_trans(skb, xdpf->dev_rx);
171
172 /* Optional SKB info, currently missing:
173 * - HW checksum info (skb->ip_summed)
174 * - HW RX hash (skb_set_hash)
175 * - RX ring dev queue index (skb_record_rx_queue)
176 */
177
178 /* Until page_pool get SKB return path, release DMA here */
179 xdp_release_frame(xdpf);
180
181 /* Allow SKB to reuse area used by xdp_frame */
182 xdp_scrub_frame(xdpf);
183
184 return skb;
185}
186
187static void __cpu_map_ring_cleanup(struct ptr_ring *ring)144static void __cpu_map_ring_cleanup(struct ptr_ring *ring)
188{145{
189 /* The tear-down procedure should have made sure that queue is146 /* The tear-down procedure should have made sure that queue is
@@ -350,7 +307,8 @@ static int cpu_map_kthread_run(void *data) · 160 unmodified lines
350 struct sk_buff *skb = skbs[i];307 struct sk_buff *skb = skbs[i];
351 int ret;308 int ret;
352 309
353 skb = cpu_map_build_skb(xdpf, skb);310 skb = __xdp_build_skb_from_frame(xdpf, skb,
311 xdpf->dev_rx);
354 if (!skb) {312 if (!skb) {
355 xdp_return_frame(xdpf);313 xdp_return_frame(xdpf);
356 continue;314 continue;
kernel/bpf/devmap.c+1−3
@@ -802,9 +802,7 @@ static int dev_map_notification(struct notifier_block *notifier, · 801 unmodified lines
802 break;802 break;
803 803
804 /* will be freed in free_netdev() */804 /* will be freed in free_netdev() */
805 netdev->xdp_bulkq =805 netdev->xdp_bulkq = alloc_percpu(struct xdp_dev_bulk_queue);
806 __alloc_percpu_gfp(sizeof(struct xdp_dev_bulk_queue),
807 sizeof(void *), GFP_ATOMIC);
808 if (!netdev->xdp_bulkq)806 if (!netdev->xdp_bulkq)
809 return NOTIFY_BAD;807 return NOTIFY_BAD;
810 808
kernel/bpf/disasm.c+1−1
@@ -161,7 +161,7 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, · 160 unmodified lines
161 insn->dst_reg,161 insn->dst_reg,
162 insn->off, insn->src_reg);162 insn->off, insn->src_reg);
163 else if (BPF_MODE(insn->code) == BPF_ATOMIC &&163 else if (BPF_MODE(insn->code) == BPF_ATOMIC &&
164 (insn->imm == BPF_ADD || insn->imm == BPF_ADD ||164 (insn->imm == BPF_ADD || insn->imm == BPF_AND ||
165 insn->imm == BPF_OR || insn->imm == BPF_XOR)) {165 insn->imm == BPF_OR || insn->imm == BPF_XOR)) {
166 verbose(cbs->private_data, "(%02x) lock *(%s *)(r%d %+d) %s r%d\n",166 verbose(cbs->private_data, "(%02x) lock *(%s *)(r%d %+d) %s r%d\n",
167 insn->code,167 insn->code,
kernel/bpf/hashtab.c+2−2
@@ -1148,7 +1148,7 @@ static int __htab_percpu_map_update_elem(struct bpf_map *map, void *key, · 1147 unmodified lines
1148 /* unknown flags */1148 /* unknown flags */
1149 return -EINVAL;1149 return -EINVAL;
1150 1150
1151 WARN_ON_ONCE(!rcu_read_lock_held());1151 WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
1152 1152
1153 key_size = map->key_size;1153 key_size = map->key_size;
1154 1154
@@ -1202,7 +1202,7 @@ static int __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, · 47 unmodified lines
1202 /* unknown flags */1202 /* unknown flags */
1203 return -EINVAL;1203 return -EINVAL;
1204 1204
1205 WARN_ON_ONCE(!rcu_read_lock_held());1205 WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
1206 1206
1207 key_size = map->key_size;1207 key_size = map->key_size;
1208 1208
kernel/bpf/helpers.c+4−8
@@ -720,14 +720,6 @@ bpf_base_func_proto(enum bpf_func_id func_id) · 719 unmodified lines
720 return &bpf_spin_lock_proto;720 return &bpf_spin_lock_proto;
721 case BPF_FUNC_spin_unlock:721 case BPF_FUNC_spin_unlock:
722 return &bpf_spin_unlock_proto;722 return &bpf_spin_unlock_proto;
723 case BPF_FUNC_trace_printk:
724 if (!perfmon_capable())
725 return NULL;
726 return bpf_get_trace_printk_proto();
727 case BPF_FUNC_snprintf_btf:
728 if (!perfmon_capable())
729 return NULL;
730 return &bpf_snprintf_btf_proto;
731 case BPF_FUNC_jiffies64:723 case BPF_FUNC_jiffies64:
732 return &bpf_jiffies64_proto;724 return &bpf_jiffies64_proto;
733 case BPF_FUNC_per_cpu_ptr:725 case BPF_FUNC_per_cpu_ptr:
@@ -742,6 +734,8 @@ bpf_base_func_proto(enum bpf_func_id func_id) · 8 unmodified lines
742 return NULL;734 return NULL;
743 735
744 switch (func_id) {736 switch (func_id) {
737 case BPF_FUNC_trace_printk:
738 return bpf_get_trace_printk_proto();
745 case BPF_FUNC_get_current_task:739 case BPF_FUNC_get_current_task:
746 return &bpf_get_current_task_proto;740 return &bpf_get_current_task_proto;
747 case BPF_FUNC_probe_read_user:741 case BPF_FUNC_probe_read_user:
@@ -752,6 +746,8 @@ bpf_base_func_proto(enum bpf_func_id func_id) · 4 unmodified lines
752 return &bpf_probe_read_user_str_proto;746 return &bpf_probe_read_user_str_proto;
753 case BPF_FUNC_probe_read_kernel_str:747 case BPF_FUNC_probe_read_kernel_str:
754 return &bpf_probe_read_kernel_str_proto;748 return &bpf_probe_read_kernel_str_proto;
749 case BPF_FUNC_snprintf_btf:
750 return &bpf_snprintf_btf_proto;
755 default:751 default:
756 return NULL;752 return NULL;
757 }753 }
kernel/bpf/syscall.c+11−5
@@ -1731,25 +1731,28 @@ static int bpf_prog_release(struct inode *inode, struct file *filp) · 1730 unmodified lines
1731static void bpf_prog_get_stats(const struct bpf_prog *prog,1731static void bpf_prog_get_stats(const struct bpf_prog *prog,
1732 struct bpf_prog_stats *stats)1732 struct bpf_prog_stats *stats)
1733{1733{
1734 u64 nsecs = 0, cnt = 0;1734 u64 nsecs = 0, cnt = 0, misses = 0;
1735 int cpu;1735 int cpu;
1736 1736
1737 for_each_possible_cpu(cpu) {1737 for_each_possible_cpu(cpu) {
1738 const struct bpf_prog_stats *st;1738 const struct bpf_prog_stats *st;
1739 unsigned int start;1739 unsigned int start;
1740 u64 tnsecs, tcnt;1740 u64 tnsecs, tcnt, tmisses;
1741 1741
1742 st = per_cpu_ptr(prog->aux->stats, cpu);1742 st = per_cpu_ptr(prog->stats, cpu);
1743 do {1743 do {
1744 start = u64_stats_fetch_begin_irq(&st->syncp);1744 start = u64_stats_fetch_begin_irq(&st->syncp);
1745 tnsecs = st->nsecs;1745 tnsecs = st->nsecs;
1746 tcnt = st->cnt;1746 tcnt = st->cnt;
1747 tmisses = st->misses;
1747 } while (u64_stats_fetch_retry_irq(&st->syncp, start));1748 } while (u64_stats_fetch_retry_irq(&st->syncp, start));
1748 nsecs += tnsecs;1749 nsecs += tnsecs;
1749 cnt += tcnt;1750 cnt += tcnt;
1751 misses += tmisses;
1750 }1752 }
1751 stats->nsecs = nsecs;1753 stats->nsecs = nsecs;
1752 stats->cnt = cnt;1754 stats->cnt = cnt;
1755 stats->misses = misses;
1753}1756}
1754 1757
1755#ifdef CONFIG_PROC_FS1758#ifdef CONFIG_PROC_FS
@@ -1768,14 +1771,16 @@ static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp) · 12 unmodified lines
1768 "memlock:\t%llu\n"1771 "memlock:\t%llu\n"
1769 "prog_id:\t%u\n"1772 "prog_id:\t%u\n"
1770 "run_time_ns:\t%llu\n"1773 "run_time_ns:\t%llu\n"
1771 "run_cnt:\t%llu\n",1774 "run_cnt:\t%llu\n"
1775 "recursion_misses:\t%llu\n",
1772 prog->type,1776 prog->type,
1773 prog->jited,1777 prog->jited,
1774 prog_tag,1778 prog_tag,
1775 prog->pages * 1ULL << PAGE_SHIFT,1779 prog->pages * 1ULL << PAGE_SHIFT,
1776 prog->aux->id,1780 prog->aux->id,
1777 stats.nsecs,1781 stats.nsecs,
1778 stats.cnt);1782 stats.cnt,
1783 stats.misses);
1779}1784}
1780#endif1785#endif
1781 1786
@@ -3438,6 +3443,7 @@ static int bpf_prog_get_info_by_fd(struct file *file, · 1656 unmodified lines
3438 bpf_prog_get_stats(prog, &stats);3443 bpf_prog_get_stats(prog, &stats);
3439 info.run_time_ns = stats.nsecs;3444 info.run_time_ns = stats.nsecs;
3440 info.run_cnt = stats.cnt;3445 info.run_cnt = stats.cnt;
3446 info.recursion_misses = stats.misses;
3441 3447
3442 if (!bpf_capable()) {3448 if (!bpf_capable()) {
3443 info.jited_prog_len = 0;3449 info.jited_prog_len = 0;
kernel/bpf/task_iter.c+266−1
@@ -286,9 +286,248 @@ static const struct seq_operations task_file_seq_ops = { · 285 unmodified lines
286 .show = task_file_seq_show,286 .show = task_file_seq_show,
287};287};
288 288
289struct bpf_iter_seq_task_vma_info {
290 /* The first field must be struct bpf_iter_seq_task_common.
291 * this is assumed by {init, fini}_seq_pidns() callback functions.
292 */
293 struct bpf_iter_seq_task_common common;
294 struct task_struct *task;
295 struct vm_area_struct *vma;
296 u32 tid;
297 unsigned long prev_vm_start;
298 unsigned long prev_vm_end;
299};
300
301enum bpf_task_vma_iter_find_op {
302 task_vma_iter_first_vma, /* use mm->mmap */
303 task_vma_iter_next_vma, /* use curr_vma->vm_next */
304 task_vma_iter_find_vma, /* use find_vma() to find next vma */
305};
306
307static struct vm_area_struct *
308task_vma_seq_get_next(struct bpf_iter_seq_task_vma_info *info)
309{
310 struct pid_namespace *ns = info->common.ns;
311 enum bpf_task_vma_iter_find_op op;
312 struct vm_area_struct *curr_vma;
313 struct task_struct *curr_task;
314 u32 curr_tid = info->tid;
315
316 /* If this function returns a non-NULL vma, it holds a reference to
317 * the task_struct, and holds read lock on vma->mm->mmap_lock.
318 * If this function returns NULL, it does not hold any reference or
319 * lock.
320 */
321 if (info->task) {
322 curr_task = info->task;
323 curr_vma = info->vma;
324 /* In case of lock contention, drop mmap_lock to unblock
325 * the writer.
326 *
327 * After relock, call find(mm, prev_vm_end - 1) to find
328 * new vma to process.
329 *
330 * +------+------+-----------+
331 * | VMA1 | VMA2 | VMA3 |
332 * +------+------+-----------+
333 * | | | |
334 * 4k 8k 16k 400k
335 *
336 * For example, curr_vma == VMA2. Before unlock, we set
337 *
338 * prev_vm_start = 8k
339 * prev_vm_end = 16k
340 *
341 * There are a few cases:
342 *
343 * 1) VMA2 is freed, but VMA3 exists.
344 *
345 * find_vma() will return VMA3, just process VMA3.
346 *
347 * 2) VMA2 still exists.
348 *
349 * find_vma() will return VMA2, process VMA2->next.
350 *
351 * 3) no more vma in this mm.
352 *
353 * Process the next task.
354 *
355 * 4) find_vma() returns a different vma, VMA2'.
356 *
357 * 4.1) If VMA2 covers same range as VMA2', skip VMA2',
358 * because we already covered the range;
359 * 4.2) VMA2 and VMA2' covers different ranges, process
360 * VMA2'.
361 */
362 if (mmap_lock_is_contended(curr_task->mm)) {
363 info->prev_vm_start = curr_vma->vm_start;
364 info->prev_vm_end = curr_vma->vm_end;
365 op = task_vma_iter_find_vma;
366 mmap_read_unlock(curr_task->mm);
367 if (mmap_read_lock_killable(curr_task->mm))
368 goto finish;
369 } else {
370 op = task_vma_iter_next_vma;
371 }
372 } else {
373again:
374 curr_task = task_seq_get_next(ns, &curr_tid, true);
375 if (!curr_task) {
376 info->tid = curr_tid + 1;
377 goto finish;
378 }
379
380 if (curr_tid != info->tid) {
381 info->tid = curr_tid;
382 /* new task, process the first vma */
383 op = task_vma_iter_first_vma;
384 } else {
385 /* Found the same tid, which means the user space
386 * finished data in previous buffer and read more.
387 * We dropped mmap_lock before returning to user
388 * space, so it is necessary to use find_vma() to
389 * find the next vma to process.
390 */
391 op = task_vma_iter_find_vma;
392 }
393
394 if (!curr_task->mm)
395 goto next_task;
396
397 if (mmap_read_lock_killable(curr_task->mm))
398 goto finish;
399 }
400
401 switch (op) {
402 case task_vma_iter_first_vma:
403 curr_vma = curr_task->mm->mmap;
404 break;
405 case task_vma_iter_next_vma:
406 curr_vma = curr_vma->vm_next;
407 break;
408 case task_vma_iter_find_vma:
409 /* We dropped mmap_lock so it is necessary to use find_vma
410 * to find the next vma. This is similar to the mechanism
411 * in show_smaps_rollup().
412 */
413 curr_vma = find_vma(curr_task->mm, info->prev_vm_end - 1);
414 /* case 1) and 4.2) above just use curr_vma */
415
416 /* check for case 2) or case 4.1) above */
417 if (curr_vma &&
418 curr_vma->vm_start == info->prev_vm_start &&
419 curr_vma->vm_end == info->prev_vm_end)
420 curr_vma = curr_vma->vm_next;
421 break;
422 }
423 if (!curr_vma) {
424 /* case 3) above, or case 2) 4.1) with vma->next == NULL */
425 mmap_read_unlock(curr_task->mm);
426 goto next_task;
427 }
428 info->task = curr_task;
429 info->vma = curr_vma;
430 return curr_vma;
431
432next_task:
433 put_task_struct(curr_task);
434 info->task = NULL;
435 curr_tid++;
436 goto again;
437
438finish:
439 if (curr_task)
440 put_task_struct(curr_task);
441 info->task = NULL;
442 info->vma = NULL;
443 return NULL;
444}
445
446static void *task_vma_seq_start(struct seq_file *seq, loff_t *pos)
447{
448 struct bpf_iter_seq_task_vma_info *info = seq->private;
449 struct vm_area_struct *vma;
450
451 vma = task_vma_seq_get_next(info);
452 if (vma && *pos == 0)
453 ++*pos;
454
455 return vma;
456}
457
458static void *task_vma_seq_next(struct seq_file *seq, void *v, loff_t *pos)
459{
460 struct bpf_iter_seq_task_vma_info *info = seq->private;
461
462 ++*pos;
463 return task_vma_seq_get_next(info);
464}
465
466struct bpf_iter__task_vma {
467 __bpf_md_ptr(struct bpf_iter_meta *, meta);
468 __bpf_md_ptr(struct task_struct *, task);
469 __bpf_md_ptr(struct vm_area_struct *, vma);
470};
471
472DEFINE_BPF_ITER_FUNC(task_vma, struct bpf_iter_meta *meta,
473 struct task_struct *task, struct vm_area_struct *vma)
474
475static int __task_vma_seq_show(struct seq_file *seq, bool in_stop)
476{
477 struct bpf_iter_seq_task_vma_info *info = seq->private;
478 struct bpf_iter__task_vma ctx;
479 struct bpf_iter_meta meta;
480 struct bpf_prog *prog;
481
482 meta.seq = seq;
483 prog = bpf_iter_get_info(&meta, in_stop);
484 if (!prog)
485 return 0;
486
487 ctx.meta = &meta;
488 ctx.task = info->task;
489 ctx.vma = info->vma;
490 return bpf_iter_run_prog(prog, &ctx);
491}
492
493static int task_vma_seq_show(struct seq_file *seq, void *v)
494{
495 return __task_vma_seq_show(seq, false);
496}
497
498static void task_vma_seq_stop(struct seq_file *seq, void *v)
499{
500 struct bpf_iter_seq_task_vma_info *info = seq->private;
501
502 if (!v) {
503 (void)__task_vma_seq_show(seq, true);
504 } else {
505 /* info->vma has not been seen by the BPF program. If the
506 * user space reads more, task_vma_seq_get_next should
507 * return this vma again. Set prev_vm_start to ~0UL,
508 * so that we don't skip the vma returned by the next
509 * find_vma() (case task_vma_iter_find_vma in
510 * task_vma_seq_get_next()).
511 */
512 info->prev_vm_start = ~0UL;
513 info->prev_vm_end = info->vma->vm_end;
514 mmap_read_unlock(info->task->mm);
515 put_task_struct(info->task);
516 info->task = NULL;
517 }
518}
519
520static const struct seq_operations task_vma_seq_ops = {
521 .start = task_vma_seq_start,
522 .next = task_vma_seq_next,
523 .stop = task_vma_seq_stop,
524 .show = task_vma_seq_show,
525};
526
289BTF_ID_LIST(btf_task_file_ids)527BTF_ID_LIST(btf_task_file_ids)
290BTF_ID(struct, task_struct)528BTF_ID(struct, task_struct)
291BTF_ID(struct, file)529BTF_ID(struct, file)
530BTF_ID(struct, vm_area_struct)
292 531
293static const struct bpf_iter_seq_info task_seq_info = {532static const struct bpf_iter_seq_info task_seq_info = {
294 .seq_ops = &task_seq_ops,533 .seq_ops = &task_seq_ops,
@@ -328,6 +567,26 @@ static struct bpf_iter_reg task_file_reg_info = { · 33 unmodified lines
328 .seq_info = &task_file_seq_info,567 .seq_info = &task_file_seq_info,
329};568};
330 569
570static const struct bpf_iter_seq_info task_vma_seq_info = {
571 .seq_ops = &task_vma_seq_ops,
572 .init_seq_private = init_seq_pidns,
573 .fini_seq_private = fini_seq_pidns,
574 .seq_priv_size = sizeof(struct bpf_iter_seq_task_vma_info),
575};
576
577static struct bpf_iter_reg task_vma_reg_info = {
578 .target = "task_vma",
579 .feature = BPF_ITER_RESCHED,
580 .ctx_arg_info_size = 2,
581 .ctx_arg_info = {
582 { offsetof(struct bpf_iter__task_vma, task),
583 PTR_TO_BTF_ID_OR_NULL },
584 { offsetof(struct bpf_iter__task_vma, vma),
585 PTR_TO_BTF_ID_OR_NULL },
586 },
587 .seq_info = &task_vma_seq_info,
588};
589
331static int __init task_iter_init(void)590static int __init task_iter_init(void)
332{591{
333 int ret;592 int ret;
@@ -339,6 +598,12 @@ static int __init task_iter_init(void) · 5 unmodified lines
339 598
340 task_file_reg_info.ctx_arg_info[0].btf_id = btf_task_file_ids[0];599 task_file_reg_info.ctx_arg_info[0].btf_id = btf_task_file_ids[0];
341 task_file_reg_info.ctx_arg_info[1].btf_id = btf_task_file_ids[1];600 task_file_reg_info.ctx_arg_info[1].btf_id = btf_task_file_ids[1];
342 return bpf_iter_reg_target(&task_file_reg_info);601 ret = bpf_iter_reg_target(&task_file_reg_info);
602 if (ret)
603 return ret;
604
605 task_vma_reg_info.ctx_arg_info[0].btf_id = btf_task_file_ids[0];
606 task_vma_reg_info.ctx_arg_info[1].btf_id = btf_task_file_ids[2];
607 return bpf_iter_reg_target(&task_vma_reg_info);
343}608}
344late_initcall(task_iter_init);609late_initcall(task_iter_init);
kernel/bpf/trampoline.c+61−16
@@ -381,55 +381,100 @@ out: · 380 unmodified lines
381 mutex_unlock(&trampoline_mutex);381 mutex_unlock(&trampoline_mutex);
382}382}
383 383
384#define NO_START_TIME 1
385static u64 notrace bpf_prog_start_time(void)
386{
387 u64 start = NO_START_TIME;
388
389 if (static_branch_unlikely(&bpf_stats_enabled_key)) {
390 start = sched_clock();
391 if (unlikely(!start))
392 start = NO_START_TIME;
393 }
394 return start;
395}
396
397static void notrace inc_misses_counter(struct bpf_prog *prog)
398{
399 struct bpf_prog_stats *stats;
400
401 stats = this_cpu_ptr(prog->stats);
402 u64_stats_update_begin(&stats->syncp);
403 stats->misses++;
404 u64_stats_update_end(&stats->syncp);
405}
406
384/* The logic is similar to BPF_PROG_RUN, but with an explicit407/* The logic is similar to BPF_PROG_RUN, but with an explicit
385 * rcu_read_lock() and migrate_disable() which are required408 * rcu_read_lock() and migrate_disable() which are required
386 * for the trampoline. The macro is split into409 * for the trampoline. The macro is split into
387 * call _bpf_prog_enter410 * call __bpf_prog_enter
388 * call prog->bpf_func411 * call prog->bpf_func
389 * call __bpf_prog_exit412 * call __bpf_prog_exit
413 *
414 * __bpf_prog_enter returns:
415 * 0 - skip execution of the bpf prog
416 * 1 - execute bpf prog
417 * [2..MAX_U64] - excute bpf prog and record execution time.
418 * This is start time.
390 */419 */
391u64 notrace __bpf_prog_enter(void)420u64 notrace __bpf_prog_enter(struct bpf_prog *prog)
392 __acquires(RCU)421 __acquires(RCU)
393{422{
394 u64 start = 0;
395
396 rcu_read_lock();423 rcu_read_lock();
397 migrate_disable();424 migrate_disable();
398 if (static_branch_unlikely(&bpf_stats_enabled_key))425 if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) {
399 start = sched_clock();426 inc_misses_counter(prog);
400 return start;427 return 0;
428 }
429 return bpf_prog_start_time();
401}430}
402 431
403void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start)432static void notrace update_prog_stats(struct bpf_prog *prog,
404 __releases(RCU)433 u64 start)
405{434{
406 struct bpf_prog_stats *stats;435 struct bpf_prog_stats *stats;
407 436
408 if (static_branch_unlikely(&bpf_stats_enabled_key) &&437 if (static_branch_unlikely(&bpf_stats_enabled_key) &&
409 /* static_key could be enabled in __bpf_prog_enter438 /* static_key could be enabled in __bpf_prog_enter*
410 * and disabled in __bpf_prog_exit.439 * and disabled in __bpf_prog_exit*.
411 * And vice versa.440 * And vice versa.
412 * Hence check that 'start' is not zero.441 * Hence check that 'start' is valid.
413 */442 */
414 start) {443 start > NO_START_TIME) {
415 stats = this_cpu_ptr(prog->aux->stats);444 stats = this_cpu_ptr(prog->stats);
416 u64_stats_update_begin(&stats->syncp);445 u64_stats_update_begin(&stats->syncp);
417 stats->cnt++;446 stats->cnt++;
418 stats->nsecs += sched_clock() - start;447 stats->nsecs += sched_clock() - start;
419 u64_stats_update_end(&stats->syncp);448 u64_stats_update_end(&stats->syncp);
420 }449 }
450}
451
452void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start)
453 __releases(RCU)
454{
455 update_prog_stats(prog, start);
456 __this_cpu_dec(*(prog->active));
421 migrate_enable();457 migrate_enable();
422 rcu_read_unlock();458 rcu_read_unlock();
423}459}
424 460
425void notrace __bpf_prog_enter_sleepable(void)461u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog)
426{462{
427 rcu_read_lock_trace();463 rcu_read_lock_trace();
464 migrate_disable();
428 might_fault();465 might_fault();
466 if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) {
467 inc_misses_counter(prog);
468 return 0;
469 }
470 return bpf_prog_start_time();
429}471}
430 472
431void notrace __bpf_prog_exit_sleepable(void)473void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start)
432{474{
475 update_prog_stats(prog, start);
476 __this_cpu_dec(*(prog->active));
477 migrate_enable();
433 rcu_read_unlock_trace();478 rcu_read_unlock_trace();
434}479}
435 480
kernel/bpf/verifier.c+662−215
@@ -228,6 +228,12 @@ static void bpf_map_key_store(struct bpf_insn_aux_data *aux, u64 state) · 227 unmodified lines
228 (poisoned ? BPF_MAP_KEY_POISON : 0ULL);228 (poisoned ? BPF_MAP_KEY_POISON : 0ULL);
229}229}
230 230
231static bool bpf_pseudo_call(const struct bpf_insn *insn)
232{
233 return insn->code == (BPF_JMP | BPF_CALL) &&
234 insn->src_reg == BPF_PSEUDO_CALL;
235}
236
231struct bpf_call_arg_meta {237struct bpf_call_arg_meta {
232 struct bpf_map *map_ptr;238 struct bpf_map *map_ptr;
233 bool raw_mode;239 bool raw_mode;
@@ -1073,6 +1079,51 @@ static void mark_reg_known_zero(struct bpf_verifier_env *env, · 839 unmodified lines
1073 __mark_reg_known_zero(regs + regno);1079 __mark_reg_known_zero(regs + regno);
1074}1080}
1075 1081
1082static void mark_ptr_not_null_reg(struct bpf_reg_state *reg)
1083{
1084 switch (reg->type) {
1085 case PTR_TO_MAP_VALUE_OR_NULL: {
1086 const struct bpf_map *map = reg->map_ptr;
1087
1088 if (map->inner_map_meta) {
1089 reg->type = CONST_PTR_TO_MAP;
1090 reg->map_ptr = map->inner_map_meta;
1091 } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) {
1092 reg->type = PTR_TO_XDP_SOCK;
1093 } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP ||
1094 map->map_type == BPF_MAP_TYPE_SOCKHASH) {
1095 reg->type = PTR_TO_SOCKET;
1096 } else {
1097 reg->type = PTR_TO_MAP_VALUE;
1098 }
1099 break;
1100 }
1101 case PTR_TO_SOCKET_OR_NULL:
1102 reg->type = PTR_TO_SOCKET;
1103 break;
1104 case PTR_TO_SOCK_COMMON_OR_NULL:
1105 reg->type = PTR_TO_SOCK_COMMON;
1106 break;
1107 case PTR_TO_TCP_SOCK_OR_NULL:
1108 reg->type = PTR_TO_TCP_SOCK;
1109 break;
1110 case PTR_TO_BTF_ID_OR_NULL:
1111 reg->type = PTR_TO_BTF_ID;
1112 break;
1113 case PTR_TO_MEM_OR_NULL:
1114 reg->type = PTR_TO_MEM;
1115 break;
1116 case PTR_TO_RDONLY_BUF_OR_NULL:
1117 reg->type = PTR_TO_RDONLY_BUF;
1118 break;
1119 case PTR_TO_RDWR_BUF_OR_NULL:
1120 reg->type = PTR_TO_RDWR_BUF;
1121 break;
1122 default:
1123 WARN_ON("unknown nullable register type");
1124 }
1125}
1126
1076static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg)1127static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg)
1077{1128{
1078 return type_is_pkt_pointer(reg->type);1129 return type_is_pkt_pointer(reg->type);
@@ -1486,9 +1537,7 @@ static int check_subprogs(struct bpf_verifier_env *env) · 407 unmodified lines
1486 1537
1487 /* determine subprog starts. The end is one before the next starts */1538 /* determine subprog starts. The end is one before the next starts */
1488 for (i = 0; i < insn_cnt; i++) {1539 for (i = 0; i < insn_cnt; i++) {
1489 if (insn[i].code != (BPF_JMP | BPF_CALL))1540 if (!bpf_pseudo_call(insn + i))
1490 continue;
1491 if (insn[i].src_reg != BPF_PSEUDO_CALL)
1492 continue;1541 continue;
1493 if (!env->bpf_capable) {1542 if (!env->bpf_capable) {
1494 verbose(env,1543 verbose(env,
@@ -2271,12 +2320,14 @@ static void save_register_state(struct bpf_func_state *state, · 776 unmodified lines
2271 state->stack[spi].slot_type[i] = STACK_SPILL;2320 state->stack[spi].slot_type[i] = STACK_SPILL;
2272}2321}
2273 2322
2274/* check_stack_read/write functions track spill/fill of registers,2323/* check_stack_{read,write}_fixed_off functions track spill/fill of registers,
2275 * stack boundary and alignment are checked in check_mem_access()2324 * stack boundary and alignment are checked in check_mem_access()
2276 */2325 */
2277static int check_stack_write(struct bpf_verifier_env *env,2326static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
2278 struct bpf_func_state *state, /* func where register points to */2327 /* stack frame we're writing to */
2279 int off, int size, int value_regno, int insn_idx)2328 struct bpf_func_state *state,
2329 int off, int size, int value_regno,
2330 int insn_idx)
2280{2331{
2281 struct bpf_func_state *cur; /* state of the current function */2332 struct bpf_func_state *cur; /* state of the current function */
2282 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err;2333 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err;
@@ -2402,9 +2453,175 @@ static int check_stack_write(struct bpf_verifier_env *env, · 119 unmodified lines
2402 return 0;2453 return 0;
2403}2454}
2404 2455
2405static int check_stack_read(struct bpf_verifier_env *env,2456/* Write the stack: 'stack[ptr_regno + off] = value_regno'. 'ptr_regno' is
2406 struct bpf_func_state *reg_state /* func where register points to */,2457 * known to contain a variable offset.
2407 int off, int size, int value_regno)2458 * This function checks whether the write is permitted and conservatively
2459 * tracks the effects of the write, considering that each stack slot in the
2460 * dynamic range is potentially written to.
2461 *
2462 * 'off' includes 'regno->off'.
2463 * 'value_regno' can be -1, meaning that an unknown value is being written to
2464 * the stack.
2465 *
2466 * Spilled pointers in range are not marked as written because we don't know
2467 * what's going to be actually written. This means that read propagation for
2468 * future reads cannot be terminated by this write.
2469 *
2470 * For privileged programs, uninitialized stack slots are considered
2471 * initialized by this write (even though we don't know exactly what offsets
2472 * are going to be written to). The idea is that we don't want the verifier to
2473 * reject future reads that access slots written to through variable offsets.
2474 */
2475static int check_stack_write_var_off(struct bpf_verifier_env *env,
2476 /* func where register points to */
2477 struct bpf_func_state *state,
2478 int ptr_regno, int off, int size,
2479 int value_regno, int insn_idx)
2480{
2481 struct bpf_func_state *cur; /* state of the current function */
2482 int min_off, max_off;
2483 int i, err;
2484 struct bpf_reg_state *ptr_reg = NULL, *value_reg = NULL;
2485 bool writing_zero = false;
2486 /* set if the fact that we're writing a zero is used to let any
2487 * stack slots remain STACK_ZERO
2488 */
2489 bool zero_used = false;
2490
2491 cur = env->cur_state->frame[env->cur_state->curframe];
2492 ptr_reg = &cur->regs[ptr_regno];
2493 min_off = ptr_reg->smin_value + off;
2494 max_off = ptr_reg->smax_value + off + size;
2495 if (value_regno >= 0)
2496 value_reg = &cur->regs[value_regno];
2497 if (value_reg && register_is_null(value_reg))
2498 writing_zero = true;
2499
2500 err = realloc_func_state(state, round_up(-min_off, BPF_REG_SIZE),
2501 state->acquired_refs, true);
2502 if (err)
2503 return err;
2504
2505
2506 /* Variable offset writes destroy any spilled pointers in range. */
2507 for (i = min_off; i < max_off; i++) {
2508 u8 new_type, *stype;
2509 int slot, spi;
2510
2511 slot = -i - 1;
2512 spi = slot / BPF_REG_SIZE;
2513 stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE];
2514
2515 if (!env->allow_ptr_leaks
2516 && *stype != NOT_INIT
2517 && *stype != SCALAR_VALUE) {
2518 /* Reject the write if there's are spilled pointers in
2519 * range. If we didn't reject here, the ptr status
2520 * would be erased below (even though not all slots are
2521 * actually overwritten), possibly opening the door to
2522 * leaks.
2523 */
2524 verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d",
2525 insn_idx, i);
2526 return -EINVAL;
2527 }
2528
2529 /* Erase all spilled pointers. */
2530 state->stack[spi].spilled_ptr.type = NOT_INIT;
2531
2532 /* Update the slot type. */
2533 new_type = STACK_MISC;
2534 if (writing_zero && *stype == STACK_ZERO) {
2535 new_type = STACK_ZERO;
2536 zero_used = true;
2537 }
2538 /* If the slot is STACK_INVALID, we check whether it's OK to
2539 * pretend that it will be initialized by this write. The slot
2540 * might not actually be written to, and so if we mark it as
2541 * initialized future reads might leak uninitialized memory.
2542 * For privileged programs, we will accept such reads to slots
2543 * that may or may not be written because, if we're reject
2544 * them, the error would be too confusing.
2545 */
2546 if (*stype == STACK_INVALID && !env->allow_uninit_stack) {
2547 verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d",
2548 insn_idx, i);
2549 return -EINVAL;
2550 }
2551 *stype = new_type;
2552 }
2553 if (zero_used) {
2554 /* backtracking doesn't work for STACK_ZERO yet. */
2555 err = mark_chain_precision(env, value_regno);
2556 if (err)
2557 return err;
2558 }
2559 return 0;
2560}
2561
2562/* When register 'dst_regno' is assigned some values from stack[min_off,
2563 * max_off), we set the register's type according to the types of the
2564 * respective stack slots. If all the stack values are known to be zeros, then
2565 * so is the destination reg. Otherwise, the register is considered to be
2566 * SCALAR. This function does not deal with register filling; the caller must
2567 * ensure that all spilled registers in the stack range have been marked as
2568 * read.
2569 */
2570static void mark_reg_stack_read(struct bpf_verifier_env *env,
2571 /* func where src register points to */
2572 struct bpf_func_state *ptr_state,
2573 int min_off, int max_off, int dst_regno)
2574{
2575 struct bpf_verifier_state *vstate = env->cur_state;
2576 struct bpf_func_state *state = vstate->frame[vstate->curframe];
2577 int i, slot, spi;
2578 u8 *stype;
2579 int zeros = 0;
2580
2581 for (i = min_off; i < max_off; i++) {
2582 slot = -i - 1;
2583 spi = slot / BPF_REG_SIZE;
2584 stype = ptr_state->stack[spi].slot_type;
2585 if (stype[slot % BPF_REG_SIZE] != STACK_ZERO)
2586 break;
2587 zeros++;
2588 }
2589 if (zeros == max_off - min_off) {
2590 /* any access_size read into register is zero extended,
2591 * so the whole register == const_zero
2592 */
2593 __mark_reg_const_zero(&state->regs[dst_regno]);
2594 /* backtracking doesn't support STACK_ZERO yet,
2595 * so mark it precise here, so that later
2596 * backtracking can stop here.
2597 * Backtracking may not need this if this register
2598 * doesn't participate in pointer adjustment.
2599 * Forward propagation of precise flag is not
2600 * necessary either. This mark is only to stop
2601 * backtracking. Any register that contributed
2602 * to const 0 was marked precise before spill.
2603 */
2604 state->regs[dst_regno].precise = true;
2605 } else {
2606 /* have read misc data from the stack */
2607 mark_reg_unknown(env, state->regs, dst_regno);
2608 }
2609 state->regs[dst_regno].live |= REG_LIVE_WRITTEN;
2610}
2611
2612/* Read the stack at 'off' and put the results into the register indicated by
2613 * 'dst_regno'. It handles reg filling if the addressed stack slot is a
2614 * spilled reg.
2615 *
2616 * 'dst_regno' can be -1, meaning that the read value is not going to a
2617 * register.
2618 *
2619 * The access is assumed to be within the current stack bounds.
2620 */
2621static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
2622 /* func where src register points to */
2623 struct bpf_func_state *reg_state,
2624 int off, int size, int dst_regno)
2408{2625{
2409 struct bpf_verifier_state *vstate = env->cur_state;2626 struct bpf_verifier_state *vstate = env->cur_state;
2410 struct bpf_func_state *state = vstate->frame[vstate->curframe];2627 struct bpf_func_state *state = vstate->frame[vstate->curframe];
@@ -2412,11 +2629,6 @@ static int check_stack_read(struct bpf_verifier_env *env, · 1 unmodified line
2412 struct bpf_reg_state *reg;2629 struct bpf_reg_state *reg;
2413 u8 *stype;2630 u8 *stype;
2414 2631
2415 if (reg_state->allocated_stack <= slot) {
2416 verbose(env, "invalid read from stack off %d+0 size %d\n",
2417 off, size);
2418 return -EACCES;
2419 }
2420 stype = reg_state->stack[spi].slot_type;2632 stype = reg_state->stack[spi].slot_type;
2421 reg = &reg_state->stack[spi].spilled_ptr;2633 reg = &reg_state->stack[spi].spilled_ptr;
2422 2634
@@ -2427,9 +2639,9 @@ static int check_stack_read(struct bpf_verifier_env *env, · 4 unmodified lines
2427 verbose(env, "invalid size of register fill\n");2639 verbose(env, "invalid size of register fill\n");
2428 return -EACCES;2640 return -EACCES;
2429 }2641 }
2430 if (value_regno >= 0) {2642 if (dst_regno >= 0) {
2431 mark_reg_unknown(env, state->regs, value_regno);2643 mark_reg_unknown(env, state->regs, dst_regno);
2432 state->regs[value_regno].live |= REG_LIVE_WRITTEN;2644 state->regs[dst_regno].live |= REG_LIVE_WRITTEN;
2433 }2645 }
2434 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);2646 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
2435 return 0;2647 return 0;
@@ -2441,16 +2653,16 @@ static int check_stack_read(struct bpf_verifier_env *env, · 5 unmodified lines
2441 }2653 }
2442 }2654 }
2443 2655
2444 if (value_regno >= 0) {2656 if (dst_regno >= 0) {
2445 /* restore register state from stack */2657 /* restore register state from stack */
2446 state->regs[value_regno] = *reg;2658 state->regs[dst_regno] = *reg;
2447 /* mark reg as written since spilled pointer state likely2659 /* mark reg as written since spilled pointer state likely
2448 * has its liveness marks cleared by is_state_visited()2660 * has its liveness marks cleared by is_state_visited()
2449 * which resets stack/reg liveness for state transitions2661 * which resets stack/reg liveness for state transitions
2450 */2662 */
2451 state->regs[value_regno].live |= REG_LIVE_WRITTEN;2663 state->regs[dst_regno].live |= REG_LIVE_WRITTEN;
2452 } else if (__is_pointer_value(env->allow_ptr_leaks, reg)) {2664 } else if (__is_pointer_value(env->allow_ptr_leaks, reg)) {
2453 /* If value_regno==-1, the caller is asking us whether2665 /* If dst_regno==-1, the caller is asking us whether
2454 * it is acceptable to use this value as a SCALAR_VALUE2666 * it is acceptable to use this value as a SCALAR_VALUE
2455 * (e.g. for XADD).2667 * (e.g. for XADD).
2456 * We must not allow unprivileged callers to do that2668 * We must not allow unprivileged callers to do that
@@ -2462,70 +2674,167 @@ static int check_stack_read(struct bpf_verifier_env *env, · 5 unmodified lines
2462 }2674 }
2463 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);2675 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
2464 } else {2676 } else {
2465 int zeros = 0;2677 u8 type;
2466 2678
2467 for (i = 0; i < size; i++) {2679 for (i = 0; i < size; i++) {
2468 if (stype[(slot - i) % BPF_REG_SIZE] == STACK_MISC)2680 type = stype[(slot - i) % BPF_REG_SIZE];
2681 if (type == STACK_MISC)
2469 continue;2682 continue;
2470 if (stype[(slot - i) % BPF_REG_SIZE] == STACK_ZERO) {2683 if (type == STACK_ZERO)
2471 zeros++;
2472 continue;2684 continue;
2473 }
2474 verbose(env, "invalid read from stack off %d+%d size %d\n",2685 verbose(env, "invalid read from stack off %d+%d size %d\n",
2475 off, i, size);2686 off, i, size);
2476 return -EACCES;2687 return -EACCES;
2477 }2688 }
2478 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);2689 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
2479 if (value_regno >= 0) {2690 if (dst_regno >= 0)
2480 if (zeros == size) {2691 mark_reg_stack_read(env, reg_state, off, off + size, dst_regno);
2481 /* any size read into register is zero extended,
2482 * so the whole register == const_zero
2483 */
2484 __mark_reg_const_zero(&state->regs[value_regno]);
2485 /* backtracking doesn't support STACK_ZERO yet,
2486 * so mark it precise here, so that later
2487 * backtracking can stop here.
2488 * Backtracking may not need this if this register
2489 * doesn't participate in pointer adjustment.
2490 * Forward propagation of precise flag is not
2491 * necessary either. This mark is only to stop
2492 * backtracking. Any register that contributed
2493 * to const 0 was marked precise before spill.
2494 */
2495 state->regs[value_regno].precise = true;
2496 } else {
2497 /* have read misc data from the stack */
2498 mark_reg_unknown(env, state->regs, value_regno);
2499 }
2500 state->regs[value_regno].live |= REG_LIVE_WRITTEN;
2501 }
2502 }2692 }
2503 return 0;2693 return 0;
2504}2694}
2505 2695
2506static int check_stack_access(struct bpf_verifier_env *env,2696enum stack_access_src {
2507 const struct bpf_reg_state *reg,2697 ACCESS_DIRECT = 1, /* the access is performed by an instruction */
2508 int off, int size)2698 ACCESS_HELPER = 2, /* the access is performed by a helper */
2699};
2700
2701static int check_stack_range_initialized(struct bpf_verifier_env *env,
2702 int regno, int off, int access_size,
2703 bool zero_size_allowed,
2704 enum stack_access_src type,
2705 struct bpf_call_arg_meta *meta);
2706
2707static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno)
2708{
2709 return cur_regs(env) + regno;
2710}
2711
2712/* Read the stack at 'ptr_regno + off' and put the result into the register
2713 * 'dst_regno'.
2714 * 'off' includes the pointer register's fixed offset(i.e. 'ptr_regno.off'),
2715 * but not its variable offset.
2716 * 'size' is assumed to be <= reg size and the access is assumed to be aligned.
2717 *
2718 * As opposed to check_stack_read_fixed_off, this function doesn't deal with
2719 * filling registers (i.e. reads of spilled register cannot be detected when
2720 * the offset is not fixed). We conservatively mark 'dst_regno' as containing
2721 * SCALAR_VALUE. That's why we assert that the 'ptr_regno' has a variable
2722 * offset; for a fixed offset check_stack_read_fixed_off should be used
2723 * instead.
2724 */
2725static int check_stack_read_var_off(struct bpf_verifier_env *env,
2726 int ptr_regno, int off, int size, int dst_regno)
2509{2727{
2510 /* Stack accesses must be at a fixed offset, so that we2728 /* The state of the source register. */
2511 * can determine what type of data were returned. See2729 struct bpf_reg_state *reg = reg_state(env, ptr_regno);
2512 * check_stack_read().2730 struct bpf_func_state *ptr_state = func(env, reg);
2731 int err;
2732 int min_off, max_off;
2733
2734 /* Note that we pass a NULL meta, so raw access will not be permitted.
2513 */2735 */
2514 if (!tnum_is_const(reg->var_off)) {2736 err = check_stack_range_initialized(env, ptr_regno, off, size,
2737 false, ACCESS_DIRECT, NULL);
2738 if (err)
2739 return err;
2740
2741 min_off = reg->smin_value + off;
2742 max_off = reg->smax_value + off;
2743 mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno);
2744 return 0;
2745}
2746
2747/* check_stack_read dispatches to check_stack_read_fixed_off or
2748 * check_stack_read_var_off.
2749 *
2750 * The caller must ensure that the offset falls within the allocated stack
2751 * bounds.
2752 *
2753 * 'dst_regno' is a register which will receive the value from the stack. It
2754 * can be -1, meaning that the read value is not going to a register.
2755 */
2756static int check_stack_read(struct bpf_verifier_env *env,
2757 int ptr_regno, int off, int size,
2758 int dst_regno)
2759{
2760 struct bpf_reg_state *reg = reg_state(env, ptr_regno);
2761 struct bpf_func_state *state = func(env, reg);
2762 int err;
2763 /* Some accesses are only permitted with a static offset. */
2764 bool var_off = !tnum_is_const(reg->var_off);
2765
2766 /* The offset is required to be static when reads don't go to a
2767 * register, in order to not leak pointers (see
2768 * check_stack_read_fixed_off).
2769 */
2770 if (dst_regno < 0 && var_off) {
2515 char tn_buf[48];2771 char tn_buf[48];
2516 2772
2517 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);2773 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
2518 verbose(env, "variable stack access var_off=%s off=%d size=%d\n",2774 verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n",
2519 tn_buf, off, size);2775 tn_buf, off, size);
2520 return -EACCES;2776 return -EACCES;
2521 }2777 }
2778 /* Variable offset is prohibited for unprivileged mode for simplicity
2779 * since it requires corresponding support in Spectre masking for stack
2780 * ALU. See also retrieve_ptr_limit().
2781 */
2782 if (!env->bypass_spec_v1 && var_off) {
2783 char tn_buf[48];
2522 2784
2523 if (off >= 0 || off < -MAX_BPF_STACK) {2785 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
2524 verbose(env, "invalid stack off=%d size=%d\n", off, size);2786 verbose(env, "R%d variable offset stack access prohibited for !root, var_off=%s\n",
2787 ptr_regno, tn_buf);
2525 return -EACCES;2788 return -EACCES;
2526 }2789 }
2527 2790
2528 return 0;2791 if (!var_off) {
2792 off += reg->var_off.value;
2793 err = check_stack_read_fixed_off(env, state, off, size,
2794 dst_regno);
2795 } else {
2796 /* Variable offset stack reads need more conservative handling
2797 * than fixed offset ones. Note that dst_regno >= 0 on this
2798 * branch.
2799 */
2800 err = check_stack_read_var_off(env, ptr_regno, off, size,
2801 dst_regno);
2802 }
2803 return err;
2804}
2805
2806
2807/* check_stack_write dispatches to check_stack_write_fixed_off or
2808 * check_stack_write_var_off.
2809 *
2810 * 'ptr_regno' is the register used as a pointer into the stack.
2811 * 'off' includes 'ptr_regno->off', but not its variable offset (if any).
2812 * 'value_regno' is the register whose value we're writing to the stack. It can
2813 * be -1, meaning that we're not writing from a register.
2814 *
2815 * The caller must ensure that the offset falls within the maximum stack size.
2816 */
2817static int check_stack_write(struct bpf_verifier_env *env,
2818 int ptr_regno, int off, int size,
2819 int value_regno, int insn_idx)
2820{
2821 struct bpf_reg_state *reg = reg_state(env, ptr_regno);
2822 struct bpf_func_state *state = func(env, reg);
2823 int err;
2824
2825 if (tnum_is_const(reg->var_off)) {
2826 off += reg->var_off.value;
2827 err = check_stack_write_fixed_off(env, state, off, size,
2828 value_regno, insn_idx);
2829 } else {
2830 /* Variable offset stack reads need more conservative handling
2831 * than fixed offset ones.
2832 */
2833 err = check_stack_write_var_off(env, state,
2834 ptr_regno, off, size,
2835 value_regno, insn_idx);
2836 }
2837 return err;
2529}2838}
2530 2839
2531static int check_map_access_type(struct bpf_verifier_env *env, u32 regno,2840static int check_map_access_type(struct bpf_verifier_env *env, u32 regno,
@@ -2858,11 +3167,6 @@ static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, · 326 unmodified lines
2858 return -EACCES;3167 return -EACCES;
2859}3168}
2860 3169
2861static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno)
2862{
2863 return cur_regs(env) + regno;
2864}
2865
2866static bool is_pointer_value(struct bpf_verifier_env *env, int regno)3170static bool is_pointer_value(struct bpf_verifier_env *env, int regno)
2867{3171{
2868 return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno));3172 return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno));
@@ -2981,8 +3285,8 @@ static int check_ptr_alignment(struct bpf_verifier_env *env, · 112 unmodified lines
2981 break;3285 break;
2982 case PTR_TO_STACK:3286 case PTR_TO_STACK:
2983 pointer_desc = "stack ";3287 pointer_desc = "stack ";
2984 /* The stack spill tracking logic in check_stack_write()3288 /* The stack spill tracking logic in check_stack_write_fixed_off()
2985 * and check_stack_read() relies on stack accesses being3289 * and check_stack_read_fixed_off() relies on stack accesses being
2986 * aligned.3290 * aligned.
2987 */3291 */
2988 strict = true;3292 strict = true;
@@ -3074,9 +3378,7 @@ process_func: · 85 unmodified lines
3074continue_func:3378continue_func:
3075 subprog_end = subprog[idx + 1].start;3379 subprog_end = subprog[idx + 1].start;
3076 for (; i < subprog_end; i++) {3380 for (; i < subprog_end; i++) {
3077 if (insn[i].code != (BPF_JMP | BPF_CALL))3381 if (!bpf_pseudo_call(insn + i))
3078 continue;
3079 if (insn[i].src_reg != BPF_PSEUDO_CALL)
3080 continue;3382 continue;
3081 /* remember insn and function to return to */3383 /* remember insn and function to return to */
3082 ret_insn[frame] = i + 1;3384 ret_insn[frame] = i + 1;
@@ -3400,6 +3702,91 @@ static int check_ptr_to_map_access(struct bpf_verifier_env *env, · 317 unmodified lines
3400 return 0;3702 return 0;
3401}3703}
3402 3704
3705/* Check that the stack access at the given offset is within bounds. The
3706 * maximum valid offset is -1.
3707 *
3708 * The minimum valid offset is -MAX_BPF_STACK for writes, and
3709 * -state->allocated_stack for reads.
3710 */
3711static int check_stack_slot_within_bounds(int off,
3712 struct bpf_func_state *state,
3713 enum bpf_access_type t)
3714{
3715 int min_valid_off;
3716
3717 if (t == BPF_WRITE)
3718 min_valid_off = -MAX_BPF_STACK;
3719 else
3720 min_valid_off = -state->allocated_stack;
3721
3722 if (off < min_valid_off || off > -1)
3723 return -EACCES;
3724 return 0;
3725}
3726
3727/* Check that the stack access at 'regno + off' falls within the maximum stack
3728 * bounds.
3729 *
3730 * 'off' includes `regno->offset`, but not its dynamic part (if any).
3731 */
3732static int check_stack_access_within_bounds(
3733 struct bpf_verifier_env *env,
3734 int regno, int off, int access_size,
3735 enum stack_access_src src, enum bpf_access_type type)
3736{
3737 struct bpf_reg_state *regs = cur_regs(env);
3738 struct bpf_reg_state *reg = regs + regno;
3739 struct bpf_func_state *state = func(env, reg);
3740 int min_off, max_off;
3741 int err;
3742 char *err_extra;
3743
3744 if (src == ACCESS_HELPER)
3745 /* We don't know if helpers are reading or writing (or both). */
3746 err_extra = " indirect access to";
3747 else if (type == BPF_READ)
3748 err_extra = " read from";
3749 else
3750 err_extra = " write to";
3751
3752 if (tnum_is_const(reg->var_off)) {
3753 min_off = reg->var_off.value + off;
3754 if (access_size > 0)
3755 max_off = min_off + access_size - 1;
3756 else
3757 max_off = min_off;
3758 } else {
3759 if (reg->smax_value >= BPF_MAX_VAR_OFF ||
3760 reg->smin_value <= -BPF_MAX_VAR_OFF) {
3761 verbose(env, "invalid unbounded variable-offset%s stack R%d\n",
3762 err_extra, regno);
3763 return -EACCES;
3764 }
3765 min_off = reg->smin_value + off;
3766 if (access_size > 0)
3767 max_off = reg->smax_value + off + access_size - 1;
3768 else
3769 max_off = min_off;
3770 }
3771
3772 err = check_stack_slot_within_bounds(min_off, state, type);
3773 if (!err)
3774 err = check_stack_slot_within_bounds(max_off, state, type);
3775
3776 if (err) {
3777 if (tnum_is_const(reg->var_off)) {
3778 verbose(env, "invalid%s stack R%d off=%d size=%d\n",
3779 err_extra, regno, off, access_size);
3780 } else {
3781 char tn_buf[48];
3782
3783 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
3784 verbose(env, "invalid variable-offset%s stack R%d var_off=%s size=%d\n",
3785 err_extra, regno, tn_buf, access_size);
3786 }
3787 }
3788 return err;
3789}
3403 3790
3404/* check whether memory at (regno + off) is accessible for t = (read | write)3791/* check whether memory at (regno + off) is accessible for t = (read | write)
3405 * if t==write, value_regno is a register which value is stored into memory3792 * if t==write, value_regno is a register which value is stored into memory
@@ -3515,8 +3902,8 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 109 unmodified lines
3515 }3902 }
3516 3903
3517 } else if (reg->type == PTR_TO_STACK) {3904 } else if (reg->type == PTR_TO_STACK) {
3518 off += reg->var_off.value;3905 /* Basic bounds checks. */
3519 err = check_stack_access(env, reg, off, size);3906 err = check_stack_access_within_bounds(env, regno, off, size, ACCESS_DIRECT, t);
3520 if (err)3907 if (err)
3521 return err;3908 return err;
3522 3909
@@ -3525,12 +3912,12 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 2 unmodified lines
3525 if (err)3912 if (err)
3526 return err;3913 return err;
3527 3914
3528 if (t == BPF_WRITE)3915 if (t == BPF_READ)
3529 err = check_stack_write(env, state, off, size,3916 err = check_stack_read(env, regno, off, size,
3530 value_regno, insn_idx);
3531 else
3532 err = check_stack_read(env, state, off, size,
3533 value_regno);3917 value_regno);
3918 else
3919 err = check_stack_write(env, regno, off, size,
3920 value_regno, insn_idx);
3534 } else if (reg_is_pkt_pointer(reg)) {3921 } else if (reg_is_pkt_pointer(reg)) {
3535 if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) {3922 if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) {
3536 verbose(env, "cannot write into packet\n");3923 verbose(env, "cannot write into packet\n");
@@ -3665,9 +4052,26 @@ static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_i · 128 unmodified lines
3665 return -EACCES;4052 return -EACCES;
3666 }4053 }
3667 4054
4055 if (insn->imm & BPF_FETCH) {
4056 if (insn->imm == BPF_CMPXCHG)
4057 load_reg = BPF_REG_0;
4058 else
4059 load_reg = insn->src_reg;
4060
4061 /* check and record load of old value */
4062 err = check_reg_arg(env, load_reg, DST_OP);
4063 if (err)
4064 return err;
4065 } else {
4066 /* This instruction accesses a memory location but doesn't
4067 * actually load it into a register.
4068 */
4069 load_reg = -1;
4070 }
4071
3668 /* check whether we can read the memory */4072 /* check whether we can read the memory */
3669 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,4073 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
3670 BPF_SIZE(insn->code), BPF_READ, -1, true);4074 BPF_SIZE(insn->code), BPF_READ, load_reg, true);
3671 if (err)4075 if (err)
3672 return err;4076 return err;
3673 4077
@@ -3677,65 +4081,56 @@ static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_i · 3 unmodified lines
3677 if (err)4081 if (err)
3678 return err;4082 return err;
3679 4083
3680 if (!(insn->imm & BPF_FETCH))
3681 return 0;
3682
3683 if (insn->imm == BPF_CMPXCHG)
3684 load_reg = BPF_REG_0;
3685 else
3686 load_reg = insn->src_reg;
3687
3688 /* check and record load of old value */
3689 err = check_reg_arg(env, load_reg, DST_OP);
3690 if (err)
3691 return err;
3692
3693 return 0;4084 return 0;
3694}4085}
3695 4086
3696static int __check_stack_boundary(struct bpf_verifier_env *env, u32 regno,4087/* When register 'regno' is used to read the stack (either directly or through
3697 int off, int access_size,4088 * a helper function) make sure that it's within stack boundary and, depending
3698 bool zero_size_allowed)4089 * on the access type, that all elements of the stack are initialized.
4090 *
4091 * 'off' includes 'regno->off', but not its dynamic part (if any).
4092 *
4093 * All registers that have been spilled on the stack in the slots within the
4094 * read offsets are marked as read.
4095 */
4096static int check_stack_range_initialized(
4097 struct bpf_verifier_env *env, int regno, int off,
4098 int access_size, bool zero_size_allowed,
4099 enum stack_access_src type, struct bpf_call_arg_meta *meta)
3699{4100{
3700 struct bpf_reg_state *reg = reg_state(env, regno);4101 struct bpf_reg_state *reg = reg_state(env, regno);
4102 struct bpf_func_state *state = func(env, reg);
4103 int err, min_off, max_off, i, j, slot, spi;
4104 char *err_extra = type == ACCESS_HELPER ? " indirect" : "";
4105 enum bpf_access_type bounds_check_type;
4106 /* Some accesses can write anything into the stack, others are
4107 * read-only.
4108 */
4109 bool clobber = false;
3701 4110
3702 if (off >= 0 || off < -MAX_BPF_STACK || off + access_size > 0 ||4111 if (access_size == 0 && !zero_size_allowed) {
3703 access_size < 0 || (access_size == 0 && !zero_size_allowed)) {4112 verbose(env, "invalid zero-sized read\n");
3704 if (tnum_is_const(reg->var_off)) {
3705 verbose(env, "invalid stack type R%d off=%d access_size=%d\n",
3706 regno, off, access_size);
3707 } else {
3708 char tn_buf[48];
3709
3710 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
3711 verbose(env, "invalid stack type R%d var_off=%s access_size=%d\n",
3712 regno, tn_buf, access_size);
3713 }
3714 return -EACCES;4113 return -EACCES;
3715 }4114 }
3716 return 0;
3717}
3718 4115
3719/* when register 'regno' is passed into function that will read 'access_size'4116 if (type == ACCESS_HELPER) {
3720 * bytes from that pointer, make sure that it's within stack boundary4117 /* The bounds checks for writes are more permissive than for
3721 * and all elements of stack are initialized.4118 * reads. However, if raw_mode is not set, we'll do extra
3722 * Unlike most pointer bounds-checking functions, this one doesn't take an4119 * checks below.
3723 * 'off' argument, so it has to add in reg->off itself.4120 */
3724 */4121 bounds_check_type = BPF_WRITE;
3725static int check_stack_boundary(struct bpf_verifier_env *env, int regno,4122 clobber = true;
3726 int access_size, bool zero_size_allowed,4123 } else {
3727 struct bpf_call_arg_meta *meta)4124 bounds_check_type = BPF_READ;
3728{4125 }
3729 struct bpf_reg_state *reg = reg_state(env, regno);4126 err = check_stack_access_within_bounds(env, regno, off, access_size,
3730 struct bpf_func_state *state = func(env, reg);4127 type, bounds_check_type);
3731 int err, min_off, max_off, i, j, slot, spi;4128 if (err)
4129 return err;
4130
3732 4131
3733 if (tnum_is_const(reg->var_off)) {4132 if (tnum_is_const(reg->var_off)) {
3734 min_off = max_off = reg->var_off.value + reg->off;4133 min_off = max_off = reg->var_off.value + off;
3735 err = __check_stack_boundary(env, regno, min_off, access_size,
3736 zero_size_allowed);
3737 if (err)
3738 return err;
3739 } else {4134 } else {
3740 /* Variable offset is prohibited for unprivileged mode for4135 /* Variable offset is prohibited for unprivileged mode for
3741 * simplicity since it requires corresponding support in4136 * simplicity since it requires corresponding support in
@@ -3746,8 +4141,8 @@ static int check_stack_boundary(struct bpf_verifier_env *env, int regno, · 4 unmodified lines
3746 char tn_buf[48];4141 char tn_buf[48];
3747 4142
3748 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);4143 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
3749 verbose(env, "R%d indirect variable offset stack access prohibited for !root, var_off=%s\n",4144 verbose(env, "R%d%s variable offset stack access prohibited for !root, var_off=%s\n",
3750 regno, tn_buf);4145 regno, err_extra, tn_buf);
3751 return -EACCES;4146 return -EACCES;
3752 }4147 }
3753 /* Only initialized buffer on stack is allowed to be accessed4148 /* Only initialized buffer on stack is allowed to be accessed
@@ -3759,28 +4154,8 @@ static int check_stack_boundary(struct bpf_verifier_env *env, int regno, · 5 unmodified lines
3759 if (meta && meta->raw_mode)4154 if (meta && meta->raw_mode)
3760 meta = NULL;4155 meta = NULL;
3761 4156
3762 if (reg->smax_value >= BPF_MAX_VAR_OFF ||4157 min_off = reg->smin_value + off;
3763 reg->smax_value <= -BPF_MAX_VAR_OFF) {4158 max_off = reg->smax_value + off;
3764 verbose(env, "R%d unbounded indirect variable offset stack access\n",
3765 regno);
3766 return -EACCES;
3767 }
3768 min_off = reg->smin_value + reg->off;
3769 max_off = reg->smax_value + reg->off;
3770 err = __check_stack_boundary(env, regno, min_off, access_size,
3771 zero_size_allowed);
3772 if (err) {
3773 verbose(env, "R%d min value is outside of stack bound\n",
3774 regno);
3775 return err;
3776 }
3777 err = __check_stack_boundary(env, regno, max_off, access_size,
3778 zero_size_allowed);
3779 if (err) {
3780 verbose(env, "R%d max value is outside of stack bound\n",
3781 regno);
3782 return err;
3783 }
3784 }4159 }
3785 4160
3786 if (meta && meta->raw_mode) {4161 if (meta && meta->raw_mode) {
@@ -3800,8 +4175,10 @@ static int check_stack_boundary(struct bpf_verifier_env *env, int regno, · 13 unmodified lines
3800 if (*stype == STACK_MISC)4175 if (*stype == STACK_MISC)
3801 goto mark;4176 goto mark;
3802 if (*stype == STACK_ZERO) {4177 if (*stype == STACK_ZERO) {
3803 /* helper can write anything into the stack */4178 if (clobber) {
3804 *stype = STACK_MISC;4179 /* helper can write anything into the stack */
4180 *stype = STACK_MISC;
4181 }
3805 goto mark;4182 goto mark;
3806 }4183 }
3807 4184
@@ -3812,22 +4189,24 @@ static int check_stack_boundary(struct bpf_verifier_env *env, int regno, · 4 unmodified lines
3812 if (state->stack[spi].slot_type[0] == STACK_SPILL &&4189 if (state->stack[spi].slot_type[0] == STACK_SPILL &&
3813 (state->stack[spi].spilled_ptr.type == SCALAR_VALUE ||4190 (state->stack[spi].spilled_ptr.type == SCALAR_VALUE ||
3814 env->allow_ptr_leaks)) {4191 env->allow_ptr_leaks)) {
3815 __mark_reg_unknown(env, &state->stack[spi].spilled_ptr);4192 if (clobber) {
3816 for (j = 0; j < BPF_REG_SIZE; j++)4193 __mark_reg_unknown(env, &state->stack[spi].spilled_ptr);
3817 state->stack[spi].slot_type[j] = STACK_MISC;4194 for (j = 0; j < BPF_REG_SIZE; j++)
4195 state->stack[spi].slot_type[j] = STACK_MISC;
4196 }
3818 goto mark;4197 goto mark;
3819 }4198 }
3820 4199
3821err:4200err:
3822 if (tnum_is_const(reg->var_off)) {4201 if (tnum_is_const(reg->var_off)) {
3823 verbose(env, "invalid indirect read from stack off %d+%d size %d\n",4202 verbose(env, "invalid%s read from stack R%d off %d+%d size %d\n",
3824 min_off, i - min_off, access_size);4203 err_extra, regno, min_off, i - min_off, access_size);
3825 } else {4204 } else {
3826 char tn_buf[48];4205 char tn_buf[48];
3827 4206
3828 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);4207 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
3829 verbose(env, "invalid indirect read from stack var_off %s+%d size %d\n",4208 verbose(env, "invalid%s read from stack R%d var_off %s+%d size %d\n",
3830 tn_buf, i - min_off, access_size);4209 err_extra, regno, tn_buf, i - min_off, access_size);
3831 }4210 }
3832 return -EACCES;4211 return -EACCES;
3833mark:4212mark:
@@ -3876,8 +4255,10 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, · 42 unmodified lines
3876 "rdwr",4255 "rdwr",
3877 &env->prog->aux->max_rdwr_access);4256 &env->prog->aux->max_rdwr_access);
3878 case PTR_TO_STACK:4257 case PTR_TO_STACK:
3879 return check_stack_boundary(env, regno, access_size,4258 return check_stack_range_initialized(
3880 zero_size_allowed, meta);4259 env,
4260 regno, reg->off, access_size,
4261 zero_size_allowed, ACCESS_HELPER, meta);
3881 default: /* scalar_value or invalid ptr */4262 default: /* scalar_value or invalid ptr */
3882 /* Allow zero-byte read from NULL, regardless of pointer type */4263 /* Allow zero-byte read from NULL, regardless of pointer type */
3883 if (zero_size_allowed && access_size == 0 &&4264 if (zero_size_allowed && access_size == 0 &&
@@ -3891,6 +4272,29 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, · 7 unmodified lines
3891 }4272 }
3892}4273}
3893 4274
4275int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
4276 u32 regno, u32 mem_size)
4277{
4278 if (register_is_null(reg))
4279 return 0;
4280
4281 if (reg_type_may_be_null(reg->type)) {
4282 /* Assuming that the register contains a value check if the memory
4283 * access is safe. Temporarily save and restore the register's state as
4284 * the conversion shouldn't be visible to a caller.
4285 */
4286 const struct bpf_reg_state saved_reg = *reg;
4287 int rv;
4288
4289 mark_ptr_not_null_reg(reg);
4290 rv = check_helper_mem_access(env, regno, mem_size, true, NULL);
4291 *reg = saved_reg;
4292 return rv;
4293 }
4294
4295 return check_helper_mem_access(env, regno, mem_size, true, NULL);
4296}
4297
3894/* Implementation details:4298/* Implementation details:
3895 * bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL4299 * bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL
3896 * Two bpf_map_lookups (even with the same key) will have different reg->id.4300 * Two bpf_map_lookups (even with the same key) will have different reg->id.
@@ -4875,8 +5279,9 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · 978 unmodified lines
4875 subprog);5279 subprog);
4876 clear_caller_saved_regs(env, caller->regs);5280 clear_caller_saved_regs(env, caller->regs);
4877 5281
4878 /* All global functions return SCALAR_VALUE */5282 /* All global functions return a 64-bit SCALAR_VALUE */
4879 mark_reg_unknown(env, caller->regs, BPF_REG_0);5283 mark_reg_unknown(env, caller->regs, BPF_REG_0);
5284 caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;
4880 5285
4881 /* continue with next insn after call */5286 /* continue with next insn after call */
4882 return 0;5287 return 0;
@@ -5541,6 +5946,41 @@ do_sim: · 658 unmodified lines
5541 return !ret ? -EFAULT : 0;5946 return !ret ? -EFAULT : 0;
5542}5947}
5543 5948
5949/* check that stack access falls within stack limits and that 'reg' doesn't
5950 * have a variable offset.
5951 *
5952 * Variable offset is prohibited for unprivileged mode for simplicity since it
5953 * requires corresponding support in Spectre masking for stack ALU. See also
5954 * retrieve_ptr_limit().
5955 *
5956 *
5957 * 'off' includes 'reg->off'.
5958 */
5959static int check_stack_access_for_ptr_arithmetic(
5960 struct bpf_verifier_env *env,
5961 int regno,
5962 const struct bpf_reg_state *reg,
5963 int off)
5964{
5965 if (!tnum_is_const(reg->var_off)) {
5966 char tn_buf[48];
5967
5968 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
5969 verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n",
5970 regno, tn_buf, off);
5971 return -EACCES;
5972 }
5973
5974 if (off >= 0 || off < -MAX_BPF_STACK) {
5975 verbose(env, "R%d stack pointer arithmetic goes out of range, "
5976 "prohibited for !root; off=%d\n", regno, off);
5977 return -EACCES;
5978 }
5979
5980 return 0;
5981}
5982
5983
5544/* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off.5984/* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off.
5545 * Caller should also handle BPF_MOV case separately.5985 * Caller should also handle BPF_MOV case separately.
5546 * If we return -EACCES, caller may want to try again treating pointer as a5986 * If we return -EACCES, caller may want to try again treating pointer as a
@@ -5784,10 +6224,9 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, · 237 unmodified lines
5784 "prohibited for !root\n", dst);6224 "prohibited for !root\n", dst);
5785 return -EACCES;6225 return -EACCES;
5786 } else if (dst_reg->type == PTR_TO_STACK &&6226 } else if (dst_reg->type == PTR_TO_STACK &&
5787 check_stack_access(env, dst_reg, dst_reg->off +6227 check_stack_access_for_ptr_arithmetic(
5788 dst_reg->var_off.value, 1)) {6228 env, dst, dst_reg, dst_reg->off +
5789 verbose(env, "R%d stack pointer arithmetic goes out of range, "6229 dst_reg->var_off.value)) {
5790 "prohibited for !root\n", dst);
5791 return -EACCES;6230 return -EACCES;
5792 }6231 }
5793 }6232 }
@@ -6266,7 +6705,7 @@ static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg, · 472 unmodified lines
6266 * 3) the signed bounds cross zero, so they tell us nothing6705 * 3) the signed bounds cross zero, so they tell us nothing
6267 * about the result6706 * about the result
6268 * If the value in dst_reg is known nonnegative, then again the6707 * If the value in dst_reg is known nonnegative, then again the
6269 * unsigned bounts capture the signed bounds.6708 * unsigned bounds capture the signed bounds.
6270 * Thus, in all cases it suffices to blow away our signed bounds6709 * Thus, in all cases it suffices to blow away our signed bounds
6271 * and rely on inferring new ones from the unsigned bounds and6710 * and rely on inferring new ones from the unsigned bounds and
6272 * var_off of the result.6711 * var_off of the result.
@@ -6297,7 +6736,7 @@ static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg, · 24 unmodified lines
6297 * 3) the signed bounds cross zero, so they tell us nothing6736 * 3) the signed bounds cross zero, so they tell us nothing
6298 * about the result6737 * about the result
6299 * If the value in dst_reg is known nonnegative, then again the6738 * If the value in dst_reg is known nonnegative, then again the
6300 * unsigned bounts capture the signed bounds.6739 * unsigned bounds capture the signed bounds.
6301 * Thus, in all cases it suffices to blow away our signed bounds6740 * Thus, in all cases it suffices to blow away our signed bounds
6302 * and rely on inferring new ones from the unsigned bounds and6741 * and rely on inferring new ones from the unsigned bounds and
6303 * var_off of the result.6742 * var_off of the result.
@@ -7367,43 +7806,19 @@ static void mark_ptr_or_null_reg(struct bpf_func_state *state, · 1063 unmodified lines
7367 }7806 }
7368 if (is_null) {7807 if (is_null) {
7369 reg->type = SCALAR_VALUE;7808 reg->type = SCALAR_VALUE;
7370 } else if (reg->type == PTR_TO_MAP_VALUE_OR_NULL) {
7371 const struct bpf_map *map = reg->map_ptr;
7372
7373 if (map->inner_map_meta) {
7374 reg->type = CONST_PTR_TO_MAP;
7375 reg->map_ptr = map->inner_map_meta;
7376 } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) {
7377 reg->type = PTR_TO_XDP_SOCK;
7378 } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP ||
7379 map->map_type == BPF_MAP_TYPE_SOCKHASH) {
7380 reg->type = PTR_TO_SOCKET;
7381 } else {
7382 reg->type = PTR_TO_MAP_VALUE;
7383 }
7384 } else if (reg->type == PTR_TO_SOCKET_OR_NULL) {
7385 reg->type = PTR_TO_SOCKET;
7386 } else if (reg->type == PTR_TO_SOCK_COMMON_OR_NULL) {
7387 reg->type = PTR_TO_SOCK_COMMON;
7388 } else if (reg->type == PTR_TO_TCP_SOCK_OR_NULL) {
7389 reg->type = PTR_TO_TCP_SOCK;
7390 } else if (reg->type == PTR_TO_BTF_ID_OR_NULL) {
7391 reg->type = PTR_TO_BTF_ID;
7392 } else if (reg->type == PTR_TO_MEM_OR_NULL) {
7393 reg->type = PTR_TO_MEM;
7394 } else if (reg->type == PTR_TO_RDONLY_BUF_OR_NULL) {
7395 reg->type = PTR_TO_RDONLY_BUF;
7396 } else if (reg->type == PTR_TO_RDWR_BUF_OR_NULL) {
7397 reg->type = PTR_TO_RDWR_BUF;
7398 }
7399 if (is_null) {
7400 /* We don't need id and ref_obj_id from this point7809 /* We don't need id and ref_obj_id from this point
7401 * onwards anymore, thus we should better reset it,7810 * onwards anymore, thus we should better reset it,
7402 * so that state pruning has chances to take effect.7811 * so that state pruning has chances to take effect.
7403 */7812 */
7404 reg->id = 0;7813 reg->id = 0;
7405 reg->ref_obj_id = 0;7814 reg->ref_obj_id = 0;
7406 } else if (!reg_may_point_to_spin_lock(reg)) {7815
7816 return;
7817 }
7818
7819 mark_ptr_not_null_reg(reg);
7820
7821 if (!reg_may_point_to_spin_lock(reg)) {
7407 /* For not-NULL ptr, reg->ref_obj_id will be reset7822 /* For not-NULL ptr, reg->ref_obj_id will be reset
7408 * in release_reg_references().7823 * in release_reg_references().
7409 *7824 *
@@ -7986,6 +8401,9 @@ static int check_return_code(struct bpf_verifier_env *env) · 576 unmodified lines
7986 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME ||8401 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME ||
7987 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME)8402 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME)
7988 range = tnum_range(1, 1);8403 range = tnum_range(1, 1);
8404 if (env->prog->expected_attach_type == BPF_CGROUP_INET4_BIND ||
8405 env->prog->expected_attach_type == BPF_CGROUP_INET6_BIND)
8406 range = tnum_range(0, 3);
7989 break;8407 break;
7990 case BPF_PROG_TYPE_CGROUP_SKB:8408 case BPF_PROG_TYPE_CGROUP_SKB:
7991 if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) {8409 if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) {
@@ -10015,15 +10433,22 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, · 2023 unmodified lines
10015 case BPF_MAP_TYPE_HASH:10433 case BPF_MAP_TYPE_HASH:
10016 case BPF_MAP_TYPE_LRU_HASH:10434 case BPF_MAP_TYPE_LRU_HASH:
10017 case BPF_MAP_TYPE_ARRAY:10435 case BPF_MAP_TYPE_ARRAY:
10436 case BPF_MAP_TYPE_PERCPU_HASH:
10437 case BPF_MAP_TYPE_PERCPU_ARRAY:
10438 case BPF_MAP_TYPE_LRU_PERCPU_HASH:
10439 case BPF_MAP_TYPE_ARRAY_OF_MAPS:
10440 case BPF_MAP_TYPE_HASH_OF_MAPS:
10018 if (!is_preallocated_map(map)) {10441 if (!is_preallocated_map(map)) {
10019 verbose(env,10442 verbose(env,
10020 "Sleepable programs can only use preallocated hash maps\n");10443 "Sleepable programs can only use preallocated maps\n");
10021 return -EINVAL;10444 return -EINVAL;
10022 }10445 }
10023 break;10446 break;
10447 case BPF_MAP_TYPE_RINGBUF:
10448 break;
10024 default:10449 default:
10025 verbose(env,10450 verbose(env,
10026 "Sleepable programs can only use array and hash maps\n");10451 "Sleepable programs can only use array, hash, and ringbuf maps\n");
10027 return -EINVAL;10452 return -EINVAL;
10028 }10453 }
10029 10454
@@ -10581,6 +11006,7 @@ static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, · 551 unmodified lines
10581 for (i = 0; i < len; i++) {11006 for (i = 0; i < len; i++) {
10582 int adj_idx = i + delta;11007 int adj_idx = i + delta;
10583 struct bpf_insn insn;11008 struct bpf_insn insn;
11009 u8 load_reg;
10584 11010
10585 insn = insns[adj_idx];11011 insn = insns[adj_idx];
10586 if (!aux[adj_idx].zext_dst) {11012 if (!aux[adj_idx].zext_dst) {
@@ -10623,9 +11049,27 @@ static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, · 36 unmodified lines
10623 if (!bpf_jit_needs_zext())11049 if (!bpf_jit_needs_zext())
10624 continue;11050 continue;
10625 11051
11052 /* zext_dst means that we want to zero-extend whatever register
11053 * the insn defines, which is dst_reg most of the time, with
11054 * the notable exception of BPF_STX + BPF_ATOMIC + BPF_FETCH.
11055 */
11056 if (BPF_CLASS(insn.code) == BPF_STX &&
11057 BPF_MODE(insn.code) == BPF_ATOMIC) {
11058 /* BPF_STX + BPF_ATOMIC insns without BPF_FETCH do not
11059 * define any registers, therefore zext_dst cannot be
11060 * set.
11061 */
11062 if (WARN_ON(!(insn.imm & BPF_FETCH)))
11063 return -EINVAL;
11064 load_reg = insn.imm == BPF_CMPXCHG ? BPF_REG_0
11065 : insn.src_reg;
11066 } else {
11067 load_reg = insn.dst_reg;
11068 }
11069
10626 zext_patch[0] = insn;11070 zext_patch[0] = insn;
10627 zext_patch[1].dst_reg = insn.dst_reg;11071 zext_patch[1].dst_reg = load_reg;
10628 zext_patch[1].src_reg = insn.dst_reg;11072 zext_patch[1].src_reg = load_reg;
10629 patch = zext_patch;11073 patch = zext_patch;
10630 patch_len = 2;11074 patch_len = 2;
10631apply_patch_buffer:11075apply_patch_buffer:
@@ -10841,8 +11285,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 209 unmodified lines
10841 return 0;11285 return 0;
10842 11286
10843 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {11287 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
10844 if (insn->code != (BPF_JMP | BPF_CALL) ||11288 if (!bpf_pseudo_call(insn))
10845 insn->src_reg != BPF_PSEUDO_CALL)
10846 continue;11289 continue;
10847 /* Upon error here we cannot fall back to interpreter but11290 /* Upon error here we cannot fall back to interpreter but
10848 * need a hard reject of the program. Thus -EFAULT is11291 * need a hard reject of the program. Thus -EFAULT is
@@ -10883,7 +11326,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 34 unmodified lines
10883 /* BPF_PROG_RUN doesn't call subprogs directly,11326 /* BPF_PROG_RUN doesn't call subprogs directly,
10884 * hence main prog stats include the runtime of subprogs.11327 * hence main prog stats include the runtime of subprogs.
10885 * subprogs don't have IDs and not reachable via prog_get_next_id11328 * subprogs don't have IDs and not reachable via prog_get_next_id
10886 * func[i]->aux->stats will never be accessed and stays NULL11329 * func[i]->stats will never be accessed and stays NULL
10887 */11330 */
10888 func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER);11331 func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER);
10889 if (!func[i])11332 if (!func[i])
@@ -10971,8 +11414,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 81 unmodified lines
10971 for (i = 0; i < env->subprog_cnt; i++) {11414 for (i = 0; i < env->subprog_cnt; i++) {
10972 insn = func[i]->insnsi;11415 insn = func[i]->insnsi;
10973 for (j = 0; j < func[i]->len; j++, insn++) {11416 for (j = 0; j < func[i]->len; j++, insn++) {
10974 if (insn->code != (BPF_JMP | BPF_CALL) ||11417 if (!bpf_pseudo_call(insn))
10975 insn->src_reg != BPF_PSEUDO_CALL)
10976 continue;11418 continue;
10977 subprog = insn->off;11419 subprog = insn->off;
10978 insn->imm = BPF_CAST_CALL(func[subprog]->bpf_func) -11420 insn->imm = BPF_CAST_CALL(func[subprog]->bpf_func) -
@@ -11017,8 +11459,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 38 unmodified lines
11017 * later look the same as if they were interpreted only.11459 * later look the same as if they were interpreted only.
11018 */11460 */
11019 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {11461 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
11020 if (insn->code != (BPF_JMP | BPF_CALL) ||11462 if (!bpf_pseudo_call(insn))
11021 insn->src_reg != BPF_PSEUDO_CALL)
11022 continue;11463 continue;
11023 insn->off = env->insn_aux_data[i].call_imm;11464 insn->off = env->insn_aux_data[i].call_imm;
11024 subprog = find_subprog(env, i + insn->off + 1);11465 subprog = find_subprog(env, i + insn->off + 1);
@@ -11047,8 +11488,7 @@ out_undo_insn: · 22 unmodified lines
11047 /* cleanup main prog to be interpreted */11488 /* cleanup main prog to be interpreted */
11048 prog->jit_requested = 0;11489 prog->jit_requested = 0;
11049 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {11490 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
11050 if (insn->code != (BPF_JMP | BPF_CALL) ||11491 if (!bpf_pseudo_call(insn))
11051 insn->src_reg != BPF_PSEUDO_CALL)
11052 continue;11492 continue;
11053 insn->off = 0;11493 insn->off = 0;
11054 insn->imm = env->insn_aux_data[i].call_imm;11494 insn->imm = env->insn_aux_data[i].call_imm;
@@ -11083,8 +11523,7 @@ static int fixup_call_args(struct bpf_verifier_env *env) · 28 unmodified lines
11083 return -EINVAL;11523 return -EINVAL;
11084 }11524 }
11085 for (i = 0; i < prog->len; i++, insn++) {11525 for (i = 0; i < prog->len; i++, insn++) {
11086 if (insn->code != (BPF_JMP | BPF_CALL) ||11526 if (!bpf_pseudo_call(insn))
11087 insn->src_reg != BPF_PSEUDO_CALL)
11088 continue;11527 continue;
11089 depth = get_callee_stack_depth(env, insn, i);11528 depth = get_callee_stack_depth(env, insn, i);
11090 if (depth < 0)11529 if (depth < 0)
@@ -11547,6 +11986,13 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) · 456 unmodified lines
11547 mark_reg_known_zero(env, regs, i);11986 mark_reg_known_zero(env, regs, i);
11548 else if (regs[i].type == SCALAR_VALUE)11987 else if (regs[i].type == SCALAR_VALUE)
11549 mark_reg_unknown(env, regs, i);11988 mark_reg_unknown(env, regs, i);
11989 else if (regs[i].type == PTR_TO_MEM_OR_NULL) {
11990 const u32 mem_size = regs[i].mem_size;
11991
11992 mark_reg_known_zero(env, regs, i);
11993 regs[i].mem_size = mem_size;
11994 regs[i].id = ++env->id_gen;
11995 }
11550 }11996 }
11551 } else {11997 } else {
11552 /* 1st arg to a function */11998 /* 1st arg to a function */
@@ -12125,6 +12571,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, · 572 unmodified lines
12125 env->strict_alignment = false;12571 env->strict_alignment = false;
12126 12572
12127 env->allow_ptr_leaks = bpf_allow_ptr_leaks();12573 env->allow_ptr_leaks = bpf_allow_ptr_leaks();
12574 env->allow_uninit_stack = bpf_allow_uninit_stack();
12128 env->allow_ptr_to_map_access = bpf_allow_ptr_to_map_access();12575 env->allow_ptr_to_map_access = bpf_allow_ptr_to_map_access();
12129 env->bypass_spec_v1 = bpf_bypass_spec_v1();12576 env->bypass_spec_v1 = bpf_bypass_spec_v1();
12130 env->bypass_spec_v4 = bpf_bypass_spec_v4();12577 env->bypass_spec_v4 = bpf_bypass_spec_v4();
kernel/trace/bpf_trace.c+6−0
@@ -1188,6 +1188,10 @@ BTF_SET_END(btf_allowlist_d_path) · 1187 unmodified lines
1188 1188
1189static bool bpf_d_path_allowed(const struct bpf_prog *prog)1189static bool bpf_d_path_allowed(const struct bpf_prog *prog)
1190{1190{
1191 if (prog->type == BPF_PROG_TYPE_TRACING &&
1192 prog->expected_attach_type == BPF_TRACE_ITER)
1193 return true;
1194
1191 if (prog->type == BPF_PROG_TYPE_LSM)1195 if (prog->type == BPF_PROG_TYPE_LSM)
1192 return bpf_lsm_is_sleepable_hook(prog->aux->attach_btf_id);1196 return bpf_lsm_is_sleepable_hook(prog->aux->attach_btf_id);
1193 1197
@@ -1757,6 +1761,8 @@ tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 563 unmodified lines
1757 return &bpf_sk_storage_delete_tracing_proto;1761 return &bpf_sk_storage_delete_tracing_proto;
1758 case BPF_FUNC_sock_from_file:1762 case BPF_FUNC_sock_from_file:
1759 return &bpf_sock_from_file_proto;1763 return &bpf_sock_from_file_proto;
1764 case BPF_FUNC_get_socket_cookie:
1765 return &bpf_get_socket_ptr_cookie_proto;
1760#endif1766#endif
1761 case BPF_FUNC_seq_printf:1767 case BPF_FUNC_seq_printf:
1762 return prog->expected_attach_type == BPF_TRACE_ITER ?1768 return prog->expected_attach_type == BPF_TRACE_ITER ?
lib/test_bpf.c+1−6
@@ -345,7 +345,7 @@ static int __bpf_fill_ja(struct bpf_test *self, unsigned int len, · 344 unmodified lines
345 345
346static int bpf_fill_maxinsns11(struct bpf_test *self)346static int bpf_fill_maxinsns11(struct bpf_test *self)
347{347{
348 /* Hits 70 passes on x86_64, so cannot get JITed there. */348 /* Hits 70 passes on x86_64 and triggers NOPs padding. */
349 return __bpf_fill_ja(self, BPF_MAXINSNS, 68);349 return __bpf_fill_ja(self, BPF_MAXINSNS, 68);
350}350}
351 351
@@ -5318,15 +5318,10 @@ static struct bpf_test tests[] = { · 4966 unmodified lines
5318 {5318 {
5319 "BPF_MAXINSNS: Jump, gap, jump, ...",5319 "BPF_MAXINSNS: Jump, gap, jump, ...",
5320 { },5320 { },
5321#if defined(CONFIG_BPF_JIT_ALWAYS_ON) && defined(CONFIG_X86)
5322 CLASSIC | FLAG_NO_DATA | FLAG_EXPECTED_FAIL,
5323#else
5324 CLASSIC | FLAG_NO_DATA,5321 CLASSIC | FLAG_NO_DATA,
5325#endif
5326 { },5322 { },
5327 { { 0, 0xababcbac } },5323 { { 0, 0xababcbac } },
5328 .fill_helper = bpf_fill_maxinsns11,5324 .fill_helper = bpf_fill_maxinsns11,
5329 .expected_errcode = -ENOTSUPP,
5330 },5325 },
5331 {5326 {
5332 "BPF_MAXINSNS: jump over MSH",5327 "BPF_MAXINSNS: jump over MSH",
net/core/dev.c+14−18
@@ -2217,28 +2217,14 @@ static inline void net_timestamp_set(struct sk_buff *skb) · 2216 unmodified lines
2217 2217
2218bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)2218bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)
2219{2219{
2220 unsigned int len;2220 return __is_skb_forwardable(dev, skb, true);
2221
2222 if (!(dev->flags & IFF_UP))
2223 return false;
2224
2225 len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
2226 if (skb->len <= len)
2227 return true;
2228
2229 /* if TSO is enabled, we don't care about the length as the packet
2230 * could be forwarded without being segmented before
2231 */
2232 if (skb_is_gso(skb))
2233 return true;
2234
2235 return false;
2236}2221}
2237EXPORT_SYMBOL_GPL(is_skb_forwardable);2222EXPORT_SYMBOL_GPL(is_skb_forwardable);
2238 2223
2239int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)2224static int __dev_forward_skb2(struct net_device *dev, struct sk_buff *skb,
2225 bool check_mtu)
2240{2226{
2241 int ret = ____dev_forward_skb(dev, skb);2227 int ret = ____dev_forward_skb(dev, skb, check_mtu);
2242 2228
2243 if (likely(!ret)) {2229 if (likely(!ret)) {
2244 skb->protocol = eth_type_trans(skb, dev);2230 skb->protocol = eth_type_trans(skb, dev);
@@ -2247,6 +2233,11 @@ int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb) · 2 unmodified lines
2247 2233
2248 return ret;2234 return ret;
2249}2235}
2236
2237int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
2238{
2239 return __dev_forward_skb2(dev, skb, true);
2240}
2250EXPORT_SYMBOL_GPL(__dev_forward_skb);2241EXPORT_SYMBOL_GPL(__dev_forward_skb);
2251 2242
2252/**2243/**
@@ -2273,6 +2264,11 @@ int dev_forward_skb(struct net_device *dev, struct sk_buff *skb) · 20 unmodified lines
2273}2264}
2274EXPORT_SYMBOL_GPL(dev_forward_skb);2265EXPORT_SYMBOL_GPL(dev_forward_skb);
2275 2266
2267int dev_forward_skb_nomtu(struct net_device *dev, struct sk_buff *skb)
2268{
2269 return __dev_forward_skb2(dev, skb, false) ?: netif_rx_internal(skb);
2270}
2271
2276static inline int deliver_skb(struct sk_buff *skb,2272static inline int deliver_skb(struct sk_buff *skb,
2277 struct packet_type *pt_prev,2273 struct packet_type *pt_prev,
2278 struct net_device *orig_dev)2274 struct net_device *orig_dev)
net/core/filter.c+174−21
@@ -2083,13 +2083,13 @@ static const struct bpf_func_proto bpf_csum_level_proto = { · 2082 unmodified lines
2083 2083
2084static inline int __bpf_rx_skb(struct net_device *dev, struct sk_buff *skb)2084static inline int __bpf_rx_skb(struct net_device *dev, struct sk_buff *skb)
2085{2085{
2086 return dev_forward_skb(dev, skb);2086 return dev_forward_skb_nomtu(dev, skb);
2087}2087}
2088 2088
2089static inline int __bpf_rx_skb_no_mac(struct net_device *dev,2089static inline int __bpf_rx_skb_no_mac(struct net_device *dev,
2090 struct sk_buff *skb)2090 struct sk_buff *skb)
2091{2091{
2092 int ret = ____dev_forward_skb(dev, skb);2092 int ret = ____dev_forward_skb(dev, skb, false);
2093 2093
2094 if (likely(!ret)) {2094 if (likely(!ret)) {
2095 skb->dev = dev;2095 skb->dev = dev;
@@ -2480,7 +2480,7 @@ int skb_do_redirect(struct sk_buff *skb) · 384 unmodified lines
2480 goto out_drop;2480 goto out_drop;
2481 dev = ops->ndo_get_peer_dev(dev);2481 dev = ops->ndo_get_peer_dev(dev);
2482 if (unlikely(!dev ||2482 if (unlikely(!dev ||
2483 !is_skb_forwardable(dev, skb) ||2483 !(dev->flags & IFF_UP) ||
2484 net_eq(net, dev_net(dev))))2484 net_eq(net, dev_net(dev))))
2485 goto out_drop;2485 goto out_drop;
2486 skb->dev = dev;2486 skb->dev = dev;
@@ -3552,11 +3552,7 @@ static int bpf_skb_net_shrink(struct sk_buff *skb, u32 off, u32 len_diff, · 1065 unmodified lines
3552 return 0;3552 return 0;
3553}3553}
3554 3554
3555static u32 __bpf_skb_max_len(const struct sk_buff *skb)3555#define BPF_SKB_MAX_LEN SKB_MAX_ALLOC
3556{
3557 return skb->dev ? skb->dev->mtu + skb->dev->hard_header_len :
3558 SKB_MAX_ALLOC;
3559}
3560 3556
3561BPF_CALL_4(sk_skb_adjust_room, struct sk_buff *, skb, s32, len_diff,3557BPF_CALL_4(sk_skb_adjust_room, struct sk_buff *, skb, s32, len_diff,
3562 u32, mode, u64, flags)3558 u32, mode, u64, flags)
@@ -3605,7 +3601,7 @@ BPF_CALL_4(bpf_skb_adjust_room, struct sk_buff *, skb, s32, len_diff, · 42 unmodified lines
3605{3601{
3606 u32 len_cur, len_diff_abs = abs(len_diff);3602 u32 len_cur, len_diff_abs = abs(len_diff);
3607 u32 len_min = bpf_skb_net_base_len(skb);3603 u32 len_min = bpf_skb_net_base_len(skb);
3608 u32 len_max = __bpf_skb_max_len(skb);3604 u32 len_max = BPF_SKB_MAX_LEN;
3609 __be16 proto = skb->protocol;3605 __be16 proto = skb->protocol;
3610 bool shrink = len_diff < 0;3606 bool shrink = len_diff < 0;
3611 u32 off;3607 u32 off;
@@ -3688,7 +3684,7 @@ static int bpf_skb_trim_rcsum(struct sk_buff *skb, unsigned int new_len) · 76 unmodified lines
3688static inline int __bpf_skb_change_tail(struct sk_buff *skb, u32 new_len,3684static inline int __bpf_skb_change_tail(struct sk_buff *skb, u32 new_len,
3689 u64 flags)3685 u64 flags)
3690{3686{
3691 u32 max_len = __bpf_skb_max_len(skb);3687 u32 max_len = BPF_SKB_MAX_LEN;
3692 u32 min_len = __bpf_skb_min_len(skb);3688 u32 min_len = __bpf_skb_min_len(skb);
3693 int ret;3689 int ret;
3694 3690
@@ -3764,7 +3760,7 @@ static const struct bpf_func_proto sk_skb_change_tail_proto = { · 69 unmodified lines
3764static inline int __bpf_skb_change_head(struct sk_buff *skb, u32 head_room,3760static inline int __bpf_skb_change_head(struct sk_buff *skb, u32 head_room,
3765 u64 flags)3761 u64 flags)
3766{3762{
3767 u32 max_len = __bpf_skb_max_len(skb);3763 u32 max_len = BPF_SKB_MAX_LEN;
3768 u32 new_len = skb->len + head_room;3764 u32 new_len = skb->len + head_room;
3769 int ret;3765 int ret;
3770 3766
@@ -4631,6 +4627,18 @@ static const struct bpf_func_proto bpf_get_socket_cookie_sock_proto = { · 860 unmodified lines
4631 .arg1_type = ARG_PTR_TO_CTX,4627 .arg1_type = ARG_PTR_TO_CTX,
4632};4628};
4633 4629
4630BPF_CALL_1(bpf_get_socket_ptr_cookie, struct sock *, sk)
4631{
4632 return sk ? sock_gen_cookie(sk) : 0;
4633}
4634
4635const struct bpf_func_proto bpf_get_socket_ptr_cookie_proto = {
4636 .func = bpf_get_socket_ptr_cookie,
4637 .gpl_only = false,
4638 .ret_type = RET_INTEGER,
4639 .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
4640};
4641
4634BPF_CALL_1(bpf_get_socket_cookie_sock_ops, struct bpf_sock_ops_kern *, ctx)4642BPF_CALL_1(bpf_get_socket_cookie_sock_ops, struct bpf_sock_ops_kern *, ctx)
4635{4643{
4636 return __sock_gen_cookie(ctx->sk);4644 return __sock_gen_cookie(ctx->sk);
@@ -5291,12 +5299,14 @@ static const struct bpf_func_proto bpf_skb_get_xfrm_state_proto = { · 654 unmodified lines
5291#if IS_ENABLED(CONFIG_INET) || IS_ENABLED(CONFIG_IPV6)5299#if IS_ENABLED(CONFIG_INET) || IS_ENABLED(CONFIG_IPV6)
5292static int bpf_fib_set_fwd_params(struct bpf_fib_lookup *params,5300static int bpf_fib_set_fwd_params(struct bpf_fib_lookup *params,
5293 const struct neighbour *neigh,5301 const struct neighbour *neigh,
5294 const struct net_device *dev)5302 const struct net_device *dev, u32 mtu)
5295{5303{
5296 memcpy(params->dmac, neigh->ha, ETH_ALEN);5304 memcpy(params->dmac, neigh->ha, ETH_ALEN);
5297 memcpy(params->smac, dev->dev_addr, ETH_ALEN);5305 memcpy(params->smac, dev->dev_addr, ETH_ALEN);
5298 params->h_vlan_TCI = 0;5306 params->h_vlan_TCI = 0;
5299 params->h_vlan_proto = 0;5307 params->h_vlan_proto = 0;
5308 if (mtu)
5309 params->mtu_result = mtu; /* union with tot_len */
5300 5310
5301 return 0;5311 return 0;
5302}5312}
@@ -5312,8 +5322,8 @@ static int bpf_ipv4_fib_lookup(struct net *net, struct bpf_fib_lookup *params, · 9 unmodified lines
5312 struct net_device *dev;5322 struct net_device *dev;
5313 struct fib_result res;5323 struct fib_result res;
5314 struct flowi4 fl4;5324 struct flowi4 fl4;
5325 u32 mtu = 0;
5315 int err;5326 int err;
5316 u32 mtu;
5317 5327
5318 dev = dev_get_by_index_rcu(net, params->ifindex);5328 dev = dev_get_by_index_rcu(net, params->ifindex);
5319 if (unlikely(!dev))5329 if (unlikely(!dev))
@@ -5380,8 +5390,10 @@ static int bpf_ipv4_fib_lookup(struct net *net, struct bpf_fib_lookup *params, · 60 unmodified lines
5380 5390
5381 if (check_mtu) {5391 if (check_mtu) {
5382 mtu = ip_mtu_from_fib_result(&res, params->ipv4_dst);5392 mtu = ip_mtu_from_fib_result(&res, params->ipv4_dst);
5383 if (params->tot_len > mtu)5393 if (params->tot_len > mtu) {
5394 params->mtu_result = mtu; /* union with tot_len */
5384 return BPF_FIB_LKUP_RET_FRAG_NEEDED;5395 return BPF_FIB_LKUP_RET_FRAG_NEEDED;
5396 }
5385 }5397 }
5386 5398
5387 nhc = res.nhc;5399 nhc = res.nhc;
@@ -5415,7 +5427,7 @@ static int bpf_ipv4_fib_lookup(struct net *net, struct bpf_fib_lookup *params, · 27 unmodified lines
5415 if (!neigh)5427 if (!neigh)
5416 return BPF_FIB_LKUP_RET_NO_NEIGH;5428 return BPF_FIB_LKUP_RET_NO_NEIGH;
5417 5429
5418 return bpf_fib_set_fwd_params(params, neigh, dev);5430 return bpf_fib_set_fwd_params(params, neigh, dev, mtu);
5419}5431}
5420#endif5432#endif
5421 5433
@@ -5432,7 +5444,7 @@ static int bpf_ipv6_fib_lookup(struct net *net, struct bpf_fib_lookup *params, · 10 unmodified lines
5432 struct flowi6 fl6;5444 struct flowi6 fl6;
5433 int strict = 0;5445 int strict = 0;
5434 int oif, err;5446 int oif, err;
5435 u32 mtu;5447 u32 mtu = 0;
5436 5448
5437 /* link local addresses are never forwarded */5449 /* link local addresses are never forwarded */
5438 if (rt6_need_strict(dst) || rt6_need_strict(src))5450 if (rt6_need_strict(dst) || rt6_need_strict(src))
@@ -5507,8 +5519,10 @@ static int bpf_ipv6_fib_lookup(struct net *net, struct bpf_fib_lookup *params, · 68 unmodified lines
5507 5519
5508 if (check_mtu) {5520 if (check_mtu) {
5509 mtu = ipv6_stub->ip6_mtu_from_fib6(&res, dst, src);5521 mtu = ipv6_stub->ip6_mtu_from_fib6(&res, dst, src);
5510 if (params->tot_len > mtu)5522 if (params->tot_len > mtu) {
5523 params->mtu_result = mtu; /* union with tot_len */
5511 return BPF_FIB_LKUP_RET_FRAG_NEEDED;5524 return BPF_FIB_LKUP_RET_FRAG_NEEDED;
5525 }
5512 }5526 }
5513 5527
5514 if (res.nh->fib_nh_lws)5528 if (res.nh->fib_nh_lws)
@@ -5528,7 +5542,7 @@ static int bpf_ipv6_fib_lookup(struct net *net, struct bpf_fib_lookup *params, · 13 unmodified lines
5528 if (!neigh)5542 if (!neigh)
5529 return BPF_FIB_LKUP_RET_NO_NEIGH;5543 return BPF_FIB_LKUP_RET_NO_NEIGH;
5530 5544
5531 return bpf_fib_set_fwd_params(params, neigh, dev);5545 return bpf_fib_set_fwd_params(params, neigh, dev, mtu);
5532}5546}
5533#endif5547#endif
5534 5548
@@ -5571,6 +5585,7 @@ BPF_CALL_4(bpf_skb_fib_lookup, struct sk_buff *, skb, · 36 unmodified lines
5571{5585{
5572 struct net *net = dev_net(skb->dev);5586 struct net *net = dev_net(skb->dev);
5573 int rc = -EAFNOSUPPORT;5587 int rc = -EAFNOSUPPORT;
5588 bool check_mtu = false;
5574 5589
5575 if (plen < sizeof(*params))5590 if (plen < sizeof(*params))
5576 return -EINVAL;5591 return -EINVAL;
@@ -5578,25 +5593,33 @@ BPF_CALL_4(bpf_skb_fib_lookup, struct sk_buff *, skb, · 1 unmodified line
5578 if (flags & ~(BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_OUTPUT))5593 if (flags & ~(BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_OUTPUT))
5579 return -EINVAL;5594 return -EINVAL;
5580 5595
5596 if (params->tot_len)
5597 check_mtu = true;
5598
5581 switch (params->family) {5599 switch (params->family) {
5582#if IS_ENABLED(CONFIG_INET)5600#if IS_ENABLED(CONFIG_INET)
5583 case AF_INET:5601 case AF_INET:
5584 rc = bpf_ipv4_fib_lookup(net, params, flags, false);5602 rc = bpf_ipv4_fib_lookup(net, params, flags, check_mtu);
5585 break;5603 break;
5586#endif5604#endif
5587#if IS_ENABLED(CONFIG_IPV6)5605#if IS_ENABLED(CONFIG_IPV6)
5588 case AF_INET6:5606 case AF_INET6:
5589 rc = bpf_ipv6_fib_lookup(net, params, flags, false);5607 rc = bpf_ipv6_fib_lookup(net, params, flags, check_mtu);
5590 break;5608 break;
5591#endif5609#endif
5592 }5610 }
5593 5611
5594 if (!rc) {5612 if (rc == BPF_FIB_LKUP_RET_SUCCESS && !check_mtu) {
5595 struct net_device *dev;5613 struct net_device *dev;
5596 5614
5615 /* When tot_len isn't provided by user, check skb
5616 * against MTU of FIB lookup resulting net_device
5617 */
5597 dev = dev_get_by_index_rcu(net, params->ifindex);5618 dev = dev_get_by_index_rcu(net, params->ifindex);
5598 if (!is_skb_forwardable(dev, skb))5619 if (!is_skb_forwardable(dev, skb))
5599 rc = BPF_FIB_LKUP_RET_FRAG_NEEDED;5620 rc = BPF_FIB_LKUP_RET_FRAG_NEEDED;
5621
5622 params->mtu_result = dev->mtu; /* union with tot_len */
5600 }5623 }
5601 5624
5602 return rc;5625 return rc;
@@ -5612,6 +5635,116 @@ static const struct bpf_func_proto bpf_skb_fib_lookup_proto = { · 9 unmodified lines
5612 .arg4_type = ARG_ANYTHING,5635 .arg4_type = ARG_ANYTHING,
5613};5636};
5614 5637
5638static struct net_device *__dev_via_ifindex(struct net_device *dev_curr,
5639 u32 ifindex)
5640{
5641 struct net *netns = dev_net(dev_curr);
5642
5643 /* Non-redirect use-cases can use ifindex=0 and save ifindex lookup */
5644 if (ifindex == 0)
5645 return dev_curr;
5646
5647 return dev_get_by_index_rcu(netns, ifindex);
5648}
5649
5650BPF_CALL_5(bpf_skb_check_mtu, struct sk_buff *, skb,
5651 u32, ifindex, u32 *, mtu_len, s32, len_diff, u64, flags)
5652{
5653 int ret = BPF_MTU_CHK_RET_FRAG_NEEDED;
5654 struct net_device *dev = skb->dev;
5655 int skb_len, dev_len;
5656 int mtu;
5657
5658 if (unlikely(flags & ~(BPF_MTU_CHK_SEGS)))
5659 return -EINVAL;
5660
5661 if (unlikely(flags & BPF_MTU_CHK_SEGS && len_diff))
5662 return -EINVAL;
5663
5664 dev = __dev_via_ifindex(dev, ifindex);
5665 if (unlikely(!dev))
5666 return -ENODEV;
5667
5668 mtu = READ_ONCE(dev->mtu);
5669
5670 dev_len = mtu + dev->hard_header_len;
5671 skb_len = skb->len + len_diff; /* minus result pass check */
5672 if (skb_len <= dev_len) {
5673 ret = BPF_MTU_CHK_RET_SUCCESS;
5674 goto out;
5675 }
5676 /* At this point, skb->len exceed MTU, but as it include length of all
5677 * segments, it can still be below MTU. The SKB can possibly get
5678 * re-segmented in transmit path (see validate_xmit_skb). Thus, user
5679 * must choose if segs are to be MTU checked.
5680 */
5681 if (skb_is_gso(skb)) {
5682 ret = BPF_MTU_CHK_RET_SUCCESS;
5683
5684 if (flags & BPF_MTU_CHK_SEGS &&
5685 !skb_gso_validate_network_len(skb, mtu))
5686 ret = BPF_MTU_CHK_RET_SEGS_TOOBIG;
5687 }
5688out:
5689 /* BPF verifier guarantees valid pointer */
5690 *mtu_len = mtu;
5691
5692 return ret;
5693}
5694
5695BPF_CALL_5(bpf_xdp_check_mtu, struct xdp_buff *, xdp,
5696 u32, ifindex, u32 *, mtu_len, s32, len_diff, u64, flags)
5697{
5698 struct net_device *dev = xdp->rxq->dev;
5699 int xdp_len = xdp->data_end - xdp->data;
5700 int ret = BPF_MTU_CHK_RET_SUCCESS;
5701 int mtu, dev_len;
5702
5703 /* XDP variant doesn't support multi-buffer segment check (yet) */
5704 if (unlikely(flags))
5705 return -EINVAL;
5706
5707 dev = __dev_via_ifindex(dev, ifindex);
5708 if (unlikely(!dev))
5709 return -ENODEV;
5710
5711 mtu = READ_ONCE(dev->mtu);
5712
5713 /* Add L2-header as dev MTU is L3 size */
5714 dev_len = mtu + dev->hard_header_len;
5715
5716 xdp_len += len_diff; /* minus result pass check */
5717 if (xdp_len > dev_len)
5718 ret = BPF_MTU_CHK_RET_FRAG_NEEDED;
5719
5720 /* BPF verifier guarantees valid pointer */
5721 *mtu_len = mtu;
5722
5723 return ret;
5724}
5725
5726static const struct bpf_func_proto bpf_skb_check_mtu_proto = {
5727 .func = bpf_skb_check_mtu,
5728 .gpl_only = true,
5729 .ret_type = RET_INTEGER,
5730 .arg1_type = ARG_PTR_TO_CTX,
5731 .arg2_type = ARG_ANYTHING,
5732 .arg3_type = ARG_PTR_TO_INT,
5733 .arg4_type = ARG_ANYTHING,
5734 .arg5_type = ARG_ANYTHING,
5735};
5736
5737static const struct bpf_func_proto bpf_xdp_check_mtu_proto = {
5738 .func = bpf_xdp_check_mtu,
5739 .gpl_only = true,
5740 .ret_type = RET_INTEGER,
5741 .arg1_type = ARG_PTR_TO_CTX,
5742 .arg2_type = ARG_ANYTHING,
5743 .arg3_type = ARG_PTR_TO_INT,
5744 .arg4_type = ARG_ANYTHING,
5745 .arg5_type = ARG_ANYTHING,
5746};
5747
5615#if IS_ENABLED(CONFIG_IPV6_SEG6_BPF)5748#if IS_ENABLED(CONFIG_IPV6_SEG6_BPF)
5616static int bpf_push_seg6_encap(struct sk_buff *skb, u32 type, void *hdr, u32 len)5749static int bpf_push_seg6_encap(struct sk_buff *skb, u32 type, void *hdr, u32 len)
5617{5750{
@@ -7021,6 +7154,14 @@ sock_addr_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 1403 unmodified lines
7021 case BPF_CGROUP_INET6_BIND:7154 case BPF_CGROUP_INET6_BIND:
7022 case BPF_CGROUP_INET4_CONNECT:7155 case BPF_CGROUP_INET4_CONNECT:
7023 case BPF_CGROUP_INET6_CONNECT:7156 case BPF_CGROUP_INET6_CONNECT:
7157 case BPF_CGROUP_UDP4_RECVMSG:
7158 case BPF_CGROUP_UDP6_RECVMSG:
7159 case BPF_CGROUP_UDP4_SENDMSG:
7160 case BPF_CGROUP_UDP6_SENDMSG:
7161 case BPF_CGROUP_INET4_GETPEERNAME:
7162 case BPF_CGROUP_INET6_GETPEERNAME:
7163 case BPF_CGROUP_INET4_GETSOCKNAME:
7164 case BPF_CGROUP_INET6_GETSOCKNAME:
7024 return &bpf_sock_addr_setsockopt_proto;7165 return &bpf_sock_addr_setsockopt_proto;
7025 default:7166 default:
7026 return NULL;7167 return NULL;
@@ -7031,6 +7172,14 @@ sock_addr_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 4 unmodified lines
7031 case BPF_CGROUP_INET6_BIND:7172 case BPF_CGROUP_INET6_BIND:
7032 case BPF_CGROUP_INET4_CONNECT:7173 case BPF_CGROUP_INET4_CONNECT:
7033 case BPF_CGROUP_INET6_CONNECT:7174 case BPF_CGROUP_INET6_CONNECT:
7175 case BPF_CGROUP_UDP4_RECVMSG:
7176 case BPF_CGROUP_UDP6_RECVMSG:
7177 case BPF_CGROUP_UDP4_SENDMSG:
7178 case BPF_CGROUP_UDP6_SENDMSG:
7179 case BPF_CGROUP_INET4_GETPEERNAME:
7180 case BPF_CGROUP_INET6_GETPEERNAME:
7181 case BPF_CGROUP_INET4_GETSOCKNAME:
7182 case BPF_CGROUP_INET6_GETSOCKNAME:
7034 return &bpf_sock_addr_getsockopt_proto;7183 return &bpf_sock_addr_getsockopt_proto;
7035 default:7184 default:
7036 return NULL;7185 return NULL;
@@ -7181,6 +7330,8 @@ tc_cls_act_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 144 unmodified lines
7181 return &bpf_get_socket_uid_proto;7330 return &bpf_get_socket_uid_proto;
7182 case BPF_FUNC_fib_lookup:7331 case BPF_FUNC_fib_lookup:
7183 return &bpf_skb_fib_lookup_proto;7332 return &bpf_skb_fib_lookup_proto;
7333 case BPF_FUNC_check_mtu:
7334 return &bpf_skb_check_mtu_proto;
7184 case BPF_FUNC_sk_fullsock:7335 case BPF_FUNC_sk_fullsock:
7185 return &bpf_sk_fullsock_proto;7336 return &bpf_sk_fullsock_proto;
7186 case BPF_FUNC_sk_storage_get:7337 case BPF_FUNC_sk_storage_get:
@@ -7250,6 +7401,8 @@ xdp_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 63 unmodified lines
7250 return &bpf_xdp_adjust_tail_proto;7401 return &bpf_xdp_adjust_tail_proto;
7251 case BPF_FUNC_fib_lookup:7402 case BPF_FUNC_fib_lookup:
7252 return &bpf_xdp_fib_lookup_proto;7403 return &bpf_xdp_fib_lookup_proto;
7404 case BPF_FUNC_check_mtu:
7405 return &bpf_xdp_check_mtu_proto;
7253#ifdef CONFIG_INET7406#ifdef CONFIG_INET
7254 case BPF_FUNC_sk_lookup_udp:7407 case BPF_FUNC_sk_lookup_udp:
7255 return &bpf_xdp_sk_lookup_udp_proto;7408 return &bpf_xdp_sk_lookup_udp_proto;
net/core/skmsg.c+1−2
@@ -669,14 +669,13 @@ static void sk_psock_destroy_deferred(struct work_struct *gc) · 668 unmodified lines
669 kfree(psock);669 kfree(psock);
670}670}
671 671
672void sk_psock_destroy(struct rcu_head *rcu)672static void sk_psock_destroy(struct rcu_head *rcu)
673{673{
674 struct sk_psock *psock = container_of(rcu, struct sk_psock, rcu);674 struct sk_psock *psock = container_of(rcu, struct sk_psock, rcu);
675 675
676 INIT_WORK(&psock->gc, sk_psock_destroy_deferred);676 INIT_WORK(&psock->gc, sk_psock_destroy_deferred);
677 schedule_work(&psock->gc);677 schedule_work(&psock->gc);
678}678}
679EXPORT_SYMBOL_GPL(sk_psock_destroy);
680 679
681void sk_psock_drop(struct sock *sk, struct sk_psock *psock)680void sk_psock_drop(struct sock *sk, struct sk_psock *psock)
682{681{
net/core/xdp.c+70−0
@@ -513,3 +513,73 @@ void xdp_warn(const char *msg, const char *func, const int line) · 512 unmodified lines
513 WARN(1, "XDP_WARN: %s(line:%d): %s\n", func, line, msg);513 WARN(1, "XDP_WARN: %s(line:%d): %s\n", func, line, msg);
514};514};
515EXPORT_SYMBOL_GPL(xdp_warn);515EXPORT_SYMBOL_GPL(xdp_warn);
516
517int xdp_alloc_skb_bulk(void **skbs, int n_skb, gfp_t gfp)
518{
519 n_skb = kmem_cache_alloc_bulk(skbuff_head_cache, gfp,
520 n_skb, skbs);
521 if (unlikely(!n_skb))
522 return -ENOMEM;
523
524 return 0;
525}
526EXPORT_SYMBOL_GPL(xdp_alloc_skb_bulk);
527
528struct sk_buff *__xdp_build_skb_from_frame(struct xdp_frame *xdpf,
529 struct sk_buff *skb,
530 struct net_device *dev)
531{
532 unsigned int headroom, frame_size;
533 void *hard_start;
534
535 /* Part of headroom was reserved to xdpf */
536 headroom = sizeof(*xdpf) + xdpf->headroom;
537
538 /* Memory size backing xdp_frame data already have reserved
539 * room for build_skb to place skb_shared_info in tailroom.
540 */
541 frame_size = xdpf->frame_sz;
542
543 hard_start = xdpf->data - headroom;
544 skb = build_skb_around(skb, hard_start, frame_size);
545 if (unlikely(!skb))
546 return NULL;
547
548 skb_reserve(skb, headroom);
549 __skb_put(skb, xdpf->len);
550 if (xdpf->metasize)
551 skb_metadata_set(skb, xdpf->metasize);
552
553 /* Essential SKB info: protocol and skb->dev */
554 skb->protocol = eth_type_trans(skb, dev);
555
556 /* Optional SKB info, currently missing:
557 * - HW checksum info (skb->ip_summed)
558 * - HW RX hash (skb_set_hash)
559 * - RX ring dev queue index (skb_record_rx_queue)
560 */
561
562 /* Until page_pool get SKB return path, release DMA here */
563 xdp_release_frame(xdpf);
564
565 /* Allow SKB to reuse area used by xdp_frame */
566 xdp_scrub_frame(xdpf);
567
568 return skb;
569}
570EXPORT_SYMBOL_GPL(__xdp_build_skb_from_frame);
571
572struct sk_buff *xdp_build_skb_from_frame(struct xdp_frame *xdpf,
573 struct net_device *dev)
574{
575 struct sk_buff *skb;
576
577 skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC);
578 if (unlikely(!skb))
579 return NULL;
580
581 memset(skb, 0, offsetof(struct sk_buff, tail));
582
583 return __xdp_build_skb_from_frame(xdpf, skb, dev);
584}
585EXPORT_SYMBOL_GPL(xdp_build_skb_from_frame);
net/ipv4/af_inet.c+11−7
@@ -438,6 +438,7 @@ EXPORT_SYMBOL(inet_release); · 437 unmodified lines
438int inet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)438int inet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
439{439{
440 struct sock *sk = sock->sk;440 struct sock *sk = sock->sk;
441 u32 flags = BIND_WITH_LOCK;
441 int err;442 int err;
442 443
443 /* If the socket has its own bind function then use it. (RAW) */444 /* If the socket has its own bind function then use it. (RAW) */
@@ -450,11 +451,12 @@ int inet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) · 6 unmodified lines
450 /* BPF prog is run before any checks are done so that if the prog451 /* BPF prog is run before any checks are done so that if the prog
451 * changes context in a wrong way it will be caught.452 * changes context in a wrong way it will be caught.
452 */453 */
453 err = BPF_CGROUP_RUN_PROG_INET4_BIND_LOCK(sk, uaddr);454 err = BPF_CGROUP_RUN_PROG_INET_BIND_LOCK(sk, uaddr,
455 BPF_CGROUP_INET4_BIND, &flags);
454 if (err)456 if (err)
455 return err;457 return err;
456 458
457 return __inet_bind(sk, uaddr, addr_len, BIND_WITH_LOCK);459 return __inet_bind(sk, uaddr, addr_len, flags);
458}460}
459EXPORT_SYMBOL(inet_bind);461EXPORT_SYMBOL(inet_bind);
460 462
@@ -499,7 +501,8 @@ int __inet_bind(struct sock *sk, struct sockaddr *uaddr, int addr_len, · 38 unmodified lines
499 501
500 snum = ntohs(addr->sin_port);502 snum = ntohs(addr->sin_port);
501 err = -EACCES;503 err = -EACCES;
502 if (snum && inet_port_requires_bind_service(net, snum) &&504 if (!(flags & BIND_NO_CAP_NET_BIND_SERVICE) &&
505 snum && inet_port_requires_bind_service(net, snum) &&
503 !ns_capable(net->user_ns, CAP_NET_BIND_SERVICE))506 !ns_capable(net->user_ns, CAP_NET_BIND_SERVICE))
504 goto out;507 goto out;
505 508
@@ -777,18 +780,19 @@ int inet_getname(struct socket *sock, struct sockaddr *uaddr, · 271 unmodified lines
777 return -ENOTCONN;780 return -ENOTCONN;
778 sin->sin_port = inet->inet_dport;781 sin->sin_port = inet->inet_dport;
779 sin->sin_addr.s_addr = inet->inet_daddr;782 sin->sin_addr.s_addr = inet->inet_daddr;
783 BPF_CGROUP_RUN_SA_PROG_LOCK(sk, (struct sockaddr *)sin,
784 BPF_CGROUP_INET4_GETPEERNAME,
785 NULL);
780 } else {786 } else {
781 __be32 addr = inet->inet_rcv_saddr;787 __be32 addr = inet->inet_rcv_saddr;
782 if (!addr)788 if (!addr)
783 addr = inet->inet_saddr;789 addr = inet->inet_saddr;
784 sin->sin_port = inet->inet_sport;790 sin->sin_port = inet->inet_sport;
785 sin->sin_addr.s_addr = addr;791 sin->sin_addr.s_addr = addr;
786 }
787 if (cgroup_bpf_enabled)
788 BPF_CGROUP_RUN_SA_PROG_LOCK(sk, (struct sockaddr *)sin,792 BPF_CGROUP_RUN_SA_PROG_LOCK(sk, (struct sockaddr *)sin,
789 peer ? BPF_CGROUP_INET4_GETPEERNAME :793 BPF_CGROUP_INET4_GETSOCKNAME,
790 BPF_CGROUP_INET4_GETSOCKNAME,
791 NULL);794 NULL);
795 }
792 memset(sin->sin_zero, 0, sizeof(sin->sin_zero));796 memset(sin->sin_zero, 0, sizeof(sin->sin_zero));
793 return sizeof(*sin);797 return sizeof(*sin);
794}798}
net/ipv4/tcp.c+14−0
@@ -4162,6 +4162,8 @@ static int do_tcp_getsockopt(struct sock *sk, int level, · 4161 unmodified lines
4162 return -EINVAL;4162 return -EINVAL;
4163 lock_sock(sk);4163 lock_sock(sk);
4164 err = tcp_zerocopy_receive(sk, &zc, &tss);4164 err = tcp_zerocopy_receive(sk, &zc, &tss);
4165 err = BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sk, level, optname,
4166 &zc, &len, err);
4165 release_sock(sk);4167 release_sock(sk);
4166 if (len >= offsetofend(struct tcp_zerocopy_receive, msg_flags))4168 if (len >= offsetofend(struct tcp_zerocopy_receive, msg_flags))
4167 goto zerocopy_rcv_cmsg;4169 goto zerocopy_rcv_cmsg;
@@ -4208,6 +4210,18 @@ zerocopy_rcv_out: · 40 unmodified lines
4208 return 0;4210 return 0;
4209}4211}
4210 4212
4213bool tcp_bpf_bypass_getsockopt(int level, int optname)
4214{
4215 /* TCP do_tcp_getsockopt has optimized getsockopt implementation
4216 * to avoid extra socket lock for TCP_ZEROCOPY_RECEIVE.
4217 */
4218 if (level == SOL_TCP && optname == TCP_ZEROCOPY_RECEIVE)
4219 return true;
4220
4221 return false;
4222}
4223EXPORT_SYMBOL(tcp_bpf_bypass_getsockopt);
4224
4211int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval,4225int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval,
4212 int __user *optlen)4226 int __user *optlen)
4213{4227{
net/ipv4/tcp_ipv4.c+1−0
@@ -2796,6 +2796,7 @@ struct proto tcp_prot = { · 2795 unmodified lines
2796 .shutdown = tcp_shutdown,2796 .shutdown = tcp_shutdown,
2797 .setsockopt = tcp_setsockopt,2797 .setsockopt = tcp_setsockopt,
2798 .getsockopt = tcp_getsockopt,2798 .getsockopt = tcp_getsockopt,
2799 .bpf_bypass_getsockopt = tcp_bpf_bypass_getsockopt,
2799 .keepalive = tcp_set_keepalive,2800 .keepalive = tcp_set_keepalive,
2800 .recvmsg = tcp_recvmsg,2801 .recvmsg = tcp_recvmsg,
2801 .sendmsg = tcp_sendmsg,2802 .sendmsg = tcp_sendmsg,
net/ipv4/udp.c+3−4
@@ -1130,7 +1130,7 @@ int udp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) · 1129 unmodified lines
1130 rcu_read_unlock();1130 rcu_read_unlock();
1131 }1131 }
1132 1132
1133 if (cgroup_bpf_enabled && !connected) {1133 if (cgroup_bpf_enabled(BPF_CGROUP_UDP4_SENDMSG) && !connected) {
1134 err = BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk,1134 err = BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk,
1135 (struct sockaddr *)usin, &ipc.addr);1135 (struct sockaddr *)usin, &ipc.addr);
1136 if (err)1136 if (err)
@@ -1864,9 +1864,8 @@ try_again: · 727 unmodified lines
1864 memset(sin->sin_zero, 0, sizeof(sin->sin_zero));1864 memset(sin->sin_zero, 0, sizeof(sin->sin_zero));
1865 *addr_len = sizeof(*sin);1865 *addr_len = sizeof(*sin);
1866 1866
1867 if (cgroup_bpf_enabled)1867 BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk,
1868 BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk,1868 (struct sockaddr *)sin);
1869 (struct sockaddr *)sin);
1870 }1869 }
1871 1870
1872 if (udp_sk(sk)->gro_enabled)1871 if (udp_sk(sk)->gro_enabled)
net/ipv6/af_inet6.c+11−7
@@ -295,7 +295,8 @@ static int __inet6_bind(struct sock *sk, struct sockaddr *uaddr, int addr_len, · 294 unmodified lines
295 return -EINVAL;295 return -EINVAL;
296 296
297 snum = ntohs(addr->sin6_port);297 snum = ntohs(addr->sin6_port);
298 if (snum && inet_port_requires_bind_service(net, snum) &&298 if (!(flags & BIND_NO_CAP_NET_BIND_SERVICE) &&
299 snum && inet_port_requires_bind_service(net, snum) &&
299 !ns_capable(net->user_ns, CAP_NET_BIND_SERVICE))300 !ns_capable(net->user_ns, CAP_NET_BIND_SERVICE))
300 return -EACCES;301 return -EACCES;
301 302
@@ -439,6 +440,7 @@ out_unlock: · 137 unmodified lines
439int inet6_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)440int inet6_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
440{441{
441 struct sock *sk = sock->sk;442 struct sock *sk = sock->sk;
443 u32 flags = BIND_WITH_LOCK;
442 int err = 0;444 int err = 0;
443 445
444 /* If the socket has its own bind function then use it. */446 /* If the socket has its own bind function then use it. */
@@ -451,11 +453,12 @@ int inet6_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) · 6 unmodified lines
451 /* BPF prog is run before any checks are done so that if the prog453 /* BPF prog is run before any checks are done so that if the prog
452 * changes context in a wrong way it will be caught.454 * changes context in a wrong way it will be caught.
453 */455 */
454 err = BPF_CGROUP_RUN_PROG_INET6_BIND_LOCK(sk, uaddr);456 err = BPF_CGROUP_RUN_PROG_INET_BIND_LOCK(sk, uaddr,
457 BPF_CGROUP_INET6_BIND, &flags);
455 if (err)458 if (err)
456 return err;459 return err;
457 460
458 return __inet6_bind(sk, uaddr, addr_len, BIND_WITH_LOCK);461 return __inet6_bind(sk, uaddr, addr_len, flags);
459}462}
460EXPORT_SYMBOL(inet6_bind);463EXPORT_SYMBOL(inet6_bind);
461 464
@@ -527,18 +530,19 @@ int inet6_getname(struct socket *sock, struct sockaddr *uaddr, · 65 unmodified lines
527 sin->sin6_addr = sk->sk_v6_daddr;530 sin->sin6_addr = sk->sk_v6_daddr;
528 if (np->sndflow)531 if (np->sndflow)
529 sin->sin6_flowinfo = np->flow_label;532 sin->sin6_flowinfo = np->flow_label;
533 BPF_CGROUP_RUN_SA_PROG_LOCK(sk, (struct sockaddr *)sin,
534 BPF_CGROUP_INET6_GETPEERNAME,
535 NULL);
530 } else {536 } else {
531 if (ipv6_addr_any(&sk->sk_v6_rcv_saddr))537 if (ipv6_addr_any(&sk->sk_v6_rcv_saddr))
532 sin->sin6_addr = np->saddr;538 sin->sin6_addr = np->saddr;
533 else539 else
534 sin->sin6_addr = sk->sk_v6_rcv_saddr;540 sin->sin6_addr = sk->sk_v6_rcv_saddr;
535 sin->sin6_port = inet->inet_sport;541 sin->sin6_port = inet->inet_sport;
536 }
537 if (cgroup_bpf_enabled)
538 BPF_CGROUP_RUN_SA_PROG_LOCK(sk, (struct sockaddr *)sin,542 BPF_CGROUP_RUN_SA_PROG_LOCK(sk, (struct sockaddr *)sin,
539 peer ? BPF_CGROUP_INET6_GETPEERNAME :543 BPF_CGROUP_INET6_GETSOCKNAME,
540 BPF_CGROUP_INET6_GETSOCKNAME,
541 NULL);544 NULL);
545 }
542 sin->sin6_scope_id = ipv6_iface_scope_id(&sin->sin6_addr,546 sin->sin6_scope_id = ipv6_iface_scope_id(&sin->sin6_addr,
543 sk->sk_bound_dev_if);547 sk->sk_bound_dev_if);
544 return sizeof(*sin);548 return sizeof(*sin);
net/ipv6/tcp_ipv6.c+1−0
@@ -2124,6 +2124,7 @@ struct proto tcpv6_prot = { · 2123 unmodified lines
2124 .shutdown = tcp_shutdown,2124 .shutdown = tcp_shutdown,
2125 .setsockopt = tcp_setsockopt,2125 .setsockopt = tcp_setsockopt,
2126 .getsockopt = tcp_getsockopt,2126 .getsockopt = tcp_getsockopt,
2127 .bpf_bypass_getsockopt = tcp_bpf_bypass_getsockopt,
2127 .keepalive = tcp_set_keepalive,2128 .keepalive = tcp_set_keepalive,
2128 .recvmsg = tcp_recvmsg,2129 .recvmsg = tcp_recvmsg,
2129 .sendmsg = tcp_sendmsg,2130 .sendmsg = tcp_sendmsg,
net/ipv6/udp.c+3−4
@@ -409,9 +409,8 @@ try_again: · 408 unmodified lines
409 }409 }
410 *addr_len = sizeof(*sin6);410 *addr_len = sizeof(*sin6);
411 411
412 if (cgroup_bpf_enabled)412 BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK(sk,
413 BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK(sk,413 (struct sockaddr *)sin6);
414 (struct sockaddr *)sin6);
415 }414 }
416 415
417 if (udp_sk(sk)->gro_enabled)416 if (udp_sk(sk)->gro_enabled)
@@ -1462,7 +1461,7 @@ do_udp_sendmsg: · 1044 unmodified lines
1462 fl6.saddr = np->saddr;1461 fl6.saddr = np->saddr;
1463 fl6.fl6_sport = inet->inet_sport;1462 fl6.fl6_sport = inet->inet_sport;
1464 1463
1465 if (cgroup_bpf_enabled && !connected) {1464 if (cgroup_bpf_enabled(BPF_CGROUP_UDP6_SENDMSG) && !connected) {
1466 err = BPF_CGROUP_RUN_PROG_UDP6_SENDMSG_LOCK(sk,1465 err = BPF_CGROUP_RUN_PROG_UDP6_SENDMSG_LOCK(sk,
1467 (struct sockaddr *)sin6, &fl6.saddr);1466 (struct sockaddr *)sin6, &fl6.saddr);
1468 if (err)1467 if (err)
net/socket.c+3−0
@@ -2126,6 +2126,9 @@ SYSCALL_DEFINE5(setsockopt, int, fd, int, level, int, optname, · 2125 unmodified lines
2126 return __sys_setsockopt(fd, level, optname, optval, optlen);2126 return __sys_setsockopt(fd, level, optname, optval, optlen);
2127}2127}
2128 2128
2129INDIRECT_CALLABLE_DECLARE(bool tcp_bpf_bypass_getsockopt(int level,
2130 int optname));
2131
2129/*2132/*
2130 * Get a socket option. Because we don't know the option lengths we have2133 * Get a socket option. Because we don't know the option lengths we have
2131 * to pass a user mode parameter for the protocols to sort out.2134 * to pass a user mode parameter for the protocols to sort out.
net/xdp/xsk.c+31−16
@@ -184,12 +184,13 @@ static void xsk_copy_xdp(struct xdp_buff *to, struct xdp_buff *from, u32 len) · 183 unmodified lines
184 memcpy(to_buf, from_buf, len + metalen);184 memcpy(to_buf, from_buf, len + metalen);
185}185}
186 186
187static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len,187static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
188 bool explicit_free)
189{188{
190 struct xdp_buff *xsk_xdp;189 struct xdp_buff *xsk_xdp;
191 int err;190 int err;
191 u32 len;
192 192
193 len = xdp->data_end - xdp->data;
193 if (len > xsk_pool_get_rx_frame_size(xs->pool)) {194 if (len > xsk_pool_get_rx_frame_size(xs->pool)) {
194 xs->rx_dropped++;195 xs->rx_dropped++;
195 return -ENOSPC;196 return -ENOSPC;
@@ -207,8 +208,6 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len, · 11 unmodified lines
207 xsk_buff_free(xsk_xdp);208 xsk_buff_free(xsk_xdp);
208 return err;209 return err;
209 }210 }
210 if (explicit_free)
211 xdp_return_buff(xdp);
212 return 0;211 return 0;
213}212}
214 213
@@ -230,11 +229,8 @@ static bool xsk_is_bound(struct xdp_sock *xs) · 15 unmodified lines
230 return false;229 return false;
231}230}
232 231
233static int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp,232static int xsk_rcv_check(struct xdp_sock *xs, struct xdp_buff *xdp)
234 bool explicit_free)
235{233{
236 u32 len;
237
238 if (!xsk_is_bound(xs))234 if (!xsk_is_bound(xs))
239 return -EINVAL;235 return -EINVAL;
240 236
@@ -242,11 +238,7 @@ static int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, · 1 unmodified line
242 return -EINVAL;238 return -EINVAL;
243 239
244 sk_mark_napi_id_once_xdp(&xs->sk, xdp);240 sk_mark_napi_id_once_xdp(&xs->sk, xdp);
245 len = xdp->data_end - xdp->data;241 return 0;
246
247 return xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL ?
248 __xsk_rcv_zc(xs, xdp, len) :
249 __xsk_rcv(xs, xdp, len, explicit_free);
250}242}
251 243
252static void xsk_flush(struct xdp_sock *xs)244static void xsk_flush(struct xdp_sock *xs)
@@ -261,18 +253,41 @@ int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) · 8 unmodified lines
261 int err;253 int err;
262 254
263 spin_lock_bh(&xs->rx_lock);255 spin_lock_bh(&xs->rx_lock);
264 err = xsk_rcv(xs, xdp, false);256 err = xsk_rcv_check(xs, xdp);
265 xsk_flush(xs);257 if (!err) {
258 err = __xsk_rcv(xs, xdp);
259 xsk_flush(xs);
260 }
266 spin_unlock_bh(&xs->rx_lock);261 spin_unlock_bh(&xs->rx_lock);
267 return err;262 return err;
268}263}
269 264
265static int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
266{
267 int err;
268 u32 len;
269
270 err = xsk_rcv_check(xs, xdp);
271 if (err)
272 return err;
273
274 if (xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL) {
275 len = xdp->data_end - xdp->data;
276 return __xsk_rcv_zc(xs, xdp, len);
277 }
278
279 err = __xsk_rcv(xs, xdp);
280 if (!err)
281 xdp_return_buff(xdp);
282 return err;
283}
284
270int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp)285int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp)
271{286{
272 struct list_head *flush_list = this_cpu_ptr(&xskmap_flush_list);287 struct list_head *flush_list = this_cpu_ptr(&xskmap_flush_list);
273 int err;288 int err;
274 289
275 err = xsk_rcv(xs, xdp, true);290 err = xsk_rcv(xs, xdp);
276 if (err)291 if (err)
277 return err;292 return err;
278 293
net/xdp/xsk_buff_pool.c+3−9
@@ -119,8 +119,8 @@ static void xp_disable_drv_zc(struct xsk_buff_pool *pool) · 118 unmodified lines
119 }119 }
120}120}
121 121
122static int __xp_assign_dev(struct xsk_buff_pool *pool,122int xp_assign_dev(struct xsk_buff_pool *pool,
123 struct net_device *netdev, u16 queue_id, u16 flags)123 struct net_device *netdev, u16 queue_id, u16 flags)
124{124{
125 bool force_zc, force_copy;125 bool force_zc, force_copy;
126 struct netdev_bpf bpf;126 struct netdev_bpf bpf;
@@ -191,12 +191,6 @@ err_unreg_pool: · 64 unmodified lines
191 return err;191 return err;
192}192}
193 193
194int xp_assign_dev(struct xsk_buff_pool *pool, struct net_device *dev,
195 u16 queue_id, u16 flags)
196{
197 return __xp_assign_dev(pool, dev, queue_id, flags);
198}
199
200int xp_assign_dev_shared(struct xsk_buff_pool *pool, struct xdp_umem *umem,194int xp_assign_dev_shared(struct xsk_buff_pool *pool, struct xdp_umem *umem,
201 struct net_device *dev, u16 queue_id)195 struct net_device *dev, u16 queue_id)
202{196{
@@ -210,7 +204,7 @@ int xp_assign_dev_shared(struct xsk_buff_pool *pool, struct xdp_umem *umem, · 7 unmodified lines
210 if (pool->uses_need_wakeup)204 if (pool->uses_need_wakeup)
211 flags |= XDP_USE_NEED_WAKEUP;205 flags |= XDP_USE_NEED_WAKEUP;
212 206
213 return __xp_assign_dev(pool, dev, queue_id, flags);207 return xp_assign_dev(pool, dev, queue_id, flags);
214}208}
215 209
216void xp_clear_dev(struct xsk_buff_pool *pool)210void xp_clear_dev(struct xsk_buff_pool *pool)
samples/bpf/Makefile+9−1
@@ -183,6 +183,14 @@ BPF_EXTRA_CFLAGS := $(ARM_ARCH_SELECTOR) · 182 unmodified lines
183TPROGS_CFLAGS += $(ARM_ARCH_SELECTOR)183TPROGS_CFLAGS += $(ARM_ARCH_SELECTOR)
184endif184endif
185 185
186ifeq ($(ARCH), mips)
187TPROGS_CFLAGS += -D__SANE_USERSPACE_TYPES__
188ifdef CONFIG_MACH_LOONGSON64
189BPF_EXTRA_CFLAGS += -I$(srctree)/arch/mips/include/asm/mach-loongson64
190BPF_EXTRA_CFLAGS += -I$(srctree)/arch/mips/include/asm/mach-generic
191endif
192endif
193
186TPROGS_CFLAGS += -Wall -O2194TPROGS_CFLAGS += -Wall -O2
187TPROGS_CFLAGS += -Wmissing-prototypes195TPROGS_CFLAGS += -Wmissing-prototypes
188TPROGS_CFLAGS += -Wstrict-prototypes196TPROGS_CFLAGS += -Wstrict-prototypes
@@ -208,7 +216,7 @@ TPROGLDLIBS_xdpsock += -pthread -lcap · 19 unmodified lines
208TPROGLDLIBS_xsk_fwd += -pthread216TPROGLDLIBS_xsk_fwd += -pthread
209 217
210# Allows pointing LLC/CLANG to a LLVM backend with bpf support, redefine on cmdline:218# Allows pointing LLC/CLANG to a LLVM backend with bpf support, redefine on cmdline:
211# make M=samples/bpf/ LLC=~/git/llvm/build/bin/llc CLANG=~/git/llvm/build/bin/clang219# make M=samples/bpf LLC=~/git/llvm-project/llvm/build/bin/llc CLANG=~/git/llvm-project/llvm/build/bin/clang
212LLC ?= llc220LLC ?= llc
213CLANG ?= clang221CLANG ?= clang
214OPT ?= opt222OPT ?= opt
samples/bpf/README.rst+14−8
@@ -62,20 +62,26 @@ To generate a smaller llc binary one can use:: · 61 unmodified lines
62 62
63 -DLLVM_TARGETS_TO_BUILD="BPF"63 -DLLVM_TARGETS_TO_BUILD="BPF"
64 64
65We recommend that developers who want the fastest incremental builds
66use the Ninja build system, you can find it in your system's package
67manager, usually the package is ninja or ninja-build.
68
65Quick sniplet for manually compiling LLVM and clang69Quick sniplet for manually compiling LLVM and clang
66(build dependencies are cmake and gcc-c++)::70(build dependencies are ninja, cmake and gcc-c++)::
67 71
68 $ git clone http://llvm.org/git/llvm.git72 $ git clone https://github.com/llvm/llvm-project.git
69 $ cd llvm/tools73 $ mkdir -p llvm-project/llvm/build
70 $ git clone --depth 1 http://llvm.org/git/clang.git74 $ cd llvm-project/llvm/build
71 $ cd ..; mkdir build; cd build75 $ cmake .. -G "Ninja" -DLLVM_TARGETS_TO_BUILD="BPF;X86" \
72 $ cmake .. -DLLVM_TARGETS_TO_BUILD="BPF;X86"76 -DLLVM_ENABLE_PROJECTS="clang" \
73 $ make -j $(getconf _NPROCESSORS_ONLN)77 -DCMAKE_BUILD_TYPE=Release \
78 -DLLVM_BUILD_RUNTIME=OFF
79 $ ninja
74 80
75It is also possible to point make to the newly compiled 'llc' or81It is also possible to point make to the newly compiled 'llc' or
76'clang' command via redefining LLC or CLANG on the make command line::82'clang' command via redefining LLC or CLANG on the make command line::
77 83
78 make M=samples/bpf LLC=~/git/llvm/build/bin/llc CLANG=~/git/llvm/build/bin/clang84 make M=samples/bpf LLC=~/git/llvm-project/llvm/build/bin/llc CLANG=~/git/llvm-project/llvm/build/bin/clang
79 85
80Cross compiling samples86Cross compiling samples
81-----------------------87-----------------------
samples/bpf/bpf_insn.h+20−4
@@ -134,15 +134,31 @@ struct bpf_insn; · 133 unmodified lines
134 .off = OFF, \134 .off = OFF, \
135 .imm = 0 })135 .imm = 0 })
136 136
137/* Atomic memory add, *(uint *)(dst_reg + off16) += src_reg */137/*
138 138 * Atomic operations:
139#define BPF_STX_XADD(SIZE, DST, SRC, OFF) \139 *
140 * BPF_ADD *(uint *) (dst_reg + off16) += src_reg
141 * BPF_AND *(uint *) (dst_reg + off16) &= src_reg
142 * BPF_OR *(uint *) (dst_reg + off16) |= src_reg
143 * BPF_XOR *(uint *) (dst_reg + off16) ^= src_reg
144 * BPF_ADD | BPF_FETCH src_reg = atomic_fetch_add(dst_reg + off16, src_reg);
145 * BPF_AND | BPF_FETCH src_reg = atomic_fetch_and(dst_reg + off16, src_reg);
146 * BPF_OR | BPF_FETCH src_reg = atomic_fetch_or(dst_reg + off16, src_reg);
147 * BPF_XOR | BPF_FETCH src_reg = atomic_fetch_xor(dst_reg + off16, src_reg);
148 * BPF_XCHG src_reg = atomic_xchg(dst_reg + off16, src_reg)
149 * BPF_CMPXCHG r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg)
150 */
151
152#define BPF_ATOMIC_OP(SIZE, OP, DST, SRC, OFF) \
140 ((struct bpf_insn) { \153 ((struct bpf_insn) { \
141 .code = BPF_STX | BPF_SIZE(SIZE) | BPF_ATOMIC, \154 .code = BPF_STX | BPF_SIZE(SIZE) | BPF_ATOMIC, \
142 .dst_reg = DST, \155 .dst_reg = DST, \
143 .src_reg = SRC, \156 .src_reg = SRC, \
144 .off = OFF, \157 .off = OFF, \
145 .imm = BPF_ADD })158 .imm = OP })
159
160/* Legacy alias */
161#define BPF_STX_XADD(SIZE, DST, SRC, OFF) BPF_ATOMIC_OP(SIZE, BPF_ADD, DST, SRC, OFF)
146 162
147/* Memory store, *(uint *) (dst_reg + off16) = imm32 */163/* Memory store, *(uint *) (dst_reg + off16) = imm32 */
148 164
samples/bpf/cookie_uid_helper_example.c+1−1
@@ -313,7 +313,7 @@ int main(int argc, char *argv[]) · 312 unmodified lines
313 print_table();313 print_table();
314 printf("\n");314 printf("\n");
315 sleep(1);315 sleep(1);
316 };316 }
317 } else if (cfg_test_cookie) {317 } else if (cfg_test_cookie) {
318 udp_client();318 udp_client();
319 }319 }
samples/bpf/xdp_redirect_map_kern.c+55−5
@@ -19,12 +19,22 @@ · 18 unmodified lines
19#include <linux/ipv6.h>19#include <linux/ipv6.h>
20#include <bpf/bpf_helpers.h>20#include <bpf/bpf_helpers.h>
21 21
22/* The 2nd xdp prog on egress does not support skb mode, so we define two
23 * maps, tx_port_general and tx_port_native.
24 */
22struct {25struct {
23 __uint(type, BPF_MAP_TYPE_DEVMAP);26 __uint(type, BPF_MAP_TYPE_DEVMAP);
24 __uint(key_size, sizeof(int));27 __uint(key_size, sizeof(int));
25 __uint(value_size, sizeof(int));28 __uint(value_size, sizeof(int));
26 __uint(max_entries, 100);29 __uint(max_entries, 100);
27} tx_port SEC(".maps");30} tx_port_general SEC(".maps");
31
32struct {
33 __uint(type, BPF_MAP_TYPE_DEVMAP);
34 __uint(key_size, sizeof(int));
35 __uint(value_size, sizeof(struct bpf_devmap_val));
36 __uint(max_entries, 100);
37} tx_port_native SEC(".maps");
28 38
29/* Count RX packets, as XDP bpf_prog doesn't get direct TX-success39/* Count RX packets, as XDP bpf_prog doesn't get direct TX-success
30 * feedback. Redirect TX errors can be caught via a tracepoint.40 * feedback. Redirect TX errors can be caught via a tracepoint.
@@ -36,6 +46,14 @@ struct { · 5 unmodified lines
36 __uint(max_entries, 1);46 __uint(max_entries, 1);
37} rxcnt SEC(".maps");47} rxcnt SEC(".maps");
38 48
49/* map to store egress interface mac address */
50struct {
51 __uint(type, BPF_MAP_TYPE_ARRAY);
52 __type(key, u32);
53 __type(value, __be64);
54 __uint(max_entries, 1);
55} tx_mac SEC(".maps");
56
39static void swap_src_dst_mac(void *data)57static void swap_src_dst_mac(void *data)
40{58{
41 unsigned short *p = data;59 unsigned short *p = data;
@@ -52,17 +70,16 @@ static void swap_src_dst_mac(void *data) · 10 unmodified lines
52 p[5] = dst[2];70 p[5] = dst[2];
53}71}
54 72
55SEC("xdp_redirect_map")73static __always_inline int xdp_redirect_map(struct xdp_md *ctx, void *redirect_map)
56int xdp_redirect_map_prog(struct xdp_md *ctx)
57{74{
58 void *data_end = (void *)(long)ctx->data_end;75 void *data_end = (void *)(long)ctx->data_end;
59 void *data = (void *)(long)ctx->data;76 void *data = (void *)(long)ctx->data;
60 struct ethhdr *eth = data;77 struct ethhdr *eth = data;
61 int rc = XDP_DROP;78 int rc = XDP_DROP;
62 int vport, port = 0, m = 0;
63 long *value;79 long *value;
64 u32 key = 0;80 u32 key = 0;
65 u64 nh_off;81 u64 nh_off;
82 int vport;
66 83
67 nh_off = sizeof(*eth);84 nh_off = sizeof(*eth);
68 if (data + nh_off > data_end)85 if (data + nh_off > data_end)
@@ -79,7 +96,40 @@ int xdp_redirect_map_prog(struct xdp_md *ctx) · 10 unmodified lines
79 swap_src_dst_mac(data);96 swap_src_dst_mac(data);
80 97
81 /* send packet out physical port */98 /* send packet out physical port */
82 return bpf_redirect_map(&tx_port, vport, 0);99 return bpf_redirect_map(redirect_map, vport, 0);
100}
101
102SEC("xdp_redirect_general")
103int xdp_redirect_map_general(struct xdp_md *ctx)
104{
105 return xdp_redirect_map(ctx, &tx_port_general);
106}
107
108SEC("xdp_redirect_native")
109int xdp_redirect_map_native(struct xdp_md *ctx)
110{
111 return xdp_redirect_map(ctx, &tx_port_native);
112}
113
114SEC("xdp_devmap/map_prog")
115int xdp_redirect_map_egress(struct xdp_md *ctx)
116{
117 void *data_end = (void *)(long)ctx->data_end;
118 void *data = (void *)(long)ctx->data;
119 struct ethhdr *eth = data;
120 __be64 *mac;
121 u32 key = 0;
122 u64 nh_off;
123
124 nh_off = sizeof(*eth);
125 if (data + nh_off > data_end)
126 return XDP_DROP;
127
128 mac = bpf_map_lookup_elem(&tx_mac, &key);
129 if (mac)
130 __builtin_memcpy(eth->h_source, mac, ETH_ALEN);
131
132 return XDP_PASS;
83}133}
84 134
85/* Redirect require an XDP bpf_prog loaded on the TX device */135/* Redirect require an XDP bpf_prog loaded on the TX device */
samples/bpf/xdp_redirect_map_user.c+92−20
@@ -14,6 +14,10 @@ · 13 unmodified lines
14#include <unistd.h>14#include <unistd.h>
15#include <libgen.h>15#include <libgen.h>
16#include <sys/resource.h>16#include <sys/resource.h>
17#include <sys/ioctl.h>
18#include <sys/types.h>
19#include <sys/socket.h>
20#include <netinet/in.h>
17 21
18#include "bpf_util.h"22#include "bpf_util.h"
19#include <bpf/bpf.h>23#include <bpf/bpf.h>
@@ -22,6 +26,7 @@ · 2 unmodified lines
22static int ifindex_in;26static int ifindex_in;
23static int ifindex_out;27static int ifindex_out;
24static bool ifindex_out_xdp_dummy_attached = true;28static bool ifindex_out_xdp_dummy_attached = true;
29static bool xdp_devmap_attached;
25static __u32 prog_id;30static __u32 prog_id;
26static __u32 dummy_prog_id;31static __u32 dummy_prog_id;
27 32
@@ -83,6 +88,32 @@ static void poll_stats(int interval, int ifindex) · 55 unmodified lines
83 }88 }
84}89}
85 90
91static int get_mac_addr(unsigned int ifindex_out, void *mac_addr)
92{
93 char ifname[IF_NAMESIZE];
94 struct ifreq ifr;
95 int fd, ret = -1;
96
97 fd = socket(AF_INET, SOCK_DGRAM, 0);
98 if (fd < 0)
99 return ret;
100
101 if (!if_indextoname(ifindex_out, ifname))
102 goto err_out;
103
104 strcpy(ifr.ifr_name, ifname);
105
106 if (ioctl(fd, SIOCGIFHWADDR, &ifr) != 0)
107 goto err_out;
108
109 memcpy(mac_addr, ifr.ifr_hwaddr.sa_data, 6 * sizeof(char));
110 ret = 0;
111
112err_out:
113 close(fd);
114 return ret;
115}
116
86static void usage(const char *prog)117static void usage(const char *prog)
87{118{
88 fprintf(stderr,119 fprintf(stderr,
@@ -90,24 +121,26 @@ static void usage(const char *prog) · 1 unmodified line
90 "OPTS:\n"121 "OPTS:\n"
91 " -S use skb-mode\n"122 " -S use skb-mode\n"
92 " -N enforce native mode\n"123 " -N enforce native mode\n"
93 " -F force loading prog\n",124 " -F force loading prog\n"
125 " -X load xdp program on egress\n",
94 prog);126 prog);
95}127}
96 128
97int main(int argc, char **argv)129int main(int argc, char **argv)
98{130{
99 struct bpf_prog_load_attr prog_load_attr = {131 struct bpf_prog_load_attr prog_load_attr = {
100 .prog_type = BPF_PROG_TYPE_XDP,132 .prog_type = BPF_PROG_TYPE_UNSPEC,
101 };133 };
102 struct bpf_program *prog, *dummy_prog;134 struct bpf_program *prog, *dummy_prog, *devmap_prog;
135 int prog_fd, dummy_prog_fd, devmap_prog_fd = 0;
136 int tx_port_map_fd, tx_mac_map_fd;
137 struct bpf_devmap_val devmap_val;
103 struct bpf_prog_info info = {};138 struct bpf_prog_info info = {};
104 __u32 info_len = sizeof(info);139 __u32 info_len = sizeof(info);
105 int prog_fd, dummy_prog_fd;140 const char *optstr = "FSNX";
106 const char *optstr = "FSN";
107 struct bpf_object *obj;141 struct bpf_object *obj;
108 int ret, opt, key = 0;142 int ret, opt, key = 0;
109 char filename[256];143 char filename[256];
110 int tx_port_map_fd;
111 144
112 while ((opt = getopt(argc, argv, optstr)) != -1) {145 while ((opt = getopt(argc, argv, optstr)) != -1) {
113 switch (opt) {146 switch (opt) {
@@ -120,14 +153,21 @@ int main(int argc, char **argv) · 6 unmodified lines
120 case 'F':153 case 'F':
121 xdp_flags &= ~XDP_FLAGS_UPDATE_IF_NOEXIST;154 xdp_flags &= ~XDP_FLAGS_UPDATE_IF_NOEXIST;
122 break;155 break;
156 case 'X':
157 xdp_devmap_attached = true;
158 break;
123 default:159 default:
124 usage(basename(argv[0]));160 usage(basename(argv[0]));
125 return 1;161 return 1;
126 }162 }
127 }163 }
128 164
129 if (!(xdp_flags & XDP_FLAGS_SKB_MODE))165 if (!(xdp_flags & XDP_FLAGS_SKB_MODE)) {
130 xdp_flags |= XDP_FLAGS_DRV_MODE;166 xdp_flags |= XDP_FLAGS_DRV_MODE;
167 } else if (xdp_devmap_attached) {
168 printf("Load xdp program on egress with SKB mode not supported yet\n");
169 return 1;
170 }
131 171
132 if (optind == argc) {172 if (optind == argc) {
133 printf("usage: %s <IFNAME|IFINDEX>_IN <IFNAME|IFINDEX>_OUT\n", argv[0]);173 printf("usage: %s <IFNAME|IFINDEX>_IN <IFNAME|IFINDEX>_OUT\n", argv[0]);
@@ -150,24 +190,28 @@ int main(int argc, char **argv) · 16 unmodified lines
150 if (bpf_prog_load_xattr(&prog_load_attr, &obj, &prog_fd))190 if (bpf_prog_load_xattr(&prog_load_attr, &obj, &prog_fd))
151 return 1;191 return 1;
152 192
153 prog = bpf_program__next(NULL, obj);193 if (xdp_flags & XDP_FLAGS_SKB_MODE) {
154 dummy_prog = bpf_program__next(prog, obj);194 prog = bpf_object__find_program_by_name(obj, "xdp_redirect_map_general");
155 if (!prog || !dummy_prog) {195 tx_port_map_fd = bpf_object__find_map_fd_by_name(obj, "tx_port_general");
156 printf("finding a prog in obj file failed\n");196 } else {
157 return 1;197 prog = bpf_object__find_program_by_name(obj, "xdp_redirect_map_native");
198 tx_port_map_fd = bpf_object__find_map_fd_by_name(obj, "tx_port_native");
199 }
200 dummy_prog = bpf_object__find_program_by_name(obj, "xdp_redirect_dummy_prog");
201 if (!prog || dummy_prog < 0 || tx_port_map_fd < 0) {
202 printf("finding prog/dummy_prog/tx_port_map in obj file failed\n");
203 goto out;
158 }204 }
159 /* bpf_prog_load_xattr gives us the pointer to first prog's fd,205 prog_fd = bpf_program__fd(prog);
160 * so we're missing only the fd for dummy prog
161 */
162 dummy_prog_fd = bpf_program__fd(dummy_prog);206 dummy_prog_fd = bpf_program__fd(dummy_prog);
163 if (prog_fd < 0 || dummy_prog_fd < 0) {207 if (prog_fd < 0 || dummy_prog_fd < 0 || tx_port_map_fd < 0) {
164 printf("bpf_prog_load_xattr: %s\n", strerror(errno));208 printf("bpf_prog_load_xattr: %s\n", strerror(errno));
165 return 1;209 return 1;
166 }210 }
167 211
168 tx_port_map_fd = bpf_object__find_map_fd_by_name(obj, "tx_port");212 tx_mac_map_fd = bpf_object__find_map_fd_by_name(obj, "tx_mac");
169 rxcnt_map_fd = bpf_object__find_map_fd_by_name(obj, "rxcnt");213 rxcnt_map_fd = bpf_object__find_map_fd_by_name(obj, "rxcnt");
170 if (tx_port_map_fd < 0 || rxcnt_map_fd < 0) {214 if (tx_mac_map_fd < 0 || rxcnt_map_fd < 0) {
171 printf("bpf_object__find_map_fd_by_name failed\n");215 printf("bpf_object__find_map_fd_by_name failed\n");
172 return 1;216 return 1;
173 }217 }
@@ -199,11 +243,39 @@ int main(int argc, char **argv) · 25 unmodified lines
199 }243 }
200 dummy_prog_id = info.id;244 dummy_prog_id = info.id;
201 245
246 /* Load 2nd xdp prog on egress. */
247 if (xdp_devmap_attached) {
248 unsigned char mac_addr[6];
249
250 devmap_prog = bpf_object__find_program_by_name(obj, "xdp_redirect_map_egress");
251 if (!devmap_prog) {
252 printf("finding devmap_prog in obj file failed\n");
253 goto out;
254 }
255 devmap_prog_fd = bpf_program__fd(devmap_prog);
256 if (devmap_prog_fd < 0) {
257 printf("finding devmap_prog fd failed\n");
258 goto out;
259 }
260
261 if (get_mac_addr(ifindex_out, mac_addr) < 0) {
262 printf("get interface %d mac failed\n", ifindex_out);
263 goto out;
264 }
265
266 ret = bpf_map_update_elem(tx_mac_map_fd, &key, mac_addr, 0);
267 if (ret) {
268 perror("bpf_update_elem tx_mac_map_fd");
269 goto out;
270 }
271 }
272
202 signal(SIGINT, int_exit);273 signal(SIGINT, int_exit);
203 signal(SIGTERM, int_exit);274 signal(SIGTERM, int_exit);
204 275
205 /* populate virtual to physical port map */276 devmap_val.ifindex = ifindex_out;
206 ret = bpf_map_update_elem(tx_port_map_fd, &key, &ifindex_out, 0);277 devmap_val.bpf_prog.fd = devmap_prog_fd;
278 ret = bpf_map_update_elem(tx_port_map_fd, &key, &devmap_val, 0);
207 if (ret) {279 if (ret) {
208 perror("bpf_update_elem");280 perror("bpf_update_elem");
209 goto out;281 goto out;
tools/bpf/bpf_dbg.c+1−1
@@ -890,7 +890,7 @@ static int bpf_run_stepping(struct sock_filter *f, uint16_t bpf_len, · 889 unmodified lines
890 bool stop = false;890 bool stop = false;
891 int i = 1;891 int i = 1;
892 892
893 while (bpf_curr.Rs == false && stop == false) {893 while (!bpf_curr.Rs && !stop) {
894 bpf_safe_regs();894 bpf_safe_regs();
895 895
896 if (i++ == next)896 if (i++ == next)
tools/bpf/bpftool/Makefile+0−2
@@ -75,8 +75,6 @@ endif · 74 unmodified lines
75 75
76INSTALL ?= install76INSTALL ?= install
77RM ?= rm -f77RM ?= rm -f
78CLANG ?= clang
79LLVM_STRIP ?= llvm-strip
80 78
81FEATURE_USER = .bpftool79FEATURE_USER = .bpftool
82FEATURE_TESTS = libbfd disassembler-four-args reallocarray zlib libcap \80FEATURE_TESTS = libbfd disassembler-four-args reallocarray zlib libcap \
tools/bpf/bpftool/prog.c+4−0
@@ -368,6 +368,8 @@ static void print_prog_header_json(struct bpf_prog_info *info) · 367 unmodified lines
368 jsonw_uint_field(json_wtr, "run_time_ns", info->run_time_ns);368 jsonw_uint_field(json_wtr, "run_time_ns", info->run_time_ns);
369 jsonw_uint_field(json_wtr, "run_cnt", info->run_cnt);369 jsonw_uint_field(json_wtr, "run_cnt", info->run_cnt);
370 }370 }
371 if (info->recursion_misses)
372 jsonw_uint_field(json_wtr, "recursion_misses", info->recursion_misses);
371}373}
372 374
373static void print_prog_json(struct bpf_prog_info *info, int fd)375static void print_prog_json(struct bpf_prog_info *info, int fd)
@@ -446,6 +448,8 @@ static void print_prog_header_plain(struct bpf_prog_info *info) · 72 unmodified lines
446 if (info->run_time_ns)448 if (info->run_time_ns)
447 printf(" run_time_ns %lld run_cnt %lld",449 printf(" run_time_ns %lld run_cnt %lld",
448 info->run_time_ns, info->run_cnt);450 info->run_time_ns, info->run_cnt);
451 if (info->recursion_misses)
452 printf(" recursion_misses %lld", info->recursion_misses);
449 printf("\n");453 printf("\n");
450}454}
451 455
tools/bpf/resolve_btfids/.gitignore+1−2
@@ -1,4 +1,3 @@
1/FEATURE-DUMP.libbpf
2/bpf_helper_defs.h
3/fixdep1/fixdep
4/resolve_btfids2/resolve_btfids
3/libbpf/
tools/bpf/resolve_btfids/Makefile+22−22
@@ -2,11 +2,7 @@ · 1 unmodified line
2include ../../scripts/Makefile.include2include ../../scripts/Makefile.include
3include ../../scripts/Makefile.arch3include ../../scripts/Makefile.arch
4 4
5ifeq ($(srctree),)5srctree := $(abspath $(CURDIR)/../../../)
6srctree := $(patsubst %/,%,$(dir $(CURDIR)))
7srctree := $(patsubst %/,%,$(dir $(srctree)))
8srctree := $(patsubst %/,%,$(dir $(srctree)))
9endif
10 6
11ifeq ($(V),1)7ifeq ($(V),1)
12 Q =8 Q =
@@ -22,28 +18,29 @@ AR = $(HOSTAR) · 9 unmodified lines
22CC = $(HOSTCC)18CC = $(HOSTCC)
23LD = $(HOSTLD)19LD = $(HOSTLD)
24ARCH = $(HOSTARCH)20ARCH = $(HOSTARCH)
21RM ?= rm
25 22
26OUTPUT ?= $(srctree)/tools/bpf/resolve_btfids/23OUTPUT ?= $(srctree)/tools/bpf/resolve_btfids/
27 24
28LIBBPF_SRC := $(srctree)/tools/lib/bpf/25LIBBPF_SRC := $(srctree)/tools/lib/bpf/
29SUBCMD_SRC := $(srctree)/tools/lib/subcmd/26SUBCMD_SRC := $(srctree)/tools/lib/subcmd/
30 27
31BPFOBJ := $(OUTPUT)/libbpf.a28BPFOBJ := $(OUTPUT)/libbpf/libbpf.a
32SUBCMDOBJ := $(OUTPUT)/libsubcmd.a29SUBCMDOBJ := $(OUTPUT)/libsubcmd/libsubcmd.a
33 30
34BINARY := $(OUTPUT)/resolve_btfids31BINARY := $(OUTPUT)/resolve_btfids
35BINARY_IN := $(BINARY)-in.o32BINARY_IN := $(BINARY)-in.o
36 33
37all: $(BINARY)34all: $(BINARY)
38 35
39$(OUTPUT):36$(OUTPUT) $(OUTPUT)/libbpf $(OUTPUT)/libsubcmd:
40 $(call msg,MKDIR,,$@)37 $(call msg,MKDIR,,$@)
41 $(Q)mkdir -p $(OUTPUT)38 $(Q)mkdir -p $(@)
42 39
43$(SUBCMDOBJ): fixdep FORCE40$(SUBCMDOBJ): fixdep FORCE | $(OUTPUT)/libsubcmd
44 $(Q)$(MAKE) -C $(SUBCMD_SRC) OUTPUT=$(OUTPUT)41 $(Q)$(MAKE) -C $(SUBCMD_SRC) OUTPUT=$(abspath $(dir $@))/ $(abspath $@)
45 42
46$(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(OUTPUT)43$(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(OUTPUT)/libbpf
47 $(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) OUTPUT=$(abspath $(dir $@))/ $(abspath $@)44 $(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) OUTPUT=$(abspath $(dir $@))/ $(abspath $@)
48 45
49CFLAGS := -g \46CFLAGS := -g \
@@ -57,24 +54,27 @@ LIBS = -lelf -lz · 7 unmodified lines
57export srctree OUTPUT CFLAGS Q54export srctree OUTPUT CFLAGS Q
58include $(srctree)/tools/build/Makefile.include55include $(srctree)/tools/build/Makefile.include
59 56
60$(BINARY_IN): fixdep FORCE57$(BINARY_IN): fixdep FORCE | $(OUTPUT)
61 $(Q)$(MAKE) $(build)=resolve_btfids58 $(Q)$(MAKE) $(build)=resolve_btfids
62 59
63$(BINARY): $(BPFOBJ) $(SUBCMDOBJ) $(BINARY_IN)60$(BINARY): $(BPFOBJ) $(SUBCMDOBJ) $(BINARY_IN)
64 $(call msg,LINK,$@)61 $(call msg,LINK,$@)
65 $(Q)$(CC) $(BINARY_IN) $(LDFLAGS) -o $@ $(BPFOBJ) $(SUBCMDOBJ) $(LIBS)62 $(Q)$(CC) $(BINARY_IN) $(LDFLAGS) -o $@ $(BPFOBJ) $(SUBCMDOBJ) $(LIBS)
66 63
67libsubcmd-clean:64clean_objects := $(wildcard $(OUTPUT)/*.o \
68 $(Q)$(MAKE) -C $(SUBCMD_SRC) OUTPUT=$(OUTPUT) clean65 $(OUTPUT)/.*.o.cmd \
69 66 $(OUTPUT)/.*.o.d \
70libbpf-clean:67 $(OUTPUT)/libbpf \
71 $(Q)$(MAKE) -C $(LIBBPF_SRC) OUTPUT=$(OUTPUT) clean68 $(OUTPUT)/libsubcmd \
69 $(OUTPUT)/resolve_btfids)
72 70
73clean: libsubcmd-clean libbpf-clean fixdep-clean71ifneq ($(clean_objects),)
72clean: fixdep-clean
74 $(call msg,CLEAN,$(BINARY))73 $(call msg,CLEAN,$(BINARY))
75 $(Q)$(RM) -f $(BINARY); \74 $(Q)$(RM) -rf $(clean_objects)
76 $(RM) -rf $(if $(OUTPUT),$(OUTPUT),.)/feature; \75else
77 find $(if $(OUTPUT),$(OUTPUT),.) -name \*.o -or -name \*.o.cmd -or -name \*.o.d | xargs $(RM)76clean:
77endif
78 78
79tags:79tags:
80 $(call msg,GEN,,tags)80 $(call msg,GEN,,tags)
tools/bpf/runqslower/Makefile+0−3
@@ -3,9 +3,6 @@ include ../../scripts/Makefile.include · 2 unmodified lines
3 3
4OUTPUT ?= $(abspath .output)/4OUTPUT ?= $(abspath .output)/
5 5
6CLANG ?= clang
7LLC ?= llc
8LLVM_STRIP ?= llvm-strip
9BPFTOOL_OUTPUT := $(OUTPUT)bpftool/6BPFTOOL_OUTPUT := $(OUTPUT)bpftool/
10DEFAULT_BPFTOOL := $(BPFTOOL_OUTPUT)bpftool7DEFAULT_BPFTOOL := $(BPFTOOL_OUTPUT)bpftool
11BPFTOOL ?= $(DEFAULT_BPFTOOL)8BPFTOOL ?= $(DEFAULT_BPFTOOL)
tools/build/feature/Makefile+2−2
@@ -1,4 +1,6 @@
1# SPDX-License-Identifier: GPL-2.01# SPDX-License-Identifier: GPL-2.0
2include ../../scripts/Makefile.include
3
2FILES= \4FILES= \
3 test-all.bin \5 test-all.bin \
4 test-backtrace.bin \6 test-backtrace.bin \
@@ -76,8 +78,6 @@ FILES= \ · 71 unmodified lines
76FILES := $(addprefix $(OUTPUT),$(FILES))78FILES := $(addprefix $(OUTPUT),$(FILES))
77 79
78PKG_CONFIG ?= $(CROSS_COMPILE)pkg-config80PKG_CONFIG ?= $(CROSS_COMPILE)pkg-config
79LLVM_CONFIG ?= llvm-config
80CLANG ?= clang
81 81
82all: $(FILES)82all: $(FILES)
83 83
tools/include/linux/types.h+3−0
@@ -6,7 +6,10 @@ · 5 unmodified lines
6#include <stddef.h>6#include <stddef.h>
7#include <stdint.h>7#include <stdint.h>
8 8
9#ifndef __SANE_USERSPACE_TYPES__
9#define __SANE_USERSPACE_TYPES__ /* For PPC64, to get LL64 types */10#define __SANE_USERSPACE_TYPES__ /* For PPC64, to get LL64 types */
11#endif
12
10#include <asm/types.h>13#include <asm/types.h>
11#include <asm/posix_types.h>14#include <asm/posix_types.h>
12 15
tools/include/uapi/linux/bpf.h+97−6
@@ -1656,22 +1656,30 @@ union bpf_attr { · 1655 unmodified lines
1656 * networking traffic statistics as it provides a global socket1656 * networking traffic statistics as it provides a global socket
1657 * identifier that can be assumed unique.1657 * identifier that can be assumed unique.
1658 * Return1658 * Return
1659 * A 8-byte long non-decreasing number on success, or 0 if the1659 * A 8-byte long unique number on success, or 0 if the socket
1660 * socket field is missing inside *skb*.1660 * field is missing inside *skb*.
1661 *1661 *
1662 * u64 bpf_get_socket_cookie(struct bpf_sock_addr *ctx)1662 * u64 bpf_get_socket_cookie(struct bpf_sock_addr *ctx)
1663 * Description1663 * Description
1664 * Equivalent to bpf_get_socket_cookie() helper that accepts1664 * Equivalent to bpf_get_socket_cookie() helper that accepts
1665 * *skb*, but gets socket from **struct bpf_sock_addr** context.1665 * *skb*, but gets socket from **struct bpf_sock_addr** context.
1666 * Return1666 * Return
1667 * A 8-byte long non-decreasing number.1667 * A 8-byte long unique number.
1668 *1668 *
1669 * u64 bpf_get_socket_cookie(struct bpf_sock_ops *ctx)1669 * u64 bpf_get_socket_cookie(struct bpf_sock_ops *ctx)
1670 * Description1670 * Description
1671 * Equivalent to **bpf_get_socket_cookie**\ () helper that accepts1671 * Equivalent to **bpf_get_socket_cookie**\ () helper that accepts
1672 * *skb*, but gets socket from **struct bpf_sock_ops** context.1672 * *skb*, but gets socket from **struct bpf_sock_ops** context.
1673 * Return1673 * Return
1674 * A 8-byte long non-decreasing number.1674 * A 8-byte long unique number.
1675 *
1676 * u64 bpf_get_socket_cookie(struct sock *sk)
1677 * Description
1678 * Equivalent to **bpf_get_socket_cookie**\ () helper that accepts
1679 * *sk*, but gets socket from a BTF **struct sock**. This helper
1680 * also works for sleepable programs.
1681 * Return
1682 * A 8-byte long unique number or 0 if *sk* is NULL.
1675 *1683 *
1676 * u32 bpf_get_socket_uid(struct sk_buff *skb)1684 * u32 bpf_get_socket_uid(struct sk_buff *skb)
1677 * Return1685 * Return
@@ -2231,6 +2239,9 @@ union bpf_attr { · 553 unmodified lines
2231 * * > 0 one of **BPF_FIB_LKUP_RET_** codes explaining why the2239 * * > 0 one of **BPF_FIB_LKUP_RET_** codes explaining why the
2232 * packet is not forwarded or needs assist from full stack2240 * packet is not forwarded or needs assist from full stack
2233 *2241 *
2242 * If lookup fails with BPF_FIB_LKUP_RET_FRAG_NEEDED, then the MTU
2243 * was exceeded and output params->mtu_result contains the MTU.
2244 *
2234 * long bpf_sock_hash_update(struct bpf_sock_ops *skops, struct bpf_map *map, void *key, u64 flags)2245 * long bpf_sock_hash_update(struct bpf_sock_ops *skops, struct bpf_map *map, void *key, u64 flags)
2235 * Description2246 * Description
2236 * Add an entry to, or update a sockhash *map* referencing sockets.2247 * Add an entry to, or update a sockhash *map* referencing sockets.
@@ -3836,6 +3847,69 @@ union bpf_attr { · 1599 unmodified lines
3836 * Return3847 * Return
3837 * A pointer to a struct socket on success or NULL if the file is3848 * A pointer to a struct socket on success or NULL if the file is
3838 * not a socket.3849 * not a socket.
3850 *
3851 * long bpf_check_mtu(void *ctx, u32 ifindex, u32 *mtu_len, s32 len_diff, u64 flags)
3852 * Description
3853
3854 * Check ctx packet size against exceeding MTU of net device (based
3855 * on *ifindex*). This helper will likely be used in combination
3856 * with helpers that adjust/change the packet size.
3857 *
3858 * The argument *len_diff* can be used for querying with a planned
3859 * size change. This allows to check MTU prior to changing packet
3860 * ctx. Providing an *len_diff* adjustment that is larger than the
3861 * actual packet size (resulting in negative packet size) will in
3862 * principle not exceed the MTU, why it is not considered a
3863 * failure. Other BPF-helpers are needed for performing the
3864 * planned size change, why the responsability for catch a negative
3865 * packet size belong in those helpers.
3866 *
3867 * Specifying *ifindex* zero means the MTU check is performed
3868 * against the current net device. This is practical if this isn't
3869 * used prior to redirect.
3870 *
3871 * The Linux kernel route table can configure MTUs on a more
3872 * specific per route level, which is not provided by this helper.
3873 * For route level MTU checks use the **bpf_fib_lookup**\ ()
3874 * helper.
3875 *
3876 * *ctx* is either **struct xdp_md** for XDP programs or
3877 * **struct sk_buff** for tc cls_act programs.
3878 *
3879 * The *flags* argument can be a combination of one or more of the
3880 * following values:
3881 *
3882 * **BPF_MTU_CHK_SEGS**
3883 * This flag will only works for *ctx* **struct sk_buff**.
3884 * If packet context contains extra packet segment buffers
3885 * (often knows as GSO skb), then MTU check is harder to
3886 * check at this point, because in transmit path it is
3887 * possible for the skb packet to get re-segmented
3888 * (depending on net device features). This could still be
3889 * a MTU violation, so this flag enables performing MTU
3890 * check against segments, with a different violation
3891 * return code to tell it apart. Check cannot use len_diff.
3892 *
3893 * On return *mtu_len* pointer contains the MTU value of the net
3894 * device. Remember the net device configured MTU is the L3 size,
3895 * which is returned here and XDP and TX length operate at L2.
3896 * Helper take this into account for you, but remember when using
3897 * MTU value in your BPF-code. On input *mtu_len* must be a valid
3898 * pointer and be initialized (to zero), else verifier will reject
3899 * BPF program.
3900 *
3901 * Return
3902 * * 0 on success, and populate MTU value in *mtu_len* pointer.
3903 *
3904 * * < 0 if any input argument is invalid (*mtu_len* not updated)
3905 *
3906 * MTU violations return positive values, but also populate MTU
3907 * value in *mtu_len* pointer, as this can be needed for
3908 * implementing PMTU handing:
3909 *
3910 * * **BPF_MTU_CHK_RET_FRAG_NEEDED**
3911 * * **BPF_MTU_CHK_RET_SEGS_TOOBIG**
3912 *
3839 */3913 */
3840#define __BPF_FUNC_MAPPER(FN) \3914#define __BPF_FUNC_MAPPER(FN) \
3841 FN(unspec), \3915 FN(unspec), \
@@ -4001,6 +4075,7 @@ union bpf_attr { · 159 unmodified lines
4001 FN(ktime_get_coarse_ns), \4075 FN(ktime_get_coarse_ns), \
4002 FN(ima_inode_hash), \4076 FN(ima_inode_hash), \
4003 FN(sock_from_file), \4077 FN(sock_from_file), \
4078 FN(check_mtu), \
4004 /* */4079 /* */
4005 4080
4006/* integer value in 'imm' field of BPF_CALL instruction selects which helper4081/* integer value in 'imm' field of BPF_CALL instruction selects which helper
@@ -4501,6 +4576,7 @@ struct bpf_prog_info { · 494 unmodified lines
4501 __aligned_u64 prog_tags;4576 __aligned_u64 prog_tags;
4502 __u64 run_time_ns;4577 __u64 run_time_ns;
4503 __u64 run_cnt;4578 __u64 run_cnt;
4579 __u64 recursion_misses;
4504} __attribute__((aligned(8)));4580} __attribute__((aligned(8)));
4505 4581
4506struct bpf_map_info {4582struct bpf_map_info {
@@ -4981,9 +5057,13 @@ struct bpf_fib_lookup { · 474 unmodified lines
4981 __be16 sport;5057 __be16 sport;
4982 __be16 dport;5058 __be16 dport;
4983 5059
4984 /* total length of packet from network header - used for MTU check */5060 union { /* used for MTU check */
4985 __u16 tot_len;5061 /* input to lookup */
5062 __u16 tot_len; /* L3 length from network hdr (iph->tot_len) */
4986 5063
5064 /* output: MTU value */
5065 __u16 mtu_result;
5066 };
4987 /* input: L3 device index for lookup5067 /* input: L3 device index for lookup
4988 * output: device index from FIB lookup5068 * output: device index from FIB lookup
4989 */5069 */
@@ -5029,6 +5109,17 @@ struct bpf_redir_neigh { · 39 unmodified lines
5029 };5109 };
5030};5110};
5031 5111
5112/* bpf_check_mtu flags*/
5113enum bpf_check_mtu_flags {
5114 BPF_MTU_CHK_SEGS = (1U << 0),
5115};
5116
5117enum bpf_check_mtu_ret {
5118 BPF_MTU_CHK_RET_SUCCESS, /* check and lookup successful */
5119 BPF_MTU_CHK_RET_FRAG_NEEDED, /* fragmentation required to fwd */
5120 BPF_MTU_CHK_RET_SEGS_TOOBIG, /* GSO re-segmentation needed to fwd */
5121};
5122
5032enum bpf_task_fd_type {5123enum bpf_task_fd_type {
5033 BPF_FD_TYPE_RAW_TRACEPOINT, /* tp name */5124 BPF_FD_TYPE_RAW_TRACEPOINT, /* tp name */
5034 BPF_FD_TYPE_TRACEPOINT, /* tp name */5125 BPF_FD_TYPE_TRACEPOINT, /* tp name */
tools/include/uapi/linux/bpf_perf_event.h+1−0
@@ -13,6 +13,7 @@ · 12 unmodified lines
13struct bpf_perf_event_data {13struct bpf_perf_event_data {
14 bpf_user_pt_regs_t regs;14 bpf_user_pt_regs_t regs;
15 __u64 sample_period;15 __u64 sample_period;
16 __u64 addr;
16};17};
17 18
18#endif /* _UAPI__LINUX_BPF_PERF_EVENT_H__ */19#endif /* _UAPI__LINUX_BPF_PERF_EVENT_H__ */
tools/include/uapi/linux/tcp.h+357−0
@@ -0,0 +1,357 @@
1/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */
2/*
3 * INET An implementation of the TCP/IP protocol suite for the LINUX
4 * operating system. INET is implemented using the BSD Socket
5 * interface as the means of communication with the user level.
6 *
7 * Definitions for the TCP protocol.
8 *
9 * Version: @(#)tcp.h 1.0.2 04/28/93
10 *
11 * Author: Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 *
13 * This program is free software; you can redistribute it and/or
14 * modify it under the terms of the GNU General Public License
15 * as published by the Free Software Foundation; either version
16 * 2 of the License, or (at your option) any later version.
17 */
18#ifndef _UAPI_LINUX_TCP_H
19#define _UAPI_LINUX_TCP_H
20
21#include <linux/types.h>
22#include <asm/byteorder.h>
23#include <linux/socket.h>
24
25struct tcphdr {
26 __be16 source;
27 __be16 dest;
28 __be32 seq;
29 __be32 ack_seq;
30#if defined(__LITTLE_ENDIAN_BITFIELD)
31 __u16 res1:4,
32 doff:4,
33 fin:1,
34 syn:1,
35 rst:1,
36 psh:1,
37 ack:1,
38 urg:1,
39 ece:1,
40 cwr:1;
41#elif defined(__BIG_ENDIAN_BITFIELD)
42 __u16 doff:4,
43 res1:4,
44 cwr:1,
45 ece:1,
46 urg:1,
47 ack:1,
48 psh:1,
49 rst:1,
50 syn:1,
51 fin:1;
52#else
53#error "Adjust your <asm/byteorder.h> defines"
54#endif
55 __be16 window;
56 __sum16 check;
57 __be16 urg_ptr;
58};
59
60/*
61 * The union cast uses a gcc extension to avoid aliasing problems
62 * (union is compatible to any of its members)
63 * This means this part of the code is -fstrict-aliasing safe now.
64 */
65union tcp_word_hdr {
66 struct tcphdr hdr;
67 __be32 words[5];
68};
69
70#define tcp_flag_word(tp) ( ((union tcp_word_hdr *)(tp))->words [3])
71
72enum {
73 TCP_FLAG_CWR = __constant_cpu_to_be32(0x00800000),
74 TCP_FLAG_ECE = __constant_cpu_to_be32(0x00400000),
75 TCP_FLAG_URG = __constant_cpu_to_be32(0x00200000),
76 TCP_FLAG_ACK = __constant_cpu_to_be32(0x00100000),
77 TCP_FLAG_PSH = __constant_cpu_to_be32(0x00080000),
78 TCP_FLAG_RST = __constant_cpu_to_be32(0x00040000),
79 TCP_FLAG_SYN = __constant_cpu_to_be32(0x00020000),
80 TCP_FLAG_FIN = __constant_cpu_to_be32(0x00010000),
81 TCP_RESERVED_BITS = __constant_cpu_to_be32(0x0F000000),
82 TCP_DATA_OFFSET = __constant_cpu_to_be32(0xF0000000)
83};
84
85/*
86 * TCP general constants
87 */
88#define TCP_MSS_DEFAULT 536U /* IPv4 (RFC1122, RFC2581) */
89#define TCP_MSS_DESIRED 1220U /* IPv6 (tunneled), EDNS0 (RFC3226) */
90
91/* TCP socket options */
92#define TCP_NODELAY 1 /* Turn off Nagle's algorithm. */
93#define TCP_MAXSEG 2 /* Limit MSS */
94#define TCP_CORK 3 /* Never send partially complete segments */
95#define TCP_KEEPIDLE 4 /* Start keeplives after this period */
96#define TCP_KEEPINTVL 5 /* Interval between keepalives */
97#define TCP_KEEPCNT 6 /* Number of keepalives before death */
98#define TCP_SYNCNT 7 /* Number of SYN retransmits */
99#define TCP_LINGER2 8 /* Life time of orphaned FIN-WAIT-2 state */
100#define TCP_DEFER_ACCEPT 9 /* Wake up listener only when data arrive */
101#define TCP_WINDOW_CLAMP 10 /* Bound advertised window */
102#define TCP_INFO 11 /* Information about this connection. */
103#define TCP_QUICKACK 12 /* Block/reenable quick acks */
104#define TCP_CONGESTION 13 /* Congestion control algorithm */
105#define TCP_MD5SIG 14 /* TCP MD5 Signature (RFC2385) */
106#define TCP_THIN_LINEAR_TIMEOUTS 16 /* Use linear timeouts for thin streams*/
107#define TCP_THIN_DUPACK 17 /* Fast retrans. after 1 dupack */
108#define TCP_USER_TIMEOUT 18 /* How long for loss retry before timeout */
109#define TCP_REPAIR 19 /* TCP sock is under repair right now */
110#define TCP_REPAIR_QUEUE 20
111#define TCP_QUEUE_SEQ 21
112#define TCP_REPAIR_OPTIONS 22
113#define TCP_FASTOPEN 23 /* Enable FastOpen on listeners */
114#define TCP_TIMESTAMP 24
115#define TCP_NOTSENT_LOWAT 25 /* limit number of unsent bytes in write queue */
116#define TCP_CC_INFO 26 /* Get Congestion Control (optional) info */
117#define TCP_SAVE_SYN 27 /* Record SYN headers for new connections */
118#define TCP_SAVED_SYN 28 /* Get SYN headers recorded for connection */
119#define TCP_REPAIR_WINDOW 29 /* Get/set window parameters */
120#define TCP_FASTOPEN_CONNECT 30 /* Attempt FastOpen with connect */
121#define TCP_ULP 31 /* Attach a ULP to a TCP connection */
122#define TCP_MD5SIG_EXT 32 /* TCP MD5 Signature with extensions */
123#define TCP_FASTOPEN_KEY 33 /* Set the key for Fast Open (cookie) */
124#define TCP_FASTOPEN_NO_COOKIE 34 /* Enable TFO without a TFO cookie */
125#define TCP_ZEROCOPY_RECEIVE 35
126#define TCP_INQ 36 /* Notify bytes available to read as a cmsg on read */
127
128#define TCP_CM_INQ TCP_INQ
129
130#define TCP_TX_DELAY 37 /* delay outgoing packets by XX usec */
131
132
133#define TCP_REPAIR_ON 1
134#define TCP_REPAIR_OFF 0
135#define TCP_REPAIR_OFF_NO_WP -1 /* Turn off without window probes */
136
137struct tcp_repair_opt {
138 __u32 opt_code;
139 __u32 opt_val;
140};
141
142struct tcp_repair_window {
143 __u32 snd_wl1;
144 __u32 snd_wnd;
145 __u32 max_window;
146
147 __u32 rcv_wnd;
148 __u32 rcv_wup;
149};
150
151enum {
152 TCP_NO_QUEUE,
153 TCP_RECV_QUEUE,
154 TCP_SEND_QUEUE,
155 TCP_QUEUES_NR,
156};
157
158/* why fastopen failed from client perspective */
159enum tcp_fastopen_client_fail {
160 TFO_STATUS_UNSPEC, /* catch-all */
161 TFO_COOKIE_UNAVAILABLE, /* if not in TFO_CLIENT_NO_COOKIE mode */
162 TFO_DATA_NOT_ACKED, /* SYN-ACK did not ack SYN data */
163 TFO_SYN_RETRANSMITTED, /* SYN-ACK did not ack SYN data after timeout */
164};
165
166/* for TCP_INFO socket option */
167#define TCPI_OPT_TIMESTAMPS 1
168#define TCPI_OPT_SACK 2
169#define TCPI_OPT_WSCALE 4
170#define TCPI_OPT_ECN 8 /* ECN was negociated at TCP session init */
171#define TCPI_OPT_ECN_SEEN 16 /* we received at least one packet with ECT */
172#define TCPI_OPT_SYN_DATA 32 /* SYN-ACK acked data in SYN sent or rcvd */
173
174/*
175 * Sender's congestion state indicating normal or abnormal situations
176 * in the last round of packets sent. The state is driven by the ACK
177 * information and timer events.
178 */
179enum tcp_ca_state {
180 /*
181 * Nothing bad has been observed recently.
182 * No apparent reordering, packet loss, or ECN marks.
183 */
184 TCP_CA_Open = 0,
185#define TCPF_CA_Open (1<<TCP_CA_Open)
186 /*
187 * The sender enters disordered state when it has received DUPACKs or
188 * SACKs in the last round of packets sent. This could be due to packet
189 * loss or reordering but needs further information to confirm packets
190 * have been lost.
191 */
192 TCP_CA_Disorder = 1,
193#define TCPF_CA_Disorder (1<<TCP_CA_Disorder)
194 /*
195 * The sender enters Congestion Window Reduction (CWR) state when it
196 * has received ACKs with ECN-ECE marks, or has experienced congestion
197 * or packet discard on the sender host (e.g. qdisc).
198 */
199 TCP_CA_CWR = 2,
200#define TCPF_CA_CWR (1<<TCP_CA_CWR)
201 /*
202 * The sender is in fast recovery and retransmitting lost packets,
203 * typically triggered by ACK events.
204 */
205 TCP_CA_Recovery = 3,
206#define TCPF_CA_Recovery (1<<TCP_CA_Recovery)
207 /*
208 * The sender is in loss recovery triggered by retransmission timeout.
209 */
210 TCP_CA_Loss = 4
211#define TCPF_CA_Loss (1<<TCP_CA_Loss)
212};
213
214struct tcp_info {
215 __u8 tcpi_state;
216 __u8 tcpi_ca_state;
217 __u8 tcpi_retransmits;
218 __u8 tcpi_probes;
219 __u8 tcpi_backoff;
220 __u8 tcpi_options;
221 __u8 tcpi_snd_wscale : 4, tcpi_rcv_wscale : 4;
222 __u8 tcpi_delivery_rate_app_limited:1, tcpi_fastopen_client_fail:2;
223
224 __u32 tcpi_rto;
225 __u32 tcpi_ato;
226 __u32 tcpi_snd_mss;
227 __u32 tcpi_rcv_mss;
228
229 __u32 tcpi_unacked;
230 __u32 tcpi_sacked;
231 __u32 tcpi_lost;
232 __u32 tcpi_retrans;
233 __u32 tcpi_fackets;
234
235 /* Times. */
236 __u32 tcpi_last_data_sent;
237 __u32 tcpi_last_ack_sent; /* Not remembered, sorry. */
238 __u32 tcpi_last_data_recv;
239 __u32 tcpi_last_ack_recv;
240
241 /* Metrics. */
242 __u32 tcpi_pmtu;
243 __u32 tcpi_rcv_ssthresh;
244 __u32 tcpi_rtt;
245 __u32 tcpi_rttvar;
246 __u32 tcpi_snd_ssthresh;
247 __u32 tcpi_snd_cwnd;
248 __u32 tcpi_advmss;
249 __u32 tcpi_reordering;
250
251 __u32 tcpi_rcv_rtt;
252 __u32 tcpi_rcv_space;
253
254 __u32 tcpi_total_retrans;
255
256 __u64 tcpi_pacing_rate;
257 __u64 tcpi_max_pacing_rate;
258 __u64 tcpi_bytes_acked; /* RFC4898 tcpEStatsAppHCThruOctetsAcked */
259 __u64 tcpi_bytes_received; /* RFC4898 tcpEStatsAppHCThruOctetsReceived */
260 __u32 tcpi_segs_out; /* RFC4898 tcpEStatsPerfSegsOut */
261 __u32 tcpi_segs_in; /* RFC4898 tcpEStatsPerfSegsIn */
262
263 __u32 tcpi_notsent_bytes;
264 __u32 tcpi_min_rtt;
265 __u32 tcpi_data_segs_in; /* RFC4898 tcpEStatsDataSegsIn */
266 __u32 tcpi_data_segs_out; /* RFC4898 tcpEStatsDataSegsOut */
267
268 __u64 tcpi_delivery_rate;
269
270 __u64 tcpi_busy_time; /* Time (usec) busy sending data */
271 __u64 tcpi_rwnd_limited; /* Time (usec) limited by receive window */
272 __u64 tcpi_sndbuf_limited; /* Time (usec) limited by send buffer */
273
274 __u32 tcpi_delivered;
275 __u32 tcpi_delivered_ce;
276
277 __u64 tcpi_bytes_sent; /* RFC4898 tcpEStatsPerfHCDataOctetsOut */
278 __u64 tcpi_bytes_retrans; /* RFC4898 tcpEStatsPerfOctetsRetrans */
279 __u32 tcpi_dsack_dups; /* RFC4898 tcpEStatsStackDSACKDups */
280 __u32 tcpi_reord_seen; /* reordering events seen */
281
282 __u32 tcpi_rcv_ooopack; /* Out-of-order packets received */
283
284 __u32 tcpi_snd_wnd; /* peer's advertised receive window after
285 * scaling (bytes)
286 */
287};
288
289/* netlink attributes types for SCM_TIMESTAMPING_OPT_STATS */
290enum {
291 TCP_NLA_PAD,
292 TCP_NLA_BUSY, /* Time (usec) busy sending data */
293 TCP_NLA_RWND_LIMITED, /* Time (usec) limited by receive window */
294 TCP_NLA_SNDBUF_LIMITED, /* Time (usec) limited by send buffer */
295 TCP_NLA_DATA_SEGS_OUT, /* Data pkts sent including retransmission */
296 TCP_NLA_TOTAL_RETRANS, /* Data pkts retransmitted */
297 TCP_NLA_PACING_RATE, /* Pacing rate in bytes per second */
298 TCP_NLA_DELIVERY_RATE, /* Delivery rate in bytes per second */
299 TCP_NLA_SND_CWND, /* Sending congestion window */
300 TCP_NLA_REORDERING, /* Reordering metric */
301 TCP_NLA_MIN_RTT, /* minimum RTT */
302 TCP_NLA_RECUR_RETRANS, /* Recurring retransmits for the current pkt */
303 TCP_NLA_DELIVERY_RATE_APP_LMT, /* delivery rate application limited ? */
304 TCP_NLA_SNDQ_SIZE, /* Data (bytes) pending in send queue */
305 TCP_NLA_CA_STATE, /* ca_state of socket */
306 TCP_NLA_SND_SSTHRESH, /* Slow start size threshold */
307 TCP_NLA_DELIVERED, /* Data pkts delivered incl. out-of-order */
308 TCP_NLA_DELIVERED_CE, /* Like above but only ones w/ CE marks */
309 TCP_NLA_BYTES_SENT, /* Data bytes sent including retransmission */
310 TCP_NLA_BYTES_RETRANS, /* Data bytes retransmitted */
311 TCP_NLA_DSACK_DUPS, /* DSACK blocks received */
312 TCP_NLA_REORD_SEEN, /* reordering events seen */
313 TCP_NLA_SRTT, /* smoothed RTT in usecs */
314 TCP_NLA_TIMEOUT_REHASH, /* Timeout-triggered rehash attempts */
315 TCP_NLA_BYTES_NOTSENT, /* Bytes in write queue not yet sent */
316 TCP_NLA_EDT, /* Earliest departure time (CLOCK_MONOTONIC) */
317};
318
319/* for TCP_MD5SIG socket option */
320#define TCP_MD5SIG_MAXKEYLEN 80
321
322/* tcp_md5sig extension flags for TCP_MD5SIG_EXT */
323#define TCP_MD5SIG_FLAG_PREFIX 0x1 /* address prefix length */
324#define TCP_MD5SIG_FLAG_IFINDEX 0x2 /* ifindex set */
325
326struct tcp_md5sig {
327 struct __kernel_sockaddr_storage tcpm_addr; /* address associated */
328 __u8 tcpm_flags; /* extension flags */
329 __u8 tcpm_prefixlen; /* address prefix */
330 __u16 tcpm_keylen; /* key length */
331 int tcpm_ifindex; /* device index for scope */
332 __u8 tcpm_key[TCP_MD5SIG_MAXKEYLEN]; /* key (binary) */
333};
334
335/* INET_DIAG_MD5SIG */
336struct tcp_diag_md5sig {
337 __u8 tcpm_family;
338 __u8 tcpm_prefixlen;
339 __u16 tcpm_keylen;
340 __be32 tcpm_addr[4];
341 __u8 tcpm_key[TCP_MD5SIG_MAXKEYLEN];
342};
343
344/* setsockopt(fd, IPPROTO_TCP, TCP_ZEROCOPY_RECEIVE, ...) */
345
346#define TCP_RECEIVE_ZEROCOPY_FLAG_TLB_CLEAN_HINT 0x1
347struct tcp_zerocopy_receive {
348 __u64 address; /* in: address of mapping */
349 __u32 length; /* in/out: number of bytes to map/mapped */
350 __u32 recv_skip_hint; /* out: amount of bytes to skip */
351 __u32 inq; /* out: amount of bytes in read queue */
352 __s32 err; /* out: socket error */
353 __u64 copybuf_address; /* in: copybuf address (small reads) */
354 __s32 copybuf_len; /* in/out: copybuf bytes avail/used or error */
355 __u32 flags; /* in: flags */
356};
357#endif /* _UAPI_LINUX_TCP_H */
tools/lib/bpf/.gitignore+0−1
@@ -1,7 +1,6 @@
1# SPDX-License-Identifier: GPL-2.0-only1# SPDX-License-Identifier: GPL-2.0-only
2libbpf_version.h2libbpf_version.h
3libbpf.pc3libbpf.pc
4FEATURE-DUMP.libbpf
5libbpf.so.*4libbpf.so.*
6TAGS5TAGS
7tags6tags
tools/lib/bpf/Makefile+4−43
@@ -58,28 +58,7 @@ ifndef VERBOSE · 57 unmodified lines
58 VERBOSE = 058 VERBOSE = 0
59endif59endif
60 60
61FEATURE_USER = .libbpf
62FEATURE_TESTS = libelf zlib bpf
63FEATURE_DISPLAY = libelf zlib bpf
64
65INCLUDES = -I. -I$(srctree)/tools/include -I$(srctree)/tools/include/uapi61INCLUDES = -I. -I$(srctree)/tools/include -I$(srctree)/tools/include/uapi
66FEATURE_CHECK_CFLAGS-bpf = $(INCLUDES)
67
68check_feat := 1
69NON_CHECK_FEAT_TARGETS := clean TAGS tags cscope help
70ifdef MAKECMDGOALS
71ifeq ($(filter-out $(NON_CHECK_FEAT_TARGETS),$(MAKECMDGOALS)),)
72 check_feat := 0
73endif
74endif
75
76ifeq ($(check_feat),1)
77ifeq ($(FEATURES_DUMP),)
78include $(srctree)/tools/build/Makefile.feature
79else
80include $(FEATURES_DUMP)
81endif
82endif
83 62
84export prefix libdir src obj63export prefix libdir src obj
85 64
@@ -157,7 +136,7 @@ all: fixdep · 71 unmodified lines
157 136
158all_cmd: $(CMD_TARGETS) check137all_cmd: $(CMD_TARGETS) check
159 138
160$(BPF_IN_SHARED): force elfdep zdep bpfdep $(BPF_HELPER_DEFS)139$(BPF_IN_SHARED): force $(BPF_HELPER_DEFS)
161 @(test -f ../../include/uapi/linux/bpf.h -a -f ../../../include/uapi/linux/bpf.h && ( \140 @(test -f ../../include/uapi/linux/bpf.h -a -f ../../../include/uapi/linux/bpf.h && ( \
162 (diff -B ../../include/uapi/linux/bpf.h ../../../include/uapi/linux/bpf.h >/dev/null) || \141 (diff -B ../../include/uapi/linux/bpf.h ../../../include/uapi/linux/bpf.h >/dev/null) || \
163 echo "Warning: Kernel ABI header at 'tools/include/uapi/linux/bpf.h' differs from latest version at 'include/uapi/linux/bpf.h'" >&2 )) || true142 echo "Warning: Kernel ABI header at 'tools/include/uapi/linux/bpf.h' differs from latest version at 'include/uapi/linux/bpf.h'" >&2 )) || true
@@ -175,7 +154,7 @@ $(BPF_IN_SHARED): force elfdep zdep bpfdep $(BPF_HELPER_DEFS) · 11 unmodified lines
175 echo "Warning: Kernel ABI header at 'tools/include/uapi/linux/if_xdp.h' differs from latest version at 'include/uapi/linux/if_xdp.h'" >&2 )) || true154 echo "Warning: Kernel ABI header at 'tools/include/uapi/linux/if_xdp.h' differs from latest version at 'include/uapi/linux/if_xdp.h'" >&2 )) || true
176 $(Q)$(MAKE) $(build)=libbpf OUTPUT=$(SHARED_OBJDIR) CFLAGS="$(CFLAGS) $(SHLIB_FLAGS)"155 $(Q)$(MAKE) $(build)=libbpf OUTPUT=$(SHARED_OBJDIR) CFLAGS="$(CFLAGS) $(SHLIB_FLAGS)"
177 156
178$(BPF_IN_STATIC): force elfdep zdep bpfdep $(BPF_HELPER_DEFS)157$(BPF_IN_STATIC): force $(BPF_HELPER_DEFS)
179 $(Q)$(MAKE) $(build)=libbpf OUTPUT=$(STATIC_OBJDIR)158 $(Q)$(MAKE) $(build)=libbpf OUTPUT=$(STATIC_OBJDIR)
180 159
181$(BPF_HELPER_DEFS): $(srctree)/tools/include/uapi/linux/bpf.h160$(BPF_HELPER_DEFS): $(srctree)/tools/include/uapi/linux/bpf.h
@@ -264,34 +243,16 @@ install_pkgconfig: $(PC_FILE) · 82 unmodified lines
264 243
265install: install_lib install_pkgconfig install_headers244install: install_lib install_pkgconfig install_headers
266 245
267### Cleaning rules246clean:
268
269config-clean:
270 $(call QUIET_CLEAN, feature-detect)
271 $(Q)$(MAKE) -C $(srctree)/tools/build/feature/ clean >/dev/null
272
273clean: config-clean
274 $(call QUIET_CLEAN, libbpf) $(RM) -rf $(CMD_TARGETS) \247 $(call QUIET_CLEAN, libbpf) $(RM) -rf $(CMD_TARGETS) \
275 *~ .*.d .*.cmd LIBBPF-CFLAGS $(BPF_HELPER_DEFS) \248 *~ .*.d .*.cmd LIBBPF-CFLAGS $(BPF_HELPER_DEFS) \
276 $(SHARED_OBJDIR) $(STATIC_OBJDIR) \249 $(SHARED_OBJDIR) $(STATIC_OBJDIR) \
277 $(addprefix $(OUTPUT), \250 $(addprefix $(OUTPUT), \
278 *.o *.a *.so *.so.$(LIBBPF_MAJOR_VERSION) *.pc)251 *.o *.a *.so *.so.$(LIBBPF_MAJOR_VERSION) *.pc)
279 $(call QUIET_CLEAN, core-gen) $(RM) $(OUTPUT)FEATURE-DUMP.libbpf
280
281
282 252
283PHONY += force elfdep zdep bpfdep cscope tags253PHONY += force cscope tags
284force:254force:
285 255
286elfdep:
287 @if [ "$(feature-libelf)" != "1" ]; then echo "No libelf found"; exit 1 ; fi
288
289zdep:
290 @if [ "$(feature-zlib)" != "1" ]; then echo "No zlib found"; exit 1 ; fi
291
292bpfdep:
293 @if [ "$(feature-bpf)" != "1" ]; then echo "BPF API too old"; exit 1 ; fi
294
295cscope:256cscope:
296 ls *.c *.h > cscope.files257 ls *.c *.h > cscope.files
297 cscope -b -q -I $(srctree)/include -f cscope.out258 cscope -b -q -I $(srctree)/include -f cscope.out
tools/lib/bpf/btf.c+10−2
@@ -858,6 +858,7 @@ static struct btf *btf_parse_elf(const char *path, struct btf *base_btf, · 857 unmodified lines
858 Elf_Scn *scn = NULL;858 Elf_Scn *scn = NULL;
859 Elf *elf = NULL;859 Elf *elf = NULL;
860 GElf_Ehdr ehdr;860 GElf_Ehdr ehdr;
861 size_t shstrndx;
861 862
862 if (elf_version(EV_CURRENT) == EV_NONE) {863 if (elf_version(EV_CURRENT) == EV_NONE) {
863 pr_warn("failed to init libelf for %s\n", path);864 pr_warn("failed to init libelf for %s\n", path);
@@ -882,7 +883,14 @@ static struct btf *btf_parse_elf(const char *path, struct btf *base_btf, · 18 unmodified lines
882 pr_warn("failed to get EHDR from %s\n", path);883 pr_warn("failed to get EHDR from %s\n", path);
883 goto done;884 goto done;
884 }885 }
885 if (!elf_rawdata(elf_getscn(elf, ehdr.e_shstrndx), NULL)) {886
887 if (elf_getshdrstrndx(elf, &shstrndx)) {
888 pr_warn("failed to get section names section index for %s\n",
889 path);
890 goto done;
891 }
892
893 if (!elf_rawdata(elf_getscn(elf, shstrndx), NULL)) {
886 pr_warn("failed to get e_shstrndx from %s\n", path);894 pr_warn("failed to get e_shstrndx from %s\n", path);
887 goto done;895 goto done;
888 }896 }
@@ -897,7 +905,7 @@ static struct btf *btf_parse_elf(const char *path, struct btf *base_btf, · 8 unmodified lines
897 idx, path);905 idx, path);
898 goto done;906 goto done;
899 }907 }
900 name = elf_strptr(elf, ehdr.e_shstrndx, sh.sh_name);908 name = elf_strptr(elf, shstrndx, sh.sh_name);
901 if (!name) {909 if (!name) {
902 pr_warn("failed to get section(%d) name from %s\n",910 pr_warn("failed to get section(%d) name from %s\n",
903 idx, path);911 idx, path);
tools/lib/bpf/libbpf.c+11−11
@@ -884,24 +884,24 @@ static int bpf_map__init_kern_struct_ops(struct bpf_map *map, · 883 unmodified lines
884 if (btf_is_ptr(mtype)) {884 if (btf_is_ptr(mtype)) {
885 struct bpf_program *prog;885 struct bpf_program *prog;
886 886
887 mtype = skip_mods_and_typedefs(btf, mtype->type, &mtype_id);887 prog = st_ops->progs[i];
888 if (!prog)
889 continue;
890
888 kern_mtype = skip_mods_and_typedefs(kern_btf,891 kern_mtype = skip_mods_and_typedefs(kern_btf,
889 kern_mtype->type,892 kern_mtype->type,
890 &kern_mtype_id);893 &kern_mtype_id);
891 if (!btf_is_func_proto(mtype) ||894
892 !btf_is_func_proto(kern_mtype)) {895 /* mtype->type must be a func_proto which was
893 pr_warn("struct_ops init_kern %s: non func ptr %s is not supported\n",896 * guaranteed in bpf_object__collect_st_ops_relos(),
897 * so only check kern_mtype for func_proto here.
898 */
899 if (!btf_is_func_proto(kern_mtype)) {
900 pr_warn("struct_ops init_kern %s: kernel member %s is not a func ptr\n",
894 map->name, mname);901 map->name, mname);
895 return -ENOTSUP;902 return -ENOTSUP;
896 }903 }
897 904
898 prog = st_ops->progs[i];
899 if (!prog) {
900 pr_debug("struct_ops init_kern %s: func ptr %s is not set\n",
901 map->name, mname);
902 continue;
903 }
904
905 prog->attach_btf_id = kern_type_id;905 prog->attach_btf_id = kern_type_id;
906 prog->expected_attach_type = kern_member_idx;906 prog->expected_attach_type = kern_member_idx;
907 907
tools/lib/bpf/xsk.c+79−4
@@ -46,6 +46,11 @@ · 45 unmodified lines
46 #define PF_XDP AF_XDP46 #define PF_XDP AF_XDP
47#endif47#endif
48 48
49enum xsk_prog {
50 XSK_PROG_FALLBACK,
51 XSK_PROG_REDIRECT_FLAGS,
52};
53
49struct xsk_umem {54struct xsk_umem {
50 struct xsk_ring_prod *fill_save;55 struct xsk_ring_prod *fill_save;
51 struct xsk_ring_cons *comp_save;56 struct xsk_ring_cons *comp_save;
@@ -351,6 +356,54 @@ int xsk_umem__create_v0_0_2(struct xsk_umem **umem_ptr, void *umem_area, · 299 unmodified lines
351COMPAT_VERSION(xsk_umem__create_v0_0_2, xsk_umem__create, LIBBPF_0.0.2)356COMPAT_VERSION(xsk_umem__create_v0_0_2, xsk_umem__create, LIBBPF_0.0.2)
352DEFAULT_VERSION(xsk_umem__create_v0_0_4, xsk_umem__create, LIBBPF_0.0.4)357DEFAULT_VERSION(xsk_umem__create_v0_0_4, xsk_umem__create, LIBBPF_0.0.4)
353 358
359static enum xsk_prog get_xsk_prog(void)
360{
361 enum xsk_prog detected = XSK_PROG_FALLBACK;
362 struct bpf_load_program_attr prog_attr;
363 struct bpf_create_map_attr map_attr;
364 __u32 size_out, retval, duration;
365 char data_in = 0, data_out;
366 struct bpf_insn insns[] = {
367 BPF_LD_MAP_FD(BPF_REG_1, 0),
368 BPF_MOV64_IMM(BPF_REG_2, 0),
369 BPF_MOV64_IMM(BPF_REG_3, XDP_PASS),
370 BPF_EMIT_CALL(BPF_FUNC_redirect_map),
371 BPF_EXIT_INSN(),
372 };
373 int prog_fd, map_fd, ret;
374
375 memset(&map_attr, 0, sizeof(map_attr));
376 map_attr.map_type = BPF_MAP_TYPE_XSKMAP;
377 map_attr.key_size = sizeof(int);
378 map_attr.value_size = sizeof(int);
379 map_attr.max_entries = 1;
380
381 map_fd = bpf_create_map_xattr(&map_attr);
382 if (map_fd < 0)
383 return detected;
384
385 insns[0].imm = map_fd;
386
387 memset(&prog_attr, 0, sizeof(prog_attr));
388 prog_attr.prog_type = BPF_PROG_TYPE_XDP;
389 prog_attr.insns = insns;
390 prog_attr.insns_cnt = ARRAY_SIZE(insns);
391 prog_attr.license = "GPL";
392
393 prog_fd = bpf_load_program_xattr(&prog_attr, NULL, 0);
394 if (prog_fd < 0) {
395 close(map_fd);
396 return detected;
397 }
398
399 ret = bpf_prog_test_run(prog_fd, 0, &data_in, 1, &data_out, &size_out, &retval, &duration);
400 if (!ret && retval == XDP_PASS)
401 detected = XSK_PROG_REDIRECT_FLAGS;
402 close(prog_fd);
403 close(map_fd);
404 return detected;
405}
406
354static int xsk_load_xdp_prog(struct xsk_socket *xsk)407static int xsk_load_xdp_prog(struct xsk_socket *xsk)
355{408{
356 static const int log_buf_size = 16 * 1024;409 static const int log_buf_size = 16 * 1024;
@@ -358,7 +411,7 @@ static int xsk_load_xdp_prog(struct xsk_socket *xsk) · 1 unmodified line
358 char log_buf[log_buf_size];411 char log_buf[log_buf_size];
359 int err, prog_fd;412 int err, prog_fd;
360 413
361 /* This is the C-program:414 /* This is the fallback C-program:
362 * SEC("xdp_sock") int xdp_sock_prog(struct xdp_md *ctx)415 * SEC("xdp_sock") int xdp_sock_prog(struct xdp_md *ctx)
363 * {416 * {
364 * int ret, index = ctx->rx_queue_index;417 * int ret, index = ctx->rx_queue_index;
@@ -414,9 +467,31 @@ static int xsk_load_xdp_prog(struct xsk_socket *xsk) · 49 unmodified lines
414 /* The jumps are to this instruction */467 /* The jumps are to this instruction */
415 BPF_EXIT_INSN(),468 BPF_EXIT_INSN(),
416 };469 };
417 size_t insns_cnt = sizeof(prog) / sizeof(struct bpf_insn);
418 470
419 prog_fd = bpf_load_program(BPF_PROG_TYPE_XDP, prog, insns_cnt,471 /* This is the post-5.3 kernel C-program:
472 * SEC("xdp_sock") int xdp_sock_prog(struct xdp_md *ctx)
473 * {
474 * return bpf_redirect_map(&xsks_map, ctx->rx_queue_index, XDP_PASS);
475 * }
476 */
477 struct bpf_insn prog_redirect_flags[] = {
478 /* r2 = *(u32 *)(r1 + 16) */
479 BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 16),
480 /* r1 = xskmap[] */
481 BPF_LD_MAP_FD(BPF_REG_1, ctx->xsks_map_fd),
482 /* r3 = XDP_PASS */
483 BPF_MOV64_IMM(BPF_REG_3, 2),
484 /* call bpf_redirect_map */
485 BPF_EMIT_CALL(BPF_FUNC_redirect_map),
486 BPF_EXIT_INSN(),
487 };
488 size_t insns_cnt[] = {sizeof(prog) / sizeof(struct bpf_insn),
489 sizeof(prog_redirect_flags) / sizeof(struct bpf_insn),
490 };
491 struct bpf_insn *progs[] = {prog, prog_redirect_flags};
492 enum xsk_prog option = get_xsk_prog();
493
494 prog_fd = bpf_load_program(BPF_PROG_TYPE_XDP, progs[option], insns_cnt[option],
420 "LGPL-2.1 or BSD-2-Clause", 0, log_buf,495 "LGPL-2.1 or BSD-2-Clause", 0, log_buf,
421 log_buf_size);496 log_buf_size);
422 if (prog_fd < 0) {497 if (prog_fd < 0) {
@@ -442,7 +517,7 @@ static int xsk_get_max_queues(struct xsk_socket *xsk) · 19 unmodified lines
442 struct ifreq ifr = {};517 struct ifreq ifr = {};
443 int fd, err, ret;518 int fd, err, ret;
444 519
445 fd = socket(AF_INET, SOCK_DGRAM, 0);520 fd = socket(AF_LOCAL, SOCK_DGRAM, 0);
446 if (fd < 0)521 if (fd < 0)
447 return -errno;522 return -errno;
448 523
tools/perf/Makefile.perf+0−1
@@ -176,7 +176,6 @@ endef · 175 unmodified lines
176LD += $(EXTRA_LDFLAGS)176LD += $(EXTRA_LDFLAGS)
177 177
178PKG_CONFIG = $(CROSS_COMPILE)pkg-config178PKG_CONFIG = $(CROSS_COMPILE)pkg-config
179LLVM_CONFIG ?= llvm-config
180 179
181RM = rm -f180RM = rm -f
182LN = ln -f181LN = ln -f
tools/scripts/Makefile.include+7−0
@@ -69,6 +69,13 @@ HOSTCC ?= gcc · 68 unmodified lines
69HOSTLD ?= ld69HOSTLD ?= ld
70endif70endif
71 71
72# Some tools require Clang, LLC and/or LLVM utils
73CLANG ?= clang
74LLC ?= llc
75LLVM_CONFIG ?= llvm-config
76LLVM_OBJCOPY ?= llvm-objcopy
77LLVM_STRIP ?= llvm-strip
78
72ifeq ($(CC_NO_CLANG), 1)79ifeq ($(CC_NO_CLANG), 1)
73EXTRA_WARNINGS += -Wstrict-aliasing=380EXTRA_WARNINGS += -Wstrict-aliasing=3
74endif81endif
tools/testing/selftests/bpf/.gitignore+0−2
@@ -17,7 +17,6 @@ test_sockmap · 16 unmodified lines
17test_lirc_mode2_user17test_lirc_mode2_user
18get_cgroup_id_user18get_cgroup_id_user
19test_skb_cgroup_id_user19test_skb_cgroup_id_user
20test_socket_cookie
21test_cgroup_storage20test_cgroup_storage
22test_flow_dissector21test_flow_dissector
23flow_dissector_load22flow_dissector_load
@@ -26,7 +25,6 @@ test_tcpnotify_user · 2 unmodified lines
26test_libbpf25test_libbpf
27test_tcp_check_syncookie_user26test_tcp_check_syncookie_user
28test_sysctl27test_sysctl
29test_current_pid_tgid_new_ns
30xdping28xdping
31test_cpp29test_cpp
32*.skel.h30*.skel.h
tools/testing/selftests/bpf/Makefile+2−6
@@ -19,8 +19,6 @@ ifneq ($(wildcard $(GENHDR)),) · 18 unmodified lines
19 GENFLAGS := -DHAVE_GENHDR19 GENFLAGS := -DHAVE_GENHDR
20endif20endif
21 21
22CLANG ?= clang
23LLVM_OBJCOPY ?= llvm-objcopy
24BPF_GCC ?= $(shell command -v bpf-gcc;)22BPF_GCC ?= $(shell command -v bpf-gcc;)
25SAN_CFLAGS ?=23SAN_CFLAGS ?=
26CFLAGS += -g -rdynamic -Wall -O2 $(GENFLAGS) $(SAN_CFLAGS) \24CFLAGS += -g -rdynamic -Wall -O2 $(GENFLAGS) $(SAN_CFLAGS) \
@@ -33,11 +31,10 @@ LDLIBS += -lcap -lelf -lz -lrt -lpthread · 6 unmodified lines
33# Order correspond to 'make run_tests' order31# Order correspond to 'make run_tests' order
34TEST_GEN_PROGS = test_verifier test_tag test_maps test_lru_map test_lpm_map test_progs \32TEST_GEN_PROGS = test_verifier test_tag test_maps test_lru_map test_lpm_map test_progs \
35 test_verifier_log test_dev_cgroup \33 test_verifier_log test_dev_cgroup \
36 test_sock test_sockmap get_cgroup_id_user test_socket_cookie \34 test_sock test_sockmap get_cgroup_id_user \
37 test_cgroup_storage \35 test_cgroup_storage \
38 test_netcnt test_tcpnotify_user test_sysctl \36 test_netcnt test_tcpnotify_user test_sysctl \
39 test_progs-no_alu32 \37 test_progs-no_alu32
40 test_current_pid_tgid_new_ns
41 38
42# Also test bpf-gcc, if present39# Also test bpf-gcc, if present
43ifneq ($(BPF_GCC),)40ifneq ($(BPF_GCC),)
@@ -188,7 +185,6 @@ $(OUTPUT)/test_dev_cgroup: cgroup_helpers.c · 144 unmodified lines
188$(OUTPUT)/test_skb_cgroup_id_user: cgroup_helpers.c185$(OUTPUT)/test_skb_cgroup_id_user: cgroup_helpers.c
189$(OUTPUT)/test_sock: cgroup_helpers.c186$(OUTPUT)/test_sock: cgroup_helpers.c
190$(OUTPUT)/test_sock_addr: cgroup_helpers.c187$(OUTPUT)/test_sock_addr: cgroup_helpers.c
191$(OUTPUT)/test_socket_cookie: cgroup_helpers.c
192$(OUTPUT)/test_sockmap: cgroup_helpers.c188$(OUTPUT)/test_sockmap: cgroup_helpers.c
193$(OUTPUT)/test_tcpnotify_user: cgroup_helpers.c trace_helpers.c189$(OUTPUT)/test_tcpnotify_user: cgroup_helpers.c trace_helpers.c
194$(OUTPUT)/get_cgroup_id_user: cgroup_helpers.c190$(OUTPUT)/get_cgroup_id_user: cgroup_helpers.c
tools/testing/selftests/bpf/README.rst+24−0
@@ -6,6 +6,30 @@ General instructions on running selftests can be found in · 5 unmodified lines
6 6
7__ /Documentation/bpf/bpf_devel_QA.rst#q-how-to-run-bpf-selftests7__ /Documentation/bpf/bpf_devel_QA.rst#q-how-to-run-bpf-selftests
8 8
9=========================
10Running Selftests in a VM
11=========================
12
13It's now possible to run the selftests using ``tools/testing/selftests/bpf/vmtest.sh``.
14The script tries to ensure that the tests are run with the same environment as they
15would be run post-submit in the CI used by the Maintainers.
16
17This script downloads a suitable Kconfig and VM userspace image from the system used by
18the CI. It builds the kernel (without overwriting your existing Kconfig), recompiles the
19bpf selftests, runs them (by default ``tools/testing/selftests/bpf/test_progs``) and
20saves the resulting output (by default in ``~/.bpf_selftests``).
21
22For more information on about using the script, run:
23
24.. code-block:: console
25
26 $ tools/testing/selftests/bpf/vmtest.sh -h
27
28.. note:: The script uses pahole and clang based on host environment setting.
29 If you want to change pahole and llvm, you can change `PATH` environment
30 variable in the beginning of script.
31
32.. note:: The script currently only supports x86_64.
9 33
10Additional information about selftest failures are34Additional information about selftest failures are
11documented here.35documented here.
tools/testing/selftests/bpf/benchs/bench_ringbufs.c+1−1
@@ -319,7 +319,7 @@ static void ringbuf_custom_process_ring(struct ringbuf_custom *r) · 318 unmodified lines
319 smp_store_release(r->consumer_pos, cons_pos);319 smp_store_release(r->consumer_pos, cons_pos);
320 else320 else
321 break;321 break;
322 };322 }
323}323}
324 324
325static void *ringbuf_custom_consumer(void *input)325static void *ringbuf_custom_consumer(void *input)
tools/testing/selftests/bpf/bpf_sockopt_helpers.h+21−0
@@ -0,0 +1,21 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2
3#include <sys/socket.h>
4#include <bpf/bpf_helpers.h>
5
6int get_set_sk_priority(void *ctx)
7{
8 int prio;
9
10 /* Verify that context allows calling bpf_getsockopt and
11 * bpf_setsockopt by reading and writing back socket
12 * priority.
13 */
14
15 if (bpf_getsockopt(ctx, SOL_SOCKET, SO_PRIORITY, &prio, sizeof(prio)))
16 return 0;
17 if (bpf_setsockopt(ctx, SOL_SOCKET, SO_PRIORITY, &prio, sizeof(prio)))
18 return 0;
19
20 return 1;
21}
tools/testing/selftests/bpf/bpf_tcp_helpers.h+1−0
@@ -177,6 +177,7 @@ struct tcp_congestion_ops { · 176 unmodified lines
177 * after all the ca_state processing. (optional)177 * after all the ca_state processing. (optional)
178 */178 */
179 void (*cong_control)(struct sock *sk, const struct rate_sample *rs);179 void (*cong_control)(struct sock *sk, const struct rate_sample *rs);
180 void *owner;
180};181};
181 182
182#define min(a, b) ((a) < (b) ? (a) : (b))183#define min(a, b) ((a) < (b) ? (a) : (b))
tools/testing/selftests/bpf/bpf_testmod/bpf_testmod-events.h+6−0
@@ -28,6 +28,12 @@ TRACE_EVENT(bpf_testmod_test_read, · 27 unmodified lines
28 __entry->pid, __entry->comm, __entry->off, __entry->len)28 __entry->pid, __entry->comm, __entry->off, __entry->len)
29);29);
30 30
31/* A bare tracepoint with no event associated with it */
32DECLARE_TRACE(bpf_testmod_test_write_bare,
33 TP_PROTO(struct task_struct *task, struct bpf_testmod_test_write_ctx *ctx),
34 TP_ARGS(task, ctx)
35);
36
31#endif /* _BPF_TESTMOD_EVENTS_H */37#endif /* _BPF_TESTMOD_EVENTS_H */
32 38
33#undef TRACE_INCLUDE_PATH39#undef TRACE_INCLUDE_PATH
tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c+20−1
@@ -31,9 +31,28 @@ bpf_testmod_test_read(struct file *file, struct kobject *kobj, · 30 unmodified lines
31EXPORT_SYMBOL(bpf_testmod_test_read);31EXPORT_SYMBOL(bpf_testmod_test_read);
32ALLOW_ERROR_INJECTION(bpf_testmod_test_read, ERRNO);32ALLOW_ERROR_INJECTION(bpf_testmod_test_read, ERRNO);
33 33
34noinline ssize_t
35bpf_testmod_test_write(struct file *file, struct kobject *kobj,
36 struct bin_attribute *bin_attr,
37 char *buf, loff_t off, size_t len)
38{
39 struct bpf_testmod_test_write_ctx ctx = {
40 .buf = buf,
41 .off = off,
42 .len = len,
43 };
44
45 trace_bpf_testmod_test_write_bare(current, &ctx);
46
47 return -EIO; /* always fail */
48}
49EXPORT_SYMBOL(bpf_testmod_test_write);
50ALLOW_ERROR_INJECTION(bpf_testmod_test_write, ERRNO);
51
34static struct bin_attribute bin_attr_bpf_testmod_file __ro_after_init = {52static struct bin_attribute bin_attr_bpf_testmod_file __ro_after_init = {
35 .attr = { .name = "bpf_testmod", .mode = 0444, },53 .attr = { .name = "bpf_testmod", .mode = 0666, },
36 .read = bpf_testmod_test_read,54 .read = bpf_testmod_test_read,
55 .write = bpf_testmod_test_write,
37};56};
38 57
39static int bpf_testmod_init(void)58static int bpf_testmod_init(void)
tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.h+6−0
@@ -11,4 +11,10 @@ struct bpf_testmod_test_read_ctx { · 10 unmodified lines
11 size_t len;11 size_t len;
12};12};
13 13
14struct bpf_testmod_test_write_ctx {
15 char *buf;
16 loff_t off;
17 size_t len;
18};
19
14#endif /* _BPF_TESTMOD_H */20#endif /* _BPF_TESTMOD_H */
tools/testing/selftests/bpf/prog_tests/atomic_bounds.c+17−0
@@ -0,0 +1,17 @@
1// SPDX-License-Identifier: GPL-2.0
2
3#include <test_progs.h>
4
5#include "atomic_bounds.skel.h"
6
7void test_atomic_bounds(void)
8{
9 struct atomic_bounds *skel;
10 __u32 duration = 0;
11
12 skel = atomic_bounds__open_and_load();
13 if (CHECK(!skel, "skel_load", "couldn't load program\n"))
14 return;
15
16 atomic_bounds__destroy(skel);
17}
tools/testing/selftests/bpf/prog_tests/bind_perm.c+109−0
@@ -0,0 +1,109 @@
1// SPDX-License-Identifier: GPL-2.0
2#include <test_progs.h>
3#include "bind_perm.skel.h"
4
5#include <sys/types.h>
6#include <sys/socket.h>
7#include <sys/capability.h>
8
9static int duration;
10
11void try_bind(int family, int port, int expected_errno)
12{
13 struct sockaddr_storage addr = {};
14 struct sockaddr_in6 *sin6;
15 struct sockaddr_in *sin;
16 int fd = -1;
17
18 fd = socket(family, SOCK_STREAM, 0);
19 if (CHECK(fd < 0, "fd", "errno %d", errno))
20 goto close_socket;
21
22 if (family == AF_INET) {
23 sin = (struct sockaddr_in *)&addr;
24 sin->sin_family = family;
25 sin->sin_port = htons(port);
26 } else {
27 sin6 = (struct sockaddr_in6 *)&addr;
28 sin6->sin6_family = family;
29 sin6->sin6_port = htons(port);
30 }
31
32 errno = 0;
33 bind(fd, (struct sockaddr *)&addr, sizeof(addr));
34 ASSERT_EQ(errno, expected_errno, "bind");
35
36close_socket:
37 if (fd >= 0)
38 close(fd);
39}
40
41bool cap_net_bind_service(cap_flag_value_t flag)
42{
43 const cap_value_t cap_net_bind_service = CAP_NET_BIND_SERVICE;
44 cap_flag_value_t original_value;
45 bool was_effective = false;
46 cap_t caps;
47
48 caps = cap_get_proc();
49 if (CHECK(!caps, "cap_get_proc", "errno %d", errno))
50 goto free_caps;
51
52 if (CHECK(cap_get_flag(caps, CAP_NET_BIND_SERVICE, CAP_EFFECTIVE,
53 &original_value),
54 "cap_get_flag", "errno %d", errno))
55 goto free_caps;
56
57 was_effective = (original_value == CAP_SET);
58
59 if (CHECK(cap_set_flag(caps, CAP_EFFECTIVE, 1, &cap_net_bind_service,
60 flag),
61 "cap_set_flag", "errno %d", errno))
62 goto free_caps;
63
64 if (CHECK(cap_set_proc(caps), "cap_set_proc", "errno %d", errno))
65 goto free_caps;
66
67free_caps:
68 CHECK(cap_free(caps), "cap_free", "errno %d", errno);
69 return was_effective;
70}
71
72void test_bind_perm(void)
73{
74 bool cap_was_effective;
75 struct bind_perm *skel;
76 int cgroup_fd;
77
78 cgroup_fd = test__join_cgroup("/bind_perm");
79 if (CHECK(cgroup_fd < 0, "cg-join", "errno %d", errno))
80 return;
81
82 skel = bind_perm__open_and_load();
83 if (!ASSERT_OK_PTR(skel, "skel"))
84 goto close_cgroup_fd;
85
86 skel->links.bind_v4_prog = bpf_program__attach_cgroup(skel->progs.bind_v4_prog, cgroup_fd);
87 if (!ASSERT_OK_PTR(skel, "bind_v4_prog"))
88 goto close_skeleton;
89
90 skel->links.bind_v6_prog = bpf_program__attach_cgroup(skel->progs.bind_v6_prog, cgroup_fd);
91 if (!ASSERT_OK_PTR(skel, "bind_v6_prog"))
92 goto close_skeleton;
93
94 cap_was_effective = cap_net_bind_service(CAP_CLEAR);
95
96 try_bind(AF_INET, 110, EACCES);
97 try_bind(AF_INET6, 110, EACCES);
98
99 try_bind(AF_INET, 111, 0);
100 try_bind(AF_INET6, 111, 0);
101
102 if (cap_was_effective)
103 cap_net_bind_service(CAP_SET);
104
105close_skeleton:
106 bind_perm__destroy(skel);
107close_cgroup_fd:
108 close(cgroup_fd);
109}
tools/testing/selftests/bpf/prog_tests/bpf_iter.c+108−10
@@ -7,6 +7,7 @@ · 6 unmodified lines
7#include "bpf_iter_task.skel.h"7#include "bpf_iter_task.skel.h"
8#include "bpf_iter_task_stack.skel.h"8#include "bpf_iter_task_stack.skel.h"
9#include "bpf_iter_task_file.skel.h"9#include "bpf_iter_task_file.skel.h"
10#include "bpf_iter_task_vma.skel.h"
10#include "bpf_iter_task_btf.skel.h"11#include "bpf_iter_task_btf.skel.h"
11#include "bpf_iter_tcp4.skel.h"12#include "bpf_iter_tcp4.skel.h"
12#include "bpf_iter_tcp6.skel.h"13#include "bpf_iter_tcp6.skel.h"
@@ -64,6 +65,22 @@ free_link: · 51 unmodified lines
64 bpf_link__destroy(link);65 bpf_link__destroy(link);
65}66}
66 67
68static int read_fd_into_buffer(int fd, char *buf, int size)
69{
70 int bufleft = size;
71 int len;
72
73 do {
74 len = read(fd, buf, bufleft);
75 if (len > 0) {
76 buf += len;
77 bufleft -= len;
78 }
79 } while (len > 0);
80
81 return len < 0 ? len : size - bufleft;
82}
83
67static void test_ipv6_route(void)84static void test_ipv6_route(void)
68{85{
69 struct bpf_iter_ipv6_route *skel;86 struct bpf_iter_ipv6_route *skel;
@@ -177,7 +194,7 @@ static int do_btf_read(struct bpf_iter_task_btf *skel) · 107 unmodified lines
177{194{
178 struct bpf_program *prog = skel->progs.dump_task_struct;195 struct bpf_program *prog = skel->progs.dump_task_struct;
179 struct bpf_iter_task_btf__bss *bss = skel->bss;196 struct bpf_iter_task_btf__bss *bss = skel->bss;
180 int iter_fd = -1, len = 0, bufleft = TASKBUFSZ;197 int iter_fd = -1, err;
181 struct bpf_link *link;198 struct bpf_link *link;
182 char *buf = taskbuf;199 char *buf = taskbuf;
183 int ret = 0;200 int ret = 0;
@@ -190,14 +207,7 @@ static int do_btf_read(struct bpf_iter_task_btf *skel) · 6 unmodified lines
190 if (CHECK(iter_fd < 0, "create_iter", "create_iter failed\n"))207 if (CHECK(iter_fd < 0, "create_iter", "create_iter failed\n"))
191 goto free_link;208 goto free_link;
192 209
193 do {210 err = read_fd_into_buffer(iter_fd, buf, TASKBUFSZ);
194 len = read(iter_fd, buf, bufleft);
195 if (len > 0) {
196 buf += len;
197 bufleft -= len;
198 }
199 } while (len > 0);
200
201 if (bss->skip) {211 if (bss->skip) {
202 printf("%s:SKIP:no __builtin_btf_type_id\n", __func__);212 printf("%s:SKIP:no __builtin_btf_type_id\n", __func__);
203 ret = 1;213 ret = 1;
@@ -205,7 +215,7 @@ static int do_btf_read(struct bpf_iter_task_btf *skel) · 1 unmodified line
205 goto free_link;215 goto free_link;
206 }216 }
207 217
208 if (CHECK(len < 0, "read", "read failed: %s\n", strerror(errno)))218 if (CHECK(err < 0, "read", "read failed: %s\n", strerror(errno)))
209 goto free_link;219 goto free_link;
210 220
211 CHECK(strstr(taskbuf, "(struct task_struct)") == NULL,221 CHECK(strstr(taskbuf, "(struct task_struct)") == NULL,
@@ -1133,6 +1143,92 @@ static void test_buf_neg_offset(void) · 921 unmodified lines
1133 bpf_iter_test_kern6__destroy(skel);1143 bpf_iter_test_kern6__destroy(skel);
1134}1144}
1135 1145
1146#define CMP_BUFFER_SIZE 1024
1147static char task_vma_output[CMP_BUFFER_SIZE];
1148static char proc_maps_output[CMP_BUFFER_SIZE];
1149
1150/* remove \0 and \t from str, and only keep the first line */
1151static void str_strip_first_line(char *str)
1152{
1153 char *dst = str, *src = str;
1154
1155 do {
1156 if (*src == ' ' || *src == '\t')
1157 src++;
1158 else
1159 *(dst++) = *(src++);
1160
1161 } while (*src != '\0' && *src != '\n');
1162
1163 *dst = '\0';
1164}
1165
1166#define min(a, b) ((a) < (b) ? (a) : (b))
1167
1168static void test_task_vma(void)
1169{
1170 int err, iter_fd = -1, proc_maps_fd = -1;
1171 struct bpf_iter_task_vma *skel;
1172 int len, read_size = 4;
1173 char maps_path[64];
1174
1175 skel = bpf_iter_task_vma__open();
1176 if (CHECK(!skel, "bpf_iter_task_vma__open", "skeleton open failed\n"))
1177 return;
1178
1179 skel->bss->pid = getpid();
1180
1181 err = bpf_iter_task_vma__load(skel);
1182 if (CHECK(err, "bpf_iter_task_vma__load", "skeleton load failed\n"))
1183 goto out;
1184
1185 skel->links.proc_maps = bpf_program__attach_iter(
1186 skel->progs.proc_maps, NULL);
1187
1188 if (CHECK(IS_ERR(skel->links.proc_maps), "bpf_program__attach_iter",
1189 "attach iterator failed\n")) {
1190 skel->links.proc_maps = NULL;
1191 goto out;
1192 }
1193
1194 iter_fd = bpf_iter_create(bpf_link__fd(skel->links.proc_maps));
1195 if (CHECK(iter_fd < 0, "create_iter", "create_iter failed\n"))
1196 goto out;
1197
1198 /* Read CMP_BUFFER_SIZE (1kB) from bpf_iter. Read in small chunks
1199 * to trigger seq_file corner cases. The expected output is much
1200 * longer than 1kB, so the while loop will terminate.
1201 */
1202 len = 0;
1203 while (len < CMP_BUFFER_SIZE) {
1204 err = read_fd_into_buffer(iter_fd, task_vma_output + len,
1205 min(read_size, CMP_BUFFER_SIZE - len));
1206 if (CHECK(err < 0, "read_iter_fd", "read_iter_fd failed\n"))
1207 goto out;
1208 len += err;
1209 }
1210
1211 /* read CMP_BUFFER_SIZE (1kB) from /proc/pid/maps */
1212 snprintf(maps_path, 64, "/proc/%u/maps", skel->bss->pid);
1213 proc_maps_fd = open(maps_path, O_RDONLY);
1214 if (CHECK(proc_maps_fd < 0, "open_proc_maps", "open_proc_maps failed\n"))
1215 goto out;
1216 err = read_fd_into_buffer(proc_maps_fd, proc_maps_output, CMP_BUFFER_SIZE);
1217 if (CHECK(err < 0, "read_prog_maps_fd", "read_prog_maps_fd failed\n"))
1218 goto out;
1219
1220 /* strip and compare the first line of the two files */
1221 str_strip_first_line(task_vma_output);
1222 str_strip_first_line(proc_maps_output);
1223
1224 CHECK(strcmp(task_vma_output, proc_maps_output), "compare_output",
1225 "found mismatch\n");
1226out:
1227 close(proc_maps_fd);
1228 close(iter_fd);
1229 bpf_iter_task_vma__destroy(skel);
1230}
1231
1136void test_bpf_iter(void)1232void test_bpf_iter(void)
1137{1233{
1138 if (test__start_subtest("btf_id_or_null"))1234 if (test__start_subtest("btf_id_or_null"))
@@ -1149,6 +1245,8 @@ void test_bpf_iter(void) · 10 unmodified lines
1149 test_task_stack();1245 test_task_stack();
1150 if (test__start_subtest("task_file"))1246 if (test__start_subtest("task_file"))
1151 test_task_file();1247 test_task_file();
1248 if (test__start_subtest("task_vma"))
1249 test_task_vma();
1152 if (test__start_subtest("task_btf"))1250 if (test__start_subtest("task_btf"))
1153 test_task_btf();1251 test_task_btf();
1154 if (test__start_subtest("tcp4"))1252 if (test__start_subtest("tcp4"))
tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c+1−0
@@ -2,6 +2,7 @@ · 1 unmodified line
2/* Copyright (c) 2019 Facebook */2/* Copyright (c) 2019 Facebook */
3 3
4#include <linux/err.h>4#include <linux/err.h>
5#include <netinet/tcp.h>
5#include <test_progs.h>6#include <test_progs.h>
6#include "bpf_dctcp.skel.h"7#include "bpf_dctcp.skel.h"
7#include "bpf_cubic.skel.h"8#include "bpf_cubic.skel.h"
tools/testing/selftests/bpf/prog_tests/btf.c+23−2
@@ -914,7 +914,7 @@ static struct btf_raw_test raw_tests[] = { · 913 unmodified lines
914 .err_str = "Member exceeds struct_size",914 .err_str = "Member exceeds struct_size",
915},915},
916 916
917/* Test member exeeds the size of struct917/* Test member exceeds the size of struct
918 *918 *
919 * struct A {919 * struct A {
920 * int m;920 * int m;
@@ -948,7 +948,7 @@ static struct btf_raw_test raw_tests[] = { · 27 unmodified lines
948 .err_str = "Member exceeds struct_size",948 .err_str = "Member exceeds struct_size",
949},949},
950 950
951/* Test member exeeds the size of struct951/* Test member exceeds the size of struct
952 *952 *
953 * struct A {953 * struct A {
954 * int m;954 * int m;
@@ -3509,6 +3509,27 @@ static struct btf_raw_test raw_tests[] = { · 2554 unmodified lines
3509 .value_type_id = 3 /* arr_t */,3509 .value_type_id = 3 /* arr_t */,
3510 .max_entries = 4,3510 .max_entries = 4,
3511},3511},
3512/*
3513 * elf .rodata section size 4 and btf .rodata section vlen 0.
3514 */
3515{
3516 .descr = "datasec: vlen == 0",
3517 .raw_types = {
3518 /* int */
3519 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
3520 /* .rodata section */
3521 BTF_TYPE_ENC(NAME_NTH(1), BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 0), 4),
3522 /* [2] */
3523 BTF_END_RAW,
3524 },
3525 BTF_STR_SEC("\0.rodata"),
3526 .map_type = BPF_MAP_TYPE_ARRAY,
3527 .key_size = sizeof(int),
3528 .value_size = sizeof(int),
3529 .key_type_id = 1,
3530 .value_type_id = 1,
3531 .max_entries = 1,
3532},
3512 3533
3513}; /* struct btf_raw_test raw_tests[] */3534}; /* struct btf_raw_test raw_tests[] */
3514 3535
tools/testing/selftests/bpf/prog_tests/check_mtu.c+216−0
@@ -0,0 +1,216 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2020 Jesper Dangaard Brouer */
3
4#include <linux/if_link.h> /* before test_progs.h, avoid bpf_util.h redefines */
5#include <test_progs.h>
6#include "test_check_mtu.skel.h"
7#include "network_helpers.h"
8
9#include <stdlib.h>
10#include <inttypes.h>
11
12#define IFINDEX_LO 1
13
14static __u32 duration; /* Hint: needed for CHECK macro */
15
16static int read_mtu_device_lo(void)
17{
18 const char *filename = "/sys/class/net/lo/mtu";
19 char buf[11] = {};
20 int value, n, fd;
21
22 fd = open(filename, 0, O_RDONLY);
23 if (fd == -1)
24 return -1;
25
26 n = read(fd, buf, sizeof(buf));
27 close(fd);
28
29 if (n == -1)
30 return -2;
31
32 value = strtoimax(buf, NULL, 10);
33 if (errno == ERANGE)
34 return -3;
35
36 return value;
37}
38
39static void test_check_mtu_xdp_attach(void)
40{
41 struct bpf_link_info link_info;
42 __u32 link_info_len = sizeof(link_info);
43 struct test_check_mtu *skel;
44 struct bpf_program *prog;
45 struct bpf_link *link;
46 int err = 0;
47 int fd;
48
49 skel = test_check_mtu__open_and_load();
50 if (CHECK(!skel, "open and load skel", "failed"))
51 return; /* Exit if e.g. helper unknown to kernel */
52
53 prog = skel->progs.xdp_use_helper_basic;
54
55 link = bpf_program__attach_xdp(prog, IFINDEX_LO);
56 if (CHECK(IS_ERR(link), "link_attach", "failed: %ld\n", PTR_ERR(link)))
57 goto out;
58 skel->links.xdp_use_helper_basic = link;
59
60 memset(&link_info, 0, sizeof(link_info));
61 fd = bpf_link__fd(link);
62 err = bpf_obj_get_info_by_fd(fd, &link_info, &link_info_len);
63 if (CHECK(err, "link_info", "failed: %d\n", err))
64 goto out;
65
66 CHECK(link_info.type != BPF_LINK_TYPE_XDP, "link_type",
67 "got %u != exp %u\n", link_info.type, BPF_LINK_TYPE_XDP);
68 CHECK(link_info.xdp.ifindex != IFINDEX_LO, "link_ifindex",
69 "got %u != exp %u\n", link_info.xdp.ifindex, IFINDEX_LO);
70
71 err = bpf_link__detach(link);
72 CHECK(err, "link_detach", "failed %d\n", err);
73
74out:
75 test_check_mtu__destroy(skel);
76}
77
78static void test_check_mtu_run_xdp(struct test_check_mtu *skel,
79 struct bpf_program *prog,
80 __u32 mtu_expect)
81{
82 const char *prog_name = bpf_program__name(prog);
83 int retval_expect = XDP_PASS;
84 __u32 mtu_result = 0;
85 char buf[256] = {};
86 int err;
87 struct bpf_prog_test_run_attr tattr = {
88 .repeat = 1,
89 .data_in = &pkt_v4,
90 .data_size_in = sizeof(pkt_v4),
91 .data_out = buf,
92 .data_size_out = sizeof(buf),
93 .prog_fd = bpf_program__fd(prog),
94 };
95
96 err = bpf_prog_test_run_xattr(&tattr);
97 CHECK_ATTR(err != 0, "bpf_prog_test_run",
98 "prog_name:%s (err %d errno %d retval %d)\n",
99 prog_name, err, errno, tattr.retval);
100
101 CHECK(tattr.retval != retval_expect, "retval",
102 "progname:%s unexpected retval=%d expected=%d\n",
103 prog_name, tattr.retval, retval_expect);
104
105 /* Extract MTU that BPF-prog got */
106 mtu_result = skel->bss->global_bpf_mtu_xdp;
107 ASSERT_EQ(mtu_result, mtu_expect, "MTU-compare-user");
108}
109
110
111static void test_check_mtu_xdp(__u32 mtu, __u32 ifindex)
112{
113 struct test_check_mtu *skel;
114 int err;
115
116 skel = test_check_mtu__open();
117 if (CHECK(!skel, "skel_open", "failed"))
118 return;
119
120 /* Update "constants" in BPF-prog *BEFORE* libbpf load */
121 skel->rodata->GLOBAL_USER_MTU = mtu;
122 skel->rodata->GLOBAL_USER_IFINDEX = ifindex;
123
124 err = test_check_mtu__load(skel);
125 if (CHECK(err, "skel_load", "failed: %d\n", err))
126 goto cleanup;
127
128 test_check_mtu_run_xdp(skel, skel->progs.xdp_use_helper, mtu);
129 test_check_mtu_run_xdp(skel, skel->progs.xdp_exceed_mtu, mtu);
130 test_check_mtu_run_xdp(skel, skel->progs.xdp_minus_delta, mtu);
131
132cleanup:
133 test_check_mtu__destroy(skel);
134}
135
136static void test_check_mtu_run_tc(struct test_check_mtu *skel,
137 struct bpf_program *prog,
138 __u32 mtu_expect)
139{
140 const char *prog_name = bpf_program__name(prog);
141 int retval_expect = BPF_OK;
142 __u32 mtu_result = 0;
143 char buf[256] = {};
144 int err;
145 struct bpf_prog_test_run_attr tattr = {
146 .repeat = 1,
147 .data_in = &pkt_v4,
148 .data_size_in = sizeof(pkt_v4),
149 .data_out = buf,
150 .data_size_out = sizeof(buf),
151 .prog_fd = bpf_program__fd(prog),
152 };
153
154 err = bpf_prog_test_run_xattr(&tattr);
155 CHECK_ATTR(err != 0, "bpf_prog_test_run",
156 "prog_name:%s (err %d errno %d retval %d)\n",
157 prog_name, err, errno, tattr.retval);
158
159 CHECK(tattr.retval != retval_expect, "retval",
160 "progname:%s unexpected retval=%d expected=%d\n",
161 prog_name, tattr.retval, retval_expect);
162
163 /* Extract MTU that BPF-prog got */
164 mtu_result = skel->bss->global_bpf_mtu_tc;
165 ASSERT_EQ(mtu_result, mtu_expect, "MTU-compare-user");
166}
167
168
169static void test_check_mtu_tc(__u32 mtu, __u32 ifindex)
170{
171 struct test_check_mtu *skel;
172 int err;
173
174 skel = test_check_mtu__open();
175 if (CHECK(!skel, "skel_open", "failed"))
176 return;
177
178 /* Update "constants" in BPF-prog *BEFORE* libbpf load */
179 skel->rodata->GLOBAL_USER_MTU = mtu;
180 skel->rodata->GLOBAL_USER_IFINDEX = ifindex;
181
182 err = test_check_mtu__load(skel);
183 if (CHECK(err, "skel_load", "failed: %d\n", err))
184 goto cleanup;
185
186 test_check_mtu_run_tc(skel, skel->progs.tc_use_helper, mtu);
187 test_check_mtu_run_tc(skel, skel->progs.tc_exceed_mtu, mtu);
188 test_check_mtu_run_tc(skel, skel->progs.tc_exceed_mtu_da, mtu);
189 test_check_mtu_run_tc(skel, skel->progs.tc_minus_delta, mtu);
190cleanup:
191 test_check_mtu__destroy(skel);
192}
193
194void test_check_mtu(void)
195{
196 __u32 mtu_lo;
197
198 if (test__start_subtest("bpf_check_mtu XDP-attach"))
199 test_check_mtu_xdp_attach();
200
201 mtu_lo = read_mtu_device_lo();
202 if (CHECK(mtu_lo < 0, "reading MTU value", "failed (err:%d)", mtu_lo))
203 return;
204
205 if (test__start_subtest("bpf_check_mtu XDP-run"))
206 test_check_mtu_xdp(mtu_lo, 0);
207
208 if (test__start_subtest("bpf_check_mtu XDP-run ifindex-lookup"))
209 test_check_mtu_xdp(mtu_lo, IFINDEX_LO);
210
211 if (test__start_subtest("bpf_check_mtu TC-run"))
212 test_check_mtu_tc(mtu_lo, 0);
213
214 if (test__start_subtest("bpf_check_mtu TC-run ifindex-lookup"))
215 test_check_mtu_tc(mtu_lo, IFINDEX_LO);
216}
tools/testing/selftests/bpf/prog_tests/cls_redirect.c+1−0
@@ -7,6 +7,7 @@ · 6 unmodified lines
7#include <string.h>7#include <string.h>
8 8
9#include <linux/pkt_cls.h>9#include <linux/pkt_cls.h>
10#include <netinet/tcp.h>
10 11
11#include <test_progs.h>12#include <test_progs.h>
12 13
tools/testing/selftests/bpf/prog_tests/fexit_stress.c+2−2
@@ -2,8 +2,8 @@ · 1 unmodified line
2/* Copyright (c) 2019 Facebook */2/* Copyright (c) 2019 Facebook */
3#include <test_progs.h>3#include <test_progs.h>
4 4
5/* x86-64 fits 55 JITed and 43 interpreted progs into half page */5/* that's kernel internal BPF_MAX_TRAMP_PROGS define */
6#define CNT 406#define CNT 38
7 7
8void test_fexit_stress(void)8void test_fexit_stress(void)
9{9{
tools/testing/selftests/bpf/prog_tests/global_func_args.c+60−0
@@ -0,0 +1,60 @@
1// SPDX-License-Identifier: GPL-2.0
2#include "test_progs.h"
3#include "network_helpers.h"
4
5static __u32 duration;
6
7static void test_global_func_args0(struct bpf_object *obj)
8{
9 int err, i, map_fd, actual_value;
10 const char *map_name = "values";
11
12 map_fd = bpf_find_map(__func__, obj, map_name);
13 if (CHECK(map_fd < 0, "bpf_find_map", "cannot find BPF map %s: %s\n",
14 map_name, strerror(errno)))
15 return;
16
17 struct {
18 const char *descr;
19 int expected_value;
20 } tests[] = {
21 {"passing NULL pointer", 0},
22 {"returning value", 1},
23 {"reading local variable", 100 },
24 {"writing local variable", 101 },
25 {"reading global variable", 42 },
26 {"writing global variable", 43 },
27 {"writing to pointer-to-pointer", 1 },
28 };
29
30 for (i = 0; i < ARRAY_SIZE(tests); ++i) {
31 const int expected_value = tests[i].expected_value;
32
33 err = bpf_map_lookup_elem(map_fd, &i, &actual_value);
34
35 CHECK(err || actual_value != expected_value, tests[i].descr,
36 "err %d result %d expected %d\n", err, actual_value, expected_value);
37 }
38}
39
40void test_global_func_args(void)
41{
42 const char *file = "./test_global_func_args.o";
43 __u32 retval;
44 struct bpf_object *obj;
45 int err, prog_fd;
46
47 err = bpf_prog_load(file, BPF_PROG_TYPE_CGROUP_SKB, &obj, &prog_fd);
48 if (CHECK(err, "load program", "error %d loading %s\n", err, file))
49 return;
50
51 err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4),
52 NULL, NULL, &retval, &duration);
53 CHECK(err || retval, "pass global func args run",
54 "err %d errno %d retval %d duration %d\n",
55 err, errno, retval, duration);
56
57 test_global_func_args0(obj);
58
59 bpf_object__close(obj);
60}
tools/testing/selftests/bpf/prog_tests/module_attach.c+27−0
@@ -21,9 +21,34 @@ static int trigger_module_test_read(int read_sz) · 20 unmodified lines
21 return 0;21 return 0;
22}22}
23 23
24static int trigger_module_test_write(int write_sz)
25{
26 int fd, err;
27 char *buf = malloc(write_sz);
28
29 if (!buf)
30 return -ENOMEM;
31
32 memset(buf, 'a', write_sz);
33 buf[write_sz-1] = '\0';
34
35 fd = open("/sys/kernel/bpf_testmod", O_WRONLY);
36 err = -errno;
37 if (CHECK(fd < 0, "testmod_file_open", "failed: %d\n", err)) {
38 free(buf);
39 return err;
40 }
41
42 write(fd, buf, write_sz);
43 close(fd);
44 free(buf);
45 return 0;
46}
47
24void test_module_attach(void)48void test_module_attach(void)
25{49{
26 const int READ_SZ = 456;50 const int READ_SZ = 456;
51 const int WRITE_SZ = 457;
27 struct test_module_attach* skel;52 struct test_module_attach* skel;
28 struct test_module_attach__bss *bss;53 struct test_module_attach__bss *bss;
29 int err;54 int err;
@@ -48,8 +73,10 @@ void test_module_attach(void) · 18 unmodified lines
48 73
49 /* trigger tracepoint */74 /* trigger tracepoint */
50 ASSERT_OK(trigger_module_test_read(READ_SZ), "trigger_read");75 ASSERT_OK(trigger_module_test_read(READ_SZ), "trigger_read");
76 ASSERT_OK(trigger_module_test_write(WRITE_SZ), "trigger_write");
51 77
52 ASSERT_EQ(bss->raw_tp_read_sz, READ_SZ, "raw_tp");78 ASSERT_EQ(bss->raw_tp_read_sz, READ_SZ, "raw_tp");
79 ASSERT_EQ(bss->raw_tp_bare_write_sz, WRITE_SZ, "raw_tp_bare");
53 ASSERT_EQ(bss->tp_btf_read_sz, READ_SZ, "tp_btf");80 ASSERT_EQ(bss->tp_btf_read_sz, READ_SZ, "tp_btf");
54 ASSERT_EQ(bss->fentry_read_sz, READ_SZ, "fentry");81 ASSERT_EQ(bss->fentry_read_sz, READ_SZ, "fentry");
55 ASSERT_EQ(bss->fentry_manual_read_sz, READ_SZ, "fentry_manual");82 ASSERT_EQ(bss->fentry_manual_read_sz, READ_SZ, "fentry_manual");
tools/testing/selftests/bpf/prog_tests/ns_current_pid_tgid.c+60−58
@@ -1,85 +1,87 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2020 Carlos Neira cneirabustos@gmail.com */2/* Copyright (c) 2020 Carlos Neira cneirabustos@gmail.com */
3
4#define _GNU_SOURCE
3#include <test_progs.h>5#include <test_progs.h>
6#include "test_ns_current_pid_tgid.skel.h"
4#include <sys/stat.h>7#include <sys/stat.h>
5#include <sys/types.h>8#include <sys/types.h>
6#include <unistd.h>9#include <unistd.h>
7#include <sys/syscall.h>10#include <sys/syscall.h>
11#include <sched.h>
12#include <sys/wait.h>
13#include <sys/mount.h>
14#include <sys/fcntl.h>
8 15
9struct bss {16#define STACK_SIZE (1024 * 1024)
10 __u64 dev;17static char child_stack[STACK_SIZE];
11 __u64 ino;
12 __u64 pid_tgid;
13 __u64 user_pid_tgid;
14};
15 18
16void test_ns_current_pid_tgid(void)19static int test_current_pid_tgid(void *args)
17{20{
18 const char *probe_name = "raw_tracepoint/sys_enter";21 struct test_ns_current_pid_tgid__bss *bss;
19 const char *file = "test_ns_current_pid_tgid.o";22 struct test_ns_current_pid_tgid *skel;
20 int err, key = 0, duration = 0;23 int err = -1, duration = 0;
21 struct bpf_link *link = NULL;24 pid_t tgid, pid;
22 struct bpf_program *prog;
23 struct bpf_map *bss_map;
24 struct bpf_object *obj;
25 struct bss bss;
26 struct stat st;25 struct stat st;
27 __u64 id;
28
29 obj = bpf_object__open_file(file, NULL);
30 if (CHECK(IS_ERR(obj), "obj_open", "err %ld\n", PTR_ERR(obj)))
31 return;
32 26
33 err = bpf_object__load(obj);27 skel = test_ns_current_pid_tgid__open_and_load();
34 if (CHECK(err, "obj_load", "err %d errno %d\n", err, errno))28 if (CHECK(!skel, "skel_open_load", "failed to load skeleton\n"))
35 goto cleanup;29 goto cleanup;
36 30
37 bss_map = bpf_object__find_map_by_name(obj, "test_ns_.bss");31 pid = syscall(SYS_gettid);
38 if (CHECK(!bss_map, "find_bss_map", "failed\n"))32 tgid = getpid();
33
34 err = stat("/proc/self/ns/pid", &st);
35 if (CHECK(err, "stat", "failed /proc/self/ns/pid: %d\n", err))
39 goto cleanup;36 goto cleanup;
40 37
41 prog = bpf_object__find_program_by_title(obj, probe_name);38 bss = skel->bss;
42 if (CHECK(!prog, "find_prog", "prog '%s' not found\n",39 bss->dev = st.st_dev;
43 probe_name))40 bss->ino = st.st_ino;
41 bss->user_pid = 0;
42 bss->user_tgid = 0;
43
44 err = test_ns_current_pid_tgid__attach(skel);
45 if (CHECK(err, "skel_attach", "skeleton attach failed: %d\n", err))
44 goto cleanup;46 goto cleanup;
45 47
46 memset(&bss, 0, sizeof(bss));48 /* trigger tracepoint */
47 pid_t tid = syscall(SYS_gettid);49 usleep(1);
48 pid_t pid = getpid();50 ASSERT_EQ(bss->user_pid, pid, "pid");
51 ASSERT_EQ(bss->user_tgid, tgid, "tgid");
52 err = 0;
49 53
50 id = (__u64) tid << 32 | pid;54cleanup:
51 bss.user_pid_tgid = id;55 test_ns_current_pid_tgid__destroy(skel);
52 56
53 if (CHECK_FAIL(stat("/proc/self/ns/pid", &st))) {57 return err;
54 perror("Failed to stat /proc/self/ns/pid");58}
55 goto cleanup;
56 }
57 59
58 bss.dev = st.st_dev;60static void test_ns_current_pid_tgid_new_ns(void)
59 bss.ino = st.st_ino;61{
62 int wstatus, duration = 0;
63 pid_t cpid;
60 64
61 err = bpf_map_update_elem(bpf_map__fd(bss_map), &key, &bss, 0);65 /* Create a process in a new namespace, this process
62 if (CHECK(err, "setting_bss", "failed to set bss : %d\n", err))66 * will be the init process of this new namespace hence will be pid 1.
63 goto cleanup;67 */
68 cpid = clone(test_current_pid_tgid, child_stack + STACK_SIZE,
69 CLONE_NEWPID | SIGCHLD, NULL);
64 70
65 link = bpf_program__attach_raw_tracepoint(prog, "sys_enter");71 if (CHECK(cpid == -1, "clone", strerror(errno)))
66 if (CHECK(IS_ERR(link), "attach_raw_tp", "err %ld\n",72 return;
67 PTR_ERR(link))) {
68 link = NULL;
69 goto cleanup;
70 }
71 73
72 /* trigger some syscalls */74 if (CHECK(waitpid(cpid, &wstatus, 0) == -1, "waitpid", strerror(errno)))
73 usleep(1);75 return;
74 76
75 err = bpf_map_lookup_elem(bpf_map__fd(bss_map), &key, &bss);77 if (CHECK(WEXITSTATUS(wstatus) != 0, "newns_pidtgid", "failed"))
76 if (CHECK(err, "set_bss", "failed to get bss : %d\n", err))78 return;
77 goto cleanup;79}
78 80
79 if (CHECK(id != bss.pid_tgid, "Compare user pid/tgid vs. bpf pid/tgid",81void test_ns_current_pid_tgid(void)
80 "User pid/tgid %llu BPF pid/tgid %llu\n", id, bss.pid_tgid))82{
81 goto cleanup;83 if (test__start_subtest("ns_current_pid_tgid_root_ns"))
82cleanup:84 test_current_pid_tgid(NULL);
83 bpf_link__destroy(link);85 if (test__start_subtest("ns_current_pid_tgid_new_ns"))
84 bpf_object__close(obj);86 test_ns_current_pid_tgid_new_ns();
85}87}
tools/testing/selftests/bpf/prog_tests/recursion.c+41−0
@@ -0,0 +1,41 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2021 Facebook */
3#include <test_progs.h>
4#include "recursion.skel.h"
5
6void test_recursion(void)
7{
8 struct bpf_prog_info prog_info = {};
9 __u32 prog_info_len = sizeof(prog_info);
10 struct recursion *skel;
11 int key = 0;
12 int err;
13
14 skel = recursion__open_and_load();
15 if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))
16 return;
17
18 err = recursion__attach(skel);
19 if (!ASSERT_OK(err, "skel_attach"))
20 goto out;
21
22 ASSERT_EQ(skel->bss->pass1, 0, "pass1 == 0");
23 bpf_map_lookup_elem(bpf_map__fd(skel->maps.hash1), &key, 0);
24 ASSERT_EQ(skel->bss->pass1, 1, "pass1 == 1");
25 bpf_map_lookup_elem(bpf_map__fd(skel->maps.hash1), &key, 0);
26 ASSERT_EQ(skel->bss->pass1, 2, "pass1 == 2");
27
28 ASSERT_EQ(skel->bss->pass2, 0, "pass2 == 0");
29 bpf_map_lookup_elem(bpf_map__fd(skel->maps.hash2), &key, 0);
30 ASSERT_EQ(skel->bss->pass2, 1, "pass2 == 1");
31 bpf_map_lookup_elem(bpf_map__fd(skel->maps.hash2), &key, 0);
32 ASSERT_EQ(skel->bss->pass2, 2, "pass2 == 2");
33
34 err = bpf_obj_get_info_by_fd(bpf_program__fd(skel->progs.on_lookup),
35 &prog_info, &prog_info_len);
36 if (!ASSERT_OK(err, "get_prog_info"))
37 goto out;
38 ASSERT_EQ(prog_info.recursion_misses, 2, "recursion_misses");
39out:
40 recursion__destroy(skel);
41}
tools/testing/selftests/bpf/prog_tests/socket_cookie.c+76−0
@@ -0,0 +1,76 @@
1// SPDX-License-Identifier: GPL-2.0
2// Copyright (c) 2020 Google LLC.
3// Copyright (c) 2018 Facebook
4
5#include <test_progs.h>
6#include "socket_cookie_prog.skel.h"
7#include "network_helpers.h"
8
9static int duration;
10
11struct socket_cookie {
12 __u64 cookie_key;
13 __u32 cookie_value;
14};
15
16void test_socket_cookie(void)
17{
18 int server_fd = 0, client_fd = 0, cgroup_fd = 0, err = 0;
19 socklen_t addr_len = sizeof(struct sockaddr_in6);
20 struct socket_cookie_prog *skel;
21 __u32 cookie_expected_value;
22 struct sockaddr_in6 addr;
23 struct socket_cookie val;
24
25 skel = socket_cookie_prog__open_and_load();
26 if (!ASSERT_OK_PTR(skel, "skel_open"))
27 return;
28
29 cgroup_fd = test__join_cgroup("/socket_cookie");
30 if (CHECK(cgroup_fd < 0, "join_cgroup", "cgroup creation failed\n"))
31 goto out;
32
33 skel->links.set_cookie = bpf_program__attach_cgroup(
34 skel->progs.set_cookie, cgroup_fd);
35 if (!ASSERT_OK_PTR(skel->links.set_cookie, "prog_attach"))
36 goto close_cgroup_fd;
37
38 skel->links.update_cookie_sockops = bpf_program__attach_cgroup(
39 skel->progs.update_cookie_sockops, cgroup_fd);
40 if (!ASSERT_OK_PTR(skel->links.update_cookie_sockops, "prog_attach"))
41 goto close_cgroup_fd;
42
43 skel->links.update_cookie_tracing = bpf_program__attach(
44 skel->progs.update_cookie_tracing);
45 if (!ASSERT_OK_PTR(skel->links.update_cookie_tracing, "prog_attach"))
46 goto close_cgroup_fd;
47
48 server_fd = start_server(AF_INET6, SOCK_STREAM, "::1", 0, 0);
49 if (CHECK(server_fd < 0, "start_server", "errno %d\n", errno))
50 goto close_cgroup_fd;
51
52 client_fd = connect_to_fd(server_fd, 0);
53 if (CHECK(client_fd < 0, "connect_to_fd", "errno %d\n", errno))
54 goto close_server_fd;
55
56 err = bpf_map_lookup_elem(bpf_map__fd(skel->maps.socket_cookies),
57 &client_fd, &val);
58 if (!ASSERT_OK(err, "map_lookup(socket_cookies)"))
59 goto close_client_fd;
60
61 err = getsockname(client_fd, (struct sockaddr *)&addr, &addr_len);
62 if (!ASSERT_OK(err, "getsockname"))
63 goto close_client_fd;
64
65 cookie_expected_value = (ntohs(addr.sin6_port) << 8) | 0xFF;
66 ASSERT_EQ(val.cookie_value, cookie_expected_value, "cookie_value");
67
68close_client_fd:
69 close(client_fd);
70close_server_fd:
71 close(server_fd);
72close_cgroup_fd:
73 close(cgroup_fd);
74out:
75 socket_cookie_prog__destroy(skel);
76}
tools/testing/selftests/bpf/prog_tests/sockmap_basic.c+1−0
@@ -1,6 +1,7 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2// Copyright (c) 2020 Cloudflare2// Copyright (c) 2020 Cloudflare
3#include <error.h>3#include <error.h>
4#include <netinet/tcp.h>
4 5
5#include "test_progs.h"6#include "test_progs.h"
6#include "test_skmsg_load_helpers.skel.h"7#include "test_skmsg_load_helpers.skel.h"
tools/testing/selftests/bpf/prog_tests/sockopt_sk.c+28−0
@@ -2,6 +2,12 @@ · 1 unmodified line
2#include <test_progs.h>2#include <test_progs.h>
3#include "cgroup_helpers.h"3#include "cgroup_helpers.h"
4 4
5#include <linux/tcp.h>
6
7#ifndef SOL_TCP
8#define SOL_TCP IPPROTO_TCP
9#endif
10
5#define SOL_CUSTOM 0xdeadbeef11#define SOL_CUSTOM 0xdeadbeef
6 12
7static int getsetsockopt(void)13static int getsetsockopt(void)
@@ -11,6 +17,7 @@ static int getsetsockopt(void) · 3 unmodified lines
11 char u8[4];17 char u8[4];
12 __u32 u32;18 __u32 u32;
13 char cc[16]; /* TCP_CA_NAME_MAX */19 char cc[16]; /* TCP_CA_NAME_MAX */
20 struct tcp_zerocopy_receive zc;
14 } buf = {};21 } buf = {};
15 socklen_t optlen;22 socklen_t optlen;
16 char *big_buf = NULL;23 char *big_buf = NULL;
@@ -154,6 +161,27 @@ static int getsetsockopt(void) · 137 unmodified lines
154 goto err;161 goto err;
155 }162 }
156 163
164 /* TCP_ZEROCOPY_RECEIVE triggers */
165 memset(&buf, 0, sizeof(buf));
166 optlen = sizeof(buf.zc);
167 err = getsockopt(fd, SOL_TCP, TCP_ZEROCOPY_RECEIVE, &buf, &optlen);
168 if (err) {
169 log_err("Unexpected getsockopt(TCP_ZEROCOPY_RECEIVE) err=%d errno=%d",
170 err, errno);
171 goto err;
172 }
173
174 memset(&buf, 0, sizeof(buf));
175 buf.zc.address = 12345; /* rejected by BPF */
176 optlen = sizeof(buf.zc);
177 errno = 0;
178 err = getsockopt(fd, SOL_TCP, TCP_ZEROCOPY_RECEIVE, &buf, &optlen);
179 if (errno != EPERM) {
180 log_err("Unexpected getsockopt(TCP_ZEROCOPY_RECEIVE) err=%d errno=%d",
181 err, errno);
182 goto err;
183 }
184
157 free(big_buf);185 free(big_buf);
158 close(fd);186 close(fd);
159 return 0;187 return 0;
tools/testing/selftests/bpf/prog_tests/stack_var_off.c+35−0
@@ -0,0 +1,35 @@
1// SPDX-License-Identifier: GPL-2.0
2#include <test_progs.h>
3#include "test_stack_var_off.skel.h"
4
5/* Test read and writes to the stack performed with offsets that are not
6 * statically known.
7 */
8void test_stack_var_off(void)
9{
10 int duration = 0;
11 struct test_stack_var_off *skel;
12
13 skel = test_stack_var_off__open_and_load();
14 if (CHECK(!skel, "skel_open", "failed to open skeleton\n"))
15 return;
16
17 /* Give pid to bpf prog so it doesn't trigger for anyone else. */
18 skel->bss->test_pid = getpid();
19 /* Initialize the probe's input. */
20 skel->bss->input[0] = 2;
21 skel->bss->input[1] = 42; /* This will be returned in probe_res. */
22
23 if (!ASSERT_OK(test_stack_var_off__attach(skel), "skel_attach"))
24 goto cleanup;
25
26 /* Trigger probe. */
27 usleep(1);
28
29 if (CHECK(skel->bss->probe_res != 42, "check_probe_res",
30 "wrong probe res: %d\n", skel->bss->probe_res))
31 goto cleanup;
32
33cleanup:
34 test_stack_var_off__destroy(skel);
35}
tools/testing/selftests/bpf/prog_tests/test_global_funcs.c+8−0
@@ -61,6 +61,14 @@ void test_test_global_funcs(void) · 60 unmodified lines
61 { "test_global_func6.o" , "modified ctx ptr R2" },61 { "test_global_func6.o" , "modified ctx ptr R2" },
62 { "test_global_func7.o" , "foo() doesn't return scalar" },62 { "test_global_func7.o" , "foo() doesn't return scalar" },
63 { "test_global_func8.o" },63 { "test_global_func8.o" },
64 { "test_global_func9.o" },
65 { "test_global_func10.o", "invalid indirect read from stack" },
66 { "test_global_func11.o", "Caller passes invalid args into func#1" },
67 { "test_global_func12.o", "invalid mem access 'mem_or_null'" },
68 { "test_global_func13.o", "Caller passes invalid args into func#1" },
69 { "test_global_func14.o", "reference type('FWD S') size cannot be determined" },
70 { "test_global_func15.o", "At program exit the register R0 has value" },
71 { "test_global_func16.o", "invalid indirect read from stack" },
64 };72 };
65 libbpf_print_fn_t old_print_fn = NULL;73 libbpf_print_fn_t old_print_fn = NULL;
66 int err, i, duration = 0;74 int err, i, duration = 0;
tools/testing/selftests/bpf/prog_tests/test_ima.c+18−5
@@ -9,6 +9,7 @@ · 8 unmodified lines
9#include <unistd.h>9#include <unistd.h>
10#include <sys/wait.h>10#include <sys/wait.h>
11#include <test_progs.h>11#include <test_progs.h>
12#include <linux/ring_buffer.h>
12 13
13#include "ima.skel.h"14#include "ima.skel.h"
14 15
@@ -31,9 +32,18 @@ static int run_measured_process(const char *measured_dir, u32 *monitored_pid) · 16 unmodified lines
31 return -EINVAL;32 return -EINVAL;
32}33}
33 34
35static u64 ima_hash_from_bpf;
36
37static int process_sample(void *ctx, void *data, size_t len)
38{
39 ima_hash_from_bpf = *((u64 *)data);
40 return 0;
41}
42
34void test_test_ima(void)43void test_test_ima(void)
35{44{
36 char measured_dir_template[] = "/tmp/ima_measuredXXXXXX";45 char measured_dir_template[] = "/tmp/ima_measuredXXXXXX";
46 struct ring_buffer *ringbuf;
37 const char *measured_dir;47 const char *measured_dir;
38 char cmd[256];48 char cmd[256];
39 49
@@ -44,6 +54,11 @@ void test_test_ima(void) · 4 unmodified lines
44 if (CHECK(!skel, "skel_load", "skeleton failed\n"))54 if (CHECK(!skel, "skel_load", "skeleton failed\n"))
45 goto close_prog;55 goto close_prog;
46 56
57 ringbuf = ring_buffer__new(bpf_map__fd(skel->maps.ringbuf),
58 process_sample, NULL, NULL);
59 if (!ASSERT_OK_PTR(ringbuf, "ringbuf"))
60 goto close_prog;
61
47 err = ima__attach(skel);62 err = ima__attach(skel);
48 if (CHECK(err, "attach", "attach failed: %d\n", err))63 if (CHECK(err, "attach", "attach failed: %d\n", err))
49 goto close_prog;64 goto close_prog;
@@ -60,11 +75,9 @@ void test_test_ima(void) · 10 unmodified lines
60 if (CHECK(err, "run_measured_process", "err = %d\n", err))75 if (CHECK(err, "run_measured_process", "err = %d\n", err))
61 goto close_clean;76 goto close_clean;
62 77
63 CHECK(skel->data->ima_hash_ret < 0, "ima_hash_ret",78 err = ring_buffer__consume(ringbuf);
64 "ima_hash_ret = %ld\n", skel->data->ima_hash_ret);79 ASSERT_EQ(err, 1, "num_samples_or_err");
65 80 ASSERT_NEQ(ima_hash_from_bpf, 0, "ima_hash");
66 CHECK(skel->bss->ima_hash == 0, "ima_hash",
67 "ima_hash = %lu\n", skel->bss->ima_hash);
68 81
69close_clean:82close_clean:
70 snprintf(cmd, sizeof(cmd), "./ima_setup.sh cleanup %s", measured_dir);83 snprintf(cmd, sizeof(cmd), "./ima_setup.sh cleanup %s", measured_dir);
tools/testing/selftests/bpf/prog_tests/test_local_storage.c+1−1
@@ -113,7 +113,7 @@ static bool check_syscall_operations(int map_fd, int obj_fd) · 112 unmodified lines
113 113
114void test_test_local_storage(void)114void test_test_local_storage(void)
115{115{
116 char tmp_dir_path[64] = "/tmp/local_storageXXXXXX";116 char tmp_dir_path[] = "/tmp/local_storageXXXXXX";
117 int err, serv_sk = -1, task_fd = -1, rm_fd = -1;117 int err, serv_sk = -1, task_fd = -1, rm_fd = -1;
118 struct local_storage *skel = NULL;118 struct local_storage *skel = NULL;
119 char tmp_exec_path[64];119 char tmp_exec_path[64];
tools/testing/selftests/bpf/prog_tests/trampoline_count.c+2−2
@@ -4,7 +4,7 @@ · 3 unmodified lines
4#include <sys/prctl.h>4#include <sys/prctl.h>
5#include <test_progs.h>5#include <test_progs.h>
6 6
7#define MAX_TRAMP_PROGS 407#define MAX_TRAMP_PROGS 38
8 8
9struct inst {9struct inst {
10 struct bpf_object *obj;10 struct bpf_object *obj;
@@ -52,7 +52,7 @@ void test_trampoline_count(void) · 41 unmodified lines
52 struct bpf_link *link;52 struct bpf_link *link;
53 char comm[16] = {};53 char comm[16] = {};
54 54
55 /* attach 'allowed' 40 trampoline programs */55 /* attach 'allowed' trampoline programs */
56 for (i = 0; i < MAX_TRAMP_PROGS; i++) {56 for (i = 0; i < MAX_TRAMP_PROGS; i++) {
57 obj = bpf_object__open_file(object, NULL);57 obj = bpf_object__open_file(object, NULL);
58 if (CHECK(IS_ERR(obj), "obj_open_file", "err %ld\n", PTR_ERR(obj))) {58 if (CHECK(IS_ERR(obj), "obj_open_file", "err %ld\n", PTR_ERR(obj))) {
tools/testing/selftests/bpf/progs/atomic_bounds.c+24−0
@@ -0,0 +1,24 @@
1// SPDX-License-Identifier: GPL-2.0
2#include <linux/bpf.h>
3#include <bpf/bpf_helpers.h>
4#include <bpf/bpf_tracing.h>
5#include <stdbool.h>
6
7#ifdef ENABLE_ATOMICS_TESTS
8bool skip_tests __attribute((__section__(".data"))) = false;
9#else
10bool skip_tests = true;
11#endif
12
13SEC("fentry/bpf_fentry_test1")
14int BPF_PROG(sub, int x)
15{
16#ifdef ENABLE_ATOMICS_TESTS
17 int a = 0;
18 int b = __sync_fetch_and_add(&a, 1);
19 /* b is certainly 0 here. Can the verifier tell? */
20 while (b)
21 continue;
22#endif
23 return 0;
24}
tools/testing/selftests/bpf/progs/bind_perm.c+45−0
@@ -0,0 +1,45 @@
1// SPDX-License-Identifier: GPL-2.0
2
3#include <linux/stddef.h>
4#include <linux/bpf.h>
5#include <sys/types.h>
6#include <sys/socket.h>
7#include <bpf/bpf_helpers.h>
8#include <bpf/bpf_endian.h>
9
10static __always_inline int bind_prog(struct bpf_sock_addr *ctx, int family)
11{
12 struct bpf_sock *sk;
13
14 sk = ctx->sk;
15 if (!sk)
16 return 0;
17
18 if (sk->family != family)
19 return 0;
20
21 if (ctx->type != SOCK_STREAM)
22 return 0;
23
24 /* Return 1 OR'ed with the first bit set to indicate
25 * that CAP_NET_BIND_SERVICE should be bypassed.
26 */
27 if (ctx->user_port == bpf_htons(111))
28 return (1 | 2);
29
30 return 1;
31}
32
33SEC("cgroup/bind4")
34int bind_v4_prog(struct bpf_sock_addr *ctx)
35{
36 return bind_prog(ctx, AF_INET);
37}
38
39SEC("cgroup/bind6")
40int bind_v6_prog(struct bpf_sock_addr *ctx)
41{
42 return bind_prog(ctx, AF_INET6);
43}
44
45char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/bpf_iter.h+8−0
@@ -7,6 +7,7 @@ · 6 unmodified lines
7#define bpf_iter__netlink bpf_iter__netlink___not_used7#define bpf_iter__netlink bpf_iter__netlink___not_used
8#define bpf_iter__task bpf_iter__task___not_used8#define bpf_iter__task bpf_iter__task___not_used
9#define bpf_iter__task_file bpf_iter__task_file___not_used9#define bpf_iter__task_file bpf_iter__task_file___not_used
10#define bpf_iter__task_vma bpf_iter__task_vma___not_used
10#define bpf_iter__tcp bpf_iter__tcp___not_used11#define bpf_iter__tcp bpf_iter__tcp___not_used
11#define tcp6_sock tcp6_sock___not_used12#define tcp6_sock tcp6_sock___not_used
12#define bpf_iter__udp bpf_iter__udp___not_used13#define bpf_iter__udp bpf_iter__udp___not_used
@@ -26,6 +27,7 @@ · 13 unmodified lines
26#undef bpf_iter__netlink27#undef bpf_iter__netlink
27#undef bpf_iter__task28#undef bpf_iter__task
28#undef bpf_iter__task_file29#undef bpf_iter__task_file
30#undef bpf_iter__task_vma
29#undef bpf_iter__tcp31#undef bpf_iter__tcp
30#undef tcp6_sock32#undef tcp6_sock
31#undef bpf_iter__udp33#undef bpf_iter__udp
@@ -67,6 +69,12 @@ struct bpf_iter__task_file { · 35 unmodified lines
67 struct file *file;69 struct file *file;
68} __attribute__((preserve_access_index));70} __attribute__((preserve_access_index));
69 71
72struct bpf_iter__task_vma {
73 struct bpf_iter_meta *meta;
74 struct task_struct *task;
75 struct vm_area_struct *vma;
76} __attribute__((preserve_access_index));
77
70struct bpf_iter__bpf_map {78struct bpf_iter__bpf_map {
71 struct bpf_iter_meta *meta;79 struct bpf_iter_meta *meta;
72 struct bpf_map *map;80 struct bpf_map *map;
tools/testing/selftests/bpf/progs/bpf_iter_task_vma.c+58−0
@@ -0,0 +1,58 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2020 Facebook */
3#include "bpf_iter.h"
4#include <bpf/bpf_helpers.h>
5#include <bpf/bpf_tracing.h>
6
7char _license[] SEC("license") = "GPL";
8
9/* Copied from mm.h */
10#define VM_READ 0x00000001
11#define VM_WRITE 0x00000002
12#define VM_EXEC 0x00000004
13#define VM_MAYSHARE 0x00000080
14
15/* Copied from kdev_t.h */
16#define MINORBITS 20
17#define MINORMASK ((1U << MINORBITS) - 1)
18#define MAJOR(dev) ((unsigned int) ((dev) >> MINORBITS))
19#define MINOR(dev) ((unsigned int) ((dev) & MINORMASK))
20
21#define D_PATH_BUF_SIZE 1024
22char d_path_buf[D_PATH_BUF_SIZE] = {};
23__u32 pid = 0;
24
25SEC("iter/task_vma") int proc_maps(struct bpf_iter__task_vma *ctx)
26{
27 struct vm_area_struct *vma = ctx->vma;
28 struct seq_file *seq = ctx->meta->seq;
29 struct task_struct *task = ctx->task;
30 struct file *file;
31 char perm_str[] = "----";
32
33 if (task == (void *)0 || vma == (void *)0)
34 return 0;
35
36 file = vma->vm_file;
37 if (task->tgid != pid)
38 return 0;
39 perm_str[0] = (vma->vm_flags & VM_READ) ? 'r' : '-';
40 perm_str[1] = (vma->vm_flags & VM_WRITE) ? 'w' : '-';
41 perm_str[2] = (vma->vm_flags & VM_EXEC) ? 'x' : '-';
42 perm_str[3] = (vma->vm_flags & VM_MAYSHARE) ? 's' : 'p';
43 BPF_SEQ_PRINTF(seq, "%08llx-%08llx %s ", vma->vm_start, vma->vm_end, perm_str);
44
45 if (file) {
46 __u32 dev = file->f_inode->i_sb->s_dev;
47
48 bpf_d_path(&file->f_path, d_path_buf, D_PATH_BUF_SIZE);
49
50 BPF_SEQ_PRINTF(seq, "%08llx ", vma->vm_pgoff << 12);
51 BPF_SEQ_PRINTF(seq, "%02x:%02x %u", MAJOR(dev), MINOR(dev),
52 file->f_inode->i_ino);
53 BPF_SEQ_PRINTF(seq, "\t%s\n", d_path_buf);
54 } else {
55 BPF_SEQ_PRINTF(seq, "%08llx 00:00 0\n", 0ULL);
56 }
57 return 0;
58}
tools/testing/selftests/bpf/progs/connect_force_port4.c+8−0
@@ -10,6 +10,8 @@ · 9 unmodified lines
10#include <bpf/bpf_helpers.h>10#include <bpf/bpf_helpers.h>
11#include <bpf/bpf_endian.h>11#include <bpf/bpf_endian.h>
12 12
13#include <bpf_sockopt_helpers.h>
14
13char _license[] SEC("license") = "GPL";15char _license[] SEC("license") = "GPL";
14int _version SEC("version") = 1;16int _version SEC("version") = 1;
15 17
@@ -58,6 +60,9 @@ int connect4(struct bpf_sock_addr *ctx) · 42 unmodified lines
58SEC("cgroup/getsockname4")60SEC("cgroup/getsockname4")
59int getsockname4(struct bpf_sock_addr *ctx)61int getsockname4(struct bpf_sock_addr *ctx)
60{62{
63 if (!get_set_sk_priority(ctx))
64 return 1;
65
61 /* Expose local server as 1.2.3.4:60000 to client. */66 /* Expose local server as 1.2.3.4:60000 to client. */
62 if (ctx->user_port == bpf_htons(60123)) {67 if (ctx->user_port == bpf_htons(60123)) {
63 ctx->user_ip4 = bpf_htonl(0x01020304);68 ctx->user_ip4 = bpf_htonl(0x01020304);
@@ -71,6 +76,9 @@ int getpeername4(struct bpf_sock_addr *ctx) · 7 unmodified lines
71{76{
72 struct svc_addr *orig;77 struct svc_addr *orig;
73 78
79 if (!get_set_sk_priority(ctx))
80 return 1;
81
74 /* Expose service 1.2.3.4:60000 as peer instead of backend. */82 /* Expose service 1.2.3.4:60000 as peer instead of backend. */
75 if (ctx->user_port == bpf_htons(60123)) {83 if (ctx->user_port == bpf_htons(60123)) {
76 orig = bpf_sk_storage_get(&service_mapping, ctx->sk, 0, 0);84 orig = bpf_sk_storage_get(&service_mapping, ctx->sk, 0, 0);
tools/testing/selftests/bpf/progs/connect_force_port6.c+8−0
@@ -9,6 +9,8 @@ · 8 unmodified lines
9#include <bpf/bpf_helpers.h>9#include <bpf/bpf_helpers.h>
10#include <bpf/bpf_endian.h>10#include <bpf/bpf_endian.h>
11 11
12#include <bpf_sockopt_helpers.h>
13
12char _license[] SEC("license") = "GPL";14char _license[] SEC("license") = "GPL";
13int _version SEC("version") = 1;15int _version SEC("version") = 1;
14 16
@@ -63,6 +65,9 @@ int connect6(struct bpf_sock_addr *ctx) · 48 unmodified lines
63SEC("cgroup/getsockname6")65SEC("cgroup/getsockname6")
64int getsockname6(struct bpf_sock_addr *ctx)66int getsockname6(struct bpf_sock_addr *ctx)
65{67{
68 if (!get_set_sk_priority(ctx))
69 return 1;
70
66 /* Expose local server as [fc00::1]:60000 to client. */71 /* Expose local server as [fc00::1]:60000 to client. */
67 if (ctx->user_port == bpf_htons(60124)) {72 if (ctx->user_port == bpf_htons(60124)) {
68 ctx->user_ip6[0] = bpf_htonl(0xfc000000);73 ctx->user_ip6[0] = bpf_htonl(0xfc000000);
@@ -79,6 +84,9 @@ int getpeername6(struct bpf_sock_addr *ctx) · 10 unmodified lines
79{84{
80 struct svc_addr *orig;85 struct svc_addr *orig;
81 86
87 if (!get_set_sk_priority(ctx))
88 return 1;
89
82 /* Expose service [fc00::1]:60000 as peer instead of backend. */90 /* Expose service [fc00::1]:60000 as peer instead of backend. */
83 if (ctx->user_port == bpf_htons(60124)) {91 if (ctx->user_port == bpf_htons(60124)) {
84 orig = bpf_sk_storage_get(&service_mapping, ctx->sk, 0, 0);92 orig = bpf_sk_storage_get(&service_mapping, ctx->sk, 0, 0);
tools/testing/selftests/bpf/progs/ima.c+25−8
@@ -9,20 +9,37 @@ · 8 unmodified lines
9#include <bpf/bpf_helpers.h>9#include <bpf/bpf_helpers.h>
10#include <bpf/bpf_tracing.h>10#include <bpf/bpf_tracing.h>
11 11
12long ima_hash_ret = -1;
13u64 ima_hash = 0;
14u32 monitored_pid = 0;12u32 monitored_pid = 0;
15 13
14struct {
15 __uint(type, BPF_MAP_TYPE_RINGBUF);
16 __uint(max_entries, 1 << 12);
17} ringbuf SEC(".maps");
18
16char _license[] SEC("license") = "GPL";19char _license[] SEC("license") = "GPL";
17 20
18SEC("lsm.s/bprm_committed_creds")21SEC("lsm.s/bprm_committed_creds")
19int BPF_PROG(ima, struct linux_binprm *bprm)22void BPF_PROG(ima, struct linux_binprm *bprm)
20{23{
21 u32 pid = bpf_get_current_pid_tgid() >> 32;24 u64 ima_hash = 0;
25 u64 *sample;
26 int ret;
27 u32 pid;
28
29 pid = bpf_get_current_pid_tgid() >> 32;
30 if (pid == monitored_pid) {
31 ret = bpf_ima_inode_hash(bprm->file->f_inode, &ima_hash,
32 sizeof(ima_hash));
33 if (ret < 0 || ima_hash == 0)
34 return;
35
36 sample = bpf_ringbuf_reserve(&ringbuf, sizeof(u64), 0);
37 if (!sample)
38 return;
22 39
23 if (pid == monitored_pid)40 *sample = ima_hash;
24 ima_hash_ret = bpf_ima_inode_hash(bprm->file->f_inode,41 bpf_ringbuf_submit(sample, 0);
25 &ima_hash, sizeof(ima_hash));42 }
26 43
27 return 0;44 return;
28}45}
tools/testing/selftests/bpf/progs/lsm.c+69−0
@@ -30,6 +30,53 @@ struct { · 29 unmodified lines
30 __type(value, __u64);30 __type(value, __u64);
31} lru_hash SEC(".maps");31} lru_hash SEC(".maps");
32 32
33struct {
34 __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
35 __uint(max_entries, 1);
36 __type(key, __u32);
37 __type(value, __u64);
38} percpu_array SEC(".maps");
39
40struct {
41 __uint(type, BPF_MAP_TYPE_PERCPU_HASH);
42 __uint(max_entries, 1);
43 __type(key, __u32);
44 __type(value, __u64);
45} percpu_hash SEC(".maps");
46
47struct {
48 __uint(type, BPF_MAP_TYPE_LRU_PERCPU_HASH);
49 __uint(max_entries, 1);
50 __type(key, __u32);
51 __type(value, __u64);
52} lru_percpu_hash SEC(".maps");
53
54struct inner_map {
55 __uint(type, BPF_MAP_TYPE_ARRAY);
56 __uint(max_entries, 1);
57 __type(key, int);
58 __type(value, __u64);
59} inner_map SEC(".maps");
60
61struct outer_arr {
62 __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS);
63 __uint(max_entries, 1);
64 __uint(key_size, sizeof(int));
65 __uint(value_size, sizeof(int));
66 __array(values, struct inner_map);
67} outer_arr SEC(".maps") = {
68 .values = { [0] = &inner_map },
69};
70
71struct outer_hash {
72 __uint(type, BPF_MAP_TYPE_HASH_OF_MAPS);
73 __uint(max_entries, 1);
74 __uint(key_size, sizeof(int));
75 __array(values, struct inner_map);
76} outer_hash SEC(".maps") = {
77 .values = { [0] = &inner_map },
78};
79
33char _license[] SEC("license") = "GPL";80char _license[] SEC("license") = "GPL";
34 81
35int monitored_pid = 0;82int monitored_pid = 0;
@@ -61,6 +108,7 @@ SEC("lsm.s/bprm_committed_creds") · 25 unmodified lines
61int BPF_PROG(test_void_hook, struct linux_binprm *bprm)108int BPF_PROG(test_void_hook, struct linux_binprm *bprm)
62{109{
63 __u32 pid = bpf_get_current_pid_tgid() >> 32;110 __u32 pid = bpf_get_current_pid_tgid() >> 32;
111 struct inner_map *inner_map;
64 char args[64];112 char args[64];
65 __u32 key = 0;113 __u32 key = 0;
66 __u64 *value;114 __u64 *value;
@@ -80,6 +128,27 @@ int BPF_PROG(test_void_hook, struct linux_binprm *bprm) · 13 unmodified lines
80 value = bpf_map_lookup_elem(&lru_hash, &key);128 value = bpf_map_lookup_elem(&lru_hash, &key);
81 if (value)129 if (value)
82 *value = 0;130 *value = 0;
131 value = bpf_map_lookup_elem(&percpu_array, &key);
132 if (value)
133 *value = 0;
134 value = bpf_map_lookup_elem(&percpu_hash, &key);
135 if (value)
136 *value = 0;
137 value = bpf_map_lookup_elem(&lru_percpu_hash, &key);
138 if (value)
139 *value = 0;
140 inner_map = bpf_map_lookup_elem(&outer_arr, &key);
141 if (inner_map) {
142 value = bpf_map_lookup_elem(inner_map, &key);
143 if (value)
144 *value = 0;
145 }
146 inner_map = bpf_map_lookup_elem(&outer_hash, &key);
147 if (inner_map) {
148 value = bpf_map_lookup_elem(inner_map, &key);
149 if (value)
150 *value = 0;
151 }
83 152
84 return 0;153 return 0;
85}154}
tools/testing/selftests/bpf/progs/recursion.c+46−0
@@ -0,0 +1,46 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2021 Facebook */
3
4#include "vmlinux.h"
5#include <bpf/bpf_helpers.h>
6#include <bpf/bpf_tracing.h>
7
8char _license[] SEC("license") = "GPL";
9
10struct {
11 __uint(type, BPF_MAP_TYPE_HASH);
12 __uint(max_entries, 1);
13 __type(key, int);
14 __type(value, long);
15} hash1 SEC(".maps");
16
17struct {
18 __uint(type, BPF_MAP_TYPE_HASH);
19 __uint(max_entries, 1);
20 __type(key, int);
21 __type(value, long);
22} hash2 SEC(".maps");
23
24int pass1 = 0;
25int pass2 = 0;
26
27SEC("fentry/__htab_map_lookup_elem")
28int BPF_PROG(on_lookup, struct bpf_map *map)
29{
30 int key = 0;
31
32 if (map == (void *)&hash1) {
33 pass1++;
34 return 0;
35 }
36 if (map == (void *)&hash2) {
37 pass2++;
38 /* htab_map_gen_lookup() will inline below call
39 * into direct call to __htab_map_lookup_elem()
40 */
41 bpf_map_lookup_elem(&hash2, &key);
42 return 0;
43 }
44
45 return 0;
46}
tools/testing/selftests/bpf/progs/recvmsg4_prog.c+42−0
@@ -0,0 +1,42 @@
1// SPDX-License-Identifier: GPL-2.0
2
3#include <linux/stddef.h>
4#include <linux/bpf.h>
5#include <linux/in.h>
6#include <sys/socket.h>
7
8#include <bpf/bpf_helpers.h>
9#include <bpf/bpf_endian.h>
10
11#include <bpf_sockopt_helpers.h>
12
13#define SERV4_IP 0xc0a801feU /* 192.168.1.254 */
14#define SERV4_PORT 4040
15
16SEC("cgroup/recvmsg4")
17int recvmsg4_prog(struct bpf_sock_addr *ctx)
18{
19 struct bpf_sock *sk;
20 __u32 user_ip4;
21 __u16 user_port;
22
23 sk = ctx->sk;
24 if (!sk)
25 return 1;
26
27 if (sk->family != AF_INET)
28 return 1;
29
30 if (ctx->type != SOCK_STREAM && ctx->type != SOCK_DGRAM)
31 return 1;
32
33 if (!get_set_sk_priority(ctx))
34 return 1;
35
36 ctx->user_ip4 = bpf_htonl(SERV4_IP);
37 ctx->user_port = bpf_htons(SERV4_PORT);
38
39 return 1;
40}
41
42char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/recvmsg6_prog.c+48−0
@@ -0,0 +1,48 @@
1// SPDX-License-Identifier: GPL-2.0
2
3#include <linux/stddef.h>
4#include <linux/bpf.h>
5#include <linux/in6.h>
6#include <sys/socket.h>
7
8#include <bpf/bpf_helpers.h>
9#include <bpf/bpf_endian.h>
10
11#include <bpf_sockopt_helpers.h>
12
13#define SERV6_IP_0 0xfaceb00c /* face:b00c:1234:5678::abcd */
14#define SERV6_IP_1 0x12345678
15#define SERV6_IP_2 0x00000000
16#define SERV6_IP_3 0x0000abcd
17#define SERV6_PORT 6060
18
19SEC("cgroup/recvmsg6")
20int recvmsg6_prog(struct bpf_sock_addr *ctx)
21{
22 struct bpf_sock *sk;
23 __u32 user_ip4;
24 __u16 user_port;
25
26 sk = ctx->sk;
27 if (!sk)
28 return 1;
29
30 if (sk->family != AF_INET6)
31 return 1;
32
33 if (ctx->type != SOCK_STREAM && ctx->type != SOCK_DGRAM)
34 return 1;
35
36 if (!get_set_sk_priority(ctx))
37 return 1;
38
39 ctx->user_ip6[0] = bpf_htonl(SERV6_IP_0);
40 ctx->user_ip6[1] = bpf_htonl(SERV6_IP_1);
41 ctx->user_ip6[2] = bpf_htonl(SERV6_IP_2);
42 ctx->user_ip6[3] = bpf_htonl(SERV6_IP_3);
43 ctx->user_port = bpf_htons(SERV6_PORT);
44
45 return 1;
46}
47
48char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/sendmsg4_prog.c+7−0
@@ -8,6 +8,8 @@ · 7 unmodified lines
8#include <bpf/bpf_helpers.h>8#include <bpf/bpf_helpers.h>
9#include <bpf/bpf_endian.h>9#include <bpf/bpf_endian.h>
10 10
11#include <bpf_sockopt_helpers.h>
12
11#define SRC1_IP4 0xAC100001U /* 172.16.0.1 */13#define SRC1_IP4 0xAC100001U /* 172.16.0.1 */
12#define SRC2_IP4 0x00000000U14#define SRC2_IP4 0x00000000U
13#define SRC_REWRITE_IP4 0x7f000004U15#define SRC_REWRITE_IP4 0x7f000004U
@@ -21,9 +23,14 @@ int _version SEC("version") = 1; · 7 unmodified lines
21SEC("cgroup/sendmsg4")23SEC("cgroup/sendmsg4")
22int sendmsg_v4_prog(struct bpf_sock_addr *ctx)24int sendmsg_v4_prog(struct bpf_sock_addr *ctx)
23{25{
26 int prio;
27
24 if (ctx->type != SOCK_DGRAM)28 if (ctx->type != SOCK_DGRAM)
25 return 0;29 return 0;
26 30
31 if (!get_set_sk_priority(ctx))
32 return 0;
33
27 /* Rewrite source. */34 /* Rewrite source. */
28 if (ctx->msg_src_ip4 == bpf_htonl(SRC1_IP4) ||35 if (ctx->msg_src_ip4 == bpf_htonl(SRC1_IP4) ||
29 ctx->msg_src_ip4 == bpf_htonl(SRC2_IP4)) {36 ctx->msg_src_ip4 == bpf_htonl(SRC2_IP4)) {
tools/testing/selftests/bpf/progs/sendmsg6_prog.c+5−0
@@ -8,6 +8,8 @@ · 7 unmodified lines
8#include <bpf/bpf_helpers.h>8#include <bpf/bpf_helpers.h>
9#include <bpf/bpf_endian.h>9#include <bpf/bpf_endian.h>
10 10
11#include <bpf_sockopt_helpers.h>
12
11#define SRC_REWRITE_IP6_0 013#define SRC_REWRITE_IP6_0 0
12#define SRC_REWRITE_IP6_1 014#define SRC_REWRITE_IP6_1 0
13#define SRC_REWRITE_IP6_2 015#define SRC_REWRITE_IP6_2 0
@@ -28,6 +30,9 @@ int sendmsg_v6_prog(struct bpf_sock_addr *ctx) · 14 unmodified lines
28 if (ctx->type != SOCK_DGRAM)30 if (ctx->type != SOCK_DGRAM)
29 return 0;31 return 0;
30 32
33 if (!get_set_sk_priority(ctx))
34 return 0;
35
31 /* Rewrite source. */36 /* Rewrite source. */
32 if (ctx->msg_src_ip6[3] == bpf_htonl(1) ||37 if (ctx->msg_src_ip6[3] == bpf_htonl(1) ||
33 ctx->msg_src_ip6[3] == bpf_htonl(0)) {38 ctx->msg_src_ip6[3] == bpf_htonl(0)) {
tools/testing/selftests/bpf/progs/socket_cookie_prog.c+38−9
@@ -1,11 +1,13 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2// Copyright (c) 2018 Facebook2// Copyright (c) 2018 Facebook
3 3
4#include <linux/bpf.h>4#include "vmlinux.h"
5#include <sys/socket.h>
6 5
7#include <bpf/bpf_helpers.h>6#include <bpf/bpf_helpers.h>
8#include <bpf/bpf_endian.h>7#include <bpf/bpf_endian.h>
8#include <bpf/bpf_tracing.h>
9
10#define AF_INET6 10
9 11
10struct socket_cookie {12struct socket_cookie {
11 __u64 cookie_key;13 __u64 cookie_key;
@@ -19,6 +21,14 @@ struct { · 7 unmodified lines
19 __type(value, struct socket_cookie);21 __type(value, struct socket_cookie);
20} socket_cookies SEC(".maps");22} socket_cookies SEC(".maps");
21 23
24/*
25 * These three programs get executed in a row on connect() syscalls. The
26 * userspace side of the test creates a client socket, issues a connect() on it
27 * and then checks that the local storage associated with this socket has:
28 * cookie_value == local_port << 8 | 0xFF
29 * The different parts of this cookie_value are appended by those hooks if they
30 * all agree on the output of bpf_get_socket_cookie().
31 */
22SEC("cgroup/connect6")32SEC("cgroup/connect6")
23int set_cookie(struct bpf_sock_addr *ctx)33int set_cookie(struct bpf_sock_addr *ctx)
24{34{
@@ -32,16 +42,16 @@ int set_cookie(struct bpf_sock_addr *ctx) · 7 unmodified lines
32 if (!p)42 if (!p)
33 return 1;43 return 1;
34 44
35 p->cookie_value = 0xFF;45 p->cookie_value = 0xF;
36 p->cookie_key = bpf_get_socket_cookie(ctx);46 p->cookie_key = bpf_get_socket_cookie(ctx);
37 47
38 return 1;48 return 1;
39}49}
40 50
41SEC("sockops")51SEC("sockops")
42int update_cookie(struct bpf_sock_ops *ctx)52int update_cookie_sockops(struct bpf_sock_ops *ctx)
43{53{
44 struct bpf_sock *sk;54 struct bpf_sock *sk = ctx->sk;
45 struct socket_cookie *p;55 struct socket_cookie *p;
46 56
47 if (ctx->family != AF_INET6)57 if (ctx->family != AF_INET6)
@@ -50,21 +60,40 @@ int update_cookie(struct bpf_sock_ops *ctx) · 2 unmodified lines
50 if (ctx->op != BPF_SOCK_OPS_TCP_CONNECT_CB)60 if (ctx->op != BPF_SOCK_OPS_TCP_CONNECT_CB)
51 return 1;61 return 1;
52 62
53 if (!ctx->sk)63 if (!sk)
54 return 1;64 return 1;
55 65
56 p = bpf_sk_storage_get(&socket_cookies, ctx->sk, 0, 0);66 p = bpf_sk_storage_get(&socket_cookies, sk, 0, 0);
57 if (!p)67 if (!p)
58 return 1;68 return 1;
59 69
60 if (p->cookie_key != bpf_get_socket_cookie(ctx))70 if (p->cookie_key != bpf_get_socket_cookie(ctx))
61 return 1;71 return 1;
62 72
63 p->cookie_value = (ctx->local_port << 8) | p->cookie_value;73 p->cookie_value |= (ctx->local_port << 8);
64 74
65 return 1;75 return 1;
66}76}
67 77
68int _version SEC("version") = 1;78SEC("fexit/inet_stream_connect")
79int BPF_PROG(update_cookie_tracing, struct socket *sock,
80 struct sockaddr *uaddr, int addr_len, int flags)
81{
82 struct socket_cookie *p;
83
84 if (uaddr->sa_family != AF_INET6)
85 return 0;
86
87 p = bpf_sk_storage_get(&socket_cookies, sock->sk, 0, 0);
88 if (!p)
89 return 0;
90
91 if (p->cookie_key != bpf_get_socket_cookie(sock->sk))
92 return 0;
93
94 p->cookie_value |= 0xF0;
95
96 return 0;
97}
69 98
70char _license[] SEC("license") = "GPL";99char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/sockopt_sk.c+21−2
@@ -1,8 +1,8 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2#include <string.h>2#include <string.h>
3#include <netinet/in.h>3#include <linux/tcp.h>
4#include <netinet/tcp.h>
5#include <linux/bpf.h>4#include <linux/bpf.h>
5#include <netinet/in.h>
6#include <bpf/bpf_helpers.h>6#include <bpf/bpf_helpers.h>
7 7
8char _license[] SEC("license") = "GPL";8char _license[] SEC("license") = "GPL";
@@ -12,6 +12,10 @@ __u32 _version SEC("version") = 1; · 3 unmodified lines
12#define PAGE_SIZE 409612#define PAGE_SIZE 4096
13#endif13#endif
14 14
15#ifndef SOL_TCP
16#define SOL_TCP IPPROTO_TCP
17#endif
18
15#define SOL_CUSTOM 0xdeadbeef19#define SOL_CUSTOM 0xdeadbeef
16 20
17struct sockopt_sk {21struct sockopt_sk {
@@ -57,6 +61,21 @@ int _getsockopt(struct bpf_sockopt *ctx) · 39 unmodified lines
57 return 1;61 return 1;
58 }62 }
59 63
64 if (ctx->level == SOL_TCP && ctx->optname == TCP_ZEROCOPY_RECEIVE) {
65 /* Verify that TCP_ZEROCOPY_RECEIVE triggers.
66 * It has a custom implementation for performance
67 * reasons.
68 */
69
70 if (optval + sizeof(struct tcp_zerocopy_receive) > optval_end)
71 return 0; /* EPERM, bounds check */
72
73 if (((struct tcp_zerocopy_receive *)optval)->address != 0)
74 return 0; /* EPERM, unexpected data */
75
76 return 1;
77 }
78
60 if (ctx->level == SOL_IP && ctx->optname == IP_FREEBIND) {79 if (ctx->level == SOL_IP && ctx->optname == IP_FREEBIND) {
61 if (optval + 1 > optval_end)80 if (optval + 1 > optval_end)
62 return 0; /* EPERM, bounds check */81 return 0; /* EPERM, bounds check */
tools/testing/selftests/bpf/progs/test_check_mtu.c+198−0
@@ -0,0 +1,198 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2020 Jesper Dangaard Brouer */
3
4#include <linux/bpf.h>
5#include <bpf/bpf_helpers.h>
6#include <linux/if_ether.h>
7
8#include <stddef.h>
9#include <stdint.h>
10
11char _license[] SEC("license") = "GPL";
12
13/* Userspace will update with MTU it can see on device */
14static volatile const int GLOBAL_USER_MTU;
15static volatile const __u32 GLOBAL_USER_IFINDEX;
16
17/* BPF-prog will update these with MTU values it can see */
18__u32 global_bpf_mtu_xdp = 0;
19__u32 global_bpf_mtu_tc = 0;
20
21SEC("xdp")
22int xdp_use_helper_basic(struct xdp_md *ctx)
23{
24 __u32 mtu_len = 0;
25
26 if (bpf_check_mtu(ctx, 0, &mtu_len, 0, 0))
27 return XDP_ABORTED;
28
29 return XDP_PASS;
30}
31
32SEC("xdp")
33int xdp_use_helper(struct xdp_md *ctx)
34{
35 int retval = XDP_PASS; /* Expected retval on successful test */
36 __u32 mtu_len = 0;
37 __u32 ifindex = 0;
38 int delta = 0;
39
40 /* When ifindex is zero, save net_device lookup and use ctx netdev */
41 if (GLOBAL_USER_IFINDEX > 0)
42 ifindex = GLOBAL_USER_IFINDEX;
43
44 if (bpf_check_mtu(ctx, ifindex, &mtu_len, delta, 0)) {
45 /* mtu_len is also valid when check fail */
46 retval = XDP_ABORTED;
47 goto out;
48 }
49
50 if (mtu_len != GLOBAL_USER_MTU)
51 retval = XDP_DROP;
52
53out:
54 global_bpf_mtu_xdp = mtu_len;
55 return retval;
56}
57
58SEC("xdp")
59int xdp_exceed_mtu(struct xdp_md *ctx)
60{
61 void *data_end = (void *)(long)ctx->data_end;
62 void *data = (void *)(long)ctx->data;
63 __u32 ifindex = GLOBAL_USER_IFINDEX;
64 __u32 data_len = data_end - data;
65 int retval = XDP_ABORTED; /* Fail */
66 __u32 mtu_len = 0;
67 int delta;
68 int err;
69
70 /* Exceed MTU with 1 via delta adjust */
71 delta = GLOBAL_USER_MTU - (data_len - ETH_HLEN) + 1;
72
73 err = bpf_check_mtu(ctx, ifindex, &mtu_len, delta, 0);
74 if (err) {
75 retval = XDP_PASS; /* Success in exceeding MTU check */
76 if (err != BPF_MTU_CHK_RET_FRAG_NEEDED)
77 retval = XDP_DROP;
78 }
79
80 global_bpf_mtu_xdp = mtu_len;
81 return retval;
82}
83
84SEC("xdp")
85int xdp_minus_delta(struct xdp_md *ctx)
86{
87 int retval = XDP_PASS; /* Expected retval on successful test */
88 void *data_end = (void *)(long)ctx->data_end;
89 void *data = (void *)(long)ctx->data;
90 __u32 ifindex = GLOBAL_USER_IFINDEX;
91 __u32 data_len = data_end - data;
92 __u32 mtu_len = 0;
93 int delta;
94
95 /* Borderline test case: Minus delta exceeding packet length allowed */
96 delta = -((data_len - ETH_HLEN) + 1);
97
98 /* Minus length (adjusted via delta) still pass MTU check, other helpers
99 * are responsible for catching this, when doing actual size adjust
100 */
101 if (bpf_check_mtu(ctx, ifindex, &mtu_len, delta, 0))
102 retval = XDP_ABORTED;
103
104 global_bpf_mtu_xdp = mtu_len;
105 return retval;
106}
107
108SEC("classifier")
109int tc_use_helper(struct __sk_buff *ctx)
110{
111 int retval = BPF_OK; /* Expected retval on successful test */
112 __u32 mtu_len = 0;
113 int delta = 0;
114
115 if (bpf_check_mtu(ctx, 0, &mtu_len, delta, 0)) {
116 retval = BPF_DROP;
117 goto out;
118 }
119
120 if (mtu_len != GLOBAL_USER_MTU)
121 retval = BPF_REDIRECT;
122out:
123 global_bpf_mtu_tc = mtu_len;
124 return retval;
125}
126
127SEC("classifier")
128int tc_exceed_mtu(struct __sk_buff *ctx)
129{
130 __u32 ifindex = GLOBAL_USER_IFINDEX;
131 int retval = BPF_DROP; /* Fail */
132 __u32 skb_len = ctx->len;
133 __u32 mtu_len = 0;
134 int delta;
135 int err;
136
137 /* Exceed MTU with 1 via delta adjust */
138 delta = GLOBAL_USER_MTU - (skb_len - ETH_HLEN) + 1;
139
140 err = bpf_check_mtu(ctx, ifindex, &mtu_len, delta, 0);
141 if (err) {
142 retval = BPF_OK; /* Success in exceeding MTU check */
143 if (err != BPF_MTU_CHK_RET_FRAG_NEEDED)
144 retval = BPF_DROP;
145 }
146
147 global_bpf_mtu_tc = mtu_len;
148 return retval;
149}
150
151SEC("classifier")
152int tc_exceed_mtu_da(struct __sk_buff *ctx)
153{
154 /* SKB Direct-Access variant */
155 void *data_end = (void *)(long)ctx->data_end;
156 void *data = (void *)(long)ctx->data;
157 __u32 ifindex = GLOBAL_USER_IFINDEX;
158 __u32 data_len = data_end - data;
159 int retval = BPF_DROP; /* Fail */
160 __u32 mtu_len = 0;
161 int delta;
162 int err;
163
164 /* Exceed MTU with 1 via delta adjust */
165 delta = GLOBAL_USER_MTU - (data_len - ETH_HLEN) + 1;
166
167 err = bpf_check_mtu(ctx, ifindex, &mtu_len, delta, 0);
168 if (err) {
169 retval = BPF_OK; /* Success in exceeding MTU check */
170 if (err != BPF_MTU_CHK_RET_FRAG_NEEDED)
171 retval = BPF_DROP;
172 }
173
174 global_bpf_mtu_tc = mtu_len;
175 return retval;
176}
177
178SEC("classifier")
179int tc_minus_delta(struct __sk_buff *ctx)
180{
181 int retval = BPF_OK; /* Expected retval on successful test */
182 __u32 ifindex = GLOBAL_USER_IFINDEX;
183 __u32 skb_len = ctx->len;
184 __u32 mtu_len = 0;
185 int delta;
186
187 /* Borderline test case: Minus delta exceeding packet length allowed */
188 delta = -((skb_len - ETH_HLEN) + 1);
189
190 /* Minus length (adjusted via delta) still pass MTU check, other helpers
191 * are responsible for catching this, when doing actual size adjust
192 */
193 if (bpf_check_mtu(ctx, ifindex, &mtu_len, delta, 0))
194 retval = BPF_DROP;
195
196 global_bpf_mtu_xdp = mtu_len;
197 return retval;
198}
tools/testing/selftests/bpf/progs/test_cls_redirect.c+7−0
@@ -70,6 +70,7 @@ typedef struct { · 69 unmodified lines
70 uint64_t errors_total_encap_adjust_failed;70 uint64_t errors_total_encap_adjust_failed;
71 uint64_t errors_total_encap_buffer_too_small;71 uint64_t errors_total_encap_buffer_too_small;
72 uint64_t errors_total_redirect_loop;72 uint64_t errors_total_redirect_loop;
73 uint64_t errors_total_encap_mtu_violate;
73} metrics_t;74} metrics_t;
74 75
75typedef enum {76typedef enum {
@@ -407,6 +408,7 @@ static INLINING ret_t forward_with_gre(struct __sk_buff *skb, encap_headers_t *e · 331 unmodified lines
407 payload_off - sizeof(struct ethhdr) - sizeof(struct iphdr);408 payload_off - sizeof(struct ethhdr) - sizeof(struct iphdr);
408 int32_t delta = sizeof(struct gre_base_hdr) - encap_overhead;409 int32_t delta = sizeof(struct gre_base_hdr) - encap_overhead;
409 uint16_t proto = ETH_P_IP;410 uint16_t proto = ETH_P_IP;
411 uint32_t mtu_len = 0;
410 412
411 /* Loop protection: the inner packet's TTL is decremented as a safeguard413 /* Loop protection: the inner packet's TTL is decremented as a safeguard
412 * against any forwarding loop. As the only interesting field is the TTL414 * against any forwarding loop. As the only interesting field is the TTL
@@ -479,6 +481,11 @@ static INLINING ret_t forward_with_gre(struct __sk_buff *skb, encap_headers_t *e · 66 unmodified lines
479 }481 }
480 }482 }
481 483
484 if (bpf_check_mtu(skb, skb->ifindex, &mtu_len, delta, 0)) {
485 metrics->errors_total_encap_mtu_violate++;
486 return TC_ACT_SHOT;
487 }
488
482 if (bpf_skb_adjust_room(skb, delta, BPF_ADJ_ROOM_NET,489 if (bpf_skb_adjust_room(skb, delta, BPF_ADJ_ROOM_NET,
483 BPF_F_ADJ_ROOM_FIXED_GSO |490 BPF_F_ADJ_ROOM_FIXED_GSO |
484 BPF_F_ADJ_ROOM_NO_CSUM_RESET) ||491 BPF_F_ADJ_ROOM_NO_CSUM_RESET) ||
tools/testing/selftests/bpf/progs/test_global_func10.c+29−0
@@ -0,0 +1,29 @@
1// SPDX-License-Identifier: GPL-2.0-only
2#include <stddef.h>
3#include <linux/bpf.h>
4#include <bpf/bpf_helpers.h>
5
6struct Small {
7 int x;
8};
9
10struct Big {
11 int x;
12 int y;
13};
14
15__noinline int foo(const struct Big *big)
16{
17 if (big == 0)
18 return 0;
19
20 return bpf_get_prandom_u32() < big->y;
21}
22
23SEC("cgroup_skb/ingress")
24int test_cls(struct __sk_buff *skb)
25{
26 const struct Small small = {.x = skb->len };
27
28 return foo((struct Big *)&small) ? 1 : 0;
29}
tools/testing/selftests/bpf/progs/test_global_func11.c+19−0
@@ -0,0 +1,19 @@
1// SPDX-License-Identifier: GPL-2.0-only
2#include <stddef.h>
3#include <linux/bpf.h>
4#include <bpf/bpf_helpers.h>
5
6struct S {
7 int x;
8};
9
10__noinline int foo(const struct S *s)
11{
12 return s ? bpf_get_prandom_u32() < s->x : 0;
13}
14
15SEC("cgroup_skb/ingress")
16int test_cls(struct __sk_buff *skb)
17{
18 return foo(skb);
19}
tools/testing/selftests/bpf/progs/test_global_func12.c+21−0
@@ -0,0 +1,21 @@
1// SPDX-License-Identifier: GPL-2.0-only
2#include <stddef.h>
3#include <linux/bpf.h>
4#include <bpf/bpf_helpers.h>
5
6struct S {
7 int x;
8};
9
10__noinline int foo(const struct S *s)
11{
12 return bpf_get_prandom_u32() < s->x;
13}
14
15SEC("cgroup_skb/ingress")
16int test_cls(struct __sk_buff *skb)
17{
18 const struct S s = {.x = skb->len };
19
20 return foo(&s);
21}
tools/testing/selftests/bpf/progs/test_global_func13.c+24−0
@@ -0,0 +1,24 @@
1// SPDX-License-Identifier: GPL-2.0-only
2#include <stddef.h>
3#include <linux/bpf.h>
4#include <bpf/bpf_helpers.h>
5
6struct S {
7 int x;
8};
9
10__noinline int foo(const struct S *s)
11{
12 if (s)
13 return bpf_get_prandom_u32() < s->x;
14
15 return 0;
16}
17
18SEC("cgroup_skb/ingress")
19int test_cls(struct __sk_buff *skb)
20{
21 const struct S *s = (const struct S *)(0xbedabeda);
22
23 return foo(s);
24}
tools/testing/selftests/bpf/progs/test_global_func14.c+21−0
@@ -0,0 +1,21 @@
1// SPDX-License-Identifier: GPL-2.0-only
2#include <stddef.h>
3#include <linux/bpf.h>
4#include <bpf/bpf_helpers.h>
5
6struct S;
7
8__noinline int foo(const struct S *s)
9{
10 if (s)
11 return bpf_get_prandom_u32() < *(const int *) s;
12
13 return 0;
14}
15
16SEC("cgroup_skb/ingress")
17int test_cls(struct __sk_buff *skb)
18{
19
20 return foo(NULL);
21}
tools/testing/selftests/bpf/progs/test_global_func15.c+22−0
@@ -0,0 +1,22 @@
1// SPDX-License-Identifier: GPL-2.0-only
2#include <stddef.h>
3#include <linux/bpf.h>
4#include <bpf/bpf_helpers.h>
5
6__noinline int foo(unsigned int *v)
7{
8 if (v)
9 *v = bpf_get_prandom_u32();
10
11 return 0;
12}
13
14SEC("cgroup_skb/ingress")
15int test_cls(struct __sk_buff *skb)
16{
17 unsigned int v = 1;
18
19 foo(&v);
20
21 return v;
22}
tools/testing/selftests/bpf/progs/test_global_func16.c+22−0
@@ -0,0 +1,22 @@
1// SPDX-License-Identifier: GPL-2.0-only
2#include <stddef.h>
3#include <linux/bpf.h>
4#include <bpf/bpf_helpers.h>
5
6__noinline int foo(int (*arr)[10])
7{
8 if (arr)
9 return (*arr)[9];
10
11 return 0;
12}
13
14SEC("cgroup_skb/ingress")
15int test_cls(struct __sk_buff *skb)
16{
17 int array[10];
18
19 const int rv = foo(&array);
20
21 return rv ? 1 : 0;
22}
tools/testing/selftests/bpf/progs/test_global_func9.c+132−0
@@ -0,0 +1,132 @@
1// SPDX-License-Identifier: GPL-2.0-only
2#include <stddef.h>
3#include <linux/bpf.h>
4#include <bpf/bpf_helpers.h>
5
6struct S {
7 int x;
8};
9
10struct C {
11 int x;
12 int y;
13};
14
15struct {
16 __uint(type, BPF_MAP_TYPE_ARRAY);
17 __uint(max_entries, 1);
18 __type(key, __u32);
19 __type(value, struct S);
20} map SEC(".maps");
21
22enum E {
23 E_ITEM
24};
25
26static int global_data_x = 100;
27static int volatile global_data_y = 500;
28
29__noinline int foo(const struct S *s)
30{
31 if (s)
32 return bpf_get_prandom_u32() < s->x;
33
34 return 0;
35}
36
37__noinline int bar(int *x)
38{
39 if (x)
40 *x &= bpf_get_prandom_u32();
41
42 return 0;
43}
44__noinline int baz(volatile int *x)
45{
46 if (x)
47 *x &= bpf_get_prandom_u32();
48
49 return 0;
50}
51
52__noinline int qux(enum E *e)
53{
54 if (e)
55 return *e;
56
57 return 0;
58}
59
60__noinline int quux(int (*arr)[10])
61{
62 if (arr)
63 return (*arr)[9];
64
65 return 0;
66}
67
68__noinline int quuz(int **p)
69{
70 if (p)
71 *p = NULL;
72
73 return 0;
74}
75
76SEC("cgroup_skb/ingress")
77int test_cls(struct __sk_buff *skb)
78{
79 int result = 0;
80
81 {
82 const struct S s = {.x = skb->len };
83
84 result |= foo(&s);
85 }
86
87 {
88 const __u32 key = 1;
89 const struct S *s = bpf_map_lookup_elem(&map, &key);
90
91 result |= foo(s);
92 }
93
94 {
95 const struct C c = {.x = skb->len, .y = skb->family };
96
97 result |= foo((const struct S *)&c);
98 }
99
100 {
101 result |= foo(NULL);
102 }
103
104 {
105 bar(&result);
106 bar(&global_data_x);
107 }
108
109 {
110 result |= baz(&global_data_y);
111 }
112
113 {
114 enum E e = E_ITEM;
115
116 result |= qux(&e);
117 }
118
119 {
120 int array[10] = {0};
121
122 result |= quux(&array);
123 }
124
125 {
126 int *p;
127
128 result |= quuz(&p);
129 }
130
131 return result ? 1 : 0;
132}
tools/testing/selftests/bpf/progs/test_global_func_args.c+91−0
@@ -0,0 +1,91 @@
1// SPDX-License-Identifier: GPL-2.0
2
3#include <linux/bpf.h>
4
5#include <bpf/bpf_helpers.h>
6
7struct S {
8 int v;
9};
10
11static volatile struct S global_variable;
12
13struct {
14 __uint(type, BPF_MAP_TYPE_ARRAY);
15 __uint(max_entries, 7);
16 __type(key, __u32);
17 __type(value, int);
18} values SEC(".maps");
19
20static void save_value(__u32 index, int value)
21{
22 bpf_map_update_elem(&values, &index, &value, 0);
23}
24
25__noinline int foo(__u32 index, struct S *s)
26{
27 if (s) {
28 save_value(index, s->v);
29 return ++s->v;
30 }
31
32 save_value(index, 0);
33
34 return 1;
35}
36
37__noinline int bar(__u32 index, volatile struct S *s)
38{
39 if (s) {
40 save_value(index, s->v);
41 return ++s->v;
42 }
43
44 save_value(index, 0);
45
46 return 1;
47}
48
49__noinline int baz(struct S **s)
50{
51 if (s)
52 *s = 0;
53
54 return 0;
55}
56
57SEC("cgroup_skb/ingress")
58int test_cls(struct __sk_buff *skb)
59{
60 __u32 index = 0;
61
62 {
63 const int v = foo(index++, 0);
64
65 save_value(index++, v);
66 }
67
68 {
69 struct S s = { .v = 100 };
70
71 foo(index++, &s);
72 save_value(index++, s.v);
73 }
74
75 {
76 global_variable.v = 42;
77 bar(index++, &global_variable);
78 save_value(index++, global_variable.v);
79 }
80
81 {
82 struct S v, *p = &v;
83
84 baz(&p);
85 save_value(index++, !p);
86 }
87
88 return 0;
89}
90
91char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/test_module_attach.c+10−0
@@ -17,6 +17,16 @@ int BPF_PROG(handle_raw_tp, · 16 unmodified lines
17 return 0;17 return 0;
18}18}
19 19
20__u32 raw_tp_bare_write_sz = 0;
21
22SEC("raw_tp/bpf_testmod_test_write_bare")
23int BPF_PROG(handle_raw_tp_bare,
24 struct task_struct *task, struct bpf_testmod_test_write_ctx *write_ctx)
25{
26 raw_tp_bare_write_sz = BPF_CORE_READ(write_ctx, len);
27 return 0;
28}
29
20__u32 tp_btf_read_sz = 0;30__u32 tp_btf_read_sz = 0;
21 31
22SEC("tp_btf/bpf_testmod_test_read")32SEC("tp_btf/bpf_testmod_test_read")
tools/testing/selftests/bpf/progs/test_ns_current_pid_tgid.c+9−19
@@ -5,31 +5,21 @@ · 4 unmodified lines
5#include <stdint.h>5#include <stdint.h>
6#include <bpf/bpf_helpers.h>6#include <bpf/bpf_helpers.h>
7 7
8static volatile struct {8__u64 user_pid = 0;
9 __u64 dev;9__u64 user_tgid = 0;
10 __u64 ino;10__u64 dev = 0;
11 __u64 pid_tgid;11__u64 ino = 0;
12 __u64 user_pid_tgid;
13} res;
14 12
15SEC("raw_tracepoint/sys_enter")13SEC("tracepoint/syscalls/sys_enter_nanosleep")
16int trace(void *ctx)14int handler(const void *ctx)
17{15{
18 __u64 ns_pid_tgid, expected_pid;
19 struct bpf_pidns_info nsdata;16 struct bpf_pidns_info nsdata;
20 __u32 key = 0;
21 17
22 if (bpf_get_ns_current_pid_tgid(res.dev, res.ino, &nsdata,18 if (bpf_get_ns_current_pid_tgid(dev, ino, &nsdata, sizeof(struct bpf_pidns_info)))
23 sizeof(struct bpf_pidns_info)))
24 return 0;19 return 0;
25 20
26 ns_pid_tgid = (__u64)nsdata.tgid << 32 | nsdata.pid;21 user_pid = nsdata.pid;
27 expected_pid = res.user_pid_tgid;22 user_tgid = nsdata.tgid;
28
29 if (expected_pid != ns_pid_tgid)
30 return 0;
31
32 res.pid_tgid = ns_pid_tgid;
33 23
34 return 0;24 return 0;
35}25}
tools/testing/selftests/bpf/progs/test_stack_var_off.c+51−0
@@ -0,0 +1,51 @@
1// SPDX-License-Identifier: GPL-2.0
2
3#include <linux/bpf.h>
4#include <bpf/bpf_helpers.h>
5
6int probe_res;
7
8char input[4] = {};
9int test_pid;
10
11SEC("tracepoint/syscalls/sys_enter_nanosleep")
12int probe(void *ctx)
13{
14 /* This BPF program performs variable-offset reads and writes on a
15 * stack-allocated buffer.
16 */
17 char stack_buf[16];
18 unsigned long len;
19 unsigned long last;
20
21 if ((bpf_get_current_pid_tgid() >> 32) != test_pid)
22 return 0;
23
24 /* Copy the input to the stack. */
25 __builtin_memcpy(stack_buf, input, 4);
26
27 /* The first byte in the buffer indicates the length. */
28 len = stack_buf[0] & 0xf;
29 last = (len - 1) & 0xf;
30
31 /* Append something to the buffer. The offset where we write is not
32 * statically known; this is a variable-offset stack write.
33 */
34 stack_buf[len] = 42;
35
36 /* Index into the buffer at an unknown offset. This is a
37 * variable-offset stack read.
38 *
39 * Note that if it wasn't for the preceding variable-offset write, this
40 * read would be rejected because the stack slot cannot be verified as
41 * being initialized. With the preceding variable-offset write, the
42 * stack slot still cannot be verified, but the write inhibits the
43 * respective check on the reasoning that, if there was a
44 * variable-offset to a higher-or-equal spot, we're probably reading
45 * what we just wrote.
46 */
47 probe_res = stack_buf[last];
48 return 0;
49}
50
51char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/test_current_pid_tgid_new_ns.c+0−160
@@ -1,160 +0,0 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2020 Carlos Neira cneirabustos@gmail.com */
3#define _GNU_SOURCE
4#include <sys/stat.h>
5#include <sys/types.h>
6#include <unistd.h>
7#include <sys/syscall.h>
8#include <sched.h>
9#include <sys/wait.h>
10#include <sys/mount.h>
11#include "test_progs.h"
12
13#define CHECK_NEWNS(condition, tag, format...) ({ \
14 int __ret = !!(condition); \
15 if (__ret) { \
16 printf("%s:FAIL:%s ", __func__, tag); \
17 printf(format); \
18 } else { \
19 printf("%s:PASS:%s\n", __func__, tag); \
20 } \
21 __ret; \
22})
23
24struct bss {
25 __u64 dev;
26 __u64 ino;
27 __u64 pid_tgid;
28 __u64 user_pid_tgid;
29};
30
31int main(int argc, char **argv)
32{
33 pid_t pid;
34 int exit_code = 1;
35 struct stat st;
36
37 printf("Testing bpf_get_ns_current_pid_tgid helper in new ns\n");
38
39 if (stat("/proc/self/ns/pid", &st)) {
40 perror("stat failed on /proc/self/ns/pid ns\n");
41 printf("%s:FAILED\n", argv[0]);
42 return exit_code;
43 }
44
45 if (CHECK_NEWNS(unshare(CLONE_NEWPID | CLONE_NEWNS),
46 "unshare CLONE_NEWPID | CLONE_NEWNS", "error errno=%d\n", errno))
47 return exit_code;
48
49 pid = fork();
50 if (pid == -1) {
51 perror("Fork() failed\n");
52 printf("%s:FAILED\n", argv[0]);
53 return exit_code;
54 }
55
56 if (pid > 0) {
57 int status;
58
59 usleep(5);
60 waitpid(pid, &status, 0);
61 return 0;
62 } else {
63
64 pid = fork();
65 if (pid == -1) {
66 perror("Fork() failed\n");
67 printf("%s:FAILED\n", argv[0]);
68 return exit_code;
69 }
70
71 if (pid > 0) {
72 int status;
73 waitpid(pid, &status, 0);
74 return 0;
75 } else {
76 if (CHECK_NEWNS(mount("none", "/proc", NULL, MS_PRIVATE|MS_REC, NULL),
77 "Unmounting proc", "Cannot umount proc! errno=%d\n", errno))
78 return exit_code;
79
80 if (CHECK_NEWNS(mount("proc", "/proc", "proc", MS_NOSUID|MS_NOEXEC|MS_NODEV, NULL),
81 "Mounting proc", "Cannot mount proc! errno=%d\n", errno))
82 return exit_code;
83
84 const char *probe_name = "raw_tracepoint/sys_enter";
85 const char *file = "test_ns_current_pid_tgid.o";
86 struct bpf_link *link = NULL;
87 struct bpf_program *prog;
88 struct bpf_map *bss_map;
89 struct bpf_object *obj;
90 int exit_code = 1;
91 int err, key = 0;
92 struct bss bss;
93 struct stat st;
94 __u64 id;
95
96 obj = bpf_object__open_file(file, NULL);
97 if (CHECK_NEWNS(IS_ERR(obj), "obj_open", "err %ld\n", PTR_ERR(obj)))
98 return exit_code;
99
100 err = bpf_object__load(obj);
101 if (CHECK_NEWNS(err, "obj_load", "err %d errno %d\n", err, errno))
102 goto cleanup;
103
104 bss_map = bpf_object__find_map_by_name(obj, "test_ns_.bss");
105 if (CHECK_NEWNS(!bss_map, "find_bss_map", "failed\n"))
106 goto cleanup;
107
108 prog = bpf_object__find_program_by_title(obj, probe_name);
109 if (CHECK_NEWNS(!prog, "find_prog", "prog '%s' not found\n",
110 probe_name))
111 goto cleanup;
112
113 memset(&bss, 0, sizeof(bss));
114 pid_t tid = syscall(SYS_gettid);
115 pid_t pid = getpid();
116
117 id = (__u64) tid << 32 | pid;
118 bss.user_pid_tgid = id;
119
120 if (CHECK_NEWNS(stat("/proc/self/ns/pid", &st),
121 "stat new ns", "Failed to stat /proc/self/ns/pid errno=%d\n", errno))
122 goto cleanup;
123
124 bss.dev = st.st_dev;
125 bss.ino = st.st_ino;
126
127 err = bpf_map_update_elem(bpf_map__fd(bss_map), &key, &bss, 0);
128 if (CHECK_NEWNS(err, "setting_bss", "failed to set bss : %d\n", err))
129 goto cleanup;
130
131 link = bpf_program__attach_raw_tracepoint(prog, "sys_enter");
132 if (CHECK_NEWNS(IS_ERR(link), "attach_raw_tp", "err %ld\n",
133 PTR_ERR(link))) {
134 link = NULL;
135 goto cleanup;
136 }
137
138 /* trigger some syscalls */
139 usleep(1);
140
141 err = bpf_map_lookup_elem(bpf_map__fd(bss_map), &key, &bss);
142 if (CHECK_NEWNS(err, "set_bss", "failed to get bss : %d\n", err))
143 goto cleanup;
144
145 if (CHECK_NEWNS(id != bss.pid_tgid, "Compare user pid/tgid vs. bpf pid/tgid",
146 "User pid/tgid %llu BPF pid/tgid %llu\n", id, bss.pid_tgid))
147 goto cleanup;
148
149 exit_code = 0;
150 printf("%s:PASS\n", argv[0]);
151cleanup:
152 if (!link) {
153 bpf_link__destroy(link);
154 link = NULL;
155 }
156 bpf_object__close(obj);
157 }
158 }
159 return 0;
160}
tools/testing/selftests/bpf/test_flow_dissector.c+1−1
@@ -503,7 +503,7 @@ static int do_rx(int fd) · 502 unmodified lines
503 if (rbuf != cfg_payload_char)503 if (rbuf != cfg_payload_char)
504 error(1, 0, "recv: payload mismatch");504 error(1, 0, "recv: payload mismatch");
505 num++;505 num++;
506 };506 }
507 507
508 return num;508 return num;
509}509}
tools/testing/selftests/bpf/test_progs.c+1−1
@@ -390,7 +390,7 @@ static void unload_bpf_testmod(void) · 389 unmodified lines
390 return;390 return;
391 }391 }
392 fprintf(env.stderr, "Failed to unload bpf_testmod.ko from kernel: %d\n", -errno);392 fprintf(env.stderr, "Failed to unload bpf_testmod.ko from kernel: %d\n", -errno);
393 exit(1);393 return;
394 }394 }
395 if (env.verbosity > VERBOSE_NONE)395 if (env.verbosity > VERBOSE_NONE)
396 fprintf(stdout, "Successfully unloaded bpf_testmod.ko.\n");396 fprintf(stdout, "Successfully unloaded bpf_testmod.ko.\n");
tools/testing/selftests/bpf/test_progs.h+0−1
@@ -16,7 +16,6 @@ typedef __u16 __sum16; · 15 unmodified lines
16#include <linux/if_packet.h>16#include <linux/if_packet.h>
17#include <linux/ip.h>17#include <linux/ip.h>
18#include <linux/ipv6.h>18#include <linux/ipv6.h>
19#include <netinet/tcp.h>
20#include <linux/filter.h>19#include <linux/filter.h>
21#include <linux/perf_event.h>20#include <linux/perf_event.h>
22#include <linux/socket.h>21#include <linux/socket.h>
tools/testing/selftests/bpf/test_sock_addr.c+12−74
@@ -31,6 +31,8 @@ · 30 unmodified lines
31#define CONNECT6_PROG_PATH "./connect6_prog.o"31#define CONNECT6_PROG_PATH "./connect6_prog.o"
32#define SENDMSG4_PROG_PATH "./sendmsg4_prog.o"32#define SENDMSG4_PROG_PATH "./sendmsg4_prog.o"
33#define SENDMSG6_PROG_PATH "./sendmsg6_prog.o"33#define SENDMSG6_PROG_PATH "./sendmsg6_prog.o"
34#define RECVMSG4_PROG_PATH "./recvmsg4_prog.o"
35#define RECVMSG6_PROG_PATH "./recvmsg6_prog.o"
34#define BIND4_PROG_PATH "./bind4_prog.o"36#define BIND4_PROG_PATH "./bind4_prog.o"
35#define BIND6_PROG_PATH "./bind6_prog.o"37#define BIND6_PROG_PATH "./bind6_prog.o"
36 38
@@ -94,10 +96,10 @@ static int sendmsg_deny_prog_load(const struct sock_addr_test *test); · 57 unmodified lines
94static int recvmsg_allow_prog_load(const struct sock_addr_test *test);96static int recvmsg_allow_prog_load(const struct sock_addr_test *test);
95static int recvmsg_deny_prog_load(const struct sock_addr_test *test);97static int recvmsg_deny_prog_load(const struct sock_addr_test *test);
96static int sendmsg4_rw_asm_prog_load(const struct sock_addr_test *test);98static int sendmsg4_rw_asm_prog_load(const struct sock_addr_test *test);
97static int recvmsg4_rw_asm_prog_load(const struct sock_addr_test *test);99static int recvmsg4_rw_c_prog_load(const struct sock_addr_test *test);
98static int sendmsg4_rw_c_prog_load(const struct sock_addr_test *test);100static int sendmsg4_rw_c_prog_load(const struct sock_addr_test *test);
99static int sendmsg6_rw_asm_prog_load(const struct sock_addr_test *test);101static int sendmsg6_rw_asm_prog_load(const struct sock_addr_test *test);
100static int recvmsg6_rw_asm_prog_load(const struct sock_addr_test *test);102static int recvmsg6_rw_c_prog_load(const struct sock_addr_test *test);
101static int sendmsg6_rw_c_prog_load(const struct sock_addr_test *test);103static int sendmsg6_rw_c_prog_load(const struct sock_addr_test *test);
102static int sendmsg6_rw_v4mapped_prog_load(const struct sock_addr_test *test);104static int sendmsg6_rw_v4mapped_prog_load(const struct sock_addr_test *test);
103static int sendmsg6_rw_wildcard_prog_load(const struct sock_addr_test *test);105static int sendmsg6_rw_wildcard_prog_load(const struct sock_addr_test *test);
@@ -573,8 +575,8 @@ static struct sock_addr_test tests[] = { · 469 unmodified lines
573 LOAD_REJECT,575 LOAD_REJECT,
574 },576 },
575 {577 {
576 "recvmsg4: rewrite IP & port (asm)",578 "recvmsg4: rewrite IP & port (C)",
577 recvmsg4_rw_asm_prog_load,579 recvmsg4_rw_c_prog_load,
578 BPF_CGROUP_UDP4_RECVMSG,580 BPF_CGROUP_UDP4_RECVMSG,
579 BPF_CGROUP_UDP4_RECVMSG,581 BPF_CGROUP_UDP4_RECVMSG,
580 AF_INET,582 AF_INET,
@@ -587,8 +589,8 @@ static struct sock_addr_test tests[] = { · 6 unmodified lines
587 SUCCESS,589 SUCCESS,
588 },590 },
589 {591 {
590 "recvmsg6: rewrite IP & port (asm)",592 "recvmsg6: rewrite IP & port (C)",
591 recvmsg6_rw_asm_prog_load,593 recvmsg6_rw_c_prog_load,
592 BPF_CGROUP_UDP6_RECVMSG,594 BPF_CGROUP_UDP6_RECVMSG,
593 BPF_CGROUP_UDP6_RECVMSG,595 BPF_CGROUP_UDP6_RECVMSG,
594 AF_INET6,596 AF_INET6,
@@ -786,45 +788,9 @@ static int sendmsg4_rw_asm_prog_load(const struct sock_addr_test *test) · 191 unmodified lines
786 return load_insns(test, insns, sizeof(insns) / sizeof(struct bpf_insn));788 return load_insns(test, insns, sizeof(insns) / sizeof(struct bpf_insn));
787}789}
788 790
789static int recvmsg4_rw_asm_prog_load(const struct sock_addr_test *test)791static int recvmsg4_rw_c_prog_load(const struct sock_addr_test *test)
790{792{
791 struct sockaddr_in src4_rw_addr;793 return load_path(test, RECVMSG4_PROG_PATH);
792
793 if (mk_sockaddr(AF_INET, SERV4_IP, SERV4_PORT,
794 (struct sockaddr *)&src4_rw_addr,
795 sizeof(src4_rw_addr)) == -1)
796 return -1;
797
798 struct bpf_insn insns[] = {
799 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
800
801 /* if (sk.family == AF_INET && */
802 BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_6,
803 offsetof(struct bpf_sock_addr, family)),
804 BPF_JMP_IMM(BPF_JNE, BPF_REG_7, AF_INET, 6),
805
806 /* sk.type == SOCK_DGRAM) { */
807 BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_6,
808 offsetof(struct bpf_sock_addr, type)),
809 BPF_JMP_IMM(BPF_JNE, BPF_REG_7, SOCK_DGRAM, 4),
810
811 /* user_ip4 = src4_rw_addr.sin_addr */
812 BPF_MOV32_IMM(BPF_REG_7, src4_rw_addr.sin_addr.s_addr),
813 BPF_STX_MEM(BPF_W, BPF_REG_6, BPF_REG_7,
814 offsetof(struct bpf_sock_addr, user_ip4)),
815
816 /* user_port = src4_rw_addr.sin_port */
817 BPF_MOV32_IMM(BPF_REG_7, src4_rw_addr.sin_port),
818 BPF_STX_MEM(BPF_W, BPF_REG_6, BPF_REG_7,
819 offsetof(struct bpf_sock_addr, user_port)),
820 /* } */
821
822 /* return 1 */
823 BPF_MOV64_IMM(BPF_REG_0, 1),
824 BPF_EXIT_INSN(),
825 };
826
827 return load_insns(test, insns, sizeof(insns) / sizeof(struct bpf_insn));
828}794}
829 795
830static int sendmsg4_rw_c_prog_load(const struct sock_addr_test *test)796static int sendmsg4_rw_c_prog_load(const struct sock_addr_test *test)
@@ -890,37 +856,9 @@ static int sendmsg6_rw_asm_prog_load(const struct sock_addr_test *test) · 59 unmodified lines
890 return sendmsg6_rw_dst_asm_prog_load(test, SERV6_REWRITE_IP);856 return sendmsg6_rw_dst_asm_prog_load(test, SERV6_REWRITE_IP);
891}857}
892 858
893static int recvmsg6_rw_asm_prog_load(const struct sock_addr_test *test)859static int recvmsg6_rw_c_prog_load(const struct sock_addr_test *test)
894{860{
895 struct sockaddr_in6 src6_rw_addr;861 return load_path(test, RECVMSG6_PROG_PATH);
896
897 if (mk_sockaddr(AF_INET6, SERV6_IP, SERV6_PORT,
898 (struct sockaddr *)&src6_rw_addr,
899 sizeof(src6_rw_addr)) == -1)
900 return -1;
901
902 struct bpf_insn insns[] = {
903 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
904
905 /* if (sk.family == AF_INET6) { */
906 BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_6,
907 offsetof(struct bpf_sock_addr, family)),
908 BPF_JMP_IMM(BPF_JNE, BPF_REG_7, AF_INET6, 10),
909
910 STORE_IPV6(user_ip6, src6_rw_addr.sin6_addr.s6_addr32),
911
912 /* user_port = dst6_rw_addr.sin6_port */
913 BPF_MOV32_IMM(BPF_REG_7, src6_rw_addr.sin6_port),
914 BPF_STX_MEM(BPF_W, BPF_REG_6, BPF_REG_7,
915 offsetof(struct bpf_sock_addr, user_port)),
916 /* } */
917
918 /* return 1 */
919 BPF_MOV64_IMM(BPF_REG_0, 1),
920 BPF_EXIT_INSN(),
921 };
922
923 return load_insns(test, insns, sizeof(insns) / sizeof(struct bpf_insn));
924}862}
925 863
926static int sendmsg6_rw_v4mapped_prog_load(const struct sock_addr_test *test)864static int sendmsg6_rw_v4mapped_prog_load(const struct sock_addr_test *test)
tools/testing/selftests/bpf/test_socket_cookie.c+0−208
@@ -1,208 +0,0 @@
1// SPDX-License-Identifier: GPL-2.0
2// Copyright (c) 2018 Facebook
3
4#include <string.h>
5#include <unistd.h>
6
7#include <arpa/inet.h>
8#include <netinet/in.h>
9#include <sys/types.h>
10#include <sys/socket.h>
11
12#include <bpf/bpf.h>
13#include <bpf/libbpf.h>
14
15#include "bpf_rlimit.h"
16#include "cgroup_helpers.h"
17
18#define CG_PATH "/foo"
19#define SOCKET_COOKIE_PROG "./socket_cookie_prog.o"
20
21struct socket_cookie {
22 __u64 cookie_key;
23 __u32 cookie_value;
24};
25
26static int start_server(void)
27{
28 struct sockaddr_in6 addr;
29 int fd;
30
31 fd = socket(AF_INET6, SOCK_STREAM, 0);
32 if (fd == -1) {
33 log_err("Failed to create server socket");
34 goto out;
35 }
36
37 memset(&addr, 0, sizeof(addr));
38 addr.sin6_family = AF_INET6;
39 addr.sin6_addr = in6addr_loopback;
40 addr.sin6_port = 0;
41
42 if (bind(fd, (const struct sockaddr *)&addr, sizeof(addr)) == -1) {
43 log_err("Failed to bind server socket");
44 goto close_out;
45 }
46
47 if (listen(fd, 128) == -1) {
48 log_err("Failed to listen on server socket");
49 goto close_out;
50 }
51
52 goto out;
53
54close_out:
55 close(fd);
56 fd = -1;
57out:
58 return fd;
59}
60
61static int connect_to_server(int server_fd)
62{
63 struct sockaddr_storage addr;
64 socklen_t len = sizeof(addr);
65 int fd;
66
67 fd = socket(AF_INET6, SOCK_STREAM, 0);
68 if (fd == -1) {
69 log_err("Failed to create client socket");
70 goto out;
71 }
72
73 if (getsockname(server_fd, (struct sockaddr *)&addr, &len)) {
74 log_err("Failed to get server addr");
75 goto close_out;
76 }
77
78 if (connect(fd, (const struct sockaddr *)&addr, len) == -1) {
79 log_err("Fail to connect to server");
80 goto close_out;
81 }
82
83 goto out;
84
85close_out:
86 close(fd);
87 fd = -1;
88out:
89 return fd;
90}
91
92static int validate_map(struct bpf_map *map, int client_fd)
93{
94 __u32 cookie_expected_value;
95 struct sockaddr_in6 addr;
96 socklen_t len = sizeof(addr);
97 struct socket_cookie val;
98 int err = 0;
99 int map_fd;
100
101 if (!map) {
102 log_err("Map not found in BPF object");
103 goto err;
104 }
105
106 map_fd = bpf_map__fd(map);
107
108 err = bpf_map_lookup_elem(map_fd, &client_fd, &val);
109
110 err = getsockname(client_fd, (struct sockaddr *)&addr, &len);
111 if (err) {
112 log_err("Can't get client local addr");
113 goto out;
114 }
115
116 cookie_expected_value = (ntohs(addr.sin6_port) << 8) | 0xFF;
117 if (val.cookie_value != cookie_expected_value) {
118 log_err("Unexpected value in map: %x != %x", val.cookie_value,
119 cookie_expected_value);
120 goto err;
121 }
122
123 goto out;
124err:
125 err = -1;
126out:
127 return err;
128}
129
130static int run_test(int cgfd)
131{
132 enum bpf_attach_type attach_type;
133 struct bpf_prog_load_attr attr;
134 struct bpf_program *prog;
135 struct bpf_object *pobj;
136 const char *prog_name;
137 int server_fd = -1;
138 int client_fd = -1;
139 int prog_fd = -1;
140 int err = 0;
141
142 memset(&attr, 0, sizeof(attr));
143 attr.file = SOCKET_COOKIE_PROG;
144 attr.prog_type = BPF_PROG_TYPE_UNSPEC;
145 attr.prog_flags = BPF_F_TEST_RND_HI32;
146
147 err = bpf_prog_load_xattr(&attr, &pobj, &prog_fd);
148 if (err) {
149 log_err("Failed to load %s", attr.file);
150 goto out;
151 }
152
153 bpf_object__for_each_program(prog, pobj) {
154 prog_name = bpf_program__section_name(prog);
155
156 if (libbpf_attach_type_by_name(prog_name, &attach_type))
157 goto err;
158
159 err = bpf_prog_attach(bpf_program__fd(prog), cgfd, attach_type,
160 BPF_F_ALLOW_OVERRIDE);
161 if (err) {
162 log_err("Failed to attach prog %s", prog_name);
163 goto out;
164 }
165 }
166
167 server_fd = start_server();
168 if (server_fd == -1)
169 goto err;
170
171 client_fd = connect_to_server(server_fd);
172 if (client_fd == -1)
173 goto err;
174
175 if (validate_map(bpf_map__next(NULL, pobj), client_fd))
176 goto err;
177
178 goto out;
179err:
180 err = -1;
181out:
182 close(client_fd);
183 close(server_fd);
184 bpf_object__close(pobj);
185 printf("%s\n", err ? "FAILED" : "PASSED");
186 return err;
187}
188
189int main(int argc, char **argv)
190{
191 int cgfd = -1;
192 int err = 0;
193
194 cgfd = cgroup_setup_and_join(CG_PATH);
195 if (cgfd < 0)
196 goto err;
197
198 if (run_test(cgfd))
199 goto err;
200
201 goto out;
202err:
203 err = -1;
204out:
205 close(cgfd);
206 cleanup_cgroup_environment();
207 return err;
208}
tools/testing/selftests/bpf/test_verifier.c+85−3
@@ -88,6 +88,10 @@ struct bpf_test { · 87 unmodified lines
88 int fixup_map_event_output[MAX_FIXUPS];88 int fixup_map_event_output[MAX_FIXUPS];
89 int fixup_map_reuseport_array[MAX_FIXUPS];89 int fixup_map_reuseport_array[MAX_FIXUPS];
90 int fixup_map_ringbuf[MAX_FIXUPS];90 int fixup_map_ringbuf[MAX_FIXUPS];
91 /* Expected verifier log output for result REJECT or VERBOSE_ACCEPT.
92 * Can be a tab-separated sequence of expected strings. An empty string
93 * means no log verification.
94 */
91 const char *errstr;95 const char *errstr;
92 const char *errstr_unpriv;96 const char *errstr_unpriv;
93 uint32_t insn_processed;97 uint32_t insn_processed;
@@ -297,6 +301,78 @@ static void bpf_fill_scale(struct bpf_test *self) · 203 unmodified lines
297 }301 }
298}302}
299 303
304static int bpf_fill_torturous_jumps_insn_1(struct bpf_insn *insn)
305{
306 unsigned int len = 259, hlen = 128;
307 int i;
308
309 insn[0] = BPF_EMIT_CALL(BPF_FUNC_get_prandom_u32);
310 for (i = 1; i <= hlen; i++) {
311 insn[i] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, i, hlen);
312 insn[i + hlen] = BPF_JMP_A(hlen - i);
313 }
314 insn[len - 2] = BPF_MOV64_IMM(BPF_REG_0, 1);
315 insn[len - 1] = BPF_EXIT_INSN();
316
317 return len;
318}
319
320static int bpf_fill_torturous_jumps_insn_2(struct bpf_insn *insn)
321{
322 unsigned int len = 4100, jmp_off = 2048;
323 int i, j;
324
325 insn[0] = BPF_EMIT_CALL(BPF_FUNC_get_prandom_u32);
326 for (i = 1; i <= jmp_off; i++) {
327 insn[i] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, i, jmp_off);
328 }
329 insn[i++] = BPF_JMP_A(jmp_off);
330 for (; i <= jmp_off * 2 + 1; i+=16) {
331 for (j = 0; j < 16; j++) {
332 insn[i + j] = BPF_JMP_A(16 - j - 1);
333 }
334 }
335
336 insn[len - 2] = BPF_MOV64_IMM(BPF_REG_0, 2);
337 insn[len - 1] = BPF_EXIT_INSN();
338
339 return len;
340}
341
342static void bpf_fill_torturous_jumps(struct bpf_test *self)
343{
344 struct bpf_insn *insn = self->fill_insns;
345 int i = 0;
346
347 switch (self->retval) {
348 case 1:
349 self->prog_len = bpf_fill_torturous_jumps_insn_1(insn);
350 return;
351 case 2:
352 self->prog_len = bpf_fill_torturous_jumps_insn_2(insn);
353 return;
354 case 3:
355 /* main */
356 insn[i++] = BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 4);
357 insn[i++] = BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 262);
358 insn[i++] = BPF_ST_MEM(BPF_B, BPF_REG_10, -32, 0);
359 insn[i++] = BPF_MOV64_IMM(BPF_REG_0, 3);
360 insn[i++] = BPF_EXIT_INSN();
361
362 /* subprog 1 */
363 i += bpf_fill_torturous_jumps_insn_1(insn + i);
364
365 /* subprog 2 */
366 i += bpf_fill_torturous_jumps_insn_2(insn + i);
367
368 self->prog_len = i;
369 return;
370 default:
371 self->prog_len = 0;
372 break;
373 }
374}
375
300/* BPF_SK_LOOKUP contains 13 instructions, if you need to fix up maps */376/* BPF_SK_LOOKUP contains 13 instructions, if you need to fix up maps */
301#define BPF_SK_LOOKUP(func) \377#define BPF_SK_LOOKUP(func) \
302 /* struct bpf_sock_tuple tuple = {} */ \378 /* struct bpf_sock_tuple tuple = {} */ \
@@ -923,13 +999,19 @@ static int do_prog_test_run(int fd_prog, bool unpriv, uint32_t expected_val, · 620 unmodified lines
923 return 0;999 return 0;
924}1000}
925 1001
1002/* Returns true if every part of exp (tab-separated) appears in log, in order.
1003 *
1004 * If exp is an empty string, returns true.
1005 */
926static bool cmp_str_seq(const char *log, const char *exp)1006static bool cmp_str_seq(const char *log, const char *exp)
927{1007{
928 char needle[80];1008 char needle[200];
929 const char *p, *q;1009 const char *p, *q;
930 int len;1010 int len;
931 1011
932 do {1012 do {
1013 if (!strlen(exp))
1014 break;
933 p = strchr(exp, '\t');1015 p = strchr(exp, '\t');
934 if (!p)1016 if (!p)
935 p = exp + strlen(exp);1017 p = exp + strlen(exp);
@@ -943,7 +1025,7 @@ static bool cmp_str_seq(const char *log, const char *exp) · 7 unmodified lines
943 needle[len] = 0;1025 needle[len] = 0;
944 q = strstr(log, needle);1026 q = strstr(log, needle);
945 if (!q) {1027 if (!q) {
946 printf("FAIL\nUnexpected verifier log in successful load!\n"1028 printf("FAIL\nUnexpected verifier log!\n"
947 "EXP: %s\nRES:\n", needle);1029 "EXP: %s\nRES:\n", needle);
948 return false;1030 return false;
949 }1031 }
@@ -1058,7 +1140,7 @@ static void do_test_single(struct bpf_test *test, bool unpriv, · 108 unmodified lines
1058 printf("FAIL\nUnexpected success to load!\n");1140 printf("FAIL\nUnexpected success to load!\n");
1059 goto fail_log;1141 goto fail_log;
1060 }1142 }
1061 if (!expected_err || !strstr(bpf_vlog, expected_err)) {1143 if (!expected_err || !cmp_str_seq(bpf_vlog, expected_err)) {
1062 printf("FAIL\nUnexpected error message!\n\tEXP: %s\n\tRES: %s\n",1144 printf("FAIL\nUnexpected error message!\n\tEXP: %s\n\tRES: %s\n",
1063 expected_err, bpf_vlog);1145 expected_err, bpf_vlog);
1064 goto fail_log;1146 goto fail_log;
tools/testing/selftests/bpf/verifier/atomic_and.c+1−1
@@ -7,7 +7,7 @@ · 6 unmodified lines
7 BPF_MOV64_IMM(BPF_REG_1, 0x011),7 BPF_MOV64_IMM(BPF_REG_1, 0x011),
8 BPF_ATOMIC_OP(BPF_DW, BPF_AND, BPF_REG_10, BPF_REG_1, -8),8 BPF_ATOMIC_OP(BPF_DW, BPF_AND, BPF_REG_10, BPF_REG_1, -8),
9 /* if (val != 0x010) exit(2); */9 /* if (val != 0x010) exit(2); */
10 BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_10, -8),10 BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_10, -8),
11 BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0x010, 2),11 BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0x010, 2),
12 BPF_MOV64_IMM(BPF_REG_0, 2),12 BPF_MOV64_IMM(BPF_REG_0, 2),
13 BPF_EXIT_INSN(),13 BPF_EXIT_INSN(),
tools/testing/selftests/bpf/verifier/atomic_bounds.c+27−0
@@ -0,0 +1,27 @@
1{
2 "BPF_ATOMIC bounds propagation, mem->reg",
3 .insns = {
4 /* a = 0; */
5 /*
6 * Note this is implemented with two separate instructions,
7 * where you might think one would suffice:
8 *
9 * BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
10 *
11 * This is because BPF_ST_MEM doesn't seem to set the stack slot
12 * type to 0 when storing an immediate.
13 */
14 BPF_MOV64_IMM(BPF_REG_0, 0),
15 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, -8),
16 /* b = atomic_fetch_add(&a, 1); */
17 BPF_MOV64_IMM(BPF_REG_1, 1),
18 BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, BPF_REG_10, BPF_REG_1, -8),
19 /* Verifier should be able to tell that this infinite loop isn't reachable. */
20 /* if (b) while (true) continue; */
21 BPF_JMP_IMM(BPF_JNE, BPF_REG_1, 0, -1),
22 BPF_EXIT_INSN(),
23 },
24 .result = ACCEPT,
25 .result_unpriv = REJECT,
26 .errstr_unpriv = "back-edge",
27},
tools/testing/selftests/bpf/verifier/atomic_or.c+1−1
@@ -7,7 +7,7 @@ · 6 unmodified lines
7 BPF_MOV64_IMM(BPF_REG_1, 0x011),7 BPF_MOV64_IMM(BPF_REG_1, 0x011),
8 BPF_ATOMIC_OP(BPF_DW, BPF_OR, BPF_REG_10, BPF_REG_1, -8),8 BPF_ATOMIC_OP(BPF_DW, BPF_OR, BPF_REG_10, BPF_REG_1, -8),
9 /* if (val != 0x111) exit(2); */9 /* if (val != 0x111) exit(2); */
10 BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_10, -8),10 BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_10, -8),
11 BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0x111, 2),11 BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0x111, 2),
12 BPF_MOV64_IMM(BPF_REG_0, 2),12 BPF_MOV64_IMM(BPF_REG_0, 2),
13 BPF_EXIT_INSN(),13 BPF_EXIT_INSN(),
tools/testing/selftests/bpf/verifier/atomic_xor.c+1−1
@@ -7,7 +7,7 @@ · 6 unmodified lines
7 BPF_MOV64_IMM(BPF_REG_1, 0x011),7 BPF_MOV64_IMM(BPF_REG_1, 0x011),
8 BPF_ATOMIC_OP(BPF_DW, BPF_XOR, BPF_REG_10, BPF_REG_1, -8),8 BPF_ATOMIC_OP(BPF_DW, BPF_XOR, BPF_REG_10, BPF_REG_1, -8),
9 /* if (val != 0x101) exit(2); */9 /* if (val != 0x101) exit(2); */
10 BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_10, -8),10 BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_10, -8),
11 BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0x101, 2),11 BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0x101, 2),
12 BPF_MOV64_IMM(BPF_REG_0, 2),12 BPF_MOV64_IMM(BPF_REG_0, 2),
13 BPF_EXIT_INSN(),13 BPF_EXIT_INSN(),
tools/testing/selftests/bpf/verifier/basic_stack.c+1−1
@@ -4,7 +4,7 @@ · 3 unmodified lines
4 BPF_ST_MEM(BPF_DW, BPF_REG_10, 8, 0),4 BPF_ST_MEM(BPF_DW, BPF_REG_10, 8, 0),
5 BPF_EXIT_INSN(),5 BPF_EXIT_INSN(),
6 },6 },
7 .errstr = "invalid stack",7 .errstr = "invalid write to stack",
8 .result = REJECT,8 .result = REJECT,
9},9},
10{10{
tools/testing/selftests/bpf/verifier/calls.c+2−2
@@ -1228,7 +1228,7 @@ · 1227 unmodified lines
1228 .prog_type = BPF_PROG_TYPE_XDP,1228 .prog_type = BPF_PROG_TYPE_XDP,
1229 .fixup_map_hash_8b = { 23 },1229 .fixup_map_hash_8b = { 23 },
1230 .result = REJECT,1230 .result = REJECT,
1231 .errstr = "invalid read from stack off -16+0 size 8",1231 .errstr = "invalid read from stack R7 off=-16 size=8",
1232},1232},
1233{1233{
1234 "calls: two calls that receive map_value via arg=ptr_stack_of_caller. test1",1234 "calls: two calls that receive map_value via arg=ptr_stack_of_caller. test1",
@@ -1958,7 +1958,7 @@ · 723 unmodified lines
1958 BPF_EXIT_INSN(),1958 BPF_EXIT_INSN(),
1959 },1959 },
1960 .fixup_map_hash_48b = { 6 },1960 .fixup_map_hash_48b = { 6 },
1961 .errstr = "invalid indirect read from stack off -8+0 size 8",1961 .errstr = "invalid indirect read from stack R2 off -8+0 size 8",
1962 .result = REJECT,1962 .result = REJECT,
1963 .prog_type = BPF_PROG_TYPE_XDP,1963 .prog_type = BPF_PROG_TYPE_XDP,
1964},1964},
tools/testing/selftests/bpf/verifier/const_or.c+2−2
@@ -23,7 +23,7 @@ · 22 unmodified lines
23 BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel),23 BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel),
24 BPF_EXIT_INSN(),24 BPF_EXIT_INSN(),
25 },25 },
26 .errstr = "invalid stack type R1 off=-48 access_size=58",26 .errstr = "invalid indirect access to stack R1 off=-48 size=58",
27 .result = REJECT,27 .result = REJECT,
28 .prog_type = BPF_PROG_TYPE_TRACEPOINT,28 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
29},29},
@@ -54,7 +54,7 @@ · 24 unmodified lines
54 BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel),54 BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel),
55 BPF_EXIT_INSN(),55 BPF_EXIT_INSN(),
56 },56 },
57 .errstr = "invalid stack type R1 off=-48 access_size=58",57 .errstr = "invalid indirect access to stack R1 off=-48 size=58",
58 .result = REJECT,58 .result = REJECT,
59 .prog_type = BPF_PROG_TYPE_TRACEPOINT,59 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
60},60},
tools/testing/selftests/bpf/verifier/helper_access_var_len.c+6−6
@@ -39,7 +39,7 @@ · 38 unmodified lines
39 BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel),39 BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel),
40 BPF_EXIT_INSN(),40 BPF_EXIT_INSN(),
41 },41 },
42 .errstr = "invalid indirect read from stack off -64+0 size 64",42 .errstr = "invalid indirect read from stack R1 off -64+0 size 64",
43 .result = REJECT,43 .result = REJECT,
44 .prog_type = BPF_PROG_TYPE_TRACEPOINT,44 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
45},45},
@@ -59,7 +59,7 @@ · 13 unmodified lines
59 BPF_MOV64_IMM(BPF_REG_0, 0),59 BPF_MOV64_IMM(BPF_REG_0, 0),
60 BPF_EXIT_INSN(),60 BPF_EXIT_INSN(),
61 },61 },
62 .errstr = "invalid stack type R1 off=-64 access_size=65",62 .errstr = "invalid indirect access to stack R1 off=-64 size=65",
63 .result = REJECT,63 .result = REJECT,
64 .prog_type = BPF_PROG_TYPE_TRACEPOINT,64 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
65},65},
@@ -136,7 +136,7 @@ · 70 unmodified lines
136 BPF_MOV64_IMM(BPF_REG_0, 0),136 BPF_MOV64_IMM(BPF_REG_0, 0),
137 BPF_EXIT_INSN(),137 BPF_EXIT_INSN(),
138 },138 },
139 .errstr = "invalid stack type R1 off=-64 access_size=65",139 .errstr = "invalid indirect access to stack R1 off=-64 size=65",
140 .result = REJECT,140 .result = REJECT,
141 .prog_type = BPF_PROG_TYPE_TRACEPOINT,141 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
142},142},
@@ -156,7 +156,7 @@ · 13 unmodified lines
156 BPF_MOV64_IMM(BPF_REG_0, 0),156 BPF_MOV64_IMM(BPF_REG_0, 0),
157 BPF_EXIT_INSN(),157 BPF_EXIT_INSN(),
158 },158 },
159 .errstr = "invalid stack type R1 off=-64 access_size=65",159 .errstr = "invalid indirect access to stack R1 off=-64 size=65",
160 .result = REJECT,160 .result = REJECT,
161 .prog_type = BPF_PROG_TYPE_TRACEPOINT,161 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
162},162},
@@ -194,7 +194,7 @@ · 31 unmodified lines
194 BPF_MOV64_IMM(BPF_REG_0, 0),194 BPF_MOV64_IMM(BPF_REG_0, 0),
195 BPF_EXIT_INSN(),195 BPF_EXIT_INSN(),
196 },196 },
197 .errstr = "invalid indirect read from stack off -64+0 size 64",197 .errstr = "invalid indirect read from stack R1 off -64+0 size 64",
198 .result = REJECT,198 .result = REJECT,
199 .prog_type = BPF_PROG_TYPE_TRACEPOINT,199 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
200},200},
@@ -584,7 +584,7 @@ · 383 unmodified lines
584 BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, -16),584 BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, -16),
585 BPF_EXIT_INSN(),585 BPF_EXIT_INSN(),
586 },586 },
587 .errstr = "invalid indirect read from stack off -64+32 size 64",587 .errstr = "invalid indirect read from stack R1 off -64+32 size 64",
588 .result = REJECT,588 .result = REJECT,
589 .prog_type = BPF_PROG_TYPE_TRACEPOINT,589 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
590},590},
tools/testing/selftests/bpf/verifier/int_ptr.c+3−3
@@ -27,7 +27,7 @@ · 26 unmodified lines
27 },27 },
28 .result = REJECT,28 .result = REJECT,
29 .prog_type = BPF_PROG_TYPE_CGROUP_SYSCTL,29 .prog_type = BPF_PROG_TYPE_CGROUP_SYSCTL,
30 .errstr = "invalid indirect read from stack off -16+0 size 8",30 .errstr = "invalid indirect read from stack R4 off -16+0 size 8",
31},31},
32{32{
33 "ARG_PTR_TO_LONG half-uninitialized",33 "ARG_PTR_TO_LONG half-uninitialized",
@@ -59,7 +59,7 @@ · 25 unmodified lines
59 },59 },
60 .result = REJECT,60 .result = REJECT,
61 .prog_type = BPF_PROG_TYPE_CGROUP_SYSCTL,61 .prog_type = BPF_PROG_TYPE_CGROUP_SYSCTL,
62 .errstr = "invalid indirect read from stack off -16+4 size 8",62 .errstr = "invalid indirect read from stack R4 off -16+4 size 8",
63},63},
64{64{
65 "ARG_PTR_TO_LONG misaligned",65 "ARG_PTR_TO_LONG misaligned",
@@ -125,7 +125,7 @@ · 59 unmodified lines
125 },125 },
126 .result = REJECT,126 .result = REJECT,
127 .prog_type = BPF_PROG_TYPE_CGROUP_SYSCTL,127 .prog_type = BPF_PROG_TYPE_CGROUP_SYSCTL,
128 .errstr = "invalid stack type R4 off=-4 access_size=8",128 .errstr = "invalid indirect access to stack R4 off=-4 size=8",
129},129},
130{130{
131 "ARG_PTR_TO_LONG initialized",131 "ARG_PTR_TO_LONG initialized",
tools/testing/selftests/bpf/verifier/jit.c+24−0
@@ -105,3 +105,27 @@ · 104 unmodified lines
105 .result = ACCEPT,105 .result = ACCEPT,
106 .retval = 2,106 .retval = 2,
107},107},
108{
109 "jit: torturous jumps, imm8 nop jmp and pure jump padding",
110 .insns = { },
111 .fill_helper = bpf_fill_torturous_jumps,
112 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
113 .result = ACCEPT,
114 .retval = 1,
115},
116{
117 "jit: torturous jumps, imm32 nop jmp and jmp_cond padding",
118 .insns = { },
119 .fill_helper = bpf_fill_torturous_jumps,
120 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
121 .result = ACCEPT,
122 .retval = 2,
123},
124{
125 "jit: torturous jumps in subprog",
126 .insns = { },
127 .fill_helper = bpf_fill_torturous_jumps,
128 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
129 .result = ACCEPT,
130 .retval = 3,
131},
tools/testing/selftests/bpf/verifier/raw_stack.c+5−5
@@ -11,7 +11,7 @@ · 10 unmodified lines
11 BPF_EXIT_INSN(),11 BPF_EXIT_INSN(),
12 },12 },
13 .result = REJECT,13 .result = REJECT,
14 .errstr = "invalid read from stack off -8+0 size 8",14 .errstr = "invalid read from stack R6 off=-8 size=8",
15 .prog_type = BPF_PROG_TYPE_SCHED_CLS,15 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
16},16},
17{17{
@@ -59,7 +59,7 @@ · 41 unmodified lines
59 BPF_EXIT_INSN(),59 BPF_EXIT_INSN(),
60 },60 },
61 .result = REJECT,61 .result = REJECT,
62 .errstr = "invalid stack type R3",62 .errstr = "invalid zero-sized read",
63 .prog_type = BPF_PROG_TYPE_SCHED_CLS,63 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
64},64},
65{65{
@@ -205,7 +205,7 @@ · 139 unmodified lines
205 BPF_EXIT_INSN(),205 BPF_EXIT_INSN(),
206 },206 },
207 .result = REJECT,207 .result = REJECT,
208 .errstr = "invalid stack type R3 off=-513 access_size=8",208 .errstr = "invalid indirect access to stack R3 off=-513 size=8",
209 .prog_type = BPF_PROG_TYPE_SCHED_CLS,209 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
210},210},
211{211{
@@ -221,7 +221,7 @@ · 9 unmodified lines
221 BPF_EXIT_INSN(),221 BPF_EXIT_INSN(),
222 },222 },
223 .result = REJECT,223 .result = REJECT,
224 .errstr = "invalid stack type R3 off=-1 access_size=8",224 .errstr = "invalid indirect access to stack R3 off=-1 size=8",
225 .prog_type = BPF_PROG_TYPE_SCHED_CLS,225 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
226},226},
227{227{
@@ -285,7 +285,7 @@ · 57 unmodified lines
285 BPF_EXIT_INSN(),285 BPF_EXIT_INSN(),
286 },286 },
287 .result = REJECT,287 .result = REJECT,
288 .errstr = "invalid stack type R3 off=-512 access_size=0",288 .errstr = "invalid zero-sized read",
289 .prog_type = BPF_PROG_TYPE_SCHED_CLS,289 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
290},290},
291{291{
tools/testing/selftests/bpf/verifier/stack_ptr.c+13−9
@@ -44,7 +44,7 @@ · 43 unmodified lines
44 BPF_EXIT_INSN(),44 BPF_EXIT_INSN(),
45 },45 },
46 .result = REJECT,46 .result = REJECT,
47 .errstr = "invalid stack off=-79992 size=8",47 .errstr = "invalid write to stack R1 off=-79992 size=8",
48 .errstr_unpriv = "R1 stack pointer arithmetic goes out of range",48 .errstr_unpriv = "R1 stack pointer arithmetic goes out of range",
49},49},
50{50{
@@ -57,7 +57,7 @@ · 6 unmodified lines
57 BPF_EXIT_INSN(),57 BPF_EXIT_INSN(),
58 },58 },
59 .result = REJECT,59 .result = REJECT,
60 .errstr = "invalid stack off=0 size=8",60 .errstr = "invalid write to stack R1 off=0 size=8",
61},61},
62{62{
63 "PTR_TO_STACK check high 1",63 "PTR_TO_STACK check high 1",
@@ -106,7 +106,7 @@ · 42 unmodified lines
106 BPF_EXIT_INSN(),106 BPF_EXIT_INSN(),
107 },107 },
108 .errstr_unpriv = "R1 stack pointer arithmetic goes out of range",108 .errstr_unpriv = "R1 stack pointer arithmetic goes out of range",
109 .errstr = "invalid stack off=0 size=1",109 .errstr = "invalid write to stack R1 off=0 size=1",
110 .result = REJECT,110 .result = REJECT,
111},111},
112{112{
@@ -119,7 +119,8 @@ · 6 unmodified lines
119 BPF_EXIT_INSN(),119 BPF_EXIT_INSN(),
120 },120 },
121 .result = REJECT,121 .result = REJECT,
122 .errstr = "invalid stack off",122 .errstr_unpriv = "R1 stack pointer arithmetic goes out of range",
123 .errstr = "invalid write to stack R1",
123},124},
124{125{
125 "PTR_TO_STACK check high 6",126 "PTR_TO_STACK check high 6",
@@ -131,7 +132,8 @@ · 5 unmodified lines
131 BPF_EXIT_INSN(),132 BPF_EXIT_INSN(),
132 },133 },
133 .result = REJECT,134 .result = REJECT,
134 .errstr = "invalid stack off",135 .errstr_unpriv = "R1 stack pointer arithmetic goes out of range",
136 .errstr = "invalid write to stack",
135},137},
136{138{
137 "PTR_TO_STACK check high 7",139 "PTR_TO_STACK check high 7",
@@ -183,7 +185,7 @@ · 45 unmodified lines
183 BPF_EXIT_INSN(),185 BPF_EXIT_INSN(),
184 },186 },
185 .errstr_unpriv = "R1 stack pointer arithmetic goes out of range",187 .errstr_unpriv = "R1 stack pointer arithmetic goes out of range",
186 .errstr = "invalid stack off=-513 size=1",188 .errstr = "invalid write to stack R1 off=-513 size=1",
187 .result = REJECT,189 .result = REJECT,
188},190},
189{191{
@@ -208,7 +210,8 @@ · 18 unmodified lines
208 BPF_EXIT_INSN(),210 BPF_EXIT_INSN(),
209 },211 },
210 .result = REJECT,212 .result = REJECT,
211 .errstr = "invalid stack off",213 .errstr_unpriv = "R1 stack pointer arithmetic goes out of range",
214 .errstr = "invalid write to stack",
212},215},
213{216{
214 "PTR_TO_STACK check low 6",217 "PTR_TO_STACK check low 6",
@@ -220,7 +223,8 @@ · 5 unmodified lines
220 BPF_EXIT_INSN(),223 BPF_EXIT_INSN(),
221 },224 },
222 .result = REJECT,225 .result = REJECT,
223 .errstr = "invalid stack off",226 .errstr = "invalid write to stack",
227 .errstr_unpriv = "R1 stack pointer arithmetic goes out of range",
224},228},
225{229{
226 "PTR_TO_STACK check low 7",230 "PTR_TO_STACK check low 7",
@@ -292,7 +296,7 @@ · 65 unmodified lines
292 BPF_EXIT_INSN(),296 BPF_EXIT_INSN(),
293 },297 },
294 .result_unpriv = REJECT,298 .result_unpriv = REJECT,
295 .errstr_unpriv = "invalid stack off=0 size=1",299 .errstr_unpriv = "invalid write to stack R1 off=0 size=1",
296 .result = ACCEPT,300 .result = ACCEPT,
297 .retval = 42,301 .retval = 42,
298},302},
tools/testing/selftests/bpf/verifier/unpriv.c+1−1
@@ -108,7 +108,7 @@ · 107 unmodified lines
108 BPF_EXIT_INSN(),108 BPF_EXIT_INSN(),
109 },109 },
110 .fixup_map_hash_8b = { 3 },110 .fixup_map_hash_8b = { 3 },
111 .errstr_unpriv = "invalid indirect read from stack off -8+0 size 8",111 .errstr_unpriv = "invalid indirect read from stack R2 off -8+0 size 8",
112 .result_unpriv = REJECT,112 .result_unpriv = REJECT,
113 .result = ACCEPT,113 .result = ACCEPT,
114},114},
tools/testing/selftests/bpf/verifier/var_off.c+105−10
@@ -18,7 +18,7 @@ · 17 unmodified lines
18 .prog_type = BPF_PROG_TYPE_LWT_IN,18 .prog_type = BPF_PROG_TYPE_LWT_IN,
19},19},
20{20{
21 "variable-offset stack access",21 "variable-offset stack read, priv vs unpriv",
22 .insns = {22 .insns = {
23 /* Fill the top 8 bytes of the stack */23 /* Fill the top 8 bytes of the stack */
24 BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),24 BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
@@ -31,14 +31,109 @@ · 6 unmodified lines
31 * we don't know which31 * we don't know which
32 */32 */
33 BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_10),33 BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_10),
34 /* dereference it */34 /* dereference it for a stack read */
35 BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_2, 0),
36 BPF_MOV64_IMM(BPF_REG_0, 0),
37 BPF_EXIT_INSN(),
38 },
39 .result = ACCEPT,
40 .result_unpriv = REJECT,
41 .errstr_unpriv = "R2 variable stack access prohibited for !root",
42 .prog_type = BPF_PROG_TYPE_CGROUP_SKB,
43},
44{
45 "variable-offset stack read, uninitialized",
46 .insns = {
47 /* Get an unknown value */
48 BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 0),
49 /* Make it small and 4-byte aligned */
50 BPF_ALU64_IMM(BPF_AND, BPF_REG_2, 4),
51 BPF_ALU64_IMM(BPF_SUB, BPF_REG_2, 8),
52 /* add it to fp. We now have either fp-4 or fp-8, but
53 * we don't know which
54 */
55 BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_10),
56 /* dereference it for a stack read */
35 BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_2, 0),57 BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_2, 0),
58 BPF_MOV64_IMM(BPF_REG_0, 0),
36 BPF_EXIT_INSN(),59 BPF_EXIT_INSN(),
37 },60 },
38 .errstr = "variable stack access var_off=(0xfffffffffffffff8; 0x4)",
39 .result = REJECT,61 .result = REJECT,
62 .errstr = "invalid variable-offset read from stack R2",
40 .prog_type = BPF_PROG_TYPE_LWT_IN,63 .prog_type = BPF_PROG_TYPE_LWT_IN,
41},64},
65{
66 "variable-offset stack write, priv vs unpriv",
67 .insns = {
68 /* Get an unknown value */
69 BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 0),
70 /* Make it small and 8-byte aligned */
71 BPF_ALU64_IMM(BPF_AND, BPF_REG_2, 8),
72 BPF_ALU64_IMM(BPF_SUB, BPF_REG_2, 16),
73 /* Add it to fp. We now have either fp-8 or fp-16, but
74 * we don't know which
75 */
76 BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_10),
77 /* Dereference it for a stack write */
78 BPF_ST_MEM(BPF_DW, BPF_REG_2, 0, 0),
79 /* Now read from the address we just wrote. This shows
80 * that, after a variable-offset write, a priviledged
81 * program can read the slots that were in the range of
82 * that write (even if the verifier doesn't actually know
83 * if the slot being read was really written to or not.
84 */
85 BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_2, 0),
86 BPF_MOV64_IMM(BPF_REG_0, 0),
87 BPF_EXIT_INSN(),
88 },
89 /* Variable stack access is rejected for unprivileged.
90 */
91 .errstr_unpriv = "R2 variable stack access prohibited for !root",
92 .result_unpriv = REJECT,
93 .result = ACCEPT,
94},
95{
96 "variable-offset stack write clobbers spilled regs",
97 .insns = {
98 /* Dummy instruction; needed because we need to patch the next one
99 * and we can't patch the first instruction.
100 */
101 BPF_MOV64_IMM(BPF_REG_6, 0),
102 /* Make R0 a map ptr */
103 BPF_LD_MAP_FD(BPF_REG_0, 0),
104 /* Get an unknown value */
105 BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 0),
106 /* Make it small and 8-byte aligned */
107 BPF_ALU64_IMM(BPF_AND, BPF_REG_2, 8),
108 BPF_ALU64_IMM(BPF_SUB, BPF_REG_2, 16),
109 /* Add it to fp. We now have either fp-8 or fp-16, but
110 * we don't know which.
111 */
112 BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_10),
113 /* Spill R0(map ptr) into stack */
114 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, -8),
115 /* Dereference the unknown value for a stack write */
116 BPF_ST_MEM(BPF_DW, BPF_REG_2, 0, 0),
117 /* Fill the register back into R2 */
118 BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_10, -8),
119 /* Try to dereference R2 for a memory load */
120 BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 8),
121 BPF_EXIT_INSN(),
122 },
123 .fixup_map_hash_8b = { 1 },
124 /* The unpriviledged case is not too interesting; variable
125 * stack access is rejected.
126 */
127 .errstr_unpriv = "R2 variable stack access prohibited for !root",
128 .result_unpriv = REJECT,
129 /* In the priviledged case, dereferencing a spilled-and-then-filled
130 * register is rejected because the previous variable offset stack
131 * write might have overwritten the spilled pointer (i.e. we lose track
132 * of the spilled register when we analyze the write).
133 */
134 .errstr = "R2 invalid mem access 'inv'",
135 .result = REJECT,
136},
42{137{
43 "indirect variable-offset stack access, unbounded",138 "indirect variable-offset stack access, unbounded",
44 .insns = {139 .insns = {
@@ -63,7 +158,7 @@ · 18 unmodified lines
63 BPF_MOV64_IMM(BPF_REG_0, 0),158 BPF_MOV64_IMM(BPF_REG_0, 0),
64 BPF_EXIT_INSN(),159 BPF_EXIT_INSN(),
65 },160 },
66 .errstr = "R4 unbounded indirect variable offset stack access",161 .errstr = "invalid unbounded variable-offset indirect access to stack R4",
67 .result = REJECT,162 .result = REJECT,
68 .prog_type = BPF_PROG_TYPE_SOCK_OPS,163 .prog_type = BPF_PROG_TYPE_SOCK_OPS,
69},164},
@@ -88,7 +183,7 @@ · 18 unmodified lines
88 BPF_EXIT_INSN(),183 BPF_EXIT_INSN(),
89 },184 },
90 .fixup_map_hash_8b = { 5 },185 .fixup_map_hash_8b = { 5 },
91 .errstr = "R2 max value is outside of stack bound",186 .errstr = "invalid variable-offset indirect access to stack R2",
92 .result = REJECT,187 .result = REJECT,
93 .prog_type = BPF_PROG_TYPE_LWT_IN,188 .prog_type = BPF_PROG_TYPE_LWT_IN,
94},189},
@@ -113,7 +208,7 @@ · 18 unmodified lines
113 BPF_EXIT_INSN(),208 BPF_EXIT_INSN(),
114 },209 },
115 .fixup_map_hash_8b = { 5 },210 .fixup_map_hash_8b = { 5 },
116 .errstr = "R2 min value is outside of stack bound",211 .errstr = "invalid variable-offset indirect access to stack R2",
117 .result = REJECT,212 .result = REJECT,
118 .prog_type = BPF_PROG_TYPE_LWT_IN,213 .prog_type = BPF_PROG_TYPE_LWT_IN,
119},214},
@@ -138,7 +233,7 @@ · 18 unmodified lines
138 BPF_EXIT_INSN(),233 BPF_EXIT_INSN(),
139 },234 },
140 .fixup_map_hash_8b = { 5 },235 .fixup_map_hash_8b = { 5 },
141 .errstr = "invalid indirect read from stack var_off",236 .errstr = "invalid indirect read from stack R2 var_off",
142 .result = REJECT,237 .result = REJECT,
143 .prog_type = BPF_PROG_TYPE_LWT_IN,238 .prog_type = BPF_PROG_TYPE_LWT_IN,
144},239},
@@ -163,7 +258,7 @@ · 18 unmodified lines
163 BPF_EXIT_INSN(),258 BPF_EXIT_INSN(),
164 },259 },
165 .fixup_map_hash_8b = { 5 },260 .fixup_map_hash_8b = { 5 },
166 .errstr = "invalid indirect read from stack var_off",261 .errstr = "invalid indirect read from stack R2 var_off",
167 .result = REJECT,262 .result = REJECT,
168 .prog_type = BPF_PROG_TYPE_LWT_IN,263 .prog_type = BPF_PROG_TYPE_LWT_IN,
169},264},
@@ -189,7 +284,7 @@ · 19 unmodified lines
189 BPF_EXIT_INSN(),284 BPF_EXIT_INSN(),
190 },285 },
191 .fixup_map_hash_8b = { 6 },286 .fixup_map_hash_8b = { 6 },
192 .errstr_unpriv = "R2 stack pointer arithmetic goes out of range, prohibited for !root",287 .errstr_unpriv = "R2 variable stack access prohibited for !root",
193 .result_unpriv = REJECT,288 .result_unpriv = REJECT,
194 .result = ACCEPT,289 .result = ACCEPT,
195 .prog_type = BPF_PROG_TYPE_CGROUP_SKB,290 .prog_type = BPF_PROG_TYPE_CGROUP_SKB,
@@ -217,7 +312,7 @@ · 21 unmodified lines
217 BPF_MOV64_IMM(BPF_REG_0, 0),312 BPF_MOV64_IMM(BPF_REG_0, 0),
218 BPF_EXIT_INSN(),313 BPF_EXIT_INSN(),
219 },314 },
220 .errstr = "invalid indirect read from stack var_off",315 .errstr = "invalid indirect read from stack R4 var_off",
221 .result = REJECT,316 .result = REJECT,
222 .prog_type = BPF_PROG_TYPE_SOCK_OPS,317 .prog_type = BPF_PROG_TYPE_SOCK_OPS,
223},318},
tools/testing/selftests/bpf/vmtest.sh+368−0
@@ -0,0 +1,368 @@
1#!/bin/bash
2# SPDX-License-Identifier: GPL-2.0
3
4set -u
5set -e
6
7# This script currently only works for x86_64, as
8# it is based on the VM image used by the BPF CI which is
9# x86_64.
10QEMU_BINARY="${QEMU_BINARY:="qemu-system-x86_64"}"
11X86_BZIMAGE="arch/x86/boot/bzImage"
12DEFAULT_COMMAND="./test_progs"
13MOUNT_DIR="mnt"
14ROOTFS_IMAGE="root.img"
15OUTPUT_DIR="$HOME/.bpf_selftests"
16KCONFIG_URL="https://raw.githubusercontent.com/libbpf/libbpf/master/travis-ci/vmtest/configs/latest.config"
17KCONFIG_API_URL="https://api.github.com/repos/libbpf/libbpf/contents/travis-ci/vmtest/configs/latest.config"
18INDEX_URL="https://raw.githubusercontent.com/libbpf/libbpf/master/travis-ci/vmtest/configs/INDEX"
19NUM_COMPILE_JOBS="$(nproc)"
20
21usage()
22{
23 cat <<EOF
24Usage: $0 [-i] [-d <output_dir>] -- [<command>]
25
26<command> is the command you would normally run when you are in
27tools/testing/selftests/bpf. e.g:
28
29 $0 -- ./test_progs -t test_lsm
30
31If no command is specified, "${DEFAULT_COMMAND}" will be run by
32default.
33
34If you build your kernel using KBUILD_OUTPUT= or O= options, these
35can be passed as environment variables to the script:
36
37 O=<kernel_build_path> $0 -- ./test_progs -t test_lsm
38
39or
40
41 KBUILD_OUTPUT=<kernel_build_path> $0 -- ./test_progs -t test_lsm
42
43Options:
44
45 -i) Update the rootfs image with a newer version.
46 -d) Update the output directory (default: ${OUTPUT_DIR})
47 -j) Number of jobs for compilation, similar to -j in make
48 (default: ${NUM_COMPILE_JOBS})
49EOF
50}
51
52unset URLS
53populate_url_map()
54{
55 if ! declare -p URLS &> /dev/null; then
56 # URLS contain the mapping from file names to URLs where
57 # those files can be downloaded from.
58 declare -gA URLS
59 while IFS=$'\t' read -r name url; do
60 URLS["$name"]="$url"
61 done < <(curl -Lsf ${INDEX_URL})
62 fi
63}
64
65download()
66{
67 local file="$1"
68
69 if [[ ! -v URLS[$file] ]]; then
70 echo "$file not found" >&2
71 return 1
72 fi
73
74 echo "Downloading $file..." >&2
75 curl -Lsf "${URLS[$file]}" "${@:2}"
76}
77
78newest_rootfs_version()
79{
80 {
81 for file in "${!URLS[@]}"; do
82 if [[ $file =~ ^libbpf-vmtest-rootfs-(.*)\.tar\.zst$ ]]; then
83 echo "${BASH_REMATCH[1]}"
84 fi
85 done
86 } | sort -rV | head -1
87}
88
89download_rootfs()
90{
91 local rootfsversion="$1"
92 local dir="$2"
93
94 if ! which zstd &> /dev/null; then
95 echo 'Could not find "zstd" on the system, please install zstd'
96 exit 1
97 fi
98
99 download "libbpf-vmtest-rootfs-$rootfsversion.tar.zst" |
100 zstd -d | sudo tar -C "$dir" -x
101}
102
103recompile_kernel()
104{
105 local kernel_checkout="$1"
106 local make_command="$2"
107
108 cd "${kernel_checkout}"
109
110 ${make_command} olddefconfig
111 ${make_command}
112}
113
114mount_image()
115{
116 local rootfs_img="${OUTPUT_DIR}/${ROOTFS_IMAGE}"
117 local mount_dir="${OUTPUT_DIR}/${MOUNT_DIR}"
118
119 sudo mount -o loop "${rootfs_img}" "${mount_dir}"
120}
121
122unmount_image()
123{
124 local mount_dir="${OUTPUT_DIR}/${MOUNT_DIR}"
125
126 sudo umount "${mount_dir}" &> /dev/null
127}
128
129update_selftests()
130{
131 local kernel_checkout="$1"
132 local selftests_dir="${kernel_checkout}/tools/testing/selftests/bpf"
133
134 cd "${selftests_dir}"
135 ${make_command}
136
137 # Mount the image and copy the selftests to the image.
138 mount_image
139 sudo rm -rf "${mount_dir}/root/bpf"
140 sudo cp -r "${selftests_dir}" "${mount_dir}/root"
141 unmount_image
142}
143
144update_init_script()
145{
146 local init_script_dir="${OUTPUT_DIR}/${MOUNT_DIR}/etc/rcS.d"
147 local init_script="${init_script_dir}/S50-startup"
148 local command="$1"
149 local log_file="$2"
150
151 mount_image
152
153 if [[ ! -d "${init_script_dir}" ]]; then
154 cat <<EOF
155Could not find ${init_script_dir} in the mounted image.
156This likely indicates a bad rootfs image, Please download
157a new image by passing "-i" to the script
158EOF
159 exit 1
160
161 fi
162
163 sudo bash -c "cat >${init_script}" <<EOF
164#!/bin/bash
165
166{
167 cd /root/bpf
168 echo ${command}
169 stdbuf -oL -eL ${command}
170} 2>&1 | tee /root/${log_file}
171poweroff -f
172EOF
173
174 sudo chmod a+x "${init_script}"
175 unmount_image
176}
177
178create_vm_image()
179{
180 local rootfs_img="${OUTPUT_DIR}/${ROOTFS_IMAGE}"
181 local mount_dir="${OUTPUT_DIR}/${MOUNT_DIR}"
182
183 rm -rf "${rootfs_img}"
184 touch "${rootfs_img}"
185 chattr +C "${rootfs_img}" >/dev/null 2>&1 || true
186
187 truncate -s 2G "${rootfs_img}"
188 mkfs.ext4 -q "${rootfs_img}"
189
190 mount_image
191 download_rootfs "$(newest_rootfs_version)" "${mount_dir}"
192 unmount_image
193}
194
195run_vm()
196{
197 local kernel_bzimage="$1"
198 local rootfs_img="${OUTPUT_DIR}/${ROOTFS_IMAGE}"
199
200 if ! which "${QEMU_BINARY}" &> /dev/null; then
201 cat <<EOF
202Could not find ${QEMU_BINARY}
203Please install qemu or set the QEMU_BINARY environment variable.
204EOF
205 exit 1
206 fi
207
208 ${QEMU_BINARY} \
209 -nodefaults \
210 -display none \
211 -serial mon:stdio \
212 -cpu kvm64 \
213 -enable-kvm \
214 -smp 4 \
215 -m 2G \
216 -drive file="${rootfs_img}",format=raw,index=1,media=disk,if=virtio,cache=none \
217 -kernel "${kernel_bzimage}" \
218 -append "root=/dev/vda rw console=ttyS0,115200"
219}
220
221copy_logs()
222{
223 local mount_dir="${OUTPUT_DIR}/${MOUNT_DIR}"
224 local log_file="${mount_dir}/root/$1"
225
226 mount_image
227 sudo cp ${log_file} "${OUTPUT_DIR}"
228 sudo rm -f ${log_file}
229 unmount_image
230}
231
232is_rel_path()
233{
234 local path="$1"
235
236 [[ ${path:0:1} != "/" ]]
237}
238
239update_kconfig()
240{
241 local kconfig_file="$1"
242 local update_command="curl -sLf ${KCONFIG_URL} -o ${kconfig_file}"
243 # Github does not return the "last-modified" header when retrieving the
244 # raw contents of the file. Use the API call to get the last-modified
245 # time of the kernel config and only update the config if it has been
246 # updated after the previously cached config was created. This avoids
247 # unnecessarily compiling the kernel and selftests.
248 if [[ -f "${kconfig_file}" ]]; then
249 local last_modified_date="$(curl -sL -D - "${KCONFIG_API_URL}" -o /dev/null | \
250 grep "last-modified" | awk -F ': ' '{print $2}')"
251 local remote_modified_timestamp="$(date -d "${last_modified_date}" +"%s")"
252 local local_creation_timestamp="$(stat -c %Y "${kconfig_file}")"
253
254 if [[ "${remote_modified_timestamp}" -gt "${local_creation_timestamp}" ]]; then
255 ${update_command}
256 fi
257 else
258 ${update_command}
259 fi
260}
261
262main()
263{
264 local script_dir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
265 local kernel_checkout=$(realpath "${script_dir}"/../../../../)
266 local log_file="$(date +"bpf_selftests.%Y-%m-%d_%H-%M-%S.log")"
267 # By default the script searches for the kernel in the checkout directory but
268 # it also obeys environment variables O= and KBUILD_OUTPUT=
269 local kernel_bzimage="${kernel_checkout}/${X86_BZIMAGE}"
270 local command="${DEFAULT_COMMAND}"
271 local update_image="no"
272
273 while getopts 'hkid:j:' opt; do
274 case ${opt} in
275 i)
276 update_image="yes"
277 ;;
278 d)
279 OUTPUT_DIR="$OPTARG"
280 ;;
281 j)
282 NUM_COMPILE_JOBS="$OPTARG"
283 ;;
284 h)
285 usage
286 exit 0
287 ;;
288 \? )
289 echo "Invalid Option: -$OPTARG"
290 usage
291 exit 1
292 ;;
293 : )
294 echo "Invalid Option: -$OPTARG requires an argument"
295 usage
296 exit 1
297 ;;
298 esac
299 done
300 shift $((OPTIND -1))
301
302 if [[ $# -eq 0 ]]; then
303 echo "No command specified, will run ${DEFAULT_COMMAND} in the vm"
304 else
305 command="$@"
306 fi
307
308 local kconfig_file="${OUTPUT_DIR}/latest.config"
309 local make_command="make -j ${NUM_COMPILE_JOBS} KCONFIG_CONFIG=${kconfig_file}"
310
311 # Figure out where the kernel is being built.
312 # O takes precedence over KBUILD_OUTPUT.
313 if [[ "${O:=""}" != "" ]]; then
314 if is_rel_path "${O}"; then
315 O="$(realpath "${PWD}/${O}")"
316 fi
317 kernel_bzimage="${O}/${X86_BZIMAGE}"
318 make_command="${make_command} O=${O}"
319 elif [[ "${KBUILD_OUTPUT:=""}" != "" ]]; then
320 if is_rel_path "${KBUILD_OUTPUT}"; then
321 KBUILD_OUTPUT="$(realpath "${PWD}/${KBUILD_OUTPUT}")"
322 fi
323 kernel_bzimage="${KBUILD_OUTPUT}/${X86_BZIMAGE}"
324 make_command="${make_command} KBUILD_OUTPUT=${KBUILD_OUTPUT}"
325 fi
326
327 populate_url_map
328
329 local rootfs_img="${OUTPUT_DIR}/${ROOTFS_IMAGE}"
330 local mount_dir="${OUTPUT_DIR}/${MOUNT_DIR}"
331
332 echo "Output directory: ${OUTPUT_DIR}"
333
334 mkdir -p "${OUTPUT_DIR}"
335 mkdir -p "${mount_dir}"
336 update_kconfig "${kconfig_file}"
337
338 recompile_kernel "${kernel_checkout}" "${make_command}"
339
340 if [[ "${update_image}" == "no" && ! -f "${rootfs_img}" ]]; then
341 echo "rootfs image not found in ${rootfs_img}"
342 update_image="yes"
343 fi
344
345 if [[ "${update_image}" == "yes" ]]; then
346 create_vm_image
347 fi
348
349 update_selftests "${kernel_checkout}" "${make_command}"
350 update_init_script "${command}" "${log_file}"
351 run_vm "${kernel_bzimage}"
352 copy_logs "${log_file}"
353 echo "Logs saved in ${OUTPUT_DIR}/${log_file}"
354}
355
356catch()
357{
358 local exit_code=$1
359 # This is just a cleanup and the directory may
360 # have already been unmounted. So, don't let this
361 # clobber the error code we intend to return.
362 unmount_image || true
363 exit ${exit_code}
364}
365
366trap 'catch "$?"' EXIT
367
368main "$@"
tools/testing/selftests/bpf/xdpxceiver.c+108−117
@@ -224,14 +224,14 @@ static inline u16 udp_csum(u32 saddr, u32 daddr, u32 len, u8 proto, u16 *udp_pkt · 223 unmodified lines
224 return csum_tcpudp_magic(saddr, daddr, len, proto, csum);224 return csum_tcpudp_magic(saddr, daddr, len, proto, csum);
225}225}
226 226
227static void gen_eth_hdr(void *data, struct ethhdr *eth_hdr)227static void gen_eth_hdr(struct ifobject *ifobject, struct ethhdr *eth_hdr)
228{228{
229 memcpy(eth_hdr->h_dest, ((struct ifobject *)data)->dst_mac, ETH_ALEN);229 memcpy(eth_hdr->h_dest, ifobject->dst_mac, ETH_ALEN);
230 memcpy(eth_hdr->h_source, ((struct ifobject *)data)->src_mac, ETH_ALEN);230 memcpy(eth_hdr->h_source, ifobject->src_mac, ETH_ALEN);
231 eth_hdr->h_proto = htons(ETH_P_IP);231 eth_hdr->h_proto = htons(ETH_P_IP);
232}232}
233 233
234static void gen_ip_hdr(void *data, struct iphdr *ip_hdr)234static void gen_ip_hdr(struct ifobject *ifobject, struct iphdr *ip_hdr)
235{235{
236 ip_hdr->version = IP_PKT_VER;236 ip_hdr->version = IP_PKT_VER;
237 ip_hdr->ihl = 0x5;237 ip_hdr->ihl = 0x5;
@@ -241,18 +241,18 @@ static void gen_ip_hdr(void *data, struct iphdr *ip_hdr) · 3 unmodified lines
241 ip_hdr->frag_off = 0;241 ip_hdr->frag_off = 0;
242 ip_hdr->ttl = IPDEFTTL;242 ip_hdr->ttl = IPDEFTTL;
243 ip_hdr->protocol = IPPROTO_UDP;243 ip_hdr->protocol = IPPROTO_UDP;
244 ip_hdr->saddr = ((struct ifobject *)data)->src_ip;244 ip_hdr->saddr = ifobject->src_ip;
245 ip_hdr->daddr = ((struct ifobject *)data)->dst_ip;245 ip_hdr->daddr = ifobject->dst_ip;
246 ip_hdr->check = 0;246 ip_hdr->check = 0;
247}247}
248 248
249static void gen_udp_hdr(void *data, void *arg, struct udphdr *udp_hdr)249static void gen_udp_hdr(struct generic_data *data, struct ifobject *ifobject,
250 struct udphdr *udp_hdr)
250{251{
251 udp_hdr->source = htons(((struct ifobject *)arg)->src_port);252 udp_hdr->source = htons(ifobject->src_port);
252 udp_hdr->dest = htons(((struct ifobject *)arg)->dst_port);253 udp_hdr->dest = htons(ifobject->dst_port);
253 udp_hdr->len = htons(UDP_PKT_SIZE);254 udp_hdr->len = htons(UDP_PKT_SIZE);
254 memset32_htonl(pkt_data + PKT_HDR_SIZE,255 memset32_htonl(pkt_data + PKT_HDR_SIZE, htonl(data->seqnum), UDP_PKT_DATA_SIZE);
255 htonl(((struct generic_data *)data)->seqnum), UDP_PKT_DATA_SIZE);
256}256}
257 257
258static void gen_udp_csum(struct udphdr *udp_hdr, struct iphdr *ip_hdr)258static void gen_udp_csum(struct udphdr *udp_hdr, struct iphdr *ip_hdr)
@@ -382,21 +382,19 @@ static bool switch_namespace(int idx) · 123 unmodified lines
382 382
383static void *nsswitchthread(void *args)383static void *nsswitchthread(void *args)
384{384{
385 if (switch_namespace(((struct targs *)args)->idx)) {385 struct targs *targs = args;
386 ifdict[((struct targs *)args)->idx]->ifindex =386
387 if_nametoindex(ifdict[((struct targs *)args)->idx]->ifname);387 targs->retptr = false;
388 if (!ifdict[((struct targs *)args)->idx]->ifindex) {388
389 ksft_test_result_fail389 if (switch_namespace(targs->idx)) {
390 ("ERROR: [%s] interface \"%s\" does not exist\n",390 ifdict[targs->idx]->ifindex = if_nametoindex(ifdict[targs->idx]->ifname);
391 __func__, ifdict[((struct targs *)args)->idx]->ifname);391 if (!ifdict[targs->idx]->ifindex) {
392 ((struct targs *)args)->retptr = false;392 ksft_test_result_fail("ERROR: [%s] interface \"%s\" does not exist\n",
393 __func__, ifdict[targs->idx]->ifname);
393 } else {394 } else {
394 ksft_print_msg("Interface found: %s\n",395 ksft_print_msg("Interface found: %s\n", ifdict[targs->idx]->ifname);
395 ifdict[((struct targs *)args)->idx]->ifname);396 targs->retptr = true;
396 ((struct targs *)args)->retptr = true;
397 }397 }
398 } else {
399 ((struct targs *)args)->retptr = false;
400 }398 }
401 pthread_exit(NULL);399 pthread_exit(NULL);
402}400}
@@ -413,12 +411,12 @@ static int validate_interfaces(void) · 10 unmodified lines
413 if (strcmp(ifdict[i]->nsname, "")) {411 if (strcmp(ifdict[i]->nsname, "")) {
414 struct targs *targs;412 struct targs *targs;
415 413
416 targs = (struct targs *)malloc(sizeof(struct targs));414 targs = malloc(sizeof(*targs));
417 if (!targs)415 if (!targs)
418 exit_with_error(errno);416 exit_with_error(errno);
419 417
420 targs->idx = i;418 targs->idx = i;
421 if (pthread_create(&ns_thread, NULL, nsswitchthread, (void *)targs))419 if (pthread_create(&ns_thread, NULL, nsswitchthread, targs))
422 exit_with_error(errno);420 exit_with_error(errno);
423 421
424 pthread_join(ns_thread, NULL);422 pthread_join(ns_thread, NULL);
@@ -569,16 +567,18 @@ static void rx_pkt(struct xsk_socket_info *xsk, struct pollfd *fds) · 144 unmodified lines
569 }567 }
570 568
571 for (i = 0; i < rcvd; i++) {569 for (i = 0; i < rcvd; i++) {
572 u64 addr = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx)->addr;570 u64 addr, orig;
573 (void)xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++)->len;571
574 u64 orig = xsk_umem__extract_addr(addr);572 addr = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx)->addr;
573 xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++);
574 orig = xsk_umem__extract_addr(addr);
575 575
576 addr = xsk_umem__add_offset_to_addr(addr);576 addr = xsk_umem__add_offset_to_addr(addr);
577 pkt_node_rx = malloc(sizeof(struct pkt) + PKT_SIZE);577 pkt_node_rx = malloc(sizeof(struct pkt) + PKT_SIZE);
578 if (!pkt_node_rx)578 if (!pkt_node_rx)
579 exit_with_error(errno);579 exit_with_error(errno);
580 580
581 pkt_node_rx->pkt_frame = (char *)malloc(PKT_SIZE);581 pkt_node_rx->pkt_frame = malloc(PKT_SIZE);
582 if (!pkt_node_rx->pkt_frame)582 if (!pkt_node_rx->pkt_frame)
583 exit_with_error(errno);583 exit_with_error(errno);
584 584
@@ -628,28 +628,27 @@ static inline int get_batch_size(int pkt_cnt) · 43 unmodified lines
628 return opt_pkt_count - pkt_cnt;628 return opt_pkt_count - pkt_cnt;
629}629}
630 630
631static void complete_tx_only_all(void *arg)631static void complete_tx_only_all(struct ifobject *ifobject)
632{632{
633 bool pending;633 bool pending;
634 634
635 do {635 do {
636 pending = false;636 pending = false;
637 if (((struct ifobject *)arg)->xsk->outstanding_tx) {637 if (ifobject->xsk->outstanding_tx) {
638 complete_tx_only(((struct ifobject *)638 complete_tx_only(ifobject->xsk, BATCH_SIZE);
639 arg)->xsk, BATCH_SIZE);639 pending = !!ifobject->xsk->outstanding_tx;
640 pending = !!((struct ifobject *)arg)->xsk->outstanding_tx;
641 }640 }
642 } while (pending);641 } while (pending);
643}642}
644 643
645static void tx_only_all(void *arg)644static void tx_only_all(struct ifobject *ifobject)
646{645{
647 struct pollfd fds[MAX_SOCKS] = { };646 struct pollfd fds[MAX_SOCKS] = { };
648 u32 frame_nb = 0;647 u32 frame_nb = 0;
649 int pkt_cnt = 0;648 int pkt_cnt = 0;
650 int ret;649 int ret;
651 650
652 fds[0].fd = xsk_socket__fd(((struct ifobject *)arg)->xsk->xsk);651 fds[0].fd = xsk_socket__fd(ifobject->xsk->xsk);
653 fds[0].events = POLLOUT;652 fds[0].events = POLLOUT;
654 653
655 while ((opt_pkt_count && pkt_cnt < opt_pkt_count) || !opt_pkt_count) {654 while ((opt_pkt_count && pkt_cnt < opt_pkt_count) || !opt_pkt_count) {
@@ -664,12 +663,12 @@ static void tx_only_all(void *arg) · 8 unmodified lines
664 continue;663 continue;
665 }664 }
666 665
667 tx_only(((struct ifobject *)arg)->xsk, &frame_nb, batch_size);666 tx_only(ifobject->xsk, &frame_nb, batch_size);
668 pkt_cnt += batch_size;667 pkt_cnt += batch_size;
669 }668 }
670 669
671 if (opt_pkt_count)670 if (opt_pkt_count)
672 complete_tx_only_all(arg);671 complete_tx_only_all(ifobject);
673}672}
674 673
675static void worker_pkt_dump(void)674static void worker_pkt_dump(void)
@@ -727,21 +726,21 @@ static void worker_pkt_dump(void) · 51 unmodified lines
727static void worker_pkt_validate(void)726static void worker_pkt_validate(void)
728{727{
729 u32 payloadseqnum = -2;728 u32 payloadseqnum = -2;
729 struct iphdr *iphdr;
730 730
731 while (1) {731 while (1) {
732 pkt_node_rx_q = malloc(sizeof(struct pkt));
733 pkt_node_rx_q = TAILQ_LAST(&head, head_s);732 pkt_node_rx_q = TAILQ_LAST(&head, head_s);
734 if (!pkt_node_rx_q)733 if (!pkt_node_rx_q)
735 break;734 break;
735
736 iphdr = (struct iphdr *)(pkt_node_rx_q->pkt_frame + sizeof(struct ethhdr));
737
736 /*do not increment pktcounter if !(tos=0x9 and ipv4) */738 /*do not increment pktcounter if !(tos=0x9 and ipv4) */
737 if ((((struct iphdr *)(pkt_node_rx_q->pkt_frame +739 if (iphdr->version == IP_PKT_VER && iphdr->tos == IP_PKT_TOS) {
738 sizeof(struct ethhdr)))->version == IP_PKT_VER)740 payloadseqnum = *((uint32_t *)(pkt_node_rx_q->pkt_frame + PKT_HDR_SIZE));
739 && (((struct iphdr *)(pkt_node_rx_q->pkt_frame + sizeof(struct ethhdr)))->tos ==
740 IP_PKT_TOS)) {
741 payloadseqnum = *((uint32_t *) (pkt_node_rx_q->pkt_frame + PKT_HDR_SIZE));
742 if (debug_pkt_dump && payloadseqnum != EOT) {741 if (debug_pkt_dump && payloadseqnum != EOT) {
743 pkt_obj = (struct pkt_frame *)malloc(sizeof(struct pkt_frame));742 pkt_obj = malloc(sizeof(*pkt_obj));
744 pkt_obj->payload = (char *)malloc(PKT_SIZE);743 pkt_obj->payload = malloc(PKT_SIZE);
745 memcpy(pkt_obj->payload, pkt_node_rx_q->pkt_frame, PKT_SIZE);744 memcpy(pkt_obj->payload, pkt_node_rx_q->pkt_frame, PKT_SIZE);
746 pkt_buf[payloadseqnum] = pkt_obj;745 pkt_buf[payloadseqnum] = pkt_obj;
747 }746 }
@@ -759,35 +758,29 @@ static void worker_pkt_validate(void) · 11 unmodified lines
759 ksft_exit_xfail();758 ksft_exit_xfail();
760 }759 }
761 760
762 TAILQ_REMOVE(&head, pkt_node_rx_q, pkt_nodes);
763 free(pkt_node_rx_q->pkt_frame);
764 free(pkt_node_rx_q);
765 pkt_node_rx_q = NULL;
766 prev_pkt = payloadseqnum;761 prev_pkt = payloadseqnum;
767 pkt_counter++;762 pkt_counter++;
768 } else {763 } else {
769 ksft_print_msg("Invalid frame received: ");764 ksft_print_msg("Invalid frame received: ");
770 ksft_print_msg("[IP_PKT_VER: %02X], [IP_PKT_TOS: %02X]\n",765 ksft_print_msg("[IP_PKT_VER: %02X], [IP_PKT_TOS: %02X]\n", iphdr->version,
771 ((struct iphdr *)(pkt_node_rx_q->pkt_frame +766 iphdr->tos);
772 sizeof(struct ethhdr)))->version,
773 ((struct iphdr *)(pkt_node_rx_q->pkt_frame +
774 sizeof(struct ethhdr)))->tos);
775 TAILQ_REMOVE(&head, pkt_node_rx_q, pkt_nodes);
776 free(pkt_node_rx_q->pkt_frame);
777 free(pkt_node_rx_q);
778 pkt_node_rx_q = NULL;
779 }767 }
768
769 TAILQ_REMOVE(&head, pkt_node_rx_q, pkt_nodes);
770 free(pkt_node_rx_q->pkt_frame);
771 free(pkt_node_rx_q);
772 pkt_node_rx_q = NULL;
780 }773 }
781}774}
782 775
783static void thread_common_ops(void *arg, void *bufs, pthread_mutex_t *mutexptr,776static void thread_common_ops(struct ifobject *ifobject, void *bufs, pthread_mutex_t *mutexptr,
784 atomic_int *spinningptr)777 atomic_int *spinningptr)
785{778{
786 int ctr = 0;779 int ctr = 0;
787 int ret;780 int ret;
788 781
789 xsk_configure_umem((struct ifobject *)arg, bufs, num_frames * XSK_UMEM__DEFAULT_FRAME_SIZE);782 xsk_configure_umem(ifobject, bufs, num_frames * XSK_UMEM__DEFAULT_FRAME_SIZE);
790 ret = xsk_configure_socket((struct ifobject *)arg);783 ret = xsk_configure_socket(ifobject);
791 784
792 /* Retry Create Socket if it fails as xsk_socket__create()785 /* Retry Create Socket if it fails as xsk_socket__create()
793 * is asynchronous786 * is asynchronous
@@ -798,9 +791,8 @@ static void thread_common_ops(void *arg, void *bufs, pthread_mutex_t *mutexptr, · 4 unmodified lines
798 pthread_mutex_lock(mutexptr);791 pthread_mutex_lock(mutexptr);
799 while (ret && ctr < SOCK_RECONF_CTR) {792 while (ret && ctr < SOCK_RECONF_CTR) {
800 atomic_store(spinningptr, 1);793 atomic_store(spinningptr, 1);
801 xsk_configure_umem((struct ifobject *)arg,794 xsk_configure_umem(ifobject, bufs, num_frames * XSK_UMEM__DEFAULT_FRAME_SIZE);
802 bufs, num_frames * XSK_UMEM__DEFAULT_FRAME_SIZE);795 ret = xsk_configure_socket(ifobject);
803 ret = xsk_configure_socket((struct ifobject *)arg);
804 usleep(USLEEP_MAX);796 usleep(USLEEP_MAX);
805 ctr++;797 ctr++;
806 }798 }
@@ -815,9 +807,10 @@ static void *worker_testapp_validate(void *arg) · 8 unmodified lines
815{807{
816 struct udphdr *udp_hdr =808 struct udphdr *udp_hdr =
817 (struct udphdr *)(pkt_data + sizeof(struct ethhdr) + sizeof(struct iphdr));809 (struct udphdr *)(pkt_data + sizeof(struct ethhdr) + sizeof(struct iphdr));
818 struct generic_data *data = (struct generic_data *)malloc(sizeof(struct generic_data));
819 struct iphdr *ip_hdr = (struct iphdr *)(pkt_data + sizeof(struct ethhdr));810 struct iphdr *ip_hdr = (struct iphdr *)(pkt_data + sizeof(struct ethhdr));
820 struct ethhdr *eth_hdr = (struct ethhdr *)pkt_data;811 struct ethhdr *eth_hdr = (struct ethhdr *)pkt_data;
812 struct ifobject *ifobject = (struct ifobject *)arg;
813 struct generic_data data;
821 void *bufs = NULL;814 void *bufs = NULL;
822 815
823 pthread_attr_setstacksize(&attr, THREAD_STACK);816 pthread_attr_setstacksize(&attr, THREAD_STACK);
@@ -828,58 +821,56 @@ static void *worker_testapp_validate(void *arg) · 4 unmodified lines
828 if (bufs == MAP_FAILED)821 if (bufs == MAP_FAILED)
829 exit_with_error(errno);822 exit_with_error(errno);
830 823
831 if (strcmp(((struct ifobject *)arg)->nsname, ""))824 if (strcmp(ifobject->nsname, ""))
832 switch_namespace(((struct ifobject *)arg)->ifdict_index);825 switch_namespace(ifobject->ifdict_index);
833 }826 }
834 827
835 if (((struct ifobject *)arg)->fv.vector == tx) {828 if (ifobject->fv.vector == tx) {
836 int spinningrxctr = 0;829 int spinningrxctr = 0;
837 830
838 if (!bidi_pass)831 if (!bidi_pass)
839 thread_common_ops(arg, bufs, &sync_mutex_tx, &spinning_tx);832 thread_common_ops(ifobject, bufs, &sync_mutex_tx, &spinning_tx);
840 833
841 while (atomic_load(&spinning_rx) && spinningrxctr < SOCK_RECONF_CTR) {834 while (atomic_load(&spinning_rx) && spinningrxctr < SOCK_RECONF_CTR) {
842 spinningrxctr++;835 spinningrxctr++;
843 usleep(USLEEP_MAX);836 usleep(USLEEP_MAX);
844 }837 }
845 838
846 ksft_print_msg("Interface [%s] vector [Tx]\n", ((struct ifobject *)arg)->ifname);839 ksft_print_msg("Interface [%s] vector [Tx]\n", ifobject->ifname);
847 for (int i = 0; i < num_frames; i++) {840 for (int i = 0; i < num_frames; i++) {
848 /*send EOT frame */841 /*send EOT frame */
849 if (i == (num_frames - 1))842 if (i == (num_frames - 1))
850 data->seqnum = -1;843 data.seqnum = -1;
851 else844 else
852 data->seqnum = i;845 data.seqnum = i;
853 gen_udp_hdr((void *)data, (void *)arg, udp_hdr);846 gen_udp_hdr(&data, ifobject, udp_hdr);
854 gen_ip_hdr((void *)arg, ip_hdr);847 gen_ip_hdr(ifobject, ip_hdr);
855 gen_udp_csum(udp_hdr, ip_hdr);848 gen_udp_csum(udp_hdr, ip_hdr);
856 gen_eth_hdr((void *)arg, eth_hdr);849 gen_eth_hdr(ifobject, eth_hdr);
857 gen_eth_frame(((struct ifobject *)arg)->umem,850 gen_eth_frame(ifobject->umem, i * XSK_UMEM__DEFAULT_FRAME_SIZE);
858 i * XSK_UMEM__DEFAULT_FRAME_SIZE);
859 }851 }
860 852
861 free(data);
862 ksft_print_msg("Sending %d packets on interface %s\n",853 ksft_print_msg("Sending %d packets on interface %s\n",
863 (opt_pkt_count - 1), ((struct ifobject *)arg)->ifname);854 (opt_pkt_count - 1), ifobject->ifname);
864 tx_only_all(arg);855 tx_only_all(ifobject);
865 } else if (((struct ifobject *)arg)->fv.vector == rx) {856 } else if (ifobject->fv.vector == rx) {
866 struct pollfd fds[MAX_SOCKS] = { };857 struct pollfd fds[MAX_SOCKS] = { };
867 int ret;858 int ret;
868 859
869 if (!bidi_pass)860 if (!bidi_pass)
870 thread_common_ops(arg, bufs, &sync_mutex_tx, &spinning_rx);861 thread_common_ops(ifobject, bufs, &sync_mutex_tx, &spinning_rx);
871 862
872 ksft_print_msg("Interface [%s] vector [Rx]\n", ((struct ifobject *)arg)->ifname);863 ksft_print_msg("Interface [%s] vector [Rx]\n", ifobject->ifname);
873 xsk_populate_fill_ring(((struct ifobject *)arg)->umem);864 xsk_populate_fill_ring(ifobject->umem);
874 865
875 TAILQ_INIT(&head);866 TAILQ_INIT(&head);
876 if (debug_pkt_dump) {867 if (debug_pkt_dump) {
877 pkt_buf = malloc(sizeof(struct pkt_frame **) * num_frames);868 pkt_buf = calloc(num_frames, sizeof(*pkt_buf));
878 if (!pkt_buf)869 if (!pkt_buf)
879 exit_with_error(errno);870 exit_with_error(errno);
880 }871 }
881 872
882 fds[0].fd = xsk_socket__fd(((struct ifobject *)arg)->xsk->xsk);873 fds[0].fd = xsk_socket__fd(ifobject->xsk->xsk);
883 fds[0].events = POLLIN;874 fds[0].events = POLLIN;
884 875
885 pthread_mutex_lock(&sync_mutex);876 pthread_mutex_lock(&sync_mutex);
@@ -892,7 +883,7 @@ static void *worker_testapp_validate(void *arg) · 6 unmodified lines
892 if (ret <= 0)883 if (ret <= 0)
893 continue;884 continue;
894 }885 }
895 rx_pkt(((struct ifobject *)arg)->xsk, fds);886 rx_pkt(ifobject->xsk, fds);
896 worker_pkt_validate();887 worker_pkt_validate();
897 888
898 if (sigvar)889 if (sigvar)
@@ -900,21 +891,23 @@ static void *worker_testapp_validate(void *arg) · 1 unmodified line
900 }891 }
901 892
902 ksft_print_msg("Received %d packets on interface %s\n",893 ksft_print_msg("Received %d packets on interface %s\n",
903 pkt_counter, ((struct ifobject *)arg)->ifname);894 pkt_counter, ifobject->ifname);
904 895
905 if (opt_teardown)896 if (opt_teardown)
906 ksft_print_msg("Destroying socket\n");897 ksft_print_msg("Destroying socket\n");
907 }898 }
908 899
909 if (!opt_bidi || (opt_bidi && bidi_pass)) {900 if (!opt_bidi || bidi_pass) {
910 xsk_socket__delete(((struct ifobject *)arg)->xsk->xsk);901 xsk_socket__delete(ifobject->xsk->xsk);
911 (void)xsk_umem__delete(((struct ifobject *)arg)->umem->umem);902 (void)xsk_umem__delete(ifobject->umem->umem);
912 }903 }
913 pthread_exit(NULL);904 pthread_exit(NULL);
914}905}
915 906
916static void testapp_validate(void)907static void testapp_validate(void)
917{908{
909 struct timespec max_wait = { 0, 0 };
910
918 pthread_attr_init(&attr);911 pthread_attr_init(&attr);
919 pthread_attr_setstacksize(&attr, THREAD_STACK);912 pthread_attr_setstacksize(&attr, THREAD_STACK);
920 913
@@ -929,18 +922,16 @@ static void testapp_validate(void) · 8 unmodified lines
929 pthread_mutex_lock(&sync_mutex);922 pthread_mutex_lock(&sync_mutex);
930 923
931 /*Spawn RX thread */924 /*Spawn RX thread */
932 if (!opt_bidi || (opt_bidi && !bidi_pass)) {925 if (!opt_bidi || !bidi_pass) {
933 if (pthread_create(&t0, &attr, worker_testapp_validate, (void *)ifdict[1]))926 if (pthread_create(&t0, &attr, worker_testapp_validate, ifdict[1]))
934 exit_with_error(errno);927 exit_with_error(errno);
935 } else if (opt_bidi && bidi_pass) {928 } else if (opt_bidi && bidi_pass) {
936 /*switch Tx/Rx vectors */929 /*switch Tx/Rx vectors */
937 ifdict[0]->fv.vector = rx;930 ifdict[0]->fv.vector = rx;
938 if (pthread_create(&t0, &attr, worker_testapp_validate, (void *)ifdict[0]))931 if (pthread_create(&t0, &attr, worker_testapp_validate, ifdict[0]))
939 exit_with_error(errno);932 exit_with_error(errno);
940 }933 }
941 934
942 struct timespec max_wait = { 0, 0 };
943
944 if (clock_gettime(CLOCK_REALTIME, &max_wait))935 if (clock_gettime(CLOCK_REALTIME, &max_wait))
945 exit_with_error(errno);936 exit_with_error(errno);
946 max_wait.tv_sec += TMOUT_SEC;937 max_wait.tv_sec += TMOUT_SEC;
@@ -951,13 +942,13 @@ static void testapp_validate(void) · 4 unmodified lines
951 pthread_mutex_unlock(&sync_mutex);942 pthread_mutex_unlock(&sync_mutex);
952 943
953 /*Spawn TX thread */944 /*Spawn TX thread */
954 if (!opt_bidi || (opt_bidi && !bidi_pass)) {945 if (!opt_bidi || !bidi_pass) {
955 if (pthread_create(&t1, &attr, worker_testapp_validate, (void *)ifdict[0]))946 if (pthread_create(&t1, &attr, worker_testapp_validate, ifdict[0]))
956 exit_with_error(errno);947 exit_with_error(errno);
957 } else if (opt_bidi && bidi_pass) {948 } else if (opt_bidi && bidi_pass) {
958 /*switch Tx/Rx vectors */949 /*switch Tx/Rx vectors */
959 ifdict[1]->fv.vector = tx;950 ifdict[1]->fv.vector = tx;
960 if (pthread_create(&t1, &attr, worker_testapp_validate, (void *)ifdict[1]))951 if (pthread_create(&t1, &attr, worker_testapp_validate, ifdict[1]))
961 exit_with_error(errno);952 exit_with_error(errno);
962 }953 }
963 954
@@ -991,25 +982,25 @@ static void testapp_sockets(void) · 27 unmodified lines
991 print_ksft_result();982 print_ksft_result();
992}983}
993 984
994static void init_iface_config(void *ifaceconfig)985static void init_iface_config(struct ifaceconfigobj *ifaceconfig)
995{986{
996 /*Init interface0 */987 /*Init interface0 */
997 ifdict[0]->fv.vector = tx;988 ifdict[0]->fv.vector = tx;
998 memcpy(ifdict[0]->dst_mac, ((struct ifaceconfigobj *)ifaceconfig)->dst_mac, ETH_ALEN);989 memcpy(ifdict[0]->dst_mac, ifaceconfig->dst_mac, ETH_ALEN);
999 memcpy(ifdict[0]->src_mac, ((struct ifaceconfigobj *)ifaceconfig)->src_mac, ETH_ALEN);990 memcpy(ifdict[0]->src_mac, ifaceconfig->src_mac, ETH_ALEN);
1000 ifdict[0]->dst_ip = ((struct ifaceconfigobj *)ifaceconfig)->dst_ip.s_addr;991 ifdict[0]->dst_ip = ifaceconfig->dst_ip.s_addr;
1001 ifdict[0]->src_ip = ((struct ifaceconfigobj *)ifaceconfig)->src_ip.s_addr;992 ifdict[0]->src_ip = ifaceconfig->src_ip.s_addr;
1002 ifdict[0]->dst_port = ((struct ifaceconfigobj *)ifaceconfig)->dst_port;993 ifdict[0]->dst_port = ifaceconfig->dst_port;
1003 ifdict[0]->src_port = ((struct ifaceconfigobj *)ifaceconfig)->src_port;994 ifdict[0]->src_port = ifaceconfig->src_port;
1004 995
1005 /*Init interface1 */996 /*Init interface1 */
1006 ifdict[1]->fv.vector = rx;997 ifdict[1]->fv.vector = rx;
1007 memcpy(ifdict[1]->dst_mac, ((struct ifaceconfigobj *)ifaceconfig)->src_mac, ETH_ALEN);998 memcpy(ifdict[1]->dst_mac, ifaceconfig->src_mac, ETH_ALEN);
1008 memcpy(ifdict[1]->src_mac, ((struct ifaceconfigobj *)ifaceconfig)->dst_mac, ETH_ALEN);999 memcpy(ifdict[1]->src_mac, ifaceconfig->dst_mac, ETH_ALEN);
1009 ifdict[1]->dst_ip = ((struct ifaceconfigobj *)ifaceconfig)->src_ip.s_addr;1000 ifdict[1]->dst_ip = ifaceconfig->src_ip.s_addr;
1010 ifdict[1]->src_ip = ((struct ifaceconfigobj *)ifaceconfig)->dst_ip.s_addr;1001 ifdict[1]->src_ip = ifaceconfig->dst_ip.s_addr;
1011 ifdict[1]->dst_port = ((struct ifaceconfigobj *)ifaceconfig)->src_port;1002 ifdict[1]->dst_port = ifaceconfig->src_port;
1012 ifdict[1]->src_port = ((struct ifaceconfigobj *)ifaceconfig)->dst_port;1003 ifdict[1]->src_port = ifaceconfig->dst_port;
1013}1004}
1014 1005
1015int main(int argc, char **argv)1006int main(int argc, char **argv)
@@ -1026,7 +1017,7 @@ int main(int argc, char **argv) · 10 unmodified lines
1026 u16 UDP_DST_PORT = 2020;1017 u16 UDP_DST_PORT = 2020;
1027 u16 UDP_SRC_PORT = 2121;1018 u16 UDP_SRC_PORT = 2121;
1028 1019
1029 ifaceconfig = (struct ifaceconfigobj *)malloc(sizeof(struct ifaceconfigobj));1020 ifaceconfig = malloc(sizeof(struct ifaceconfigobj));
1030 memcpy(ifaceconfig->dst_mac, MAC1, ETH_ALEN);1021 memcpy(ifaceconfig->dst_mac, MAC1, ETH_ALEN);
1031 memcpy(ifaceconfig->src_mac, MAC2, ETH_ALEN);1022 memcpy(ifaceconfig->src_mac, MAC2, ETH_ALEN);
1032 inet_aton(IP1, &ifaceconfig->dst_ip);1023 inet_aton(IP1, &ifaceconfig->dst_ip);
@@ -1035,7 +1026,7 @@ int main(int argc, char **argv) · 2 unmodified lines
1035 ifaceconfig->src_port = UDP_SRC_PORT;1026 ifaceconfig->src_port = UDP_SRC_PORT;
1036 1027
1037 for (int i = 0; i < MAX_INTERFACES; i++) {1028 for (int i = 0; i < MAX_INTERFACES; i++) {
1038 ifdict[i] = (struct ifobject *)malloc(sizeof(struct ifobject));1029 ifdict[i] = malloc(sizeof(struct ifobject));
1039 if (!ifdict[i])1030 if (!ifdict[i])
1040 exit_with_error(errno);1031 exit_with_error(errno);
1041 1032
@@ -1048,7 +1039,7 @@ int main(int argc, char **argv) · 6 unmodified lines
1048 1039
1049 num_frames = ++opt_pkt_count;1040 num_frames = ++opt_pkt_count;
1050 1041
1051 init_iface_config((void *)ifaceconfig);1042 init_iface_config(ifaceconfig);
1052 1043
1053 pthread_init_mutex();1044 pthread_init_mutex();
1054 1045
tools/testing/selftests/bpf/xdpxceiver.h+0−2
@@ -92,8 +92,6 @@ struct flow_vector { · 91 unmodified lines
92 enum fvector {92 enum fvector {
93 tx,93 tx,
94 rx,94 rx,
95 bidi,
96 undef,
97 } vector;95 } vector;
98};96};
99 97
tools/testing/selftests/tc-testing/Makefile+1−2
@@ -1,4 +1,5 @@
1# SPDX-License-Identifier: GPL-2.01# SPDX-License-Identifier: GPL-2.0
2include ../../../scripts/Makefile.include
2 3
3top_srcdir = $(abspath ../../../..)4top_srcdir = $(abspath ../../../..)
4APIDIR := $(top_scrdir)/include/uapi5APIDIR := $(top_scrdir)/include/uapi
@@ -7,8 +8,6 @@ TEST_GEN_FILES = action.o · 2 unmodified lines
7KSFT_KHDR_INSTALL := 18KSFT_KHDR_INSTALL := 1
8include ../lib.mk9include ../lib.mk
9 10
10CLANG ?= clang
11LLC ?= llc
12PROBE := $(shell $(LLC) -march=bpf -mcpu=probe -filetype=null /dev/null 2>&1)11PROBE := $(shell $(LLC) -march=bpf -mcpu=probe -filetype=null /dev/null 2>&1)
13 12
14ifeq ($(PROBE),)13ifeq ($(PROBE),)