History

Merge tag 'bpf-next-6.13' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Pull bpf updates from Alexei Starovoitov: - Add BPF uprobe session support (Jiri Olsa) - Optimize uprobe performance (Andrii Nakryiko) - Add bpf_fastcall support to helpers and kfuncs (Eduard Zingerman) - Avoid calling free_htab_elem() under hash map bucket lock (Hou Tao) - Prevent tailcall infinite loop caused by freplace (Leon Hwang) - Mark raw_tracepoint arguments as nullable (Kumar Kartikeya Dwivedi) - Introduce uptr support in the task local storage map (Martin KaFai Lau) - Stringify errno log messages in libbpf (Mykyta Yatsenko) - Add kmem_cache BPF iterator for perf's lock profiling (Namhyung Kim) - Support BPF objects of either endianness in libbpf (Tony Ambardar) - Add ksym to struct_ops trampoline to fix stack trace (Xu Kuohai) - Introduce private stack for eligible BPF programs (Yonghong Song) - Migrate samples/bpf tests to selftests/bpf test_progs (Daniel T. Lee) - Migrate test_sock to selftests/bpf test_progs (Jordan Rife) * tag 'bpf-next-6.13' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next: (152 commits) libbpf: Change hash_combine parameters from long to unsigned long selftests/bpf: Fix build error with llvm 19 libbpf: Fix memory leak in bpf_program__attach_uprobe_multi bpf: use common instruction history across all states bpf: Add necessary migrate_disable to range_tree. bpf: Do not alloc arena on unsupported arches selftests/bpf: Set test path for token/obj_priv_implicit_token_envvar selftests/bpf: Add a test for arena range tree algorithm bpf: Introduce range_tree data structure and use it in bpf arena samples/bpf: Remove unused variable in xdp2skb_meta_kern.c samples/bpf: Remove unused variables in tc_l2_redirect_kern.c bpftool: Cast variable `var` to long long bpf, x86: Propagate tailcall info only for subprogs bpf: Add kernel symbol for struct_ops trampoline bpf: Use function pointers count as struct_ops links count bpf: Remove unused member rcu from bpf_struct_ops_map selftests/bpf: Add struct_ops prog private stack tests bpf: Support private stack for struct_ops progs selftests/bpf: Add tracing prog private stack tests bpf, x86: Support private stack in jit ...

Linus Torvalds committed 1 year ago

Commit 6e95ef02 parents 43fb83c 2c8b09a211 files · +6961 −3473Browse files

Unified · Split

211 files
  1. Documentation/bpf/btf.rst +76 −1
  2. Documentation/bpf/verifier.rst +2 −2
  3. arch/arm64/net/bpf_jit_comp.c +31 −16
  4. arch/x86/net/bpf_jit_comp.c +144 −5
  5. include/linux/bpf.h +54 −9
  6. include/linux/bpf_local_storage.h +9 −3
  7. include/linux/bpf_verifier.h +31 −36
  8. include/linux/btf.h +11 −11
  9. include/linux/btf_ids.h +1 −0
  10. include/linux/filter.h +1 −0
  11. include/uapi/linux/bpf.h +4 −5
  12. kernel/bpf/Makefile +2 −1
  13. kernel/bpf/arena.c +24 −14
  14. kernel/bpf/arraymap.c +24 −2
  15. kernel/bpf/bpf_cgrp_storage.c +2 −2
  16. kernel/bpf/bpf_inode_storage.c +2 −2
  17. kernel/bpf/bpf_local_storage.c +63 −16
  18. kernel/bpf/bpf_struct_ops.c +103 −12
  19. kernel/bpf/bpf_task_storage.c +5 −2
  20. kernel/bpf/btf.c +42 −15
  21. kernel/bpf/core.c +6 −0
  22. kernel/bpf/dispatcher.c +2 −1
  23. kernel/bpf/hashtab.c +39 −17
  24. kernel/bpf/helpers.c +27 −2
  25. kernel/bpf/kmem_cache_iter.c +238 −0
  26. kernel/bpf/memalloc.c +1 −4
  27. kernel/bpf/range_tree.c +272 −0
  28. kernel/bpf/range_tree.h +21 −0
  29. kernel/bpf/syscall.c +115 −9
  30. kernel/bpf/trampoline.c +50 −10
  31. kernel/bpf/verifier.c +418 −179
  32. kernel/trace/bpf_trace.c +86 −24
  33. mm/slab_common.c +19 −0
  34. net/core/bpf_sk_storage.c +3 −3
  35. samples/bpf/Makefile +0 −25
  36. samples/bpf/sock_flags.bpf.c +0 −47
  37. samples/bpf/syscall_nrs.c +5 −0
  38. samples/bpf/tc_l2_redirect_kern.c +0 −6
  39. samples/bpf/test_cgrp2_array_pin.c +0 −106
  40. samples/bpf/test_cgrp2_attach.c +0 −177
  41. samples/bpf/test_cgrp2_sock.c +0 −294
  42. samples/bpf/test_cgrp2_sock.sh +0 −137
  43. samples/bpf/test_cgrp2_sock2.c +0 −95
  44. samples/bpf/test_cgrp2_sock2.sh +0 −103
  45. samples/bpf/test_cgrp2_tc.bpf.c +0 −56
  46. samples/bpf/test_cgrp2_tc.sh +0 −187
  47. samples/bpf/test_current_task_under_cgroup.bpf.c +0 −43
  48. samples/bpf/test_current_task_under_cgroup_user.c +0 −115
  49. samples/bpf/test_overhead_kprobe.bpf.c +0 −41
  50. samples/bpf/test_overhead_raw_tp.bpf.c +0 −17
  51. samples/bpf/test_overhead_tp.bpf.c +0 −23
  52. samples/bpf/test_overhead_user.c +0 −225
  53. samples/bpf/test_override_return.sh +0 −16
  54. samples/bpf/test_probe_write_user.bpf.c +0 −52
  55. samples/bpf/test_probe_write_user_user.c +0 −108
  56. samples/bpf/tracex7.bpf.c +0 −15
  57. samples/bpf/tracex7_user.c +0 −56
  58. samples/bpf/xdp2skb_meta_kern.c +1 −1
  59. samples/bpf/xdp_adjust_tail_kern.c +1 −0
  60. scripts/Makefile.btf +4 −2
  61. scripts/bpf_doc.py +50 −3
  62. tools/bpf/bpf_jit_disasm.c +1 −1
  63. tools/bpf/bpftool/Makefile +5 −1
  64. tools/bpf/bpftool/btf.c +89 −11
  65. tools/bpf/bpftool/jit_disasm.c +29 −11
  66. tools/bpf/resolve_btfids/main.c +2 −2
  67. tools/bpf/runqslower/runqslower.bpf.c +0 −1
  68. tools/include/uapi/linux/bpf.h +4 −5
  69. tools/lib/bpf/Makefile +2 −1
  70. tools/lib/bpf/bpf.c +1 −0
  71. tools/lib/bpf/bpf_gen_internal.h +1 −0
  72. tools/lib/bpf/bpf_helpers.h +1 −0
  73. tools/lib/bpf/btf.c +236 −72
  74. tools/lib/bpf/btf.h +3 −0
  75. tools/lib/bpf/btf_dump.c +4 −3
  76. tools/lib/bpf/btf_relocate.c +1 −1
  77. tools/lib/bpf/elf.c +1 −3
  78. tools/lib/bpf/features.c +6 −9
  79. tools/lib/bpf/gen_loader.c +137 −53
  80. tools/lib/bpf/hashmap.h +10 −10
  81. tools/lib/bpf/libbpf.c +266 −260
  82. tools/lib/bpf/libbpf.h +3 −1
  83. tools/lib/bpf/libbpf.map +5 −0
  84. tools/lib/bpf/libbpf_internal.h +41 −2
  85. tools/lib/bpf/libbpf_version.h +1 −1
  86. tools/lib/bpf/linker.c +80 −25
  87. tools/lib/bpf/relo_core.c +1 −1
  88. tools/lib/bpf/ringbuf.c +18 −16
  89. tools/lib/bpf/skel_internal.h +2 −1
  90. tools/lib/bpf/str_error.c +71 −0
  91. tools/lib/bpf/str_error.h +7 −0
  92. tools/lib/bpf/usdt.c +17 −15
  93. tools/lib/bpf/zip.c +1 −1
  94. tools/lib/subcmd/parse-options.c +1 −1
  95. tools/testing/selftests/bpf/.gitignore +0 −1
  96. tools/testing/selftests/bpf/Makefile +38 −22
  97. tools/testing/selftests/bpf/benchs/bench_trigger.c +2 −1
  98. tools/testing/selftests/bpf/bpf_experimental.h +6 −0
  99. tools/testing/selftests/bpf/bpf_testmod/bpf_testmod-events.h +8 −0
  100. tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c +106 −0
  101. tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.h +5 −0
  102. tools/testing/selftests/bpf/bpf_util.h +12 −0
  103. tools/testing/selftests/bpf/config.vm +5 −2
  104. tools/testing/selftests/bpf/io_helpers.c +21 −0
  105. tools/testing/selftests/bpf/io_helpers.h +7 −0
  106. tools/testing/selftests/bpf/map_tests/task_storage_map.c +2 −1
  107. tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +1 −1
  108. tools/testing/selftests/bpf/prog_tests/bpf_iter.c +7 −7
  109. tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c +0 −4
  110. tools/testing/selftests/bpf/prog_tests/cb_refs.c +2 −2
  111. tools/testing/selftests/bpf/prog_tests/cgrp_local_storage.c +5 −5
  112. tools/testing/selftests/bpf/prog_tests/core_reloc.c +1 −1
  113. tools/testing/selftests/bpf/prog_tests/iters.c +2 −2
  114. tools/testing/selftests/bpf/prog_tests/kmem_cache_iter.c +126 −0
  115. tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c +2 −0
  116. tools/testing/selftests/bpf/prog_tests/linked_funcs.c +1 −1
  117. tools/testing/selftests/bpf/prog_tests/log_buf.c +0 −3
  118. tools/testing/selftests/bpf/prog_tests/lsm_cgroup.c +0 −4
  119. tools/testing/selftests/bpf/prog_tests/map_in_map.c +131 −1
  120. tools/testing/selftests/bpf/prog_tests/ns_current_pid_tgid.c +1 −1
  121. tools/testing/selftests/bpf/prog_tests/perf_link.c +13 −2
  122. tools/testing/selftests/bpf/prog_tests/raw_tp_null.c +25 −0
  123. tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c +2 −2
  124. tools/testing/selftests/bpf/prog_tests/send_signal.c +111 −35
  125. tools/testing/selftests/bpf/prog_tests/sock_addr.c +0 −4
  126. tools/testing/selftests/bpf/prog_tests/sock_create.c +348 −0
  127. tools/testing/selftests/bpf/prog_tests/sock_post_bind.c +62 −192
  128. tools/testing/selftests/bpf/prog_tests/struct_ops_private_stack.c +106 −0
  129. tools/testing/selftests/bpf/prog_tests/subskeleton.c +75 −1
  130. tools/testing/selftests/bpf/prog_tests/tailcalls.c +114 −14
  131. tools/testing/selftests/bpf/prog_tests/task_kfunc.c +80 −0
  132. tools/testing/selftests/bpf/prog_tests/task_local_storage.c +282 −4
  133. tools/testing/selftests/bpf/prog_tests/timer_lockup.c +6 −0
  134. tools/testing/selftests/bpf/prog_tests/token.c +12 −7
  135. tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c +311 −50
  136. tools/testing/selftests/bpf/prog_tests/verifier.c +2 −0
  137. tools/testing/selftests/bpf/progs/bpf_iter.h +0 −167
  138. tools/testing/selftests/bpf/progs/bpf_iter_bpf_array_map.c +1 −1
  139. tools/testing/selftests/bpf/progs/bpf_iter_bpf_hash_map.c +1 −1
  140. tools/testing/selftests/bpf/progs/bpf_iter_bpf_link.c +1 −1
  141. tools/testing/selftests/bpf/progs/bpf_iter_bpf_map.c +1 −1
  142. tools/testing/selftests/bpf/progs/bpf_iter_bpf_percpu_array_map.c +1 −1
  143. tools/testing/selftests/bpf/progs/bpf_iter_bpf_percpu_hash_map.c +1 −1
  144. tools/testing/selftests/bpf/progs/bpf_iter_bpf_sk_storage_helpers.c +1 −1
  145. tools/testing/selftests/bpf/progs/bpf_iter_bpf_sk_storage_map.c +1 −1
  146. tools/testing/selftests/bpf/progs/bpf_iter_ipv6_route.c +1 −1
  147. tools/testing/selftests/bpf/progs/bpf_iter_ksym.c +1 −1
  148. tools/testing/selftests/bpf/progs/bpf_iter_netlink.c +1 −1
  149. tools/testing/selftests/bpf/progs/bpf_iter_setsockopt.c +1 −1
  150. tools/testing/selftests/bpf/progs/bpf_iter_setsockopt_unix.c +1 −1
  151. tools/testing/selftests/bpf/progs/bpf_iter_sockmap.c +1 −1
  152. tools/testing/selftests/bpf/progs/bpf_iter_task_btf.c +1 −1
  153. tools/testing/selftests/bpf/progs/bpf_iter_task_file.c +1 −1
  154. tools/testing/selftests/bpf/progs/bpf_iter_task_stack.c +1 −1
  155. tools/testing/selftests/bpf/progs/bpf_iter_task_vmas.c +1 −1
  156. tools/testing/selftests/bpf/progs/bpf_iter_tasks.c +1 −1
  157. tools/testing/selftests/bpf/progs/bpf_iter_tcp4.c +1 −1
  158. tools/testing/selftests/bpf/progs/bpf_iter_tcp6.c +1 −1
  159. tools/testing/selftests/bpf/progs/bpf_iter_test_kern3.c +1 −1
  160. tools/testing/selftests/bpf/progs/bpf_iter_test_kern4.c +1 −1
  161. tools/testing/selftests/bpf/progs/bpf_iter_test_kern5.c +1 −1
  162. tools/testing/selftests/bpf/progs/bpf_iter_test_kern6.c +1 −1
  163. tools/testing/selftests/bpf/progs/bpf_iter_test_kern_common.h +1 −1
  164. tools/testing/selftests/bpf/progs/bpf_iter_udp4.c +1 −1
  165. tools/testing/selftests/bpf/progs/bpf_iter_udp6.c +1 −1
  166. tools/testing/selftests/bpf/progs/bpf_iter_unix.c +1 −1
  167. tools/testing/selftests/bpf/progs/bpf_iter_vma_offset.c +1 −1
  168. tools/testing/selftests/bpf/progs/cgroup_iter.c +1 −2
  169. tools/testing/selftests/bpf/progs/cgrp_ls_sleepable.c +1 −2
  170. tools/testing/selftests/bpf/progs/exceptions_fail.c +2 −2
  171. tools/testing/selftests/bpf/progs/kmem_cache_iter.c +108 −0
  172. tools/testing/selftests/bpf/progs/kprobe_multi_verifier.c +31 −0
  173. tools/testing/selftests/bpf/progs/linked_funcs1.c +8 −0
  174. tools/testing/selftests/bpf/progs/linked_funcs2.c +8 −0
  175. tools/testing/selftests/bpf/progs/preempt_lock.c +7 −7
  176. tools/testing/selftests/bpf/progs/raw_tp_null.c +32 −0
  177. tools/testing/selftests/bpf/progs/struct_ops_detach.c +12 −0
  178. tools/testing/selftests/bpf/progs/struct_ops_private_stack.c +62 −0
  179. tools/testing/selftests/bpf/progs/struct_ops_private_stack_fail.c +62 −0
  180. tools/testing/selftests/bpf/progs/struct_ops_private_stack_recur.c +50 −0
  181. tools/testing/selftests/bpf/progs/tailcall_fail.c +64 −0
  182. tools/testing/selftests/bpf/progs/task_kfunc_common.h +1 −0
  183. tools/testing/selftests/bpf/progs/task_kfunc_failure.c +14 −0
  184. tools/testing/selftests/bpf/progs/task_kfunc_success.c +51 −0
  185. tools/testing/selftests/bpf/progs/task_ls_uptr.c +63 −0
  186. tools/testing/selftests/bpf/progs/tc_bpf2bpf.c +3 −2
  187. tools/testing/selftests/bpf/progs/test_send_signal_kern.c +29 −6
  188. tools/testing/selftests/bpf/progs/test_spin_lock_fail.c +2 −2
  189. tools/testing/selftests/bpf/progs/test_tp_btf_nullable.c +5 −1
  190. tools/testing/selftests/bpf/progs/update_map_in_htab.c +30 −0
  191. tools/testing/selftests/bpf/progs/uprobe_multi_consumers.c +3 −3
  192. tools/testing/selftests/bpf/progs/uprobe_multi_session.c +71 −0
  193. tools/testing/selftests/bpf/progs/uprobe_multi_session_cookie.c +48 −0
  194. tools/testing/selftests/bpf/progs/uprobe_multi_session_recursive.c +44 −0
  195. tools/testing/selftests/bpf/progs/uprobe_multi_session_single.c +44 −0
  196. tools/testing/selftests/bpf/progs/uprobe_multi_verifier.c +31 −0
  197. tools/testing/selftests/bpf/progs/uptr_failure.c +105 −0
  198. tools/testing/selftests/bpf/progs/uptr_map_failure.c +27 −0
  199. tools/testing/selftests/bpf/progs/uptr_update_failure.c +42 −0
  200. tools/testing/selftests/bpf/progs/verifier_arena_large.c +108 −2
  201. tools/testing/selftests/bpf/progs/verifier_private_stack.c +272 −0
  202. tools/testing/selftests/bpf/progs/verifier_ref_tracking.c +2 −2
  203. tools/testing/selftests/bpf/progs/verifier_sock.c +60 −0
  204. tools/testing/selftests/bpf/progs/verifier_spin_lock.c +1 −1
  205. tools/testing/selftests/bpf/test_maps.c +0 −4
  206. tools/testing/selftests/bpf/test_progs.c +111 −3
  207. tools/testing/selftests/bpf/test_progs.h +14 −0
  208. tools/testing/selftests/bpf/test_verifier.c +0 −4
  209. tools/testing/selftests/bpf/uprobe_multi.c +4 −0
  210. tools/testing/selftests/bpf/uptr_test_common.h +63 −0
  211. tools/testing/selftests/bpf/veristat.c +159 −2
Documentation/bpf/btf.rst+76−1
@@ -835,7 +835,7 @@ section named by ``btf_ext_info_sec->sec_name_off``. · 834 unmodified lines
835835 See :ref:`Documentation/bpf/llvm_reloc.rst <btf-co-re-relocations>`
836836 for more information on CO-RE relocations.
837837
838−4.2 .BTF_ids section
838+4.3 .BTF_ids section
839839 --------------------
840840
841841 The .BTF_ids section encodes BTF ID values that are used within the kernel.
@@ -896,6 +896,81 @@ and is used as a filter when resolving the BTF ID value. · 54 unmodified lines
896896 All the BTF ID lists and sets are compiled in the .BTF_ids section and
897897 resolved during the linking phase of kernel build by ``resolve_btfids`` tool.
898898
899+4.4 .BTF.base section
900+---------------------
901+Split BTF - where the .BTF section only contains types not in the associated
902+base .BTF section - is an extremely efficient way to encode type information
903+for kernel modules, since they generally consist of a few module-specific
904+types along with a large set of shared kernel types. The former are encoded
905+in split BTF, while the latter are encoded in base BTF, resulting in more
906+compact representations. A type in split BTF that refers to a type in
907+base BTF refers to it using its base BTF ID, and split BTF IDs start
908+at last_base_BTF_ID + 1.
909+
910+The downside of this approach however is that this makes the split BTF
911+somewhat brittle - when the base BTF changes, base BTF ID references are
912+no longer valid and the split BTF itself becomes useless. The role of the
913+.BTF.base section is to make split BTF more resilient for cases where
914+the base BTF may change, as is the case for kernel modules not built every
915+time the kernel is for example. .BTF.base contains named base types; INTs,
916+FLOATs, STRUCTs, UNIONs, ENUM[64]s and FWDs. INTs and FLOATs are fully
917+described in .BTF.base sections, while composite types like structs
918+and unions are not fully defined - the .BTF.base type simply serves as
919+a description of the type the split BTF referred to, so structs/unions
920+have 0 members in the .BTF.base section. ENUM[64]s are similarly recorded
921+with 0 members. Any other types are added to the split BTF. This
922+distillation process then leaves us with a .BTF.base section with
923+such minimal descriptions of base types and .BTF split section which refers
924+to those base types. Later, we can relocate the split BTF using both the
925+information stored in the .BTF.base section and the new .BTF base; the type
926+information in the .BTF.base section allows us to update the split BTF
927+references to point at the corresponding new base BTF IDs.
928+
929+BTF relocation happens on kernel module load when a kernel module has a
930+.BTF.base section, and libbpf also provides a btf__relocate() API to
931+accomplish this.
932+
933+As an example consider the following base BTF::
934+
935+ [1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED
936+ [2] STRUCT 'foo' size=8 vlen=2
937+ 'f1' type_id=1 bits_offset=0
938+ 'f2' type_id=1 bits_offset=32
939+
940+...and associated split BTF::
941+
942+ [3] PTR '(anon)' type_id=2
943+
944+i.e. split BTF describes a pointer to struct foo { int f1; int f2 };
945+
946+.BTF.base will consist of::
947+
948+ [1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED
949+ [2] STRUCT 'foo' size=8 vlen=0
950+
951+If we relocate the split BTF later using the following new base BTF::
952+
953+ [1] INT 'long unsigned int' size=8 bits_offset=0 nr_bits=64 encoding=(none)
954+ [2] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED
955+ [3] STRUCT 'foo' size=8 vlen=2
956+ 'f1' type_id=2 bits_offset=0
957+ 'f2' type_id=2 bits_offset=32
958+
959+...we can use our .BTF.base description to know that the split BTF reference
960+is to struct foo, and relocation results in new split BTF::
961+
962+ [4] PTR '(anon)' type_id=3
963+
964+Note that we had to update BTF ID and start BTF ID for the split BTF.
965+
966+So we see how .BTF.base plays the role of facilitating later relocation,
967+leading to more resilient split BTF.
968+
969+.BTF.base sections will be generated automatically for out-of-tree kernel module
970+builds - i.e. where KBUILD_EXTMOD is set (as it would be for "make M=path/2/mod"
971+cases). .BTF.base generation requires pahole support for the "distilled_base"
972+BTF feature; this is available in pahole v1.28 and later.
973+
899974 5. Using BTF
900975 ============
901976
Documentation/bpf/verifier.rst+2−2
@@ -507,7 +507,7 @@ Notes: · 506 unmodified lines
507507 from the parent state to the current state.
508508
509509 * Details about REG_LIVE_READ32 are omitted.
510−
510+
511511 * Function ``propagate_liveness()`` (see section :ref:`read_marks_for_cache_hits`)
512512 might override the first parent link. Please refer to the comments in the
513513 ``propagate_liveness()`` and ``mark_reg_read()`` source code for further
@@ -571,7 +571,7 @@ works:: · 57 unmodified lines
571571 are considered equivalent.
572572
573573 .. _read_marks_for_cache_hits:
574−
574+
575575 Read marks propagation for cache hits
576576 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
577577
arch/arm64/net/bpf_jit_comp.c+31−16
@@ -2094,6 +2094,12 @@ static void restore_args(struct jit_ctx *ctx, int args_off, int nregs) · 2093 unmodified lines
20942094 }
20952095 }
20962096
2097+static bool is_struct_ops_tramp(const struct bpf_tramp_links *fentry_links)
2098+{
2099+ return fentry_links->nr_links == 1 &&
2100+ fentry_links->links[0]->link.type == BPF_LINK_TYPE_STRUCT_OPS;
2101+}
2102+
20972103 /* Based on the x86's implementation of arch_prepare_bpf_trampoline().
20982104 *
20992105 * bpf prog and function entry before bpf trampoline hooked:
@@ -2123,6 +2129,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, · 23 unmodified lines
21232129 struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];
21242130 bool save_ret;
21252131 __le32 **branches = NULL;
2132+ bool is_struct_ops = is_struct_ops_tramp(fentry);
21262133
21272134 /* trampoline stack layout:
21282135 * [ parent ip ]
@@ -2191,11 +2198,14 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, · 62 unmodified lines
21912198 */
21922199 emit_bti(A64_BTI_JC, ctx);
21932200
2194− /* frame for parent function */
2195− emit(A64_PUSH(A64_FP, A64_R(9), A64_SP), ctx);
2196− emit(A64_MOV(1, A64_FP, A64_SP), ctx);
2201+ /* x9 is not set for struct_ops */
2202+ if (!is_struct_ops) {
2203+ /* frame for parent function */
2204+ emit(A64_PUSH(A64_FP, A64_R(9), A64_SP), ctx);
2205+ emit(A64_MOV(1, A64_FP, A64_SP), ctx);
2206+ }
21972207
2198− /* frame for patched function */
2208+ /* frame for patched function for tracing, or caller for struct_ops */
21992209 emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx);
22002210 emit(A64_MOV(1, A64_FP, A64_SP), ctx);
22012211
@@ -2289,19 +2299,24 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, · 87 unmodified lines
22892299 /* reset SP */
22902300 emit(A64_MOV(1, A64_SP, A64_FP), ctx);
22912301
2292− /* pop frames */
2293− emit(A64_POP(A64_FP, A64_LR, A64_SP), ctx);
2294− emit(A64_POP(A64_FP, A64_R(9), A64_SP), ctx);
2295−
2296− if (flags & BPF_TRAMP_F_SKIP_FRAME) {
2297− /* skip patched function, return to parent */
2298− emit(A64_MOV(1, A64_LR, A64_R(9)), ctx);
2299− emit(A64_RET(A64_R(9)), ctx);
2302+ if (is_struct_ops) {
2303+ emit(A64_POP(A64_FP, A64_LR, A64_SP), ctx);
2304+ emit(A64_RET(A64_LR), ctx);
23002305 } else {
2301− /* return to patched function */
2302− emit(A64_MOV(1, A64_R(10), A64_LR), ctx);
2303− emit(A64_MOV(1, A64_LR, A64_R(9)), ctx);
2304− emit(A64_RET(A64_R(10)), ctx);
2306+ /* pop frames */
2307+ emit(A64_POP(A64_FP, A64_LR, A64_SP), ctx);
2308+ emit(A64_POP(A64_FP, A64_R(9), A64_SP), ctx);
2309+
2310+ if (flags & BPF_TRAMP_F_SKIP_FRAME) {
2311+ /* skip patched function, return to parent */
2312+ emit(A64_MOV(1, A64_LR, A64_R(9)), ctx);
2313+ emit(A64_RET(A64_R(9)), ctx);
2314+ } else {
2315+ /* return to patched function */
2316+ emit(A64_MOV(1, A64_R(10), A64_LR), ctx);
2317+ emit(A64_MOV(1, A64_LR, A64_R(9)), ctx);
2318+ emit(A64_RET(A64_R(10)), ctx);
2319+ }
23052320 }
23062321
23072322 kfree(branches);
arch/x86/net/bpf_jit_comp.c+144−5
@@ -325,6 +325,22 @@ struct jit_context { · 324 unmodified lines
325325 /* Number of bytes that will be skipped on tailcall */
326326 #define X86_TAIL_CALL_OFFSET (12 + ENDBR_INSN_SIZE)
327327
328+static void push_r9(u8 **pprog)
329+{
330+ u8 *prog = *pprog;
331+
332+ EMIT2(0x41, 0x51); /* push r9 */
333+ *pprog = prog;
334+}
335+
336+static void pop_r9(u8 **pprog)
337+{
338+ u8 *prog = *pprog;
339+
340+ EMIT2(0x41, 0x59); /* pop r9 */
341+ *pprog = prog;
342+}
343+
328344 static void push_r12(u8 **pprog)
329345 {
330346 u8 *prog = *pprog;
@@ -1404,6 +1420,24 @@ static void emit_shiftx(u8 **pprog, u32 dst_reg, u8 src_reg, bool is64, u8 op) · 1073 unmodified lines
14041420 *pprog = prog;
14051421 }
14061422
1423+static void emit_priv_frame_ptr(u8 **pprog, void __percpu *priv_frame_ptr)
1424+{
1425+ u8 *prog = *pprog;
1426+
1427+ /* movabs r9, priv_frame_ptr */
1428+ emit_mov_imm64(&prog, X86_REG_R9, (__force long) priv_frame_ptr >> 32,
1429+ (u32) (__force long) priv_frame_ptr);
1430+
1431+#ifdef CONFIG_SMP
1432+ /* add <r9>, gs:[<off>] */
1433+ EMIT2(0x65, 0x4c);
1434+ EMIT3(0x03, 0x0c, 0x25);
1435+ EMIT((u32)(unsigned long)&this_cpu_off, 4);
1436+#endif
1437+
1438+ *pprog = prog;
1439+}
1440+
14071441 #define INSN_SZ_DIFF (((addrs[i] - addrs[i - 1]) - (prog - temp)))
14081442
14091443 #define __LOAD_TCC_PTR(off) \
@@ -1412,6 +1446,10 @@ static void emit_shiftx(u8 **pprog, u32 dst_reg, u8 src_reg, bool is64, u8 op) · 2 unmodified lines
14121446 #define LOAD_TAIL_CALL_CNT_PTR(stack) \
14131447 __LOAD_TCC_PTR(BPF_TAIL_CALL_CNT_PTR_STACK_OFF(stack))
14141448
1449+/* Memory size/value to protect private stack overflow/underflow */
1450+#define PRIV_STACK_GUARD_SZ 8
1451+#define PRIV_STACK_GUARD_VAL 0xEB9F12345678eb9fULL
1452+
14151453 static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image,
14161454 int oldproglen, struct jit_context *ctx, bool jmp_padding)
14171455 {
@@ -1421,18 +1459,28 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image · 3 unmodified lines
14211459 int insn_cnt = bpf_prog->len;
14221460 bool seen_exit = false;
14231461 u8 temp[BPF_MAX_INSN_SIZE + BPF_INSN_SAFETY];
1462+ void __percpu *priv_frame_ptr = NULL;
14241463 u64 arena_vm_start, user_vm_start;
1464+ void __percpu *priv_stack_ptr;
14251465 int i, excnt = 0;
14261466 int ilen, proglen = 0;
14271467 u8 *prog = temp;
1468+ u32 stack_depth;
14281469 int err;
14291470
1471+ stack_depth = bpf_prog->aux->stack_depth;
1472+ priv_stack_ptr = bpf_prog->aux->priv_stack_ptr;
1473+ if (priv_stack_ptr) {
1474+ priv_frame_ptr = priv_stack_ptr + PRIV_STACK_GUARD_SZ + round_up(stack_depth, 8);
1475+ stack_depth = 0;
1476+ }
1477+
14301478 arena_vm_start = bpf_arena_get_kern_vm_start(bpf_prog->aux->arena);
14311479 user_vm_start = bpf_arena_get_user_vm_start(bpf_prog->aux->arena);
14321480
14331481 detect_reg_usage(insn, insn_cnt, callee_regs_used);
14341482
1435− emit_prologue(&prog, bpf_prog->aux->stack_depth,
1483+ emit_prologue(&prog, stack_depth,
14361484 bpf_prog_was_classic(bpf_prog), tail_call_reachable,
14371485 bpf_is_subprog(bpf_prog), bpf_prog->aux->exception_cb);
14381486 /* Exception callback will clobber callee regs for its own use, and
@@ -1454,6 +1502,9 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image · 15 unmodified lines
14541502 emit_mov_imm64(&prog, X86_REG_R12,
14551503 arena_vm_start >> 32, (u32) arena_vm_start);
14561504
1505+ if (priv_frame_ptr)
1506+ emit_priv_frame_ptr(&prog, priv_frame_ptr);
1507+
14571508 ilen = prog - temp;
14581509 if (rw_image)
14591510 memcpy(rw_image + proglen, temp, ilen);
@@ -1473,6 +1524,14 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image · 13 unmodified lines
14731524 u8 *func;
14741525 int nops;
14751526
1527+ if (priv_frame_ptr) {
1528+ if (src_reg == BPF_REG_FP)
1529+ src_reg = X86_REG_R9;
1530+
1531+ if (dst_reg == BPF_REG_FP)
1532+ dst_reg = X86_REG_R9;
1533+ }
1534+
14761535 switch (insn->code) {
14771536 /* ALU */
14781537 case BPF_ALU | BPF_ADD | BPF_X:
@@ -2127,15 +2186,21 @@ populate_extable: · 648 unmodified lines
21272186 u8 *ip = image + addrs[i - 1];
21282187
21292188 func = (u8 *) __bpf_call_base + imm32;
2130− if (tail_call_reachable) {
2131− LOAD_TAIL_CALL_CNT_PTR(bpf_prog->aux->stack_depth);
2189+ if (src_reg == BPF_PSEUDO_CALL && tail_call_reachable) {
2190+ LOAD_TAIL_CALL_CNT_PTR(stack_depth);
21322191 ip += 7;
21332192 }
21342193 if (!imm32)
21352194 return -EINVAL;
2195+ if (priv_frame_ptr) {
2196+ push_r9(&prog);
2197+ ip += 2;
2198+ }
21362199 ip += x86_call_depth_emit_accounting(&prog, func, ip);
21372200 if (emit_call(&prog, func, ip))
21382201 return -EINVAL;
2202+ if (priv_frame_ptr)
2203+ pop_r9(&prog);
21392204 break;
21402205 }
21412206
@@ -2145,13 +2210,13 @@ populate_extable: · 3 unmodified lines
21452210 &bpf_prog->aux->poke_tab[imm32 - 1],
21462211 &prog, image + addrs[i - 1],
21472212 callee_regs_used,
2148− bpf_prog->aux->stack_depth,
2213+ stack_depth,
21492214 ctx);
21502215 else
21512216 emit_bpf_tail_call_indirect(bpf_prog,
21522217 &prog,
21532218 callee_regs_used,
2154− bpf_prog->aux->stack_depth,
2219+ stack_depth,
21552220 image + addrs[i - 1],
21562221 ctx);
21572222 break;
@@ -3303,6 +3368,42 @@ int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_func · 1145 unmodified lines
33033368 return emit_bpf_dispatcher(&prog, 0, num_funcs - 1, funcs, image, buf);
33043369 }
33053370
3371+static const char *bpf_get_prog_name(struct bpf_prog *prog)
3372+{
3373+ if (prog->aux->ksym.prog)
3374+ return prog->aux->ksym.name;
3375+ return prog->aux->name;
3376+}
3377+
3378+static void priv_stack_init_guard(void __percpu *priv_stack_ptr, int alloc_size)
3379+{
3380+ int cpu, underflow_idx = (alloc_size - PRIV_STACK_GUARD_SZ) >> 3;
3381+ u64 *stack_ptr;
3382+
3383+ for_each_possible_cpu(cpu) {
3384+ stack_ptr = per_cpu_ptr(priv_stack_ptr, cpu);
3385+ stack_ptr[0] = PRIV_STACK_GUARD_VAL;
3386+ stack_ptr[underflow_idx] = PRIV_STACK_GUARD_VAL;
3387+ }
3388+}
3389+
3390+static void priv_stack_check_guard(void __percpu *priv_stack_ptr, int alloc_size,
3391+ struct bpf_prog *prog)
3392+{
3393+ int cpu, underflow_idx = (alloc_size - PRIV_STACK_GUARD_SZ) >> 3;
3394+ u64 *stack_ptr;
3395+
3396+ for_each_possible_cpu(cpu) {
3397+ stack_ptr = per_cpu_ptr(priv_stack_ptr, cpu);
3398+ if (stack_ptr[0] != PRIV_STACK_GUARD_VAL ||
3399+ stack_ptr[underflow_idx] != PRIV_STACK_GUARD_VAL) {
3400+ pr_err("BPF private stack overflow/underflow detected for prog %sx\n",
3401+ bpf_get_prog_name(prog));
3402+ break;
3403+ }
3404+ }
3405+}
3406+
33063407 struct x64_jit_data {
33073408 struct bpf_binary_header *rw_header;
33083409 struct bpf_binary_header *header;
@@ -3320,7 +3421,9 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 11 unmodified lines
33203421 struct bpf_binary_header *rw_header = NULL;
33213422 struct bpf_binary_header *header = NULL;
33223423 struct bpf_prog *tmp, *orig_prog = prog;
3424+ void __percpu *priv_stack_ptr = NULL;
33233425 struct x64_jit_data *jit_data;
3426+ int priv_stack_alloc_sz;
33243427 int proglen, oldproglen = 0;
33253428 struct jit_context ctx = {};
33263429 bool tmp_blinded = false;
@@ -3356,6 +3459,23 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 29 unmodified lines
33563459 }
33573460 prog->aux->jit_data = jit_data;
33583461 }
3462+ priv_stack_ptr = prog->aux->priv_stack_ptr;
3463+ if (!priv_stack_ptr && prog->aux->jits_use_priv_stack) {
3464+ /* Allocate actual private stack size with verifier-calculated
3465+ * stack size plus two memory guards to protect overflow and
3466+ * underflow.
3467+ */
3468+ priv_stack_alloc_sz = round_up(prog->aux->stack_depth, 8) +
3469+ 2 * PRIV_STACK_GUARD_SZ;
3470+ priv_stack_ptr = __alloc_percpu_gfp(priv_stack_alloc_sz, 8, GFP_KERNEL);
3471+ if (!priv_stack_ptr) {
3472+ prog = orig_prog;
3473+ goto out_priv_stack;
3474+ }
3475+
3476+ priv_stack_init_guard(priv_stack_ptr, priv_stack_alloc_sz);
3477+ prog->aux->priv_stack_ptr = priv_stack_ptr;
3478+ }
33593479 addrs = jit_data->addrs;
33603480 if (addrs) {
33613481 ctx = jit_data->ctx;
@@ -3491,6 +3611,11 @@ out_image: · 129 unmodified lines
34913611 bpf_prog_fill_jited_linfo(prog, addrs + 1);
34923612 out_addrs:
34933613 kvfree(addrs);
3614+ if (!image && priv_stack_ptr) {
3615+ free_percpu(priv_stack_ptr);
3616+ prog->aux->priv_stack_ptr = NULL;
3617+ }
3618+out_priv_stack:
34943619 kfree(jit_data);
34953620 prog->aux->jit_data = NULL;
34963621 }
@@ -3529,6 +3654,8 @@ void bpf_jit_free(struct bpf_prog *prog) · 32 unmodified lines
35293654 if (prog->jited) {
35303655 struct x64_jit_data *jit_data = prog->aux->jit_data;
35313656 struct bpf_binary_header *hdr;
3657+ void __percpu *priv_stack_ptr;
3658+ int priv_stack_alloc_sz;
35323659
35333660 /*
35343661 * If we fail the final pass of JIT (from jit_subprogs),
@@ -3544,6 +3671,13 @@ void bpf_jit_free(struct bpf_prog *prog) · 9 unmodified lines
35443671 prog->bpf_func = (void *)prog->bpf_func - cfi_get_offset();
35453672 hdr = bpf_jit_binary_pack_hdr(prog);
35463673 bpf_jit_binary_pack_free(hdr, NULL);
3674+ priv_stack_ptr = prog->aux->priv_stack_ptr;
3675+ if (priv_stack_ptr) {
3676+ priv_stack_alloc_sz = round_up(prog->aux->stack_depth, 8) +
3677+ 2 * PRIV_STACK_GUARD_SZ;
3678+ priv_stack_check_guard(priv_stack_ptr, priv_stack_alloc_sz, prog);
3679+ free_percpu(prog->aux->priv_stack_ptr);
3680+ }
35473681 WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog));
35483682 }
35493683
@@ -3559,6 +3693,11 @@ bool bpf_jit_supports_exceptions(void) · 9 unmodified lines
35593693 return IS_ENABLED(CONFIG_UNWINDER_ORC);
35603694 }
35613695
3696+bool bpf_jit_supports_private_stack(void)
3697+{
3698+ return true;
3699+}
3700+
35623701 void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie)
35633702 {
35643703 #if defined(CONFIG_UNWINDER_ORC)
include/linux/bpf.h+54−9
@@ -203,6 +203,7 @@ enum btf_field_type { · 202 unmodified lines
203203 BPF_GRAPH_ROOT = BPF_RB_ROOT | BPF_LIST_HEAD,
204204 BPF_REFCOUNT = (1 << 9),
205205 BPF_WORKQUEUE = (1 << 10),
206+ BPF_UPTR = (1 << 11),
206207 };
207208
208209 typedef void (*btf_dtor_kfunc_t)(void *);
@@ -322,6 +323,8 @@ static inline const char *btf_field_type_name(enum btf_field_type type) · 113 unmodified lines
322323 return "kptr";
323324 case BPF_KPTR_PERCPU:
324325 return "percpu_kptr";
326+ case BPF_UPTR:
327+ return "uptr";
325328 case BPF_LIST_HEAD:
326329 return "bpf_list_head";
327330 case BPF_LIST_NODE:
@@ -350,6 +353,7 @@ static inline u32 btf_field_type_size(enum btf_field_type type) · 22 unmodified lines
350353 case BPF_KPTR_UNREF:
351354 case BPF_KPTR_REF:
352355 case BPF_KPTR_PERCPU:
356+ case BPF_UPTR:
353357 return sizeof(u64);
354358 case BPF_LIST_HEAD:
355359 return sizeof(struct bpf_list_head);
@@ -379,6 +383,7 @@ static inline u32 btf_field_type_align(enum btf_field_type type) · 23 unmodified lines
379383 case BPF_KPTR_UNREF:
380384 case BPF_KPTR_REF:
381385 case BPF_KPTR_PERCPU:
386+ case BPF_UPTR:
382387 return __alignof__(u64);
383388 case BPF_LIST_HEAD:
384389 return __alignof__(struct bpf_list_head);
@@ -419,6 +424,7 @@ static inline void bpf_obj_init_field(const struct btf_field *field, void *addr) · 34 unmodified lines
419424 case BPF_KPTR_UNREF:
420425 case BPF_KPTR_REF:
421426 case BPF_KPTR_PERCPU:
427+ case BPF_UPTR:
422428 break;
423429 default:
424430 WARN_ON_ONCE(1);
@@ -507,6 +513,25 @@ static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src · 82 unmodified lines
507513 bpf_obj_memcpy(map->record, dst, src, map->value_size, true);
508514 }
509515
516+static inline void bpf_obj_swap_uptrs(const struct btf_record *rec, void *dst, void *src)
517+{
518+ unsigned long *src_uptr, *dst_uptr;
519+ const struct btf_field *field;
520+ int i;
521+
522+ if (!btf_record_has_field(rec, BPF_UPTR))
523+ return;
524+
525+ for (i = 0, field = rec->fields; i < rec->cnt; i++, field++) {
526+ if (field->type != BPF_UPTR)
527+ continue;
528+
529+ src_uptr = src + field->offset;
530+ dst_uptr = dst + field->offset;
531+ swap(*src_uptr, *dst_uptr);
532+ }
533+}
534+
510535 static inline void bpf_obj_memzero(struct btf_record *rec, void *dst, u32 size)
511536 {
512537 u32 curr_off = 0;
@@ -907,10 +932,6 @@ enum bpf_reg_type { · 394 unmodified lines
907932 * additional context, assume the value is non-null.
908933 */
909934 PTR_TO_BTF_ID,
910− /* PTR_TO_BTF_ID_OR_NULL points to a kernel struct that has not
911− * been checked for null. Used primarily to inform the verifier
912− * an explicit null check is required for this struct.
913− */
914935 PTR_TO_MEM, /* reg points to valid memory region */
915936 PTR_TO_ARENA,
916937 PTR_TO_BUF, /* reg points to a read/write buffer */
@@ -923,6 +944,10 @@ enum bpf_reg_type { · 6 unmodified lines
923944 PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | PTR_TO_SOCKET,
924945 PTR_TO_SOCK_COMMON_OR_NULL = PTR_MAYBE_NULL | PTR_TO_SOCK_COMMON,
925946 PTR_TO_TCP_SOCK_OR_NULL = PTR_MAYBE_NULL | PTR_TO_TCP_SOCK,
947+ /* PTR_TO_BTF_ID_OR_NULL points to a kernel struct that has not
948+ * been checked for null. Used primarily to inform the verifier
949+ * an explicit null check is required for this struct.
950+ */
926951 PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | PTR_TO_BTF_ID,
927952
928953 /* This must be the last entry. Its purpose is to ensure the enum is
@@ -1300,8 +1325,12 @@ void *__bpf_dynptr_data_rw(const struct bpf_dynptr_kern *ptr, u32 len); · 371 unmodified lines
13001325 bool __bpf_dynptr_is_rdonly(const struct bpf_dynptr_kern *ptr);
13011326
13021327 #ifdef CONFIG_BPF_JIT
1303−int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
1304−int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
1328+int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
1329+ struct bpf_trampoline *tr,
1330+ struct bpf_prog *tgt_prog);
1331+int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
1332+ struct bpf_trampoline *tr,
1333+ struct bpf_prog *tgt_prog);
13051334 struct bpf_trampoline *bpf_trampoline_get(u64 key,
13061335 struct bpf_attach_target_info *tgt_info);
13071336 void bpf_trampoline_put(struct bpf_trampoline *tr);
@@ -1373,7 +1402,8 @@ int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_func · 65 unmodified lines
13731402 void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from,
13741403 struct bpf_prog *to);
13751404 /* Called only from JIT-enabled code, so there's no need for stubs. */
1376−void bpf_image_ksym_add(void *data, unsigned int size, struct bpf_ksym *ksym);
1405+void bpf_image_ksym_init(void *data, unsigned int size, struct bpf_ksym *ksym);
1406+void bpf_image_ksym_add(struct bpf_ksym *ksym);
13771407 void bpf_image_ksym_del(struct bpf_ksym *ksym);
13781408 void bpf_ksym_add(struct bpf_ksym *ksym);
13791409 void bpf_ksym_del(struct bpf_ksym *ksym);
@@ -1382,12 +1412,14 @@ void bpf_jit_uncharge_modmem(u32 size); · 2 unmodified lines
13821412 bool bpf_prog_has_trampoline(const struct bpf_prog *prog);
13831413 #else
13841414 static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
1385− struct bpf_trampoline *tr)
1415+ struct bpf_trampoline *tr,
1416+ struct bpf_prog *tgt_prog)
13861417 {
13871418 return -ENOTSUPP;
13881419 }
13891420 static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
1390− struct bpf_trampoline *tr)
1421+ struct bpf_trampoline *tr,
1422+ struct bpf_prog *tgt_prog)
13911423 {
13921424 return -ENOTSUPP;
13931425 }
@@ -1476,6 +1508,7 @@ struct bpf_prog_aux { · 82 unmodified lines
14761508 u32 max_rdwr_access;
14771509 struct btf *attach_btf;
14781510 const struct bpf_ctx_arg_aux *ctx_arg_info;
1511+ void __percpu *priv_stack_ptr;
14791512 struct mutex dst_mutex; /* protects dst_* pointers below, *after* prog becomes visible */
14801513 struct bpf_prog *dst_prog;
14811514 struct bpf_trampoline *dst_trampoline;
@@ -1491,7 +1524,13 @@ struct bpf_prog_aux { · 9 unmodified lines
14911524 bool xdp_has_frags;
14921525 bool exception_cb;
14931526 bool exception_boundary;
1527+ bool is_extended; /* true if extended by freplace program */
1528+ bool jits_use_priv_stack;
1529+ bool priv_stack_requested;
1530+ u64 prog_array_member_cnt; /* counts how many times as member of prog_array */
1531+ struct mutex ext_mutex; /* mutex for is_extended and prog_array_member_cnt */
14941532 struct bpf_arena *arena;
1533+ void (*recursion_detected)(struct bpf_prog *prog); /* callback if recursion is detected */
14951534 /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */
14961535 const struct btf_type *attach_func_proto;
14971536 /* function name for valid attach_btf_id */
@@ -3461,4 +3500,10 @@ static inline bool bpf_is_subprog(const struct bpf_prog *prog) · 1963 unmodified lines
34613500 return prog->aux->func_idx != 0;
34623501 }
34633502
3503+static inline bool bpf_prog_is_raw_tp(const struct bpf_prog *prog)
3504+{
3505+ return prog->type == BPF_PROG_TYPE_TRACING &&
3506+ prog->expected_attach_type == BPF_TRACE_RAW_TP;
3507+}
3508+
34643509 #endif /* _LINUX_BPF_H */
include/linux/bpf_local_storage.h+9−3
@@ -77,7 +77,13 @@ struct bpf_local_storage_elem { · 76 unmodified lines
7777 struct hlist_node map_node; /* Linked to bpf_local_storage_map */
7878 struct hlist_node snode; /* Linked to bpf_local_storage */
7979 struct bpf_local_storage __rcu *local_storage;
80− struct rcu_head rcu;
80+ union {
81+ struct rcu_head rcu;
82+ struct hlist_node free_node; /* used to postpone
83+ * bpf_selem_free
84+ * after raw_spin_unlock
85+ */
86+ };
8187 /* 8 bytes hole */
8288 /* The data is stored in another cacheline to minimize
8389 * the number of cachelines access during a cache hit.
@@ -181,7 +187,7 @@ void bpf_selem_link_map(struct bpf_local_storage_map *smap, · 97 unmodified lines
181187
182188 struct bpf_local_storage_elem *
183189 bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, void *value,
184− bool charge_mem, gfp_t gfp_flags);
190+ bool charge_mem, bool swap_uptrs, gfp_t gfp_flags);
185191
186192 void bpf_selem_free(struct bpf_local_storage_elem *selem,
187193 struct bpf_local_storage_map *smap,
@@ -195,7 +201,7 @@ bpf_local_storage_alloc(void *owner, · 7 unmodified lines
195201
196202 struct bpf_local_storage_data *
197203 bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
198− void *value, u64 map_flags, gfp_t gfp_flags);
204+ void *value, u64 map_flags, bool swap_uptrs, gfp_t gfp_flags);
199205
200206 u64 bpf_local_storage_map_mem_usage(const struct bpf_map *map);
201207
include/linux/bpf_verifier.h+31−36
@@ -48,22 +48,6 @@ enum bpf_reg_liveness { · 47 unmodified lines
4848 REG_LIVE_DONE = 0x8, /* liveness won't be updating this register anymore */
4949 };
5050
51−/* For every reg representing a map value or allocated object pointer,
52− * we consider the tuple of (ptr, id) for them to be unique in verifier
53− * context and conside them to not alias each other for the purposes of
54− * tracking lock state.
55− */
56−struct bpf_active_lock {
57− /* This can either be reg->map_ptr or reg->btf. If ptr is NULL,
58− * there's no active lock held, and other fields have no
59− * meaning. If non-NULL, it indicates that a lock is held and
60− * id member has the reg->id of the register which can be >= 0.
61− */
62− void *ptr;
63− /* This will be reg->id */
64− u32 id;
65−};
66−
6751 #define ITER_PREFIX "bpf_iter_"
6852
6953 enum bpf_iter_state {
@@ -266,6 +250,13 @@ struct bpf_stack_state { · 196 unmodified lines
266250 };
267251
268252 struct bpf_reference_state {
253+ /* Each reference object has a type. Ensure REF_TYPE_PTR is zero to
254+ * default to pointer reference on zero initialization of a state.
255+ */
256+ enum ref_state_type {
257+ REF_TYPE_PTR = 0,
258+ REF_TYPE_LOCK,
259+ } type;
269260 /* Track each reference created with a unique id, even if the same
270261 * instruction creates the reference multiple times (eg, via CALL).
271262 */
@@ -274,17 +265,10 @@ struct bpf_reference_state { · 2 unmodified lines
274265 * is used purely to inform the user of a reference leak.
275266 */
276267 int insn_idx;
277− /* There can be a case like:
278− * main (frame 0)
279− * cb (frame 1)
280− * func (frame 3)
281− * cb (frame 4)
282− * Hence for frame 4, if callback_ref just stored boolean, it would be
283− * impossible to distinguish nested callback refs. Hence store the
284− * frameno and compare that to callback_ref in check_reference_leak when
285− * exiting a callback function.
286− */
287− int callback_ref;
268+ /* Use to keep track of the source object of a lock, to ensure
269+ * it matches on unlock.
270+ */
271+ void *ptr;
288272 };
289273
290274 struct bpf_retval_range {
@@ -332,6 +316,7 @@ struct bpf_func_state { · 41 unmodified lines
332316
333317 /* The following fields should be last. See copy_func_state() */
334318 int acquired_refs;
319+ int active_locks;
335320 struct bpf_reference_state *refs;
336321 /* The state of the stack. Each element of the array describes BPF_REG_SIZE
337322 * (i.e. 8) bytes worth of stack memory.
@@ -349,7 +334,7 @@ struct bpf_func_state { · 11 unmodified lines
349334
350335 #define MAX_CALL_FRAMES 8
351336
352−/* instruction history flags, used in bpf_jmp_history_entry.flags field */
337+/* instruction history flags, used in bpf_insn_hist_entry.flags field */
353338 enum {
354339 /* instruction references stack slot through PTR_TO_STACK register;
355340 * we also store stack's frame number in lower 3 bits (MAX_CALL_FRAMES is 8)
@@ -367,7 +352,7 @@ enum { · 11 unmodified lines
367352 static_assert(INSN_F_FRAMENO_MASK + 1 >= MAX_CALL_FRAMES);
368353 static_assert(INSN_F_SPI_MASK + 1 >= MAX_BPF_STACK / 8);
369354
370−struct bpf_jmp_history_entry {
355+struct bpf_insn_hist_entry {
371356 u32 idx;
372357 /* insn idx can't be bigger than 1 million */
373358 u32 prev_idx : 22;
@@ -434,7 +419,6 @@ struct bpf_verifier_state { · 60 unmodified lines
434419 u32 insn_idx;
435420 u32 curframe;
436421
437− struct bpf_active_lock active_lock;
438422 bool speculative;
439423 bool active_rcu_lock;
440424 u32 active_preempt_lock;
@@ -458,13 +442,14 @@ struct bpf_verifier_state { · 17 unmodified lines
458442 * See get_loop_entry() for more information.
459443 */
460444 struct bpf_verifier_state *loop_entry;
461− /* jmp history recorded from first to last.
462− * backtracking is using it to go from last to first.
463− * For most states jmp_history_cnt is [0-3].
445+ /* Sub-range of env->insn_hist[] corresponding to this state's
446+ * instruction history.
447+ * Backtracking is using it to go from last to first.
448+ * For most states instruction history is short, 0-3 instructions.
464449 * For loops can go up to ~40.
465450 */
466− struct bpf_jmp_history_entry *jmp_history;
467− u32 jmp_history_cnt;
451+ u32 insn_hist_start;
452+ u32 insn_hist_end;
468453 u32 dfs_depth;
469454 u32 callback_unroll_depth;
470455 u32 may_goto_depth;
@@ -649,6 +634,12 @@ struct bpf_subprog_arg_info { · 178 unmodified lines
649634 };
650635 };
651636
637+enum priv_stack_mode {
638+ PRIV_STACK_UNKNOWN,
639+ NO_PRIV_STACK,
640+ PRIV_STACK_ADAPTIVE,
641+};
642+
652643 struct bpf_subprog_info {
653644 /* 'start' has to be the first field otherwise find_subprog() won't work */
654645 u32 start; /* insn idx of function entry point */
@@ -669,6 +660,7 @@ struct bpf_subprog_info { · 14 unmodified lines
669660 /* true if bpf_fastcall stack region is used by functions that can't be inlined */
670661 bool keep_fastcall_stack: 1;
671662
663+ enum priv_stack_mode priv_stack_mode;
672664 u8 arg_cnt;
673665 struct bpf_subprog_arg_info args[MAX_BPF_FUNC_REG_ARGS];
674666 };
@@ -747,7 +739,9 @@ struct bpf_verifier_env { · 72 unmodified lines
747739 int cur_stack;
748740 } cfg;
749741 struct backtrack_state bt;
750− struct bpf_jmp_history_entry *cur_hist_ent;
742+ struct bpf_insn_hist_entry *insn_hist;
743+ struct bpf_insn_hist_entry *cur_hist_ent;
744+ u32 insn_hist_cap;
751745 u32 pass_cnt; /* number of times do_check() was called */
752746 u32 subprog_cnt;
753747 /* number of instructions analyzed by the verifier */
@@ -888,6 +882,7 @@ static inline bool bpf_prog_check_recur(const struct bpf_prog *prog) · 134 unmodified lines
888882 case BPF_PROG_TYPE_TRACING:
889883 return prog->expected_attach_type != BPF_TRACE_ITER;
890884 case BPF_PROG_TYPE_STRUCT_OPS:
885+ return prog->aux->jits_use_priv_stack;
891886 case BPF_PROG_TYPE_LSM:
892887 return false;
893888 default:
include/linux/btf.h+11−11
@@ -75,6 +75,7 @@ · 74 unmodified lines
7575 #define KF_ITER_NEXT (1 << 9) /* kfunc implements BPF iter next method */
7676 #define KF_ITER_DESTROY (1 << 10) /* kfunc implements BPF iter destructor */
7777 #define KF_RCU_PROTECTED (1 << 11) /* kfunc should be protected by rcu cs when they are invoked */
78+#define KF_FASTCALL (1 << 12) /* kfunc supports bpf_fastcall protocol */
7879
7980 /*
8081 * Tag marking a kernel function as a kfunc. This is meant to minimize the
@@ -581,6 +582,16 @@ int get_kern_ctx_btf_id(struct bpf_verifier_log *log, enum bpf_prog_type prog_ty · 500 unmodified lines
581582 bool btf_types_are_same(const struct btf *btf1, u32 id1,
582583 const struct btf *btf2, u32 id2);
583584 int btf_check_iter_arg(struct btf *btf, const struct btf_type *func, int arg_idx);
585+
586+static inline bool btf_type_is_struct_ptr(struct btf *btf, const struct btf_type *t)
587+{
588+ if (!btf_type_is_ptr(t))
589+ return false;
590+
591+ t = btf_type_skip_modifiers(btf, t->type, NULL);
592+
593+ return btf_type_is_struct(t);
594+}
584595 #else
585596 static inline const struct btf_type *btf_type_by_id(const struct btf *btf,
586597 u32 type_id)
@@ -660,15 +671,4 @@ static inline int btf_check_iter_arg(struct btf *btf, const struct btf_type *fun · 73 unmodified lines
660671 return -EOPNOTSUPP;
661672 }
662673 #endif
663−
664−static inline bool btf_type_is_struct_ptr(struct btf *btf, const struct btf_type *t)
665−{
666− if (!btf_type_is_ptr(t))
667− return false;
668−
669− t = btf_type_skip_modifiers(btf, t->type, NULL);
670−
671− return btf_type_is_struct(t);
672−}
673−
674674 #endif
include/linux/btf_ids.h+1−0
@@ -283,5 +283,6 @@ extern u32 btf_tracing_ids[]; · 282 unmodified lines
283283 extern u32 bpf_cgroup_btf_id[];
284284 extern u32 bpf_local_storage_map_btf_id[];
285285 extern u32 btf_bpf_map_id[];
286+extern u32 bpf_kmem_cache_btf_id[];
286287
287288 #endif
include/linux/filter.h+1−0
@@ -1119,6 +1119,7 @@ bool bpf_jit_supports_exceptions(void); · 1118 unmodified lines
11191119 bool bpf_jit_supports_ptr_xchg(void);
11201120 bool bpf_jit_supports_arena(void);
11211121 bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena);
1122+bool bpf_jit_supports_private_stack(void);
11221123 u64 bpf_arch_uaddress_limit(void);
11231124 void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie);
11241125 bool bpf_helper_changes_pkt_data(void *func);
include/uapi/linux/bpf.h+4−5
@@ -1116,6 +1116,7 @@ enum bpf_attach_type { · 1115 unmodified lines
11161116 BPF_NETKIT_PRIMARY,
11171117 BPF_NETKIT_PEER,
11181118 BPF_TRACE_KPROBE_SESSION,
1119+ BPF_TRACE_UPROBE_SESSION,
11191120 __MAX_BPF_ATTACH_TYPE
11201121 };
11211122
@@ -1973,6 +1974,8 @@ union bpf_attr { · 851 unmodified lines
19731974 * program.
19741975 * Return
19751976 * The SMP id of the processor running the program.
1977+ * Attributes
1978+ * __bpf_fastcall
19761979 *
19771980 * long bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from, u32 len, u64 flags)
19781981 * Description
@@ -3104,10 +3107,6 @@ union bpf_attr { · 1125 unmodified lines
31043107 * with the **CONFIG_BPF_KPROBE_OVERRIDE** configuration
31053108 * option, and in this case it only works on functions tagged with
31063109 * **ALLOW_ERROR_INJECTION** in the kernel code.
3107− *
3108− * Also, the helper is only available for the architectures having
3109− * the CONFIG_FUNCTION_ERROR_INJECTION option. As of this writing,
3110− * x86 architecture is the only one to support this feature.
31113110 * Return
31123111 * 0
31133112 *
@@ -5372,7 +5371,7 @@ union bpf_attr { · 2258 unmodified lines
53725371 * Currently, the **flags** must be 0. Currently, nr_loops is
53735372 * limited to 1 << 23 (~8 million) loops.
53745373 *
5375− * long (\*callback_fn)(u32 index, void \*ctx);
5374+ * long (\*callback_fn)(u64 index, void \*ctx);
53765375 *
53775376 * where **index** is the current index in the loop. The index
53785377 * is zero-indexed.
kernel/bpf/Makefile+2−1
@@ -16,7 +16,7 @@ obj-$(CONFIG_BPF_SYSCALL) += disasm.o mprog.o · 15 unmodified lines
1616 obj-$(CONFIG_BPF_JIT) += trampoline.o
1717 obj-$(CONFIG_BPF_SYSCALL) += btf.o memalloc.o
1818 ifeq ($(CONFIG_MMU)$(CONFIG_64BIT),yy)
19−obj-$(CONFIG_BPF_SYSCALL) += arena.o
19+obj-$(CONFIG_BPF_SYSCALL) += arena.o range_tree.o
2020 endif
2121 obj-$(CONFIG_BPF_JIT) += dispatcher.o
2222 ifeq ($(CONFIG_NET),y)
@@ -52,3 +52,4 @@ obj-$(CONFIG_BPF_PRELOAD) += preload/ · 29 unmodified lines
5252 obj-$(CONFIG_BPF_SYSCALL) += relo_core.o
5353 obj-$(CONFIG_BPF_SYSCALL) += btf_iter.o
5454 obj-$(CONFIG_BPF_SYSCALL) += btf_relocate.o
55+obj-$(CONFIG_BPF_SYSCALL) += kmem_cache_iter.o
kernel/bpf/arena.c+24−14
@@ -3,9 +3,11 @@ · 2 unmodified lines
33 #include <linux/bpf.h>
44 #include <linux/btf.h>
55 #include <linux/err.h>
6+#include "linux/filter.h"
67 #include <linux/btf_ids.h>
78 #include <linux/vmalloc.h>
89 #include <linux/pagemap.h>
10+#include "range_tree.h"
911
1012 /*
1113 * bpf_arena is a sparsely populated shared memory region between bpf program and
@@ -45,7 +47,7 @@ struct bpf_arena { · 33 unmodified lines
4547 u64 user_vm_start;
4648 u64 user_vm_end;
4749 struct vm_struct *kern_vm;
48− struct maple_tree mt;
50+ struct range_tree rt;
4951 struct list_head vma_list;
5052 struct mutex lock;
5153 };
@@ -98,6 +100,9 @@ static struct bpf_map *arena_map_alloc(union bpf_attr *attr) · 46 unmodified lines
98100 u64 vm_range;
99101 int err = -ENOMEM;
100102
103+ if (!bpf_jit_supports_arena())
104+ return ERR_PTR(-EOPNOTSUPP);
105+
101106 if (attr->key_size || attr->value_size || attr->max_entries == 0 ||
102107 /* BPF_F_MMAPABLE must be set */
103108 !(attr->map_flags & BPF_F_MMAPABLE) ||
@@ -132,7 +137,8 @@ static struct bpf_map *arena_map_alloc(union bpf_attr *attr) · 28 unmodified lines
132137
133138 INIT_LIST_HEAD(&arena->vma_list);
134139 bpf_map_init_from_attr(&arena->map, attr);
135− mt_init_flags(&arena->mt, MT_FLAGS_ALLOC_RANGE);
140+ range_tree_init(&arena->rt);
141+ range_tree_set(&arena->rt, 0, attr->max_entries);
136142 mutex_init(&arena->lock);
137143
138144 return &arena->map;
@@ -183,7 +189,7 @@ static void arena_map_free(struct bpf_map *map) · 44 unmodified lines
183189 apply_to_existing_page_range(&init_mm, bpf_arena_get_kern_vm_start(arena),
184190 KERN_VM_SZ - GUARD_SZ, existing_page_cb, NULL);
185191 free_vm_area(arena->kern_vm);
186− mtree_destroy(&arena->mt);
192+ range_tree_destroy(&arena->rt);
187193 bpf_map_area_free(arena);
188194 }
189195
@@ -274,20 +280,20 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) · 84 unmodified lines
274280 /* User space requested to segfault when page is not allocated by bpf prog */
275281 return VM_FAULT_SIGSEGV;
276282
277− ret = mtree_insert(&arena->mt, vmf->pgoff, MT_ENTRY, GFP_KERNEL);
283+ ret = range_tree_clear(&arena->rt, vmf->pgoff, 1);
278284 if (ret)
279285 return VM_FAULT_SIGSEGV;
280286
281287 /* Account into memcg of the process that created bpf_arena */
282288 ret = bpf_map_alloc_pages(map, GFP_KERNEL | __GFP_ZERO, NUMA_NO_NODE, 1, &page);
283289 if (ret) {
284− mtree_erase(&arena->mt, vmf->pgoff);
290+ range_tree_set(&arena->rt, vmf->pgoff, 1);
285291 return VM_FAULT_SIGSEGV;
286292 }
287293
288294 ret = vm_area_map_pages(arena->kern_vm, kaddr, kaddr + PAGE_SIZE, &page);
289295 if (ret) {
290− mtree_erase(&arena->mt, vmf->pgoff);
296+ range_tree_set(&arena->rt, vmf->pgoff, 1);
291297 __free_page(page);
292298 return VM_FAULT_SIGSEGV;
293299 }
@@ -444,12 +450,16 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt · 150 unmodified lines
444450
445451 guard(mutex)(&arena->lock);
446452
447− if (uaddr)
448− ret = mtree_insert_range(&arena->mt, pgoff, pgoff + page_cnt - 1,
449− MT_ENTRY, GFP_KERNEL);
450− else
451− ret = mtree_alloc_range(&arena->mt, &pgoff, MT_ENTRY,
452− page_cnt, 0, page_cnt_max - 1, GFP_KERNEL);
453+ if (uaddr) {
454+ ret = is_range_tree_set(&arena->rt, pgoff, page_cnt);
455+ if (ret)
456+ goto out_free_pages;
457+ ret = range_tree_clear(&arena->rt, pgoff, page_cnt);
458+ } else {
459+ ret = pgoff = range_tree_find(&arena->rt, page_cnt);
460+ if (pgoff >= 0)
461+ ret = range_tree_clear(&arena->rt, pgoff, page_cnt);
462+ }
453463 if (ret)
454464 goto out_free_pages;
455465
@@ -476,7 +486,7 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt · 20 unmodified lines
476486 kvfree(pages);
477487 return clear_lo32(arena->user_vm_start) + uaddr32;
478488 out:
479− mtree_erase(&arena->mt, pgoff);
489+ range_tree_set(&arena->rt, pgoff, page_cnt);
480490 out_free_pages:
481491 kvfree(pages);
482492 return 0;
@@ -516,7 +526,7 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt) · 33 unmodified lines
516526
517527 pgoff = compute_pgoff(arena, uaddr);
518528 /* clear range */
519− mtree_store_range(&arena->mt, pgoff, pgoff + page_cnt - 1, NULL, GFP_KERNEL);
529+ range_tree_set(&arena->rt, pgoff, page_cnt);
520530
521531 if (page_cnt > 1)
522532 /* bulk zap if multiple pages being freed */
kernel/bpf/arraymap.c+24−2
@@ -947,22 +947,44 @@ static void *prog_fd_array_get_ptr(struct bpf_map *map, · 946 unmodified lines
947947 struct file *map_file, int fd)
948948 {
949949 struct bpf_prog *prog = bpf_prog_get(fd);
950+ bool is_extended;
950951
951952 if (IS_ERR(prog))
952953 return prog;
953954
954− if (!bpf_prog_map_compatible(map, prog)) {
955+ if (prog->type == BPF_PROG_TYPE_EXT ||
956+ !bpf_prog_map_compatible(map, prog)) {
955957 bpf_prog_put(prog);
956958 return ERR_PTR(-EINVAL);
957959 }
958960
961+ mutex_lock(&prog->aux->ext_mutex);
962+ is_extended = prog->aux->is_extended;
963+ if (!is_extended)
964+ prog->aux->prog_array_member_cnt++;
965+ mutex_unlock(&prog->aux->ext_mutex);
966+ if (is_extended) {
967+ /* Extended prog can not be tail callee. It's to prevent a
968+ * potential infinite loop like:
969+ * tail callee prog entry -> tail callee prog subprog ->
970+ * freplace prog entry --tailcall-> tail callee prog entry.
971+ */
972+ bpf_prog_put(prog);
973+ return ERR_PTR(-EBUSY);
974+ }
975+
959976 return prog;
960977 }
961978
962979 static void prog_fd_array_put_ptr(struct bpf_map *map, void *ptr, bool need_defer)
963980 {
981+ struct bpf_prog *prog = ptr;
982+
983+ mutex_lock(&prog->aux->ext_mutex);
984+ prog->aux->prog_array_member_cnt--;
985+ mutex_unlock(&prog->aux->ext_mutex);
964986 /* bpf_prog is freed after one RCU or tasks trace grace period */
965− bpf_prog_put(ptr);
987+ bpf_prog_put(prog);
966988 }
967989
968990 static u32 prog_fd_array_sys_lookup_elem(void *ptr)
kernel/bpf/bpf_cgrp_storage.c+2−2
@@ -107,7 +107,7 @@ static long bpf_cgrp_storage_update_elem(struct bpf_map *map, void *key, · 106 unmodified lines
107107
108108 bpf_cgrp_storage_lock();
109109 sdata = bpf_local_storage_update(cgroup, (struct bpf_local_storage_map *)map,
110− value, map_flags, GFP_ATOMIC);
110+ value, map_flags, false, GFP_ATOMIC);
111111 bpf_cgrp_storage_unlock();
112112 cgroup_put(cgroup);
113113 return PTR_ERR_OR_ZERO(sdata);
@@ -181,7 +181,7 @@ BPF_CALL_5(bpf_cgrp_storage_get, struct bpf_map *, map, struct cgroup *, cgroup, · 67 unmodified lines
181181 if (!percpu_ref_is_dying(&cgroup->self.refcnt) &&
182182 (flags & BPF_LOCAL_STORAGE_GET_F_CREATE))
183183 sdata = bpf_local_storage_update(cgroup, (struct bpf_local_storage_map *)map,
184− value, BPF_NOEXIST, gfp_flags);
184+ value, BPF_NOEXIST, false, gfp_flags);
185185
186186 unlock:
187187 bpf_cgrp_storage_unlock();
kernel/bpf/bpf_inode_storage.c+2−2
@@ -99,7 +99,7 @@ static long bpf_fd_inode_storage_update_elem(struct bpf_map *map, void *key, · 98 unmodified lines
9999
100100 sdata = bpf_local_storage_update(file_inode(fd_file(f)),
101101 (struct bpf_local_storage_map *)map,
102− value, map_flags, GFP_ATOMIC);
102+ value, map_flags, false, GFP_ATOMIC);
103103 return PTR_ERR_OR_ZERO(sdata);
104104 }
105105
@@ -153,7 +153,7 @@ BPF_CALL_5(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode, · 47 unmodified lines
153153 if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) {
154154 sdata = bpf_local_storage_update(
155155 inode, (struct bpf_local_storage_map *)map, value,
156− BPF_NOEXIST, gfp_flags);
156+ BPF_NOEXIST, false, gfp_flags);
157157 return IS_ERR(sdata) ? (unsigned long)NULL :
158158 (unsigned long)sdata->data;
159159 }
kernel/bpf/bpf_local_storage.c+63−16
@@ -73,7 +73,7 @@ static bool selem_linked_to_map(const struct bpf_local_storage_elem *selem) · 72 unmodified lines
7373
7474 struct bpf_local_storage_elem *
7575 bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner,
76− void *value, bool charge_mem, gfp_t gfp_flags)
76+ void *value, bool charge_mem, bool swap_uptrs, gfp_t gfp_flags)
7777 {
7878 struct bpf_local_storage_elem *selem;
7979
@@ -99,9 +99,12 @@ bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, · 19 unmodified lines
9999 }
100100
101101 if (selem) {
102− if (value)
102+ if (value) {
103+ /* No need to call check_and_init_map_value as memory is zero init */
103104 copy_map_value(&smap->map, SDATA(selem)->data, value);
104− /* No need to call check_and_init_map_value as memory is zero init */
105+ if (swap_uptrs)
106+ bpf_obj_swap_uptrs(smap->map.record, SDATA(selem)->data, value);
107+ }
105108 return selem;
106109 }
107110
@@ -209,8 +212,12 @@ static void __bpf_selem_free(struct bpf_local_storage_elem *selem, · 101 unmodified lines
209212 static void bpf_selem_free_rcu(struct rcu_head *rcu)
210213 {
211214 struct bpf_local_storage_elem *selem;
215+ struct bpf_local_storage_map *smap;
212216
213217 selem = container_of(rcu, struct bpf_local_storage_elem, rcu);
218+ /* The bpf_local_storage_map_free will wait for rcu_barrier */
219+ smap = rcu_dereference_check(SDATA(selem)->smap, 1);
220+ bpf_obj_free_fields(smap->map.record, SDATA(selem)->data);
214221 bpf_mem_cache_raw_free(selem);
215222 }
216223
@@ -226,16 +233,25 @@ void bpf_selem_free(struct bpf_local_storage_elem *selem, · 9 unmodified lines
226233 struct bpf_local_storage_map *smap,
227234 bool reuse_now)
228235 {
229− bpf_obj_free_fields(smap->map.record, SDATA(selem)->data);
230−
231236 if (!smap->bpf_ma) {
237+ /* Only task storage has uptrs and task storage
238+ * has moved to bpf_mem_alloc. Meaning smap->bpf_ma == true
239+ * for task storage, so this bpf_obj_free_fields() won't unpin
240+ * any uptr.
241+ */
242+ bpf_obj_free_fields(smap->map.record, SDATA(selem)->data);
232243 __bpf_selem_free(selem, reuse_now);
233244 return;
234245 }
235246
236− if (!reuse_now) {
237− call_rcu_tasks_trace(&selem->rcu, bpf_selem_free_trace_rcu);
238− } else {
247+ if (reuse_now) {
248+ /* reuse_now == true only happens when the storage owner
249+ * (e.g. task_struct) is being destructed or the map itself
250+ * is being destructed (ie map_free). In both cases,
251+ * no bpf prog can have a hold on the selem. It is
252+ * safe to unpin the uptrs and free the selem now.
253+ */
254+ bpf_obj_free_fields(smap->map.record, SDATA(selem)->data);
239255 /* Instead of using the vanilla call_rcu(),
240256 * bpf_mem_cache_free will be able to reuse selem
241257 * immediately.
@@ -243,6 +259,26 @@ void bpf_selem_free(struct bpf_local_storage_elem *selem, · 1 unmodified line
243259 migrate_disable();
244260 bpf_mem_cache_free(&smap->selem_ma, selem);
245261 migrate_enable();
262+ return;
263+ }
264+
265+ call_rcu_tasks_trace(&selem->rcu, bpf_selem_free_trace_rcu);
266+}
267+
268+static void bpf_selem_free_list(struct hlist_head *list, bool reuse_now)
269+{
270+ struct bpf_local_storage_elem *selem;
271+ struct bpf_local_storage_map *smap;
272+ struct hlist_node *n;
273+
274+ /* The "_safe" iteration is needed.
275+ * The loop is not removing the selem from the list
276+ * but bpf_selem_free will use the selem->rcu_head
277+ * which is union-ized with the selem->free_node.
278+ */
279+ hlist_for_each_entry_safe(selem, n, list, free_node) {
280+ smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held());
281+ bpf_selem_free(selem, smap, reuse_now);
246282 }
247283 }
248284
@@ -252,7 +288,7 @@ void bpf_selem_free(struct bpf_local_storage_elem *selem, · 3 unmodified lines
252288 */
253289 static bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage,
254290 struct bpf_local_storage_elem *selem,
255− bool uncharge_mem, bool reuse_now)
291+ bool uncharge_mem, struct hlist_head *free_selem_list)
256292 {
257293 struct bpf_local_storage_map *smap;
258294 bool free_local_storage;
@@ -296,7 +332,7 @@ static bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_stor · 37 unmodified lines
296332 SDATA(selem))
297333 RCU_INIT_POINTER(local_storage->cache[smap->cache_idx], NULL);
298334
299− bpf_selem_free(selem, smap, reuse_now);
335+ hlist_add_head(&selem->free_node, free_selem_list);
300336
301337 if (rcu_access_pointer(local_storage->smap) == smap)
302338 RCU_INIT_POINTER(local_storage->smap, NULL);
@@ -345,6 +381,7 @@ static void bpf_selem_unlink_storage(struct bpf_local_storage_elem *selem, · 42 unmodified lines
345381 struct bpf_local_storage_map *storage_smap;
346382 struct bpf_local_storage *local_storage;
347383 bool bpf_ma, free_local_storage = false;
384+ HLIST_HEAD(selem_free_list);
348385 unsigned long flags;
349386
350387 if (unlikely(!selem_linked_to_storage_lockless(selem)))
@@ -360,9 +397,11 @@ static void bpf_selem_unlink_storage(struct bpf_local_storage_elem *selem, · 9 unmodified lines
360397 raw_spin_lock_irqsave(&local_storage->lock, flags);
361398 if (likely(selem_linked_to_storage(selem)))
362399 free_local_storage = bpf_selem_unlink_storage_nolock(
363− local_storage, selem, true, reuse_now);
400+ local_storage, selem, true, &selem_free_list);
364401 raw_spin_unlock_irqrestore(&local_storage->lock, flags);
365402
403+ bpf_selem_free_list(&selem_free_list, reuse_now);
404+
366405 if (free_local_storage)
367406 bpf_local_storage_free(local_storage, storage_smap, bpf_ma, reuse_now);
368407 }
@@ -524,11 +563,12 @@ uncharge: · 155 unmodified lines
524563 */
525564 struct bpf_local_storage_data *
526565 bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
527− void *value, u64 map_flags, gfp_t gfp_flags)
566+ void *value, u64 map_flags, bool swap_uptrs, gfp_t gfp_flags)
528567 {
529568 struct bpf_local_storage_data *old_sdata = NULL;
530569 struct bpf_local_storage_elem *alloc_selem, *selem = NULL;
531570 struct bpf_local_storage *local_storage;
571+ HLIST_HEAD(old_selem_free_list);
532572 unsigned long flags;
533573 int err;
534574
@@ -550,7 +590,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, · 15 unmodified lines
550590 if (err)
551591 return ERR_PTR(err);
552592
553− selem = bpf_selem_alloc(smap, owner, value, true, gfp_flags);
593+ selem = bpf_selem_alloc(smap, owner, value, true, swap_uptrs, gfp_flags);
554594 if (!selem)
555595 return ERR_PTR(-ENOMEM);
556596
@@ -584,7 +624,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, · 27 unmodified lines
584624 /* A lookup has just been done before and concluded a new selem is
585625 * needed. The chance of an unnecessary alloc is unlikely.
586626 */
587− alloc_selem = selem = bpf_selem_alloc(smap, owner, value, true, gfp_flags);
627+ alloc_selem = selem = bpf_selem_alloc(smap, owner, value, true, swap_uptrs, gfp_flags);
588628 if (!alloc_selem)
589629 return ERR_PTR(-ENOMEM);
590630
@@ -624,11 +664,12 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, · 33 unmodified lines
624664 if (old_sdata) {
625665 bpf_selem_unlink_map(SELEM(old_sdata));
626666 bpf_selem_unlink_storage_nolock(local_storage, SELEM(old_sdata),
627− true, false);
667+ true, &old_selem_free_list);
628668 }
629669
630670 unlock:
631671 raw_spin_unlock_irqrestore(&local_storage->lock, flags);
672+ bpf_selem_free_list(&old_selem_free_list, false);
632673 if (alloc_selem) {
633674 mem_uncharge(smap, owner, smap->elem_size);
634675 bpf_selem_free(alloc_selem, smap, true);
@@ -706,6 +747,7 @@ void bpf_local_storage_destroy(struct bpf_local_storage *local_storage) · 71 unmodified lines
706747 struct bpf_local_storage_map *storage_smap;
707748 struct bpf_local_storage_elem *selem;
708749 bool bpf_ma, free_storage = false;
750+ HLIST_HEAD(free_selem_list);
709751 struct hlist_node *n;
710752 unsigned long flags;
711753
@@ -734,10 +776,12 @@ void bpf_local_storage_destroy(struct bpf_local_storage *local_storage) · 22 unmodified lines
734776 * of the loop will set the free_cgroup_storage to true.
735777 */
736778 free_storage = bpf_selem_unlink_storage_nolock(
737− local_storage, selem, true, true);
779+ local_storage, selem, true, &free_selem_list);
738780 }
739781 raw_spin_unlock_irqrestore(&local_storage->lock, flags);
740782
783+ bpf_selem_free_list(&free_selem_list, true);
784+
741785 if (free_storage)
742786 bpf_local_storage_free(local_storage, storage_smap, bpf_ma, true);
743787 }
@@ -883,6 +927,9 @@ void bpf_local_storage_map_free(struct bpf_map *map, · 139 unmodified lines
883927 synchronize_rcu();
884928
885929 if (smap->bpf_ma) {
930+ rcu_barrier_tasks_trace();
931+ if (!rcu_trace_implies_rcu_gp())
932+ rcu_barrier();
886933 bpf_mem_alloc_destroy(&smap->selem_ma);
887934 bpf_mem_alloc_destroy(&smap->storage_ma);
888935 }
kernel/bpf/bpf_struct_ops.c+103−12
@@ -23,7 +23,6 @@ struct bpf_struct_ops_value { · 22 unmodified lines
2323
2424 struct bpf_struct_ops_map {
2525 struct bpf_map map;
26− struct rcu_head rcu;
2726 const struct bpf_struct_ops_desc *st_ops_desc;
2827 /* protect map_update */
2928 struct mutex lock;
@@ -32,7 +31,9 @@ struct bpf_struct_ops_map { · 2 unmodified lines
3231 * (in kvalue.data).
3332 */
3433 struct bpf_link **links;
35− u32 links_cnt;
34+ /* ksyms for bpf trampolines */
35+ struct bpf_ksym **ksyms;
36+ u32 funcs_cnt;
3637 u32 image_pages_cnt;
3738 /* image_pages is an array of pages that has all the trampolines
3839 * that stores the func args before calling the bpf_prog.
@@ -481,11 +482,11 @@ static void bpf_struct_ops_map_put_progs(struct bpf_struct_ops_map *st_map) · 442 unmodified lines
481482 {
482483 u32 i;
483484
484− for (i = 0; i < st_map->links_cnt; i++) {
485− if (st_map->links[i]) {
486− bpf_link_put(st_map->links[i]);
487− st_map->links[i] = NULL;
488− }
485+ for (i = 0; i < st_map->funcs_cnt; i++) {
486+ if (!st_map->links[i])
487+ break;
488+ bpf_link_put(st_map->links[i]);
489+ st_map->links[i] = NULL;
489490 }
490491 }
491492
@@ -586,6 +587,49 @@ int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks, · 94 unmodified lines
586587 return 0;
587588 }
588589
590+static void bpf_struct_ops_ksym_init(const char *tname, const char *mname,
591+ void *image, unsigned int size,
592+ struct bpf_ksym *ksym)
593+{
594+ snprintf(ksym->name, KSYM_NAME_LEN, "bpf__%s_%s", tname, mname);
595+ INIT_LIST_HEAD_RCU(&ksym->lnode);
596+ bpf_image_ksym_init(image, size, ksym);
597+}
598+
599+static void bpf_struct_ops_map_add_ksyms(struct bpf_struct_ops_map *st_map)
600+{
601+ u32 i;
602+
603+ for (i = 0; i < st_map->funcs_cnt; i++) {
604+ if (!st_map->ksyms[i])
605+ break;
606+ bpf_image_ksym_add(st_map->ksyms[i]);
607+ }
608+}
609+
610+static void bpf_struct_ops_map_del_ksyms(struct bpf_struct_ops_map *st_map)
611+{
612+ u32 i;
613+
614+ for (i = 0; i < st_map->funcs_cnt; i++) {
615+ if (!st_map->ksyms[i])
616+ break;
617+ bpf_image_ksym_del(st_map->ksyms[i]);
618+ }
619+}
620+
621+static void bpf_struct_ops_map_free_ksyms(struct bpf_struct_ops_map *st_map)
622+{
623+ u32 i;
624+
625+ for (i = 0; i < st_map->funcs_cnt; i++) {
626+ if (!st_map->ksyms[i])
627+ break;
628+ kfree(st_map->ksyms[i]);
629+ st_map->ksyms[i] = NULL;
630+ }
631+}
632+
589633 static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
590634 void *value, u64 flags)
591635 {
@@ -601,6 +645,9 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, · 9 unmodified lines
601645 int prog_fd, err;
602646 u32 i, trampoline_start, image_off = 0;
603647 void *cur_image = NULL, *image = NULL;
648+ struct bpf_link **plink;
649+ struct bpf_ksym **pksym;
650+ const char *tname, *mname;
604651
605652 if (flags)
606653 return -EINVAL;
@@ -639,14 +686,19 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, · 32 unmodified lines
639686 udata = &uvalue->data;
640687 kdata = &kvalue->data;
641688
689+ plink = st_map->links;
690+ pksym = st_map->ksyms;
691+ tname = btf_name_by_offset(st_map->btf, t->name_off);
642692 module_type = btf_type_by_id(btf_vmlinux, st_ops_ids[IDX_MODULE_ID]);
643693 for_each_member(i, t, member) {
644694 const struct btf_type *mtype, *ptype;
645695 struct bpf_prog *prog;
646696 struct bpf_tramp_link *link;
697+ struct bpf_ksym *ksym;
647698 u32 moff;
648699
649700 moff = __btf_member_bit_offset(t, member) / 8;
701+ mname = btf_name_by_offset(st_map->btf, member->name_off);
650702 ptype = btf_type_resolve_ptr(st_map->btf, member->type, NULL);
651703 if (ptype == module_type) {
652704 if (*(void **)(udata + moff))
@@ -714,7 +766,14 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, · 61 unmodified lines
714766 }
715767 bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS,
716768 &bpf_struct_ops_link_lops, prog);
717− st_map->links[i] = &link->link;
769+ *plink++ = &link->link;
770+
771+ ksym = kzalloc(sizeof(*ksym), GFP_USER);
772+ if (!ksym) {
773+ err = -ENOMEM;
774+ goto reset_unlock;
775+ }
776+ *pksym++ = ksym;
718777
719778 trampoline_start = image_off;
720779 err = bpf_struct_ops_prepare_trampoline(tlinks, link,
@@ -735,6 +794,12 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, · 14 unmodified lines
735794
736795 /* put prog_id to udata */
737796 *(unsigned long *)(udata + moff) = prog->aux->id;
797+
798+ /* init ksym for this trampoline */
799+ bpf_struct_ops_ksym_init(tname, mname,
800+ image + trampoline_start,
801+ image_off - trampoline_start,
802+ ksym);
738803 }
739804
740805 if (st_ops->validate) {
@@ -783,6 +848,7 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, · 42 unmodified lines
783848 */
784849
785850 reset_unlock:
851+ bpf_struct_ops_map_free_ksyms(st_map);
786852 bpf_struct_ops_map_free_image(st_map);
787853 bpf_struct_ops_map_put_progs(st_map);
788854 memset(uvalue, 0, map->value_size);
@@ -790,6 +856,8 @@ reset_unlock: · 1 unmodified line
790856 unlock:
791857 kfree(tlinks);
792858 mutex_unlock(&st_map->lock);
859+ if (!err)
860+ bpf_struct_ops_map_add_ksyms(st_map);
793861 return err;
794862 }
795863
@@ -849,7 +917,10 @@ static void __bpf_struct_ops_map_free(struct bpf_map *map) · 53 unmodified lines
849917
850918 if (st_map->links)
851919 bpf_struct_ops_map_put_progs(st_map);
920+ if (st_map->ksyms)
921+ bpf_struct_ops_map_free_ksyms(st_map);
852922 bpf_map_area_free(st_map->links);
923+ bpf_map_area_free(st_map->ksyms);
853924 bpf_struct_ops_map_free_image(st_map);
854925 bpf_map_area_free(st_map->uvalue);
855926 bpf_map_area_free(st_map);
@@ -866,6 +937,8 @@ static void bpf_struct_ops_map_free(struct bpf_map *map) · 10 unmodified lines
866937 if (btf_is_module(st_map->btf))
867938 module_put(st_map->st_ops_desc->st_ops->owner);
868939
940+ bpf_struct_ops_map_del_ksyms(st_map);
941+
869942 /* The struct_ops's function may switch to another struct_ops.
870943 *
871944 * For example, bpf_tcp_cc_x->init() may switch to
@@ -895,6 +968,19 @@ static int bpf_struct_ops_map_alloc_check(union bpf_attr *attr) · 23 unmodified lines
895968 return 0;
896969 }
897970
971+static u32 count_func_ptrs(const struct btf *btf, const struct btf_type *t)
972+{
973+ int i;
974+ u32 count;
975+ const struct btf_member *member;
976+
977+ count = 0;
978+ for_each_member(i, t, member)
979+ if (btf_type_resolve_func_ptr(btf, member->type, NULL))
980+ count++;
981+ return count;
982+}
983+
898984 static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr)
899985 {
900986 const struct bpf_struct_ops_desc *st_ops_desc;
@@ -961,11 +1047,15 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) · 60 unmodified lines
9611047 map = &st_map->map;
9621048
9631049 st_map->uvalue = bpf_map_area_alloc(vt->size, NUMA_NO_NODE);
964− st_map->links_cnt = btf_type_vlen(t);
1050+ st_map->funcs_cnt = count_func_ptrs(btf, t);
9651051 st_map->links =
966− bpf_map_area_alloc(st_map->links_cnt * sizeof(struct bpf_links *),
1052+ bpf_map_area_alloc(st_map->funcs_cnt * sizeof(struct bpf_link *),
1053+ NUMA_NO_NODE);
1054+
1055+ st_map->ksyms =
1056+ bpf_map_area_alloc(st_map->funcs_cnt * sizeof(struct bpf_ksym *),
9671057 NUMA_NO_NODE);
968− if (!st_map->uvalue || !st_map->links) {
1058+ if (!st_map->uvalue || !st_map->links || !st_map->ksyms) {
9691059 ret = -ENOMEM;
9701060 goto errout_free;
9711061 }
@@ -994,7 +1084,8 @@ static u64 bpf_struct_ops_map_mem_usage(const struct bpf_map *map) · 22 unmodified lines
9941084 usage = sizeof(*st_map) +
9951085 vt->size - sizeof(struct bpf_struct_ops_value);
9961086 usage += vt->size;
997− usage += btf_type_vlen(vt) * sizeof(struct bpf_links *);
1087+ usage += st_map->funcs_cnt * sizeof(struct bpf_link *);
1088+ usage += st_map->funcs_cnt * sizeof(struct bpf_ksym *);
9981089 usage += PAGE_SIZE;
9991090 return usage;
10001091 }
kernel/bpf/bpf_task_storage.c+5−2
@@ -128,6 +128,9 @@ static long bpf_pid_task_storage_update_elem(struct bpf_map *map, void *key, · 127 unmodified lines
128128 struct pid *pid;
129129 int fd, err;
130130
131+ if ((map_flags & BPF_F_LOCK) && btf_record_has_field(map->record, BPF_UPTR))
132+ return -EOPNOTSUPP;
133+
131134 fd = *(int *)key;
132135 pid = pidfd_get_pid(fd, &f_flags);
133136 if (IS_ERR(pid))
@@ -146,7 +149,7 @@ static long bpf_pid_task_storage_update_elem(struct bpf_map *map, void *key, · 12 unmodified lines
146149 bpf_task_storage_lock();
147150 sdata = bpf_local_storage_update(
148151 task, (struct bpf_local_storage_map *)map, value, map_flags,
149− GFP_ATOMIC);
152+ true, GFP_ATOMIC);
150153 bpf_task_storage_unlock();
151154
152155 err = PTR_ERR_OR_ZERO(sdata);
@@ -218,7 +221,7 @@ static void *__bpf_task_storage_get(struct bpf_map *map, · 65 unmodified lines
218221 (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) && nobusy) {
219222 sdata = bpf_local_storage_update(
220223 task, (struct bpf_local_storage_map *)map, value,
221− BPF_NOEXIST, gfp_flags);
224+ BPF_NOEXIST, false, gfp_flags);
222225 return IS_ERR(sdata) ? NULL : sdata->data;
223226 }
224227
kernel/bpf/btf.c+42−15
@@ -2808,7 +2808,7 @@ static void btf_ref_type_log(struct btf_verifier_env *env, · 2807 unmodified lines
28082808 btf_verifier_log(env, "type_id=%u", t->type);
28092809 }
28102810
2811−static struct btf_kind_operations modifier_ops = {
2811+static const struct btf_kind_operations modifier_ops = {
28122812 .check_meta = btf_ref_type_check_meta,
28132813 .resolve = btf_modifier_resolve,
28142814 .check_member = btf_modifier_check_member,
@@ -2817,7 +2817,7 @@ static struct btf_kind_operations modifier_ops = { · 2 unmodified lines
28172817 .show = btf_modifier_show,
28182818 };
28192819
2820−static struct btf_kind_operations ptr_ops = {
2820+static const struct btf_kind_operations ptr_ops = {
28212821 .check_meta = btf_ref_type_check_meta,
28222822 .resolve = btf_ptr_resolve,
28232823 .check_member = btf_ptr_check_member,
@@ -2858,7 +2858,7 @@ static void btf_fwd_type_log(struct btf_verifier_env *env, · 34 unmodified lines
28582858 btf_verifier_log(env, "%s", btf_type_kflag(t) ? "union" : "struct");
28592859 }
28602860
2861−static struct btf_kind_operations fwd_ops = {
2861+static const struct btf_kind_operations fwd_ops = {
28622862 .check_meta = btf_fwd_check_meta,
28632863 .resolve = btf_df_resolve,
28642864 .check_member = btf_df_check_member,
@@ -3109,7 +3109,7 @@ static void btf_array_show(const struct btf *btf, const struct btf_type *t, · 244 unmodified lines
31093109 __btf_array_show(btf, t, type_id, data, bits_offset, show);
31103110 }
31113111
3112−static struct btf_kind_operations array_ops = {
3112+static const struct btf_kind_operations array_ops = {
31133113 .check_meta = btf_array_check_meta,
31143114 .resolve = btf_array_resolve,
31153115 .check_member = btf_array_check_member,
@@ -3334,7 +3334,7 @@ static int btf_find_struct(const struct btf *btf, const struct btf_type *t, · 218 unmodified lines
33343334 }
33353335
33363336 static int btf_find_kptr(const struct btf *btf, const struct btf_type *t,
3337− u32 off, int sz, struct btf_field_info *info)
3337+ u32 off, int sz, struct btf_field_info *info, u32 field_mask)
33383338 {
33393339 enum btf_field_type type;
33403340 u32 res_id;
@@ -3358,9 +3358,14 @@ static int btf_find_kptr(const struct btf *btf, const struct btf_type *t, · 17 unmodified lines
33583358 type = BPF_KPTR_REF;
33593359 else if (!strcmp("percpu_kptr", __btf_name_by_offset(btf, t->name_off)))
33603360 type = BPF_KPTR_PERCPU;
3361+ else if (!strcmp("uptr", __btf_name_by_offset(btf, t->name_off)))
3362+ type = BPF_UPTR;
33613363 else
33623364 return -EINVAL;
33633365
3366+ if (!(type & field_mask))
3367+ return BTF_FIELD_IGNORE;
3368+
33643369 /* Get the base type */
33653370 t = btf_type_skip_modifiers(btf, t->type, &res_id);
33663371 /* Only pointer to struct is allowed */
@@ -3502,7 +3507,7 @@ static int btf_get_field_type(const struct btf *btf, const struct btf_type *var_ · 135 unmodified lines
35023507 field_mask_test_name(BPF_REFCOUNT, "bpf_refcount");
35033508
35043509 /* Only return BPF_KPTR when all other types with matchable names fail */
3505− if (field_mask & BPF_KPTR && !__btf_type_is_struct(var_type)) {
3510+ if (field_mask & (BPF_KPTR | BPF_UPTR) && !__btf_type_is_struct(var_type)) {
35063511 type = BPF_KPTR_REF;
35073512 goto end;
35083513 }
@@ -3535,6 +3540,7 @@ static int btf_repeat_fields(struct btf_field_info *info, int info_cnt, · 26 unmodified lines
35353540 case BPF_KPTR_UNREF:
35363541 case BPF_KPTR_REF:
35373542 case BPF_KPTR_PERCPU:
3543+ case BPF_UPTR:
35383544 case BPF_LIST_HEAD:
35393545 case BPF_RB_ROOT:
35403546 break;
@@ -3667,8 +3673,9 @@ static int btf_find_field_one(const struct btf *btf, · 126 unmodified lines
36673673 case BPF_KPTR_UNREF:
36683674 case BPF_KPTR_REF:
36693675 case BPF_KPTR_PERCPU:
3676+ case BPF_UPTR:
36703677 ret = btf_find_kptr(btf, var_type, off, sz,
3671− info_cnt ? &info[0] : &tmp);
3678+ info_cnt ? &info[0] : &tmp, field_mask);
36723679 if (ret < 0)
36733680 return ret;
36743681 break;
@@ -3991,6 +3998,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type · 316 unmodified lines
39913998 case BPF_KPTR_UNREF:
39923999 case BPF_KPTR_REF:
39934000 case BPF_KPTR_PERCPU:
4001+ case BPF_UPTR:
39944002 ret = btf_parse_kptr(btf, &rec->fields[i], &info_arr[i]);
39954003 if (ret < 0)
39964004 goto end;
@@ -4050,12 +4058,28 @@ int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec) · 53 unmodified lines
40504058 * Hence we only need to ensure that bpf_{list_head,rb_root} ownership
40514059 * does not form cycles.
40524060 */
4053− if (IS_ERR_OR_NULL(rec) || !(rec->field_mask & BPF_GRAPH_ROOT))
4061+ if (IS_ERR_OR_NULL(rec) || !(rec->field_mask & (BPF_GRAPH_ROOT | BPF_UPTR)))
40544062 return 0;
40554063 for (i = 0; i < rec->cnt; i++) {
40564064 struct btf_struct_meta *meta;
4065+ const struct btf_type *t;
40574066 u32 btf_id;
40584067
4068+ if (rec->fields[i].type == BPF_UPTR) {
4069+ /* The uptr only supports pinning one page and cannot
4070+ * point to a kernel struct
4071+ */
4072+ if (btf_is_kernel(rec->fields[i].kptr.btf))
4073+ return -EINVAL;
4074+ t = btf_type_by_id(rec->fields[i].kptr.btf,
4075+ rec->fields[i].kptr.btf_id);
4076+ if (!t->size)
4077+ return -EINVAL;
4078+ if (t->size > PAGE_SIZE)
4079+ return -E2BIG;
4080+ continue;
4081+ }
4082+
40594083 if (!(rec->fields[i].type & BPF_GRAPH_ROOT))
40604084 continue;
40614085 btf_id = rec->fields[i].graph_root.value_btf_id;
@@ -4191,7 +4215,7 @@ static void btf_struct_show(const struct btf *btf, const struct btf_type *t, · 129 unmodified lines
41914215 __btf_struct_show(btf, t, type_id, data, bits_offset, show);
41924216 }
41934217
4194−static struct btf_kind_operations struct_ops = {
4218+static const struct btf_kind_operations struct_ops = {
41954219 .check_meta = btf_struct_check_meta,
41964220 .resolve = btf_struct_resolve,
41974221 .check_member = btf_struct_check_member,
@@ -4359,7 +4383,7 @@ static void btf_enum_show(const struct btf *btf, const struct btf_type *t, · 161 unmodified lines
43594383 btf_show_end_type(show);
43604384 }
43614385
4362−static struct btf_kind_operations enum_ops = {
4386+static const struct btf_kind_operations enum_ops = {
43634387 .check_meta = btf_enum_check_meta,
43644388 .resolve = btf_df_resolve,
43654389 .check_member = btf_enum_check_member,
@@ -4462,7 +4486,7 @@ static void btf_enum64_show(const struct btf *btf, const struct btf_type *t, · 96 unmodified lines
44624486 btf_show_end_type(show);
44634487 }
44644488
4465−static struct btf_kind_operations enum64_ops = {
4489+static const struct btf_kind_operations enum64_ops = {
44664490 .check_meta = btf_enum64_check_meta,
44674491 .resolve = btf_df_resolve,
44684492 .check_member = btf_enum_check_member,
@@ -4540,7 +4564,7 @@ done: · 71 unmodified lines
45404564 btf_verifier_log(env, ")");
45414565 }
45424566
4543−static struct btf_kind_operations func_proto_ops = {
4567+static const struct btf_kind_operations func_proto_ops = {
45444568 .check_meta = btf_func_proto_check_meta,
45454569 .resolve = btf_df_resolve,
45464570 /*
@@ -4598,7 +4622,7 @@ static int btf_func_resolve(struct btf_verifier_env *env, · 51 unmodified lines
45984622 return 0;
45994623 }
46004624
4601−static struct btf_kind_operations func_ops = {
4625+static const struct btf_kind_operations func_ops = {
46024626 .check_meta = btf_func_check_meta,
46034627 .resolve = btf_func_resolve,
46044628 .check_member = btf_df_check_member,
@@ -5566,7 +5590,7 @@ btf_parse_struct_metas(struct bpf_verifier_log *log, struct btf *btf) · 961 unmodified lines
55665590 goto free_aof;
55675591 }
55685592
5569− ret = btf_find_kptr(btf, t, 0, 0, &tmp);
5593+ ret = btf_find_kptr(btf, t, 0, 0, &tmp, BPF_KPTR);
55705594 if (ret != BTF_FIELD_FOUND)
55715595 continue;
55725596
@@ -6564,7 +6588,10 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, · 991 unmodified lines
65646588 if (prog_args_trusted(prog))
65656589 info->reg_type |= PTR_TRUSTED;
65666590
6567− if (btf_param_match_suffix(btf, &args[arg], "__nullable"))
6591+ /* Raw tracepoint arguments always get marked as maybe NULL */
6592+ if (bpf_prog_is_raw_tp(prog))
6593+ info->reg_type |= PTR_MAYBE_NULL;
6594+ else if (btf_param_match_suffix(btf, &args[arg], "__nullable"))
65686595 info->reg_type |= PTR_MAYBE_NULL;
65696596
65706597 if (tgt_prog) {
kernel/bpf/core.c+6−0
@@ -131,6 +131,7 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag · 130 unmodified lines
131131 INIT_LIST_HEAD_RCU(&fp->aux->ksym_prefix.lnode);
132132 #endif
133133 mutex_init(&fp->aux->used_maps_mutex);
134+ mutex_init(&fp->aux->ext_mutex);
134135 mutex_init(&fp->aux->dst_mutex);
135136
136137 return fp;
@@ -3044,6 +3045,11 @@ bool __weak bpf_jit_supports_exceptions(void) · 2907 unmodified lines
30443045 return false;
30453046 }
30463047
3048+bool __weak bpf_jit_supports_private_stack(void)
3049+{
3050+ return false;
3051+}
3052+
30473053 void __weak arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie)
30483054 {
30493055 }
kernel/bpf/dispatcher.c+2−1
@@ -154,7 +154,8 @@ void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, · 153 unmodified lines
154154 d->image = NULL;
155155 goto out;
156156 }
157− bpf_image_ksym_add(d->image, PAGE_SIZE, &d->ksym);
157+ bpf_image_ksym_init(d->image, PAGE_SIZE, &d->ksym);
158+ bpf_image_ksym_add(&d->ksym);
158159 }
159160
160161 prev_num_progs = d->num_progs;
kernel/bpf/hashtab.c+39−17
@@ -896,9 +896,12 @@ find_first_elem: · 895 unmodified lines
896896 static void htab_elem_free(struct bpf_htab *htab, struct htab_elem *l)
897897 {
898898 check_and_free_fields(htab, l);
899+
900+ migrate_disable();
899901 if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH)
900902 bpf_mem_cache_free(&htab->pcpu_ma, l->ptr_to_pptr);
901903 bpf_mem_cache_free(&htab->ma, l);
904+ migrate_enable();
902905 }
903906
904907 static void htab_put_fd_value(struct bpf_htab *htab, struct htab_elem *l)
@@ -948,7 +951,7 @@ static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l) · 43 unmodified lines
948951 if (htab_is_prealloc(htab)) {
949952 bpf_map_dec_elem_count(&htab->map);
950953 check_and_free_fields(htab, l);
951− __pcpu_freelist_push(&htab->freelist, &l->fnode);
954+ pcpu_freelist_push(&htab->freelist, &l->fnode);
952955 } else {
953956 dec_elem_count(htab);
954957 htab_elem_free(htab, l);
@@ -1018,7 +1021,6 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, · 63 unmodified lines
10181021 */
10191022 pl_new = this_cpu_ptr(htab->extra_elems);
10201023 l_new = *pl_new;
1021− htab_put_fd_value(htab, old_elem);
10221024 *pl_new = old_elem;
10231025 } else {
10241026 struct pcpu_freelist_node *l;
@@ -1105,6 +1107,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value, · 80 unmodified lines
11051107 struct htab_elem *l_new = NULL, *l_old;
11061108 struct hlist_nulls_head *head;
11071109 unsigned long flags;
1110+ void *old_map_ptr;
11081111 struct bucket *b;
11091112 u32 key_size, hash;
11101113 int ret;
@@ -1183,12 +1186,27 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value, · 72 unmodified lines
11831186 hlist_nulls_add_head_rcu(&l_new->hash_node, head);
11841187 if (l_old) {
11851188 hlist_nulls_del_rcu(&l_old->hash_node);
1189+
1190+ /* l_old has already been stashed in htab->extra_elems, free
1191+ * its special fields before it is available for reuse. Also
1192+ * save the old map pointer in htab of maps before unlock
1193+ * and release it after unlock.
1194+ */
1195+ old_map_ptr = NULL;
1196+ if (htab_is_prealloc(htab)) {
1197+ if (map->ops->map_fd_put_ptr)
1198+ old_map_ptr = fd_htab_map_get_ptr(map, l_old);
1199+ check_and_free_fields(htab, l_old);
1200+ }
1201+ }
1202+ htab_unlock_bucket(htab, b, hash, flags);
1203+ if (l_old) {
1204+ if (old_map_ptr)
1205+ map->ops->map_fd_put_ptr(map, old_map_ptr, true);
11861206 if (!htab_is_prealloc(htab))
11871207 free_htab_elem(htab, l_old);
1188− else
1189− check_and_free_fields(htab, l_old);
11901208 }
1191− ret = 0;
1209+ return 0;
11921210 err:
11931211 htab_unlock_bucket(htab, b, hash, flags);
11941212 return ret;
@@ -1432,15 +1450,15 @@ static long htab_map_delete_elem(struct bpf_map *map, void *key) · 237 unmodified lines
14321450 return ret;
14331451
14341452 l = lookup_elem_raw(head, hash, key, key_size);
1435−
1436− if (l) {
1453+ if (l)
14371454 hlist_nulls_del_rcu(&l->hash_node);
1438− free_htab_elem(htab, l);
1439− } else {
1455+ else
14401456 ret = -ENOENT;
1441− }
14421457
14431458 htab_unlock_bucket(htab, b, hash, flags);
1459+
1460+ if (l)
1461+ free_htab_elem(htab, l);
14441462 return ret;
14451463 }
14461464
@@ -1853,13 +1871,14 @@ again_nocopy: · 406 unmodified lines
18531871 * may cause deadlock. See comments in function
18541872 * prealloc_lru_pop(). Let us do bpf_lru_push_free()
18551873 * after releasing the bucket lock.
1874+ *
1875+ * For htab of maps, htab_put_fd_value() in
1876+ * free_htab_elem() may acquire a spinlock with bucket
1877+ * lock being held and it violates the lock rule, so
1878+ * invoke free_htab_elem() after unlock as well.
18561879 */
1857− if (is_lru_map) {
1858− l->batch_flink = node_to_free;
1859− node_to_free = l;
1860− } else {
1861− free_htab_elem(htab, l);
1862− }
1880+ l->batch_flink = node_to_free;
1881+ node_to_free = l;
18631882 }
18641883 dst_key += key_size;
18651884 dst_val += value_size;
@@ -1871,7 +1890,10 @@ again_nocopy: · 5 unmodified lines
18711890 while (node_to_free) {
18721891 l = node_to_free;
18731892 node_to_free = node_to_free->batch_flink;
1874− htab_lru_push_free(htab, l);
1893+ if (is_lru_map)
1894+ htab_lru_push_free(htab, l);
1895+ else
1896+ free_htab_elem(htab, l);
18751897 }
18761898
18771899 next_batch:
kernel/bpf/helpers.c+27−2
@@ -2521,6 +2521,25 @@ __bpf_kfunc struct task_struct *bpf_task_from_pid(s32 pid) · 2520 unmodified lines
25212521 return p;
25222522 }
25232523
2524+/**
2525+ * bpf_task_from_vpid - Find a struct task_struct from its vpid by looking it up
2526+ * in the pid namespace of the current task. If a task is returned, it must
2527+ * either be stored in a map, or released with bpf_task_release().
2528+ * @vpid: The vpid of the task being looked up.
2529+ */
2530+__bpf_kfunc struct task_struct *bpf_task_from_vpid(s32 vpid)
2531+{
2532+ struct task_struct *p;
2533+
2534+ rcu_read_lock();
2535+ p = find_task_by_vpid(vpid);
2536+ if (p)
2537+ p = bpf_task_acquire(p);
2538+ rcu_read_unlock();
2539+
2540+ return p;
2541+}
2542+
25242543 /**
25252544 * bpf_dynptr_slice() - Obtain a read-only pointer to the dynptr data.
25262545 * @p: The dynptr whose data slice to retrieve
@@ -3068,7 +3087,9 @@ BTF_ID_FLAGS(func, bpf_task_under_cgroup, KF_RCU) · 541 unmodified lines
30683087 BTF_ID_FLAGS(func, bpf_task_get_cgroup1, KF_ACQUIRE | KF_RCU | KF_RET_NULL)
30693088 #endif
30703089 BTF_ID_FLAGS(func, bpf_task_from_pid, KF_ACQUIRE | KF_RET_NULL)
3090+BTF_ID_FLAGS(func, bpf_task_from_vpid, KF_ACQUIRE | KF_RET_NULL)
30713091 BTF_ID_FLAGS(func, bpf_throw)
3092+BTF_ID_FLAGS(func, bpf_send_signal_task, KF_TRUSTED_ARGS)
30723093 BTF_KFUNCS_END(generic_btf_ids)
30733094
30743095 static const struct btf_kfunc_id_set generic_kfunc_set = {
@@ -3086,8 +3107,8 @@ BTF_ID(func, bpf_cgroup_release_dtor) · 11 unmodified lines
30863107 #endif
30873108
30883109 BTF_KFUNCS_START(common_btf_ids)
3089−BTF_ID_FLAGS(func, bpf_cast_to_kern_ctx)
3090−BTF_ID_FLAGS(func, bpf_rdonly_cast)
3110+BTF_ID_FLAGS(func, bpf_cast_to_kern_ctx, KF_FASTCALL)
3111+BTF_ID_FLAGS(func, bpf_rdonly_cast, KF_FASTCALL)
30913112 BTF_ID_FLAGS(func, bpf_rcu_read_lock)
30923113 BTF_ID_FLAGS(func, bpf_rcu_read_unlock)
30933114 BTF_ID_FLAGS(func, bpf_dynptr_slice, KF_RET_NULL)
@@ -3124,6 +3145,10 @@ BTF_ID_FLAGS(func, bpf_iter_bits_new, KF_ITER_NEW) · 30 unmodified lines
31243145 BTF_ID_FLAGS(func, bpf_iter_bits_next, KF_ITER_NEXT | KF_RET_NULL)
31253146 BTF_ID_FLAGS(func, bpf_iter_bits_destroy, KF_ITER_DESTROY)
31263147 BTF_ID_FLAGS(func, bpf_copy_from_user_str, KF_SLEEPABLE)
3148+BTF_ID_FLAGS(func, bpf_get_kmem_cache)
3149+BTF_ID_FLAGS(func, bpf_iter_kmem_cache_new, KF_ITER_NEW | KF_SLEEPABLE)
3150+BTF_ID_FLAGS(func, bpf_iter_kmem_cache_next, KF_ITER_NEXT | KF_RET_NULL | KF_SLEEPABLE)
3151+BTF_ID_FLAGS(func, bpf_iter_kmem_cache_destroy, KF_ITER_DESTROY | KF_SLEEPABLE)
31273152 BTF_KFUNCS_END(common_btf_ids)
31283153
31293154 static const struct btf_kfunc_id_set common_kfunc_set = {
kernel/bpf/kmem_cache_iter.c+238−0
@@ -0,0 +1,238 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+/* Copyright (c) 2024 Google */
3+#include <linux/bpf.h>
4+#include <linux/btf_ids.h>
5+#include <linux/slab.h>
6+#include <linux/kernel.h>
7+#include <linux/seq_file.h>
8+
9+#include "../../mm/slab.h" /* kmem_cache, slab_caches and slab_mutex */
10+
11+/* open-coded version */
12+struct bpf_iter_kmem_cache {
13+ __u64 __opaque[1];
14+} __attribute__((aligned(8)));
15+
16+struct bpf_iter_kmem_cache_kern {
17+ struct kmem_cache *pos;
18+} __attribute__((aligned(8)));
19+
20+#define KMEM_CACHE_POS_START ((void *)1L)
21+
22+__bpf_kfunc_start_defs();
23+
24+__bpf_kfunc int bpf_iter_kmem_cache_new(struct bpf_iter_kmem_cache *it)
25+{
26+ struct bpf_iter_kmem_cache_kern *kit = (void *)it;
27+
28+ BUILD_BUG_ON(sizeof(*kit) > sizeof(*it));
29+ BUILD_BUG_ON(__alignof__(*kit) != __alignof__(*it));
30+
31+ kit->pos = KMEM_CACHE_POS_START;
32+ return 0;
33+}
34+
35+__bpf_kfunc struct kmem_cache *bpf_iter_kmem_cache_next(struct bpf_iter_kmem_cache *it)
36+{
37+ struct bpf_iter_kmem_cache_kern *kit = (void *)it;
38+ struct kmem_cache *prev = kit->pos;
39+ struct kmem_cache *next;
40+ bool destroy = false;
41+
42+ if (!prev)
43+ return NULL;
44+
45+ mutex_lock(&slab_mutex);
46+
47+ if (list_empty(&slab_caches)) {
48+ mutex_unlock(&slab_mutex);
49+ return NULL;
50+ }
51+
52+ if (prev == KMEM_CACHE_POS_START)
53+ next = list_first_entry(&slab_caches, struct kmem_cache, list);
54+ else if (list_last_entry(&slab_caches, struct kmem_cache, list) == prev)
55+ next = NULL;
56+ else
57+ next = list_next_entry(prev, list);
58+
59+ /* boot_caches have negative refcount, don't touch them */
60+ if (next && next->refcount > 0)
61+ next->refcount++;
62+
63+ /* Skip kmem_cache_destroy() for active entries */
64+ if (prev && prev != KMEM_CACHE_POS_START) {
65+ if (prev->refcount > 1)
66+ prev->refcount--;
67+ else if (prev->refcount == 1)
68+ destroy = true;
69+ }
70+
71+ mutex_unlock(&slab_mutex);
72+
73+ if (destroy)
74+ kmem_cache_destroy(prev);
75+
76+ kit->pos = next;
77+ return next;
78+}
79+
80+__bpf_kfunc void bpf_iter_kmem_cache_destroy(struct bpf_iter_kmem_cache *it)
81+{
82+ struct bpf_iter_kmem_cache_kern *kit = (void *)it;
83+ struct kmem_cache *s = kit->pos;
84+ bool destroy = false;
85+
86+ if (s == NULL || s == KMEM_CACHE_POS_START)
87+ return;
88+
89+ mutex_lock(&slab_mutex);
90+
91+ /* Skip kmem_cache_destroy() for active entries */
92+ if (s->refcount > 1)
93+ s->refcount--;
94+ else if (s->refcount == 1)
95+ destroy = true;
96+
97+ mutex_unlock(&slab_mutex);
98+
99+ if (destroy)
100+ kmem_cache_destroy(s);
101+}
102+
103+__bpf_kfunc_end_defs();
104+
105+struct bpf_iter__kmem_cache {
106+ __bpf_md_ptr(struct bpf_iter_meta *, meta);
107+ __bpf_md_ptr(struct kmem_cache *, s);
108+};
109+
110+union kmem_cache_iter_priv {
111+ struct bpf_iter_kmem_cache it;
112+ struct bpf_iter_kmem_cache_kern kit;
113+};
114+
115+static void *kmem_cache_iter_seq_start(struct seq_file *seq, loff_t *pos)
116+{
117+ loff_t cnt = 0;
118+ bool found = false;
119+ struct kmem_cache *s;
120+ union kmem_cache_iter_priv *p = seq->private;
121+
122+ mutex_lock(&slab_mutex);
123+
124+ /* Find an entry at the given position in the slab_caches list instead
125+ * of keeping a reference (of the last visited entry, if any) out of
126+ * slab_mutex. It might miss something if one is deleted in the middle
127+ * while it releases the lock. But it should be rare and there's not
128+ * much we can do about it.
129+ */
130+ list_for_each_entry(s, &slab_caches, list) {
131+ if (cnt == *pos) {
132+ /* Make sure this entry remains in the list by getting
133+ * a new reference count. Note that boot_cache entries
134+ * have a negative refcount, so don't touch them.
135+ */
136+ if (s->refcount > 0)
137+ s->refcount++;
138+ found = true;
139+ break;
140+ }
141+ cnt++;
142+ }
143+ mutex_unlock(&slab_mutex);
144+
145+ if (!found)
146+ s = NULL;
147+
148+ p->kit.pos = s;
149+ return s;
150+}
151+
152+static void kmem_cache_iter_seq_stop(struct seq_file *seq, void *v)
153+{
154+ struct bpf_iter_meta meta;
155+ struct bpf_iter__kmem_cache ctx = {
156+ .meta = &meta,
157+ .s = v,
158+ };
159+ union kmem_cache_iter_priv *p = seq->private;
160+ struct bpf_prog *prog;
161+
162+ meta.seq = seq;
163+ prog = bpf_iter_get_info(&meta, true);
164+ if (prog && !ctx.s)
165+ bpf_iter_run_prog(prog, &ctx);
166+
167+ bpf_iter_kmem_cache_destroy(&p->it);
168+}
169+
170+static void *kmem_cache_iter_seq_next(struct seq_file *seq, void *v, loff_t *pos)
171+{
172+ union kmem_cache_iter_priv *p = seq->private;
173+
174+ ++*pos;
175+
176+ return bpf_iter_kmem_cache_next(&p->it);
177+}
178+
179+static int kmem_cache_iter_seq_show(struct seq_file *seq, void *v)
180+{
181+ struct bpf_iter_meta meta;
182+ struct bpf_iter__kmem_cache ctx = {
183+ .meta = &meta,
184+ .s = v,
185+ };
186+ struct bpf_prog *prog;
187+ int ret = 0;
188+
189+ meta.seq = seq;
190+ prog = bpf_iter_get_info(&meta, false);
191+ if (prog)
192+ ret = bpf_iter_run_prog(prog, &ctx);
193+
194+ return ret;
195+}
196+
197+static const struct seq_operations kmem_cache_iter_seq_ops = {
198+ .start = kmem_cache_iter_seq_start,
199+ .next = kmem_cache_iter_seq_next,
200+ .stop = kmem_cache_iter_seq_stop,
201+ .show = kmem_cache_iter_seq_show,
202+};
203+
204+BTF_ID_LIST_GLOBAL_SINGLE(bpf_kmem_cache_btf_id, struct, kmem_cache)
205+
206+static const struct bpf_iter_seq_info kmem_cache_iter_seq_info = {
207+ .seq_ops = &kmem_cache_iter_seq_ops,
208+ .seq_priv_size = sizeof(union kmem_cache_iter_priv),
209+};
210+
211+static void bpf_iter_kmem_cache_show_fdinfo(const struct bpf_iter_aux_info *aux,
212+ struct seq_file *seq)
213+{
214+ seq_puts(seq, "kmem_cache iter\n");
215+}
216+
217+DEFINE_BPF_ITER_FUNC(kmem_cache, struct bpf_iter_meta *meta,
218+ struct kmem_cache *s)
219+
220+static struct bpf_iter_reg bpf_kmem_cache_reg_info = {
221+ .target = "kmem_cache",
222+ .feature = BPF_ITER_RESCHED,
223+ .show_fdinfo = bpf_iter_kmem_cache_show_fdinfo,
224+ .ctx_arg_info_size = 1,
225+ .ctx_arg_info = {
226+ { offsetof(struct bpf_iter__kmem_cache, s),
227+ PTR_TO_BTF_ID_OR_NULL | PTR_TRUSTED },
228+ },
229+ .seq_info = &kmem_cache_iter_seq_info,
230+};
231+
232+static int __init bpf_kmem_cache_iter_init(void)
233+{
234+ bpf_kmem_cache_reg_info.ctx_arg_info[0].btf_id = bpf_kmem_cache_btf_id[0];
235+ return bpf_iter_reg_target(&bpf_kmem_cache_reg_info);
236+}
237+
238+late_initcall(bpf_kmem_cache_iter_init);
kernel/bpf/memalloc.c+1−4
@@ -254,11 +254,8 @@ static void alloc_bulk(struct bpf_mem_cache *c, int cnt, int node, bool atomic) · 253 unmodified lines
254254
255255 static void free_one(void *obj, bool percpu)
256256 {
257− if (percpu) {
257+ if (percpu)
258258 free_percpu(((void __percpu **)obj)[1]);
259− kfree(obj);
260− return;
261− }
262259
263260 kfree(obj);
264261 }
kernel/bpf/range_tree.c+272−0
@@ -0,0 +1,272 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
3+#include <linux/interval_tree_generic.h>
4+#include <linux/slab.h>
5+#include <linux/bpf_mem_alloc.h>
6+#include <linux/bpf.h>
7+#include "range_tree.h"
8+
9+/*
10+ * struct range_tree is a data structure used to allocate contiguous memory
11+ * ranges in bpf arena. It's a large bitmap. The contiguous sequence of bits is
12+ * represented by struct range_node or 'rn' for short.
13+ * rn->rn_rbnode links it into an interval tree while
14+ * rn->rb_range_size links it into a second rbtree sorted by size of the range.
15+ * __find_range() performs binary search and best fit algorithm to find the
16+ * range less or equal requested size.
17+ * range_tree_clear/set() clears or sets a range of bits in this bitmap. The
18+ * adjacent ranges are merged or split at the same time.
19+ *
20+ * The split/merge logic is based/borrowed from XFS's xbitmap32 added
21+ * in commit 6772fcc8890a ("xfs: convert xbitmap to interval tree").
22+ *
23+ * The implementation relies on external lock to protect rbtree-s.
24+ * The alloc/free of range_node-s is done via bpf_mem_alloc.
25+ *
26+ * bpf arena is using range_tree to represent unallocated slots.
27+ * At init time:
28+ * range_tree_set(rt, 0, max);
29+ * Then:
30+ * start = range_tree_find(rt, len);
31+ * if (start >= 0)
32+ * range_tree_clear(rt, start, len);
33+ * to find free range and mark slots as allocated and later:
34+ * range_tree_set(rt, start, len);
35+ * to mark as unallocated after use.
36+ */
37+struct range_node {
38+ struct rb_node rn_rbnode;
39+ struct rb_node rb_range_size;
40+ u32 rn_start;
41+ u32 rn_last; /* inclusive */
42+ u32 __rn_subtree_last;
43+};
44+
45+static struct range_node *rb_to_range_node(struct rb_node *rb)
46+{
47+ return rb_entry(rb, struct range_node, rb_range_size);
48+}
49+
50+static u32 rn_size(struct range_node *rn)
51+{
52+ return rn->rn_last - rn->rn_start + 1;
53+}
54+
55+/* Find range that fits best to requested size */
56+static inline struct range_node *__find_range(struct range_tree *rt, u32 len)
57+{
58+ struct rb_node *rb = rt->range_size_root.rb_root.rb_node;
59+ struct range_node *best = NULL;
60+
61+ while (rb) {
62+ struct range_node *rn = rb_to_range_node(rb);
63+
64+ if (len <= rn_size(rn)) {
65+ best = rn;
66+ rb = rb->rb_right;
67+ } else {
68+ rb = rb->rb_left;
69+ }
70+ }
71+
72+ return best;
73+}
74+
75+s64 range_tree_find(struct range_tree *rt, u32 len)
76+{
77+ struct range_node *rn;
78+
79+ rn = __find_range(rt, len);
80+ if (!rn)
81+ return -ENOENT;
82+ return rn->rn_start;
83+}
84+
85+/* Insert the range into rbtree sorted by the range size */
86+static inline void __range_size_insert(struct range_node *rn,
87+ struct rb_root_cached *root)
88+{
89+ struct rb_node **link = &root->rb_root.rb_node, *rb = NULL;
90+ u64 size = rn_size(rn);
91+ bool leftmost = true;
92+
93+ while (*link) {
94+ rb = *link;
95+ if (size > rn_size(rb_to_range_node(rb))) {
96+ link = &rb->rb_left;
97+ } else {
98+ link = &rb->rb_right;
99+ leftmost = false;
100+ }
101+ }
102+
103+ rb_link_node(&rn->rb_range_size, rb, link);
104+ rb_insert_color_cached(&rn->rb_range_size, root, leftmost);
105+}
106+
107+#define START(node) ((node)->rn_start)
108+#define LAST(node) ((node)->rn_last)
109+
110+INTERVAL_TREE_DEFINE(struct range_node, rn_rbnode, u32,
111+ __rn_subtree_last, START, LAST,
112+ static inline __maybe_unused,
113+ __range_it)
114+
115+static inline __maybe_unused void
116+range_it_insert(struct range_node *rn, struct range_tree *rt)
117+{
118+ __range_size_insert(rn, &rt->range_size_root);
119+ __range_it_insert(rn, &rt->it_root);
120+}
121+
122+static inline __maybe_unused void
123+range_it_remove(struct range_node *rn, struct range_tree *rt)
124+{
125+ rb_erase_cached(&rn->rb_range_size, &rt->range_size_root);
126+ RB_CLEAR_NODE(&rn->rb_range_size);
127+ __range_it_remove(rn, &rt->it_root);
128+}
129+
130+static inline __maybe_unused struct range_node *
131+range_it_iter_first(struct range_tree *rt, u32 start, u32 last)
132+{
133+ return __range_it_iter_first(&rt->it_root, start, last);
134+}
135+
136+/* Clear the range in this range tree */
137+int range_tree_clear(struct range_tree *rt, u32 start, u32 len)
138+{
139+ u32 last = start + len - 1;
140+ struct range_node *new_rn;
141+ struct range_node *rn;
142+
143+ while ((rn = range_it_iter_first(rt, start, last))) {
144+ if (rn->rn_start < start && rn->rn_last > last) {
145+ u32 old_last = rn->rn_last;
146+
147+ /* Overlaps with the entire clearing range */
148+ range_it_remove(rn, rt);
149+ rn->rn_last = start - 1;
150+ range_it_insert(rn, rt);
151+
152+ /* Add a range */
153+ migrate_disable();
154+ new_rn = bpf_mem_alloc(&bpf_global_ma, sizeof(struct range_node));
155+ migrate_enable();
156+ if (!new_rn)
157+ return -ENOMEM;
158+ new_rn->rn_start = last + 1;
159+ new_rn->rn_last = old_last;
160+ range_it_insert(new_rn, rt);
161+ } else if (rn->rn_start < start) {
162+ /* Overlaps with the left side of the clearing range */
163+ range_it_remove(rn, rt);
164+ rn->rn_last = start - 1;
165+ range_it_insert(rn, rt);
166+ } else if (rn->rn_last > last) {
167+ /* Overlaps with the right side of the clearing range */
168+ range_it_remove(rn, rt);
169+ rn->rn_start = last + 1;
170+ range_it_insert(rn, rt);
171+ break;
172+ } else {
173+ /* in the middle of the clearing range */
174+ range_it_remove(rn, rt);
175+ migrate_disable();
176+ bpf_mem_free(&bpf_global_ma, rn);
177+ migrate_enable();
178+ }
179+ }
180+ return 0;
181+}
182+
183+/* Is the whole range set ? */
184+int is_range_tree_set(struct range_tree *rt, u32 start, u32 len)
185+{
186+ u32 last = start + len - 1;
187+ struct range_node *left;
188+
189+ /* Is this whole range set ? */
190+ left = range_it_iter_first(rt, start, last);
191+ if (left && left->rn_start <= start && left->rn_last >= last)
192+ return 0;
193+ return -ESRCH;
194+}
195+
196+/* Set the range in this range tree */
197+int range_tree_set(struct range_tree *rt, u32 start, u32 len)
198+{
199+ u32 last = start + len - 1;
200+ struct range_node *right;
201+ struct range_node *left;
202+ int err;
203+
204+ /* Is this whole range already set ? */
205+ left = range_it_iter_first(rt, start, last);
206+ if (left && left->rn_start <= start && left->rn_last >= last)
207+ return 0;
208+
209+ /* Clear out everything in the range we want to set. */
210+ err = range_tree_clear(rt, start, len);
211+ if (err)
212+ return err;
213+
214+ /* Do we have a left-adjacent range ? */
215+ left = range_it_iter_first(rt, start - 1, start - 1);
216+ if (left && left->rn_last + 1 != start)
217+ return -EFAULT;
218+
219+ /* Do we have a right-adjacent range ? */
220+ right = range_it_iter_first(rt, last + 1, last + 1);
221+ if (right && right->rn_start != last + 1)
222+ return -EFAULT;
223+
224+ if (left && right) {
225+ /* Combine left and right adjacent ranges */
226+ range_it_remove(left, rt);
227+ range_it_remove(right, rt);
228+ left->rn_last = right->rn_last;
229+ range_it_insert(left, rt);
230+ migrate_disable();
231+ bpf_mem_free(&bpf_global_ma, right);
232+ migrate_enable();
233+ } else if (left) {
234+ /* Combine with the left range */
235+ range_it_remove(left, rt);
236+ left->rn_last = last;
237+ range_it_insert(left, rt);
238+ } else if (right) {
239+ /* Combine with the right range */
240+ range_it_remove(right, rt);
241+ right->rn_start = start;
242+ range_it_insert(right, rt);
243+ } else {
244+ migrate_disable();
245+ left = bpf_mem_alloc(&bpf_global_ma, sizeof(struct range_node));
246+ migrate_enable();
247+ if (!left)
248+ return -ENOMEM;
249+ left->rn_start = start;
250+ left->rn_last = last;
251+ range_it_insert(left, rt);
252+ }
253+ return 0;
254+}
255+
256+void range_tree_destroy(struct range_tree *rt)
257+{
258+ struct range_node *rn;
259+
260+ while ((rn = range_it_iter_first(rt, 0, -1U))) {
261+ range_it_remove(rn, rt);
262+ migrate_disable();
263+ bpf_mem_free(&bpf_global_ma, rn);
264+ migrate_enable();
265+ }
266+}
267+
268+void range_tree_init(struct range_tree *rt)
269+{
270+ rt->it_root = RB_ROOT_CACHED;
271+ rt->range_size_root = RB_ROOT_CACHED;
272+}
kernel/bpf/range_tree.h+21−0
@@ -0,0 +1,21 @@
1+/* SPDX-License-Identifier: GPL-2.0-only */
2+/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
3+#ifndef _RANGE_TREE_H
4+#define _RANGE_TREE_H 1
5+
6+struct range_tree {
7+ /* root of interval tree */
8+ struct rb_root_cached it_root;
9+ /* root of rbtree of interval sizes */
10+ struct rb_root_cached range_size_root;
11+};
12+
13+void range_tree_init(struct range_tree *rt);
14+void range_tree_destroy(struct range_tree *rt);
15+
16+int range_tree_clear(struct range_tree *rt, u32 start, u32 len);
17+int range_tree_set(struct range_tree *rt, u32 start, u32 len);
18+int is_range_tree_set(struct range_tree *rt, u32 start, u32 len);
19+s64 range_tree_find(struct range_tree *rt, u32 len);
20+
21+#endif
kernel/bpf/syscall.c+115−9
@@ -155,6 +155,89 @@ static void maybe_wait_bpf_programs(struct bpf_map *map) · 154 unmodified lines
155155 synchronize_rcu();
156156 }
157157
158+static void unpin_uptr_kaddr(void *kaddr)
159+{
160+ if (kaddr)
161+ unpin_user_page(virt_to_page(kaddr));
162+}
163+
164+static void __bpf_obj_unpin_uptrs(struct btf_record *rec, u32 cnt, void *obj)
165+{
166+ const struct btf_field *field;
167+ void **uptr_addr;
168+ int i;
169+
170+ for (i = 0, field = rec->fields; i < cnt; i++, field++) {
171+ if (field->type != BPF_UPTR)
172+ continue;
173+
174+ uptr_addr = obj + field->offset;
175+ unpin_uptr_kaddr(*uptr_addr);
176+ }
177+}
178+
179+static void bpf_obj_unpin_uptrs(struct btf_record *rec, void *obj)
180+{
181+ if (!btf_record_has_field(rec, BPF_UPTR))
182+ return;
183+
184+ __bpf_obj_unpin_uptrs(rec, rec->cnt, obj);
185+}
186+
187+static int bpf_obj_pin_uptrs(struct btf_record *rec, void *obj)
188+{
189+ const struct btf_field *field;
190+ const struct btf_type *t;
191+ unsigned long start, end;
192+ struct page *page;
193+ void **uptr_addr;
194+ int i, err;
195+
196+ if (!btf_record_has_field(rec, BPF_UPTR))
197+ return 0;
198+
199+ for (i = 0, field = rec->fields; i < rec->cnt; i++, field++) {
200+ if (field->type != BPF_UPTR)
201+ continue;
202+
203+ uptr_addr = obj + field->offset;
204+ start = *(unsigned long *)uptr_addr;
205+ if (!start)
206+ continue;
207+
208+ t = btf_type_by_id(field->kptr.btf, field->kptr.btf_id);
209+ /* t->size was checked for zero before */
210+ if (check_add_overflow(start, t->size - 1, &end)) {
211+ err = -EFAULT;
212+ goto unpin_all;
213+ }
214+
215+ /* The uptr's struct cannot span across two pages */
216+ if ((start & PAGE_MASK) != (end & PAGE_MASK)) {
217+ err = -EOPNOTSUPP;
218+ goto unpin_all;
219+ }
220+
221+ err = pin_user_pages_fast(start, 1, FOLL_LONGTERM | FOLL_WRITE, &page);
222+ if (err != 1)
223+ goto unpin_all;
224+
225+ if (PageHighMem(page)) {
226+ err = -EOPNOTSUPP;
227+ unpin_user_page(page);
228+ goto unpin_all;
229+ }
230+
231+ *uptr_addr = page_address(page) + offset_in_page(start);
232+ }
233+
234+ return 0;
235+
236+unpin_all:
237+ __bpf_obj_unpin_uptrs(rec, i, obj);
238+ return err;
239+}
240+
158241 static int bpf_map_update_value(struct bpf_map *map, struct file *map_file,
159242 void *key, void *value, __u64 flags)
160243 {
@@ -199,9 +282,14 @@ static int bpf_map_update_value(struct bpf_map *map, struct file *map_file, · 38 unmodified lines
199282 map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) {
200283 err = map->ops->map_push_elem(map, value, flags);
201284 } else {
202− rcu_read_lock();
203− err = map->ops->map_update_elem(map, key, value, flags);
204− rcu_read_unlock();
285+ err = bpf_obj_pin_uptrs(map->record, value);
286+ if (!err) {
287+ rcu_read_lock();
288+ err = map->ops->map_update_elem(map, key, value, flags);
289+ rcu_read_unlock();
290+ if (err)
291+ bpf_obj_unpin_uptrs(map->record, value);
292+ }
205293 }
206294 bpf_enable_instrumentation();
207295
@@ -548,6 +636,7 @@ void btf_record_free(struct btf_record *rec) · 340 unmodified lines
548636 case BPF_KPTR_UNREF:
549637 case BPF_KPTR_REF:
550638 case BPF_KPTR_PERCPU:
639+ case BPF_UPTR:
551640 if (rec->fields[i].kptr.module)
552641 module_put(rec->fields[i].kptr.module);
553642 if (btf_is_kernel(rec->fields[i].kptr.btf))
@@ -597,6 +686,7 @@ struct btf_record *btf_record_dup(const struct btf_record *rec) · 43 unmodified lines
597686 case BPF_KPTR_UNREF:
598687 case BPF_KPTR_REF:
599688 case BPF_KPTR_PERCPU:
689+ case BPF_UPTR:
600690 if (btf_is_kernel(fields[i].kptr.btf))
601691 btf_get(fields[i].kptr.btf);
602692 if (fields[i].kptr.module && !try_module_get(fields[i].kptr.module)) {
@@ -714,6 +804,10 @@ void bpf_obj_free_fields(const struct btf_record *rec, void *obj) · 111 unmodified lines
714804 field->kptr.dtor(xchgd_field);
715805 }
716806 break;
807+ case BPF_UPTR:
808+ /* The caller ensured that no one is using the uptr */
809+ unpin_uptr_kaddr(*(void **)field_ptr);
810+ break;
717811 case BPF_LIST_HEAD:
718812 if (WARN_ON_ONCE(rec->spin_lock_off < 0))
719813 continue;
@@ -1105,7 +1199,7 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token, · 385 unmodified lines
11051199
11061200 map->record = btf_parse_fields(btf, value_type,
11071201 BPF_SPIN_LOCK | BPF_TIMER | BPF_KPTR | BPF_LIST_HEAD |
1108− BPF_RB_ROOT | BPF_REFCOUNT | BPF_WORKQUEUE,
1202+ BPF_RB_ROOT | BPF_REFCOUNT | BPF_WORKQUEUE | BPF_UPTR,
11091203 map->value_size);
11101204 if (!IS_ERR_OR_NULL(map->record)) {
11111205 int i;
@@ -1161,6 +1255,12 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token, · 49 unmodified lines
11611255 goto free_map_tab;
11621256 }
11631257 break;
1258+ case BPF_UPTR:
1259+ if (map->map_type != BPF_MAP_TYPE_TASK_STORAGE) {
1260+ ret = -EOPNOTSUPP;
1261+ goto free_map_tab;
1262+ }
1263+ break;
11641264 case BPF_LIST_HEAD:
11651265 case BPF_RB_ROOT:
11661266 if (map->map_type != BPF_MAP_TYPE_HASH &&
@@ -3218,7 +3318,8 @@ static void bpf_tracing_link_release(struct bpf_link *link) · 2051 unmodified lines
32183318 container_of(link, struct bpf_tracing_link, link.link);
32193319
32203320 WARN_ON_ONCE(bpf_trampoline_unlink_prog(&tr_link->link,
3221− tr_link->trampoline));
3321+ tr_link->trampoline,
3322+ tr_link->tgt_prog));
32223323
32233324 bpf_trampoline_put(tr_link->trampoline);
32243325
@@ -3358,7 +3459,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog, · 133 unmodified lines
33583459 * in prog->aux
33593460 *
33603461 * - if prog->aux->dst_trampoline is NULL, the program has already been
3361− * attached to a target and its initial target was cleared (below)
3462+ * attached to a target and its initial target was cleared (below)
33623463 *
33633464 * - if tgt_prog != NULL, the caller specified tgt_prog_fd +
33643465 * target_btf_id using the link_create API.
@@ -3433,7 +3534,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog, · 68 unmodified lines
34333534 if (err)
34343535 goto out_unlock;
34353536
3436− err = bpf_trampoline_link_prog(&link->link, tr);
3537+ err = bpf_trampoline_link_prog(&link->link, tr, tgt_prog);
34373538 if (err) {
34383539 bpf_link_cleanup(&link_primer);
34393540 link = NULL;
@@ -4002,10 +4103,14 @@ static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog, · 562 unmodified lines
40024103 if (prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI &&
40034104 attach_type != BPF_TRACE_UPROBE_MULTI)
40044105 return -EINVAL;
4106+ if (prog->expected_attach_type == BPF_TRACE_UPROBE_SESSION &&
4107+ attach_type != BPF_TRACE_UPROBE_SESSION)
4108+ return -EINVAL;
40054109 if (attach_type != BPF_PERF_EVENT &&
40064110 attach_type != BPF_TRACE_KPROBE_MULTI &&
40074111 attach_type != BPF_TRACE_KPROBE_SESSION &&
4008− attach_type != BPF_TRACE_UPROBE_MULTI)
4112+ attach_type != BPF_TRACE_UPROBE_MULTI &&
4113+ attach_type != BPF_TRACE_UPROBE_SESSION)
40094114 return -EINVAL;
40104115 return 0;
40114116 case BPF_PROG_TYPE_SCHED_CLS:
@@ -5258,7 +5363,8 @@ static int link_create(union bpf_attr *attr, bpfptr_t uattr) · 1246 unmodified lines
52585363 else if (attr->link_create.attach_type == BPF_TRACE_KPROBE_MULTI ||
52595364 attr->link_create.attach_type == BPF_TRACE_KPROBE_SESSION)
52605365 ret = bpf_kprobe_multi_link_attach(attr, prog);
5261− else if (attr->link_create.attach_type == BPF_TRACE_UPROBE_MULTI)
5366+ else if (attr->link_create.attach_type == BPF_TRACE_UPROBE_MULTI ||
5367+ attr->link_create.attach_type == BPF_TRACE_UPROBE_SESSION)
52625368 ret = bpf_uprobe_multi_link_attach(attr, prog);
52635369 break;
52645370 default:
kernel/bpf/trampoline.c+50−10
@@ -115,10 +115,14 @@ bool bpf_prog_has_trampoline(const struct bpf_prog *prog) · 114 unmodified lines
115115 (ptype == BPF_PROG_TYPE_LSM && eatype == BPF_LSM_MAC);
116116 }
117117
118−void bpf_image_ksym_add(void *data, unsigned int size, struct bpf_ksym *ksym)
118+void bpf_image_ksym_init(void *data, unsigned int size, struct bpf_ksym *ksym)
119119 {
120120 ksym->start = (unsigned long) data;
121121 ksym->end = ksym->start + size;
122+}
123+
124+void bpf_image_ksym_add(struct bpf_ksym *ksym)
125+{
122126 bpf_ksym_add(ksym);
123127 perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_BPF, ksym->start,
124128 PAGE_SIZE, false, ksym->name);
@@ -377,7 +381,8 @@ static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size) · 252 unmodified lines
377381 ksym = &im->ksym;
378382 INIT_LIST_HEAD_RCU(&ksym->lnode);
379383 snprintf(ksym->name, KSYM_NAME_LEN, "bpf_trampoline_%llu", key);
380− bpf_image_ksym_add(image, size, ksym);
384+ bpf_image_ksym_init(image, size, ksym);
385+ bpf_image_ksym_add(ksym);
381386 return im;
382387
383388 out_free_image:
@@ -523,7 +528,27 @@ static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog) · 139 unmodified lines
523528 }
524529 }
525530
526−static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr)
531+static int bpf_freplace_check_tgt_prog(struct bpf_prog *tgt_prog)
532+{
533+ struct bpf_prog_aux *aux = tgt_prog->aux;
534+
535+ guard(mutex)(&aux->ext_mutex);
536+ if (aux->prog_array_member_cnt)
537+ /* Program extensions can not extend target prog when the target
538+ * prog has been updated to any prog_array map as tail callee.
539+ * It's to prevent a potential infinite loop like:
540+ * tgt prog entry -> tgt prog subprog -> freplace prog entry
541+ * --tailcall-> tgt prog entry.
542+ */
543+ return -EBUSY;
544+
545+ aux->is_extended = true;
546+ return 0;
547+}
548+
549+static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link,
550+ struct bpf_trampoline *tr,
551+ struct bpf_prog *tgt_prog)
527552 {
528553 enum bpf_tramp_prog_type kind;
529554 struct bpf_tramp_link *link_exiting;
@@ -544,6 +569,9 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_tr · 14 unmodified lines
544569 /* Cannot attach extension if fentry/fexit are in use. */
545570 if (cnt)
546571 return -EBUSY;
572+ err = bpf_freplace_check_tgt_prog(tgt_prog);
573+ if (err)
574+ return err;
547575 tr->extension_prog = link->link.prog;
548576 return bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP, NULL,
549577 link->link.prog->bpf_func);
@@ -570,17 +598,21 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_tr · 20 unmodified lines
570598 return err;
571599 }
572600
573−int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr)
601+int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
602+ struct bpf_trampoline *tr,
603+ struct bpf_prog *tgt_prog)
574604 {
575605 int err;
576606
577607 mutex_lock(&tr->mutex);
578− err = __bpf_trampoline_link_prog(link, tr);
608+ err = __bpf_trampoline_link_prog(link, tr, tgt_prog);
579609 mutex_unlock(&tr->mutex);
580610 return err;
581611 }
582612
583−static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr)
613+static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
614+ struct bpf_trampoline *tr,
615+ struct bpf_prog *tgt_prog)
584616 {
585617 enum bpf_tramp_prog_type kind;
586618 int err;
@@ -591,6 +623,8 @@ static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_ · 4 unmodified lines
591623 err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP,
592624 tr->extension_prog->bpf_func, NULL);
593625 tr->extension_prog = NULL;
626+ guard(mutex)(&tgt_prog->aux->ext_mutex);
627+ tgt_prog->aux->is_extended = false;
594628 return err;
595629 }
596630 hlist_del_init(&link->tramp_hlist);
@@ -599,12 +633,14 @@ static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_ · 2 unmodified lines
599633 }
600634
601635 /* bpf_trampoline_unlink_prog() should never fail. */
602−int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr)
636+int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
637+ struct bpf_trampoline *tr,
638+ struct bpf_prog *tgt_prog)
603639 {
604640 int err;
605641
606642 mutex_lock(&tr->mutex);
607− err = __bpf_trampoline_unlink_prog(link, tr);
643+ err = __bpf_trampoline_unlink_prog(link, tr, tgt_prog);
608644 mutex_unlock(&tr->mutex);
609645 return err;
610646 }
@@ -619,7 +655,7 @@ static void bpf_shim_tramp_link_release(struct bpf_link *link) · 8 unmodified lines
619655 if (!shim_link->trampoline)
620656 return;
621657
622− WARN_ON_ONCE(bpf_trampoline_unlink_prog(&shim_link->link, shim_link->trampoline));
658+ WARN_ON_ONCE(bpf_trampoline_unlink_prog(&shim_link->link, shim_link->trampoline, NULL));
623659 bpf_trampoline_put(shim_link->trampoline);
624660 }
625661
@@ -733,7 +769,7 @@ int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog, · 107 unmodified lines
733769 goto err;
734770 }
735771
736− err = __bpf_trampoline_link_prog(&shim_link->link, tr);
772+ err = __bpf_trampoline_link_prog(&shim_link->link, tr, NULL);
737773 if (err)
738774 goto err;
739775
@@ -868,6 +904,8 @@ static u64 notrace __bpf_prog_enter_recur(struct bpf_prog *prog, struct bpf_tram · 128 unmodified lines
868904
869905 if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {
870906 bpf_prog_inc_misses_counter(prog);
907+ if (prog->aux->recursion_detected)
908+ prog->aux->recursion_detected(prog);
871909 return 0;
872910 }
873911 return bpf_prog_start_time();
@@ -944,6 +982,8 @@ u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog, · 70 unmodified lines
944982
945983 if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {
946984 bpf_prog_inc_misses_counter(prog);
985+ if (prog->aux->recursion_detected)
986+ prog->aux->recursion_detected(prog);
947987 return 0;
948988 }
949989 return bpf_prog_start_time();
kernel/bpf/verifier.c+418−179
@@ -194,6 +194,8 @@ struct bpf_verifier_stack_elem { · 193 unmodified lines
194194
195195 #define BPF_GLOBAL_PERCPU_MA_MAX_SIZE 512
196196
197+#define BPF_PRIV_STACK_MIN_SIZE 64
198+
197199 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx);
198200 static int release_reference(struct bpf_verifier_env *env, int ref_obj_id);
199201 static void invalidate_non_owning_refs(struct bpf_verifier_env *env);
@@ -418,6 +420,25 @@ static struct btf_record *reg_btf_record(const struct bpf_reg_state *reg) · 218 unmodified lines
418420 return rec;
419421 }
420422
423+static bool mask_raw_tp_reg_cond(const struct bpf_verifier_env *env, struct bpf_reg_state *reg) {
424+ return reg->type == (PTR_TO_BTF_ID | PTR_TRUSTED | PTR_MAYBE_NULL) &&
425+ bpf_prog_is_raw_tp(env->prog) && !reg->ref_obj_id;
426+}
427+
428+static bool mask_raw_tp_reg(const struct bpf_verifier_env *env, struct bpf_reg_state *reg)
429+{
430+ if (!mask_raw_tp_reg_cond(env, reg))
431+ return false;
432+ reg->type &= ~PTR_MAYBE_NULL;
433+ return true;
434+}
435+
436+static void unmask_raw_tp_reg(struct bpf_reg_state *reg, bool result)
437+{
438+ if (result)
439+ reg->type |= PTR_MAYBE_NULL;
440+}
441+
421442 static bool subprog_is_global(const struct bpf_verifier_env *env, int subprog)
422443 {
423444 struct bpf_func_info_aux *aux = env->prog->aux->func_info_aux;
@@ -1265,6 +1286,7 @@ static int copy_reference_state(struct bpf_func_state *dst, const struct bpf_fun · 841 unmodified lines
12651286 if (!dst->refs)
12661287 return -ENOMEM;
12671288
1289+ dst->active_locks = src->active_locks;
12681290 dst->acquired_refs = src->acquired_refs;
12691291 return 0;
12701292 }
@@ -1335,13 +1357,32 @@ static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx) · 64 unmodified lines
13351357 if (err)
13361358 return err;
13371359 id = ++env->id_gen;
1360+ state->refs[new_ofs].type = REF_TYPE_PTR;
13381361 state->refs[new_ofs].id = id;
13391362 state->refs[new_ofs].insn_idx = insn_idx;
1340− state->refs[new_ofs].callback_ref = state->in_callback_fn ? state->frameno : 0;
13411363
13421364 return id;
13431365 }
13441366
1367+static int acquire_lock_state(struct bpf_verifier_env *env, int insn_idx, enum ref_state_type type,
1368+ int id, void *ptr)
1369+{
1370+ struct bpf_func_state *state = cur_func(env);
1371+ int new_ofs = state->acquired_refs;
1372+ int err;
1373+
1374+ err = resize_reference_state(state, state->acquired_refs + 1);
1375+ if (err)
1376+ return err;
1377+ state->refs[new_ofs].type = type;
1378+ state->refs[new_ofs].id = id;
1379+ state->refs[new_ofs].insn_idx = insn_idx;
1380+ state->refs[new_ofs].ptr = ptr;
1381+
1382+ state->active_locks++;
1383+ return 0;
1384+}
1385+
13451386 /* release function corresponding to acquire_reference_state(). Idempotent. */
13461387 static int release_reference_state(struct bpf_func_state *state, int ptr_id)
13471388 {
@@ -1349,10 +1390,9 @@ static int release_reference_state(struct bpf_func_state *state, int ptr_id) · 1 unmodified line
13491390
13501391 last_idx = state->acquired_refs - 1;
13511392 for (i = 0; i < state->acquired_refs; i++) {
1393+ if (state->refs[i].type != REF_TYPE_PTR)
1394+ continue;
13521395 if (state->refs[i].id == ptr_id) {
1353− /* Cannot release caller references in callbacks */
1354− if (state->in_callback_fn && state->refs[i].callback_ref != state->frameno)
1355− return -EINVAL;
13561396 if (last_idx && i != last_idx)
13571397 memcpy(&state->refs[i], &state->refs[last_idx],
13581398 sizeof(*state->refs));
@@ -1364,6 +1404,45 @@ static int release_reference_state(struct bpf_func_state *state, int ptr_id) · 5 unmodified lines
13641404 return -EINVAL;
13651405 }
13661406
1407+static int release_lock_state(struct bpf_func_state *state, int type, int id, void *ptr)
1408+{
1409+ int i, last_idx;
1410+
1411+ last_idx = state->acquired_refs - 1;
1412+ for (i = 0; i < state->acquired_refs; i++) {
1413+ if (state->refs[i].type != type)
1414+ continue;
1415+ if (state->refs[i].id == id && state->refs[i].ptr == ptr) {
1416+ if (last_idx && i != last_idx)
1417+ memcpy(&state->refs[i], &state->refs[last_idx],
1418+ sizeof(*state->refs));
1419+ memset(&state->refs[last_idx], 0, sizeof(*state->refs));
1420+ state->acquired_refs--;
1421+ state->active_locks--;
1422+ return 0;
1423+ }
1424+ }
1425+ return -EINVAL;
1426+}
1427+
1428+static struct bpf_reference_state *find_lock_state(struct bpf_verifier_env *env, enum ref_state_type type,
1429+ int id, void *ptr)
1430+{
1431+ struct bpf_func_state *state = cur_func(env);
1432+ int i;
1433+
1434+ for (i = 0; i < state->acquired_refs; i++) {
1435+ struct bpf_reference_state *s = &state->refs[i];
1436+
1437+ if (s->type == REF_TYPE_PTR || s->type != type)
1438+ continue;
1439+
1440+ if (s->id == id && s->ptr == ptr)
1441+ return s;
1442+ }
1443+ return NULL;
1444+}
1445+
13671446 static void free_func_state(struct bpf_func_state *state)
13681447 {
13691448 if (!state)
@@ -1373,13 +1452,6 @@ static void free_func_state(struct bpf_func_state *state) · 3 unmodified lines
13731452 kfree(state);
13741453 }
13751454
1376−static void clear_jmp_history(struct bpf_verifier_state *state)
1377−{
1378− kfree(state->jmp_history);
1379− state->jmp_history = NULL;
1380− state->jmp_history_cnt = 0;
1381−}
1382−
13831455 static void free_verifier_state(struct bpf_verifier_state *state,
13841456 bool free_self)
13851457 {
@@ -1389,7 +1461,6 @@ static void free_verifier_state(struct bpf_verifier_state *state, · 3 unmodified lines
13891461 free_func_state(state->frame[i]);
13901462 state->frame[i] = NULL;
13911463 }
1392− clear_jmp_history(state);
13931464 if (free_self)
13941465 kfree(state);
13951466 }
@@ -1415,13 +1486,6 @@ static int copy_verifier_state(struct bpf_verifier_state *dst_state, · 19 unmodified lines
14151486 struct bpf_func_state *dst;
14161487 int i, err;
14171488
1418− dst_state->jmp_history = copy_array(dst_state->jmp_history, src->jmp_history,
1419− src->jmp_history_cnt, sizeof(*dst_state->jmp_history),
1420− GFP_USER);
1421− if (!dst_state->jmp_history)
1422− return -ENOMEM;
1423− dst_state->jmp_history_cnt = src->jmp_history_cnt;
1424−
14251489 /* if dst has more stack frames then src frame, free them, this is also
14261490 * necessary in case of exceptional exits using bpf_throw.
14271491 */
@@ -1434,12 +1498,12 @@ static int copy_verifier_state(struct bpf_verifier_state *dst_state, · 6 unmodified lines
14341498 dst_state->active_preempt_lock = src->active_preempt_lock;
14351499 dst_state->in_sleepable = src->in_sleepable;
14361500 dst_state->curframe = src->curframe;
1437− dst_state->active_lock.ptr = src->active_lock.ptr;
1438− dst_state->active_lock.id = src->active_lock.id;
14391501 dst_state->branches = src->branches;
14401502 dst_state->parent = src->parent;
14411503 dst_state->first_insn_idx = src->first_insn_idx;
14421504 dst_state->last_insn_idx = src->last_insn_idx;
1505+ dst_state->insn_hist_start = src->insn_hist_start;
1506+ dst_state->insn_hist_end = src->insn_hist_end;
14431507 dst_state->dfs_depth = src->dfs_depth;
14441508 dst_state->callback_unroll_depth = src->callback_unroll_depth;
14451509 dst_state->used_as_loop_entry = src->used_as_loop_entry;
@@ -2492,9 +2556,14 @@ static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env, · 1046 unmodified lines
24922556 * The caller state doesn't matter.
24932557 * This is async callback. It starts in a fresh stack.
24942558 * Initialize it similar to do_check_common().
2559+ * But we do need to make sure to not clobber insn_hist, so we keep
2560+ * chaining insn_hist_start/insn_hist_end indices as for a normal
2561+ * child state.
24952562 */
24962563 elem->st.branches = 1;
24972564 elem->st.in_sleepable = is_sleepable;
2565+ elem->st.insn_hist_start = env->cur_state->insn_hist_end;
2566+ elem->st.insn_hist_end = elem->st.insn_hist_start;
24982567 frame = kzalloc(sizeof(*frame), GFP_KERNEL);
24992568 if (!frame)
25002569 goto err;
@@ -3474,11 +3543,10 @@ static void linked_regs_unpack(u64 val, struct linked_regs *s) · 973 unmodified lines
34743543 }
34753544
34763545 /* for any branch, call, exit record the history of jmps in the given state */
3477−static int push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur,
3478− int insn_flags, u64 linked_regs)
3546+static int push_insn_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur,
3547+ int insn_flags, u64 linked_regs)
34793548 {
3480− u32 cnt = cur->jmp_history_cnt;
3481− struct bpf_jmp_history_entry *p;
3549+ struct bpf_insn_hist_entry *p;
34823550 size_t alloc_size;
34833551
34843552 /* combine instruction flags if we already recorded this instruction */
@@ -3498,29 +3566,32 @@ static int push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_st · 13 unmodified lines
34983566 return 0;
34993567 }
35003568
3501− cnt++;
3502− alloc_size = kmalloc_size_roundup(size_mul(cnt, sizeof(*p)));
3503− p = krealloc(cur->jmp_history, alloc_size, GFP_USER);
3504− if (!p)
3505− return -ENOMEM;
3506− cur->jmp_history = p;
3569+ if (cur->insn_hist_end + 1 > env->insn_hist_cap) {
3570+ alloc_size = size_mul(cur->insn_hist_end + 1, sizeof(*p));
3571+ p = kvrealloc(env->insn_hist, alloc_size, GFP_USER);
3572+ if (!p)
3573+ return -ENOMEM;
3574+ env->insn_hist = p;
3575+ env->insn_hist_cap = alloc_size / sizeof(*p);
3576+ }
35073577
3508− p = &cur->jmp_history[cnt - 1];
3578+ p = &env->insn_hist[cur->insn_hist_end];
35093579 p->idx = env->insn_idx;
35103580 p->prev_idx = env->prev_insn_idx;
35113581 p->flags = insn_flags;
35123582 p->linked_regs = linked_regs;
3513− cur->jmp_history_cnt = cnt;
3583+
3584+ cur->insn_hist_end++;
35143585 env->cur_hist_ent = p;
35153586
35163587 return 0;
35173588 }
35183589
3519−static struct bpf_jmp_history_entry *get_jmp_hist_entry(struct bpf_verifier_state *st,
3520− u32 hist_end, int insn_idx)
3590+static struct bpf_insn_hist_entry *get_insn_hist_entry(struct bpf_verifier_env *env,
3591+ u32 hist_start, u32 hist_end, int insn_idx)
35213592 {
3522− if (hist_end > 0 && st->jmp_history[hist_end - 1].idx == insn_idx)
3523− return &st->jmp_history[hist_end - 1];
3593+ if (hist_end > hist_start && env->insn_hist[hist_end - 1].idx == insn_idx)
3594+ return &env->insn_hist[hist_end - 1];
35243595 return NULL;
35253596 }
35263597
@@ -3537,25 +3608,26 @@ static struct bpf_jmp_history_entry *get_jmp_hist_entry(struct bpf_verifier_stat · 10 unmodified lines
35373608 * history entry recording a jump from last instruction of parent state and
35383609 * first instruction of given state.
35393610 */
3540−static int get_prev_insn_idx(struct bpf_verifier_state *st, int i,
3541− u32 *history)
3611+static int get_prev_insn_idx(const struct bpf_verifier_env *env,
3612+ struct bpf_verifier_state *st,
3613+ int insn_idx, u32 hist_start, u32 *hist_endp)
35423614 {
3543− u32 cnt = *history;
3615+ u32 hist_end = *hist_endp;
3616+ u32 cnt = hist_end - hist_start;
35443617
3545− if (i == st->first_insn_idx) {
3618+ if (insn_idx == st->first_insn_idx) {
35463619 if (cnt == 0)
35473620 return -ENOENT;
3548− if (cnt == 1 && st->jmp_history[0].idx == i)
3621+ if (cnt == 1 && env->insn_hist[hist_start].idx == insn_idx)
35493622 return -ENOENT;
35503623 }
35513624
3552− if (cnt && st->jmp_history[cnt - 1].idx == i) {
3553− i = st->jmp_history[cnt - 1].prev_idx;
3554− (*history)--;
3625+ if (cnt && env->insn_hist[hist_end - 1].idx == insn_idx) {
3626+ (*hist_endp)--;
3627+ return env->insn_hist[hist_end - 1].prev_idx;
35553628 } else {
3556− i--;
3629+ return insn_idx - 1;
35573630 }
3558− return i;
35593631 }
35603632
35613633 static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn)
@@ -3727,7 +3799,7 @@ static void fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask) · 165 unmodified lines
37273799 /* If any register R in hist->linked_regs is marked as precise in bt,
37283800 * do bt_set_frame_{reg,slot}(bt, R) for all registers in hist->linked_regs.
37293801 */
3730−static void bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_history_entry *hist)
3802+static void bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_insn_hist_entry *hist)
37313803 {
37323804 struct linked_regs linked_regs;
37333805 bool some_precise = false;
@@ -3772,7 +3844,7 @@ static bool calls_callback(struct bpf_verifier_env *env, int insn_idx); · 38 unmodified lines
37723844 * - *was* processed previously during backtracking.
37733845 */
37743846 static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
3775− struct bpf_jmp_history_entry *hist, struct backtrack_state *bt)
3847+ struct bpf_insn_hist_entry *hist, struct backtrack_state *bt)
37763848 {
37773849 const struct bpf_insn_cbs cbs = {
37783850 .cb_call = disasm_kfunc_name,
@@ -4191,7 +4263,7 @@ static void mark_all_scalars_imprecise(struct bpf_verifier_env *env, struct bpf_ · 412 unmodified lines
41914263 * SCALARS, as well as any other registers and slots that contribute to
41924264 * a tracked state of given registers/stack slots, depending on specific BPF
41934265 * assembly instructions (see backtrack_insns() for exact instruction handling
4194− * logic). This backtracking relies on recorded jmp_history and is able to
4266+ * logic). This backtracking relies on recorded insn_hist and is able to
41954267 * traverse entire chain of parent states. This process ends only when all the
41964268 * necessary registers/slots and their transitive dependencies are marked as
41974269 * precise.
@@ -4308,8 +4380,9 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int regno) · 110 unmodified lines
43084380
43094381 for (;;) {
43104382 DECLARE_BITMAP(mask, 64);
4311− u32 history = st->jmp_history_cnt;
4312− struct bpf_jmp_history_entry *hist;
4383+ u32 hist_start = st->insn_hist_start;
4384+ u32 hist_end = st->insn_hist_end;
4385+ struct bpf_insn_hist_entry *hist;
43134386
43144387 if (env->log.level & BPF_LOG_LEVEL2) {
43154388 verbose(env, "mark_precise: frame%d: last_idx %d first_idx %d subseq_idx %d \n",
@@ -4348,7 +4421,7 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int regno) · 32 unmodified lines
43484421 err = 0;
43494422 skip_first = false;
43504423 } else {
4351− hist = get_jmp_hist_entry(st, history, i);
4424+ hist = get_insn_hist_entry(env, hist_start, hist_end, i);
43524425 err = backtrack_insn(env, i, subseq_idx, hist, bt);
43534426 }
43544427 if (err == -ENOTSUPP) {
@@ -4365,7 +4438,7 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int regno) · 10 unmodified lines
43654438 */
43664439 return 0;
43674440 subseq_idx = i;
4368− i = get_prev_insn_idx(st, i, &history);
4441+ i = get_prev_insn_idx(env, st, i, hist_start, &hist_end);
43694442 if (i == -ENOENT)
43704443 break;
43714444 if (i >= env->prog->len) {
@@ -4731,7 +4804,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, · 359 unmodified lines
47314804 }
47324805
47334806 if (insn_flags)
4734− return push_jmp_history(env, env->cur_state, insn_flags, 0);
4807+ return push_insn_history(env, env->cur_state, insn_flags, 0);
47354808 return 0;
47364809 }
47374810
@@ -5038,7 +5111,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, · 300 unmodified lines
50385111 insn_flags = 0; /* we are not restoring spilled register */
50395112 }
50405113 if (insn_flags)
5041− return push_jmp_history(env, env->cur_state, insn_flags, 0);
5114+ return push_insn_history(env, env->cur_state, insn_flags, 0);
50425115 return 0;
50435116 }
50445117
@@ -5423,7 +5496,7 @@ static bool in_sleepable(struct bpf_verifier_env *env) · 378 unmodified lines
54235496 static bool in_rcu_cs(struct bpf_verifier_env *env)
54245497 {
54255498 return env->cur_state->active_rcu_lock ||
5426− env->cur_state->active_lock.ptr ||
5499+ cur_func(env)->active_locks ||
54275500 !in_sleepable(env);
54285501 }
54295502
@@ -5491,6 +5564,22 @@ static u32 btf_ld_kptr_type(struct bpf_verifier_env *env, struct btf_field *kptr · 61 unmodified lines
54915564 return ret;
54925565 }
54935566
5567+static int mark_uptr_ld_reg(struct bpf_verifier_env *env, u32 regno,
5568+ struct btf_field *field)
5569+{
5570+ struct bpf_reg_state *reg;
5571+ const struct btf_type *t;
5572+
5573+ t = btf_type_by_id(field->kptr.btf, field->kptr.btf_id);
5574+ mark_reg_known_zero(env, cur_regs(env), regno);
5575+ reg = reg_state(env, regno);
5576+ reg->type = PTR_TO_MEM | PTR_MAYBE_NULL;
5577+ reg->mem_size = t->size;
5578+ reg->id = ++env->id_gen;
5579+
5580+ return 0;
5581+}
5582+
54945583 static int check_map_kptr_access(struct bpf_verifier_env *env, u32 regno,
54955584 int value_regno, int insn_idx,
54965585 struct btf_field *kptr_field)
@@ -5519,9 +5608,15 @@ static int check_map_kptr_access(struct bpf_verifier_env *env, u32 regno, · 22 unmodified lines
55195608 verbose(env, "store to referenced kptr disallowed\n");
55205609 return -EACCES;
55215610 }
5611+ if (class != BPF_LDX && kptr_field->type == BPF_UPTR) {
5612+ verbose(env, "store to uptr disallowed\n");
5613+ return -EACCES;
5614+ }
55225615
55235616 if (class == BPF_LDX) {
5524− val_reg = reg_state(env, value_regno);
5617+ if (kptr_field->type == BPF_UPTR)
5618+ return mark_uptr_ld_reg(env, value_regno, kptr_field);
5619+
55255620 /* We can simply mark the value_regno receiving the pointer
55265621 * value from map as PTR_TO_BTF_ID, with the correct type.
55275622 */
@@ -5579,21 +5674,26 @@ static int check_map_access(struct bpf_verifier_env *env, u32 regno, · 51 unmodified lines
55795674 case BPF_KPTR_UNREF:
55805675 case BPF_KPTR_REF:
55815676 case BPF_KPTR_PERCPU:
5677+ case BPF_UPTR:
55825678 if (src != ACCESS_DIRECT) {
5583− verbose(env, "kptr cannot be accessed indirectly by helper\n");
5679+ verbose(env, "%s cannot be accessed indirectly by helper\n",
5680+ btf_field_type_name(field->type));
55845681 return -EACCES;
55855682 }
55865683 if (!tnum_is_const(reg->var_off)) {
5587− verbose(env, "kptr access cannot have variable offset\n");
5684+ verbose(env, "%s access cannot have variable offset\n",
5685+ btf_field_type_name(field->type));
55885686 return -EACCES;
55895687 }
55905688 if (p != off + reg->var_off.value) {
5591− verbose(env, "kptr access misaligned expected=%u off=%llu\n",
5689+ verbose(env, "%s access misaligned expected=%u off=%llu\n",
5690+ btf_field_type_name(field->type),
55925691 p, off + reg->var_off.value);
55935692 return -EACCES;
55945693 }
55955694 if (size != bpf_size_to_bytes(BPF_DW)) {
5596− verbose(env, "kptr access size must be BPF_DW\n");
5695+ verbose(env, "%s access size must be BPF_DW\n",
5696+ btf_field_type_name(field->type));
55975697 return -EACCES;
55985698 }
55995699 break;
@@ -5988,6 +6088,34 @@ static int check_ptr_alignment(struct bpf_verifier_env *env, · 388 unmodified lines
59886088 strict);
59896089 }
59906090
6091+static enum priv_stack_mode bpf_enable_priv_stack(struct bpf_prog *prog)
6092+{
6093+ if (!bpf_jit_supports_private_stack())
6094+ return NO_PRIV_STACK;
6095+
6096+ /* bpf_prog_check_recur() checks all prog types that use bpf trampoline
6097+ * while kprobe/tp/perf_event/raw_tp don't use trampoline hence checked
6098+ * explicitly.
6099+ */
6100+ switch (prog->type) {
6101+ case BPF_PROG_TYPE_KPROBE:
6102+ case BPF_PROG_TYPE_TRACEPOINT:
6103+ case BPF_PROG_TYPE_PERF_EVENT:
6104+ case BPF_PROG_TYPE_RAW_TRACEPOINT:
6105+ return PRIV_STACK_ADAPTIVE;
6106+ case BPF_PROG_TYPE_TRACING:
6107+ case BPF_PROG_TYPE_LSM:
6108+ case BPF_PROG_TYPE_STRUCT_OPS:
6109+ if (prog->aux->priv_stack_requested || bpf_prog_check_recur(prog))
6110+ return PRIV_STACK_ADAPTIVE;
6111+ fallthrough;
6112+ default:
6113+ break;
6114+ }
6115+
6116+ return NO_PRIV_STACK;
6117+}
6118+
59916119 static int round_up_stack_depth(struct bpf_verifier_env *env, int stack_depth)
59926120 {
59936121 if (env->prog->jit_requested)
@@ -6005,17 +6133,20 @@ static int round_up_stack_depth(struct bpf_verifier_env *env, int stack_depth) · 11 unmodified lines
60056133 * Since recursion is prevented by check_cfg() this algorithm
60066134 * only needs a local stack of MAX_CALL_FRAMES to remember callsites
60076135 */
6008−static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx)
6136+static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,
6137+ bool priv_stack_supported)
60096138 {
60106139 struct bpf_subprog_info *subprog = env->subprog_info;
60116140 struct bpf_insn *insn = env->prog->insnsi;
6012− int depth = 0, frame = 0, i, subprog_end;
6141+ int depth = 0, frame = 0, i, subprog_end, subprog_depth;
60136142 bool tail_call_reachable = false;
60146143 int ret_insn[MAX_CALL_FRAMES];
60156144 int ret_prog[MAX_CALL_FRAMES];
60166145 int j;
60176146
60186147 i = subprog[idx].start;
6148+ if (!priv_stack_supported)
6149+ subprog[idx].priv_stack_mode = NO_PRIV_STACK;
60196150 process_func:
60206151 /* protect against potential stack overflow that might happen when
60216152 * bpf2bpf calls get combined with tailcalls. Limit the caller's stack
@@ -6042,11 +6173,31 @@ process_func: · 20 unmodified lines
60426173 depth);
60436174 return -EACCES;
60446175 }
6045− depth += round_up_stack_depth(env, subprog[idx].stack_depth);
6046− if (depth > MAX_BPF_STACK) {
6047− verbose(env, "combined stack size of %d calls is %d. Too large\n",
6048− frame + 1, depth);
6049− return -EACCES;
6176+
6177+ subprog_depth = round_up_stack_depth(env, subprog[idx].stack_depth);
6178+ if (priv_stack_supported) {
6179+ /* Request private stack support only if the subprog stack
6180+ * depth is no less than BPF_PRIV_STACK_MIN_SIZE. This is to
6181+ * avoid jit penalty if the stack usage is small.
6182+ */
6183+ if (subprog[idx].priv_stack_mode == PRIV_STACK_UNKNOWN &&
6184+ subprog_depth >= BPF_PRIV_STACK_MIN_SIZE)
6185+ subprog[idx].priv_stack_mode = PRIV_STACK_ADAPTIVE;
6186+ }
6187+
6188+ if (subprog[idx].priv_stack_mode == PRIV_STACK_ADAPTIVE) {
6189+ if (subprog_depth > MAX_BPF_STACK) {
6190+ verbose(env, "stack size of subprog %d is %d. Too large\n",
6191+ idx, subprog_depth);
6192+ return -EACCES;
6193+ }
6194+ } else {
6195+ depth += subprog_depth;
6196+ if (depth > MAX_BPF_STACK) {
6197+ verbose(env, "combined stack size of %d calls is %d. Too large\n",
6198+ frame + 1, depth);
6199+ return -EACCES;
6200+ }
60506201 }
60516202 continue_func:
60526203 subprog_end = subprog[idx + 1].start;
@@ -6103,6 +6254,8 @@ continue_func: · 50 unmodified lines
61036254 }
61046255 i = next_insn;
61056256 idx = sidx;
6257+ if (!priv_stack_supported)
6258+ subprog[idx].priv_stack_mode = NO_PRIV_STACK;
61066259
61076260 if (subprog[idx].has_tail_call)
61086261 tail_call_reachable = true;
@@ -6136,7 +6289,8 @@ continue_func: · 27 unmodified lines
61366289 */
61376290 if (frame == 0)
61386291 return 0;
6139− depth -= round_up_stack_depth(env, subprog[idx].stack_depth);
6292+ if (subprog[idx].priv_stack_mode != PRIV_STACK_ADAPTIVE)
6293+ depth -= round_up_stack_depth(env, subprog[idx].stack_depth);
61406294 frame--;
61416295 i = ret_insn[frame];
61426296 idx = ret_prog[frame];
@@ -6145,17 +6299,45 @@ continue_func: · 2 unmodified lines
61456299
61466300 static int check_max_stack_depth(struct bpf_verifier_env *env)
61476301 {
6302+ enum priv_stack_mode priv_stack_mode = PRIV_STACK_UNKNOWN;
61486303 struct bpf_subprog_info *si = env->subprog_info;
6304+ bool priv_stack_supported;
61496305 int ret;
61506306
61516307 for (int i = 0; i < env->subprog_cnt; i++) {
6308+ if (si[i].has_tail_call) {
6309+ priv_stack_mode = NO_PRIV_STACK;
6310+ break;
6311+ }
6312+ }
6313+
6314+ if (priv_stack_mode == PRIV_STACK_UNKNOWN)
6315+ priv_stack_mode = bpf_enable_priv_stack(env->prog);
6316+
6317+ /* All async_cb subprogs use normal kernel stack. If a particular
6318+ * subprog appears in both main prog and async_cb subtree, that
6319+ * subprog will use normal kernel stack to avoid potential nesting.
6320+ * The reverse subprog traversal ensures when main prog subtree is
6321+ * checked, the subprogs appearing in async_cb subtrees are already
6322+ * marked as using normal kernel stack, so stack size checking can
6323+ * be done properly.
6324+ */
6325+ for (int i = env->subprog_cnt - 1; i >= 0; i--) {
61526326 if (!i || si[i].is_async_cb) {
6153− ret = check_max_stack_depth_subprog(env, i);
6327+ priv_stack_supported = !i && priv_stack_mode == PRIV_STACK_ADAPTIVE;
6328+ ret = check_max_stack_depth_subprog(env, i, priv_stack_supported);
61546329 if (ret < 0)
61556330 return ret;
61566331 }
6157− continue;
61586332 }
6333+
6334+ for (int i = 0; i < env->subprog_cnt; i++) {
6335+ if (si[i].priv_stack_mode == PRIV_STACK_ADAPTIVE) {
6336+ env->prog->aux->jits_use_priv_stack = true;
6337+ break;
6338+ }
6339+ }
6340+
61596341 return 0;
61606342 }
61616343
@@ -6595,6 +6777,7 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, · 433 unmodified lines
65956777 const char *field_name = NULL;
65966778 enum bpf_type_flag flag = 0;
65976779 u32 btf_id = 0;
6780+ bool mask;
65986781 int ret;
65996782
66006783 if (!env->allow_ptr_leaks) {
@@ -6666,7 +6849,21 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, · 65 unmodified lines
66666849
66676850 if (ret < 0)
66686851 return ret;
6669−
6852+ /* For raw_tp progs, we allow dereference of PTR_MAYBE_NULL
6853+ * trusted PTR_TO_BTF_ID, these are the ones that are possibly
6854+ * arguments to the raw_tp. Since internal checks in for trusted
6855+ * reg in check_ptr_to_btf_access would consider PTR_MAYBE_NULL
6856+ * modifier as problematic, mask it out temporarily for the
6857+ * check. Don't apply this to pointers with ref_obj_id > 0, as
6858+ * those won't be raw_tp args.
6859+ *
6860+ * We may end up applying this relaxation to other trusted
6861+ * PTR_TO_BTF_ID with maybe null flag, since we cannot
6862+ * distinguish PTR_MAYBE_NULL tagged for arguments vs normal
6863+ * tagging, but that should expand allowed behavior, and not
6864+ * cause regression for existing behavior.
6865+ */
6866+ mask = mask_raw_tp_reg(env, reg);
66706867 if (ret != PTR_TO_BTF_ID) {
66716868 /* just mark; */
66726869
@@ -6727,8 +6924,13 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, · 54 unmodified lines
67276924 clear_trusted_flags(&flag);
67286925 }
67296926
6730− if (atype == BPF_READ && value_regno >= 0)
6927+ if (atype == BPF_READ && value_regno >= 0) {
67316928 mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag);
6929+ /* We've assigned a new type to regno, so don't undo masking. */
6930+ if (regno == value_regno)
6931+ mask = false;
6932+ }
6933+ unmask_raw_tp_reg(reg, mask);
67326934
67336935 return 0;
67346936 }
@@ -6949,7 +7151,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 214 unmodified lines
69497151 return err;
69507152 if (tnum_is_const(reg->var_off))
69517153 kptr_field = btf_record_find(reg->map_ptr->record,
6952− off + reg->var_off.value, BPF_KPTR);
7154+ off + reg->var_off.value, BPF_KPTR | BPF_UPTR);
69537155 if (kptr_field) {
69547156 err = check_map_kptr_access(env, regno, value_regno, insn_idx, kptr_field);
69557157 } else if (t == BPF_READ && value_regno >= 0) {
@@ -7103,7 +7305,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 147 unmodified lines
71037305 if (!err && t == BPF_READ && value_regno >= 0)
71047306 mark_reg_unknown(env, regs, value_regno);
71057307 } else if (base_type(reg->type) == PTR_TO_BTF_ID &&
7106− !type_may_be_null(reg->type)) {
7308+ (mask_raw_tp_reg_cond(env, reg) || !type_may_be_null(reg->type))) {
71077309 err = check_ptr_to_btf_access(env, regs, regno, off, size, t,
71087310 value_regno);
71097311 } else if (reg->type == CONST_PTR_TO_MAP) {
@@ -7648,19 +7850,20 @@ static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg · 538 unmodified lines
76487850 * Since only one bpf_spin_lock is allowed the checks are simpler than
76497851 * reg_is_refcounted() logic. The verifier needs to remember only
76507852 * one spin_lock instead of array of acquired_refs.
7651− * cur_state->active_lock remembers which map value element or allocated
7853+ * cur_func(env)->active_locks remembers which map value element or allocated
76527854 * object got locked and clears it after bpf_spin_unlock.
76537855 */
76547856 static int process_spin_lock(struct bpf_verifier_env *env, int regno,
76557857 bool is_lock)
76567858 {
76577859 struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
7658− struct bpf_verifier_state *cur = env->cur_state;
76597860 bool is_const = tnum_is_const(reg->var_off);
7861+ struct bpf_func_state *cur = cur_func(env);
76607862 u64 val = reg->var_off.value;
76617863 struct bpf_map *map = NULL;
76627864 struct btf *btf = NULL;
76637865 struct btf_record *rec;
7866+ int err;
76647867
76657868 if (!is_const) {
76667869 verbose(env,
@@ -7692,16 +7895,23 @@ static int process_spin_lock(struct bpf_verifier_env *env, int regno, · 25 unmodified lines
76927895 return -EINVAL;
76937896 }
76947897 if (is_lock) {
7695− if (cur->active_lock.ptr) {
7898+ void *ptr;
7899+
7900+ if (map)
7901+ ptr = map;
7902+ else
7903+ ptr = btf;
7904+
7905+ if (cur->active_locks) {
76967906 verbose(env,
76977907 "Locking two bpf_spin_locks are not allowed\n");
76987908 return -EINVAL;
76997909 }
7700− if (map)
7701− cur->active_lock.ptr = map;
7702− else
7703− cur->active_lock.ptr = btf;
7704− cur->active_lock.id = reg->id;
7910+ err = acquire_lock_state(env, env->insn_idx, REF_TYPE_LOCK, reg->id, ptr);
7911+ if (err < 0) {
7912+ verbose(env, "Failed to acquire lock state\n");
7913+ return err;
7914+ }
77057915 } else {
77067916 void *ptr;
77077917
@@ -7710,20 +7920,17 @@ static int process_spin_lock(struct bpf_verifier_env *env, int regno, · 2 unmodified lines
77107920 else
77117921 ptr = btf;
77127922
7713− if (!cur->active_lock.ptr) {
7923+ if (!cur->active_locks) {
77147924 verbose(env, "bpf_spin_unlock without taking a lock\n");
77157925 return -EINVAL;
77167926 }
7717− if (cur->active_lock.ptr != ptr ||
7718− cur->active_lock.id != reg->id) {
7927+
7928+ if (release_lock_state(cur_func(env), REF_TYPE_LOCK, reg->id, ptr)) {
77197929 verbose(env, "bpf_spin_unlock of different lock\n");
77207930 return -EINVAL;
77217931 }
77227932
77237933 invalidate_non_owning_refs(env);
7724−
7725− cur->active_lock.ptr = NULL;
7726− cur->active_lock.id = 0;
77277934 }
77287935 return 0;
77297936 }
@@ -8796,6 +9003,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, · 1066 unmodified lines
87969003 enum bpf_reg_type type = reg->type;
87979004 u32 *arg_btf_id = NULL;
87989005 int err = 0;
9006+ bool mask;
87999007
88009008 if (arg_type == ARG_DONTCARE)
88019009 return 0;
@@ -8836,11 +9044,11 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, · 34 unmodified lines
88369044 base_type(arg_type) == ARG_PTR_TO_SPIN_LOCK)
88379045 arg_btf_id = fn->arg_btf_id[arg];
88389046
9047+ mask = mask_raw_tp_reg(env, reg);
88399048 err = check_reg_type(env, regno, arg_type, arg_btf_id, meta);
8840− if (err)
8841− return err;
88429049
8843− err = check_func_arg_reg_off(env, reg, regno, arg_type);
9050+ err = err ?: check_func_arg_reg_off(env, reg, regno, arg_type);
9051+ unmask_raw_tp_reg(reg, mask);
88449052 if (err)
88459053 return err;
88469054
@@ -9635,14 +9843,17 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, · 788 unmodified lines
96359843 return ret;
96369844 } else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) {
96379845 struct bpf_call_arg_meta meta;
9846+ bool mask;
96389847 int err;
96399848
96409849 if (register_is_null(reg) && type_may_be_null(arg->arg_type))
96419850 continue;
96429851
96439852 memset(&meta, 0, sizeof(meta)); /* leave func_id as zero */
9853+ mask = mask_raw_tp_reg(env, reg);
96449854 err = check_reg_type(env, regno, arg->arg_type, &arg->btf_id, &meta);
96459855 err = err ?: check_func_arg_reg_off(env, reg, regno, arg->arg_type);
9856+ unmask_raw_tp_reg(reg, mask);
96469857 if (err)
96479858 return err;
96489859 } else {
@@ -9781,7 +9992,7 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · 132 unmodified lines
97819992 const char *sub_name = subprog_name(env, subprog);
97829993
97839994 /* Only global subprogs cannot be called with a lock held. */
9784− if (env->cur_state->active_lock.ptr) {
9995+ if (cur_func(env)->active_locks) {
97859996 verbose(env, "global function calls are not allowed while holding a lock,\n"
97869997 "use static function instead\n");
97879998 return -EINVAL;
@@ -9910,7 +10121,7 @@ static int set_loop_callback_state(struct bpf_verifier_env *env, · 122 unmodified lines
991010121 {
991110122 /* bpf_loop(u32 nr_loops, void *callback_fn, void *callback_ctx,
991210123 * u64 flags);
9913− * callback_fn(u32 index, void *callback_ctx);
10124+ * callback_fn(u64 index, void *callback_ctx);
991410125 */
991510126 callee->regs[BPF_REG_1].type = SCALAR_VALUE;
991610127 callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3];
@@ -10122,17 +10333,10 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) · 205 unmodified lines
1012210333 caller->regs[BPF_REG_0] = *r0;
1012310334 }
1012410335
10125− /* callback_fn frame should have released its own additions to parent's
10126− * reference state at this point, or check_reference_leak would
10127− * complain, hence it must be the same as the caller. There is no need
10128− * to copy it back.
10129− */
10130− if (!callee->in_callback_fn) {
10131− /* Transfer references to the caller */
10132− err = copy_reference_state(caller, callee);
10133− if (err)
10134− return err;
10135− }
10336+ /* Transfer references to the caller */
10337+ err = copy_reference_state(caller, callee);
10338+ if (err)
10339+ return err;
1013610340
1013710341 /* for callbacks like bpf_loop or bpf_for_each_map_elem go back to callsite,
1013810342 * there function call logic would reschedule callback visit. If iteration
@@ -10302,11 +10506,11 @@ static int check_reference_leak(struct bpf_verifier_env *env, bool exception_exi · 163 unmodified lines
1030210506 bool refs_lingering = false;
1030310507 int i;
1030410508
10305− if (!exception_exit && state->frameno && !state->in_callback_fn)
10509+ if (!exception_exit && state->frameno)
1030610510 return 0;
1030710511
1030810512 for (i = 0; i < state->acquired_refs; i++) {
10309− if (!exception_exit && state->in_callback_fn && state->refs[i].callback_ref != state->frameno)
10513+ if (state->refs[i].type != REF_TYPE_PTR)
1031010514 continue;
1031110515 verbose(env, "Unreleased reference id=%d alloc_insn=%d\n",
1031210516 state->refs[i].id, state->refs[i].insn_idx);
@@ -10315,6 +10519,34 @@ static int check_reference_leak(struct bpf_verifier_env *env, bool exception_exi · 2 unmodified lines
1031510519 return refs_lingering ? -EINVAL : 0;
1031610520 }
1031710521
10522+static int check_resource_leak(struct bpf_verifier_env *env, bool exception_exit, bool check_lock, const char *prefix)
10523+{
10524+ int err;
10525+
10526+ if (check_lock && cur_func(env)->active_locks) {
10527+ verbose(env, "%s cannot be used inside bpf_spin_lock-ed region\n", prefix);
10528+ return -EINVAL;
10529+ }
10530+
10531+ err = check_reference_leak(env, exception_exit);
10532+ if (err) {
10533+ verbose(env, "%s would lead to reference leak\n", prefix);
10534+ return err;
10535+ }
10536+
10537+ if (check_lock && env->cur_state->active_rcu_lock) {
10538+ verbose(env, "%s cannot be used inside bpf_rcu_read_lock-ed region\n", prefix);
10539+ return -EINVAL;
10540+ }
10541+
10542+ if (check_lock && env->cur_state->active_preempt_lock) {
10543+ verbose(env, "%s cannot be used inside bpf_preempt_disable-ed region\n", prefix);
10544+ return -EINVAL;
10545+ }
10546+
10547+ return 0;
10548+}
10549+
1031810550 static int check_bpf_snprintf_call(struct bpf_verifier_env *env,
1031910551 struct bpf_reg_state *regs)
1032010552 {
@@ -10583,11 +10815,9 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn · 262 unmodified lines
1058310815
1058410816 switch (func_id) {
1058510817 case BPF_FUNC_tail_call:
10586− err = check_reference_leak(env, false);
10587− if (err) {
10588− verbose(env, "tail_call would lead to reference leak\n");
10818+ err = check_resource_leak(env, false, true, "tail_call");
10819+ if (err)
1058910820 return err;
10590− }
1059110821 break;
1059210822 case BPF_FUNC_get_local_storage:
1059310823 /* check that flags argument in get_local_storage(map, flags) is 0,
@@ -11252,6 +11482,7 @@ enum special_kfunc_type { · 658 unmodified lines
1125211482 KF_bpf_preempt_enable,
1125311483 KF_bpf_iter_css_task_new,
1125411484 KF_bpf_session_cookie,
11485+ KF_bpf_get_kmem_cache,
1125511486 };
1125611487
1125711488 BTF_SET_START(special_kfunc_set)
@@ -11317,6 +11548,7 @@ BTF_ID(func, bpf_session_cookie) · 59 unmodified lines
1131711548 #else
1131811549 BTF_ID_UNUSED
1131911550 #endif
11551+BTF_ID(func, bpf_get_kmem_cache)
1132011552
1132111553 static bool is_kfunc_ret_null(struct bpf_kfunc_call_arg_meta *meta)
1132211554 {
@@ -11512,10 +11744,9 @@ static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, · 189 unmodified lines
1151211744
1151311745 static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
1151411746 {
11515− struct bpf_verifier_state *state = env->cur_state;
1151611747 struct btf_record *rec = reg_btf_record(reg);
1151711748
11518− if (!state->active_lock.ptr) {
11749+ if (!cur_func(env)->active_locks) {
1151911750 verbose(env, "verifier internal error: ref_set_non_owning w/o active lock\n");
1152011751 return -EFAULT;
1152111752 }
@@ -11612,6 +11843,7 @@ static int ref_convert_owning_non_owning(struct bpf_verifier_env *env, u32 ref_o · 90 unmodified lines
1161211843 */
1161311844 static int check_reg_allocation_locked(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
1161411845 {
11846+ struct bpf_reference_state *s;
1161511847 void *ptr;
1161611848 u32 id;
1161711849
@@ -11628,10 +11860,10 @@ static int check_reg_allocation_locked(struct bpf_verifier_env *env, struct bpf_ · 10 unmodified lines
1162811860 }
1162911861 id = reg->id;
1163011862
11631− if (!env->cur_state->active_lock.ptr)
11863+ if (!cur_func(env)->active_locks)
1163211864 return -EINVAL;
11633− if (env->cur_state->active_lock.ptr != ptr ||
11634− env->cur_state->active_lock.id != id) {
11865+ s = find_lock_state(env, REF_TYPE_LOCK, id, ptr);
11866+ if (!s) {
1163511867 verbose(env, "held lock and object are not in the same allocation\n");
1163611868 return -EINVAL;
1163711869 }
@@ -11942,6 +12174,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ · 304 unmodified lines
1194212174 enum bpf_arg_type arg_type = ARG_DONTCARE;
1194312175 u32 regno = i + 1, ref_id, type_size;
1194412176 bool is_ret_buf_sz = false;
12177+ bool mask = false;
1194512178 int kf_arg_type;
1194612179
1194712180 t = btf_type_skip_modifiers(btf, args[i].type, NULL);
@@ -12000,12 +12233,15 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ · 52 unmodified lines
1200012233 return -EINVAL;
1200112234 }
1200212235
12236+ mask = mask_raw_tp_reg(env, reg);
1200312237 if ((is_kfunc_trusted_args(meta) || is_kfunc_rcu(meta)) &&
1200412238 (register_is_null(reg) || type_may_be_null(reg->type)) &&
1200512239 !is_kfunc_arg_nullable(meta->btf, &args[i])) {
1200612240 verbose(env, "Possibly NULL pointer passed to trusted arg%d\n", i);
12241+ unmask_raw_tp_reg(reg, mask);
1200712242 return -EACCES;
1200812243 }
12244+ unmask_raw_tp_reg(reg, mask);
1200912245
1201012246 if (reg->ref_obj_id) {
1201112247 if (is_kfunc_release(meta) && meta->ref_obj_id) {
@@ -12063,16 +12299,24 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ · 51 unmodified lines
1206312299 if (!is_kfunc_trusted_args(meta) && !is_kfunc_rcu(meta))
1206412300 break;
1206512301
12302+ /* Allow passing maybe NULL raw_tp arguments to
12303+ * kfuncs for compatibility. Don't apply this to
12304+ * arguments with ref_obj_id > 0.
12305+ */
12306+ mask = mask_raw_tp_reg(env, reg);
1206612307 if (!is_trusted_reg(reg)) {
1206712308 if (!is_kfunc_rcu(meta)) {
1206812309 verbose(env, "R%d must be referenced or trusted\n", regno);
12310+ unmask_raw_tp_reg(reg, mask);
1206912311 return -EINVAL;
1207012312 }
1207112313 if (!is_rcu_reg(reg)) {
1207212314 verbose(env, "R%d must be a rcu pointer\n", regno);
12315+ unmask_raw_tp_reg(reg, mask);
1207312316 return -EINVAL;
1207412317 }
1207512318 }
12319+ unmask_raw_tp_reg(reg, mask);
1207612320 fallthrough;
1207712321 case KF_ARG_PTR_TO_CTX:
1207812322 case KF_ARG_PTR_TO_DYNPTR:
@@ -12095,7 +12339,9 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ · 16 unmodified lines
1209512339
1209612340 if (is_kfunc_release(meta) && reg->ref_obj_id)
1209712341 arg_type |= OBJ_RELEASE;
12342+ mask = mask_raw_tp_reg(env, reg);
1209812343 ret = check_func_arg_reg_off(env, reg, regno, arg_type);
12344+ unmask_raw_tp_reg(reg, mask);
1209912345 if (ret < 0)
1210012346 return ret;
1210112347
@@ -12272,6 +12518,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ · 170 unmodified lines
1227212518 ref_tname = btf_name_by_offset(btf, ref_t->name_off);
1227312519 fallthrough;
1227412520 case KF_ARG_PTR_TO_BTF_ID:
12521+ mask = mask_raw_tp_reg(env, reg);
1227512522 /* Only base_type is checked, further checks are done here */
1227612523 if ((base_type(reg->type) != PTR_TO_BTF_ID ||
1227712524 (bpf_type_has_unsafe_modifiers(reg->type) && !is_rcu_reg(reg))) &&
@@ -12280,9 +12527,11 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ · 2 unmodified lines
1228012527 verbose(env, "expected %s or socket\n",
1228112528 reg_type_str(env, base_type(reg->type) |
1228212529 (type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS)));
12530+ unmask_raw_tp_reg(reg, mask);
1228312531 return -EINVAL;
1228412532 }
1228512533 ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i);
12534+ unmask_raw_tp_reg(reg, mask);
1228612535 if (ret < 0)
1228712536 return ret;
1228812537 break;
@@ -12827,6 +13076,9 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · 538 unmodified lines
1282713076 regs[BPF_REG_0].type = PTR_TO_BTF_ID;
1282813077 regs[BPF_REG_0].btf_id = ptr_type_id;
1282913078
13079+ if (meta.func_id == special_kfunc_list[KF_bpf_get_kmem_cache])
13080+ regs[BPF_REG_0].type |= PTR_UNTRUSTED;
13081+
1283013082 if (is_iter_next_kfunc(&meta)) {
1283113083 struct bpf_reg_state *cur_iter;
1283213084
@@ -13252,7 +13504,7 @@ static int sanitize_check_bounds(struct bpf_verifier_env *env, · 419 unmodified lines
1325213504 */
1325313505 static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
1325413506 struct bpf_insn *insn,
13255− const struct bpf_reg_state *ptr_reg,
13507+ struct bpf_reg_state *ptr_reg,
1325613508 const struct bpf_reg_state *off_reg)
1325713509 {
1325813510 struct bpf_verifier_state *vstate = env->cur_state;
@@ -13266,6 +13518,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, · 7 unmodified lines
1326613518 struct bpf_sanitize_info info = {};
1326713519 u8 opcode = BPF_OP(insn->code);
1326813520 u32 dst = insn->dst_reg;
13521+ bool mask;
1326913522 int ret;
1327013523
1327113524 dst_reg = &regs[dst];
@@ -13292,11 +13545,14 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, · 20 unmodified lines
1329213545 return -EACCES;
1329313546 }
1329413547
13548+ mask = mask_raw_tp_reg(env, ptr_reg);
1329513549 if (ptr_reg->type & PTR_MAYBE_NULL) {
1329613550 verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n",
1329713551 dst, reg_type_str(env, ptr_reg->type));
13552+ unmask_raw_tp_reg(ptr_reg, mask);
1329813553 return -EACCES;
1329913554 }
13555+ unmask_raw_tp_reg(ptr_reg, mask);
1330013556
1330113557 switch (base_type(ptr_reg->type)) {
1330213558 case PTR_TO_CTX:
@@ -15480,7 +15736,7 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, · 2177 unmodified lines
1548015736 if (dst_reg->type == SCALAR_VALUE && dst_reg->id)
1548115737 collect_linked_regs(this_branch, dst_reg->id, &linked_regs);
1548215738 if (linked_regs.cnt > 1) {
15483− err = push_jmp_history(env, this_branch, 0, linked_regs_pack(&linked_regs));
15739+ err = push_insn_history(env, this_branch, 0, linked_regs_pack(&linked_regs));
1548415740 if (err)
1548515741 return err;
1548615742 }
@@ -15744,26 +16000,9 @@ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) · 257 unmodified lines
1574416000 * gen_ld_abs() may terminate the program at runtime, leading to
1574516001 * reference leak.
1574616002 */
15747− err = check_reference_leak(env, false);
15748− if (err) {
15749− verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n");
16003+ err = check_resource_leak(env, false, true, "BPF_LD_[ABS|IND]");
16004+ if (err)
1575016005 return err;
15751− }
15752−
15753− if (env->cur_state->active_lock.ptr) {
15754− verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_spin_lock-ed region\n");
15755− return -EINVAL;
15756− }
15757−
15758− if (env->cur_state->active_rcu_lock) {
15759− verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_rcu_read_lock-ed region\n");
15760− return -EINVAL;
15761− }
15762−
15763− if (env->cur_state->active_preempt_lock) {
15764− verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_preempt_disable-ed region\n");
15765− return -EINVAL;
15766− }
1576716006
1576816007 if (regs[ctx_reg].type != PTR_TO_CTX) {
1576916008 verbose(env,
@@ -15909,6 +16148,16 @@ static int check_return_code(struct bpf_verifier_env *env, int regno, const char · 139 unmodified lines
1590916148 return -ENOTSUPP;
1591016149 }
1591116150 break;
16151+ case BPF_PROG_TYPE_KPROBE:
16152+ switch (env->prog->expected_attach_type) {
16153+ case BPF_TRACE_KPROBE_SESSION:
16154+ case BPF_TRACE_UPROBE_SESSION:
16155+ range = retval_range(0, 1);
16156+ break;
16157+ default:
16158+ return 0;
16159+ }
16160+ break;
1591216161 case BPF_PROG_TYPE_SK_LOOKUP:
1591316162 range = retval_range(SK_DROP, SK_PASS);
1591416163 break;
@@ -16175,10 +16424,7 @@ static u32 kfunc_fastcall_clobber_mask(struct bpf_kfunc_call_arg_meta *meta) · 260 unmodified lines
1617516424 /* Same as verifier_inlines_helper_call() but for kfuncs, see comment above */
1617616425 static bool is_fastcall_kfunc_call(struct bpf_kfunc_call_arg_meta *meta)
1617716426 {
16178− if (meta->btf == btf_vmlinux)
16179− return meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] ||
16180− meta->func_id == special_kfunc_list[KF_bpf_rdonly_cast];
16181− return false;
16427+ return meta->kfunc_flags & KF_FASTCALL;
1618216428 }
1618316429
1618416430 /* LLVM define a bpf_fastcall function attribute.
@@ -17513,8 +17759,20 @@ static bool refsafe(struct bpf_func_state *old, struct bpf_func_state *cur, · 1328 unmodified lines
1751317759 return false;
1751417760
1751517761 for (i = 0; i < old->acquired_refs; i++) {
17516− if (!check_ids(old->refs[i].id, cur->refs[i].id, idmap))
17762+ if (!check_ids(old->refs[i].id, cur->refs[i].id, idmap) ||
17763+ old->refs[i].type != cur->refs[i].type)
1751717764 return false;
17765+ switch (old->refs[i].type) {
17766+ case REF_TYPE_PTR:
17767+ break;
17768+ case REF_TYPE_LOCK:
17769+ if (old->refs[i].ptr != cur->refs[i].ptr)
17770+ return false;
17771+ break;
17772+ default:
17773+ WARN_ONCE(1, "Unhandled enum type for reference state: %d\n", old->refs[i].type);
17774+ return false;
17775+ }
1751817776 }
1751917777
1752017778 return true;
@@ -17592,19 +17850,6 @@ static bool states_equal(struct bpf_verifier_env *env, · 71 unmodified lines
1759217850 if (old->speculative && !cur->speculative)
1759317851 return false;
1759417852
17595− if (old->active_lock.ptr != cur->active_lock.ptr)
17596− return false;
17597−
17598− /* Old and cur active_lock's have to be either both present
17599− * or both absent.
17600− */
17601− if (!!old->active_lock.id != !!cur->active_lock.id)
17602− return false;
17603−
17604− if (old->active_lock.id &&
17605− !check_ids(old->active_lock.id, cur->active_lock.id, &env->idmap_scratch))
17606− return false;
17607−
1760817853 if (old->active_rcu_lock != cur->active_rcu_lock)
1760917854 return false;
1761017855
@@ -17880,7 +18125,7 @@ static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) · 269 unmodified lines
1788018125
1788118126 force_new_state = env->test_state_freq || is_force_checkpoint(env, insn_idx) ||
1788218127 /* Avoid accumulating infinitely long jmp history */
17883− cur->jmp_history_cnt > 40;
18128+ cur->insn_hist_end - cur->insn_hist_start > 40;
1788418129
1788518130 /* bpf progs typically have pruning point every 4 instructions
1788618131 * http://vger.kernel.org/bpfconf2019.html#session-1
@@ -18078,7 +18323,7 @@ hit: · 191 unmodified lines
1807818323 * the current state.
1807918324 */
1808018325 if (is_jmp_point(env, env->insn_idx))
18081− err = err ? : push_jmp_history(env, cur, 0, 0);
18326+ err = err ? : push_insn_history(env, cur, 0, 0);
1808218327 err = err ? : propagate_precision(env, &sl->state);
1808318328 if (err)
1808418329 return err;
@@ -18177,8 +18422,8 @@ next: · 92 unmodified lines
1817718422
1817818423 cur->parent = new;
1817918424 cur->first_insn_idx = insn_idx;
18425+ cur->insn_hist_start = cur->insn_hist_end;
1818018426 cur->dfs_depth = new->dfs_depth + 1;
18181− clear_jmp_history(cur);
1818218427 new_sl->next = *explored_state(env, insn_idx);
1818318428 *explored_state(env, insn_idx) = new_sl;
1818418429 /* connect new state to parentage chain. Current frame needs all
@@ -18346,7 +18591,7 @@ static int do_check(struct bpf_verifier_env *env) · 161 unmodified lines
1834618591 }
1834718592
1834818593 if (is_jmp_point(env, env->insn_idx)) {
18349− err = push_jmp_history(env, state, 0, 0);
18594+ err = push_insn_history(env, state, 0, 0);
1835018595 if (err)
1835118596 return err;
1835218597 }
@@ -18506,7 +18751,7 @@ static int do_check(struct bpf_verifier_env *env) · 153 unmodified lines
1850618751 return -EINVAL;
1850718752 }
1850818753
18509− if (env->cur_state->active_lock.ptr) {
18754+ if (cur_func(env)->active_locks) {
1851018755 if ((insn->src_reg == BPF_REG_0 && insn->imm != BPF_FUNC_spin_unlock) ||
1851118756 (insn->src_reg == BPF_PSEUDO_KFUNC_CALL &&
1851218757 (insn->off != 0 || !is_bpf_graph_api_kfunc(insn->imm)))) {
@@ -18555,30 +18800,14 @@ static int do_check(struct bpf_verifier_env *env) · 42 unmodified lines
1855518800 return -EINVAL;
1855618801 }
1855718802 process_bpf_exit_full:
18558− if (env->cur_state->active_lock.ptr && !env->cur_state->curframe) {
18559− verbose(env, "bpf_spin_unlock is missing\n");
18560− return -EINVAL;
18561− }
18562−
18563− if (env->cur_state->active_rcu_lock && !env->cur_state->curframe) {
18564− verbose(env, "bpf_rcu_read_unlock is missing\n");
18565− return -EINVAL;
18566− }
18567−
18568− if (env->cur_state->active_preempt_lock && !env->cur_state->curframe) {
18569− verbose(env, "%d bpf_preempt_enable%s missing\n",
18570− env->cur_state->active_preempt_lock,
18571− env->cur_state->active_preempt_lock == 1 ? " is" : "(s) are");
18572− return -EINVAL;
18573− }
18574−
1857518803 /* We must do check_reference_leak here before
1857618804 * prepare_func_exit to handle the case when
1857718805 * state->curframe > 0, it may be a callback
1857818806 * function, for which reference_state must
1857918807 * match caller reference state when it exits.
1858018808 */
18581− err = check_reference_leak(env, exception_exit);
18809+ err = check_resource_leak(env, exception_exit, !env->cur_state->curframe,
18810+ "BPF_EXIT instruction");
1858218811 if (err)
1858318812 return err;
1858418813
@@ -19837,6 +20066,7 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env) · 1252 unmodified lines
1983720066 * for this case.
1983820067 */
1983920068 case PTR_TO_BTF_ID | MEM_ALLOC | PTR_UNTRUSTED:
20069+ case PTR_TO_BTF_ID | PTR_TRUSTED | PTR_MAYBE_NULL:
1984020070 if (type == BPF_READ) {
1984120071 if (BPF_MODE(insn->code) == BPF_MEM)
1984220072 insn->code = BPF_LDX | BPF_PROBE_MEM |
@@ -20041,6 +20271,9 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 198 unmodified lines
2004120271
2004220272 func[i]->aux->name[0] = 'F';
2004320273 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;
20274+ if (env->subprog_info[i].priv_stack_mode == PRIV_STACK_ADAPTIVE)
20275+ func[i]->aux->jits_use_priv_stack = true;
20276+
2004420277 func[i]->jit_requested = 1;
2004520278 func[i]->blinding_requested = prog->blinding_requested;
2004620279 func[i]->aux->kfunc_tab = prog->aux->kfunc_tab;
@@ -21809,6 +22042,11 @@ static int check_struct_ops_btf_id(struct bpf_verifier_env *env) · 1762 unmodified lines
2180922042 }
2181022043 }
2181122044
22045+ if (prog->aux->priv_stack_requested && !bpf_jit_supports_private_stack()) {
22046+ verbose(env, "Private stack not supported by jit\n");
22047+ return -EACCES;
22048+ }
22049+
2181222050 /* btf_ctx_access() used this to provide argument type info */
2181322051 prog->aux->ctx_arg_info =
2181422052 st_ops_desc->arg_info[member_idx].info;
@@ -22547,6 +22785,7 @@ err_unlock: · 732 unmodified lines
2254722785 if (!is_priv)
2254822786 mutex_unlock(&bpf_verifier_lock);
2254922787 vfree(env->insn_aux_data);
22788+ kvfree(env->insn_hist);
2255022789 err_free_env:
2255122790 kvfree(env);
2255222791 return ret;
kernel/trace/bpf_trace.c+86−24
@@ -802,6 +802,8 @@ struct send_signal_irq_work { · 801 unmodified lines
802802 struct task_struct *task;
803803 u32 sig;
804804 enum pid_type type;
805+ bool has_siginfo;
806+ struct kernel_siginfo info;
805807 };
806808
807809 static DEFINE_PER_CPU(struct send_signal_irq_work, send_signal_work);
@@ -809,27 +811,46 @@ static DEFINE_PER_CPU(struct send_signal_irq_work, send_signal_work); · 1 unmodified line
809811 static void do_bpf_send_signal(struct irq_work *entry)
810812 {
811813 struct send_signal_irq_work *work;
814+ struct kernel_siginfo *siginfo;
812815
813816 work = container_of(entry, struct send_signal_irq_work, irq_work);
814− group_send_sig_info(work->sig, SEND_SIG_PRIV, work->task, work->type);
817+ siginfo = work->has_siginfo ? &work->info : SEND_SIG_PRIV;
818+
819+ group_send_sig_info(work->sig, siginfo, work->task, work->type);
815820 put_task_struct(work->task);
816821 }
817822
818−static int bpf_send_signal_common(u32 sig, enum pid_type type)
823+static int bpf_send_signal_common(u32 sig, enum pid_type type, struct task_struct *task, u64 value)
819824 {
820825 struct send_signal_irq_work *work = NULL;
826+ struct kernel_siginfo info;
827+ struct kernel_siginfo *siginfo;
828+
829+ if (!task) {
830+ task = current;
831+ siginfo = SEND_SIG_PRIV;
832+ } else {
833+ clear_siginfo(&info);
834+ info.si_signo = sig;
835+ info.si_errno = 0;
836+ info.si_code = SI_KERNEL;
837+ info.si_pid = 0;
838+ info.si_uid = 0;
839+ info.si_value.sival_ptr = (void *)(unsigned long)value;
840+ siginfo = &info;
841+ }
821842
822843 /* Similar to bpf_probe_write_user, task needs to be
823844 * in a sound condition and kernel memory access be
824845 * permitted in order to send signal to the current
825846 * task.
826847 */
827− if (unlikely(current->flags & (PF_KTHREAD | PF_EXITING)))
848+ if (unlikely(task->flags & (PF_KTHREAD | PF_EXITING)))
828849 return -EPERM;
829850 if (unlikely(!nmi_uaccess_okay()))
830851 return -EPERM;
831852 /* Task should not be pid=1 to avoid kernel panic. */
832− if (unlikely(is_global_init(current)))
853+ if (unlikely(is_global_init(task)))
833854 return -EPERM;
834855
835856 if (irqs_disabled()) {
@@ -847,19 +868,22 @@ static int bpf_send_signal_common(u32 sig, enum pid_type type) · 11 unmodified lines
847868 * to the irq_work. The current task may change when queued
848869 * irq works get executed.
849870 */
850− work->task = get_task_struct(current);
871+ work->task = get_task_struct(task);
872+ work->has_siginfo = siginfo == &info;
873+ if (work->has_siginfo)
874+ copy_siginfo(&work->info, &info);
851875 work->sig = sig;
852876 work->type = type;
853877 irq_work_queue(&work->irq_work);
854878 return 0;
855879 }
856880
857− return group_send_sig_info(sig, SEND_SIG_PRIV, current, type);
881+ return group_send_sig_info(sig, siginfo, task, type);
858882 }
859883
860884 BPF_CALL_1(bpf_send_signal, u32, sig)
861885 {
862− return bpf_send_signal_common(sig, PIDTYPE_TGID);
886+ return bpf_send_signal_common(sig, PIDTYPE_TGID, NULL, 0);
863887 }
864888
865889 static const struct bpf_func_proto bpf_send_signal_proto = {
@@ -871,7 +895,7 @@ static const struct bpf_func_proto bpf_send_signal_proto = { · 5 unmodified lines
871895
872896 BPF_CALL_1(bpf_send_signal_thread, u32, sig)
873897 {
874− return bpf_send_signal_common(sig, PIDTYPE_PID);
898+ return bpf_send_signal_common(sig, PIDTYPE_PID, NULL, 0);
875899 }
876900
877901 static const struct bpf_func_proto bpf_send_signal_thread_proto = {
@@ -1557,6 +1581,17 @@ static inline bool is_kprobe_session(const struct bpf_prog *prog) · 679 unmodified lines
15571581 return prog->expected_attach_type == BPF_TRACE_KPROBE_SESSION;
15581582 }
15591583
1584+static inline bool is_uprobe_multi(const struct bpf_prog *prog)
1585+{
1586+ return prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI ||
1587+ prog->expected_attach_type == BPF_TRACE_UPROBE_SESSION;
1588+}
1589+
1590+static inline bool is_uprobe_session(const struct bpf_prog *prog)
1591+{
1592+ return prog->expected_attach_type == BPF_TRACE_UPROBE_SESSION;
1593+}
1594+
15601595 static const struct bpf_func_proto *
15611596 kprobe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
15621597 {
@@ -1574,13 +1609,13 @@ kprobe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 11 unmodified lines
15741609 case BPF_FUNC_get_func_ip:
15751610 if (is_kprobe_multi(prog))
15761611 return &bpf_get_func_ip_proto_kprobe_multi;
1577− if (prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI)
1612+ if (is_uprobe_multi(prog))
15781613 return &bpf_get_func_ip_proto_uprobe_multi;
15791614 return &bpf_get_func_ip_proto_kprobe;
15801615 case BPF_FUNC_get_attach_cookie:
15811616 if (is_kprobe_multi(prog))
15821617 return &bpf_get_attach_cookie_proto_kmulti;
1583− if (prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI)
1618+ if (is_uprobe_multi(prog))
15841619 return &bpf_get_attach_cookie_proto_umulti;
15851620 return &bpf_get_attach_cookie_proto_trace;
15861621 default:
@@ -3072,6 +3107,7 @@ struct bpf_uprobe { · 1485 unmodified lines
30723107 u64 cookie;
30733108 struct uprobe *uprobe;
30743109 struct uprobe_consumer consumer;
3110+ bool session;
30753111 };
30763112
30773113 struct bpf_uprobe_multi_link {
@@ -3084,7 +3120,7 @@ struct bpf_uprobe_multi_link { · 6 unmodified lines
30843120 };
30853121
30863122 struct bpf_uprobe_multi_run_ctx {
3087− struct bpf_run_ctx run_ctx;
3123+ struct bpf_session_run_ctx session_ctx;
30883124 unsigned long entry_ip;
30893125 struct bpf_uprobe *uprobe;
30903126 };
@@ -3195,17 +3231,22 @@ static const struct bpf_link_ops bpf_uprobe_multi_link_lops = { · 104 unmodified lines
31953231
31963232 static int uprobe_prog_run(struct bpf_uprobe *uprobe,
31973233 unsigned long entry_ip,
3198− struct pt_regs *regs)
3234+ struct pt_regs *regs,
3235+ bool is_return, void *data)
31993236 {
32003237 struct bpf_uprobe_multi_link *link = uprobe->link;
32013238 struct bpf_uprobe_multi_run_ctx run_ctx = {
3239+ .session_ctx = {
3240+ .is_return = is_return,
3241+ .data = data,
3242+ },
32023243 .entry_ip = entry_ip,
32033244 .uprobe = uprobe,
32043245 };
32053246 struct bpf_prog *prog = link->link.prog;
32063247 bool sleepable = prog->sleepable;
32073248 struct bpf_run_ctx *old_run_ctx;
3208− int err = 0;
3249+ int err;
32093250
32103251 if (link->task && !same_thread_group(current, link->task))
32113252 return 0;
@@ -3217,7 +3258,7 @@ static int uprobe_prog_run(struct bpf_uprobe *uprobe, · 5 unmodified lines
32173258
32183259 migrate_disable();
32193260
3220− old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
3261+ old_run_ctx = bpf_set_run_ctx(&run_ctx.session_ctx.run_ctx);
32213262 err = bpf_prog_run(link->link.prog, regs);
32223263 bpf_reset_run_ctx(old_run_ctx);
32233264
@@ -3244,9 +3285,13 @@ uprobe_multi_link_handler(struct uprobe_consumer *con, struct pt_regs *regs, · 20 unmodified lines
32443285 __u64 *data)
32453286 {
32463287 struct bpf_uprobe *uprobe;
3288+ int ret;
32473289
32483290 uprobe = container_of(con, struct bpf_uprobe, consumer);
3249− return uprobe_prog_run(uprobe, instruction_pointer(regs), regs);
3291+ ret = uprobe_prog_run(uprobe, instruction_pointer(regs), regs, false, data);
3292+ if (uprobe->session)
3293+ return ret ? UPROBE_HANDLER_IGNORE : 0;
3294+ return 0;
32503295 }
32513296
32523297 static int
@@ -3256,14 +3301,16 @@ uprobe_multi_link_ret_handler(struct uprobe_consumer *con, unsigned long func, s · 3 unmodified lines
32563301 struct bpf_uprobe *uprobe;
32573302
32583303 uprobe = container_of(con, struct bpf_uprobe, consumer);
3259− return uprobe_prog_run(uprobe, func, regs);
3304+ uprobe_prog_run(uprobe, func, regs, true, data);
3305+ return 0;
32603306 }
32613307
32623308 static u64 bpf_uprobe_multi_entry_ip(struct bpf_run_ctx *ctx)
32633309 {
32643310 struct bpf_uprobe_multi_run_ctx *run_ctx;
32653311
3266− run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx, run_ctx);
3312+ run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx,
3313+ session_ctx.run_ctx);
32673314 return run_ctx->entry_ip;
32683315 }
32693316
@@ -3271,7 +3318,8 @@ static u64 bpf_uprobe_multi_cookie(struct bpf_run_ctx *ctx) · 1 unmodified line
32713318 {
32723319 struct bpf_uprobe_multi_run_ctx *run_ctx;
32733320
3274− run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx, run_ctx);
3321+ run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx,
3322+ session_ctx.run_ctx);
32753323 return run_ctx->uprobe->cookie;
32763324 }
32773325
@@ -3295,7 +3343,7 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr · 17 unmodified lines
32953343 if (sizeof(u64) != sizeof(void *))
32963344 return -EOPNOTSUPP;
32973345
3298− if (prog->expected_attach_type != BPF_TRACE_UPROBE_MULTI)
3346+ if (!is_uprobe_multi(prog))
32993347 return -EINVAL;
33003348
33013349 flags = attr->link_create.uprobe_multi.flags;
@@ -3371,11 +3419,12 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr · 69 unmodified lines
33713419
33723420 uprobes[i].link = link;
33733421
3374− if (flags & BPF_F_UPROBE_MULTI_RETURN)
3375− uprobes[i].consumer.ret_handler = uprobe_multi_link_ret_handler;
3376− else
3422+ if (!(flags & BPF_F_UPROBE_MULTI_RETURN))
33773423 uprobes[i].consumer.handler = uprobe_multi_link_handler;
3378−
3424+ if (flags & BPF_F_UPROBE_MULTI_RETURN || is_uprobe_session(prog))
3425+ uprobes[i].consumer.ret_handler = uprobe_multi_link_ret_handler;
3426+ if (is_uprobe_session(prog))
3427+ uprobes[i].session = true;
33793428 if (pid)
33803429 uprobes[i].consumer.filter = uprobe_multi_link_filter;
33813430 }
@@ -3464,7 +3513,7 @@ static int bpf_kprobe_multi_filter(const struct bpf_prog *prog, u32 kfunc_id) · 82 unmodified lines
34643513 if (!btf_id_set8_contains(&kprobe_multi_kfunc_set_ids, kfunc_id))
34653514 return 0;
34663515
3467− if (!is_kprobe_session(prog))
3516+ if (!is_kprobe_session(prog) && !is_uprobe_session(prog))
34683517 return -EACCES;
34693518
34703519 return 0;
@@ -3482,3 +3531,16 @@ static int __init bpf_kprobe_multi_kfuncs_init(void) · 11 unmodified lines
34823531 }
34833532
34843533 late_initcall(bpf_kprobe_multi_kfuncs_init);
3534+
3535+__bpf_kfunc_start_defs();
3536+
3537+__bpf_kfunc int bpf_send_signal_task(struct task_struct *task, int sig, enum pid_type type,
3538+ u64 value)
3539+{
3540+ if (type != PIDTYPE_PID && type != PIDTYPE_TGID)
3541+ return -EINVAL;
3542+
3543+ return bpf_send_signal_common(sig, type, task, value);
3544+}
3545+
3546+__bpf_kfunc_end_defs();
mm/slab_common.c+19−0
@@ -1332,6 +1332,25 @@ size_t ksize(const void *objp) · 1331 unmodified lines
13321332 }
13331333 EXPORT_SYMBOL(ksize);
13341334
1335+#ifdef CONFIG_BPF_SYSCALL
1336+#include <linux/btf.h>
1337+
1338+__bpf_kfunc_start_defs();
1339+
1340+__bpf_kfunc struct kmem_cache *bpf_get_kmem_cache(u64 addr)
1341+{
1342+ struct slab *slab;
1343+
1344+ if (!virt_addr_valid((void *)(long)addr))
1345+ return NULL;
1346+
1347+ slab = virt_to_slab((void *)(long)addr);
1348+ return slab ? slab->slab_cache : NULL;
1349+}
1350+
1351+__bpf_kfunc_end_defs();
1352+#endif /* CONFIG_BPF_SYSCALL */
1353+
13351354 /* Tracepoints definitions. */
13361355 EXPORT_TRACEPOINT_SYMBOL(kmalloc);
13371356 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc);
net/core/bpf_sk_storage.c+3−3
@@ -106,7 +106,7 @@ static long bpf_fd_sk_storage_update_elem(struct bpf_map *map, void *key, · 105 unmodified lines
106106 if (sock) {
107107 sdata = bpf_local_storage_update(
108108 sock->sk, (struct bpf_local_storage_map *)map, value,
109− map_flags, GFP_ATOMIC);
109+ map_flags, false, GFP_ATOMIC);
110110 sockfd_put(sock);
111111 return PTR_ERR_OR_ZERO(sdata);
112112 }
@@ -137,7 +137,7 @@ bpf_sk_storage_clone_elem(struct sock *newsk, · 24 unmodified lines
137137 {
138138 struct bpf_local_storage_elem *copy_selem;
139139
140− copy_selem = bpf_selem_alloc(smap, newsk, NULL, true, GFP_ATOMIC);
140+ copy_selem = bpf_selem_alloc(smap, newsk, NULL, true, false, GFP_ATOMIC);
141141 if (!copy_selem)
142142 return NULL;
143143
@@ -243,7 +243,7 @@ BPF_CALL_5(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk, · 99 unmodified lines
243243 refcount_inc_not_zero(&sk->sk_refcnt)) {
244244 sdata = bpf_local_storage_update(
245245 sk, (struct bpf_local_storage_map *)map, value,
246− BPF_NOEXIST, gfp_flags);
246+ BPF_NOEXIST, false, gfp_flags);
247247 /* sk must be a fullsock (guaranteed by verifier),
248248 * so sock_gen_put() is unnecessary.
249249 */
samples/bpf/Makefile+0−25
@@ -17,20 +17,12 @@ tprogs-y += tracex3 · 16 unmodified lines
1717 tprogs-y += tracex4
1818 tprogs-y += tracex5
1919 tprogs-y += tracex6
20−tprogs-y += tracex7
21−tprogs-y += test_probe_write_user
2220 tprogs-y += trace_output
2321 tprogs-y += lathist
2422 tprogs-y += offwaketime
2523 tprogs-y += spintest
2624 tprogs-y += map_perf_test
27−tprogs-y += test_overhead
28−tprogs-y += test_cgrp2_array_pin
29−tprogs-y += test_cgrp2_attach
30−tprogs-y += test_cgrp2_sock
31−tprogs-y += test_cgrp2_sock2
3225 tprogs-y += xdp_router_ipv4
33−tprogs-y += test_current_task_under_cgroup
3426 tprogs-y += trace_event
3527 tprogs-y += sampleip
3628 tprogs-y += tc_l2_redirect
@@ -66,20 +58,12 @@ tracex3-objs := tracex3_user.o · 29 unmodified lines
6658 tracex4-objs := tracex4_user.o
6759 tracex5-objs := tracex5_user.o $(TRACE_HELPERS)
6860 tracex6-objs := tracex6_user.o
69−tracex7-objs := tracex7_user.o
70−test_probe_write_user-objs := test_probe_write_user_user.o
7161 trace_output-objs := trace_output_user.o
7262 lathist-objs := lathist_user.o
7363 offwaketime-objs := offwaketime_user.o $(TRACE_HELPERS)
7464 spintest-objs := spintest_user.o $(TRACE_HELPERS)
7565 map_perf_test-objs := map_perf_test_user.o
7666 test_overhead-objs := test_overhead_user.o
77−test_cgrp2_array_pin-objs := test_cgrp2_array_pin.o
78−test_cgrp2_attach-objs := test_cgrp2_attach.o
79−test_cgrp2_sock-objs := test_cgrp2_sock.o
80−test_cgrp2_sock2-objs := test_cgrp2_sock2.o
81−test_current_task_under_cgroup-objs := $(CGROUP_HELPERS) \
82− test_current_task_under_cgroup_user.o
8367 trace_event-objs := trace_event_user.o $(TRACE_HELPERS)
8468 sampleip-objs := sampleip_user.o $(TRACE_HELPERS)
8569 tc_l2_redirect-objs := tc_l2_redirect_user.o
@@ -107,9 +91,6 @@ always-y += tracex3.bpf.o · 21 unmodified lines
10791 always-y += tracex4.bpf.o
10892 always-y += tracex5.bpf.o
10993 always-y += tracex6.bpf.o
110−always-y += tracex7.bpf.o
111−always-y += sock_flags.bpf.o
112−always-y += test_probe_write_user.bpf.o
11394 always-y += trace_output.bpf.o
11495 always-y += tcbpf1_kern.o
11596 always-y += tc_l2_redirect_kern.o
@@ -117,12 +98,7 @@ always-y += lathist_kern.o · 1 unmodified line
11798 always-y += offwaketime.bpf.o
11899 always-y += spintest.bpf.o
119100 always-y += map_perf_test.bpf.o
120−always-y += test_overhead_tp.bpf.o
121−always-y += test_overhead_raw_tp.bpf.o
122−always-y += test_overhead_kprobe.bpf.o
123101 always-y += parse_varlen.o parse_simple.o parse_ldabs.o
124−always-y += test_cgrp2_tc.bpf.o
125−always-y += test_current_task_under_cgroup.bpf.o
126102 always-y += trace_event_kern.o
127103 always-y += sampleip_kern.o
128104 always-y += lwt_len_hist.bpf.o
@@ -195,7 +171,6 @@ TPROGLDLIBS_xdp_router_ipv4 += -lm -pthread · 66 unmodified lines
195171 TPROGLDLIBS_tracex4 += -lrt
196172 TPROGLDLIBS_trace_output += -lrt
197173 TPROGLDLIBS_map_perf_test += -lrt
198−TPROGLDLIBS_test_overhead += -lrt
199174
200175 # Allows pointing LLC/CLANG to a LLVM backend with bpf support, redefine on cmdline:
201176 # make M=samples/bpf LLC=~/git/llvm-project/llvm/build/bin/llc CLANG=~/git/llvm-project/llvm/build/bin/clang
samples/bpf/sock_flags.bpf.c+0−47
@@ -1,47 +0,0 @@
1−// SPDX-License-Identifier: GPL-2.0
2−#include "vmlinux.h"
3−#include "net_shared.h"
4−#include <bpf/bpf_helpers.h>
5−
6−SEC("cgroup/sock")
7−int bpf_prog1(struct bpf_sock *sk)
8−{
9− char fmt[] = "socket: family %d type %d protocol %d\n";
10− char fmt2[] = "socket: uid %u gid %u\n";
11− __u64 gid_uid = bpf_get_current_uid_gid();
12− __u32 uid = gid_uid & 0xffffffff;
13− __u32 gid = gid_uid >> 32;
14−
15− bpf_trace_printk(fmt, sizeof(fmt), sk->family, sk->type, sk->protocol);
16− bpf_trace_printk(fmt2, sizeof(fmt2), uid, gid);
17−
18− /* block AF_INET6, SOCK_DGRAM, IPPROTO_ICMPV6 sockets
19− * ie., make ping6 fail
20− */
21− if (sk->family == AF_INET6 &&
22− sk->type == SOCK_DGRAM &&
23− sk->protocol == IPPROTO_ICMPV6)
24− return 0;
25−
26− return 1;
27−}
28−
29−SEC("cgroup/sock")
30−int bpf_prog2(struct bpf_sock *sk)
31−{
32− char fmt[] = "socket: family %d type %d protocol %d\n";
33−
34− bpf_trace_printk(fmt, sizeof(fmt), sk->family, sk->type, sk->protocol);
35−
36− /* block AF_INET, SOCK_DGRAM, IPPROTO_ICMP sockets
37− * ie., make ping fail
38− */
39− if (sk->family == AF_INET &&
40− sk->type == SOCK_DGRAM &&
41− sk->protocol == IPPROTO_ICMP)
42− return 0;
43−
44− return 1;
45−}
46−
47−char _license[] SEC("license") = "GPL";
samples/bpf/syscall_nrs.c+5−0
@@ -2,6 +2,9 @@ · 1 unmodified line
22 #include <uapi/linux/unistd.h>
33 #include <linux/kbuild.h>
44
5+#pragma GCC diagnostic push
6+#pragma GCC diagnostic ignored "-Wmissing-prototypes"
7+
58 #define SYSNR(_NR) DEFINE(SYS ## _NR, _NR)
69
710 void syscall_defines(void)
@@ -17,3 +20,5 @@ void syscall_defines(void) · 9 unmodified lines
1720 #endif
1821
1922 }
23+
24+#pragma GCC diagnostic pop
samples/bpf/tc_l2_redirect_kern.c+0−6
@@ -58,14 +58,11 @@ static __always_inline bool is_vip_addr(__be16 eth_proto, __be32 daddr) · 57 unmodified lines
5858 SEC("l2_to_iptun_ingress_forward")
5959 int _l2_to_iptun_ingress_forward(struct __sk_buff *skb)
6060 {
61− struct bpf_tunnel_key tkey = {};
6261 void *data = (void *)(long)skb->data;
6362 struct eth_hdr *eth = data;
6463 void *data_end = (void *)(long)skb->data_end;
6564 int key = 0, *ifindex;
6665
67− int ret;
68−
6966 if (data + sizeof(*eth) > data_end)
7067 return TC_ACT_OK;
7168
@@ -115,8 +112,6 @@ int _l2_to_iptun_ingress_redirect(struct __sk_buff *skb) · 43 unmodified lines
115112 void *data_end = (void *)(long)skb->data_end;
116113 int key = 0, *ifindex;
117114
118− int ret;
119−
120115 if (data + sizeof(*eth) > data_end)
121116 return TC_ACT_OK;
122117
@@ -205,7 +200,6 @@ int _l2_to_ip6tun_ingress_redirect(struct __sk_buff *skb) · 82 unmodified lines
205200 SEC("drop_non_tun_vip")
206201 int _drop_non_tun_vip(struct __sk_buff *skb)
207202 {
208− struct bpf_tunnel_key tkey = {};
209203 void *data = (void *)(long)skb->data;
210204 struct eth_hdr *eth = data;
211205 void *data_end = (void *)(long)skb->data_end;
samples/bpf/test_cgrp2_array_pin.c+0−106
@@ -1,106 +0,0 @@
1−// SPDX-License-Identifier: GPL-2.0-only
2−/* Copyright (c) 2016 Facebook
3− */
4−#include <linux/unistd.h>
5−#include <linux/bpf.h>
6−
7−#include <stdio.h>
8−#include <stdint.h>
9−#include <unistd.h>
10−#include <string.h>
11−#include <errno.h>
12−#include <fcntl.h>
13−
14−#include <bpf/bpf.h>
15−
16−static void usage(void)
17−{
18− printf("Usage: test_cgrp2_array_pin [...]\n");
19− printf(" -F <file> File to pin an BPF cgroup array\n");
20− printf(" -U <file> Update an already pinned BPF cgroup array\n");
21− printf(" -v <value> Full path of the cgroup2\n");
22− printf(" -h Display this help\n");
23−}
24−
25−int main(int argc, char **argv)
26−{
27− const char *pinned_file = NULL, *cg2 = NULL;
28− int create_array = 1;
29− int array_key = 0;
30− int array_fd = -1;
31− int cg2_fd = -1;
32− int ret = -1;
33− int opt;
34−
35− while ((opt = getopt(argc, argv, "F:U:v:")) != -1) {
36− switch (opt) {
37− /* General args */
38− case 'F':
39− pinned_file = optarg;
40− break;
41− case 'U':
42− pinned_file = optarg;
43− create_array = 0;
44− break;
45− case 'v':
46− cg2 = optarg;
47− break;
48− default:
49− usage();
50− goto out;
51− }
52− }
53−
54− if (!cg2 || !pinned_file) {
55− usage();
56− goto out;
57− }
58−
59− cg2_fd = open(cg2, O_RDONLY);
60− if (cg2_fd < 0) {
61− fprintf(stderr, "open(%s,...): %s(%d)\n",
62− cg2, strerror(errno), errno);
63− goto out;
64− }
65−
66− if (create_array) {
67− array_fd = bpf_map_create(BPF_MAP_TYPE_CGROUP_ARRAY, NULL,
68− sizeof(uint32_t), sizeof(uint32_t),
69− 1, NULL);
70− if (array_fd < 0) {
71− fprintf(stderr,
72− "bpf_create_map(BPF_MAP_TYPE_CGROUP_ARRAY,...): %s(%d)\n",
73− strerror(errno), errno);
74− goto out;
75− }
76− } else {
77− array_fd = bpf_obj_get(pinned_file);
78− if (array_fd < 0) {
79− fprintf(stderr, "bpf_obj_get(%s): %s(%d)\n",
80− pinned_file, strerror(errno), errno);
81− goto out;
82− }
83− }
84−
85− ret = bpf_map_update_elem(array_fd, &array_key, &cg2_fd, 0);
86− if (ret) {
87− perror("bpf_map_update_elem");
88− goto out;
89− }
90−
91− if (create_array) {
92− ret = bpf_obj_pin(array_fd, pinned_file);
93− if (ret) {
94− fprintf(stderr, "bpf_obj_pin(..., %s): %s(%d)\n",
95− pinned_file, strerror(errno), errno);
96− goto out;
97− }
98− }
99−
100−out:
101− if (array_fd != -1)
102− close(array_fd);
103− if (cg2_fd != -1)
104− close(cg2_fd);
105− return ret;
106−}
samples/bpf/test_cgrp2_attach.c+0−177
@@ -1,177 +0,0 @@
1−/* eBPF example program:
2− *
3− * - Creates arraymap in kernel with 4 bytes keys and 8 byte values
4− *
5− * - Loads eBPF program
6− *
7− * The eBPF program accesses the map passed in to store two pieces of
8− * information. The number of invocations of the program, which maps
9− * to the number of packets received, is stored to key 0. Key 1 is
10− * incremented on each iteration by the number of bytes stored in
11− * the skb.
12− *
13− * - Attaches the new program to a cgroup using BPF_PROG_ATTACH
14− *
15− * - Every second, reads map[0] and map[1] to see how many bytes and
16− * packets were seen on any socket of tasks in the given cgroup.
17− */
18−
19−#define _GNU_SOURCE
20−
21−#include <stdio.h>
22−#include <stdlib.h>
23−#include <stddef.h>
24−#include <string.h>
25−#include <unistd.h>
26−#include <assert.h>
27−#include <errno.h>
28−#include <fcntl.h>
29−
30−#include <linux/bpf.h>
31−#include <bpf/bpf.h>
32−
33−#include "bpf_insn.h"
34−#include "bpf_util.h"
35−
36−enum {
37− MAP_KEY_PACKETS,
38− MAP_KEY_BYTES,
39−};
40−
41−char bpf_log_buf[BPF_LOG_BUF_SIZE];
42−
43−static int prog_load(int map_fd, int verdict)
44−{
45− struct bpf_insn prog[] = {
46− BPF_MOV64_REG(BPF_REG_6, BPF_REG_1), /* save r6 so it's not clobbered by BPF_CALL */
47−
48− /* Count packets */
49− BPF_MOV64_IMM(BPF_REG_0, MAP_KEY_PACKETS), /* r0 = 0 */
50− BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_0, -4), /* *(u32 *)(fp - 4) = r0 */
51− BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
52− BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), /* r2 = fp - 4 */
53− BPF_LD_MAP_FD(BPF_REG_1, map_fd), /* load map fd to r1 */
54− BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem),
55− BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 2),
56− BPF_MOV64_IMM(BPF_REG_1, 1), /* r1 = 1 */
57− BPF_ATOMIC_OP(BPF_DW, BPF_ADD, BPF_REG_0, BPF_REG_1, 0),
58−
59− /* Count bytes */
60− BPF_MOV64_IMM(BPF_REG_0, MAP_KEY_BYTES), /* r0 = 1 */
61− BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_0, -4), /* *(u32 *)(fp - 4) = r0 */
62− BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
63− BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), /* r2 = fp - 4 */
64− BPF_LD_MAP_FD(BPF_REG_1, map_fd),
65− BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem),
66− BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 2),
67− BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_6, offsetof(struct __sk_buff, len)), /* r1 = skb->len */
68−
69− BPF_ATOMIC_OP(BPF_DW, BPF_ADD, BPF_REG_0, BPF_REG_1, 0),
70−
71− BPF_MOV64_IMM(BPF_REG_0, verdict), /* r0 = verdict */
72− BPF_EXIT_INSN(),
73− };
74− size_t insns_cnt = ARRAY_SIZE(prog);
75− LIBBPF_OPTS(bpf_prog_load_opts, opts,
76− .log_buf = bpf_log_buf,
77− .log_size = BPF_LOG_BUF_SIZE,
78− );
79−
80− return bpf_prog_load(BPF_PROG_TYPE_CGROUP_SKB, NULL, "GPL",
81− prog, insns_cnt, &opts);
82−}
83−
84−static int usage(const char *argv0)
85−{
86− printf("Usage: %s [-d] [-D] <cg-path> <egress|ingress>\n", argv0);
87− printf(" -d Drop Traffic\n");
88− printf(" -D Detach filter, and exit\n");
89− return EXIT_FAILURE;
90−}
91−
92−static int attach_filter(int cg_fd, int type, int verdict)
93−{
94− int prog_fd, map_fd, ret, key;
95− long long pkt_cnt, byte_cnt;
96−
97− map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL,
98− sizeof(key), sizeof(byte_cnt),
99− 256, NULL);
100− if (map_fd < 0) {
101− printf("Failed to create map: '%s'\n", strerror(errno));
102− return EXIT_FAILURE;
103− }
104−
105− prog_fd = prog_load(map_fd, verdict);
106− printf("Output from kernel verifier:\n%s\n-------\n", bpf_log_buf);
107−
108− if (prog_fd < 0) {
109− printf("Failed to load prog: '%s'\n", strerror(errno));
110− return EXIT_FAILURE;
111− }
112−
113− ret = bpf_prog_attach(prog_fd, cg_fd, type, 0);
114− if (ret < 0) {
115− printf("Failed to attach prog to cgroup: '%s'\n",
116− strerror(errno));
117− return EXIT_FAILURE;
118− }
119− while (1) {
120− key = MAP_KEY_PACKETS;
121− assert(bpf_map_lookup_elem(map_fd, &key, &pkt_cnt) == 0);
122−
123− key = MAP_KEY_BYTES;
124− assert(bpf_map_lookup_elem(map_fd, &key, &byte_cnt) == 0);
125−
126− printf("cgroup received %lld packets, %lld bytes\n",
127− pkt_cnt, byte_cnt);
128− sleep(1);
129− }
130−
131− return EXIT_SUCCESS;
132−}
133−
134−int main(int argc, char **argv)
135−{
136− int detach_only = 0, verdict = 1;
137− enum bpf_attach_type type;
138− int opt, cg_fd, ret;
139−
140− while ((opt = getopt(argc, argv, "Dd")) != -1) {
141− switch (opt) {
142− case 'd':
143− verdict = 0;
144− break;
145− case 'D':
146− detach_only = 1;
147− break;
148− default:
149− return usage(argv[0]);
150− }
151− }
152−
153− if (argc - optind < 2)
154− return usage(argv[0]);
155−
156− if (strcmp(argv[optind + 1], "ingress") == 0)
157− type = BPF_CGROUP_INET_INGRESS;
158− else if (strcmp(argv[optind + 1], "egress") == 0)
159− type = BPF_CGROUP_INET_EGRESS;
160− else
161− return usage(argv[0]);
162−
163− cg_fd = open(argv[optind], O_DIRECTORY | O_RDONLY);
164− if (cg_fd < 0) {
165− printf("Failed to open cgroup path: '%s'\n", strerror(errno));
166− return EXIT_FAILURE;
167− }
168−
169− if (detach_only) {
170− ret = bpf_prog_detach(cg_fd, type);
171− printf("bpf_prog_detach() returned '%s' (%d)\n",
172− strerror(errno), errno);
173− } else
174− ret = attach_filter(cg_fd, type, verdict);
175−
176− return ret;
177−}
samples/bpf/test_cgrp2_sock.c+0−294
@@ -1,294 +0,0 @@
1−/* eBPF example program:
2− *
3− * - Loads eBPF program
4− *
5− * The eBPF program sets the sk_bound_dev_if index in new AF_INET{6}
6− * sockets opened by processes in the cgroup.
7− *
8− * - Attaches the new program to a cgroup using BPF_PROG_ATTACH
9− */
10−
11−#define _GNU_SOURCE
12−
13−#include <stdio.h>
14−#include <stdlib.h>
15−#include <stddef.h>
16−#include <string.h>
17−#include <unistd.h>
18−#include <assert.h>
19−#include <errno.h>
20−#include <fcntl.h>
21−#include <net/if.h>
22−#include <inttypes.h>
23−#include <linux/bpf.h>
24−#include <bpf/bpf.h>
25−
26−#include "bpf_insn.h"
27−
28−char bpf_log_buf[BPF_LOG_BUF_SIZE];
29−
30−static int prog_load(__u32 idx, __u32 mark, __u32 prio)
31−{
32− /* save pointer to context */
33− struct bpf_insn prog_start[] = {
34− BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
35− };
36− struct bpf_insn prog_end[] = {
37− BPF_MOV64_IMM(BPF_REG_0, 1), /* r0 = verdict */
38− BPF_EXIT_INSN(),
39− };
40−
41− /* set sk_bound_dev_if on socket */
42− struct bpf_insn prog_dev[] = {
43− BPF_MOV64_IMM(BPF_REG_3, idx),
44− BPF_MOV64_IMM(BPF_REG_2, offsetof(struct bpf_sock, bound_dev_if)),
45− BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_3, offsetof(struct bpf_sock, bound_dev_if)),
46− };
47−
48− /* set mark on socket */
49− struct bpf_insn prog_mark[] = {
50− /* get uid of process */
51− BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
52− BPF_FUNC_get_current_uid_gid),
53− BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xffffffff),
54−
55− /* if uid is 0, use given mark, else use the uid as the mark */
56− BPF_MOV64_REG(BPF_REG_3, BPF_REG_0),
57− BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 1),
58− BPF_MOV64_IMM(BPF_REG_3, mark),
59−
60− /* set the mark on the new socket */
61− BPF_MOV64_REG(BPF_REG_1, BPF_REG_6),
62− BPF_MOV64_IMM(BPF_REG_2, offsetof(struct bpf_sock, mark)),
63− BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_3, offsetof(struct bpf_sock, mark)),
64− };
65−
66− /* set priority on socket */
67− struct bpf_insn prog_prio[] = {
68− BPF_MOV64_REG(BPF_REG_1, BPF_REG_6),
69− BPF_MOV64_IMM(BPF_REG_3, prio),
70− BPF_MOV64_IMM(BPF_REG_2, offsetof(struct bpf_sock, priority)),
71− BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_3, offsetof(struct bpf_sock, priority)),
72− };
73− LIBBPF_OPTS(bpf_prog_load_opts, opts,
74− .log_buf = bpf_log_buf,
75− .log_size = BPF_LOG_BUF_SIZE,
76− );
77−
78− struct bpf_insn *prog;
79− size_t insns_cnt;
80− void *p;
81− int ret;
82−
83− insns_cnt = sizeof(prog_start) + sizeof(prog_end);
84− if (idx)
85− insns_cnt += sizeof(prog_dev);
86−
87− if (mark)
88− insns_cnt += sizeof(prog_mark);
89−
90− if (prio)
91− insns_cnt += sizeof(prog_prio);
92−
93− p = prog = malloc(insns_cnt);
94− if (!prog) {
95− fprintf(stderr, "Failed to allocate memory for instructions\n");
96− return EXIT_FAILURE;
97− }
98−
99− memcpy(p, prog_start, sizeof(prog_start));
100− p += sizeof(prog_start);
101−
102− if (idx) {
103− memcpy(p, prog_dev, sizeof(prog_dev));
104− p += sizeof(prog_dev);
105− }
106−
107− if (mark) {
108− memcpy(p, prog_mark, sizeof(prog_mark));
109− p += sizeof(prog_mark);
110− }
111−
112− if (prio) {
113− memcpy(p, prog_prio, sizeof(prog_prio));
114− p += sizeof(prog_prio);
115− }
116−
117− memcpy(p, prog_end, sizeof(prog_end));
118− p += sizeof(prog_end);
119−
120− insns_cnt /= sizeof(struct bpf_insn);
121−
122− ret = bpf_prog_load(BPF_PROG_TYPE_CGROUP_SOCK, NULL, "GPL",
123− prog, insns_cnt, &opts);
124−
125− free(prog);
126−
127− return ret;
128−}
129−
130−static int get_bind_to_device(int sd, char *name, size_t len)
131−{
132− socklen_t optlen = len;
133− int rc;
134−
135− name[0] = '\0';
136− rc = getsockopt(sd, SOL_SOCKET, SO_BINDTODEVICE, name, &optlen);
137− if (rc < 0)
138− perror("setsockopt(SO_BINDTODEVICE)");
139−
140− return rc;
141−}
142−
143−static unsigned int get_somark(int sd)
144−{
145− unsigned int mark = 0;
146− socklen_t optlen = sizeof(mark);
147− int rc;
148−
149− rc = getsockopt(sd, SOL_SOCKET, SO_MARK, &mark, &optlen);
150− if (rc < 0)
151− perror("getsockopt(SO_MARK)");
152−
153− return mark;
154−}
155−
156−static unsigned int get_priority(int sd)
157−{
158− unsigned int prio = 0;
159− socklen_t optlen = sizeof(prio);
160− int rc;
161−
162− rc = getsockopt(sd, SOL_SOCKET, SO_PRIORITY, &prio, &optlen);
163− if (rc < 0)
164− perror("getsockopt(SO_PRIORITY)");
165−
166− return prio;
167−}
168−
169−static int show_sockopts(int family)
170−{
171− unsigned int mark, prio;
172− char name[16];
173− int sd;
174−
175− sd = socket(family, SOCK_DGRAM, 17);
176− if (sd < 0) {
177− perror("socket");
178− return 1;
179− }
180−
181− if (get_bind_to_device(sd, name, sizeof(name)) < 0)
182− return 1;
183−
184− mark = get_somark(sd);
185− prio = get_priority(sd);
186−
187− close(sd);
188−
189− printf("sd %d: dev %s, mark %u, priority %u\n", sd, name, mark, prio);
190−
191− return 0;
192−}
193−
194−static int usage(const char *argv0)
195−{
196− printf("Usage:\n");
197− printf(" Attach a program\n");
198− printf(" %s -b bind-to-dev -m mark -p prio cg-path\n", argv0);
199− printf("\n");
200− printf(" Detach a program\n");
201− printf(" %s -d cg-path\n", argv0);
202− printf("\n");
203− printf(" Show inherited socket settings (mark, priority, and device)\n");
204− printf(" %s [-6]\n", argv0);
205− return EXIT_FAILURE;
206−}
207−
208−int main(int argc, char **argv)
209−{
210− __u32 idx = 0, mark = 0, prio = 0;
211− const char *cgrp_path = NULL;
212− int cg_fd, prog_fd, ret;
213− int family = PF_INET;
214− int do_attach = 1;
215− int rc;
216−
217− while ((rc = getopt(argc, argv, "db:m:p:6")) != -1) {
218− switch (rc) {
219− case 'd':
220− do_attach = 0;
221− break;
222− case 'b':
223− idx = if_nametoindex(optarg);
224− if (!idx) {
225− idx = strtoumax(optarg, NULL, 0);
226− if (!idx) {
227− printf("Invalid device name\n");
228− return EXIT_FAILURE;
229− }
230− }
231− break;
232− case 'm':
233− mark = strtoumax(optarg, NULL, 0);
234− break;
235− case 'p':
236− prio = strtoumax(optarg, NULL, 0);
237− break;
238− case '6':
239− family = PF_INET6;
240− break;
241− default:
242− return usage(argv[0]);
243− }
244− }
245−
246− if (optind == argc)
247− return show_sockopts(family);
248−
249− cgrp_path = argv[optind];
250− if (!cgrp_path) {
251− fprintf(stderr, "cgroup path not given\n");
252− return EXIT_FAILURE;
253− }
254−
255− if (do_attach && !idx && !mark && !prio) {
256− fprintf(stderr,
257− "One of device, mark or priority must be given\n");
258− return EXIT_FAILURE;
259− }
260−
261− cg_fd = open(cgrp_path, O_DIRECTORY | O_RDONLY);
262− if (cg_fd < 0) {
263− printf("Failed to open cgroup path: '%s'\n", strerror(errno));
264− return EXIT_FAILURE;
265− }
266−
267− if (do_attach) {
268− prog_fd = prog_load(idx, mark, prio);
269− if (prog_fd < 0) {
270− printf("Failed to load prog: '%s'\n", strerror(errno));
271− printf("Output from kernel verifier:\n%s\n-------\n",
272− bpf_log_buf);
273− return EXIT_FAILURE;
274− }
275−
276− ret = bpf_prog_attach(prog_fd, cg_fd,
277− BPF_CGROUP_INET_SOCK_CREATE, 0);
278− if (ret < 0) {
279− printf("Failed to attach prog to cgroup: '%s'\n",
280− strerror(errno));
281− return EXIT_FAILURE;
282− }
283− } else {
284− ret = bpf_prog_detach(cg_fd, BPF_CGROUP_INET_SOCK_CREATE);
285− if (ret < 0) {
286− printf("Failed to detach prog from cgroup: '%s'\n",
287− strerror(errno));
288− return EXIT_FAILURE;
289− }
290− }
291−
292− close(cg_fd);
293− return EXIT_SUCCESS;
294−}
samples/bpf/test_cgrp2_sock.sh+0−137
@@ -1,137 +0,0 @@
1−#!/bin/sh
2−# SPDX-License-Identifier: GPL-2.0
3−
4−# Test various socket options that can be set by attaching programs to cgroups.
5−
6−MY_DIR=$(dirname $0)
7−TEST=$MY_DIR/test_cgrp2_sock
8−CGRP_MNT="/tmp/cgroupv2-test_cgrp2_sock"
9−
10−################################################################################
11−#
12−print_result()
13−{
14− local rc=$1
15− local status=" OK "
16−
17− [ $rc -ne 0 ] && status="FAIL"
18−
19− printf "%-50s [%4s]\n" "$2" "$status"
20−}
21−
22−check_sock()
23−{
24− out=$($TEST)
25− echo $out | grep -q "$1"
26− if [ $? -ne 0 ]; then
27− print_result 1 "IPv4: $2"
28− echo " expected: $1"
29− echo " have: $out"
30− rc=1
31− else
32− print_result 0 "IPv4: $2"
33− fi
34−}
35−
36−check_sock6()
37−{
38− out=$($TEST -6)
39− echo $out | grep -q "$1"
40− if [ $? -ne 0 ]; then
41− print_result 1 "IPv6: $2"
42− echo " expected: $1"
43− echo " have: $out"
44− rc=1
45− else
46− print_result 0 "IPv6: $2"
47− fi
48−}
49−
50−################################################################################
51−#
52−
53−cleanup()
54−{
55− echo $$ >> ${CGRP_MNT}/cgroup.procs
56− rmdir ${CGRP_MNT}/sockopts
57−}
58−
59−cleanup_and_exit()
60−{
61− local rc=$1
62− local msg="$2"
63−
64− [ -n "$msg" ] && echo "ERROR: $msg"
65−
66− $TEST -d ${CGRP_MNT}/sockopts
67− ip li del cgrp2_sock
68− umount ${CGRP_MNT}
69−
70− exit $rc
71−}
72−
73−
74−################################################################################
75−# main
76−
77−rc=0
78−
79−ip li add cgrp2_sock type dummy 2>/dev/null
80−
81−set -e
82−mkdir -p ${CGRP_MNT}
83−mount -t cgroup2 none ${CGRP_MNT}
84−set +e
85−
86−
87−# make sure we have a known start point
88−cleanup 2>/dev/null
89−
90−mkdir -p ${CGRP_MNT}/sockopts
91−[ $? -ne 0 ] && cleanup_and_exit 1 "Failed to create cgroup hierarchy"
92−
93−
94−# set pid into cgroup
95−echo $$ > ${CGRP_MNT}/sockopts/cgroup.procs
96−
97−# no bpf program attached, so socket should show no settings
98−check_sock "dev , mark 0, priority 0" "No programs attached"
99−check_sock6 "dev , mark 0, priority 0" "No programs attached"
100−
101−# verify device is set
102−#
103−$TEST -b cgrp2_sock ${CGRP_MNT}/sockopts
104−if [ $? -ne 0 ]; then
105− cleanup_and_exit 1 "Failed to install program to set device"
106−fi
107−check_sock "dev cgrp2_sock, mark 0, priority 0" "Device set"
108−check_sock6 "dev cgrp2_sock, mark 0, priority 0" "Device set"
109−
110−# verify mark is set
111−#
112−$TEST -m 666 ${CGRP_MNT}/sockopts
113−if [ $? -ne 0 ]; then
114− cleanup_and_exit 1 "Failed to install program to set mark"
115−fi
116−check_sock "dev , mark 666, priority 0" "Mark set"
117−check_sock6 "dev , mark 666, priority 0" "Mark set"
118−
119−# verify priority is set
120−#
121−$TEST -p 123 ${CGRP_MNT}/sockopts
122−if [ $? -ne 0 ]; then
123− cleanup_and_exit 1 "Failed to install program to set priority"
124−fi
125−check_sock "dev , mark 0, priority 123" "Priority set"
126−check_sock6 "dev , mark 0, priority 123" "Priority set"
127−
128−# all 3 at once
129−#
130−$TEST -b cgrp2_sock -m 666 -p 123 ${CGRP_MNT}/sockopts
131−if [ $? -ne 0 ]; then
132− cleanup_and_exit 1 "Failed to install program to set device, mark and priority"
133−fi
134−check_sock "dev cgrp2_sock, mark 666, priority 123" "Priority set"
135−check_sock6 "dev cgrp2_sock, mark 666, priority 123" "Priority set"
136−
137−cleanup_and_exit $rc
samples/bpf/test_cgrp2_sock2.c+0−95
@@ -1,95 +0,0 @@
1−// SPDX-License-Identifier: GPL-2.0
2−/* eBPF example program:
3− *
4− * - Loads eBPF program
5− *
6− * The eBPF program loads a filter from file and attaches the
7− * program to a cgroup using BPF_PROG_ATTACH
8− */
9−
10−#define _GNU_SOURCE
11−
12−#include <stdio.h>
13−#include <stdlib.h>
14−#include <stddef.h>
15−#include <string.h>
16−#include <unistd.h>
17−#include <assert.h>
18−#include <errno.h>
19−#include <fcntl.h>
20−#include <net/if.h>
21−#include <linux/bpf.h>
22−#include <bpf/bpf.h>
23−#include <bpf/libbpf.h>
24−
25−#include "bpf_insn.h"
26−
27−static int usage(const char *argv0)
28−{
29− printf("Usage: %s cg-path filter-path [filter-id]\n", argv0);
30− return EXIT_FAILURE;
31−}
32−
33−int main(int argc, char **argv)
34−{
35− int cg_fd, err, ret = EXIT_FAILURE, filter_id = 0, prog_cnt = 0;
36− const char *link_pin_path = "/sys/fs/bpf/test_cgrp2_sock2";
37− struct bpf_link *link = NULL;
38− struct bpf_program *progs[2];
39− struct bpf_program *prog;
40− struct bpf_object *obj;
41−
42− if (argc < 3)
43− return usage(argv[0]);
44−
45− if (argc > 3)
46− filter_id = atoi(argv[3]);
47−
48− cg_fd = open(argv[1], O_DIRECTORY | O_RDONLY);
49− if (cg_fd < 0) {
50− printf("Failed to open cgroup path: '%s'\n", strerror(errno));
51− return ret;
52− }
53−
54− obj = bpf_object__open_file(argv[2], NULL);
55− if (libbpf_get_error(obj)) {
56− printf("ERROR: opening BPF object file failed\n");
57− return ret;
58− }
59−
60− bpf_object__for_each_program(prog, obj) {
61− progs[prog_cnt] = prog;
62− prog_cnt++;
63− }
64−
65− if (filter_id >= prog_cnt) {
66− printf("Invalid program id; program not found in file\n");
67− goto cleanup;
68− }
69−
70− /* load BPF program */
71− if (bpf_object__load(obj)) {
72− printf("ERROR: loading BPF object file failed\n");
73− goto cleanup;
74− }
75−
76− link = bpf_program__attach_cgroup(progs[filter_id], cg_fd);
77− if (libbpf_get_error(link)) {
78− printf("ERROR: bpf_program__attach failed\n");
79− link = NULL;
80− goto cleanup;
81− }
82−
83− err = bpf_link__pin(link, link_pin_path);
84− if (err < 0) {
85− printf("ERROR: bpf_link__pin failed: %d\n", err);
86− goto cleanup;
87− }
88−
89− ret = EXIT_SUCCESS;
90−
91−cleanup:
92− bpf_link__destroy(link);
93− bpf_object__close(obj);
94− return ret;
95−}
samples/bpf/test_cgrp2_sock2.sh+0−103
@@ -1,103 +0,0 @@
1−#!/bin/bash
2−# SPDX-License-Identifier: GPL-2.0
3−
4−BPFFS=/sys/fs/bpf
5−MY_DIR=$(dirname $0)
6−TEST=$MY_DIR/test_cgrp2_sock2
7−LINK_PIN=$BPFFS/test_cgrp2_sock2
8−BPF_PROG=$MY_DIR/sock_flags.bpf.o
9−
10−function config_device {
11− ip netns add at_ns0
12− ip link add veth0 type veth peer name veth0b
13− ip link set veth0 netns at_ns0
14− ip netns exec at_ns0 sysctl -q net.ipv6.conf.veth0.disable_ipv6=0
15− ip netns exec at_ns0 ip addr add 172.16.1.100/24 dev veth0
16− ip netns exec at_ns0 ip addr add 2401:db00::1/64 dev veth0 nodad
17− ip netns exec at_ns0 ip link set dev veth0 up
18− sysctl -q net.ipv6.conf.veth0b.disable_ipv6=0
19− ip addr add 172.16.1.101/24 dev veth0b
20− ip addr add 2401:db00::2/64 dev veth0b nodad
21− ip link set veth0b up
22−}
23−
24−function config_cgroup {
25− rm -rf /tmp/cgroupv2
26− mkdir -p /tmp/cgroupv2
27− mount -t cgroup2 none /tmp/cgroupv2
28− mkdir -p /tmp/cgroupv2/foo
29− echo $$ >> /tmp/cgroupv2/foo/cgroup.procs
30−}
31−
32−function config_bpffs {
33− if mount | grep $BPFFS > /dev/null; then
34− echo "bpffs already mounted"
35− else
36− echo "bpffs not mounted. Mounting..."
37− mount -t bpf none $BPFFS
38− fi
39−}
40−
41−function attach_bpf {
42− $TEST /tmp/cgroupv2/foo $BPF_PROG $1
43− [ $? -ne 0 ] && exit 1
44−}
45−
46−function cleanup {
47− rm -rf $LINK_PIN
48− ip link del veth0b
49− ip netns delete at_ns0
50− umount /tmp/cgroupv2
51− rm -rf /tmp/cgroupv2
52−}
53−
54−cleanup 2>/dev/null
55−
56−set -e
57−config_device
58−config_cgroup
59−config_bpffs
60−set +e
61−
62−#
63−# Test 1 - fail ping6
64−#
65−attach_bpf 0
66−ping -c1 -w1 172.16.1.100
67−if [ $? -ne 0 ]; then
68− echo "ping failed when it should succeed"
69− cleanup
70− exit 1
71−fi
72−
73−ping6 -c1 -w1 2401:db00::1
74−if [ $? -eq 0 ]; then
75− echo "ping6 succeeded when it should not"
76− cleanup
77− exit 1
78−fi
79−
80−rm -rf $LINK_PIN
81−sleep 1 # Wait for link detach
82−
83−#
84−# Test 2 - fail ping
85−#
86−attach_bpf 1
87−ping6 -c1 -w1 2401:db00::1
88−if [ $? -ne 0 ]; then
89− echo "ping6 failed when it should succeed"
90− cleanup
91− exit 1
92−fi
93−
94−ping -c1 -w1 172.16.1.100
95−if [ $? -eq 0 ]; then
96− echo "ping succeeded when it should not"
97− cleanup
98− exit 1
99−fi
100−
101−cleanup
102−echo
103−echo "*** PASS ***"
samples/bpf/test_cgrp2_tc.bpf.c+0−56
@@ -1,56 +0,0 @@
1−/* Copyright (c) 2016 Facebook
2− *
3− * This program is free software; you can redistribute it and/or
4− * modify it under the terms of version 2 of the GNU General Public
5− * License as published by the Free Software Foundation.
6− */
7−#define KBUILD_MODNAME "foo"
8−#include "vmlinux.h"
9−#include "net_shared.h"
10−#include <bpf/bpf_helpers.h>
11−
12−/* copy of 'struct ethhdr' without __packed */
13−struct eth_hdr {
14− unsigned char h_dest[ETH_ALEN];
15− unsigned char h_source[ETH_ALEN];
16− unsigned short h_proto;
17−};
18−
19−struct {
20− __uint(type, BPF_MAP_TYPE_CGROUP_ARRAY);
21− __type(key, u32);
22− __type(value, u32);
23− __uint(pinning, LIBBPF_PIN_BY_NAME);
24− __uint(max_entries, 1);
25−} test_cgrp2_array_pin SEC(".maps");
26−
27−SEC("filter")
28−int handle_egress(struct __sk_buff *skb)
29−{
30− void *data = (void *)(long)skb->data;
31− struct eth_hdr *eth = data;
32− struct ipv6hdr *ip6h = data + sizeof(*eth);
33− void *data_end = (void *)(long)skb->data_end;
34− char dont_care_msg[] = "dont care %04x %d\n";
35− char pass_msg[] = "pass\n";
36− char reject_msg[] = "reject\n";
37−
38− /* single length check */
39− if (data + sizeof(*eth) + sizeof(*ip6h) > data_end)
40− return TC_ACT_OK;
41−
42− if (eth->h_proto != bpf_htons(ETH_P_IPV6) ||
43− ip6h->nexthdr != IPPROTO_ICMPV6) {
44− bpf_trace_printk(dont_care_msg, sizeof(dont_care_msg),
45− eth->h_proto, ip6h->nexthdr);
46− return TC_ACT_OK;
47− } else if (bpf_skb_under_cgroup(skb, &test_cgrp2_array_pin, 0) != 1) {
48− bpf_trace_printk(pass_msg, sizeof(pass_msg));
49− return TC_ACT_OK;
50− } else {
51− bpf_trace_printk(reject_msg, sizeof(reject_msg));
52− return TC_ACT_SHOT;
53− }
54−}
55−
56−char _license[] SEC("license") = "GPL";
samples/bpf/test_cgrp2_tc.sh+0−187
@@ -1,187 +0,0 @@
1−#!/bin/bash
2−# SPDX-License-Identifier: GPL-2.0
3−
4−MY_DIR=$(dirname $0)
5−# Details on the bpf prog
6−BPF_CGRP2_ARRAY_NAME='test_cgrp2_array_pin'
7−BPF_PROG="$MY_DIR/test_cgrp2_tc.bpf.o"
8−BPF_SECTION='filter'
9−
10−[ -z "$TC" ] && TC='tc'
11−[ -z "$IP" ] && IP='ip'
12−
13−# Names of the veth interface, net namespace...etc.
14−HOST_IFC='ve'
15−NS_IFC='vens'
16−NS='ns'
17−
18−find_mnt() {
19− cat /proc/mounts | \
20− awk '{ if ($3 == "'$1'" && mnt == "") { mnt = $2 }} END { print mnt }'
21−}
22−
23−# Init cgroup2 vars
24−init_cgrp2_vars() {
25− CGRP2_ROOT=$(find_mnt cgroup2)
26− if [ -z "$CGRP2_ROOT" ]
27− then
28− CGRP2_ROOT='/mnt/cgroup2'
29− MOUNT_CGRP2="yes"
30− fi
31− CGRP2_TC="$CGRP2_ROOT/tc"
32− CGRP2_TC_LEAF="$CGRP2_TC/leaf"
33−}
34−
35−# Init bpf fs vars
36−init_bpf_fs_vars() {
37− local bpf_fs_root=$(find_mnt bpf)
38− [ -n "$bpf_fs_root" ] || return -1
39− BPF_FS_TC_SHARE="$bpf_fs_root/tc/globals"
40−}
41−
42−setup_cgrp2() {
43− case $1 in
44− start)
45− if [ "$MOUNT_CGRP2" == 'yes' ]
46− then
47− [ -d $CGRP2_ROOT ] || mkdir -p $CGRP2_ROOT
48− mount -t cgroup2 none $CGRP2_ROOT || return $?
49− fi
50− mkdir -p $CGRP2_TC_LEAF
51− ;;
52− *)
53− rmdir $CGRP2_TC_LEAF && rmdir $CGRP2_TC
54− [ "$MOUNT_CGRP2" == 'yes' ] && umount $CGRP2_ROOT
55− ;;
56− esac
57−}
58−
59−setup_bpf_cgrp2_array() {
60− local bpf_cgrp2_array="$BPF_FS_TC_SHARE/$BPF_CGRP2_ARRAY_NAME"
61− case $1 in
62− start)
63− $MY_DIR/test_cgrp2_array_pin -U $bpf_cgrp2_array -v $CGRP2_TC
64− ;;
65− *)
66− [ -d "$BPF_FS_TC_SHARE" ] && rm -f $bpf_cgrp2_array
67− ;;
68− esac
69−}
70−
71−setup_net() {
72− case $1 in
73− start)
74− $IP link add $HOST_IFC type veth peer name $NS_IFC || return $?
75− $IP link set dev $HOST_IFC up || return $?
76− sysctl -q net.ipv6.conf.$HOST_IFC.disable_ipv6=0
77− sysctl -q net.ipv6.conf.$HOST_IFC.accept_dad=0
78−
79− $IP netns add $NS || return $?
80− $IP link set dev $NS_IFC netns $NS || return $?
81− $IP -n $NS link set dev $NS_IFC up || return $?
82− $IP netns exec $NS sysctl -q net.ipv6.conf.$NS_IFC.disable_ipv6=0
83− $IP netns exec $NS sysctl -q net.ipv6.conf.$NS_IFC.accept_dad=0
84− $TC qdisc add dev $HOST_IFC clsact || return $?
85− $TC filter add dev $HOST_IFC egress bpf da obj $BPF_PROG sec $BPF_SECTION || return $?
86− ;;
87− *)
88− $IP netns del $NS
89− $IP link del $HOST_IFC
90− ;;
91− esac
92−}
93−
94−run_in_cgrp() {
95− # Fork another bash and move it under the specified cgroup.
96− # It makes the cgroup cleanup easier at the end of the test.
97− cmd='echo $$ > '
98− cmd="$cmd $1/cgroup.procs; exec $2"
99− bash -c "$cmd"
100−}
101−
102−do_test() {
103− run_in_cgrp $CGRP2_TC_LEAF "ping -6 -c3 ff02::1%$HOST_IFC >& /dev/null"
104− local dropped=$($TC -s qdisc show dev $HOST_IFC | tail -3 | \
105− awk '/drop/{print substr($7, 0, index($7, ",")-1)}')
106− if [[ $dropped -eq 0 ]]
107− then
108− echo "FAIL"
109− return 1
110− else
111− echo "Successfully filtered $dropped packets"
112− return 0
113− fi
114−}
115−
116−do_exit() {
117− if [ "$DEBUG" == "yes" ] && [ "$MODE" != 'cleanuponly' ]
118− then
119− echo "------ DEBUG ------"
120− echo "mount: "; mount | grep -E '(cgroup2|bpf)'; echo
121− echo "$CGRP2_TC_LEAF: "; ls -l $CGRP2_TC_LEAF; echo
122− if [ -d "$BPF_FS_TC_SHARE" ]
123− then
124− echo "$BPF_FS_TC_SHARE: "; ls -l $BPF_FS_TC_SHARE; echo
125− fi
126− echo "Host net:"
127− $IP netns
128− $IP link show dev $HOST_IFC
129− $IP -6 a show dev $HOST_IFC
130− $TC -s qdisc show dev $HOST_IFC
131− echo
132− echo "$NS net:"
133− $IP -n $NS link show dev $NS_IFC
134− $IP -n $NS -6 link show dev $NS_IFC
135− echo "------ DEBUG ------"
136− echo
137− fi
138−
139− if [ "$MODE" != 'nocleanup' ]
140− then
141− setup_net stop
142− setup_bpf_cgrp2_array stop
143− setup_cgrp2 stop
144− fi
145−}
146−
147−init_cgrp2_vars
148−init_bpf_fs_vars
149−
150−while [[ $# -ge 1 ]]
151−do
152− a="$1"
153− case $a in
154− debug)
155− DEBUG='yes'
156− shift 1
157− ;;
158− cleanup-only)
159− MODE='cleanuponly'
160− shift 1
161− ;;
162− no-cleanup)
163− MODE='nocleanup'
164− shift 1
165− ;;
166− *)
167− echo "test_cgrp2_tc [debug] [cleanup-only | no-cleanup]"
168− echo " debug: Print cgrp and network setup details at the end of the test"
169− echo " cleanup-only: Try to cleanup things from last test. No test will be run"
170− echo " no-cleanup: Run the test but don't do cleanup at the end"
171− echo "[Note: If no arg is given, it will run the test and do cleanup at the end]"
172− echo
173− exit -1
174− ;;
175− esac
176−done
177−
178−trap do_exit 0
179−
180−[ "$MODE" == 'cleanuponly' ] && exit
181−
182−setup_cgrp2 start || exit $?
183−setup_net start || exit $?
184−init_bpf_fs_vars || exit $?
185−setup_bpf_cgrp2_array start || exit $?
186−do_test
187−echo
samples/bpf/test_current_task_under_cgroup.bpf.c+0−43
@@ -1,43 +0,0 @@
1−/* Copyright (c) 2016 Sargun Dhillon <sargun@sargun.me>
2− *
3− * This program is free software; you can redistribute it and/or
4− * modify it under the terms of version 2 of the GNU General Public
5− * License as published by the Free Software Foundation.
6− */
7−
8−#include "vmlinux.h"
9−#include <linux/version.h>
10−#include <bpf/bpf_helpers.h>
11−#include <bpf/bpf_tracing.h>
12−#include <bpf/bpf_core_read.h>
13−
14−struct {
15− __uint(type, BPF_MAP_TYPE_CGROUP_ARRAY);
16− __uint(key_size, sizeof(u32));
17− __uint(value_size, sizeof(u32));
18− __uint(max_entries, 1);
19−} cgroup_map SEC(".maps");
20−
21−struct {
22− __uint(type, BPF_MAP_TYPE_ARRAY);
23− __type(key, u32);
24− __type(value, u64);
25− __uint(max_entries, 1);
26−} perf_map SEC(".maps");
27−
28−/* Writes the last PID that called sync to a map at index 0 */
29−SEC("ksyscall/sync")
30−int BPF_KSYSCALL(bpf_prog1)
31−{
32− u64 pid = bpf_get_current_pid_tgid();
33− int idx = 0;
34−
35− if (!bpf_current_task_under_cgroup(&cgroup_map, 0))
36− return 0;
37−
38− bpf_map_update_elem(&perf_map, &idx, &pid, BPF_ANY);
39− return 0;
40−}
41−
42−char _license[] SEC("license") = "GPL";
43−u32 _version SEC("version") = LINUX_VERSION_CODE;
samples/bpf/test_current_task_under_cgroup_user.c+0−115
@@ -1,115 +0,0 @@
1−// SPDX-License-Identifier: GPL-2.0-only
2−/* Copyright (c) 2016 Sargun Dhillon <sargun@sargun.me>
3− */
4−
5−#define _GNU_SOURCE
6−#include <stdio.h>
7−#include <unistd.h>
8−#include <bpf/bpf.h>
9−#include <bpf/libbpf.h>
10−#include "cgroup_helpers.h"
11−
12−#define CGROUP_PATH "/my-cgroup"
13−
14−int main(int argc, char **argv)
15−{
16− pid_t remote_pid, local_pid = getpid();
17− int cg2 = -1, idx = 0, rc = 1;
18− struct bpf_link *link = NULL;
19− struct bpf_program *prog;
20− struct bpf_object *obj;
21− char filename[256];
22− int map_fd[2];
23−
24− snprintf(filename, sizeof(filename), "%s.bpf.o", argv[0]);
25− obj = bpf_object__open_file(filename, NULL);
26− if (libbpf_get_error(obj)) {
27− fprintf(stderr, "ERROR: opening BPF object file failed\n");
28− return 0;
29− }
30−
31− prog = bpf_object__find_program_by_name(obj, "bpf_prog1");
32− if (!prog) {
33− printf("finding a prog in obj file failed\n");
34− goto cleanup;
35− }
36−
37− /* load BPF program */
38− if (bpf_object__load(obj)) {
39− fprintf(stderr, "ERROR: loading BPF object file failed\n");
40− goto cleanup;
41− }
42−
43− map_fd[0] = bpf_object__find_map_fd_by_name(obj, "cgroup_map");
44− map_fd[1] = bpf_object__find_map_fd_by_name(obj, "perf_map");
45− if (map_fd[0] < 0 || map_fd[1] < 0) {
46− fprintf(stderr, "ERROR: finding a map in obj file failed\n");
47− goto cleanup;
48− }
49−
50− link = bpf_program__attach(prog);
51− if (libbpf_get_error(link)) {
52− fprintf(stderr, "ERROR: bpf_program__attach failed\n");
53− link = NULL;
54− goto cleanup;
55− }
56−
57− if (setup_cgroup_environment())
58− goto err;
59−
60− cg2 = create_and_get_cgroup(CGROUP_PATH);
61−
62− if (cg2 < 0)
63− goto err;
64−
65− if (bpf_map_update_elem(map_fd[0], &idx, &cg2, BPF_ANY)) {
66− log_err("Adding target cgroup to map");
67− goto err;
68− }
69−
70− if (join_cgroup(CGROUP_PATH))
71− goto err;
72−
73− /*
74− * The installed helper program catched the sync call, and should
75− * write it to the map.
76− */
77−
78− sync();
79− bpf_map_lookup_elem(map_fd[1], &idx, &remote_pid);
80−
81− if (local_pid != remote_pid) {
82− fprintf(stderr,
83− "BPF Helper didn't write correct PID to map, but: %d\n",
84− remote_pid);
85− goto err;
86− }
87−
88− /* Verify the negative scenario; leave the cgroup */
89− if (join_cgroup("/"))
90− goto err;
91−
92− remote_pid = 0;
93− bpf_map_update_elem(map_fd[1], &idx, &remote_pid, BPF_ANY);
94−
95− sync();
96− bpf_map_lookup_elem(map_fd[1], &idx, &remote_pid);
97−
98− if (local_pid == remote_pid) {
99− fprintf(stderr, "BPF cgroup negative test did not work\n");
100− goto err;
101− }
102−
103− rc = 0;
104−
105−err:
106− if (cg2 != -1)
107− close(cg2);
108−
109− cleanup_cgroup_environment();
110−
111−cleanup:
112− bpf_link__destroy(link);
113− bpf_object__close(obj);
114− return rc;
115−}
samples/bpf/test_overhead_kprobe.bpf.c+0−41
@@ -1,41 +0,0 @@
1−/* Copyright (c) 2016 Facebook
2− *
3− * This program is free software; you can redistribute it and/or
4− * modify it under the terms of version 2 of the GNU General Public
5− * License as published by the Free Software Foundation.
6− */
7−#include "vmlinux.h"
8−#include <linux/version.h>
9−#include <bpf/bpf_helpers.h>
10−#include <bpf/bpf_tracing.h>
11−#include <bpf/bpf_core_read.h>
12−
13−SEC("kprobe/__set_task_comm")
14−int prog(struct pt_regs *ctx)
15−{
16− struct signal_struct *signal;
17− struct task_struct *tsk;
18− char oldcomm[TASK_COMM_LEN] = {};
19− char newcomm[TASK_COMM_LEN] = {};
20− u16 oom_score_adj;
21− u32 pid;
22−
23− tsk = (void *)PT_REGS_PARM1_CORE(ctx);
24−
25− pid = BPF_CORE_READ(tsk, pid);
26− bpf_core_read_str(oldcomm, sizeof(oldcomm), &tsk->comm);
27− bpf_core_read_str(newcomm, sizeof(newcomm),
28− (void *)PT_REGS_PARM2(ctx));
29− signal = BPF_CORE_READ(tsk, signal);
30− oom_score_adj = BPF_CORE_READ(signal, oom_score_adj);
31− return 0;
32−}
33−
34−SEC("kprobe/fib_table_lookup")
35−int prog2(struct pt_regs *ctx)
36−{
37− return 0;
38−}
39−
40−char _license[] SEC("license") = "GPL";
41−u32 _version SEC("version") = LINUX_VERSION_CODE;
samples/bpf/test_overhead_raw_tp.bpf.c+0−17
@@ -1,17 +0,0 @@
1−// SPDX-License-Identifier: GPL-2.0
2−/* Copyright (c) 2018 Facebook */
3−#include "vmlinux.h"
4−#include <bpf/bpf_helpers.h>
5−
6−SEC("raw_tracepoint/task_rename")
7−int prog(struct bpf_raw_tracepoint_args *ctx)
8−{
9− return 0;
10−}
11−
12−SEC("raw_tracepoint/fib_table_lookup")
13−int prog2(struct bpf_raw_tracepoint_args *ctx)
14−{
15− return 0;
16−}
17−char _license[] SEC("license") = "GPL";
samples/bpf/test_overhead_tp.bpf.c+0−23
@@ -1,23 +0,0 @@
1−/* Copyright (c) 2016 Facebook
2− *
3− * This program is free software; you can redistribute it and/or
4− * modify it under the terms of version 2 of the GNU General Public
5− * License as published by the Free Software Foundation.
6− */
7−#include "vmlinux.h"
8−#include <bpf/bpf_helpers.h>
9−
10−/* from /sys/kernel/tracing/events/task/task_rename/format */
11−SEC("tracepoint/task/task_rename")
12−int prog(struct trace_event_raw_task_rename *ctx)
13−{
14− return 0;
15−}
16−
17−/* from /sys/kernel/tracing/events/fib/fib_table_lookup/format */
18−SEC("tracepoint/fib/fib_table_lookup")
19−int prog2(struct trace_event_raw_fib_table_lookup *ctx)
20−{
21− return 0;
22−}
23−char _license[] SEC("license") = "GPL";
samples/bpf/test_overhead_user.c+0−225
@@ -1,225 +0,0 @@
1−// SPDX-License-Identifier: GPL-2.0-only
2−/* Copyright (c) 2016 Facebook
3− */
4−#define _GNU_SOURCE
5−#include <sched.h>
6−#include <errno.h>
7−#include <stdio.h>
8−#include <sys/types.h>
9−#include <asm/unistd.h>
10−#include <fcntl.h>
11−#include <unistd.h>
12−#include <assert.h>
13−#include <sys/wait.h>
14−#include <sys/socket.h>
15−#include <arpa/inet.h>
16−#include <stdlib.h>
17−#include <signal.h>
18−#include <linux/bpf.h>
19−#include <string.h>
20−#include <time.h>
21−#include <bpf/bpf.h>
22−#include <bpf/libbpf.h>
23−
24−#define MAX_CNT 1000000
25−#define DUMMY_IP "127.0.0.1"
26−#define DUMMY_PORT 80
27−
28−static struct bpf_link *links[2];
29−static struct bpf_object *obj;
30−static int cnt;
31−
32−static __u64 time_get_ns(void)
33−{
34− struct timespec ts;
35−
36− clock_gettime(CLOCK_MONOTONIC, &ts);
37− return ts.tv_sec * 1000000000ull + ts.tv_nsec;
38−}
39−
40−static void test_task_rename(int cpu)
41−{
42− char buf[] = "test\n";
43− __u64 start_time;
44− int i, fd;
45−
46− fd = open("/proc/self/comm", O_WRONLY|O_TRUNC);
47− if (fd < 0) {
48− printf("couldn't open /proc\n");
49− exit(1);
50− }
51− start_time = time_get_ns();
52− for (i = 0; i < MAX_CNT; i++) {
53− if (write(fd, buf, sizeof(buf)) < 0) {
54− printf("task rename failed: %s\n", strerror(errno));
55− close(fd);
56− return;
57− }
58− }
59− printf("task_rename:%d: %lld events per sec\n",
60− cpu, MAX_CNT * 1000000000ll / (time_get_ns() - start_time));
61− close(fd);
62−}
63−
64−static void test_fib_table_lookup(int cpu)
65−{
66− struct sockaddr_in addr;
67− char buf[] = "test\n";
68− __u64 start_time;
69− int i, fd;
70−
71− fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
72− if (fd < 0) {
73− printf("couldn't open socket\n");
74− exit(1);
75− }
76− memset((char *)&addr, 0, sizeof(addr));
77− addr.sin_addr.s_addr = inet_addr(DUMMY_IP);
78− addr.sin_port = htons(DUMMY_PORT);
79− addr.sin_family = AF_INET;
80− start_time = time_get_ns();
81− for (i = 0; i < MAX_CNT; i++) {
82− if (sendto(fd, buf, strlen(buf), 0,
83− (struct sockaddr *)&addr, sizeof(addr)) < 0) {
84− printf("failed to start ping: %s\n", strerror(errno));
85− close(fd);
86− return;
87− }
88− }
89− printf("fib_table_lookup:%d: %lld events per sec\n",
90− cpu, MAX_CNT * 1000000000ll / (time_get_ns() - start_time));
91− close(fd);
92−}
93−
94−static void loop(int cpu, int flags)
95−{
96− cpu_set_t cpuset;
97−
98− CPU_ZERO(&cpuset);
99− CPU_SET(cpu, &cpuset);
100− sched_setaffinity(0, sizeof(cpuset), &cpuset);
101−
102− if (flags & 1)
103− test_task_rename(cpu);
104− if (flags & 2)
105− test_fib_table_lookup(cpu);
106−}
107−
108−static void run_perf_test(int tasks, int flags)
109−{
110− pid_t pid[tasks];
111− int i;
112−
113− for (i = 0; i < tasks; i++) {
114− pid[i] = fork();
115− if (pid[i] == 0) {
116− loop(i, flags);
117− exit(0);
118− } else if (pid[i] == -1) {
119− printf("couldn't spawn #%d process\n", i);
120− exit(1);
121− }
122− }
123− for (i = 0; i < tasks; i++) {
124− int status;
125−
126− assert(waitpid(pid[i], &status, 0) == pid[i]);
127− assert(status == 0);
128− }
129−}
130−
131−static int load_progs(char *filename)
132−{
133− struct bpf_program *prog;
134− int err = 0;
135−
136− obj = bpf_object__open_file(filename, NULL);
137− err = libbpf_get_error(obj);
138− if (err < 0) {
139− fprintf(stderr, "ERROR: opening BPF object file failed\n");
140− return err;
141− }
142−
143− /* load BPF program */
144− err = bpf_object__load(obj);
145− if (err < 0) {
146− fprintf(stderr, "ERROR: loading BPF object file failed\n");
147− return err;
148− }
149−
150− bpf_object__for_each_program(prog, obj) {
151− links[cnt] = bpf_program__attach(prog);
152− err = libbpf_get_error(links[cnt]);
153− if (err < 0) {
154− fprintf(stderr, "ERROR: bpf_program__attach failed\n");
155− links[cnt] = NULL;
156− return err;
157− }
158− cnt++;
159− }
160−
161− return err;
162−}
163−
164−static void unload_progs(void)
165−{
166− while (cnt)
167− bpf_link__destroy(links[--cnt]);
168−
169− bpf_object__close(obj);
170−}
171−
172−int main(int argc, char **argv)
173−{
174− int num_cpu = sysconf(_SC_NPROCESSORS_ONLN);
175− int test_flags = ~0;
176− char filename[256];
177− int err = 0;
178−
179−
180− if (argc > 1)
181− test_flags = atoi(argv[1]) ? : test_flags;
182− if (argc > 2)
183− num_cpu = atoi(argv[2]) ? : num_cpu;
184−
185− if (test_flags & 0x3) {
186− printf("BASE\n");
187− run_perf_test(num_cpu, test_flags);
188− }
189−
190− if (test_flags & 0xC) {
191− snprintf(filename, sizeof(filename),
192− "%s_kprobe.bpf.o", argv[0]);
193−
194− printf("w/KPROBE\n");
195− err = load_progs(filename);
196− if (!err)
197− run_perf_test(num_cpu, test_flags >> 2);
198−
199− unload_progs();
200− }
201−
202− if (test_flags & 0x30) {
203− snprintf(filename, sizeof(filename),
204− "%s_tp.bpf.o", argv[0]);
205− printf("w/TRACEPOINT\n");
206− err = load_progs(filename);
207− if (!err)
208− run_perf_test(num_cpu, test_flags >> 4);
209−
210− unload_progs();
211− }
212−
213− if (test_flags & 0xC0) {
214− snprintf(filename, sizeof(filename),
215− "%s_raw_tp.bpf.o", argv[0]);
216− printf("w/RAW_TRACEPOINT\n");
217− err = load_progs(filename);
218− if (!err)
219− run_perf_test(num_cpu, test_flags >> 6);
220−
221− unload_progs();
222− }
223−
224− return err;
225−}
samples/bpf/test_override_return.sh+0−16
@@ -1,16 +0,0 @@
1−#!/bin/bash
2−
3−rm -r tmpmnt
4−rm -f testfile.img
5−dd if=/dev/zero of=testfile.img bs=1M seek=1000 count=1
6−DEVICE=$(losetup --show -f testfile.img)
7−mkfs.btrfs -f $DEVICE
8−mkdir tmpmnt
9−./tracex7 $DEVICE
10−if [ $? -eq 0 ]
11−then
12− echo "SUCCESS!"
13−else
14− echo "FAILED!"
15−fi
16−losetup -d $DEVICE
samples/bpf/test_probe_write_user.bpf.c+0−52
@@ -1,52 +0,0 @@
1−/* Copyright (c) 2016 Sargun Dhillon <sargun@sargun.me>
2− *
3− * This program is free software; you can redistribute it and/or
4− * modify it under the terms of version 2 of the GNU General Public
5− * License as published by the Free Software Foundation.
6− */
7−#include "vmlinux.h"
8−#include <string.h>
9−#include <linux/version.h>
10−#include <bpf/bpf_helpers.h>
11−#include <bpf/bpf_tracing.h>
12−#include <bpf/bpf_core_read.h>
13−
14−struct {
15− __uint(type, BPF_MAP_TYPE_HASH);
16− __type(key, struct sockaddr_in);
17− __type(value, struct sockaddr_in);
18− __uint(max_entries, 256);
19−} dnat_map SEC(".maps");
20−
21−/* kprobe is NOT a stable ABI
22− * kernel functions can be removed, renamed or completely change semantics.
23− * Number of arguments and their positions can change, etc.
24− * In such case this bpf+kprobe example will no longer be meaningful
25− *
26− * This example sits on a syscall, and the syscall ABI is relatively stable
27− * of course, across platforms, and over time, the ABI may change.
28− */
29−SEC("ksyscall/connect")
30−int BPF_KSYSCALL(bpf_prog1, int fd, struct sockaddr_in *uservaddr,
31− int addrlen)
32−{
33− struct sockaddr_in new_addr, orig_addr = {};
34− struct sockaddr_in *mapped_addr;
35−
36− if (addrlen > sizeof(orig_addr))
37− return 0;
38−
39− if (bpf_probe_read_user(&orig_addr, sizeof(orig_addr), uservaddr) != 0)
40− return 0;
41−
42− mapped_addr = bpf_map_lookup_elem(&dnat_map, &orig_addr);
43− if (mapped_addr != NULL) {
44− memcpy(&new_addr, mapped_addr, sizeof(new_addr));
45− bpf_probe_write_user(uservaddr, &new_addr,
46− sizeof(new_addr));
47− }
48− return 0;
49−}
50−
51−char _license[] SEC("license") = "GPL";
52−u32 _version SEC("version") = LINUX_VERSION_CODE;
samples/bpf/test_probe_write_user_user.c+0−108
@@ -1,108 +0,0 @@
1−// SPDX-License-Identifier: GPL-2.0
2−#include <stdio.h>
3−#include <assert.h>
4−#include <unistd.h>
5−#include <bpf/bpf.h>
6−#include <bpf/libbpf.h>
7−#include <sys/socket.h>
8−#include <netinet/in.h>
9−#include <arpa/inet.h>
10−
11−int main(int ac, char **argv)
12−{
13− struct sockaddr_in *serv_addr_in, *mapped_addr_in, *tmp_addr_in;
14− struct sockaddr serv_addr, mapped_addr, tmp_addr;
15− int serverfd, serverconnfd, clientfd, map_fd;
16− struct bpf_link *link = NULL;
17− struct bpf_program *prog;
18− struct bpf_object *obj;
19− socklen_t sockaddr_len;
20− char filename[256];
21− char *ip;
22−
23− serv_addr_in = (struct sockaddr_in *)&serv_addr;
24− mapped_addr_in = (struct sockaddr_in *)&mapped_addr;
25− tmp_addr_in = (struct sockaddr_in *)&tmp_addr;
26−
27− snprintf(filename, sizeof(filename), "%s.bpf.o", argv[0]);
28− obj = bpf_object__open_file(filename, NULL);
29− if (libbpf_get_error(obj)) {
30− fprintf(stderr, "ERROR: opening BPF object file failed\n");
31− return 0;
32− }
33−
34− prog = bpf_object__find_program_by_name(obj, "bpf_prog1");
35− if (libbpf_get_error(prog)) {
36− fprintf(stderr, "ERROR: finding a prog in obj file failed\n");
37− goto cleanup;
38− }
39−
40− /* load BPF program */
41− if (bpf_object__load(obj)) {
42− fprintf(stderr, "ERROR: loading BPF object file failed\n");
43− goto cleanup;
44− }
45−
46− map_fd = bpf_object__find_map_fd_by_name(obj, "dnat_map");
47− if (map_fd < 0) {
48− fprintf(stderr, "ERROR: finding a map in obj file failed\n");
49− goto cleanup;
50− }
51−
52− link = bpf_program__attach(prog);
53− if (libbpf_get_error(link)) {
54− fprintf(stderr, "ERROR: bpf_program__attach failed\n");
55− link = NULL;
56− goto cleanup;
57− }
58−
59− assert((serverfd = socket(AF_INET, SOCK_STREAM, 0)) > 0);
60− assert((clientfd = socket(AF_INET, SOCK_STREAM, 0)) > 0);
61−
62− /* Bind server to ephemeral port on lo */
63− memset(&serv_addr, 0, sizeof(serv_addr));
64− serv_addr_in->sin_family = AF_INET;
65− serv_addr_in->sin_port = 0;
66− serv_addr_in->sin_addr.s_addr = htonl(INADDR_LOOPBACK);
67−
68− assert(bind(serverfd, &serv_addr, sizeof(serv_addr)) == 0);
69−
70− sockaddr_len = sizeof(serv_addr);
71− assert(getsockname(serverfd, &serv_addr, &sockaddr_len) == 0);
72− ip = inet_ntoa(serv_addr_in->sin_addr);
73− printf("Server bound to: %s:%d\n", ip, ntohs(serv_addr_in->sin_port));
74−
75− memset(&mapped_addr, 0, sizeof(mapped_addr));
76− mapped_addr_in->sin_family = AF_INET;
77− mapped_addr_in->sin_port = htons(5555);
78− mapped_addr_in->sin_addr.s_addr = inet_addr("255.255.255.255");
79−
80− assert(!bpf_map_update_elem(map_fd, &mapped_addr, &serv_addr, BPF_ANY));
81−
82− assert(listen(serverfd, 5) == 0);
83−
84− ip = inet_ntoa(mapped_addr_in->sin_addr);
85− printf("Client connecting to: %s:%d\n",
86− ip, ntohs(mapped_addr_in->sin_port));
87− assert(connect(clientfd, &mapped_addr, sizeof(mapped_addr)) == 0);
88−
89− sockaddr_len = sizeof(tmp_addr);
90− ip = inet_ntoa(tmp_addr_in->sin_addr);
91− assert((serverconnfd = accept(serverfd, &tmp_addr, &sockaddr_len)) > 0);
92− printf("Server received connection from: %s:%d\n",
93− ip, ntohs(tmp_addr_in->sin_port));
94−
95− sockaddr_len = sizeof(tmp_addr);
96− assert(getpeername(clientfd, &tmp_addr, &sockaddr_len) == 0);
97− ip = inet_ntoa(tmp_addr_in->sin_addr);
98− printf("Client's peer address: %s:%d\n",
99− ip, ntohs(tmp_addr_in->sin_port));
100−
101− /* Is the server's getsockname = the socket getpeername */
102− assert(memcmp(&serv_addr, &tmp_addr, sizeof(struct sockaddr_in)) == 0);
103−
104−cleanup:
105− bpf_link__destroy(link);
106− bpf_object__close(obj);
107− return 0;
108−}
samples/bpf/tracex7.bpf.c+0−15
@@ -1,15 +0,0 @@
1−#include "vmlinux.h"
2−#include <linux/version.h>
3−#include <bpf/bpf_helpers.h>
4−
5−SEC("kprobe/open_ctree")
6−int bpf_prog1(struct pt_regs *ctx)
7−{
8− unsigned long rc = -12;
9−
10− bpf_override_return(ctx, rc);
11− return 0;
12−}
13−
14−char _license[] SEC("license") = "GPL";
15−u32 _version SEC("version") = LINUX_VERSION_CODE;
samples/bpf/tracex7_user.c+0−56
@@ -1,56 +0,0 @@
1−#define _GNU_SOURCE
2−
3−#include <stdio.h>
4−#include <unistd.h>
5−#include <bpf/libbpf.h>
6−
7−int main(int argc, char **argv)
8−{
9− struct bpf_link *link = NULL;
10− struct bpf_program *prog;
11− struct bpf_object *obj;
12− char filename[256];
13− char command[256];
14− int ret = 0;
15− FILE *f;
16−
17− if (!argv[1]) {
18− fprintf(stderr, "ERROR: Run with the btrfs device argument!\n");
19− return 0;
20− }
21−
22− snprintf(filename, sizeof(filename), "%s.bpf.o", argv[0]);
23− obj = bpf_object__open_file(filename, NULL);
24− if (libbpf_get_error(obj)) {
25− fprintf(stderr, "ERROR: opening BPF object file failed\n");
26− return 0;
27− }
28−
29− prog = bpf_object__find_program_by_name(obj, "bpf_prog1");
30− if (!prog) {
31− fprintf(stderr, "ERROR: finding a prog in obj file failed\n");
32− goto cleanup;
33− }
34−
35− /* load BPF program */
36− if (bpf_object__load(obj)) {
37− fprintf(stderr, "ERROR: loading BPF object file failed\n");
38− goto cleanup;
39− }
40−
41− link = bpf_program__attach(prog);
42− if (libbpf_get_error(link)) {
43− fprintf(stderr, "ERROR: bpf_program__attach failed\n");
44− link = NULL;
45− goto cleanup;
46− }
47−
48− snprintf(command, 256, "mount %s tmpmnt/", argv[1]);
49− f = popen(command, "r");
50− ret = pclose(f);
51−
52−cleanup:
53− bpf_link__destroy(link);
54− bpf_object__close(obj);
55− return ret ? 0 : 1;
56−}
samples/bpf/xdp2skb_meta_kern.c+1−1
@@ -32,7 +32,7 @@ SEC("xdp_mark") · 31 unmodified lines
3232 int _xdp_mark(struct xdp_md *ctx)
3333 {
3434 struct meta_info *meta;
35− void *data, *data_end;
35+ void *data;
3636 int ret;
3737
3838 /* Reserve space in-front of data pointer for our meta info.
samples/bpf/xdp_adjust_tail_kern.c+1−0
@@ -57,6 +57,7 @@ static __always_inline void swap_mac(void *data, struct ethhdr *orig_eth) · 56 unmodified lines
5757
5858 static __always_inline __u16 csum_fold_helper(__u32 csum)
5959 {
60+ csum = (csum & 0xffff) + (csum >> 16);
6061 return ~((csum & 0xffff) + (csum >> 16));
6162 }
6263
scripts/Makefile.btf+4−2
@@ -3,6 +3,8 @@ · 2 unmodified lines
33 pahole-ver := $(CONFIG_PAHOLE_VERSION)
44 pahole-flags-y :=
55
6+JOBS := $(patsubst -j%,%,$(filter -j%,$(MAKEFLAGS)))
7+
68 ifeq ($(call test-le, $(pahole-ver), 125),y)
79
810 # pahole 1.18 through 1.21 can't handle zero-sized per-CPU vars
@@ -12,14 +14,14 @@ endif · 3 unmodified lines
1214
1315 pahole-flags-$(call test-ge, $(pahole-ver), 121) += --btf_gen_floats
1416
15−pahole-flags-$(call test-ge, $(pahole-ver), 122) += -j
17+pahole-flags-$(call test-ge, $(pahole-ver), 122) += -j$(JOBS)
1618
1719 pahole-flags-$(call test-ge, $(pahole-ver), 125) += --skip_encoding_btf_inconsistent_proto --btf_gen_optimized
1820
1921 else
2022
2123 # Switch to using --btf_features for v1.26 and later.
22−pahole-flags-$(call test-ge, $(pahole-ver), 126) = -j --btf_features=encode_force,var,float,enum64,decl_tag,type_tag,optimized_func,consistent_func,decl_tag_kfuncs
24+pahole-flags-$(call test-ge, $(pahole-ver), 126) = -j$(JOBS) --btf_features=encode_force,var,float,enum64,decl_tag,type_tag,optimized_func,consistent_func,decl_tag_kfuncs
2325
2426 ifneq ($(KBUILD_EXTMOD),)
2527 module-pahole-flags-$(call test-ge, $(pahole-ver), 126) += --btf_features=distilled_base
scripts/bpf_doc.py+50−3
@@ -37,10 +37,11 @@ class APIElement(object): · 36 unmodified lines
3737 @desc: textual description of the symbol
3838 @ret: (optional) description of any associated return value
3939 """
40− def __init__(self, proto='', desc='', ret=''):
40+ def __init__(self, proto='', desc='', ret='', attrs=[]):
4141 self.proto = proto
4242 self.desc = desc
4343 self.ret = ret
44+ self.attrs = attrs
4445
4546
4647 class Helper(APIElement):
@@ -81,6 +82,11 @@ class Helper(APIElement): · 34 unmodified lines
8182 return res
8283
8384
85+ATTRS = {
86+ '__bpf_fastcall': 'bpf_fastcall'
87+}
88+
89+
8490 class HeaderParser(object):
8591 """
8692 An object used to parse a file in order to extract the documentation of a
@@ -111,7 +117,8 @@ class HeaderParser(object): · 24 unmodified lines
111117 proto = self.parse_proto()
112118 desc = self.parse_desc(proto)
113119 ret = self.parse_ret(proto)
114− return Helper(proto=proto, desc=desc, ret=ret)
120+ attrs = self.parse_attrs(proto)
121+ return Helper(proto=proto, desc=desc, ret=ret, attrs=attrs)
115122
116123 def parse_symbol(self):
117124 p = re.compile(r' \* ?(BPF\w+)$')
@@ -192,6 +199,28 @@ class HeaderParser(object): · 74 unmodified lines
192199 raise Exception("No return found for " + proto)
193200 return ret
194201
202+ def parse_attrs(self, proto):
203+ p = re.compile(r' \* ?(?:\t| {5,8})Attributes$')
204+ capture = p.match(self.line)
205+ if not capture:
206+ return []
207+ # Expect a single line with mnemonics for attributes separated by spaces
208+ self.line = self.reader.readline()
209+ p = re.compile(r' \* ?(?:\t| {5,8})(?:\t| {8})(.*)')
210+ capture = p.match(self.line)
211+ if not capture:
212+ raise Exception("Incomplete 'Attributes' section for " + proto)
213+ attrs = capture.group(1).split(' ')
214+ for attr in attrs:
215+ if attr not in ATTRS:
216+ raise Exception("Unexpected attribute '" + attr + "' specified for " + proto)
217+ self.line = self.reader.readline()
218+ if self.line != ' *\n':
219+ raise Exception("Expecting empty line after 'Attributes' section for " + proto)
220+ # Prepare a line for next self.parse_* to consume
221+ self.line = self.reader.readline()
222+ return attrs
223+
195224 def seek_to(self, target, help_message, discard_lines = 1):
196225 self.reader.seek(0)
197226 offset = self.reader.read().find(target)
@@ -789,6 +818,21 @@ class PrinterHelpers(Printer): · 591 unmodified lines
789818 print('%s;' % fwd)
790819 print('')
791820
821+ used_attrs = set()
822+ for helper in self.elements:
823+ for attr in helper.attrs:
824+ used_attrs.add(attr)
825+ for attr in sorted(used_attrs):
826+ print('#ifndef %s' % attr)
827+ print('#if __has_attribute(%s)' % ATTRS[attr])
828+ print('#define %s __attribute__((%s))' % (attr, ATTRS[attr]))
829+ print('#else')
830+ print('#define %s' % attr)
831+ print('#endif')
832+ print('#endif')
833+ if used_attrs:
834+ print('')
835+
792836 def print_footer(self):
793837 footer = ''
794838 print(footer)
@@ -827,7 +871,10 @@ class PrinterHelpers(Printer): · 32 unmodified lines
827871 print(' *{}{}'.format(' \t' if line else '', line))
828872
829873 print(' */')
830− print('static %s %s(* const %s)(' % (self.map_type(proto['ret_type']),
874+ print('static ', end='')
875+ if helper.attrs:
876+ print('%s ' % (" ".join(helper.attrs)), end='')
877+ print('%s %s(* const %s)(' % (self.map_type(proto['ret_type']),
831878 proto['ret_star'], proto['name']), end='')
832879 comma = ''
833880 for i, a in enumerate(proto['args']):
tools/bpf/bpf_jit_disasm.c+1−1
@@ -210,7 +210,7 @@ static uint8_t *get_last_jit_image(char *haystack, size_t hlen, · 209 unmodified lines
210210 return NULL;
211211 }
212212 if (proglen > 1000000) {
213− printf("proglen of %d too big, stopping\n", proglen);
213+ printf("proglen of %u too big, stopping\n", proglen);
214214 return NULL;
215215 }
216216
tools/bpf/bpftool/Makefile+5−1
@@ -147,7 +147,11 @@ ifeq ($(feature-llvm),1) · 146 unmodified lines
147147 # If LLVM is available, use it for JIT disassembly
148148 CFLAGS += -DHAVE_LLVM_SUPPORT
149149 LLVM_CONFIG_LIB_COMPONENTS := mcdisassembler all-targets
150− CFLAGS += $(shell $(LLVM_CONFIG) --cflags)
150+ # llvm-config always adds -D_GNU_SOURCE, however, it may already be in CFLAGS
151+ # (e.g. when bpftool build is called from selftests build as selftests
152+ # Makefile includes lib.mk which sets -D_GNU_SOURCE) which would cause
153+ # compilation error due to redefinition. Let's filter it out here.
154+ CFLAGS += $(filter-out -D_GNU_SOURCE,$(shell $(LLVM_CONFIG) --cflags))
151155 LIBS += $(shell $(LLVM_CONFIG) --libs $(LLVM_CONFIG_LIB_COMPONENTS))
152156 ifeq ($(shell $(LLVM_CONFIG) --shared-mode),static)
153157 LIBS += $(shell $(LLVM_CONFIG) --system-libs $(LLVM_CONFIG_LIB_COMPONENTS))
tools/bpf/bpftool/btf.c+89−11
@@ -1,11 +1,15 @@
11 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
22 /* Copyright (C) 2019 Facebook */
33
4+#ifndef _GNU_SOURCE
5+#define _GNU_SOURCE
6+#endif
47 #include <errno.h>
58 #include <fcntl.h>
69 #include <linux/err.h>
710 #include <stdbool.h>
811 #include <stdio.h>
12+#include <stdlib.h>
913 #include <string.h>
1014 #include <unistd.h>
1115 #include <linux/btf.h>
@@ -21,6 +25,7 @@ · 9 unmodified lines
2125 #include "main.h"
2226
2327 #define KFUNC_DECL_TAG "bpf_kfunc"
28+#define FASTCALL_DECL_TAG "bpf_fastcall"
2429
2530 static const char * const btf_kind_str[NR_BTF_KINDS] = {
2631 [BTF_KIND_UNKN] = "UNKNOWN",
@@ -284,7 +289,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id, · 257 unmodified lines
284289 } else {
285290 if (btf_kflag(t))
286291 printf("\n\t'%s' val=%lldLL", name,
287− (unsigned long long)val);
292+ (long long)val);
288293 else
289294 printf("\n\t'%s' val=%lluULL", name,
290295 (unsigned long long)val);
@@ -464,19 +469,59 @@ static int dump_btf_raw(const struct btf *btf, · 173 unmodified lines
464469 return 0;
465470 }
466471
472+struct ptr_array {
473+ __u32 cnt;
474+ __u32 cap;
475+ const void **elems;
476+};
477+
478+static int ptr_array_push(const void *ptr, struct ptr_array *arr)
479+{
480+ __u32 new_cap;
481+ void *tmp;
482+
483+ if (arr->cnt == arr->cap) {
484+ new_cap = (arr->cap ?: 16) * 2;
485+ tmp = realloc(arr->elems, sizeof(*arr->elems) * new_cap);
486+ if (!tmp)
487+ return -ENOMEM;
488+ arr->elems = tmp;
489+ arr->cap = new_cap;
490+ }
491+ arr->elems[arr->cnt++] = ptr;
492+ return 0;
493+}
494+
495+static void ptr_array_free(struct ptr_array *arr)
496+{
497+ free(arr->elems);
498+}
499+
500+static int cmp_kfuncs(const void *pa, const void *pb, void *ctx)
501+{
502+ struct btf *btf = ctx;
503+ const struct btf_type *a = *(void **)pa;
504+ const struct btf_type *b = *(void **)pb;
505+
506+ return strcmp(btf__str_by_offset(btf, a->name_off),
507+ btf__str_by_offset(btf, b->name_off));
508+}
509+
467510 static int dump_btf_kfuncs(struct btf_dump *d, const struct btf *btf)
468511 {
469512 LIBBPF_OPTS(btf_dump_emit_type_decl_opts, opts);
470− int cnt = btf__type_cnt(btf);
471− int i;
513+ __u32 cnt = btf__type_cnt(btf), i, j;
514+ struct ptr_array fastcalls = {};
515+ struct ptr_array kfuncs = {};
516+ int err = 0;
472517
473518 printf("\n/* BPF kfuncs */\n");
474519 printf("#ifndef BPF_NO_KFUNC_PROTOTYPES\n");
475520
476521 for (i = 1; i < cnt; i++) {
477522 const struct btf_type *t = btf__type_by_id(btf, i);
523+ const struct btf_type *ft;
478524 const char *name;
479− int err;
480525
481526 if (!btf_is_decl_tag(t))
482527 continue;
@@ -484,27 +529,53 @@ static int dump_btf_kfuncs(struct btf_dump *d, const struct btf *btf) · 1 unmodified line
484529 if (btf_decl_tag(t)->component_idx != -1)
485530 continue;
486531
487− name = btf__name_by_offset(btf, t->name_off);
488− if (strncmp(name, KFUNC_DECL_TAG, sizeof(KFUNC_DECL_TAG)))
532+ ft = btf__type_by_id(btf, t->type);
533+ if (!btf_is_func(ft))
489534 continue;
490535
491− t = btf__type_by_id(btf, t->type);
492− if (!btf_is_func(t))
493− continue;
536+ name = btf__name_by_offset(btf, t->name_off);
537+ if (strncmp(name, KFUNC_DECL_TAG, sizeof(KFUNC_DECL_TAG)) == 0) {
538+ err = ptr_array_push(ft, &kfuncs);
539+ if (err)
540+ goto out;
541+ }
542+
543+ if (strncmp(name, FASTCALL_DECL_TAG, sizeof(FASTCALL_DECL_TAG)) == 0) {
544+ err = ptr_array_push(ft, &fastcalls);
545+ if (err)
546+ goto out;
547+ }
548+ }
549+
550+ /* Sort kfuncs by name for improved vmlinux.h stability */
551+ qsort_r(kfuncs.elems, kfuncs.cnt, sizeof(*kfuncs.elems), cmp_kfuncs, (void *)btf);
552+ for (i = 0; i < kfuncs.cnt; i++) {
553+ const struct btf_type *t = kfuncs.elems[i];
494554
495555 printf("extern ");
496556
557+ /* Assume small amount of fastcall kfuncs */
558+ for (j = 0; j < fastcalls.cnt; j++) {
559+ if (fastcalls.elems[j] == t) {
560+ printf("__bpf_fastcall ");
561+ break;
562+ }
563+ }
564+
497565 opts.field_name = btf__name_by_offset(btf, t->name_off);
498566 err = btf_dump__emit_type_decl(d, t->type, &opts);
499567 if (err)
500− return err;
568+ goto out;
501569
502570 printf(" __weak __ksym;\n");
503571 }
504572
505573 printf("#endif\n\n");
506574
507− return 0;
575+out:
576+ ptr_array_free(&fastcalls);
577+ ptr_array_free(&kfuncs);
578+ return err;
508579 }
509580
510581 static void __printf(2, 0) btf_dump_printf(void *ctx,
@@ -718,6 +789,13 @@ static int dump_btf_c(const struct btf *btf, · 207 unmodified lines
718789 printf("#ifndef __weak\n");
719790 printf("#define __weak __attribute__((weak))\n");
720791 printf("#endif\n\n");
792+ printf("#ifndef __bpf_fastcall\n");
793+ printf("#if __has_attribute(bpf_fastcall)\n");
794+ printf("#define __bpf_fastcall __attribute__((bpf_fastcall))\n");
795+ printf("#else\n");
796+ printf("#define __bpf_fastcall\n");
797+ printf("#endif\n");
798+ printf("#endif\n\n");
721799
722800 if (root_type_cnt) {
723801 for (i = 0; i < root_type_cnt; i++) {
tools/bpf/bpftool/jit_disasm.c+29−11
@@ -80,7 +80,8 @@ symbol_lookup_callback(__maybe_unused void *disasm_info, · 79 unmodified lines
8080 static int
8181 init_context(disasm_ctx_t *ctx, const char *arch,
8282 __maybe_unused const char *disassembler_options,
83− __maybe_unused unsigned char *image, __maybe_unused ssize_t len)
83+ __maybe_unused unsigned char *image, __maybe_unused ssize_t len,
84+ __maybe_unused __u64 func_ksym)
8485 {
8586 char *triple;
8687
@@ -109,12 +110,13 @@ static void destroy_context(disasm_ctx_t *ctx) · 22 unmodified lines
109110 }
110111
111112 static int
112−disassemble_insn(disasm_ctx_t *ctx, unsigned char *image, ssize_t len, int pc)
113+disassemble_insn(disasm_ctx_t *ctx, unsigned char *image, ssize_t len, int pc,
114+ __u64 func_ksym)
113115 {
114116 char buf[256];
115117 int count;
116118
117− count = LLVMDisasmInstruction(*ctx, image + pc, len - pc, pc,
119+ count = LLVMDisasmInstruction(*ctx, image + pc, len - pc, func_ksym + pc,
118120 buf, sizeof(buf));
119121 if (json_output)
120122 printf_json(buf);
@@ -136,8 +138,21 @@ int disasm_init(void) · 15 unmodified lines
136138 #ifdef HAVE_LIBBFD_SUPPORT
137139 #define DISASM_SPACER "\t"
138140
141+struct disasm_info {
142+ struct disassemble_info info;
143+ __u64 func_ksym;
144+};
145+
146+static void disasm_print_addr(bfd_vma addr, struct disassemble_info *info)
147+{
148+ struct disasm_info *dinfo = container_of(info, struct disasm_info, info);
149+
150+ addr += dinfo->func_ksym;
151+ generic_print_address(addr, info);
152+}
153+
139154 typedef struct {
140− struct disassemble_info *info;
155+ struct disasm_info *info;
141156 disassembler_ftype disassemble;
142157 bfd *bfdf;
143158 } disasm_ctx_t;
@@ -215,7 +230,7 @@ static int fprintf_json_styled(void *out, · 71 unmodified lines
215230
216231 static int init_context(disasm_ctx_t *ctx, const char *arch,
217232 const char *disassembler_options,
218− unsigned char *image, ssize_t len)
233+ unsigned char *image, ssize_t len, __u64 func_ksym)
219234 {
220235 struct disassemble_info *info;
221236 char tpath[PATH_MAX];
@@ -238,12 +253,13 @@ static int init_context(disasm_ctx_t *ctx, const char *arch, · 16 unmodified lines
238253 }
239254 bfdf = ctx->bfdf;
240255
241− ctx->info = malloc(sizeof(struct disassemble_info));
256+ ctx->info = malloc(sizeof(struct disasm_info));
242257 if (!ctx->info) {
243258 p_err("mem alloc failed");
244259 goto err_close;
245260 }
246− info = ctx->info;
261+ ctx->info->func_ksym = func_ksym;
262+ info = &ctx->info->info;
247263
248264 if (json_output)
249265 init_disassemble_info_compat(info, stdout,
@@ -272,6 +288,7 @@ static int init_context(disasm_ctx_t *ctx, const char *arch, · 22 unmodified lines
272288 info->disassembler_options = disassembler_options;
273289 info->buffer = image;
274290 info->buffer_length = len;
291+ info->print_address_func = disasm_print_addr;
275292
276293 disassemble_init_for_target(info);
277294
@@ -304,9 +321,10 @@ static void destroy_context(disasm_ctx_t *ctx) · 26 unmodified lines
304321
305322 static int
306323 disassemble_insn(disasm_ctx_t *ctx, __maybe_unused unsigned char *image,
307− __maybe_unused ssize_t len, int pc)
324+ __maybe_unused ssize_t len, int pc,
325+ __maybe_unused __u64 func_ksym)
308326 {
309− return ctx->disassemble(pc, ctx->info);
327+ return ctx->disassemble(pc, &ctx->info->info);
310328 }
311329
312330 int disasm_init(void)
@@ -331,7 +349,7 @@ int disasm_print_insn(unsigned char *image, ssize_t len, int opcodes, · 18 unmodified lines
331349 if (!len)
332350 return -1;
333351
334− if (init_context(&ctx, arch, disassembler_options, image, len))
352+ if (init_context(&ctx, arch, disassembler_options, image, len, func_ksym))
335353 return -1;
336354
337355 if (json_output)
@@ -360,7 +378,7 @@ int disasm_print_insn(unsigned char *image, ssize_t len, int opcodes, · 22 unmodified lines
360378 printf("%4x:" DISASM_SPACER, pc);
361379 }
362380
363− count = disassemble_insn(&ctx, image, len, pc);
381+ count = disassemble_insn(&ctx, image, len, pc, func_ksym);
364382
365383 if (json_output) {
366384 /* Operand array, was started in fprintf_json. Before
tools/bpf/resolve_btfids/main.c+2−2
@@ -679,8 +679,8 @@ static int sets_patch(struct object *obj) · 678 unmodified lines
679679
680680 next = rb_first(&obj->sets);
681681 while (next) {
682− struct btf_id_set8 *set8;
683− struct btf_id_set *set;
682+ struct btf_id_set8 *set8 = NULL;
683+ struct btf_id_set *set = NULL;
684684 unsigned long addr, off;
685685 struct btf_id *id;
686686
tools/bpf/runqslower/runqslower.bpf.c+0−1
@@ -70,7 +70,6 @@ int handle__sched_switch(u64 *ctx) · 69 unmodified lines
7070 struct task_struct *next = (struct task_struct *)ctx[2];
7171 struct runq_event event = {};
7272 u64 *tsp, delta_us;
73− long state;
7473 u32 pid;
7574
7675 /* ivcsw: treat like an enqueue event and store timestamp */
tools/include/uapi/linux/bpf.h+4−5
@@ -1116,6 +1116,7 @@ enum bpf_attach_type { · 1115 unmodified lines
11161116 BPF_NETKIT_PRIMARY,
11171117 BPF_NETKIT_PEER,
11181118 BPF_TRACE_KPROBE_SESSION,
1119+ BPF_TRACE_UPROBE_SESSION,
11191120 __MAX_BPF_ATTACH_TYPE
11201121 };
11211122
@@ -1973,6 +1974,8 @@ union bpf_attr { · 851 unmodified lines
19731974 * program.
19741975 * Return
19751976 * The SMP id of the processor running the program.
1977+ * Attributes
1978+ * __bpf_fastcall
19761979 *
19771980 * long bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from, u32 len, u64 flags)
19781981 * Description
@@ -3104,10 +3107,6 @@ union bpf_attr { · 1125 unmodified lines
31043107 * with the **CONFIG_BPF_KPROBE_OVERRIDE** configuration
31053108 * option, and in this case it only works on functions tagged with
31063109 * **ALLOW_ERROR_INJECTION** in the kernel code.
3107− *
3108− * Also, the helper is only available for the architectures having
3109− * the CONFIG_FUNCTION_ERROR_INJECTION option. As of this writing,
3110− * x86 architecture is the only one to support this feature.
31113110 * Return
31123111 * 0
31133112 *
@@ -5372,7 +5371,7 @@ union bpf_attr { · 2258 unmodified lines
53725371 * Currently, the **flags** must be 0. Currently, nr_loops is
53735372 * limited to 1 << 23 (~8 million) loops.
53745373 *
5375− * long (\*callback_fn)(u32 index, void \*ctx);
5374+ * long (\*callback_fn)(u64 index, void \*ctx);
53765375 *
53775376 * where **index** is the current index in the loop. The index
53785377 * is zero-indexed.
tools/lib/bpf/Makefile+2−1
@@ -61,7 +61,8 @@ ifndef VERBOSE · 60 unmodified lines
6161 endif
6262
6363 INCLUDES = -I$(or $(OUTPUT),.) \
64− -I$(srctree)/tools/include -I$(srctree)/tools/include/uapi
64+ -I$(srctree)/tools/include -I$(srctree)/tools/include/uapi \
65+ -I$(srctree)/tools/arch/$(SRCARCH)/include
6566
6667 export prefix libdir src obj
6768
tools/lib/bpf/bpf.c+1−0
@@ -776,6 +776,7 @@ int bpf_link_create(int prog_fd, int target_fd, · 775 unmodified lines
776776 return libbpf_err(-EINVAL);
777777 break;
778778 case BPF_TRACE_UPROBE_MULTI:
779+ case BPF_TRACE_UPROBE_SESSION:
779780 attr.link_create.uprobe_multi.flags = OPTS_GET(opts, uprobe_multi.flags, 0);
780781 attr.link_create.uprobe_multi.cnt = OPTS_GET(opts, uprobe_multi.cnt, 0);
781782 attr.link_create.uprobe_multi.path = ptr_to_u64(OPTS_GET(opts, uprobe_multi.path, 0));
tools/lib/bpf/bpf_gen_internal.h+1−0
@@ -34,6 +34,7 @@ struct bpf_gen { · 33 unmodified lines
3434 void *data_cur;
3535 void *insn_start;
3636 void *insn_cur;
37+ bool swapped_endian;
3738 ssize_t cleanup_label;
3839 __u32 nr_progs;
3940 __u32 nr_maps;
tools/lib/bpf/bpf_helpers.h+1−0
@@ -185,6 +185,7 @@ enum libbpf_tristate { · 184 unmodified lines
185185 #define __kptr_untrusted __attribute__((btf_type_tag("kptr_untrusted")))
186186 #define __kptr __attribute__((btf_type_tag("kptr")))
187187 #define __percpu_kptr __attribute__((btf_type_tag("percpu_kptr")))
188+#define __uptr __attribute__((btf_type_tag("uptr")))
188189
189190 #if defined (__clang__)
190191 #define bpf_ksym_exists(sym) ({ \
tools/lib/bpf/btf.c+236−72
@@ -22,6 +22,7 @@ · 21 unmodified lines
2222 #include "libbpf_internal.h"
2323 #include "hashmap.h"
2424 #include "strset.h"
25+#include "str_error.h"
2526
2627 #define BTF_MAX_NR_TYPES 0x7fffffffU
2728 #define BTF_MAX_STR_OFFSET 0x7fffffffU
@@ -1179,7 +1180,7 @@ static struct btf *btf_parse_elf(const char *path, struct btf *base_btf, · 1151 unmodified lines
11791180 fd = open(path, O_RDONLY | O_CLOEXEC);
11801181 if (fd < 0) {
11811182 err = -errno;
1182− pr_warn("failed to open %s: %s\n", path, strerror(errno));
1183+ pr_warn("failed to open %s: %s\n", path, errstr(err));
11831184 return ERR_PTR(err);
11841185 }
11851186
@@ -1445,7 +1446,7 @@ retry_load: · 259 unmodified lines
14451446 goto retry_load;
14461447
14471448 err = -errno;
1448− pr_warn("BTF loading error: %d\n", err);
1449+ pr_warn("BTF loading error: %s\n", errstr(err));
14491450 /* don't print out contents of custom log_buf */
14501451 if (!log_buf && buf[0])
14511452 pr_warn("-- BEGIN BTF LOAD LOG ---\n%s\n-- END BTF LOAD LOG --\n", buf);
@@ -2885,7 +2886,7 @@ int btf__add_decl_tag(struct btf *btf, const char *value, int ref_type_id, · 1433 unmodified lines
28852886 return btf_commit_type(btf, sz);
28862887 }
28872888
2888−struct btf_ext_sec_setup_param {
2889+struct btf_ext_sec_info_param {
28892890 __u32 off;
28902891 __u32 len;
28912892 __u32 min_rec_size;
@@ -2893,14 +2894,20 @@ struct btf_ext_sec_setup_param { · 1 unmodified line
28932894 const char *desc;
28942895 };
28952896
2896−static int btf_ext_setup_info(struct btf_ext *btf_ext,
2897− struct btf_ext_sec_setup_param *ext_sec)
2897+/*
2898+ * Parse a single info subsection of the BTF.ext info data:
2899+ * - validate subsection structure and elements
2900+ * - save info subsection start and sizing details in struct btf_ext
2901+ * - endian-independent operation, for calling before byte-swapping
2902+ */
2903+static int btf_ext_parse_sec_info(struct btf_ext *btf_ext,
2904+ struct btf_ext_sec_info_param *ext_sec,
2905+ bool is_native)
28982906 {
28992907 const struct btf_ext_info_sec *sinfo;
29002908 struct btf_ext_info *ext_info;
29012909 __u32 info_left, record_size;
29022910 size_t sec_cnt = 0;
2903− /* The start of the info sec (including the __u32 record_size). */
29042911 void *info;
29052912
29062913 if (ext_sec->len == 0)
@@ -2912,6 +2919,7 @@ static int btf_ext_setup_info(struct btf_ext *btf_ext, · 5 unmodified lines
29122919 return -EINVAL;
29132920 }
29142921
2922+ /* The start of the info sec (including the __u32 record_size). */
29152923 info = btf_ext->data + btf_ext->hdr->hdr_len + ext_sec->off;
29162924 info_left = ext_sec->len;
29172925
@@ -2927,9 +2935,13 @@ static int btf_ext_setup_info(struct btf_ext *btf_ext, · 9 unmodified lines
29272935 return -EINVAL;
29282936 }
29292937
2930− /* The record size needs to meet the minimum standard */
2931− record_size = *(__u32 *)info;
2938+ /* The record size needs to meet either the minimum standard or, when
2939+ * handling non-native endianness data, the exact standard so as
2940+ * to allow safe byte-swapping.
2941+ */
2942+ record_size = is_native ? *(__u32 *)info : bswap_32(*(__u32 *)info);
29322943 if (record_size < ext_sec->min_rec_size ||
2944+ (!is_native && record_size != ext_sec->min_rec_size) ||
29332945 record_size & 0x03) {
29342946 pr_debug("%s section in .BTF.ext has invalid record size %u\n",
29352947 ext_sec->desc, record_size);
@@ -2941,7 +2953,7 @@ static int btf_ext_setup_info(struct btf_ext *btf_ext, · 5 unmodified lines
29412953
29422954 /* If no records, return failure now so .BTF.ext won't be used. */
29432955 if (!info_left) {
2944− pr_debug("%s section in .BTF.ext has no records", ext_sec->desc);
2956+ pr_debug("%s section in .BTF.ext has no records\n", ext_sec->desc);
29452957 return -EINVAL;
29462958 }
29472959
@@ -2956,7 +2968,7 @@ static int btf_ext_setup_info(struct btf_ext *btf_ext, · 8 unmodified lines
29562968 return -EINVAL;
29572969 }
29582970
2959− num_records = sinfo->num_info;
2971+ num_records = is_native ? sinfo->num_info : bswap_32(sinfo->num_info);
29602972 if (num_records == 0) {
29612973 pr_debug("%s section has incorrect num_records in .BTF.ext\n",
29622974 ext_sec->desc);
@@ -2984,64 +2996,157 @@ static int btf_ext_setup_info(struct btf_ext *btf_ext, · 21 unmodified lines
29842996 return 0;
29852997 }
29862998
2987−static int btf_ext_setup_func_info(struct btf_ext *btf_ext)
2999+/* Parse all info secs in the BTF.ext info data */
3000+static int btf_ext_parse_info(struct btf_ext *btf_ext, bool is_native)
29883001 {
2989− struct btf_ext_sec_setup_param param = {
3002+ struct btf_ext_sec_info_param func_info = {
29903003 .off = btf_ext->hdr->func_info_off,
29913004 .len = btf_ext->hdr->func_info_len,
29923005 .min_rec_size = sizeof(struct bpf_func_info_min),
29933006 .ext_info = &btf_ext->func_info,
29943007 .desc = "func_info"
29953008 };
2996−
2997− return btf_ext_setup_info(btf_ext, &param);
2998−}
2999−
3000−static int btf_ext_setup_line_info(struct btf_ext *btf_ext)
3001−{
3002− struct btf_ext_sec_setup_param param = {
3009+ struct btf_ext_sec_info_param line_info = {
30033010 .off = btf_ext->hdr->line_info_off,
30043011 .len = btf_ext->hdr->line_info_len,
30053012 .min_rec_size = sizeof(struct bpf_line_info_min),
30063013 .ext_info = &btf_ext->line_info,
30073014 .desc = "line_info",
30083015 };
3009−
3010− return btf_ext_setup_info(btf_ext, &param);
3011−}
3012−
3013−static int btf_ext_setup_core_relos(struct btf_ext *btf_ext)
3014−{
3015− struct btf_ext_sec_setup_param param = {
3016+ struct btf_ext_sec_info_param core_relo = {
30163017 .off = btf_ext->hdr->core_relo_off,
30173018 .len = btf_ext->hdr->core_relo_len,
30183019 .min_rec_size = sizeof(struct bpf_core_relo),
30193020 .ext_info = &btf_ext->core_relo_info,
30203021 .desc = "core_relo",
30213022 };
3023+ int err;
3024+
3025+ err = btf_ext_parse_sec_info(btf_ext, &func_info, is_native);
3026+ if (err)
3027+ return err;
3028+
3029+ err = btf_ext_parse_sec_info(btf_ext, &line_info, is_native);
3030+ if (err)
3031+ return err;
3032+
3033+ if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, core_relo_len))
3034+ return 0; /* skip core relos parsing */
3035+
3036+ err = btf_ext_parse_sec_info(btf_ext, &core_relo, is_native);
3037+ if (err)
3038+ return err;
3039+
3040+ return 0;
3041+}
3042+
3043+/* Swap byte-order of BTF.ext header with any endianness */
3044+static void btf_ext_bswap_hdr(struct btf_ext_header *h)
3045+{
3046+ bool is_native = h->magic == BTF_MAGIC;
3047+ __u32 hdr_len;
3048+
3049+ hdr_len = is_native ? h->hdr_len : bswap_32(h->hdr_len);
3050+
3051+ h->magic = bswap_16(h->magic);
3052+ h->hdr_len = bswap_32(h->hdr_len);
3053+ h->func_info_off = bswap_32(h->func_info_off);
3054+ h->func_info_len = bswap_32(h->func_info_len);
3055+ h->line_info_off = bswap_32(h->line_info_off);
3056+ h->line_info_len = bswap_32(h->line_info_len);
3057+
3058+ if (hdr_len < offsetofend(struct btf_ext_header, core_relo_len))
3059+ return;
3060+
3061+ h->core_relo_off = bswap_32(h->core_relo_off);
3062+ h->core_relo_len = bswap_32(h->core_relo_len);
3063+}
3064+
3065+/* Swap byte-order of generic info subsection */
3066+static void btf_ext_bswap_info_sec(void *info, __u32 len, bool is_native,
3067+ info_rec_bswap_fn bswap_fn)
3068+{
3069+ struct btf_ext_info_sec *sec;
3070+ __u32 info_left, rec_size, *rs;
3071+
3072+ if (len == 0)
3073+ return;
3074+
3075+ rs = info; /* info record size */
3076+ rec_size = is_native ? *rs : bswap_32(*rs);
3077+ *rs = bswap_32(*rs);
3078+
3079+ sec = info + sizeof(__u32); /* info sec #1 */
3080+ info_left = len - sizeof(__u32);
3081+ while (info_left) {
3082+ unsigned int sec_hdrlen = sizeof(struct btf_ext_info_sec);
3083+ __u32 i, num_recs;
3084+ void *p;
3085+
3086+ num_recs = is_native ? sec->num_info : bswap_32(sec->num_info);
3087+ sec->sec_name_off = bswap_32(sec->sec_name_off);
3088+ sec->num_info = bswap_32(sec->num_info);
3089+ p = sec->data; /* info rec #1 */
3090+ for (i = 0; i < num_recs; i++, p += rec_size)
3091+ bswap_fn(p);
3092+ sec = p;
3093+ info_left -= sec_hdrlen + (__u64)rec_size * num_recs;
3094+ }
3095+}
3096+
3097+/*
3098+ * Swap byte-order of all info data in a BTF.ext section
3099+ * - requires BTF.ext hdr in native endianness
3100+ */
3101+static void btf_ext_bswap_info(struct btf_ext *btf_ext, void *data)
3102+{
3103+ const bool is_native = btf_ext->swapped_endian;
3104+ const struct btf_ext_header *h = data;
3105+ void *info;
3106+
3107+ /* Swap func_info subsection byte-order */
3108+ info = data + h->hdr_len + h->func_info_off;
3109+ btf_ext_bswap_info_sec(info, h->func_info_len, is_native,
3110+ (info_rec_bswap_fn)bpf_func_info_bswap);
3111+
3112+ /* Swap line_info subsection byte-order */
3113+ info = data + h->hdr_len + h->line_info_off;
3114+ btf_ext_bswap_info_sec(info, h->line_info_len, is_native,
3115+ (info_rec_bswap_fn)bpf_line_info_bswap);
3116+
3117+ /* Swap core_relo subsection byte-order (if present) */
3118+ if (h->hdr_len < offsetofend(struct btf_ext_header, core_relo_len))
3119+ return;
30223120
3023− return btf_ext_setup_info(btf_ext, &param);
3121+ info = data + h->hdr_len + h->core_relo_off;
3122+ btf_ext_bswap_info_sec(info, h->core_relo_len, is_native,
3123+ (info_rec_bswap_fn)bpf_core_relo_bswap);
30243124 }
30253125
3026−static int btf_ext_parse_hdr(__u8 *data, __u32 data_size)
3126+/* Parse hdr data and info sections: check and convert to native endianness */
3127+static int btf_ext_parse(struct btf_ext *btf_ext)
30273128 {
3028− const struct btf_ext_header *hdr = (struct btf_ext_header *)data;
3129+ __u32 hdr_len, data_size = btf_ext->data_size;
3130+ struct btf_ext_header *hdr = btf_ext->hdr;
3131+ bool swapped_endian = false;
3132+ int err;
30293133
3030− if (data_size < offsetofend(struct btf_ext_header, hdr_len) ||
3031− data_size < hdr->hdr_len) {
3032− pr_debug("BTF.ext header not found");
3134+ if (data_size < offsetofend(struct btf_ext_header, hdr_len)) {
3135+ pr_debug("BTF.ext header too short\n");
30333136 return -EINVAL;
30343137 }
30353138
3139+ hdr_len = hdr->hdr_len;
30363140 if (hdr->magic == bswap_16(BTF_MAGIC)) {
3037− pr_warn("BTF.ext in non-native endianness is not supported\n");
3038− return -ENOTSUP;
3141+ swapped_endian = true;
3142+ hdr_len = bswap_32(hdr_len);
30393143 } else if (hdr->magic != BTF_MAGIC) {
30403144 pr_debug("Invalid BTF.ext magic:%x\n", hdr->magic);
30413145 return -EINVAL;
30423146 }
30433147
3044− if (hdr->version != BTF_VERSION) {
3148+ /* Ensure known version of structs, current BTF_VERSION == 1 */
3149+ if (hdr->version != 1) {
30453150 pr_debug("Unsupported BTF.ext version:%u\n", hdr->version);
30463151 return -ENOTSUP;
30473152 }
@@ -3051,11 +3156,39 @@ static int btf_ext_parse_hdr(__u8 *data, __u32 data_size) · 3 unmodified lines
30513156 return -ENOTSUP;
30523157 }
30533158
3054− if (data_size == hdr->hdr_len) {
3159+ if (data_size < hdr_len) {
3160+ pr_debug("BTF.ext header not found\n");
3161+ return -EINVAL;
3162+ } else if (data_size == hdr_len) {
30553163 pr_debug("BTF.ext has no data\n");
30563164 return -EINVAL;
30573165 }
30583166
3167+ /* Verify mandatory hdr info details present */
3168+ if (hdr_len < offsetofend(struct btf_ext_header, line_info_len)) {
3169+ pr_warn("BTF.ext header missing func_info, line_info\n");
3170+ return -EINVAL;
3171+ }
3172+
3173+ /* Keep hdr native byte-order in memory for introspection */
3174+ if (swapped_endian)
3175+ btf_ext_bswap_hdr(btf_ext->hdr);
3176+
3177+ /* Validate info subsections and cache key metadata */
3178+ err = btf_ext_parse_info(btf_ext, !swapped_endian);
3179+ if (err)
3180+ return err;
3181+
3182+ /* Keep infos native byte-order in memory for introspection */
3183+ if (swapped_endian)
3184+ btf_ext_bswap_info(btf_ext, btf_ext->data);
3185+
3186+ /*
3187+ * Set btf_ext->swapped_endian only after all header and info data has
3188+ * been swapped, helping bswap functions determine if their data are
3189+ * in native byte-order when called.
3190+ */
3191+ btf_ext->swapped_endian = swapped_endian;
30593192 return 0;
30603193 }
30613194
@@ -3067,6 +3200,7 @@ void btf_ext__free(struct btf_ext *btf_ext) · 5 unmodified lines
30673200 free(btf_ext->line_info.sec_idxs);
30683201 free(btf_ext->core_relo_info.sec_idxs);
30693202 free(btf_ext->data);
3203+ free(btf_ext->data_swapped);
30703204 free(btf_ext);
30713205 }
30723206
@@ -3087,29 +3221,7 @@ struct btf_ext *btf_ext__new(const __u8 *data, __u32 size) · 14 unmodified lines
30873221 }
30883222 memcpy(btf_ext->data, data, size);
30893223
3090− err = btf_ext_parse_hdr(btf_ext->data, size);
3091− if (err)
3092− goto done;
3093−
3094− if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, line_info_len)) {
3095− err = -EINVAL;
3096− goto done;
3097− }
3098−
3099− err = btf_ext_setup_func_info(btf_ext);
3100− if (err)
3101− goto done;
3102−
3103− err = btf_ext_setup_line_info(btf_ext);
3104− if (err)
3105− goto done;
3106−
3107− if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, core_relo_len))
3108− goto done; /* skip core relos parsing */
3109−
3110− err = btf_ext_setup_core_relos(btf_ext);
3111− if (err)
3112− goto done;
3224+ err = btf_ext_parse(btf_ext);
31133225
31143226 done:
31153227 if (err) {
@@ -3120,15 +3232,66 @@ done: · 4 unmodified lines
31203232 return btf_ext;
31213233 }
31223234
3235+static void *btf_ext_raw_data(const struct btf_ext *btf_ext_ro, bool swap_endian)
3236+{
3237+ struct btf_ext *btf_ext = (struct btf_ext *)btf_ext_ro;
3238+ const __u32 data_sz = btf_ext->data_size;
3239+ void *data;
3240+
3241+ /* Return native data (always present) or swapped data if present */
3242+ if (!swap_endian)
3243+ return btf_ext->data;
3244+ else if (btf_ext->data_swapped)
3245+ return btf_ext->data_swapped;
3246+
3247+ /* Recreate missing swapped data, then cache and return */
3248+ data = calloc(1, data_sz);
3249+ if (!data)
3250+ return NULL;
3251+ memcpy(data, btf_ext->data, data_sz);
3252+
3253+ btf_ext_bswap_info(btf_ext, data);
3254+ btf_ext_bswap_hdr(data);
3255+ btf_ext->data_swapped = data;
3256+ return data;
3257+}
3258+
31233259 const void *btf_ext__raw_data(const struct btf_ext *btf_ext, __u32 *size)
31243260 {
3261+ void *data;
3262+
3263+ data = btf_ext_raw_data(btf_ext, btf_ext->swapped_endian);
3264+ if (!data)
3265+ return errno = ENOMEM, NULL;
3266+
31253267 *size = btf_ext->data_size;
3126− return btf_ext->data;
3268+ return data;
31273269 }
31283270
31293271 __attribute__((alias("btf_ext__raw_data")))
31303272 const void *btf_ext__get_raw_data(const struct btf_ext *btf_ext, __u32 *size);
31313273
3274+enum btf_endianness btf_ext__endianness(const struct btf_ext *btf_ext)
3275+{
3276+ if (is_host_big_endian())
3277+ return btf_ext->swapped_endian ? BTF_LITTLE_ENDIAN : BTF_BIG_ENDIAN;
3278+ else
3279+ return btf_ext->swapped_endian ? BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN;
3280+}
3281+
3282+int btf_ext__set_endianness(struct btf_ext *btf_ext, enum btf_endianness endian)
3283+{
3284+ if (endian != BTF_LITTLE_ENDIAN && endian != BTF_BIG_ENDIAN)
3285+ return libbpf_err(-EINVAL);
3286+
3287+ btf_ext->swapped_endian = is_host_big_endian() != (endian == BTF_BIG_ENDIAN);
3288+
3289+ if (!btf_ext->swapped_endian) {
3290+ free(btf_ext->data_swapped);
3291+ btf_ext->data_swapped = NULL;
3292+ }
3293+ return 0;
3294+}
31323295
31333296 struct btf_dedup;
31343297
@@ -3291,7 +3454,7 @@ int btf__dedup(struct btf *btf, const struct btf_dedup_opts *opts) · 156 unmodified lines
32913454
32923455 d = btf_dedup_new(btf, opts);
32933456 if (IS_ERR(d)) {
3294− pr_debug("btf_dedup_new failed: %ld", PTR_ERR(d));
3457+ pr_debug("btf_dedup_new failed: %ld\n", PTR_ERR(d));
32953458 return libbpf_err(-EINVAL);
32963459 }
32973460
@@ -3302,42 +3465,42 @@ int btf__dedup(struct btf *btf, const struct btf_dedup_opts *opts) · 4 unmodified lines
33023465
33033466 err = btf_dedup_prep(d);
33043467 if (err) {
3305− pr_debug("btf_dedup_prep failed:%d\n", err);
3468+ pr_debug("btf_dedup_prep failed: %s\n", errstr(err));
33063469 goto done;
33073470 }
33083471 err = btf_dedup_strings(d);
33093472 if (err < 0) {
3310− pr_debug("btf_dedup_strings failed:%d\n", err);
3473+ pr_debug("btf_dedup_strings failed: %s\n", errstr(err));
33113474 goto done;
33123475 }
33133476 err = btf_dedup_prim_types(d);
33143477 if (err < 0) {
3315− pr_debug("btf_dedup_prim_types failed:%d\n", err);
3478+ pr_debug("btf_dedup_prim_types failed: %s\n", errstr(err));
33163479 goto done;
33173480 }
33183481 err = btf_dedup_struct_types(d);
33193482 if (err < 0) {
3320− pr_debug("btf_dedup_struct_types failed:%d\n", err);
3483+ pr_debug("btf_dedup_struct_types failed: %s\n", errstr(err));
33213484 goto done;
33223485 }
33233486 err = btf_dedup_resolve_fwds(d);
33243487 if (err < 0) {
3325− pr_debug("btf_dedup_resolve_fwds failed:%d\n", err);
3488+ pr_debug("btf_dedup_resolve_fwds failed: %s\n", errstr(err));
33263489 goto done;
33273490 }
33283491 err = btf_dedup_ref_types(d);
33293492 if (err < 0) {
3330− pr_debug("btf_dedup_ref_types failed:%d\n", err);
3493+ pr_debug("btf_dedup_ref_types failed: %s\n", errstr(err));
33313494 goto done;
33323495 }
33333496 err = btf_dedup_compact_types(d);
33343497 if (err < 0) {
3335− pr_debug("btf_dedup_compact_types failed:%d\n", err);
3498+ pr_debug("btf_dedup_compact_types failed: %s\n", errstr(err));
33363499 goto done;
33373500 }
33383501 err = btf_dedup_remap_types(d);
33393502 if (err < 0) {
3340− pr_debug("btf_dedup_remap_types failed:%d\n", err);
3503+ pr_debug("btf_dedup_remap_types failed: %s\n", errstr(err));
33413504 goto done;
33423505 }
33433506
@@ -3385,7 +3548,7 @@ struct btf_dedup { · 41 unmodified lines
33853548 struct strset *strs_set;
33863549 };
33873550
3388−static long hash_combine(long h, long value)
3551+static unsigned long hash_combine(unsigned long h, unsigned long value)
33893552 {
33903553 return h * 31 + value;
33913554 }
@@ -5056,7 +5219,8 @@ struct btf *btf__load_vmlinux_btf(void) · 1664 unmodified lines
50565219 btf = btf__parse(sysfs_btf_path, NULL);
50575220 if (!btf) {
50585221 err = -errno;
5059− pr_warn("failed to read kernel BTF from '%s': %d\n", sysfs_btf_path, err);
5222+ pr_warn("failed to read kernel BTF from '%s': %s\n",
5223+ sysfs_btf_path, errstr(err));
50605224 return libbpf_err_ptr(err);
50615225 }
50625226 pr_debug("loaded kernel BTF from '%s'\n", sysfs_btf_path);
@@ -5073,7 +5237,7 @@ struct btf *btf__load_vmlinux_btf(void) · 10 unmodified lines
50735237
50745238 btf = btf__parse(path, NULL);
50755239 err = libbpf_get_error(btf);
5076− pr_debug("loading kernel BTF '%s': %d\n", path, err);
5240+ pr_debug("loading kernel BTF '%s': %s\n", path, errstr(err));
50775241 if (err)
50785242 continue;
50795243
tools/lib/bpf/btf.h+3−0
@@ -167,6 +167,9 @@ LIBBPF_API const char *btf__str_by_offset(const struct btf *btf, __u32 offset); · 166 unmodified lines
167167 LIBBPF_API struct btf_ext *btf_ext__new(const __u8 *data, __u32 size);
168168 LIBBPF_API void btf_ext__free(struct btf_ext *btf_ext);
169169 LIBBPF_API const void *btf_ext__raw_data(const struct btf_ext *btf_ext, __u32 *size);
170+LIBBPF_API enum btf_endianness btf_ext__endianness(const struct btf_ext *btf_ext);
171+LIBBPF_API int btf_ext__set_endianness(struct btf_ext *btf_ext,
172+ enum btf_endianness endian);
170173
171174 LIBBPF_API int btf__find_str(struct btf *btf, const char *s);
172175 LIBBPF_API int btf__add_str(struct btf *btf, const char *s);
tools/lib/bpf/btf_dump.c+4−3
@@ -21,6 +21,7 @@ · 20 unmodified lines
2121 #include "hashmap.h"
2222 #include "libbpf.h"
2323 #include "libbpf_internal.h"
24+#include "str_error.h"
2425
2526 static const char PREFIXES[] = "\t\t\t\t\t\t\t\t\t\t\t\t\t";
2627 static const size_t PREFIX_CNT = sizeof(PREFIXES) - 1;
@@ -867,8 +868,8 @@ static void btf_dump_emit_bit_padding(const struct btf_dump *d, · 840 unmodified lines
867868 } pads[] = {
868869 {"long", d->ptr_sz * 8}, {"int", 32}, {"short", 16}, {"char", 8}
869870 };
870− int new_off, pad_bits, bits, i;
871− const char *pad_type;
871+ int new_off = 0, pad_bits = 0, bits, i;
872+ const char *pad_type = NULL;
872873
873874 if (cur_off >= next_off)
874875 return; /* no gap */
@@ -1304,7 +1305,7 @@ static void btf_dump_emit_type_decl(struct btf_dump *d, __u32 id, · 429 unmodified lines
13041305 * chain, restore stack, emit warning, and try to
13051306 * proceed nevertheless
13061307 */
1307− pr_warn("not enough memory for decl stack:%d", err);
1308+ pr_warn("not enough memory for decl stack: %s\n", errstr(err));
13081309 d->decl_stack_cnt = stack_start;
13091310 return;
13101311 }
tools/lib/bpf/btf_relocate.c+1−1
@@ -428,7 +428,7 @@ static int btf_relocate_rewrite_strs(struct btf_relocate *r, __u32 i) · 427 unmodified lines
428428 } else {
429429 off = r->str_map[*str_off];
430430 if (!off) {
431− pr_warn("string '%s' [offset %u] is not mapped to base BTF",
431+ pr_warn("string '%s' [offset %u] is not mapped to base BTF\n",
432432 btf__str_by_offset(r->btf, off), *str_off);
433433 return -ENOENT;
434434 }
tools/lib/bpf/elf.c+1−3
@@ -24,7 +24,6 @@ · 23 unmodified lines
2424
2525 int elf_open(const char *binary_path, struct elf_fd *elf_fd)
2626 {
27− char errmsg[STRERR_BUFSIZE];
2827 int fd, ret;
2928 Elf *elf;
3029
@@ -38,8 +37,7 @@ int elf_open(const char *binary_path, struct elf_fd *elf_fd) · 7 unmodified lines
3837 fd = open(binary_path, O_RDONLY | O_CLOEXEC);
3938 if (fd < 0) {
4039 ret = -errno;
41− pr_warn("elf: failed to open %s: %s\n", binary_path,
42− libbpf_strerror_r(ret, errmsg, sizeof(errmsg)));
40+ pr_warn("elf: failed to open %s: %s\n", binary_path, errstr(ret));
4341 return ret;
4442 }
4543 elf = elf_begin(fd, ELF_C_READ_MMAP, NULL);
tools/lib/bpf/features.c+6−9
@@ -47,7 +47,6 @@ static int probe_kern_prog_name(int token_fd) · 46 unmodified lines
4747
4848 static int probe_kern_global_data(int token_fd)
4949 {
50− char *cp, errmsg[STRERR_BUFSIZE];
5150 struct bpf_insn insns[] = {
5251 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16),
5352 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42),
@@ -67,9 +66,8 @@ static int probe_kern_global_data(int token_fd) · 13 unmodified lines
6766 map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_global", sizeof(int), 32, 1, &map_opts);
6867 if (map < 0) {
6968 ret = -errno;
70− cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
71− pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n",
72− __func__, cp, -ret);
69+ pr_warn("Error in %s(): %s. Couldn't create simple array map.\n",
70+ __func__, errstr(ret));
7371 return ret;
7472 }
7573
@@ -267,7 +265,6 @@ static int probe_kern_probe_read_kernel(int token_fd) · 191 unmodified lines
267265
268266 static int probe_prog_bind_map(int token_fd)
269267 {
270− char *cp, errmsg[STRERR_BUFSIZE];
271268 struct bpf_insn insns[] = {
272269 BPF_MOV64_IMM(BPF_REG_0, 0),
273270 BPF_EXIT_INSN(),
@@ -285,9 +282,8 @@ static int probe_prog_bind_map(int token_fd) · 11 unmodified lines
285282 map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_det_bind", sizeof(int), 32, 1, &map_opts);
286283 if (map < 0) {
287284 ret = -errno;
288− cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
289− pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n",
290− __func__, cp, -ret);
285+ pr_warn("Error in %s(): %s. Couldn't create simple array map.\n",
286+ __func__, errstr(ret));
291287 return ret;
292288 }
293289
@@ -604,7 +600,8 @@ bool feat_supported(struct kern_feature_cache *cache, enum kern_feature_id feat_ · 310 unmodified lines
604600 } else if (ret == 0) {
605601 WRITE_ONCE(cache->res[feat_id], FEAT_MISSING);
606602 } else {
607− pr_warn("Detection of kernel %s support failed: %d\n", feat->desc, ret);
603+ pr_warn("Detection of kernel %s support failed: %s\n",
604+ feat->desc, errstr(ret));
608605 WRITE_ONCE(cache->res[feat_id], FEAT_MISSING);
609606 }
610607 }
tools/lib/bpf/gen_loader.c+137−53
@@ -14,6 +14,7 @@ · 13 unmodified lines
1414 #include "bpf_gen_internal.h"
1515 #include "skel_internal.h"
1616 #include <asm/byteorder.h>
17+#include "str_error.h"
1718
1819 #define MAX_USED_MAPS 64
1920 #define MAX_USED_PROGS 32
@@ -393,7 +394,7 @@ int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps) · 373 unmodified lines
393394 blob_fd_array_off(gen, i));
394395 emit(gen, BPF_MOV64_IMM(BPF_REG_0, 0));
395396 emit(gen, BPF_EXIT_INSN());
396− pr_debug("gen: finish %d\n", gen->error);
397+ pr_debug("gen: finish %s\n", errstr(gen->error));
397398 if (!gen->error) {
398399 struct gen_loader_opts *opts = gen->opts;
399400
@@ -401,6 +402,15 @@ int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps) · 1 unmodified line
401402 opts->insns_sz = gen->insn_cur - gen->insn_start;
402403 opts->data = gen->data_start;
403404 opts->data_sz = gen->data_cur - gen->data_start;
405+
406+ /* use target endianness for embedded loader */
407+ if (gen->swapped_endian) {
408+ struct bpf_insn *insn = (struct bpf_insn *)opts->insns;
409+ int insn_cnt = opts->insns_sz / sizeof(struct bpf_insn);
410+
411+ for (i = 0; i < insn_cnt; i++)
412+ bpf_insn_bswap(insn++);
413+ }
404414 }
405415 return gen->error;
406416 }
@@ -414,6 +424,28 @@ void bpf_gen__free(struct bpf_gen *gen) · 7 unmodified lines
414424 free(gen);
415425 }
416426
427+/*
428+ * Fields of bpf_attr are set to values in native byte-order before being
429+ * written to the target-bound data blob, and may need endian conversion.
430+ * This macro allows providing the correct value in situ more simply than
431+ * writing a separate converter for *all fields* of *all records* included
432+ * in union bpf_attr. Note that sizeof(rval) should match the assignment
433+ * target to avoid runtime problems.
434+ */
435+#define tgt_endian(rval) ({ \
436+ typeof(rval) _val = (rval); \
437+ if (gen->swapped_endian) { \
438+ switch (sizeof(_val)) { \
439+ case 1: break; \
440+ case 2: _val = bswap_16(_val); break; \
441+ case 4: _val = bswap_32(_val); break; \
442+ case 8: _val = bswap_64(_val); break; \
443+ default: pr_warn("unsupported bswap size!\n"); \
444+ } \
445+ } \
446+ _val; \
447+})
448+
417449 void bpf_gen__load_btf(struct bpf_gen *gen, const void *btf_raw_data,
418450 __u32 btf_raw_size)
419451 {
@@ -422,11 +454,12 @@ void bpf_gen__load_btf(struct bpf_gen *gen, const void *btf_raw_data, · 2 unmodified lines
422454 union bpf_attr attr;
423455
424456 memset(&attr, 0, attr_size);
425− pr_debug("gen: load_btf: size %d\n", btf_raw_size);
426457 btf_data = add_data(gen, btf_raw_data, btf_raw_size);
427458
428− attr.btf_size = btf_raw_size;
459+ attr.btf_size = tgt_endian(btf_raw_size);
429460 btf_load_attr = add_data(gen, &attr, attr_size);
461+ pr_debug("gen: load_btf: off %d size %d, attr: off %d size %d\n",
462+ btf_data, btf_raw_size, btf_load_attr, attr_size);
430463
431464 /* populate union bpf_attr with user provided log details */
432465 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_level), 4,
@@ -457,28 +490,29 @@ void bpf_gen__map_create(struct bpf_gen *gen, · 24 unmodified lines
457490 union bpf_attr attr;
458491
459492 memset(&attr, 0, attr_size);
460− attr.map_type = map_type;
461− attr.key_size = key_size;
462− attr.value_size = value_size;
463− attr.map_flags = map_attr->map_flags;
464− attr.map_extra = map_attr->map_extra;
493+ attr.map_type = tgt_endian(map_type);
494+ attr.key_size = tgt_endian(key_size);
495+ attr.value_size = tgt_endian(value_size);
496+ attr.map_flags = tgt_endian(map_attr->map_flags);
497+ attr.map_extra = tgt_endian(map_attr->map_extra);
465498 if (map_name)
466499 libbpf_strlcpy(attr.map_name, map_name, sizeof(attr.map_name));
467− attr.numa_node = map_attr->numa_node;
468− attr.map_ifindex = map_attr->map_ifindex;
469− attr.max_entries = max_entries;
470− attr.btf_key_type_id = map_attr->btf_key_type_id;
471− attr.btf_value_type_id = map_attr->btf_value_type_id;
472−
473− pr_debug("gen: map_create: %s idx %d type %d value_type_id %d\n",
474− attr.map_name, map_idx, map_type, attr.btf_value_type_id);
500+ attr.numa_node = tgt_endian(map_attr->numa_node);
501+ attr.map_ifindex = tgt_endian(map_attr->map_ifindex);
502+ attr.max_entries = tgt_endian(max_entries);
503+ attr.btf_key_type_id = tgt_endian(map_attr->btf_key_type_id);
504+ attr.btf_value_type_id = tgt_endian(map_attr->btf_value_type_id);
475505
476506 map_create_attr = add_data(gen, &attr, attr_size);
477− if (attr.btf_value_type_id)
507+ pr_debug("gen: map_create: %s idx %d type %d value_type_id %d, attr: off %d size %d\n",
508+ map_name, map_idx, map_type, map_attr->btf_value_type_id,
509+ map_create_attr, attr_size);
510+
511+ if (map_attr->btf_value_type_id)
478512 /* populate union bpf_attr with btf_fd saved in the stack earlier */
479513 move_stack2blob(gen, attr_field(map_create_attr, btf_fd), 4,
480514 stack_off(btf_fd));
481− switch (attr.map_type) {
515+ switch (map_type) {
482516 case BPF_MAP_TYPE_ARRAY_OF_MAPS:
483517 case BPF_MAP_TYPE_HASH_OF_MAPS:
484518 move_stack2blob(gen, attr_field(map_create_attr, inner_map_fd), 4,
@@ -498,8 +532,8 @@ void bpf_gen__map_create(struct bpf_gen *gen, · 13 unmodified lines
498532 /* emit MAP_CREATE command */
499533 emit_sys_bpf(gen, BPF_MAP_CREATE, map_create_attr, attr_size);
500534 debug_ret(gen, "map_create %s idx %d type %d value_size %d value_btf_id %d",
501− attr.map_name, map_idx, map_type, value_size,
502− attr.btf_value_type_id);
535+ map_name, map_idx, map_type, value_size,
536+ map_attr->btf_value_type_id);
503537 emit_check_err(gen);
504538 /* remember map_fd in the stack, if successful */
505539 if (map_idx < 0) {
@@ -784,12 +818,12 @@ log: · 278 unmodified lines
784818 emit_ksym_relo_log(gen, relo, kdesc->ref);
785819 }
786820
787−static __u32 src_reg_mask(void)
821+static __u32 src_reg_mask(struct bpf_gen *gen)
788822 {
789−#if defined(__LITTLE_ENDIAN_BITFIELD)
790− return 0x0f; /* src_reg,dst_reg,... */
791−#elif defined(__BIG_ENDIAN_BITFIELD)
792− return 0xf0; /* dst_reg,src_reg,... */
823+#if defined(__LITTLE_ENDIAN_BITFIELD) /* src_reg,dst_reg,... */
824+ return gen->swapped_endian ? 0xf0 : 0x0f;
825+#elif defined(__BIG_ENDIAN_BITFIELD) /* dst_reg,src_reg,... */
826+ return gen->swapped_endian ? 0x0f : 0xf0;
793827 #else
794828 #error "Unsupported bit endianness, cannot proceed"
795829 #endif
@@ -840,7 +874,7 @@ static void emit_relo_ksym_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, · 44 unmodified lines
840874 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 3));
841875 clear_src_reg:
842876 /* clear bpf_object__relocate_data's src_reg assignment, otherwise we get a verifier failure */
843− reg_mask = src_reg_mask();
877+ reg_mask = src_reg_mask(gen);
844878 emit(gen, BPF_LDX_MEM(BPF_B, BPF_REG_9, BPF_REG_8, offsetofend(struct bpf_insn, code)));
845879 emit(gen, BPF_ALU32_IMM(BPF_AND, BPF_REG_9, reg_mask));
846880 emit(gen, BPF_STX_MEM(BPF_B, BPF_REG_8, BPF_REG_9, offsetofend(struct bpf_insn, code)));
@@ -931,48 +965,94 @@ static void cleanup_relos(struct bpf_gen *gen, int insns) · 84 unmodified lines
931965 cleanup_core_relo(gen);
932966 }
933967
968+/* Convert func, line, and core relo info blobs to target endianness */
969+static void info_blob_bswap(struct bpf_gen *gen, int func_info, int line_info,
970+ int core_relos, struct bpf_prog_load_opts *load_attr)
971+{
972+ struct bpf_func_info *fi = gen->data_start + func_info;
973+ struct bpf_line_info *li = gen->data_start + line_info;
974+ struct bpf_core_relo *cr = gen->data_start + core_relos;
975+ int i;
976+
977+ for (i = 0; i < load_attr->func_info_cnt; i++)
978+ bpf_func_info_bswap(fi++);
979+
980+ for (i = 0; i < load_attr->line_info_cnt; i++)
981+ bpf_line_info_bswap(li++);
982+
983+ for (i = 0; i < gen->core_relo_cnt; i++)
984+ bpf_core_relo_bswap(cr++);
985+}
986+
934987 void bpf_gen__prog_load(struct bpf_gen *gen,
935988 enum bpf_prog_type prog_type, const char *prog_name,
936989 const char *license, struct bpf_insn *insns, size_t insn_cnt,
937990 struct bpf_prog_load_opts *load_attr, int prog_idx)
938991 {
992+ int func_info_tot_sz = load_attr->func_info_cnt *
993+ load_attr->func_info_rec_size;
994+ int line_info_tot_sz = load_attr->line_info_cnt *
995+ load_attr->line_info_rec_size;
996+ int core_relo_tot_sz = gen->core_relo_cnt *
997+ sizeof(struct bpf_core_relo);
939998 int prog_load_attr, license_off, insns_off, func_info, line_info, core_relos;
940999 int attr_size = offsetofend(union bpf_attr, core_relo_rec_size);
9411000 union bpf_attr attr;
9421001
9431002 memset(&attr, 0, attr_size);
944− pr_debug("gen: prog_load: type %d insns_cnt %zd progi_idx %d\n",
945− prog_type, insn_cnt, prog_idx);
9461003 /* add license string to blob of bytes */
9471004 license_off = add_data(gen, license, strlen(license) + 1);
9481005 /* add insns to blob of bytes */
9491006 insns_off = add_data(gen, insns, insn_cnt * sizeof(struct bpf_insn));
1007+ pr_debug("gen: prog_load: prog_idx %d type %d insn off %d insns_cnt %zd license off %d\n",
1008+ prog_idx, prog_type, insns_off, insn_cnt, license_off);
9501009
951− attr.prog_type = prog_type;
952− attr.expected_attach_type = load_attr->expected_attach_type;
953− attr.attach_btf_id = load_attr->attach_btf_id;
954− attr.prog_ifindex = load_attr->prog_ifindex;
955− attr.kern_version = 0;
956− attr.insn_cnt = (__u32)insn_cnt;
957− attr.prog_flags = load_attr->prog_flags;
958−
959− attr.func_info_rec_size = load_attr->func_info_rec_size;
960− attr.func_info_cnt = load_attr->func_info_cnt;
961− func_info = add_data(gen, load_attr->func_info,
962− attr.func_info_cnt * attr.func_info_rec_size);
1010+ /* convert blob insns to target endianness */
1011+ if (gen->swapped_endian) {
1012+ struct bpf_insn *insn = gen->data_start + insns_off;
1013+ int i;
9631014
964− attr.line_info_rec_size = load_attr->line_info_rec_size;
965− attr.line_info_cnt = load_attr->line_info_cnt;
966− line_info = add_data(gen, load_attr->line_info,
967− attr.line_info_cnt * attr.line_info_rec_size);
1015+ for (i = 0; i < insn_cnt; i++, insn++)
1016+ bpf_insn_bswap(insn);
1017+ }
9681018
969− attr.core_relo_rec_size = sizeof(struct bpf_core_relo);
970− attr.core_relo_cnt = gen->core_relo_cnt;
971− core_relos = add_data(gen, gen->core_relos,
972− attr.core_relo_cnt * attr.core_relo_rec_size);
1019+ attr.prog_type = tgt_endian(prog_type);
1020+ attr.expected_attach_type = tgt_endian(load_attr->expected_attach_type);
1021+ attr.attach_btf_id = tgt_endian(load_attr->attach_btf_id);
1022+ attr.prog_ifindex = tgt_endian(load_attr->prog_ifindex);
1023+ attr.kern_version = 0;
1024+ attr.insn_cnt = tgt_endian((__u32)insn_cnt);
1025+ attr.prog_flags = tgt_endian(load_attr->prog_flags);
1026+
1027+ attr.func_info_rec_size = tgt_endian(load_attr->func_info_rec_size);
1028+ attr.func_info_cnt = tgt_endian(load_attr->func_info_cnt);
1029+ func_info = add_data(gen, load_attr->func_info, func_info_tot_sz);
1030+ pr_debug("gen: prog_load: func_info: off %d cnt %d rec size %d\n",
1031+ func_info, load_attr->func_info_cnt,
1032+ load_attr->func_info_rec_size);
1033+
1034+ attr.line_info_rec_size = tgt_endian(load_attr->line_info_rec_size);
1035+ attr.line_info_cnt = tgt_endian(load_attr->line_info_cnt);
1036+ line_info = add_data(gen, load_attr->line_info, line_info_tot_sz);
1037+ pr_debug("gen: prog_load: line_info: off %d cnt %d rec size %d\n",
1038+ line_info, load_attr->line_info_cnt,
1039+ load_attr->line_info_rec_size);
1040+
1041+ attr.core_relo_rec_size = tgt_endian((__u32)sizeof(struct bpf_core_relo));
1042+ attr.core_relo_cnt = tgt_endian(gen->core_relo_cnt);
1043+ core_relos = add_data(gen, gen->core_relos, core_relo_tot_sz);
1044+ pr_debug("gen: prog_load: core_relos: off %d cnt %d rec size %zd\n",
1045+ core_relos, gen->core_relo_cnt,
1046+ sizeof(struct bpf_core_relo));
1047+
1048+ /* convert all info blobs to target endianness */
1049+ if (gen->swapped_endian)
1050+ info_blob_bswap(gen, func_info, line_info, core_relos, load_attr);
9731051
9741052 libbpf_strlcpy(attr.prog_name, prog_name, sizeof(attr.prog_name));
9751053 prog_load_attr = add_data(gen, &attr, attr_size);
1054+ pr_debug("gen: prog_load: attr: off %d size %d\n",
1055+ prog_load_attr, attr_size);
9761056
9771057 /* populate union bpf_attr with a pointer to license */
9781058 emit_rel_store(gen, attr_field(prog_load_attr, license), license_off);
@@ -1040,7 +1120,6 @@ void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, · 61 unmodified lines
10401120 int zero = 0;
10411121
10421122 memset(&attr, 0, attr_size);
1043− pr_debug("gen: map_update_elem: idx %d\n", map_idx);
10441123
10451124 value = add_data(gen, pvalue, value_size);
10461125 key = add_data(gen, &zero, sizeof(zero));
@@ -1068,6 +1147,8 @@ void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, · 21 unmodified lines
10681147 emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel));
10691148
10701149 map_update_attr = add_data(gen, &attr, attr_size);
1150+ pr_debug("gen: map_update_elem: idx %d, value: off %d size %d, attr: off %d size %d\n",
1151+ map_idx, value, value_size, map_update_attr, attr_size);
10711152 move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4,
10721153 blob_fd_array_off(gen, map_idx));
10731154 emit_rel_store(gen, attr_field(map_update_attr, key), key);
@@ -1084,14 +1165,16 @@ void bpf_gen__populate_outer_map(struct bpf_gen *gen, int outer_map_idx, int slo · 10 unmodified lines
10841165 int attr_size = offsetofend(union bpf_attr, flags);
10851166 int map_update_attr, key;
10861167 union bpf_attr attr;
1168+ int tgt_slot;
10871169
10881170 memset(&attr, 0, attr_size);
1089− pr_debug("gen: populate_outer_map: outer %d key %d inner %d\n",
1090− outer_map_idx, slot, inner_map_idx);
10911171
1092− key = add_data(gen, &slot, sizeof(slot));
1172+ tgt_slot = tgt_endian(slot);
1173+ key = add_data(gen, &tgt_slot, sizeof(tgt_slot));
10931174
10941175 map_update_attr = add_data(gen, &attr, attr_size);
1176+ pr_debug("gen: populate_outer_map: outer %d key %d inner %d, attr: off %d size %d\n",
1177+ outer_map_idx, slot, inner_map_idx, map_update_attr, attr_size);
10951178 move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4,
10961179 blob_fd_array_off(gen, outer_map_idx));
10971180 emit_rel_store(gen, attr_field(map_update_attr, key), key);
@@ -1112,8 +1195,9 @@ void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx) · 14 unmodified lines
11121195 union bpf_attr attr;
11131196
11141197 memset(&attr, 0, attr_size);
1115− pr_debug("gen: map_freeze: idx %d\n", map_idx);
11161198 map_freeze_attr = add_data(gen, &attr, attr_size);
1199+ pr_debug("gen: map_freeze: idx %d, attr: off %d size %d\n",
1200+ map_idx, map_freeze_attr, attr_size);
11171201 move_blob2blob(gen, attr_field(map_freeze_attr, map_fd), 4,
11181202 blob_fd_array_off(gen, map_idx));
11191203 /* emit MAP_FREEZE command */
tools/lib/bpf/hashmap.h+10−10
@@ -166,8 +166,8 @@ bool hashmap_find(const struct hashmap *map, long key, long *value); · 165 unmodified lines
166166 * @bkt: integer used as a bucket loop cursor
167167 */
168168 #define hashmap__for_each_entry(map, cur, bkt) \
169− for (bkt = 0; bkt < map->cap; bkt++) \
170− for (cur = map->buckets[bkt]; cur; cur = cur->next)
169+ for (bkt = 0; bkt < (map)->cap; bkt++) \
170+ for (cur = (map)->buckets[bkt]; cur; cur = cur->next)
171171
172172 /*
173173 * hashmap__for_each_entry_safe - iterate over all entries in hashmap, safe
@@ -178,8 +178,8 @@ bool hashmap_find(const struct hashmap *map, long key, long *value); · 4 unmodified lines
178178 * @bkt: integer used as a bucket loop cursor
179179 */
180180 #define hashmap__for_each_entry_safe(map, cur, tmp, bkt) \
181− for (bkt = 0; bkt < map->cap; bkt++) \
182− for (cur = map->buckets[bkt]; \
181+ for (bkt = 0; bkt < (map)->cap; bkt++) \
182+ for (cur = (map)->buckets[bkt]; \
183183 cur && ({tmp = cur->next; true; }); \
184184 cur = tmp)
185185
@@ -190,19 +190,19 @@ bool hashmap_find(const struct hashmap *map, long key, long *value); · 4 unmodified lines
190190 * @key: key to iterate entries for
191191 */
192192 #define hashmap__for_each_key_entry(map, cur, _key) \
193− for (cur = map->buckets \
194− ? map->buckets[hash_bits(map->hash_fn((_key), map->ctx), map->cap_bits)] \
193+ for (cur = (map)->buckets \
194+ ? (map)->buckets[hash_bits((map)->hash_fn((_key), (map)->ctx), (map)->cap_bits)] \
195195 : NULL; \
196196 cur; \
197197 cur = cur->next) \
198− if (map->equal_fn(cur->key, (_key), map->ctx))
198+ if ((map)->equal_fn(cur->key, (_key), (map)->ctx))
199199
200200 #define hashmap__for_each_key_entry_safe(map, cur, tmp, _key) \
201− for (cur = map->buckets \
202− ? map->buckets[hash_bits(map->hash_fn((_key), map->ctx), map->cap_bits)] \
201+ for (cur = (map)->buckets \
202+ ? (map)->buckets[hash_bits((map)->hash_fn((_key), (map)->ctx), (map)->cap_bits)] \
203203 : NULL; \
204204 cur && ({ tmp = cur->next; true; }); \
205205 cur = tmp) \
206− if (map->equal_fn(cur->key, (_key), map->ctx))
206+ if ((map)->equal_fn(cur->key, (_key), (map)->ctx))
207207
208208 #endif /* __LIBBPF_HASHMAP_H */
tools/lib/bpf/libbpf.c+266−260
@@ -133,6 +133,7 @@ static const char * const attach_type_name[] = { · 132 unmodified lines
133133 [BPF_NETKIT_PRIMARY] = "netkit_primary",
134134 [BPF_NETKIT_PEER] = "netkit_peer",
135135 [BPF_TRACE_KPROBE_SESSION] = "trace_kprobe_session",
136+ [BPF_TRACE_UPROBE_SESSION] = "trace_uprobe_session",
136137 };
137138
138139 static const char * const link_type_name[] = {
@@ -694,6 +695,8 @@ struct bpf_object { · 555 unmodified lines
694695 /* Information when doing ELF related work. Only valid if efile.elf is not NULL */
695696 struct elf_state efile;
696697
698+ unsigned char byteorder;
699+
697700 struct btf *btf;
698701 struct btf_ext *btf_ext;
699702
@@ -940,6 +943,20 @@ bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data, · 240 unmodified lines
940943 return 0;
941944 }
942945
946+static void bpf_object_bswap_progs(struct bpf_object *obj)
947+{
948+ struct bpf_program *prog = obj->programs;
949+ struct bpf_insn *insn;
950+ int p, i;
951+
952+ for (p = 0; p < obj->nr_programs; p++, prog++) {
953+ insn = prog->insns;
954+ for (i = 0; i < prog->insns_cnt; i++, insn++)
955+ bpf_insn_bswap(insn);
956+ }
957+ pr_debug("converted %zu BPF programs to native byte order\n", obj->nr_programs);
958+}
959+
943960 static const struct btf_member *
944961 find_member_by_offset(const struct btf_type *t, __u32 bit_offset)
945962 {
@@ -1506,6 +1523,7 @@ static void bpf_object__elf_finish(struct bpf_object *obj) · 560 unmodified lines
15061523
15071524 elf_end(obj->efile.elf);
15081525 obj->efile.elf = NULL;
1526+ obj->efile.ehdr = NULL;
15091527 obj->efile.symbols = NULL;
15101528 obj->efile.arena_data = NULL;
15111529
@@ -1533,11 +1551,8 @@ static int bpf_object__elf_init(struct bpf_object *obj) · 21 unmodified lines
15331551 } else {
15341552 obj->efile.fd = open(obj->path, O_RDONLY | O_CLOEXEC);
15351553 if (obj->efile.fd < 0) {
1536− char errmsg[STRERR_BUFSIZE], *cp;
1537−
15381554 err = -errno;
1539− cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
1540− pr_warn("elf: failed to open %s: %s\n", obj->path, cp);
1555+ pr_warn("elf: failed to open %s: %s\n", obj->path, errstr(err));
15411556 return err;
15421557 }
15431558
@@ -1571,6 +1586,16 @@ static int bpf_object__elf_init(struct bpf_object *obj) · 27 unmodified lines
15711586 goto errout;
15721587 }
15731588
1589+ /* Validate ELF object endianness... */
1590+ if (ehdr->e_ident[EI_DATA] != ELFDATA2LSB &&
1591+ ehdr->e_ident[EI_DATA] != ELFDATA2MSB) {
1592+ err = -LIBBPF_ERRNO__ENDIAN;
1593+ pr_warn("elf: '%s' has unknown byte order\n", obj->path);
1594+ goto errout;
1595+ }
1596+ /* and save after bpf_object_open() frees ELF data */
1597+ obj->byteorder = ehdr->e_ident[EI_DATA];
1598+
15741599 if (elf_getshdrstrndx(elf, &obj->efile.shstrndx)) {
15751600 pr_warn("elf: failed to get section names section index for %s: %s\n",
15761601 obj->path, elf_errmsg(-1));
@@ -1599,19 +1624,15 @@ errout: · 22 unmodified lines
15991624 return err;
16001625 }
16011626
1602−static int bpf_object__check_endianness(struct bpf_object *obj)
1627+static bool is_native_endianness(struct bpf_object *obj)
16031628 {
16041629 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
1605− if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2LSB)
1606− return 0;
1630+ return obj->byteorder == ELFDATA2LSB;
16071631 #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
1608− if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2MSB)
1609− return 0;
1632+ return obj->byteorder == ELFDATA2MSB;
16101633 #else
16111634 # error "Unrecognized __BYTE_ORDER__"
16121635 #endif
1613− pr_warn("elf: endianness mismatch in %s.\n", obj->path);
1614− return -LIBBPF_ERRNO__ENDIAN;
16151636 }
16161637
16171638 static int
@@ -1937,8 +1958,7 @@ bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, · 319 unmodified lines
19371958 if (map->mmaped == MAP_FAILED) {
19381959 err = -errno;
19391960 map->mmaped = NULL;
1940− pr_warn("failed to alloc map '%s' content buffer: %d\n",
1941− map->name, err);
1961+ pr_warn("failed to alloc map '%s' content buffer: %s\n", map->name, errstr(err));
19421962 zfree(&map->real_name);
19431963 zfree(&map->name);
19441964 return err;
@@ -2102,7 +2122,7 @@ static int parse_u64(const char *value, __u64 *res) · 157 unmodified lines
21022122 *res = strtoull(value, &value_end, 0);
21032123 if (errno) {
21042124 err = -errno;
2105− pr_warn("failed to parse '%s' as integer: %d\n", value, err);
2125+ pr_warn("failed to parse '%s': %s\n", value, errstr(err));
21062126 return err;
21072127 }
21082128 if (*value_end) {
@@ -2268,8 +2288,8 @@ static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data) · 159 unmodified lines
22682288 while (gzgets(file, buf, sizeof(buf))) {
22692289 err = bpf_object__process_kconfig_line(obj, buf, data);
22702290 if (err) {
2271− pr_warn("error parsing system Kconfig line '%s': %d\n",
2272− buf, err);
2291+ pr_warn("error parsing system Kconfig line '%s': %s\n",
2292+ buf, errstr(err));
22732293 goto out;
22742294 }
22752295 }
@@ -2289,15 +2309,15 @@ static int bpf_object__read_kconfig_mem(struct bpf_object *obj, · 13 unmodified lines
22892309 file = fmemopen((void *)config, strlen(config), "r");
22902310 if (!file) {
22912311 err = -errno;
2292− pr_warn("failed to open in-memory Kconfig: %d\n", err);
2312+ pr_warn("failed to open in-memory Kconfig: %s\n", errstr(err));
22932313 return err;
22942314 }
22952315
22962316 while (fgets(buf, sizeof(buf), file)) {
22972317 err = bpf_object__process_kconfig_line(obj, buf, data);
22982318 if (err) {
2299− pr_warn("error parsing in-memory Kconfig line '%s': %d\n",
2300− buf, err);
2319+ pr_warn("error parsing in-memory Kconfig line '%s': %s\n",
2320+ buf, errstr(err));
23012321 break;
23022322 }
23032323 }
@@ -3212,7 +3232,7 @@ static int bpf_object__init_btf(struct bpf_object *obj, · 908 unmodified lines
32123232 err = libbpf_get_error(obj->btf);
32133233 if (err) {
32143234 obj->btf = NULL;
3215− pr_warn("Error loading ELF section %s: %d.\n", BTF_ELF_SEC, err);
3235+ pr_warn("Error loading ELF section %s: %s.\n", BTF_ELF_SEC, errstr(err));
32163236 goto out;
32173237 }
32183238 /* enforce 8-byte pointers for BPF-targeted BTFs */
@@ -3230,8 +3250,8 @@ static int bpf_object__init_btf(struct bpf_object *obj, · 11 unmodified lines
32303250 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size);
32313251 err = libbpf_get_error(obj->btf_ext);
32323252 if (err) {
3233− pr_warn("Error loading ELF section %s: %d. Ignored and continue.\n",
3234− BTF_EXT_ELF_SEC, err);
3253+ pr_warn("Error loading ELF section %s: %s. Ignored and continue.\n",
3254+ BTF_EXT_ELF_SEC, errstr(err));
32353255 obj->btf_ext = NULL;
32363256 goto out;
32373257 }
@@ -3323,8 +3343,8 @@ static int btf_fixup_datasec(struct bpf_object *obj, struct btf *btf, · 85 unmodified lines
33233343 if (t->size == 0) {
33243344 err = find_elf_sec_sz(obj, sec_name, &size);
33253345 if (err || !size) {
3326− pr_debug("sec '%s': failed to determine size from ELF: size %u, err %d\n",
3327− sec_name, size, err);
3346+ pr_debug("sec '%s': failed to determine size from ELF: size %u, err %s\n",
3347+ sec_name, size, errstr(err));
33283348 return -ENOENT;
33293349 }
33303350
@@ -3478,7 +3498,7 @@ static int bpf_object__load_vmlinux_btf(struct bpf_object *obj, bool force) · 147 unmodified lines
34783498 obj->btf_vmlinux = btf__load_vmlinux_btf();
34793499 err = libbpf_get_error(obj->btf_vmlinux);
34803500 if (err) {
3481− pr_warn("Error loading vmlinux BTF: %d\n", err);
3501+ pr_warn("Error loading vmlinux BTF: %s\n", errstr(err));
34823502 obj->btf_vmlinux = NULL;
34833503 return err;
34843504 }
@@ -3581,11 +3601,14 @@ static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) · 96 unmodified lines
35813601 report:
35823602 if (err) {
35833603 btf_mandatory = kernel_needs_btf(obj);
3584− pr_warn("Error loading .BTF into kernel: %d. %s\n", err,
3585− btf_mandatory ? "BTF is mandatory, can't proceed."
3586− : "BTF is optional, ignoring.");
3587− if (!btf_mandatory)
3604+ if (btf_mandatory) {
3605+ pr_warn("Error loading .BTF into kernel: %s. BTF is mandatory, can't proceed.\n",
3606+ errstr(err));
3607+ } else {
3608+ pr_info("Error loading .BTF into kernel: %s. BTF is optional, ignoring.\n",
3609+ errstr(err));
35883610 err = 0;
3611+ }
35893612 }
35903613 return err;
35913614 }
@@ -3953,6 +3976,10 @@ static int bpf_object__elf_collect(struct bpf_object *obj) · 361 unmodified lines
39533976 return -LIBBPF_ERRNO__FORMAT;
39543977 }
39553978
3979+ /* change BPF program insns to native endianness for introspection */
3980+ if (!is_native_endianness(obj))
3981+ bpf_object_bswap_progs(obj);
3982+
39563983 /* sort BPF programs by section name and in-section instruction offset
39573984 * for faster search
39583985 */
@@ -3985,7 +4012,7 @@ static bool sym_is_subprog(const Elf64_Sym *sym, int text_shndx) · 26 unmodified lines
39854012 return true;
39864013
39874014 /* global function */
3988− return bind == STB_GLOBAL && type == STT_FUNC;
4015+ return (bind == STB_GLOBAL || bind == STB_WEAK) && type == STT_FUNC;
39894016 }
39904017
39914018 static int find_extern_btf_id(const struct btf *btf, const char *ext_name)
@@ -4389,7 +4416,7 @@ static int bpf_object__collect_externs(struct bpf_object *obj) · 397 unmodified lines
43894416
43904417 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog)
43914418 {
4392− return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1;
4419+ return prog->sec_idx == obj->efile.text_shndx;
43934420 }
43944421
43954422 struct bpf_program *
@@ -4783,8 +4810,8 @@ static int bpf_get_map_info_from_fdinfo(int fd, struct bpf_map_info *info) · 387 unmodified lines
47834810 fp = fopen(file, "re");
47844811 if (!fp) {
47854812 err = -errno;
4786− pr_warn("failed to open %s: %d. No procfs support?\n", file,
4787− err);
4813+ pr_warn("failed to open %s: %s. No procfs support?\n", file,
4814+ errstr(err));
47884815 return err;
47894816 }
47904817
@@ -4939,8 +4966,8 @@ static int bpf_object_prepare_token(struct bpf_object *obj) · 148 unmodified lines
49394966 bpffs_fd = open(bpffs_path, O_DIRECTORY, O_RDWR);
49404967 if (bpffs_fd < 0) {
49414968 err = -errno;
4942− __pr(level, "object '%s': failed (%d) to open BPF FS mount at '%s'%s\n",
4943− obj->name, err, bpffs_path,
4969+ __pr(level, "object '%s': failed (%s) to open BPF FS mount at '%s'%s\n",
4970+ obj->name, errstr(err), bpffs_path,
49444971 mandatory ? "" : ", skipping optional step...");
49454972 return mandatory ? err : 0;
49464973 }
@@ -4974,7 +5001,6 @@ static int bpf_object_prepare_token(struct bpf_object *obj) · 27 unmodified lines
49745001 static int
49755002 bpf_object__probe_loading(struct bpf_object *obj)
49765003 {
4977− char *cp, errmsg[STRERR_BUFSIZE];
49785004 struct bpf_insn insns[] = {
49795005 BPF_MOV64_IMM(BPF_REG_0, 0),
49805006 BPF_EXIT_INSN(),
@@ -4990,7 +5016,8 @@ bpf_object__probe_loading(struct bpf_object *obj) · 9 unmodified lines
49905016
49915017 ret = bump_rlimit_memlock();
49925018 if (ret)
4993− pr_warn("Failed to bump RLIMIT_MEMLOCK (err = %d), you might need to do it explicitly!\n", ret);
5019+ pr_warn("Failed to bump RLIMIT_MEMLOCK (err = %s), you might need to do it explicitly!\n",
5020+ errstr(ret));
49945021
49955022 /* make sure basic loading works */
49965023 ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &opts);
@@ -4998,11 +5025,8 @@ bpf_object__probe_loading(struct bpf_object *obj) · 1 unmodified line
49985025 ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, &opts);
49995026 if (ret < 0) {
50005027 ret = errno;
5001− cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
5002− pr_warn("Error in %s():%s(%d). Couldn't load trivial BPF "
5003− "program. Make sure your kernel supports BPF "
5004− "(CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is "
5005− "set to big enough value.\n", __func__, cp, ret);
5028+ pr_warn("Error in %s(): %s. Couldn't load trivial BPF program. Make sure your kernel supports BPF (CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is set to big enough value.\n",
5029+ __func__, errstr(ret));
50065030 return -ret;
50075031 }
50085032 close(ret);
@@ -5027,7 +5051,6 @@ bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id) · 18 unmodified lines
50275051 static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd)
50285052 {
50295053 struct bpf_map_info map_info;
5030− char msg[STRERR_BUFSIZE];
50315054 __u32 map_info_len = sizeof(map_info);
50325055 int err;
50335056
@@ -5037,7 +5060,7 @@ static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd) · 3 unmodified lines
50375060 err = bpf_get_map_info_from_fdinfo(map_fd, &map_info);
50385061 if (err) {
50395062 pr_warn("failed to get map info for map FD %d: %s\n", map_fd,
5040− libbpf_strerror_r(errno, msg, sizeof(msg)));
5063+ errstr(err));
50415064 return false;
50425065 }
50435066
@@ -5052,7 +5075,6 @@ static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd) · 8 unmodified lines
50525075 static int
50535076 bpf_object__reuse_map(struct bpf_map *map)
50545077 {
5055− char *cp, errmsg[STRERR_BUFSIZE];
50565078 int err, pin_fd;
50575079
50585080 pin_fd = bpf_obj_get(map->pin_path);
@@ -5064,9 +5086,8 @@ bpf_object__reuse_map(struct bpf_map *map) · 5 unmodified lines
50645086 return 0;
50655087 }
50665088
5067− cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
50685089 pr_warn("couldn't retrieve pinned map '%s': %s\n",
5069− map->pin_path, cp);
5090+ map->pin_path, errstr(err));
50705091 return err;
50715092 }
50725093
@@ -5092,8 +5113,8 @@ static int · 19 unmodified lines
50925113 bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map)
50935114 {
50945115 enum libbpf_map_type map_type = map->libbpf_type;
5095− char *cp, errmsg[STRERR_BUFSIZE];
50965116 int err, zero = 0;
5117+ size_t mmap_sz;
50975118
50985119 if (obj->gen_loader) {
50995120 bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps,
@@ -5106,9 +5127,8 @@ bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) · 6 unmodified lines
51065127 err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0);
51075128 if (err) {
51085129 err = -errno;
5109− cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
5110− pr_warn("Error setting initial map(%s) contents: %s\n",
5111− map->name, cp);
5130+ pr_warn("map '%s': failed to set initial contents: %s\n",
5131+ bpf_map__name(map), errstr(err));
51125132 return err;
51135133 }
51145134
@@ -5117,12 +5137,43 @@ bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) · 2 unmodified lines
51175137 err = bpf_map_freeze(map->fd);
51185138 if (err) {
51195139 err = -errno;
5120− cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
5121− pr_warn("Error freezing map(%s) as read-only: %s\n",
5122− map->name, cp);
5140+ pr_warn("map '%s': failed to freeze as read-only: %s\n",
5141+ bpf_map__name(map), errstr(err));
51235142 return err;
51245143 }
51255144 }
5145+
5146+ /* Remap anonymous mmap()-ed "map initialization image" as
5147+ * a BPF map-backed mmap()-ed memory, but preserving the same
5148+ * memory address. This will cause kernel to change process'
5149+ * page table to point to a different piece of kernel memory,
5150+ * but from userspace point of view memory address (and its
5151+ * contents, being identical at this point) will stay the
5152+ * same. This mapping will be released by bpf_object__close()
5153+ * as per normal clean up procedure.
5154+ */
5155+ mmap_sz = bpf_map_mmap_sz(map);
5156+ if (map->def.map_flags & BPF_F_MMAPABLE) {
5157+ void *mmaped;
5158+ int prot;
5159+
5160+ if (map->def.map_flags & BPF_F_RDONLY_PROG)
5161+ prot = PROT_READ;
5162+ else
5163+ prot = PROT_READ | PROT_WRITE;
5164+ mmaped = mmap(map->mmaped, mmap_sz, prot, MAP_SHARED | MAP_FIXED, map->fd, 0);
5165+ if (mmaped == MAP_FAILED) {
5166+ err = -errno;
5167+ pr_warn("map '%s': failed to re-mmap() contents: %s\n",
5168+ bpf_map__name(map), errstr(err));
5169+ return err;
5170+ }
5171+ map->mmaped = mmaped;
5172+ } else if (map->mmaped) {
5173+ munmap(map->mmaped, mmap_sz);
5174+ map->mmaped = NULL;
5175+ }
5176+
51265177 return 0;
51275178 }
51285179
@@ -5171,8 +5222,8 @@ static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, b · 42 unmodified lines
51715222 return err;
51725223 err = bpf_object__create_map(obj, map->inner_map, true);
51735224 if (err) {
5174− pr_warn("map '%s': failed to create inner map: %d\n",
5175− map->name, err);
5225+ pr_warn("map '%s': failed to create inner map: %s\n",
5226+ map->name, errstr(err));
51765227 return err;
51775228 }
51785229 map->inner_map_fd = map->inner_map->fd;
@@ -5226,12 +5277,9 @@ static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, b · 47 unmodified lines
52265277 def->max_entries, &create_attr);
52275278 }
52285279 if (map_fd < 0 && (create_attr.btf_key_type_id || create_attr.btf_value_type_id)) {
5229− char *cp, errmsg[STRERR_BUFSIZE];
5230−
52315280 err = -errno;
5232− cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
5233− pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",
5234− map->name, cp, err);
5281+ pr_warn("Error in bpf_create_map_xattr(%s): %s. Retrying without BTF.\n",
5282+ map->name, errstr(err));
52355283 create_attr.btf_fd = 0;
52365284 create_attr.btf_key_type_id = 0;
52375285 create_attr.btf_value_type_id = 0;
@@ -5286,8 +5334,8 @@ static int init_map_in_map_slots(struct bpf_object *obj, struct bpf_map *map) · 48 unmodified lines
52865334 }
52875335 if (err) {
52885336 err = -errno;
5289− pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %d\n",
5290− map->name, i, targ_map->name, fd, err);
5337+ pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %s\n",
5338+ map->name, i, targ_map->name, fd, errstr(err));
52915339 return err;
52925340 }
52935341 pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n",
@@ -5319,8 +5367,8 @@ static int init_prog_array_slots(struct bpf_object *obj, struct bpf_map *map) · 25 unmodified lines
53195367 err = bpf_map_update_elem(map->fd, &i, &fd, 0);
53205368 if (err) {
53215369 err = -errno;
5322− pr_warn("map '%s': failed to initialize slot [%d] to prog '%s' fd=%d: %d\n",
5323− map->name, i, targ_prog->name, fd, err);
5370+ pr_warn("map '%s': failed to initialize slot [%d] to prog '%s' fd=%d: %s\n",
5371+ map->name, i, targ_prog->name, fd, errstr(err));
53245372 return err;
53255373 }
53265374 pr_debug("map '%s': slot [%d] set to prog '%s' fd=%d\n",
@@ -5373,7 +5421,6 @@ static int · 46 unmodified lines
53735421 bpf_object__create_maps(struct bpf_object *obj)
53745422 {
53755423 struct bpf_map *map;
5376− char *cp, errmsg[STRERR_BUFSIZE];
53775424 unsigned int i, j;
53785425 int err;
53795426 bool retried;
@@ -5439,8 +5486,7 @@ retry: · 59 unmodified lines
54395486 err = bpf_object__populate_internal_map(obj, map);
54405487 if (err < 0)
54415488 goto err_out;
5442− }
5443− if (map->def.type == BPF_MAP_TYPE_ARENA) {
5489+ } else if (map->def.type == BPF_MAP_TYPE_ARENA) {
54445490 map->mmaped = mmap((void *)(long)map->map_extra,
54455491 bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE,
54465492 map->map_extra ? MAP_SHARED | MAP_FIXED : MAP_SHARED,
@@ -5448,8 +5494,8 @@ retry: · 1 unmodified line
54485494 if (map->mmaped == MAP_FAILED) {
54495495 err = -errno;
54505496 map->mmaped = NULL;
5451− pr_warn("map '%s': failed to mmap arena: %d\n",
5452− map->name, err);
5497+ pr_warn("map '%s': failed to mmap arena: %s\n",
5498+ map->name, errstr(err));
54535499 return err;
54545500 }
54555501 if (obj->arena_data) {
@@ -5471,8 +5517,8 @@ retry: · 15 unmodified lines
54715517 retried = true;
54725518 goto retry;
54735519 }
5474− pr_warn("map '%s': failed to auto-pin at '%s': %d\n",
5475− map->name, map->pin_path, err);
5520+ pr_warn("map '%s': failed to auto-pin at '%s': %s\n",
5521+ map->name, map->pin_path, errstr(err));
54765522 goto err_out;
54775523 }
54785524 }
@@ -5481,8 +5527,7 @@ retry: · 2 unmodified lines
54815527 return 0;
54825528
54835529 err_out:
5484− cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
5485− pr_warn("map '%s': failed to create: %s(%d)\n", map->name, cp, err);
5530+ pr_warn("map '%s': failed to create: %s\n", map->name, errstr(err));
54865531 pr_perm_msg(err);
54875532 for (j = 0; j < i; j++)
54885533 zclose(obj->maps[j].fd);
@@ -5606,7 +5651,7 @@ static int load_module_btfs(struct bpf_object *obj) · 117 unmodified lines
56065651 }
56075652 if (err) {
56085653 err = -errno;
5609− pr_warn("failed to iterate BTF objects: %d\n", err);
5654+ pr_warn("failed to iterate BTF objects: %s\n", errstr(err));
56105655 return err;
56115656 }
56125657
@@ -5615,7 +5660,7 @@ static int load_module_btfs(struct bpf_object *obj) · 2 unmodified lines
56155660 if (errno == ENOENT)
56165661 continue; /* expected race: BTF was unloaded */
56175662 err = -errno;
5618− pr_warn("failed to get BTF object #%d FD: %d\n", id, err);
5663+ pr_warn("failed to get BTF object #%d FD: %s\n", id, errstr(err));
56195664 return err;
56205665 }
56215666
@@ -5627,7 +5672,7 @@ static int load_module_btfs(struct bpf_object *obj) · 5 unmodified lines
56275672 err = bpf_btf_get_info_by_fd(fd, &info, &len);
56285673 if (err) {
56295674 err = -errno;
5630− pr_warn("failed to get BTF object #%d info: %d\n", id, err);
5675+ pr_warn("failed to get BTF object #%d info: %s\n", id, errstr(err));
56315676 goto err_out;
56325677 }
56335678
@@ -5640,8 +5685,8 @@ static int load_module_btfs(struct bpf_object *obj) · 6 unmodified lines
56405685 btf = btf_get_from_fd(fd, obj->btf_vmlinux);
56415686 err = libbpf_get_error(btf);
56425687 if (err) {
5643− pr_warn("failed to load module [%s]'s BTF object #%d: %d\n",
5644− name, id, err);
5688+ pr_warn("failed to load module [%s]'s BTF object #%d: %s\n",
5689+ name, id, errstr(err));
56455690 goto err_out;
56465691 }
56475692
@@ -5870,7 +5915,7 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) · 222 unmodified lines
58705915 obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL);
58715916 err = libbpf_get_error(obj->btf_vmlinux_override);
58725917 if (err) {
5873− pr_warn("failed to parse target BTF: %d\n", err);
5918+ pr_warn("failed to parse target BTF: %s\n", errstr(err));
58745919 return err;
58755920 }
58765921 }
@@ -5930,8 +5975,8 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) · 53 unmodified lines
59305975
59315976 err = record_relo_core(prog, rec, insn_idx);
59325977 if (err) {
5933− pr_warn("prog '%s': relo #%d: failed to record relocation: %d\n",
5934− prog->name, i, err);
5978+ pr_warn("prog '%s': relo #%d: failed to record relocation: %s\n",
5979+ prog->name, i, errstr(err));
59355980 goto out;
59365981 }
59375982
@@ -5940,15 +5985,15 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) · 2 unmodified lines
59405985
59415986 err = bpf_core_resolve_relo(prog, rec, i, obj->btf, cand_cache, &targ_res);
59425987 if (err) {
5943− pr_warn("prog '%s': relo #%d: failed to relocate: %d\n",
5944− prog->name, i, err);
5988+ pr_warn("prog '%s': relo #%d: failed to relocate: %s\n",
5989+ prog->name, i, errstr(err));
59455990 goto out;
59465991 }
59475992
59485993 err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res);
59495994 if (err) {
5950− pr_warn("prog '%s': relo #%d: failed to patch insn #%u: %d\n",
5951− prog->name, i, insn_idx, err);
5995+ pr_warn("prog '%s': relo #%d: failed to patch insn #%u: %s\n",
5996+ prog->name, i, insn_idx, errstr(err));
59525997 goto out;
59535998 }
59545999 }
@@ -6216,8 +6261,8 @@ reloc_prog_func_and_line_info(const struct bpf_object *obj, · 261 unmodified lines
62166261 &main_prog->func_info_rec_size);
62176262 if (err) {
62186263 if (err != -ENOENT) {
6219− pr_warn("prog '%s': error relocating .BTF.ext function info: %d\n",
6220− prog->name, err);
6264+ pr_warn("prog '%s': error relocating .BTF.ext function info: %s\n",
6265+ prog->name, errstr(err));
62216266 return err;
62226267 }
62236268 if (main_prog->func_info) {
@@ -6244,8 +6289,8 @@ line_info: · 20 unmodified lines
62446289 &main_prog->line_info_rec_size);
62456290 if (err) {
62466291 if (err != -ENOENT) {
6247− pr_warn("prog '%s': error relocating .BTF.ext line info: %d\n",
6248− prog->name, err);
6292+ pr_warn("prog '%s': error relocating .BTF.ext line info: %s\n",
6293+ prog->name, errstr(err));
62496294 return err;
62506295 }
62516296 if (main_prog->line_info) {
@@ -7009,8 +7054,8 @@ static int bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_pat · 757 unmodified lines
70097054 if (obj->btf_ext) {
70107055 err = bpf_object__relocate_core(obj, targ_btf_path);
70117056 if (err) {
7012− pr_warn("failed to perform CO-RE relocations: %d\n",
7013− err);
7057+ pr_warn("failed to perform CO-RE relocations: %s\n",
7058+ errstr(err));
70147059 return err;
70157060 }
70167061 bpf_object__sort_relos(obj);
@@ -7054,8 +7099,8 @@ static int bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_pat · 37 unmodified lines
70547099
70557100 err = bpf_object__relocate_calls(obj, prog);
70567101 if (err) {
7057− pr_warn("prog '%s': failed to relocate calls: %d\n",
7058− prog->name, err);
7102+ pr_warn("prog '%s': failed to relocate calls: %s\n",
7103+ prog->name, errstr(err));
70597104 return err;
70607105 }
70617106
@@ -7091,16 +7136,16 @@ static int bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_pat · 29 unmodified lines
70917136 /* Process data relos for main programs */
70927137 err = bpf_object__relocate_data(obj, prog);
70937138 if (err) {
7094− pr_warn("prog '%s': failed to relocate data references: %d\n",
7095− prog->name, err);
7139+ pr_warn("prog '%s': failed to relocate data references: %s\n",
7140+ prog->name, errstr(err));
70967141 return err;
70977142 }
70987143
70997144 /* Fix up .BTF.ext information, if necessary */
71007145 err = bpf_program_fixup_func_info(obj, prog);
71017146 if (err) {
7102− pr_warn("prog '%s': failed to perform .BTF.ext fix ups: %d\n",
7103− prog->name, err);
7147+ pr_warn("prog '%s': failed to perform .BTF.ext fix ups: %s\n",
7148+ prog->name, errstr(err));
71047149 return err;
71057150 }
71067151 }
@@ -7352,8 +7397,14 @@ static int libbpf_prepare_prog_load(struct bpf_program *prog, · 245 unmodified lines
73527397 opts->prog_flags |= BPF_F_XDP_HAS_FRAGS;
73537398
73547399 /* special check for usdt to use uprobe_multi link */
7355− if ((def & SEC_USDT) && kernel_supports(prog->obj, FEAT_UPROBE_MULTI_LINK))
7400+ if ((def & SEC_USDT) && kernel_supports(prog->obj, FEAT_UPROBE_MULTI_LINK)) {
7401+ /* for BPF_TRACE_UPROBE_MULTI, user might want to query expected_attach_type
7402+ * in prog, and expected_attach_type we set in kernel is from opts, so we
7403+ * update both.
7404+ */
73567405 prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI;
7406+ opts->expected_attach_type = BPF_TRACE_UPROBE_MULTI;
7407+ }
73577408
73587409 if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) {
73597410 int btf_obj_fd = 0, btf_type_id = 0, err;
@@ -7403,7 +7454,6 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog · 43 unmodified lines
74037454 {
74047455 LIBBPF_OPTS(bpf_prog_load_opts, load_attr);
74057456 const char *prog_name = NULL;
7406− char *cp, errmsg[STRERR_BUFSIZE];
74077457 size_t log_buf_size = 0;
74087458 char *log_buf = NULL, *tmp;
74097459 bool own_log_buf = true;
@@ -7443,6 +7493,7 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog · 33 unmodified lines
74437493 load_attr.attach_btf_id = prog->attach_btf_id;
74447494 load_attr.kern_version = kern_version;
74457495 load_attr.prog_ifindex = prog->prog_ifindex;
7496+ load_attr.expected_attach_type = prog->expected_attach_type;
74467497
74477498 /* specify func_info/line_info only if kernel supports them */
74487499 if (obj->btf && btf__fd(obj->btf) >= 0 && kernel_supports(obj, FEAT_BTF_FUNC)) {
@@ -7466,17 +7517,14 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog · 17 unmodified lines
74667517 if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) {
74677518 err = prog->sec_def->prog_prepare_load_fn(prog, &load_attr, prog->sec_def->cookie);
74687519 if (err < 0) {
7469− pr_warn("prog '%s': failed to prepare load attributes: %d\n",
7470− prog->name, err);
7520+ pr_warn("prog '%s': failed to prepare load attributes: %s\n",
7521+ prog->name, errstr(err));
74717522 return err;
74727523 }
74737524 insns = prog->insns;
74747525 insns_cnt = prog->insns_cnt;
74757526 }
74767527
7477− /* allow prog_prepare_load_fn to change expected_attach_type */
7478− load_attr.expected_attach_type = prog->expected_attach_type;
7479−
74807528 if (obj->gen_loader) {
74817529 bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name,
74827530 license, insns, insns_cnt, &load_attr,
@@ -7534,9 +7582,8 @@ retry_load: · 51 unmodified lines
75347582 continue;
75357583
75367584 if (bpf_prog_bind_map(ret, map->fd, NULL)) {
7537− cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
75387585 pr_warn("prog '%s': failed to bind map '%s': %s\n",
7539− prog->name, map->real_name, cp);
7586+ prog->name, map->real_name, errstr(errno));
75407587 /* Don't fail hard if can't bind rodata. */
75417588 }
75427589 }
@@ -7566,8 +7613,7 @@ retry_load: · 23 unmodified lines
75667613 /* post-process verifier log to improve error descriptions */
75677614 fixup_verifier_log(prog, log_buf, log_buf_size);
75687615
7569− cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
7570− pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, cp);
7616+ pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, errstr(errno));
75717617 pr_perm_msg(ret);
75727618
75737619 if (own_log_buf && log_buf && log_buf[0] != '\0') {
@@ -7860,7 +7906,7 @@ bpf_object__load_progs(struct bpf_object *obj, int log_level) · 286 unmodified lines
78607906 err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt,
78617907 obj->license, obj->kern_version, &prog->fd);
78627908 if (err) {
7863− pr_warn("prog '%s': failed to load: %d\n", prog->name, err);
7909+ pr_warn("prog '%s': failed to load: %s\n", prog->name, errstr(err));
78647910 return err;
78657911 }
78667912 }
@@ -7894,8 +7940,8 @@ static int bpf_object_init_progs(struct bpf_object *obj, const struct bpf_object · 27 unmodified lines
78947940 if (prog->sec_def->prog_setup_fn) {
78957941 err = prog->sec_def->prog_setup_fn(prog, prog->sec_def->cookie);
78967942 if (err < 0) {
7897− pr_warn("prog '%s': failed to initialize: %d\n",
7898− prog->name, err);
7943+ pr_warn("prog '%s': failed to initialize: %s\n",
7944+ prog->name, errstr(err));
78997945 return err;
79007946 }
79017947 }
@@ -7992,7 +8038,6 @@ static struct bpf_object *bpf_object_open(const char *path, const void *obj_buf, · 90 unmodified lines
79928038 }
79938039
79948040 err = bpf_object__elf_init(obj);
7995− err = err ? : bpf_object__check_endianness(obj);
79968041 err = err ? : bpf_object__elf_collect(obj);
79978042 err = err ? : bpf_object__collect_externs(obj);
79988043 err = err ? : bpf_object_fixup_btf(obj);
@@ -8085,7 +8130,7 @@ static int libbpf_kallsyms_parse(kallsyms_cb_t cb, void *ctx) · 86 unmodified lines
80858130 f = fopen("/proc/kallsyms", "re");
80868131 if (!f) {
80878132 err = -errno;
8088− pr_warn("failed to open /proc/kallsyms: %d\n", err);
8133+ pr_warn("failed to open /proc/kallsyms: %s\n", errstr(err));
80898134 return err;
80908135 }
80918136
@@ -8498,8 +8543,15 @@ static int bpf_object_load(struct bpf_object *obj, int extra_log_level, const ch · 406 unmodified lines
84988543 return libbpf_err(-EINVAL);
84998544 }
85008545
8501− if (obj->gen_loader)
8546+ /* Disallow kernel loading programs of non-native endianness but
8547+ * permit cross-endian creation of "light skeleton".
8548+ */
8549+ if (obj->gen_loader) {
85028550 bpf_gen__init(obj->gen_loader, extra_log_level, obj->nr_programs, obj->nr_maps);
8551+ } else if (!is_native_endianness(obj)) {
8552+ pr_warn("object '%s': loading non-native endianness is unsupported\n", obj->name);
8553+ return libbpf_err(-LIBBPF_ERRNO__ENDIAN);
8554+ }
85038555
85048556 err = bpf_object_prepare_token(obj);
85058557 err = err ? : bpf_object__probe_loading(obj);
@@ -8562,7 +8614,6 @@ int bpf_object__load(struct bpf_object *obj) · 56 unmodified lines
85628614
85638615 static int make_parent_dir(const char *path)
85648616 {
8565− char *cp, errmsg[STRERR_BUFSIZE];
85668617 char *dname, *dir;
85678618 int err = 0;
85688619
@@ -8576,15 +8627,13 @@ static int make_parent_dir(const char *path) · 7 unmodified lines
85768627
85778628 free(dname);
85788629 if (err) {
8579− cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
8580− pr_warn("failed to mkdir %s: %s\n", path, cp);
8630+ pr_warn("failed to mkdir %s: %s\n", path, errstr(err));
85818631 }
85828632 return err;
85838633 }
85848634
85858635 static int check_path(const char *path)
85868636 {
8587− char *cp, errmsg[STRERR_BUFSIZE];
85888637 struct statfs st_fs;
85898638 char *dname, *dir;
85908639 int err = 0;
@@ -8598,8 +8647,7 @@ static int check_path(const char *path) · 7 unmodified lines
85988647
85998648 dir = dirname(dname);
86008649 if (statfs(dir, &st_fs)) {
8601− cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
8602− pr_warn("failed to statfs %s: %s\n", dir, cp);
8650+ pr_warn("failed to statfs %s: %s\n", dir, errstr(errno));
86038651 err = -errno;
86048652 }
86058653 free(dname);
@@ -8614,7 +8662,6 @@ static int check_path(const char *path) · 8 unmodified lines
86148662
86158663 int bpf_program__pin(struct bpf_program *prog, const char *path)
86168664 {
8617− char *cp, errmsg[STRERR_BUFSIZE];
86188665 int err;
86198666
86208667 if (prog->fd < 0) {
@@ -8632,8 +8679,7 @@ int bpf_program__pin(struct bpf_program *prog, const char *path) · 11 unmodified lines
86328679
86338680 if (bpf_obj_pin(prog->fd, path)) {
86348681 err = -errno;
8635− cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
8636− pr_warn("prog '%s': failed to pin at '%s': %s\n", prog->name, path, cp);
8682+ pr_warn("prog '%s': failed to pin at '%s': %s\n", prog->name, path, errstr(err));
86378683 return libbpf_err(err);
86388684 }
86398685
@@ -8664,7 +8710,6 @@ int bpf_program__unpin(struct bpf_program *prog, const char *path) · 24 unmodified lines
86648710
86658711 int bpf_map__pin(struct bpf_map *map, const char *path)
86668712 {
8667− char *cp, errmsg[STRERR_BUFSIZE];
86688713 int err;
86698714
86708715 if (map == NULL) {
@@ -8723,8 +8768,7 @@ int bpf_map__pin(struct bpf_map *map, const char *path) · 52 unmodified lines
87238768 return 0;
87248769
87258770 out_err:
8726− cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
8727− pr_warn("failed to pin map: %s\n", cp);
8771+ pr_warn("failed to pin map: %s\n", errstr(err));
87288772 return libbpf_err(err);
87298773 }
87308774
@@ -9096,6 +9140,7 @@ int bpf_object__gen_loader(struct bpf_object *obj, struct gen_loader_opts *opts) · 365 unmodified lines
90969140 if (!gen)
90979141 return -ENOMEM;
90989142 gen->opts = opts;
9143+ gen->swapped_endian = !is_native_endianness(obj);
90999144 obj->gen_loader = gen;
91009145 return 0;
91019146 }
@@ -9370,8 +9415,10 @@ static const struct bpf_sec_def section_defs[] = { · 268 unmodified lines
93709415 SEC_DEF("kprobe.session+", KPROBE, BPF_TRACE_KPROBE_SESSION, SEC_NONE, attach_kprobe_session),
93719416 SEC_DEF("uprobe.multi+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_NONE, attach_uprobe_multi),
93729417 SEC_DEF("uretprobe.multi+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_NONE, attach_uprobe_multi),
9418+ SEC_DEF("uprobe.session+", KPROBE, BPF_TRACE_UPROBE_SESSION, SEC_NONE, attach_uprobe_multi),
93739419 SEC_DEF("uprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi),
93749420 SEC_DEF("uretprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi),
9421+ SEC_DEF("uprobe.session.s+", KPROBE, BPF_TRACE_UPROBE_SESSION, SEC_SLEEPABLE, attach_uprobe_multi),
93759422 SEC_DEF("ksyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall),
93769423 SEC_DEF("kretsyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall),
93779424 SEC_DEF("usdt+", KPROBE, 0, SEC_USDT, attach_usdt),
@@ -9910,8 +9957,8 @@ static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd) · 532 unmodified lines
99109957 memset(&info, 0, info_len);
99119958 err = bpf_prog_get_info_by_fd(attach_prog_fd, &info, &info_len);
99129959 if (err) {
9913− pr_warn("failed bpf_prog_get_info_by_fd for FD %d: %d\n",
9914− attach_prog_fd, err);
9960+ pr_warn("failed bpf_prog_get_info_by_fd for FD %d: %s\n",
9961+ attach_prog_fd, errstr(err));
99159962 return err;
99169963 }
99179964
@@ -9923,7 +9970,7 @@ static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd) · 5 unmodified lines
99239970 btf = btf__load_from_kernel_by_id(info.btf_id);
99249971 err = libbpf_get_error(btf);
99259972 if (err) {
9926− pr_warn("Failed to get BTF %d of the program: %d\n", info.btf_id, err);
9973+ pr_warn("Failed to get BTF %d of the program: %s\n", info.btf_id, errstr(err));
99279974 goto out;
99289975 }
99299976 err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC);
@@ -10005,8 +10052,8 @@ static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attac · 75 unmodified lines
1000510052 }
1000610053 err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd);
1000710054 if (err < 0) {
10008− pr_warn("prog '%s': failed to find BPF program (FD %d) BTF ID for '%s': %d\n",
10009− prog->name, attach_prog_fd, attach_name, err);
10055+ pr_warn("prog '%s': failed to find BPF program (FD %d) BTF ID for '%s': %s\n",
10056+ prog->name, attach_prog_fd, attach_name, errstr(err));
1001010057 return err;
1001110058 }
1001210059 *btf_obj_fd = 0;
@@ -10025,8 +10072,8 @@ static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attac · 12 unmodified lines
1002510072 btf_type_id);
1002610073 }
1002710074 if (err) {
10028− pr_warn("prog '%s': failed to find kernel BTF type ID of '%s': %d\n",
10029− prog->name, attach_name, err);
10075+ pr_warn("prog '%s': failed to find kernel BTF type ID of '%s': %s\n",
10076+ prog->name, attach_name, errstr(err));
1003010077 return err;
1003110078 }
1003210079 return 0;
@@ -10254,14 +10301,14 @@ int bpf_map__set_value_size(struct bpf_map *map, __u32 size) · 221 unmodified lines
1025410301 mmap_new_sz = array_map_mmap_sz(size, map->def.max_entries);
1025510302 err = bpf_map_mmap_resize(map, mmap_old_sz, mmap_new_sz);
1025610303 if (err) {
10257− pr_warn("map '%s': failed to resize memory-mapped region: %d\n",
10258− bpf_map__name(map), err);
10304+ pr_warn("map '%s': failed to resize memory-mapped region: %s\n",
10305+ bpf_map__name(map), errstr(err));
1025910306 return err;
1026010307 }
1026110308 err = map_btf_datasec_resize(map, size);
1026210309 if (err && err != -ENOENT) {
10263− pr_warn("map '%s': failed to adjust resized BTF, clearing BTF key/value info: %d\n",
10264− bpf_map__name(map), err);
10310+ pr_warn("map '%s': failed to adjust resized BTF, clearing BTF key/value info: %s\n",
10311+ bpf_map__name(map), errstr(err));
1026510312 map->btf_value_type_id = 0;
1026610313 map->btf_key_type_id = 0;
1026710314 }
@@ -10752,7 +10799,6 @@ static void bpf_link_perf_dealloc(struct bpf_link *link) · 484 unmodified lines
1075210799 struct bpf_link *bpf_program__attach_perf_event_opts(const struct bpf_program *prog, int pfd,
1075310800 const struct bpf_perf_event_opts *opts)
1075410801 {
10755− char errmsg[STRERR_BUFSIZE];
1075610802 struct bpf_link_perf *link;
1075710803 int prog_fd, link_fd = -1, err;
1075810804 bool force_ioctl_attach;
@@ -10787,9 +10833,8 @@ struct bpf_link *bpf_program__attach_perf_event_opts(const struct bpf_program *p · 28 unmodified lines
1078710833 link_fd = bpf_link_create(prog_fd, pfd, BPF_PERF_EVENT, &link_opts);
1078810834 if (link_fd < 0) {
1078910835 err = -errno;
10790− pr_warn("prog '%s': failed to create BPF link for perf_event FD %d: %d (%s)\n",
10791− prog->name, pfd,
10792− err, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10836+ pr_warn("prog '%s': failed to create BPF link for perf_event FD %d: %s\n",
10837+ prog->name, pfd, errstr(err));
1079310838 goto err_out;
1079410839 }
1079510840 link->link.fd = link_fd;
@@ -10803,7 +10848,7 @@ struct bpf_link *bpf_program__attach_perf_event_opts(const struct bpf_program *p · 7 unmodified lines
1080310848 if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) {
1080410849 err = -errno;
1080510850 pr_warn("prog '%s': failed to attach to perf_event FD %d: %s\n",
10806− prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10851+ prog->name, pfd, errstr(err));
1080710852 if (err == -EPROTO)
1080810853 pr_warn("prog '%s': try add PERF_SAMPLE_CALLCHAIN to or remove exclude_callchain_[kernel|user] from pfd %d\n",
1080910854 prog->name, pfd);
@@ -10814,7 +10859,7 @@ struct bpf_link *bpf_program__attach_perf_event_opts(const struct bpf_program *p · 4 unmodified lines
1081410859 if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
1081510860 err = -errno;
1081610861 pr_warn("prog '%s': failed to enable perf_event FD %d: %s\n",
10817− prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10862+ prog->name, pfd, errstr(err));
1081810863 goto err_out;
1081910864 }
1082010865
@@ -10838,22 +10883,19 @@ struct bpf_link *bpf_program__attach_perf_event(const struct bpf_program *prog, · 17 unmodified lines
1083810883 */
1083910884 static int parse_uint_from_file(const char *file, const char *fmt)
1084010885 {
10841− char buf[STRERR_BUFSIZE];
1084210886 int err, ret;
1084310887 FILE *f;
1084410888
1084510889 f = fopen(file, "re");
1084610890 if (!f) {
1084710891 err = -errno;
10848− pr_debug("failed to open '%s': %s\n", file,
10849− libbpf_strerror_r(err, buf, sizeof(buf)));
10892+ pr_debug("failed to open '%s': %s\n", file, errstr(err));
1085010893 return err;
1085110894 }
1085210895 err = fscanf(f, fmt, &ret);
1085310896 if (err != 1) {
1085410897 err = err == EOF ? -EIO : -errno;
10855− pr_debug("failed to parse '%s': %s\n", file,
10856− libbpf_strerror_r(err, buf, sizeof(buf)));
10898+ pr_debug("failed to parse '%s': %s\n", file, errstr(err));
1085710899 fclose(f);
1085810900 return err;
1085910901 }
@@ -10897,7 +10939,6 @@ static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, · 37 unmodified lines
1089710939 {
1089810940 const size_t attr_sz = sizeof(struct perf_event_attr);
1089910941 struct perf_event_attr attr;
10900− char errmsg[STRERR_BUFSIZE];
1090110942 int type, pfd;
1090210943
1090310944 if ((__u64)ref_ctr_off >= (1ULL << PERF_UPROBE_REF_CTR_OFFSET_BITS))
@@ -10910,7 +10951,7 @@ static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, · 6 unmodified lines
1091010951 if (type < 0) {
1091110952 pr_warn("failed to determine %s perf type: %s\n",
1091210953 uprobe ? "uprobe" : "kprobe",
10913− libbpf_strerror_r(type, errmsg, sizeof(errmsg)));
10954+ errstr(type));
1091410955 return type;
1091510956 }
1091610957 if (retprobe) {
@@ -10920,7 +10961,7 @@ static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, · 3 unmodified lines
1092010961 if (bit < 0) {
1092110962 pr_warn("failed to determine %s retprobe bit: %s\n",
1092210963 uprobe ? "uprobe" : "kprobe",
10923− libbpf_strerror_r(bit, errmsg, sizeof(errmsg)));
10964+ errstr(bit));
1092410965 return bit;
1092510966 }
1092610967 attr.config |= 1 << bit;
@@ -11049,14 +11090,13 @@ static int perf_event_kprobe_open_legacy(const char *probe_name, bool retprobe, · 122 unmodified lines
1104911090 {
1105011091 const size_t attr_sz = sizeof(struct perf_event_attr);
1105111092 struct perf_event_attr attr;
11052− char errmsg[STRERR_BUFSIZE];
1105311093 int type, pfd, err;
1105411094
1105511095 err = add_kprobe_event_legacy(probe_name, retprobe, kfunc_name, offset);
1105611096 if (err < 0) {
1105711097 pr_warn("failed to add legacy kprobe event for '%s+0x%zx': %s\n",
1105811098 kfunc_name, offset,
11059− libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
11099+ errstr(err));
1106011100 return err;
1106111101 }
1106211102 type = determine_kprobe_perf_type_legacy(probe_name, retprobe);
@@ -11064,7 +11104,7 @@ static int perf_event_kprobe_open_legacy(const char *probe_name, bool retprobe, · 1 unmodified line
1106411104 err = type;
1106511105 pr_warn("failed to determine legacy kprobe event id for '%s+0x%zx': %s\n",
1106611106 kfunc_name, offset,
11067− libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
11107+ errstr(err));
1106811108 goto err_clean_legacy;
1106911109 }
1107011110
@@ -11080,7 +11120,7 @@ static int perf_event_kprobe_open_legacy(const char *probe_name, bool retprobe, · 9 unmodified lines
1108011120 if (pfd < 0) {
1108111121 err = -errno;
1108211122 pr_warn("legacy kprobe perf_event_open() failed: %s\n",
11083− libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
11123+ errstr(err));
1108411124 goto err_clean_legacy;
1108511125 }
1108611126 return pfd;
@@ -11156,7 +11196,6 @@ bpf_program__attach_kprobe_opts(const struct bpf_program *prog, · 69 unmodified lines
1115611196 {
1115711197 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);
1115811198 enum probe_attach_mode attach_mode;
11159− char errmsg[STRERR_BUFSIZE];
1116011199 char *legacy_probe = NULL;
1116111200 struct bpf_link *link;
1116211201 size_t offset;
@@ -11214,7 +11253,7 @@ bpf_program__attach_kprobe_opts(const struct bpf_program *prog, · 51 unmodified lines
1121411253 pr_warn("prog '%s': failed to create %s '%s+0x%zx' perf event: %s\n",
1121511254 prog->name, retprobe ? "kretprobe" : "kprobe",
1121611255 func_name, offset,
11217− libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
11256+ errstr(err));
1121811257 goto err_out;
1121911258 }
1122011259 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
@@ -11224,7 +11263,7 @@ bpf_program__attach_kprobe_opts(const struct bpf_program *prog, · 3 unmodified lines
1122411263 pr_warn("prog '%s': failed to attach to %s '%s+0x%zx': %s\n",
1122511264 prog->name, retprobe ? "kretprobe" : "kprobe",
1122611265 func_name, offset,
11227− libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
11266+ errstr(err));
1122811267 goto err_clean_legacy;
1122911268 }
1123011269 if (legacy) {
@@ -11360,7 +11399,7 @@ static int libbpf_available_kallsyms_parse(struct kprobe_multi_resolve *res) · 129 unmodified lines
1136011399 f = fopen(available_functions_file, "re");
1136111400 if (!f) {
1136211401 err = -errno;
11363− pr_warn("failed to open %s: %d\n", available_functions_file, err);
11402+ pr_warn("failed to open %s: %s\n", available_functions_file, errstr(err));
1136411403 return err;
1136511404 }
1136611405
@@ -11435,7 +11474,7 @@ static int libbpf_available_kprobes_parse(struct kprobe_multi_resolve *res) · 68 unmodified lines
1143511474 f = fopen(available_path, "re");
1143611475 if (!f) {
1143711476 err = -errno;
11438− pr_warn("failed to open %s: %d\n", available_path, err);
11477+ pr_warn("failed to open %s: %s\n", available_path, errstr(err));
1143911478 return err;
1144011479 }
1144111480
@@ -11481,7 +11520,6 @@ bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog, · 39 unmodified lines
1148111520 };
1148211521 enum bpf_attach_type attach_type;
1148311522 struct bpf_link *link = NULL;
11484− char errmsg[STRERR_BUFSIZE];
1148511523 const unsigned long *addrs;
1148611524 int err, link_fd, prog_fd;
1148711525 bool retprobe, session;
@@ -11549,7 +11587,7 @@ bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog, · 61 unmodified lines
1154911587 if (link_fd < 0) {
1155011588 err = -errno;
1155111589 pr_warn("prog '%s': failed to attach: %s\n",
11552− prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
11590+ prog->name, errstr(err));
1155311591 goto error;
1155411592 }
1155511593 link->fd = link_fd;
@@ -11693,7 +11731,9 @@ static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, stru · 137 unmodified lines
1169311731 ret = 0;
1169411732 break;
1169511733 case 3:
11734+ opts.session = str_has_pfx(probe_type, "uprobe.session");
1169611735 opts.retprobe = str_has_pfx(probe_type, "uretprobe.multi");
11736+
1169711737 *link = bpf_program__attach_uprobe_multi(prog, -1, binary_path, func_name, &opts);
1169811738 ret = libbpf_get_error(*link);
1169911739 break;
@@ -11756,15 +11796,15 @@ static int perf_event_uprobe_open_legacy(const char *probe_name, bool retprobe, · 56 unmodified lines
1175611796
1175711797 err = add_uprobe_event_legacy(probe_name, retprobe, binary_path, offset);
1175811798 if (err < 0) {
11759− pr_warn("failed to add legacy uprobe event for %s:0x%zx: %d\n",
11760− binary_path, (size_t)offset, err);
11799+ pr_warn("failed to add legacy uprobe event for %s:0x%zx: %s\n",
11800+ binary_path, (size_t)offset, errstr(err));
1176111801 return err;
1176211802 }
1176311803 type = determine_uprobe_perf_type_legacy(probe_name, retprobe);
1176411804 if (type < 0) {
1176511805 err = type;
11766− pr_warn("failed to determine legacy uprobe event id for %s:0x%zx: %d\n",
11767− binary_path, offset, err);
11806+ pr_warn("failed to determine legacy uprobe event id for %s:0x%zx: %s\n",
11807+ binary_path, offset, errstr(err));
1176811808 goto err_clean_legacy;
1176911809 }
1177011810
@@ -11779,7 +11819,7 @@ static int perf_event_uprobe_open_legacy(const char *probe_name, bool retprobe, · 8 unmodified lines
1177911819 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
1178011820 if (pfd < 0) {
1178111821 err = -errno;
11782− pr_warn("legacy uprobe perf_event_open() failed: %d\n", err);
11822+ pr_warn("legacy uprobe perf_event_open() failed: %s\n", errstr(err));
1178311823 goto err_clean_legacy;
1178411824 }
1178511825 return pfd;
@@ -11942,10 +11982,11 @@ bpf_program__attach_uprobe_multi(const struct bpf_program *prog, · 156 unmodified lines
1194211982 const unsigned long *ref_ctr_offsets = NULL, *offsets = NULL;
1194311983 LIBBPF_OPTS(bpf_link_create_opts, lopts);
1194411984 unsigned long *resolved_offsets = NULL;
11985+ enum bpf_attach_type attach_type;
1194511986 int err = 0, link_fd, prog_fd;
1194611987 struct bpf_link *link = NULL;
11947− char errmsg[STRERR_BUFSIZE];
1194811988 char full_path[PATH_MAX];
11989+ bool retprobe, session;
1194911990 const __u64 *cookies;
1195011991 const char **syms;
1195111992 size_t cnt;
@@ -11965,6 +12006,8 @@ bpf_program__attach_uprobe_multi(const struct bpf_program *prog, · 13 unmodified lines
1196512006 ref_ctr_offsets = OPTS_GET(opts, ref_ctr_offsets, NULL);
1196612007 cookies = OPTS_GET(opts, cookies, NULL);
1196712008 cnt = OPTS_GET(opts, cnt, 0);
12009+ retprobe = OPTS_GET(opts, retprobe, false);
12010+ session = OPTS_GET(opts, session, false);
1196812011
1196912012 /*
1197012013 * User can specify 2 mutually exclusive set of inputs:
@@ -11993,12 +12036,15 @@ bpf_program__attach_uprobe_multi(const struct bpf_program *prog, · 22 unmodified lines
1199312036 return libbpf_err_ptr(-EINVAL);
1199412037 }
1199512038
12039+ if (retprobe && session)
12040+ return libbpf_err_ptr(-EINVAL);
12041+
1199612042 if (func_pattern) {
1199712043 if (!strchr(path, '/')) {
1199812044 err = resolve_full_path(path, full_path, sizeof(full_path));
1199912045 if (err) {
12000− pr_warn("prog '%s': failed to resolve full path for '%s': %d\n",
12001− prog->name, path, err);
12046+ pr_warn("prog '%s': failed to resolve full path for '%s': %s\n",
12047+ prog->name, path, errstr(err));
1200212048 return libbpf_err_ptr(err);
1200312049 }
1200412050 path = full_path;
@@ -12016,12 +12062,14 @@ bpf_program__attach_uprobe_multi(const struct bpf_program *prog, · 11 unmodified lines
1201612062 offsets = resolved_offsets;
1201712063 }
1201812064
12065+ attach_type = session ? BPF_TRACE_UPROBE_SESSION : BPF_TRACE_UPROBE_MULTI;
12066+
1201912067 lopts.uprobe_multi.path = path;
1202012068 lopts.uprobe_multi.offsets = offsets;
1202112069 lopts.uprobe_multi.ref_ctr_offsets = ref_ctr_offsets;
1202212070 lopts.uprobe_multi.cookies = cookies;
1202312071 lopts.uprobe_multi.cnt = cnt;
12024− lopts.uprobe_multi.flags = OPTS_GET(opts, retprobe, false) ? BPF_F_UPROBE_MULTI_RETURN : 0;
12072+ lopts.uprobe_multi.flags = retprobe ? BPF_F_UPROBE_MULTI_RETURN : 0;
1202512073
1202612074 if (pid == 0)
1202712075 pid = getpid();
@@ -12035,11 +12083,11 @@ bpf_program__attach_uprobe_multi(const struct bpf_program *prog, · 7 unmodified lines
1203512083 }
1203612084 link->detach = &bpf_link__detach_fd;
1203712085
12038− link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &lopts);
12086+ link_fd = bpf_link_create(prog_fd, 0, attach_type, &lopts);
1203912087 if (link_fd < 0) {
1204012088 err = -errno;
1204112089 pr_warn("prog '%s': failed to attach multi-uprobe: %s\n",
12042− prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
12090+ prog->name, errstr(err));
1204312091 goto error;
1204412092 }
1204512093 link->fd = link_fd;
@@ -12058,7 +12106,7 @@ bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid, · 12 unmodified lines
1205812106 const struct bpf_uprobe_opts *opts)
1205912107 {
1206012108 const char *archive_path = NULL, *archive_sep = NULL;
12061− char errmsg[STRERR_BUFSIZE], *legacy_probe = NULL;
12109+ char *legacy_probe = NULL;
1206212110 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);
1206312111 enum probe_attach_mode attach_mode;
1206412112 char full_path[PATH_MAX];
@@ -12090,8 +12138,8 @@ bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid, · 25 unmodified lines
1209012138 } else if (!strchr(binary_path, '/')) {
1209112139 err = resolve_full_path(binary_path, full_path, sizeof(full_path));
1209212140 if (err) {
12093− pr_warn("prog '%s': failed to resolve full path for '%s': %d\n",
12094− prog->name, binary_path, err);
12141+ pr_warn("prog '%s': failed to resolve full path for '%s': %s\n",
12142+ prog->name, binary_path, errstr(err));
1209512143 return libbpf_err_ptr(err);
1209612144 }
1209712145 binary_path = full_path;
@@ -12157,7 +12205,7 @@ bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid, · 59 unmodified lines
1215712205 pr_warn("prog '%s': failed to create %s '%s:0x%zx' perf event: %s\n",
1215812206 prog->name, retprobe ? "uretprobe" : "uprobe",
1215912207 binary_path, func_offset,
12160− libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
12208+ errstr(err));
1216112209 goto err_out;
1216212210 }
1216312211
@@ -12168,7 +12216,7 @@ bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid, · 4 unmodified lines
1216812216 pr_warn("prog '%s': failed to attach to %s '%s:0x%zx': %s\n",
1216912217 prog->name, retprobe ? "uretprobe" : "uprobe",
1217012218 binary_path, func_offset,
12171− libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
12219+ errstr(err));
1217212220 goto err_clean_legacy;
1217312221 }
1217412222 if (legacy) {
@@ -12289,8 +12337,8 @@ struct bpf_link *bpf_program__attach_usdt(const struct bpf_program *prog, · 114 unmodified lines
1228912337 if (!strchr(binary_path, '/')) {
1229012338 err = resolve_full_path(binary_path, resolved_path, sizeof(resolved_path));
1229112339 if (err) {
12292− pr_warn("prog '%s': failed to resolve full path for '%s': %d\n",
12293− prog->name, binary_path, err);
12340+ pr_warn("prog '%s': failed to resolve full path for '%s': %s\n",
12341+ prog->name, binary_path, errstr(err));
1229412342 return libbpf_err_ptr(err);
1229512343 }
1229612344 binary_path = resolved_path;
@@ -12368,14 +12416,13 @@ static int perf_event_open_tracepoint(const char *tp_category, · 71 unmodified lines
1236812416 {
1236912417 const size_t attr_sz = sizeof(struct perf_event_attr);
1237012418 struct perf_event_attr attr;
12371− char errmsg[STRERR_BUFSIZE];
1237212419 int tp_id, pfd, err;
1237312420
1237412421 tp_id = determine_tracepoint_id(tp_category, tp_name);
1237512422 if (tp_id < 0) {
1237612423 pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n",
1237712424 tp_category, tp_name,
12378− libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg)));
12425+ errstr(tp_id));
1237912426 return tp_id;
1238012427 }
1238112428
@@ -12390,7 +12437,7 @@ static int perf_event_open_tracepoint(const char *tp_category, · 8 unmodified lines
1239012437 err = -errno;
1239112438 pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n",
1239212439 tp_category, tp_name,
12393− libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
12440+ errstr(err));
1239412441 return err;
1239512442 }
1239612443 return pfd;
@@ -12402,7 +12449,6 @@ struct bpf_link *bpf_program__attach_tracepoint_opts(const struct bpf_program *p · 5 unmodified lines
1240212449 const struct bpf_tracepoint_opts *opts)
1240312450 {
1240412451 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);
12405− char errmsg[STRERR_BUFSIZE];
1240612452 struct bpf_link *link;
1240712453 int pfd, err;
1240812454
@@ -12415,7 +12461,7 @@ struct bpf_link *bpf_program__attach_tracepoint_opts(const struct bpf_program *p · 6 unmodified lines
1241512461 if (pfd < 0) {
1241612462 pr_warn("prog '%s': failed to create tracepoint '%s/%s' perf event: %s\n",
1241712463 prog->name, tp_category, tp_name,
12418− libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
12464+ errstr(pfd));
1241912465 return libbpf_err_ptr(pfd);
1242012466 }
1242112467 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
@@ -12424,7 +12470,7 @@ struct bpf_link *bpf_program__attach_tracepoint_opts(const struct bpf_program *p · 2 unmodified lines
1242412470 close(pfd);
1242512471 pr_warn("prog '%s': failed to attach to tracepoint '%s/%s': %s\n",
1242612472 prog->name, tp_category, tp_name,
12427− libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
12473+ errstr(err));
1242812474 return libbpf_err_ptr(err);
1242912475 }
1243012476 return link;
@@ -12475,7 +12521,6 @@ bpf_program__attach_raw_tracepoint_opts(const struct bpf_program *prog, · 44 unmodified lines
1247512521 struct bpf_raw_tracepoint_opts *opts)
1247612522 {
1247712523 LIBBPF_OPTS(bpf_raw_tp_opts, raw_opts);
12478− char errmsg[STRERR_BUFSIZE];
1247912524 struct bpf_link *link;
1248012525 int prog_fd, pfd;
1248112526
@@ -12500,7 +12545,7 @@ bpf_program__attach_raw_tracepoint_opts(const struct bpf_program *prog, · 18 unmodified lines
1250012545 pfd = -errno;
1250112546 free(link);
1250212547 pr_warn("prog '%s': failed to attach to raw tracepoint '%s': %s\n",
12503− prog->name, tp_name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
12548+ prog->name, tp_name, errstr(pfd));
1250412549 return libbpf_err_ptr(pfd);
1250512550 }
1250612551 link->fd = pfd;
@@ -12559,7 +12604,6 @@ static struct bpf_link *bpf_program__attach_btf_id(const struct bpf_program *pro · 52 unmodified lines
1255912604 const struct bpf_trace_opts *opts)
1256012605 {
1256112606 LIBBPF_OPTS(bpf_link_create_opts, link_opts);
12562− char errmsg[STRERR_BUFSIZE];
1256312607 struct bpf_link *link;
1256412608 int prog_fd, pfd;
1256512609
@@ -12584,7 +12628,7 @@ static struct bpf_link *bpf_program__attach_btf_id(const struct bpf_program *pro · 18 unmodified lines
1258412628 pfd = -errno;
1258512629 free(link);
1258612630 pr_warn("prog '%s': failed to attach: %s\n",
12587− prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
12631+ prog->name, errstr(pfd));
1258812632 return libbpf_err_ptr(pfd);
1258912633 }
1259012634 link->fd = pfd;
@@ -12625,7 +12669,6 @@ bpf_program_attach_fd(const struct bpf_program *prog, · 34 unmodified lines
1262512669 const struct bpf_link_create_opts *opts)
1262612670 {
1262712671 enum bpf_attach_type attach_type;
12628− char errmsg[STRERR_BUFSIZE];
1262912672 struct bpf_link *link;
1263012673 int prog_fd, link_fd;
1263112674
@@ -12647,7 +12690,7 @@ bpf_program_attach_fd(const struct bpf_program *prog, · 15 unmodified lines
1264712690 free(link);
1264812691 pr_warn("prog '%s': failed to attach to %s: %s\n",
1264912692 prog->name, target_name,
12650− libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
12693+ errstr(link_fd));
1265112694 return libbpf_err_ptr(link_fd);
1265212695 }
1265312696 link->fd = link_fd;
@@ -12760,7 +12803,7 @@ struct bpf_link *bpf_program__attach_freplace(const struct bpf_program *prog, · 106 unmodified lines
1276012803 }
1276112804
1276212805 if (prog->type != BPF_PROG_TYPE_EXT) {
12763− pr_warn("prog '%s': only BPF_PROG_TYPE_EXT can attach as freplace",
12806+ pr_warn("prog '%s': only BPF_PROG_TYPE_EXT can attach as freplace\n",
1276412807 prog->name);
1276512808 return libbpf_err_ptr(-EINVAL);
1276612809 }
@@ -12789,7 +12832,6 @@ bpf_program__attach_iter(const struct bpf_program *prog, · 22 unmodified lines
1278912832 const struct bpf_iter_attach_opts *opts)
1279012833 {
1279112834 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_create_opts);
12792− char errmsg[STRERR_BUFSIZE];
1279312835 struct bpf_link *link;
1279412836 int prog_fd, link_fd;
1279512837 __u32 target_fd = 0;
@@ -12817,7 +12859,7 @@ bpf_program__attach_iter(const struct bpf_program *prog, · 21 unmodified lines
1281712859 link_fd = -errno;
1281812860 free(link);
1281912861 pr_warn("prog '%s': failed to attach to iterator: %s\n",
12820− prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
12862+ prog->name, errstr(link_fd));
1282112863 return libbpf_err_ptr(link_fd);
1282212864 }
1282312865 link->fd = link_fd;
@@ -12859,12 +12901,10 @@ struct bpf_link *bpf_program__attach_netfilter(const struct bpf_program *prog, · 35 unmodified lines
1285912901
1286012902 link_fd = bpf_link_create(prog_fd, 0, BPF_NETFILTER, &lopts);
1286112903 if (link_fd < 0) {
12862− char errmsg[STRERR_BUFSIZE];
12863−
1286412904 link_fd = -errno;
1286512905 free(link);
1286612906 pr_warn("prog '%s': failed to attach to netfilter: %s\n",
12867− prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
12907+ prog->name, errstr(link_fd));
1286812908 return libbpf_err_ptr(link_fd);
1286912909 }
1287012910 link->fd = link_fd;
@@ -13149,7 +13189,6 @@ perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr, · 278 unmodified lines
1314913189 int cpu, int map_key)
1315013190 {
1315113191 struct perf_cpu_buf *cpu_buf;
13152− char msg[STRERR_BUFSIZE];
1315313192 int err;
1315413193
1315513194 cpu_buf = calloc(1, sizeof(*cpu_buf));
@@ -13165,7 +13204,7 @@ perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr, · 9 unmodified lines
1316513204 if (cpu_buf->fd < 0) {
1316613205 err = -errno;
1316713206 pr_warn("failed to open perf buffer event on cpu #%d: %s\n",
13168− cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
13207+ cpu, errstr(err));
1316913208 goto error;
1317013209 }
1317113210
@@ -13176,14 +13215,14 @@ perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr, · 4 unmodified lines
1317613215 cpu_buf->base = NULL;
1317713216 err = -errno;
1317813217 pr_warn("failed to mmap perf buffer on cpu #%d: %s\n",
13179− cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
13218+ cpu, errstr(err));
1318013219 goto error;
1318113220 }
1318213221
1318313222 if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
1318413223 err = -errno;
1318513224 pr_warn("failed to enable perf buffer event on cpu #%d: %s\n",
13186− cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
13225+ cpu, errstr(err));
1318713226 goto error;
1318813227 }
1318913228
@@ -13259,7 +13298,6 @@ static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, · 69 unmodified lines
1325913298 {
1326013299 const char *online_cpus_file = "/sys/devices/system/cpu/online";
1326113300 struct bpf_map_info map;
13262− char msg[STRERR_BUFSIZE];
1326313301 struct perf_buffer *pb;
1326413302 bool *online = NULL;
1326513303 __u32 map_info_len;
@@ -13282,7 +13320,7 @@ static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, · 16 unmodified lines
1328213320 */
1328313321 if (err != -EINVAL) {
1328413322 pr_warn("failed to get map info for map FD %d: %s\n",
13285− map_fd, libbpf_strerror_r(err, msg, sizeof(msg)));
13323+ map_fd, errstr(err));
1328613324 return ERR_PTR(err);
1328713325 }
1328813326 pr_debug("failed to get map info for FD %d; API not supported? Ignoring...\n",
@@ -13312,7 +13350,7 @@ static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, · 23 unmodified lines
1331213350 if (pb->epoll_fd < 0) {
1331313351 err = -errno;
1331413352 pr_warn("failed to create epoll instance: %s\n",
13315− libbpf_strerror_r(err, msg, sizeof(msg)));
13353+ errstr(err));
1331613354 goto error;
1331713355 }
1331813356
@@ -13343,7 +13381,7 @@ static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, · 24 unmodified lines
1334313381
1334413382 err = parse_cpu_mask_file(online_cpus_file, &online, &n);
1334513383 if (err) {
13346− pr_warn("failed to get online CPU mask: %d\n", err);
13384+ pr_warn("failed to get online CPU mask: %s\n", errstr(err));
1334713385 goto error;
1334813386 }
1334913387
@@ -13374,7 +13412,7 @@ static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, · 24 unmodified lines
1337413412 err = -errno;
1337513413 pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n",
1337613414 cpu, map_key, cpu_buf->fd,
13377− libbpf_strerror_r(err, msg, sizeof(msg)));
13415+ errstr(err));
1337813416 goto error;
1337913417 }
1338013418
@@ -13385,7 +13423,7 @@ static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, · 4 unmodified lines
1338513423 err = -errno;
1338613424 pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n",
1338713425 cpu, cpu_buf->fd,
13388− libbpf_strerror_r(err, msg, sizeof(msg)));
13426+ errstr(err));
1338913427 goto error;
1339013428 }
1339113429 j++;
@@ -13480,7 +13518,7 @@ int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms) · 88 unmodified lines
1348013518
1348113519 err = perf_buffer__process_records(pb, cpu_buf);
1348213520 if (err) {
13483− pr_warn("error while processing records: %d\n", err);
13521+ pr_warn("error while processing records: %s\n", errstr(err));
1348413522 return libbpf_err(err);
1348513523 }
1348613524 }
@@ -13564,7 +13602,8 @@ int perf_buffer__consume(struct perf_buffer *pb) · 77 unmodified lines
1356413602
1356513603 err = perf_buffer__process_records(pb, cpu_buf);
1356613604 if (err) {
13567− pr_warn("perf_buffer: failed to process records in buffer #%d: %d\n", i, err);
13605+ pr_warn("perf_buffer: failed to process records in buffer #%d: %s\n",
13606+ i, errstr(err));
1356813607 return libbpf_err(err);
1356913608 }
1357013609 }
@@ -13675,14 +13714,14 @@ int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz) · 104 unmodified lines
1367513714 fd = open(fcpu, O_RDONLY | O_CLOEXEC);
1367613715 if (fd < 0) {
1367713716 err = -errno;
13678− pr_warn("Failed to open cpu mask file %s: %d\n", fcpu, err);
13717+ pr_warn("Failed to open cpu mask file %s: %s\n", fcpu, errstr(err));
1367913718 return err;
1368013719 }
1368113720 len = read(fd, buf, sizeof(buf));
1368213721 close(fd);
1368313722 if (len <= 0) {
1368413723 err = len ? -errno : -EINVAL;
13685− pr_warn("Failed to read cpu mask from %s: %d\n", fcpu, err);
13724+ pr_warn("Failed to read cpu mask from %s: %s\n", fcpu, errstr(err));
1368613725 return err;
1368713726 }
1368813727 if (len >= sizeof(buf)) {
@@ -13774,20 +13813,21 @@ int bpf_object__open_skeleton(struct bpf_object_skeleton *s, · 85 unmodified lines
1377413813 obj = bpf_object_open(NULL, s->data, s->data_sz, s->name, opts);
1377513814 if (IS_ERR(obj)) {
1377613815 err = PTR_ERR(obj);
13777− pr_warn("failed to initialize skeleton BPF object '%s': %d\n", s->name, err);
13816+ pr_warn("failed to initialize skeleton BPF object '%s': %s\n",
13817+ s->name, errstr(err));
1377813818 return libbpf_err(err);
1377913819 }
1378013820
1378113821 *s->obj = obj;
1378213822 err = populate_skeleton_maps(obj, s->maps, s->map_cnt, s->map_skel_sz);
1378313823 if (err) {
13784− pr_warn("failed to populate skeleton maps for '%s': %d\n", s->name, err);
13824+ pr_warn("failed to populate skeleton maps for '%s': %s\n", s->name, errstr(err));
1378513825 return libbpf_err(err);
1378613826 }
1378713827
1378813828 err = populate_skeleton_progs(obj, s->progs, s->prog_cnt, s->prog_skel_sz);
1378913829 if (err) {
13790− pr_warn("failed to populate skeleton progs for '%s': %d\n", s->name, err);
13830+ pr_warn("failed to populate skeleton progs for '%s': %s\n", s->name, errstr(err));
1379113831 return libbpf_err(err);
1379213832 }
1379313833
@@ -13817,13 +13857,13 @@ int bpf_object__open_subskeleton(struct bpf_object_subskeleton *s) · 23 unmodified lines
1381713857
1381813858 err = populate_skeleton_maps(s->obj, s->maps, s->map_cnt, s->map_skel_sz);
1381913859 if (err) {
13820− pr_warn("failed to populate subskeleton maps: %d\n", err);
13860+ pr_warn("failed to populate subskeleton maps: %s\n", errstr(err));
1382113861 return libbpf_err(err);
1382213862 }
1382313863
1382413864 err = populate_skeleton_progs(s->obj, s->progs, s->prog_cnt, s->prog_skel_sz);
1382513865 if (err) {
13826− pr_warn("failed to populate subskeleton maps: %d\n", err);
13866+ pr_warn("failed to populate subskeleton maps: %s\n", errstr(err));
1382713867 return libbpf_err(err);
1382813868 }
1382913869
@@ -13834,7 +13874,7 @@ int bpf_object__open_subskeleton(struct bpf_object_subskeleton *s) · 4 unmodified lines
1383413874 map_type = btf__type_by_id(btf, map_type_id);
1383513875
1383613876 if (!btf_is_datasec(map_type)) {
13837− pr_warn("type for map '%1$s' is not a datasec: %2$s",
13877+ pr_warn("type for map '%1$s' is not a datasec: %2$s\n",
1383813878 bpf_map__name(map),
1383913879 __btf_kind_str(btf_kind(map_type)));
1384013880 return libbpf_err(-EINVAL);
@@ -13870,53 +13910,18 @@ int bpf_object__load_skeleton(struct bpf_object_skeleton *s) · 29 unmodified lines
1387013910
1387113911 err = bpf_object__load(*s->obj);
1387213912 if (err) {
13873− pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err);
13913+ pr_warn("failed to load BPF skeleton '%s': %s\n", s->name, errstr(err));
1387413914 return libbpf_err(err);
1387513915 }
1387613916
1387713917 for (i = 0; i < s->map_cnt; i++) {
1387813918 struct bpf_map_skeleton *map_skel = (void *)s->maps + i * s->map_skel_sz;
1387913919 struct bpf_map *map = *map_skel->map;
13880− size_t mmap_sz = bpf_map_mmap_sz(map);
13881− int prot, map_fd = map->fd;
13882− void **mmaped = map_skel->mmaped;
1388313920
13884− if (!mmaped)
13921+ if (!map_skel->mmaped)
1388513922 continue;
1388613923
13887− if (!(map->def.map_flags & BPF_F_MMAPABLE)) {
13888− *mmaped = NULL;
13889− continue;
13890− }
13891−
13892− if (map->def.type == BPF_MAP_TYPE_ARENA) {
13893− *mmaped = map->mmaped;
13894− continue;
13895− }
13896−
13897− if (map->def.map_flags & BPF_F_RDONLY_PROG)
13898− prot = PROT_READ;
13899− else
13900− prot = PROT_READ | PROT_WRITE;
13901−
13902− /* Remap anonymous mmap()-ed "map initialization image" as
13903− * a BPF map-backed mmap()-ed memory, but preserving the same
13904− * memory address. This will cause kernel to change process'
13905− * page table to point to a different piece of kernel memory,
13906− * but from userspace point of view memory address (and its
13907− * contents, being identical at this point) will stay the
13908− * same. This mapping will be released by bpf_object__close()
13909− * as per normal clean up procedure, so we don't need to worry
13910− * about it from skeleton's clean up perspective.
13911− */
13912− *mmaped = mmap(map->mmaped, mmap_sz, prot, MAP_SHARED | MAP_FIXED, map_fd, 0);
13913− if (*mmaped == MAP_FAILED) {
13914− err = -errno;
13915− *mmaped = NULL;
13916− pr_warn("failed to re-mmap() map '%s': %d\n",
13917− bpf_map__name(map), err);
13918− return libbpf_err(err);
13919− }
13924+ *map_skel->mmaped = map->mmaped;
1392013925 }
1392113926
1392213927 return 0;
@@ -13944,8 +13949,8 @@ int bpf_object__attach_skeleton(struct bpf_object_skeleton *s) · 21 unmodified lines
1394413949
1394513950 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, link);
1394613951 if (err) {
13947− pr_warn("prog '%s': failed to auto-attach: %d\n",
13948− bpf_program__name(prog), err);
13952+ pr_warn("prog '%s': failed to auto-attach: %s\n",
13953+ bpf_program__name(prog), errstr(err));
1394913954 return libbpf_err(err);
1395013955 }
1395113956
@@ -13988,7 +13993,8 @@ int bpf_object__attach_skeleton(struct bpf_object_skeleton *s) · 36 unmodified lines
1398813993 *link = bpf_map__attach_struct_ops(map);
1398913994 if (!*link) {
1399013995 err = -errno;
13991− pr_warn("map '%s': failed to auto-attach: %d\n", bpf_map__name(map), err);
13996+ pr_warn("map '%s': failed to auto-attach: %s\n",
13997+ bpf_map__name(map), errstr(err));
1399213998 return libbpf_err(err);
1399313999 }
1399414000 }
tools/lib/bpf/libbpf.h+3−1
@@ -577,10 +577,12 @@ struct bpf_uprobe_multi_opts { · 576 unmodified lines
577577 size_t cnt;
578578 /* create return uprobes */
579579 bool retprobe;
580+ /* create session kprobes */
581+ bool session;
580582 size_t :0;
581583 };
582584
583−#define bpf_uprobe_multi_opts__last_field retprobe
585+#define bpf_uprobe_multi_opts__last_field session
584586
585587 /**
586588 * @brief **bpf_program__attach_uprobe_multi()** attaches a BPF program
tools/lib/bpf/libbpf.map+5−0
@@ -421,6 +421,8 @@ LIBBPF_1.5.0 { · 420 unmodified lines
421421 global:
422422 btf__distill_base;
423423 btf__relocate;
424+ btf_ext__endianness;
425+ btf_ext__set_endianness;
424426 bpf_map__autoattach;
425427 bpf_map__set_autoattach;
426428 bpf_object__token_fd;
@@ -428,3 +430,6 @@ LIBBPF_1.5.0 { · 1 unmodified line
428430 ring__consume_n;
429431 ring_buffer__consume_n;
430432 } LIBBPF_1.4.0;
433+
434+LIBBPF_1.6.0 {
435+} LIBBPF_1.5.0;
tools/lib/bpf/libbpf_internal.h+41−2
@@ -10,6 +10,7 @@ · 9 unmodified lines
1010 #define __LIBBPF_LIBBPF_INTERNAL_H
1111
1212 #include <stdlib.h>
13+#include <byteswap.h>
1314 #include <limits.h>
1415 #include <errno.h>
1516 #include <linux/err.h>
@@ -448,11 +449,11 @@ struct btf_ext_info { · 432 unmodified lines
448449 *
449450 * The func_info subsection layout:
450451 * record size for struct bpf_func_info in the func_info subsection
451− * struct btf_sec_func_info for section #1
452+ * struct btf_ext_info_sec for section #1
452453 * a list of bpf_func_info records for section #1
453454 * where struct bpf_func_info mimics one in include/uapi/linux/bpf.h
454455 * but may not be identical
455− * struct btf_sec_func_info for section #2
456+ * struct btf_ext_info_sec for section #2
456457 * a list of bpf_func_info records for section #2
457458 * ......
458459 *
@@ -484,6 +485,8 @@ struct btf_ext { · 25 unmodified lines
484485 struct btf_ext_header *hdr;
485486 void *data;
486487 };
488+ void *data_swapped;
489+ bool swapped_endian;
487490 struct btf_ext_info func_info;
488491 struct btf_ext_info line_info;
489492 struct btf_ext_info core_relo_info;
@@ -511,6 +514,32 @@ struct bpf_line_info_min { · 21 unmodified lines
511514 __u32 line_col;
512515 };
513516
517+/* Functions to byte-swap info records */
518+
519+typedef void (*info_rec_bswap_fn)(void *);
520+
521+static inline void bpf_func_info_bswap(struct bpf_func_info *i)
522+{
523+ i->insn_off = bswap_32(i->insn_off);
524+ i->type_id = bswap_32(i->type_id);
525+}
526+
527+static inline void bpf_line_info_bswap(struct bpf_line_info *i)
528+{
529+ i->insn_off = bswap_32(i->insn_off);
530+ i->file_name_off = bswap_32(i->file_name_off);
531+ i->line_off = bswap_32(i->line_off);
532+ i->line_col = bswap_32(i->line_col);
533+}
534+
535+static inline void bpf_core_relo_bswap(struct bpf_core_relo *i)
536+{
537+ i->insn_off = bswap_32(i->insn_off);
538+ i->type_id = bswap_32(i->type_id);
539+ i->access_str_off = bswap_32(i->access_str_off);
540+ i->kind = bswap_32(i->kind);
541+}
542+
514543 enum btf_field_iter_kind {
515544 BTF_FIELD_ITER_IDS,
516545 BTF_FIELD_ITER_STRS,
@@ -588,6 +617,16 @@ static inline bool is_ldimm64_insn(struct bpf_insn *insn) · 71 unmodified lines
588617 return insn->code == (BPF_LD | BPF_IMM | BPF_DW);
589618 }
590619
620+static inline void bpf_insn_bswap(struct bpf_insn *insn)
621+{
622+ __u8 tmp_reg = insn->dst_reg;
623+
624+ insn->dst_reg = insn->src_reg;
625+ insn->src_reg = tmp_reg;
626+ insn->off = bswap_16(insn->off);
627+ insn->imm = bswap_32(insn->imm);
628+}
629+
591630 /* Unconditionally dup FD, ensuring it doesn't use [0, 2] range.
592631 * Original FD is not closed or altered in any other way.
593632 * Preserves original FD value, if it's invalid (negative).
tools/lib/bpf/libbpf_version.h+1−1
@@ -4,6 +4,6 @@ · 3 unmodified lines
44 #define __LIBBPF_VERSION_H
55
66 #define LIBBPF_MAJOR_VERSION 1
7−#define LIBBPF_MINOR_VERSION 5
7+#define LIBBPF_MINOR_VERSION 6
88
99 #endif /* __LIBBPF_VERSION_H */
tools/lib/bpf/linker.c+80−25
@@ -20,6 +20,7 @@ · 19 unmodified lines
2020 #include "btf.h"
2121 #include "libbpf_internal.h"
2222 #include "strset.h"
23+#include "str_error.h"
2324
2425 #define BTF_EXTERN_SEC ".extern"
2526
@@ -135,6 +136,7 @@ struct bpf_linker { · 109 unmodified lines
135136 int fd;
136137 Elf *elf;
137138 Elf64_Ehdr *elf_hdr;
139+ bool swapped_endian;
138140
139141 /* Output sections metadata */
140142 struct dst_sec *secs;
@@ -305,7 +307,7 @@ static int init_output_elf(struct bpf_linker *linker, const char *file) · 164 unmodified lines
305307 linker->fd = open(file, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644);
306308 if (linker->fd < 0) {
307309 err = -errno;
308− pr_warn("failed to create '%s': %d\n", file, err);
310+ pr_warn("failed to create '%s': %s\n", file, errstr(err));
309311 return err;
310312 }
311313
@@ -324,13 +326,8 @@ static int init_output_elf(struct bpf_linker *linker, const char *file) · 12 unmodified lines
324326
325327 linker->elf_hdr->e_machine = EM_BPF;
326328 linker->elf_hdr->e_type = ET_REL;
327−#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
328− linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2LSB;
329−#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
330− linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2MSB;
331−#else
332−#error "Unknown __BYTE_ORDER__"
333−#endif
329+ /* Set unknown ELF endianness, assign later from input files */
330+ linker->elf_hdr->e_ident[EI_DATA] = ELFDATANONE;
334331
335332 /* STRTAB */
336333 /* initialize strset with an empty string to conform to ELF */
@@ -396,6 +393,8 @@ static int init_output_elf(struct bpf_linker *linker, const char *file) · 59 unmodified lines
396393 pr_warn_elf("failed to create SYMTAB data");
397394 return -EINVAL;
398395 }
396+ /* Ensure libelf translates byte-order of symbol records */
397+ sec->data->d_type = ELF_T_SYM;
399398
400399 str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
401400 if (str_off < 0)
@@ -539,19 +538,21 @@ static int linker_load_obj_file(struct bpf_linker *linker, const char *filename, · 137 unmodified lines
539538 const struct bpf_linker_file_opts *opts,
540539 struct src_obj *obj)
541540 {
542−#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
543− const int host_endianness = ELFDATA2LSB;
544−#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
545− const int host_endianness = ELFDATA2MSB;
546−#else
547−#error "Unknown __BYTE_ORDER__"
548−#endif
549541 int err = 0;
550542 Elf_Scn *scn;
551543 Elf_Data *data;
552544 Elf64_Ehdr *ehdr;
553545 Elf64_Shdr *shdr;
554546 struct src_sec *sec;
547+ unsigned char obj_byteorder;
548+ unsigned char link_byteorder = linker->elf_hdr->e_ident[EI_DATA];
549+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
550+ const unsigned char host_byteorder = ELFDATA2LSB;
551+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
552+ const unsigned char host_byteorder = ELFDATA2MSB;
553+#else
554+#error "Unknown __BYTE_ORDER__"
555+#endif
555556
556557 pr_debug("linker: adding object file '%s'...\n", filename);
557558
@@ -560,7 +561,7 @@ static int linker_load_obj_file(struct bpf_linker *linker, const char *filename, · 2 unmodified lines
560561 obj->fd = open(filename, O_RDONLY | O_CLOEXEC);
561562 if (obj->fd < 0) {
562563 err = -errno;
563− pr_warn("failed to open file '%s': %d\n", filename, err);
564+ pr_warn("failed to open file '%s': %s\n", filename, errstr(err));
564565 return err;
565566 }
566567 obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL);
@@ -577,11 +578,25 @@ static int linker_load_obj_file(struct bpf_linker *linker, const char *filename, · 10 unmodified lines
577578 pr_warn_elf("failed to get ELF header for %s", filename);
578579 return err;
579580 }
580− if (ehdr->e_ident[EI_DATA] != host_endianness) {
581+
582+ /* Linker output endianness set by first input object */
583+ obj_byteorder = ehdr->e_ident[EI_DATA];
584+ if (obj_byteorder != ELFDATA2LSB && obj_byteorder != ELFDATA2MSB) {
581585 err = -EOPNOTSUPP;
582− pr_warn_elf("unsupported byte order of ELF file %s", filename);
586+ pr_warn("unknown byte order of ELF file %s\n", filename);
583587 return err;
584588 }
589+ if (link_byteorder == ELFDATANONE) {
590+ linker->elf_hdr->e_ident[EI_DATA] = obj_byteorder;
591+ linker->swapped_endian = obj_byteorder != host_byteorder;
592+ pr_debug("linker: set %s-endian output byte order\n",
593+ obj_byteorder == ELFDATA2MSB ? "big" : "little");
594+ } else if (link_byteorder != obj_byteorder) {
595+ err = -EOPNOTSUPP;
596+ pr_warn("byte order mismatch with ELF file %s\n", filename);
597+ return err;
598+ }
599+
585600 if (ehdr->e_type != ET_REL
586601 || ehdr->e_machine != EM_BPF
587602 || ehdr->e_ident[EI_CLASS] != ELFCLASS64) {
@@ -656,7 +671,8 @@ static int linker_load_obj_file(struct bpf_linker *linker, const char *filename, · 68 unmodified lines
656671 obj->btf = btf__new(data->d_buf, shdr->sh_size);
657672 err = libbpf_get_error(obj->btf);
658673 if (err) {
659− pr_warn("failed to parse .BTF from %s: %d\n", filename, err);
674+ pr_warn("failed to parse .BTF from %s: %s\n",
675+ filename, errstr(err));
660676 return err;
661677 }
662678 sec->skipped = true;
@@ -666,7 +682,8 @@ static int linker_load_obj_file(struct bpf_linker *linker, const char *filename, · 3 unmodified lines
666682 obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size);
667683 err = libbpf_get_error(obj->btf_ext);
668684 if (err) {
669− pr_warn("failed to parse .BTF.ext from '%s': %d\n", filename, err);
685+ pr_warn("failed to parse .BTF.ext from '%s': %s\n",
686+ filename, errstr(err));
670687 return err;
671688 }
672689 sec->skipped = true;
@@ -1109,6 +1126,24 @@ static bool sec_content_is_same(struct dst_sec *dst_sec, struct src_sec *src_sec · 436 unmodified lines
11091126 return true;
11101127 }
11111128
1129+static bool is_exec_sec(struct dst_sec *sec)
1130+{
1131+ if (!sec || sec->ephemeral)
1132+ return false;
1133+ return (sec->shdr->sh_type == SHT_PROGBITS) &&
1134+ (sec->shdr->sh_flags & SHF_EXECINSTR);
1135+}
1136+
1137+static void exec_sec_bswap(void *raw_data, int size)
1138+{
1139+ const int insn_cnt = size / sizeof(struct bpf_insn);
1140+ struct bpf_insn *insn = raw_data;
1141+ int i;
1142+
1143+ for (i = 0; i < insn_cnt; i++, insn++)
1144+ bpf_insn_bswap(insn);
1145+}
1146+
11121147 static int extend_sec(struct bpf_linker *linker, struct dst_sec *dst, struct src_sec *src)
11131148 {
11141149 void *tmp;
@@ -1168,6 +1203,10 @@ static int extend_sec(struct bpf_linker *linker, struct dst_sec *dst, struct src · 53 unmodified lines
11681203 memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz);
11691204 /* now copy src data at a properly aligned offset */
11701205 memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size);
1206+
1207+ /* convert added bpf insns to native byte-order */
1208+ if (linker->swapped_endian && is_exec_sec(dst))
1209+ exec_sec_bswap(dst->raw_data + dst_align_sz, src->shdr->sh_size);
11711210 }
11721211
11731212 dst->sec_sz = dst_final_sz;
@@ -2415,6 +2454,10 @@ static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj) · 1241 unmodified lines
24152454 if (glob_sym && glob_sym->var_idx >= 0) {
24162455 __s64 sz;
24172456
2457+ /* FUNCs don't have size, nothing to update */
2458+ if (btf_is_func(t))
2459+ continue;
2460+
24182461 dst_var = &dst_sec->sec_vars[glob_sym->var_idx];
24192462 /* Because underlying BTF type might have
24202463 * changed, so might its size have changed, so
@@ -2628,6 +2671,10 @@ int bpf_linker__finalize(struct bpf_linker *linker) · 207 unmodified lines
26282671 if (!sec->scn)
26292672 continue;
26302673
2674+ /* restore sections with bpf insns to target byte-order */
2675+ if (linker->swapped_endian && is_exec_sec(sec))
2676+ exec_sec_bswap(sec->raw_data, sec->sec_sz);
2677+
26312678 sec->data->d_buf = sec->raw_data;
26322679 }
26332680
@@ -2696,6 +2743,7 @@ static int emit_elf_data_sec(struct bpf_linker *linker, const char *sec_name, · 62 unmodified lines
26962743
26972744 static int finalize_btf(struct bpf_linker *linker)
26982745 {
2746+ enum btf_endianness link_endianness;
26992747 LIBBPF_OPTS(btf_dedup_opts, opts);
27002748 struct btf *btf = linker->btf;
27012749 const void *raw_data;
@@ -2729,17 +2777,24 @@ static int finalize_btf(struct bpf_linker *linker) · 27 unmodified lines
27292777
27302778 err = finalize_btf_ext(linker);
27312779 if (err) {
2732− pr_warn(".BTF.ext generation failed: %d\n", err);
2780+ pr_warn(".BTF.ext generation failed: %s\n", errstr(err));
27332781 return err;
27342782 }
27352783
27362784 opts.btf_ext = linker->btf_ext;
27372785 err = btf__dedup(linker->btf, &opts);
27382786 if (err) {
2739− pr_warn("BTF dedup failed: %d\n", err);
2787+ pr_warn("BTF dedup failed: %s\n", errstr(err));
27402788 return err;
27412789 }
27422790
2791+ /* Set .BTF and .BTF.ext output byte order */
2792+ link_endianness = linker->elf_hdr->e_ident[EI_DATA] == ELFDATA2MSB ?
2793+ BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN;
2794+ btf__set_endianness(linker->btf, link_endianness);
2795+ if (linker->btf_ext)
2796+ btf_ext__set_endianness(linker->btf_ext, link_endianness);
2797+
27432798 /* Emit .BTF section */
27442799 raw_data = btf__raw_data(linker->btf, &raw_sz);
27452800 if (!raw_data)
@@ -2747,7 +2802,7 @@ static int finalize_btf(struct bpf_linker *linker) · 1 unmodified line
27472802
27482803 err = emit_elf_data_sec(linker, BTF_ELF_SEC, 8, raw_data, raw_sz);
27492804 if (err) {
2750− pr_warn("failed to write out .BTF ELF section: %d\n", err);
2805+ pr_warn("failed to write out .BTF ELF section: %s\n", errstr(err));
27512806 return err;
27522807 }
27532808
@@ -2759,7 +2814,7 @@ static int finalize_btf(struct bpf_linker *linker) · 5 unmodified lines
27592814
27602815 err = emit_elf_data_sec(linker, BTF_EXT_ELF_SEC, 8, raw_data, raw_sz);
27612816 if (err) {
2762− pr_warn("failed to write out .BTF.ext ELF section: %d\n", err);
2817+ pr_warn("failed to write out .BTF.ext ELF section: %s\n", errstr(err));
27632818 return err;
27642819 }
27652820 }
@@ -2935,7 +2990,7 @@ static int finalize_btf_ext(struct bpf_linker *linker) · 169 unmodified lines
29352990 err = libbpf_get_error(linker->btf_ext);
29362991 if (err) {
29372992 linker->btf_ext = NULL;
2938− pr_warn("failed to parse final .BTF.ext data: %d\n", err);
2993+ pr_warn("failed to parse final .BTF.ext data: %s\n", errstr(err));
29392994 goto out;
29402995 }
29412996
tools/lib/bpf/relo_core.c+1−1
@@ -1339,7 +1339,7 @@ int bpf_core_calc_relo_insn(const char *prog_name, · 1338 unmodified lines
13391339 cands->cands[i].id, cand_spec);
13401340 if (err < 0) {
13411341 bpf_core_format_spec(spec_buf, sizeof(spec_buf), cand_spec);
1342− pr_warn("prog '%s': relo #%d: error matching candidate #%d %s: %d\n ",
1342+ pr_warn("prog '%s': relo #%d: error matching candidate #%d %s: %d\n",
13431343 prog_name, relo_idx, i, spec_buf, err);
13441344 return err;
13451345 }
tools/lib/bpf/ringbuf.c+18−16
@@ -21,6 +21,7 @@ · 20 unmodified lines
2121 #include "libbpf.h"
2222 #include "libbpf_internal.h"
2323 #include "bpf.h"
24+#include "str_error.h"
2425
2526 struct ring {
2627 ring_buffer_sample_fn sample_cb;
@@ -88,8 +89,8 @@ int ring_buffer__add(struct ring_buffer *rb, int map_fd, · 61 unmodified lines
8889 err = bpf_map_get_info_by_fd(map_fd, &info, &len);
8990 if (err) {
9091 err = -errno;
91− pr_warn("ringbuf: failed to get map info for fd=%d: %d\n",
92− map_fd, err);
92+ pr_warn("ringbuf: failed to get map info for fd=%d: %s\n",
93+ map_fd, errstr(err));
9394 return libbpf_err(err);
9495 }
9596
@@ -123,8 +124,8 @@ int ring_buffer__add(struct ring_buffer *rb, int map_fd, · 27 unmodified lines
123124 tmp = mmap(NULL, rb->page_size, PROT_READ | PROT_WRITE, MAP_SHARED, map_fd, 0);
124125 if (tmp == MAP_FAILED) {
125126 err = -errno;
126− pr_warn("ringbuf: failed to mmap consumer page for map fd=%d: %d\n",
127− map_fd, err);
127+ pr_warn("ringbuf: failed to mmap consumer page for map fd=%d: %s\n",
128+ map_fd, errstr(err));
128129 goto err_out;
129130 }
130131 r->consumer_pos = tmp;
@@ -142,8 +143,8 @@ int ring_buffer__add(struct ring_buffer *rb, int map_fd, · 11 unmodified lines
142143 tmp = mmap(NULL, (size_t)mmap_sz, PROT_READ, MAP_SHARED, map_fd, rb->page_size);
143144 if (tmp == MAP_FAILED) {
144145 err = -errno;
145− pr_warn("ringbuf: failed to mmap data pages for map fd=%d: %d\n",
146− map_fd, err);
146+ pr_warn("ringbuf: failed to mmap data pages for map fd=%d: %s\n",
147+ map_fd, errstr(err));
147148 goto err_out;
148149 }
149150 r->producer_pos = tmp;
@@ -156,8 +157,8 @@ int ring_buffer__add(struct ring_buffer *rb, int map_fd, · 6 unmodified lines
156157 e->data.fd = rb->ring_cnt;
157158 if (epoll_ctl(rb->epoll_fd, EPOLL_CTL_ADD, map_fd, e) < 0) {
158159 err = -errno;
159− pr_warn("ringbuf: failed to epoll add map fd=%d: %d\n",
160− map_fd, err);
160+ pr_warn("ringbuf: failed to epoll add map fd=%d: %s\n",
161+ map_fd, errstr(err));
161162 goto err_out;
162163 }
163164
@@ -205,7 +206,7 @@ ring_buffer__new(int map_fd, ring_buffer_sample_fn sample_cb, void *ctx, · 41 unmodified lines
205206 rb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
206207 if (rb->epoll_fd < 0) {
207208 err = -errno;
208− pr_warn("ringbuf: failed to create epoll instance: %d\n", err);
209+ pr_warn("ringbuf: failed to create epoll instance: %s\n", errstr(err));
209210 goto err_out;
210211 }
211212
@@ -458,7 +459,8 @@ static int user_ringbuf_map(struct user_ring_buffer *rb, int map_fd) · 246 unmodified lines
458459 err = bpf_map_get_info_by_fd(map_fd, &info, &len);
459460 if (err) {
460461 err = -errno;
461− pr_warn("user ringbuf: failed to get map info for fd=%d: %d\n", map_fd, err);
462+ pr_warn("user ringbuf: failed to get map info for fd=%d: %s\n",
463+ map_fd, errstr(err));
462464 return err;
463465 }
464466
@@ -474,8 +476,8 @@ static int user_ringbuf_map(struct user_ring_buffer *rb, int map_fd) · 9 unmodified lines
474476 tmp = mmap(NULL, rb->page_size, PROT_READ, MAP_SHARED, map_fd, 0);
475477 if (tmp == MAP_FAILED) {
476478 err = -errno;
477− pr_warn("user ringbuf: failed to mmap consumer page for map fd=%d: %d\n",
478− map_fd, err);
479+ pr_warn("user ringbuf: failed to mmap consumer page for map fd=%d: %s\n",
480+ map_fd, errstr(err));
479481 return err;
480482 }
481483 rb->consumer_pos = tmp;
@@ -494,8 +496,8 @@ static int user_ringbuf_map(struct user_ring_buffer *rb, int map_fd) · 12 unmodified lines
494496 map_fd, rb->page_size);
495497 if (tmp == MAP_FAILED) {
496498 err = -errno;
497− pr_warn("user ringbuf: failed to mmap data pages for map fd=%d: %d\n",
498− map_fd, err);
499+ pr_warn("user ringbuf: failed to mmap data pages for map fd=%d: %s\n",
500+ map_fd, errstr(err));
499501 return err;
500502 }
501503
@@ -506,7 +508,7 @@ static int user_ringbuf_map(struct user_ring_buffer *rb, int map_fd) · 4 unmodified lines
506508 rb_epoll->events = EPOLLOUT;
507509 if (epoll_ctl(rb->epoll_fd, EPOLL_CTL_ADD, map_fd, rb_epoll) < 0) {
508510 err = -errno;
509− pr_warn("user ringbuf: failed to epoll add map fd=%d: %d\n", map_fd, err);
511+ pr_warn("user ringbuf: failed to epoll add map fd=%d: %s\n", map_fd, errstr(err));
510512 return err;
511513 }
512514
@@ -531,7 +533,7 @@ user_ring_buffer__new(int map_fd, const struct user_ring_buffer_opts *opts) · 18 unmodified lines
531533 rb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
532534 if (rb->epoll_fd < 0) {
533535 err = -errno;
534− pr_warn("user ringbuf: failed to create epoll instance: %d\n", err);
536+ pr_warn("user ringbuf: failed to create epoll instance: %s\n", errstr(err));
535537 goto err_out;
536538 }
537539
tools/lib/bpf/skel_internal.h+2−1
@@ -351,10 +351,11 @@ static inline int bpf_load_and_run(struct bpf_load_and_run_opts *opts) · 350 unmodified lines
351351 attr.test.ctx_size_in = opts->ctx->sz;
352352 err = skel_sys_bpf(BPF_PROG_RUN, &attr, test_run_attr_sz);
353353 if (err < 0 || (int)attr.test.retval < 0) {
354− opts->errstr = "failed to execute loader prog";
355354 if (err < 0) {
355+ opts->errstr = "failed to execute loader prog";
356356 set_err;
357357 } else {
358+ opts->errstr = "error returned by loader prog";
358359 err = (int)attr.test.retval;
359360 #ifndef __KERNEL__
360361 errno = -err;
tools/lib/bpf/str_error.c+71−0
@@ -5,6 +5,10 @@ · 4 unmodified lines
55 #include <errno.h>
66 #include "str_error.h"
77
8+#ifndef ENOTSUPP
9+#define ENOTSUPP 524
10+#endif
11+
812 /* make sure libbpf doesn't use kernel-only integer typedefs */
913 #pragma GCC poison u8 u16 u32 u64 s8 s16 s32 s64
1014
@@ -31,3 +35,70 @@ char *libbpf_strerror_r(int err, char *dst, int len) · 20 unmodified lines
3135 }
3236 return dst;
3337 }
38+
39+const char *errstr(int err)
40+{
41+ static __thread char buf[12];
42+
43+ if (err > 0)
44+ err = -err;
45+
46+ switch (err) {
47+ case -E2BIG: return "-E2BIG";
48+ case -EACCES: return "-EACCES";
49+ case -EADDRINUSE: return "-EADDRINUSE";
50+ case -EADDRNOTAVAIL: return "-EADDRNOTAVAIL";
51+ case -EAGAIN: return "-EAGAIN";
52+ case -EALREADY: return "-EALREADY";
53+ case -EBADF: return "-EBADF";
54+ case -EBADFD: return "-EBADFD";
55+ case -EBUSY: return "-EBUSY";
56+ case -ECANCELED: return "-ECANCELED";
57+ case -ECHILD: return "-ECHILD";
58+ case -EDEADLK: return "-EDEADLK";
59+ case -EDOM: return "-EDOM";
60+ case -EEXIST: return "-EEXIST";
61+ case -EFAULT: return "-EFAULT";
62+ case -EFBIG: return "-EFBIG";
63+ case -EILSEQ: return "-EILSEQ";
64+ case -EINPROGRESS: return "-EINPROGRESS";
65+ case -EINTR: return "-EINTR";
66+ case -EINVAL: return "-EINVAL";
67+ case -EIO: return "-EIO";
68+ case -EISDIR: return "-EISDIR";
69+ case -ELOOP: return "-ELOOP";
70+ case -EMFILE: return "-EMFILE";
71+ case -EMLINK: return "-EMLINK";
72+ case -EMSGSIZE: return "-EMSGSIZE";
73+ case -ENAMETOOLONG: return "-ENAMETOOLONG";
74+ case -ENFILE: return "-ENFILE";
75+ case -ENODATA: return "-ENODATA";
76+ case -ENODEV: return "-ENODEV";
77+ case -ENOENT: return "-ENOENT";
78+ case -ENOEXEC: return "-ENOEXEC";
79+ case -ENOLINK: return "-ENOLINK";
80+ case -ENOMEM: return "-ENOMEM";
81+ case -ENOSPC: return "-ENOSPC";
82+ case -ENOTBLK: return "-ENOTBLK";
83+ case -ENOTDIR: return "-ENOTDIR";
84+ case -ENOTSUPP: return "-ENOTSUPP";
85+ case -ENOTTY: return "-ENOTTY";
86+ case -ENXIO: return "-ENXIO";
87+ case -EOPNOTSUPP: return "-EOPNOTSUPP";
88+ case -EOVERFLOW: return "-EOVERFLOW";
89+ case -EPERM: return "-EPERM";
90+ case -EPIPE: return "-EPIPE";
91+ case -EPROTO: return "-EPROTO";
92+ case -EPROTONOSUPPORT: return "-EPROTONOSUPPORT";
93+ case -ERANGE: return "-ERANGE";
94+ case -EROFS: return "-EROFS";
95+ case -ESPIPE: return "-ESPIPE";
96+ case -ESRCH: return "-ESRCH";
97+ case -ETXTBSY: return "-ETXTBSY";
98+ case -EUCLEAN: return "-EUCLEAN";
99+ case -EXDEV: return "-EXDEV";
100+ default:
101+ snprintf(buf, sizeof(buf), "%d", err);
102+ return buf;
103+ }
104+}
tools/lib/bpf/str_error.h+7−0
@@ -6,4 +6,11 @@ · 5 unmodified lines
66
77 char *libbpf_strerror_r(int err, char *dst, int len);
88
9+/**
10+ * @brief **errstr()** returns string corresponding to numeric errno
11+ * @param err negative numeric errno
12+ * @return pointer to string representation of the errno, that is invalidated
13+ * upon the next call.
14+ */
15+const char *errstr(int err);
916 #endif /* __LIBBPF_STR_ERROR_H */
tools/lib/bpf/usdt.c+17−15
@@ -20,6 +20,7 @@ · 19 unmodified lines
2020 #include "libbpf_common.h"
2121 #include "libbpf_internal.h"
2222 #include "hashmap.h"
23+#include "str_error.h"
2324
2425 /* libbpf's USDT support consists of BPF-side state/code and user-space
2526 * state/code working together in concert. BPF-side parts are defined in
@@ -465,8 +466,8 @@ static int parse_vma_segs(int pid, const char *lib_path, struct elf_seg **segs, · 439 unmodified lines
465466 goto proceed;
466467
467468 if (!realpath(lib_path, path)) {
468− pr_warn("usdt: failed to get absolute path of '%s' (err %d), using path as is...\n",
469− lib_path, -errno);
469+ pr_warn("usdt: failed to get absolute path of '%s' (err %s), using path as is...\n",
470+ lib_path, errstr(-errno));
470471 libbpf_strlcpy(path, lib_path, sizeof(path));
471472 }
472473
@@ -475,8 +476,8 @@ proceed: · 2 unmodified lines
475476 f = fopen(line, "re");
476477 if (!f) {
477478 err = -errno;
478− pr_warn("usdt: failed to open '%s' to get base addr of '%s': %d\n",
479− line, lib_path, err);
479+ pr_warn("usdt: failed to open '%s' to get base addr of '%s': %s\n",
480+ line, lib_path, errstr(err));
480481 return err;
481482 }
482483
@@ -606,7 +607,8 @@ static int collect_usdt_targets(struct usdt_manager *man, Elf *elf, const char * · 123 unmodified lines
606607
607608 err = parse_elf_segs(elf, path, &segs, &seg_cnt);
608609 if (err) {
609− pr_warn("usdt: failed to process ELF program segments for '%s': %d\n", path, err);
610+ pr_warn("usdt: failed to process ELF program segments for '%s': %s\n",
611+ path, errstr(err));
610612 goto err_out;
611613 }
612614
@@ -708,8 +710,8 @@ static int collect_usdt_targets(struct usdt_manager *man, Elf *elf, const char * · 95 unmodified lines
708710 if (vma_seg_cnt == 0) {
709711 err = parse_vma_segs(pid, path, &vma_segs, &vma_seg_cnt);
710712 if (err) {
711− pr_warn("usdt: failed to get memory segments in PID %d for shared library '%s': %d\n",
712− pid, path, err);
713+ pr_warn("usdt: failed to get memory segments in PID %d for shared library '%s': %s\n",
714+ pid, path, errstr(err));
713715 goto err_out;
714716 }
715717 }
@@ -1047,8 +1049,8 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct · 331 unmodified lines
10471049
10481050 if (is_new && bpf_map_update_elem(spec_map_fd, &spec_id, &target->spec, BPF_ANY)) {
10491051 err = -errno;
1050− pr_warn("usdt: failed to set USDT spec #%d for '%s:%s' in '%s': %d\n",
1051− spec_id, usdt_provider, usdt_name, path, err);
1052+ pr_warn("usdt: failed to set USDT spec #%d for '%s:%s' in '%s': %s\n",
1053+ spec_id, usdt_provider, usdt_name, path, errstr(err));
10521054 goto err_out;
10531055 }
10541056 if (!man->has_bpf_cookie &&
@@ -1058,9 +1060,9 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct · 3 unmodified lines
10581060 pr_warn("usdt: IP collision detected for spec #%d for '%s:%s' in '%s'\n",
10591061 spec_id, usdt_provider, usdt_name, path);
10601062 } else {
1061− pr_warn("usdt: failed to map IP 0x%lx to spec #%d for '%s:%s' in '%s': %d\n",
1063+ pr_warn("usdt: failed to map IP 0x%lx to spec #%d for '%s:%s' in '%s': %s\n",
10621064 target->abs_ip, spec_id, usdt_provider, usdt_name,
1063− path, err);
1065+ path, errstr(err));
10641066 }
10651067 goto err_out;
10661068 }
@@ -1076,8 +1078,8 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct · 9 unmodified lines
10761078 target->rel_ip, &opts);
10771079 err = libbpf_get_error(uprobe_link);
10781080 if (err) {
1079− pr_warn("usdt: failed to attach uprobe #%d for '%s:%s' in '%s': %d\n",
1080− i, usdt_provider, usdt_name, path, err);
1081+ pr_warn("usdt: failed to attach uprobe #%d for '%s:%s' in '%s': %s\n",
1082+ i, usdt_provider, usdt_name, path, errstr(err));
10811083 goto err_out;
10821084 }
10831085
@@ -1099,8 +1101,8 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct · 15 unmodified lines
10991101 NULL, &opts_multi);
11001102 if (!link->multi_link) {
11011103 err = -errno;
1102− pr_warn("usdt: failed to attach uprobe multi for '%s:%s' in '%s': %d\n",
1103− usdt_provider, usdt_name, path, err);
1104+ pr_warn("usdt: failed to attach uprobe multi for '%s:%s' in '%s': %s\n",
1105+ usdt_provider, usdt_name, path, errstr(err));
11041106 goto err_out;
11051107 }
11061108
tools/lib/bpf/zip.c+1−1
@@ -223,7 +223,7 @@ struct zip_archive *zip_archive_open(const char *path) · 222 unmodified lines
223223 if (!archive) {
224224 munmap(data, size);
225225 return ERR_PTR(-ENOMEM);
226− };
226+ }
227227
228228 archive->data = data;
229229 archive->size = size;
tools/lib/subcmd/parse-options.c+1−1
@@ -807,7 +807,7 @@ static int option__cmp(const void *va, const void *vb) · 806 unmodified lines
807807 static struct option *options__order(const struct option *opts)
808808 {
809809 int nr_opts = 0, nr_group = 0, nr_parent = 0, len;
810− const struct option *o, *p = opts;
810+ const struct option *o = NULL, *p = opts;
811811 struct option *opt, *ordered = NULL, *group;
812812
813813 /* flatten the options that have parents */
tools/testing/selftests/bpf/.gitignore+0−1
@@ -16,7 +16,6 @@ fixdep · 15 unmodified lines
1616 /test_progs-cpuv4
1717 test_verifier_log
1818 feature
19−test_sock
2019 urandom_read
2120 test_sockmap
2221 test_lirc_mode2_user
tools/testing/selftests/bpf/Makefile+38−22
@@ -10,6 +10,7 @@ TOOLSDIR := $(abspath ../../..) · 9 unmodified lines
1010 LIBDIR := $(TOOLSDIR)/lib
1111 BPFDIR := $(LIBDIR)/bpf
1212 TOOLSINCDIR := $(TOOLSDIR)/include
13+TOOLSARCHINCDIR := $(TOOLSDIR)/arch/$(SRCARCH)/include
1314 BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool
1415 APIDIR := $(TOOLSINCDIR)/uapi
1516 ifneq ($(O),)
@@ -44,7 +45,7 @@ CFLAGS += -g $(OPT_FLAGS) -rdynamic \ · 28 unmodified lines
4445 -Wall -Werror -fno-omit-frame-pointer \
4546 $(GENFLAGS) $(SAN_CFLAGS) $(LIBELF_CFLAGS) \
4647 -I$(CURDIR) -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \
47− -I$(TOOLSINCDIR) -I$(APIDIR) -I$(OUTPUT)
48+ -I$(TOOLSINCDIR) -I$(TOOLSARCHINCDIR) -I$(APIDIR) -I$(OUTPUT)
4849 LDFLAGS += $(SAN_LDFLAGS)
4950 LDLIBS += $(LIBELF_LIBS) -lz -lrt -lpthread
5051
@@ -83,7 +84,7 @@ endif · 32 unmodified lines
8384
8485 # Order correspond to 'make run_tests' order
8586 TEST_GEN_PROGS = test_verifier test_tag test_maps test_lru_map test_lpm_map test_progs \
86− test_sock test_sockmap \
87+ test_sockmap \
8788 test_tcpnotify_user test_sysctl \
8889 test_progs-no_alu32
8990 TEST_INST_SUBDIRS := no_alu32
@@ -132,7 +133,6 @@ TEST_PROGS := test_kmod.sh \ · 42 unmodified lines
132133 test_tunnel.sh \
133134 test_lwt_seg6local.sh \
134135 test_lirc_mode2.sh \
135− test_skb_cgroup_id.sh \
136136 test_flow_dissector.sh \
137137 test_xdp_vlan_mode_generic.sh \
138138 test_xdp_vlan_mode_native.sh \
@@ -274,6 +274,7 @@ $(OUTPUT)/liburandom_read.so: urandom_read_lib1.c urandom_read_lib2.c liburandom · 135 unmodified lines
274274 $(Q)$(CLANG) $(CLANG_TARGET_ARCH) \
275275 $(filter-out -static,$(CFLAGS) $(LDFLAGS)) \
276276 $(filter %.c,$^) $(filter-out -static,$(LDLIBS)) \
277+ -Wno-unused-command-line-argument \
277278 -fuse-ld=$(LLD) -Wl,-znoseparate-code -Wl,--build-id=sha1 \
278279 -Wl,--version-script=liburandom_read.map \
279280 -fPIC -shared -o $@
@@ -282,6 +283,7 @@ $(OUTPUT)/urandom_read: urandom_read.c urandom_read_aux.c $(OUTPUT)/liburandom_r · 2 unmodified lines
282283 $(call msg,BINARY,,$@)
283284 $(Q)$(CLANG) $(CLANG_TARGET_ARCH) \
284285 $(filter-out -static,$(CFLAGS) $(LDFLAGS)) $(filter %.c,$^) \
286+ -Wno-unused-command-line-argument \
285287 -lurandom_read $(filter-out -static,$(LDLIBS)) -L$(OUTPUT) \
286288 -fuse-ld=$(LLD) -Wl,-znoseparate-code -Wl,--build-id=sha1 \
287289 -Wl,-rpath=. -o $@
@@ -295,25 +297,33 @@ $(OUTPUT)/sign-file: ../../../../scripts/sign-file.c · 7 unmodified lines
295297 $(OUTPUT)/bpf_testmod.ko: $(VMLINUX_BTF) $(RESOLVE_BTFIDS) $(wildcard bpf_testmod/Makefile bpf_testmod/*.[ch])
296298 $(call msg,MOD,,$@)
297299 $(Q)$(RM) bpf_testmod/bpf_testmod.ko # force re-compilation
298− $(Q)$(MAKE) $(submake_extras) RESOLVE_BTFIDS=$(RESOLVE_BTFIDS) -C bpf_testmod
300+ $(Q)$(MAKE) $(submake_extras) -C bpf_testmod \
301+ RESOLVE_BTFIDS=$(RESOLVE_BTFIDS) \
302+ EXTRA_CFLAGS='' EXTRA_LDFLAGS=''
299303 $(Q)cp bpf_testmod/bpf_testmod.ko $@
300304
301305 $(OUTPUT)/bpf_test_no_cfi.ko: $(VMLINUX_BTF) $(RESOLVE_BTFIDS) $(wildcard bpf_test_no_cfi/Makefile bpf_test_no_cfi/*.[ch])
302306 $(call msg,MOD,,$@)
303307 $(Q)$(RM) bpf_test_no_cfi/bpf_test_no_cfi.ko # force re-compilation
304− $(Q)$(MAKE) $(submake_extras) RESOLVE_BTFIDS=$(RESOLVE_BTFIDS) -C bpf_test_no_cfi
308+ $(Q)$(MAKE) $(submake_extras) -C bpf_test_no_cfi \
309+ RESOLVE_BTFIDS=$(RESOLVE_BTFIDS) \
310+ EXTRA_CFLAGS='' EXTRA_LDFLAGS=''
305311 $(Q)cp bpf_test_no_cfi/bpf_test_no_cfi.ko $@
306312
307313 $(OUTPUT)/bpf_test_modorder_x.ko: $(VMLINUX_BTF) $(RESOLVE_BTFIDS) $(wildcard bpf_test_modorder_x/Makefile bpf_test_modorder_x/*.[ch])
308314 $(call msg,MOD,,$@)
309315 $(Q)$(RM) bpf_test_modorder_x/bpf_test_modorder_x.ko # force re-compilation
310− $(Q)$(MAKE) $(submake_extras) RESOLVE_BTFIDS=$(RESOLVE_BTFIDS) -C bpf_test_modorder_x
316+ $(Q)$(MAKE) $(submake_extras) -C bpf_test_modorder_x \
317+ RESOLVE_BTFIDS=$(RESOLVE_BTFIDS) \
318+ EXTRA_CFLAGS='' EXTRA_LDFLAGS=''
311319 $(Q)cp bpf_test_modorder_x/bpf_test_modorder_x.ko $@
312320
313321 $(OUTPUT)/bpf_test_modorder_y.ko: $(VMLINUX_BTF) $(RESOLVE_BTFIDS) $(wildcard bpf_test_modorder_y/Makefile bpf_test_modorder_y/*.[ch])
314322 $(call msg,MOD,,$@)
315323 $(Q)$(RM) bpf_test_modorder_y/bpf_test_modorder_y.ko # force re-compilation
316− $(Q)$(MAKE) $(submake_extras) RESOLVE_BTFIDS=$(RESOLVE_BTFIDS) -C bpf_test_modorder_y
324+ $(Q)$(MAKE) $(submake_extras) -C bpf_test_modorder_y \
325+ RESOLVE_BTFIDS=$(RESOLVE_BTFIDS) \
326+ EXTRA_CFLAGS='' EXTRA_LDFLAGS=''
317327 $(Q)cp bpf_test_modorder_y/bpf_test_modorder_y.ko $@
318328
319329
@@ -333,8 +343,8 @@ $(OUTPUT)/runqslower: $(BPFOBJ) | $(DEFAULT_BPFTOOL) $(RUNQSLOWER_OUTPUT) · 13 unmodified lines
333343 BPFTOOL_OUTPUT=$(HOST_BUILD_DIR)/bpftool/ \
334344 BPFOBJ_OUTPUT=$(BUILD_DIR)/libbpf/ \
335345 BPFOBJ=$(BPFOBJ) BPF_INCLUDE=$(INCLUDE_DIR) \
336− EXTRA_CFLAGS='-g $(OPT_FLAGS) $(SAN_CFLAGS)' \
337− EXTRA_LDFLAGS='$(SAN_LDFLAGS)' && \
346+ EXTRA_CFLAGS='-g $(OPT_FLAGS) $(SAN_CFLAGS) $(EXTRA_CFLAGS)' \
347+ EXTRA_LDFLAGS='$(SAN_LDFLAGS) $(EXTRA_LDFLAGS)' && \
338348 cp $(RUNQSLOWER_OUTPUT)runqslower $@
339349
340350 TEST_GEN_PROGS_EXTENDED += $(TRUNNER_BPFTOOL)
@@ -349,7 +359,6 @@ JSON_WRITER := $(OUTPUT)/json_writer.o · 8 unmodified lines
349359 CAP_HELPERS := $(OUTPUT)/cap_helpers.o
350360 NETWORK_HELPERS := $(OUTPUT)/network_helpers.o
351361
352−$(OUTPUT)/test_sock: $(CGROUP_HELPERS) $(TESTING_HELPERS)
353362 $(OUTPUT)/test_sockmap: $(CGROUP_HELPERS) $(TESTING_HELPERS)
354363 $(OUTPUT)/test_tcpnotify_user: $(CGROUP_HELPERS) $(TESTING_HELPERS) $(TRACE_HELPERS)
355364 $(OUTPUT)/test_sock_fields: $(CGROUP_HELPERS) $(TESTING_HELPERS)
@@ -368,7 +377,8 @@ $(DEFAULT_BPFTOOL): $(wildcard $(BPFTOOLDIR)/*.[ch] $(BPFTOOLDIR)/Makefile) \ · 12 unmodified lines
368377 $(HOST_BPFOBJ) | $(HOST_BUILD_DIR)/bpftool
369378 $(Q)$(MAKE) $(submake_extras) -C $(BPFTOOLDIR) \
370379 ARCH= CROSS_COMPILE= CC="$(HOSTCC)" LD="$(HOSTLD)" \
371− EXTRA_CFLAGS='-g $(OPT_FLAGS)' \
380+ EXTRA_CFLAGS='-g $(OPT_FLAGS) $(EXTRA_CFLAGS)' \
381+ EXTRA_LDFLAGS='$(EXTRA_LDFLAGS)' \
372382 OUTPUT=$(HOST_BUILD_DIR)/bpftool/ \
373383 LIBBPF_OUTPUT=$(HOST_BUILD_DIR)/libbpf/ \
374384 LIBBPF_DESTDIR=$(HOST_SCRATCH_DIR)/ \
@@ -379,7 +389,8 @@ $(CROSS_BPFTOOL): $(wildcard $(BPFTOOLDIR)/*.[ch] $(BPFTOOLDIR)/Makefile) \ · 4 unmodified lines
379389 $(BPFOBJ) | $(BUILD_DIR)/bpftool
380390 $(Q)$(MAKE) $(submake_extras) -C $(BPFTOOLDIR) \
381391 ARCH=$(ARCH) CROSS_COMPILE=$(CROSS_COMPILE) \
382− EXTRA_CFLAGS='-g $(OPT_FLAGS)' \
392+ EXTRA_CFLAGS='-g $(OPT_FLAGS) $(EXTRA_CFLAGS)' \
393+ EXTRA_LDFLAGS='$(EXTRA_LDFLAGS)' \
383394 OUTPUT=$(BUILD_DIR)/bpftool/ \
384395 LIBBPF_OUTPUT=$(BUILD_DIR)/libbpf/ \
385396 LIBBPF_DESTDIR=$(SCRATCH_DIR)/ \
@@ -402,8 +413,8 @@ $(BPFOBJ): $(wildcard $(BPFDIR)/*.[ch] $(BPFDIR)/Makefile) \ · 16 unmodified lines
402413 $(APIDIR)/linux/bpf.h \
403414 | $(BUILD_DIR)/libbpf
404415 $(Q)$(MAKE) $(submake_extras) -C $(BPFDIR) OUTPUT=$(BUILD_DIR)/libbpf/ \
405− EXTRA_CFLAGS='-g $(OPT_FLAGS) $(SAN_CFLAGS)' \
406− EXTRA_LDFLAGS='$(SAN_LDFLAGS)' \
416+ EXTRA_CFLAGS='-g $(OPT_FLAGS) $(SAN_CFLAGS) $(EXTRA_CFLAGS)' \
417+ EXTRA_LDFLAGS='$(SAN_LDFLAGS) $(EXTRA_LDFLAGS)' \
407418 DESTDIR=$(SCRATCH_DIR) prefix= all install_headers
408419
409420 ifneq ($(BPFOBJ),$(HOST_BPFOBJ))
@@ -411,7 +422,9 @@ $(HOST_BPFOBJ): $(wildcard $(BPFDIR)/*.[ch] $(BPFDIR)/Makefile) \ · 1 unmodified line
411422 $(APIDIR)/linux/bpf.h \
412423 | $(HOST_BUILD_DIR)/libbpf
413424 $(Q)$(MAKE) $(submake_extras) -C $(BPFDIR) \
414− EXTRA_CFLAGS='-g $(OPT_FLAGS)' ARCH= CROSS_COMPILE= \
425+ ARCH= CROSS_COMPILE= \
426+ EXTRA_CFLAGS='-g $(OPT_FLAGS) $(EXTRA_CFLAGS)' \
427+ EXTRA_LDFLAGS='$(EXTRA_LDFLAGS)' \
415428 OUTPUT=$(HOST_BUILD_DIR)/libbpf/ \
416429 CC="$(HOSTCC)" LD="$(HOSTLD)" \
417430 DESTDIR=$(HOST_SCRATCH_DIR)/ prefix= all install_headers
@@ -460,6 +473,7 @@ endef · 42 unmodified lines
460473 IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - </dev/null | \
461474 grep 'define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__')
462475 MENDIAN=$(if $(IS_LITTLE_ENDIAN),-mlittle-endian,-mbig-endian)
476+BPF_TARGET_ENDIAN=$(if $(IS_LITTLE_ENDIAN),--target=bpfel,--target=bpfeb)
463477
464478 ifneq ($(CROSS_COMPILE),)
465479 CLANG_TARGET_ARCH = --target=$(notdir $(CROSS_COMPILE:%-=%))
@@ -487,17 +501,17 @@ $(OUTPUT)/cgroup_getset_retval_hooks.o: cgroup_getset_retval_hooks.h · 21 unmodified lines
487501 # $4 - binary name
488502 define CLANG_BPF_BUILD_RULE
489503 $(call msg,CLNG-BPF,$4,$2)
490− $(Q)$(CLANG) $3 -O2 --target=bpf -c $1 -mcpu=v3 -o $2
504+ $(Q)$(CLANG) $3 -O2 $(BPF_TARGET_ENDIAN) -c $1 -mcpu=v3 -o $2
491505 endef
492506 # Similar to CLANG_BPF_BUILD_RULE, but with disabled alu32
493507 define CLANG_NOALU32_BPF_BUILD_RULE
494508 $(call msg,CLNG-BPF,$4,$2)
495− $(Q)$(CLANG) $3 -O2 --target=bpf -c $1 -mcpu=v2 -o $2
509+ $(Q)$(CLANG) $3 -O2 $(BPF_TARGET_ENDIAN) -c $1 -mcpu=v2 -o $2
496510 endef
497511 # Similar to CLANG_BPF_BUILD_RULE, but with cpu-v4
498512 define CLANG_CPUV4_BPF_BUILD_RULE
499513 $(call msg,CLNG-BPF,$4,$2)
500− $(Q)$(CLANG) $3 -O2 --target=bpf -c $1 -mcpu=v4 -o $2
514+ $(Q)$(CLANG) $3 -O2 $(BPF_TARGET_ENDIAN) -c $1 -mcpu=v4 -o $2
501515 endef
502516 # Build BPF object using GCC
503517 define GCC_BPF_BUILD_RULE
@@ -637,10 +651,11 @@ $(TRUNNER_BPF_SKELS_LINKED): $(TRUNNER_OUTPUT)/%: $$$$(%-deps) $(BPFTOOL) | $(TR · 133 unmodified lines
637651
638652 # When the compiler generates a %.d file, only skel basenames (not
639653 # full paths) are specified as prerequisites for corresponding %.o
640−# file. This target makes %.skel.h basename dependent on full paths,
641−# linking generated %.d dependency with actual %.skel.h files.
642−$(notdir %.skel.h): $(TRUNNER_OUTPUT)/%.skel.h
643− @true
654+# file. vpath directives below instruct make to search for skel files
655+# in TRUNNER_OUTPUT, if they are not present in the working directory.
656+vpath %.skel.h $(TRUNNER_OUTPUT)
657+vpath %.lskel.h $(TRUNNER_OUTPUT)
658+vpath %.subskel.h $(TRUNNER_OUTPUT)
644659
645660 endif
646661
@@ -727,6 +742,7 @@ TRUNNER_EXTRA_SOURCES := test_progs.c \ · 80 unmodified lines
727742 unpriv_helpers.c \
728743 netlink_helpers.c \
729744 jit_disasm_helpers.c \
745+ io_helpers.c \
730746 test_loader.c \
731747 xsk.c \
732748 disasm.c \
tools/testing/selftests/bpf/benchs/bench_trigger.c+2−1
@@ -4,6 +4,7 @@ · 3 unmodified lines
44 #include <argp.h>
55 #include <unistd.h>
66 #include <stdint.h>
7+#include "bpf_util.h"
78 #include "bench.h"
89 #include "trigger_bench.skel.h"
910 #include "trace_helpers.h"
@@ -72,7 +73,7 @@ static __always_inline void inc_counter(struct counter *counters) · 62 unmodified lines
7273 unsigned slot;
7374
7475 if (unlikely(tid == 0))
75− tid = syscall(SYS_gettid);
76+ tid = sys_gettid();
7677
7778 /* multiplicative hashing, it's fast */
7879 slot = 2654435769U * tid;
tools/testing/selftests/bpf/bpf_experimental.h+6−0
@@ -582,4 +582,10 @@ extern int bpf_wq_set_callback_impl(struct bpf_wq *wq, · 581 unmodified lines
582582 unsigned int flags__k, void *aux__ign) __ksym;
583583 #define bpf_wq_set_callback(timer, cb, flags) \
584584 bpf_wq_set_callback_impl(timer, cb, flags, NULL)
585+
586+struct bpf_iter_kmem_cache;
587+extern int bpf_iter_kmem_cache_new(struct bpf_iter_kmem_cache *it) __weak __ksym;
588+extern struct kmem_cache *bpf_iter_kmem_cache_next(struct bpf_iter_kmem_cache *it) __weak __ksym;
589+extern void bpf_iter_kmem_cache_destroy(struct bpf_iter_kmem_cache *it) __weak __ksym;
590+
585591 #endif
tools/testing/selftests/bpf/bpf_testmod/bpf_testmod-events.h+8−0
@@ -40,6 +40,14 @@ DECLARE_TRACE(bpf_testmod_test_nullable_bare, · 39 unmodified lines
4040 TP_ARGS(ctx__nullable)
4141 );
4242
43+struct sk_buff;
44+
45+DECLARE_TRACE(bpf_testmod_test_raw_tp_null,
46+ TP_PROTO(struct sk_buff *skb),
47+ TP_ARGS(skb)
48+);
49+
50+
4351 #undef BPF_TESTMOD_DECLARE_TRACE
4452 #ifdef DECLARE_TRACE_WRITABLE
4553 #define BPF_TESTMOD_DECLARE_TRACE(call, proto, args, size) \
tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c+106−0
@@ -245,6 +245,39 @@ __bpf_kfunc void bpf_testmod_ctx_release(struct bpf_testmod_ctx *ctx) · 244 unmodified lines
245245 call_rcu(&ctx->rcu, testmod_free_cb);
246246 }
247247
248+static struct bpf_testmod_ops3 *st_ops3;
249+
250+static int bpf_testmod_test_3(void)
251+{
252+ return 0;
253+}
254+
255+static int bpf_testmod_test_4(void)
256+{
257+ return 0;
258+}
259+
260+static struct bpf_testmod_ops3 __bpf_testmod_ops3 = {
261+ .test_1 = bpf_testmod_test_3,
262+ .test_2 = bpf_testmod_test_4,
263+};
264+
265+static void bpf_testmod_test_struct_ops3(void)
266+{
267+ if (st_ops3)
268+ st_ops3->test_1();
269+}
270+
271+__bpf_kfunc void bpf_testmod_ops3_call_test_1(void)
272+{
273+ st_ops3->test_1();
274+}
275+
276+__bpf_kfunc void bpf_testmod_ops3_call_test_2(void)
277+{
278+ st_ops3->test_2();
279+}
280+
248281 struct bpf_testmod_btf_type_tag_1 {
249282 int a;
250283 };
@@ -380,6 +413,10 @@ bpf_testmod_test_read(struct file *file, struct kobject *kobj, · 129 unmodified lines
380413
381414 (void)bpf_testmod_test_arg_ptr_to_struct(&struct_arg1_2);
382415
416+ (void)trace_bpf_testmod_test_raw_tp_null(NULL);
417+
418+ bpf_testmod_test_struct_ops3();
419+
383420 struct_arg3 = kmalloc((sizeof(struct bpf_testmod_struct_arg_3) +
384421 sizeof(int)), GFP_KERNEL);
385422 if (struct_arg3 != NULL) {
@@ -584,6 +621,8 @@ BTF_ID_FLAGS(func, bpf_kfunc_trusted_num_test, KF_TRUSTED_ARGS) · 198 unmodified lines
584621 BTF_ID_FLAGS(func, bpf_kfunc_rcu_task_test, KF_RCU)
585622 BTF_ID_FLAGS(func, bpf_testmod_ctx_create, KF_ACQUIRE | KF_RET_NULL)
586623 BTF_ID_FLAGS(func, bpf_testmod_ctx_release, KF_RELEASE)
624+BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_1)
625+BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_2)
587626 BTF_KFUNCS_END(bpf_testmod_common_kfunc_ids)
588627
589628 BTF_ID_LIST(bpf_testmod_dtor_ids)
@@ -1094,6 +1133,10 @@ static const struct bpf_verifier_ops bpf_testmod_verifier_ops = { · 504 unmodified lines
10941133 .is_valid_access = bpf_testmod_ops_is_valid_access,
10951134 };
10961135
1136+static const struct bpf_verifier_ops bpf_testmod_verifier_ops3 = {
1137+ .is_valid_access = bpf_testmod_ops_is_valid_access,
1138+};
1139+
10971140 static int bpf_dummy_reg(void *kdata, struct bpf_link *link)
10981141 {
10991142 struct bpf_testmod_ops *ops = kdata;
@@ -1173,6 +1216,68 @@ struct bpf_struct_ops bpf_testmod_ops2 = { · 73 unmodified lines
11731216 .owner = THIS_MODULE,
11741217 };
11751218
1219+static int st_ops3_reg(void *kdata, struct bpf_link *link)
1220+{
1221+ int err = 0;
1222+
1223+ mutex_lock(&st_ops_mutex);
1224+ if (st_ops3) {
1225+ pr_err("st_ops has already been registered\n");
1226+ err = -EEXIST;
1227+ goto unlock;
1228+ }
1229+ st_ops3 = kdata;
1230+
1231+unlock:
1232+ mutex_unlock(&st_ops_mutex);
1233+ return err;
1234+}
1235+
1236+static void st_ops3_unreg(void *kdata, struct bpf_link *link)
1237+{
1238+ mutex_lock(&st_ops_mutex);
1239+ st_ops3 = NULL;
1240+ mutex_unlock(&st_ops_mutex);
1241+}
1242+
1243+static void test_1_recursion_detected(struct bpf_prog *prog)
1244+{
1245+ struct bpf_prog_stats *stats;
1246+
1247+ stats = this_cpu_ptr(prog->stats);
1248+ printk("bpf_testmod: oh no, recursing into test_1, recursion_misses %llu",
1249+ u64_stats_read(&stats->misses));
1250+}
1251+
1252+static int st_ops3_check_member(const struct btf_type *t,
1253+ const struct btf_member *member,
1254+ const struct bpf_prog *prog)
1255+{
1256+ u32 moff = __btf_member_bit_offset(t, member) / 8;
1257+
1258+ switch (moff) {
1259+ case offsetof(struct bpf_testmod_ops3, test_1):
1260+ prog->aux->priv_stack_requested = true;
1261+ prog->aux->recursion_detected = test_1_recursion_detected;
1262+ fallthrough;
1263+ default:
1264+ break;
1265+ }
1266+ return 0;
1267+}
1268+
1269+struct bpf_struct_ops bpf_testmod_ops3 = {
1270+ .verifier_ops = &bpf_testmod_verifier_ops3,
1271+ .init = bpf_testmod_ops_init,
1272+ .init_member = bpf_testmod_ops_init_member,
1273+ .reg = st_ops3_reg,
1274+ .unreg = st_ops3_unreg,
1275+ .check_member = st_ops3_check_member,
1276+ .cfi_stubs = &__bpf_testmod_ops3,
1277+ .name = "bpf_testmod_ops3",
1278+ .owner = THIS_MODULE,
1279+};
1280+
11761281 static int bpf_test_mod_st_ops__test_prologue(struct st_ops_args *args)
11771282 {
11781283 return 0;
@@ -1331,6 +1436,7 @@ static int bpf_testmod_init(void) · 152 unmodified lines
13311436 ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &bpf_testmod_kfunc_set);
13321437 ret = ret ?: register_bpf_struct_ops(&bpf_bpf_testmod_ops, bpf_testmod_ops);
13331438 ret = ret ?: register_bpf_struct_ops(&bpf_testmod_ops2, bpf_testmod_ops2);
1439+ ret = ret ?: register_bpf_struct_ops(&bpf_testmod_ops3, bpf_testmod_ops3);
13341440 ret = ret ?: register_bpf_struct_ops(&testmod_st_ops, bpf_testmod_st_ops);
13351441 ret = ret ?: register_btf_id_dtor_kfuncs(bpf_testmod_dtors,
13361442 ARRAY_SIZE(bpf_testmod_dtors),
tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.h+5−0
@@ -94,6 +94,11 @@ struct bpf_testmod_ops2 { · 93 unmodified lines
9494 int (*test_1)(void);
9595 };
9696
97+struct bpf_testmod_ops3 {
98+ int (*test_1)(void);
99+ int (*test_2)(void);
100+};
101+
97102 struct st_ops_args {
98103 u64 a;
99104 };
tools/testing/selftests/bpf/bpf_util.h+12−0
@@ -6,6 +6,7 @@ · 5 unmodified lines
66 #include <stdlib.h>
77 #include <string.h>
88 #include <errno.h>
9+#include <syscall.h>
910 #include <bpf/libbpf.h> /* libbpf_num_possible_cpus */
1011
1112 static inline unsigned int bpf_num_possible_cpus(void)
@@ -59,4 +60,15 @@ static inline void bpf_strlcpy(char *dst, const char *src, size_t sz) · 47 unmodified lines
5960 (offsetof(TYPE, MEMBER) + sizeof_field(TYPE, MEMBER))
6061 #endif
6162
63+/* Availability of gettid across glibc versions is hit-and-miss, therefore
64+ * fallback to syscall in this macro and use it everywhere.
65+ */
66+#ifndef sys_gettid
67+#define sys_gettid() syscall(SYS_gettid)
68+#endif
69+
70+#ifndef ENOTSUPP
71+#define ENOTSUPP 524
72+#endif
73+
6274 #endif /* __BPF_UTIL__ */
tools/testing/selftests/bpf/config.vm+5−2
@@ -1,12 +1,15 @@
1−CONFIG_9P_FS=y
21 CONFIG_9P_FS_POSIX_ACL=y
32 CONFIG_9P_FS_SECURITY=y
3+CONFIG_9P_FS=y
44 CONFIG_CRYPTO_DEV_VIRTIO=y
5−CONFIG_NET_9P=y
5+CONFIG_FUSE_FS=y
6+CONFIG_FUSE_PASSTHROUGH=y
67 CONFIG_NET_9P_VIRTIO=y
8+CONFIG_NET_9P=y
79 CONFIG_VIRTIO_BALLOON=y
810 CONFIG_VIRTIO_BLK=y
911 CONFIG_VIRTIO_CONSOLE=y
12+CONFIG_VIRTIO_FS=y
1013 CONFIG_VIRTIO_NET=y
1114 CONFIG_VIRTIO_PCI=y
1215 CONFIG_VIRTIO_VSOCKETS_COMMON=y
tools/testing/selftests/bpf/io_helpers.c+21−0
@@ -0,0 +1,21 @@
1+// SPDX-License-Identifier: GPL-2.0
2+#include <sys/select.h>
3+#include <unistd.h>
4+#include <errno.h>
5+
6+int read_with_timeout(int fd, char *buf, size_t count, long usec)
7+{
8+ const long M = 1000 * 1000;
9+ struct timeval tv = { usec / M, usec % M };
10+ fd_set fds;
11+ int err;
12+
13+ FD_ZERO(&fds);
14+ FD_SET(fd, &fds);
15+ err = select(fd + 1, &fds, NULL, NULL, &tv);
16+ if (err < 0)
17+ return err;
18+ if (FD_ISSET(fd, &fds))
19+ return read(fd, buf, count);
20+ return -EAGAIN;
21+}
tools/testing/selftests/bpf/io_helpers.h+7−0
@@ -0,0 +1,7 @@
1+// SPDX-License-Identifier: GPL-2.0
2+#include <unistd.h>
3+
4+/* As a regular read(2), but allows to specify a timeout in micro-seconds.
5+ * Returns -EAGAIN on timeout.
6+ */
7+int read_with_timeout(int fd, char *buf, size_t count, long usec);
tools/testing/selftests/bpf/map_tests/task_storage_map.c+2−1
@@ -12,6 +12,7 @@ · 11 unmodified lines
1212 #include <bpf/bpf.h>
1313 #include <bpf/libbpf.h>
1414
15+#include "bpf_util.h"
1516 #include "test_maps.h"
1617 #include "task_local_storage_helpers.h"
1718 #include "read_bpf_task_storage_busy.skel.h"
@@ -115,7 +116,7 @@ void test_task_storage_map_stress_lookup(void) · 97 unmodified lines
115116 CHECK(err, "attach", "error %d\n", err);
116117
117118 /* Trigger program */
118− syscall(SYS_gettid);
119+ sys_gettid();
119120 skel->bss->pid = 0;
120121
121122 CHECK(skel->bss->busy != 0, "bad bpf_task_storage_busy", "got %d\n", skel->bss->busy);
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c+1−1
@@ -690,7 +690,7 @@ void test_bpf_cookie(void) · 689 unmodified lines
690690 if (!ASSERT_OK_PTR(skel, "skel_open"))
691691 return;
692692
693− skel->bss->my_tid = syscall(SYS_gettid);
693+ skel->bss->my_tid = sys_gettid();
694694
695695 if (test__start_subtest("kprobe"))
696696 kprobe_subtest(skel);
tools/testing/selftests/bpf/prog_tests/bpf_iter.c+7−7
@@ -226,7 +226,7 @@ static void test_task_common_nocheck(struct bpf_iter_attach_opts *opts, · 225 unmodified lines
226226 ASSERT_OK(pthread_create(&thread_id, NULL, &do_nothing_wait, NULL),
227227 "pthread_create");
228228
229− skel->bss->tid = gettid();
229+ skel->bss->tid = sys_gettid();
230230
231231 do_dummy_read_opts(skel->progs.dump_task, opts);
232232
@@ -255,20 +255,20 @@ static void *run_test_task_tid(void *arg) · 22 unmodified lines
255255 union bpf_iter_link_info linfo;
256256 int num_unknown_tid, num_known_tid;
257257
258− ASSERT_NEQ(getpid(), gettid(), "check_new_thread_id");
258+ ASSERT_NEQ(getpid(), sys_gettid(), "check_new_thread_id");
259259
260260 memset(&linfo, 0, sizeof(linfo));
261− linfo.task.tid = gettid();
261+ linfo.task.tid = sys_gettid();
262262 opts.link_info = &linfo;
263263 opts.link_info_len = sizeof(linfo);
264264 test_task_common(&opts, 0, 1);
265265
266266 linfo.task.tid = 0;
267267 linfo.task.pid = getpid();
268− /* This includes the parent thread, this thread,
268+ /* This includes the parent thread, this thread, watchdog timer thread
269269 * and the do_nothing_wait thread
270270 */
271− test_task_common(&opts, 2, 1);
271+ test_task_common(&opts, 3, 1);
272272
273273 test_task_common_nocheck(NULL, &num_unknown_tid, &num_known_tid);
274274 ASSERT_GT(num_unknown_tid, 2, "check_num_unknown_tid");
@@ -297,7 +297,7 @@ static void test_task_pid(void) · 22 unmodified lines
297297 opts.link_info = &linfo;
298298 opts.link_info_len = sizeof(linfo);
299299
300− test_task_common(&opts, 1, 1);
300+ test_task_common(&opts, 2, 1);
301301 }
302302
303303 static void test_task_pidfd(void)
@@ -315,7 +315,7 @@ static void test_task_pidfd(void) · 11 unmodified lines
315315 opts.link_info = &linfo;
316316 opts.link_info_len = sizeof(linfo);
317317
318− test_task_common(&opts, 1, 1);
318+ test_task_common(&opts, 2, 1);
319319
320320 close(pidfd);
321321 }
tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c+0−4
@@ -16,10 +16,6 @@ · 15 unmodified lines
1616 #include "tcp_ca_kfunc.skel.h"
1717 #include "bpf_cc_cubic.skel.h"
1818
19−#ifndef ENOTSUPP
20−#define ENOTSUPP 524
21−#endif
22−
2319 static const unsigned int total_bytes = 10 * 1024 * 1024;
2420 static int expected_stg = 0xeB9F;
2521
tools/testing/selftests/bpf/prog_tests/cb_refs.c+2−2
@@ -11,8 +11,8 @@ struct { · 10 unmodified lines
1111 const char *prog_name;
1212 const char *err_msg;
1313 } cb_refs_tests[] = {
14− { "underflow_prog", "reference has not been acquired before" },
15− { "leak_prog", "Unreleased reference" },
14+ { "underflow_prog", "must point to scalar, or struct with scalar" },
15+ { "leak_prog", "Possibly NULL pointer passed to helper arg2" },
1616 { "nested_cb", "Unreleased reference id=4 alloc_insn=2" }, /* alloc_insn=2{4,5} */
1717 { "non_cb_transfer_ref", "Unreleased reference id=4 alloc_insn=1" }, /* alloc_insn=1{1,2} */
1818 };
tools/testing/selftests/bpf/prog_tests/cgrp_local_storage.c+5−5
@@ -63,14 +63,14 @@ static void test_tp_btf(int cgroup_fd) · 62 unmodified lines
6363 if (!ASSERT_OK(err, "map_delete_elem"))
6464 goto out;
6565
66− skel->bss->target_pid = syscall(SYS_gettid);
66+ skel->bss->target_pid = sys_gettid();
6767
6868 err = cgrp_ls_tp_btf__attach(skel);
6969 if (!ASSERT_OK(err, "skel_attach"))
7070 goto out;
7171
72− syscall(SYS_gettid);
73− syscall(SYS_gettid);
72+ sys_gettid();
73+ sys_gettid();
7474
7575 skel->bss->target_pid = 0;
7676
@@ -154,7 +154,7 @@ static void test_recursion(int cgroup_fd) · 77 unmodified lines
154154 goto out;
155155
156156 /* trigger sys_enter, make sure it does not cause deadlock */
157− syscall(SYS_gettid);
157+ sys_gettid();
158158
159159 out:
160160 cgrp_ls_recursion__destroy(skel);
@@ -224,7 +224,7 @@ static void test_yes_rcu_lock(__u64 cgroup_id) · 63 unmodified lines
224224 return;
225225
226226 CGROUP_MODE_SET(skel);
227− skel->bss->target_pid = syscall(SYS_gettid);
227+ skel->bss->target_pid = sys_gettid();
228228
229229 bpf_program__set_autoload(skel->progs.yes_rcu_lock, true);
230230 err = cgrp_ls_sleepable__load(skel);
tools/testing/selftests/bpf/prog_tests/core_reloc.c+1−1
@@ -1010,7 +1010,7 @@ static void run_core_reloc_tests(bool use_btfgen) · 1009 unmodified lines
10101010 struct data *data;
10111011 void *mmap_data = NULL;
10121012
1013− my_pid_tgid = getpid() | ((uint64_t)syscall(SYS_gettid) << 32);
1013+ my_pid_tgid = getpid() | ((uint64_t)sys_gettid() << 32);
10141014
10151015 for (i = 0; i < ARRAY_SIZE(test_cases); i++) {
10161016 char btf_file[] = "/tmp/core_reloc.btf.XXXXXX";
tools/testing/selftests/bpf/prog_tests/iters.c+2−2
@@ -192,8 +192,8 @@ static void subtest_task_iters(void) · 191 unmodified lines
192192 syscall(SYS_getpgid);
193193 iters_task__detach(skel);
194194 ASSERT_EQ(skel->bss->procs_cnt, 1, "procs_cnt");
195− ASSERT_EQ(skel->bss->threads_cnt, thread_num + 1, "threads_cnt");
196− ASSERT_EQ(skel->bss->proc_threads_cnt, thread_num + 1, "proc_threads_cnt");
195+ ASSERT_EQ(skel->bss->threads_cnt, thread_num + 2, "threads_cnt");
196+ ASSERT_EQ(skel->bss->proc_threads_cnt, thread_num + 2, "proc_threads_cnt");
197197 ASSERT_EQ(skel->bss->invalid_cnt, 0, "invalid_cnt");
198198 pthread_mutex_unlock(&do_nothing_mutex);
199199 for (int i = 0; i < thread_num; i++)
tools/testing/selftests/bpf/prog_tests/kmem_cache_iter.c+126−0
@@ -0,0 +1,126 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/* Copyright (c) 2024 Google */
3+
4+#include <test_progs.h>
5+#include <bpf/libbpf.h>
6+#include <bpf/btf.h>
7+#include "kmem_cache_iter.skel.h"
8+
9+#define SLAB_NAME_MAX 32
10+
11+struct kmem_cache_result {
12+ char name[SLAB_NAME_MAX];
13+ long obj_size;
14+};
15+
16+static void subtest_kmem_cache_iter_check_task_struct(struct kmem_cache_iter *skel)
17+{
18+ LIBBPF_OPTS(bpf_test_run_opts, opts,
19+ .flags = 0, /* Run it with the current task */
20+ );
21+ int prog_fd = bpf_program__fd(skel->progs.check_task_struct);
22+
23+ /* Get task_struct and check it if's from a slab cache */
24+ ASSERT_OK(bpf_prog_test_run_opts(prog_fd, &opts), "prog_test_run");
25+
26+ /* The BPF program should set 'found' variable */
27+ ASSERT_EQ(skel->bss->task_struct_found, 1, "task_struct_found");
28+}
29+
30+static void subtest_kmem_cache_iter_check_slabinfo(struct kmem_cache_iter *skel)
31+{
32+ FILE *fp;
33+ int map_fd;
34+ char name[SLAB_NAME_MAX];
35+ unsigned long objsize;
36+ char rest_of_line[1000];
37+ struct kmem_cache_result r;
38+ int seen = 0;
39+
40+ fp = fopen("/proc/slabinfo", "r");
41+ if (fp == NULL) {
42+ /* CONFIG_SLUB_DEBUG is not enabled */
43+ return;
44+ }
45+
46+ map_fd = bpf_map__fd(skel->maps.slab_result);
47+
48+ /* Ignore first two lines for header */
49+ fscanf(fp, "slabinfo - version: %*d.%*d\n");
50+ fscanf(fp, "# %*s %*s %*s %*s %*s %*s : %[^\n]\n", rest_of_line);
51+
52+ /* Compare name and objsize only - others can be changes frequently */
53+ while (fscanf(fp, "%s %*u %*u %lu %*u %*u : %[^\n]\n",
54+ name, &objsize, rest_of_line) == 3) {
55+ int ret = bpf_map_lookup_elem(map_fd, &seen, &r);
56+
57+ if (!ASSERT_OK(ret, "kmem_cache_lookup"))
58+ break;
59+
60+ ASSERT_STREQ(r.name, name, "kmem_cache_name");
61+ ASSERT_EQ(r.obj_size, objsize, "kmem_cache_objsize");
62+
63+ seen++;
64+ }
65+
66+ ASSERT_EQ(skel->bss->kmem_cache_seen, seen, "kmem_cache_seen_eq");
67+
68+ fclose(fp);
69+}
70+
71+static void subtest_kmem_cache_iter_open_coded(struct kmem_cache_iter *skel)
72+{
73+ LIBBPF_OPTS(bpf_test_run_opts, topts);
74+ int err, fd;
75+
76+ /* No need to attach it, just run it directly */
77+ fd = bpf_program__fd(skel->progs.open_coded_iter);
78+
79+ err = bpf_prog_test_run_opts(fd, &topts);
80+ if (!ASSERT_OK(err, "test_run_opts err"))
81+ return;
82+ if (!ASSERT_OK(topts.retval, "test_run_opts retval"))
83+ return;
84+
85+ /* It should be same as we've seen from the explicit iterator */
86+ ASSERT_EQ(skel->bss->open_coded_seen, skel->bss->kmem_cache_seen, "open_code_seen_eq");
87+}
88+
89+void test_kmem_cache_iter(void)
90+{
91+ struct kmem_cache_iter *skel = NULL;
92+ char buf[256];
93+ int iter_fd;
94+
95+ skel = kmem_cache_iter__open_and_load();
96+ if (!ASSERT_OK_PTR(skel, "kmem_cache_iter__open_and_load"))
97+ return;
98+
99+ if (!ASSERT_OK(kmem_cache_iter__attach(skel), "skel_attach"))
100+ goto destroy;
101+
102+ iter_fd = bpf_iter_create(bpf_link__fd(skel->links.slab_info_collector));
103+ if (!ASSERT_GE(iter_fd, 0, "iter_create"))
104+ goto destroy;
105+
106+ memset(buf, 0, sizeof(buf));
107+ while (read(iter_fd, buf, sizeof(buf) > 0)) {
108+ /* Read out all contents */
109+ printf("%s", buf);
110+ }
111+
112+ /* Next reads should return 0 */
113+ ASSERT_EQ(read(iter_fd, buf, sizeof(buf)), 0, "read");
114+
115+ if (test__start_subtest("check_task_struct"))
116+ subtest_kmem_cache_iter_check_task_struct(skel);
117+ if (test__start_subtest("check_slabinfo"))
118+ subtest_kmem_cache_iter_check_slabinfo(skel);
119+ if (test__start_subtest("open_coded_iter"))
120+ subtest_kmem_cache_iter_open_coded(skel);
121+
122+ close(iter_fd);
123+
124+destroy:
125+ kmem_cache_iter__destroy(skel);
126+}
tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c+2−0
@@ -6,6 +6,7 @@ · 5 unmodified lines
66 #include "kprobe_multi_override.skel.h"
77 #include "kprobe_multi_session.skel.h"
88 #include "kprobe_multi_session_cookie.skel.h"
9+#include "kprobe_multi_verifier.skel.h"
910 #include "bpf/libbpf_internal.h"
1011 #include "bpf/hashmap.h"
1112
@@ -764,4 +765,5 @@ void test_kprobe_multi_test(void) · 752 unmodified lines
764765 test_session_skel_api();
765766 if (test__start_subtest("session_cookie"))
766767 test_session_cookie_skel_api();
768+ RUN_TESTS(kprobe_multi_verifier);
767769 }
tools/testing/selftests/bpf/prog_tests/linked_funcs.c+1−1
@@ -20,7 +20,7 @@ void test_linked_funcs(void) · 19 unmodified lines
2020 bpf_program__set_autoload(skel->progs.handler1, true);
2121 bpf_program__set_autoload(skel->progs.handler2, true);
2222
23− skel->rodata->my_tid = syscall(SYS_gettid);
23+ skel->rodata->my_tid = sys_gettid();
2424 skel->bss->syscall_id = SYS_getpgid;
2525
2626 err = linked_funcs__load(skel);
tools/testing/selftests/bpf/prog_tests/log_buf.c+0−3
@@ -169,7 +169,6 @@ static void bpf_prog_load_log_buf(void) · 168 unmodified lines
169169 ASSERT_GE(fd, 0, "good_fd1");
170170 if (fd >= 0)
171171 close(fd);
172− fd = -1;
173172
174173 /* log_level == 2 should always fill log_buf, even for good prog */
175174 log_buf[0] = '\0';
@@ -180,7 +179,6 @@ static void bpf_prog_load_log_buf(void) · 4 unmodified lines
180179 ASSERT_GE(fd, 0, "good_fd2");
181180 if (fd >= 0)
182181 close(fd);
183− fd = -1;
184182
185183 /* log_level == 0 should fill log_buf for bad prog */
186184 log_buf[0] = '\0';
@@ -191,7 +189,6 @@ static void bpf_prog_load_log_buf(void) · 4 unmodified lines
191189 ASSERT_LT(fd, 0, "bad_fd");
192190 if (fd >= 0)
193191 close(fd);
194− fd = -1;
195192
196193 free(log_buf);
197194 }
tools/testing/selftests/bpf/prog_tests/lsm_cgroup.c+0−4
@@ -10,10 +10,6 @@ · 9 unmodified lines
1010 #include "cgroup_helpers.h"
1111 #include "network_helpers.h"
1212
13−#ifndef ENOTSUPP
14−#define ENOTSUPP 524
15−#endif
16−
1713 static struct btf *btf;
1814
1915 static __u32 query_prog_cnt(int cgroup_fd, const char *attach_func)
tools/testing/selftests/bpf/prog_tests/map_in_map.c+131−1
@@ -5,7 +5,9 @@ · 4 unmodified lines
55 #include <sys/syscall.h>
66 #include <test_progs.h>
77 #include <bpf/btf.h>
8+
89 #include "access_map_in_map.skel.h"
10+#include "update_map_in_htab.skel.h"
911
1012 struct thread_ctx {
1113 pthread_barrier_t barrier;
@@ -127,6 +129,131 @@ out: · 115 unmodified lines
127129 access_map_in_map__destroy(skel);
128130 }
129131
132+static void add_del_fd_htab(int outer_fd)
133+{
134+ int inner_fd, err;
135+ int key = 1;
136+
137+ inner_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "arr1", 4, 4, 1, NULL);
138+ if (!ASSERT_OK_FD(inner_fd, "inner1"))
139+ return;
140+ err = bpf_map_update_elem(outer_fd, &key, &inner_fd, BPF_NOEXIST);
141+ close(inner_fd);
142+ if (!ASSERT_OK(err, "add"))
143+ return;
144+
145+ /* Delete */
146+ err = bpf_map_delete_elem(outer_fd, &key);
147+ ASSERT_OK(err, "del");
148+}
149+
150+static void overwrite_fd_htab(int outer_fd)
151+{
152+ int inner_fd, err;
153+ int key = 1;
154+
155+ inner_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "arr1", 4, 4, 1, NULL);
156+ if (!ASSERT_OK_FD(inner_fd, "inner1"))
157+ return;
158+ err = bpf_map_update_elem(outer_fd, &key, &inner_fd, BPF_NOEXIST);
159+ close(inner_fd);
160+ if (!ASSERT_OK(err, "add"))
161+ return;
162+
163+ /* Overwrite */
164+ inner_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "arr2", 4, 4, 1, NULL);
165+ if (!ASSERT_OK_FD(inner_fd, "inner2"))
166+ goto out;
167+ err = bpf_map_update_elem(outer_fd, &key, &inner_fd, BPF_EXIST);
168+ close(inner_fd);
169+ if (!ASSERT_OK(err, "overwrite"))
170+ goto out;
171+
172+ err = bpf_map_delete_elem(outer_fd, &key);
173+ ASSERT_OK(err, "del");
174+ return;
175+out:
176+ bpf_map_delete_elem(outer_fd, &key);
177+}
178+
179+static void lookup_delete_fd_htab(int outer_fd)
180+{
181+ int key = 1, value;
182+ int inner_fd, err;
183+
184+ inner_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "arr1", 4, 4, 1, NULL);
185+ if (!ASSERT_OK_FD(inner_fd, "inner1"))
186+ return;
187+ err = bpf_map_update_elem(outer_fd, &key, &inner_fd, BPF_NOEXIST);
188+ close(inner_fd);
189+ if (!ASSERT_OK(err, "add"))
190+ return;
191+
192+ /* lookup_and_delete is not supported for htab of maps */
193+ err = bpf_map_lookup_and_delete_elem(outer_fd, &key, &value);
194+ ASSERT_EQ(err, -ENOTSUPP, "lookup_del");
195+
196+ err = bpf_map_delete_elem(outer_fd, &key);
197+ ASSERT_OK(err, "del");
198+}
199+
200+static void batched_lookup_delete_fd_htab(int outer_fd)
201+{
202+ int keys[2] = {1, 2}, values[2];
203+ unsigned int cnt, batch;
204+ int inner_fd, err;
205+
206+ inner_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "arr1", 4, 4, 1, NULL);
207+ if (!ASSERT_OK_FD(inner_fd, "inner1"))
208+ return;
209+
210+ err = bpf_map_update_elem(outer_fd, &keys[0], &inner_fd, BPF_NOEXIST);
211+ close(inner_fd);
212+ if (!ASSERT_OK(err, "add1"))
213+ return;
214+
215+ inner_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "arr2", 4, 4, 1, NULL);
216+ if (!ASSERT_OK_FD(inner_fd, "inner2"))
217+ goto out;
218+ err = bpf_map_update_elem(outer_fd, &keys[1], &inner_fd, BPF_NOEXIST);
219+ close(inner_fd);
220+ if (!ASSERT_OK(err, "add2"))
221+ goto out;
222+
223+ /* batched lookup_and_delete */
224+ cnt = ARRAY_SIZE(keys);
225+ err = bpf_map_lookup_and_delete_batch(outer_fd, NULL, &batch, keys, values, &cnt, NULL);
226+ ASSERT_TRUE((!err || err == -ENOENT), "delete_batch ret");
227+ ASSERT_EQ(cnt, ARRAY_SIZE(keys), "delete_batch cnt");
228+
229+out:
230+ bpf_map_delete_elem(outer_fd, &keys[0]);
231+}
232+
233+static void test_update_map_in_htab(bool preallocate)
234+{
235+ struct update_map_in_htab *skel;
236+ int err, fd;
237+
238+ skel = update_map_in_htab__open();
239+ if (!ASSERT_OK_PTR(skel, "open"))
240+ return;
241+
242+ err = update_map_in_htab__load(skel);
243+ if (!ASSERT_OK(err, "load"))
244+ goto out;
245+
246+ fd = preallocate ? bpf_map__fd(skel->maps.outer_htab_map) :
247+ bpf_map__fd(skel->maps.outer_alloc_htab_map);
248+
249+ add_del_fd_htab(fd);
250+ overwrite_fd_htab(fd);
251+ lookup_delete_fd_htab(fd);
252+ batched_lookup_delete_fd_htab(fd);
253+out:
254+ update_map_in_htab__destroy(skel);
255+}
256+
130257 void test_map_in_map(void)
131258 {
132259 if (test__start_subtest("acc_map_in_array"))
@@ -137,5 +264,8 @@ void test_map_in_map(void) · 4 unmodified lines
137264 test_map_in_map_access("access_map_in_htab", "outer_htab_map");
138265 if (test__start_subtest("sleepable_acc_map_in_htab"))
139266 test_map_in_map_access("sleepable_access_map_in_htab", "outer_htab_map");
267+ if (test__start_subtest("update_map_in_htab"))
268+ test_update_map_in_htab(true);
269+ if (test__start_subtest("update_map_in_alloc_htab"))
270+ test_update_map_in_htab(false);
140271 }
141−
tools/testing/selftests/bpf/prog_tests/ns_current_pid_tgid.c+1−1
@@ -23,7 +23,7 @@ static int get_pid_tgid(pid_t *pid, pid_t *tgid, · 22 unmodified lines
2323 struct stat st;
2424 int err;
2525
26− *pid = syscall(SYS_gettid);
26+ *pid = sys_gettid();
2727 *tgid = getpid();
2828
2929 err = stat("/proc/self/ns/pid", &st);
tools/testing/selftests/bpf/prog_tests/perf_link.c+13−2
@@ -4,8 +4,12 @@ · 3 unmodified lines
44 #include <pthread.h>
55 #include <sched.h>
66 #include <test_progs.h>
7+#include "testing_helpers.h"
78 #include "test_perf_link.skel.h"
89
10+#define BURN_TIMEOUT_MS 100
11+#define BURN_TIMEOUT_NS BURN_TIMEOUT_MS * 1000000
12+
913 static void burn_cpu(void)
1014 {
1115 volatile int j = 0;
@@ -32,6 +36,7 @@ void serial_test_perf_link(void) · 20 unmodified lines
3236 int run_cnt_before, run_cnt_after;
3337 struct bpf_link_info info;
3438 __u32 info_len = sizeof(info);
39+ __u64 timeout_time_ns;
3540
3641 /* create perf event */
3742 memset(&attr, 0, sizeof(attr));
@@ -63,8 +68,14 @@ void serial_test_perf_link(void) · 25 unmodified lines
6368 ASSERT_GT(info.prog_id, 0, "link_prog_id");
6469
6570 /* ensure we get at least one perf_event prog execution */
66− burn_cpu();
67− ASSERT_GT(skel->bss->run_cnt, 0, "run_cnt");
71+ timeout_time_ns = get_time_ns() + BURN_TIMEOUT_NS;
72+ while (true) {
73+ burn_cpu();
74+ if (skel->bss->run_cnt > 0)
75+ break;
76+ if (!ASSERT_LT(get_time_ns(), timeout_time_ns, "run_cnt_timeout"))
77+ break;
78+ }
6879
6980 /* perf_event is still active, but we close link and BPF program
7081 * shouldn't be executed anymore
tools/testing/selftests/bpf/prog_tests/raw_tp_null.c+25−0
@@ -0,0 +1,25 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
3+
4+#include <test_progs.h>
5+#include "raw_tp_null.skel.h"
6+
7+void test_raw_tp_null(void)
8+{
9+ struct raw_tp_null *skel;
10+
11+ skel = raw_tp_null__open_and_load();
12+ if (!ASSERT_OK_PTR(skel, "raw_tp_null__open_and_load"))
13+ return;
14+
15+ skel->bss->tid = sys_gettid();
16+
17+ if (!ASSERT_OK(raw_tp_null__attach(skel), "raw_tp_null__attach"))
18+ goto end;
19+
20+ ASSERT_OK(trigger_module_test_read(2), "trigger testmod read");
21+ ASSERT_EQ(skel->bss->i, 3, "invocations");
22+
23+end:
24+ raw_tp_null__destroy(skel);
25+}
tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c+2−2
@@ -21,7 +21,7 @@ static void test_success(void) · 20 unmodified lines
2121 if (!ASSERT_OK_PTR(skel, "skel_open"))
2222 return;
2323
24− skel->bss->target_pid = syscall(SYS_gettid);
24+ skel->bss->target_pid = sys_gettid();
2525
2626 bpf_program__set_autoload(skel->progs.get_cgroup_id, true);
2727 bpf_program__set_autoload(skel->progs.task_succ, true);
@@ -58,7 +58,7 @@ static void test_rcuptr_acquire(void) · 30 unmodified lines
5858 if (!ASSERT_OK_PTR(skel, "skel_open"))
5959 return;
6060
61− skel->bss->target_pid = syscall(SYS_gettid);
61+ skel->bss->target_pid = sys_gettid();
6262
6363 bpf_program__set_autoload(skel->progs.task_acquire, true);
6464 err = rcu_read_lock__load(skel);
tools/testing/selftests/bpf/prog_tests/send_signal.c+111−35
@@ -3,22 +3,32 @@ · 2 unmodified lines
33 #include <sys/time.h>
44 #include <sys/resource.h>
55 #include "test_send_signal_kern.skel.h"
6+#include "io_helpers.h"
67
78 static int sigusr1_received;
89
910 static void sigusr1_handler(int signum)
1011 {
11− sigusr1_received = 1;
12+ sigusr1_received = 8;
13+}
14+
15+static void sigusr1_siginfo_handler(int s, siginfo_t *i, void *v)
16+{
17+ sigusr1_received = (int)(long long)i->si_value.sival_ptr;
1218 }
1319
1420 static void test_send_signal_common(struct perf_event_attr *attr,
15− bool signal_thread)
21+ bool signal_thread, bool remote)
1622 {
1723 struct test_send_signal_kern *skel;
24+ struct sigaction sa;
1825 int pipe_c2p[2], pipe_p2c[2];
1926 int err = -1, pmu_fd = -1;
27+ volatile int j = 0;
28+ int retry_count;
2029 char buf[256];
2130 pid_t pid;
31+ int old_prio;
2232
2333 if (!ASSERT_OK(pipe(pipe_c2p), "pipe_c2p"))
2434 return;
@@ -39,11 +49,14 @@ static void test_send_signal_common(struct perf_event_attr *attr, · 14 unmodified lines
3949 }
4050
4151 if (pid == 0) {
42− int old_prio;
43− volatile int j = 0;
44−
4552 /* install signal handler and notify parent */
46− ASSERT_NEQ(signal(SIGUSR1, sigusr1_handler), SIG_ERR, "signal");
53+ if (remote) {
54+ sa.sa_sigaction = sigusr1_siginfo_handler;
55+ sa.sa_flags = SA_RESTART | SA_SIGINFO;
56+ ASSERT_NEQ(sigaction(SIGUSR1, &sa, NULL), -1, "sigaction");
57+ } else {
58+ ASSERT_NEQ(signal(SIGUSR1, sigusr1_handler), SIG_ERR, "signal");
59+ }
4760
4861 close(pipe_c2p[0]); /* close read */
4962 close(pipe_p2c[1]); /* close write */
@@ -52,10 +65,12 @@ static void test_send_signal_common(struct perf_event_attr *attr, · 2 unmodified lines
5265 * that if an interrupt happens, the underlying task
5366 * is this process.
5467 */
55− errno = 0;
56− old_prio = getpriority(PRIO_PROCESS, 0);
57− ASSERT_OK(errno, "getpriority");
58− ASSERT_OK(setpriority(PRIO_PROCESS, 0, -20), "setpriority");
68+ if (!remote) {
69+ errno = 0;
70+ old_prio = getpriority(PRIO_PROCESS, 0);
71+ ASSERT_OK(errno, "getpriority");
72+ ASSERT_OK(setpriority(PRIO_PROCESS, 0, -20), "setpriority");
73+ }
5974
6075 /* notify parent signal handler is installed */
6176 ASSERT_EQ(write(pipe_c2p[1], buf, 1), 1, "pipe_write");
@@ -66,20 +81,25 @@ static void test_send_signal_common(struct perf_event_attr *attr, · 4 unmodified lines
6681 /* wait a little for signal handler */
6782 for (int i = 0; i < 1000000000 && !sigusr1_received; i++) {
6883 j /= i + j + 1;
69− if (!attr)
70− /* trigger the nanosleep tracepoint program. */
71− usleep(1);
84+ if (remote)
85+ sleep(1);
86+ else
87+ if (!attr)
88+ /* trigger the nanosleep tracepoint program. */
89+ usleep(1);
7290 }
7391
74− buf[0] = sigusr1_received ? '2' : '0';
75− ASSERT_EQ(sigusr1_received, 1, "sigusr1_received");
92+ buf[0] = sigusr1_received;
93+
94+ ASSERT_EQ(sigusr1_received, 8, "sigusr1_received");
7695 ASSERT_EQ(write(pipe_c2p[1], buf, 1), 1, "pipe_write");
7796
7897 /* wait for parent notification and exit */
7998 ASSERT_EQ(read(pipe_p2c[0], buf, 1), 1, "pipe_read");
8099
81100 /* restore the old priority */
82− ASSERT_OK(setpriority(PRIO_PROCESS, 0, old_prio), "setpriority");
101+ if (!remote)
102+ ASSERT_OK(setpriority(PRIO_PROCESS, 0, old_prio), "setpriority");
83103
84104 close(pipe_c2p[1]);
85105 close(pipe_p2c[0]);
@@ -93,6 +113,17 @@ static void test_send_signal_common(struct perf_event_attr *attr, · 7 unmodified lines
93113 if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))
94114 goto skel_open_load_failure;
95115
116+ /* boost with a high priority so we got a higher chance
117+ * that if an interrupt happens, the underlying task
118+ * is this process.
119+ */
120+ if (remote) {
121+ errno = 0;
122+ old_prio = getpriority(PRIO_PROCESS, 0);
123+ ASSERT_OK(errno, "getpriority");
124+ ASSERT_OK(setpriority(PRIO_PROCESS, 0, -20), "setpriority");
125+ }
126+
96127 if (!attr) {
97128 err = test_send_signal_kern__attach(skel);
98129 if (!ASSERT_OK(err, "skel_attach")) {
@@ -100,8 +131,12 @@ static void test_send_signal_common(struct perf_event_attr *attr, · 1 unmodified line
100131 goto destroy_skel;
101132 }
102133 } else {
103− pmu_fd = syscall(__NR_perf_event_open, attr, pid, -1 /* cpu */,
104− -1 /* group id */, 0 /* flags */);
134+ if (!remote)
135+ pmu_fd = syscall(__NR_perf_event_open, attr, pid, -1 /* cpu */,
136+ -1 /* group id */, 0 /* flags */);
137+ else
138+ pmu_fd = syscall(__NR_perf_event_open, attr, getpid(), -1 /* cpu */,
139+ -1 /* group id */, 0 /* flags */);
105140 if (!ASSERT_GE(pmu_fd, 0, "perf_event_open")) {
106141 err = -1;
107142 goto destroy_skel;
@@ -119,13 +154,36 @@ static void test_send_signal_common(struct perf_event_attr *attr, · 11 unmodified lines
119154 /* trigger the bpf send_signal */
120155 skel->bss->signal_thread = signal_thread;
121156 skel->bss->sig = SIGUSR1;
122− skel->bss->pid = pid;
157+ if (!remote) {
158+ skel->bss->target_pid = 0;
159+ skel->bss->pid = pid;
160+ } else {
161+ skel->bss->target_pid = pid;
162+ skel->bss->pid = getpid();
163+ }
123164
124165 /* notify child that bpf program can send_signal now */
125166 ASSERT_EQ(write(pipe_p2c[1], buf, 1), 1, "pipe_write");
126167
127− /* wait for result */
128− err = read(pipe_c2p[0], buf, 1);
168+ for (retry_count = 0;;) {
169+ /* For the remote test, the BPF program is triggered from this
170+ * process but the other process/thread is signaled.
171+ */
172+ if (remote) {
173+ if (!attr) {
174+ for (int i = 0; i < 10; i++)
175+ usleep(1);
176+ } else {
177+ for (int i = 0; i < 100000000; i++)
178+ j /= i + 1;
179+ }
180+ }
181+ /* wait for result */
182+ err = read_with_timeout(pipe_c2p[0], buf, 1, 100);
183+ if (err == -EAGAIN && retry_count++ < 10000)
184+ continue;
185+ break;
186+ }
129187 if (!ASSERT_GE(err, 0, "reading pipe"))
130188 goto disable_pmu;
131189 if (!ASSERT_GT(err, 0, "reading pipe error: size 0")) {
@@ -133,7 +191,7 @@ static void test_send_signal_common(struct perf_event_attr *attr, · 1 unmodified line
133191 goto disable_pmu;
134192 }
135193
136− ASSERT_EQ(buf[0], '2', "incorrect result");
194+ ASSERT_EQ(buf[0], 8, "incorrect result");
137195
138196 /* notify child safe to exit */
139197 ASSERT_EQ(write(pipe_p2c[1], buf, 1), 1, "pipe_write");
@@ -142,18 +200,21 @@ disable_pmu: · 2 unmodified lines
142200 close(pmu_fd);
143201 destroy_skel:
144202 test_send_signal_kern__destroy(skel);
203+ /* restore the old priority */
204+ if (remote)
205+ ASSERT_OK(setpriority(PRIO_PROCESS, 0, old_prio), "setpriority");
145206 skel_open_load_failure:
146207 close(pipe_c2p[0]);
147208 close(pipe_p2c[1]);
148209 wait(NULL);
149210 }
150211
151−static void test_send_signal_tracepoint(bool signal_thread)
212+static void test_send_signal_tracepoint(bool signal_thread, bool remote)
152213 {
153− test_send_signal_common(NULL, signal_thread);
214+ test_send_signal_common(NULL, signal_thread, remote);
154215 }
155216
156−static void test_send_signal_perf(bool signal_thread)
217+static void test_send_signal_perf(bool signal_thread, bool remote)
157218 {
158219 struct perf_event_attr attr = {
159220 .freq = 1,
@@ -162,13 +223,14 @@ static void test_send_signal_perf(bool signal_thread) · 2 unmodified lines
162223 .config = PERF_COUNT_SW_CPU_CLOCK,
163224 };
164225
165− test_send_signal_common(&attr, signal_thread);
226+ test_send_signal_common(&attr, signal_thread, remote);
166227 }
167228
168−static void test_send_signal_nmi(bool signal_thread)
229+static void test_send_signal_nmi(bool signal_thread, bool remote)
169230 {
170231 struct perf_event_attr attr = {
171− .sample_period = 1,
232+ .freq = 1,
233+ .sample_freq = 1000,
172234 .type = PERF_TYPE_HARDWARE,
173235 .config = PERF_COUNT_HW_CPU_CYCLES,
174236 };
@@ -191,21 +253,35 @@ static void test_send_signal_nmi(bool signal_thread) · 16 unmodified lines
191253 close(pmu_fd);
192254 }
193255
194− test_send_signal_common(&attr, signal_thread);
256+ test_send_signal_common(&attr, signal_thread, remote);
195257 }
196258
197259 void test_send_signal(void)
198260 {
199261 if (test__start_subtest("send_signal_tracepoint"))
200− test_send_signal_tracepoint(false);
262+ test_send_signal_tracepoint(false, false);
201263 if (test__start_subtest("send_signal_perf"))
202− test_send_signal_perf(false);
264+ test_send_signal_perf(false, false);
203265 if (test__start_subtest("send_signal_nmi"))
204− test_send_signal_nmi(false);
266+ test_send_signal_nmi(false, false);
205267 if (test__start_subtest("send_signal_tracepoint_thread"))
206− test_send_signal_tracepoint(true);
268+ test_send_signal_tracepoint(true, false);
207269 if (test__start_subtest("send_signal_perf_thread"))
208− test_send_signal_perf(true);
270+ test_send_signal_perf(true, false);
209271 if (test__start_subtest("send_signal_nmi_thread"))
210− test_send_signal_nmi(true);
272+ test_send_signal_nmi(true, false);
273+
274+ /* Signal remote thread and thread group */
275+ if (test__start_subtest("send_signal_tracepoint_remote"))
276+ test_send_signal_tracepoint(false, true);
277+ if (test__start_subtest("send_signal_perf_remote"))
278+ test_send_signal_perf(false, true);
279+ if (test__start_subtest("send_signal_nmi_remote"))
280+ test_send_signal_nmi(false, true);
281+ if (test__start_subtest("send_signal_tracepoint_thread_remote"))
282+ test_send_signal_tracepoint(true, true);
283+ if (test__start_subtest("send_signal_perf_thread_remote"))
284+ test_send_signal_perf(true, true);
285+ if (test__start_subtest("send_signal_nmi_thread_remote"))
286+ test_send_signal_nmi(true, true);
211287 }
tools/testing/selftests/bpf/prog_tests/sock_addr.c+0−4
@@ -23,10 +23,6 @@ · 22 unmodified lines
2323 #include "getpeername_unix_prog.skel.h"
2424 #include "network_helpers.h"
2525
26−#ifndef ENOTSUPP
27−# define ENOTSUPP 524
28−#endif
29−
3026 #define TEST_NS "sock_addr"
3127 #define TEST_IF_PREFIX "test_sock_addr"
3228 #define TEST_IPV4 "127.0.0.4"
tools/testing/selftests/bpf/prog_tests/sock_create.c+348−0
@@ -0,0 +1,348 @@
1+// SPDX-License-Identifier: GPL-2.0
2+#include <linux/bpf.h>
3+#include <test_progs.h>
4+#include "cgroup_helpers.h"
5+
6+static char bpf_log_buf[4096];
7+static bool verbose;
8+
9+enum sock_create_test_error {
10+ OK = 0,
11+ DENY_CREATE,
12+};
13+
14+static struct sock_create_test {
15+ const char *descr;
16+ const struct bpf_insn insns[64];
17+ enum bpf_attach_type attach_type;
18+ enum bpf_attach_type expected_attach_type;
19+
20+ int domain;
21+ int type;
22+ int protocol;
23+
24+ int optname;
25+ int optval;
26+ enum sock_create_test_error error;
27+} tests[] = {
28+ {
29+ .descr = "AF_INET set priority",
30+ .insns = {
31+ /* r3 = 123 (priority) */
32+ BPF_MOV64_IMM(BPF_REG_3, 123),
33+ BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_3,
34+ offsetof(struct bpf_sock, priority)),
35+
36+ /* return 1 */
37+ BPF_MOV64_IMM(BPF_REG_0, 1),
38+ BPF_EXIT_INSN(),
39+ },
40+ .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE,
41+ .attach_type = BPF_CGROUP_INET_SOCK_CREATE,
42+
43+ .domain = AF_INET,
44+ .type = SOCK_DGRAM,
45+
46+ .optname = SO_PRIORITY,
47+ .optval = 123,
48+ },
49+ {
50+ .descr = "AF_INET6 set priority",
51+ .insns = {
52+ /* r3 = 123 (priority) */
53+ BPF_MOV64_IMM(BPF_REG_3, 123),
54+ BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_3,
55+ offsetof(struct bpf_sock, priority)),
56+
57+ /* return 1 */
58+ BPF_MOV64_IMM(BPF_REG_0, 1),
59+ BPF_EXIT_INSN(),
60+ },
61+ .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE,
62+ .attach_type = BPF_CGROUP_INET_SOCK_CREATE,
63+
64+ .domain = AF_INET6,
65+ .type = SOCK_DGRAM,
66+
67+ .optname = SO_PRIORITY,
68+ .optval = 123,
69+ },
70+ {
71+ .descr = "AF_INET set mark",
72+ .insns = {
73+ BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
74+
75+ /* get uid of process */
76+ BPF_EMIT_CALL(BPF_FUNC_get_current_uid_gid),
77+ BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xffffffff),
78+
79+ /* if uid is 0, use given mark(666), else use uid as the mark */
80+ BPF_MOV64_REG(BPF_REG_3, BPF_REG_0),
81+ BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 1),
82+ BPF_MOV64_IMM(BPF_REG_3, 666),
83+
84+ BPF_MOV64_REG(BPF_REG_1, BPF_REG_6),
85+ BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_3,
86+ offsetof(struct bpf_sock, mark)),
87+
88+ /* return 1 */
89+ BPF_MOV64_IMM(BPF_REG_0, 1),
90+ BPF_EXIT_INSN(),
91+ },
92+ .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE,
93+ .attach_type = BPF_CGROUP_INET_SOCK_CREATE,
94+
95+ .domain = AF_INET,
96+ .type = SOCK_DGRAM,
97+
98+ .optname = SO_MARK,
99+ .optval = 666,
100+ },
101+ {
102+ .descr = "AF_INET6 set mark",
103+ .insns = {
104+ BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
105+
106+ /* get uid of process */
107+ BPF_EMIT_CALL(BPF_FUNC_get_current_uid_gid),
108+ BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xffffffff),
109+
110+ /* if uid is 0, use given mark(666), else use uid as the mark */
111+ BPF_MOV64_REG(BPF_REG_3, BPF_REG_0),
112+ BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 1),
113+ BPF_MOV64_IMM(BPF_REG_3, 666),
114+
115+ BPF_MOV64_REG(BPF_REG_1, BPF_REG_6),
116+ BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_3,
117+ offsetof(struct bpf_sock, mark)),
118+
119+ /* return 1 */
120+ BPF_MOV64_IMM(BPF_REG_0, 1),
121+ BPF_EXIT_INSN(),
122+ },
123+ .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE,
124+ .attach_type = BPF_CGROUP_INET_SOCK_CREATE,
125+
126+ .domain = AF_INET6,
127+ .type = SOCK_DGRAM,
128+
129+ .optname = SO_MARK,
130+ .optval = 666,
131+ },
132+ {
133+ .descr = "AF_INET bound to iface",
134+ .insns = {
135+ /* r3 = 1 (lo interface) */
136+ BPF_MOV64_IMM(BPF_REG_3, 1),
137+ BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_3,
138+ offsetof(struct bpf_sock, bound_dev_if)),
139+
140+ /* return 1 */
141+ BPF_MOV64_IMM(BPF_REG_0, 1),
142+ BPF_EXIT_INSN(),
143+ },
144+ .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE,
145+ .attach_type = BPF_CGROUP_INET_SOCK_CREATE,
146+
147+ .domain = AF_INET,
148+ .type = SOCK_DGRAM,
149+
150+ .optname = SO_BINDTOIFINDEX,
151+ .optval = 1,
152+ },
153+ {
154+ .descr = "AF_INET6 bound to iface",
155+ .insns = {
156+ /* r3 = 1 (lo interface) */
157+ BPF_MOV64_IMM(BPF_REG_3, 1),
158+ BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_3,
159+ offsetof(struct bpf_sock, bound_dev_if)),
160+
161+ /* return 1 */
162+ BPF_MOV64_IMM(BPF_REG_0, 1),
163+ BPF_EXIT_INSN(),
164+ },
165+ .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE,
166+ .attach_type = BPF_CGROUP_INET_SOCK_CREATE,
167+
168+ .domain = AF_INET6,
169+ .type = SOCK_DGRAM,
170+
171+ .optname = SO_BINDTOIFINDEX,
172+ .optval = 1,
173+ },
174+ {
175+ .descr = "block AF_INET, SOCK_DGRAM, IPPROTO_ICMP socket",
176+ .insns = {
177+ BPF_MOV64_IMM(BPF_REG_0, 1), /* r0 = verdict */
178+
179+ /* sock->family == AF_INET */
180+ BPF_LDX_MEM(BPF_H, BPF_REG_2, BPF_REG_1,
181+ offsetof(struct bpf_sock, family)),
182+ BPF_JMP_IMM(BPF_JNE, BPF_REG_2, AF_INET, 5),
183+
184+ /* sock->type == SOCK_DGRAM */
185+ BPF_LDX_MEM(BPF_H, BPF_REG_2, BPF_REG_1,
186+ offsetof(struct bpf_sock, type)),
187+ BPF_JMP_IMM(BPF_JNE, BPF_REG_2, SOCK_DGRAM, 3),
188+
189+ /* sock->protocol == IPPROTO_ICMP */
190+ BPF_LDX_MEM(BPF_H, BPF_REG_2, BPF_REG_1,
191+ offsetof(struct bpf_sock, protocol)),
192+ BPF_JMP_IMM(BPF_JNE, BPF_REG_2, IPPROTO_ICMP, 1),
193+
194+ /* return 0 (block) */
195+ BPF_MOV64_IMM(BPF_REG_0, 0),
196+ BPF_EXIT_INSN(),
197+ },
198+ .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE,
199+ .attach_type = BPF_CGROUP_INET_SOCK_CREATE,
200+
201+ .domain = AF_INET,
202+ .type = SOCK_DGRAM,
203+ .protocol = IPPROTO_ICMP,
204+
205+ .error = DENY_CREATE,
206+ },
207+ {
208+ .descr = "block AF_INET6, SOCK_DGRAM, IPPROTO_ICMPV6 socket",
209+ .insns = {
210+ BPF_MOV64_IMM(BPF_REG_0, 1), /* r0 = verdict */
211+
212+ /* sock->family == AF_INET6 */
213+ BPF_LDX_MEM(BPF_H, BPF_REG_2, BPF_REG_1,
214+ offsetof(struct bpf_sock, family)),
215+ BPF_JMP_IMM(BPF_JNE, BPF_REG_2, AF_INET6, 5),
216+
217+ /* sock->type == SOCK_DGRAM */
218+ BPF_LDX_MEM(BPF_H, BPF_REG_2, BPF_REG_1,
219+ offsetof(struct bpf_sock, type)),
220+ BPF_JMP_IMM(BPF_JNE, BPF_REG_2, SOCK_DGRAM, 3),
221+
222+ /* sock->protocol == IPPROTO_ICMPV6 */
223+ BPF_LDX_MEM(BPF_H, BPF_REG_2, BPF_REG_1,
224+ offsetof(struct bpf_sock, protocol)),
225+ BPF_JMP_IMM(BPF_JNE, BPF_REG_2, IPPROTO_ICMPV6, 1),
226+
227+ /* return 0 (block) */
228+ BPF_MOV64_IMM(BPF_REG_0, 0),
229+ BPF_EXIT_INSN(),
230+ },
231+ .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE,
232+ .attach_type = BPF_CGROUP_INET_SOCK_CREATE,
233+
234+ .domain = AF_INET,
235+ .type = SOCK_DGRAM,
236+ .protocol = IPPROTO_ICMPV6,
237+
238+ .error = DENY_CREATE,
239+ },
240+ {
241+ .descr = "load w/o expected_attach_type (compat mode)",
242+ .insns = {
243+ /* return 1 */
244+ BPF_MOV64_IMM(BPF_REG_0, 1),
245+ BPF_EXIT_INSN(),
246+ },
247+ .expected_attach_type = 0,
248+ .attach_type = BPF_CGROUP_INET_SOCK_CREATE,
249+
250+ .domain = AF_INET,
251+ .type = SOCK_STREAM,
252+ },
253+};
254+
255+static int load_prog(const struct bpf_insn *insns,
256+ enum bpf_attach_type expected_attach_type)
257+{
258+ LIBBPF_OPTS(bpf_prog_load_opts, opts,
259+ .expected_attach_type = expected_attach_type,
260+ .log_level = 2,
261+ .log_buf = bpf_log_buf,
262+ .log_size = sizeof(bpf_log_buf),
263+ );
264+ int fd, insns_cnt = 0;
265+
266+ for (;
267+ insns[insns_cnt].code != (BPF_JMP | BPF_EXIT);
268+ insns_cnt++) {
269+ }
270+ insns_cnt++;
271+
272+ fd = bpf_prog_load(BPF_PROG_TYPE_CGROUP_SOCK, NULL, "GPL", insns,
273+ insns_cnt, &opts);
274+ if (verbose && fd < 0)
275+ fprintf(stderr, "%s\n", bpf_log_buf);
276+
277+ return fd;
278+}
279+
280+static int run_test(int cgroup_fd, struct sock_create_test *test)
281+{
282+ int sock_fd, err, prog_fd, optval, ret = -1;
283+ socklen_t optlen = sizeof(optval);
284+
285+ prog_fd = load_prog(test->insns, test->expected_attach_type);
286+ if (prog_fd < 0) {
287+ log_err("Failed to load BPF program");
288+ return -1;
289+ }
290+
291+ err = bpf_prog_attach(prog_fd, cgroup_fd, test->attach_type, 0);
292+ if (err < 0) {
293+ log_err("Failed to attach BPF program");
294+ goto close_prog_fd;
295+ }
296+
297+ sock_fd = socket(test->domain, test->type, test->protocol);
298+ if (sock_fd < 0) {
299+ if (test->error == DENY_CREATE)
300+ ret = 0;
301+ else
302+ log_err("Failed to create socket");
303+
304+ goto detach_prog;
305+ }
306+
307+ if (test->optname) {
308+ err = getsockopt(sock_fd, SOL_SOCKET, test->optname, &optval, &optlen);
309+ if (err) {
310+ log_err("Failed to call getsockopt");
311+ goto cleanup;
312+ }
313+
314+ if (optval != test->optval) {
315+ errno = 0;
316+ log_err("getsockopt returned unexpected optval");
317+ goto cleanup;
318+ }
319+ }
320+
321+ ret = test->error != OK;
322+
323+cleanup:
324+ close(sock_fd);
325+detach_prog:
326+ bpf_prog_detach2(prog_fd, cgroup_fd, test->attach_type);
327+close_prog_fd:
328+ close(prog_fd);
329+ return ret;
330+}
331+
332+void test_sock_create(void)
333+{
334+ int cgroup_fd, i;
335+
336+ cgroup_fd = test__join_cgroup("/sock_create");
337+ if (!ASSERT_GE(cgroup_fd, 0, "join_cgroup"))
338+ return;
339+
340+ for (i = 0; i < ARRAY_SIZE(tests); i++) {
341+ if (!test__start_subtest(tests[i].descr))
342+ continue;
343+
344+ ASSERT_OK(run_test(cgroup_fd, &tests[i]), tests[i].descr);
345+ }
346+
347+ close(cgroup_fd);
348+}
tools/testing/selftests/bpf/prog_tests/sock_post_bind.crenamed from tools/testing/selftests/bpf/test_sock.c+62−192
@@ -1,132 +1,35 @@
11 // SPDX-License-Identifier: GPL-2.0
2−// Copyright (c) 2018 Facebook
3−
4−#include <stdio.h>
5−#include <unistd.h>
6−
7−#include <arpa/inet.h>
8−#include <sys/types.h>
9−#include <sys/socket.h>
10−
11−#include <linux/filter.h>
12−
13−#include <bpf/bpf.h>
14−
2+#include <linux/bpf.h>
3+#include <test_progs.h>
154 #include "cgroup_helpers.h"
16−#include <bpf/bpf_endian.h>
17−#include "bpf_util.h"
185
19−#define CG_PATH "/foo"
20−#define MAX_INSNS 512
6+#define TEST_NS "sock_post_bind"
217
22−char bpf_log_buf[BPF_LOG_BUF_SIZE];
23−static bool verbose = false;
8+static char bpf_log_buf[4096];
249
25−struct sock_test {
26− const char *descr;
10+static struct sock_post_bind_test {
11+ const char *descr;
2712 /* BPF prog properties */
28− struct bpf_insn insns[MAX_INSNS];
29− enum bpf_attach_type expected_attach_type;
30− enum bpf_attach_type attach_type;
13+ const struct bpf_insn insns[64];
14+ enum bpf_attach_type attach_type;
15+ enum bpf_attach_type expected_attach_type;
3116 /* Socket properties */
32− int domain;
33− int type;
17+ int domain;
18+ int type;
3419 /* Endpoint to bind() to */
3520 const char *ip;
3621 unsigned short port;
3722 unsigned short port_retry;
23+
3824 /* Expected test result */
3925 enum {
40− LOAD_REJECT,
4126 ATTACH_REJECT,
4227 BIND_REJECT,
4328 SUCCESS,
4429 RETRY_SUCCESS,
4530 RETRY_REJECT
4631 } result;
47−};
48−
49−static struct sock_test tests[] = {
50− {
51− .descr = "bind4 load with invalid access: src_ip6",
52− .insns = {
53− BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
54− BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_6,
55− offsetof(struct bpf_sock, src_ip6[0])),
56− BPF_MOV64_IMM(BPF_REG_0, 1),
57− BPF_EXIT_INSN(),
58− },
59− .expected_attach_type = BPF_CGROUP_INET4_POST_BIND,
60− .attach_type = BPF_CGROUP_INET4_POST_BIND,
61− .result = LOAD_REJECT,
62− },
63− {
64− .descr = "bind4 load with invalid access: mark",
65− .insns = {
66− BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
67− BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_6,
68− offsetof(struct bpf_sock, mark)),
69− BPF_MOV64_IMM(BPF_REG_0, 1),
70− BPF_EXIT_INSN(),
71− },
72− .expected_attach_type = BPF_CGROUP_INET4_POST_BIND,
73− .attach_type = BPF_CGROUP_INET4_POST_BIND,
74− .result = LOAD_REJECT,
75− },
76− {
77− .descr = "bind6 load with invalid access: src_ip4",
78− .insns = {
79− BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
80− BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_6,
81− offsetof(struct bpf_sock, src_ip4)),
82− BPF_MOV64_IMM(BPF_REG_0, 1),
83− BPF_EXIT_INSN(),
84− },
85− .expected_attach_type = BPF_CGROUP_INET6_POST_BIND,
86− .attach_type = BPF_CGROUP_INET6_POST_BIND,
87− .result = LOAD_REJECT,
88− },
89− {
90− .descr = "sock_create load with invalid access: src_port",
91− .insns = {
92− BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
93− BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_6,
94− offsetof(struct bpf_sock, src_port)),
95− BPF_MOV64_IMM(BPF_REG_0, 1),
96− BPF_EXIT_INSN(),
97− },
98− .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE,
99− .attach_type = BPF_CGROUP_INET_SOCK_CREATE,
100− .result = LOAD_REJECT,
101− },
102− {
103− .descr = "sock_create load w/o expected_attach_type (compat mode)",
104− .insns = {
105− BPF_MOV64_IMM(BPF_REG_0, 1),
106− BPF_EXIT_INSN(),
107− },
108− .expected_attach_type = 0,
109− .attach_type = BPF_CGROUP_INET_SOCK_CREATE,
110− .domain = AF_INET,
111− .type = SOCK_STREAM,
112− .ip = "127.0.0.1",
113− .port = 8097,
114− .result = SUCCESS,
115− },
116− {
117− .descr = "sock_create load w/ expected_attach_type",
118− .insns = {
119− BPF_MOV64_IMM(BPF_REG_0, 1),
120− BPF_EXIT_INSN(),
121− },
122− .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE,
123− .attach_type = BPF_CGROUP_INET_SOCK_CREATE,
124− .domain = AF_INET,
125− .type = SOCK_STREAM,
126− .ip = "127.0.0.1",
127− .port = 8097,
128− .result = SUCCESS,
129− },
32+} tests[] = {
13033 {
13134 .descr = "attach type mismatch bind4 vs bind6",
13235 .insns = {
@@ -374,40 +277,29 @@ static struct sock_test tests[] = { · 241 unmodified lines
374277 },
375278 };
376279
377−static size_t probe_prog_length(const struct bpf_insn *fp)
378−{
379− size_t len;
380−
381− for (len = MAX_INSNS - 1; len > 0; --len)
382− if (fp[len].code != 0 || fp[len].imm != 0)
383− break;
384− return len + 1;
385−}
386−
387−static int load_sock_prog(const struct bpf_insn *prog,
388− enum bpf_attach_type attach_type)
280+static int load_prog(const struct bpf_insn *insns,
281+ enum bpf_attach_type expected_attach_type)
389282 {
390− LIBBPF_OPTS(bpf_prog_load_opts, opts);
391− int ret, insn_cnt;
392−
393− insn_cnt = probe_prog_length(prog);
394−
395− opts.expected_attach_type = attach_type;
396− opts.log_buf = bpf_log_buf;
397− opts.log_size = BPF_LOG_BUF_SIZE;
398− opts.log_level = 2;
283+ LIBBPF_OPTS(bpf_prog_load_opts, opts,
284+ .expected_attach_type = expected_attach_type,
285+ .log_level = 2,
286+ .log_buf = bpf_log_buf,
287+ .log_size = sizeof(bpf_log_buf),
288+ );
289+ int fd, insns_cnt = 0;
290+
291+ for (;
292+ insns[insns_cnt].code != (BPF_JMP | BPF_EXIT);
293+ insns_cnt++) {
294+ }
295+ insns_cnt++;
399296
400− ret = bpf_prog_load(BPF_PROG_TYPE_CGROUP_SOCK, NULL, "GPL", prog, insn_cnt, &opts);
401− if (verbose && ret < 0)
297+ fd = bpf_prog_load(BPF_PROG_TYPE_CGROUP_SOCK, NULL, "GPL", insns,
298+ insns_cnt, &opts);
299+ if (fd < 0)
402300 fprintf(stderr, "%s\n", bpf_log_buf);
403301
404− return ret;
405−}
406−
407−static int attach_sock_prog(int cgfd, int progfd,
408− enum bpf_attach_type attach_type)
409−{
410− return bpf_prog_attach(progfd, cgfd, attach_type, BPF_F_ALLOW_OVERRIDE);
302+ return fd;
411303 }
412304
413305 static int bind_sock(int domain, int type, const char *ip,
@@ -477,22 +369,16 @@ out: · 63 unmodified lines
477369 return res;
478370 }
479371
480−static int run_test_case(int cgfd, const struct sock_test *test)
372+static int run_test(int cgroup_fd, struct sock_post_bind_test *test)
481373 {
482− int progfd = -1;
483− int err = 0;
484− int res;
485−
486− printf("Test case: %s .. ", test->descr);
487− progfd = load_sock_prog(test->insns, test->expected_attach_type);
488− if (progfd < 0) {
489− if (test->result == LOAD_REJECT)
490− goto out;
491− else
492− goto err;
493− }
374+ int err, prog_fd, res, ret = 0;
494375
495− if (attach_sock_prog(cgfd, progfd, test->attach_type) < 0) {
376+ prog_fd = load_prog(test->insns, test->expected_attach_type);
377+ if (prog_fd < 0)
378+ goto err;
379+
380+ err = bpf_prog_attach(prog_fd, cgroup_fd, test->attach_type, 0);
381+ if (err < 0) {
496382 if (test->result == ATTACH_REJECT)
497383 goto out;
498384 else
@@ -503,54 +389,38 @@ static int run_test_case(int cgfd, const struct sock_test *test) · 4 unmodified lines
503389 test->port_retry);
504390 if (res > 0 && test->result == res)
505391 goto out;
506−
507392 err:
508− err = -1;
393+ ret = -1;
509394 out:
510395 /* Detaching w/o checking return code: best effort attempt. */
511− if (progfd != -1)
512− bpf_prog_detach(cgfd, test->attach_type);
513− close(progfd);
514− printf("[%s]\n", err ? "FAIL" : "PASS");
515− return err;
396+ if (prog_fd != -1)
397+ bpf_prog_detach(cgroup_fd, test->attach_type);
398+ close(prog_fd);
399+ return ret;
516400 }
517401
518−static int run_tests(int cgfd)
402+void test_sock_post_bind(void)
519403 {
520− int passes = 0;
521− int fails = 0;
404+ struct netns_obj *ns;
405+ int cgroup_fd;
522406 int i;
523407
524− for (i = 0; i < ARRAY_SIZE(tests); ++i) {
525− if (run_test_case(cgfd, &tests[i]))
526− ++fails;
527− else
528− ++passes;
529− }
530− printf("Summary: %d PASSED, %d FAILED\n", passes, fails);
531− return fails ? -1 : 0;
532−}
533−
534−int main(int argc, char **argv)
535−{
536− int cgfd = -1;
537− int err = 0;
408+ cgroup_fd = test__join_cgroup("/post_bind");
409+ if (!ASSERT_OK_FD(cgroup_fd, "join_cgroup"))
410+ return;
538411
539− cgfd = cgroup_setup_and_join(CG_PATH);
540− if (cgfd < 0)
541− goto err;
412+ ns = netns_new(TEST_NS, true);
413+ if (!ASSERT_OK_PTR(ns, "netns_new"))
414+ goto cleanup;
542415
543− /* Use libbpf 1.0 API mode */
544− libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
416+ for (i = 0; i < ARRAY_SIZE(tests); i++) {
417+ if (!test__start_subtest(tests[i].descr))
418+ continue;
545419
546− if (run_tests(cgfd))
547− goto err;
420+ ASSERT_OK(run_test(cgroup_fd, &tests[i]), tests[i].descr);
421+ }
548422
549− goto out;
550−err:
551− err = -1;
552−out:
553− close(cgfd);
554− cleanup_cgroup_environment();
555− return err;
423+cleanup:
424+ netns_free(ns);
425+ close(cgroup_fd);
556426 }
tools/testing/selftests/bpf/prog_tests/struct_ops_private_stack.c+106−0
@@ -0,0 +1,106 @@
1+// SPDX-License-Identifier: GPL-2.0
2+
3+#include <test_progs.h>
4+#include "struct_ops_private_stack.skel.h"
5+#include "struct_ops_private_stack_fail.skel.h"
6+#include "struct_ops_private_stack_recur.skel.h"
7+
8+static void test_private_stack(void)
9+{
10+ struct struct_ops_private_stack *skel;
11+ struct bpf_link *link;
12+ int err;
13+
14+ skel = struct_ops_private_stack__open();
15+ if (!ASSERT_OK_PTR(skel, "struct_ops_private_stack__open"))
16+ return;
17+
18+ if (skel->data->skip) {
19+ test__skip();
20+ goto cleanup;
21+ }
22+
23+ err = struct_ops_private_stack__load(skel);
24+ if (!ASSERT_OK(err, "struct_ops_private_stack__load"))
25+ goto cleanup;
26+
27+ link = bpf_map__attach_struct_ops(skel->maps.testmod_1);
28+ if (!ASSERT_OK_PTR(link, "attach_struct_ops"))
29+ goto cleanup;
30+
31+ ASSERT_OK(trigger_module_test_read(256), "trigger_read");
32+
33+ ASSERT_EQ(skel->bss->val_i, 3, "val_i");
34+ ASSERT_EQ(skel->bss->val_j, 8, "val_j");
35+
36+ bpf_link__destroy(link);
37+
38+cleanup:
39+ struct_ops_private_stack__destroy(skel);
40+}
41+
42+static void test_private_stack_fail(void)
43+{
44+ struct struct_ops_private_stack_fail *skel;
45+ int err;
46+
47+ skel = struct_ops_private_stack_fail__open();
48+ if (!ASSERT_OK_PTR(skel, "struct_ops_private_stack_fail__open"))
49+ return;
50+
51+ if (skel->data->skip) {
52+ test__skip();
53+ goto cleanup;
54+ }
55+
56+ err = struct_ops_private_stack_fail__load(skel);
57+ if (!ASSERT_ERR(err, "struct_ops_private_stack_fail__load"))
58+ goto cleanup;
59+ return;
60+
61+cleanup:
62+ struct_ops_private_stack_fail__destroy(skel);
63+}
64+
65+static void test_private_stack_recur(void)
66+{
67+ struct struct_ops_private_stack_recur *skel;
68+ struct bpf_link *link;
69+ int err;
70+
71+ skel = struct_ops_private_stack_recur__open();
72+ if (!ASSERT_OK_PTR(skel, "struct_ops_private_stack_recur__open"))
73+ return;
74+
75+ if (skel->data->skip) {
76+ test__skip();
77+ goto cleanup;
78+ }
79+
80+ err = struct_ops_private_stack_recur__load(skel);
81+ if (!ASSERT_OK(err, "struct_ops_private_stack_recur__load"))
82+ goto cleanup;
83+
84+ link = bpf_map__attach_struct_ops(skel->maps.testmod_1);
85+ if (!ASSERT_OK_PTR(link, "attach_struct_ops"))
86+ goto cleanup;
87+
88+ ASSERT_OK(trigger_module_test_read(256), "trigger_read");
89+
90+ ASSERT_EQ(skel->bss->val_j, 3, "val_j");
91+
92+ bpf_link__destroy(link);
93+
94+cleanup:
95+ struct_ops_private_stack_recur__destroy(skel);
96+}
97+
98+void test_struct_ops_private_stack(void)
99+{
100+ if (test__start_subtest("private_stack"))
101+ test_private_stack();
102+ if (test__start_subtest("private_stack_fail"))
103+ test_private_stack_fail();
104+ if (test__start_subtest("private_stack_recur"))
105+ test_private_stack_recur();
106+}
tools/testing/selftests/bpf/prog_tests/subskeleton.c+75−1
@@ -46,7 +46,8 @@ static int subskeleton_lib_subresult(struct bpf_object *obj) · 45 unmodified lines
4646 return result;
4747 }
4848
49−void test_subskeleton(void)
49+/* initialize and load through skeleton, then instantiate subskeleton out of it */
50+static void subtest_skel_subskeleton(void)
5051 {
5152 int err, result;
5253 struct test_subskeleton *skel;
@@ -76,3 +77,76 @@ void test_subskeleton(void) · 23 unmodified lines
7677 cleanup:
7778 test_subskeleton__destroy(skel);
7879 }
80+
81+/* initialize and load through generic bpf_object API, then instantiate subskeleton out of it */
82+static void subtest_obj_subskeleton(void)
83+{
84+ int err, result;
85+ const void *elf_bytes;
86+ size_t elf_bytes_sz = 0, rodata_sz = 0, bss_sz = 0;
87+ struct bpf_object *obj;
88+ const struct bpf_map *map;
89+ const struct bpf_program *prog;
90+ struct bpf_link *link = NULL;
91+ struct test_subskeleton__rodata *rodata;
92+ struct test_subskeleton__bss *bss;
93+
94+ elf_bytes = test_subskeleton__elf_bytes(&elf_bytes_sz);
95+ if (!ASSERT_OK_PTR(elf_bytes, "elf_bytes"))
96+ return;
97+
98+ obj = bpf_object__open_mem(elf_bytes, elf_bytes_sz, NULL);
99+ if (!ASSERT_OK_PTR(obj, "obj_open_mem"))
100+ return;
101+
102+ map = bpf_object__find_map_by_name(obj, ".rodata");
103+ if (!ASSERT_OK_PTR(map, "rodata_map_by_name"))
104+ goto cleanup;
105+
106+ rodata = bpf_map__initial_value(map, &rodata_sz);
107+ if (!ASSERT_OK_PTR(rodata, "rodata_get"))
108+ goto cleanup;
109+
110+ rodata->rovar1 = 10;
111+ rodata->var1 = 1;
112+ subskeleton_lib_setup(obj);
113+
114+ err = bpf_object__load(obj);
115+ if (!ASSERT_OK(err, "obj_load"))
116+ goto cleanup;
117+
118+ prog = bpf_object__find_program_by_name(obj, "handler1");
119+ if (!ASSERT_OK_PTR(prog, "prog_by_name"))
120+ goto cleanup;
121+
122+ link = bpf_program__attach(prog);
123+ if (!ASSERT_OK_PTR(link, "prog_attach"))
124+ goto cleanup;
125+
126+ /* trigger tracepoint */
127+ usleep(1);
128+
129+ map = bpf_object__find_map_by_name(obj, ".bss");
130+ if (!ASSERT_OK_PTR(map, "bss_map_by_name"))
131+ goto cleanup;
132+
133+ bss = bpf_map__initial_value(map, &bss_sz);
134+ if (!ASSERT_OK_PTR(rodata, "rodata_get"))
135+ goto cleanup;
136+
137+ result = subskeleton_lib_subresult(obj) * 10;
138+ ASSERT_EQ(bss->out1, result, "out1");
139+
140+cleanup:
141+ bpf_link__destroy(link);
142+ bpf_object__close(obj);
143+}
144+
145+
146+void test_subskeleton(void)
147+{
148+ if (test__start_subtest("skel_subskel"))
149+ subtest_skel_subskeleton();
150+ if (test__start_subtest("obj_subskel"))
151+ subtest_obj_subskeleton();
152+}
tools/testing/selftests/bpf/prog_tests/tailcalls.c+114−14
@@ -7,6 +7,7 @@ · 6 unmodified lines
77 #include "tailcall_bpf2bpf_hierarchy3.skel.h"
88 #include "tailcall_freplace.skel.h"
99 #include "tc_bpf2bpf.skel.h"
10+#include "tailcall_fail.skel.h"
1011
1112 /* test_tailcall_1 checks basic functionality by patching multiple locations
1213 * in a single program for a single tail call slot with nop->jmp, jmp->nop
@@ -1496,8 +1497,8 @@ static void test_tailcall_bpf2bpf_hierarchy_3(void) · 1483 unmodified lines
14961497 RUN_TESTS(tailcall_bpf2bpf_hierarchy3);
14971498 }
14981499
1499−/* test_tailcall_freplace checks that the attached freplace prog is OK to
1500− * update the prog_array map.
1500+/* test_tailcall_freplace checks that the freplace prog fails to update the
1501+ * prog_array map, no matter whether the freplace prog attaches to its target.
15011502 */
15021503 static void test_tailcall_freplace(void)
15031504 {
@@ -1505,7 +1506,7 @@ static void test_tailcall_freplace(void) · 1 unmodified line
15051506 struct bpf_link *freplace_link = NULL;
15061507 struct bpf_program *freplace_prog;
15071508 struct tc_bpf2bpf *tc_skel = NULL;
1508− int prog_fd, map_fd;
1509+ int prog_fd, tc_prog_fd, map_fd;
15091510 char buff[128] = {};
15101511 int err, key;
15111512
@@ -1523,9 +1524,10 @@ static void test_tailcall_freplace(void) · 11 unmodified lines
15231524 if (!ASSERT_OK_PTR(tc_skel, "tc_bpf2bpf__open_and_load"))
15241525 goto out;
15251526
1526− prog_fd = bpf_program__fd(tc_skel->progs.entry_tc);
1527+ tc_prog_fd = bpf_program__fd(tc_skel->progs.entry_tc);
15271528 freplace_prog = freplace_skel->progs.entry_freplace;
1528− err = bpf_program__set_attach_target(freplace_prog, prog_fd, "subprog");
1529+ err = bpf_program__set_attach_target(freplace_prog, tc_prog_fd,
1530+ "subprog_tc");
15291531 if (!ASSERT_OK(err, "set_attach_target"))
15301532 goto out;
15311533
@@ -1533,27 +1535,121 @@ static void test_tailcall_freplace(void) · 1 unmodified line
15331535 if (!ASSERT_OK(err, "tailcall_freplace__load"))
15341536 goto out;
15351537
1536− freplace_link = bpf_program__attach_freplace(freplace_prog, prog_fd,
1537− "subprog");
1538+ map_fd = bpf_map__fd(freplace_skel->maps.jmp_table);
1539+ prog_fd = bpf_program__fd(freplace_prog);
1540+ key = 0;
1541+ err = bpf_map_update_elem(map_fd, &key, &prog_fd, BPF_ANY);
1542+ ASSERT_ERR(err, "update jmp_table failure");
1543+
1544+ freplace_link = bpf_program__attach_freplace(freplace_prog, tc_prog_fd,
1545+ "subprog_tc");
15381546 if (!ASSERT_OK_PTR(freplace_link, "attach_freplace"))
15391547 goto out;
15401548
1541− map_fd = bpf_map__fd(freplace_skel->maps.jmp_table);
1542− prog_fd = bpf_program__fd(freplace_prog);
1549+ err = bpf_map_update_elem(map_fd, &key, &prog_fd, BPF_ANY);
1550+ ASSERT_ERR(err, "update jmp_table failure");
1551+
1552+out:
1553+ bpf_link__destroy(freplace_link);
1554+ tailcall_freplace__destroy(freplace_skel);
1555+ tc_bpf2bpf__destroy(tc_skel);
1556+}
1557+
1558+/* test_tailcall_bpf2bpf_freplace checks the failure that fails to attach a tail
1559+ * callee prog with freplace prog or fails to update an extended prog to
1560+ * prog_array map.
1561+ */
1562+static void test_tailcall_bpf2bpf_freplace(void)
1563+{
1564+ struct tailcall_freplace *freplace_skel = NULL;
1565+ struct bpf_link *freplace_link = NULL;
1566+ struct tc_bpf2bpf *tc_skel = NULL;
1567+ char buff[128] = {};
1568+ int prog_fd, map_fd;
1569+ int err, key;
1570+
1571+ LIBBPF_OPTS(bpf_test_run_opts, topts,
1572+ .data_in = buff,
1573+ .data_size_in = sizeof(buff),
1574+ .repeat = 1,
1575+ );
1576+
1577+ tc_skel = tc_bpf2bpf__open_and_load();
1578+ if (!ASSERT_OK_PTR(tc_skel, "tc_bpf2bpf__open_and_load"))
1579+ goto out;
1580+
1581+ prog_fd = bpf_program__fd(tc_skel->progs.entry_tc);
1582+ freplace_skel = tailcall_freplace__open();
1583+ if (!ASSERT_OK_PTR(freplace_skel, "tailcall_freplace__open"))
1584+ goto out;
1585+
1586+ err = bpf_program__set_attach_target(freplace_skel->progs.entry_freplace,
1587+ prog_fd, "subprog_tc");
1588+ if (!ASSERT_OK(err, "set_attach_target"))
1589+ goto out;
1590+
1591+ err = tailcall_freplace__load(freplace_skel);
1592+ if (!ASSERT_OK(err, "tailcall_freplace__load"))
1593+ goto out;
1594+
1595+ /* OK to attach then detach freplace prog. */
1596+
1597+ freplace_link = bpf_program__attach_freplace(freplace_skel->progs.entry_freplace,
1598+ prog_fd, "subprog_tc");
1599+ if (!ASSERT_OK_PTR(freplace_link, "attach_freplace"))
1600+ goto out;
1601+
1602+ err = bpf_link__destroy(freplace_link);
1603+ if (!ASSERT_OK(err, "destroy link"))
1604+ goto out;
1605+
1606+ /* OK to update prog_array map then delete element from the map. */
1607+
15431608 key = 0;
1609+ map_fd = bpf_map__fd(freplace_skel->maps.jmp_table);
15441610 err = bpf_map_update_elem(map_fd, &key, &prog_fd, BPF_ANY);
15451611 if (!ASSERT_OK(err, "update jmp_table"))
15461612 goto out;
15471613
1548− prog_fd = bpf_program__fd(tc_skel->progs.entry_tc);
1549− err = bpf_prog_test_run_opts(prog_fd, &topts);
1550− ASSERT_OK(err, "test_run");
1551− ASSERT_EQ(topts.retval, 34, "test_run retval");
1614+ err = bpf_map_delete_elem(map_fd, &key);
1615+ if (!ASSERT_OK(err, "delete_elem from jmp_table"))
1616+ goto out;
1617+
1618+ /* Fail to attach a tail callee prog with freplace prog. */
1619+
1620+ err = bpf_map_update_elem(map_fd, &key, &prog_fd, BPF_ANY);
1621+ if (!ASSERT_OK(err, "update jmp_table"))
1622+ goto out;
1623+
1624+ freplace_link = bpf_program__attach_freplace(freplace_skel->progs.entry_freplace,
1625+ prog_fd, "subprog_tc");
1626+ if (!ASSERT_ERR_PTR(freplace_link, "attach_freplace failure"))
1627+ goto out;
1628+
1629+ err = bpf_map_delete_elem(map_fd, &key);
1630+ if (!ASSERT_OK(err, "delete_elem from jmp_table"))
1631+ goto out;
1632+
1633+ /* Fail to update an extended prog to prog_array map. */
1634+
1635+ freplace_link = bpf_program__attach_freplace(freplace_skel->progs.entry_freplace,
1636+ prog_fd, "subprog_tc");
1637+ if (!ASSERT_OK_PTR(freplace_link, "attach_freplace"))
1638+ goto out;
1639+
1640+ err = bpf_map_update_elem(map_fd, &key, &prog_fd, BPF_ANY);
1641+ if (!ASSERT_ERR(err, "update jmp_table failure"))
1642+ goto out;
15521643
15531644 out:
15541645 bpf_link__destroy(freplace_link);
1555− tc_bpf2bpf__destroy(tc_skel);
15561646 tailcall_freplace__destroy(freplace_skel);
1647+ tc_bpf2bpf__destroy(tc_skel);
1648+}
1649+
1650+static void test_tailcall_failure()
1651+{
1652+ RUN_TESTS(tailcall_fail);
15571653 }
15581654
15591655 void test_tailcalls(void)
@@ -1606,4 +1702,8 @@ void test_tailcalls(void) · 46 unmodified lines
16061702 test_tailcall_bpf2bpf_hierarchy_3();
16071703 if (test__start_subtest("tailcall_freplace"))
16081704 test_tailcall_freplace();
1705+ if (test__start_subtest("tailcall_bpf2bpf_freplace"))
1706+ test_tailcall_bpf2bpf_freplace();
1707+ if (test__start_subtest("tailcall_failure"))
1708+ test_tailcall_failure();
16091709 }
tools/testing/selftests/bpf/prog_tests/task_kfunc.c+80−0
@@ -68,6 +68,74 @@ cleanup: · 67 unmodified lines
6868 task_kfunc_success__destroy(skel);
6969 }
7070
71+static int run_vpid_test(void *prog_name)
72+{
73+ struct task_kfunc_success *skel;
74+ struct bpf_program *prog;
75+ int prog_fd, err = 0;
76+
77+ if (getpid() != 1)
78+ return 1;
79+
80+ skel = open_load_task_kfunc_skel();
81+ if (!skel)
82+ return 2;
83+
84+ if (skel->bss->err) {
85+ err = 3;
86+ goto cleanup;
87+ }
88+
89+ prog = bpf_object__find_program_by_name(skel->obj, prog_name);
90+ if (!prog) {
91+ err = 4;
92+ goto cleanup;
93+ }
94+
95+ prog_fd = bpf_program__fd(prog);
96+ if (prog_fd < 0) {
97+ err = 5;
98+ goto cleanup;
99+ }
100+
101+ if (bpf_prog_test_run_opts(prog_fd, NULL)) {
102+ err = 6;
103+ goto cleanup;
104+ }
105+
106+ if (skel->bss->err)
107+ err = 7 + skel->bss->err;
108+cleanup:
109+ task_kfunc_success__destroy(skel);
110+ return err;
111+}
112+
113+static void run_vpid_success_test(const char *prog_name)
114+{
115+ const int stack_size = 1024 * 1024;
116+ int child_pid, wstatus;
117+ char *stack;
118+
119+ stack = (char *)malloc(stack_size);
120+ if (!ASSERT_OK_PTR(stack, "clone_stack"))
121+ return;
122+
123+ child_pid = clone(run_vpid_test, stack + stack_size,
124+ CLONE_NEWPID | SIGCHLD, (void *)prog_name);
125+ if (!ASSERT_GT(child_pid, -1, "child_pid"))
126+ goto cleanup;
127+
128+ if (!ASSERT_GT(waitpid(child_pid, &wstatus, 0), -1, "waitpid"))
129+ goto cleanup;
130+
131+ if (WEXITSTATUS(wstatus) > 7)
132+ ASSERT_OK(WEXITSTATUS(wstatus) - 7, "vpid_test_failure");
133+ else
134+ ASSERT_OK(WEXITSTATUS(wstatus), "run_vpid_test_err");
135+cleanup:
136+ free(stack);
137+}
138+
71139 static const char * const success_tests[] = {
72140 "test_task_acquire_release_argument",
73141 "test_task_acquire_release_current",
@@ -83,6 +151,11 @@ static const char * const success_tests[] = { · 9 unmodified lines
83151 "test_task_kfunc_flavor_relo_not_found",
84152 };
85153
154+static const char * const vpid_success_tests[] = {
155+ "test_task_from_vpid_current",
156+ "test_task_from_vpid_invalid",
157+};
158+
86159 void test_task_kfunc(void)
87160 {
88161 int i;
@@ -94,5 +167,12 @@ void test_task_kfunc(void) · 5 unmodified lines
94167 run_success_test(success_tests[i]);
95168 }
96169
170+ for (i = 0; i < ARRAY_SIZE(vpid_success_tests); i++) {
171+ if (!test__start_subtest(vpid_success_tests[i]))
172+ continue;
173+
174+ run_vpid_success_test(vpid_success_tests[i]);
175+ }
176+
97177 RUN_TESTS(task_kfunc_failure);
98178 }
tools/testing/selftests/bpf/prog_tests/task_local_storage.c+282−4
@@ -7,12 +7,20 @@ · 6 unmodified lines
77 #include <pthread.h>
88 #include <sys/syscall.h> /* For SYS_xxx definitions */
99 #include <sys/types.h>
10+#include <sys/eventfd.h>
11+#include <sys/mman.h>
1012 #include <test_progs.h>
13+#include <bpf/btf.h>
1114 #include "task_local_storage_helpers.h"
1215 #include "task_local_storage.skel.h"
1316 #include "task_local_storage_exit_creds.skel.h"
1417 #include "task_ls_recursion.skel.h"
1518 #include "task_storage_nodeadlock.skel.h"
19+#include "uptr_test_common.h"
20+#include "task_ls_uptr.skel.h"
21+#include "uptr_update_failure.skel.h"
22+#include "uptr_failure.skel.h"
23+#include "uptr_map_failure.skel.h"
1624
1725 static void test_sys_enter_exit(void)
1826 {
@@ -23,14 +31,14 @@ static void test_sys_enter_exit(void) · 4 unmodified lines
2331 if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))
2432 return;
2533
26− skel->bss->target_pid = syscall(SYS_gettid);
34+ skel->bss->target_pid = sys_gettid();
2735
2836 err = task_local_storage__attach(skel);
2937 if (!ASSERT_OK(err, "skel_attach"))
3038 goto out;
3139
32− syscall(SYS_gettid);
33− syscall(SYS_gettid);
40+ sys_gettid();
41+ sys_gettid();
3442
3543 /* 3x syscalls: 1x attach and 2x gettid */
3644 ASSERT_EQ(skel->bss->enter_cnt, 3, "enter_cnt");
@@ -99,7 +107,7 @@ static void test_recursion(void) · 62 unmodified lines
99107
100108 /* trigger sys_enter, make sure it does not cause deadlock */
101109 skel->bss->test_pid = getpid();
102− syscall(SYS_gettid);
110+ sys_gettid();
103111 skel->bss->test_pid = 0;
104112 task_ls_recursion__detach(skel);
105113
@@ -227,6 +235,259 @@ done: · 121 unmodified lines
227235 sched_setaffinity(getpid(), sizeof(old), &old);
228236 }
229237
238+static struct user_data udata __attribute__((aligned(16))) = {
239+ .a = 1,
240+ .b = 2,
241+};
242+
243+static struct user_data udata2 __attribute__((aligned(16))) = {
244+ .a = 3,
245+ .b = 4,
246+};
247+
248+static void check_udata2(int expected)
249+{
250+ udata2.result = udata2.nested_result = 0;
251+ usleep(1);
252+ ASSERT_EQ(udata2.result, expected, "udata2.result");
253+ ASSERT_EQ(udata2.nested_result, expected, "udata2.nested_result");
254+}
255+
256+static void test_uptr_basic(void)
257+{
258+ int map_fd, parent_task_fd, ev_fd;
259+ struct value_type value = {};
260+ struct task_ls_uptr *skel;
261+ pid_t child_pid, my_tid;
262+ __u64 ev_dummy_data = 1;
263+ int err;
264+
265+ my_tid = sys_gettid();
266+ parent_task_fd = sys_pidfd_open(my_tid, 0);
267+ if (!ASSERT_OK_FD(parent_task_fd, "parent_task_fd"))
268+ return;
269+
270+ ev_fd = eventfd(0, 0);
271+ if (!ASSERT_OK_FD(ev_fd, "ev_fd")) {
272+ close(parent_task_fd);
273+ return;
274+ }
275+
276+ skel = task_ls_uptr__open_and_load();
277+ if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))
278+ goto out;
279+
280+ map_fd = bpf_map__fd(skel->maps.datamap);
281+ value.udata = &udata;
282+ value.nested.udata = &udata;
283+ err = bpf_map_update_elem(map_fd, &parent_task_fd, &value, BPF_NOEXIST);
284+ if (!ASSERT_OK(err, "update_elem(udata)"))
285+ goto out;
286+
287+ err = task_ls_uptr__attach(skel);
288+ if (!ASSERT_OK(err, "skel_attach"))
289+ goto out;
290+
291+ child_pid = fork();
292+ if (!ASSERT_NEQ(child_pid, -1, "fork"))
293+ goto out;
294+
295+ /* Call syscall in the child process, but access the map value of
296+ * the parent process in the BPF program to check if the user kptr
297+ * is translated/mapped correctly.
298+ */
299+ if (child_pid == 0) {
300+ /* child */
301+
302+ /* Overwrite the user_data in the child process to check if
303+ * the BPF program accesses the user_data of the parent.
304+ */
305+ udata.a = 0;
306+ udata.b = 0;
307+
308+ /* Wait for the parent to set child_pid */
309+ read(ev_fd, &ev_dummy_data, sizeof(ev_dummy_data));
310+ exit(0);
311+ }
312+
313+ skel->bss->parent_pid = my_tid;
314+ skel->bss->target_pid = child_pid;
315+
316+ write(ev_fd, &ev_dummy_data, sizeof(ev_dummy_data));
317+
318+ err = waitpid(child_pid, NULL, 0);
319+ ASSERT_EQ(err, child_pid, "waitpid");
320+ ASSERT_EQ(udata.result, MAGIC_VALUE + udata.a + udata.b, "udata.result");
321+ ASSERT_EQ(udata.nested_result, MAGIC_VALUE + udata.a + udata.b, "udata.nested_result");
322+
323+ skel->bss->target_pid = my_tid;
324+
325+ /* update_elem: uptr changes from udata1 to udata2 */
326+ value.udata = &udata2;
327+ value.nested.udata = &udata2;
328+ err = bpf_map_update_elem(map_fd, &parent_task_fd, &value, BPF_EXIST);
329+ if (!ASSERT_OK(err, "update_elem(udata2)"))
330+ goto out;
331+ check_udata2(MAGIC_VALUE + udata2.a + udata2.b);
332+
333+ /* update_elem: uptr changes from udata2 uptr to NULL */
334+ memset(&value, 0, sizeof(value));
335+ err = bpf_map_update_elem(map_fd, &parent_task_fd, &value, BPF_EXIST);
336+ if (!ASSERT_OK(err, "update_elem(udata2)"))
337+ goto out;
338+ check_udata2(0);
339+
340+ /* update_elem: uptr changes from NULL to udata2 */
341+ value.udata = &udata2;
342+ value.nested.udata = &udata2;
343+ err = bpf_map_update_elem(map_fd, &parent_task_fd, &value, BPF_EXIST);
344+ if (!ASSERT_OK(err, "update_elem(udata2)"))
345+ goto out;
346+ check_udata2(MAGIC_VALUE + udata2.a + udata2.b);
347+
348+ /* Check if user programs can access the value of user kptrs
349+ * through bpf_map_lookup_elem(). Make sure the kernel value is not
350+ * leaked.
351+ */
352+ err = bpf_map_lookup_elem(map_fd, &parent_task_fd, &value);
353+ if (!ASSERT_OK(err, "bpf_map_lookup_elem"))
354+ goto out;
355+ ASSERT_EQ(value.udata, NULL, "value.udata");
356+ ASSERT_EQ(value.nested.udata, NULL, "value.nested.udata");
357+
358+ /* delete_elem */
359+ err = bpf_map_delete_elem(map_fd, &parent_task_fd);
360+ ASSERT_OK(err, "delete_elem(udata2)");
361+ check_udata2(0);
362+
363+ /* update_elem: add uptr back to test map_free */
364+ value.udata = &udata2;
365+ value.nested.udata = &udata2;
366+ err = bpf_map_update_elem(map_fd, &parent_task_fd, &value, BPF_NOEXIST);
367+ ASSERT_OK(err, "update_elem(udata2)");
368+
369+out:
370+ task_ls_uptr__destroy(skel);
371+ close(ev_fd);
372+ close(parent_task_fd);
373+}
374+
375+static void test_uptr_across_pages(void)
376+{
377+ int page_size = getpagesize();
378+ struct value_type value = {};
379+ struct task_ls_uptr *skel;
380+ int err, task_fd, map_fd;
381+ void *mem;
382+
383+ task_fd = sys_pidfd_open(getpid(), 0);
384+ if (!ASSERT_OK_FD(task_fd, "task_fd"))
385+ return;
386+
387+ mem = mmap(NULL, page_size * 2, PROT_READ | PROT_WRITE,
388+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
389+ if (!ASSERT_OK_PTR(mem, "mmap(page_size * 2)")) {
390+ close(task_fd);
391+ return;
392+ }
393+
394+ skel = task_ls_uptr__open_and_load();
395+ if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))
396+ goto out;
397+
398+ map_fd = bpf_map__fd(skel->maps.datamap);
399+ value.udata = mem + page_size - offsetof(struct user_data, b);
400+ err = bpf_map_update_elem(map_fd, &task_fd, &value, 0);
401+ if (!ASSERT_ERR(err, "update_elem(udata)"))
402+ goto out;
403+ ASSERT_EQ(errno, EOPNOTSUPP, "errno");
404+
405+ value.udata = mem + page_size - sizeof(struct user_data);
406+ err = bpf_map_update_elem(map_fd, &task_fd, &value, 0);
407+ ASSERT_OK(err, "update_elem(udata)");
408+
409+out:
410+ task_ls_uptr__destroy(skel);
411+ close(task_fd);
412+ munmap(mem, page_size * 2);
413+}
414+
415+static void test_uptr_update_failure(void)
416+{
417+ struct value_lock_type value = {};
418+ struct uptr_update_failure *skel;
419+ int err, task_fd, map_fd;
420+
421+ task_fd = sys_pidfd_open(getpid(), 0);
422+ if (!ASSERT_OK_FD(task_fd, "task_fd"))
423+ return;
424+
425+ skel = uptr_update_failure__open_and_load();
426+ if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))
427+ goto out;
428+
429+ map_fd = bpf_map__fd(skel->maps.datamap);
430+
431+ value.udata = &udata;
432+ err = bpf_map_update_elem(map_fd, &task_fd, &value, BPF_F_LOCK);
433+ if (!ASSERT_ERR(err, "update_elem(udata, BPF_F_LOCK)"))
434+ goto out;
435+ ASSERT_EQ(errno, EOPNOTSUPP, "errno");
436+
437+ err = bpf_map_update_elem(map_fd, &task_fd, &value, BPF_EXIST);
438+ if (!ASSERT_ERR(err, "update_elem(udata, BPF_EXIST)"))
439+ goto out;
440+ ASSERT_EQ(errno, ENOENT, "errno");
441+
442+ err = bpf_map_update_elem(map_fd, &task_fd, &value, BPF_NOEXIST);
443+ if (!ASSERT_OK(err, "update_elem(udata, BPF_NOEXIST)"))
444+ goto out;
445+
446+ value.udata = &udata2;
447+ err = bpf_map_update_elem(map_fd, &task_fd, &value, BPF_NOEXIST);
448+ if (!ASSERT_ERR(err, "update_elem(udata2, BPF_NOEXIST)"))
449+ goto out;
450+ ASSERT_EQ(errno, EEXIST, "errno");
451+
452+out:
453+ uptr_update_failure__destroy(skel);
454+ close(task_fd);
455+}
456+
457+static void test_uptr_map_failure(const char *map_name, int expected_errno)
458+{
459+ LIBBPF_OPTS(bpf_map_create_opts, create_attr);
460+ struct uptr_map_failure *skel;
461+ struct bpf_map *map;
462+ struct btf *btf;
463+ int map_fd, err;
464+
465+ skel = uptr_map_failure__open();
466+ if (!ASSERT_OK_PTR(skel, "uptr_map_failure__open"))
467+ return;
468+
469+ map = bpf_object__find_map_by_name(skel->obj, map_name);
470+ btf = bpf_object__btf(skel->obj);
471+ err = btf__load_into_kernel(btf);
472+ if (!ASSERT_OK(err, "btf__load_into_kernel"))
473+ goto done;
474+
475+ create_attr.map_flags = bpf_map__map_flags(map);
476+ create_attr.btf_fd = btf__fd(btf);
477+ create_attr.btf_key_type_id = bpf_map__btf_key_type_id(map);
478+ create_attr.btf_value_type_id = bpf_map__btf_value_type_id(map);
479+ map_fd = bpf_map_create(bpf_map__type(map), map_name,
480+ bpf_map__key_size(map), bpf_map__value_size(map),
481+ 0, &create_attr);
482+ if (ASSERT_ERR_FD(map_fd, "map_create"))
483+ ASSERT_EQ(errno, expected_errno, "errno");
484+ else
485+ close(map_fd);
486+
487+done:
488+ uptr_map_failure__destroy(skel);
489+}
490+
230491 void test_task_local_storage(void)
231492 {
232493 if (test__start_subtest("sys_enter_exit"))
@@ -237,4 +498,21 @@ void test_task_local_storage(void) · 4 unmodified lines
237498 test_recursion();
238499 if (test__start_subtest("nodeadlock"))
239500 test_nodeadlock();
501+ if (test__start_subtest("uptr_basic"))
502+ test_uptr_basic();
503+ if (test__start_subtest("uptr_across_pages"))
504+ test_uptr_across_pages();
505+ if (test__start_subtest("uptr_update_failure"))
506+ test_uptr_update_failure();
507+ if (test__start_subtest("uptr_map_failure_e2big")) {
508+ if (getpagesize() == PAGE_SIZE)
509+ test_uptr_map_failure("large_uptr_map", E2BIG);
510+ else
511+ test__skip();
512+ }
513+ if (test__start_subtest("uptr_map_failure_size0"))
514+ test_uptr_map_failure("empty_uptr_map", EINVAL);
515+ if (test__start_subtest("uptr_map_failure_kstruct"))
516+ test_uptr_map_failure("kstruct_uptr_map", EINVAL);
517+ RUN_TESTS(uptr_failure);
240518 }
tools/testing/selftests/bpf/prog_tests/timer_lockup.c+6−0
@@ -5,6 +5,7 @@ · 4 unmodified lines
55 #include <test_progs.h>
66 #include <pthread.h>
77 #include <network_helpers.h>
8+#include <sys/sysinfo.h>
89
910 #include "timer_lockup.skel.h"
1011
@@ -52,6 +53,11 @@ void test_timer_lockup(void) · 41 unmodified lines
5253 pthread_t thrds[2];
5354 void *ret;
5455
56+ if (get_nprocs() < 2) {
57+ test__skip();
58+ return;
59+ }
60+
5561 skel = timer_lockup__open_and_load();
5662 if (!ASSERT_OK_PTR(skel, "timer_lockup__open_and_load"))
5763 return;
tools/testing/selftests/bpf/prog_tests/token.c+12−7
@@ -828,8 +828,12 @@ static int userns_obj_priv_btf_success(int mnt_fd, struct token_lsm *lsm_skel) · 827 unmodified lines
828828 return validate_struct_ops_load(mnt_fd, true /* should succeed */);
829829 }
830830
831+static const char *token_bpffs_custom_dir()
832+{
833+ return getenv("BPF_SELFTESTS_BPF_TOKEN_DIR") ?: "/tmp/bpf-token-fs";
834+}
835+
831836 #define TOKEN_ENVVAR "LIBBPF_BPF_TOKEN_PATH"
832−#define TOKEN_BPFFS_CUSTOM "/bpf-token-fs"
833837
834838 static int userns_obj_priv_implicit_token(int mnt_fd, struct token_lsm *lsm_skel)
835839 {
@@ -892,6 +896,7 @@ static int userns_obj_priv_implicit_token(int mnt_fd, struct token_lsm *lsm_skel · 56 unmodified lines
892896
893897 static int userns_obj_priv_implicit_token_envvar(int mnt_fd, struct token_lsm *lsm_skel)
894898 {
899+ const char *custom_dir = token_bpffs_custom_dir();
895900 LIBBPF_OPTS(bpf_object_open_opts, opts);
896901 struct dummy_st_ops_success *skel;
897902 int err;
@@ -909,10 +914,10 @@ static int userns_obj_priv_implicit_token_envvar(int mnt_fd, struct token_lsm *l · 11 unmodified lines
909914 * BPF token implicitly, unless pointed to it through
910915 * LIBBPF_BPF_TOKEN_PATH envvar
911916 */
912− rmdir(TOKEN_BPFFS_CUSTOM);
913− if (!ASSERT_OK(mkdir(TOKEN_BPFFS_CUSTOM, 0777), "mkdir_bpffs_custom"))
917+ rmdir(custom_dir);
918+ if (!ASSERT_OK(mkdir(custom_dir, 0777), "mkdir_bpffs_custom"))
914919 goto err_out;
915− err = sys_move_mount(mnt_fd, "", AT_FDCWD, TOKEN_BPFFS_CUSTOM, MOVE_MOUNT_F_EMPTY_PATH);
920+ err = sys_move_mount(mnt_fd, "", AT_FDCWD, custom_dir, MOVE_MOUNT_F_EMPTY_PATH);
916921 if (!ASSERT_OK(err, "move_mount_bpffs"))
917922 goto err_out;
918923
@@ -925,7 +930,7 @@ static int userns_obj_priv_implicit_token_envvar(int mnt_fd, struct token_lsm *l · 6 unmodified lines
925930 goto err_out;
926931 }
927932
928− err = setenv(TOKEN_ENVVAR, TOKEN_BPFFS_CUSTOM, 1 /*overwrite*/);
933+ err = setenv(TOKEN_ENVVAR, custom_dir, 1 /*overwrite*/);
929934 if (!ASSERT_OK(err, "setenv_token_path"))
930935 goto err_out;
931936
@@ -951,11 +956,11 @@ static int userns_obj_priv_implicit_token_envvar(int mnt_fd, struct token_lsm *l · 19 unmodified lines
951956 if (!ASSERT_ERR(err, "obj_empty_token_path_load"))
952957 goto err_out;
953958
954− rmdir(TOKEN_BPFFS_CUSTOM);
959+ rmdir(custom_dir);
955960 unsetenv(TOKEN_ENVVAR);
956961 return 0;
957962 err_out:
958− rmdir(TOKEN_BPFFS_CUSTOM);
963+ rmdir(custom_dir);
959964 unsetenv(TOKEN_ENVVAR);
960965 return -EINVAL;
961966 }
tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c+311−50
@@ -8,6 +8,11 @@ · 7 unmodified lines
88 #include "uprobe_multi_usdt.skel.h"
99 #include "uprobe_multi_consumers.skel.h"
1010 #include "uprobe_multi_pid_filter.skel.h"
11+#include "uprobe_multi_session.skel.h"
12+#include "uprobe_multi_session_single.skel.h"
13+#include "uprobe_multi_session_cookie.skel.h"
14+#include "uprobe_multi_session_recursive.skel.h"
15+#include "uprobe_multi_verifier.skel.h"
1116 #include "bpf/libbpf_internal.h"
1217 #include "testing_helpers.h"
1318 #include "../sdt.h"
@@ -34,6 +39,12 @@ noinline void usdt_trigger(void) · 20 unmodified lines
3439 STAP_PROBE(test, pid_filter_usdt);
3540 }
3641
42+noinline void uprobe_session_recursive(int i)
43+{
44+ if (i)
45+ uprobe_session_recursive(i - 1);
46+}
47+
3748 struct child {
3849 int go[2];
3950 int c2p[2]; /* child -> parent channel */
@@ -125,7 +136,7 @@ static void *child_thread(void *ctx) · 85 unmodified lines
125136 struct child *child = ctx;
126137 int c = 0, err;
127138
128− child->tid = syscall(SYS_gettid);
139+ child->tid = sys_gettid();
129140
130141 /* let parent know we are ready */
131142 err = write(child->c2p[1], &c, 1);
@@ -778,7 +789,7 @@ get_link(struct uprobe_multi_consumers *skel, int link) · 646 unmodified lines
778789 }
779790 }
780791
781−static int uprobe_attach(struct uprobe_multi_consumers *skel, int idx)
792+static int uprobe_attach(struct uprobe_multi_consumers *skel, int idx, unsigned long offset)
782793 {
783794 struct bpf_program *prog = get_program(skel, idx);
784795 struct bpf_link **link = get_link(skel, idx);
@@ -787,15 +798,19 @@ static int uprobe_attach(struct uprobe_multi_consumers *skel, int idx) · 2 unmodified lines
787798 if (!prog || !link)
788799 return -1;
789800
801+ opts.offsets = &offset;
802+ opts.cnt = 1;
803+
790804 /*
791− * bit/prog: 0,1 uprobe entry
792− * bit/prog: 2,3 uprobe return
805+ * bit/prog: 0 uprobe entry
806+ * bit/prog: 1 uprobe return
807+ * bit/prog: 2 uprobe session without return
808+ * bit/prog: 3 uprobe session with return
793809 */
794− opts.retprobe = idx == 2 || idx == 3;
810+ opts.retprobe = idx == 1;
811+ opts.session = idx == 2 || idx == 3;
795812
796− *link = bpf_program__attach_uprobe_multi(prog, 0, "/proc/self/exe",
797− "uprobe_consumer_test",
798− &opts);
813+ *link = bpf_program__attach_uprobe_multi(prog, 0, "/proc/self/exe", NULL, &opts);
799814 if (!ASSERT_OK_PTR(*link, "bpf_program__attach_uprobe_multi"))
800815 return -1;
801816 return 0;
@@ -816,7 +831,8 @@ static bool test_bit(int bit, unsigned long val) · 14 unmodified lines
816831
817832 noinline int
818833 uprobe_consumer_test(struct uprobe_multi_consumers *skel,
819− unsigned long before, unsigned long after)
834+ unsigned long before, unsigned long after,
835+ unsigned long offset)
820836 {
821837 int idx;
822838
@@ -829,89 +845,174 @@ uprobe_consumer_test(struct uprobe_multi_consumers *skel, · 6 unmodified lines
829845 /* ... and attach all new programs in 'after' state */
830846 for (idx = 0; idx < 4; idx++) {
831847 if (!test_bit(idx, before) && test_bit(idx, after)) {
832− if (!ASSERT_OK(uprobe_attach(skel, idx), "uprobe_attach_after"))
848+ if (!ASSERT_OK(uprobe_attach(skel, idx, offset), "uprobe_attach_after"))
833849 return -1;
834850 }
835851 }
836852 return 0;
837853 }
838854
839−static void consumer_test(struct uprobe_multi_consumers *skel,
840− unsigned long before, unsigned long after)
855+/*
856+ * We generate 16 consumer_testX functions that will have uprobe installed on
857+ * and will be called in separate threads. All function pointer are stored in
858+ * "consumers" section and each thread will pick one function based on index.
859+ */
860+
861+extern const void *__start_consumers;
862+
863+#define __CONSUMER_TEST(func) \
864+noinline int func(struct uprobe_multi_consumers *skel, unsigned long before, \
865+ unsigned long after, unsigned long offset) \
866+{ \
867+ return uprobe_consumer_test(skel, before, after, offset); \
868+} \
869+void *__ ## func __used __attribute__((section("consumers"))) = (void *) func;
870+
871+#define CONSUMER_TEST(func) __CONSUMER_TEST(func)
872+
873+#define C1 CONSUMER_TEST(__PASTE(consumer_test, __COUNTER__))
874+#define C4 C1 C1 C1 C1
875+#define C16 C4 C4 C4 C4
876+
877+C16
878+
879+typedef int (*test_t)(struct uprobe_multi_consumers *, unsigned long,
880+ unsigned long, unsigned long);
881+
882+static int consumer_test(struct uprobe_multi_consumers *skel,
883+ unsigned long before, unsigned long after,
884+ test_t test, unsigned long offset)
841885 {
842− int err, idx;
886+ int err, idx, ret = -1;
843887
844888 printf("consumer_test before %lu after %lu\n", before, after);
845889
846890 /* 'before' is each, we attach uprobe for every set idx */
847891 for (idx = 0; idx < 4; idx++) {
848892 if (test_bit(idx, before)) {
849− if (!ASSERT_OK(uprobe_attach(skel, idx), "uprobe_attach_before"))
893+ if (!ASSERT_OK(uprobe_attach(skel, idx, offset), "uprobe_attach_before"))
850894 goto cleanup;
851895 }
852896 }
853897
854− err = uprobe_consumer_test(skel, before, after);
898+ err = test(skel, before, after, offset);
855899 if (!ASSERT_EQ(err, 0, "uprobe_consumer_test"))
856900 goto cleanup;
857901
858902 for (idx = 0; idx < 4; idx++) {
903+ bool uret_stays, uret_survives;
859904 const char *fmt = "BUG";
860905 __u64 val = 0;
861906
862− if (idx < 2) {
907+ switch (idx) {
908+ case 0:
863909 /*
864910 * uprobe entry
865911 * +1 if define in 'before'
866912 */
867913 if (test_bit(idx, before))
868914 val++;
869− fmt = "prog 0/1: uprobe";
870− } else {
915+ fmt = "prog 0: uprobe";
916+ break;
917+ case 1:
871918 /*
872− * uprobe return is tricky ;-)
873− *
874− * to trigger uretprobe consumer, the uretprobe needs to be installed,
875− * which means one of the 'return' uprobes was alive when probe was hit:
876− *
877− * idxs: 2/3 uprobe return in 'installed' mask
878− *
879− * in addition if 'after' state removes everything that was installed in
880− * 'before' state, then uprobe kernel object goes away and return uprobe
881− * is not installed and we won't hit it even if it's in 'after' state.
919+ * To trigger uretprobe consumer, the uretprobe under test either stayed from
920+ * before to after (uret_stays + test_bit) or uretprobe instance survived and
921+ * we have uretprobe active in after (uret_survives + test_bit)
882922 */
883− unsigned long had_uretprobes = before & 0b1100; /* is uretprobe installed */
884− unsigned long probe_preserved = before & after; /* did uprobe go away */
923+ uret_stays = before & after & 0b0110;
924+ uret_survives = ((before & 0b0110) && (after & 0b0110) && (before & 0b1001));
885925
886− if (had_uretprobes && probe_preserved && test_bit(idx, after))
926+ if ((uret_stays || uret_survives) && test_bit(idx, after))
887927 val++;
888− fmt = "idx 2/3: uretprobe";
928+ fmt = "prog 1: uretprobe";
929+ break;
930+ case 2:
931+ /*
932+ * session with return
933+ * +1 if defined in 'before'
934+ * +1 if defined in 'after'
935+ */
936+ if (test_bit(idx, before)) {
937+ val++;
938+ if (test_bit(idx, after))
939+ val++;
940+ }
941+ fmt = "prog 2: session with return";
942+ break;
943+ case 3:
944+ /*
945+ * session without return
946+ * +1 if defined in 'before'
947+ */
948+ if (test_bit(idx, before))
949+ val++;
950+ fmt = "prog 3: session with NO return";
951+ break;
889952 }
890953
891− ASSERT_EQ(skel->bss->uprobe_result[idx], val, fmt);
954+ if (!ASSERT_EQ(skel->bss->uprobe_result[idx], val, fmt))
955+ goto cleanup;
892956 skel->bss->uprobe_result[idx] = 0;
893957 }
894958
959+ ret = 0;
960+
895961 cleanup:
896962 for (idx = 0; idx < 4; idx++)
897963 uprobe_detach(skel, idx);
964+ return ret;
898965 }
899966
900−static void test_consumers(void)
967+#define CONSUMER_MAX 16
968+
969+/*
970+ * Each thread runs 1/16 of the load by running test for single
971+ * 'before' number (based on thread index) and full scale of
972+ * 'after' numbers.
973+ */
974+static void *consumer_thread(void *arg)
901975 {
976+ unsigned long idx = (unsigned long) arg;
902977 struct uprobe_multi_consumers *skel;
903− int before, after;
978+ unsigned long offset;
979+ const void *func;
980+ int after;
904981
905982 skel = uprobe_multi_consumers__open_and_load();
906983 if (!ASSERT_OK_PTR(skel, "uprobe_multi_consumers__open_and_load"))
907− return;
984+ return NULL;
985+
986+ func = *((&__start_consumers) + idx);
987+
988+ offset = get_uprobe_offset(func);
989+ if (!ASSERT_GE(offset, 0, "uprobe_offset"))
990+ goto out;
991+
992+ for (after = 0; after < CONSUMER_MAX; after++)
993+ if (consumer_test(skel, idx, after, func, offset))
994+ goto out;
995+
996+out:
997+ uprobe_multi_consumers__destroy(skel);
998+ return NULL;
999+}
1000+
1001+
1002+static void test_consumers(void)
1003+{
1004+ pthread_t pt[CONSUMER_MAX];
1005+ unsigned long idx;
1006+ int err;
9081007
9091008 /*
9101009 * The idea of this test is to try all possible combinations of
9111010 * uprobes consumers attached on single function.
9121011 *
913− * - 2 uprobe entry consumer
914− * - 2 uprobe exit consumers
1012+ * - 1 uprobe entry consumer
1013+ * - 1 uprobe exit consumer
1014+ * - 1 uprobe session with return
1015+ * - 1 uprobe session without return
9151016 *
9161017 * The test uses 4 uprobes attached on single function, but that
9171018 * translates into single uprobe with 4 consumers in kernel.
@@ -919,37 +1020,38 @@ static void test_consumers(void) · 1 unmodified line
9191020 * The before/after values present the state of attached consumers
9201021 * before and after the probed function:
9211022 *
922− * bit/prog 0,1 : uprobe entry
923− * bit/prog 2,3 : uprobe return
1023+ * bit/prog 0 : uprobe entry
1024+ * bit/prog 1 : uprobe return
9241025 *
9251026 * For example for:
9261027 *
927− * before = 0b0101
928− * after = 0b0110
1028+ * before = 0b01
1029+ * after = 0b10
9291030 *
9301031 * it means that before we call 'uprobe_consumer_test' we attach
9311032 * uprobes defined in 'before' value:
9321033 *
933− * - bit/prog 0: uprobe entry
934− * - bit/prog 2: uprobe return
1034+ * - bit/prog 1: uprobe entry
9351035 *
9361036 * uprobe_consumer_test is called and inside it we attach and detach
9371037 * uprobes based on 'after' value:
9381038 *
939− * - bit/prog 0: stays untouched
940− * - bit/prog 2: uprobe return is detached
1039+ * - bit/prog 0: is detached
1040+ * - bit/prog 1: is attached
9411041 *
9421042 * uprobe_consumer_test returns and we check counters values increased
9431043 * by bpf programs on each uprobe to match the expected count based on
9441044 * before/after bits.
9451045 */
9461046
947− for (before = 0; before < 16; before++) {
948− for (after = 0; after < 16; after++)
949− consumer_test(skel, before, after);
1047+ for (idx = 0; idx < CONSUMER_MAX; idx++) {
1048+ err = pthread_create(&pt[idx], NULL, consumer_thread, (void *) idx);
1049+ if (!ASSERT_OK(err, "pthread_create"))
1050+ break;
9501051 }
9511052
952− uprobe_multi_consumers__destroy(skel);
1053+ while (idx)
1054+ pthread_join(pt[--idx], NULL);
9531055 }
9541056
9551057 static struct bpf_program *uprobe_multi_program(struct uprobe_multi_pid_filter *skel, int idx)
@@ -1016,6 +1118,156 @@ static void test_pid_filter_process(bool clone_vm) · 60 unmodified lines
10161118 uprobe_multi_pid_filter__destroy(skel);
10171119 }
10181120
1121+static void test_session_skel_api(void)
1122+{
1123+ struct uprobe_multi_session *skel = NULL;
1124+ LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
1125+ struct bpf_link *link = NULL;
1126+ int err;
1127+
1128+ skel = uprobe_multi_session__open_and_load();
1129+ if (!ASSERT_OK_PTR(skel, "uprobe_multi_session__open_and_load"))
1130+ goto cleanup;
1131+
1132+ skel->bss->pid = getpid();
1133+ skel->bss->user_ptr = test_data;
1134+
1135+ err = uprobe_multi_session__attach(skel);
1136+ if (!ASSERT_OK(err, "uprobe_multi_session__attach"))
1137+ goto cleanup;
1138+
1139+ /* trigger all probes */
1140+ skel->bss->uprobe_multi_func_1_addr = (__u64) uprobe_multi_func_1;
1141+ skel->bss->uprobe_multi_func_2_addr = (__u64) uprobe_multi_func_2;
1142+ skel->bss->uprobe_multi_func_3_addr = (__u64) uprobe_multi_func_3;
1143+
1144+ uprobe_multi_func_1();
1145+ uprobe_multi_func_2();
1146+ uprobe_multi_func_3();
1147+
1148+ /*
1149+ * We expect 2 for uprobe_multi_func_2 because it runs both entry/return probe,
1150+ * uprobe_multi_func_[13] run just the entry probe. All expected numbers are
1151+ * doubled, because we run extra test for sleepable session.
1152+ */
1153+ ASSERT_EQ(skel->bss->uprobe_session_result[0], 2, "uprobe_multi_func_1_result");
1154+ ASSERT_EQ(skel->bss->uprobe_session_result[1], 4, "uprobe_multi_func_2_result");
1155+ ASSERT_EQ(skel->bss->uprobe_session_result[2], 2, "uprobe_multi_func_3_result");
1156+
1157+ /* We expect increase in 3 entry and 1 return session calls -> 4 */
1158+ ASSERT_EQ(skel->bss->uprobe_multi_sleep_result, 4, "uprobe_multi_sleep_result");
1159+
1160+cleanup:
1161+ bpf_link__destroy(link);
1162+ uprobe_multi_session__destroy(skel);
1163+}
1164+
1165+static void test_session_single_skel_api(void)
1166+{
1167+ struct uprobe_multi_session_single *skel = NULL;
1168+ LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
1169+ int err;
1170+
1171+ skel = uprobe_multi_session_single__open_and_load();
1172+ if (!ASSERT_OK_PTR(skel, "uprobe_multi_session_single__open_and_load"))
1173+ goto cleanup;
1174+
1175+ skel->bss->pid = getpid();
1176+
1177+ err = uprobe_multi_session_single__attach(skel);
1178+ if (!ASSERT_OK(err, "uprobe_multi_session_single__attach"))
1179+ goto cleanup;
1180+
1181+ uprobe_multi_func_1();
1182+
1183+ /*
1184+ * We expect consumer 0 and 2 to trigger just entry handler (value 1)
1185+ * and consumer 1 to hit both (value 2).
1186+ */
1187+ ASSERT_EQ(skel->bss->uprobe_session_result[0], 1, "uprobe_session_result_0");
1188+ ASSERT_EQ(skel->bss->uprobe_session_result[1], 2, "uprobe_session_result_1");
1189+ ASSERT_EQ(skel->bss->uprobe_session_result[2], 1, "uprobe_session_result_2");
1190+
1191+cleanup:
1192+ uprobe_multi_session_single__destroy(skel);
1193+}
1194+
1195+static void test_session_cookie_skel_api(void)
1196+{
1197+ struct uprobe_multi_session_cookie *skel = NULL;
1198+ int err;
1199+
1200+ skel = uprobe_multi_session_cookie__open_and_load();
1201+ if (!ASSERT_OK_PTR(skel, "uprobe_multi_session_cookie__open_and_load"))
1202+ goto cleanup;
1203+
1204+ skel->bss->pid = getpid();
1205+
1206+ err = uprobe_multi_session_cookie__attach(skel);
1207+ if (!ASSERT_OK(err, "uprobe_multi_session_cookie__attach"))
1208+ goto cleanup;
1209+
1210+ /* trigger all probes */
1211+ uprobe_multi_func_1();
1212+ uprobe_multi_func_2();
1213+ uprobe_multi_func_3();
1214+
1215+ ASSERT_EQ(skel->bss->test_uprobe_1_result, 1, "test_uprobe_1_result");
1216+ ASSERT_EQ(skel->bss->test_uprobe_2_result, 2, "test_uprobe_2_result");
1217+ ASSERT_EQ(skel->bss->test_uprobe_3_result, 3, "test_uprobe_3_result");
1218+
1219+cleanup:
1220+ uprobe_multi_session_cookie__destroy(skel);
1221+}
1222+
1223+static void test_session_recursive_skel_api(void)
1224+{
1225+ struct uprobe_multi_session_recursive *skel = NULL;
1226+ int i, err;
1227+
1228+ skel = uprobe_multi_session_recursive__open_and_load();
1229+ if (!ASSERT_OK_PTR(skel, "uprobe_multi_session_recursive__open_and_load"))
1230+ goto cleanup;
1231+
1232+ skel->bss->pid = getpid();
1233+
1234+ err = uprobe_multi_session_recursive__attach(skel);
1235+ if (!ASSERT_OK(err, "uprobe_multi_session_recursive__attach"))
1236+ goto cleanup;
1237+
1238+ for (i = 0; i < ARRAY_SIZE(skel->bss->test_uprobe_cookie_entry); i++)
1239+ skel->bss->test_uprobe_cookie_entry[i] = i + 1;
1240+
1241+ uprobe_session_recursive(5);
1242+
1243+ /*
1244+ * entry uprobe:
1245+ * uprobe_session_recursive(5) { *cookie = 1, return 0
1246+ * uprobe_session_recursive(4) { *cookie = 2, return 1
1247+ * uprobe_session_recursive(3) { *cookie = 3, return 0
1248+ * uprobe_session_recursive(2) { *cookie = 4, return 1
1249+ * uprobe_session_recursive(1) { *cookie = 5, return 0
1250+ * uprobe_session_recursive(0) { *cookie = 6, return 1
1251+ * return uprobe:
1252+ * } i = 0 not executed
1253+ * } i = 1 test_uprobe_cookie_return[0] = 5
1254+ * } i = 2 not executed
1255+ * } i = 3 test_uprobe_cookie_return[1] = 3
1256+ * } i = 4 not executed
1257+ * } i = 5 test_uprobe_cookie_return[2] = 1
1258+ */
1259+
1260+ ASSERT_EQ(skel->bss->idx_entry, 6, "idx_entry");
1261+ ASSERT_EQ(skel->bss->idx_return, 3, "idx_return");
1262+
1263+ ASSERT_EQ(skel->bss->test_uprobe_cookie_return[0], 5, "test_uprobe_cookie_return[0]");
1264+ ASSERT_EQ(skel->bss->test_uprobe_cookie_return[1], 3, "test_uprobe_cookie_return[1]");
1265+ ASSERT_EQ(skel->bss->test_uprobe_cookie_return[2], 1, "test_uprobe_cookie_return[2]");
1266+
1267+cleanup:
1268+ uprobe_multi_session_recursive__destroy(skel);
1269+}
1270+
10191271 static void test_bench_attach_uprobe(void)
10201272 {
10211273 long attach_start_ns = 0, attach_end_ns = 0;
@@ -1112,4 +1364,13 @@ void test_uprobe_multi_test(void) · 90 unmodified lines
11121364 test_pid_filter_process(false);
11131365 if (test__start_subtest("filter_clone_vm"))
11141366 test_pid_filter_process(true);
1367+ if (test__start_subtest("session"))
1368+ test_session_skel_api();
1369+ if (test__start_subtest("session_single"))
1370+ test_session_single_skel_api();
1371+ if (test__start_subtest("session_cookie"))
1372+ test_session_cookie_skel_api();
1373+ if (test__start_subtest("session_cookie_recursive"))
1374+ test_session_recursive_skel_api();
1375+ RUN_TESTS(uprobe_multi_verifier);
11151376 }
tools/testing/selftests/bpf/prog_tests/verifier.c+2−0
@@ -61,6 +61,7 @@ · 60 unmodified lines
6161 #include "verifier_or_jmp32_k.skel.h"
6262 #include "verifier_precision.skel.h"
6363 #include "verifier_prevent_map_lookup.skel.h"
64+#include "verifier_private_stack.skel.h"
6465 #include "verifier_raw_stack.skel.h"
6566 #include "verifier_raw_tp_writable.skel.h"
6667 #include "verifier_reg_equal.skel.h"
@@ -188,6 +189,7 @@ void test_verifier_bpf_fastcall(void) { RUN(verifier_bpf_fastcall); } · 121 unmodified lines
188189 void test_verifier_or_jmp32_k(void) { RUN(verifier_or_jmp32_k); }
189190 void test_verifier_precision(void) { RUN(verifier_precision); }
190191 void test_verifier_prevent_map_lookup(void) { RUN(verifier_prevent_map_lookup); }
192+void test_verifier_private_stack(void) { RUN(verifier_private_stack); }
191193 void test_verifier_raw_stack(void) { RUN(verifier_raw_stack); }
192194 void test_verifier_raw_tp_writable(void) { RUN(verifier_raw_tp_writable); }
193195 void test_verifier_reg_equal(void) { RUN(verifier_reg_equal); }
tools/testing/selftests/bpf/progs/bpf_iter.h+0−167
@@ -1,167 +0,0 @@
1−/* SPDX-License-Identifier: GPL-2.0 */
2−/* Copyright (c) 2020 Facebook */
3−/* "undefine" structs in vmlinux.h, because we "override" them below */
4−#define bpf_iter_meta bpf_iter_meta___not_used
5−#define bpf_iter__bpf_map bpf_iter__bpf_map___not_used
6−#define bpf_iter__ipv6_route bpf_iter__ipv6_route___not_used
7−#define bpf_iter__netlink bpf_iter__netlink___not_used
8−#define bpf_iter__task bpf_iter__task___not_used
9−#define bpf_iter__task_file bpf_iter__task_file___not_used
10−#define bpf_iter__task_vma bpf_iter__task_vma___not_used
11−#define bpf_iter__tcp bpf_iter__tcp___not_used
12−#define tcp6_sock tcp6_sock___not_used
13−#define bpf_iter__udp bpf_iter__udp___not_used
14−#define udp6_sock udp6_sock___not_used
15−#define bpf_iter__unix bpf_iter__unix___not_used
16−#define bpf_iter__bpf_map_elem bpf_iter__bpf_map_elem___not_used
17−#define bpf_iter__bpf_sk_storage_map bpf_iter__bpf_sk_storage_map___not_used
18−#define bpf_iter__sockmap bpf_iter__sockmap___not_used
19−#define bpf_iter__bpf_link bpf_iter__bpf_link___not_used
20−#define bpf_iter__cgroup bpf_iter__cgroup___not_used
21−#define btf_ptr btf_ptr___not_used
22−#define BTF_F_COMPACT BTF_F_COMPACT___not_used
23−#define BTF_F_NONAME BTF_F_NONAME___not_used
24−#define BTF_F_PTR_RAW BTF_F_PTR_RAW___not_used
25−#define BTF_F_ZERO BTF_F_ZERO___not_used
26−#define bpf_iter__ksym bpf_iter__ksym___not_used
27−#include "vmlinux.h"
28−#undef bpf_iter_meta
29−#undef bpf_iter__bpf_map
30−#undef bpf_iter__ipv6_route
31−#undef bpf_iter__netlink
32−#undef bpf_iter__task
33−#undef bpf_iter__task_file
34−#undef bpf_iter__task_vma
35−#undef bpf_iter__tcp
36−#undef tcp6_sock
37−#undef bpf_iter__udp
38−#undef udp6_sock
39−#undef bpf_iter__unix
40−#undef bpf_iter__bpf_map_elem
41−#undef bpf_iter__bpf_sk_storage_map
42−#undef bpf_iter__sockmap
43−#undef bpf_iter__bpf_link
44−#undef bpf_iter__cgroup
45−#undef btf_ptr
46−#undef BTF_F_COMPACT
47−#undef BTF_F_NONAME
48−#undef BTF_F_PTR_RAW
49−#undef BTF_F_ZERO
50−#undef bpf_iter__ksym
51−
52−struct bpf_iter_meta {
53− struct seq_file *seq;
54− __u64 session_id;
55− __u64 seq_num;
56−} __attribute__((preserve_access_index));
57−
58−struct bpf_iter__ipv6_route {
59− struct bpf_iter_meta *meta;
60− struct fib6_info *rt;
61−} __attribute__((preserve_access_index));
62−
63−struct bpf_iter__netlink {
64− struct bpf_iter_meta *meta;
65− struct netlink_sock *sk;
66−} __attribute__((preserve_access_index));
67−
68−struct bpf_iter__task {
69− struct bpf_iter_meta *meta;
70− struct task_struct *task;
71−} __attribute__((preserve_access_index));
72−
73−struct bpf_iter__task_file {
74− struct bpf_iter_meta *meta;
75− struct task_struct *task;
76− __u32 fd;
77− struct file *file;
78−} __attribute__((preserve_access_index));
79−
80−struct bpf_iter__task_vma {
81− struct bpf_iter_meta *meta;
82− struct task_struct *task;
83− struct vm_area_struct *vma;
84−} __attribute__((preserve_access_index));
85−
86−struct bpf_iter__bpf_map {
87− struct bpf_iter_meta *meta;
88− struct bpf_map *map;
89−} __attribute__((preserve_access_index));
90−
91−struct bpf_iter__tcp {
92− struct bpf_iter_meta *meta;
93− struct sock_common *sk_common;
94− uid_t uid;
95−} __attribute__((preserve_access_index));
96−
97−struct tcp6_sock {
98− struct tcp_sock tcp;
99− struct ipv6_pinfo inet6;
100−} __attribute__((preserve_access_index));
101−
102−struct bpf_iter__udp {
103− struct bpf_iter_meta *meta;
104− struct udp_sock *udp_sk;
105− uid_t uid __attribute__((aligned(8)));
106− int bucket __attribute__((aligned(8)));
107−} __attribute__((preserve_access_index));
108−
109−struct udp6_sock {
110− struct udp_sock udp;
111− struct ipv6_pinfo inet6;
112−} __attribute__((preserve_access_index));
113−
114−struct bpf_iter__unix {
115− struct bpf_iter_meta *meta;
116− struct unix_sock *unix_sk;
117− uid_t uid;
118−} __attribute__((preserve_access_index));
119−
120−struct bpf_iter__bpf_map_elem {
121− struct bpf_iter_meta *meta;
122− struct bpf_map *map;
123− void *key;
124− void *value;
125−};
126−
127−struct bpf_iter__bpf_sk_storage_map {
128− struct bpf_iter_meta *meta;
129− struct bpf_map *map;
130− struct sock *sk;
131− void *value;
132−};
133−
134−struct bpf_iter__sockmap {
135− struct bpf_iter_meta *meta;
136− struct bpf_map *map;
137− void *key;
138− struct sock *sk;
139−};
140−
141−struct bpf_iter__bpf_link {
142− struct bpf_iter_meta *meta;
143− struct bpf_link *link;
144−};
145−
146−struct bpf_iter__cgroup {
147− struct bpf_iter_meta *meta;
148− struct cgroup *cgroup;
149−} __attribute__((preserve_access_index));
150−
151−struct btf_ptr {
152− void *ptr;
153− __u32 type_id;
154− __u32 flags;
155−};
156−
157−enum {
158− BTF_F_COMPACT = (1ULL << 0),
159− BTF_F_NONAME = (1ULL << 1),
160− BTF_F_PTR_RAW = (1ULL << 2),
161− BTF_F_ZERO = (1ULL << 3),
162−};
163−
164−struct bpf_iter__ksym {
165− struct bpf_iter_meta *meta;
166− struct kallsym_iter *ksym;
167−};
tools/testing/selftests/bpf/progs/bpf_iter_bpf_array_map.c+1−1
@@ -1,6 +1,6 @@
11 // SPDX-License-Identifier: GPL-2.0
22 /* Copyright (c) 2020 Facebook */
3−#include "bpf_iter.h"
3+#include <vmlinux.h>
44 #include <bpf/bpf_helpers.h>
55 #include <bpf/bpf_tracing.h>
66
tools/testing/selftests/bpf/progs/bpf_iter_bpf_hash_map.c+1−1
@@ -1,6 +1,6 @@
11 // SPDX-License-Identifier: GPL-2.0
22 /* Copyright (c) 2020 Facebook */
3−#include "bpf_iter.h"
3+#include <vmlinux.h>
44 #include <bpf/bpf_helpers.h>
55
66 char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/bpf_iter_bpf_link.c+1−1
@@ -1,6 +1,6 @@
11 // SPDX-License-Identifier: GPL-2.0
22 /* Copyright (c) 2022 Red Hat, Inc. */
3−#include "bpf_iter.h"
3+#include <vmlinux.h>
44 #include <bpf/bpf_helpers.h>
55
66 char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/bpf_iter_bpf_map.c+1−1
@@ -1,6 +1,6 @@
11 // SPDX-License-Identifier: GPL-2.0
22 /* Copyright (c) 2020 Facebook */
3−#include "bpf_iter.h"
3+#include <vmlinux.h>
44 #include <bpf/bpf_helpers.h>
55
66 char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/bpf_iter_bpf_percpu_array_map.c+1−1
@@ -1,6 +1,6 @@
11 // SPDX-License-Identifier: GPL-2.0
22 /* Copyright (c) 2020 Facebook */
3−#include "bpf_iter.h"
3+#include <vmlinux.h>
44 #include <bpf/bpf_helpers.h>
55 #include <bpf/bpf_tracing.h>
66
tools/testing/selftests/bpf/progs/bpf_iter_bpf_percpu_hash_map.c+1−1
@@ -1,6 +1,6 @@
11 // SPDX-License-Identifier: GPL-2.0
22 /* Copyright (c) 2020 Facebook */
3−#include "bpf_iter.h"
3+#include <vmlinux.h>
44 #include <bpf/bpf_helpers.h>
55 #include <bpf/bpf_tracing.h>
66
tools/testing/selftests/bpf/progs/bpf_iter_bpf_sk_storage_helpers.c+1−1
@@ -1,6 +1,6 @@
11 // SPDX-License-Identifier: GPL-2.0
22 /* Copyright (c) 2020 Google LLC. */
3−#include "bpf_iter.h"
3+#include <vmlinux.h>
44 #include <bpf/bpf_helpers.h>
55 #include <bpf/bpf_tracing.h>
66
tools/testing/selftests/bpf/progs/bpf_iter_bpf_sk_storage_map.c+1−1
@@ -1,6 +1,6 @@
11 // SPDX-License-Identifier: GPL-2.0
22 /* Copyright (c) 2020 Facebook */
3−#include "bpf_iter.h"
3+#include <vmlinux.h>
44 #include "bpf_tracing_net.h"
55 #include <bpf/bpf_helpers.h>
66 #include <bpf/bpf_tracing.h>
tools/testing/selftests/bpf/progs/bpf_iter_ipv6_route.c+1−1
@@ -1,6 +1,6 @@
11 // SPDX-License-Identifier: GPL-2.0
22 /* Copyright (c) 2020 Facebook */
3−#include "bpf_iter.h"
3+#include <vmlinux.h>
44 #include "bpf_tracing_net.h"
55 #include <bpf/bpf_helpers.h>
66
tools/testing/selftests/bpf/progs/bpf_iter_ksym.c+1−1
@@ -1,6 +1,6 @@
11 // SPDX-License-Identifier: GPL-2.0
22 /* Copyright (c) 2022, Oracle and/or its affiliates. */
3−#include "bpf_iter.h"
3+#include <vmlinux.h>
44 #include <bpf/bpf_helpers.h>
55
66 char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/bpf_iter_netlink.c+1−1
@@ -1,6 +1,6 @@
11 // SPDX-License-Identifier: GPL-2.0
22 /* Copyright (c) 2020 Facebook */
3−#include "bpf_iter.h"
3+#include <vmlinux.h>
44 #include "bpf_tracing_net.h"
55 #include <bpf/bpf_helpers.h>
66
tools/testing/selftests/bpf/progs/bpf_iter_setsockopt.c+1−1
@@ -1,6 +1,6 @@
11 // SPDX-License-Identifier: GPL-2.0
22 /* Copyright (c) 2021 Facebook */
3−#include "bpf_iter.h"
3+#include <vmlinux.h>
44 #include "bpf_tracing_net.h"
55 #include <bpf/bpf_helpers.h>
66 #include <bpf/bpf_endian.h>
tools/testing/selftests/bpf/progs/bpf_iter_setsockopt_unix.c+1−1
@@ -1,6 +1,6 @@
11 // SPDX-License-Identifier: GPL-2.0
22 /* Copyright Amazon.com Inc. or its affiliates. */
3−#include "bpf_iter.h"
3+#include <vmlinux.h>
44 #include "bpf_tracing_net.h"
55 #include <bpf/bpf_helpers.h>
66 #include <limits.h>
tools/testing/selftests/bpf/progs/bpf_iter_sockmap.c+1−1
@@ -1,6 +1,6 @@
11 // SPDX-License-Identifier: GPL-2.0
22 /* Copyright (c) 2020 Cloudflare */
3−#include "bpf_iter.h"
3+#include <vmlinux.h>
44 #include "bpf_tracing_net.h"
55 #include <bpf/bpf_helpers.h>
66 #include <bpf/bpf_tracing.h>
tools/testing/selftests/bpf/progs/bpf_iter_task_btf.c+1−1
@@ -1,6 +1,6 @@
11 // SPDX-License-Identifier: GPL-2.0
22 /* Copyright (c) 2020, Oracle and/or its affiliates. */
3−#include "bpf_iter.h"
3+#include <vmlinux.h>
44 #include <bpf/bpf_helpers.h>
55 #include <bpf/bpf_core_read.h>
66
tools/testing/selftests/bpf/progs/bpf_iter_task_file.c+1−1
@@ -1,6 +1,6 @@
11 // SPDX-License-Identifier: GPL-2.0
22 /* Copyright (c) 2020 Facebook */
3−#include "bpf_iter.h"
3+#include <vmlinux.h>
44 #include <bpf/bpf_helpers.h>
55
66 char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/bpf_iter_task_stack.c+1−1
@@ -1,6 +1,6 @@
11 // SPDX-License-Identifier: GPL-2.0
22 /* Copyright (c) 2020 Facebook */
3−#include "bpf_iter.h"
3+#include <vmlinux.h>
44 #include <bpf/bpf_helpers.h>
55
66 char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/bpf_iter_task_vmas.c+1−1
@@ -1,6 +1,6 @@
11 // SPDX-License-Identifier: GPL-2.0
22 /* Copyright (c) 2020 Facebook */
3−#include "bpf_iter.h"
3+#include <vmlinux.h>
44 #include <bpf/bpf_helpers.h>
55
66 char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/bpf_iter_tasks.c+1−1
@@ -1,6 +1,6 @@
11 // SPDX-License-Identifier: GPL-2.0
22 /* Copyright (c) 2020 Facebook */
3−#include "bpf_iter.h"
3+#include <vmlinux.h>
44 #include <bpf/bpf_helpers.h>
55 #include <bpf/bpf_tracing.h>
66
tools/testing/selftests/bpf/progs/bpf_iter_tcp4.c+1−1
@@ -1,6 +1,6 @@
11 // SPDX-License-Identifier: GPL-2.0
22 /* Copyright (c) 2020 Facebook */
3−#include "bpf_iter.h"
3+#include <vmlinux.h>
44 #include "bpf_tracing_net.h"
55 #include <bpf/bpf_helpers.h>
66 #include <bpf/bpf_endian.h>
tools/testing/selftests/bpf/progs/bpf_iter_tcp6.c+1−1
@@ -1,6 +1,6 @@
11 // SPDX-License-Identifier: GPL-2.0
22 /* Copyright (c) 2020 Facebook */
3−#include "bpf_iter.h"
3+#include <vmlinux.h>
44 #include "bpf_tracing_net.h"
55 #include <bpf/bpf_helpers.h>
66 #include <bpf/bpf_endian.h>
tools/testing/selftests/bpf/progs/bpf_iter_test_kern3.c+1−1
@@ -1,6 +1,6 @@
11 // SPDX-License-Identifier: GPL-2.0
22 /* Copyright (c) 2020 Facebook */
3−#include "bpf_iter.h"
3+#include <vmlinux.h>
44 #include <bpf/bpf_helpers.h>
55
66 char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/bpf_iter_test_kern4.c+1−1
@@ -1,6 +1,6 @@
11 // SPDX-License-Identifier: GPL-2.0
22 /* Copyright (c) 2020 Facebook */
3−#include "bpf_iter.h"
3+#include <vmlinux.h>
44 #include <bpf/bpf_helpers.h>
55
66 char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/bpf_iter_test_kern5.c+1−1
@@ -1,6 +1,6 @@
11 // SPDX-License-Identifier: GPL-2.0
22 /* Copyright (c) 2020 Facebook */
3−#include "bpf_iter.h"
3+#include <vmlinux.h>
44 #include <bpf/bpf_helpers.h>
55 #include <bpf/bpf_tracing.h>
66
tools/testing/selftests/bpf/progs/bpf_iter_test_kern6.c+1−1
@@ -1,6 +1,6 @@
11 // SPDX-License-Identifier: GPL-2.0
22 /* Copyright (c) 2020 Facebook */
3−#include "bpf_iter.h"
3+#include <vmlinux.h>
44 #include <bpf/bpf_helpers.h>
55
66 char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/bpf_iter_test_kern_common.h+1−1
@@ -1,6 +1,6 @@
11 /* SPDX-License-Identifier: GPL-2.0 */
22 /* Copyright (c) 2020 Facebook */
3−#include "bpf_iter.h"
3+#include <vmlinux.h>
44 #include <bpf/bpf_helpers.h>
55
66 char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/bpf_iter_udp4.c+1−1
@@ -1,6 +1,6 @@
11 // SPDX-License-Identifier: GPL-2.0
22 /* Copyright (c) 2020 Facebook */
3−#include "bpf_iter.h"
3+#include <vmlinux.h>
44 #include "bpf_tracing_net.h"
55 #include <bpf/bpf_helpers.h>
66 #include <bpf/bpf_endian.h>
tools/testing/selftests/bpf/progs/bpf_iter_udp6.c+1−1
@@ -1,6 +1,6 @@
11 // SPDX-License-Identifier: GPL-2.0
22 /* Copyright (c) 2020 Facebook */
3−#include "bpf_iter.h"
3+#include <vmlinux.h>
44 #include "bpf_tracing_net.h"
55 #include <bpf/bpf_helpers.h>
66 #include <bpf/bpf_endian.h>
tools/testing/selftests/bpf/progs/bpf_iter_unix.c+1−1
@@ -1,6 +1,6 @@
11 // SPDX-License-Identifier: GPL-2.0
22 /* Copyright Amazon.com Inc. or its affiliates. */
3−#include "bpf_iter.h"
3+#include <vmlinux.h>
44 #include "bpf_tracing_net.h"
55 #include <bpf/bpf_helpers.h>
66 #include <bpf/bpf_endian.h>
tools/testing/selftests/bpf/progs/bpf_iter_vma_offset.c+1−1
@@ -1,6 +1,6 @@
11 // SPDX-License-Identifier: GPL-2.0
22 /* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
3−#include "bpf_iter.h"
3+#include <vmlinux.h>
44 #include <bpf/bpf_helpers.h>
55
66 char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/cgroup_iter.c+1−2
@@ -1,7 +1,6 @@
11 // SPDX-License-Identifier: GPL-2.0
22 /* Copyright (c) 2022 Google */
3−
4−#include "bpf_iter.h"
3+#include <vmlinux.h>
54 #include <bpf/bpf_helpers.h>
65 #include <bpf/bpf_tracing.h>
76
tools/testing/selftests/bpf/progs/cgrp_ls_sleepable.c+1−2
@@ -1,7 +1,6 @@
11 // SPDX-License-Identifier: GPL-2.0
22 /* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
3−
4−#include "bpf_iter.h"
3+#include <vmlinux.h>
54 #include <bpf/bpf_helpers.h>
65 #include <bpf/bpf_tracing.h>
76 #include "bpf_misc.h"
tools/testing/selftests/bpf/progs/exceptions_fail.c+2−2
@@ -131,7 +131,7 @@ int reject_subprog_with_lock(void *ctx) · 130 unmodified lines
131131 }
132132
133133 SEC("?tc")
134−__failure __msg("bpf_rcu_read_unlock is missing")
134+__failure __msg("BPF_EXIT instruction cannot be used inside bpf_rcu_read_lock-ed region")
135135 int reject_with_rcu_read_lock(void *ctx)
136136 {
137137 bpf_rcu_read_lock();
@@ -147,7 +147,7 @@ __noinline static int throwing_subprog(struct __sk_buff *ctx) · 9 unmodified lines
147147 }
148148
149149 SEC("?tc")
150−__failure __msg("bpf_rcu_read_unlock is missing")
150+__failure __msg("BPF_EXIT instruction cannot be used inside bpf_rcu_read_lock-ed region")
151151 int reject_subprog_with_rcu_read_lock(void *ctx)
152152 {
153153 bpf_rcu_read_lock();
tools/testing/selftests/bpf/progs/kmem_cache_iter.c+108−0
@@ -0,0 +1,108 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/* Copyright (c) 2024 Google */
3+#include <vmlinux.h>
4+#include <bpf/bpf_helpers.h>
5+#include <bpf/bpf_tracing.h>
6+#include "bpf_experimental.h"
7+
8+char _license[] SEC("license") = "GPL";
9+
10+#define SLAB_NAME_MAX 32
11+
12+struct kmem_cache_result {
13+ char name[SLAB_NAME_MAX];
14+ long obj_size;
15+};
16+
17+struct {
18+ __uint(type, BPF_MAP_TYPE_HASH);
19+ __uint(key_size, sizeof(void *));
20+ __uint(value_size, SLAB_NAME_MAX);
21+ __uint(max_entries, 1);
22+} slab_hash SEC(".maps");
23+
24+struct {
25+ __uint(type, BPF_MAP_TYPE_ARRAY);
26+ __uint(key_size, sizeof(int));
27+ __uint(value_size, sizeof(struct kmem_cache_result));
28+ __uint(max_entries, 1024);
29+} slab_result SEC(".maps");
30+
31+extern struct kmem_cache *bpf_get_kmem_cache(u64 addr) __ksym;
32+
33+/* Result, will be checked by userspace */
34+int task_struct_found;
35+int kmem_cache_seen;
36+int open_coded_seen;
37+
38+SEC("iter/kmem_cache")
39+int slab_info_collector(struct bpf_iter__kmem_cache *ctx)
40+{
41+ struct seq_file *seq = ctx->meta->seq;
42+ struct kmem_cache *s = ctx->s;
43+ struct kmem_cache_result *r;
44+ int idx;
45+
46+ if (s) {
47+ /* To make sure if the slab_iter implements the seq interface
48+ * properly and it's also useful for debugging.
49+ */
50+ BPF_SEQ_PRINTF(seq, "%s: %u\n", s->name, s->size);
51+
52+ idx = kmem_cache_seen;
53+ r = bpf_map_lookup_elem(&slab_result, &idx);
54+ if (r == NULL)
55+ return 0;
56+
57+ kmem_cache_seen++;
58+
59+ /* Save name and size to match /proc/slabinfo */
60+ bpf_probe_read_kernel_str(r->name, sizeof(r->name), s->name);
61+ r->obj_size = s->size;
62+
63+ if (!bpf_strncmp(r->name, 11, "task_struct"))
64+ bpf_map_update_elem(&slab_hash, &s, r->name, BPF_NOEXIST);
65+ }
66+
67+ return 0;
68+}
69+
70+SEC("raw_tp/bpf_test_finish")
71+int BPF_PROG(check_task_struct)
72+{
73+ u64 curr = bpf_get_current_task();
74+ struct kmem_cache *s;
75+ char *name;
76+
77+ s = bpf_get_kmem_cache(curr);
78+ if (s == NULL) {
79+ task_struct_found = -1;
80+ return 0;
81+ }
82+ name = bpf_map_lookup_elem(&slab_hash, &s);
83+ if (name && !bpf_strncmp(name, 11, "task_struct"))
84+ task_struct_found = 1;
85+ else
86+ task_struct_found = -2;
87+ return 0;
88+}
89+
90+SEC("syscall")
91+int open_coded_iter(const void *ctx)
92+{
93+ struct kmem_cache *s;
94+
95+ bpf_for_each(kmem_cache, s) {
96+ struct kmem_cache_result *r;
97+
98+ r = bpf_map_lookup_elem(&slab_result, &open_coded_seen);
99+ if (!r)
100+ break;
101+
102+ if (r->obj_size != s->size)
103+ break;
104+
105+ open_coded_seen++;
106+ }
107+ return 0;
108+}
tools/testing/selftests/bpf/progs/kprobe_multi_verifier.c+31−0
@@ -0,0 +1,31 @@
1+// SPDX-License-Identifier: GPL-2.0
2+#include "vmlinux.h"
3+#include <bpf/bpf_helpers.h>
4+#include <bpf/bpf_tracing.h>
5+#include <bpf/usdt.bpf.h>
6+#include "bpf_misc.h"
7+
8+char _license[] SEC("license") = "GPL";
9+
10+
11+SEC("kprobe.session")
12+__success
13+int kprobe_session_return_0(struct pt_regs *ctx)
14+{
15+ return 0;
16+}
17+
18+SEC("kprobe.session")
19+__success
20+int kprobe_session_return_1(struct pt_regs *ctx)
21+{
22+ return 1;
23+}
24+
25+SEC("kprobe.session")
26+__failure
27+__msg("At program exit the register R0 has smin=2 smax=2 should have been in [0, 1]")
28+int kprobe_session_return_2(struct pt_regs *ctx)
29+{
30+ return 2;
31+}
tools/testing/selftests/bpf/progs/linked_funcs1.c+8−0
@@ -63,6 +63,8 @@ extern int set_output_val2(int x); · 62 unmodified lines
6363 /* here we'll force set_output_ctx2() to be __hidden in the final obj file */
6464 __hidden extern void set_output_ctx2(__u64 *ctx);
6565
66+void *bpf_cast_to_kern_ctx(void *obj) __ksym;
67+
6668 SEC("?raw_tp/sys_enter")
6769 int BPF_PROG(handler1, struct pt_regs *regs, long id)
6870 {
@@ -86,4 +88,10 @@ int BPF_PROG(handler1, struct pt_regs *regs, long id) · 17 unmodified lines
8688 return 0;
8789 }
8890
91+/* Generate BTF FUNC record and test linking with duplicate extern functions */
92+void kfunc_gen1(void)
93+{
94+ bpf_cast_to_kern_ctx(0);
95+}
96+
8997 char LICENSE[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/linked_funcs2.c+8−0
@@ -63,6 +63,8 @@ extern int set_output_val1(int x); · 62 unmodified lines
6363 /* here we'll force set_output_ctx1() to be __hidden in the final obj file */
6464 __hidden extern void set_output_ctx1(__u64 *ctx);
6565
66+void *bpf_cast_to_kern_ctx(void *obj) __ksym;
67+
6668 SEC("?raw_tp/sys_enter")
6769 int BPF_PROG(handler2, struct pt_regs *regs, long id)
6870 {
@@ -86,4 +88,10 @@ int BPF_PROG(handler2, struct pt_regs *regs, long id) · 17 unmodified lines
8688 return 0;
8789 }
8890
91+/* Generate BTF FUNC record and test linking with duplicate extern functions */
92+void kfunc_gen2(void)
93+{
94+ bpf_cast_to_kern_ctx(0);
95+}
96+
8997 char LICENSE[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/preempt_lock.c+7−7
@@ -6,7 +6,7 @@ · 5 unmodified lines
66 #include "bpf_experimental.h"
77
88 SEC("?tc")
9−__failure __msg("1 bpf_preempt_enable is missing")
9+__failure __msg("BPF_EXIT instruction cannot be used inside bpf_preempt_disable-ed region")
1010 int preempt_lock_missing_1(struct __sk_buff *ctx)
1111 {
1212 bpf_preempt_disable();
@@ -14,7 +14,7 @@ int preempt_lock_missing_1(struct __sk_buff *ctx) · 1 unmodified line
1414 }
1515
1616 SEC("?tc")
17−__failure __msg("2 bpf_preempt_enable(s) are missing")
17+__failure __msg("BPF_EXIT instruction cannot be used inside bpf_preempt_disable-ed region")
1818 int preempt_lock_missing_2(struct __sk_buff *ctx)
1919 {
2020 bpf_preempt_disable();
@@ -23,7 +23,7 @@ int preempt_lock_missing_2(struct __sk_buff *ctx) · 2 unmodified lines
2323 }
2424
2525 SEC("?tc")
26−__failure __msg("3 bpf_preempt_enable(s) are missing")
26+__failure __msg("BPF_EXIT instruction cannot be used inside bpf_preempt_disable-ed region")
2727 int preempt_lock_missing_3(struct __sk_buff *ctx)
2828 {
2929 bpf_preempt_disable();
@@ -33,7 +33,7 @@ int preempt_lock_missing_3(struct __sk_buff *ctx) · 3 unmodified lines
3333 }
3434
3535 SEC("?tc")
36−__failure __msg("1 bpf_preempt_enable is missing")
36+__failure __msg("BPF_EXIT instruction cannot be used inside bpf_preempt_disable-ed region")
3737 int preempt_lock_missing_3_minus_2(struct __sk_buff *ctx)
3838 {
3939 bpf_preempt_disable();
@@ -55,7 +55,7 @@ static __noinline void preempt_enable(void) · 15 unmodified lines
5555 }
5656
5757 SEC("?tc")
58−__failure __msg("1 bpf_preempt_enable is missing")
58+__failure __msg("BPF_EXIT instruction cannot be used inside bpf_preempt_disable-ed region")
5959 int preempt_lock_missing_1_subprog(struct __sk_buff *ctx)
6060 {
6161 preempt_disable();
@@ -63,7 +63,7 @@ int preempt_lock_missing_1_subprog(struct __sk_buff *ctx) · 1 unmodified line
6363 }
6464
6565 SEC("?tc")
66−__failure __msg("2 bpf_preempt_enable(s) are missing")
66+__failure __msg("BPF_EXIT instruction cannot be used inside bpf_preempt_disable-ed region")
6767 int preempt_lock_missing_2_subprog(struct __sk_buff *ctx)
6868 {
6969 preempt_disable();
@@ -72,7 +72,7 @@ int preempt_lock_missing_2_subprog(struct __sk_buff *ctx) · 2 unmodified lines
7272 }
7373
7474 SEC("?tc")
75−__failure __msg("1 bpf_preempt_enable is missing")
75+__failure __msg("BPF_EXIT instruction cannot be used inside bpf_preempt_disable-ed region")
7676 int preempt_lock_missing_2_minus_1_subprog(struct __sk_buff *ctx)
7777 {
7878 preempt_disable();
tools/testing/selftests/bpf/progs/raw_tp_null.c+32−0
@@ -0,0 +1,32 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
3+
4+#include <vmlinux.h>
5+#include <bpf/bpf_tracing.h>
6+
7+char _license[] SEC("license") = "GPL";
8+
9+int tid;
10+int i;
11+
12+SEC("tp_btf/bpf_testmod_test_raw_tp_null")
13+int BPF_PROG(test_raw_tp_null, struct sk_buff *skb)
14+{
15+ struct task_struct *task = bpf_get_current_task_btf();
16+
17+ if (task->pid != tid)
18+ return 0;
19+
20+ i = i + skb->mark + 1;
21+ /* The compiler may move the NULL check before this deref, which causes
22+ * the load to fail as deref of scalar. Prevent that by using a barrier.
23+ */
24+ barrier();
25+ /* If dead code elimination kicks in, the increment below will
26+ * be removed. For raw_tp programs, we mark input arguments as
27+ * PTR_MAYBE_NULL, so branch prediction should never kick in.
28+ */
29+ if (!skb)
30+ i += 2;
31+ return 0;
32+}
tools/testing/selftests/bpf/progs/struct_ops_detach.c+12−0
@@ -6,5 +6,17 @@ · 5 unmodified lines
66
77 char _license[] SEC("license") = "GPL";
88
9+/*
10+ * This subprogram validates that libbpf handles the situation in which BPF
11+ * object has subprograms in .text section, but has no entry BPF programs.
12+ * At some point that was causing issues due to legacy logic of treating such
13+ * subprogram as entry program (with unknown program type, which would fail).
14+ */
15+int dangling_subprog(void)
16+{
17+ /* do nothing, just be here */
18+ return 0;
19+}
20+
921 SEC(".struct_ops.link")
1022 struct bpf_testmod_ops testmod_do_detach;
tools/testing/selftests/bpf/progs/struct_ops_private_stack.c+62−0
@@ -0,0 +1,62 @@
1+// SPDX-License-Identifier: GPL-2.0
2+
3+#include <vmlinux.h>
4+#include <bpf/bpf_helpers.h>
5+#include <bpf/bpf_tracing.h>
6+#include "../bpf_testmod/bpf_testmod.h"
7+
8+char _license[] SEC("license") = "GPL";
9+
10+#if defined(__TARGET_ARCH_x86)
11+bool skip __attribute((__section__(".data"))) = false;
12+#else
13+bool skip = true;
14+#endif
15+
16+void bpf_testmod_ops3_call_test_2(void) __ksym;
17+
18+int val_i, val_j;
19+
20+__noinline static int subprog2(int *a, int *b)
21+{
22+ return val_i + a[10] + b[20];
23+}
24+
25+__noinline static int subprog1(int *a)
26+{
27+ /* stack size 200 bytes */
28+ int b[50] = {};
29+
30+ b[20] = 2;
31+ return subprog2(a, b);
32+}
33+
34+
35+SEC("struct_ops")
36+int BPF_PROG(test_1)
37+{
38+ /* stack size 400 bytes */
39+ int a[100] = {};
40+
41+ a[10] = 1;
42+ val_i = subprog1(a);
43+ bpf_testmod_ops3_call_test_2();
44+ return 0;
45+}
46+
47+SEC("struct_ops")
48+int BPF_PROG(test_2)
49+{
50+ /* stack size 200 bytes */
51+ int a[50] = {};
52+
53+ a[10] = 3;
54+ val_j = subprog1(a);
55+ return 0;
56+}
57+
58+SEC(".struct_ops")
59+struct bpf_testmod_ops3 testmod_1 = {
60+ .test_1 = (void *)test_1,
61+ .test_2 = (void *)test_2,
62+};
tools/testing/selftests/bpf/progs/struct_ops_private_stack_fail.c+62−0
@@ -0,0 +1,62 @@
1+// SPDX-License-Identifier: GPL-2.0
2+
3+#include <vmlinux.h>
4+#include <bpf/bpf_helpers.h>
5+#include <bpf/bpf_tracing.h>
6+#include "../bpf_testmod/bpf_testmod.h"
7+
8+char _license[] SEC("license") = "GPL";
9+
10+#if defined(__TARGET_ARCH_x86)
11+bool skip __attribute((__section__(".data"))) = false;
12+#else
13+bool skip = true;
14+#endif
15+
16+void bpf_testmod_ops3_call_test_2(void) __ksym;
17+
18+int val_i, val_j;
19+
20+__noinline static int subprog2(int *a, int *b)
21+{
22+ return val_i + a[10] + b[20];
23+}
24+
25+__noinline static int subprog1(int *a)
26+{
27+ /* stack size 200 bytes */
28+ int b[50] = {};
29+
30+ b[20] = 2;
31+ return subprog2(a, b);
32+}
33+
34+
35+SEC("struct_ops")
36+int BPF_PROG(test_1)
37+{
38+ /* stack size 100 bytes */
39+ int a[25] = {};
40+
41+ a[10] = 1;
42+ val_i = subprog1(a);
43+ bpf_testmod_ops3_call_test_2();
44+ return 0;
45+}
46+
47+SEC("struct_ops")
48+int BPF_PROG(test_2)
49+{
50+ /* stack size 400 bytes */
51+ int a[100] = {};
52+
53+ a[10] = 3;
54+ val_j = subprog1(a);
55+ return 0;
56+}
57+
58+SEC(".struct_ops")
59+struct bpf_testmod_ops3 testmod_1 = {
60+ .test_1 = (void *)test_1,
61+ .test_2 = (void *)test_2,
62+};
tools/testing/selftests/bpf/progs/struct_ops_private_stack_recur.c+50−0
@@ -0,0 +1,50 @@
1+// SPDX-License-Identifier: GPL-2.0
2+
3+#include <vmlinux.h>
4+#include <bpf/bpf_helpers.h>
5+#include <bpf/bpf_tracing.h>
6+#include "../bpf_testmod/bpf_testmod.h"
7+
8+char _license[] SEC("license") = "GPL";
9+
10+#if defined(__TARGET_ARCH_x86)
11+bool skip __attribute((__section__(".data"))) = false;
12+#else
13+bool skip = true;
14+#endif
15+
16+void bpf_testmod_ops3_call_test_1(void) __ksym;
17+
18+int val_i, val_j;
19+
20+__noinline static int subprog2(int *a, int *b)
21+{
22+ return val_i + a[1] + b[20];
23+}
24+
25+__noinline static int subprog1(int *a)
26+{
27+ /* stack size 400 bytes */
28+ int b[100] = {};
29+
30+ b[20] = 2;
31+ return subprog2(a, b);
32+}
33+
34+
35+SEC("struct_ops")
36+int BPF_PROG(test_1)
37+{
38+ /* stack size 20 bytes */
39+ int a[5] = {};
40+
41+ a[1] = 1;
42+ val_j += subprog1(a);
43+ bpf_testmod_ops3_call_test_1();
44+ return 0;
45+}
46+
47+SEC(".struct_ops")
48+struct bpf_testmod_ops3 testmod_1 = {
49+ .test_1 = (void *)test_1,
50+};
tools/testing/selftests/bpf/progs/tailcall_fail.c+64−0
@@ -0,0 +1,64 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
3+#include <vmlinux.h>
4+#include <bpf/bpf_tracing.h>
5+#include <bpf/bpf_helpers.h>
6+#include <bpf/bpf_core_read.h>
7+
8+#include "bpf_misc.h"
9+#include "bpf_experimental.h"
10+
11+extern void bpf_rcu_read_lock(void) __ksym;
12+extern void bpf_rcu_read_unlock(void) __ksym;
13+
14+#define private(name) SEC(".bss." #name) __hidden __attribute__((aligned(8)))
15+
16+private(A) struct bpf_spin_lock lock;
17+
18+struct {
19+ __uint(type, BPF_MAP_TYPE_PROG_ARRAY);
20+ __uint(max_entries, 3);
21+ __uint(key_size, sizeof(__u32));
22+ __uint(value_size, sizeof(__u32));
23+} jmp_table SEC(".maps");
24+
25+SEC("?tc")
26+__failure __msg("function calls are not allowed while holding a lock")
27+int reject_tail_call_spin_lock(struct __sk_buff *ctx)
28+{
29+ bpf_spin_lock(&lock);
30+ bpf_tail_call_static(ctx, &jmp_table, 0);
31+ return 0;
32+}
33+
34+SEC("?tc")
35+__failure __msg("tail_call cannot be used inside bpf_rcu_read_lock-ed region")
36+int reject_tail_call_rcu_lock(struct __sk_buff *ctx)
37+{
38+ bpf_rcu_read_lock();
39+ bpf_tail_call_static(ctx, &jmp_table, 0);
40+ bpf_rcu_read_unlock();
41+ return 0;
42+}
43+
44+SEC("?tc")
45+__failure __msg("tail_call cannot be used inside bpf_preempt_disable-ed region")
46+int reject_tail_call_preempt_lock(struct __sk_buff *ctx)
47+{
48+ bpf_guard_preempt();
49+ bpf_tail_call_static(ctx, &jmp_table, 0);
50+ return 0;
51+}
52+
53+SEC("?tc")
54+__failure __msg("tail_call would lead to reference leak")
55+int reject_tail_call_ref(struct __sk_buff *ctx)
56+{
57+ struct foo { int i; } *p;
58+
59+ p = bpf_obj_new(typeof(*p));
60+ bpf_tail_call_static(ctx, &jmp_table, 0);
61+ return 0;
62+}
63+
64+char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/task_kfunc_common.h+1−0
@@ -23,6 +23,7 @@ struct { · 22 unmodified lines
2323 struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym;
2424 void bpf_task_release(struct task_struct *p) __ksym;
2525 struct task_struct *bpf_task_from_pid(s32 pid) __ksym;
26+struct task_struct *bpf_task_from_vpid(s32 vpid) __ksym;
2627 void bpf_rcu_read_lock(void) __ksym;
2728 void bpf_rcu_read_unlock(void) __ksym;
2829
tools/testing/selftests/bpf/progs/task_kfunc_failure.c+14−0
@@ -247,6 +247,20 @@ int BPF_PROG(task_kfunc_from_pid_no_null_check, struct task_struct *task, u64 cl · 246 unmodified lines
247247 return 0;
248248 }
249249
250+SEC("tp_btf/task_newtask")
251+__failure __msg("Possibly NULL pointer passed to trusted arg0")
252+int BPF_PROG(task_kfunc_from_vpid_no_null_check, struct task_struct *task, u64 clone_flags)
253+{
254+ struct task_struct *acquired;
255+
256+ acquired = bpf_task_from_vpid(task->pid);
257+
258+ /* Releasing bpf_task_from_vpid() lookup without a NULL check. */
259+ bpf_task_release(acquired);
260+
261+ return 0;
262+}
263+
250264 SEC("lsm/task_free")
251265 __failure __msg("R1 must be a rcu pointer")
252266 int BPF_PROG(task_kfunc_from_lsm_task_free, struct task_struct *task)
tools/testing/selftests/bpf/progs/task_kfunc_success.c+51−0
@@ -366,3 +366,54 @@ int BPF_PROG(task_kfunc_acquire_trusted_walked, struct task_struct *task, u64 cl · 365 unmodified lines
366366
367367 return 0;
368368 }
369+
370+SEC("syscall")
371+int test_task_from_vpid_current(const void *ctx)
372+{
373+ struct task_struct *current, *v_task;
374+
375+ v_task = bpf_task_from_vpid(1);
376+ if (!v_task) {
377+ err = 1;
378+ return 0;
379+ }
380+
381+ current = bpf_get_current_task_btf();
382+
383+ /* The current process should be the init process (pid 1) in the new pid namespace. */
384+ if (current != v_task)
385+ err = 2;
386+
387+ bpf_task_release(v_task);
388+ return 0;
389+}
390+
391+SEC("syscall")
392+int test_task_from_vpid_invalid(const void *ctx)
393+{
394+ struct task_struct *v_task;
395+
396+ v_task = bpf_task_from_vpid(-1);
397+ if (v_task) {
398+ err = 1;
399+ goto err;
400+ }
401+
402+ /* There should be only one process (current process) in the new pid namespace. */
403+ v_task = bpf_task_from_vpid(2);
404+ if (v_task) {
405+ err = 2;
406+ goto err;
407+ }
408+
409+ v_task = bpf_task_from_vpid(9999);
410+ if (v_task) {
411+ err = 3;
412+ goto err;
413+ }
414+
415+ return 0;
416+err:
417+ bpf_task_release(v_task);
418+ return 0;
419+}
tools/testing/selftests/bpf/progs/task_ls_uptr.c+63−0
@@ -0,0 +1,63 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
3+
4+#include <vmlinux.h>
5+#include <bpf/bpf_helpers.h>
6+#include "uptr_test_common.h"
7+
8+struct task_struct *bpf_task_from_pid(s32 pid) __ksym;
9+void bpf_task_release(struct task_struct *p) __ksym;
10+void bpf_cgroup_release(struct cgroup *cgrp) __ksym;
11+
12+struct {
13+ __uint(type, BPF_MAP_TYPE_TASK_STORAGE);
14+ __uint(map_flags, BPF_F_NO_PREALLOC);
15+ __type(key, int);
16+ __type(value, struct value_type);
17+} datamap SEC(".maps");
18+
19+pid_t target_pid = 0;
20+pid_t parent_pid = 0;
21+
22+SEC("tp_btf/sys_enter")
23+int on_enter(__u64 *ctx)
24+{
25+ struct task_struct *task, *data_task;
26+ struct value_type *ptr;
27+ struct user_data *udata;
28+ struct cgroup *cgrp;
29+
30+ task = bpf_get_current_task_btf();
31+ if (task->pid != target_pid)
32+ return 0;
33+
34+ data_task = bpf_task_from_pid(parent_pid);
35+ if (!data_task)
36+ return 0;
37+
38+ ptr = bpf_task_storage_get(&datamap, data_task, 0, 0);
39+ bpf_task_release(data_task);
40+ if (!ptr)
41+ return 0;
42+
43+ cgrp = bpf_kptr_xchg(&ptr->cgrp, NULL);
44+ if (cgrp) {
45+ int lvl = cgrp->level;
46+
47+ bpf_cgroup_release(cgrp);
48+ return lvl;
49+ }
50+
51+ udata = ptr->udata;
52+ if (!udata || udata->result)
53+ return 0;
54+ udata->result = MAGIC_VALUE + udata->a + udata->b;
55+
56+ udata = ptr->nested.udata;
57+ if (udata && !udata->nested_result)
58+ udata->nested_result = udata->result;
59+
60+ return 0;
61+}
62+
63+char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/tc_bpf2bpf.c+3−2
@@ -5,10 +5,11 @@ · 4 unmodified lines
55 #include "bpf_misc.h"
66
77 __noinline
8−int subprog(struct __sk_buff *skb)
8+int subprog_tc(struct __sk_buff *skb)
99 {
1010 int ret = 1;
1111
12+ __sink(skb);
1213 __sink(ret);
1314 return ret;
1415 }
@@ -16,7 +17,7 @@ int subprog(struct __sk_buff *skb) · 1 unmodified line
1617 SEC("tc")
1718 int entry_tc(struct __sk_buff *skb)
1819 {
19− return subprog(skb);
20+ return subprog_tc(skb);
2021 }
2122
2223 char __license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/test_send_signal_kern.c+29−6
@@ -1,27 +1,50 @@
11 // SPDX-License-Identifier: GPL-2.0
22 // Copyright (c) 2019 Facebook
3−#include <linux/bpf.h>
3+#include <vmlinux.h>
44 #include <linux/version.h>
55 #include <bpf/bpf_helpers.h>
66
7−__u32 sig = 0, pid = 0, status = 0, signal_thread = 0;
7+struct task_struct *bpf_task_from_pid(int pid) __ksym;
8+void bpf_task_release(struct task_struct *p) __ksym;
9+int bpf_send_signal_task(struct task_struct *task, int sig, enum pid_type type, u64 value) __ksym;
10+
11+__u32 sig = 0, pid = 0, status = 0, signal_thread = 0, target_pid = 0;
812
913 static __always_inline int bpf_send_signal_test(void *ctx)
1014 {
15+ struct task_struct *target_task = NULL;
1116 int ret;
17+ u64 value;
1218
1319 if (status != 0 || pid == 0)
1420 return 0;
1521
1622 if ((bpf_get_current_pid_tgid() >> 32) == pid) {
17− if (signal_thread)
18− ret = bpf_send_signal_thread(sig);
19− else
20− ret = bpf_send_signal(sig);
23+ if (target_pid) {
24+ target_task = bpf_task_from_pid(target_pid);
25+ if (!target_task)
26+ return 0;
27+ value = 8;
28+ }
29+
30+ if (signal_thread) {
31+ if (target_pid)
32+ ret = bpf_send_signal_task(target_task, sig, PIDTYPE_PID, value);
33+ else
34+ ret = bpf_send_signal_thread(sig);
35+ } else {
36+ if (target_pid)
37+ ret = bpf_send_signal_task(target_task, sig, PIDTYPE_TGID, value);
38+ else
39+ ret = bpf_send_signal(sig);
40+ }
2141 if (ret == 0)
2242 status = 1;
2343 }
2444
45+ if (target_task)
46+ bpf_task_release(target_task);
47+
2548 return 0;
2649 }
2750
tools/testing/selftests/bpf/progs/test_spin_lock_fail.c+2−2
@@ -28,8 +28,8 @@ struct { · 27 unmodified lines
2828 },
2929 };
3030
31−SEC(".data.A") struct bpf_spin_lock lockA;
32−SEC(".data.B") struct bpf_spin_lock lockB;
31+static struct bpf_spin_lock lockA SEC(".data.A");
32+static struct bpf_spin_lock lockB SEC(".data.B");
3333
3434 SEC("?tc")
3535 int lock_id_kptr_preserve(void *ctx)
tools/testing/selftests/bpf/progs/test_tp_btf_nullable.c+5−1
@@ -7,7 +7,11 @@ · 6 unmodified lines
77 #include "bpf_misc.h"
88
99 SEC("tp_btf/bpf_testmod_test_nullable_bare")
10−__failure __msg("R1 invalid mem access 'trusted_ptr_or_null_'")
10+/* This used to be a failure test, but raw_tp nullable arguments can now
11+ * directly be dereferenced, whether they have nullable annotation or not,
12+ * and don't need to be explicitly checked.
13+ */
14+__success
1115 int BPF_PROG(handle_tp_btf_nullable_bare1, struct bpf_testmod_test_read_ctx *nullable_ctx)
1216 {
1317 return nullable_ctx->len;
tools/testing/selftests/bpf/progs/update_map_in_htab.c+30−0
@@ -0,0 +1,30 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/* Copyright (C) 2024. Huawei Technologies Co., Ltd */
3+#include <linux/bpf.h>
4+#include <bpf/bpf_helpers.h>
5+
6+struct inner_map_type {
7+ __uint(type, BPF_MAP_TYPE_ARRAY);
8+ __uint(key_size, 4);
9+ __uint(value_size, 4);
10+ __uint(max_entries, 1);
11+} inner_map SEC(".maps");
12+
13+struct {
14+ __uint(type, BPF_MAP_TYPE_HASH_OF_MAPS);
15+ __type(key, int);
16+ __type(value, int);
17+ __uint(max_entries, 2);
18+ __array(values, struct inner_map_type);
19+} outer_htab_map SEC(".maps");
20+
21+struct {
22+ __uint(type, BPF_MAP_TYPE_HASH_OF_MAPS);
23+ __uint(map_flags, BPF_F_NO_PREALLOC);
24+ __type(key, int);
25+ __type(value, int);
26+ __uint(max_entries, 2);
27+ __array(values, struct inner_map_type);
28+} outer_alloc_htab_map SEC(".maps");
29+
30+char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/uprobe_multi_consumers.c+3−3
@@ -24,16 +24,16 @@ int uprobe_1(struct pt_regs *ctx) · 23 unmodified lines
2424 return 0;
2525 }
2626
27−SEC("uprobe.multi")
27+SEC("uprobe.session")
2828 int uprobe_2(struct pt_regs *ctx)
2929 {
3030 uprobe_result[2]++;
3131 return 0;
3232 }
3333
34−SEC("uprobe.multi")
34+SEC("uprobe.session")
3535 int uprobe_3(struct pt_regs *ctx)
3636 {
3737 uprobe_result[3]++;
38− return 0;
38+ return 1;
3939 }
tools/testing/selftests/bpf/progs/uprobe_multi_session.c+71−0
@@ -0,0 +1,71 @@
1+// SPDX-License-Identifier: GPL-2.0
2+#include <linux/bpf.h>
3+#include <bpf/bpf_helpers.h>
4+#include <bpf/bpf_tracing.h>
5+#include <stdbool.h>
6+#include "bpf_kfuncs.h"
7+#include "bpf_misc.h"
8+
9+char _license[] SEC("license") = "GPL";
10+
11+__u64 uprobe_multi_func_1_addr = 0;
12+__u64 uprobe_multi_func_2_addr = 0;
13+__u64 uprobe_multi_func_3_addr = 0;
14+
15+__u64 uprobe_session_result[3] = {};
16+__u64 uprobe_multi_sleep_result = 0;
17+
18+void *user_ptr = 0;
19+int pid = 0;
20+
21+static int uprobe_multi_check(void *ctx, bool is_return)
22+{
23+ const __u64 funcs[] = {
24+ uprobe_multi_func_1_addr,
25+ uprobe_multi_func_2_addr,
26+ uprobe_multi_func_3_addr,
27+ };
28+ unsigned int i;
29+ __u64 addr;
30+
31+ if (bpf_get_current_pid_tgid() >> 32 != pid)
32+ return 1;
33+
34+ addr = bpf_get_func_ip(ctx);
35+
36+ for (i = 0; i < ARRAY_SIZE(funcs); i++) {
37+ if (funcs[i] == addr) {
38+ uprobe_session_result[i]++;
39+ break;
40+ }
41+ }
42+
43+ /* only uprobe_multi_func_2 executes return probe */
44+ if ((addr == uprobe_multi_func_1_addr) ||
45+ (addr == uprobe_multi_func_3_addr))
46+ return 1;
47+
48+ return 0;
49+}
50+
51+SEC("uprobe.session//proc/self/exe:uprobe_multi_func_*")
52+int uprobe(struct pt_regs *ctx)
53+{
54+ return uprobe_multi_check(ctx, bpf_session_is_return());
55+}
56+
57+static __always_inline bool verify_sleepable_user_copy(void)
58+{
59+ char data[9];
60+
61+ bpf_copy_from_user(data, sizeof(data), user_ptr);
62+ return bpf_strncmp(data, sizeof(data), "test_data") == 0;
63+}
64+
65+SEC("uprobe.session.s//proc/self/exe:uprobe_multi_func_*")
66+int uprobe_sleepable(struct pt_regs *ctx)
67+{
68+ if (verify_sleepable_user_copy())
69+ uprobe_multi_sleep_result++;
70+ return uprobe_multi_check(ctx, bpf_session_is_return());
71+}
tools/testing/selftests/bpf/progs/uprobe_multi_session_cookie.c+48−0
@@ -0,0 +1,48 @@
1+// SPDX-License-Identifier: GPL-2.0
2+#include <linux/bpf.h>
3+#include <bpf/bpf_helpers.h>
4+#include <bpf/bpf_tracing.h>
5+#include <stdbool.h>
6+#include "bpf_kfuncs.h"
7+
8+char _license[] SEC("license") = "GPL";
9+
10+int pid = 0;
11+
12+__u64 test_uprobe_1_result = 0;
13+__u64 test_uprobe_2_result = 0;
14+__u64 test_uprobe_3_result = 0;
15+
16+static int check_cookie(__u64 val, __u64 *result)
17+{
18+ __u64 *cookie;
19+
20+ if (bpf_get_current_pid_tgid() >> 32 != pid)
21+ return 1;
22+
23+ cookie = bpf_session_cookie();
24+
25+ if (bpf_session_is_return())
26+ *result = *cookie == val ? val : 0;
27+ else
28+ *cookie = val;
29+ return 0;
30+}
31+
32+SEC("uprobe.session//proc/self/exe:uprobe_multi_func_1")
33+int uprobe_1(struct pt_regs *ctx)
34+{
35+ return check_cookie(1, &test_uprobe_1_result);
36+}
37+
38+SEC("uprobe.session//proc/self/exe:uprobe_multi_func_2")
39+int uprobe_2(struct pt_regs *ctx)
40+{
41+ return check_cookie(2, &test_uprobe_2_result);
42+}
43+
44+SEC("uprobe.session//proc/self/exe:uprobe_multi_func_3")
45+int uprobe_3(struct pt_regs *ctx)
46+{
47+ return check_cookie(3, &test_uprobe_3_result);
48+}
tools/testing/selftests/bpf/progs/uprobe_multi_session_recursive.c+44−0
@@ -0,0 +1,44 @@
1+// SPDX-License-Identifier: GPL-2.0
2+#include <linux/bpf.h>
3+#include <bpf/bpf_helpers.h>
4+#include <bpf/bpf_tracing.h>
5+#include <stdbool.h>
6+#include "bpf_kfuncs.h"
7+#include "bpf_misc.h"
8+
9+char _license[] SEC("license") = "GPL";
10+
11+int pid = 0;
12+
13+int idx_entry = 0;
14+int idx_return = 0;
15+
16+__u64 test_uprobe_cookie_entry[6];
17+__u64 test_uprobe_cookie_return[3];
18+
19+static int check_cookie(void)
20+{
21+ __u64 *cookie = bpf_session_cookie();
22+
23+ if (bpf_session_is_return()) {
24+ if (idx_return >= ARRAY_SIZE(test_uprobe_cookie_return))
25+ return 1;
26+ test_uprobe_cookie_return[idx_return++] = *cookie;
27+ return 0;
28+ }
29+
30+ if (idx_entry >= ARRAY_SIZE(test_uprobe_cookie_entry))
31+ return 1;
32+ *cookie = test_uprobe_cookie_entry[idx_entry];
33+ return idx_entry++ % 2;
34+}
35+
36+
37+SEC("uprobe.session//proc/self/exe:uprobe_session_recursive")
38+int uprobe_recursive(struct pt_regs *ctx)
39+{
40+ if (bpf_get_current_pid_tgid() >> 32 != pid)
41+ return 1;
42+
43+ return check_cookie();
44+}
tools/testing/selftests/bpf/progs/uprobe_multi_session_single.c+44−0
@@ -0,0 +1,44 @@
1+// SPDX-License-Identifier: GPL-2.0
2+#include <linux/bpf.h>
3+#include <bpf/bpf_helpers.h>
4+#include <bpf/bpf_tracing.h>
5+#include <stdbool.h>
6+#include "bpf_kfuncs.h"
7+#include "bpf_misc.h"
8+
9+char _license[] SEC("license") = "GPL";
10+
11+__u64 uprobe_session_result[3] = {};
12+int pid = 0;
13+
14+static int uprobe_multi_check(void *ctx, int idx)
15+{
16+ if (bpf_get_current_pid_tgid() >> 32 != pid)
17+ return 1;
18+
19+ uprobe_session_result[idx]++;
20+
21+ /* only consumer 1 executes return probe */
22+ if (idx == 0 || idx == 2)
23+ return 1;
24+
25+ return 0;
26+}
27+
28+SEC("uprobe.session//proc/self/exe:uprobe_multi_func_1")
29+int uprobe_0(struct pt_regs *ctx)
30+{
31+ return uprobe_multi_check(ctx, 0);
32+}
33+
34+SEC("uprobe.session//proc/self/exe:uprobe_multi_func_1")
35+int uprobe_1(struct pt_regs *ctx)
36+{
37+ return uprobe_multi_check(ctx, 1);
38+}
39+
40+SEC("uprobe.session//proc/self/exe:uprobe_multi_func_1")
41+int uprobe_2(struct pt_regs *ctx)
42+{
43+ return uprobe_multi_check(ctx, 2);
44+}
tools/testing/selftests/bpf/progs/uprobe_multi_verifier.c+31−0
@@ -0,0 +1,31 @@
1+// SPDX-License-Identifier: GPL-2.0
2+#include "vmlinux.h"
3+#include <bpf/bpf_helpers.h>
4+#include <bpf/bpf_tracing.h>
5+#include <bpf/usdt.bpf.h>
6+#include "bpf_misc.h"
7+
8+char _license[] SEC("license") = "GPL";
9+
10+
11+SEC("uprobe.session")
12+__success
13+int uprobe_sesison_return_0(struct pt_regs *ctx)
14+{
15+ return 0;
16+}
17+
18+SEC("uprobe.session")
19+__success
20+int uprobe_sesison_return_1(struct pt_regs *ctx)
21+{
22+ return 1;
23+}
24+
25+SEC("uprobe.session")
26+__failure
27+__msg("At program exit the register R0 has smin=2 smax=2 should have been in [0, 1]")
28+int uprobe_sesison_return_2(struct pt_regs *ctx)
29+{
30+ return 2;
31+}
tools/testing/selftests/bpf/progs/uptr_failure.c+105−0
@@ -0,0 +1,105 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
3+
4+#include <vmlinux.h>
5+#include <bpf/bpf_helpers.h>
6+#include "bpf_experimental.h"
7+#include "bpf_misc.h"
8+#include "uptr_test_common.h"
9+
10+struct {
11+ __uint(type, BPF_MAP_TYPE_TASK_STORAGE);
12+ __uint(map_flags, BPF_F_NO_PREALLOC);
13+ __type(key, int);
14+ __type(value, struct value_type);
15+} datamap SEC(".maps");
16+
17+SEC("?syscall")
18+__failure __msg("store to uptr disallowed")
19+int uptr_write(const void *ctx)
20+{
21+ struct task_struct *task;
22+ struct value_type *v;
23+
24+ task = bpf_get_current_task_btf();
25+ v = bpf_task_storage_get(&datamap, task, 0,
26+ BPF_LOCAL_STORAGE_GET_F_CREATE);
27+ if (!v)
28+ return 0;
29+
30+ v->udata = NULL;
31+ return 0;
32+}
33+
34+SEC("?syscall")
35+__failure __msg("store to uptr disallowed")
36+int uptr_write_nested(const void *ctx)
37+{
38+ struct task_struct *task;
39+ struct value_type *v;
40+
41+ task = bpf_get_current_task_btf();
42+ v = bpf_task_storage_get(&datamap, task, 0,
43+ BPF_LOCAL_STORAGE_GET_F_CREATE);
44+ if (!v)
45+ return 0;
46+
47+ v->nested.udata = NULL;
48+ return 0;
49+}
50+
51+SEC("?syscall")
52+__failure __msg("R1 invalid mem access 'mem_or_null'")
53+int uptr_no_null_check(const void *ctx)
54+{
55+ struct task_struct *task;
56+ struct value_type *v;
57+
58+ task = bpf_get_current_task_btf();
59+ v = bpf_task_storage_get(&datamap, task, 0,
60+ BPF_LOCAL_STORAGE_GET_F_CREATE);
61+ if (!v)
62+ return 0;
63+
64+ v->udata->result = 0;
65+
66+ return 0;
67+}
68+
69+SEC("?syscall")
70+__failure __msg("doesn't point to kptr")
71+int uptr_kptr_xchg(const void *ctx)
72+{
73+ struct task_struct *task;
74+ struct value_type *v;
75+
76+ task = bpf_get_current_task_btf();
77+ v = bpf_task_storage_get(&datamap, task, 0,
78+ BPF_LOCAL_STORAGE_GET_F_CREATE);
79+ if (!v)
80+ return 0;
81+
82+ bpf_kptr_xchg(&v->udata, NULL);
83+
84+ return 0;
85+}
86+
87+SEC("?syscall")
88+__failure __msg("invalid mem access 'scalar'")
89+int uptr_obj_new(const void *ctx)
90+{
91+ struct value_type *v;
92+
93+ v = bpf_obj_new(typeof(*v));
94+ if (!v)
95+ return 0;
96+
97+ if (v->udata)
98+ v->udata->result = 0;
99+
100+ bpf_obj_drop(v);
101+
102+ return 0;
103+}
104+
105+char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/uptr_map_failure.c+27−0
@@ -0,0 +1,27 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
3+
4+#include <vmlinux.h>
5+#include <bpf/bpf_helpers.h>
6+#include "uptr_test_common.h"
7+
8+struct {
9+ __uint(type, BPF_MAP_TYPE_TASK_STORAGE);
10+ __uint(map_flags, BPF_F_NO_PREALLOC);
11+ __type(key, int);
12+ __type(value, struct large_uptr);
13+} large_uptr_map SEC(".maps");
14+
15+struct {
16+ __uint(type, BPF_MAP_TYPE_TASK_STORAGE);
17+ __uint(map_flags, BPF_F_NO_PREALLOC);
18+ __type(key, int);
19+ __type(value, struct empty_uptr);
20+} empty_uptr_map SEC(".maps");
21+
22+struct {
23+ __uint(type, BPF_MAP_TYPE_TASK_STORAGE);
24+ __uint(map_flags, BPF_F_NO_PREALLOC);
25+ __type(key, int);
26+ __type(value, struct kstruct_uptr);
27+} kstruct_uptr_map SEC(".maps");
tools/testing/selftests/bpf/progs/uptr_update_failure.c+42−0
@@ -0,0 +1,42 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
3+
4+#include <vmlinux.h>
5+#include <bpf/bpf_helpers.h>
6+#include "uptr_test_common.h"
7+
8+struct {
9+ __uint(type, BPF_MAP_TYPE_TASK_STORAGE);
10+ __uint(map_flags, BPF_F_NO_PREALLOC);
11+ __type(key, int);
12+ __type(value, struct value_lock_type);
13+} datamap SEC(".maps");
14+
15+/* load test only. not used */
16+SEC("syscall")
17+int not_used(void *ctx)
18+{
19+ struct value_lock_type *ptr;
20+ struct task_struct *task;
21+ struct user_data *udata;
22+
23+ task = bpf_get_current_task_btf();
24+ ptr = bpf_task_storage_get(&datamap, task, 0, 0);
25+ if (!ptr)
26+ return 0;
27+
28+ bpf_spin_lock(&ptr->lock);
29+
30+ udata = ptr->udata;
31+ if (!udata) {
32+ bpf_spin_unlock(&ptr->lock);
33+ return 0;
34+ }
35+ udata->result = MAGIC_VALUE + udata->a + udata->b;
36+
37+ bpf_spin_unlock(&ptr->lock);
38+
39+ return 0;
40+}
41+
42+char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/verifier_arena_large.c+108−2
@@ -29,12 +29,12 @@ int big_alloc1(void *ctx) · 28 unmodified lines
2929 if (!page1)
3030 return 1;
3131 *page1 = 1;
32− page2 = bpf_arena_alloc_pages(&arena, base + ARENA_SIZE - PAGE_SIZE,
32+ page2 = bpf_arena_alloc_pages(&arena, base + ARENA_SIZE - PAGE_SIZE * 2,
3333 1, NUMA_NO_NODE, 0);
3434 if (!page2)
3535 return 2;
3636 *page2 = 2;
37− no_page = bpf_arena_alloc_pages(&arena, base + ARENA_SIZE,
37+ no_page = bpf_arena_alloc_pages(&arena, base + ARENA_SIZE - PAGE_SIZE,
3838 1, NUMA_NO_NODE, 0);
3939 if (no_page)
4040 return 3;
@@ -66,4 +66,110 @@ int big_alloc1(void *ctx) · 25 unmodified lines
6666 #endif
6767 return 0;
6868 }
69+
70+#if defined(__BPF_FEATURE_ADDR_SPACE_CAST)
71+#define PAGE_CNT 100
72+__u8 __arena * __arena page[PAGE_CNT]; /* occupies the first page */
73+__u8 __arena *base;
74+
75+/*
76+ * Check that arena's range_tree algorithm allocates pages sequentially
77+ * on the first pass and then fills in all gaps on the second pass.
78+ */
79+__noinline int alloc_pages(int page_cnt, int pages_atonce, bool first_pass,
80+ int max_idx, int step)
81+{
82+ __u8 __arena *pg;
83+ int i, pg_idx;
84+
85+ for (i = 0; i < page_cnt; i++) {
86+ pg = bpf_arena_alloc_pages(&arena, NULL, pages_atonce,
87+ NUMA_NO_NODE, 0);
88+ if (!pg)
89+ return step;
90+ pg_idx = (unsigned long) (pg - base) / PAGE_SIZE;
91+ if (first_pass) {
92+ /* Pages must be allocated sequentially */
93+ if (pg_idx != i)
94+ return step + 100;
95+ } else {
96+ /* Allocator must fill into gaps */
97+ if (pg_idx >= max_idx || (pg_idx & 1))
98+ return step + 200;
99+ }
100+ *pg = pg_idx;
101+ page[pg_idx] = pg;
102+ cond_break;
103+ }
104+ return 0;
105+}
106+
107+SEC("syscall")
108+__success __retval(0)
109+int big_alloc2(void *ctx)
110+{
111+ __u8 __arena *pg;
112+ int i, err;
113+
114+ base = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0);
115+ if (!base)
116+ return 1;
117+ bpf_arena_free_pages(&arena, (void __arena *)base, 1);
118+
119+ err = alloc_pages(PAGE_CNT, 1, true, PAGE_CNT, 2);
120+ if (err)
121+ return err;
122+
123+ /* Clear all even pages */
124+ for (i = 0; i < PAGE_CNT; i += 2) {
125+ pg = page[i];
126+ if (*pg != i)
127+ return 3;
128+ bpf_arena_free_pages(&arena, (void __arena *)pg, 1);
129+ page[i] = NULL;
130+ cond_break;
131+ }
132+
133+ /* Allocate into freed gaps */
134+ err = alloc_pages(PAGE_CNT / 2, 1, false, PAGE_CNT, 4);
135+ if (err)
136+ return err;
137+
138+ /* Free pairs of pages */
139+ for (i = 0; i < PAGE_CNT; i += 4) {
140+ pg = page[i];
141+ if (*pg != i)
142+ return 5;
143+ bpf_arena_free_pages(&arena, (void __arena *)pg, 2);
144+ page[i] = NULL;
145+ page[i + 1] = NULL;
146+ cond_break;
147+ }
148+
149+ /* Allocate 2 pages at a time into freed gaps */
150+ err = alloc_pages(PAGE_CNT / 4, 2, false, PAGE_CNT, 6);
151+ if (err)
152+ return err;
153+
154+ /* Check pages without freeing */
155+ for (i = 0; i < PAGE_CNT; i += 2) {
156+ pg = page[i];
157+ if (*pg != i)
158+ return 7;
159+ cond_break;
160+ }
161+
162+ pg = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0);
163+
164+ if (!pg)
165+ return 8;
166+ /*
167+ * The first PAGE_CNT pages are occupied. The new page
168+ * must be above.
169+ */
170+ if ((pg - base) / PAGE_SIZE < PAGE_CNT)
171+ return 9;
172+ return 0;
173+}
174+#endif
69175 char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/verifier_private_stack.c+272−0
@@ -0,0 +1,272 @@
1+// SPDX-License-Identifier: GPL-2.0
2+
3+#include <vmlinux.h>
4+#include <bpf/bpf_helpers.h>
5+#include "bpf_misc.h"
6+#include "bpf_experimental.h"
7+
8+/* From include/linux/filter.h */
9+#define MAX_BPF_STACK 512
10+
11+#if defined(__TARGET_ARCH_x86)
12+
13+struct elem {
14+ struct bpf_timer t;
15+ char pad[256];
16+};
17+
18+struct {
19+ __uint(type, BPF_MAP_TYPE_ARRAY);
20+ __uint(max_entries, 1);
21+ __type(key, int);
22+ __type(value, struct elem);
23+} array SEC(".maps");
24+
25+SEC("kprobe")
26+__description("Private stack, single prog")
27+__success
28+__arch_x86_64
29+__jited(" movabsq $0x{{.*}}, %r9")
30+__jited(" addq %gs:0x{{.*}}, %r9")
31+__jited(" movl $0x2a, %edi")
32+__jited(" movq %rdi, -0x100(%r9)")
33+__naked void private_stack_single_prog(void)
34+{
35+ asm volatile (" \
36+ r1 = 42; \
37+ *(u64 *)(r10 - 256) = r1; \
38+ r0 = 0; \
39+ exit; \
40+" ::: __clobber_all);
41+}
42+
43+SEC("raw_tp")
44+__description("No private stack")
45+__success
46+__arch_x86_64
47+__jited(" subq $0x8, %rsp")
48+__naked void no_private_stack_nested(void)
49+{
50+ asm volatile (" \
51+ r1 = 42; \
52+ *(u64 *)(r10 - 8) = r1; \
53+ r0 = 0; \
54+ exit; \
55+" ::: __clobber_all);
56+}
57+
58+__used
59+__naked static void cumulative_stack_depth_subprog(void)
60+{
61+ asm volatile (" \
62+ r1 = 41; \
63+ *(u64 *)(r10 - 32) = r1; \
64+ call %[bpf_get_smp_processor_id]; \
65+ exit; \
66+" :
67+ : __imm(bpf_get_smp_processor_id)
68+ : __clobber_all);
69+}
70+
71+SEC("kprobe")
72+__description("Private stack, subtree > MAX_BPF_STACK")
73+__success
74+__arch_x86_64
75+/* private stack fp for the main prog */
76+__jited(" movabsq $0x{{.*}}, %r9")
77+__jited(" addq %gs:0x{{.*}}, %r9")
78+__jited(" movl $0x2a, %edi")
79+__jited(" movq %rdi, -0x200(%r9)")
80+__jited(" pushq %r9")
81+__jited(" callq 0x{{.*}}")
82+__jited(" popq %r9")
83+__jited(" xorl %eax, %eax")
84+__naked void private_stack_nested_1(void)
85+{
86+ asm volatile (" \
87+ r1 = 42; \
88+ *(u64 *)(r10 - %[max_bpf_stack]) = r1; \
89+ call cumulative_stack_depth_subprog; \
90+ r0 = 0; \
91+ exit; \
92+" :
93+ : __imm_const(max_bpf_stack, MAX_BPF_STACK)
94+ : __clobber_all);
95+}
96+
97+__naked __noinline __used
98+static unsigned long loop_callback(void)
99+{
100+ asm volatile (" \
101+ call %[bpf_get_prandom_u32]; \
102+ r1 = 42; \
103+ *(u64 *)(r10 - 512) = r1; \
104+ call cumulative_stack_depth_subprog; \
105+ r0 = 0; \
106+ exit; \
107+" :
108+ : __imm(bpf_get_prandom_u32)
109+ : __clobber_common);
110+}
111+
112+SEC("raw_tp")
113+__description("Private stack, callback")
114+__success
115+__arch_x86_64
116+/* for func loop_callback */
117+__jited("func #1")
118+__jited(" endbr64")
119+__jited(" nopl (%rax,%rax)")
120+__jited(" nopl (%rax)")
121+__jited(" pushq %rbp")
122+__jited(" movq %rsp, %rbp")
123+__jited(" endbr64")
124+__jited(" movabsq $0x{{.*}}, %r9")
125+__jited(" addq %gs:0x{{.*}}, %r9")
126+__jited(" pushq %r9")
127+__jited(" callq")
128+__jited(" popq %r9")
129+__jited(" movl $0x2a, %edi")
130+__jited(" movq %rdi, -0x200(%r9)")
131+__jited(" pushq %r9")
132+__jited(" callq")
133+__jited(" popq %r9")
134+__naked void private_stack_callback(void)
135+{
136+ asm volatile (" \
137+ r1 = 1; \
138+ r2 = %[loop_callback]; \
139+ r3 = 0; \
140+ r4 = 0; \
141+ call %[bpf_loop]; \
142+ r0 = 0; \
143+ exit; \
144+" :
145+ : __imm_ptr(loop_callback),
146+ __imm(bpf_loop)
147+ : __clobber_common);
148+}
149+
150+SEC("fentry/bpf_fentry_test9")
151+__description("Private stack, exception in main prog")
152+__success __retval(0)
153+__arch_x86_64
154+__jited(" pushq %r9")
155+__jited(" callq")
156+__jited(" popq %r9")
157+int private_stack_exception_main_prog(void)
158+{
159+ asm volatile (" \
160+ r1 = 42; \
161+ *(u64 *)(r10 - 512) = r1; \
162+" ::: __clobber_common);
163+
164+ bpf_throw(0);
165+ return 0;
166+}
167+
168+__used static int subprog_exception(void)
169+{
170+ bpf_throw(0);
171+ return 0;
172+}
173+
174+SEC("fentry/bpf_fentry_test9")
175+__description("Private stack, exception in subprog")
176+__success __retval(0)
177+__arch_x86_64
178+__jited(" movq %rdi, -0x200(%r9)")
179+__jited(" pushq %r9")
180+__jited(" callq")
181+__jited(" popq %r9")
182+int private_stack_exception_sub_prog(void)
183+{
184+ asm volatile (" \
185+ r1 = 42; \
186+ *(u64 *)(r10 - 512) = r1; \
187+ call subprog_exception; \
188+" ::: __clobber_common);
189+
190+ return 0;
191+}
192+
193+int glob;
194+__noinline static void subprog2(int *val)
195+{
196+ glob += val[0] * 2;
197+}
198+
199+__noinline static void subprog1(int *val)
200+{
201+ int tmp[64] = {};
202+
203+ tmp[0] = *val;
204+ subprog2(tmp);
205+}
206+
207+__noinline static int timer_cb1(void *map, int *key, struct bpf_timer *timer)
208+{
209+ subprog1(key);
210+ return 0;
211+}
212+
213+__noinline static int timer_cb2(void *map, int *key, struct bpf_timer *timer)
214+{
215+ return 0;
216+}
217+
218+SEC("fentry/bpf_fentry_test9")
219+__description("Private stack, async callback, not nested")
220+__success __retval(0)
221+__arch_x86_64
222+__jited(" movabsq $0x{{.*}}, %r9")
223+int private_stack_async_callback_1(void)
224+{
225+ struct bpf_timer *arr_timer;
226+ int array_key = 0;
227+
228+ arr_timer = bpf_map_lookup_elem(&array, &array_key);
229+ if (!arr_timer)
230+ return 0;
231+
232+ bpf_timer_init(arr_timer, &array, 1);
233+ bpf_timer_set_callback(arr_timer, timer_cb2);
234+ bpf_timer_start(arr_timer, 0, 0);
235+ subprog1(&array_key);
236+ return 0;
237+}
238+
239+SEC("fentry/bpf_fentry_test9")
240+__description("Private stack, async callback, potential nesting")
241+__success __retval(0)
242+__arch_x86_64
243+__jited(" subq $0x100, %rsp")
244+int private_stack_async_callback_2(void)
245+{
246+ struct bpf_timer *arr_timer;
247+ int array_key = 0;
248+
249+ arr_timer = bpf_map_lookup_elem(&array, &array_key);
250+ if (!arr_timer)
251+ return 0;
252+
253+ bpf_timer_init(arr_timer, &array, 1);
254+ bpf_timer_set_callback(arr_timer, timer_cb1);
255+ bpf_timer_start(arr_timer, 0, 0);
256+ subprog1(&array_key);
257+ return 0;
258+}
259+
260+#else
261+
262+SEC("kprobe")
263+__description("private stack is not supported, use a dummy test")
264+__success
265+int dummy_test(void)
266+{
267+ return 0;
268+}
269+
270+#endif
271+
272+char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/verifier_ref_tracking.c+2−2
@@ -791,7 +791,7 @@ l0_%=: r0 = *(u8*)skb[0]; \ · 790 unmodified lines
791791
792792 SEC("tc")
793793 __description("reference tracking: forbid LD_ABS while holding reference")
794−__failure __msg("BPF_LD_[ABS|IND] cannot be mixed with socket references")
794+__failure __msg("BPF_LD_[ABS|IND] would lead to reference leak")
795795 __naked void ld_abs_while_holding_reference(void)
796796 {
797797 asm volatile (" \
@@ -836,7 +836,7 @@ l0_%=: r7 = 1; \ · 38 unmodified lines
836836
837837 SEC("tc")
838838 __description("reference tracking: forbid LD_IND while holding reference")
839−__failure __msg("BPF_LD_[ABS|IND] cannot be mixed with socket references")
839+__failure __msg("BPF_LD_[ABS|IND] would lead to reference leak")
840840 __naked void ld_ind_while_holding_reference(void)
841841 {
842842 asm volatile (" \
tools/testing/selftests/bpf/progs/verifier_sock.c+60−0
@@ -977,4 +977,64 @@ l1_%=: r0 = *(u8*)(r7 + 0); \ · 976 unmodified lines
977977 : __clobber_all);
978978 }
979979
980+SEC("cgroup/post_bind4")
981+__description("sk->src_ip6[0] [load 1st byte]")
982+__failure __msg("invalid bpf_context access off=28 size=2")
983+__naked void post_bind4_read_src_ip6(void)
984+{
985+ asm volatile (" \
986+ r6 = r1; \
987+ r7 = *(u16*)(r6 + %[bpf_sock_src_ip6_0]); \
988+ r0 = 1; \
989+ exit; \
990+" :
991+ : __imm_const(bpf_sock_src_ip6_0, offsetof(struct bpf_sock, src_ip6[0]))
992+ : __clobber_all);
993+}
994+
995+SEC("cgroup/post_bind4")
996+__description("sk->mark [load mark]")
997+__failure __msg("invalid bpf_context access off=16 size=2")
998+__naked void post_bind4_read_mark(void)
999+{
1000+ asm volatile (" \
1001+ r6 = r1; \
1002+ r7 = *(u16*)(r6 + %[bpf_sock_mark]); \
1003+ r0 = 1; \
1004+ exit; \
1005+" :
1006+ : __imm_const(bpf_sock_mark, offsetof(struct bpf_sock, mark))
1007+ : __clobber_all);
1008+}
1009+
1010+SEC("cgroup/post_bind6")
1011+__description("sk->src_ip4 [load src_ip4]")
1012+__failure __msg("invalid bpf_context access off=24 size=2")
1013+__naked void post_bind6_read_src_ip4(void)
1014+{
1015+ asm volatile (" \
1016+ r6 = r1; \
1017+ r7 = *(u16*)(r6 + %[bpf_sock_src_ip4]); \
1018+ r0 = 1; \
1019+ exit; \
1020+" :
1021+ : __imm_const(bpf_sock_src_ip4, offsetof(struct bpf_sock, src_ip4))
1022+ : __clobber_all);
1023+}
1024+
1025+SEC("cgroup/sock_create")
1026+__description("sk->src_port [word load]")
1027+__failure __msg("invalid bpf_context access off=44 size=2")
1028+__naked void sock_create_read_src_port(void)
1029+{
1030+ asm volatile (" \
1031+ r6 = r1; \
1032+ r7 = *(u16*)(r6 + %[bpf_sock_src_port]); \
1033+ r0 = 1; \
1034+ exit; \
1035+" :
1036+ : __imm_const(bpf_sock_src_port, offsetof(struct bpf_sock, src_port))
1037+ : __clobber_all);
1038+}
1039+
9801040 char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/verifier_spin_lock.c+1−1
@@ -187,7 +187,7 @@ l0_%=: r6 = r0; \ · 186 unmodified lines
187187
188188 SEC("cgroup/skb")
189189 __description("spin_lock: test6 missing unlock")
190−__failure __msg("unlock is missing")
190+__failure __msg("BPF_EXIT instruction cannot be used inside bpf_spin_lock-ed region")
191191 __failure_unpriv __msg_unpriv("")
192192 __naked void spin_lock_test6_missing_unlock(void)
193193 {
tools/testing/selftests/bpf/test_maps.c+0−4
@@ -26,10 +26,6 @@ · 25 unmodified lines
2626 #include "test_maps.h"
2727 #include "testing_helpers.h"
2828
29−#ifndef ENOTSUPP
30−#define ENOTSUPP 524
31−#endif
32−
3329 int skips;
3430
3531 static struct bpf_map_create_opts map_opts = { .sz = sizeof(map_opts) };
tools/testing/selftests/bpf/test_progs.c+111−3
@@ -16,15 +16,18 @@ · 15 unmodified lines
1616 #include <sys/socket.h>
1717 #include <sys/un.h>
1818 #include <bpf/btf.h>
19+#include <time.h>
1920 #include "json_writer.h"
2021
2122 #include "network_helpers.h"
2223
24+/* backtrace() and backtrace_symbols_fd() are glibc specific,
25+ * use header file when glibc is available and provide stub
26+ * implementations when another libc implementation is used.
27+ */
2328 #ifdef __GLIBC__
2429 #include <execinfo.h> /* backtrace */
25−#endif
26−
27−/* Default backtrace funcs if missing at link */
30+#else
2831 __weak int backtrace(void **buffer, int size)
2932 {
3033 return 0;
@@ -34,6 +37,7 @@ __weak void backtrace_symbols_fd(void *const *buffer, int size, int fd) · 3 unmodified lines
3437 {
3538 dprintf(fd, "<backtrace not supported>\n");
3639 }
40+#endif /*__GLIBC__ */
3741
3842 int env_verbosity = 0;
3943
@@ -176,6 +180,88 @@ int usleep(useconds_t usec) · 136 unmodified lines
176180 return syscall(__NR_nanosleep, &ts, NULL);
177181 }
178182
183+/* Watchdog timer is started by watchdog_start() and stopped by watchdog_stop().
184+ * If timer is active for longer than env.secs_till_notify,
185+ * it prints the name of the current test to the stderr.
186+ * If timer is active for longer than env.secs_till_kill,
187+ * it kills the thread executing the test by sending a SIGSEGV signal to it.
188+ */
189+static void watchdog_timer_func(union sigval sigval)
190+{
191+ struct itimerspec timeout = {};
192+ char test_name[256];
193+ int err;
194+
195+ if (env.subtest_state)
196+ snprintf(test_name, sizeof(test_name), "%s/%s",
197+ env.test->test_name, env.subtest_state->name);
198+ else
199+ snprintf(test_name, sizeof(test_name), "%s",
200+ env.test->test_name);
201+
202+ switch (env.watchdog_state) {
203+ case WD_NOTIFY:
204+ fprintf(env.stderr_saved, "WATCHDOG: test case %s executes for %d seconds...\n",
205+ test_name, env.secs_till_notify);
206+ timeout.it_value.tv_sec = env.secs_till_kill - env.secs_till_notify;
207+ env.watchdog_state = WD_KILL;
208+ err = timer_settime(env.watchdog, 0, &timeout, NULL);
209+ if (err)
210+ fprintf(env.stderr_saved, "Failed to arm watchdog timer\n");
211+ break;
212+ case WD_KILL:
213+ fprintf(env.stderr_saved,
214+ "WATCHDOG: test case %s executes for %d seconds, terminating with SIGSEGV\n",
215+ test_name, env.secs_till_kill);
216+ pthread_kill(env.main_thread, SIGSEGV);
217+ break;
218+ }
219+}
220+
221+static void watchdog_start(void)
222+{
223+ struct itimerspec timeout = {};
224+ int err;
225+
226+ if (env.secs_till_kill == 0)
227+ return;
228+ if (env.secs_till_notify > 0) {
229+ env.watchdog_state = WD_NOTIFY;
230+ timeout.it_value.tv_sec = env.secs_till_notify;
231+ } else {
232+ env.watchdog_state = WD_KILL;
233+ timeout.it_value.tv_sec = env.secs_till_kill;
234+ }
235+ err = timer_settime(env.watchdog, 0, &timeout, NULL);
236+ if (err)
237+ fprintf(env.stderr_saved, "Failed to start watchdog timer\n");
238+}
239+
240+static void watchdog_stop(void)
241+{
242+ struct itimerspec timeout = {};
243+ int err;
244+
245+ env.watchdog_state = WD_NOTIFY;
246+ err = timer_settime(env.watchdog, 0, &timeout, NULL);
247+ if (err)
248+ fprintf(env.stderr_saved, "Failed to stop watchdog timer\n");
249+}
250+
251+static void watchdog_init(void)
252+{
253+ struct sigevent watchdog_sev = {
254+ .sigev_notify = SIGEV_THREAD,
255+ .sigev_notify_function = watchdog_timer_func,
256+ };
257+ int err;
258+
259+ env.main_thread = pthread_self();
260+ err = timer_create(CLOCK_MONOTONIC, &watchdog_sev, &env.watchdog);
261+ if (err)
262+ fprintf(stderr, "Failed to initialize watchdog timer\n");
263+}
264+
179265 static bool should_run(struct test_selector *sel, int num, const char *name)
180266 {
181267 int i;
@@ -512,6 +598,7 @@ bool test__start_subtest(const char *subtest_name) · 330 unmodified lines
512598
513599 env.subtest_state = subtest_state;
514600 stdio_hijack_init(&subtest_state->log_buf, &subtest_state->log_cnt);
601+ watchdog_start();
515602
516603 return true;
517604 }
@@ -777,6 +864,7 @@ enum ARG_KEYS { · 259 unmodified lines
777864 ARG_DEBUG = -1,
778865 ARG_JSON_SUMMARY = 'J',
779866 ARG_TRAFFIC_MONITOR = 'm',
867+ ARG_WATCHDOG_TIMEOUT = 'w',
780868 };
781869
782870 static const struct argp_option opts[] = {
@@ -807,6 +895,8 @@ static const struct argp_option opts[] = { · 24 unmodified lines
807895 { "traffic-monitor", ARG_TRAFFIC_MONITOR, "NAMES", 0,
808896 "Monitor network traffic of tests with name matching the pattern (supports '*' wildcard)." },
809897 #endif
898+ { "watchdog-timeout", ARG_WATCHDOG_TIMEOUT, "SECONDS", 0,
899+ "Kill the process if tests are not making progress for specified number of seconds." },
810900 {},
811901 };
812902
@@ -868,6 +958,7 @@ static int libbpf_print_fn(enum libbpf_print_level level, · 55 unmodified lines
868958
869959 va_copy(args2, args);
870960 vfprintf(libbpf_capture_stream, format, args2);
961+ va_end(args2);
871962 }
872963
873964 if (env.verbosity < VERBOSE_VERY && level == LIBBPF_DEBUG)
@@ -1031,6 +1122,16 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) · 157 unmodified lines
10311122 true);
10321123 break;
10331124 #endif
1125+ case ARG_WATCHDOG_TIMEOUT:
1126+ env->secs_till_kill = atoi(arg);
1127+ if (env->secs_till_kill < 0) {
1128+ fprintf(stderr, "Invalid watchdog timeout: %s.\n", arg);
1129+ return -EINVAL;
1130+ }
1131+ if (env->secs_till_kill < env->secs_till_notify) {
1132+ env->secs_till_notify = 0;
1133+ }
1134+ break;
10341135 default:
10351136 return ARGP_ERR_UNKNOWN;
10361137 }
@@ -1259,10 +1360,12 @@ static void run_one_test(int test_num) · 222 unmodified lines
12591360
12601361 stdio_hijack(&state->log_buf, &state->log_cnt);
12611362
1363+ watchdog_start();
12621364 if (test->run_test)
12631365 test->run_test();
12641366 else if (test->run_serial_test)
12651367 test->run_serial_test();
1368+ watchdog_stop();
12661369
12671370 /* ensure last sub-test is finalized properly */
12681371 if (env.subtest_state)
@@ -1703,6 +1806,7 @@ out: · 434 unmodified lines
17031806 static int worker_main(int sock)
17041807 {
17051808 save_netns();
1809+ watchdog_init();
17061810
17071811 while (true) {
17081812 /* receive command */
@@ -1812,6 +1916,8 @@ int main(int argc, char **argv) · 103 unmodified lines
18121916
18131917 sigaction(SIGSEGV, &sigact, NULL);
18141918
1919+ env.secs_till_notify = 10;
1920+ env.secs_till_kill = 120;
18151921 err = argp_parse(&argp, argc, argv, 0, NULL, &env);
18161922 if (err)
18171923 return err;
@@ -1820,6 +1926,8 @@ int main(int argc, char **argv) · 2 unmodified lines
18201926 if (err)
18211927 return err;
18221928
1929+ watchdog_init();
1930+
18231931 /* Use libbpf 1.0 API mode */
18241932 libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
18251933 libbpf_set_print(libbpf_print_fn);
tools/testing/selftests/bpf/test_progs.h+14−0
@@ -131,6 +131,12 @@ struct test_env { · 130 unmodified lines
131131 pid_t *worker_pids; /* array of worker pids */
132132 int *worker_socks; /* array of worker socks */
133133 int *worker_current_test; /* array of current running test for each worker */
134+
135+ pthread_t main_thread;
136+ int secs_till_notify;
137+ int secs_till_kill;
138+ timer_t watchdog; /* watch for stalled tests/subtests */
139+ enum { WD_NOTIFY, WD_KILL } watchdog_state;
134140 };
135141
136142 #define MAX_LOG_TRUNK_SIZE 8192
@@ -390,6 +396,14 @@ int test__join_cgroup(const char *path); · 253 unmodified lines
390396 ___ok; \
391397 })
392398
399+#define ASSERT_ERR_FD(fd, name) ({ \
400+ static int duration = 0; \
401+ int ___fd = (fd); \
402+ bool ___ok = ___fd < 0; \
403+ CHECK(!___ok, (name), "unexpected fd: %d\n", ___fd); \
404+ ___ok; \
405+})
406+
393407 #define SYS(goto_label, fmt, ...) \
394408 ({ \
395409 char cmd[1024]; \
tools/testing/selftests/bpf/test_verifier.c+0−4
@@ -42,10 +42,6 @@ · 41 unmodified lines
4242 #include "../../../include/linux/filter.h"
4343 #include "testing_helpers.h"
4444
45−#ifndef ENOTSUPP
46−#define ENOTSUPP 524
47−#endif
48−
4945 #define MAX_INSNS BPF_MAXINSNS
5046 #define MAX_EXPECTED_INSNS 32
5147 #define MAX_UNEXPECTED_INSNS 32
tools/testing/selftests/bpf/uprobe_multi.c+4−0
@@ -12,6 +12,10 @@ · 11 unmodified lines
1212 #define MADV_POPULATE_READ 22
1313 #endif
1414
15+#ifndef MADV_PAGEOUT
16+#define MADV_PAGEOUT 21
17+#endif
18+
1519 int __attribute__((weak)) uprobe(void)
1620 {
1721 return 0;
tools/testing/selftests/bpf/uptr_test_common.h+63−0
@@ -0,0 +1,63 @@
1+/* SPDX-License-Identifier: GPL-2.0 */
2+/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
3+
4+#ifndef _UPTR_TEST_COMMON_H
5+#define _UPTR_TEST_COMMON_H
6+
7+#define MAGIC_VALUE 0xabcd1234
8+#define PAGE_SIZE 4096
9+
10+#ifdef __BPF__
11+/* Avoid fwd btf type being generated for the following struct */
12+struct large_data *dummy_large_data;
13+struct empty_data *dummy_empty_data;
14+struct user_data *dummy_data;
15+struct cgroup *dummy_cgrp;
16+#else
17+#define __uptr
18+#define __kptr
19+#endif
20+
21+struct user_data {
22+ int a;
23+ int b;
24+ int result;
25+ int nested_result;
26+};
27+
28+struct nested_udata {
29+ struct user_data __uptr *udata;
30+};
31+
32+struct value_type {
33+ struct user_data __uptr *udata;
34+ struct cgroup __kptr *cgrp;
35+ struct nested_udata nested;
36+};
37+
38+struct value_lock_type {
39+ struct user_data __uptr *udata;
40+ struct bpf_spin_lock lock;
41+};
42+
43+struct large_data {
44+ __u8 one_page[PAGE_SIZE];
45+ int a;
46+};
47+
48+struct large_uptr {
49+ struct large_data __uptr *udata;
50+};
51+
52+struct empty_data {
53+};
54+
55+struct empty_uptr {
56+ struct empty_data __uptr *udata;
57+};
58+
59+struct kstruct_uptr {
60+ struct cgroup __uptr *cgrp;
61+};
62+
63+#endif
tools/testing/selftests/bpf/veristat.c+159−2
@@ -16,6 +16,7 @@ · 15 unmodified lines
1616 #include <sys/stat.h>
1717 #include <bpf/libbpf.h>
1818 #include <bpf/btf.h>
19+#include <bpf/bpf.h>
1920 #include <libelf.h>
2021 #include <gelf.h>
2122 #include <float.h>
@@ -179,6 +180,7 @@ static struct env { · 157 unmodified lines
179180 int files_skipped;
180181 int progs_processed;
181182 int progs_skipped;
183+ int top_src_lines;
182184 } env;
183185
184186 static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args)
@@ -228,6 +230,7 @@ static const struct argp_option opts[] = { · 43 unmodified lines
228230 "Force frequent BPF verifier state checkpointing (set BPF_F_TEST_STATE_FREQ program flag)" },
229231 { "test-reg-invariants", 'r', NULL, 0,
230232 "Force BPF verifier failure on register invariant violation (BPF_F_TEST_REG_INVARIANTS program flag)" },
233+ { "top-src-lines", 'S', "N", 0, "Emit N most frequent source code lines" },
231234 {},
232235 };
233236
@@ -327,6 +330,14 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) · 93 unmodified lines
327330 return err;
328331 }
329332 break;
333+ case 'S':
334+ errno = 0;
335+ env.top_src_lines = strtol(arg, NULL, 10);
336+ if (errno) {
337+ fprintf(stderr, "invalid top lines N specifier: %s\n", arg);
338+ argp_usage(state);
339+ }
340+ break;
330341 case ARGP_KEY_ARG:
331342 tmp = realloc(env.filenames, (env.filename_cnt + 1) * sizeof(*env.filenames));
332343 if (!tmp)
@@ -854,6 +865,118 @@ static int parse_verif_log(char * const buf, size_t buf_sz, struct verif_stats * · 521 unmodified lines
854865 return 0;
855866 }
856867
868+struct line_cnt {
869+ char *line;
870+ int cnt;
871+};
872+
873+static int str_cmp(const void *a, const void *b)
874+{
875+ const char **str1 = (const char **)a;
876+ const char **str2 = (const char **)b;
877+
878+ return strcmp(*str1, *str2);
879+}
880+
881+static int line_cnt_cmp(const void *a, const void *b)
882+{
883+ const struct line_cnt *a_cnt = (const struct line_cnt *)a;
884+ const struct line_cnt *b_cnt = (const struct line_cnt *)b;
885+
886+ if (a_cnt->cnt != b_cnt->cnt)
887+ return a_cnt->cnt < b_cnt->cnt ? -1 : 1;
888+ return strcmp(a_cnt->line, b_cnt->line);
889+}
890+
891+static int print_top_src_lines(char * const buf, size_t buf_sz, const char *prog_name)
892+{
893+ int lines_cap = 0;
894+ int lines_size = 0;
895+ char **lines = NULL;
896+ char *line = NULL;
897+ char *state;
898+ struct line_cnt *freq = NULL;
899+ struct line_cnt *cur;
900+ int unique_lines;
901+ int err = 0;
902+ int i;
903+
904+ while ((line = strtok_r(line ? NULL : buf, "\n", &state))) {
905+ if (strncmp(line, "; ", 2) != 0)
906+ continue;
907+ line += 2;
908+
909+ if (lines_size == lines_cap) {
910+ char **tmp;
911+
912+ lines_cap = max(16, lines_cap * 2);
913+ tmp = realloc(lines, lines_cap * sizeof(*tmp));
914+ if (!tmp) {
915+ err = -ENOMEM;
916+ goto cleanup;
917+ }
918+ lines = tmp;
919+ }
920+ lines[lines_size] = line;
921+ lines_size++;
922+ }
923+
924+ if (lines_size == 0)
925+ goto cleanup;
926+
927+ qsort(lines, lines_size, sizeof(*lines), str_cmp);
928+
929+ freq = calloc(lines_size, sizeof(*freq));
930+ if (!freq) {
931+ err = -ENOMEM;
932+ goto cleanup;
933+ }
934+
935+ cur = freq;
936+ cur->line = lines[0];
937+ cur->cnt = 1;
938+ for (i = 1; i < lines_size; ++i) {
939+ if (strcmp(lines[i], cur->line) != 0) {
940+ cur++;
941+ cur->line = lines[i];
942+ cur->cnt = 0;
943+ }
944+ cur->cnt++;
945+ }
946+ unique_lines = cur - freq + 1;
947+
948+ qsort(freq, unique_lines, sizeof(struct line_cnt), line_cnt_cmp);
949+
950+ printf("Top source lines (%s):\n", prog_name);
951+ for (i = 0; i < min(unique_lines, env.top_src_lines); ++i) {
952+ const char *src_code = freq[i].line;
953+ const char *src_line = NULL;
954+ char *split = strrchr(freq[i].line, '@');
955+
956+ if (split) {
957+ src_line = split + 1;
958+
959+ while (*src_line && isspace(*src_line))
960+ src_line++;
961+
962+ while (split > src_code && isspace(*split))
963+ split--;
964+ *split = '\0';
965+ }
966+
967+ if (src_line)
968+ printf("%5d: (%s)\t%s\n", freq[i].cnt, src_line, src_code);
969+ else
970+ printf("%5d: %s\n", freq[i].cnt, src_code);
971+ }
972+ printf("\n");
973+
974+cleanup:
975+ free(freq);
976+ free(lines);
977+ return err;
978+}
979+
857980 static int guess_prog_type_by_ctx_name(const char *ctx_name,
858981 enum bpf_prog_type *prog_type,
859982 enum bpf_attach_type *attach_type)
@@ -987,6 +1110,35 @@ skip_freplace_fixup: · 127 unmodified lines
9871110 return;
9881111 }
9891112
1113+static int max_verifier_log_size(void)
1114+{
1115+ const int SMALL_LOG_SIZE = UINT_MAX >> 8;
1116+ const int BIG_LOG_SIZE = UINT_MAX >> 2;
1117+ struct bpf_insn insns[] = {
1118+ { .code = BPF_ALU | BPF_MOV | BPF_X, .dst_reg = BPF_REG_0, },
1119+ { .code = BPF_JMP | BPF_EXIT, },
1120+ };
1121+ LIBBPF_OPTS(bpf_prog_load_opts, opts,
1122+ .log_size = BIG_LOG_SIZE,
1123+ .log_buf = (void *)-1,
1124+ .log_level = 4
1125+ );
1126+ int ret, insn_cnt = ARRAY_SIZE(insns);
1127+ static int log_size;
1128+
1129+ if (log_size != 0)
1130+ return log_size;
1131+
1132+ ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, &opts);
1133+
1134+ if (ret == -EFAULT)
1135+ log_size = BIG_LOG_SIZE;
1136+ else /* ret == -EINVAL, big log size is not supported by the verifier */
1137+ log_size = SMALL_LOG_SIZE;
1138+
1139+ return log_size;
1140+}
1141+
9901142 static int process_prog(const char *filename, struct bpf_object *obj, struct bpf_program *prog)
9911143 {
9921144 const char *base_filename = basename(strdupa(filename));
@@ -1009,13 +1161,16 @@ static int process_prog(const char *filename, struct bpf_object *obj, struct bpf · 16 unmodified lines
10091161 stats = &env.prog_stats[env.prog_stat_cnt++];
10101162 memset(stats, 0, sizeof(*stats));
10111163
1012− if (env.verbose) {
1013− buf_sz = env.log_size ? env.log_size : 16 * 1024 * 1024;
1164+ if (env.verbose || env.top_src_lines > 0) {
1165+ buf_sz = env.log_size ? env.log_size : max_verifier_log_size();
10141166 buf = malloc(buf_sz);
10151167 if (!buf)
10161168 return -ENOMEM;
10171169 /* ensure we always request stats */
10181170 log_level = env.log_level | 4 | (env.log_fixed ? 8 : 0);
1171+ /* --top-src-lines needs verifier log */
1172+ if (env.top_src_lines > 0 && env.log_level == 0)
1173+ log_level |= 2;
10191174 } else {
10201175 buf = verif_log_buf;
10211176 buf_sz = sizeof(verif_log_buf);
@@ -1048,6 +1203,8 @@ static int process_prog(const char *filename, struct bpf_object *obj, struct bpf · 26 unmodified lines
10481203 filename, prog_name, stats->stats[DURATION],
10491204 err ? "failure" : "success", buf);
10501205 }
1206+ if (env.top_src_lines > 0)
1207+ print_top_src_lines(buf, buf_sz, stats->prog_name);
10511208
10521209 if (verif_log_buf != buf)
10531210 free(buf);